From 2c2988518d4aa71bd3e83ba7c2c767884e92801c Mon Sep 17 00:00:00 2001 From: Jukka Ojanen Date: Thu, 22 Jul 2021 13:59:19 +0300 Subject: [PATCH 01/33] Remove all calls to clFlush() --- src/backend.c | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/src/backend.c b/src/backend.c index 31d6bade1..ae728fbe7 100644 --- a/src/backend.c +++ b/src/backend.c @@ -5377,7 +5377,7 @@ int run_opencl_kernel_atinit (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *dev if (hc_clEnqueueNDRangeKernel (hashcat_ctx, device_param->opencl_command_queue, kernel, 1, NULL, global_work_size, local_work_size, 0, NULL, NULL) == -1) return -1; - if (hc_clFlush (hashcat_ctx, device_param->opencl_command_queue) == -1) return -1; + /*if (hc_clFlush (hashcat_ctx, device_param->opencl_command_queue) == -1) return -1;*/ if (hc_clFinish (hashcat_ctx, device_param->opencl_command_queue) == -1) return -1; @@ -5405,7 +5405,7 @@ int run_opencl_kernel_utf8toutf16le (hashcat_ctx_t *hashcat_ctx, hc_device_param if (hc_clEnqueueNDRangeKernel (hashcat_ctx, device_param->opencl_command_queue, kernel, 1, NULL, global_work_size, local_work_size, 0, NULL, NULL) == -1) return -1; - if (hc_clFlush (hashcat_ctx, device_param->opencl_command_queue) == -1) return -1; + /*if (hc_clFlush (hashcat_ctx, device_param->opencl_command_queue) == -1) return -1;*/ if (hc_clFinish (hashcat_ctx, device_param->opencl_command_queue) == -1) return -1; @@ -5446,6 +5446,8 @@ int run_opencl_kernel_memset (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *dev if (hc_clEnqueueWriteBuffer (hashcat_ctx, device_param->opencl_command_queue, buf, CL_FALSE, num16d * 16, num16m, tmp, 0, NULL, NULL) == -1) return -1; } + /*if (hc_clFlush (hashcat_ctx, device_param->opencl_command_queue) == -1) return -1;*/ + if (hc_clFinish (hashcat_ctx, device_param->opencl_command_queue) == -1) return -1; return 0; @@ -5484,6 +5486,8 @@ int run_opencl_kernel_bzero (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *devi if (hc_clEnqueueWriteBuffer (hashcat_ctx, device_param->opencl_command_queue, buf, CL_FALSE, num16d * 16, num16m, tmp, 0, NULL, NULL) == -1) return -1; } + /*if (hc_clFlush (hashcat_ctx, device_param->opencl_command_queue) == -1) return -1;*/ + if (hc_clFinish (hashcat_ctx, device_param->opencl_command_queue) == -1) return -1; return 0; @@ -5857,7 +5861,7 @@ int run_kernel (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param, con if (hc_clEnqueueNDRangeKernel (hashcat_ctx, device_param->opencl_command_queue, opencl_kernel, 1, NULL, global_work_size, local_work_size, 0, NULL, &opencl_event) == -1) return -1; - if (hc_clFlush (hashcat_ctx, device_param->opencl_command_queue) == -1) return -1; + /*if (hc_clFlush (hashcat_ctx, device_param->opencl_command_queue) == -1) return -1;*/ // spin damper section @@ -6082,7 +6086,7 @@ int run_kernel_mp (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param, if (hc_clEnqueueNDRangeKernel (hashcat_ctx, device_param->opencl_command_queue, opencl_kernel, 1, NULL, global_work_size, local_work_size, 0, NULL, NULL) == -1) return -1; - if (hc_clFlush (hashcat_ctx, device_param->opencl_command_queue) == -1) return -1; + /*if (hc_clFlush (hashcat_ctx, device_param->opencl_command_queue) == -1) return -1;*/ if (hc_clFinish (hashcat_ctx, device_param->opencl_command_queue) == -1) return -1; } @@ -6123,7 +6127,7 @@ int run_kernel_tm (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param) if (hc_clEnqueueNDRangeKernel (hashcat_ctx, device_param->opencl_command_queue, cuda_kernel, 1, NULL, global_work_size, local_work_size, 0, NULL, NULL) == -1) return -1; - if (hc_clFlush (hashcat_ctx, device_param->opencl_command_queue) == -1) return -1; + /*if (hc_clFlush (hashcat_ctx, device_param->opencl_command_queue) == -1) return -1;*/ if (hc_clFinish (hashcat_ctx, device_param->opencl_command_queue) == -1) return -1; } @@ -6174,7 +6178,7 @@ int run_kernel_amp (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param, if (hc_clEnqueueNDRangeKernel (hashcat_ctx, device_param->opencl_command_queue, opencl_kernel, 1, NULL, global_work_size, local_work_size, 0, NULL, NULL) == -1) return -1; - if (hc_clFlush (hashcat_ctx, device_param->opencl_command_queue) == -1) return -1; + /*if (hc_clFlush (hashcat_ctx, device_param->opencl_command_queue) == -1) return -1;*/ if (hc_clFinish (hashcat_ctx, device_param->opencl_command_queue) == -1) return -1; } @@ -6225,7 +6229,7 @@ int run_kernel_decompress (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device if (hc_clEnqueueNDRangeKernel (hashcat_ctx, device_param->opencl_command_queue, opencl_kernel, 1, NULL, global_work_size, local_work_size, 0, NULL, NULL) == -1) return -1; - if (hc_clFlush (hashcat_ctx, device_param->opencl_command_queue) == -1) return -1; + /*if (hc_clFlush (hashcat_ctx, device_param->opencl_command_queue) == -1) return -1;*/ if (hc_clFinish (hashcat_ctx, device_param->opencl_command_queue) == -1) return -1; } From fafd24237d8bad4f662d0eb63e98ea9d16c49b38 Mon Sep 17 00:00:00 2001 From: Jukka Ojanen Date: Thu, 22 Jul 2021 14:24:03 +0300 Subject: [PATCH 02/33] Define HC_ALIGN macro to control data alignment and use common constant zero buffer in run_cuda_kernel_bzero(), run_hip_kernel_bzero() and run_opencl_kernel_bzero(). --- include/common.h | 8 ++++++++ src/backend.c | 27 ++++++--------------------- 2 files changed, 14 insertions(+), 21 deletions(-) diff --git a/include/common.h b/include/common.h index 445ebc405..3da61bc6c 100644 --- a/include/common.h +++ b/include/common.h @@ -63,6 +63,14 @@ #define HC_API_CALL #endif +#if defined (__GNUC__) +#define HC_ALIGN(x) __attribute__((aligned(x))) +#elif defined (_MSC_VER) +#define HC_ALIGN(x) __declspec(align(x)) +#else +#define HC_ALIGN(x) +#endif + #if defined (_WIN) #define WIN32_LEAN_AND_MEAN #endif diff --git a/src/backend.c b/src/backend.c index ae728fbe7..af8cd7c2a 100644 --- a/src/backend.c +++ b/src/backend.c @@ -38,6 +38,9 @@ static const u32 full80 = 0x80808080; static double TARGET_MSEC_PROFILE[4] = { 2, 12, 96, 480 }; +HC_ALIGN(16) +static const u32 bzeros[4] = { 0, 0, 0, 0 }; + static bool is_same_device (const hc_device_param_t *src, const hc_device_param_t *dst) { // First check by PCI address @@ -5214,7 +5217,6 @@ int run_cuda_kernel_bzero (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device { const u64 num16d = size / 16; const u64 num16m = size % 16; - u32 tmp[4]; if (num16d) { @@ -5232,12 +5234,7 @@ int run_cuda_kernel_bzero (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device if (num16m) { - tmp[0] = 0; - tmp[1] = 0; - tmp[2] = 0; - tmp[3] = 0; - - if (hc_cuMemcpyHtoDAsync (hashcat_ctx, buf + (num16d * 16), tmp, num16m, device_param->cuda_stream) == -1) return -1; + if (hc_cuMemcpyHtoDAsync (hashcat_ctx, buf + (num16d * 16), bzeros, num16m, device_param->cuda_stream) == -1) return -1; } if (hc_cuStreamSynchronize (hashcat_ctx, device_param->cuda_stream) == -1) return -1; @@ -5325,7 +5322,6 @@ int run_hip_kernel_bzero (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_ { const u64 num16d = size / 16; const u64 num16m = size % 16; - u32 tmp[4]; if (num16d) { @@ -5343,12 +5339,7 @@ int run_hip_kernel_bzero (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_ if (num16m) { - tmp[0] = 0; - tmp[1] = 0; - tmp[2] = 0; - tmp[3] = 0; - - if (hc_hipMemcpyHtoDAsync (hashcat_ctx, buf + (num16d * 16), tmp, num16m, device_param->hip_stream) == -1) return -1; + if (hc_hipMemcpyHtoDAsync (hashcat_ctx, buf + (num16d * 16), bzeros, num16m, device_param->hip_stream) == -1) return -1; } if (hc_hipStreamSynchronize (hashcat_ctx, device_param->hip_stream) == -1) return -1; @@ -5457,7 +5448,6 @@ int run_opencl_kernel_bzero (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *devi { const u64 num16d = size / 16; const u64 num16m = size % 16; - u32 tmp[4]; if (num16d) { @@ -5478,12 +5468,7 @@ int run_opencl_kernel_bzero (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *devi if (num16m) { - tmp[0] = 0; - tmp[1] = 0; - tmp[2] = 0; - tmp[3] = 0; - - if (hc_clEnqueueWriteBuffer (hashcat_ctx, device_param->opencl_command_queue, buf, CL_FALSE, num16d * 16, num16m, tmp, 0, NULL, NULL) == -1) return -1; + if (hc_clEnqueueWriteBuffer (hashcat_ctx, device_param->opencl_command_queue, buf, CL_FALSE, num16d * 16, num16m, bzeros, 0, NULL, NULL) == -1) return -1; } /*if (hc_clFlush (hashcat_ctx, device_param->opencl_command_queue) == -1) return -1;*/ From 9f9333f2efeb9c0858cca9e8cebec9a4b64e2bbc Mon Sep 17 00:00:00 2001 From: Jukka Ojanen Date: Thu, 22 Jul 2021 15:18:10 +0300 Subject: [PATCH 03/33] Allow async execution of run_opencl_kernel_bzero(), run_hip_kernel_bzero() and run_opencl_kernel_bzero() --- src/backend.c | 10 +++++----- src/selftest.c | 9 +++++++++ 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/src/backend.c b/src/backend.c index af8cd7c2a..2b52b4cde 100644 --- a/src/backend.c +++ b/src/backend.c @@ -5068,12 +5068,12 @@ int choose_kernel (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param, { if (device_param->is_cuda == true) { - if (run_cuda_kernel_bzero (hashcat_ctx, device_param, device_param->cuda_d_hooks, pws_cnt * hashconfig->hook_size) == -1) return -1; + if (run_cuda_kernel_bzero (hashcat_ctx, device_param, device_param->cuda_d_hooks, pws_cnt * hashconfig->hook_size) == -1) return -1; } if (device_param->is_hip == true) { - if (run_hip_kernel_bzero (hashcat_ctx, device_param, device_param->hip_d_hooks, pws_cnt * hashconfig->hook_size) == -1) return -1; + if (run_hip_kernel_bzero (hashcat_ctx, device_param, device_param->hip_d_hooks, pws_cnt * hashconfig->hook_size) == -1) return -1; } if (device_param->is_opencl == true) @@ -5237,7 +5237,7 @@ int run_cuda_kernel_bzero (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device if (hc_cuMemcpyHtoDAsync (hashcat_ctx, buf + (num16d * 16), bzeros, num16m, device_param->cuda_stream) == -1) return -1; } - if (hc_cuStreamSynchronize (hashcat_ctx, device_param->cuda_stream) == -1) return -1; + /*if (hc_cuStreamSynchronize (hashcat_ctx, device_param->cuda_stream) == -1) return -1;*/ return 0; } @@ -5342,7 +5342,7 @@ int run_hip_kernel_bzero (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_ if (hc_hipMemcpyHtoDAsync (hashcat_ctx, buf + (num16d * 16), bzeros, num16m, device_param->hip_stream) == -1) return -1; } - if (hc_hipStreamSynchronize (hashcat_ctx, device_param->hip_stream) == -1) return -1; + /*if (hc_hipStreamSynchronize (hashcat_ctx, device_param->hip_stream) == -1) return -1;*/ return 0; } @@ -5473,7 +5473,7 @@ int run_opencl_kernel_bzero (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *devi /*if (hc_clFlush (hashcat_ctx, device_param->opencl_command_queue) == -1) return -1;*/ - if (hc_clFinish (hashcat_ctx, device_param->opencl_command_queue) == -1) return -1; + /*if (hc_clFinish (hashcat_ctx, device_param->opencl_command_queue) == -1) return -1;*/ return 0; } diff --git a/src/selftest.c b/src/selftest.c index 4f8dc1092..41906ad73 100644 --- a/src/selftest.c +++ b/src/selftest.c @@ -903,13 +903,22 @@ HC_API_CALL void *thread_selftest (void *p) if (device_param->is_cuda == true) { + if (hc_cuStreamSynchronize (hashcat_ctx, device_param->cuda_stream) == -1) return NULL; + if (hc_cuCtxPopCurrent (hashcat_ctx, &device_param->cuda_context) == -1) return NULL; } if (device_param->is_hip == true) { + if (hc_hipStreamSynchronize (hashcat_ctx, device_param->hip_stream) == -1) return NULL; + if (hc_hipCtxPopCurrent (hashcat_ctx, &device_param->hip_context) == -1) return NULL; } + + if (device_param->is_opencl == true) + { + if (hc_clFinish (hashcat_ctx, device_param->opencl_command_queue) == -1) return NULL; + } return NULL; } From 4bd736367420b399b04738f5d33e9756842b51d5 Mon Sep 17 00:00:00 2001 From: Jukka Ojanen Date: Sun, 25 Jul 2021 22:18:16 +0300 Subject: [PATCH 04/33] Add missing HIP declarations --- include/backend.h | 3 +++ 1 file changed, 3 insertions(+) diff --git a/include/backend.h b/include/backend.h index 957ac229d..baa94b50c 100644 --- a/include/backend.h +++ b/include/backend.h @@ -120,8 +120,11 @@ int hc_hipInit (hashcat_ctx_t *hashcat_ctx, unsigned int Flags int hc_hipLaunchKernel (hashcat_ctx_t *hashcat_ctx, HIPfunction f, unsigned int gridDimX, unsigned int gridDimY, unsigned int gridDimZ, unsigned int blockDimX, unsigned int blockDimY, unsigned int blockDimZ, unsigned int sharedMemBytes, HIPstream hStream, void **kernelParams, void **extra); int hc_hipMemAlloc (hashcat_ctx_t *hashcat_ctx, HIPdeviceptr *dptr, size_t bytesize); int hc_hipMemcpyDtoD (hashcat_ctx_t *hashcat_ctx, HIPdeviceptr dstDevice, HIPdeviceptr srcDevice, size_t ByteCount); +int hc_hipMemcpyDtoDAsync (hashcat_ctx_t *hashcat_ctx, HIPdeviceptr dstDevice, HIPdeviceptr srcDevice, size_t ByteCount, HIPstream hStream); int hc_hipMemcpyDtoH (hashcat_ctx_t *hashcat_ctx, void *dstHost, HIPdeviceptr srcDevice, size_t ByteCount); +int hc_hipMemcpyDtoHAsync (hashcat_ctx_t *hashcat_ctx, void *dstHost, HIPdeviceptr srcDevice, size_t ByteCount, HIPstream hStream); int hc_hipMemcpyHtoD (hashcat_ctx_t *hashcat_ctx, HIPdeviceptr dstDevice, const void *srcHost, size_t ByteCount); +int hc_hipMemcpyHtoDAsync (hashcat_ctx_t *hashcat_ctx, HIPdeviceptr dstDevice, const void *srcHost, size_t ByteCount, HIPstream hStream); int hc_hipMemFree (hashcat_ctx_t *hashcat_ctx, HIPdeviceptr dptr); int hc_hipModuleGetFunction (hashcat_ctx_t *hashcat_ctx, HIPfunction *hfunc, HIPmodule hmod, const char *name); int hc_hipModuleLoadDataEx (hashcat_ctx_t *hashcat_ctx, HIPmodule *module, const void *image, unsigned int numOptions, HIPjit_option *options, void **optionValues); From 79404b1ff12ca30686cfb0014126d873cd957cad Mon Sep 17 00:00:00 2001 From: Jukka Ojanen Date: Sun, 25 Jul 2021 22:19:50 +0300 Subject: [PATCH 05/33] Allow async execution of selftest() --- src/selftest.c | 131 ++++++++++++++++++++++++++++++------------------- 1 file changed, 81 insertions(+), 50 deletions(-) diff --git a/src/selftest.c b/src/selftest.c index 41906ad73..77ad8f23b 100644 --- a/src/selftest.c +++ b/src/selftest.c @@ -95,17 +95,17 @@ static int selftest (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param if (device_param->is_cuda == true) { - if (hc_cuMemcpyHtoD (hashcat_ctx, device_param->cuda_d_pws_buf, &pw, 1 * sizeof (pw_t)) == -1) return -1; + if (hc_cuMemcpyHtoDAsync (hashcat_ctx, device_param->cuda_d_pws_buf, &pw, 1 * sizeof (pw_t), device_param->cuda_stream) == -1) return -1; } if (device_param->is_hip == true) { - if (hc_hipMemcpyHtoD (hashcat_ctx, device_param->hip_d_pws_buf, &pw, 1 * sizeof (pw_t)) == -1) return -1; + if (hc_hipMemcpyHtoDAsync (hashcat_ctx, device_param->hip_d_pws_buf, &pw, 1 * sizeof (pw_t), device_param->hip_stream) == -1) return -1; } if (device_param->is_opencl == true) { - if (hc_clEnqueueWriteBuffer (hashcat_ctx, device_param->opencl_command_queue, device_param->opencl_d_pws_buf, CL_TRUE, 0, 1 * sizeof (pw_t), &pw, 0, NULL, NULL) == -1) return -1; + if (hc_clEnqueueWriteBuffer (hashcat_ctx, device_param->opencl_command_queue, device_param->opencl_d_pws_buf, CL_FALSE, 0, 1 * sizeof (pw_t), &pw, 0, NULL, NULL) == -1) return -1; } } else @@ -135,17 +135,17 @@ static int selftest (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param if (device_param->is_cuda == true) { - if (hc_cuMemcpyHtoD (hashcat_ctx, device_param->cuda_d_pws_buf, &pw, 1 * sizeof (pw_t)) == -1) return -1; + if (hc_cuMemcpyHtoDAsync (hashcat_ctx, device_param->cuda_d_pws_buf, &pw, 1 * sizeof (pw_t), device_param->cuda_stream) == -1) return -1; } if (device_param->is_hip == true) { - if (hc_hipMemcpyHtoD (hashcat_ctx, device_param->hip_d_pws_buf, &pw, 1 * sizeof (pw_t)) == -1) return -1; + if (hc_hipMemcpyHtoDAsync (hashcat_ctx, device_param->hip_d_pws_buf, &pw, 1 * sizeof (pw_t), device_param->hip_stream) == -1) return -1; } if (device_param->is_opencl == true) { - if (hc_clEnqueueWriteBuffer (hashcat_ctx, device_param->opencl_command_queue, device_param->opencl_d_pws_buf, CL_TRUE, 0, 1 * sizeof (pw_t), &pw, 0, NULL, NULL) == -1) return -1; + if (hc_clEnqueueWriteBuffer (hashcat_ctx, device_param->opencl_command_queue, device_param->opencl_d_pws_buf, CL_FALSE, 0, 1 * sizeof (pw_t), &pw, 0, NULL, NULL) == -1) return -1; } } else if (user_options_extra->attack_kern == ATTACK_KERN_COMBI) @@ -202,23 +202,23 @@ static int selftest (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param if (device_param->is_cuda == true) { - if (hc_cuMemcpyHtoD (hashcat_ctx, device_param->cuda_d_combs_c, &comb, 1 * sizeof (pw_t)) == -1) return -1; + if (hc_cuMemcpyHtoDAsync (hashcat_ctx, device_param->cuda_d_combs_c, &comb, 1 * sizeof (pw_t), device_param->cuda_stream) == -1) return -1; - if (hc_cuMemcpyHtoD (hashcat_ctx, device_param->cuda_d_pws_buf, &pw, 1 * sizeof (pw_t)) == -1) return -1; + if (hc_cuMemcpyHtoDAsync (hashcat_ctx, device_param->cuda_d_pws_buf, &pw, 1 * sizeof (pw_t), device_param->cuda_stream) == -1) return -1; } if (device_param->is_hip == true) { - if (hc_hipMemcpyHtoD (hashcat_ctx, device_param->hip_d_combs_c, &comb, 1 * sizeof (pw_t)) == -1) return -1; + if (hc_hipMemcpyHtoDAsync (hashcat_ctx, device_param->hip_d_combs_c, &comb, 1 * sizeof (pw_t), device_param->hip_stream) == -1) return -1; - if (hc_hipMemcpyHtoD (hashcat_ctx, device_param->hip_d_pws_buf, &pw, 1 * sizeof (pw_t)) == -1) return -1; + if (hc_hipMemcpyHtoDAsync (hashcat_ctx, device_param->hip_d_pws_buf, &pw, 1 * sizeof (pw_t), device_param->hip_stream) == -1) return -1; } if (device_param->is_opencl == true) { - if (hc_clEnqueueWriteBuffer (hashcat_ctx, device_param->opencl_command_queue, device_param->opencl_d_combs_c, CL_TRUE, 0, 1 * sizeof (pw_t), &comb, 0, NULL, NULL) == -1) return -1; + if (hc_clEnqueueWriteBuffer (hashcat_ctx, device_param->opencl_command_queue, device_param->opencl_d_combs_c, CL_FALSE, 0, 1 * sizeof (pw_t), &comb, 0, NULL, NULL) == -1) return -1; - if (hc_clEnqueueWriteBuffer (hashcat_ctx, device_param->opencl_command_queue, device_param->opencl_d_pws_buf, CL_TRUE, 0, 1 * sizeof (pw_t), &pw, 0, NULL, NULL) == -1) return -1; + if (hc_clEnqueueWriteBuffer (hashcat_ctx, device_param->opencl_command_queue, device_param->opencl_d_pws_buf, CL_FALSE, 0, 1 * sizeof (pw_t), &pw, 0, NULL, NULL) == -1) return -1; } } else if (user_options_extra->attack_kern == ATTACK_KERN_BF) @@ -246,17 +246,17 @@ static int selftest (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param if (device_param->is_cuda == true) { - if (hc_cuMemcpyHtoD (hashcat_ctx, device_param->cuda_d_pws_buf, &pw, 1 * sizeof (pw_t)) == -1) return -1; + if (hc_cuMemcpyHtoDAsync (hashcat_ctx, device_param->cuda_d_pws_buf, &pw, 1 * sizeof (pw_t), device_param->cuda_stream) == -1) return -1; } if (device_param->is_hip == true) { - if (hc_hipMemcpyHtoD (hashcat_ctx, device_param->hip_d_pws_buf, &pw, 1 * sizeof (pw_t)) == -1) return -1; + if (hc_hipMemcpyHtoDAsync (hashcat_ctx, device_param->hip_d_pws_buf, &pw, 1 * sizeof (pw_t), device_param->hip_stream) == -1) return -1; } if (device_param->is_opencl == true) { - if (hc_clEnqueueWriteBuffer (hashcat_ctx, device_param->opencl_command_queue, device_param->opencl_d_pws_buf, CL_TRUE, 0, 1 * sizeof (pw_t), &pw, 0, NULL, NULL) == -1) return -1; + if (hc_clEnqueueWriteBuffer (hashcat_ctx, device_param->opencl_command_queue, device_param->opencl_d_pws_buf, CL_FALSE, 0, 1 * sizeof (pw_t), &pw, 0, NULL, NULL) == -1) return -1; } } else @@ -302,17 +302,17 @@ static int selftest (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param if (device_param->is_cuda == true) { - if (hc_cuMemcpyHtoD (hashcat_ctx, device_param->cuda_d_bfs_c, &bf, 1 * sizeof (bf_t)) == -1) return -1; + if (hc_cuMemcpyHtoDAsync (hashcat_ctx, device_param->cuda_d_bfs_c, &bf, 1 * sizeof (bf_t), device_param->cuda_stream) == -1) return -1; } if (device_param->is_hip == true) { - if (hc_hipMemcpyHtoD (hashcat_ctx, device_param->hip_d_bfs_c, &bf, 1 * sizeof (bf_t)) == -1) return -1; + if (hc_hipMemcpyHtoDAsync (hashcat_ctx, device_param->hip_d_bfs_c, &bf, 1 * sizeof (bf_t), device_param->hip_stream) == -1) return -1; } if (device_param->is_opencl == true) { - if (hc_clEnqueueWriteBuffer (hashcat_ctx, device_param->opencl_command_queue, device_param->opencl_d_bfs_c, CL_TRUE, 0, 1 * sizeof (bf_t), &bf, 0, NULL, NULL) == -1) return -1; + if (hc_clEnqueueWriteBuffer (hashcat_ctx, device_param->opencl_command_queue, device_param->opencl_d_bfs_c, CL_FALSE, 0, 1 * sizeof (bf_t), &bf, 0, NULL, NULL) == -1) return -1; } pw_t pw; @@ -403,17 +403,17 @@ static int selftest (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param if (device_param->is_cuda == true) { - if (hc_cuMemcpyHtoD (hashcat_ctx, device_param->cuda_d_pws_buf, &pw, 1 * sizeof (pw_t)) == -1) return -1; + if (hc_cuMemcpyHtoDAsync (hashcat_ctx, device_param->cuda_d_pws_buf, &pw, 1 * sizeof (pw_t), device_param->cuda_stream) == -1) return -1; } if (device_param->is_hip == true) { - if (hc_hipMemcpyHtoD (hashcat_ctx, device_param->hip_d_pws_buf, &pw, 1 * sizeof (pw_t)) == -1) return -1; + if (hc_hipMemcpyHtoDAsync (hashcat_ctx, device_param->hip_d_pws_buf, &pw, 1 * sizeof (pw_t), device_param->hip_stream) == -1) return -1; } if (device_param->is_opencl == true) { - if (hc_clEnqueueWriteBuffer (hashcat_ctx, device_param->opencl_command_queue, device_param->opencl_d_pws_buf, CL_TRUE, 0, 1 * sizeof (pw_t), &pw, 0, NULL, NULL) == -1) return -1; + if (hc_clEnqueueWriteBuffer (hashcat_ctx, device_param->opencl_command_queue, device_param->opencl_d_pws_buf, CL_FALSE, 0, 1 * sizeof (pw_t), &pw, 0, NULL, NULL) == -1) return -1; } highest_pw_len = pw.pw_len; @@ -436,17 +436,17 @@ static int selftest (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param if (device_param->is_cuda == true) { - if (hc_cuMemcpyHtoD (hashcat_ctx, device_param->cuda_d_pws_buf, &pw, 1 * sizeof (pw_t)) == -1) return -1; + if (hc_cuMemcpyHtoDAsync (hashcat_ctx, device_param->cuda_d_pws_buf, &pw, 1 * sizeof (pw_t), device_param->cuda_stream) == -1) return -1; } if (device_param->is_hip == true) { - if (hc_hipMemcpyHtoD (hashcat_ctx, device_param->hip_d_pws_buf, &pw, 1 * sizeof (pw_t)) == -1) return -1; + if (hc_hipMemcpyHtoDAsync (hashcat_ctx, device_param->hip_d_pws_buf, &pw, 1 * sizeof (pw_t), device_param->hip_stream) == -1) return -1; } if (device_param->is_opencl == true) { - if (hc_clEnqueueWriteBuffer (hashcat_ctx, device_param->opencl_command_queue, device_param->opencl_d_pws_buf, CL_TRUE, 0, 1 * sizeof (pw_t), &pw, 0, NULL, NULL) == -1) return -1; + if (hc_clEnqueueWriteBuffer (hashcat_ctx, device_param->opencl_command_queue, device_param->opencl_d_pws_buf, CL_FALSE, 0, 1 * sizeof (pw_t), &pw, 0, NULL, NULL) == -1) return -1; } } } @@ -509,12 +509,16 @@ static int selftest (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param if (device_param->is_cuda == true) { - if (hc_cuMemcpyDtoH (hashcat_ctx, device_param->hooks_buf, device_param->cuda_d_hooks, device_param->size_hooks) == -1) return -1; + if (hc_cuMemcpyDtoHAsync (hashcat_ctx, device_param->hooks_buf, device_param->cuda_d_hooks, device_param->size_hooks, device_param->cuda_stream) == -1) return -1; + + if (hc_cuStreamSynchronize (hashcat_ctx, device_param->cuda_stream) == -1) return -1; } if (device_param->is_hip == true) { - if (hc_hipMemcpyDtoH (hashcat_ctx, device_param->hooks_buf, device_param->hip_d_hooks, device_param->size_hooks) == -1) return -1; + if (hc_hipMemcpyDtoHAsync (hashcat_ctx, device_param->hooks_buf, device_param->hip_d_hooks, device_param->size_hooks, device_param->hip_stream) == -1) return -1; + + if (hc_hipStreamSynchronize (hashcat_ctx, device_param->hip_stream) == -1) return -1; } if (device_param->is_opencl == true) @@ -526,17 +530,17 @@ static int selftest (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param if (device_param->is_cuda == true) { - if (hc_cuMemcpyHtoD (hashcat_ctx, device_param->cuda_d_hooks, device_param->hooks_buf, device_param->size_hooks) == -1) return -1; + if (hc_cuMemcpyHtoDAsync (hashcat_ctx, device_param->cuda_d_hooks, device_param->hooks_buf, device_param->size_hooks, device_param->cuda_stream) == -1) return -1; } if (device_param->is_hip == true) { - if (hc_hipMemcpyHtoD (hashcat_ctx, device_param->hip_d_hooks, device_param->hooks_buf, device_param->size_hooks) == -1) return -1; + if (hc_hipMemcpyHtoDAsync (hashcat_ctx, device_param->hip_d_hooks, device_param->hooks_buf, device_param->size_hooks, device_param->hip_stream) == -1) return -1; } if (device_param->is_opencl == true) { - if (hc_clEnqueueWriteBuffer (hashcat_ctx, device_param->opencl_command_queue, device_param->opencl_d_hooks, CL_TRUE, 0, device_param->size_hooks, device_param->hooks_buf, 0, NULL, NULL) == -1) return -1; + if (hc_clEnqueueWriteBuffer (hashcat_ctx, device_param->opencl_command_queue, device_param->opencl_d_hooks, CL_FALSE, 0, device_param->size_hooks, device_param->hooks_buf, 0, NULL, NULL) == -1) return -1; } } @@ -582,12 +586,16 @@ static int selftest (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param if (device_param->is_cuda == true) { - if (hc_cuMemcpyDtoH (hashcat_ctx, device_param->hooks_buf, device_param->cuda_d_hooks, device_param->size_hooks) == -1) return -1; + if (hc_cuMemcpyDtoHAsync (hashcat_ctx, device_param->hooks_buf, device_param->cuda_d_hooks, device_param->size_hooks, device_param->cuda_stream) == -1) return -1; + + if (hc_cuStreamSynchronize (hashcat_ctx, device_param->cuda_stream) == -1) return -1; } if (device_param->is_hip == true) { - if (hc_hipMemcpyDtoH (hashcat_ctx, device_param->hooks_buf, device_param->hip_d_hooks, device_param->size_hooks) == -1) return -1; + if (hc_hipMemcpyDtoHAsync (hashcat_ctx, device_param->hooks_buf, device_param->hip_d_hooks, device_param->size_hooks, device_param->hip_stream) == -1) return -1; + + if (hc_hipStreamSynchronize (hashcat_ctx, device_param->hip_stream) == -1) return -1; } if (device_param->is_opencl == true) @@ -599,17 +607,17 @@ static int selftest (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param if (device_param->is_cuda == true) { - if (hc_cuMemcpyHtoD (hashcat_ctx, device_param->cuda_d_hooks, device_param->hooks_buf, device_param->size_hooks) == -1) return -1; + if (hc_cuMemcpyHtoDAsync (hashcat_ctx, device_param->cuda_d_hooks, device_param->hooks_buf, device_param->size_hooks, device_param->cuda_stream) == -1) return -1; } if (device_param->is_hip == true) { - if (hc_hipMemcpyHtoD (hashcat_ctx, device_param->hip_d_hooks, device_param->hooks_buf, device_param->size_hooks) == -1) return -1; + if (hc_hipMemcpyHtoDAsync (hashcat_ctx, device_param->hip_d_hooks, device_param->hooks_buf, device_param->size_hooks, device_param->hip_stream) == -1) return -1; } if (device_param->is_opencl == true) { - if (hc_clEnqueueWriteBuffer (hashcat_ctx, device_param->opencl_command_queue, device_param->opencl_d_hooks, CL_TRUE, 0, device_param->size_hooks, device_param->hooks_buf, 0, NULL, NULL) == -1) return -1; + if (hc_clEnqueueWriteBuffer (hashcat_ctx, device_param->opencl_command_queue, device_param->opencl_d_hooks, CL_FALSE, 0, device_param->size_hooks, device_param->hooks_buf, 0, NULL, NULL) == -1) return -1; } } } @@ -681,19 +689,25 @@ static int selftest (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param u32 num_cracked = 0; + cl_event opencl_event; + if (device_param->is_cuda == true) { - if (hc_cuMemcpyDtoH (hashcat_ctx, &num_cracked, device_param->cuda_d_result, sizeof (u32)) == -1) return -1; + if (hc_cuMemcpyDtoHAsync (hashcat_ctx, &num_cracked, device_param->cuda_d_result, sizeof (u32), device_param->cuda_stream) == -1) return -1; + + if (hc_cuEventRecord (hashcat_ctx, device_param->cuda_event1, device_param->cuda_stream) == -1) return -1; } if (device_param->is_hip == true) { - if (hc_hipMemcpyDtoH (hashcat_ctx, &num_cracked, device_param->hip_d_result, sizeof (u32)) == -1) return -1; + if (hc_hipMemcpyDtoHAsync (hashcat_ctx, &num_cracked, device_param->hip_d_result, sizeof (u32), device_param->hip_stream) == -1) return -1; + + if (hc_hipEventRecord (hashcat_ctx, device_param->hip_event1, device_param->hip_stream) == -1) return -1; } if (device_param->is_opencl == true) { - if (hc_clEnqueueReadBuffer (hashcat_ctx, device_param->opencl_command_queue, device_param->opencl_d_result, CL_TRUE, 0, sizeof (u32), &num_cracked, 0, NULL, NULL) == -1) return -1; + if (hc_clEnqueueReadBuffer (hashcat_ctx, device_param->opencl_command_queue, device_param->opencl_d_result, CL_FALSE, 0, sizeof (u32), &num_cracked, 0, NULL, &opencl_event) == -1) return -1; } // finish : cleanup and restore @@ -712,12 +726,12 @@ static int selftest (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param device_param->kernel_params[17] = &device_param->cuda_d_salt_bufs; device_param->kernel_params[18] = &device_param->cuda_d_esalt_bufs; - if (run_cuda_kernel_bzero (hashcat_ctx, device_param, device_param->cuda_d_pws_buf, device_param->size_pws) == -1) return -1; - if (run_cuda_kernel_bzero (hashcat_ctx, device_param, device_param->cuda_d_tmps, device_param->size_tmps) == -1) return -1; - if (run_cuda_kernel_bzero (hashcat_ctx, device_param, device_param->cuda_d_hooks, device_param->size_hooks) == -1) return -1; - if (run_cuda_kernel_bzero (hashcat_ctx, device_param, device_param->cuda_d_plain_bufs, device_param->size_plains) == -1) return -1; - if (run_cuda_kernel_bzero (hashcat_ctx, device_param, device_param->cuda_d_digests_shown, device_param->size_shown) == -1) return -1; - if (run_cuda_kernel_bzero (hashcat_ctx, device_param, device_param->cuda_d_result, device_param->size_results) == -1) return -1; + if (run_cuda_kernel_bzero (hashcat_ctx, device_param, device_param->cuda_d_pws_buf, device_param->size_pws) == -1) return -1; + if (run_cuda_kernel_bzero (hashcat_ctx, device_param, device_param->cuda_d_tmps, device_param->size_tmps) == -1) return -1; + if (run_cuda_kernel_bzero (hashcat_ctx, device_param, device_param->cuda_d_hooks, device_param->size_hooks) == -1) return -1; + if (run_cuda_kernel_bzero (hashcat_ctx, device_param, device_param->cuda_d_plain_bufs, device_param->size_plains) == -1) return -1; + if (run_cuda_kernel_bzero (hashcat_ctx, device_param, device_param->cuda_d_digests_shown, device_param->size_shown) == -1) return -1; + if (run_cuda_kernel_bzero (hashcat_ctx, device_param, device_param->cuda_d_result, device_param->size_results) == -1) return -1; } if (device_param->is_hip == true) @@ -726,12 +740,12 @@ static int selftest (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param device_param->kernel_params[17] = &device_param->hip_d_salt_bufs; device_param->kernel_params[18] = &device_param->hip_d_esalt_bufs; - if (run_hip_kernel_bzero (hashcat_ctx, device_param, device_param->hip_d_pws_buf, device_param->size_pws) == -1) return -1; - if (run_hip_kernel_bzero (hashcat_ctx, device_param, device_param->hip_d_tmps, device_param->size_tmps) == -1) return -1; - if (run_hip_kernel_bzero (hashcat_ctx, device_param, device_param->hip_d_hooks, device_param->size_hooks) == -1) return -1; - if (run_hip_kernel_bzero (hashcat_ctx, device_param, device_param->hip_d_plain_bufs, device_param->size_plains) == -1) return -1; - if (run_hip_kernel_bzero (hashcat_ctx, device_param, device_param->hip_d_digests_shown, device_param->size_shown) == -1) return -1; - if (run_hip_kernel_bzero (hashcat_ctx, device_param, device_param->hip_d_result, device_param->size_results) == -1) return -1; + if (run_hip_kernel_bzero (hashcat_ctx, device_param, device_param->hip_d_pws_buf, device_param->size_pws) == -1) return -1; + if (run_hip_kernel_bzero (hashcat_ctx, device_param, device_param->hip_d_tmps, device_param->size_tmps) == -1) return -1; + if (run_hip_kernel_bzero (hashcat_ctx, device_param, device_param->hip_d_hooks, device_param->size_hooks) == -1) return -1; + if (run_hip_kernel_bzero (hashcat_ctx, device_param, device_param->hip_d_plain_bufs, device_param->size_plains) == -1) return -1; + if (run_hip_kernel_bzero (hashcat_ctx, device_param, device_param->hip_d_digests_shown, device_param->size_shown) == -1) return -1; + if (run_hip_kernel_bzero (hashcat_ctx, device_param, device_param->hip_d_result, device_param->size_results) == -1) return -1; } if (device_param->is_opencl == true) @@ -820,8 +834,25 @@ static int selftest (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param } } - // check return + // synchronize and .. + if (device_param->is_cuda == true) + { + if (hc_cuEventSynchronize (hashcat_ctx, device_param->cuda_event1) == -1) return -1; + } + if (device_param->is_hip == true) + { + if (hc_hipEventSynchronize (hashcat_ctx, device_param->hip_event1) == -1) return -1; + } + + if (device_param->is_opencl == true) + { + if (hc_clWaitForEvents (hashcat_ctx, 1, &opencl_event) == -1) return -1; + + if (hc_clReleaseEvent (hashcat_ctx, opencl_event) == -1) return -1; + } + + // check return if (num_cracked == 0) { hc_thread_mutex_lock (status_ctx->mux_display); From a86b44a7501569378b148d3999cd07e6fb1fb954 Mon Sep 17 00:00:00 2001 From: Jukka Ojanen Date: Mon, 26 Jul 2021 01:51:19 +0300 Subject: [PATCH 06/33] Allow async execution of autotune() and prepare try_run() for async kernel execution --- src/autotune.c | 38 ++++++++++++++++++++++++++++++++++++-- 1 file changed, 36 insertions(+), 2 deletions(-) diff --git a/src/autotune.c b/src/autotune.c index cdeb73553..e8148daf7 100644 --- a/src/autotune.c +++ b/src/autotune.c @@ -37,6 +37,23 @@ static double try_run (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_par device_param->spin_damp = 0; + if (device_param->is_cuda == true) + { + if (hc_cuStreamSynchronize (hashcat_ctx, device_param->cuda_stream) == -1) return -1; + } + + if (device_param->is_hip == true) + { + if (hc_hipStreamSynchronize (hashcat_ctx, device_param->hip_stream) == -1) return -1; + } + + if (device_param->is_opencl == true) + { + /*if (hc_clFlush (hashcat_ctx, device_param->opencl_command_queue) == -1) return -1;*/ + + if (hc_clFinish (hashcat_ctx, device_param->opencl_command_queue) == -1) return -1; + } + if (hashconfig->attack_exec == ATTACK_EXEC_INSIDE_KERNEL) { if (hashconfig->opti_type & OPTI_TYPE_OPTIMIZED_KERNEL) @@ -60,6 +77,23 @@ static double try_run (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_par run_kernel (hashcat_ctx, device_param, KERN_RUN_2, 0, kernel_power_try, true, 0); } + if (device_param->is_cuda == true) + { + if (hc_cuStreamSynchronize (hashcat_ctx, device_param->cuda_stream) == -1) return -1; + } + + if (device_param->is_hip == true) + { + if (hc_hipStreamSynchronize (hashcat_ctx, device_param->hip_stream) == -1) return -1; + } + + if (device_param->is_opencl == true) + { + /*if (hc_clFlush (hashcat_ctx, device_param->opencl_command_queue) == -1) return -1;*/ + + if (hc_clFinish (hashcat_ctx, device_param->opencl_command_queue) == -1) return -1; + } + device_param->spin_damp = spin_damp_sav; const double exec_msec_prev = get_avg_exec_time (device_param, 1); @@ -200,14 +234,14 @@ static int autotune (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param { if (device_param->is_cuda == true) { - CU_rc = hc_cuMemcpyDtoD (hashcat_ctx, device_param->cuda_d_rules_c, device_param->cuda_d_rules, MIN (kernel_loops_max, KERNEL_RULES) * sizeof (kernel_rule_t)); + CU_rc = hc_cuMemcpyDtoDAsync (hashcat_ctx, device_param->cuda_d_rules_c, device_param->cuda_d_rules, MIN (kernel_loops_max, KERNEL_RULES) * sizeof (kernel_rule_t), device_param->cuda_stream); if (CU_rc == -1) return -1; } if (device_param->is_hip == true) { - HIP_rc = hc_hipMemcpyDtoD (hashcat_ctx, device_param->hip_d_rules_c, device_param->hip_d_rules, MIN (kernel_loops_max, KERNEL_RULES) * sizeof (kernel_rule_t)); + HIP_rc = hc_hipMemcpyDtoDAsync (hashcat_ctx, device_param->hip_d_rules_c, device_param->hip_d_rules, MIN (kernel_loops_max, KERNEL_RULES) * sizeof (kernel_rule_t), device_param->hip_stream); if (HIP_rc == -1) return -1; } From de5200cffccabe389ddceca6ac1f23f301bafb6e Mon Sep 17 00:00:00 2001 From: Jukka Ojanen Date: Mon, 26 Jul 2021 15:17:25 +0300 Subject: [PATCH 07/33] Allow async execution of backend --- src/backend.c | 300 ++++++++++++++++++++++---------------------------- 1 file changed, 130 insertions(+), 170 deletions(-) diff --git a/src/backend.c b/src/backend.c index 2b52b4cde..9631e8dd2 100644 --- a/src/backend.c +++ b/src/backend.c @@ -4513,7 +4513,9 @@ int gidd_to_pw_t (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param, c { if (hc_cuCtxPushCurrent (hashcat_ctx, device_param->cuda_context) == -1) return -1; - if (hc_cuMemcpyDtoH (hashcat_ctx, &pw_idx, device_param->cuda_d_pws_idx + (gidd * sizeof (pw_idx_t)), sizeof (pw_idx_t)) == -1) return -1; + if (hc_cuMemcpyDtoHAsync (hashcat_ctx, &pw_idx, device_param->cuda_d_pws_idx + (gidd * sizeof (pw_idx_t)), sizeof (pw_idx_t), device_param->cuda_stream) == -1) return -1; + + if (hc_cuStreamSynchronize (hashcat_ctx, device_param->cuda_stream) == -1) return -1; if (hc_cuCtxPopCurrent (hashcat_ctx, &device_param->cuda_context) == -1) return -1; } @@ -4522,7 +4524,9 @@ int gidd_to_pw_t (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param, c { if (hc_hipCtxPushCurrent (hashcat_ctx, device_param->hip_context) == -1) return -1; - if (hc_hipMemcpyDtoH (hashcat_ctx, &pw_idx, device_param->hip_d_pws_idx + (gidd * sizeof (pw_idx_t)), sizeof (pw_idx_t)) == -1) return -1; + if (hc_hipMemcpyDtoHAsync (hashcat_ctx, &pw_idx, device_param->hip_d_pws_idx + (gidd * sizeof (pw_idx_t)), sizeof (pw_idx_t), device_param->hip_stream) == -1) return -1; + + if (hc_hipStreamSynchronize (hashcat_ctx, device_param->hip_stream) == -1) return -1; if (hc_hipCtxPopCurrent (hashcat_ctx, &device_param->hip_context) == -1) return -1; } @@ -4542,7 +4546,9 @@ int gidd_to_pw_t (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param, c { if (hc_cuCtxPushCurrent (hashcat_ctx, device_param->cuda_context) == -1) return -1; - if (hc_cuMemcpyDtoH (hashcat_ctx,pw->i, device_param->cuda_d_pws_comp_buf + (off * sizeof (u32)), cnt * sizeof (u32)) == -1) return -1; + if (hc_cuMemcpyDtoHAsync (hashcat_ctx,pw->i, device_param->cuda_d_pws_comp_buf + (off * sizeof (u32)), cnt * sizeof (u32), device_param->cuda_stream) == -1) return -1; + + if (hc_cuStreamSynchronize (hashcat_ctx, device_param->cuda_stream) == -1) return -1; if (hc_cuCtxPopCurrent (hashcat_ctx, &device_param->cuda_context) == -1) return -1; } @@ -4554,7 +4560,9 @@ int gidd_to_pw_t (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param, c { if (hc_hipCtxPushCurrent (hashcat_ctx, device_param->hip_context) == -1) return -1; - if (hc_hipMemcpyDtoH (hashcat_ctx,pw->i, device_param->hip_d_pws_comp_buf + (off * sizeof (u32)), cnt * sizeof (u32)) == -1) return -1; + if (hc_hipMemcpyDtoHAsync (hashcat_ctx,pw->i, device_param->hip_d_pws_comp_buf + (off * sizeof (u32)), cnt * sizeof (u32), device_param->hip_stream) == -1) return -1; + + if (hc_hipStreamSynchronize (hashcat_ctx, device_param->hip_stream) == -1) return -1; if (hc_hipCtxPopCurrent (hashcat_ctx, &device_param->hip_context) == -1) return -1; } @@ -4623,12 +4631,12 @@ int choose_kernel (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param, if (device_param->is_cuda == true) { - if (hc_cuMemcpyDtoD (hashcat_ctx, device_param->cuda_d_bfs_c, device_param->cuda_d_tm_c, size_tm) == -1) return -1; + if (hc_cuMemcpyDtoDAsync (hashcat_ctx, device_param->cuda_d_bfs_c, device_param->cuda_d_tm_c, size_tm, device_param->cuda_stream) == -1) return -1; } if (device_param->is_hip == true) { - if (hc_hipMemcpyDtoD (hashcat_ctx, device_param->hip_d_bfs_c, device_param->hip_d_tm_c, size_tm) == -1) return -1; + if (hc_hipMemcpyDtoDAsync (hashcat_ctx, device_param->hip_d_bfs_c, device_param->hip_d_tm_c, size_tm, device_param->hip_stream) == -1) return -1; } if (device_param->is_opencl == true) @@ -4689,12 +4697,12 @@ int choose_kernel (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param, { if (device_param->is_cuda == true) { - if (hc_cuMemcpyDtoD (hashcat_ctx, device_param->cuda_d_pws_buf, device_param->cuda_d_pws_amp_buf, pws_cnt * sizeof (pw_t)) == -1) return -1; + if (hc_cuMemcpyDtoDAsync (hashcat_ctx, device_param->cuda_d_pws_buf, device_param->cuda_d_pws_amp_buf, pws_cnt * sizeof (pw_t), device_param->cuda_stream) == -1) return -1; } if (device_param->is_hip == true) { - if (hc_hipMemcpyDtoD (hashcat_ctx, device_param->hip_d_pws_buf, device_param->hip_d_pws_amp_buf, pws_cnt * sizeof (pw_t)) == -1) return -1; + if (hc_hipMemcpyDtoDAsync (hashcat_ctx, device_param->hip_d_pws_buf, device_param->hip_d_pws_amp_buf, pws_cnt * sizeof (pw_t), device_param->hip_stream) == -1) return -1; } if (device_param->is_opencl == true) @@ -4736,17 +4744,17 @@ int choose_kernel (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param, if (device_param->is_cuda == true) { - if (hc_cuMemcpyDtoH (hashcat_ctx, device_param->hooks_buf, device_param->cuda_d_hooks, pws_cnt * hashconfig->hook_size) == -1) return -1; + if (hc_cuMemcpyDtoHAsync (hashcat_ctx, device_param->hooks_buf, device_param->cuda_d_hooks, pws_cnt * hashconfig->hook_size, device_param->cuda_stream) == -1) return -1; } if (device_param->is_hip == true) { - if (hc_hipMemcpyDtoH (hashcat_ctx, device_param->hooks_buf, device_param->hip_d_hooks, pws_cnt * hashconfig->hook_size) == -1) return -1; + if (hc_hipMemcpyDtoHAsync (hashcat_ctx, device_param->hooks_buf, device_param->hip_d_hooks, pws_cnt * hashconfig->hook_size, device_param->hip_stream) == -1) return -1; } if (device_param->is_opencl == true) { - if (hc_clEnqueueReadBuffer (hashcat_ctx, device_param->opencl_command_queue, device_param->opencl_d_hooks, CL_TRUE, 0, pws_cnt * hashconfig->hook_size, device_param->hooks_buf, 0, NULL, NULL) == -1) return -1; + if (hc_clEnqueueReadBuffer (hashcat_ctx, device_param->opencl_command_queue, device_param->opencl_d_hooks, CL_FALSE, 0, pws_cnt * hashconfig->hook_size, device_param->hooks_buf, 0, NULL, NULL) == -1) return -1; } const int hook_threads = (int) user_options->hook_threads; @@ -4790,17 +4798,17 @@ int choose_kernel (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param, if (device_param->is_cuda == true) { - if (hc_cuMemcpyHtoD (hashcat_ctx, device_param->cuda_d_hooks, device_param->hooks_buf, pws_cnt * hashconfig->hook_size) == -1) return -1; + if (hc_cuMemcpyHtoDAsync (hashcat_ctx, device_param->cuda_d_hooks, device_param->hooks_buf, pws_cnt * hashconfig->hook_size, device_param->cuda_stream) == -1) return -1; } if (device_param->is_hip == true) { - if (hc_hipMemcpyHtoD (hashcat_ctx, device_param->hip_d_hooks, device_param->hooks_buf, pws_cnt * hashconfig->hook_size) == -1) return -1; + if (hc_hipMemcpyHtoDAsync (hashcat_ctx, device_param->hip_d_hooks, device_param->hooks_buf, pws_cnt * hashconfig->hook_size, device_param->hip_stream) == -1) return -1; } if (device_param->is_opencl == true) { - if (hc_clEnqueueWriteBuffer (hashcat_ctx, device_param->opencl_command_queue, device_param->opencl_d_hooks, CL_TRUE, 0, pws_cnt * hashconfig->hook_size, device_param->hooks_buf, 0, NULL, NULL) == -1) return -1; + if (hc_clEnqueueWriteBuffer (hashcat_ctx, device_param->opencl_command_queue, device_param->opencl_d_hooks, CL_FALSE, 0, pws_cnt * hashconfig->hook_size, device_param->hooks_buf, 0, NULL, NULL) == -1) return -1; } } } @@ -4884,17 +4892,17 @@ int choose_kernel (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param, if (device_param->is_cuda == true) { - if (hc_cuMemcpyDtoH (hashcat_ctx, device_param->hooks_buf, device_param->cuda_d_hooks, pws_cnt * hashconfig->hook_size) == -1) return -1; + if (hc_cuMemcpyDtoHAsync (hashcat_ctx, device_param->hooks_buf, device_param->cuda_d_hooks, pws_cnt * hashconfig->hook_size, device_param->cuda_stream) == -1) return -1; } if (device_param->is_hip == true) { - if (hc_hipMemcpyDtoH (hashcat_ctx, device_param->hooks_buf, device_param->hip_d_hooks, pws_cnt * hashconfig->hook_size) == -1) return -1; + if (hc_hipMemcpyDtoHAsync (hashcat_ctx, device_param->hooks_buf, device_param->hip_d_hooks, pws_cnt * hashconfig->hook_size, device_param->hip_stream) == -1) return -1; } if (device_param->is_opencl == true) { - if (hc_clEnqueueReadBuffer (hashcat_ctx, device_param->opencl_command_queue, device_param->opencl_d_hooks, CL_TRUE, 0, pws_cnt * hashconfig->hook_size, device_param->hooks_buf, 0, NULL, NULL) == -1) return -1; + if (hc_clEnqueueReadBuffer (hashcat_ctx, device_param->opencl_command_queue, device_param->opencl_d_hooks, CL_FALSE, 0, pws_cnt * hashconfig->hook_size, device_param->hooks_buf, 0, NULL, NULL) == -1) return -1; } const int hook_threads = (int) user_options->hook_threads; @@ -4938,17 +4946,17 @@ int choose_kernel (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param, if (device_param->is_cuda == true) { - if (hc_cuMemcpyHtoD (hashcat_ctx, device_param->cuda_d_hooks, device_param->hooks_buf, pws_cnt * hashconfig->hook_size) == -1) return -1; + if (hc_cuMemcpyHtoDAsync (hashcat_ctx, device_param->cuda_d_hooks, device_param->hooks_buf, pws_cnt * hashconfig->hook_size, device_param->cuda_stream) == -1) return -1; } if (device_param->is_hip == true) { - if (hc_hipMemcpyHtoD (hashcat_ctx, device_param->hip_d_hooks, device_param->hooks_buf, pws_cnt * hashconfig->hook_size) == -1) return -1; + if (hc_hipMemcpyHtoDAsync (hashcat_ctx, device_param->hip_d_hooks, device_param->hooks_buf, pws_cnt * hashconfig->hook_size, device_param->hip_stream) == -1) return -1; } if (device_param->is_opencl == true) { - if (hc_clEnqueueWriteBuffer (hashcat_ctx, device_param->opencl_command_queue, device_param->opencl_d_hooks, CL_TRUE, 0, pws_cnt * hashconfig->hook_size, device_param->hooks_buf, 0, NULL, NULL) == -1) return -1; + if (hc_clEnqueueWriteBuffer (hashcat_ctx, device_param->opencl_command_queue, device_param->opencl_d_hooks, CL_FALSE, 0, pws_cnt * hashconfig->hook_size, device_param->hooks_buf, 0, NULL, NULL) == -1) return -1; } } } @@ -5152,8 +5160,6 @@ int run_cuda_kernel_atinit (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *devic if (hc_cuLaunchKernel (hashcat_ctx, function, num_elements, 1, 1, kernel_threads, 1, 1, 0, device_param->cuda_stream, device_param->kernel_params_atinit, NULL) == -1) return -1; - if (hc_cuStreamSynchronize (hashcat_ctx, device_param->cuda_stream) == -1) return -1; - return 0; } @@ -5172,8 +5178,6 @@ int run_cuda_kernel_utf8toutf16le (hashcat_ctx_t *hashcat_ctx, hc_device_param_t if (hc_cuLaunchKernel (hashcat_ctx, function, num_elements, 1, 1, kernel_threads, 1, 1, 0, device_param->cuda_stream, device_param->kernel_params_utf8toutf16le, NULL) == -1) return -1; - if (hc_cuStreamSynchronize (hashcat_ctx, device_param->cuda_stream) == -1) return -1; - return 0; } @@ -5237,8 +5241,6 @@ int run_cuda_kernel_bzero (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device if (hc_cuMemcpyHtoDAsync (hashcat_ctx, buf + (num16d * 16), bzeros, num16m, device_param->cuda_stream) == -1) return -1; } - /*if (hc_cuStreamSynchronize (hashcat_ctx, device_param->cuda_stream) == -1) return -1;*/ - return 0; } @@ -5257,8 +5259,6 @@ int run_hip_kernel_atinit (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device if (hc_hipLaunchKernel (hashcat_ctx, function, num_elements, 1, 1, kernel_threads, 1, 1, 0, device_param->hip_stream, device_param->kernel_params_atinit, NULL) == -1) return -1; - if (hc_hipStreamSynchronize (hashcat_ctx, device_param->hip_stream) == -1) return -1; - return 0; } @@ -5277,8 +5277,6 @@ int run_hip_kernel_utf8toutf16le (hashcat_ctx_t *hashcat_ctx, hc_device_param_t if (hc_hipLaunchKernel (hashcat_ctx, function, num_elements, 1, 1, kernel_threads, 1, 1, 0, device_param->hip_stream, device_param->kernel_params_utf8toutf16le, NULL) == -1) return -1; - if (hc_hipStreamSynchronize (hashcat_ctx, device_param->hip_stream) == -1) return -1; - return 0; } @@ -5342,8 +5340,6 @@ int run_hip_kernel_bzero (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_ if (hc_hipMemcpyHtoDAsync (hashcat_ctx, buf + (num16d * 16), bzeros, num16m, device_param->hip_stream) == -1) return -1; } - /*if (hc_hipStreamSynchronize (hashcat_ctx, device_param->hip_stream) == -1) return -1;*/ - return 0; } @@ -5370,8 +5366,6 @@ int run_opencl_kernel_atinit (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *dev /*if (hc_clFlush (hashcat_ctx, device_param->opencl_command_queue) == -1) return -1;*/ - if (hc_clFinish (hashcat_ctx, device_param->opencl_command_queue) == -1) return -1; - return 0; } @@ -5398,8 +5392,6 @@ int run_opencl_kernel_utf8toutf16le (hashcat_ctx_t *hashcat_ctx, hc_device_param /*if (hc_clFlush (hashcat_ctx, device_param->opencl_command_queue) == -1) return -1;*/ - if (hc_clFinish (hashcat_ctx, device_param->opencl_command_queue) == -1) return -1; - return 0; } @@ -5473,8 +5465,6 @@ int run_opencl_kernel_bzero (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *devi /*if (hc_clFlush (hashcat_ctx, device_param->opencl_command_queue) == -1) return -1;*/ - /*if (hc_clFinish (hashcat_ctx, device_param->opencl_command_queue) == -1) return -1;*/ - return 0; } @@ -5648,8 +5638,6 @@ int run_kernel (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param, con if (hc_cuEventRecord (hashcat_ctx, device_param->cuda_event2, device_param->cuda_stream) == -1) return -1; - if (hc_cuStreamSynchronize (hashcat_ctx, device_param->cuda_stream) == -1) return -1; - if (hc_cuEventSynchronize (hashcat_ctx, device_param->cuda_event2) == -1) return -1; float exec_ms; @@ -5747,8 +5735,6 @@ int run_kernel (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param, con if (hc_hipEventRecord (hashcat_ctx, device_param->hip_event2, device_param->hip_stream) == -1) return -1; - if (hc_hipStreamSynchronize (hashcat_ctx, device_param->hip_stream) == -1) return -1; - if (hc_hipEventSynchronize (hashcat_ctx, device_param->hip_event2) == -1) return -1; float exec_ms; @@ -5948,8 +5934,6 @@ int run_kernel (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param, con } if (hc_clReleaseEvent (hashcat_ctx, opencl_event) == -1) return -1; - - if (hc_clFinish (hashcat_ctx, device_param->opencl_command_queue) == -1) return -1; } return 0; @@ -5997,8 +5981,6 @@ int run_kernel_mp (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param, num_elements = CEILDIV (num_elements, kernel_threads); if (hc_cuLaunchKernel (hashcat_ctx, cuda_function, num_elements, 1, 1, kernel_threads, 1, 1, 0, device_param->cuda_stream, cuda_args, NULL) == -1) return -1; - - if (hc_cuStreamSynchronize (hashcat_ctx, device_param->cuda_stream) == -1) return -1; } if (device_param->is_hip == true) @@ -6023,8 +6005,6 @@ int run_kernel_mp (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param, num_elements = CEILDIV (num_elements, kernel_threads); if (hc_hipLaunchKernel (hashcat_ctx, hip_function, num_elements, 1, 1, kernel_threads, 1, 1, 0, device_param->hip_stream, hip_args, NULL) == -1) return -1; - - if (hc_hipStreamSynchronize (hashcat_ctx, device_param->hip_stream) == -1) return -1; } if (device_param->is_opencl == true) @@ -6072,8 +6052,6 @@ int run_kernel_mp (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param, if (hc_clEnqueueNDRangeKernel (hashcat_ctx, device_param->opencl_command_queue, opencl_kernel, 1, NULL, global_work_size, local_work_size, 0, NULL, NULL) == -1) return -1; /*if (hc_clFlush (hashcat_ctx, device_param->opencl_command_queue) == -1) return -1;*/ - - if (hc_clFinish (hashcat_ctx, device_param->opencl_command_queue) == -1) return -1; } return 0; @@ -6090,8 +6068,6 @@ int run_kernel_tm (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param) CUfunction cuda_function = device_param->cuda_function_tm; if (hc_cuLaunchKernel (hashcat_ctx, cuda_function, num_elements / kernel_threads, 1, 1, kernel_threads, 1, 1, 0, device_param->cuda_stream, device_param->kernel_params_tm, NULL) == -1) return -1; - - if (hc_cuStreamSynchronize (hashcat_ctx, device_param->cuda_stream) == -1) return -1; } if (device_param->is_hip == true) @@ -6099,8 +6075,6 @@ int run_kernel_tm (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param) HIPfunction hip_function = device_param->hip_function_tm; if (hc_hipLaunchKernel (hashcat_ctx, hip_function, num_elements / kernel_threads, 1, 1, kernel_threads, 1, 1, 0, device_param->hip_stream, device_param->kernel_params_tm, NULL) == -1) return -1; - - if (hc_hipStreamSynchronize (hashcat_ctx, device_param->hip_stream) == -1) return -1; } if (device_param->is_opencl == true) @@ -6113,8 +6087,6 @@ int run_kernel_tm (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param) if (hc_clEnqueueNDRangeKernel (hashcat_ctx, device_param->opencl_command_queue, cuda_kernel, 1, NULL, global_work_size, local_work_size, 0, NULL, NULL) == -1) return -1; /*if (hc_clFlush (hashcat_ctx, device_param->opencl_command_queue) == -1) return -1;*/ - - if (hc_clFinish (hashcat_ctx, device_param->opencl_command_queue) == -1) return -1; } return 0; @@ -6135,8 +6107,6 @@ int run_kernel_amp (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param, CUfunction cuda_function = device_param->cuda_function_amp; if (hc_cuLaunchKernel (hashcat_ctx, cuda_function, num_elements, 1, 1, kernel_threads, 1, 1, 0, device_param->cuda_stream, device_param->kernel_params_amp, NULL) == -1) return -1; - - if (hc_cuStreamSynchronize (hashcat_ctx, device_param->cuda_stream) == -1) return -1; } if (device_param->is_hip == true) @@ -6146,8 +6116,6 @@ int run_kernel_amp (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param, HIPfunction hip_function = device_param->hip_function_amp; if (hc_hipLaunchKernel (hashcat_ctx, hip_function, num_elements, 1, 1, kernel_threads, 1, 1, 0, device_param->hip_stream, device_param->kernel_params_amp, NULL) == -1) return -1; - - if (hc_hipStreamSynchronize (hashcat_ctx, device_param->hip_stream) == -1) return -1; } if (device_param->is_opencl == true) @@ -6164,8 +6132,6 @@ int run_kernel_amp (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param, if (hc_clEnqueueNDRangeKernel (hashcat_ctx, device_param->opencl_command_queue, opencl_kernel, 1, NULL, global_work_size, local_work_size, 0, NULL, NULL) == -1) return -1; /*if (hc_clFlush (hashcat_ctx, device_param->opencl_command_queue) == -1) return -1;*/ - - if (hc_clFinish (hashcat_ctx, device_param->opencl_command_queue) == -1) return -1; } return 0; @@ -6186,8 +6152,6 @@ int run_kernel_decompress (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device CUfunction cuda_function = device_param->cuda_function_decompress; if (hc_cuLaunchKernel (hashcat_ctx, cuda_function, num_elements, 1, 1, kernel_threads, 1, 1, 0, device_param->cuda_stream, device_param->kernel_params_decompress, NULL) == -1) return -1; - - if (hc_cuStreamSynchronize (hashcat_ctx, device_param->cuda_stream) == -1) return -1; } if (device_param->is_hip == true) @@ -6197,8 +6161,6 @@ int run_kernel_decompress (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device HIPfunction hip_function = device_param->hip_function_decompress; if (hc_hipLaunchKernel (hashcat_ctx, hip_function, num_elements, 1, 1, kernel_threads, 1, 1, 0, device_param->hip_stream, device_param->kernel_params_decompress, NULL) == -1) return -1; - - if (hc_hipStreamSynchronize (hashcat_ctx, device_param->hip_stream) == -1) return -1; } if (device_param->is_opencl == true) @@ -6215,8 +6177,6 @@ int run_kernel_decompress (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device if (hc_clEnqueueNDRangeKernel (hashcat_ctx, device_param->opencl_command_queue, opencl_kernel, 1, NULL, global_work_size, local_work_size, 0, NULL, NULL) == -1) return -1; /*if (hc_clFlush (hashcat_ctx, device_param->opencl_command_queue) == -1) return -1;*/ - - if (hc_clFinish (hashcat_ctx, device_param->opencl_command_queue) == -1) return -1; } return 0; @@ -6247,7 +6207,7 @@ int run_copy (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param, const { if (device_param->is_cuda == true) { - if (hc_cuMemcpyHtoD (hashcat_ctx, device_param->cuda_d_pws_idx, device_param->pws_idx, pws_cnt * sizeof (pw_idx_t)) == -1) return -1; + if (hc_cuMemcpyHtoDAsync (hashcat_ctx, device_param->cuda_d_pws_idx, device_param->pws_idx, pws_cnt * sizeof (pw_idx_t), device_param->cuda_stream) == -1) return -1; const pw_idx_t *pw_idx = device_param->pws_idx + pws_cnt; @@ -6255,13 +6215,13 @@ int run_copy (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param, const if (off) { - if (hc_cuMemcpyHtoD (hashcat_ctx, device_param->cuda_d_pws_comp_buf, device_param->pws_comp, off * sizeof (u32)) == -1) return -1; + if (hc_cuMemcpyHtoDAsync (hashcat_ctx, device_param->cuda_d_pws_comp_buf, device_param->pws_comp, off * sizeof (u32), device_param->cuda_stream) == -1) return -1; } } if (device_param->is_hip == true) { - if (hc_hipMemcpyHtoD (hashcat_ctx, device_param->hip_d_pws_idx, device_param->pws_idx, pws_cnt * sizeof (pw_idx_t)) == -1) return -1; + if (hc_hipMemcpyHtoDAsync (hashcat_ctx, device_param->hip_d_pws_idx, device_param->pws_idx, pws_cnt * sizeof (pw_idx_t), device_param->hip_stream) == -1) return -1; const pw_idx_t *pw_idx = device_param->pws_idx + pws_cnt; @@ -6269,13 +6229,13 @@ int run_copy (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param, const if (off) { - if (hc_hipMemcpyHtoD (hashcat_ctx, device_param->hip_d_pws_comp_buf, device_param->pws_comp, off * sizeof (u32)) == -1) return -1; + if (hc_hipMemcpyHtoDAsync (hashcat_ctx, device_param->hip_d_pws_comp_buf, device_param->pws_comp, off * sizeof (u32), device_param->hip_stream) == -1) return -1; } } if (device_param->is_opencl == true) { - if (hc_clEnqueueWriteBuffer (hashcat_ctx, device_param->opencl_command_queue, device_param->opencl_d_pws_idx, CL_TRUE, 0, pws_cnt * sizeof (pw_idx_t), device_param->pws_idx, 0, NULL, NULL) == -1) return -1; + if (hc_clEnqueueWriteBuffer (hashcat_ctx, device_param->opencl_command_queue, device_param->opencl_d_pws_idx, CL_FALSE, 0, pws_cnt * sizeof (pw_idx_t), device_param->pws_idx, 0, NULL, NULL) == -1) return -1; const pw_idx_t *pw_idx = device_param->pws_idx + pws_cnt; @@ -6283,7 +6243,7 @@ int run_copy (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param, const if (off) { - if (hc_clEnqueueWriteBuffer (hashcat_ctx, device_param->opencl_command_queue, device_param->opencl_d_pws_comp_buf, CL_TRUE, 0, off * sizeof (u32), device_param->pws_comp, 0, NULL, NULL) == -1) return -1; + if (hc_clEnqueueWriteBuffer (hashcat_ctx, device_param->opencl_command_queue, device_param->opencl_d_pws_comp_buf, CL_FALSE, 0, off * sizeof (u32), device_param->pws_comp, 0, NULL, NULL) == -1) return -1; } } @@ -6295,7 +6255,7 @@ int run_copy (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param, const { if (device_param->is_cuda == true) { - if (hc_cuMemcpyHtoD (hashcat_ctx, device_param->cuda_d_pws_idx, device_param->pws_idx, pws_cnt * sizeof (pw_idx_t)) == -1) return -1; + if (hc_cuMemcpyHtoDAsync (hashcat_ctx, device_param->cuda_d_pws_idx, device_param->pws_idx, pws_cnt * sizeof (pw_idx_t), device_param->cuda_stream) == -1) return -1; const pw_idx_t *pw_idx = device_param->pws_idx + pws_cnt; @@ -6303,13 +6263,13 @@ int run_copy (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param, const if (off) { - if (hc_cuMemcpyHtoD (hashcat_ctx, device_param->cuda_d_pws_comp_buf, device_param->pws_comp, off * sizeof (u32)) == -1) return -1; + if (hc_cuMemcpyHtoDAsync (hashcat_ctx, device_param->cuda_d_pws_comp_buf, device_param->pws_comp, off * sizeof (u32), device_param->cuda_stream) == -1) return -1; } } if (device_param->is_hip == true) { - if (hc_hipMemcpyHtoD (hashcat_ctx, device_param->hip_d_pws_idx, device_param->pws_idx, pws_cnt * sizeof (pw_idx_t)) == -1) return -1; + if (hc_hipMemcpyHtoDAsync (hashcat_ctx, device_param->hip_d_pws_idx, device_param->pws_idx, pws_cnt * sizeof (pw_idx_t), device_param->hip_stream) == -1) return -1; const pw_idx_t *pw_idx = device_param->pws_idx + pws_cnt; @@ -6317,13 +6277,13 @@ int run_copy (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param, const if (off) { - if (hc_hipMemcpyHtoD (hashcat_ctx, device_param->hip_d_pws_comp_buf, device_param->pws_comp, off * sizeof (u32)) == -1) return -1; + if (hc_hipMemcpyHtoDAsync (hashcat_ctx, device_param->hip_d_pws_comp_buf, device_param->pws_comp, off * sizeof (u32), device_param->hip_stream) == -1) return -1; } } if (device_param->is_opencl == true) { - if (hc_clEnqueueWriteBuffer (hashcat_ctx, device_param->opencl_command_queue, device_param->opencl_d_pws_idx, CL_TRUE, 0, pws_cnt * sizeof (pw_idx_t), device_param->pws_idx, 0, NULL, NULL) == -1) return -1; + if (hc_clEnqueueWriteBuffer (hashcat_ctx, device_param->opencl_command_queue, device_param->opencl_d_pws_idx, CL_FALSE, 0, pws_cnt * sizeof (pw_idx_t), device_param->pws_idx, 0, NULL, NULL) == -1) return -1; const pw_idx_t *pw_idx = device_param->pws_idx + pws_cnt; @@ -6331,7 +6291,7 @@ int run_copy (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param, const if (off) { - if (hc_clEnqueueWriteBuffer (hashcat_ctx, device_param->opencl_command_queue, device_param->opencl_d_pws_comp_buf, CL_TRUE, 0, off * sizeof (u32), device_param->pws_comp, 0, NULL, NULL) == -1) return -1; + if (hc_clEnqueueWriteBuffer (hashcat_ctx, device_param->opencl_command_queue, device_param->opencl_d_pws_comp_buf, CL_FALSE, 0, off * sizeof (u32), device_param->pws_comp, 0, NULL, NULL) == -1) return -1; } } @@ -6377,7 +6337,7 @@ int run_copy (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param, const if (device_param->is_cuda == true) { - if (hc_cuMemcpyHtoD (hashcat_ctx, device_param->cuda_d_pws_idx, device_param->pws_idx, pws_cnt * sizeof (pw_idx_t)) == -1) return -1; + if (hc_cuMemcpyHtoDAsync (hashcat_ctx, device_param->cuda_d_pws_idx, device_param->pws_idx, pws_cnt * sizeof (pw_idx_t), device_param->cuda_stream) == -1) return -1; const pw_idx_t *pw_idx = device_param->pws_idx + pws_cnt; @@ -6385,13 +6345,13 @@ int run_copy (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param, const if (off) { - if (hc_cuMemcpyHtoD (hashcat_ctx, device_param->cuda_d_pws_comp_buf, device_param->pws_comp, off * sizeof (u32)) == -1) return -1; + if (hc_cuMemcpyHtoDAsync (hashcat_ctx, device_param->cuda_d_pws_comp_buf, device_param->pws_comp, off * sizeof (u32), device_param->cuda_stream) == -1) return -1; } } if (device_param->is_hip == true) { - if (hc_hipMemcpyHtoD (hashcat_ctx, device_param->hip_d_pws_idx, device_param->pws_idx, pws_cnt * sizeof (pw_idx_t)) == -1) return -1; + if (hc_hipMemcpyHtoDAsync (hashcat_ctx, device_param->hip_d_pws_idx, device_param->pws_idx, pws_cnt * sizeof (pw_idx_t), device_param->hip_stream) == -1) return -1; const pw_idx_t *pw_idx = device_param->pws_idx + pws_cnt; @@ -6399,13 +6359,13 @@ int run_copy (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param, const if (off) { - if (hc_hipMemcpyHtoD (hashcat_ctx, device_param->hip_d_pws_comp_buf, device_param->pws_comp, off * sizeof (u32)) == -1) return -1; + if (hc_hipMemcpyHtoDAsync (hashcat_ctx, device_param->hip_d_pws_comp_buf, device_param->pws_comp, off * sizeof (u32), device_param->hip_stream) == -1) return -1; } } if (device_param->is_opencl == true) { - if (hc_clEnqueueWriteBuffer (hashcat_ctx, device_param->opencl_command_queue, device_param->opencl_d_pws_idx, CL_TRUE, 0, pws_cnt * sizeof (pw_idx_t), device_param->pws_idx, 0, NULL, NULL) == -1) return -1; + if (hc_clEnqueueWriteBuffer (hashcat_ctx, device_param->opencl_command_queue, device_param->opencl_d_pws_idx, CL_FALSE, 0, pws_cnt * sizeof (pw_idx_t), device_param->pws_idx, 0, NULL, NULL) == -1) return -1; const pw_idx_t *pw_idx = device_param->pws_idx + pws_cnt; @@ -6413,7 +6373,7 @@ int run_copy (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param, const if (off) { - if (hc_clEnqueueWriteBuffer (hashcat_ctx, device_param->opencl_command_queue, device_param->opencl_d_pws_comp_buf, CL_TRUE, 0, off * sizeof (u32), device_param->pws_comp, 0, NULL, NULL) == -1) return -1; + if (hc_clEnqueueWriteBuffer (hashcat_ctx, device_param->opencl_command_queue, device_param->opencl_d_pws_comp_buf, CL_FALSE, 0, off * sizeof (u32), device_param->pws_comp, 0, NULL, NULL) == -1) return -1; } } @@ -6425,7 +6385,7 @@ int run_copy (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param, const { if (device_param->is_cuda == true) { - if (hc_cuMemcpyHtoD (hashcat_ctx, device_param->cuda_d_pws_idx, device_param->pws_idx, pws_cnt * sizeof (pw_idx_t)) == -1) return -1; + if (hc_cuMemcpyHtoDAsync (hashcat_ctx, device_param->cuda_d_pws_idx, device_param->pws_idx, pws_cnt * sizeof (pw_idx_t), device_param->cuda_stream) == -1) return -1; const pw_idx_t *pw_idx = device_param->pws_idx + pws_cnt; @@ -6433,13 +6393,13 @@ int run_copy (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param, const if (off) { - if (hc_cuMemcpyHtoD (hashcat_ctx, device_param->cuda_d_pws_comp_buf, device_param->pws_comp, off * sizeof (u32)) == -1) return -1; + if (hc_cuMemcpyHtoDAsync (hashcat_ctx, device_param->cuda_d_pws_comp_buf, device_param->pws_comp, off * sizeof (u32), device_param->cuda_stream) == -1) return -1; } } if (device_param->is_hip == true) { - if (hc_hipMemcpyHtoD (hashcat_ctx, device_param->hip_d_pws_idx, device_param->pws_idx, pws_cnt * sizeof (pw_idx_t)) == -1) return -1; + if (hc_hipMemcpyHtoDAsync (hashcat_ctx, device_param->hip_d_pws_idx, device_param->pws_idx, pws_cnt * sizeof (pw_idx_t), device_param->hip_stream) == -1) return -1; const pw_idx_t *pw_idx = device_param->pws_idx + pws_cnt; @@ -6447,13 +6407,13 @@ int run_copy (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param, const if (off) { - if (hc_hipMemcpyHtoD (hashcat_ctx, device_param->hip_d_pws_comp_buf, device_param->pws_comp, off * sizeof (u32)) == -1) return -1; + if (hc_hipMemcpyHtoDAsync (hashcat_ctx, device_param->hip_d_pws_comp_buf, device_param->pws_comp, off * sizeof (u32), device_param->hip_stream) == -1) return -1; } } if (device_param->is_opencl == true) { - if (hc_clEnqueueWriteBuffer (hashcat_ctx, device_param->opencl_command_queue, device_param->opencl_d_pws_idx, CL_TRUE, 0, pws_cnt * sizeof (pw_idx_t), device_param->pws_idx, 0, NULL, NULL) == -1) return -1; + if (hc_clEnqueueWriteBuffer (hashcat_ctx, device_param->opencl_command_queue, device_param->opencl_d_pws_idx, CL_FALSE, 0, pws_cnt * sizeof (pw_idx_t), device_param->pws_idx, 0, NULL, NULL) == -1) return -1; const pw_idx_t *pw_idx = device_param->pws_idx + pws_cnt; @@ -6461,7 +6421,7 @@ int run_copy (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param, const if (off) { - if (hc_clEnqueueWriteBuffer (hashcat_ctx, device_param->opencl_command_queue, device_param->opencl_d_pws_comp_buf, CL_TRUE, 0, off * sizeof (u32), device_param->pws_comp, 0, NULL, NULL) == -1) return -1; + if (hc_clEnqueueWriteBuffer (hashcat_ctx, device_param->opencl_command_queue, device_param->opencl_d_pws_comp_buf, CL_FALSE, 0, off * sizeof (u32), device_param->pws_comp, 0, NULL, NULL) == -1) return -1; } } @@ -6471,7 +6431,7 @@ int run_copy (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param, const { if (device_param->is_cuda == true) { - if (hc_cuMemcpyHtoD (hashcat_ctx, device_param->cuda_d_pws_idx, device_param->pws_idx, pws_cnt * sizeof (pw_idx_t)) == -1) return -1; + if (hc_cuMemcpyHtoDAsync (hashcat_ctx, device_param->cuda_d_pws_idx, device_param->pws_idx, pws_cnt * sizeof (pw_idx_t), device_param->cuda_stream) == -1) return -1; const pw_idx_t *pw_idx = device_param->pws_idx + pws_cnt; @@ -6479,13 +6439,13 @@ int run_copy (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param, const if (off) { - if (hc_cuMemcpyHtoD (hashcat_ctx, device_param->cuda_d_pws_comp_buf, device_param->pws_comp, off * sizeof (u32)) == -1) return -1; + if (hc_cuMemcpyHtoDAsync (hashcat_ctx, device_param->cuda_d_pws_comp_buf, device_param->pws_comp, off * sizeof (u32), device_param->cuda_stream) == -1) return -1; } } if (device_param->is_hip == true) { - if (hc_hipMemcpyHtoD (hashcat_ctx, device_param->hip_d_pws_idx, device_param->pws_idx, pws_cnt * sizeof (pw_idx_t)) == -1) return -1; + if (hc_hipMemcpyHtoDAsync (hashcat_ctx, device_param->hip_d_pws_idx, device_param->pws_idx, pws_cnt * sizeof (pw_idx_t), device_param->hip_stream) == -1) return -1; const pw_idx_t *pw_idx = device_param->pws_idx + pws_cnt; @@ -6493,13 +6453,13 @@ int run_copy (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param, const if (off) { - if (hc_hipMemcpyHtoD (hashcat_ctx, device_param->hip_d_pws_comp_buf, device_param->pws_comp, off * sizeof (u32)) == -1) return -1; + if (hc_hipMemcpyHtoDAsync (hashcat_ctx, device_param->hip_d_pws_comp_buf, device_param->pws_comp, off * sizeof (u32), device_param->hip_stream) == -1) return -1; } } if (device_param->is_opencl == true) { - if (hc_clEnqueueWriteBuffer (hashcat_ctx, device_param->opencl_command_queue, device_param->opencl_d_pws_idx, CL_TRUE, 0, pws_cnt * sizeof (pw_idx_t), device_param->pws_idx, 0, NULL, NULL) == -1) return -1; + if (hc_clEnqueueWriteBuffer (hashcat_ctx, device_param->opencl_command_queue, device_param->opencl_d_pws_idx, CL_FALSE, 0, pws_cnt * sizeof (pw_idx_t), device_param->pws_idx, 0, NULL, NULL) == -1) return -1; const pw_idx_t *pw_idx = device_param->pws_idx + pws_cnt; @@ -6507,7 +6467,7 @@ int run_copy (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param, const if (off) { - if (hc_clEnqueueWriteBuffer (hashcat_ctx, device_param->opencl_command_queue, device_param->opencl_d_pws_comp_buf, CL_TRUE, 0, off * sizeof (u32), device_param->pws_comp, 0, NULL, NULL) == -1) return -1; + if (hc_clEnqueueWriteBuffer (hashcat_ctx, device_param->opencl_command_queue, device_param->opencl_d_pws_comp_buf, CL_FALSE, 0, off * sizeof (u32), device_param->pws_comp, 0, NULL, NULL) == -1) return -1; } } @@ -6715,12 +6675,12 @@ int run_cracker (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param, co { if (device_param->is_cuda == true) { - if (hc_cuMemcpyDtoD (hashcat_ctx, device_param->cuda_d_rules_c, device_param->cuda_d_rules + (innerloop_pos * sizeof (kernel_rule_t)), innerloop_left * sizeof (kernel_rule_t)) == -1) return -1; + if (hc_cuMemcpyDtoDAsync (hashcat_ctx, device_param->cuda_d_rules_c, device_param->cuda_d_rules + (innerloop_pos * sizeof (kernel_rule_t)), innerloop_left * sizeof (kernel_rule_t), device_param->cuda_stream) == -1) return -1; } if (device_param->is_hip == true) { - if (hc_hipMemcpyDtoD (hashcat_ctx, device_param->hip_d_rules_c, device_param->hip_d_rules + (innerloop_pos * sizeof (kernel_rule_t)), innerloop_left * sizeof (kernel_rule_t)) == -1) return -1; + if (hc_hipMemcpyDtoDAsync (hashcat_ctx, device_param->hip_d_rules_c, device_param->hip_d_rules + (innerloop_pos * sizeof (kernel_rule_t)), innerloop_left * sizeof (kernel_rule_t), device_param->hip_stream) == -1) return -1; } if (device_param->is_opencl == true) @@ -6840,12 +6800,12 @@ int run_cracker (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param, co if (device_param->is_cuda == true) { - if (hc_cuMemcpyHtoD (hashcat_ctx, device_param->cuda_d_combs_c, device_param->combs_buf, innerloop_left * sizeof (pw_t)) == -1) return -1; + if (hc_cuMemcpyHtoDAsync (hashcat_ctx, device_param->cuda_d_combs_c, device_param->combs_buf, innerloop_left * sizeof (pw_t), device_param->cuda_stream) == -1) return -1; } if (device_param->is_hip == true) { - if (hc_hipMemcpyHtoD (hashcat_ctx, device_param->hip_d_combs_c, device_param->combs_buf, innerloop_left * sizeof (pw_t)) == -1) return -1; + if (hc_hipMemcpyHtoDAsync (hashcat_ctx, device_param->hip_d_combs_c, device_param->combs_buf, innerloop_left * sizeof (pw_t), device_param->hip_stream) == -1) return -1; } if (device_param->is_opencl == true) @@ -6863,12 +6823,12 @@ int run_cracker (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param, co if (device_param->is_cuda == true) { - if (hc_cuMemcpyDtoD (hashcat_ctx, device_param->cuda_d_combs_c, device_param->cuda_d_combs, innerloop_left * sizeof (pw_t)) == -1) return -1; + if (hc_cuMemcpyDtoDAsync (hashcat_ctx, device_param->cuda_d_combs_c, device_param->cuda_d_combs, innerloop_left * sizeof (pw_t), device_param->cuda_stream) == -1) return -1; } if (device_param->is_hip == true) { - if (hc_hipMemcpyDtoD (hashcat_ctx, device_param->hip_d_combs_c, device_param->hip_d_combs, innerloop_left * sizeof (pw_t)) == -1) return -1; + if (hc_hipMemcpyDtoDAsync (hashcat_ctx, device_param->hip_d_combs_c, device_param->hip_d_combs, innerloop_left * sizeof (pw_t), device_param->hip_stream) == -1) return -1; } if (device_param->is_opencl == true) @@ -6886,12 +6846,12 @@ int run_cracker (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param, co if (device_param->is_cuda == true) { - if (hc_cuMemcpyDtoD (hashcat_ctx, device_param->cuda_d_combs_c, device_param->cuda_d_combs, innerloop_left * sizeof (pw_t)) == -1) return -1; + if (hc_cuMemcpyDtoDAsync (hashcat_ctx, device_param->cuda_d_combs_c, device_param->cuda_d_combs, innerloop_left * sizeof (pw_t), device_param->cuda_stream) == -1) return -1; } if (device_param->is_hip == true) { - if (hc_hipMemcpyDtoD (hashcat_ctx, device_param->hip_d_combs_c, device_param->hip_d_combs, innerloop_left * sizeof (pw_t)) == -1) return -1; + if (hc_hipMemcpyDtoDAsync (hashcat_ctx, device_param->hip_d_combs_c, device_param->hip_d_combs, innerloop_left * sizeof (pw_t), device_param->hip_stream) == -1) return -1; } if (device_param->is_opencl == true) @@ -7012,17 +6972,17 @@ int run_cracker (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param, co if (device_param->is_cuda == true) { - if (hc_cuMemcpyHtoD (hashcat_ctx, device_param->cuda_d_combs_c, device_param->combs_buf, innerloop_left * sizeof (pw_t)) == -1) return -1; + if (hc_cuMemcpyHtoDAsync (hashcat_ctx, device_param->cuda_d_combs_c, device_param->combs_buf, innerloop_left * sizeof (pw_t), device_param->cuda_stream) == -1) return -1; } if (device_param->is_hip == true) { - if (hc_hipMemcpyHtoD (hashcat_ctx, device_param->hip_d_combs_c, device_param->combs_buf, innerloop_left * sizeof (pw_t)) == -1) return -1; + if (hc_hipMemcpyHtoDAsync (hashcat_ctx, device_param->hip_d_combs_c, device_param->combs_buf, innerloop_left * sizeof (pw_t), device_param->hip_stream) == -1) return -1; } if (device_param->is_opencl == true) { - if (hc_clEnqueueWriteBuffer (hashcat_ctx, device_param->opencl_command_queue, device_param->opencl_d_combs_c, CL_TRUE, 0, innerloop_left * sizeof (pw_t), device_param->combs_buf, 0, NULL, NULL) == -1) return -1; + if (hc_clEnqueueWriteBuffer (hashcat_ctx, device_param->opencl_command_queue, device_param->opencl_d_combs_c, CL_FALSE, 0, innerloop_left * sizeof (pw_t), device_param->combs_buf, 0, NULL, NULL) == -1) return -1; } } else if (user_options->attack_mode == ATTACK_MODE_HYBRID1) @@ -7035,12 +6995,12 @@ int run_cracker (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param, co if (device_param->is_cuda == true) { - if (hc_cuMemcpyDtoD (hashcat_ctx, device_param->cuda_d_combs_c, device_param->cuda_d_combs, innerloop_left * sizeof (pw_t)) == -1) return -1; + if (hc_cuMemcpyDtoDAsync (hashcat_ctx, device_param->cuda_d_combs_c, device_param->cuda_d_combs, innerloop_left * sizeof (pw_t), device_param->cuda_stream) == -1) return -1; } if (device_param->is_hip == true) { - if (hc_hipMemcpyDtoD (hashcat_ctx, device_param->hip_d_combs_c, device_param->hip_d_combs, innerloop_left * sizeof (pw_t)) == -1) return -1; + if (hc_hipMemcpyDtoDAsync (hashcat_ctx, device_param->hip_d_combs_c, device_param->hip_d_combs, innerloop_left * sizeof (pw_t), device_param->hip_stream) == -1) return -1; } if (device_param->is_opencl == true) @@ -7060,12 +7020,12 @@ int run_cracker (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param, co if (device_param->is_cuda == true) { - if (hc_cuMemcpyDtoD (hashcat_ctx, device_param->cuda_d_bfs_c, device_param->cuda_d_bfs, innerloop_left * sizeof (bf_t)) == -1) return -1; + if (hc_cuMemcpyDtoDAsync (hashcat_ctx, device_param->cuda_d_bfs_c, device_param->cuda_d_bfs, innerloop_left * sizeof (bf_t), device_param->cuda_stream) == -1) return -1; } if (device_param->is_hip == true) { - if (hc_hipMemcpyDtoD (hashcat_ctx, device_param->hip_d_bfs_c, device_param->hip_d_bfs, innerloop_left * sizeof (bf_t)) == -1) return -1; + if (hc_hipMemcpyDtoDAsync (hashcat_ctx, device_param->hip_d_bfs_c, device_param->hip_d_bfs, innerloop_left * sizeof (bf_t), device_param->hip_stream) == -1) return -1; } if (device_param->is_opencl == true) @@ -12334,16 +12294,16 @@ int backend_session_begin (hashcat_ctx_t *hashcat_ctx) if (hc_cuMemAlloc (hashcat_ctx, &device_param->cuda_d_st_digests_buf, size_st_digests) == -1) return -1; if (hc_cuMemAlloc (hashcat_ctx, &device_param->cuda_d_st_salts_buf, size_st_salts) == -1) return -1; - if (hc_cuMemcpyHtoD (hashcat_ctx, device_param->cuda_d_bitmap_s1_a, bitmap_ctx->bitmap_s1_a, bitmap_ctx->bitmap_size) == -1) return -1; - if (hc_cuMemcpyHtoD (hashcat_ctx, device_param->cuda_d_bitmap_s1_b, bitmap_ctx->bitmap_s1_b, bitmap_ctx->bitmap_size) == -1) return -1; - if (hc_cuMemcpyHtoD (hashcat_ctx, device_param->cuda_d_bitmap_s1_c, bitmap_ctx->bitmap_s1_c, bitmap_ctx->bitmap_size) == -1) return -1; - if (hc_cuMemcpyHtoD (hashcat_ctx, device_param->cuda_d_bitmap_s1_d, bitmap_ctx->bitmap_s1_d, bitmap_ctx->bitmap_size) == -1) return -1; - if (hc_cuMemcpyHtoD (hashcat_ctx, device_param->cuda_d_bitmap_s2_a, bitmap_ctx->bitmap_s2_a, bitmap_ctx->bitmap_size) == -1) return -1; - if (hc_cuMemcpyHtoD (hashcat_ctx, device_param->cuda_d_bitmap_s2_b, bitmap_ctx->bitmap_s2_b, bitmap_ctx->bitmap_size) == -1) return -1; - if (hc_cuMemcpyHtoD (hashcat_ctx, device_param->cuda_d_bitmap_s2_c, bitmap_ctx->bitmap_s2_c, bitmap_ctx->bitmap_size) == -1) return -1; - if (hc_cuMemcpyHtoD (hashcat_ctx, device_param->cuda_d_bitmap_s2_d, bitmap_ctx->bitmap_s2_d, bitmap_ctx->bitmap_size) == -1) return -1; - if (hc_cuMemcpyHtoD (hashcat_ctx, device_param->cuda_d_digests_buf, hashes->digests_buf, size_digests) == -1) return -1; - if (hc_cuMemcpyHtoD (hashcat_ctx, device_param->cuda_d_salt_bufs, hashes->salts_buf, size_salts) == -1) return -1; + if (hc_cuMemcpyHtoDAsync (hashcat_ctx, device_param->cuda_d_bitmap_s1_a, bitmap_ctx->bitmap_s1_a, bitmap_ctx->bitmap_size, device_param->cuda_stream) == -1) return -1; + if (hc_cuMemcpyHtoDAsync (hashcat_ctx, device_param->cuda_d_bitmap_s1_b, bitmap_ctx->bitmap_s1_b, bitmap_ctx->bitmap_size, device_param->cuda_stream) == -1) return -1; + if (hc_cuMemcpyHtoDAsync (hashcat_ctx, device_param->cuda_d_bitmap_s1_c, bitmap_ctx->bitmap_s1_c, bitmap_ctx->bitmap_size, device_param->cuda_stream) == -1) return -1; + if (hc_cuMemcpyHtoDAsync (hashcat_ctx, device_param->cuda_d_bitmap_s1_d, bitmap_ctx->bitmap_s1_d, bitmap_ctx->bitmap_size, device_param->cuda_stream) == -1) return -1; + if (hc_cuMemcpyHtoDAsync (hashcat_ctx, device_param->cuda_d_bitmap_s2_a, bitmap_ctx->bitmap_s2_a, bitmap_ctx->bitmap_size, device_param->cuda_stream) == -1) return -1; + if (hc_cuMemcpyHtoDAsync (hashcat_ctx, device_param->cuda_d_bitmap_s2_b, bitmap_ctx->bitmap_s2_b, bitmap_ctx->bitmap_size, device_param->cuda_stream) == -1) return -1; + if (hc_cuMemcpyHtoDAsync (hashcat_ctx, device_param->cuda_d_bitmap_s2_c, bitmap_ctx->bitmap_s2_c, bitmap_ctx->bitmap_size, device_param->cuda_stream) == -1) return -1; + if (hc_cuMemcpyHtoDAsync (hashcat_ctx, device_param->cuda_d_bitmap_s2_d, bitmap_ctx->bitmap_s2_d, bitmap_ctx->bitmap_size, device_param->cuda_stream) == -1) return -1; + if (hc_cuMemcpyHtoDAsync (hashcat_ctx, device_param->cuda_d_digests_buf, hashes->digests_buf, size_digests, device_param->cuda_stream) == -1) return -1; + if (hc_cuMemcpyHtoDAsync (hashcat_ctx, device_param->cuda_d_salt_bufs, hashes->salts_buf, size_salts, device_param->cuda_stream) == -1) return -1; /** * special buffers @@ -12370,7 +12330,7 @@ int backend_session_begin (hashcat_ctx_t *hashcat_ctx) if (hc_cuMemAlloc (hashcat_ctx, &device_param->cuda_d_rules_c, size_rules_c) == -1) return -1; } - if (hc_cuMemcpyHtoD (hashcat_ctx, device_param->cuda_d_rules, straight_ctx->kernel_rules_buf, size_rules) == -1) return -1; + if (hc_cuMemcpyHtoDAsync (hashcat_ctx, device_param->cuda_d_rules, straight_ctx->kernel_rules_buf, size_rules, device_param->cuda_stream) == -1) return -1; } else if (user_options_extra->attack_kern == ATTACK_KERN_COMBI) { @@ -12405,19 +12365,19 @@ int backend_session_begin (hashcat_ctx_t *hashcat_ctx) { if (hc_cuMemAlloc (hashcat_ctx, &device_param->cuda_d_esalt_bufs, size_esalts) == -1) return -1; - if (hc_cuMemcpyHtoD (hashcat_ctx, device_param->cuda_d_esalt_bufs, hashes->esalts_buf, size_esalts) == -1) return -1; + if (hc_cuMemcpyHtoDAsync (hashcat_ctx, device_param->cuda_d_esalt_bufs, hashes->esalts_buf, size_esalts, device_param->cuda_stream) == -1) return -1; } if (hashconfig->st_hash != NULL) { - if (hc_cuMemcpyHtoD (hashcat_ctx, device_param->cuda_d_st_digests_buf, hashes->st_digests_buf, size_st_digests) == -1) return -1; - if (hc_cuMemcpyHtoD (hashcat_ctx, device_param->cuda_d_st_salts_buf, hashes->st_salts_buf, size_st_salts) == -1) return -1; + if (hc_cuMemcpyHtoDAsync (hashcat_ctx, device_param->cuda_d_st_digests_buf, hashes->st_digests_buf, size_st_digests, device_param->cuda_stream) == -1) return -1; + if (hc_cuMemcpyHtoDAsync (hashcat_ctx, device_param->cuda_d_st_salts_buf, hashes->st_salts_buf, size_st_salts, device_param->cuda_stream) == -1) return -1; if (size_esalts) { if (hc_cuMemAlloc (hashcat_ctx, &device_param->cuda_d_st_esalts_buf, size_st_esalts) == -1) return -1; - if (hc_cuMemcpyHtoD (hashcat_ctx, device_param->cuda_d_st_esalts_buf, hashes->st_esalts_buf, size_st_esalts) == -1) return -1; + if (hc_cuMemcpyHtoDAsync (hashcat_ctx, device_param->cuda_d_st_esalts_buf, hashes->st_esalts_buf, size_st_esalts, device_param->cuda_stream) == -1) return -1; } } } @@ -12444,16 +12404,16 @@ int backend_session_begin (hashcat_ctx_t *hashcat_ctx) if (hc_hipMemAlloc (hashcat_ctx, &device_param->hip_d_st_digests_buf, size_st_digests) == -1) return -1; if (hc_hipMemAlloc (hashcat_ctx, &device_param->hip_d_st_salts_buf, size_st_salts) == -1) return -1; - if (hc_hipMemcpyHtoD (hashcat_ctx, device_param->hip_d_bitmap_s1_a, bitmap_ctx->bitmap_s1_a, bitmap_ctx->bitmap_size) == -1) return -1; - if (hc_hipMemcpyHtoD (hashcat_ctx, device_param->hip_d_bitmap_s1_b, bitmap_ctx->bitmap_s1_b, bitmap_ctx->bitmap_size) == -1) return -1; - if (hc_hipMemcpyHtoD (hashcat_ctx, device_param->hip_d_bitmap_s1_c, bitmap_ctx->bitmap_s1_c, bitmap_ctx->bitmap_size) == -1) return -1; - if (hc_hipMemcpyHtoD (hashcat_ctx, device_param->hip_d_bitmap_s1_d, bitmap_ctx->bitmap_s1_d, bitmap_ctx->bitmap_size) == -1) return -1; - if (hc_hipMemcpyHtoD (hashcat_ctx, device_param->hip_d_bitmap_s2_a, bitmap_ctx->bitmap_s2_a, bitmap_ctx->bitmap_size) == -1) return -1; - if (hc_hipMemcpyHtoD (hashcat_ctx, device_param->hip_d_bitmap_s2_b, bitmap_ctx->bitmap_s2_b, bitmap_ctx->bitmap_size) == -1) return -1; - if (hc_hipMemcpyHtoD (hashcat_ctx, device_param->hip_d_bitmap_s2_c, bitmap_ctx->bitmap_s2_c, bitmap_ctx->bitmap_size) == -1) return -1; - if (hc_hipMemcpyHtoD (hashcat_ctx, device_param->hip_d_bitmap_s2_d, bitmap_ctx->bitmap_s2_d, bitmap_ctx->bitmap_size) == -1) return -1; - if (hc_hipMemcpyHtoD (hashcat_ctx, device_param->hip_d_digests_buf, hashes->digests_buf, size_digests) == -1) return -1; - if (hc_hipMemcpyHtoD (hashcat_ctx, device_param->hip_d_salt_bufs, hashes->salts_buf, size_salts) == -1) return -1; + if (hc_hipMemcpyHtoDAsync (hashcat_ctx, device_param->hip_d_bitmap_s1_a, bitmap_ctx->bitmap_s1_a, bitmap_ctx->bitmap_size, device_param->hip_stream) == -1) return -1; + if (hc_hipMemcpyHtoDAsync (hashcat_ctx, device_param->hip_d_bitmap_s1_b, bitmap_ctx->bitmap_s1_b, bitmap_ctx->bitmap_size, device_param->hip_stream) == -1) return -1; + if (hc_hipMemcpyHtoDAsync (hashcat_ctx, device_param->hip_d_bitmap_s1_c, bitmap_ctx->bitmap_s1_c, bitmap_ctx->bitmap_size, device_param->hip_stream) == -1) return -1; + if (hc_hipMemcpyHtoDAsync (hashcat_ctx, device_param->hip_d_bitmap_s1_d, bitmap_ctx->bitmap_s1_d, bitmap_ctx->bitmap_size, device_param->hip_stream) == -1) return -1; + if (hc_hipMemcpyHtoDAsync (hashcat_ctx, device_param->hip_d_bitmap_s2_a, bitmap_ctx->bitmap_s2_a, bitmap_ctx->bitmap_size, device_param->hip_stream) == -1) return -1; + if (hc_hipMemcpyHtoDAsync (hashcat_ctx, device_param->hip_d_bitmap_s2_b, bitmap_ctx->bitmap_s2_b, bitmap_ctx->bitmap_size, device_param->hip_stream) == -1) return -1; + if (hc_hipMemcpyHtoDAsync (hashcat_ctx, device_param->hip_d_bitmap_s2_c, bitmap_ctx->bitmap_s2_c, bitmap_ctx->bitmap_size, device_param->hip_stream) == -1) return -1; + if (hc_hipMemcpyHtoDAsync (hashcat_ctx, device_param->hip_d_bitmap_s2_d, bitmap_ctx->bitmap_s2_d, bitmap_ctx->bitmap_size, device_param->hip_stream) == -1) return -1; + if (hc_hipMemcpyHtoDAsync (hashcat_ctx, device_param->hip_d_digests_buf, hashes->digests_buf, size_digests, device_param->hip_stream) == -1) return -1; + if (hc_hipMemcpyHtoDAsync (hashcat_ctx, device_param->hip_d_salt_bufs, hashes->salts_buf, size_salts, device_param->hip_stream) == -1) return -1; /** * special buffers @@ -12480,7 +12440,7 @@ int backend_session_begin (hashcat_ctx_t *hashcat_ctx) if (hc_hipMemAlloc (hashcat_ctx, &device_param->hip_d_rules_c, size_rules_c) == -1) return -1; } - if (hc_hipMemcpyHtoD (hashcat_ctx, device_param->hip_d_rules, straight_ctx->kernel_rules_buf, size_rules) == -1) return -1; + if (hc_hipMemcpyHtoDAsync (hashcat_ctx, device_param->hip_d_rules, straight_ctx->kernel_rules_buf, size_rules, device_param->hip_stream) == -1) return -1; } else if (user_options_extra->attack_kern == ATTACK_KERN_COMBI) { @@ -12515,19 +12475,19 @@ int backend_session_begin (hashcat_ctx_t *hashcat_ctx) { if (hc_hipMemAlloc (hashcat_ctx, &device_param->hip_d_esalt_bufs, size_esalts) == -1) return -1; - if (hc_hipMemcpyHtoD (hashcat_ctx, device_param->hip_d_esalt_bufs, hashes->esalts_buf, size_esalts) == -1) return -1; + if (hc_hipMemcpyHtoDAsync (hashcat_ctx, device_param->hip_d_esalt_bufs, hashes->esalts_buf, size_esalts, device_param->hip_stream) == -1) return -1; } if (hashconfig->st_hash != NULL) { - if (hc_hipMemcpyHtoD (hashcat_ctx, device_param->hip_d_st_digests_buf, hashes->st_digests_buf, size_st_digests) == -1) return -1; - if (hc_hipMemcpyHtoD (hashcat_ctx, device_param->hip_d_st_salts_buf, hashes->st_salts_buf, size_st_salts) == -1) return -1; + if (hc_hipMemcpyHtoDAsync (hashcat_ctx, device_param->hip_d_st_digests_buf, hashes->st_digests_buf, size_st_digests, device_param->hip_stream) == -1) return -1; + if (hc_hipMemcpyHtoDAsync (hashcat_ctx, device_param->hip_d_st_salts_buf, hashes->st_salts_buf, size_st_salts, device_param->hip_stream) == -1) return -1; if (size_esalts) { if (hc_hipMemAlloc (hashcat_ctx, &device_param->hip_d_st_esalts_buf, size_st_esalts) == -1) return -1; - if (hc_hipMemcpyHtoD (hashcat_ctx, device_param->hip_d_st_esalts_buf, hashes->st_esalts_buf, size_st_esalts) == -1) return -1; + if (hc_hipMemcpyHtoDAsync (hashcat_ctx, device_param->hip_d_st_esalts_buf, hashes->st_esalts_buf, size_st_esalts, device_param->hip_stream) == -1) return -1; } } } @@ -12554,16 +12514,16 @@ int backend_session_begin (hashcat_ctx_t *hashcat_ctx) if (hc_clCreateBuffer (hashcat_ctx, device_param->opencl_context, CL_MEM_READ_ONLY, size_st_digests, NULL, &device_param->opencl_d_st_digests_buf) == -1) return -1; if (hc_clCreateBuffer (hashcat_ctx, device_param->opencl_context, CL_MEM_READ_ONLY, size_st_salts, NULL, &device_param->opencl_d_st_salts_buf) == -1) return -1; - if (hc_clEnqueueWriteBuffer (hashcat_ctx, device_param->opencl_command_queue, device_param->opencl_d_bitmap_s1_a, CL_TRUE, 0, bitmap_ctx->bitmap_size, bitmap_ctx->bitmap_s1_a, 0, NULL, NULL) == -1) return -1; - if (hc_clEnqueueWriteBuffer (hashcat_ctx, device_param->opencl_command_queue, device_param->opencl_d_bitmap_s1_b, CL_TRUE, 0, bitmap_ctx->bitmap_size, bitmap_ctx->bitmap_s1_b, 0, NULL, NULL) == -1) return -1; - if (hc_clEnqueueWriteBuffer (hashcat_ctx, device_param->opencl_command_queue, device_param->opencl_d_bitmap_s1_c, CL_TRUE, 0, bitmap_ctx->bitmap_size, bitmap_ctx->bitmap_s1_c, 0, NULL, NULL) == -1) return -1; - if (hc_clEnqueueWriteBuffer (hashcat_ctx, device_param->opencl_command_queue, device_param->opencl_d_bitmap_s1_d, CL_TRUE, 0, bitmap_ctx->bitmap_size, bitmap_ctx->bitmap_s1_d, 0, NULL, NULL) == -1) return -1; - if (hc_clEnqueueWriteBuffer (hashcat_ctx, device_param->opencl_command_queue, device_param->opencl_d_bitmap_s2_a, CL_TRUE, 0, bitmap_ctx->bitmap_size, bitmap_ctx->bitmap_s2_a, 0, NULL, NULL) == -1) return -1; - if (hc_clEnqueueWriteBuffer (hashcat_ctx, device_param->opencl_command_queue, device_param->opencl_d_bitmap_s2_b, CL_TRUE, 0, bitmap_ctx->bitmap_size, bitmap_ctx->bitmap_s2_b, 0, NULL, NULL) == -1) return -1; - if (hc_clEnqueueWriteBuffer (hashcat_ctx, device_param->opencl_command_queue, device_param->opencl_d_bitmap_s2_c, CL_TRUE, 0, bitmap_ctx->bitmap_size, bitmap_ctx->bitmap_s2_c, 0, NULL, NULL) == -1) return -1; - if (hc_clEnqueueWriteBuffer (hashcat_ctx, device_param->opencl_command_queue, device_param->opencl_d_bitmap_s2_d, CL_TRUE, 0, bitmap_ctx->bitmap_size, bitmap_ctx->bitmap_s2_d, 0, NULL, NULL) == -1) return -1; - if (hc_clEnqueueWriteBuffer (hashcat_ctx, device_param->opencl_command_queue, device_param->opencl_d_digests_buf, CL_TRUE, 0, size_digests, hashes->digests_buf, 0, NULL, NULL) == -1) return -1; - if (hc_clEnqueueWriteBuffer (hashcat_ctx, device_param->opencl_command_queue, device_param->opencl_d_salt_bufs, CL_TRUE, 0, size_salts, hashes->salts_buf, 0, NULL, NULL) == -1) return -1; + if (hc_clEnqueueWriteBuffer (hashcat_ctx, device_param->opencl_command_queue, device_param->opencl_d_bitmap_s1_a, CL_FALSE, 0, bitmap_ctx->bitmap_size, bitmap_ctx->bitmap_s1_a, 0, NULL, NULL) == -1) return -1; + if (hc_clEnqueueWriteBuffer (hashcat_ctx, device_param->opencl_command_queue, device_param->opencl_d_bitmap_s1_b, CL_FALSE, 0, bitmap_ctx->bitmap_size, bitmap_ctx->bitmap_s1_b, 0, NULL, NULL) == -1) return -1; + if (hc_clEnqueueWriteBuffer (hashcat_ctx, device_param->opencl_command_queue, device_param->opencl_d_bitmap_s1_c, CL_FALSE, 0, bitmap_ctx->bitmap_size, bitmap_ctx->bitmap_s1_c, 0, NULL, NULL) == -1) return -1; + if (hc_clEnqueueWriteBuffer (hashcat_ctx, device_param->opencl_command_queue, device_param->opencl_d_bitmap_s1_d, CL_FALSE, 0, bitmap_ctx->bitmap_size, bitmap_ctx->bitmap_s1_d, 0, NULL, NULL) == -1) return -1; + if (hc_clEnqueueWriteBuffer (hashcat_ctx, device_param->opencl_command_queue, device_param->opencl_d_bitmap_s2_a, CL_FALSE, 0, bitmap_ctx->bitmap_size, bitmap_ctx->bitmap_s2_a, 0, NULL, NULL) == -1) return -1; + if (hc_clEnqueueWriteBuffer (hashcat_ctx, device_param->opencl_command_queue, device_param->opencl_d_bitmap_s2_b, CL_FALSE, 0, bitmap_ctx->bitmap_size, bitmap_ctx->bitmap_s2_b, 0, NULL, NULL) == -1) return -1; + if (hc_clEnqueueWriteBuffer (hashcat_ctx, device_param->opencl_command_queue, device_param->opencl_d_bitmap_s2_c, CL_FALSE, 0, bitmap_ctx->bitmap_size, bitmap_ctx->bitmap_s2_c, 0, NULL, NULL) == -1) return -1; + if (hc_clEnqueueWriteBuffer (hashcat_ctx, device_param->opencl_command_queue, device_param->opencl_d_bitmap_s2_d, CL_FALSE, 0, bitmap_ctx->bitmap_size, bitmap_ctx->bitmap_s2_d, 0, NULL, NULL) == -1) return -1; + if (hc_clEnqueueWriteBuffer (hashcat_ctx, device_param->opencl_command_queue, device_param->opencl_d_digests_buf, CL_FALSE, 0, size_digests, hashes->digests_buf, 0, NULL, NULL) == -1) return -1; + if (hc_clEnqueueWriteBuffer (hashcat_ctx, device_param->opencl_command_queue, device_param->opencl_d_salt_bufs, CL_FALSE, 0, size_salts, hashes->salts_buf, 0, NULL, NULL) == -1) return -1; /** * special buffers @@ -12580,7 +12540,7 @@ int backend_session_begin (hashcat_ctx_t *hashcat_ctx) if (hc_clCreateBuffer (hashcat_ctx, device_param->opencl_context, CL_MEM_READ_ONLY, size_rules, NULL, &device_param->opencl_d_rules) == -1) return -1; if (hc_clCreateBuffer (hashcat_ctx, device_param->opencl_context, CL_MEM_READ_ONLY, size_rules_c, NULL, &device_param->opencl_d_rules_c) == -1) return -1; - if (hc_clEnqueueWriteBuffer (hashcat_ctx, device_param->opencl_command_queue, device_param->opencl_d_rules, CL_TRUE, 0, size_rules, straight_ctx->kernel_rules_buf, 0, NULL, NULL) == -1) return -1; + if (hc_clEnqueueWriteBuffer (hashcat_ctx, device_param->opencl_command_queue, device_param->opencl_d_rules, CL_FALSE, 0, size_rules, straight_ctx->kernel_rules_buf, 0, NULL, NULL) == -1) return -1; } else if (user_options_extra->attack_kern == ATTACK_KERN_COMBI) { @@ -12603,19 +12563,19 @@ int backend_session_begin (hashcat_ctx_t *hashcat_ctx) { if (hc_clCreateBuffer (hashcat_ctx, device_param->opencl_context, CL_MEM_READ_ONLY, size_esalts, NULL, &device_param->opencl_d_esalt_bufs) == -1) return -1; - if (hc_clEnqueueWriteBuffer (hashcat_ctx, device_param->opencl_command_queue, device_param->opencl_d_esalt_bufs, CL_TRUE, 0, size_esalts, hashes->esalts_buf, 0, NULL, NULL) == -1) return -1; + if (hc_clEnqueueWriteBuffer (hashcat_ctx, device_param->opencl_command_queue, device_param->opencl_d_esalt_bufs, CL_FALSE, 0, size_esalts, hashes->esalts_buf, 0, NULL, NULL) == -1) return -1; } if (hashconfig->st_hash != NULL) { - if (hc_clEnqueueWriteBuffer (hashcat_ctx, device_param->opencl_command_queue, device_param->opencl_d_st_digests_buf, CL_TRUE, 0, size_st_digests, hashes->st_digests_buf, 0, NULL, NULL) == -1) return -1; - if (hc_clEnqueueWriteBuffer (hashcat_ctx, device_param->opencl_command_queue, device_param->opencl_d_st_salts_buf, CL_TRUE, 0, size_st_salts, hashes->st_salts_buf, 0, NULL, NULL) == -1) return -1; + if (hc_clEnqueueWriteBuffer (hashcat_ctx, device_param->opencl_command_queue, device_param->opencl_d_st_digests_buf, CL_FALSE, 0, size_st_digests, hashes->st_digests_buf, 0, NULL, NULL) == -1) return -1; + if (hc_clEnqueueWriteBuffer (hashcat_ctx, device_param->opencl_command_queue, device_param->opencl_d_st_salts_buf, CL_FALSE, 0, size_st_salts, hashes->st_salts_buf, 0, NULL, NULL) == -1) return -1; if (size_esalts) { if (hc_clCreateBuffer (hashcat_ctx, device_param->opencl_context, CL_MEM_READ_ONLY, size_st_esalts, NULL, &device_param->opencl_d_st_esalts_buf) == -1) return -1; - if (hc_clEnqueueWriteBuffer (hashcat_ctx, device_param->opencl_command_queue, device_param->opencl_d_st_esalts_buf, CL_TRUE, 0, size_st_esalts, hashes->st_esalts_buf, 0, NULL, NULL) == -1) return -1; + if (hc_clEnqueueWriteBuffer (hashcat_ctx, device_param->opencl_command_queue, device_param->opencl_d_st_esalts_buf, CL_FALSE, 0, size_st_esalts, hashes->st_esalts_buf, 0, NULL, NULL) == -1) return -1; } } } @@ -15928,8 +15888,8 @@ int backend_session_update_mp (hashcat_ctx_t *hashcat_ctx) //for (u32 i = 3; i < 4; i++) { CL_rc = hc_clSetKernelArg (hashcat_ctx, device_param->opencl_kernel_mp, i, sizeof (cl_ulong), device_param->kernel_params_mp[i]); if (CL_rc == -1) return -1; } //for (u32 i = 4; i < 8; i++) { CL_rc = hc_clSetKernelArg (hashcat_ctx, device_param->opencl_kernel_mp, i, sizeof (cl_uint), device_param->kernel_params_mp[i]); if (CL_rc == -1) return -1; } - if (hc_cuMemcpyHtoD (hashcat_ctx, device_param->cuda_d_root_css_buf, mask_ctx->root_css_buf, device_param->size_root_css) == -1) return -1; - if (hc_cuMemcpyHtoD (hashcat_ctx, device_param->cuda_d_markov_css_buf, mask_ctx->markov_css_buf, device_param->size_markov_css) == -1) return -1; + if (hc_cuMemcpyHtoDAsync (hashcat_ctx, device_param->cuda_d_root_css_buf, mask_ctx->root_css_buf, device_param->size_root_css, device_param->cuda_stream) == -1) return -1; + if (hc_cuMemcpyHtoDAsync (hashcat_ctx, device_param->cuda_d_markov_css_buf, mask_ctx->markov_css_buf, device_param->size_markov_css, device_param->cuda_stream) == -1) return -1; } if (device_param->is_hip == true) @@ -15937,8 +15897,8 @@ int backend_session_update_mp (hashcat_ctx_t *hashcat_ctx) //for (u32 i = 3; i < 4; i++) { CL_rc = hc_clSetKernelArg (hashcat_ctx, device_param->opencl_kernel_mp, i, sizeof (cl_ulong), device_param->kernel_params_mp[i]); if (CL_rc == -1) return -1; } //for (u32 i = 4; i < 8; i++) { CL_rc = hc_clSetKernelArg (hashcat_ctx, device_param->opencl_kernel_mp, i, sizeof (cl_uint), device_param->kernel_params_mp[i]); if (CL_rc == -1) return -1; } - if (hc_hipMemcpyHtoD (hashcat_ctx, device_param->hip_d_root_css_buf, mask_ctx->root_css_buf, device_param->size_root_css) == -1) return -1; - if (hc_hipMemcpyHtoD (hashcat_ctx, device_param->hip_d_markov_css_buf, mask_ctx->markov_css_buf, device_param->size_markov_css) == -1) return -1; + if (hc_hipMemcpyHtoDAsync (hashcat_ctx, device_param->hip_d_root_css_buf, mask_ctx->root_css_buf, device_param->size_root_css, device_param->hip_stream) == -1) return -1; + if (hc_hipMemcpyHtoDAsync (hashcat_ctx, device_param->hip_d_markov_css_buf, mask_ctx->markov_css_buf, device_param->size_markov_css, device_param->hip_stream) == -1) return -1; } if (device_param->is_opencl == true) @@ -15946,8 +15906,8 @@ int backend_session_update_mp (hashcat_ctx_t *hashcat_ctx) for (u32 i = 3; i < 4; i++) { if (hc_clSetKernelArg (hashcat_ctx, device_param->opencl_kernel_mp, i, sizeof (cl_ulong), device_param->kernel_params_mp[i]) == -1) return -1; } for (u32 i = 4; i < 8; i++) { if (hc_clSetKernelArg (hashcat_ctx, device_param->opencl_kernel_mp, i, sizeof (cl_uint), device_param->kernel_params_mp[i]) == -1) return -1; } - if (hc_clEnqueueWriteBuffer (hashcat_ctx, device_param->opencl_command_queue, device_param->opencl_d_root_css_buf, CL_TRUE, 0, device_param->size_root_css, mask_ctx->root_css_buf, 0, NULL, NULL) == -1) return -1; - if (hc_clEnqueueWriteBuffer (hashcat_ctx, device_param->opencl_command_queue, device_param->opencl_d_markov_css_buf, CL_TRUE, 0, device_param->size_markov_css, mask_ctx->markov_css_buf, 0, NULL, NULL) == -1) return -1; + if (hc_clEnqueueWriteBuffer (hashcat_ctx, device_param->opencl_command_queue, device_param->opencl_d_root_css_buf, CL_FALSE, 0, device_param->size_root_css, mask_ctx->root_css_buf, 0, NULL, NULL) == -1) return -1; + if (hc_clEnqueueWriteBuffer (hashcat_ctx, device_param->opencl_command_queue, device_param->opencl_d_markov_css_buf, CL_FALSE, 0, device_param->size_markov_css, mask_ctx->markov_css_buf, 0, NULL, NULL) == -1) return -1; } } @@ -15989,8 +15949,8 @@ int backend_session_update_mp_rl (hashcat_ctx_t *hashcat_ctx, const u32 css_cnt_ //for (u32 i = 4; i < 7; i++) { CL_rc = hc_clSetKernelArg (hashcat_ctx, device_param->opencl_kernel_mp_r, i, sizeof (cl_uint), device_param->kernel_params_mp_r[i]); if (CL_rc == -1) return -1; } //for (u32 i = 8; i < 8; i++) { CL_rc = hc_clSetKernelArg (hashcat_ctx, device_param->opencl_kernel_mp_r, i, sizeof (cl_ulong), device_param->kernel_params_mp_r[i]); if (CL_rc == -1) return -1; } - if (hc_cuMemcpyHtoD (hashcat_ctx, device_param->cuda_d_root_css_buf, mask_ctx->root_css_buf, device_param->size_root_css) == -1) return -1; - if (hc_cuMemcpyHtoD (hashcat_ctx, device_param->cuda_d_markov_css_buf, mask_ctx->markov_css_buf, device_param->size_markov_css) == -1) return -1; + if (hc_cuMemcpyHtoDAsync (hashcat_ctx, device_param->cuda_d_root_css_buf, mask_ctx->root_css_buf, device_param->size_root_css, device_param->cuda_stream) == -1) return -1; + if (hc_cuMemcpyHtoDAsync (hashcat_ctx, device_param->cuda_d_markov_css_buf, mask_ctx->markov_css_buf, device_param->size_markov_css, device_param->cuda_stream) == -1) return -1; } if (device_param->is_hip == true) @@ -16003,8 +15963,8 @@ int backend_session_update_mp_rl (hashcat_ctx_t *hashcat_ctx, const u32 css_cnt_ //for (u32 i = 4; i < 7; i++) { CL_rc = hc_clSetKernelArg (hashcat_ctx, device_param->opencl_kernel_mp_r, i, sizeof (cl_uint), device_param->kernel_params_mp_r[i]); if (CL_rc == -1) return -1; } //for (u32 i = 8; i < 8; i++) { CL_rc = hc_clSetKernelArg (hashcat_ctx, device_param->opencl_kernel_mp_r, i, sizeof (cl_ulong), device_param->kernel_params_mp_r[i]); if (CL_rc == -1) return -1; } - if (hc_hipMemcpyHtoD (hashcat_ctx, device_param->hip_d_root_css_buf, mask_ctx->root_css_buf, device_param->size_root_css) == -1) return -1; - if (hc_hipMemcpyHtoD (hashcat_ctx, device_param->hip_d_markov_css_buf, mask_ctx->markov_css_buf, device_param->size_markov_css) == -1) return -1; + if (hc_hipMemcpyHtoDAsync (hashcat_ctx, device_param->hip_d_root_css_buf, mask_ctx->root_css_buf, device_param->size_root_css, device_param->hip_stream) == -1) return -1; + if (hc_hipMemcpyHtoDAsync (hashcat_ctx, device_param->hip_d_markov_css_buf, mask_ctx->markov_css_buf, device_param->size_markov_css, device_param->hip_stream) == -1) return -1; } if (device_param->is_opencl == true) @@ -16017,8 +15977,8 @@ int backend_session_update_mp_rl (hashcat_ctx_t *hashcat_ctx, const u32 css_cnt_ for (u32 i = 4; i < 7; i++) { if (hc_clSetKernelArg (hashcat_ctx, device_param->opencl_kernel_mp_r, i, sizeof (cl_uint), device_param->kernel_params_mp_r[i]) == -1) return -1; } for (u32 i = 8; i < 8; i++) { if (hc_clSetKernelArg (hashcat_ctx, device_param->opencl_kernel_mp_r, i, sizeof (cl_ulong), device_param->kernel_params_mp_r[i]) == -1) return -1; } - if (hc_clEnqueueWriteBuffer (hashcat_ctx, device_param->opencl_command_queue, device_param->opencl_d_root_css_buf, CL_TRUE, 0, device_param->size_root_css, mask_ctx->root_css_buf, 0, NULL, NULL) == -1) return -1; - if (hc_clEnqueueWriteBuffer (hashcat_ctx, device_param->opencl_command_queue, device_param->opencl_d_markov_css_buf, CL_TRUE, 0, device_param->size_markov_css, mask_ctx->markov_css_buf, 0, NULL, NULL) == -1) return -1; + if (hc_clEnqueueWriteBuffer (hashcat_ctx, device_param->opencl_command_queue, device_param->opencl_d_root_css_buf, CL_FALSE, 0, device_param->size_root_css, mask_ctx->root_css_buf, 0, NULL, NULL) == -1) return -1; + if (hc_clEnqueueWriteBuffer (hashcat_ctx, device_param->opencl_command_queue, device_param->opencl_d_markov_css_buf, CL_FALSE, 0, device_param->size_markov_css, mask_ctx->markov_css_buf, 0, NULL, NULL) == -1) return -1; } } From 507d40fff638eb94d813891faf5c1db4de6f74f0 Mon Sep 17 00:00:00 2001 From: Jukka Ojanen Date: Mon, 26 Jul 2021 15:35:59 +0300 Subject: [PATCH 08/33] Allow async execution of check_hash() and check_cracked() --- src/hashes.c | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/src/hashes.c b/src/hashes.c index 27aa94370..0355a50ad 100644 --- a/src/hashes.c +++ b/src/hashes.c @@ -319,17 +319,17 @@ void check_hash (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param, pl if (device_param->is_cuda == true) { - hc_cuMemcpyDtoH (hashcat_ctx, tmps, device_param->cuda_d_tmps + (plain->gidvid * hashconfig->tmp_size), hashconfig->tmp_size); + hc_cuMemcpyDtoHAsync (hashcat_ctx, tmps, device_param->cuda_d_tmps + (plain->gidvid * hashconfig->tmp_size), hashconfig->tmp_size, device_param->cuda_stream); } if (device_param->is_hip == true) { - hc_hipMemcpyDtoH (hashcat_ctx, tmps, device_param->hip_d_tmps + (plain->gidvid * hashconfig->tmp_size), hashconfig->tmp_size); + hc_hipMemcpyDtoHAsync (hashcat_ctx, tmps, device_param->hip_d_tmps + (plain->gidvid * hashconfig->tmp_size), hashconfig->tmp_size, device_param->hip_stream); } if (device_param->is_opencl == true) { - hc_clEnqueueReadBuffer (hashcat_ctx, device_param->opencl_command_queue, device_param->opencl_d_tmps, CL_TRUE, plain->gidvid * hashconfig->tmp_size, hashconfig->tmp_size, tmps, 0, NULL, NULL); + hc_clEnqueueReadBuffer (hashcat_ctx, device_param->opencl_command_queue, device_param->opencl_d_tmps, CL_FALSE, plain->gidvid * hashconfig->tmp_size, hashconfig->tmp_size, tmps, 0, NULL, NULL); } } @@ -491,14 +491,14 @@ int check_cracked (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param) if (device_param->is_cuda == true) { - CU_rc = hc_cuMemcpyDtoH (hashcat_ctx, &num_cracked, device_param->cuda_d_result, sizeof (u32)); + CU_rc = hc_cuMemcpyDtoHAsync (hashcat_ctx, &num_cracked, device_param->cuda_d_result, sizeof (u32), device_param->cuda_stream); if (CU_rc == -1) return -1; } if (device_param->is_hip == true) { - HIP_rc = hc_hipMemcpyDtoH (hashcat_ctx, &num_cracked, device_param->hip_d_result, sizeof (u32)); + HIP_rc = hc_hipMemcpyDtoHAsync (hashcat_ctx, &num_cracked, device_param->hip_d_result, sizeof (u32), device_param->hip_stream); if (HIP_rc == -1) return -1; } @@ -524,14 +524,14 @@ int check_cracked (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param) if (device_param->is_cuda == true) { - CU_rc = hc_cuMemcpyDtoH (hashcat_ctx, cracked, device_param->cuda_d_plain_bufs, num_cracked * sizeof (plain_t)); + CU_rc = hc_cuMemcpyDtoHAsync (hashcat_ctx, cracked, device_param->cuda_d_plain_bufs, num_cracked * sizeof (plain_t), device_param->cuda_stream); if (CU_rc == -1) return -1; } if (device_param->is_hip == true) { - HIP_rc = hc_hipMemcpyDtoH (hashcat_ctx, cracked, device_param->hip_d_plain_bufs, num_cracked * sizeof (plain_t)); + HIP_rc = hc_hipMemcpyDtoHAsync (hashcat_ctx, cracked, device_param->hip_d_plain_bufs, num_cracked * sizeof (plain_t), device_param->hip_stream); if (HIP_rc == -1) return -1; } @@ -588,21 +588,21 @@ int check_cracked (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param) if (device_param->is_cuda == true) { - CU_rc = hc_cuMemcpyHtoD (hashcat_ctx, device_param->cuda_d_digests_shown + (salt_buf->digests_offset * sizeof (u32)), &hashes->digests_shown_tmp[salt_buf->digests_offset], salt_buf->digests_cnt * sizeof (u32)); + CU_rc = hc_cuMemcpyHtoDAsync (hashcat_ctx, device_param->cuda_d_digests_shown + (salt_buf->digests_offset * sizeof (u32)), &hashes->digests_shown_tmp[salt_buf->digests_offset], salt_buf->digests_cnt * sizeof (u32), device_param->cuda_stream); if (CU_rc == -1) return -1; } if (device_param->is_hip == true) { - HIP_rc = hc_hipMemcpyHtoD (hashcat_ctx, device_param->hip_d_digests_shown + (salt_buf->digests_offset * sizeof (u32)), &hashes->digests_shown_tmp[salt_buf->digests_offset], salt_buf->digests_cnt * sizeof (u32)); + HIP_rc = hc_hipMemcpyHtoDAsync (hashcat_ctx, device_param->hip_d_digests_shown + (salt_buf->digests_offset * sizeof (u32)), &hashes->digests_shown_tmp[salt_buf->digests_offset], salt_buf->digests_cnt * sizeof (u32), device_param->hip_stream); if (HIP_rc == -1) return -1; } if (device_param->is_opencl == true) { - CL_rc = hc_clEnqueueWriteBuffer (hashcat_ctx, device_param->opencl_command_queue, device_param->opencl_d_digests_shown, CL_TRUE, salt_buf->digests_offset * sizeof (u32), salt_buf->digests_cnt * sizeof (u32), &hashes->digests_shown_tmp[salt_buf->digests_offset], 0, NULL, NULL); + CL_rc = hc_clEnqueueWriteBuffer (hashcat_ctx, device_param->opencl_command_queue, device_param->opencl_d_digests_shown, CL_FALSE, salt_buf->digests_offset * sizeof (u32), salt_buf->digests_cnt * sizeof (u32), &hashes->digests_shown_tmp[salt_buf->digests_offset], 0, NULL, NULL); if (CL_rc == -1) return -1; } @@ -633,21 +633,21 @@ int check_cracked (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param) if (device_param->is_cuda == true) { - CU_rc = hc_cuMemcpyHtoD (hashcat_ctx, device_param->cuda_d_result, &num_cracked, sizeof (u32)); + CU_rc = hc_cuMemcpyHtoDAsync (hashcat_ctx, device_param->cuda_d_result, &num_cracked, sizeof (u32), device_param->cuda_stream); if (CU_rc == -1) return -1; } if (device_param->is_hip == true) { - HIP_rc = hc_hipMemcpyHtoD (hashcat_ctx, device_param->hip_d_result, &num_cracked, sizeof (u32)); + HIP_rc = hc_hipMemcpyHtoDAsync (hashcat_ctx, device_param->hip_d_result, &num_cracked, sizeof (u32), device_param->hip_stream); if (HIP_rc == -1) return -1; } if (device_param->is_opencl == true) { - CL_rc = hc_clEnqueueWriteBuffer (hashcat_ctx, device_param->opencl_command_queue, device_param->opencl_d_result, CL_TRUE, 0, sizeof (u32), &num_cracked, 0, NULL, NULL); + CL_rc = hc_clEnqueueWriteBuffer (hashcat_ctx, device_param->opencl_command_queue, device_param->opencl_d_result, CL_FALSE, 0, sizeof (u32), &num_cracked, 0, NULL, NULL); if (CL_rc == -1) return -1; } From a642f7b23344309fe8dafe6d3d73285ae49af354 Mon Sep 17 00:00:00 2001 From: Jukka Ojanen Date: Mon, 26 Jul 2021 15:36:42 +0300 Subject: [PATCH 09/33] Remove synchronous GPU memory copy functions --- include/backend.h | 6 -- include/ext_cuda.h | 6 -- include/ext_hip.h | 6 -- src/backend.c | 168 --------------------------------------------- 4 files changed, 186 deletions(-) diff --git a/include/backend.h b/include/backend.h index baa94b50c..a54ba357e 100644 --- a/include/backend.h +++ b/include/backend.h @@ -68,11 +68,8 @@ int hc_cuFuncSetAttribute (hashcat_ctx_t *hashcat_ctx, CUfunction hfunc, int hc_cuInit (hashcat_ctx_t *hashcat_ctx, unsigned int Flags); int hc_cuLaunchKernel (hashcat_ctx_t *hashcat_ctx, CUfunction f, unsigned int gridDimX, unsigned int gridDimY, unsigned int gridDimZ, unsigned int blockDimX, unsigned int blockDimY, unsigned int blockDimZ, unsigned int sharedMemBytes, CUstream hStream, void **kernelParams, void **extra); int hc_cuMemAlloc (hashcat_ctx_t *hashcat_ctx, CUdeviceptr *dptr, size_t bytesize); -int hc_cuMemcpyDtoD (hashcat_ctx_t *hashcat_ctx, CUdeviceptr dstDevice, CUdeviceptr srcDevice, size_t ByteCount); int hc_cuMemcpyDtoDAsync (hashcat_ctx_t *hashcat_ctx, CUdeviceptr dstDevice, CUdeviceptr srcDevice, size_t ByteCount, CUstream hStream); -int hc_cuMemcpyDtoH (hashcat_ctx_t *hashcat_ctx, void *dstHost, CUdeviceptr srcDevice, size_t ByteCount); int hc_cuMemcpyDtoHAsync (hashcat_ctx_t *hashcat_ctx, void *dstHost, CUdeviceptr srcDevice, size_t ByteCount, CUstream hStream); -int hc_cuMemcpyHtoD (hashcat_ctx_t *hashcat_ctx, CUdeviceptr dstDevice, const void *srcHost, size_t ByteCount); int hc_cuMemcpyHtoDAsync (hashcat_ctx_t *hashcat_ctx, CUdeviceptr dstDevice, const void *srcHost, size_t ByteCount, CUstream hStream); int hc_cuMemFree (hashcat_ctx_t *hashcat_ctx, CUdeviceptr dptr); int hc_cuModuleGetFunction (hashcat_ctx_t *hashcat_ctx, CUfunction *hfunc, CUmodule hmod, const char *name); @@ -119,11 +116,8 @@ int hc_hipFuncGetAttribute (hashcat_ctx_t *hashcat_ctx, int *pi, HIPfuncti int hc_hipInit (hashcat_ctx_t *hashcat_ctx, unsigned int Flags); int hc_hipLaunchKernel (hashcat_ctx_t *hashcat_ctx, HIPfunction f, unsigned int gridDimX, unsigned int gridDimY, unsigned int gridDimZ, unsigned int blockDimX, unsigned int blockDimY, unsigned int blockDimZ, unsigned int sharedMemBytes, HIPstream hStream, void **kernelParams, void **extra); int hc_hipMemAlloc (hashcat_ctx_t *hashcat_ctx, HIPdeviceptr *dptr, size_t bytesize); -int hc_hipMemcpyDtoD (hashcat_ctx_t *hashcat_ctx, HIPdeviceptr dstDevice, HIPdeviceptr srcDevice, size_t ByteCount); int hc_hipMemcpyDtoDAsync (hashcat_ctx_t *hashcat_ctx, HIPdeviceptr dstDevice, HIPdeviceptr srcDevice, size_t ByteCount, HIPstream hStream); -int hc_hipMemcpyDtoH (hashcat_ctx_t *hashcat_ctx, void *dstHost, HIPdeviceptr srcDevice, size_t ByteCount); int hc_hipMemcpyDtoHAsync (hashcat_ctx_t *hashcat_ctx, void *dstHost, HIPdeviceptr srcDevice, size_t ByteCount, HIPstream hStream); -int hc_hipMemcpyHtoD (hashcat_ctx_t *hashcat_ctx, HIPdeviceptr dstDevice, const void *srcHost, size_t ByteCount); int hc_hipMemcpyHtoDAsync (hashcat_ctx_t *hashcat_ctx, HIPdeviceptr dstDevice, const void *srcHost, size_t ByteCount, HIPstream hStream); int hc_hipMemFree (hashcat_ctx_t *hashcat_ctx, HIPdeviceptr dptr); int hc_hipModuleGetFunction (hashcat_ctx_t *hashcat_ctx, HIPfunction *hfunc, HIPmodule hmod, const char *name); diff --git a/include/ext_cuda.h b/include/ext_cuda.h index 8048c729f..7f40e1410 100644 --- a/include/ext_cuda.h +++ b/include/ext_cuda.h @@ -1028,11 +1028,8 @@ typedef CUresult (CUDA_API_CALL *CUDA_CUINIT) (unsigned int); typedef CUresult (CUDA_API_CALL *CUDA_CULAUNCHKERNEL) (CUfunction, unsigned int, unsigned int, unsigned int, unsigned int, unsigned int, unsigned int, unsigned int, CUstream, void **, void **); typedef CUresult (CUDA_API_CALL *CUDA_CUMEMALLOC) (CUdeviceptr *, size_t); typedef CUresult (CUDA_API_CALL *CUDA_CUMEMALLOCHOST) (void **, size_t); -typedef CUresult (CUDA_API_CALL *CUDA_CUMEMCPYDTOD) (CUdeviceptr, CUdeviceptr, size_t); typedef CUresult (CUDA_API_CALL *CUDA_CUMEMCPYDTODASYNC) (CUdeviceptr, CUdeviceptr, size_t, CUstream); -typedef CUresult (CUDA_API_CALL *CUDA_CUMEMCPYDTOH) (void *, CUdeviceptr, size_t); typedef CUresult (CUDA_API_CALL *CUDA_CUMEMCPYDTOHASYNC) (void *, CUdeviceptr, size_t, CUstream); -typedef CUresult (CUDA_API_CALL *CUDA_CUMEMCPYHTOD) (CUdeviceptr, const void *, size_t); typedef CUresult (CUDA_API_CALL *CUDA_CUMEMCPYHTODASYNC) (CUdeviceptr, const void *, size_t, CUstream); typedef CUresult (CUDA_API_CALL *CUDA_CUMEMFREE) (CUdeviceptr); typedef CUresult (CUDA_API_CALL *CUDA_CUMEMFREEHOST) (void *); @@ -1093,11 +1090,8 @@ typedef struct hc_cuda_lib CUDA_CULAUNCHKERNEL cuLaunchKernel; CUDA_CUMEMALLOC cuMemAlloc; CUDA_CUMEMALLOCHOST cuMemAllocHost; - CUDA_CUMEMCPYDTOD cuMemcpyDtoD; CUDA_CUMEMCPYDTODASYNC cuMemcpyDtoDAsync; - CUDA_CUMEMCPYDTOH cuMemcpyDtoH; CUDA_CUMEMCPYDTOHASYNC cuMemcpyDtoHAsync; - CUDA_CUMEMCPYHTOD cuMemcpyHtoD; CUDA_CUMEMCPYHTODASYNC cuMemcpyHtoDAsync; CUDA_CUMEMFREE cuMemFree; CUDA_CUMEMFREEHOST cuMemFreeHost; diff --git a/include/ext_hip.h b/include/ext_hip.h index a99d1e5a9..631dbfb33 100644 --- a/include/ext_hip.h +++ b/include/ext_hip.h @@ -1037,11 +1037,8 @@ typedef HIPresult (HIP_API_CALL *HIP_HIPINIT) (unsigned int); typedef HIPresult (HIP_API_CALL *HIP_HIPLAUNCHKERNEL) (HIPfunction, unsigned int, unsigned int, unsigned int, unsigned int, unsigned int, unsigned int, unsigned int, HIPstream, void **, void **); typedef HIPresult (HIP_API_CALL *HIP_HIPMEMALLOC) (HIPdeviceptr *, size_t); typedef HIPresult (HIP_API_CALL *HIP_HIPMEMALLOCHOST) (void **, size_t); -typedef HIPresult (HIP_API_CALL *HIP_HIPMEMCPYDTOD) (HIPdeviceptr, HIPdeviceptr, size_t); typedef HIPresult (HIP_API_CALL *HIP_HIPMEMCPYDTODASYNC) (HIPdeviceptr, HIPdeviceptr, size_t, HIPstream); -typedef HIPresult (HIP_API_CALL *HIP_HIPMEMCPYDTOH) (void *, HIPdeviceptr, size_t); typedef HIPresult (HIP_API_CALL *HIP_HIPMEMCPYDTOHASYNC) (void *, HIPdeviceptr, size_t, HIPstream); -typedef HIPresult (HIP_API_CALL *HIP_HIPMEMCPYHTOD) (HIPdeviceptr, const void *, size_t); typedef HIPresult (HIP_API_CALL *HIP_HIPMEMCPYHTODASYNC) (HIPdeviceptr, const void *, size_t, HIPstream); typedef HIPresult (HIP_API_CALL *HIP_HIPMEMFREE) (HIPdeviceptr); typedef HIPresult (HIP_API_CALL *HIP_HIPMEMFREEHOST) (void *); @@ -1102,11 +1099,8 @@ typedef struct hc_hip_lib HIP_HIPLAUNCHKERNEL hipLaunchKernel; HIP_HIPMEMALLOC hipMemAlloc; HIP_HIPMEMALLOCHOST hipMemAllocHost; - HIP_HIPMEMCPYDTOD hipMemcpyDtoD; HIP_HIPMEMCPYDTODASYNC hipMemcpyDtoDAsync; - HIP_HIPMEMCPYDTOH hipMemcpyDtoH; HIP_HIPMEMCPYDTOHASYNC hipMemcpyDtoHAsync; - HIP_HIPMEMCPYHTOD hipMemcpyHtoD; HIP_HIPMEMCPYHTODASYNC hipMemcpyHtoDAsync; HIP_HIPMEMFREE hipMemFree; HIP_HIPMEMFREEHOST hipMemFreeHost; diff --git a/src/backend.c b/src/backend.c index 9631e8dd2..a6f42b672 100644 --- a/src/backend.c +++ b/src/backend.c @@ -1257,11 +1257,8 @@ int cuda_init (hashcat_ctx_t *hashcat_ctx) HC_LOAD_FUNC_CUDA (cuda, cuLaunchKernel, cuLaunchKernel, CUDA_CULAUNCHKERNEL, CUDA, 1); HC_LOAD_FUNC_CUDA (cuda, cuMemAlloc, cuMemAlloc_v2, CUDA_CUMEMALLOC, CUDA, 1); HC_LOAD_FUNC_CUDA (cuda, cuMemAllocHost, cuMemAllocHost_v2, CUDA_CUMEMALLOCHOST, CUDA, 1); - HC_LOAD_FUNC_CUDA (cuda, cuMemcpyDtoD, cuMemcpyDtoD_v2, CUDA_CUMEMCPYDTOD, CUDA, 1); HC_LOAD_FUNC_CUDA (cuda, cuMemcpyDtoDAsync, cuMemcpyDtoDAsync_v2, CUDA_CUMEMCPYDTODASYNC, CUDA, 1); - HC_LOAD_FUNC_CUDA (cuda, cuMemcpyDtoH, cuMemcpyDtoH_v2, CUDA_CUMEMCPYDTOH, CUDA, 1); HC_LOAD_FUNC_CUDA (cuda, cuMemcpyDtoHAsync, cuMemcpyDtoHAsync_v2, CUDA_CUMEMCPYDTOHASYNC, CUDA, 1); - HC_LOAD_FUNC_CUDA (cuda, cuMemcpyHtoD, cuMemcpyHtoD_v2, CUDA_CUMEMCPYHTOD, CUDA, 1); HC_LOAD_FUNC_CUDA (cuda, cuMemcpyHtoDAsync, cuMemcpyHtoDAsync_v2, CUDA_CUMEMCPYHTODASYNC, CUDA, 1); HC_LOAD_FUNC_CUDA (cuda, cuMemFree, cuMemFree_v2, CUDA_CUMEMFREE, CUDA, 1); HC_LOAD_FUNC_CUDA (cuda, cuMemFreeHost, cuMemFreeHost, CUDA_CUMEMFREEHOST, CUDA, 1); @@ -1687,33 +1684,6 @@ int hc_cuMemFree (hashcat_ctx_t *hashcat_ctx, CUdeviceptr dptr) return 0; } -int hc_cuMemcpyDtoH (hashcat_ctx_t *hashcat_ctx, void *dstHost, CUdeviceptr srcDevice, size_t ByteCount) -{ - backend_ctx_t *backend_ctx = hashcat_ctx->backend_ctx; - - CUDA_PTR *cuda = (CUDA_PTR *) backend_ctx->cuda; - - const CUresult CU_err = cuda->cuMemcpyDtoH (dstHost, srcDevice, ByteCount); - - if (CU_err != CUDA_SUCCESS) - { - const char *pStr = NULL; - - if (cuda->cuGetErrorString (CU_err, &pStr) == CUDA_SUCCESS) - { - event_log_error (hashcat_ctx, "cuMemcpyDtoH(): %s", pStr); - } - else - { - event_log_error (hashcat_ctx, "cuMemcpyDtoH(): %d", CU_err); - } - - return -1; - } - - return 0; -} - int hc_cuMemcpyDtoHAsync (hashcat_ctx_t *hashcat_ctx, void *dstHost, CUdeviceptr srcDevice, size_t ByteCount, CUstream hStream) { backend_ctx_t *backend_ctx = hashcat_ctx->backend_ctx; @@ -1741,33 +1711,6 @@ int hc_cuMemcpyDtoHAsync (hashcat_ctx_t *hashcat_ctx, void *dstHost, CUdeviceptr return 0; } -int hc_cuMemcpyDtoD (hashcat_ctx_t *hashcat_ctx, CUdeviceptr dstDevice, CUdeviceptr srcDevice, size_t ByteCount) -{ - backend_ctx_t *backend_ctx = hashcat_ctx->backend_ctx; - - CUDA_PTR *cuda = (CUDA_PTR *) backend_ctx->cuda; - - const CUresult CU_err = cuda->cuMemcpyDtoD (dstDevice, srcDevice, ByteCount); - - if (CU_err != CUDA_SUCCESS) - { - const char *pStr = NULL; - - if (cuda->cuGetErrorString (CU_err, &pStr) == CUDA_SUCCESS) - { - event_log_error (hashcat_ctx, "cuMemcpyDtoD(): %s", pStr); - } - else - { - event_log_error (hashcat_ctx, "cuMemcpyDtoD(): %d", CU_err); - } - - return -1; - } - - return 0; -} - int hc_cuMemcpyDtoDAsync (hashcat_ctx_t *hashcat_ctx, CUdeviceptr dstDevice, CUdeviceptr srcDevice, size_t ByteCount, CUstream hStream) { backend_ctx_t *backend_ctx = hashcat_ctx->backend_ctx; @@ -1795,33 +1738,6 @@ int hc_cuMemcpyDtoDAsync (hashcat_ctx_t *hashcat_ctx, CUdeviceptr dstDevice, CUd return 0; } -int hc_cuMemcpyHtoD (hashcat_ctx_t *hashcat_ctx, CUdeviceptr dstDevice, const void *srcHost, size_t ByteCount) -{ - backend_ctx_t *backend_ctx = hashcat_ctx->backend_ctx; - - CUDA_PTR *cuda = (CUDA_PTR *) backend_ctx->cuda; - - const CUresult CU_err = cuda->cuMemcpyHtoD (dstDevice, srcHost, ByteCount); - - if (CU_err != CUDA_SUCCESS) - { - const char *pStr = NULL; - - if (cuda->cuGetErrorString (CU_err, &pStr) == CUDA_SUCCESS) - { - event_log_error (hashcat_ctx, "cuMemcpyHtoD(): %s", pStr); - } - else - { - event_log_error (hashcat_ctx, "cuMemcpyHtoD(): %d", CU_err); - } - - return -1; - } - - return 0; -} - int hc_cuMemcpyHtoDAsync (hashcat_ctx_t *hashcat_ctx, CUdeviceptr dstDevice, const void *srcHost, size_t ByteCount, CUstream hStream) { backend_ctx_t *backend_ctx = hashcat_ctx->backend_ctx; @@ -2550,11 +2466,8 @@ int hip_init (hashcat_ctx_t *hashcat_ctx) HC_LOAD_FUNC_HIP (hip, hipLaunchKernel, hipModuleLaunchKernel, HIP_HIPLAUNCHKERNEL, HIP, 1); HC_LOAD_FUNC_HIP (hip, hipMemAlloc, hipMalloc, HIP_HIPMEMALLOC, HIP, 1); HC_LOAD_FUNC_HIP (hip, hipMemAllocHost, hipMemAllocHost, HIP_HIPMEMALLOCHOST, HIP, 1); - HC_LOAD_FUNC_HIP (hip, hipMemcpyDtoD, hipMemcpyDtoD, HIP_HIPMEMCPYDTOD, HIP, 1); HC_LOAD_FUNC_HIP (hip, hipMemcpyDtoDAsync, hipMemcpyDtoDAsync, HIP_HIPMEMCPYDTODASYNC, HIP, 1); - HC_LOAD_FUNC_HIP (hip, hipMemcpyDtoH, hipMemcpyDtoH, HIP_HIPMEMCPYDTOH, HIP, 1); HC_LOAD_FUNC_HIP (hip, hipMemcpyDtoHAsync, hipMemcpyDtoHAsync, HIP_HIPMEMCPYDTOHASYNC, HIP, 1); - HC_LOAD_FUNC_HIP (hip, hipMemcpyHtoD, hipMemcpyHtoD, HIP_HIPMEMCPYHTOD, HIP, 1); HC_LOAD_FUNC_HIP (hip, hipMemcpyHtoDAsync, hipMemcpyHtoDAsync, HIP_HIPMEMCPYHTODASYNC, HIP, 1); HC_LOAD_FUNC_HIP (hip, hipMemFree, hipFree, HIP_HIPMEMFREE, HIP, 1); HC_LOAD_FUNC_HIP (hip, hipMemFreeHost, hipFreeHost, HIP_HIPMEMFREEHOST, HIP, 1); @@ -2981,33 +2894,6 @@ int hc_hipMemFree (hashcat_ctx_t *hashcat_ctx, HIPdeviceptr dptr) return 0; } -int hc_hipMemcpyDtoH (hashcat_ctx_t *hashcat_ctx, void *dstHost, HIPdeviceptr srcDevice, size_t ByteCount) -{ - backend_ctx_t *backend_ctx = hashcat_ctx->backend_ctx; - - HIP_PTR *hip = (HIP_PTR *) backend_ctx->hip; - - const HIPresult HIP_err = hip->hipMemcpyDtoH (dstHost, srcDevice, ByteCount); - - if (HIP_err != HIP_SUCCESS) - { - const char *pStr = NULL; - - if (hip->hipGetErrorString (HIP_err, &pStr) == HIP_SUCCESS) - { - event_log_error (hashcat_ctx, "hipMemcpyDtoH(): %s", pStr); - } - else - { - event_log_error (hashcat_ctx, "hipMemcpyDtoH(): %d", HIP_err); - } - - return -1; - } - - return 0; -} - int hc_hipMemcpyDtoHAsync (hashcat_ctx_t *hashcat_ctx, void *dstHost, HIPdeviceptr srcDevice, size_t ByteCount, HIPstream hStream) { backend_ctx_t *backend_ctx = hashcat_ctx->backend_ctx; @@ -3035,33 +2921,6 @@ int hc_hipMemcpyDtoHAsync (hashcat_ctx_t *hashcat_ctx, void *dstHost, HIPdevicep return 0; } -int hc_hipMemcpyDtoD (hashcat_ctx_t *hashcat_ctx, HIPdeviceptr dstDevice, HIPdeviceptr srcDevice, size_t ByteCount) -{ - backend_ctx_t *backend_ctx = hashcat_ctx->backend_ctx; - - HIP_PTR *hip = (HIP_PTR *) backend_ctx->hip; - - const HIPresult HIP_err = hip->hipMemcpyDtoD (dstDevice, srcDevice, ByteCount); - - if (HIP_err != HIP_SUCCESS) - { - const char *pStr = NULL; - - if (hip->hipGetErrorString (HIP_err, &pStr) == HIP_SUCCESS) - { - event_log_error (hashcat_ctx, "hipMemcpyDtoD(): %s", pStr); - } - else - { - event_log_error (hashcat_ctx, "hipMemcpyDtoD(): %d", HIP_err); - } - - return -1; - } - - return 0; -} - int hc_hipMemcpyDtoDAsync (hashcat_ctx_t *hashcat_ctx, HIPdeviceptr dstDevice, HIPdeviceptr srcDevice, size_t ByteCount, HIPstream hStream) { backend_ctx_t *backend_ctx = hashcat_ctx->backend_ctx; @@ -3089,33 +2948,6 @@ int hc_hipMemcpyDtoDAsync (hashcat_ctx_t *hashcat_ctx, HIPdeviceptr dstDevice, H return 0; } -int hc_hipMemcpyHtoD (hashcat_ctx_t *hashcat_ctx, HIPdeviceptr dstDevice, const void *srcHost, size_t ByteCount) -{ - backend_ctx_t *backend_ctx = hashcat_ctx->backend_ctx; - - HIP_PTR *hip = (HIP_PTR *) backend_ctx->hip; - - const HIPresult HIP_err = hip->hipMemcpyHtoD (dstDevice, srcHost, ByteCount); - - if (HIP_err != HIP_SUCCESS) - { - const char *pStr = NULL; - - if (hip->hipGetErrorString (HIP_err, &pStr) == HIP_SUCCESS) - { - event_log_error (hashcat_ctx, "hipMemcpyHtoD(): %s", pStr); - } - else - { - event_log_error (hashcat_ctx, "hipMemcpyHtoD(): %d", HIP_err); - } - - return -1; - } - - return 0; -} - int hc_hipMemcpyHtoDAsync (hashcat_ctx_t *hashcat_ctx, HIPdeviceptr dstDevice, const void *srcHost, size_t ByteCount, HIPstream hStream) { backend_ctx_t *backend_ctx = hashcat_ctx->backend_ctx; From 43a3622a7711e398c7bf561724ef4f9796ca481b Mon Sep 17 00:00:00 2001 From: Jukka Ojanen Date: Mon, 26 Jul 2021 20:57:24 +0300 Subject: [PATCH 10/33] Add necessary synchronization to check_hash() and check_cracked() --- src/hashes.c | 274 +++++++++++++++++++++++++++------------------------ 1 file changed, 147 insertions(+), 127 deletions(-) diff --git a/src/hashes.c b/src/hashes.c index 0355a50ad..86616afad 100644 --- a/src/hashes.c +++ b/src/hashes.c @@ -313,6 +313,8 @@ void check_hash (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param, pl void *tmps = NULL; + cl_event opencl_event; + if (hashconfig->opts_type & OPTS_TYPE_COPY_TMPS) { tmps = hcmalloc (hashconfig->tmp_size); @@ -329,7 +331,7 @@ void check_hash (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param, pl if (device_param->is_opencl == true) { - hc_clEnqueueReadBuffer (hashcat_ctx, device_param->opencl_command_queue, device_param->opencl_d_tmps, CL_FALSE, plain->gidvid * hashconfig->tmp_size, hashconfig->tmp_size, tmps, 0, NULL, NULL); + hc_clEnqueueReadBuffer (hashcat_ctx, device_param->opencl_command_queue, device_param->opencl_d_tmps, CL_FALSE, plain->gidvid * hashconfig->tmp_size, hashconfig->tmp_size, tmps, 0, NULL, &opencl_event); } } @@ -343,9 +345,7 @@ void check_hash (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param, pl // plain - u8 plain_buf[0x1000]; // while the password itself can have only length 256, the module could encode it with something like base64 which inflates the requires buffer size - - memset (plain_buf, 0, sizeof (plain_buf)); + u8 plain_buf[HCBUFSIZ_TINY] = { 0 }; // while the password itself can have only length 256, the module could encode it with something like base64 which inflates the requires buffer size u8 *plain_ptr = plain_buf; @@ -355,9 +355,25 @@ void check_hash (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param, pl if (module_ctx->module_build_plain_postprocess != MODULE_DEFAULT) { - u8 temp_buf[0x1000]; + u8 temp_buf[HCBUFSIZ_TINY] = { 0 }; - memset (temp_buf, 0, sizeof (temp_buf)); + if (hashconfig->opts_type & OPTS_TYPE_COPY_TMPS) + { + if (device_param->is_cuda == true) + { + if (hc_cuStreamSynchronize (hashcat_ctx, device_param->cuda_stream) == -1) return; + } + + if (device_param->is_hip == true) + { + if (hc_hipStreamSynchronize (hashcat_ctx, device_param->hip_stream) == -1) return; + } + + if (device_param->is_opencl == true) + { + if (hc_clWaitForEvents (hashcat_ctx, 1, &opencl_event) == -1) return; + } + } const int temp_len = module_ctx->module_build_plain_postprocess (hashcat_ctx->hashconfig, hashcat_ctx->hashes, tmps, (u32 *) plain_buf, sizeof (plain_buf), plain_len, (u32 *)temp_buf, sizeof (temp_buf)); @@ -407,6 +423,24 @@ void check_hash (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param, pl if (module_ctx->module_hash_encode_potfile != MODULE_DEFAULT) { + if (hashconfig->opts_type & OPTS_TYPE_COPY_TMPS) + { + if (device_param->is_cuda == true) + { + if (hc_cuStreamSynchronize (hashcat_ctx, device_param->cuda_stream) == -1) return; + } + + if (device_param->is_hip == true) + { + if (hc_hipStreamSynchronize (hashcat_ctx, device_param->hip_stream) == -1) return; + } + + if (device_param->is_opencl == true) + { + if (hc_clWaitForEvents (hashcat_ctx, 1, &opencl_event) == -1) return; + } + } + salt_t *salts_buf = hashes->salts_buf; salts_buf += salt_pos; @@ -470,6 +504,11 @@ void check_hash (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param, pl if (hashconfig->opts_type & OPTS_TYPE_COPY_TMPS) { + if (device_param->is_opencl == true) + { + if (hc_clReleaseEvent (hashcat_ctx, opencl_event) == -1) return; + } + hcfree (tmps); } } @@ -485,172 +524,153 @@ int check_cracked (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param) u32 num_cracked = 0; - int CU_rc; - int HIP_rc; - int CL_rc; - if (device_param->is_cuda == true) { - CU_rc = hc_cuMemcpyDtoHAsync (hashcat_ctx, &num_cracked, device_param->cuda_d_result, sizeof (u32), device_param->cuda_stream); + if (hc_cuMemcpyDtoHAsync (hashcat_ctx, &num_cracked, device_param->cuda_d_result, sizeof (u32), device_param->cuda_stream) == -1) return -1; - if (CU_rc == -1) return -1; + if (hc_cuStreamSynchronize (hashcat_ctx, device_param->cuda_stream) == -1) return -1; } if (device_param->is_hip == true) { - HIP_rc = hc_hipMemcpyDtoHAsync (hashcat_ctx, &num_cracked, device_param->hip_d_result, sizeof (u32), device_param->hip_stream); + if (hc_hipMemcpyDtoHAsync (hashcat_ctx, &num_cracked, device_param->hip_d_result, sizeof (u32), device_param->hip_stream) == -1) return -1; - if (HIP_rc == -1) return -1; + if (hc_hipStreamSynchronize (hashcat_ctx, device_param->hip_stream) == -1) return -1; } if (device_param->is_opencl == true) { - CL_rc = hc_clEnqueueReadBuffer (hashcat_ctx, device_param->opencl_command_queue, device_param->opencl_d_result, CL_TRUE, 0, sizeof (u32), &num_cracked, 0, NULL, NULL); - - if (CL_rc == -1) return -1; + if (hc_clEnqueueReadBuffer (hashcat_ctx, device_param->opencl_command_queue, device_param->opencl_d_result, CL_TRUE, 0, sizeof (u32), &num_cracked, 0, NULL, NULL) == -1) return -1; } - if (user_options->speed_only == true) + if (num_cracked == 0 || user_options->speed_only == true) { - // we want the hc_clEnqueueReadBuffer to run in benchmark mode because it has an influence in performance + // we want get num_cracked in benchmark mode because it has an influence in performance // however if the benchmark cracks the artificial hash used for benchmarks we don't want to see that! return 0; } - if (num_cracked) + plain_t *cracked = (plain_t *) hccalloc (num_cracked, sizeof (plain_t)); + + if (device_param->is_cuda == true) { - plain_t *cracked = (plain_t *) hccalloc (num_cracked, sizeof (plain_t)); + if (hc_cuMemcpyDtoHAsync (hashcat_ctx, cracked, device_param->cuda_d_plain_bufs, num_cracked * sizeof (plain_t), device_param->cuda_stream) == -1) return -1; - if (device_param->is_cuda == true) + if (hc_cuStreamSynchronize (hashcat_ctx, device_param->cuda_stream) == -1) return -1; + } + + if (device_param->is_hip == true) + { + if (hc_hipMemcpyDtoHAsync (hashcat_ctx, cracked, device_param->hip_d_plain_bufs, num_cracked * sizeof (plain_t), device_param->hip_stream) == -1) return -1; + + if (hc_hipStreamSynchronize (hashcat_ctx, device_param->hip_stream) == -1) return -1; + } + + if (device_param->is_opencl == true) + { + if (hc_clEnqueueReadBuffer (hashcat_ctx, device_param->opencl_command_queue, device_param->opencl_d_plain_bufs, CL_TRUE, 0, num_cracked * sizeof (plain_t), cracked, 0, NULL, NULL) == -1) return -1; + } + + u32 cpt_cracked = 0; + + hc_thread_mutex_lock (status_ctx->mux_display); + + for (u32 i = 0; i < num_cracked; i++) + { + const u32 hash_pos = cracked[i].hash_pos; + + if (hashes->digests_shown[hash_pos] == 1) continue; + + const u32 salt_pos = cracked[i].salt_pos; + salt_t *salt_buf = &hashes->salts_buf[salt_pos]; + + if ((hashconfig->opts_type & OPTS_TYPE_PT_NEVERCRACK) == 0) { - CU_rc = hc_cuMemcpyDtoHAsync (hashcat_ctx, cracked, device_param->cuda_d_plain_bufs, num_cracked * sizeof (plain_t), device_param->cuda_stream); + hashes->digests_shown[hash_pos] = 1; - if (CU_rc == -1) return -1; + hashes->digests_done++; + + cpt_cracked++; + + salt_buf->digests_done++; + + if (salt_buf->digests_done == salt_buf->digests_cnt) + { + hashes->salts_shown[salt_pos] = 1; + + hashes->salts_done++; + } } - if (device_param->is_hip == true) + if (hashes->salts_done == hashes->salts_cnt) mycracked (hashcat_ctx); + + check_hash (hashcat_ctx, device_param, &cracked[i]); + + if (hashconfig->opts_type & OPTS_TYPE_PT_NEVERCRACK) { - HIP_rc = hc_hipMemcpyDtoHAsync (hashcat_ctx, cracked, device_param->hip_d_plain_bufs, num_cracked * sizeof (plain_t), device_param->hip_stream); + // we need to reset cracked state on the device + // otherwise host thinks again and again the hash was cracked + // and returns invalid password each time - if (HIP_rc == -1) return -1; + memset (hashes->digests_shown_tmp, 0, salt_buf->digests_cnt * sizeof (u32)); + + if (device_param->is_cuda == true) + { + if (hc_cuMemcpyHtoDAsync (hashcat_ctx, device_param->cuda_d_digests_shown + (salt_buf->digests_offset * sizeof (u32)), &hashes->digests_shown_tmp[salt_buf->digests_offset], salt_buf->digests_cnt * sizeof (u32), device_param->cuda_stream) == -1) return -1; + } + + if (device_param->is_hip == true) + { + if (hc_hipMemcpyHtoDAsync (hashcat_ctx, device_param->hip_d_digests_shown + (salt_buf->digests_offset * sizeof (u32)), &hashes->digests_shown_tmp[salt_buf->digests_offset], salt_buf->digests_cnt * sizeof (u32), device_param->hip_stream) == -1) return -1; + } + + if (device_param->is_opencl == true) + { + if (hc_clEnqueueWriteBuffer (hashcat_ctx, device_param->opencl_command_queue, device_param->opencl_d_digests_shown, CL_FALSE, salt_buf->digests_offset * sizeof (u32), salt_buf->digests_cnt * sizeof (u32), &hashes->digests_shown_tmp[salt_buf->digests_offset], 0, NULL, NULL) == -1) return -1; + } } + } - if (device_param->is_opencl == true) - { - CL_rc = hc_clEnqueueReadBuffer (hashcat_ctx, device_param->opencl_command_queue, device_param->opencl_d_plain_bufs, CL_TRUE, 0, num_cracked * sizeof (plain_t), cracked, 0, NULL, NULL); + hc_thread_mutex_unlock (status_ctx->mux_display); - if (CL_rc == -1) return -1; - } - - u32 cpt_cracked = 0; + hcfree (cracked); + if (cpt_cracked > 0) + { hc_thread_mutex_lock (status_ctx->mux_display); - for (u32 i = 0; i < num_cracked; i++) - { - const u32 hash_pos = cracked[i].hash_pos; + cpt_ctx->cpt_buf[cpt_ctx->cpt_pos].timestamp = time (NULL); + cpt_ctx->cpt_buf[cpt_ctx->cpt_pos].cracked = cpt_cracked; - if (hashes->digests_shown[hash_pos] == 1) continue; + cpt_ctx->cpt_pos++; - const u32 salt_pos = cracked[i].salt_pos; - salt_t *salt_buf = &hashes->salts_buf[salt_pos]; + cpt_ctx->cpt_total += cpt_cracked; - if ((hashconfig->opts_type & OPTS_TYPE_PT_NEVERCRACK) == 0) - { - hashes->digests_shown[hash_pos] = 1; - - hashes->digests_done++; - - cpt_cracked++; - - salt_buf->digests_done++; - - if (salt_buf->digests_done == salt_buf->digests_cnt) - { - hashes->salts_shown[salt_pos] = 1; - - hashes->salts_done++; - } - } - - if (hashes->salts_done == hashes->salts_cnt) mycracked (hashcat_ctx); - - check_hash (hashcat_ctx, device_param, &cracked[i]); - - if (hashconfig->opts_type & OPTS_TYPE_PT_NEVERCRACK) - { - // we need to reset cracked state on the device - // otherwise host thinks again and again the hash was cracked - // and returns invalid password each time - - memset (hashes->digests_shown_tmp, 0, salt_buf->digests_cnt * sizeof (u32)); - - if (device_param->is_cuda == true) - { - CU_rc = hc_cuMemcpyHtoDAsync (hashcat_ctx, device_param->cuda_d_digests_shown + (salt_buf->digests_offset * sizeof (u32)), &hashes->digests_shown_tmp[salt_buf->digests_offset], salt_buf->digests_cnt * sizeof (u32), device_param->cuda_stream); - - if (CU_rc == -1) return -1; - } - - if (device_param->is_hip == true) - { - HIP_rc = hc_hipMemcpyHtoDAsync (hashcat_ctx, device_param->hip_d_digests_shown + (salt_buf->digests_offset * sizeof (u32)), &hashes->digests_shown_tmp[salt_buf->digests_offset], salt_buf->digests_cnt * sizeof (u32), device_param->hip_stream); - - if (HIP_rc == -1) return -1; - } - - if (device_param->is_opencl == true) - { - CL_rc = hc_clEnqueueWriteBuffer (hashcat_ctx, device_param->opencl_command_queue, device_param->opencl_d_digests_shown, CL_FALSE, salt_buf->digests_offset * sizeof (u32), salt_buf->digests_cnt * sizeof (u32), &hashes->digests_shown_tmp[salt_buf->digests_offset], 0, NULL, NULL); - - if (CL_rc == -1) return -1; - } - } - } + if (cpt_ctx->cpt_pos == CPT_CACHE) cpt_ctx->cpt_pos = 0; hc_thread_mutex_unlock (status_ctx->mux_display); + } - hcfree (cracked); + num_cracked = 0; - if (cpt_cracked > 0) - { - hc_thread_mutex_lock (status_ctx->mux_display); + if (device_param->is_cuda == true) + { + if (hc_cuMemcpyHtoDAsync (hashcat_ctx, device_param->cuda_d_result, &num_cracked, sizeof (u32), device_param->cuda_stream) == -1) return -1; - cpt_ctx->cpt_buf[cpt_ctx->cpt_pos].timestamp = time (NULL); - cpt_ctx->cpt_buf[cpt_ctx->cpt_pos].cracked = cpt_cracked; + if (hc_cuStreamSynchronize (hashcat_ctx, device_param->cuda_stream) == -1) return -1; + } - cpt_ctx->cpt_pos++; + if (device_param->is_hip == true) + { + if (hc_hipMemcpyHtoDAsync (hashcat_ctx, device_param->hip_d_result, &num_cracked, sizeof (u32), device_param->hip_stream) == -1) return -1; - cpt_ctx->cpt_total += cpt_cracked; + if (hc_hipStreamSynchronize (hashcat_ctx, device_param->hip_stream) == -1) return -1; + } - if (cpt_ctx->cpt_pos == CPT_CACHE) cpt_ctx->cpt_pos = 0; - - hc_thread_mutex_unlock (status_ctx->mux_display); - } - - num_cracked = 0; - - if (device_param->is_cuda == true) - { - CU_rc = hc_cuMemcpyHtoDAsync (hashcat_ctx, device_param->cuda_d_result, &num_cracked, sizeof (u32), device_param->cuda_stream); - - if (CU_rc == -1) return -1; - } - - if (device_param->is_hip == true) - { - HIP_rc = hc_hipMemcpyHtoDAsync (hashcat_ctx, device_param->hip_d_result, &num_cracked, sizeof (u32), device_param->hip_stream); - - if (HIP_rc == -1) return -1; - } - - if (device_param->is_opencl == true) - { - CL_rc = hc_clEnqueueWriteBuffer (hashcat_ctx, device_param->opencl_command_queue, device_param->opencl_d_result, CL_FALSE, 0, sizeof (u32), &num_cracked, 0, NULL, NULL); - - if (CL_rc == -1) return -1; - } + if (device_param->is_opencl == true) + { + if (hc_clEnqueueWriteBuffer (hashcat_ctx, device_param->opencl_command_queue, device_param->opencl_d_result, CL_TRUE, 0, sizeof (u32), &num_cracked, 0, NULL, NULL) == -1) return -1; } return 0; From 73b8cda1764353ece4699708ed8fa32ec0632563 Mon Sep 17 00:00:00 2001 From: Jukka Ojanen Date: Mon, 26 Jul 2021 22:25:50 +0300 Subject: [PATCH 11/33] Avoid unnecessary memcpy in check_hash() --- src/hashes.c | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/src/hashes.c b/src/hashes.c index 86616afad..946f3b1f6 100644 --- a/src/hashes.c +++ b/src/hashes.c @@ -346,6 +346,7 @@ void check_hash (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param, pl // plain u8 plain_buf[HCBUFSIZ_TINY] = { 0 }; // while the password itself can have only length 256, the module could encode it with something like base64 which inflates the requires buffer size + u8 postprocess_buf[HCBUFSIZ_TINY] = { 0 }; u8 *plain_ptr = plain_buf; @@ -355,8 +356,6 @@ void check_hash (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param, pl if (module_ctx->module_build_plain_postprocess != MODULE_DEFAULT) { - u8 temp_buf[HCBUFSIZ_TINY] = { 0 }; - if (hashconfig->opts_type & OPTS_TYPE_COPY_TMPS) { if (device_param->is_cuda == true) @@ -375,14 +374,9 @@ void check_hash (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param, pl } } - const int temp_len = module_ctx->module_build_plain_postprocess (hashcat_ctx->hashconfig, hashcat_ctx->hashes, tmps, (u32 *) plain_buf, sizeof (plain_buf), plain_len, (u32 *)temp_buf, sizeof (temp_buf)); + plain_len = module_ctx->module_build_plain_postprocess (hashcat_ctx->hashconfig, hashcat_ctx->hashes, tmps, (u32 *) plain_buf, sizeof (plain_buf), plain_len, (u32 *) postprocess_buf, sizeof (postprocess_buf)); - if (temp_len < (int) sizeof (plain_buf)) - { - memcpy (plain_buf, temp_buf, temp_len); - - plain_len = temp_len; - } + plain_ptr = postprocess_buf; } // crackpos From d7de3550b19b808a853ef45021fa24e2a947db49 Mon Sep 17 00:00:00 2001 From: Jukka Ojanen Date: Tue, 27 Jul 2021 12:57:26 +0300 Subject: [PATCH 12/33] Add fast event for CUDA and HIP, and use it to synchronize check_hash() --- include/types.h | 2 ++ src/backend.c | 16 ++++++++++++++++ src/hashes.c | 36 ++++++++++++++++++------------------ 3 files changed, 36 insertions(+), 18 deletions(-) diff --git a/include/types.h b/include/types.h index 43e5a55da..93cb42bf4 100644 --- a/include/types.h +++ b/include/types.h @@ -1428,6 +1428,7 @@ typedef struct hc_device_param CUevent cuda_event1; CUevent cuda_event2; + CUevent cuda_event3; CUmodule cuda_module; CUmodule cuda_module_shared; @@ -1509,6 +1510,7 @@ typedef struct hc_device_param HIPevent hip_event1; HIPevent hip_event2; + HIPevent hip_event3; HIPmodule hip_module; HIPmodule hip_module_shared; diff --git a/src/backend.c b/src/backend.c index a6f42b672..73edce1ce 100644 --- a/src/backend.c +++ b/src/backend.c @@ -11360,6 +11360,12 @@ int backend_session_begin (hashcat_ctx_t *hashcat_ctx) device_param->skipped = true; continue; } + + if (hc_cuEventCreate (hashcat_ctx, &device_param->cuda_event3, CU_EVENT_DISABLE_TIMING) == -1) + { + device_param->skipped = true; + continue; + } } /** @@ -11379,6 +11385,12 @@ int backend_session_begin (hashcat_ctx_t *hashcat_ctx) device_param->skipped = true; continue; } + + if (hc_hipEventCreate (hashcat_ctx, &device_param->hip_event3, HIP_EVENT_DISABLE_TIMING) == -1) + { + device_param->skipped = true; + continue; + } } /** @@ -15214,6 +15226,7 @@ void backend_session_destroy (hashcat_ctx_t *hashcat_ctx) if (device_param->cuda_event1) hc_cuEventDestroy (hashcat_ctx, device_param->cuda_event1); if (device_param->cuda_event2) hc_cuEventDestroy (hashcat_ctx, device_param->cuda_event2); + if (device_param->cuda_event3) hc_cuEventDestroy (hashcat_ctx, device_param->cuda_event3); if (device_param->cuda_stream) hc_cuStreamDestroy (hashcat_ctx, device_param->cuda_stream); @@ -15289,6 +15302,7 @@ void backend_session_destroy (hashcat_ctx_t *hashcat_ctx) device_param->cuda_event1 = NULL; device_param->cuda_event2 = NULL; + device_param->cuda_event3 = NULL; device_param->cuda_stream = NULL; @@ -15341,6 +15355,7 @@ void backend_session_destroy (hashcat_ctx_t *hashcat_ctx) if (device_param->hip_event1) hc_hipEventDestroy (hashcat_ctx, device_param->hip_event1); if (device_param->hip_event2) hc_hipEventDestroy (hashcat_ctx, device_param->hip_event2); + if (device_param->hip_event3) hc_hipEventDestroy (hashcat_ctx, device_param->hip_event3); if (device_param->hip_stream) hc_hipStreamDestroy (hashcat_ctx, device_param->hip_stream); @@ -15416,6 +15431,7 @@ void backend_session_destroy (hashcat_ctx_t *hashcat_ctx) device_param->hip_event1 = NULL; device_param->hip_event2 = NULL; + device_param->hip_event3 = NULL; device_param->hip_stream = NULL; diff --git a/src/hashes.c b/src/hashes.c index 946f3b1f6..aa0d5e234 100644 --- a/src/hashes.c +++ b/src/hashes.c @@ -322,16 +322,22 @@ void check_hash (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param, pl if (device_param->is_cuda == true) { hc_cuMemcpyDtoHAsync (hashcat_ctx, tmps, device_param->cuda_d_tmps + (plain->gidvid * hashconfig->tmp_size), hashconfig->tmp_size, device_param->cuda_stream); + + hc_cuEventRecord (hashcat_ctx, device_param->cuda_event3, device_param->cuda_stream); } if (device_param->is_hip == true) { hc_hipMemcpyDtoHAsync (hashcat_ctx, tmps, device_param->hip_d_tmps + (plain->gidvid * hashconfig->tmp_size), hashconfig->tmp_size, device_param->hip_stream); + + hc_hipEventRecord (hashcat_ctx, device_param->hip_event3, device_param->hip_stream); } if (device_param->is_opencl == true) { hc_clEnqueueReadBuffer (hashcat_ctx, device_param->opencl_command_queue, device_param->opencl_d_tmps, CL_FALSE, plain->gidvid * hashconfig->tmp_size, hashconfig->tmp_size, tmps, 0, NULL, &opencl_event); + + hc_clFlush (hashcat_ctx, device_param->opencl_command_queue); } } @@ -360,12 +366,12 @@ void check_hash (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param, pl { if (device_param->is_cuda == true) { - if (hc_cuStreamSynchronize (hashcat_ctx, device_param->cuda_stream) == -1) return; + if (hc_cuEventSynchronize (hashcat_ctx, device_param->cuda_event3) == -1) return; } if (device_param->is_hip == true) { - if (hc_hipStreamSynchronize (hashcat_ctx, device_param->hip_stream) == -1) return; + if (hc_hipEventSynchronize (hashcat_ctx, device_param->hip_event3) == -1) return; } if (device_param->is_opencl == true) @@ -421,12 +427,12 @@ void check_hash (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param, pl { if (device_param->is_cuda == true) { - if (hc_cuStreamSynchronize (hashcat_ctx, device_param->cuda_stream) == -1) return; + if (hc_cuEventSynchronize (hashcat_ctx, device_param->cuda_event3) == -1) return; } if (device_param->is_hip == true) { - if (hc_hipStreamSynchronize (hashcat_ctx, device_param->hip_stream) == -1) return; + if (hc_hipEventSynchronize (hashcat_ctx, device_param->hip_event3) == -1) return; } if (device_param->is_opencl == true) @@ -607,21 +613,19 @@ int check_cracked (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param) // otherwise host thinks again and again the hash was cracked // and returns invalid password each time - memset (hashes->digests_shown_tmp, 0, salt_buf->digests_cnt * sizeof (u32)); - if (device_param->is_cuda == true) { - if (hc_cuMemcpyHtoDAsync (hashcat_ctx, device_param->cuda_d_digests_shown + (salt_buf->digests_offset * sizeof (u32)), &hashes->digests_shown_tmp[salt_buf->digests_offset], salt_buf->digests_cnt * sizeof (u32), device_param->cuda_stream) == -1) return -1; + if (hc_cuMemcpyHtoDAsync (hashcat_ctx, device_param->cuda_d_digests_shown + (salt_buf->digests_offset * sizeof (u32)), hashes->digests_shown_tmp, salt_buf->digests_cnt * sizeof (u32), device_param->cuda_stream) == -1) return -1; } if (device_param->is_hip == true) { - if (hc_hipMemcpyHtoDAsync (hashcat_ctx, device_param->hip_d_digests_shown + (salt_buf->digests_offset * sizeof (u32)), &hashes->digests_shown_tmp[salt_buf->digests_offset], salt_buf->digests_cnt * sizeof (u32), device_param->hip_stream) == -1) return -1; + if (hc_hipMemcpyHtoDAsync (hashcat_ctx, device_param->hip_d_digests_shown + (salt_buf->digests_offset * sizeof (u32)), hashes->digests_shown_tmp, salt_buf->digests_cnt * sizeof (u32), device_param->hip_stream) == -1) return -1; } if (device_param->is_opencl == true) { - if (hc_clEnqueueWriteBuffer (hashcat_ctx, device_param->opencl_command_queue, device_param->opencl_d_digests_shown, CL_FALSE, salt_buf->digests_offset * sizeof (u32), salt_buf->digests_cnt * sizeof (u32), &hashes->digests_shown_tmp[salt_buf->digests_offset], 0, NULL, NULL) == -1) return -1; + if (hc_clEnqueueWriteBuffer (hashcat_ctx, device_param->opencl_command_queue, device_param->opencl_d_digests_shown, CL_FALSE, salt_buf->digests_offset * sizeof (u32), salt_buf->digests_cnt * sizeof (u32), hashes->digests_shown_tmp, 0, NULL, NULL) == -1) return -1; } } } @@ -646,25 +650,21 @@ int check_cracked (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param) hc_thread_mutex_unlock (status_ctx->mux_display); } - num_cracked = 0; - if (device_param->is_cuda == true) { - if (hc_cuMemcpyHtoDAsync (hashcat_ctx, device_param->cuda_d_result, &num_cracked, sizeof (u32), device_param->cuda_stream) == -1) return -1; - - if (hc_cuStreamSynchronize (hashcat_ctx, device_param->cuda_stream) == -1) return -1; + if (run_cuda_kernel_bzero (hashcat_ctx, device_param, device_param->cuda_d_result, sizeof (u32)) == -1) return -1; } if (device_param->is_hip == true) { - if (hc_hipMemcpyHtoDAsync (hashcat_ctx, device_param->hip_d_result, &num_cracked, sizeof (u32), device_param->hip_stream) == -1) return -1; - - if (hc_hipStreamSynchronize (hashcat_ctx, device_param->hip_stream) == -1) return -1; + if (run_hip_kernel_bzero (hashcat_ctx, device_param, device_param->hip_d_result, sizeof (u32)) == -1) return -1; } if (device_param->is_opencl == true) { - if (hc_clEnqueueWriteBuffer (hashcat_ctx, device_param->opencl_command_queue, device_param->opencl_d_result, CL_TRUE, 0, sizeof (u32), &num_cracked, 0, NULL, NULL) == -1) return -1; + if (run_opencl_kernel_bzero (hashcat_ctx, device_param, device_param->opencl_d_result, sizeof (u32)) == -1) return -1; + + if (hc_clFlush (hashcat_ctx, device_param->opencl_command_queue) == -1) return -1; } return 0; From ae44a3022b383276ea12775646ad88c2422b8870 Mon Sep 17 00:00:00 2001 From: Jukka Ojanen Date: Tue, 27 Jul 2021 13:00:39 +0300 Subject: [PATCH 13/33] Use fast event to synchronize selftest() --- src/selftest.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/selftest.c b/src/selftest.c index 77ad8f23b..d5fff9433 100644 --- a/src/selftest.c +++ b/src/selftest.c @@ -695,14 +695,14 @@ static int selftest (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param { if (hc_cuMemcpyDtoHAsync (hashcat_ctx, &num_cracked, device_param->cuda_d_result, sizeof (u32), device_param->cuda_stream) == -1) return -1; - if (hc_cuEventRecord (hashcat_ctx, device_param->cuda_event1, device_param->cuda_stream) == -1) return -1; + if (hc_cuEventRecord (hashcat_ctx, device_param->cuda_event3, device_param->cuda_stream) == -1) return -1; } if (device_param->is_hip == true) { if (hc_hipMemcpyDtoHAsync (hashcat_ctx, &num_cracked, device_param->hip_d_result, sizeof (u32), device_param->hip_stream) == -1) return -1; - if (hc_hipEventRecord (hashcat_ctx, device_param->hip_event1, device_param->hip_stream) == -1) return -1; + if (hc_hipEventRecord (hashcat_ctx, device_param->hip_event3, device_param->hip_stream) == -1) return -1; } if (device_param->is_opencl == true) @@ -837,12 +837,12 @@ static int selftest (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param // synchronize and .. if (device_param->is_cuda == true) { - if (hc_cuEventSynchronize (hashcat_ctx, device_param->cuda_event1) == -1) return -1; + if (hc_cuEventSynchronize (hashcat_ctx, device_param->cuda_event3) == -1) return -1; } if (device_param->is_hip == true) { - if (hc_hipEventSynchronize (hashcat_ctx, device_param->hip_event1) == -1) return -1; + if (hc_hipEventSynchronize (hashcat_ctx, device_param->hip_event3) == -1) return -1; } if (device_param->is_opencl == true) From be33ccaa6c750897706abccf9a6067edd8b64b6e Mon Sep 17 00:00:00 2001 From: Jukka Ojanen Date: Tue, 27 Jul 2021 14:36:48 +0300 Subject: [PATCH 14/33] check_hash() needs to return value to indicate possible error --- include/hashes.h | 2 +- src/hashes.c | 70 +++++++++++++++++++++++++++++++++++------------- 2 files changed, 53 insertions(+), 19 deletions(-) diff --git a/include/hashes.h b/include/hashes.h index ca88cab13..1e043aad4 100644 --- a/include/hashes.h +++ b/include/hashes.h @@ -15,7 +15,7 @@ int hash_encode (const hashconfig_t *hashconfig, const hashes_t *hashes, const m int save_hash (hashcat_ctx_t *hashcat_ctx); -void check_hash (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param, plain_t *plain); +int check_hash (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param, plain_t *plain); //int check_cracked (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param, const u32 salt_pos); int check_cracked (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param); diff --git a/src/hashes.c b/src/hashes.c index aa0d5e234..b143ed406 100644 --- a/src/hashes.c +++ b/src/hashes.c @@ -300,7 +300,7 @@ int save_hash (hashcat_ctx_t *hashcat_ctx) return 0; } -void check_hash (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param, plain_t *plain) +int check_hash (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param, plain_t *plain) { const debugfile_ctx_t *debugfile_ctx = hashcat_ctx->debugfile_ctx; const hashes_t *hashes = hashcat_ctx->hashes; @@ -314,6 +314,7 @@ void check_hash (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param, pl void *tmps = NULL; cl_event opencl_event; + int rc; if (hashconfig->opts_type & OPTS_TYPE_COPY_TMPS) { @@ -321,23 +322,53 @@ void check_hash (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param, pl if (device_param->is_cuda == true) { - hc_cuMemcpyDtoHAsync (hashcat_ctx, tmps, device_param->cuda_d_tmps + (plain->gidvid * hashconfig->tmp_size), hashconfig->tmp_size, device_param->cuda_stream); + rc = hc_cuMemcpyDtoHAsync (hashcat_ctx, tmps, device_param->cuda_d_tmps + (plain->gidvid * hashconfig->tmp_size), hashconfig->tmp_size, device_param->cuda_stream); - hc_cuEventRecord (hashcat_ctx, device_param->cuda_event3, device_param->cuda_stream); + if (rc == 0) + { + rc = hc_cuEventRecord (hashcat_ctx, device_param->cuda_event3, device_param->cuda_stream); + } + + if (rc == -1) + { + free(tmps); + + return -1; + } } if (device_param->is_hip == true) { - hc_hipMemcpyDtoHAsync (hashcat_ctx, tmps, device_param->hip_d_tmps + (plain->gidvid * hashconfig->tmp_size), hashconfig->tmp_size, device_param->hip_stream); + rc = hc_hipMemcpyDtoHAsync (hashcat_ctx, tmps, device_param->hip_d_tmps + (plain->gidvid * hashconfig->tmp_size), hashconfig->tmp_size, device_param->hip_stream); - hc_hipEventRecord (hashcat_ctx, device_param->hip_event3, device_param->hip_stream); + if (rc == 0) + { + rc = hc_hipEventRecord (hashcat_ctx, device_param->hip_event3, device_param->hip_stream); + } + + if (rc == -1) + { + free(tmps); + + return -1; + } } if (device_param->is_opencl == true) { - hc_clEnqueueReadBuffer (hashcat_ctx, device_param->opencl_command_queue, device_param->opencl_d_tmps, CL_FALSE, plain->gidvid * hashconfig->tmp_size, hashconfig->tmp_size, tmps, 0, NULL, &opencl_event); + rc = hc_clEnqueueReadBuffer (hashcat_ctx, device_param->opencl_command_queue, device_param->opencl_d_tmps, CL_FALSE, plain->gidvid * hashconfig->tmp_size, hashconfig->tmp_size, tmps, 0, NULL, &opencl_event); - hc_clFlush (hashcat_ctx, device_param->opencl_command_queue); + if (rc == 0) + { + rc = hc_clFlush (hashcat_ctx, device_param->opencl_command_queue); + } + + if (rc == -1) + { + free(tmps); + + return -1; + } } } @@ -345,7 +376,7 @@ void check_hash (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param, pl u8 *out_buf = hashes->out_buf; - int out_len = hash_encode (hashcat_ctx->hashconfig, hashcat_ctx->hashes, hashcat_ctx->module_ctx, (char *) out_buf, HCBUFSIZ_LARGE, salt_pos, digest_pos); + int out_len = hash_encode (hashconfig, hashes, module_ctx, (char *) out_buf, HCBUFSIZ_LARGE, salt_pos, digest_pos); out_buf[out_len] = 0; @@ -366,21 +397,21 @@ void check_hash (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param, pl { if (device_param->is_cuda == true) { - if (hc_cuEventSynchronize (hashcat_ctx, device_param->cuda_event3) == -1) return; + if (hc_cuEventSynchronize (hashcat_ctx, device_param->cuda_event3) == -1) return -1; } if (device_param->is_hip == true) { - if (hc_hipEventSynchronize (hashcat_ctx, device_param->hip_event3) == -1) return; + if (hc_hipEventSynchronize (hashcat_ctx, device_param->hip_event3) == -1) return -1; } if (device_param->is_opencl == true) { - if (hc_clWaitForEvents (hashcat_ctx, 1, &opencl_event) == -1) return; + if (hc_clWaitForEvents (hashcat_ctx, 1, &opencl_event) == -1) return -1; } } - plain_len = module_ctx->module_build_plain_postprocess (hashcat_ctx->hashconfig, hashcat_ctx->hashes, tmps, (u32 *) plain_buf, sizeof (plain_buf), plain_len, (u32 *) postprocess_buf, sizeof (postprocess_buf)); + plain_len = module_ctx->module_build_plain_postprocess (hashconfig, hashes, tmps, (u32 *) plain_buf, sizeof (plain_buf), plain_len, (u32 *) postprocess_buf, sizeof (postprocess_buf)); plain_ptr = postprocess_buf; } @@ -427,17 +458,17 @@ void check_hash (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param, pl { if (device_param->is_cuda == true) { - if (hc_cuEventSynchronize (hashcat_ctx, device_param->cuda_event3) == -1) return; + if (hc_cuEventSynchronize (hashcat_ctx, device_param->cuda_event3) == -1) return -1; } if (device_param->is_hip == true) { - if (hc_hipEventSynchronize (hashcat_ctx, device_param->hip_event3) == -1) return; + if (hc_hipEventSynchronize (hashcat_ctx, device_param->hip_event3) == -1) return -1; } if (device_param->is_opencl == true) { - if (hc_clWaitForEvents (hashcat_ctx, 1, &opencl_event) == -1) return; + if (hc_clWaitForEvents (hashcat_ctx, 1, &opencl_event) == -1) return -1; } } @@ -504,13 +535,15 @@ void check_hash (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param, pl if (hashconfig->opts_type & OPTS_TYPE_COPY_TMPS) { + hcfree (tmps); + if (device_param->is_opencl == true) { - if (hc_clReleaseEvent (hashcat_ctx, opencl_event) == -1) return; + if (hc_clReleaseEvent (hashcat_ctx, opencl_event) == -1) return -1; } - - hcfree (tmps); } + + return 0; } //int check_cracked (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param, const u32 salt_pos) @@ -605,6 +638,7 @@ int check_cracked (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param) if (hashes->salts_done == hashes->salts_cnt) mycracked (hashcat_ctx); + /* TODO: handle check_hash return value */ check_hash (hashcat_ctx, device_param, &cracked[i]); if (hashconfig->opts_type & OPTS_TYPE_PT_NEVERCRACK) From e8be7028cdb048aa9e56ddad0c3d193c7d2df45c Mon Sep 17 00:00:00 2001 From: Jukka Ojanen Date: Tue, 27 Jul 2021 16:28:07 +0300 Subject: [PATCH 15/33] Handle errors inside check_cracked() --- src/hashes.c | 78 +++++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 68 insertions(+), 10 deletions(-) diff --git a/src/hashes.c b/src/hashes.c index b143ed406..5679a5442 100644 --- a/src/hashes.c +++ b/src/hashes.c @@ -556,6 +556,7 @@ int check_cracked (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param) user_options_t *user_options = hashcat_ctx->user_options; u32 num_cracked = 0; + int rc; if (device_param->is_cuda == true) { @@ -588,21 +589,48 @@ int check_cracked (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param) if (device_param->is_cuda == true) { - if (hc_cuMemcpyDtoHAsync (hashcat_ctx, cracked, device_param->cuda_d_plain_bufs, num_cracked * sizeof (plain_t), device_param->cuda_stream) == -1) return -1; + rc = hc_cuMemcpyDtoHAsync (hashcat_ctx, cracked, device_param->cuda_d_plain_bufs, num_cracked * sizeof (plain_t), device_param->cuda_stream); - if (hc_cuStreamSynchronize (hashcat_ctx, device_param->cuda_stream) == -1) return -1; + if (rc == 0) + { + rc = hc_cuStreamSynchronize (hashcat_ctx, device_param->cuda_stream); + } + + if (rc == -1) + { + hcfree (cracked); + + return -1; + } } if (device_param->is_hip == true) { - if (hc_hipMemcpyDtoHAsync (hashcat_ctx, cracked, device_param->hip_d_plain_bufs, num_cracked * sizeof (plain_t), device_param->hip_stream) == -1) return -1; + rc = hc_hipMemcpyDtoHAsync (hashcat_ctx, cracked, device_param->hip_d_plain_bufs, num_cracked * sizeof (plain_t), device_param->hip_stream); - if (hc_hipStreamSynchronize (hashcat_ctx, device_param->hip_stream) == -1) return -1; + if (rc == 0) + { + rc = hc_hipStreamSynchronize (hashcat_ctx, device_param->hip_stream); + } + + if (rc == -1) + { + hcfree (cracked); + + return -1; + } } if (device_param->is_opencl == true) { - if (hc_clEnqueueReadBuffer (hashcat_ctx, device_param->opencl_command_queue, device_param->opencl_d_plain_bufs, CL_TRUE, 0, num_cracked * sizeof (plain_t), cracked, 0, NULL, NULL) == -1) return -1; + rc = hc_clEnqueueReadBuffer (hashcat_ctx, device_param->opencl_command_queue, device_param->opencl_d_plain_bufs, CL_TRUE, 0, num_cracked * sizeof (plain_t), cracked, 0, NULL, NULL); + + if (rc == -1) + { + hcfree (cracked); + + return -1; + } } u32 cpt_cracked = 0; @@ -638,8 +666,12 @@ int check_cracked (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param) if (hashes->salts_done == hashes->salts_cnt) mycracked (hashcat_ctx); - /* TODO: handle check_hash return value */ - check_hash (hashcat_ctx, device_param, &cracked[i]); + rc = check_hash (hashcat_ctx, device_param, &cracked[i]); + + if (rc == -1) + { + break; + } if (hashconfig->opts_type & OPTS_TYPE_PT_NEVERCRACK) { @@ -647,19 +679,40 @@ int check_cracked (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param) // otherwise host thinks again and again the hash was cracked // and returns invalid password each time + /* TODO: this is bad design -- digests_shown_tmp is only used for zeroing */ if (device_param->is_cuda == true) { - if (hc_cuMemcpyHtoDAsync (hashcat_ctx, device_param->cuda_d_digests_shown + (salt_buf->digests_offset * sizeof (u32)), hashes->digests_shown_tmp, salt_buf->digests_cnt * sizeof (u32), device_param->cuda_stream) == -1) return -1; + rc = hc_cuMemcpyHtoDAsync (hashcat_ctx, device_param->cuda_d_digests_shown + (salt_buf->digests_offset * sizeof (u32)), hashes->digests_shown_tmp, salt_buf->digests_cnt * sizeof (u32), device_param->cuda_stream); + + if (rc == -1) + { + break; + } } if (device_param->is_hip == true) { - if (hc_hipMemcpyHtoDAsync (hashcat_ctx, device_param->hip_d_digests_shown + (salt_buf->digests_offset * sizeof (u32)), hashes->digests_shown_tmp, salt_buf->digests_cnt * sizeof (u32), device_param->hip_stream) == -1) return -1; + rc = hc_hipMemcpyHtoDAsync (hashcat_ctx, device_param->hip_d_digests_shown + (salt_buf->digests_offset * sizeof (u32)), hashes->digests_shown_tmp, salt_buf->digests_cnt * sizeof (u32), device_param->hip_stream); + + if (rc == -1) + { + break; + } } if (device_param->is_opencl == true) { - if (hc_clEnqueueWriteBuffer (hashcat_ctx, device_param->opencl_command_queue, device_param->opencl_d_digests_shown, CL_FALSE, salt_buf->digests_offset * sizeof (u32), salt_buf->digests_cnt * sizeof (u32), hashes->digests_shown_tmp, 0, NULL, NULL) == -1) return -1; + rc = hc_clEnqueueWriteBuffer (hashcat_ctx, device_param->opencl_command_queue, device_param->opencl_d_digests_shown, CL_FALSE, salt_buf->digests_offset * sizeof (u32), salt_buf->digests_cnt * sizeof (u32), hashes->digests_shown_tmp, 0, NULL, NULL); + + if (rc == 0) + { + rc = hc_clFlush (hashcat_ctx, device_param->opencl_command_queue); + } + + if (rc == -1) + { + break; + } } } } @@ -668,6 +721,11 @@ int check_cracked (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param) hcfree (cracked); + if (rc == -1) + { + return -1; + } + if (cpt_cracked > 0) { hc_thread_mutex_lock (status_ctx->mux_display); From cdf27a1cb351a026375d974836340a8e7a7c6a2e Mon Sep 17 00:00:00 2001 From: Jukka Ojanen Date: Tue, 27 Jul 2021 18:56:59 +0300 Subject: [PATCH 16/33] Implement async run_cuda_kernel_memset() and run_cuda_kernel_memset32() --- include/backend.h | 14 ++- include/ext_OpenCL.h | 2 + include/ext_cuda.h | 8 +- include/ext_hip.h | 8 +- src/backend.c | 284 ++++++++++++++++++++++++++++--------------- 5 files changed, 210 insertions(+), 106 deletions(-) diff --git a/include/backend.h b/include/backend.h index a54ba357e..c2c984b24 100644 --- a/include/backend.h +++ b/include/backend.h @@ -72,6 +72,8 @@ int hc_cuMemcpyDtoDAsync (hashcat_ctx_t *hashcat_ctx, CUdeviceptr dstDev int hc_cuMemcpyDtoHAsync (hashcat_ctx_t *hashcat_ctx, void *dstHost, CUdeviceptr srcDevice, size_t ByteCount, CUstream hStream); int hc_cuMemcpyHtoDAsync (hashcat_ctx_t *hashcat_ctx, CUdeviceptr dstDevice, const void *srcHost, size_t ByteCount, CUstream hStream); int hc_cuMemFree (hashcat_ctx_t *hashcat_ctx, CUdeviceptr dptr); +int hc_cuMemsetD32Async (hashcat_ctx_t *hashcat_ctx, CUdeviceptr dstDevice, unsigned int ui, size_t N, CUstream hStream); +int hc_cuMemsetD8Async (hashcat_ctx_t *hashcat_ctx, CUdeviceptr dstDevice, unsigned char uc, size_t N, CUstream hStream); int hc_cuModuleGetFunction (hashcat_ctx_t *hashcat_ctx, CUfunction *hfunc, CUmodule hmod, const char *name); int hc_cuModuleLoadDataEx (hashcat_ctx_t *hashcat_ctx, CUmodule *module, const void *image, unsigned int numOptions, CUjit_option *options, void **optionValues); int hc_cuModuleUnload (hashcat_ctx_t *hashcat_ctx, CUmodule hmod); @@ -120,6 +122,8 @@ int hc_hipMemcpyDtoDAsync (hashcat_ctx_t *hashcat_ctx, HIPdeviceptr dstDe int hc_hipMemcpyDtoHAsync (hashcat_ctx_t *hashcat_ctx, void *dstHost, HIPdeviceptr srcDevice, size_t ByteCount, HIPstream hStream); int hc_hipMemcpyHtoDAsync (hashcat_ctx_t *hashcat_ctx, HIPdeviceptr dstDevice, const void *srcHost, size_t ByteCount, HIPstream hStream); int hc_hipMemFree (hashcat_ctx_t *hashcat_ctx, HIPdeviceptr dptr); +int hc_hipMemsetD32Async (hashcat_ctx_t *hashcat_ctx, HIPdeviceptr dstDevice, unsigned int ui, size_t N, HIPstream hStream); +int hc_hipMemsetD8Async (hashcat_ctx_t *hashcat_ctx, HIPdeviceptr dstDevice, unsigned char uc, size_t N, HIPstream hStream); int hc_hipModuleGetFunction (hashcat_ctx_t *hashcat_ctx, HIPfunction *hfunc, HIPmodule hmod, const char *name); int hc_hipModuleLoadDataEx (hashcat_ctx_t *hashcat_ctx, HIPmodule *module, const void *image, unsigned int numOptions, HIPjit_option *options, void **optionValues); int hc_hipModuleUnload (hashcat_ctx_t *hashcat_ctx, HIPmodule hmod); @@ -142,6 +146,7 @@ int hc_clCreateKernel (hashcat_ctx_t *hashcat_ctx, cl_program program int hc_clCreateProgramWithBinary (hashcat_ctx_t *hashcat_ctx, cl_context context, cl_uint num_devices, const cl_device_id *device_list, const size_t *lengths, const unsigned char **binaries, cl_int *binary_status, cl_program *program); int hc_clCreateProgramWithSource (hashcat_ctx_t *hashcat_ctx, cl_context context, cl_uint count, const char **strings, const size_t *lengths, cl_program *program); int hc_clEnqueueCopyBuffer (hashcat_ctx_t *hashcat_ctx, cl_command_queue command_queue, cl_mem src_buffer, cl_mem dst_buffer, size_t src_offset, size_t dst_offset, size_t size, cl_uint num_events_in_wait_list, const cl_event *event_wait_list, cl_event *event); +int hc_clEnqueueFillBuffer (hashcat_ctx_t *hashcat_ctx, cl_command_queue command_queue, cl_mem buffer, const void *pattern, size_t pattern_size, size_t offset, size_t size, cl_uint num_events_in_wait_list, const cl_event *event_wait_list, cl_event *event); int hc_clEnqueueMapBuffer (hashcat_ctx_t *hashcat_ctx, cl_command_queue command_queue, cl_mem buffer, cl_bool blocking_map, cl_map_flags map_flags, size_t offset, size_t size, cl_uint num_events_in_wait_list, const cl_event *event_wait_list, cl_event *event, void **buf); int hc_clEnqueueNDRangeKernel (hashcat_ctx_t *hashcat_ctx, cl_command_queue command_queue, cl_kernel kernel, cl_uint work_dim, const size_t *global_work_offset, const size_t *global_work_size, const size_t *local_work_size, cl_uint num_events_in_wait_list, const cl_event *event_wait_list, cl_event *event); int hc_clEnqueueReadBuffer (hashcat_ctx_t *hashcat_ctx, cl_command_queue command_queue, cl_mem buffer, cl_bool blocking_read, size_t offset, size_t size, void *ptr, cl_uint num_events_in_wait_list, const cl_event *event_wait_list, cl_event *event); @@ -177,17 +182,20 @@ void rebuild_pws_compressed_append (hc_device_param_t *device_param, const u64 p int run_cuda_kernel_atinit (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param, CUdeviceptr buf, const u64 num); int run_cuda_kernel_utf8toutf16le (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param, CUdeviceptr buf, const u64 num); -int run_cuda_kernel_memset (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param, CUdeviceptr buf, const u32 value, const u64 size); +int run_cuda_kernel_memset (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param, CUdeviceptr buf, const u64 offset, const u8 value, const u64 size); +int run_cuda_kernel_memset32 (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param, CUdeviceptr buf, const u64 offset, const u32 value, const u64 size); int run_cuda_kernel_bzero (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param, CUdeviceptr buf, const u64 size); int run_hip_kernel_atinit (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param, HIPdeviceptr buf, const u64 num); int run_hip_kernel_utf8toutf16le (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param, HIPdeviceptr buf, const u64 num); -int run_hip_kernel_memset (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param, HIPdeviceptr buf, const u32 value, const u64 size); +int run_hip_kernel_memset (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param, HIPdeviceptr buf, const u64 offset, const u8 value, const u64 size); +int run_hip_kernel_memset32 (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param, HIPdeviceptr buf, const u64 offset, const u32 value, const u64 size); int run_hip_kernel_bzero (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param, HIPdeviceptr buf, const u64 size); int run_opencl_kernel_atinit (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param, cl_mem buf, const u64 num); int run_opencl_kernel_utf8toutf16le (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param, cl_mem buf, const u64 num); -int run_opencl_kernel_memset (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param, cl_mem buf, const u32 value, const u64 size); +int run_opencl_kernel_memset (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param, cl_mem buf, const u64 offset, const u8 value, const u64 size); +int run_opencl_kernel_memset32 (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param, cl_mem buf, const u64 offset, const u32 value, const u64 size); int run_opencl_kernel_bzero (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param, cl_mem buf, const u64 size); int run_kernel (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param, const u32 kern_run, const u64 pws_pos, const u64 num, const u32 event_update, const u32 iteration); diff --git a/include/ext_OpenCL.h b/include/ext_OpenCL.h index 42e37deed..40f3515ed 100644 --- a/include/ext_OpenCL.h +++ b/include/ext_OpenCL.h @@ -46,6 +46,7 @@ typedef cl_context (CL_API_CALL *OCL_CLCREATECONTEXT) (const cl_ typedef cl_kernel (CL_API_CALL *OCL_CLCREATEKERNEL) (cl_program, const char *, cl_int *); typedef cl_program (CL_API_CALL *OCL_CLCREATEPROGRAMWITHBINARY) (cl_context, cl_uint, const cl_device_id *, const size_t *, const unsigned char **, cl_int *, cl_int *); typedef cl_program (CL_API_CALL *OCL_CLCREATEPROGRAMWITHSOURCE) (cl_context, cl_uint, const char **, const size_t *, cl_int *); +typedef cl_int (CL_API_CALL *OCL_CLENQUEUEFILLBUFFER) (cl_command_queue, cl_mem, const void *, size_t, size_t, size_t, cl_uint, const cl_event *, cl_event *); typedef cl_int (CL_API_CALL *OCL_CLENQUEUECOPYBUFFER) (cl_command_queue, cl_mem, cl_mem, size_t, size_t, size_t, cl_uint, const cl_event *, cl_event *); typedef void * (CL_API_CALL *OCL_CLENQUEUEMAPBUFFER) (cl_command_queue, cl_mem, cl_bool, cl_map_flags, size_t, size_t, cl_uint, const cl_event *, cl_event *, cl_int *); typedef cl_int (CL_API_CALL *OCL_CLENQUEUENDRANGEKERNEL) (cl_command_queue, cl_kernel, cl_uint, const size_t *, const size_t *, const size_t *, cl_uint, const cl_event *, cl_event *); @@ -87,6 +88,7 @@ typedef struct hc_opencl_lib OCL_CLCREATEPROGRAMWITHBINARY clCreateProgramWithBinary; OCL_CLCREATEPROGRAMWITHSOURCE clCreateProgramWithSource; OCL_CLENQUEUECOPYBUFFER clEnqueueCopyBuffer; + OCL_CLENQUEUEFILLBUFFER clEnqueueFillBuffer; OCL_CLENQUEUEMAPBUFFER clEnqueueMapBuffer; OCL_CLENQUEUENDRANGEKERNEL clEnqueueNDRangeKernel; OCL_CLENQUEUEREADBUFFER clEnqueueReadBuffer; diff --git a/include/ext_cuda.h b/include/ext_cuda.h index 7f40e1410..11ed86048 100644 --- a/include/ext_cuda.h +++ b/include/ext_cuda.h @@ -1034,8 +1034,8 @@ typedef CUresult (CUDA_API_CALL *CUDA_CUMEMCPYHTODASYNC) (CUdeviceptr, co typedef CUresult (CUDA_API_CALL *CUDA_CUMEMFREE) (CUdeviceptr); typedef CUresult (CUDA_API_CALL *CUDA_CUMEMFREEHOST) (void *); typedef CUresult (CUDA_API_CALL *CUDA_CUMEMGETINFO) (size_t *, size_t *); -typedef CUresult (CUDA_API_CALL *CUDA_CUMEMSETD32) (CUdeviceptr, unsigned int, size_t); -typedef CUresult (CUDA_API_CALL *CUDA_CUMEMSETD8) (CUdeviceptr, unsigned char, size_t); +typedef CUresult (CUDA_API_CALL *CUDA_CUMEMSETD32ASYNC) (CUdeviceptr, unsigned int, size_t, CUstream); +typedef CUresult (CUDA_API_CALL *CUDA_CUMEMSETD8ASYNC) (CUdeviceptr, unsigned char, size_t, CUstream); typedef CUresult (CUDA_API_CALL *CUDA_CUMODULEGETFUNCTION) (CUfunction *, CUmodule, const char *); typedef CUresult (CUDA_API_CALL *CUDA_CUMODULEGETGLOBAL) (CUdeviceptr *, size_t *, CUmodule, const char *); typedef CUresult (CUDA_API_CALL *CUDA_CUMODULELOAD) (CUmodule *, const char *); @@ -1096,8 +1096,8 @@ typedef struct hc_cuda_lib CUDA_CUMEMFREE cuMemFree; CUDA_CUMEMFREEHOST cuMemFreeHost; CUDA_CUMEMGETINFO cuMemGetInfo; - CUDA_CUMEMSETD32 cuMemsetD32; - CUDA_CUMEMSETD8 cuMemsetD8; + CUDA_CUMEMSETD32ASYNC cuMemsetD32Async; + CUDA_CUMEMSETD8ASYNC cuMemsetD8Async; CUDA_CUMODULEGETFUNCTION cuModuleGetFunction; CUDA_CUMODULEGETGLOBAL cuModuleGetGlobal; CUDA_CUMODULELOAD cuModuleLoad; diff --git a/include/ext_hip.h b/include/ext_hip.h index 631dbfb33..ee344a80b 100644 --- a/include/ext_hip.h +++ b/include/ext_hip.h @@ -1043,8 +1043,8 @@ typedef HIPresult (HIP_API_CALL *HIP_HIPMEMCPYHTODASYNC) (HIPdeviceptr, c typedef HIPresult (HIP_API_CALL *HIP_HIPMEMFREE) (HIPdeviceptr); typedef HIPresult (HIP_API_CALL *HIP_HIPMEMFREEHOST) (void *); typedef HIPresult (HIP_API_CALL *HIP_HIPMEMGETINFO) (size_t *, size_t *); -typedef HIPresult (HIP_API_CALL *HIP_HIPMEMSETD32) (HIPdeviceptr, unsigned int, size_t); -typedef HIPresult (HIP_API_CALL *HIP_HIPMEMSETD8) (HIPdeviceptr, unsigned char, size_t); +typedef HIPresult (HIP_API_CALL *HIP_HIPMEMSETD32ASYNC) (HIPdeviceptr, unsigned int, size_t, HIPstream); +typedef HIPresult (HIP_API_CALL *HIP_HIPMEMSETD8ASYNC) (HIPdeviceptr, unsigned char, size_t, HIPstream); typedef HIPresult (HIP_API_CALL *HIP_HIPMODULEGETFUNCTION) (HIPfunction *, HIPmodule, const char *); typedef HIPresult (HIP_API_CALL *HIP_HIPMODULEGETGLOBAL) (HIPdeviceptr *, size_t *, HIPmodule, const char *); typedef HIPresult (HIP_API_CALL *HIP_HIPMODULELOAD) (HIPmodule *, const char *); @@ -1105,8 +1105,8 @@ typedef struct hc_hip_lib HIP_HIPMEMFREE hipMemFree; HIP_HIPMEMFREEHOST hipMemFreeHost; HIP_HIPMEMGETINFO hipMemGetInfo; - HIP_HIPMEMSETD32 hipMemsetD32; - HIP_HIPMEMSETD8 hipMemsetD8; + HIP_HIPMEMSETD32ASYNC hipMemsetD32Async; + HIP_HIPMEMSETD8ASYNC hipMemsetD8Async; HIP_HIPMODULEGETFUNCTION hipModuleGetFunction; HIP_HIPMODULEGETGLOBAL hipModuleGetGlobal; HIP_HIPMODULELOAD hipModuleLoad; diff --git a/src/backend.c b/src/backend.c index 73edce1ce..69b6345c6 100644 --- a/src/backend.c +++ b/src/backend.c @@ -1263,8 +1263,8 @@ int cuda_init (hashcat_ctx_t *hashcat_ctx) HC_LOAD_FUNC_CUDA (cuda, cuMemFree, cuMemFree_v2, CUDA_CUMEMFREE, CUDA, 1); HC_LOAD_FUNC_CUDA (cuda, cuMemFreeHost, cuMemFreeHost, CUDA_CUMEMFREEHOST, CUDA, 1); HC_LOAD_FUNC_CUDA (cuda, cuMemGetInfo, cuMemGetInfo_v2, CUDA_CUMEMGETINFO, CUDA, 1); - HC_LOAD_FUNC_CUDA (cuda, cuMemsetD32, cuMemsetD32_v2, CUDA_CUMEMSETD32, CUDA, 1); - HC_LOAD_FUNC_CUDA (cuda, cuMemsetD8, cuMemsetD8_v2, CUDA_CUMEMSETD8, CUDA, 1); + HC_LOAD_FUNC_CUDA (cuda, cuMemsetD32Async, cuMemsetD32Async, CUDA_CUMEMSETD32ASYNC, CUDA, 1); + HC_LOAD_FUNC_CUDA (cuda, cuMemsetD8Async, cuMemsetD8Async, CUDA_CUMEMSETD8ASYNC, CUDA, 1); HC_LOAD_FUNC_CUDA (cuda, cuModuleGetFunction, cuModuleGetFunction, CUDA_CUMODULEGETFUNCTION, CUDA, 1); HC_LOAD_FUNC_CUDA (cuda, cuModuleGetGlobal, cuModuleGetGlobal_v2, CUDA_CUMODULEGETGLOBAL, CUDA, 1); HC_LOAD_FUNC_CUDA (cuda, cuModuleLoad, cuModuleLoad, CUDA_CUMODULELOAD, CUDA, 1); @@ -1765,6 +1765,60 @@ int hc_cuMemcpyHtoDAsync (hashcat_ctx_t *hashcat_ctx, CUdeviceptr dstDevice, con return 0; } +int hc_cuMemsetD32Async (hashcat_ctx_t *hashcat_ctx, CUdeviceptr dstDevice, unsigned int ui, size_t N, CUstream hStream) +{ + backend_ctx_t *backend_ctx = hashcat_ctx->backend_ctx; + + CUDA_PTR *cuda = (CUDA_PTR *) backend_ctx->cuda; + + const CUresult CU_err = cuda->cuMemsetD32Async (dstDevice, ui, N, hStream); + + if (CU_err != CUDA_SUCCESS) + { + const char *pStr = NULL; + + if (cuda->cuGetErrorString (CU_err, &pStr) == CUDA_SUCCESS) + { + event_log_error (hashcat_ctx, "cuMemsetD32Async(): %s", pStr); + } + else + { + event_log_error (hashcat_ctx, "cuMemsetD32Async(): %d", CU_err); + } + + return -1; + } + + return 0; +} + +int hc_cuMemsetD8Async (hashcat_ctx_t *hashcat_ctx, CUdeviceptr dstDevice, unsigned char uc, size_t N, CUstream hStream) +{ + backend_ctx_t *backend_ctx = hashcat_ctx->backend_ctx; + + CUDA_PTR *cuda = (CUDA_PTR *) backend_ctx->cuda; + + const CUresult CU_err = cuda->cuMemsetD8Async (dstDevice, uc, N, hStream); + + if (CU_err != CUDA_SUCCESS) + { + const char *pStr = NULL; + + if (cuda->cuGetErrorString (CU_err, &pStr) == CUDA_SUCCESS) + { + event_log_error (hashcat_ctx, "cuMemsetD8Async(): %s", pStr); + } + else + { + event_log_error (hashcat_ctx, "cuMemsetD8Async(): %d", CU_err); + } + + return -1; + } + + return 0; +} + int hc_cuModuleGetFunction (hashcat_ctx_t *hashcat_ctx, CUfunction *hfunc, CUmodule hmod, const char *name) { backend_ctx_t *backend_ctx = hashcat_ctx->backend_ctx; @@ -2472,8 +2526,8 @@ int hip_init (hashcat_ctx_t *hashcat_ctx) HC_LOAD_FUNC_HIP (hip, hipMemFree, hipFree, HIP_HIPMEMFREE, HIP, 1); HC_LOAD_FUNC_HIP (hip, hipMemFreeHost, hipFreeHost, HIP_HIPMEMFREEHOST, HIP, 1); HC_LOAD_FUNC_HIP (hip, hipMemGetInfo, hipMemGetInfo, HIP_HIPMEMGETINFO, HIP, 1); - //HC_LOAD_FUNC_HIP (hip, hipMemsetD32, hipMemsetD32, HIP_HIPMEMSETD32, HIP, 1); - //HC_LOAD_FUNC_HIP (hip, hipMemsetD8, hipMemsetD8, HIP_HIPMEMSETD8, HIP, 1); + HC_LOAD_FUNC_HIP (hip, hipMemsetD32Async, hipMemsetD32Async, HIP_HIPMEMSETD32ASYNC, HIP, -1); + HC_LOAD_FUNC_HIP (hip, hipMemsetD8Async, hipMemsetD8Async, HIP_HIPMEMSETD8ASYNC, HIP, -1); HC_LOAD_FUNC_HIP (hip, hipModuleGetFunction, hipModuleGetFunction, HIP_HIPMODULEGETFUNCTION, HIP, 1); HC_LOAD_FUNC_HIP (hip, hipModuleGetGlobal, hipModuleGetGlobal, HIP_HIPMODULEGETGLOBAL, HIP, 1); HC_LOAD_FUNC_HIP (hip, hipModuleLoad, hipModuleLoad, HIP_HIPMODULELOAD, HIP, 1); @@ -2975,6 +3029,60 @@ int hc_hipMemcpyHtoDAsync (hashcat_ctx_t *hashcat_ctx, HIPdeviceptr dstDevice, c return 0; } +int hc_hipMemsetD32Async (hashcat_ctx_t *hashcat_ctx, HIPdeviceptr dstDevice, unsigned int ui, size_t N, HIPstream hStream) +{ + backend_ctx_t *backend_ctx = hashcat_ctx->backend_ctx; + + HIP_PTR *hip = (HIP_PTR *) backend_ctx->hip; + + const HIPresult HIP_err = hip->hipMemsetD32Async (dstDevice, ui, N, hStream); + + if (HIP_err != HIP_SUCCESS) + { + const char *pStr = NULL; + + if (hip->hipGetErrorString (HIP_err, &pStr) == HIP_SUCCESS) + { + event_log_error (hashcat_ctx, "hipMemsetD32Async(): %s", pStr); + } + else + { + event_log_error (hashcat_ctx, "hipMemsetD32Async(): %d", HIP_err); + } + + return -1; + } + + return 0; +} + +int hc_hipMemsetD8Async (hashcat_ctx_t *hashcat_ctx, HIPdeviceptr dstDevice, unsigned char uc, size_t N, HIPstream hStream) +{ + backend_ctx_t *backend_ctx = hashcat_ctx->backend_ctx; + + HIP_PTR *hip = (HIP_PTR *) backend_ctx->hip; + + const HIPresult HIP_err = hip->hipMemsetD8Async (dstDevice, uc, N, hStream); + + if (HIP_err != HIP_SUCCESS) + { + const char *pStr = NULL; + + if (hip->hipGetErrorString (HIP_err, &pStr) == HIP_SUCCESS) + { + event_log_error (hashcat_ctx, "hipMemsetD8Async(): %s", pStr); + } + else + { + event_log_error (hashcat_ctx, "hipMemsetD8Async(): %d", HIP_err); + } + + return -1; + } + + return 0; +} + int hc_hipModuleGetFunction (hashcat_ctx_t *hashcat_ctx, HIPfunction *hfunc, HIPmodule hmod, const char *name) { backend_ctx_t *backend_ctx = hashcat_ctx->backend_ctx; @@ -3636,6 +3744,7 @@ int ocl_init (hashcat_ctx_t *hashcat_ctx) HC_LOAD_FUNC (ocl, clCreateProgramWithBinary, OCL_CLCREATEPROGRAMWITHBINARY, OpenCL, 1); HC_LOAD_FUNC (ocl, clCreateProgramWithSource, OCL_CLCREATEPROGRAMWITHSOURCE, OpenCL, 1); HC_LOAD_FUNC (ocl, clEnqueueCopyBuffer, OCL_CLENQUEUECOPYBUFFER, OpenCL, 1); + HC_LOAD_FUNC (ocl, clEnqueueFillBuffer, OCL_CLENQUEUEFILLBUFFER, OpenCL, -1); HC_LOAD_FUNC (ocl, clEnqueueMapBuffer, OCL_CLENQUEUEMAPBUFFER, OpenCL, 1); HC_LOAD_FUNC (ocl, clEnqueueNDRangeKernel, OCL_CLENQUEUENDRANGEKERNEL, OpenCL, 1); HC_LOAD_FUNC (ocl, clEnqueueReadBuffer, OCL_CLENQUEUEREADBUFFER, OpenCL, 1); @@ -3811,6 +3920,23 @@ int hc_clEnqueueCopyBuffer (hashcat_ctx_t *hashcat_ctx, cl_command_queue command return 0; } +int hc_clEnqueueFillBuffer (hashcat_ctx_t *hashcat_ctx, cl_command_queue command_queue, cl_mem buffer, const void *pattern, size_t pattern_size, size_t offset, size_t size, cl_uint num_events_in_wait_list, const cl_event *event_wait_list, cl_event *event) +{ + const backend_ctx_t *backend_ctx = hashcat_ctx->backend_ctx; + const OCL_PTR *ocl = backend_ctx->ocl; + + cl_int CL_err = ocl->clEnqueueFillBuffer (command_queue, buffer, pattern, pattern_size, offset, size, num_events_in_wait_list, event_wait_list, event); + + if (CL_err != CL_SUCCESS) + { + event_log_error (hashcat_ctx, "clEnqueueFillBuffer(): %s", val2cstr_cl (CL_err)); + + return -1; + } + + return 0; +} + int hc_clEnqueueReadBuffer (hashcat_ctx_t *hashcat_ctx, cl_command_queue command_queue, cl_mem buffer, cl_bool blocking_read, size_t offset, size_t size, void *ptr, cl_uint num_events_in_wait_list, const cl_event *event_wait_list, cl_event *event) { backend_ctx_t *backend_ctx = hashcat_ctx->backend_ctx; @@ -5013,40 +5139,14 @@ int run_cuda_kernel_utf8toutf16le (hashcat_ctx_t *hashcat_ctx, hc_device_param_t return 0; } -int run_cuda_kernel_memset (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param, CUdeviceptr buf, const u32 value, const u64 size) +int run_cuda_kernel_memset (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param, CUdeviceptr buf, const u64 offset, const u8 value, const u64 size) { - const u64 num16d = size / 16; - const u64 num16m = size % 16; - u32 tmp[4]; + return hc_cuMemsetD8Async (hashcat_ctx, buf + offset, value, size, device_param->cuda_stream); +} - if (num16d) - { - device_param->kernel_params_memset[0] = (void *) &buf; - device_param->kernel_params_memset_buf32[1] = value; - device_param->kernel_params_memset_buf64[2] = num16d; - - const u64 kernel_threads = device_param->kernel_wgs_memset; - - u64 num_elements = CEILDIV (num16d, kernel_threads); - - CUfunction function = device_param->cuda_function_memset; - - if (hc_cuLaunchKernel (hashcat_ctx, function, num_elements, 1, 1, kernel_threads, 1, 1, 0, device_param->cuda_stream, device_param->kernel_params_memset, NULL) == -1) return -1; - } - - if (num16m) - { - tmp[0] = value; - tmp[1] = value; - tmp[2] = value; - tmp[3] = value; - - if (hc_cuMemcpyHtoDAsync (hashcat_ctx, buf + (num16d * 16), tmp, num16m, device_param->cuda_stream) == -1) return -1; - } - - if (hc_cuStreamSynchronize (hashcat_ctx, device_param->cuda_stream) == -1) return -1; - - return 0; +int run_cuda_kernel_memset32 (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param, CUdeviceptr buf, const u64 offset, const u32 value, const u64 size) +{ + return hc_cuMemsetD32Async (hashcat_ctx, buf + offset * sizeof (u32), value, size, device_param->cuda_stream); } int run_cuda_kernel_bzero (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param, CUdeviceptr buf, const u64 size) @@ -5112,40 +5212,14 @@ int run_hip_kernel_utf8toutf16le (hashcat_ctx_t *hashcat_ctx, hc_device_param_t return 0; } -int run_hip_kernel_memset (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param, HIPdeviceptr buf, const u32 value, const u64 size) +int run_hip_kernel_memset (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param, HIPdeviceptr buf, const u64 offset, const u8 value, const u64 size) { - const u64 num16d = size / 16; - const u64 num16m = size % 16; - u32 tmp[4]; + return hc_hipMemsetD8Async (hashcat_ctx, buf + offset, value, size, device_param->hip_stream); +} - if (num16d) - { - device_param->kernel_params_memset[0] = (void *) &buf; - device_param->kernel_params_memset_buf32[1] = value; - device_param->kernel_params_memset_buf64[2] = num16d; - - const u64 kernel_threads = device_param->kernel_wgs_memset; - - u64 num_elements = CEILDIV (num16d, kernel_threads); - - HIPfunction function = device_param->hip_function_memset; - - if (hc_hipLaunchKernel (hashcat_ctx, function, num_elements, 1, 1, kernel_threads, 1, 1, 0, device_param->hip_stream, device_param->kernel_params_memset, NULL) == -1) return -1; - } - - if (num16m) - { - tmp[0] = value; - tmp[1] = value; - tmp[2] = value; - tmp[3] = value; - - if (hc_hipMemcpyHtoDAsync (hashcat_ctx, buf + (num16d * 16), tmp, num16m, device_param->hip_stream) == -1) return -1; - } - - if (hc_hipStreamSynchronize (hashcat_ctx, device_param->hip_stream) == -1) return -1; - - return 0; +int run_hip_kernel_memset32 (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param, HIPdeviceptr buf, const u64 offset, const u32 value, const u64 size) +{ + return hc_hipMemsetD32Async (hashcat_ctx, buf + offset * sizeof (u32), value, size, device_param->hip_stream); } int run_hip_kernel_bzero (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param, HIPdeviceptr buf, const u64 size) @@ -5227,45 +5301,65 @@ int run_opencl_kernel_utf8toutf16le (hashcat_ctx_t *hashcat_ctx, hc_device_param return 0; } -int run_opencl_kernel_memset (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param, cl_mem buf, const u32 value, const u64 size) +int run_opencl_kernel_memset (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param, cl_mem buf, const u64 offset, const u8 value, const u64 size) { - const u64 num16d = size / 16; - const u64 num16m = size % 16; - u32 tmp[4]; + const backend_ctx_t *backend_ctx = hashcat_ctx->backend_ctx; + const OCL_PTR *ocl = backend_ctx->ocl; - if (num16d) + int rc; + + /* workaround if missing clEnqueueFillBuffer() */ + if (ocl->clEnqueueFillBuffer == NULL) { - const u64 kernel_threads = device_param->kernel_wgs_memset; + const u64 len = offset + size; - u64 num_elements = round_up_multiple_64 (num16d, kernel_threads); + char *tmp = hcmalloc (len * sizeof (u8)); - cl_kernel kernel = device_param->opencl_kernel_memset; + memset(tmp, value, len); - if (hc_clSetKernelArg (hashcat_ctx, kernel, 0, sizeof(cl_mem), (void *) &buf) == -1) return -1; - if (hc_clSetKernelArg (hashcat_ctx, kernel, 1, sizeof(cl_uint), (void *) &value) == -1) return -1; - if (hc_clSetKernelArg (hashcat_ctx, kernel, 2, sizeof(cl_ulong), (void *) &num16d) == -1) return -1; + /* blocking */ + rc = hc_clEnqueueWriteBuffer (hashcat_ctx, device_param->opencl_command_queue, buf, CL_TRUE, offset, size, tmp, 0, NULL, NULL); - const size_t global_work_size[3] = { num_elements, 1, 1 }; - const size_t local_work_size[3] = { kernel_threads, 1, 1 }; - - if (hc_clEnqueueNDRangeKernel (hashcat_ctx, device_param->opencl_command_queue, kernel, 1, NULL, global_work_size, local_work_size, 0, NULL, NULL) == -1) return -1; + free(tmp); + } + else + { + rc = hc_clEnqueueFillBuffer (hashcat_ctx, device_param->opencl_command_queue, buf, &value, sizeof (u8), offset, size, 0, NULL, NULL); } - if (num16m) - { - tmp[0] = value; - tmp[1] = value; - tmp[2] = value; - tmp[3] = value; + return rc; +} - if (hc_clEnqueueWriteBuffer (hashcat_ctx, device_param->opencl_command_queue, buf, CL_FALSE, num16d * 16, num16m, tmp, 0, NULL, NULL) == -1) return -1; +int run_opencl_kernel_memset32 (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param, cl_mem buf, const u64 offset, const u32 value, const u64 size) +{ + const backend_ctx_t *backend_ctx = hashcat_ctx->backend_ctx; + const OCL_PTR *ocl = backend_ctx->ocl; + + int rc; + + /* workaround if missing clEnqueueFillBuffer() */ + if (ocl->clEnqueueFillBuffer == NULL) + { + const u64 len = offset + size; + + u32 *tmp = (u32 *) hcmalloc ((offset + size) * sizeof (u32)); + + for (u64 i = 0; i < len; i++) + { + tmp[i] = value; + } + + /* blocking */ + rc = hc_clEnqueueWriteBuffer (hashcat_ctx, device_param->opencl_command_queue, buf, CL_TRUE, offset * sizeof (u32), size * sizeof (u32), tmp, 0, NULL, NULL); + + free(tmp); + } + else + { + rc = hc_clEnqueueFillBuffer (hashcat_ctx, device_param->opencl_command_queue, buf, &value, sizeof (u32), offset, size, 0, NULL, NULL); } - /*if (hc_clFlush (hashcat_ctx, device_param->opencl_command_queue) == -1) return -1;*/ - - if (hc_clFinish (hashcat_ctx, device_param->opencl_command_queue) == -1) return -1; - - return 0; + return rc; } int run_opencl_kernel_bzero (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param, cl_mem buf, const u64 size) From 5dbe4958b88fc3b37543eae2b7391fb35b590647 Mon Sep 17 00:00:00 2001 From: Jukka Ojanen Date: Tue, 27 Jul 2021 19:01:13 +0300 Subject: [PATCH 17/33] Remove digests_shown_tmp buffer, used in check_cracked() to zero out hashes --- include/types.h | 1 - src/hashes.c | 11 ++++------- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/include/types.h b/include/types.h index 93cb42bf4..e0de045a5 100644 --- a/include/types.h +++ b/include/types.h @@ -927,7 +927,6 @@ typedef struct hashes void *digests_buf; u32 *digests_shown; - u32 *digests_shown_tmp; u32 salts_cnt; u32 salts_done; diff --git a/src/hashes.c b/src/hashes.c index 5679a5442..e1f8402c2 100644 --- a/src/hashes.c +++ b/src/hashes.c @@ -679,10 +679,9 @@ int check_cracked (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param) // otherwise host thinks again and again the hash was cracked // and returns invalid password each time - /* TODO: this is bad design -- digests_shown_tmp is only used for zeroing */ if (device_param->is_cuda == true) { - rc = hc_cuMemcpyHtoDAsync (hashcat_ctx, device_param->cuda_d_digests_shown + (salt_buf->digests_offset * sizeof (u32)), hashes->digests_shown_tmp, salt_buf->digests_cnt * sizeof (u32), device_param->cuda_stream); + rc = run_cuda_kernel_bzero (hashcat_ctx, device_param, device_param->cuda_d_digests_shown + (salt_buf->digests_offset * sizeof (u32)), salt_buf->digests_cnt * sizeof (u32)); if (rc == -1) { @@ -692,7 +691,7 @@ int check_cracked (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param) if (device_param->is_hip == true) { - rc = hc_hipMemcpyHtoDAsync (hashcat_ctx, device_param->hip_d_digests_shown + (salt_buf->digests_offset * sizeof (u32)), hashes->digests_shown_tmp, salt_buf->digests_cnt * sizeof (u32), device_param->hip_stream); + rc = run_hip_kernel_bzero (hashcat_ctx, device_param, device_param->hip_d_digests_shown + (salt_buf->digests_offset * sizeof (u32)), salt_buf->digests_cnt * sizeof (u32)); if (rc == -1) { @@ -702,7 +701,8 @@ int check_cracked (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param) if (device_param->is_opencl == true) { - rc = hc_clEnqueueWriteBuffer (hashcat_ctx, device_param->opencl_command_queue, device_param->opencl_d_digests_shown, CL_FALSE, salt_buf->digests_offset * sizeof (u32), salt_buf->digests_cnt * sizeof (u32), hashes->digests_shown_tmp, 0, NULL, NULL); + /* NOTE: run_opencl_kernel_bzero() does not handle buffer offset */ + rc = run_opencl_kernel_memset32 (hashcat_ctx, device_param, device_param->opencl_d_digests_shown, salt_buf->digests_offset, 0, salt_buf->digests_cnt); if (rc == 0) { @@ -1675,7 +1675,6 @@ int hashes_init_stage2 (hashcat_ctx_t *hashcat_ctx) u32 digests_done = 0; u32 *digests_shown = (u32 *) hccalloc (digests_cnt, sizeof (u32)); - u32 *digests_shown_tmp = (u32 *) hccalloc (digests_cnt, sizeof (u32)); u32 salts_cnt = 0; u32 salts_done = 0; @@ -1812,7 +1811,6 @@ int hashes_init_stage2 (hashcat_ctx_t *hashcat_ctx) hashes->digests_done = digests_done; hashes->digests_buf = digests_buf_new; hashes->digests_shown = digests_shown; - hashes->digests_shown_tmp = digests_shown_tmp; hashes->salts_cnt = salts_cnt; hashes->salts_done = salts_done; @@ -2299,7 +2297,6 @@ void hashes_destroy (hashcat_ctx_t *hashcat_ctx) hcfree (hashes->digests_buf); hcfree (hashes->digests_shown); - hcfree (hashes->digests_shown_tmp); hcfree (hashes->salts_buf); hcfree (hashes->salts_shown); From 8b590f651b058e56e68678d1f3b0cda1f0ad8405 Mon Sep 17 00:00:00 2001 From: Jukka Ojanen Date: Wed, 28 Jul 2021 19:26:30 +0300 Subject: [PATCH 18/33] Remove unused gpu_memset and its references --- OpenCL/shared.cl | 27 ---------------------- include/types.h | 11 --------- src/backend.c | 59 ------------------------------------------------ 3 files changed, 97 deletions(-) diff --git a/OpenCL/shared.cl b/OpenCL/shared.cl index fe4ff4087..267ce5c85 100644 --- a/OpenCL/shared.cl +++ b/OpenCL/shared.cl @@ -106,33 +106,6 @@ KERNEL_FQ void gpu_decompress (GLOBAL_AS pw_idx_t *pws_idx, GLOBAL_AS u32 *pws_c pws_buf[gid] = pw; } -KERNEL_FQ void gpu_memset (GLOBAL_AS uint4 *buf, const u32 value, const u64 gid_max) -{ - const u64 gid = get_global_id (0); - - if (gid >= gid_max) return; - - uint4 r; - - #if defined IS_NATIVE - r = value; - #elif defined IS_OPENCL - r = (uint4) (value); - #elif defined IS_CUDA - r.x = value; - r.y = value; - r.z = value; - r.w = value; - #elif defined IS_HIP - r.x = value; - r.y = value; - r.z = value; - r.w = value; - #endif - - buf[gid] = r; -} - KERNEL_FQ void gpu_bzero(GLOBAL_AS uint4* buf, const u64 gid_max) { const u64 gid = get_global_id(0); diff --git a/include/types.h b/include/types.h index e0de045a5..88a4e98dc 100644 --- a/include/types.h +++ b/include/types.h @@ -1139,7 +1139,6 @@ typedef struct hc_device_param u32 kernel_wgs_mp_r; u32 kernel_wgs_amp; u32 kernel_wgs_tm; - u32 kernel_wgs_memset; u32 kernel_wgs_bzero; u32 kernel_wgs_atinit; u32 kernel_wgs_utf8toutf16le; @@ -1165,7 +1164,6 @@ typedef struct hc_device_param u32 kernel_preferred_wgs_multiple_mp_r; u32 kernel_preferred_wgs_multiple_amp; u32 kernel_preferred_wgs_multiple_tm; - u32 kernel_preferred_wgs_multiple_memset; u32 kernel_preferred_wgs_multiple_bzero; u32 kernel_preferred_wgs_multiple_atinit; u32 kernel_preferred_wgs_multiple_utf8toutf16le; @@ -1191,7 +1189,6 @@ typedef struct hc_device_param u64 kernel_local_mem_size_mp_r; u64 kernel_local_mem_size_amp; u64 kernel_local_mem_size_tm; - u64 kernel_local_mem_size_memset; u64 kernel_local_mem_size_bzero; u64 kernel_local_mem_size_atinit; u64 kernel_local_mem_size_utf8toutf16le; @@ -1217,7 +1214,6 @@ typedef struct hc_device_param u64 kernel_dynamic_local_mem_size_mp_r; u64 kernel_dynamic_local_mem_size_amp; u64 kernel_dynamic_local_mem_size_tm; - u64 kernel_dynamic_local_mem_size_memset; u64 kernel_dynamic_local_mem_size_bzero; u64 kernel_dynamic_local_mem_size_atinit; u64 kernel_dynamic_local_mem_size_utf8toutf16le; @@ -1379,7 +1375,6 @@ typedef struct hc_device_param void *kernel_params_mp_l[PARAMCNT]; void *kernel_params_amp[PARAMCNT]; void *kernel_params_tm[PARAMCNT]; - void *kernel_params_memset[PARAMCNT]; void *kernel_params_bzero[PARAMCNT]; void *kernel_params_atinit[PARAMCNT]; void *kernel_params_utf8toutf16le[PARAMCNT]; @@ -1400,9 +1395,6 @@ typedef struct hc_device_param u32 kernel_params_amp_buf32[PARAMCNT]; u64 kernel_params_amp_buf64[PARAMCNT]; - u32 kernel_params_memset_buf32[PARAMCNT]; - u64 kernel_params_memset_buf64[PARAMCNT]; - u32 kernel_params_bzero_buf32[PARAMCNT]; u64 kernel_params_bzero_buf64[PARAMCNT]; @@ -1450,7 +1442,6 @@ typedef struct hc_device_param CUfunction cuda_function_mp_r; CUfunction cuda_function_amp; CUfunction cuda_function_tm; - CUfunction cuda_function_memset; CUfunction cuda_function_bzero; CUfunction cuda_function_atinit; CUfunction cuda_function_utf8toutf16le; @@ -1532,7 +1523,6 @@ typedef struct hc_device_param HIPfunction hip_function_mp_r; HIPfunction hip_function_amp; HIPfunction hip_function_tm; - HIPfunction hip_function_memset; HIPfunction hip_function_bzero; HIPfunction hip_function_atinit; HIPfunction hip_function_utf8toutf16le; @@ -1618,7 +1608,6 @@ typedef struct hc_device_param cl_kernel opencl_kernel_mp_r; cl_kernel opencl_kernel_amp; cl_kernel opencl_kernel_tm; - cl_kernel opencl_kernel_memset; cl_kernel opencl_kernel_bzero; cl_kernel opencl_kernel_atinit; cl_kernel opencl_kernel_utf8toutf16le; diff --git a/src/backend.c b/src/backend.c index 69b6345c6..e9fc8c64c 100644 --- a/src/backend.c +++ b/src/backend.c @@ -5471,18 +5471,6 @@ int run_kernel (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param, con dynamic_shared_mem = 0; } - //if (device_param->is_cuda == true) - //{ - //if ((device_param->kernel_dynamic_local_mem_size_memset % device_param->device_local_mem_size) == 0) - //{ - // this is the case Compute Capability 7.5 - // there is also Compute Capability 7.0 which offers a larger dynamic local size access - // however, if it's an exact multiple the driver can optimize this for us more efficient - - //dynamic_shared_mem = 0; - //} - //} - kernel_threads = MIN (kernel_threads, device_param->kernel_threads); device_param->kernel_params_buf64[35] = pws_pos; @@ -11750,18 +11738,6 @@ int backend_session_begin (hashcat_ctx_t *hashcat_ctx) if (device_param->is_cuda == true) { - // GPU memset - - if (hc_cuModuleGetFunction (hashcat_ctx, &device_param->cuda_function_memset, device_param->cuda_module_shared, "gpu_memset") == -1) return -1; - - if (get_cuda_kernel_wgs (hashcat_ctx, device_param->cuda_function_memset, &device_param->kernel_wgs_memset) == -1) return -1; - - if (get_cuda_kernel_local_mem_size (hashcat_ctx, device_param->cuda_function_memset, &device_param->kernel_local_mem_size_memset) == -1) return -1; - - device_param->kernel_dynamic_local_mem_size_memset = device_param->device_local_mem_size - device_param->kernel_local_mem_size_memset; - - device_param->kernel_preferred_wgs_multiple_memset = device_param->cuda_warp_size; - // GPU bzero if (hc_cuModuleGetFunction (hashcat_ctx, &device_param->cuda_function_bzero, device_param->cuda_module_shared, "gpu_bzero") == -1) return -1; @@ -11816,18 +11792,6 @@ int backend_session_begin (hashcat_ctx_t *hashcat_ctx) if (device_param->is_hip == true) { - // GPU memset - - if (hc_hipModuleGetFunction (hashcat_ctx, &device_param->hip_function_memset, device_param->hip_module_shared, "gpu_memset") == -1) return -1; - - if (get_hip_kernel_wgs (hashcat_ctx, device_param->hip_function_memset, &device_param->kernel_wgs_memset) == -1) return -1; - - if (get_hip_kernel_local_mem_size (hashcat_ctx, device_param->hip_function_memset, &device_param->kernel_local_mem_size_memset) == -1) return -1; - - device_param->kernel_dynamic_local_mem_size_memset = device_param->device_local_mem_size - device_param->kernel_local_mem_size_memset; - - device_param->kernel_preferred_wgs_multiple_memset = device_param->hip_warp_size; - // GPU bzero if (hc_hipModuleGetFunction (hashcat_ctx, &device_param->hip_function_bzero, device_param->hip_module_shared, "gpu_bzero") == -1) return -1; @@ -11882,18 +11846,6 @@ int backend_session_begin (hashcat_ctx_t *hashcat_ctx) if (device_param->is_opencl == true) { - // GPU memset - - if (hc_clCreateKernel (hashcat_ctx, device_param->opencl_program_shared, "gpu_memset", &device_param->opencl_kernel_memset) == -1) return -1; - - if (get_opencl_kernel_wgs (hashcat_ctx, device_param, device_param->opencl_kernel_memset, &device_param->kernel_wgs_memset) == -1) return -1; - - if (get_opencl_kernel_local_mem_size (hashcat_ctx, device_param, device_param->opencl_kernel_memset, &device_param->kernel_local_mem_size_memset) == -1) return -1; - - if (get_opencl_kernel_dynamic_local_mem_size (hashcat_ctx, device_param, device_param->opencl_kernel_memset, &device_param->kernel_dynamic_local_mem_size_memset) == -1) return -1; - - if (get_opencl_kernel_preferred_wgs_multiple (hashcat_ctx, device_param, device_param->opencl_kernel_memset, &device_param->kernel_preferred_wgs_multiple_memset) == -1) return -1; - // GPU bzero if (hc_clCreateKernel (hashcat_ctx, device_param->opencl_program_shared, "gpu_bzero", &device_param->opencl_kernel_bzero) == -1) return -1; @@ -12840,13 +12792,6 @@ int backend_session_begin (hashcat_ctx_t *hashcat_ctx) } } - device_param->kernel_params_memset_buf32[1] = 0; // value - device_param->kernel_params_memset_buf64[2] = 0; // gid_max - - device_param->kernel_params_memset[0] = NULL; - device_param->kernel_params_memset[1] = &device_param->kernel_params_memset_buf32[1]; - device_param->kernel_params_memset[2] = &device_param->kernel_params_memset_buf64[2]; - device_param->kernel_params_bzero_buf64[1] = 0; // gid_max device_param->kernel_params_bzero[0] = NULL; @@ -15384,7 +15329,6 @@ void backend_session_destroy (hashcat_ctx_t *hashcat_ctx) device_param->cuda_function_mp_r = NULL; device_param->cuda_function_tm = NULL; device_param->cuda_function_amp = NULL; - device_param->cuda_function_memset = NULL; device_param->cuda_function_bzero = NULL; device_param->cuda_function_atinit = NULL; device_param->cuda_function_utf8toutf16le = NULL; @@ -15513,7 +15457,6 @@ void backend_session_destroy (hashcat_ctx_t *hashcat_ctx) device_param->hip_function_mp_r = NULL; device_param->hip_function_tm = NULL; device_param->hip_function_amp = NULL; - device_param->hip_function_memset = NULL; device_param->hip_function_bzero = NULL; device_param->hip_function_atinit = NULL; device_param->hip_function_utf8toutf16le = NULL; @@ -15592,7 +15535,6 @@ void backend_session_destroy (hashcat_ctx_t *hashcat_ctx) if (device_param->opencl_kernel_mp_r) hc_clReleaseKernel (hashcat_ctx, device_param->opencl_kernel_mp_r); if (device_param->opencl_kernel_tm) hc_clReleaseKernel (hashcat_ctx, device_param->opencl_kernel_tm); if (device_param->opencl_kernel_amp) hc_clReleaseKernel (hashcat_ctx, device_param->opencl_kernel_amp); - if (device_param->opencl_kernel_memset) hc_clReleaseKernel (hashcat_ctx, device_param->opencl_kernel_memset); if (device_param->opencl_kernel_bzero) hc_clReleaseKernel (hashcat_ctx, device_param->opencl_kernel_bzero); if (device_param->opencl_kernel_atinit) hc_clReleaseKernel (hashcat_ctx, device_param->opencl_kernel_atinit); if (device_param->opencl_kernel_utf8toutf16le) hc_clReleaseKernel (hashcat_ctx, device_param->opencl_kernel_utf8toutf16le); @@ -15663,7 +15605,6 @@ void backend_session_destroy (hashcat_ctx_t *hashcat_ctx) device_param->opencl_kernel_mp_r = NULL; device_param->opencl_kernel_tm = NULL; device_param->opencl_kernel_amp = NULL; - device_param->opencl_kernel_memset = NULL; device_param->opencl_kernel_bzero = NULL; device_param->opencl_kernel_atinit = NULL; device_param->opencl_kernel_utf8toutf16le = NULL; From d7cc8d7cd2247615bdba6fad9e34e7b29d53a4fe Mon Sep 17 00:00:00 2001 From: Jukka Ojanen Date: Wed, 28 Jul 2021 19:49:02 +0300 Subject: [PATCH 19/33] Revert due to module_03200.c, module_25600.c and module_25800.c using device_param->kernel_dynamic_local_mem_size_memset --- OpenCL/shared.cl | 27 ++++++++++++++++++++++ include/types.h | 11 +++++++++ src/backend.c | 59 ++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 97 insertions(+) diff --git a/OpenCL/shared.cl b/OpenCL/shared.cl index 267ce5c85..fe4ff4087 100644 --- a/OpenCL/shared.cl +++ b/OpenCL/shared.cl @@ -106,6 +106,33 @@ KERNEL_FQ void gpu_decompress (GLOBAL_AS pw_idx_t *pws_idx, GLOBAL_AS u32 *pws_c pws_buf[gid] = pw; } +KERNEL_FQ void gpu_memset (GLOBAL_AS uint4 *buf, const u32 value, const u64 gid_max) +{ + const u64 gid = get_global_id (0); + + if (gid >= gid_max) return; + + uint4 r; + + #if defined IS_NATIVE + r = value; + #elif defined IS_OPENCL + r = (uint4) (value); + #elif defined IS_CUDA + r.x = value; + r.y = value; + r.z = value; + r.w = value; + #elif defined IS_HIP + r.x = value; + r.y = value; + r.z = value; + r.w = value; + #endif + + buf[gid] = r; +} + KERNEL_FQ void gpu_bzero(GLOBAL_AS uint4* buf, const u64 gid_max) { const u64 gid = get_global_id(0); diff --git a/include/types.h b/include/types.h index 88a4e98dc..e0de045a5 100644 --- a/include/types.h +++ b/include/types.h @@ -1139,6 +1139,7 @@ typedef struct hc_device_param u32 kernel_wgs_mp_r; u32 kernel_wgs_amp; u32 kernel_wgs_tm; + u32 kernel_wgs_memset; u32 kernel_wgs_bzero; u32 kernel_wgs_atinit; u32 kernel_wgs_utf8toutf16le; @@ -1164,6 +1165,7 @@ typedef struct hc_device_param u32 kernel_preferred_wgs_multiple_mp_r; u32 kernel_preferred_wgs_multiple_amp; u32 kernel_preferred_wgs_multiple_tm; + u32 kernel_preferred_wgs_multiple_memset; u32 kernel_preferred_wgs_multiple_bzero; u32 kernel_preferred_wgs_multiple_atinit; u32 kernel_preferred_wgs_multiple_utf8toutf16le; @@ -1189,6 +1191,7 @@ typedef struct hc_device_param u64 kernel_local_mem_size_mp_r; u64 kernel_local_mem_size_amp; u64 kernel_local_mem_size_tm; + u64 kernel_local_mem_size_memset; u64 kernel_local_mem_size_bzero; u64 kernel_local_mem_size_atinit; u64 kernel_local_mem_size_utf8toutf16le; @@ -1214,6 +1217,7 @@ typedef struct hc_device_param u64 kernel_dynamic_local_mem_size_mp_r; u64 kernel_dynamic_local_mem_size_amp; u64 kernel_dynamic_local_mem_size_tm; + u64 kernel_dynamic_local_mem_size_memset; u64 kernel_dynamic_local_mem_size_bzero; u64 kernel_dynamic_local_mem_size_atinit; u64 kernel_dynamic_local_mem_size_utf8toutf16le; @@ -1375,6 +1379,7 @@ typedef struct hc_device_param void *kernel_params_mp_l[PARAMCNT]; void *kernel_params_amp[PARAMCNT]; void *kernel_params_tm[PARAMCNT]; + void *kernel_params_memset[PARAMCNT]; void *kernel_params_bzero[PARAMCNT]; void *kernel_params_atinit[PARAMCNT]; void *kernel_params_utf8toutf16le[PARAMCNT]; @@ -1395,6 +1400,9 @@ typedef struct hc_device_param u32 kernel_params_amp_buf32[PARAMCNT]; u64 kernel_params_amp_buf64[PARAMCNT]; + u32 kernel_params_memset_buf32[PARAMCNT]; + u64 kernel_params_memset_buf64[PARAMCNT]; + u32 kernel_params_bzero_buf32[PARAMCNT]; u64 kernel_params_bzero_buf64[PARAMCNT]; @@ -1442,6 +1450,7 @@ typedef struct hc_device_param CUfunction cuda_function_mp_r; CUfunction cuda_function_amp; CUfunction cuda_function_tm; + CUfunction cuda_function_memset; CUfunction cuda_function_bzero; CUfunction cuda_function_atinit; CUfunction cuda_function_utf8toutf16le; @@ -1523,6 +1532,7 @@ typedef struct hc_device_param HIPfunction hip_function_mp_r; HIPfunction hip_function_amp; HIPfunction hip_function_tm; + HIPfunction hip_function_memset; HIPfunction hip_function_bzero; HIPfunction hip_function_atinit; HIPfunction hip_function_utf8toutf16le; @@ -1608,6 +1618,7 @@ typedef struct hc_device_param cl_kernel opencl_kernel_mp_r; cl_kernel opencl_kernel_amp; cl_kernel opencl_kernel_tm; + cl_kernel opencl_kernel_memset; cl_kernel opencl_kernel_bzero; cl_kernel opencl_kernel_atinit; cl_kernel opencl_kernel_utf8toutf16le; diff --git a/src/backend.c b/src/backend.c index e9fc8c64c..69b6345c6 100644 --- a/src/backend.c +++ b/src/backend.c @@ -5471,6 +5471,18 @@ int run_kernel (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param, con dynamic_shared_mem = 0; } + //if (device_param->is_cuda == true) + //{ + //if ((device_param->kernel_dynamic_local_mem_size_memset % device_param->device_local_mem_size) == 0) + //{ + // this is the case Compute Capability 7.5 + // there is also Compute Capability 7.0 which offers a larger dynamic local size access + // however, if it's an exact multiple the driver can optimize this for us more efficient + + //dynamic_shared_mem = 0; + //} + //} + kernel_threads = MIN (kernel_threads, device_param->kernel_threads); device_param->kernel_params_buf64[35] = pws_pos; @@ -11738,6 +11750,18 @@ int backend_session_begin (hashcat_ctx_t *hashcat_ctx) if (device_param->is_cuda == true) { + // GPU memset + + if (hc_cuModuleGetFunction (hashcat_ctx, &device_param->cuda_function_memset, device_param->cuda_module_shared, "gpu_memset") == -1) return -1; + + if (get_cuda_kernel_wgs (hashcat_ctx, device_param->cuda_function_memset, &device_param->kernel_wgs_memset) == -1) return -1; + + if (get_cuda_kernel_local_mem_size (hashcat_ctx, device_param->cuda_function_memset, &device_param->kernel_local_mem_size_memset) == -1) return -1; + + device_param->kernel_dynamic_local_mem_size_memset = device_param->device_local_mem_size - device_param->kernel_local_mem_size_memset; + + device_param->kernel_preferred_wgs_multiple_memset = device_param->cuda_warp_size; + // GPU bzero if (hc_cuModuleGetFunction (hashcat_ctx, &device_param->cuda_function_bzero, device_param->cuda_module_shared, "gpu_bzero") == -1) return -1; @@ -11792,6 +11816,18 @@ int backend_session_begin (hashcat_ctx_t *hashcat_ctx) if (device_param->is_hip == true) { + // GPU memset + + if (hc_hipModuleGetFunction (hashcat_ctx, &device_param->hip_function_memset, device_param->hip_module_shared, "gpu_memset") == -1) return -1; + + if (get_hip_kernel_wgs (hashcat_ctx, device_param->hip_function_memset, &device_param->kernel_wgs_memset) == -1) return -1; + + if (get_hip_kernel_local_mem_size (hashcat_ctx, device_param->hip_function_memset, &device_param->kernel_local_mem_size_memset) == -1) return -1; + + device_param->kernel_dynamic_local_mem_size_memset = device_param->device_local_mem_size - device_param->kernel_local_mem_size_memset; + + device_param->kernel_preferred_wgs_multiple_memset = device_param->hip_warp_size; + // GPU bzero if (hc_hipModuleGetFunction (hashcat_ctx, &device_param->hip_function_bzero, device_param->hip_module_shared, "gpu_bzero") == -1) return -1; @@ -11846,6 +11882,18 @@ int backend_session_begin (hashcat_ctx_t *hashcat_ctx) if (device_param->is_opencl == true) { + // GPU memset + + if (hc_clCreateKernel (hashcat_ctx, device_param->opencl_program_shared, "gpu_memset", &device_param->opencl_kernel_memset) == -1) return -1; + + if (get_opencl_kernel_wgs (hashcat_ctx, device_param, device_param->opencl_kernel_memset, &device_param->kernel_wgs_memset) == -1) return -1; + + if (get_opencl_kernel_local_mem_size (hashcat_ctx, device_param, device_param->opencl_kernel_memset, &device_param->kernel_local_mem_size_memset) == -1) return -1; + + if (get_opencl_kernel_dynamic_local_mem_size (hashcat_ctx, device_param, device_param->opencl_kernel_memset, &device_param->kernel_dynamic_local_mem_size_memset) == -1) return -1; + + if (get_opencl_kernel_preferred_wgs_multiple (hashcat_ctx, device_param, device_param->opencl_kernel_memset, &device_param->kernel_preferred_wgs_multiple_memset) == -1) return -1; + // GPU bzero if (hc_clCreateKernel (hashcat_ctx, device_param->opencl_program_shared, "gpu_bzero", &device_param->opencl_kernel_bzero) == -1) return -1; @@ -12792,6 +12840,13 @@ int backend_session_begin (hashcat_ctx_t *hashcat_ctx) } } + device_param->kernel_params_memset_buf32[1] = 0; // value + device_param->kernel_params_memset_buf64[2] = 0; // gid_max + + device_param->kernel_params_memset[0] = NULL; + device_param->kernel_params_memset[1] = &device_param->kernel_params_memset_buf32[1]; + device_param->kernel_params_memset[2] = &device_param->kernel_params_memset_buf64[2]; + device_param->kernel_params_bzero_buf64[1] = 0; // gid_max device_param->kernel_params_bzero[0] = NULL; @@ -15329,6 +15384,7 @@ void backend_session_destroy (hashcat_ctx_t *hashcat_ctx) device_param->cuda_function_mp_r = NULL; device_param->cuda_function_tm = NULL; device_param->cuda_function_amp = NULL; + device_param->cuda_function_memset = NULL; device_param->cuda_function_bzero = NULL; device_param->cuda_function_atinit = NULL; device_param->cuda_function_utf8toutf16le = NULL; @@ -15457,6 +15513,7 @@ void backend_session_destroy (hashcat_ctx_t *hashcat_ctx) device_param->hip_function_mp_r = NULL; device_param->hip_function_tm = NULL; device_param->hip_function_amp = NULL; + device_param->hip_function_memset = NULL; device_param->hip_function_bzero = NULL; device_param->hip_function_atinit = NULL; device_param->hip_function_utf8toutf16le = NULL; @@ -15535,6 +15592,7 @@ void backend_session_destroy (hashcat_ctx_t *hashcat_ctx) if (device_param->opencl_kernel_mp_r) hc_clReleaseKernel (hashcat_ctx, device_param->opencl_kernel_mp_r); if (device_param->opencl_kernel_tm) hc_clReleaseKernel (hashcat_ctx, device_param->opencl_kernel_tm); if (device_param->opencl_kernel_amp) hc_clReleaseKernel (hashcat_ctx, device_param->opencl_kernel_amp); + if (device_param->opencl_kernel_memset) hc_clReleaseKernel (hashcat_ctx, device_param->opencl_kernel_memset); if (device_param->opencl_kernel_bzero) hc_clReleaseKernel (hashcat_ctx, device_param->opencl_kernel_bzero); if (device_param->opencl_kernel_atinit) hc_clReleaseKernel (hashcat_ctx, device_param->opencl_kernel_atinit); if (device_param->opencl_kernel_utf8toutf16le) hc_clReleaseKernel (hashcat_ctx, device_param->opencl_kernel_utf8toutf16le); @@ -15605,6 +15663,7 @@ void backend_session_destroy (hashcat_ctx_t *hashcat_ctx) device_param->opencl_kernel_mp_r = NULL; device_param->opencl_kernel_tm = NULL; device_param->opencl_kernel_amp = NULL; + device_param->opencl_kernel_memset = NULL; device_param->opencl_kernel_bzero = NULL; device_param->opencl_kernel_atinit = NULL; device_param->opencl_kernel_utf8toutf16le = NULL; From e889cf98a9456415cd168d10661687e5a7bb1df4 Mon Sep 17 00:00:00 2001 From: Jukka Ojanen Date: Wed, 28 Jul 2021 23:50:58 +0300 Subject: [PATCH 20/33] Few cleanups in autotune --- src/autotune.c | 113 ++++++++++--------------------------------------- 1 file changed, 22 insertions(+), 91 deletions(-) diff --git a/src/autotune.c b/src/autotune.c index e8148daf7..0db68efda 100644 --- a/src/autotune.c +++ b/src/autotune.c @@ -37,23 +37,6 @@ static double try_run (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_par device_param->spin_damp = 0; - if (device_param->is_cuda == true) - { - if (hc_cuStreamSynchronize (hashcat_ctx, device_param->cuda_stream) == -1) return -1; - } - - if (device_param->is_hip == true) - { - if (hc_hipStreamSynchronize (hashcat_ctx, device_param->hip_stream) == -1) return -1; - } - - if (device_param->is_opencl == true) - { - /*if (hc_clFlush (hashcat_ctx, device_param->opencl_command_queue) == -1) return -1;*/ - - if (hc_clFinish (hashcat_ctx, device_param->opencl_command_queue) == -1) return -1; - } - if (hashconfig->attack_exec == ATTACK_EXEC_INSIDE_KERNEL) { if (hashconfig->opti_type & OPTI_TYPE_OPTIMIZED_KERNEL) @@ -89,8 +72,6 @@ static double try_run (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_par if (device_param->is_opencl == true) { - /*if (hc_clFlush (hashcat_ctx, device_param->opencl_command_queue) == -1) return -1;*/ - if (hc_clFinish (hashcat_ctx, device_param->opencl_command_queue) == -1) return -1; } @@ -198,29 +179,19 @@ static int autotune (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param const u32 kernel_power_max = device_param->hardware_power * kernel_accel_max; - int CU_rc; - int HIP_rc; - int CL_rc; - if (device_param->is_cuda == true) { - CU_rc = run_cuda_kernel_atinit (hashcat_ctx, device_param, device_param->cuda_d_pws_buf, kernel_power_max); - - if (CU_rc == -1) return -1; + if (run_cuda_kernel_atinit (hashcat_ctx, device_param, device_param->cuda_d_pws_buf, kernel_power_max) == -1) return -1; } if (device_param->is_hip == true) { - HIP_rc = run_hip_kernel_atinit (hashcat_ctx, device_param, device_param->hip_d_pws_buf, kernel_power_max); - - if (HIP_rc == -1) return -1; + if (run_hip_kernel_atinit (hashcat_ctx, device_param, device_param->hip_d_pws_buf, kernel_power_max) == -1) return -1; } if (device_param->is_opencl == true) { - CL_rc = run_opencl_kernel_atinit (hashcat_ctx, device_param, device_param->opencl_d_pws_buf, kernel_power_max); - - if (CL_rc == -1) return -1; + if (run_opencl_kernel_atinit (hashcat_ctx, device_param, device_param->opencl_d_pws_buf, kernel_power_max) == -1) return -1; } if (user_options->slow_candidates == true) @@ -234,23 +205,19 @@ static int autotune (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param { if (device_param->is_cuda == true) { - CU_rc = hc_cuMemcpyDtoDAsync (hashcat_ctx, device_param->cuda_d_rules_c, device_param->cuda_d_rules, MIN (kernel_loops_max, KERNEL_RULES) * sizeof (kernel_rule_t), device_param->cuda_stream); - - if (CU_rc == -1) return -1; + if (hc_cuMemcpyDtoDAsync (hashcat_ctx, device_param->cuda_d_rules_c, device_param->cuda_d_rules, MIN (kernel_loops_max, KERNEL_RULES) * sizeof (kernel_rule_t), device_param->cuda_stream) == -1) return -1; } if (device_param->is_hip == true) { - HIP_rc = hc_hipMemcpyDtoDAsync (hashcat_ctx, device_param->hip_d_rules_c, device_param->hip_d_rules, MIN (kernel_loops_max, KERNEL_RULES) * sizeof (kernel_rule_t), device_param->hip_stream); - - if (HIP_rc == -1) return -1; + if (hc_hipMemcpyDtoDAsync (hashcat_ctx, device_param->hip_d_rules_c, device_param->hip_d_rules, MIN (kernel_loops_max, KERNEL_RULES) * sizeof (kernel_rule_t), device_param->hip_stream) == -1) return -1; } if (device_param->is_opencl == true) { - CL_rc = hc_clEnqueueCopyBuffer (hashcat_ctx, device_param->opencl_command_queue, device_param->opencl_d_rules, device_param->opencl_d_rules_c, 0, 0, MIN (kernel_loops_max, KERNEL_RULES) * sizeof (kernel_rule_t), 0, NULL, NULL); + if (hc_clEnqueueCopyBuffer (hashcat_ctx, device_param->opencl_command_queue, device_param->opencl_d_rules, device_param->opencl_d_rules_c, 0, 0, MIN (kernel_loops_max, KERNEL_RULES) * sizeof (kernel_rule_t), 0, NULL, NULL) == -1) return -1; - if (CL_rc == -1) return -1; + if (hc_clFlush (hashcat_ctx, device_param->opencl_command_queue) == -1) return -1; } } } @@ -420,77 +387,41 @@ static int autotune (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param if (device_param->is_cuda == true) { - int CU_rc; + if (run_cuda_kernel_bzero (hashcat_ctx, device_param, device_param->cuda_d_pws_buf, device_param->size_pws) == -1) return -1; - CU_rc = run_cuda_kernel_bzero (hashcat_ctx, device_param, device_param->cuda_d_pws_buf, device_param->size_pws); + if (run_cuda_kernel_bzero (hashcat_ctx, device_param, device_param->cuda_d_plain_bufs, device_param->size_plains) == -1) return -1; - if (CU_rc == -1) return -1; + if (run_cuda_kernel_bzero (hashcat_ctx, device_param, device_param->cuda_d_digests_shown, device_param->size_shown) == -1) return -1; - CU_rc = run_cuda_kernel_bzero (hashcat_ctx, device_param, device_param->cuda_d_plain_bufs, device_param->size_plains); + if (run_cuda_kernel_bzero (hashcat_ctx, device_param, device_param->cuda_d_result, device_param->size_results) == -1) return -1; - if (CU_rc == -1) return -1; - - CU_rc = run_cuda_kernel_bzero (hashcat_ctx, device_param, device_param->cuda_d_digests_shown, device_param->size_shown); - - if (CU_rc == -1) return -1; - - CU_rc = run_cuda_kernel_bzero (hashcat_ctx, device_param, device_param->cuda_d_result, device_param->size_results); - - if (CU_rc == -1) return -1; - - CU_rc = run_cuda_kernel_bzero (hashcat_ctx, device_param, device_param->cuda_d_tmps, device_param->size_tmps); - - if (CU_rc == -1) return -1; + if (run_cuda_kernel_bzero (hashcat_ctx, device_param, device_param->cuda_d_tmps, device_param->size_tmps) == -1) return -1; } if (device_param->is_hip == true) { - int HIP_rc; + if (run_hip_kernel_bzero (hashcat_ctx, device_param, device_param->hip_d_pws_buf, device_param->size_pws) == -1) return -1; - HIP_rc = run_hip_kernel_bzero (hashcat_ctx, device_param, device_param->hip_d_pws_buf, device_param->size_pws); + if (run_hip_kernel_bzero (hashcat_ctx, device_param, device_param->hip_d_plain_bufs, device_param->size_plains) == -1) return -1; - if (HIP_rc == -1) return -1; + if (run_hip_kernel_bzero (hashcat_ctx, device_param, device_param->hip_d_digests_shown, device_param->size_shown) == -1) return -1; - HIP_rc = run_hip_kernel_bzero (hashcat_ctx, device_param, device_param->hip_d_plain_bufs, device_param->size_plains); + if (run_hip_kernel_bzero (hashcat_ctx, device_param, device_param->hip_d_result, device_param->size_results) == -1) return -1; - if (HIP_rc == -1) return -1; - - HIP_rc = run_hip_kernel_bzero (hashcat_ctx, device_param, device_param->hip_d_digests_shown, device_param->size_shown); - - if (HIP_rc == -1) return -1; - - HIP_rc = run_hip_kernel_bzero (hashcat_ctx, device_param, device_param->hip_d_result, device_param->size_results); - - if (HIP_rc == -1) return -1; - - HIP_rc = run_hip_kernel_bzero (hashcat_ctx, device_param, device_param->hip_d_tmps, device_param->size_tmps); - - if (HIP_rc == -1) return -1; + if (run_hip_kernel_bzero (hashcat_ctx, device_param, device_param->hip_d_tmps, device_param->size_tmps) == -1) return -1; } if (device_param->is_opencl == true) { - int CL_rc; + if (run_opencl_kernel_bzero (hashcat_ctx, device_param, device_param->opencl_d_pws_buf, device_param->size_pws) == -1) return -1; - CL_rc = run_opencl_kernel_bzero (hashcat_ctx, device_param, device_param->opencl_d_pws_buf, device_param->size_pws); + if (run_opencl_kernel_bzero (hashcat_ctx, device_param, device_param->opencl_d_plain_bufs, device_param->size_plains) == -1) return -1; - if (CL_rc == -1) return -1; + if (run_opencl_kernel_bzero (hashcat_ctx, device_param, device_param->opencl_d_digests_shown, device_param->size_shown) == -1) return -1; - CL_rc = run_opencl_kernel_bzero (hashcat_ctx, device_param, device_param->opencl_d_plain_bufs, device_param->size_plains); + if (run_opencl_kernel_bzero (hashcat_ctx, device_param, device_param->opencl_d_result, device_param->size_results) == -1) return -1; - if (CL_rc == -1) return -1; - - CL_rc = run_opencl_kernel_bzero (hashcat_ctx, device_param, device_param->opencl_d_digests_shown, device_param->size_shown); - - if (CL_rc == -1) return -1; - - CL_rc = run_opencl_kernel_bzero (hashcat_ctx, device_param, device_param->opencl_d_result, device_param->size_results); - - if (CL_rc == -1) return -1; - - CL_rc = run_opencl_kernel_bzero (hashcat_ctx, device_param, device_param->opencl_d_tmps, device_param->size_tmps); - - if (CL_rc == -1) return -1; + if (run_opencl_kernel_bzero (hashcat_ctx, device_param, device_param->opencl_d_tmps, device_param->size_tmps) == -1) return -1; } // reset timer From 9ed231c99c9728c1b5036636261ea642f39b9d02 Mon Sep 17 00:00:00 2001 From: Jukka Ojanen Date: Thu, 29 Jul 2021 00:07:10 +0300 Subject: [PATCH 21/33] Add comment to blocking OpenCL calls --- src/backend.c | 2 ++ src/hashes.c | 2 ++ src/selftest.c | 2 ++ 3 files changed, 6 insertions(+) diff --git a/src/backend.c b/src/backend.c index 69b6345c6..59458e5e8 100644 --- a/src/backend.c +++ b/src/backend.c @@ -4491,6 +4491,7 @@ int gidd_to_pw_t (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param, c if (device_param->is_opencl == true) { + /* blocking */ if (hc_clEnqueueReadBuffer (hashcat_ctx, device_param->opencl_command_queue, device_param->opencl_d_pws_idx, CL_TRUE, gidd * sizeof (pw_idx_t), sizeof (pw_idx_t), &pw_idx, 0, NULL, NULL) == -1) return -1; } @@ -4530,6 +4531,7 @@ int gidd_to_pw_t (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param, c { if (cnt > 0) { + /* blocking */ if (hc_clEnqueueReadBuffer (hashcat_ctx, device_param->opencl_command_queue, device_param->opencl_d_pws_comp_buf, CL_TRUE, off * sizeof (u32), cnt * sizeof (u32), pw->i, 0, NULL, NULL) == -1) return -1; } } diff --git a/src/hashes.c b/src/hashes.c index e1f8402c2..9145c9834 100644 --- a/src/hashes.c +++ b/src/hashes.c @@ -574,6 +574,7 @@ int check_cracked (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param) if (device_param->is_opencl == true) { + /* blocking */ if (hc_clEnqueueReadBuffer (hashcat_ctx, device_param->opencl_command_queue, device_param->opencl_d_result, CL_TRUE, 0, sizeof (u32), &num_cracked, 0, NULL, NULL) == -1) return -1; } @@ -623,6 +624,7 @@ int check_cracked (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param) if (device_param->is_opencl == true) { + /* blocking */ rc = hc_clEnqueueReadBuffer (hashcat_ctx, device_param->opencl_command_queue, device_param->opencl_d_plain_bufs, CL_TRUE, 0, num_cracked * sizeof (plain_t), cracked, 0, NULL, NULL); if (rc == -1) diff --git a/src/selftest.c b/src/selftest.c index d5fff9433..af3a94550 100644 --- a/src/selftest.c +++ b/src/selftest.c @@ -523,6 +523,7 @@ static int selftest (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param if (device_param->is_opencl == true) { + /* blocking */ if (hc_clEnqueueReadBuffer (hashcat_ctx, device_param->opencl_command_queue, device_param->opencl_d_hooks, CL_TRUE, 0, device_param->size_hooks, device_param->hooks_buf, 0, NULL, NULL) == -1) return -1; } @@ -600,6 +601,7 @@ static int selftest (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param if (device_param->is_opencl == true) { + /* blocking */ if (hc_clEnqueueReadBuffer (hashcat_ctx, device_param->opencl_command_queue, device_param->opencl_d_hooks, CL_TRUE, 0, device_param->size_hooks, device_param->hooks_buf, 0, NULL, NULL) == -1) return -1; } From c48e6a25a811f46adfced436eca81edc7115d22f Mon Sep 17 00:00:00 2001 From: Jukka Ojanen Date: Thu, 29 Jul 2021 14:28:01 +0300 Subject: [PATCH 22/33] Enqueue several commands before clFlush() --- src/autotune.c | 4 ++-- src/backend.c | 29 ++++++++++------------------- src/hashes.c | 5 ----- 3 files changed, 12 insertions(+), 26 deletions(-) diff --git a/src/autotune.c b/src/autotune.c index 0db68efda..6d9f6da9a 100644 --- a/src/autotune.c +++ b/src/autotune.c @@ -216,8 +216,6 @@ static int autotune (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param if (device_param->is_opencl == true) { if (hc_clEnqueueCopyBuffer (hashcat_ctx, device_param->opencl_command_queue, device_param->opencl_d_rules, device_param->opencl_d_rules_c, 0, 0, MIN (kernel_loops_max, KERNEL_RULES) * sizeof (kernel_rule_t), 0, NULL, NULL) == -1) return -1; - - if (hc_clFlush (hashcat_ctx, device_param->opencl_command_queue) == -1) return -1; } } } @@ -422,6 +420,8 @@ static int autotune (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param if (run_opencl_kernel_bzero (hashcat_ctx, device_param, device_param->opencl_d_result, device_param->size_results) == -1) return -1; if (run_opencl_kernel_bzero (hashcat_ctx, device_param, device_param->opencl_d_tmps, device_param->size_tmps) == -1) return -1; + + if (hc_clFlush (hashcat_ctx, device_param->opencl_command_queue) == -1) return -1; } // reset timer diff --git a/src/backend.c b/src/backend.c index 59458e5e8..4d08dc875 100644 --- a/src/backend.c +++ b/src/backend.c @@ -4602,6 +4602,8 @@ int choose_kernel (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param, if (device_param->is_opencl == true) { if (hc_clEnqueueCopyBuffer (hashcat_ctx, device_param->opencl_command_queue, device_param->opencl_d_tm_c, device_param->opencl_d_bfs_c, 0, 0, size_tm, 0, NULL, NULL) == -1) return -1; + + if (hc_clFlush (hashcat_ctx, device_param->opencl_command_queue) == -1) return -1; } } } @@ -5270,11 +5272,7 @@ int run_opencl_kernel_atinit (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *dev if (hc_clSetKernelArg (hashcat_ctx, kernel, 1, sizeof (cl_ulong), device_param->kernel_params_atinit[1]) == -1) return -1; - if (hc_clEnqueueNDRangeKernel (hashcat_ctx, device_param->opencl_command_queue, kernel, 1, NULL, global_work_size, local_work_size, 0, NULL, NULL) == -1) return -1; - - /*if (hc_clFlush (hashcat_ctx, device_param->opencl_command_queue) == -1) return -1;*/ - - return 0; + return hc_clEnqueueNDRangeKernel (hashcat_ctx, device_param->opencl_command_queue, kernel, 1, NULL, global_work_size, local_work_size, 0, NULL, NULL); } int run_opencl_kernel_utf8toutf16le (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param, cl_mem buf, const u64 num) @@ -5296,11 +5294,7 @@ int run_opencl_kernel_utf8toutf16le (hashcat_ctx_t *hashcat_ctx, hc_device_param if (hc_clSetKernelArg (hashcat_ctx, kernel, 1, sizeof (cl_ulong), device_param->kernel_params_utf8toutf16le[1]) == -1) return -1; - if (hc_clEnqueueNDRangeKernel (hashcat_ctx, device_param->opencl_command_queue, kernel, 1, NULL, global_work_size, local_work_size, 0, NULL, NULL) == -1) return -1; - - /*if (hc_clFlush (hashcat_ctx, device_param->opencl_command_queue) == -1) return -1;*/ - - return 0; + return hc_clEnqueueNDRangeKernel (hashcat_ctx, device_param->opencl_command_queue, kernel, 1, NULL, global_work_size, local_work_size, 0, NULL, NULL); } int run_opencl_kernel_memset (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param, cl_mem buf, const u64 offset, const u8 value, const u64 size) @@ -5391,8 +5385,6 @@ int run_opencl_kernel_bzero (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *devi if (hc_clEnqueueWriteBuffer (hashcat_ctx, device_param->opencl_command_queue, buf, CL_FALSE, num16d * 16, num16m, bzeros, 0, NULL, NULL) == -1) return -1; } - /*if (hc_clFlush (hashcat_ctx, device_param->opencl_command_queue) == -1) return -1;*/ - return 0; } @@ -5760,7 +5752,7 @@ int run_kernel (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param, con if (hc_clEnqueueNDRangeKernel (hashcat_ctx, device_param->opencl_command_queue, opencl_kernel, 1, NULL, global_work_size, local_work_size, 0, NULL, &opencl_event) == -1) return -1; - /*if (hc_clFlush (hashcat_ctx, device_param->opencl_command_queue) == -1) return -1;*/ + if (hc_clFlush (hashcat_ctx, device_param->opencl_command_queue) == -1) return -1; // spin damper section @@ -5978,8 +5970,6 @@ int run_kernel_mp (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param, const size_t local_work_size[3] = { kernel_threads, 1, 1 }; if (hc_clEnqueueNDRangeKernel (hashcat_ctx, device_param->opencl_command_queue, opencl_kernel, 1, NULL, global_work_size, local_work_size, 0, NULL, NULL) == -1) return -1; - - /*if (hc_clFlush (hashcat_ctx, device_param->opencl_command_queue) == -1) return -1;*/ } return 0; @@ -6013,8 +6003,6 @@ int run_kernel_tm (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param) const size_t local_work_size[3] = { kernel_threads, 1, 1 }; if (hc_clEnqueueNDRangeKernel (hashcat_ctx, device_param->opencl_command_queue, cuda_kernel, 1, NULL, global_work_size, local_work_size, 0, NULL, NULL) == -1) return -1; - - /*if (hc_clFlush (hashcat_ctx, device_param->opencl_command_queue) == -1) return -1;*/ } return 0; @@ -6103,8 +6091,6 @@ int run_kernel_decompress (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device if (hc_clSetKernelArg (hashcat_ctx, opencl_kernel, 3, sizeof (cl_ulong), device_param->kernel_params_decompress[3]) == -1) return -1; if (hc_clEnqueueNDRangeKernel (hashcat_ctx, device_param->opencl_command_queue, opencl_kernel, 1, NULL, global_work_size, local_work_size, 0, NULL, NULL) == -1) return -1; - - /*if (hc_clFlush (hashcat_ctx, device_param->opencl_command_queue) == -1) return -1;*/ } return 0; @@ -6421,6 +6407,11 @@ int run_copy (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param, const } } + if (device_param->is_opencl == true) + { + if (hc_clFlush (hashcat_ctx, device_param->opencl_command_queue) == -1) return -1; + } + return 0; } diff --git a/src/hashes.c b/src/hashes.c index 9145c9834..241b653cb 100644 --- a/src/hashes.c +++ b/src/hashes.c @@ -706,11 +706,6 @@ int check_cracked (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param) /* NOTE: run_opencl_kernel_bzero() does not handle buffer offset */ rc = run_opencl_kernel_memset32 (hashcat_ctx, device_param, device_param->opencl_d_digests_shown, salt_buf->digests_offset, 0, salt_buf->digests_cnt); - if (rc == 0) - { - rc = hc_clFlush (hashcat_ctx, device_param->opencl_command_queue); - } - if (rc == -1) { break; From 7a8065d090027418d452876109f386ee3aaf64c1 Mon Sep 17 00:00:00 2001 From: Jukka Ojanen Date: Thu, 29 Jul 2021 14:39:11 +0300 Subject: [PATCH 23/33] Do not call clWaitForEvents() after spin damper when we know that event status is CL_COMPLETE --- src/backend.c | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/src/backend.c b/src/backend.c index 4d08dc875..4e4581d99 100644 --- a/src/backend.c +++ b/src/backend.c @@ -5752,8 +5752,6 @@ int run_kernel (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param, con if (hc_clEnqueueNDRangeKernel (hashcat_ctx, device_param->opencl_command_queue, opencl_kernel, 1, NULL, global_work_size, local_work_size, 0, NULL, &opencl_event) == -1) return -1; - if (hc_clFlush (hashcat_ctx, device_param->opencl_command_queue) == -1) return -1; - // spin damper section const u32 iterationm = iteration % EXPECTED_ITERATIONS; @@ -5764,6 +5762,8 @@ int run_kernel (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param, con size_t param_value_size_ret; + if (hc_clFlush (hashcat_ctx, device_param->opencl_command_queue) == -1) return -1; + if (hc_clGetEventInfo (hashcat_ctx, opencl_event, CL_EVENT_COMMAND_EXECUTION_STATUS, sizeof (opencl_event_status), &opencl_event_status, ¶m_value_size_ret) == -1) return -1; double spin_total = device_param->spin_damp; @@ -5800,11 +5800,18 @@ int run_kernel (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param, con spin_total += device_param->spin_damp; - if (spin_total > 1) break; + if (spin_total > 1) + { + if (hc_clWaitForEvents (hashcat_ctx, 1, &opencl_event) == -1) return -1; + + break; + } } } - - if (hc_clWaitForEvents (hashcat_ctx, 1, &opencl_event) == -1) return -1; + else + { + if (hc_clWaitForEvents (hashcat_ctx, 1, &opencl_event) == -1) return -1; + } cl_ulong time_start; cl_ulong time_end; From 8d51fca192040a6e5b30ec8b8049d3f27ff3525c Mon Sep 17 00:00:00 2001 From: Jukka Ojanen Date: Thu, 29 Jul 2021 14:40:45 +0300 Subject: [PATCH 24/33] No need to synchronize after run_kernel() --- src/autotune.c | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/src/autotune.c b/src/autotune.c index 6d9f6da9a..8941c71b9 100644 --- a/src/autotune.c +++ b/src/autotune.c @@ -60,21 +60,6 @@ static double try_run (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_par run_kernel (hashcat_ctx, device_param, KERN_RUN_2, 0, kernel_power_try, true, 0); } - if (device_param->is_cuda == true) - { - if (hc_cuStreamSynchronize (hashcat_ctx, device_param->cuda_stream) == -1) return -1; - } - - if (device_param->is_hip == true) - { - if (hc_hipStreamSynchronize (hashcat_ctx, device_param->hip_stream) == -1) return -1; - } - - if (device_param->is_opencl == true) - { - if (hc_clFinish (hashcat_ctx, device_param->opencl_command_queue) == -1) return -1; - } - device_param->spin_damp = spin_damp_sav; const double exec_msec_prev = get_avg_exec_time (device_param, 1); From 81c2ec3caf7780eb26bf061bd89cde353342ee85 Mon Sep 17 00:00:00 2001 From: Jukka Ojanen Date: Thu, 29 Jul 2021 14:48:36 +0300 Subject: [PATCH 25/33] Small cleanup in gidd_to_pw_t() --- src/backend.c | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/src/backend.c b/src/backend.c index 4e4581d99..1dbe2ec16 100644 --- a/src/backend.c +++ b/src/backend.c @@ -4499,9 +4499,9 @@ int gidd_to_pw_t (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param, c const u32 cnt = pw_idx.cnt; const u32 len = pw_idx.len; - if (device_param->is_cuda == true) + if (cnt > 0) { - if (cnt > 0) + if (device_param->is_cuda == true) { if (hc_cuCtxPushCurrent (hashcat_ctx, device_param->cuda_context) == -1) return -1; @@ -4511,11 +4511,8 @@ int gidd_to_pw_t (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param, c if (hc_cuCtxPopCurrent (hashcat_ctx, &device_param->cuda_context) == -1) return -1; } - } - if (device_param->is_hip == true) - { - if (cnt > 0) + if (device_param->is_hip == true) { if (hc_hipCtxPushCurrent (hashcat_ctx, device_param->hip_context) == -1) return -1; @@ -4525,11 +4522,8 @@ int gidd_to_pw_t (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param, c if (hc_hipCtxPopCurrent (hashcat_ctx, &device_param->hip_context) == -1) return -1; } - } - if (device_param->is_opencl == true) - { - if (cnt > 0) + if (device_param->is_opencl == true) { /* blocking */ if (hc_clEnqueueReadBuffer (hashcat_ctx, device_param->opencl_command_queue, device_param->opencl_d_pws_comp_buf, CL_TRUE, off * sizeof (u32), cnt * sizeof (u32), pw->i, 0, NULL, NULL) == -1) return -1; From 1064cce08cc85001f30af73fc6e644977422dc8f Mon Sep 17 00:00:00 2001 From: Jukka Ojanen Date: Thu, 29 Jul 2021 15:22:25 +0300 Subject: [PATCH 26/33] Synchronize before hooks --- src/backend.c | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/src/backend.c b/src/backend.c index 1dbe2ec16..595fd3508 100644 --- a/src/backend.c +++ b/src/backend.c @@ -4701,16 +4701,21 @@ int choose_kernel (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param, if (device_param->is_cuda == true) { if (hc_cuMemcpyDtoHAsync (hashcat_ctx, device_param->hooks_buf, device_param->cuda_d_hooks, pws_cnt * hashconfig->hook_size, device_param->cuda_stream) == -1) return -1; + + if (hc_cuStreamSynchronize (hashcat_ctx, device_param->cuda_stream) == -1) return -1; } if (device_param->is_hip == true) { if (hc_hipMemcpyDtoHAsync (hashcat_ctx, device_param->hooks_buf, device_param->hip_d_hooks, pws_cnt * hashconfig->hook_size, device_param->hip_stream) == -1) return -1; + + if (hc_hipStreamSynchronize (hashcat_ctx, device_param->hip_stream) == -1) return -1; } if (device_param->is_opencl == true) { - if (hc_clEnqueueReadBuffer (hashcat_ctx, device_param->opencl_command_queue, device_param->opencl_d_hooks, CL_FALSE, 0, pws_cnt * hashconfig->hook_size, device_param->hooks_buf, 0, NULL, NULL) == -1) return -1; + /* blocking */ + if (hc_clEnqueueReadBuffer (hashcat_ctx, device_param->opencl_command_queue, device_param->opencl_d_hooks, CL_TRUE, 0, pws_cnt * hashconfig->hook_size, device_param->hooks_buf, 0, NULL, NULL) == -1) return -1; } const int hook_threads = (int) user_options->hook_threads; @@ -4849,16 +4854,21 @@ int choose_kernel (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param, if (device_param->is_cuda == true) { if (hc_cuMemcpyDtoHAsync (hashcat_ctx, device_param->hooks_buf, device_param->cuda_d_hooks, pws_cnt * hashconfig->hook_size, device_param->cuda_stream) == -1) return -1; + + if (hc_cuStreamSynchronize (hashcat_ctx, device_param->cuda_stream) == -1) return -1; } if (device_param->is_hip == true) { if (hc_hipMemcpyDtoHAsync (hashcat_ctx, device_param->hooks_buf, device_param->hip_d_hooks, pws_cnt * hashconfig->hook_size, device_param->hip_stream) == -1) return -1; + + if (hc_hipStreamSynchronize (hashcat_ctx, device_param->hip_stream) == -1) return -1; } if (device_param->is_opencl == true) { - if (hc_clEnqueueReadBuffer (hashcat_ctx, device_param->opencl_command_queue, device_param->opencl_d_hooks, CL_FALSE, 0, pws_cnt * hashconfig->hook_size, device_param->hooks_buf, 0, NULL, NULL) == -1) return -1; + /* blocking */ + if (hc_clEnqueueReadBuffer (hashcat_ctx, device_param->opencl_command_queue, device_param->opencl_d_hooks, CL_TRUE, 0, pws_cnt * hashconfig->hook_size, device_param->hooks_buf, 0, NULL, NULL) == -1) return -1; } const int hook_threads = (int) user_options->hook_threads; @@ -6047,8 +6057,6 @@ int run_kernel_amp (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param, const size_t local_work_size[3] = { kernel_threads, 1, 1 }; if (hc_clEnqueueNDRangeKernel (hashcat_ctx, device_param->opencl_command_queue, opencl_kernel, 1, NULL, global_work_size, local_work_size, 0, NULL, NULL) == -1) return -1; - - /*if (hc_clFlush (hashcat_ctx, device_param->opencl_command_queue) == -1) return -1;*/ } return 0; @@ -12510,6 +12518,8 @@ int backend_session_begin (hashcat_ctx_t *hashcat_ctx) if (hc_clEnqueueWriteBuffer (hashcat_ctx, device_param->opencl_command_queue, device_param->opencl_d_st_esalts_buf, CL_FALSE, 0, size_st_esalts, hashes->st_esalts_buf, 0, NULL, NULL) == -1) return -1; } } + + if (hc_clFlush (hashcat_ctx, device_param->opencl_command_queue) == -1) return -1; } /** From d721deb17a5df30ed57cc0c78a4dc5b7ebab4db5 Mon Sep 17 00:00:00 2001 From: Jukka Ojanen Date: Thu, 29 Jul 2021 15:23:24 +0300 Subject: [PATCH 27/33] Move variables higher to avoid async execution stack problems --- src/selftest.c | 22 ++++++---------------- 1 file changed, 6 insertions(+), 16 deletions(-) diff --git a/src/selftest.c b/src/selftest.c index af3a94550..e9367e152 100644 --- a/src/selftest.c +++ b/src/selftest.c @@ -72,6 +72,10 @@ static int selftest (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param tmp.pw_len = (u32) tmp_len; } + pw_t pw; + pw_t comb; + bf_t bf; + u32 highest_pw_len = 0; if (user_options->slow_candidates == true) @@ -81,8 +85,6 @@ static int selftest (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param device_param->kernel_params_buf32[30] = 1; } - pw_t pw; - memset (&pw, 0, sizeof (pw)); char *pw_ptr = (char *) &pw.i; @@ -116,8 +118,6 @@ static int selftest (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param { device_param->kernel_params_buf32[30] = 1; - pw_t pw; - memset (&pw, 0, sizeof (pw)); char *pw_ptr = (char *) &pw.i; @@ -153,8 +153,6 @@ static int selftest (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param device_param->kernel_params_buf32[30] = 1; device_param->kernel_params_buf32[33] = COMBINATOR_MODE_BASE_LEFT; - pw_t pw; - memset (&pw, 0, sizeof (pw)); char *pw_ptr = (char *) &pw.i; @@ -170,8 +168,6 @@ static int selftest (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param uppercase ((u8 *) pw_ptr, pw.pw_len); } - pw_t comb; - memset (&comb, 0, sizeof (comb)); char *comb_ptr = (char *) &comb.i; @@ -227,8 +223,6 @@ static int selftest (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param if (hashconfig->opts_type & OPTS_TYPE_TM_KERNEL) { - pw_t pw; - memset (&pw, 0, sizeof (pw)); char *pw_ptr = (char *) &pw.i; @@ -261,8 +255,6 @@ static int selftest (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param } else { - bf_t bf; - memset (&bf, 0, sizeof (bf)); char *bf_ptr = (char *) &bf.i; @@ -315,8 +307,6 @@ static int selftest (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param if (hc_clEnqueueWriteBuffer (hashcat_ctx, device_param->opencl_command_queue, device_param->opencl_d_bfs_c, CL_FALSE, 0, 1 * sizeof (bf_t), &bf, 0, NULL, NULL) == -1) return -1; } - pw_t pw; - memset (&pw, 0, sizeof (pw)); char *pw_ptr = (char *) &pw.i; @@ -422,8 +412,6 @@ static int selftest (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param } else { - pw_t pw; - memset (&pw, 0, sizeof (pw)); char *pw_ptr = (char *) &pw.i; @@ -710,6 +698,8 @@ static int selftest (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param if (device_param->is_opencl == true) { if (hc_clEnqueueReadBuffer (hashcat_ctx, device_param->opencl_command_queue, device_param->opencl_d_result, CL_FALSE, 0, sizeof (u32), &num_cracked, 0, NULL, &opencl_event) == -1) return -1; + + if (hc_clFlush (hashcat_ctx, device_param->opencl_command_queue) == -1) return -1; } // finish : cleanup and restore From e154f9e781cfc15b3eba10ba1136da2135358a92 Mon Sep 17 00:00:00 2001 From: Jukka Ojanen Date: Fri, 30 Jul 2021 12:24:21 +0300 Subject: [PATCH 28/33] Few cleanups --- src/backend.c | 24 +++++------------------- src/hashes.c | 4 ++-- 2 files changed, 7 insertions(+), 21 deletions(-) diff --git a/src/backend.c b/src/backend.c index b399ec492..435d06097 100644 --- a/src/backend.c +++ b/src/backend.c @@ -4480,7 +4480,8 @@ int choose_kernel (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param, const int hook_threads = (int) user_options->hook_threads; - hook_thread_param_t *hook_threads_param = (hook_thread_param_t *) hccalloc (hook_threads, sizeof (hook_thread_param_t)); + hook_thread_param_t *hook_threads_param = (hook_thread_param_t *) hcmalloc (hook_threads * sizeof (hook_thread_param_t)); + hc_thread_t *c_threads = (hc_thread_t *) hcmalloc (hook_threads * sizeof (hc_thread_t)); for (int i = 0; i < hook_threads; i++) { @@ -4500,13 +4501,6 @@ int choose_kernel (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param, hook_thread_param->salt_pos = salt_pos; hook_thread_param->pws_cnt = pws_cnt; - } - - hc_thread_t *c_threads = (hc_thread_t *) hccalloc (hook_threads, sizeof (hc_thread_t)); - - for (int i = 0; i < hook_threads; i++) - { - hook_thread_param_t *hook_thread_param = hook_threads_param + i; hc_thread_create (c_threads[i], hook12_thread, hook_thread_param); } @@ -4514,7 +4508,6 @@ int choose_kernel (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param, hc_thread_wait (hook_threads, c_threads); hcfree (c_threads); - hcfree (hook_threads_param); if (device_param->is_cuda == true) @@ -4633,7 +4626,8 @@ int choose_kernel (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param, const int hook_threads = (int) user_options->hook_threads; - hook_thread_param_t *hook_threads_param = (hook_thread_param_t *) hccalloc (hook_threads, sizeof (hook_thread_param_t)); + hook_thread_param_t *hook_threads_param = (hook_thread_param_t *) hcmalloc (hook_threads * sizeof (hook_thread_param_t)); + hc_thread_t *c_threads = (hc_thread_t *) hcmalloc (hook_threads * sizeof (hc_thread_t)); for (int i = 0; i < hook_threads; i++) { @@ -4653,13 +4647,6 @@ int choose_kernel (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param, hook_thread_param->salt_pos = salt_pos; hook_thread_param->pws_cnt = pws_cnt; - } - - hc_thread_t *c_threads = (hc_thread_t *) hccalloc (hook_threads, sizeof (hc_thread_t)); - - for (int i = 0; i < hook_threads; i++) - { - hook_thread_param_t *hook_thread_param = hook_threads_param + i; hc_thread_create (c_threads[i], hook23_thread, hook_thread_param); } @@ -4667,7 +4654,6 @@ int choose_kernel (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param, hc_thread_wait (hook_threads, c_threads); hcfree (c_threads); - hcfree (hook_threads_param); if (device_param->is_cuda == true) @@ -5102,7 +5088,7 @@ int run_opencl_kernel_memset32 (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *d { const u64 len = offset + size; - u32 *tmp = (u32 *) hcmalloc ((offset + size) * sizeof (u32)); + u32 *tmp = (u32 *) hcmalloc (len * sizeof (u32)); for (u64 i = 0; i < len; i++) { diff --git a/src/hashes.c b/src/hashes.c index 2fbf9d0cf..ef654e672 100644 --- a/src/hashes.c +++ b/src/hashes.c @@ -580,13 +580,13 @@ int check_cracked (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param) if (num_cracked == 0 || user_options->speed_only == true) { - // we want get num_cracked in benchmark mode because it has an influence in performance + // we want to get the num_cracked in benchmark mode because it has an influence in performance // however if the benchmark cracks the artificial hash used for benchmarks we don't want to see that! return 0; } - plain_t *cracked = (plain_t *) hccalloc (num_cracked, sizeof (plain_t)); + plain_t *cracked = (plain_t *) hcmalloc (num_cracked * sizeof (plain_t)); if (device_param->is_cuda == true) { From e352a79a05fd60dec8f63b6454c96c0e49f87a55 Mon Sep 17 00:00:00 2001 From: Jukka Ojanen Date: Sun, 1 Aug 2021 14:15:03 +0300 Subject: [PATCH 29/33] Extend context in gidd_to_pw_t() --- src/backend.c | 26 ++++++++++++-------------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/src/backend.c b/src/backend.c index 435d06097..0d7741593 100644 --- a/src/backend.c +++ b/src/backend.c @@ -4234,8 +4234,6 @@ int gidd_to_pw_t (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param, c if (hc_cuMemcpyDtoHAsync (hashcat_ctx, &pw_idx, device_param->cuda_d_pws_idx + (gidd * sizeof (pw_idx_t)), sizeof (pw_idx_t), device_param->cuda_stream) == -1) return -1; if (hc_cuStreamSynchronize (hashcat_ctx, device_param->cuda_stream) == -1) return -1; - - if (hc_cuCtxPopCurrent (hashcat_ctx, &device_param->cuda_context) == -1) return -1; } if (device_param->is_hip == true) @@ -4245,8 +4243,6 @@ int gidd_to_pw_t (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param, c if (hc_hipMemcpyDtoHAsync (hashcat_ctx, &pw_idx, device_param->hip_d_pws_idx + (gidd * sizeof (pw_idx_t)), sizeof (pw_idx_t), device_param->hip_stream) == -1) return -1; if (hc_hipStreamSynchronize (hashcat_ctx, device_param->hip_stream) == -1) return -1; - - if (hc_hipCtxPopCurrent (hashcat_ctx, &device_param->hip_context) == -1) return -1; } if (device_param->is_opencl == true) @@ -4263,24 +4259,16 @@ int gidd_to_pw_t (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param, c { if (device_param->is_cuda == true) { - if (hc_cuCtxPushCurrent (hashcat_ctx, device_param->cuda_context) == -1) return -1; - - if (hc_cuMemcpyDtoHAsync (hashcat_ctx,pw->i, device_param->cuda_d_pws_comp_buf + (off * sizeof (u32)), cnt * sizeof (u32), device_param->cuda_stream) == -1) return -1; + if (hc_cuMemcpyDtoHAsync (hashcat_ctx, pw->i, device_param->cuda_d_pws_comp_buf + (off * sizeof (u32)), cnt * sizeof (u32), device_param->cuda_stream) == -1) return -1; if (hc_cuStreamSynchronize (hashcat_ctx, device_param->cuda_stream) == -1) return -1; - - if (hc_cuCtxPopCurrent (hashcat_ctx, &device_param->cuda_context) == -1) return -1; } if (device_param->is_hip == true) { - if (hc_hipCtxPushCurrent (hashcat_ctx, device_param->hip_context) == -1) return -1; - - if (hc_hipMemcpyDtoHAsync (hashcat_ctx,pw->i, device_param->hip_d_pws_comp_buf + (off * sizeof (u32)), cnt * sizeof (u32), device_param->hip_stream) == -1) return -1; + if (hc_hipMemcpyDtoHAsync (hashcat_ctx, pw->i, device_param->hip_d_pws_comp_buf + (off * sizeof (u32)), cnt * sizeof (u32), device_param->hip_stream) == -1) return -1; if (hc_hipStreamSynchronize (hashcat_ctx, device_param->hip_stream) == -1) return -1; - - if (hc_hipCtxPopCurrent (hashcat_ctx, &device_param->hip_context) == -1) return -1; } if (device_param->is_opencl == true) @@ -4297,6 +4285,16 @@ int gidd_to_pw_t (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param, c pw->pw_len = len; + if (device_param->is_cuda == true) + { + if (hc_cuCtxPopCurrent (hashcat_ctx, &device_param->cuda_context) == -1) return -1; + } + + if (device_param->is_hip == true) + { + if (hc_hipCtxPopCurrent (hashcat_ctx, &device_param->hip_context) == -1) return -1; + } + return 0; } From 1ee222d43f3d923f2539f7f2931c1ae7761aee0a Mon Sep 17 00:00:00 2001 From: Jukka Ojanen Date: Sun, 1 Aug 2021 14:21:18 +0300 Subject: [PATCH 30/33] Replace free() with hcfree() --- src/backend.c | 6 +++--- src/hashes.c | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/backend.c b/src/backend.c index 0d7741593..d870602e0 100644 --- a/src/backend.c +++ b/src/backend.c @@ -5059,12 +5059,12 @@ int run_opencl_kernel_memset (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *dev char *tmp = hcmalloc (len * sizeof (u8)); - memset(tmp, value, len); + memset (tmp, value, len); /* blocking */ rc = hc_clEnqueueWriteBuffer (hashcat_ctx, device_param->opencl_command_queue, buf, CL_TRUE, offset, size, tmp, 0, NULL, NULL); - free(tmp); + hcfree (tmp); } else { @@ -5096,7 +5096,7 @@ int run_opencl_kernel_memset32 (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *d /* blocking */ rc = hc_clEnqueueWriteBuffer (hashcat_ctx, device_param->opencl_command_queue, buf, CL_TRUE, offset * sizeof (u32), size * sizeof (u32), tmp, 0, NULL, NULL); - free(tmp); + hcfree (tmp); } else { diff --git a/src/hashes.c b/src/hashes.c index ef654e672..28f5396b7 100644 --- a/src/hashes.c +++ b/src/hashes.c @@ -331,7 +331,7 @@ int check_hash (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param, pla if (rc == -1) { - free(tmps); + hcfree (tmps); return -1; } @@ -348,7 +348,7 @@ int check_hash (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param, pla if (rc == -1) { - free(tmps); + hcfree (tmps); return -1; } @@ -365,7 +365,7 @@ int check_hash (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param, pla if (rc == -1) { - free(tmps); + hcfree (tmps); return -1; } From dbe2bad0989e94592d6ea6b8f01256aeefd642c2 Mon Sep 17 00:00:00 2001 From: Jukka Ojanen Date: Sun, 1 Aug 2021 14:26:33 +0300 Subject: [PATCH 31/33] Maintain code style --- src/backend.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/backend.c b/src/backend.c index d870602e0..bc1064c34 100644 --- a/src/backend.c +++ b/src/backend.c @@ -5020,7 +5020,9 @@ int run_opencl_kernel_atinit (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *dev if (hc_clSetKernelArg (hashcat_ctx, kernel, 1, sizeof (cl_ulong), device_param->kernel_params_atinit[1]) == -1) return -1; - return hc_clEnqueueNDRangeKernel (hashcat_ctx, device_param->opencl_command_queue, kernel, 1, NULL, global_work_size, local_work_size, 0, NULL, NULL); + if (hc_clEnqueueNDRangeKernel (hashcat_ctx, device_param->opencl_command_queue, kernel, 1, NULL, global_work_size, local_work_size, 0, NULL, NULL) == -1) return -1; + + return 0; } int run_opencl_kernel_utf8toutf16le (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param, cl_mem buf, const u64 num) @@ -5042,7 +5044,9 @@ int run_opencl_kernel_utf8toutf16le (hashcat_ctx_t *hashcat_ctx, hc_device_param if (hc_clSetKernelArg (hashcat_ctx, kernel, 1, sizeof (cl_ulong), device_param->kernel_params_utf8toutf16le[1]) == -1) return -1; - return hc_clEnqueueNDRangeKernel (hashcat_ctx, device_param->opencl_command_queue, kernel, 1, NULL, global_work_size, local_work_size, 0, NULL, NULL); + if (hc_clEnqueueNDRangeKernel (hashcat_ctx, device_param->opencl_command_queue, kernel, 1, NULL, global_work_size, local_work_size, 0, NULL, NULL) == -1) return -1; + + return 0; } int run_opencl_kernel_memset (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param, cl_mem buf, const u64 offset, const u8 value, const u64 size) From fdbfae9a287e694539e50a9eb4e7ce3ebcc43819 Mon Sep 17 00:00:00 2001 From: Jukka Ojanen Date: Sun, 1 Aug 2021 14:47:08 +0300 Subject: [PATCH 32/33] Modify OpenCL clEnqueueFillBuffer() workaround --- src/backend.c | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/src/backend.c b/src/backend.c index bc1064c34..2b43c008b 100644 --- a/src/backend.c +++ b/src/backend.c @@ -5059,11 +5059,9 @@ int run_opencl_kernel_memset (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *dev /* workaround if missing clEnqueueFillBuffer() */ if (ocl->clEnqueueFillBuffer == NULL) { - const u64 len = offset + size; + char *tmp = hcmalloc (size * sizeof (u8)); - char *tmp = hcmalloc (len * sizeof (u8)); - - memset (tmp, value, len); + memset (tmp, value, size); /* blocking */ rc = hc_clEnqueueWriteBuffer (hashcat_ctx, device_param->opencl_command_queue, buf, CL_TRUE, offset, size, tmp, 0, NULL, NULL); @@ -5088,11 +5086,9 @@ int run_opencl_kernel_memset32 (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *d /* workaround if missing clEnqueueFillBuffer() */ if (ocl->clEnqueueFillBuffer == NULL) { - const u64 len = offset + size; + u32 *tmp = (u32 *) hcmalloc (size * sizeof (u32)); - u32 *tmp = (u32 *) hcmalloc (len * sizeof (u32)); - - for (u64 i = 0; i < len; i++) + for (u64 i = 0; i < size; i++) { tmp[i] = value; } From 62a06f735f7b7c10e7a50eea36e81bed06e4586f Mon Sep 17 00:00:00 2001 From: Jukka Ojanen Date: Sun, 1 Aug 2021 15:19:15 +0300 Subject: [PATCH 33/33] kernel_memset32(): assume offset and size are bytes, not elements --- src/backend.c | 30 +++++++++++++++++++++++++----- src/hashes.c | 2 +- 2 files changed, 26 insertions(+), 6 deletions(-) diff --git a/src/backend.c b/src/backend.c index 2b43c008b..4801acf38 100644 --- a/src/backend.c +++ b/src/backend.c @@ -4898,7 +4898,13 @@ int run_cuda_kernel_memset (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *devic int run_cuda_kernel_memset32 (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param, CUdeviceptr buf, const u64 offset, const u32 value, const u64 size) { - return hc_cuMemsetD32Async (hashcat_ctx, buf + offset * sizeof (u32), value, size, device_param->cuda_stream); + /* check that the size is multiple of element size */ + if (size % 4 != 0) + { + return CUDA_ERROR_INVALID_VALUE; + } + + return hc_cuMemsetD32Async (hashcat_ctx, buf + offset, value, size / 4, device_param->cuda_stream); } int run_cuda_kernel_bzero (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param, CUdeviceptr buf, const u64 size) @@ -4971,7 +4977,13 @@ int run_hip_kernel_memset (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device int run_hip_kernel_memset32 (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param, hipDeviceptr_t buf, const u64 offset, const u32 value, const u64 size) { - return hc_hipMemsetD32Async (hashcat_ctx, buf + offset * sizeof (u32), value, size, device_param->hip_stream); + /* check that the size is multiple of element size */ + if (size % 4 != 0) + { + return hipErrorInvalidValue; + } + + return hc_hipMemsetD32Async (hashcat_ctx, buf + offset, value, size / 4, device_param->hip_stream); } int run_hip_kernel_bzero (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param, hipDeviceptr_t buf, const u64 size) @@ -5086,15 +5098,23 @@ int run_opencl_kernel_memset32 (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *d /* workaround if missing clEnqueueFillBuffer() */ if (ocl->clEnqueueFillBuffer == NULL) { - u32 *tmp = (u32 *) hcmalloc (size * sizeof (u32)); + const u64 N = size / 4; - for (u64 i = 0; i < size; i++) + /* check that the size is multiple of element size */ + if (size % 4 != 0) + { + return CL_INVALID_VALUE; + } + + u32 *tmp = (u32 *) hcmalloc (size); + + for (u64 i = 0; i < N; i++) { tmp[i] = value; } /* blocking */ - rc = hc_clEnqueueWriteBuffer (hashcat_ctx, device_param->opencl_command_queue, buf, CL_TRUE, offset * sizeof (u32), size * sizeof (u32), tmp, 0, NULL, NULL); + rc = hc_clEnqueueWriteBuffer (hashcat_ctx, device_param->opencl_command_queue, buf, CL_TRUE, offset, size, tmp, 0, NULL, NULL); hcfree (tmp); } diff --git a/src/hashes.c b/src/hashes.c index 28f5396b7..80fa6cede 100644 --- a/src/hashes.c +++ b/src/hashes.c @@ -704,7 +704,7 @@ int check_cracked (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param) if (device_param->is_opencl == true) { /* NOTE: run_opencl_kernel_bzero() does not handle buffer offset */ - rc = run_opencl_kernel_memset32 (hashcat_ctx, device_param, device_param->opencl_d_digests_shown, salt_buf->digests_offset, 0, salt_buf->digests_cnt); + rc = run_opencl_kernel_memset32 (hashcat_ctx, device_param, device_param->opencl_d_digests_shown, salt_buf->digests_offset * sizeof (u32), 0, salt_buf->digests_cnt * sizeof (u32)); if (rc == -1) {