From 1b4edf879b5f37dd32da8c525c730ebfcb110eef Mon Sep 17 00:00:00 2001 From: Jukka Ojanen Date: Wed, 1 Sep 2021 14:28:24 +0300 Subject: [PATCH 1/3] Set GZIP internal buffer size to 256k (default 8k) --- src/filehandling.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/filehandling.c b/src/filehandling.c index 48c2831b2..d1c094535 100644 --- a/src/filehandling.c +++ b/src/filehandling.c @@ -9,6 +9,10 @@ #include "shared.h" #include "filehandling.h" +#ifndef HCFILE_BUFFER_SIZE +#define HCFILE_BUFFER_SIZE 256 * 1024 +#endif + #if defined (__CYGWIN__) // workaround for zlib with cygwin build int _wopen (const char *path, int oflag, ...) @@ -112,6 +116,8 @@ bool hc_fopen (HCFILE *fp, const char *path, const char *mode) if (is_gzip) { if ((fp->gfp = gzdopen (fp->fd, mode)) == NULL) return false; + + gzbuffer (fp->gfp, HCFILE_BUFFER_SIZE); } else { From 213b5339758b6eb6e457b77e642604e59a3577fe Mon Sep 17 00:00:00 2001 From: Jukka Ojanen Date: Thu, 2 Sep 2021 12:00:46 +0300 Subject: [PATCH 2/3] Use HCFILE_BUFFER_SIZE for xz stream --- src/filehandling.c | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/src/filehandling.c b/src/filehandling.c index 3c3a8cffa..3ca4a97d6 100644 --- a/src/filehandling.c +++ b/src/filehandling.c @@ -21,8 +21,8 @@ _Static_assert(sizeof (size_t) == sizeof (SizeT), "Check why sizeof(size_t) != sizeof(SizeT)"); #endif -#ifndef XZFILE_BUFFER_SIZE -#define XZFILE_BUFFER_SIZE 1024 * 1024 +#ifndef HCFILE_BUFFER_SIZE +#define HCFILE_BUFFER_SIZE 256 * 1024 #endif static bool xz_initialized = false; @@ -45,10 +45,6 @@ struct xzfile CXzs streams; }; -#ifndef HCFILE_BUFFER_SIZE -#define HCFILE_BUFFER_SIZE 256 * 1024 -#endif - #if defined (__CYGWIN__) // workaround for zlib with cygwin build int _wopen (const char *path, int oflag, ...) @@ -181,7 +177,7 @@ bool hc_fopen (HCFILE *fp, const char *path, const char *mode) xfp->alloc.numAlignBits = 7; xfp->alloc.baseAlloc = &xz_alloc; ISzAllocPtr alloc = &xfp->alloc.vt; - xfp->inBuf = (Byte *) ISzAlloc_Alloc (alloc, XZFILE_BUFFER_SIZE); + xfp->inBuf = (Byte *) ISzAlloc_Alloc (alloc, HCFILE_BUFFER_SIZE); if (xfp->inBuf == NULL) { hcfree (xfp); @@ -207,7 +203,7 @@ bool hc_fopen (HCFILE *fp, const char *path, const char *mode) CLookToRead2 lookStream; LookToRead2_CreateVTable (&lookStream, false); lookStream.buf = xfp->inBuf; - lookStream.bufSize = XZFILE_BUFFER_SIZE; + lookStream.bufSize = HCFILE_BUFFER_SIZE; lookStream.realStream = &inStream->vt; LookToRead2_Init (&lookStream); Xzs_Construct (&xfp->streams); @@ -227,7 +223,7 @@ bool hc_fopen (HCFILE *fp, const char *path, const char *mode) xfp->outSize = Xzs_GetUnpackSize (&xfp->streams); /* seek to start of the file and fill the buffer */ - SizeT inLen = XZFILE_BUFFER_SIZE; + SizeT inLen = HCFILE_BUFFER_SIZE; res = ISeekInStream_Seek (&inStream->vt, &offset, SZ_SEEK_SET); if (res == SZ_OK) { @@ -430,7 +426,7 @@ size_t hc_fread (void *ptr, size_t size, size_t nmemb, HCFILE *fp) if (xfp->inLen == xfp->inPos && !xfp->inEof) { xfp->inPos = 0; - xfp->inLen = XZFILE_BUFFER_SIZE; + xfp->inLen = HCFILE_BUFFER_SIZE; res = ISeekInStream_Read (&xfp->inStream.vt, xfp->inBuf, &xfp->inLen); if (res != SZ_OK || xfp->inLen == 0) xfp->inEof = true; } @@ -591,7 +587,7 @@ void hc_rewind (HCFILE *fp) XzUnpacker_Init (&xfp->state); /* fill the buffer */ - SizeT inLen = XZFILE_BUFFER_SIZE; + SizeT inLen = HCFILE_BUFFER_SIZE; res = ISeekInStream_Read (&inStream->vt, xfp->inBuf, &inLen); if (res != SZ_OK || inLen == 0) return; @@ -712,7 +708,7 @@ int hc_fgetc (HCFILE *fp) if (xfp->inLen == xfp->inPos && !xfp->inEof) { xfp->inPos = 0; - xfp->inLen = XZFILE_BUFFER_SIZE; + xfp->inLen = HCFILE_BUFFER_SIZE; res = ISeekInStream_Read (&xfp->inStream.vt, xfp->inBuf, &xfp->inLen); if (res != SZ_OK || xfp->inLen == 0) xfp->inEof = true; } @@ -764,7 +760,7 @@ char *hc_fgets (char *buf, int len, HCFILE *fp) if (xfp->inLen == xfp->inPos && !xfp->inEof) { xfp->inPos = 0; - xfp->inLen = XZFILE_BUFFER_SIZE; + xfp->inLen = HCFILE_BUFFER_SIZE; res = ISeekInStream_Read (&xfp->inStream.vt, xfp->inBuf, &xfp->inLen); if (res != SZ_OK || xfp->inLen == 0) xfp->inEof = true; } From 5baaa4fb0fb5410f97f71db4e6b3b22e21400f0d Mon Sep 17 00:00:00 2001 From: Jukka Ojanen Date: Thu, 2 Sep 2021 12:08:36 +0300 Subject: [PATCH 3/3] Fix merge mistake --- src/filehandling.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/filehandling.c b/src/filehandling.c index 3ca4a97d6..11d56af16 100644 --- a/src/filehandling.c +++ b/src/filehandling.c @@ -149,6 +149,8 @@ bool hc_fopen (HCFILE *fp, const char *path, const char *mode) if (is_gzip) { if ((fp->gfp = gzdopen (fp->fd, mode)) == NULL) return false; + + gzbuffer (fp->gfp, HCFILE_BUFFER_SIZE); } else if (is_zip) { @@ -165,8 +167,6 @@ bool hc_fopen (HCFILE *fp, const char *path, const char *mode) Crc64GenerateTable (); Sha256Prepare (); xz_initialized = true; - - gzbuffer (fp->gfp, HCFILE_BUFFER_SIZE); } xzfile_t *xfp = (xzfile_t *) hccalloc (1, sizeof (*xfp));