From 2cf662496529117de53731d576b4d23044e792d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Verschelde?= Date: Thu, 30 Apr 2020 14:24:03 +0200 Subject: [PATCH] FastLZ: Update to upstream version 0.5.0 Upstream development restarted after 13 years. Changes: 2020-02-02: Version 0.5.0 Minor speed improvement on the decompressor. Prevent memory violation when decompressing corrupted input. 2020-01-10: Version 0.4.0 Only code & infrastructure clean-up, no new functionality. (cherry picked from commit 5167c9186a1fd9cea58803d3a36ec51d9f654f6f) --- COPYRIGHT.txt | 2 +- thirdparty/README.md | 2 +- thirdparty/misc/fastlz.c | 872 ++++++++++++++++++++------------------- thirdparty/misc/fastlz.h | 86 ++-- 4 files changed, 500 insertions(+), 462 deletions(-) diff --git a/COPYRIGHT.txt b/COPYRIGHT.txt index 8a1720b43ef..5c7e9298e8b 100644 --- a/COPYRIGHT.txt +++ b/COPYRIGHT.txt @@ -253,7 +253,7 @@ License: BSD-3-clause Files: ./thirdparty/misc/fastlz.c ./thirdparty/misc/fastlz.h Comment: FastLZ -Copyright: 2005-2007, Ariya Hidayat +Copyright: 2005-2020, Ariya Hidayat License: Expat Files: ./thirdparty/misc/hq2x.cpp diff --git a/thirdparty/README.md b/thirdparty/README.md index 02ddeb7b5c5..9de0ffa2d86 100644 --- a/thirdparty/README.md +++ b/thirdparty/README.md @@ -324,7 +324,7 @@ Collection of single-file libraries used in Godot components. * License: BSL-1.0 - `fastlz.{c,h}` * Upstream: https://github.com/ariya/FastLZ - * Version: git (f121734, 2007) + * Version: 0.5.0 (4f20f54d46f5a6dd4fae4def134933369b7602d2, 2020) * License: MIT - `hq2x.{cpp,h}` * Upstream: https://github.com/brunexgeek/hqx diff --git a/thirdparty/misc/fastlz.c b/thirdparty/misc/fastlz.c index 508f6ea2aea..b4d2dd3c29e 100644 --- a/thirdparty/misc/fastlz.c +++ b/thirdparty/misc/fastlz.c @@ -1,9 +1,6 @@ - /* - FastLZ - lightning-fast lossless compression library - - Copyright (C) 2007 Ariya Hidayat (ariya@kde.org) - Copyright (C) 2006 Ariya Hidayat (ariya@kde.org) - Copyright (C) 2005 Ariya Hidayat (ariya@kde.org) +/* + FastLZ - Byte-aligned LZ77 compression library + Copyright (C) 2005-2020 Ariya Hidayat Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal @@ -24,274 +21,148 @@ THE SOFTWARE. */ -#if !defined(FASTLZ_COMPRESSOR) && !defined(FASTLZ_DECOMPRESSOR) +#include "fastlz.h" + +#include /* * Always check for bound when decompressing. * Generally it is best to leave it defined. */ #define FASTLZ_SAFE +#if defined(FASTLZ_USE_SAFE_DECOMPRESSOR) && (FASTLZ_USE_SAFE_DECOMPRESSOR == 0) +#undef FASTLZ_SAFE +#endif /* * Give hints to the compiler for branch prediction optimization. */ -#if defined(__GNUC__) && (__GNUC__ > 2) -#define FASTLZ_EXPECT_CONDITIONAL(c) (__builtin_expect((c), 1)) -#define FASTLZ_UNEXPECT_CONDITIONAL(c) (__builtin_expect((c), 0)) +#if defined(__clang__) || (defined(__GNUC__) && (__GNUC__ > 2)) +#define FASTLZ_LIKELY(c) (__builtin_expect(!!(c), 1)) +#define FASTLZ_UNLIKELY(c) (__builtin_expect(!!(c), 0)) #else -#define FASTLZ_EXPECT_CONDITIONAL(c) (c) -#define FASTLZ_UNEXPECT_CONDITIONAL(c) (c) +#define FASTLZ_LIKELY(c) (c) +#define FASTLZ_UNLIKELY(c) (c) #endif -/* - * Use inlined functions for supported systems. - */ -#if defined(__GNUC__) || defined(__DMC__) || defined(__POCC__) || defined(__WATCOMC__) || defined(__SUNPRO_C) -#define FASTLZ_INLINE inline -#elif defined(__BORLANDC__) || defined(_MSC_VER) || defined(__LCC__) -#define FASTLZ_INLINE __inline -#else -#define FASTLZ_INLINE -#endif - -/* - * Prevent accessing more than 8-bit at once, except on x86 architectures. - */ -#if !defined(FASTLZ_STRICT_ALIGN) -#define FASTLZ_STRICT_ALIGN -#if defined(__i386__) || defined(__386) /* GNU C, Sun Studio */ -#undef FASTLZ_STRICT_ALIGN -#elif defined(__i486__) || defined(__i586__) || defined(__i686__) /* GNU C */ -#undef FASTLZ_STRICT_ALIGN -#elif defined(_M_IX86) /* Intel, MSVC */ -#undef FASTLZ_STRICT_ALIGN -#elif defined(__386) -#undef FASTLZ_STRICT_ALIGN -#elif defined(_X86_) /* MinGW */ -#undef FASTLZ_STRICT_ALIGN -#elif defined(__I86__) /* Digital Mars */ -#undef FASTLZ_STRICT_ALIGN -#endif -#endif - -/* - * FIXME: use preprocessor magic to set this on different platforms! - */ -typedef unsigned char flzuint8; -typedef unsigned short flzuint16; -typedef unsigned int flzuint32; - -/* prototypes */ -int fastlz_compress(const void* input, int length, void* output); -int fastlz_compress_level(int level, const void* input, int length, void* output); -int fastlz_decompress(const void* input, int length, void* output, int maxout); - -#define MAX_COPY 32 -#define MAX_LEN 264 /* 256 + 8 */ -#define MAX_DISTANCE 8192 - -#if !defined(FASTLZ_STRICT_ALIGN) -#define FASTLZ_READU16(p) *((const flzuint16*)(p)) +#if defined(FASTLZ_SAFE) +#define FASTLZ_BOUND_CHECK(cond) \ + if (FASTLZ_UNLIKELY(!(cond))) return 0; #else -#define FASTLZ_READU16(p) ((p)[0] | (p)[1]<<8) +#define FASTLZ_BOUND_CHECK(cond) \ + do { \ + } while (0) #endif -#define HASH_LOG 13 -#define HASH_SIZE (1<< HASH_LOG) -#define HASH_MASK (HASH_SIZE-1) -#define HASH_FUNCTION(v,p) { v = FASTLZ_READU16(p); v ^= FASTLZ_READU16(p+1)^(v>>(16-HASH_LOG));v &= HASH_MASK; } +#define MAX_COPY 32 +#define MAX_LEN 264 /* 256 + 8 */ +#define MAX_L1_DISTANCE 8192 +#define MAX_L2_DISTANCE 8191 +#define MAX_FARDISTANCE (65535 + MAX_L2_DISTANCE - 1) -#undef FASTLZ_LEVEL -#define FASTLZ_LEVEL 1 +#define FASTLZ_READU16(p) ((p)[0] | (p)[1] << 8) -#undef FASTLZ_COMPRESSOR -#undef FASTLZ_DECOMPRESSOR -#define FASTLZ_COMPRESSOR fastlz1_compress -#define FASTLZ_DECOMPRESSOR fastlz1_decompress -static FASTLZ_INLINE int FASTLZ_COMPRESSOR(const void* input, int length, void* output); -static FASTLZ_INLINE int FASTLZ_DECOMPRESSOR(const void* input, int length, void* output, int maxout); -#include "fastlz.c" +#define HASH_LOG 13 +#define HASH_SIZE (1 << HASH_LOG) +#define HASH_MASK (HASH_SIZE - 1) +#define HASH_FUNCTION(v, p) \ + { \ + v = FASTLZ_READU16(p); \ + v ^= FASTLZ_READU16(p + 1) ^ (v >> (16 - HASH_LOG)); \ + v &= HASH_MASK; \ + } -#undef FASTLZ_LEVEL -#define FASTLZ_LEVEL 2 +int fastlz1_compress(const void* input, int length, void* output) { + const uint8_t* ip = (const uint8_t*)input; + const uint8_t* ip_bound = ip + length - 2; + const uint8_t* ip_limit = ip + length - 12 - 1; + uint8_t* op = (uint8_t*)output; -#undef MAX_DISTANCE -#define MAX_DISTANCE 8191 -#define MAX_FARDISTANCE (65535+MAX_DISTANCE-1) + const uint8_t* htab[HASH_SIZE]; + uint32_t hval; -#undef FASTLZ_COMPRESSOR -#undef FASTLZ_DECOMPRESSOR -#define FASTLZ_COMPRESSOR fastlz2_compress -#define FASTLZ_DECOMPRESSOR fastlz2_decompress -static FASTLZ_INLINE int FASTLZ_COMPRESSOR(const void* input, int length, void* output); -static FASTLZ_INLINE int FASTLZ_DECOMPRESSOR(const void* input, int length, void* output, int maxout); -#include "fastlz.c" - -int fastlz_compress(const void* input, int length, void* output) -{ - /* for short block, choose fastlz1 */ - if(length < 65536) - return fastlz1_compress(input, length, output); - - /* else... */ - return fastlz2_compress(input, length, output); -} - -int fastlz_decompress(const void* input, int length, void* output, int maxout) -{ - /* magic identifier for compression level */ - int level = ((*(const flzuint8*)input) >> 5) + 1; - - if(level == 1) - return fastlz1_decompress(input, length, output, maxout); - if(level == 2) - return fastlz2_decompress(input, length, output, maxout); - - /* unknown level, trigger error */ - return 0; -} - -int fastlz_compress_level(int level, const void* input, int length, void* output) -{ - if(level == 1) - return fastlz1_compress(input, length, output); - if(level == 2) - return fastlz2_compress(input, length, output); - - return 0; -} - -#else /* !defined(FASTLZ_COMPRESSOR) && !defined(FASTLZ_DECOMPRESSOR) */ - -static FASTLZ_INLINE int FASTLZ_COMPRESSOR(const void* input, int length, void* output) -{ - const flzuint8* ip = (const flzuint8*) input; - const flzuint8* ip_bound = ip + length - 2; - const flzuint8* ip_limit = ip + length - 12; - flzuint8* op = (flzuint8*) output; - - const flzuint8* htab[HASH_SIZE]; - const flzuint8** hslot; - flzuint32 hval; - - flzuint32 copy; + uint32_t copy; /* sanity check */ - if(FASTLZ_UNEXPECT_CONDITIONAL(length < 4)) - { - if(length) - { + if (FASTLZ_UNLIKELY(length < 4)) { + if (length) { /* create literal copy only */ - *op++ = length-1; + *op++ = length - 1; ip_bound++; - while(ip <= ip_bound) - *op++ = *ip++; - return length+1; - } - else + while (ip <= ip_bound) *op++ = *ip++; + return length + 1; + } else return 0; } /* initializes hash table */ - for (hslot = htab; hslot < htab + HASH_SIZE; hslot++) - *hslot = ip; + for (hval = 0; hval < HASH_SIZE; ++hval) htab[hval] = ip; /* we start with literal copy */ copy = 2; - *op++ = MAX_COPY-1; + *op++ = MAX_COPY - 1; *op++ = *ip++; *op++ = *ip++; /* main loop */ - while(FASTLZ_EXPECT_CONDITIONAL(ip < ip_limit)) - { - const flzuint8* ref; - flzuint32 distance; + while (FASTLZ_LIKELY(ip < ip_limit)) { + const uint8_t* ref; + uint32_t distance; /* minimum match length */ - flzuint32 len = 3; + uint32_t len = 3; /* comparison starting-point */ - const flzuint8* anchor = ip; - - /* check for a run */ -#if FASTLZ_LEVEL==2 - if(ip[0] == ip[-1] && FASTLZ_READU16(ip-1)==FASTLZ_READU16(ip+1)) - { - distance = 1; - ip += 3; - ref = anchor - 1 + 3; - goto match; - } -#endif + const uint8_t* anchor = ip; /* find potential match */ - HASH_FUNCTION(hval,ip); - hslot = htab + hval; + HASH_FUNCTION(hval, ip); ref = htab[hval]; + /* update hash table */ + htab[hval] = anchor; + /* calculate distance to the match */ distance = anchor - ref; - /* update hash table */ - *hslot = anchor; - /* is this a match? check the first 3 bytes */ - if(distance==0 || -#if FASTLZ_LEVEL==1 - (distance >= MAX_DISTANCE) || -#else - (distance >= MAX_FARDISTANCE) || -#endif - *ref++ != *ip++ || *ref++!=*ip++ || *ref++!=*ip++) + if (distance == 0 || (distance >= MAX_L1_DISTANCE) || *ref++ != *ip++ || + *ref++ != *ip++ || *ref++ != *ip++) goto literal; -#if FASTLZ_LEVEL==2 - /* far, needs at least 5-byte match */ - if(distance >= MAX_DISTANCE) - { - if(*ip++ != *ref++ || *ip++!= *ref++) - goto literal; - len += 2; - } - - match: -#endif - /* last matched byte */ ip = anchor + len; /* distance is biased */ distance--; - if(!distance) - { + if (!distance) { /* zero distance means a run */ - flzuint8 x = ip[-1]; - while(ip < ip_bound) - if(*ref++ != x) break; else ip++; - } - else - for(;;) - { - /* safe because the outer check against ip limit */ - if(*ref++ != *ip++) break; - if(*ref++ != *ip++) break; - if(*ref++ != *ip++) break; - if(*ref++ != *ip++) break; - if(*ref++ != *ip++) break; - if(*ref++ != *ip++) break; - if(*ref++ != *ip++) break; - if(*ref++ != *ip++) break; - while(ip < ip_bound) - if(*ref++ != *ip++) break; - break; - } + uint8_t x = ip[-1]; + while (ip < ip_bound) + if (*ref++ != x) + break; + else + ip++; + } else + for (;;) { + /* safe because the outer check against ip limit */ + if (*ref++ != *ip++) break; + if (*ref++ != *ip++) break; + if (*ref++ != *ip++) break; + if (*ref++ != *ip++) break; + if (*ref++ != *ip++) break; + if (*ref++ != *ip++) break; + if (*ref++ != *ip++) break; + if (*ref++ != *ip++) break; + while (ip < ip_bound) + if (*ref++ != *ip++) break; + break; + } /* if we have copied something, adjust the copy count */ - if(copy) - /* copy is biased, '0' means 1 byte copy */ - *(op-copy-1) = copy-1; + if (copy) /* copy is biased, '0' means 1 byte copy */ + *(op - copy - 1) = copy - 1; else /* back, to overwrite the copy count */ op--; @@ -304,248 +175,417 @@ static FASTLZ_INLINE int FASTLZ_COMPRESSOR(const void* input, int length, void* len = ip - anchor; /* encode the match */ -#if FASTLZ_LEVEL==2 - if(distance < MAX_DISTANCE) - { - if(len < 7) - { - *op++ = (len << 5) + (distance >> 8); - *op++ = (distance & 255); - } - else - { + if (FASTLZ_UNLIKELY(len > MAX_LEN - 2)) + while (len > MAX_LEN - 2) { *op++ = (7 << 5) + (distance >> 8); - for(len-=7; len >= 255; len-= 255) - *op++ = 255; - *op++ = len; + *op++ = MAX_LEN - 2 - 7 - 2; *op++ = (distance & 255); - } - } - else - { - /* far away, but not yet in the another galaxy... */ - if(len < 7) - { - distance -= MAX_DISTANCE; - *op++ = (len << 5) + 31; - *op++ = 255; - *op++ = distance >> 8; - *op++ = distance & 255; - } - else - { - distance -= MAX_DISTANCE; - *op++ = (7 << 5) + 31; - for(len-=7; len >= 255; len-= 255) - *op++ = 255; - *op++ = len; - *op++ = 255; - *op++ = distance >> 8; - *op++ = distance & 255; - } - } -#else - - if(FASTLZ_UNEXPECT_CONDITIONAL(len > MAX_LEN-2)) - while(len > MAX_LEN-2) - { - *op++ = (7 << 5) + (distance >> 8); - *op++ = MAX_LEN - 2 - 7 -2; - *op++ = (distance & 255); - len -= MAX_LEN-2; + len -= MAX_LEN - 2; } - if(len < 7) - { + if (len < 7) { *op++ = (len << 5) + (distance >> 8); *op++ = (distance & 255); - } - else - { + } else { *op++ = (7 << 5) + (distance >> 8); *op++ = len - 7; *op++ = (distance & 255); } -#endif /* update the hash at match boundary */ - HASH_FUNCTION(hval,ip); + HASH_FUNCTION(hval, ip); htab[hval] = ip++; - HASH_FUNCTION(hval,ip); + HASH_FUNCTION(hval, ip); htab[hval] = ip++; /* assuming literal copy */ - *op++ = MAX_COPY-1; + *op++ = MAX_COPY - 1; continue; - literal: - *op++ = *anchor++; - ip = anchor; - copy++; - if(FASTLZ_UNEXPECT_CONDITIONAL(copy == MAX_COPY)) - { - copy = 0; - *op++ = MAX_COPY-1; - } + literal: + *op++ = *anchor++; + ip = anchor; + copy++; + if (FASTLZ_UNLIKELY(copy == MAX_COPY)) { + copy = 0; + *op++ = MAX_COPY - 1; + } } /* left-over as literal copy */ ip_bound++; - while(ip <= ip_bound) - { + while (ip <= ip_bound) { *op++ = *ip++; copy++; - if(copy == MAX_COPY) - { + if (copy == MAX_COPY) { copy = 0; - *op++ = MAX_COPY-1; + *op++ = MAX_COPY - 1; } } /* if we have copied something, adjust the copy length */ - if(copy) - *(op-copy-1) = copy-1; + if (copy) + *(op - copy - 1) = copy - 1; else op--; -#if FASTLZ_LEVEL==2 - /* marker for fastlz2 */ - *(flzuint8*)output |= (1 << 5); -#endif - - return op - (flzuint8*)output; + return op - (uint8_t*)output; } -static FASTLZ_INLINE int FASTLZ_DECOMPRESSOR(const void* input, int length, void* output, int maxout) -{ - const flzuint8* ip = (const flzuint8*) input; - const flzuint8* ip_limit = ip + length; - flzuint8* op = (flzuint8*) output; - flzuint8* op_limit = op + maxout; - flzuint32 ctrl = (*ip++) & 31; - int loop = 1; +#if defined(FASTLZ_USE_MEMMOVE) && (FASTLZ_USE_MEMMOVE == 0) - do - { - const flzuint8* ref = op; - flzuint32 len = ctrl >> 5; - flzuint32 ofs = (ctrl & 31) << 8; +static void fastlz_memmove(uint8_t* dest, const uint8_t* src, uint32_t count) { + do { + *dest++ = *src++; + } while (--count); +} + +static void fastlz_memcpy(uint8_t* dest, const uint8_t* src, uint32_t count) { + return fastlz_memmove(dest, src, count); +} - if(ctrl >= 32) - { -#if FASTLZ_LEVEL==2 - flzuint8 code; -#endif - len--; - ref -= ofs; - if (len == 7-1) -#if FASTLZ_LEVEL==1 - len += *ip++; - ref -= *ip++; #else - do - { - code = *ip++; - len += code; - } while (code==255); - code = *ip++; - ref -= code; - /* match from 16-bit distance */ - if(FASTLZ_UNEXPECT_CONDITIONAL(code==255)) - if(FASTLZ_EXPECT_CONDITIONAL(ofs==(31 << 8))) - { - ofs = (*ip++) << 8; - ofs += *ip++; - ref = op - ofs - MAX_DISTANCE; - } -#endif - -#ifdef FASTLZ_SAFE - if (FASTLZ_UNEXPECT_CONDITIONAL(op + len + 3 > op_limit)) - return 0; +#include - if (FASTLZ_UNEXPECT_CONDITIONAL(ref-1 < (flzuint8 *)output)) - return 0; -#endif - - if(FASTLZ_EXPECT_CONDITIONAL(ip < ip_limit)) - ctrl = *ip++; - else - loop = 0; - - if(ref == op) - { - /* optimize copy for a run */ - flzuint8 b = ref[-1]; - *op++ = b; - *op++ = b; - *op++ = b; - for(; len; --len) - *op++ = b; - } - else - { -#if !defined(FASTLZ_STRICT_ALIGN) - const flzuint16* p; - flzuint16* q; -#endif - /* copy from reference */ - ref--; - *op++ = *ref++; - *op++ = *ref++; - *op++ = *ref++; - -#if !defined(FASTLZ_STRICT_ALIGN) - /* copy a byte, so that now it's word aligned */ - if(len & 1) - { - *op++ = *ref++; - len--; - } - - /* copy 16-bit at once */ - q = (flzuint16*) op; - op += len; - p = (const flzuint16*) ref; - for(len>>=1; len > 4; len-=4) - { - *q++ = *p++; - *q++ = *p++; - *q++ = *p++; - *q++ = *p++; - } - for(; len; --len) - *q++ = *p++; -#else - for(; len; --len) - *op++ = *ref++; -#endif - } - } - else - { - ctrl++; -#ifdef FASTLZ_SAFE - if (FASTLZ_UNEXPECT_CONDITIONAL(op + ctrl > op_limit)) - return 0; - if (FASTLZ_UNEXPECT_CONDITIONAL(ip + ctrl > ip_limit)) - return 0; -#endif - - *op++ = *ip++; - for(--ctrl; ctrl; ctrl--) - *op++ = *ip++; - - loop = FASTLZ_EXPECT_CONDITIONAL(ip < ip_limit); - if(loop) - ctrl = *ip++; +static void fastlz_memmove(uint8_t* dest, const uint8_t* src, uint32_t count) { + if ((count > 4) && (dest >= src + count)) { + memmove(dest, src, count); + } else { + switch (count) { + default: + do { + *dest++ = *src++; + } while (--count); + break; + case 3: + *dest++ = *src++; + case 2: + *dest++ = *src++; + case 1: + *dest++ = *src++; + case 0: + break; } } - while(FASTLZ_EXPECT_CONDITIONAL(loop)); - - return op - (flzuint8*)output; } -#endif /* !defined(FASTLZ_COMPRESSOR) && !defined(FASTLZ_DECOMPRESSOR) */ +static void fastlz_memcpy(uint8_t* dest, const uint8_t* src, uint32_t count) { + memcpy(dest, src, count); +} + +#endif + +int fastlz1_decompress(const void* input, int length, void* output, + int maxout) { + const uint8_t* ip = (const uint8_t*)input; + const uint8_t* ip_limit = ip + length; + const uint8_t* ip_bound = ip_limit - 2; + uint8_t* op = (uint8_t*)output; + uint8_t* op_limit = op + maxout; + uint32_t ctrl = (*ip++) & 31; + + while (1) { + if (ctrl >= 32) { + uint32_t len = (ctrl >> 5) - 1; + uint32_t ofs = (ctrl & 31) << 8; + const uint8_t* ref = op - ofs - 1; + if (len == 7 - 1) { + FASTLZ_BOUND_CHECK(ip <= ip_bound); + len += *ip++; + } + ref -= *ip++; + len += 3; + FASTLZ_BOUND_CHECK(op + len <= op_limit); + FASTLZ_BOUND_CHECK(ref >= (uint8_t*)output); + fastlz_memmove(op, ref, len); + op += len; + } else { + ctrl++; + FASTLZ_BOUND_CHECK(op + ctrl <= op_limit); + FASTLZ_BOUND_CHECK(ip + ctrl <= ip_limit); + fastlz_memcpy(op, ip, ctrl); + ip += ctrl; + op += ctrl; + } + + if (FASTLZ_UNLIKELY(ip > ip_bound)) break; + ctrl = *ip++; + } + + return op - (uint8_t*)output; +} + +int fastlz2_compress(const void* input, int length, void* output) { + const uint8_t* ip = (const uint8_t*)input; + const uint8_t* ip_bound = ip + length - 2; + const uint8_t* ip_limit = ip + length - 12 - 1; + uint8_t* op = (uint8_t*)output; + + const uint8_t* htab[HASH_SIZE]; + uint32_t hval; + + uint32_t copy; + + /* sanity check */ + if (FASTLZ_UNLIKELY(length < 4)) { + if (length) { + /* create literal copy only */ + *op++ = length - 1; + ip_bound++; + while (ip <= ip_bound) *op++ = *ip++; + return length + 1; + } else + return 0; + } + + /* initializes hash table */ + for (hval = 0; hval < HASH_SIZE; ++hval) htab[hval] = ip; + + /* we start with literal copy */ + copy = 2; + *op++ = MAX_COPY - 1; + *op++ = *ip++; + *op++ = *ip++; + + /* main loop */ + while (FASTLZ_LIKELY(ip < ip_limit)) { + const uint8_t* ref; + uint32_t distance; + + /* minimum match length */ + uint32_t len = 3; + + /* comparison starting-point */ + const uint8_t* anchor = ip; + + /* check for a run */ + if (ip[0] == ip[-1] && ip[0] == ip[1] && ip[1] == ip[2]) { + distance = 1; + ip += 3; + ref = anchor - 1 + 3; + goto match; + } + + /* find potential match */ + HASH_FUNCTION(hval, ip); + ref = htab[hval]; + + /* update hash table */ + htab[hval] = anchor; + + /* calculate distance to the match */ + distance = anchor - ref; + + /* is this a match? check the first 3 bytes */ + if (distance == 0 || (distance >= MAX_FARDISTANCE) || *ref++ != *ip++ || + *ref++ != *ip++ || *ref++ != *ip++) + goto literal; + + /* far, needs at least 5-byte match */ + if (distance >= MAX_L2_DISTANCE) { + if (*ip++ != *ref++ || *ip++ != *ref++) goto literal; + len += 2; + } + + match: + + /* last matched byte */ + ip = anchor + len; + + /* distance is biased */ + distance--; + + if (!distance) { + /* zero distance means a run */ + uint8_t x = ip[-1]; + while (ip < ip_bound) + if (*ref++ != x) + break; + else + ip++; + } else + for (;;) { + /* safe because the outer check against ip limit */ + if (*ref++ != *ip++) break; + if (*ref++ != *ip++) break; + if (*ref++ != *ip++) break; + if (*ref++ != *ip++) break; + if (*ref++ != *ip++) break; + if (*ref++ != *ip++) break; + if (*ref++ != *ip++) break; + if (*ref++ != *ip++) break; + while (ip < ip_bound) + if (*ref++ != *ip++) break; + break; + } + + /* if we have copied something, adjust the copy count */ + if (copy) /* copy is biased, '0' means 1 byte copy */ + *(op - copy - 1) = copy - 1; + else + /* back, to overwrite the copy count */ + op--; + + /* reset literal counter */ + copy = 0; + + /* length is biased, '1' means a match of 3 bytes */ + ip -= 3; + len = ip - anchor; + + /* encode the match */ + if (distance < MAX_L2_DISTANCE) { + if (len < 7) { + *op++ = (len << 5) + (distance >> 8); + *op++ = (distance & 255); + } else { + *op++ = (7 << 5) + (distance >> 8); + for (len -= 7; len >= 255; len -= 255) *op++ = 255; + *op++ = len; + *op++ = (distance & 255); + } + } else { + /* far away, but not yet in the another galaxy... */ + if (len < 7) { + distance -= MAX_L2_DISTANCE; + *op++ = (len << 5) + 31; + *op++ = 255; + *op++ = distance >> 8; + *op++ = distance & 255; + } else { + distance -= MAX_L2_DISTANCE; + *op++ = (7 << 5) + 31; + for (len -= 7; len >= 255; len -= 255) *op++ = 255; + *op++ = len; + *op++ = 255; + *op++ = distance >> 8; + *op++ = distance & 255; + } + } + + /* update the hash at match boundary */ + HASH_FUNCTION(hval, ip); + htab[hval] = ip++; + HASH_FUNCTION(hval, ip); + htab[hval] = ip++; + + /* assuming literal copy */ + *op++ = MAX_COPY - 1; + + continue; + + literal: + *op++ = *anchor++; + ip = anchor; + copy++; + if (FASTLZ_UNLIKELY(copy == MAX_COPY)) { + copy = 0; + *op++ = MAX_COPY - 1; + } + } + + /* left-over as literal copy */ + ip_bound++; + while (ip <= ip_bound) { + *op++ = *ip++; + copy++; + if (copy == MAX_COPY) { + copy = 0; + *op++ = MAX_COPY - 1; + } + } + + /* if we have copied something, adjust the copy length */ + if (copy) + *(op - copy - 1) = copy - 1; + else + op--; + + /* marker for fastlz2 */ + *(uint8_t*)output |= (1 << 5); + + return op - (uint8_t*)output; +} + +int fastlz2_decompress(const void* input, int length, void* output, + int maxout) { + const uint8_t* ip = (const uint8_t*)input; + const uint8_t* ip_limit = ip + length; + const uint8_t* ip_bound = ip_limit - 2; + uint8_t* op = (uint8_t*)output; + uint8_t* op_limit = op + maxout; + uint32_t ctrl = (*ip++) & 31; + + while (1) { + if (ctrl >= 32) { + uint32_t len = (ctrl >> 5) - 1; + uint32_t ofs = (ctrl & 31) << 8; + const uint8_t* ref = op - ofs - 1; + + uint8_t code; + if (len == 7 - 1) do { + FASTLZ_BOUND_CHECK(ip <= ip_bound); + code = *ip++; + len += code; + } while (code == 255); + code = *ip++; + ref -= code; + len += 3; + + /* match from 16-bit distance */ + if (FASTLZ_UNLIKELY(code == 255)) + if (FASTLZ_LIKELY(ofs == (31 << 8))) { + FASTLZ_BOUND_CHECK(ip < ip_bound); + ofs = (*ip++) << 8; + ofs += *ip++; + ref = op - ofs - MAX_L2_DISTANCE - 1; + } + + FASTLZ_BOUND_CHECK(op + len <= op_limit); + FASTLZ_BOUND_CHECK(ref >= (uint8_t*)output); + fastlz_memmove(op, ref, len); + op += len; + } else { + ctrl++; + FASTLZ_BOUND_CHECK(op + ctrl <= op_limit); + FASTLZ_BOUND_CHECK(ip + ctrl <= ip_limit); + fastlz_memcpy(op, ip, ctrl); + ip += ctrl; + op += ctrl; + } + + if (FASTLZ_UNLIKELY(ip >= ip_limit)) break; + ctrl = *ip++; + } + + return op - (uint8_t*)output; +} + +int fastlz_compress(const void* input, int length, void* output) { + /* for short block, choose fastlz1 */ + if (length < 65536) return fastlz1_compress(input, length, output); + + /* else... */ + return fastlz2_compress(input, length, output); +} + +int fastlz_decompress(const void* input, int length, void* output, int maxout) { + /* magic identifier for compression level */ + int level = ((*(const uint8_t*)input) >> 5) + 1; + + if (level == 1) return fastlz1_decompress(input, length, output, maxout); + if (level == 2) return fastlz2_decompress(input, length, output, maxout); + + /* unknown level, trigger error */ + return 0; +} + +int fastlz_compress_level(int level, const void* input, int length, + void* output) { + if (level == 1) return fastlz1_compress(input, length, output); + if (level == 2) return fastlz2_compress(input, length, output); + + return 0; +} diff --git a/thirdparty/misc/fastlz.h b/thirdparty/misc/fastlz.h index e5ca8dfc020..e53dbfbb563 100644 --- a/thirdparty/misc/fastlz.h +++ b/thirdparty/misc/fastlz.h @@ -1,9 +1,6 @@ /* - FastLZ - lightning-fast lossless compression library - - Copyright (C) 2007 Ariya Hidayat (ariya@kde.org) - Copyright (C) 2006 Ariya Hidayat (ariya@kde.org) - Copyright (C) 2005 Ariya Hidayat (ariya@kde.org) + FastLZ - Byte-aligned LZ77 compression library + Copyright (C) 2005-2020 Ariya Hidayat Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal @@ -27,48 +24,18 @@ #ifndef FASTLZ_H #define FASTLZ_H -#define FASTLZ_VERSION 0x000100 +#define FASTLZ_VERSION 0x000500 -#define FASTLZ_VERSION_MAJOR 0 -#define FASTLZ_VERSION_MINOR 0 -#define FASTLZ_VERSION_REVISION 0 +#define FASTLZ_VERSION_MAJOR 0 +#define FASTLZ_VERSION_MINOR 5 +#define FASTLZ_VERSION_REVISION 0 -#define FASTLZ_VERSION_STRING "0.1.0" +#define FASTLZ_VERSION_STRING "0.5.0" -#if defined (__cplusplus) +#if defined(__cplusplus) extern "C" { #endif -/** - Compress a block of data in the input buffer and returns the size of - compressed block. The size of input buffer is specified by length. The - minimum input buffer size is 16. - - The output buffer must be at least 5% larger than the input buffer - and can not be smaller than 66 bytes. - - If the input is not compressible, the return value might be larger than - length (input buffer size). - - The input buffer and the output buffer can not overlap. -*/ - -int fastlz_compress(const void* input, int length, void* output); - -/** - Decompress a block of compressed data and returns the size of the - decompressed block. If error occurs, e.g. the compressed data is - corrupted or the output buffer is not large enough, then 0 (zero) - will be returned instead. - - The input buffer and the output buffer can not overlap. - - Decompression is memory safe and guaranteed not to write the output buffer - more than what is specified in maxout. - */ - -int fastlz_decompress(const void* input, int length, void* output, int maxout); - /** Compress a block of data in the input buffer and returns the size of compressed block. The size of input buffer is specified by length. The @@ -88,12 +55,43 @@ int fastlz_decompress(const void* input, int length, void* output, int maxout); Level 2 is slightly slower but it gives better compression ratio. Note that the compressed data, regardless of the level, can always be - decompressed using the function fastlz_decompress above. + decompressed using the function fastlz_decompress below. */ -int fastlz_compress_level(int level, const void* input, int length, void* output); +int fastlz_compress_level(int level, const void* input, int length, + void* output); -#if defined (__cplusplus) +/** + Decompress a block of compressed data and returns the size of the + decompressed block. If error occurs, e.g. the compressed data is + corrupted or the output buffer is not large enough, then 0 (zero) + will be returned instead. + + The input buffer and the output buffer can not overlap. + + Decompression is memory safe and guaranteed not to write the output buffer + more than what is specified in maxout. + + Note that the decompression will always work, regardless of the + compression level specified in fastlz_compress_level above (when + producing the compressed block). + */ + +int fastlz_decompress(const void* input, int length, void* output, int maxout); + +/** + DEPRECATED. + + This is similar to fastlz_compress_level above, but with the level + automatically chosen. + + This function is deprecated and it will be completely removed in some future + version. +*/ + +int fastlz_compress(const void* input, int length, void* output); + +#if defined(__cplusplus) } #endif