godotengine
/
godot
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Expose brotli decompression to the scripting API.

This commit is contained in:
bruvzg 2023-03-29 09:31:25 +03:00
parent c29866dbc0
commit 0e4bd964cc
No known key found for this signature in database
GPG Key ID: 7960FCF39844EC38
10 changed files with 182 additions and 94 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
SConstruct
View File

@ -181,6 +181,7 @@ opts.Add(BoolVariable("production", "Set defaults to build Godot for use in prod
opts.Add(BoolVariable("deprecated", "Enable compatibility code for deprecated and removed features", True))
opts.Add(EnumVariable("precision", "Set the floating-point precision level", "single", ("single", "double")))
opts.Add(BoolVariable("minizip", "Enable ZIP archive support using minizip", True))
opts.Add(BoolVariable("brotli", "Enable Brotli for decompresson and WOFF2 fonts support", True))
opts.Add(BoolVariable("xaudio2", "Enable the XAudio2 audio driver", False))
opts.Add(BoolVariable("vulkan", "Enable the vulkan rendering driver", True))
opts.Add(BoolVariable("opengl3", "Enable the OpenGL/GLES3 rendering driver", True))
@ -855,6 +856,8 @@ if selected_platform in platform_list:
env.Append(CPPDEFINES=["ADVANCED_GUI_DISABLED"])
if env["minizip"]:
env.Append(CPPDEFINES=["MINIZIP_ENABLED"])
if env["brotli"]:
env.Append(CPPDEFINES=["BROTLI_ENABLED"])
if not env["verbose"]:
methods.no_verbose(sys, env)

25
core/SCsub
View File

@ -64,6 +64,31 @@ thirdparty_misc_sources = [
thirdparty_misc_sources = [thirdparty_misc_dir + file for file in thirdparty_misc_sources]
env_thirdparty.add_source_files(thirdparty_obj, thirdparty_misc_sources)
# Brotli
if env["brotli"]:
thirdparty_brotli_dir = "#thirdparty/brotli/"
thirdparty_brotli_sources = [
"common/constants.c",
"common/context.c",
"common/dictionary.c",
"common/platform.c",
"common/shared_dictionary.c",
"common/transform.c",
"dec/bit_reader.c",
"dec/decode.c",
"dec/huffman.c",
"dec/state.c",
]
thirdparty_brotli_sources = [thirdparty_brotli_dir + file for file in thirdparty_brotli_sources]
env_thirdparty.Prepend(CPPPATH=[thirdparty_brotli_dir + "include"])
env.Prepend(CPPPATH=[thirdparty_brotli_dir + "include"])
if env.get("use_ubsan") or env.get("use_asan") or env.get("use_tsan") or env.get("use_lsan") or env.get("use_msan"):
env_thirdparty.Append(CPPDEFINES=["BROTLI_BUILD_PORTABLE"])
env_thirdparty.add_source_files(thirdparty_obj, thirdparty_brotli_sources)
# Zlib library, can be unbundled
if env["builtin_zlib"]:
thirdparty_zlib_dir = "#thirdparty/zlib/"

210
core/io/compression.cpp
View File

@ -35,11 +35,18 @@
#include "thirdparty/misc/fastlz.h"
#ifdef BROTLI_ENABLED
#include "thirdparty/brotli/include/brotli/decode.h"
#endif
#include <zlib.h>
#include <zstd.h>
int Compression::compress(uint8_t *p_dst, const uint8_t *p_src, int p_src_size, Mode p_mode) {
switch (p_mode) {
case MODE_BROTLI: {
ERR_FAIL_V_MSG(-1, "Only brotli decompression is supported.");
} break;
case MODE_FASTLZ: {
if (p_src_size < 16) {
uint8_t src[16];
@ -95,6 +102,9 @@ int Compression::compress(uint8_t *p_dst, const uint8_t *p_src, int p_src_size,
int Compression::get_max_compressed_buffer_size(int p_src_size, Mode p_mode) {
switch (p_mode) {
case MODE_BROTLI: {
ERR_FAIL_V_MSG(-1, "Only brotli decompression is supported.");
} break;
case MODE_FASTLZ: {
int ss = p_src_size + p_src_size * 6 / 100;
if (ss < 66) {
@ -129,6 +139,16 @@ int Compression::get_max_compressed_buffer_size(int p_src_size, Mode p_mode) {
int Compression::decompress(uint8_t *p_dst, int p_dst_max_size, const uint8_t *p_src, int p_src_size, Mode p_mode) {
switch (p_mode) {
case MODE_BROTLI: {
#ifdef BROTLI_ENABLED
size_t ret_size = p_dst_max_size;
BrotliDecoderResult res = BrotliDecoderDecompress(p_src_size, p_src, &ret_size, p_dst);
ERR_FAIL_COND_V(res != BROTLI_DECODER_RESULT_SUCCESS, -1);
return ret_size;
#else
ERR_FAIL_V_MSG(-1, "Godot was compiled without brotli support.");
#endif
} break;
case MODE_FASTLZ: {
int ret_size = 0;
@ -186,87 +206,147 @@ int Compression::decompress(uint8_t *p_dst, int p_dst_max_size, const uint8_t *p
This is much slower however than using Compression::decompress because it may result in multiple full copies of the output buffer.
*/
int Compression::decompress_dynamic(Vector<uint8_t> *p_dst_vect, int p_max_dst_size, const uint8_t *p_src, int p_src_size, Mode p_mode) {
int ret;
uint8_t *dst = nullptr;
int out_mark = 0;
z_stream strm;
ERR_FAIL_COND_V(p_src_size <= 0, Z_DATA_ERROR);
// This function only supports GZip and Deflate
int window_bits = p_mode == MODE_DEFLATE ? 15 : 15 + 16;
ERR_FAIL_COND_V(p_mode != MODE_DEFLATE && p_mode != MODE_GZIP, Z_ERRNO);
if (p_mode == MODE_BROTLI) {
#ifdef BROTLI_ENABLED
BrotliDecoderResult ret;
BrotliDecoderState *state = BrotliDecoderCreateInstance(nullptr, nullptr, nullptr);
ERR_FAIL_COND_V(state == nullptr, Z_DATA_ERROR);
// Initialize the stream
strm.zalloc = Z_NULL;
strm.zfree = Z_NULL;
strm.opaque = Z_NULL;
strm.avail_in = 0;
strm.next_in = Z_NULL;
// Setup the stream inputs.
const uint8_t *next_in = p_src;
size_t avail_in = p_src_size;
uint8_t *next_out = nullptr;
size_t avail_out = 0;
size_t total_out = 0;
int err = inflateInit2(&strm, window_bits);
ERR_FAIL_COND_V(err != Z_OK, -1);
// Ensure the destination buffer is empty.
p_dst_vect->clear();
// Setup the stream inputs
strm.next_in = (Bytef *)p_src;
strm.avail_in = p_src_size;
// Ensure the destination buffer is empty
p_dst_vect->clear();
// decompress until deflate stream ends or end of file
do {
// Add another chunk size to the output buffer
// This forces a copy of the whole buffer
p_dst_vect->resize(p_dst_vect->size() + gzip_chunk);
// Get pointer to the actual output buffer
dst = p_dst_vect->ptrw();
// Set the stream to the new output stream
// Since it was copied, we need to reset the stream to the new buffer
strm.next_out = &(dst[out_mark]);
strm.avail_out = gzip_chunk;
// run inflate() on input until output buffer is full and needs to be resized
// or input runs out
// Decompress until stream ends or end of file.
do {
ret = inflate(&strm, Z_SYNC_FLUSH);
// Add another chunk size to the output buffer.
// This forces a copy of the whole buffer.
p_dst_vect->resize(p_dst_vect->size() + gzip_chunk);
// Get pointer to the actual output buffer.
dst = p_dst_vect->ptrw();
switch (ret) {
case Z_NEED_DICT:
ret = Z_DATA_ERROR;
[[fallthrough]];
case Z_DATA_ERROR:
case Z_MEM_ERROR:
case Z_STREAM_ERROR:
case Z_BUF_ERROR:
if (strm.msg) {
WARN_PRINT(strm.msg);
}
(void)inflateEnd(&strm);
p_dst_vect->clear();
return ret;
// Set the stream to the new output stream.
// Since it was copied, we need to reset the stream to the new buffer.
next_out = &(dst[out_mark]);
avail_out += gzip_chunk;
ret = BrotliDecoderDecompressStream(state, &avail_in, &next_in, &avail_out, &next_out, &total_out);
if (ret == BROTLI_DECODER_RESULT_ERROR) {
WARN_PRINT(BrotliDecoderErrorString(BrotliDecoderGetErrorCode(state)));
BrotliDecoderDestroyInstance(state);
p_dst_vect->clear();
return Z_DATA_ERROR;
}
} while (strm.avail_out > 0 && strm.avail_in > 0);
out_mark += gzip_chunk;
out_mark += gzip_chunk - avail_out;
// Enforce max output size
if (p_max_dst_size > -1 && strm.total_out > (uint64_t)p_max_dst_size) {
(void)inflateEnd(&strm);
p_dst_vect->clear();
return Z_BUF_ERROR;
// Enforce max output size.
if (p_max_dst_size > -1 && total_out > (uint64_t)p_max_dst_size) {
BrotliDecoderDestroyInstance(state);
p_dst_vect->clear();
return Z_BUF_ERROR;
}
} while (ret != BROTLI_DECODER_RESULT_SUCCESS);
// If all done successfully, resize the output if it's larger than the actual output.
if ((unsigned long)p_dst_vect->size() > total_out) {
p_dst_vect->resize(total_out);
}
} while (ret != Z_STREAM_END);
// If all done successfully, resize the output if it's larger than the actual output
if ((unsigned long)p_dst_vect->size() > strm.total_out) {
p_dst_vect->resize(strm.total_out);
// Clean up and return.
BrotliDecoderDestroyInstance(state);
return Z_OK;
#else
ERR_FAIL_V_MSG(Z_ERRNO, "Godot was compiled without brotli support.");
#endif
} else {
// This function only supports GZip and Deflate.
ERR_FAIL_COND_V(p_mode != MODE_DEFLATE && p_mode != MODE_GZIP, Z_ERRNO);
int ret;
z_stream strm;
int window_bits = p_mode == MODE_DEFLATE ? 15 : 15 + 16;
// Initialize the stream.
strm.zalloc = Z_NULL;
strm.zfree = Z_NULL;
strm.opaque = Z_NULL;
strm.avail_in = 0;
strm.next_in = Z_NULL;
int err = inflateInit2(&strm, window_bits);
ERR_FAIL_COND_V(err != Z_OK, -1);
// Setup the stream inputs.
strm.next_in = (Bytef *)p_src;
strm.avail_in = p_src_size;
// Ensure the destination buffer is empty.
p_dst_vect->clear();
// Decompress until deflate stream ends or end of file.
do {
// Add another chunk size to the output buffer.
// This forces a copy of the whole buffer.
p_dst_vect->resize(p_dst_vect->size() + gzip_chunk);
// Get pointer to the actual output buffer.
dst = p_dst_vect->ptrw();
// Set the stream to the new output stream.
// Since it was copied, we need to reset the stream to the new buffer.
strm.next_out = &(dst[out_mark]);
strm.avail_out = gzip_chunk;
// Run inflate() on input until output buffer is full and needs to be resized or input runs out.
do {
ret = inflate(&strm, Z_SYNC_FLUSH);
switch (ret) {
case Z_NEED_DICT:
ret = Z_DATA_ERROR;
[[fallthrough]];
case Z_DATA_ERROR:
case Z_MEM_ERROR:
case Z_STREAM_ERROR:
case Z_BUF_ERROR:
if (strm.msg) {
WARN_PRINT(strm.msg);
}
(void)inflateEnd(&strm);
p_dst_vect->clear();
return ret;
}
} while (strm.avail_out > 0 && strm.avail_in > 0);
out_mark += gzip_chunk;
// Enforce max output size.
if (p_max_dst_size > -1 && strm.total_out > (uint64_t)p_max_dst_size) {
(void)inflateEnd(&strm);
p_dst_vect->clear();
return Z_BUF_ERROR;
}
} while (ret != Z_STREAM_END);
// If all done successfully, resize the output if it's larger than the actual output.
if ((unsigned long)p_dst_vect->size() > strm.total_out) {
p_dst_vect->resize(strm.total_out);
}
// Clean up and return.
(void)inflateEnd(&strm);
return Z_OK;
}
// clean up and return
(void)inflateEnd(&strm);
return Z_OK;
}
int Compression::zlib_level = Z_DEFAULT_COMPRESSION;

3
core/io/compression.h
View File

@ -47,7 +47,8 @@ public:
MODE_FASTLZ,
MODE_DEFLATE,
MODE_ZSTD,
MODE_GZIP
MODE_GZIP,
MODE_BROTLI
};
static int compress(uint8_t *p_dst, const uint8_t *p_src, int p_src_size, Mode p_mode = MODE_ZSTD);

1
core/io/file_access.cpp
View File

@ -871,4 +871,5 @@ void FileAccess::_bind_methods() {
BIND_ENUM_CONSTANT(COMPRESSION_DEFLATE);
BIND_ENUM_CONSTANT(COMPRESSION_ZSTD);
BIND_ENUM_CONSTANT(COMPRESSION_GZIP);
BIND_ENUM_CONSTANT(COMPRESSION_BROTLI);
}

3
core/io/file_access.h
View File

@ -64,7 +64,8 @@ public:
COMPRESSION_FASTLZ = Compression::MODE_FASTLZ,
COMPRESSION_DEFLATE = Compression::MODE_DEFLATE,
COMPRESSION_ZSTD = Compression::MODE_ZSTD,
COMPRESSION_GZIP = Compression::MODE_GZIP
COMPRESSION_GZIP = Compression::MODE_GZIP,
COMPRESSION_BROTLI = Compression::MODE_BROTLI,
};
typedef void (*FileCloseFailNotify)(const String &);

3
doc/classes/FileAccess.xml
View File

@ -492,5 +492,8 @@
<constant name="COMPRESSION_GZIP" value="3" enum="CompressionMode">
Uses the [url=https://www.gzip.org/]gzip[/url] compression method.
</constant>
<constant name="COMPRESSION_BROTLI" value="4" enum="CompressionMode">
Uses the [url=https://github.com/google/brotli]brotli[/url] compression method (only decompression is supported).
</constant>
</constants>
</class>

2
doc/classes/PackedByteArray.xml
View File

@ -181,7 +181,7 @@
<param index="0" name="max_output_size" type="int" />
<param index="1" name="compression_mode" type="int" default="0" />
<description>
Returns a new [PackedByteArray] with the data decompressed. Set the compression mode using one of [enum FileAccess.CompressionMode]'s constants. [b]This method only accepts gzip and deflate compression modes.[/b]
Returns a new [PackedByteArray] with the data decompressed. Set the compression mode using one of [enum FileAccess.CompressionMode]'s constants. [b]This method only accepts brotli, gzip, and deflate compression modes.[/b]
This method is potentially slower than [code]decompress[/code], as it may have to re-allocate its output buffer multiple times while decompressing, whereas [code]decompress[/code] knows it's output buffer size from the beginning.
GZIP has a maximal compression ratio of 1032:1, meaning it's very possible for a small compressed payload to decompress to a potentially very large output. To guard against this, you may provide a maximum size this function is allowed to allocate in bytes via [param max_output_size]. Passing -1 will allow for unbounded output. If any positive value is passed, and the decompression exceeds that amount in bytes, then an error will be returned.
</description>

18
modules/freetype/SCsub
View File

@ -59,25 +59,7 @@ if env["builtin_freetype"]:
thirdparty_sources = [thirdparty_dir + file for file in thirdparty_sources]
if env["brotli"]:
thirdparty_brotli_dir = "#thirdparty/brotli/"
thirdparty_brotli_sources = [
"common/constants.c",
"common/context.c",
"common/dictionary.c",
"common/platform.c",
"common/shared_dictionary.c",
"common/transform.c",
"dec/bit_reader.c",
"dec/decode.c",
"dec/huffman.c",
"dec/state.c",
]
thirdparty_sources += [thirdparty_brotli_dir + file for file in thirdparty_brotli_sources]
env_freetype.Append(CPPDEFINES=["FT_CONFIG_OPTION_USE_BROTLI"])
env_freetype.Prepend(CPPPATH=[thirdparty_brotli_dir + "include"])
if env.get("use_ubsan") or env.get("use_asan") or env.get("use_tsan") or env.get("use_lsan") or env.get("use_msan"):
env_freetype.Append(CPPDEFINES=["BROTLI_BUILD_PORTABLE"])
if env["platform"] == "uwp":
# Include header for UWP to fix build issues

8
modules/freetype/config.py
View File

@ -2,13 +2,5 @@ def can_build(env, platform):
return True
def get_opts(platform):
from SCons.Variables import BoolVariable
return [
BoolVariable("brotli", "Enable Brotli decompressor for WOFF2 fonts support", True),
]
def configure(env):
pass