Nixyri/godot
1
0
Fork 0
mirror of https://github.com/godotengine/godot.git synced 2024-11-10 14:12:51 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

[IO] Add StreamPeerGZIP for streaming compression/decompression.

Browse Source 
Putting data results in writing the compressed/decompressed output into
a RingBuffer.
You can retrieve the content of the RingBuffer via get_data.

Support both gzip and deflate.
This commit is contained in:
Fabio Alessandrelli 2022-07-30 11:43:05 +02:00
parent 6f5704d86f
commit 273ba0794f
4 changed files with 329 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

209
core/io/stream_peer_gzip.cpp Normal file
View File

@ -0,0 +1,209 @@
/*************************************************************************/
/* stream_peer_gzip.cpp */
/*************************************************************************/
/* This file is part of: */
/* GODOT ENGINE */
/* https://godotengine.org */
/*************************************************************************/
/* Copyright (c) 2007-2022 Juan Linietsky, Ariel Manzur. */
/* Copyright (c) 2014-2022 Godot Engine contributors (cf. AUTHORS.md). */
/* */
/* Permission is hereby granted, free of charge, to any person obtaining */
/* a copy of this software and associated documentation files (the */
/* "Software"), to deal in the Software without restriction, including */
/* without limitation the rights to use, copy, modify, merge, publish, */
/* distribute, sublicense, and/or sell copies of the Software, and to */
/* permit persons to whom the Software is furnished to do so, subject to */
/* the following conditions: */
/* */
/* The above copyright notice and this permission notice shall be */
/* included in all copies or substantial portions of the Software. */
/* */
/* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, */
/* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF */
/* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.*/
/* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY */
/* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, */
/* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE */
/* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */
/*************************************************************************/
#include "core/io/stream_peer_gzip.h"
#include "core/io/zip_io.h"
#include <zlib.h>
void StreamPeerGZIP::_bind_methods() {
ClassDB::bind_method(D_METHOD("start_compression", "use_deflate", "buffer_size"), &StreamPeerGZIP::start_compression, DEFVAL(false), DEFVAL(65535));
ClassDB::bind_method(D_METHOD("start_decompression", "use_deflate", "buffer_size"), &StreamPeerGZIP::start_decompression, DEFVAL(false), DEFVAL(65535));
ClassDB::bind_method(D_METHOD("finish"), &StreamPeerGZIP::finish);
ClassDB::bind_method(D_METHOD("clear"), &StreamPeerGZIP::clear);
}
StreamPeerGZIP::StreamPeerGZIP() {
}
StreamPeerGZIP::~StreamPeerGZIP() {
_close();
}
void StreamPeerGZIP::_close() {
if (ctx) {
z_stream *strm = (z_stream *)ctx;
if (compressing) {
deflateEnd(strm);
} else {
inflateEnd(strm);
}
memfree(strm);
ctx = nullptr;
}
}
void StreamPeerGZIP::clear() {
_close();
rb.clear();
buffer.clear();
}
Error StreamPeerGZIP::start_compression(bool p_is_deflate, int buffer_size) {
return _start(true, p_is_deflate, buffer_size);
}
Error StreamPeerGZIP::start_decompression(bool p_is_deflate, int buffer_size) {
return _start(false, p_is_deflate, buffer_size);
}
Error StreamPeerGZIP::_start(bool p_compress, bool p_is_deflate, int buffer_size) {
ERR_FAIL_COND_V(ctx != nullptr, ERR_ALREADY_IN_USE);
clear();
compressing = p_compress;
rb.resize(nearest_shift(buffer_size - 1));
buffer.resize(1024);
// Create ctx.
ctx = memalloc(sizeof(z_stream));
z_stream &strm = *(z_stream *)ctx;
strm.next_in = Z_NULL;
strm.avail_in = 0;
strm.zalloc = zipio_alloc;
strm.zfree = zipio_free;
strm.opaque = Z_NULL;
int window_bits = p_is_deflate ? 15 : (15 + 16);
int err = Z_OK;
int level = Z_DEFAULT_COMPRESSION;
if (compressing) {
err = deflateInit2(&strm, level, Z_DEFLATED, window_bits, 8, Z_DEFAULT_STRATEGY);
} else {
err = inflateInit2(&strm, window_bits);
}
ERR_FAIL_COND_V(err != Z_OK, FAILED);
return OK;
}
Error StreamPeerGZIP::_process(uint8_t *p_dst, int p_dst_size, const uint8_t *p_src, int p_src_size, int &r_consumed, int &r_out, bool p_close) {
ERR_FAIL_COND_V(!ctx, ERR_UNCONFIGURED);
z_stream &strm = *(z_stream *)ctx;
strm.avail_in = p_src_size;
strm.avail_out = p_dst_size;
strm.next_in = (Bytef *)p_src;
strm.next_out = (Bytef *)p_dst;
int flush = p_close ? Z_FINISH : Z_NO_FLUSH;
if (compressing) {
int err = deflate(&strm, flush);
ERR_FAIL_COND_V(err != (p_close ? Z_STREAM_END : Z_OK), FAILED);
} else {
int err = inflate(&strm, flush);
ERR_FAIL_COND_V(err != Z_OK && err != Z_STREAM_END, FAILED);
}
r_out = p_dst_size - strm.avail_out;
r_consumed = p_src_size - strm.avail_in;
return OK;
}
Error StreamPeerGZIP::put_data(const uint8_t *p_data, int p_bytes) {
int wrote = 0;
Error err = put_partial_data(p_data, p_bytes, wrote);
if (err != OK) {
return err;
}
ERR_FAIL_COND_V(p_bytes != wrote, ERR_OUT_OF_MEMORY);
return OK;
}
Error StreamPeerGZIP::put_partial_data(const uint8_t *p_data, int p_bytes, int &r_sent) {
ERR_FAIL_COND_V(!ctx, ERR_UNCONFIGURED);
ERR_FAIL_COND_V(p_bytes < 0, ERR_INVALID_PARAMETER);
// Ensure we have enough space in temporary buffer.
if (buffer.size() < p_bytes) {
buffer.resize(p_bytes);
}
r_sent = 0;
while (r_sent < p_bytes && rb.space_left() > 1024) { // Keep the ring buffer size meaningful.
int sent = 0;
int to_write = 0;
// Compress or decompress
Error err = _process(buffer.ptrw(), MIN(buffer.size(), rb.space_left()), p_data + r_sent, p_bytes - r_sent, sent, to_write);
if (err != OK) {
return err;
}
// When decompressing, we might need to do another round.
r_sent += sent;
// We can't write more than this buffer is full.
if (sent == 0 && to_write == 0) {
return OK;
}
if (to_write) {
// Copy to ring buffer.
int wrote = rb.write(buffer.ptr(), to_write);
ERR_FAIL_COND_V(wrote != to_write, ERR_BUG);
}
}
return OK;
}
Error StreamPeerGZIP::get_data(uint8_t *p_buffer, int p_bytes) {
int received = 0;
Error err = get_partial_data(p_buffer, p_bytes, received);
if (err != OK) {
return err;
}
ERR_FAIL_COND_V(p_bytes != received, ERR_UNAVAILABLE);
return OK;
}
Error StreamPeerGZIP::get_partial_data(uint8_t *p_buffer, int p_bytes, int &r_received) {
ERR_FAIL_COND_V(p_bytes < 0, ERR_INVALID_PARAMETER);
r_received = MIN(p_bytes, rb.data_left());
if (r_received == 0) {
return OK;
}
int received = rb.read(p_buffer, r_received);
ERR_FAIL_COND_V(received != r_received, ERR_BUG);
return OK;
}
int StreamPeerGZIP::get_available_bytes() const {
return rb.data_left();
}
Error StreamPeerGZIP::finish() {
ERR_FAIL_COND_V(!ctx || !compressing, ERR_UNAVAILABLE);
// Ensure we have enough space in temporary buffer.
if (buffer.size() < 1024) {
buffer.resize(1024); // 1024 should be more than enough.
}
int consumed = 0;
int to_write = 0;
Error err = _process(buffer.ptrw(), 1024, nullptr, 0, consumed, to_write, true); // compress
if (err != OK) {
return err;
}
int wrote = rb.write(buffer.ptr(), to_write);
ERR_FAIL_COND_V(wrote != to_write, ERR_OUT_OF_MEMORY);
return OK;
}

76
core/io/stream_peer_gzip.h Normal file
View File

@ -0,0 +1,76 @@
/*************************************************************************/
/* stream_peer_gzip.h */
/*************************************************************************/
/* This file is part of: */
/* GODOT ENGINE */
/* https://godotengine.org */
/*************************************************************************/
/* Copyright (c) 2007-2022 Juan Linietsky, Ariel Manzur. */
/* Copyright (c) 2014-2022 Godot Engine contributors (cf. AUTHORS.md). */
/* */
/* Permission is hereby granted, free of charge, to any person obtaining */
/* a copy of this software and associated documentation files (the */
/* "Software"), to deal in the Software without restriction, including */
/* without limitation the rights to use, copy, modify, merge, publish, */
/* distribute, sublicense, and/or sell copies of the Software, and to */
/* permit persons to whom the Software is furnished to do so, subject to */
/* the following conditions: */
/* */
/* The above copyright notice and this permission notice shall be */
/* included in all copies or substantial portions of the Software. */
/* */
/* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, */
/* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF */
/* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.*/
/* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY */
/* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, */
/* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE */
/* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */
/*************************************************************************/
#ifndef STREAM_PEER_GZIP_H
#define STREAM_PEER_GZIP_H
#include "core/io/stream_peer.h"
#include "core/core_bind.h"
#include "core/io/compression.h"
#include "core/templates/ring_buffer.h"
class StreamPeerGZIP : public StreamPeer {
GDCLASS(StreamPeerGZIP, StreamPeer);
private:
void *ctx = nullptr; // Will hold our z_stream instance.
bool compressing = true;
RingBuffer<uint8_t> rb;
Vector<uint8_t> buffer;
Error _process(uint8_t *p_dst, int p_dst_size, const uint8_t *p_src, int p_src_size, int &r_consumed, int &r_out, bool p_close = false);
void _close();
Error _start(bool p_compress, bool p_is_deflate, int buffer_size = 65535);
protected:
static void _bind_methods();
public:
Error start_compression(bool p_is_deflate, int buffer_size = 65535);
Error start_decompression(bool p_is_deflate, int buffer_size = 65535);
Error finish();
void clear();
virtual Error put_data(const uint8_t *p_data, int p_bytes) override;
virtual Error put_partial_data(const uint8_t *p_data, int p_bytes, int &r_sent) override;
virtual Error get_data(uint8_t *p_buffer, int p_bytes) override;
virtual Error get_partial_data(uint8_t *p_buffer, int p_bytes, int &r_received) override;
virtual int get_available_bytes() const override;
StreamPeerGZIP();
~StreamPeerGZIP();
};
#endif // STREAM_PEER_GZIP_H

2
core/register_core_types.cpp
View File

@ -58,6 +58,7 @@
#include "core/io/resource_format_binary.h"
#include "core/io/resource_importer.h"
#include "core/io/resource_uid.h"
#include "core/io/stream_peer_gzip.h"
#include "core/io/stream_peer_tls.h"
#include "core/io/tcp_server.h"
#include "core/io/translation_loader_po.h"
@ -184,6 +185,7 @@ void register_core_types() {
GDREGISTER_ABSTRACT_CLASS(StreamPeer);
GDREGISTER_CLASS(StreamPeerExtension);
GDREGISTER_CLASS(StreamPeerBuffer);
GDREGISTER_CLASS(StreamPeerGZIP);
GDREGISTER_CLASS(StreamPeerTCP);
GDREGISTER_CLASS(TCPServer);

42
doc/classes/StreamPeerGZIP.xml Normal file
View File

@ -0,0 +1,42 @@
<?xml version="1.0" encoding="UTF-8" ?>
<class name="StreamPeerGZIP" inherits="StreamPeer" is_experimental="true" version="4.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:noNamespaceSchemaLocation="../class.xsd">
<brief_description>
Stream peer handling GZIP and deflate compression/decompresison.
</brief_description>
<description>
This class allows to compress or decompress data using GZIP/deflate in a streaming fashion. This is particularly useful when compressing or decompressing files that has to be sent through the network without having to allocate them all in memory.
After starting the stream via [method start_compression] (or [method start_decompression]), calling [method StreamPeer.put_partial_data] on this stream will compress (or decompress) the data, writing it to the internal buffer. Calling [method StreamPeer.get_available_bytes] will return the pending bytes in the internal buffer, and [method StreamPeer.get_partial_data] will retrieve the compressed (or decompressed) bytes from it. When the stream is over, you must call [method finish] to ensure the internal buffer is properly flushed (make sure to call [method StreamPeer.get_available_bytes] on last time to check if more data needs to be read after that).
</description>
<tutorials>
</tutorials>
<methods>
<method name="clear">
<return type="void" />
<description>
Clears this stream, resetting the internal state.
</description>
</method>
<method name="finish">
<return type="int" enum="Error" />
<description>
Finalizes the stream, compressing or decompressing any buffered chunk left.
</description>
</method>
<method name="start_compression">
<return type="int" enum="Error" />
<param index="0" name="use_deflate" type="bool" default="false" />
<param index="1" name="buffer_size" type="int" default="65535" />
<description>
Start the stream in compression mode with the given [param buffer_size], if [param use_deflate] is [code]true[/code] uses deflate instead of GZIP.
</description>
</method>
<method name="start_decompression">
<return type="int" enum="Error" />
<param index="0" name="use_deflate" type="bool" default="false" />
<param index="1" name="buffer_size" type="int" default="65535" />
<description>
Start the stream in decompression mode with the given [param buffer_size], if [param use_deflate] is [code]true[/code] uses deflate instead of GZIP.
</description>
</method>
</methods>
</class>