diff options
| -rw-r--r-- | common.h | 1 | ||||
| -rw-r--r-- | config.m4 | 31 | ||||
| -rw-r--r-- | library.c | 138 | ||||
| -rw-r--r-- | redis.c | 14 | ||||
| -rw-r--r-- | redis_commands.c | 3 | ||||
| -rw-r--r-- | tests/RedisTest.php | 22 |
6 files changed, 208 insertions, 1 deletions
@@ -100,6 +100,7 @@ typedef enum { #define REDIS_COMPRESSION_NONE 0 #define REDIS_COMPRESSION_LZF 1 #define REDIS_COMPRESSION_ZSTD 2 +#define REDIS_COMPRESSION_LZ4 3 /* SCAN options */ #define REDIS_SCAN_NORETRY 0 @@ -29,6 +29,12 @@ PHP_ARG_ENABLE(redis-zstd, whether to enable Zstd compression, PHP_ARG_WITH(libzstd, use system libsztd, [ --with-libzstd[=DIR] Use system libzstd], yes, no) +PHP_ARG_ENABLE(redis-lz4, whether to enable lz4 compression, +[ --enable-redis-lz4 Enable lz4 compression support], no, no) + +PHP_ARG_WITH(liblz4, use system liblz4, +[ --with-liblz4[=DIR] Use system liblz4], no, no) + if test "$PHP_REDIS" != "no"; then if test "$PHP_REDIS_SESSION" != "no"; then @@ -194,6 +200,31 @@ if test "$PHP_REDIS" != "no"; then fi fi + if test "$PHP_REDIS_LZ4" != "no"; then + AC_DEFINE(HAVE_REDIS_LZ4, 1, [ ]) + AC_MSG_CHECKING(for liblz4 files in default path) + for i in $PHP_LIBLZ4 /usr/local /usr; do + if test -r $i/include/lz4.h; then + AC_MSG_RESULT(found in $i) + LIBLZ4_DIR=$i + break + fi + done + if test -z "$LIBLZ4_DIR"; then + AC_MSG_RESULT([not found]) + AC_MSG_ERROR([Please reinstall the liblz4 distribution]) + fi + PHP_CHECK_LIBRARY(lz4, LZ4_compress, + [ + PHP_ADD_LIBRARY_WITH_PATH(zstd, $LIBLZ4_DIR/$PHP_LIBDIR, REDIS_SHARED_LIBADD) + ], [ + AC_MSG_ERROR([could not find usable liblz4]) + ], [ + -L$LIBLZ4_DIR/$PHP_LIBDIR + ]) + PHP_SUBST(REDIS_SHARED_LIBADD) + fi + if test "$PHP_REDIS_ZSTD" != "no"; then AC_DEFINE(HAVE_REDIS_ZSTD, 1, [ ]) if test "$PHP_LIBZSTD" != "no"; then @@ -25,6 +25,26 @@ #include <zstd.h> #endif +#ifdef HAVE_REDIS_LZ4 +#include <lz4.h> +#include <lz4hc.h> + +/* uint8_t crf + int length */ +#define REDIS_LZ4_HDR_SIZE (sizeof(uint8_t) + sizeof(int)) +#if defined(LZ4HC_CLEVEL_MAX) +/* version >= 1.7.5 */ +#define REDIS_LZ4_MAX_CLEVEL LZ4HC_CLEVEL_MAX + +#elif defined (LZ4HC_MAX_CLEVEL) +/* version >= 1.7.3 */ +#define REDIS_LZ4_MAX_CLEVEL LZ4HC_MAX_CLEVEL + +#else +/* older versions */ +#define REDIS_LZ4_MAX_CLEVEL 12 +#endif +#endif + #include <zend_exceptions.h> #include "php_redis.h" #include "library.h" @@ -2280,6 +2300,26 @@ PHP_REDIS_API void redis_free_socket(RedisSock *redis_sock) efree(redis_sock); } +#ifdef HAVE_REDIS_LZ4 +/* Implementation of CRC8 for our LZ4 checksum value */ +static uint8_t crc8(unsigned char *input, size_t len) { + size_t i; + uint8_t crc = 0xFF; + + while (len--) { + crc ^= *input++; + for (i = 0; i < 8; i++) { + if (crc & 0x80) + crc = (uint8_t)(crc << 1) ^ 0x31; + else + crc <<= 1; + } + } + + return crc; +} +#endif + PHP_REDIS_API int redis_pack(RedisSock *redis_sock, zval *z, char **val, size_t *val_len) { @@ -2288,6 +2328,12 @@ redis_pack(RedisSock *redis_sock, zval *z, char **val, size_t *val_len) size_t len; valfree = redis_serialize(redis_sock, z, &buf, &len); + if (redis_sock->compression == REDIS_COMPRESSION_NONE) { + *val = buf; + *val_len = len; + return valfree; + } + switch (redis_sock->compression) { case REDIS_COMPRESSION_LZF: #ifdef HAVE_REDIS_LZF @@ -2342,6 +2388,54 @@ redis_pack(RedisSock *redis_sock, zval *z, char **val, size_t *val_len) } #endif break; + case REDIS_COMPRESSION_LZ4: +#ifdef HAVE_REDIS_LZ4 + { + /* Compressing empty data is pointless */ + if (len < 1) + break; + + /* Compressing more than INT_MAX bytes would require multiple blocks */ + if (len > INT_MAX) { + php_error_docref(NULL, E_WARNING, + "LZ4: compressing > %d bytes not supported", INT_MAX); + break; + } + + int old_len = len, lz4len, lz4bound; + uint8_t crc = crc8((unsigned char*)&old_len, sizeof(old_len)); + char *lz4buf, *lz4pos; + + lz4bound = LZ4_compressBound(len); + lz4buf = emalloc(REDIS_LZ4_HDR_SIZE + lz4bound); + lz4pos = lz4buf; + + /* Copy and move past crc8 length checksum */ + memcpy(lz4pos, &crc, sizeof(crc)); + lz4pos += sizeof(crc); + + /* Copy and advance past length */ + memcpy(lz4pos, &old_len, sizeof(old_len)); + lz4pos += sizeof(old_len); + + if (redis_sock->compression_level <= 0 || redis_sock->compression_level > REDIS_LZ4_MAX_CLEVEL) { + lz4len = LZ4_compress_default(buf, lz4pos, old_len, lz4bound); + } else { + lz4len = LZ4_compress_HC(buf, lz4pos, old_len, lz4bound, redis_sock->compression_level); + } + + if (lz4len <= 0) { + efree(lz4buf); + break; + } + + if (valfree) efree(buf); + *val = lz4buf; + *val_len = lz4len + REDIS_LZ4_HDR_SIZE; + return 1; + } +#endif + break; } *val = buf; *val_len = len; @@ -2359,9 +2453,12 @@ redis_unpack(RedisSock *redis_sock, const char *val, int val_len, zval *z_ret) int i; uint32_t res; - errno = E2BIG; + if (val_len == 0) + break; + /* start from two-times bigger buffer and * increase it exponentially if needed */ + errno = E2BIG; for (i = 2; errno == E2BIG; i *= 2) { data = emalloc(i * val_len); if ((res = lzf_decompress(val, val_len, data, i * val_len)) == 0) { @@ -2399,6 +2496,45 @@ redis_unpack(RedisSock *redis_sock, const char *val, int val_len, zval *z_ret) } #endif break; + case REDIS_COMPRESSION_LZ4: +#ifdef HAVE_REDIS_LZ4 + { + char *data; + int datalen; + uint8_t lz4crc; + + /* We must have at least enough bytes for our header, and can't have more than + * INT_MAX + our header size. */ + if (val_len < REDIS_LZ4_HDR_SIZE || val_len > INT_MAX + REDIS_LZ4_HDR_SIZE) + break; + + /* Operate on copies in case our CRC fails */ + const char *copy = val; + size_t copylen = val_len; + + /* Read in our header bytes */ + memcpy(&lz4crc, copy, sizeof(uint8_t)); + copy += sizeof(uint8_t); copylen -= sizeof(uint8_t); + memcpy(&datalen, copy, sizeof(int)); + copy += sizeof(int); copylen -= sizeof(int); + + /* Make sure our CRC matches (TODO: Maybe issue a docref error?) */ + if (crc8((unsigned char*)&datalen, sizeof(datalen)) != lz4crc) + break; + + /* Finally attempt decompression */ + data = emalloc(datalen); + if (LZ4_decompress_safe(copy, data, copylen, datalen) > 0) { + if (redis_unserialize(redis_sock, data, datalen, z_ret) == 0) { + ZVAL_STRINGL(z_ret, data, datalen); + } + efree(data); + return 1; + } + efree(data); + } +#endif + break; } return redis_unserialize(redis_sock, val, val_len, z_ret); } @@ -40,6 +40,10 @@ #include <zstd.h> #endif +#ifdef HAVE_REDIS_LZ4 +#include <lz4.h> +#endif + #ifdef PHP_SESSION extern ps_module ps_mod_redis; extern ps_module ps_mod_redis_cluster; @@ -719,6 +723,10 @@ static void add_class_constants(zend_class_entry *ce, int is_cluster) { zend_declare_class_constant_long(ce, ZEND_STRL("COMPRESSION_ZSTD_MAX"), ZSTD_maxCLevel()); #endif +#ifdef HAVE_REDIS_LZ4 + zend_declare_class_constant_long(ce, ZEND_STRL("COMPRESSION_LZ4"), REDIS_COMPRESSION_LZ4); +#endif + /* scan options*/ zend_declare_class_constant_long(ce, ZEND_STRL("OPT_SCAN"), REDIS_OPT_SCAN); zend_declare_class_constant_long(ce, ZEND_STRL("SCAN_RETRY"), REDIS_SCAN_RETRY); @@ -887,6 +895,12 @@ PHP_MINFO_FUNCTION(redis) } smart_str_appends(&names, "zstd"); #endif +#ifdef HAVE_REDIS_LZ4 + if (names.s) { + smart_str_appends(&names, ", "); + } + smart_str_appends(&names, "lz4"); +#endif if (names.s) { smart_str_0(&names); php_info_print_table_row(2, "Available compression", ZSTR_VAL(names.s)); diff --git a/redis_commands.c b/redis_commands.c index f07fdfcb..97442de3 100644 --- a/redis_commands.c +++ b/redis_commands.c @@ -4003,6 +4003,9 @@ void redis_setoption_handler(INTERNAL_FUNCTION_PARAMETERS, #ifdef HAVE_REDIS_ZSTD || val_long == REDIS_COMPRESSION_ZSTD #endif +#ifdef HAVE_REDIS_LZ4 + || val_long == REDIS_COMPRESSION_LZ4 +#endif ) { redis_sock->compression = val_long; RETURN_TRUE; diff --git a/tests/RedisTest.php b/tests/RedisTest.php index f029d80d..3cf753d8 100644 --- a/tests/RedisTest.php +++ b/tests/RedisTest.php @@ -4519,6 +4519,16 @@ class Redis_Test extends TestSuite $this->checkCompression(Redis::COMPRESSION_ZSTD, 9); } + + public function testCompressionLZ4() + { + if (!defined('Redis::COMPRESSION_LZ4')) { + $this->markTestSkipped(); + } + $this->checkCompression(Redis::COMPRESSION_LZ4, 0); + $this->checkCompression(Redis::COMPRESSION_LZ4, 9); + } + private function checkCompression($mode, $level) { $this->assertTrue($this->redis->setOption(Redis::OPT_COMPRESSION, $mode) === TRUE); // set ok @@ -4530,6 +4540,18 @@ class Redis_Test extends TestSuite $val = 'xxxxxxxxxx'; $this->redis->set('key', $val); $this->assertEquals($val, $this->redis->get('key')); + + /* Empty data */ + $this->redis->set('key', ''); + $this->assertEquals('', $this->redis->get('key')); + + /* Iterate through class sizes */ + for ($i = 1; $i <= 65536; $i *= 2) { + foreach ([str_repeat('A', $i), random_bytes($i)] as $val) { + $this->redis->set('key', $val); + $this->assertEquals($val, $this->redis->get('key')); + } + } } public function testDumpRestore() { |