Menu
▾
▴
[Openvpn-devel] [PATCH v2] Implement the compression V2 data format for stub and lz4.
|
From: Arne S. <ar...@rf...> - 2016-01-01 13:43:12
|
Patch V2: Fix minor issues found by Steffan
It has been tested against v3 server and again itself. From James Mail:
Compression V2
I have observed that compression in many cases, even when
enabled, often does not produce packet size reduction
because much of the packet data typically generated by web
sessions is already compressed. Further, the single byte that
precedes the packet and indicates whether or not compression
occurred has the unfortunate side effect of misaligning the IP
packet in cases where compression did not occur. To remedy this,
I propose a Compression V2 header that is optimized for the
case where compression does not occur.
a. No compression occurred and first byte of IP/Ethernet packet
is NOT 0x50 (0 bytes of overhead and maintains alignment):
[ uncompressed IP/Ethernet packet ]
b. No compression occurred and first byte of IP/Ethernet packet
is 0x50 (2 bytes of overhead but unlikely since no known
IP packet can begin with 0x50):
[ 0x50 ] [ 0x00 ] [ uncompressed IP/Ethernet packet ]
c. Compression occurred (2 bytes of overhead):
[ 0x50 ] [ compression Alg ID ] [ compressed IP/Ethernet packet ]
Compression Alg ID is one-byte algorithm identifier
for LZ4 (0x1), LZO (0x2), or Snappy (0x3).
This approach has several beneficial effects:
1. In the common case where compression does not occur, no
compression op is required, therefore there is zero overhead.
2. When compression does not occur, the IP/Ethernet packet
alignment is retained.
3. This technique does not require any byte swapping with
the tail of the packet which can potentially incur an
expensive cache miss.
---
src/openvpn/comp-lz4.c | 198 ++++++++++++++++++++++++++++++++++++++-----------
src/openvpn/comp-lz4.h | 1 +
src/openvpn/comp.c | 81 +++++++++++++++-----
src/openvpn/comp.h | 13 ++++
src/openvpn/compstub.c | 52 +++++++++++++
src/openvpn/options.c | 10 +++
6 files changed, 294 insertions(+), 61 deletions(-)
diff --git a/src/openvpn/comp-lz4.c b/src/openvpn/comp-lz4.c
index 4651148..9274535 100644
--- a/src/openvpn/comp-lz4.c
+++ b/src/openvpn/comp-lz4.c
@@ -51,7 +51,7 @@ static void
lz4_compress_init (struct compress_context *compctx)
{
msg (D_INIT_MEDIUM, "LZ4 compression initializing");
- ASSERT(compctx->flags & COMP_F_SWAP);
+ ASSERT(compctx->flags & COMP_F_SWAP || compctx->flags & COMP_F_COMPV2);
}
static void
@@ -59,16 +59,12 @@ lz4_compress_uninit (struct compress_context *compctx)
{
}
-static void
-lz4_compress (struct buffer *buf, struct buffer work,
- struct compress_context *compctx,
- const struct frame* frame)
+static bool
+do_lz4_compress (struct buffer *buf,
+ struct buffer *work,
+ struct compress_context *compctx,
+ const struct frame* frame)
{
- bool compressed = false;
-
- if (buf->len <= 0)
- return;
-
/*
* In order to attempt compression, length must be at least COMPRESS_THRESHOLD.
*/
@@ -78,38 +74,58 @@ lz4_compress (struct buffer *buf, struct buffer work,
int zlen_max = ps + COMP_EXTRA_BUFFER (ps);
int zlen;
- ASSERT (buf_init (&work, FRAME_HEADROOM (frame)));
- ASSERT (buf_safe (&work, zlen_max));
+ ASSERT (buf_init (work, FRAME_HEADROOM (frame)));
+ ASSERT (buf_safe (work, zlen_max));
if (buf->len > ps)
{
dmsg (D_COMP_ERRORS, "LZ4 compression buffer overflow");
buf->len = 0;
- return;
+ return false;
}
- zlen = LZ4_compress_limitedOutput((const char *)BPTR(buf), (char *)BPTR(&work), BLEN(buf), zlen_max );
+ zlen = LZ4_compress_limitedOutput((const char *)BPTR(buf), (char *)BPTR(work), BLEN(buf), zlen_max );
if (zlen <= 0)
{
dmsg (D_COMP_ERRORS, "LZ4 compression error");
buf->len = 0;
- return;
+ return false;
}
- ASSERT (buf_safe (&work, zlen));
- work.len = zlen;
- compressed = true;
+ ASSERT (buf_safe (work, zlen));
+ work->len = zlen;
+
- dmsg (D_COMP, "LZ4 compress %d -> %d", buf->len, work.len);
+ dmsg (D_COMP, "LZ4 compress %d -> %d", buf->len, work->len);
compctx->pre_compress += buf->len;
- compctx->post_compress += work.len;
+ compctx->post_compress += work->len;
+ return true;
}
+ return false;
+}
- /* did compression save us anything? */
+
+static void
+lz4_compress (struct buffer *buf, struct buffer work,
+ struct compress_context *compctx,
+ const struct frame* frame)
+{
+ bool compressed;
+ if (buf->len <= 0)
+ return;
+
+ compressed = do_lz4_compress(buf, &work, compctx, frame);
+
+ /* On Error just return */
+ if (buf->len == 0)
+ return;
+
+ /* did compression save us anything? Include 2 byte compression header
+ in calculation */
{
uint8_t comp_head_byte = NO_COMPRESS_BYTE_SWAP;
- if (compressed && work.len < buf->len)
+ if (compressed && work.len +2 < buf->len)
{
*buf = work;
comp_head_byte = LZ4_COMPRESS_BYTE;
@@ -128,13 +144,73 @@ lz4_compress (struct buffer *buf, struct buffer work,
}
}
+
+static void
+lz4v2_compress (struct buffer *buf, struct buffer work,
+ struct compress_context *compctx,
+ const struct frame* frame)
+{
+ bool compressed;
+ if (buf->len <= 0)
+ return;
+
+ compressed = do_lz4_compress(buf, &work, compctx, frame);
+ if (buf->len <= 0)
+ return;
+
+ /* On Error just return */
+ if (buf->len == 0)
+ return;
+
+ /* did compression save us anything? */
+ {
+ if (compressed && work.len < buf->len)
+ {
+ ASSERT(buf_prepend(&work, 2));
+ uint8_t *head = BPTR (&work);
+ head[0] = COMP_ALGV2_INDICATOR_BYTE;
+ head[1] = COMP_ALGV2_LZ4;
+ *buf = work;
+ }
+ else
+ {
+ compv2_escape_data_ifneeded(buf);
+ }
+ }
+}
+
+void
+do_lz4_decompress(size_t zlen_max,
+ struct buffer *work,
+ struct buffer *buf,
+ struct compress_context *compctx)
+{
+ int uncomp_len;
+ ASSERT (buf_safe (work, zlen_max));
+ uncomp_len = LZ4_decompress_safe((const char *)BPTR(buf), (char *)BPTR(work), (size_t)BLEN(buf), zlen_max);
+ if (uncomp_len <= 0)
+ {
+ dmsg (D_COMP_ERRORS, "LZ4 decompression error: %d", uncomp_len);
+ buf->len = 0;
+ return;
+ }
+
+ ASSERT (buf_safe (work, uncomp_len));
+ work->len = uncomp_len;
+
+ dmsg (D_COMP, "LZ4 decompress %d -> %d", buf->len, work->len);
+ compctx->pre_decompress += buf->len;
+ compctx->post_decompress += work->len;
+
+ *buf = *work;
+}
+
static void
lz4_decompress (struct buffer *buf, struct buffer work,
- struct compress_context *compctx,
- const struct frame* frame)
+ struct compress_context *compctx,
+ const struct frame* frame)
{
size_t zlen_max = EXPANDED_SIZE (frame);
- int uncomp_len;
uint8_t c; /* flag indicating whether or not our peer compressed */
if (buf->len <= 0)
@@ -152,23 +228,7 @@ lz4_decompress (struct buffer *buf, struct buffer work,
if (c == LZ4_COMPRESS_BYTE) /* packet was compressed */
{
- ASSERT (buf_safe (&work, zlen_max));
- uncomp_len = LZ4_decompress_safe((const char *)BPTR(buf), (char *)BPTR(&work), (size_t)BLEN(buf), zlen_max);
- if (uncomp_len <= 0)
- {
- dmsg (D_COMP_ERRORS, "LZ4 decompression error: %d", uncomp_len);
- buf->len = 0;
- return;
- }
-
- ASSERT (buf_safe (&work, uncomp_len));
- work.len = uncomp_len;
-
- dmsg (D_COMP, "LZ4 decompress %d -> %d", buf->len, work.len);
- compctx->pre_decompress += buf->len;
- compctx->post_decompress += work.len;
-
- *buf = work;
+ do_lz4_decompress(zlen_max, &work, buf, compctx);
}
else if (c == NO_COMPRESS_BYTE_SWAP) /* packet was not compressed */
{
@@ -181,6 +241,52 @@ lz4_decompress (struct buffer *buf, struct buffer work,
}
}
+static void
+lz4v2_decompress (struct buffer *buf, struct buffer work,
+ struct compress_context *compctx,
+ const struct frame* frame)
+{
+ size_t zlen_max = EXPANDED_SIZE (frame);
+ uint8_t c; /* flag indicating whether or not our peer compressed */
+
+ if (buf->len <= 0)
+ return;
+
+ ASSERT (buf_init (&work, FRAME_HEADROOM (frame)));
+
+ /* do unframing/swap (assumes buf->len > 0) */
+ uint8_t *head = BPTR (buf);
+ c = *head;
+
+ /* Not compressed */
+ if (c != COMP_ALGV2_INDICATOR_BYTE) {
+ return;
+ }
+
+ /* Packet to short to make sense */
+ if (buf->len <= 1)
+ {
+ buf->len=0;
+ return;
+ }
+
+ c = head[1];
+ if (c == COMP_ALGV2_LZ4) /* packet was compressed */
+ {
+ buf_advance(buf,2);
+ do_lz4_decompress(zlen_max, &work, buf, compctx);
+ }
+ else if (c == COMP_ALGV2_UNCOMPRESSED)
+ {
+ buf_advance(buf,2);
+ }
+ else
+ {
+ dmsg (D_COMP_ERRORS, "Bad LZ4v2 decompression header byte: %d", c);
+ buf->len = 0;
+ }
+}
+
const struct compress_alg lz4_alg = {
"lz4",
lz4_compress_init,
@@ -189,6 +295,14 @@ const struct compress_alg lz4_alg = {
lz4_decompress
};
+const struct compress_alg lz4v2_alg = {
+ "lz4v2",
+ lz4_compress_init,
+ lz4_compress_uninit,
+ lz4v2_compress,
+ lz4v2_decompress
+};
+
#else
static void dummy(void) {}
#endif /* ENABLE_LZ4 */
diff --git a/src/openvpn/comp-lz4.h b/src/openvpn/comp-lz4.h
index ca1dfa9..9d3c664 100644
--- a/src/openvpn/comp-lz4.h
+++ b/src/openvpn/comp-lz4.h
@@ -31,6 +31,7 @@
#include "buffer.h"
extern const struct compress_alg lz4_alg;
+extern const struct compress_alg lz4v2_alg;
struct lz4_workspace
{
diff --git a/src/openvpn/comp.c b/src/openvpn/comp.c
index 706ad7e..6cf9a83 100644
--- a/src/openvpn/comp.c
+++ b/src/openvpn/comp.c
@@ -42,34 +42,75 @@ struct compress_context *
comp_init(const struct compress_options *opt)
{
struct compress_context *compctx = NULL;
- switch (opt->alg)
+ if (opt->flags & COMP_F_COMPV2) {
+ switch (opt->alg)
+ {
+ case COMP_ALGV2_UNCOMPRESSED:
+ ALLOC_OBJ_CLEAR (compctx, struct compress_context);
+ compctx->flags = opt->flags;
+ compctx->alg = compv2_stub_alg;
+ break;
+#ifdef ENABLE_LZ4
+ case COMP_ALGV2_LZ4:
+ ALLOC_OBJ_CLEAR (compctx, struct compress_context);
+ compctx->flags = opt->flags;
+ compctx->alg = lz4v2_alg;
+ break;
+#endif
+ default:
+ break;
+ }
+ }
+ else
{
- case COMP_ALG_STUB:
- ALLOC_OBJ_CLEAR (compctx, struct compress_context);
- compctx->flags = opt->flags;
- compctx->alg = comp_stub_alg;
- (*compctx->alg.compress_init)(compctx);
- break;
+ switch (opt->alg)
+ {
+ case COMP_ALG_STUB:
+ ALLOC_OBJ_CLEAR (compctx, struct compress_context);
+ compctx->flags = opt->flags;
+ compctx->alg = comp_stub_alg;
+ (*compctx->alg.compress_init)(compctx);
+ break;
#ifdef ENABLE_LZO
- case COMP_ALG_LZO:
- ALLOC_OBJ_CLEAR (compctx, struct compress_context);
- compctx->flags = opt->flags;
- compctx->alg = lzo_alg;
- (*compctx->alg.compress_init)(compctx);
- break;
+ case COMP_ALG_LZO:
+ ALLOC_OBJ_CLEAR (compctx, struct compress_context);
+ compctx->flags = opt->flags;
+ compctx->alg = lzo_alg;
+ (*compctx->alg.compress_init)(compctx);
+ break;
#endif
#ifdef ENABLE_LZ4
- case COMP_ALG_LZ4:
- ALLOC_OBJ_CLEAR (compctx, struct compress_context);
- compctx->flags = opt->flags;
- compctx->alg = lz4_alg;
- (*compctx->alg.compress_init)(compctx);
- break;
+ case COMP_ALG_LZ4:
+ ALLOC_OBJ_CLEAR (compctx, struct compress_context);
+ compctx->flags = opt->flags;
+ compctx->alg = lz4_alg;
+ (*compctx->alg.compress_init)(compctx);
+ break;
#endif
+ }
}
return compctx;
}
+/* In the v2 compression schmemes, an uncompressed packet has
+ * has no opcode in front, unless the first byte is 0x50. In this
+ * case the packet needs to be escaped */
+void
+compv2_escape_data_ifneeded (struct buffer *buf)
+{
+ uint8_t *head = BPTR (buf);
+ if (head[0] != COMP_ALGV2_INDICATOR_BYTE)
+ return;
+
+ /* Header is 0x50 */
+ ASSERT(buf_prepend(buf, 2));
+
+ head = BPTR (buf);
+ head[0] = COMP_ALGV2_INDICATOR_BYTE;
+ head[1] = COMP_ALGV2_UNCOMPRESSED;
+}
+
+
void
comp_uninit(struct compress_context *compctx)
{
@@ -120,6 +161,7 @@ comp_generate_peer_info_string(const struct compress_options *opt, struct buffer
{
#if defined(ENABLE_LZ4)
buf_printf (out, "IV_LZ4=1\n");
+ buf_printf (out, "IV_LZ4v2=1\n");
#endif
#if defined(ENABLE_LZO)
buf_printf (out, "IV_LZO=1\n");
@@ -129,6 +171,7 @@ comp_generate_peer_info_string(const struct compress_options *opt, struct buffer
if (!lzo_avail)
buf_printf (out, "IV_LZO_STUB=1\n");
buf_printf (out, "IV_COMP_STUB=1\n");
+ buf_printf (out, "IV_COMP_STUBv2=1\n");
}
}
diff --git a/src/openvpn/comp.h b/src/openvpn/comp.h
index 716b1c0..2cfabef 100644
--- a/src/openvpn/comp.h
+++ b/src/openvpn/comp.h
@@ -43,11 +43,21 @@
#define COMP_ALG_SNAPPY 3 /* Snappy algorithm (no longer supported) */
#define COMP_ALG_LZ4 4 /* LZ4 algorithm */
+
+/* algorithm v2 */
+#define COMP_ALGV2_INDICATOR_BYTE 0x50
+#define COMP_ALGV2_UNCOMPRESSED 0
+#define COMP_ALGV2_LZ4 1
+#define COMP_ALGV2_LZO 2
+#define COMP_ALGV2_SNAPPY 3
+
/* Compression flags */
#define COMP_F_ADAPTIVE (1<<0) /* COMP_ALG_LZO only */
#define COMP_F_ASYM (1<<1) /* only downlink is compressed, not uplink */
#define COMP_F_SWAP (1<<2) /* initial command byte is swapped with last byte in buffer to preserve payload alignment */
#define COMP_F_ADVERTISE_STUBS_ONLY (1<<3) /* tell server that we only support compression stubs */
+#define COMP_F_COMPV2 (1<<8) /* Use the V2 compression methode that only uses opcodes if compression is actually needed */
+
/*
* Length of prepended prefix on compressed packets
@@ -145,6 +155,7 @@ struct compress_context
};
extern const struct compress_alg comp_stub_alg;
+extern const struct compress_alg compv2_stub_alg;
struct compress_context *comp_init(const struct compress_options *opt);
@@ -157,6 +168,8 @@ void comp_print_stats (const struct compress_context *compctx, struct status_out
void comp_generate_peer_info_string(const struct compress_options *opt, struct buffer *out);
+void compv2_escape_data_ifneeded (struct buffer *buf);
+
static inline bool
comp_enabled(const struct compress_options *info)
{
diff --git a/src/openvpn/compstub.c b/src/openvpn/compstub.c
index 2ab7163..0f21a6f 100644
--- a/src/openvpn/compstub.c
+++ b/src/openvpn/compstub.c
@@ -73,6 +73,8 @@ stub_compress (struct buffer *buf, struct buffer work,
}
}
+
+
static void
stub_decompress (struct buffer *buf, struct buffer work,
struct compress_context *compctx,
@@ -105,6 +107,56 @@ stub_decompress (struct buffer *buf, struct buffer work,
}
}
+
+static void
+stubv2_compress (struct buffer *buf, struct buffer work,
+ struct compress_context *compctx,
+ const struct frame* frame)
+{
+ if (buf->len <= 0)
+ return;
+
+ compv2_escape_data_ifneeded (buf);
+}
+
+static void
+stubv2_decompress (struct buffer *buf, struct buffer work,
+ struct compress_context *compctx,
+ const struct frame* frame)
+{
+ struct gc_arena gc = gc_new();
+ if (buf->len <= 0)
+ return;
+
+ uint8_t *head = BPTR (buf);
+
+ /* no compression */
+ if (head[0] != COMP_ALGV2_INDICATOR_BYTE)
+ return;
+
+ /* compression header (0x50) is present */
+ buf_advance(buf, 1);
+
+ /* Packet buffer too short (only 1 byte) */
+ if (buf->len <= 0)
+ return;
+
+ head = BPTR (buf);
+ if (head[0] != COMP_ALGV2_UNCOMPRESSED) {
+ dmsg (D_COMP_ERRORS, "Bad compression stubv2 decompression header byte: %d", *head);
+ buf->len = 0;
+ }
+ gc_free(&gc);
+}
+
+const struct compress_alg compv2_stub_alg = {
+ "stubv2",
+ stub_compress_init,
+ stub_compress_uninit,
+ stubv2_compress,
+ stubv2_decompress
+};
+
const struct compress_alg comp_stub_alg = {
"stub",
stub_compress_init,
diff --git a/src/openvpn/options.c b/src/openvpn/options.c
index 5a83897..2b29704 100644
--- a/src/openvpn/options.c
+++ b/src/openvpn/options.c
@@ -6323,6 +6323,11 @@ add_option (struct options *options,
options->comp.alg = COMP_ALG_STUB;
options->comp.flags = (COMP_F_SWAP|COMP_F_ADVERTISE_STUBS_ONLY);
}
+ else if (streq(p[1], "stub-v2"))
+ {
+ options->comp.alg = COMP_ALGV2_UNCOMPRESSED;
+ options->comp.flags = (COMP_F_COMPV2);
+ }
#if defined(ENABLE_LZO)
else if (streq (p[1], "lzo"))
{
@@ -6336,6 +6341,11 @@ add_option (struct options *options,
options->comp.alg = COMP_ALG_LZ4;
options->comp.flags = COMP_F_SWAP;
}
+ else if (streq (p[1], "lz4-v2"))
+ {
+ options->comp.alg = COMP_ALGV2_LZ4;
+ options->comp.flags = COMP_F_COMPV2;
+ }
#endif
else
{
--
2.5.4 (Apple Git-61)
|
