Update Files

Kinc/Sources/kinc/libs/core/compression/README.md

@@ -0,0 +1,17 @@
+HTTP compression
+----------------
+
+This directory contains generic compression transforms that can be applied to
+specifically HTTP content streams, after the header, be it h1 or h2.
+
+The compression transforms expose an "ops" type struct and a compressor name
+as used by `content-encoding`... the ops struct definition can be found in
+./private-lib-roles-http-compression.h.
+
+Because the compression transform depends on being able to send on its output
+before it can process new input, the transform adds a new kind of buflist
+`wsi->buflist_comp` that represents pre-compression transform data
+("input data" from the perspective of the compression transform) that was
+delivered to be processed but couldn't be accepted.
+
+Currently, zlib 'deflate' and brotli 'br' are supported on the server side.

Kinc/Sources/kinc/libs/core/compression/brotli/brotli.c

@@ -0,0 +1,124 @@
+/*
+ * libwebsockets - small server side websockets and web server implementation
+ *
+ * Copyright (C) 2010 - 2019 Andy Green <andy@warmcat.com>
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ */
+
+#include "private-lib-core.h"
+
+static int
+lcs_init_compression_brotli(lws_comp_ctx_t *ctx, int decomp)
+{
+	ctx->is_decompression = (unsigned char)!!decomp;
+
+	if (!decomp) {
+		ctx->u.br_en = BrotliEncoderCreateInstance(NULL, NULL, NULL);
+		if (ctx->u.br_en) {
+			BrotliEncoderSetParameter(ctx->u.br_en,
+					BROTLI_PARAM_MODE, BROTLI_MODE_TEXT);
+			BrotliEncoderSetParameter(ctx->u.br_en,
+				BROTLI_PARAM_QUALITY, BROTLI_MIN_QUALITY);
+		}
+	}
+	else
+		ctx->u.br_de = BrotliDecoderCreateInstance(NULL, NULL, NULL);
+
+	return !ctx->u.br_de;
+}
+
+static int
+lcs_process_brotli(lws_comp_ctx_t *ctx, const void *in, size_t *ilen_iused,
+		   void *out, size_t *olen_oused)
+{
+	size_t a_in, a_out, t_out;
+	const uint8_t *n_in;
+	uint8_t *n_out;
+	int n;
+
+	n_in = (void *)in;
+	a_in = *ilen_iused;
+	a_out = *olen_oused;
+	n_out = out;
+	t_out = 0;
+
+	if (!ctx->is_decompression) {
+
+		if (!a_in && !BrotliEncoderHasMoreOutput(ctx->u.br_en)) {
+			*olen_oused = 0;
+
+			goto bail;
+		}
+
+		n = BROTLI_OPERATION_PROCESS;
+		if (!ctx->buflist_comp && ctx->final_on_input_side)
+			n = BROTLI_OPERATION_FINISH;
+
+		if (BrotliEncoderCompressStream(ctx->u.br_en, n, &a_in, &n_in,
+						&a_out, &n_out, &t_out) ==
+		    BROTLI_FALSE) {
+			lwsl_err("brotli encode failed\n");
+
+			return -1;
+		}
+
+		ctx->may_have_more = !a_out;
+
+	} else {
+		n = BrotliDecoderDecompressStream(ctx->u.br_de, &a_in, &n_in,
+						  &a_out, &n_out, &t_out);
+
+		switch (n) {
+		case BROTLI_DECODER_RESULT_ERROR:
+			lwsl_err("brotli decoder error\n");
+			return -1;
+		}
+	}
+
+	*ilen_iused -= a_in;
+	*olen_oused -= a_out;
+
+bail:
+	if (!ctx->is_decompression)
+		return BrotliEncoderIsFinished(ctx->u.br_en);
+	else
+		return BrotliDecoderIsFinished(ctx->u.br_de);
+}
+
+static void
+lcs_destroy_brotli(lws_comp_ctx_t *ctx)
+{
+	if (!ctx)
+		return;
+
+	if (!(*ctx).is_decompression)
+		BrotliEncoderDestroyInstance((*ctx).u.br_en);
+	else
+		BrotliDecoderDestroyInstance((*ctx).u.br_de);
+
+	(*ctx).u.generic_ctx_ptr = NULL;
+}
+
+struct lws_compression_support lcs_brotli = {
+	/* .encoding_name */		"br",
+	/* .init_compression */		lcs_init_compression_brotli,
+	/* .process */			lcs_process_brotli,
+	/* .destroy */			lcs_destroy_brotli,
+};

Kinc/Sources/kinc/libs/core/compression/deflate/deflate.c

@@ -0,0 +1,113 @@
+/*
+ * libwebsockets - small server side websockets and web server implementation
+ *
+ * Copyright (C) 2010 - 2019 Andy Green <andy@warmcat.com>
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ */
+
+#include "private-lib-core.h"
+
+static int
+lcs_init_compression_deflate(lws_comp_ctx_t *ctx, int decomp)
+{
+	int n;
+
+	ctx->is_decompression = !!decomp;
+	ctx->u.deflate = lws_malloc(sizeof(*ctx->u.deflate), __func__);
+
+	if (!ctx->u.deflate)
+		return 2;
+
+	memset(ctx->u.deflate, 0, sizeof(*ctx->u.deflate));
+
+	if (!decomp &&
+	    (n = deflateInit2(ctx->u.deflate, 1, Z_DEFLATED, -15, 8,
+			 Z_DEFAULT_STRATEGY)) != Z_OK) {
+		lwsl_err("deflate init failed: %d\n", n);
+		lws_free_set_NULL(ctx->u.deflate);
+
+		return 1;
+	}
+
+	if (decomp &&
+	    inflateInit2(ctx->u.deflate, 16 + 15) != Z_OK) {
+		lws_free_set_NULL(ctx->u.deflate);
+		return 1;
+	}
+
+	return 0;
+}
+
+static int
+lcs_process_deflate(lws_comp_ctx_t *ctx, const void *in, size_t *ilen_iused,
+		    void *out, size_t *olen_oused)
+{
+	size_t olen_oused_in = *olen_oused;
+	int n;
+
+	ctx->u.deflate->next_in = (void *)in;
+	ctx->u.deflate->avail_in = (unsigned int)*ilen_iused;
+
+	ctx->u.deflate->next_out = out;
+	ctx->u.deflate->avail_out = (unsigned int)*olen_oused;
+
+	if (!ctx->is_decompression)
+		n = deflate(ctx->u.deflate, Z_SYNC_FLUSH);
+	else
+		n = inflate(ctx->u.deflate, Z_SYNC_FLUSH);
+
+	switch (n) {
+	case Z_NEED_DICT:
+	case Z_STREAM_ERROR:
+	case Z_DATA_ERROR:
+	case Z_MEM_ERROR:
+		lwsl_err("zlib error inflate %d\n", n);
+		return -1;
+	}
+
+	*ilen_iused -= ctx->u.deflate->avail_in;
+	*olen_oused -= ctx->u.deflate->avail_out;
+
+	/* it's ambiguous with zlib... */
+	ctx->may_have_more = (*olen_oused == olen_oused_in);
+
+	return n == Z_STREAM_END;
+}
+
+static void
+lcs_destroy_deflate(lws_comp_ctx_t *ctx)
+{
+	if (!ctx)
+		return;
+
+	if (!(*ctx).is_decompression)
+		deflateEnd((*ctx).u.deflate);
+	else
+		inflateEnd((*ctx).u.deflate);
+
+	lws_free_set_NULL(ctx->u.deflate);
+}
+
+struct lws_compression_support lcs_deflate = {
+	/* .encoding_name */		"deflate",
+	/* .init_compression */		lcs_init_compression_deflate,
+	/* .process */			lcs_process_deflate,
+	/* .destroy */			lcs_destroy_deflate,
+};

Kinc/Sources/kinc/libs/core/compression/private-lib-roles-http-compression.h

@@ -0,0 +1,87 @@
+ /*
+ * libwebsockets - small server side websockets and web server implementation
+ *
+ * Copyright (C) 2010 - 2019 Andy Green <andy@warmcat.com>
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ *
+ *  This is included from private-lib-core.h if LWS_WITH_HTTP_STREAM_COMPRESSION
+ */
+
+#if defined(LWS_WITH_MINIZ)
+#include <miniz.h>
+#else
+#include <zlib.h>
+#endif
+#if defined(LWS_WITH_HTTP_BROTLI)
+#include <brotli/encode.h>
+#include <brotli/decode.h>
+#endif
+
+/*
+ * struct holding union of all the available compression methods' context data,
+ * and state if it's compressing or decompressing
+ */
+
+typedef struct lws_compression_ctx {
+	union {
+
+#if defined(LWS_WITH_HTTP_BROTLI)
+		BrotliEncoderState *br_en;
+		BrotliDecoderState *br_de;
+#endif
+		z_stream *deflate;
+		void *generic_ctx_ptr;
+	} u;
+
+	struct lws_buflist *buflist_comp;
+
+	unsigned int is_decompression:1;
+	unsigned int final_on_input_side:1;
+	unsigned int may_have_more:1;
+	unsigned int chunking:1;
+} lws_comp_ctx_t;
+
+/* generic structure defining the interface to a compression method */
+
+struct lws_compression_support {
+	/** compression name as used by, eg, content-ecoding */
+	const char *encoding_name;
+	/** create a compression context for the compression method, or NULL */
+	int (*init_compression)(lws_comp_ctx_t *ctx, int decomp);
+	/** pass data into the context to be processed */
+	int (*process)(lws_comp_ctx_t *ctx, const void *in, size_t *ilen_iused,
+		       void *out, size_t *olen_oused);
+	/** destroy the de/compression context */
+	void (*destroy)(lws_comp_ctx_t *ctx);
+};
+
+extern struct lws_compression_support lcs_deflate;
+extern struct lws_compression_support lcs_brotli;
+
+int
+lws_http_compression_validate(struct lws *wsi);
+
+int
+lws_http_compression_transform(struct lws *wsi, unsigned char *buf,
+			       size_t len, enum lws_write_protocol *wp,
+			       unsigned char **outbuf, size_t *olen_oused);
+
+void
+lws_http_compression_destroy(struct lws *wsi);

Kinc/Sources/kinc/libs/core/compression/stream.c

@@ -0,0 +1,227 @@
+/*
+ * libwebsockets - small server side websockets and web server implementation
+ *
+ * Copyright (C) 2010 - 2019 Andy Green <andy@warmcat.com>
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ */
+
+#include "private-lib-core.h"
+
+/* compression methods listed in order of preference */
+
+struct lws_compression_support *lcs_available[] = {
+#if defined(LWS_WITH_HTTP_BROTLI)
+	&lcs_brotli,
+#endif
+	&lcs_deflate,
+};
+
+/* compute acceptable compression encodings while we still have an ah */
+
+int
+lws_http_compression_validate(struct lws *wsi)
+{
+	const char *a;
+	size_t n;
+
+	wsi->http.comp_accept_mask = 0;
+
+	if (!wsi->http.ah || !lwsi_role_server(wsi))
+		return 0;
+
+	a = lws_hdr_simple_ptr(wsi, WSI_TOKEN_HTTP_ACCEPT_ENCODING);
+	if (!a)
+		return 0;
+
+	for (n = 0; n < LWS_ARRAY_SIZE(lcs_available); n++)
+		if (strstr(a, lcs_available[n]->encoding_name))
+			wsi->http.comp_accept_mask = (uint8_t)(wsi->http.comp_accept_mask | (1 << n));
+
+	return 0;
+}
+
+int
+lws_http_compression_apply(struct lws *wsi, const char *name,
+			   unsigned char **p, unsigned char *end, char decomp)
+{
+	size_t n;
+
+	for (n = 0; n < LWS_ARRAY_SIZE(lcs_available); n++) {
+		/* if name is non-NULL, choose only that compression method */
+		if (name && strcmp(lcs_available[n]->encoding_name, name))
+			continue;
+		/*
+		 * If we're the server, confirm that the client told us he could
+		 * handle this kind of compression transform...
+		 */
+		if (!decomp && !(wsi->http.comp_accept_mask & (1 << n)))
+			continue;
+
+		/* let's go with this one then... */
+		break;
+	}
+
+	if (n == LWS_ARRAY_SIZE(lcs_available))
+		return 1;
+
+	lcs_available[n]->init_compression(&wsi->http.comp_ctx, decomp);
+	if (!wsi->http.comp_ctx.u.generic_ctx_ptr) {
+		lwsl_err("%s: init_compression %d failed\n", __func__, (int)n);
+		return 1;
+	}
+
+	wsi->http.lcs = lcs_available[n];
+	wsi->http.comp_ctx.may_have_more = 0;
+	wsi->http.comp_ctx.final_on_input_side = 0;
+	wsi->http.comp_ctx.chunking = 0;
+	wsi->http.comp_ctx.is_decompression = !!decomp;
+
+	if (lws_add_http_header_by_token(wsi, WSI_TOKEN_HTTP_CONTENT_ENCODING,
+			(unsigned char *)lcs_available[n]->encoding_name,
+			(int)strlen(lcs_available[n]->encoding_name), p, end))
+		return -1;
+
+	lwsl_info("%s: %s: applied %s content-encoding\n", __func__,
+		    lws_wsi_tag(wsi), lcs_available[n]->encoding_name);
+
+	return 0;
+}
+
+void
+lws_http_compression_destroy(struct lws *wsi)
+{
+	if (!wsi->http.lcs || !wsi->http.comp_ctx.u.generic_ctx_ptr)
+		return;
+
+	wsi->http.lcs->destroy(&wsi->http.comp_ctx);
+
+	wsi->http.lcs = NULL;
+}
+
+/*
+ * This manages the compression transform independent of h1 or h2.
+ *
+ * wsi->buflist_comp stashes pre-transform input that was not yet compressed
+ */
+
+int
+lws_http_compression_transform(struct lws *wsi, unsigned char *buf,
+			       size_t len, enum lws_write_protocol *wp,
+			       unsigned char **outbuf, size_t *olen_oused)
+{
+	size_t ilen_iused = len;
+	int n, use = 0, wp1f = (*wp) & 0x1f;
+	lws_comp_ctx_t *ctx = &wsi->http.comp_ctx;
+
+	ctx->may_have_more = 0;
+
+	if (!wsi->http.lcs ||
+	    (wp1f != LWS_WRITE_HTTP && wp1f != LWS_WRITE_HTTP_FINAL)) {
+		*outbuf = buf;
+		*olen_oused = len;
+
+		return 0;
+	}
+
+	if (wp1f == LWS_WRITE_HTTP_FINAL) {
+		/*
+		 * ...we may get a large buffer that represents the final input
+		 * buffer, but it may form multiple frames after being
+		 * tranformed by compression; only the last of those is actually
+		 * the final frame on the output stream.
+		 *
+		 * Note that we have received the FINAL input, and downgrade it
+		 * to a non-final for now.
+		 */
+		ctx->final_on_input_side = 1;
+		*wp = (unsigned int)(LWS_WRITE_HTTP | ((*wp) & ~0x1fu));
+	}
+
+	if (ctx->buflist_comp) {
+		/*
+		 * we can't send this new stuff when we have old stuff
+		 * buffered and not compressed yet.  Add it to the tail
+		 * and switch to trying to process the head.
+		 */
+		if (buf && len) {
+			if (lws_buflist_append_segment(
+					&ctx->buflist_comp, buf, len) < 0)
+				return -1;
+			lwsl_debug("%s: %s: adding %d to comp buflist\n",
+				   __func__, lws_wsi_tag(wsi), (int)len);
+		}
+
+		len = lws_buflist_next_segment_len(&ctx->buflist_comp, &buf);
+		ilen_iused = len;
+		use = 1;
+		lwsl_debug("%s: %s: trying comp buflist %d\n", __func__,
+				lws_wsi_tag(wsi), (int)len);
+	}
+
+	if (!buf && ilen_iused)
+		return 0;
+
+	lwsl_debug("%s: %s: pre-process: ilen_iused %d, olen_oused %d\n",
+		   __func__, lws_wsi_tag(wsi), (int)ilen_iused, (int)*olen_oused);
+
+	n = wsi->http.lcs->process(ctx, buf, &ilen_iused, *outbuf, olen_oused);
+
+	if (n && n != 1) {
+		lwsl_err("%s: problem with compression\n", __func__);
+
+		return -1;
+	}
+
+	if (!ctx->may_have_more && ctx->final_on_input_side)
+
+		*wp = (unsigned int)(LWS_WRITE_HTTP_FINAL | ((*wp) & ~0x1fu));
+
+	lwsl_debug("%s: %s: more %d, ilen_iused %d\n", __func__, lws_wsi_tag(wsi),
+		   ctx->may_have_more, (int)ilen_iused);
+
+	if (use && ilen_iused) {
+		/*
+		 * we were flushing stuff from the buflist head... account for
+		 * however much actually got processed by the compression
+		 * transform
+		 */
+		lws_buflist_use_segment(&ctx->buflist_comp, ilen_iused);
+		lwsl_debug("%s: %s: marking %d of comp buflist as used "
+			   "(ctx->buflist_comp %p)\n", __func__,
+			   lws_wsi_tag(wsi), (int)len, ctx->buflist_comp);
+	}
+
+	if (!use && ilen_iused != len) {
+		 /*
+		  * ...we were sending stuff from the caller directly and not
+		  * all of it got processed... stash on the buflist tail
+		  */
+		if (lws_buflist_append_segment(&ctx->buflist_comp,
+					   buf + ilen_iused, len - ilen_iused) < 0)
+			return -1;
+
+		lwsl_debug("%s: buffering %d unused comp input\n", __func__,
+			   (int)(len - ilen_iused));
+	}
+	if (ctx->buflist_comp || ctx->may_have_more)
+		lws_callback_on_writable(wsi);
+
+	return 0;
+}