From b32b384666cd7ef61f61d7ba4761b03bd61af776 Mon Sep 17 00:00:00 2001
From: Balint Reczey <balint.reczey@canonical.com>
Date: Thu, 8 Mar 2018 09:53:36 +0100
Subject: [PATCH] dpkg: Add Zstandard compression and decompression support for
 binary packages

---
 README               |   1 +
 configure.ac         |   2 +
 debian/control       |   3 +
 debian/rules         |   1 +
 dpkg-deb/Makefile.am |   1 +
 dpkg-deb/extract.c   |   1 +
 dpkg-deb/main.c      |   7 ++
 lib/dpkg/compress.c  | 157 ++++++++++++++++++++++++++++++++++++++++++-
 lib/dpkg/compress.h  |   1 +
 m4/dpkg-libs.m4      |   7 ++
 man/deb.pod          |   6 +-
 t-func/deb-format.at |  26 +++++++
 12 files changed, 210 insertions(+), 3 deletions(-)

diff --git a/README b/README
index dd5a70fac..898369b7c 100644
--- a/README
+++ b/README
@@ -73,6 +73,7 @@ To enable optional functionality or programs, this software might be needed:
 
   libmd (used by libdpkg, currently falling back to embedded code)
   libz (from zlib, used instead of gzip command-line tool)
+  libzstd (from libzstd, used instead of zstd command-line tool)
   liblzma (from xz utils, used instead of xz command-line tool)
   libbz2 (from bzip2, used instead of bzip2 command-line tool)
   libselinux
diff --git a/configure.ac b/configure.ac
index 10d2ad278..7d9151376 100644
--- a/configure.ac
+++ b/configure.ac
@@ -88,6 +88,7 @@ AC_SYS_LARGEFILE
 # Checks for libraries.
 DPKG_LIB_MD
 DPKG_LIB_Z
+DPKG_LIB_ZSTD
 DPKG_LIB_BZ2
 DPKG_LIB_LZMA
 DPKG_LIB_SELINUX
@@ -279,6 +280,7 @@ Configuration:
     libselinux  . . . . . . . . . : $have_libselinux
     libmd . . . . . . . . . . . . : $have_libmd
     libz  . . . . . . . . . . . . : $have_libz
+    libzstd  . . . . . . . . . .  : $have_libzstd
     liblzma . . . . . . . . . . . : $have_liblzma
     libbz2  . . . . . . . . . . . : $have_libbz2
     libcurses . . . . . . . . . . : ${have_libcurses:-no}
diff --git a/debian/control b/debian/control
index 52f4d8986..0df054b0e 100644
--- a/debian/control
+++ b/debian/control
@@ -16,7 +16,9 @@ Build-Depends:
 # Needed for --porefs defaults, conditional addenda and mode=eof.
  po4a (>= 0.59),
  zlib1g-dev,
+ zstd,
  libbz2-dev,
+ libzstd-dev,
  liblzma-dev,
  libselinux1-dev [linux-any],
  libncurses-dev (>= 6.1+20180210) | libncursesw5-dev,
@@ -56,6 +58,7 @@ Multi-Arch: same
 Depends:
  ${misc:Depends},
  zlib1g-dev,
+ libzstd-dev,
  liblzma-dev,
  libbz2-dev,
 Description: Debian package management static library
diff --git a/debian/rules b/debian/rules
index fde5388a5..87c74bfee 100755
--- a/debian/rules
+++ b/debian/rules
@@ -65,6 +65,7 @@ build-tree/config.status:
 		--with-devlibdir=\$${prefix}/lib/$(DEB_HOST_MULTIARCH) \
 		--without-libmd \
 		--with-libz \
+		--with-libzstd \
 		--with-liblzma \
 		--with-libbz2
 
diff --git a/dpkg-deb/Makefile.am b/dpkg-deb/Makefile.am
index 02d79ed7d..bbd30e02c 100644
--- a/dpkg-deb/Makefile.am
+++ b/dpkg-deb/Makefile.am
@@ -21,5 +21,6 @@ dpkg_deb_LDADD = \
 	../lib/dpkg/libdpkg.la \
 	$(LIBINTL) \
 	$(Z_LIBS) \
+	$(ZSTD_LIBS) \
 	$(LZMA_LIBS) \
 	$(BZ2_LIBS)
diff --git a/dpkg-deb/extract.c b/dpkg-deb/extract.c
index d0f6cb6c4..bf2d782a4 100644
--- a/dpkg-deb/extract.c
+++ b/dpkg-deb/extract.c
@@ -180,6 +180,7 @@ extracthalf(const char *debar, const char *dir,
           decompressor = compressor_find_by_extension(extension);
           if (decompressor != COMPRESSOR_TYPE_NONE &&
               decompressor != COMPRESSOR_TYPE_GZIP &&
+              decompressor != COMPRESSOR_TYPE_ZSTD &&
               decompressor != COMPRESSOR_TYPE_XZ)
             ohshit(_("archive '%s' uses unknown compression for member '%.*s', "
                      "giving up"),
diff --git a/dpkg-deb/main.c b/dpkg-deb/main.c
index 17ed92b18..d728af748 100644
--- a/dpkg-deb/main.c
+++ b/dpkg-deb/main.c
@@ -108,7 +108,11 @@ usage(const struct cmdinfo *cip, const char *value)
 "      --[no-]uniform-compression   Use the compression params on all members.\n"
 "  -z#                              Set the compression level when building.\n"
 "  -Z<type>                         Set the compression type used when building.\n"
+#ifdef ENABLE_ZSTD_COMPRESSOR
+"                                     Allowed types: gzip, xz, zstd, none.\n"
+#else
 "                                     Allowed types: gzip, xz, none.\n"
+#endif
 "  -S<strategy>                     Set the compression strategy when building.\n"
 "                                     Allowed values: none; extreme (xz);\n"
 "                                     filtered, huffman, rle, fixed (gzip).\n"
@@ -245,6 +249,9 @@ int main(int argc, const char *const *argv) {
   if (opt_uniform_compression &&
       (compress_params.type != COMPRESSOR_TYPE_NONE &&
        compress_params.type != COMPRESSOR_TYPE_GZIP &&
+#ifdef ENABLE_ZSTD_COMPRESSOR
+       compress_params.type != COMPRESSOR_TYPE_ZSTD &&
+#endif
        compress_params.type != COMPRESSOR_TYPE_XZ))
     badusage(_("unsupported compression type '%s' with uniform compression"),
              compressor_get_name(compress_params.type));
diff --git a/lib/dpkg/compress.c b/lib/dpkg/compress.c
index 41991317a..53a1e9167 100644
--- a/lib/dpkg/compress.c
+++ b/lib/dpkg/compress.c
@@ -32,6 +32,9 @@
 #ifdef WITH_LIBZ
 #include <zlib.h>
 #endif
+#ifdef WITH_LIBZSTD
+#include <zstd.h>
+#endif
 #ifdef WITH_LIBLZMA
 #include <lzma.h>
 #endif
@@ -47,7 +50,7 @@
 #include <dpkg/buffer.h>
 #include <dpkg/command.h>
 #include <dpkg/compress.h>
-#if !defined(WITH_LIBZ) || !defined(WITH_LIBLZMA) || !defined(WITH_LIBBZ2)
+#if !defined(WITH_LIBZ) || !defined(WITH_LIBZSTD) || !defined(WITH_LIBLZMA) || !defined(WITH_LIBBZ2)
 #include <dpkg/subproc.h>
 
 static void DPKG_ATTR_SENTINEL
@@ -763,6 +766,157 @@ static const struct compressor compressor_lzma = {
 	.decompress = decompress_lzma,
 };
 
+/*
+ * Zstd compressor.
+ */
+
+#define ZSTD		"zstd"
+
+#ifdef WITH_LIBZSTD
+
+static void
+decompress_zstd(int fd_in, int fd_out, const char *desc)
+{
+	size_t const buf_in_size = ZSTD_DStreamInSize();
+	void*  const buf_in = m_malloc(buf_in_size);
+	size_t const buf_out_size = ZSTD_DStreamOutSize();
+	void*  const buf_out = m_malloc(buf_out_size);
+	size_t init_result, just_read, to_read;
+	ZSTD_DStream* const dstream = ZSTD_createDStream();
+	if (dstream == NULL) {
+		ohshit(_("ZSTD_createDStream error creating stream"));
+	}
+
+	init_result = ZSTD_initDStream(dstream);
+	if (ZSTD_isError(init_result)) {
+		ohshit(_("ZSTD_initDStream error : %s"), ZSTD_getErrorName(init_result));
+	}
+	to_read = init_result;
+	while ((just_read = fd_read(fd_in, buf_in, to_read))) {
+		ZSTD_inBuffer input = { buf_in, just_read, 0 };
+		while (input.pos < input.size) {
+			size_t actualwrite;
+			ZSTD_outBuffer output = { buf_out, buf_out_size, 0 };
+			to_read = ZSTD_decompressStream(dstream, &output , &input);
+			if (ZSTD_isError(to_read)) {
+				ohshit(_("ZSTD_decompressStream error : %s \n"),
+				       ZSTD_getErrorName(to_read));
+			}
+			actualwrite = fd_write(fd_out, output.dst, output.pos);
+			if (actualwrite != output.pos) {
+				const char *errmsg = strerror(errno);
+				ohshite(_("%s: internal zstd write error: '%s'"), desc, errmsg);
+			}
+			/* possible next frame */
+			if (to_read == 0) {
+				init_result = ZSTD_initDStream(dstream);
+				if (ZSTD_isError(init_result)) {
+					ohshit(_("ZSTD_initDStream error : %s"), ZSTD_getErrorName(init_result));
+				}
+				to_read = init_result;
+			}
+		}
+	}
+
+	ZSTD_freeDStream(dstream);
+	free(buf_in);
+	free(buf_out);
+	if (close(fd_out))
+		ohshite(_("%s: internal zstd write error"), desc);
+}
+
+static void
+compress_zstd(int fd_in, int fd_out, struct compress_params *params, const char *desc)
+{
+	size_t const buf_in_size = ZSTD_CStreamInSize();
+	void*  const buf_in = m_malloc(buf_in_size);
+	size_t const buf_out_size = ZSTD_CStreamOutSize();
+	void*  const buf_out = m_malloc(buf_out_size);
+	size_t init_result, end_res;
+	size_t just_read, to_read;
+	ZSTD_CStream* const cstream = ZSTD_createCStream();
+	if (cstream == NULL) {
+		ohshit(_("ZSTD_createCStream error"));
+	}
+
+	init_result = ZSTD_initCStream(cstream, params->level);
+	if (ZSTD_isError(init_result)) {
+		ohshit(_("ZSTD_initCStream error : %s"), ZSTD_getErrorName(init_result));
+	}
+	to_read = buf_in_size;
+	while ((just_read = fd_read(fd_in, buf_in, to_read))) {
+		ZSTD_inBuffer input = { buf_in, just_read, 0 };
+		while (input.pos < input.size) {
+			size_t actualwrite;
+			ZSTD_outBuffer output = { buf_out, buf_out_size, 0 };
+			to_read = ZSTD_compressStream(cstream, &output , &input);
+			if (ZSTD_isError(to_read)) {
+				ohshit(_("ZSTD_decompressStream error : %s \n"),
+				       ZSTD_getErrorName(to_read));
+			}
+			actualwrite = fd_write(fd_out, output.dst, output.pos);
+			if (actualwrite != output.pos) {
+				const char *errmsg = strerror(errno);
+				ohshite(_("%s: internal zstd write error: '%s'"),
+					desc, errmsg);
+			}
+		}
+	}
+	do {
+		size_t actualwrite;
+		ZSTD_outBuffer output = { buf_out, buf_out_size, 0 };
+		end_res = ZSTD_endStream(cstream, &output);
+		if (ZSTD_isError(end_res)) {
+			ohshit(_("ZSTD_endStream error : %s \n"),
+			       ZSTD_getErrorName(end_res));
+		}
+		actualwrite = fd_write(fd_out, output.dst, output.pos);
+		if (actualwrite != output.pos) {
+			const char *errmsg = strerror(errno);
+			ohshite(_("%s: internal zstd write error: '%s'"), desc,
+				errmsg);
+		}
+	} while (end_res > 0);
+
+	ZSTD_freeCStream(cstream);
+	free(buf_in);
+	free(buf_out);
+
+	/* ZSTD_endStream() already flushed the output buffers */
+	if (close(fd_out))
+		ohshite(_("%s: internal zstd write error"), desc);
+}
+
+#else
+static const char *env_zstd[] = {};
+
+static void
+decompress_zstd(int fd_in, int fd_out, const char *desc)
+{
+	fd_fd_filter(fd_in, fd_out, desc, env_zstd, ZSTD, "-dcq", NULL);
+}
+
+static void
+compress_zstd(int fd_in, int fd_out, struct compress_params *params, const char *desc)
+{
+	char combuf[6];
+
+	snprintf(combuf, sizeof(combuf), "-c%d", params->level);
+	fd_fd_filter(fd_in, fd_out, desc, env_zstd, ZSTD, combuf, "-q", NULL);
+}
+#endif
+
+static const struct compressor compressor_zstd = {
+	.name = "zstd",
+	.extension = ".zst",
+        /* zstd commands's default is 3 but the aim is to be closer to xz's
+         * default compression efficiency */
+	.default_level = 19,
+	.fixup_params = fixup_none_params,
+	.compress = compress_zstd,
+	.decompress = decompress_zstd,
+};
+
 /*
  * Generic compressor filter.
  */
@@ -773,6 +927,7 @@ static const struct compressor *compressor_array[] = {
 	[COMPRESSOR_TYPE_XZ] = &compressor_xz,
 	[COMPRESSOR_TYPE_BZIP2] = &compressor_bzip2,
 	[COMPRESSOR_TYPE_LZMA] = &compressor_lzma,
+	[COMPRESSOR_TYPE_ZSTD] = &compressor_zstd,
 };
 
 static const struct compressor *
diff --git a/lib/dpkg/compress.h b/lib/dpkg/compress.h
index 08aaf2516..1af8a3490 100644
--- a/lib/dpkg/compress.h
+++ b/lib/dpkg/compress.h
@@ -42,6 +42,7 @@ enum compressor_type {
 	COMPRESSOR_TYPE_XZ,
 	COMPRESSOR_TYPE_BZIP2,
 	COMPRESSOR_TYPE_LZMA,
+	COMPRESSOR_TYPE_ZSTD,
 };
 
 enum compressor_strategy {
diff --git a/m4/dpkg-libs.m4 b/m4/dpkg-libs.m4
index ffe373b7d..d9f869f22 100644
--- a/m4/dpkg-libs.m4
+++ b/m4/dpkg-libs.m4
@@ -75,6 +75,13 @@ AC_DEFUN([DPKG_LIB_Z], [
   DPKG_WITH_COMPRESS_LIB([z], [zlib.h], [gzdopen])
 ])# DPKG_LIB_Z
 
+# DPKG_LIB_ZSTD
+# -------------
+# Check for zstd library.
+AC_DEFUN([DPKG_LIB_ZSTD], [
+  DPKG_WITH_COMPRESS_LIB([zstd], [zstd.h], [ZSTD_decompressStream])
+])# DPKG_LIB_ZSTD
+
 # DPKG_LIB_LZMA
 # -------------
 # Check for lzma library.
diff --git a/man/deb.pod b/man/deb.pod
index cd42f1ca2..e13349851 100644
--- a/man/deb.pod
+++ b/man/deb.pod
@@ -82,8 +82,9 @@ The second required member is named
 B<control.tar>.
 It is a tar archive containing the package control information, either
 not compressed (supported since dpkg 1.17.6), or compressed with
-gzip (with B<.gz> extension) or
-xz (with B<.xz> extension, supported since 1.17.6),
+gzip (with B<.gz> extension),
+xz (with B<.xz> extension, supported since 1.17.6) or
+zstd (with B<.zst>B extension, decompression is supported since 1.20.7),
 as a series of plain files, of which the file
 B<control>
 is mandatory and contains the core control information, the
@@ -105,6 +106,7 @@ It contains the filesystem as a tar archive, either
 not compressed (supported since dpkg 1.10.24), or compressed with
 gzip (with B<.gz> extension),
 xz (with B<.xz> extension, supported since dpkg 1.15.6),
+zstd (with B<.zst> extension, supported since 1.19.1),
 bzip2 (with B<.bz2> extension, supported since dpkg 1.10.24) or
 lzma (with B<.lzma> extension, supported since dpkg 1.13.25).
 
diff --git a/t-func/deb-format.at b/t-func/deb-format.at
index 956100ce9..ed9de2d25 100644
--- a/t-func/deb-format.at
+++ b/t-func/deb-format.at
@@ -28,6 +28,8 @@ xz -c control.tar >control.tar.xz
 xz -c data.tar >data.tar.xz
 bzip2 -c data.tar >data.tar.bz2
 lzma -c data.tar >data.tar.lzma
+pzstd -q -c control.tar >control.tar.zst
+pzstd -q -c data.tar >data.tar.zst
 touch _ignore
 touch unknown
 ])
@@ -242,6 +244,18 @@ drwxr-xr-x root/root         0 1970-01-01 00:00 ./
 -rw-r--r-- root/root         5 1970-01-01 00:00 ./file-templ
 ])
 
+AT_CHECK([
+# Test control.tar.zst member
+ar rc pkg-control-zst.deb debian-binary control.tar.zst data.tar.xz
+ar t pkg-control-zst.deb
+dpkg-deb -c pkg-control-zst.deb
+], [], [debian-binary
+control.tar.zst
+data.tar.xz
+drwxr-xr-x root/root         0 1970-01-01 00:00 ./
+-rw-r--r-- root/root         5 1970-01-01 00:00 ./file-templ
+])
+
 AT_CHECK([
 # Test data.tar member
 ar rc pkg-data-none.deb debian-binary control.tar.gz data.tar
@@ -290,6 +304,18 @@ drwxr-xr-x root/root         0 1970-01-01 00:00 ./
 -rw-r--r-- root/root         5 1970-01-01 00:00 ./file-templ
 ])
 
+AT_CHECK([
+# Test data.tar.zst member
+ar rc pkg-data-zst.deb debian-binary control.tar.gz data.tar.zst
+ar t pkg-data-zst.deb
+dpkg-deb -c pkg-data-zst.deb
+], [], [debian-binary
+control.tar.gz
+data.tar.zst
+drwxr-xr-x root/root         0 1970-01-01 00:00 ./
+-rw-r--r-- root/root         5 1970-01-01 00:00 ./file-templ
+])
+
 AT_CHECK([
 # Test data.tar.lzma member
 ar rc pkg-data-lzma.deb debian-binary control.tar.gz data.tar.lzma
-- 
2.25.1

