libarchive: add capability to unpack to mem.buffer

The performance and number of processes for a "depmod -a" with gzipped
modules was abysmal. This patch adds a code path without fork,
benefiting all users of xmalloc_open_zipped_read_close.

"modinfo radeon.ko.gz", a single-file reader, got 30% faster.
"depmod -a", which used to fork over 800 times, got 20% faster.

Heavily based on a patch by Lauri Kasanen <curaga@operamail.com>

function                                             old     new   delta
setup_transformer_on_fd                                -     159    +159
transformer_write                                      -     122    +122
fork_transformer                                       -     112    +112
xmalloc_open_zipped_read_close                        63     118     +55
read_bunzip                                         1866    1896     +30
xtransformer_write                                     -      19     +19
unzip_main                                          2449    2462     +13
bbunpack                                             755     766     +11
unpack_lzma_stream                                  2717    2723      +6
unpack_xz_stream                                    2393    2397      +4
unpack_Z_stream                                     1173    1175      +2
inflate_unzip                                        111     105      -6
check_signature16                                     70      63      -7
unpack_bz2_stream                                    359     349     -10
unpack_unxz                                           12       -     -12
unpack_unlzma                                         12       -     -12
unpack_uncompress                                     12       -     -12
unpack_gunzip                                         12       -     -12
unpack_bunzip2                                        12       -     -12
open_transformer                                     106      92     -14
inflate_unzip_internal                              1945    1916     -29
unpack_gz_stream                                     693     655     -38
open_zipped                                           89      47     -42
setup_unzip_on_fd                                    142      53     -89
------------------------------------------------------------------------------
(add/remove: 4/5 grow/shrink: 7/8 up/down: 533/-295)          Total: 238 bytes

Signed-off-by: Denys Vlasenko <vda.linux@googlemail.com>

2 files changed, 30 insertions, 15 deletions

diff --git a/include/bb_archive.h b/include/bb_archive.h
index 8a2dd2735..a6b166fe3 100644
--- a/include/bb_archive.h
+++ b/include/bb_archive.h
@@ -205,6 +205,18 @@ void dealloc_bunzip(bunzip_data *bd) FAST_FUNC;
 /* Meaning and direction (input/output) of the fields are transformer-specific */
 typedef struct transformer_state_t {
 	smallint check_signature; /* most often referenced member */
+
+	IF_DESKTOP(long long) int FAST_FUNC (*xformer)(struct transformer_state_t *xstate);
+	USE_FOR_NOMMU(const char *xformer_prog;)
+
+	/* Source */
+	int      src_fd;
+	/* Output */
+	int      dst_fd;
+	size_t   mem_output_size_max; /* if non-zero, decompress to RAM instead of fd */
+	size_t   mem_output_size;
+	char     *mem_output_buf;
+
 	off_t    bytes_out;
 	off_t    bytes_in;  /* used in unzip code only: needs to know packed size */
 	uint32_t crc32;
@@ -212,14 +224,16 @@ typedef struct transformer_state_t {
 } transformer_state_t;
 
 void init_transformer_state(transformer_state_t *xstate) FAST_FUNC;
-int FAST_FUNC check_signature16(transformer_state_t *xstate, int src_fd, unsigned magic16) FAST_FUNC;
+ssize_t transformer_write(transformer_state_t *xstate, const void *buf, size_t bufsize) FAST_FUNC;
+ssize_t xtransformer_write(transformer_state_t *xstate, const void *buf, size_t bufsize) FAST_FUNC;
+int check_signature16(transformer_state_t *xstate, unsigned magic16) FAST_FUNC;
 
-IF_DESKTOP(long long) int inflate_unzip(transformer_state_t *xstate, int src_fd, int dst_fd) FAST_FUNC;
-IF_DESKTOP(long long) int unpack_Z_stream(transformer_state_t *xstate, int src_fd, int dst_fd) FAST_FUNC;
-IF_DESKTOP(long long) int unpack_gz_stream(transformer_state_t *xstate, int src_fd, int dst_fd) FAST_FUNC;
-IF_DESKTOP(long long) int unpack_bz2_stream(transformer_state_t *xstate, int src_fd, int dst_fd) FAST_FUNC;
-IF_DESKTOP(long long) int unpack_lzma_stream(transformer_state_t *xstate, int src_fd, int dst_fd) FAST_FUNC;
-IF_DESKTOP(long long) int unpack_xz_stream(transformer_state_t *xstate, int src_fd, int dst_fd) FAST_FUNC;
+IF_DESKTOP(long long) int inflate_unzip(transformer_state_t *xstate) FAST_FUNC;
+IF_DESKTOP(long long) int unpack_Z_stream(transformer_state_t *xstate) FAST_FUNC;
+IF_DESKTOP(long long) int unpack_gz_stream(transformer_state_t *xstate) FAST_FUNC;
+IF_DESKTOP(long long) int unpack_bz2_stream(transformer_state_t *xstate) FAST_FUNC;
+IF_DESKTOP(long long) int unpack_lzma_stream(transformer_state_t *xstate) FAST_FUNC;
+IF_DESKTOP(long long) int unpack_xz_stream(transformer_state_t *xstate) FAST_FUNC;
 
 char* append_ext(char *filename, const char *expected_ext) FAST_FUNC;
 int bbunpack(char **argv,
@@ -230,16 +244,16 @@ int bbunpack(char **argv,
 
 void check_errors_in_children(int signo);
 #if BB_MMU
-void open_transformer(int fd,
+void fork_transformer(int fd,
 	int check_signature,
-	IF_DESKTOP(long long) int FAST_FUNC (*transformer)(transformer_state_t *xstate, int src_fd, int dst_fd)
+	IF_DESKTOP(long long) int FAST_FUNC (*transformer)(transformer_state_t *xstate)
 ) FAST_FUNC;
-#define open_transformer_with_sig(fd, transformer, transform_prog) open_transformer((fd), 1, (transformer))
-#define open_transformer_with_no_sig(fd, transformer)              open_transformer((fd), 0, (transformer))
+#define fork_transformer_with_sig(fd, transformer, transform_prog) fork_transformer((fd), 1, (transformer))
+#define fork_transformer_with_no_sig(fd, transformer)              fork_transformer((fd), 0, (transformer))
 #else
-void open_transformer(int fd, const char *transform_prog) FAST_FUNC;
-#define open_transformer_with_sig(fd, transformer, transform_prog) open_transformer((fd), (transform_prog))
-/* open_transformer_with_no_sig() does not exist on NOMMU */
+void fork_transformer(int fd, const char *transform_prog) FAST_FUNC;
+#define fork_transformer_with_sig(fd, transformer, transform_prog) fork_transformer((fd), (transform_prog))
+/* fork_transformer_with_no_sig() does not exist on NOMMU */
 #endif
 
 
diff --git a/include/libbb.h b/include/libbb.h
index 17a0089d8..8e8b9ca0e 100644
--- a/include/libbb.h
+++ b/include/libbb.h
@@ -755,11 +755,12 @@ unsigned bb_clk_tck(void) FAST_FUNC;
 extern int setup_unzip_on_fd(int fd, int fail_if_not_compressed) FAST_FUNC;
 /* Autodetects .gz etc */
 extern int open_zipped(const char *fname, int fail_if_not_compressed) FAST_FUNC;
+extern void *xmalloc_open_zipped_read_close(const char *fname, size_t *maxsz_p) FAST_FUNC RETURNS_MALLOC;
 #else
 # define setup_unzip_on_fd(...) (0)
 # define open_zipped(fname, fail_if_not_compressed)  open((fname), O_RDONLY);
+# define xmalloc_open_zipped_read_close(fname, maxsz_p) xmalloc_open_read_close((fname), (maxsz_p))
 #endif
-extern void *xmalloc_open_zipped_read_close(const char *fname, size_t *maxsz_p) FAST_FUNC RETURNS_MALLOC;
 
 extern ssize_t safe_write(int fd, const void *buf, size_t count) FAST_FUNC;
 // NB: will return short write on error, not -1,