Patch to add zlib:crc32_combine/4 and zlib:adler32_combine/4

Previous Topic Next Topic
 
classic Classic list List threaded Threaded
6 messages Options
Reply | Threaded
Open this post in threaded view
|

Patch to add zlib:crc32_combine/4 and zlib:adler32_combine/4

Matthew Dempsky-3
I wrote an Erlang module to concatenate zlib compressed data, but to
generate proper zlib and gzip trailers, I need to compute the adler32
and crc32 checksums.  adler32 has a simple algebraic structure that
makes it easy to implement in just a few lines of Erlang code (give it
a shot---it's fun :-), but combining crc32 checksums involves
computing x*2^n in a Galois field (which isn't so fun in Erlang).

Anyways, I noticed the zlib C library already provides functions for
these, but they're not exposed by the Erlang interface... so I wrote
the patch below.  (I'm mainly interested in crc32_combine, but I
thought it would be nice to have adler32_combine for completeness.)

--- erts/emulator/drivers/common/zlib_drv.c.orig 2007-11-06
11:44:12.000000000 -0800
+++ erts/emulator/drivers/common/zlib_drv.c 2007-11-06 11:30:11.000000000 -0800
@@ -55,6 +55,9 @@
 #define ADLER32_1         21
 #define ADLER32_2         22

+#define CRC32_COMBINE   23
+#define ADLER32_COMBINE 24
+
 #define DEFAULT_BUFSZ   4000

 static int zlib_init(void);
@@ -597,6 +600,26 @@
  adler = adler32(adler, buf+4, len-4);
  return zlib_value(adler, rbuf, rlen);
     }
+
+    case CRC32_COMBINE: {
+ uLong crc1, crc2, len2;
+ if (len != 12) goto badarg;
+ crc1 = i32(buf);
+ crc2 = i32(buf+4);
+ len2 = i32(buf+8);
+ crc1 = crc32_combine(crc1, crc2, len2);
+ return zlib_value(crc1, rbuf, rlen);
+    }
+
+    case ADLER32_COMBINE: {
+ uLong adler1, adler2, len2;
+ if (len != 12) goto badarg;
+ adler1 = i32(buf);
+ adler2 = i32(buf+4);
+ len2 = i32(buf+8);
+ adler1 = adler32_combine(adler1, adler2, len2);
+ return zlib_value(adler1, rbuf, rlen);
+    }
     }

  badarg:
--- lib/kernel/src/zlib.erl.orig 2007-11-06 11:20:14.000000000 -0800
+++ lib/kernel/src/zlib.erl 2007-11-06 11:43:28.000000000 -0800
@@ -25,6 +25,7 @@
  inflateSync/1,inflateReset/1,inflate/2,inflateEnd/1,
  setBufSize/2,getBufSize/1,
  crc32/1,crc32/2,crc32/3,adler32/2,adler32/3,getQSize/1,
+ crc32_combine/4,adler32_combine/4,
  compress/1,uncompress/1,zip/1,unzip/1,
  gzip/1,gunzip/1]).

@@ -108,6 +109,9 @@
 -define(ADLER32_1,       21).
 -define(ADLER32_2,       22).

+-define(CRC32_COMBINE,   23).
+-define(ADLER32_COMBINE, 24).
+
 %% open a z_stream
 open() ->
     open_port({spawn, zlib_drv}, [binary]).
@@ -213,6 +217,16 @@
 adler32(_Z, _Adler, _Binary)  ->
     erlang:error(badarg).

+crc32_combine(Z, CRC1, CRC2, Len2) when is_integer(CRC1),
is_integer(CRC2), is_integer(Len2) ->
+    call(Z, ?CRC32_COMBINE, <<CRC1:32, CRC2:32, Len2:32>>);
+crc32_combine(_Z, _CRC1, _CRC2, _Len2) ->
+    erlang:error(badarg).
+
+adler32_combine(Z, Adler1, Adler2, Len2) when is_integer(Adler1),
is_integer(Adler2), is_integer(Len2) ->
+    call(Z, ?ADLER32_COMBINE, <<Adler1:32, Adler2:32, Len2:32>>);
+adler32_combine(_Z, _Adler1, _Adler2, _Len2) ->
+    erlang:error(badarg).
+
 getQSize(Z) ->
     call(Z, ?GET_QSIZE, []).


Reply | Threaded
Open this post in threaded view
|

Patch to add zlib:crc32_combine/4 and zlib:adler32_combine/4

Matthew Dempsky-3
On 11/6/07, Matthew Dempsky <matthew> wrote:
> I wrote an Erlang module to concatenate zlib compressed data, but to
> generate proper zlib and gzip trailers, I need to compute the adler32
> and crc32 checksums.

Also, for what it's worth, this code has been released as part of our
eswf project on google code.  You can find the module source at:

    http://eswf.googlecode.com/svn/trunk/src/zipchunk.erl

Right now the public API only supports zlib encoding, but the code
internally could handle raw deflate and gzip encoding without much
effort.  Support for crc32 checksum joining is currently using a slow
workaround until the otp patch I submitted is accepted.

As such, I'm interested in feedback on my patch.  Will it be merged
into R12B?  Is there anything else I can do to help ensure this
happens?

Thanks.


Reply | Threaded
Open this post in threaded view
|

Patch to add zlib:crc32_combine/4 and zlib:adler32_combine/4

Matthew Dempsky-3
Some further testing with the zipchunk module revealed that the
zlib_drv does not properly flush the zlib buffers.  In particular, a
call to zlib:deflate(Z, Binary, full) can leave part of Binary still
in Z's internal state.

The patch below properly follows the advice of zlib.h that ``If
deflate returns with avail_out == 0, this function must be called
again with the same value of the flush parameter and more output space
(updated avail_out).''

Also, Z_STREAM_END is only returned when Z_FINISH is given.

--- otp_src_R11B-5/erts/emulator/drivers/common/zlib_drv.c.orig 2007-11-06
11:44:12.000000000 -0800
+++ otp_src_R11B-5/erts/emulator/drivers/common/zlib_drv.c 2007-11-13
16:39:48.000000000 -0800
@@ -323,12 +323,15 @@
  }
     }
  } else {
-    while (d->s.avail_out < d->binsz) {
+    while (d->s.avail_out == 0) {
  zlib_output(d);
- if (res == Z_STREAM_END) {
-    break;
+ if ((res = deflate(&d->s, flush)) < 0) {
+    return res;
  }
     }
+    if (d->s.avail_out < d->binsz) {
+ zlib_output(d);
+    }
  }
     }
     return res;


Reply | Threaded
Open this post in threaded view
|

Patch to add zlib:crc32_combine/4 and zlib:adler32_combine/4

Tomas Abrahamsson-5
In reply to this post by Matthew Dempsky-3


> On 11/6/07, Matthew Dempsky <matthew> wrote:
>> I wrote an Erlang module to concatenate zlib compressed data, but to
>> generate proper zlib and gzip trailers, I need to compute the adler32
>> and crc32 checksums.

> ...
> As such, I'm interested in feedback on my patch.  Will it be merged
> into R12B?  ...

I, too, would be interested in the any answer to this.

Incidentally, concatenating compressed data happens to
be the reason for my interest in this :-) I wrote a
piece of C code to do what I needed, but having this
directly in the zlib of Erlang/OTP would of couse
be much better.


BRs
Tomas


Reply | Threaded
Open this post in threaded view
|

Patch to add zlib:crc32_combine/4 and zlib:adler32_combine/4

Björn Gustavsson-3
In reply to this post by Matthew Dempsky-3
"Matthew Dempsky" <matthew> writes:

> As such, I'm interested in feedback on my patch.  Will it be merged
> into R12B?

Barred any unforseen problems, you patches should make it to the R12B release.

/Bjorn
--
Bj?rn Gustavsson, Erlang/OTP, Ericsson AB


Reply | Threaded
Open this post in threaded view
|

Patch to add zlib:crc32_combine/4 and zlib:adler32_combine/4

Matthew Dempsky-3
On 14 Nov 2007 14:52:06 +0100, Bjorn Gustavsson <bjorn> wrote:
> Barred any unforseen problems, you patches should make it to the R12B release.

Awesome, thanks. :-)