zlib: support concatenated gzip files

Reviewed-By: Fedor Indutny <fedor@indutny.com> PR-URL: nodejs/node-v0.x-archive#6442

Author: eendeego

lib/zlib.js

@@ -580,14 +580,21 @@ Zlib.prototype._processChunk = function(chunk, flushFlag, cb) {
  self._buffer = new Buffer(self._chunkSize);
  }
 
- if (availOutAfter === 0) {
+ if (availOutAfter === 0 || availInAfter > 0) {
  // Not actually done. Need to reprocess.
  // Also, update the availInBefore to the availInAfter value,
  // so that if we have to hit it a third (fourth, etc.) time,
  // it'll have the correct byte counts.
  inOff += (availInBefore - availInAfter);
  availInBefore = availInAfter;
 
+ if (availOutAfter !== 0) {
+ // There is still some data available for reading.
+ // This is usually a concatenated stream, so, reset and restart.
+ self.reset();
+ self._offset = 0;
+ }
+
  if (!async)
  return true;
 

src/node_zlib.cc

@@ -63,6 +63,11 @@ enum node_zlib_mode {
  UNZIP
 };
 
+enum node_zlib_error {
+ NO_ERROR,
+ FAILED,
+ WRITE_PENDING
+};
 
 void InitZlib(v8::Handle<v8::Object> target);
 
@@ -203,7 +208,7 @@ class ZCtx : public AsyncWrap {
  if (!async) {
  // sync version
  Process(work_req);
- if (CheckError(ctx))
+ if (CheckError(ctx) == NO_ERROR)
  AfterSync(ctx, args);
  return;
  }
@@ -287,7 +292,7 @@ class ZCtx : public AsyncWrap {
  }
 
 
- static bool CheckError(ZCtx* ctx) {
+ static node_zlib_error CheckError(ZCtx* ctx) {
  // Acceptable error states depend on the type of zlib stream.
  switch (ctx->err_) {
  case Z_OK:
@@ -300,14 +305,18 @@ class ZCtx : public AsyncWrap {
  ZCtx::Error(ctx, "Missing dictionary");
  else
  ZCtx::Error(ctx, "Bad dictionary");
- return false;
+ return FAILED;
  default:
  // something else.
- ZCtx::Error(ctx, "Zlib error");
- return false;
+ if (ctx->strm_.total_out == 0) {
+ ZCtx::Error(ctx, "Zlib error");
+ return FAILED;
+ } else {
+ return WRITE_PENDING;
+ }
  }
 
- return true;
+ return NO_ERROR;
  }
 
 
@@ -321,7 +330,8 @@ class ZCtx : public AsyncWrap {
  HandleScope handle_scope(env->isolate());
  Context::Scope context_scope(env->context());
 
- if (!CheckError(ctx))
+ node_zlib_error error = CheckError(ctx);
+ if (error == FAILED)
  return;
 
  Local<Integer> avail_out = Integer::New(env->isolate(),
@@ -335,6 +345,11 @@ class ZCtx : public AsyncWrap {
  Local<Value> args[2] = { avail_in, avail_out };
  ctx->MakeCallback(env->callback_string(), ARRAY_SIZE(args), args);
 
+ if (error == WRITE_PENDING) {
+ ZCtx::Error(ctx, "Zlib error");
+ return;
+ }
+
  ctx->Unref();
  if (ctx->pending_close_)
  ctx->Close();
@@ -539,10 +554,12 @@ class ZCtx : public AsyncWrap {
  switch (ctx->mode_) {
  case DEFLATE:
  case DEFLATERAW:
+ case GZIP:
  ctx->err_ = deflateReset(&ctx->strm_);
  break;
  case INFLATE:
  case INFLATERAW:
+ case GUNZIP:
  ctx->err_ = inflateReset(&ctx->strm_);
  break;
  default:

test/simple/test-zlib-from-multiple-gzip-with-garbage.js

@@ -0,0 +1,83 @@
+// Copyright Joyent, Inc. and other Node contributors.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a
+// copy of this software and associated documentation files (the
+// "Software"), to deal in the Software without restriction, including
+// without limitation the rights to use, copy, modify, merge, publish,
+// distribute, sublicense, and/or sell copies of the Software, and to permit
+// persons to whom the Software is furnished to do so, subject to the
+// following conditions:
+//
+// The above copyright notice and this permission notice shall be included
+// in all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN
+// NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
+// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
+// OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
+// USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+// test unzipping a file that was created by concatenating multiple gzip
+// streams.
+
+var common = require('../common');
+var assert = require('assert');
+var zlib = require('zlib');
+
+var util = require('util');
+
+var gzipBuffer = new Buffer(128);
+var gzipOffset = 0;
+
+var stream1 = '123\n';
+var stream2 = '456\n';
+var stream3 = '789\n';
+
+function gzipAppend(data) {
+ data.copy(gzipBuffer, gzipOffset);
+ gzipOffset += data.length;
+}
+
+function writeGzipStream(text, cb) {
+ var gzip = zlib.createGzip();
+ gzip.on('data', gzipAppend);
+ gzip.write(text, function() {
+ gzip.flush(function() {
+ gzip.end(function() {
+ cb();
+ });
+ });
+ });
+}
+
+function writeGarbageStream(text, cb) {
+ gzipAppend(new Buffer(text));
+ cb();
+}
+
+writeGzipStream(stream1, function() {
+ writeGzipStream(stream2, function() {
+ writeGarbageStream(stream3, function() {
+ var gunzip = zlib.createGunzip();
+ var gunzippedData = new Buffer(2 * 1024);
+ var gunzippedOffset = 0;
+ gunzip.on('data', function (data) {
+ data.copy(gunzippedData, gunzippedOffset);
+ gunzippedOffset += data.length;
+ });
+ gunzip.on('error', function() {
+ assert.equal(gunzippedData.toString('utf8', 0, gunzippedOffset),
+ stream1 + stream2);
+ });
+ gunzip.on('end', function() {
+ assert.fail('end event not expected');
+ });
+
+ gunzip.write(gzipBuffer.slice(0, gzipOffset), 'binary', function() {
+ gunzip.end();
+ });
+ });
+ });
+});

test/simple/test-zlib-from-multiple-gzip.js

@@ -0,0 +1,74 @@
+// Copyright Joyent, Inc. and other Node contributors.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a
+// copy of this software and associated documentation files (the
+// "Software"), to deal in the Software without restriction, including
+// without limitation the rights to use, copy, modify, merge, publish,
+// distribute, sublicense, and/or sell copies of the Software, and to permit
+// persons to whom the Software is furnished to do so, subject to the
+// following conditions:
+//
+// The above copyright notice and this permission notice shall be included
+// in all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN
+// NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
+// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
+// OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
+// USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+// test unzipping a file that was created by concatenating multiple gzip
+// streams.
+
+var common = require('../common');
+var assert = require('assert');
+var zlib = require('zlib');
+
+var util = require('util');
+
+var gzipBuffer = new Buffer(128);
+var gzipOffset = 0;
+
+var stream1 = '123\n';
+var stream2 = '456\n';
+var stream3 = '789\n';
+
+function gzipAppend(data) {
+ data.copy(gzipBuffer, gzipOffset);
+ gzipOffset += data.length;
+}
+
+function writeGzipStream(text, cb) {
+ var gzip = zlib.createGzip();
+ gzip.on('data', gzipAppend);
+ gzip.write(text, function() {
+ gzip.flush(function() {
+ gzip.end(function() {
+ cb();
+ });
+ });
+ });
+}
+
+writeGzipStream(stream1, function() {
+ writeGzipStream(stream2, function() {
+ writeGzipStream(stream3, function() {
+ var gunzip = zlib.createGunzip();
+ var gunzippedData = new Buffer(2 * 1024);
+ var gunzippedOffset = 0;
+ gunzip.on('data', function (data) {
+ data.copy(gunzippedData, gunzippedOffset);
+ gunzippedOffset += data.length;
+ });
+ gunzip.on('end', function() {
+ assert.equal(gunzippedData.toString('utf8', 0, gunzippedOffset), stream1 + stream2 + stream3);
+ });
+
+ gunzip.write(gzipBuffer.slice(0, gzipOffset), 'binary', function() {
+ gunzip.end();
+ });
+ });
+ });
+});

test/simple/test-zlib-from-multiple-huge-gzip.js

@@ -0,0 +1,93 @@
+// Copyright Joyent, Inc. and other Node contributors.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a
+// copy of this software and associated documentation files (the
+// "Software"), to deal in the Software without restriction, including
+// without limitation the rights to use, copy, modify, merge, publish,
+// distribute, sublicense, and/or sell copies of the Software, and to permit
+// persons to whom the Software is furnished to do so, subject to the
+// following conditions:
+//
+// The above copyright notice and this permission notice shall be included
+// in all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN
+// NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
+// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
+// OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
+// USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+// test unzipping a file that was created by concatenating multiple gzip
+// streams.
+
+var common = require('../common');
+var assert = require('assert');
+var zlib = require('zlib');
+
+var util = require('util');
+
+var HUGE = 64 * 1024;
+
+var originalBuffer = new Buffer(3 * HUGE);
+var originalOffset = 0;
+
+var gzipBuffer = new Buffer(3 * HUGE);
+var gzipOffset = 0;
+
+function getRandomLetter() {
+ return (Math.random() * (122 - 97)) + 97;
+}
+
+function generateHugeStream() {
+ var buffer = new Buffer(HUGE);
+ for (var i = 0; i < HUGE; i++)
+ buffer.writeUInt8(getRandomLetter(), i);
+
+ buffer.copy(originalBuffer, originalOffset);
+ originalOffset += HUGE;
+
+ return buffer;
+}
+
+function gzipAppend(data) {
+ data.copy(gzipBuffer, gzipOffset);
+ gzipOffset += data.length;
+}
+
+function writeGzipStream(text, cb) {
+ var gzip = zlib.createGzip();
+ gzip.on('data', gzipAppend);
+ gzip.write(text, function() {
+ gzip.flush(function() {
+ gzip.end(function() {
+ cb();
+ });
+ });
+ });
+}
+
+writeGzipStream(generateHugeStream(), function() {
+ writeGzipStream(generateHugeStream(), function() {
+ writeGzipStream(generateHugeStream(), function() {
+ var gunzip = zlib.createGunzip();
+ var gunzippedData = new Buffer(3 * HUGE);
+ var gunzippedOffset = 0;
+ gunzip.on('data', function (data) {
+ data.copy(gunzippedData, gunzippedOffset);
+ gunzippedOffset += data.length;
+ });
+ gunzip.on('end', function() {
+ var gunzippedStr = gunzippedData.toString('utf8', 0, gunzippedOffset);
+ var originalStr = originalBuffer.toString('utf8', 0, 3 * HUGE);
+
+ assert.equal(gunzippedStr, originalStr);
+ });
+
+ gunzip.write(gzipBuffer.slice(0, gzipOffset), 'binary', function() {
+ gunzip.end();
+ });
+ });
+ });
+});