source upload

contrib/preflate/support/arithmetic_coder.cpp

@@ -0,0 +1,232 @@
+/* Copyright 2018 Dirk Steinke
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License. */
+
+#include "pch.h"
+#include <algorithm>
+#include "arithmetic_coder.h"
+#include "array_helper.h"
+#include "bit_helper.h"
+
+const uint8_t ArithmeticCodecBase::_normCheckLUT[8] = {
+  0x33, 0x77, 0xff, 0xff, 0x33, 0x77, 0xff, 0xff
+};
+
+ArithmeticCodecBase::ArithmeticCodecBase()
+  : _low(0)
+  , _high(0x7fffffff) {}
+
+
+ArithmeticEncoder::ArithmeticEncoder(BitOutputStream& bos)
+  : _bos(bos)
+  , _e3cnt(0) {}
+
+void ArithmeticEncoder::_writeE3(const unsigned w) {
+  while (_e3cnt > 0) {
+    uint32_t todo = min(_e3cnt, 16u);
+    _bos.put(w, todo);
+    _e3cnt -= todo;
+  }
+}
+
+void ArithmeticEncoder::flush() {
+  if (_low < 0x20000000) { // case a.) 
+    _bos.put(2, 2); // write 0, 1, E3
+    _writeE3(~0u);
+  } else {
+    _bos.put(1, 1);
+  }
+  _low = 0;
+  _high = 0x7fffffff;
+}
+
+void ArithmeticEncoder::_normalize() {
+#ifdef _DEBUG
+  _ASSERT(_low <= _high && _high < 0x80000000);
+#endif
+  // write determinated bits
+  // this is the case if _low features 1 bits
+  // or _high features 0 bits
+  uint32_t lh = ~_low & _high;
+  if ((lh & 0x40000000) == 0) {
+    unsigned w = (_low & 0x40000000) != 0;
+    _bos.put(w, 1);
+    _writeE3(w - 1);
+    if ((lh & 0x20000000) == 0) {
+      unsigned l = bitLeadingZeroes((lh << 2) + 3);
+      if (l <= 16) {
+        _bos.putReverse(_low >> (30 - l), l);
+      } else {
+        _bos.putReverse(_low >> (30 - 16), 16);
+        _bos.putReverse(_low >> (30 - l), l - 16);
+      }
+      _low  = (_low << (l + 1)) & 0x7fffffff;
+      _high = (((_high + 1) << (l + 1)) - 1) & 0x7fffffff;
+    } else {
+      _low = (_low << 1) & 0x7fffffff;
+      _high = ((_high << 1) + 1) & 0x7fffffff;
+    }
+  }
+
+  // count indeterminated bits
+  lh = ~_low | _high;
+  if ((lh & 0x20000000) == 0) {
+    // low starts with 01, high starts with 10
+    unsigned l = bitLeadingZeroes((lh << 2) + 3);
+    _e3cnt += l;
+    _low = (_low << l) & 0x3fffffff;
+    _high = ((((_high + 1) << l) - 1) & 0x3fffffff)
+            | 0x40000000;
+  }
+#ifdef _DEBUG
+  _ASSERT(_low <= _high && _high < 0x80000000);
+#endif
+}
+
+ArithmeticDecoder::ArithmeticDecoder(BitInputStream& bis) 
+  : _bis(bis)
+  , _value(0) {
+  _value = _bis.getReverse(16) << 15;
+  _value |= _bis.getReverse(15);
+}
+void ArithmeticDecoder::_normalize() {
+#ifdef _DEBUG
+  _ASSERT(_low <= _value && _value <= _high && _high < 0x80000000);
+#endif
+  // skip determinated bits
+  // this is the case if _low features 1 bits
+  // or _high features 0 bits
+  uint32_t lh = ~_low & _high;
+  if ((lh & 0x40000000) == 0) {
+    //unsigned w = (_low & 0x40000000) != 0;
+    if ((lh & 0x20000000) == 0) {
+      unsigned l = bitLeadingZeroes((lh << 2) + 3);
+      _low = (_low << (l + 1)) & 0x7fffffff;
+      _high = (((_high + 1) << (l + 1)) - 1) & 0x7fffffff;
+      if (l <= 15) {
+        _value = ((_value << (l + 1)) + _bis.getReverse(l + 1)) & 0x7fffffff;
+      } else {
+        _value = ((_value << 16) + _bis.getReverse(16)) & 0x7fffffff;
+        _value = ((_value << (l - 15)) + _bis.getReverse(l - 15)) & 0x7fffffff;
+      }
+    } else {
+      _low = (_low << 1) & 0x7fffffff;
+      _high = ((_high << 1) + 1) & 0x7fffffff;
+      _value = ((_value << 1) + _bis.get(1)) & 0x7fffffff;
+    }
+  }
+
+  // count indeterminated bits
+  lh = ~_low | _high;
+  if ((lh & 0x20000000) == 0) {
+    // low starts with 01, high starts with 10
+    unsigned l = bitLeadingZeroes((lh << 2) + 3);
+    _low = (_low << l) & 0x3fffffff;
+    _high = ((((_high + 1) << l) - 1) & 0x3fffffff)
+      | 0x40000000;
+    if (l <= 16) {
+      _value = (((_value << l) + _bis.getReverse(l)) -0x40000000) & 0x7fffffff;
+    } else {
+      _value = ((_value << 16) + _bis.getReverse(16));
+      _value = (((_value << (l - 16)) + _bis.getReverse(l - 16)) - 0x40000000) & 0x7fffffff;
+    }
+  }
+#ifdef _DEBUG
+  _ASSERT(_low <= _value && _value <= _high && _high < 0x80000000);
+#endif
+}
+
+bool modelCheckFixed(unsigned bounds[], unsigned short ids[], unsigned short rids[],
+                     const unsigned N) {
+  unsigned idx = N;
+  for (unsigned i = 0; i < N; ++i) {
+    if (bounds[i]) {
+      if (idx != N) {
+        return false;
+      }
+      idx = i;
+    }
+  }
+  ids[N - 1] = idx;
+  rids[idx] = N - 1;
+  bounds[idx] = 0;
+  bounds[N] = 1 << 16;
+  return true;
+}
+
+void modelSortBounds(unsigned bounds[], unsigned short ids[], unsigned short rids[],
+                     unsigned backup[], const unsigned N) {
+  for (unsigned i = 0; i < N; ++i) {
+    ids[i] = i;
+    backup[i] = bounds[i];
+  }
+  std::sort(ids, ids + N, [=](unsigned i1, unsigned i2) {
+    if (backup[i1] != backup[i2]) {
+      return backup[i1] < backup[i2];
+    }
+    return i1 < i2;
+  });
+  for (unsigned i = 0; i < N; ++i) {
+    bounds[i] = backup[ids[i]];
+    rids[ids[i]] = i;
+  }
+}
+
+void modelRecreateBounds(unsigned bounds[], const unsigned N) {
+  unsigned sum = sumArray(bounds, N), acc, prev;
+  prev = bounds[0];
+  bounds[0] = acc = 0;
+  for (unsigned i = 0; i < N; ++i) {
+    if (prev) {
+      acc += prev;
+      prev = bounds[i + 1];
+      int diff = (((uint64_t)acc) << 16) / sum - bounds[i];
+      unsigned diff_bits = bitLength(diff);
+      const unsigned k = 5;
+      if (diff > 0 && diff_bits > k) {
+        diff = diff & (((1 << k) - 1) << (diff_bits - k));
+      }
+      bounds[i + 1] = bounds[i] + diff;
+      if (bounds[i + 1] <= bounds[i]) {
+        bounds[i + 1] = bounds[i] + 1;
+      }
+    } else {
+      prev = bounds[i + 1];
+      bounds[i + 1] = bounds[i];
+    }
+  }
+  if (bounds[N] > 0) {
+    bounds[N] = 1 << 16;
+  }
+}
+
+void ACFixedScaleBinaryModel::build() {
+  if (bounds[0] == 0 || bounds[1] == 0) {
+    _fixed = true;
+    ids[1] = bounds[0] == 0;
+    rids[ids[1]] = 1;
+    bounds[1] = bounds[0] = 0;
+    bounds[2] = 1 << 16;
+    return;
+  }
+  ids[0] = 0;
+  ids[1] = 1;
+  if (bounds[1] < bounds[0]) {
+    std::swap(ids[0], ids[1]);
+    std::swap(bounds[0], bounds[1]);
+  }
+  rids[ids[0]] = 0;
+  rids[ids[1]] = 1;
+
+  modelRecreateBounds(bounds, 2);
+}

contrib/preflate/support/arithmetic_coder.h

@@ -0,0 +1,260 @@
+/* Copyright 2018 Dirk Steinke
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License. */
+
+#ifndef ARITHMETIC_CODER_H
+#define ARITHMETIC_CODER_H
+
+#include <stdint.h>
+#include <string.h>
+#include "bitstream.h"
+#include "const_division.h"
+
+class ArithmeticCodecBase {
+public:
+  ArithmeticCodecBase();
+  
+  // array for fast check if normalization is required
+  static const uint8_t _normCheckLUT[8];
+  bool _needsNormalization() const {
+    return (_normCheckLUT[_low >> 29] & (1 << (_high >> 29))) != 0;
+  }
+  // arithmetic coding variables
+  uint32_t _low;
+  uint32_t _high;
+};
+
+class ArithmeticEncoder : public ArithmeticCodecBase {
+public:
+  ArithmeticEncoder(BitOutputStream& bos);
+  void flush();
+  void encode(const uint32_t scale, const uint32_t low, const uint32_t high) {
+    // update steps, low count, high count
+    uint32_t step = ((_high - _low) + 1) / scale;
+    _high = _low + step * high - 1;
+    _low += step * low;
+    _checkNormalize();
+  }
+  void encodeShiftScale(const uint32_t shift, const uint32_t low, const uint32_t high) {
+    // update steps, low count, high count
+    uint32_t step = ((_high - _low) + 1) >> shift;
+    _high = _low + step * high - 1;
+    _low += step * low;
+    _checkNormalize();
+  }
+  void encode(const udivider_t<32>& scale, const uint32_t low, const uint32_t high) {
+    // update steps, low count, high count
+    uint32_t step = divide((_high - _low) + 1, scale);
+    _high = _low + step * high - 1;
+    _low += step * low;
+    _checkNormalize();
+  }
+  void encodeBits(const uint32_t value, const uint32_t bits) {
+    uint32_t step = ((_high - _low) + 1) >> bits;
+    _low += step * value;
+    _high = _low + step - 1;
+    _normalize();
+  }
+
+private:
+  void _checkNormalize() {
+    if (_needsNormalization()) {
+      _normalize();
+    }
+  }
+  void _normalize();
+  void _writeE3(const unsigned w);
+
+  BitOutputStream& _bos;
+
+  // arithmetic coding variables
+  uint32_t _e3cnt;
+};
+
+class ArithmeticDecoder : public ArithmeticCodecBase {
+public:
+  ArithmeticDecoder(BitInputStream& bis);
+  unsigned decode(const uint32_t scale, const unsigned bounds[], const unsigned N) {
+    uint32_t step = ((_high - _low) + 1) / scale;
+    return _decode(step, bounds, N);
+  }
+  unsigned decodeShiftScale(const uint32_t shift, const unsigned bounds[], const unsigned N) {
+    uint32_t step = ((_high - _low) + 1) >> shift;
+    return _decode(step, bounds, N);
+  }
+  unsigned decode(const udivider_t<32>& scale, const unsigned bounds[], const unsigned N) {
+    uint32_t step = divide((_high - _low) + 1, scale);
+    return _decode(step, bounds, N);
+  }
+
+  unsigned decodeBinary(const uint32_t scale, const unsigned bounds[]) {
+    uint32_t step = ((_high - _low) + 1) / scale;
+    return _decodeBinary(step, bounds);
+  }
+  unsigned decodeBinaryShiftScale(const uint32_t shift, const unsigned bounds[]) {
+    uint32_t step = ((_high - _low) + 1) >> shift;
+    return _decodeBinary(step, bounds);
+  }
+  unsigned decodeBinary(const udivider_t<32>& scale, const unsigned bounds[]) {
+    uint32_t step = divide((_high - _low) + 1, scale);
+    return _decodeBinary(step, bounds);
+  }
+
+  unsigned decodeBits(const uint32_t bits) {
+    uint32_t step = ((_high - _low) + 1) >> bits;
+    unsigned result = (_value - _low) / step;
+    _low += step * result;
+    _high = _low + step - 1;
+    _normalize();
+    return result;
+  }
+  
+private:
+  unsigned _findIndex(const unsigned bounds[],
+                      const unsigned N,
+                      const unsigned val) {
+    for (unsigned i = N; i > 1; --i) {
+      if (val >= bounds[i - 1]) {
+        return i - 1;
+      }
+    }
+    return 0;
+  }
+
+  unsigned _decode(const uint32_t step, const unsigned bounds[], const unsigned N) {
+    uint32_t val = (_value - _low) / step;
+    unsigned result = _findIndex(bounds, N, val);
+    _high = _low + step * bounds[result + 1] - 1;
+    _low += step * bounds[result];
+    _checkNormalize();
+    return result;
+  }
+  unsigned _decodeBinary(const uint32_t step, const unsigned bounds[]) {
+    unsigned result = (_value >= _low + bounds[1] * step);
+    _high = _low + step * bounds[result + 1] - 1;
+    _low += step * bounds[result];
+    _checkNormalize();
+    return result;
+  }
+  void _checkNormalize() {
+    if (_needsNormalization()) {
+      _normalize();
+    }
+  }
+  void _normalize();
+
+  BitInputStream& _bis;
+
+  // arithmetic coding variables
+  uint32_t _value;
+};
+
+bool modelCheckFixed(unsigned bounds[], unsigned short ids[], unsigned short rids[],
+                     const unsigned N);
+void modelSortBounds(unsigned bounds[], unsigned short ids[], unsigned short rids[],
+                     unsigned backup[], const unsigned N);
+void modelRecreateBounds(unsigned bounds[], const unsigned N);
+
+template <unsigned N>
+struct ACModelBase {
+  static const unsigned L = N;
+  bool isEqualTo(const ACModelBase& m) const {
+    for (unsigned i = 0; i < N; ++i) {
+      if (bounds[i] != m.bounds[i]) {
+        return false;
+      }
+      if (bounds[i + 1] > 0 && ids[i] != m.ids[i]) {
+        return false;
+      }
+    }
+    if (bounds[N] != m.bounds[N]) {
+      return false;
+    }
+    return true;
+  }
+
+  unsigned bounds[N + 1];
+  unsigned short ids[N], rids[N];
+  bool _fixed;
+};
+
+struct ACFixedScaleBinaryModel : public ACModelBase<2> {
+  ACFixedScaleBinaryModel() {}
+  ACFixedScaleBinaryModel(const unsigned(&arr)[2]) {
+    memcpy(this->bounds, arr, sizeof(arr));
+    build();
+  }
+  void build();
+  void encode(ArithmeticEncoder* encoder, const unsigned item) {
+    if (!this->_fixed) {
+      unsigned pos = this->rids[item];
+      encoder->encodeShiftScale(16, this->bounds[pos], this->bounds[pos + 1]);
+    }
+  }
+#if 0
+  unsigned decode(aricoder* codec) {
+    symbol s;
+    s.scale = 1 << 16;
+    unsigned cnt = codec->decode_count(&s);
+    for (unsigned i = 0; i < N; ++i) {
+      if (cnt < bounds[i + 1]) {
+        s.low_count = bounds[i];
+        s.high_count = bounds[i + 1];
+        codec->decode(&s);
+        return ids[i];
+      }
+    }
+    return 0;
+  }
+#endif
+};
+
+template <unsigned N>
+struct ACFixedScaleModel : public ACModelBase<N> {
+  ACFixedScaleModel() {}
+  ACFixedScaleModel(const unsigned(&arr)[N]) {
+    memcpy(this->bounds, arr, sizeof(arr));
+    build();
+  }
+  void build() {
+    unsigned backup[N];
+    if (!(this->_fixed = modelCheckFixed(this->bounds, this->ids, this->rids, N))) {
+      modelSortBounds(this->bounds, this->ids, this->rids, backup, N);
+      modelRecreateBounds(this->bounds, N);
+    }
+  }
+  void encode(ArithmeticEncoder* encoder, const unsigned item) {
+    if (!this->_fixed) {
+      unsigned pos =this->rids[item];
+      encoder->encodeShiftScale(16, this->bounds[pos], this->bounds[pos + 1]);
+    }
+  }
+#if 0
+  unsigned decode(aricoder* codec) {
+    symbol s;
+    s.scale = 1 << 16;
+    unsigned cnt = codec->decode_count(&s);
+    for (unsigned i = 0; i < N; ++i) {
+      if (cnt < bounds[i + 1]) {
+        s.low_count = bounds[i];
+        s.high_count = bounds[i + 1];
+        codec->decode(&s);
+        return this->ids[i];
+      }
+    }
+    return 0;
+  }
+#endif
+};
+
+#endif /* ARITHMETIC_CODER_H */

contrib/preflate/support/array_helper.cpp

@@ -0,0 +1,24 @@
+/* Copyright 2018 Dirk Steinke
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License. */
+
+#include "pch.h"
+#include "array_helper.h"
+
+unsigned sumArray(const unsigned* data, const unsigned n) {
+  unsigned sum = 0;
+  for (unsigned i = 0; i < n; ++i) {
+    sum += data[i];
+  }
+  return sum;
+}

contrib/preflate/support/array_helper.h

@@ -0,0 +1,25 @@
+/* Copyright 2018 Dirk Steinke
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License. */
+
+#ifndef ARRAY_HELPER_H
+#define ARRAY_HELPER_H
+
+unsigned sumArray(const unsigned* data, const unsigned n);
+
+template <unsigned N>
+inline unsigned sumArray(const unsigned (&data)[N]) {
+  return sumArray(data, N);
+}
+
+#endif /* ARRAY_HELPER_H */

contrib/preflate/support/bit_helper.cpp

@@ -0,0 +1,73 @@
+/* Copyright 2018 Dirk Steinke
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License. */
+
+#include "pch.h"
+#include "bit_helper.h"
+
+unsigned bitLength(unsigned value) {
+  unsigned l = 0;
+  while (value > 0) {
+    l++;
+    value >>= 1;
+  }
+  return l;
+}
+
+static unsigned char reverse4[16] = {0, 8, 4, 12, 2, 10, 6, 14, 1, 9, 5, 13, 3, 11, 7, 15};
+static unsigned bitReverse8(const unsigned value) {
+  return (reverse4[value & 0x0f] << 4) | reverse4[(value >> 4) & 0x0f];
+}
+static unsigned bitReverse16(const unsigned value) {
+  return (bitReverse8(value & 0xff) << 8) | bitReverse8(value >> 8);
+}
+static unsigned bitReverse32(const unsigned value) {
+  return (bitReverse16(value & 0xffff) << 16) | bitReverse16(value >> 16);
+}
+unsigned bitReverse(const unsigned value, const unsigned bits) {
+  if (bits <= 8) {
+    return bitReverse8(value) >> (8 - bits);
+  }
+  if (bits <= 16) {
+    return bitReverse16(value) >> (16 - bits);
+  }
+  return bitReverse32(value) >> (32 - bits);
+}
+
+static unsigned char leading4[16] = {4, 3, 2, 2, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0};
+unsigned bitLeadingZeroes(const unsigned value_) {
+  if (value_ == 0) {
+    return 32;
+  }
+  unsigned value = value_;
+  unsigned result = 0;
+  while ((value & 0xf0000000) == 0) {
+    value <<= 4;
+    result += 4;
+  }
+  return result + leading4[value >> 28];
+}
+static unsigned char trailing4[16] = {4, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0};
+unsigned bitTrailingZeroes(const unsigned value_) {
+  if (value_ == 0) {
+    return 32;
+  }
+  unsigned value = value_;
+  unsigned result = 0;
+  while ((value & 0xf) == 0) {
+    value >>= 4;
+    result += 4;
+  }
+  return result + trailing4[value & 0xf];
+}
+

contrib/preflate/support/bit_helper.h

@@ -0,0 +1,23 @@
+/* Copyright 2018 Dirk Steinke
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License. */
+
+#ifndef BIT_HELPER_H
+#define BIT_HELPER_H
+
+unsigned bitLength(unsigned value);
+unsigned bitReverse(const unsigned value, const unsigned bits);
+unsigned bitLeadingZeroes(const unsigned value);
+unsigned bitTrailingZeroes(const unsigned value);
+
+#endif /* BIT_HELPER_H */

contrib/preflate/support/bitstream.cpp

@@ -0,0 +1,176 @@
+/* Copyright 2018 Dirk Steinke
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License. */
+
+#include "pch.h"
+#include <algorithm>
+#include <memory.h>
+#include "bitstream.h"
+
+BitInputStream::BitInputStream(InputStream& is)
+  : _input(is)
+  , _bufPos(0)
+  , _bufSize(0)
+  , _bufFastLimit(0)
+  , _eof(false)
+  , _bits(0)
+  , _bitsRemaining(0)
+  , _totalBitPos(0)
+{}
+
+void BitInputStream::_fillBytes() {
+  // free space in bit buffer
+  if (_bufPos >= _bufFastLimit) {
+    if (!_eof) {
+      unsigned remaining = _bufSize - _bufPos;
+      memcpy(_buffer + PRE_BUF_EXTRA - remaining,
+             _buffer + _bufPos, remaining);
+      _bufPos = PRE_BUF_EXTRA - remaining;
+      _bufSize = PRE_BUF_EXTRA + _input.read(_buffer + PRE_BUF_EXTRA, BUF_SIZE);
+      _bufFastLimit = max(_bufPos, _bufSize - PRE_BUF_EXTRA);
+      _eof = _bufSize != PRE_BUF_EXTRA + BUF_SIZE;
+    }
+  }
+}
+void BitInputStream::_fill() {
+  // free space in bit buffer
+  if (_bufPos >= _bufFastLimit) {
+    if (!_eof) {
+      _fillBytes();
+    }
+    while (_bitsRemaining <= BITS - 8 && _bufPos < _bufSize) {
+      _bits |= ((size_t)_buffer[_bufPos++]) << _bitsRemaining;
+      _bitsRemaining += 8;
+    }
+    return;
+  }
+  while (_bitsRemaining <= BITS - 8) {
+    _bits |= ((size_t)_buffer[_bufPos++]) << _bitsRemaining;
+    _bitsRemaining += 8;
+  }
+}
+size_t BitInputStream::copyBytesTo(OutputStream& output, const size_t len) {
+  if (_bitsRemaining & 7) {
+    return 0;
+  }
+  uint8_t a[sizeof(_bits)];
+  size_t l = 0;
+  while (_bitsRemaining > 0 && l < len) {
+    a[l++] = _bits & 0xff;
+    _bitsRemaining -= 8;
+    _bits >>= 8;
+    _totalBitPos += 8;
+  }
+  size_t w = output.write(a, l);
+  if (w != l) {
+    return w;
+  }
+  while (l < len) {
+    unsigned todo = min(len - l, (size_t)(_bufSize - _bufPos));
+    w = output.write(_buffer + _bufPos, todo);
+    _totalBitPos += 8 * w;
+    _bufPos += w;
+    l += w;
+    if (w != todo || eof()) {
+      return l;
+    }
+    _fillBytes();
+  }
+  return l;
+}
+size_t BitInputStream::getBytes(uint8_t* data, const size_t size_) {
+  skipToByte();
+  size_t size = size_;
+  while (_bitsRemaining > 0 && size > 0) {
+    *data++ = _bits & 0xff;
+    _bitsRemaining -= 8;
+    _bits >>= 8;
+    _totalBitPos += 8;
+    size--;
+  }
+  while (size > 0) {
+    unsigned todo = min(size, (size_t)(_bufSize - _bufPos));
+    memcpy(data, _buffer + _bufPos, todo);
+    data += todo;
+    _totalBitPos += 8 * todo;
+    _bufPos += todo;
+    size -= todo;
+    if (eof()) {
+      return size_ - size;
+    }
+    _fillBytes();
+  }
+  return size_;
+}
+uint64_t BitInputStream::getVLI() {
+  uint64_t result = 0, o = 0;
+  unsigned s = 0, c;
+  unsigned bitsRemaining = ((_bitsRemaining - 1) & 7) + 1;
+  unsigned limit = 1 << (bitsRemaining - 1);
+  while ((c = get(bitsRemaining)) >= limit) {
+    result += ((uint64_t)(c & (limit - 1))) << s;
+    s += (bitsRemaining - 1);
+    o = (o + 1) << (bitsRemaining - 1);
+    bitsRemaining = 8;
+    limit = 128;
+  }
+  return result + o + (((uint64_t)c) << s);
+}
+
+BitOutputStream::BitOutputStream(OutputStream& output)
+  : _output(output)
+  , _bufPos(0)
+  , _bits(0)
+  , _bitPos(0) {}
+
+void BitOutputStream::_flush() {
+  while (_bitPos >= 8) {
+    _buffer[_bufPos++] = _bits & 0xff;
+    _bits >>= 8;
+    _bitPos -= 8;
+  }
+  if (_bufPos >= BUF_SIZE) {
+    _output.write(_buffer, BUF_SIZE);
+    memcpy(_buffer, _buffer + BUF_SIZE, _bufPos - BUF_SIZE);
+    _bufPos -= BUF_SIZE;
+  }
+}
+void BitOutputStream::flush() {
+  _flush();
+
+  if (_bitPos > 0) {
+    _buffer[_bufPos++] = _bits & 0xff;
+    _bits   = 0;
+    _bitPos = 0;
+  }
+
+  _output.write(_buffer, _bufPos);
+  _bufPos = 0;
+}
+void BitOutputStream::putBytes(const uint8_t* data, const size_t size) {
+  flush();
+  _output.write(data, size);
+}
+void BitOutputStream::putVLI(const uint64_t size_) {
+  uint64_t size = size_;
+  unsigned bitsRemaining = 8 - (_bitPos & 7);
+  unsigned limit = 1 << (bitsRemaining - 1);
+  while (size >= limit) {
+    put(size | limit, bitsRemaining);
+    size = (size >> (bitsRemaining - 1)) - 1;
+    bitsRemaining = 8;
+    limit = 128;
+  }
+  put(size, bitsRemaining);
+ 
+}

contrib/preflate/support/bitstream.h

@@ -0,0 +1,130 @@
+/* Copyright 2018 Dirk Steinke
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License. */
+
+#ifndef BITSTREAM_H
+#define BITSTREAM_H
+
+#include <algorithm>
+#include "bit_helper.h"
+#include "stream.h"
+
+// Huffman decoder for little endian 
+class BitInputStream {
+public:
+  BitInputStream(InputStream&);
+
+  bool eof() const {
+    return _eof && _bufPos == _bufSize && !_bitsRemaining;
+  }
+
+  size_t bitPos() const {
+    return _totalBitPos;
+  }
+
+  size_t peek(const unsigned n) {
+    if (_bitsRemaining < n) {
+      _fill();
+    }
+    return _bits & ((1 << n) - 1);
+  }
+  void skip(const unsigned n) {
+    _bitsRemaining -= min(n, _bitsRemaining);
+    _bits >>= n;
+    _totalBitPos += n;
+  }
+  size_t get(const unsigned n) {
+    size_t v = peek(n);
+    skip(n);
+    return v;
+  }
+  size_t getReverse(const unsigned n) {
+    return bitReverse(get(n), n);
+  }
+  void skipToByte() {
+    skip(_bitsRemaining & 7);
+  }
+  bool checkLastBitsOfByteAreZero() {
+    return peek(_bitsRemaining & 7) == 0;
+  }
+  void fastFill(const unsigned n) {
+    if (_bitsRemaining < n) {
+      _fill();
+    }
+  }
+  size_t fastPeek(const unsigned n) {
+    return _bits & ((1 << n) - 1);
+  }
+  size_t fastGet(const unsigned n) {
+    size_t v = fastPeek(n);
+    skip(n);
+    return v;
+  }
+  size_t copyBytesTo(OutputStream& output, const size_t len);
+  size_t getBytes(uint8_t* data, const size_t size);
+  uint64_t getVLI();
+
+private:
+  void _fillBytes();
+  void _fill();
+
+  enum { BUF_SIZE = 1024, PRE_BUF_EXTRA = 16, BITS = sizeof(size_t)*8 };
+
+  InputStream& _input;
+  unsigned char _buffer[PRE_BUF_EXTRA + BUF_SIZE];
+  unsigned _bufPos, _bufSize, _bufFastLimit;
+  bool _eof;
+  size_t _bits;
+  unsigned _bitsRemaining;
+  size_t _totalBitPos;
+};
+
+class BitOutputStream {
+public:
+  BitOutputStream(OutputStream&);
+
+  void put(const size_t value, const unsigned n) {
+    if (_bitPos + n >= BITS) {
+      _flush();
+    }
+    _bits   |= (value & ((1 << n) - 1)) << _bitPos;
+    _bitPos += n;
+  }
+  void putReverse(const size_t value, const unsigned n) {
+    put(bitReverse(value, n), n);
+  }
+  void fillByte() {
+    _bitPos = (_bitPos + 7) & ~7;
+  }
+  void flush();
+  unsigned bitPos() const {
+    return _bitPos;
+  }
+  void putBytes(const uint8_t* data, const size_t size);
+  void putVLI(const uint64_t size);
+
+private:
+  void _flush();
+
+  enum {
+    BUF_SIZE = 1024, BUF_EXTRA = 64, BITS = sizeof(size_t) * 8
+  };
+
+  OutputStream& _output;
+  unsigned char _buffer[BUF_SIZE + BUF_EXTRA];
+  unsigned _bufPos;
+  size_t _bits;
+  unsigned _bitPos;
+};
+
+#endif /* BITSTREAM_H */

contrib/preflate/support/const_division.cpp

@@ -0,0 +1,115 @@
+/* Copyright 2018 Dirk Steinke
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License. */
+
+#include "pch.h"
+#include "bit_helper.h"
+#include "const_division.h"
+
+// Based on "N-Bit Unsigned Division Via N-Bit Multiply-Add"
+// by Robison
+
+template <unsigned N>
+udivider_t<N> build_udivider(const typename divider_uint_t<N>::type d) {
+  typedef typename divider_uint_t<N * 2>::type T1;
+  typedef typename divider_uint_t<N>::type T2;
+  udivider_t<N> result;
+  result.shift = bitLength(d) - 1;
+  if ((d & (d - 1)) == 0) {
+    result.magic1 = result.magic2 = ~(T2)0;
+  } else {
+    T2 shm = 1 << result.shift;
+    T2 t = (((T1)shm) << N) / d;
+    T2 r = t * d + d;
+    if (r <= shm) {
+      result.magic1 = t + 1;
+      result.magic2 = 0;
+    } else {
+      result.magic1 = t;
+      result.magic2 = t;
+    }
+  }
+  return result;
+}
+
+udivider_t<16> build_udivider_16(const uint16_t d) {
+  return build_udivider<16>(d);
+}
+udivider_t<32> build_udivider_32(const uint32_t d) {
+  return build_udivider<32>(d);
+}
+
+template <unsigned N>
+ucdivider_t<N> build_ucdivider(const typename divider_uint_t<N>::type d) {
+  typedef typename divider_uint_t<N * 2>::type T1;
+  typedef typename divider_uint_t<N>::type T2;
+  ucdivider_t<N> result;
+  result.ctrl = bitLength(d) - 1;
+  if ((d & (d - 1)) == 0) {
+    result.magic = ~(T2)0;
+    result.ctrl |= 0x80;
+  } else {
+    T2 shm = 1 << result.ctrl;
+    T2 t = (((T1)shm) << N) / d;
+    T2 r = t * d + d;
+    if (r <= shm) {
+      result.magic = t + 1;
+    } else {
+      result.magic = t;
+      result.ctrl |= 0x80;
+    }
+  }
+  return result;
+}
+
+ucdivider_t<16> build_ucdivider_16(const uint16_t d) {
+  return build_ucdivider<16>(d);
+}
+ucdivider_t<32> build_ucdivider_32(const uint32_t d) {
+  return build_ucdivider<32>(d);
+}
+
+template <unsigned N>
+sdivider_t<N> build_sdivider(const typename divider_int_t<N>::type d_) {
+  sdivider_t<N> result;
+  udivider_t<N> uresult = build_udivider<N>(d_ < 0 ? -d_ : d_);
+  result.magic1 = uresult.magic1;
+  result.magic2 = uresult.magic2;
+  result.shift = uresult.shift;
+  result.sign = d_ < 0 ? -1 : 0;
+  return result;
+}
+sdivider_t<16> build_sdivider_16(const int16_t d) {
+  return build_sdivider<16>(d);
+}
+sdivider_t<32> build_sdivider_32(const int32_t d) {
+  return build_sdivider<32>(d);
+}
+
+template <unsigned N>
+scdivider_t<N> build_scdivider(const typename divider_int_t<N>::type d_) {
+  scdivider_t<N> result;
+  ucdivider_t<N> uresult = build_ucdivider<N>(d_ < 0 ? -d_ : d_);
+  result.magic = uresult.magic;
+  result.ctrl  = uresult.ctrl;
+  if (d_ < 0) {
+    result.ctrl |= 0x40;
+  }
+  return result;
+}
+scdivider_t<16> build_scdivider_16(const int16_t d) {
+  return build_scdivider<16>(d);
+}
+scdivider_t<32> build_scdivider_32(const int32_t d) {
+  return build_scdivider<32>(d);
+}

contrib/preflate/support/const_division.h

@@ -0,0 +1,170 @@
+/* Copyright 2018 Dirk Steinke
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License. */
+
+#ifndef CONST_DIVISION_H
+#define CONST_DIVISION_H
+
+#include <stdint.h>
+
+template <unsigned N> struct divider_int_t;
+template <unsigned N> struct divider_uint_t;
+template <> struct divider_int_t<16> {
+  typedef int16_t type; 
+};
+template <> struct divider_int_t<32> {
+  typedef int32_t type;
+};
+template <> struct divider_uint_t<16> {
+  typedef uint16_t type;
+};
+template <> struct divider_uint_t<32> {
+  typedef uint32_t type;
+};
+template <> struct divider_uint_t<64> {
+  typedef uint64_t type;
+};
+
+
+template <unsigned N>
+struct udivider_t {
+  typename divider_uint_t<N>::type magic1; // factor
+  typename divider_uint_t<N>::type magic2; // addend
+  uint8_t shift;
+};
+
+template <unsigned N>
+struct ucdivider_t {
+  typename divider_uint_t<N>::type magic; // factor/addend
+  uint8_t ctrl; // bits 0..3/4/5 - shift, bit 7 - add required
+};
+
+// If it wasn't for +/-1, the signed dividers wouldn't
+// need the add-term (magic2), as they could just
+// use a factor (magic1) with one more bit precision.
+template <unsigned N>
+struct sdivider_t {
+  typename divider_uint_t<N>::type magic1; // factor
+  typename divider_uint_t<N>::type magic2; // addend
+  uint8_t shift; 
+  int8_t sign; // -1 if negative, 0 otherwise
+};
+
+template <unsigned N>
+struct scdivider_t {
+  typename divider_uint_t<N>::type magic;
+  uint8_t ctrl; // bits 0..3/4/5 - shift, bit 6 - negative, bit 7 - add required
+};
+
+udivider_t<16> build_udivider_16(const uint16_t d);
+udivider_t<32> build_udivider_32(const uint32_t d);
+
+ucdivider_t<16> build_ucdivider_16(const uint16_t d);
+ucdivider_t<32> build_ucdivider_32(const uint32_t d);
+
+sdivider_t<16> build_sdivider_16(const int16_t d);
+sdivider_t<32> build_sdivider_32(const int32_t d);
+
+scdivider_t<16> build_scdivider_16(const int16_t d);
+scdivider_t<32> build_scdivider_32(const int32_t d);
+
+template <unsigned N1, unsigned N2>
+inline typename divider_uint_t<N1>::type 
+divide_template(const typename divider_uint_t<N1>::type dividend, 
+                const udivider_t<N2>& divisor) {
+  typedef typename divider_uint_t<N1 * 2>::type T1;
+  typedef typename divider_uint_t<N1>::type T2;
+  T1 t = ((T1)dividend) * divisor.magic1 + divisor.magic2;
+  T2 u = (T2)(t >> N2);
+  return u >> divisor.shift;
+}
+template <unsigned N1, unsigned N2>
+inline typename divider_uint_t<N1>::type
+divide_template(const typename divider_uint_t<N1>::type dividend,
+                const ucdivider_t<N2>& divisor) {
+  typedef typename divider_uint_t<N1 * 2>::type T1;
+  typedef typename divider_uint_t<N1>::type T2;
+  T1 t = ((T1)dividend) * divisor.magic
+        + (divisor.ctrl & 0x80 ? divisor.magic : 0);
+  T2 u = (T2)(t >> N2);
+  return u >> (divisor.ctrl & (N2 - 1));
+}
+template <unsigned N1, unsigned N2>
+inline typename divider_int_t<N1>::type
+divide_template(const typename divider_int_t<N1>::type dividend,
+                const sdivider_t<N2>& divisor) {
+  typedef typename divider_uint_t<N1 * 2>::type T1;
+  typedef typename divider_uint_t<N1>::type T2;
+  T2 s = dividend < 0 ? -1 : 0;
+  T1 t = ((T1)(T2)((dividend ^ s) - s)) * divisor.magic1 
+        + divisor.magic2;
+  T2 u = (T2)(t >> N2) >> divisor.shift;
+  s ^= divisor.sign;
+  return (u ^ s) - s;
+}
+template <unsigned N1, unsigned N2>
+inline typename divider_int_t<N1>::type
+divide_template(const typename divider_int_t<N1>::type dividend,
+                const scdivider_t<N2>& divisor) {
+  typedef typename divider_uint_t<N1 * 2>::type T1;
+  typedef typename divider_uint_t<N1>::type T2;
+  
+  T2 s = dividend < 0 ? -1 : 0;
+  T1 t = ((T1)(T2)((dividend ^ s) - s)) * divisor.magic
+        + (divisor.ctrl & 0x80 ? divisor.magic : 0);
+  T2 u = (T2)(t >> N2) >> (divisor.ctrl & (N2 - 1));
+  s ^= (divisor.ctrl & 0x40 ? -1 : 0);
+  return (u ^ s) - s;
+}
+
+inline uint16_t divide(const uint16_t dividend, const udivider_t<16>& divisor) {
+  return divide_template<16, 16>(dividend, divisor);
+}
+inline uint32_t divide(const uint32_t dividend, const udivider_t<16>& divisor) {
+  return divide_template<32, 16>(dividend, divisor);
+}
+inline uint32_t divide(const uint32_t dividend, const udivider_t<32>& divisor) {
+  return divide_template<32, 32>(dividend, divisor);
+}
+
+inline uint16_t divide(const uint16_t dividend, const ucdivider_t<16>& divisor) {
+  return divide_template<16, 16>(dividend, divisor);
+}
+inline uint32_t divide(const uint32_t dividend, const ucdivider_t<16>& divisor) {
+  return divide_template<32, 16>(dividend, divisor);
+}
+inline uint32_t divide(const uint32_t dividend, const ucdivider_t<32>& divisor) {
+  return divide_template<32, 32>(dividend, divisor);
+}
+
+inline int16_t divide(const int16_t dividend, const sdivider_t<16>& divisor) {
+  return divide_template<16, 16>(dividend, divisor);
+}
+inline int32_t divide(const int32_t dividend, const sdivider_t<16>& divisor) {
+  return divide_template<32, 16>(dividend, divisor);
+}
+inline int32_t divide(const int32_t dividend, const sdivider_t<32>& divisor) {
+  return divide_template<32, 32>(dividend, divisor);
+}
+
+inline int16_t divide(const int16_t dividend, const scdivider_t<16>& divisor) {
+  return divide_template<16, 16>(dividend, divisor);
+}
+inline int32_t divide(const int32_t dividend, const scdivider_t<16>& divisor) {
+  return divide_template<32, 16>(dividend, divisor);
+}
+inline int32_t divide(const int32_t dividend, const scdivider_t<32>& divisor) {
+  return divide_template<32, 32>(dividend, divisor);
+}
+
+#endif /* CONST_DIVISION_H */

contrib/preflate/support/filestream.cpp

@@ -0,0 +1,41 @@
+/* Copyright 2018 Dirk Steinke
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License. */
+
+#include "pch.h"
+#include <algorithm>
+#include <string.h>
+#include <stdio.h>
+#include "filestream.h"
+
+FileStream::FileStream(FILE* f) : _f(f) {}
+
+bool FileStream::eof() const {
+  return feof(_f);
+}
+size_t FileStream::read(unsigned char* buffer, const size_t size) {
+  return fread(buffer, 1, size, _f);
+}
+
+size_t FileStream::write(const unsigned char* buffer, const size_t size) {
+  return fwrite(buffer, 1, size, _f);
+}
+
+uint64_t FileStream::tell() const {
+  return _ftelli64(_f);
+}
+uint64_t FileStream::seek(const uint64_t newPos) {
+  uint64_t oldPos = tell();
+  _fseeki64(_f, newPos, SEEK_SET);
+  return oldPos;
+}

contrib/preflate/support/filestream.h

@@ -0,0 +1,38 @@
+/* Copyright 2018 Dirk Steinke
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License. */
+
+#ifndef FILESTREAM_H
+#define FILESTREAM_H
+
+#include <stdint.h>
+#include <vector>
+#include "stream.h"
+
+class FileStream : public SeekableInputOutputStream {
+public:
+  FileStream(FILE* f);
+
+  virtual bool eof() const;
+  virtual size_t read(unsigned char* buffer, const size_t size);
+
+  virtual size_t write(const unsigned char* buffer, const size_t size);
+
+  virtual uint64_t tell() const;
+  virtual uint64_t seek(const uint64_t newPos);
+
+private:
+  FILE* _f;
+};
+
+#endif /* FILESTREAM_H */

contrib/preflate/support/huffman_decoder.cpp

@@ -0,0 +1,111 @@
+/* Copyright 2018 Dirk Steinke
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License. */
+
+#include "pch.h"
+#include <algorithm>
+#include "huffman_decoder.h"
+#include "huffman_helper.h"
+#include "bit_helper.h"
+
+HuffmanDecoder::HuffmanDecoder(
+    const unsigned char* symbolBitLengths,
+    const size_t symbolCount,
+    const bool disableZeroBitSymbols,
+    const unsigned char maxBitsPerTable
+) : _error(false) {
+  if (!_constructTables(symbolBitLengths, symbolCount, disableZeroBitSymbols, maxBitsPerTable)) {
+    _constructErrorTable();
+  }
+}
+
+size_t HuffmanDecoder::_decodeDeeper(
+    BitInputStream& bis, 
+    const size_t tableId_
+) const {
+  bis.skip(_table0.peekBits);
+  size_t tableId = tableId_;
+  do {
+    const Table* table = &_tables[tableId];
+    size_t v = bis.peek(table->peekBits);
+    signed short w = table->lookup[v];
+    if (w >= 0) {
+      bis.skip(w & 0xf);
+      return w >> 4;
+    }
+    bis.skip(table->peekBits);
+    tableId = ~w;
+  } while (true);
+}
+bool HuffmanDecoder::_constructTables(
+    const unsigned char* symbolBitLengths,
+    const size_t symbolCount,
+    const bool disableZeroBitSymbols,
+    const unsigned char maxBitsPerTable
+) {
+  if (maxBitsPerTable < 1 || maxBitsPerTable > 15) {
+    return false;
+  }
+  unsigned nextCode[HuffmanHelper::MAX_BL + 2];
+  unsigned char minLength, maxLength;
+  if (!HuffmanHelper::countSymbols(nextCode, minLength, maxLength,
+                                   symbolBitLengths, symbolCount, 
+                                   disableZeroBitSymbols)) {
+    return false;
+  }
+
+  _table0.peekBits = min((unsigned char)(maxLength - 1), maxBitsPerTable);
+  _table0.lookup.resize(1 << _table0.peekBits);
+  std::fill(_table0.lookup.begin(), _table0.lookup.end(), 0);
+
+  unsigned char minL = disableZeroBitSymbols ? 2 : 1;
+
+  for (unsigned i = 0; i < symbolCount; ++i) {
+    unsigned char l = (unsigned char)(symbolBitLengths[i] + 1);
+    if (l < minL) {
+      continue;
+    }
+    unsigned char k = l - 1, maxK = maxLength - 1;
+    unsigned code = bitReverse(nextCode[l]++, k);
+    Table* t = &_table0;
+    while (k > t->peekBits) {
+      k -= t->peekBits;
+      maxK -= t->peekBits;
+      unsigned subbits = code & ((1 << t->peekBits) - 1);
+      code >>= t->peekBits;
+      signed short v = t->lookup[subbits];
+      if (v >= 0) {
+        unsigned newTableId = _tables.size();
+        t->lookup[subbits] = ~newTableId;
+        _tables.push_back(Table());
+        t = &_tables[newTableId];
+        t->peekBits = min(maxK, maxBitsPerTable);
+        t->lookup.resize(1 << t->peekBits);
+        std::fill(t->lookup.begin(), t->lookup.end(), 0);
+      } else {
+        t = &_tables[~v];
+      }
+    }
+    do {
+      t->lookup[code] = (i << 4) | k;
+      code += 1 << k;
+    } while (code < t->lookup.size());
+  }
+  return true;
+}
+void HuffmanDecoder::_constructErrorTable() {
+  _error = true;
+  _table0.peekBits = 0;
+  _table0.lookup.resize(1);
+  _table0.lookup[0] = 0;
+}

contrib/preflate/support/huffman_decoder.h

@@ -0,0 +1,62 @@
+/* Copyright 2018 Dirk Steinke
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License. */
+
+#ifndef HUFFMAN_DECODER_H
+#define HUFFMAN_DECODER_H
+
+#include <vector>
+#include "bitstream.h"
+
+// Huffman decoder
+class HuffmanDecoder {
+public:
+  HuffmanDecoder(const unsigned char* symbolBitLengths,
+                 const size_t symbolCount,
+                 const bool disableZeroBitSymbols,
+                 const unsigned char maxBitsPerTable);
+
+  bool error() const {
+    return _error;
+  }
+
+  size_t decode(BitInputStream& bis) const {
+    size_t v = bis.peek(_table0.peekBits);
+    signed short w = _table0.lookup[v];
+    if (w >= 0) {
+      bis.skip(w & 0xf);
+      return w >> 4;
+    }
+    return _decodeDeeper(bis, ~w);
+  }
+
+private:
+  size_t _decodeDeeper(BitInputStream& bis, const size_t tableId) const;
+  bool _constructTables(const unsigned char* symbolBitLengths,
+                        const size_t symbolCount,
+                        const bool disableZeroBitSymbols,
+                        const unsigned char maxBitsPerTable);
+  void _constructErrorTable();
+
+private:
+  struct Table {
+    unsigned char peekBits;
+    std::vector<signed short> lookup;
+  };
+
+  Table _table0;
+  std::vector<Table> _tables;
+  bool _error;
+};
+
+#endif /* HUFFMAN_DECODER_H */

contrib/preflate/support/huffman_encoder.cpp

@@ -0,0 +1,65 @@
+/* Copyright 2018 Dirk Steinke
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License. */
+
+#include "pch.h"
+#include <algorithm>
+#include "huffman_encoder.h"
+#include "huffman_helper.h"
+#include "bit_helper.h"
+
+HuffmanEncoder::HuffmanEncoder(
+    const unsigned char* symbolBitLengths,
+    const unsigned symbolCount,
+    const bool disableZeroBitSymbols
+) : _error(false) {
+  if (!_constructTables(symbolBitLengths, symbolCount, disableZeroBitSymbols)) {
+    _constructErrorTable(symbolCount);
+  }
+}
+
+bool HuffmanEncoder::_constructTables(
+    const unsigned char* symbolBitLengths,
+    const unsigned symbolCount,
+    const bool disableZeroBitSymbols
+) {
+  unsigned nextCode[HuffmanHelper::MAX_BL + 2];
+  unsigned char minLength, maxLength;
+  if (!HuffmanHelper::countSymbols(nextCode, minLength, maxLength,
+                                   symbolBitLengths, symbolCount,
+                                   disableZeroBitSymbols)) {
+    return false;
+  }
+
+  unsigned char minL = disableZeroBitSymbols ? 2 : 1;
+
+  _lookup.resize(symbolCount);
+  for (unsigned i = 0; i < symbolCount; ++i) {
+    unsigned char l = (unsigned char)(symbolBitLengths[i] + 1);
+    if (l < minL) {
+      _lookup[i] = 0;
+      continue;
+    }
+    unsigned char k = l - 1;
+    unsigned code = bitReverse(nextCode[l]++, k);
+    _lookup[i] = (code << 5) | k;
+  }
+  return true;
+}
+void HuffmanEncoder::_constructErrorTable(
+    const unsigned symbolCount
+) {
+  _error = true;
+  _lookup.resize(symbolCount);
+  std::fill(_lookup.begin(), _lookup.end(), 0);
+}

contrib/preflate/support/huffman_encoder.h

@@ -0,0 +1,48 @@
+/* Copyright 2018 Dirk Steinke
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License. */
+
+#ifndef HUFFMAN_ENCODER_H
+#define HUFFMAN_ENCODER_H
+
+#include <vector>
+#include "bitstream.h"
+
+// Huffman decoder
+class HuffmanEncoder {
+public:
+  HuffmanEncoder(const unsigned char* symbolBitLengths,
+                 const unsigned symbolCount,
+                 const bool disableZeroBitSymbols);
+
+  bool error() const {
+    return _error;
+  }
+
+  void encode(BitOutputStream& bos, const unsigned symbol) const {
+    unsigned v = _lookup[symbol];
+    bos.put(v >> 5, v & 0x1f);
+  }
+
+private:
+  bool _constructTables(const unsigned char* symbolBitLengths,
+                        const unsigned symbolCount,
+                        const bool disableZeroBitSymbols);
+  void _constructErrorTable(const unsigned symbolCount);
+
+private:
+  std::vector<unsigned> _lookup;
+  bool _error;
+};
+
+#endif /* HUFFMAN_ENCODER_H */

contrib/preflate/support/huffman_helper.cpp

@@ -0,0 +1,75 @@
+/* Copyright 2018 Dirk Steinke
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License. */
+
+#include "pch.h"
+#include <algorithm>
+#include <string.h>
+#include "huffman_helper.h"
+#include "bit_helper.h"
+
+bool HuffmanHelper::countSymbols(
+    unsigned(&nextCode)[MAX_BL + 2],
+    unsigned char& minLength,
+    unsigned char& maxLength,
+    const unsigned char* symbolBitLengths,
+    const unsigned symbolCount,
+    const bool disableZeroBitSymbols
+) {
+  if (symbolCount < 1 || symbolCount >= 1024) {
+    return false;
+  }
+  unsigned short blCount[MAX_BL + 2];
+
+  // Count symbol frequencies
+  memset(blCount, 0, sizeof(blCount));
+  for (unsigned i = 0; i < symbolCount; ++i) {
+    unsigned char l = (unsigned char)(symbolBitLengths[i] + 1);
+    if (l > MAX_BL + 1) {
+      return false;
+    }
+    blCount[l]++;
+  }
+  for (minLength = 1; minLength <= MAX_BL + 1; ++minLength) {
+    if (blCount[minLength]) {
+      break;
+    }
+  }
+  for (maxLength = MAX_BL + 1; maxLength >= minLength; --maxLength) {
+    if (blCount[maxLength]) {
+      break;
+    }
+  }
+  if (minLength > maxLength) {
+    return false;
+  }
+  // Remove deleted symbols
+  blCount[0] = 0;
+  if (disableZeroBitSymbols) {
+    blCount[1] = 0;
+  }
+
+  // Calculate start codes
+  unsigned code = 0;
+  for (unsigned i = minLength; i <= maxLength; ++i) {
+    code = (code + blCount[i - 1]) << 1;
+    nextCode[i] = code;
+  }
+
+  if (minLength == maxLength && blCount[maxLength] == 1) {
+    return true;
+  }
+
+  // Check that we don't have holes
+  return nextCode[maxLength] + blCount[maxLength] == (unsigned)(1 << (maxLength - 1));
+}

contrib/preflate/support/huffman_helper.h

@@ -0,0 +1,34 @@
+/* Copyright 2018 Dirk Steinke
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License. */
+
+#ifndef HUFFMAN_HELPER_H
+#define HUFFMAN_HELPER_H
+
+#include <vector>
+
+// Huffman decoder
+class HuffmanHelper {
+public:
+  enum {
+    MAX_BL = 25
+  };
+  static bool countSymbols(unsigned(&nextCode)[MAX_BL + 2],
+                           unsigned char& minLength,
+                           unsigned char& maxLength,
+                           const unsigned char* symbolBitLengths,
+                           const unsigned symbolCount,
+                           const bool disableZeroBitSymbols);
+};
+
+#endif /* HUFFMAN_HELPER_H */

contrib/preflate/support/memstream.cpp

@@ -0,0 +1,57 @@
+/* Copyright 2018 Dirk Steinke
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License. */
+
+#include "pch.h"
+#include <algorithm>
+#include <string.h>
+#include "memstream.h"
+
+MemStream::MemStream() : _pos(0) {}
+MemStream::MemStream(const std::vector<uint8_t>& content)
+  : _data(content)
+  , _pos(0) {}
+MemStream::MemStream(const std::vector<uint8_t>& content, const size_t off, const size_t sz)
+  : _data(max(min(content.size(), off + sz), off) - off)
+  , _pos(0) {
+  memcpy(_data.data(), content.data() + off, _data.size());
+}
+
+bool MemStream::eof() const {
+  return _pos == _data.size();
+}
+size_t MemStream::read(unsigned char* buffer, const size_t size) {
+  size_t toCopy = min(size, _data.size() - _pos);
+  memcpy(buffer, _data.data() + _pos, toCopy);
+  _pos += toCopy;
+  return toCopy;
+}
+
+size_t MemStream::write(const unsigned char* buffer, const size_t size) {
+  size_t remaining = _data.size() - _pos;
+  if (size > remaining) {
+    _data.resize(_pos + size);
+  }
+  memcpy(_data.data() + _pos, buffer, size);
+  _pos += size;
+  return size;
+}
+
+uint64_t MemStream::tell() const {
+  return _pos;
+}
+uint64_t MemStream::seek(const uint64_t newPos) {
+  size_t oldPos = _pos;
+  _pos = min(newPos, (uint64_t)_data.size());
+  return oldPos;
+}

contrib/preflate/support/memstream.h

@@ -0,0 +1,52 @@
+/* Copyright 2018 Dirk Steinke
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License. */
+
+#ifndef MEMSTREAM_H
+#define MEMSTREAM_H
+
+#include <stdint.h>
+#include <vector>
+#include "stream.h"
+
+class MemStream : public SeekableInputOutputStream {
+public:
+  MemStream();
+  MemStream(const std::vector<uint8_t>& content);
+  MemStream(const std::vector<uint8_t>& content, const size_t off, const size_t sz);
+
+  virtual bool eof() const;
+  virtual size_t read(unsigned char* buffer, const size_t size);
+
+  virtual size_t write(const unsigned char* buffer, const size_t size);
+
+  virtual uint64_t tell() const;
+  virtual uint64_t seek(const uint64_t newPos);
+
+  void replaceData(const std::vector<uint8_t>& content) {
+    _data = content;
+  }
+
+  const std::vector<uint8_t>& data() const {
+    return _data;
+  }
+  std::vector<uint8_t> extractData() {
+    return std::move(_data);
+  }
+
+private:
+  std::vector<uint8_t> _data;
+  size_t _pos;
+};
+
+#endif /* MEMSTREAM_H */

contrib/preflate/support/outputcachestream.cpp

@@ -0,0 +1,30 @@
+/* Copyright 2018 Dirk Steinke
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License. */
+
+#include "pch.h"
+#include <algorithm>
+#include "outputcachestream.h"
+
+OutputCacheStream::OutputCacheStream(OutputStream& os)
+  : _os(os)
+  , _cacheStartPos(0) {}
+OutputCacheStream::~OutputCacheStream() {
+}
+
+void OutputCacheStream::flushUpTo(const uint64_t newStartPos) {
+  size_t toWrite = min(newStartPos - _cacheStartPos, (uint64_t)_cache.size());
+  size_t written = _os.write(_cache.data(), toWrite);
+  _cacheStartPos += written;
+  _cache.erase(_cache.begin(), _cache.begin() + written);
+}

contrib/preflate/support/outputcachestream.h

@@ -0,0 +1,67 @@
+/* Copyright 2018 Dirk Steinke
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License. */
+
+#ifndef OUTPUTCACHESTREAM_H
+#define OUTPUTCACHESTREAM_H
+
+#include <algorithm>
+#include <vector>
+#include "stream.h"
+
+class OutputCacheStream : public OutputStream {
+public:
+  OutputCacheStream(OutputStream& os);
+  virtual ~OutputCacheStream();
+
+  size_t write(const unsigned char* buffer, const size_t size) {
+/*    if (size == 1) {
+      _cache.push_back(*buffer);
+      return 1;
+    }*/
+    _cache.insert(_cache.end(), buffer, buffer + size);
+    return size;
+  }
+  void reserve(const size_t len) {
+    size_t cap = _cache.capacity();
+    if (_cache.size() + len > cap) {
+      _cache.reserve(cap + max(cap >> 1, len));
+    }
+  }
+  void flush() {
+    flushUpTo(cacheEndPos());
+  }
+  void flushUpTo(const uint64_t newStartPos);
+  uint64_t cacheStartPos() const {
+    return _cacheStartPos;
+  }
+  uint64_t cacheEndPos() const {
+    return _cacheStartPos + _cache.size();
+  }
+  const unsigned char* cacheData(const uint64_t pos) const {
+    return _cache.data() + (ptrdiff_t)(pos - _cacheStartPos);
+  }
+  const unsigned char* cacheEnd() const {
+    return _cache.data() + _cache.size();
+  }
+  const size_t cacheSize() const {
+    return _cache.size();
+  }
+
+private:
+  OutputStream& _os;
+  std::vector<unsigned char> _cache;
+  uint64_t _cacheStartPos;
+};
+
+#endif /* OUTPUTCACHESTREAM_H */

contrib/preflate/support/stream.h

@@ -0,0 +1,51 @@
+/* Copyright 2018 Dirk Steinke
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License. */
+
+#ifndef STREAM_H
+#define STREAM_H
+
+#include <stdint.h>
+
+class InputStream {
+public:
+  virtual ~InputStream() {}
+
+  virtual bool eof() const = 0;
+  virtual size_t read(unsigned char* buffer, const size_t size) = 0;
+};
+
+class OutputStream {
+public:
+  virtual ~OutputStream() {}
+  
+  virtual size_t write(const unsigned char* buffer, const size_t size) = 0;
+};
+
+class SeekableStream {
+public:
+  virtual ~SeekableStream() {}
+
+  virtual uint64_t tell() const = 0;
+  virtual uint64_t seek(const uint64_t newPos) = 0;
+};
+
+class SeekableInputStream 
+  : public InputStream
+  , public SeekableStream {};
+class SeekableInputOutputStream 
+  : public SeekableInputStream
+  , public OutputStream {};
+
+
+#endif /* STREAM_H */

contrib/preflate/support/support_tests.cpp

@@ -0,0 +1,181 @@
+/* Copyright 2018 Dirk Steinke
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License. */
+
+#include "pch.h"
+#include <stdio.h>
+#include "array_helper.h"
+#include "bit_helper.h"
+#include "bitstream.h"
+#include "const_division.h"
+#include "huffman_decoder.h"
+#include "huffman_encoder.h"
+#include "huffman_helper.h"
+#include "memstream.h"
+#include "outputcachestream.h"
+#include "stream.h"
+
+bool support_self_tests() {
+  unsigned arr[] = {1,2,3,4,5};
+  if (sumArray(arr) != 15
+      || sumArray(arr, sizeof(arr) / sizeof(arr[0])) != 15) {
+    printf("sumArray failed\n");
+    return false;
+  }
+  if (bitLength(0) != 0 
+      || bitLength(15) != 4
+      || bitLength(0xffffffff) != 32) {
+    printf("bitLength failed\n");
+    return false;
+  }
+  if (bitReverse(1, 3) != 4
+      || bitReverse(0x12345678, 32) != 0x1e6a2c48
+      || bitReverse(0xfedcba90, 32) != 0x095d3b7f) {
+    printf("bitReverse failed\n");
+    return false;
+  }
+
+  MemStream mem;
+  mem.write((const uint8_t*)"Hello", 5);
+  if (mem.tell() != 5 || !mem.eof()) {
+    printf("MemStream/1 failed\n");
+    return false;
+  }
+  mem.write((const uint8_t*)"!", 1);
+  uint8_t tmp[5], tmp2[2];
+  if (mem.read(tmp, 5) != 0) {
+    printf("MemStream/2 failed\n");
+    return false;
+  }
+  if (mem.seek(0) != 6) {
+    printf("MemStream/3 failed\n");
+    return false;
+  }
+  if (mem.tell() != 0) {
+    printf("MemStream/4 failed\n");
+    return false;
+  }
+  if (mem.read(tmp, 5) != 5 || tmp[0] != 'H' || tmp[4] != 'o') {
+    printf("MemStream/5 failed\n");
+    return false;
+  }
+  if (mem.read(tmp2, 2) != 1 || tmp2[0] != '!') {
+    printf("MemStream/6 failed\n");
+    return false;
+  }
+  if (!mem.eof()) {
+    printf("MemStream/7 failed\n");
+    return false;
+  }
+
+  mem.seek(0);
+  {
+    BitOutputStream bos(mem);
+    for (unsigned i = 0; i <= HuffmanHelper::MAX_BL; ++i) {
+      bos.put(i, i);
+    }
+    bos.flush();
+  }
+  mem.seek(0);
+  {
+    BitInputStream bis(mem);
+    for (unsigned i = 0; i <= HuffmanHelper::MAX_BL; ++i) {
+      if (bis.get(i) != i) {
+       printf("BitStreams failed\n");
+       return false;
+      }
+    }
+  }
+
+  unsigned char lengths[] = {
+    1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,
+    17,18,19,20,21,22,23,24,25,25
+  };
+  unsigned count = sizeof(lengths) / sizeof(lengths[0]);
+  HuffmanEncoder henc(lengths, count, false);
+  HuffmanDecoder hdec(lengths, count, false, 7);
+  if (henc.error() || hdec.error()) {
+    printf("HuffmanEncoder failed\n");
+    return false;
+  }
+  mem.seek(0);
+  {
+    BitOutputStream bos(mem);
+    for (unsigned i = 0; i < count; ++i) {
+      henc.encode(bos, i);
+    }
+    bos.flush();
+  }
+  mem.seek(0);
+  {
+    BitInputStream bis(mem);
+    for (unsigned i = 0; i < count; ++i) {
+      if (hdec.decode(bis) != i) {
+        printf("HuffmanDecoder failed\n");
+        return false;
+      }
+    }
+  }
+
+  uint16_t divtest16[] = {1, 3, 5, 7, 9, 11, 13, 17, 32767};
+  for (int i = 0, n = sizeof(divtest16) / sizeof(divtest16[0]); i < n; ++i) {
+    udivider_t<16> du   = build_udivider_16(divtest16[i]);
+    ucdivider_t<16> duc = build_ucdivider_16(divtest16[i]);
+    sdivider_t<16> ds = build_sdivider_16(divtest16[i]);
+    scdivider_t<16> dsc = build_scdivider_16(divtest16[i]);
+
+    for (int k = 0; k < 65536; ++k) {
+      uint16_t c1 = divide((uint16_t)k, du);
+      uint16_t c2 = divide((uint16_t)k, duc);
+      uint16_t r = k / divtest16[i];
+      if (c1 != r || c2 != r) {
+        printf("16bit divider/1 failed\n");
+        return false;
+      }
+
+      int16_t d1 = divide((int16_t)(k - 32768), ds);
+      int16_t d2 = divide((int16_t)(k - 32768), dsc);
+      int16_t s = ((int16_t)(k - 32768)) / (int16_t)divtest16[i];
+      if (d1 != s || d2 != s) {
+        printf("16bit divider/2 failed\n");
+        return false;
+      }
+    }
+  }
+  uint32_t divtest32[] = {1, 3, 5, 7, 9, 11, 13, 17, 0x7fff, 0x7fffffff};
+  for (int i = 0, n = sizeof(divtest32) / sizeof(divtest32[0]); i < n; ++i) {
+    udivider_t<32> du = build_udivider_32(divtest32[i]);
+    ucdivider_t<32> duc = build_ucdivider_32(divtest32[i]);
+    sdivider_t<32> ds = build_sdivider_32(divtest32[i]);
+    scdivider_t<32> dsc = build_scdivider_32(divtest32[i]);
+
+    for (int k = 0; k < 65536; ++k) {
+      uint32_t c1 = divide(((uint32_t)k)* 65536, du);
+      uint32_t c2 = divide(((uint32_t)k) * 65536, duc);
+      uint32_t r = (((uint32_t)k) * 65536) / divtest32[i];
+      if (c1 != r || c2 != r) {
+        printf("32bit divider/1 failed\n");
+        return false;
+      }
+
+      int32_t d1 = divide((int32_t)(k - 32768) * 65536, ds);
+      int32_t d2 = divide((int32_t)(k - 32768) * 65536, dsc);
+      int32_t s = ((int32_t)(k - 32768)) * 65536 / (int32_t)divtest32[i];
+      if (d1 != s || d2 != s) {
+        printf("32bit divider/2 failed\n");
+        return false;
+      }
+    }
+  }
+  return true;
+}

contrib/preflate/support/support_tests.h

@@ -0,0 +1,20 @@
+/* Copyright 2018 Dirk Steinke
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License. */
+
+#ifndef SUPPORT_TESTS_H
+#define SUPPORT_TESTS_H
+
+bool support_self_tests();
+
+#endif /* SUPPORT_TESTS_H */

contrib/preflate/support/task_pool.cpp

@@ -0,0 +1,58 @@
+/* Copyright 2018 Dirk Steinke
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License. */
+#include "pch.h"
+#include "task_pool.h"
+
+#include <memory>
+
+/*TaskPool globalTaskPool;
+
+TaskPool::TaskPool()
+  : _state(INIT)
+  , _threadLimit(max(1u, std::thread::hardware_concurrency()) - 1) {
+}
+
+void TaskPool::_init() {
+  _state = RUN;
+  std::function<void(void)> workerLoop = [this] {
+    for (;;) {
+      std::function<void()> task;
+
+      {
+        std::unique_lock<std::mutex> lock(this->_mutex);
+        this->_condition.wait(lock,
+                             [this] { return this->_state == FINISH || !this->_tasks.empty(); });
+        if (this->_state == FINISH) {
+          return;
+        }
+        task = std::move(this->_tasks.front());
+        this->_tasks.pop();
+      }
+      task();
+    }
+  };
+  for (unsigned i = 0, n = max((size_t)1, _threadLimit); i < n; ++i) {
+    _workers.emplace_back(workerLoop);
+  }
+}
+
+TaskPool::~TaskPool() {
+  _state = FINISH;
+  _condition.notify_all();
+  for (auto& thr : _workers) {
+    if (thr.joinable()) {
+      thr.join();
+    }
+  }
+} */

contrib/preflate/support/task_pool.h

@@ -0,0 +1,70 @@
+/* Copyright 2018 Dirk Steinke
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License. */
+
+// #ifndef TASK_POOL_H
+// #define TASK_POOL_H
+
+#include <condition_variable>
+#include <functional>
+#include <future>
+#include <memory>
+#include <mutex>
+#include <queue>
+#include <thread>
+#include <vector>
+
+ /* class TaskPool {
+public:
+  TaskPool();
+  ~TaskPool();
+
+  template<class F, class... Args>
+  auto addTask(F&& f, Args&&... args)
+    -> std::future<typename std::result_of<F(Args...)>::type> {
+    using R = typename std::result_of<F(Args...)>::type;
+    auto task = std::make_shared<std::packaged_task<R()>>(
+      std::bind(std::forward<F>(f), std::forward<Args>(args)...));
+
+    if (_state == INIT) {
+      _init();
+    }
+    std::future<R> res = task->get_future();
+    {
+      std::unique_lock<std::mutex> lock(_mutex);
+      _tasks.emplace([task]() { (*task)(); });
+    }
+    _condition.notify_one();
+    return res;
+  }
+
+  size_t extraThreadCount() const {
+    return _threadLimit;
+  }
+
+private:
+  enum State { INIT, RUN, FINISH };
+
+  void _init();
+
+  State _state;
+  size_t _threadLimit;
+  std::vector<std::thread> _workers;
+  std::mutex _mutex;
+  std::condition_variable _condition;
+  std::queue<std::function<void()>> _tasks;
+};
+
+extern TaskPool globalTaskPool;
+
+#endif */ /* TASK_POOL_H */