This is an automated email from the ASF dual-hosted git repository. mmerli pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/pulsar-client-go.git
The following commit(s) were added to refs/heads/master by this push: new a0d96c8 Updated zstd go library to avoid cgo dependency for compression (#102) a0d96c8 is described below commit a0d96c89ad5dc2091f5e28c4386f73ebddf99a4a Author: Matteo Merli <mme...@apache.org> AuthorDate: Mon Nov 18 14:26:20 2019 -0800 Updated zstd go library to avoid cgo dependency for compression (#102) --- go.mod | 6 +--- go.sum | 15 ++-------- pkg/compression/zstd.go | 45 +++++++++++++++++------------ pkg/compression/zstd_native.go | 64 ------------------------------------------ 4 files changed, 30 insertions(+), 100 deletions(-) diff --git a/go.mod b/go.mod index dddb428..339748e 100644 --- a/go.mod +++ b/go.mod @@ -3,16 +3,12 @@ module github.com/apache/pulsar-client-go go 1.12 require ( - github.com/DataDog/zstd v1.4.0 // indirect github.com/beefsack/go-rate v0.0.0-20180408011153-efa7637bb9b6 github.com/bmizerany/perks v0.0.0-20141205001514-d9a9656a3a4b - github.com/cespare/xxhash v1.1.0 // indirect - github.com/deckarep/golang-set v1.7.1 github.com/golang/protobuf v1.3.1 github.com/google/go-cmp v0.3.0 // indirect github.com/inconshreveable/mousetrap v1.0.0 // indirect - github.com/klauspost/compress v1.6.2 - github.com/klauspost/cpuid v1.2.1 // indirect + github.com/klauspost/compress v1.9.2 github.com/pierrec/lz4 v2.0.5+incompatible github.com/pkg/errors v0.8.1 github.com/sirupsen/logrus v1.4.1 diff --git a/go.sum b/go.sum index a1ccf2f..c3fdcf6 100644 --- a/go.sum +++ b/go.sum @@ -1,28 +1,18 @@ -github.com/DataDog/zstd v1.4.0 h1:vhoV+DUHnRZdKW1i5UMjAk2G4JY8wN4ayRfYDNdEhwo= -github.com/DataDog/zstd v1.4.0/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= -github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= -github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/beefsack/go-rate v0.0.0-20180408011153-efa7637bb9b6 h1:KXlsf+qt/X5ttPGEjR0tPH1xaWWoKBEg9Q1THAj2h3I= github.com/beefsack/go-rate v0.0.0-20180408011153-efa7637bb9b6/go.mod h1:6YNgTHLutezwnBvyneBbwvB8C82y3dcoOj5EQJIdGXA= github.com/bmizerany/perks v0.0.0-20141205001514-d9a9656a3a4b h1:AP/Y7sqYicnjGDfD5VcY4CIfh1hRXBUavxrvELjTiOE= github.com/bmizerany/perks v0.0.0-20141205001514-d9a9656a3a4b/go.mod h1:ac9efd0D1fsDb3EJvhqgXRbFx7bs2wqZ10HQPeU8U/Q= -github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= -github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/deckarep/golang-set v1.7.1 h1:SCQV0S6gTtp6itiFrTqI+pfmJ4LN85S1YzhDf9rTHJQ= -github.com/deckarep/golang-set v1.7.1/go.mod h1:93vsz/8Wt4joVM7c2AVqh+YRMiUSc14yDtF28KmMOgQ= github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= -github.com/klauspost/compress v1.6.2 h1:D9kM6nOc1x+yA/DW/k81uG1xdmwqCMQ/A266P1edQEw= -github.com/klauspost/compress v1.6.2/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= -github.com/klauspost/cpuid v1.2.1 h1:vJi+O/nMdFt0vqm8NZBI6wzALWdA2X+egi0ogNyrC/w= -github.com/klauspost/cpuid v1.2.1/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= +github.com/klauspost/compress v1.9.2 h1:LfVyl+ZlLlLDeQ/d2AqfGIIH4qEDu0Ed2S5GyhCWIWY= +github.com/klauspost/compress v1.9.2/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/pierrec/lz4 v2.0.5+incompatible h1:2xWsjqPFWcplujydGg4WmhC/6fZqK42wMM8aXeqhl0I= @@ -33,7 +23,6 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/sirupsen/logrus v1.4.1 h1:GL2rEmy6nsikmW0r8opw9JIRScdMF5hA8cOYLH7In1k= github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= -github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/cobra v0.0.3 h1:ZlrZ4XsMRm04Fr5pSFxBgfND2EBVa1nLpiy1stUsX/8= diff --git a/pkg/compression/zstd.go b/pkg/compression/zstd.go index 529020c..b385c34 100644 --- a/pkg/compression/zstd.go +++ b/pkg/compression/zstd.go @@ -15,36 +15,45 @@ // specific language governing permissions and limitations // under the License. -// +build cgo - -// If CGO is enabled, use ZSTD library that links with official -// C based zstd. This can perform both compression and decompression. - package compression import ( - zstd "github.com/valyala/gozstd" + "bytes" + "github.com/klauspost/compress/zstd" + "github.com/pkg/errors" ) -// NewZStdProvider returns a Provider interface. -func NewZStdProvider() Provider { - return newCGoZStdProvider() +type zstdProvider struct { + encoder *zstd.Encoder } -type zstdProvider struct{} - -func newCGoZStdProvider() Provider { - return &zstdProvider{} +func NewZStdProvider() Provider { + p := &zstdProvider{} + p.encoder, _ = zstd.NewWriter(nil) + return p } -func (zstdProvider) CanCompress() bool { +func (p *zstdProvider) CanCompress() bool { return true } -func (zstdProvider) Compress(data []byte) []byte { - return zstd.Compress(nil, data) +func (p *zstdProvider) Compress(data []byte) []byte { + return p.encoder.EncodeAll(data, []byte{}) } -func (zstdProvider) Decompress(compressedData []byte, originalSize int) ([]byte, error) { - return zstd.Decompress(nil, compressedData) +func (p* zstdProvider) Decompress(compressedData []byte, originalSize int) ([]byte, error) { + d, err := zstd.NewReader(bytes.NewReader(compressedData)) + if err != nil { + return nil, err + } + + uncompressed := make([]byte, originalSize) + size, err := d.Read(uncompressed) + if err != nil { + return nil, err + } else if size != originalSize { + return nil, errors.New("Invalid uncompressed size") + } else { + return uncompressed, nil + } } diff --git a/pkg/compression/zstd_native.go b/pkg/compression/zstd_native.go deleted file mode 100644 index fa3649e..0000000 --- a/pkg/compression/zstd_native.go +++ /dev/null @@ -1,64 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -// +build !cgo - -// Pure GO ZStd library only supports decompression - -package compression - -import ( - "bytes" - "github.com/klauspost/compress/zstd" - "github.com/pkg/errors" - log "github.com/sirupsen/logrus" -) - -// Pure GO ZStd library only supports decompression - -type zstdProvider struct { -} - -func NewZStdProvider() Provider { - return &zstdProvider{} -} - -func (zstdProvider) CanCompress() bool { - return false -} - -func (zstdProvider) Compress(data []byte) []byte { - log.Panic("ZSTD compression is not supported if CGo is disabled") - return nil -} - -func (zstdProvider) Decompress(compressedData []byte, originalSize int) ([]byte, error) { - d, err := zstd.NewReader(bytes.NewReader(compressedData)) - if err != nil { - return nil, err - } - - uncompressed := make([]byte, originalSize) - size, err := d.Read(uncompressed) - if err != nil { - return nil, err - } else if size != originalSize { - return nil, errors.New("Invalid uncompressed size") - } else { - return uncompressed, nil - } -}