Skip to content

Instantly share code, notes, and snippets.

@regalius

regalius/compression.go

Created July 17, 2019 15:03
Show Gist options
  • Save regalius/956ee319d432a767b5fe7cf3596a779a to your computer and use it in GitHub Desktop.
Save regalius/956ee319d432a767b5fe7cf3596a779a to your computer and use it in GitHub Desktop.
Download ZIP
Comparing compression algorithm in Go
Raw
compression.go
package main
import (
"bytes"
"compress/gzip"
"compress/lzw"
b64 "encoding/base64"
"encoding/binary"
"encoding/json"
"fmt"
"io/ioutil"
"log"
"strconv"
"strings"
"time"
lz4 "github.com/bkaradzic/go-lz4"
)
type Performance struct {
EncodeTime time.Duration
DecodeTime time.Duration
EncodeChar string
Correct bool
}
var summary = map[string]Performance{}
var str = []int64{956382892, 959657555, 986192491, 930288616, 961265578, 977652190, 915927984, 911397602, 999653837, 997359124, 974714714, 933357047, 982147098, 991011501, 926257391, 913330609, 978481537, 900303559, 918441486, 931717899, 925334099, 961400250, 937338224, 901725021, 948626512, 912875369, 987364123, 943466645, 983099727, 931573065, 935398960, 981238448, 946106915, 979315854, 906720440, 909521980, 970789428, 978091034, 958352405, 901087918, 926223614, 947355086, 972821998, 907741120, 945801210, 924748474, 914191033, 978890209, 952012406, 985495877, 944697457, 903742345, 927476950, 904879541, 908907381, 915964805, 961236823, 954537779, 917414304, 964468093, 904040274, 959518821, 981531855, 962659643, 910656032, 926357417, 978753909, 942737690, 967878345, 971783335, 904200423, 952290929, 945612919, 988156420, 939100353, 955451241, 976816632, 977083564, 997470921, 924410424, 991646530, 940757440, 956999486, 918956085, 955845708, 911277183, 911960956, 945431042, 940907794, 910302234, 939021405, 926806250, 966049498, 984221626, 954969240, 918587334, 931770293, 926773599, 946502015, 924050539, 998513663, 993316821, 935359685, 934939106, 911284774, 967453200, 911211525, 914346299, 914266898, 980886994, 978578577, 906507061, 941519281, 906181697, 908366617, 984219088, 985548657, 915082674, 947682339, 927958522, 993255278, 994358823, 900503551, 966434201, 947757807, 939040831, 923094530, 909310066, 948983594, 929519314, 902800430, 943988932, 923113422, 960916820, 966772613, 931060823, 990060808, 976091483, 904250253, 937433894, 934150932, 949561800, 917361585, 952637511, 989874786, 926258348, 902493190, 924102996, 912447979, 965318988, 937712333, 944925439, 952427802, 948790935, 971830169, 979995341, 948287602, 994344639, 907553858, 933650936, 909980989, 921330258, 939970699, 914356308, 952295937, 993767503, 961830818, 945833290, 977625682, 907433967, 918078341, 988534900, 942569140, 927031826, 921408492, 964560449, 953127524, 979890608, 964883155, 950837747, 910268959, 941892226, 904209349, 903264557, 932998370, 931763578, 917778080, 966761864, 975256115, 950025527, 989995161, 919225226, 997184677, 984324112, 923963824, 975694830, 911718761, 978218544, 945908724, 974889069, 942063116, 953851385, 993295954, 918438970, 915764897, 905615733, 950589338, 925684892, 997422267, 949603952, 925733447, 902424667, 902942183, 911159371, 961798812, 913661624, 978683158, 990515988, 931757443, 945707692, 988032341, 955853442, 923550292, 991764538, 936811893, 924493305, 927259039, 954768053, 941875078, 922882197, 967626617, 964125604, 966189507, 912851011, 968542133, 909603793, 955434074, 955867945, 962694824, 972796431, 994235021, 907630136, 955987504, 964526169, 978844079, 904905927, 969081204, 993636443, 937460622, 972315197, 951062574, 916868575, 988448690, 949109082, 955186500, 974618019, 917078461, 996811140, 911643968, 920211485, 959423086, 903167010, 920785043, 963062813, 934398448, 947938502, 985133884, 945160020, 901495270, 963644915, 920735174, 940297881, 932994201, 962842414, 964510649, 919758416, 945182028, 922788622, 998200677, 991832029, 940995703, 947458618, 963105509, 951481134, 988053090, 943897304, 977210619, 907553913, 946831961, 978130860, 978169445, 924933669, 936913939, 997994928, 952384525, 924303461, 963492572, 912345824, 976744726, 990560326, 934734631, 915782465, 904391415, 978270572, 924479778, 924574238, 981545320, 909456311, 911555463, 955513892, 994801385, 973446454, 983980173, 993097093, 979808160, 956911278, 907323316, 986685916, 957182810, 943236706, 943890362, 930090799, 907533647, 956706234, 920035633, 965848647, 995165809, 941042386, 953868208, 917441748, 981573440, 913219772, 927920095, 911549145, 933333814, 967701091, 928055677, 918704642, 976228254, 907854061, 993235914, 975380991, 902505515, 976882546, 906063386, 961085472, 936050572, 915600105, 976139375, 922775530, 957398696, 951169655, 978968070, 971902455, 953570438, 973160670, 936962490, 929821118, 940974785, 967563089, 949095468, 974614469, 908073266, 957474983, 925386462, 933583126, 952172151, 914292855, 957352177, 960335496, 913187297, 938225789, 935445687, 977463509, 907430084, 930335199, 948221061, 938862062, 942292956, 992410601, 992404793, 912464461, 948176819, 909734747, 971111380, 957093042, 963255725, 992063264, 991808802, 998637705, 922067203, 983275476, 916595935, 902253548, 999398663, 973541496, 968089697, 909077596, 964144890, 995553684, 964422196, 907428499, 912238338, 940253837, 920488676, 978430935, 945932916, 989456735, 983775707, 996744463, 968474143, 978440409, 931200825, 980454504, 935985124, 951668929, 935094405, 968951390, 985529401, 956582514, 907841637, 904499649, 966236462, 938150201, 984326566, 959569625, 962681470, 941094772, 902317606, 941479734, 941240289, 969285401, 964562859, 905149410, 958617676, 941190570, 929406447, 922812479, 983902732, 907954870, 902614543, 968779824, 990964032, 926544283, 952055594, 957674288, 921526168, 960973257, 947986190, 935196197, 925475961, 907038179, 961128677, 906754722, 907742290, 922917029, 975387160, 917852375, 929190902, 921783488, 916987686, 911377959, 977659256, 957162636, 936711952, 913157486, 919709026, 940191486, 973194221, 934413074, 959623259, 931283612, 958871858, 926831480, 935440516, 993592227, 942968594, 933279603, 914397717, 961466328, 997368334, 922083216, 994502305, 992931691, 992268064, 967940832, 904559681, 949538634, 957347104, 982214789, 916544795, 954063937, 942368926, 970730496, 962303953, 907259579, 900428525, 984831521, 914823790, 968377835}
var buf = new(bytes.Buffer)
var gbuf = new(bytes.Buffer)
func main() {
runMe("lz4", encodeLZ4, decodeLZ4)
runMe("base64", encodeBase64, decodeBase64)
runMe("lzw", encodeLZW, decodeLZW)
runMe("base64-gzip", encodeBase64Gzip, decodeBase64Gzip)
for algo, perf := range summary {
log.Println("Using", algo)
// log.Println("Result", perf.EncodeChar)
log.Printf("\n%v | Encode Time: %v\n Decode Time: %v\n Length: %v\n Correctness: %v", algo, perf.EncodeTime, perf.DecodeTime, perf. ,perf.Correct)
}
}
func runMe(method string, encoder func([]int64) string, decoder func(string) []int64) {
encodeStartTime := time.Now()
encodeResult := encoder(str)
encodeEndTime := time.Now()
// log.Println("Using", method)
// log.Println(encodeResult)
encodeTime := encodeEndTime.Sub(encodeStartTime)
// log.Printf("Encode Took %v with character length %v from %v", encodeTime, len(encodeResult), len(str)*10)
// log.Printf("Encoded")
decodeStartTime := time.Now()
decodeResult := decoder(encodeResult)
decodeEndTime := time.Now()
decodeTime := decodeEndTime.Sub(decodeStartTime)
// log.Printf("Decoded")
// log.Println(decodeResult)
correct := false
if len(str) == len(decodeResult) {
correct = true
}
// log.Println("Decoded Result is", correct, len(str), len(decodeResult))
summary[method] = Performance{
EncodeTime: encodeTime,
DecodeTime: decodeTime,
EncodeChar: encodeResult,
Correct: correct,
}
}
func encodeLZW(str []int64) string {
lzwWriter := lzw.NewWriter(buf, lzw.LSB, 8)
a := arrToComma(str)
_, err := lzwWriter.Write([]byte(a))
if err != nil {
log.Fatal(err)
return ""
}
if err := lzwWriter.Close(); err != nil {
log.Fatal(err)
return ""
}
return buf.String()
}
func decodeLZW(str string) []int64 {
lzwReader := lzw.NewReader(buf, lzw.LSB, 8)
result, err := ioutil.ReadAll(lzwReader)
if err != nil {
log.Fatal(err)
return []int64{}
}
return commaToArr(string(result))
}
func encodeBase64(str []int64) string {
wbuf := new(bytes.Buffer)
err := binary.Write(wbuf, binary.LittleEndian, str)
if err != nil {
fmt.Println("binary.Write failed:", err)
}
result := b64.StdEncoding.EncodeToString(wbuf.Bytes())
return result
}
func decodeBase64(str string) []int64 {
result, err := b64.StdEncoding.DecodeString(str)
if err != nil {
log.Fatal(err)
return []int64{}
}
rbuf := bytes.NewBuffer(result)
r64 := make([]int64, 500)
err = binary.Read(rbuf, binary.LittleEndian, &r64)
if err != nil {
fmt.Println("binary.Read failed:", err)
}
return r64
}
func encodeBase64Gzip(str []int64) string {
wbuf := new(bytes.Buffer)
err := binary.Write(wbuf, binary.LittleEndian, str)
if err != nil {
fmt.Println("binary.Write failed:", err)
}
result := b64.StdEncoding.EncodeToString(wbuf.Bytes())
gzWriter := gzip.NewWriter(gbuf)
if _, err := gzWriter.Write([]byte(result)); err != nil {
log.Fatal(err)
}
if err := gzWriter.Flush(); err != nil {
log.Fatal(err)
}
if err := gzWriter.Close(); err != nil {
log.Fatal(err)
}
return string(gbuf.Bytes())
}
func decodeBase64Gzip(str string) []int64 {
rbuf := bytes.NewBuffer([]byte(str))
r, err := gzip.NewReader(rbuf)
if err != nil {
log.Fatal(err)
return []int64{}
}
s, err := ioutil.ReadAll(r)
if err != nil {
log.Fatal(err)
return []int64{}
}
result, err := b64.StdEncoding.DecodeString(string(s))
if err != nil {
log.Fatal(err)
return []int64{}
}
r64 := make([]int64, 500)
bbuf := bytes.NewBuffer(result)
err = binary.Read(bbuf, binary.LittleEndian, &r64)
if err != nil {
fmt.Println("binary.Read failed:", err)
}
return commaToArr(string(s))
}
func encodeLZ4(str []int64) string {
var dst []byte
a := arrToComma(str)
data, err := lz4.Encode(dst, []byte(a))
if err != nil {
log.Fatal(err)
return ""
}
return string(data)
}
func decodeLZ4(str string) []int64 {
var asd []byte
datb, err := lz4.Decode(asd, []byte(str))
if err != nil {
log.Fatal(err)
return []int64{}
}
return commaToArr(string(datb))
}
func arrToComma(str []int64) string {
a, _ := json.Marshal(str)
return string(a)
}
func commaToArr(str string) []int64 {
strs := strings.Split(str, ",")
ary := make([]int64, len(strs))
for i := range ary {
a, _ := strconv.Atoi(strs[i])
ary[i] = int64(a)
}
return ary
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment