// Copyright 2009 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. package flate import ( "math" "math/bits" "slices" "sync" ) const ( maxBitsLimit = 16 // number of valid literals literalCount = 286 ) // hcode is a huffman code with a bit code and bit length. type hcode uint32 // len returns the length of the code in bits. func (h hcode) len() uint8 { return uint8(h) } // code64 returns the code as a uint64. func (h hcode) code64() uint64 { return uint64(h >> 8) } // zero returns true if the code is unset. func (h hcode) zero() bool { return h == 0 } // set sets the code and length of an hcode. func (h *hcode) set(code uint16, length uint8) { *h = newhcode(code, length) } // newhcode combines a code and length into an hcode. func newhcode(code uint16, length uint8) hcode { return hcode(length) | (hcode(code) << 8) } // huffmanEncoder provides a fast way to generate Huffman codes for a given // frequency table. It is based on the algorithm described in RFC 1951, // section 3.2.2. type huffmanEncoder struct { codes []hcode bitCount [17]int32 // freqcache is a reusable buffer with the longest possible frequency table. // Possible lengths are codegenCodeCount, offsetCodeCount and literalCount. // The largest of these is literalCount, so we allocate for that case. freqcache [literalCount + 1]literalNode } // newHuffmanEncoder returns a new huffmanEncoder with the given size. func newHuffmanEncoder(size int) *huffmanEncoder { // Make capacity to next power of two. c := uint(bits.Len32(uint32(size - 1))) return &huffmanEncoder{codes: make([]hcode, size, 1<= 3. // The cases of 0, 1, and 2 literals are handled by special case code. // // list is an array of the literals with non-zero frequencies // and their associated frequencies. The array is in order of increasing // frequency and has as its last element a special element with frequency // MaxInt32. // // maxBits is the maximum number of bits that should be used to encode any literal. // It must be less than 16. func (h *huffmanEncoder) bitCounts(list []literalNode, maxBits int32) []int32 { if maxBits >= maxBitsLimit { panic("flate: maxBits too large") } n := int32(len(list)) list = list[0 : n+1] list[n] = maxNode() // The tree can't have greater depth than n - 1, no matter what. This // saves a little bit of work in some small cases if maxBits > n-1 { maxBits = n - 1 } // Create information about each of the levels. // A bogus "Level 0" whose sole purpose is so that // level1.prev.needed==0. This makes level1.nextPairFreq // be a legitimate value that never gets chosen. var levels [maxBitsLimit]levelInfo // leafCounts[i] counts the number of literals at the left // of ancestors of the rightmost node at level i. // leafCounts[i][j] is the number of literals at the left // of the level j ancestor. var leafCounts [maxBitsLimit][maxBitsLimit]int32 _ = list[2] // check bounds here instead of in loop for level := int32(1); level <= maxBits; level++ { // For every level, the first two items are the first two characters. // We initialize the levels as if we had already figured this out. levels[level] = levelInfo{ level: level, lastFreq: int32(list[1].freq), nextCharFreq: int32(list[2].freq), nextPairFreq: int32(list[0].freq) + int32(list[1].freq), } leafCounts[level][level] = 2 if level == 1 { levels[level].nextPairFreq = math.MaxInt32 } } // We need a total of 2*n - 2 items at top level and have already generated 2. levels[maxBits].needed = 2*n - 4 level := uint32(maxBits) for level < 16 { l := &levels[level] if l.nextPairFreq == math.MaxInt32 && l.nextCharFreq == math.MaxInt32 { // We've run out of both leafs and pairs. // End all calculations for this level. // To make sure we never come back to this level or any lower level, // set nextPairFreq impossibly large. l.needed = 0 levels[level+1].nextPairFreq = math.MaxInt32 level++ continue } prevFreq := l.lastFreq if l.nextCharFreq < l.nextPairFreq { // The next item on this row is a leaf node. n := leafCounts[level][level] + 1 l.lastFreq = l.nextCharFreq // Lower leafCounts are the same of the previous node. leafCounts[level][level] = n e := list[n] if e.literal < math.MaxUint16 { l.nextCharFreq = int32(e.freq) } else { l.nextCharFreq = math.MaxInt32 } } else { // The next item on this row is a pair from the previous row. // nextPairFreq isn't valid until we generate two // more values in the level below l.lastFreq = l.nextPairFreq // Take leaf counts from the lower level, except counts[level] remains the same. save := leafCounts[level][level] leafCounts[level] = leafCounts[level-1] leafCounts[level][level] = save levels[l.level-1].needed = 2 } if l.needed--; l.needed == 0 { // We've done everything we need to do for this level. // Continue calculating one level up. Fill in nextPairFreq // of that level with the sum of the two nodes we've just calculated on // this level. if l.level == maxBits { // All done! break } levels[l.level+1].nextPairFreq = prevFreq + l.lastFreq level++ } else { // If we stole from below, move down temporarily to replenish it. for levels[level-1].needed > 0 { level-- } } } // Somethings is wrong if at the end, the top level is null or hasn't used // all of the leaves. if leafCounts[maxBits][maxBits] != n { panic("leafCounts[maxBits][maxBits] != n") } bitCount := h.bitCount[:maxBits+1] bits := 1 counts := &leafCounts[maxBits] for level := maxBits; level > 0; level-- { // chain.leafCount gives the number of literals requiring at least "bits" // bits to encode. bitCount[bits] = counts[level] - counts[level-1] bits++ } return bitCount } // assignEncodingAndSize assigns bit counts and encodings to the leaves // as specified in RFC 1951 3.2.2. func (h *huffmanEncoder) assignEncodingAndSize(bitCount []int32, list []literalNode) { code := uint16(0) for n, bits := range bitCount { code <<= 1 if n == 0 || bits == 0 { continue } // The literals list[len(list)-bits] .. list[len(list)-bits] // are encoded using "bits" bits, and get the values // code, code + 1, .... The code values are // assigned in literal order (not frequency order). chunk := list[len(list)-int(bits):] slices.SortFunc(chunk, func(a, b literalNode) int { return int(a.literal) - int(b.literal) }) for _, node := range chunk { h.codes[node.literal] = newhcode(reverseBits(code, uint8(n)), uint8(n)) code++ } list = list[0 : len(list)-int(bits)] } } // generate rewrites h to be the Huffman code for the given frequency count. // freq[i] is the frequency of literal i, and maxBits is the maximum number // of bits to use for any literal. func (h *huffmanEncoder) generate(freq []uint16, maxBits int32) { list := h.freqcache[:len(freq)+1] codes := h.codes[:len(freq)] // Number of non-zero literals count := 0 // Set list to be the set of all non-zero literals and their frequencies for i, f := range freq { if f != 0 { list[count] = literalNode{uint16(i), f} count++ } else { codes[i] = 0 } } list[count] = literalNode{} list = list[:count] if count <= 2 { // Handle the small cases here, because they are awkward for the general case code. With // two or fewer literals, everything has bit length 1. for i, node := range list { // "list" is in order of increasing literal value. h.codes[node.literal].set(uint16(i), 1) } return } slices.SortFunc(list, func(a, b literalNode) int { // Literals can be contained in 9 bits, so we shift freq to be branchless. return (int(a.freq)<<10 + int(a.literal)) - (int(b.freq)<<10 + int(b.literal)) }) // Get the number of literals for each bit count bitCount := h.bitCounts(list, maxBits) // And do the assignment h.assignEncodingAndSize(bitCount, list) } func histogram(b []byte, h []uint16) { if len(b) >= 8<<10 { histogramSplit(b, h) return } h = h[:256] for _, t := range b { h[t]++ } } func histogramSplit(b []byte, h []uint16) { // Walk four quarters in parallel. // Tested to be faster than walking halves. h = h[:256] // Make size divisible by 4 for len(b)&3 != 0 { h[b[0]]++ b = b[1:] } n := len(b) / 4 x, y, z, w := b[:n], b[n:], b[n+n:], b[n+n+n:] y, z, w = y[:len(x)], z[:len(x)], w[:len(x)] for i, t := range x { v0 := &h[t] v1 := &h[y[i]] v2 := &h[z[i]] v3 := &h[w[i]] *v0++ *v1++ *v2++ *v3++ } }