internal/mem/buffer_pool.go

/*
 *
 * Copyright 2026 gRPC authors.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 */

// Package mem provides utilities that facilitate memory reuse in byte slices
// that are used as buffers.
package mem

import (
	"fmt"
	"math/bits"
	"slices"
	"sort"
	"sync"

	"google.golang.org/grpc/internal"
)

const (
	goPageSize = 4 * 1024 // 4KiB. N.B. this must be a power of 2.
)

var (
	// BufferPoolingThreshold is the minimum size of a buffer that can be pooled.
	// This is used to determine whether to pool buffers or allocate them directly.
	BufferPoolingThreshold = 1 << 10
)

func init() {
	internal.SetBufferPoolingThresholdForTesting = func(threshold int) {
		BufferPoolingThreshold = threshold
	}
}

var uintSize = bits.UintSize // use a variable for mocking during tests.

// bufferPool is a copy of the public bufferPool interface used to avoid
// circular dependencies.
type bufferPool interface {
	// Get returns a buffer with specified length from the pool.
	Get(length int) *[]byte

	// Put returns a buffer to the pool.
	//
	// The provided pointer must hold a prefix of the buffer obtained via
	// BufferPool.Get to ensure the buffer's entire capacity can be re-used.
	Put(*[]byte)
}

// BinaryTieredBufferPool is a buffer pool that uses multiple sub-pools with
// power-of-two sizes.
type BinaryTieredBufferPool struct {
	// exponentToNextLargestPoolMap maps a power-of-two exponent (e.g., 12 for
	// 4KB) to the index of the next largest sizedBufferPool. This is used by
	// Get() to find the smallest pool that can satisfy a request for a given
	// size.
	exponentToNextLargestPoolMap []int
	// exponentToPreviousLargestPoolMap maps a power-of-two exponent to the
	// index of the previous largest sizedBufferPool. This is used by Put()
	// to return a buffer to the most appropriate pool based on its capacity.
	exponentToPreviousLargestPoolMap []int
	sizedPools                       []bufferPool
	fallbackPool                     bufferPool
	maxPoolCap                       int // Optimization: Cache max capacity
}

// NewBinaryTieredBufferPool returns a BufferPool backed by multiple sub-pools.
// This structure enables O(1) lookup time for Get and Put operations.
//
// The arguments provided are the exponents for the buffer capacities (powers
// of 2), not the raw byte sizes. For example, to create a pool of 16KB buffers
// (2^14 bytes), pass 14 as the argument.
func NewBinaryTieredBufferPool(powerOfTwoExponents ...uint8) (*BinaryTieredBufferPool, error) {
	return newBinaryTiered(func(size int) bufferPool {
		return newSizedBufferPool(size, true)
	}, &SimpleBufferPool{shouldZero: true}, powerOfTwoExponents...)
}

// NewDirtyBinaryTieredBufferPool returns a BufferPool backed by multiple
// sub-pools. It is similar to NewBinaryTieredBufferPool but it does not
// initialize the buffers before returning them.
func NewDirtyBinaryTieredBufferPool(powerOfTwoExponents ...uint8) (*BinaryTieredBufferPool, error) {
	return newBinaryTiered(func(size int) bufferPool {
		return newSizedBufferPool(size, false)
	}, NewDirtySimplePool(), powerOfTwoExponents...)
}

func newBinaryTiered(sizedPoolFactory func(int) bufferPool, fallbackPool bufferPool, powerOfTwoExponents ...uint8) (*BinaryTieredBufferPool, error) {
	slices.Sort(powerOfTwoExponents)
	powerOfTwoExponents = slices.Compact(powerOfTwoExponents)

	// Determine the maximum exponent we need to support. This depends on the
	// word size (32-bit vs 64-bit).
	maxExponent := uintSize - 2
	indexOfNextLargestBit := slices.Repeat([]int{-1}, maxExponent+1)
	indexOfPreviousLargestBit := slices.Repeat([]int{-1}, maxExponent+1)

	maxTier := 0
	pools := make([]bufferPool, 0, len(powerOfTwoExponents))

	for i, exp := range powerOfTwoExponents {
		// Allocating slices of size > 2^maxExponent isn't possible on
		// maxExponent-bit machines.
		if int(exp) > maxExponent {
			return nil, fmt.Errorf("mem: allocating slice of size 2^%d is not possible", exp)
		}
		tierSize := 1 << exp
		pools = append(pools, sizedPoolFactory(tierSize))
		maxTier = max(maxTier, tierSize)

		// Map the exact power of 2 to this pool index.
		indexOfNextLargestBit[exp] = i
		indexOfPreviousLargestBit[exp] = i
	}

	// Fill gaps for Get() (Next Largest)
	// We iterate backwards. If current is empty, take the value from the right (larger).
	for i := maxExponent - 1; i >= 0; i-- {
		if indexOfNextLargestBit[i] == -1 {
			indexOfNextLargestBit[i] = indexOfNextLargestBit[i+1]
		}
	}

	// Fill gaps for Put() (Previous Largest)
	// We iterate forwards. If current is empty, take the value from the left (smaller).
	for i := 1; i <= maxExponent; i++ {
		if indexOfPreviousLargestBit[i] == -1 {
			indexOfPreviousLargestBit[i] = indexOfPreviousLargestBit[i-1]
		}
	}

	return &BinaryTieredBufferPool{
		exponentToNextLargestPoolMap:     indexOfNextLargestBit,
		exponentToPreviousLargestPoolMap: indexOfPreviousLargestBit,
		sizedPools:                       pools,
		maxPoolCap:                       maxTier,
		fallbackPool:                     fallbackPool,
	}, nil
}

// Get returns a buffer with specified length from the pool.
func (b *BinaryTieredBufferPool) Get(size int) *[]byte {
	return b.poolForGet(size).Get(size)
}

func (b *BinaryTieredBufferPool) poolForGet(size int) bufferPool {
	if size == 0 || size > b.maxPoolCap {
		return b.fallbackPool
	}

	// Calculate the exponent of the smallest power of 2 >= size.
	// We subtract 1 from size to handle exact powers of 2 correctly.
	//
	// Examples:
	// size=16 (0b10000) -> size-1=15 (0b01111) -> bits.Len=4 -> Pool for 2^4
	// size=17 (0b10001) -> size-1=16 (0b10000) -> bits.Len=5 -> Pool for 2^5
	querySize := uint(size - 1)
	poolIdx := b.exponentToNextLargestPoolMap[bits.Len(querySize)]

	return b.sizedPools[poolIdx]
}

// Put returns a buffer to the pool.
func (b *BinaryTieredBufferPool) Put(buf *[]byte) {
	// We pass the capacity of the buffer, and not the size of the buffer here.
	// If we did the latter, all buffers would eventually move to the smallest
	// pool.
	b.poolForPut(cap(*buf)).Put(buf)
}

func (b *BinaryTieredBufferPool) poolForPut(bCap int) bufferPool {
	if bCap == 0 {
		return NopBufferPool{}
	}
	if bCap > b.maxPoolCap {
		return b.fallbackPool
	}
	// Find the pool with the largest capacity <= bCap.
	//
	// We calculate the exponent of the largest power of 2 <= bCap.
	// bits.Len(x) returns the minimum number of bits required to represent x;
	// i.e. the number of bits up to and including the most significant bit.
	// Subtracting 1 gives the 0-based index of the most significant bit,
	// which is the exponent of the largest power of 2 <= bCap.
	//
	// Examples:
	// cap=16 (0b10000) -> Len=5 -> 5-1=4 -> 2^4
	// cap=15 (0b01111) -> Len=4 -> 4-1=3 -> 2^3
	largestPowerOfTwo := bits.Len(uint(bCap)) - 1
	poolIdx := b.exponentToPreviousLargestPoolMap[largestPowerOfTwo]
	// The buffer is smaller than the smallest power of 2, discard it.
	if poolIdx == -1 {
		// Buffer is smaller than our smallest pool bucket.
		return NopBufferPool{}
	}
	return b.sizedPools[poolIdx]
}

// NopBufferPool is a buffer pool that returns new buffers without pooling.
type NopBufferPool struct{}

// Get returns a buffer with specified length from the pool.
func (NopBufferPool) Get(length int) *[]byte {
	b := make([]byte, length)
	return &b
}

// Put returns a buffer to the pool.
func (NopBufferPool) Put(*[]byte) {
}

// sizedBufferPool is a BufferPool implementation that is optimized for specific
// buffer sizes. For example, HTTP/2 frames within gRPC have a default max size
// of 16kb and a sizedBufferPool can be configured to only return buffers with a
// capacity of 16kb. Note that however it does not support returning larger
// buffers and in fact panics if such a buffer is requested. Because of this,
// this BufferPool implementation is not meant to be used on its own and rather
// is intended to be embedded in a TieredBufferPool such that Get is only
// invoked when the required size is smaller than or equal to defaultSize.
type sizedBufferPool struct {
	pool        sync.Pool
	defaultSize int
	shouldZero  bool
}

func (p *sizedBufferPool) Get(size int) *[]byte {
	buf, ok := p.pool.Get().(*[]byte)
	if !ok {
		buf := make([]byte, size, p.defaultSize)
		return &buf
	}
	b := *buf
	if p.shouldZero {
		clear(b[:cap(b)])
	}
	*buf = b[:size]
	return buf
}

func (p *sizedBufferPool) Put(buf *[]byte) {
	if cap(*buf) < p.defaultSize {
		// Ignore buffers that are too small to fit in the pool. Otherwise, when
		// Get is called it will panic as it tries to index outside the bounds
		// of the buffer.
		return
	}
	p.pool.Put(buf)
}

func newSizedBufferPool(size int, zero bool) *sizedBufferPool {
	return &sizedBufferPool{
		defaultSize: size,
		shouldZero:  zero,
	}
}

// TieredBufferPool implements the BufferPool interface with multiple tiers of
// buffer pools for different sizes of buffers.
type TieredBufferPool struct {
	sizedPools   []*sizedBufferPool
	fallbackPool SimpleBufferPool
}

// NewTieredBufferPool returns a BufferPool implementation that uses multiple
// underlying pools of the given pool sizes.
func NewTieredBufferPool(poolSizes ...int) *TieredBufferPool {
	sort.Ints(poolSizes)
	pools := make([]*sizedBufferPool, len(poolSizes))
	for i, s := range poolSizes {
		pools[i] = newSizedBufferPool(s, true)
	}
	return &TieredBufferPool{
		sizedPools:   pools,
		fallbackPool: SimpleBufferPool{shouldZero: true},
	}
}

// Get returns a buffer with specified length from the pool.
func (p *TieredBufferPool) Get(size int) *[]byte {
	return p.getPool(size).Get(size)
}

// Put returns a buffer to the pool.
func (p *TieredBufferPool) Put(buf *[]byte) {
	p.getPool(cap(*buf)).Put(buf)
}

func (p *TieredBufferPool) getPool(size int) bufferPool {
	poolIdx := sort.Search(len(p.sizedPools), func(i int) bool {
		return p.sizedPools[i].defaultSize >= size
	})

	if poolIdx == len(p.sizedPools) {
		return &p.fallbackPool
	}

	return p.sizedPools[poolIdx]
}

// SimpleBufferPool is an implementation of the mem.BufferPool interface that
// attempts to pool buffers with a sync.Pool. When Get is invoked, it tries to
// acquire a buffer from the pool but if that buffer is too small, it returns it
// to the pool and creates a new one.
type SimpleBufferPool struct {
	pool       sync.Pool
	shouldZero bool
}

// NewDirtySimplePool constructs a [SimpleBufferPool]. It does not initialize
// the buffers before returning them. Callers must ensure they don't read the
// buffers before writing data to them.
func NewDirtySimplePool() *SimpleBufferPool {
	return &SimpleBufferPool{
		shouldZero: false,
	}
}

// Get returns a buffer with specified length from the pool.
func (p *SimpleBufferPool) Get(size int) *[]byte {
	bs, ok := p.pool.Get().(*[]byte)
	if ok && cap(*bs) >= size {
		if p.shouldZero {
			clear((*bs)[:cap(*bs)])
		}
		*bs = (*bs)[:size]
		return bs
	}

	// A buffer was pulled from the pool, but it is too small. Put it back in
	// the pool and create one large enough.
	if ok {
		p.pool.Put(bs)
	}

	// If we're going to allocate, round up to the nearest page. This way if
	// requests frequently arrive with small variation we don't allocate
	// repeatedly if we get unlucky and they increase over time. By default we
	// only allocate here if size > 1MiB. Because goPageSize is a power of 2, we
	// can round up efficiently.
	allocSize := (size + goPageSize - 1) & ^(goPageSize - 1)

	b := make([]byte, size, allocSize)
	return &b
}

// Put returns a buffer to the pool.
func (p *SimpleBufferPool) Put(buf *[]byte) {
	p.pool.Put(buf)
}