capture nics (#1000)

1. move the `isDevice(l.host, pi)` to be first, as no need to iterate on all nics if it returns `true`
2. first compare by name, as same nics will have same names
3. if not found by name, compare by ips.

the bug was the `strings.HasPrefix`
2 different nics with ipv6:
```
#nic1 ip: f1234::55
#nic2 ip: f1234::55::66::66
```

so because of the `strings.HasPrefix` it was evaluated as the name nics. but they are not.

.deepsource.toml

@@ -0,0 +1,28 @@
+version = 1
+
+exclude_patterns = [
+  "vendor/**"
+]
+
+[[analyzers]]
+name = "go"
+enabled = true
+
+  [analyzers.meta]
+  import_paths = ["github.com/ankitdobhal/goreplay"]
+
+[[analyzers]]
+name = "docker"
+enabled = true
+
+[[analyzers]]
+name = "ruby"
+enabled = true
+
+[[analyzers]]
+name = "javascript"
+enabled = true
+
+  [analyzers.meta]
+  environment = ["nodejs"]
+

.github/dependabot.yaml

@@ -0,0 +1,10 @@
+version: 2
+updates:
+  # Maintain dependencies for GitHub Actions
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+    pull-request-branch-name:
+      # Separate sections of the branch name with a hyphen
+      separator: "-"

.github/workflows/ci-docker.yaml

@@ -0,0 +1,38 @@
+name: ci
+on:
+  release:
+    types: [published]
+jobs:
+  docker-build-and-push:
+    runs-on: ubuntu-latest
+    steps:
+      -
+        name: "Checkout repository"
+        uses: actions/checkout@v2
+      -
+        name: "Set up Docker Buildx"
+        uses: docker/setup-buildx-action@v1
+      -
+        name: "Cache Docker layers"
+        uses: actions/cache@v2
+        with:
+          path: /tmp/.buildx-cache
+          key: ${{ runner.os }}-buildx-${{ github.sha }}
+          restore-keys: |
+            ${{ runner.os }}-buildx-
+      -
+        name: "Login to Container Registry"
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      -
+        name: "Build and push docker image"
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          push: true
+          tags: ${{ github.repository }}:${{ github.event.release.tag_name }}
+          build-args: RELEASE_VERSION=${{ github.event.release.tag_name }}
+          cache-from: type=local,src=/tmp/.buildx-cache
+          cache-to: type=local,mode=max,dest=/tmp/.buildx-cache

.github/workflows/ci-test.yaml

@@ -0,0 +1,29 @@
+name: test
+on: [push, pull_request]
+jobs:
+  test:
+    strategy:
+      matrix:
+        go-version: [1.15.x, 1.16.x] # two latest minor versions
+    runs-on: ubuntu-latest
+    steps:
+    - name: update package index
+      run: sudo apt-get update
+    - name: install libpcap
+      run: sudo apt-get install libpcap-dev -y
+    - name: install Go
+      uses: actions/setup-go@v2
+      with:
+        go-version: ${{ matrix.go-version }}
+    - name: checkout code
+      uses: actions/checkout@v2
+    - uses: actions/cache@v2
+      with:
+        path: |
+          ~/go/pkg/mod              # Module download cache
+          ~/.cache/go-build         # Build cache (Linux)
+        key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
+        restore-keys: |
+          ${{ runner.os }}-go-
+    - name: test
+      run: sudo go test ./... -v -timeout 120s

.gitignore

@@ -1,10 +1,14 @@
+vendor
 *.swp
 *.gor
 *.rpm
 *.dep
+*.deb
 *.pkg
-
+*.exe
+*.pprof
 *.out
+hey
 
 *.bin
 

.travis.yml

@@ -1,9 +0,0 @@
-dist: focal
-
-language: go
-go: 1.14
-
-script: sudo -E bash -c "source /etc/profile && eval '$(gimme 1.14)' && go test ./... -v -timeout 120s"
-
-before_install:
-  - sudo apt-get install libpcap-dev -y

Dockerfile

@@ -1,6 +1,9 @@
-FROM alpine:latest as builder
+FROM alpine:3.12 as builder
+
+ARG RELEASE_VERSION
+
 RUN apk add --no-cache ca-certificates openssl
-RUN wget https://github.com/buger/goreplay/releases/download/v1.2.0/gor_v1.2.0_x64.tar.gz -O gor.tar.gz
+RUN wget https://github.com/buger/goreplay/releases/download/${RELEASE_VERSION}/gor_${RELEASE_VERSION}_x64.tar.gz -O gor.tar.gz
 RUN tar xzf gor.tar.gz
 
 FROM scratch

Dockerfile.dev

@@ -1,9 +1,9 @@
-FROM golang:1.14
+FROM golang:1.15
 
 RUN apt-get update && apt-get install ruby vim-common -y
 
 RUN apt-get install flex bison -y
-RUN wget http://www.tcpdump.org/release/libpcap-1.9.1.tar.gz && tar xzf libpcap-1.9.1.tar.gz && cd libpcap-1.9.1 && ./configure && make install
+RUN wget http://www.tcpdump.org/release/libpcap-1.10.0.tar.gz && tar xzf libpcap-1.10.0.tar.gz && cd libpcap-1.10.0 && ./configure && make install
 
 RUN go get github.com/google/gopacket
 RUN go get -u golang.org/x/lint/golint

Makefile

@@ -23,16 +23,19 @@ FPMCOMMON= \
     -s dir \
     -C /tmp/gor-build \
 
-release: release-x64 release-mac
+release: vendor release-x64 release-mac release-windows
+
+vendor:
+	go mod vendor
 
 release-bin:
-	docker run -v `pwd`:$(SOURCE_PATH) -t --env GOOS=linux --env GOARCH=amd64  -i $(CONTAINER) go build -o $(BIN_NAME) -tags netgo $(LDFLAGS)
+	docker run -v `pwd`:$(SOURCE_PATH) -t --env GOOS=linux --env GOARCH=amd64  -i $(CONTAINER) go build -mod=vendor -o $(BIN_NAME) -tags netgo $(LDFLAGS)
 
 release-bin-mac:
 	GOOS=darwin go build -o $(BIN_NAME) $(MAC_LDFLAGS)
 
 release-x64:
-	docker run -v `pwd`:$(SOURCE_PATH) -t --env GOOS=linux --env GOARCH=amd64  -i $(CONTAINER) go build -o $(BIN_NAME) -tags netgo $(LDFLAGS)
+	docker run -v `pwd`:$(SOURCE_PATH) -t --env GOOS=linux --env GOARCH=amd64  -i $(CONTAINER) go build -mod=vendor -o $(BIN_NAME) -tags netgo $(LDFLAGS)
 	tar -czf gor_$(VERSION)$(PREFIX)_x64.tar.gz $(BIN_NAME)
 	mkdir -p /tmp/gor-build
 	mv ./$(BIN_NAME) /tmp/gor-build/$(BIN_NAME)
@@ -44,12 +47,12 @@ release-x64:
 	rm -rf /tmp/gor-build
 
 release-x86:
-	docker run -v `pwd`:$(SOURCE_PATH) -t --env GOOS=linux --env GOARCH=386 -i $(CONTAINER) go build -o $(BIN_NAME) -tags netgo $(LDFLAGS)
+	docker run -v `pwd`:$(SOURCE_PATH) -t --env GOOS=linux --env GOARCH=386 -i $(CONTAINER) go build -mod=vendor -o $(BIN_NAME) -tags netgo $(LDFLAGS)
 	tar -czf gor_$(VERSION)$(PREFIX)_x86.tar.gz $(BIN_NAME)
 	rm $(BIN_NAME)
 
 release-mac:
-	go build -o $(BIN_NAME) $(MAC_LDFLAGS)
+	go build -mod=vendor -o $(BIN_NAME) $(MAC_LDFLAGS)
 	tar -czf gor_$(VERSION)$(PREFIX)_mac.tar.gz $(BIN_NAME)
 	mkdir -p /tmp/gor-build
 	mv ./$(BIN_NAME) /tmp/gor-build/$(BIN_NAME)
@@ -58,11 +61,25 @@ release-mac:
 	fpm $(FPMCOMMON) -a amd64 -t osxpkg ./=/usr/local/bin
 	rm -rf /tmp/gor-build
 
+release-windows:
+	docker run -it --rm \
+	  -v `pwd`:/go/src/github.com/buger/goreplay \
+	  -w /go/src/github.com/buger/goreplay \
+	  -e CGO_ENABLED=1 \
+	  docker.elastic.co/beats-dev/golang-crossbuild:1.16.4-main \
+	  --build-cmd "make VERSION=$(VERSION) build" \
+	  -p "windows/amd64"
+	mv ./gor ./gor.exe
+	zip gor-$(VERSION)$(PREFIX)_windows.zip ./gor.exe
+	rm -rf ./gor.exe
+
+build:
+	go build -mod=vendor -o $(BIN_NAME) $(LDFLAGS)
 
 install:
 	go install $(MAC_LDFLAGS)
 
-build:
+build-env:
 	docker build -t $(CONTAINER) -f Dockerfile.dev .
 
 profile:
@@ -102,7 +119,7 @@ profile_test:
 
 # Used mainly for debugging, because docker container do not have access to parent machine ports
 run:
-	$(RUN) go run $(LDFLAGS) $(SOURCE) --input-dummy=0 --output-http="http://localhost:9000" --input-raw-track-response --input-raw 127.0.0.1:9000 --verbose --debug --middleware "./examples/middleware/echo.sh" --output-file requests.gor
+	$(RUN) go run $(LDFLAGS) $(SOURCE) --input-dummy=0 --output-http="http://localhost:9000" --input-raw-track-response --input-raw 127.0.0.1:9000 --verbose 0 --middleware "./examples/middleware/echo.sh" --output-file requests.gor
 
 run-2:
 	$(RUN) go run $(LDFLAGS) $(SOURCE) --input-raw :8000 --input-raw-bpf-filter "dst port 8000" --output-stdout --output-http "http://localhost:8000" --input-dummy=0
@@ -130,7 +147,7 @@ bash:
 
 
 FPMCOMMON= \
-    --name goreplay \
+    --name gor \
     --description "GoReplay is an open-source network monitoring tool which can record your live traffic, and use it for shadowing, load testing, monitoring and detailed analysis." \
     -v $(VERSION) \
     --vendor "Leonid Bugaev" \

capture/af_packet.go

@@ -0,0 +1,32 @@
+// +build !linux
+
+package capture
+
+import (
+	"fmt"
+	"time"
+
+	"github.com/google/gopacket"
+)
+
+func newAfpacketHandle(device string, snaplen int, block_size int, num_blocks int,
+	useVLAN bool, timeout time.Duration) (*afpacketHandle, error) {
+	return nil, fmt.Errorf("Not implemented")
+}
+
+func afpacketComputeSize(targetSizeMb int, snaplen int, pageSize int) (
+	frameSize int, blockSize int, numBlocks int, err error) {
+	return 0, 0, 0, fmt.Errorf("Not implemented")
+}
+
+type afpacketHandle struct{}
+
+// ReadPacketData satisfies PacketDataSource interface
+func (h *afpacketHandle) ReadPacketData() (data []byte, ci gopacket.CaptureInfo, err error) {
+	return nil, gopacket.CaptureInfo{}, fmt.Errorf("Not implemented")
+}
+
+// SetBPFFilter translates a BPF filter string into BPF RawInstruction and applies them.
+func (h *afpacketHandle) SetBPFFilter(filter string, snaplen int) (err error) {
+	return fmt.Errorf("Not implemented")
+}

capture/af_packet_linux.go

@@ -0,0 +1,115 @@
+// +build linux
+
+package capture
+
+import (
+	"fmt"
+	"time"
+
+	"github.com/google/gopacket"
+	"github.com/google/gopacket/afpacket"
+	"github.com/google/gopacket/layers"
+	"github.com/google/gopacket/pcap"
+	"golang.org/x/net/bpf"
+
+	_ "github.com/google/gopacket/layers"
+)
+
+type afpacketHandle struct {
+	TPacket *afpacket.TPacket
+}
+
+func newAfpacketHandle(device string, snaplen int, block_size int, num_blocks int,
+	useVLAN bool, timeout time.Duration) (*afpacketHandle, error) {
+
+	h := &afpacketHandle{}
+	var err error
+
+	if device == "any" {
+		h.TPacket, err = afpacket.NewTPacket(
+			afpacket.OptFrameSize(snaplen),
+			afpacket.OptBlockSize(block_size),
+			afpacket.OptNumBlocks(num_blocks),
+			afpacket.OptAddVLANHeader(false),
+			afpacket.OptPollTimeout(timeout),
+			afpacket.SocketRaw,
+			afpacket.TPacketVersion3)
+	} else {
+		h.TPacket, err = afpacket.NewTPacket(
+			afpacket.OptInterface(device),
+			afpacket.OptFrameSize(snaplen),
+			afpacket.OptBlockSize(block_size),
+			afpacket.OptNumBlocks(num_blocks),
+			afpacket.OptAddVLANHeader(false),
+			afpacket.OptPollTimeout(timeout),
+			afpacket.SocketRaw,
+			afpacket.TPacketVersion3)
+	}
+	return h, err
+}
+
+// ReadPacketData satisfies PacketDataSource interface
+func (h *afpacketHandle) ReadPacketData() (data []byte, ci gopacket.CaptureInfo, err error) {
+	return h.TPacket.ReadPacketData()
+}
+
+// SetBPFFilter translates a BPF filter string into BPF RawInstruction and applies them.
+func (h *afpacketHandle) SetBPFFilter(filter string, snaplen int) (err error) {
+	pcapBPF, err := pcap.CompileBPFFilter(layers.LinkTypeEthernet, snaplen, filter)
+	if err != nil {
+		return err
+	}
+	bpfIns := []bpf.RawInstruction{}
+	for _, ins := range pcapBPF {
+		bpfIns2 := bpf.RawInstruction{
+			Op: ins.Code,
+			Jt: ins.Jt,
+			Jf: ins.Jf,
+			K:  ins.K,
+		}
+		bpfIns = append(bpfIns, bpfIns2)
+	}
+	if h.TPacket.SetBPF(bpfIns); err != nil {
+		return err
+	}
+	return h.TPacket.SetBPF(bpfIns)
+}
+
+// LinkType returns ethernet link type.
+func (h *afpacketHandle) LinkType() layers.LinkType {
+	return layers.LinkTypeEthernet
+}
+
+// Close will close afpacket source.
+func (h *afpacketHandle) Close() {
+	h.TPacket.Close()
+}
+
+// SocketStats prints received, dropped, queue-freeze packet stats.
+func (h *afpacketHandle) SocketStats() (as afpacket.SocketStats, asv afpacket.SocketStatsV3, err error) {
+	return h.TPacket.SocketStats()
+}
+
+// afpacketComputeSize computes the block_size and the num_blocks in such a way that the
+// allocated mmap buffer is close to but smaller than target_size_mb.
+// The restriction is that the block_size must be divisible by both the
+// frame size and page size.
+func afpacketComputeSize(targetSizeMb int, snaplen int, pageSize int) (
+	frameSize int, blockSize int, numBlocks int, err error) {
+
+	if snaplen < pageSize {
+		frameSize = pageSize / (pageSize / snaplen)
+	} else {
+		frameSize = (snaplen/pageSize + 1) * pageSize
+	}
+
+	// 128 is the default from the gopacket library so just use that
+	blockSize = frameSize * 128
+	numBlocks = (targetSizeMb * 1024 * 1024) / blockSize
+
+	if numBlocks == 0 {
+		return 0, 0, 0, fmt.Errorf("Interface buffersize is too small")
+	}
+
+	return frameSize, blockSize, numBlocks, nil
+}

capture/capture.go

@@ -3,21 +3,40 @@ package capture
 import (
 	"context"
 	"errors"
+	"expvar"
 	"fmt"
+	"io"
+	"log"
 	"net"
 	"os"
 	"runtime"
+	"strings"
 	"sync"
+	"syscall"
 	"time"
 
+	"github.com/buger/goreplay/proto"
 	"github.com/buger/goreplay/size"
+	"github.com/buger/goreplay/tcp"
+
 	"github.com/google/gopacket"
 	"github.com/google/gopacket/layers"
 	"github.com/google/gopacket/pcap"
 )
 
-// Handler is a function that is used to handle packets
-type Handler func(gopacket.Packet)
+var stats *expvar.Map
+
+func init() {
+	stats = expvar.NewMap("raw")
+	stats.Init()
+}
+
+// PacketHandler is a function that is used to handle packets
+type PacketHandler func(*tcp.Packet)
+
+type PcapStatProvider interface {
+	Stats() (*pcap.Stats, error)
+}
 
 // PcapOptions options that can be set on a pcap capture handle,
 // these options take effect on inactive pcap handles
@@ -31,30 +50,33 @@ type PcapOptions struct {
 	Snaplen       bool          `json:"input-raw-override-snaplen"`
 }
 
-// NetInterface represents network interface
-type NetInterface struct {
-	net.Interface
-	IPs []string
-}
-
 // Listener handle traffic capture, this is its representation.
 type Listener struct {
 	sync.Mutex
 	Transport  string       // transport layer default to tcp
 	Activate   func() error // function is used to activate the engine. it must be called before reading packets
-	Handles    map[string]gopacket.PacketDataSource
-	Interfaces []NetInterface
+	Handles    map[string]packetHandle
+	Interfaces []pcap.Interface
 	loopIndex  int
 	Reading    chan bool // this channel is closed when the listener has started reading packets
 	PcapOptions
-	Engine        EngineType
-	port          uint16 // src or/and dst port
-	trackResponse bool
+	Engine          EngineType
+	ports           []uint16 // src or/and dst ports
+	trackResponse   bool
+	expiry          time.Duration
+	allowIncomplete bool
+	messages        chan *tcp.Message
+	protocol        tcp.TCPProtocol
 
 	host string // pcap file name or interface (name, hardware addr, index or ip address)
 
-	quit    chan bool
-	packets chan gopacket.Packet
+	closeDone chan struct{}
+	quit      chan struct{}
+}
+
+type packetHandle struct {
+	handler gopacket.PacketDataSource
+	ips     []net.IP
 }
 
 // EngineType ...
@@ -65,6 +87,7 @@ const (
 	EnginePcap EngineType = 1 << iota
 	EnginePcapFile
 	EngineRawSocket
+	EngineAFPacket
 )
 
 // Set is here so that EngineType can implement flag.Var
@@ -74,8 +97,10 @@ func (eng *EngineType) Set(v string) error {
 		*eng = EnginePcap
 	case "pcap_file":
 		*eng = EnginePcapFile
-	case "raw_socket", "af_packet":
+	case "raw_socket":
 		*eng = EngineRawSocket
+	case "af_packet":
+		*eng = EngineAFPacket
 	default:
 		return fmt.Errorf("invalid engine %s", v)
 	}
@@ -90,6 +115,8 @@ func (eng *EngineType) String() (e string) {
 		e = "libpcap"
 	case EngineRawSocket:
 		e = "raw_socket"
+	case EngineAFPacket:
+		e = "af_packet"
 	default:
 		e = ""
 	}
@@ -99,20 +126,29 @@ func (eng *EngineType) String() (e string) {
 // NewListener creates and initialize a new Listener. if transport or/and engine are invalid/unsupported
 // is "tcp" and "pcap", are assumed. l.Engine and l.Transport can help to get the values used.
 // if there is an error it will be associated with getting network interfaces
-func NewListener(host string, port uint16, transport string, engine EngineType, trackResponse bool) (l *Listener, err error) {
+func NewListener(host string, ports []uint16, transport string, engine EngineType, protocol tcp.TCPProtocol, trackResponse bool, expiry time.Duration, allowIncomplete bool) (l *Listener, err error) {
 	l = &Listener{}
 
 	l.host = host
-	l.port = port
+	if l.host == "localhost" {
+		l.host = "127.0.0.1"
+	}
+	l.ports = ports
+
 	l.Transport = "tcp"
 	if transport != "" {
 		l.Transport = transport
 	}
-	l.Handles = make(map[string]gopacket.PacketDataSource)
+	l.Handles = make(map[string]packetHandle)
 	l.trackResponse = trackResponse
-	l.packets = make(chan gopacket.Packet, 1000)
-	l.quit = make(chan bool, 1)
-	l.Reading = make(chan bool, 1)
+	l.closeDone = make(chan struct{})
+	l.quit = make(chan struct{})
+	l.Reading = make(chan bool)
+	l.expiry = expiry
+	l.allowIncomplete = allowIncomplete
+	l.protocol = protocol
+	l.messages = make(chan *tcp.Message, 10000)
+
 	switch engine {
 	default:
 		l.Engine = EnginePcap
@@ -120,11 +156,15 @@ func NewListener(host string, port uint16, transport string, engine EngineType,
 	case EngineRawSocket:
 		l.Engine = EngineRawSocket
 		l.Activate = l.activateRawSocket
+	case EngineAFPacket:
+		l.Engine = EngineAFPacket
+		l.Activate = l.activateAFPacket
 	case EnginePcapFile:
 		l.Engine = EnginePcapFile
 		l.Activate = l.activatePcapFile
 		return
 	}
+
 	err = l.setInterfaces()
 	if err != nil {
 		return nil, err
@@ -139,38 +179,27 @@ func (l *Listener) SetPcapOptions(opts PcapOptions) {
 }
 
 // Listen listens for packets from the handles, and call handler on every packet received
-// until the context done signal is sent or EOF on handles.
-// this function should be called after activating pcap handles
-func (l *Listener) Listen(ctx context.Context, handler Handler) (err error) {
+// until the context done signal is sent or there is unrecoverable error on all handles.
+// this function must be called after activating pcap handles
+func (l *Listener) Listen(ctx context.Context) (err error) {
 	l.read()
 	done := ctx.Done()
-	var p gopacket.Packet
-	var ok bool
-	for {
-		select {
-		case <-done:
-			l.quit <- true
-			close(l.quit)
-			err = ctx.Err()
-			done = nil
-		case p, ok = <-l.packets:
-			if !ok {
-				return
-			}
-			if p == nil {
-				continue
-			}
-			handler(p)
-		}
+	select {
+	case <-done:
+		close(l.quit) // signal close on all handles
+		<-l.closeDone // wait all handles to be closed
+		err = ctx.Err()
+	case <-l.closeDone: // all handles closed voluntarily
 	}
+	return
 }
 
 // ListenBackground is like listen but can run concurrently and signal error through channel
-func (l *Listener) ListenBackground(ctx context.Context, handler Handler) chan error {
+func (l *Listener) ListenBackground(ctx context.Context) chan error {
 	err := make(chan error, 1)
 	go func() {
 		defer close(err)
-		if e := l.Listen(ctx, handler); err != nil {
+		if e := l.Listen(ctx); err != nil {
 			err <- e
 		}
 	}()
@@ -179,59 +208,51 @@ func (l *Listener) ListenBackground(ctx context.Context, handler Handler) chan e
 
 // Filter returns automatic filter applied by goreplay
 // to a pcap handle of a specific interface
-func (l *Listener) Filter(ifi NetInterface) (filter string) {
+func (l *Listener) Filter(ifi pcap.Interface) (filter string) {
 	// https://www.tcpdump.org/manpages/pcap-filter.7.html
 
-	port := fmt.Sprintf("portrange 0-%d", 1<<16-1)
-	if l.port != 0 {
-		port = fmt.Sprintf("port %d", l.port)
-	}
-	dir := " dst " // direction
-	if l.trackResponse {
-		dir = " "
-	}
-	filter = fmt.Sprintf("(%s%s%s)", l.Transport, dir, port)
+	hosts := []string{l.host}
 	if listenAll(l.host) || isDevice(l.host, ifi) {
-		return
+		hosts = interfaceAddresses(ifi)
 	}
-	filter = fmt.Sprintf("(%s%s%s and host %s)", l.Transport, dir, port, l.host)
-	return
-}
 
-// PcapDumpHandler returns a handler to write packet data in PCAP
-// format, See http://wiki.wireshark.org/Development/LibpcapFileFormathandler.
-// if link layer is invalid Ethernet is assumed
-func PcapDumpHandler(file *os.File, link layers.LinkType, debugger func(int, ...interface{})) (handler func(packet gopacket.Packet), err error) {
-	if link.String() == "" {
-		link = layers.LinkTypeEthernet
+	filter = portsFilter(l.Transport, "dst", l.ports)
+
+	if len(hosts) != 0 && !l.Promiscuous {
+		filter = fmt.Sprintf("((%s) and (%s))", filter, hostsFilter("dst", hosts))
+	} else {
+		filter = fmt.Sprintf("(%s)", filter)
 	}
-	w := NewWriterNanos(file)
-	err = w.WriteFileHeader(64<<10, link)
-	if err != nil {
-		return nil, err
-	}
-	return func(packet gopacket.Packet) {
-		err = w.WritePacket(packet.Metadata().CaptureInfo, packet.Data())
-		if err != nil && debugger != nil {
-			go debugger(3, err)
+
+	if l.trackResponse {
+		responseFilter := portsFilter(l.Transport, "src", l.ports)
+
+		if len(hosts) != 0 && !l.Promiscuous {
+			responseFilter = fmt.Sprintf("((%s) and (%s))", responseFilter, hostsFilter("src", hosts))
+		} else {
+			responseFilter = fmt.Sprintf("(%s)", responseFilter)
 		}
-	}, nil
+
+		filter = fmt.Sprintf("%s or %s", filter, responseFilter)
+	}
+
+	return
 }
 
 // PcapHandle returns new pcap Handle from dev on success.
 // this function should be called after setting all necessary options for this listener
-func (l *Listener) PcapHandle(ifi NetInterface) (handle *pcap.Handle, err error) {
+func (l *Listener) PcapHandle(ifi pcap.Interface) (handle *pcap.Handle, err error) {
 	var inactive *pcap.InactiveHandle
 	inactive, err = pcap.NewInactiveHandle(ifi.Name)
-	if inactive != nil && err != nil {
-		defer inactive.CleanUp()
-	}
 	if err != nil {
 		return nil, fmt.Errorf("inactive handle error: %q, interface: %q", err, ifi.Name)
 	}
-	if l.TimestampType != "" {
+	defer inactive.CleanUp()
+
+	if l.TimestampType != "" && l.TimestampType != "go" {
 		var ts pcap.TimestampSource
 		ts, err = pcap.TimestampSourceFromString(l.TimestampType)
+		fmt.Println("Setting custom Timestamp Source. Supported values: `go`, ", inactive.SupportedTimestamps())
 		err = inactive.SetTimestampSource(ts)
 		if err != nil {
 			return nil, fmt.Errorf("%q: supported timestamps: %q, interface: %q", err, inactive.SupportedTimestamps(), ifi.Name)
@@ -247,12 +268,22 @@ func (l *Listener) PcapHandle(ifi NetInterface) (handle *pcap.Handle, err error)
 			return nil, fmt.Errorf("monitor mode error: %q, interface: %q", err, ifi.Name)
 		}
 	}
+
 	var snap int
-	if l.Snaplen {
-		snap = 64<<10 + 200
-	} else if ifi.MTU > 0 {
-		snap = ifi.MTU + 200
+
+	if !l.Snaplen {
+		infs, _ := net.Interfaces()
+		for _, i := range infs {
+			if i.Name == ifi.Name {
+				snap = i.MTU + 200
+			}
+		}
 	}
+
+	if snap == 0 {
+		snap = 64<<10 + 200
+	}
+
 	err = inactive.SetSnapLen(snap)
 	if err != nil {
 		return nil, fmt.Errorf("snapshot length error: %q, interface: %q", err, ifi.Name)
@@ -264,7 +295,7 @@ func (l *Listener) PcapHandle(ifi NetInterface) (handle *pcap.Handle, err error)
 		}
 	}
 	if l.BufferTimeout == 0 {
-		l.BufferTimeout = pcap.BlockForever
+		l.BufferTimeout = 2000 * time.Millisecond
 	}
 	err = inactive.SetTimeout(l.BufferTimeout)
 	if err != nil {
@@ -274,37 +305,33 @@ func (l *Listener) PcapHandle(ifi NetInterface) (handle *pcap.Handle, err error)
 	if err != nil {
 		return nil, fmt.Errorf("PCAP Activate device error: %q, interface: %q", err, ifi.Name)
 	}
-	if l.BPFFilter != "" {
-		if l.BPFFilter[0] != '(' || l.BPFFilter[len(l.BPFFilter)-1] != ')' {
-			l.BPFFilter = "(" + l.BPFFilter + ")"
-		}
-	} else {
-		l.BPFFilter = l.Filter(ifi)
+
+	bpfFilter := l.BPFFilter
+	if bpfFilter == "" {
+		bpfFilter = l.Filter(ifi)
 	}
-	err = handle.SetBPFFilter(l.BPFFilter)
+	fmt.Println("Interface:", ifi.Name, ". BPF Filter:", bpfFilter)
+	err = handle.SetBPFFilter(bpfFilter)
 	if err != nil {
 		handle.Close()
-		return nil, fmt.Errorf("BPF filter error: %q%s, interface: %q", err, l.BPFFilter, ifi.Name)
+		return nil, fmt.Errorf("BPF filter error: %q%s, interface: %q", err, bpfFilter, ifi.Name)
 	}
 	return
 }
 
 // SocketHandle returns new unix ethernet handle associated with this listener settings
-func (l *Listener) SocketHandle(ifi NetInterface) (handle Socket, err error) {
-	handle, err = NewSocket(ifi.Interface)
+func (l *Listener) SocketHandle(ifi pcap.Interface) (handle Socket, err error) {
+	handle, err = NewSocket(ifi)
 	if err != nil {
 		return nil, fmt.Errorf("sock raw error: %q, interface: %q", err, ifi.Name)
 	}
 	if err = handle.SetPromiscuous(l.Promiscuous || l.Monitor); err != nil {
 		return nil, fmt.Errorf("promiscuous mode error: %q, interface: %q", err, ifi.Name)
 	}
-	if l.BPFFilter != "" {
-		if l.BPFFilter[0] != '(' || l.BPFFilter[len(l.BPFFilter)-1] != ')' {
-			l.BPFFilter = "(" + l.BPFFilter + ")"
-		}
-	} else {
+	if l.BPFFilter == "" {
 		l.BPFFilter = l.Filter(ifi)
 	}
+	fmt.Println("BPF Filter: ", l.BPFFilter)
 	if err = handle.SetBPFFilter(l.BPFFilter); err != nil {
 		handle.Close()
 		return nil, fmt.Errorf("BPF filter error: %q%s, interface: %q", err, l.BPFFilter, ifi.Name)
@@ -313,50 +340,121 @@ func (l *Listener) SocketHandle(ifi NetInterface) (handle Socket, err error) {
 	return
 }
 
+func http1StartHint(pckt *tcp.Packet) (isRequest, isResponse bool) {
+	if proto.HasRequestTitle(pckt.Payload) {
+		return true, false
+	}
+
+	if proto.HasResponseTitle(pckt.Payload) {
+		return false, true
+	}
+
+	// No request or response detected
+	return false, false
+}
+
+func http1EndHint(m *tcp.Message) bool {
+	if m.MissingChunk() {
+		return false
+	}
+
+	return proto.HasFullPayload(m, m.PacketData()...)
+}
+
 func (l *Listener) read() {
 	l.Lock()
 	defer l.Unlock()
 	for key, handle := range l.Handles {
-		var source *gopacket.PacketSource
-		linkType := layers.LinkTypeEthernet
-		if _, ok := handle.(*pcap.Handle); ok {
-			linkType = handle.(*pcap.Handle).LinkType()
-		}
-		source = gopacket.NewPacketSource(handle, linkType)
-		source.Lazy = true
-		source.NoCopy = true
-		ch := source.Packets()
-		go func(key string) {
+		go func(key string, hndl packetHandle) {
+			runtime.LockOSThread()
+
 			defer l.closeHandles(key)
+			linkSize := 14
+			linkType := int(layers.LinkTypeEthernet)
+			if _, ok := hndl.handler.(*pcap.Handle); ok {
+				linkType = int(hndl.handler.(*pcap.Handle).LinkType())
+				linkSize, ok = pcapLinkTypeLength(linkType)
+				if !ok {
+					if os.Getenv("GORDEBUG") != "0" {
+						log.Printf("can not identify link type of an interface '%s'\n", key)
+					}
+					return // can't find the linktype size
+				}
+			}
+
+			messageParser := tcp.NewMessageParser(l.messages, l.ports, hndl.ips, l.expiry, l.allowIncomplete)
+
+			if l.protocol == tcp.ProtocolHTTP {
+				messageParser.Start = http1StartHint
+				messageParser.End = http1EndHint
+			}
+
+			timer := time.NewTicker(1 * time.Second)
+
 			for {
 				select {
 				case <-l.quit:
 					return
-				case p, ok := <-ch:
-					if !ok {
+				case <-timer.C:
+					if h, ok := hndl.handler.(PcapStatProvider); ok {
+						s, _ := h.Stats()
+						stats.Add("packets_received", int64(s.PacketsReceived))
+						stats.Add("packets_dropped", int64(s.PacketsDropped))
+						stats.Add("packets_if_dropped", int64(s.PacketsIfDropped))
+					}
+				default:
+					data, ci, err := hndl.handler.ReadPacketData()
+					if err == nil {
+						if l.TimestampType == "go" {
+							ci.Timestamp = time.Now()
+						}
+
+						messageParser.PacketHandler(&tcp.PcapPacket{
+							Data:     data,
+							LType:    linkType,
+							LTypeLen: linkSize,
+							Ci:       &ci,
+						})
+						continue
+					}
+					if enext, ok := err.(pcap.NextError); ok && enext == pcap.NextErrorTimeoutExpired {
+						continue
+					}
+					if eno, ok := err.(syscall.Errno); ok && eno.Temporary() {
+						continue
+					}
+					if enet, ok := err.(*net.OpError); ok && (enet.Temporary() || enet.Timeout()) {
+						continue
+					}
+					if err == io.EOF || err == io.ErrClosedPipe {
+						log.Printf("stopped reading from %s interface with error %s\n", key, err)
 						return
 					}
-					l.packets <- p
+
+					log.Printf("stopped reading from %s interface with error %s\n", key, err)
+					return
 				}
 			}
-		}(key)
+		}(key, handle)
 	}
-	l.Reading <- true
 	close(l.Reading)
 }
 
+func (l *Listener) Messages() chan *tcp.Message {
+	return l.messages
+}
+
 func (l *Listener) closeHandles(key string) {
 	l.Lock()
 	defer l.Unlock()
 	if handle, ok := l.Handles[key]; ok {
-		if _, ok = handle.(Socket); ok {
-			handle.(Socket).Close()
-		} else {
-			handle.(*pcap.Handle).Close()
+		if c, ok := handle.handler.(io.Closer); ok {
+			c.Close()
 		}
+
 		delete(l.Handles, key)
 		if len(l.Handles) == 0 {
-			close(l.packets)
+			close(l.closeDone)
 		}
 	}
 }
@@ -371,7 +469,10 @@ func (l *Listener) activatePcap() error {
 			msg += ("\n" + e.Error())
 			continue
 		}
-		l.Handles[ifi.Name] = handle
+		l.Handles[ifi.Name] = packetHandle{
+			handler: handle,
+			ips:     interfaceIPs(ifi),
+		}
 	}
 	if len(l.Handles) == 0 {
 		return fmt.Errorf("pcap handles error:%s", msg)
@@ -392,9 +493,15 @@ func (l *Listener) activateRawSocket() error {
 			msg += ("\n" + e.Error())
 			continue
 		}
-		l.Handles[ifi.Name] = handle
+		l.Handles[ifi.Name] = packetHandle{
+			handler: handle,
+			ips:     interfaceIPs(ifi),
+		}
 	}
-	return e
+	if len(l.Handles) == 0 {
+		return fmt.Errorf("raw socket handles error:%s", msg)
+	}
+	return nil
 }
 
 func (l *Listener) activatePcapFile() (err error) {
@@ -403,92 +510,140 @@ func (l *Listener) activatePcapFile() (err error) {
 	if handle, e = pcap.OpenOffline(l.host); e != nil {
 		return fmt.Errorf("open pcap file error: %q", e)
 	}
-	if l.BPFFilter != "" {
-		if l.BPFFilter[0] != '(' || l.BPFFilter[len(l.BPFFilter)-1] != ')' {
-			l.BPFFilter = "(" + l.BPFFilter + ")"
-		}
-	} else {
-		addr := l.host
-		l.host = ""
-		l.BPFFilter = l.Filter(NetInterface{})
-		l.host = addr
-	}
+
+	tmp := l.host
+	l.host = ""
+	l.BPFFilter = l.Filter(pcap.Interface{})
+	l.host = tmp
+
 	if e = handle.SetBPFFilter(l.BPFFilter); e != nil {
 		handle.Close()
 		return fmt.Errorf("BPF filter error: %q, filter: %s", e, l.BPFFilter)
 	}
-	l.Handles["pcap_file"] = handle
+	l.Handles["pcap_file"] = packetHandle{
+		handler: handle,
+	}
 	return
 }
 
-func (l *Listener) setInterfaces() (err error) {
-	var Ifis []NetInterface
-	var ifis []net.Interface
-	ifis, err = net.Interfaces()
+func (l *Listener) activateAFPacket() error {
+	szFrame, szBlock, numBlocks, err := afpacketComputeSize(32, 32<<10, os.Getpagesize())
 	if err != nil {
 		return err
 	}
 
-	for i := 0; i < len(ifis); i++ {
-		if ifis[i].Flags&net.FlagLoopback != 0 {
-			l.loopIndex = ifis[i].Index
-		}
-		if ifis[i].Flags&net.FlagUp == 0 {
-			continue
-		}
-		var addrs []net.Addr
-		addrs, err = ifis[i].Addrs()
+	var msg string
+	for _, ifi := range l.Interfaces {
+		handle, err := newAfpacketHandle(ifi.Name, szFrame, szBlock, numBlocks, false, pcap.BlockForever)
+
 		if err != nil {
-			return err
-		}
-		if len(addrs) == 0 {
+			msg += ("\n" + err.Error())
 			continue
 		}
-		ifi := NetInterface{}
-		ifi.Interface = ifis[i]
-		ifi.IPs = make([]string, len(addrs))
-		for j, addr := range addrs {
-			ifi.IPs[j] = cutMask(addr)
+
+		if l.BPFFilter == "" {
+			l.BPFFilter = l.Filter(ifi)
+		}
+		fmt.Println("Interface:", ifi.Name, ". BPF Filter:", l.BPFFilter)
+		handle.SetBPFFilter(l.BPFFilter, 64<<10)
+
+		l.Handles[ifi.Name] = packetHandle{
+			handler: handle,
+			ips:     interfaceIPs(ifi),
 		}
-		Ifis = append(Ifis, ifi)
 	}
 
-	if listenAll(l.host) {
-		l.Interfaces = Ifis
+	if len(l.Handles) == 0 {
+		return fmt.Errorf("pcap handles error:%s", msg)
+	}
+
+	return nil
+}
+
+func (l *Listener) setInterfaces() (err error) {
+	var pifis []pcap.Interface
+	pifis, err = pcap.FindAllDevs()
+	ifis, _ := net.Interfaces()
+	if err != nil {
 		return
 	}
-	found := false
-	for _, ifi := range Ifis {
-		if isDevice(l.host, ifi) {
-			found = true
-		}
-		for _, ip := range ifi.IPs {
-			if ip == l.host {
-				found = true
-				break
-			}
-		}
-		if found {
-			l.Interfaces = []NetInterface{ifi}
+
+	for _, pi := range pifis {
+		if isDevice(l.host, pi) {
+			l.Interfaces = []pcap.Interface{pi}
 			return
 		}
+
+		var ni net.Interface
+		for _, i := range ifis {
+			if i.Name == pi.Name {
+				ni = i
+				break
+			}
+
+			addrs, _ := i.Addrs()
+			for _, a := range addrs {
+				for _, pa := range pi.Addresses {
+					if a.String() == pa.IP.String() {
+						ni = i
+						break
+					}
+				}
+			}
+		}
+
+		if ni.Flags&net.FlagLoopback != 0 {
+			l.loopIndex = ni.Index
+		}
+
+		if runtime.GOOS != "windows" {
+			if len(pi.Addresses) == 0 {
+				continue
+			}
+
+			if ni.Flags&net.FlagUp == 0 {
+				continue
+			}
+		}
+
+		l.Interfaces = append(l.Interfaces, pi)
 	}
-	err = fmt.Errorf("can not find interface with addr, name or index %s", l.host)
-	return err
+	return
 }
 
-func cutMask(addr net.Addr) string {
-	mask := addr.String()
-	for i, v := range mask {
-		if v == '/' {
-			return mask[:i]
+func isDevice(addr string, ifi pcap.Interface) bool {
+	// Windows npcap loopback have no IPs
+	if addr == "127.0.0.1" && ifi.Name == `\Device\NPF_Loopback` {
+		return true
+	}
+
+	if addr == ifi.Name {
+		return true
+	}
+
+	for _, _addr := range ifi.Addresses {
+		if _addr.IP.String() == addr {
+			return true
 		}
 	}
-	return mask
+
+	return false
 }
 
-func isDevice(addr string, ifi NetInterface) bool {
-	return addr == ifi.Name || addr == fmt.Sprintf("%d", ifi.Index) || addr == ifi.HardwareAddr.String()
+func interfaceAddresses(ifi pcap.Interface) []string {
+	var hosts []string
+	for _, addr := range ifi.Addresses {
+		hosts = append(hosts, addr.IP.String())
+	}
+	return hosts
+}
+
+func interfaceIPs(ifi pcap.Interface) []net.IP {
+	var ips []net.IP
+	for _, addr := range ifi.Addresses {
+		ips = append(ips, addr.IP)
+	}
+	return ips
 }
 
 func listenAll(addr string) bool {
@@ -498,3 +653,47 @@ func listenAll(addr string) bool {
 	}
 	return false
 }
+
+func portsFilter(transport string, direction string, ports []uint16) string {
+	if len(ports) == 0 || ports[0] == 0 {
+		return fmt.Sprintf("%s %s portrange 0-%d", transport, direction, 1<<16-1)
+	}
+
+	var filters []string
+	for _, port := range ports {
+		filters = append(filters, fmt.Sprintf("%s %s port %d", transport, direction, port))
+	}
+	return strings.Join(filters, " or ")
+}
+
+func hostsFilter(direction string, hosts []string) string {
+	var hostsFilters []string
+	for _, host := range hosts {
+		hostsFilters = append(hostsFilters, fmt.Sprintf("%s host %s", direction, host))
+	}
+
+	return strings.Join(hostsFilters, " or ")
+}
+
+func pcapLinkTypeLength(lType int) (int, bool) {
+	switch layers.LinkType(lType) {
+	case layers.LinkTypeEthernet:
+		return 14, true
+	case layers.LinkTypeNull, layers.LinkTypeLoop:
+		return 4, true
+	case layers.LinkTypeRaw, 12, 14:
+		return 0, true
+	case layers.LinkTypeIPv4, layers.LinkTypeIPv6:
+		// (TODO:) look out for IP encapsulation?
+		return 0, true
+	case layers.LinkTypeLinuxSLL:
+		return 16, true
+	case layers.LinkTypeFDDI:
+		return 13, true
+	case 226 /*DLT_IPNET*/ :
+		// https://www.tcpdump.org/linktypes/LINKTYPE_IPNET.html
+		return 24, true
+	default:
+		return 0, false
+	}
+}

capture/capture_test.go

@@ -1,360 +0,0 @@
-package capture
-
-import (
-	"context"
-	"encoding/binary"
-	"io/ioutil"
-	"net"
-	"os"
-	"sync/atomic"
-	"testing"
-	"time"
-
-	"github.com/google/gopacket"
-	"github.com/google/gopacket/layers"
-	"github.com/google/gopacket/pcap"
-)
-
-var LoopBack = func() net.Interface {
-	ifis, _ := net.Interfaces()
-	for _, v := range ifis {
-		if v.Flags&net.FlagLoopback != 0 {
-			return v
-		}
-	}
-	return ifis[0]
-}()
-
-func TestSetInterfaces(t *testing.T) {
-	l := &Listener{}
-	l.host = "127.0.0.1"
-	l.setInterfaces()
-	if len(l.Interfaces) != 1 {
-		t.Error("expected a single interface")
-	}
-	l.host = LoopBack.HardwareAddr.String()
-	l.setInterfaces()
-	if l.Interfaces[0].Name != LoopBack.Name && len(l.Interfaces) != 1 {
-		t.Error("interface should be loop back interface")
-	}
-	l.host = ""
-	l.setInterfaces()
-	if len(l.Interfaces) < 1 {
-		t.Error("should get all interfaces")
-	}
-}
-
-func TestBPFFilter(t *testing.T) {
-	l := &Listener{}
-	l.host = "127.0.0.1"
-	l.Transport = "tcp"
-	l.setInterfaces()
-	filter := l.Filter(l.Interfaces[0])
-	if filter != "(tcp dst portrange 0-65535 and host 127.0.0.1)" {
-		t.Error("wrong filter", filter)
-	}
-	l.port = 8000
-	l.trackResponse = true
-	filter = l.Filter(l.Interfaces[0])
-	if filter != "(tcp port 8000 and host 127.0.0.1)" {
-		t.Error("wrong filter")
-	}
-}
-
-var decodeOpts = gopacket.DecodeOptions{Lazy: true, NoCopy: true}
-
-func generateHeaders(seq uint32, length uint16) (headers [44]byte) {
-	// set ethernet headers
-	binary.BigEndian.PutUint32(headers[0:4], uint32(layers.ProtocolFamilyIPv4))
-
-	// set ip header
-	ip := headers[4:]
-	copy(ip[0:2], []byte{4<<4 | 5, 0x28<<2 | 0x00})
-	binary.BigEndian.PutUint16(ip[2:4], length+54)
-	ip[9] = uint8(layers.IPProtocolTCP)
-	copy(ip[12:16], []byte{127, 0, 0, 1})
-	copy(ip[16:], []byte{127, 0, 0, 1})
-
-	// set tcp header
-	tcp := ip[20:]
-	binary.BigEndian.PutUint16(tcp[0:2], 45678)
-	binary.BigEndian.PutUint16(tcp[2:4], 8000)
-	tcp[12] = 5 << 4
-	return
-}
-
-func randomPackets(start uint32, _len int, length uint16) []gopacket.Packet {
-	var packets = make([]gopacket.Packet, _len)
-	for i := start; i < start+uint32(_len); i++ {
-		h := generateHeaders(i, length)
-		d := make([]byte, int(length)+len(h))
-		copy(d, h[0:])
-		packet := gopacket.NewPacket(d, layers.LinkTypeLoop, decodeOpts)
-		packets[i-start] = packet
-		inf := packets[i-start].Metadata()
-		_len := len(d)
-		inf.CaptureInfo = gopacket.CaptureInfo{CaptureLength: _len, Length: _len, Timestamp: time.Now()}
-	}
-	return packets
-}
-
-func TestPcapDump(t *testing.T) {
-	f, err := ioutil.TempFile("", "pcap_file")
-	if err != nil {
-		t.Error(err)
-	}
-	waiter := make(chan bool, 1)
-	h, _ := PcapDumpHandler(f, layers.LinkTypeLoop, func(level int, a ...interface{}) {
-		if level != 3 {
-			t.Errorf("expected debug level to be 3, got %d", level)
-		}
-		waiter <- true
-	})
-	packets := randomPackets(1, 5, 5)
-	for i := 0; i < len(packets); i++ {
-		if i == 1 {
-			tcp := packets[i].Data()[4:][20:]
-			// change dst port
-			binary.BigEndian.PutUint16(tcp[2:], 8001)
-		}
-		if i == 4 {
-			inf := packets[i].Metadata()
-			inf.CaptureLength = 40
-		}
-		h(packets[i])
-	}
-	<-waiter
-	name := f.Name()
-	f.Close()
-	testPcapDumpEngine(name, t)
-}
-
-func testPcapDumpEngine(f string, t *testing.T) {
-	defer os.Remove(f)
-	l, err := NewListener(f, 8000, "", EnginePcapFile, true)
-	err = l.Activate()
-	if err != nil {
-		t.Errorf("expected error to be nil, got %q", err)
-		return
-	}
-	pckts := 0
-	ctx, cancel := context.WithCancel(context.Background())
-	defer cancel()
-	err = l.Listen(ctx, func(packet gopacket.Packet) {
-		if packet.Metadata().CaptureLength != 49 {
-			t.Errorf("expected packet length to be %d, got %d", 49, packet.Metadata().CaptureLength)
-		}
-		pckts++
-	})
-
-	if err != nil {
-		t.Errorf("expected error to be nil, got %q", err)
-	}
-	if pckts != 3 {
-		t.Errorf("expected %d packets, got %d packets", 3, pckts)
-	}
-}
-
-func TestPcapHandler(t *testing.T) {
-	l, err := NewListener(LoopBack.Name, 8000, "", EnginePcap, true)
-	if err != nil {
-		t.Errorf("expected error to be nil, got %v", err)
-		return
-	}
-	err = l.Activate()
-	if err != nil {
-		t.Errorf("expected error to be nil, got %v", err)
-		return
-	}
-	defer l.Handles[LoopBack.Name].(*pcap.Handle).Close()
-	if err != nil {
-		t.Errorf("expected error to be nil, got %v", err)
-		return
-	}
-	for i := 0; i < 5; i++ {
-		_, _ = net.Dial("tcp", "127.0.0.1:8000")
-	}
-	sts, _ := l.Handles[LoopBack.Name].(*pcap.Handle).Stats()
-	if sts.PacketsReceived < 5 {
-		t.Errorf("expected >=5 packets got %d", sts.PacketsReceived)
-	}
-}
-
-func TestSocketHandler(t *testing.T) {
-	l, err := NewListener(LoopBack.Name, 8000, "", EngineRawSocket, true)
-	err = l.Activate()
-	if err != nil {
-		return
-	}
-	defer l.Handles[LoopBack.Name].(*SockRaw).Close()
-	if err != nil {
-		t.Errorf("expected error to be nil, got %v", err)
-		return
-	}
-	for i := 0; i < 5; i++ {
-		_, _ = net.Dial("tcp", "127.0.0.1:8000")
-	}
-	sts, _ := l.Handles[LoopBack.Name].(*SockRaw).Stats()
-	if sts.Packets < 5 {
-		t.Errorf("expected >=5 packets got %d", sts.Packets)
-	}
-}
-
-func BenchmarkPcapDump(b *testing.B) {
-	f, err := ioutil.TempFile("", "pcap_file")
-	if err != nil {
-		b.Error(err)
-		return
-	}
-	now := time.Now()
-	defer os.Remove(f.Name())
-	h, _ := PcapDumpHandler(f, layers.LinkTypeLoop, nil)
-	packets := randomPackets(1, b.N, 5)
-	for i := 0; i < len(packets); i++ {
-		h(packets[i])
-	}
-	f.Close()
-	b.Logf("%d packets in %s", b.N, time.Since(now))
-}
-
-func BenchmarkPcapFile(b *testing.B) {
-	f, err := ioutil.TempFile("", "pcap_file")
-	if err != nil {
-		b.Error(err)
-		return
-	}
-	defer os.Remove(f.Name())
-	h, _ := PcapDumpHandler(f, layers.LinkTypeLoop, nil)
-	packets := randomPackets(1, b.N, 5)
-	for i := 0; i < len(packets); i++ {
-		h(packets[i])
-	}
-	name := f.Name()
-	f.Close()
-	b.ResetTimer()
-	var l *Listener
-	l, err = NewListener(name, 8000, "", EnginePcapFile, true)
-	if err != nil {
-		b.Error(err)
-		return
-	}
-	err = l.Activate()
-	if err != nil {
-		b.Error(err)
-		return
-	}
-	now := time.Now()
-	pckts := 0
-	ctx, cancel := context.WithCancel(context.Background())
-	defer cancel()
-	if err = l.Listen(ctx, func(packet gopacket.Packet) {
-		if packet.Metadata().CaptureLength != 49 {
-			b.Errorf("expected packet length to be %d, got %d", 49, packet.Metadata().CaptureLength)
-		}
-		pckts++
-	}); err != nil {
-		b.Error(err)
-	}
-	b.Logf("%d/%d packets in %s", pckts, b.N, time.Since(now))
-}
-
-// used to benchmark sock engine
-var buf [1024]byte
-
-func init() {
-	for i := 0; i < len(buf); i++ {
-		buf[i] = 0xff
-	}
-}
-
-func handler(n, counter *int32) Handler {
-	return func(p gopacket.Packet) {
-		nn := int32(len(p.Data()))
-		atomic.AddInt32(n, nn)
-		atomic.AddInt32(counter, 1)
-	}
-}
-
-func BenchmarkPcap(b *testing.B) {
-	var err error
-	n := new(int32)
-	counter := new(int32)
-	l, err := NewListener(LoopBack.Name, 8000, "", EnginePcap, false)
-	if err != nil {
-		b.Error(err)
-		return
-	}
-	l.PcapOptions.BPFFilter = "udp dst port 8000 and host 127.0.0.1"
-	err = l.Activate()
-	if err != nil {
-		b.Error(err)
-		return
-	}
-	errCh := l.ListenBackground(context.Background(), handler(n, counter))
-	select {
-	case <-l.Reading:
-	case err = <-errCh:
-		b.Error(err)
-		return
-	}
-	var conn net.Conn
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		b.StopTimer()
-		conn, err = net.Dial("udp", "127.0.0.1:8000")
-		if err != nil {
-			b.Error(err)
-			return
-		}
-		b.StartTimer()
-		_, err = conn.Write(buf[:])
-		if err != nil {
-			b.Error(err)
-			return
-		}
-	}
-	b.ReportMetric(float64(atomic.LoadInt32(n)), "buf")
-	b.ReportMetric(float64(atomic.LoadInt32(counter)), "packets")
-}
-
-func BenchmarkRawSocket(b *testing.B) {
-	var err error
-	n := new(int32)
-	counter := new(int32)
-	l, err := NewListener(LoopBack.Name, 8000, "", EngineRawSocket, false)
-	if err != nil {
-		b.Error(err)
-		return
-	}
-	l.PcapOptions.BPFFilter = "udp dst port 8000 and host 127.0.0.1"
-	err = l.Activate()
-	if err != nil {
-		b.Error(err)
-		return
-	}
-	errCh := l.ListenBackground(context.Background(), handler(n, counter))
-	select {
-	case <-l.Reading:
-	case err = <-errCh:
-		b.Error(err)
-		return
-	}
-	var conn net.Conn
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		b.StopTimer()
-		conn, err = net.Dial("udp", "127.0.0.1:8000")
-		if err != nil {
-			b.Error(err)
-			return
-		}
-		b.StartTimer()
-		_, err = conn.Write(buf[:])
-		if err != nil {
-			b.Error(err)
-			return
-		}
-	}
-	b.ReportMetric(float64(atomic.LoadInt32(n)), "buf")
-	b.ReportMetric(float64(atomic.LoadInt32(counter)), "packets")
-}

capture/doc.go

@@ -22,7 +22,7 @@ if err := listener.Listen(context.Background(), handler); err != nil {
 	 // handle error
 }
 // or
-errCh := listener.ListenBackground(context.Background(), handler) // runs in the backgorund
+errCh := listener.ListenBackground(context.Background(), handler) // runs in the background
 select {
 case err := <- errCh:
 	// handle error

capture/dump.go

@@ -54,7 +54,7 @@ func NewWriterNanos(w io.Writer) *Writer {
 // NewWriter returns a new writer object, for writing packet data out
 // to the given writer.  If this is a new empty writer (as opposed to
 // an append), you must call WriteFileHeader before WritePacket.
-// Packet timestamps are written witn microsecond precision.
+// Packet timestamps are written with microsecond precision.
 //
 //  // Write a new file:
 //  f, _ := os.Create("/tmp/file.pcap")

capture/sock_linux.go

@@ -1,3 +1,5 @@
+// +build linux
+
 package capture
 
 import (
@@ -44,7 +46,23 @@ type SockRaw struct {
 }
 
 // NewSocket returns new M'maped sock_raw on packet version 2.
-func NewSocket(ifi net.Interface) (*SockRaw, error) {
+func NewSocket(pifi pcap.Interface) (*SockRaw, error) {
+	var ifi net.Interface
+
+	infs, _ := net.Interfaces()
+	found := false
+	for _, i := range infs {
+		if i.Name == pifi.Name {
+			ifi = i
+			found = true
+			break
+		}
+	}
+
+	if !found {
+		return nil, fmt.Errorf("can't find matching interface")
+	}
+
 	// sock create
 	fd, err := unix.Socket(unix.AF_PACKET, unix.SOCK_RAW, int(ETHALL))
 	if err != nil {
@@ -197,7 +215,7 @@ func (sock *SockRaw) GetSnapLen() int {
 func (sock *SockRaw) SetBPFFilter(expr string) error {
 	sock.mu.Lock()
 	defer sock.mu.Unlock()
-	if len(expr) == 0 {
+	if expr == "" {
 		return unix.SetsockoptInt(sock.fd, unix.SOL_SOCKET, unix.SO_DETACH_FILTER, 0)
 	}
 	filter, err := pcap.CompileBPFFilter(layers.LinkTypeEthernet, sock.snaplen, expr)
@@ -217,7 +235,7 @@ func (sock *SockRaw) SetBPFFilter(expr string) error {
 	return unix.SetsockoptSockFprog(sock.fd, unix.SOL_SOCKET, unix.SO_ATTACH_FILTER, fprog)
 }
 
-// SetPromiscuous sets promiscous mode to the required value. for better result capture on all interfaces instead.
+// SetPromiscuous sets promiscuous mode to the required value. for better result capture on all interfaces instead.
 // If it is enabled, traffic not destined for the interface will also be captured.
 func (sock *SockRaw) SetPromiscuous(b bool) error {
 	sock.mu.Lock()

capture/sock_others.go

@@ -4,10 +4,11 @@ package capture
 
 import (
 	"errors"
-	"net"
+
+	"github.com/google/gopacket/pcap"
 )
 
 // NewSocket returns new M'maped sock_raw on packet version 2.
-func NewSocket(_ net.Interface) (Socket, error) {
+func NewSocket(_ pcap.Interface) (Socket, error) {
 	return nil, errors.New("afpacket socket is only available on linux")
 }

elasticsearch.go

@@ -57,7 +57,7 @@ type ESRequestResponse struct {
 //
 // Proper format is: scheme://[userinfo@]host/index_name
 // userinfo is: user[:password]
-// net/url.Parse() does not fail if scheme is not provided but actualy does not
+// net/url.Parse() does not fail if scheme is not provided but actually does not
 // handle URI properly.
 // So we must 'validate' URI format to match requirements to use net/url.Parse()
 func parseURI(URI string) (err error, index string) {

emitter.go

@@ -152,7 +152,7 @@ func CopyMulty(src PluginReader, writers ...PluginWriter) error {
 				}
 			} else {
 				for _, dst := range writers {
-					if _, err := dst.PluginWrite(msg); err != nil {
+					if _, err := dst.PluginWrite(msg); err != nil && err != io.ErrClosedPipe {
 						return err
 					}
 				}

examples/middleware/echo.sh

@@ -8,7 +8,7 @@
 # 
 
 function log {
-    if $GOR_TEST != ""; then # if we are not testing
+    if [[ ! -v GOR_TEST ]]; then # if we are not testing
     # Logging to stderr, because stdout/stdin used for data transfer
     >&2 echo "[DEBUG][ECHO] $1"
     fi

examples/middleware/token_modifier.go

@@ -115,7 +115,7 @@ func encode(buf []byte) []byte {
 }
 
 func Debug(args ...interface{}) {
-	if os.Getenv("GOR_TEST") != "" { // if we are not testing
+	if os.Getenv("GOR_TEST") == "" { // if we are not testing
 		fmt.Fprint(os.Stderr, "[DEBUG][TOKEN-MOD] ")
 		fmt.Fprintln(os.Stderr, args...)
 	}

go.mod

@@ -1,6 +1,6 @@
 module github.com/buger/goreplay
 
-go 1.14
+go 1.15
 
 require (
 	github.com/Shopify/sarama v1.26.4
@@ -8,12 +8,13 @@ require (
 	github.com/aws/aws-sdk-go v1.33.2
 	github.com/bitly/go-hostpool v0.1.0 // indirect
 	github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 // indirect
-	github.com/google/gopacket v1.1.18
+	github.com/google/gopacket v1.1.20-0.20210429153827-3eaba0894325
 	github.com/klauspost/compress v1.10.10 // indirect
 	github.com/mattbaird/elastigo v0.0.0-20170123220020-2fe47fd29e4b
 	github.com/pierrec/lz4 v2.5.2+incompatible // indirect
 	github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0 // indirect
 	github.com/smartystreets/goconvey v1.6.4 // indirect
-	golang.org/x/net v0.0.0-20200707034311-ab3426394381 // indirect
+	github.com/stretchr/testify v1.5.1
+	golang.org/x/net v0.0.0-20200707034311-ab3426394381
 	golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd
 )

go.sum

@@ -10,7 +10,6 @@ github.com/bitly/go-hostpool v0.1.0 h1:XKmsF6k5el6xHG3WPJ8U0Ku/ye7njX7W81Ng7O2io
 github.com/bitly/go-hostpool v0.1.0/go.mod h1:4gOCgp6+NZnVqlKyZ/iBZFTAJKembaVENUpMkpg42fw=
 github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 h1:DDGfHa7BWjL4YnC6+E63dPcxHo2sUxDIu8g3QgEJdRY=
 github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4=
-github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -30,8 +29,8 @@ github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEW
 github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4=
 github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/gopacket v1.1.18 h1:lum7VRA9kdlvBi7/v2p7/zcbkduHaCH/SVVyurs7OpY=
-github.com/google/gopacket v1.1.18/go.mod h1:UdDNZ1OO62aGYVnPhxT1U6aI7ukYtA/kB8vaU0diBUM=
+github.com/google/gopacket v1.1.20-0.20210429153827-3eaba0894325 h1:YmIcZ5Var3BAQ64AW98Iiys5Ih4fiU0xK41+8isC5Ec=
+github.com/google/gopacket v1.1.20-0.20210429153827-3eaba0894325/go.mod h1:riddUzxTSBpJXk3qBHtYr4qOhFhT6k/1c0E3qkQjQpA=
 github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8=
 github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
 github.com/hashicorp/go-uuid v1.0.2 h1:cfejS+Tpcp13yd5nYHWDI6qVCny6wyX2Mt5SGur2IGE=
@@ -42,7 +41,6 @@ github.com/jmespath/go-jmespath v0.3.0 h1:OS12ieG61fsCg5+qLJ+SsW9NicxNkg3b25OyT2
 github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik=
 github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
 github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
-github.com/klauspost/compress v1.9.8 h1:VMAMUUOh+gaxKTMk+zqbjsSjsIcUcL/LF4o63i82QyA=
 github.com/klauspost/compress v1.9.8/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
 github.com/klauspost/compress v1.10.10 h1:a/y8CglcM7gLGYmlbP/stPE5sR3hbhFRUjCBfd/0B3I=
 github.com/klauspost/compress v1.10.10/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
@@ -71,33 +69,40 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4=
 github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
+github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE=
+github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU=
+github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0=
 github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I=
 github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/crypto v0.0.0-20200204104054-c9f3fb736b72 h1:+ELyKg6m8UBf0nPFSqD0mi7zUfwPyXo23HNjMnXPz7w=
+golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20200204104054-c9f3fb736b72/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
+golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20200202094626-16171245cfb2 h1:CCH4IOTTfewWjGOlSp+zGcjutRKlBEZQ6wTn8ozI/nI=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200707034311-ab3426394381 h1:VXak5I6aEWmAXeQjA+QSZzlgNrpq9mjcfDemuexIKsU=
 golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20190405154228-4b34438f7a67/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190606203320-7fc4e5ec1444/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200217220822-9197077df867/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884=
 golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
@@ -111,7 +116,6 @@ gopkg.in/jcmturner/gokrb5.v7 v7.5.0 h1:a9tsXlIDD9SKxotJMK3niV7rPZAJeX2aD/0yg3qlI
 gopkg.in/jcmturner/gokrb5.v7 v7.5.0/go.mod h1:l8VISx+WGYp+Fp7KRbsiUuXTTOnxIc3Tuvyavf11/WM=
 gopkg.in/jcmturner/rpc.v1 v1.1.0 h1:QHIUxTX1ISuAv9dD2wJ9HWQVuWDX/Zc0PfeC2tjc4rU=
 gopkg.in/jcmturner/rpc.v1 v1.1.0/go.mod h1:YIdkC4XfD6GXbzje11McwsDuOlZQSb9W4vfLvuNnlv8=
-gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=
 gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

gor.go

@@ -3,15 +3,16 @@
 package main
 
 import (
+	"expvar"
 	"flag"
+	"fmt"
 	"log"
 	"net/http"
 	"net/http/httputil"
-	_ "net/http/pprof"
+	httppptof "net/http/pprof"
 	"os"
 	"os/signal"
 	"runtime"
-	_ "runtime/debug"
 	"runtime/pprof"
 	"syscall"
 	"time"
@@ -22,8 +23,42 @@ var (
 	memprofile = flag.String("memprofile", "", "write memory profile to this file")
 )
 
-func loggingMiddleware(next http.Handler) http.Handler {
+func init() {
+	var defaultServeMux http.ServeMux
+	http.DefaultServeMux = &defaultServeMux
+
+	http.HandleFunc("/debug/vars", func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "application/json; charset=utf-8")
+		fmt.Fprintf(w, "{\n")
+		first := true
+		expvar.Do(func(kv expvar.KeyValue) {
+			if kv.Key == "memstats" || kv.Key == "cmdline" {
+				return
+			}
+
+			if !first {
+				fmt.Fprintf(w, ",\n")
+			}
+			first = false
+			fmt.Fprintf(w, "%q: %s", kv.Key, kv.Value)
+		})
+		fmt.Fprintf(w, "\n}\n")
+	})
+
+	http.HandleFunc("/debug/pprof/", httppptof.Index)
+	http.HandleFunc("/debug/pprof/cmdline", httppptof.Cmdline)
+	http.HandleFunc("/debug/pprof/profile", httppptof.Profile)
+	http.HandleFunc("/debug/pprof/symbol", httppptof.Symbol)
+	http.HandleFunc("/debug/pprof/trace", httppptof.Trace)
+}
+
+func loggingMiddleware(addr string, next http.Handler) http.Handler {
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path == "/loop" {
+			_, err := http.Get("http://" + addr)
+			log.Println(err)
+		}
+
 		rb, _ := httputil.DumpRequest(r, false)
 		log.Println(string(rb))
 		next.ServeHTTP(w, r)
@@ -31,7 +66,7 @@ func loggingMiddleware(next http.Handler) http.Handler {
 }
 
 func main() {
-	if len(os.Getenv("GOMAXPROCS")) == 0 {
+	if os.Getenv("GOMAXPROCS") == "" {
 		runtime.GOMAXPROCS(runtime.NumCPU() * 2)
 	}
 
@@ -45,7 +80,7 @@ func main() {
 
 		Debug(0, "Started example file server for current directory on address ", args[1])
 
-		log.Fatal(http.ListenAndServe(args[1], loggingMiddleware(http.FileServer(http.Dir(dir)))))
+		log.Fatal(http.ListenAndServe(args[1], loggingMiddleware(args[1], http.FileServer(http.Dir(dir)))))
 	} else {
 		flag.Parse()
 		checkSettings()
@@ -94,7 +129,6 @@ func main() {
 	}
 	emitter.Close()
 	os.Exit(exit)
-
 }
 
 func profileCPU(cpuprofile string) {

input_file.go

@@ -4,9 +4,12 @@ import (
 	"bufio"
 	"bytes"
 	"compress/gzip"
+	"container/heap"
 	"errors"
+	"expvar"
 	"fmt"
 	"io"
+	"math"
 	"os"
 	"path/filepath"
 	"strconv"
@@ -20,27 +23,68 @@ import (
 	"github.com/aws/aws-sdk-go/service/s3"
 )
 
-type fileInputReader struct {
-	reader    *bufio.Reader
+type filePayload struct {
 	data      []byte
-	file      io.ReadCloser
 	timestamp int64
-	closed    int32 // Value of 0 indicates that the file is still open.
-	s3        bool
 }
 
-func (f *fileInputReader) parseNext() error {
+// An IntHeap is a min-heap of ints.
+type payloadQueue struct {
+	sync.RWMutex
+	s []*filePayload
+}
+
+func (h payloadQueue) Len() int           { return len(h.s) }
+func (h payloadQueue) Less(i, j int) bool { return h.s[i].timestamp < h.s[j].timestamp }
+func (h payloadQueue) Swap(i, j int)      { h.s[i], h.s[j] = h.s[j], h.s[i] }
+
+func (h *payloadQueue) Push(x interface{}) {
+	// Push and Pop use pointer receivers because they modify the slice's length,
+	// not just its contents.
+	h.s = append(h.s, x.(*filePayload))
+}
+
+func (h *payloadQueue) Pop() interface{} {
+	old := h.s
+	n := len(old)
+	x := old[n-1]
+	h.s = old[0 : n-1]
+	return x
+}
+
+func (h payloadQueue) Idx(i int) *filePayload {
+	return h.s[i]
+}
+
+type fileInputReader struct {
+	reader    *bufio.Reader
+	file      io.ReadCloser
+	closed    int32 // Value of 0 indicates that the file is still open.
+	s3        bool
+	queue     payloadQueue
+	readDepth int
+}
+
+func (f *fileInputReader) parse(init chan struct{}) error {
 	payloadSeparatorAsBytes := []byte(payloadSeparator)
 	var buffer bytes.Buffer
+	var initialized bool
+
 	for {
 		line, err := f.reader.ReadBytes('\n')
 
 		if err != nil {
 			if err != io.EOF {
 				Debug(1, err)
-			} else {
-				f.Close()
 			}
+
+			f.Close()
+
+			if !initialized {
+				close(init)
+				initialized = true
+			}
+
 			return err
 		}
 
@@ -48,21 +92,51 @@ func (f *fileInputReader) parseNext() error {
 			asBytes := buffer.Bytes()
 			meta := payloadMeta(asBytes)
 
-			f.timestamp, _ = strconv.ParseInt(string(meta[2]), 10, 64)
-			f.data = asBytes[:len(asBytes)-1]
+			timestamp, _ := strconv.ParseInt(string(meta[2]), 10, 64)
+			data := asBytes[:len(asBytes)-1]
 
-			return nil
+			f.queue.Lock()
+			heap.Push(&f.queue, &filePayload{
+				timestamp: timestamp,
+				data:      data,
+			})
+			f.queue.Unlock()
+
+			for {
+				if f.queue.Len() < f.readDepth {
+					break
+				}
+
+				if !initialized {
+					close(init)
+					initialized = true
+				}
+
+				time.Sleep(100 * time.Millisecond)
+			}
+
+			buffer = bytes.Buffer{}
+			continue
 		}
 
 		buffer.Write(line)
 	}
-
 }
 
-func (f *fileInputReader) ReadPayload() []byte {
-	defer f.parseNext()
+func (f *fileInputReader) wait() {
+	for {
+		if atomic.LoadInt32(&f.closed) == 1 {
+			return
+		}
 
-	return f.data
+		if f.queue.Len() > 0 {
+			return
+		}
+
+		time.Sleep(100 * time.Millisecond)
+	}
+
+	return
 }
 
 // Close closes this plugin
@@ -75,7 +149,7 @@ func (f *fileInputReader) Close() error {
 	return nil
 }
 
-func newFileInputReader(path string) *fileInputReader {
+func newFileInputReader(path string, readDepth int) *fileInputReader {
 	var file io.ReadCloser
 	var err error
 
@@ -90,7 +164,7 @@ func newFileInputReader(path string) *fileInputReader {
 		return nil
 	}
 
-	r := &fileInputReader{file: file, closed: 0}
+	r := &fileInputReader{file: file, closed: 0, readDepth: readDepth}
 	if strings.HasSuffix(path, ".gz") {
 		gzReader, err := gzip.NewReader(file)
 		if err != nil {
@@ -102,7 +176,11 @@ func newFileInputReader(path string) *fileInputReader {
 		r.reader = bufio.NewReader(file)
 	}
 
-	r.parseNext()
+	heap.Init(&r.queue)
+
+	init := make(chan struct{})
+	go r.parse(init)
+	<-init
 
 	return r
 }
@@ -116,16 +194,25 @@ type FileInput struct {
 	readers     []*fileInputReader
 	speedFactor float64
 	loop        bool
+	readDepth   int
+	dryRun      bool
+	maxWait     time.Duration
+
+	stats *expvar.Map
 }
 
 // NewFileInput constructor for FileInput. Accepts file path as argument.
-func NewFileInput(path string, loop bool) (i *FileInput) {
+func NewFileInput(path string, loop bool, readDepth int, maxWait time.Duration, dryRun bool) (i *FileInput) {
 	i = new(FileInput)
 	i.data = make(chan []byte, 1000)
 	i.exit = make(chan bool)
 	i.path = path
 	i.speedFactor = 1
 	i.loop = loop
+	i.readDepth = readDepth
+	i.stats = expvar.NewMap("file-" + path)
+	i.dryRun = dryRun
+	i.maxWait = maxWait
 
 	if err := i.init(); err != nil {
 		return
@@ -162,11 +249,9 @@ func (i *FileInput) init() (err error) {
 		for _, c := range resp.Contents {
 			matches = append(matches, "s3://"+bucket+"/"+(*c.Key))
 		}
-	} else {
-		if matches, err = filepath.Glob(i.path); err != nil {
-			Debug(0, "[INPUT-FILE] Wrong file pattern", i.path, err)
-			return
-		}
+	} else if matches, err = filepath.Glob(i.path); err != nil {
+		Debug(0, "[INPUT-FILE] Wrong file pattern", i.path, err)
+		return
 	}
 
 	if len(matches) == 0 {
@@ -177,9 +262,11 @@ func (i *FileInput) init() (err error) {
 	i.readers = make([]*fileInputReader, len(matches))
 
 	for idx, p := range matches {
-		i.readers[idx] = newFileInputReader(p)
+		i.readers[idx] = newFileInputReader(p, i.readDepth)
 	}
 
+	i.stats.Add("reader_count", int64(len(matches)))
+
 	return nil
 }
 
@@ -190,6 +277,7 @@ func (i *FileInput) PluginRead() (*Message, error) {
 	case <-i.exit:
 		return nil, ErrorStopped
 	case buf := <-i.data:
+		i.stats.Add("read_from", 1)
 		msg.Meta, msg.Data = payloadMetaWithBody(buf)
 		return &msg, nil
 	}
@@ -202,11 +290,17 @@ func (i *FileInput) String() string {
 // Find reader with smallest timestamp e.g next payload in row
 func (i *FileInput) nextReader() (next *fileInputReader) {
 	for _, r := range i.readers {
-		if r == nil || atomic.LoadInt32(&r.closed) != 0 {
+		if r == nil {
 			continue
 		}
 
-		if next == nil || r.timestamp < next.timestamp {
+		r.wait()
+
+		if r.queue.Len() == 0 {
+			continue
+		}
+
+		if next == nil || r.queue.Idx(0).timestamp < next.queue.Idx(0).timestamp {
 			next = r
 			continue
 		}
@@ -218,6 +312,11 @@ func (i *FileInput) nextReader() (next *fileInputReader) {
 func (i *FileInput) emit() {
 	var lastTime int64 = -1
 
+	var maxWait, firstWait, minWait int64
+	minWait = math.MaxInt64
+
+	i.stats.Add("negative_wait", 0)
+
 	for {
 		select {
 		case <-i.exit:
@@ -237,17 +336,48 @@ func (i *FileInput) emit() {
 			}
 		}
 
+		reader.queue.RLock()
+		payload := heap.Pop(&reader.queue).(*filePayload)
+		i.stats.Add("total_counter", 1)
+		i.stats.Add("total_bytes", int64(len(payload.data)))
+		reader.queue.RUnlock()
+
 		if lastTime != -1 {
-			diff := reader.timestamp - lastTime
-			lastTime = reader.timestamp
+			diff := payload.timestamp - lastTime
+
+			if firstWait == 0 {
+				firstWait = diff
+			}
 
 			if i.speedFactor != 1 {
 				diff = int64(float64(diff) / i.speedFactor)
 			}
 
-			time.Sleep(time.Duration(diff))
+			if i.maxWait > 0 && diff > int64(i.maxWait) {
+				diff = int64(i.maxWait)
+			}
+
+			if diff >= 0 {
+				lastTime = payload.timestamp
+
+				if !i.dryRun {
+					time.Sleep(time.Duration(diff))
+				}
+
+				i.stats.Add("total_wait", diff)
+
+				if diff > maxWait {
+					maxWait = diff
+				}
+
+				if diff < minWait {
+					minWait = diff
+				}
+			} else {
+				i.stats.Add("negative_wait", 1)
+			}
 		} else {
-			lastTime = reader.timestamp
+			lastTime = payload.timestamp
 		}
 
 		// Recheck if we have exited since last check.
@@ -255,12 +385,30 @@ func (i *FileInput) emit() {
 		case <-i.exit:
 			return
 		default:
-			i.data <- reader.ReadPayload()
+			if !i.dryRun {
+				i.data <- payload.data
+			}
 		}
 	}
 
+	i.stats.Set("first_wait", time.Duration(firstWait))
+	i.stats.Set("max_wait", time.Duration(maxWait))
+	i.stats.Set("min_wait", time.Duration(minWait))
+
 	Debug(0, fmt.Sprintf("[INPUT-FILE] FileInput: end of file '%s'\n", i.path))
 
+	if i.dryRun {
+		fmt.Printf("Records found: %v\nFiles processed: %v\nBytes processed: %v\nMax wait: %v\nMin wait: %v\nFirst wait: %v\nIt will take `%v` to replay at current speed.\nFound %v records with out of order timestamp\n",
+			i.stats.Get("total_counter"),
+			i.stats.Get("reader_count"),
+			i.stats.Get("total_bytes"),
+			i.stats.Get("max_wait"),
+			i.stats.Get("min_wait"),
+			i.stats.Get("first_wait"),
+			time.Duration(i.stats.Get("total_wait").(*expvar.Int).Value()),
+			i.stats.Get("negative_wait"),
+		)
+	}
 }
 
 // Close closes this plugin

input_file_test.go

@@ -29,10 +29,9 @@ func TestInputFileWithGET(t *testing.T) {
 	// The read request should match the original request
 	if err != nil {
 		t.Error(err)
-	} else {
-		if !expectedCaptureFile.PayloadsEqual(readPayloads) {
-			t.Error("Request read back from file should match")
-		}
+	} else if !expectedCaptureFile.PayloadsEqual(readPayloads) {
+		t.Error("Request read back from file should match")
+
 	}
 }
 
@@ -53,10 +52,9 @@ func TestInputFileWithPayloadLargerThan64Kb(t *testing.T) {
 	// The read request should match the original request
 	if err != nil {
 		t.Error(err)
-	} else {
-		if !expectedCaptureFile.PayloadsEqual(readPayloads) {
-			t.Error("Request read back from file should match")
-		}
+	} else if !expectedCaptureFile.PayloadsEqual(readPayloads) {
+		t.Error("Request read back from file should match")
+
 	}
 
 }
@@ -82,10 +80,9 @@ func TestInputFileWithGETAndPOST(t *testing.T) {
 	// The read requests should match the original request
 	if err != nil {
 		t.Error(err)
-	} else {
-		if !expectedCaptureFile.PayloadsEqual(readPayloads) {
-			t.Error("Request read back from file should match")
-		}
+	} else if !expectedCaptureFile.PayloadsEqual(readPayloads) {
+		t.Error("Request read back from file should match")
+
 	}
 
 }
@@ -107,7 +104,7 @@ func TestInputFileMultipleFilesWithRequestsOnly(t *testing.T) {
 	file2.Write([]byte(payloadSeparator))
 	file2.Close()
 
-	input := NewFileInput(fmt.Sprintf("/tmp/%d*", rnd), false)
+	input := NewFileInput(fmt.Sprintf("/tmp/%d*", rnd), false, 100, 0, false)
 
 	for i := '1'; i <= '4'; i++ {
 		msg, _ := input.PluginRead()
@@ -133,7 +130,7 @@ func TestInputFileRequestsWithLatency(t *testing.T) {
 	file.Write([]byte("1 3 250000000\nrequest3"))
 	file.Write([]byte(payloadSeparator))
 
-	input := NewFileInput(fmt.Sprintf("/tmp/%d", rnd), false)
+	input := NewFileInput(fmt.Sprintf("/tmp/%d", rnd), false, 100, 0, false)
 
 	start := time.Now().UnixNano()
 	for i := 0; i < 3; i++ {
@@ -173,7 +170,7 @@ func TestInputFileMultipleFilesWithRequestsAndResponses(t *testing.T) {
 	file2.Write([]byte(payloadSeparator))
 	file2.Close()
 
-	input := NewFileInput(fmt.Sprintf("/tmp/%d*", rnd), false)
+	input := NewFileInput(fmt.Sprintf("/tmp/%d*", rnd), false, 100, 0, false)
 
 	for i := '1'; i <= '4'; i++ {
 		msg, _ := input.PluginRead()
@@ -201,7 +198,7 @@ func TestInputFileLoop(t *testing.T) {
 	file.Write([]byte(payloadSeparator))
 	file.Close()
 
-	input := NewFileInput(fmt.Sprintf("/tmp/%d", rnd), true)
+	input := NewFileInput(fmt.Sprintf("/tmp/%d", rnd), true, 100, 0, false)
 
 	// Even if we have just 2 requests in file, it should indifinitly loop
 	for i := 0; i < 1000; i++ {
@@ -229,7 +226,7 @@ func TestInputFileCompressed(t *testing.T) {
 	name2 := output2.file.Name()
 	output2.Close()
 
-	input := NewFileInput(fmt.Sprintf("/tmp/%d*", rnd), false)
+	input := NewFileInput(fmt.Sprintf("/tmp/%d*", rnd), false, 100, 0, false)
 	for i := 0; i < 2000; i++ {
 		input.PluginRead()
 	}
@@ -329,7 +326,7 @@ func CreateCaptureFile(requestGenerator *RequestGenerator) *CaptureFile {
 func ReadFromCaptureFile(captureFile *os.File, count int, callback writeCallback) (err error) {
 	wg := new(sync.WaitGroup)
 
-	input := NewFileInput(captureFile.Name(), false)
+	input := NewFileInput(captureFile.Name(), false, 100, 0, false)
 	output := NewTestOutput(func(msg *Message) {
 		callback(msg)
 		wg.Done()

input_kafka.go

@@ -9,7 +9,7 @@ import (
 	"github.com/Shopify/sarama/mocks"
 )
 
-// KafkaInput is used for recieving Kafka messages and
+// KafkaInput is used for receiving Kafka messages and
 // transforming them into HTTP payloads.
 type KafkaInput struct {
 	config    *InputKafkaConfig

input_raw.go

@@ -6,6 +6,7 @@ import (
 	"log"
 	"net"
 	"strconv"
+	"strings"
 	"sync"
 	"time"
 
@@ -15,53 +16,20 @@ import (
 	"github.com/buger/goreplay/tcp"
 )
 
-// TCPProtocol is a number to indicate type of protocol
-type TCPProtocol uint8
-
-const (
-	// ProtocolHTTP ...
-	ProtocolHTTP TCPProtocol = iota
-	// ProtocolBinary ...
-	ProtocolBinary
-)
-
-// Set is here so that TCPProtocol can implement flag.Var
-func (protocol *TCPProtocol) Set(v string) error {
-	switch v {
-	case "", "http":
-		*protocol = ProtocolHTTP
-	case "binary":
-		*protocol = ProtocolBinary
-	default:
-		return fmt.Errorf("unsupported protocol %s", v)
-	}
-	return nil
-}
-
-func (protocol *TCPProtocol) String() string {
-	switch *protocol {
-	case ProtocolBinary:
-		return "binary"
-	case ProtocolHTTP:
-		return "http"
-	default:
-		return ""
-	}
-}
-
 // RAWInputConfig represents configuration that can be applied on raw input
 type RAWInputConfig struct {
 	capture.PcapOptions
-	Expire         time.Duration      `json:"input-raw-expire"`
-	CopyBufferSize size.Size          `json:"copy-buffer-size"`
-	Engine         capture.EngineType `json:"input-raw-engine"`
-	TrackResponse  bool               `json:"input-raw-track-response"`
-	Protocol       TCPProtocol        `json:"input-raw-protocol"`
-	RealIPHeader   string             `json:"input-raw-realip-header"`
-	Stats          bool               `json:"input-raw-stats"`
-	quit           chan bool          // Channel used only to indicate goroutine should shutdown
-	host           string
-	port           uint16
+	Expire          time.Duration      `json:"input-raw-expire"`
+	CopyBufferSize  size.Size          `json:"copy-buffer-size"`
+	Engine          capture.EngineType `json:"input-raw-engine"`
+	TrackResponse   bool               `json:"input-raw-track-response"`
+	Protocol        tcp.TCPProtocol    `json:"input-raw-protocol"`
+	RealIPHeader    string             `json:"input-raw-realip-header"`
+	Stats           bool               `json:"input-raw-stats"`
+	AllowIncomplete bool               `json:"input-raw-allow-incomplete"`
+	quit            chan bool          // Channel used only to indicate goroutine should shutdown
+	host            string
+	ports           []uint16
 }
 
 // RAWInput used for intercepting traffic for given address
@@ -70,32 +38,38 @@ type RAWInput struct {
 	RAWInputConfig
 	messageStats   []tcp.Stats
 	listener       *capture.Listener
-	message        chan *tcp.Message
+	messageParser  *tcp.MessageParser
 	cancelListener context.CancelFunc
+	closed         bool
 }
 
 // NewRAWInput constructor for RAWInput. Accepts raw input config as arguments.
 func NewRAWInput(address string, config RAWInputConfig) (i *RAWInput) {
 	i = new(RAWInput)
 	i.RAWInputConfig = config
-	i.message = make(chan *tcp.Message, 1000)
 	i.quit = make(chan bool)
-	var host, _port string
-	var err error
-	var port int
-	host, _port, err = net.SplitHostPort(address)
+
+	host, _ports, err := net.SplitHostPort(address)
 	if err != nil {
 		log.Fatalf("input-raw: error while parsing address: %s", err)
 	}
-	if _port != "" {
-		port, err = strconv.Atoi(_port)
+
+	var ports []uint16
+	if _ports != "" {
+		portsStr := strings.Split(_ports, ",")
+
+		for _, portStr := range portsStr {
+			port, err := strconv.Atoi(strings.TrimSpace(portStr))
+			if err != nil {
+				log.Fatalf("parsing port error: %v", err)
+			}
+			ports = append(ports, uint16(port))
+
+		}
 	}
 
-	if err != nil {
-		log.Fatalf("parsing port error: %v", err)
-	}
 	i.host = host
-	i.port = uint16(port)
+	i.ports = ports
 
 	i.listen(address)
 
@@ -109,11 +83,12 @@ func (i *RAWInput) PluginRead() (*Message, error) {
 	select {
 	case <-i.quit:
 		return nil, ErrorStopped
-	case msgTCP = <-i.message:
+	case msgTCP = <-i.listener.Messages():
 		msg.Data = msgTCP.Data()
 	}
+
 	var msgType byte = ResponsePayload
-	if msgTCP.IsIncoming {
+	if msgTCP.Direction == tcp.DirIncoming {
 		msgType = RequestPayload
 		if i.RealIPHeader != "" {
 			msg.Data = proto.SetHeader(msg.Data, []byte(i.RealIPHeader), []byte(msgTCP.SrcAddr))
@@ -139,7 +114,7 @@ func (i *RAWInput) PluginRead() (*Message, error) {
 
 func (i *RAWInput) listen(address string) {
 	var err error
-	i.listener, err = capture.NewListener(i.host, i.port, "", i.Engine, i.TrackResponse)
+	i.listener, err = capture.NewListener(i.host, i.ports, "", i.Engine, i.Protocol, i.TrackResponse, i.Expire, i.AllowIncomplete)
 	if err != nil {
 		log.Fatal(err)
 	}
@@ -148,26 +123,20 @@ func (i *RAWInput) listen(address string) {
 	if err != nil {
 		log.Fatal(err)
 	}
-	pool := tcp.NewMessagePool(i.CopyBufferSize, i.Expire, Debug, i.handler)
-	pool.End = endHint
-	pool.Start = startHint
+
 	var ctx context.Context
 	ctx, i.cancelListener = context.WithCancel(context.Background())
-	errCh := i.listener.ListenBackground(ctx, pool.Handler)
-	select {
-	case err := <-errCh:
-		log.Fatal(err)
-	case <-i.listener.Reading:
-		Debug(1, i)
-	}
-}
-
-func (i *RAWInput) handler(m *tcp.Message) {
-	i.message <- m
+	errCh := i.listener.ListenBackground(ctx)
+	<-i.listener.Reading
+	Debug(1, i)
+	go func() {
+		<-errCh // the listener closed voluntarily
+		i.Close()
+	}()
 }
 
 func (i *RAWInput) String() string {
-	return fmt.Sprintf("Intercepting traffic from: %s:%d", i.host, i.port)
+	return fmt.Sprintf("Intercepting traffic from: %s:%s", i.host, strings.Join(strings.Fields(fmt.Sprint(i.ports)), ","))
 }
 
 // GetStats returns the stats so far and reset the stats
@@ -182,8 +151,14 @@ func (i *RAWInput) GetStats() []tcp.Stats {
 
 // Close closes the input raw listener
 func (i *RAWInput) Close() error {
+	i.Lock()
+	defer i.Unlock()
+	if i.closed {
+		return nil
+	}
 	i.cancelListener()
 	close(i.quit)
+	i.closed = true
 	return nil
 }
 
@@ -195,15 +170,3 @@ func (i *RAWInput) addStats(mStats tcp.Stats) {
 	i.messageStats = append(i.messageStats, mStats)
 	i.Unlock()
 }
-
-func startHint(pckt *tcp.Packet) (isIncoming, isOutgoing bool) {
-	isIncoming = proto.HasRequestTitle(pckt.Payload)
-	if isIncoming {
-		return
-	}
-	return false, proto.HasResponseTitle(pckt.Payload)
-}
-
-func endHint(m *tcp.Message) bool {
-	return proto.HasFullPayload(m.Data())
-}

input_raw_test.go

@@ -16,6 +16,7 @@ import (
 
 	"github.com/buger/goreplay/capture"
 	"github.com/buger/goreplay/proto"
+	"github.com/buger/goreplay/tcp"
 )
 
 const testRawExpire = time.Millisecond * 200
@@ -43,7 +44,7 @@ func TestRAWInputIPv4(t *testing.T) {
 	conf := RAWInputConfig{
 		Engine:        capture.EnginePcap,
 		Expire:        0,
-		Protocol:      ProtocolHTTP,
+		Protocol:      tcp.ProtocolHTTP,
 		TrackResponse: true,
 		RealIPHeader:  "X-Real-IP",
 	}
@@ -58,6 +59,7 @@ func TestRAWInputIPv4(t *testing.T) {
 		} else {
 			respCounter++
 		}
+
 		wg.Done()
 	})
 
@@ -71,14 +73,18 @@ func TestRAWInputIPv4(t *testing.T) {
 	emitter := NewEmitter()
 	defer emitter.Close()
 	go emitter.Start(plugins, Settings.Middleware)
+
+	// time.Sleep(time.Second)
 	for i := 0; i < 1; i++ {
 		wg.Add(2)
 		_, err = http.Get(addr)
+
 		if err != nil {
 			t.Error(err)
 			return
 		}
 	}
+
 	wg.Wait()
 	const want = 10
 	if reqCounter != respCounter && reqCounter != want {
@@ -108,7 +114,7 @@ func TestRAWInputNoKeepAlive(t *testing.T) {
 	conf := RAWInputConfig{
 		Engine:        capture.EnginePcap,
 		Expire:        testRawExpire,
-		Protocol:      ProtocolHTTP,
+		Protocol:      tcp.ProtocolHTTP,
 		TrackResponse: true,
 	}
 	input := NewRAWInput(":"+port, conf)
@@ -173,7 +179,7 @@ func TestRAWInputIPv6(t *testing.T) {
 	var respCounter, reqCounter int64
 	conf := RAWInputConfig{
 		Engine:        capture.EnginePcap,
-		Protocol:      ProtocolHTTP,
+		Protocol:      tcp.ProtocolHTTP,
 		TrackResponse: true,
 	}
 	input := NewRAWInput(originAddr, conf)
@@ -228,10 +234,11 @@ func TestInputRAWChunkedEncoding(t *testing.T) {
 
 	originAddr := strings.Replace(origin.Listener.Addr().String(), "[::]", "127.0.0.1", -1)
 	conf := RAWInputConfig{
-		Engine:        capture.EnginePcap,
-		Expire:        time.Second,
-		Protocol:      ProtocolHTTP,
-		TrackResponse: true,
+		Engine:          capture.EnginePcap,
+		Expire:          time.Second,
+		Protocol:        tcp.ProtocolHTTP,
+		TrackResponse:   true,
+		AllowIncomplete: true,
 	}
 	input := NewRAWInput(originAddr, conf)
 
@@ -309,7 +316,7 @@ func BenchmarkRAWInputWithReplay(b *testing.B) {
 	conf := RAWInputConfig{
 		Engine:        capture.EnginePcap,
 		Expire:        testRawExpire,
-		Protocol:      ProtocolHTTP,
+		Protocol:      tcp.ProtocolHTTP,
 		TrackResponse: true,
 	}
 	input := NewRAWInput(originAddr, conf)

kafka.go

@@ -51,7 +51,6 @@ type KafkaMessage struct {
 	ReqHeaders map[string]string `json:"Req_Headers,omitempty"`
 }
 
-
 // NewTLSConfig loads TLS certificates
 func NewTLSConfig(clientCertFile, clientKeyFile, caCertFile string) (*tls.Config, error) {
 	tlsConfig := tls.Config{}
@@ -63,7 +62,7 @@ func NewTLSConfig(clientCertFile, clientKeyFile, caCertFile string) (*tls.Config
 		return &tlsConfig, errors.New("missing TLS client certificate in kafka")
 	}
 	// Load client cert
-	if  (clientCertFile != "") && (clientKeyFile != "") {
+	if (clientCertFile != "") && (clientKeyFile != "") {
 		cert, err := tls.LoadX509KeyPair(clientCertFile, clientKeyFile)
 		if err != nil {
 			return &tlsConfig, err

middleware.go

@@ -6,10 +6,11 @@ import (
 	"encoding/hex"
 	"fmt"
 	"io"
-	"log"
 	"os"
 	"os/exec"
 	"strings"
+	"sync"
+	"syscall"
 )
 
 // Middleware represents a middleware object
@@ -20,6 +21,8 @@ type Middleware struct {
 	Stdout        io.Reader
 	commandCancel context.CancelFunc
 	stop          chan bool // Channel used only to indicate goroutine should shutdown
+	closed        bool
+	mu            sync.RWMutex
 }
 
 // NewMiddleware returns new middleware
@@ -42,16 +45,19 @@ func NewMiddleware(command string) *Middleware {
 	go m.read(m.Stdout)
 
 	go func() {
-		err := cmd.Start()
-
-		if err != nil {
-			log.Fatal(err)
+		defer m.Close()
+		var err error
+		if err = cmd.Start(); err == nil {
+			err = cmd.Wait()
 		}
-
-		err = cmd.Wait()
-
 		if err != nil {
-			log.Fatal(err)
+			if e, ok := err.(*exec.ExitError); ok {
+				status := e.Sys().(syscall.WaitStatus)
+				if status.Signal() == syscall.SIGKILL /*killed or context canceld */ {
+					return
+				}
+			}
+			Debug(0, fmt.Sprintf("[MIDDLEWARE] command[%q] error: %q", command, err.Error()))
 		}
 	}()
 
@@ -60,7 +66,7 @@ func NewMiddleware(command string) *Middleware {
 
 // ReadFrom start a worker to read from this plugin
 func (m *Middleware) ReadFrom(plugin PluginReader) {
-	Debug(2, "[MIDDLEWARE-MASTER] Starting reading from", plugin)
+	Debug(2, fmt.Sprintf("[MIDDLEWARE] command[%q] Starting reading from %q", m.command, plugin))
 	go m.copy(m.Stdin, plugin)
 }
 
@@ -75,16 +81,26 @@ func (m *Middleware) copy(to io.Writer, from PluginReader) {
 		if msg == nil || len(msg.Data) == 0 {
 			continue
 		}
-
+		buf = msg.Data
 		if Settings.PrettifyHTTP {
 			buf = prettifyHTTP(msg.Data)
 		}
-		dst = make([]byte, len(buf)*2+1)
-		hex.Encode(dst, buf)
-		dst[len(buf)*2] = '\n'
-
-		to.Write(dst)
+		dstLen := (len(buf)+len(msg.Meta))*2 + 1
+		// if enough space was previously allocated use it instead
+		if dstLen > len(dst) {
+			dst = make([]byte, dstLen)
+		}
+		n := hex.Encode(dst, msg.Meta)
+		n += hex.Encode(dst[n:], buf)
+		dst[n] = '\n'
 
+		n, err = to.Write(dst[:n+1])
+		if err == nil {
+			continue
+		}
+		if m.isClosed() {
+			return
+		}
 	}
 }
 
@@ -92,19 +108,16 @@ func (m *Middleware) read(from io.Reader) {
 	reader := bufio.NewReader(from)
 	var line []byte
 	var e error
-
 	for {
 		if line, e = reader.ReadBytes('\n'); e != nil {
-			if e == io.EOF {
-				continue
-			} else {
-				break
+			if m.isClosed() {
+				return
 			}
+			continue
 		}
-
-		buf := make([]byte, len(line)/2-1)
+		buf := make([]byte, (len(line)-1)/2)
 		if _, err := hex.Decode(buf, line[:len(line)-1]); err != nil {
-			Debug(0, fmt.Sprintf("[MIDDLEWARE] failed to decode err: %q", err))
+			Debug(0, fmt.Sprintf("[MIDDLEWARE] command[%q] failed to decode err: %q", m.command, err))
 			continue
 		}
 		var msg Message
@@ -116,7 +129,6 @@ func (m *Middleware) read(from io.Reader) {
 		}
 	}
 
-	return
 }
 
 // PluginRead reads message from this plugin
@@ -134,9 +146,21 @@ func (m *Middleware) String() string {
 	return fmt.Sprintf("Modifying traffic using %q command", m.command)
 }
 
+func (m *Middleware) isClosed() bool {
+	m.mu.RLock()
+	defer m.mu.RUnlock()
+	return m.closed
+}
+
 // Close closes this plugin
 func (m *Middleware) Close() error {
+	if m.isClosed() {
+		return nil
+	}
+	m.mu.Lock()
+	defer m.mu.Unlock()
 	m.commandCancel()
 	close(m.stop)
+	m.closed = true
 	return nil
 }

middleware_test.go

@@ -1,225 +1,154 @@
 package main
 
-// import (
-// 	"bytes"
-// 	"crypto/rand"
-// 	"encoding/hex"
-// 	"io"
-// 	"net/http"
-// 	"net/http/httptest"
-// 	"strings"
-// 	"sync"
-// 	"testing"
-// 	"time"
+import (
+	"bytes"
+	"context"
+	"os"
+	"os/exec"
+	"strings"
+	"sync/atomic"
+	"syscall"
+	"testing"
 
-// 	"github.com/buger/goreplay/capture"
-// 	"github.com/buger/goreplay/proto"
-// )
+	"github.com/buger/goreplay/proto"
+)
 
-// type fakeServiceCb func(string, int, []byte)
+const echoSh = "./examples/middleware/echo.sh"
+const tokenModifier = "go run ./examples/middleware/token_modifier.go"
 
-// // Simple service that generate token on request, and require this token for accesing to secure area
-// func NewFakeSecureService(wg *sync.WaitGroup, cb fakeServiceCb) *httptest.Server {
-// 	active_tokens := make([]string, 0)
-// 	var mu sync.Mutex
+var noDebug = append(syscall.Environ(), "GOR_TEST=1")
 
-// 	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
-// 		mu.Lock()
-// 		defer mu.Unlock()
+func initMiddleware(cmd *exec.Cmd, cancl context.CancelFunc, l PluginReader, c func(error)) *Middleware {
+	var m Middleware
+	m.data = make(chan *Message, 1000)
+	m.stop = make(chan bool)
+	m.commandCancel = cancl
+	m.Stdout, _ = cmd.StdoutPipe()
+	m.Stdin, _ = cmd.StdinPipe()
+	cmd.Stderr = os.Stderr
+	go m.read(m.Stdout)
+	go func() {
+		defer m.Close()
+		var err error
+		if err = cmd.Start(); err == nil {
+			err = cmd.Wait()
+		}
+		if err != nil {
+			c(err)
+		}
+	}()
+	m.ReadFrom(l)
+	return &m
+}
 
-// 		switch req.URL.Path {
-// 		case "/token":
-// 			// Generate random token
-// 			token_length := 10
-// 			buf := make([]byte, token_length)
-// 			rand.Read(buf)
-// 			token := hex.EncodeToString(buf)
-// 			active_tokens = append(active_tokens, token)
+func initCmd(command string, env []string) (*exec.Cmd, context.CancelFunc) {
+	commands := strings.Split(command, " ")
+	ctx, cancl := context.WithCancel(context.Background())
+	cmd := exec.CommandContext(ctx, commands[0], commands[1:]...)
+	cmd.Env = env
+	return cmd, cancl
+}
 
-// 			w.Write([]byte(token))
+func TestMiddlewareEarlyClose(t *testing.T) {
+	quit := make(chan struct{})
+	in := NewTestInput()
+	cmd, cancl := initCmd(echoSh, noDebug)
+	midd := initMiddleware(cmd, cancl, in, func(err error) {
+		if err != nil {
+			if e, ok := err.(*exec.ExitError); ok {
+				status := e.Sys().(syscall.WaitStatus)
+				if status.Signal() != syscall.SIGKILL {
+					t.Errorf("expected error to be signal killed. got %s", status.Signal().String())
+				}
+			}
+		}
+		quit <- struct{}{}
+	})
+	var body = []byte("OPTIONS / HTTP/1.1\r\nHost: example.org\r\n\r\n")
+	count := uint32(0)
+	out := NewTestOutput(func(msg *Message) {
+		if !bytes.Equal(body, msg.Data) {
+			t.Errorf("expected %q to equal %q", body, msg.Data)
+		}
+		atomic.AddUint32(&count, 1)
+		if atomic.LoadUint32(&count) == 5 {
+			quit <- struct{}{}
+		}
+	})
+	pl := &InOutPlugins{}
+	pl.Inputs = []PluginReader{midd, in}
+	pl.Outputs = []PluginWriter{out}
+	pl.All = []interface{}{midd, out, in}
+	e := NewEmitter()
+	go e.Start(pl, "")
+	for i := 0; i < 5; i++ {
+		in.EmitBytes(body)
+	}
+	<-quit
+	midd.Close()
+	<-quit
+}
 
-// 			cb(req.URL.Path, 200, []byte(token))
-// 		case "/secure":
-// 			token := req.URL.Query().Get("token")
-// 			token_found := false
+func TestTokenMiddleware(t *testing.T) {
+	quit := make(chan struct{})
+	in := NewTestInput()
+	in.skipHeader = true
+	cmd, cancl := initCmd(tokenModifier, noDebug)
+	midd := initMiddleware(cmd, cancl, in, func(err error) {})
+	req := []byte("1 932079936fa4306fc308d67588178d17d823647c 1439818823587396305 200\nGET /token HTTP/1.1\r\nHost: example.org\r\n\r\n")
+	res := []byte("2 932079936fa4306fc308d67588178d17d823647c 1439818823587396305 200\nHTTP/1.1 200 OK\r\nContent-Length: 10\r\nContent-Type: text/plain; charset=utf-8\r\n\r\n17d823647c")
+	rep := []byte("3 932079936fa4306fc308d67588178d17d823647c 1439818823587396305 200\nHTTP/1.1 200 OK\r\nContent-Length: 15\r\nContent-Type: text/plain; charset=utf-8\r\n\r\n932079936fa4306")
+	count := uint32(0)
+	out := NewTestOutput(func(msg *Message) {
+		if msg.Meta[0] == '1' && !bytes.Equal(payloadID(msg.Meta), payloadID(req)) {
+			token, _, _ := proto.PathParam(msg.Data, []byte("token"))
+			if !bytes.Equal(token, proto.Body(rep)) {
+				t.Error("expected the token to be equal to the replayed responses's token")
+			}
+		}
+		atomic.AddUint32(&count, 1)
+		if atomic.LoadUint32(&count) == 2 {
+			quit <- struct{}{}
+		}
+	})
+	pl := &InOutPlugins{}
+	pl.Inputs = []PluginReader{midd, in}
+	pl.Outputs = []PluginWriter{out}
+	pl.All = []interface{}{midd, out, in}
+	e := NewEmitter()
+	go e.Start(pl, "")
+	in.EmitBytes(req) // emit original request
+	in.EmitBytes(res) // emit its response
+	in.EmitBytes(rep) // emit replayed response
+	// emit the request which should have modified token
+	token := []byte("1 8e091765ae902fef8a2b7d9dd96 14398188235873 100\nGET /?token=17d823647c HTTP/1.1\r\nHost: example.org\r\n\r\n")
+	in.EmitBytes(token)
+	<-quit
+	midd.Close()
+}
 
-// 			for _, t := range active_tokens {
-// 				if t == token {
-// 					token_found = true
-// 					break
-// 				}
-// 			}
-
-// 			if token_found {
-// 				w.WriteHeader(http.StatusAccepted)
-// 				cb(req.URL.Path, 202, []byte(nil))
-// 			} else {
-// 				w.WriteHeader(http.StatusForbidden)
-// 				cb(req.URL.Path, 403, []byte(nil))
-// 			}
-// 		}
-
-// 		wg.Done()
-// 	}))
-
-// 	return server
-// }
-
-// func TestFakeSecureService(t *testing.T) {
-// 	var resp, token []byte
-
-// 	wg := new(sync.WaitGroup)
-
-// 	server := NewFakeSecureService(wg, func(path string, status int, resp []byte) {
-// 	})
-// 	defer server.Close()
-
-// 	wg.Add(3)
-
-// 	client := NewHTTPClient(server.URL, &HTTPClientConfig{Debug: true})
-// 	resp, _ = client.Get("/token")
-// 	token = proto.Body(resp)
-
-// 	// Right token
-// 	resp, _ = client.Get("/secure?token=" + string(token))
-// 	if !bytes.Equal(proto.Status(resp), []byte("202")) {
-// 		t.Error("Valid token should return status 202:", string(proto.Status(resp)))
-// 	}
-
-// 	// Wrong tokens forbidden
-// 	resp, _ = client.Get("/secure?token=wrong")
-// 	if !bytes.Equal(proto.Status(resp), []byte("403")) {
-// 		t.Error("Wrong token should returns status 403:", string(proto.Status(resp)))
-// 	}
-
-// 	wg.Wait()
-// }
-
-// func TestEchoMiddleware(t *testing.T) {
-// 	wg := new(sync.WaitGroup)
-
-// 	from := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-// 		w.Header().Set("Env", "prod")
-// 		w.Header().Set("RequestPath", r.URL.Path)
-// 		wg.Done()
-// 	}))
-// 	defer from.Close()
-
-// 	to := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-// 		w.Header().Set("Env", "test")
-// 		w.Header().Set("RequestPath", r.URL.Path)
-// 		wg.Done()
-// 	}))
-// 	defer to.Close()
-
-// 	quit := make(chan int)
-
-// 	// Catch traffic from one service
-// 	fromAddr := strings.Replace(from.Listener.Addr().String(), "[::]", "127.0.0.1", -1)
-// 	conf := RAWInputConfig{
-// 		engine:        capture.EnginePcap,
-// 		expire:        testRawExpire,
-// 		protocol:      ProtocolHTTP,
-// 		trackResponse: true,
-// 	}
-// 	input := NewRAWInput(fromAddr, conf)
-
-// 	// And redirect to another
-// 	output := NewHTTPOutput(to.URL, &HTTPOutputConfig{Debug: false})
-
-// 	plugins := &InOutPlugins{
-// 		Inputs:  []io.Reader{input},
-// 		Outputs: []io.Writer{output},
-// 	}
-// 	plugins.All = append(plugins.All, input, output)
-
-// 	// Start Gor
-// 	emitter := NewEmitter(quit)
-// 	go emitter.Start(plugins, "echo -n && GOR_TEST=true && ./examples/middleware/echo.sh")
-
-// 	// Wait till middleware initialization
-// 	time.Sleep(100 * time.Millisecond)
-
-// 	// Should receive 2 requests from original + 2 from replayed
-// 	client := NewHTTPClient(from.URL, &HTTPClientConfig{Debug: false})
-
-// 	for i := 0; i < 10; i++ {
-// 		wg.Add(2)
-// 		// Request should be echoed
-// 		client.Get("/a")
-// 		time.Sleep(5 * time.Millisecond)
-// 		client.Get("/b")
-// 		time.Sleep(5 * time.Millisecond)
-// 	}
-
-// 	wg.Wait()
-// 	emitter.Close()
-// }
-
-// func TestTokenMiddleware(t *testing.T) {
-// 	var resp, token []byte
-
-// 	wg := new(sync.WaitGroup)
-
-// 	from := NewFakeSecureService(wg, func(path string, status int, tok []byte) {
-// 		time.Sleep(10 * time.Millisecond)
-// 	})
-// 	defer from.Close()
-
-// 	to := NewFakeSecureService(wg, func(path string, status int, tok []byte) {
-// 		switch path {
-// 		case "/secure":
-// 			if status != 202 {
-// 				t.Error("Server should receive valid rewritten token")
-// 			}
-// 		}
-
-// 		time.Sleep(10 * time.Millisecond)
-// 	})
-// 	defer to.Close()
-
-// 	quit := make(chan int)
-
-// 	Settings.middleware = "echo -n && GOR_TEST=true && go run ./examples/middleware/token_modifier.go"
-
-// 	fromAddr := strings.Replace(from.Listener.Addr().String(), "[::]", "127.0.0.1", -1)
-// 	conf := RAWInputConfig{
-// 		engine:        capture.EnginePcap,
-// 		expire:        testRawExpire,
-// 		protocol:      ProtocolHTTP,
-// 		trackResponse: true,
-// 	}
-// 	// Catch traffic from one service
-// 	input := NewRAWInput(fromAddr, conf)
-
-// 	// And redirect to another
-// 	output := NewHTTPOutput(to.URL, &HTTPOutputConfig{Debug: true})
-
-// 	plugins := &InOutPlugins{
-// 		Inputs:  []io.Reader{input},
-// 		Outputs: []io.Writer{output},
-// 	}
-// 	plugins.All = append(plugins.All, input, output)
-
-// 	// Start Gor
-// 	emitter := NewEmitter(quit)
-// 	go emitter.Start(plugins, Settings.middleware)
-
-// 	// Should receive 2 requests from original + 2 from replayed
-// 	wg.Add(2)
-
-// 	client := NewHTTPClient(from.URL, &HTTPClientConfig{Debug: true})
-
-// 	// Sending traffic to original service
-// 	resp, _ = client.Get("/token")
-// 	token = proto.Body(resp)
-
-// 	resp, _ = client.Get("/secure?token=" + string(token))
-// 	if !bytes.Equal(proto.Status(resp), []byte("202")) {
-// 		t.Error("Valid token should return 202:", proto.Status(resp))
-// 	}
-
-// 	wg.Wait()
-// 	emitter.Close()
-// 	Settings.middleware = ""
-// }
+func TestMiddlewareWithPrettify(t *testing.T) {
+	Settings.PrettifyHTTP = true
+	quit := make(chan struct{})
+	in := NewTestInput()
+	cmd, cancl := initCmd(echoSh, noDebug)
+	midd := initMiddleware(cmd, cancl, in, func(err error) {})
+	var b1 = []byte("POST / HTTP/1.1\r\nHost: example.org\r\nTransfer-Encoding: chunked\r\n\r\n4\r\nWiki\r\n5\r\npedia\r\nE\r\n in\r\n\r\nchunks.\r\n0\r\n\r\n")
+	var b2 = []byte("POST / HTTP/1.1\r\nHost: example.org\r\nContent-Length: 25\r\n\r\nWikipedia in\r\n\r\nchunks.")
+	out := NewTestOutput(func(msg *Message) {
+		if !bytes.Equal(proto.Body(b2), proto.Body(msg.Data)) {
+			t.Errorf("expected %q body to equal %q body", b2, msg.Data)
+		}
+		quit <- struct{}{}
+	})
+	pl := &InOutPlugins{}
+	pl.Inputs = []PluginReader{midd, in}
+	pl.Outputs = []PluginWriter{out}
+	pl.All = []interface{}{midd, out, in}
+	e := NewEmitter()
+	go e.Start(pl, "")
+	in.EmitBytes(b1)
+	<-quit
+	midd.Close()
+	Settings.PrettifyHTTP = false
+}

output_dummy.go

@@ -24,6 +24,7 @@ func (i *DummyOutput) PluginWrite(msg *Message) (int, error) {
 	n += nn
 	nn, err = os.Stdout.Write(payloadSeparatorAsBytes)
 	n += nn
+
 	return n, err
 }
 

output_file.go

@@ -7,6 +7,7 @@ import (
 	"fmt"
 	"io"
 	"log"
+	"math/rand"
 	"os"
 	"path/filepath"
 	"runtime/debug"
@@ -19,6 +20,21 @@ import (
 	"github.com/buger/goreplay/size"
 )
 
+var letters = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ")
+var instanceID string
+
+func init() {
+	instanceID = randSeq(8)
+}
+
+func randSeq(n int) string {
+	b := make([]rune, n)
+	for i := range b {
+		b[i] = letters[rand.Intn(len(letters))]
+	}
+	return string(b)
+}
+
 var dateFileNameFuncs = map[string]func(*FileOutput) string{
 	"%Y":  func(o *FileOutput) string { return time.Now().Format("2006") },
 	"%m":  func(o *FileOutput) string { return time.Now().Format("01") },
@@ -29,6 +45,7 @@ var dateFileNameFuncs = map[string]func(*FileOutput) string{
 	"%NS": func(o *FileOutput) string { return fmt.Sprint(time.Now().Nanosecond()) },
 	"%r":  func(o *FileOutput) string { return string(o.currentID) },
 	"%t":  func(o *FileOutput) string { return string(o.payloadType) },
+	"%i":  func(o *FileOutput) string { return instanceID },
 }
 
 // FileOutputConfig ...
@@ -45,17 +62,17 @@ type FileOutputConfig struct {
 // FileOutput output plugin
 type FileOutput struct {
 	sync.RWMutex
-	pathTemplate   string
-	currentName    string
-	file           *os.File
-	QueueLength    int
-	chunkSize      int
-	writer         io.Writer
-	requestPerFile bool
-	currentID      []byte
-	payloadType    []byte
-	closed         bool
-	totalFileSize  size.Size
+	pathTemplate    string
+	currentName     string
+	file            *os.File
+	QueueLength     int
+	writer          io.Writer
+	requestPerFile  bool
+	currentID       []byte
+	payloadType     []byte
+	closed          bool
+	currentFileSize int
+	totalFileSize   size.Size
 
 	config *FileOutputConfig
 }
@@ -157,7 +174,7 @@ func (o *FileOutput) filename() string {
 
 		if o.currentName == "" ||
 			((o.config.QueueLimit > 0 && o.QueueLength >= o.config.QueueLimit) ||
-				(o.config.SizeLimit > 0 && o.chunkSize >= int(o.config.SizeLimit))) {
+				(o.config.SizeLimit > 0 && o.currentFileSize >= int(o.config.SizeLimit))) {
 			nextChunk = true
 		}
 
@@ -178,6 +195,7 @@ func (o *FileOutput) filename() string {
 
 				if nextChunk {
 					fileIndex++
+					o.currentFileSize = 0
 				}
 			}
 
@@ -236,6 +254,7 @@ func (o *FileOutput) PluginWrite(msg *Message) (n int, err error) {
 	n += nn
 
 	o.totalFileSize += size.Size(n)
+	o.currentFileSize += n
 	o.QueueLength++
 
 	if Settings.OutputFileConfig.OutputFileMaxSize > 0 && o.totalFileSize >= Settings.OutputFileConfig.OutputFileMaxSize {
@@ -264,7 +283,7 @@ func (o *FileOutput) flush() {
 		}
 
 		if stat, err := o.file.Stat(); err == nil {
-			o.chunkSize = int(stat.Size())
+			o.currentFileSize = int(stat.Size())
 		} else {
 			Debug(0, "[OUTPUT-HTTP] error accessing file size", err)
 		}

output_file_test.go

@@ -39,7 +39,7 @@ func TestFileOutput(t *testing.T) {
 	emitter.Close()
 
 	var counter int64
-	input2 := NewFileInput("/tmp/test_requests.gor", false)
+	input2 := NewFileInput("/tmp/test_requests.gor", false, 100, 0, false)
 	output2 := NewTestOutput(func(*Message) {
 		atomic.AddInt64(&counter, 1)
 		wg.Done()

output_http.go

@@ -215,6 +215,7 @@ func (o *HTTPOutput) sendRequest(client *HTTPClient, msg *Message) {
 	if !isRequestPayload(msg.Meta) {
 		return
 	}
+
 	uuid := payloadID(msg.Meta)
 	start := time.Now()
 	resp, err := client.Send(msg.Data)
@@ -301,7 +302,14 @@ func (c *HTTPClient) Send(data []byte) ([]byte, error) {
 		req.Host = c.config.url.Host
 	}
 
-	req.URL = c.config.url
+	// fix #862
+	if c.config.url.Path == "" && c.config.url.RawQuery == "" {
+		req.URL.Scheme = c.config.url.Scheme
+		req.URL.Host = c.config.url.Host
+	} else {
+		req.URL = c.config.url
+	}
+
 	// force connection to not be closed, which can affect the global client
 	req.Close = false
 	// it's an error if this is not equal to empty string
@@ -314,5 +322,6 @@ func (c *HTTPClient) Send(data []byte) ([]byte, error) {
 	if c.config.TrackResponses {
 		return httputil.DumpResponse(resp, true)
 	}
+	_ = resp.Body.Close()
 	return nil, nil
 }

output_kafka.go

@@ -70,7 +70,7 @@ func (o *KafkaOutput) PluginWrite(msg *Message) (n int, err error) {
 		message = sarama.StringEncoder(byteutils.SliceToString(msg.Meta) + byteutils.SliceToString(msg.Data))
 	} else {
 		mimeHeader := proto.ParseHeaders(msg.Data)
-		var header map[string]string
+		header := make(map[string]string)
 		for k, v := range mimeHeader {
 			header[k] = strings.Join(v, ", ")
 		}

output_s3.go

@@ -63,7 +63,7 @@ func NewS3Output(pathTemplate string, config *FileOutputConfig) *S3Output {
 func (o *S3Output) connect() {
 	if o.session == nil {
 		o.session = session.Must(session.NewSession(awsConfig()))
-		log.Println("[S3 Output] S3 connection succesfully initialized")
+		log.Println("[S3 Output] S3 connection successfully initialized")
 	}
 }
 

output_tcp.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"context"
 	"crypto/tls"
 	"fmt"
 	"hash/fnv"
@@ -10,7 +11,7 @@ import (
 
 // TCPOutput used for sending raw tcp payloads
 // Currently used for internal communication between listener and replay server
-// Can be used for transfering binary payloads like protocol buffers
+// Can be used for transferring binary payloads like protocol buffers
 type TCPOutput struct {
 	address     string
 	limit       int
@@ -18,6 +19,8 @@ type TCPOutput struct {
 	bufStats    *GorStat
 	config      *TCPOutputConfig
 	workerIndex uint32
+
+	close bool
 }
 
 // TCPOutputConfig tcp output configuration
@@ -54,6 +57,10 @@ func (o *TCPOutput) worker(bufferIndex int) {
 	retries := 0
 	conn, err := o.connect(o.address)
 	for {
+		if o.close {
+			return
+		}
+
 		if err == nil {
 			break
 		}
@@ -88,16 +95,15 @@ func (o *TCPOutput) worker(bufferIndex int) {
 	}
 }
 
-func (o *TCPOutput) getBufferIndex(data []byte) int {
+func (o *TCPOutput) getBufferIndex(msg *Message) int {
 	if !o.config.Sticky {
 		o.workerIndex++
 		return int(o.workerIndex) % o.config.Workers
 	}
 
 	hasher := fnv.New32a()
-	hasher.Write(payloadMeta(data)[1])
+	hasher.Write(payloadID(msg.Meta))
 	return int(hasher.Sum32()) % o.config.Workers
-
 }
 
 // PluginWrite writes message to this plugin
@@ -106,7 +112,7 @@ func (o *TCPOutput) PluginWrite(msg *Message) (n int, err error) {
 		return len(msg.Data), nil
 	}
 
-	bufferIndex := o.getBufferIndex(msg.Data)
+	bufferIndex := o.getBufferIndex(msg)
 	o.buf[bufferIndex] <- msg
 
 	if Settings.OutputTCPStats {
@@ -118,9 +124,16 @@ func (o *TCPOutput) PluginWrite(msg *Message) (n int, err error) {
 
 func (o *TCPOutput) connect(address string) (conn net.Conn, err error) {
 	if o.config.Secure {
-		conn, err = tls.Dial("tcp", address, &tls.Config{InsecureSkipVerify: o.config.SkipVerify})
+		var d tls.Dialer
+		d.Config = &tls.Config{InsecureSkipVerify: o.config.SkipVerify}
+		ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+		defer cancel()
+		conn, err = d.DialContext(ctx, "tcp", address)
 	} else {
-		conn, err = net.Dial("tcp", address)
+		var d net.Dialer
+		ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+		defer cancel()
+		conn, err = d.DialContext(ctx, "tcp", address)
 	}
 
 	return
@@ -129,3 +142,7 @@ func (o *TCPOutput) connect(address string) (conn net.Conn, err error) {
 func (o *TCPOutput) String() string {
 	return fmt.Sprintf("TCP output %s, limit: %d", o.address, o.limit)
 }
+
+func (o *TCPOutput) Close() {
+	o.close = true
+}

output_tcp_test.go

@@ -125,8 +125,9 @@ func TestBufferDistribution(t *testing.T) {
 	}
 }
 
-func getTestBytes() []byte {
-	reqh := payloadHeader(RequestPayload, uuid(), time.Now().UnixNano(), -1)
-	reqb := append(reqh, []byte("GET / HTTP/1.1\r\nHost: www.w3.org\r\nUser-Agent: Go 1.1 package http\r\nAccept-Encoding: gzip\r\n\r\n")...)
-	return reqb
+func getTestBytes() *Message {
+	return &Message{
+		Meta: payloadHeader(RequestPayload, uuid(), time.Now().UnixNano(), -1),
+		Data: []byte("GET / HTTP/1.1\r\nHost: www.w3.org\r\nUser-Agent: Go 1.1 package http\r\nAccept-Encoding: gzip\r\n\r\n"),
+	}
 }

plugins.go

@@ -5,7 +5,7 @@ import (
 	"strings"
 )
 
-// Message represents data accross plugins
+// Message represents data across plugins
 type Message struct {
 	Meta []byte // metadata
 	Data []byte // actual data
@@ -83,7 +83,6 @@ func (plugins *InOutPlugins) registerPlugin(constructor interface{}, options ...
 		plugins.Outputs = append(plugins.Outputs, w)
 	}
 	plugins.All = append(plugins.All, plugin)
-
 }
 
 // NewPlugins specify and initialize all available plugins
@@ -119,7 +118,7 @@ func NewPlugins() *InOutPlugins {
 	}
 
 	for _, options := range Settings.InputFile {
-		plugins.registerPlugin(NewFileInput, options, Settings.InputFileLoop)
+		plugins.registerPlugin(NewFileInput, options, Settings.InputFileLoop, Settings.InputFileReadDepth, Settings.InputFileMaxWait, Settings.InputFileDryRun)
 	}
 
 	for _, path := range Settings.OutputFile {

proto/proto.go

@@ -19,6 +19,7 @@ package proto
 import (
 	"bufio"
 	"bytes"
+	_ "fmt"
 	"net/http"
 	"net/textproto"
 	"strings"
@@ -58,10 +59,15 @@ func MIMEHeadersStartPos(payload []byte) int {
 // If not found, value will be blank, and headerStart will be -1
 // Do not support multi-line headers.
 func header(payload []byte, name []byte) (value []byte, headerStart, headerEnd, valueStart, valueEnd int) {
-	headerStart = MIMEHeadersStartPos(payload)
-	if headerStart < 0 {
-		return
+	if HasTitle(payload) {
+		headerStart = MIMEHeadersStartPos(payload)
+		if headerStart < 0 {
+			return
+		}
+	} else {
+		headerStart = 0
 	}
+
 	var colonIndex int
 	for headerStart < len(payload) {
 		headerEnd = bytes.IndexByte(payload[headerStart:], '\n')
@@ -125,6 +131,11 @@ func ParseHeaders(p []byte) textproto.MIMEHeader {
 	if headerEnd > 1 {
 		p = p[:headerEnd]
 	}
+	return GetHeaders(p)
+}
+
+// GetHeaders returns mime headers from the payload
+func GetHeaders(p []byte) textproto.MIMEHeader {
 	reader := textproto.NewReader(bufio.NewReader(bytes.NewReader(p)))
 	mime, err := reader.ReadMIMEHeader()
 	if err != nil {
@@ -182,15 +193,15 @@ func DeleteHeader(payload, name []byte) []byte {
 // Body returns request/response body
 func Body(payload []byte) []byte {
 	pos := MIMEHeadersEndPos(payload)
-	if pos == -1 {
+	if pos == -1 || len(payload) <= pos {
 		return nil
 	}
 	return payload[pos:]
 }
 
-// Path takes payload and retuns request path: Split(firstLine, ' ')[1]
+// Path takes payload and returns request path: Split(firstLine, ' ')[1]
 func Path(payload []byte) []byte {
-	if !HasTitle(payload) {
+	if !HasRequestTitle(payload) {
 		return nil
 	}
 	start := bytes.IndexByte(payload, ' ') + 1
@@ -304,7 +315,12 @@ func Method(payload []byte) []byte {
 // Status returns response status.
 // It happens to be in same position as request payload path
 func Status(payload []byte) []byte {
-	return Path(payload)
+	if !HasResponseTitle(payload) {
+		return nil
+	}
+	start := bytes.IndexByte(payload, ' ') + 1
+	// status code are in range 100-600
+	return payload[start : start+3]
 }
 
 // Methods holds the http methods ordered in ascending order
@@ -317,8 +333,8 @@ var Methods = [...]string{
 const (
 	//MinRequestCount GET / HTTP/1.1\r\n
 	MinRequestCount = 16
-	// MinResponseCount HTTP/1.1 200 OK\r\n
-	MinResponseCount = 17
+	// MinResponseCount HTTP/1.1 200\r\n
+	MinResponseCount = 14
 	// VersionLen HTTP/1.1
 	VersionLen = 8
 )
@@ -341,17 +357,15 @@ func HasResponseTitle(payload []byte) bool {
 		return false
 	}
 	status, ok := atoI(payload[VersionLen+1:VersionLen+4], 10)
-	if !ok || s[VersionLen+4] != ' ' {
+	if !ok {
 		return false
 	}
-	statusText := http.StatusText(status)
-	if statusText == "" {
+	// only validate status codes mentioned in rfc2616.
+	if http.StatusText(status) == "" {
 		return false
 	}
-	if titleLen+len(CRLF) > len(s) {
-		return false
-	}
-	return s[VersionLen+5:titleLen] == statusText
+	// handle cases from #875
+	return payload[VersionLen+4] == ' ' || payload[VersionLen+4] == '\r'
 }
 
 // HasRequestTitle reports whether this payload has an HTTP/1 request title
@@ -390,102 +404,172 @@ func HasTitle(payload []byte) bool {
 	return HasRequestTitle(payload) || HasResponseTitle(payload)
 }
 
-// CheckChunked checks HTTP/1 chunked data integrity and return the final index
-// of chunks(index after '0\r\n\r\n') or -1 if there is missing data
-// or there is bad format
-func CheckChunked(buf []byte) (chunkEnd int) {
-	var (
-		ok     bool
-		chkLen int
-		sz     int
-		ext    int
-	)
-	for {
-		sz = bytes.IndexByte(buf[chunkEnd:], '\r')
-		if sz < 1 {
-			return -1
-		}
-		// ignoring chunks extensions https://github.com/golang/go/issues/13135
-		// but chunks extensions are no longer a thing
-		ext = bytes.IndexByte(buf[chunkEnd:chunkEnd+sz], ';')
-		if ext < 0 {
-			ext = sz
-		}
-		chkLen, ok = atoI(buf[chunkEnd:chunkEnd+ext], 16)
-		if !ok {
-			return -1
-		}
-		chunkEnd += (sz + 2)
-		if chkLen == 0 {
-			if !bytes.Equal(buf[chunkEnd:chunkEnd+2], CRLF) {
-				return -1
-			}
-			return chunkEnd + 2
-		}
-		// ideally chunck length and at least len("\r\n0\r\n\r\n")
-		if len(buf[chunkEnd:]) < chkLen+7 {
-			return -1
-		}
-		chunkEnd += chkLen
-		// chunks must end with CRLF
-		if !bytes.Equal(buf[chunkEnd:chunkEnd+2], CRLF) {
-			return -1
-		}
-		chunkEnd += 2
+// CheckChunked checks HTTP/1 chunked data integrity(https://tools.ietf.org/html/rfc7230#section-4.1)
+// and returns the length of total valid scanned chunks(including chunk size, extensions and CRLFs) and
+// full is true if all chunks was scanned.
+func CheckChunked(bufs ...[]byte) (chunkEnd int, full bool) {
+	var buf []byte
+	if len(bufs) > 0 {
+		buf = bufs[0]
 	}
+	for chunkEnd < len(buf) {
+		sz := bytes.IndexByte(buf[chunkEnd:], '\r')
+		if sz < 1 {
+			break
+		}
+		// don't parse chunk extensions https://github.com/golang/go/issues/13135.
+		// chunks extensions are no longer a thing, but we do check if the byte
+		// following the parsed hex number is ';'
+		sz += chunkEnd
+		chkLen, ok := atoI(buf[chunkEnd:sz], 16)
+		if !ok && bytes.IndexByte(buf[chunkEnd:sz], ';') < 1 {
+			break
+		}
+		sz++ // + '\n'
+		// total length = SIZE + CRLF + OCTETS + CRLF
+		allChunk := sz + chkLen + 2
+		if allChunk >= len(buf) ||
+			buf[sz]&buf[allChunk] != '\n' ||
+			buf[allChunk-1] != '\r' {
+			break
+		}
+		chunkEnd = allChunk + 1
+		if chkLen == 0 {
+			full = true
+			break
+		}
+	}
+	return
 }
 
-// HasFullPayload reports if this http has full payloads
-func HasFullPayload(payload []byte) bool {
-	body := Body(payload)
+// ProtocolStateSetter is an interface used to provide protocol state for future use
+type ProtocolStateSetter interface {
+	SetProtocolState(interface{})
+	ProtocolState() interface{}
+}
 
-	// check for chunked transfer-encoding
-	header := Header(payload, []byte("Transfer-Encoding"))
-	if bytes.Contains(header, []byte("chunked")) {
+type httpProto struct {
+	body         int // body index
+	headerStart  int
+	headerParsed bool // we checked necessary headers
+	hasFullBody  bool // all chunks has been parsed
+	isChunked    bool // Transfer-Encoding: chunked
+	bodyLen      int  // Content-Length's value
+	hasTrailer   bool // Trailer header?
+}
 
-		// check chunks
-		if len(body) < 1 {
-			return false
+// HasFullPayload checks if this message has full or valid payloads and returns true.
+// Message param is optional but recommended on cases where 'data' is storing
+// partial-to-full stream of bytes(packets).
+func HasFullPayload(m ProtocolStateSetter, payloads ...[]byte) bool {
+	var state *httpProto
+	if m != nil {
+		state, _ = m.ProtocolState().(*httpProto)
+	}
+	if state == nil {
+		state = new(httpProto)
+		if m != nil {
+			m.SetProtocolState(state)
 		}
-		var chunkEnd int
-		if chunkEnd = CheckChunked(body); chunkEnd < 1 {
+	}
+	if state.headerStart < 1 {
+		for _, data := range payloads {
+			state.headerStart = MIMEHeadersStartPos(data)
+			if state.headerStart < 0 {
+				return false
+			} else {
+				break
+			}
+		}
+	}
+
+	if state.body < 1 {
+		var pos int
+		for _, data := range payloads {
+			endPos := MIMEHeadersEndPos(data)
+			if endPos < 0 {
+				pos += len(data)
+			} else {
+				pos += endPos
+			}
+
+			if endPos > 0 {
+				state.body = pos
+				break
+			}
+		}
+	}
+	if !state.headerParsed {
+		var pos int
+		for _, data := range payloads {
+			chunked := Header(data, []byte("Transfer-Encoding"))
+
+			if len(chunked) > 0 && bytes.Index(data, []byte("chunked")) > 0 {
+				state.isChunked = true
+				// trailers are generally not allowed in non-chunks body
+				state.hasTrailer = len(Header(data, []byte("Trailer"))) > 0
+			} else {
+				contentLen := Header(data, []byte("Content-Length"))
+				state.bodyLen, _ = atoI(contentLen, 10)
+			}
+
+			pos += len(data)
+
+			if state.bodyLen > 0 || pos >= state.body {
+				state.headerParsed = true
+				break
+			}
+		}
+	}
+
+	bodyLen := 0
+	for _, data := range payloads {
+		bodyLen += len(data)
+	}
+	bodyLen -= state.body
+
+	if state.isChunked {
+		// check chunks
+		if bodyLen < 1 {
 			return false
 		}
 
 		// check trailer headers
-		if len(Header(payload, []byte("Trailer"))) < 1 {
-			return true
+		if state.hasTrailer {
+			if bytes.HasSuffix(payloads[len(payloads)-1], []byte("\r\n\r\n")) {
+				return true
+			}
+		} else {
+			if bytes.HasSuffix(payloads[len(payloads)-1], []byte("0\r\n\r\n")) {
+				state.hasFullBody = true
+				return true
+			}
 		}
-		// trailer headers(whether chunked or plain) should end with empty line
-		return len(body) > chunkEnd && MIMEHeadersEndPos(body[chunkEnd:]) != -1
+
+		return false
 	}
 
 	// check for content-length header
-	// trailers are generally not allowed in non-chunks body
-	header = Header(payload, []byte("Content-Length"))
-	if len(header) > 1 {
-		num, ok := atoI(header, 10)
-		return ok && num == len(body)
-	}
-
-	// for empty body, check for emptyline
-	return MIMEHeadersEndPos(payload) != -1
+	return state.bodyLen == bodyLen
 }
 
 // this works with positive integers
 func atoI(s []byte, base int) (num int, ok bool) {
 	var v int
+	ok = true
 	for i := 0; i < len(s); i++ {
 		if s[i] > 127 {
-			return 0, false
+			ok = false
+			break
 		}
 		v = int(hexTable[s[i]])
 		if v >= base || (v == 0 && s[i] != '0') {
-			return 0, false
+			ok = false
+			break
 		}
 		num = (num * base) + v
 	}
-	return num, true
+	return
 }
 
 var hexTable = [128]byte{

proto/proto_test.go

@@ -136,6 +136,24 @@ func TestParseHeaders(t *testing.T) {
 	if !reflect.DeepEqual(headers, expected) {
 		t.Error("Headers do not properly parsed", headers)
 	}
+
+	// Response with Reason phrase
+	payload = [][]byte{[]byte("HTTP/1.1 200 OK\r\nContent-Length: 7\r\nHost: www.w3.org\r\nUser-Agent:Chrome\r\n\r\nbody")}
+
+	headers = ParseHeaders(bytes.Join(payload, nil))
+
+	if !reflect.DeepEqual(headers, expected) {
+		t.Error("Headers do not properly parsed", headers)
+	}
+
+	// Response without Reason phrase
+	payload = [][]byte{[]byte("HTTP/1.1 200\r\nContent-Length: 7\r\nHost: www.w3.org\r\nUser-Agent:Chrome\r\n\r\nbody")}
+
+	headers = ParseHeaders(bytes.Join(payload, nil))
+
+	if !reflect.DeepEqual(headers, expected) {
+		t.Error("Headers do not properly parsed", headers)
+	}
 }
 
 // See https://github.com/dvyukov/go-fuzz and fuzz.go
@@ -219,6 +237,25 @@ func TestPath(t *testing.T) {
 	}
 }
 
+func TestStatus(t *testing.T) {
+	var status, payload []byte
+
+	payload = []byte("HTTP/1.1 200 OK\r\n")
+	if status = Status(payload); !bytes.Equal(status, []byte("200")) {
+		t.Error("Should find status 200 but:", string(status))
+	}
+
+	payload = []byte("HTTP/1.1 200\r\n")
+	if status = Status(payload); !bytes.Equal(status, []byte("200")) {
+		t.Error("1Should find status 200 but:", string(status))
+	}
+
+	payload = []byte("HTTP/1.1 404 Not Found\r\n")
+	if status = Status(payload); !bytes.Equal(status, []byte("404")) {
+		t.Error("2Should find status 404 but:", string(status))
+	}
+}
+
 func TestSetPath(t *testing.T) {
 	var payload, payloadAfter []byte
 
@@ -318,9 +355,10 @@ func TestHasResponseTitle(t *testing.T) {
 		"HTTP/1.1 100 Continue\r\n": true,
 		"HTTP/1.1  \r\n":            false,
 		"HTTP/4.0 100Continue\r\n":  false,
+		"HTTP/1.0 100Continue\r\n":  false,
 		"HTTP/1.0 10r Continue\r\n": false,
-		"HTTP/1.1 200\r\n":          false,
-		"HTTP/1.1 200\r\nServer: Tengine\r\nContent-Length: 0\r\nConnection: close\r\n\r\n": false,
+		"HTTP/1.1 200\r\n":          true,
+		"HTTP/1.1 200\r\nServer: Tengine\r\nContent-Length: 0\r\nConnection: close\r\n\r\n": true,
 	}
 	for k, v := range m {
 		if HasResponseTitle([]byte(k)) != v {
@@ -350,63 +388,61 @@ func TestHasRequestTitle(t *testing.T) {
 
 func TestCheckChunks(t *testing.T) {
 	var m = "4\r\nWiki\r\n5\r\npedia\r\nE\r\n in\r\n\r\nchunks.\r\n0\r\n\r\n"
-	chunkEnd := CheckChunked([]byte(m))
-	expected := bytes.Index([]byte(m), []byte("0\r\n")) + 5
+	chunkEnd, _ := CheckChunked([]byte(m))
+	expected := len(m)
 	if chunkEnd != expected {
 		t.Errorf("expected %d to equal %d", chunkEnd, expected)
 	}
 
 	m = "7\r\nMozia\r\n9\r\nDeveloper\r\n7\r\nNetwork\r\n0\r\n\r\n"
-	chunkEnd = CheckChunked([]byte(m))
-	if chunkEnd != -1 {
-		t.Errorf("expected %d to equal %d", chunkEnd, -1)
+	chunkEnd, _ = CheckChunked([]byte(m))
+	if chunkEnd != 0 {
+		t.Errorf("expected %d to equal %d", chunkEnd, 0)
 	}
 
 	// with trailers
 	m = "4\r\nWiki\r\n5\r\npedia\r\nE\r\n in\r\n\r\nchunks.\r\n0\r\n\r\nEXpires"
-	chunkEnd = CheckChunked([]byte(m))
-	expected = bytes.Index([]byte(m), []byte("0\r\n")) + 5
+	chunkEnd, _ = CheckChunked([]byte(m))
+	expected = len(m) - 7
 	if chunkEnd != expected {
 		t.Errorf("expected %d to equal %d", chunkEnd, expected)
 	}
 
-	// last chunk inside the the body
+	// last chunk inside the body
 	// with trailers
 	m = "4\r\nWiki\r\n5\r\npedia\r\nE\r\n in\r\n\r\nchunks.\r\n3\r\n0\r\n\r\n0\r\n\r\nEXpires"
-	chunkEnd = CheckChunked([]byte(m))
-	expected = bytes.Index([]byte(m), []byte("0\r\n")) + 10
+	chunkEnd, _ = CheckChunked([]byte(m))
+	expected = len(m) - 7
 	if chunkEnd != expected {
 		t.Errorf("expected %d to equal %d", chunkEnd, expected)
 	}
 
 	// checks with chucks-extensions
 	m = "4\r\nWiki\r\n5\r\npedia\r\nE; name='quoted string'\r\n in\r\n\r\nchunks.\r\n3\r\n0\r\n\r\n0\r\n\r\nEXpires"
-	chunkEnd = CheckChunked([]byte(m))
-	expected = bytes.Index([]byte(m), []byte("0\r\n")) + 10
+	chunkEnd, _ = CheckChunked([]byte(m))
+	expected = len(m) - 7
 	if chunkEnd != expected {
 		t.Errorf("expected %d to equal %d", chunkEnd, expected)
 	}
 }
 
 func TestHasFullPayload(t *testing.T) {
-	var m = "HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nTransfer-Encoding: chunked\r\n\r\n7\r\nMozilla\r\n9\r\nDeveloper\r\n7\r\nNetwork\r\n0\r\n\r\n"
-	got := HasFullPayload([]byte(m))
-	expected := true
-	if got != expected {
-		t.Errorf("expected %v to equal %v", got, expected)
-	}
+	var m string
+	var got, expected bool
 
-	// check with invalid chunk format
-	m = "HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nTransfer-Encoding: chunked\r\n\r\n7\r\nMozia\r\n9\r\nDeveloper\r\n7\r\nNetwork\r\n0\r\n\r\n"
-	got = HasFullPayload([]byte(m))
-	expected = false
+	got = HasFullPayload(nil,
+		[]byte("HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\n"),
+		[]byte("Transfer-Encoding: chunked\r\n\r\n"),
+		[]byte("7\r\nMozilla\r\n9\r\nDeveloper\r\n"),
+		[]byte("7\r\nNetwork\r\n0\r\n\r\n"))
+	expected = true
 	if got != expected {
 		t.Errorf("expected %v to equal %v", got, expected)
 	}
 
 	// check chunks with trailers
 	m = "HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nTransfer-Encoding: chunked\r\nTrailer: Expires\r\n\r\n7\r\nMozilla\r\n9\r\nDeveloper\r\n7\r\nNetwork\r\n0\r\n\r\nExpires: Wed, 21 Oct 2015 07:28:00 GMT\r\n\r\n"
-	got = HasFullPayload([]byte(m))
+	got = HasFullPayload(nil, []byte(m))
 	expected = true
 	if got != expected {
 		t.Errorf("expected %v to equal %v", got, expected)
@@ -414,7 +450,7 @@ func TestHasFullPayload(t *testing.T) {
 
 	// check with missing trailers
 	m = "HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nTransfer-Encoding: chunked\r\nTrailer: Expires\r\n\r\n7\r\nMozilla\r\n9\r\nDeveloper\r\n7\r\nNetwork\r\n0\r\n\r\nExpires: Wed, 21 Oct 2015 07:28:00"
-	got = HasFullPayload([]byte(m))
+	got = HasFullPayload(nil, []byte(m))
 	expected = false
 	if got != expected {
 		t.Errorf("expected %v to equal %v", got, expected)
@@ -422,7 +458,7 @@ func TestHasFullPayload(t *testing.T) {
 
 	// check with content-length
 	m = "HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nContent-Length: 23\r\n\r\nMozillaDeveloperNetwork"
-	got = HasFullPayload([]byte(m))
+	got = HasFullPayload(nil, []byte(m))
 	expected = true
 	if got != expected {
 		t.Errorf("expected %v to equal %v", got, expected)
@@ -430,7 +466,7 @@ func TestHasFullPayload(t *testing.T) {
 
 	// check missing total length
 	m = "HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nContent-Length: 23\r\n\r\nMozillaDeveloperNet"
-	got = HasFullPayload([]byte(m))
+	got = HasFullPayload(nil, []byte(m))
 	expected = false
 	if got != expected {
 		t.Errorf("expected %v to equal %v", got, expected)
@@ -438,7 +474,7 @@ func TestHasFullPayload(t *testing.T) {
 
 	// check with no body
 	m = "HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\n\r\n"
-	got = HasFullPayload([]byte(m))
+	got = HasFullPayload(nil, []byte(m))
 	expected = true
 	if got != expected {
 		t.Errorf("expected %v to equal %v", got, expected)
@@ -446,21 +482,10 @@ func TestHasFullPayload(t *testing.T) {
 }
 
 func BenchmarkHasFullPayload(b *testing.B) {
-	var buf bytes.Buffer
-	buf.Write([]byte("HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nTransfer-Encoding: chunked\r\n\r\n"))
-	var chunk = []byte("1e\r\n111111111111111111111111111111\r\n")
-	for i := 0; i < 5000; i++ {
-		buf.Write(chunk)
-	}
-	buf.Write([]byte("0\r\n\r\n"))
-	data := buf.Bytes()
-	var ok bool
-	b.ResetTimer() // ignores the upper initialization
-	b.ReportMetric(float64(5000), "chunks/op")
+	data := []byte("HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nTransfer-Encoding: chunked\r\n\r\n1e\r\n111111111111111111111111111111\r\n0\r\n\r\n")
 	for i := 0; i < b.N; i++ {
-		ok = HasFullPayload(data)
-	}
-	if !ok {
-		b.Fail()
+		if !HasFullPayload(nil, data) {
+			b.Fail()
+		}
 	}
 }

ring/ring.go

@@ -0,0 +1,205 @@
+package ring
+
+import (
+	"errors"
+	"runtime"
+	"sync/atomic"
+	"time"
+)
+
+var (
+	// ErrDisposed is returned when an operation is performed on a disposed
+	// queue.
+	ErrDisposed = errors.New(`queue: disposed`)
+
+	// ErrTimeout is returned when an applicable queue operation times out.
+	ErrTimeout = errors.New(`queue: poll timed out`)
+
+	// ErrEmptyQueue is returned when an non-applicable queue operation was called
+	// due to the queue's empty item state
+	ErrEmptyQueue = errors.New(`queue: empty queue`)
+)
+
+// roundUp takes a uint64 greater than 0 and rounds it up to the next
+// power of 2.
+func roundUp(v uint64) uint64 {
+	v--
+	v |= v >> 1
+	v |= v >> 2
+	v |= v >> 4
+	v |= v >> 8
+	v |= v >> 16
+	v |= v >> 32
+	v++
+	return v
+}
+
+type node struct {
+	position uint64
+	data     interface{}
+}
+
+type nodes []node
+
+// RingBuffer is a MPMC buffer that achieves threadsafety with CAS operations
+// only.  A put on full or get on empty call will block until an item
+// is put or retrieved.  Calling Dispose on the RingBuffer will unblock
+// any blocked threads with an error.  This buffer is similar to the buffer
+// described here: http://www.1024cores.net/home/lock-free-algorithms/queues/bounded-mpmc-queue
+// with some minor additions.
+type RingBuffer struct {
+	_padding0      [8]uint64
+	queue          uint64
+	_padding1      [8]uint64
+	dequeue        uint64
+	_padding2      [8]uint64
+	mask, disposed uint64
+	_padding3      [8]uint64
+	nodes          nodes
+}
+
+func (rb *RingBuffer) init(size uint64) {
+	size = roundUp(size)
+	rb.nodes = make(nodes, size)
+	for i := uint64(0); i < size; i++ {
+		rb.nodes[i] = node{position: i}
+	}
+	rb.mask = size - 1 // so we don't have to do this with every put/get operation
+}
+
+// Put adds the provided item to the queue.  If the queue is full, this
+// call will block until an item is added to the queue or Dispose is called
+// on the queue.  An error will be returned if the queue is disposed.
+func (rb *RingBuffer) Put(item interface{}) error {
+	_, err := rb.put(item, false)
+	return err
+}
+
+// Offer adds the provided item to the queue if there is space.  If the queue
+// is full, this call will return false.  An error will be returned if the
+// queue is disposed.
+func (rb *RingBuffer) Offer(item interface{}) (bool, error) {
+	return rb.put(item, true)
+}
+
+func (rb *RingBuffer) put(item interface{}, offer bool) (bool, error) {
+	var n *node
+	pos := atomic.LoadUint64(&rb.queue)
+L:
+	for {
+		if atomic.LoadUint64(&rb.disposed) == 1 {
+			return false, ErrDisposed
+		}
+
+		n = &rb.nodes[pos&rb.mask]
+		seq := atomic.LoadUint64(&n.position)
+		switch dif := seq - pos; {
+		case dif == 0:
+			if atomic.CompareAndSwapUint64(&rb.queue, pos, pos+1) {
+				break L
+			}
+		case dif < 0:
+			panic(`Ring buffer in a compromised state during a put operation.`)
+		default:
+			pos = atomic.LoadUint64(&rb.queue)
+		}
+
+		if offer {
+			return false, nil
+		}
+
+		runtime.Gosched() // free up the cpu before the next iteration
+	}
+
+	n.data = item
+	atomic.StoreUint64(&n.position, pos+1)
+	return true, nil
+}
+
+// Get will return the next item in the queue.  This call will block
+// if the queue is empty.  This call will unblock when an item is added
+// to the queue or Dispose is called on the queue.  An error will be returned
+// if the queue is disposed.
+func (rb *RingBuffer) Get() (interface{}, error) {
+	return rb.Poll(0)
+}
+
+// Poll will return the next item in the queue.  This call will block
+// if the queue is empty.  This call will unblock when an item is added
+// to the queue, Dispose is called on the queue, or the timeout is reached. An
+// error will be returned if the queue is disposed or a timeout occurs. A
+// non-positive timeout will block indefinitely.
+func (rb *RingBuffer) Poll(timeout time.Duration) (interface{}, error) {
+	var (
+		n     *node
+		pos   = atomic.LoadUint64(&rb.dequeue)
+		start time.Time
+	)
+	if timeout > 0 {
+		start = time.Now()
+	}
+L:
+	for {
+		if atomic.LoadUint64(&rb.disposed) == 1 {
+			return nil, ErrDisposed
+		}
+
+		n = &rb.nodes[pos&rb.mask]
+		seq := atomic.LoadUint64(&n.position)
+		switch dif := seq - (pos + 1); {
+		case dif == 0:
+			if atomic.CompareAndSwapUint64(&rb.dequeue, pos, pos+1) {
+				break L
+			}
+		case dif < 0:
+			panic(`Ring buffer in compromised state during a get operation.`)
+		default:
+			pos = atomic.LoadUint64(&rb.dequeue)
+		}
+
+		if timeout > 0 && time.Since(start) >= timeout {
+			return nil, ErrTimeout
+		}
+
+		if timeout < 0 {
+			return nil, ErrTimeout
+		}
+
+		runtime.Gosched() // free up the cpu before the next iteration
+	}
+	data := n.data
+	n.data = nil
+	atomic.StoreUint64(&n.position, pos+rb.mask+1)
+	return data, nil
+}
+
+// Len returns the number of items in the queue.
+func (rb *RingBuffer) Len() uint64 {
+	return atomic.LoadUint64(&rb.queue) - atomic.LoadUint64(&rb.dequeue)
+}
+
+// Cap returns the capacity of this ring buffer.
+func (rb *RingBuffer) Cap() uint64 {
+	return uint64(len(rb.nodes))
+}
+
+// Dispose will dispose of this queue and free any blocked threads
+// in the Put and/or Get methods.  Calling those methods on a disposed
+// queue will return an error.
+func (rb *RingBuffer) Dispose() {
+	atomic.CompareAndSwapUint64(&rb.disposed, 0, 1)
+}
+
+// IsDisposed will return a bool indicating if this queue has been
+// disposed.
+func (rb *RingBuffer) IsDisposed() bool {
+	return atomic.LoadUint64(&rb.disposed) == 1
+}
+
+// NewRingBuffer will allocate, initialize, and return a ring buffer
+// with the specified size.
+func NewRingBuffer(size uint64) *RingBuffer {
+	rb := &RingBuffer{}
+	rb.init(size)
+	return rb
+}

s3_reader.go

@@ -60,7 +60,7 @@ func NewS3ReadCloser(path string) *S3ReadCloser {
 	bucket, key := parseS3Url(path)
 	sess := session.Must(session.NewSession(awsConfig()))
 
-	log.Println("[S3 Input] S3 connection succesfully initialized", path)
+	log.Println("[S3 Input] S3 connection successfully initialized", path)
 
 	return &S3ReadCloser{
 		bucket: bucket,

s3_test.go

@@ -127,7 +127,7 @@ func TestInputFileFromS3(t *testing.T) {
 		<-output.closeCh
 	}
 
-	input := NewFileInput(fmt.Sprintf("s3://test-gor-eu/%d", rnd), false)
+	input := NewFileInput(fmt.Sprintf("s3://test-gor-eu/%d", rnd), false, 100, 0, false)
 
 	buf := make([]byte, 1000)
 	for i := 0; i <= 19999; i++ {

settings.go

@@ -45,10 +45,13 @@ type AppSettings struct {
 	OutputTCPConfig TCPOutputConfig
 	OutputTCPStats  bool `json:"output-tcp-stats"`
 
-	InputFile        MultiOption `json:"input-file"`
-	InputFileLoop    bool        `json:"input-file-loop"`
-	OutputFile       MultiOption `json:"output-file"`
-	OutputFileConfig FileOutputConfig
+	InputFile          MultiOption   `json:"input-file"`
+	InputFileLoop      bool          `json:"input-file-loop"`
+	InputFileReadDepth int           `json:"input-file-read-depth"`
+	InputFileDryRun    bool          `json:"input-file-dry-run"`
+	InputFileMaxWait   time.Duration `json:"input-file-max-wait"`
+	OutputFile         MultiOption   `json:"output-file"`
+	OutputFileConfig   FileOutputConfig
 
 	InputRAW MultiOption `json:"input_raw"`
 	RAWInputConfig
@@ -113,6 +116,9 @@ func init() {
 
 	flag.Var(&Settings.InputFile, "input-file", "Read requests from file: \n\tgor --input-file ./requests.gor --output-http staging.com")
 	flag.BoolVar(&Settings.InputFileLoop, "input-file-loop", false, "Loop input files, useful for performance testing.")
+	flag.IntVar(&Settings.InputFileReadDepth, "input-file-read-depth", 100, "GoReplay tries to read and cache multiple records, in advance. In parallel it also perform sorting of requests, if they came out of order. Since it needs hold this buffer in memory, bigger values can cause worse performance")
+	flag.BoolVar(&Settings.InputFileDryRun, "input-file-dry-run", false, "Simulate reading from the data source without replaying it. You will get information about expected replay time, number of found records etc.")
+	flag.DurationVar(&Settings.InputFileMaxWait, "input-file-max-wait", 0, "Set the maximum time between requests. Can help in situations when you have too long periods between request, and you want to skip them. Example: --input-raw-max-wait 1s")
 
 	flag.Var(&Settings.OutputFile, "output-file", "Write incoming requests to file: \n\tgor --input-raw :80 --output-file ./requests.gor")
 	flag.DurationVar(&Settings.OutputFileConfig.FlushInterval, "output-file-flush-interval", time.Second, "Interval for forcing buffer flush to the file, default: 1s.")
@@ -123,7 +129,7 @@ func init() {
 
 	flag.StringVar(&Settings.OutputFileConfig.BufferPath, "output-file-buffer", "/tmp", "The path for temporary storing current buffer: \n\tgor --input-raw :80 --output-file s3://mybucket/logs/%Y-%m-%d.gz --output-file-buffer /mnt/logs")
 
-	flag.BoolVar(&Settings.PrettifyHTTP, "prettify-http", false, "If enabled, will automatically decode requests and responses with: Content-Encoding: gzip and Transfer-Encoding: chunked. Useful for debugging, in conjuction with --output-stdout")
+	flag.BoolVar(&Settings.PrettifyHTTP, "prettify-http", false, "If enabled, will automatically decode requests and responses with: Content-Encoding: gzip and Transfer-Encoding: chunked. Useful for debugging, in conjunction with --output-stdout")
 
 	// input raw flags
 	flag.Var(&Settings.InputRAW, "input-raw", "Capture traffic from given port (use RAW sockets and require *sudo* access):\n\t# Capture traffic from 8080 port\n\tgor --input-raw :8080 --output-http staging.com")
@@ -141,6 +147,7 @@ func init() {
 	flag.BoolVar(&Settings.Promiscuous, "input-raw-promisc", false, "enable promiscuous mode")
 	flag.BoolVar(&Settings.Monitor, "input-raw-monitor", false, "enable RF monitor mode")
 	flag.BoolVar(&Settings.Stats, "input-raw-stats", false, "enable stats generator on raw TCP messages")
+	flag.BoolVar(&Settings.AllowIncomplete, "input-raw-allow-incomplete", false, "If turned on Gor will record HTTP messages with missing packets")
 
 	flag.StringVar(&Settings.Middleware, "middleware", "", "Used for modifying traffic using external command")
 
@@ -187,7 +194,7 @@ func init() {
 	flag.StringVar(&Settings.KafkaTLSConfig.ClientCert, "kafka-tls-client-cert", "", "Client certificate for Kafka TLS Config (mandatory with to kafka-tls-ca-cert and kafka-tls-client-key)")
 	flag.StringVar(&Settings.KafkaTLSConfig.ClientKey, "kafka-tls-client-key", "", "Client Key for Kafka TLS Config (mandatory with to kafka-tls-client-cert and kafka-tls-client-key)")
 
-	flag.Var(&Settings.ModifierConfig.Headers, "http-set-header", "Inject additional headers to http reqest:\n\tgor --input-raw :8080 --output-http staging.com --http-set-header 'User-Agent: Gor'")
+	flag.Var(&Settings.ModifierConfig.Headers, "http-set-header", "Inject additional headers to http request:\n\tgor --input-raw :8080 --output-http staging.com --http-set-header 'User-Agent: Gor'")
 	flag.Var(&Settings.ModifierConfig.HeaderRewrite, "http-rewrite-header", "Rewrite the request header based on a mapping:\n\tgor --input-raw :8080 --output-http staging.com --http-rewrite-header Host: (.*).example.com,$1.beta.example.com")
 	flag.Var(&Settings.ModifierConfig.Params, "http-set-param", "Set request url param, if param already exists it will be overwritten:\n\tgor --input-raw :8080 --output-http staging.com --http-set-param api_key=1")
 	flag.Var(&Settings.ModifierConfig.Methods, "http-allow-method", "Whitelist of HTTP methods to replay. Anything else will be dropped:\n\tgor --input-raw :8080 --output-http staging.com --http-allow-method GET --http-allow-method OPTIONS")

simpletime/time.go

@@ -0,0 +1,17 @@
+package simpletime
+
+import (
+	"time"
+)
+
+var Now time.Time
+
+func init() {
+	go func() {
+		for {
+			// Accurate enough
+			Now = time.Now()
+			time.Sleep(100 * time.Millisecond)
+		}
+	}()
+}

tcp/doc.go

@@ -15,7 +15,7 @@ maxSize := 5 << 20
 debugger := func(debugLevel int, data ...interface{}){} // debugger can also be nil
 messageHandler := func(mssg *tcp.Message){}
 
-mssgPool := tcp.NewMessagePool(maxMessageSize, messageExpire, debugger, messageHandler)
+mssgPool := tcp.NewMessageParser(maxMessageSize, messageExpire, debugger, messageHandler)
 listener.Listen(ctx, mssgPool.Handler)
 
 you can use pool.End or/and pool.Start to set custom session behaviors

tcp/tcp_message.go

@@ -1,90 +1,184 @@
 package tcp
 
 import (
-	"bytes"
-	"crypto/sha1"
+	"encoding/binary"
 	"encoding/hex"
 	"fmt"
+	"net"
+	"reflect"
 	"sort"
 	"sync"
 	"time"
-
-	"github.com/buger/goreplay/size"
-	"github.com/google/gopacket"
+	"unsafe"
 )
 
+// TCPProtocol is a number to indicate type of protocol
+type TCPProtocol uint8
+
+const (
+	// ProtocolHTTP ...
+	ProtocolHTTP TCPProtocol = iota
+	// ProtocolBinary ...
+	ProtocolBinary
+)
+
+// Set is here so that TCPProtocol can implement flag.Var
+func (protocol *TCPProtocol) Set(v string) error {
+	switch v {
+	case "", "http":
+		*protocol = ProtocolHTTP
+	case "binary":
+		*protocol = ProtocolBinary
+	default:
+		return fmt.Errorf("unsupported protocol %s", v)
+	}
+	return nil
+}
+
+func (protocol *TCPProtocol) String() string {
+	switch *protocol {
+	case ProtocolBinary:
+		return "binary"
+	case ProtocolHTTP:
+		return "http"
+	default:
+		return ""
+	}
+}
+
 // Stats every message carry its own stats object
 type Stats struct {
-	LostData   int
-	Length     int       // length of the data
-	Start      time.Time // first packet's timestamp
-	End        time.Time // last packet's timestamp
-	SrcAddr    string
-	DstAddr    string
-	IsIncoming bool
-	TimedOut   bool // timeout before getting the whole message
-	Truncated  bool // last packet truncated due to max message size
-	IPversion  byte
+	LostData  int
+	Length    int       // length of the data
+	Start     time.Time // first packet's timestamp
+	End       time.Time // last packet's timestamp
+	SrcAddr   string
+	DstAddr   string
+	Direction Dir
+	TimedOut  bool // timeout before getting the whole message
+	Truncated bool // last packet truncated due to max message size
+	IPversion byte
 }
 
 // Message is the representation of a tcp message
 type Message struct {
-	packets []*Packet
-	done    chan bool
-	buf     bytes.Buffer
+	packets  []*Packet
+	parser   *MessageParser
+	feedback interface{}
+	Idx      uint16
 	Stats
 }
 
-// NewMessage ...
-func NewMessage(srcAddr, dstAddr string, ipVersion uint8) (m *Message) {
-	m = new(Message)
-	m.DstAddr = dstAddr
-	m.SrcAddr = srcAddr
-	m.IPversion = ipVersion
-	m.done = make(chan bool)
-	return
-}
-
-// UUID the unique id of a TCP session it is not granted to be unique!
+// UUID returns the UUID of a TCP request and its response.
 func (m *Message) UUID() []byte {
-	var src, dst string
-	if m.IsIncoming {
-		src = m.SrcAddr
-		dst = m.DstAddr
+	var streamID uint64
+	pckt := m.packets[0]
+
+	// check if response or request have generated the ID before.
+	if m.Direction == DirIncoming {
+		streamID = uint64(pckt.SrcPort)<<48 | uint64(pckt.DstPort)<<32 |
+			uint64(ip2int(pckt.SrcIP))
 	} else {
-		src = m.DstAddr
-		dst = m.SrcAddr
+		streamID = uint64(pckt.DstPort)<<48 | uint64(pckt.SrcPort)<<32 |
+			uint64(ip2int(pckt.DstIP))
 	}
 
-	length := len(src) + len(dst)
-	uuid := make([]byte, length)
-	copy(uuid, src)
-	copy(uuid[len(src):], dst)
-	sha := sha1.Sum(uuid)
-	uuid = make([]byte, 40)
-	hex.Encode(uuid, sha[:])
+	id := make([]byte, 12)
+	binary.BigEndian.PutUint64(id, streamID)
 
-	return uuid
-}
-
-func (m *Message) add(pckt *Packet) {
-	m.Length += len(pckt.Payload)
-	m.LostData += int(pckt.Lost)
-	m.packets = append(m.packets, pckt)
-	if len(pckt.Payload) > 0 {
-		m.buf.Write(pckt.Payload)
+	if m.Direction == DirIncoming {
+		binary.BigEndian.PutUint32(id[8:], pckt.Ack)
+	} else {
+		binary.BigEndian.PutUint32(id[8:], pckt.Seq)
 	}
-	m.End = pckt.Timestamp
+
+	uuidHex := make([]byte, 24)
+	hex.Encode(uuidHex[:], id[:])
+
+	return uuidHex
 }
 
-// Packets returns packets of this message
+func (m *Message) add(packet *Packet) bool {
+	// Skip duplicates
+	for _, p := range m.packets {
+		if p.Seq == packet.Seq {
+			return false
+		}
+	}
+
+	// Packets not always captured in same Seq order, and sometimes we need to prepend
+	if len(m.packets) == 0 || packet.Seq > m.packets[len(m.packets)-1].Seq {
+		m.packets = append(m.packets, packet)
+	} else if packet.Seq < m.packets[0].Seq {
+		m.packets = append([]*Packet{packet}, m.packets...)
+	} else { // insert somewhere in the middle...
+		for i, p := range m.packets {
+			if packet.Seq < p.Seq {
+				m.packets = append(m.packets[:i], append([]*Packet{packet}, m.packets[i:]...)...)
+				break
+			}
+		}
+	}
+
+	m.Length += len(packet.Payload)
+	m.LostData += int(packet.Lost)
+
+	if packet.Timestamp.After(m.End) || m.End.IsZero() {
+		m.End = packet.Timestamp
+	}
+
+	return true
+}
+
+// Packets returns packets of the message
 func (m *Message) Packets() []*Packet {
 	return m.packets
 }
 
+func (m *Message) MissingChunk() bool {
+	nextSeq := m.packets[0].Seq
+
+	for _, p := range m.packets {
+		if p.Seq != nextSeq {
+			return true
+		}
+
+		nextSeq += uint32(len(p.Payload))
+	}
+
+	return false
+}
+
+func (m *Message) PacketData() [][]byte {
+	tmp := make([][]byte, len(m.packets))
+
+	for i, p := range m.packets {
+		tmp[i] = p.Payload
+	}
+
+	return tmp
+}
+
 // Data returns data in this message
 func (m *Message) Data() []byte {
-	return m.buf.Bytes()
+	packetData := m.PacketData()
+	tmp := packetData[0]
+
+	if len(packetData) > 0 {
+		tmp, _ = copySlice(tmp, len(packetData[0]), packetData[1:]...)
+	}
+
+	return tmp
+}
+
+// SetProtocolState set feedback/data that can be used later, e.g with End or Start hint
+func (m *Message) SetProtocolState(feedback interface{}) {
+	m.feedback = feedback
+}
+
+// ProtocolState returns feedback associated to this message
+func (m *Message) ProtocolState() interface{} {
+	return m.feedback
 }
 
 // Sort a helper to sort packets
@@ -92,144 +186,243 @@ func (m *Message) Sort() {
 	sort.SliceStable(m.packets, func(i, j int) bool { return m.packets[i].Seq < m.packets[j].Seq })
 }
 
-// Handler message handler
-type Handler func(*Message)
+// Emitter message handler
+type Emitter func(*Message)
 
-// Debugger is the debugger function. first params is the indicator of the issue's priority
-// the higher the number, the lower the priority. it can be 4 <= level <= 6.
-type Debugger func(int, ...interface{})
-
-// HintEnd hints the pool to stop the session, see MessagePool.End
+// HintEnd hints the parser to stop the session, see MessageParser.End
 // when set, it will be executed before checking FIN or RST flag
 type HintEnd func(*Message) bool
 
-// HintStart hints the pool to start the reassembling the message, see MessagePool.Start
-// when set, it will be used instead of checking SYN flag
-type HintStart func(*Packet) (IsIncoming, IsOutgoing bool)
+// HintStart hints the parser to start the reassembling the message, see MessageParser.Start
+// when set, it will be called after checking SYN flag
+type HintStart func(*Packet) (IsRequest, IsOutgoing bool)
 
-// MessagePool holds data of all tcp messages in progress(still receiving/sending packets).
+// MessageParser holds data of all tcp messages in progress(still receiving/sending packets).
 // message is identified by its source port and dst port, and last 4bytes of src IP.
-type MessagePool struct {
-	sync.Mutex
-	debug         Debugger
-	maxSize       size.Size // maximum message size, default 5mb
-	pool          map[uint64]*Message
-	handler       Handler
-	messageExpire time.Duration // the maximum time to wait for the final packet, minimum is 100ms
-	End           HintEnd
-	Start         HintStart
+type MessageParser struct {
+	m  []map[uint64]*Message
+	mL []sync.RWMutex
+
+	messageExpire  time.Duration // the maximum time to wait for the final packet, minimum is 100ms
+	allowIncompete bool
+	End            HintEnd
+	Start          HintStart
+	ticker         *time.Ticker
+	messages       chan *Message
+	packets        chan *PcapPacket
+	close          chan struct{} // to signal that we are able to close
+	ports          []uint16
+	ips            []net.IP
 }
 
-// NewMessagePool returns a new instance of message pool
-func NewMessagePool(maxSize size.Size, messageExpire time.Duration, debugger Debugger, handler Handler) (pool *MessagePool) {
-	pool = new(MessagePool)
-	pool.debug = debugger
-	pool.handler = handler
-	pool.messageExpire = time.Millisecond * 100
-	if pool.messageExpire < messageExpire {
-		pool.messageExpire = messageExpire
+// NewMessageParser returns a new instance of message parser
+func NewMessageParser(messages chan *Message, ports []uint16, ips []net.IP, messageExpire time.Duration, allowIncompete bool) (parser *MessageParser) {
+	parser = new(MessageParser)
+
+	parser.messageExpire = messageExpire
+	if parser.messageExpire == 0 {
+		parser.messageExpire = time.Millisecond * 1000
 	}
-	pool.maxSize = maxSize
-	if pool.maxSize < 1 {
-		pool.maxSize = 5 << 20
+
+	parser.allowIncompete = allowIncompete
+
+	parser.packets = make(chan *PcapPacket, 10000)
+
+	if messages == nil {
+		messages = make(chan *Message, 100)
 	}
-	pool.pool = make(map[uint64]*Message)
-	return pool
+	parser.messages = messages
+	parser.ticker = time.NewTicker(time.Millisecond * 100)
+	parser.close = make(chan struct{}, 1)
+
+	parser.ports = ports
+	parser.ips = ips
+
+	for i := 0; i < 10; i++ {
+		parser.m = append(parser.m, make(map[uint64]*Message))
+		parser.mL = append(parser.mL, sync.RWMutex{})
+	}
+
+	for i := 0; i < 10; i++ {
+		go parser.wait(i)
+	}
+
+	return parser
 }
 
-// Handler returns packet handler
-func (pool *MessagePool) Handler(packet gopacket.Packet) {
-	var in, out bool
-	pckt, err := ParsePacket(packet)
-	if err != nil || pckt == nil {
-		go pool.say(4, fmt.Sprintf("error decoding packet(%dBytes):%s\n", packet.Metadata().CaptureLength, err))
+var packetLen int
+
+// Packet returns packet handler
+func (parser *MessageParser) PacketHandler(packet *PcapPacket) {
+	packetLen++
+	parser.packets <- packet
+}
+
+func (parser *MessageParser) wait(index int) {
+	var (
+		now time.Time
+	)
+	for {
+		select {
+		case pckt := <-parser.packets:
+			parser.processPacket(parser.parsePacket(pckt))
+		case now = <-parser.ticker.C:
+			parser.timer(now, index)
+		case <-parser.close:
+			parser.ticker.Stop()
+			// parser.Close should wait for this function to return
+			parser.close <- struct{}{}
+			return
+			// default:
+		}
+	}
+}
+
+func (parser *MessageParser) parsePacket(pcapPkt *PcapPacket) *Packet {
+	pckt, err := ParsePacket(pcapPkt.Data, pcapPkt.LType, pcapPkt.LTypeLen, pcapPkt.Ci, false)
+	if err != nil {
+		stats.Add("packet_error", 1)
+		return nil
+	}
+
+	for _, p := range parser.ports {
+		if pckt.DstPort == p {
+			for _, ip := range parser.ips {
+				if pckt.DstIP.Equal(ip) {
+					pckt.Direction = DirIncoming
+					break
+				}
+			}
+			break
+		}
+	}
+
+	return pckt
+}
+
+func (parser *MessageParser) processPacket(pckt *Packet) {
+	if pckt == nil {
 		return
 	}
-	pool.Lock()
-	defer pool.Unlock()
-	lst := 3
-	if pckt.Version == 6 {
-		lst = 15
-	}
-	key := uint64(pckt.SrcPort)<<48 | uint64(pckt.DstPort)<<32 |
-		uint64(pckt.SrcIP[lst])<<24 | uint64(pckt.SrcIP[lst-1])<<16 |
-		uint64(pckt.SrcIP[lst-2])<<8 | uint64(pckt.SrcIP[lst-3])
-	m, ok := pool.pool[key]
-	if pckt.RST {
-		if ok {
-			m.done <- true
-			<-m.done
+
+	// Trying to build unique hash, but there is small chance of collision
+	// No matter if it is request or response, all packets in the same message have same
+	mID := pckt.MessageID()
+	mIDX := pckt.SrcPort % 10
+
+	parser.mL[mIDX].Lock()
+	m, ok := parser.m[mIDX][mID]
+	if !ok {
+		parser.mL[mIDX].Unlock()
+
+		mIDX = pckt.DstPort % 10
+		parser.mL[mIDX].Lock()
+		m, ok = parser.m[mIDX][mID]
+
+		if !ok {
+			parser.mL[mIDX].Unlock()
 		}
-		key = uint64(pckt.DstPort)<<48 | uint64(pckt.SrcPort)<<32 |
-			uint64(pckt.DstIP[lst])<<24 | uint64(pckt.DstIP[lst-1])<<16 |
-			uint64(pckt.DstIP[lst-2])<<8 | uint64(pckt.DstIP[lst-3])
-		m, ok = pool.pool[key]
-		if ok {
-			m.done <- true
-			<-m.done
-		}
-		go pool.say(4, fmt.Sprintf("RST flag from %s to %s at %s\n", pckt.Src(), pckt.Dst(), pckt.Timestamp))
-		return
 	}
+
 	switch {
 	case ok:
-		pool.addPacket(m, pckt)
+		parser.addPacket(m, pckt)
+
+		parser.mL[mIDX].Unlock()
 		return
-	case pckt.SYN:
-		in = !pckt.ACK
-	case pool.Start != nil:
-		if in, out = pool.Start(pckt); !(in || out) {
-			return
+	case pckt.Direction == DirUnknown && parser.Start != nil:
+		if in, out := parser.Start(pckt); in || out {
+			if in {
+				pckt.Direction = DirIncoming
+			} else {
+				pckt.Direction = DirOutcoming
+			}
 		}
-	default:
-		return
 	}
-	m = NewMessage(pckt.Src(), pckt.Dst(), pckt.Version)
-	m.IsIncoming = in
-	pool.pool[key] = m
+
+	if pckt.Direction == DirIncoming {
+		mIDX = pckt.SrcPort % 10
+	} else {
+		mIDX = pckt.DstPort % 10
+	}
+
+	parser.mL[mIDX].Lock()
+
+	m = new(Message)
+	m.Direction = pckt.Direction
+	m.SrcAddr = pckt.SrcIP.String()
+	m.DstAddr = pckt.DstIP.String()
+
+	parser.m[mIDX][mID] = m
+
+	m.Idx = mIDX
 	m.Start = pckt.Timestamp
-	go pool.dispatch(key, m)
-	pool.addPacket(m, pckt)
+	m.parser = parser
+	parser.addPacket(m, pckt)
+
+	parser.mL[mIDX].Unlock()
 }
 
-func (pool *MessagePool) dispatch(key uint64, m *Message) {
-	select {
-	case <-m.done:
-		defer func() { m.done <- true }() // signal that message was dispatched
-	case <-time.After(pool.messageExpire):
-		pool.Lock()
-		defer pool.Unlock()
-		// avoid dispathing message twice
-		if _, ok := pool.pool[key]; !ok {
-			return
+func (parser *MessageParser) addPacket(m *Message, pckt *Packet) bool {
+	if !m.add(pckt) {
+		return false
+	}
+
+	// If we are using protocol parsing, like HTTP, depend on its parsing func.
+	// For the binary procols wait for message to expire
+	if parser.End != nil {
+		if parser.End(m) {
+			parser.Emit(m)
 		}
-		m.TimedOut = true
 	}
-	delete(pool.pool, key)
-	pool.handler(m)
+
+	return true
 }
 
-func (pool *MessagePool) addPacket(m *Message, pckt *Packet) {
-	trunc := m.Length + len(pckt.Payload) - int(pool.maxSize)
-	if trunc > 0 {
-		m.Truncated = true
-		pckt.Payload = pckt.Payload[:int(pool.maxSize)-m.Length]
-	}
-	m.add(pckt)
-	switch {
-	case trunc >= 0:
-	case pckt.FIN:
-	case pool.End != nil && pool.End(m):
-	default:
-		return
-	}
-	m.done <- true
-	<-m.done
+func (parser *MessageParser) Read() *Message {
+	m := <-parser.messages
+	return m
 }
 
-// this function should not block other pool operations
-func (pool *MessagePool) say(level int, args ...interface{}) {
-	if pool.debug != nil {
-		pool.debug(level, args...)
+func (parser *MessageParser) Emit(m *Message) {
+	stats.Add("message_count", 1)
+
+	delete(parser.m[m.Idx], m.packets[0].MessageID())
+
+	parser.messages <- m
+}
+
+func GetUnexportedField(field reflect.Value) interface{} {
+	return reflect.NewAt(field.Type(), unsafe.Pointer(field.UnsafeAddr())).Elem().Interface()
+}
+
+var failMsg int
+
+func (parser *MessageParser) timer(now time.Time, index int) {
+	packetLen = 0
+	parser.mL[index].Lock()
+
+	packetQueueLen.Set(int64(len(parser.packets)))
+	messageQueueLen.Set(int64(len(parser.m[index])))
+
+	for _, m := range parser.m[index] {
+		if now.Sub(m.End) > parser.messageExpire {
+			m.TimedOut = true
+			stats.Add("message_timeout_count", 1)
+			failMsg++
+			if parser.End == nil || parser.allowIncompete {
+				parser.Emit(m)
+			}
+
+			delete(parser.m[index], m.packets[0].MessageID())
+		}
 	}
+
+	parser.mL[index].Unlock()
+}
+
+func (parser *MessageParser) Close() error {
+	parser.close <- struct{}{}
+	<-parser.close // wait for timer to be closed!
+	return nil
 }

tcp/tcp_packet.go

@@ -2,12 +2,52 @@ package tcp
 
 import (
 	"encoding/binary"
+	"expvar"
 	"fmt"
 	"net"
 	"time"
 
 	"github.com/google/gopacket"
-	"github.com/google/gopacket/layers"
+)
+
+func copySlice(to []byte, skip int, from ...[]byte) ([]byte, int) {
+	var totalLen int
+	for _, s := range from {
+		totalLen += len(s)
+	}
+	totalLen += skip
+
+	if cap(to) < totalLen {
+		diff := totalLen - cap(to)
+		to = append(to, make([]byte, diff)...)
+	}
+
+	for _, s := range from {
+		skip += copy(to[skip:], s)
+	}
+
+	return to, skip
+}
+
+var stats *expvar.Map
+var packetQueueLen, messageQueueLen *expvar.Int
+
+func init() {
+	packetQueueLen = new(expvar.Int)
+	messageQueueLen = new(expvar.Int)
+
+	stats = expvar.NewMap("tcp")
+	stats.Init()
+	stats.Set("packet_queue", packetQueueLen)
+	stats.Set("message_queue", messageQueueLen)
+}
+
+type Dir int
+
+const (
+	DirUnknown = iota
+	DirIncoming
+	DirOutcoming
 )
 
 /*
@@ -17,82 +57,160 @@ calllers must make sure that ParsePacket has'nt returned any error before callin
 function.
 */
 type Packet struct {
-	// Link layer
-	gopacket.LinkLayer
+	Direction          Dir
+	messageID          uint64
+	SrcIP, DstIP       net.IP
+	Version            uint8
+	SrcPort, DstPort   uint16
+	Ack, Seq           uint32
+	ACK, SYN, FIN, RST bool
+	Lost               uint32
+	Retry              int
+	CaptureLength      int
+	Timestamp          time.Time
+	Payload            []byte
+	buf                []byte
 
-	// IP Header
-	Version uint8 // Ip version
-	SrcIP   net.IP
-	DstIP   net.IP
-	IHL     uint8
-	Length  uint16
+	created time.Time
+	gc      bool
+}
 
-	// TCP Segment Header
-	*layers.TCP
-
-	// Data info
-	Lost      uint16
-	Timestamp time.Time
+type PcapPacket struct {
+	Data     []byte
+	LType    int
+	LTypeLen int
+	Ci       *gopacket.CaptureInfo
 }
 
 // ParsePacket parse raw packets
-func ParsePacket(packet gopacket.Packet) (pckt *Packet, err error) {
-	// early check of error
-	if packet == nil {
-		return
-	}
-	defer func() {
-		if packet.ErrorLayer() != nil {
-			err = packet.ErrorLayer().Error()
-			return
-		}
-	}()
-
-	// initialization
+func ParsePacket(data []byte, lType, lTypeLen int, ci *gopacket.CaptureInfo, allowEmpty bool) (pckt *Packet, err error) {
 	pckt = new(Packet)
-	pckt.Timestamp = packet.Metadata().Timestamp
-	if pckt.Timestamp.IsZero() {
-		pckt.Timestamp = time.Now()
+	if err := pckt.parse(data, lType, lTypeLen, ci, allowEmpty); err != nil {
+		return nil, err
 	}
 
-	// parsing link layer
-	pckt.LinkLayer = packet.LinkLayer()
+	return pckt, nil
+}
 
-	// parsing network layer
-	if net4, ok := packet.NetworkLayer().(*layers.IPv4); ok {
+func (pckt *Packet) parse(data []byte, lType, lTypeLen int, cp *gopacket.CaptureInfo, allowEmpty bool) error {
+	pckt.Retry = 0
+	pckt.messageID = 0
+	pckt.buf = pckt.buf[:]
+
+	// TODO: check resolution
+	pckt.Timestamp = cp.Timestamp
+
+	if len(data) < lTypeLen {
+		return ErrHdrLength("Link")
+	}
+	if len(data) <= lTypeLen {
+		return ErrHdrMissing("IPv4 or IPv6")
+	}
+
+	ldata := data[lTypeLen:]
+	var proto byte
+	var netLayer, transLayer []byte
+
+	if ldata[0]>>4 == 4 {
+		// IPv4 header
+		if len(ldata) < 20 {
+			return ErrHdrLength("IPv4")
+		}
+		proto = ldata[9]
+		ihl := int(ldata[0]&0x0F) * 4
+		if ihl < 20 {
+			return ErrHdrInvalid("IPv4's IHL")
+		}
+		if len(ldata) < ihl {
+			return ErrHdrLength("IPv4 opts")
+		}
+		netLayer = ldata[:ihl]
+	} else if ldata[0]>>4 == 6 {
+		if len(ldata) < 40 {
+			return ErrHdrLength("IPv6")
+		}
+		proto = ldata[6]
+		totalLen := 40
+		for ipv6ExtensionHdr(proto) {
+			hdr := len(ldata) - totalLen
+			if hdr < 8 {
+				return ErrHdrExpected("IPv6 opts")
+			}
+			extLen := 8
+			if proto != 44 {
+				extLen = int(ldata[totalLen+1]+1) * 8
+			}
+			if hdr < extLen {
+				return ErrHdrLength("IPv6 opts")
+			}
+			proto = ldata[totalLen]
+			totalLen += extLen
+		}
+		netLayer = ldata[:totalLen]
+	} else {
+		return ErrHdrExpected("IPv4 or IPv6")
+	}
+	if proto != 6 {
+		return ErrHdrExpected("TCP")
+	}
+	if len(data) <= len(netLayer) {
+		return ErrHdrMissing("TCP")
+	}
+	ndata := ldata[len(netLayer):]
+	// TCP header
+	if len(ndata) < 20 {
+		return ErrHdrLength("TCP")
+	}
+	dOf := int(ndata[12]>>4) * 4
+	if dOf < 20 {
+		return ErrHdrInvalid("TCP's ndata offset")
+	}
+	if len(ndata) < dOf {
+		return ErrHdrLength("TCP opts")
+	}
+
+	if !allowEmpty && len(ndata[dOf:]) == 0 {
+		return EmptyPacket("")
+	}
+
+	if (netLayer[0] >> 4) == 4 {
+		// IPv4 header
 		pckt.Version = 4
-		pckt.SrcIP = net4.SrcIP
-		pckt.DstIP = net4.DstIP
-		pckt.IHL = net4.IHL * 4
-		pckt.Length = net4.Length
-	} else if net6, ok := packet.NetworkLayer().(*layers.IPv6); ok {
+		pckt.SrcIP = netLayer[12:16]
+		pckt.DstIP = netLayer[16:20]
+	} else {
+		// IPv6 header
 		pckt.Version = 6
-		pckt.SrcIP = net6.SrcIP
-		pckt.DstIP = net6.DstIP
-		pckt.IHL = 40
-		pckt.Length = net6.Length
-	} else {
-		pckt = nil
-		return
+		pckt.SrcIP = netLayer[8:24]
+		pckt.DstIP = netLayer[24:40]
 	}
 
-	// parsing tcp header(transportation layer)
-	if tcp, ok := packet.TransportLayer().(*layers.TCP); ok {
-		pckt.TCP = tcp
-	} else {
-		pckt = nil
-		return
-	}
-	pckt.DataOffset *= 4
+	transLayer = ndata[:dOf]
 
-	// calculating lost data
-	headerSize := int(uint32(pckt.DataOffset) + uint32(pckt.IHL))
-	if pckt.Version == 6 {
-		headerSize -= 40 // in ipv6 the length of payload doesn't include the IPheader size
-	}
-	pckt.Lost = pckt.Length - uint16(headerSize+len(pckt.Payload))
+	pckt.CaptureLength = cp.CaptureLength
+	pckt.SrcPort = binary.BigEndian.Uint16(transLayer[0:2])
+	pckt.DstPort = binary.BigEndian.Uint16(transLayer[2:4])
+	pckt.Seq = binary.BigEndian.Uint32(transLayer[4:8])
+	pckt.Ack = binary.BigEndian.Uint32(transLayer[8:12])
+	pckt.FIN = transLayer[13]&0x01 != 0
+	pckt.SYN = transLayer[13]&0x02 != 0
+	pckt.RST = transLayer[13]&0x04 != 0
+	pckt.ACK = transLayer[13]&0x10 != 0
+	pckt.Lost = uint32(cp.Length - cp.CaptureLength)
 
-	return
+	pckt.Payload = ndata[dOf:]
+
+	return nil
+}
+
+func (pckt *Packet) MessageID() uint64 {
+	if pckt.messageID == 0 {
+		// All packets in the same message will share the same ID
+		pckt.messageID = uint64(pckt.SrcPort)<<48 | uint64(pckt.DstPort)<<32 |
+			(uint64(ip2int(pckt.SrcIP)) + uint64(ip2int(pckt.DstIP)) + uint64(pckt.Ack))
+	}
+
+	return pckt.messageID
 }
 
 // Src returns the source socket of a packet
@@ -105,93 +223,53 @@ func (pckt *Packet) Dst() string {
 	return fmt.Sprintf("%s:%d", pckt.DstIP, pckt.DstPort)
 }
 
-// SYNOptions returns MSS and windowscale of syn packets
-func (pckt *Packet) SYNOptions() (mss uint16, windowscale byte) {
-	if !pckt.SYN {
-		return
-	}
-	for _, v := range pckt.Options {
-		if v.OptionType == layers.TCPOptionKindMSS {
-			mss = binary.BigEndian.Uint16(v.OptionData)
-			continue
-		}
-		if v.OptionType == layers.TCPOptionKindWindowScale {
-			if v.OptionLength > 0 {
-				windowscale = v.OptionData[0]
-			}
-		}
-	}
-	return
+type EmptyPacket string
+
+func (err EmptyPacket) Error() string {
+	return "Empty packet"
 }
 
-// LinkInfo returns info about the link layer
-func (pckt *Packet) LinkInfo() string {
-	if l, ok := pckt.LinkLayer.(*layers.Ethernet); ok {
-		return fmt.Sprintf(
-			"Source Mac: %s\nDestination Mac: %s\nProtocol: %s",
-			l.SrcMAC,
-			l.DstMAC,
-			l.EthernetType,
-		)
-	}
-	return "<Not Ethernet>"
+// ErrHdrLength returned on short header length
+type ErrHdrLength string
+
+func (err ErrHdrLength) Error() string {
+	return "short " + string(err) + " length"
 }
 
-// Flag returns formatted tcp flags
-func (pckt *Packet) Flag() (flag string) {
-	if pckt.FIN {
-		flag += "FIN, "
-	}
-	if pckt.SYN {
-		flag += "SYN, "
-	}
-	if pckt.RST {
-		flag += "RST, "
-	}
-	if pckt.PSH {
-		flag += "PSH, "
-	}
-	if pckt.ACK {
-		flag += "ACK, "
-	}
-	if pckt.URG {
-		flag += "URG, "
-	}
-	if len(flag) != 0 {
-		return flag[:len(flag)-2]
-	}
-	return flag
+// ErrHdrMissing returned on missing header(s)
+type ErrHdrMissing string
+
+func (err ErrHdrMissing) Error() string {
+	return "missing " + string(err) + " header(s)"
 }
 
-// String output for a TCP Packet
-func (pckt *Packet) String() string {
-	return fmt.Sprintf(`Time: %s
-%s
-Source: %s
-Destination: %s
-IHL: %d
-Total Length: %d
-Sequence: %d
-Acknowledgment: %d
-DataOffset: %d
-Window: %d
-Flag: %s
-Options: %s
-Data Size: %d
-Lost Data: %d`,
-		pckt.Timestamp.Format(time.StampNano),
-		pckt.LinkInfo(),
-		pckt.Src(),
-		pckt.Dst(),
-		pckt.IHL,
-		pckt.Length,
-		pckt.Seq,
-		pckt.Ack,
-		pckt.DataOffset,
-		pckt.Window,
-		pckt.Flag(),
-		pckt.Options,
-		len(pckt.Payload),
-		pckt.Lost,
-	)
+// ErrHdrExpected returned when header(s) are different from the one expected
+type ErrHdrExpected string
+
+func (err ErrHdrExpected) Error() string {
+	return "expected " + string(err) + " header(s)"
+}
+
+// ErrHdrInvalid returned when header(s) are different from the one expected
+type ErrHdrInvalid string
+
+func (err ErrHdrInvalid) Error() string {
+	return "invalid " + string(err) + " value"
+}
+
+// https://en.wikipedia.org/wiki/IPv6_packet#Extension_headers
+func ipv6ExtensionHdr(b byte) bool {
+	// TODO: support all extension headers
+	return b == 0 || b == 43 || b == 44
+}
+
+func ip2int(ip net.IP) uint32 {
+	if len(ip) == 0 {
+		return 0
+	}
+
+	if len(ip) == 16 {
+		return binary.BigEndian.Uint32(ip[12:16])
+	}
+	return binary.BigEndian.Uint32(ip)
 }

tcp/tcp_test.go

@@ -3,185 +3,223 @@ package tcp
 import (
 	"bytes"
 	"encoding/binary"
+
+	// "runtime"
 	"testing"
 	"time"
 
+	"github.com/stretchr/testify/assert"
+
 	"github.com/buger/goreplay/proto"
 	"github.com/google/gopacket"
 	"github.com/google/gopacket/layers"
 )
 
-var decodeOpts = gopacket.DecodeOptions{Lazy: true, NoCopy: true}
+func generateHeader(request bool, seq uint32, length uint16) []byte {
+	hdr := make([]byte, 4+24+24)
+	binary.BigEndian.PutUint32(hdr, uint32(layers.ProtocolFamilyIPv4))
 
-func headersIP4(seq uint32, length uint16) (headers [54]byte) {
-	// set ethernet headers
-	binary.BigEndian.PutUint16(headers[12:14], uint16(layers.EthernetTypeIPv4))
-
-	// set ip header
-	ip := headers[14:]
-	copy(ip[0:2], []byte{4<<4 | 5, 0x28<<2 | 0x00})
-	binary.BigEndian.PutUint16(ip[2:4], length+40)
+	ip := hdr[4:]
+	ip[0] = 4<<4 | 6
+	binary.BigEndian.PutUint16(ip[2:4], length+24+24)
 	ip[9] = uint8(layers.IPProtocolTCP)
-	copy(ip[12:16], []byte{192, 168, 1, 2})
-	copy(ip[16:], []byte{192, 168, 1, 3})
+	copy(ip[12:16], []byte{127, 0, 0, 1})
+	copy(ip[16:], []byte{127, 0, 0, 1})
 
 	// set tcp header
-	tcp := ip[20:]
-	binary.BigEndian.PutUint16(tcp[0:2], 45678)
-	binary.BigEndian.PutUint16(tcp[2:4], 8001)
-	tcp[12] = 5 << 4
-	return
+	tcp := ip[24:]
+	tcp[12] = 6 << 4
+
+	if request {
+		binary.BigEndian.PutUint16(tcp, 5535)
+		binary.BigEndian.PutUint16(tcp[2:], 8000)
+	} else {
+		binary.BigEndian.PutUint16(tcp, 8000)
+		binary.BigEndian.PutUint16(tcp[2:], 5535)
+	}
+	binary.BigEndian.PutUint32(tcp[4:], seq)
+	return hdr
 }
 
-func GetPackets(start uint32, _len int, payload []byte) []gopacket.Packet {
-	var packets = make([]gopacket.Packet, _len)
+func GetPackets(request bool, start uint32, _len int, payload []byte) []*Packet {
+	var packets = make([]*Packet, _len)
+	var err error
 	for i := start; i < start+uint32(_len); i++ {
-		data := make([]byte, 54+len(payload))
-		h := headersIP4(i, uint16(len(payload)))
-		copy(data, h[:])
-		copy(data[len(h):], payload)
-		packets[i-start] = gopacket.NewPacket(data, layers.LinkTypeEthernet, decodeOpts)
+		d := append(generateHeader(request, i, uint16(len(payload))), payload...)
+		ci := &gopacket.CaptureInfo{Length: len(d), CaptureLength: len(d), Timestamp: time.Now()}
+
+		packets[i-start], err = ParsePacket(d, int(layers.LinkTypeLoop), 4, ci, true)
+		if request {
+			packets[i-start].Direction = DirIncoming
+		} else {
+			packets[i-start].Direction = DirOutcoming
+		}
+		if err != nil {
+			panic(err)
+		}
 	}
 	return packets
 }
 
-func TestMessageParserWithHint(t *testing.T) {
-	var mssg = make(chan *Message, 3)
-	pool := NewMessagePool(1<<20, time.Second, nil, func(m *Message) { mssg <- m })
-	pool.Start = func(pckt *Packet) (bool, bool) {
+func TestRequestResponseMapping(t *testing.T) {
+	packets := []*Packet{
+		{SrcPort: 60000, DstPort: 80, Ack: 1, Seq: 1, Direction: DirIncoming, Timestamp: time.Unix(1, 0), Payload: []byte("GET / HTTP/1.1\r\n")},
+		{SrcPort: 60000, DstPort: 80, Ack: 1, Seq: 17, Direction: DirIncoming, Timestamp: time.Unix(2, 0), Payload: []byte("Host: localhost\r\n\r\n")},
+
+		// Seq of first response packet match Ack of first request packet
+		{SrcPort: 80, DstPort: 60000, Ack: 36, Seq: 1, Direction: DirOutcoming, Timestamp: time.Unix(3, 0), Payload: []byte("HTTP/1.1 200 OK\r\n")},
+		{SrcPort: 80, DstPort: 60000, Ack: 36, Seq: 18, Direction: DirOutcoming, Timestamp: time.Unix(4, 0), Payload: []byte("Content-Length: 0\r\n\r\n")},
+
+		// Same TCP stream
+		{SrcPort: 60000, DstPort: 80, Ack: 39, Seq: 36, Direction: DirIncoming, Timestamp: time.Unix(5, 0), Payload: []byte("GET / HTTP/1.1\r\n")},
+		{SrcPort: 60000, DstPort: 80, Ack: 39, Seq: 52, Direction: DirIncoming, Timestamp: time.Unix(6, 0), Payload: []byte("Host: localhost\r\n\r\n")},
+
+		// Seq of first response packet match Ack of first request packet
+		{SrcPort: 80, DstPort: 60000, Ack: 71, Seq: 39, Direction: DirOutcoming, Timestamp: time.Unix(7, 0), Payload: []byte("HTTP/1.1 200 OK\r\n")},
+		{SrcPort: 80, DstPort: 60000, Ack: 71, Seq: 56, Direction: DirOutcoming, Timestamp: time.Unix(8, 0), Payload: []byte("Content-Length: 0\r\n\r\n")},
+	}
+
+	parser := NewMessageParser(nil, nil, nil, time.Second, false)
+	parser.Start = func(pckt *Packet) (bool, bool) {
 		return proto.HasRequestTitle(pckt.Payload), proto.HasResponseTitle(pckt.Payload)
 	}
-	pool.End = func(m *Message) bool {
-		return proto.HasFullPayload(m.Data())
+	parser.End = func(m *Message) bool {
+		return proto.HasFullPayload(m, m.PacketData()...)
 	}
-	packets := GetPackets(1, 30, nil)
-	packets[0].Data()[14:][20:][13] = 2  // SYN flag
-	packets[10].Data()[14:][20:][13] = 2 // SYN flag
-	packets[29].Data()[14:][20:][13] = 1 // FIN flag
-	packets[4] = GetPackets(5, 1, []byte("HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nTransfer-Encoding: chunked\r\n\r\n7"))[0]
-	packets[5] = GetPackets(6, 1, []byte("\r\nMozilla\r\n9\r\nDeveloper\r"))[0]
-	packets[6] = GetPackets(7, 1, []byte("\n7\r\nNetwork\r\n0\r\n\r\n"))[0]
-	packets[14] = GetPackets(5, 1, []byte("POST / HTTP/1.1\r\nContent-Type: text/plain\r\nContent-Length: 23\r\n\r\n"))[0]
-	packets[15] = GetPackets(6, 1, []byte("MozillaDeveloper"))[0]
-	packets[16] = GetPackets(7, 1, []byte("Network"))[0]
-	packets[24] = GetPackets(5, 1, []byte("HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nContent-Length: 0\r\n\r"))[0]
-	for i := 0; i < 30; i++ {
-		pool.Handler(packets[i])
+
+	for _, packet := range packets {
+		parser.processPacket(packet)
 	}
-	var m *Message
-	select {
-	case <-time.After(time.Second):
-		t.Errorf("can't parse packets fast enough")
-		return
-	case m = <-mssg:
+
+	messages := []*Message{}
+	for i := 0; i < 4; i++ {
+		m := parser.Read()
+		messages = append(messages, m)
 	}
-	if len(m.packets) != 7 {
-		t.Errorf("expected to have 7 packets got %d", len(m.packets))
+
+	assert.Equal(t, int(messages[0].Direction), int(DirIncoming))
+	assert.Equal(t, int(messages[1].Direction), int(DirOutcoming))
+	assert.Equal(t, int(messages[2].Direction), int(DirIncoming))
+	assert.Equal(t, int(messages[3].Direction), int(DirOutcoming))
+
+	assert.Equal(t, messages[0].UUID(), messages[1].UUID())
+	assert.Equal(t, messages[2].UUID(), messages[3].UUID())
+
+	assert.NotEqual(t, messages[0].UUID(), messages[2].UUID())
+}
+
+func TestMessageParserWithHint(t *testing.T) {
+	parser := NewMessageParser(nil, nil, nil, time.Second, false)
+	parser.Start = func(pckt *Packet) (bool, bool) {
+		return proto.HasRequestTitle(pckt.Payload), proto.HasResponseTitle(pckt.Payload)
 	}
+	parser.End = func(m *Message) bool {
+		return proto.HasFullPayload(m, m.PacketData()...)
+	}
+
+	packets := []*Packet{
+		// Seq of first response packet match Ack of first request packet
+		{SrcPort: 80, DstPort: 60000, Ack: 1, Seq: 1, Direction: DirOutcoming, Timestamp: time.Unix(1, 0), Payload: []byte("HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nTransfer-Encoding: chunked\r\n\r\n7\r\n")},
+		{SrcPort: 80, DstPort: 60000, Ack: 1, Seq: 18, Direction: DirOutcoming, Timestamp: time.Unix(2, 0), Payload: []byte("\r\nMozilla\r\n9\r\nDeveloper\r")},
+		{SrcPort: 80, DstPort: 60000, Ack: 1, Seq: 42, Direction: DirOutcoming, Timestamp: time.Unix(3, 0), Payload: []byte("\n7\r\nNetwork\r\n0\r\n\r\n")},
+
+		{SrcPort: 60000, DstPort: 80, Ack: 60, Seq: 1, Direction: DirIncoming, Timestamp: time.Unix(4, 0), Payload: []byte("POST / HTTP/1.1\r\nContent-Type: text/plain\r\nContent-Length: 23\r\n\r\n")},
+		{SrcPort: 60000, DstPort: 80, Ack: 60, Seq: 66, Direction: DirIncoming, Timestamp: time.Unix(5, 0), Payload: []byte("MozillaDeveloper")},
+		{SrcPort: 60000, DstPort: 80, Ack: 60, Seq: 82, Direction: DirIncoming, Timestamp: time.Unix(6, 0), Payload: []byte("Network")},
+
+		{SrcPort: 80, DstPort: 60000, Ack: 89, Seq: 1, Direction: DirOutcoming, Timestamp: time.Unix(7, 0), Payload: []byte("HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nContent-Length: 0\r\n\r\n")},
+	}
+
+	for _, p := range packets {
+		parser.processPacket(p)
+	}
+
+	messages := []*Message{}
+	for i := 0; i < 3; i++ {
+		m := parser.Read()
+		messages = append(messages, m)
+	}
+
+	if !bytes.HasSuffix(messages[0].Data(), []byte("\n7\r\nNetwork\r\n0\r\n\r\n")) {
+		t.Errorf("expected to %q to have suffix %q", messages[0].Data(), []byte("\n7\r\nNetwork\r\n0\r\n\r\n"))
+	}
+
+	if !bytes.HasSuffix(messages[1].Data(), []byte("Network")) {
+		t.Errorf("expected to %q to have suffix %q", messages[1].Data(), []byte("Network"))
+	}
+
+	if !bytes.HasSuffix(messages[2].Data(), []byte("Content-Length: 0\r\n\r\n")) {
+		t.Errorf("expected to %q to have suffix %q", messages[2].Data(), []byte("Content-Length: 0\r\n\r\n"))
+	}
+}
+
+func TestMessageParserWrongOrder(t *testing.T) {
+	parser := NewMessageParser(nil, nil, nil, time.Second, false)
+	parser.Start = func(pckt *Packet) (bool, bool) {
+		return proto.HasRequestTitle(pckt.Payload), proto.HasResponseTitle(pckt.Payload)
+	}
+	parser.End = func(m *Message) bool {
+		return proto.HasFullPayload(m, m.PacketData()...)
+	}
+	packets := []*Packet{
+		// Seq of first response packet match Ack of first request packet
+		{SrcPort: 60000, DstPort: 80, Ack: 60, Seq: 66, Direction: DirIncoming, Timestamp: time.Unix(5, 0), Payload: []byte("MozillaDeveloper")},
+		{SrcPort: 80, DstPort: 60000, Ack: 1, Seq: 1, Direction: DirOutcoming, Timestamp: time.Unix(1, 0), Payload: []byte("HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nTransfer-Encoding: chunked\r\n\r\n7\r\n")},
+		{SrcPort: 80, DstPort: 60000, Ack: 1, Seq: 42, Direction: DirOutcoming, Timestamp: time.Unix(3, 0), Payload: []byte("\n7\r\nNetwork\r\n0\r\n\r\n")},
+
+		{SrcPort: 60000, DstPort: 80, Ack: 60, Seq: 1, Direction: DirIncoming, Timestamp: time.Unix(4, 0), Payload: []byte("POST / HTTP/1.1\r\nContent-Type: text/plain\r\nContent-Length: 23\r\n\r\n")},
+		{SrcPort: 80, DstPort: 60000, Ack: 1, Seq: 18, Direction: DirOutcoming, Timestamp: time.Unix(2, 0), Payload: []byte("\r\nMozilla\r\n9\r\nDeveloper\r")},
+
+		{SrcPort: 80, DstPort: 60000, Ack: 89, Seq: 1, Direction: DirOutcoming, Timestamp: time.Unix(7, 0), Payload: []byte("HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nContent-Length: 0\r\n\r\n")},
+		{SrcPort: 60000, DstPort: 80, Ack: 60, Seq: 82, Direction: DirIncoming, Timestamp: time.Unix(6, 0), Payload: []byte("Network")},
+	}
+
+	for _, p := range packets {
+		parser.processPacket(p)
+	}
+
+	m := parser.Read()
+
 	if !bytes.HasSuffix(m.Data(), []byte("\n7\r\nNetwork\r\n0\r\n\r\n")) {
 		t.Errorf("expected to %q to have suffix %q", m.Data(), []byte("\n7\r\nNetwork\r\n0\r\n\r\n"))
 	}
 
-	select {
-	case <-time.After(time.Second):
-		t.Errorf("can't parse packets fast enough")
-		return
-	case m = <-mssg:
-	}
-	if len(m.packets) != 7 {
-		t.Errorf("expected to have 7 packets got %d", len(m.packets))
+	m = parser.Read()
+
+	if !bytes.HasSuffix(m.Data(), []byte("Content-Length: 0\r\n\r\n")) {
+		t.Errorf("expected to %q to have suffix %q", m.Data(), []byte("Content-Length: 0\r\n\r\n"))
 	}
+
+	m = parser.Read()
+
 	if !bytes.HasSuffix(m.Data(), []byte("Network")) {
 		t.Errorf("expected to %q to have suffix %q", m.Data(), []byte("Network"))
 	}
-
-	select {
-	case <-time.After(time.Second):
-		t.Errorf("can't parse packets fast enough")
-		return
-	case m = <-mssg:
-	}
-	if len(m.packets) != 6 {
-		t.Errorf("expected to have 6 packets got %d", len(m.packets))
-	}
-	if !bytes.HasSuffix(m.Data(), []byte("Content-Length: 0\r\n\r")) {
-		t.Errorf("expected to %q to have suffix %q", m.Data(), []byte("Content-Length: 0\r\n\r"))
-	}
-
 }
 
 func TestMessageParserWithoutHint(t *testing.T) {
-	var mssg = make(chan *Message, 1)
 	var data [63 << 10]byte
-	packets := GetPackets(1, 10, data[:])
-	packets[0].Data()[14:][20:][13] = 2 // SYN flag
-	packets[9].Data()[14:][20:][13] = 1 // FIN flag
-	p := NewMessagePool(63<<10*10, time.Second, nil, func(m *Message) { mssg <- m })
+	packets := GetPackets(true, 1, 10, data[:])
+
+	p := NewMessageParser(nil, nil, nil, time.Second, false)
 	for _, v := range packets {
-		p.Handler(v)
-	}
-	var m *Message
-	select {
-	case <-time.After(time.Second):
-		t.Errorf("can't parse packets fast enough")
-		return
-	case m = <-mssg:
+		p.processPacket(v)
 	}
+	m := p.Read()
+
 	if m.Length != 63<<10*10 {
 		t.Errorf("expected %d to equal %d", m.Length, 63<<10*10)
 	}
 }
 
-func TestMessageMaxSizeReached(t *testing.T) {
-	var mssg = make(chan *Message, 2)
-	var data [63 << 10]byte
-	packets := GetPackets(1, 2, data[:])
-	packets = append(packets, GetPackets(3, 1, make([]byte, 63<<10+10))...)
-	packets[0].Data()[14:][20:][13] = 2 // SYN flag
-	packets[2].Data()[14:][20:][13] = 2 // SYN flag
-	packets[2].Data()[14:][15] = 3      // changing address
-	p := NewMessagePool(63<<10+10, time.Second, nil, func(m *Message) { mssg <- m })
-	for _, v := range packets {
-		p.Handler(v)
-	}
-	var m *Message
-	select {
-	case <-time.After(time.Second):
-		t.Errorf("can't parse packets fast enough")
-		return
-	case m = <-mssg:
-	}
-	if m.Length != 63<<10+10 {
-		t.Errorf("expected %d to equal %d", m.Length, 63<<10+10)
-	}
-	if !m.Truncated {
-		t.Error("expected message to be truncated")
-	}
-
-	select {
-	case <-time.After(time.Second):
-		t.Errorf("can't parse packets fast enough")
-		return
-	case m = <-mssg:
-	}
-	if m.Length != 63<<10+10 {
-		t.Errorf("expected %d to equal %d", m.Length, 63<<10+10)
-	}
-	if m.Truncated {
-		t.Error("expected message to not be truncated")
-	}
-}
-
 func TestMessageTimeoutReached(t *testing.T) {
-	var mssg = make(chan *Message, 2)
 	var data [63 << 10]byte
-	packets := GetPackets(1, 2, data[:])
-	packets[0].Data()[14:][20:][13] = 2 // SYN flag
-	p := NewMessagePool(1<<20, 0, nil, func(m *Message) { mssg <- m })
-	p.Handler(packets[0])
-	time.Sleep(time.Millisecond * 200)
-	p.Handler(packets[1])
-	m := <-mssg
+	packets := GetPackets(true, 1, 2, data[:])
+	p := NewMessageParser(nil, nil, nil, 10*time.Millisecond, true)
+	p.processPacket(packets[0])
+	time.Sleep(time.Millisecond * 100)
+	p.processPacket(packets[1])
+	m := p.Read()
 	if m.Length != 63<<10 {
 		t.Errorf("expected %d to equal %d", m.Length, 63<<10)
 	}
@@ -190,24 +228,29 @@ func TestMessageTimeoutReached(t *testing.T) {
 	}
 }
 
-func TestMessageUUID(t *testing.T) {
-	m1 := &Message{}
-	m1.IsIncoming = true
-	m1.SrcAddr = "src"
-	m1.DstAddr = "dst"
-	m2 := &Message{}
-	m2.SrcAddr = "dst"
-	m2.DstAddr = "src"
-	if string(m1.UUID()) != string(m2.UUID()) {
-		t.Errorf("expected %s, to equal %s", m1.UUID(), m2.UUID())
+func BenchmarkMessageUUID(b *testing.B) {
+	packets := GetPackets(true, 1, 5, nil)
+
+	var uuid []byte
+	parser := NewMessageParser(nil, nil, nil, 10*time.Millisecond, true)
+	for _, p := range packets {
+		parser.processPacket(p)
 	}
+
+	msg := parser.Read()
+
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		uuid = msg.UUID()
+	}
+	_ = uuid
 }
 
 func BenchmarkPacketParseAndSort(b *testing.B) {
 	m := new(Message)
 	m.packets = make([]*Packet, 100)
-	for i, v := range GetPackets(1, 100, nil) {
-		m.packets[i], _ = ParsePacket(v)
+	for i, v := range GetPackets(true, 1, 100, nil) {
+		m.packets[i] = v
 	}
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
@@ -216,21 +259,16 @@ func BenchmarkPacketParseAndSort(b *testing.B) {
 }
 
 func BenchmarkMessageParserWithoutHint(b *testing.B) {
-	var mssg = make(chan *Message, 1)
 	var chunk = []byte("111111111111111111111111111111")
-	packets := GetPackets(1, 1000, chunk)
-	packets[0].Data()[14:][20:][13] = 2      // SYN flag
-	packets[1000-1].Data()[14:][20:][13] = 1 // FIN flag
-	p := NewMessagePool(1<<20, time.Second*2, nil, func(m *Message) {
-		mssg <- m
-	})
+	packets := GetPackets(true, 1, 1000, chunk)
+	p := NewMessageParser(nil, nil, nil, 2*time.Second, false)
 	b.ResetTimer()
 	b.ReportMetric(float64(1000), "packets/op")
 	for i := 0; i < b.N; i++ {
 		for _, v := range packets {
-			p.Handler(v)
+			p.processPacket(v)
 		}
-		<-mssg
+		p.Read()
 	}
 }
 
@@ -242,25 +280,33 @@ func BenchmarkMessageParserWithHint(b *testing.B) {
 		buf[i] = chunk
 	}
 	buf[1001] = []byte("0\r\n\r\n")
-	packets := make([]gopacket.Packet, len(buf))
+	packets := make([]*Packet, len(buf))
 	for i := 0; i < len(buf); i++ {
-		packets[i] = GetPackets(uint32(i+10), 1, buf[i])[0]
+		packets[i] = GetPackets(false, 1, 1, buf[i])[0]
 	}
-	var mssg = make(chan *Message, 1)
-	pool := NewMessagePool(1<<30, time.Second*10, nil, func(m *Message) { mssg <- m })
-	pool.Start = func(pckt *Packet) (bool, bool) {
+
+	parser := NewMessageParser(nil, nil, nil, 2*time.Second, false)
+	parser.Start = func(pckt *Packet) (bool, bool) {
 		return false, proto.HasResponseTitle(pckt.Payload)
 	}
-	pool.End = func(m *Message) bool {
-		return proto.HasFullPayload(m.Data())
+	parser.End = func(m *Message) bool {
+		return proto.HasFullPayload(m, m.PacketData()...)
 	}
 	b.ResetTimer()
 	b.ReportMetric(float64(len(packets)), "packets/op")
 	b.ReportMetric(float64(1000), "chunks/op")
 	for i := 0; i < b.N; i++ {
 		for j := range packets {
-			pool.Handler(packets[j])
+			parser.processPacket(packets[j])
 		}
-		<-mssg
+		parser.Read()
+	}
+}
+
+func BenchmarkNewAndParsePacket(b *testing.B) {
+	data := append(generateHeader(true, 1024, 10), make([]byte, 10)...)
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		ParsePacket(data, int(layers.LinkTypeLoop), 4, &gopacket.CaptureInfo{}, true)
 	}
 }

vendor/github.com/Shopify/sarama/.gitignore

@@ -1,27 +0,0 @@
-# Compiled Object files, Static and Dynamic libs (Shared Objects)
-*.o
-*.a
-*.so
-*.test
-
-# Folders
-_obj
-_test
-.vagrant
-
-# Architecture specific extensions/prefixes
-*.[568vq]
-[568vq].out
-
-*.cgo1.go
-*.cgo2.c
-_cgo_defun.c
-_cgo_gotypes.go
-_cgo_export.*
-
-_testmain.go
-
-*.exe
-
-coverage.txt
-profile.out

vendor/github.com/Shopify/sarama/.golangci.yml

@@ -1,74 +0,0 @@
-run:
-  timeout: 5m
-  deadline: 10m
-
-linters-settings:
-  govet:
-    check-shadowing: false
-  golint:
-    min-confidence: 0
-  gocyclo:
-    min-complexity: 99
-  maligned:
-    suggest-new: true
-  dupl:
-    threshold: 100
-  goconst:
-    min-len: 2
-    min-occurrences: 3
-  misspell:
-    locale: US
-  goimports:
-    local-prefixes: github.com/Shopify/sarama
-  gocritic:
-    enabled-tags:
-      - diagnostic
-      - experimental
-      - opinionated
-      - performance
-      - style
-    disabled-checks:
-      - wrapperFunc
-      - ifElseChain
-  funlen:
-    lines: 300
-    statements: 300
-
-linters:
-  disable-all: true
-  enable:
-    - bodyclose
-    - deadcode
-    - depguard
-    - dogsled
-    # - dupl
-    - errcheck
-    - funlen
-    # - gocritic
-    - gocyclo
-    - gofmt
-    - goimports
-    # - golint
-    - gosec
-    # - gosimple
-    - govet
-    # - ineffassign
-    - interfacer
-    # - misspell
-    # - nakedret
-    # - scopelint
-    # - staticcheck
-    - structcheck
-    # - stylecheck
-    - typecheck
-    - unconvert
-    - unused
-    - varcheck
-    - whitespace
-    # - goconst
-    # - gochecknoinits
-
-issues:
-  exclude:
-    - consider giving a name to these results
-    - include an explanation for nolint directive

vendor/github.com/Shopify/sarama/CHANGELOG.md

@@ -1,912 +0,0 @@
-# Changelog
-
-#### Unreleased
-
-#### Version 1.26.1 (2020-02-04)
-
-Improvements:
-- Add requests-in-flight metric ([1539](https://github.com/Shopify/sarama/pull/1539))
-- Fix misleading example for cluster admin ([1595](https://github.com/Shopify/sarama/pull/1595))
-- Replace Travis with GitHub Actions, linters housekeeping ([1573](https://github.com/Shopify/sarama/pull/1573))
-- Allow BalanceStrategy to provide custom assignment data ([1592](https://github.com/Shopify/sarama/pull/1592))
-
-Bug Fixes:
-- Adds back Consumer.Offsets.CommitInterval to fix API ([1590](https://github.com/Shopify/sarama/pull/1590))
-- Fix error message s/CommitInterval/AutoCommit.Interval ([1589](https://github.com/Shopify/sarama/pull/1589))
-
-#### Version 1.26.0 (2020-01-24)
-
-New Features:
-- Enable zstd compression
-  ([1574](https://github.com/Shopify/sarama/pull/1574),
-  [1582](https://github.com/Shopify/sarama/pull/1582))
-- Support headers in tools kafka-console-producer
-  ([1549](https://github.com/Shopify/sarama/pull/1549))
-
-Improvements:
-- Add SASL AuthIdentity to SASL frames (authzid)
-  ([1585](https://github.com/Shopify/sarama/pull/1585)).
-
-Bug Fixes:
-- Sending messages with ZStd compression enabled fails in multiple ways
-  ([1252](https://github.com/Shopify/sarama/issues/1252)).
-- Use the broker for any admin on BrokerConfig
-  ([1571](https://github.com/Shopify/sarama/pull/1571)).
-- Set DescribeConfigRequest Version field
-  ([1576](https://github.com/Shopify/sarama/pull/1576)).
-- ConsumerGroup flooding logs with client/metadata update req
-  ([1578](https://github.com/Shopify/sarama/pull/1578)).
-- MetadataRequest version in DescribeCluster
-  ([1580](https://github.com/Shopify/sarama/pull/1580)).
-- Fix deadlock in consumer group handleError
-  ([1581](https://github.com/Shopify/sarama/pull/1581))
-- Fill in the Fetch{Request,Response} protocol
-  ([1582](https://github.com/Shopify/sarama/pull/1582)).
-- Retry topic request on ControllerNotAvailable
-  ([1586](https://github.com/Shopify/sarama/pull/1586)).
-
-#### Version 1.25.0 (2020-01-13)
-
-New Features:
-- Support TLS protocol in kafka-producer-performance
-  ([1538](https://github.com/Shopify/sarama/pull/1538)).
-- Add support for kafka 2.4.0
-  ([1552](https://github.com/Shopify/sarama/pull/1552)).
-
-Improvements:
-- Allow the Consumer to disable auto-commit offsets
-  ([1164](https://github.com/Shopify/sarama/pull/1164)).
-- Produce records with consistent timestamps
-  ([1455](https://github.com/Shopify/sarama/pull/1455)).
-
-Bug Fixes:
-- Fix incorrect SetTopicMetadata name mentions
-  ([1534](https://github.com/Shopify/sarama/pull/1534)).
-- Fix client.tryRefreshMetadata Println
-  ([1535](https://github.com/Shopify/sarama/pull/1535)).
-- Fix panic on calling updateMetadata on closed client
-  ([1531](https://github.com/Shopify/sarama/pull/1531)).
-- Fix possible faulty metrics in TestFuncProducing
-  ([1545](https://github.com/Shopify/sarama/pull/1545)).
-
-#### Version 1.24.1 (2019-10-31)
-
-New Features:
-- Add DescribeLogDirs Request/Response pair
-  ([1520](https://github.com/Shopify/sarama/pull/1520)).
-
-Bug Fixes:
-- Fix ClusterAdmin returning invalid controller ID on DescribeCluster
-  ([1518](https://github.com/Shopify/sarama/pull/1518)).
-- Fix issue with consumergroup not rebalancing when new partition is added
-  ([1525](https://github.com/Shopify/sarama/pull/1525)).
-- Ensure consistent use of read/write deadlines
-  ([1529](https://github.com/Shopify/sarama/pull/1529)).
-
-#### Version 1.24.0 (2019-10-09)
-
-New Features:
-- Add sticky partition assignor
-  ([1416](https://github.com/Shopify/sarama/pull/1416)).
-- Switch from cgo zstd package to pure Go implementation
-  ([1477](https://github.com/Shopify/sarama/pull/1477)).
-
-Improvements:
-- Allow creating ClusterAdmin from client
-  ([1415](https://github.com/Shopify/sarama/pull/1415)).
-- Set KafkaVersion in ListAcls method
-  ([1452](https://github.com/Shopify/sarama/pull/1452)).
-- Set request version in CreateACL ClusterAdmin method
-  ([1458](https://github.com/Shopify/sarama/pull/1458)).
-- Set request version in DeleteACL ClusterAdmin method
-  ([1461](https://github.com/Shopify/sarama/pull/1461)).
-- Handle missed error codes on TopicMetaDataRequest and GroupCoordinatorRequest
-  ([1464](https://github.com/Shopify/sarama/pull/1464)).
-- Remove direct usage of gofork
-  ([1465](https://github.com/Shopify/sarama/pull/1465)).
-- Add support for Go 1.13
-  ([1478](https://github.com/Shopify/sarama/pull/1478)).
-- Improve behavior of NewMockListAclsResponse
-  ([1481](https://github.com/Shopify/sarama/pull/1481)).
-
-Bug Fixes:
-- Fix race condition in consumergroup example
-  ([1434](https://github.com/Shopify/sarama/pull/1434)).
-- Fix brokerProducer goroutine leak
-  ([1442](https://github.com/Shopify/sarama/pull/1442)).
-- Use released version of lz4 library
-  ([1469](https://github.com/Shopify/sarama/pull/1469)).
-- Set correct version in MockDeleteTopicsResponse
-  ([1484](https://github.com/Shopify/sarama/pull/1484)).
-- Fix CLI help message typo
-  ([1494](https://github.com/Shopify/sarama/pull/1494)).
-
-Known Issues:
-- Please **don't** use Zstd, as it doesn't work right now.
-  See https://github.com/Shopify/sarama/issues/1252
-
-#### Version 1.23.1 (2019-07-22)
-
-Bug Fixes:
-- Fix fetch delete bug record
-  ([1425](https://github.com/Shopify/sarama/pull/1425)).
-- Handle SASL/OAUTHBEARER token rejection
-  ([1428](https://github.com/Shopify/sarama/pull/1428)).
-
-#### Version 1.23.0 (2019-07-02)
-
-New Features:
-- Add support for Kafka 2.3.0
-  ([1418](https://github.com/Shopify/sarama/pull/1418)).
-- Add support for ListConsumerGroupOffsets v2
-  ([1374](https://github.com/Shopify/sarama/pull/1374)).
-- Add support for DeleteConsumerGroup
-  ([1417](https://github.com/Shopify/sarama/pull/1417)).
-- Add support for SASLVersion configuration
-  ([1410](https://github.com/Shopify/sarama/pull/1410)).
-- Add kerberos support
-  ([1366](https://github.com/Shopify/sarama/pull/1366)).
-
-Improvements:
-- Improve sasl_scram_client example
-  ([1406](https://github.com/Shopify/sarama/pull/1406)).
-- Fix shutdown and race-condition in consumer-group example
-  ([1404](https://github.com/Shopify/sarama/pull/1404)).
-- Add support for error codes 77—81
-  ([1397](https://github.com/Shopify/sarama/pull/1397)).
-- Pool internal objects allocated per message
-  ([1385](https://github.com/Shopify/sarama/pull/1385)).
-- Reduce packet decoder allocations
-  ([1373](https://github.com/Shopify/sarama/pull/1373)).
-- Support timeout when fetching metadata
-  ([1359](https://github.com/Shopify/sarama/pull/1359)).
-
-Bug Fixes:
-- Fix fetch size integer overflow
-  ([1376](https://github.com/Shopify/sarama/pull/1376)).
-- Handle and log throttled FetchResponses
-  ([1383](https://github.com/Shopify/sarama/pull/1383)).
-- Refactor misspelled word Resouce to Resource
-  ([1368](https://github.com/Shopify/sarama/pull/1368)).
-
-#### Version 1.22.1 (2019-04-29)
-
-Improvements:
-- Use zstd 1.3.8
-  ([1350](https://github.com/Shopify/sarama/pull/1350)).
-- Add support for SaslHandshakeRequest v1
-  ([1354](https://github.com/Shopify/sarama/pull/1354)).
-
-Bug Fixes:
-- Fix V5 MetadataRequest nullable topics array
-  ([1353](https://github.com/Shopify/sarama/pull/1353)).
-- Use a different SCRAM client for each broker connection
-  ([1349](https://github.com/Shopify/sarama/pull/1349)).
-- Fix AllowAutoTopicCreation for MetadataRequest greater than v3
-  ([1344](https://github.com/Shopify/sarama/pull/1344)).
-
-#### Version 1.22.0 (2019-04-09)
-
-New Features:
-- Add Offline Replicas Operation to Client
-  ([1318](https://github.com/Shopify/sarama/pull/1318)).
-- Allow using proxy when connecting to broker
-  ([1326](https://github.com/Shopify/sarama/pull/1326)).
-- Implement ReadCommitted
-  ([1307](https://github.com/Shopify/sarama/pull/1307)).
-- Add support for Kafka 2.2.0
-  ([1331](https://github.com/Shopify/sarama/pull/1331)).
-- Add SASL SCRAM-SHA-512 and SCRAM-SHA-256 mechanismes
-  ([1331](https://github.com/Shopify/sarama/pull/1295)).
-
-Improvements:
-- Unregister all broker metrics on broker stop
-  ([1232](https://github.com/Shopify/sarama/pull/1232)).
-- Add SCRAM authentication example
-  ([1303](https://github.com/Shopify/sarama/pull/1303)).
-- Add consumergroup examples
-  ([1304](https://github.com/Shopify/sarama/pull/1304)).
-- Expose consumer batch size metric
-  ([1296](https://github.com/Shopify/sarama/pull/1296)).
-- Add TLS options to console producer and consumer
-  ([1300](https://github.com/Shopify/sarama/pull/1300)).
-- Reduce client close bookkeeping
-  ([1297](https://github.com/Shopify/sarama/pull/1297)).
-- Satisfy error interface in create responses
-  ([1154](https://github.com/Shopify/sarama/pull/1154)).
-- Please lint gods
-  ([1346](https://github.com/Shopify/sarama/pull/1346)).
-
-Bug Fixes:
-- Fix multi consumer group instance crash
-  ([1338](https://github.com/Shopify/sarama/pull/1338)).
-- Update lz4 to latest version
-  ([1347](https://github.com/Shopify/sarama/pull/1347)).
-- Retry ErrNotCoordinatorForConsumer in new consumergroup session
-  ([1231](https://github.com/Shopify/sarama/pull/1231)).
-- Fix cleanup error handler
-  ([1332](https://github.com/Shopify/sarama/pull/1332)).
-- Fix rate condition in PartitionConsumer
-  ([1156](https://github.com/Shopify/sarama/pull/1156)).
-
-#### Version 1.21.0 (2019-02-24)
-
-New Features:
-- Add CreateAclRequest, DescribeAclRequest, DeleteAclRequest
-  ([1236](https://github.com/Shopify/sarama/pull/1236)).
-- Add DescribeTopic, DescribeConsumerGroup, ListConsumerGroups, ListConsumerGroupOffsets admin requests
-  ([1178](https://github.com/Shopify/sarama/pull/1178)).
-- Implement SASL/OAUTHBEARER
-  ([1240](https://github.com/Shopify/sarama/pull/1240)).
-
-Improvements:
-- Add Go mod support
-  ([1282](https://github.com/Shopify/sarama/pull/1282)).
-- Add error codes 73—76
-  ([1239](https://github.com/Shopify/sarama/pull/1239)).
-- Add retry backoff function
-  ([1160](https://github.com/Shopify/sarama/pull/1160)).
-- Maintain metadata in the producer even when retries are disabled
-  ([1189](https://github.com/Shopify/sarama/pull/1189)).
-- Include ReplicaAssignment in ListTopics
-  ([1274](https://github.com/Shopify/sarama/pull/1274)).
-- Add producer performance tool
-  ([1222](https://github.com/Shopify/sarama/pull/1222)).
-- Add support LogAppend timestamps
-  ([1258](https://github.com/Shopify/sarama/pull/1258)).
-
-Bug Fixes:
-- Fix potential deadlock when a heartbeat request fails
-  ([1286](https://github.com/Shopify/sarama/pull/1286)).
-- Fix consuming compacted topic
-  ([1227](https://github.com/Shopify/sarama/pull/1227)).
-- Set correct Kafka version for DescribeConfigsRequest v1
-  ([1277](https://github.com/Shopify/sarama/pull/1277)).
-- Update kafka test version
-  ([1273](https://github.com/Shopify/sarama/pull/1273)).
-
-#### Version 1.20.1 (2019-01-10)
-
-New Features:
-- Add optional replica id in offset request
-  ([1100](https://github.com/Shopify/sarama/pull/1100)).
-
-Improvements:
-- Implement DescribeConfigs Request + Response v1 & v2
-  ([1230](https://github.com/Shopify/sarama/pull/1230)).
-- Reuse compression objects
-  ([1185](https://github.com/Shopify/sarama/pull/1185)).
-- Switch from png to svg for GoDoc link in README
-  ([1243](https://github.com/Shopify/sarama/pull/1243)).
-- Fix typo in deprecation notice for FetchResponseBlock.Records
-  ([1242](https://github.com/Shopify/sarama/pull/1242)).
-- Fix typos in consumer metadata response file
-  ([1244](https://github.com/Shopify/sarama/pull/1244)).
-
-Bug Fixes:
-- Revert to individual msg retries for non-idempotent
-  ([1203](https://github.com/Shopify/sarama/pull/1203)).
-- Respect MaxMessageBytes limit for uncompressed messages
-  ([1141](https://github.com/Shopify/sarama/pull/1141)).
-
-#### Version 1.20.0 (2018-12-10)
-
-New Features:
- - Add support for zstd compression
-   ([#1170](https://github.com/Shopify/sarama/pull/1170)).
- - Add support for Idempotent Producer
-   ([#1152](https://github.com/Shopify/sarama/pull/1152)).
- - Add support support for Kafka 2.1.0
-   ([#1229](https://github.com/Shopify/sarama/pull/1229)).
- - Add support support for OffsetCommit request/response pairs versions v1 to v5
-   ([#1201](https://github.com/Shopify/sarama/pull/1201)).
- - Add support support for OffsetFetch request/response pair up to version v5
-   ([#1198](https://github.com/Shopify/sarama/pull/1198)).
-
-Improvements:
- - Export broker's Rack setting
-   ([#1173](https://github.com/Shopify/sarama/pull/1173)).
- - Always use latest patch version of Go on CI
-   ([#1202](https://github.com/Shopify/sarama/pull/1202)).
- - Add error codes 61 to 72
-   ([#1195](https://github.com/Shopify/sarama/pull/1195)).
-
-Bug Fixes:
- - Fix build without cgo
-   ([#1182](https://github.com/Shopify/sarama/pull/1182)).
- - Fix go vet suggestion in consumer group file
-   ([#1209](https://github.com/Shopify/sarama/pull/1209)).
- - Fix typos in code and comments
-   ([#1228](https://github.com/Shopify/sarama/pull/1228)).
-
-#### Version 1.19.0 (2018-09-27)
-
-New Features:
- - Implement a higher-level consumer group
-   ([#1099](https://github.com/Shopify/sarama/pull/1099)).
-
-Improvements:
- - Add support for Go 1.11
-   ([#1176](https://github.com/Shopify/sarama/pull/1176)).
-
-Bug Fixes:
- - Fix encoding of `MetadataResponse` with version 2 and higher
-   ([#1174](https://github.com/Shopify/sarama/pull/1174)).
- - Fix race condition in mock async producer
-   ([#1174](https://github.com/Shopify/sarama/pull/1174)).
-
-#### Version 1.18.0 (2018-09-07)
-
-New Features:
- - Make `Partitioner.RequiresConsistency` vary per-message
-   ([#1112](https://github.com/Shopify/sarama/pull/1112)).
- - Add customizable partitioner
-   ([#1118](https://github.com/Shopify/sarama/pull/1118)).
- - Add `ClusterAdmin` support for `CreateTopic`, `DeleteTopic`, `CreatePartitions`,
-   `DeleteRecords`, `DescribeConfig`, `AlterConfig`, `CreateACL`, `ListAcls`, `DeleteACL`
-   ([#1055](https://github.com/Shopify/sarama/pull/1055)).
-
-Improvements:
- - Add support for Kafka 2.0.0
-   ([#1149](https://github.com/Shopify/sarama/pull/1149)).
- - Allow setting `LocalAddr` when dialing an address to support multi-homed hosts
-   ([#1123](https://github.com/Shopify/sarama/pull/1123)).
- - Simpler offset management
-   ([#1127](https://github.com/Shopify/sarama/pull/1127)).
-
-Bug Fixes:
- - Fix mutation of `ProducerMessage.MetaData` when producing to Kafka
-   ([#1110](https://github.com/Shopify/sarama/pull/1110)).
- - Fix consumer block when response did not contain all the
-   expected topic/partition blocks
-   ([#1086](https://github.com/Shopify/sarama/pull/1086)).
- - Fix consumer block when response contains only constrol messages
-   ([#1115](https://github.com/Shopify/sarama/pull/1115)).
- - Add timeout config for ClusterAdmin requests
-   ([#1142](https://github.com/Shopify/sarama/pull/1142)).
- - Add version check when producing message with headers
-   ([#1117](https://github.com/Shopify/sarama/pull/1117)).
- - Fix `MetadataRequest` for empty list of topics
-   ([#1132](https://github.com/Shopify/sarama/pull/1132)).
- - Fix producer topic metadata on-demand fetch when topic error happens in metadata response
-   ([#1125](https://github.com/Shopify/sarama/pull/1125)).
-
-#### Version 1.17.0 (2018-05-30)
-
-New Features:
- - Add support for gzip compression levels
-   ([#1044](https://github.com/Shopify/sarama/pull/1044)).
- - Add support for Metadata request/response pairs versions v1 to v5
-   ([#1047](https://github.com/Shopify/sarama/pull/1047),
-    [#1069](https://github.com/Shopify/sarama/pull/1069)).
- - Add versioning to JoinGroup request/response pairs
-   ([#1098](https://github.com/Shopify/sarama/pull/1098))
- - Add support for CreatePartitions, DeleteGroups, DeleteRecords request/response pairs
-   ([#1065](https://github.com/Shopify/sarama/pull/1065),
-    [#1096](https://github.com/Shopify/sarama/pull/1096),
-    [#1027](https://github.com/Shopify/sarama/pull/1027)).
- - Add `Controller()` method to Client interface
-   ([#1063](https://github.com/Shopify/sarama/pull/1063)).
-
-Improvements:
- - ConsumerMetadataReq/Resp has been migrated to FindCoordinatorReq/Resp
-   ([#1010](https://github.com/Shopify/sarama/pull/1010)).
- - Expose missing protocol parts: `msgSet` and `recordBatch`
-   ([#1049](https://github.com/Shopify/sarama/pull/1049)).
- - Add support for v1 DeleteTopics Request
-   ([#1052](https://github.com/Shopify/sarama/pull/1052)).
- - Add support for Go 1.10
-   ([#1064](https://github.com/Shopify/sarama/pull/1064)).
- - Claim support for Kafka 1.1.0
-   ([#1073](https://github.com/Shopify/sarama/pull/1073)).
-
-Bug Fixes:
- - Fix FindCoordinatorResponse.encode to allow nil Coordinator
-   ([#1050](https://github.com/Shopify/sarama/pull/1050),
-    [#1051](https://github.com/Shopify/sarama/pull/1051)).
- - Clear all metadata when we have the latest topic info
-   ([#1033](https://github.com/Shopify/sarama/pull/1033)).
- - Make `PartitionConsumer.Close` idempotent
-   ([#1092](https://github.com/Shopify/sarama/pull/1092)).
-
-#### Version 1.16.0 (2018-02-12)
-
-New Features:
- - Add support for the Create/Delete Topics request/response pairs
-   ([#1007](https://github.com/Shopify/sarama/pull/1007),
-    [#1008](https://github.com/Shopify/sarama/pull/1008)).
- - Add support for the Describe/Create/Delete ACL request/response pairs
-   ([#1009](https://github.com/Shopify/sarama/pull/1009)).
- - Add support for the five transaction-related request/response pairs
-   ([#1016](https://github.com/Shopify/sarama/pull/1016)).
-
-Improvements:
- - Permit setting version on mock producer responses
-   ([#999](https://github.com/Shopify/sarama/pull/999)).
- - Add `NewMockBrokerListener` helper for testing TLS connections
-   ([#1019](https://github.com/Shopify/sarama/pull/1019)).
- - Changed the default value for `Consumer.Fetch.Default` from 32KiB to 1MiB
-   which results in much higher throughput in most cases
-   ([#1024](https://github.com/Shopify/sarama/pull/1024)).
- - Reuse the `time.Ticker` across fetch requests in the PartitionConsumer to
-   reduce CPU and memory usage when processing many partitions
-   ([#1028](https://github.com/Shopify/sarama/pull/1028)).
- - Assign relative offsets to messages in the producer to save the brokers a
-   recompression pass
-   ([#1002](https://github.com/Shopify/sarama/pull/1002),
-    [#1015](https://github.com/Shopify/sarama/pull/1015)).
-
-Bug Fixes:
- - Fix producing uncompressed batches with the new protocol format
-   ([#1032](https://github.com/Shopify/sarama/issues/1032)).
- - Fix consuming compacted topics with the new protocol format
-   ([#1005](https://github.com/Shopify/sarama/issues/1005)).
- - Fix consuming topics with a mix of protocol formats
-   ([#1021](https://github.com/Shopify/sarama/issues/1021)).
- - Fix consuming when the broker includes multiple batches in a single response
-   ([#1022](https://github.com/Shopify/sarama/issues/1022)).
- - Fix detection of `PartialTrailingMessage` when the partial message was
-   truncated before the magic value indicating its version
-   ([#1030](https://github.com/Shopify/sarama/pull/1030)).
- - Fix expectation-checking in the mock of `SyncProducer.SendMessages`
-   ([#1035](https://github.com/Shopify/sarama/pull/1035)).
-
-#### Version 1.15.0 (2017-12-08)
-
-New Features:
- - Claim official support for Kafka 1.0, though it did already work
-   ([#984](https://github.com/Shopify/sarama/pull/984)).
- - Helper methods for Kafka version numbers to/from strings
-   ([#989](https://github.com/Shopify/sarama/pull/989)).
- - Implement CreatePartitions request/response
-   ([#985](https://github.com/Shopify/sarama/pull/985)).
-
-Improvements:
- - Add error codes 45-60
-   ([#986](https://github.com/Shopify/sarama/issues/986)).
-
-Bug Fixes:
- - Fix slow consuming for certain Kafka 0.11/1.0 configurations
-   ([#982](https://github.com/Shopify/sarama/pull/982)).
- - Correctly determine when a FetchResponse contains the new message format
-   ([#990](https://github.com/Shopify/sarama/pull/990)).
- - Fix producing with multiple headers
-   ([#996](https://github.com/Shopify/sarama/pull/996)).
- - Fix handling of truncated record batches
-   ([#998](https://github.com/Shopify/sarama/pull/998)).
- - Fix leaking metrics when closing brokers
-   ([#991](https://github.com/Shopify/sarama/pull/991)).
-
-#### Version 1.14.0 (2017-11-13)
-
-New Features:
- - Add support for the new Kafka 0.11 record-batch format, including the wire
-   protocol and the necessary behavioural changes in the producer and consumer.
-   Transactions and idempotency are not yet supported, but producing and
-   consuming should work with all the existing bells and whistles (batching,
-   compression, etc) as well as the new custom headers. Thanks to Vlad Hanciuta
-   of Arista Networks for this work. Part of
-   ([#901](https://github.com/Shopify/sarama/issues/901)).
-
-Bug Fixes:
- - Fix encoding of ProduceResponse versions in test
-   ([#970](https://github.com/Shopify/sarama/pull/970)).
- - Return partial replicas list when we have it
-   ([#975](https://github.com/Shopify/sarama/pull/975)).
-
-#### Version 1.13.0 (2017-10-04)
-
-New Features:
- - Support for FetchRequest version 3
-   ([#905](https://github.com/Shopify/sarama/pull/905)).
- - Permit setting version on mock FetchResponses
-   ([#939](https://github.com/Shopify/sarama/pull/939)).
- - Add a configuration option to support storing only minimal metadata for
-   extremely large clusters
-   ([#937](https://github.com/Shopify/sarama/pull/937)).
- - Add `PartitionOffsetManager.ResetOffset` for backtracking tracked offsets
-   ([#932](https://github.com/Shopify/sarama/pull/932)).
-
-Improvements:
- - Provide the block-level timestamp when consuming compressed messages
-   ([#885](https://github.com/Shopify/sarama/issues/885)).
- - `Client.Replicas` and `Client.InSyncReplicas` now respect the order returned
-   by the broker, which can be meaningful
-   ([#930](https://github.com/Shopify/sarama/pull/930)).
- - Use a `Ticker` to reduce consumer timer overhead at the cost of higher
-   variance in the actual timeout
-   ([#933](https://github.com/Shopify/sarama/pull/933)).
-
-Bug Fixes:
- - Gracefully handle messages with negative timestamps
-   ([#907](https://github.com/Shopify/sarama/pull/907)).
- - Raise a proper error when encountering an unknown message version
-   ([#940](https://github.com/Shopify/sarama/pull/940)).
-
-#### Version 1.12.0 (2017-05-08)
-
-New Features:
- - Added support for the `ApiVersions` request and response pair, and Kafka
-   version 0.10.2 ([#867](https://github.com/Shopify/sarama/pull/867)). Note
-   that you still need to specify the Kafka version in the Sarama configuration
-   for the time being.
- - Added a `Brokers` method to the Client which returns the complete set of
-   active brokers ([#813](https://github.com/Shopify/sarama/pull/813)).
- - Added an `InSyncReplicas` method to the Client which returns the set of all
-   in-sync broker IDs for the given partition, now that the Kafka versions for
-   which this was misleading are no longer in our supported set
-   ([#872](https://github.com/Shopify/sarama/pull/872)).
- - Added a `NewCustomHashPartitioner` method which allows constructing a hash
-   partitioner with a custom hash method in case the default (FNV-1a) is not
-   suitable
-   ([#837](https://github.com/Shopify/sarama/pull/837),
-    [#841](https://github.com/Shopify/sarama/pull/841)).
-
-Improvements:
- - Recognize more Kafka error codes
-   ([#859](https://github.com/Shopify/sarama/pull/859)).
-
-Bug Fixes:
- - Fix an issue where decoding a malformed FetchRequest would not return the
-   correct error ([#818](https://github.com/Shopify/sarama/pull/818)).
- - Respect ordering of group protocols in JoinGroupRequests. This fix is
-   transparent if you're using the `AddGroupProtocol` or
-   `AddGroupProtocolMetadata` helpers; otherwise you will need to switch from
-   the `GroupProtocols` field (now deprecated) to use `OrderedGroupProtocols`
-   ([#812](https://github.com/Shopify/sarama/issues/812)).
- - Fix an alignment-related issue with atomics on 32-bit architectures
-   ([#859](https://github.com/Shopify/sarama/pull/859)).
-
-#### Version 1.11.0 (2016-12-20)
-
-_Important:_ As of Sarama 1.11 it is necessary to set the config value of
-`Producer.Return.Successes` to true in order to use the SyncProducer. Previous
-versions would silently override this value when instantiating a SyncProducer
-which led to unexpected values and data races.
-
-New Features:
- - Metrics! Thanks to Sébastien Launay for all his work on this feature
-   ([#701](https://github.com/Shopify/sarama/pull/701),
-    [#746](https://github.com/Shopify/sarama/pull/746),
-    [#766](https://github.com/Shopify/sarama/pull/766)).
- - Add support for LZ4 compression
-   ([#786](https://github.com/Shopify/sarama/pull/786)).
- - Add support for ListOffsetRequest v1 and Kafka 0.10.1
-   ([#775](https://github.com/Shopify/sarama/pull/775)).
- - Added a `HighWaterMarks` method to the Consumer which aggregates the
-   `HighWaterMarkOffset` values of its child topic/partitions
-   ([#769](https://github.com/Shopify/sarama/pull/769)).
-
-Bug Fixes:
- - Fixed producing when using timestamps, compression and Kafka 0.10
-   ([#759](https://github.com/Shopify/sarama/pull/759)).
- - Added missing decoder methods to DescribeGroups response
-   ([#756](https://github.com/Shopify/sarama/pull/756)).
- - Fix producer shutdown when `Return.Errors` is disabled
-   ([#787](https://github.com/Shopify/sarama/pull/787)).
- - Don't mutate configuration in SyncProducer
-   ([#790](https://github.com/Shopify/sarama/pull/790)).
- - Fix crash on SASL initialization failure
-   ([#795](https://github.com/Shopify/sarama/pull/795)).
-
-#### Version 1.10.1 (2016-08-30)
-
-Bug Fixes:
- - Fix the documentation for `HashPartitioner` which was incorrect
-   ([#717](https://github.com/Shopify/sarama/pull/717)).
- - Permit client creation even when it is limited by ACLs
-   ([#722](https://github.com/Shopify/sarama/pull/722)).
- - Several fixes to the consumer timer optimization code, regressions introduced
-   in v1.10.0. Go's timers are finicky
-   ([#730](https://github.com/Shopify/sarama/pull/730),
-    [#733](https://github.com/Shopify/sarama/pull/733),
-    [#734](https://github.com/Shopify/sarama/pull/734)).
- - Handle consuming compressed relative offsets with Kafka 0.10
-   ([#735](https://github.com/Shopify/sarama/pull/735)).
-
-#### Version 1.10.0 (2016-08-02)
-
-_Important:_ As of Sarama 1.10 it is necessary to tell Sarama the version of
-Kafka you are running against (via the `config.Version` value) in order to use
-features that may not be compatible with old Kafka versions. If you don't
-specify this value it will default to 0.8.2 (the minimum supported), and trying
-to use more recent features (like the offset manager) will fail with an error.
-
-_Also:_ The offset-manager's behaviour has been changed to match the upstream
-java consumer (see [#705](https://github.com/Shopify/sarama/pull/705) and
-[#713](https://github.com/Shopify/sarama/pull/713)). If you use the
-offset-manager, please ensure that you are committing one *greater* than the
-last consumed message offset or else you may end up consuming duplicate
-messages.
-
-New Features:
- - Support for Kafka 0.10
-   ([#672](https://github.com/Shopify/sarama/pull/672),
-    [#678](https://github.com/Shopify/sarama/pull/678),
-    [#681](https://github.com/Shopify/sarama/pull/681), and others).
- - Support for configuring the target Kafka version
-   ([#676](https://github.com/Shopify/sarama/pull/676)).
- - Batch producing support in the SyncProducer
-   ([#677](https://github.com/Shopify/sarama/pull/677)).
- - Extend producer mock to allow setting expectations on message contents
-   ([#667](https://github.com/Shopify/sarama/pull/667)).
-
-Improvements:
- - Support `nil` compressed messages for deleting in compacted topics
-   ([#634](https://github.com/Shopify/sarama/pull/634)).
- - Pre-allocate decoding errors, greatly reducing heap usage and GC time against
-   misbehaving brokers ([#690](https://github.com/Shopify/sarama/pull/690)).
- - Re-use consumer expiry timers, removing one allocation per consumed message
-   ([#707](https://github.com/Shopify/sarama/pull/707)).
-
-Bug Fixes:
- - Actually default the client ID to "sarama" like we say we do
-   ([#664](https://github.com/Shopify/sarama/pull/664)).
- - Fix a rare issue where `Client.Leader` could return the wrong error
-   ([#685](https://github.com/Shopify/sarama/pull/685)).
- - Fix a possible tight loop in the consumer
-   ([#693](https://github.com/Shopify/sarama/pull/693)).
- - Match upstream's offset-tracking behaviour
-   ([#705](https://github.com/Shopify/sarama/pull/705)).
- - Report UnknownTopicOrPartition errors from the offset manager
-   ([#706](https://github.com/Shopify/sarama/pull/706)).
- - Fix possible negative partition value from the HashPartitioner
-   ([#709](https://github.com/Shopify/sarama/pull/709)).
-
-#### Version 1.9.0 (2016-05-16)
-
-New Features:
- - Add support for custom offset manager retention durations
-   ([#602](https://github.com/Shopify/sarama/pull/602)).
- - Publish low-level mocks to enable testing of third-party producer/consumer
-   implementations ([#570](https://github.com/Shopify/sarama/pull/570)).
- - Declare support for Golang 1.6
-   ([#611](https://github.com/Shopify/sarama/pull/611)).
- - Support for SASL plain-text auth
-   ([#648](https://github.com/Shopify/sarama/pull/648)).
-
-Improvements:
- - Simplified broker locking scheme slightly
-   ([#604](https://github.com/Shopify/sarama/pull/604)).
- - Documentation cleanup
-   ([#605](https://github.com/Shopify/sarama/pull/605),
-    [#621](https://github.com/Shopify/sarama/pull/621),
-    [#654](https://github.com/Shopify/sarama/pull/654)).
-
-Bug Fixes:
- - Fix race condition shutting down the OffsetManager
-   ([#658](https://github.com/Shopify/sarama/pull/658)).
-
-#### Version 1.8.0 (2016-02-01)
-
-New Features:
- - Full support for Kafka 0.9:
-   - All protocol messages and fields
-   ([#586](https://github.com/Shopify/sarama/pull/586),
-   [#588](https://github.com/Shopify/sarama/pull/588),
-   [#590](https://github.com/Shopify/sarama/pull/590)).
-   - Verified that TLS support works
-   ([#581](https://github.com/Shopify/sarama/pull/581)).
-   - Fixed the OffsetManager compatibility
-   ([#585](https://github.com/Shopify/sarama/pull/585)).
-
-Improvements:
- - Optimize for fewer system calls when reading from the network
-   ([#584](https://github.com/Shopify/sarama/pull/584)).
- - Automatically retry `InvalidMessage` errors to match upstream behaviour
-   ([#589](https://github.com/Shopify/sarama/pull/589)).
-
-#### Version 1.7.0 (2015-12-11)
-
-New Features:
- - Preliminary support for Kafka 0.9
-   ([#572](https://github.com/Shopify/sarama/pull/572)). This comes with several
-   caveats:
-   - Protocol-layer support is mostly in place
-     ([#577](https://github.com/Shopify/sarama/pull/577)), however Kafka 0.9
-     renamed some messages and fields, which we did not in order to preserve API
-     compatibility.
-   - The producer and consumer work against 0.9, but the offset manager does
-     not ([#573](https://github.com/Shopify/sarama/pull/573)).
-   - TLS support may or may not work
-     ([#581](https://github.com/Shopify/sarama/pull/581)).
-
-Improvements:
- - Don't wait for request timeouts on dead brokers, greatly speeding recovery
-   when the TCP connection is left hanging
-   ([#548](https://github.com/Shopify/sarama/pull/548)).
- - Refactored part of the producer. The new version provides a much more elegant
-   solution to [#449](https://github.com/Shopify/sarama/pull/449). It is also
-   slightly more efficient, and much more precise in calculating batch sizes
-   when compression is used
-   ([#549](https://github.com/Shopify/sarama/pull/549),
-   [#550](https://github.com/Shopify/sarama/pull/550),
-   [#551](https://github.com/Shopify/sarama/pull/551)).
-
-Bug Fixes:
- - Fix race condition in consumer test mock
-   ([#553](https://github.com/Shopify/sarama/pull/553)).
-
-#### Version 1.6.1 (2015-09-25)
-
-Bug Fixes:
- - Fix panic that could occur if a user-supplied message value failed to encode
-   ([#449](https://github.com/Shopify/sarama/pull/449)).
-
-#### Version 1.6.0 (2015-09-04)
-
-New Features:
- - Implementation of a consumer offset manager using the APIs introduced in
-   Kafka 0.8.2. The API is designed mainly for integration into a future
-   high-level consumer, not for direct use, although it is *possible* to use it
-   directly.
-   ([#461](https://github.com/Shopify/sarama/pull/461)).
-
-Improvements:
- - CRC32 calculation is much faster on machines with SSE4.2 instructions,
-   removing a major hotspot from most profiles
-   ([#255](https://github.com/Shopify/sarama/pull/255)).
-
-Bug Fixes:
- - Make protocol decoding more robust against some malformed packets generated
-   by go-fuzz ([#523](https://github.com/Shopify/sarama/pull/523),
-   [#525](https://github.com/Shopify/sarama/pull/525)) or found in other ways
-   ([#528](https://github.com/Shopify/sarama/pull/528)).
- - Fix a potential race condition panic in the consumer on shutdown
-   ([#529](https://github.com/Shopify/sarama/pull/529)).
-
-#### Version 1.5.0 (2015-08-17)
-
-New Features:
- - TLS-encrypted network connections are now supported. This feature is subject
-   to change when Kafka releases built-in TLS support, but for now this is
-   enough to work with TLS-terminating proxies
-   ([#154](https://github.com/Shopify/sarama/pull/154)).
-
-Improvements:
- - The consumer will not block if a single partition is not drained by the user;
-   all other partitions will continue to consume normally
-   ([#485](https://github.com/Shopify/sarama/pull/485)).
- - Formatting of error strings has been much improved
-   ([#495](https://github.com/Shopify/sarama/pull/495)).
- - Internal refactoring of the producer for code cleanliness and to enable
-   future work ([#300](https://github.com/Shopify/sarama/pull/300)).
-
-Bug Fixes:
- - Fix a potential deadlock in the consumer on shutdown
-   ([#475](https://github.com/Shopify/sarama/pull/475)).
-
-#### Version 1.4.3 (2015-07-21)
-
-Bug Fixes:
- - Don't include the partitioner in the producer's "fetch partitions"
-   circuit-breaker ([#466](https://github.com/Shopify/sarama/pull/466)).
- - Don't retry messages until the broker is closed when abandoning a broker in
-   the producer ([#468](https://github.com/Shopify/sarama/pull/468)).
- - Update the import path for snappy-go, it has moved again and the API has
-   changed slightly ([#486](https://github.com/Shopify/sarama/pull/486)).
-
-#### Version 1.4.2 (2015-05-27)
-
-Bug Fixes:
- - Update the import path for snappy-go, it has moved from google code to github
-   ([#456](https://github.com/Shopify/sarama/pull/456)).
-
-#### Version 1.4.1 (2015-05-25)
-
-Improvements:
- - Optimizations when decoding snappy messages, thanks to John Potocny
-   ([#446](https://github.com/Shopify/sarama/pull/446)).
-
-Bug Fixes:
- - Fix hypothetical race conditions on producer shutdown
-   ([#450](https://github.com/Shopify/sarama/pull/450),
-   [#451](https://github.com/Shopify/sarama/pull/451)).
-
-#### Version 1.4.0 (2015-05-01)
-
-New Features:
- - The consumer now implements `Topics()` and `Partitions()` methods to enable
-   users to dynamically choose what topics/partitions to consume without
-   instantiating a full client
-   ([#431](https://github.com/Shopify/sarama/pull/431)).
- - The partition-consumer now exposes the high water mark offset value returned
-   by the broker via the `HighWaterMarkOffset()` method ([#339](https://github.com/Shopify/sarama/pull/339)).
- - Added a `kafka-console-consumer` tool capable of handling multiple
-   partitions, and deprecated the now-obsolete `kafka-console-partitionConsumer`
-   ([#439](https://github.com/Shopify/sarama/pull/439),
-   [#442](https://github.com/Shopify/sarama/pull/442)).
-
-Improvements:
- - The producer's logging during retry scenarios is more consistent, more
-   useful, and slightly less verbose
-   ([#429](https://github.com/Shopify/sarama/pull/429)).
- - The client now shuffles its initial list of seed brokers in order to prevent
-   thundering herd on the first broker in the list
-   ([#441](https://github.com/Shopify/sarama/pull/441)).
-
-Bug Fixes:
- - The producer now correctly manages its state if retries occur when it is
-   shutting down, fixing several instances of confusing behaviour and at least
-   one potential deadlock ([#419](https://github.com/Shopify/sarama/pull/419)).
- - The consumer now handles messages for different partitions asynchronously,
-   making it much more resilient to specific user code ordering
-   ([#325](https://github.com/Shopify/sarama/pull/325)).
-
-#### Version 1.3.0 (2015-04-16)
-
-New Features:
- - The client now tracks consumer group coordinators using
-   ConsumerMetadataRequests similar to how it tracks partition leadership using
-   regular MetadataRequests ([#411](https://github.com/Shopify/sarama/pull/411)).
-   This adds two methods to the client API:
-   - `Coordinator(consumerGroup string) (*Broker, error)`
-   - `RefreshCoordinator(consumerGroup string) error`
-
-Improvements:
- - ConsumerMetadataResponses now automatically create a Broker object out of the
-   ID/address/port combination for the Coordinator; accessing the fields
-   individually has been deprecated
-   ([#413](https://github.com/Shopify/sarama/pull/413)).
- - Much improved handling of `OffsetOutOfRange` errors in the consumer.
-   Consumers will fail to start if the provided offset is out of range
-   ([#418](https://github.com/Shopify/sarama/pull/418))
-   and they will automatically shut down if the offset falls out of range
-   ([#424](https://github.com/Shopify/sarama/pull/424)).
- - Small performance improvement in encoding and decoding protocol messages
-   ([#427](https://github.com/Shopify/sarama/pull/427)).
-
-Bug Fixes:
- - Fix a rare race condition in the client's background metadata refresher if
-   it happens to be activated while the client is being closed
-   ([#422](https://github.com/Shopify/sarama/pull/422)).
-
-#### Version 1.2.0 (2015-04-07)
-
-Improvements:
- - The producer's behaviour when `Flush.Frequency` is set is now more intuitive
-   ([#389](https://github.com/Shopify/sarama/pull/389)).
- - The producer is now somewhat more memory-efficient during and after retrying
-   messages due to an improved queue implementation
-   ([#396](https://github.com/Shopify/sarama/pull/396)).
- - The consumer produces much more useful logging output when leadership
-   changes ([#385](https://github.com/Shopify/sarama/pull/385)).
- - The client's `GetOffset` method will now automatically refresh metadata and
-   retry once in the event of stale information or similar
-   ([#394](https://github.com/Shopify/sarama/pull/394)).
- - Broker connections now have support for using TCP keepalives
-   ([#407](https://github.com/Shopify/sarama/issues/407)).
-
-Bug Fixes:
- - The OffsetCommitRequest message now correctly implements all three possible
-   API versions ([#390](https://github.com/Shopify/sarama/pull/390),
-   [#400](https://github.com/Shopify/sarama/pull/400)).
-
-#### Version 1.1.0 (2015-03-20)
-
-Improvements:
- - Wrap the producer's partitioner call in a circuit-breaker so that repeatedly
-   broken topics don't choke throughput
-   ([#373](https://github.com/Shopify/sarama/pull/373)).
-
-Bug Fixes:
- - Fix the producer's internal reference counting in certain unusual scenarios
-   ([#367](https://github.com/Shopify/sarama/pull/367)).
- - Fix the consumer's internal reference counting in certain unusual scenarios
-   ([#369](https://github.com/Shopify/sarama/pull/369)).
- - Fix a condition where the producer's internal control messages could have
-   gotten stuck ([#368](https://github.com/Shopify/sarama/pull/368)).
- - Fix an issue where invalid partition lists would be cached when asking for
-   metadata for a non-existant topic ([#372](https://github.com/Shopify/sarama/pull/372)).
-
-
-#### Version 1.0.0 (2015-03-17)
-
-Version 1.0.0 is the first tagged version, and is almost a complete rewrite. The primary differences with previous untagged versions are:
-
-- The producer has been rewritten; there is now a `SyncProducer` with a blocking API, and an `AsyncProducer` that is non-blocking.
-- The consumer has been rewritten to only open one connection per broker instead of one connection per partition.
-- The main types of Sarama are now interfaces to make depedency injection easy; mock implementations for `Consumer`, `SyncProducer` and `AsyncProducer` are provided in the `github.com/Shopify/sarama/mocks` package.
-- For most uses cases, it is no longer necessary to open a `Client`; this will be done for you.
-- All the configuration values have been unified in the `Config` struct.
-- Much improved test suite.

vendor/github.com/Shopify/sarama/LICENSE

@@ -1,20 +0,0 @@
-Copyright (c) 2013 Shopify
-
-Permission is hereby granted, free of charge, to any person obtaining
-a copy of this software and associated documentation files (the
-"Software"), to deal in the Software without restriction, including
-without limitation the rights to use, copy, modify, merge, publish,
-distribute, sublicense, and/or sell copies of the Software, and to
-permit persons to whom the Software is furnished to do so, subject to
-the following conditions:
-
-The above copyright notice and this permission notice shall be
-included in all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
-NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
-LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
-OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
-WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

vendor/github.com/Shopify/sarama/Makefile

@@ -1,27 +0,0 @@
-default: fmt get update test lint
-
-GO       := GO111MODULE=on GOPRIVATE=github.com/linkedin GOSUMDB=off go
-GOBUILD  := CGO_ENABLED=0 $(GO) build $(BUILD_FLAG)
-GOTEST   := $(GO) test -gcflags='-l' -p 3 -v -race -timeout 6m -coverprofile=profile.out -covermode=atomic
-
-FILES    := $(shell find . -name '*.go' -type f -not -name '*.pb.go' -not -name '*_generated.go' -not -name '*_test.go')
-TESTS    := $(shell find . -name '*.go' -type f -not -name '*.pb.go' -not -name '*_generated.go' -name '*_test.go')
-
-get:
-	$(GO) get ./...
-	$(GO) mod verify
-	$(GO) mod tidy
-
-update:
-	$(GO) get -u -v all
-	$(GO) mod verify
-	$(GO) mod tidy
-
-fmt:
-	gofmt -s -l -w $(FILES) $(TESTS)
-
-lint:
-	golangci-lint run
-
-test:
-	$(GOTEST) ./...

vendor/github.com/Shopify/sarama/README.md

@@ -1,36 +0,0 @@
-# sarama
-
-[![GoDoc](https://godoc.org/github.com/Shopify/sarama?status.svg)](https://godoc.org/github.com/Shopify/sarama)
-[![Build Status](https://travis-ci.org/Shopify/sarama.svg?branch=master)](https://travis-ci.org/Shopify/sarama)
-[![Coverage](https://codecov.io/gh/Shopify/sarama/branch/master/graph/badge.svg)](https://codecov.io/gh/Shopify/sarama)
-
-Sarama is an MIT-licensed Go client library for [Apache Kafka](https://kafka.apache.org/) version 0.8 (and later).
-
-## Getting started
-
-- API documentation and examples are available via [godoc](https://godoc.org/github.com/Shopify/sarama).
-- Mocks for testing are available in the [mocks](./mocks) subpackage.
-- The [examples](./examples) directory contains more elaborate example applications.
-- The [tools](./tools) directory contains command line tools that can be useful for testing, diagnostics, and instrumentation.
-
-You might also want to look at the [Frequently Asked Questions](https://github.com/Shopify/sarama/wiki/Frequently-Asked-Questions).
-
-## Compatibility and API stability
-
-Sarama provides a "2 releases + 2 months" compatibility guarantee: we support
-the two latest stable releases of Kafka and Go, and we provide a two month
-grace period for older releases. This means we currently officially support
-Go 1.12 through 1.14, and Kafka 2.1 through 2.4, although older releases are
-still likely to work.
-
-Sarama follows semantic versioning and provides API stability via the gopkg.in service.
-You can import a version with a guaranteed stable API via http://gopkg.in/Shopify/sarama.v1.
-A changelog is available [here](CHANGELOG.md).
-
-## Contributing
-
-- Get started by checking our [contribution guidelines](https://github.com/Shopify/sarama/blob/master/.github/CONTRIBUTING.md).
-- Read the [Sarama wiki](https://github.com/Shopify/sarama/wiki) for more technical and design details.
-- The [Kafka Protocol Specification](https://cwiki.apache.org/confluence/display/KAFKA/A+Guide+To+The+Kafka+Protocol) contains a wealth of useful information.
-- For more general issues, there is [a google group](https://groups.google.com/forum/#!forum/kafka-clients) for Kafka client developers.
-- If you have any questions, just ask!

vendor/github.com/Shopify/sarama/Vagrantfile

@@ -1,14 +0,0 @@
-# We have 5 * 192MB ZK processes and 5 * 320MB Kafka processes => 2560MB
-MEMORY = 3072
-
-Vagrant.configure("2") do |config|
-  config.vm.box = "ubuntu/bionic64"
-
-  config.vm.provision :shell, path: "vagrant/provision.sh"
-
-  config.vm.network "private_network", ip: "192.168.100.67"
-
-  config.vm.provider "virtualbox" do |v|
-    v.memory = MEMORY
-  end
-end

vendor/github.com/Shopify/sarama/acl_bindings.go

@@ -1,138 +0,0 @@
-package sarama
-
-//Resource holds information about acl resource type
-type Resource struct {
-	ResourceType        AclResourceType
-	ResourceName        string
-	ResourcePatternType AclResourcePatternType
-}
-
-func (r *Resource) encode(pe packetEncoder, version int16) error {
-	pe.putInt8(int8(r.ResourceType))
-
-	if err := pe.putString(r.ResourceName); err != nil {
-		return err
-	}
-
-	if version == 1 {
-		if r.ResourcePatternType == AclPatternUnknown {
-			Logger.Print("Cannot encode an unknown resource pattern type, using Literal instead")
-			r.ResourcePatternType = AclPatternLiteral
-		}
-		pe.putInt8(int8(r.ResourcePatternType))
-	}
-
-	return nil
-}
-
-func (r *Resource) decode(pd packetDecoder, version int16) (err error) {
-	resourceType, err := pd.getInt8()
-	if err != nil {
-		return err
-	}
-	r.ResourceType = AclResourceType(resourceType)
-
-	if r.ResourceName, err = pd.getString(); err != nil {
-		return err
-	}
-	if version == 1 {
-		pattern, err := pd.getInt8()
-		if err != nil {
-			return err
-		}
-		r.ResourcePatternType = AclResourcePatternType(pattern)
-	}
-
-	return nil
-}
-
-//Acl holds information about acl type
-type Acl struct {
-	Principal      string
-	Host           string
-	Operation      AclOperation
-	PermissionType AclPermissionType
-}
-
-func (a *Acl) encode(pe packetEncoder) error {
-	if err := pe.putString(a.Principal); err != nil {
-		return err
-	}
-
-	if err := pe.putString(a.Host); err != nil {
-		return err
-	}
-
-	pe.putInt8(int8(a.Operation))
-	pe.putInt8(int8(a.PermissionType))
-
-	return nil
-}
-
-func (a *Acl) decode(pd packetDecoder, version int16) (err error) {
-	if a.Principal, err = pd.getString(); err != nil {
-		return err
-	}
-
-	if a.Host, err = pd.getString(); err != nil {
-		return err
-	}
-
-	operation, err := pd.getInt8()
-	if err != nil {
-		return err
-	}
-	a.Operation = AclOperation(operation)
-
-	permissionType, err := pd.getInt8()
-	if err != nil {
-		return err
-	}
-	a.PermissionType = AclPermissionType(permissionType)
-
-	return nil
-}
-
-//ResourceAcls is an acl resource type
-type ResourceAcls struct {
-	Resource
-	Acls []*Acl
-}
-
-func (r *ResourceAcls) encode(pe packetEncoder, version int16) error {
-	if err := r.Resource.encode(pe, version); err != nil {
-		return err
-	}
-
-	if err := pe.putArrayLength(len(r.Acls)); err != nil {
-		return err
-	}
-	for _, acl := range r.Acls {
-		if err := acl.encode(pe); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-func (r *ResourceAcls) decode(pd packetDecoder, version int16) error {
-	if err := r.Resource.decode(pd, version); err != nil {
-		return err
-	}
-
-	n, err := pd.getArrayLength()
-	if err != nil {
-		return err
-	}
-
-	r.Acls = make([]*Acl, n)
-	for i := 0; i < n; i++ {
-		r.Acls[i] = new(Acl)
-		if err := r.Acls[i].decode(pd, version); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}

vendor/github.com/Shopify/sarama/acl_create_request.go

@@ -1,89 +0,0 @@
-package sarama
-
-//CreateAclsRequest is an acl creation request
-type CreateAclsRequest struct {
-	Version      int16
-	AclCreations []*AclCreation
-}
-
-func (c *CreateAclsRequest) encode(pe packetEncoder) error {
-	if err := pe.putArrayLength(len(c.AclCreations)); err != nil {
-		return err
-	}
-
-	for _, aclCreation := range c.AclCreations {
-		if err := aclCreation.encode(pe, c.Version); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-func (c *CreateAclsRequest) decode(pd packetDecoder, version int16) (err error) {
-	c.Version = version
-	n, err := pd.getArrayLength()
-	if err != nil {
-		return err
-	}
-
-	c.AclCreations = make([]*AclCreation, n)
-
-	for i := 0; i < n; i++ {
-		c.AclCreations[i] = new(AclCreation)
-		if err := c.AclCreations[i].decode(pd, version); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-func (c *CreateAclsRequest) key() int16 {
-	return 30
-}
-
-func (c *CreateAclsRequest) version() int16 {
-	return c.Version
-}
-
-func (c *CreateAclsRequest) headerVersion() int16 {
-	return 1
-}
-
-func (c *CreateAclsRequest) requiredVersion() KafkaVersion {
-	switch c.Version {
-	case 1:
-		return V2_0_0_0
-	default:
-		return V0_11_0_0
-	}
-}
-
-//AclCreation is a wrapper around Resource and Acl type
-type AclCreation struct {
-	Resource
-	Acl
-}
-
-func (a *AclCreation) encode(pe packetEncoder, version int16) error {
-	if err := a.Resource.encode(pe, version); err != nil {
-		return err
-	}
-	if err := a.Acl.encode(pe); err != nil {
-		return err
-	}
-
-	return nil
-}
-
-func (a *AclCreation) decode(pd packetDecoder, version int16) (err error) {
-	if err := a.Resource.decode(pd, version); err != nil {
-		return err
-	}
-	if err := a.Acl.decode(pd, version); err != nil {
-		return err
-	}
-
-	return nil
-}

vendor/github.com/Shopify/sarama/acl_create_response.go

@@ -1,94 +0,0 @@
-package sarama
-
-import "time"
-
-//CreateAclsResponse is a an acl response creation type
-type CreateAclsResponse struct {
-	ThrottleTime         time.Duration
-	AclCreationResponses []*AclCreationResponse
-}
-
-func (c *CreateAclsResponse) encode(pe packetEncoder) error {
-	pe.putInt32(int32(c.ThrottleTime / time.Millisecond))
-
-	if err := pe.putArrayLength(len(c.AclCreationResponses)); err != nil {
-		return err
-	}
-
-	for _, aclCreationResponse := range c.AclCreationResponses {
-		if err := aclCreationResponse.encode(pe); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-func (c *CreateAclsResponse) decode(pd packetDecoder, version int16) (err error) {
-	throttleTime, err := pd.getInt32()
-	if err != nil {
-		return err
-	}
-	c.ThrottleTime = time.Duration(throttleTime) * time.Millisecond
-
-	n, err := pd.getArrayLength()
-	if err != nil {
-		return err
-	}
-
-	c.AclCreationResponses = make([]*AclCreationResponse, n)
-	for i := 0; i < n; i++ {
-		c.AclCreationResponses[i] = new(AclCreationResponse)
-		if err := c.AclCreationResponses[i].decode(pd, version); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-func (c *CreateAclsResponse) key() int16 {
-	return 30
-}
-
-func (c *CreateAclsResponse) version() int16 {
-	return 0
-}
-
-func (c *CreateAclsResponse) headerVersion() int16 {
-	return 0
-}
-
-func (c *CreateAclsResponse) requiredVersion() KafkaVersion {
-	return V0_11_0_0
-}
-
-//AclCreationResponse is an acl creation response type
-type AclCreationResponse struct {
-	Err    KError
-	ErrMsg *string
-}
-
-func (a *AclCreationResponse) encode(pe packetEncoder) error {
-	pe.putInt16(int16(a.Err))
-
-	if err := pe.putNullableString(a.ErrMsg); err != nil {
-		return err
-	}
-
-	return nil
-}
-
-func (a *AclCreationResponse) decode(pd packetDecoder, version int16) (err error) {
-	kerr, err := pd.getInt16()
-	if err != nil {
-		return err
-	}
-	a.Err = KError(kerr)
-
-	if a.ErrMsg, err = pd.getNullableString(); err != nil {
-		return err
-	}
-
-	return nil
-}

vendor/github.com/Shopify/sarama/acl_delete_request.go

@@ -1,62 +0,0 @@
-package sarama
-
-//DeleteAclsRequest is a delete acl request
-type DeleteAclsRequest struct {
-	Version int
-	Filters []*AclFilter
-}
-
-func (d *DeleteAclsRequest) encode(pe packetEncoder) error {
-	if err := pe.putArrayLength(len(d.Filters)); err != nil {
-		return err
-	}
-
-	for _, filter := range d.Filters {
-		filter.Version = d.Version
-		if err := filter.encode(pe); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-func (d *DeleteAclsRequest) decode(pd packetDecoder, version int16) (err error) {
-	d.Version = int(version)
-	n, err := pd.getArrayLength()
-	if err != nil {
-		return err
-	}
-
-	d.Filters = make([]*AclFilter, n)
-	for i := 0; i < n; i++ {
-		d.Filters[i] = new(AclFilter)
-		d.Filters[i].Version = int(version)
-		if err := d.Filters[i].decode(pd, version); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-func (d *DeleteAclsRequest) key() int16 {
-	return 31
-}
-
-func (d *DeleteAclsRequest) version() int16 {
-	return int16(d.Version)
-}
-
-func (c *DeleteAclsRequest) headerVersion() int16 {
-	return 1
-}
-
-func (d *DeleteAclsRequest) requiredVersion() KafkaVersion {
-	switch d.Version {
-	case 1:
-		return V2_0_0_0
-	default:
-		return V0_11_0_0
-	}
-}

vendor/github.com/Shopify/sarama/acl_delete_response.go

@@ -1,163 +0,0 @@
-package sarama
-
-import "time"
-
-//DeleteAclsResponse is a delete acl response
-type DeleteAclsResponse struct {
-	Version         int16
-	ThrottleTime    time.Duration
-	FilterResponses []*FilterResponse
-}
-
-func (d *DeleteAclsResponse) encode(pe packetEncoder) error {
-	pe.putInt32(int32(d.ThrottleTime / time.Millisecond))
-
-	if err := pe.putArrayLength(len(d.FilterResponses)); err != nil {
-		return err
-	}
-
-	for _, filterResponse := range d.FilterResponses {
-		if err := filterResponse.encode(pe, d.Version); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-func (d *DeleteAclsResponse) decode(pd packetDecoder, version int16) (err error) {
-	throttleTime, err := pd.getInt32()
-	if err != nil {
-		return err
-	}
-	d.ThrottleTime = time.Duration(throttleTime) * time.Millisecond
-
-	n, err := pd.getArrayLength()
-	if err != nil {
-		return err
-	}
-	d.FilterResponses = make([]*FilterResponse, n)
-
-	for i := 0; i < n; i++ {
-		d.FilterResponses[i] = new(FilterResponse)
-		if err := d.FilterResponses[i].decode(pd, version); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-func (d *DeleteAclsResponse) key() int16 {
-	return 31
-}
-
-func (d *DeleteAclsResponse) version() int16 {
-	return d.Version
-}
-
-func (d *DeleteAclsResponse) headerVersion() int16 {
-	return 0
-}
-
-func (d *DeleteAclsResponse) requiredVersion() KafkaVersion {
-	return V0_11_0_0
-}
-
-//FilterResponse is a filter response type
-type FilterResponse struct {
-	Err          KError
-	ErrMsg       *string
-	MatchingAcls []*MatchingAcl
-}
-
-func (f *FilterResponse) encode(pe packetEncoder, version int16) error {
-	pe.putInt16(int16(f.Err))
-	if err := pe.putNullableString(f.ErrMsg); err != nil {
-		return err
-	}
-
-	if err := pe.putArrayLength(len(f.MatchingAcls)); err != nil {
-		return err
-	}
-	for _, matchingAcl := range f.MatchingAcls {
-		if err := matchingAcl.encode(pe, version); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-func (f *FilterResponse) decode(pd packetDecoder, version int16) (err error) {
-	kerr, err := pd.getInt16()
-	if err != nil {
-		return err
-	}
-	f.Err = KError(kerr)
-
-	if f.ErrMsg, err = pd.getNullableString(); err != nil {
-		return err
-	}
-
-	n, err := pd.getArrayLength()
-	if err != nil {
-		return err
-	}
-	f.MatchingAcls = make([]*MatchingAcl, n)
-	for i := 0; i < n; i++ {
-		f.MatchingAcls[i] = new(MatchingAcl)
-		if err := f.MatchingAcls[i].decode(pd, version); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-//MatchingAcl is a matching acl type
-type MatchingAcl struct {
-	Err    KError
-	ErrMsg *string
-	Resource
-	Acl
-}
-
-func (m *MatchingAcl) encode(pe packetEncoder, version int16) error {
-	pe.putInt16(int16(m.Err))
-	if err := pe.putNullableString(m.ErrMsg); err != nil {
-		return err
-	}
-
-	if err := m.Resource.encode(pe, version); err != nil {
-		return err
-	}
-
-	if err := m.Acl.encode(pe); err != nil {
-		return err
-	}
-
-	return nil
-}
-
-func (m *MatchingAcl) decode(pd packetDecoder, version int16) (err error) {
-	kerr, err := pd.getInt16()
-	if err != nil {
-		return err
-	}
-	m.Err = KError(kerr)
-
-	if m.ErrMsg, err = pd.getNullableString(); err != nil {
-		return err
-	}
-
-	if err := m.Resource.decode(pd, version); err != nil {
-		return err
-	}
-
-	if err := m.Acl.decode(pd, version); err != nil {
-		return err
-	}
-
-	return nil
-}

vendor/github.com/Shopify/sarama/acl_describe_request.go

@@ -1,39 +0,0 @@
-package sarama
-
-//DescribeAclsRequest is a secribe acl request type
-type DescribeAclsRequest struct {
-	Version int
-	AclFilter
-}
-
-func (d *DescribeAclsRequest) encode(pe packetEncoder) error {
-	d.AclFilter.Version = d.Version
-	return d.AclFilter.encode(pe)
-}
-
-func (d *DescribeAclsRequest) decode(pd packetDecoder, version int16) (err error) {
-	d.Version = int(version)
-	d.AclFilter.Version = int(version)
-	return d.AclFilter.decode(pd, version)
-}
-
-func (d *DescribeAclsRequest) key() int16 {
-	return 29
-}
-
-func (d *DescribeAclsRequest) version() int16 {
-	return int16(d.Version)
-}
-
-func (d *DescribeAclsRequest) headerVersion() int16 {
-	return 1
-}
-
-func (d *DescribeAclsRequest) requiredVersion() KafkaVersion {
-	switch d.Version {
-	case 1:
-		return V2_0_0_0
-	default:
-		return V0_11_0_0
-	}
-}

vendor/github.com/Shopify/sarama/acl_describe_response.go

@@ -1,91 +0,0 @@
-package sarama
-
-import "time"
-
-//DescribeAclsResponse is a describe acl response type
-type DescribeAclsResponse struct {
-	Version      int16
-	ThrottleTime time.Duration
-	Err          KError
-	ErrMsg       *string
-	ResourceAcls []*ResourceAcls
-}
-
-func (d *DescribeAclsResponse) encode(pe packetEncoder) error {
-	pe.putInt32(int32(d.ThrottleTime / time.Millisecond))
-	pe.putInt16(int16(d.Err))
-
-	if err := pe.putNullableString(d.ErrMsg); err != nil {
-		return err
-	}
-
-	if err := pe.putArrayLength(len(d.ResourceAcls)); err != nil {
-		return err
-	}
-
-	for _, resourceAcl := range d.ResourceAcls {
-		if err := resourceAcl.encode(pe, d.Version); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-func (d *DescribeAclsResponse) decode(pd packetDecoder, version int16) (err error) {
-	throttleTime, err := pd.getInt32()
-	if err != nil {
-		return err
-	}
-	d.ThrottleTime = time.Duration(throttleTime) * time.Millisecond
-
-	kerr, err := pd.getInt16()
-	if err != nil {
-		return err
-	}
-	d.Err = KError(kerr)
-
-	errmsg, err := pd.getString()
-	if err != nil {
-		return err
-	}
-	if errmsg != "" {
-		d.ErrMsg = &errmsg
-	}
-
-	n, err := pd.getArrayLength()
-	if err != nil {
-		return err
-	}
-	d.ResourceAcls = make([]*ResourceAcls, n)
-
-	for i := 0; i < n; i++ {
-		d.ResourceAcls[i] = new(ResourceAcls)
-		if err := d.ResourceAcls[i].decode(pd, version); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-func (d *DescribeAclsResponse) key() int16 {
-	return 29
-}
-
-func (d *DescribeAclsResponse) version() int16 {
-	return d.Version
-}
-
-func (d *DescribeAclsResponse) headerVersion() int16 {
-	return 0
-}
-
-func (d *DescribeAclsResponse) requiredVersion() KafkaVersion {
-	switch d.Version {
-	case 1:
-		return V2_0_0_0
-	default:
-		return V0_11_0_0
-	}
-}

vendor/github.com/Shopify/sarama/acl_filter.go

@@ -1,78 +0,0 @@
-package sarama
-
-type AclFilter struct {
-	Version                   int
-	ResourceType              AclResourceType
-	ResourceName              *string
-	ResourcePatternTypeFilter AclResourcePatternType
-	Principal                 *string
-	Host                      *string
-	Operation                 AclOperation
-	PermissionType            AclPermissionType
-}
-
-func (a *AclFilter) encode(pe packetEncoder) error {
-	pe.putInt8(int8(a.ResourceType))
-	if err := pe.putNullableString(a.ResourceName); err != nil {
-		return err
-	}
-
-	if a.Version == 1 {
-		pe.putInt8(int8(a.ResourcePatternTypeFilter))
-	}
-
-	if err := pe.putNullableString(a.Principal); err != nil {
-		return err
-	}
-	if err := pe.putNullableString(a.Host); err != nil {
-		return err
-	}
-	pe.putInt8(int8(a.Operation))
-	pe.putInt8(int8(a.PermissionType))
-
-	return nil
-}
-
-func (a *AclFilter) decode(pd packetDecoder, version int16) (err error) {
-	resourceType, err := pd.getInt8()
-	if err != nil {
-		return err
-	}
-	a.ResourceType = AclResourceType(resourceType)
-
-	if a.ResourceName, err = pd.getNullableString(); err != nil {
-		return err
-	}
-
-	if a.Version == 1 {
-		pattern, err := pd.getInt8()
-
-		if err != nil {
-			return err
-		}
-
-		a.ResourcePatternTypeFilter = AclResourcePatternType(pattern)
-	}
-
-	if a.Principal, err = pd.getNullableString(); err != nil {
-		return err
-	}
-
-	if a.Host, err = pd.getNullableString(); err != nil {
-		return err
-	}
-
-	operation, err := pd.getInt8()
-	if err != nil {
-		return err
-	}
-	a.Operation = AclOperation(operation)
-
-	permissionType, err := pd.getInt8()
-	if err != nil {
-		return err
-	}
-	a.PermissionType = AclPermissionType(permissionType)
-
-	return nil
-}

vendor/github.com/Shopify/sarama/acl_types.go

@@ -1,55 +0,0 @@
-package sarama
-
-type (
-	AclOperation int
-
-	AclPermissionType int
-
-	AclResourceType int
-
-	AclResourcePatternType int
-)
-
-// ref: https://github.com/apache/kafka/blob/trunk/clients/src/main/java/org/apache/kafka/common/acl/AclOperation.java
-const (
-	AclOperationUnknown AclOperation = iota
-	AclOperationAny
-	AclOperationAll
-	AclOperationRead
-	AclOperationWrite
-	AclOperationCreate
-	AclOperationDelete
-	AclOperationAlter
-	AclOperationDescribe
-	AclOperationClusterAction
-	AclOperationDescribeConfigs
-	AclOperationAlterConfigs
-	AclOperationIdempotentWrite
-)
-
-// ref: https://github.com/apache/kafka/blob/trunk/clients/src/main/java/org/apache/kafka/common/acl/AclPermissionType.java
-const (
-	AclPermissionUnknown AclPermissionType = iota
-	AclPermissionAny
-	AclPermissionDeny
-	AclPermissionAllow
-)
-
-// ref: https://github.com/apache/kafka/blob/trunk/clients/src/main/java/org/apache/kafka/common/resource/ResourceType.java
-const (
-	AclResourceUnknown AclResourceType = iota
-	AclResourceAny
-	AclResourceTopic
-	AclResourceGroup
-	AclResourceCluster
-	AclResourceTransactionalID
-)
-
-// ref: https://github.com/apache/kafka/blob/trunk/clients/src/main/java/org/apache/kafka/common/resource/PatternType.java
-const (
-	AclPatternUnknown AclResourcePatternType = iota
-	AclPatternAny
-	AclPatternMatch
-	AclPatternLiteral
-	AclPatternPrefixed
-)

vendor/github.com/Shopify/sarama/add_offsets_to_txn_request.go

@@ -1,57 +0,0 @@
-package sarama
-
-//AddOffsetsToTxnRequest adds offsets to a transaction request
-type AddOffsetsToTxnRequest struct {
-	TransactionalID string
-	ProducerID      int64
-	ProducerEpoch   int16
-	GroupID         string
-}
-
-func (a *AddOffsetsToTxnRequest) encode(pe packetEncoder) error {
-	if err := pe.putString(a.TransactionalID); err != nil {
-		return err
-	}
-
-	pe.putInt64(a.ProducerID)
-
-	pe.putInt16(a.ProducerEpoch)
-
-	if err := pe.putString(a.GroupID); err != nil {
-		return err
-	}
-
-	return nil
-}
-
-func (a *AddOffsetsToTxnRequest) decode(pd packetDecoder, version int16) (err error) {
-	if a.TransactionalID, err = pd.getString(); err != nil {
-		return err
-	}
-	if a.ProducerID, err = pd.getInt64(); err != nil {
-		return err
-	}
-	if a.ProducerEpoch, err = pd.getInt16(); err != nil {
-		return err
-	}
-	if a.GroupID, err = pd.getString(); err != nil {
-		return err
-	}
-	return nil
-}
-
-func (a *AddOffsetsToTxnRequest) key() int16 {
-	return 25
-}
-
-func (a *AddOffsetsToTxnRequest) version() int16 {
-	return 0
-}
-
-func (a *AddOffsetsToTxnRequest) headerVersion() int16 {
-	return 1
-}
-
-func (a *AddOffsetsToTxnRequest) requiredVersion() KafkaVersion {
-	return V0_11_0_0
-}

vendor/github.com/Shopify/sarama/add_offsets_to_txn_response.go

@@ -1,49 +0,0 @@
-package sarama
-
-import (
-	"time"
-)
-
-//AddOffsetsToTxnResponse is a response type for adding offsets to txns
-type AddOffsetsToTxnResponse struct {
-	ThrottleTime time.Duration
-	Err          KError
-}
-
-func (a *AddOffsetsToTxnResponse) encode(pe packetEncoder) error {
-	pe.putInt32(int32(a.ThrottleTime / time.Millisecond))
-	pe.putInt16(int16(a.Err))
-	return nil
-}
-
-func (a *AddOffsetsToTxnResponse) decode(pd packetDecoder, version int16) (err error) {
-	throttleTime, err := pd.getInt32()
-	if err != nil {
-		return err
-	}
-	a.ThrottleTime = time.Duration(throttleTime) * time.Millisecond
-
-	kerr, err := pd.getInt16()
-	if err != nil {
-		return err
-	}
-	a.Err = KError(kerr)
-
-	return nil
-}
-
-func (a *AddOffsetsToTxnResponse) key() int16 {
-	return 25
-}
-
-func (a *AddOffsetsToTxnResponse) version() int16 {
-	return 0
-}
-
-func (a *AddOffsetsToTxnResponse) headerVersion() int16 {
-	return 0
-}
-
-func (a *AddOffsetsToTxnResponse) requiredVersion() KafkaVersion {
-	return V0_11_0_0
-}

vendor/github.com/Shopify/sarama/add_partitions_to_txn_request.go

@@ -1,81 +0,0 @@
-package sarama
-
-//AddPartitionsToTxnRequest is a add paartition request
-type AddPartitionsToTxnRequest struct {
-	TransactionalID string
-	ProducerID      int64
-	ProducerEpoch   int16
-	TopicPartitions map[string][]int32
-}
-
-func (a *AddPartitionsToTxnRequest) encode(pe packetEncoder) error {
-	if err := pe.putString(a.TransactionalID); err != nil {
-		return err
-	}
-	pe.putInt64(a.ProducerID)
-	pe.putInt16(a.ProducerEpoch)
-
-	if err := pe.putArrayLength(len(a.TopicPartitions)); err != nil {
-		return err
-	}
-	for topic, partitions := range a.TopicPartitions {
-		if err := pe.putString(topic); err != nil {
-			return err
-		}
-		if err := pe.putInt32Array(partitions); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-func (a *AddPartitionsToTxnRequest) decode(pd packetDecoder, version int16) (err error) {
-	if a.TransactionalID, err = pd.getString(); err != nil {
-		return err
-	}
-	if a.ProducerID, err = pd.getInt64(); err != nil {
-		return err
-	}
-	if a.ProducerEpoch, err = pd.getInt16(); err != nil {
-		return err
-	}
-
-	n, err := pd.getArrayLength()
-	if err != nil {
-		return err
-	}
-
-	a.TopicPartitions = make(map[string][]int32)
-	for i := 0; i < n; i++ {
-		topic, err := pd.getString()
-		if err != nil {
-			return err
-		}
-
-		partitions, err := pd.getInt32Array()
-		if err != nil {
-			return err
-		}
-
-		a.TopicPartitions[topic] = partitions
-	}
-
-	return nil
-}
-
-func (a *AddPartitionsToTxnRequest) key() int16 {
-	return 24
-}
-
-func (a *AddPartitionsToTxnRequest) version() int16 {
-	return 0
-}
-
-func (a *AddPartitionsToTxnRequest) headerVersion() int16 {
-	return 1
-}
-
-func (a *AddPartitionsToTxnRequest) requiredVersion() KafkaVersion {
-	return V0_11_0_0
-}

vendor/github.com/Shopify/sarama/add_partitions_to_txn_response.go

@@ -1,114 +0,0 @@
-package sarama
-
-import (
-	"time"
-)
-
-//AddPartitionsToTxnResponse is a partition errors to transaction type
-type AddPartitionsToTxnResponse struct {
-	ThrottleTime time.Duration
-	Errors       map[string][]*PartitionError
-}
-
-func (a *AddPartitionsToTxnResponse) encode(pe packetEncoder) error {
-	pe.putInt32(int32(a.ThrottleTime / time.Millisecond))
-	if err := pe.putArrayLength(len(a.Errors)); err != nil {
-		return err
-	}
-
-	for topic, e := range a.Errors {
-		if err := pe.putString(topic); err != nil {
-			return err
-		}
-		if err := pe.putArrayLength(len(e)); err != nil {
-			return err
-		}
-		for _, partitionError := range e {
-			if err := partitionError.encode(pe); err != nil {
-				return err
-			}
-		}
-	}
-
-	return nil
-}
-
-func (a *AddPartitionsToTxnResponse) decode(pd packetDecoder, version int16) (err error) {
-	throttleTime, err := pd.getInt32()
-	if err != nil {
-		return err
-	}
-	a.ThrottleTime = time.Duration(throttleTime) * time.Millisecond
-
-	n, err := pd.getArrayLength()
-	if err != nil {
-		return err
-	}
-
-	a.Errors = make(map[string][]*PartitionError)
-
-	for i := 0; i < n; i++ {
-		topic, err := pd.getString()
-		if err != nil {
-			return err
-		}
-
-		m, err := pd.getArrayLength()
-		if err != nil {
-			return err
-		}
-
-		a.Errors[topic] = make([]*PartitionError, m)
-
-		for j := 0; j < m; j++ {
-			a.Errors[topic][j] = new(PartitionError)
-			if err := a.Errors[topic][j].decode(pd, version); err != nil {
-				return err
-			}
-		}
-	}
-
-	return nil
-}
-
-func (a *AddPartitionsToTxnResponse) key() int16 {
-	return 24
-}
-
-func (a *AddPartitionsToTxnResponse) version() int16 {
-	return 0
-}
-
-func (a *AddPartitionsToTxnResponse) headerVersion() int16 {
-	return 0
-}
-
-func (a *AddPartitionsToTxnResponse) requiredVersion() KafkaVersion {
-	return V0_11_0_0
-}
-
-//PartitionError is a partition error type
-type PartitionError struct {
-	Partition int32
-	Err       KError
-}
-
-func (p *PartitionError) encode(pe packetEncoder) error {
-	pe.putInt32(p.Partition)
-	pe.putInt16(int16(p.Err))
-	return nil
-}
-
-func (p *PartitionError) decode(pd packetDecoder, version int16) (err error) {
-	if p.Partition, err = pd.getInt32(); err != nil {
-		return err
-	}
-
-	kerr, err := pd.getInt16()
-	if err != nil {
-		return err
-	}
-	p.Err = KError(kerr)
-
-	return nil
-}

vendor/github.com/Shopify/sarama/admin.go

@@ -1,934 +0,0 @@
-package sarama
-
-import (
-	"errors"
-	"fmt"
-	"math/rand"
-	"strconv"
-	"sync"
-	"time"
-)
-
-// ClusterAdmin is the administrative client for Kafka, which supports managing and inspecting topics,
-// brokers, configurations and ACLs. The minimum broker version required is 0.10.0.0.
-// Methods with stricter requirements will specify the minimum broker version required.
-// You MUST call Close() on a client to avoid leaks
-type ClusterAdmin interface {
-	// Creates a new topic. This operation is supported by brokers with version 0.10.1.0 or higher.
-	// It may take several seconds after CreateTopic returns success for all the brokers
-	// to become aware that the topic has been created. During this time, listTopics
-	// may not return information about the new topic.The validateOnly option is supported from version 0.10.2.0.
-	CreateTopic(topic string, detail *TopicDetail, validateOnly bool) error
-
-	// List the topics available in the cluster with the default options.
-	ListTopics() (map[string]TopicDetail, error)
-
-	// Describe some topics in the cluster.
-	DescribeTopics(topics []string) (metadata []*TopicMetadata, err error)
-
-	// Delete a topic. It may take several seconds after the DeleteTopic to returns success
-	// and for all the brokers to become aware that the topics are gone.
-	// During this time, listTopics  may continue to return information about the deleted topic.
-	// If delete.topic.enable is false on the brokers, deleteTopic will mark
-	// the topic for deletion, but not actually delete them.
-	// This operation is supported by brokers with version 0.10.1.0 or higher.
-	DeleteTopic(topic string) error
-
-	// Increase the number of partitions of the topics  according to the corresponding values.
-	// If partitions are increased for a topic that has a key, the partition logic or ordering of
-	// the messages will be affected. It may take several seconds after this method returns
-	// success for all the brokers to become aware that the partitions have been created.
-	// During this time, ClusterAdmin#describeTopics may not return information about the
-	// new partitions. This operation is supported by brokers with version 1.0.0 or higher.
-	CreatePartitions(topic string, count int32, assignment [][]int32, validateOnly bool) error
-
-	// Alter the replica assignment for partitions.
-	// This operation is supported by brokers with version 2.4.0.0 or higher.
-	AlterPartitionReassignments(topic string, assignment [][]int32) error
-
-	// Provides info on ongoing partitions replica reassignments.
-	// This operation is supported by brokers with version 2.4.0.0 or higher.
-	ListPartitionReassignments(topics string, partitions []int32) (topicStatus map[string]map[int32]*PartitionReplicaReassignmentsStatus, err error)
-
-	// Delete records whose offset is smaller than the given offset of the corresponding partition.
-	// This operation is supported by brokers with version 0.11.0.0 or higher.
-	DeleteRecords(topic string, partitionOffsets map[int32]int64) error
-
-	// Get the configuration for the specified resources.
-	// The returned configuration includes default values and the Default is true
-	// can be used to distinguish them from user supplied values.
-	// Config entries where ReadOnly is true cannot be updated.
-	// The value of config entries where Sensitive is true is always nil so
-	// sensitive information is not disclosed.
-	// This operation is supported by brokers with version 0.11.0.0 or higher.
-	DescribeConfig(resource ConfigResource) ([]ConfigEntry, error)
-
-	// Update the configuration for the specified resources with the default options.
-	// This operation is supported by brokers with version 0.11.0.0 or higher.
-	// The resources with their configs (topic is the only resource type with configs
-	// that can be updated currently Updates are not transactional so they may succeed
-	// for some resources while fail for others. The configs for a particular resource are updated automatically.
-	AlterConfig(resourceType ConfigResourceType, name string, entries map[string]*string, validateOnly bool) error
-
-	// Creates access control lists (ACLs) which are bound to specific resources.
-	// This operation is not transactional so it may succeed for some ACLs while fail for others.
-	// If you attempt to add an ACL that duplicates an existing ACL, no error will be raised, but
-	// no changes will be made. This operation is supported by brokers with version 0.11.0.0 or higher.
-	CreateACL(resource Resource, acl Acl) error
-
-	// Lists access control lists (ACLs) according to the supplied filter.
-	// it may take some time for changes made by createAcls or deleteAcls to be reflected in the output of ListAcls
-	// This operation is supported by brokers with version 0.11.0.0 or higher.
-	ListAcls(filter AclFilter) ([]ResourceAcls, error)
-
-	// Deletes access control lists (ACLs) according to the supplied filters.
-	// This operation is not transactional so it may succeed for some ACLs while fail for others.
-	// This operation is supported by brokers with version 0.11.0.0 or higher.
-	DeleteACL(filter AclFilter, validateOnly bool) ([]MatchingAcl, error)
-
-	// List the consumer groups available in the cluster.
-	ListConsumerGroups() (map[string]string, error)
-
-	// Describe the given consumer groups.
-	DescribeConsumerGroups(groups []string) ([]*GroupDescription, error)
-
-	// List the consumer group offsets available in the cluster.
-	ListConsumerGroupOffsets(group string, topicPartitions map[string][]int32) (*OffsetFetchResponse, error)
-
-	// Delete a consumer group.
-	DeleteConsumerGroup(group string) error
-
-	// Get information about the nodes in the cluster
-	DescribeCluster() (brokers []*Broker, controllerID int32, err error)
-
-	// Get information about all log directories on the given set of brokers
-	DescribeLogDirs(brokers []int32) (map[int32][]DescribeLogDirsResponseDirMetadata, error)
-
-	// Close shuts down the admin and closes underlying client.
-	Close() error
-}
-
-type clusterAdmin struct {
-	client Client
-	conf   *Config
-}
-
-// NewClusterAdmin creates a new ClusterAdmin using the given broker addresses and configuration.
-func NewClusterAdmin(addrs []string, conf *Config) (ClusterAdmin, error) {
-	client, err := NewClient(addrs, conf)
-	if err != nil {
-		return nil, err
-	}
-	return NewClusterAdminFromClient(client)
-}
-
-// NewClusterAdminFromClient creates a new ClusterAdmin using the given client.
-// Note that underlying client will also be closed on admin's Close() call.
-func NewClusterAdminFromClient(client Client) (ClusterAdmin, error) {
-	//make sure we can retrieve the controller
-	_, err := client.Controller()
-	if err != nil {
-		return nil, err
-	}
-
-	ca := &clusterAdmin{
-		client: client,
-		conf:   client.Config(),
-	}
-	return ca, nil
-}
-
-func (ca *clusterAdmin) Close() error {
-	return ca.client.Close()
-}
-
-func (ca *clusterAdmin) Controller() (*Broker, error) {
-	return ca.client.Controller()
-}
-
-func (ca *clusterAdmin) refreshController() (*Broker, error) {
-	return ca.client.RefreshController()
-}
-
-// isErrNoController returns `true` if the given error type unwraps to an
-// `ErrNotController` response from Kafka
-func isErrNoController(err error) bool {
-	switch e := err.(type) {
-	case *TopicError:
-		return e.Err == ErrNotController
-	case *TopicPartitionError:
-		return e.Err == ErrNotController
-	case KError:
-		return e == ErrNotController
-	}
-	return false
-}
-
-// retryOnError will repeatedly call the given (error-returning) func in the
-// case that its response is non-nil and retriable (as determined by the
-// provided retriable func) up to the maximum number of tries permitted by
-// the admin client configuration
-func (ca *clusterAdmin) retryOnError(retriable func(error) bool, fn func() error) error {
-	var err error
-	for attempt := 0; attempt < ca.conf.Admin.Retry.Max; attempt++ {
-		err = fn()
-		if err == nil || !retriable(err) {
-			return err
-		}
-		Logger.Printf(
-			"admin/request retrying after %dms... (%d attempts remaining)\n",
-			ca.conf.Admin.Retry.Backoff/time.Millisecond, ca.conf.Admin.Retry.Max-attempt)
-		time.Sleep(ca.conf.Admin.Retry.Backoff)
-		continue
-	}
-	return err
-}
-
-func (ca *clusterAdmin) CreateTopic(topic string, detail *TopicDetail, validateOnly bool) error {
-	if topic == "" {
-		return ErrInvalidTopic
-	}
-
-	if detail == nil {
-		return errors.New("you must specify topic details")
-	}
-
-	topicDetails := make(map[string]*TopicDetail)
-	topicDetails[topic] = detail
-
-	request := &CreateTopicsRequest{
-		TopicDetails: topicDetails,
-		ValidateOnly: validateOnly,
-		Timeout:      ca.conf.Admin.Timeout,
-	}
-
-	if ca.conf.Version.IsAtLeast(V0_11_0_0) {
-		request.Version = 1
-	}
-	if ca.conf.Version.IsAtLeast(V1_0_0_0) {
-		request.Version = 2
-	}
-
-	return ca.retryOnError(isErrNoController, func() error {
-		b, err := ca.Controller()
-		if err != nil {
-			return err
-		}
-
-		rsp, err := b.CreateTopics(request)
-		if err != nil {
-			return err
-		}
-
-		topicErr, ok := rsp.TopicErrors[topic]
-		if !ok {
-			return ErrIncompleteResponse
-		}
-
-		if topicErr.Err != ErrNoError {
-			if topicErr.Err == ErrNotController {
-				_, _ = ca.refreshController()
-			}
-			return topicErr
-		}
-
-		return nil
-	})
-}
-
-func (ca *clusterAdmin) DescribeTopics(topics []string) (metadata []*TopicMetadata, err error) {
-	controller, err := ca.Controller()
-	if err != nil {
-		return nil, err
-	}
-
-	request := &MetadataRequest{
-		Topics:                 topics,
-		AllowAutoTopicCreation: false,
-	}
-
-	if ca.conf.Version.IsAtLeast(V1_0_0_0) {
-		request.Version = 5
-	} else if ca.conf.Version.IsAtLeast(V0_11_0_0) {
-		request.Version = 4
-	}
-
-	response, err := controller.GetMetadata(request)
-	if err != nil {
-		return nil, err
-	}
-	return response.Topics, nil
-}
-
-func (ca *clusterAdmin) DescribeCluster() (brokers []*Broker, controllerID int32, err error) {
-	controller, err := ca.Controller()
-	if err != nil {
-		return nil, int32(0), err
-	}
-
-	request := &MetadataRequest{
-		Topics: []string{},
-	}
-
-	if ca.conf.Version.IsAtLeast(V0_10_0_0) {
-		request.Version = 1
-	}
-
-	response, err := controller.GetMetadata(request)
-	if err != nil {
-		return nil, int32(0), err
-	}
-
-	return response.Brokers, response.ControllerID, nil
-}
-
-func (ca *clusterAdmin) findBroker(id int32) (*Broker, error) {
-	brokers := ca.client.Brokers()
-	for _, b := range brokers {
-		if b.ID() == id {
-			return b, nil
-		}
-	}
-	return nil, fmt.Errorf("could not find broker id %d", id)
-}
-
-func (ca *clusterAdmin) findAnyBroker() (*Broker, error) {
-	brokers := ca.client.Brokers()
-	if len(brokers) > 0 {
-		index := rand.Intn(len(brokers))
-		return brokers[index], nil
-	}
-	return nil, errors.New("no available broker")
-}
-
-func (ca *clusterAdmin) ListTopics() (map[string]TopicDetail, error) {
-	// In order to build TopicDetails we need to first get the list of all
-	// topics using a MetadataRequest and then get their configs using a
-	// DescribeConfigsRequest request. To avoid sending many requests to the
-	// broker, we use a single DescribeConfigsRequest.
-
-	// Send the all-topic MetadataRequest
-	b, err := ca.findAnyBroker()
-	if err != nil {
-		return nil, err
-	}
-	_ = b.Open(ca.client.Config())
-
-	metadataReq := &MetadataRequest{}
-	metadataResp, err := b.GetMetadata(metadataReq)
-	if err != nil {
-		return nil, err
-	}
-
-	topicsDetailsMap := make(map[string]TopicDetail)
-
-	var describeConfigsResources []*ConfigResource
-
-	for _, topic := range metadataResp.Topics {
-		topicDetails := TopicDetail{
-			NumPartitions: int32(len(topic.Partitions)),
-		}
-		if len(topic.Partitions) > 0 {
-			topicDetails.ReplicaAssignment = map[int32][]int32{}
-			for _, partition := range topic.Partitions {
-				topicDetails.ReplicaAssignment[partition.ID] = partition.Replicas
-			}
-			topicDetails.ReplicationFactor = int16(len(topic.Partitions[0].Replicas))
-		}
-		topicsDetailsMap[topic.Name] = topicDetails
-
-		// we populate the resources we want to describe from the MetadataResponse
-		topicResource := ConfigResource{
-			Type: TopicResource,
-			Name: topic.Name,
-		}
-		describeConfigsResources = append(describeConfigsResources, &topicResource)
-	}
-
-	// Send the DescribeConfigsRequest
-	describeConfigsReq := &DescribeConfigsRequest{
-		Resources: describeConfigsResources,
-	}
-
-	if ca.conf.Version.IsAtLeast(V1_1_0_0) {
-		describeConfigsReq.Version = 1
-	}
-
-	if ca.conf.Version.IsAtLeast(V2_0_0_0) {
-		describeConfigsReq.Version = 2
-	}
-
-	describeConfigsResp, err := b.DescribeConfigs(describeConfigsReq)
-	if err != nil {
-		return nil, err
-	}
-
-	for _, resource := range describeConfigsResp.Resources {
-		topicDetails := topicsDetailsMap[resource.Name]
-		topicDetails.ConfigEntries = make(map[string]*string)
-
-		for _, entry := range resource.Configs {
-			// only include non-default non-sensitive config
-			// (don't actually think topic config will ever be sensitive)
-			if entry.Default || entry.Sensitive {
-				continue
-			}
-			topicDetails.ConfigEntries[entry.Name] = &entry.Value
-		}
-
-		topicsDetailsMap[resource.Name] = topicDetails
-	}
-
-	return topicsDetailsMap, nil
-}
-
-func (ca *clusterAdmin) DeleteTopic(topic string) error {
-	if topic == "" {
-		return ErrInvalidTopic
-	}
-
-	request := &DeleteTopicsRequest{
-		Topics:  []string{topic},
-		Timeout: ca.conf.Admin.Timeout,
-	}
-
-	if ca.conf.Version.IsAtLeast(V0_11_0_0) {
-		request.Version = 1
-	}
-
-	return ca.retryOnError(isErrNoController, func() error {
-		b, err := ca.Controller()
-		if err != nil {
-			return err
-		}
-
-		rsp, err := b.DeleteTopics(request)
-		if err != nil {
-			return err
-		}
-
-		topicErr, ok := rsp.TopicErrorCodes[topic]
-		if !ok {
-			return ErrIncompleteResponse
-		}
-
-		if topicErr != ErrNoError {
-			if topicErr == ErrNotController {
-				_, _ = ca.refreshController()
-			}
-			return topicErr
-		}
-
-		return nil
-	})
-}
-
-func (ca *clusterAdmin) CreatePartitions(topic string, count int32, assignment [][]int32, validateOnly bool) error {
-	if topic == "" {
-		return ErrInvalidTopic
-	}
-
-	topicPartitions := make(map[string]*TopicPartition)
-	topicPartitions[topic] = &TopicPartition{Count: count, Assignment: assignment}
-
-	request := &CreatePartitionsRequest{
-		TopicPartitions: topicPartitions,
-		Timeout:         ca.conf.Admin.Timeout,
-	}
-
-	return ca.retryOnError(isErrNoController, func() error {
-		b, err := ca.Controller()
-		if err != nil {
-			return err
-		}
-
-		rsp, err := b.CreatePartitions(request)
-		if err != nil {
-			return err
-		}
-
-		topicErr, ok := rsp.TopicPartitionErrors[topic]
-		if !ok {
-			return ErrIncompleteResponse
-		}
-
-		if topicErr.Err != ErrNoError {
-			if topicErr.Err == ErrNotController {
-				_, _ = ca.refreshController()
-			}
-			return topicErr
-		}
-
-		return nil
-	})
-}
-
-func (ca *clusterAdmin) AlterPartitionReassignments(topic string, assignment [][]int32) error {
-	if topic == "" {
-		return ErrInvalidTopic
-	}
-
-	request := &AlterPartitionReassignmentsRequest{
-		TimeoutMs: int32(60000),
-		Version:   int16(0),
-	}
-
-	for i := 0; i < len(assignment); i++ {
-		request.AddBlock(topic, int32(i), assignment[i])
-	}
-
-	return ca.retryOnError(isErrNoController, func() error {
-		b, err := ca.Controller()
-		if err != nil {
-			return err
-		}
-
-		errs := make([]error, 0)
-
-		rsp, err := b.AlterPartitionReassignments(request)
-
-		if err != nil {
-			errs = append(errs, err)
-		} else {
-			if rsp.ErrorCode > 0 {
-				errs = append(errs, errors.New(rsp.ErrorCode.Error()))
-			}
-
-			for topic, topicErrors := range rsp.Errors {
-				for partition, partitionError := range topicErrors {
-					if partitionError.errorCode != ErrNoError {
-						errStr := fmt.Sprintf("[%s-%d]: %s", topic, partition, partitionError.errorCode.Error())
-						errs = append(errs, errors.New(errStr))
-					}
-				}
-			}
-		}
-
-		if len(errs) > 0 {
-			return ErrReassignPartitions{MultiError{&errs}}
-		}
-
-		return nil
-	})
-}
-
-func (ca *clusterAdmin) ListPartitionReassignments(topic string, partitions []int32) (topicStatus map[string]map[int32]*PartitionReplicaReassignmentsStatus, err error) {
-	if topic == "" {
-		return nil, ErrInvalidTopic
-	}
-
-	request := &ListPartitionReassignmentsRequest{
-		TimeoutMs: int32(60000),
-		Version:   int16(0),
-	}
-
-	request.AddBlock(topic, partitions)
-
-	b, err := ca.Controller()
-	if err != nil {
-		return nil, err
-	}
-	_ = b.Open(ca.client.Config())
-
-	rsp, err := b.ListPartitionReassignments(request)
-
-	if err == nil && rsp != nil {
-		return rsp.TopicStatus, nil
-	} else {
-		return nil, err
-	}
-}
-
-func (ca *clusterAdmin) DeleteRecords(topic string, partitionOffsets map[int32]int64) error {
-	if topic == "" {
-		return ErrInvalidTopic
-	}
-	partitionPerBroker := make(map[*Broker][]int32)
-	for partition := range partitionOffsets {
-		broker, err := ca.client.Leader(topic, partition)
-		if err != nil {
-			return err
-		}
-		if _, ok := partitionPerBroker[broker]; ok {
-			partitionPerBroker[broker] = append(partitionPerBroker[broker], partition)
-		} else {
-			partitionPerBroker[broker] = []int32{partition}
-		}
-	}
-	errs := make([]error, 0)
-	for broker, partitions := range partitionPerBroker {
-		topics := make(map[string]*DeleteRecordsRequestTopic)
-		recordsToDelete := make(map[int32]int64)
-		for _, p := range partitions {
-			recordsToDelete[p] = partitionOffsets[p]
-		}
-		topics[topic] = &DeleteRecordsRequestTopic{PartitionOffsets: recordsToDelete}
-		request := &DeleteRecordsRequest{
-			Topics:  topics,
-			Timeout: ca.conf.Admin.Timeout,
-		}
-
-		rsp, err := broker.DeleteRecords(request)
-		if err != nil {
-			errs = append(errs, err)
-		} else {
-			deleteRecordsResponseTopic, ok := rsp.Topics[topic]
-			if !ok {
-				errs = append(errs, ErrIncompleteResponse)
-			} else {
-				for _, deleteRecordsResponsePartition := range deleteRecordsResponseTopic.Partitions {
-					if deleteRecordsResponsePartition.Err != ErrNoError {
-						errs = append(errs, errors.New(deleteRecordsResponsePartition.Err.Error()))
-					}
-				}
-			}
-		}
-	}
-	if len(errs) > 0 {
-		return ErrDeleteRecords{MultiError{&errs}}
-	}
-	//todo since we are dealing with couple of partitions it would be good if we return slice of errors
-	//for each partition instead of one error
-	return nil
-}
-
-// Returns a bool indicating whether the resource request needs to go to a
-// specific broker
-func dependsOnSpecificNode(resource ConfigResource) bool {
-	return (resource.Type == BrokerResource && resource.Name != "") ||
-		resource.Type == BrokerLoggerResource
-}
-
-func (ca *clusterAdmin) DescribeConfig(resource ConfigResource) ([]ConfigEntry, error) {
-	var entries []ConfigEntry
-	var resources []*ConfigResource
-	resources = append(resources, &resource)
-
-	request := &DescribeConfigsRequest{
-		Resources: resources,
-	}
-
-	if ca.conf.Version.IsAtLeast(V1_1_0_0) {
-		request.Version = 1
-	}
-
-	if ca.conf.Version.IsAtLeast(V2_0_0_0) {
-		request.Version = 2
-	}
-
-	var (
-		b   *Broker
-		err error
-	)
-
-	// DescribeConfig of broker/broker logger must be sent to the broker in question
-	if dependsOnSpecificNode(resource) {
-		id, _ := strconv.Atoi(resource.Name)
-		b, err = ca.findBroker(int32(id))
-	} else {
-		b, err = ca.findAnyBroker()
-	}
-	if err != nil {
-		return nil, err
-	}
-
-	_ = b.Open(ca.client.Config())
-	rsp, err := b.DescribeConfigs(request)
-	if err != nil {
-		return nil, err
-	}
-
-	for _, rspResource := range rsp.Resources {
-		if rspResource.Name == resource.Name {
-			if rspResource.ErrorMsg != "" {
-				return nil, errors.New(rspResource.ErrorMsg)
-			}
-			if rspResource.ErrorCode != 0 {
-				return nil, KError(rspResource.ErrorCode)
-			}
-			for _, cfgEntry := range rspResource.Configs {
-				entries = append(entries, *cfgEntry)
-			}
-		}
-	}
-	return entries, nil
-}
-
-func (ca *clusterAdmin) AlterConfig(resourceType ConfigResourceType, name string, entries map[string]*string, validateOnly bool) error {
-	var resources []*AlterConfigsResource
-	resources = append(resources, &AlterConfigsResource{
-		Type:          resourceType,
-		Name:          name,
-		ConfigEntries: entries,
-	})
-
-	request := &AlterConfigsRequest{
-		Resources:    resources,
-		ValidateOnly: validateOnly,
-	}
-
-	var (
-		b   *Broker
-		err error
-	)
-
-	// AlterConfig of broker/broker logger must be sent to the broker in question
-	if dependsOnSpecificNode(ConfigResource{Name: name, Type: resourceType}) {
-		id, _ := strconv.Atoi(name)
-		b, err = ca.findBroker(int32(id))
-	} else {
-		b, err = ca.findAnyBroker()
-	}
-	if err != nil {
-		return err
-	}
-
-	_ = b.Open(ca.client.Config())
-	rsp, err := b.AlterConfigs(request)
-	if err != nil {
-		return err
-	}
-
-	for _, rspResource := range rsp.Resources {
-		if rspResource.Name == name {
-			if rspResource.ErrorMsg != "" {
-				return errors.New(rspResource.ErrorMsg)
-			}
-			if rspResource.ErrorCode != 0 {
-				return KError(rspResource.ErrorCode)
-			}
-		}
-	}
-	return nil
-}
-
-func (ca *clusterAdmin) CreateACL(resource Resource, acl Acl) error {
-	var acls []*AclCreation
-	acls = append(acls, &AclCreation{resource, acl})
-	request := &CreateAclsRequest{AclCreations: acls}
-
-	if ca.conf.Version.IsAtLeast(V2_0_0_0) {
-		request.Version = 1
-	}
-
-	b, err := ca.Controller()
-	if err != nil {
-		return err
-	}
-
-	_, err = b.CreateAcls(request)
-	return err
-}
-
-func (ca *clusterAdmin) ListAcls(filter AclFilter) ([]ResourceAcls, error) {
-	request := &DescribeAclsRequest{AclFilter: filter}
-
-	if ca.conf.Version.IsAtLeast(V2_0_0_0) {
-		request.Version = 1
-	}
-
-	b, err := ca.Controller()
-	if err != nil {
-		return nil, err
-	}
-
-	rsp, err := b.DescribeAcls(request)
-	if err != nil {
-		return nil, err
-	}
-
-	var lAcls []ResourceAcls
-	for _, rAcl := range rsp.ResourceAcls {
-		lAcls = append(lAcls, *rAcl)
-	}
-	return lAcls, nil
-}
-
-func (ca *clusterAdmin) DeleteACL(filter AclFilter, validateOnly bool) ([]MatchingAcl, error) {
-	var filters []*AclFilter
-	filters = append(filters, &filter)
-	request := &DeleteAclsRequest{Filters: filters}
-
-	if ca.conf.Version.IsAtLeast(V2_0_0_0) {
-		request.Version = 1
-	}
-
-	b, err := ca.Controller()
-	if err != nil {
-		return nil, err
-	}
-
-	rsp, err := b.DeleteAcls(request)
-	if err != nil {
-		return nil, err
-	}
-
-	var mAcls []MatchingAcl
-	for _, fr := range rsp.FilterResponses {
-		for _, mACL := range fr.MatchingAcls {
-			mAcls = append(mAcls, *mACL)
-		}
-	}
-	return mAcls, nil
-}
-
-func (ca *clusterAdmin) DescribeConsumerGroups(groups []string) (result []*GroupDescription, err error) {
-	groupsPerBroker := make(map[*Broker][]string)
-
-	for _, group := range groups {
-		controller, err := ca.client.Coordinator(group)
-		if err != nil {
-			return nil, err
-		}
-		groupsPerBroker[controller] = append(groupsPerBroker[controller], group)
-	}
-
-	for broker, brokerGroups := range groupsPerBroker {
-		response, err := broker.DescribeGroups(&DescribeGroupsRequest{
-			Groups: brokerGroups,
-		})
-		if err != nil {
-			return nil, err
-		}
-
-		result = append(result, response.Groups...)
-	}
-	return result, nil
-}
-
-func (ca *clusterAdmin) ListConsumerGroups() (allGroups map[string]string, err error) {
-	allGroups = make(map[string]string)
-
-	// Query brokers in parallel, since we have to query *all* brokers
-	brokers := ca.client.Brokers()
-	groupMaps := make(chan map[string]string, len(brokers))
-	errors := make(chan error, len(brokers))
-	wg := sync.WaitGroup{}
-
-	for _, b := range brokers {
-		wg.Add(1)
-		go func(b *Broker, conf *Config) {
-			defer wg.Done()
-			_ = b.Open(conf) // Ensure that broker is opened
-
-			response, err := b.ListGroups(&ListGroupsRequest{})
-			if err != nil {
-				errors <- err
-				return
-			}
-
-			groups := make(map[string]string)
-			for group, typ := range response.Groups {
-				groups[group] = typ
-			}
-
-			groupMaps <- groups
-		}(b, ca.conf)
-	}
-
-	wg.Wait()
-	close(groupMaps)
-	close(errors)
-
-	for groupMap := range groupMaps {
-		for group, protocolType := range groupMap {
-			allGroups[group] = protocolType
-		}
-	}
-
-	// Intentionally return only the first error for simplicity
-	err = <-errors
-	return
-}
-
-func (ca *clusterAdmin) ListConsumerGroupOffsets(group string, topicPartitions map[string][]int32) (*OffsetFetchResponse, error) {
-	coordinator, err := ca.client.Coordinator(group)
-	if err != nil {
-		return nil, err
-	}
-
-	request := &OffsetFetchRequest{
-		ConsumerGroup: group,
-		partitions:    topicPartitions,
-	}
-
-	if ca.conf.Version.IsAtLeast(V0_10_2_0) {
-		request.Version = 2
-	} else if ca.conf.Version.IsAtLeast(V0_8_2_2) {
-		request.Version = 1
-	}
-
-	return coordinator.FetchOffset(request)
-}
-
-func (ca *clusterAdmin) DeleteConsumerGroup(group string) error {
-	coordinator, err := ca.client.Coordinator(group)
-	if err != nil {
-		return err
-	}
-
-	request := &DeleteGroupsRequest{
-		Groups: []string{group},
-	}
-
-	resp, err := coordinator.DeleteGroups(request)
-	if err != nil {
-		return err
-	}
-
-	groupErr, ok := resp.GroupErrorCodes[group]
-	if !ok {
-		return ErrIncompleteResponse
-	}
-
-	if groupErr != ErrNoError {
-		return groupErr
-	}
-
-	return nil
-}
-
-func (ca *clusterAdmin) DescribeLogDirs(brokerIds []int32) (allLogDirs map[int32][]DescribeLogDirsResponseDirMetadata, err error) {
-	allLogDirs = make(map[int32][]DescribeLogDirsResponseDirMetadata)
-
-	// Query brokers in parallel, since we may have to query multiple brokers
-	logDirsMaps := make(chan map[int32][]DescribeLogDirsResponseDirMetadata, len(brokerIds))
-	errors := make(chan error, len(brokerIds))
-	wg := sync.WaitGroup{}
-
-	for _, b := range brokerIds {
-		wg.Add(1)
-		broker, err := ca.findBroker(b)
-		if err != nil {
-			Logger.Printf("Unable to find broker with ID = %v\n", b)
-			continue
-		}
-		go func(b *Broker, conf *Config) {
-			defer wg.Done()
-			_ = b.Open(conf) // Ensure that broker is opened
-
-			response, err := b.DescribeLogDirs(&DescribeLogDirsRequest{})
-			if err != nil {
-				errors <- err
-				return
-			}
-			logDirs := make(map[int32][]DescribeLogDirsResponseDirMetadata)
-			logDirs[b.ID()] = response.LogDirs
-			logDirsMaps <- logDirs
-		}(broker, ca.conf)
-	}
-
-	wg.Wait()
-	close(logDirsMaps)
-	close(errors)
-
-	for logDirsMap := range logDirsMaps {
-		for id, logDirs := range logDirsMap {
-			allLogDirs[id] = logDirs
-		}
-	}
-
-	// Intentionally return only the first error for simplicity
-	err = <-errors
-	return
-}

vendor/github.com/Shopify/sarama/alter_configs_request.go

@@ -1,126 +0,0 @@
-package sarama
-
-//AlterConfigsRequest is an alter config request type
-type AlterConfigsRequest struct {
-	Resources    []*AlterConfigsResource
-	ValidateOnly bool
-}
-
-//AlterConfigsResource is an alter config resource type
-type AlterConfigsResource struct {
-	Type          ConfigResourceType
-	Name          string
-	ConfigEntries map[string]*string
-}
-
-func (a *AlterConfigsRequest) encode(pe packetEncoder) error {
-	if err := pe.putArrayLength(len(a.Resources)); err != nil {
-		return err
-	}
-
-	for _, r := range a.Resources {
-		if err := r.encode(pe); err != nil {
-			return err
-		}
-	}
-
-	pe.putBool(a.ValidateOnly)
-	return nil
-}
-
-func (a *AlterConfigsRequest) decode(pd packetDecoder, version int16) error {
-	resourceCount, err := pd.getArrayLength()
-	if err != nil {
-		return err
-	}
-
-	a.Resources = make([]*AlterConfigsResource, resourceCount)
-	for i := range a.Resources {
-		r := &AlterConfigsResource{}
-		err = r.decode(pd, version)
-		if err != nil {
-			return err
-		}
-		a.Resources[i] = r
-	}
-
-	validateOnly, err := pd.getBool()
-	if err != nil {
-		return err
-	}
-
-	a.ValidateOnly = validateOnly
-
-	return nil
-}
-
-func (a *AlterConfigsResource) encode(pe packetEncoder) error {
-	pe.putInt8(int8(a.Type))
-
-	if err := pe.putString(a.Name); err != nil {
-		return err
-	}
-
-	if err := pe.putArrayLength(len(a.ConfigEntries)); err != nil {
-		return err
-	}
-	for configKey, configValue := range a.ConfigEntries {
-		if err := pe.putString(configKey); err != nil {
-			return err
-		}
-		if err := pe.putNullableString(configValue); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-func (a *AlterConfigsResource) decode(pd packetDecoder, version int16) error {
-	t, err := pd.getInt8()
-	if err != nil {
-		return err
-	}
-	a.Type = ConfigResourceType(t)
-
-	name, err := pd.getString()
-	if err != nil {
-		return err
-	}
-	a.Name = name
-
-	n, err := pd.getArrayLength()
-	if err != nil {
-		return err
-	}
-
-	if n > 0 {
-		a.ConfigEntries = make(map[string]*string, n)
-		for i := 0; i < n; i++ {
-			configKey, err := pd.getString()
-			if err != nil {
-				return err
-			}
-			if a.ConfigEntries[configKey], err = pd.getNullableString(); err != nil {
-				return err
-			}
-		}
-	}
-	return err
-}
-
-func (a *AlterConfigsRequest) key() int16 {
-	return 33
-}
-
-func (a *AlterConfigsRequest) version() int16 {
-	return 0
-}
-
-func (a *AlterConfigsRequest) headerVersion() int16 {
-	return 1
-}
-
-func (a *AlterConfigsRequest) requiredVersion() KafkaVersion {
-	return V0_11_0_0
-}

vendor/github.com/Shopify/sarama/alter_configs_response.go

@@ -1,101 +0,0 @@
-package sarama
-
-import "time"
-
-//AlterConfigsResponse is a response type for alter config
-type AlterConfigsResponse struct {
-	ThrottleTime time.Duration
-	Resources    []*AlterConfigsResourceResponse
-}
-
-//AlterConfigsResourceResponse is a response type for alter config resource
-type AlterConfigsResourceResponse struct {
-	ErrorCode int16
-	ErrorMsg  string
-	Type      ConfigResourceType
-	Name      string
-}
-
-func (a *AlterConfigsResponse) encode(pe packetEncoder) error {
-	pe.putInt32(int32(a.ThrottleTime / time.Millisecond))
-
-	if err := pe.putArrayLength(len(a.Resources)); err != nil {
-		return err
-	}
-
-	for i := range a.Resources {
-		pe.putInt16(a.Resources[i].ErrorCode)
-		err := pe.putString(a.Resources[i].ErrorMsg)
-		if err != nil {
-			return nil
-		}
-		pe.putInt8(int8(a.Resources[i].Type))
-		err = pe.putString(a.Resources[i].Name)
-		if err != nil {
-			return nil
-		}
-	}
-
-	return nil
-}
-
-func (a *AlterConfigsResponse) decode(pd packetDecoder, version int16) error {
-	throttleTime, err := pd.getInt32()
-	if err != nil {
-		return err
-	}
-	a.ThrottleTime = time.Duration(throttleTime) * time.Millisecond
-
-	responseCount, err := pd.getArrayLength()
-	if err != nil {
-		return err
-	}
-
-	a.Resources = make([]*AlterConfigsResourceResponse, responseCount)
-
-	for i := range a.Resources {
-		a.Resources[i] = new(AlterConfigsResourceResponse)
-
-		errCode, err := pd.getInt16()
-		if err != nil {
-			return err
-		}
-		a.Resources[i].ErrorCode = errCode
-
-		e, err := pd.getString()
-		if err != nil {
-			return err
-		}
-		a.Resources[i].ErrorMsg = e
-
-		t, err := pd.getInt8()
-		if err != nil {
-			return err
-		}
-		a.Resources[i].Type = ConfigResourceType(t)
-
-		name, err := pd.getString()
-		if err != nil {
-			return err
-		}
-		a.Resources[i].Name = name
-	}
-
-	return nil
-}
-
-func (a *AlterConfigsResponse) key() int16 {
-	return 32
-}
-
-func (a *AlterConfigsResponse) version() int16 {
-	return 0
-}
-
-func (a *AlterConfigsResponse) headerVersion() int16 {
-	return 0
-}
-
-func (a *AlterConfigsResponse) requiredVersion() KafkaVersion {
-	return V0_11_0_0
-}

vendor/github.com/Shopify/sarama/alter_partition_reassignments_request.go

@@ -1,130 +0,0 @@
-package sarama
-
-type alterPartitionReassignmentsBlock struct {
-	replicas []int32
-}
-
-func (b *alterPartitionReassignmentsBlock) encode(pe packetEncoder) error {
-	if err := pe.putNullableCompactInt32Array(b.replicas); err != nil {
-		return err
-	}
-
-	pe.putEmptyTaggedFieldArray()
-	return nil
-}
-
-func (b *alterPartitionReassignmentsBlock) decode(pd packetDecoder) (err error) {
-	if b.replicas, err = pd.getCompactInt32Array(); err != nil {
-		return err
-	}
-	return nil
-}
-
-type AlterPartitionReassignmentsRequest struct {
-	TimeoutMs int32
-	blocks    map[string]map[int32]*alterPartitionReassignmentsBlock
-	Version   int16
-}
-
-func (r *AlterPartitionReassignmentsRequest) encode(pe packetEncoder) error {
-	pe.putInt32(r.TimeoutMs)
-
-	pe.putCompactArrayLength(len(r.blocks))
-
-	for topic, partitions := range r.blocks {
-		if err := pe.putCompactString(topic); err != nil {
-			return err
-		}
-		pe.putCompactArrayLength(len(partitions))
-		for partition, block := range partitions {
-			pe.putInt32(partition)
-			if err := block.encode(pe); err != nil {
-				return err
-			}
-		}
-		pe.putEmptyTaggedFieldArray()
-	}
-
-	pe.putEmptyTaggedFieldArray()
-
-	return nil
-}
-
-func (r *AlterPartitionReassignmentsRequest) decode(pd packetDecoder, version int16) (err error) {
-	r.Version = version
-
-	if r.TimeoutMs, err = pd.getInt32(); err != nil {
-		return err
-	}
-
-	topicCount, err := pd.getCompactArrayLength()
-	if err != nil {
-		return err
-	}
-	if topicCount > 0 {
-		r.blocks = make(map[string]map[int32]*alterPartitionReassignmentsBlock)
-		for i := 0; i < topicCount; i++ {
-			topic, err := pd.getCompactString()
-			if err != nil {
-				return err
-			}
-			partitionCount, err := pd.getCompactArrayLength()
-			if err != nil {
-				return err
-			}
-			r.blocks[topic] = make(map[int32]*alterPartitionReassignmentsBlock)
-			for j := 0; j < partitionCount; j++ {
-				partition, err := pd.getInt32()
-				if err != nil {
-					return err
-				}
-				block := &alterPartitionReassignmentsBlock{}
-				if err := block.decode(pd); err != nil {
-					return err
-				}
-				r.blocks[topic][partition] = block
-
-				if _, err := pd.getEmptyTaggedFieldArray(); err != nil {
-					return err
-				}
-			}
-			if _, err := pd.getEmptyTaggedFieldArray(); err != nil {
-				return err
-			}
-		}
-	}
-
-	if _, err := pd.getEmptyTaggedFieldArray(); err != nil {
-		return err
-	}
-
-	return
-}
-
-func (r *AlterPartitionReassignmentsRequest) key() int16 {
-	return 45
-}
-
-func (r *AlterPartitionReassignmentsRequest) version() int16 {
-	return r.Version
-}
-
-func (r *AlterPartitionReassignmentsRequest) headerVersion() int16 {
-	return 2
-}
-
-func (r *AlterPartitionReassignmentsRequest) requiredVersion() KafkaVersion {
-	return V2_4_0_0
-}
-
-func (r *AlterPartitionReassignmentsRequest) AddBlock(topic string, partitionID int32, replicas []int32) {
-	if r.blocks == nil {
-		r.blocks = make(map[string]map[int32]*alterPartitionReassignmentsBlock)
-	}
-
-	if r.blocks[topic] == nil {
-		r.blocks[topic] = make(map[int32]*alterPartitionReassignmentsBlock)
-	}
-
-	r.blocks[topic][partitionID] = &alterPartitionReassignmentsBlock{replicas}
-}

vendor/github.com/Shopify/sarama/alter_partition_reassignments_response.go

@@ -1,157 +0,0 @@
-package sarama
-
-type alterPartitionReassignmentsErrorBlock struct {
-	errorCode    KError
-	errorMessage *string
-}
-
-func (b *alterPartitionReassignmentsErrorBlock) encode(pe packetEncoder) error {
-	pe.putInt16(int16(b.errorCode))
-	if err := pe.putNullableCompactString(b.errorMessage); err != nil {
-		return err
-	}
-	pe.putEmptyTaggedFieldArray()
-
-	return nil
-}
-
-func (b *alterPartitionReassignmentsErrorBlock) decode(pd packetDecoder) (err error) {
-	errorCode, err := pd.getInt16()
-	if err != nil {
-		return err
-	}
-	b.errorCode = KError(errorCode)
-	b.errorMessage, err = pd.getCompactNullableString()
-
-	if _, err := pd.getEmptyTaggedFieldArray(); err != nil {
-		return err
-	}
-	return err
-}
-
-type AlterPartitionReassignmentsResponse struct {
-	Version        int16
-	ThrottleTimeMs int32
-	ErrorCode      KError
-	ErrorMessage   *string
-	Errors         map[string]map[int32]*alterPartitionReassignmentsErrorBlock
-}
-
-func (r *AlterPartitionReassignmentsResponse) AddError(topic string, partition int32, kerror KError, message *string) {
-	if r.Errors == nil {
-		r.Errors = make(map[string]map[int32]*alterPartitionReassignmentsErrorBlock)
-	}
-	partitions := r.Errors[topic]
-	if partitions == nil {
-		partitions = make(map[int32]*alterPartitionReassignmentsErrorBlock)
-		r.Errors[topic] = partitions
-	}
-
-	partitions[partition] = &alterPartitionReassignmentsErrorBlock{errorCode: kerror, errorMessage: message}
-}
-
-func (r *AlterPartitionReassignmentsResponse) encode(pe packetEncoder) error {
-	pe.putInt32(r.ThrottleTimeMs)
-	pe.putInt16(int16(r.ErrorCode))
-	if err := pe.putNullableCompactString(r.ErrorMessage); err != nil {
-		return err
-	}
-
-	pe.putCompactArrayLength(len(r.Errors))
-	for topic, partitions := range r.Errors {
-		if err := pe.putCompactString(topic); err != nil {
-			return err
-		}
-		pe.putCompactArrayLength(len(partitions))
-		for partition, block := range partitions {
-			pe.putInt32(partition)
-
-			if err := block.encode(pe); err != nil {
-				return err
-			}
-		}
-		pe.putEmptyTaggedFieldArray()
-	}
-
-	pe.putEmptyTaggedFieldArray()
-	return nil
-}
-
-func (r *AlterPartitionReassignmentsResponse) decode(pd packetDecoder, version int16) (err error) {
-	r.Version = version
-
-	if r.ThrottleTimeMs, err = pd.getInt32(); err != nil {
-		return err
-	}
-
-	kerr, err := pd.getInt16()
-	if err != nil {
-		return err
-	}
-
-	r.ErrorCode = KError(kerr)
-
-	if r.ErrorMessage, err = pd.getCompactNullableString(); err != nil {
-		return err
-	}
-
-	numTopics, err := pd.getCompactArrayLength()
-	if err != nil {
-		return err
-	}
-
-	if numTopics > 0 {
-		r.Errors = make(map[string]map[int32]*alterPartitionReassignmentsErrorBlock, numTopics)
-		for i := 0; i < numTopics; i++ {
-			topic, err := pd.getCompactString()
-			if err != nil {
-				return err
-			}
-
-			ongoingPartitionReassignments, err := pd.getCompactArrayLength()
-			if err != nil {
-				return err
-			}
-
-			r.Errors[topic] = make(map[int32]*alterPartitionReassignmentsErrorBlock, ongoingPartitionReassignments)
-
-			for j := 0; j < ongoingPartitionReassignments; j++ {
-				partition, err := pd.getInt32()
-				if err != nil {
-					return err
-				}
-				block := &alterPartitionReassignmentsErrorBlock{}
-				if err := block.decode(pd); err != nil {
-					return err
-				}
-
-				r.Errors[topic][partition] = block
-			}
-			if _, err = pd.getEmptyTaggedFieldArray(); err != nil {
-				return err
-			}
-		}
-	}
-
-	if _, err = pd.getEmptyTaggedFieldArray(); err != nil {
-		return err
-	}
-
-	return nil
-}
-
-func (r *AlterPartitionReassignmentsResponse) key() int16 {
-	return 45
-}
-
-func (r *AlterPartitionReassignmentsResponse) version() int16 {
-	return r.Version
-}
-
-func (r *AlterPartitionReassignmentsResponse) headerVersion() int16 {
-	return 1
-}
-
-func (r *AlterPartitionReassignmentsResponse) requiredVersion() KafkaVersion {
-	return V2_4_0_0
-}

vendor/github.com/Shopify/sarama/api_versions_request.go

@@ -1,29 +0,0 @@
-package sarama
-
-//ApiVersionsRequest ...
-type ApiVersionsRequest struct {
-}
-
-func (a *ApiVersionsRequest) encode(pe packetEncoder) error {
-	return nil
-}
-
-func (a *ApiVersionsRequest) decode(pd packetDecoder, version int16) (err error) {
-	return nil
-}
-
-func (a *ApiVersionsRequest) key() int16 {
-	return 18
-}
-
-func (a *ApiVersionsRequest) version() int16 {
-	return 0
-}
-
-func (a *ApiVersionsRequest) headerVersion() int16 {
-	return 1
-}
-
-func (a *ApiVersionsRequest) requiredVersion() KafkaVersion {
-	return V0_10_0_0
-}

vendor/github.com/Shopify/sarama/api_versions_response.go

@@ -1,93 +0,0 @@
-package sarama
-
-//ApiVersionsResponseBlock is an api version response block type
-type ApiVersionsResponseBlock struct {
-	ApiKey     int16
-	MinVersion int16
-	MaxVersion int16
-}
-
-func (b *ApiVersionsResponseBlock) encode(pe packetEncoder) error {
-	pe.putInt16(b.ApiKey)
-	pe.putInt16(b.MinVersion)
-	pe.putInt16(b.MaxVersion)
-	return nil
-}
-
-func (b *ApiVersionsResponseBlock) decode(pd packetDecoder) error {
-	var err error
-
-	if b.ApiKey, err = pd.getInt16(); err != nil {
-		return err
-	}
-
-	if b.MinVersion, err = pd.getInt16(); err != nil {
-		return err
-	}
-
-	if b.MaxVersion, err = pd.getInt16(); err != nil {
-		return err
-	}
-
-	return nil
-}
-
-//ApiVersionsResponse is an api version response type
-type ApiVersionsResponse struct {
-	Err         KError
-	ApiVersions []*ApiVersionsResponseBlock
-}
-
-func (r *ApiVersionsResponse) encode(pe packetEncoder) error {
-	pe.putInt16(int16(r.Err))
-	if err := pe.putArrayLength(len(r.ApiVersions)); err != nil {
-		return err
-	}
-	for _, apiVersion := range r.ApiVersions {
-		if err := apiVersion.encode(pe); err != nil {
-			return err
-		}
-	}
-	return nil
-}
-
-func (r *ApiVersionsResponse) decode(pd packetDecoder, version int16) error {
-	kerr, err := pd.getInt16()
-	if err != nil {
-		return err
-	}
-
-	r.Err = KError(kerr)
-
-	numBlocks, err := pd.getArrayLength()
-	if err != nil {
-		return err
-	}
-
-	r.ApiVersions = make([]*ApiVersionsResponseBlock, numBlocks)
-	for i := 0; i < numBlocks; i++ {
-		block := new(ApiVersionsResponseBlock)
-		if err := block.decode(pd); err != nil {
-			return err
-		}
-		r.ApiVersions[i] = block
-	}
-
-	return nil
-}
-
-func (r *ApiVersionsResponse) key() int16 {
-	return 18
-}
-
-func (r *ApiVersionsResponse) version() int16 {
-	return 0
-}
-
-func (a *ApiVersionsResponse) headerVersion() int16 {
-	return 0
-}
-
-func (r *ApiVersionsResponse) requiredVersion() KafkaVersion {
-	return V0_10_0_0
-}

vendor/github.com/Shopify/sarama/compress.go

@@ -1,194 +0,0 @@
-package sarama
-
-import (
-	"bytes"
-	"compress/gzip"
-	"fmt"
-	"sync"
-
-	snappy "github.com/eapache/go-xerial-snappy"
-	"github.com/pierrec/lz4"
-)
-
-var (
-	lz4WriterPool = sync.Pool{
-		New: func() interface{} {
-			return lz4.NewWriter(nil)
-		},
-	}
-
-	gzipWriterPool = sync.Pool{
-		New: func() interface{} {
-			return gzip.NewWriter(nil)
-		},
-	}
-	gzipWriterPoolForCompressionLevel1 = sync.Pool{
-		New: func() interface{} {
-			gz, err := gzip.NewWriterLevel(nil, 1)
-			if err != nil {
-				panic(err)
-			}
-			return gz
-		},
-	}
-	gzipWriterPoolForCompressionLevel2 = sync.Pool{
-		New: func() interface{} {
-			gz, err := gzip.NewWriterLevel(nil, 2)
-			if err != nil {
-				panic(err)
-			}
-			return gz
-		},
-	}
-	gzipWriterPoolForCompressionLevel3 = sync.Pool{
-		New: func() interface{} {
-			gz, err := gzip.NewWriterLevel(nil, 3)
-			if err != nil {
-				panic(err)
-			}
-			return gz
-		},
-	}
-	gzipWriterPoolForCompressionLevel4 = sync.Pool{
-		New: func() interface{} {
-			gz, err := gzip.NewWriterLevel(nil, 4)
-			if err != nil {
-				panic(err)
-			}
-			return gz
-		},
-	}
-	gzipWriterPoolForCompressionLevel5 = sync.Pool{
-		New: func() interface{} {
-			gz, err := gzip.NewWriterLevel(nil, 5)
-			if err != nil {
-				panic(err)
-			}
-			return gz
-		},
-	}
-	gzipWriterPoolForCompressionLevel6 = sync.Pool{
-		New: func() interface{} {
-			gz, err := gzip.NewWriterLevel(nil, 6)
-			if err != nil {
-				panic(err)
-			}
-			return gz
-		},
-	}
-	gzipWriterPoolForCompressionLevel7 = sync.Pool{
-		New: func() interface{} {
-			gz, err := gzip.NewWriterLevel(nil, 7)
-			if err != nil {
-				panic(err)
-			}
-			return gz
-		},
-	}
-	gzipWriterPoolForCompressionLevel8 = sync.Pool{
-		New: func() interface{} {
-			gz, err := gzip.NewWriterLevel(nil, 8)
-			if err != nil {
-				panic(err)
-			}
-			return gz
-		},
-	}
-	gzipWriterPoolForCompressionLevel9 = sync.Pool{
-		New: func() interface{} {
-			gz, err := gzip.NewWriterLevel(nil, 9)
-			if err != nil {
-				panic(err)
-			}
-			return gz
-		},
-	}
-)
-
-func compress(cc CompressionCodec, level int, data []byte) ([]byte, error) {
-	switch cc {
-	case CompressionNone:
-		return data, nil
-	case CompressionGZIP:
-		var (
-			err    error
-			buf    bytes.Buffer
-			writer *gzip.Writer
-		)
-
-		switch level {
-		case CompressionLevelDefault:
-			writer = gzipWriterPool.Get().(*gzip.Writer)
-			defer gzipWriterPool.Put(writer)
-			writer.Reset(&buf)
-		case 1:
-			writer = gzipWriterPoolForCompressionLevel1.Get().(*gzip.Writer)
-			defer gzipWriterPoolForCompressionLevel1.Put(writer)
-			writer.Reset(&buf)
-		case 2:
-			writer = gzipWriterPoolForCompressionLevel2.Get().(*gzip.Writer)
-			defer gzipWriterPoolForCompressionLevel2.Put(writer)
-			writer.Reset(&buf)
-		case 3:
-			writer = gzipWriterPoolForCompressionLevel3.Get().(*gzip.Writer)
-			defer gzipWriterPoolForCompressionLevel3.Put(writer)
-			writer.Reset(&buf)
-		case 4:
-			writer = gzipWriterPoolForCompressionLevel4.Get().(*gzip.Writer)
-			defer gzipWriterPoolForCompressionLevel4.Put(writer)
-			writer.Reset(&buf)
-		case 5:
-			writer = gzipWriterPoolForCompressionLevel5.Get().(*gzip.Writer)
-			defer gzipWriterPoolForCompressionLevel5.Put(writer)
-			writer.Reset(&buf)
-		case 6:
-			writer = gzipWriterPoolForCompressionLevel6.Get().(*gzip.Writer)
-			defer gzipWriterPoolForCompressionLevel6.Put(writer)
-			writer.Reset(&buf)
-		case 7:
-			writer = gzipWriterPoolForCompressionLevel7.Get().(*gzip.Writer)
-			defer gzipWriterPoolForCompressionLevel7.Put(writer)
-			writer.Reset(&buf)
-		case 8:
-			writer = gzipWriterPoolForCompressionLevel8.Get().(*gzip.Writer)
-			defer gzipWriterPoolForCompressionLevel8.Put(writer)
-			writer.Reset(&buf)
-		case 9:
-			writer = gzipWriterPoolForCompressionLevel9.Get().(*gzip.Writer)
-			defer gzipWriterPoolForCompressionLevel9.Put(writer)
-			writer.Reset(&buf)
-		default:
-			writer, err = gzip.NewWriterLevel(&buf, level)
-			if err != nil {
-				return nil, err
-			}
-		}
-		if _, err := writer.Write(data); err != nil {
-			return nil, err
-		}
-		if err := writer.Close(); err != nil {
-			return nil, err
-		}
-		return buf.Bytes(), nil
-	case CompressionSnappy:
-		return snappy.Encode(data), nil
-	case CompressionLZ4:
-		writer := lz4WriterPool.Get().(*lz4.Writer)
-		defer lz4WriterPool.Put(writer)
-
-		var buf bytes.Buffer
-		writer.Reset(&buf)
-
-		if _, err := writer.Write(data); err != nil {
-			return nil, err
-		}
-		if err := writer.Close(); err != nil {
-			return nil, err
-		}
-		return buf.Bytes(), nil
-	case CompressionZSTD:
-		return zstdCompress(nil, data)
-	default:
-		return nil, PacketEncodingError{fmt.Sprintf("unsupported compression codec (%d)", cc)}
-	}
-}

vendor/github.com/Shopify/sarama/config.go

@@ -1,749 +0,0 @@
-package sarama
-
-import (
-	"compress/gzip"
-	"crypto/tls"
-	"fmt"
-	"io/ioutil"
-	"net"
-	"regexp"
-	"time"
-
-	"github.com/rcrowley/go-metrics"
-	"golang.org/x/net/proxy"
-)
-
-const defaultClientID = "sarama"
-
-var validID = regexp.MustCompile(`\A[A-Za-z0-9._-]+\z`)
-
-// Config is used to pass multiple configuration options to Sarama's constructors.
-type Config struct {
-	// Admin is the namespace for ClusterAdmin properties used by the administrative Kafka client.
-	Admin struct {
-		Retry struct {
-			// The total number of times to retry sending (retriable) admin requests (default 5).
-			// Similar to the `retries` setting of the JVM AdminClientConfig.
-			Max int
-			// Backoff time between retries of a failed request (default 100ms)
-			Backoff time.Duration
-		}
-		// The maximum duration the administrative Kafka client will wait for ClusterAdmin operations,
-		// including topics, brokers, configurations and ACLs (defaults to 3 seconds).
-		Timeout time.Duration
-	}
-
-	// Net is the namespace for network-level properties used by the Broker, and
-	// shared by the Client/Producer/Consumer.
-	Net struct {
-		// How many outstanding requests a connection is allowed to have before
-		// sending on it blocks (default 5).
-		MaxOpenRequests int
-
-		// All three of the below configurations are similar to the
-		// `socket.timeout.ms` setting in JVM kafka. All of them default
-		// to 30 seconds.
-		DialTimeout  time.Duration // How long to wait for the initial connection.
-		ReadTimeout  time.Duration // How long to wait for a response.
-		WriteTimeout time.Duration // How long to wait for a transmit.
-
-		TLS struct {
-			// Whether or not to use TLS when connecting to the broker
-			// (defaults to false).
-			Enable bool
-			// The TLS configuration to use for secure connections if
-			// enabled (defaults to nil).
-			Config *tls.Config
-		}
-
-		// SASL based authentication with broker. While there are multiple SASL authentication methods
-		// the current implementation is limited to plaintext (SASL/PLAIN) authentication
-		SASL struct {
-			// Whether or not to use SASL authentication when connecting to the broker
-			// (defaults to false).
-			Enable bool
-			// SASLMechanism is the name of the enabled SASL mechanism.
-			// Possible values: OAUTHBEARER, PLAIN (defaults to PLAIN).
-			Mechanism SASLMechanism
-			// Version is the SASL Protocol Version to use
-			// Kafka > 1.x should use V1, except on Azure EventHub which use V0
-			Version int16
-			// Whether or not to send the Kafka SASL handshake first if enabled
-			// (defaults to true). You should only set this to false if you're using
-			// a non-Kafka SASL proxy.
-			Handshake bool
-			// AuthIdentity is an (optional) authorization identity (authzid) to
-			// use for SASL/PLAIN authentication (if different from User) when
-			// an authenticated user is permitted to act as the presented
-			// alternative user. See RFC4616 for details.
-			AuthIdentity string
-			// User is the authentication identity (authcid) to present for
-			// SASL/PLAIN or SASL/SCRAM authentication
-			User string
-			// Password for SASL/PLAIN authentication
-			Password string
-			// authz id used for SASL/SCRAM authentication
-			SCRAMAuthzID string
-			// SCRAMClientGeneratorFunc is a generator of a user provided implementation of a SCRAM
-			// client used to perform the SCRAM exchange with the server.
-			SCRAMClientGeneratorFunc func() SCRAMClient
-			// TokenProvider is a user-defined callback for generating
-			// access tokens for SASL/OAUTHBEARER auth. See the
-			// AccessTokenProvider interface docs for proper implementation
-			// guidelines.
-			TokenProvider AccessTokenProvider
-
-			GSSAPI GSSAPIConfig
-		}
-
-		// KeepAlive specifies the keep-alive period for an active network connection (defaults to 0).
-		// If zero or positive, keep-alives are enabled.
-		// If negative, keep-alives are disabled.
-		KeepAlive time.Duration
-
-		// LocalAddr is the local address to use when dialing an
-		// address. The address must be of a compatible type for the
-		// network being dialed.
-		// If nil, a local address is automatically chosen.
-		LocalAddr net.Addr
-
-		Proxy struct {
-			// Whether or not to use proxy when connecting to the broker
-			// (defaults to false).
-			Enable bool
-			// The proxy dialer to use enabled (defaults to nil).
-			Dialer proxy.Dialer
-		}
-	}
-
-	// Metadata is the namespace for metadata management properties used by the
-	// Client, and shared by the Producer/Consumer.
-	Metadata struct {
-		Retry struct {
-			// The total number of times to retry a metadata request when the
-			// cluster is in the middle of a leader election (default 3).
-			Max int
-			// How long to wait for leader election to occur before retrying
-			// (default 250ms). Similar to the JVM's `retry.backoff.ms`.
-			Backoff time.Duration
-			// Called to compute backoff time dynamically. Useful for implementing
-			// more sophisticated backoff strategies. This takes precedence over
-			// `Backoff` if set.
-			BackoffFunc func(retries, maxRetries int) time.Duration
-		}
-		// How frequently to refresh the cluster metadata in the background.
-		// Defaults to 10 minutes. Set to 0 to disable. Similar to
-		// `topic.metadata.refresh.interval.ms` in the JVM version.
-		RefreshFrequency time.Duration
-
-		// Whether to maintain a full set of metadata for all topics, or just
-		// the minimal set that has been necessary so far. The full set is simpler
-		// and usually more convenient, but can take up a substantial amount of
-		// memory if you have many topics and partitions. Defaults to true.
-		Full bool
-
-		// How long to wait for a successful metadata response.
-		// Disabled by default which means a metadata request against an unreachable
-		// cluster (all brokers are unreachable or unresponsive) can take up to
-		// `Net.[Dial|Read]Timeout * BrokerCount * (Metadata.Retry.Max + 1) + Metadata.Retry.Backoff * Metadata.Retry.Max`
-		// to fail.
-		Timeout time.Duration
-	}
-
-	// Producer is the namespace for configuration related to producing messages,
-	// used by the Producer.
-	Producer struct {
-		// The maximum permitted size of a message (defaults to 1000000). Should be
-		// set equal to or smaller than the broker's `message.max.bytes`.
-		MaxMessageBytes int
-		// The level of acknowledgement reliability needed from the broker (defaults
-		// to WaitForLocal). Equivalent to the `request.required.acks` setting of the
-		// JVM producer.
-		RequiredAcks RequiredAcks
-		// The maximum duration the broker will wait the receipt of the number of
-		// RequiredAcks (defaults to 10 seconds). This is only relevant when
-		// RequiredAcks is set to WaitForAll or a number > 1. Only supports
-		// millisecond resolution, nanoseconds will be truncated. Equivalent to
-		// the JVM producer's `request.timeout.ms` setting.
-		Timeout time.Duration
-		// The type of compression to use on messages (defaults to no compression).
-		// Similar to `compression.codec` setting of the JVM producer.
-		Compression CompressionCodec
-		// The level of compression to use on messages. The meaning depends
-		// on the actual compression type used and defaults to default compression
-		// level for the codec.
-		CompressionLevel int
-		// Generates partitioners for choosing the partition to send messages to
-		// (defaults to hashing the message key). Similar to the `partitioner.class`
-		// setting for the JVM producer.
-		Partitioner PartitionerConstructor
-		// If enabled, the producer will ensure that exactly one copy of each message is
-		// written.
-		Idempotent bool
-
-		// Return specifies what channels will be populated. If they are set to true,
-		// you must read from the respective channels to prevent deadlock. If,
-		// however, this config is used to create a `SyncProducer`, both must be set
-		// to true and you shall not read from the channels since the producer does
-		// this internally.
-		Return struct {
-			// If enabled, successfully delivered messages will be returned on the
-			// Successes channel (default disabled).
-			Successes bool
-
-			// If enabled, messages that failed to deliver will be returned on the
-			// Errors channel, including error (default enabled).
-			Errors bool
-		}
-
-		// The following config options control how often messages are batched up and
-		// sent to the broker. By default, messages are sent as fast as possible, and
-		// all messages received while the current batch is in-flight are placed
-		// into the subsequent batch.
-		Flush struct {
-			// The best-effort number of bytes needed to trigger a flush. Use the
-			// global sarama.MaxRequestSize to set a hard upper limit.
-			Bytes int
-			// The best-effort number of messages needed to trigger a flush. Use
-			// `MaxMessages` to set a hard upper limit.
-			Messages int
-			// The best-effort frequency of flushes. Equivalent to
-			// `queue.buffering.max.ms` setting of JVM producer.
-			Frequency time.Duration
-			// The maximum number of messages the producer will send in a single
-			// broker request. Defaults to 0 for unlimited. Similar to
-			// `queue.buffering.max.messages` in the JVM producer.
-			MaxMessages int
-		}
-
-		Retry struct {
-			// The total number of times to retry sending a message (default 3).
-			// Similar to the `message.send.max.retries` setting of the JVM producer.
-			Max int
-			// How long to wait for the cluster to settle between retries
-			// (default 100ms). Similar to the `retry.backoff.ms` setting of the
-			// JVM producer.
-			Backoff time.Duration
-			// Called to compute backoff time dynamically. Useful for implementing
-			// more sophisticated backoff strategies. This takes precedence over
-			// `Backoff` if set.
-			BackoffFunc func(retries, maxRetries int) time.Duration
-		}
-	}
-
-	// Consumer is the namespace for configuration related to consuming messages,
-	// used by the Consumer.
-	Consumer struct {
-
-		// Group is the namespace for configuring consumer group.
-		Group struct {
-			Session struct {
-				// The timeout used to detect consumer failures when using Kafka's group management facility.
-				// The consumer sends periodic heartbeats to indicate its liveness to the broker.
-				// If no heartbeats are received by the broker before the expiration of this session timeout,
-				// then the broker will remove this consumer from the group and initiate a rebalance.
-				// Note that the value must be in the allowable range as configured in the broker configuration
-				// by `group.min.session.timeout.ms` and `group.max.session.timeout.ms` (default 10s)
-				Timeout time.Duration
-			}
-			Heartbeat struct {
-				// The expected time between heartbeats to the consumer coordinator when using Kafka's group
-				// management facilities. Heartbeats are used to ensure that the consumer's session stays active and
-				// to facilitate rebalancing when new consumers join or leave the group.
-				// The value must be set lower than Consumer.Group.Session.Timeout, but typically should be set no
-				// higher than 1/3 of that value.
-				// It can be adjusted even lower to control the expected time for normal rebalances (default 3s)
-				Interval time.Duration
-			}
-			Rebalance struct {
-				// Strategy for allocating topic partitions to members (default BalanceStrategyRange)
-				Strategy BalanceStrategy
-				// The maximum allowed time for each worker to join the group once a rebalance has begun.
-				// This is basically a limit on the amount of time needed for all tasks to flush any pending
-				// data and commit offsets. If the timeout is exceeded, then the worker will be removed from
-				// the group, which will cause offset commit failures (default 60s).
-				Timeout time.Duration
-
-				Retry struct {
-					// When a new consumer joins a consumer group the set of consumers attempt to "rebalance"
-					// the load to assign partitions to each consumer. If the set of consumers changes while
-					// this assignment is taking place the rebalance will fail and retry. This setting controls
-					// the maximum number of attempts before giving up (default 4).
-					Max int
-					// Backoff time between retries during rebalance (default 2s)
-					Backoff time.Duration
-				}
-			}
-			Member struct {
-				// Custom metadata to include when joining the group. The user data for all joined members
-				// can be retrieved by sending a DescribeGroupRequest to the broker that is the
-				// coordinator for the group.
-				UserData []byte
-			}
-		}
-
-		Retry struct {
-			// How long to wait after a failing to read from a partition before
-			// trying again (default 2s).
-			Backoff time.Duration
-			// Called to compute backoff time dynamically. Useful for implementing
-			// more sophisticated backoff strategies. This takes precedence over
-			// `Backoff` if set.
-			BackoffFunc func(retries int) time.Duration
-		}
-
-		// Fetch is the namespace for controlling how many bytes are retrieved by any
-		// given request.
-		Fetch struct {
-			// The minimum number of message bytes to fetch in a request - the broker
-			// will wait until at least this many are available. The default is 1,
-			// as 0 causes the consumer to spin when no messages are available.
-			// Equivalent to the JVM's `fetch.min.bytes`.
-			Min int32
-			// The default number of message bytes to fetch from the broker in each
-			// request (default 1MB). This should be larger than the majority of
-			// your messages, or else the consumer will spend a lot of time
-			// negotiating sizes and not actually consuming. Similar to the JVM's
-			// `fetch.message.max.bytes`.
-			Default int32
-			// The maximum number of message bytes to fetch from the broker in a
-			// single request. Messages larger than this will return
-			// ErrMessageTooLarge and will not be consumable, so you must be sure
-			// this is at least as large as your largest message. Defaults to 0
-			// (no limit). Similar to the JVM's `fetch.message.max.bytes`. The
-			// global `sarama.MaxResponseSize` still applies.
-			Max int32
-		}
-		// The maximum amount of time the broker will wait for Consumer.Fetch.Min
-		// bytes to become available before it returns fewer than that anyways. The
-		// default is 250ms, since 0 causes the consumer to spin when no events are
-		// available. 100-500ms is a reasonable range for most cases. Kafka only
-		// supports precision up to milliseconds; nanoseconds will be truncated.
-		// Equivalent to the JVM's `fetch.wait.max.ms`.
-		MaxWaitTime time.Duration
-
-		// The maximum amount of time the consumer expects a message takes to
-		// process for the user. If writing to the Messages channel takes longer
-		// than this, that partition will stop fetching more messages until it
-		// can proceed again.
-		// Note that, since the Messages channel is buffered, the actual grace time is
-		// (MaxProcessingTime * ChannelBufferSize). Defaults to 100ms.
-		// If a message is not written to the Messages channel between two ticks
-		// of the expiryTicker then a timeout is detected.
-		// Using a ticker instead of a timer to detect timeouts should typically
-		// result in many fewer calls to Timer functions which may result in a
-		// significant performance improvement if many messages are being sent
-		// and timeouts are infrequent.
-		// The disadvantage of using a ticker instead of a timer is that
-		// timeouts will be less accurate. That is, the effective timeout could
-		// be between `MaxProcessingTime` and `2 * MaxProcessingTime`. For
-		// example, if `MaxProcessingTime` is 100ms then a delay of 180ms
-		// between two messages being sent may not be recognized as a timeout.
-		MaxProcessingTime time.Duration
-
-		// Return specifies what channels will be populated. If they are set to true,
-		// you must read from them to prevent deadlock.
-		Return struct {
-			// If enabled, any errors that occurred while consuming are returned on
-			// the Errors channel (default disabled).
-			Errors bool
-		}
-
-		// Offsets specifies configuration for how and when to commit consumed
-		// offsets. This currently requires the manual use of an OffsetManager
-		// but will eventually be automated.
-		Offsets struct {
-			// Deprecated: CommitInterval exists for historical compatibility
-			// and should not be used. Please use Consumer.Offsets.AutoCommit
-			CommitInterval time.Duration
-
-			// AutoCommit specifies configuration for commit messages automatically.
-			AutoCommit struct {
-				// Whether or not to auto-commit updated offsets back to the broker.
-				// (default enabled).
-				Enable bool
-
-				// How frequently to commit updated offsets. Ineffective unless
-				// auto-commit is enabled (default 1s)
-				Interval time.Duration
-			}
-
-			// The initial offset to use if no offset was previously committed.
-			// Should be OffsetNewest or OffsetOldest. Defaults to OffsetNewest.
-			Initial int64
-
-			// The retention duration for committed offsets. If zero, disabled
-			// (in which case the `offsets.retention.minutes` option on the
-			// broker will be used).  Kafka only supports precision up to
-			// milliseconds; nanoseconds will be truncated. Requires Kafka
-			// broker version 0.9.0 or later.
-			// (default is 0: disabled).
-			Retention time.Duration
-
-			Retry struct {
-				// The total number of times to retry failing commit
-				// requests during OffsetManager shutdown (default 3).
-				Max int
-			}
-		}
-
-		// IsolationLevel support 2 mode:
-		// 	- use `ReadUncommitted` (default) to consume and return all messages in message channel
-		//	- use `ReadCommitted` to hide messages that are part of an aborted transaction
-		IsolationLevel IsolationLevel
-	}
-
-	// A user-provided string sent with every request to the brokers for logging,
-	// debugging, and auditing purposes. Defaults to "sarama", but you should
-	// probably set it to something specific to your application.
-	ClientID string
-	// A rack identifier for this client. This can be any string value which
-	// indicates where this client is physically located.
-	// It corresponds with the broker config 'broker.rack'
-	RackID string
-	// The number of events to buffer in internal and external channels. This
-	// permits the producer and consumer to continue processing some messages
-	// in the background while user code is working, greatly improving throughput.
-	// Defaults to 256.
-	ChannelBufferSize int
-	// The version of Kafka that Sarama will assume it is running against.
-	// Defaults to the oldest supported stable version. Since Kafka provides
-	// backwards-compatibility, setting it to a version older than you have
-	// will not break anything, although it may prevent you from using the
-	// latest features. Setting it to a version greater than you are actually
-	// running may lead to random breakage.
-	Version KafkaVersion
-	// The registry to define metrics into.
-	// Defaults to a local registry.
-	// If you want to disable metrics gathering, set "metrics.UseNilMetrics" to "true"
-	// prior to starting Sarama.
-	// See Examples on how to use the metrics registry
-	MetricRegistry metrics.Registry
-}
-
-// NewConfig returns a new configuration instance with sane defaults.
-func NewConfig() *Config {
-	c := &Config{}
-
-	c.Admin.Retry.Max = 5
-	c.Admin.Retry.Backoff = 100 * time.Millisecond
-	c.Admin.Timeout = 3 * time.Second
-
-	c.Net.MaxOpenRequests = 5
-	c.Net.DialTimeout = 30 * time.Second
-	c.Net.ReadTimeout = 30 * time.Second
-	c.Net.WriteTimeout = 30 * time.Second
-	c.Net.SASL.Handshake = true
-	c.Net.SASL.Version = SASLHandshakeV0
-
-	c.Metadata.Retry.Max = 3
-	c.Metadata.Retry.Backoff = 250 * time.Millisecond
-	c.Metadata.RefreshFrequency = 10 * time.Minute
-	c.Metadata.Full = true
-
-	c.Producer.MaxMessageBytes = 1000000
-	c.Producer.RequiredAcks = WaitForLocal
-	c.Producer.Timeout = 10 * time.Second
-	c.Producer.Partitioner = NewHashPartitioner
-	c.Producer.Retry.Max = 3
-	c.Producer.Retry.Backoff = 100 * time.Millisecond
-	c.Producer.Return.Errors = true
-	c.Producer.CompressionLevel = CompressionLevelDefault
-
-	c.Consumer.Fetch.Min = 1
-	c.Consumer.Fetch.Default = 1024 * 1024
-	c.Consumer.Retry.Backoff = 2 * time.Second
-	c.Consumer.MaxWaitTime = 250 * time.Millisecond
-	c.Consumer.MaxProcessingTime = 100 * time.Millisecond
-	c.Consumer.Return.Errors = false
-	c.Consumer.Offsets.AutoCommit.Enable = true
-	c.Consumer.Offsets.AutoCommit.Interval = 1 * time.Second
-	c.Consumer.Offsets.Initial = OffsetNewest
-	c.Consumer.Offsets.Retry.Max = 3
-
-	c.Consumer.Group.Session.Timeout = 10 * time.Second
-	c.Consumer.Group.Heartbeat.Interval = 3 * time.Second
-	c.Consumer.Group.Rebalance.Strategy = BalanceStrategyRange
-	c.Consumer.Group.Rebalance.Timeout = 60 * time.Second
-	c.Consumer.Group.Rebalance.Retry.Max = 4
-	c.Consumer.Group.Rebalance.Retry.Backoff = 2 * time.Second
-
-	c.ClientID = defaultClientID
-	c.ChannelBufferSize = 256
-	c.Version = MinVersion
-	c.MetricRegistry = metrics.NewRegistry()
-
-	return c
-}
-
-// Validate checks a Config instance. It will return a
-// ConfigurationError if the specified values don't make sense.
-func (c *Config) Validate() error {
-	// some configuration values should be warned on but not fail completely, do those first
-	if !c.Net.TLS.Enable && c.Net.TLS.Config != nil {
-		Logger.Println("Net.TLS is disabled but a non-nil configuration was provided.")
-	}
-	if !c.Net.SASL.Enable {
-		if c.Net.SASL.User != "" {
-			Logger.Println("Net.SASL is disabled but a non-empty username was provided.")
-		}
-		if c.Net.SASL.Password != "" {
-			Logger.Println("Net.SASL is disabled but a non-empty password was provided.")
-		}
-	}
-	if c.Producer.RequiredAcks > 1 {
-		Logger.Println("Producer.RequiredAcks > 1 is deprecated and will raise an exception with kafka >= 0.8.2.0.")
-	}
-	if c.Producer.MaxMessageBytes >= int(MaxRequestSize) {
-		Logger.Println("Producer.MaxMessageBytes must be smaller than MaxRequestSize; it will be ignored.")
-	}
-	if c.Producer.Flush.Bytes >= int(MaxRequestSize) {
-		Logger.Println("Producer.Flush.Bytes must be smaller than MaxRequestSize; it will be ignored.")
-	}
-	if (c.Producer.Flush.Bytes > 0 || c.Producer.Flush.Messages > 0) && c.Producer.Flush.Frequency == 0 {
-		Logger.Println("Producer.Flush: Bytes or Messages are set, but Frequency is not; messages may not get flushed.")
-	}
-	if c.Producer.Timeout%time.Millisecond != 0 {
-		Logger.Println("Producer.Timeout only supports millisecond resolution; nanoseconds will be truncated.")
-	}
-	if c.Consumer.MaxWaitTime < 100*time.Millisecond {
-		Logger.Println("Consumer.MaxWaitTime is very low, which can cause high CPU and network usage. See documentation for details.")
-	}
-	if c.Consumer.MaxWaitTime%time.Millisecond != 0 {
-		Logger.Println("Consumer.MaxWaitTime only supports millisecond precision; nanoseconds will be truncated.")
-	}
-	if c.Consumer.Offsets.Retention%time.Millisecond != 0 {
-		Logger.Println("Consumer.Offsets.Retention only supports millisecond precision; nanoseconds will be truncated.")
-	}
-	if c.Consumer.Group.Session.Timeout%time.Millisecond != 0 {
-		Logger.Println("Consumer.Group.Session.Timeout only supports millisecond precision; nanoseconds will be truncated.")
-	}
-	if c.Consumer.Group.Heartbeat.Interval%time.Millisecond != 0 {
-		Logger.Println("Consumer.Group.Heartbeat.Interval only supports millisecond precision; nanoseconds will be truncated.")
-	}
-	if c.Consumer.Group.Rebalance.Timeout%time.Millisecond != 0 {
-		Logger.Println("Consumer.Group.Rebalance.Timeout only supports millisecond precision; nanoseconds will be truncated.")
-	}
-	if c.ClientID == defaultClientID {
-		Logger.Println("ClientID is the default of 'sarama', you should consider setting it to something application-specific.")
-	}
-
-	// validate Net values
-	switch {
-	case c.Net.MaxOpenRequests <= 0:
-		return ConfigurationError("Net.MaxOpenRequests must be > 0")
-	case c.Net.DialTimeout <= 0:
-		return ConfigurationError("Net.DialTimeout must be > 0")
-	case c.Net.ReadTimeout <= 0:
-		return ConfigurationError("Net.ReadTimeout must be > 0")
-	case c.Net.WriteTimeout <= 0:
-		return ConfigurationError("Net.WriteTimeout must be > 0")
-	case c.Net.SASL.Enable:
-		if c.Net.SASL.Mechanism == "" {
-			c.Net.SASL.Mechanism = SASLTypePlaintext
-		}
-
-		switch c.Net.SASL.Mechanism {
-		case SASLTypePlaintext:
-			if c.Net.SASL.User == "" {
-				return ConfigurationError("Net.SASL.User must not be empty when SASL is enabled")
-			}
-			if c.Net.SASL.Password == "" {
-				return ConfigurationError("Net.SASL.Password must not be empty when SASL is enabled")
-			}
-		case SASLTypeOAuth:
-			if c.Net.SASL.TokenProvider == nil {
-				return ConfigurationError("An AccessTokenProvider instance must be provided to Net.SASL.TokenProvider")
-			}
-		case SASLTypeSCRAMSHA256, SASLTypeSCRAMSHA512:
-			if c.Net.SASL.User == "" {
-				return ConfigurationError("Net.SASL.User must not be empty when SASL is enabled")
-			}
-			if c.Net.SASL.Password == "" {
-				return ConfigurationError("Net.SASL.Password must not be empty when SASL is enabled")
-			}
-			if c.Net.SASL.SCRAMClientGeneratorFunc == nil {
-				return ConfigurationError("A SCRAMClientGeneratorFunc function must be provided to Net.SASL.SCRAMClientGeneratorFunc")
-			}
-		case SASLTypeGSSAPI:
-			if c.Net.SASL.GSSAPI.ServiceName == "" {
-				return ConfigurationError("Net.SASL.GSSAPI.ServiceName must not be empty when GSS-API mechanism is used")
-			}
-
-			if c.Net.SASL.GSSAPI.AuthType == KRB5_USER_AUTH {
-				if c.Net.SASL.GSSAPI.Password == "" {
-					return ConfigurationError("Net.SASL.GSSAPI.Password must not be empty when GSS-API " +
-						"mechanism is used and Net.SASL.GSSAPI.AuthType = KRB5_USER_AUTH")
-				}
-			} else if c.Net.SASL.GSSAPI.AuthType == KRB5_KEYTAB_AUTH {
-				if c.Net.SASL.GSSAPI.KeyTabPath == "" {
-					return ConfigurationError("Net.SASL.GSSAPI.KeyTabPath must not be empty when GSS-API mechanism is used" +
-						" and  Net.SASL.GSSAPI.AuthType = KRB5_KEYTAB_AUTH")
-				}
-			} else {
-				return ConfigurationError("Net.SASL.GSSAPI.AuthType is invalid. Possible values are KRB5_USER_AUTH and KRB5_KEYTAB_AUTH")
-			}
-			if c.Net.SASL.GSSAPI.KerberosConfigPath == "" {
-				return ConfigurationError("Net.SASL.GSSAPI.KerberosConfigPath must not be empty when GSS-API mechanism is used")
-			}
-			if c.Net.SASL.GSSAPI.Username == "" {
-				return ConfigurationError("Net.SASL.GSSAPI.Username must not be empty when GSS-API mechanism is used")
-			}
-			if c.Net.SASL.GSSAPI.Realm == "" {
-				return ConfigurationError("Net.SASL.GSSAPI.Realm must not be empty when GSS-API mechanism is used")
-			}
-		default:
-			msg := fmt.Sprintf("The SASL mechanism configuration is invalid. Possible values are `%s`, `%s`, `%s`, `%s` and `%s`",
-				SASLTypeOAuth, SASLTypePlaintext, SASLTypeSCRAMSHA256, SASLTypeSCRAMSHA512, SASLTypeGSSAPI)
-			return ConfigurationError(msg)
-		}
-	}
-
-	// validate the Admin values
-	switch {
-	case c.Admin.Timeout <= 0:
-		return ConfigurationError("Admin.Timeout must be > 0")
-	}
-
-	// validate the Metadata values
-	switch {
-	case c.Metadata.Retry.Max < 0:
-		return ConfigurationError("Metadata.Retry.Max must be >= 0")
-	case c.Metadata.Retry.Backoff < 0:
-		return ConfigurationError("Metadata.Retry.Backoff must be >= 0")
-	case c.Metadata.RefreshFrequency < 0:
-		return ConfigurationError("Metadata.RefreshFrequency must be >= 0")
-	}
-
-	// validate the Producer values
-	switch {
-	case c.Producer.MaxMessageBytes <= 0:
-		return ConfigurationError("Producer.MaxMessageBytes must be > 0")
-	case c.Producer.RequiredAcks < -1:
-		return ConfigurationError("Producer.RequiredAcks must be >= -1")
-	case c.Producer.Timeout <= 0:
-		return ConfigurationError("Producer.Timeout must be > 0")
-	case c.Producer.Partitioner == nil:
-		return ConfigurationError("Producer.Partitioner must not be nil")
-	case c.Producer.Flush.Bytes < 0:
-		return ConfigurationError("Producer.Flush.Bytes must be >= 0")
-	case c.Producer.Flush.Messages < 0:
-		return ConfigurationError("Producer.Flush.Messages must be >= 0")
-	case c.Producer.Flush.Frequency < 0:
-		return ConfigurationError("Producer.Flush.Frequency must be >= 0")
-	case c.Producer.Flush.MaxMessages < 0:
-		return ConfigurationError("Producer.Flush.MaxMessages must be >= 0")
-	case c.Producer.Flush.MaxMessages > 0 && c.Producer.Flush.MaxMessages < c.Producer.Flush.Messages:
-		return ConfigurationError("Producer.Flush.MaxMessages must be >= Producer.Flush.Messages when set")
-	case c.Producer.Retry.Max < 0:
-		return ConfigurationError("Producer.Retry.Max must be >= 0")
-	case c.Producer.Retry.Backoff < 0:
-		return ConfigurationError("Producer.Retry.Backoff must be >= 0")
-	}
-
-	if c.Producer.Compression == CompressionLZ4 && !c.Version.IsAtLeast(V0_10_0_0) {
-		return ConfigurationError("lz4 compression requires Version >= V0_10_0_0")
-	}
-
-	if c.Producer.Compression == CompressionGZIP {
-		if c.Producer.CompressionLevel != CompressionLevelDefault {
-			if _, err := gzip.NewWriterLevel(ioutil.Discard, c.Producer.CompressionLevel); err != nil {
-				return ConfigurationError(fmt.Sprintf("gzip compression does not work with level %d: %v", c.Producer.CompressionLevel, err))
-			}
-		}
-	}
-
-	if c.Producer.Compression == CompressionZSTD && !c.Version.IsAtLeast(V2_1_0_0) {
-		return ConfigurationError("zstd compression requires Version >= V2_1_0_0")
-	}
-
-	if c.Producer.Idempotent {
-		if !c.Version.IsAtLeast(V0_11_0_0) {
-			return ConfigurationError("Idempotent producer requires Version >= V0_11_0_0")
-		}
-		if c.Producer.Retry.Max == 0 {
-			return ConfigurationError("Idempotent producer requires Producer.Retry.Max >= 1")
-		}
-		if c.Producer.RequiredAcks != WaitForAll {
-			return ConfigurationError("Idempotent producer requires Producer.RequiredAcks to be WaitForAll")
-		}
-		if c.Net.MaxOpenRequests > 1 {
-			return ConfigurationError("Idempotent producer requires Net.MaxOpenRequests to be 1")
-		}
-	}
-
-	// validate the Consumer values
-	switch {
-	case c.Consumer.Fetch.Min <= 0:
-		return ConfigurationError("Consumer.Fetch.Min must be > 0")
-	case c.Consumer.Fetch.Default <= 0:
-		return ConfigurationError("Consumer.Fetch.Default must be > 0")
-	case c.Consumer.Fetch.Max < 0:
-		return ConfigurationError("Consumer.Fetch.Max must be >= 0")
-	case c.Consumer.MaxWaitTime < 1*time.Millisecond:
-		return ConfigurationError("Consumer.MaxWaitTime must be >= 1ms")
-	case c.Consumer.MaxProcessingTime <= 0:
-		return ConfigurationError("Consumer.MaxProcessingTime must be > 0")
-	case c.Consumer.Retry.Backoff < 0:
-		return ConfigurationError("Consumer.Retry.Backoff must be >= 0")
-	case c.Consumer.Offsets.AutoCommit.Interval <= 0:
-		return ConfigurationError("Consumer.Offsets.AutoCommit.Interval must be > 0")
-	case c.Consumer.Offsets.Initial != OffsetOldest && c.Consumer.Offsets.Initial != OffsetNewest:
-		return ConfigurationError("Consumer.Offsets.Initial must be OffsetOldest or OffsetNewest")
-	case c.Consumer.Offsets.Retry.Max < 0:
-		return ConfigurationError("Consumer.Offsets.Retry.Max must be >= 0")
-	case c.Consumer.IsolationLevel != ReadUncommitted && c.Consumer.IsolationLevel != ReadCommitted:
-		return ConfigurationError("Consumer.IsolationLevel must be ReadUncommitted or ReadCommitted")
-	}
-
-	if c.Consumer.Offsets.CommitInterval != 0 {
-		Logger.Println("Deprecation warning: Consumer.Offsets.CommitInterval exists for historical compatibility" +
-			" and should not be used. Please use Consumer.Offsets.AutoCommit, the current value will be ignored")
-	}
-
-	// validate IsolationLevel
-	if c.Consumer.IsolationLevel == ReadCommitted && !c.Version.IsAtLeast(V0_11_0_0) {
-		return ConfigurationError("ReadCommitted requires Version >= V0_11_0_0")
-	}
-
-	// validate the Consumer Group values
-	switch {
-	case c.Consumer.Group.Session.Timeout <= 2*time.Millisecond:
-		return ConfigurationError("Consumer.Group.Session.Timeout must be >= 2ms")
-	case c.Consumer.Group.Heartbeat.Interval < 1*time.Millisecond:
-		return ConfigurationError("Consumer.Group.Heartbeat.Interval must be >= 1ms")
-	case c.Consumer.Group.Heartbeat.Interval >= c.Consumer.Group.Session.Timeout:
-		return ConfigurationError("Consumer.Group.Heartbeat.Interval must be < Consumer.Group.Session.Timeout")
-	case c.Consumer.Group.Rebalance.Strategy == nil:
-		return ConfigurationError("Consumer.Group.Rebalance.Strategy must not be empty")
-	case c.Consumer.Group.Rebalance.Timeout <= time.Millisecond:
-		return ConfigurationError("Consumer.Group.Rebalance.Timeout must be >= 1ms")
-	case c.Consumer.Group.Rebalance.Retry.Max < 0:
-		return ConfigurationError("Consumer.Group.Rebalance.Retry.Max must be >= 0")
-	case c.Consumer.Group.Rebalance.Retry.Backoff < 0:
-		return ConfigurationError("Consumer.Group.Rebalance.Retry.Backoff must be >= 0")
-	}
-
-	// validate misc shared values
-	switch {
-	case c.ChannelBufferSize < 0:
-		return ConfigurationError("ChannelBufferSize must be >= 0")
-	case !validID.MatchString(c.ClientID):
-		return ConfigurationError("ClientID is invalid")
-	}
-
-	return nil
-}
-
-func (c *Config) getDialer() proxy.Dialer {
-	if c.Net.Proxy.Enable {
-		Logger.Printf("using proxy %s", c.Net.Proxy.Dialer)
-		return c.Net.Proxy.Dialer
-	} else {
-		return &net.Dialer{
-			Timeout:   c.Net.DialTimeout,
-			KeepAlive: c.Net.KeepAlive,
-			LocalAddr: c.Net.LocalAddr,
-		}
-	}
-}

vendor/github.com/Shopify/sarama/config_resource_type.go

@@ -1,18 +0,0 @@
-package sarama
-
-// ConfigResourceType is a type for resources that have configs.
-type ConfigResourceType int8
-
-// Taken from:
-// https://github.com/apache/kafka/blob/ed7c071e07f1f90e4c2895582f61ca090ced3c42/clients/src/main/java/org/apache/kafka/common/config/ConfigResource.java#L32-L55
-
-const (
-	// UnknownResource constant type
-	UnknownResource ConfigResourceType = 0
-	// TopicResource constant type
-	TopicResource ConfigResourceType = 2
-	// BrokerResource constant type
-	BrokerResource ConfigResourceType = 4
-	// BrokerLoggerResource constant type
-	BrokerLoggerResource ConfigResourceType = 8
-)

vendor/github.com/Shopify/sarama/consumer.go

@@ -1,911 +0,0 @@
-package sarama
-
-import (
-	"errors"
-	"fmt"
-	"math"
-	"sync"
-	"sync/atomic"
-	"time"
-
-	"github.com/rcrowley/go-metrics"
-)
-
-// ConsumerMessage encapsulates a Kafka message returned by the consumer.
-type ConsumerMessage struct {
-	Headers        []*RecordHeader // only set if kafka is version 0.11+
-	Timestamp      time.Time       // only set if kafka is version 0.10+, inner message timestamp
-	BlockTimestamp time.Time       // only set if kafka is version 0.10+, outer (compressed) block timestamp
-
-	Key, Value []byte
-	Topic      string
-	Partition  int32
-	Offset     int64
-}
-
-// ConsumerError is what is provided to the user when an error occurs.
-// It wraps an error and includes the topic and partition.
-type ConsumerError struct {
-	Topic     string
-	Partition int32
-	Err       error
-}
-
-func (ce ConsumerError) Error() string {
-	return fmt.Sprintf("kafka: error while consuming %s/%d: %s", ce.Topic, ce.Partition, ce.Err)
-}
-
-// ConsumerErrors is a type that wraps a batch of errors and implements the Error interface.
-// It can be returned from the PartitionConsumer's Close methods to avoid the need to manually drain errors
-// when stopping.
-type ConsumerErrors []*ConsumerError
-
-func (ce ConsumerErrors) Error() string {
-	return fmt.Sprintf("kafka: %d errors while consuming", len(ce))
-}
-
-// Consumer manages PartitionConsumers which process Kafka messages from brokers. You MUST call Close()
-// on a consumer to avoid leaks, it will not be garbage-collected automatically when it passes out of
-// scope.
-type Consumer interface {
-	// Topics returns the set of available topics as retrieved from the cluster
-	// metadata. This method is the same as Client.Topics(), and is provided for
-	// convenience.
-	Topics() ([]string, error)
-
-	// Partitions returns the sorted list of all partition IDs for the given topic.
-	// This method is the same as Client.Partitions(), and is provided for convenience.
-	Partitions(topic string) ([]int32, error)
-
-	// ConsumePartition creates a PartitionConsumer on the given topic/partition with
-	// the given offset. It will return an error if this Consumer is already consuming
-	// on the given topic/partition. Offset can be a literal offset, or OffsetNewest
-	// or OffsetOldest
-	ConsumePartition(topic string, partition int32, offset int64) (PartitionConsumer, error)
-
-	// HighWaterMarks returns the current high water marks for each topic and partition.
-	// Consistency between partitions is not guaranteed since high water marks are updated separately.
-	HighWaterMarks() map[string]map[int32]int64
-
-	// Close shuts down the consumer. It must be called after all child
-	// PartitionConsumers have already been closed.
-	Close() error
-}
-
-type consumer struct {
-	conf            *Config
-	children        map[string]map[int32]*partitionConsumer
-	brokerConsumers map[*Broker]*brokerConsumer
-	client          Client
-	lock            sync.Mutex
-}
-
-// NewConsumer creates a new consumer using the given broker addresses and configuration.
-func NewConsumer(addrs []string, config *Config) (Consumer, error) {
-	client, err := NewClient(addrs, config)
-	if err != nil {
-		return nil, err
-	}
-	return newConsumer(client)
-}
-
-// NewConsumerFromClient creates a new consumer using the given client. It is still
-// necessary to call Close() on the underlying client when shutting down this consumer.
-func NewConsumerFromClient(client Client) (Consumer, error) {
-	// For clients passed in by the client, ensure we don't
-	// call Close() on it.
-	cli := &nopCloserClient{client}
-	return newConsumer(cli)
-}
-
-func newConsumer(client Client) (Consumer, error) {
-	// Check that we are not dealing with a closed Client before processing any other arguments
-	if client.Closed() {
-		return nil, ErrClosedClient
-	}
-
-	c := &consumer{
-		client:          client,
-		conf:            client.Config(),
-		children:        make(map[string]map[int32]*partitionConsumer),
-		brokerConsumers: make(map[*Broker]*brokerConsumer),
-	}
-
-	return c, nil
-}
-
-func (c *consumer) Close() error {
-	return c.client.Close()
-}
-
-func (c *consumer) Topics() ([]string, error) {
-	return c.client.Topics()
-}
-
-func (c *consumer) Partitions(topic string) ([]int32, error) {
-	return c.client.Partitions(topic)
-}
-
-func (c *consumer) ConsumePartition(topic string, partition int32, offset int64) (PartitionConsumer, error) {
-	child := &partitionConsumer{
-		consumer:  c,
-		conf:      c.conf,
-		topic:     topic,
-		partition: partition,
-		messages:  make(chan *ConsumerMessage, c.conf.ChannelBufferSize),
-		errors:    make(chan *ConsumerError, c.conf.ChannelBufferSize),
-		feeder:    make(chan *FetchResponse, 1),
-		trigger:   make(chan none, 1),
-		dying:     make(chan none),
-		fetchSize: c.conf.Consumer.Fetch.Default,
-	}
-
-	if err := child.chooseStartingOffset(offset); err != nil {
-		return nil, err
-	}
-
-	var leader *Broker
-	var err error
-	if leader, err = c.client.Leader(child.topic, child.partition); err != nil {
-		return nil, err
-	}
-
-	if err := c.addChild(child); err != nil {
-		return nil, err
-	}
-
-	go withRecover(child.dispatcher)
-	go withRecover(child.responseFeeder)
-
-	child.broker = c.refBrokerConsumer(leader)
-	child.broker.input <- child
-
-	return child, nil
-}
-
-func (c *consumer) HighWaterMarks() map[string]map[int32]int64 {
-	c.lock.Lock()
-	defer c.lock.Unlock()
-
-	hwms := make(map[string]map[int32]int64)
-	for topic, p := range c.children {
-		hwm := make(map[int32]int64, len(p))
-		for partition, pc := range p {
-			hwm[partition] = pc.HighWaterMarkOffset()
-		}
-		hwms[topic] = hwm
-	}
-
-	return hwms
-}
-
-func (c *consumer) addChild(child *partitionConsumer) error {
-	c.lock.Lock()
-	defer c.lock.Unlock()
-
-	topicChildren := c.children[child.topic]
-	if topicChildren == nil {
-		topicChildren = make(map[int32]*partitionConsumer)
-		c.children[child.topic] = topicChildren
-	}
-
-	if topicChildren[child.partition] != nil {
-		return ConfigurationError("That topic/partition is already being consumed")
-	}
-
-	topicChildren[child.partition] = child
-	return nil
-}
-
-func (c *consumer) removeChild(child *partitionConsumer) {
-	c.lock.Lock()
-	defer c.lock.Unlock()
-
-	delete(c.children[child.topic], child.partition)
-}
-
-func (c *consumer) refBrokerConsumer(broker *Broker) *brokerConsumer {
-	c.lock.Lock()
-	defer c.lock.Unlock()
-
-	bc := c.brokerConsumers[broker]
-	if bc == nil {
-		bc = c.newBrokerConsumer(broker)
-		c.brokerConsumers[broker] = bc
-	}
-
-	bc.refs++
-
-	return bc
-}
-
-func (c *consumer) unrefBrokerConsumer(brokerWorker *brokerConsumer) {
-	c.lock.Lock()
-	defer c.lock.Unlock()
-
-	brokerWorker.refs--
-
-	if brokerWorker.refs == 0 {
-		close(brokerWorker.input)
-		if c.brokerConsumers[brokerWorker.broker] == brokerWorker {
-			delete(c.brokerConsumers, brokerWorker.broker)
-		}
-	}
-}
-
-func (c *consumer) abandonBrokerConsumer(brokerWorker *brokerConsumer) {
-	c.lock.Lock()
-	defer c.lock.Unlock()
-
-	delete(c.brokerConsumers, brokerWorker.broker)
-}
-
-// PartitionConsumer
-
-// PartitionConsumer processes Kafka messages from a given topic and partition. You MUST call one of Close() or
-// AsyncClose() on a PartitionConsumer to avoid leaks; it will not be garbage-collected automatically when it passes out
-// of scope.
-//
-// The simplest way of using a PartitionConsumer is to loop over its Messages channel using a for/range
-// loop. The PartitionConsumer will only stop itself in one case: when the offset being consumed is reported
-// as out of range by the brokers. In this case you should decide what you want to do (try a different offset,
-// notify a human, etc) and handle it appropriately. For all other error cases, it will just keep retrying.
-// By default, it logs these errors to sarama.Logger; if you want to be notified directly of all errors, set
-// your config's Consumer.Return.Errors to true and read from the Errors channel, using a select statement
-// or a separate goroutine. Check out the Consumer examples to see implementations of these different approaches.
-//
-// To terminate such a for/range loop while the loop is executing, call AsyncClose. This will kick off the process of
-// consumer tear-down & return immediately. Continue to loop, servicing the Messages channel until the teardown process
-// AsyncClose initiated closes it (thus terminating the for/range loop). If you've already ceased reading Messages, call
-// Close; this will signal the PartitionConsumer's goroutines to begin shutting down (just like AsyncClose), but will
-// also drain the Messages channel, harvest all errors & return them once cleanup has completed.
-type PartitionConsumer interface {
-	// AsyncClose initiates a shutdown of the PartitionConsumer. This method will return immediately, after which you
-	// should continue to service the 'Messages' and 'Errors' channels until they are empty. It is required to call this
-	// function, or Close before a consumer object passes out of scope, as it will otherwise leak memory. You must call
-	// this before calling Close on the underlying client.
-	AsyncClose()
-
-	// Close stops the PartitionConsumer from fetching messages. It will initiate a shutdown just like AsyncClose, drain
-	// the Messages channel, harvest any errors & return them to the caller. Note that if you are continuing to service
-	// the Messages channel when this function is called, you will be competing with Close for messages; consider
-	// calling AsyncClose, instead. It is required to call this function (or AsyncClose) before a consumer object passes
-	// out of scope, as it will otherwise leak memory. You must call this before calling Close on the underlying client.
-	Close() error
-
-	// Messages returns the read channel for the messages that are returned by
-	// the broker.
-	Messages() <-chan *ConsumerMessage
-
-	// Errors returns a read channel of errors that occurred during consuming, if
-	// enabled. By default, errors are logged and not returned over this channel.
-	// If you want to implement any custom error handling, set your config's
-	// Consumer.Return.Errors setting to true, and read from this channel.
-	Errors() <-chan *ConsumerError
-
-	// HighWaterMarkOffset returns the high water mark offset of the partition,
-	// i.e. the offset that will be used for the next message that will be produced.
-	// You can use this to determine how far behind the processing is.
-	HighWaterMarkOffset() int64
-}
-
-type partitionConsumer struct {
-	highWaterMarkOffset int64 // must be at the top of the struct because https://golang.org/pkg/sync/atomic/#pkg-note-BUG
-
-	consumer *consumer
-	conf     *Config
-	broker   *brokerConsumer
-	messages chan *ConsumerMessage
-	errors   chan *ConsumerError
-	feeder   chan *FetchResponse
-
-	trigger, dying chan none
-	closeOnce      sync.Once
-	topic          string
-	partition      int32
-	responseResult error
-	fetchSize      int32
-	offset         int64
-	retries        int32
-}
-
-var errTimedOut = errors.New("timed out feeding messages to the user") // not user-facing
-
-func (child *partitionConsumer) sendError(err error) {
-	cErr := &ConsumerError{
-		Topic:     child.topic,
-		Partition: child.partition,
-		Err:       err,
-	}
-
-	if child.conf.Consumer.Return.Errors {
-		child.errors <- cErr
-	} else {
-		Logger.Println(cErr)
-	}
-}
-
-func (child *partitionConsumer) computeBackoff() time.Duration {
-	if child.conf.Consumer.Retry.BackoffFunc != nil {
-		retries := atomic.AddInt32(&child.retries, 1)
-		return child.conf.Consumer.Retry.BackoffFunc(int(retries))
-	}
-	return child.conf.Consumer.Retry.Backoff
-}
-
-func (child *partitionConsumer) dispatcher() {
-	for range child.trigger {
-		select {
-		case <-child.dying:
-			close(child.trigger)
-		case <-time.After(child.computeBackoff()):
-			if child.broker != nil {
-				child.consumer.unrefBrokerConsumer(child.broker)
-				child.broker = nil
-			}
-
-			Logger.Printf("consumer/%s/%d finding new broker\n", child.topic, child.partition)
-			if err := child.dispatch(); err != nil {
-				child.sendError(err)
-				child.trigger <- none{}
-			}
-		}
-	}
-
-	if child.broker != nil {
-		child.consumer.unrefBrokerConsumer(child.broker)
-	}
-	child.consumer.removeChild(child)
-	close(child.feeder)
-}
-
-func (child *partitionConsumer) dispatch() error {
-	if err := child.consumer.client.RefreshMetadata(child.topic); err != nil {
-		return err
-	}
-
-	var leader *Broker
-	var err error
-	if leader, err = child.consumer.client.Leader(child.topic, child.partition); err != nil {
-		return err
-	}
-
-	child.broker = child.consumer.refBrokerConsumer(leader)
-
-	child.broker.input <- child
-
-	return nil
-}
-
-func (child *partitionConsumer) chooseStartingOffset(offset int64) error {
-	newestOffset, err := child.consumer.client.GetOffset(child.topic, child.partition, OffsetNewest)
-	if err != nil {
-		return err
-	}
-	oldestOffset, err := child.consumer.client.GetOffset(child.topic, child.partition, OffsetOldest)
-	if err != nil {
-		return err
-	}
-
-	switch {
-	case offset == OffsetNewest:
-		child.offset = newestOffset
-	case offset == OffsetOldest:
-		child.offset = oldestOffset
-	case offset >= oldestOffset && offset <= newestOffset:
-		child.offset = offset
-	default:
-		return ErrOffsetOutOfRange
-	}
-
-	return nil
-}
-
-func (child *partitionConsumer) Messages() <-chan *ConsumerMessage {
-	return child.messages
-}
-
-func (child *partitionConsumer) Errors() <-chan *ConsumerError {
-	return child.errors
-}
-
-func (child *partitionConsumer) AsyncClose() {
-	// this triggers whatever broker owns this child to abandon it and close its trigger channel, which causes
-	// the dispatcher to exit its loop, which removes it from the consumer then closes its 'messages' and
-	// 'errors' channel (alternatively, if the child is already at the dispatcher for some reason, that will
-	// also just close itself)
-	child.closeOnce.Do(func() {
-		close(child.dying)
-	})
-}
-
-func (child *partitionConsumer) Close() error {
-	child.AsyncClose()
-
-	var errors ConsumerErrors
-	for err := range child.errors {
-		errors = append(errors, err)
-	}
-
-	if len(errors) > 0 {
-		return errors
-	}
-	return nil
-}
-
-func (child *partitionConsumer) HighWaterMarkOffset() int64 {
-	return atomic.LoadInt64(&child.highWaterMarkOffset)
-}
-
-func (child *partitionConsumer) responseFeeder() {
-	var msgs []*ConsumerMessage
-	expiryTicker := time.NewTicker(child.conf.Consumer.MaxProcessingTime)
-	firstAttempt := true
-
-feederLoop:
-	for response := range child.feeder {
-		msgs, child.responseResult = child.parseResponse(response)
-
-		if child.responseResult == nil {
-			atomic.StoreInt32(&child.retries, 0)
-		}
-
-		for i, msg := range msgs {
-		messageSelect:
-			select {
-			case <-child.dying:
-				child.broker.acks.Done()
-				continue feederLoop
-			case child.messages <- msg:
-				firstAttempt = true
-			case <-expiryTicker.C:
-				if !firstAttempt {
-					child.responseResult = errTimedOut
-					child.broker.acks.Done()
-				remainingLoop:
-					for _, msg = range msgs[i:] {
-						select {
-						case child.messages <- msg:
-						case <-child.dying:
-							break remainingLoop
-						}
-					}
-					child.broker.input <- child
-					continue feederLoop
-				} else {
-					// current message has not been sent, return to select
-					// statement
-					firstAttempt = false
-					goto messageSelect
-				}
-			}
-		}
-
-		child.broker.acks.Done()
-	}
-
-	expiryTicker.Stop()
-	close(child.messages)
-	close(child.errors)
-}
-
-func (child *partitionConsumer) parseMessages(msgSet *MessageSet) ([]*ConsumerMessage, error) {
-	var messages []*ConsumerMessage
-	for _, msgBlock := range msgSet.Messages {
-		for _, msg := range msgBlock.Messages() {
-			offset := msg.Offset
-			timestamp := msg.Msg.Timestamp
-			if msg.Msg.Version >= 1 {
-				baseOffset := msgBlock.Offset - msgBlock.Messages()[len(msgBlock.Messages())-1].Offset
-				offset += baseOffset
-				if msg.Msg.LogAppendTime {
-					timestamp = msgBlock.Msg.Timestamp
-				}
-			}
-			if offset < child.offset {
-				continue
-			}
-			messages = append(messages, &ConsumerMessage{
-				Topic:          child.topic,
-				Partition:      child.partition,
-				Key:            msg.Msg.Key,
-				Value:          msg.Msg.Value,
-				Offset:         offset,
-				Timestamp:      timestamp,
-				BlockTimestamp: msgBlock.Msg.Timestamp,
-			})
-			child.offset = offset + 1
-		}
-	}
-	if len(messages) == 0 {
-		child.offset++
-	}
-	return messages, nil
-}
-
-func (child *partitionConsumer) parseRecords(batch *RecordBatch) ([]*ConsumerMessage, error) {
-	messages := make([]*ConsumerMessage, 0, len(batch.Records))
-
-	for _, rec := range batch.Records {
-		offset := batch.FirstOffset + rec.OffsetDelta
-		if offset < child.offset {
-			continue
-		}
-		timestamp := batch.FirstTimestamp.Add(rec.TimestampDelta)
-		if batch.LogAppendTime {
-			timestamp = batch.MaxTimestamp
-		}
-		messages = append(messages, &ConsumerMessage{
-			Topic:     child.topic,
-			Partition: child.partition,
-			Key:       rec.Key,
-			Value:     rec.Value,
-			Offset:    offset,
-			Timestamp: timestamp,
-			Headers:   rec.Headers,
-		})
-		child.offset = offset + 1
-	}
-	if len(messages) == 0 {
-		child.offset++
-	}
-	return messages, nil
-}
-
-func (child *partitionConsumer) parseResponse(response *FetchResponse) ([]*ConsumerMessage, error) {
-	var (
-		metricRegistry          = child.conf.MetricRegistry
-		consumerBatchSizeMetric metrics.Histogram
-	)
-
-	if metricRegistry != nil {
-		consumerBatchSizeMetric = getOrRegisterHistogram("consumer-batch-size", metricRegistry)
-	}
-
-	// If request was throttled and empty we log and return without error
-	if response.ThrottleTime != time.Duration(0) && len(response.Blocks) == 0 {
-		Logger.Printf(
-			"consumer/broker/%d FetchResponse throttled %v\n",
-			child.broker.broker.ID(), response.ThrottleTime)
-		return nil, nil
-	}
-
-	block := response.GetBlock(child.topic, child.partition)
-	if block == nil {
-		return nil, ErrIncompleteResponse
-	}
-
-	if block.Err != ErrNoError {
-		return nil, block.Err
-	}
-
-	nRecs, err := block.numRecords()
-	if err != nil {
-		return nil, err
-	}
-
-	consumerBatchSizeMetric.Update(int64(nRecs))
-
-	if nRecs == 0 {
-		partialTrailingMessage, err := block.isPartial()
-		if err != nil {
-			return nil, err
-		}
-		// We got no messages. If we got a trailing one then we need to ask for more data.
-		// Otherwise we just poll again and wait for one to be produced...
-		if partialTrailingMessage {
-			if child.conf.Consumer.Fetch.Max > 0 && child.fetchSize == child.conf.Consumer.Fetch.Max {
-				// we can't ask for more data, we've hit the configured limit
-				child.sendError(ErrMessageTooLarge)
-				child.offset++ // skip this one so we can keep processing future messages
-			} else {
-				child.fetchSize *= 2
-				// check int32 overflow
-				if child.fetchSize < 0 {
-					child.fetchSize = math.MaxInt32
-				}
-				if child.conf.Consumer.Fetch.Max > 0 && child.fetchSize > child.conf.Consumer.Fetch.Max {
-					child.fetchSize = child.conf.Consumer.Fetch.Max
-				}
-			}
-		}
-
-		return nil, nil
-	}
-
-	// we got messages, reset our fetch size in case it was increased for a previous request
-	child.fetchSize = child.conf.Consumer.Fetch.Default
-	atomic.StoreInt64(&child.highWaterMarkOffset, block.HighWaterMarkOffset)
-
-	// abortedProducerIDs contains producerID which message should be ignored as uncommitted
-	// - producerID are added when the partitionConsumer iterate over the offset at which an aborted transaction begins (abortedTransaction.FirstOffset)
-	// - producerID are removed when partitionConsumer iterate over an aborted controlRecord, meaning the aborted transaction for this producer is over
-	abortedProducerIDs := make(map[int64]struct{}, len(block.AbortedTransactions))
-	abortedTransactions := block.getAbortedTransactions()
-
-	messages := []*ConsumerMessage{}
-	for _, records := range block.RecordsSet {
-		switch records.recordsType {
-		case legacyRecords:
-			messageSetMessages, err := child.parseMessages(records.MsgSet)
-			if err != nil {
-				return nil, err
-			}
-
-			messages = append(messages, messageSetMessages...)
-		case defaultRecords:
-			// Consume remaining abortedTransaction up to last offset of current batch
-			for _, txn := range abortedTransactions {
-				if txn.FirstOffset > records.RecordBatch.LastOffset() {
-					break
-				}
-				abortedProducerIDs[txn.ProducerID] = struct{}{}
-				// Pop abortedTransactions so that we never add it again
-				abortedTransactions = abortedTransactions[1:]
-			}
-
-			recordBatchMessages, err := child.parseRecords(records.RecordBatch)
-			if err != nil {
-				return nil, err
-			}
-
-			// Parse and commit offset but do not expose messages that are:
-			// - control records
-			// - part of an aborted transaction when set to `ReadCommitted`
-
-			// control record
-			isControl, err := records.isControl()
-			if err != nil {
-				// I don't know why there is this continue in case of error to begin with
-				// Safe bet is to ignore control messages if ReadUncommitted
-				// and block on them in case of error and ReadCommitted
-				if child.conf.Consumer.IsolationLevel == ReadCommitted {
-					return nil, err
-				}
-				continue
-			}
-			if isControl {
-				controlRecord, err := records.getControlRecord()
-				if err != nil {
-					return nil, err
-				}
-
-				if controlRecord.Type == ControlRecordAbort {
-					delete(abortedProducerIDs, records.RecordBatch.ProducerID)
-				}
-				continue
-			}
-
-			// filter aborted transactions
-			if child.conf.Consumer.IsolationLevel == ReadCommitted {
-				_, isAborted := abortedProducerIDs[records.RecordBatch.ProducerID]
-				if records.RecordBatch.IsTransactional && isAborted {
-					continue
-				}
-			}
-
-			messages = append(messages, recordBatchMessages...)
-		default:
-			return nil, fmt.Errorf("unknown records type: %v", records.recordsType)
-		}
-	}
-
-	return messages, nil
-}
-
-type brokerConsumer struct {
-	consumer         *consumer
-	broker           *Broker
-	input            chan *partitionConsumer
-	newSubscriptions chan []*partitionConsumer
-	subscriptions    map[*partitionConsumer]none
-	wait             chan none
-	acks             sync.WaitGroup
-	refs             int
-}
-
-func (c *consumer) newBrokerConsumer(broker *Broker) *brokerConsumer {
-	bc := &brokerConsumer{
-		consumer:         c,
-		broker:           broker,
-		input:            make(chan *partitionConsumer),
-		newSubscriptions: make(chan []*partitionConsumer),
-		wait:             make(chan none),
-		subscriptions:    make(map[*partitionConsumer]none),
-		refs:             0,
-	}
-
-	go withRecover(bc.subscriptionManager)
-	go withRecover(bc.subscriptionConsumer)
-
-	return bc
-}
-
-// The subscriptionManager constantly accepts new subscriptions on `input` (even when the main subscriptionConsumer
-// goroutine is in the middle of a network request) and batches it up. The main worker goroutine picks
-// up a batch of new subscriptions between every network request by reading from `newSubscriptions`, so we give
-// it nil if no new subscriptions are available. We also write to `wait` only when new subscriptions is available,
-// so the main goroutine can block waiting for work if it has none.
-func (bc *brokerConsumer) subscriptionManager() {
-	var buffer []*partitionConsumer
-
-	for {
-		if len(buffer) > 0 {
-			select {
-			case event, ok := <-bc.input:
-				if !ok {
-					goto done
-				}
-				buffer = append(buffer, event)
-			case bc.newSubscriptions <- buffer:
-				buffer = nil
-			case bc.wait <- none{}:
-			}
-		} else {
-			select {
-			case event, ok := <-bc.input:
-				if !ok {
-					goto done
-				}
-				buffer = append(buffer, event)
-			case bc.newSubscriptions <- nil:
-			}
-		}
-	}
-
-done:
-	close(bc.wait)
-	if len(buffer) > 0 {
-		bc.newSubscriptions <- buffer
-	}
-	close(bc.newSubscriptions)
-}
-
-//subscriptionConsumer ensures we will get nil right away if no new subscriptions is available
-func (bc *brokerConsumer) subscriptionConsumer() {
-	<-bc.wait // wait for our first piece of work
-
-	for newSubscriptions := range bc.newSubscriptions {
-		bc.updateSubscriptions(newSubscriptions)
-
-		if len(bc.subscriptions) == 0 {
-			// We're about to be shut down or we're about to receive more subscriptions.
-			// Either way, the signal just hasn't propagated to our goroutine yet.
-			<-bc.wait
-			continue
-		}
-
-		response, err := bc.fetchNewMessages()
-
-		if err != nil {
-			Logger.Printf("consumer/broker/%d disconnecting due to error processing FetchRequest: %s\n", bc.broker.ID(), err)
-			bc.abort(err)
-			return
-		}
-
-		bc.acks.Add(len(bc.subscriptions))
-		for child := range bc.subscriptions {
-			child.feeder <- response
-		}
-		bc.acks.Wait()
-		bc.handleResponses()
-	}
-}
-
-func (bc *brokerConsumer) updateSubscriptions(newSubscriptions []*partitionConsumer) {
-	for _, child := range newSubscriptions {
-		bc.subscriptions[child] = none{}
-		Logger.Printf("consumer/broker/%d added subscription to %s/%d\n", bc.broker.ID(), child.topic, child.partition)
-	}
-
-	for child := range bc.subscriptions {
-		select {
-		case <-child.dying:
-			Logger.Printf("consumer/broker/%d closed dead subscription to %s/%d\n", bc.broker.ID(), child.topic, child.partition)
-			close(child.trigger)
-			delete(bc.subscriptions, child)
-		default:
-			// no-op
-		}
-	}
-}
-
-//handleResponses handles the response codes left for us by our subscriptions, and abandons ones that have been closed
-func (bc *brokerConsumer) handleResponses() {
-	for child := range bc.subscriptions {
-		result := child.responseResult
-		child.responseResult = nil
-
-		switch result {
-		case nil:
-			// no-op
-		case errTimedOut:
-			Logger.Printf("consumer/broker/%d abandoned subscription to %s/%d because consuming was taking too long\n",
-				bc.broker.ID(), child.topic, child.partition)
-			delete(bc.subscriptions, child)
-		case ErrOffsetOutOfRange:
-			// there's no point in retrying this it will just fail the same way again
-			// shut it down and force the user to choose what to do
-			child.sendError(result)
-			Logger.Printf("consumer/%s/%d shutting down because %s\n", child.topic, child.partition, result)
-			close(child.trigger)
-			delete(bc.subscriptions, child)
-		case ErrUnknownTopicOrPartition, ErrNotLeaderForPartition, ErrLeaderNotAvailable, ErrReplicaNotAvailable:
-			// not an error, but does need redispatching
-			Logger.Printf("consumer/broker/%d abandoned subscription to %s/%d because %s\n",
-				bc.broker.ID(), child.topic, child.partition, result)
-			child.trigger <- none{}
-			delete(bc.subscriptions, child)
-		default:
-			// dunno, tell the user and try redispatching
-			child.sendError(result)
-			Logger.Printf("consumer/broker/%d abandoned subscription to %s/%d because %s\n",
-				bc.broker.ID(), child.topic, child.partition, result)
-			child.trigger <- none{}
-			delete(bc.subscriptions, child)
-		}
-	}
-}
-
-func (bc *brokerConsumer) abort(err error) {
-	bc.consumer.abandonBrokerConsumer(bc)
-	_ = bc.broker.Close() // we don't care about the error this might return, we already have one
-
-	for child := range bc.subscriptions {
-		child.sendError(err)
-		child.trigger <- none{}
-	}
-
-	for newSubscriptions := range bc.newSubscriptions {
-		if len(newSubscriptions) == 0 {
-			<-bc.wait
-			continue
-		}
-		for _, child := range newSubscriptions {
-			child.sendError(err)
-			child.trigger <- none{}
-		}
-	}
-}
-
-func (bc *brokerConsumer) fetchNewMessages() (*FetchResponse, error) {
-	request := &FetchRequest{
-		MinBytes:    bc.consumer.conf.Consumer.Fetch.Min,
-		MaxWaitTime: int32(bc.consumer.conf.Consumer.MaxWaitTime / time.Millisecond),
-	}
-	if bc.consumer.conf.Version.IsAtLeast(V0_9_0_0) {
-		request.Version = 1
-	}
-	if bc.consumer.conf.Version.IsAtLeast(V0_10_0_0) {
-		request.Version = 2
-	}
-	if bc.consumer.conf.Version.IsAtLeast(V0_10_1_0) {
-		request.Version = 3
-		request.MaxBytes = MaxResponseSize
-	}
-	if bc.consumer.conf.Version.IsAtLeast(V0_11_0_0) {
-		request.Version = 4
-		request.Isolation = bc.consumer.conf.Consumer.IsolationLevel
-	}
-	if bc.consumer.conf.Version.IsAtLeast(V1_1_0_0) {
-		request.Version = 7
-		// We do not currently implement KIP-227 FetchSessions. Setting the id to 0
-		// and the epoch to -1 tells the broker not to generate as session ID we're going
-		// to just ignore anyway.
-		request.SessionID = 0
-		request.SessionEpoch = -1
-	}
-	if bc.consumer.conf.Version.IsAtLeast(V2_1_0_0) {
-		request.Version = 10
-	}
-	if bc.consumer.conf.Version.IsAtLeast(V2_3_0_0) {
-		request.Version = 11
-		request.RackID = bc.consumer.conf.RackID
-	}
-
-	for child := range bc.subscriptions {
-		request.AddBlock(child.topic, child.partition, child.offset, child.fetchSize)
-	}
-
-	return bc.broker.Fetch(request)
-}

vendor/github.com/Shopify/sarama/consumer_group.go

@@ -1,867 +0,0 @@
-package sarama
-
-import (
-	"context"
-	"errors"
-	"fmt"
-	"sort"
-	"sync"
-	"time"
-)
-
-// ErrClosedConsumerGroup is the error returned when a method is called on a consumer group that has been closed.
-var ErrClosedConsumerGroup = errors.New("kafka: tried to use a consumer group that was closed")
-
-// ConsumerGroup is responsible for dividing up processing of topics and partitions
-// over a collection of processes (the members of the consumer group).
-type ConsumerGroup interface {
-	// Consume joins a cluster of consumers for a given list of topics and
-	// starts a blocking ConsumerGroupSession through the ConsumerGroupHandler.
-	//
-	// The life-cycle of a session is represented by the following steps:
-	//
-	// 1. The consumers join the group (as explained in https://kafka.apache.org/documentation/#intro_consumers)
-	//    and is assigned their "fair share" of partitions, aka 'claims'.
-	// 2. Before processing starts, the handler's Setup() hook is called to notify the user
-	//    of the claims and allow any necessary preparation or alteration of state.
-	// 3. For each of the assigned claims the handler's ConsumeClaim() function is then called
-	//    in a separate goroutine which requires it to be thread-safe. Any state must be carefully protected
-	//    from concurrent reads/writes.
-	// 4. The session will persist until one of the ConsumeClaim() functions exits. This can be either when the
-	//    parent context is cancelled or when a server-side rebalance cycle is initiated.
-	// 5. Once all the ConsumeClaim() loops have exited, the handler's Cleanup() hook is called
-	//    to allow the user to perform any final tasks before a rebalance.
-	// 6. Finally, marked offsets are committed one last time before claims are released.
-	//
-	// Please note, that once a rebalance is triggered, sessions must be completed within
-	// Config.Consumer.Group.Rebalance.Timeout. This means that ConsumeClaim() functions must exit
-	// as quickly as possible to allow time for Cleanup() and the final offset commit. If the timeout
-	// is exceeded, the consumer will be removed from the group by Kafka, which will cause offset
-	// commit failures.
-	// This method should be called inside an infinite loop, when a
-	// server-side rebalance happens, the consumer session will need to be
-	// recreated to get the new claims.
-	Consume(ctx context.Context, topics []string, handler ConsumerGroupHandler) error
-
-	// Errors returns a read channel of errors that occurred during the consumer life-cycle.
-	// By default, errors are logged and not returned over this channel.
-	// If you want to implement any custom error handling, set your config's
-	// Consumer.Return.Errors setting to true, and read from this channel.
-	Errors() <-chan error
-
-	// Close stops the ConsumerGroup and detaches any running sessions. It is required to call
-	// this function before the object passes out of scope, as it will otherwise leak memory.
-	Close() error
-}
-
-type consumerGroup struct {
-	client Client
-
-	config   *Config
-	consumer Consumer
-	groupID  string
-	memberID string
-	errors   chan error
-
-	lock      sync.Mutex
-	closed    chan none
-	closeOnce sync.Once
-
-	userData []byte
-}
-
-// NewConsumerGroup creates a new consumer group the given broker addresses and configuration.
-func NewConsumerGroup(addrs []string, groupID string, config *Config) (ConsumerGroup, error) {
-	client, err := NewClient(addrs, config)
-	if err != nil {
-		return nil, err
-	}
-
-	c, err := newConsumerGroup(groupID, client)
-	if err != nil {
-		_ = client.Close()
-	}
-	return c, err
-}
-
-// NewConsumerGroupFromClient creates a new consumer group using the given client. It is still
-// necessary to call Close() on the underlying client when shutting down this consumer.
-// PLEASE NOTE: consumer groups can only re-use but not share clients.
-func NewConsumerGroupFromClient(groupID string, client Client) (ConsumerGroup, error) {
-	// For clients passed in by the client, ensure we don't
-	// call Close() on it.
-	cli := &nopCloserClient{client}
-	return newConsumerGroup(groupID, cli)
-}
-
-func newConsumerGroup(groupID string, client Client) (ConsumerGroup, error) {
-	config := client.Config()
-	if !config.Version.IsAtLeast(V0_10_2_0) {
-		return nil, ConfigurationError("consumer groups require Version to be >= V0_10_2_0")
-	}
-
-	consumer, err := NewConsumerFromClient(client)
-	if err != nil {
-		return nil, err
-	}
-
-	return &consumerGroup{
-		client:   client,
-		consumer: consumer,
-		config:   config,
-		groupID:  groupID,
-		errors:   make(chan error, config.ChannelBufferSize),
-		closed:   make(chan none),
-	}, nil
-}
-
-// Errors implements ConsumerGroup.
-func (c *consumerGroup) Errors() <-chan error { return c.errors }
-
-// Close implements ConsumerGroup.
-func (c *consumerGroup) Close() (err error) {
-	c.closeOnce.Do(func() {
-		close(c.closed)
-
-		// leave group
-		if e := c.leave(); e != nil {
-			err = e
-		}
-
-		// drain errors
-		go func() {
-			close(c.errors)
-		}()
-		for e := range c.errors {
-			err = e
-		}
-
-		if e := c.client.Close(); e != nil {
-			err = e
-		}
-	})
-	return
-}
-
-// Consume implements ConsumerGroup.
-func (c *consumerGroup) Consume(ctx context.Context, topics []string, handler ConsumerGroupHandler) error {
-	// Ensure group is not closed
-	select {
-	case <-c.closed:
-		return ErrClosedConsumerGroup
-	default:
-	}
-
-	c.lock.Lock()
-	defer c.lock.Unlock()
-
-	// Quick exit when no topics are provided
-	if len(topics) == 0 {
-		return fmt.Errorf("no topics provided")
-	}
-
-	// Refresh metadata for requested topics
-	if err := c.client.RefreshMetadata(topics...); err != nil {
-		return err
-	}
-
-	// Init session
-	sess, err := c.newSession(ctx, topics, handler, c.config.Consumer.Group.Rebalance.Retry.Max)
-	if err == ErrClosedClient {
-		return ErrClosedConsumerGroup
-	} else if err != nil {
-		return err
-	}
-
-	// loop check topic partition numbers changed
-	// will trigger rebalance when any topic partitions number had changed
-	// avoid Consume function called again that will generate more than loopCheckPartitionNumbers coroutine
-	go c.loopCheckPartitionNumbers(topics, sess)
-
-	// Wait for session exit signal
-	<-sess.ctx.Done()
-
-	// Gracefully release session claims
-	return sess.release(true)
-}
-
-func (c *consumerGroup) retryNewSession(ctx context.Context, topics []string, handler ConsumerGroupHandler, retries int, refreshCoordinator bool) (*consumerGroupSession, error) {
-	select {
-	case <-c.closed:
-		return nil, ErrClosedConsumerGroup
-	case <-time.After(c.config.Consumer.Group.Rebalance.Retry.Backoff):
-	}
-
-	if refreshCoordinator {
-		err := c.client.RefreshCoordinator(c.groupID)
-		if err != nil {
-			return c.retryNewSession(ctx, topics, handler, retries, true)
-		}
-	}
-
-	return c.newSession(ctx, topics, handler, retries-1)
-}
-
-func (c *consumerGroup) newSession(ctx context.Context, topics []string, handler ConsumerGroupHandler, retries int) (*consumerGroupSession, error) {
-	coordinator, err := c.client.Coordinator(c.groupID)
-	if err != nil {
-		if retries <= 0 {
-			return nil, err
-		}
-
-		return c.retryNewSession(ctx, topics, handler, retries, true)
-	}
-
-	// Join consumer group
-	join, err := c.joinGroupRequest(coordinator, topics)
-	if err != nil {
-		_ = coordinator.Close()
-		return nil, err
-	}
-	switch join.Err {
-	case ErrNoError:
-		c.memberID = join.MemberId
-	case ErrUnknownMemberId, ErrIllegalGeneration: // reset member ID and retry immediately
-		c.memberID = ""
-		return c.newSession(ctx, topics, handler, retries)
-	case ErrNotCoordinatorForConsumer: // retry after backoff with coordinator refresh
-		if retries <= 0 {
-			return nil, join.Err
-		}
-
-		return c.retryNewSession(ctx, topics, handler, retries, true)
-	case ErrRebalanceInProgress: // retry after backoff
-		if retries <= 0 {
-			return nil, join.Err
-		}
-
-		return c.retryNewSession(ctx, topics, handler, retries, false)
-	default:
-		return nil, join.Err
-	}
-
-	// Prepare distribution plan if we joined as the leader
-	var plan BalanceStrategyPlan
-	if join.LeaderId == join.MemberId {
-		members, err := join.GetMembers()
-		if err != nil {
-			return nil, err
-		}
-
-		plan, err = c.balance(members)
-		if err != nil {
-			return nil, err
-		}
-	}
-
-	// Sync consumer group
-	sync, err := c.syncGroupRequest(coordinator, plan, join.GenerationId)
-	if err != nil {
-		_ = coordinator.Close()
-		return nil, err
-	}
-	switch sync.Err {
-	case ErrNoError:
-	case ErrUnknownMemberId, ErrIllegalGeneration: // reset member ID and retry immediately
-		c.memberID = ""
-		return c.newSession(ctx, topics, handler, retries)
-	case ErrNotCoordinatorForConsumer: // retry after backoff with coordinator refresh
-		if retries <= 0 {
-			return nil, sync.Err
-		}
-
-		return c.retryNewSession(ctx, topics, handler, retries, true)
-	case ErrRebalanceInProgress: // retry after backoff
-		if retries <= 0 {
-			return nil, sync.Err
-		}
-
-		return c.retryNewSession(ctx, topics, handler, retries, false)
-	default:
-		return nil, sync.Err
-	}
-
-	// Retrieve and sort claims
-	var claims map[string][]int32
-	if len(sync.MemberAssignment) > 0 {
-		members, err := sync.GetMemberAssignment()
-		if err != nil {
-			return nil, err
-		}
-		claims = members.Topics
-		c.userData = members.UserData
-
-		for _, partitions := range claims {
-			sort.Sort(int32Slice(partitions))
-		}
-	}
-
-	return newConsumerGroupSession(ctx, c, claims, join.MemberId, join.GenerationId, handler)
-}
-
-func (c *consumerGroup) joinGroupRequest(coordinator *Broker, topics []string) (*JoinGroupResponse, error) {
-	req := &JoinGroupRequest{
-		GroupId:        c.groupID,
-		MemberId:       c.memberID,
-		SessionTimeout: int32(c.config.Consumer.Group.Session.Timeout / time.Millisecond),
-		ProtocolType:   "consumer",
-	}
-	if c.config.Version.IsAtLeast(V0_10_1_0) {
-		req.Version = 1
-		req.RebalanceTimeout = int32(c.config.Consumer.Group.Rebalance.Timeout / time.Millisecond)
-	}
-
-	// use static user-data if configured, otherwise use consumer-group userdata from the last sync
-	userData := c.config.Consumer.Group.Member.UserData
-	if len(userData) == 0 {
-		userData = c.userData
-	}
-	meta := &ConsumerGroupMemberMetadata{
-		Topics:   topics,
-		UserData: userData,
-	}
-	strategy := c.config.Consumer.Group.Rebalance.Strategy
-	if err := req.AddGroupProtocolMetadata(strategy.Name(), meta); err != nil {
-		return nil, err
-	}
-
-	return coordinator.JoinGroup(req)
-}
-
-func (c *consumerGroup) syncGroupRequest(coordinator *Broker, plan BalanceStrategyPlan, generationID int32) (*SyncGroupResponse, error) {
-	req := &SyncGroupRequest{
-		GroupId:      c.groupID,
-		MemberId:     c.memberID,
-		GenerationId: generationID,
-	}
-	strategy := c.config.Consumer.Group.Rebalance.Strategy
-	for memberID, topics := range plan {
-		assignment := &ConsumerGroupMemberAssignment{Topics: topics}
-		userDataBytes, err := strategy.AssignmentData(memberID, topics, generationID)
-		if err != nil {
-			return nil, err
-		}
-		assignment.UserData = userDataBytes
-		if err := req.AddGroupAssignmentMember(memberID, assignment); err != nil {
-			return nil, err
-		}
-	}
-	return coordinator.SyncGroup(req)
-}
-
-func (c *consumerGroup) heartbeatRequest(coordinator *Broker, memberID string, generationID int32) (*HeartbeatResponse, error) {
-	req := &HeartbeatRequest{
-		GroupId:      c.groupID,
-		MemberId:     memberID,
-		GenerationId: generationID,
-	}
-
-	return coordinator.Heartbeat(req)
-}
-
-func (c *consumerGroup) balance(members map[string]ConsumerGroupMemberMetadata) (BalanceStrategyPlan, error) {
-	topics := make(map[string][]int32)
-	for _, meta := range members {
-		for _, topic := range meta.Topics {
-			topics[topic] = nil
-		}
-	}
-
-	for topic := range topics {
-		partitions, err := c.client.Partitions(topic)
-		if err != nil {
-			return nil, err
-		}
-		topics[topic] = partitions
-	}
-
-	strategy := c.config.Consumer.Group.Rebalance.Strategy
-	return strategy.Plan(members, topics)
-}
-
-// Leaves the cluster, called by Close.
-func (c *consumerGroup) leave() error {
-	c.lock.Lock()
-	defer c.lock.Unlock()
-	if c.memberID == "" {
-		return nil
-	}
-
-	coordinator, err := c.client.Coordinator(c.groupID)
-	if err != nil {
-		return err
-	}
-
-	resp, err := coordinator.LeaveGroup(&LeaveGroupRequest{
-		GroupId:  c.groupID,
-		MemberId: c.memberID,
-	})
-	if err != nil {
-		_ = coordinator.Close()
-		return err
-	}
-
-	// Unset memberID
-	c.memberID = ""
-
-	// Check response
-	switch resp.Err {
-	case ErrRebalanceInProgress, ErrUnknownMemberId, ErrNoError:
-		return nil
-	default:
-		return resp.Err
-	}
-}
-
-func (c *consumerGroup) handleError(err error, topic string, partition int32) {
-	if _, ok := err.(*ConsumerError); !ok && topic != "" && partition > -1 {
-		err = &ConsumerError{
-			Topic:     topic,
-			Partition: partition,
-			Err:       err,
-		}
-	}
-
-	if !c.config.Consumer.Return.Errors {
-		Logger.Println(err)
-		return
-	}
-
-	select {
-	case <-c.closed:
-		//consumer is closed
-		return
-	default:
-	}
-
-	select {
-	case c.errors <- err:
-	default:
-		// no error listener
-	}
-}
-
-func (c *consumerGroup) loopCheckPartitionNumbers(topics []string, session *consumerGroupSession) {
-	pause := time.NewTicker(c.config.Metadata.RefreshFrequency)
-	defer session.cancel()
-	defer pause.Stop()
-	var oldTopicToPartitionNum map[string]int
-	var err error
-	if oldTopicToPartitionNum, err = c.topicToPartitionNumbers(topics); err != nil {
-		return
-	}
-	for {
-		if newTopicToPartitionNum, err := c.topicToPartitionNumbers(topics); err != nil {
-			return
-		} else {
-			for topic, num := range oldTopicToPartitionNum {
-				if newTopicToPartitionNum[topic] != num {
-					return // trigger the end of the session on exit
-				}
-			}
-		}
-		select {
-		case <-pause.C:
-		case <-session.ctx.Done():
-			Logger.Printf("loop check partition number coroutine will exit, topics %s", topics)
-			// if session closed by other, should be exited
-			return
-		case <-c.closed:
-			return
-		}
-	}
-}
-
-func (c *consumerGroup) topicToPartitionNumbers(topics []string) (map[string]int, error) {
-	topicToPartitionNum := make(map[string]int, len(topics))
-	for _, topic := range topics {
-		if partitionNum, err := c.client.Partitions(topic); err != nil {
-			Logger.Printf("Consumer Group topic %s get partition number failed %v", topic, err)
-			return nil, err
-		} else {
-			topicToPartitionNum[topic] = len(partitionNum)
-		}
-	}
-	return topicToPartitionNum, nil
-}
-
-// --------------------------------------------------------------------
-
-// ConsumerGroupSession represents a consumer group member session.
-type ConsumerGroupSession interface {
-	// Claims returns information about the claimed partitions by topic.
-	Claims() map[string][]int32
-
-	// MemberID returns the cluster member ID.
-	MemberID() string
-
-	// GenerationID returns the current generation ID.
-	GenerationID() int32
-
-	// MarkOffset marks the provided offset, alongside a metadata string
-	// that represents the state of the partition consumer at that point in time. The
-	// metadata string can be used by another consumer to restore that state, so it
-	// can resume consumption.
-	//
-	// To follow upstream conventions, you are expected to mark the offset of the
-	// next message to read, not the last message read. Thus, when calling `MarkOffset`
-	// you should typically add one to the offset of the last consumed message.
-	//
-	// Note: calling MarkOffset does not necessarily commit the offset to the backend
-	// store immediately for efficiency reasons, and it may never be committed if
-	// your application crashes. This means that you may end up processing the same
-	// message twice, and your processing should ideally be idempotent.
-	MarkOffset(topic string, partition int32, offset int64, metadata string)
-
-	// ResetOffset resets to the provided offset, alongside a metadata string that
-	// represents the state of the partition consumer at that point in time. Reset
-	// acts as a counterpart to MarkOffset, the difference being that it allows to
-	// reset an offset to an earlier or smaller value, where MarkOffset only
-	// allows incrementing the offset. cf MarkOffset for more details.
-	ResetOffset(topic string, partition int32, offset int64, metadata string)
-
-	// MarkMessage marks a message as consumed.
-	MarkMessage(msg *ConsumerMessage, metadata string)
-
-	// Context returns the session context.
-	Context() context.Context
-}
-
-type consumerGroupSession struct {
-	parent       *consumerGroup
-	memberID     string
-	generationID int32
-	handler      ConsumerGroupHandler
-
-	claims  map[string][]int32
-	offsets *offsetManager
-	ctx     context.Context
-	cancel  func()
-
-	waitGroup       sync.WaitGroup
-	releaseOnce     sync.Once
-	hbDying, hbDead chan none
-}
-
-func newConsumerGroupSession(ctx context.Context, parent *consumerGroup, claims map[string][]int32, memberID string, generationID int32, handler ConsumerGroupHandler) (*consumerGroupSession, error) {
-	// init offset manager
-	offsets, err := newOffsetManagerFromClient(parent.groupID, memberID, generationID, parent.client)
-	if err != nil {
-		return nil, err
-	}
-
-	// init context
-	ctx, cancel := context.WithCancel(ctx)
-
-	// init session
-	sess := &consumerGroupSession{
-		parent:       parent,
-		memberID:     memberID,
-		generationID: generationID,
-		handler:      handler,
-		offsets:      offsets,
-		claims:       claims,
-		ctx:          ctx,
-		cancel:       cancel,
-		hbDying:      make(chan none),
-		hbDead:       make(chan none),
-	}
-
-	// start heartbeat loop
-	go sess.heartbeatLoop()
-
-	// create a POM for each claim
-	for topic, partitions := range claims {
-		for _, partition := range partitions {
-			pom, err := offsets.ManagePartition(topic, partition)
-			if err != nil {
-				_ = sess.release(false)
-				return nil, err
-			}
-
-			// handle POM errors
-			go func(topic string, partition int32) {
-				for err := range pom.Errors() {
-					sess.parent.handleError(err, topic, partition)
-				}
-			}(topic, partition)
-		}
-	}
-
-	// perform setup
-	if err := handler.Setup(sess); err != nil {
-		_ = sess.release(true)
-		return nil, err
-	}
-
-	// start consuming
-	for topic, partitions := range claims {
-		for _, partition := range partitions {
-			sess.waitGroup.Add(1)
-
-			go func(topic string, partition int32) {
-				defer sess.waitGroup.Done()
-
-				// cancel the as session as soon as the first
-				// goroutine exits
-				defer sess.cancel()
-
-				// consume a single topic/partition, blocking
-				sess.consume(topic, partition)
-			}(topic, partition)
-		}
-	}
-	return sess, nil
-}
-
-func (s *consumerGroupSession) Claims() map[string][]int32 { return s.claims }
-func (s *consumerGroupSession) MemberID() string           { return s.memberID }
-func (s *consumerGroupSession) GenerationID() int32        { return s.generationID }
-
-func (s *consumerGroupSession) MarkOffset(topic string, partition int32, offset int64, metadata string) {
-	if pom := s.offsets.findPOM(topic, partition); pom != nil {
-		pom.MarkOffset(offset, metadata)
-	}
-}
-
-func (s *consumerGroupSession) ResetOffset(topic string, partition int32, offset int64, metadata string) {
-	if pom := s.offsets.findPOM(topic, partition); pom != nil {
-		pom.ResetOffset(offset, metadata)
-	}
-}
-
-func (s *consumerGroupSession) MarkMessage(msg *ConsumerMessage, metadata string) {
-	s.MarkOffset(msg.Topic, msg.Partition, msg.Offset+1, metadata)
-}
-
-func (s *consumerGroupSession) Context() context.Context {
-	return s.ctx
-}
-
-func (s *consumerGroupSession) consume(topic string, partition int32) {
-	// quick exit if rebalance is due
-	select {
-	case <-s.ctx.Done():
-		return
-	case <-s.parent.closed:
-		return
-	default:
-	}
-
-	// get next offset
-	offset := s.parent.config.Consumer.Offsets.Initial
-	if pom := s.offsets.findPOM(topic, partition); pom != nil {
-		offset, _ = pom.NextOffset()
-	}
-
-	// create new claim
-	claim, err := newConsumerGroupClaim(s, topic, partition, offset)
-	if err != nil {
-		s.parent.handleError(err, topic, partition)
-		return
-	}
-
-	// handle errors
-	go func() {
-		for err := range claim.Errors() {
-			s.parent.handleError(err, topic, partition)
-		}
-	}()
-
-	// trigger close when session is done
-	go func() {
-		select {
-		case <-s.ctx.Done():
-		case <-s.parent.closed:
-		}
-		claim.AsyncClose()
-	}()
-
-	// start processing
-	if err := s.handler.ConsumeClaim(s, claim); err != nil {
-		s.parent.handleError(err, topic, partition)
-	}
-
-	// ensure consumer is closed & drained
-	claim.AsyncClose()
-	for _, err := range claim.waitClosed() {
-		s.parent.handleError(err, topic, partition)
-	}
-}
-
-func (s *consumerGroupSession) release(withCleanup bool) (err error) {
-	// signal release, stop heartbeat
-	s.cancel()
-
-	// wait for consumers to exit
-	s.waitGroup.Wait()
-
-	// perform release
-	s.releaseOnce.Do(func() {
-		if withCleanup {
-			if e := s.handler.Cleanup(s); e != nil {
-				s.parent.handleError(e, "", -1)
-				err = e
-			}
-		}
-
-		if e := s.offsets.Close(); e != nil {
-			err = e
-		}
-
-		close(s.hbDying)
-		<-s.hbDead
-	})
-
-	return
-}
-
-func (s *consumerGroupSession) heartbeatLoop() {
-	defer close(s.hbDead)
-	defer s.cancel() // trigger the end of the session on exit
-
-	pause := time.NewTicker(s.parent.config.Consumer.Group.Heartbeat.Interval)
-	defer pause.Stop()
-
-	retries := s.parent.config.Metadata.Retry.Max
-	for {
-		coordinator, err := s.parent.client.Coordinator(s.parent.groupID)
-		if err != nil {
-			if retries <= 0 {
-				s.parent.handleError(err, "", -1)
-				return
-			}
-
-			select {
-			case <-s.hbDying:
-				return
-			case <-time.After(s.parent.config.Metadata.Retry.Backoff):
-				retries--
-			}
-			continue
-		}
-
-		resp, err := s.parent.heartbeatRequest(coordinator, s.memberID, s.generationID)
-		if err != nil {
-			_ = coordinator.Close()
-
-			if retries <= 0 {
-				s.parent.handleError(err, "", -1)
-				return
-			}
-
-			retries--
-			continue
-		}
-
-		switch resp.Err {
-		case ErrNoError:
-			retries = s.parent.config.Metadata.Retry.Max
-		case ErrRebalanceInProgress, ErrUnknownMemberId, ErrIllegalGeneration:
-			return
-		default:
-			s.parent.handleError(resp.Err, "", -1)
-			return
-		}
-
-		select {
-		case <-pause.C:
-		case <-s.hbDying:
-			return
-		}
-	}
-}
-
-// --------------------------------------------------------------------
-
-// ConsumerGroupHandler instances are used to handle individual topic/partition claims.
-// It also provides hooks for your consumer group session life-cycle and allow you to
-// trigger logic before or after the consume loop(s).
-//
-// PLEASE NOTE that handlers are likely be called from several goroutines concurrently,
-// ensure that all state is safely protected against race conditions.
-type ConsumerGroupHandler interface {
-	// Setup is run at the beginning of a new session, before ConsumeClaim.
-	Setup(ConsumerGroupSession) error
-
-	// Cleanup is run at the end of a session, once all ConsumeClaim goroutines have exited
-	// but before the offsets are committed for the very last time.
-	Cleanup(ConsumerGroupSession) error
-
-	// ConsumeClaim must start a consumer loop of ConsumerGroupClaim's Messages().
-	// Once the Messages() channel is closed, the Handler must finish its processing
-	// loop and exit.
-	ConsumeClaim(ConsumerGroupSession, ConsumerGroupClaim) error
-}
-
-// ConsumerGroupClaim processes Kafka messages from a given topic and partition within a consumer group.
-type ConsumerGroupClaim interface {
-	// Topic returns the consumed topic name.
-	Topic() string
-
-	// Partition returns the consumed partition.
-	Partition() int32
-
-	// InitialOffset returns the initial offset that was used as a starting point for this claim.
-	InitialOffset() int64
-
-	// HighWaterMarkOffset returns the high water mark offset of the partition,
-	// i.e. the offset that will be used for the next message that will be produced.
-	// You can use this to determine how far behind the processing is.
-	HighWaterMarkOffset() int64
-
-	// Messages returns the read channel for the messages that are returned by
-	// the broker. The messages channel will be closed when a new rebalance cycle
-	// is due. You must finish processing and mark offsets within
-	// Config.Consumer.Group.Session.Timeout before the topic/partition is eventually
-	// re-assigned to another group member.
-	Messages() <-chan *ConsumerMessage
-}
-
-type consumerGroupClaim struct {
-	topic     string
-	partition int32
-	offset    int64
-	PartitionConsumer
-}
-
-func newConsumerGroupClaim(sess *consumerGroupSession, topic string, partition int32, offset int64) (*consumerGroupClaim, error) {
-	pcm, err := sess.parent.consumer.ConsumePartition(topic, partition, offset)
-	if err == ErrOffsetOutOfRange {
-		offset = sess.parent.config.Consumer.Offsets.Initial
-		pcm, err = sess.parent.consumer.ConsumePartition(topic, partition, offset)
-	}
-	if err != nil {
-		return nil, err
-	}
-
-	go func() {
-		for err := range pcm.Errors() {
-			sess.parent.handleError(err, topic, partition)
-		}
-	}()
-
-	return &consumerGroupClaim{
-		topic:             topic,
-		partition:         partition,
-		offset:            offset,
-		PartitionConsumer: pcm,
-	}, nil
-}
-
-func (c *consumerGroupClaim) Topic() string        { return c.topic }
-func (c *consumerGroupClaim) Partition() int32     { return c.partition }
-func (c *consumerGroupClaim) InitialOffset() int64 { return c.offset }
-
-// Drains messages and errors, ensures the claim is fully closed.
-func (c *consumerGroupClaim) waitClosed() (errs ConsumerErrors) {
-	go func() {
-		for range c.Messages() {
-		}
-	}()
-
-	for err := range c.Errors() {
-		errs = append(errs, err)
-	}
-	return
-}

vendor/github.com/Shopify/sarama/consumer_group_members.go

@@ -1,96 +0,0 @@
-package sarama
-
-//ConsumerGroupMemberMetadata holds the metadata for consumer group
-type ConsumerGroupMemberMetadata struct {
-	Version  int16
-	Topics   []string
-	UserData []byte
-}
-
-func (m *ConsumerGroupMemberMetadata) encode(pe packetEncoder) error {
-	pe.putInt16(m.Version)
-
-	if err := pe.putStringArray(m.Topics); err != nil {
-		return err
-	}
-
-	if err := pe.putBytes(m.UserData); err != nil {
-		return err
-	}
-
-	return nil
-}
-
-func (m *ConsumerGroupMemberMetadata) decode(pd packetDecoder) (err error) {
-	if m.Version, err = pd.getInt16(); err != nil {
-		return
-	}
-
-	if m.Topics, err = pd.getStringArray(); err != nil {
-		return
-	}
-
-	if m.UserData, err = pd.getBytes(); err != nil {
-		return
-	}
-
-	return nil
-}
-
-//ConsumerGroupMemberAssignment holds the member assignment for a consume group
-type ConsumerGroupMemberAssignment struct {
-	Version  int16
-	Topics   map[string][]int32
-	UserData []byte
-}
-
-func (m *ConsumerGroupMemberAssignment) encode(pe packetEncoder) error {
-	pe.putInt16(m.Version)
-
-	if err := pe.putArrayLength(len(m.Topics)); err != nil {
-		return err
-	}
-
-	for topic, partitions := range m.Topics {
-		if err := pe.putString(topic); err != nil {
-			return err
-		}
-		if err := pe.putInt32Array(partitions); err != nil {
-			return err
-		}
-	}
-
-	if err := pe.putBytes(m.UserData); err != nil {
-		return err
-	}
-
-	return nil
-}
-
-func (m *ConsumerGroupMemberAssignment) decode(pd packetDecoder) (err error) {
-	if m.Version, err = pd.getInt16(); err != nil {
-		return
-	}
-
-	var topicLen int
-	if topicLen, err = pd.getArrayLength(); err != nil {
-		return
-	}
-
-	m.Topics = make(map[string][]int32, topicLen)
-	for i := 0; i < topicLen; i++ {
-		var topic string
-		if topic, err = pd.getString(); err != nil {
-			return
-		}
-		if m.Topics[topic], err = pd.getInt32Array(); err != nil {
-			return
-		}
-	}
-
-	if m.UserData, err = pd.getBytes(); err != nil {
-		return
-	}
-
-	return nil
-}

vendor/github.com/Shopify/sarama/consumer_metadata_request.go

@@ -1,38 +0,0 @@
-package sarama
-
-//ConsumerMetadataRequest is used for metadata requests
-type ConsumerMetadataRequest struct {
-	ConsumerGroup string
-}
-
-func (r *ConsumerMetadataRequest) encode(pe packetEncoder) error {
-	tmp := new(FindCoordinatorRequest)
-	tmp.CoordinatorKey = r.ConsumerGroup
-	tmp.CoordinatorType = CoordinatorGroup
-	return tmp.encode(pe)
-}
-
-func (r *ConsumerMetadataRequest) decode(pd packetDecoder, version int16) (err error) {
-	tmp := new(FindCoordinatorRequest)
-	if err := tmp.decode(pd, version); err != nil {
-		return err
-	}
-	r.ConsumerGroup = tmp.CoordinatorKey
-	return nil
-}
-
-func (r *ConsumerMetadataRequest) key() int16 {
-	return 10
-}
-
-func (r *ConsumerMetadataRequest) version() int16 {
-	return 0
-}
-
-func (r *ConsumerMetadataRequest) headerVersion() int16 {
-	return 1
-}
-
-func (r *ConsumerMetadataRequest) requiredVersion() KafkaVersion {
-	return V0_8_2_0
-}

vendor/github.com/Shopify/sarama/consumer_metadata_response.go

@@ -1,82 +0,0 @@
-package sarama
-
-import (
-	"net"
-	"strconv"
-)
-
-//ConsumerMetadataResponse holds the response for a consumer group meta data requests
-type ConsumerMetadataResponse struct {
-	Err             KError
-	Coordinator     *Broker
-	CoordinatorID   int32  // deprecated: use Coordinator.ID()
-	CoordinatorHost string // deprecated: use Coordinator.Addr()
-	CoordinatorPort int32  // deprecated: use Coordinator.Addr()
-}
-
-func (r *ConsumerMetadataResponse) decode(pd packetDecoder, version int16) (err error) {
-	tmp := new(FindCoordinatorResponse)
-
-	if err := tmp.decode(pd, version); err != nil {
-		return err
-	}
-
-	r.Err = tmp.Err
-
-	r.Coordinator = tmp.Coordinator
-	if tmp.Coordinator == nil {
-		return nil
-	}
-
-	// this can all go away in 2.0, but we have to fill in deprecated fields to maintain
-	// backwards compatibility
-	host, portstr, err := net.SplitHostPort(r.Coordinator.Addr())
-	if err != nil {
-		return err
-	}
-	port, err := strconv.ParseInt(portstr, 10, 32)
-	if err != nil {
-		return err
-	}
-	r.CoordinatorID = r.Coordinator.ID()
-	r.CoordinatorHost = host
-	r.CoordinatorPort = int32(port)
-
-	return nil
-}
-
-func (r *ConsumerMetadataResponse) encode(pe packetEncoder) error {
-	if r.Coordinator == nil {
-		r.Coordinator = new(Broker)
-		r.Coordinator.id = r.CoordinatorID
-		r.Coordinator.addr = net.JoinHostPort(r.CoordinatorHost, strconv.Itoa(int(r.CoordinatorPort)))
-	}
-
-	tmp := &FindCoordinatorResponse{
-		Version:     0,
-		Err:         r.Err,
-		Coordinator: r.Coordinator,
-	}
-
-	if err := tmp.encode(pe); err != nil {
-		return err
-	}
-
-	return nil
-}
-
-func (r *ConsumerMetadataResponse) key() int16 {
-	return 10
-}
-
-func (r *ConsumerMetadataResponse) version() int16 {
-	return 0
-}
-
-func (r *ConsumerMetadataResponse) headerVersion() int16 {
-	return 0
-}
-
-func (r *ConsumerMetadataResponse) requiredVersion() KafkaVersion {
-	return V0_8_2_0
-}

vendor/github.com/Shopify/sarama/control_record.go

@@ -1,72 +0,0 @@
-package sarama
-
-//ControlRecordType ...
-type ControlRecordType int
-
-const (
-	//ControlRecordAbort is a control record for abort
-	ControlRecordAbort ControlRecordType = iota
-	//ControlRecordCommit is a control record for commit
-	ControlRecordCommit
-	//ControlRecordUnknown is a control record of unknown type
-	ControlRecordUnknown
-)
-
-// Control records are returned as a record by fetchRequest
-// However unlike "normal" records, they mean nothing application wise.
-// They only serve internal logic for supporting transactions.
-type ControlRecord struct {
-	Version          int16
-	CoordinatorEpoch int32
-	Type             ControlRecordType
-}
-
-func (cr *ControlRecord) decode(key, value packetDecoder) error {
-	var err error
-	cr.Version, err = value.getInt16()
-	if err != nil {
-		return err
-	}
-
-	cr.CoordinatorEpoch, err = value.getInt32()
-	if err != nil {
-		return err
-	}
-
-	// There a version for the value part AND the key part. And I have no idea if they are supposed to match or not
-	// Either way, all these version can only be 0 for now
-	cr.Version, err = key.getInt16()
-	if err != nil {
-		return err
-	}
-
-	recordType, err := key.getInt16()
-	if err != nil {
-		return err
-	}
-
-	switch recordType {
-	case 0:
-		cr.Type = ControlRecordAbort
-	case 1:
-		cr.Type = ControlRecordCommit
-	default:
-		// from JAVA implementation:
-		// UNKNOWN is used to indicate a control type which the client is not aware of and should be ignored
-		cr.Type = ControlRecordUnknown
-	}
-	return nil
-}
-
-func (cr *ControlRecord) encode(key, value packetEncoder) {
-	value.putInt16(cr.Version)
-	value.putInt32(cr.CoordinatorEpoch)
-	key.putInt16(cr.Version)
-
-	switch cr.Type {
-	case ControlRecordAbort:
-		key.putInt16(0)
-	case ControlRecordCommit:
-		key.putInt16(1)
-	}
-}

vendor/github.com/Shopify/sarama/crc32_field.go

@@ -1,86 +0,0 @@
-package sarama
-
-import (
-	"encoding/binary"
-	"fmt"
-	"hash/crc32"
-	"sync"
-)
-
-type crcPolynomial int8
-
-const (
-	crcIEEE crcPolynomial = iota
-	crcCastagnoli
-)
-
-var crc32FieldPool = sync.Pool{}
-
-func acquireCrc32Field(polynomial crcPolynomial) *crc32Field {
-	val := crc32FieldPool.Get()
-	if val != nil {
-		c := val.(*crc32Field)
-		c.polynomial = polynomial
-		return c
-	}
-	return newCRC32Field(polynomial)
-}
-
-func releaseCrc32Field(c *crc32Field) {
-	crc32FieldPool.Put(c)
-}
-
-var castagnoliTable = crc32.MakeTable(crc32.Castagnoli)
-
-// crc32Field implements the pushEncoder and pushDecoder interfaces for calculating CRC32s.
-type crc32Field struct {
-	startOffset int
-	polynomial  crcPolynomial
-}
-
-func (c *crc32Field) saveOffset(in int) {
-	c.startOffset = in
-}
-
-func (c *crc32Field) reserveLength() int {
-	return 4
-}
-
-func newCRC32Field(polynomial crcPolynomial) *crc32Field {
-	return &crc32Field{polynomial: polynomial}
-}
-
-func (c *crc32Field) run(curOffset int, buf []byte) error {
-	crc, err := c.crc(curOffset, buf)
-	if err != nil {
-		return err
-	}
-	binary.BigEndian.PutUint32(buf[c.startOffset:], crc)
-	return nil
-}
-
-func (c *crc32Field) check(curOffset int, buf []byte) error {
-	crc, err := c.crc(curOffset, buf)
-	if err != nil {
-		return err
-	}
-
-	expected := binary.BigEndian.Uint32(buf[c.startOffset:])
-	if crc != expected {
-		return PacketDecodingError{fmt.Sprintf("CRC didn't match expected %#x got %#x", expected, crc)}
-	}
-
-	return nil
-}
-func (c *crc32Field) crc(curOffset int, buf []byte) (uint32, error) {
-	var tab *crc32.Table
-	switch c.polynomial {
-	case crcIEEE:
-		tab = crc32.IEEETable
-	case crcCastagnoli:
-		tab = castagnoliTable
-	default:
-		return 0, PacketDecodingError{"invalid CRC type"}
-	}
-	return crc32.Checksum(buf[c.startOffset+4:curOffset], tab), nil
-}

vendor/github.com/Shopify/sarama/create_partitions_request.go

@@ -1,125 +0,0 @@
-package sarama
-
-import "time"
-
-type CreatePartitionsRequest struct {
-	TopicPartitions map[string]*TopicPartition
-	Timeout         time.Duration
-	ValidateOnly    bool
-}
-
-func (c *CreatePartitionsRequest) encode(pe packetEncoder) error {
-	if err := pe.putArrayLength(len(c.TopicPartitions)); err != nil {
-		return err
-	}
-
-	for topic, partition := range c.TopicPartitions {
-		if err := pe.putString(topic); err != nil {
-			return err
-		}
-		if err := partition.encode(pe); err != nil {
-			return err
-		}
-	}
-
-	pe.putInt32(int32(c.Timeout / time.Millisecond))
-
-	pe.putBool(c.ValidateOnly)
-
-	return nil
-}
-
-func (c *CreatePartitionsRequest) decode(pd packetDecoder, version int16) (err error) {
-	n, err := pd.getArrayLength()
-	if err != nil {
-		return err
-	}
-	c.TopicPartitions = make(map[string]*TopicPartition, n)
-	for i := 0; i < n; i++ {
-		topic, err := pd.getString()
-		if err != nil {
-			return err
-		}
-		c.TopicPartitions[topic] = new(TopicPartition)
-		if err := c.TopicPartitions[topic].decode(pd, version); err != nil {
-			return err
-		}
-	}
-
-	timeout, err := pd.getInt32()
-	if err != nil {
-		return err
-	}
-	c.Timeout = time.Duration(timeout) * time.Millisecond
-
-	if c.ValidateOnly, err = pd.getBool(); err != nil {
-		return err
-	}
-
-	return nil
-}
-
-func (r *CreatePartitionsRequest) key() int16 {
-	return 37
-}
-
-func (r *CreatePartitionsRequest) version() int16 {
-	return 0
-}
-
-func (r *CreatePartitionsRequest) headerVersion() int16 {
-	return 1
-}
-
-func (r *CreatePartitionsRequest) requiredVersion() KafkaVersion {
-	return V1_0_0_0
-}
-
-type TopicPartition struct {
-	Count      int32
-	Assignment [][]int32
-}
-
-func (t *TopicPartition) encode(pe packetEncoder) error {
-	pe.putInt32(t.Count)
-
-	if len(t.Assignment) == 0 {
-		pe.putInt32(-1)
-		return nil
-	}
-
-	if err := pe.putArrayLength(len(t.Assignment)); err != nil {
-		return err
-	}
-
-	for _, assign := range t.Assignment {
-		if err := pe.putInt32Array(assign); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-func (t *TopicPartition) decode(pd packetDecoder, version int16) (err error) {
-	if t.Count, err = pd.getInt32(); err != nil {
-		return err
-	}
-
-	n, err := pd.getInt32()
-	if err != nil {
-		return err
-	}
-	if n <= 0 {
-		return nil
-	}
-	t.Assignment = make([][]int32, n)
-
-	for i := 0; i < int(n); i++ {
-		if t.Assignment[i], err = pd.getInt32Array(); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}

vendor/github.com/Shopify/sarama/create_partitions_response.go

@@ -1,109 +0,0 @@
-package sarama
-
-import (
-	"fmt"
-	"time"
-)
-
-type CreatePartitionsResponse struct {
-	ThrottleTime         time.Duration
-	TopicPartitionErrors map[string]*TopicPartitionError
-}
-
-func (c *CreatePartitionsResponse) encode(pe packetEncoder) error {
-	pe.putInt32(int32(c.ThrottleTime / time.Millisecond))
-	if err := pe.putArrayLength(len(c.TopicPartitionErrors)); err != nil {
-		return err
-	}
-
-	for topic, partitionError := range c.TopicPartitionErrors {
-		if err := pe.putString(topic); err != nil {
-			return err
-		}
-		if err := partitionError.encode(pe); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-func (c *CreatePartitionsResponse) decode(pd packetDecoder, version int16) (err error) {
-	throttleTime, err := pd.getInt32()
-	if err != nil {
-		return err
-	}
-	c.ThrottleTime = time.Duration(throttleTime) * time.Millisecond
-
-	n, err := pd.getArrayLength()
-	if err != nil {
-		return err
-	}
-
-	c.TopicPartitionErrors = make(map[string]*TopicPartitionError, n)
-	for i := 0; i < n; i++ {
-		topic, err := pd.getString()
-		if err != nil {
-			return err
-		}
-		c.TopicPartitionErrors[topic] = new(TopicPartitionError)
-		if err := c.TopicPartitionErrors[topic].decode(pd, version); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-func (r *CreatePartitionsResponse) key() int16 {
-	return 37
-}
-
-func (r *CreatePartitionsResponse) version() int16 {
-	return 0
-}
-
-func (r *CreatePartitionsResponse) headerVersion() int16 {
-	return 0
-}
-
-func (r *CreatePartitionsResponse) requiredVersion() KafkaVersion {
-	return V1_0_0_0
-}
-
-type TopicPartitionError struct {
-	Err    KError
-	ErrMsg *string
-}
-
-func (t *TopicPartitionError) Error() string {
-	text := t.Err.Error()
-	if t.ErrMsg != nil {
-		text = fmt.Sprintf("%s - %s", text, *t.ErrMsg)
-	}
-	return text
-}
-
-func (t *TopicPartitionError) encode(pe packetEncoder) error {
-	pe.putInt16(int16(t.Err))
-
-	if err := pe.putNullableString(t.ErrMsg); err != nil {
-		return err
-	}
-
-	return nil
-}
-
-func (t *TopicPartitionError) decode(pd packetDecoder, version int16) (err error) {
-	kerr, err := pd.getInt16()
-	if err != nil {
-		return err
-	}
-	t.Err = KError(kerr)
-
-	if t.ErrMsg, err = pd.getNullableString(); err != nil {
-		return err
-	}
-
-	return nil
-}

vendor/github.com/Shopify/sarama/create_topics_request.go

@@ -1,178 +0,0 @@
-package sarama
-
-import (
-	"time"
-)
-
-type CreateTopicsRequest struct {
-	Version int16
-
-	TopicDetails map[string]*TopicDetail
-	Timeout      time.Duration
-	ValidateOnly bool
-}
-
-func (c *CreateTopicsRequest) encode(pe packetEncoder) error {
-	if err := pe.putArrayLength(len(c.TopicDetails)); err != nil {
-		return err
-	}
-	for topic, detail := range c.TopicDetails {
-		if err := pe.putString(topic); err != nil {
-			return err
-		}
-		if err := detail.encode(pe); err != nil {
-			return err
-		}
-	}
-
-	pe.putInt32(int32(c.Timeout / time.Millisecond))
-
-	if c.Version >= 1 {
-		pe.putBool(c.ValidateOnly)
-	}
-
-	return nil
-}
-
-func (c *CreateTopicsRequest) decode(pd packetDecoder, version int16) (err error) {
-	n, err := pd.getArrayLength()
-	if err != nil {
-		return err
-	}
-
-	c.TopicDetails = make(map[string]*TopicDetail, n)
-
-	for i := 0; i < n; i++ {
-		topic, err := pd.getString()
-		if err != nil {
-			return err
-		}
-		c.TopicDetails[topic] = new(TopicDetail)
-		if err = c.TopicDetails[topic].decode(pd, version); err != nil {
-			return err
-		}
-	}
-
-	timeout, err := pd.getInt32()
-	if err != nil {
-		return err
-	}
-	c.Timeout = time.Duration(timeout) * time.Millisecond
-
-	if version >= 1 {
-		c.ValidateOnly, err = pd.getBool()
-		if err != nil {
-			return err
-		}
-
-		c.Version = version
-	}
-
-	return nil
-}
-
-func (c *CreateTopicsRequest) key() int16 {
-	return 19
-}
-
-func (c *CreateTopicsRequest) version() int16 {
-	return c.Version
-}
-
-func (r *CreateTopicsRequest) headerVersion() int16 {
-	return 1
-}
-
-func (c *CreateTopicsRequest) requiredVersion() KafkaVersion {
-	switch c.Version {
-	case 2:
-		return V1_0_0_0
-	case 1:
-		return V0_11_0_0
-	default:
-		return V0_10_1_0
-	}
-}
-
-type TopicDetail struct {
-	NumPartitions     int32
-	ReplicationFactor int16
-	ReplicaAssignment map[int32][]int32
-	ConfigEntries     map[string]*string
-}
-
-func (t *TopicDetail) encode(pe packetEncoder) error {
-	pe.putInt32(t.NumPartitions)
-	pe.putInt16(t.ReplicationFactor)
-
-	if err := pe.putArrayLength(len(t.ReplicaAssignment)); err != nil {
-		return err
-	}
-	for partition, assignment := range t.ReplicaAssignment {
-		pe.putInt32(partition)
-		if err := pe.putInt32Array(assignment); err != nil {
-			return err
-		}
-	}
-
-	if err := pe.putArrayLength(len(t.ConfigEntries)); err != nil {
-		return err
-	}
-	for configKey, configValue := range t.ConfigEntries {
-		if err := pe.putString(configKey); err != nil {
-			return err
-		}
-		if err := pe.putNullableString(configValue); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-func (t *TopicDetail) decode(pd packetDecoder, version int16) (err error) {
-	if t.NumPartitions, err = pd.getInt32(); err != nil {
-		return err
-	}
-	if t.ReplicationFactor, err = pd.getInt16(); err != nil {
-		return err
-	}
-
-	n, err := pd.getArrayLength()
-	if err != nil {
-		return err
-	}
-
-	if n > 0 {
-		t.ReplicaAssignment = make(map[int32][]int32, n)
-		for i := 0; i < n; i++ {
-			replica, err := pd.getInt32()
-			if err != nil {
-				return err
-			}
-			if t.ReplicaAssignment[replica], err = pd.getInt32Array(); err != nil {
-				return err
-			}
-		}
-	}
-
-	n, err = pd.getArrayLength()
-	if err != nil {
-		return err
-	}
-
-	if n > 0 {
-		t.ConfigEntries = make(map[string]*string, n)
-		for i := 0; i < n; i++ {
-			configKey, err := pd.getString()
-			if err != nil {
-				return err
-			}
-			if t.ConfigEntries[configKey], err = pd.getNullableString(); err != nil {
-				return err
-			}
-		}
-	}
-
-	return nil
-}

vendor/github.com/Shopify/sarama/create_topics_response.go

@@ -1,127 +0,0 @@
-package sarama
-
-import (
-	"fmt"
-	"time"
-)
-
-type CreateTopicsResponse struct {
-	Version      int16
-	ThrottleTime time.Duration
-	TopicErrors  map[string]*TopicError
-}
-
-func (c *CreateTopicsResponse) encode(pe packetEncoder) error {
-	if c.Version >= 2 {
-		pe.putInt32(int32(c.ThrottleTime / time.Millisecond))
-	}
-
-	if err := pe.putArrayLength(len(c.TopicErrors)); err != nil {
-		return err
-	}
-	for topic, topicError := range c.TopicErrors {
-		if err := pe.putString(topic); err != nil {
-			return err
-		}
-		if err := topicError.encode(pe, c.Version); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-func (c *CreateTopicsResponse) decode(pd packetDecoder, version int16) (err error) {
-	c.Version = version
-
-	if version >= 2 {
-		throttleTime, err := pd.getInt32()
-		if err != nil {
-			return err
-		}
-		c.ThrottleTime = time.Duration(throttleTime) * time.Millisecond
-	}
-
-	n, err := pd.getArrayLength()
-	if err != nil {
-		return err
-	}
-
-	c.TopicErrors = make(map[string]*TopicError, n)
-	for i := 0; i < n; i++ {
-		topic, err := pd.getString()
-		if err != nil {
-			return err
-		}
-		c.TopicErrors[topic] = new(TopicError)
-		if err := c.TopicErrors[topic].decode(pd, version); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-func (c *CreateTopicsResponse) key() int16 {
-	return 19
-}
-
-func (c *CreateTopicsResponse) version() int16 {
-	return c.Version
-}
-
-func (c *CreateTopicsResponse) headerVersion() int16 {
-	return 0
-}
-
-func (c *CreateTopicsResponse) requiredVersion() KafkaVersion {
-	switch c.Version {
-	case 2:
-		return V1_0_0_0
-	case 1:
-		return V0_11_0_0
-	default:
-		return V0_10_1_0
-	}
-}
-
-type TopicError struct {
-	Err    KError
-	ErrMsg *string
-}
-
-func (t *TopicError) Error() string {
-	text := t.Err.Error()
-	if t.ErrMsg != nil {
-		text = fmt.Sprintf("%s - %s", text, *t.ErrMsg)
-	}
-	return text
-}
-
-func (t *TopicError) encode(pe packetEncoder, version int16) error {
-	pe.putInt16(int16(t.Err))
-
-	if version >= 1 {
-		if err := pe.putNullableString(t.ErrMsg); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-func (t *TopicError) decode(pd packetDecoder, version int16) (err error) {
-	kErr, err := pd.getInt16()
-	if err != nil {
-		return err
-	}
-	t.Err = KError(kErr)
-
-	if version >= 1 {
-		if t.ErrMsg, err = pd.getNullableString(); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}

vendor/github.com/Shopify/sarama/decompress.go

@@ -1,63 +0,0 @@
-package sarama
-
-import (
-	"bytes"
-	"compress/gzip"
-	"fmt"
-	"io/ioutil"
-	"sync"
-
-	snappy "github.com/eapache/go-xerial-snappy"
-	"github.com/pierrec/lz4"
-)
-
-var (
-	lz4ReaderPool = sync.Pool{
-		New: func() interface{} {
-			return lz4.NewReader(nil)
-		},
-	}
-
-	gzipReaderPool sync.Pool
-)
-
-func decompress(cc CompressionCodec, data []byte) ([]byte, error) {
-	switch cc {
-	case CompressionNone:
-		return data, nil
-	case CompressionGZIP:
-		var (
-			err        error
-			reader     *gzip.Reader
-			readerIntf = gzipReaderPool.Get()
-		)
-		if readerIntf != nil {
-			reader = readerIntf.(*gzip.Reader)
-		} else {
-			reader, err = gzip.NewReader(bytes.NewReader(data))
-			if err != nil {
-				return nil, err
-			}
-		}
-
-		defer gzipReaderPool.Put(reader)
-
-		if err := reader.Reset(bytes.NewReader(data)); err != nil {
-			return nil, err
-		}
-
-		return ioutil.ReadAll(reader)
-	case CompressionSnappy:
-		return snappy.Decode(data)
-	case CompressionLZ4:
-		reader := lz4ReaderPool.Get().(*lz4.Reader)
-		defer lz4ReaderPool.Put(reader)
-
-		reader.Reset(bytes.NewReader(data))
-		return ioutil.ReadAll(reader)
-	case CompressionZSTD:
-		return zstdDecompress(nil, data)
-	default:
-		return nil, PacketDecodingError{fmt.Sprintf("invalid compression specified (%d)", cc)}
-	}
-}

vendor/github.com/Shopify/sarama/delete_groups_request.go

@@ -1,34 +0,0 @@
-package sarama
-
-type DeleteGroupsRequest struct {
-	Groups []string
-}
-
-func (r *DeleteGroupsRequest) encode(pe packetEncoder) error {
-	return pe.putStringArray(r.Groups)
-}
-
-func (r *DeleteGroupsRequest) decode(pd packetDecoder, version int16) (err error) {
-	r.Groups, err = pd.getStringArray()
-	return
-}
-
-func (r *DeleteGroupsRequest) key() int16 {
-	return 42
-}
-
-func (r *DeleteGroupsRequest) version() int16 {
-	return 0
-}
-
-func (r *DeleteGroupsRequest) headerVersion() int16 {
-	return 1
-}
-
-func (r *DeleteGroupsRequest) requiredVersion() KafkaVersion {
-	return V1_1_0_0
-}
-
-func (r *DeleteGroupsRequest) AddGroup(group string) {
-	r.Groups = append(r.Groups, group)
-}

vendor/github.com/Shopify/sarama/delete_groups_response.go

@@ -1,74 +0,0 @@
-package sarama
-
-import (
-	"time"
-)
-
-type DeleteGroupsResponse struct {
-	ThrottleTime    time.Duration
-	GroupErrorCodes map[string]KError
-}
-
-func (r *DeleteGroupsResponse) encode(pe packetEncoder) error {
-	pe.putInt32(int32(r.ThrottleTime / time.Millisecond))
-
-	if err := pe.putArrayLength(len(r.GroupErrorCodes)); err != nil {
-		return err
-	}
-	for groupID, errorCode := range r.GroupErrorCodes {
-		if err := pe.putString(groupID); err != nil {
-			return err
-		}
-		pe.putInt16(int16(errorCode))
-	}
-
-	return nil
-}
-
-func (r *DeleteGroupsResponse) decode(pd packetDecoder, version int16) error {
-	throttleTime, err := pd.getInt32()
-	if err != nil {
-		return err
-	}
-	r.ThrottleTime = time.Duration(throttleTime) * time.Millisecond
-
-	n, err := pd.getArrayLength()
-	if err != nil {
-		return err
-	}
-	if n == 0 {
-		return nil
-	}
-
-	r.GroupErrorCodes = make(map[string]KError, n)
-	for i := 0; i < n; i++ {
-		groupID, err := pd.getString()
-		if err != nil {
-			return err
-		}
-		errorCode, err := pd.getInt16()
-		if err != nil {
-			return err
-		}
-
-		r.GroupErrorCodes[groupID] = KError(errorCode)
-	}
-
-	return nil
-}
-
-func (r *DeleteGroupsResponse) key() int16 {
-	return 42
-}
-
-func (r *DeleteGroupsResponse) version() int16 {
-	return 0
-}
-
-func (r *DeleteGroupsResponse) headerVersion() int16 {
-	return 0
-}
-
-func (r *DeleteGroupsResponse) requiredVersion() KafkaVersion {
-	return V1_1_0_0
-}