Fix: message size check in timeout test (#1014)

Fix message size check in `TestMessageTimeoutReached`. Since the message parser has parsed two packets of size 63 << 10, then the message size should be 63 << 11

.deepsource.toml

@@ -0,0 +1,28 @@
+version = 1
+
+exclude_patterns = [
+  "vendor/**"
+]
+
+[[analyzers]]
+name = "go"
+enabled = true
+
+  [analyzers.meta]
+  import_paths = ["github.com/ankitdobhal/goreplay"]
+
+[[analyzers]]
+name = "docker"
+enabled = true
+
+[[analyzers]]
+name = "ruby"
+enabled = true
+
+[[analyzers]]
+name = "javascript"
+enabled = true
+
+  [analyzers.meta]
+  environment = ["nodejs"]
+

.github/dependabot.yaml

@@ -0,0 +1,10 @@
+version: 2
+updates:
+  # Maintain dependencies for GitHub Actions
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+    pull-request-branch-name:
+      # Separate sections of the branch name with a hyphen
+      separator: "-"

.github/workflows/ci-docker.yaml

@@ -0,0 +1,38 @@
+name: ci
+on:
+  release:
+    types: [published]
+jobs:
+  docker-build-and-push:
+    runs-on: ubuntu-latest
+    steps:
+      -
+        name: "Checkout repository"
+        uses: actions/checkout@v2
+      -
+        name: "Set up Docker Buildx"
+        uses: docker/setup-buildx-action@v1
+      -
+        name: "Cache Docker layers"
+        uses: actions/cache@v2
+        with:
+          path: /tmp/.buildx-cache
+          key: ${{ runner.os }}-buildx-${{ github.sha }}
+          restore-keys: |
+            ${{ runner.os }}-buildx-
+      -
+        name: "Login to Container Registry"
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      -
+        name: "Build and push docker image"
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          push: true
+          tags: ${{ github.repository }}:${{ github.event.release.tag_name }}
+          build-args: RELEASE_VERSION=${{ github.event.release.tag_name }}
+          cache-from: type=local,src=/tmp/.buildx-cache
+          cache-to: type=local,mode=max,dest=/tmp/.buildx-cache

.github/workflows/ci-test.yaml

@@ -0,0 +1,29 @@
+name: test
+on: [push, pull_request]
+jobs:
+  test:
+    strategy:
+      matrix:
+        go-version: [1.15.x, 1.16.x] # two latest minor versions
+    runs-on: ubuntu-latest
+    steps:
+    - name: update package index
+      run: sudo apt-get update
+    - name: install libpcap
+      run: sudo apt-get install libpcap-dev -y
+    - name: install Go
+      uses: actions/setup-go@v2
+      with:
+        go-version: ${{ matrix.go-version }}
+    - name: checkout code
+      uses: actions/checkout@v2
+    - uses: actions/cache@v2
+      with:
+        path: |
+          ~/go/pkg/mod              # Module download cache
+          ~/.cache/go-build         # Build cache (Linux)
+        key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
+        restore-keys: |
+          ${{ runner.os }}-go-
+    - name: test
+      run: sudo go test ./... -v -timeout 120s

.gitignore

@@ -1,10 +1,14 @@
+vendor
 *.swp
 *.gor
 *.rpm
 *.dep
+*.deb
 *.pkg
-
+*.exe
+*.pprof
 *.out
+hey
 
 *.bin
 

.travis.yml

@@ -1,9 +0,0 @@
-dist: focal
-
-language: go
-go: 1.14
-
-script: sudo -E bash -c "source /etc/profile && eval '$(gimme 1.14)' && go test ./... -v -timeout 120s"
-
-before_install:
-  - sudo apt-get install libpcap-dev -y

Dockerfile

@@ -1,6 +1,9 @@
-FROM alpine:latest as builder
+FROM alpine:3.12 as builder
+
+ARG RELEASE_VERSION
+
 RUN apk add --no-cache ca-certificates openssl
-RUN wget https://github.com/buger/goreplay/releases/download/v1.2.0/gor_v1.2.0_x64.tar.gz -O gor.tar.gz
+RUN wget https://github.com/buger/goreplay/releases/download/${RELEASE_VERSION}/gor_${RELEASE_VERSION}_x64.tar.gz -O gor.tar.gz
 RUN tar xzf gor.tar.gz
 
 FROM scratch

Dockerfile.dev

@@ -1,9 +1,9 @@
-FROM golang:1.14
+FROM golang:1.15
 
 RUN apt-get update && apt-get install ruby vim-common -y
 
 RUN apt-get install flex bison -y
-RUN wget http://www.tcpdump.org/release/libpcap-1.9.1.tar.gz && tar xzf libpcap-1.9.1.tar.gz && cd libpcap-1.9.1 && ./configure && make install
+RUN wget http://www.tcpdump.org/release/libpcap-1.10.0.tar.gz && tar xzf libpcap-1.10.0.tar.gz && cd libpcap-1.10.0 && ./configure && make install
 
 RUN go get github.com/google/gopacket
 RUN go get -u golang.org/x/lint/golint

Makefile

@@ -23,16 +23,24 @@ FPMCOMMON= \
     -s dir \
     -C /tmp/gor-build \
 
-release: release-x64 release-mac
+release: release-x64 release-x86 release-mac release-windows
 
-release-bin:
-	docker run -v `pwd`:$(SOURCE_PATH) -t --env GOOS=linux --env GOARCH=amd64  -i $(CONTAINER) go build -o $(BIN_NAME) -tags netgo $(LDFLAGS)
+vendor:
+	go mod vendor
 
-release-bin-mac:
-	GOOS=darwin go build -o $(BIN_NAME) $(MAC_LDFLAGS)
+release-bin: vendor
+	docker run --rm -v `pwd`:$(SOURCE_PATH) -t --env GOOS=linux --env GOARCH=amd64  -i $(CONTAINER) go build -mod=vendor -o $(BIN_NAME) -tags netgo $(LDFLAGS)
 
-release-x64:
-	docker run -v `pwd`:$(SOURCE_PATH) -t --env GOOS=linux --env GOARCH=amd64  -i $(CONTAINER) go build -o $(BIN_NAME) -tags netgo $(LDFLAGS)
+release-bin-x64: vendor
+	docker run --rm -v `pwd`:$(SOURCE_PATH) -t --env GOOS=linux --env GOARCH=386 -i $(CONTAINER) go build -mod=vendor -o $(BIN_NAME) -tags netgo $(LDFLAGS)
+
+release-bin-mac: vendor
+	GOOS=darwin go build -mod=vendor -o $(BIN_NAME) $(MAC_LDFLAGS)
+
+release-bin-windows: vendor
+	docker run -it --rm -v `pwd`:$(SOURCE_PATH) -w $(SOURCE_PATH) -e CGO_ENABLED=1 docker.elastic.co/beats-dev/golang-crossbuild:1.16.4-main --build-cmd "make VERSION=$(VERSION) build" -p "windows/amd64"
+
+release-x64: release-bin
 	tar -czf gor_$(VERSION)$(PREFIX)_x64.tar.gz $(BIN_NAME)
 	mkdir -p /tmp/gor-build
 	mv ./$(BIN_NAME) /tmp/gor-build/$(BIN_NAME)
@@ -43,13 +51,11 @@ release-x64:
 	fpm $(FPMCOMMON) -a amd64 -t rpm ./=/usr/local/bin
 	rm -rf /tmp/gor-build
 
-release-x86:
-	docker run -v `pwd`:$(SOURCE_PATH) -t --env GOOS=linux --env GOARCH=386 -i $(CONTAINER) go build -o $(BIN_NAME) -tags netgo $(LDFLAGS)
+release-x86: release-bin-x64
 	tar -czf gor_$(VERSION)$(PREFIX)_x86.tar.gz $(BIN_NAME)
 	rm $(BIN_NAME)
 
-release-mac:
-	go build -o $(BIN_NAME) $(MAC_LDFLAGS)
+release-mac: release-bin-mac
 	tar -czf gor_$(VERSION)$(PREFIX)_mac.tar.gz $(BIN_NAME)
 	mkdir -p /tmp/gor-build
 	mv ./$(BIN_NAME) /tmp/gor-build/$(BIN_NAME)
@@ -58,11 +64,18 @@ release-mac:
 	fpm $(FPMCOMMON) -a amd64 -t osxpkg ./=/usr/local/bin
 	rm -rf /tmp/gor-build
 
+release-windows: release-bin-windows
+	mv ./gor ./gor.exe
+	zip gor-$(VERSION)$(PREFIX)_windows.zip ./gor.exe
+	rm -rf ./gor.exe
+
+build:
+	go build -mod=vendor -o $(BIN_NAME) $(LDFLAGS)
 
 install:
 	go install $(MAC_LDFLAGS)
 
-build:
+build-env:
 	docker build -t $(CONTAINER) -f Dockerfile.dev .
 
 profile:
@@ -102,7 +115,7 @@ profile_test:
 
 # Used mainly for debugging, because docker container do not have access to parent machine ports
 run:
-	$(RUN) go run $(LDFLAGS) $(SOURCE) --input-dummy=0 --output-http="http://localhost:9000" --input-raw-track-response --input-raw 127.0.0.1:9000 --verbose --debug --middleware "./examples/middleware/echo.sh" --output-file requests.gor
+	$(RUN) go run $(LDFLAGS) $(SOURCE) --input-dummy=0 --output-http="http://localhost:9000" --input-raw-track-response --input-raw 127.0.0.1:9000 --verbose 0 --middleware "./examples/middleware/echo.sh" --output-file requests.gor
 
 run-2:
 	$(RUN) go run $(LDFLAGS) $(SOURCE) --input-raw :8000 --input-raw-bpf-filter "dst port 8000" --output-stdout --output-http "http://localhost:8000" --input-dummy=0
@@ -130,7 +143,7 @@ bash:
 
 
 FPMCOMMON= \
-    --name goreplay \
+    --name gor \
     --description "GoReplay is an open-source network monitoring tool which can record your live traffic, and use it for shadowing, load testing, monitoring and detailed analysis." \
     -v $(VERSION) \
     --vendor "Leonid Bugaev" \

capture/af_packet.go

@@ -0,0 +1,32 @@
+// +build !linux
+
+package capture
+
+import (
+	"fmt"
+	"time"
+
+	"github.com/google/gopacket"
+)
+
+func newAfpacketHandle(device string, snaplen int, block_size int, num_blocks int,
+	useVLAN bool, timeout time.Duration) (*afpacketHandle, error) {
+	return nil, fmt.Errorf("Not implemented")
+}
+
+func afpacketComputeSize(targetSizeMb int, snaplen int, pageSize int) (
+	frameSize int, blockSize int, numBlocks int, err error) {
+	return 0, 0, 0, fmt.Errorf("Not implemented")
+}
+
+type afpacketHandle struct{}
+
+// ReadPacketData satisfies PacketDataSource interface
+func (h *afpacketHandle) ReadPacketData() (data []byte, ci gopacket.CaptureInfo, err error) {
+	return nil, gopacket.CaptureInfo{}, fmt.Errorf("Not implemented")
+}
+
+// SetBPFFilter translates a BPF filter string into BPF RawInstruction and applies them.
+func (h *afpacketHandle) SetBPFFilter(filter string, snaplen int) (err error) {
+	return fmt.Errorf("Not implemented")
+}

capture/af_packet_linux.go

@@ -0,0 +1,115 @@
+// +build linux
+
+package capture
+
+import (
+	"fmt"
+	"time"
+
+	"github.com/google/gopacket"
+	"github.com/google/gopacket/afpacket"
+	"github.com/google/gopacket/layers"
+	"github.com/google/gopacket/pcap"
+	"golang.org/x/net/bpf"
+
+	_ "github.com/google/gopacket/layers"
+)
+
+type afpacketHandle struct {
+	TPacket *afpacket.TPacket
+}
+
+func newAfpacketHandle(device string, snaplen int, block_size int, num_blocks int,
+	useVLAN bool, timeout time.Duration) (*afpacketHandle, error) {
+
+	h := &afpacketHandle{}
+	var err error
+
+	if device == "any" {
+		h.TPacket, err = afpacket.NewTPacket(
+			afpacket.OptFrameSize(snaplen),
+			afpacket.OptBlockSize(block_size),
+			afpacket.OptNumBlocks(num_blocks),
+			afpacket.OptAddVLANHeader(false),
+			afpacket.OptPollTimeout(timeout),
+			afpacket.SocketRaw,
+			afpacket.TPacketVersion3)
+	} else {
+		h.TPacket, err = afpacket.NewTPacket(
+			afpacket.OptInterface(device),
+			afpacket.OptFrameSize(snaplen),
+			afpacket.OptBlockSize(block_size),
+			afpacket.OptNumBlocks(num_blocks),
+			afpacket.OptAddVLANHeader(false),
+			afpacket.OptPollTimeout(timeout),
+			afpacket.SocketRaw,
+			afpacket.TPacketVersion3)
+	}
+	return h, err
+}
+
+// ReadPacketData satisfies PacketDataSource interface
+func (h *afpacketHandle) ReadPacketData() (data []byte, ci gopacket.CaptureInfo, err error) {
+	return h.TPacket.ReadPacketData()
+}
+
+// SetBPFFilter translates a BPF filter string into BPF RawInstruction and applies them.
+func (h *afpacketHandle) SetBPFFilter(filter string, snaplen int) (err error) {
+	pcapBPF, err := pcap.CompileBPFFilter(layers.LinkTypeEthernet, snaplen, filter)
+	if err != nil {
+		return err
+	}
+	bpfIns := []bpf.RawInstruction{}
+	for _, ins := range pcapBPF {
+		bpfIns2 := bpf.RawInstruction{
+			Op: ins.Code,
+			Jt: ins.Jt,
+			Jf: ins.Jf,
+			K:  ins.K,
+		}
+		bpfIns = append(bpfIns, bpfIns2)
+	}
+	if h.TPacket.SetBPF(bpfIns); err != nil {
+		return err
+	}
+	return h.TPacket.SetBPF(bpfIns)
+}
+
+// LinkType returns ethernet link type.
+func (h *afpacketHandle) LinkType() layers.LinkType {
+	return layers.LinkTypeEthernet
+}
+
+// Close will close afpacket source.
+func (h *afpacketHandle) Close() {
+	h.TPacket.Close()
+}
+
+// SocketStats prints received, dropped, queue-freeze packet stats.
+func (h *afpacketHandle) SocketStats() (as afpacket.SocketStats, asv afpacket.SocketStatsV3, err error) {
+	return h.TPacket.SocketStats()
+}
+
+// afpacketComputeSize computes the block_size and the num_blocks in such a way that the
+// allocated mmap buffer is close to but smaller than target_size_mb.
+// The restriction is that the block_size must be divisible by both the
+// frame size and page size.
+func afpacketComputeSize(targetSizeMb int, snaplen int, pageSize int) (
+	frameSize int, blockSize int, numBlocks int, err error) {
+
+	if snaplen < pageSize {
+		frameSize = pageSize / (pageSize / snaplen)
+	} else {
+		frameSize = (snaplen/pageSize + 1) * pageSize
+	}
+
+	// 128 is the default from the gopacket library so just use that
+	blockSize = frameSize * 128
+	numBlocks = (targetSizeMb * 1024 * 1024) / blockSize
+
+	if numBlocks == 0 {
+		return 0, 0, 0, fmt.Errorf("Interface buffersize is too small")
+	}
+
+	return frameSize, blockSize, numBlocks, nil
+}

capture/capture.go

@@ -3,21 +3,40 @@ package capture
 import (
 	"context"
 	"errors"
+	"expvar"
 	"fmt"
+	"io"
+	"log"
 	"net"
 	"os"
 	"runtime"
+	"strings"
 	"sync"
+	"syscall"
 	"time"
 
+	"github.com/buger/goreplay/proto"
 	"github.com/buger/goreplay/size"
+	"github.com/buger/goreplay/tcp"
+
 	"github.com/google/gopacket"
 	"github.com/google/gopacket/layers"
 	"github.com/google/gopacket/pcap"
 )
 
-// Handler is a function that is used to handle packets
-type Handler func(gopacket.Packet)
+var stats *expvar.Map
+
+func init() {
+	stats = expvar.NewMap("raw")
+	stats.Init()
+}
+
+// PacketHandler is a function that is used to handle packets
+type PacketHandler func(*tcp.Packet)
+
+type PcapStatProvider interface {
+	Stats() (*pcap.Stats, error)
+}
 
 // PcapOptions options that can be set on a pcap capture handle,
 // these options take effect on inactive pcap handles
@@ -31,30 +50,33 @@ type PcapOptions struct {
 	Snaplen       bool          `json:"input-raw-override-snaplen"`
 }
 
-// NetInterface represents network interface
-type NetInterface struct {
-	net.Interface
-	IPs []string
-}
-
 // Listener handle traffic capture, this is its representation.
 type Listener struct {
 	sync.Mutex
 	Transport  string       // transport layer default to tcp
 	Activate   func() error // function is used to activate the engine. it must be called before reading packets
-	Handles    map[string]gopacket.PacketDataSource
-	Interfaces []NetInterface
+	Handles    map[string]packetHandle
+	Interfaces []pcap.Interface
 	loopIndex  int
 	Reading    chan bool // this channel is closed when the listener has started reading packets
 	PcapOptions
-	Engine        EngineType
-	port          uint16 // src or/and dst port
-	trackResponse bool
+	Engine          EngineType
+	ports           []uint16 // src or/and dst ports
+	trackResponse   bool
+	expiry          time.Duration
+	allowIncomplete bool
+	messages        chan *tcp.Message
+	protocol        tcp.TCPProtocol
 
 	host string // pcap file name or interface (name, hardware addr, index or ip address)
 
-	quit    chan bool
-	packets chan gopacket.Packet
+	closeDone chan struct{}
+	quit      chan struct{}
+}
+
+type packetHandle struct {
+	handler gopacket.PacketDataSource
+	ips     []net.IP
 }
 
 // EngineType ...
@@ -65,6 +87,7 @@ const (
 	EnginePcap EngineType = 1 << iota
 	EnginePcapFile
 	EngineRawSocket
+	EngineAFPacket
 )
 
 // Set is here so that EngineType can implement flag.Var
@@ -74,8 +97,10 @@ func (eng *EngineType) Set(v string) error {
 		*eng = EnginePcap
 	case "pcap_file":
 		*eng = EnginePcapFile
-	case "raw_socket", "af_packet":
+	case "raw_socket":
 		*eng = EngineRawSocket
+	case "af_packet":
+		*eng = EngineAFPacket
 	default:
 		return fmt.Errorf("invalid engine %s", v)
 	}
@@ -90,6 +115,8 @@ func (eng *EngineType) String() (e string) {
 		e = "libpcap"
 	case EngineRawSocket:
 		e = "raw_socket"
+	case EngineAFPacket:
+		e = "af_packet"
 	default:
 		e = ""
 	}
@@ -99,20 +126,29 @@ func (eng *EngineType) String() (e string) {
 // NewListener creates and initialize a new Listener. if transport or/and engine are invalid/unsupported
 // is "tcp" and "pcap", are assumed. l.Engine and l.Transport can help to get the values used.
 // if there is an error it will be associated with getting network interfaces
-func NewListener(host string, port uint16, transport string, engine EngineType, trackResponse bool) (l *Listener, err error) {
+func NewListener(host string, ports []uint16, transport string, engine EngineType, protocol tcp.TCPProtocol, trackResponse bool, expiry time.Duration, allowIncomplete bool) (l *Listener, err error) {
 	l = &Listener{}
 
 	l.host = host
-	l.port = port
+	if l.host == "localhost" {
+		l.host = "127.0.0.1"
+	}
+	l.ports = ports
+
 	l.Transport = "tcp"
 	if transport != "" {
 		l.Transport = transport
 	}
-	l.Handles = make(map[string]gopacket.PacketDataSource)
+	l.Handles = make(map[string]packetHandle)
 	l.trackResponse = trackResponse
-	l.packets = make(chan gopacket.Packet, 1000)
-	l.quit = make(chan bool, 1)
-	l.Reading = make(chan bool, 1)
+	l.closeDone = make(chan struct{})
+	l.quit = make(chan struct{})
+	l.Reading = make(chan bool)
+	l.expiry = expiry
+	l.allowIncomplete = allowIncomplete
+	l.protocol = protocol
+	l.messages = make(chan *tcp.Message, 10000)
+
 	switch engine {
 	default:
 		l.Engine = EnginePcap
@@ -120,11 +156,15 @@ func NewListener(host string, port uint16, transport string, engine EngineType,
 	case EngineRawSocket:
 		l.Engine = EngineRawSocket
 		l.Activate = l.activateRawSocket
+	case EngineAFPacket:
+		l.Engine = EngineAFPacket
+		l.Activate = l.activateAFPacket
 	case EnginePcapFile:
 		l.Engine = EnginePcapFile
 		l.Activate = l.activatePcapFile
 		return
 	}
+
 	err = l.setInterfaces()
 	if err != nil {
 		return nil, err
@@ -139,38 +179,27 @@ func (l *Listener) SetPcapOptions(opts PcapOptions) {
 }
 
 // Listen listens for packets from the handles, and call handler on every packet received
-// until the context done signal is sent or EOF on handles.
-// this function should be called after activating pcap handles
-func (l *Listener) Listen(ctx context.Context, handler Handler) (err error) {
+// until the context done signal is sent or there is unrecoverable error on all handles.
+// this function must be called after activating pcap handles
+func (l *Listener) Listen(ctx context.Context) (err error) {
 	l.read()
 	done := ctx.Done()
-	var p gopacket.Packet
-	var ok bool
-	for {
-		select {
-		case <-done:
-			l.quit <- true
-			close(l.quit)
-			err = ctx.Err()
-			done = nil
-		case p, ok = <-l.packets:
-			if !ok {
-				return
-			}
-			if p == nil {
-				continue
-			}
-			handler(p)
-		}
+	select {
+	case <-done:
+		close(l.quit) // signal close on all handles
+		<-l.closeDone // wait all handles to be closed
+		err = ctx.Err()
+	case <-l.closeDone: // all handles closed voluntarily
 	}
+	return
 }
 
 // ListenBackground is like listen but can run concurrently and signal error through channel
-func (l *Listener) ListenBackground(ctx context.Context, handler Handler) chan error {
+func (l *Listener) ListenBackground(ctx context.Context) chan error {
 	err := make(chan error, 1)
 	go func() {
 		defer close(err)
-		if e := l.Listen(ctx, handler); err != nil {
+		if e := l.Listen(ctx); err != nil {
 			err <- e
 		}
 	}()
@@ -179,59 +208,51 @@ func (l *Listener) ListenBackground(ctx context.Context, handler Handler) chan e
 
 // Filter returns automatic filter applied by goreplay
 // to a pcap handle of a specific interface
-func (l *Listener) Filter(ifi NetInterface) (filter string) {
+func (l *Listener) Filter(ifi pcap.Interface) (filter string) {
 	// https://www.tcpdump.org/manpages/pcap-filter.7.html
 
-	port := fmt.Sprintf("portrange 0-%d", 1<<16-1)
-	if l.port != 0 {
-		port = fmt.Sprintf("port %d", l.port)
-	}
-	dir := " dst " // direction
-	if l.trackResponse {
-		dir = " "
-	}
-	filter = fmt.Sprintf("(%s%s%s)", l.Transport, dir, port)
+	hosts := []string{l.host}
 	if listenAll(l.host) || isDevice(l.host, ifi) {
-		return
+		hosts = interfaceAddresses(ifi)
 	}
-	filter = fmt.Sprintf("(%s%s%s and host %s)", l.Transport, dir, port, l.host)
-	return
-}
 
-// PcapDumpHandler returns a handler to write packet data in PCAP
-// format, See http://wiki.wireshark.org/Development/LibpcapFileFormathandler.
-// if link layer is invalid Ethernet is assumed
-func PcapDumpHandler(file *os.File, link layers.LinkType, debugger func(int, ...interface{})) (handler func(packet gopacket.Packet), err error) {
-	if link.String() == "" {
-		link = layers.LinkTypeEthernet
+	filter = portsFilter(l.Transport, "dst", l.ports)
+
+	if len(hosts) != 0 && !l.Promiscuous {
+		filter = fmt.Sprintf("((%s) and (%s))", filter, hostsFilter("dst", hosts))
+	} else {
+		filter = fmt.Sprintf("(%s)", filter)
 	}
-	w := NewWriterNanos(file)
-	err = w.WriteFileHeader(64<<10, link)
-	if err != nil {
-		return nil, err
-	}
-	return func(packet gopacket.Packet) {
-		err = w.WritePacket(packet.Metadata().CaptureInfo, packet.Data())
-		if err != nil && debugger != nil {
-			go debugger(3, err)
+
+	if l.trackResponse {
+		responseFilter := portsFilter(l.Transport, "src", l.ports)
+
+		if len(hosts) != 0 && !l.Promiscuous {
+			responseFilter = fmt.Sprintf("((%s) and (%s))", responseFilter, hostsFilter("src", hosts))
+		} else {
+			responseFilter = fmt.Sprintf("(%s)", responseFilter)
 		}
-	}, nil
+
+		filter = fmt.Sprintf("%s or %s", filter, responseFilter)
+	}
+
+	return
 }
 
 // PcapHandle returns new pcap Handle from dev on success.
 // this function should be called after setting all necessary options for this listener
-func (l *Listener) PcapHandle(ifi NetInterface) (handle *pcap.Handle, err error) {
+func (l *Listener) PcapHandle(ifi pcap.Interface) (handle *pcap.Handle, err error) {
 	var inactive *pcap.InactiveHandle
 	inactive, err = pcap.NewInactiveHandle(ifi.Name)
-	if inactive != nil && err != nil {
-		defer inactive.CleanUp()
-	}
 	if err != nil {
 		return nil, fmt.Errorf("inactive handle error: %q, interface: %q", err, ifi.Name)
 	}
-	if l.TimestampType != "" {
+	defer inactive.CleanUp()
+
+	if l.TimestampType != "" && l.TimestampType != "go" {
 		var ts pcap.TimestampSource
 		ts, err = pcap.TimestampSourceFromString(l.TimestampType)
+		fmt.Println("Setting custom Timestamp Source. Supported values: `go`, ", inactive.SupportedTimestamps())
 		err = inactive.SetTimestampSource(ts)
 		if err != nil {
 			return nil, fmt.Errorf("%q: supported timestamps: %q, interface: %q", err, inactive.SupportedTimestamps(), ifi.Name)
@@ -247,12 +268,22 @@ func (l *Listener) PcapHandle(ifi NetInterface) (handle *pcap.Handle, err error)
 			return nil, fmt.Errorf("monitor mode error: %q, interface: %q", err, ifi.Name)
 		}
 	}
+
 	var snap int
-	if l.Snaplen {
-		snap = 64<<10 + 200
-	} else if ifi.MTU > 0 {
-		snap = ifi.MTU + 200
+
+	if !l.Snaplen {
+		infs, _ := net.Interfaces()
+		for _, i := range infs {
+			if i.Name == ifi.Name {
+				snap = i.MTU + 200
+			}
+		}
 	}
+
+	if snap == 0 {
+		snap = 64<<10 + 200
+	}
+
 	err = inactive.SetSnapLen(snap)
 	if err != nil {
 		return nil, fmt.Errorf("snapshot length error: %q, interface: %q", err, ifi.Name)
@@ -264,7 +295,7 @@ func (l *Listener) PcapHandle(ifi NetInterface) (handle *pcap.Handle, err error)
 		}
 	}
 	if l.BufferTimeout == 0 {
-		l.BufferTimeout = pcap.BlockForever
+		l.BufferTimeout = 2000 * time.Millisecond
 	}
 	err = inactive.SetTimeout(l.BufferTimeout)
 	if err != nil {
@@ -274,37 +305,33 @@ func (l *Listener) PcapHandle(ifi NetInterface) (handle *pcap.Handle, err error)
 	if err != nil {
 		return nil, fmt.Errorf("PCAP Activate device error: %q, interface: %q", err, ifi.Name)
 	}
-	if l.BPFFilter != "" {
-		if l.BPFFilter[0] != '(' || l.BPFFilter[len(l.BPFFilter)-1] != ')' {
-			l.BPFFilter = "(" + l.BPFFilter + ")"
-		}
-	} else {
-		l.BPFFilter = l.Filter(ifi)
+
+	bpfFilter := l.BPFFilter
+	if bpfFilter == "" {
+		bpfFilter = l.Filter(ifi)
 	}
-	err = handle.SetBPFFilter(l.BPFFilter)
+	fmt.Println("Interface:", ifi.Name, ". BPF Filter:", bpfFilter)
+	err = handle.SetBPFFilter(bpfFilter)
 	if err != nil {
 		handle.Close()
-		return nil, fmt.Errorf("BPF filter error: %q%s, interface: %q", err, l.BPFFilter, ifi.Name)
+		return nil, fmt.Errorf("BPF filter error: %q%s, interface: %q", err, bpfFilter, ifi.Name)
 	}
 	return
 }
 
 // SocketHandle returns new unix ethernet handle associated with this listener settings
-func (l *Listener) SocketHandle(ifi NetInterface) (handle Socket, err error) {
-	handle, err = NewSocket(ifi.Interface)
+func (l *Listener) SocketHandle(ifi pcap.Interface) (handle Socket, err error) {
+	handle, err = NewSocket(ifi)
 	if err != nil {
 		return nil, fmt.Errorf("sock raw error: %q, interface: %q", err, ifi.Name)
 	}
 	if err = handle.SetPromiscuous(l.Promiscuous || l.Monitor); err != nil {
 		return nil, fmt.Errorf("promiscuous mode error: %q, interface: %q", err, ifi.Name)
 	}
-	if l.BPFFilter != "" {
-		if l.BPFFilter[0] != '(' || l.BPFFilter[len(l.BPFFilter)-1] != ')' {
-			l.BPFFilter = "(" + l.BPFFilter + ")"
-		}
-	} else {
+	if l.BPFFilter == "" {
 		l.BPFFilter = l.Filter(ifi)
 	}
+	fmt.Println("BPF Filter: ", l.BPFFilter)
 	if err = handle.SetBPFFilter(l.BPFFilter); err != nil {
 		handle.Close()
 		return nil, fmt.Errorf("BPF filter error: %q%s, interface: %q", err, l.BPFFilter, ifi.Name)
@@ -313,50 +340,123 @@ func (l *Listener) SocketHandle(ifi NetInterface) (handle Socket, err error) {
 	return
 }
 
+func http1StartHint(pckt *tcp.Packet) (isRequest, isResponse bool) {
+	if proto.HasRequestTitle(pckt.Payload) {
+		return true, false
+	}
+
+	if proto.HasResponseTitle(pckt.Payload) {
+		return false, true
+	}
+
+	// No request or response detected
+	return false, false
+}
+
+func http1EndHint(m *tcp.Message) bool {
+	if m.MissingChunk() {
+		return false
+	}
+
+	return proto.HasFullPayload(m, m.PacketData()...)
+}
+
 func (l *Listener) read() {
 	l.Lock()
 	defer l.Unlock()
 	for key, handle := range l.Handles {
-		var source *gopacket.PacketSource
-		linkType := layers.LinkTypeEthernet
-		if _, ok := handle.(*pcap.Handle); ok {
-			linkType = handle.(*pcap.Handle).LinkType()
-		}
-		source = gopacket.NewPacketSource(handle, linkType)
-		source.Lazy = true
-		source.NoCopy = true
-		ch := source.Packets()
-		go func(key string) {
+		go func(key string, hndl packetHandle) {
+			runtime.LockOSThread()
+
 			defer l.closeHandles(key)
+			linkSize := 14
+			linkType := int(layers.LinkTypeEthernet)
+			if _, ok := hndl.handler.(*pcap.Handle); ok {
+				linkType = int(hndl.handler.(*pcap.Handle).LinkType())
+				linkSize, ok = pcapLinkTypeLength(linkType)
+				if !ok {
+					if os.Getenv("GORDEBUG") != "0" {
+						log.Printf("can not identify link type of an interface '%s'\n", key)
+					}
+					return // can't find the linktype size
+				}
+			}
+
+			messageParser := tcp.NewMessageParser(l.messages, l.ports, hndl.ips, l.expiry, l.allowIncomplete)
+
+			if l.protocol == tcp.ProtocolHTTP {
+				messageParser.Start = http1StartHint
+				messageParser.End = http1EndHint
+			}
+
+			timer := time.NewTicker(1 * time.Second)
+
 			for {
 				select {
 				case <-l.quit:
 					return
-				case p, ok := <-ch:
-					if !ok {
+				case <-timer.C:
+					if h, ok := hndl.handler.(PcapStatProvider); ok {
+						s, err := h.Stats()
+						if err == nil {
+							stats.Add("packets_received", int64(s.PacketsReceived))
+							stats.Add("packets_dropped", int64(s.PacketsDropped))
+							stats.Add("packets_if_dropped", int64(s.PacketsIfDropped))
+						}
+					}
+				default:
+					data, ci, err := hndl.handler.ReadPacketData()
+					if err == nil {
+						if l.TimestampType == "go" {
+							ci.Timestamp = time.Now()
+						}
+
+						messageParser.PacketHandler(&tcp.PcapPacket{
+							Data:     data,
+							LType:    linkType,
+							LTypeLen: linkSize,
+							Ci:       &ci,
+						})
+						continue
+					}
+					if enext, ok := err.(pcap.NextError); ok && enext == pcap.NextErrorTimeoutExpired {
+						continue
+					}
+					if eno, ok := err.(syscall.Errno); ok && eno.Temporary() {
+						continue
+					}
+					if enet, ok := err.(*net.OpError); ok && (enet.Temporary() || enet.Timeout()) {
+						continue
+					}
+					if err == io.EOF || err == io.ErrClosedPipe {
+						log.Printf("stopped reading from %s interface with error %s\n", key, err)
 						return
 					}
-					l.packets <- p
+
+					log.Printf("stopped reading from %s interface with error %s\n", key, err)
+					return
 				}
 			}
-		}(key)
+		}(key, handle)
 	}
-	l.Reading <- true
 	close(l.Reading)
 }
 
+func (l *Listener) Messages() chan *tcp.Message {
+	return l.messages
+}
+
 func (l *Listener) closeHandles(key string) {
 	l.Lock()
 	defer l.Unlock()
 	if handle, ok := l.Handles[key]; ok {
-		if _, ok = handle.(Socket); ok {
-			handle.(Socket).Close()
-		} else {
-			handle.(*pcap.Handle).Close()
+		if c, ok := handle.handler.(io.Closer); ok {
+			c.Close()
 		}
+
 		delete(l.Handles, key)
 		if len(l.Handles) == 0 {
-			close(l.packets)
+			close(l.closeDone)
 		}
 	}
 }
@@ -371,7 +471,10 @@ func (l *Listener) activatePcap() error {
 			msg += ("\n" + e.Error())
 			continue
 		}
-		l.Handles[ifi.Name] = handle
+		l.Handles[ifi.Name] = packetHandle{
+			handler: handle,
+			ips:     interfaceIPs(ifi),
+		}
 	}
 	if len(l.Handles) == 0 {
 		return fmt.Errorf("pcap handles error:%s", msg)
@@ -392,9 +495,15 @@ func (l *Listener) activateRawSocket() error {
 			msg += ("\n" + e.Error())
 			continue
 		}
-		l.Handles[ifi.Name] = handle
+		l.Handles[ifi.Name] = packetHandle{
+			handler: handle,
+			ips:     interfaceIPs(ifi),
+		}
 	}
-	return e
+	if len(l.Handles) == 0 {
+		return fmt.Errorf("raw socket handles error:%s", msg)
+	}
+	return nil
 }
 
 func (l *Listener) activatePcapFile() (err error) {
@@ -403,92 +512,143 @@ func (l *Listener) activatePcapFile() (err error) {
 	if handle, e = pcap.OpenOffline(l.host); e != nil {
 		return fmt.Errorf("open pcap file error: %q", e)
 	}
-	if l.BPFFilter != "" {
-		if l.BPFFilter[0] != '(' || l.BPFFilter[len(l.BPFFilter)-1] != ')' {
-			l.BPFFilter = "(" + l.BPFFilter + ")"
-		}
-	} else {
-		addr := l.host
-		l.host = ""
-		l.BPFFilter = l.Filter(NetInterface{})
-		l.host = addr
-	}
+
+	tmp := l.host
+	l.host = ""
+	l.BPFFilter = l.Filter(pcap.Interface{})
+	l.host = tmp
+
 	if e = handle.SetBPFFilter(l.BPFFilter); e != nil {
 		handle.Close()
 		return fmt.Errorf("BPF filter error: %q, filter: %s", e, l.BPFFilter)
 	}
-	l.Handles["pcap_file"] = handle
+
+	fmt.Println("BPF Filter:", l.BPFFilter)
+
+	l.Handles["pcap_file"] = packetHandle{
+		handler: handle,
+	}
 	return
 }
 
-func (l *Listener) setInterfaces() (err error) {
-	var Ifis []NetInterface
-	var ifis []net.Interface
-	ifis, err = net.Interfaces()
+func (l *Listener) activateAFPacket() error {
+	szFrame, szBlock, numBlocks, err := afpacketComputeSize(32, 32<<10, os.Getpagesize())
 	if err != nil {
 		return err
 	}
 
-	for i := 0; i < len(ifis); i++ {
-		if ifis[i].Flags&net.FlagLoopback != 0 {
-			l.loopIndex = ifis[i].Index
-		}
-		if ifis[i].Flags&net.FlagUp == 0 {
-			continue
-		}
-		var addrs []net.Addr
-		addrs, err = ifis[i].Addrs()
+	var msg string
+	for _, ifi := range l.Interfaces {
+		handle, err := newAfpacketHandle(ifi.Name, szFrame, szBlock, numBlocks, false, pcap.BlockForever)
+
 		if err != nil {
-			return err
-		}
-		if len(addrs) == 0 {
+			msg += ("\n" + err.Error())
 			continue
 		}
-		ifi := NetInterface{}
-		ifi.Interface = ifis[i]
-		ifi.IPs = make([]string, len(addrs))
-		for j, addr := range addrs {
-			ifi.IPs[j] = cutMask(addr)
+
+		if l.BPFFilter == "" {
+			l.BPFFilter = l.Filter(ifi)
+		}
+		fmt.Println("Interface:", ifi.Name, ". BPF Filter:", l.BPFFilter)
+		handle.SetBPFFilter(l.BPFFilter, 64<<10)
+
+		l.Handles[ifi.Name] = packetHandle{
+			handler: handle,
+			ips:     interfaceIPs(ifi),
 		}
-		Ifis = append(Ifis, ifi)
 	}
 
-	if listenAll(l.host) {
-		l.Interfaces = Ifis
+	if len(l.Handles) == 0 {
+		return fmt.Errorf("pcap handles error:%s", msg)
+	}
+
+	return nil
+}
+
+func (l *Listener) setInterfaces() (err error) {
+	var pifis []pcap.Interface
+	pifis, err = pcap.FindAllDevs()
+	ifis, _ := net.Interfaces()
+	if err != nil {
 		return
 	}
-	found := false
-	for _, ifi := range Ifis {
-		if isDevice(l.host, ifi) {
-			found = true
-		}
-		for _, ip := range ifi.IPs {
-			if ip == l.host {
-				found = true
-				break
-			}
-		}
-		if found {
-			l.Interfaces = []NetInterface{ifi}
+
+	for _, pi := range pifis {
+		if isDevice(l.host, pi) {
+			l.Interfaces = []pcap.Interface{pi}
 			return
 		}
+
+		var ni net.Interface
+		for _, i := range ifis {
+			if i.Name == pi.Name {
+				ni = i
+				break
+			}
+
+			addrs, _ := i.Addrs()
+			for _, a := range addrs {
+				for _, pa := range pi.Addresses {
+					if a.String() == pa.IP.String() {
+						ni = i
+						break
+					}
+				}
+			}
+		}
+
+		if ni.Flags&net.FlagLoopback != 0 {
+			l.loopIndex = ni.Index
+		}
+
+		if runtime.GOOS != "windows" {
+			if len(pi.Addresses) == 0 {
+				continue
+			}
+
+			if ni.Flags&net.FlagUp == 0 {
+				continue
+			}
+		}
+
+		l.Interfaces = append(l.Interfaces, pi)
 	}
-	err = fmt.Errorf("can not find interface with addr, name or index %s", l.host)
-	return err
+	return
 }
 
-func cutMask(addr net.Addr) string {
-	mask := addr.String()
-	for i, v := range mask {
-		if v == '/' {
-			return mask[:i]
+func isDevice(addr string, ifi pcap.Interface) bool {
+	// Windows npcap loopback have no IPs
+	if addr == "127.0.0.1" && ifi.Name == `\Device\NPF_Loopback` {
+		return true
+	}
+
+	if addr == ifi.Name {
+		return true
+	}
+
+	for _, _addr := range ifi.Addresses {
+		if _addr.IP.String() == addr {
+			return true
 		}
 	}
-	return mask
+
+	return false
 }
 
-func isDevice(addr string, ifi NetInterface) bool {
-	return addr == ifi.Name || addr == fmt.Sprintf("%d", ifi.Index) || addr == ifi.HardwareAddr.String()
+func interfaceAddresses(ifi pcap.Interface) []string {
+	var hosts []string
+	for _, addr := range ifi.Addresses {
+		hosts = append(hosts, addr.IP.String())
+	}
+	return hosts
+}
+
+func interfaceIPs(ifi pcap.Interface) []net.IP {
+	var ips []net.IP
+	for _, addr := range ifi.Addresses {
+		ips = append(ips, addr.IP)
+	}
+	return ips
 }
 
 func listenAll(addr string) bool {
@@ -498,3 +658,47 @@ func listenAll(addr string) bool {
 	}
 	return false
 }
+
+func portsFilter(transport string, direction string, ports []uint16) string {
+	if len(ports) == 0 || ports[0] == 0 {
+		return fmt.Sprintf("%s %s portrange 0-%d", transport, direction, 1<<16-1)
+	}
+
+	var filters []string
+	for _, port := range ports {
+		filters = append(filters, fmt.Sprintf("%s %s port %d", transport, direction, port))
+	}
+	return strings.Join(filters, " or ")
+}
+
+func hostsFilter(direction string, hosts []string) string {
+	var hostsFilters []string
+	for _, host := range hosts {
+		hostsFilters = append(hostsFilters, fmt.Sprintf("%s host %s", direction, host))
+	}
+
+	return strings.Join(hostsFilters, " or ")
+}
+
+func pcapLinkTypeLength(lType int) (int, bool) {
+	switch layers.LinkType(lType) {
+	case layers.LinkTypeEthernet:
+		return 14, true
+	case layers.LinkTypeNull, layers.LinkTypeLoop:
+		return 4, true
+	case layers.LinkTypeRaw, 12, 14:
+		return 0, true
+	case layers.LinkTypeIPv4, layers.LinkTypeIPv6:
+		// (TODO:) look out for IP encapsulation?
+		return 0, true
+	case layers.LinkTypeLinuxSLL:
+		return 16, true
+	case layers.LinkTypeFDDI:
+		return 13, true
+	case 226 /*DLT_IPNET*/ :
+		// https://www.tcpdump.org/linktypes/LINKTYPE_IPNET.html
+		return 24, true
+	default:
+		return 0, false
+	}
+}

capture/capture_test.go

@@ -1,297 +0,0 @@
-package capture
-
-import (
-	"context"
-	"encoding/binary"
-	"io/ioutil"
-	"net"
-	"os"
-	"testing"
-	"time"
-
-	"github.com/google/gopacket"
-	"github.com/google/gopacket/layers"
-	"github.com/google/gopacket/pcap"
-)
-
-var LoopBack = func() net.Interface {
-	ifis, _ := net.Interfaces()
-	for _, v := range ifis {
-		if v.Flags&net.FlagLoopback != 0 {
-			return v
-		}
-	}
-	return ifis[0]
-}()
-
-func TestSetInterfaces(t *testing.T) {
-	l := &Listener{}
-	l.host = "127.0.0.1"
-	l.setInterfaces()
-	if len(l.Interfaces) != 1 {
-		t.Error("expected a single interface")
-	}
-	l.host = LoopBack.HardwareAddr.String()
-	l.setInterfaces()
-	if l.Interfaces[0].Name != LoopBack.Name && len(l.Interfaces) != 1 {
-		t.Error("interface should be loop back interface")
-	}
-	l.host = ""
-	l.setInterfaces()
-	if len(l.Interfaces) < 1 {
-		t.Error("should get all interfaces")
-	}
-}
-
-func TestBPFFilter(t *testing.T) {
-	l := &Listener{}
-	l.host = "127.0.0.1"
-	l.Transport = "tcp"
-	l.setInterfaces()
-	filter := l.Filter(l.Interfaces[0])
-	if filter != "(tcp dst portrange 0-65535 and host 127.0.0.1)" {
-		t.Error("wrong filter", filter)
-	}
-	l.port = 8000
-	l.trackResponse = true
-	filter = l.Filter(l.Interfaces[0])
-	if filter != "(tcp port 8000 and host 127.0.0.1)" {
-		t.Error("wrong filter")
-	}
-}
-
-var decodeOpts = gopacket.DecodeOptions{Lazy: true, NoCopy: true}
-
-func generateHeaders(seq uint32, length uint16) (headers [44]byte) {
-	// set ethernet headers
-	binary.BigEndian.PutUint32(headers[0:4], uint32(layers.ProtocolFamilyIPv4))
-
-	// set ip header
-	ip := headers[4:]
-	copy(ip[0:2], []byte{4<<4 | 5, 0x28<<2 | 0x00})
-	binary.BigEndian.PutUint16(ip[2:4], length+54)
-	ip[9] = uint8(layers.IPProtocolTCP)
-	copy(ip[12:16], []byte{127, 0, 0, 1})
-	copy(ip[16:], []byte{127, 0, 0, 1})
-
-	// set tcp header
-	tcp := ip[20:]
-	binary.BigEndian.PutUint16(tcp[0:2], 45678)
-	binary.BigEndian.PutUint16(tcp[2:4], 8000)
-	tcp[12] = 5 << 4
-	return
-}
-
-func randomPackets(start uint32, _len int, length uint16) []gopacket.Packet {
-	var packets = make([]gopacket.Packet, _len)
-	for i := start; i < start+uint32(_len); i++ {
-		h := generateHeaders(i, length)
-		d := make([]byte, int(length)+len(h))
-		copy(d, h[0:])
-		packet := gopacket.NewPacket(d, layers.LinkTypeLoop, decodeOpts)
-		packets[i-start] = packet
-		inf := packets[i-start].Metadata()
-		_len := len(d)
-		inf.CaptureInfo = gopacket.CaptureInfo{CaptureLength: _len, Length: _len, Timestamp: time.Now()}
-	}
-	return packets
-}
-
-func TestPcapDump(t *testing.T) {
-	f, err := ioutil.TempFile("", "pcap_file")
-	if err != nil {
-		t.Error(err)
-	}
-	waiter := make(chan bool, 1)
-	h, _ := PcapDumpHandler(f, layers.LinkTypeLoop, func(level int, a ...interface{}) {
-		if level != 3 {
-			t.Errorf("expected debug level to be 3, got %d", level)
-		}
-		waiter <- true
-	})
-	packets := randomPackets(1, 5, 5)
-	for i := 0; i < len(packets); i++ {
-		if i == 1 {
-			tcp := packets[i].Data()[4:][20:]
-			// change dst port
-			binary.BigEndian.PutUint16(tcp[2:], 8001)
-		}
-		if i == 4 {
-			inf := packets[i].Metadata()
-			inf.CaptureLength = 40
-		}
-		h(packets[i])
-	}
-	<-waiter
-	name := f.Name()
-	f.Close()
-	testPcapDumpEngine(name, t)
-}
-
-func testPcapDumpEngine(f string, t *testing.T) {
-	defer os.Remove(f)
-	l, err := NewListener(f, 8000, "", EnginePcapFile, true)
-	err = l.Activate()
-	if err != nil {
-		t.Errorf("expected error to be nil, got %q", err)
-		return
-	}
-	pckts := 0
-	ctx, cancel := context.WithCancel(context.Background())
-	defer cancel()
-	err = l.Listen(ctx, func(packet gopacket.Packet) {
-		if packet.Metadata().CaptureLength != 49 {
-			t.Errorf("expected packet length to be %d, got %d", 49, packet.Metadata().CaptureLength)
-		}
-		pckts++
-	})
-
-	if err != nil {
-		t.Errorf("expected error to be nil, got %q", err)
-	}
-	if pckts != 3 {
-		t.Errorf("expected %d packets, got %d packets", 3, pckts)
-	}
-}
-
-func TestPcapHandler(t *testing.T) {
-	l, err := NewListener(LoopBack.Name, 8000, "", EnginePcap, true)
-	if err != nil {
-		t.Errorf("expected error to be nil, got %v", err)
-		return
-	}
-	err = l.Activate()
-	if err != nil {
-		t.Errorf("expected error to be nil, got %v", err)
-		return
-	}
-	defer l.Handles[LoopBack.Name].(*pcap.Handle).Close()
-	if err != nil {
-		t.Errorf("expected error to be nil, got %v", err)
-		return
-	}
-	for i := 0; i < 5; i++ {
-		_, _ = net.Dial("tcp", "127.0.0.1:8000")
-	}
-	sts, _ := l.Handles[LoopBack.Name].(*pcap.Handle).Stats()
-	if sts.PacketsReceived < 5 {
-		t.Errorf("expected >=5 packets got %d", sts.PacketsReceived)
-	}
-}
-
-func TestSocketHandler(t *testing.T) {
-	l, err := NewListener(LoopBack.Name, 8000, "", EngineRawSocket, true)
-	err = l.Activate()
-	if err != nil {
-		return
-	}
-	defer l.Handles[LoopBack.Name].(*SockRaw).Close()
-	if err != nil {
-		t.Errorf("expected error to be nil, got %v", err)
-		return
-	}
-	for i := 0; i < 5; i++ {
-		_, _ = net.Dial("tcp", "127.0.0.1:8000")
-	}
-	sts, _ := l.Handles[LoopBack.Name].(*SockRaw).Stats()
-	if sts.Packets < 5 {
-		t.Errorf("expected >=5 packets got %d", sts.Packets)
-	}
-}
-
-func BenchmarkPcapDump(b *testing.B) {
-	f, err := ioutil.TempFile("", "pcap_file")
-	if err != nil {
-		b.Error(err)
-		return
-	}
-	now := time.Now()
-	defer os.Remove(f.Name())
-	h, _ := PcapDumpHandler(f, layers.LinkTypeLoop, nil)
-	packets := randomPackets(1, b.N, 5)
-	for i := 0; i < len(packets); i++ {
-		h(packets[i])
-	}
-	f.Close()
-	b.Logf("%d packets in %s", b.N, time.Since(now))
-}
-
-func BenchmarkPcapFile(b *testing.B) {
-	f, err := ioutil.TempFile("", "pcap_file")
-	if err != nil {
-		b.Error(err)
-		return
-	}
-	defer os.Remove(f.Name())
-	h, _ := PcapDumpHandler(f, layers.LinkTypeLoop, nil)
-	packets := randomPackets(1, b.N, 5)
-	for i := 0; i < len(packets); i++ {
-		h(packets[i])
-	}
-	name := f.Name()
-	f.Close()
-	b.ResetTimer()
-	var l *Listener
-	l, err = NewListener(name, 8000, "", EnginePcapFile, true)
-	if err != nil {
-		b.Error(err)
-		return
-	}
-	err = l.Activate()
-	if err != nil {
-		b.Error(err)
-		return
-	}
-	now := time.Now()
-	pckts := 0
-	ctx, cancel := context.WithCancel(context.Background())
-	defer cancel()
-	if err = l.Listen(ctx, func(packet gopacket.Packet) {
-		if packet.Metadata().CaptureLength != 49 {
-			b.Errorf("expected packet length to be %d, got %d", 49, packet.Metadata().CaptureLength)
-		}
-		pckts++
-	}); err != nil {
-		b.Error(err)
-	}
-	b.Logf("%d/%d packets in %s", pckts, b.N, time.Since(now))
-}
-
-func BenchmarkPcap(b *testing.B) {
-	now := time.Now()
-	var err error
-
-	l, err := NewListener(LoopBack.Name, 8000, "", EnginePcap, true)
-	if err != nil {
-		b.Errorf("expected error to be nil, got %v", err)
-		return
-	}
-	err = l.Activate()
-	if err != nil {
-		b.Errorf("expected error to be nil, got %v", err)
-		return
-	}
-	defer l.Handles[LoopBack.Name].(*pcap.Handle).Close()
-	for i := 0; i < b.N; i++ {
-		_, _ = net.Dial("tcp", "127.0.0.1:8000")
-	}
-	sts, _ := l.Handles[LoopBack.Name].(*pcap.Handle).Stats()
-	b.Logf("%d packets in %s", sts.PacketsReceived, time.Since(now))
-}
-
-func BenchmarkRawSocket(b *testing.B) {
-	now := time.Now()
-	var err error
-
-	l, err := NewListener(LoopBack.Name, 0, "", EngineRawSocket, true)
-	err = l.Activate()
-	if err != nil {
-		return
-	}
-	defer l.Handles[LoopBack.Name].(*SockRaw).Close()
-	for i := 0; i < b.N; i++ {
-		_, _ = net.Dial("tcp", "127.0.0.1:8000")
-	}
-	sts, _ := l.Handles[LoopBack.Name].(*SockRaw).Stats()
-	b.Logf("%d packets in %s", sts.Packets, time.Since(now))
-}

capture/doc.go

@@ -22,7 +22,7 @@ if err := listener.Listen(context.Background(), handler); err != nil {
 	 // handle error
 }
 // or
-errCh := listener.ListenBackground(context.Background(), handler) // runs in the backgorund
+errCh := listener.ListenBackground(context.Background(), handler) // runs in the background
 select {
 case err := <- errCh:
 	// handle error

capture/dump.go

@@ -54,7 +54,7 @@ func NewWriterNanos(w io.Writer) *Writer {
 // NewWriter returns a new writer object, for writing packet data out
 // to the given writer.  If this is a new empty writer (as opposed to
 // an append), you must call WriteFileHeader before WritePacket.
-// Packet timestamps are written witn microsecond precision.
+// Packet timestamps are written with microsecond precision.
 //
 //  // Write a new file:
 //  f, _ := os.Create("/tmp/file.pcap")

capture/sock_linux.go

@@ -1,3 +1,5 @@
+// +build linux
+
 package capture
 
 import (
@@ -44,7 +46,23 @@ type SockRaw struct {
 }
 
 // NewSocket returns new M'maped sock_raw on packet version 2.
-func NewSocket(ifi net.Interface) (*SockRaw, error) {
+func NewSocket(pifi pcap.Interface) (*SockRaw, error) {
+	var ifi net.Interface
+
+	infs, _ := net.Interfaces()
+	found := false
+	for _, i := range infs {
+		if i.Name == pifi.Name {
+			ifi = i
+			found = true
+			break
+		}
+	}
+
+	if !found {
+		return nil, fmt.Errorf("can't find matching interface")
+	}
+
 	// sock create
 	fd, err := unix.Socket(unix.AF_PACKET, unix.SOCK_RAW, int(ETHALL))
 	if err != nil {
@@ -197,7 +215,7 @@ func (sock *SockRaw) GetSnapLen() int {
 func (sock *SockRaw) SetBPFFilter(expr string) error {
 	sock.mu.Lock()
 	defer sock.mu.Unlock()
-	if len(expr) == 0 {
+	if expr == "" {
 		return unix.SetsockoptInt(sock.fd, unix.SOL_SOCKET, unix.SO_DETACH_FILTER, 0)
 	}
 	filter, err := pcap.CompileBPFFilter(layers.LinkTypeEthernet, sock.snaplen, expr)
@@ -217,7 +235,7 @@ func (sock *SockRaw) SetBPFFilter(expr string) error {
 	return unix.SetsockoptSockFprog(sock.fd, unix.SOL_SOCKET, unix.SO_ATTACH_FILTER, fprog)
 }
 
-// SetPromiscuous sets promiscous mode to the required value. for better result capture on all interfaces instead.
+// SetPromiscuous sets promiscuous mode to the required value. for better result capture on all interfaces instead.
 // If it is enabled, traffic not destined for the interface will also be captured.
 func (sock *SockRaw) SetPromiscuous(b bool) error {
 	sock.mu.Lock()

capture/sock_others.go

@@ -4,10 +4,11 @@ package capture
 
 import (
 	"errors"
-	"net"
+
+	"github.com/google/gopacket/pcap"
 )
 
 // NewSocket returns new M'maped sock_raw on packet version 2.
-func NewSocket(_ net.Interface) (Socket, error) {
+func NewSocket(_ pcap.Interface) (Socket, error) {
 	return nil, errors.New("afpacket socket is only available on linux")
 }

elasticsearch.go

@@ -57,7 +57,7 @@ type ESRequestResponse struct {
 //
 // Proper format is: scheme://[userinfo@]host/index_name
 // userinfo is: user[:password]
-// net/url.Parse() does not fail if scheme is not provided but actualy does not
+// net/url.Parse() does not fail if scheme is not provided but actually does not
 // handle URI properly.
 // So we must 'validate' URI format to match requirements to use net/url.Parse()
 func parseURI(URI string) (err error, index string) {
@@ -124,6 +124,7 @@ func (p *ESPlugin) RttDurationToMs(d time.Duration) int64 {
 	return int64(fl)
 }
 
+// ResponseAnalyze send req and resp to ES
 func (p *ESPlugin) ResponseAnalyze(req, resp []byte, start, stop time.Time) {
 	if len(resp) == 0 {
 		// nil http response - skipped elasticsearch export for this request
@@ -131,7 +132,6 @@ func (p *ESPlugin) ResponseAnalyze(req, resp []byte, start, stop time.Time) {
 	}
 	t := time.Now()
 	rtt := p.RttDurationToMs(stop.Sub(start))
-	req = payloadBody(req)
 
 	esResp := ESRequestResponse{
 		ReqURL:               string(proto.Path(req)),

emitter.go

@@ -1,31 +1,29 @@
 package main
 
 import (
-	"bytes"
+	"fmt"
 	"hash/fnv"
 	"io"
 	"log"
 	"sync"
 	"time"
+
+	"github.com/buger/goreplay/byteutils"
 )
 
-type emitter struct {
-	sync.Mutex
+// Emitter represents an abject to manage plugins communication
+type Emitter struct {
 	sync.WaitGroup
-	quit    chan int
 	plugins *InOutPlugins
 }
 
-// NewEmitter creates and initializes new `emitter` object.
-func NewEmitter(quit chan int) *emitter {
-	return &emitter{
-		quit: quit,
-	}
+// NewEmitter creates and initializes new Emitter object.
+func NewEmitter() *Emitter {
+	return &Emitter{}
 }
 
 // Start initialize loop for sending data from inputs to outputs
-func (e *emitter) Start(plugins *InOutPlugins, middlewareCmd string) {
-	defer e.Wait()
+func (e *Emitter) Start(plugins *InOutPlugins, middlewareCmd string) {
 	if Settings.CopyBufferSize < 1 {
 		Settings.CopyBufferSize = 5 << 20
 	}
@@ -38,147 +36,96 @@ func (e *emitter) Start(plugins *InOutPlugins, middlewareCmd string) {
 			middleware.ReadFrom(in)
 		}
 
-		// We are going only to read responses, so using same ReadFrom method
-		for _, out := range plugins.Outputs {
-			if r, ok := out.(io.Reader); ok {
-				middleware.ReadFrom(r)
-			}
-		}
+		e.plugins.Inputs = append(e.plugins.Inputs, middleware)
+		e.plugins.All = append(e.plugins.All, middleware)
 		e.Add(1)
 		go func() {
 			defer e.Done()
-			if err := CopyMulty(e.quit, middleware, plugins.Outputs...); err != nil {
-				Debug(2, "Error during copy: ", err)
-				e.Close()
-			}
-		}()
-		go func() {
-			for {
-				select {
-				case <-e.quit:
-					middleware.Close()
-					return
-				}
+			if err := CopyMulty(middleware, plugins.Outputs...); err != nil {
+				Debug(2, fmt.Sprintf("[EMITTER] error during copy: %q", err))
 			}
 		}()
 	} else {
 		for _, in := range plugins.Inputs {
 			e.Add(1)
-			go func(in io.Reader) {
+			go func(in PluginReader) {
 				defer e.Done()
-				if err := CopyMulty(e.quit, in, plugins.Outputs...); err != nil {
-					Debug(2, "Error during copy: ", err)
-					e.Close()
+				if err := CopyMulty(in, plugins.Outputs...); err != nil {
+					Debug(2, fmt.Sprintf("[EMITTER] error during copy: %q", err))
 				}
 			}(in)
 		}
-
-		for _, out := range plugins.Outputs {
-			if r, ok := out.(io.Reader); ok {
-				e.Add(1)
-				go func(r io.Reader) {
-					defer e.Done()
-					if err := CopyMulty(e.quit, r, plugins.Outputs...); err != nil {
-						Debug(2, "Error during copy: ", err)
-						e.Close()
-					}
-				}(r)
-			}
-		}
-	}
-}
-
-func (e *emitter) close() {
-	select {
-	case <-e.quit:
-	default:
-		close(e.quit)
 	}
 }
 
 // Close closes all the goroutine and waits for it to finish.
-func (e *emitter) Close() {
-	e.close()
-	for _, p := range e.plugins.Inputs {
+func (e *Emitter) Close() {
+	for _, p := range e.plugins.All {
 		if cp, ok := p.(io.Closer); ok {
 			cp.Close()
 		}
 	}
-	for _, p := range e.plugins.Outputs {
-		if cp, ok := p.(io.Closer); ok {
-			cp.Close()
-		}
+	if len(e.plugins.All) > 0 {
+		// wait for everything to stop
+		e.Wait()
 	}
-	e.plugins = nil // avoid further accidental usage
+	e.plugins.All = nil // avoid Close to make changes again
 }
 
 // CopyMulty copies from 1 reader to multiple writers
-func CopyMulty(stop chan int, src io.Reader, writers ...io.Writer) error {
-	buf := make([]byte, Settings.CopyBufferSize)
+func CopyMulty(src PluginReader, writers ...PluginWriter) error {
 	wIndex := 0
 	modifier := NewHTTPModifier(&Settings.ModifierConfig)
-	filteredRequests := make(map[string]time.Time)
-	filteredRequestsLastCleanTime := time.Now()
+	filteredRequests := make(map[string]int64)
+	filteredRequestsLastCleanTime := time.Now().UnixNano()
+	filteredCount := 0
 
-	i := 0
 	for {
-		var nr int
-		nr, err := src.Read(buf)
-
-		select {
-		case <-stop:
-			return nil
-		default:
-		}
+		msg, err := src.PluginRead()
 		if err != nil {
+			if err == ErrorStopped || err == io.EOF {
+				return nil
+			}
 			return err
 		}
-
-		_maxN := nr
-		if nr > 500 {
-			_maxN = 500
-		}
-		if nr > 0 {
-			payload := buf[:nr]
-			meta := payloadMeta(payload)
+		if msg != nil && len(msg.Data) > 0 {
+			if len(msg.Data) > int(Settings.CopyBufferSize) {
+				msg.Data = msg.Data[:Settings.CopyBufferSize]
+			}
+			meta := payloadMeta(msg.Meta)
 			if len(meta) < 3 {
-				Debug(2, "[EMITTER] Found malformed record", string(payload[0:_maxN]), nr, "from:", src)
+				Debug(2, fmt.Sprintf("[EMITTER] Found malformed record %q from %q", msg.Meta, src))
 				continue
 			}
-			requestID := string(meta[1])
-
-			Debug(3, "[EMITTER] input:", string(payload[0:_maxN]), nr, "from:", src)
-
+			requestID := byteutils.SliceToString(meta[1])
+			// start a subroutine only when necessary
+			if Settings.Verbose >= 3 {
+				Debug(3, "[EMITTER] input: ", byteutils.SliceToString(msg.Meta[:len(msg.Meta)-1]), " from: ", src)
+			}
 			if modifier != nil {
-				if isRequestPayload(payload) {
-					headSize := bytes.IndexByte(payload, '\n') + 1
-					body := payload[headSize:]
-					originalBodyLen := len(body)
-					body = modifier.Rewrite(body)
-
+				Debug(3, "[EMITTER] modifier:", requestID, "from:", src)
+				if isRequestPayload(msg.Meta) {
+					msg.Data = modifier.Rewrite(msg.Data)
 					// If modifier tells to skip request
-					if len(body) == 0 {
-						filteredRequests[requestID] = time.Now()
+					if len(msg.Data) == 0 {
+						filteredRequests[requestID] = time.Now().UnixNano()
+						filteredCount++
 						continue
 					}
-
-					if originalBodyLen != len(body) {
-						payload = append(payload[:headSize], body...)
-					}
-
-					Debug(3, "[EMITTER] Rewritten input:", len(payload), "First %d bytes:", _maxN, string(payload[0:_maxN]))
+					Debug(3, "[EMITTER] Rewritten input:", requestID, "from:", src)
 
 				} else {
 					if _, ok := filteredRequests[requestID]; ok {
 						delete(filteredRequests, requestID)
+						filteredCount--
 						continue
 					}
 				}
 			}
 
 			if Settings.PrettifyHTTP {
-				payload = prettifyHTTP(payload)
-				if len(payload) == 0 {
+				msg.Data = prettifyHTTP(msg.Data)
+				if len(msg.Data) == 0 {
 					continue
 				}
 			}
@@ -189,27 +136,23 @@ func CopyMulty(stop chan int, src io.Reader, writers ...io.Writer) error {
 						log.Fatal("Detailed TCP sessions work only with PRO license")
 					}
 					hasher := fnv.New32a()
-					// First 20 bytes contain tcp session
-					id := payloadID(payload)
-					hasher.Write(id)
+					hasher.Write(meta[1])
 
 					wIndex = int(hasher.Sum32()) % len(writers)
-					writers[wIndex].Write(payload)
+					if _, err := writers[wIndex].PluginWrite(msg); err != nil {
+						return err
+					}
 				} else {
 					// Simple round robin
-					if _, err := writers[wIndex].Write(payload); err != nil {
+					if _, err := writers[wIndex].PluginWrite(msg); err != nil {
 						return err
 					}
 
-					wIndex++
-
-					if wIndex >= len(writers) {
-						wIndex = 0
-					}
+					wIndex = (wIndex + 1) % len(writers)
 				}
 			} else {
 				for _, dst := range writers {
-					if _, err := dst.Write(payload); err != nil {
+					if _, err := dst.PluginWrite(msg); err != nil && err != io.ErrClosedPipe {
 						return err
 					}
 				}
@@ -217,19 +160,18 @@ func CopyMulty(stop chan int, src io.Reader, writers ...io.Writer) error {
 		}
 
 		// Run GC on each 1000 request
-		if i%1000 == 0 {
+		if filteredCount > 0 && filteredCount%1000 == 0 {
 			// Clean up filtered requests for which we didn't get a response to filter
-			now := time.Now()
-			if now.Sub(filteredRequestsLastCleanTime) > 60*time.Second {
+			now := time.Now().UnixNano()
+			if now-filteredRequestsLastCleanTime > int64(60*time.Second) {
 				for k, v := range filteredRequests {
-					if now.Sub(v) > 60*time.Second {
+					if now-v > int64(60*time.Second) {
 						delete(filteredRequests, k)
+						filteredCount--
 					}
 				}
-				filteredRequestsLastCleanTime = time.Now()
+				filteredRequestsLastCleanTime = time.Now().UnixNano()
 			}
 		}
-
-		i++
 	}
 }

emitter_test.go

@@ -2,7 +2,6 @@ package main
 
 import (
 	"fmt"
-	"io"
 	"os"
 	"sync"
 	"sync/atomic"
@@ -18,20 +17,19 @@ func TestMain(m *testing.M) {
 
 func TestEmitter(t *testing.T) {
 	wg := new(sync.WaitGroup)
-	quit := make(chan int)
 
 	input := NewTestInput()
-	output := NewTestOutput(func(data []byte) {
+	output := NewTestOutput(func(*Message) {
 		wg.Done()
 	})
 
 	plugins := &InOutPlugins{
-		Inputs:  []io.Reader{input},
-		Outputs: []io.Writer{output},
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{output},
 	}
 	plugins.All = append(plugins.All, input, output)
 
-	emitter := NewEmitter(quit)
+	emitter := NewEmitter()
 	go emitter.Start(plugins, Settings.Middleware)
 
 	for i := 0; i < 1000; i++ {
@@ -45,32 +43,31 @@ func TestEmitter(t *testing.T) {
 
 func TestEmitterFiltered(t *testing.T) {
 	wg := new(sync.WaitGroup)
-	quit := make(chan int)
 
 	input := NewTestInput()
 	input.skipHeader = true
 
-	output := NewTestOutput(func(data []byte) {
+	output := NewTestOutput(func(*Message) {
 		wg.Done()
 	})
 
 	plugins := &InOutPlugins{
-		Inputs:  []io.Reader{input},
-		Outputs: []io.Writer{output},
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{output},
 	}
 	plugins.All = append(plugins.All, input, output)
 
 	methods := HTTPMethods{[]byte("GET")}
 	Settings.ModifierConfig = HTTPModifierConfig{Methods: methods}
 
-	emitter := &emitter{quit: quit}
+	emitter := &Emitter{}
 	go emitter.Start(plugins, "")
 
 	wg.Add(2)
 
 	id := uuid()
 	reqh := payloadHeader(RequestPayload, id, time.Now().UnixNano(), -1)
-	reqb := append(reqh, []byte("GET / HTTP/1.1\r\nHost: www.w3.org\r\nUser-Agent: Go 1.1 package http\r\nAccept-Encoding: gzip\r\n\r\n")...)
+	reqb := append(reqh, []byte("POST / HTTP/1.1\r\nHost: www.w3.org\r\nUser-Agent: Go 1.1 package http\r\nAccept-Encoding: gzip\r\n\r\n")...)
 
 	resh := payloadHeader(ResponsePayload, id, time.Now().UnixNano()+1, 1)
 	respb := append(resh, []byte("HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n")...)
@@ -80,7 +77,7 @@ func TestEmitterFiltered(t *testing.T) {
 
 	id = uuid()
 	reqh = payloadHeader(RequestPayload, id, time.Now().UnixNano(), -1)
-	reqb = append(reqh, []byte("POST / HTTP/1.1\r\nHost: www.w3.org\r\nUser-Agent: Go 1.1 package http\r\nAccept-Encoding: gzip\r\n\r\n")...)
+	reqb = append(reqh, []byte("GET / HTTP/1.1\r\nHost: www.w3.org\r\nUser-Agent: Go 1.1 package http\r\nAccept-Encoding: gzip\r\n\r\n")...)
 
 	resh = payloadHeader(ResponsePayload, id, time.Now().UnixNano()+1, 1)
 	respb = append(resh, []byte("HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n")...)
@@ -96,30 +93,29 @@ func TestEmitterFiltered(t *testing.T) {
 
 func TestEmitterSplitRoundRobin(t *testing.T) {
 	wg := new(sync.WaitGroup)
-	quit := make(chan int)
 
 	input := NewTestInput()
 
 	var counter1, counter2 int32
 
-	output1 := NewTestOutput(func(data []byte) {
+	output1 := NewTestOutput(func(*Message) {
 		atomic.AddInt32(&counter1, 1)
 		wg.Done()
 	})
 
-	output2 := NewTestOutput(func(data []byte) {
+	output2 := NewTestOutput(func(*Message) {
 		atomic.AddInt32(&counter2, 1)
 		wg.Done()
 	})
 
 	plugins := &InOutPlugins{
-		Inputs:  []io.Reader{input},
-		Outputs: []io.Writer{output1, output2},
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{output1, output2},
 	}
 
 	Settings.SplitOutput = true
 
-	emitter := NewEmitter(quit)
+	emitter := NewEmitter()
 	go emitter.Start(plugins, Settings.Middleware)
 
 	for i := 0; i < 1000; i++ {
@@ -140,31 +136,30 @@ func TestEmitterSplitRoundRobin(t *testing.T) {
 
 func TestEmitterRoundRobin(t *testing.T) {
 	wg := new(sync.WaitGroup)
-	quit := make(chan int)
 
 	input := NewTestInput()
 
 	var counter1, counter2 int32
 
-	output1 := NewTestOutput(func(data []byte) {
-		atomic.AddInt32(&counter1, 1)
+	output1 := NewTestOutput(func(*Message) {
+		counter1++
 		wg.Done()
 	})
 
-	output2 := NewTestOutput(func(data []byte) {
-		atomic.AddInt32(&counter2, 1)
+	output2 := NewTestOutput(func(*Message) {
+		counter2++
 		wg.Done()
 	})
 
 	plugins := &InOutPlugins{
-		Inputs:  []io.Reader{input},
-		Outputs: []io.Writer{output1, output2},
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{output1, output2},
 	}
 	plugins.All = append(plugins.All, input, output1, output2)
 
 	Settings.SplitOutput = true
 
-	emitter := NewEmitter(quit)
+	emitter := NewEmitter()
 	go emitter.Start(plugins, Settings.Middleware)
 
 	for i := 0; i < 1000; i++ {
@@ -186,36 +181,34 @@ func TestEmitterSplitSession(t *testing.T) {
 	wg := new(sync.WaitGroup)
 	wg.Add(200)
 
-	quit := make(chan int)
-
 	input := NewTestInput()
 	input.skipHeader = true
 
 	var counter1, counter2 int32
 
-	output1 := NewTestOutput(func(data []byte) {
-		if payloadID(data)[0] == 'a' {
-			atomic.AddInt32(&counter1, 1)
+	output1 := NewTestOutput(func(msg *Message) {
+		if payloadID(msg.Meta)[0] == 'a' {
+			counter1++
 		}
 		wg.Done()
 	})
 
-	output2 := NewTestOutput(func(data []byte) {
-		if payloadID(data)[0] == 'b' {
-			atomic.AddInt32(&counter2, 1)
+	output2 := NewTestOutput(func(msg *Message) {
+		if payloadID(msg.Meta)[0] == 'b' {
+			counter2++
 		}
 		wg.Done()
 	})
 
 	plugins := &InOutPlugins{
-		Inputs:  []io.Reader{input},
-		Outputs: []io.Writer{output1, output2},
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{output1, output2},
 	}
 
 	Settings.SplitOutput = true
 	Settings.RecognizeTCPSessions = true
 
-	emitter := NewEmitter(quit)
+	emitter := NewEmitter()
 	go emitter.Start(plugins, Settings.Middleware)
 
 	for i := 0; i < 200; i++ {
@@ -242,21 +235,20 @@ func TestEmitterSplitSession(t *testing.T) {
 
 func BenchmarkEmitter(b *testing.B) {
 	wg := new(sync.WaitGroup)
-	quit := make(chan int)
 
 	input := NewTestInput()
 
-	output := NewTestOutput(func(data []byte) {
+	output := NewTestOutput(func(*Message) {
 		wg.Done()
 	})
 
 	plugins := &InOutPlugins{
-		Inputs:  []io.Reader{input},
-		Outputs: []io.Writer{output},
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{output},
 	}
 	plugins.All = append(plugins.All, input, output)
 
-	emitter := NewEmitter(quit)
+	emitter := NewEmitter()
 	go emitter.Start(plugins, Settings.Middleware)
 
 	b.ResetTimer()

examples/middleware/echo.sh

@@ -8,7 +8,7 @@
 # 
 
 function log {
-    if $GOR_TEST != ""; then # if we are not testing
+    if [[ ! -v GOR_TEST ]]; then # if we are not testing
     # Logging to stderr, because stdout/stdin used for data transfer
     >&2 echo "[DEBUG][ECHO] $1"
     fi

examples/middleware/token_modifier.go

@@ -115,7 +115,7 @@ func encode(buf []byte) []byte {
 }
 
 func Debug(args ...interface{}) {
-	if os.Getenv("GOR_TEST") != "" { // if we are not testing
+	if os.Getenv("GOR_TEST") == "" { // if we are not testing
 		fmt.Fprint(os.Stderr, "[DEBUG][TOKEN-MOD] ")
 		fmt.Fprintln(os.Stderr, args...)
 	}

go.mod

@@ -1,6 +1,6 @@
 module github.com/buger/goreplay
 
-go 1.14
+go 1.15
 
 require (
 	github.com/Shopify/sarama v1.26.4
@@ -8,12 +8,13 @@ require (
 	github.com/aws/aws-sdk-go v1.33.2
 	github.com/bitly/go-hostpool v0.1.0 // indirect
 	github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 // indirect
-	github.com/google/gopacket v1.1.18
+	github.com/google/gopacket v1.1.20-0.20210429153827-3eaba0894325
 	github.com/klauspost/compress v1.10.10 // indirect
 	github.com/mattbaird/elastigo v0.0.0-20170123220020-2fe47fd29e4b
 	github.com/pierrec/lz4 v2.5.2+incompatible // indirect
 	github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0 // indirect
 	github.com/smartystreets/goconvey v1.6.4 // indirect
-	golang.org/x/net v0.0.0-20200707034311-ab3426394381 // indirect
+	github.com/stretchr/testify v1.5.1
+	golang.org/x/net v0.0.0-20200707034311-ab3426394381
 	golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd
 )

go.sum

@@ -10,7 +10,6 @@ github.com/bitly/go-hostpool v0.1.0 h1:XKmsF6k5el6xHG3WPJ8U0Ku/ye7njX7W81Ng7O2io
 github.com/bitly/go-hostpool v0.1.0/go.mod h1:4gOCgp6+NZnVqlKyZ/iBZFTAJKembaVENUpMkpg42fw=
 github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 h1:DDGfHa7BWjL4YnC6+E63dPcxHo2sUxDIu8g3QgEJdRY=
 github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4=
-github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -30,8 +29,8 @@ github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEW
 github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4=
 github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/gopacket v1.1.18 h1:lum7VRA9kdlvBi7/v2p7/zcbkduHaCH/SVVyurs7OpY=
-github.com/google/gopacket v1.1.18/go.mod h1:UdDNZ1OO62aGYVnPhxT1U6aI7ukYtA/kB8vaU0diBUM=
+github.com/google/gopacket v1.1.20-0.20210429153827-3eaba0894325 h1:YmIcZ5Var3BAQ64AW98Iiys5Ih4fiU0xK41+8isC5Ec=
+github.com/google/gopacket v1.1.20-0.20210429153827-3eaba0894325/go.mod h1:riddUzxTSBpJXk3qBHtYr4qOhFhT6k/1c0E3qkQjQpA=
 github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8=
 github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
 github.com/hashicorp/go-uuid v1.0.2 h1:cfejS+Tpcp13yd5nYHWDI6qVCny6wyX2Mt5SGur2IGE=
@@ -42,7 +41,6 @@ github.com/jmespath/go-jmespath v0.3.0 h1:OS12ieG61fsCg5+qLJ+SsW9NicxNkg3b25OyT2
 github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik=
 github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
 github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
-github.com/klauspost/compress v1.9.8 h1:VMAMUUOh+gaxKTMk+zqbjsSjsIcUcL/LF4o63i82QyA=
 github.com/klauspost/compress v1.9.8/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
 github.com/klauspost/compress v1.10.10 h1:a/y8CglcM7gLGYmlbP/stPE5sR3hbhFRUjCBfd/0B3I=
 github.com/klauspost/compress v1.10.10/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
@@ -71,33 +69,40 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4=
 github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
+github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE=
+github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU=
+github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0=
 github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I=
 github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/crypto v0.0.0-20200204104054-c9f3fb736b72 h1:+ELyKg6m8UBf0nPFSqD0mi7zUfwPyXo23HNjMnXPz7w=
+golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20200204104054-c9f3fb736b72/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
+golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20200202094626-16171245cfb2 h1:CCH4IOTTfewWjGOlSp+zGcjutRKlBEZQ6wTn8ozI/nI=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200707034311-ab3426394381 h1:VXak5I6aEWmAXeQjA+QSZzlgNrpq9mjcfDemuexIKsU=
 golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20190405154228-4b34438f7a67/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190606203320-7fc4e5ec1444/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200217220822-9197077df867/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884=
 golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
@@ -111,7 +116,6 @@ gopkg.in/jcmturner/gokrb5.v7 v7.5.0 h1:a9tsXlIDD9SKxotJMK3niV7rPZAJeX2aD/0yg3qlI
 gopkg.in/jcmturner/gokrb5.v7 v7.5.0/go.mod h1:l8VISx+WGYp+Fp7KRbsiUuXTTOnxIc3Tuvyavf11/WM=
 gopkg.in/jcmturner/rpc.v1 v1.1.0 h1:QHIUxTX1ISuAv9dD2wJ9HWQVuWDX/Zc0PfeC2tjc4rU=
 gopkg.in/jcmturner/rpc.v1 v1.1.0/go.mod h1:YIdkC4XfD6GXbzje11McwsDuOlZQSb9W4vfLvuNnlv8=
-gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=
 gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

gor.go

@@ -3,16 +3,16 @@
 package main
 
 import (
+	"expvar"
 	"flag"
 	"fmt"
 	"log"
 	"net/http"
 	"net/http/httputil"
-	_ "net/http/pprof"
+	httppptof "net/http/pprof"
 	"os"
 	"os/signal"
 	"runtime"
-	_ "runtime/debug"
 	"runtime/pprof"
 	"syscall"
 	"time"
@@ -23,8 +23,42 @@ var (
 	memprofile = flag.String("memprofile", "", "write memory profile to this file")
 )
 
-func loggingMiddleware(next http.Handler) http.Handler {
+func init() {
+	var defaultServeMux http.ServeMux
+	http.DefaultServeMux = &defaultServeMux
+
+	http.HandleFunc("/debug/vars", func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "application/json; charset=utf-8")
+		fmt.Fprintf(w, "{\n")
+		first := true
+		expvar.Do(func(kv expvar.KeyValue) {
+			if kv.Key == "memstats" || kv.Key == "cmdline" {
+				return
+			}
+
+			if !first {
+				fmt.Fprintf(w, ",\n")
+			}
+			first = false
+			fmt.Fprintf(w, "%q: %s", kv.Key, kv.Value)
+		})
+		fmt.Fprintf(w, "\n}\n")
+	})
+
+	http.HandleFunc("/debug/pprof/", httppptof.Index)
+	http.HandleFunc("/debug/pprof/cmdline", httppptof.Cmdline)
+	http.HandleFunc("/debug/pprof/profile", httppptof.Profile)
+	http.HandleFunc("/debug/pprof/symbol", httppptof.Symbol)
+	http.HandleFunc("/debug/pprof/trace", httppptof.Trace)
+}
+
+func loggingMiddleware(addr string, next http.Handler) http.Handler {
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path == "/loop" {
+			_, err := http.Get("http://" + addr)
+			log.Println(err)
+		}
+
 		rb, _ := httputil.DumpRequest(r, false)
 		log.Println(string(rb))
 		next.ServeHTTP(w, r)
@@ -32,7 +66,7 @@ func loggingMiddleware(next http.Handler) http.Handler {
 }
 
 func main() {
-	if len(os.Getenv("GOMAXPROCS")) == 0 {
+	if os.Getenv("GOMAXPROCS") == "" {
 		runtime.GOMAXPROCS(runtime.NumCPU() * 2)
 	}
 
@@ -46,7 +80,7 @@ func main() {
 
 		Debug(0, "Started example file server for current directory on address ", args[1])
 
-		log.Fatal(http.ListenAndServe(args[1], loggingMiddleware(http.FileServer(http.Dir(dir)))))
+		log.Fatal(http.ListenAndServe(args[1], loggingMiddleware(args[1], http.FileServer(http.Dir(dir)))))
 	} else {
 		flag.Parse()
 		checkSettings()
@@ -74,13 +108,13 @@ func main() {
 	}
 
 	closeCh := make(chan int)
-	emitter := NewEmitter(closeCh)
+	emitter := NewEmitter()
 	go emitter.Start(plugins, Settings.Middleware)
 	if Settings.ExitAfter > 0 {
 		log.Printf("Running gor for a duration of %s\n", Settings.ExitAfter)
 
 		time.AfterFunc(Settings.ExitAfter, func() {
-			fmt.Printf("gor run timeout %s\n", Settings.ExitAfter)
+			log.Printf("gor run timeout %s\n", Settings.ExitAfter)
 			close(closeCh)
 		})
 	}
@@ -95,7 +129,6 @@ func main() {
 	}
 	emitter.Close()
 	os.Exit(exit)
-
 }
 
 func profileCPU(cpuprofile string) {

http_modifier_settings.go

@@ -19,10 +19,9 @@ type HTTPModifierConfig struct {
 	HeaderBasicAuthFilters HTTPHeaderBasicAuthFilters `json:"http-basic-auth-filter"`
 	HeaderHashFilters      HTTPHashFilters            `json:"http-header-limiter"`
 	ParamHashFilters       HTTPHashFilters            `json:"http-param-limiter"`
-
-	Params  HTTPParams  `json:"http-set-param"`
-	Headers HTTPHeaders `json:"http-set-header"`
-	Methods HTTPMethods `json:"http-allow-method"`
+	Params                 HTTPParams                 `json:"http-set-param"`
+	Headers                HTTPHeaders                `json:"http-set-header"`
+	Methods                HTTPMethods                `json:"http-allow-method"`
 }
 
 //
@@ -40,10 +39,11 @@ func (h *HTTPHeaderFilters) String() string {
 	return fmt.Sprint(*h)
 }
 
+// Set method to implement flags.Value
 func (h *HTTPHeaderFilters) Set(value string) error {
 	valArr := strings.SplitN(value, ":", 2)
 	if len(valArr) < 2 {
-		return errors.New("need both header and value, colon-delimited (ex. user_id:^169$).")
+		return errors.New("need both header and value, colon-delimited (ex. user_id:^169$)")
 	}
 	val := strings.TrimSpace(valArr[1])
 	r, err := regexp.Compile(val)
@@ -63,13 +63,14 @@ type basicAuthFilter struct {
 	regexp *regexp.Regexp
 }
 
-// HTTPHeaderFilters holds list of headers and their regexps
+// HTTPHeaderBasicAuthFilters holds list of regxp to match basic Auth header values
 type HTTPHeaderBasicAuthFilters []basicAuthFilter
 
 func (h *HTTPHeaderBasicAuthFilters) String() string {
 	return fmt.Sprint(*h)
 }
 
+// Set method to implement flags.Value
 func (h *HTTPHeaderBasicAuthFilters) Set(value string) error {
 	r, err := regexp.Compile(value)
 	if err != nil {
@@ -89,12 +90,14 @@ type hashFilter struct {
 	percent uint32
 }
 
+// HTTPHashFilters represents a slice of header hash filters
 type HTTPHashFilters []hashFilter
 
 func (h *HTTPHashFilters) String() string {
 	return fmt.Sprint(*h)
 }
 
+// Set method to implement flags.Value
 func (h *HTTPHashFilters) Set(value string) error {
 	valArr := strings.SplitN(value, ":", 2)
 	if len(valArr) < 2 {
@@ -129,23 +132,26 @@ func (h *HTTPHashFilters) Set(value string) error {
 //
 // Handling of --http-set-header option
 //
-type HTTPHeaders []HTTPHeader
-type HTTPHeader struct {
+type httpHeader struct {
 	Name  string
 	Value string
 }
 
+// HTTPHeaders is a slice of headers that must appended
+type HTTPHeaders []httpHeader
+
 func (h *HTTPHeaders) String() string {
 	return fmt.Sprint(*h)
 }
 
+// Set method to implement flags.Value
 func (h *HTTPHeaders) Set(value string) error {
 	v := strings.SplitN(value, ":", 2)
 	if len(v) != 2 {
 		return errors.New("Expected `Key: Value`")
 	}
 
-	header := HTTPHeader{
+	header := httpHeader{
 		strings.TrimSpace(v[0]),
 		strings.TrimSpace(v[1]),
 	}
@@ -157,23 +163,26 @@ func (h *HTTPHeaders) Set(value string) error {
 //
 // Handling of --http-set-param option
 //
-type HTTPParams []HTTPParam
-type HTTPParam struct {
+type httpParam struct {
 	Name  []byte
 	Value []byte
 }
 
+// HTTPParams filters for --http-set-param
+type HTTPParams []httpParam
+
 func (h *HTTPParams) String() string {
 	return fmt.Sprint(*h)
 }
 
+// Set method to implement flags.Value
 func (h *HTTPParams) Set(value string) error {
 	v := strings.SplitN(value, "=", 2)
 	if len(v) != 2 {
 		return errors.New("Expected `Key=Value`")
 	}
 
-	param := HTTPParam{
+	param := httpParam{
 		[]byte(strings.TrimSpace(v[0])),
 		[]byte(strings.TrimSpace(v[1])),
 	}
@@ -185,12 +194,15 @@ func (h *HTTPParams) Set(value string) error {
 //
 // Handling of --http-allow-method option
 //
+
+// HTTPMethods holds values for method allowed
 type HTTPMethods [][]byte
 
 func (h *HTTPMethods) String() string {
 	return fmt.Sprint(*h)
 }
 
+// Set method to implement flags.Value
 func (h *HTTPMethods) Set(value string) error {
 	*h = append(*h, []byte(value))
 	return nil
@@ -204,12 +216,14 @@ type urlRewrite struct {
 	target []byte
 }
 
+// URLRewriteMap holds regexp and data to modify URL
 type URLRewriteMap []urlRewrite
 
 func (r *URLRewriteMap) String() string {
 	return fmt.Sprint(*r)
 }
 
+// Set method to implement flags.Value
 func (r *URLRewriteMap) Set(value string) error {
 	valArr := strings.SplitN(value, ":", 2)
 	if len(valArr) < 2 {
@@ -232,12 +246,14 @@ type headerRewrite struct {
 	target []byte
 }
 
+// HeaderRewriteMap holds regexp and data to rewrite headers
 type HeaderRewriteMap []headerRewrite
 
 func (r *HeaderRewriteMap) String() string {
 	return fmt.Sprint(*r)
 }
 
+// Set method to implement flags.Value
 func (r *HeaderRewriteMap) Set(value string) error {
 	headerArr := strings.SplitN(value, ":", 2)
 	if len(headerArr) < 2 {
@@ -266,12 +282,14 @@ type urlRegexp struct {
 	regexp *regexp.Regexp
 }
 
+// HTTPURLRegexp a slice of regexp to match URLs
 type HTTPURLRegexp []urlRegexp
 
 func (r *HTTPURLRegexp) String() string {
 	return fmt.Sprint(*r)
 }
 
+// Set method to implement flags.Value
 func (r *HTTPURLRegexp) Set(value string) error {
 	regexp, err := regexp.Compile(value)
 

http_prettifier.go

@@ -12,25 +12,22 @@ import (
 )
 
 func prettifyHTTP(p []byte) []byte {
-	headSize := bytes.IndexByte(p, '\n') + 1
-	head := p[:headSize]
-	body := p[headSize:]
 
-	tEnc := bytes.Equal(proto.Header(body, []byte("Transfer-Encoding")), []byte("chunked"))
-	cEnc := bytes.Equal(proto.Header(body, []byte("Content-Encoding")), []byte("gzip"))
+	tEnc := bytes.Equal(proto.Header(p, []byte("Transfer-Encoding")), []byte("chunked"))
+	cEnc := bytes.Equal(proto.Header(p, []byte("Content-Encoding")), []byte("gzip"))
 
 	if !(tEnc || cEnc) {
 		return p
 	}
 
-	headersPos := proto.MIMEHeadersEndPos(body)
+	headersPos := proto.MIMEHeadersEndPos(p)
 
-	if headersPos < 5 || headersPos > len(body) {
+	if headersPos < 5 || headersPos > len(p) {
 		return p
 	}
 
-	headers := body[:headersPos]
-	content := body[headersPos:]
+	headers := p[:headersPos]
+	content := p[headersPos:]
 
 	if tEnc {
 		buf := bytes.NewReader(content)
@@ -64,7 +61,7 @@ func prettifyHTTP(p []byte) []byte {
 		headers = proto.SetHeader(headers, []byte("Content-Length"), []byte(newLen))
 	}
 
-	newPayload := append(append(head, headers...), content...)
+	newPayload := append(headers, content...)
 
 	return newPayload
 }

http_prettifier_test.go

@@ -5,6 +5,8 @@ import (
 	"compress/gzip"
 	"strconv"
 	"testing"
+
+	"github.com/buger/goreplay/proto"
 )
 
 func TestHTTPPrettifierGzip(t *testing.T) {
@@ -15,22 +17,21 @@ func TestHTTPPrettifierGzip(t *testing.T) {
 
 	size := strconv.Itoa(len(b.Bytes()))
 
-	payload := []byte("2 1 1\nHTTP/1.1 200 OK\r\nContent-Length: " + size + "\r\nContent-Encoding: gzip\r\n\r\n")
+	payload := []byte("HTTP/1.1 200 OK\r\nContent-Length: " + size + "\r\nContent-Encoding: gzip\r\n\r\n")
 	payload = append(payload, b.Bytes()...)
 
 	newPayload := prettifyHTTP(payload)
 
-	if string(newPayload) != "2 1 1\nHTTP/1.1 200 OK\r\nContent-Length: 4\r\n\r\ntest" {
-		t.Error("Payload not match:", string(newPayload))
+	if string(newPayload) != "HTTP/1.1 200 OK\r\nContent-Length: 4\r\n\r\ntest" {
+		t.Errorf("Payload not match %q", string(newPayload))
 	}
 }
 
 func TestHTTPPrettifierChunked(t *testing.T) {
 	payload := []byte("POST / HTTP/1.1\r\nHost: www.w3.org\r\nTransfer-Encoding: chunked\r\n\r\n4\r\nWiki\r\n5\r\npedia\r\ne\r\n in\r\n\r\nchunks.\r\n0\r\n\r\n")
 
-	newPayload := prettifyHTTP(payload)
-
-	if string(newPayload) != "POST / HTTP/1.1\r\nHost: www.w3.org\r\nContent-Length: 23\r\n\r\nWikipedia in\r\n\r\nchunks." {
-		t.Error("Payload not match:", string(newPayload))
+	payload = prettifyHTTP(payload)
+	if string(proto.Header(payload, []byte("Content-Length"))) != "23" {
+		t.Errorf("payload should have content length of 23")
 	}
 }

input_dummy.go

@@ -7,24 +7,30 @@ import (
 // DummyInput used for debugging. It generate 1 "GET /"" request per second.
 type DummyInput struct {
 	data chan []byte
+	quit chan struct{}
 }
 
 // NewDummyInput constructor for DummyInput
 func NewDummyInput(options string) (di *DummyInput) {
 	di = new(DummyInput)
 	di.data = make(chan []byte)
+	di.quit = make(chan struct{})
 
 	go di.emit()
 
 	return
 }
 
-func (i *DummyInput) Read(data []byte) (int, error) {
-	buf := <-i.data
-
-	copy(data, buf)
-
-	return len(buf), nil
+// PluginRead reads message from this plugin
+func (i *DummyInput) PluginRead() (*Message, error) {
+	var msg Message
+	select {
+	case <-i.quit:
+		return nil, ErrorStopped
+	case buf := <-i.data:
+		msg.Meta, msg.Data = payloadMetaWithBody(buf)
+		return &msg, nil
+	}
 }
 
 func (i *DummyInput) emit() {
@@ -46,3 +52,9 @@ func (i *DummyInput) emit() {
 func (i *DummyInput) String() string {
 	return "Dummy Input"
 }
+
+// Close closes this plugins
+func (i *DummyInput) Close() error {
+	close(i.quit)
+	return nil
+}

input_file.go

@@ -4,9 +4,12 @@ import (
 	"bufio"
 	"bytes"
 	"compress/gzip"
+	"container/heap"
 	"errors"
+	"expvar"
+	"fmt"
 	"io"
-	"log"
+	"math"
 	"os"
 	"path/filepath"
 	"strconv"
@@ -20,27 +23,73 @@ import (
 	"github.com/aws/aws-sdk-go/service/s3"
 )
 
-type fileInputReader struct {
-	reader    *bufio.Reader
+type filePayload struct {
 	data      []byte
-	file      io.ReadCloser
 	timestamp int64
-	closed    int32 // Value of 0 indicates that the file is still open.
-	s3        bool
 }
 
-func (f *fileInputReader) parseNext() error {
+// An IntHeap is a min-heap of ints.
+type payloadQueue struct {
+	sync.RWMutex
+	s []*filePayload
+}
+
+func (h payloadQueue) Len() int           { return len(h.s) }
+func (h payloadQueue) Less(i, j int) bool { return h.s[i].timestamp < h.s[j].timestamp }
+func (h payloadQueue) Swap(i, j int)      { h.s[i], h.s[j] = h.s[j], h.s[i] }
+
+func (h *payloadQueue) Push(x interface{}) {
+	// Push and Pop use pointer receivers because they modify the slice's length,
+	// not just its contents.
+	h.s = append(h.s, x.(*filePayload))
+}
+
+func (h *payloadQueue) Pop() interface{} {
+	old := h.s
+	n := len(old)
+	x := old[n-1]
+	h.s = old[0 : n-1]
+	return x
+}
+
+func (h payloadQueue) Idx(i int) *filePayload {
+	return h.s[i]
+}
+
+type fileInputReader struct {
+	reader    *bufio.Reader
+	file      io.ReadCloser
+	closed    int32 // Value of 0 indicates that the file is still open.
+	s3        bool
+	queue     payloadQueue
+	readDepth int
+	dryRun    bool
+	path      string
+}
+
+func (f *fileInputReader) parse(init chan struct{}) error {
 	payloadSeparatorAsBytes := []byte(payloadSeparator)
 	var buffer bytes.Buffer
+	var initialized bool
+
+	lineNum := 0
+
 	for {
 		line, err := f.reader.ReadBytes('\n')
+		lineNum++
 
 		if err != nil {
 			if err != io.EOF {
 				Debug(1, err)
-			} else {
-				f.Close()
 			}
+
+			f.Close()
+
+			if !initialized {
+				close(init)
+				initialized = true
+			}
+
 			return err
 		}
 
@@ -48,22 +97,64 @@ func (f *fileInputReader) parseNext() error {
 			asBytes := buffer.Bytes()
 			meta := payloadMeta(asBytes)
 
-			f.timestamp, _ = strconv.ParseInt(string(meta[2]), 10, 64)
-			f.data = asBytes[:len(asBytes)-1]
+			if len(meta) < 3 {
+				Debug(1, fmt.Sprintf("Found malformed record, file: %s, line %d", f.path, lineNum))
+				buffer = bytes.Buffer{}
+				continue
+			}
 
-			return nil
+			timestamp, _ := strconv.ParseInt(string(meta[2]), 10, 64)
+			data := asBytes[:len(asBytes)-1]
+
+			f.queue.Lock()
+			heap.Push(&f.queue, &filePayload{
+				timestamp: timestamp,
+				data:      data,
+			})
+			f.queue.Unlock()
+
+			for {
+				if f.queue.Len() < f.readDepth {
+					break
+				}
+
+				if !initialized {
+					close(init)
+					initialized = true
+				}
+
+				if !f.dryRun {
+					time.Sleep(100 * time.Millisecond)
+				}
+			}
+
+			buffer = bytes.Buffer{}
+			continue
 		}
 
 		buffer.Write(line)
 	}
-
 }
 
-func (f *fileInputReader) ReadPayload() []byte {
-	defer f.parseNext()
+func (f *fileInputReader) wait() {
+	for {
+		if atomic.LoadInt32(&f.closed) == 1 {
+			return
+		}
 
-	return f.data
+		if f.queue.Len() > 0 {
+			return
+		}
+
+		if !f.dryRun {
+			time.Sleep(100 * time.Millisecond)
+		}
+	}
+
+	return
 }
+
+// Close closes this plugin
 func (f *fileInputReader) Close() error {
 	if atomic.LoadInt32(&f.closed) == 0 {
 		atomic.StoreInt32(&f.closed, 1)
@@ -73,7 +164,7 @@ func (f *fileInputReader) Close() error {
 	return nil
 }
 
-func NewFileInputReader(path string) *fileInputReader {
+func newFileInputReader(path string, readDepth int, dryRun bool) *fileInputReader {
 	var file io.ReadCloser
 	var err error
 
@@ -84,15 +175,15 @@ func NewFileInputReader(path string) *fileInputReader {
 	}
 
 	if err != nil {
-		log.Println(err)
+		Debug(0, fmt.Sprintf("[INPUT-FILE] err: %q", err))
 		return nil
 	}
 
-	r := &fileInputReader{file: file, closed: 0}
+	r := &fileInputReader{path: path, file: file, closed: 0, readDepth: readDepth, dryRun: dryRun}
 	if strings.HasSuffix(path, ".gz") {
 		gzReader, err := gzip.NewReader(file)
 		if err != nil {
-			log.Println(err)
+			Debug(0, fmt.Sprintf("[INPUT-FILE] err: %q", err))
 			return nil
 		}
 		r.reader = bufio.NewReader(gzReader)
@@ -100,7 +191,11 @@ func NewFileInputReader(path string) *fileInputReader {
 		r.reader = bufio.NewReader(file)
 	}
 
-	r.parseNext()
+	heap.Init(&r.queue)
+
+	init := make(chan struct{})
+	go r.parse(init)
+	<-init
 
 	return r
 }
@@ -114,16 +209,25 @@ type FileInput struct {
 	readers     []*fileInputReader
 	speedFactor float64
 	loop        bool
+	readDepth   int
+	dryRun      bool
+	maxWait     time.Duration
+
+	stats *expvar.Map
 }
 
 // NewFileInput constructor for FileInput. Accepts file path as argument.
-func NewFileInput(path string, loop bool) (i *FileInput) {
+func NewFileInput(path string, loop bool, readDepth int, maxWait time.Duration, dryRun bool) (i *FileInput) {
 	i = new(FileInput)
 	i.data = make(chan []byte, 1000)
 	i.exit = make(chan bool)
 	i.path = path
 	i.speedFactor = 1
 	i.loop = loop
+	i.readDepth = readDepth
+	i.stats = expvar.NewMap("file-" + path)
+	i.dryRun = dryRun
+	i.maxWait = maxWait
 
 	if err := i.init(); err != nil {
 		return
@@ -153,43 +257,45 @@ func (i *FileInput) init() (err error) {
 
 		resp, err := svc.ListObjects(params)
 		if err != nil {
-			log.Println("Error while retreiving list of files from S3", i.path, err)
+			Debug(0, "[INPUT-FILE] Error while retreiving list of files from S3", i.path, err)
 			return err
 		}
 
 		for _, c := range resp.Contents {
 			matches = append(matches, "s3://"+bucket+"/"+(*c.Key))
 		}
-	} else {
-		if matches, err = filepath.Glob(i.path); err != nil {
-			log.Println("Wrong file pattern", i.path, err)
-			return
-		}
+	} else if matches, err = filepath.Glob(i.path); err != nil {
+		Debug(0, "[INPUT-FILE] Wrong file pattern", i.path, err)
+		return
 	}
 
 	if len(matches) == 0 {
-		log.Println("No files match pattern: ", i.path)
+		Debug(0, "[INPUT-FILE] No files match pattern: ", i.path)
 		return errors.New("No matching files")
 	}
 
 	i.readers = make([]*fileInputReader, len(matches))
 
 	for idx, p := range matches {
-		i.readers[idx] = NewFileInputReader(p)
+		i.readers[idx] = newFileInputReader(p, i.readDepth, i.dryRun)
 	}
 
+	i.stats.Add("reader_count", int64(len(matches)))
+
 	return nil
 }
 
-func (i *FileInput) Read(data []byte) (int, error) {
-	var buf []byte
+// PluginRead reads message from this plugin
+func (i *FileInput) PluginRead() (*Message, error) {
+	var msg Message
 	select {
 	case <-i.exit:
-		return 0, ErrorStopped
-	case buf = <-i.data:
+		return nil, ErrorStopped
+	case buf := <-i.data:
+		i.stats.Add("read_from", 1)
+		msg.Meta, msg.Data = payloadMetaWithBody(buf)
+		return &msg, nil
 	}
-	n := copy(data, buf)
-	return n, nil
 }
 
 func (i *FileInput) String() string {
@@ -199,11 +305,17 @@ func (i *FileInput) String() string {
 // Find reader with smallest timestamp e.g next payload in row
 func (i *FileInput) nextReader() (next *fileInputReader) {
 	for _, r := range i.readers {
-		if r == nil || atomic.LoadInt32(&r.closed) != 0 {
+		if r == nil {
 			continue
 		}
 
-		if next == nil || r.timestamp < next.timestamp {
+		r.wait()
+
+		if r.queue.Len() == 0 {
+			continue
+		}
+
+		if next == nil || r.queue.Idx(0).timestamp < next.queue.Idx(0).timestamp {
 			next = r
 			continue
 		}
@@ -215,6 +327,11 @@ func (i *FileInput) nextReader() (next *fileInputReader) {
 func (i *FileInput) emit() {
 	var lastTime int64 = -1
 
+	var maxWait, firstWait, minWait int64
+	minWait = math.MaxInt64
+
+	i.stats.Add("negative_wait", 0)
+
 	for {
 		select {
 		case <-i.exit:
@@ -234,17 +351,48 @@ func (i *FileInput) emit() {
 			}
 		}
 
+		reader.queue.RLock()
+		payload := heap.Pop(&reader.queue).(*filePayload)
+		i.stats.Add("total_counter", 1)
+		i.stats.Add("total_bytes", int64(len(payload.data)))
+		reader.queue.RUnlock()
+
 		if lastTime != -1 {
-			diff := reader.timestamp - lastTime
-			lastTime = reader.timestamp
+			diff := payload.timestamp - lastTime
+
+			if firstWait == 0 {
+				firstWait = diff
+			}
 
 			if i.speedFactor != 1 {
 				diff = int64(float64(diff) / i.speedFactor)
 			}
 
-			time.Sleep(time.Duration(diff))
+			if i.maxWait > 0 && diff > int64(i.maxWait) {
+				diff = int64(i.maxWait)
+			}
+
+			if diff >= 0 {
+				lastTime = payload.timestamp
+
+				if !i.dryRun {
+					time.Sleep(time.Duration(diff))
+				}
+
+				i.stats.Add("total_wait", diff)
+
+				if diff > maxWait {
+					maxWait = diff
+				}
+
+				if diff < minWait {
+					minWait = diff
+				}
+			} else {
+				i.stats.Add("negative_wait", 1)
+			}
 		} else {
-			lastTime = reader.timestamp
+			lastTime = payload.timestamp
 		}
 
 		// Recheck if we have exited since last check.
@@ -252,12 +400,30 @@ func (i *FileInput) emit() {
 		case <-i.exit:
 			return
 		default:
-			i.data <- reader.ReadPayload()
+			if !i.dryRun {
+				i.data <- payload.data
+			}
 		}
 	}
 
-	log.Printf("FileInput: end of file '%s'\n", i.path)
+	i.stats.Set("first_wait", time.Duration(firstWait))
+	i.stats.Set("max_wait", time.Duration(maxWait))
+	i.stats.Set("min_wait", time.Duration(minWait))
 
+	Debug(0, fmt.Sprintf("[INPUT-FILE] FileInput: end of file '%s'\n", i.path))
+
+	if i.dryRun {
+		fmt.Printf("Records found: %v\nFiles processed: %v\nBytes processed: %v\nMax wait: %v\nMin wait: %v\nFirst wait: %v\nIt will take `%v` to replay at current speed.\nFound %v records with out of order timestamp\n",
+			i.stats.Get("total_counter"),
+			i.stats.Get("reader_count"),
+			i.stats.Get("total_bytes"),
+			i.stats.Get("max_wait"),
+			i.stats.Get("min_wait"),
+			i.stats.Get("first_wait"),
+			time.Duration(i.stats.Get("total_wait").(*expvar.Int).Value()),
+			i.stats.Get("negative_wait"),
+		)
+	}
 }
 
 // Close closes this plugin

input_file_test.go

@@ -4,9 +4,7 @@ import (
 	"bytes"
 	"errors"
 	"fmt"
-	"io"
 	"io/ioutil"
-	"log"
 	"math/rand"
 	"os"
 	"sync"
@@ -14,54 +12,49 @@ import (
 	"time"
 )
 
-var _ = log.Println
-
 func TestInputFileWithGET(t *testing.T) {
 	input := NewTestInput()
-	rg := NewRequestGenerator([]io.Reader{input}, func() { input.EmitGET() }, 1)
-	readPayloads := [][]byte{}
+	rg := NewRequestGenerator([]PluginReader{input}, func() { input.EmitGET() }, 1)
+	readPayloads := []*Message{}
 
 	// Given a capture file with a GET request
 	expectedCaptureFile := CreateCaptureFile(rg)
 	defer expectedCaptureFile.TearDown()
 
 	// When the request is read from the capture file
-	err := ReadFromCaptureFile(expectedCaptureFile.file, 1, func(data []byte) {
-		readPayloads = append(readPayloads, Duplicate(data))
+	err := ReadFromCaptureFile(expectedCaptureFile.file, 1, func(msg *Message) {
+		readPayloads = append(readPayloads, msg)
 	})
 
 	// The read request should match the original request
 	if err != nil {
 		t.Error(err)
-	} else {
-		if !expectedCaptureFile.PayloadsEqual(readPayloads) {
-			t.Error("Request read back from file should match")
-		}
+	} else if !expectedCaptureFile.PayloadsEqual(readPayloads) {
+		t.Error("Request read back from file should match")
+
 	}
 }
 
 func TestInputFileWithPayloadLargerThan64Kb(t *testing.T) {
-
 	input := NewTestInput()
-	rg := NewRequestGenerator([]io.Reader{input}, func() { input.EmitSizedPOST(64 * 1024) }, 1)
-	readPayloads := [][]byte{}
+	rg := NewRequestGenerator([]PluginReader{input}, func() { input.EmitSizedPOST(64 * 1024) }, 1)
+	readPayloads := []*Message{}
 
 	// Given a capture file with a request over 64Kb
 	expectedCaptureFile := CreateCaptureFile(rg)
 	defer expectedCaptureFile.TearDown()
 
 	// When the request is read from the capture file
-	err := ReadFromCaptureFile(expectedCaptureFile.file, 1, func(data []byte) {
-		readPayloads = append(readPayloads, Duplicate(data))
+	err := ReadFromCaptureFile(expectedCaptureFile.file, 1, func(msg *Message) {
+		readPayloads = append(readPayloads, msg)
 	})
 
 	// The read request should match the original request
 	if err != nil {
 		t.Error(err)
-	} else {
-		if !expectedCaptureFile.PayloadsEqual(readPayloads) {
-			t.Error("Request read back from file should match")
-		}
+	} else if !expectedCaptureFile.PayloadsEqual(readPayloads) {
+		t.Error("Request read back from file should match")
+
 	}
 
 }
@@ -69,28 +62,27 @@ func TestInputFileWithPayloadLargerThan64Kb(t *testing.T) {
 func TestInputFileWithGETAndPOST(t *testing.T) {
 
 	input := NewTestInput()
-	rg := NewRequestGenerator([]io.Reader{input}, func() {
+	rg := NewRequestGenerator([]PluginReader{input}, func() {
 		input.EmitGET()
 		input.EmitPOST()
 	}, 2)
-	readPayloads := [][]byte{}
+	readPayloads := []*Message{}
 
 	// Given a capture file with a GET request
 	expectedCaptureFile := CreateCaptureFile(rg)
 	defer expectedCaptureFile.TearDown()
 
 	// When the requests are read from the capture file
-	err := ReadFromCaptureFile(expectedCaptureFile.file, 2, func(data []byte) {
-		readPayloads = append(readPayloads, Duplicate(data))
+	err := ReadFromCaptureFile(expectedCaptureFile.file, 2, func(msg *Message) {
+		readPayloads = append(readPayloads, msg)
 	})
 
 	// The read requests should match the original request
 	if err != nil {
 		t.Error(err)
-	} else {
-		if !expectedCaptureFile.PayloadsEqual(readPayloads) {
-			t.Error("Request read back from file should match")
-		}
+	} else if !expectedCaptureFile.PayloadsEqual(readPayloads) {
+		t.Error("Request read back from file should match")
+
 	}
 
 }
@@ -112,13 +104,12 @@ func TestInputFileMultipleFilesWithRequestsOnly(t *testing.T) {
 	file2.Write([]byte(payloadSeparator))
 	file2.Close()
 
-	input := NewFileInput(fmt.Sprintf("/tmp/%d*", rnd), false)
-	buf := make([]byte, 1000)
+	input := NewFileInput(fmt.Sprintf("/tmp/%d*", rnd), false, 100, 0, false)
 
 	for i := '1'; i <= '4'; i++ {
-		n, _ := input.Read(buf)
-		if buf[4] != byte(i) {
-			t.Error("Should emit requests in right order", string(buf[:n]))
+		msg, _ := input.PluginRead()
+		if msg.Meta[4] != byte(i) {
+			t.Error("Should emit requests in right order", string(msg.Meta))
 		}
 	}
 
@@ -139,12 +130,11 @@ func TestInputFileRequestsWithLatency(t *testing.T) {
 	file.Write([]byte("1 3 250000000\nrequest3"))
 	file.Write([]byte(payloadSeparator))
 
-	input := NewFileInput(fmt.Sprintf("/tmp/%d", rnd), false)
-	buf := make([]byte, 1000)
+	input := NewFileInput(fmt.Sprintf("/tmp/%d", rnd), false, 100, 0, false)
 
 	start := time.Now().UnixNano()
 	for i := 0; i < 3; i++ {
-		input.Read(buf)
+		input.PluginRead()
 	}
 	end := time.Now().UnixNano()
 
@@ -180,18 +170,17 @@ func TestInputFileMultipleFilesWithRequestsAndResponses(t *testing.T) {
 	file2.Write([]byte(payloadSeparator))
 	file2.Close()
 
-	input := NewFileInput(fmt.Sprintf("/tmp/%d*", rnd), false)
-	buf := make([]byte, 1000)
+	input := NewFileInput(fmt.Sprintf("/tmp/%d*", rnd), false, 100, 0, false)
 
 	for i := '1'; i <= '4'; i++ {
-		n, _ := input.Read(buf)
-		if buf[0] != '1' && buf[4] != byte(i) {
-			t.Error("Shound emit requests in right order", string(buf[:n]))
+		msg, _ := input.PluginRead()
+		if msg.Meta[0] != '1' && msg.Meta[4] != byte(i) {
+			t.Error("Shound emit requests in right order", string(msg.Meta))
 		}
 
-		n, _ = input.Read(buf)
-		if buf[0] != '2' && buf[4] != byte(i) {
-			t.Error("Shound emit responses in right order", string(buf[:n]))
+		msg, _ = input.PluginRead()
+		if msg.Meta[0] != '2' && msg.Meta[4] != byte(i) {
+			t.Error("Shound emit responses in right order", string(msg.Meta))
 		}
 	}
 
@@ -209,12 +198,11 @@ func TestInputFileLoop(t *testing.T) {
 	file.Write([]byte(payloadSeparator))
 	file.Close()
 
-	input := NewFileInput(fmt.Sprintf("/tmp/%d", rnd), true)
-	buf := make([]byte, 1000)
+	input := NewFileInput(fmt.Sprintf("/tmp/%d", rnd), true, 100, 0, false)
 
 	// Even if we have just 2 requests in file, it should indifinitly loop
 	for i := 0; i < 1000; i++ {
-		input.Read(buf)
+		input.PluginRead()
 	}
 
 	input.Close()
@@ -226,22 +214,21 @@ func TestInputFileCompressed(t *testing.T) {
 
 	output := NewFileOutput(fmt.Sprintf("/tmp/%d_0.gz", rnd), &FileOutputConfig{FlushInterval: time.Minute, Append: true})
 	for i := 0; i < 1000; i++ {
-		output.Write([]byte("1 1 1\r\ntest"))
+		output.PluginWrite(&Message{Meta: []byte("1 1 1\r\n"), Data: []byte("test")})
 	}
 	name1 := output.file.Name()
 	output.Close()
 
 	output2 := NewFileOutput(fmt.Sprintf("/tmp/%d_1.gz", rnd), &FileOutputConfig{FlushInterval: time.Minute, Append: true})
 	for i := 0; i < 1000; i++ {
-		output2.Write([]byte("1 1 1\r\ntest"))
+		output2.PluginWrite(&Message{Meta: []byte("1 1 1\r\n"), Data: []byte("test")})
 	}
 	name2 := output2.file.Name()
 	output2.Close()
 
-	input := NewFileInput(fmt.Sprintf("/tmp/%d*", rnd), false)
-	buf := make([]byte, 1000)
+	input := NewFileInput(fmt.Sprintf("/tmp/%d*", rnd), false, 100, 0, false)
 	for i := 0; i < 2000; i++ {
-		input.Read(buf)
+		input.PluginRead()
 	}
 
 	os.Remove(name1)
@@ -249,14 +236,14 @@ func TestInputFileCompressed(t *testing.T) {
 }
 
 type CaptureFile struct {
-	data [][]byte
+	msgs []*Message
 	file *os.File
 }
 
-func NewExpectedCaptureFile(data [][]byte, file *os.File) *CaptureFile {
+func NewExpectedCaptureFile(msgs []*Message, file *os.File) *CaptureFile {
 	ecf := new(CaptureFile)
 	ecf.file = file
-	ecf.data = data
+	ecf.msgs = msgs
 	return ecf
 }
 
@@ -267,12 +254,12 @@ func (expectedCaptureFile *CaptureFile) TearDown() {
 }
 
 type RequestGenerator struct {
-	inputs []io.Reader
+	inputs []PluginReader
 	emit   func()
 	wg     *sync.WaitGroup
 }
 
-func NewRequestGenerator(inputs []io.Reader, emit func(), count int) (rg *RequestGenerator) {
+func NewRequestGenerator(inputs []PluginReader, emit func(), count int) (rg *RequestGenerator) {
 	rg = new(RequestGenerator)
 	rg.inputs = inputs
 	rg.emit = emit
@@ -281,14 +268,17 @@ func NewRequestGenerator(inputs []io.Reader, emit func(), count int) (rg *Reques
 	return
 }
 
-func (expectedCaptureFile *CaptureFile) PayloadsEqual(other [][]byte) bool {
+func (expectedCaptureFile *CaptureFile) PayloadsEqual(other []*Message) bool {
 
-	if len(expectedCaptureFile.data) != len(other) {
+	if len(expectedCaptureFile.msgs) != len(other) {
 		return false
 	}
 
 	for i, payload := range other {
-		if !bytes.Equal(expectedCaptureFile.data[i], payload) {
+		if !bytes.Equal(expectedCaptureFile.msgs[i].Meta, payload.Meta) {
+			return false
+		}
+		if !bytes.Equal(expectedCaptureFile.msgs[i].Data, payload.Data) {
 			return false
 		}
 	}
@@ -303,26 +293,24 @@ func CreateCaptureFile(requestGenerator *RequestGenerator) *CaptureFile {
 		panic(err)
 	}
 
-	quit := make(chan int)
-
-	readPayloads := [][]byte{}
-	output := NewTestOutput(func(data []byte) {
-		readPayloads = append(readPayloads, Duplicate(data))
+	readPayloads := []*Message{}
+	output := NewTestOutput(func(msg *Message) {
+		readPayloads = append(readPayloads, msg)
 		requestGenerator.wg.Done()
 	})
 
-	outputFile := NewFileOutput(f.Name(), &FileOutputConfig{FlushInterval: time.Minute, Append: true})
+	outputFile := NewFileOutput(f.Name(), &FileOutputConfig{FlushInterval: time.Second, Append: true})
 
 	plugins := &InOutPlugins{
 		Inputs:  requestGenerator.inputs,
-		Outputs: []io.Writer{output, outputFile},
+		Outputs: []PluginWriter{output, outputFile},
 	}
 	for _, input := range requestGenerator.inputs {
 		plugins.All = append(plugins.All, input)
 	}
 	plugins.All = append(plugins.All, output, outputFile)
 
-	emitter := NewEmitter(quit)
+	emitter := NewEmitter()
 	go emitter.Start(plugins, Settings.Middleware)
 
 	requestGenerator.emit()
@@ -336,23 +324,22 @@ func CreateCaptureFile(requestGenerator *RequestGenerator) *CaptureFile {
 }
 
 func ReadFromCaptureFile(captureFile *os.File, count int, callback writeCallback) (err error) {
-	quit := make(chan int)
 	wg := new(sync.WaitGroup)
 
-	input := NewFileInput(captureFile.Name(), false)
-	output := NewTestOutput(func(data []byte) {
-		callback(data)
+	input := NewFileInput(captureFile.Name(), false, 100, 0, false)
+	output := NewTestOutput(func(msg *Message) {
+		callback(msg)
 		wg.Done()
 	})
 
 	plugins := &InOutPlugins{
-		Inputs:  []io.Reader{input},
-		Outputs: []io.Writer{output},
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{output},
 	}
 	plugins.All = append(plugins.All, input, output)
 
 	wg.Add(count)
-	emitter := NewEmitter(quit)
+	emitter := NewEmitter()
 	go emitter.Start(plugins, Settings.Middleware)
 
 	done := make(chan int, 1)
@@ -371,10 +358,3 @@ func ReadFromCaptureFile(captureFile *os.File, count int, callback writeCallback
 	return
 
 }
-
-func Duplicate(data []byte) (duplicate []byte) {
-	duplicate = make([]byte, len(data))
-	copy(duplicate, data)
-
-	return
-}

input_http.go

@@ -27,25 +27,17 @@ func NewHTTPInput(address string) (i *HTTPInput) {
 	return
 }
 
-func (i *HTTPInput) Read(data []byte) (int, error) {
-	var buf []byte
+// PluginRead reads message from this plugin
+func (i *HTTPInput) PluginRead() (*Message, error) {
+	var msg Message
 	select {
 	case <-i.stop:
-		return 0, ErrorStopped
-	case buf = <-i.data:
+		return nil, ErrorStopped
+	case buf := <-i.data:
+		msg.Data = buf
+		msg.Meta = payloadHeader(RequestPayload, uuid(), time.Now().UnixNano(), -1)
+		return &msg, nil
 	}
-	header := payloadHeader(RequestPayload, uuid(), time.Now().UnixNano(), -1)
-
-	n := copy(data, header)
-	if len(data) > len(header) {
-		n += copy(data[len(header):], buf)
-	}
-	dis := len(header) + len(buf) - n
-	if dis > 0 {
-		Debug(2, "[INPUT-HTTP] discarded", dis, "increase copy buffer size")
-	}
-
-	return n, nil
 }
 
 // Close closes this plugin

input_http_test.go

@@ -2,33 +2,29 @@ package main
 
 import (
 	"bytes"
-	"io"
 	"net/http"
 	"strings"
 	"sync"
 	"testing"
 	"time"
-
-	"github.com/buger/goreplay/proto"
 )
 
 func TestHTTPInput(t *testing.T) {
 	wg := new(sync.WaitGroup)
-	quit := make(chan int)
 
 	input := NewHTTPInput("127.0.0.1:0")
 	time.Sleep(time.Millisecond)
-	output := NewTestOutput(func(data []byte) {
+	output := NewTestOutput(func(*Message) {
 		wg.Done()
 	})
 
 	plugins := &InOutPlugins{
-		Inputs:  []io.Reader{input},
-		Outputs: []io.Writer{output},
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{output},
 	}
 	plugins.All = append(plugins.All, input, output)
 
-	emitter := NewEmitter(quit)
+	emitter := NewEmitter()
 	go emitter.Start(plugins, Settings.Middleware)
 
 	address := strings.Replace(input.address, "[::]", "127.0.0.1", -1)
@@ -44,27 +40,25 @@ func TestHTTPInput(t *testing.T) {
 
 func TestInputHTTPLargePayload(t *testing.T) {
 	wg := new(sync.WaitGroup)
-	quit := make(chan int, 1)
 	const n = 10 << 20 // 10MB
 	var large [n]byte
 	large[n-1] = '0'
 
 	input := NewHTTPInput("127.0.0.1:0")
-	time.Sleep(time.Millisecond)
-	output := NewTestOutput(func(data []byte) {
-		_len := len(proto.Body(payloadBody(data)))
+	output := NewTestOutput(func(msg *Message) {
+		_len := len(msg.Data)
 		if _len >= n { // considering http body CRLF
 			t.Errorf("expected body to be >= %d", n)
 		}
 		wg.Done()
 	})
 	plugins := &InOutPlugins{
-		Inputs:  []io.Reader{input},
-		Outputs: []io.Writer{output},
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{output},
 	}
 	plugins.All = append(plugins.All, input, output)
 
-	emitter := NewEmitter(quit)
+	emitter := NewEmitter()
 	defer emitter.Close()
 	go emitter.Start(plugins, Settings.Middleware)
 

input_kafka.go

@@ -9,21 +9,17 @@ import (
 	"github.com/Shopify/sarama/mocks"
 )
 
-// KafkaInput is used for recieving Kafka messages and
+// KafkaInput is used for receiving Kafka messages and
 // transforming them into HTTP payloads.
 type KafkaInput struct {
 	config    *InputKafkaConfig
 	consumers []sarama.PartitionConsumer
 	messages  chan *sarama.ConsumerMessage
+	quit      chan struct{}
 }
 
-// NewKafkaInput creates instance of kafka consumer client.
-func NewKafkaInput(address string, config *InputKafkaConfig) *KafkaInput {
-	return NewKafkaInputWithTLS(address, config, nil)
-}
-
-// NewKafkaInputWithTLS creates instance of kafka consumer client with TLS
-func NewKafkaInputWithTLS(address string, config *InputKafkaConfig, tlsConfig *KafkaTLSConfig) *KafkaInput {
+// NewKafkaInput creates instance of kafka consumer client with TLS config
+func NewKafkaInput(address string, config *InputKafkaConfig, tlsConfig *KafkaTLSConfig) *KafkaInput {
 	c := NewKafkaConfig(tlsConfig)
 
 	var con sarama.Consumer
@@ -48,6 +44,7 @@ func NewKafkaInputWithTLS(address string, config *InputKafkaConfig, tlsConfig *K
 		config:    config,
 		consumers: make([]sarama.PartitionConsumer, len(partitions)),
 		messages:  make(chan *sarama.ConsumerMessage, 256),
+		quit:      make(chan struct{}),
 	}
 
 	for index, partition := range partitions {
@@ -79,29 +76,45 @@ func (i *KafkaInput) ErrorHandler(consumer sarama.PartitionConsumer) {
 	}
 }
 
-func (i *KafkaInput) Read(data []byte) (int, error) {
-	message := <-i.messages
-
-	if !i.config.UseJSON {
-		copy(data, message.Value)
-		return len(message.Value), nil
+// PluginRead a reads message from this plugin
+func (i *KafkaInput) PluginRead() (*Message, error) {
+	var message *sarama.ConsumerMessage
+	var msg Message
+	select {
+	case <-i.quit:
+		return nil, ErrorStopped
+	case message = <-i.messages:
 	}
 
-	var kafkaMessage KafkaMessage
-	json.Unmarshal(message.Value, &kafkaMessage)
+	msg.Data = message.Value
+	if i.config.UseJSON {
 
-	buf, err := kafkaMessage.Dump()
-	if err != nil {
-		Debug(1, "Failed to decode access log entry:", err)
-		return 0, err
+		var kafkaMessage KafkaMessage
+		json.Unmarshal(message.Value, &kafkaMessage)
+
+		var err error
+		msg.Data, err = kafkaMessage.Dump()
+		if err != nil {
+			Debug(1, "[INPUT-KAFKA] failed to decode access log entry:", err)
+			return nil, err
+		}
 	}
 
-	n := copy(data, buf)
+	// does it have meta
+	if isOriginPayload(msg.Data) {
+		msg.Meta, msg.Data = payloadMetaWithBody(msg.Data)
+	}
 
-	return n, nil
+	return &msg, nil
 
 }
 
 func (i *KafkaInput) String() string {
 	return "Kafka Input: " + i.config.Host + "/" + i.config.Topic
 }
+
+// Close closes this plugin
+func (i *KafkaInput) Close() error {
+	close(i.quit)
+	return nil
+}

input_kafka_test.go

@@ -20,17 +20,16 @@ func TestInputKafkaRAW(t *testing.T) {
 		consumer: consumer,
 		Topic:    "test",
 		UseJSON:  false,
-	})
+	}, nil)
 
-	buf := make([]byte, 1024)
-	n, err := input.Read(buf)
+	msg, err := input.PluginRead()
 
 	if err != nil {
 		t.Fatal(err)
 	}
 
-	if string(buf[:n]) != "1 2 3\nGET / HTTP1.1\r\nHeader: 1\r\n\r\n" {
-		t.Error("Message not properly decoded: ", string(buf[:n]), n)
+	if string(append(msg.Meta, msg.Data...)) != "1 2 3\nGET / HTTP1.1\r\nHeader: 1\r\n\r\n" {
+		t.Error("Message not properly decoded")
 	}
 }
 
@@ -47,16 +46,15 @@ func TestInputKafkaJSON(t *testing.T) {
 		consumer: consumer,
 		Topic:    "test",
 		UseJSON:  true,
-	})
+	}, nil)
 
-	buf := make([]byte, 1024)
-	n, err := input.Read(buf)
+	msg, err := input.PluginRead()
 
 	if err != nil {
 		t.Fatal(err)
 	}
 
-	if string(buf[:n]) != "1 2 3\nGET / HTTP/1.1\r\nHeader: 1\r\n\r\n" {
-		t.Error("Message not properly decoded: ", string(buf[:n]), n)
+	if string(append(msg.Meta, msg.Data...)) != "1 2 3\nGET / HTTP/1.1\r\nHeader: 1\r\n\r\n" {
+		t.Error("Message not properly decoded")
 	}
 }

input_raw.go

@@ -6,6 +6,7 @@ import (
 	"log"
 	"net"
 	"strconv"
+	"strings"
 	"sync"
 	"time"
 
@@ -15,53 +16,20 @@ import (
 	"github.com/buger/goreplay/tcp"
 )
 
-// TCPProtocol is a number to indicate type of protocol
-type TCPProtocol uint8
-
-const (
-	// ProtocolHTTP ...
-	ProtocolHTTP TCPProtocol = iota
-	// ProtocolBinary ...
-	ProtocolBinary
-)
-
-// Set is here so that TCPProtocol can implement flag.Var
-func (protocol *TCPProtocol) Set(v string) error {
-	switch v {
-	case "", "http":
-		*protocol = ProtocolHTTP
-	case "binary":
-		*protocol = ProtocolBinary
-	default:
-		return fmt.Errorf("unsupported protocol %s", v)
-	}
-	return nil
-}
-
-func (protocol *TCPProtocol) String() string {
-	switch *protocol {
-	case ProtocolBinary:
-		return "binary"
-	case ProtocolHTTP:
-		return "http"
-	default:
-		return ""
-	}
-}
-
 // RAWInputConfig represents configuration that can be applied on raw input
 type RAWInputConfig struct {
 	capture.PcapOptions
-	Expire         time.Duration      `json:"input-raw-expire"`
-	CopyBufferSize size.Size          `json:"copy-buffer-size"`
-	Engine         capture.EngineType `json:"input-raw-engine"`
-	TrackResponse  bool               `json:"input-raw-track-response"`
-	Protocol       TCPProtocol        `json:"input-raw-protocol"`
-	RealIPHeader   string             `json:"input-raw-realip-header"`
-	Stats          bool               `json:"input-raw-stats"`
-	quit           chan bool          // Channel used only to indicate goroutine should shutdown
-	host           string
-	port           uint16
+	Expire          time.Duration      `json:"input-raw-expire"`
+	CopyBufferSize  size.Size          `json:"copy-buffer-size"`
+	Engine          capture.EngineType `json:"input-raw-engine"`
+	TrackResponse   bool               `json:"input-raw-track-response"`
+	Protocol        tcp.TCPProtocol    `json:"input-raw-protocol"`
+	RealIPHeader    string             `json:"input-raw-realip-header"`
+	Stats           bool               `json:"input-raw-stats"`
+	AllowIncomplete bool               `json:"input-raw-allow-incomplete"`
+	quit            chan bool          // Channel used only to indicate goroutine should shutdown
+	host            string
+	ports           []uint16
 }
 
 // RAWInput used for intercepting traffic for given address
@@ -70,81 +38,83 @@ type RAWInput struct {
 	RAWInputConfig
 	messageStats   []tcp.Stats
 	listener       *capture.Listener
-	message        chan *tcp.Message
+	messageParser  *tcp.MessageParser
 	cancelListener context.CancelFunc
+	closed         bool
 }
 
 // NewRAWInput constructor for RAWInput. Accepts raw input config as arguments.
 func NewRAWInput(address string, config RAWInputConfig) (i *RAWInput) {
 	i = new(RAWInput)
 	i.RAWInputConfig = config
-	i.message = make(chan *tcp.Message, 1000)
 	i.quit = make(chan bool)
-	var host, _port string
-	var err error
-	var port int
-	host, _port, err = net.SplitHostPort(address)
+
+	host, _ports, err := net.SplitHostPort(address)
 	if err != nil {
 		log.Fatalf("input-raw: error while parsing address: %s", err)
 	}
-	if _port != "" {
-		port, err = strconv.Atoi(_port)
+
+	var ports []uint16
+	if _ports != "" {
+		portsStr := strings.Split(_ports, ",")
+
+		for _, portStr := range portsStr {
+			port, err := strconv.Atoi(strings.TrimSpace(portStr))
+			if err != nil {
+				log.Fatalf("parsing port error: %v", err)
+			}
+			ports = append(ports, uint16(port))
+
+		}
 	}
 
-	if err != nil {
-		log.Fatalf("parsing port error: %v", err)
-	}
 	i.host = host
-	i.port = uint16(port)
+	i.ports = ports
 
 	i.listen(address)
 
 	return
 }
 
-func (i *RAWInput) Read(data []byte) (n int, err error) {
-	var msg *tcp.Message
-	var buf []byte
+// PluginRead reads meassage from this plugin
+func (i *RAWInput) PluginRead() (*Message, error) {
+	var msgTCP *tcp.Message
+	var msg Message
 	select {
 	case <-i.quit:
-		return 0, ErrorStopped
-	case msg = <-i.message:
-		buf = msg.Data()
+		return nil, ErrorStopped
+	case msgTCP = <-i.listener.Messages():
+		msg.Data = msgTCP.Data()
 	}
-	var header []byte
 
 	var msgType byte = ResponsePayload
-	if msg.IsIncoming {
+	if msgTCP.Direction == tcp.DirIncoming {
 		msgType = RequestPayload
 		if i.RealIPHeader != "" {
-			buf = proto.SetHeader(buf, []byte(i.RealIPHeader), []byte(msg.SrcAddr))
+			msg.Data = proto.SetHeader(msg.Data, []byte(i.RealIPHeader), []byte(msgTCP.SrcAddr))
 		}
 	}
-	header = payloadHeader(msgType, msg.UUID(), msg.Start.UnixNano(), msg.End.UnixNano()-msg.Start.UnixNano())
+	msg.Meta = payloadHeader(msgType, msgTCP.UUID(), msgTCP.Start.UnixNano(), msgTCP.End.UnixNano()-msgTCP.Start.UnixNano())
 
-	n = copy(data, header)
-	if len(data) > len(header) {
-		n += copy(data[len(header):], buf)
-	}
 	// to be removed....
-	if msg.Truncated || len(header)+len(buf)-n > 0 {
-		go Debug(2, "[INPUT-RAW] message truncated, increase copy-buffer-size")
+	if msgTCP.Truncated {
+		Debug(2, "[INPUT-RAW] message truncated, increase copy-buffer-size")
 	}
 	// to be removed...
-	if msg.TimedOut {
-		go Debug(2, "[INPUT-RAW] message timeout reached, increase input-raw-expire")
+	if msgTCP.TimedOut {
+		Debug(2, "[INPUT-RAW] message timeout reached, increase input-raw-expire")
 	}
 	if i.Stats {
-		stat := msg.Stats
+		stat := msgTCP.Stats
 		go i.addStats(stat)
 	}
-	msg = nil
-	return n, nil
+	msgTCP = nil
+	return &msg, nil
 }
 
 func (i *RAWInput) listen(address string) {
 	var err error
-	i.listener, err = capture.NewListener(i.host, i.port, "", i.Engine, i.TrackResponse)
+	i.listener, err = capture.NewListener(i.host, i.ports, "", i.Engine, i.Protocol, i.TrackResponse, i.Expire, i.AllowIncomplete)
 	if err != nil {
 		log.Fatal(err)
 	}
@@ -153,26 +123,20 @@ func (i *RAWInput) listen(address string) {
 	if err != nil {
 		log.Fatal(err)
 	}
-	pool := tcp.NewMessagePool(i.CopyBufferSize, i.Expire, Debug, i.handler)
-	pool.End = endHint
-	pool.Start = startHint
+
 	var ctx context.Context
 	ctx, i.cancelListener = context.WithCancel(context.Background())
-	errCh := i.listener.ListenBackground(ctx, pool.Handler)
-	select {
-	case err := <-errCh:
-		log.Fatal(err)
-	case <-i.listener.Reading:
-		Debug(1, i)
-	}
-}
-
-func (i *RAWInput) handler(m *tcp.Message) {
-	i.message <- m
+	errCh := i.listener.ListenBackground(ctx)
+	<-i.listener.Reading
+	Debug(1, i)
+	go func() {
+		<-errCh // the listener closed voluntarily
+		i.Close()
+	}()
 }
 
 func (i *RAWInput) String() string {
-	return fmt.Sprintf("Intercepting traffic from: %s:%d", i.host, i.port)
+	return fmt.Sprintf("Intercepting traffic from: %s:%s", i.host, strings.Join(strings.Fields(fmt.Sprint(i.ports)), ","))
 }
 
 // GetStats returns the stats so far and reset the stats
@@ -187,8 +151,14 @@ func (i *RAWInput) GetStats() []tcp.Stats {
 
 // Close closes the input raw listener
 func (i *RAWInput) Close() error {
+	i.Lock()
+	defer i.Unlock()
+	if i.closed {
+		return nil
+	}
 	i.cancelListener()
 	close(i.quit)
+	i.closed = true
 	return nil
 }
 
@@ -200,15 +170,3 @@ func (i *RAWInput) addStats(mStats tcp.Stats) {
 	i.messageStats = append(i.messageStats, mStats)
 	i.Unlock()
 }
-
-func startHint(pckt *tcp.Packet) (isIncoming, isOutgoing bool) {
-	isIncoming = proto.HasRequestTitle(pckt.Payload)
-	if isIncoming {
-		return
-	}
-	return false, proto.HasResponseTitle(pckt.Payload)
-}
-
-func endHint(m *tcp.Message) bool {
-	return proto.HasFullPayload(m.Data())
-}

input_raw_test.go

@@ -2,7 +2,6 @@ package main
 
 import (
 	"bytes"
-	"io"
 	"io/ioutil"
 	"net"
 	"net/http"
@@ -17,15 +16,15 @@ import (
 
 	"github.com/buger/goreplay/capture"
 	"github.com/buger/goreplay/proto"
+	"github.com/buger/goreplay/tcp"
 )
 
 const testRawExpire = time.Millisecond * 200
 
 func TestRAWInputIPv4(t *testing.T) {
 	wg := new(sync.WaitGroup)
-	quit := make(chan int)
 
-	listener, err := net.Listen("tcp", ":0")
+	listener, err := net.Listen("tcp4", "127.0.0.1:0")
 	if err != nil {
 		t.Error(err)
 		return
@@ -45,43 +44,47 @@ func TestRAWInputIPv4(t *testing.T) {
 	conf := RAWInputConfig{
 		Engine:        capture.EnginePcap,
 		Expire:        0,
-		Protocol:      ProtocolHTTP,
+		Protocol:      tcp.ProtocolHTTP,
 		TrackResponse: true,
 		RealIPHeader:  "X-Real-IP",
 	}
-	input := NewRAWInput(":"+port, conf)
+	input := NewRAWInput(listener.Addr().String(), conf)
 
-	output := NewTestOutput(func(data []byte) {
-		if data[0] == '1' {
-			body := payloadBody(data)
-			if len(proto.Header(body, []byte("X-Real-IP"))) == 0 {
-				t.Error("Should have X-Real-IP header", string(body))
+	output := NewTestOutput(func(msg *Message) {
+		if msg.Meta[0] == '1' {
+			if len(proto.Header(msg.Data, []byte("X-Real-IP"))) == 0 {
+				t.Error("Should have X-Real-IP header")
 			}
-			atomic.AddInt64(&reqCounter, 1)
+			reqCounter++
 		} else {
-			atomic.AddInt64(&respCounter, 1)
+			respCounter++
 		}
+
 		wg.Done()
 	})
 
 	plugins := &InOutPlugins{
-		Inputs:  []io.Reader{input},
-		Outputs: []io.Writer{output},
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{output},
 	}
 	plugins.All = append(plugins.All, input, output)
 
 	addr := "http://127.0.0.1:" + port
-	emitter := NewEmitter(quit)
+	emitter := NewEmitter()
 	defer emitter.Close()
 	go emitter.Start(plugins, Settings.Middleware)
-	for i := 0; i < 10; i++ {
+
+	// time.Sleep(time.Second)
+	for i := 0; i < 1; i++ {
 		wg.Add(2)
 		_, err = http.Get(addr)
+
 		if err != nil {
 			t.Error(err)
 			return
 		}
 	}
+
 	wg.Wait()
 	const want = 10
 	if reqCounter != respCounter && reqCounter != want {
@@ -91,7 +94,6 @@ func TestRAWInputIPv4(t *testing.T) {
 
 func TestRAWInputNoKeepAlive(t *testing.T) {
 	wg := new(sync.WaitGroup)
-	quit := make(chan int)
 
 	listener, err := net.Listen("tcp", "127.0.0.1:0")
 	if err != nil {
@@ -112,13 +114,13 @@ func TestRAWInputNoKeepAlive(t *testing.T) {
 	conf := RAWInputConfig{
 		Engine:        capture.EnginePcap,
 		Expire:        testRawExpire,
-		Protocol:      ProtocolHTTP,
+		Protocol:      tcp.ProtocolHTTP,
 		TrackResponse: true,
 	}
 	input := NewRAWInput(":"+port, conf)
 	var respCounter, reqCounter int64
-	output := NewTestOutput(func(data []byte) {
-		if data[0] == '1' {
+	output := NewTestOutput(func(msg *Message) {
+		if msg.Meta[0] == '1' {
 			atomic.AddInt64(&reqCounter, 1)
 		} else {
 			atomic.AddInt64(&respCounter, 1)
@@ -127,14 +129,14 @@ func TestRAWInputNoKeepAlive(t *testing.T) {
 	})
 
 	plugins := &InOutPlugins{
-		Inputs:  []io.Reader{input},
-		Outputs: []io.Writer{output},
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{output},
 	}
 	plugins.All = append(plugins.All, input, output)
 
 	addr := "http://127.0.0.1:" + port
 
-	emitter := NewEmitter(quit)
+	emitter := NewEmitter()
 	go emitter.Start(plugins, Settings.Middleware)
 
 	for i := 0; i < 10; i++ {
@@ -157,7 +159,6 @@ func TestRAWInputNoKeepAlive(t *testing.T) {
 
 func TestRAWInputIPv6(t *testing.T) {
 	wg := new(sync.WaitGroup)
-	quit := make(chan int)
 
 	listener, err := net.Listen("tcp", "[::1]:0")
 	if err != nil {
@@ -178,13 +179,13 @@ func TestRAWInputIPv6(t *testing.T) {
 	var respCounter, reqCounter int64
 	conf := RAWInputConfig{
 		Engine:        capture.EnginePcap,
-		Protocol:      ProtocolHTTP,
+		Protocol:      tcp.ProtocolHTTP,
 		TrackResponse: true,
 	}
 	input := NewRAWInput(originAddr, conf)
 
-	output := NewTestOutput(func(data []byte) {
-		if data[0] == '1' {
+	output := NewTestOutput(func(msg *Message) {
+		if msg.Meta[0] == '1' {
 			atomic.AddInt64(&reqCounter, 1)
 		} else {
 			atomic.AddInt64(&respCounter, 1)
@@ -193,11 +194,11 @@ func TestRAWInputIPv6(t *testing.T) {
 	})
 
 	plugins := &InOutPlugins{
-		Inputs:  []io.Reader{input},
-		Outputs: []io.Writer{output},
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{output},
 	}
 
-	emitter := NewEmitter(quit)
+	emitter := NewEmitter()
 	addr := "http://" + originAddr
 	go emitter.Start(plugins, Settings.Middleware)
 	for i := 0; i < 10; i++ {
@@ -220,7 +221,6 @@ func TestRAWInputIPv6(t *testing.T) {
 
 func TestInputRAWChunkedEncoding(t *testing.T) {
 	wg := new(sync.WaitGroup)
-	quit := make(chan int)
 
 	fileContent, _ := ioutil.ReadFile("README.md")
 
@@ -234,10 +234,11 @@ func TestInputRAWChunkedEncoding(t *testing.T) {
 
 	originAddr := strings.Replace(origin.Listener.Addr().String(), "[::]", "127.0.0.1", -1)
 	conf := RAWInputConfig{
-		Engine:        capture.EnginePcap,
-		Expire:        time.Second,
-		Protocol:      ProtocolHTTP,
-		TrackResponse: true,
+		Engine:          capture.EnginePcap,
+		Expire:          time.Second,
+		Protocol:        tcp.ProtocolHTTP,
+		TrackResponse:   true,
+		AllowIncomplete: true,
 	}
 	input := NewRAWInput(originAddr, conf)
 
@@ -257,12 +258,12 @@ func TestInputRAWChunkedEncoding(t *testing.T) {
 	httpOutput := NewHTTPOutput(replay.URL, &HTTPOutputConfig{})
 
 	plugins := &InOutPlugins{
-		Inputs:  []io.Reader{input},
-		Outputs: []io.Writer{httpOutput},
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{httpOutput},
 	}
 	plugins.All = append(plugins.All, input, httpOutput)
 
-	emitter := NewEmitter(quit)
+	emitter := NewEmitter()
 	defer emitter.Close()
 	go emitter.Start(plugins, Settings.Middleware)
 	wg.Add(2)
@@ -278,17 +279,15 @@ func TestInputRAWChunkedEncoding(t *testing.T) {
 }
 
 func BenchmarkRAWInputWithReplay(b *testing.B) {
-	var respCounter, reqCounter, replayCounter uint64
+	var respCounter, reqCounter, replayCounter uint32
 	wg := &sync.WaitGroup{}
-	wg.Add(b.N * 3) // reqCounter + replayCounter + respCounter
 
-	quit := make(chan int)
-	listener, err := net.Listen("tcp", "127.0.0.1:0")
+	listener, err := net.Listen("tcp4", "127.0.0.1:0")
 	if err != nil {
 		b.Error(err)
 		return
 	}
-	listener0, err := net.Listen("tcp", "127.0.0.1:0")
+	listener0, err := net.Listen("tcp4", "127.0.0.1:0")
 	if err != nil {
 		b.Error(err)
 		return
@@ -305,6 +304,8 @@ func BenchmarkRAWInputWithReplay(b *testing.B) {
 
 	replay := http.Server{
 		Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			atomic.AddUint32(&replayCounter, 1)
+			w.Write(nil)
 			wg.Done()
 		}),
 	}
@@ -315,39 +316,42 @@ func BenchmarkRAWInputWithReplay(b *testing.B) {
 	conf := RAWInputConfig{
 		Engine:        capture.EnginePcap,
 		Expire:        testRawExpire,
-		Protocol:      ProtocolHTTP,
+		Protocol:      tcp.ProtocolHTTP,
 		TrackResponse: true,
 	}
 	input := NewRAWInput(originAddr, conf)
 
-	testOutput := NewTestOutput(func(data []byte) {
-		if data[0] == '1' {
-			atomic.AddUint64(&reqCounter, 1)
+	testOutput := NewTestOutput(func(msg *Message) {
+		if msg.Meta[0] == '1' {
+			reqCounter++
 		} else {
-			atomic.AddUint64(&respCounter, 1)
+			respCounter++
 		}
 		wg.Done()
 	})
-	httpOutput := NewHTTPOutput(replayAddr, &HTTPOutputConfig{})
+	httpOutput := NewHTTPOutput("http://"+replayAddr, &HTTPOutputConfig{})
 
 	plugins := &InOutPlugins{
-		Inputs:  []io.Reader{input},
-		Outputs: []io.Writer{testOutput, httpOutput},
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{testOutput, httpOutput},
 	}
 
-	emitter := NewEmitter(quit)
+	emitter := NewEmitter()
 	go emitter.Start(plugins, Settings.Middleware)
-	now := time.Now()
 	addr := "http://" + originAddr
+	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
-		_, err = http.Get(addr)
+		wg.Add(3) // reqCounter + replayCounter + respCounter
+		resp, err := http.Get(addr)
 		if err != nil {
-			b.Log(err)
 			wg.Add(-3)
 		}
+		resp.Body.Close()
 	}
 
 	wg.Wait()
-	b.Logf("%d/%d Requests, %d/%d Responses, %d/%d Replayed in %s\n", reqCounter, b.N, respCounter, b.N, replayCounter, b.N, time.Since(now))
+	b.ReportMetric(float64(reqCounter), "requests")
+	b.ReportMetric(float64(respCounter), "responses")
+	b.ReportMetric(float64(replayCounter), "replayed")
 	emitter.Close()
 }

input_tcp.go

@@ -8,18 +8,18 @@ import (
 	"io"
 	"log"
 	"net"
-	"os"
 )
 
 // TCPInput used for internal communication
 type TCPInput struct {
-	data     chan []byte
+	data     chan *Message
 	listener net.Listener
 	address  string
 	config   *TCPInputConfig
 	stop     chan bool // Channel used only to indicate goroutine should shutdown
 }
 
+// TCPInputConfig represents configuration of a TCP input plugin
 type TCPInputConfig struct {
 	Secure          bool   `json:"input-tcp-secure"`
 	CertificatePath string `json:"input-tcp-certificate"`
@@ -29,7 +29,7 @@ type TCPInputConfig struct {
 // NewTCPInput constructor for TCPInput, accepts address with port
 func NewTCPInput(address string, config *TCPInputConfig) (i *TCPInput) {
 	i = new(TCPInput)
-	i.data = make(chan []byte, 1000)
+	i.data = make(chan *Message, 1000)
 	i.address = address
 	i.config = config
 	i.stop = make(chan bool)
@@ -39,20 +39,21 @@ func NewTCPInput(address string, config *TCPInputConfig) (i *TCPInput) {
 	return
 }
 
-func (i *TCPInput) Read(data []byte) (int, error) {
-	var buf []byte
+// PluginRead returns data and details read from plugin
+func (i *TCPInput) PluginRead() (msg *Message, err error) {
 	select {
 	case <-i.stop:
-		return 0, ErrorStopped
-	case buf = <-i.data:
+		return nil, ErrorStopped
+	case msg = <-i.data:
+		return msg, nil
 	}
-	copy(data, buf)
 
-	return len(buf), nil
 }
 
+// Close closes the plugin
 func (i *TCPInput) Close() error {
 	close(i.stop)
+	i.listener.Close()
 	return nil
 }
 
@@ -60,64 +61,67 @@ func (i *TCPInput) listen(address string) {
 	if i.config.Secure {
 		cer, err := tls.LoadX509KeyPair(i.config.CertificatePath, i.config.KeyPath)
 		if err != nil {
-			log.Fatal("Error while loading --input-file certificate:", err)
+			log.Fatalln("error while loading --input-tcp TLS certificate:", err)
 		}
 
 		config := &tls.Config{Certificates: []tls.Certificate{cer}}
 		listener, err := tls.Listen("tcp", address, config)
 		if err != nil {
-			log.Fatal("Can't start --input-tcp with secure connection:", err)
+			log.Fatalln("[INPUT-TCP] failed to start INPUT-TCP listener:", err)
 		}
 		i.listener = listener
 	} else {
 		listener, err := net.Listen("tcp", address)
 		if err != nil {
-			log.Fatal("Can't start:", err)
+			log.Fatalln("failed to start INPUT-TCP listener:", err)
 		}
-
 		i.listener = listener
 	}
-
 	go func() {
 		for {
 			conn, err := i.listener.Accept()
-
-			if err != nil {
-				log.Println("Error while Accept()", err)
+			if err == nil {
+				go i.handleConnection(conn)
 				continue
 			}
-
-			go i.handleConnection(conn)
+			if isTemporaryNetworkError(err) {
+				continue
+			}
+			if operr, ok := err.(*net.OpError); ok && operr.Err.Error() != "use of closed network connection" {
+				Debug(0, fmt.Sprintf("[INPUT-TCP] listener closed, err: %q", err))
+			}
+			break
 		}
 	}()
 }
 
+var payloadSeparatorAsBytes = []byte(payloadSeparator)
+
 func (i *TCPInput) handleConnection(conn net.Conn) {
 	defer conn.Close()
 
-	payloadSeparatorAsBytes := []byte(payloadSeparator)
 	reader := bufio.NewReader(conn)
 	var buffer bytes.Buffer
 
 	for {
 		line, err := reader.ReadBytes('\n')
-
 		if err != nil {
+			if isTemporaryNetworkError(err) {
+				continue
+			}
 			if err != io.EOF {
-				fmt.Fprintln(os.Stderr, "Unexpected error in input tcp connection:", err)
+				Debug(0, fmt.Sprintf("[INPUT-TCP] connection error: %q", err))
 			}
 			break
-
 		}
 
 		if bytes.Equal(payloadSeparatorAsBytes[1:], line) {
-			asBytes := buffer.Bytes()
+			// unread the '\n' before monkeys
+			buffer.UnreadByte()
+			var msg Message
+			msg.Meta, msg.Data = payloadMetaWithBody(buffer.Bytes())
+			i.data <- &msg
 			buffer.Reset()
-
-			newBuf := make([]byte, len(asBytes)-1)
-			copy(newBuf, asBytes)
-
-			i.data <- newBuf
 		} else {
 			buffer.Write(line)
 		}
@@ -127,3 +131,13 @@ func (i *TCPInput) handleConnection(conn net.Conn) {
 func (i *TCPInput) String() string {
 	return "TCP input: " + i.address
 }
+
+func isTemporaryNetworkError(err error) bool {
+	if nerr, ok := err.(net.Error); ok && nerr.Temporary() {
+		return true
+	}
+	if operr, ok := err.(*net.OpError); ok && operr.Temporary() {
+		return true
+	}
+	return false
+}

input_tcp_test.go

@@ -7,7 +7,6 @@ import (
 	"crypto/tls"
 	"crypto/x509"
 	"encoding/pem"
-	"io"
 	"io/ioutil"
 	"log"
 	"math/big"
@@ -20,20 +19,19 @@ import (
 
 func TestTCPInput(t *testing.T) {
 	wg := new(sync.WaitGroup)
-	quit := make(chan int)
 
 	input := NewTCPInput("127.0.0.1:0", &TCPInputConfig{})
-	output := NewTestOutput(func(data []byte) {
+	output := NewTestOutput(func(*Message) {
 		wg.Done()
 	})
 
 	plugins := &InOutPlugins{
-		Inputs:  []io.Reader{input},
-		Outputs: []io.Writer{output},
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{output},
 	}
 	plugins.All = append(plugins.All, input, output)
 
-	emitter := NewEmitter(quit)
+	emitter := NewEmitter()
 	go emitter.Start(plugins, Settings.Middleware)
 
 	tcpAddr, err := net.ResolveTCPAddr("tcp", input.listener.Addr().String())
@@ -43,7 +41,6 @@ func TestTCPInput(t *testing.T) {
 	}
 
 	conn, err := net.DialTCP("tcp", nil, tcpAddr)
-
 	if err != nil {
 		log.Fatal(err)
 	}
@@ -52,10 +49,14 @@ func TestTCPInput(t *testing.T) {
 
 	for i := 0; i < 100; i++ {
 		wg.Add(1)
-		conn.Write(msg)
-		conn.Write([]byte(payloadSeparator))
+		if _, err = conn.Write(msg); err == nil {
+			_, err = conn.Write(payloadSeparatorAsBytes)
+		}
+		if err != nil {
+			t.Error(err)
+			return
+		}
 	}
-
 	wg.Wait()
 	emitter.Close()
 }
@@ -99,24 +100,23 @@ func TestTCPInputSecure(t *testing.T) {
 	}()
 
 	wg := new(sync.WaitGroup)
-	quit := make(chan int)
 
 	input := NewTCPInput("127.0.0.1:0", &TCPInputConfig{
 		Secure:          true,
 		CertificatePath: serverCertPemFile.Name(),
 		KeyPath:         serverPrivPemFile.Name(),
 	})
-	output := NewTestOutput(func(data []byte) {
+	output := NewTestOutput(func(*Message) {
 		wg.Done()
 	})
 
 	plugins := &InOutPlugins{
-		Inputs:  []io.Reader{input},
-		Outputs: []io.Writer{output},
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{output},
 	}
 	plugins.All = append(plugins.All, input, output)
 
-	emitter := NewEmitter(quit)
+	emitter := NewEmitter()
 	go emitter.Start(plugins, Settings.Middleware)
 
 	conf := &tls.Config{

kafka.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"crypto/tls"
 	"crypto/x509"
+	"errors"
 	"fmt"
 	"io/ioutil"
 	"log"
@@ -54,30 +55,38 @@ type KafkaMessage struct {
 func NewTLSConfig(clientCertFile, clientKeyFile, caCertFile string) (*tls.Config, error) {
 	tlsConfig := tls.Config{}
 
+	if clientCertFile != "" && clientKeyFile == "" {
+		return &tlsConfig, errors.New("Missing key of client certificate in kafka")
+	}
+	if clientCertFile == "" && clientKeyFile != "" {
+		return &tlsConfig, errors.New("missing TLS client certificate in kafka")
+	}
 	// Load client cert
-	cert, err := tls.LoadX509KeyPair(clientCertFile, clientKeyFile)
-	if err != nil {
-		return &tlsConfig, err
+	if (clientCertFile != "") && (clientKeyFile != "") {
+		cert, err := tls.LoadX509KeyPair(clientCertFile, clientKeyFile)
+		if err != nil {
+			return &tlsConfig, err
+		}
+		tlsConfig.Certificates = []tls.Certificate{cert}
 	}
-	tlsConfig.Certificates = []tls.Certificate{cert}
-
 	// Load CA cert
-	caCert, err := ioutil.ReadFile(caCertFile)
-	if err != nil {
-		return &tlsConfig, err
+	if caCertFile != "" {
+		caCert, err := ioutil.ReadFile(caCertFile)
+		if err != nil {
+			return &tlsConfig, err
+		}
+		caCertPool := x509.NewCertPool()
+		caCertPool.AppendCertsFromPEM(caCert)
+		tlsConfig.RootCAs = caCertPool
 	}
-	caCertPool := x509.NewCertPool()
-	caCertPool.AppendCertsFromPEM(caCert)
-	tlsConfig.RootCAs = caCertPool
-
-	return &tlsConfig, err
+	return &tlsConfig, nil
 }
 
 // NewKafkaConfig returns Kafka config with or without TLS
 func NewKafkaConfig(tlsConfig *KafkaTLSConfig) *sarama.Config {
 	config := sarama.NewConfig()
 	// Configuration options go here
-	if (tlsConfig != nil) && (tlsConfig.CACert != "") && (tlsConfig.ClientCert != "") && (tlsConfig.ClientKey != "") {
+	if tlsConfig != nil && (tlsConfig.ClientCert != "" || tlsConfig.CACert != "") {
 		config.Net.TLS.Enable = true
 		tlsConfig, err := NewTLSConfig(tlsConfig.ClientCert, tlsConfig.ClientKey, tlsConfig.CACert)
 		if err != nil {

limiter.go

@@ -20,8 +20,8 @@ type Limiter struct {
 }
 
 func parseLimitOptions(options string) (limit int, isPercent bool) {
-	if strings.Contains(options, "%") {
-		limit, _ = strconv.Atoi(strings.Split(options, "%")[0])
+	if n := strings.Index(options, "%"); n > 0 {
+		limit, _ = strconv.Atoi(options[:n])
 		isPercent = true
 	} else {
 		limit, _ = strconv.Atoi(options)
@@ -33,7 +33,7 @@ func parseLimitOptions(options string) (limit int, isPercent bool) {
 
 // NewLimiter constructor for Limiter, accepts plugin and options
 // `options` allow to sprcify relatve or absolute limiting
-func NewLimiter(plugin interface{}, options string) io.ReadWriter {
+func NewLimiter(plugin interface{}, options string) PluginReadWriter {
 	l := new(Limiter)
 	l.limit, l.isPercent = parseLimitOptions(options)
 	l.plugin = plugin
@@ -71,24 +71,29 @@ func (l *Limiter) isLimited() bool {
 	return false
 }
 
-func (l *Limiter) Write(data []byte) (n int, err error) {
+// PluginWrite writes message to this plugin
+func (l *Limiter) PluginWrite(msg *Message) (n int, err error) {
 	if l.isLimited() {
 		return 0, nil
 	}
-
-	n, err = l.plugin.(io.Writer).Write(data)
-	return
+	if w, ok := l.plugin.(PluginWriter); ok {
+		return w.PluginWrite(msg)
+	}
+	// avoid further writing
+	return 0, io.ErrClosedPipe
 }
 
-func (l *Limiter) Read(data []byte) (n int, err error) {
-	if r, ok := l.plugin.(io.Reader); ok {
-		n, err = r.Read(data)
+// PluginRead reads message from this plugin
+func (l *Limiter) PluginRead() (msg *Message, err error) {
+	if r, ok := l.plugin.(PluginReader); ok {
+		msg, err = r.PluginRead()
 	} else {
-		return 0, nil
+		// avoid further reading
+		return nil, io.ErrClosedPipe
 	}
 
 	if l.isLimited() {
-		return 0, nil
+		return nil, nil
 	}
 
 	return
@@ -100,7 +105,7 @@ func (l *Limiter) String() string {
 
 // Close closes the resources.
 func (l *Limiter) Close() error {
-	if fi, ok := l.plugin.(io.ReadCloser); ok {
+	if fi, ok := l.plugin.(io.Closer); ok {
 		fi.Close()
 	}
 	return nil

limiter_test.go

@@ -3,28 +3,26 @@
 package main
 
 import (
-	"io"
 	"sync"
 	"testing"
 )
 
 func TestOutputLimiter(t *testing.T) {
 	wg := new(sync.WaitGroup)
-	quit := make(chan int)
 
 	input := NewTestInput()
-	output := NewLimiter(NewTestOutput(func(data []byte) {
+	output := NewLimiter(NewTestOutput(func(*Message) {
 		wg.Done()
 	}), "10")
 	wg.Add(10)
 
 	plugins := &InOutPlugins{
-		Inputs:  []io.Reader{input},
-		Outputs: []io.Writer{output},
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{output},
 	}
 	plugins.All = append(plugins.All, input, output)
 
-	emitter := NewEmitter(quit)
+	emitter := NewEmitter()
 	go emitter.Start(plugins, Settings.Middleware)
 
 	for i := 0; i < 100; i++ {
@@ -37,21 +35,20 @@ func TestOutputLimiter(t *testing.T) {
 
 func TestInputLimiter(t *testing.T) {
 	wg := new(sync.WaitGroup)
-	quit := make(chan int)
 
 	input := NewLimiter(NewTestInput(), "10")
-	output := NewTestOutput(func(data []byte) {
+	output := NewTestOutput(func(*Message) {
 		wg.Done()
 	})
 	wg.Add(10)
 
 	plugins := &InOutPlugins{
-		Inputs:  []io.Reader{input},
-		Outputs: []io.Writer{output},
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{output},
 	}
 	plugins.All = append(plugins.All, input, output)
 
-	emitter := NewEmitter(quit)
+	emitter := NewEmitter()
 	go emitter.Start(plugins, Settings.Middleware)
 
 	for i := 0; i < 100; i++ {
@@ -65,20 +62,19 @@ func TestInputLimiter(t *testing.T) {
 // Should limit all requests
 func TestPercentLimiter1(t *testing.T) {
 	wg := new(sync.WaitGroup)
-	quit := make(chan int)
 
 	input := NewTestInput()
-	output := NewLimiter(NewTestOutput(func(data []byte) {
+	output := NewLimiter(NewTestOutput(func(*Message) {
 		wg.Done()
 	}), "0%")
 
 	plugins := &InOutPlugins{
-		Inputs:  []io.Reader{input},
-		Outputs: []io.Writer{output},
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{output},
 	}
 	plugins.All = append(plugins.All, input, output)
 
-	emitter := NewEmitter(quit)
+	emitter := NewEmitter()
 	go emitter.Start(plugins, Settings.Middleware)
 
 	for i := 0; i < 100; i++ {
@@ -91,21 +87,20 @@ func TestPercentLimiter1(t *testing.T) {
 // Should not limit at all
 func TestPercentLimiter2(t *testing.T) {
 	wg := new(sync.WaitGroup)
-	quit := make(chan int)
 
 	input := NewTestInput()
-	output := NewLimiter(NewTestOutput(func(data []byte) {
+	output := NewLimiter(NewTestOutput(func(*Message) {
 		wg.Done()
 	}), "100%")
 	wg.Add(100)
 
 	plugins := &InOutPlugins{
-		Inputs:  []io.Reader{input},
-		Outputs: []io.Writer{output},
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{output},
 	}
 	plugins.All = append(plugins.All, input, output)
 
-	emitter := NewEmitter(quit)
+	emitter := NewEmitter()
 	go emitter.Start(plugins, Settings.Middleware)
 
 	for i := 0; i < 100; i++ {

middleware.go

@@ -2,39 +2,40 @@ package main
 
 import (
 	"bufio"
+	"context"
 	"encoding/hex"
 	"fmt"
 	"io"
-	"log"
 	"os"
 	"os/exec"
 	"strings"
 	"sync"
+	"syscall"
 )
 
 // Middleware represents a middleware object
 type Middleware struct {
-	command string
-
-	data chan []byte
-
-	mu sync.Mutex
-
-	Stdin  io.Writer
-	Stdout io.Reader
-
-	stop chan bool // Channel used only to indicate goroutine should shutdown
+	command       string
+	data          chan *Message
+	Stdin         io.Writer
+	Stdout        io.Reader
+	commandCancel context.CancelFunc
+	stop          chan bool // Channel used only to indicate goroutine should shutdown
+	closed        bool
+	mu            sync.RWMutex
 }
 
 // NewMiddleware returns new middleware
 func NewMiddleware(command string) *Middleware {
 	m := new(Middleware)
 	m.command = command
-	m.data = make(chan []byte, 1000)
+	m.data = make(chan *Message, 1000)
 	m.stop = make(chan bool)
 
 	commands := strings.Split(command, " ")
-	cmd := exec.Command(commands[0], commands[1:]...)
+	ctx, cancl := context.WithCancel(context.Background())
+	m.commandCancel = cancl
+	cmd := exec.CommandContext(ctx, commands[0], commands[1:]...)
 
 	m.Stdout, _ = cmd.StdoutPipe()
 	m.Stdin, _ = cmd.StdinPipe()
@@ -44,16 +45,19 @@ func NewMiddleware(command string) *Middleware {
 	go m.read(m.Stdout)
 
 	go func() {
-		err := cmd.Start()
-
-		if err != nil {
-			log.Fatal(err)
+		defer m.Close()
+		var err error
+		if err = cmd.Start(); err == nil {
+			err = cmd.Wait()
 		}
-
-		err = cmd.Wait()
-
 		if err != nil {
-			log.Fatal(err)
+			if e, ok := err.(*exec.ExitError); ok {
+				status := e.Sys().(syscall.WaitStatus)
+				if status.Signal() == syscall.SIGKILL /*killed or context canceld */ {
+					return
+				}
+			}
+			Debug(0, fmt.Sprintf("[MIDDLEWARE] command[%q] error: %q", command, err.Error()))
 		}
 	}()
 
@@ -61,45 +65,42 @@ func NewMiddleware(command string) *Middleware {
 }
 
 // ReadFrom start a worker to read from this plugin
-func (m *Middleware) ReadFrom(plugin io.Reader) {
-	Debug(2, "[MIDDLEWARE-MASTER] Starting reading from", plugin)
+func (m *Middleware) ReadFrom(plugin PluginReader) {
+	Debug(2, fmt.Sprintf("[MIDDLEWARE] command[%q] Starting reading from %q", m.command, plugin))
 	go m.copy(m.Stdin, plugin)
 }
 
-func (m *Middleware) copy(to io.Writer, from io.Reader) {
-	buf := make([]byte, 5*1024*1024)
-	dst := make([]byte, len(buf)*4)
+func (m *Middleware) copy(to io.Writer, from PluginReader) {
+	var buf, dst []byte
 
 	for {
-		nr, _ := from.Read(buf)
-		if nr == 0 || nr > len(buf) {
+		msg, err := from.PluginRead()
+		if err != nil {
+			return
+		}
+		if msg == nil || len(msg.Data) == 0 {
 			continue
 		}
-
-		payload := buf[0:nr]
-
+		buf = msg.Data
 		if Settings.PrettifyHTTP {
-			payload = prettifyHTTP(payload)
-			nr = len(payload)
-
-			if nr*2 > len(dst) {
-				continue
-			}
+			buf = prettifyHTTP(msg.Data)
 		}
-
-		if Settings.PrettifyHTTP {
-			payload = prettifyHTTP(payload)
-			nr = len(payload)
+		dstLen := (len(buf)+len(msg.Meta))*2 + 1
+		// if enough space was previously allocated use it instead
+		if dstLen > len(dst) {
+			dst = make([]byte, dstLen)
 		}
+		n := hex.Encode(dst, msg.Meta)
+		n += hex.Encode(dst[n:], buf)
+		dst[n] = '\n'
 
-		hex.Encode(dst, payload)
-		dst[nr*2] = '\n'
-
-		m.mu.Lock()
-		to.Write(dst[0 : nr*2+1])
-		m.mu.Unlock()
-
-		Debug(3, "[MIDDLEWARE-MASTER] Sending:", string(buf[0:nr]), "From:", from)
+		n, err = to.Write(dst[:n+1])
+		if err == nil {
+			continue
+		}
+		if m.isClosed() {
+			return
+		}
 	}
 }
 
@@ -107,51 +108,59 @@ func (m *Middleware) read(from io.Reader) {
 	reader := bufio.NewReader(from)
 	var line []byte
 	var e error
-
 	for {
 		if line, e = reader.ReadBytes('\n'); e != nil {
-			if e == io.EOF {
-				continue
-			} else {
-				break
+			if m.isClosed() {
+				return
 			}
+			continue
 		}
-
-		buf := make([]byte, len(line)/2)
+		buf := make([]byte, (len(line)-1)/2)
 		if _, err := hex.Decode(buf, line[:len(line)-1]); err != nil {
-			fmt.Fprintln(os.Stderr, "Failed to decode input payload", err, len(line), string(line[:len(line)-1]))
+			Debug(0, fmt.Sprintf("[MIDDLEWARE] command[%q] failed to decode err: %q", m.command, err))
+			continue
 		}
-
-		Debug(3, "[MIDDLEWARE-MASTER] Received:", string(buf))
-
+		var msg Message
+		msg.Meta, msg.Data = payloadMetaWithBody(buf)
 		select {
 		case <-m.stop:
 			return
-		case m.data <- buf:
+		case m.data <- &msg:
 		}
 	}
 
+}
+
+// PluginRead reads message from this plugin
+func (m *Middleware) PluginRead() (msg *Message, err error) {
+	select {
+	case <-m.stop:
+		return nil, ErrorStopped
+	case msg = <-m.data:
+	}
+
 	return
 }
 
-func (m *Middleware) Read(data []byte) (int, error) {
-	var buf []byte
-	select {
-	case <-m.stop:
-		return 0, ErrorStopped
-	case buf = <-m.data:
-	}
-
-	n := copy(data, buf)
-	return n, nil
+func (m *Middleware) String() string {
+	return fmt.Sprintf("Modifying traffic using %q command", m.command)
 }
 
-func (m *Middleware) String() string {
-	return fmt.Sprintf("Modifying traffic using '%s' command", m.command)
+func (m *Middleware) isClosed() bool {
+	m.mu.RLock()
+	defer m.mu.RUnlock()
+	return m.closed
 }
 
 // Close closes this plugin
 func (m *Middleware) Close() error {
+	if m.isClosed() {
+		return nil
+	}
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	m.commandCancel()
 	close(m.stop)
+	m.closed = true
 	return nil
 }

middleware_test.go

@@ -1,225 +1,154 @@
 package main
 
-// import (
-// 	"bytes"
-// 	"crypto/rand"
-// 	"encoding/hex"
-// 	"io"
-// 	"net/http"
-// 	"net/http/httptest"
-// 	"strings"
-// 	"sync"
-// 	"testing"
-// 	"time"
+import (
+	"bytes"
+	"context"
+	"os"
+	"os/exec"
+	"strings"
+	"sync/atomic"
+	"syscall"
+	"testing"
 
-// 	"github.com/buger/goreplay/capture"
-// 	"github.com/buger/goreplay/proto"
-// )
+	"github.com/buger/goreplay/proto"
+)
 
-// type fakeServiceCb func(string, int, []byte)
+const echoSh = "./examples/middleware/echo.sh"
+const tokenModifier = "go run ./examples/middleware/token_modifier.go"
 
-// // Simple service that generate token on request, and require this token for accesing to secure area
-// func NewFakeSecureService(wg *sync.WaitGroup, cb fakeServiceCb) *httptest.Server {
-// 	active_tokens := make([]string, 0)
-// 	var mu sync.Mutex
+var noDebug = append(syscall.Environ(), "GOR_TEST=1")
 
-// 	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
-// 		mu.Lock()
-// 		defer mu.Unlock()
+func initMiddleware(cmd *exec.Cmd, cancl context.CancelFunc, l PluginReader, c func(error)) *Middleware {
+	var m Middleware
+	m.data = make(chan *Message, 1000)
+	m.stop = make(chan bool)
+	m.commandCancel = cancl
+	m.Stdout, _ = cmd.StdoutPipe()
+	m.Stdin, _ = cmd.StdinPipe()
+	cmd.Stderr = os.Stderr
+	go m.read(m.Stdout)
+	go func() {
+		defer m.Close()
+		var err error
+		if err = cmd.Start(); err == nil {
+			err = cmd.Wait()
+		}
+		if err != nil {
+			c(err)
+		}
+	}()
+	m.ReadFrom(l)
+	return &m
+}
 
-// 		switch req.URL.Path {
-// 		case "/token":
-// 			// Generate random token
-// 			token_length := 10
-// 			buf := make([]byte, token_length)
-// 			rand.Read(buf)
-// 			token := hex.EncodeToString(buf)
-// 			active_tokens = append(active_tokens, token)
+func initCmd(command string, env []string) (*exec.Cmd, context.CancelFunc) {
+	commands := strings.Split(command, " ")
+	ctx, cancl := context.WithCancel(context.Background())
+	cmd := exec.CommandContext(ctx, commands[0], commands[1:]...)
+	cmd.Env = env
+	return cmd, cancl
+}
 
-// 			w.Write([]byte(token))
+func TestMiddlewareEarlyClose(t *testing.T) {
+	quit := make(chan struct{})
+	in := NewTestInput()
+	cmd, cancl := initCmd(echoSh, noDebug)
+	midd := initMiddleware(cmd, cancl, in, func(err error) {
+		if err != nil {
+			if e, ok := err.(*exec.ExitError); ok {
+				status := e.Sys().(syscall.WaitStatus)
+				if status.Signal() != syscall.SIGKILL {
+					t.Errorf("expected error to be signal killed. got %s", status.Signal().String())
+				}
+			}
+		}
+		quit <- struct{}{}
+	})
+	var body = []byte("OPTIONS / HTTP/1.1\r\nHost: example.org\r\n\r\n")
+	count := uint32(0)
+	out := NewTestOutput(func(msg *Message) {
+		if !bytes.Equal(body, msg.Data) {
+			t.Errorf("expected %q to equal %q", body, msg.Data)
+		}
+		atomic.AddUint32(&count, 1)
+		if atomic.LoadUint32(&count) == 5 {
+			quit <- struct{}{}
+		}
+	})
+	pl := &InOutPlugins{}
+	pl.Inputs = []PluginReader{midd, in}
+	pl.Outputs = []PluginWriter{out}
+	pl.All = []interface{}{midd, out, in}
+	e := NewEmitter()
+	go e.Start(pl, "")
+	for i := 0; i < 5; i++ {
+		in.EmitBytes(body)
+	}
+	<-quit
+	midd.Close()
+	<-quit
+}
 
-// 			cb(req.URL.Path, 200, []byte(token))
-// 		case "/secure":
-// 			token := req.URL.Query().Get("token")
-// 			token_found := false
+func TestTokenMiddleware(t *testing.T) {
+	quit := make(chan struct{})
+	in := NewTestInput()
+	in.skipHeader = true
+	cmd, cancl := initCmd(tokenModifier, noDebug)
+	midd := initMiddleware(cmd, cancl, in, func(err error) {})
+	req := []byte("1 932079936fa4306fc308d67588178d17d823647c 1439818823587396305 200\nGET /token HTTP/1.1\r\nHost: example.org\r\n\r\n")
+	res := []byte("2 932079936fa4306fc308d67588178d17d823647c 1439818823587396305 200\nHTTP/1.1 200 OK\r\nContent-Length: 10\r\nContent-Type: text/plain; charset=utf-8\r\n\r\n17d823647c")
+	rep := []byte("3 932079936fa4306fc308d67588178d17d823647c 1439818823587396305 200\nHTTP/1.1 200 OK\r\nContent-Length: 15\r\nContent-Type: text/plain; charset=utf-8\r\n\r\n932079936fa4306")
+	count := uint32(0)
+	out := NewTestOutput(func(msg *Message) {
+		if msg.Meta[0] == '1' && !bytes.Equal(payloadID(msg.Meta), payloadID(req)) {
+			token, _, _ := proto.PathParam(msg.Data, []byte("token"))
+			if !bytes.Equal(token, proto.Body(rep)) {
+				t.Error("expected the token to be equal to the replayed responses's token")
+			}
+		}
+		atomic.AddUint32(&count, 1)
+		if atomic.LoadUint32(&count) == 2 {
+			quit <- struct{}{}
+		}
+	})
+	pl := &InOutPlugins{}
+	pl.Inputs = []PluginReader{midd, in}
+	pl.Outputs = []PluginWriter{out}
+	pl.All = []interface{}{midd, out, in}
+	e := NewEmitter()
+	go e.Start(pl, "")
+	in.EmitBytes(req) // emit original request
+	in.EmitBytes(res) // emit its response
+	in.EmitBytes(rep) // emit replayed response
+	// emit the request which should have modified token
+	token := []byte("1 8e091765ae902fef8a2b7d9dd96 14398188235873 100\nGET /?token=17d823647c HTTP/1.1\r\nHost: example.org\r\n\r\n")
+	in.EmitBytes(token)
+	<-quit
+	midd.Close()
+}
 
-// 			for _, t := range active_tokens {
-// 				if t == token {
-// 					token_found = true
-// 					break
-// 				}
-// 			}
-
-// 			if token_found {
-// 				w.WriteHeader(http.StatusAccepted)
-// 				cb(req.URL.Path, 202, []byte(nil))
-// 			} else {
-// 				w.WriteHeader(http.StatusForbidden)
-// 				cb(req.URL.Path, 403, []byte(nil))
-// 			}
-// 		}
-
-// 		wg.Done()
-// 	}))
-
-// 	return server
-// }
-
-// func TestFakeSecureService(t *testing.T) {
-// 	var resp, token []byte
-
-// 	wg := new(sync.WaitGroup)
-
-// 	server := NewFakeSecureService(wg, func(path string, status int, resp []byte) {
-// 	})
-// 	defer server.Close()
-
-// 	wg.Add(3)
-
-// 	client := NewHTTPClient(server.URL, &HTTPClientConfig{Debug: true})
-// 	resp, _ = client.Get("/token")
-// 	token = proto.Body(resp)
-
-// 	// Right token
-// 	resp, _ = client.Get("/secure?token=" + string(token))
-// 	if !bytes.Equal(proto.Status(resp), []byte("202")) {
-// 		t.Error("Valid token should return status 202:", string(proto.Status(resp)))
-// 	}
-
-// 	// Wrong tokens forbidden
-// 	resp, _ = client.Get("/secure?token=wrong")
-// 	if !bytes.Equal(proto.Status(resp), []byte("403")) {
-// 		t.Error("Wrong token should returns status 403:", string(proto.Status(resp)))
-// 	}
-
-// 	wg.Wait()
-// }
-
-// func TestEchoMiddleware(t *testing.T) {
-// 	wg := new(sync.WaitGroup)
-
-// 	from := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-// 		w.Header().Set("Env", "prod")
-// 		w.Header().Set("RequestPath", r.URL.Path)
-// 		wg.Done()
-// 	}))
-// 	defer from.Close()
-
-// 	to := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-// 		w.Header().Set("Env", "test")
-// 		w.Header().Set("RequestPath", r.URL.Path)
-// 		wg.Done()
-// 	}))
-// 	defer to.Close()
-
-// 	quit := make(chan int)
-
-// 	// Catch traffic from one service
-// 	fromAddr := strings.Replace(from.Listener.Addr().String(), "[::]", "127.0.0.1", -1)
-// 	conf := RAWInputConfig{
-// 		engine:        capture.EnginePcap,
-// 		expire:        testRawExpire,
-// 		protocol:      ProtocolHTTP,
-// 		trackResponse: true,
-// 	}
-// 	input := NewRAWInput(fromAddr, conf)
-
-// 	// And redirect to another
-// 	output := NewHTTPOutput(to.URL, &HTTPOutputConfig{Debug: false})
-
-// 	plugins := &InOutPlugins{
-// 		Inputs:  []io.Reader{input},
-// 		Outputs: []io.Writer{output},
-// 	}
-// 	plugins.All = append(plugins.All, input, output)
-
-// 	// Start Gor
-// 	emitter := NewEmitter(quit)
-// 	go emitter.Start(plugins, "echo -n && GOR_TEST=true && ./examples/middleware/echo.sh")
-
-// 	// Wait till middleware initialization
-// 	time.Sleep(100 * time.Millisecond)
-
-// 	// Should receive 2 requests from original + 2 from replayed
-// 	client := NewHTTPClient(from.URL, &HTTPClientConfig{Debug: false})
-
-// 	for i := 0; i < 10; i++ {
-// 		wg.Add(2)
-// 		// Request should be echoed
-// 		client.Get("/a")
-// 		time.Sleep(5 * time.Millisecond)
-// 		client.Get("/b")
-// 		time.Sleep(5 * time.Millisecond)
-// 	}
-
-// 	wg.Wait()
-// 	emitter.Close()
-// }
-
-// func TestTokenMiddleware(t *testing.T) {
-// 	var resp, token []byte
-
-// 	wg := new(sync.WaitGroup)
-
-// 	from := NewFakeSecureService(wg, func(path string, status int, tok []byte) {
-// 		time.Sleep(10 * time.Millisecond)
-// 	})
-// 	defer from.Close()
-
-// 	to := NewFakeSecureService(wg, func(path string, status int, tok []byte) {
-// 		switch path {
-// 		case "/secure":
-// 			if status != 202 {
-// 				t.Error("Server should receive valid rewritten token")
-// 			}
-// 		}
-
-// 		time.Sleep(10 * time.Millisecond)
-// 	})
-// 	defer to.Close()
-
-// 	quit := make(chan int)
-
-// 	Settings.middleware = "echo -n && GOR_TEST=true && go run ./examples/middleware/token_modifier.go"
-
-// 	fromAddr := strings.Replace(from.Listener.Addr().String(), "[::]", "127.0.0.1", -1)
-// 	conf := RAWInputConfig{
-// 		engine:        capture.EnginePcap,
-// 		expire:        testRawExpire,
-// 		protocol:      ProtocolHTTP,
-// 		trackResponse: true,
-// 	}
-// 	// Catch traffic from one service
-// 	input := NewRAWInput(fromAddr, conf)
-
-// 	// And redirect to another
-// 	output := NewHTTPOutput(to.URL, &HTTPOutputConfig{Debug: true})
-
-// 	plugins := &InOutPlugins{
-// 		Inputs:  []io.Reader{input},
-// 		Outputs: []io.Writer{output},
-// 	}
-// 	plugins.All = append(plugins.All, input, output)
-
-// 	// Start Gor
-// 	emitter := NewEmitter(quit)
-// 	go emitter.Start(plugins, Settings.middleware)
-
-// 	// Should receive 2 requests from original + 2 from replayed
-// 	wg.Add(2)
-
-// 	client := NewHTTPClient(from.URL, &HTTPClientConfig{Debug: true})
-
-// 	// Sending traffic to original service
-// 	resp, _ = client.Get("/token")
-// 	token = proto.Body(resp)
-
-// 	resp, _ = client.Get("/secure?token=" + string(token))
-// 	if !bytes.Equal(proto.Status(resp), []byte("202")) {
-// 		t.Error("Valid token should return 202:", proto.Status(resp))
-// 	}
-
-// 	wg.Wait()
-// 	emitter.Close()
-// 	Settings.middleware = ""
-// }
+func TestMiddlewareWithPrettify(t *testing.T) {
+	Settings.PrettifyHTTP = true
+	quit := make(chan struct{})
+	in := NewTestInput()
+	cmd, cancl := initCmd(echoSh, noDebug)
+	midd := initMiddleware(cmd, cancl, in, func(err error) {})
+	var b1 = []byte("POST / HTTP/1.1\r\nHost: example.org\r\nTransfer-Encoding: chunked\r\n\r\n4\r\nWiki\r\n5\r\npedia\r\nE\r\n in\r\n\r\nchunks.\r\n0\r\n\r\n")
+	var b2 = []byte("POST / HTTP/1.1\r\nHost: example.org\r\nContent-Length: 25\r\n\r\nWikipedia in\r\n\r\nchunks.")
+	out := NewTestOutput(func(msg *Message) {
+		if !bytes.Equal(proto.Body(b2), proto.Body(msg.Data)) {
+			t.Errorf("expected %q body to equal %q body", b2, msg.Data)
+		}
+		quit <- struct{}{}
+	})
+	pl := &InOutPlugins{}
+	pl.Inputs = []PluginReader{midd, in}
+	pl.Outputs = []PluginWriter{out}
+	pl.All = []interface{}{midd, out, in}
+	e := NewEmitter()
+	go e.Start(pl, "")
+	in.EmitBytes(b1)
+	<-quit
+	midd.Close()
+	Settings.PrettifyHTTP = false
+}

output_binary.go

@@ -1,7 +1,6 @@
 package main
 
 import (
-	"io"
 	"sync/atomic"
 	"time"
 
@@ -25,30 +24,27 @@ type BinaryOutput struct {
 	// alignment. atomic.* functions crash on 32bit machines if operand is not
 	// aligned at 64bit. See https://github.com/golang/go/issues/599
 	activeWorkers int64
-
-	address string
-	queue   chan []byte
-
-	responses chan response
-
-	needWorker chan int
-
-	config *BinaryOutputConfig
-
-	queueStats *GorStat
+	address       string
+	queue         chan *Message
+	responses     chan response
+	needWorker    chan int
+	quit          chan struct{}
+	config        *BinaryOutputConfig
+	queueStats    *GorStat
 }
 
 // NewBinaryOutput constructor for BinaryOutput
 // Initialize workers
-func NewBinaryOutput(address string, config *BinaryOutputConfig) io.Writer {
+func NewBinaryOutput(address string, config *BinaryOutputConfig) PluginReadWriter {
 	o := new(BinaryOutput)
 
 	o.address = address
 	o.config = config
 
-	o.queue = make(chan []byte, 1000)
+	o.queue = make(chan *Message, 1000)
 	o.responses = make(chan response, 1000)
 	o.needWorker = make(chan int, 1)
+	o.quit = make(chan struct{})
 
 	// Initial workers count
 	if o.config.Workers == 0 {
@@ -89,8 +85,8 @@ func (o *BinaryOutput) startWorker() {
 
 	for {
 		select {
-		case data := <-o.queue:
-			o.sendRequest(client, data)
+		case msg := <-o.queue:
+			o.sendRequest(client, msg)
 			deathCount = 0
 		case <-time.After(time.Millisecond * 100):
 			// When dynamic scaling enabled workers die after 2s of inactivity
@@ -113,15 +109,13 @@ func (o *BinaryOutput) startWorker() {
 	}
 }
 
-func (o *BinaryOutput) Write(data []byte) (n int, err error) {
-	if !isRequestPayload(data) {
-		return len(data), nil
+// PluginWrite writes a message tothis plugin
+func (o *BinaryOutput) PluginWrite(msg *Message) (n int, err error) {
+	if !isRequestPayload(msg.Meta) {
+		return len(msg.Data), nil
 	}
 
-	buf := make([]byte, len(data))
-	copy(buf, data)
-
-	o.queue <- buf
+	o.queue <- msg
 
 	if o.config.Workers == 0 {
 		workersCount := atomic.LoadInt64(&o.activeWorkers)
@@ -131,37 +125,33 @@ func (o *BinaryOutput) Write(data []byte) (n int, err error) {
 		}
 	}
 
-	return len(data), nil
+	return len(msg.Data) + len(msg.Meta), nil
 }
 
-func (o *BinaryOutput) Read(data []byte) (int, error) {
-	resp := <-o.responses
+// PluginRead reads a message from this plugin
+func (o *BinaryOutput) PluginRead() (*Message, error) {
+	var resp response
+	var msg Message
+	select {
+	case <-o.quit:
+		return nil, ErrorStopped
+	case resp = <-o.responses:
+	}
+	msg.Data = resp.payload
+	msg.Meta = payloadHeader(ReplayedResponsePayload, resp.uuid, resp.startedAt, resp.roundTripTime)
 
-	Debug(2, "[OUTPUT-TCP] Received response:", string(resp.payload))
-
-	header := payloadHeader(ReplayedResponsePayload, resp.uuid, resp.startedAt, resp.roundTripTime)
-	copy(data[0:len(header)], header)
-	copy(data[len(header):], resp.payload)
-
-	return len(resp.payload) + len(header), nil
+	return &msg, nil
 }
 
-func (o *BinaryOutput) sendRequest(client *TCPClient, request []byte) {
-	meta := payloadMeta(request)
-	if len(meta) < 2 {
+func (o *BinaryOutput) sendRequest(client *TCPClient, msg *Message) {
+	if !isRequestPayload(msg.Meta) {
 		return
 	}
 
-	if !isRequestPayload(request) {
-		return
-	}
-
-	uuid := meta[1]
-
-	body := payloadBody(request)
+	uuid := payloadID(msg.Meta)
 
 	start := time.Now()
-	resp, err := client.Send(body)
+	resp, err := client.Send(msg.Data)
 	stop := time.Now()
 
 	if err != nil {
@@ -176,3 +166,9 @@ func (o *BinaryOutput) sendRequest(client *TCPClient, request []byte) {
 func (o *BinaryOutput) String() string {
 	return "Binary output: " + o.address
 }
+
+// Close closes this plugin for reading
+func (o *BinaryOutput) Close() error {
+	close(o.quit)
+	return nil
+}

output_dummy.go

@@ -15,9 +15,16 @@ func NewDummyOutput() (di *DummyOutput) {
 	return
 }
 
-func (i *DummyOutput) Write(data []byte) (int, error) {
-	n, err := os.Stdout.Write(data)
-	os.Stdout.Write([]byte{'\n'})
+// PluginWrite writes message to this plugin
+func (i *DummyOutput) PluginWrite(msg *Message) (int, error) {
+	var n, nn int
+	var err error
+	n, err = os.Stdout.Write(msg.Meta)
+	nn, err = os.Stdout.Write(msg.Data)
+	n += nn
+	nn, err = os.Stdout.Write(payloadSeparatorAsBytes)
+	n += nn
+
 	return n, err
 }
 

output_file.go

@@ -7,6 +7,7 @@ import (
 	"fmt"
 	"io"
 	"log"
+	"math/rand"
 	"os"
 	"path/filepath"
 	"runtime/debug"
@@ -19,6 +20,21 @@ import (
 	"github.com/buger/goreplay/size"
 )
 
+var letters = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ")
+var instanceID string
+
+func init() {
+	instanceID = randSeq(8)
+}
+
+func randSeq(n int) string {
+	b := make([]rune, n)
+	for i := range b {
+		b[i] = letters[rand.Intn(len(letters))]
+	}
+	return string(b)
+}
+
 var dateFileNameFuncs = map[string]func(*FileOutput) string{
 	"%Y":  func(o *FileOutput) string { return time.Now().Format("2006") },
 	"%m":  func(o *FileOutput) string { return time.Now().Format("01") },
@@ -29,6 +45,7 @@ var dateFileNameFuncs = map[string]func(*FileOutput) string{
 	"%NS": func(o *FileOutput) string { return fmt.Sprint(time.Now().Nanosecond()) },
 	"%r":  func(o *FileOutput) string { return string(o.currentID) },
 	"%t":  func(o *FileOutput) string { return string(o.payloadType) },
+	"%i":  func(o *FileOutput) string { return instanceID },
 }
 
 // FileOutputConfig ...
@@ -45,17 +62,17 @@ type FileOutputConfig struct {
 // FileOutput output plugin
 type FileOutput struct {
 	sync.RWMutex
-	pathTemplate   string
-	currentName    string
-	file           *os.File
-	QueueLength    int
-	chunkSize      int
-	writer         io.Writer
-	requestPerFile bool
-	currentID      []byte
-	payloadType    []byte
-	closed         bool
-	totalFileSize  size.Size
+	pathTemplate    string
+	currentName     string
+	file            *os.File
+	QueueLength     int
+	writer          io.Writer
+	requestPerFile  bool
+	currentID       []byte
+	payloadType     []byte
+	closed          bool
+	currentFileSize int
+	totalFileSize   size.Size
 
 	config *FileOutputConfig
 }
@@ -157,7 +174,7 @@ func (o *FileOutput) filename() string {
 
 		if o.currentName == "" ||
 			((o.config.QueueLimit > 0 && o.QueueLength >= o.config.QueueLimit) ||
-				(o.config.SizeLimit > 0 && o.chunkSize >= int(o.config.SizeLimit))) {
+				(o.config.SizeLimit > 0 && o.currentFileSize >= int(o.config.SizeLimit))) {
 			nextChunk = true
 		}
 
@@ -195,10 +212,11 @@ func (o *FileOutput) updateName() {
 	o.Unlock()
 }
 
-func (o *FileOutput) Write(data []byte) (n int, err error) {
+// PluginWrite writes message to this plugin
+func (o *FileOutput) PluginWrite(msg *Message) (n int, err error) {
 	if o.requestPerFile {
 		o.Lock()
-		meta := payloadMeta(data)
+		meta := payloadMeta(msg.Meta)
 		o.currentID = meta[1]
 		o.payloadType = meta[0]
 		o.Unlock()
@@ -227,26 +245,29 @@ func (o *FileOutput) Write(data []byte) (n int, err error) {
 		o.QueueLength = 0
 	}
 
-	n, _ = o.writer.Write(data)
-	nSeparator, _ := o.writer.Write([]byte(payloadSeparator))
-
-	n += nSeparator
+	var nn int
+	n, err = o.writer.Write(msg.Meta)
+	nn, err = o.writer.Write(msg.Data)
+	n += nn
+	nn, err = o.writer.Write(payloadSeparatorAsBytes)
+	n += nn
 
 	o.totalFileSize += size.Size(n)
+	o.currentFileSize += n
 	o.QueueLength++
 
 	if Settings.OutputFileConfig.OutputFileMaxSize > 0 && o.totalFileSize >= Settings.OutputFileConfig.OutputFileMaxSize {
 		return n, errors.New("File output reached size limit")
 	}
 
-	return n, nil
+	return n, err
 }
 
 func (o *FileOutput) flush() {
 	// Don't exit on panic
 	defer func() {
 		if r := recover(); r != nil {
-			log.Println("PANIC while file flush: ", r, o, string(debug.Stack()))
+			Debug(0, "[OUTPUT-FILE] PANIC while file flush: ", r, o, string(debug.Stack()))
 		}
 	}()
 
@@ -261,9 +282,9 @@ func (o *FileOutput) flush() {
 		}
 
 		if stat, err := o.file.Stat(); err == nil {
-			o.chunkSize = int(stat.Size())
+			o.currentFileSize = int(stat.Size())
 		} else {
-			log.Println("Error accessing file sats", err)
+			Debug(0, "[OUTPUT-HTTP] error accessing file size", err)
 		}
 	}
 }
@@ -287,6 +308,8 @@ func (o *FileOutput) closeLocked() error {
 	}
 
 	o.closed = true
+	o.currentFileSize = 0
+
 	return nil
 }
 

output_file_test.go

@@ -2,7 +2,6 @@ package main
 
 import (
 	"fmt"
-	"io"
 	"math/rand"
 	"os"
 	"reflect"
@@ -17,18 +16,17 @@ import (
 
 func TestFileOutput(t *testing.T) {
 	wg := new(sync.WaitGroup)
-	quit := make(chan int)
 
 	input := NewTestInput()
 	output := NewFileOutput("/tmp/test_requests.gor", &FileOutputConfig{FlushInterval: time.Minute, Append: true})
 
 	plugins := &InOutPlugins{
-		Inputs:  []io.Reader{input},
-		Outputs: []io.Writer{output},
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{output},
 	}
 	plugins.All = append(plugins.All, input, output)
 
-	emitter := NewEmitter(quit)
+	emitter := NewEmitter()
 	go emitter.Start(plugins, Settings.Middleware)
 
 	for i := 0; i < 100; i++ {
@@ -41,20 +39,19 @@ func TestFileOutput(t *testing.T) {
 	emitter.Close()
 
 	var counter int64
-	input2 := NewFileInput("/tmp/test_requests.gor", false)
-	output2 := NewTestOutput(func(data []byte) {
+	input2 := NewFileInput("/tmp/test_requests.gor", false, 100, 0, false)
+	output2 := NewTestOutput(func(*Message) {
 		atomic.AddInt64(&counter, 1)
 		wg.Done()
 	})
 
 	plugins2 := &InOutPlugins{
-		Inputs:  []io.Reader{input2},
-		Outputs: []io.Writer{output2},
+		Inputs:  []PluginReader{input2},
+		Outputs: []PluginWriter{output2},
 	}
 	plugins2.All = append(plugins2.All, input2, output2)
 
-	quit2 := make(chan int)
-	emitter2 := NewEmitter(quit2)
+	emitter2 := NewEmitter()
 	go emitter2.Start(plugins2, Settings.Middleware)
 
 	wg.Wait()
@@ -91,16 +88,16 @@ func TestFileOutputMultipleFiles(t *testing.T) {
 		t.Error("Should not initialize file if no writes")
 	}
 
-	output.Write([]byte("1 1 1\r\ntest"))
+	output.PluginWrite(&Message{Meta: []byte("1 1 1\r\n"), Data: []byte("test")})
 	name1 := output.file.Name()
 
-	output.Write([]byte("1 1 1\r\ntest"))
+	output.PluginWrite(&Message{Meta: []byte("1 1 1\r\n"), Data: []byte("test")})
 	name2 := output.file.Name()
 
 	time.Sleep(time.Second)
 	output.updateName()
 
-	output.Write([]byte("1 1 1\r\ntest"))
+	output.PluginWrite(&Message{Meta: []byte("1 1 1\r\n"), Data: []byte("test")})
 	name3 := output.file.Name()
 
 	if name2 != name1 {
@@ -122,16 +119,16 @@ func TestFileOutputFilePerRequest(t *testing.T) {
 		t.Error("Should not initialize file if no writes")
 	}
 
-	output.Write([]byte("1 1 1\ntest"))
+	output.PluginWrite(&Message{Meta: []byte("1 1 1\r\n"), Data: []byte("test")})
 	name1 := output.file.Name()
 
-	output.Write([]byte("1 2 1\ntest"))
+	output.PluginWrite(&Message{Meta: []byte("1 2 1\r\n"), Data: []byte("test")})
 	name2 := output.file.Name()
 
 	time.Sleep(time.Second)
 	output.updateName()
 
-	output.Write([]byte("1 3 1\ntest"))
+	output.PluginWrite(&Message{Meta: []byte("1 3 1\r\n"), Data: []byte("test")})
 	name3 := output.file.Name()
 
 	if name3 == name2 || name2 == name1 || name3 == name1 {
@@ -151,7 +148,7 @@ func TestFileOutputCompression(t *testing.T) {
 	}
 
 	for i := 0; i < 1000; i++ {
-		output.Write([]byte("1 1 1\r\ntest"))
+		output.PluginWrite(&Message{Meta: []byte("1 1 1\r\n"), Data: []byte("test")})
 	}
 
 	name := output.file.Name()
@@ -210,15 +207,15 @@ func TestFileOutputAppendQueueLimitOverflow(t *testing.T) {
 
 	output := NewFileOutput(name, &FileOutputConfig{Append: false, FlushInterval: time.Minute, QueueLimit: 2})
 
-	output.Write([]byte("1 1 1\r\ntest"))
+	output.PluginWrite(&Message{Meta: []byte("1 1 1\r\n"), Data: []byte("test")})
 	name1 := output.file.Name()
 
-	output.Write([]byte("1 1 1\r\ntest"))
+	output.PluginWrite(&Message{Meta: []byte("1 1 1\r\n"), Data: []byte("test")})
 	name2 := output.file.Name()
 
 	output.updateName()
 
-	output.Write([]byte("1 1 1\r\ntest"))
+	output.PluginWrite(&Message{Meta: []byte("1 1 1\r\n"), Data: []byte("test")})
 	name3 := output.file.Name()
 
 	if name2 != name1 || name1 != fmt.Sprintf("/tmp/%d_0", rnd) {
@@ -239,15 +236,15 @@ func TestFileOutputAppendQueueLimitNoOverflow(t *testing.T) {
 
 	output := NewFileOutput(name, &FileOutputConfig{Append: false, FlushInterval: time.Minute, QueueLimit: 3})
 
-	output.Write([]byte("1 1 1\r\ntest"))
+	output.PluginWrite(&Message{Meta: []byte("1 1 1\r\n"), Data: []byte("test")})
 	name1 := output.file.Name()
 
-	output.Write([]byte("1 1 1\r\ntest"))
+	output.PluginWrite(&Message{Meta: []byte("1 1 1\r\n"), Data: []byte("test")})
 	name2 := output.file.Name()
 
 	output.updateName()
 
-	output.Write([]byte("1 1 1\r\ntest"))
+	output.PluginWrite(&Message{Meta: []byte("1 1 1\r\n"), Data: []byte("test")})
 	name3 := output.file.Name()
 
 	if name2 != name1 || name1 != fmt.Sprintf("/tmp/%d_0", rnd) {
@@ -268,15 +265,15 @@ func TestFileOutputAppendQueueLimitGzips(t *testing.T) {
 
 	output := NewFileOutput(name, &FileOutputConfig{Append: false, FlushInterval: time.Minute, QueueLimit: 2})
 
-	output.Write([]byte("1 1 1\r\ntest"))
+	output.PluginWrite(&Message{Meta: []byte("1 1 1\r\n"), Data: []byte("test")})
 	name1 := output.file.Name()
 
-	output.Write([]byte("1 1 1\r\ntest"))
+	output.PluginWrite(&Message{Meta: []byte("1 1 1\r\n"), Data: []byte("test")})
 	name2 := output.file.Name()
 
 	output.updateName()
 
-	output.Write([]byte("1 1 1\r\ntest"))
+	output.PluginWrite(&Message{Meta: []byte("1 1 1\r\n"), Data: []byte("test")})
 	name3 := output.file.Name()
 
 	if name2 != name1 || name1 != fmt.Sprintf("/tmp/%d_0.gz", rnd) {
@@ -311,16 +308,16 @@ func TestFileOutputAppendSizeLimitOverflow(t *testing.T) {
 
 	output := NewFileOutput(name, &FileOutputConfig{Append: false, FlushInterval: time.Minute, SizeLimit: size.Size(2 * messageSize)})
 
-	output.Write([]byte("1 1 1\r\ntest"))
+	output.PluginWrite(&Message{Meta: []byte("1 1 1\r\n"), Data: []byte("test")})
 	name1 := output.file.Name()
 
-	output.Write([]byte("1 1 1\r\ntest"))
+	output.PluginWrite(&Message{Meta: []byte("1 1 1\r\n"), Data: []byte("test")})
 	name2 := output.file.Name()
 
 	output.flush()
 	output.updateName()
 
-	output.Write([]byte("1 1 1\r\ntest"))
+	output.PluginWrite(&Message{Meta: []byte("1 1 1\r\n"), Data: []byte("test")})
 	name3 := output.file.Name()
 
 	if name2 != name1 || name1 != fmt.Sprintf("/tmp/%d_0", rnd) {

output_http.go

@@ -5,7 +5,6 @@ import (
 	"bytes"
 	"crypto/tls"
 	"fmt"
-	"io"
 	"log"
 	"math"
 	"net/http"
@@ -59,14 +58,14 @@ type HTTPOutput struct {
 	elasticSearch *ESPlugin
 	client        *HTTPClient
 	stopWorker    chan struct{}
-	queue         chan []byte
-	responses     chan response
+	queue         chan *Message
+	responses     chan *response
 	stop          chan bool // Channel used only to indicate goroutine should shutdown
 }
 
 // NewHTTPOutput constructor for HTTPOutput
 // Initialize workers
-func NewHTTPOutput(address string, config *HTTPOutputConfig) io.Writer {
+func NewHTTPOutput(address string, config *HTTPOutputConfig) PluginReadWriter {
 	o := new(HTTPOutput)
 	var err error
 	config.url, err = url.Parse(address)
@@ -110,8 +109,10 @@ func NewHTTPOutput(address string, config *HTTPOutputConfig) io.Writer {
 		o.queueStats = NewGorStat("output_http", o.config.StatsMs)
 	}
 
-	o.queue = make(chan []byte, o.config.QueueLen)
-	o.responses = make(chan response, o.config.QueueLen)
+	o.queue = make(chan *Message, o.config.QueueLen)
+	if o.config.TrackResponses {
+		o.responses = make(chan *response, o.config.QueueLen)
+	}
 	// it should not be buffered to avoid races
 	o.stopWorker = make(chan struct{})
 
@@ -160,23 +161,22 @@ func (o *HTTPOutput) startWorker() {
 		select {
 		case <-o.stopWorker:
 			return
-		case data := <-o.queue:
-			o.sendRequest(o.client, data)
+		case msg := <-o.queue:
+			o.sendRequest(o.client, msg)
 		}
 	}
 }
 
-func (o *HTTPOutput) Write(data []byte) (n int, err error) {
-	if !isRequestPayload(data) {
-		return len(data), nil
+// PluginWrite writes message to this plugin
+func (o *HTTPOutput) PluginWrite(msg *Message) (n int, err error) {
+	if !isRequestPayload(msg.Meta) {
+		return len(msg.Data), nil
 	}
 
-	buf := make([]byte, len(data))
-	copy(buf, data)
 	select {
 	case <-o.stop:
 		return 0, ErrorStopped
-	case o.queue <- buf:
+	case o.queue <- msg:
 	}
 
 	if o.config.Stats {
@@ -189,51 +189,52 @@ func (o *HTTPOutput) Write(data []byte) (n int, err error) {
 			atomic.AddInt32(&o.activeWorkers, 1)
 		}
 	}
-	return len(data), nil
+	return len(msg.Data) + len(msg.Meta), nil
 }
 
-func (o *HTTPOutput) Read(data []byte) (int, error) {
-	var resp response
+// PluginRead reads message from this plugin
+func (o *HTTPOutput) PluginRead() (*Message, error) {
+	if !o.config.TrackResponses {
+		return nil, ErrorStopped
+	}
+	var resp *response
+	var msg Message
 	select {
 	case <-o.stop:
-		return 0, ErrorStopped
+		return nil, ErrorStopped
 	case resp = <-o.responses:
+		msg.Data = resp.payload
 	}
 
-	header := payloadHeader(ReplayedResponsePayload, resp.uuid, resp.roundTripTime, resp.startedAt)
-	n := copy(data, header)
-	if len(data) > len(header) {
-		n += copy(data[len(header):], resp.payload)
-	}
-	dis := len(header) + len(data) - n
-	if dis > 0 {
-		Debug(2, fmt.Sprintf("[OUTPUT-HTTP] %dB discarded increase copy buffer size", dis))
-	}
+	msg.Meta = payloadHeader(ReplayedResponsePayload, resp.uuid, resp.roundTripTime, resp.startedAt)
 
-	return n, nil
+	return &msg, nil
 }
 
-func (o *HTTPOutput) sendRequest(client *HTTPClient, request []byte) {
-	if !isRequestPayload(request) {
+func (o *HTTPOutput) sendRequest(client *HTTPClient, msg *Message) {
+	if !isRequestPayload(msg.Meta) {
 		return
 	}
-	uuid := payloadID(request)
-	body := payloadBody(request)
+
+	uuid := payloadID(msg.Meta)
 	start := time.Now()
-	resp, err := client.Send(body)
+	resp, err := client.Send(msg.Data)
 	stop := time.Now()
 
 	if err != nil {
 		Debug(1, fmt.Sprintf("[HTTP-OUTPUT] error when sending: %q", err))
 		return
 	}
+	if resp == nil {
+		return
+	}
 
 	if o.config.TrackResponses {
-		o.responses <- response{resp, uuid, start.UnixNano(), stop.UnixNano() - start.UnixNano()}
+		o.responses <- &response{resp, uuid, start.UnixNano(), stop.UnixNano() - start.UnixNano()}
 	}
 
 	if o.elasticSearch != nil {
-		o.elasticSearch.ResponseAnalyze(request, resp, start, stop)
+		o.elasticSearch.ResponseAnalyze(msg.Data, resp, start, stop)
 	}
 }
 
@@ -301,7 +302,14 @@ func (c *HTTPClient) Send(data []byte) ([]byte, error) {
 		req.Host = c.config.url.Host
 	}
 
-	req.URL = c.config.url
+	// fix #862
+	if c.config.url.Path == "" && c.config.url.RawQuery == "" {
+		req.URL.Scheme = c.config.url.Scheme
+		req.URL.Host = c.config.url.Host
+	} else {
+		req.URL = c.config.url
+	}
+
 	// force connection to not be closed, which can affect the global client
 	req.Close = false
 	// it's an error if this is not equal to empty string
@@ -314,5 +322,6 @@ func (c *HTTPClient) Send(data []byte) ([]byte, error) {
 	if c.config.TrackResponses {
 		return httputil.DumpResponse(resp, true)
 	}
+	_ = resp.Body.Close()
 	return nil, nil
 }

output_http_test.go

@@ -1,7 +1,6 @@
 package main
 
 import (
-	"io"
 	"io/ioutil"
 	"net/http"
 	"net/http/httptest"
@@ -12,7 +11,6 @@ import (
 
 func TestHTTPOutput(t *testing.T) {
 	wg := new(sync.WaitGroup)
-	quit := make(chan int)
 
 	input := NewTestInput()
 
@@ -38,27 +36,27 @@ func TestHTTPOutput(t *testing.T) {
 	}))
 	defer server.Close()
 
-	headers := HTTPHeaders{HTTPHeader{"User-Agent", "Gor"}}
+	headers := HTTPHeaders{httpHeader{"User-Agent", "Gor"}}
 	methods := HTTPMethods{[]byte("GET"), []byte("PUT"), []byte("POST")}
 	Settings.ModifierConfig = HTTPModifierConfig{Headers: headers, Methods: methods}
 
-	httpOutput := NewHTTPOutput(server.URL, &HTTPOutputConfig{TrackResponses: true})
-	output := NewTestOutput(func(data []byte) {
+	httpOutput := NewHTTPOutput(server.URL, &HTTPOutputConfig{TrackResponses: false})
+	output := NewTestOutput(func(*Message) {
 		wg.Done()
 	})
 
 	plugins := &InOutPlugins{
-		Inputs:  []io.Reader{input},
-		Outputs: []io.Writer{httpOutput, output},
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{httpOutput, output},
 	}
 	plugins.All = append(plugins.All, input, output, httpOutput)
 
-	emitter := NewEmitter(quit)
+	emitter := NewEmitter()
 	go emitter.Start(plugins, Settings.Middleware)
 
 	for i := 0; i < 10; i++ {
-		// 2 http-output, 2 - test output request, 2 - test output http response
-		wg.Add(6) // OPTIONS should be ignored
+		// 2 http-output, 2 - test output request
+		wg.Add(4) // OPTIONS should be ignored
 		input.EmitPOST()
 		input.EmitOPTIONS()
 		input.EmitGET()
@@ -72,7 +70,6 @@ func TestHTTPOutput(t *testing.T) {
 
 func TestHTTPOutputKeepOriginalHost(t *testing.T) {
 	wg := new(sync.WaitGroup)
-	quit := make(chan int)
 
 	input := NewTestInput()
 
@@ -85,18 +82,18 @@ func TestHTTPOutputKeepOriginalHost(t *testing.T) {
 	}))
 	defer server.Close()
 
-	headers := HTTPHeaders{HTTPHeader{"Host", "custom-host.com"}}
+	headers := HTTPHeaders{httpHeader{"Host", "custom-host.com"}}
 	Settings.ModifierConfig = HTTPModifierConfig{Headers: headers}
 
 	output := NewHTTPOutput(server.URL, &HTTPOutputConfig{OriginalHost: true, SkipVerify: true})
 
 	plugins := &InOutPlugins{
-		Inputs:  []io.Reader{input},
-		Outputs: []io.Writer{output},
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{output},
 	}
 	plugins.All = append(plugins.All, input, output)
 
-	emitter := NewEmitter(quit)
+	emitter := NewEmitter()
 	go emitter.Start(plugins, Settings.Middleware)
 
 	wg.Add(1)
@@ -109,7 +106,6 @@ func TestHTTPOutputKeepOriginalHost(t *testing.T) {
 
 func TestHTTPOutputSSL(t *testing.T) {
 	wg := new(sync.WaitGroup)
-	quit := make(chan int)
 
 	// Origing and Replay server initialization
 	server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
@@ -120,12 +116,12 @@ func TestHTTPOutputSSL(t *testing.T) {
 	output := NewHTTPOutput(server.URL, &HTTPOutputConfig{SkipVerify: true})
 
 	plugins := &InOutPlugins{
-		Inputs:  []io.Reader{input},
-		Outputs: []io.Writer{output},
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{output},
 	}
 	plugins.All = append(plugins.All, input, output)
 
-	emitter := NewEmitter(quit)
+	emitter := NewEmitter()
 	go emitter.Start(plugins, Settings.Middleware)
 
 	wg.Add(2)
@@ -139,7 +135,6 @@ func TestHTTPOutputSSL(t *testing.T) {
 
 func TestHTTPOutputSessions(t *testing.T) {
 	wg := new(sync.WaitGroup)
-	quit := make(chan int)
 
 	input := NewTestInput()
 	input.skipHeader = true
@@ -150,25 +145,27 @@ func TestHTTPOutputSessions(t *testing.T) {
 	defer server.Close()
 
 	Settings.RecognizeTCPSessions = true
+	Settings.SplitOutput = true
 	output := NewHTTPOutput(server.URL, &HTTPOutputConfig{})
 
 	plugins := &InOutPlugins{
-		Inputs:  []io.Reader{input},
-		Outputs: []io.Writer{output},
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{output},
 	}
-	emitter := NewEmitter(quit)
+	plugins.All = append(plugins.All, input, output)
+	emitter := NewEmitter()
 	go emitter.Start(plugins, Settings.Middleware)
 
 	uuid1 := []byte("1234567890123456789a0000")
 	uuid2 := []byte("1234567890123456789d0000")
 
-	for i := 0; i < 100; i++ {
+	for i := 0; i < 10; i++ {
 		wg.Add(1) // OPTIONS should be ignored
 		copy(uuid1[20:], randByte(4))
 		input.EmitBytes([]byte("1 " + string(uuid1) + " 1\n" + "GET / HTTP/1.1\r\n\r\n"))
 	}
 
-	for i := 0; i < 100; i++ {
+	for i := 0; i < 10; i++ {
 		wg.Add(1) // OPTIONS should be ignored
 		copy(uuid2[20:], randByte(4))
 		input.EmitBytes([]byte("1 " + string(uuid2) + " 1\n" + "GET / HTTP/1.1\r\n\r\n"))
@@ -179,11 +176,11 @@ func TestHTTPOutputSessions(t *testing.T) {
 	emitter.Close()
 
 	Settings.RecognizeTCPSessions = false
+	Settings.SplitOutput = false
 }
 
 func BenchmarkHTTPOutput(b *testing.B) {
 	wg := new(sync.WaitGroup)
-	quit := make(chan int)
 
 	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		wg.Done()
@@ -194,12 +191,12 @@ func BenchmarkHTTPOutput(b *testing.B) {
 	output := NewHTTPOutput(server.URL, &HTTPOutputConfig{WorkersMax: 1})
 
 	plugins := &InOutPlugins{
-		Inputs:  []io.Reader{input},
-		Outputs: []io.Writer{output},
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{output},
 	}
 	plugins.All = append(plugins.All, input, output)
 
-	emitter := NewEmitter(quit)
+	emitter := NewEmitter()
 	go emitter.Start(plugins, Settings.Middleware)
 
 	for i := 0; i < b.N; i++ {
@@ -213,7 +210,6 @@ func BenchmarkHTTPOutput(b *testing.B) {
 
 func BenchmarkHTTPOutputTLS(b *testing.B) {
 	wg := new(sync.WaitGroup)
-	quit := make(chan int)
 
 	server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		wg.Done()
@@ -224,12 +220,12 @@ func BenchmarkHTTPOutputTLS(b *testing.B) {
 	output := NewHTTPOutput(server.URL, &HTTPOutputConfig{SkipVerify: true, WorkersMax: 1})
 
 	plugins := &InOutPlugins{
-		Inputs:  []io.Reader{input},
-		Outputs: []io.Writer{output},
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{output},
 	}
 	plugins.All = append(plugins.All, input, output)
 
-	emitter := NewEmitter(quit)
+	emitter := NewEmitter()
 	go emitter.Start(plugins, Settings.Middleware)
 
 	for i := 0; i < b.N; i++ {

output_kafka.go

@@ -2,7 +2,6 @@ package main
 
 import (
 	"encoding/json"
-	"io"
 	"log"
 	"strings"
 	"time"
@@ -23,13 +22,8 @@ type KafkaOutput struct {
 // KafkaOutputFrequency in milliseconds
 const KafkaOutputFrequency = 500
 
-// NewKafkaOutput creates instance of kafka producer client.
-func NewKafkaOutput(address string, config *OutputKafkaConfig) io.Writer {
-	return NewKafkaOutputWithTLS(address, config, nil)
-}
-
-// NewKafkaOutputWithTLS creates instance of kafka producer client.
-func NewKafkaOutputWithTLS(address string, config *OutputKafkaConfig, tlsConfig *KafkaTLSConfig) io.Writer {
+// NewKafkaOutput creates instance of kafka producer client  with TLS config
+func NewKafkaOutput(address string, config *OutputKafkaConfig, tlsConfig *KafkaTLSConfig) PluginWriter {
 	c := NewKafkaConfig(tlsConfig)
 
 	var producer sarama.AsyncProducer
@@ -68,20 +62,21 @@ func (o *KafkaOutput) ErrorHandler() {
 	}
 }
 
-func (o *KafkaOutput) Write(data []byte) (n int, err error) {
+// PluginWrite writes a message to this plugin
+func (o *KafkaOutput) PluginWrite(msg *Message) (n int, err error) {
 	var message sarama.StringEncoder
 
 	if !o.config.UseJSON {
-		message = sarama.StringEncoder(byteutils.SliceToString(data))
+		message = sarama.StringEncoder(byteutils.SliceToString(msg.Meta) + byteutils.SliceToString(msg.Data))
 	} else {
-		mimeHeader := proto.ParseHeaders(data)
-		var header map[string]string
+		mimeHeader := proto.ParseHeaders(msg.Data)
+		header := make(map[string]string)
 		for k, v := range mimeHeader {
 			header[k] = strings.Join(v, ", ")
 		}
 
-		meta := payloadMeta(data)
-		req := payloadBody(data)
+		meta := payloadMeta(msg.Meta)
+		req := msg.Data
 
 		kafkaMessage := KafkaMessage{
 			ReqURL:     byteutils.SliceToString(proto.Path(req)),

output_kafka_test.go

@@ -17,16 +17,16 @@ func TestOutputKafkaRAW(t *testing.T) {
 		producer: producer,
 		Topic:    "test",
 		UseJSON:  false,
-	})
+	}, nil)
 
-	output.Write([]byte("1 2 3\nGET / HTTP1.1\r\nHeader: 1\r\n\r\n"))
+	output.PluginWrite(&Message{Meta: []byte("1 2 3\n"), Data: []byte("GET / HTTP1.1\r\nHeader: 1\r\n\r\n")})
 
 	resp := <-producer.Successes()
 
 	data, _ := resp.Value.Encode()
 
 	if string(data) != "1 2 3\nGET / HTTP1.1\r\nHeader: 1\r\n\r\n" {
-		t.Error("Message not properly encoded: ", string(data))
+		t.Errorf("Message not properly encoded: %q", data)
 	}
 }
 
@@ -40,9 +40,9 @@ func TestOutputKafkaJSON(t *testing.T) {
 		producer: producer,
 		Topic:    "test",
 		UseJSON:  true,
-	})
+	}, nil)
 
-	output.Write([]byte("1 2 3\nGET / HTTP1.1\r\nHeader: 1\r\n\r\n"))
+	output.PluginWrite(&Message{Meta: []byte("1 2 3\n"), Data: []byte("GET / HTTP1.1\r\nHeader: 1\r\n\r\n")})
 
 	resp := <-producer.Successes()
 

output_null.go

@@ -9,8 +9,9 @@ func NewNullOutput() (o *NullOutput) {
 	return new(NullOutput)
 }
 
-func (o *NullOutput) Write(data []byte) (int, error) {
-	return len(data), nil
+// PluginWrite writes message to this plugin
+func (o *NullOutput) PluginWrite(msg *Message) (int, error) {
+	return len(msg.Data) + len(msg.Meta), nil
 }
 
 func (o *NullOutput) String() string {

output_s3.go

@@ -63,12 +63,13 @@ func NewS3Output(pathTemplate string, config *FileOutputConfig) *S3Output {
 func (o *S3Output) connect() {
 	if o.session == nil {
 		o.session = session.Must(session.NewSession(awsConfig()))
-		log.Println("[S3 Output] S3 connection succesfully initialized")
+		log.Println("[S3 Output] S3 connection successfully initialized")
 	}
 }
 
-func (o *S3Output) Write(data []byte) (n int, err error) {
-	return o.buffer.Write(data)
+// PluginWrite writes message to this plugin
+func (o *S3Output) PluginWrite(msg *Message) (n int, err error) {
+	return o.buffer.PluginWrite(msg)
 }
 
 func (o *S3Output) String() string {
@@ -107,22 +108,23 @@ func (o *S3Output) onBufferUpdate(path string) {
 	idx := getFileIndex(path)
 	bucket, key := o.keyPath(idx)
 
-	file, _ := os.Open(path)
-	// reader := bufio.NewReader(file)
+	file, err := os.Open(path)
+	if err != nil {
+		Debug(0, fmt.Sprintf("[S3 Output] Failed to open file %q. err: %q", path, err))
+		return
+	}
+	defer os.Remove(path)
 
-	_, err := svc.PutObject(&s3.PutObjectInput{
+	_, err = svc.PutObject(&s3.PutObjectInput{
 		Body:   file,
 		Bucket: aws.String(bucket),
 		Key:    aws.String(key),
 	})
 	if err != nil {
-		log.Printf("[S3 Output] Failed to upload data to %s/%s, %s\n", bucket, key, err)
-		os.Remove(path)
+		Debug(0, fmt.Sprintf("[S3 Output] Failed to upload data to %q/%q, %q", bucket, key, err))
 		return
 	}
 
-	os.Remove(path)
-
 	if o.closeCh != nil {
 		o.closeCh <- struct{}{}
 	}

output_tcp.go

@@ -1,24 +1,26 @@
 package main
 
 import (
+	"context"
 	"crypto/tls"
 	"fmt"
 	"hash/fnv"
-	"io"
 	"net"
 	"time"
 )
 
 // TCPOutput used for sending raw tcp payloads
 // Currently used for internal communication between listener and replay server
-// Can be used for transfering binary payloads like protocol buffers
+// Can be used for transferring binary payloads like protocol buffers
 type TCPOutput struct {
 	address     string
 	limit       int
-	buf         []chan []byte
+	buf         []chan *Message
 	bufStats    *GorStat
 	config      *TCPOutputConfig
 	workerIndex uint32
+
+	close bool
 }
 
 // TCPOutputConfig tcp output configuration
@@ -31,7 +33,7 @@ type TCPOutputConfig struct {
 
 // NewTCPOutput constructor for TCPOutput
 // Initialize X workers which hold keep-alive connection
-func NewTCPOutput(address string, config *TCPOutputConfig) io.Writer {
+func NewTCPOutput(address string, config *TCPOutputConfig) PluginWriter {
 	o := new(TCPOutput)
 
 	o.address = address
@@ -42,9 +44,9 @@ func NewTCPOutput(address string, config *TCPOutputConfig) io.Writer {
 	}
 
 	// create X buffers and send the buffer index to the worker
-	o.buf = make([]chan []byte, o.config.Workers)
+	o.buf = make([]chan *Message, o.config.Workers)
 	for i := 0; i < o.config.Workers; i++ {
-		o.buf[i] = make(chan []byte, 100)
+		o.buf[i] = make(chan *Message, 100)
 		go o.worker(i)
 	}
 
@@ -55,6 +57,10 @@ func (o *TCPOutput) worker(bufferIndex int) {
 	retries := 0
 	conn, err := o.connect(o.address)
 	for {
+		if o.close {
+			return
+		}
+
 		if err == nil {
 			break
 		}
@@ -73,56 +79,61 @@ func (o *TCPOutput) worker(bufferIndex int) {
 	defer conn.Close()
 
 	for {
-		data := <-o.buf[bufferIndex]
-		if _, err = conn.Write(data); err == nil {
-			_, err = conn.Write([]byte(payloadSeparator))
+		msg := <-o.buf[bufferIndex]
+		if _, err = conn.Write(msg.Meta); err == nil {
+			if _, err = conn.Write(msg.Data); err == nil {
+				_, err = conn.Write(payloadSeparatorAsBytes)
+			}
 		}
 
 		if err != nil {
 			Debug(2, "INFO: TCP output connection closed, reconnecting")
-			o.buf[bufferIndex] <- data
+			o.buf[bufferIndex] <- msg
 			go o.worker(bufferIndex)
 			break
 		}
 	}
 }
 
-func (o *TCPOutput) getBufferIndex(data []byte) int {
+func (o *TCPOutput) getBufferIndex(msg *Message) int {
 	if !o.config.Sticky {
 		o.workerIndex++
 		return int(o.workerIndex) % o.config.Workers
 	}
 
 	hasher := fnv.New32a()
-	hasher.Write(payloadMeta(data)[1])
+	hasher.Write(payloadID(msg.Meta))
 	return int(hasher.Sum32()) % o.config.Workers
-
 }
 
-func (o *TCPOutput) Write(data []byte) (n int, err error) {
-	if !isOriginPayload(data) {
-		return len(data), nil
+// PluginWrite writes message to this plugin
+func (o *TCPOutput) PluginWrite(msg *Message) (n int, err error) {
+	if !isOriginPayload(msg.Meta) {
+		return len(msg.Data), nil
 	}
 
-	// We have to copy, because sending data in multiple threads
-	newBuf := make([]byte, len(data))
-	copy(newBuf, data)
-
-	bufferIndex := o.getBufferIndex(data)
-	o.buf[bufferIndex] <- newBuf
+	bufferIndex := o.getBufferIndex(msg)
+	o.buf[bufferIndex] <- msg
 
 	if Settings.OutputTCPStats {
 		o.bufStats.Write(len(o.buf[bufferIndex]))
 	}
 
-	return len(data), nil
+	return len(msg.Data) + len(msg.Meta), nil
 }
 
 func (o *TCPOutput) connect(address string) (conn net.Conn, err error) {
 	if o.config.Secure {
-		conn, err = tls.Dial("tcp", address, &tls.Config{InsecureSkipVerify: o.config.SkipVerify})
+		var d tls.Dialer
+		d.Config = &tls.Config{InsecureSkipVerify: o.config.SkipVerify}
+		ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+		defer cancel()
+		conn, err = d.DialContext(ctx, "tcp", address)
 	} else {
-		conn, err = net.Dial("tcp", address)
+		var d net.Dialer
+		ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+		defer cancel()
+		conn, err = d.DialContext(ctx, "tcp", address)
 	}
 
 	return
@@ -131,3 +142,7 @@ func (o *TCPOutput) connect(address string) (conn net.Conn, err error) {
 func (o *TCPOutput) String() string {
 	return fmt.Sprintf("TCP output %s, limit: %d", o.address, o.limit)
 }
+
+func (o *TCPOutput) Close() {
+	o.close = true
+}

output_tcp_test.go

@@ -2,7 +2,6 @@ package main
 
 import (
 	"bufio"
-	"io"
 	"log"
 	"net"
 	"sync"
@@ -11,9 +10,7 @@ import (
 )
 
 func TestTCPOutput(t *testing.T) {
-	Settings.Verbose = 2
 	wg := new(sync.WaitGroup)
-	quit := make(chan int)
 
 	listener := startTCP(func(data []byte) {
 		wg.Done()
@@ -22,12 +19,11 @@ func TestTCPOutput(t *testing.T) {
 	output := NewTCPOutput(listener.Addr().String(), &TCPOutputConfig{Workers: 10})
 
 	plugins := &InOutPlugins{
-		Inputs:  []io.Reader{input},
-		Outputs: []io.Writer{output},
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{output},
 	}
-	plugins.All = append(plugins.All, input, output)
 
-	emitter := NewEmitter(quit)
+	emitter := NewEmitter()
 	go emitter.Start(plugins, Settings.Middleware)
 
 	for i := 0; i < 10; i++ {
@@ -68,7 +64,6 @@ func startTCP(cb func([]byte)) net.Listener {
 
 func BenchmarkTCPOutput(b *testing.B) {
 	wg := new(sync.WaitGroup)
-	quit := make(chan int)
 
 	listener := startTCP(func(data []byte) {
 		wg.Done()
@@ -82,12 +77,11 @@ func BenchmarkTCPOutput(b *testing.B) {
 	output := NewTCPOutput(listener.Addr().String(), &TCPOutputConfig{Workers: 10})
 
 	plugins := &InOutPlugins{
-		Inputs:  []io.Reader{input},
-		Outputs: []io.Writer{output},
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{output},
 	}
-	plugins.All = append(plugins.All, input, output)
 
-	emitter := NewEmitter(quit)
+	emitter := NewEmitter()
 	// avoid counting above initialization
 	b.ResetTimer()
 	go emitter.Start(plugins, Settings.Middleware)
@@ -109,7 +103,7 @@ func TestStickyDisable(t *testing.T) {
 
 func TestBufferDistribution(t *testing.T) {
 	numberOfWorkers := 10
-	numberOfMessages := 1000000
+	numberOfMessages := 10000
 	percentDistributionErrorRange := 20
 
 	buffer := make([]int, numberOfWorkers)
@@ -131,8 +125,9 @@ func TestBufferDistribution(t *testing.T) {
 	}
 }
 
-func getTestBytes() []byte {
-	reqh := payloadHeader(RequestPayload, uuid(), time.Now().UnixNano(), -1)
-	reqb := append(reqh, []byte("GET / HTTP/1.1\r\nHost: www.w3.org\r\nUser-Agent: Go 1.1 package http\r\nAccept-Encoding: gzip\r\n\r\n")...)
-	return reqb
+func getTestBytes() *Message {
+	return &Message{
+		Meta: payloadHeader(RequestPayload, uuid(), time.Now().UnixNano(), -1),
+		Data: []byte("GET / HTTP/1.1\r\nHost: www.w3.org\r\nUser-Agent: Go 1.1 package http\r\nAccept-Encoding: gzip\r\n\r\n"),
+	}
 }

plugins.go

@@ -1,15 +1,36 @@
 package main
 
 import (
-	"io"
 	"reflect"
 	"strings"
 )
 
+// Message represents data across plugins
+type Message struct {
+	Meta []byte // metadata
+	Data []byte // actual data
+}
+
+// PluginReader is an interface for input plugins
+type PluginReader interface {
+	PluginRead() (msg *Message, err error)
+}
+
+// PluginWriter is an interface for output plugins
+type PluginWriter interface {
+	PluginWrite(msg *Message) (n int, err error)
+}
+
+// PluginReadWriter is an interface for plugins that support reading and writing
+type PluginReadWriter interface {
+	PluginReader
+	PluginWriter
+}
+
 // InOutPlugins struct for holding references to plugins
 type InOutPlugins struct {
-	Inputs  []io.Reader
-	Outputs []io.Writer
+	Inputs  []PluginReader
+	Outputs []PluginWriter
 	All     []interface{}
 }
 
@@ -48,26 +69,19 @@ func (plugins *InOutPlugins) registerPlugin(constructor interface{}, options ...
 
 	// Calling our constructor with list of given options
 	plugin := vc.Call(vo)[0].Interface()
-	pluginWrapper := plugin
 
 	if limit != "" {
-		pluginWrapper = NewLimiter(plugin, limit)
-	} else {
-		pluginWrapper = plugin
+		plugin = NewLimiter(plugin, limit)
 	}
 
-	_, isR := plugin.(io.Reader)
-	_, isW := plugin.(io.Writer)
-
 	// Some of the output can be Readers as well because return responses
-	if isR && !isW {
-		plugins.Inputs = append(plugins.Inputs, pluginWrapper.(io.Reader))
+	if r, ok := plugin.(PluginReader); ok {
+		plugins.Inputs = append(plugins.Inputs, r)
 	}
 
-	if isW {
-		plugins.Outputs = append(plugins.Outputs, pluginWrapper.(io.Writer))
+	if w, ok := plugin.(PluginWriter); ok {
+		plugins.Outputs = append(plugins.Outputs, w)
 	}
-
 	plugins.All = append(plugins.All, plugin)
 }
 
@@ -104,7 +118,7 @@ func NewPlugins() *InOutPlugins {
 	}
 
 	for _, options := range Settings.InputFile {
-		plugins.registerPlugin(NewFileInput, options, Settings.InputFileLoop)
+		plugins.registerPlugin(NewFileInput, options, Settings.InputFileLoop, Settings.InputFileReadDepth, Settings.InputFileMaxWait, Settings.InputFileDryRun)
 	}
 
 	for _, path := range Settings.OutputFile {

plugins_test.go

@@ -12,8 +12,8 @@ func TestPluginsRegistration(t *testing.T) {
 
 	plugins := NewPlugins()
 
-	if len(plugins.Inputs) != 2 {
-		t.Errorf("Should be 2 inputs %d", len(plugins.Inputs))
+	if len(plugins.Inputs) != 3 {
+		t.Errorf("Should be 3 inputs got %d", len(plugins.Inputs))
 	}
 
 	if _, ok := plugins.Inputs[0].(*DummyInput); !ok {

proto/proto.go

@@ -19,6 +19,7 @@ package proto
 import (
 	"bufio"
 	"bytes"
+	_ "fmt"
 	"net/http"
 	"net/textproto"
 	"strings"
@@ -58,10 +59,15 @@ func MIMEHeadersStartPos(payload []byte) int {
 // If not found, value will be blank, and headerStart will be -1
 // Do not support multi-line headers.
 func header(payload []byte, name []byte) (value []byte, headerStart, headerEnd, valueStart, valueEnd int) {
-	headerStart = MIMEHeadersStartPos(payload)
-	if headerStart < 0 {
-		return
+	if HasTitle(payload) {
+		headerStart = MIMEHeadersStartPos(payload)
+		if headerStart < 0 {
+			return
+		}
+	} else {
+		headerStart = 0
 	}
+
 	var colonIndex int
 	for headerStart < len(payload) {
 		headerEnd = bytes.IndexByte(payload[headerStart:], '\n')
@@ -71,9 +77,12 @@ func header(payload []byte, name []byte) (value []byte, headerStart, headerEnd,
 		headerEnd += headerStart
 		colonIndex = bytes.IndexByte(payload[headerStart:headerEnd], ':')
 		if colonIndex == -1 {
-			break
+			// Malformed header, skip, most likely packet with partial headers
+			headerStart = headerEnd + 1
+			continue
 		}
 		colonIndex += headerStart
+
 		if bytes.EqualFold(payload[headerStart:colonIndex], name) {
 			valueStart = colonIndex + 1
 			valueEnd = headerEnd - 2
@@ -125,6 +134,11 @@ func ParseHeaders(p []byte) textproto.MIMEHeader {
 	if headerEnd > 1 {
 		p = p[:headerEnd]
 	}
+	return GetHeaders(p)
+}
+
+// GetHeaders returns mime headers from the payload
+func GetHeaders(p []byte) textproto.MIMEHeader {
 	reader := textproto.NewReader(bufio.NewReader(bytes.NewReader(p)))
 	mime, err := reader.ReadMIMEHeader()
 	if err != nil {
@@ -182,15 +196,15 @@ func DeleteHeader(payload, name []byte) []byte {
 // Body returns request/response body
 func Body(payload []byte) []byte {
 	pos := MIMEHeadersEndPos(payload)
-	if pos == -1 {
+	if pos == -1 || len(payload) <= pos {
 		return nil
 	}
 	return payload[pos:]
 }
 
-// Path takes payload and retuns request path: Split(firstLine, ' ')[1]
+// Path takes payload and returns request path: Split(firstLine, ' ')[1]
 func Path(payload []byte) []byte {
-	if !HasTitle(payload) {
+	if !HasRequestTitle(payload) {
 		return nil
 	}
 	start := bytes.IndexByte(payload, ' ') + 1
@@ -304,7 +318,12 @@ func Method(payload []byte) []byte {
 // Status returns response status.
 // It happens to be in same position as request payload path
 func Status(payload []byte) []byte {
-	return Path(payload)
+	if !HasResponseTitle(payload) {
+		return nil
+	}
+	start := bytes.IndexByte(payload, ' ') + 1
+	// status code are in range 100-600
+	return payload[start : start+3]
 }
 
 // Methods holds the http methods ordered in ascending order
@@ -317,8 +336,8 @@ var Methods = [...]string{
 const (
 	//MinRequestCount GET / HTTP/1.1\r\n
 	MinRequestCount = 16
-	// MinResponseCount HTTP/1.1 200 OK\r\n
-	MinResponseCount = 17
+	// MinResponseCount HTTP/1.1 200\r\n
+	MinResponseCount = 14
 	// VersionLen HTTP/1.1
 	VersionLen = 8
 )
@@ -341,17 +360,15 @@ func HasResponseTitle(payload []byte) bool {
 		return false
 	}
 	status, ok := atoI(payload[VersionLen+1:VersionLen+4], 10)
-	if !ok || s[VersionLen+4] != ' ' {
+	if !ok {
 		return false
 	}
-	statusText := http.StatusText(status)
-	if statusText == "" {
+	// only validate status codes mentioned in rfc2616.
+	if http.StatusText(status) == "" {
 		return false
 	}
-	if titleLen+len(CRLF) > len(s) {
-		return false
-	}
-	return s[VersionLen+5:titleLen] == statusText
+	// handle cases from #875
+	return payload[VersionLen+4] == ' ' || payload[VersionLen+4] == '\r'
 }
 
 // HasRequestTitle reports whether this payload has an HTTP/1 request title
@@ -390,102 +407,184 @@ func HasTitle(payload []byte) bool {
 	return HasRequestTitle(payload) || HasResponseTitle(payload)
 }
 
-// CheckChunked checks HTTP/1 chunked data integrity and return the final index
-// of chunks(index after '0\r\n\r\n') or -1 if there is missing data
-// or there is bad format
-func CheckChunked(buf []byte) (chunkEnd int) {
-	var (
-		ok     bool
-		chkLen int
-		sz     int
-		ext    int
-	)
-	for {
-		sz = bytes.IndexByte(buf[chunkEnd:], '\r')
-		if sz < 1 {
-			return -1
-		}
-		// ignoring chunks extensions https://github.com/golang/go/issues/13135
-		// but chunks extensions are no longer a thing
-		ext = bytes.IndexByte(buf[chunkEnd:chunkEnd+sz], ';')
-		if ext < 0 {
-			ext = sz
-		}
-		chkLen, ok = atoI(buf[chunkEnd:chunkEnd+ext], 16)
-		if !ok {
-			return -1
-		}
-		chunkEnd += (sz + 2)
-		if chkLen == 0 {
-			if !bytes.Equal(buf[chunkEnd:chunkEnd+2], CRLF) {
-				return -1
-			}
-			return chunkEnd + 2
-		}
-		// ideally chunck length and at least len("\r\n0\r\n\r\n")
-		if len(buf[chunkEnd:]) < chkLen+7 {
-			return -1
-		}
-		chunkEnd += chkLen
-		// chunks must end with CRLF
-		if !bytes.Equal(buf[chunkEnd:chunkEnd+2], CRLF) {
-			return -1
-		}
-		chunkEnd += 2
+// CheckChunked checks HTTP/1 chunked data integrity(https://tools.ietf.org/html/rfc7230#section-4.1)
+// and returns the length of total valid scanned chunks(including chunk size, extensions and CRLFs) and
+// full is true if all chunks was scanned.
+func CheckChunked(bufs ...[]byte) (chunkEnd int, full bool) {
+	var buf []byte
+	if len(bufs) > 0 {
+		buf = bufs[0]
 	}
+	for chunkEnd < len(buf) {
+		sz := bytes.IndexByte(buf[chunkEnd:], '\r')
+		if sz < 1 {
+			break
+		}
+		// don't parse chunk extensions https://github.com/golang/go/issues/13135.
+		// chunks extensions are no longer a thing, but we do check if the byte
+		// following the parsed hex number is ';'
+		sz += chunkEnd
+		chkLen, ok := atoI(buf[chunkEnd:sz], 16)
+		if !ok && bytes.IndexByte(buf[chunkEnd:sz], ';') < 1 {
+			break
+		}
+		sz++ // + '\n'
+		// total length = SIZE + CRLF + OCTETS + CRLF
+		allChunk := sz + chkLen + 2
+		if allChunk >= len(buf) ||
+			buf[sz]&buf[allChunk] != '\n' ||
+			buf[allChunk-1] != '\r' {
+			break
+		}
+		chunkEnd = allChunk + 1
+		if chkLen == 0 {
+			full = true
+			break
+		}
+	}
+	return
 }
 
-// HasFullPayload reports if this http has full payloads
-func HasFullPayload(payload []byte) bool {
-	body := Body(payload)
+// ProtocolStateSetter is an interface used to provide protocol state for future use
+type ProtocolStateSetter interface {
+	SetProtocolState(interface{})
+	ProtocolState() interface{}
+}
 
-	// check for chunked transfer-encoding
-	header := Header(payload, []byte("Transfer-Encoding"))
-	if bytes.Contains(header, []byte("chunked")) {
+type HTTPState struct {
+	Body           int // body index
+	HeaderStart    int
+	HeaderEnd      int
+	HeaderParsed   bool // we checked necessary headers
+	HasFullPayload bool // all chunks has been parsed
+	IsChunked      bool // Transfer-Encoding: chunked
+	BodyLen        int  // Content-Length's value
+	HasTrailer     bool // Trailer header?
+	Continue100    bool
+}
 
-		// check chunks
-		if len(body) < 1 {
-			return false
+// HasFullPayload checks if this message has full or valid payloads and returns true.
+// Message param is optional but recommended on cases where 'data' is storing
+// partial-to-full stream of bytes(packets).
+func HasFullPayload(m ProtocolStateSetter, payloads ...[]byte) bool {
+	var state *HTTPState
+	if m != nil {
+		state, _ = m.ProtocolState().(*HTTPState)
+	}
+	if state == nil {
+		state = new(HTTPState)
+		if m != nil {
+			m.SetProtocolState(state)
 		}
-		var chunkEnd int
-		if chunkEnd = CheckChunked(body); chunkEnd < 1 {
+	}
+	if state.HeaderStart < 1 {
+		for _, data := range payloads {
+			state.HeaderStart = MIMEHeadersStartPos(data)
+			if state.HeaderStart < 0 {
+				return false
+			} else {
+				break
+			}
+		}
+	}
+
+	if state.Body < 1 || state.HeaderEnd < 1 {
+		var pos int
+		for _, data := range payloads {
+			endPos := MIMEHeadersEndPos(data)
+			if endPos < 0 {
+				pos += len(data)
+			} else {
+				pos += endPos
+				state.HeaderEnd = pos
+			}
+
+			if endPos > 0 {
+				state.Body = pos
+				break
+			}
+		}
+	}
+
+	if state.HeaderEnd < 1 {
+		return false
+	}
+
+	if !state.HeaderParsed {
+		var pos int
+		for _, data := range payloads {
+			chunked := Header(data, []byte("Transfer-Encoding"))
+
+			if len(chunked) > 0 && bytes.Index(data, []byte("chunked")) > 0 {
+				state.IsChunked = true
+				// trailers are generally not allowed in non-chunks body
+				state.HasTrailer = len(Header(data, []byte("Trailer"))) > 0
+			} else {
+				contentLen := Header(data, []byte("Content-Length"))
+				state.BodyLen, _ = atoI(contentLen, 10)
+			}
+
+			pos += len(data)
+
+			if string(Header(data, []byte("Expect"))) == "100-continue" {
+				state.Continue100 = true
+			}
+
+			if state.BodyLen > 0 || pos >= state.Body {
+				state.HeaderParsed = true
+				break
+			}
+		}
+	}
+
+	bodyLen := 0
+	for _, data := range payloads {
+		bodyLen += len(data)
+	}
+	bodyLen -= state.Body
+
+	if state.IsChunked {
+		// check chunks
+		if bodyLen < 1 {
 			return false
 		}
 
 		// check trailer headers
-		if len(Header(payload, []byte("Trailer"))) < 1 {
-			return true
+		if state.HasTrailer {
+			if bytes.HasSuffix(payloads[len(payloads)-1], []byte("\r\n\r\n")) {
+				return true
+			}
+		} else {
+			if bytes.HasSuffix(payloads[len(payloads)-1], []byte("0\r\n\r\n")) {
+				state.HasFullPayload = true
+				return true
+			}
 		}
-		// trailer headers(whether chunked or plain) should end with empty line
-		return len(body) > chunkEnd && MIMEHeadersEndPos(body[chunkEnd:]) != -1
+
+		return false
 	}
 
 	// check for content-length header
-	// trailers are generally not allowed in non-chunks body
-	header = Header(payload, []byte("Content-Length"))
-	if len(header) > 1 {
-		num, ok := atoI(header, 10)
-		return ok && num == len(body)
-	}
-
-	// for empty body, check for emptyline
-	return MIMEHeadersEndPos(payload) != -1
+	return state.BodyLen == bodyLen
 }
 
 // this works with positive integers
 func atoI(s []byte, base int) (num int, ok bool) {
 	var v int
+	ok = true
 	for i := 0; i < len(s); i++ {
 		if s[i] > 127 {
-			return 0, false
+			ok = false
+			break
 		}
 		v = int(hexTable[s[i]])
 		if v >= base || (v == 0 && s[i] != '0') {
-			return 0, false
+			ok = false
+			break
 		}
 		num = (num * base) + v
 	}
-	return num, true
+	return
 }
 
 var hexTable = [128]byte{

proto/proto_test.go

@@ -56,6 +56,12 @@ func TestHeader(t *testing.T) {
 	if val = Header(payload, []byte("host")); !bytes.Equal(val, []byte("www.w3.org")) {
 		t.Error("Should find lower case 1 word header")
 	}
+
+	payload = []byte("GT\r\nContent-Length: 10\r\n\r\n")
+
+	if val = Header(payload, []byte("Content-Length")); !bytes.Equal(val, []byte("10")) {
+		t.Error("Should find in partial payload")
+	}
 }
 
 func TestMIMEHeadersEndPos(t *testing.T) {
@@ -136,6 +142,24 @@ func TestParseHeaders(t *testing.T) {
 	if !reflect.DeepEqual(headers, expected) {
 		t.Error("Headers do not properly parsed", headers)
 	}
+
+	// Response with Reason phrase
+	payload = [][]byte{[]byte("HTTP/1.1 200 OK\r\nContent-Length: 7\r\nHost: www.w3.org\r\nUser-Agent:Chrome\r\n\r\nbody")}
+
+	headers = ParseHeaders(bytes.Join(payload, nil))
+
+	if !reflect.DeepEqual(headers, expected) {
+		t.Error("Headers do not properly parsed", headers)
+	}
+
+	// Response without Reason phrase
+	payload = [][]byte{[]byte("HTTP/1.1 200\r\nContent-Length: 7\r\nHost: www.w3.org\r\nUser-Agent:Chrome\r\n\r\nbody")}
+
+	headers = ParseHeaders(bytes.Join(payload, nil))
+
+	if !reflect.DeepEqual(headers, expected) {
+		t.Error("Headers do not properly parsed", headers)
+	}
 }
 
 // See https://github.com/dvyukov/go-fuzz and fuzz.go
@@ -219,6 +243,25 @@ func TestPath(t *testing.T) {
 	}
 }
 
+func TestStatus(t *testing.T) {
+	var status, payload []byte
+
+	payload = []byte("HTTP/1.1 200 OK\r\n")
+	if status = Status(payload); !bytes.Equal(status, []byte("200")) {
+		t.Error("Should find status 200 but:", string(status))
+	}
+
+	payload = []byte("HTTP/1.1 200\r\n")
+	if status = Status(payload); !bytes.Equal(status, []byte("200")) {
+		t.Error("1Should find status 200 but:", string(status))
+	}
+
+	payload = []byte("HTTP/1.1 404 Not Found\r\n")
+	if status = Status(payload); !bytes.Equal(status, []byte("404")) {
+		t.Error("2Should find status 404 but:", string(status))
+	}
+}
+
 func TestSetPath(t *testing.T) {
 	var payload, payloadAfter []byte
 
@@ -318,9 +361,10 @@ func TestHasResponseTitle(t *testing.T) {
 		"HTTP/1.1 100 Continue\r\n": true,
 		"HTTP/1.1  \r\n":            false,
 		"HTTP/4.0 100Continue\r\n":  false,
+		"HTTP/1.0 100Continue\r\n":  false,
 		"HTTP/1.0 10r Continue\r\n": false,
-		"HTTP/1.1 200\r\n":          false,
-		"HTTP/1.1 200\r\nServer: Tengine\r\nContent-Length: 0\r\nConnection: close\r\n\r\n": false,
+		"HTTP/1.1 200\r\n":          true,
+		"HTTP/1.1 200\r\nServer: Tengine\r\nContent-Length: 0\r\nConnection: close\r\n\r\n": true,
 	}
 	for k, v := range m {
 		if HasResponseTitle([]byte(k)) != v {
@@ -350,63 +394,61 @@ func TestHasRequestTitle(t *testing.T) {
 
 func TestCheckChunks(t *testing.T) {
 	var m = "4\r\nWiki\r\n5\r\npedia\r\nE\r\n in\r\n\r\nchunks.\r\n0\r\n\r\n"
-	chunkEnd := CheckChunked([]byte(m))
-	expected := bytes.Index([]byte(m), []byte("0\r\n")) + 5
+	chunkEnd, _ := CheckChunked([]byte(m))
+	expected := len(m)
 	if chunkEnd != expected {
 		t.Errorf("expected %d to equal %d", chunkEnd, expected)
 	}
 
 	m = "7\r\nMozia\r\n9\r\nDeveloper\r\n7\r\nNetwork\r\n0\r\n\r\n"
-	chunkEnd = CheckChunked([]byte(m))
-	if chunkEnd != -1 {
-		t.Errorf("expected %d to equal %d", chunkEnd, -1)
+	chunkEnd, _ = CheckChunked([]byte(m))
+	if chunkEnd != 0 {
+		t.Errorf("expected %d to equal %d", chunkEnd, 0)
 	}
 
 	// with trailers
 	m = "4\r\nWiki\r\n5\r\npedia\r\nE\r\n in\r\n\r\nchunks.\r\n0\r\n\r\nEXpires"
-	chunkEnd = CheckChunked([]byte(m))
-	expected = bytes.Index([]byte(m), []byte("0\r\n")) + 5
+	chunkEnd, _ = CheckChunked([]byte(m))
+	expected = len(m) - 7
 	if chunkEnd != expected {
 		t.Errorf("expected %d to equal %d", chunkEnd, expected)
 	}
 
-	// last chunk inside the the body
+	// last chunk inside the body
 	// with trailers
 	m = "4\r\nWiki\r\n5\r\npedia\r\nE\r\n in\r\n\r\nchunks.\r\n3\r\n0\r\n\r\n0\r\n\r\nEXpires"
-	chunkEnd = CheckChunked([]byte(m))
-	expected = bytes.Index([]byte(m), []byte("0\r\n")) + 10
+	chunkEnd, _ = CheckChunked([]byte(m))
+	expected = len(m) - 7
 	if chunkEnd != expected {
 		t.Errorf("expected %d to equal %d", chunkEnd, expected)
 	}
 
 	// checks with chucks-extensions
 	m = "4\r\nWiki\r\n5\r\npedia\r\nE; name='quoted string'\r\n in\r\n\r\nchunks.\r\n3\r\n0\r\n\r\n0\r\n\r\nEXpires"
-	chunkEnd = CheckChunked([]byte(m))
-	expected = bytes.Index([]byte(m), []byte("0\r\n")) + 10
+	chunkEnd, _ = CheckChunked([]byte(m))
+	expected = len(m) - 7
 	if chunkEnd != expected {
 		t.Errorf("expected %d to equal %d", chunkEnd, expected)
 	}
 }
 
 func TestHasFullPayload(t *testing.T) {
-	var m = "HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nTransfer-Encoding: chunked\r\n\r\n7\r\nMozilla\r\n9\r\nDeveloper\r\n7\r\nNetwork\r\n0\r\n\r\n"
-	got := HasFullPayload([]byte(m))
-	expected := true
-	if got != expected {
-		t.Errorf("expected %v to equal %v", got, expected)
-	}
+	var m string
+	var got, expected bool
 
-	// check with invalid chunk format
-	m = "HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nTransfer-Encoding: chunked\r\n\r\n7\r\nMozia\r\n9\r\nDeveloper\r\n7\r\nNetwork\r\n0\r\n\r\n"
-	got = HasFullPayload([]byte(m))
-	expected = false
+	got = HasFullPayload(nil,
+		[]byte("HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\n"),
+		[]byte("Transfer-Encoding: chunked\r\n\r\n"),
+		[]byte("7\r\nMozilla\r\n9\r\nDeveloper\r\n"),
+		[]byte("7\r\nNetwork\r\n0\r\n\r\n"))
+	expected = true
 	if got != expected {
 		t.Errorf("expected %v to equal %v", got, expected)
 	}
 
 	// check chunks with trailers
 	m = "HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nTransfer-Encoding: chunked\r\nTrailer: Expires\r\n\r\n7\r\nMozilla\r\n9\r\nDeveloper\r\n7\r\nNetwork\r\n0\r\n\r\nExpires: Wed, 21 Oct 2015 07:28:00 GMT\r\n\r\n"
-	got = HasFullPayload([]byte(m))
+	got = HasFullPayload(nil, []byte(m))
 	expected = true
 	if got != expected {
 		t.Errorf("expected %v to equal %v", got, expected)
@@ -414,7 +456,7 @@ func TestHasFullPayload(t *testing.T) {
 
 	// check with missing trailers
 	m = "HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nTransfer-Encoding: chunked\r\nTrailer: Expires\r\n\r\n7\r\nMozilla\r\n9\r\nDeveloper\r\n7\r\nNetwork\r\n0\r\n\r\nExpires: Wed, 21 Oct 2015 07:28:00"
-	got = HasFullPayload([]byte(m))
+	got = HasFullPayload(nil, []byte(m))
 	expected = false
 	if got != expected {
 		t.Errorf("expected %v to equal %v", got, expected)
@@ -422,7 +464,7 @@ func TestHasFullPayload(t *testing.T) {
 
 	// check with content-length
 	m = "HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nContent-Length: 23\r\n\r\nMozillaDeveloperNetwork"
-	got = HasFullPayload([]byte(m))
+	got = HasFullPayload(nil, []byte(m))
 	expected = true
 	if got != expected {
 		t.Errorf("expected %v to equal %v", got, expected)
@@ -430,7 +472,7 @@ func TestHasFullPayload(t *testing.T) {
 
 	// check missing total length
 	m = "HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nContent-Length: 23\r\n\r\nMozillaDeveloperNet"
-	got = HasFullPayload([]byte(m))
+	got = HasFullPayload(nil, []byte(m))
 	expected = false
 	if got != expected {
 		t.Errorf("expected %v to equal %v", got, expected)
@@ -438,7 +480,7 @@ func TestHasFullPayload(t *testing.T) {
 
 	// check with no body
 	m = "HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\n\r\n"
-	got = HasFullPayload([]byte(m))
+	got = HasFullPayload(nil, []byte(m))
 	expected = true
 	if got != expected {
 		t.Errorf("expected %v to equal %v", got, expected)
@@ -446,21 +488,10 @@ func TestHasFullPayload(t *testing.T) {
 }
 
 func BenchmarkHasFullPayload(b *testing.B) {
-	var buf bytes.Buffer
-	buf.Write([]byte("HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nTransfer-Encoding: chunked\r\n\r\n"))
-	var chunk = []byte("1e\r\n111111111111111111111111111111\r\n")
-	for i := 0; i < 5000; i++ {
-		buf.Write(chunk)
-	}
-	buf.Write([]byte("0\r\n\r\n"))
-	data := buf.Bytes()
-	var ok bool
-	b.ResetTimer() // ignores the upper initialization
-	b.ReportMetric(float64(5000), "chunks/op")
+	data := []byte("HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nTransfer-Encoding: chunked\r\n\r\n1e\r\n111111111111111111111111111111\r\n0\r\n\r\n")
 	for i := 0; i < b.N; i++ {
-		ok = HasFullPayload(data)
-	}
-	if !ok {
-		b.Fail()
+		if !HasFullPayload(nil, data) {
+			b.Fail()
+		}
 	}
 }

protocol.go

@@ -66,6 +66,16 @@ func payloadMeta(payload []byte) [][]byte {
 	return bytes.Split(payload[:headerSize], []byte{' '})
 }
 
+func payloadMetaWithBody(payload []byte) (meta, body []byte) {
+	if i := bytes.IndexByte(payload, '\n'); i > 0 && len(payload) > i+1 {
+		meta = payload[:i+1]
+		body = payload[i+1:]
+		return
+	}
+	// we assume the message did not have meta data
+	return nil, payload
+}
+
 func payloadID(payload []byte) (id []byte) {
 	meta := payloadMeta(payload)
 

ring/ring.go

@@ -0,0 +1,205 @@
+package ring
+
+import (
+	"errors"
+	"runtime"
+	"sync/atomic"
+	"time"
+)
+
+var (
+	// ErrDisposed is returned when an operation is performed on a disposed
+	// queue.
+	ErrDisposed = errors.New(`queue: disposed`)
+
+	// ErrTimeout is returned when an applicable queue operation times out.
+	ErrTimeout = errors.New(`queue: poll timed out`)
+
+	// ErrEmptyQueue is returned when an non-applicable queue operation was called
+	// due to the queue's empty item state
+	ErrEmptyQueue = errors.New(`queue: empty queue`)
+)
+
+// roundUp takes a uint64 greater than 0 and rounds it up to the next
+// power of 2.
+func roundUp(v uint64) uint64 {
+	v--
+	v |= v >> 1
+	v |= v >> 2
+	v |= v >> 4
+	v |= v >> 8
+	v |= v >> 16
+	v |= v >> 32
+	v++
+	return v
+}
+
+type node struct {
+	position uint64
+	data     interface{}
+}
+
+type nodes []node
+
+// RingBuffer is a MPMC buffer that achieves threadsafety with CAS operations
+// only.  A put on full or get on empty call will block until an item
+// is put or retrieved.  Calling Dispose on the RingBuffer will unblock
+// any blocked threads with an error.  This buffer is similar to the buffer
+// described here: http://www.1024cores.net/home/lock-free-algorithms/queues/bounded-mpmc-queue
+// with some minor additions.
+type RingBuffer struct {
+	_padding0      [8]uint64
+	queue          uint64
+	_padding1      [8]uint64
+	dequeue        uint64
+	_padding2      [8]uint64
+	mask, disposed uint64
+	_padding3      [8]uint64
+	nodes          nodes
+}
+
+func (rb *RingBuffer) init(size uint64) {
+	size = roundUp(size)
+	rb.nodes = make(nodes, size)
+	for i := uint64(0); i < size; i++ {
+		rb.nodes[i] = node{position: i}
+	}
+	rb.mask = size - 1 // so we don't have to do this with every put/get operation
+}
+
+// Put adds the provided item to the queue.  If the queue is full, this
+// call will block until an item is added to the queue or Dispose is called
+// on the queue.  An error will be returned if the queue is disposed.
+func (rb *RingBuffer) Put(item interface{}) error {
+	_, err := rb.put(item, false)
+	return err
+}
+
+// Offer adds the provided item to the queue if there is space.  If the queue
+// is full, this call will return false.  An error will be returned if the
+// queue is disposed.
+func (rb *RingBuffer) Offer(item interface{}) (bool, error) {
+	return rb.put(item, true)
+}
+
+func (rb *RingBuffer) put(item interface{}, offer bool) (bool, error) {
+	var n *node
+	pos := atomic.LoadUint64(&rb.queue)
+L:
+	for {
+		if atomic.LoadUint64(&rb.disposed) == 1 {
+			return false, ErrDisposed
+		}
+
+		n = &rb.nodes[pos&rb.mask]
+		seq := atomic.LoadUint64(&n.position)
+		switch dif := seq - pos; {
+		case dif == 0:
+			if atomic.CompareAndSwapUint64(&rb.queue, pos, pos+1) {
+				break L
+			}
+		case dif < 0:
+			panic(`Ring buffer in a compromised state during a put operation.`)
+		default:
+			pos = atomic.LoadUint64(&rb.queue)
+		}
+
+		if offer {
+			return false, nil
+		}
+
+		runtime.Gosched() // free up the cpu before the next iteration
+	}
+
+	n.data = item
+	atomic.StoreUint64(&n.position, pos+1)
+	return true, nil
+}
+
+// Get will return the next item in the queue.  This call will block
+// if the queue is empty.  This call will unblock when an item is added
+// to the queue or Dispose is called on the queue.  An error will be returned
+// if the queue is disposed.
+func (rb *RingBuffer) Get() (interface{}, error) {
+	return rb.Poll(0)
+}
+
+// Poll will return the next item in the queue.  This call will block
+// if the queue is empty.  This call will unblock when an item is added
+// to the queue, Dispose is called on the queue, or the timeout is reached. An
+// error will be returned if the queue is disposed or a timeout occurs. A
+// non-positive timeout will block indefinitely.
+func (rb *RingBuffer) Poll(timeout time.Duration) (interface{}, error) {
+	var (
+		n     *node
+		pos   = atomic.LoadUint64(&rb.dequeue)
+		start time.Time
+	)
+	if timeout > 0 {
+		start = time.Now()
+	}
+L:
+	for {
+		if atomic.LoadUint64(&rb.disposed) == 1 {
+			return nil, ErrDisposed
+		}
+
+		n = &rb.nodes[pos&rb.mask]
+		seq := atomic.LoadUint64(&n.position)
+		switch dif := seq - (pos + 1); {
+		case dif == 0:
+			if atomic.CompareAndSwapUint64(&rb.dequeue, pos, pos+1) {
+				break L
+			}
+		case dif < 0:
+			panic(`Ring buffer in compromised state during a get operation.`)
+		default:
+			pos = atomic.LoadUint64(&rb.dequeue)
+		}
+
+		if timeout > 0 && time.Since(start) >= timeout {
+			return nil, ErrTimeout
+		}
+
+		if timeout < 0 {
+			return nil, ErrTimeout
+		}
+
+		runtime.Gosched() // free up the cpu before the next iteration
+	}
+	data := n.data
+	n.data = nil
+	atomic.StoreUint64(&n.position, pos+rb.mask+1)
+	return data, nil
+}
+
+// Len returns the number of items in the queue.
+func (rb *RingBuffer) Len() uint64 {
+	return atomic.LoadUint64(&rb.queue) - atomic.LoadUint64(&rb.dequeue)
+}
+
+// Cap returns the capacity of this ring buffer.
+func (rb *RingBuffer) Cap() uint64 {
+	return uint64(len(rb.nodes))
+}
+
+// Dispose will dispose of this queue and free any blocked threads
+// in the Put and/or Get methods.  Calling those methods on a disposed
+// queue will return an error.
+func (rb *RingBuffer) Dispose() {
+	atomic.CompareAndSwapUint64(&rb.disposed, 0, 1)
+}
+
+// IsDisposed will return a bool indicating if this queue has been
+// disposed.
+func (rb *RingBuffer) IsDisposed() bool {
+	return atomic.LoadUint64(&rb.disposed) == 1
+}
+
+// NewRingBuffer will allocate, initialize, and return a ring buffer
+// with the specified size.
+func NewRingBuffer(size uint64) *RingBuffer {
+	rb := &RingBuffer{}
+	rb.init(size)
+	return rb
+}

s3_reader.go

@@ -60,7 +60,7 @@ func NewS3ReadCloser(path string) *S3ReadCloser {
 	bucket, key := parseS3Url(path)
 	sess := session.Must(session.NewSession(awsConfig()))
 
-	log.Println("[S3 Input] S3 connection succesfully initialized", path)
+	log.Println("[S3 Input] S3 connection successfully initialized", path)
 
 	return &S3ReadCloser{
 		bucket: bucket,

s3_test.go

@@ -127,7 +127,7 @@ func TestInputFileFromS3(t *testing.T) {
 		<-output.closeCh
 	}
 
-	input := NewFileInput(fmt.Sprintf("s3://test-gor-eu/%d", rnd), false)
+	input := NewFileInput(fmt.Sprintf("s3://test-gor-eu/%d", rnd), false, 100, 0, false)
 
 	buf := make([]byte, 1000)
 	for i := 0; i <= 19999; i++ {

settings.go

@@ -45,10 +45,13 @@ type AppSettings struct {
 	OutputTCPConfig TCPOutputConfig
 	OutputTCPStats  bool `json:"output-tcp-stats"`
 
-	InputFile        MultiOption `json:"input-file"`
-	InputFileLoop    bool        `json:"input-file-loop"`
-	OutputFile       MultiOption `json:"output-file"`
-	OutputFileConfig FileOutputConfig
+	InputFile          MultiOption   `json:"input-file"`
+	InputFileLoop      bool          `json:"input-file-loop"`
+	InputFileReadDepth int           `json:"input-file-read-depth"`
+	InputFileDryRun    bool          `json:"input-file-dry-run"`
+	InputFileMaxWait   time.Duration `json:"input-file-max-wait"`
+	OutputFile         MultiOption   `json:"output-file"`
+	OutputFileConfig   FileOutputConfig
 
 	InputRAW MultiOption `json:"input_raw"`
 	RAWInputConfig
@@ -93,14 +96,10 @@ func init() {
 	}
 
 	flag.BoolVar(&Settings.SplitOutput, "split-output", false, "By default each output gets same traffic. If set to `true` it splits traffic equally among all outputs.")
-
 	flag.BoolVar(&Settings.RecognizeTCPSessions, "recognize-tcp-sessions", false, "[PRO] If turned on http output will create separate worker for each TCP session. Splitting output will session based as well.")
 
 	flag.Var(&Settings.InputDummy, "input-dummy", "Used for testing outputs. Emits 'Get /' request every 1s")
-	flag.Var(&Settings.OutputDummy, "output-dummy", "DEPRECATED: use --output-stdout instead")
-
 	flag.BoolVar(&Settings.OutputStdout, "output-stdout", false, "Used for testing inputs. Just prints to console data coming from inputs.")
-
 	flag.BoolVar(&Settings.OutputNull, "output-null", false, "Used for testing inputs. Drops all requests.")
 
 	flag.Var(&Settings.InputTCP, "input-tcp", "Used for internal communication between Gor instances. Example: \n\t# Receive requests from other Gor instances on 28020 port, and redirect output to staging\n\tgor --input-tcp :28020 --output-http staging.com")
@@ -117,6 +116,9 @@ func init() {
 
 	flag.Var(&Settings.InputFile, "input-file", "Read requests from file: \n\tgor --input-file ./requests.gor --output-http staging.com")
 	flag.BoolVar(&Settings.InputFileLoop, "input-file-loop", false, "Loop input files, useful for performance testing.")
+	flag.IntVar(&Settings.InputFileReadDepth, "input-file-read-depth", 100, "GoReplay tries to read and cache multiple records, in advance. In parallel it also perform sorting of requests, if they came out of order. Since it needs hold this buffer in memory, bigger values can cause worse performance")
+	flag.BoolVar(&Settings.InputFileDryRun, "input-file-dry-run", false, "Simulate reading from the data source without replaying it. You will get information about expected replay time, number of found records etc.")
+	flag.DurationVar(&Settings.InputFileMaxWait, "input-file-max-wait", 0, "Set the maximum time between requests. Can help in situations when you have too long periods between request, and you want to skip them. Example: --input-raw-max-wait 1s")
 
 	flag.Var(&Settings.OutputFile, "output-file", "Write incoming requests to file: \n\tgor --input-raw :80 --output-file ./requests.gor")
 	flag.DurationVar(&Settings.OutputFileConfig.FlushInterval, "output-file-flush-interval", time.Second, "Interval for forcing buffer flush to the file, default: 1s.")
@@ -127,7 +129,7 @@ func init() {
 
 	flag.StringVar(&Settings.OutputFileConfig.BufferPath, "output-file-buffer", "/tmp", "The path for temporary storing current buffer: \n\tgor --input-raw :80 --output-file s3://mybucket/logs/%Y-%m-%d.gz --output-file-buffer /mnt/logs")
 
-	flag.BoolVar(&Settings.PrettifyHTTP, "prettify-http", false, "If enabled, will automatically decode requests and responses with: Content-Encoding: gzip and Transfer-Encoding: chunked. Useful for debugging, in conjuction with --output-stdout")
+	flag.BoolVar(&Settings.PrettifyHTTP, "prettify-http", false, "If enabled, will automatically decode requests and responses with: Content-Encoding: gzip and Transfer-Encoding: chunked. Useful for debugging, in conjunction with --output-stdout")
 
 	// input raw flags
 	flag.Var(&Settings.InputRAW, "input-raw", "Capture traffic from given port (use RAW sockets and require *sudo* access):\n\t# Capture traffic from 8080 port\n\tgor --input-raw :8080 --output-http staging.com")
@@ -145,6 +147,7 @@ func init() {
 	flag.BoolVar(&Settings.Promiscuous, "input-raw-promisc", false, "enable promiscuous mode")
 	flag.BoolVar(&Settings.Monitor, "input-raw-monitor", false, "enable RF monitor mode")
 	flag.BoolVar(&Settings.Stats, "input-raw-stats", false, "enable stats generator on raw TCP messages")
+	flag.BoolVar(&Settings.AllowIncomplete, "input-raw-allow-incomplete", false, "If turned on Gor will record HTTP messages with missing packets")
 
 	flag.StringVar(&Settings.Middleware, "middleware", "", "Used for modifying traffic using external command")
 
@@ -191,33 +194,17 @@ func init() {
 	flag.StringVar(&Settings.KafkaTLSConfig.ClientCert, "kafka-tls-client-cert", "", "Client certificate for Kafka TLS Config (mandatory with to kafka-tls-ca-cert and kafka-tls-client-key)")
 	flag.StringVar(&Settings.KafkaTLSConfig.ClientKey, "kafka-tls-client-key", "", "Client Key for Kafka TLS Config (mandatory with to kafka-tls-client-cert and kafka-tls-client-key)")
 
-	flag.Var(&Settings.ModifierConfig.Headers, "http-set-header", "Inject additional headers to http reqest:\n\tgor --input-raw :8080 --output-http staging.com --http-set-header 'User-Agent: Gor'")
-	flag.Var(&Settings.ModifierConfig.Headers, "output-http-header", "WARNING: `--output-http-header` DEPRECATED, use `--http-set-header` instead")
-
+	flag.Var(&Settings.ModifierConfig.Headers, "http-set-header", "Inject additional headers to http request:\n\tgor --input-raw :8080 --output-http staging.com --http-set-header 'User-Agent: Gor'")
 	flag.Var(&Settings.ModifierConfig.HeaderRewrite, "http-rewrite-header", "Rewrite the request header based on a mapping:\n\tgor --input-raw :8080 --output-http staging.com --http-rewrite-header Host: (.*).example.com,$1.beta.example.com")
-
 	flag.Var(&Settings.ModifierConfig.Params, "http-set-param", "Set request url param, if param already exists it will be overwritten:\n\tgor --input-raw :8080 --output-http staging.com --http-set-param api_key=1")
-
 	flag.Var(&Settings.ModifierConfig.Methods, "http-allow-method", "Whitelist of HTTP methods to replay. Anything else will be dropped:\n\tgor --input-raw :8080 --output-http staging.com --http-allow-method GET --http-allow-method OPTIONS")
-
 	flag.Var(&Settings.ModifierConfig.URLRegexp, "http-allow-url", "A regexp to match requests against. Filter get matched against full url with domain. Anything else will be dropped:\n\t gor --input-raw :8080 --output-http staging.com --http-allow-url ^www.")
-
 	flag.Var(&Settings.ModifierConfig.URLNegativeRegexp, "http-disallow-url", "A regexp to match requests against. Filter get matched against full url with domain. Anything else will be forwarded:\n\t gor --input-raw :8080 --output-http staging.com --http-disallow-url ^www.")
-
 	flag.Var(&Settings.ModifierConfig.URLRewrite, "http-rewrite-url", "Rewrite the request url based on a mapping:\n\tgor --input-raw :8080 --output-http staging.com --http-rewrite-url /v1/user/([^\\/]+)/ping:/v2/user/$1/ping")
-	flag.Var(&Settings.ModifierConfig.URLRewrite, "output-http-rewrite-url", "WARNING: `--output-http-rewrite-url` DEPRECATED, use `--http-rewrite-url` instead")
-
 	flag.Var(&Settings.ModifierConfig.HeaderFilters, "http-allow-header", "A regexp to match a specific header against. Requests with non-matching headers will be dropped:\n\t gor --input-raw :8080 --output-http staging.com --http-allow-header api-version:^v1")
-	flag.Var(&Settings.ModifierConfig.HeaderFilters, "output-http-header-filter", "WARNING: `--output-http-header-filter` DEPRECATED, use `--http-allow-header` instead")
-
 	flag.Var(&Settings.ModifierConfig.HeaderNegativeFilters, "http-disallow-header", "A regexp to match a specific header against. Requests with matching headers will be dropped:\n\t gor --input-raw :8080 --output-http staging.com --http-disallow-header \"User-Agent: Replayed by Gor\"")
-
 	flag.Var(&Settings.ModifierConfig.HeaderBasicAuthFilters, "http-basic-auth-filter", "A regexp to match the decoded basic auth string against. Requests with non-matching headers will be dropped:\n\t gor --input-raw :8080 --output-http staging.com --http-basic-auth-filter \"^customer[0-9].*\"")
-
 	flag.Var(&Settings.ModifierConfig.HeaderHashFilters, "http-header-limiter", "Takes a fraction of requests, consistently taking or rejecting a request based on the FNV32-1A hash of a specific header:\n\t gor --input-raw :8080 --output-http staging.com --http-header-limiter user-id:25%")
-
-	flag.Var(&Settings.ModifierConfig.HeaderHashFilters, "output-http-header-hash-filter", "WARNING: `output-http-header-hash-filter` DEPRECATED, use `--http-header-hash-limiter` instead")
-
 	flag.Var(&Settings.ModifierConfig.ParamHashFilters, "http-param-limiter", "Takes a fraction of requests, consistently taking or rejecting a request based on the FNV32-1A hash of a specific GET param:\n\t gor --input-raw :8080 --output-http staging.com --http-param-limiter user_id:25%")
 
 	// default values, using for tests

simpletime/time.go

@@ -0,0 +1,17 @@
+package simpletime
+
+import (
+	"time"
+)
+
+var Now time.Time
+
+func init() {
+	go func() {
+		for {
+			// Accurate enough
+			Now = time.Now()
+			time.Sleep(100 * time.Millisecond)
+		}
+	}()
+}

tcp/doc.go

@@ -15,7 +15,7 @@ maxSize := 5 << 20
 debugger := func(debugLevel int, data ...interface{}){} // debugger can also be nil
 messageHandler := func(mssg *tcp.Message){}
 
-mssgPool := tcp.NewMessagePool(maxMessageSize, messageExpire, debugger, messageHandler)
+mssgPool := tcp.NewMessageParser(maxMessageSize, messageExpire, debugger, messageHandler)
 listener.Listen(ctx, mssgPool.Handler)
 
 you can use pool.End or/and pool.Start to set custom session behaviors

tcp/tcp_message.go

@@ -1,90 +1,191 @@
 package tcp
 
 import (
-	"bytes"
-	"crypto/sha1"
+	"encoding/binary"
 	"encoding/hex"
 	"fmt"
+	"net"
+	"reflect"
 	"sort"
-	"sync"
 	"time"
+	"unsafe"
 
-	"github.com/buger/goreplay/size"
-	"github.com/google/gopacket"
+	"github.com/buger/goreplay/proto"
 )
 
+// TCPProtocol is a number to indicate type of protocol
+type TCPProtocol uint8
+
+const (
+	// ProtocolHTTP ...
+	ProtocolHTTP TCPProtocol = iota
+	// ProtocolBinary ...
+	ProtocolBinary
+)
+
+// Set is here so that TCPProtocol can implement flag.Var
+func (protocol *TCPProtocol) Set(v string) error {
+	switch v {
+	case "", "http":
+		*protocol = ProtocolHTTP
+	case "binary":
+		*protocol = ProtocolBinary
+	default:
+		return fmt.Errorf("unsupported protocol %s", v)
+	}
+	return nil
+}
+
+func (protocol *TCPProtocol) String() string {
+	switch *protocol {
+	case ProtocolBinary:
+		return "binary"
+	case ProtocolHTTP:
+		return "http"
+	default:
+		return ""
+	}
+}
+
 // Stats every message carry its own stats object
 type Stats struct {
-	LostData   int
-	Length     int       // length of the data
-	Start      time.Time // first packet's timestamp
-	End        time.Time // last packet's timestamp
-	SrcAddr    string
-	DstAddr    string
-	IsIncoming bool
-	TimedOut   bool // timeout before getting the whole message
-	Truncated  bool // last packet truncated due to max message size
-	IPversion  byte
+	LostData  int
+	Length    int       // length of the data
+	Start     time.Time // first packet's timestamp
+	End       time.Time // last packet's timestamp
+	SrcAddr   string
+	DstAddr   string
+	Direction Dir
+	TimedOut  bool // timeout before getting the whole message
+	Truncated bool // last packet truncated due to max message size
+	IPversion byte
 }
 
 // Message is the representation of a tcp message
 type Message struct {
-	packets []*Packet
-	done    chan bool
-	buf     bytes.Buffer
+	packets  []*Packet
+	parser   *MessageParser
+	feedback interface{}
 	Stats
 }
 
-// NewMessage ...
-func NewMessage(srcAddr, dstAddr string, ipVersion uint8) (m *Message) {
-	m = new(Message)
-	m.DstAddr = dstAddr
-	m.SrcAddr = srcAddr
-	m.IPversion = ipVersion
-	m.done = make(chan bool)
-	return
-}
-
-// UUID the unique id of a TCP session it is not granted to be unique!
+// UUID returns the UUID of a TCP request and its response.
 func (m *Message) UUID() []byte {
-	var src, dst string
-	if m.IsIncoming {
-		src = m.SrcAddr
-		dst = m.DstAddr
+	var streamID uint64
+	pckt := m.packets[0]
+
+	// check if response or request have generated the ID before.
+	if m.Direction == DirIncoming {
+		streamID = uint64(pckt.SrcPort)<<48 | uint64(pckt.DstPort)<<32 |
+			uint64(ip2int(pckt.SrcIP))
 	} else {
-		src = m.DstAddr
-		dst = m.SrcAddr
+		streamID = uint64(pckt.DstPort)<<48 | uint64(pckt.SrcPort)<<32 |
+			uint64(ip2int(pckt.DstIP))
 	}
 
-	length := len(src) + len(dst)
-	uuid := make([]byte, length)
-	copy(uuid, src)
-	copy(uuid[len(src):], dst)
-	sha := sha1.Sum(uuid)
-	uuid = make([]byte, 40)
-	hex.Encode(uuid, sha[:])
+	id := make([]byte, 12)
+	binary.BigEndian.PutUint64(id, streamID)
 
-	return uuid
-}
-
-func (m *Message) add(pckt *Packet) {
-	m.Length += len(pckt.Payload)
-	m.LostData += int(pckt.Lost)
-	m.packets = append(m.packets, pckt)
-	if len(pckt.Payload) > 0 {
-		m.buf.Write(pckt.Payload)
+	if m.Direction == DirIncoming {
+		binary.BigEndian.PutUint32(id[8:], pckt.Ack)
+	} else {
+		binary.BigEndian.PutUint32(id[8:], pckt.Seq)
 	}
-	m.End = pckt.Timestamp
+
+	uuidHex := make([]byte, 24)
+	hex.Encode(uuidHex[:], id[:])
+
+	return uuidHex
 }
 
-// Packets returns packets of this message
+func (m *Message) add(packet *Packet) bool {
+	// Skip duplicates
+	for _, p := range m.packets {
+		if p.Seq == packet.Seq {
+			return false
+		}
+	}
+
+	// Packets not always captured in same Seq order, and sometimes we need to prepend
+	if len(m.packets) == 0 || packet.Seq > m.packets[len(m.packets)-1].Seq {
+		m.packets = append(m.packets, packet)
+	} else if packet.Seq < m.packets[0].Seq {
+		m.packets = append([]*Packet{packet}, m.packets...)
+	} else { // insert somewhere in the middle...
+		for i, p := range m.packets {
+			if packet.Seq < p.Seq {
+				m.packets = append(m.packets[:i], append([]*Packet{packet}, m.packets[i:]...)...)
+				break
+			}
+		}
+	}
+
+	m.Length += len(packet.Payload)
+	m.LostData += int(packet.Lost)
+
+	if packet.Timestamp.After(m.End) || m.End.IsZero() {
+		m.End = packet.Timestamp
+	}
+
+	return true
+}
+
+// Packets returns packets of the message
 func (m *Message) Packets() []*Packet {
 	return m.packets
 }
 
+func (m *Message) MissingChunk() bool {
+	nextSeq := m.packets[0].Seq
+
+	for _, p := range m.packets {
+		if p.Seq != nextSeq {
+			return true
+		}
+
+		nextSeq += uint32(len(p.Payload))
+	}
+
+	return false
+}
+
+func (m *Message) PacketData() [][]byte {
+	tmp := make([][]byte, len(m.packets))
+
+	for i, p := range m.packets {
+		tmp[i] = p.Payload
+	}
+
+	return tmp
+}
+
 // Data returns data in this message
 func (m *Message) Data() []byte {
-	return m.buf.Bytes()
+	packetData := m.PacketData()
+	tmp := packetData[0]
+
+	if len(packetData) > 0 {
+		tmp, _ = copySlice(tmp, len(packetData[0]), packetData[1:]...)
+	}
+
+	// Remove Expect header, since its replay not fully supported
+	if state, ok := m.feedback.(*proto.HTTPState); ok {
+		if state.Continue100 {
+			tmp = proto.DeleteHeader(tmp, []byte("Expect"))
+		}
+	}
+
+	return tmp
+}
+
+// SetProtocolState set feedback/data that can be used later, e.g with End or Start hint
+func (m *Message) SetProtocolState(feedback interface{}) {
+	m.feedback = feedback
+}
+
+// ProtocolState returns feedback associated to this message
+func (m *Message) ProtocolState() interface{} {
+	return m.feedback
 }
 
 // Sort a helper to sort packets
@@ -92,144 +193,239 @@ func (m *Message) Sort() {
 	sort.SliceStable(m.packets, func(i, j int) bool { return m.packets[i].Seq < m.packets[j].Seq })
 }
 
-// Handler message handler
-type Handler func(*Message)
+// Emitter message handler
+type Emitter func(*Message)
 
-// Debugger is the debugger function. first params is the indicator of the issue's priority
-// the higher the number, the lower the priority. it can be 4 <= level <= 6.
-type Debugger func(int, ...interface{})
-
-// HintEnd hints the pool to stop the session, see MessagePool.End
+// HintEnd hints the parser to stop the session, see MessageParser.End
 // when set, it will be executed before checking FIN or RST flag
 type HintEnd func(*Message) bool
 
-// HintStart hints the pool to start the reassembling the message, see MessagePool.Start
-// when set, it will be used instead of checking SYN flag
-type HintStart func(*Packet) (IsIncoming, IsOutgoing bool)
+// HintStart hints the parser to start the reassembling the message, see MessageParser.Start
+// when set, it will be called after checking SYN flag
+type HintStart func(*Packet) (IsRequest, IsOutgoing bool)
 
-// MessagePool holds data of all tcp messages in progress(still receiving/sending packets).
+// MessageParser holds data of all tcp messages in progress(still receiving/sending packets).
 // message is identified by its source port and dst port, and last 4bytes of src IP.
-type MessagePool struct {
-	sync.Mutex
-	debug         Debugger
-	maxSize       size.Size // maximum message size, default 5mb
-	pool          map[uint64]*Message
-	handler       Handler
-	messageExpire time.Duration // the maximum time to wait for the final packet, minimum is 100ms
-	End           HintEnd
-	Start         HintStart
+type MessageParser struct {
+	m map[uint64]*Message
+
+	messageExpire  time.Duration // the maximum time to wait for the final packet, minimum is 100ms
+	allowIncompete bool
+	End            HintEnd
+	Start          HintStart
+	ticker         *time.Ticker
+	messages       chan *Message
+	packets        chan *PcapPacket
+	close          chan struct{} // to signal that we are able to close
+	ports          []uint16
+	ips            []net.IP
 }
 
-// NewMessagePool returns a new instance of message pool
-func NewMessagePool(maxSize size.Size, messageExpire time.Duration, debugger Debugger, handler Handler) (pool *MessagePool) {
-	pool = new(MessagePool)
-	pool.debug = debugger
-	pool.handler = handler
-	pool.messageExpire = time.Millisecond * 100
-	if pool.messageExpire < messageExpire {
-		pool.messageExpire = messageExpire
+// NewMessageParser returns a new instance of message parser
+func NewMessageParser(messages chan *Message, ports []uint16, ips []net.IP, messageExpire time.Duration, allowIncompete bool) (parser *MessageParser) {
+	parser = new(MessageParser)
+
+	parser.messageExpire = messageExpire
+	if parser.messageExpire == 0 {
+		parser.messageExpire = time.Millisecond * 1000
 	}
-	pool.maxSize = maxSize
-	if pool.maxSize < 1 {
-		pool.maxSize = 5 << 20
+
+	parser.allowIncompete = allowIncompete
+
+	parser.packets = make(chan *PcapPacket, 10000)
+
+	if messages == nil {
+		messages = make(chan *Message, 1000)
 	}
-	pool.pool = make(map[uint64]*Message)
-	return pool
+	parser.messages = messages
+
+	parser.m = make(map[uint64]*Message)
+	parser.ticker = time.NewTicker(time.Millisecond * 100)
+	parser.close = make(chan struct{}, 1)
+
+	parser.ports = ports
+	parser.ips = ips
+
+	go parser.wait()
+	return parser
 }
 
-// Handler returns packet handler
-func (pool *MessagePool) Handler(packet gopacket.Packet) {
-	var in, out bool
-	pckt, err := ParsePacket(packet)
-	if err != nil || pckt == nil {
-		go pool.say(4, fmt.Sprintf("error decoding packet(%dBytes):%s\n", packet.Metadata().CaptureLength, err))
-		return
-	}
-	pool.Lock()
-	defer pool.Unlock()
-	lst := 3
-	if pckt.Version == 6 {
-		lst = 15
+var packetLen int
+
+// Packet returns packet handler
+func (parser *MessageParser) PacketHandler(packet *PcapPacket) {
+	packetLen++
+	parser.packets <- packet
+}
+
+func (parser *MessageParser) wait() {
+	var (
+		now time.Time
+	)
+	for {
+		select {
+		case pckt := <-parser.packets:
+			parser.processPacket(parser.parsePacket(pckt))
+		case now = <-parser.ticker.C:
+			parser.timer(now)
+		case <-parser.close:
+			parser.ticker.Stop()
+			// parser.Close should wait for this function to return
+			parser.close <- struct{}{}
+			return
+			// default:
+		}
 	}
-	key := uint64(pckt.SrcPort)<<48 | uint64(pckt.DstPort)<<32 |
-		uint64(pckt.SrcIP[lst])<<24 | uint64(pckt.SrcIP[lst-1])<<16 |
-		uint64(pckt.SrcIP[lst-2])<<8 | uint64(pckt.SrcIP[lst-3])
-	m, ok := pool.pool[key]
-	if pckt.RST {
-		if ok {
-			m.done <- true
-			<-m.done
+}
+
+func (parser *MessageParser) parsePacket(pcapPkt *PcapPacket) *Packet {
+	pckt, err := ParsePacket(pcapPkt.Data, pcapPkt.LType, pcapPkt.LTypeLen, pcapPkt.Ci, false)
+	if err != nil {
+		if _, empty := err.(EmptyPacket); !empty {
+			stats.Add("packet_error", 1)
 		}
-		key = uint64(pckt.DstPort)<<48 | uint64(pckt.SrcPort)<<32 |
-			uint64(pckt.DstIP[lst])<<24 | uint64(pckt.DstIP[lst-1])<<16 |
-			uint64(pckt.DstIP[lst-2])<<8 | uint64(pckt.DstIP[lst-3])
-		m, ok = pool.pool[key]
-		if ok {
-			m.done <- true
-			<-m.done
+		return nil
+	}
+
+	for _, p := range parser.ports {
+		if pckt.DstPort == p {
+			for _, ip := range parser.ips {
+				if pckt.DstIP.Equal(ip) {
+					pckt.Direction = DirIncoming
+					break
+				}
+			}
+			break
 		}
-		go pool.say(4, fmt.Sprintf("RST flag from %s to %s at %s\n", pckt.Src(), pckt.Dst(), pckt.Timestamp))
+	}
+
+	return pckt
+}
+
+func (parser *MessageParser) processPacket(pckt *Packet) {
+	if pckt == nil {
 		return
 	}
+
+	// Trying to build unique hash, but there is small chance of collision
+	// No matter if it is request or response, all packets in the same message have same
+	m, ok := parser.m[pckt.MessageID()]
 	switch {
 	case ok:
-		pool.addPacket(m, pckt)
-		return
-	case pckt.SYN:
-		in = !pckt.ACK
-	case pool.Start != nil:
-		if in, out = pool.Start(pckt); !(in || out) {
-			return
+		if m.Direction == DirUnknown {
+			if in, out := parser.Start(pckt); in || out {
+				if in {
+					m.Direction = DirIncoming
+				} else {
+					m.Direction = DirOutcoming
+				}
+			}
 		}
-	default:
+		parser.addPacket(m, pckt)
 		return
+	case pckt.Direction == DirUnknown && parser.Start != nil:
+		if in, out := parser.Start(pckt); in || out {
+			if in {
+				pckt.Direction = DirIncoming
+			} else {
+				pckt.Direction = DirOutcoming
+			}
+		}
 	}
-	m = NewMessage(pckt.Src(), pckt.Dst(), pckt.Version)
-	m.IsIncoming = in
-	pool.pool[key] = m
+
+	m = new(Message)
+	m.Direction = pckt.Direction
+	m.SrcAddr = pckt.SrcIP.String()
+	m.DstAddr = pckt.DstIP.String()
+	parser.m[pckt.MessageID()] = m
 	m.Start = pckt.Timestamp
-	go pool.dispatch(key, m)
-	pool.addPacket(m, pckt)
+	m.parser = parser
+	parser.addPacket(m, pckt)
 }
 
-func (pool *MessagePool) dispatch(key uint64, m *Message) {
-	select {
-	case <-m.done:
-		defer func() { m.done <- true }() // signal that message was dispatched
-	case <-time.After(pool.messageExpire):
-		pool.Lock()
-		defer pool.Unlock()
-		// avoid dispathing message twice
-		if _, ok := pool.pool[key]; !ok {
-			return
+func (parser *MessageParser) addPacket(m *Message, pckt *Packet) bool {
+	if !m.add(pckt) {
+		return false
+	}
+
+	// If we are using protocol parsing, like HTTP, depend on its parsing func.
+	// For the binary procols wait for message to expire
+	if parser.End != nil {
+		if parser.End(m) {
+			parser.Emit(m)
+			return true
 		}
-		m.TimedOut = true
+
+		parser.Fix100Continue(m)
 	}
-	delete(pool.pool, key)
-	pool.handler(m)
+
+	return true
 }
 
-func (pool *MessagePool) addPacket(m *Message, pckt *Packet) {
-	trunc := m.Length + len(pckt.Payload) - int(pool.maxSize)
-	if trunc > 0 {
-		m.Truncated = true
-		pckt.Payload = pckt.Payload[:int(pool.maxSize)-m.Length]
+func (parser *MessageParser) Fix100Continue(m *Message) {
+	if state, ok := m.feedback.(*proto.HTTPState); ok && state.Continue100 {
+		delete(parser.m, m.packets[0].MessageID())
+
+		// Shift Ack by given offset
+		// Size of "HTTP/1.1 100 Continue\r\n\r\n" message
+		for _, p := range m.packets {
+			p.messageID = 0
+			p.Ack += 25
+		}
+
+		// If next section was aready approved and received, merge messages
+		if next, found := parser.m[m.packets[0].MessageID()]; found {
+			for _, p := range next.packets {
+				parser.addPacket(m, p)
+			}
+		}
+
+		// Re-add (or override) again with new message and ID
+		parser.m[m.packets[0].MessageID()] = m
 	}
-	m.add(pckt)
-	switch {
-	case trunc >= 0:
-	case pckt.FIN:
-	case pool.End != nil && pool.End(m):
-	default:
-		return
-	}
-	m.done <- true
-	<-m.done
 }
 
-// this function should not block other pool operations
-func (pool *MessagePool) say(level int, args ...interface{}) {
-	if pool.debug != nil {
-		pool.debug(level, args...)
+func (parser *MessageParser) Read() *Message {
+	m := <-parser.messages
+	return m
+}
+
+func (parser *MessageParser) Emit(m *Message) {
+	stats.Add("message_count", 1)
+
+	delete(parser.m, m.packets[0].MessageID())
+
+	parser.messages <- m
+}
+
+func GetUnexportedField(field reflect.Value) interface{} {
+	return reflect.NewAt(field.Type(), unsafe.Pointer(field.UnsafeAddr())).Elem().Interface()
+}
+
+var failMsg int
+
+func (parser *MessageParser) timer(now time.Time) {
+	packetLen = 0
+
+	packetQueueLen.Set(int64(len(parser.packets)))
+	messageQueueLen.Set(int64(len(parser.m)))
+
+	for _, m := range parser.m {
+		if now.Sub(m.End) > parser.messageExpire {
+			m.TimedOut = true
+			stats.Add("message_timeout_count", 1)
+			failMsg++
+			if parser.End == nil || parser.allowIncompete {
+				parser.Emit(m)
+			}
+
+			delete(parser.m, m.packets[0].MessageID())
+		}
 	}
 }
+
+func (parser *MessageParser) Close() error {
+	parser.close <- struct{}{}
+	<-parser.close // wait for timer to be closed!
+	return nil
+}

tcp/tcp_packet.go

@@ -2,12 +2,52 @@ package tcp
 
 import (
 	"encoding/binary"
+	"expvar"
 	"fmt"
 	"net"
 	"time"
 
 	"github.com/google/gopacket"
-	"github.com/google/gopacket/layers"
+)
+
+func copySlice(to []byte, skip int, from ...[]byte) ([]byte, int) {
+	var totalLen int
+	for _, s := range from {
+		totalLen += len(s)
+	}
+	totalLen += skip
+
+	if cap(to) < totalLen {
+		diff := totalLen - cap(to)
+		to = append(to, make([]byte, diff)...)
+	}
+
+	for _, s := range from {
+		skip += copy(to[skip:], s)
+	}
+
+	return to, skip
+}
+
+var stats *expvar.Map
+var packetQueueLen, messageQueueLen *expvar.Int
+
+func init() {
+	packetQueueLen = new(expvar.Int)
+	messageQueueLen = new(expvar.Int)
+
+	stats = expvar.NewMap("tcp")
+	stats.Init()
+	stats.Set("packet_queue", packetQueueLen)
+	stats.Set("message_queue", messageQueueLen)
+}
+
+type Dir int
+
+const (
+	DirUnknown = iota
+	DirIncoming
+	DirOutcoming
 )
 
 /*
@@ -17,82 +57,169 @@ calllers must make sure that ParsePacket has'nt returned any error before callin
 function.
 */
 type Packet struct {
-	// Link layer
-	gopacket.LinkLayer
+	Direction          Dir
+	messageID          uint64
+	SrcIP, DstIP       net.IP
+	Version            uint8
+	SrcPort, DstPort   uint16
+	Ack, Seq           uint32
+	ACK, SYN, FIN, RST bool
+	Lost               uint32
+	Retry              int
+	CaptureLength      int
+	Timestamp          time.Time
+	Payload            []byte
+	buf                []byte
 
-	// IP Header
-	Version uint8 // Ip version
-	SrcIP   net.IP
-	DstIP   net.IP
-	IHL     uint8
-	Length  uint16
+	created time.Time
+	gc      bool
+}
 
-	// TCP Segment Header
-	*layers.TCP
-
-	// Data info
-	Lost      uint16
-	Timestamp time.Time
+type PcapPacket struct {
+	Data     []byte
+	LType    int
+	LTypeLen int
+	Ci       *gopacket.CaptureInfo
 }
 
 // ParsePacket parse raw packets
-func ParsePacket(packet gopacket.Packet) (pckt *Packet, err error) {
-	// early check of error
-	if packet == nil {
-		return
-	}
-	defer func() {
-		if packet.ErrorLayer() != nil {
-			err = packet.ErrorLayer().Error()
-			return
-		}
-	}()
-
-	// initialization
+func ParsePacket(data []byte, lType, lTypeLen int, ci *gopacket.CaptureInfo, allowEmpty bool) (pckt *Packet, err error) {
 	pckt = new(Packet)
-	pckt.Timestamp = packet.Metadata().Timestamp
-	if pckt.Timestamp.IsZero() {
-		pckt.Timestamp = time.Now()
+	if err := pckt.parse(data, lType, lTypeLen, ci, allowEmpty); err != nil {
+		return nil, err
 	}
 
-	// parsing link layer
-	pckt.LinkLayer = packet.LinkLayer()
+	return pckt, nil
+}
 
-	// parsing network layer
-	if net4, ok := packet.NetworkLayer().(*layers.IPv4); ok {
+func (pckt *Packet) parse(data []byte, lType, lTypeLen int, cp *gopacket.CaptureInfo, allowEmpty bool) error {
+	pckt.Retry = 0
+	pckt.messageID = 0
+	pckt.buf = pckt.buf[:]
+
+	// TODO: check resolution
+	pckt.Timestamp = cp.Timestamp
+
+	if len(data) < lTypeLen {
+		return ErrHdrLength("Link")
+	}
+	if len(data) <= lTypeLen {
+		return ErrHdrMissing("IPv4 or IPv6")
+	}
+
+	ldata := data[lTypeLen:]
+	var proto byte
+	var netLayer, transLayer []byte
+
+	if ldata[0]>>4 == 4 {
+		// IPv4 header
+		if len(ldata) < 20 {
+			return ErrHdrLength("IPv4")
+		}
+		proto = ldata[9]
+		ihl := int(ldata[0]&0x0F) * 4
+		if ihl < 20 {
+			return ErrHdrInvalid("IPv4's IHL")
+		}
+		if len(ldata) < ihl {
+			return ErrHdrLength("IPv4 opts")
+		}
+		netLayer = ldata[:ihl]
+	} else if ldata[0]>>4 == 6 {
+		if len(ldata) < 40 {
+			return ErrHdrLength("IPv6")
+		}
+		proto = ldata[6]
+		totalLen := 40
+		for ipv6ExtensionHdr(proto) {
+			hdr := len(ldata) - totalLen
+			if hdr < 8 {
+				return ErrHdrExpected("IPv6 opts")
+			}
+			extLen := 8
+			if proto != 44 {
+				extLen = int(ldata[totalLen+1]+1) * 8
+			}
+			if hdr < extLen {
+				return ErrHdrLength("IPv6 opts")
+			}
+			proto = ldata[totalLen]
+			totalLen += extLen
+		}
+		netLayer = ldata[:totalLen]
+	} else {
+		return ErrHdrExpected("IPv4 or IPv6")
+	}
+	if proto != 6 {
+		return ErrHdrExpected("TCP")
+	}
+	if len(data) <= len(netLayer) {
+		return ErrHdrMissing("TCP")
+	}
+	ndata := ldata[len(netLayer):]
+	// TCP header
+	if len(ndata) < 20 {
+		return ErrHdrLength("TCP")
+	}
+	dOf := int(ndata[12]>>4) * 4
+	if dOf < 20 {
+		return ErrHdrInvalid("TCP's ndata offset")
+	}
+	if len(ndata) < dOf {
+		return ErrHdrLength("TCP opts")
+	}
+
+	// There are case when packet have padding but dOf shows its not
+	empty := true
+	for i := 0; i < len(ndata[dOf:]); i++ {
+		if ndata[dOf:][i] != 0 {
+			empty = false
+			break
+		}
+	}
+
+	if !allowEmpty && empty {
+		return EmptyPacket("")
+	}
+
+	if (netLayer[0] >> 4) == 4 {
+		// IPv4 header
 		pckt.Version = 4
-		pckt.SrcIP = net4.SrcIP
-		pckt.DstIP = net4.DstIP
-		pckt.IHL = net4.IHL * 4
-		pckt.Length = net4.Length
-	} else if net6, ok := packet.NetworkLayer().(*layers.IPv6); ok {
+		pckt.SrcIP = netLayer[12:16]
+		pckt.DstIP = netLayer[16:20]
+	} else {
+		// IPv6 header
 		pckt.Version = 6
-		pckt.SrcIP = net6.SrcIP
-		pckt.DstIP = net6.DstIP
-		pckt.IHL = 40
-		pckt.Length = net6.Length
-	} else {
-		pckt = nil
-		return
+		pckt.SrcIP = netLayer[8:24]
+		pckt.DstIP = netLayer[24:40]
 	}
 
-	// parsing tcp header(transportation layer)
-	if tcp, ok := packet.TransportLayer().(*layers.TCP); ok {
-		pckt.TCP = tcp
-	} else {
-		pckt = nil
-		return
-	}
-	pckt.DataOffset *= 4
+	transLayer = ndata[:dOf]
 
-	// calculating lost data
-	headerSize := int(uint32(pckt.DataOffset) + uint32(pckt.IHL))
-	if pckt.Version == 6 {
-		headerSize -= 40 // in ipv6 the length of payload doesn't include the IPheader size
-	}
-	pckt.Lost = pckt.Length - uint16(headerSize+len(pckt.Payload))
+	pckt.CaptureLength = cp.CaptureLength
+	pckt.SrcPort = binary.BigEndian.Uint16(transLayer[0:2])
+	pckt.DstPort = binary.BigEndian.Uint16(transLayer[2:4])
+	pckt.Seq = binary.BigEndian.Uint32(transLayer[4:8])
+	pckt.Ack = binary.BigEndian.Uint32(transLayer[8:12])
+	pckt.FIN = transLayer[13]&0x01 != 0
+	pckt.SYN = transLayer[13]&0x02 != 0
+	pckt.RST = transLayer[13]&0x04 != 0
+	pckt.ACK = transLayer[13]&0x10 != 0
+	pckt.Lost = uint32(cp.Length - cp.CaptureLength)
 
-	return
+	pckt.Payload = ndata[dOf:]
+
+	return nil
+}
+
+func (pckt *Packet) MessageID() uint64 {
+	if pckt.messageID == 0 {
+		// All packets in the same message will share the same ID
+		pckt.messageID = uint64(pckt.SrcPort)<<48 | uint64(pckt.DstPort)<<32 |
+			(uint64(ip2int(pckt.SrcIP)) + uint64(ip2int(pckt.DstIP)) + uint64(pckt.Ack))
+	}
+
+	return pckt.messageID
 }
 
 // Src returns the source socket of a packet
@@ -105,93 +232,53 @@ func (pckt *Packet) Dst() string {
 	return fmt.Sprintf("%s:%d", pckt.DstIP, pckt.DstPort)
 }
 
-// SYNOptions returns MSS and windowscale of syn packets
-func (pckt *Packet) SYNOptions() (mss uint16, windowscale byte) {
-	if !pckt.SYN {
-		return
-	}
-	for _, v := range pckt.Options {
-		if v.OptionType == layers.TCPOptionKindMSS {
-			mss = binary.BigEndian.Uint16(v.OptionData)
-			continue
-		}
-		if v.OptionType == layers.TCPOptionKindWindowScale {
-			if v.OptionLength > 0 {
-				windowscale = v.OptionData[0]
-			}
-		}
-	}
-	return
+type EmptyPacket string
+
+func (err EmptyPacket) Error() string {
+	return "Empty packet"
 }
 
-// LinkInfo returns info about the link layer
-func (pckt *Packet) LinkInfo() string {
-	if l, ok := pckt.LinkLayer.(*layers.Ethernet); ok {
-		return fmt.Sprintf(
-			"Source Mac: %s\nDestination Mac: %s\nProtocol: %s",
-			l.SrcMAC,
-			l.DstMAC,
-			l.EthernetType,
-		)
-	}
-	return "<Not Ethernet>"
+// ErrHdrLength returned on short header length
+type ErrHdrLength string
+
+func (err ErrHdrLength) Error() string {
+	return "short " + string(err) + " length"
 }
 
-// Flag returns formatted tcp flags
-func (pckt *Packet) Flag() (flag string) {
-	if pckt.FIN {
-		flag += "FIN, "
-	}
-	if pckt.SYN {
-		flag += "SYN, "
-	}
-	if pckt.RST {
-		flag += "RST, "
-	}
-	if pckt.PSH {
-		flag += "PSH, "
-	}
-	if pckt.ACK {
-		flag += "ACK, "
-	}
-	if pckt.URG {
-		flag += "URG, "
-	}
-	if len(flag) != 0 {
-		return flag[:len(flag)-2]
-	}
-	return flag
+// ErrHdrMissing returned on missing header(s)
+type ErrHdrMissing string
+
+func (err ErrHdrMissing) Error() string {
+	return "missing " + string(err) + " header(s)"
 }
 
-// String output for a TCP Packet
-func (pckt *Packet) String() string {
-	return fmt.Sprintf(`Time: %s
-%s
-Source: %s
-Destination: %s
-IHL: %d
-Total Length: %d
-Sequence: %d
-Acknowledgment: %d
-DataOffset: %d
-Window: %d
-Flag: %s
-Options: %s
-Data Size: %d
-Lost Data: %d`,
-		pckt.Timestamp.Format(time.StampNano),
-		pckt.LinkInfo(),
-		pckt.Src(),
-		pckt.Dst(),
-		pckt.IHL,
-		pckt.Length,
-		pckt.Seq,
-		pckt.Ack,
-		pckt.DataOffset,
-		pckt.Window,
-		pckt.Flag(),
-		pckt.Options,
-		len(pckt.Payload),
-		pckt.Lost,
-	)
+// ErrHdrExpected returned when header(s) are different from the one expected
+type ErrHdrExpected string
+
+func (err ErrHdrExpected) Error() string {
+	return "expected " + string(err) + " header(s)"
+}
+
+// ErrHdrInvalid returned when header(s) are different from the one expected
+type ErrHdrInvalid string
+
+func (err ErrHdrInvalid) Error() string {
+	return "invalid " + string(err) + " value"
+}
+
+// https://en.wikipedia.org/wiki/IPv6_packet#Extension_headers
+func ipv6ExtensionHdr(b byte) bool {
+	// TODO: support all extension headers
+	return b == 0 || b == 43 || b == 44
+}
+
+func ip2int(ip net.IP) uint32 {
+	if len(ip) == 0 {
+		return 0
+	}
+
+	if len(ip) == 16 {
+		return binary.BigEndian.Uint32(ip[12:16])
+	}
+	return binary.BigEndian.Uint32(ip)
 }

tcp/tcp_test.go

@@ -3,211 +3,255 @@ package tcp
 import (
 	"bytes"
 	"encoding/binary"
+
+	// "runtime"
 	"testing"
 	"time"
 
+	"github.com/stretchr/testify/assert"
+
 	"github.com/buger/goreplay/proto"
 	"github.com/google/gopacket"
 	"github.com/google/gopacket/layers"
 )
 
-var decodeOpts = gopacket.DecodeOptions{Lazy: true, NoCopy: true}
+func generateHeader(request bool, seq uint32, length uint16) []byte {
+	hdr := make([]byte, 4+24+24)
+	binary.BigEndian.PutUint32(hdr, uint32(layers.ProtocolFamilyIPv4))
 
-func headersIP4(seq uint32, length uint16) (headers [54]byte) {
-	// set ethernet headers
-	binary.BigEndian.PutUint16(headers[12:14], uint16(layers.EthernetTypeIPv4))
-
-	// set ip header
-	ip := headers[14:]
-	copy(ip[0:2], []byte{4<<4 | 5, 0x28<<2 | 0x00})
-	binary.BigEndian.PutUint16(ip[2:4], length+40)
+	ip := hdr[4:]
+	ip[0] = 4<<4 | 6
+	binary.BigEndian.PutUint16(ip[2:4], length+24+24)
 	ip[9] = uint8(layers.IPProtocolTCP)
-	copy(ip[12:16], []byte{192, 168, 1, 2})
-	copy(ip[16:], []byte{192, 168, 1, 3})
+	copy(ip[12:16], []byte{127, 0, 0, 1})
+	copy(ip[16:], []byte{127, 0, 0, 1})
 
 	// set tcp header
-	tcp := ip[20:]
-	binary.BigEndian.PutUint16(tcp[0:2], 45678)
-	binary.BigEndian.PutUint16(tcp[2:4], 8001)
-	tcp[12] = 5 << 4
-	return
+	tcp := ip[24:]
+	tcp[12] = 6 << 4
+
+	if request {
+		binary.BigEndian.PutUint16(tcp, 5535)
+		binary.BigEndian.PutUint16(tcp[2:], 8000)
+	} else {
+		binary.BigEndian.PutUint16(tcp, 8000)
+		binary.BigEndian.PutUint16(tcp[2:], 5535)
+	}
+	binary.BigEndian.PutUint32(tcp[4:], seq)
+	return hdr
 }
 
-func GetPackets(start uint32, _len int, payload []byte) []gopacket.Packet {
-	var packets = make([]gopacket.Packet, _len)
+func GetPackets(request bool, start uint32, _len int, payload []byte) []*Packet {
+	var packets = make([]*Packet, _len)
+	var err error
 	for i := start; i < start+uint32(_len); i++ {
-		data := make([]byte, 54+len(payload))
-		h := headersIP4(i, uint16(len(payload)))
-		copy(data, h[:])
-		copy(data[len(h):], payload)
-		packets[i-start] = gopacket.NewPacket(data, layers.LinkTypeEthernet, decodeOpts)
+		d := append(generateHeader(request, i, uint16(len(payload))), payload...)
+		ci := &gopacket.CaptureInfo{Length: len(d), CaptureLength: len(d), Timestamp: time.Now()}
+
+		packets[i-start], err = ParsePacket(d, int(layers.LinkTypeLoop), 4, ci, true)
+		if request {
+			packets[i-start].Direction = DirIncoming
+		} else {
+			packets[i-start].Direction = DirOutcoming
+		}
+		if err != nil {
+			panic(err)
+		}
 	}
 	return packets
 }
 
-func TestMessageParserWithHint(t *testing.T) {
-	var mssg = make(chan *Message, 3)
-	pool := NewMessagePool(1<<20, time.Second, nil, func(m *Message) { mssg <- m })
-	pool.Start = func(pckt *Packet) (bool, bool) {
+func TestRequestResponseMapping(t *testing.T) {
+	packets := []*Packet{
+		{SrcPort: 60000, DstPort: 80, Ack: 1, Seq: 1, Direction: DirIncoming, Timestamp: time.Unix(1, 0), Payload: []byte("GET / HTTP/1.1\r\n")},
+		{SrcPort: 60000, DstPort: 80, Ack: 1, Seq: 17, Direction: DirIncoming, Timestamp: time.Unix(2, 0), Payload: []byte("Host: localhost\r\n\r\n")},
+
+		// Seq of first response packet match Ack of first request packet
+		{SrcPort: 80, DstPort: 60000, Ack: 36, Seq: 1, Direction: DirOutcoming, Timestamp: time.Unix(3, 0), Payload: []byte("HTTP/1.1 200 OK\r\n")},
+		{SrcPort: 80, DstPort: 60000, Ack: 36, Seq: 18, Direction: DirOutcoming, Timestamp: time.Unix(4, 0), Payload: []byte("Content-Length: 0\r\n\r\n")},
+
+		// Same TCP stream
+		{SrcPort: 60000, DstPort: 80, Ack: 39, Seq: 36, Direction: DirIncoming, Timestamp: time.Unix(5, 0), Payload: []byte("GET / HTTP/1.1\r\n")},
+		{SrcPort: 60000, DstPort: 80, Ack: 39, Seq: 52, Direction: DirIncoming, Timestamp: time.Unix(6, 0), Payload: []byte("Host: localhost\r\n\r\n")},
+
+		// Seq of first response packet match Ack of first request packet
+		{SrcPort: 80, DstPort: 60000, Ack: 71, Seq: 39, Direction: DirOutcoming, Timestamp: time.Unix(7, 0), Payload: []byte("HTTP/1.1 200 OK\r\n")},
+		{SrcPort: 80, DstPort: 60000, Ack: 71, Seq: 56, Direction: DirOutcoming, Timestamp: time.Unix(8, 0), Payload: []byte("Content-Length: 0\r\n\r\n")},
+	}
+
+	parser := NewMessageParser(nil, nil, nil, time.Second, false)
+	parser.Start = func(pckt *Packet) (bool, bool) {
 		return proto.HasRequestTitle(pckt.Payload), proto.HasResponseTitle(pckt.Payload)
 	}
-	pool.End = func(m *Message) bool {
-		return proto.HasFullPayload(m.Data())
+	parser.End = func(m *Message) bool {
+		return proto.HasFullPayload(m, m.PacketData()...)
 	}
-	packets := GetPackets(1, 30, nil)
-	packets[0].Data()[14:][20:][13] = 2  // SYN flag
-	packets[10].Data()[14:][20:][13] = 2 // SYN flag
-	packets[29].Data()[14:][20:][13] = 1 // FIN flag
-	packets[4] = GetPackets(5, 1, []byte("HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nTransfer-Encoding: chunked\r\n\r\n7"))[0]
-	packets[5] = GetPackets(6, 1, []byte("\r\nMozilla\r\n9\r\nDeveloper\r"))[0]
-	packets[6] = GetPackets(7, 1, []byte("\n7\r\nNetwork\r\n0\r\n\r\n"))[0]
-	packets[14] = GetPackets(5, 1, []byte("POST / HTTP/1.1\r\nContent-Type: text/plain\r\nContent-Length: 23\r\n\r\n"))[0]
-	packets[15] = GetPackets(6, 1, []byte("MozillaDeveloper"))[0]
-	packets[16] = GetPackets(7, 1, []byte("Network"))[0]
-	packets[24] = GetPackets(5, 1, []byte("HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nContent-Length: 0\r\n\r"))[0]
-	for i := 0; i < 30; i++ {
-		pool.Handler(packets[i])
+
+	for _, packet := range packets {
+		parser.processPacket(packet)
 	}
-	var m *Message
-	select {
-	case <-time.After(time.Second):
-		t.Errorf("can't parse packets fast enough")
-		return
-	case m = <-mssg:
+
+	messages := []*Message{}
+	for i := 0; i < 4; i++ {
+		m := parser.Read()
+		messages = append(messages, m)
 	}
-	if len(m.packets) != 7 {
-		t.Errorf("expected to have 7 packets got %d", len(m.packets))
+
+	assert.Equal(t, int(messages[0].Direction), int(DirIncoming))
+	assert.Equal(t, int(messages[1].Direction), int(DirOutcoming))
+	assert.Equal(t, int(messages[2].Direction), int(DirIncoming))
+	assert.Equal(t, int(messages[3].Direction), int(DirOutcoming))
+
+	assert.Equal(t, messages[0].UUID(), messages[1].UUID())
+	assert.Equal(t, messages[2].UUID(), messages[3].UUID())
+
+	assert.NotEqual(t, messages[0].UUID(), messages[2].UUID())
+}
+
+func TestMessageParserWithHint(t *testing.T) {
+	parser := NewMessageParser(nil, nil, nil, time.Second, false)
+	parser.Start = func(pckt *Packet) (bool, bool) {
+		return proto.HasRequestTitle(pckt.Payload), proto.HasResponseTitle(pckt.Payload)
 	}
+	parser.End = func(m *Message) bool {
+		return proto.HasFullPayload(m, m.PacketData()...)
+	}
+
+	packets := []*Packet{
+		// Seq of first response packet match Ack of first request packet
+		{SrcPort: 80, DstPort: 60000, Ack: 1, Seq: 1, Direction: DirOutcoming, Timestamp: time.Unix(1, 0), Payload: []byte("HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nTransfer-Encoding: chunked\r\n\r\n7\r\n")},
+		{SrcPort: 80, DstPort: 60000, Ack: 1, Seq: 18, Direction: DirOutcoming, Timestamp: time.Unix(2, 0), Payload: []byte("\r\nMozilla\r\n9\r\nDeveloper\r")},
+		{SrcPort: 80, DstPort: 60000, Ack: 1, Seq: 42, Direction: DirOutcoming, Timestamp: time.Unix(3, 0), Payload: []byte("\n7\r\nNetwork\r\n0\r\n\r\n")},
+
+		{SrcPort: 60000, DstPort: 80, Ack: 60, Seq: 1, Direction: DirIncoming, Timestamp: time.Unix(4, 0), Payload: []byte("POST / HTTP/1.1\r\nContent-Type: text/plain\r\nContent-Length: 23\r\n\r\n")},
+		{SrcPort: 60000, DstPort: 80, Ack: 60, Seq: 66, Direction: DirIncoming, Timestamp: time.Unix(5, 0), Payload: []byte("MozillaDeveloper")},
+		{SrcPort: 60000, DstPort: 80, Ack: 60, Seq: 82, Direction: DirIncoming, Timestamp: time.Unix(6, 0), Payload: []byte("Network")},
+
+		{SrcPort: 80, DstPort: 60000, Ack: 89, Seq: 1, Direction: DirOutcoming, Timestamp: time.Unix(7, 0), Payload: []byte("HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nContent-Length: 0\r\n\r\n")},
+	}
+
+	for _, p := range packets {
+		parser.processPacket(p)
+	}
+
+	messages := []*Message{}
+	for i := 0; i < 3; i++ {
+		m := parser.Read()
+		messages = append(messages, m)
+	}
+
+	if !bytes.HasSuffix(messages[0].Data(), []byte("\n7\r\nNetwork\r\n0\r\n\r\n")) {
+		t.Errorf("expected to %q to have suffix %q", messages[0].Data(), []byte("\n7\r\nNetwork\r\n0\r\n\r\n"))
+	}
+
+	if !bytes.HasSuffix(messages[1].Data(), []byte("Network")) {
+		t.Errorf("expected to %q to have suffix %q", messages[1].Data(), []byte("Network"))
+	}
+
+	if !bytes.HasSuffix(messages[2].Data(), []byte("Content-Length: 0\r\n\r\n")) {
+		t.Errorf("expected to %q to have suffix %q", messages[2].Data(), []byte("Content-Length: 0\r\n\r\n"))
+	}
+}
+
+func TestMessageParserWrongOrder(t *testing.T) {
+	parser := NewMessageParser(nil, nil, nil, time.Second, false)
+	parser.Start = func(pckt *Packet) (bool, bool) {
+		return proto.HasRequestTitle(pckt.Payload), proto.HasResponseTitle(pckt.Payload)
+	}
+	parser.End = func(m *Message) bool {
+		return proto.HasFullPayload(m, m.PacketData()...)
+	}
+	packets := []*Packet{
+		// Seq of first response packet match Ack of first request packet
+		{SrcPort: 60000, DstPort: 80, Ack: 60, Seq: 66, Direction: DirIncoming, Timestamp: time.Unix(5, 0), Payload: []byte("MozillaDeveloper")},
+		{SrcPort: 80, DstPort: 60000, Ack: 1, Seq: 1, Direction: DirOutcoming, Timestamp: time.Unix(1, 0), Payload: []byte("HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nTransfer-Encoding: chunked\r\n\r\n7\r\n")},
+		{SrcPort: 80, DstPort: 60000, Ack: 1, Seq: 42, Direction: DirOutcoming, Timestamp: time.Unix(3, 0), Payload: []byte("\n7\r\nNetwork\r\n0\r\n\r\n")},
+
+		{SrcPort: 60000, DstPort: 80, Ack: 60, Seq: 1, Direction: DirIncoming, Timestamp: time.Unix(4, 0), Payload: []byte("POST / HTTP/1.1\r\nContent-Type: text/plain\r\nContent-Length: 23\r\n\r\n")},
+		{SrcPort: 80, DstPort: 60000, Ack: 1, Seq: 18, Direction: DirOutcoming, Timestamp: time.Unix(2, 0), Payload: []byte("\r\nMozilla\r\n9\r\nDeveloper\r")},
+
+		{SrcPort: 80, DstPort: 60000, Ack: 89, Seq: 1, Direction: DirOutcoming, Timestamp: time.Unix(7, 0), Payload: []byte("HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nContent-Length: 0\r\n\r\n")},
+		{SrcPort: 60000, DstPort: 80, Ack: 60, Seq: 82, Direction: DirIncoming, Timestamp: time.Unix(6, 0), Payload: []byte("Network")},
+	}
+
+	for _, p := range packets {
+		parser.processPacket(p)
+	}
+
+	m := parser.Read()
+
 	if !bytes.HasSuffix(m.Data(), []byte("\n7\r\nNetwork\r\n0\r\n\r\n")) {
 		t.Errorf("expected to %q to have suffix %q", m.Data(), []byte("\n7\r\nNetwork\r\n0\r\n\r\n"))
 	}
 
-	select {
-	case <-time.After(time.Second):
-		t.Errorf("can't parse packets fast enough")
-		return
-	case m = <-mssg:
-	}
-	if len(m.packets) != 7 {
-		t.Errorf("expected to have 7 packets got %d", len(m.packets))
+	m = parser.Read()
+
+	if !bytes.HasSuffix(m.Data(), []byte("Content-Length: 0\r\n\r\n")) {
+		t.Errorf("expected to %q to have suffix %q", m.Data(), []byte("Content-Length: 0\r\n\r\n"))
 	}
+
+	m = parser.Read()
+
 	if !bytes.HasSuffix(m.Data(), []byte("Network")) {
 		t.Errorf("expected to %q to have suffix %q", m.Data(), []byte("Network"))
 	}
-
-	select {
-	case <-time.After(time.Second):
-		t.Errorf("can't parse packets fast enough")
-		return
-	case m = <-mssg:
-	}
-	if len(m.packets) != 6 {
-		t.Errorf("expected to have 6 packets got %d", len(m.packets))
-	}
-	if !bytes.HasSuffix(m.Data(), []byte("Content-Length: 0\r\n\r")) {
-		t.Errorf("expected to %q to have suffix %q", m.Data(), []byte("Content-Length: 0\r\n\r"))
-	}
-
 }
 
 func TestMessageParserWithoutHint(t *testing.T) {
-	var mssg = make(chan *Message, 1)
 	var data [63 << 10]byte
-	packets := GetPackets(1, 10, data[:])
-	packets[0].Data()[14:][20:][13] = 2 // SYN flag
-	packets[9].Data()[14:][20:][13] = 1 // FIN flag
-	p := NewMessagePool(63<<10*10, time.Second, nil, func(m *Message) { mssg <- m })
+	packets := GetPackets(true, 1, 10, data[:])
+
+	p := NewMessageParser(nil, nil, nil, time.Second, false)
 	for _, v := range packets {
-		p.Handler(v)
-	}
-	var m *Message
-	select {
-	case <-time.After(time.Second):
-		t.Errorf("can't parse packets fast enough")
-		return
-	case m = <-mssg:
+		p.processPacket(v)
 	}
+	m := p.Read()
+
 	if m.Length != 63<<10*10 {
 		t.Errorf("expected %d to equal %d", m.Length, 63<<10*10)
 	}
 }
 
-func TestMessageMaxSizeReached(t *testing.T) {
-	var mssg = make(chan *Message, 2)
-	var data [63 << 10]byte
-	packets := GetPackets(1, 2, data[:])
-	packets = append(packets, GetPackets(3, 1, make([]byte, 63<<10+10))...)
-	packets[0].Data()[14:][20:][13] = 2 // SYN flag
-	packets[2].Data()[14:][20:][13] = 2 // SYN flag
-	packets[2].Data()[14:][15] = 3      // changing address
-	p := NewMessagePool(63<<10+10, time.Second, nil, func(m *Message) { mssg <- m })
-	for _, v := range packets {
-		p.Handler(v)
-	}
-	var m *Message
-	select {
-	case <-time.After(time.Second):
-		t.Errorf("can't parse packets fast enough")
-		return
-	case m = <-mssg:
-	}
-	if m.Length != 63<<10+10 {
-		t.Errorf("expected %d to equal %d", m.Length, 63<<10+10)
-	}
-	if !m.Truncated {
-		t.Error("expected message to be truncated")
-	}
-
-	select {
-	case <-time.After(time.Second):
-		t.Errorf("can't parse packets fast enough")
-		return
-	case m = <-mssg:
-	}
-	if m.Length != 63<<10+10 {
-		t.Errorf("expected %d to equal %d", m.Length, 63<<10+10)
-	}
-	if m.Truncated {
-		t.Error("expected message to not be truncated")
-	}
-}
-
 func TestMessageTimeoutReached(t *testing.T) {
-	var mssg = make(chan *Message, 2)
-	var data [63 << 10]byte
-	packets := GetPackets(1, 2, data[:])
-	packets[0].Data()[14:][20:][13] = 2 // SYN flag
-	p := NewMessagePool(1<<20, 0, nil, func(m *Message) { mssg <- m })
-	p.Handler(packets[0])
+	const size = 63 << 11
+	var data [size >> 1]byte
+	packets := GetPackets(true, 1, 2, data[:])
+	p := NewMessageParser(nil, nil, nil, 10*time.Millisecond, true)
+	p.processPacket(packets[0])
 	time.Sleep(time.Millisecond * 200)
-	p.Handler(packets[1])
-	m := <-mssg
-	if m.Length != 63<<10 {
-		t.Errorf("expected %d to equal %d", m.Length, 63<<10)
+	p.processPacket(packets[1])
+	m := p.Read()
+	if m.Length != size {
+		t.Errorf("expected %d to equal %d", m.Length, size)
 	}
 	if !m.TimedOut {
 		t.Error("expected message to be timeout")
 	}
 }
 
-func TestMessageUUID(t *testing.T) {
-	m1 := &Message{}
-	m1.IsIncoming = true
-	m1.SrcAddr = "src"
-	m1.DstAddr = "dst"
-	m2 := &Message{}
-	m2.SrcAddr = "dst"
-	m2.DstAddr = "src"
-	if string(m1.UUID()) != string(m2.UUID()) {
-		t.Errorf("expected %s, to equal %s", m1.UUID(), m2.UUID())
+func BenchmarkMessageUUID(b *testing.B) {
+	packets := GetPackets(true, 1, 5, nil)
+
+	var uuid []byte
+	parser := NewMessageParser(nil, nil, nil, 10*time.Millisecond, true)
+	for _, p := range packets {
+		parser.processPacket(p)
 	}
+
+	msg := parser.Read()
+
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		uuid = msg.UUID()
+	}
+	_ = uuid
 }
 
 func BenchmarkPacketParseAndSort(b *testing.B) {
 	m := new(Message)
 	m.packets = make([]*Packet, 100)
-	for i, v := range GetPackets(1, 100, nil) {
-		m.packets[i], _ = ParsePacket(v)
+	for i, v := range GetPackets(true, 1, 100, nil) {
+		m.packets[i] = v
 	}
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
@@ -216,21 +260,16 @@ func BenchmarkPacketParseAndSort(b *testing.B) {
 }
 
 func BenchmarkMessageParserWithoutHint(b *testing.B) {
-	var mssg = make(chan *Message, 1)
 	var chunk = []byte("111111111111111111111111111111")
-	packets := GetPackets(1, 1000, chunk)
-	packets[0].Data()[14:][20:][13] = 2      // SYN flag
-	packets[1000-1].Data()[14:][20:][13] = 1 // FIN flag
-	p := NewMessagePool(1<<20, time.Second*2, nil, func(m *Message) {
-		mssg <- m
-	})
+	packets := GetPackets(true, 1, 1000, chunk)
+	p := NewMessageParser(nil, nil, nil, 2*time.Second, false)
 	b.ResetTimer()
 	b.ReportMetric(float64(1000), "packets/op")
 	for i := 0; i < b.N; i++ {
 		for _, v := range packets {
-			p.Handler(v)
+			p.processPacket(v)
 		}
-		<-mssg
+		p.Read()
 	}
 }
 
@@ -242,25 +281,33 @@ func BenchmarkMessageParserWithHint(b *testing.B) {
 		buf[i] = chunk
 	}
 	buf[1001] = []byte("0\r\n\r\n")
-	packets := make([]gopacket.Packet, len(buf))
+	packets := make([]*Packet, len(buf))
 	for i := 0; i < len(buf); i++ {
-		packets[i] = GetPackets(uint32(i+10), 1, buf[i])[0]
+		packets[i] = GetPackets(false, 1, 1, buf[i])[0]
 	}
-	var mssg = make(chan *Message, 1)
-	pool := NewMessagePool(1<<30, time.Second*10, nil, func(m *Message) { mssg <- m })
-	pool.Start = func(pckt *Packet) (bool, bool) {
+
+	parser := NewMessageParser(nil, nil, nil, 2*time.Second, false)
+	parser.Start = func(pckt *Packet) (bool, bool) {
 		return false, proto.HasResponseTitle(pckt.Payload)
 	}
-	pool.End = func(m *Message) bool {
-		return proto.HasFullPayload(m.Data())
+	parser.End = func(m *Message) bool {
+		return proto.HasFullPayload(m, m.PacketData()...)
 	}
 	b.ResetTimer()
 	b.ReportMetric(float64(len(packets)), "packets/op")
 	b.ReportMetric(float64(1000), "chunks/op")
 	for i := 0; i < b.N; i++ {
 		for j := range packets {
-			pool.Handler(packets[j])
+			parser.processPacket(packets[j])
 		}
-		<-mssg
+		parser.Read()
+	}
+}
+
+func BenchmarkNewAndParsePacket(b *testing.B) {
+	data := append(generateHeader(true, 1024, 10), make([]byte, 10)...)
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		ParsePacket(data, int(layers.LinkTypeLoop), 4, &gopacket.CaptureInfo{}, true)
 	}
 }

test_input.go

@@ -25,22 +25,21 @@ func NewTestInput() (i *TestInput) {
 	return
 }
 
-func (i *TestInput) Read(data []byte) (int, error) {
+// PluginRead reads message from this plugin
+func (i *TestInput) PluginRead() (*Message, error) {
+	var msg Message
 	select {
 	case buf := <-i.data:
-		var header []byte
-
+		msg.Data = buf
 		if !i.skipHeader {
-			header = payloadHeader(RequestPayload, uuid(), time.Now().UnixNano(), -1)
-			copy(data[0:len(header)], header)
-			copy(data[len(header):], buf)
+			msg.Meta = payloadHeader(RequestPayload, uuid(), time.Now().UnixNano(), -1)
 		} else {
-			copy(data, buf)
+			msg.Meta, msg.Data = payloadMetaWithBody(msg.Data)
 		}
 
-		return len(buf) + len(header), nil
+		return &msg, nil
 	case <-i.stop:
-		return 0, ErrorStopped
+		return nil, ErrorStopped
 	}
 }
 

test_output.go

@@ -1,6 +1,6 @@
 package main
 
-type writeCallback func(data []byte)
+type writeCallback func(*Message)
 
 // TestOutput used in testing to intercept any output into callback
 type TestOutput struct {
@@ -8,19 +8,20 @@ type TestOutput struct {
 }
 
 // NewTestOutput constructor for TestOutput, accepts callback which get called on each incoming Write
-func NewTestOutput(cb writeCallback) (i *TestOutput) {
-	i = new(TestOutput)
+func NewTestOutput(cb writeCallback) PluginWriter {
+	i := new(TestOutput)
 	i.cb = cb
 
-	return
+	return i
 }
 
-func (i *TestOutput) Write(data []byte) (int, error) {
-	i.cb(data)
+// PluginWrite write message to this plugin
+func (i *TestOutput) PluginWrite(msg *Message) (int, error) {
+	i.cb(msg)
 
-	return len(data), nil
+	return len(msg.Data) + len(msg.Meta), nil
 }
 
 func (i *TestOutput) String() string {
-	return "Test Input"
+	return "Test Output"
 }

vendor/github.com/Shopify/sarama/.gitignore

@@ -1,27 +0,0 @@
-# Compiled Object files, Static and Dynamic libs (Shared Objects)
-*.o
-*.a
-*.so
-*.test
-
-# Folders
-_obj
-_test
-.vagrant
-
-# Architecture specific extensions/prefixes
-*.[568vq]
-[568vq].out
-
-*.cgo1.go
-*.cgo2.c
-_cgo_defun.c
-_cgo_gotypes.go
-_cgo_export.*
-
-_testmain.go
-
-*.exe
-
-coverage.txt
-profile.out

vendor/github.com/Shopify/sarama/.golangci.yml

@@ -1,74 +0,0 @@
-run:
-  timeout: 5m
-  deadline: 10m
-
-linters-settings:
-  govet:
-    check-shadowing: false
-  golint:
-    min-confidence: 0
-  gocyclo:
-    min-complexity: 99
-  maligned:
-    suggest-new: true
-  dupl:
-    threshold: 100
-  goconst:
-    min-len: 2
-    min-occurrences: 3
-  misspell:
-    locale: US
-  goimports:
-    local-prefixes: github.com/Shopify/sarama
-  gocritic:
-    enabled-tags:
-      - diagnostic
-      - experimental
-      - opinionated
-      - performance
-      - style
-    disabled-checks:
-      - wrapperFunc
-      - ifElseChain
-  funlen:
-    lines: 300
-    statements: 300
-
-linters:
-  disable-all: true
-  enable:
-    - bodyclose
-    - deadcode
-    - depguard
-    - dogsled
-    # - dupl
-    - errcheck
-    - funlen
-    # - gocritic
-    - gocyclo
-    - gofmt
-    - goimports
-    # - golint
-    - gosec
-    # - gosimple
-    - govet
-    # - ineffassign
-    - interfacer
-    # - misspell
-    # - nakedret
-    # - scopelint
-    # - staticcheck
-    - structcheck
-    # - stylecheck
-    - typecheck
-    - unconvert
-    - unused
-    - varcheck
-    - whitespace
-    # - goconst
-    # - gochecknoinits
-
-issues:
-  exclude:
-    - consider giving a name to these results
-    - include an explanation for nolint directive

vendor/github.com/Shopify/sarama/CHANGELOG.md

@@ -1,912 +0,0 @@
-# Changelog
-
-#### Unreleased
-
-#### Version 1.26.1 (2020-02-04)
-
-Improvements:
-- Add requests-in-flight metric ([1539](https://github.com/Shopify/sarama/pull/1539))
-- Fix misleading example for cluster admin ([1595](https://github.com/Shopify/sarama/pull/1595))
-- Replace Travis with GitHub Actions, linters housekeeping ([1573](https://github.com/Shopify/sarama/pull/1573))
-- Allow BalanceStrategy to provide custom assignment data ([1592](https://github.com/Shopify/sarama/pull/1592))
-
-Bug Fixes:
-- Adds back Consumer.Offsets.CommitInterval to fix API ([1590](https://github.com/Shopify/sarama/pull/1590))
-- Fix error message s/CommitInterval/AutoCommit.Interval ([1589](https://github.com/Shopify/sarama/pull/1589))
-
-#### Version 1.26.0 (2020-01-24)
-
-New Features:
-- Enable zstd compression
-  ([1574](https://github.com/Shopify/sarama/pull/1574),
-  [1582](https://github.com/Shopify/sarama/pull/1582))
-- Support headers in tools kafka-console-producer
-  ([1549](https://github.com/Shopify/sarama/pull/1549))
-
-Improvements:
-- Add SASL AuthIdentity to SASL frames (authzid)
-  ([1585](https://github.com/Shopify/sarama/pull/1585)).
-
-Bug Fixes:
-- Sending messages with ZStd compression enabled fails in multiple ways
-  ([1252](https://github.com/Shopify/sarama/issues/1252)).
-- Use the broker for any admin on BrokerConfig
-  ([1571](https://github.com/Shopify/sarama/pull/1571)).
-- Set DescribeConfigRequest Version field
-  ([1576](https://github.com/Shopify/sarama/pull/1576)).
-- ConsumerGroup flooding logs with client/metadata update req
-  ([1578](https://github.com/Shopify/sarama/pull/1578)).
-- MetadataRequest version in DescribeCluster
-  ([1580](https://github.com/Shopify/sarama/pull/1580)).
-- Fix deadlock in consumer group handleError
-  ([1581](https://github.com/Shopify/sarama/pull/1581))
-- Fill in the Fetch{Request,Response} protocol
-  ([1582](https://github.com/Shopify/sarama/pull/1582)).
-- Retry topic request on ControllerNotAvailable
-  ([1586](https://github.com/Shopify/sarama/pull/1586)).
-
-#### Version 1.25.0 (2020-01-13)
-
-New Features:
-- Support TLS protocol in kafka-producer-performance
-  ([1538](https://github.com/Shopify/sarama/pull/1538)).
-- Add support for kafka 2.4.0
-  ([1552](https://github.com/Shopify/sarama/pull/1552)).
-
-Improvements:
-- Allow the Consumer to disable auto-commit offsets
-  ([1164](https://github.com/Shopify/sarama/pull/1164)).
-- Produce records with consistent timestamps
-  ([1455](https://github.com/Shopify/sarama/pull/1455)).
-
-Bug Fixes:
-- Fix incorrect SetTopicMetadata name mentions
-  ([1534](https://github.com/Shopify/sarama/pull/1534)).
-- Fix client.tryRefreshMetadata Println
-  ([1535](https://github.com/Shopify/sarama/pull/1535)).
-- Fix panic on calling updateMetadata on closed client
-  ([1531](https://github.com/Shopify/sarama/pull/1531)).
-- Fix possible faulty metrics in TestFuncProducing
-  ([1545](https://github.com/Shopify/sarama/pull/1545)).
-
-#### Version 1.24.1 (2019-10-31)
-
-New Features:
-- Add DescribeLogDirs Request/Response pair
-  ([1520](https://github.com/Shopify/sarama/pull/1520)).
-
-Bug Fixes:
-- Fix ClusterAdmin returning invalid controller ID on DescribeCluster
-  ([1518](https://github.com/Shopify/sarama/pull/1518)).
-- Fix issue with consumergroup not rebalancing when new partition is added
-  ([1525](https://github.com/Shopify/sarama/pull/1525)).
-- Ensure consistent use of read/write deadlines
-  ([1529](https://github.com/Shopify/sarama/pull/1529)).
-
-#### Version 1.24.0 (2019-10-09)
-
-New Features:
-- Add sticky partition assignor
-  ([1416](https://github.com/Shopify/sarama/pull/1416)).
-- Switch from cgo zstd package to pure Go implementation
-  ([1477](https://github.com/Shopify/sarama/pull/1477)).
-
-Improvements:
-- Allow creating ClusterAdmin from client
-  ([1415](https://github.com/Shopify/sarama/pull/1415)).
-- Set KafkaVersion in ListAcls method
-  ([1452](https://github.com/Shopify/sarama/pull/1452)).
-- Set request version in CreateACL ClusterAdmin method
-  ([1458](https://github.com/Shopify/sarama/pull/1458)).
-- Set request version in DeleteACL ClusterAdmin method
-  ([1461](https://github.com/Shopify/sarama/pull/1461)).
-- Handle missed error codes on TopicMetaDataRequest and GroupCoordinatorRequest
-  ([1464](https://github.com/Shopify/sarama/pull/1464)).
-- Remove direct usage of gofork
-  ([1465](https://github.com/Shopify/sarama/pull/1465)).
-- Add support for Go 1.13
-  ([1478](https://github.com/Shopify/sarama/pull/1478)).
-- Improve behavior of NewMockListAclsResponse
-  ([1481](https://github.com/Shopify/sarama/pull/1481)).
-
-Bug Fixes:
-- Fix race condition in consumergroup example
-  ([1434](https://github.com/Shopify/sarama/pull/1434)).
-- Fix brokerProducer goroutine leak
-  ([1442](https://github.com/Shopify/sarama/pull/1442)).
-- Use released version of lz4 library
-  ([1469](https://github.com/Shopify/sarama/pull/1469)).
-- Set correct version in MockDeleteTopicsResponse
-  ([1484](https://github.com/Shopify/sarama/pull/1484)).
-- Fix CLI help message typo
-  ([1494](https://github.com/Shopify/sarama/pull/1494)).
-
-Known Issues:
-- Please **don't** use Zstd, as it doesn't work right now.
-  See https://github.com/Shopify/sarama/issues/1252
-
-#### Version 1.23.1 (2019-07-22)
-
-Bug Fixes:
-- Fix fetch delete bug record
-  ([1425](https://github.com/Shopify/sarama/pull/1425)).
-- Handle SASL/OAUTHBEARER token rejection
-  ([1428](https://github.com/Shopify/sarama/pull/1428)).
-
-#### Version 1.23.0 (2019-07-02)
-
-New Features:
-- Add support for Kafka 2.3.0
-  ([1418](https://github.com/Shopify/sarama/pull/1418)).
-- Add support for ListConsumerGroupOffsets v2
-  ([1374](https://github.com/Shopify/sarama/pull/1374)).
-- Add support for DeleteConsumerGroup
-  ([1417](https://github.com/Shopify/sarama/pull/1417)).
-- Add support for SASLVersion configuration
-  ([1410](https://github.com/Shopify/sarama/pull/1410)).
-- Add kerberos support
-  ([1366](https://github.com/Shopify/sarama/pull/1366)).
-
-Improvements:
-- Improve sasl_scram_client example
-  ([1406](https://github.com/Shopify/sarama/pull/1406)).
-- Fix shutdown and race-condition in consumer-group example
-  ([1404](https://github.com/Shopify/sarama/pull/1404)).
-- Add support for error codes 77—81
-  ([1397](https://github.com/Shopify/sarama/pull/1397)).
-- Pool internal objects allocated per message
-  ([1385](https://github.com/Shopify/sarama/pull/1385)).
-- Reduce packet decoder allocations
-  ([1373](https://github.com/Shopify/sarama/pull/1373)).
-- Support timeout when fetching metadata
-  ([1359](https://github.com/Shopify/sarama/pull/1359)).
-
-Bug Fixes:
-- Fix fetch size integer overflow
-  ([1376](https://github.com/Shopify/sarama/pull/1376)).
-- Handle and log throttled FetchResponses
-  ([1383](https://github.com/Shopify/sarama/pull/1383)).
-- Refactor misspelled word Resouce to Resource
-  ([1368](https://github.com/Shopify/sarama/pull/1368)).
-
-#### Version 1.22.1 (2019-04-29)
-
-Improvements:
-- Use zstd 1.3.8
-  ([1350](https://github.com/Shopify/sarama/pull/1350)).
-- Add support for SaslHandshakeRequest v1
-  ([1354](https://github.com/Shopify/sarama/pull/1354)).
-
-Bug Fixes:
-- Fix V5 MetadataRequest nullable topics array
-  ([1353](https://github.com/Shopify/sarama/pull/1353)).
-- Use a different SCRAM client for each broker connection
-  ([1349](https://github.com/Shopify/sarama/pull/1349)).
-- Fix AllowAutoTopicCreation for MetadataRequest greater than v3
-  ([1344](https://github.com/Shopify/sarama/pull/1344)).
-
-#### Version 1.22.0 (2019-04-09)
-
-New Features:
-- Add Offline Replicas Operation to Client
-  ([1318](https://github.com/Shopify/sarama/pull/1318)).
-- Allow using proxy when connecting to broker
-  ([1326](https://github.com/Shopify/sarama/pull/1326)).
-- Implement ReadCommitted
-  ([1307](https://github.com/Shopify/sarama/pull/1307)).
-- Add support for Kafka 2.2.0
-  ([1331](https://github.com/Shopify/sarama/pull/1331)).
-- Add SASL SCRAM-SHA-512 and SCRAM-SHA-256 mechanismes
-  ([1331](https://github.com/Shopify/sarama/pull/1295)).
-
-Improvements:
-- Unregister all broker metrics on broker stop
-  ([1232](https://github.com/Shopify/sarama/pull/1232)).
-- Add SCRAM authentication example
-  ([1303](https://github.com/Shopify/sarama/pull/1303)).
-- Add consumergroup examples
-  ([1304](https://github.com/Shopify/sarama/pull/1304)).
-- Expose consumer batch size metric
-  ([1296](https://github.com/Shopify/sarama/pull/1296)).
-- Add TLS options to console producer and consumer
-  ([1300](https://github.com/Shopify/sarama/pull/1300)).
-- Reduce client close bookkeeping
-  ([1297](https://github.com/Shopify/sarama/pull/1297)).
-- Satisfy error interface in create responses
-  ([1154](https://github.com/Shopify/sarama/pull/1154)).
-- Please lint gods
-  ([1346](https://github.com/Shopify/sarama/pull/1346)).
-
-Bug Fixes:
-- Fix multi consumer group instance crash
-  ([1338](https://github.com/Shopify/sarama/pull/1338)).
-- Update lz4 to latest version
-  ([1347](https://github.com/Shopify/sarama/pull/1347)).
-- Retry ErrNotCoordinatorForConsumer in new consumergroup session
-  ([1231](https://github.com/Shopify/sarama/pull/1231)).
-- Fix cleanup error handler
-  ([1332](https://github.com/Shopify/sarama/pull/1332)).
-- Fix rate condition in PartitionConsumer
-  ([1156](https://github.com/Shopify/sarama/pull/1156)).
-
-#### Version 1.21.0 (2019-02-24)
-
-New Features:
-- Add CreateAclRequest, DescribeAclRequest, DeleteAclRequest
-  ([1236](https://github.com/Shopify/sarama/pull/1236)).
-- Add DescribeTopic, DescribeConsumerGroup, ListConsumerGroups, ListConsumerGroupOffsets admin requests
-  ([1178](https://github.com/Shopify/sarama/pull/1178)).
-- Implement SASL/OAUTHBEARER
-  ([1240](https://github.com/Shopify/sarama/pull/1240)).
-
-Improvements:
-- Add Go mod support
-  ([1282](https://github.com/Shopify/sarama/pull/1282)).
-- Add error codes 73—76
-  ([1239](https://github.com/Shopify/sarama/pull/1239)).
-- Add retry backoff function
-  ([1160](https://github.com/Shopify/sarama/pull/1160)).
-- Maintain metadata in the producer even when retries are disabled
-  ([1189](https://github.com/Shopify/sarama/pull/1189)).
-- Include ReplicaAssignment in ListTopics
-  ([1274](https://github.com/Shopify/sarama/pull/1274)).
-- Add producer performance tool
-  ([1222](https://github.com/Shopify/sarama/pull/1222)).
-- Add support LogAppend timestamps
-  ([1258](https://github.com/Shopify/sarama/pull/1258)).
-
-Bug Fixes:
-- Fix potential deadlock when a heartbeat request fails
-  ([1286](https://github.com/Shopify/sarama/pull/1286)).
-- Fix consuming compacted topic
-  ([1227](https://github.com/Shopify/sarama/pull/1227)).
-- Set correct Kafka version for DescribeConfigsRequest v1
-  ([1277](https://github.com/Shopify/sarama/pull/1277)).
-- Update kafka test version
-  ([1273](https://github.com/Shopify/sarama/pull/1273)).
-
-#### Version 1.20.1 (2019-01-10)
-
-New Features:
-- Add optional replica id in offset request
-  ([1100](https://github.com/Shopify/sarama/pull/1100)).
-
-Improvements:
-- Implement DescribeConfigs Request + Response v1 & v2
-  ([1230](https://github.com/Shopify/sarama/pull/1230)).
-- Reuse compression objects
-  ([1185](https://github.com/Shopify/sarama/pull/1185)).
-- Switch from png to svg for GoDoc link in README
-  ([1243](https://github.com/Shopify/sarama/pull/1243)).
-- Fix typo in deprecation notice for FetchResponseBlock.Records
-  ([1242](https://github.com/Shopify/sarama/pull/1242)).
-- Fix typos in consumer metadata response file
-  ([1244](https://github.com/Shopify/sarama/pull/1244)).
-
-Bug Fixes:
-- Revert to individual msg retries for non-idempotent
-  ([1203](https://github.com/Shopify/sarama/pull/1203)).
-- Respect MaxMessageBytes limit for uncompressed messages
-  ([1141](https://github.com/Shopify/sarama/pull/1141)).
-
-#### Version 1.20.0 (2018-12-10)
-
-New Features:
- - Add support for zstd compression
-   ([#1170](https://github.com/Shopify/sarama/pull/1170)).
- - Add support for Idempotent Producer
-   ([#1152](https://github.com/Shopify/sarama/pull/1152)).
- - Add support support for Kafka 2.1.0
-   ([#1229](https://github.com/Shopify/sarama/pull/1229)).
- - Add support support for OffsetCommit request/response pairs versions v1 to v5
-   ([#1201](https://github.com/Shopify/sarama/pull/1201)).
- - Add support support for OffsetFetch request/response pair up to version v5
-   ([#1198](https://github.com/Shopify/sarama/pull/1198)).
-
-Improvements:
- - Export broker's Rack setting
-   ([#1173](https://github.com/Shopify/sarama/pull/1173)).
- - Always use latest patch version of Go on CI
-   ([#1202](https://github.com/Shopify/sarama/pull/1202)).
- - Add error codes 61 to 72
-   ([#1195](https://github.com/Shopify/sarama/pull/1195)).
-
-Bug Fixes:
- - Fix build without cgo
-   ([#1182](https://github.com/Shopify/sarama/pull/1182)).
- - Fix go vet suggestion in consumer group file
-   ([#1209](https://github.com/Shopify/sarama/pull/1209)).
- - Fix typos in code and comments
-   ([#1228](https://github.com/Shopify/sarama/pull/1228)).
-
-#### Version 1.19.0 (2018-09-27)
-
-New Features:
- - Implement a higher-level consumer group
-   ([#1099](https://github.com/Shopify/sarama/pull/1099)).
-
-Improvements:
- - Add support for Go 1.11
-   ([#1176](https://github.com/Shopify/sarama/pull/1176)).
-
-Bug Fixes:
- - Fix encoding of `MetadataResponse` with version 2 and higher
-   ([#1174](https://github.com/Shopify/sarama/pull/1174)).
- - Fix race condition in mock async producer
-   ([#1174](https://github.com/Shopify/sarama/pull/1174)).
-
-#### Version 1.18.0 (2018-09-07)
-
-New Features:
- - Make `Partitioner.RequiresConsistency` vary per-message
-   ([#1112](https://github.com/Shopify/sarama/pull/1112)).
- - Add customizable partitioner
-   ([#1118](https://github.com/Shopify/sarama/pull/1118)).
- - Add `ClusterAdmin` support for `CreateTopic`, `DeleteTopic`, `CreatePartitions`,
-   `DeleteRecords`, `DescribeConfig`, `AlterConfig`, `CreateACL`, `ListAcls`, `DeleteACL`
-   ([#1055](https://github.com/Shopify/sarama/pull/1055)).
-
-Improvements:
- - Add support for Kafka 2.0.0
-   ([#1149](https://github.com/Shopify/sarama/pull/1149)).
- - Allow setting `LocalAddr` when dialing an address to support multi-homed hosts
-   ([#1123](https://github.com/Shopify/sarama/pull/1123)).
- - Simpler offset management
-   ([#1127](https://github.com/Shopify/sarama/pull/1127)).
-
-Bug Fixes:
- - Fix mutation of `ProducerMessage.MetaData` when producing to Kafka
-   ([#1110](https://github.com/Shopify/sarama/pull/1110)).
- - Fix consumer block when response did not contain all the
-   expected topic/partition blocks
-   ([#1086](https://github.com/Shopify/sarama/pull/1086)).
- - Fix consumer block when response contains only constrol messages
-   ([#1115](https://github.com/Shopify/sarama/pull/1115)).
- - Add timeout config for ClusterAdmin requests
-   ([#1142](https://github.com/Shopify/sarama/pull/1142)).
- - Add version check when producing message with headers
-   ([#1117](https://github.com/Shopify/sarama/pull/1117)).
- - Fix `MetadataRequest` for empty list of topics
-   ([#1132](https://github.com/Shopify/sarama/pull/1132)).
- - Fix producer topic metadata on-demand fetch when topic error happens in metadata response
-   ([#1125](https://github.com/Shopify/sarama/pull/1125)).
-
-#### Version 1.17.0 (2018-05-30)
-
-New Features:
- - Add support for gzip compression levels
-   ([#1044](https://github.com/Shopify/sarama/pull/1044)).
- - Add support for Metadata request/response pairs versions v1 to v5
-   ([#1047](https://github.com/Shopify/sarama/pull/1047),
-    [#1069](https://github.com/Shopify/sarama/pull/1069)).
- - Add versioning to JoinGroup request/response pairs
-   ([#1098](https://github.com/Shopify/sarama/pull/1098))
- - Add support for CreatePartitions, DeleteGroups, DeleteRecords request/response pairs
-   ([#1065](https://github.com/Shopify/sarama/pull/1065),
-    [#1096](https://github.com/Shopify/sarama/pull/1096),
-    [#1027](https://github.com/Shopify/sarama/pull/1027)).
- - Add `Controller()` method to Client interface
-   ([#1063](https://github.com/Shopify/sarama/pull/1063)).
-
-Improvements:
- - ConsumerMetadataReq/Resp has been migrated to FindCoordinatorReq/Resp
-   ([#1010](https://github.com/Shopify/sarama/pull/1010)).
- - Expose missing protocol parts: `msgSet` and `recordBatch`
-   ([#1049](https://github.com/Shopify/sarama/pull/1049)).
- - Add support for v1 DeleteTopics Request
-   ([#1052](https://github.com/Shopify/sarama/pull/1052)).
- - Add support for Go 1.10
-   ([#1064](https://github.com/Shopify/sarama/pull/1064)).
- - Claim support for Kafka 1.1.0
-   ([#1073](https://github.com/Shopify/sarama/pull/1073)).
-
-Bug Fixes:
- - Fix FindCoordinatorResponse.encode to allow nil Coordinator
-   ([#1050](https://github.com/Shopify/sarama/pull/1050),
-    [#1051](https://github.com/Shopify/sarama/pull/1051)).
- - Clear all metadata when we have the latest topic info
-   ([#1033](https://github.com/Shopify/sarama/pull/1033)).
- - Make `PartitionConsumer.Close` idempotent
-   ([#1092](https://github.com/Shopify/sarama/pull/1092)).
-
-#### Version 1.16.0 (2018-02-12)
-
-New Features:
- - Add support for the Create/Delete Topics request/response pairs
-   ([#1007](https://github.com/Shopify/sarama/pull/1007),
-    [#1008](https://github.com/Shopify/sarama/pull/1008)).
- - Add support for the Describe/Create/Delete ACL request/response pairs
-   ([#1009](https://github.com/Shopify/sarama/pull/1009)).
- - Add support for the five transaction-related request/response pairs
-   ([#1016](https://github.com/Shopify/sarama/pull/1016)).
-
-Improvements:
- - Permit setting version on mock producer responses
-   ([#999](https://github.com/Shopify/sarama/pull/999)).
- - Add `NewMockBrokerListener` helper for testing TLS connections
-   ([#1019](https://github.com/Shopify/sarama/pull/1019)).
- - Changed the default value for `Consumer.Fetch.Default` from 32KiB to 1MiB
-   which results in much higher throughput in most cases
-   ([#1024](https://github.com/Shopify/sarama/pull/1024)).
- - Reuse the `time.Ticker` across fetch requests in the PartitionConsumer to
-   reduce CPU and memory usage when processing many partitions
-   ([#1028](https://github.com/Shopify/sarama/pull/1028)).
- - Assign relative offsets to messages in the producer to save the brokers a
-   recompression pass
-   ([#1002](https://github.com/Shopify/sarama/pull/1002),
-    [#1015](https://github.com/Shopify/sarama/pull/1015)).
-
-Bug Fixes:
- - Fix producing uncompressed batches with the new protocol format
-   ([#1032](https://github.com/Shopify/sarama/issues/1032)).
- - Fix consuming compacted topics with the new protocol format
-   ([#1005](https://github.com/Shopify/sarama/issues/1005)).
- - Fix consuming topics with a mix of protocol formats
-   ([#1021](https://github.com/Shopify/sarama/issues/1021)).
- - Fix consuming when the broker includes multiple batches in a single response
-   ([#1022](https://github.com/Shopify/sarama/issues/1022)).
- - Fix detection of `PartialTrailingMessage` when the partial message was
-   truncated before the magic value indicating its version
-   ([#1030](https://github.com/Shopify/sarama/pull/1030)).
- - Fix expectation-checking in the mock of `SyncProducer.SendMessages`
-   ([#1035](https://github.com/Shopify/sarama/pull/1035)).
-
-#### Version 1.15.0 (2017-12-08)
-
-New Features:
- - Claim official support for Kafka 1.0, though it did already work
-   ([#984](https://github.com/Shopify/sarama/pull/984)).
- - Helper methods for Kafka version numbers to/from strings
-   ([#989](https://github.com/Shopify/sarama/pull/989)).
- - Implement CreatePartitions request/response
-   ([#985](https://github.com/Shopify/sarama/pull/985)).
-
-Improvements:
- - Add error codes 45-60
-   ([#986](https://github.com/Shopify/sarama/issues/986)).
-
-Bug Fixes:
- - Fix slow consuming for certain Kafka 0.11/1.0 configurations
-   ([#982](https://github.com/Shopify/sarama/pull/982)).
- - Correctly determine when a FetchResponse contains the new message format
-   ([#990](https://github.com/Shopify/sarama/pull/990)).
- - Fix producing with multiple headers
-   ([#996](https://github.com/Shopify/sarama/pull/996)).
- - Fix handling of truncated record batches
-   ([#998](https://github.com/Shopify/sarama/pull/998)).
- - Fix leaking metrics when closing brokers
-   ([#991](https://github.com/Shopify/sarama/pull/991)).
-
-#### Version 1.14.0 (2017-11-13)
-
-New Features:
- - Add support for the new Kafka 0.11 record-batch format, including the wire
-   protocol and the necessary behavioural changes in the producer and consumer.
-   Transactions and idempotency are not yet supported, but producing and
-   consuming should work with all the existing bells and whistles (batching,
-   compression, etc) as well as the new custom headers. Thanks to Vlad Hanciuta
-   of Arista Networks for this work. Part of
-   ([#901](https://github.com/Shopify/sarama/issues/901)).
-
-Bug Fixes:
- - Fix encoding of ProduceResponse versions in test
-   ([#970](https://github.com/Shopify/sarama/pull/970)).
- - Return partial replicas list when we have it
-   ([#975](https://github.com/Shopify/sarama/pull/975)).
-
-#### Version 1.13.0 (2017-10-04)
-
-New Features:
- - Support for FetchRequest version 3
-   ([#905](https://github.com/Shopify/sarama/pull/905)).
- - Permit setting version on mock FetchResponses
-   ([#939](https://github.com/Shopify/sarama/pull/939)).
- - Add a configuration option to support storing only minimal metadata for
-   extremely large clusters
-   ([#937](https://github.com/Shopify/sarama/pull/937)).
- - Add `PartitionOffsetManager.ResetOffset` for backtracking tracked offsets
-   ([#932](https://github.com/Shopify/sarama/pull/932)).
-
-Improvements:
- - Provide the block-level timestamp when consuming compressed messages
-   ([#885](https://github.com/Shopify/sarama/issues/885)).
- - `Client.Replicas` and `Client.InSyncReplicas` now respect the order returned
-   by the broker, which can be meaningful
-   ([#930](https://github.com/Shopify/sarama/pull/930)).
- - Use a `Ticker` to reduce consumer timer overhead at the cost of higher
-   variance in the actual timeout
-   ([#933](https://github.com/Shopify/sarama/pull/933)).
-
-Bug Fixes:
- - Gracefully handle messages with negative timestamps
-   ([#907](https://github.com/Shopify/sarama/pull/907)).
- - Raise a proper error when encountering an unknown message version
-   ([#940](https://github.com/Shopify/sarama/pull/940)).
-
-#### Version 1.12.0 (2017-05-08)
-
-New Features:
- - Added support for the `ApiVersions` request and response pair, and Kafka
-   version 0.10.2 ([#867](https://github.com/Shopify/sarama/pull/867)). Note
-   that you still need to specify the Kafka version in the Sarama configuration
-   for the time being.
- - Added a `Brokers` method to the Client which returns the complete set of
-   active brokers ([#813](https://github.com/Shopify/sarama/pull/813)).
- - Added an `InSyncReplicas` method to the Client which returns the set of all
-   in-sync broker IDs for the given partition, now that the Kafka versions for
-   which this was misleading are no longer in our supported set
-   ([#872](https://github.com/Shopify/sarama/pull/872)).
- - Added a `NewCustomHashPartitioner` method which allows constructing a hash
-   partitioner with a custom hash method in case the default (FNV-1a) is not
-   suitable
-   ([#837](https://github.com/Shopify/sarama/pull/837),
-    [#841](https://github.com/Shopify/sarama/pull/841)).
-
-Improvements:
- - Recognize more Kafka error codes
-   ([#859](https://github.com/Shopify/sarama/pull/859)).
-
-Bug Fixes:
- - Fix an issue where decoding a malformed FetchRequest would not return the
-   correct error ([#818](https://github.com/Shopify/sarama/pull/818)).
- - Respect ordering of group protocols in JoinGroupRequests. This fix is
-   transparent if you're using the `AddGroupProtocol` or
-   `AddGroupProtocolMetadata` helpers; otherwise you will need to switch from
-   the `GroupProtocols` field (now deprecated) to use `OrderedGroupProtocols`
-   ([#812](https://github.com/Shopify/sarama/issues/812)).
- - Fix an alignment-related issue with atomics on 32-bit architectures
-   ([#859](https://github.com/Shopify/sarama/pull/859)).
-
-#### Version 1.11.0 (2016-12-20)
-
-_Important:_ As of Sarama 1.11 it is necessary to set the config value of
-`Producer.Return.Successes` to true in order to use the SyncProducer. Previous
-versions would silently override this value when instantiating a SyncProducer
-which led to unexpected values and data races.
-
-New Features:
- - Metrics! Thanks to Sébastien Launay for all his work on this feature
-   ([#701](https://github.com/Shopify/sarama/pull/701),
-    [#746](https://github.com/Shopify/sarama/pull/746),
-    [#766](https://github.com/Shopify/sarama/pull/766)).
- - Add support for LZ4 compression
-   ([#786](https://github.com/Shopify/sarama/pull/786)).
- - Add support for ListOffsetRequest v1 and Kafka 0.10.1
-   ([#775](https://github.com/Shopify/sarama/pull/775)).
- - Added a `HighWaterMarks` method to the Consumer which aggregates the
-   `HighWaterMarkOffset` values of its child topic/partitions
-   ([#769](https://github.com/Shopify/sarama/pull/769)).
-
-Bug Fixes:
- - Fixed producing when using timestamps, compression and Kafka 0.10
-   ([#759](https://github.com/Shopify/sarama/pull/759)).
- - Added missing decoder methods to DescribeGroups response
-   ([#756](https://github.com/Shopify/sarama/pull/756)).
- - Fix producer shutdown when `Return.Errors` is disabled
-   ([#787](https://github.com/Shopify/sarama/pull/787)).
- - Don't mutate configuration in SyncProducer
-   ([#790](https://github.com/Shopify/sarama/pull/790)).
- - Fix crash on SASL initialization failure
-   ([#795](https://github.com/Shopify/sarama/pull/795)).
-
-#### Version 1.10.1 (2016-08-30)
-
-Bug Fixes:
- - Fix the documentation for `HashPartitioner` which was incorrect
-   ([#717](https://github.com/Shopify/sarama/pull/717)).
- - Permit client creation even when it is limited by ACLs
-   ([#722](https://github.com/Shopify/sarama/pull/722)).
- - Several fixes to the consumer timer optimization code, regressions introduced
-   in v1.10.0. Go's timers are finicky
-   ([#730](https://github.com/Shopify/sarama/pull/730),
-    [#733](https://github.com/Shopify/sarama/pull/733),
-    [#734](https://github.com/Shopify/sarama/pull/734)).
- - Handle consuming compressed relative offsets with Kafka 0.10
-   ([#735](https://github.com/Shopify/sarama/pull/735)).
-
-#### Version 1.10.0 (2016-08-02)
-
-_Important:_ As of Sarama 1.10 it is necessary to tell Sarama the version of
-Kafka you are running against (via the `config.Version` value) in order to use
-features that may not be compatible with old Kafka versions. If you don't
-specify this value it will default to 0.8.2 (the minimum supported), and trying
-to use more recent features (like the offset manager) will fail with an error.
-
-_Also:_ The offset-manager's behaviour has been changed to match the upstream
-java consumer (see [#705](https://github.com/Shopify/sarama/pull/705) and
-[#713](https://github.com/Shopify/sarama/pull/713)). If you use the
-offset-manager, please ensure that you are committing one *greater* than the
-last consumed message offset or else you may end up consuming duplicate
-messages.
-
-New Features:
- - Support for Kafka 0.10
-   ([#672](https://github.com/Shopify/sarama/pull/672),
-    [#678](https://github.com/Shopify/sarama/pull/678),
-    [#681](https://github.com/Shopify/sarama/pull/681), and others).
- - Support for configuring the target Kafka version
-   ([#676](https://github.com/Shopify/sarama/pull/676)).
- - Batch producing support in the SyncProducer
-   ([#677](https://github.com/Shopify/sarama/pull/677)).
- - Extend producer mock to allow setting expectations on message contents
-   ([#667](https://github.com/Shopify/sarama/pull/667)).
-
-Improvements:
- - Support `nil` compressed messages for deleting in compacted topics
-   ([#634](https://github.com/Shopify/sarama/pull/634)).
- - Pre-allocate decoding errors, greatly reducing heap usage and GC time against
-   misbehaving brokers ([#690](https://github.com/Shopify/sarama/pull/690)).
- - Re-use consumer expiry timers, removing one allocation per consumed message
-   ([#707](https://github.com/Shopify/sarama/pull/707)).
-
-Bug Fixes:
- - Actually default the client ID to "sarama" like we say we do
-   ([#664](https://github.com/Shopify/sarama/pull/664)).
- - Fix a rare issue where `Client.Leader` could return the wrong error
-   ([#685](https://github.com/Shopify/sarama/pull/685)).
- - Fix a possible tight loop in the consumer
-   ([#693](https://github.com/Shopify/sarama/pull/693)).
- - Match upstream's offset-tracking behaviour
-   ([#705](https://github.com/Shopify/sarama/pull/705)).
- - Report UnknownTopicOrPartition errors from the offset manager
-   ([#706](https://github.com/Shopify/sarama/pull/706)).
- - Fix possible negative partition value from the HashPartitioner
-   ([#709](https://github.com/Shopify/sarama/pull/709)).
-
-#### Version 1.9.0 (2016-05-16)
-
-New Features:
- - Add support for custom offset manager retention durations
-   ([#602](https://github.com/Shopify/sarama/pull/602)).
- - Publish low-level mocks to enable testing of third-party producer/consumer
-   implementations ([#570](https://github.com/Shopify/sarama/pull/570)).
- - Declare support for Golang 1.6
-   ([#611](https://github.com/Shopify/sarama/pull/611)).
- - Support for SASL plain-text auth
-   ([#648](https://github.com/Shopify/sarama/pull/648)).
-
-Improvements:
- - Simplified broker locking scheme slightly
-   ([#604](https://github.com/Shopify/sarama/pull/604)).
- - Documentation cleanup
-   ([#605](https://github.com/Shopify/sarama/pull/605),
-    [#621](https://github.com/Shopify/sarama/pull/621),
-    [#654](https://github.com/Shopify/sarama/pull/654)).
-
-Bug Fixes:
- - Fix race condition shutting down the OffsetManager
-   ([#658](https://github.com/Shopify/sarama/pull/658)).
-
-#### Version 1.8.0 (2016-02-01)
-
-New Features:
- - Full support for Kafka 0.9:
-   - All protocol messages and fields
-   ([#586](https://github.com/Shopify/sarama/pull/586),
-   [#588](https://github.com/Shopify/sarama/pull/588),
-   [#590](https://github.com/Shopify/sarama/pull/590)).
-   - Verified that TLS support works
-   ([#581](https://github.com/Shopify/sarama/pull/581)).
-   - Fixed the OffsetManager compatibility
-   ([#585](https://github.com/Shopify/sarama/pull/585)).
-
-Improvements:
- - Optimize for fewer system calls when reading from the network
-   ([#584](https://github.com/Shopify/sarama/pull/584)).
- - Automatically retry `InvalidMessage` errors to match upstream behaviour
-   ([#589](https://github.com/Shopify/sarama/pull/589)).
-
-#### Version 1.7.0 (2015-12-11)
-
-New Features:
- - Preliminary support for Kafka 0.9
-   ([#572](https://github.com/Shopify/sarama/pull/572)). This comes with several
-   caveats:
-   - Protocol-layer support is mostly in place
-     ([#577](https://github.com/Shopify/sarama/pull/577)), however Kafka 0.9
-     renamed some messages and fields, which we did not in order to preserve API
-     compatibility.
-   - The producer and consumer work against 0.9, but the offset manager does
-     not ([#573](https://github.com/Shopify/sarama/pull/573)).
-   - TLS support may or may not work
-     ([#581](https://github.com/Shopify/sarama/pull/581)).
-
-Improvements:
- - Don't wait for request timeouts on dead brokers, greatly speeding recovery
-   when the TCP connection is left hanging
-   ([#548](https://github.com/Shopify/sarama/pull/548)).
- - Refactored part of the producer. The new version provides a much more elegant
-   solution to [#449](https://github.com/Shopify/sarama/pull/449). It is also
-   slightly more efficient, and much more precise in calculating batch sizes
-   when compression is used
-   ([#549](https://github.com/Shopify/sarama/pull/549),
-   [#550](https://github.com/Shopify/sarama/pull/550),
-   [#551](https://github.com/Shopify/sarama/pull/551)).
-
-Bug Fixes:
- - Fix race condition in consumer test mock
-   ([#553](https://github.com/Shopify/sarama/pull/553)).
-
-#### Version 1.6.1 (2015-09-25)
-
-Bug Fixes:
- - Fix panic that could occur if a user-supplied message value failed to encode
-   ([#449](https://github.com/Shopify/sarama/pull/449)).
-
-#### Version 1.6.0 (2015-09-04)
-
-New Features:
- - Implementation of a consumer offset manager using the APIs introduced in
-   Kafka 0.8.2. The API is designed mainly for integration into a future
-   high-level consumer, not for direct use, although it is *possible* to use it
-   directly.
-   ([#461](https://github.com/Shopify/sarama/pull/461)).
-
-Improvements:
- - CRC32 calculation is much faster on machines with SSE4.2 instructions,
-   removing a major hotspot from most profiles
-   ([#255](https://github.com/Shopify/sarama/pull/255)).
-
-Bug Fixes:
- - Make protocol decoding more robust against some malformed packets generated
-   by go-fuzz ([#523](https://github.com/Shopify/sarama/pull/523),
-   [#525](https://github.com/Shopify/sarama/pull/525)) or found in other ways
-   ([#528](https://github.com/Shopify/sarama/pull/528)).
- - Fix a potential race condition panic in the consumer on shutdown
-   ([#529](https://github.com/Shopify/sarama/pull/529)).
-
-#### Version 1.5.0 (2015-08-17)
-
-New Features:
- - TLS-encrypted network connections are now supported. This feature is subject
-   to change when Kafka releases built-in TLS support, but for now this is
-   enough to work with TLS-terminating proxies
-   ([#154](https://github.com/Shopify/sarama/pull/154)).
-
-Improvements:
- - The consumer will not block if a single partition is not drained by the user;
-   all other partitions will continue to consume normally
-   ([#485](https://github.com/Shopify/sarama/pull/485)).
- - Formatting of error strings has been much improved
-   ([#495](https://github.com/Shopify/sarama/pull/495)).
- - Internal refactoring of the producer for code cleanliness and to enable
-   future work ([#300](https://github.com/Shopify/sarama/pull/300)).
-
-Bug Fixes:
- - Fix a potential deadlock in the consumer on shutdown
-   ([#475](https://github.com/Shopify/sarama/pull/475)).
-
-#### Version 1.4.3 (2015-07-21)
-
-Bug Fixes:
- - Don't include the partitioner in the producer's "fetch partitions"
-   circuit-breaker ([#466](https://github.com/Shopify/sarama/pull/466)).
- - Don't retry messages until the broker is closed when abandoning a broker in
-   the producer ([#468](https://github.com/Shopify/sarama/pull/468)).
- - Update the import path for snappy-go, it has moved again and the API has
-   changed slightly ([#486](https://github.com/Shopify/sarama/pull/486)).
-
-#### Version 1.4.2 (2015-05-27)
-
-Bug Fixes:
- - Update the import path for snappy-go, it has moved from google code to github
-   ([#456](https://github.com/Shopify/sarama/pull/456)).
-
-#### Version 1.4.1 (2015-05-25)
-
-Improvements:
- - Optimizations when decoding snappy messages, thanks to John Potocny
-   ([#446](https://github.com/Shopify/sarama/pull/446)).
-
-Bug Fixes:
- - Fix hypothetical race conditions on producer shutdown
-   ([#450](https://github.com/Shopify/sarama/pull/450),
-   [#451](https://github.com/Shopify/sarama/pull/451)).
-
-#### Version 1.4.0 (2015-05-01)
-
-New Features:
- - The consumer now implements `Topics()` and `Partitions()` methods to enable
-   users to dynamically choose what topics/partitions to consume without
-   instantiating a full client
-   ([#431](https://github.com/Shopify/sarama/pull/431)).
- - The partition-consumer now exposes the high water mark offset value returned
-   by the broker via the `HighWaterMarkOffset()` method ([#339](https://github.com/Shopify/sarama/pull/339)).
- - Added a `kafka-console-consumer` tool capable of handling multiple
-   partitions, and deprecated the now-obsolete `kafka-console-partitionConsumer`
-   ([#439](https://github.com/Shopify/sarama/pull/439),
-   [#442](https://github.com/Shopify/sarama/pull/442)).
-
-Improvements:
- - The producer's logging during retry scenarios is more consistent, more
-   useful, and slightly less verbose
-   ([#429](https://github.com/Shopify/sarama/pull/429)).
- - The client now shuffles its initial list of seed brokers in order to prevent
-   thundering herd on the first broker in the list
-   ([#441](https://github.com/Shopify/sarama/pull/441)).
-
-Bug Fixes:
- - The producer now correctly manages its state if retries occur when it is
-   shutting down, fixing several instances of confusing behaviour and at least
-   one potential deadlock ([#419](https://github.com/Shopify/sarama/pull/419)).
- - The consumer now handles messages for different partitions asynchronously,
-   making it much more resilient to specific user code ordering
-   ([#325](https://github.com/Shopify/sarama/pull/325)).
-
-#### Version 1.3.0 (2015-04-16)
-
-New Features:
- - The client now tracks consumer group coordinators using
-   ConsumerMetadataRequests similar to how it tracks partition leadership using
-   regular MetadataRequests ([#411](https://github.com/Shopify/sarama/pull/411)).
-   This adds two methods to the client API:
-   - `Coordinator(consumerGroup string) (*Broker, error)`
-   - `RefreshCoordinator(consumerGroup string) error`
-
-Improvements:
- - ConsumerMetadataResponses now automatically create a Broker object out of the
-   ID/address/port combination for the Coordinator; accessing the fields
-   individually has been deprecated
-   ([#413](https://github.com/Shopify/sarama/pull/413)).
- - Much improved handling of `OffsetOutOfRange` errors in the consumer.
-   Consumers will fail to start if the provided offset is out of range
-   ([#418](https://github.com/Shopify/sarama/pull/418))
-   and they will automatically shut down if the offset falls out of range
-   ([#424](https://github.com/Shopify/sarama/pull/424)).
- - Small performance improvement in encoding and decoding protocol messages
-   ([#427](https://github.com/Shopify/sarama/pull/427)).
-
-Bug Fixes:
- - Fix a rare race condition in the client's background metadata refresher if
-   it happens to be activated while the client is being closed
-   ([#422](https://github.com/Shopify/sarama/pull/422)).
-
-#### Version 1.2.0 (2015-04-07)
-
-Improvements:
- - The producer's behaviour when `Flush.Frequency` is set is now more intuitive
-   ([#389](https://github.com/Shopify/sarama/pull/389)).
- - The producer is now somewhat more memory-efficient during and after retrying
-   messages due to an improved queue implementation
-   ([#396](https://github.com/Shopify/sarama/pull/396)).
- - The consumer produces much more useful logging output when leadership
-   changes ([#385](https://github.com/Shopify/sarama/pull/385)).
- - The client's `GetOffset` method will now automatically refresh metadata and
-   retry once in the event of stale information or similar
-   ([#394](https://github.com/Shopify/sarama/pull/394)).
- - Broker connections now have support for using TCP keepalives
-   ([#407](https://github.com/Shopify/sarama/issues/407)).
-
-Bug Fixes:
- - The OffsetCommitRequest message now correctly implements all three possible
-   API versions ([#390](https://github.com/Shopify/sarama/pull/390),
-   [#400](https://github.com/Shopify/sarama/pull/400)).
-
-#### Version 1.1.0 (2015-03-20)
-
-Improvements:
- - Wrap the producer's partitioner call in a circuit-breaker so that repeatedly
-   broken topics don't choke throughput
-   ([#373](https://github.com/Shopify/sarama/pull/373)).
-
-Bug Fixes:
- - Fix the producer's internal reference counting in certain unusual scenarios
-   ([#367](https://github.com/Shopify/sarama/pull/367)).
- - Fix the consumer's internal reference counting in certain unusual scenarios
-   ([#369](https://github.com/Shopify/sarama/pull/369)).
- - Fix a condition where the producer's internal control messages could have
-   gotten stuck ([#368](https://github.com/Shopify/sarama/pull/368)).
- - Fix an issue where invalid partition lists would be cached when asking for
-   metadata for a non-existant topic ([#372](https://github.com/Shopify/sarama/pull/372)).
-
-
-#### Version 1.0.0 (2015-03-17)
-
-Version 1.0.0 is the first tagged version, and is almost a complete rewrite. The primary differences with previous untagged versions are:
-
-- The producer has been rewritten; there is now a `SyncProducer` with a blocking API, and an `AsyncProducer` that is non-blocking.
-- The consumer has been rewritten to only open one connection per broker instead of one connection per partition.
-- The main types of Sarama are now interfaces to make depedency injection easy; mock implementations for `Consumer`, `SyncProducer` and `AsyncProducer` are provided in the `github.com/Shopify/sarama/mocks` package.
-- For most uses cases, it is no longer necessary to open a `Client`; this will be done for you.
-- All the configuration values have been unified in the `Config` struct.
-- Much improved test suite.

vendor/github.com/Shopify/sarama/LICENSE

@@ -1,20 +0,0 @@
-Copyright (c) 2013 Shopify
-
-Permission is hereby granted, free of charge, to any person obtaining
-a copy of this software and associated documentation files (the
-"Software"), to deal in the Software without restriction, including
-without limitation the rights to use, copy, modify, merge, publish,
-distribute, sublicense, and/or sell copies of the Software, and to
-permit persons to whom the Software is furnished to do so, subject to
-the following conditions:
-
-The above copyright notice and this permission notice shall be
-included in all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
-NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
-LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
-OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
-WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

vendor/github.com/Shopify/sarama/Makefile

@@ -1,27 +0,0 @@
-default: fmt get update test lint
-
-GO       := GO111MODULE=on GOPRIVATE=github.com/linkedin GOSUMDB=off go
-GOBUILD  := CGO_ENABLED=0 $(GO) build $(BUILD_FLAG)
-GOTEST   := $(GO) test -gcflags='-l' -p 3 -v -race -timeout 6m -coverprofile=profile.out -covermode=atomic
-
-FILES    := $(shell find . -name '*.go' -type f -not -name '*.pb.go' -not -name '*_generated.go' -not -name '*_test.go')
-TESTS    := $(shell find . -name '*.go' -type f -not -name '*.pb.go' -not -name '*_generated.go' -name '*_test.go')
-
-get:
-	$(GO) get ./...
-	$(GO) mod verify
-	$(GO) mod tidy
-
-update:
-	$(GO) get -u -v all
-	$(GO) mod verify
-	$(GO) mod tidy
-
-fmt:
-	gofmt -s -l -w $(FILES) $(TESTS)
-
-lint:
-	golangci-lint run
-
-test:
-	$(GOTEST) ./...

vendor/github.com/Shopify/sarama/README.md

@@ -1,36 +0,0 @@
-# sarama
-
-[![GoDoc](https://godoc.org/github.com/Shopify/sarama?status.svg)](https://godoc.org/github.com/Shopify/sarama)
-[![Build Status](https://travis-ci.org/Shopify/sarama.svg?branch=master)](https://travis-ci.org/Shopify/sarama)
-[![Coverage](https://codecov.io/gh/Shopify/sarama/branch/master/graph/badge.svg)](https://codecov.io/gh/Shopify/sarama)
-
-Sarama is an MIT-licensed Go client library for [Apache Kafka](https://kafka.apache.org/) version 0.8 (and later).
-
-## Getting started
-
-- API documentation and examples are available via [godoc](https://godoc.org/github.com/Shopify/sarama).
-- Mocks for testing are available in the [mocks](./mocks) subpackage.
-- The [examples](./examples) directory contains more elaborate example applications.
-- The [tools](./tools) directory contains command line tools that can be useful for testing, diagnostics, and instrumentation.
-
-You might also want to look at the [Frequently Asked Questions](https://github.com/Shopify/sarama/wiki/Frequently-Asked-Questions).
-
-## Compatibility and API stability
-
-Sarama provides a "2 releases + 2 months" compatibility guarantee: we support
-the two latest stable releases of Kafka and Go, and we provide a two month
-grace period for older releases. This means we currently officially support
-Go 1.12 through 1.14, and Kafka 2.1 through 2.4, although older releases are
-still likely to work.
-
-Sarama follows semantic versioning and provides API stability via the gopkg.in service.
-You can import a version with a guaranteed stable API via http://gopkg.in/Shopify/sarama.v1.
-A changelog is available [here](CHANGELOG.md).
-
-## Contributing
-
-- Get started by checking our [contribution guidelines](https://github.com/Shopify/sarama/blob/master/.github/CONTRIBUTING.md).
-- Read the [Sarama wiki](https://github.com/Shopify/sarama/wiki) for more technical and design details.
-- The [Kafka Protocol Specification](https://cwiki.apache.org/confluence/display/KAFKA/A+Guide+To+The+Kafka+Protocol) contains a wealth of useful information.
-- For more general issues, there is [a google group](https://groups.google.com/forum/#!forum/kafka-clients) for Kafka client developers.
-- If you have any questions, just ask!

vendor/github.com/Shopify/sarama/Vagrantfile

@@ -1,14 +0,0 @@
-# We have 5 * 192MB ZK processes and 5 * 320MB Kafka processes => 2560MB
-MEMORY = 3072
-
-Vagrant.configure("2") do |config|
-  config.vm.box = "ubuntu/bionic64"
-
-  config.vm.provision :shell, path: "vagrant/provision.sh"
-
-  config.vm.network "private_network", ip: "192.168.100.67"
-
-  config.vm.provider "virtualbox" do |v|
-    v.memory = MEMORY
-  end
-end

vendor/github.com/Shopify/sarama/acl_bindings.go

@@ -1,138 +0,0 @@
-package sarama
-
-//Resource holds information about acl resource type
-type Resource struct {
-	ResourceType        AclResourceType
-	ResourceName        string
-	ResourcePatternType AclResourcePatternType
-}
-
-func (r *Resource) encode(pe packetEncoder, version int16) error {
-	pe.putInt8(int8(r.ResourceType))
-
-	if err := pe.putString(r.ResourceName); err != nil {
-		return err
-	}
-
-	if version == 1 {
-		if r.ResourcePatternType == AclPatternUnknown {
-			Logger.Print("Cannot encode an unknown resource pattern type, using Literal instead")
-			r.ResourcePatternType = AclPatternLiteral
-		}
-		pe.putInt8(int8(r.ResourcePatternType))
-	}
-
-	return nil
-}
-
-func (r *Resource) decode(pd packetDecoder, version int16) (err error) {
-	resourceType, err := pd.getInt8()
-	if err != nil {
-		return err
-	}
-	r.ResourceType = AclResourceType(resourceType)
-
-	if r.ResourceName, err = pd.getString(); err != nil {
-		return err
-	}
-	if version == 1 {
-		pattern, err := pd.getInt8()
-		if err != nil {
-			return err
-		}
-		r.ResourcePatternType = AclResourcePatternType(pattern)
-	}
-
-	return nil
-}
-
-//Acl holds information about acl type
-type Acl struct {
-	Principal      string
-	Host           string
-	Operation      AclOperation
-	PermissionType AclPermissionType
-}
-
-func (a *Acl) encode(pe packetEncoder) error {
-	if err := pe.putString(a.Principal); err != nil {
-		return err
-	}
-
-	if err := pe.putString(a.Host); err != nil {
-		return err
-	}
-
-	pe.putInt8(int8(a.Operation))
-	pe.putInt8(int8(a.PermissionType))
-
-	return nil
-}
-
-func (a *Acl) decode(pd packetDecoder, version int16) (err error) {
-	if a.Principal, err = pd.getString(); err != nil {
-		return err
-	}
-
-	if a.Host, err = pd.getString(); err != nil {
-		return err
-	}
-
-	operation, err := pd.getInt8()
-	if err != nil {
-		return err
-	}
-	a.Operation = AclOperation(operation)
-
-	permissionType, err := pd.getInt8()
-	if err != nil {
-		return err
-	}
-	a.PermissionType = AclPermissionType(permissionType)
-
-	return nil
-}
-
-//ResourceAcls is an acl resource type
-type ResourceAcls struct {
-	Resource
-	Acls []*Acl
-}
-
-func (r *ResourceAcls) encode(pe packetEncoder, version int16) error {
-	if err := r.Resource.encode(pe, version); err != nil {
-		return err
-	}
-
-	if err := pe.putArrayLength(len(r.Acls)); err != nil {
-		return err
-	}
-	for _, acl := range r.Acls {
-		if err := acl.encode(pe); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-func (r *ResourceAcls) decode(pd packetDecoder, version int16) error {
-	if err := r.Resource.decode(pd, version); err != nil {
-		return err
-	}
-
-	n, err := pd.getArrayLength()
-	if err != nil {
-		return err
-	}
-
-	r.Acls = make([]*Acl, n)
-	for i := 0; i < n; i++ {
-		r.Acls[i] = new(Acl)
-		if err := r.Acls[i].decode(pd, version); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}

vendor/github.com/Shopify/sarama/acl_create_request.go

@@ -1,89 +0,0 @@
-package sarama
-
-//CreateAclsRequest is an acl creation request
-type CreateAclsRequest struct {
-	Version      int16
-	AclCreations []*AclCreation
-}
-
-func (c *CreateAclsRequest) encode(pe packetEncoder) error {
-	if err := pe.putArrayLength(len(c.AclCreations)); err != nil {
-		return err
-	}
-
-	for _, aclCreation := range c.AclCreations {
-		if err := aclCreation.encode(pe, c.Version); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-func (c *CreateAclsRequest) decode(pd packetDecoder, version int16) (err error) {
-	c.Version = version
-	n, err := pd.getArrayLength()
-	if err != nil {
-		return err
-	}
-
-	c.AclCreations = make([]*AclCreation, n)
-
-	for i := 0; i < n; i++ {
-		c.AclCreations[i] = new(AclCreation)
-		if err := c.AclCreations[i].decode(pd, version); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-func (c *CreateAclsRequest) key() int16 {
-	return 30
-}
-
-func (c *CreateAclsRequest) version() int16 {
-	return c.Version
-}
-
-func (c *CreateAclsRequest) headerVersion() int16 {
-	return 1
-}
-
-func (c *CreateAclsRequest) requiredVersion() KafkaVersion {
-	switch c.Version {
-	case 1:
-		return V2_0_0_0
-	default:
-		return V0_11_0_0
-	}
-}
-
-//AclCreation is a wrapper around Resource and Acl type
-type AclCreation struct {
-	Resource
-	Acl
-}
-
-func (a *AclCreation) encode(pe packetEncoder, version int16) error {
-	if err := a.Resource.encode(pe, version); err != nil {
-		return err
-	}
-	if err := a.Acl.encode(pe); err != nil {
-		return err
-	}
-
-	return nil
-}
-
-func (a *AclCreation) decode(pd packetDecoder, version int16) (err error) {
-	if err := a.Resource.decode(pd, version); err != nil {
-		return err
-	}
-	if err := a.Acl.decode(pd, version); err != nil {
-		return err
-	}
-
-	return nil
-}

vendor/github.com/Shopify/sarama/acl_create_response.go

@@ -1,94 +0,0 @@
-package sarama
-
-import "time"
-
-//CreateAclsResponse is a an acl response creation type
-type CreateAclsResponse struct {
-	ThrottleTime         time.Duration
-	AclCreationResponses []*AclCreationResponse
-}
-
-func (c *CreateAclsResponse) encode(pe packetEncoder) error {
-	pe.putInt32(int32(c.ThrottleTime / time.Millisecond))
-
-	if err := pe.putArrayLength(len(c.AclCreationResponses)); err != nil {
-		return err
-	}
-
-	for _, aclCreationResponse := range c.AclCreationResponses {
-		if err := aclCreationResponse.encode(pe); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-func (c *CreateAclsResponse) decode(pd packetDecoder, version int16) (err error) {
-	throttleTime, err := pd.getInt32()
-	if err != nil {
-		return err
-	}
-	c.ThrottleTime = time.Duration(throttleTime) * time.Millisecond
-
-	n, err := pd.getArrayLength()
-	if err != nil {
-		return err
-	}
-
-	c.AclCreationResponses = make([]*AclCreationResponse, n)
-	for i := 0; i < n; i++ {
-		c.AclCreationResponses[i] = new(AclCreationResponse)
-		if err := c.AclCreationResponses[i].decode(pd, version); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-func (c *CreateAclsResponse) key() int16 {
-	return 30
-}
-
-func (c *CreateAclsResponse) version() int16 {
-	return 0
-}
-
-func (c *CreateAclsResponse) headerVersion() int16 {
-	return 0
-}
-
-func (c *CreateAclsResponse) requiredVersion() KafkaVersion {
-	return V0_11_0_0
-}
-
-//AclCreationResponse is an acl creation response type
-type AclCreationResponse struct {
-	Err    KError
-	ErrMsg *string
-}
-
-func (a *AclCreationResponse) encode(pe packetEncoder) error {
-	pe.putInt16(int16(a.Err))
-
-	if err := pe.putNullableString(a.ErrMsg); err != nil {
-		return err
-	}
-
-	return nil
-}
-
-func (a *AclCreationResponse) decode(pd packetDecoder, version int16) (err error) {
-	kerr, err := pd.getInt16()
-	if err != nil {
-		return err
-	}
-	a.Err = KError(kerr)
-
-	if a.ErrMsg, err = pd.getNullableString(); err != nil {
-		return err
-	}
-
-	return nil
-}

vendor/github.com/Shopify/sarama/acl_delete_request.go

@@ -1,62 +0,0 @@
-package sarama
-
-//DeleteAclsRequest is a delete acl request
-type DeleteAclsRequest struct {
-	Version int
-	Filters []*AclFilter
-}
-
-func (d *DeleteAclsRequest) encode(pe packetEncoder) error {
-	if err := pe.putArrayLength(len(d.Filters)); err != nil {
-		return err
-	}
-
-	for _, filter := range d.Filters {
-		filter.Version = d.Version
-		if err := filter.encode(pe); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-func (d *DeleteAclsRequest) decode(pd packetDecoder, version int16) (err error) {
-	d.Version = int(version)
-	n, err := pd.getArrayLength()
-	if err != nil {
-		return err
-	}
-
-	d.Filters = make([]*AclFilter, n)
-	for i := 0; i < n; i++ {
-		d.Filters[i] = new(AclFilter)
-		d.Filters[i].Version = int(version)
-		if err := d.Filters[i].decode(pd, version); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-func (d *DeleteAclsRequest) key() int16 {
-	return 31
-}
-
-func (d *DeleteAclsRequest) version() int16 {
-	return int16(d.Version)
-}
-
-func (c *DeleteAclsRequest) headerVersion() int16 {
-	return 1
-}
-
-func (d *DeleteAclsRequest) requiredVersion() KafkaVersion {
-	switch d.Version {
-	case 1:
-		return V2_0_0_0
-	default:
-		return V0_11_0_0
-	}
-}

vendor/github.com/Shopify/sarama/acl_delete_response.go

@@ -1,163 +0,0 @@
-package sarama
-
-import "time"
-
-//DeleteAclsResponse is a delete acl response
-type DeleteAclsResponse struct {
-	Version         int16
-	ThrottleTime    time.Duration
-	FilterResponses []*FilterResponse
-}
-
-func (d *DeleteAclsResponse) encode(pe packetEncoder) error {
-	pe.putInt32(int32(d.ThrottleTime / time.Millisecond))
-
-	if err := pe.putArrayLength(len(d.FilterResponses)); err != nil {
-		return err
-	}
-
-	for _, filterResponse := range d.FilterResponses {
-		if err := filterResponse.encode(pe, d.Version); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-func (d *DeleteAclsResponse) decode(pd packetDecoder, version int16) (err error) {
-	throttleTime, err := pd.getInt32()
-	if err != nil {
-		return err
-	}
-	d.ThrottleTime = time.Duration(throttleTime) * time.Millisecond
-
-	n, err := pd.getArrayLength()
-	if err != nil {
-		return err
-	}
-	d.FilterResponses = make([]*FilterResponse, n)
-
-	for i := 0; i < n; i++ {
-		d.FilterResponses[i] = new(FilterResponse)
-		if err := d.FilterResponses[i].decode(pd, version); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-func (d *DeleteAclsResponse) key() int16 {
-	return 31
-}
-
-func (d *DeleteAclsResponse) version() int16 {
-	return d.Version
-}
-
-func (d *DeleteAclsResponse) headerVersion() int16 {
-	return 0
-}
-
-func (d *DeleteAclsResponse) requiredVersion() KafkaVersion {
-	return V0_11_0_0
-}
-
-//FilterResponse is a filter response type
-type FilterResponse struct {
-	Err          KError
-	ErrMsg       *string
-	MatchingAcls []*MatchingAcl
-}
-
-func (f *FilterResponse) encode(pe packetEncoder, version int16) error {
-	pe.putInt16(int16(f.Err))
-	if err := pe.putNullableString(f.ErrMsg); err != nil {
-		return err
-	}
-
-	if err := pe.putArrayLength(len(f.MatchingAcls)); err != nil {
-		return err
-	}
-	for _, matchingAcl := range f.MatchingAcls {
-		if err := matchingAcl.encode(pe, version); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-func (f *FilterResponse) decode(pd packetDecoder, version int16) (err error) {
-	kerr, err := pd.getInt16()
-	if err != nil {
-		return err
-	}
-	f.Err = KError(kerr)
-
-	if f.ErrMsg, err = pd.getNullableString(); err != nil {
-		return err
-	}
-
-	n, err := pd.getArrayLength()
-	if err != nil {
-		return err
-	}
-	f.MatchingAcls = make([]*MatchingAcl, n)
-	for i := 0; i < n; i++ {
-		f.MatchingAcls[i] = new(MatchingAcl)
-		if err := f.MatchingAcls[i].decode(pd, version); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-//MatchingAcl is a matching acl type
-type MatchingAcl struct {
-	Err    KError
-	ErrMsg *string
-	Resource
-	Acl
-}
-
-func (m *MatchingAcl) encode(pe packetEncoder, version int16) error {
-	pe.putInt16(int16(m.Err))
-	if err := pe.putNullableString(m.ErrMsg); err != nil {
-		return err
-	}
-
-	if err := m.Resource.encode(pe, version); err != nil {
-		return err
-	}
-
-	if err := m.Acl.encode(pe); err != nil {
-		return err
-	}
-
-	return nil
-}
-
-func (m *MatchingAcl) decode(pd packetDecoder, version int16) (err error) {
-	kerr, err := pd.getInt16()
-	if err != nil {
-		return err
-	}
-	m.Err = KError(kerr)
-
-	if m.ErrMsg, err = pd.getNullableString(); err != nil {
-		return err
-	}
-
-	if err := m.Resource.decode(pd, version); err != nil {
-		return err
-	}
-
-	if err := m.Acl.decode(pd, version); err != nil {
-		return err
-	}
-
-	return nil
-}

vendor/github.com/Shopify/sarama/acl_describe_request.go

@@ -1,39 +0,0 @@
-package sarama
-
-//DescribeAclsRequest is a secribe acl request type
-type DescribeAclsRequest struct {
-	Version int
-	AclFilter
-}
-
-func (d *DescribeAclsRequest) encode(pe packetEncoder) error {
-	d.AclFilter.Version = d.Version
-	return d.AclFilter.encode(pe)
-}
-
-func (d *DescribeAclsRequest) decode(pd packetDecoder, version int16) (err error) {
-	d.Version = int(version)
-	d.AclFilter.Version = int(version)
-	return d.AclFilter.decode(pd, version)
-}
-
-func (d *DescribeAclsRequest) key() int16 {
-	return 29
-}
-
-func (d *DescribeAclsRequest) version() int16 {
-	return int16(d.Version)
-}
-
-func (d *DescribeAclsRequest) headerVersion() int16 {
-	return 1
-}
-
-func (d *DescribeAclsRequest) requiredVersion() KafkaVersion {
-	switch d.Version {
-	case 1:
-		return V2_0_0_0
-	default:
-		return V0_11_0_0
-	}
-}

vendor/github.com/Shopify/sarama/acl_describe_response.go

@@ -1,91 +0,0 @@
-package sarama
-
-import "time"
-
-//DescribeAclsResponse is a describe acl response type
-type DescribeAclsResponse struct {
-	Version      int16
-	ThrottleTime time.Duration
-	Err          KError
-	ErrMsg       *string
-	ResourceAcls []*ResourceAcls
-}
-
-func (d *DescribeAclsResponse) encode(pe packetEncoder) error {
-	pe.putInt32(int32(d.ThrottleTime / time.Millisecond))
-	pe.putInt16(int16(d.Err))
-
-	if err := pe.putNullableString(d.ErrMsg); err != nil {
-		return err
-	}
-
-	if err := pe.putArrayLength(len(d.ResourceAcls)); err != nil {
-		return err
-	}
-
-	for _, resourceAcl := range d.ResourceAcls {
-		if err := resourceAcl.encode(pe, d.Version); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-func (d *DescribeAclsResponse) decode(pd packetDecoder, version int16) (err error) {
-	throttleTime, err := pd.getInt32()
-	if err != nil {
-		return err
-	}
-	d.ThrottleTime = time.Duration(throttleTime) * time.Millisecond
-
-	kerr, err := pd.getInt16()
-	if err != nil {
-		return err
-	}
-	d.Err = KError(kerr)
-
-	errmsg, err := pd.getString()
-	if err != nil {
-		return err
-	}
-	if errmsg != "" {
-		d.ErrMsg = &errmsg
-	}
-
-	n, err := pd.getArrayLength()
-	if err != nil {
-		return err
-	}
-	d.ResourceAcls = make([]*ResourceAcls, n)
-
-	for i := 0; i < n; i++ {
-		d.ResourceAcls[i] = new(ResourceAcls)
-		if err := d.ResourceAcls[i].decode(pd, version); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-func (d *DescribeAclsResponse) key() int16 {
-	return 29
-}
-
-func (d *DescribeAclsResponse) version() int16 {
-	return d.Version
-}
-
-func (d *DescribeAclsResponse) headerVersion() int16 {
-	return 0
-}
-
-func (d *DescribeAclsResponse) requiredVersion() KafkaVersion {
-	switch d.Version {
-	case 1:
-		return V2_0_0_0
-	default:
-		return V0_11_0_0
-	}
-}

vendor/github.com/Shopify/sarama/acl_filter.go

@@ -1,78 +0,0 @@
-package sarama
-
-type AclFilter struct {
-	Version                   int
-	ResourceType              AclResourceType
-	ResourceName              *string
-	ResourcePatternTypeFilter AclResourcePatternType
-	Principal                 *string
-	Host                      *string
-	Operation                 AclOperation
-	PermissionType            AclPermissionType
-}
-
-func (a *AclFilter) encode(pe packetEncoder) error {
-	pe.putInt8(int8(a.ResourceType))
-	if err := pe.putNullableString(a.ResourceName); err != nil {
-		return err
-	}
-
-	if a.Version == 1 {
-		pe.putInt8(int8(a.ResourcePatternTypeFilter))
-	}
-
-	if err := pe.putNullableString(a.Principal); err != nil {
-		return err
-	}
-	if err := pe.putNullableString(a.Host); err != nil {
-		return err
-	}
-	pe.putInt8(int8(a.Operation))
-	pe.putInt8(int8(a.PermissionType))
-
-	return nil
-}
-
-func (a *AclFilter) decode(pd packetDecoder, version int16) (err error) {
-	resourceType, err := pd.getInt8()
-	if err != nil {
-		return err
-	}
-	a.ResourceType = AclResourceType(resourceType)
-
-	if a.ResourceName, err = pd.getNullableString(); err != nil {
-		return err
-	}
-
-	if a.Version == 1 {
-		pattern, err := pd.getInt8()
-
-		if err != nil {
-			return err
-		}
-
-		a.ResourcePatternTypeFilter = AclResourcePatternType(pattern)
-	}
-
-	if a.Principal, err = pd.getNullableString(); err != nil {
-		return err
-	}
-
-	if a.Host, err = pd.getNullableString(); err != nil {
-		return err
-	}
-
-	operation, err := pd.getInt8()
-	if err != nil {
-		return err
-	}
-	a.Operation = AclOperation(operation)
-
-	permissionType, err := pd.getInt8()
-	if err != nil {
-		return err
-	}
-	a.PermissionType = AclPermissionType(permissionType)
-
-	return nil
-}

vendor/github.com/Shopify/sarama/acl_types.go

@@ -1,55 +0,0 @@
-package sarama
-
-type (
-	AclOperation int
-
-	AclPermissionType int
-
-	AclResourceType int
-
-	AclResourcePatternType int
-)
-
-// ref: https://github.com/apache/kafka/blob/trunk/clients/src/main/java/org/apache/kafka/common/acl/AclOperation.java
-const (
-	AclOperationUnknown AclOperation = iota
-	AclOperationAny
-	AclOperationAll
-	AclOperationRead
-	AclOperationWrite
-	AclOperationCreate
-	AclOperationDelete
-	AclOperationAlter
-	AclOperationDescribe
-	AclOperationClusterAction
-	AclOperationDescribeConfigs
-	AclOperationAlterConfigs
-	AclOperationIdempotentWrite
-)
-
-// ref: https://github.com/apache/kafka/blob/trunk/clients/src/main/java/org/apache/kafka/common/acl/AclPermissionType.java
-const (
-	AclPermissionUnknown AclPermissionType = iota
-	AclPermissionAny
-	AclPermissionDeny
-	AclPermissionAllow
-)
-
-// ref: https://github.com/apache/kafka/blob/trunk/clients/src/main/java/org/apache/kafka/common/resource/ResourceType.java
-const (
-	AclResourceUnknown AclResourceType = iota
-	AclResourceAny
-	AclResourceTopic
-	AclResourceGroup
-	AclResourceCluster
-	AclResourceTransactionalID
-)
-
-// ref: https://github.com/apache/kafka/blob/trunk/clients/src/main/java/org/apache/kafka/common/resource/PatternType.java
-const (
-	AclPatternUnknown AclResourcePatternType = iota
-	AclPatternAny
-	AclPatternMatch
-	AclPatternLiteral
-	AclPatternPrefixed
-)

vendor/github.com/Shopify/sarama/add_offsets_to_txn_request.go

@@ -1,57 +0,0 @@
-package sarama
-
-//AddOffsetsToTxnRequest adds offsets to a transaction request
-type AddOffsetsToTxnRequest struct {
-	TransactionalID string
-	ProducerID      int64
-	ProducerEpoch   int16
-	GroupID         string
-}
-
-func (a *AddOffsetsToTxnRequest) encode(pe packetEncoder) error {
-	if err := pe.putString(a.TransactionalID); err != nil {
-		return err
-	}
-
-	pe.putInt64(a.ProducerID)
-
-	pe.putInt16(a.ProducerEpoch)
-
-	if err := pe.putString(a.GroupID); err != nil {
-		return err
-	}
-
-	return nil
-}
-
-func (a *AddOffsetsToTxnRequest) decode(pd packetDecoder, version int16) (err error) {
-	if a.TransactionalID, err = pd.getString(); err != nil {
-		return err
-	}
-	if a.ProducerID, err = pd.getInt64(); err != nil {
-		return err
-	}
-	if a.ProducerEpoch, err = pd.getInt16(); err != nil {
-		return err
-	}
-	if a.GroupID, err = pd.getString(); err != nil {
-		return err
-	}
-	return nil
-}
-
-func (a *AddOffsetsToTxnRequest) key() int16 {
-	return 25
-}
-
-func (a *AddOffsetsToTxnRequest) version() int16 {
-	return 0
-}
-
-func (a *AddOffsetsToTxnRequest) headerVersion() int16 {
-	return 1
-}
-
-func (a *AddOffsetsToTxnRequest) requiredVersion() KafkaVersion {
-	return V0_11_0_0
-}

vendor/github.com/Shopify/sarama/add_offsets_to_txn_response.go

@@ -1,49 +0,0 @@
-package sarama
-
-import (
-	"time"
-)
-
-//AddOffsetsToTxnResponse is a response type for adding offsets to txns
-type AddOffsetsToTxnResponse struct {
-	ThrottleTime time.Duration
-	Err          KError
-}
-
-func (a *AddOffsetsToTxnResponse) encode(pe packetEncoder) error {
-	pe.putInt32(int32(a.ThrottleTime / time.Millisecond))
-	pe.putInt16(int16(a.Err))
-	return nil
-}
-
-func (a *AddOffsetsToTxnResponse) decode(pd packetDecoder, version int16) (err error) {
-	throttleTime, err := pd.getInt32()
-	if err != nil {
-		return err
-	}
-	a.ThrottleTime = time.Duration(throttleTime) * time.Millisecond
-
-	kerr, err := pd.getInt16()
-	if err != nil {
-		return err
-	}
-	a.Err = KError(kerr)
-
-	return nil
-}
-
-func (a *AddOffsetsToTxnResponse) key() int16 {
-	return 25
-}
-
-func (a *AddOffsetsToTxnResponse) version() int16 {
-	return 0
-}
-
-func (a *AddOffsetsToTxnResponse) headerVersion() int16 {
-	return 0
-}
-
-func (a *AddOffsetsToTxnResponse) requiredVersion() KafkaVersion {
-	return V0_11_0_0
-}

vendor/github.com/Shopify/sarama/add_partitions_to_txn_request.go

@@ -1,81 +0,0 @@
-package sarama
-
-//AddPartitionsToTxnRequest is a add paartition request
-type AddPartitionsToTxnRequest struct {
-	TransactionalID string
-	ProducerID      int64
-	ProducerEpoch   int16
-	TopicPartitions map[string][]int32
-}
-
-func (a *AddPartitionsToTxnRequest) encode(pe packetEncoder) error {
-	if err := pe.putString(a.TransactionalID); err != nil {
-		return err
-	}
-	pe.putInt64(a.ProducerID)
-	pe.putInt16(a.ProducerEpoch)
-
-	if err := pe.putArrayLength(len(a.TopicPartitions)); err != nil {
-		return err
-	}
-	for topic, partitions := range a.TopicPartitions {
-		if err := pe.putString(topic); err != nil {
-			return err
-		}
-		if err := pe.putInt32Array(partitions); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-func (a *AddPartitionsToTxnRequest) decode(pd packetDecoder, version int16) (err error) {
-	if a.TransactionalID, err = pd.getString(); err != nil {
-		return err
-	}
-	if a.ProducerID, err = pd.getInt64(); err != nil {
-		return err
-	}
-	if a.ProducerEpoch, err = pd.getInt16(); err != nil {
-		return err
-	}
-
-	n, err := pd.getArrayLength()
-	if err != nil {
-		return err
-	}
-
-	a.TopicPartitions = make(map[string][]int32)
-	for i := 0; i < n; i++ {
-		topic, err := pd.getString()
-		if err != nil {
-			return err
-		}
-
-		partitions, err := pd.getInt32Array()
-		if err != nil {
-			return err
-		}
-
-		a.TopicPartitions[topic] = partitions
-	}
-
-	return nil
-}
-
-func (a *AddPartitionsToTxnRequest) key() int16 {
-	return 24
-}
-
-func (a *AddPartitionsToTxnRequest) version() int16 {
-	return 0
-}
-
-func (a *AddPartitionsToTxnRequest) headerVersion() int16 {
-	return 1
-}
-
-func (a *AddPartitionsToTxnRequest) requiredVersion() KafkaVersion {
-	return V0_11_0_0
-}

vendor/github.com/Shopify/sarama/add_partitions_to_txn_response.go

@@ -1,114 +0,0 @@
-package sarama
-
-import (
-	"time"
-)
-
-//AddPartitionsToTxnResponse is a partition errors to transaction type
-type AddPartitionsToTxnResponse struct {
-	ThrottleTime time.Duration
-	Errors       map[string][]*PartitionError
-}
-
-func (a *AddPartitionsToTxnResponse) encode(pe packetEncoder) error {
-	pe.putInt32(int32(a.ThrottleTime / time.Millisecond))
-	if err := pe.putArrayLength(len(a.Errors)); err != nil {
-		return err
-	}
-
-	for topic, e := range a.Errors {
-		if err := pe.putString(topic); err != nil {
-			return err
-		}
-		if err := pe.putArrayLength(len(e)); err != nil {
-			return err
-		}
-		for _, partitionError := range e {
-			if err := partitionError.encode(pe); err != nil {
-				return err
-			}
-		}
-	}
-
-	return nil
-}
-
-func (a *AddPartitionsToTxnResponse) decode(pd packetDecoder, version int16) (err error) {
-	throttleTime, err := pd.getInt32()
-	if err != nil {
-		return err
-	}
-	a.ThrottleTime = time.Duration(throttleTime) * time.Millisecond
-
-	n, err := pd.getArrayLength()
-	if err != nil {
-		return err
-	}
-
-	a.Errors = make(map[string][]*PartitionError)
-
-	for i := 0; i < n; i++ {
-		topic, err := pd.getString()
-		if err != nil {
-			return err
-		}
-
-		m, err := pd.getArrayLength()
-		if err != nil {
-			return err
-		}
-
-		a.Errors[topic] = make([]*PartitionError, m)
-
-		for j := 0; j < m; j++ {
-			a.Errors[topic][j] = new(PartitionError)
-			if err := a.Errors[topic][j].decode(pd, version); err != nil {
-				return err
-			}
-		}
-	}
-
-	return nil
-}
-
-func (a *AddPartitionsToTxnResponse) key() int16 {
-	return 24
-}
-
-func (a *AddPartitionsToTxnResponse) version() int16 {
-	return 0
-}
-
-func (a *AddPartitionsToTxnResponse) headerVersion() int16 {
-	return 0
-}
-
-func (a *AddPartitionsToTxnResponse) requiredVersion() KafkaVersion {
-	return V0_11_0_0
-}
-
-//PartitionError is a partition error type
-type PartitionError struct {
-	Partition int32
-	Err       KError
-}
-
-func (p *PartitionError) encode(pe packetEncoder) error {
-	pe.putInt32(p.Partition)
-	pe.putInt16(int16(p.Err))
-	return nil
-}
-
-func (p *PartitionError) decode(pd packetDecoder, version int16) (err error) {
-	if p.Partition, err = pd.getInt32(); err != nil {
-		return err
-	}
-
-	kerr, err := pd.getInt16()
-	if err != nil {
-		return err
-	}
-	p.Err = KError(kerr)
-
-	return nil
-}

vendor/github.com/Shopify/sarama/admin.go

@@ -1,934 +0,0 @@
-package sarama
-
-import (
-	"errors"
-	"fmt"
-	"math/rand"
-	"strconv"
-	"sync"
-	"time"
-)
-
-// ClusterAdmin is the administrative client for Kafka, which supports managing and inspecting topics,
-// brokers, configurations and ACLs. The minimum broker version required is 0.10.0.0.
-// Methods with stricter requirements will specify the minimum broker version required.
-// You MUST call Close() on a client to avoid leaks
-type ClusterAdmin interface {
-	// Creates a new topic. This operation is supported by brokers with version 0.10.1.0 or higher.
-	// It may take several seconds after CreateTopic returns success for all the brokers
-	// to become aware that the topic has been created. During this time, listTopics
-	// may not return information about the new topic.The validateOnly option is supported from version 0.10.2.0.
-	CreateTopic(topic string, detail *TopicDetail, validateOnly bool) error
-
-	// List the topics available in the cluster with the default options.
-	ListTopics() (map[string]TopicDetail, error)
-
-	// Describe some topics in the cluster.
-	DescribeTopics(topics []string) (metadata []*TopicMetadata, err error)
-
-	// Delete a topic. It may take several seconds after the DeleteTopic to returns success
-	// and for all the brokers to become aware that the topics are gone.
-	// During this time, listTopics  may continue to return information about the deleted topic.
-	// If delete.topic.enable is false on the brokers, deleteTopic will mark
-	// the topic for deletion, but not actually delete them.
-	// This operation is supported by brokers with version 0.10.1.0 or higher.
-	DeleteTopic(topic string) error
-
-	// Increase the number of partitions of the topics  according to the corresponding values.
-	// If partitions are increased for a topic that has a key, the partition logic or ordering of
-	// the messages will be affected. It may take several seconds after this method returns
-	// success for all the brokers to become aware that the partitions have been created.
-	// During this time, ClusterAdmin#describeTopics may not return information about the
-	// new partitions. This operation is supported by brokers with version 1.0.0 or higher.
-	CreatePartitions(topic string, count int32, assignment [][]int32, validateOnly bool) error
-
-	// Alter the replica assignment for partitions.
-	// This operation is supported by brokers with version 2.4.0.0 or higher.
-	AlterPartitionReassignments(topic string, assignment [][]int32) error
-
-	// Provides info on ongoing partitions replica reassignments.
-	// This operation is supported by brokers with version 2.4.0.0 or higher.
-	ListPartitionReassignments(topics string, partitions []int32) (topicStatus map[string]map[int32]*PartitionReplicaReassignmentsStatus, err error)
-
-	// Delete records whose offset is smaller than the given offset of the corresponding partition.
-	// This operation is supported by brokers with version 0.11.0.0 or higher.
-	DeleteRecords(topic string, partitionOffsets map[int32]int64) error
-
-	// Get the configuration for the specified resources.
-	// The returned configuration includes default values and the Default is true
-	// can be used to distinguish them from user supplied values.
-	// Config entries where ReadOnly is true cannot be updated.
-	// The value of config entries where Sensitive is true is always nil so
-	// sensitive information is not disclosed.
-	// This operation is supported by brokers with version 0.11.0.0 or higher.
-	DescribeConfig(resource ConfigResource) ([]ConfigEntry, error)
-
-	// Update the configuration for the specified resources with the default options.
-	// This operation is supported by brokers with version 0.11.0.0 or higher.
-	// The resources with their configs (topic is the only resource type with configs
-	// that can be updated currently Updates are not transactional so they may succeed
-	// for some resources while fail for others. The configs for a particular resource are updated automatically.
-	AlterConfig(resourceType ConfigResourceType, name string, entries map[string]*string, validateOnly bool) error
-
-	// Creates access control lists (ACLs) which are bound to specific resources.
-	// This operation is not transactional so it may succeed for some ACLs while fail for others.
-	// If you attempt to add an ACL that duplicates an existing ACL, no error will be raised, but
-	// no changes will be made. This operation is supported by brokers with version 0.11.0.0 or higher.
-	CreateACL(resource Resource, acl Acl) error
-
-	// Lists access control lists (ACLs) according to the supplied filter.
-	// it may take some time for changes made by createAcls or deleteAcls to be reflected in the output of ListAcls
-	// This operation is supported by brokers with version 0.11.0.0 or higher.
-	ListAcls(filter AclFilter) ([]ResourceAcls, error)
-
-	// Deletes access control lists (ACLs) according to the supplied filters.
-	// This operation is not transactional so it may succeed for some ACLs while fail for others.
-	// This operation is supported by brokers with version 0.11.0.0 or higher.
-	DeleteACL(filter AclFilter, validateOnly bool) ([]MatchingAcl, error)
-
-	// List the consumer groups available in the cluster.
-	ListConsumerGroups() (map[string]string, error)
-
-	// Describe the given consumer groups.
-	DescribeConsumerGroups(groups []string) ([]*GroupDescription, error)
-
-	// List the consumer group offsets available in the cluster.
-	ListConsumerGroupOffsets(group string, topicPartitions map[string][]int32) (*OffsetFetchResponse, error)
-
-	// Delete a consumer group.
-	DeleteConsumerGroup(group string) error
-
-	// Get information about the nodes in the cluster
-	DescribeCluster() (brokers []*Broker, controllerID int32, err error)
-
-	// Get information about all log directories on the given set of brokers
-	DescribeLogDirs(brokers []int32) (map[int32][]DescribeLogDirsResponseDirMetadata, error)
-
-	// Close shuts down the admin and closes underlying client.
-	Close() error
-}
-
-type clusterAdmin struct {
-	client Client
-	conf   *Config
-}
-
-// NewClusterAdmin creates a new ClusterAdmin using the given broker addresses and configuration.
-func NewClusterAdmin(addrs []string, conf *Config) (ClusterAdmin, error) {
-	client, err := NewClient(addrs, conf)
-	if err != nil {
-		return nil, err
-	}
-	return NewClusterAdminFromClient(client)
-}
-
-// NewClusterAdminFromClient creates a new ClusterAdmin using the given client.
-// Note that underlying client will also be closed on admin's Close() call.
-func NewClusterAdminFromClient(client Client) (ClusterAdmin, error) {
-	//make sure we can retrieve the controller
-	_, err := client.Controller()
-	if err != nil {
-		return nil, err
-	}
-
-	ca := &clusterAdmin{
-		client: client,
-		conf:   client.Config(),
-	}
-	return ca, nil
-}
-
-func (ca *clusterAdmin) Close() error {
-	return ca.client.Close()
-}
-
-func (ca *clusterAdmin) Controller() (*Broker, error) {
-	return ca.client.Controller()
-}
-
-func (ca *clusterAdmin) refreshController() (*Broker, error) {
-	return ca.client.RefreshController()
-}
-
-// isErrNoController returns `true` if the given error type unwraps to an
-// `ErrNotController` response from Kafka
-func isErrNoController(err error) bool {
-	switch e := err.(type) {
-	case *TopicError:
-		return e.Err == ErrNotController
-	case *TopicPartitionError:
-		return e.Err == ErrNotController
-	case KError:
-		return e == ErrNotController
-	}
-	return false
-}
-
-// retryOnError will repeatedly call the given (error-returning) func in the
-// case that its response is non-nil and retriable (as determined by the
-// provided retriable func) up to the maximum number of tries permitted by
-// the admin client configuration
-func (ca *clusterAdmin) retryOnError(retriable func(error) bool, fn func() error) error {
-	var err error
-	for attempt := 0; attempt < ca.conf.Admin.Retry.Max; attempt++ {
-		err = fn()
-		if err == nil || !retriable(err) {
-			return err
-		}
-		Logger.Printf(
-			"admin/request retrying after %dms... (%d attempts remaining)\n",
-			ca.conf.Admin.Retry.Backoff/time.Millisecond, ca.conf.Admin.Retry.Max-attempt)
-		time.Sleep(ca.conf.Admin.Retry.Backoff)
-		continue
-	}
-	return err
-}
-
-func (ca *clusterAdmin) CreateTopic(topic string, detail *TopicDetail, validateOnly bool) error {
-	if topic == "" {
-		return ErrInvalidTopic
-	}
-
-	if detail == nil {
-		return errors.New("you must specify topic details")
-	}
-
-	topicDetails := make(map[string]*TopicDetail)
-	topicDetails[topic] = detail
-
-	request := &CreateTopicsRequest{
-		TopicDetails: topicDetails,
-		ValidateOnly: validateOnly,
-		Timeout:      ca.conf.Admin.Timeout,
-	}
-
-	if ca.conf.Version.IsAtLeast(V0_11_0_0) {
-		request.Version = 1
-	}
-	if ca.conf.Version.IsAtLeast(V1_0_0_0) {
-		request.Version = 2
-	}
-
-	return ca.retryOnError(isErrNoController, func() error {
-		b, err := ca.Controller()
-		if err != nil {
-			return err
-		}
-
-		rsp, err := b.CreateTopics(request)
-		if err != nil {
-			return err
-		}
-
-		topicErr, ok := rsp.TopicErrors[topic]
-		if !ok {
-			return ErrIncompleteResponse
-		}
-
-		if topicErr.Err != ErrNoError {
-			if topicErr.Err == ErrNotController {
-				_, _ = ca.refreshController()
-			}
-			return topicErr
-		}
-
-		return nil
-	})
-}
-
-func (ca *clusterAdmin) DescribeTopics(topics []string) (metadata []*TopicMetadata, err error) {
-	controller, err := ca.Controller()
-	if err != nil {
-		return nil, err
-	}
-
-	request := &MetadataRequest{
-		Topics:                 topics,
-		AllowAutoTopicCreation: false,
-	}
-
-	if ca.conf.Version.IsAtLeast(V1_0_0_0) {
-		request.Version = 5
-	} else if ca.conf.Version.IsAtLeast(V0_11_0_0) {
-		request.Version = 4
-	}
-
-	response, err := controller.GetMetadata(request)
-	if err != nil {
-		return nil, err
-	}
-	return response.Topics, nil
-}
-
-func (ca *clusterAdmin) DescribeCluster() (brokers []*Broker, controllerID int32, err error) {
-	controller, err := ca.Controller()
-	if err != nil {
-		return nil, int32(0), err
-	}
-
-	request := &MetadataRequest{
-		Topics: []string{},
-	}
-
-	if ca.conf.Version.IsAtLeast(V0_10_0_0) {
-		request.Version = 1
-	}
-
-	response, err := controller.GetMetadata(request)
-	if err != nil {
-		return nil, int32(0), err
-	}
-
-	return response.Brokers, response.ControllerID, nil
-}
-
-func (ca *clusterAdmin) findBroker(id int32) (*Broker, error) {
-	brokers := ca.client.Brokers()
-	for _, b := range brokers {
-		if b.ID() == id {
-			return b, nil
-		}
-	}
-	return nil, fmt.Errorf("could not find broker id %d", id)
-}
-
-func (ca *clusterAdmin) findAnyBroker() (*Broker, error) {
-	brokers := ca.client.Brokers()
-	if len(brokers) > 0 {
-		index := rand.Intn(len(brokers))
-		return brokers[index], nil
-	}
-	return nil, errors.New("no available broker")
-}
-
-func (ca *clusterAdmin) ListTopics() (map[string]TopicDetail, error) {
-	// In order to build TopicDetails we need to first get the list of all
-	// topics using a MetadataRequest and then get their configs using a
-	// DescribeConfigsRequest request. To avoid sending many requests to the
-	// broker, we use a single DescribeConfigsRequest.
-
-	// Send the all-topic MetadataRequest
-	b, err := ca.findAnyBroker()
-	if err != nil {
-		return nil, err
-	}
-	_ = b.Open(ca.client.Config())
-
-	metadataReq := &MetadataRequest{}
-	metadataResp, err := b.GetMetadata(metadataReq)
-	if err != nil {
-		return nil, err
-	}
-
-	topicsDetailsMap := make(map[string]TopicDetail)
-
-	var describeConfigsResources []*ConfigResource
-
-	for _, topic := range metadataResp.Topics {
-		topicDetails := TopicDetail{
-			NumPartitions: int32(len(topic.Partitions)),
-		}
-		if len(topic.Partitions) > 0 {
-			topicDetails.ReplicaAssignment = map[int32][]int32{}
-			for _, partition := range topic.Partitions {
-				topicDetails.ReplicaAssignment[partition.ID] = partition.Replicas
-			}
-			topicDetails.ReplicationFactor = int16(len(topic.Partitions[0].Replicas))
-		}
-		topicsDetailsMap[topic.Name] = topicDetails
-
-		// we populate the resources we want to describe from the MetadataResponse
-		topicResource := ConfigResource{
-			Type: TopicResource,
-			Name: topic.Name,
-		}
-		describeConfigsResources = append(describeConfigsResources, &topicResource)
-	}
-
-	// Send the DescribeConfigsRequest
-	describeConfigsReq := &DescribeConfigsRequest{
-		Resources: describeConfigsResources,
-	}
-
-	if ca.conf.Version.IsAtLeast(V1_1_0_0) {
-		describeConfigsReq.Version = 1
-	}
-
-	if ca.conf.Version.IsAtLeast(V2_0_0_0) {
-		describeConfigsReq.Version = 2
-	}
-
-	describeConfigsResp, err := b.DescribeConfigs(describeConfigsReq)
-	if err != nil {
-		return nil, err
-	}
-
-	for _, resource := range describeConfigsResp.Resources {
-		topicDetails := topicsDetailsMap[resource.Name]
-		topicDetails.ConfigEntries = make(map[string]*string)
-
-		for _, entry := range resource.Configs {
-			// only include non-default non-sensitive config
-			// (don't actually think topic config will ever be sensitive)
-			if entry.Default || entry.Sensitive {
-				continue
-			}
-			topicDetails.ConfigEntries[entry.Name] = &entry.Value
-		}
-
-		topicsDetailsMap[resource.Name] = topicDetails
-	}
-
-	return topicsDetailsMap, nil
-}
-
-func (ca *clusterAdmin) DeleteTopic(topic string) error {
-	if topic == "" {
-		return ErrInvalidTopic
-	}
-
-	request := &DeleteTopicsRequest{
-		Topics:  []string{topic},
-		Timeout: ca.conf.Admin.Timeout,
-	}
-
-	if ca.conf.Version.IsAtLeast(V0_11_0_0) {
-		request.Version = 1
-	}
-
-	return ca.retryOnError(isErrNoController, func() error {
-		b, err := ca.Controller()
-		if err != nil {
-			return err
-		}
-
-		rsp, err := b.DeleteTopics(request)
-		if err != nil {
-			return err
-		}
-
-		topicErr, ok := rsp.TopicErrorCodes[topic]
-		if !ok {
-			return ErrIncompleteResponse
-		}
-
-		if topicErr != ErrNoError {
-			if topicErr == ErrNotController {
-				_, _ = ca.refreshController()
-			}
-			return topicErr
-		}
-
-		return nil
-	})
-}
-
-func (ca *clusterAdmin) CreatePartitions(topic string, count int32, assignment [][]int32, validateOnly bool) error {
-	if topic == "" {
-		return ErrInvalidTopic
-	}
-
-	topicPartitions := make(map[string]*TopicPartition)
-	topicPartitions[topic] = &TopicPartition{Count: count, Assignment: assignment}
-
-	request := &CreatePartitionsRequest{
-		TopicPartitions: topicPartitions,
-		Timeout:         ca.conf.Admin.Timeout,
-	}
-
-	return ca.retryOnError(isErrNoController, func() error {
-		b, err := ca.Controller()
-		if err != nil {
-			return err
-		}
-
-		rsp, err := b.CreatePartitions(request)
-		if err != nil {
-			return err
-		}
-
-		topicErr, ok := rsp.TopicPartitionErrors[topic]
-		if !ok {
-			return ErrIncompleteResponse
-		}
-
-		if topicErr.Err != ErrNoError {
-			if topicErr.Err == ErrNotController {
-				_, _ = ca.refreshController()
-			}
-			return topicErr
-		}
-
-		return nil
-	})
-}
-
-func (ca *clusterAdmin) AlterPartitionReassignments(topic string, assignment [][]int32) error {
-	if topic == "" {
-		return ErrInvalidTopic
-	}
-
-	request := &AlterPartitionReassignmentsRequest{
-		TimeoutMs: int32(60000),
-		Version:   int16(0),
-	}
-
-	for i := 0; i < len(assignment); i++ {
-		request.AddBlock(topic, int32(i), assignment[i])
-	}
-
-	return ca.retryOnError(isErrNoController, func() error {
-		b, err := ca.Controller()
-		if err != nil {
-			return err
-		}
-
-		errs := make([]error, 0)
-
-		rsp, err := b.AlterPartitionReassignments(request)
-
-		if err != nil {
-			errs = append(errs, err)
-		} else {
-			if rsp.ErrorCode > 0 {
-				errs = append(errs, errors.New(rsp.ErrorCode.Error()))
-			}
-
-			for topic, topicErrors := range rsp.Errors {
-				for partition, partitionError := range topicErrors {
-					if partitionError.errorCode != ErrNoError {
-						errStr := fmt.Sprintf("[%s-%d]: %s", topic, partition, partitionError.errorCode.Error())
-						errs = append(errs, errors.New(errStr))
-					}
-				}
-			}
-		}
-
-		if len(errs) > 0 {
-			return ErrReassignPartitions{MultiError{&errs}}
-		}
-
-		return nil
-	})
-}
-
-func (ca *clusterAdmin) ListPartitionReassignments(topic string, partitions []int32) (topicStatus map[string]map[int32]*PartitionReplicaReassignmentsStatus, err error) {
-	if topic == "" {
-		return nil, ErrInvalidTopic
-	}
-
-	request := &ListPartitionReassignmentsRequest{
-		TimeoutMs: int32(60000),
-		Version:   int16(0),
-	}
-
-	request.AddBlock(topic, partitions)
-
-	b, err := ca.Controller()
-	if err != nil {
-		return nil, err
-	}
-	_ = b.Open(ca.client.Config())
-
-	rsp, err := b.ListPartitionReassignments(request)
-
-	if err == nil && rsp != nil {
-		return rsp.TopicStatus, nil
-	} else {
-		return nil, err
-	}
-}
-
-func (ca *clusterAdmin) DeleteRecords(topic string, partitionOffsets map[int32]int64) error {
-	if topic == "" {
-		return ErrInvalidTopic
-	}
-	partitionPerBroker := make(map[*Broker][]int32)
-	for partition := range partitionOffsets {
-		broker, err := ca.client.Leader(topic, partition)
-		if err != nil {
-			return err
-		}
-		if _, ok := partitionPerBroker[broker]; ok {
-			partitionPerBroker[broker] = append(partitionPerBroker[broker], partition)
-		} else {
-			partitionPerBroker[broker] = []int32{partition}
-		}
-	}
-	errs := make([]error, 0)
-	for broker, partitions := range partitionPerBroker {
-		topics := make(map[string]*DeleteRecordsRequestTopic)
-		recordsToDelete := make(map[int32]int64)
-		for _, p := range partitions {
-			recordsToDelete[p] = partitionOffsets[p]
-		}
-		topics[topic] = &DeleteRecordsRequestTopic{PartitionOffsets: recordsToDelete}
-		request := &DeleteRecordsRequest{
-			Topics:  topics,
-			Timeout: ca.conf.Admin.Timeout,
-		}
-
-		rsp, err := broker.DeleteRecords(request)
-		if err != nil {
-			errs = append(errs, err)
-		} else {
-			deleteRecordsResponseTopic, ok := rsp.Topics[topic]
-			if !ok {
-				errs = append(errs, ErrIncompleteResponse)
-			} else {
-				for _, deleteRecordsResponsePartition := range deleteRecordsResponseTopic.Partitions {
-					if deleteRecordsResponsePartition.Err != ErrNoError {
-						errs = append(errs, errors.New(deleteRecordsResponsePartition.Err.Error()))
-					}
-				}
-			}
-		}
-	}
-	if len(errs) > 0 {
-		return ErrDeleteRecords{MultiError{&errs}}
-	}
-	//todo since we are dealing with couple of partitions it would be good if we return slice of errors
-	//for each partition instead of one error
-	return nil
-}
-
-// Returns a bool indicating whether the resource request needs to go to a
-// specific broker
-func dependsOnSpecificNode(resource ConfigResource) bool {
-	return (resource.Type == BrokerResource && resource.Name != "") ||
-		resource.Type == BrokerLoggerResource
-}
-
-func (ca *clusterAdmin) DescribeConfig(resource ConfigResource) ([]ConfigEntry, error) {
-	var entries []ConfigEntry
-	var resources []*ConfigResource
-	resources = append(resources, &resource)
-
-	request := &DescribeConfigsRequest{
-		Resources: resources,
-	}
-
-	if ca.conf.Version.IsAtLeast(V1_1_0_0) {
-		request.Version = 1
-	}
-
-	if ca.conf.Version.IsAtLeast(V2_0_0_0) {
-		request.Version = 2
-	}
-
-	var (
-		b   *Broker
-		err error
-	)
-
-	// DescribeConfig of broker/broker logger must be sent to the broker in question
-	if dependsOnSpecificNode(resource) {
-		id, _ := strconv.Atoi(resource.Name)
-		b, err = ca.findBroker(int32(id))
-	} else {
-		b, err = ca.findAnyBroker()
-	}
-	if err != nil {
-		return nil, err
-	}
-
-	_ = b.Open(ca.client.Config())
-	rsp, err := b.DescribeConfigs(request)
-	if err != nil {
-		return nil, err
-	}
-
-	for _, rspResource := range rsp.Resources {
-		if rspResource.Name == resource.Name {
-			if rspResource.ErrorMsg != "" {
-				return nil, errors.New(rspResource.ErrorMsg)
-			}
-			if rspResource.ErrorCode != 0 {
-				return nil, KError(rspResource.ErrorCode)
-			}
-			for _, cfgEntry := range rspResource.Configs {
-				entries = append(entries, *cfgEntry)
-			}
-		}
-	}
-	return entries, nil
-}
-
-func (ca *clusterAdmin) AlterConfig(resourceType ConfigResourceType, name string, entries map[string]*string, validateOnly bool) error {
-	var resources []*AlterConfigsResource
-	resources = append(resources, &AlterConfigsResource{
-		Type:          resourceType,
-		Name:          name,
-		ConfigEntries: entries,
-	})
-
-	request := &AlterConfigsRequest{
-		Resources:    resources,
-		ValidateOnly: validateOnly,
-	}
-
-	var (
-		b   *Broker
-		err error
-	)
-
-	// AlterConfig of broker/broker logger must be sent to the broker in question
-	if dependsOnSpecificNode(ConfigResource{Name: name, Type: resourceType}) {
-		id, _ := strconv.Atoi(name)
-		b, err = ca.findBroker(int32(id))
-	} else {
-		b, err = ca.findAnyBroker()
-	}
-	if err != nil {
-		return err
-	}
-
-	_ = b.Open(ca.client.Config())
-	rsp, err := b.AlterConfigs(request)
-	if err != nil {
-		return err
-	}
-
-	for _, rspResource := range rsp.Resources {
-		if rspResource.Name == name {
-			if rspResource.ErrorMsg != "" {
-				return errors.New(rspResource.ErrorMsg)
-			}
-			if rspResource.ErrorCode != 0 {
-				return KError(rspResource.ErrorCode)
-			}
-		}
-	}
-	return nil
-}
-
-func (ca *clusterAdmin) CreateACL(resource Resource, acl Acl) error {
-	var acls []*AclCreation
-	acls = append(acls, &AclCreation{resource, acl})
-	request := &CreateAclsRequest{AclCreations: acls}
-
-	if ca.conf.Version.IsAtLeast(V2_0_0_0) {
-		request.Version = 1
-	}
-
-	b, err := ca.Controller()
-	if err != nil {
-		return err
-	}
-
-	_, err = b.CreateAcls(request)
-	return err
-}
-
-func (ca *clusterAdmin) ListAcls(filter AclFilter) ([]ResourceAcls, error) {
-	request := &DescribeAclsRequest{AclFilter: filter}
-
-	if ca.conf.Version.IsAtLeast(V2_0_0_0) {
-		request.Version = 1
-	}
-
-	b, err := ca.Controller()
-	if err != nil {
-		return nil, err
-	}
-
-	rsp, err := b.DescribeAcls(request)
-	if err != nil {
-		return nil, err
-	}
-
-	var lAcls []ResourceAcls
-	for _, rAcl := range rsp.ResourceAcls {
-		lAcls = append(lAcls, *rAcl)
-	}
-	return lAcls, nil
-}
-
-func (ca *clusterAdmin) DeleteACL(filter AclFilter, validateOnly bool) ([]MatchingAcl, error) {
-	var filters []*AclFilter
-	filters = append(filters, &filter)
-	request := &DeleteAclsRequest{Filters: filters}
-
-	if ca.conf.Version.IsAtLeast(V2_0_0_0) {
-		request.Version = 1
-	}
-
-	b, err := ca.Controller()
-	if err != nil {
-		return nil, err
-	}
-
-	rsp, err := b.DeleteAcls(request)
-	if err != nil {
-		return nil, err
-	}
-
-	var mAcls []MatchingAcl
-	for _, fr := range rsp.FilterResponses {
-		for _, mACL := range fr.MatchingAcls {
-			mAcls = append(mAcls, *mACL)
-		}
-	}
-	return mAcls, nil
-}
-
-func (ca *clusterAdmin) DescribeConsumerGroups(groups []string) (result []*GroupDescription, err error) {
-	groupsPerBroker := make(map[*Broker][]string)
-
-	for _, group := range groups {
-		controller, err := ca.client.Coordinator(group)
-		if err != nil {
-			return nil, err
-		}
-		groupsPerBroker[controller] = append(groupsPerBroker[controller], group)
-	}
-
-	for broker, brokerGroups := range groupsPerBroker {
-		response, err := broker.DescribeGroups(&DescribeGroupsRequest{
-			Groups: brokerGroups,
-		})
-		if err != nil {
-			return nil, err
-		}
-
-		result = append(result, response.Groups...)
-	}
-	return result, nil
-}
-
-func (ca *clusterAdmin) ListConsumerGroups() (allGroups map[string]string, err error) {
-	allGroups = make(map[string]string)
-
-	// Query brokers in parallel, since we have to query *all* brokers
-	brokers := ca.client.Brokers()
-	groupMaps := make(chan map[string]string, len(brokers))
-	errors := make(chan error, len(brokers))
-	wg := sync.WaitGroup{}
-
-	for _, b := range brokers {
-		wg.Add(1)
-		go func(b *Broker, conf *Config) {
-			defer wg.Done()
-			_ = b.Open(conf) // Ensure that broker is opened
-
-			response, err := b.ListGroups(&ListGroupsRequest{})
-			if err != nil {
-				errors <- err
-				return
-			}
-
-			groups := make(map[string]string)
-			for group, typ := range response.Groups {
-				groups[group] = typ
-			}
-
-			groupMaps <- groups
-		}(b, ca.conf)
-	}
-
-	wg.Wait()
-	close(groupMaps)
-	close(errors)
-
-	for groupMap := range groupMaps {
-		for group, protocolType := range groupMap {
-			allGroups[group] = protocolType
-		}
-	}
-
-	// Intentionally return only the first error for simplicity
-	err = <-errors
-	return
-}
-
-func (ca *clusterAdmin) ListConsumerGroupOffsets(group string, topicPartitions map[string][]int32) (*OffsetFetchResponse, error) {
-	coordinator, err := ca.client.Coordinator(group)
-	if err != nil {
-		return nil, err
-	}
-
-	request := &OffsetFetchRequest{
-		ConsumerGroup: group,
-		partitions:    topicPartitions,
-	}
-
-	if ca.conf.Version.IsAtLeast(V0_10_2_0) {
-		request.Version = 2
-	} else if ca.conf.Version.IsAtLeast(V0_8_2_2) {
-		request.Version = 1
-	}
-
-	return coordinator.FetchOffset(request)
-}
-
-func (ca *clusterAdmin) DeleteConsumerGroup(group string) error {
-	coordinator, err := ca.client.Coordinator(group)
-	if err != nil {
-		return err
-	}
-
-	request := &DeleteGroupsRequest{
-		Groups: []string{group},
-	}
-
-	resp, err := coordinator.DeleteGroups(request)
-	if err != nil {
-		return err
-	}
-
-	groupErr, ok := resp.GroupErrorCodes[group]
-	if !ok {
-		return ErrIncompleteResponse
-	}
-
-	if groupErr != ErrNoError {
-		return groupErr
-	}
-
-	return nil
-}
-
-func (ca *clusterAdmin) DescribeLogDirs(brokerIds []int32) (allLogDirs map[int32][]DescribeLogDirsResponseDirMetadata, err error) {
-	allLogDirs = make(map[int32][]DescribeLogDirsResponseDirMetadata)
-
-	// Query brokers in parallel, since we may have to query multiple brokers
-	logDirsMaps := make(chan map[int32][]DescribeLogDirsResponseDirMetadata, len(brokerIds))
-	errors := make(chan error, len(brokerIds))
-	wg := sync.WaitGroup{}
-
-	for _, b := range brokerIds {
-		wg.Add(1)
-		broker, err := ca.findBroker(b)
-		if err != nil {
-			Logger.Printf("Unable to find broker with ID = %v\n", b)
-			continue
-		}
-		go func(b *Broker, conf *Config) {
-			defer wg.Done()
-			_ = b.Open(conf) // Ensure that broker is opened
-
-			response, err := b.DescribeLogDirs(&DescribeLogDirsRequest{})
-			if err != nil {
-				errors <- err
-				return
-			}
-			logDirs := make(map[int32][]DescribeLogDirsResponseDirMetadata)
-			logDirs[b.ID()] = response.LogDirs
-			logDirsMaps <- logDirs
-		}(broker, ca.conf)
-	}
-
-	wg.Wait()
-	close(logDirsMaps)
-	close(errors)
-
-	for logDirsMap := range logDirsMaps {
-		for id, logDirs := range logDirsMap {
-			allLogDirs[id] = logDirs
-		}
-	}
-
-	// Intentionally return only the first error for simplicity
-	err = <-errors
-	return
-}

vendor/github.com/Shopify/sarama/alter_configs_request.go

@@ -1,126 +0,0 @@
-package sarama
-
-//AlterConfigsRequest is an alter config request type
-type AlterConfigsRequest struct {
-	Resources    []*AlterConfigsResource
-	ValidateOnly bool
-}
-
-//AlterConfigsResource is an alter config resource type
-type AlterConfigsResource struct {
-	Type          ConfigResourceType
-	Name          string
-	ConfigEntries map[string]*string
-}
-
-func (a *AlterConfigsRequest) encode(pe packetEncoder) error {
-	if err := pe.putArrayLength(len(a.Resources)); err != nil {
-		return err
-	}
-
-	for _, r := range a.Resources {
-		if err := r.encode(pe); err != nil {
-			return err
-		}
-	}
-
-	pe.putBool(a.ValidateOnly)
-	return nil
-}
-
-func (a *AlterConfigsRequest) decode(pd packetDecoder, version int16) error {
-	resourceCount, err := pd.getArrayLength()
-	if err != nil {
-		return err
-	}
-
-	a.Resources = make([]*AlterConfigsResource, resourceCount)
-	for i := range a.Resources {
-		r := &AlterConfigsResource{}
-		err = r.decode(pd, version)
-		if err != nil {
-			return err
-		}
-		a.Resources[i] = r
-	}
-
-	validateOnly, err := pd.getBool()
-	if err != nil {
-		return err
-	}
-
-	a.ValidateOnly = validateOnly
-
-	return nil
-}
-
-func (a *AlterConfigsResource) encode(pe packetEncoder) error {
-	pe.putInt8(int8(a.Type))
-
-	if err := pe.putString(a.Name); err != nil {
-		return err
-	}
-
-	if err := pe.putArrayLength(len(a.ConfigEntries)); err != nil {
-		return err
-	}
-	for configKey, configValue := range a.ConfigEntries {
-		if err := pe.putString(configKey); err != nil {
-			return err
-		}
-		if err := pe.putNullableString(configValue); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-func (a *AlterConfigsResource) decode(pd packetDecoder, version int16) error {
-	t, err := pd.getInt8()
-	if err != nil {
-		return err
-	}
-	a.Type = ConfigResourceType(t)
-
-	name, err := pd.getString()
-	if err != nil {
-		return err
-	}
-	a.Name = name
-
-	n, err := pd.getArrayLength()
-	if err != nil {
-		return err
-	}
-
-	if n > 0 {
-		a.ConfigEntries = make(map[string]*string, n)
-		for i := 0; i < n; i++ {
-			configKey, err := pd.getString()
-			if err != nil {
-				return err
-			}
-			if a.ConfigEntries[configKey], err = pd.getNullableString(); err != nil {
-				return err
-			}
-		}
-	}
-	return err
-}
-
-func (a *AlterConfigsRequest) key() int16 {
-	return 33
-}
-
-func (a *AlterConfigsRequest) version() int16 {
-	return 0
-}
-
-func (a *AlterConfigsRequest) headerVersion() int16 {
-	return 1
-}
-
-func (a *AlterConfigsRequest) requiredVersion() KafkaVersion {
-	return V0_11_0_0
-}

vendor/github.com/Shopify/sarama/alter_configs_response.go

@@ -1,101 +0,0 @@
-package sarama
-
-import "time"
-
-//AlterConfigsResponse is a response type for alter config
-type AlterConfigsResponse struct {
-	ThrottleTime time.Duration
-	Resources    []*AlterConfigsResourceResponse
-}
-
-//AlterConfigsResourceResponse is a response type for alter config resource
-type AlterConfigsResourceResponse struct {
-	ErrorCode int16
-	ErrorMsg  string
-	Type      ConfigResourceType
-	Name      string
-}
-
-func (a *AlterConfigsResponse) encode(pe packetEncoder) error {
-	pe.putInt32(int32(a.ThrottleTime / time.Millisecond))
-
-	if err := pe.putArrayLength(len(a.Resources)); err != nil {
-		return err
-	}
-
-	for i := range a.Resources {
-		pe.putInt16(a.Resources[i].ErrorCode)
-		err := pe.putString(a.Resources[i].ErrorMsg)
-		if err != nil {
-			return nil
-		}
-		pe.putInt8(int8(a.Resources[i].Type))
-		err = pe.putString(a.Resources[i].Name)
-		if err != nil {
-			return nil
-		}
-	}
-
-	return nil
-}
-
-func (a *AlterConfigsResponse) decode(pd packetDecoder, version int16) error {
-	throttleTime, err := pd.getInt32()
-	if err != nil {
-		return err
-	}
-	a.ThrottleTime = time.Duration(throttleTime) * time.Millisecond
-
-	responseCount, err := pd.getArrayLength()
-	if err != nil {
-		return err
-	}
-
-	a.Resources = make([]*AlterConfigsResourceResponse, responseCount)
-
-	for i := range a.Resources {
-		a.Resources[i] = new(AlterConfigsResourceResponse)
-
-		errCode, err := pd.getInt16()
-		if err != nil {
-			return err
-		}
-		a.Resources[i].ErrorCode = errCode
-
-		e, err := pd.getString()
-		if err != nil {
-			return err
-		}
-		a.Resources[i].ErrorMsg = e
-
-		t, err := pd.getInt8()
-		if err != nil {
-			return err
-		}
-		a.Resources[i].Type = ConfigResourceType(t)
-
-		name, err := pd.getString()
-		if err != nil {
-			return err
-		}
-		a.Resources[i].Name = name
-	}
-
-	return nil
-}
-
-func (a *AlterConfigsResponse) key() int16 {
-	return 32
-}
-
-func (a *AlterConfigsResponse) version() int16 {
-	return 0
-}
-
-func (a *AlterConfigsResponse) headerVersion() int16 {
-	return 0
-}
-
-func (a *AlterConfigsResponse) requiredVersion() KafkaVersion {
-	return V0_11_0_0
-}

vendor/github.com/Shopify/sarama/alter_partition_reassignments_request.go

@@ -1,130 +0,0 @@
-package sarama
-
-type alterPartitionReassignmentsBlock struct {
-	replicas []int32
-}
-
-func (b *alterPartitionReassignmentsBlock) encode(pe packetEncoder) error {
-	if err := pe.putNullableCompactInt32Array(b.replicas); err != nil {
-		return err
-	}
-
-	pe.putEmptyTaggedFieldArray()
-	return nil
-}
-
-func (b *alterPartitionReassignmentsBlock) decode(pd packetDecoder) (err error) {
-	if b.replicas, err = pd.getCompactInt32Array(); err != nil {
-		return err
-	}
-	return nil
-}
-
-type AlterPartitionReassignmentsRequest struct {
-	TimeoutMs int32
-	blocks    map[string]map[int32]*alterPartitionReassignmentsBlock
-	Version   int16
-}
-
-func (r *AlterPartitionReassignmentsRequest) encode(pe packetEncoder) error {
-	pe.putInt32(r.TimeoutMs)
-
-	pe.putCompactArrayLength(len(r.blocks))
-
-	for topic, partitions := range r.blocks {
-		if err := pe.putCompactString(topic); err != nil {
-			return err
-		}
-		pe.putCompactArrayLength(len(partitions))
-		for partition, block := range partitions {
-			pe.putInt32(partition)
-			if err := block.encode(pe); err != nil {
-				return err
-			}
-		}
-		pe.putEmptyTaggedFieldArray()
-	}
-
-	pe.putEmptyTaggedFieldArray()
-
-	return nil
-}
-
-func (r *AlterPartitionReassignmentsRequest) decode(pd packetDecoder, version int16) (err error) {
-	r.Version = version
-
-	if r.TimeoutMs, err = pd.getInt32(); err != nil {
-		return err
-	}
-
-	topicCount, err := pd.getCompactArrayLength()
-	if err != nil {
-		return err
-	}
-	if topicCount > 0 {
-		r.blocks = make(map[string]map[int32]*alterPartitionReassignmentsBlock)
-		for i := 0; i < topicCount; i++ {
-			topic, err := pd.getCompactString()
-			if err != nil {
-				return err
-			}
-			partitionCount, err := pd.getCompactArrayLength()
-			if err != nil {
-				return err
-			}
-			r.blocks[topic] = make(map[int32]*alterPartitionReassignmentsBlock)
-			for j := 0; j < partitionCount; j++ {
-				partition, err := pd.getInt32()
-				if err != nil {
-					return err
-				}
-				block := &alterPartitionReassignmentsBlock{}
-				if err := block.decode(pd); err != nil {
-					return err
-				}
-				r.blocks[topic][partition] = block
-
-				if _, err := pd.getEmptyTaggedFieldArray(); err != nil {
-					return err
-				}
-			}
-			if _, err := pd.getEmptyTaggedFieldArray(); err != nil {
-				return err
-			}
-		}
-	}
-
-	if _, err := pd.getEmptyTaggedFieldArray(); err != nil {
-		return err
-	}
-
-	return
-}
-
-func (r *AlterPartitionReassignmentsRequest) key() int16 {
-	return 45
-}
-
-func (r *AlterPartitionReassignmentsRequest) version() int16 {
-	return r.Version
-}
-
-func (r *AlterPartitionReassignmentsRequest) headerVersion() int16 {
-	return 2
-}
-
-func (r *AlterPartitionReassignmentsRequest) requiredVersion() KafkaVersion {
-	return V2_4_0_0
-}
-
-func (r *AlterPartitionReassignmentsRequest) AddBlock(topic string, partitionID int32, replicas []int32) {
-	if r.blocks == nil {
-		r.blocks = make(map[string]map[int32]*alterPartitionReassignmentsBlock)
-	}
-
-	if r.blocks[topic] == nil {
-		r.blocks[topic] = make(map[int32]*alterPartitionReassignmentsBlock)
-	}
-
-	r.blocks[topic][partitionID] = &alterPartitionReassignmentsBlock{replicas}
-}

vendor/github.com/Shopify/sarama/alter_partition_reassignments_response.go

@@ -1,157 +0,0 @@
-package sarama
-
-type alterPartitionReassignmentsErrorBlock struct {
-	errorCode    KError
-	errorMessage *string
-}
-
-func (b *alterPartitionReassignmentsErrorBlock) encode(pe packetEncoder) error {
-	pe.putInt16(int16(b.errorCode))
-	if err := pe.putNullableCompactString(b.errorMessage); err != nil {
-		return err
-	}
-	pe.putEmptyTaggedFieldArray()
-
-	return nil
-}
-
-func (b *alterPartitionReassignmentsErrorBlock) decode(pd packetDecoder) (err error) {
-	errorCode, err := pd.getInt16()
-	if err != nil {
-		return err
-	}
-	b.errorCode = KError(errorCode)
-	b.errorMessage, err = pd.getCompactNullableString()
-
-	if _, err := pd.getEmptyTaggedFieldArray(); err != nil {
-		return err
-	}
-	return err
-}
-
-type AlterPartitionReassignmentsResponse struct {
-	Version        int16
-	ThrottleTimeMs int32
-	ErrorCode      KError
-	ErrorMessage   *string
-	Errors         map[string]map[int32]*alterPartitionReassignmentsErrorBlock
-}
-
-func (r *AlterPartitionReassignmentsResponse) AddError(topic string, partition int32, kerror KError, message *string) {
-	if r.Errors == nil {
-		r.Errors = make(map[string]map[int32]*alterPartitionReassignmentsErrorBlock)
-	}
-	partitions := r.Errors[topic]
-	if partitions == nil {
-		partitions = make(map[int32]*alterPartitionReassignmentsErrorBlock)
-		r.Errors[topic] = partitions
-	}
-
-	partitions[partition] = &alterPartitionReassignmentsErrorBlock{errorCode: kerror, errorMessage: message}
-}
-
-func (r *AlterPartitionReassignmentsResponse) encode(pe packetEncoder) error {
-	pe.putInt32(r.ThrottleTimeMs)
-	pe.putInt16(int16(r.ErrorCode))
-	if err := pe.putNullableCompactString(r.ErrorMessage); err != nil {
-		return err
-	}
-
-	pe.putCompactArrayLength(len(r.Errors))
-	for topic, partitions := range r.Errors {
-		if err := pe.putCompactString(topic); err != nil {
-			return err
-		}
-		pe.putCompactArrayLength(len(partitions))
-		for partition, block := range partitions {
-			pe.putInt32(partition)
-
-			if err := block.encode(pe); err != nil {
-				return err
-			}
-		}
-		pe.putEmptyTaggedFieldArray()
-	}
-
-	pe.putEmptyTaggedFieldArray()
-	return nil
-}
-
-func (r *AlterPartitionReassignmentsResponse) decode(pd packetDecoder, version int16) (err error) {
-	r.Version = version
-
-	if r.ThrottleTimeMs, err = pd.getInt32(); err != nil {
-		return err
-	}
-
-	kerr, err := pd.getInt16()
-	if err != nil {
-		return err
-	}
-
-	r.ErrorCode = KError(kerr)
-
-	if r.ErrorMessage, err = pd.getCompactNullableString(); err != nil {
-		return err
-	}
-
-	numTopics, err := pd.getCompactArrayLength()
-	if err != nil {
-		return err
-	}
-
-	if numTopics > 0 {
-		r.Errors = make(map[string]map[int32]*alterPartitionReassignmentsErrorBlock, numTopics)
-		for i := 0; i < numTopics; i++ {
-			topic, err := pd.getCompactString()
-			if err != nil {
-				return err
-			}
-
-			ongoingPartitionReassignments, err := pd.getCompactArrayLength()
-			if err != nil {
-				return err
-			}
-
-			r.Errors[topic] = make(map[int32]*alterPartitionReassignmentsErrorBlock, ongoingPartitionReassignments)
-
-			for j := 0; j < ongoingPartitionReassignments; j++ {
-				partition, err := pd.getInt32()
-				if err != nil {
-					return err
-				}
-				block := &alterPartitionReassignmentsErrorBlock{}
-				if err := block.decode(pd); err != nil {
-					return err
-				}
-
-				r.Errors[topic][partition] = block
-			}
-			if _, err = pd.getEmptyTaggedFieldArray(); err != nil {
-				return err
-			}
-		}
-	}
-
-	if _, err = pd.getEmptyTaggedFieldArray(); err != nil {
-		return err
-	}
-
-	return nil
-}
-
-func (r *AlterPartitionReassignmentsResponse) key() int16 {
-	return 45
-}
-
-func (r *AlterPartitionReassignmentsResponse) version() int16 {
-	return r.Version
-}
-
-func (r *AlterPartitionReassignmentsResponse) headerVersion() int16 {
-	return 1
-}
-
-func (r *AlterPartitionReassignmentsResponse) requiredVersion() KafkaVersion {
-	return V2_4_0_0
-}

vendor/github.com/Shopify/sarama/api_versions_request.go

@@ -1,29 +0,0 @@
-package sarama
-
-//ApiVersionsRequest ...
-type ApiVersionsRequest struct {
-}
-
-func (a *ApiVersionsRequest) encode(pe packetEncoder) error {
-	return nil
-}
-
-func (a *ApiVersionsRequest) decode(pd packetDecoder, version int16) (err error) {
-	return nil
-}
-
-func (a *ApiVersionsRequest) key() int16 {
-	return 18
-}
-
-func (a *ApiVersionsRequest) version() int16 {
-	return 0
-}
-
-func (a *ApiVersionsRequest) headerVersion() int16 {
-	return 1
-}
-
-func (a *ApiVersionsRequest) requiredVersion() KafkaVersion {
-	return V0_10_0_0
-}

vendor/github.com/Shopify/sarama/api_versions_response.go

@@ -1,93 +0,0 @@
-package sarama
-
-//ApiVersionsResponseBlock is an api version response block type
-type ApiVersionsResponseBlock struct {
-	ApiKey     int16
-	MinVersion int16
-	MaxVersion int16
-}
-
-func (b *ApiVersionsResponseBlock) encode(pe packetEncoder) error {
-	pe.putInt16(b.ApiKey)
-	pe.putInt16(b.MinVersion)
-	pe.putInt16(b.MaxVersion)
-	return nil
-}
-
-func (b *ApiVersionsResponseBlock) decode(pd packetDecoder) error {
-	var err error
-
-	if b.ApiKey, err = pd.getInt16(); err != nil {
-		return err
-	}
-
-	if b.MinVersion, err = pd.getInt16(); err != nil {
-		return err
-	}
-
-	if b.MaxVersion, err = pd.getInt16(); err != nil {
-		return err
-	}
-
-	return nil
-}
-
-//ApiVersionsResponse is an api version response type
-type ApiVersionsResponse struct {
-	Err         KError
-	ApiVersions []*ApiVersionsResponseBlock
-}
-
-func (r *ApiVersionsResponse) encode(pe packetEncoder) error {
-	pe.putInt16(int16(r.Err))
-	if err := pe.putArrayLength(len(r.ApiVersions)); err != nil {
-		return err
-	}
-	for _, apiVersion := range r.ApiVersions {
-		if err := apiVersion.encode(pe); err != nil {
-			return err
-		}
-	}
-	return nil
-}
-
-func (r *ApiVersionsResponse) decode(pd packetDecoder, version int16) error {
-	kerr, err := pd.getInt16()
-	if err != nil {
-		return err
-	}
-
-	r.Err = KError(kerr)
-
-	numBlocks, err := pd.getArrayLength()
-	if err != nil {
-		return err
-	}
-
-	r.ApiVersions = make([]*ApiVersionsResponseBlock, numBlocks)
-	for i := 0; i < numBlocks; i++ {
-		block := new(ApiVersionsResponseBlock)
-		if err := block.decode(pd); err != nil {
-			return err
-		}
-		r.ApiVersions[i] = block
-	}
-
-	return nil
-}
-
-func (r *ApiVersionsResponse) key() int16 {
-	return 18
-}
-
-func (r *ApiVersionsResponse) version() int16 {
-	return 0
-}
-
-func (a *ApiVersionsResponse) headerVersion() int16 {
-	return 0
-}
-
-func (r *ApiVersionsResponse) requiredVersion() KafkaVersion {
-	return V0_10_0_0
-}