Fix MacOS builds

Added `--input-raw-allow-incomplete` if you really need it.

Fixed Bug when outpuut binary response not tracked
Additionally fixed bug which prevents Gor from exiting.

.deepsource.toml

@@ -0,0 +1,28 @@
+version = 1
+
+exclude_patterns = [
+  "vendor/**"
+]
+
+[[analyzers]]
+name = "go"
+enabled = true
+
+  [analyzers.meta]
+  import_paths = ["github.com/ankitdobhal/goreplay"]
+
+[[analyzers]]
+name = "docker"
+enabled = true
+
+[[analyzers]]
+name = "ruby"
+enabled = true
+
+[[analyzers]]
+name = "javascript"
+enabled = true
+
+  [analyzers.meta]
+  environment = ["nodejs"]
+

.github/dependabot.yaml

@@ -0,0 +1,10 @@
+version: 2
+updates:
+  # Maintain dependencies for GitHub Actions
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+    pull-request-branch-name:
+      # Separate sections of the branch name with a hyphen
+      separator: "-"

.github/workflows/ci-docker.yaml

@@ -0,0 +1,38 @@
+name: ci
+on:
+  release:
+    types: [published]
+jobs:
+  docker-build-and-push:
+    runs-on: ubuntu-latest
+    steps:
+      -
+        name: "Checkout repository"
+        uses: actions/checkout@v2
+      -
+        name: "Set up Docker Buildx"
+        uses: docker/setup-buildx-action@v1
+      -
+        name: "Cache Docker layers"
+        uses: actions/cache@v2
+        with:
+          path: /tmp/.buildx-cache
+          key: ${{ runner.os }}-buildx-${{ github.sha }}
+          restore-keys: |
+            ${{ runner.os }}-buildx-
+      -
+        name: "Login to Container Registry"
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      -
+        name: "Build and push docker image"
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          push: true
+          tags: ${{ github.repository }}:${{ github.event.release.tag_name }}
+          build-args: RELEASE_VERSION=${{ github.event.release.tag_name }}
+          cache-from: type=local,src=/tmp/.buildx-cache
+          cache-to: type=local,mode=max,dest=/tmp/.buildx-cache

.github/workflows/ci-test.yaml

@@ -0,0 +1,29 @@
+name: test
+on: [push, pull_request]
+jobs:
+  test:
+    strategy:
+      matrix:
+        go-version: [1.15.x, 1.16.x] # two latest minor versions
+    runs-on: ubuntu-latest
+    steps:
+    - name: update package index
+      run: sudo apt-get update
+    - name: install libpcap
+      run: sudo apt-get install libpcap-dev -y
+    - name: install Go
+      uses: actions/setup-go@v2
+      with:
+        go-version: ${{ matrix.go-version }}
+    - name: checkout code
+      uses: actions/checkout@v2
+    - uses: actions/cache@v2
+      with:
+        path: |
+          ~/go/pkg/mod              # Module download cache
+          ~/.cache/go-build         # Build cache (Linux)
+        key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
+        restore-keys: |
+          ${{ runner.os }}-go-
+    - name: test
+      run: sudo go test ./... -v -timeout 120s

.gitignore

@@ -2,9 +2,12 @@
 *.gor
 *.rpm
 *.dep
+*.deb
 *.pkg
-
+*.exe
+*.pprof
 *.out
+hey
 
 *.bin
 

.travis.yml

@@ -1,9 +0,0 @@
-dist: focal
-
-language: go
-go: 1.14
-
-script: sudo -E bash -c "source /etc/profile && eval '$(gimme 1.14)' && go test ./... -v -timeout 120s"
-
-before_install:
-  - sudo apt-get install libpcap-dev -y

Dockerfile

@@ -1,6 +1,9 @@
-FROM alpine:latest as builder
+FROM alpine:3.12 as builder
+
+ARG RELEASE_VERSION
+
 RUN apk add --no-cache ca-certificates openssl
-RUN wget https://github.com/buger/goreplay/releases/download/v1.2.0/gor_v1.2.0_x64.tar.gz -O gor.tar.gz
+RUN wget https://github.com/buger/goreplay/releases/download/${RELEASE_VERSION}/gor_${RELEASE_VERSION}_x64.tar.gz -O gor.tar.gz
 RUN tar xzf gor.tar.gz
 
 FROM scratch

Dockerfile.dev

@@ -1,9 +1,9 @@
-FROM golang:1.14
+FROM golang:1.15
 
 RUN apt-get update && apt-get install ruby vim-common -y
 
 RUN apt-get install flex bison -y
-RUN wget http://www.tcpdump.org/release/libpcap-1.9.1.tar.gz && tar xzf libpcap-1.9.1.tar.gz && cd libpcap-1.9.1 && ./configure && make install
+RUN wget http://www.tcpdump.org/release/libpcap-1.10.0.tar.gz && tar xzf libpcap-1.10.0.tar.gz && cd libpcap-1.10.0 && ./configure && make install
 
 RUN go get github.com/google/gopacket
 RUN go get -u golang.org/x/lint/golint

Makefile

@@ -23,16 +23,16 @@ FPMCOMMON= \
     -s dir \
     -C /tmp/gor-build \
 
-release: release-x64 release-mac
+release: release-x64 release-mac release-windows
 
 release-bin:
-	docker run -v `pwd`:$(SOURCE_PATH) -t --env GOOS=linux --env GOARCH=amd64  -i $(CONTAINER) go build -o $(BIN_NAME) -tags netgo $(LDFLAGS)
+	docker run -v `pwd`:$(SOURCE_PATH) -t --env GOOS=linux --env GOARCH=amd64  -i $(CONTAINER) go build -mod=mod -o $(BIN_NAME) -tags netgo $(LDFLAGS)
 
 release-bin-mac:
 	GOOS=darwin go build -o $(BIN_NAME) $(MAC_LDFLAGS)
 
 release-x64:
-	docker run -v `pwd`:$(SOURCE_PATH) -t --env GOOS=linux --env GOARCH=amd64  -i $(CONTAINER) go build -o $(BIN_NAME) -tags netgo $(LDFLAGS)
+	docker run -v `pwd`:$(SOURCE_PATH) -t --env GOOS=linux --env GOARCH=amd64  -i $(CONTAINER) go build -mod=mod -o $(BIN_NAME) -tags netgo $(LDFLAGS)
 	tar -czf gor_$(VERSION)$(PREFIX)_x64.tar.gz $(BIN_NAME)
 	mkdir -p /tmp/gor-build
 	mv ./$(BIN_NAME) /tmp/gor-build/$(BIN_NAME)
@@ -44,12 +44,12 @@ release-x64:
 	rm -rf /tmp/gor-build
 
 release-x86:
-	docker run -v `pwd`:$(SOURCE_PATH) -t --env GOOS=linux --env GOARCH=386 -i $(CONTAINER) go build -o $(BIN_NAME) -tags netgo $(LDFLAGS)
+	docker run -v `pwd`:$(SOURCE_PATH) -t --env GOOS=linux --env GOARCH=386 -i $(CONTAINER) go build -mod=mod -o $(BIN_NAME) -tags netgo $(LDFLAGS)
 	tar -czf gor_$(VERSION)$(PREFIX)_x86.tar.gz $(BIN_NAME)
 	rm $(BIN_NAME)
 
 release-mac:
-	go build -o $(BIN_NAME) $(MAC_LDFLAGS)
+	go build -mod=mod -o $(BIN_NAME) $(MAC_LDFLAGS)
 	tar -czf gor_$(VERSION)$(PREFIX)_mac.tar.gz $(BIN_NAME)
 	mkdir -p /tmp/gor-build
 	mv ./$(BIN_NAME) /tmp/gor-build/$(BIN_NAME)
@@ -58,11 +58,25 @@ release-mac:
 	fpm $(FPMCOMMON) -a amd64 -t osxpkg ./=/usr/local/bin
 	rm -rf /tmp/gor-build
 
+release-windows:
+	docker run -it --rm \
+	  -v `pwd`:/go/src/github.com/buger/goreplay \
+	  -w /go/src/github.com/buger/goreplay \
+	  -e CGO_ENABLED=1 \
+	  docker.elastic.co/beats-dev/golang-crossbuild:1.16.4-main \
+	  --build-cmd "make VERSION=$(VERSION) build" \
+	  -p "windows/amd64"
+	mv ./gor ./gor.exe
+	zip gor-$(VERSION)$(PREFIX)_windows.zip ./gor.exe
+	rm -rf ./gor.exe
+
+build:
+	go build -mod=mod -o $(BIN_NAME) $(LDFLAGS)
 
 install:
 	go install $(MAC_LDFLAGS)
 
-build:
+build-env:
 	docker build -t $(CONTAINER) -f Dockerfile.dev .
 
 profile:
@@ -102,7 +116,7 @@ profile_test:
 
 # Used mainly for debugging, because docker container do not have access to parent machine ports
 run:
-	$(RUN) go run $(LDFLAGS) $(SOURCE) --input-dummy=0 --output-http="http://localhost:9000" --input-raw-track-response --input-raw 127.0.0.1:9000 --verbose --debug --middleware "./examples/middleware/echo.sh" --output-file requests.gor
+	$(RUN) go run $(LDFLAGS) $(SOURCE) --input-dummy=0 --output-http="http://localhost:9000" --input-raw-track-response --input-raw 127.0.0.1:9000 --verbose 0 --middleware "./examples/middleware/echo.sh" --output-file requests.gor
 
 run-2:
 	$(RUN) go run $(LDFLAGS) $(SOURCE) --input-raw :8000 --input-raw-bpf-filter "dst port 8000" --output-stdout --output-http "http://localhost:8000" --input-dummy=0
@@ -130,7 +144,7 @@ bash:
 
 
 FPMCOMMON= \
-    --name goreplay \
+    --name gor \
     --description "GoReplay is an open-source network monitoring tool which can record your live traffic, and use it for shadowing, load testing, monitoring and detailed analysis." \
     -v $(VERSION) \
     --vendor "Leonid Bugaev" \

capture/af_packet.go

@@ -0,0 +1,32 @@
+// +build !linux
+
+package capture
+
+import (
+	"fmt"
+	"time"
+
+	"github.com/google/gopacket"
+)
+
+func newAfpacketHandle(device string, snaplen int, block_size int, num_blocks int,
+	useVLAN bool, timeout time.Duration) (*afpacketHandle, error) {
+	return nil, fmt.Errorf("Not implemented")
+}
+
+func afpacketComputeSize(targetSizeMb int, snaplen int, pageSize int) (
+	frameSize int, blockSize int, numBlocks int, err error) {
+	return 0, 0, 0, fmt.Errorf("Not implemented")
+}
+
+type afpacketHandle struct{}
+
+// ZeroCopyReadPacketData satisfies ZeroCopyPacketDataSource interface
+func (h *afpacketHandle) ZeroCopyReadPacketData() (data []byte, ci gopacket.CaptureInfo, err error) {
+	return nil, gopacket.CaptureInfo{}, fmt.Errorf("Not implemented")
+}
+
+// SetBPFFilter translates a BPF filter string into BPF RawInstruction and applies them.
+func (h *afpacketHandle) SetBPFFilter(filter string, snaplen int) (err error) {
+	return fmt.Errorf("Not implemented")
+}

capture/af_packet_linux.go

@@ -0,0 +1,130 @@
+// +build linux
+
+package main
+
+import (
+	"flag"
+	"fmt"
+	"log"
+	"os"
+	"runtime/pprof"
+	"time"
+
+	"github.com/google/gopacket"
+	"github.com/google/gopacket/afpacket"
+	"github.com/google/gopacket/layers"
+	"github.com/google/gopacket/pcap"
+	"golang.org/x/net/bpf"
+
+	_ "github.com/google/gopacket/layers"
+)
+
+var (
+	iface      = flag.String("i", "any", "Interface to read from")
+	cpuprofile = flag.String("cpuprofile", "", "If non-empty, write CPU profile here")
+	snaplen    = flag.Int("s", 0, "Snaplen, if <= 0, use 65535")
+	bufferSize = flag.Int("b", 8, "Interface buffersize (MB)")
+	filter     = flag.String("f", "port not 22", "BPF filter")
+	count      = flag.Int64("c", -1, "If >= 0, # of packets to capture before returning")
+	verbose    = flag.Int64("log_every", 1, "Write a log every X packets")
+	addVLAN    = flag.Bool("add_vlan", false, "If true, add VLAN header")
+)
+
+type afpacketHandle struct {
+	TPacket *afpacket.TPacket
+}
+
+func newAfpacketHandle(device string, snaplen int, block_size int, num_blocks int,
+	useVLAN bool, timeout time.Duration) (*afpacketHandle, error) {
+
+	h := &afpacketHandle{}
+	var err error
+
+	if device == "any" {
+		h.TPacket, err = afpacket.NewTPacket(
+			afpacket.OptFrameSize(snaplen),
+			afpacket.OptBlockSize(block_size),
+			afpacket.OptNumBlocks(num_blocks),
+			afpacket.OptAddVLANHeader(useVLAN),
+			afpacket.OptPollTimeout(timeout),
+			afpacket.SocketRaw,
+			afpacket.TPacketVersion3)
+	} else {
+		h.TPacket, err = afpacket.NewTPacket(
+			afpacket.OptInterface(device),
+			afpacket.OptFrameSize(snaplen),
+			afpacket.OptBlockSize(block_size),
+			afpacket.OptNumBlocks(num_blocks),
+			afpacket.OptAddVLANHeader(useVLAN),
+			afpacket.OptPollTimeout(timeout),
+			afpacket.SocketRaw,
+			afpacket.TPacketVersion3)
+	}
+	return h, err
+}
+
+// ZeroCopyReadPacketData satisfies ZeroCopyPacketDataSource interface
+func (h *afpacketHandle) ZeroCopyReadPacketData() (data []byte, ci gopacket.CaptureInfo, err error) {
+	return h.TPacket.ZeroCopyReadPacketData()
+}
+
+// SetBPFFilter translates a BPF filter string into BPF RawInstruction and applies them.
+func (h *afpacketHandle) SetBPFFilter(filter string, snaplen int) (err error) {
+	pcapBPF, err := pcap.CompileBPFFilter(layers.LinkTypeEthernet, snaplen, filter)
+	if err != nil {
+		return err
+	}
+	bpfIns := []bpf.RawInstruction{}
+	for _, ins := range pcapBPF {
+		bpfIns2 := bpf.RawInstruction{
+			Op: ins.Code,
+			Jt: ins.Jt,
+			Jf: ins.Jf,
+			K:  ins.K,
+		}
+		bpfIns = append(bpfIns, bpfIns2)
+	}
+	if h.TPacket.SetBPF(bpfIns); err != nil {
+		return err
+	}
+	return h.TPacket.SetBPF(bpfIns)
+}
+
+// LinkType returns ethernet link type.
+func (h *afpacketHandle) LinkType() layers.LinkType {
+	return layers.LinkTypeEthernet
+}
+
+// Close will close afpacket source.
+func (h *afpacketHandle) Close() {
+	h.TPacket.Close()
+}
+
+// SocketStats prints received, dropped, queue-freeze packet stats.
+func (h *afpacketHandle) SocketStats() (as afpacket.SocketStats, asv afpacket.SocketStatsV3, err error) {
+	return h.TPacket.SocketStats()
+}
+
+// afpacketComputeSize computes the block_size and the num_blocks in such a way that the
+// allocated mmap buffer is close to but smaller than target_size_mb.
+// The restriction is that the block_size must be divisible by both the
+// frame size and page size.
+func afpacketComputeSize(targetSizeMb int, snaplen int, pageSize int) (
+	frameSize int, blockSize int, numBlocks int, err error) {
+
+	if snaplen < pageSize {
+		frameSize = pageSize / (pageSize / snaplen)
+	} else {
+		frameSize = (snaplen/pageSize + 1) * pageSize
+	}
+
+	// 128 is the default from the gopacket library so just use that
+	blockSize = frameSize * 128
+	numBlocks = (targetSizeMb * 1024 * 1024) / blockSize
+
+	if numBlocks == 0 {
+		return 0, 0, 0, fmt.Errorf("Interface buffersize is too small")
+	}
+
+	return frameSize, blockSize, numBlocks, nil
+}

capture/capture.go

@@ -4,20 +4,26 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"io"
+	"log"
 	"net"
 	"os"
 	"runtime"
+	"strings"
 	"sync"
+	"syscall"
 	"time"
 
 	"github.com/buger/goreplay/size"
+	"github.com/buger/goreplay/tcp"
+
 	"github.com/google/gopacket"
 	"github.com/google/gopacket/layers"
 	"github.com/google/gopacket/pcap"
 )
 
-// Handler is a function that is used to handle packets
-type Handler func(gopacket.Packet)
+// PacketHandler is a function that is used to handle packets
+type PacketHandler func(*tcp.Packet)
 
 // PcapOptions options that can be set on a pcap capture handle,
 // these options take effect on inactive pcap handles
@@ -31,30 +37,29 @@ type PcapOptions struct {
 	Snaplen       bool          `json:"input-raw-override-snaplen"`
 }
 
-// NetInterface represents network interface
-type NetInterface struct {
-	net.Interface
-	IPs []string
-}
-
 // Listener handle traffic capture, this is its representation.
 type Listener struct {
 	sync.Mutex
 	Transport  string       // transport layer default to tcp
 	Activate   func() error // function is used to activate the engine. it must be called before reading packets
-	Handles    map[string]gopacket.PacketDataSource
-	Interfaces []NetInterface
+	Handles    map[string]packetHandle
+	Interfaces []pcap.Interface
 	loopIndex  int
 	Reading    chan bool // this channel is closed when the listener has started reading packets
 	PcapOptions
 	Engine        EngineType
-	port          uint16 // src or/and dst port
+	ports         []uint16 // src or/and dst ports
 	trackResponse bool
 
 	host string // pcap file name or interface (name, hardware addr, index or ip address)
 
-	quit    chan bool
-	packets chan gopacket.Packet
+	closeDone chan struct{}
+	quit      chan struct{}
+}
+
+type packetHandle struct {
+	handler gopacket.ZeroCopyPacketDataSource
+	ips     []net.IP
 }
 
 // EngineType ...
@@ -65,6 +70,7 @@ const (
 	EnginePcap EngineType = 1 << iota
 	EnginePcapFile
 	EngineRawSocket
+	EngineAFPacket
 )
 
 // Set is here so that EngineType can implement flag.Var
@@ -74,8 +80,10 @@ func (eng *EngineType) Set(v string) error {
 		*eng = EnginePcap
 	case "pcap_file":
 		*eng = EnginePcapFile
-	case "raw_socket", "af_packet":
+	case "raw_socket":
 		*eng = EngineRawSocket
+	case "af_packet":
+		*eng = EngineAFPacket
 	default:
 		return fmt.Errorf("invalid engine %s", v)
 	}
@@ -90,6 +98,8 @@ func (eng *EngineType) String() (e string) {
 		e = "libpcap"
 	case EngineRawSocket:
 		e = "raw_socket"
+	case EngineAFPacket:
+		e = "af_packet"
 	default:
 		e = ""
 	}
@@ -99,20 +109,24 @@ func (eng *EngineType) String() (e string) {
 // NewListener creates and initialize a new Listener. if transport or/and engine are invalid/unsupported
 // is "tcp" and "pcap", are assumed. l.Engine and l.Transport can help to get the values used.
 // if there is an error it will be associated with getting network interfaces
-func NewListener(host string, port uint16, transport string, engine EngineType, trackResponse bool) (l *Listener, err error) {
+func NewListener(host string, ports []uint16, transport string, engine EngineType, trackResponse bool) (l *Listener, err error) {
 	l = &Listener{}
 
 	l.host = host
-	l.port = port
+	if l.host == "localhost" {
+		l.host = "127.0.0.1"
+	}
+	l.ports = ports
+
 	l.Transport = "tcp"
 	if transport != "" {
 		l.Transport = transport
 	}
-	l.Handles = make(map[string]gopacket.PacketDataSource)
+	l.Handles = make(map[string]packetHandle)
 	l.trackResponse = trackResponse
-	l.packets = make(chan gopacket.Packet, 1000)
-	l.quit = make(chan bool, 1)
-	l.Reading = make(chan bool, 1)
+	l.closeDone = make(chan struct{})
+	l.quit = make(chan struct{})
+	l.Reading = make(chan bool)
 	switch engine {
 	default:
 		l.Engine = EnginePcap
@@ -120,11 +134,15 @@ func NewListener(host string, port uint16, transport string, engine EngineType,
 	case EngineRawSocket:
 		l.Engine = EngineRawSocket
 		l.Activate = l.activateRawSocket
+	case EngineAFPacket:
+		l.Engine = EngineAFPacket
+		l.Activate = l.activateAFPacket
 	case EnginePcapFile:
 		l.Engine = EnginePcapFile
 		l.Activate = l.activatePcapFile
 		return
 	}
+
 	err = l.setInterfaces()
 	if err != nil {
 		return nil, err
@@ -139,34 +157,23 @@ func (l *Listener) SetPcapOptions(opts PcapOptions) {
 }
 
 // Listen listens for packets from the handles, and call handler on every packet received
-// until the context done signal is sent or EOF on handles.
-// this function should be called after activating pcap handles
-func (l *Listener) Listen(ctx context.Context, handler Handler) (err error) {
-	l.read()
+// until the context done signal is sent or there is unrecoverable error on all handles.
+// this function must be called after activating pcap handles
+func (l *Listener) Listen(ctx context.Context, handler PacketHandler) (err error) {
+	l.read(handler)
 	done := ctx.Done()
-	var p gopacket.Packet
-	var ok bool
-	for {
-		select {
-		case <-done:
-			l.quit <- true
-			close(l.quit)
-			err = ctx.Err()
-			done = nil
-		case p, ok = <-l.packets:
-			if !ok {
-				return
-			}
-			if p == nil {
-				continue
-			}
-			handler(p)
-		}
+	select {
+	case <-done:
+		close(l.quit) // signal close on all handles
+		<-l.closeDone // wait all handles to be closed
+		err = ctx.Err()
+	case <-l.closeDone: // all handles closed voluntarily
 	}
+	return
 }
 
 // ListenBackground is like listen but can run concurrently and signal error through channel
-func (l *Listener) ListenBackground(ctx context.Context, handler Handler) chan error {
+func (l *Listener) ListenBackground(ctx context.Context, handler PacketHandler) chan error {
 	err := make(chan error, 1)
 	go func() {
 		defer close(err)
@@ -179,56 +186,49 @@ func (l *Listener) ListenBackground(ctx context.Context, handler Handler) chan e
 
 // Filter returns automatic filter applied by goreplay
 // to a pcap handle of a specific interface
-func (l *Listener) Filter(ifi NetInterface) (filter string) {
+func (l *Listener) Filter(ifi pcap.Interface) (filter string) {
 	// https://www.tcpdump.org/manpages/pcap-filter.7.html
 
-	port := fmt.Sprintf("portrange 0-%d", 1<<16-1)
-	if l.port != 0 {
-		port = fmt.Sprintf("port %d", l.port)
-	}
-	dir := " dst " // direction
-	if l.trackResponse {
-		dir = " "
-	}
-	filter = fmt.Sprintf("(%s%s%s)", l.Transport, dir, port)
+	hosts := []string{l.host}
 	if listenAll(l.host) || isDevice(l.host, ifi) {
-		return
+		hosts = interfaceAddresses(ifi)
 	}
-	filter = fmt.Sprintf("(%s%s%s and host %s)", l.Transport, dir, port, l.host)
-	return
-}
 
-// PcapDumpHandler returns a handler to write packet data in PCAP
-// format, See http://wiki.wireshark.org/Development/LibpcapFileFormathandler.
-// if link layer is invalid Ethernet is assumed
-func PcapDumpHandler(file *os.File, link layers.LinkType, debugger func(int, ...interface{})) (handler func(packet gopacket.Packet), err error) {
-	if link.String() == "" {
-		link = layers.LinkTypeEthernet
+	filter = portsFilter(l.Transport, "dst", l.ports)
+
+	if len(hosts) != 0 {
+		filter = fmt.Sprintf("((%s) and (%s))", filter, hostsFilter("dst", hosts))
+	} else {
+		filter = fmt.Sprintf("(%s)", filter)
 	}
-	w := NewWriterNanos(file)
-	err = w.WriteFileHeader(64<<10, link)
-	if err != nil {
-		return nil, err
-	}
-	return func(packet gopacket.Packet) {
-		err = w.WritePacket(packet.Metadata().CaptureInfo, packet.Data())
-		if err != nil && debugger != nil {
-			go debugger(3, err)
+
+	if l.trackResponse {
+		responseFilter := portsFilter(l.Transport, "src", l.ports)
+
+		if len(hosts) != 0 {
+			responseFilter = fmt.Sprintf("((%s) and (%s))", responseFilter, hostsFilter("src", hosts))
+		} else {
+			responseFilter = fmt.Sprintf("(%s)", responseFilter)
 		}
-	}, nil
+
+		filter = fmt.Sprintf("%s or %s", filter, responseFilter)
+	}
+
+	// filter = fmt.Sprintf("((((ip[2:2] - ((ip[0]&0xf)<<2)) - ((tcp[12]&0xf0)>>2)) != 0)) and (%s)", filter)
+
+	return
 }
 
 // PcapHandle returns new pcap Handle from dev on success.
 // this function should be called after setting all necessary options for this listener
-func (l *Listener) PcapHandle(ifi NetInterface) (handle *pcap.Handle, err error) {
+func (l *Listener) PcapHandle(ifi pcap.Interface) (handle *pcap.Handle, err error) {
 	var inactive *pcap.InactiveHandle
 	inactive, err = pcap.NewInactiveHandle(ifi.Name)
-	if inactive != nil && err != nil {
-		defer inactive.CleanUp()
-	}
 	if err != nil {
 		return nil, fmt.Errorf("inactive handle error: %q, interface: %q", err, ifi.Name)
 	}
+	defer inactive.CleanUp()
+
 	if l.TimestampType != "" {
 		var ts pcap.TimestampSource
 		ts, err = pcap.TimestampSourceFromString(l.TimestampType)
@@ -247,12 +247,22 @@ func (l *Listener) PcapHandle(ifi NetInterface) (handle *pcap.Handle, err error)
 			return nil, fmt.Errorf("monitor mode error: %q, interface: %q", err, ifi.Name)
 		}
 	}
+
 	var snap int
-	if l.Snaplen {
-		snap = 64<<10 + 200
-	} else if ifi.MTU > 0 {
-		snap = ifi.MTU + 200
+
+	if !l.Snaplen {
+		infs, _ := net.Interfaces()
+		for _, i := range infs {
+			if i.Name == ifi.Name {
+				snap = i.MTU + 200
+			}
+		}
 	}
+
+	if snap == 0 {
+		snap = 64<<10 + 200
+	}
+
 	err = inactive.SetSnapLen(snap)
 	if err != nil {
 		return nil, fmt.Errorf("snapshot length error: %q, interface: %q", err, ifi.Name)
@@ -264,7 +274,7 @@ func (l *Listener) PcapHandle(ifi NetInterface) (handle *pcap.Handle, err error)
 		}
 	}
 	if l.BufferTimeout == 0 {
-		l.BufferTimeout = pcap.BlockForever
+		l.BufferTimeout = 2000 * time.Millisecond
 	}
 	err = inactive.SetTimeout(l.BufferTimeout)
 	if err != nil {
@@ -274,13 +284,8 @@ func (l *Listener) PcapHandle(ifi NetInterface) (handle *pcap.Handle, err error)
 	if err != nil {
 		return nil, fmt.Errorf("PCAP Activate device error: %q, interface: %q", err, ifi.Name)
 	}
-	if l.BPFFilter != "" {
-		if l.BPFFilter[0] != '(' || l.BPFFilter[len(l.BPFFilter)-1] != ')' {
-			l.BPFFilter = "(" + l.BPFFilter + ")"
-		}
-	} else {
-		l.BPFFilter = l.Filter(ifi)
-	}
+	l.BPFFilter = l.Filter(ifi)
+	fmt.Println("Interface:", ifi.Name, ". BPF Filter:", l.BPFFilter)
 	err = handle.SetBPFFilter(l.BPFFilter)
 	if err != nil {
 		handle.Close()
@@ -290,21 +295,16 @@ func (l *Listener) PcapHandle(ifi NetInterface) (handle *pcap.Handle, err error)
 }
 
 // SocketHandle returns new unix ethernet handle associated with this listener settings
-func (l *Listener) SocketHandle(ifi NetInterface) (handle Socket, err error) {
-	handle, err = NewSocket(ifi.Interface)
+func (l *Listener) SocketHandle(ifi pcap.Interface) (handle Socket, err error) {
+	handle, err = NewSocket(ifi)
 	if err != nil {
 		return nil, fmt.Errorf("sock raw error: %q, interface: %q", err, ifi.Name)
 	}
 	if err = handle.SetPromiscuous(l.Promiscuous || l.Monitor); err != nil {
 		return nil, fmt.Errorf("promiscuous mode error: %q, interface: %q", err, ifi.Name)
 	}
-	if l.BPFFilter != "" {
-		if l.BPFFilter[0] != '(' || l.BPFFilter[len(l.BPFFilter)-1] != ')' {
-			l.BPFFilter = "(" + l.BPFFilter + ")"
-		}
-	} else {
-		l.BPFFilter = l.Filter(ifi)
-	}
+	l.BPFFilter = l.Filter(ifi)
+	fmt.Println("BPF Filter: ", l.BPFFilter)
 	if err = handle.SetBPFFilter(l.BPFFilter); err != nil {
 		handle.Close()
 		return nil, fmt.Errorf("BPF filter error: %q%s, interface: %q", err, l.BPFFilter, ifi.Name)
@@ -313,35 +313,70 @@ func (l *Listener) SocketHandle(ifi NetInterface) (handle Socket, err error) {
 	return
 }
 
-func (l *Listener) read() {
+func (l *Listener) read(handler PacketHandler) {
 	l.Lock()
 	defer l.Unlock()
 	for key, handle := range l.Handles {
-		var source *gopacket.PacketSource
-		linkType := layers.LinkTypeEthernet
-		if _, ok := handle.(*pcap.Handle); ok {
-			linkType = handle.(*pcap.Handle).LinkType()
-		}
-		source = gopacket.NewPacketSource(handle, linkType)
-		source.Lazy = true
-		source.NoCopy = true
-		ch := source.Packets()
-		go func(key string) {
+		go func(key string, hndl packetHandle) {
 			defer l.closeHandles(key)
+			linkSize := 14
+			linkType := int(layers.LinkTypeEthernet)
+			if _, ok := hndl.handler.(*pcap.Handle); ok {
+				linkType = int(hndl.handler.(*pcap.Handle).LinkType())
+				linkSize, ok = pcapLinkTypeLength(linkType)
+				if !ok {
+					if os.Getenv("GORDEBUG") != "0" {
+						log.Printf("can not identify link type of an interface '%s'\n", key)
+					}
+					return // can't find the linktype size
+				}
+			}
+
 			for {
 				select {
 				case <-l.quit:
 					return
-				case p, ok := <-ch:
-					if !ok {
+				default:
+					data, ci, err := hndl.handler.ZeroCopyReadPacketData()
+					if err == nil {
+						pckt, err := tcp.ParsePacket(data, linkType, linkSize, &ci, false)
+						if err == nil {
+							for _, p := range l.ports {
+								if pckt.DstPort == p {
+									for _, ip := range hndl.ips {
+										if pckt.DstIP.Equal(ip) {
+											pckt.Incoming = true
+											break
+										}
+									}
+									break
+								}
+							}
+
+							handler(pckt)
+						}
+						continue
+					}
+					if enext, ok := err.(pcap.NextError); ok && enext == pcap.NextErrorTimeoutExpired {
+						continue
+					}
+					if eno, ok := err.(syscall.Errno); ok && eno.Temporary() {
+						continue
+					}
+					if enet, ok := err.(*net.OpError); ok && (enet.Temporary() || enet.Timeout()) {
+						continue
+					}
+					if err == io.EOF || err == io.ErrClosedPipe {
+						log.Printf("stopped reading from %s interface with error %s\n", key, err)
 						return
 					}
-					l.packets <- p
+
+					log.Printf("stopped reading from %s interface with error %s\n", key, err)
+					return
 				}
 			}
-		}(key)
+		}(key, handle)
 	}
-	l.Reading <- true
 	close(l.Reading)
 }
 
@@ -349,14 +384,13 @@ func (l *Listener) closeHandles(key string) {
 	l.Lock()
 	defer l.Unlock()
 	if handle, ok := l.Handles[key]; ok {
-		if _, ok = handle.(Socket); ok {
-			handle.(Socket).Close()
-		} else {
-			handle.(*pcap.Handle).Close()
+		if c, ok := handle.handler.(io.Closer); ok {
+			c.Close()
 		}
+
 		delete(l.Handles, key)
 		if len(l.Handles) == 0 {
-			close(l.packets)
+			close(l.closeDone)
 		}
 	}
 }
@@ -371,7 +405,10 @@ func (l *Listener) activatePcap() error {
 			msg += ("\n" + e.Error())
 			continue
 		}
-		l.Handles[ifi.Name] = handle
+		l.Handles[ifi.Name] = packetHandle{
+			handler: handle,
+			ips:     interfaceIPs(ifi),
+		}
 	}
 	if len(l.Handles) == 0 {
 		return fmt.Errorf("pcap handles error:%s", msg)
@@ -392,9 +429,15 @@ func (l *Listener) activateRawSocket() error {
 			msg += ("\n" + e.Error())
 			continue
 		}
-		l.Handles[ifi.Name] = handle
+		l.Handles[ifi.Name] = packetHandle{
+			handler: handle,
+			ips:     interfaceIPs(ifi),
+		}
 	}
-	return e
+	if len(l.Handles) == 0 {
+		return fmt.Errorf("raw socket handles error:%s", msg)
+	}
+	return nil
 }
 
 func (l *Listener) activatePcapFile() (err error) {
@@ -403,92 +446,118 @@ func (l *Listener) activatePcapFile() (err error) {
 	if handle, e = pcap.OpenOffline(l.host); e != nil {
 		return fmt.Errorf("open pcap file error: %q", e)
 	}
-	if l.BPFFilter != "" {
-		if l.BPFFilter[0] != '(' || l.BPFFilter[len(l.BPFFilter)-1] != ')' {
-			l.BPFFilter = "(" + l.BPFFilter + ")"
-		}
-	} else {
-		addr := l.host
-		l.host = ""
-		l.BPFFilter = l.Filter(NetInterface{})
-		l.host = addr
-	}
+
+	tmp := l.host
+	l.host = ""
+	l.BPFFilter = l.Filter(pcap.Interface{})
+	l.host = tmp
+
 	if e = handle.SetBPFFilter(l.BPFFilter); e != nil {
 		handle.Close()
 		return fmt.Errorf("BPF filter error: %q, filter: %s", e, l.BPFFilter)
 	}
-	l.Handles["pcap_file"] = handle
+	l.Handles["pcap_file"] = packetHandle{
+		handler: handle,
+	}
 	return
 }
 
-func (l *Listener) setInterfaces() (err error) {
-	var Ifis []NetInterface
-	var ifis []net.Interface
-	ifis, err = net.Interfaces()
+func (l *Listener) activateAFPacket() error {
+	szFrame, szBlock, numBlocks, err := afpacketComputeSize(32, 32<<10, os.Getpagesize())
 	if err != nil {
 		return err
 	}
 
-	for i := 0; i < len(ifis); i++ {
-		if ifis[i].Flags&net.FlagLoopback != 0 {
-			l.loopIndex = ifis[i].Index
-		}
-		if ifis[i].Flags&net.FlagUp == 0 {
-			continue
-		}
-		var addrs []net.Addr
-		addrs, err = ifis[i].Addrs()
+	var msg string
+	for _, ifi := range l.Interfaces {
+		handle, err := newAfpacketHandle(ifi.Name, szFrame, szBlock, numBlocks, false, pcap.BlockForever)
+
 		if err != nil {
-			return err
-		}
-		if len(addrs) == 0 {
+			msg += ("\n" + err.Error())
 			continue
 		}
-		ifi := NetInterface{}
-		ifi.Interface = ifis[i]
-		ifi.IPs = make([]string, len(addrs))
-		for j, addr := range addrs {
-			ifi.IPs[j] = cutMask(addr)
+
+		l.BPFFilter = l.Filter(ifi)
+		fmt.Println("Interface:", ifi.Name, ". BPF Filter:", l.BPFFilter)
+		handle.SetBPFFilter(l.BPFFilter, 64<<10)
+
+		l.Handles[ifi.Name] = packetHandle{
+			handler: handle,
+			ips:     interfaceIPs(ifi),
 		}
-		Ifis = append(Ifis, ifi)
 	}
 
-	if listenAll(l.host) {
-		l.Interfaces = Ifis
+	if len(l.Handles) == 0 {
+		return fmt.Errorf("pcap handles error:%s", msg)
+	}
+
+	return nil
+}
+
+func (l *Listener) setInterfaces() (err error) {
+	var pifis []pcap.Interface
+	pifis, err = pcap.FindAllDevs()
+	ifis, _ := net.Interfaces()
+	if err != nil {
 		return
 	}
-	found := false
-	for _, ifi := range Ifis {
-		if isDevice(l.host, ifi) {
-			found = true
-		}
-		for _, ip := range ifi.IPs {
-			if ip == l.host {
-				found = true
+
+	for _, pi := range pifis {
+		var ni net.Interface
+		for _, i := range ifis {
+			if i.Name == pi.Name {
+				ni = i
 				break
 			}
 		}
-		if found {
-			l.Interfaces = []NetInterface{ifi}
+
+		if ni.Flags&net.FlagLoopback != 0 {
+			l.loopIndex = ni.Index
+		}
+		if ni.Flags&net.FlagUp == 0 {
+			continue
+		}
+
+		if isDevice(l.host, pi) {
+			l.Interfaces = []pcap.Interface{pi}
 			return
 		}
-	}
-	err = fmt.Errorf("can not find interface with addr, name or index %s", l.host)
-	return err
-}
 
-func cutMask(addr net.Addr) string {
-	mask := addr.String()
-	for i, v := range mask {
-		if v == '/' {
-			return mask[:i]
+		if len(pi.Addresses) != 0 {
+			l.Interfaces = append(l.Interfaces, pi)
 		}
 	}
-	return mask
+	return
 }
 
-func isDevice(addr string, ifi NetInterface) bool {
-	return addr == ifi.Name || addr == fmt.Sprintf("%d", ifi.Index) || addr == ifi.HardwareAddr.String()
+func isDevice(addr string, ifi pcap.Interface) bool {
+	if addr == ifi.Name {
+		return true
+	}
+
+	for _, _addr := range ifi.Addresses {
+		if _addr.IP.String() == addr {
+			return true
+		}
+	}
+
+	return false
+}
+
+func interfaceAddresses(ifi pcap.Interface) []string {
+	var hosts []string
+	for _, addr := range ifi.Addresses {
+		hosts = append(hosts, addr.IP.String())
+	}
+	return hosts
+}
+
+func interfaceIPs(ifi pcap.Interface) []net.IP {
+	var ips []net.IP
+	for _, addr := range ifi.Addresses {
+		ips = append(ips, addr.IP)
+	}
+	return ips
 }
 
 func listenAll(addr string) bool {
@@ -498,3 +567,47 @@ func listenAll(addr string) bool {
 	}
 	return false
 }
+
+func portsFilter(transport string, direction string, ports []uint16) string {
+	if len(ports) == 0 || ports[0] == 0 {
+		return fmt.Sprintf("%s %s portrange 0-%d", transport, direction, 1<<16-1)
+	}
+
+	var filters []string
+	for _, port := range ports {
+		filters = append(filters, fmt.Sprintf("%s %s port %d", transport, direction, port))
+	}
+	return strings.Join(filters, " or ")
+}
+
+func hostsFilter(direction string, hosts []string) string {
+	var hostsFilters []string
+	for _, host := range hosts {
+		hostsFilters = append(hostsFilters, fmt.Sprintf("%s host %s", direction, host))
+	}
+
+	return strings.Join(hostsFilters, " or ")
+}
+
+func pcapLinkTypeLength(lType int) (int, bool) {
+	switch layers.LinkType(lType) {
+	case layers.LinkTypeEthernet:
+		return 14, true
+	case layers.LinkTypeNull, layers.LinkTypeLoop:
+		return 4, true
+	case layers.LinkTypeRaw, 12, 14:
+		return 0, true
+	case layers.LinkTypeIPv4, layers.LinkTypeIPv6:
+		// (TODO:) look out for IP encapsulation?
+		return 0, true
+	case layers.LinkTypeLinuxSLL:
+		return 16, true
+	case layers.LinkTypeFDDI:
+		return 13, true
+	case 226 /*DLT_IPNET*/ :
+		// https://www.tcpdump.org/linktypes/LINKTYPE_IPNET.html
+		return 24, true
+	default:
+		return 0, false
+	}
+}

capture/capture_test.go

@@ -1,297 +0,0 @@
-package capture
-
-import (
-	"context"
-	"encoding/binary"
-	"io/ioutil"
-	"net"
-	"os"
-	"testing"
-	"time"
-
-	"github.com/google/gopacket"
-	"github.com/google/gopacket/layers"
-	"github.com/google/gopacket/pcap"
-)
-
-var LoopBack = func() net.Interface {
-	ifis, _ := net.Interfaces()
-	for _, v := range ifis {
-		if v.Flags&net.FlagLoopback != 0 {
-			return v
-		}
-	}
-	return ifis[0]
-}()
-
-func TestSetInterfaces(t *testing.T) {
-	l := &Listener{}
-	l.host = "127.0.0.1"
-	l.setInterfaces()
-	if len(l.Interfaces) != 1 {
-		t.Error("expected a single interface")
-	}
-	l.host = LoopBack.HardwareAddr.String()
-	l.setInterfaces()
-	if l.Interfaces[0].Name != LoopBack.Name && len(l.Interfaces) != 1 {
-		t.Error("interface should be loop back interface")
-	}
-	l.host = ""
-	l.setInterfaces()
-	if len(l.Interfaces) < 1 {
-		t.Error("should get all interfaces")
-	}
-}
-
-func TestBPFFilter(t *testing.T) {
-	l := &Listener{}
-	l.host = "127.0.0.1"
-	l.Transport = "tcp"
-	l.setInterfaces()
-	filter := l.Filter(l.Interfaces[0])
-	if filter != "(tcp dst portrange 0-65535 and host 127.0.0.1)" {
-		t.Error("wrong filter", filter)
-	}
-	l.port = 8000
-	l.trackResponse = true
-	filter = l.Filter(l.Interfaces[0])
-	if filter != "(tcp port 8000 and host 127.0.0.1)" {
-		t.Error("wrong filter")
-	}
-}
-
-var decodeOpts = gopacket.DecodeOptions{Lazy: true, NoCopy: true}
-
-func generateHeaders(seq uint32, length uint16) (headers [44]byte) {
-	// set ethernet headers
-	binary.BigEndian.PutUint32(headers[0:4], uint32(layers.ProtocolFamilyIPv4))
-
-	// set ip header
-	ip := headers[4:]
-	copy(ip[0:2], []byte{4<<4 | 5, 0x28<<2 | 0x00})
-	binary.BigEndian.PutUint16(ip[2:4], length+54)
-	ip[9] = uint8(layers.IPProtocolTCP)
-	copy(ip[12:16], []byte{127, 0, 0, 1})
-	copy(ip[16:], []byte{127, 0, 0, 1})
-
-	// set tcp header
-	tcp := ip[20:]
-	binary.BigEndian.PutUint16(tcp[0:2], 45678)
-	binary.BigEndian.PutUint16(tcp[2:4], 8000)
-	tcp[12] = 5 << 4
-	return
-}
-
-func randomPackets(start uint32, _len int, length uint16) []gopacket.Packet {
-	var packets = make([]gopacket.Packet, _len)
-	for i := start; i < start+uint32(_len); i++ {
-		h := generateHeaders(i, length)
-		d := make([]byte, int(length)+len(h))
-		copy(d, h[0:])
-		packet := gopacket.NewPacket(d, layers.LinkTypeLoop, decodeOpts)
-		packets[i-start] = packet
-		inf := packets[i-start].Metadata()
-		_len := len(d)
-		inf.CaptureInfo = gopacket.CaptureInfo{CaptureLength: _len, Length: _len, Timestamp: time.Now()}
-	}
-	return packets
-}
-
-func TestPcapDump(t *testing.T) {
-	f, err := ioutil.TempFile("", "pcap_file")
-	if err != nil {
-		t.Error(err)
-	}
-	waiter := make(chan bool, 1)
-	h, _ := PcapDumpHandler(f, layers.LinkTypeLoop, func(level int, a ...interface{}) {
-		if level != 3 {
-			t.Errorf("expected debug level to be 3, got %d", level)
-		}
-		waiter <- true
-	})
-	packets := randomPackets(1, 5, 5)
-	for i := 0; i < len(packets); i++ {
-		if i == 1 {
-			tcp := packets[i].Data()[4:][20:]
-			// change dst port
-			binary.BigEndian.PutUint16(tcp[2:], 8001)
-		}
-		if i == 4 {
-			inf := packets[i].Metadata()
-			inf.CaptureLength = 40
-		}
-		h(packets[i])
-	}
-	<-waiter
-	name := f.Name()
-	f.Close()
-	testPcapDumpEngine(name, t)
-}
-
-func testPcapDumpEngine(f string, t *testing.T) {
-	defer os.Remove(f)
-	l, err := NewListener(f, 8000, "", EnginePcapFile, true)
-	err = l.Activate()
-	if err != nil {
-		t.Errorf("expected error to be nil, got %q", err)
-		return
-	}
-	pckts := 0
-	ctx, cancel := context.WithCancel(context.Background())
-	defer cancel()
-	err = l.Listen(ctx, func(packet gopacket.Packet) {
-		if packet.Metadata().CaptureLength != 49 {
-			t.Errorf("expected packet length to be %d, got %d", 49, packet.Metadata().CaptureLength)
-		}
-		pckts++
-	})
-
-	if err != nil {
-		t.Errorf("expected error to be nil, got %q", err)
-	}
-	if pckts != 3 {
-		t.Errorf("expected %d packets, got %d packets", 3, pckts)
-	}
-}
-
-func TestPcapHandler(t *testing.T) {
-	l, err := NewListener(LoopBack.Name, 8000, "", EnginePcap, true)
-	if err != nil {
-		t.Errorf("expected error to be nil, got %v", err)
-		return
-	}
-	err = l.Activate()
-	if err != nil {
-		t.Errorf("expected error to be nil, got %v", err)
-		return
-	}
-	defer l.Handles[LoopBack.Name].(*pcap.Handle).Close()
-	if err != nil {
-		t.Errorf("expected error to be nil, got %v", err)
-		return
-	}
-	for i := 0; i < 5; i++ {
-		_, _ = net.Dial("tcp", "127.0.0.1:8000")
-	}
-	sts, _ := l.Handles[LoopBack.Name].(*pcap.Handle).Stats()
-	if sts.PacketsReceived < 5 {
-		t.Errorf("expected >=5 packets got %d", sts.PacketsReceived)
-	}
-}
-
-func TestSocketHandler(t *testing.T) {
-	l, err := NewListener(LoopBack.Name, 8000, "", EngineRawSocket, true)
-	err = l.Activate()
-	if err != nil {
-		return
-	}
-	defer l.Handles[LoopBack.Name].(*SockRaw).Close()
-	if err != nil {
-		t.Errorf("expected error to be nil, got %v", err)
-		return
-	}
-	for i := 0; i < 5; i++ {
-		_, _ = net.Dial("tcp", "127.0.0.1:8000")
-	}
-	sts, _ := l.Handles[LoopBack.Name].(*SockRaw).Stats()
-	if sts.Packets < 5 {
-		t.Errorf("expected >=5 packets got %d", sts.Packets)
-	}
-}
-
-func BenchmarkPcapDump(b *testing.B) {
-	f, err := ioutil.TempFile("", "pcap_file")
-	if err != nil {
-		b.Error(err)
-		return
-	}
-	now := time.Now()
-	defer os.Remove(f.Name())
-	h, _ := PcapDumpHandler(f, layers.LinkTypeLoop, nil)
-	packets := randomPackets(1, b.N, 5)
-	for i := 0; i < len(packets); i++ {
-		h(packets[i])
-	}
-	f.Close()
-	b.Logf("%d packets in %s", b.N, time.Since(now))
-}
-
-func BenchmarkPcapFile(b *testing.B) {
-	f, err := ioutil.TempFile("", "pcap_file")
-	if err != nil {
-		b.Error(err)
-		return
-	}
-	defer os.Remove(f.Name())
-	h, _ := PcapDumpHandler(f, layers.LinkTypeLoop, nil)
-	packets := randomPackets(1, b.N, 5)
-	for i := 0; i < len(packets); i++ {
-		h(packets[i])
-	}
-	name := f.Name()
-	f.Close()
-	b.ResetTimer()
-	var l *Listener
-	l, err = NewListener(name, 8000, "", EnginePcapFile, true)
-	if err != nil {
-		b.Error(err)
-		return
-	}
-	err = l.Activate()
-	if err != nil {
-		b.Error(err)
-		return
-	}
-	now := time.Now()
-	pckts := 0
-	ctx, cancel := context.WithCancel(context.Background())
-	defer cancel()
-	if err = l.Listen(ctx, func(packet gopacket.Packet) {
-		if packet.Metadata().CaptureLength != 49 {
-			b.Errorf("expected packet length to be %d, got %d", 49, packet.Metadata().CaptureLength)
-		}
-		pckts++
-	}); err != nil {
-		b.Error(err)
-	}
-	b.Logf("%d/%d packets in %s", pckts, b.N, time.Since(now))
-}
-
-func BenchmarkPcap(b *testing.B) {
-	now := time.Now()
-	var err error
-
-	l, err := NewListener(LoopBack.Name, 8000, "", EnginePcap, true)
-	if err != nil {
-		b.Errorf("expected error to be nil, got %v", err)
-		return
-	}
-	err = l.Activate()
-	if err != nil {
-		b.Errorf("expected error to be nil, got %v", err)
-		return
-	}
-	defer l.Handles[LoopBack.Name].(*pcap.Handle).Close()
-	for i := 0; i < b.N; i++ {
-		_, _ = net.Dial("tcp", "127.0.0.1:8000")
-	}
-	sts, _ := l.Handles[LoopBack.Name].(*pcap.Handle).Stats()
-	b.Logf("%d packets in %s", sts.PacketsReceived, time.Since(now))
-}
-
-func BenchmarkRawSocket(b *testing.B) {
-	now := time.Now()
-	var err error
-
-	l, err := NewListener(LoopBack.Name, 0, "", EngineRawSocket, true)
-	err = l.Activate()
-	if err != nil {
-		return
-	}
-	defer l.Handles[LoopBack.Name].(*SockRaw).Close()
-	for i := 0; i < b.N; i++ {
-		_, _ = net.Dial("tcp", "127.0.0.1:8000")
-	}
-	sts, _ := l.Handles[LoopBack.Name].(*SockRaw).Stats()
-	b.Logf("%d packets in %s", sts.Packets, time.Since(now))
-}

capture/doc.go

@@ -22,7 +22,7 @@ if err := listener.Listen(context.Background(), handler); err != nil {
 	 // handle error
 }
 // or
-errCh := listener.ListenBackground(context.Background(), handler) // runs in the backgorund
+errCh := listener.ListenBackground(context.Background(), handler) // runs in the background
 select {
 case err := <- errCh:
 	// handle error

capture/dump.go

@@ -54,7 +54,7 @@ func NewWriterNanos(w io.Writer) *Writer {
 // NewWriter returns a new writer object, for writing packet data out
 // to the given writer.  If this is a new empty writer (as opposed to
 // an append), you must call WriteFileHeader before WritePacket.
-// Packet timestamps are written witn microsecond precision.
+// Packet timestamps are written with microsecond precision.
 //
 //  // Write a new file:
 //  f, _ := os.Create("/tmp/file.pcap")

capture/sock_linux.go

@@ -1,3 +1,5 @@
+// +build linux
+
 package capture
 
 import (
@@ -44,7 +46,23 @@ type SockRaw struct {
 }
 
 // NewSocket returns new M'maped sock_raw on packet version 2.
-func NewSocket(ifi net.Interface) (*SockRaw, error) {
+func NewSocket(pifi pcap.Interface) (*SockRaw, error) {
+	var ifi net.Interface
+
+	infs, _ := net.Interfaces()
+	found := false
+	for _, i := range infs {
+		if i.Name == pifi.Name {
+			ifi = i
+			found = true
+			break
+		}
+	}
+
+	if !found {
+		return nil, fmt.Errorf("can't find matching interface")
+	}
+
 	// sock create
 	fd, err := unix.Socket(unix.AF_PACKET, unix.SOCK_RAW, int(ETHALL))
 	if err != nil {
@@ -108,7 +126,7 @@ func NewSocket(ifi net.Interface) (*SockRaw, error) {
 }
 
 // ReadPacketData implements gopacket.PacketDataSource.
-func (sock *SockRaw) ReadPacketData() (buf []byte, ci gopacket.CaptureInfo, err error) {
+func (sock *SockRaw) ZeroCopyReadPacketData() (buf []byte, ci gopacket.CaptureInfo, err error) {
 	sock.mu.Lock()
 	defer sock.mu.Unlock()
 	var tpHdr *unix.Tpacket2Hdr
@@ -197,7 +215,7 @@ func (sock *SockRaw) GetSnapLen() int {
 func (sock *SockRaw) SetBPFFilter(expr string) error {
 	sock.mu.Lock()
 	defer sock.mu.Unlock()
-	if len(expr) == 0 {
+	if expr == "" {
 		return unix.SetsockoptInt(sock.fd, unix.SOL_SOCKET, unix.SO_DETACH_FILTER, 0)
 	}
 	filter, err := pcap.CompileBPFFilter(layers.LinkTypeEthernet, sock.snaplen, expr)
@@ -217,7 +235,7 @@ func (sock *SockRaw) SetBPFFilter(expr string) error {
 	return unix.SetsockoptSockFprog(sock.fd, unix.SOL_SOCKET, unix.SO_ATTACH_FILTER, fprog)
 }
 
-// SetPromiscuous sets promiscous mode to the required value. for better result capture on all interfaces instead.
+// SetPromiscuous sets promiscuous mode to the required value. for better result capture on all interfaces instead.
 // If it is enabled, traffic not destined for the interface will also be captured.
 func (sock *SockRaw) SetPromiscuous(b bool) error {
 	sock.mu.Lock()

capture/sock_others.go

@@ -4,10 +4,11 @@ package capture
 
 import (
 	"errors"
-	"net"
+
+	"github.com/google/gopacket/pcap"
 )
 
 // NewSocket returns new M'maped sock_raw on packet version 2.
-func NewSocket(_ net.Interface) (Socket, error) {
+func NewSocket(_ pcap.Interface) (Socket, error) {
 	return nil, errors.New("afpacket socket is only available on linux")
 }

capture/socket.go

@@ -8,7 +8,7 @@ import (
 
 // Socket is any interface that defines the behaviors of Socket
 type Socket interface {
-	ReadPacketData() ([]byte, gopacket.CaptureInfo, error)
+	ZeroCopyReadPacketData() ([]byte, gopacket.CaptureInfo, error)
 	WritePacketData([]byte) error
 	SetBPFFilter(string) error
 	SetPromiscuous(bool) error

elasticsearch.go

@@ -57,7 +57,7 @@ type ESRequestResponse struct {
 //
 // Proper format is: scheme://[userinfo@]host/index_name
 // userinfo is: user[:password]
-// net/url.Parse() does not fail if scheme is not provided but actualy does not
+// net/url.Parse() does not fail if scheme is not provided but actually does not
 // handle URI properly.
 // So we must 'validate' URI format to match requirements to use net/url.Parse()
 func parseURI(URI string) (err error, index string) {
@@ -124,6 +124,7 @@ func (p *ESPlugin) RttDurationToMs(d time.Duration) int64 {
 	return int64(fl)
 }
 
+// ResponseAnalyze send req and resp to ES
 func (p *ESPlugin) ResponseAnalyze(req, resp []byte, start, stop time.Time) {
 	if len(resp) == 0 {
 		// nil http response - skipped elasticsearch export for this request
@@ -131,7 +132,6 @@ func (p *ESPlugin) ResponseAnalyze(req, resp []byte, start, stop time.Time) {
 	}
 	t := time.Now()
 	rtt := p.RttDurationToMs(stop.Sub(start))
-	req = payloadBody(req)
 
 	esResp := ESRequestResponse{
 		ReqURL:               string(proto.Path(req)),

emitter.go

@@ -1,31 +1,29 @@
 package main
 
 import (
-	"bytes"
+	"fmt"
 	"hash/fnv"
 	"io"
 	"log"
 	"sync"
 	"time"
+
+	"github.com/buger/goreplay/byteutils"
 )
 
-type emitter struct {
-	sync.Mutex
+// Emitter represents an abject to manage plugins communication
+type Emitter struct {
 	sync.WaitGroup
-	quit    chan int
 	plugins *InOutPlugins
 }
 
-// NewEmitter creates and initializes new `emitter` object.
-func NewEmitter(quit chan int) *emitter {
-	return &emitter{
-		quit: quit,
-	}
+// NewEmitter creates and initializes new Emitter object.
+func NewEmitter() *Emitter {
+	return &Emitter{}
 }
 
 // Start initialize loop for sending data from inputs to outputs
-func (e *emitter) Start(plugins *InOutPlugins, middlewareCmd string) {
-	defer e.Wait()
+func (e *Emitter) Start(plugins *InOutPlugins, middlewareCmd string) {
 	if Settings.CopyBufferSize < 1 {
 		Settings.CopyBufferSize = 5 << 20
 	}
@@ -38,147 +36,96 @@ func (e *emitter) Start(plugins *InOutPlugins, middlewareCmd string) {
 			middleware.ReadFrom(in)
 		}
 
-		// We are going only to read responses, so using same ReadFrom method
-		for _, out := range plugins.Outputs {
-			if r, ok := out.(io.Reader); ok {
-				middleware.ReadFrom(r)
-			}
-		}
+		e.plugins.Inputs = append(e.plugins.Inputs, middleware)
+		e.plugins.All = append(e.plugins.All, middleware)
 		e.Add(1)
 		go func() {
 			defer e.Done()
-			if err := CopyMulty(e.quit, middleware, plugins.Outputs...); err != nil {
-				Debug(2, "Error during copy: ", err)
-				e.Close()
-			}
-		}()
-		go func() {
-			for {
-				select {
-				case <-e.quit:
-					middleware.Close()
-					return
-				}
+			if err := CopyMulty(middleware, plugins.Outputs...); err != nil {
+				Debug(2, fmt.Sprintf("[EMITTER] error during copy: %q", err))
 			}
 		}()
 	} else {
 		for _, in := range plugins.Inputs {
 			e.Add(1)
-			go func(in io.Reader) {
+			go func(in PluginReader) {
 				defer e.Done()
-				if err := CopyMulty(e.quit, in, plugins.Outputs...); err != nil {
-					Debug(2, "Error during copy: ", err)
-					e.Close()
+				if err := CopyMulty(in, plugins.Outputs...); err != nil {
+					Debug(2, fmt.Sprintf("[EMITTER] error during copy: %q", err))
 				}
 			}(in)
 		}
-
-		for _, out := range plugins.Outputs {
-			if r, ok := out.(io.Reader); ok {
-				e.Add(1)
-				go func(r io.Reader) {
-					defer e.Done()
-					if err := CopyMulty(e.quit, r, plugins.Outputs...); err != nil {
-						Debug(2, "Error during copy: ", err)
-						e.Close()
-					}
-				}(r)
-			}
-		}
-	}
-}
-
-func (e *emitter) close() {
-	select {
-	case <-e.quit:
-	default:
-		close(e.quit)
 	}
 }
 
 // Close closes all the goroutine and waits for it to finish.
-func (e *emitter) Close() {
-	e.close()
-	for _, p := range e.plugins.Inputs {
+func (e *Emitter) Close() {
+	for _, p := range e.plugins.All {
 		if cp, ok := p.(io.Closer); ok {
 			cp.Close()
 		}
 	}
-	for _, p := range e.plugins.Outputs {
-		if cp, ok := p.(io.Closer); ok {
-			cp.Close()
-		}
+	if len(e.plugins.All) > 0 {
+		// wait for everything to stop
+		e.Wait()
 	}
-	e.plugins = nil // avoid further accidental usage
+	e.plugins.All = nil // avoid Close to make changes again
 }
 
 // CopyMulty copies from 1 reader to multiple writers
-func CopyMulty(stop chan int, src io.Reader, writers ...io.Writer) error {
-	buf := make([]byte, Settings.CopyBufferSize)
+func CopyMulty(src PluginReader, writers ...PluginWriter) error {
 	wIndex := 0
 	modifier := NewHTTPModifier(&Settings.ModifierConfig)
-	filteredRequests := make(map[string]time.Time)
-	filteredRequestsLastCleanTime := time.Now()
+	filteredRequests := make(map[string]int64)
+	filteredRequestsLastCleanTime := time.Now().UnixNano()
+	filteredCount := 0
 
-	i := 0
 	for {
-		var nr int
-		nr, err := src.Read(buf)
-
-		select {
-		case <-stop:
-			return nil
-		default:
-		}
+		msg, err := src.PluginRead()
 		if err != nil {
+			if err == ErrorStopped || err == io.EOF {
+				return nil
+			}
 			return err
 		}
-
-		_maxN := nr
-		if nr > 500 {
-			_maxN = 500
-		}
-		if nr > 0 {
-			payload := buf[:nr]
-			meta := payloadMeta(payload)
+		if msg != nil && len(msg.Data) > 0 {
+			if len(msg.Data) > int(Settings.CopyBufferSize) {
+				msg.Data = msg.Data[:Settings.CopyBufferSize]
+			}
+			meta := payloadMeta(msg.Meta)
 			if len(meta) < 3 {
-				Debug(2, "[EMITTER] Found malformed record", string(payload[0:_maxN]), nr, "from:", src)
+				Debug(2, fmt.Sprintf("[EMITTER] Found malformed record %q from %q", msg.Meta, src))
 				continue
 			}
-			requestID := string(meta[1])
-
-			Debug(3, "[EMITTER] input:", string(payload[0:_maxN]), nr, "from:", src)
-
+			requestID := byteutils.SliceToString(meta[1])
+			// start a subroutine only when necessary
+			if Settings.Verbose >= 3 {
+				Debug(3, "[EMITTER] input: ", byteutils.SliceToString(msg.Meta[:len(msg.Meta)-1]), " from: ", src)
+			}
 			if modifier != nil {
-				if isRequestPayload(payload) {
-					headSize := bytes.IndexByte(payload, '\n') + 1
-					body := payload[headSize:]
-					originalBodyLen := len(body)
-					body = modifier.Rewrite(body)
-
+				Debug(3, "[EMITTER] modifier:", requestID, "from:", src)
+				if isRequestPayload(msg.Meta) {
+					msg.Data = modifier.Rewrite(msg.Data)
 					// If modifier tells to skip request
-					if len(body) == 0 {
-						filteredRequests[requestID] = time.Now()
+					if len(msg.Data) == 0 {
+						filteredRequests[requestID] = time.Now().UnixNano()
+						filteredCount++
 						continue
 					}
-
-					if originalBodyLen != len(body) {
-						payload = append(payload[:headSize], body...)
-					}
-
-					Debug(3, "[EMITTER] Rewritten input:", len(payload), "First %d bytes:", _maxN, string(payload[0:_maxN]))
+					Debug(3, "[EMITTER] Rewritten input:", requestID, "from:", src)
 
 				} else {
 					if _, ok := filteredRequests[requestID]; ok {
 						delete(filteredRequests, requestID)
+						filteredCount--
 						continue
 					}
 				}
 			}
 
 			if Settings.PrettifyHTTP {
-				payload = prettifyHTTP(payload)
-				if len(payload) == 0 {
+				msg.Data = prettifyHTTP(msg.Data)
+				if len(msg.Data) == 0 {
 					continue
 				}
 			}
@@ -189,27 +136,23 @@ func CopyMulty(stop chan int, src io.Reader, writers ...io.Writer) error {
 						log.Fatal("Detailed TCP sessions work only with PRO license")
 					}
 					hasher := fnv.New32a()
-					// First 20 bytes contain tcp session
-					id := payloadID(payload)
-					hasher.Write(id)
+					hasher.Write(meta[1])
 
 					wIndex = int(hasher.Sum32()) % len(writers)
-					writers[wIndex].Write(payload)
+					if _, err := writers[wIndex].PluginWrite(msg); err != nil {
+						return err
+					}
 				} else {
 					// Simple round robin
-					if _, err := writers[wIndex].Write(payload); err != nil {
+					if _, err := writers[wIndex].PluginWrite(msg); err != nil {
 						return err
 					}
 
-					wIndex++
-
-					if wIndex >= len(writers) {
-						wIndex = 0
-					}
+					wIndex = (wIndex + 1) % len(writers)
 				}
 			} else {
 				for _, dst := range writers {
-					if _, err := dst.Write(payload); err != nil {
+					if _, err := dst.PluginWrite(msg); err != nil {
 						return err
 					}
 				}
@@ -217,19 +160,18 @@ func CopyMulty(stop chan int, src io.Reader, writers ...io.Writer) error {
 		}
 
 		// Run GC on each 1000 request
-		if i%1000 == 0 {
+		if filteredCount > 0 && filteredCount%1000 == 0 {
 			// Clean up filtered requests for which we didn't get a response to filter
-			now := time.Now()
-			if now.Sub(filteredRequestsLastCleanTime) > 60*time.Second {
+			now := time.Now().UnixNano()
+			if now-filteredRequestsLastCleanTime > int64(60*time.Second) {
 				for k, v := range filteredRequests {
-					if now.Sub(v) > 60*time.Second {
+					if now-v > int64(60*time.Second) {
 						delete(filteredRequests, k)
+						filteredCount--
 					}
 				}
-				filteredRequestsLastCleanTime = time.Now()
+				filteredRequestsLastCleanTime = time.Now().UnixNano()
 			}
 		}
-
-		i++
 	}
 }

emitter_test.go

@@ -2,7 +2,6 @@ package main
 
 import (
 	"fmt"
-	"io"
 	"os"
 	"sync"
 	"sync/atomic"
@@ -18,20 +17,19 @@ func TestMain(m *testing.M) {
 
 func TestEmitter(t *testing.T) {
 	wg := new(sync.WaitGroup)
-	quit := make(chan int)
 
 	input := NewTestInput()
-	output := NewTestOutput(func(data []byte) {
+	output := NewTestOutput(func(*Message) {
 		wg.Done()
 	})
 
 	plugins := &InOutPlugins{
-		Inputs:  []io.Reader{input},
-		Outputs: []io.Writer{output},
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{output},
 	}
 	plugins.All = append(plugins.All, input, output)
 
-	emitter := NewEmitter(quit)
+	emitter := NewEmitter()
 	go emitter.Start(plugins, Settings.Middleware)
 
 	for i := 0; i < 1000; i++ {
@@ -45,32 +43,31 @@ func TestEmitter(t *testing.T) {
 
 func TestEmitterFiltered(t *testing.T) {
 	wg := new(sync.WaitGroup)
-	quit := make(chan int)
 
 	input := NewTestInput()
 	input.skipHeader = true
 
-	output := NewTestOutput(func(data []byte) {
+	output := NewTestOutput(func(*Message) {
 		wg.Done()
 	})
 
 	plugins := &InOutPlugins{
-		Inputs:  []io.Reader{input},
-		Outputs: []io.Writer{output},
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{output},
 	}
 	plugins.All = append(plugins.All, input, output)
 
 	methods := HTTPMethods{[]byte("GET")}
 	Settings.ModifierConfig = HTTPModifierConfig{Methods: methods}
 
-	emitter := &emitter{quit: quit}
+	emitter := &Emitter{}
 	go emitter.Start(plugins, "")
 
 	wg.Add(2)
 
 	id := uuid()
 	reqh := payloadHeader(RequestPayload, id, time.Now().UnixNano(), -1)
-	reqb := append(reqh, []byte("GET / HTTP/1.1\r\nHost: www.w3.org\r\nUser-Agent: Go 1.1 package http\r\nAccept-Encoding: gzip\r\n\r\n")...)
+	reqb := append(reqh, []byte("POST / HTTP/1.1\r\nHost: www.w3.org\r\nUser-Agent: Go 1.1 package http\r\nAccept-Encoding: gzip\r\n\r\n")...)
 
 	resh := payloadHeader(ResponsePayload, id, time.Now().UnixNano()+1, 1)
 	respb := append(resh, []byte("HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n")...)
@@ -80,7 +77,7 @@ func TestEmitterFiltered(t *testing.T) {
 
 	id = uuid()
 	reqh = payloadHeader(RequestPayload, id, time.Now().UnixNano(), -1)
-	reqb = append(reqh, []byte("POST / HTTP/1.1\r\nHost: www.w3.org\r\nUser-Agent: Go 1.1 package http\r\nAccept-Encoding: gzip\r\n\r\n")...)
+	reqb = append(reqh, []byte("GET / HTTP/1.1\r\nHost: www.w3.org\r\nUser-Agent: Go 1.1 package http\r\nAccept-Encoding: gzip\r\n\r\n")...)
 
 	resh = payloadHeader(ResponsePayload, id, time.Now().UnixNano()+1, 1)
 	respb = append(resh, []byte("HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n")...)
@@ -96,30 +93,29 @@ func TestEmitterFiltered(t *testing.T) {
 
 func TestEmitterSplitRoundRobin(t *testing.T) {
 	wg := new(sync.WaitGroup)
-	quit := make(chan int)
 
 	input := NewTestInput()
 
 	var counter1, counter2 int32
 
-	output1 := NewTestOutput(func(data []byte) {
+	output1 := NewTestOutput(func(*Message) {
 		atomic.AddInt32(&counter1, 1)
 		wg.Done()
 	})
 
-	output2 := NewTestOutput(func(data []byte) {
+	output2 := NewTestOutput(func(*Message) {
 		atomic.AddInt32(&counter2, 1)
 		wg.Done()
 	})
 
 	plugins := &InOutPlugins{
-		Inputs:  []io.Reader{input},
-		Outputs: []io.Writer{output1, output2},
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{output1, output2},
 	}
 
 	Settings.SplitOutput = true
 
-	emitter := NewEmitter(quit)
+	emitter := NewEmitter()
 	go emitter.Start(plugins, Settings.Middleware)
 
 	for i := 0; i < 1000; i++ {
@@ -140,31 +136,30 @@ func TestEmitterSplitRoundRobin(t *testing.T) {
 
 func TestEmitterRoundRobin(t *testing.T) {
 	wg := new(sync.WaitGroup)
-	quit := make(chan int)
 
 	input := NewTestInput()
 
 	var counter1, counter2 int32
 
-	output1 := NewTestOutput(func(data []byte) {
-		atomic.AddInt32(&counter1, 1)
+	output1 := NewTestOutput(func(*Message) {
+		counter1++
 		wg.Done()
 	})
 
-	output2 := NewTestOutput(func(data []byte) {
-		atomic.AddInt32(&counter2, 1)
+	output2 := NewTestOutput(func(*Message) {
+		counter2++
 		wg.Done()
 	})
 
 	plugins := &InOutPlugins{
-		Inputs:  []io.Reader{input},
-		Outputs: []io.Writer{output1, output2},
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{output1, output2},
 	}
 	plugins.All = append(plugins.All, input, output1, output2)
 
 	Settings.SplitOutput = true
 
-	emitter := NewEmitter(quit)
+	emitter := NewEmitter()
 	go emitter.Start(plugins, Settings.Middleware)
 
 	for i := 0; i < 1000; i++ {
@@ -186,36 +181,34 @@ func TestEmitterSplitSession(t *testing.T) {
 	wg := new(sync.WaitGroup)
 	wg.Add(200)
 
-	quit := make(chan int)
-
 	input := NewTestInput()
 	input.skipHeader = true
 
 	var counter1, counter2 int32
 
-	output1 := NewTestOutput(func(data []byte) {
-		if payloadID(data)[0] == 'a' {
-			atomic.AddInt32(&counter1, 1)
+	output1 := NewTestOutput(func(msg *Message) {
+		if payloadID(msg.Meta)[0] == 'a' {
+			counter1++
 		}
 		wg.Done()
 	})
 
-	output2 := NewTestOutput(func(data []byte) {
-		if payloadID(data)[0] == 'b' {
-			atomic.AddInt32(&counter2, 1)
+	output2 := NewTestOutput(func(msg *Message) {
+		if payloadID(msg.Meta)[0] == 'b' {
+			counter2++
 		}
 		wg.Done()
 	})
 
 	plugins := &InOutPlugins{
-		Inputs:  []io.Reader{input},
-		Outputs: []io.Writer{output1, output2},
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{output1, output2},
 	}
 
 	Settings.SplitOutput = true
 	Settings.RecognizeTCPSessions = true
 
-	emitter := NewEmitter(quit)
+	emitter := NewEmitter()
 	go emitter.Start(plugins, Settings.Middleware)
 
 	for i := 0; i < 200; i++ {
@@ -242,21 +235,20 @@ func TestEmitterSplitSession(t *testing.T) {
 
 func BenchmarkEmitter(b *testing.B) {
 	wg := new(sync.WaitGroup)
-	quit := make(chan int)
 
 	input := NewTestInput()
 
-	output := NewTestOutput(func(data []byte) {
+	output := NewTestOutput(func(*Message) {
 		wg.Done()
 	})
 
 	plugins := &InOutPlugins{
-		Inputs:  []io.Reader{input},
-		Outputs: []io.Writer{output},
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{output},
 	}
 	plugins.All = append(plugins.All, input, output)
 
-	emitter := NewEmitter(quit)
+	emitter := NewEmitter()
 	go emitter.Start(plugins, Settings.Middleware)
 
 	b.ResetTimer()

examples/middleware/echo.sh

@@ -8,7 +8,7 @@
 # 
 
 function log {
-    if $GOR_TEST != ""; then # if we are not testing
+    if [[ ! -v GOR_TEST ]]; then # if we are not testing
     # Logging to stderr, because stdout/stdin used for data transfer
     >&2 echo "[DEBUG][ECHO] $1"
     fi

examples/middleware/token_modifier.go

@@ -115,7 +115,7 @@ func encode(buf []byte) []byte {
 }
 
 func Debug(args ...interface{}) {
-	if os.Getenv("GOR_TEST") != "" { // if we are not testing
+	if os.Getenv("GOR_TEST") == "" { // if we are not testing
 		fmt.Fprint(os.Stderr, "[DEBUG][TOKEN-MOD] ")
 		fmt.Fprintln(os.Stderr, args...)
 	}

go.mod

@@ -1,6 +1,6 @@
 module github.com/buger/goreplay
 
-go 1.14
+go 1.15
 
 require (
 	github.com/Shopify/sarama v1.26.4
@@ -8,12 +8,13 @@ require (
 	github.com/aws/aws-sdk-go v1.33.2
 	github.com/bitly/go-hostpool v0.1.0 // indirect
 	github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 // indirect
-	github.com/google/gopacket v1.1.18
+	github.com/google/gopacket v1.1.20-0.20210429153827-3eaba0894325
 	github.com/klauspost/compress v1.10.10 // indirect
 	github.com/mattbaird/elastigo v0.0.0-20170123220020-2fe47fd29e4b
 	github.com/pierrec/lz4 v2.5.2+incompatible // indirect
 	github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0 // indirect
 	github.com/smartystreets/goconvey v1.6.4 // indirect
-	golang.org/x/net v0.0.0-20200707034311-ab3426394381 // indirect
+	github.com/stretchr/testify v1.5.1
+	golang.org/x/net v0.0.0-20200707034311-ab3426394381
 	golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd
 )

go.sum

@@ -10,7 +10,6 @@ github.com/bitly/go-hostpool v0.1.0 h1:XKmsF6k5el6xHG3WPJ8U0Ku/ye7njX7W81Ng7O2io
 github.com/bitly/go-hostpool v0.1.0/go.mod h1:4gOCgp6+NZnVqlKyZ/iBZFTAJKembaVENUpMkpg42fw=
 github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 h1:DDGfHa7BWjL4YnC6+E63dPcxHo2sUxDIu8g3QgEJdRY=
 github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4=
-github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -30,8 +29,8 @@ github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEW
 github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4=
 github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/gopacket v1.1.18 h1:lum7VRA9kdlvBi7/v2p7/zcbkduHaCH/SVVyurs7OpY=
-github.com/google/gopacket v1.1.18/go.mod h1:UdDNZ1OO62aGYVnPhxT1U6aI7ukYtA/kB8vaU0diBUM=
+github.com/google/gopacket v1.1.20-0.20210429153827-3eaba0894325 h1:YmIcZ5Var3BAQ64AW98Iiys5Ih4fiU0xK41+8isC5Ec=
+github.com/google/gopacket v1.1.20-0.20210429153827-3eaba0894325/go.mod h1:riddUzxTSBpJXk3qBHtYr4qOhFhT6k/1c0E3qkQjQpA=
 github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8=
 github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
 github.com/hashicorp/go-uuid v1.0.2 h1:cfejS+Tpcp13yd5nYHWDI6qVCny6wyX2Mt5SGur2IGE=
@@ -42,7 +41,6 @@ github.com/jmespath/go-jmespath v0.3.0 h1:OS12ieG61fsCg5+qLJ+SsW9NicxNkg3b25OyT2
 github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik=
 github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
 github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
-github.com/klauspost/compress v1.9.8 h1:VMAMUUOh+gaxKTMk+zqbjsSjsIcUcL/LF4o63i82QyA=
 github.com/klauspost/compress v1.9.8/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
 github.com/klauspost/compress v1.10.10 h1:a/y8CglcM7gLGYmlbP/stPE5sR3hbhFRUjCBfd/0B3I=
 github.com/klauspost/compress v1.10.10/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
@@ -71,33 +69,40 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4=
 github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
+github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE=
+github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU=
+github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0=
 github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I=
 github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/crypto v0.0.0-20200204104054-c9f3fb736b72 h1:+ELyKg6m8UBf0nPFSqD0mi7zUfwPyXo23HNjMnXPz7w=
+golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20200204104054-c9f3fb736b72/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
+golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20200202094626-16171245cfb2 h1:CCH4IOTTfewWjGOlSp+zGcjutRKlBEZQ6wTn8ozI/nI=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200707034311-ab3426394381 h1:VXak5I6aEWmAXeQjA+QSZzlgNrpq9mjcfDemuexIKsU=
 golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20190405154228-4b34438f7a67/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190606203320-7fc4e5ec1444/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200217220822-9197077df867/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884=
 golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
@@ -111,7 +116,6 @@ gopkg.in/jcmturner/gokrb5.v7 v7.5.0 h1:a9tsXlIDD9SKxotJMK3niV7rPZAJeX2aD/0yg3qlI
 gopkg.in/jcmturner/gokrb5.v7 v7.5.0/go.mod h1:l8VISx+WGYp+Fp7KRbsiUuXTTOnxIc3Tuvyavf11/WM=
 gopkg.in/jcmturner/rpc.v1 v1.1.0 h1:QHIUxTX1ISuAv9dD2wJ9HWQVuWDX/Zc0PfeC2tjc4rU=
 gopkg.in/jcmturner/rpc.v1 v1.1.0/go.mod h1:YIdkC4XfD6GXbzje11McwsDuOlZQSb9W4vfLvuNnlv8=
-gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=
 gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

gor.go

@@ -4,7 +4,6 @@ package main
 
 import (
 	"flag"
-	"fmt"
 	"log"
 	"net/http"
 	"net/http/httputil"
@@ -12,7 +11,7 @@ import (
 	"os"
 	"os/signal"
 	"runtime"
-	_ "runtime/debug"
+	"runtime/debug"
 	"runtime/pprof"
 	"syscall"
 	"time"
@@ -23,8 +22,13 @@ var (
 	memprofile = flag.String("memprofile", "", "write memory profile to this file")
 )
 
-func loggingMiddleware(next http.Handler) http.Handler {
+func loggingMiddleware(addr string, next http.Handler) http.Handler {
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path == "/loop" {
+			_, err := http.Get("http://" + addr)
+			log.Println(err)
+		}
+
 		rb, _ := httputil.DumpRequest(r, false)
 		log.Println(string(rb))
 		next.ServeHTTP(w, r)
@@ -32,10 +36,14 @@ func loggingMiddleware(next http.Handler) http.Handler {
 }
 
 func main() {
-	if len(os.Getenv("GOMAXPROCS")) == 0 {
+	if os.Getenv("GOMAXPROCS") == "" {
 		runtime.GOMAXPROCS(runtime.NumCPU() * 2)
 	}
 
+	if os.Getenv("GOGC") == "" {
+		debug.SetGCPercent(500)
+	}
+
 	args := os.Args[1:]
 	var plugins *InOutPlugins
 	if len(args) > 0 && args[0] == "file-server" {
@@ -46,7 +54,7 @@ func main() {
 
 		Debug(0, "Started example file server for current directory on address ", args[1])
 
-		log.Fatal(http.ListenAndServe(args[1], loggingMiddleware(http.FileServer(http.Dir(dir)))))
+		log.Fatal(http.ListenAndServe(args[1], loggingMiddleware(args[1], http.FileServer(http.Dir(dir)))))
 	} else {
 		flag.Parse()
 		checkSettings()
@@ -74,13 +82,13 @@ func main() {
 	}
 
 	closeCh := make(chan int)
-	emitter := NewEmitter(closeCh)
+	emitter := NewEmitter()
 	go emitter.Start(plugins, Settings.Middleware)
 	if Settings.ExitAfter > 0 {
 		log.Printf("Running gor for a duration of %s\n", Settings.ExitAfter)
 
 		time.AfterFunc(Settings.ExitAfter, func() {
-			fmt.Printf("gor run timeout %s\n", Settings.ExitAfter)
+			log.Printf("gor run timeout %s\n", Settings.ExitAfter)
 			close(closeCh)
 		})
 	}
@@ -95,7 +103,6 @@ func main() {
 	}
 	emitter.Close()
 	os.Exit(exit)
-
 }
 
 func profileCPU(cpuprofile string) {

http_client.go

@@ -1,521 +0,0 @@
-package main
-
-import (
-	"bufio"
-	"bytes"
-	"crypto/tls"
-	"encoding/base64"
-	"io"
-	"log"
-	"net"
-	"net/http"
-	"net/http/httputil"
-	"net/url"
-	"runtime/debug"
-	"strconv"
-	"strings"
-	"sync"
-	"syscall"
-	"time"
-
-	"github.com/buger/goreplay/proto"
-)
-
-var httpMu sync.Mutex
-
-const (
-	readChunkSize   = 64 * 1024
-	maxResponseSize = 1073741824
-)
-
-var chunkedSuffix = []byte("0\r\n\r\n")
-
-var defaultPorts = map[string]string{
-	"http":  "80",
-	"https": "443",
-}
-
-type HTTPClientConfig struct {
-	FollowRedirects    int
-	Debug              bool
-	OriginalHost       bool
-	ConnectionTimeout  time.Duration
-	Timeout            time.Duration
-	ResponseBufferSize int
-	CompatibilityMode  bool
-}
-
-type HTTPClient struct {
-	baseURL        string
-	scheme         string
-	host           string
-	auth           string
-	conn           net.Conn
-	proxy          *url.URL
-	proxyAuth      string
-	respBuf        []byte
-	config         *HTTPClientConfig
-	goClient       *http.Client
-	redirectsCount int
-}
-
-func NewHTTPClient(baseURL string, config *HTTPClientConfig) *HTTPClient {
-	if !strings.HasPrefix(baseURL, "http") {
-		baseURL = "http://" + baseURL
-	}
-
-	u, _ := url.Parse(baseURL)
-
-	if config.Timeout == 0 {
-		config.Timeout = time.Second
-	}
-
-	config.ConnectionTimeout = config.Timeout
-
-	if config.ResponseBufferSize == 0 {
-		config.ResponseBufferSize = 100 * 1024 // 100kb
-	}
-
-	client := new(HTTPClient)
-	client.baseURL = u.String()
-	client.host = u.Host
-	client.scheme = u.Scheme
-	client.respBuf = make([]byte, config.ResponseBufferSize)
-	client.config = config
-
-	if config.CompatibilityMode {
-		client.goClient = &http.Client{
-			// #TODO
-			// CheckRedirect: redirectPolicyFunc,
-		}
-	}
-
-	if u.User != nil {
-		client.auth = "Basic " + base64.StdEncoding.EncodeToString([]byte(u.User.String()))
-	}
-
-	client.proxy, _ = http.ProxyFromEnvironment(&http.Request{URL: u})
-
-	if client.isProxy() && client.proxy.User != nil {
-		client.proxyAuth = "Basic " + base64.StdEncoding.EncodeToString([]byte(client.proxy.User.String()))
-	}
-
-	return client
-}
-
-func (c *HTTPClient) Connect() (err error) {
-	c.Disconnect()
-
-	var toDial string
-	if !strings.Contains(c.host, ":") {
-		toDial = c.host + ":" + defaultPorts[c.scheme]
-	} else {
-		toDial = c.host
-	}
-
-	if c.isProxy() {
-		if c.proxy.Scheme != "http" {
-			panic("Unsupported HTTP Proxy method")
-		}
-		Debug(3, "[HTTPClient] Connecting to proxy", c.proxy.String(), "<>", toDial)
-		c.conn, err = net.DialTimeout("tcp", c.proxy.Host, c.config.ConnectionTimeout)
-		if err != nil {
-			return
-		}
-		if c.scheme == "https" {
-			c.conn.Write([]byte("CONNECT " + toDial + " HTTP/1.1\r\n"))
-			if c.proxyAuth != "" {
-				c.conn.Write([]byte("Proxy-Authorization: " + c.proxyAuth + "\r\n"))
-			}
-			c.conn.Write([]byte("\r\n"))
-			br := bufio.NewReader(c.conn)
-			l, _, err := br.ReadLine()
-			if err != nil {
-				return err
-			}
-			if len(l) < 12 {
-				panic("HTTP proxy did not respond correctly")
-			}
-			status := l[9:12]
-			if !bytes.Equal(status, []byte("200")) {
-				panic("HTTP proxy did not respond correctly")
-			}
-			for {
-				// Read until we find the empty line
-				l, _, err := br.ReadLine()
-				if err != nil {
-					return err
-				}
-				if len(l) == 0 {
-					break
-				}
-			}
-		}
-		Debug(3, "[HTTPClient] Proxy successfully connected")
-	} else {
-		c.conn, err = net.DialTimeout("tcp", toDial, c.config.ConnectionTimeout)
-		if err != nil {
-			return
-		}
-	}
-
-	if c.scheme == "https" {
-		// Wrap our socket in TLS
-		Debug(3, "[HTTPClient] Wrapping socket in TLS", c.host)
-		tlsConn := tls.Client(c.conn, &tls.Config{InsecureSkipVerify: true, ServerName: c.host})
-
-		if err = tlsConn.Handshake(); err != nil {
-			return
-		}
-
-		c.conn = tlsConn
-		Debug(3, "[HTTPClient] Successfully wrapped in TLS")
-	}
-
-	return
-}
-
-func (c *HTTPClient) Disconnect() {
-	if c.conn != nil {
-		c.conn.Close()
-		c.conn = nil
-		Debug(3, "[HTTP] Disconnected: ", c.baseURL)
-	}
-}
-
-func (c *HTTPClient) isAlive(readBytes *int) bool {
-	// Ready 1 byte from socket without timeout to check if it not closed
-	c.conn.SetReadDeadline(time.Now().Add(time.Millisecond))
-	n, err := c.conn.Read(c.respBuf[:1])
-
-	if err == io.EOF {
-		Debug(3, "[HTTPClient] connection closed, reconnecting")
-		return false
-	}
-
-	if err == syscall.EPIPE {
-		Debug(3, "Detected broken pipe.", err)
-		return false
-	}
-	if n != 0 {
-		*readBytes += n
-		Debug(3, "[HTTPClient] isAlive readBytes ", *readBytes)
-	}
-	return true
-}
-
-func (c *HTTPClient) SendGoClient(data []byte) ([]byte, error) {
-	var req *http.Request
-	var resp *http.Response
-	var err error
-
-	req, err = http.ReadRequest(bufio.NewReader(bytes.NewBuffer(data)))
-	if err != nil {
-		return nil, err
-	}
-
-	if !c.config.OriginalHost {
-		req.Host = c.host
-	}
-
-	if c.auth != "" {
-		req.Header.Add("Authorization", c.auth)
-	}
-
-	req.URL, _ = url.ParseRequestURI(c.scheme + "://" + c.host + req.RequestURI)
-	req.RequestURI = ""
-
-	resp, err = c.goClient.Do(req)
-	if err != nil {
-		return nil, err
-	}
-
-	return httputil.DumpResponse(resp, true)
-}
-
-func (c *HTTPClient) Send(data []byte) (response []byte, err error) {
-	// Don't exit on panic
-	defer func() {
-		if r := recover(); r != nil {
-			Debug(3, "[HTTPClient]", r, string(data))
-
-			if _, ok := r.(error); ok {
-				log.Println("[HTTPClient] Failed to send request: ", string(data))
-				log.Println("[HTTPClient] Response: ", string(response))
-				log.Println("PANIC: pkg:", r, string(debug.Stack()))
-			}
-		}
-	}()
-
-	if c.config.CompatibilityMode {
-		return c.SendGoClient(data)
-	}
-
-	var readBytes int
-	if c.conn == nil || !c.isAlive(&readBytes) {
-		Debug(3, "[HTTPClient] Connecting:", c.baseURL)
-		if err = c.Connect(); err != nil {
-			Debug(1, "[HTTPClient] Connection error:", err)
-			response = errorPayload(HTTP_CONNECTION_ERROR)
-			return
-		}
-	}
-
-	timeout := time.Now().Add(c.config.Timeout)
-
-	c.conn.SetWriteDeadline(timeout)
-
-	if !c.config.OriginalHost {
-		data = proto.SetHost(data, []byte(c.baseURL), []byte(c.host))
-	}
-
-	if c.isProxy() && c.scheme == "http" {
-		path := proto.Path(data)
-		if len(path) > 0 && path[0] == '/' {
-			data = proto.SetPath(data, c.proxyPath(path))
-			if c.proxyAuth != "" {
-				data = proto.SetHeader(data, []byte("Proxy-Authorization"), []byte(c.proxyAuth))
-			}
-		}
-	}
-
-	if c.auth != "" {
-		data = proto.SetHeader(data, []byte("Authorization"), []byte(c.auth))
-	}
-
-	if c.config.Debug {
-		Debug(3, "[HTTPClient] Sending:", string(data))
-	}
-
-	return c.send(data, readBytes, timeout)
-}
-
-func (c *HTTPClient) send(data []byte, readBytes int, timeout time.Time) (response []byte, err error) {
-	var payload []byte
-	var n int
-	if _, err = c.conn.Write(data); err != nil {
-		Debug(1, "[HTTPClient] Write error:", err, c.baseURL)
-		response = errorPayload(HTTP_TIMEOUT)
-		c.Disconnect()
-		return
-	}
-
-	var currentChunk []byte
-	timeout = time.Now().Add(c.config.Timeout)
-	chunked := false
-	contentLength := -1
-	currentContentLength := 0
-	chunks := 0
-
-	for {
-		c.conn.SetReadDeadline(timeout)
-
-		if readBytes < len(c.respBuf) {
-			n, err = c.conn.Read(c.respBuf[readBytes:])
-			readBytes += n
-			chunks++
-
-			// First chunk
-			if chunked || contentLength != -1 {
-				currentContentLength += n
-			} else {
-				// If headers are finished
-				var firstEmptyLine = bytes.Index(c.respBuf[:readBytes], proto.EmptyLine)
-				if firstEmptyLine != -1 {
-					if bytes.Equal(proto.Header(c.respBuf[:readBytes], []byte("Transfer-Encoding")), []byte("chunked")) {
-						chunked = true
-					} else {
-						status, _ := strconv.Atoi(string(proto.Status(c.respBuf[:readBytes])))
-						// We want to soak up all 100 Continues received to get the real result code
-						if status >= 100 && status < 200 {
-							timeout = time.Now().Add(c.config.Timeout)
-							var deleteLen = firstEmptyLine + len(proto.EmptyLine)
-							copy(c.respBuf, c.respBuf[deleteLen:readBytes])
-							readBytes -= deleteLen
-							chunks--
-							continue
-						} else if status == 204 || status == 304 {
-							contentLength = 0
-							break
-						} else {
-							l := proto.Header(c.respBuf[:readBytes], []byte("Content-Length"))
-							if len(l) > 0 {
-								contentLength, _ = strconv.Atoi(string(l))
-							}
-						}
-					}
-
-					currentContentLength += len(proto.Body(c.respBuf[:readBytes]))
-				}
-			}
-
-			if chunked {
-				// Check if chunked message finished
-				if bytes.HasSuffix(c.respBuf[:readBytes], chunkedSuffix) {
-					break
-				}
-			} else if contentLength != -1 {
-				if currentContentLength > contentLength {
-					Debug(3, "[HTTPClient] disconnected, wrong length", currentContentLength, contentLength)
-					c.Disconnect()
-					break
-				} else if currentContentLength == contentLength {
-					break
-				}
-			}
-
-			if err != nil {
-				if err == io.EOF {
-					err = nil
-				}
-				break
-			}
-		} else {
-			if currentChunk == nil {
-				currentChunk = make([]byte, readChunkSize)
-			}
-
-			n, err = c.conn.Read(currentChunk)
-
-			readBytes += int(n)
-			chunks++
-			currentContentLength += n
-
-			if chunked {
-				// Check if chunked message finished
-				if bytes.HasSuffix(currentChunk[:n], chunkedSuffix) {
-					break
-				}
-			} else if contentLength != -1 {
-				if currentContentLength > contentLength {
-					Debug(3, "[HTTPClient] disconnected, wrong length", currentContentLength, contentLength)
-					c.Disconnect()
-					break
-				} else if currentContentLength == contentLength {
-					break
-				}
-			} else {
-				Debug(3, "[HTTPClient] disconnected, can't find Content-Length or Chunked")
-				c.Disconnect()
-				break
-			}
-
-			if err == io.EOF {
-				break
-			} else if err != nil {
-				Debug(3, "[HTTPClient] Read the whole body error:", err, c.baseURL)
-				break
-			}
-
-		}
-
-		if readBytes >= maxResponseSize {
-			Debug(3, "[HTTPClient] Body is more than the max size", maxResponseSize,
-				c.baseURL)
-			break
-		}
-
-		// For following chunks expect less timeout
-		timeout = time.Now().Add(c.config.Timeout / 5)
-	}
-
-	if err != nil && readBytes == 0 {
-		Debug(3, "[HTTPClient] Response read timeout error", err, c.conn, readBytes, string(c.respBuf[:readBytes]))
-		response = errorPayload(HTTP_TIMEOUT)
-		c.Disconnect()
-		return
-	}
-
-	if readBytes < 4 || string(c.respBuf[:4]) != "HTTP" {
-		maxRead := 100
-		if readBytes < maxRead {
-			maxRead = readBytes
-		}
-		Debug(3, "[HTTPClient] Response read unknown error", err, c.conn, readBytes, string(c.respBuf[:maxRead]))
-		response = errorPayload(HTTP_UNKNOWN_ERROR)
-		c.Disconnect()
-		return
-	}
-
-	if readBytes > len(c.respBuf) {
-		readBytes = len(c.respBuf)
-	}
-	payload = make([]byte, readBytes)
-	copy(payload, c.respBuf[:readBytes])
-
-	if c.config.Debug {
-		Debug(3, "[HTTPClient] Received:", string(payload))
-	}
-
-	if c.config.FollowRedirects > 0 && c.redirectsCount < c.config.FollowRedirects {
-		status := payload[9:12]
-
-		// 3xx requests
-		if status[0] == '3' {
-			c.redirectsCount++
-
-			location := proto.Header(payload, []byte("Location"))
-			redirectPayload := proto.SetPath(data, location)
-
-			Debug(3, "[HTTPClient] Redirecting to: "+string(location))
-
-			return c.Send(redirectPayload)
-		}
-	}
-
-	if bytes.Equal(proto.Status(payload), []byte("400")) {
-		Debug(3, "[HTTPClient] Closed connection on 400 response")
-		c.Disconnect()
-	}
-
-	c.redirectsCount = 0
-
-	return payload, err
-}
-
-func (c *HTTPClient) Get(path string) (response []byte, err error) {
-	payload := "GET " + path + " HTTP/1.1\r\n\r\n"
-
-	return c.Send([]byte(payload))
-}
-
-func (c *HTTPClient) Post(path string, body []byte) (response []byte, err error) {
-	payload := "POST " + path + " HTTP/1.1\r\n"
-	payload += "Content-Length: " + strconv.Itoa(len(body)) + "\r\n\r\n"
-	payload += string(body)
-
-	return c.Send([]byte(payload))
-}
-
-func (c *HTTPClient) proxyPath(path []byte) []byte {
-	return append([]byte(c.scheme+"://"+c.host), path...)
-}
-
-func (c *HTTPClient) isProxy() bool {
-	return c.proxy != nil
-}
-
-const (
-	// https://support.cloudflare.com/hc/en-us/articles/200171936-Error-520-Web-server-is-returning-an-unknown-error
-	HTTP_UNKNOWN_ERROR = "520"
-	// https://support.cloudflare.com/hc/en-us/articles/200171916-Error-521-Web-server-is-down
-	HTTP_CONNECTION_ERROR = "521"
-	// https://support.cloudflare.com/hc/en-us/articles/200171906-Error-522-Connection-timed-out
-	HTTP_CONNECTION_TIMEOUT = "522"
-	// https://support.cloudflare.com/hc/en-us/articles/200171946-Error-523-Origin-is-unreachable
-	HTTP_UNREACHABLE = "523"
-	// https://support.cloudflare.com/hc/en-us/articles/200171926-Error-524-A-timeout-occurred
-	HTTP_TIMEOUT = "524"
-)
-
-var errorPayloadTemplate = "HTTP/1.1 202 Accepted\r\nDate: Mon, 17 Aug 2015 14:10:11 GMT\r\nContent-Length: 0\r\nContent-Type: text/plain; charset=utf-8\r\n\r\n"
-
-func errorPayload(errorCode string) []byte {
-	payload := make([]byte, len(errorPayloadTemplate))
-	copy(payload, errorPayloadTemplate)
-
-	copy(payload[29:58], []byte(time.Now().Format(time.RFC1123)))
-	copy(payload[9:12], errorCode)
-
-	return payload
-}

http_client_test.go

@@ -1,532 +0,0 @@
-package main
-
-import (
-	"bytes"
-	"crypto/rand"
-	"io/ioutil"
-	_ "log"
-	"net"
-	"net/http"
-	"net/http/httptest"
-	"net/http/httputil"
-	_ "reflect"
-	"strings"
-	"sync"
-	"testing"
-	"time"
-
-	"github.com/buger/goreplay/proto"
-)
-
-func TestHTTPClientURLPort(t *testing.T) {
-	c1 := NewHTTPClient("http://example.com", &HTTPClientConfig{})
-	if c1.baseURL != "http://example.com" {
-		t.Error("Sould not add 80 port for http:", c1.baseURL)
-	}
-
-	c2 := NewHTTPClient("https://example.com", &HTTPClientConfig{})
-	if c2.baseURL != "https://example.com" {
-		t.Error("Sould not add 443 port for https:", c2.baseURL)
-	}
-
-	c3 := NewHTTPClient("https://example.com:1", &HTTPClientConfig{})
-	if c3.baseURL != "https://example.com:1" {
-		t.Error("Sould use specified port:", c3.baseURL)
-	}
-
-	c4 := NewHTTPClient("example.com", &HTTPClientConfig{})
-	if c4.baseURL != "http://example.com" {
-		t.Error("Sould not add default protocol:", c4.baseURL)
-	}
-}
-
-func TestHTTPClientSend(t *testing.T) {
-	wg := new(sync.WaitGroup)
-
-	payload := func(reqType string) []byte {
-		switch reqType {
-		case "GET":
-			return []byte("GET / HTTP/1.1\r\n\r\n")
-		case "POST":
-			return []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
-		case "POST_CHUNKED":
-			return []byte("POST / HTTP/1.1\r\nHost: www.w3.org\r\nTransfer-Encoding: chunked\r\n\r\n4\r\nWiki\r\n5\r\npedia\r\ne\r\n in\r\n\r\nchunks.\r\n0\r\n\r\n")
-		}
-
-		return []byte("")
-	}
-
-	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-
-		if r.Method == "POST" {
-			defer r.Body.Close()
-			body, _ := ioutil.ReadAll(r.Body)
-
-			if len(r.TransferEncoding) > 0 && r.TransferEncoding[0] == "chunked" {
-				if string(body) != "Wikipedia in\r\n\r\nchunks." {
-					t.Error("Wrong POST body:", body, string(body))
-				}
-			} else {
-				if string(body) != "a=1&b=2" {
-					buf, _ := httputil.DumpRequest(r, true)
-					t.Error("Wrong POST body:", string(body), string(buf))
-				}
-			}
-		}
-
-		wg.Done()
-	}))
-	defer server.Close()
-
-	client := NewHTTPClient(server.URL, &HTTPClientConfig{Debug: true})
-
-	wg.Add(4)
-	client.Send(payload("POST"))
-	client.Send(payload("GET"))
-	client.Send(payload("POST_CHUNKED"))
-	client.Send(payload("POST"))
-
-	wg.Wait()
-
-	client = NewHTTPClient(server.URL, &HTTPClientConfig{Debug: true, CompatibilityMode: true})
-
-	wg.Add(4)
-
-	if _, err := client.Send(payload("POST")); err != nil {
-		t.Fatal(err.Error())
-	}
-	if _, err := client.Send(payload("GET")); err != nil {
-		t.Fatal(err.Error())
-	}
-	if _, err := client.Send(payload("POST_CHUNKED")); err != nil {
-		t.Fatal(err.Error())
-	}
-	if _, err := client.Send(payload("POST")); err != nil {
-		t.Fatal(err.Error())
-	}
-
-	wg.Wait()
-}
-
-func TestHTTPClientResonseByClose(t *testing.T) {
-	wg := new(sync.WaitGroup)
-
-	payload := []byte("GET / HTTP/1.1\r\n\r\n")
-	ln, _ := net.Listen("tcp", ":0")
-	go func() {
-		for {
-			conn, _ := ln.Accept()
-			buf := make([]byte, 4096)
-			conn.Read(buf)
-
-			conn.Write([]byte("HTTP/1.1 200 OK\r\n\r\n"))
-			conn.Write([]byte("ab"))
-			conn.Close()
-
-			wg.Done()
-		}
-	}()
-
-	client := NewHTTPClient(ln.Addr().String(), &HTTPClientConfig{Debug: true})
-
-	wg.Add(1)
-	resp, _ := client.Send(payload)
-
-	if !bytes.Equal(resp, []byte("HTTP/1.1 200 OK\r\n\r\nab")) {
-		t.Error("Should return valid response", string(resp))
-	}
-
-	wg.Wait()
-}
-
-// https://github.com/buger/gor/issues/184
-func TestHTTPClientResponseBuffer(t *testing.T) {
-	testCases := []struct {
-		name         string
-		responseSize int
-		buffserSize  int
-		expectedSize int
-		timeout      time.Duration
-	}{
-		{"Chunked, buffer overflow", 10 * 1024, 1024, 1024, 50 * time.Millisecond},
-
-		{"Chunked, fits buffer", 10 * 1024, 64 * 1024, 10*1024 + 145 /* headers length + chunked meta */, 50 * time.Millisecond},
-
-		{"Content-Length, buffer overflow", 1024, 1000, 1000, 50 * time.Millisecond},
-
-		{"Content-Length, fits buffer", 1024, 64 * 1024, 1024 + 118, 50 * time.Millisecond},
-	}
-
-	for _, tc := range testCases {
-		wg := new(sync.WaitGroup)
-
-		payload := []byte("GET / HTTP/1.1\r\n\r\n")
-
-		server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-
-			size := tc.responseSize // 1kb
-			rb := make([]byte, size)
-			rand.Read(rb)
-
-			w.Write(rb[:size/2])
-			w.Write(rb[size/2:])
-
-			wg.Done()
-		}))
-
-		client := NewHTTPClient(server.URL, &HTTPClientConfig{Debug: true, ResponseBufferSize: tc.buffserSize, Timeout: 100 * time.Millisecond})
-
-		wg.Add(2)
-
-		start := time.Now()
-		client.Send(payload)
-		resp, err := client.Send(payload)
-		stop := time.Now()
-
-		if err != nil {
-			t.Error("Request error", err)
-		}
-
-		if stop.Sub(start) > tc.timeout {
-			t.Error("Request took too long", stop.Sub(start), tc.timeout)
-		}
-
-		if len(resp) != tc.expectedSize {
-			t.Error(tc.name, " - Wrong response size:", tc.expectedSize, len(resp))
-		} else {
-			if !bytes.Equal(resp[0:8], []byte("HTTP/1.1")) {
-				t.Error(tc.name, " - Response buffer contains data from previous request", string(resp), len(resp))
-			}
-		}
-
-		wg.Wait()
-		server.Close()
-	}
-}
-
-func TestHTTPClientHTTPSSend(t *testing.T) {
-	wg := new(sync.WaitGroup)
-
-	payload := func(reqType string) []byte {
-		switch reqType {
-		case "GET":
-			return []byte("GET / HTTP/1.1\r\n\r\n")
-		case "POST":
-			return []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
-		case "POST_CHUNKED":
-			return []byte("POST / HTTP/1.1\r\nHost: www.w3.org\r\nTransfer-Encoding: chunked\r\n\r\n4\r\nWiki\r\n5\r\npedia\r\ne\r\n in\r\n\r\nchunks.\r\n0\r\n\r\n")
-		}
-
-		return []byte("")
-	}
-
-	server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-
-		if r.Method == "POST" {
-			defer r.Body.Close()
-			body, _ := ioutil.ReadAll(r.Body)
-
-			if len(r.TransferEncoding) > 0 && r.TransferEncoding[0] == "chunked" {
-				if string(body) != "Wikipedia in\r\n\r\nchunks." {
-					t.Error("Wrong POST body:", body, string(body))
-				}
-			} else {
-				if string(body) != "a=1&b=2" {
-					buf, _ := httputil.DumpRequest(r, true)
-					t.Error("Wrong POST body:", string(body), string(buf))
-				}
-			}
-		}
-
-		wg.Done()
-	}))
-	defer server.Close()
-
-	client := NewHTTPClient(server.URL, &HTTPClientConfig{})
-
-	wg.Add(4)
-	client.Send(payload("POST"))
-	client.Send(payload("GET"))
-	client.Send(payload("POST_CHUNKED"))
-	client.Send(payload("POST"))
-
-	wg.Wait()
-}
-
-func TestHTTPClientServerInstantDisconnect(t *testing.T) {
-	wg := new(sync.WaitGroup)
-
-	GETPayload := []byte("GET / HTTP/1.1\r\n\r\n")
-
-	ln, _ := net.Listen("tcp", "127.0.0.1:0")
-	defer ln.Close()
-
-	go func() {
-		for {
-			conn, err := ln.Accept()
-
-			if err != nil {
-				break
-			}
-			conn.Close()
-			wg.Done()
-		}
-	}()
-
-	client := NewHTTPClient(ln.Addr().String(), &HTTPClientConfig{})
-
-	wg.Add(2)
-	client.Send(GETPayload)
-	client.Send(GETPayload)
-
-	wg.Wait()
-}
-
-func TestHTTPClientServerNoKeepAlive(t *testing.T) {
-	wg := new(sync.WaitGroup)
-
-	GETPayload := []byte("GET / HTTP/1.1\r\n\r\n")
-
-	ln, _ := net.Listen("tcp", "127.0.0.1:0")
-	defer ln.Close()
-
-	go func() {
-		for {
-			conn, err := ln.Accept()
-			if err != nil {
-				break
-			}
-
-			buf := make([]byte, 4096)
-			_, err = conn.Read(buf)
-			if err != nil {
-				t.Error("Error reading:", err.Error())
-			}
-			conn.Write([]byte("OK"))
-
-			// No keep-alive connections
-			conn.Close()
-
-			wg.Done()
-		}
-	}()
-
-	client := NewHTTPClient(ln.Addr().String(), &HTTPClientConfig{})
-
-	wg.Add(2)
-	client.Send(GETPayload)
-	client.Send(GETPayload)
-
-	wg.Wait()
-}
-
-func TestHTTPClientRedirect(t *testing.T) {
-	wg := new(sync.WaitGroup)
-
-	GETPayload := []byte("GET / HTTP/1.1\r\n\r\n")
-
-	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-
-		if r.URL.Path == "/" {
-			http.Redirect(w, r, "/new", 301)
-		}
-
-		wg.Done()
-	}))
-	defer server.Close()
-
-	client := NewHTTPClient(server.URL, &HTTPClientConfig{FollowRedirects: 1, Debug: false})
-
-	// Should do 2 queries
-	wg.Add(2)
-	client.Send(GETPayload)
-
-	wg.Wait()
-}
-
-func TestHTTPClientRedirectLimit(t *testing.T) {
-	wg := new(sync.WaitGroup)
-
-	GETPayload := []byte("GET / HTTP/1.1\r\n\r\n")
-
-	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-
-		if r.URL.Path == "/" {
-			http.Redirect(w, r, "/r1", 301)
-		}
-
-		if r.URL.Path == "/r1" {
-			http.Redirect(w, r, "/r2", 301)
-		}
-
-		if r.URL.Path == "/r2" {
-			http.Redirect(w, r, "/new", 301)
-		}
-
-		wg.Done()
-	}))
-	defer server.Close()
-
-	client := NewHTTPClient(server.URL, &HTTPClientConfig{FollowRedirects: 2, Debug: false})
-
-	// Have 3 redirects + 1 GET, but should do only 2 redirects + GET
-	wg.Add(3)
-	client.Send(GETPayload)
-
-	wg.Wait()
-}
-
-func TestHTTPClientKeepHeadersRedirect(t *testing.T) {
-	wg := new(sync.WaitGroup)
-
-	GETPayload := []byte("GET / HTTP/1.1\r\n\r\n")
-
-	GETPayload = proto.AddHeader(GETPayload, []byte("keep-header"), []byte("true"))
-
-	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-
-		if r.URL.Path == "/" {
-			http.Redirect(w, r, "/new", 301)
-		}
-
-		if r.Header.Get("keep-header") != "true" {
-			t.Errorf("Header keep-header was incorrect, got: %s, want: %s.", r.Header.Get("keep-header"), "true")
-		}
-
-		wg.Done()
-	}))
-	defer server.Close()
-
-	client := NewHTTPClient(server.URL, &HTTPClientConfig{FollowRedirects: 1, Debug: false})
-
-	wg.Add(2)
-	client.Send(GETPayload)
-
-	wg.Wait()
-}
-
-func TestHTTPClientBasicAuth(t *testing.T) {
-	wg := new(sync.WaitGroup)
-	wg.Add(2)
-
-	GETPayload := []byte("GET / HTTP/1.1\r\n\r\n")
-
-	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		user, pass, _ := r.BasicAuth()
-
-		if user != "user" || pass != "pass" {
-			http.Error(w, "Unauthorized.", 401)
-			wg.Done()
-			return
-		}
-
-		wg.Done()
-	}))
-	defer server.Close()
-
-	client := NewHTTPClient(server.URL, &HTTPClientConfig{Debug: false})
-	resp, _ := client.Send(GETPayload)
-	client.Disconnect()
-
-	if !bytes.Equal(proto.Status(resp), []byte("401")) {
-		t.Error("Should return unauthorized error", string(resp))
-	}
-
-	authUrl := strings.Replace(server.URL, "http://", "http://user:pass@", -1)
-	client = NewHTTPClient(authUrl, &HTTPClientConfig{Debug: false})
-	resp, _ = client.Send(GETPayload)
-	client.Disconnect()
-
-	if !bytes.Equal(proto.Status(resp), []byte("200")) {
-		t.Error("Should return proper response", string(resp))
-	}
-
-	wg.Wait()
-}
-
-func TestHTTPClientHandleHTTP10(t *testing.T) {
-	wg := new(sync.WaitGroup)
-
-	GETPayload := []byte("GET http://foobar.com/path HTTP/1.0\r\n\r\n")
-
-	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-
-		if r.URL.Path != "/path" {
-			t.Error("Path not match:", r.URL.Path)
-		}
-
-		wg.Done()
-	}))
-	defer server.Close()
-
-	client := NewHTTPClient(server.URL, &HTTPClientConfig{Debug: true})
-
-	wg.Add(1)
-	client.Send(GETPayload)
-
-	wg.Wait()
-}
-
-// func TestHTTPClientErrors(t *testing.T) {
-// 	req := []byte("GET http://foobar.com/path HTTP/1.0\r\n\r\n")
-
-// 	// Port not exists
-// 	client := NewHTTPClient("http://127.0.0.1:1", &HTTPClientConfig{Debug: true})
-// 	if resp, err := client.Send(req); err != nil {
-// 		if s := proto.Status(resp); !bytes.Equal(s, []byte("521")) {
-// 			t.Error("Should return status 521 for connection refused, instead:", string(s), err)
-// 		}
-// 	} else {
-// 		t.Error("Should throw error")
-// 	}
-
-// 	client = NewHTTPClient("http://not.existing", &HTTPClientConfig{Debug: true})
-// 	if resp, err := client.Send(req); err != nil {
-// 		if s := proto.Status(resp); !bytes.Equal(s, []byte("521")) {
-// 			t.Error("Should return status 521 for no such host, instead:", string(s))
-// 		}
-// 	} else {
-// 		t.Error("Should throw error")
-// 	}
-
-// 	// Non routable IP address to simulate connection timeout
-// 	client = NewHTTPClient("http://10.255.255.1", &HTTPClientConfig{Debug: true, ConnectionTimeout: 100 * time.Millisecond})
-
-// 	if resp, err := client.Send(req); err != nil {
-// 		if s := proto.Status(resp); !bytes.Equal(s, []byte("521")) {
-// 			t.Error("Should return status 521 for io/timeout:", string(s))
-// 		}
-// 	} else {
-// 		t.Error("Should throw error")
-// 	}
-
-// 	// Connecting but io timeout on read
-// 	ln, _ := net.Listen("tcp", "127.0.0.1:0")
-// 	client = NewHTTPClient("http://"+ln.Addr().String(), &HTTPClientConfig{Debug: true, Timeout: 10 * time.Millisecond})
-// 	defer ln.Close()
-
-// 	if resp, err := client.Send(req); err != nil {
-// 		if s := proto.Status(resp); !bytes.Equal(s, []byte("524")) {
-// 			t.Error("Should return status 524 for io read, instead:", string(s))
-// 		}
-// 	} else {
-// 		t.Error("Should throw error")
-// 	}
-
-// 	// Response read error read tcp [::1]:51128: connection reset by peer &{{0xc20802a000}}
-// 	ln1, _ := net.Listen("tcp", "127.0.0.1:0")
-// 	go func() {
-// 		ln1.Accept()
-// 	}()
-// 	defer ln1.Close()
-
-// 	client = NewHTTPClient("http://"+ln1.Addr().String(), &HTTPClientConfig{Debug: true, Timeout: 10 * time.Millisecond})
-
-// 	if resp, err := client.Send(req); err != nil {
-// 		if s := proto.Status(resp); !bytes.Equal(s, []byte("524")) {
-// 			t.Error("Should return status 524 for connection reset by peer, instead:", string(s))
-// 		}
-// 	} else {
-// 		t.Error("Should throw error")
-// 	}
-// }

http_modifier_settings.go

@@ -19,10 +19,9 @@ type HTTPModifierConfig struct {
 	HeaderBasicAuthFilters HTTPHeaderBasicAuthFilters `json:"http-basic-auth-filter"`
 	HeaderHashFilters      HTTPHashFilters            `json:"http-header-limiter"`
 	ParamHashFilters       HTTPHashFilters            `json:"http-param-limiter"`
-
-	Params  HTTPParams  `json:"http-set-param"`
-	Headers HTTPHeaders `json:"http-set-header"`
-	Methods HTTPMethods `json:"http-allow-method"`
+	Params                 HTTPParams                 `json:"http-set-param"`
+	Headers                HTTPHeaders                `json:"http-set-header"`
+	Methods                HTTPMethods                `json:"http-allow-method"`
 }
 
 //
@@ -40,10 +39,11 @@ func (h *HTTPHeaderFilters) String() string {
 	return fmt.Sprint(*h)
 }
 
+// Set method to implement flags.Value
 func (h *HTTPHeaderFilters) Set(value string) error {
 	valArr := strings.SplitN(value, ":", 2)
 	if len(valArr) < 2 {
-		return errors.New("need both header and value, colon-delimited (ex. user_id:^169$).")
+		return errors.New("need both header and value, colon-delimited (ex. user_id:^169$)")
 	}
 	val := strings.TrimSpace(valArr[1])
 	r, err := regexp.Compile(val)
@@ -63,13 +63,14 @@ type basicAuthFilter struct {
 	regexp *regexp.Regexp
 }
 
-// HTTPHeaderFilters holds list of headers and their regexps
+// HTTPHeaderBasicAuthFilters holds list of regxp to match basic Auth header values
 type HTTPHeaderBasicAuthFilters []basicAuthFilter
 
 func (h *HTTPHeaderBasicAuthFilters) String() string {
 	return fmt.Sprint(*h)
 }
 
+// Set method to implement flags.Value
 func (h *HTTPHeaderBasicAuthFilters) Set(value string) error {
 	r, err := regexp.Compile(value)
 	if err != nil {
@@ -89,12 +90,14 @@ type hashFilter struct {
 	percent uint32
 }
 
+// HTTPHashFilters represents a slice of header hash filters
 type HTTPHashFilters []hashFilter
 
 func (h *HTTPHashFilters) String() string {
 	return fmt.Sprint(*h)
 }
 
+// Set method to implement flags.Value
 func (h *HTTPHashFilters) Set(value string) error {
 	valArr := strings.SplitN(value, ":", 2)
 	if len(valArr) < 2 {
@@ -129,23 +132,26 @@ func (h *HTTPHashFilters) Set(value string) error {
 //
 // Handling of --http-set-header option
 //
-type HTTPHeaders []HTTPHeader
-type HTTPHeader struct {
+type httpHeader struct {
 	Name  string
 	Value string
 }
 
+// HTTPHeaders is a slice of headers that must appended
+type HTTPHeaders []httpHeader
+
 func (h *HTTPHeaders) String() string {
 	return fmt.Sprint(*h)
 }
 
+// Set method to implement flags.Value
 func (h *HTTPHeaders) Set(value string) error {
 	v := strings.SplitN(value, ":", 2)
 	if len(v) != 2 {
 		return errors.New("Expected `Key: Value`")
 	}
 
-	header := HTTPHeader{
+	header := httpHeader{
 		strings.TrimSpace(v[0]),
 		strings.TrimSpace(v[1]),
 	}
@@ -157,23 +163,26 @@ func (h *HTTPHeaders) Set(value string) error {
 //
 // Handling of --http-set-param option
 //
-type HTTPParams []HTTPParam
-type HTTPParam struct {
+type httpParam struct {
 	Name  []byte
 	Value []byte
 }
 
+// HTTPParams filters for --http-set-param
+type HTTPParams []httpParam
+
 func (h *HTTPParams) String() string {
 	return fmt.Sprint(*h)
 }
 
+// Set method to implement flags.Value
 func (h *HTTPParams) Set(value string) error {
 	v := strings.SplitN(value, "=", 2)
 	if len(v) != 2 {
 		return errors.New("Expected `Key=Value`")
 	}
 
-	param := HTTPParam{
+	param := httpParam{
 		[]byte(strings.TrimSpace(v[0])),
 		[]byte(strings.TrimSpace(v[1])),
 	}
@@ -185,12 +194,15 @@ func (h *HTTPParams) Set(value string) error {
 //
 // Handling of --http-allow-method option
 //
+
+// HTTPMethods holds values for method allowed
 type HTTPMethods [][]byte
 
 func (h *HTTPMethods) String() string {
 	return fmt.Sprint(*h)
 }
 
+// Set method to implement flags.Value
 func (h *HTTPMethods) Set(value string) error {
 	*h = append(*h, []byte(value))
 	return nil
@@ -204,12 +216,14 @@ type urlRewrite struct {
 	target []byte
 }
 
+// URLRewriteMap holds regexp and data to modify URL
 type URLRewriteMap []urlRewrite
 
 func (r *URLRewriteMap) String() string {
 	return fmt.Sprint(*r)
 }
 
+// Set method to implement flags.Value
 func (r *URLRewriteMap) Set(value string) error {
 	valArr := strings.SplitN(value, ":", 2)
 	if len(valArr) < 2 {
@@ -232,12 +246,14 @@ type headerRewrite struct {
 	target []byte
 }
 
+// HeaderRewriteMap holds regexp and data to rewrite headers
 type HeaderRewriteMap []headerRewrite
 
 func (r *HeaderRewriteMap) String() string {
 	return fmt.Sprint(*r)
 }
 
+// Set method to implement flags.Value
 func (r *HeaderRewriteMap) Set(value string) error {
 	headerArr := strings.SplitN(value, ":", 2)
 	if len(headerArr) < 2 {
@@ -266,12 +282,14 @@ type urlRegexp struct {
 	regexp *regexp.Regexp
 }
 
+// HTTPURLRegexp a slice of regexp to match URLs
 type HTTPURLRegexp []urlRegexp
 
 func (r *HTTPURLRegexp) String() string {
 	return fmt.Sprint(*r)
 }
 
+// Set method to implement flags.Value
 func (r *HTTPURLRegexp) Set(value string) error {
 	regexp, err := regexp.Compile(value)
 

http_prettifier.go

@@ -3,6 +3,7 @@ package main
 import (
 	"bytes"
 	"compress/gzip"
+	"fmt"
 	"io/ioutil"
 	"net/http/httputil"
 	"strconv"
@@ -11,36 +12,25 @@ import (
 )
 
 func prettifyHTTP(p []byte) []byte {
-	headSize := bytes.IndexByte(p, '\n') + 1
-	head := p[:headSize]
-	body := p[headSize:]
 
-	headersPos := proto.MIMEHeadersEndPos(body)
+	tEnc := bytes.Equal(proto.Header(p, []byte("Transfer-Encoding")), []byte("chunked"))
+	cEnc := bytes.Equal(proto.Header(p, []byte("Content-Encoding")), []byte("gzip"))
 
-	if headersPos < 5 || headersPos > len(body) {
+	if !(tEnc || cEnc) {
 		return p
 	}
 
-	headers := body[:headersPos]
-	content := body[headersPos:]
+	headersPos := proto.MIMEHeadersEndPos(p)
 
-	var tEnc, cEnc []byte
-	proto.ParseHeaders([][]byte{headers}, func(header, value []byte) {
-		if bytes.EqualFold(header, []byte("Transfer-Encoding")) {
-			tEnc = value
-		}
-
-		if bytes.EqualFold(header, []byte("Content-Encoding")) {
-			cEnc = value
-		}
-	})
-
-	if len(tEnc) == 0 && len(cEnc) == 0 {
+	if headersPos < 5 || headersPos > len(p) {
 		return p
 	}
 
-	if bytes.Equal(tEnc, []byte("chunked")) {
-		buf := bytes.NewBuffer(content)
+	headers := p[:headersPos]
+	content := p[headersPos:]
+
+	if tEnc {
+		buf := bytes.NewReader(content)
 		r := httputil.NewChunkedReader(buf)
 		content, _ = ioutil.ReadAll(r)
 
@@ -50,8 +40,8 @@ func prettifyHTTP(p []byte) []byte {
 		headers = proto.SetHeader(headers, []byte("Content-Length"), []byte(newLen))
 	}
 
-	if bytes.Equal(cEnc, []byte("gzip")) {
-		buf := bytes.NewBuffer(content)
+	if cEnc {
+		buf := bytes.NewReader(content)
 		g, err := gzip.NewReader(buf)
 
 		if err != nil {
@@ -59,7 +49,11 @@ func prettifyHTTP(p []byte) []byte {
 			return []byte{}
 		}
 
-		content, _ = ioutil.ReadAll(g)
+		content, err = ioutil.ReadAll(g)
+		if err != nil {
+			Debug(1, fmt.Sprintf("[HTTP-PRETTIFIER] %q", err))
+			return p
+		}
 
 		headers = proto.DeleteHeader(headers, []byte("Content-Encoding"))
 
@@ -67,7 +61,7 @@ func prettifyHTTP(p []byte) []byte {
 		headers = proto.SetHeader(headers, []byte("Content-Length"), []byte(newLen))
 	}
 
-	newPayload := append(append(head, headers...), content...)
+	newPayload := append(headers, content...)
 
 	return newPayload
 }

http_prettifier_test.go

@@ -5,6 +5,8 @@ import (
 	"compress/gzip"
 	"strconv"
 	"testing"
+
+	"github.com/buger/goreplay/proto"
 )
 
 func TestHTTPPrettifierGzip(t *testing.T) {
@@ -15,22 +17,21 @@ func TestHTTPPrettifierGzip(t *testing.T) {
 
 	size := strconv.Itoa(len(b.Bytes()))
 
-	payload := []byte("2 1 1\nHTTP/1.1 200 OK\r\nContent-Length: " + size + "\r\nContent-Encoding: gzip\r\n\r\n")
+	payload := []byte("HTTP/1.1 200 OK\r\nContent-Length: " + size + "\r\nContent-Encoding: gzip\r\n\r\n")
 	payload = append(payload, b.Bytes()...)
 
 	newPayload := prettifyHTTP(payload)
 
-	if string(newPayload) != "2 1 1\nHTTP/1.1 200 OK\r\nContent-Length: 4\r\n\r\ntest" {
-		t.Error("Payload not match:", string(newPayload))
+	if string(newPayload) != "HTTP/1.1 200 OK\r\nContent-Length: 4\r\n\r\ntest" {
+		t.Errorf("Payload not match %q", string(newPayload))
 	}
 }
 
 func TestHTTPPrettifierChunked(t *testing.T) {
 	payload := []byte("POST / HTTP/1.1\r\nHost: www.w3.org\r\nTransfer-Encoding: chunked\r\n\r\n4\r\nWiki\r\n5\r\npedia\r\ne\r\n in\r\n\r\nchunks.\r\n0\r\n\r\n")
 
-	newPayload := prettifyHTTP(payload)
-
-	if string(newPayload) != "POST / HTTP/1.1\r\nHost: www.w3.org\r\nContent-Length: 23\r\n\r\nWikipedia in\r\n\r\nchunks." {
-		t.Error("Payload not match:", string(newPayload))
+	payload = prettifyHTTP(payload)
+	if string(proto.Header(payload, []byte("Content-Length"))) != "23" {
+		t.Errorf("payload should have content length of 23")
 	}
 }

input_dummy.go

@@ -7,24 +7,30 @@ import (
 // DummyInput used for debugging. It generate 1 "GET /"" request per second.
 type DummyInput struct {
 	data chan []byte
+	quit chan struct{}
 }
 
 // NewDummyInput constructor for DummyInput
 func NewDummyInput(options string) (di *DummyInput) {
 	di = new(DummyInput)
 	di.data = make(chan []byte)
+	di.quit = make(chan struct{})
 
 	go di.emit()
 
 	return
 }
 
-func (i *DummyInput) Read(data []byte) (int, error) {
-	buf := <-i.data
-
-	copy(data, buf)
-
-	return len(buf), nil
+// PluginRead reads message from this plugin
+func (i *DummyInput) PluginRead() (*Message, error) {
+	var msg Message
+	select {
+	case <-i.quit:
+		return nil, ErrorStopped
+	case buf := <-i.data:
+		msg.Meta, msg.Data = payloadMetaWithBody(buf)
+		return &msg, nil
+	}
 }
 
 func (i *DummyInput) emit() {
@@ -46,3 +52,9 @@ func (i *DummyInput) emit() {
 func (i *DummyInput) String() string {
 	return "Dummy Input"
 }
+
+// Close closes this plugins
+func (i *DummyInput) Close() error {
+	close(i.quit)
+	return nil
+}

input_file.go

@@ -5,8 +5,8 @@ import (
 	"bytes"
 	"compress/gzip"
 	"errors"
+	"fmt"
 	"io"
-	"log"
 	"os"
 	"path/filepath"
 	"strconv"
@@ -64,6 +64,8 @@ func (f *fileInputReader) ReadPayload() []byte {
 
 	return f.data
 }
+
+// Close closes this plugin
 func (f *fileInputReader) Close() error {
 	if atomic.LoadInt32(&f.closed) == 0 {
 		atomic.StoreInt32(&f.closed, 1)
@@ -73,7 +75,7 @@ func (f *fileInputReader) Close() error {
 	return nil
 }
 
-func NewFileInputReader(path string) *fileInputReader {
+func newFileInputReader(path string) *fileInputReader {
 	var file io.ReadCloser
 	var err error
 
@@ -84,7 +86,7 @@ func NewFileInputReader(path string) *fileInputReader {
 	}
 
 	if err != nil {
-		log.Println(err)
+		Debug(0, fmt.Sprintf("[INPUT-FILE] err: %q", err))
 		return nil
 	}
 
@@ -92,7 +94,7 @@ func NewFileInputReader(path string) *fileInputReader {
 	if strings.HasSuffix(path, ".gz") {
 		gzReader, err := gzip.NewReader(file)
 		if err != nil {
-			log.Println(err)
+			Debug(0, fmt.Sprintf("[INPUT-FILE] err: %q", err))
 			return nil
 		}
 		r.reader = bufio.NewReader(gzReader)
@@ -153,43 +155,43 @@ func (i *FileInput) init() (err error) {
 
 		resp, err := svc.ListObjects(params)
 		if err != nil {
-			log.Println("Error while retreiving list of files from S3", i.path, err)
+			Debug(0, "[INPUT-FILE] Error while retreiving list of files from S3", i.path, err)
 			return err
 		}
 
 		for _, c := range resp.Contents {
 			matches = append(matches, "s3://"+bucket+"/"+(*c.Key))
 		}
-	} else {
-		if matches, err = filepath.Glob(i.path); err != nil {
-			log.Println("Wrong file pattern", i.path, err)
-			return
-		}
+	} else if matches, err = filepath.Glob(i.path); err != nil {
+		Debug(0, "[INPUT-FILE] Wrong file pattern", i.path, err)
+		return
+
 	}
 
 	if len(matches) == 0 {
-		log.Println("No files match pattern: ", i.path)
+		Debug(0, "[INPUT-FILE] No files match pattern: ", i.path)
 		return errors.New("No matching files")
 	}
 
 	i.readers = make([]*fileInputReader, len(matches))
 
 	for idx, p := range matches {
-		i.readers[idx] = NewFileInputReader(p)
+		i.readers[idx] = newFileInputReader(p)
 	}
 
 	return nil
 }
 
-func (i *FileInput) Read(data []byte) (int, error) {
-	var buf []byte
+// PluginRead reads message from this plugin
+func (i *FileInput) PluginRead() (*Message, error) {
+	var msg Message
 	select {
 	case <-i.exit:
-		return 0, ErrorStopped
-	case buf = <-i.data:
+		return nil, ErrorStopped
+	case buf := <-i.data:
+		msg.Meta, msg.Data = payloadMetaWithBody(buf)
+		return &msg, nil
 	}
-	n := copy(data, buf)
-	return n, nil
 }
 
 func (i *FileInput) String() string {
@@ -256,7 +258,7 @@ func (i *FileInput) emit() {
 		}
 	}
 
-	log.Printf("FileInput: end of file '%s'\n", i.path)
+	Debug(0, fmt.Sprintf("[INPUT-FILE] FileInput: end of file '%s'\n", i.path))
 
 }
 

input_file_test.go

@@ -4,9 +4,7 @@ import (
 	"bytes"
 	"errors"
 	"fmt"
-	"io"
 	"io/ioutil"
-	"log"
 	"math/rand"
 	"os"
 	"sync"
@@ -14,54 +12,49 @@ import (
 	"time"
 )
 
-var _ = log.Println
-
 func TestInputFileWithGET(t *testing.T) {
 	input := NewTestInput()
-	rg := NewRequestGenerator([]io.Reader{input}, func() { input.EmitGET() }, 1)
-	readPayloads := [][]byte{}
+	rg := NewRequestGenerator([]PluginReader{input}, func() { input.EmitGET() }, 1)
+	readPayloads := []*Message{}
 
 	// Given a capture file with a GET request
 	expectedCaptureFile := CreateCaptureFile(rg)
 	defer expectedCaptureFile.TearDown()
 
 	// When the request is read from the capture file
-	err := ReadFromCaptureFile(expectedCaptureFile.file, 1, func(data []byte) {
-		readPayloads = append(readPayloads, Duplicate(data))
+	err := ReadFromCaptureFile(expectedCaptureFile.file, 1, func(msg *Message) {
+		readPayloads = append(readPayloads, msg)
 	})
 
 	// The read request should match the original request
 	if err != nil {
 		t.Error(err)
-	} else {
-		if !expectedCaptureFile.PayloadsEqual(readPayloads) {
-			t.Error("Request read back from file should match")
-		}
+	} else if !expectedCaptureFile.PayloadsEqual(readPayloads) {
+		t.Error("Request read back from file should match")
+
 	}
 }
 
 func TestInputFileWithPayloadLargerThan64Kb(t *testing.T) {
-
 	input := NewTestInput()
-	rg := NewRequestGenerator([]io.Reader{input}, func() { input.EmitSizedPOST(64 * 1024) }, 1)
-	readPayloads := [][]byte{}
+	rg := NewRequestGenerator([]PluginReader{input}, func() { input.EmitSizedPOST(64 * 1024) }, 1)
+	readPayloads := []*Message{}
 
 	// Given a capture file with a request over 64Kb
 	expectedCaptureFile := CreateCaptureFile(rg)
 	defer expectedCaptureFile.TearDown()
 
 	// When the request is read from the capture file
-	err := ReadFromCaptureFile(expectedCaptureFile.file, 1, func(data []byte) {
-		readPayloads = append(readPayloads, Duplicate(data))
+	err := ReadFromCaptureFile(expectedCaptureFile.file, 1, func(msg *Message) {
+		readPayloads = append(readPayloads, msg)
 	})
 
 	// The read request should match the original request
 	if err != nil {
 		t.Error(err)
-	} else {
-		if !expectedCaptureFile.PayloadsEqual(readPayloads) {
-			t.Error("Request read back from file should match")
-		}
+	} else if !expectedCaptureFile.PayloadsEqual(readPayloads) {
+		t.Error("Request read back from file should match")
+
 	}
 
 }
@@ -69,28 +62,27 @@ func TestInputFileWithPayloadLargerThan64Kb(t *testing.T) {
 func TestInputFileWithGETAndPOST(t *testing.T) {
 
 	input := NewTestInput()
-	rg := NewRequestGenerator([]io.Reader{input}, func() {
+	rg := NewRequestGenerator([]PluginReader{input}, func() {
 		input.EmitGET()
 		input.EmitPOST()
 	}, 2)
-	readPayloads := [][]byte{}
+	readPayloads := []*Message{}
 
 	// Given a capture file with a GET request
 	expectedCaptureFile := CreateCaptureFile(rg)
 	defer expectedCaptureFile.TearDown()
 
 	// When the requests are read from the capture file
-	err := ReadFromCaptureFile(expectedCaptureFile.file, 2, func(data []byte) {
-		readPayloads = append(readPayloads, Duplicate(data))
+	err := ReadFromCaptureFile(expectedCaptureFile.file, 2, func(msg *Message) {
+		readPayloads = append(readPayloads, msg)
 	})
 
 	// The read requests should match the original request
 	if err != nil {
 		t.Error(err)
-	} else {
-		if !expectedCaptureFile.PayloadsEqual(readPayloads) {
-			t.Error("Request read back from file should match")
-		}
+	} else if !expectedCaptureFile.PayloadsEqual(readPayloads) {
+		t.Error("Request read back from file should match")
+
 	}
 
 }
@@ -113,12 +105,11 @@ func TestInputFileMultipleFilesWithRequestsOnly(t *testing.T) {
 	file2.Close()
 
 	input := NewFileInput(fmt.Sprintf("/tmp/%d*", rnd), false)
-	buf := make([]byte, 1000)
 
 	for i := '1'; i <= '4'; i++ {
-		n, _ := input.Read(buf)
-		if buf[4] != byte(i) {
-			t.Error("Should emit requests in right order", string(buf[:n]))
+		msg, _ := input.PluginRead()
+		if msg.Meta[4] != byte(i) {
+			t.Error("Should emit requests in right order", string(msg.Meta))
 		}
 	}
 
@@ -140,24 +131,18 @@ func TestInputFileRequestsWithLatency(t *testing.T) {
 	file.Write([]byte(payloadSeparator))
 
 	input := NewFileInput(fmt.Sprintf("/tmp/%d", rnd), false)
-	buf := make([]byte, 1000)
 
 	start := time.Now().UnixNano()
 	for i := 0; i < 3; i++ {
-		input.Read(buf)
+		input.PluginRead()
 	}
 	end := time.Now().UnixNano()
 
-	var expectedLatency int64 = 250000000 - 100000000
+	var expectedLatency int64 = 300000000 - 100000000
 	realLatency := end - start
-	if realLatency < expectedLatency {
+	if realLatency > expectedLatency {
 		t.Errorf("Should emit requests respecting latency. Expected: %v, real: %v", expectedLatency, realLatency)
 	}
-
-	if realLatency > expectedLatency+10000000 {
-		t.Errorf("Should emit requests respecting latency. Expected: %v, real: %v", expectedLatency, realLatency)
-
-	}
 }
 
 func TestInputFileMultipleFilesWithRequestsAndResponses(t *testing.T) {
@@ -186,17 +171,16 @@ func TestInputFileMultipleFilesWithRequestsAndResponses(t *testing.T) {
 	file2.Close()
 
 	input := NewFileInput(fmt.Sprintf("/tmp/%d*", rnd), false)
-	buf := make([]byte, 1000)
 
 	for i := '1'; i <= '4'; i++ {
-		n, _ := input.Read(buf)
-		if buf[0] != '1' && buf[4] != byte(i) {
-			t.Error("Shound emit requests in right order", string(buf[:n]))
+		msg, _ := input.PluginRead()
+		if msg.Meta[0] != '1' && msg.Meta[4] != byte(i) {
+			t.Error("Shound emit requests in right order", string(msg.Meta))
 		}
 
-		n, _ = input.Read(buf)
-		if buf[0] != '2' && buf[4] != byte(i) {
-			t.Error("Shound emit responses in right order", string(buf[:n]))
+		msg, _ = input.PluginRead()
+		if msg.Meta[0] != '2' && msg.Meta[4] != byte(i) {
+			t.Error("Shound emit responses in right order", string(msg.Meta))
 		}
 	}
 
@@ -215,11 +199,10 @@ func TestInputFileLoop(t *testing.T) {
 	file.Close()
 
 	input := NewFileInput(fmt.Sprintf("/tmp/%d", rnd), true)
-	buf := make([]byte, 1000)
 
 	// Even if we have just 2 requests in file, it should indifinitly loop
 	for i := 0; i < 1000; i++ {
-		input.Read(buf)
+		input.PluginRead()
 	}
 
 	input.Close()
@@ -231,22 +214,21 @@ func TestInputFileCompressed(t *testing.T) {
 
 	output := NewFileOutput(fmt.Sprintf("/tmp/%d_0.gz", rnd), &FileOutputConfig{FlushInterval: time.Minute, Append: true})
 	for i := 0; i < 1000; i++ {
-		output.Write([]byte("1 1 1\r\ntest"))
+		output.PluginWrite(&Message{Meta: []byte("1 1 1\r\n"), Data: []byte("test")})
 	}
 	name1 := output.file.Name()
 	output.Close()
 
 	output2 := NewFileOutput(fmt.Sprintf("/tmp/%d_1.gz", rnd), &FileOutputConfig{FlushInterval: time.Minute, Append: true})
 	for i := 0; i < 1000; i++ {
-		output2.Write([]byte("1 1 1\r\ntest"))
+		output2.PluginWrite(&Message{Meta: []byte("1 1 1\r\n"), Data: []byte("test")})
 	}
 	name2 := output2.file.Name()
 	output2.Close()
 
 	input := NewFileInput(fmt.Sprintf("/tmp/%d*", rnd), false)
-	buf := make([]byte, 1000)
 	for i := 0; i < 2000; i++ {
-		input.Read(buf)
+		input.PluginRead()
 	}
 
 	os.Remove(name1)
@@ -254,14 +236,14 @@ func TestInputFileCompressed(t *testing.T) {
 }
 
 type CaptureFile struct {
-	data [][]byte
+	msgs []*Message
 	file *os.File
 }
 
-func NewExpectedCaptureFile(data [][]byte, file *os.File) *CaptureFile {
+func NewExpectedCaptureFile(msgs []*Message, file *os.File) *CaptureFile {
 	ecf := new(CaptureFile)
 	ecf.file = file
-	ecf.data = data
+	ecf.msgs = msgs
 	return ecf
 }
 
@@ -272,12 +254,12 @@ func (expectedCaptureFile *CaptureFile) TearDown() {
 }
 
 type RequestGenerator struct {
-	inputs []io.Reader
+	inputs []PluginReader
 	emit   func()
 	wg     *sync.WaitGroup
 }
 
-func NewRequestGenerator(inputs []io.Reader, emit func(), count int) (rg *RequestGenerator) {
+func NewRequestGenerator(inputs []PluginReader, emit func(), count int) (rg *RequestGenerator) {
 	rg = new(RequestGenerator)
 	rg.inputs = inputs
 	rg.emit = emit
@@ -286,14 +268,17 @@ func NewRequestGenerator(inputs []io.Reader, emit func(), count int) (rg *Reques
 	return
 }
 
-func (expectedCaptureFile *CaptureFile) PayloadsEqual(other [][]byte) bool {
+func (expectedCaptureFile *CaptureFile) PayloadsEqual(other []*Message) bool {
 
-	if len(expectedCaptureFile.data) != len(other) {
+	if len(expectedCaptureFile.msgs) != len(other) {
 		return false
 	}
 
 	for i, payload := range other {
-		if !bytes.Equal(expectedCaptureFile.data[i], payload) {
+		if !bytes.Equal(expectedCaptureFile.msgs[i].Meta, payload.Meta) {
+			return false
+		}
+		if !bytes.Equal(expectedCaptureFile.msgs[i].Data, payload.Data) {
 			return false
 		}
 	}
@@ -308,26 +293,24 @@ func CreateCaptureFile(requestGenerator *RequestGenerator) *CaptureFile {
 		panic(err)
 	}
 
-	quit := make(chan int)
-
-	readPayloads := [][]byte{}
-	output := NewTestOutput(func(data []byte) {
-		readPayloads = append(readPayloads, Duplicate(data))
+	readPayloads := []*Message{}
+	output := NewTestOutput(func(msg *Message) {
+		readPayloads = append(readPayloads, msg)
 		requestGenerator.wg.Done()
 	})
 
-	outputFile := NewFileOutput(f.Name(), &FileOutputConfig{FlushInterval: time.Minute, Append: true})
+	outputFile := NewFileOutput(f.Name(), &FileOutputConfig{FlushInterval: time.Second, Append: true})
 
 	plugins := &InOutPlugins{
 		Inputs:  requestGenerator.inputs,
-		Outputs: []io.Writer{output, outputFile},
+		Outputs: []PluginWriter{output, outputFile},
 	}
 	for _, input := range requestGenerator.inputs {
 		plugins.All = append(plugins.All, input)
 	}
 	plugins.All = append(plugins.All, output, outputFile)
 
-	emitter := NewEmitter(quit)
+	emitter := NewEmitter()
 	go emitter.Start(plugins, Settings.Middleware)
 
 	requestGenerator.emit()
@@ -341,23 +324,22 @@ func CreateCaptureFile(requestGenerator *RequestGenerator) *CaptureFile {
 }
 
 func ReadFromCaptureFile(captureFile *os.File, count int, callback writeCallback) (err error) {
-	quit := make(chan int)
 	wg := new(sync.WaitGroup)
 
 	input := NewFileInput(captureFile.Name(), false)
-	output := NewTestOutput(func(data []byte) {
-		callback(data)
+	output := NewTestOutput(func(msg *Message) {
+		callback(msg)
 		wg.Done()
 	})
 
 	plugins := &InOutPlugins{
-		Inputs:  []io.Reader{input},
-		Outputs: []io.Writer{output},
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{output},
 	}
 	plugins.All = append(plugins.All, input, output)
 
 	wg.Add(count)
-	emitter := NewEmitter(quit)
+	emitter := NewEmitter()
 	go emitter.Start(plugins, Settings.Middleware)
 
 	done := make(chan int, 1)
@@ -376,10 +358,3 @@ func ReadFromCaptureFile(captureFile *os.File, count int, callback writeCallback
 	return
 
 }
-
-func Duplicate(data []byte) (duplicate []byte) {
-	duplicate = make([]byte, len(data))
-	copy(duplicate, data)
-
-	return
-}

input_http.go

@@ -27,25 +27,17 @@ func NewHTTPInput(address string) (i *HTTPInput) {
 	return
 }
 
-func (i *HTTPInput) Read(data []byte) (int, error) {
-	var buf []byte
+// PluginRead reads message from this plugin
+func (i *HTTPInput) PluginRead() (*Message, error) {
+	var msg Message
 	select {
 	case <-i.stop:
-		return 0, ErrorStopped
-	case buf = <-i.data:
+		return nil, ErrorStopped
+	case buf := <-i.data:
+		msg.Data = buf
+		msg.Meta = payloadHeader(RequestPayload, uuid(), time.Now().UnixNano(), -1)
+		return &msg, nil
 	}
-	header := payloadHeader(RequestPayload, uuid(), time.Now().UnixNano(), -1)
-
-	n := copy(data, header)
-	if len(data) > len(header) {
-		n += copy(data[len(header):], buf)
-	}
-	dis := len(header) + len(buf) - n
-	if dis > 0 {
-		Debug(2, "[INPUT-HTTP] discarded", dis, "increase copy buffer size")
-	}
-
-	return n, nil
 }
 
 // Close closes this plugin

input_http_test.go

@@ -2,33 +2,29 @@ package main
 
 import (
 	"bytes"
-	"io"
 	"net/http"
 	"strings"
 	"sync"
 	"testing"
 	"time"
-
-	"github.com/buger/goreplay/proto"
 )
 
 func TestHTTPInput(t *testing.T) {
 	wg := new(sync.WaitGroup)
-	quit := make(chan int)
 
 	input := NewHTTPInput("127.0.0.1:0")
 	time.Sleep(time.Millisecond)
-	output := NewTestOutput(func(data []byte) {
+	output := NewTestOutput(func(*Message) {
 		wg.Done()
 	})
 
 	plugins := &InOutPlugins{
-		Inputs:  []io.Reader{input},
-		Outputs: []io.Writer{output},
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{output},
 	}
 	plugins.All = append(plugins.All, input, output)
 
-	emitter := NewEmitter(quit)
+	emitter := NewEmitter()
 	go emitter.Start(plugins, Settings.Middleware)
 
 	address := strings.Replace(input.address, "[::]", "127.0.0.1", -1)
@@ -44,27 +40,25 @@ func TestHTTPInput(t *testing.T) {
 
 func TestInputHTTPLargePayload(t *testing.T) {
 	wg := new(sync.WaitGroup)
-	quit := make(chan int, 1)
 	const n = 10 << 20 // 10MB
 	var large [n]byte
 	large[n-1] = '0'
 
 	input := NewHTTPInput("127.0.0.1:0")
-	time.Sleep(time.Millisecond)
-	output := NewTestOutput(func(data []byte) {
-		_len := len(proto.Body(payloadBody(data)))
+	output := NewTestOutput(func(msg *Message) {
+		_len := len(msg.Data)
 		if _len >= n { // considering http body CRLF
 			t.Errorf("expected body to be >= %d", n)
 		}
 		wg.Done()
 	})
 	plugins := &InOutPlugins{
-		Inputs:  []io.Reader{input},
-		Outputs: []io.Writer{output},
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{output},
 	}
 	plugins.All = append(plugins.All, input, output)
 
-	emitter := NewEmitter(quit)
+	emitter := NewEmitter()
 	defer emitter.Close()
 	go emitter.Start(plugins, Settings.Middleware)
 

input_kafka.go

@@ -9,21 +9,17 @@ import (
 	"github.com/Shopify/sarama/mocks"
 )
 
-// KafkaInput is used for recieving Kafka messages and
+// KafkaInput is used for receiving Kafka messages and
 // transforming them into HTTP payloads.
 type KafkaInput struct {
 	config    *InputKafkaConfig
 	consumers []sarama.PartitionConsumer
 	messages  chan *sarama.ConsumerMessage
+	quit      chan struct{}
 }
 
-// NewKafkaInput creates instance of kafka consumer client.
-func NewKafkaInput(address string, config *InputKafkaConfig) *KafkaInput {
-	return NewKafkaInputWithTLS(address, config, nil)
-}
-
-// NewKafkaInputWithTLS creates instance of kafka consumer client with TLS
-func NewKafkaInputWithTLS(address string, config *InputKafkaConfig, tlsConfig *KafkaTLSConfig) *KafkaInput {
+// NewKafkaInput creates instance of kafka consumer client with TLS config
+func NewKafkaInput(address string, config *InputKafkaConfig, tlsConfig *KafkaTLSConfig) *KafkaInput {
 	c := NewKafkaConfig(tlsConfig)
 
 	var con sarama.Consumer
@@ -32,7 +28,6 @@ func NewKafkaInputWithTLS(address string, config *InputKafkaConfig, tlsConfig *K
 		con = config.consumer
 	} else {
 		var err error
-		//con, err = sarama.NewConsumer([]string{config.Host}, c)
 		con, err = sarama.NewConsumer(strings.Split(config.Host, ","), c)
 
 		if err != nil {
@@ -49,6 +44,7 @@ func NewKafkaInputWithTLS(address string, config *InputKafkaConfig, tlsConfig *K
 		config:    config,
 		consumers: make([]sarama.PartitionConsumer, len(partitions)),
 		messages:  make(chan *sarama.ConsumerMessage, 256),
+		quit:      make(chan struct{}),
 	}
 
 	for index, partition := range partitions {
@@ -80,29 +76,45 @@ func (i *KafkaInput) ErrorHandler(consumer sarama.PartitionConsumer) {
 	}
 }
 
-func (i *KafkaInput) Read(data []byte) (int, error) {
-	message := <-i.messages
-
-	if !i.config.UseJSON {
-		copy(data, message.Value)
-		return len(message.Value), nil
+// PluginRead a reads message from this plugin
+func (i *KafkaInput) PluginRead() (*Message, error) {
+	var message *sarama.ConsumerMessage
+	var msg Message
+	select {
+	case <-i.quit:
+		return nil, ErrorStopped
+	case message = <-i.messages:
 	}
 
-	var kafkaMessage KafkaMessage
-	json.Unmarshal(message.Value, &kafkaMessage)
+	msg.Data = message.Value
+	if i.config.UseJSON {
 
-	buf, err := kafkaMessage.Dump()
-	if err != nil {
-		Debug(1, "Failed to decode access log entry:", err)
-		return 0, err
+		var kafkaMessage KafkaMessage
+		json.Unmarshal(message.Value, &kafkaMessage)
+
+		var err error
+		msg.Data, err = kafkaMessage.Dump()
+		if err != nil {
+			Debug(1, "[INPUT-KAFKA] failed to decode access log entry:", err)
+			return nil, err
+		}
 	}
 
-	copy(data, buf)
+	// does it have meta
+	if isOriginPayload(msg.Data) {
+		msg.Meta, msg.Data = payloadMetaWithBody(msg.Data)
+	}
 
-	return len(buf), nil
+	return &msg, nil
 
 }
 
 func (i *KafkaInput) String() string {
 	return "Kafka Input: " + i.config.Host + "/" + i.config.Topic
 }
+
+// Close closes this plugin
+func (i *KafkaInput) Close() error {
+	close(i.quit)
+	return nil
+}

input_kafka_test.go

@@ -20,17 +20,16 @@ func TestInputKafkaRAW(t *testing.T) {
 		consumer: consumer,
 		Topic:    "test",
 		UseJSON:  false,
-	})
+	}, nil)
 
-	buf := make([]byte, 1024)
-	n, err := input.Read(buf)
+	msg, err := input.PluginRead()
 
 	if err != nil {
 		t.Fatal(err)
 	}
 
-	if string(buf[:n]) != "1 2 3\nGET / HTTP1.1\r\nHeader: 1\r\n\r\n" {
-		t.Error("Message not properly decoded: ", string(buf[:n]), n)
+	if string(append(msg.Meta, msg.Data...)) != "1 2 3\nGET / HTTP1.1\r\nHeader: 1\r\n\r\n" {
+		t.Error("Message not properly decoded")
 	}
 }
 
@@ -47,16 +46,15 @@ func TestInputKafkaJSON(t *testing.T) {
 		consumer: consumer,
 		Topic:    "test",
 		UseJSON:  true,
-	})
+	}, nil)
 
-	buf := make([]byte, 1024)
-	n, err := input.Read(buf)
+	msg, err := input.PluginRead()
 
 	if err != nil {
 		t.Fatal(err)
 	}
 
-	if string(buf[:n]) != "1 2 3\nGET / HTTP/1.1\r\nHeader: 1\r\n\r\n" {
-		t.Error("Message not properly decoded: ", string(buf[:n]), n)
+	if string(append(msg.Meta, msg.Data...)) != "1 2 3\nGET / HTTP/1.1\r\nHeader: 1\r\n\r\n" {
+		t.Error("Message not properly decoded")
 	}
 }

input_raw.go

@@ -6,6 +6,7 @@ import (
 	"log"
 	"net"
 	"strconv"
+	"strings"
 	"sync"
 	"time"
 
@@ -52,16 +53,17 @@ func (protocol *TCPProtocol) String() string {
 // RAWInputConfig represents configuration that can be applied on raw input
 type RAWInputConfig struct {
 	capture.PcapOptions
-	Expire         time.Duration      `json:"input-raw-expire"`
-	CopyBufferSize size.Size          `json:"copy-buffer-size"`
-	Engine         capture.EngineType `json:"input-raw-engine"`
-	TrackResponse  bool               `json:"input-raw-track-response"`
-	Protocol       TCPProtocol        `json:"input-raw-protocol"`
-	RealIPHeader   string             `json:"input-raw-realip-header"`
-	Stats          bool               `json:"input-raw-stats"`
-	quit           chan bool          // Channel used only to indicate goroutine should shutdown
-	host           string
-	port           uint16
+	Expire          time.Duration      `json:"input-raw-expire"`
+	CopyBufferSize  size.Size          `json:"copy-buffer-size"`
+	Engine          capture.EngineType `json:"input-raw-engine"`
+	TrackResponse   bool               `json:"input-raw-track-response"`
+	Protocol        TCPProtocol        `json:"input-raw-protocol"`
+	RealIPHeader    string             `json:"input-raw-realip-header"`
+	Stats           bool               `json:"input-raw-stats"`
+	AllowIncomplete bool               `json:"input-raw-allow-incomplete"`
+	quit            chan bool          // Channel used only to indicate goroutine should shutdown
+	host            string
+	ports           []uint16
 }
 
 // RAWInput used for intercepting traffic for given address
@@ -70,81 +72,84 @@ type RAWInput struct {
 	RAWInputConfig
 	messageStats   []tcp.Stats
 	listener       *capture.Listener
-	message        chan *tcp.Message
+	messageParser  *tcp.MessageParser
 	cancelListener context.CancelFunc
+	closed         bool
 }
 
 // NewRAWInput constructor for RAWInput. Accepts raw input config as arguments.
 func NewRAWInput(address string, config RAWInputConfig) (i *RAWInput) {
 	i = new(RAWInput)
 	i.RAWInputConfig = config
-	i.message = make(chan *tcp.Message, 1000)
 	i.quit = make(chan bool)
-	var host, _port string
-	var err error
-	var port int
-	host, _port, err = net.SplitHostPort(address)
+
+	host, _ports, err := net.SplitHostPort(address)
 	if err != nil {
 		log.Fatalf("input-raw: error while parsing address: %s", err)
 	}
-	if _port != "" {
-		port, err = strconv.Atoi(_port)
+
+	var ports []uint16
+	if _ports != "" {
+		portsStr := strings.Split(_ports, ",")
+
+		for _, portStr := range portsStr {
+			port, err := strconv.Atoi(strings.TrimSpace(portStr))
+			if err != nil {
+				log.Fatalf("parsing port error: %v", err)
+			}
+			ports = append(ports, uint16(port))
+
+		}
 	}
 
-	if err != nil {
-		log.Fatalf("parsing port error: %v", err)
-	}
 	i.host = host
-	i.port = uint16(port)
+	i.ports = ports
 
 	i.listen(address)
 
 	return
 }
 
-func (i *RAWInput) Read(data []byte) (n int, err error) {
-	var msg *tcp.Message
-	var buf []byte
+// PluginRead reads meassage from this plugin
+func (i *RAWInput) PluginRead() (*Message, error) {
+	var msgTCP *tcp.Message
+	var msg Message
 	select {
 	case <-i.quit:
-		return 0, ErrorStopped
-	case msg = <-i.message:
-		buf = msg.Data()
+		return nil, ErrorStopped
+	case msgTCP = <-i.messageParser.Messages():
+		msg.Data = msgTCP.Data()
 	}
-	var header []byte
 
 	var msgType byte = ResponsePayload
-	if msg.IsIncoming {
+	if msgTCP.IsRequest {
 		msgType = RequestPayload
 		if i.RealIPHeader != "" {
-			buf = proto.SetHeader(buf, []byte(i.RealIPHeader), []byte(msg.SrcAddr))
+			msg.Data = proto.SetHeader(msg.Data, []byte(i.RealIPHeader), []byte(msgTCP.SrcAddr))
 		}
 	}
-	header = payloadHeader(msgType, msg.UUID(), msg.Start.UnixNano(), msg.End.UnixNano()-msg.Start.UnixNano())
+	msg.Meta = payloadHeader(msgType, msgTCP.UUID(), msgTCP.Start.UnixNano(), msgTCP.End.UnixNano()-msgTCP.Start.UnixNano())
 
-	n = copy(data, header)
-	if len(data) > len(header) {
-		n += copy(data[len(header):], buf)
-	}
 	// to be removed....
-	if msg.Truncated || len(header)+len(buf)-n > 0 {
-		go Debug(2, "[INPUT-RAW] message truncated, increase copy-buffer-size")
+	if msgTCP.Truncated {
+		Debug(2, "[INPUT-RAW] message truncated, increase copy-buffer-size")
 	}
 	// to be removed...
-	if msg.TimedOut {
-		go Debug(2, "[INPUT-RAW] message timeout reached, increase input-raw-expire")
+	if msgTCP.TimedOut {
+		Debug(2, "[INPUT-RAW] message timeout reached, increase input-raw-expire")
 	}
 	if i.Stats {
-		stat := msg.Stats
+		stat := msgTCP.Stats
 		go i.addStats(stat)
 	}
-	msg = nil
-	return n, nil
+	msgTCP.Finalize()
+	msgTCP = nil
+	return &msg, nil
 }
 
 func (i *RAWInput) listen(address string) {
 	var err error
-	i.listener, err = capture.NewListener(i.host, i.port, "", i.Engine, i.TrackResponse)
+	i.listener, err = capture.NewListener(i.host, i.ports, "", i.Engine, i.TrackResponse)
 	if err != nil {
 		log.Fatal(err)
 	}
@@ -153,26 +158,25 @@ func (i *RAWInput) listen(address string) {
 	if err != nil {
 		log.Fatal(err)
 	}
-	pool := tcp.NewMessagePool(i.CopyBufferSize, i.Expire, Debug, i.handler)
-	pool.End = endHint
-	pool.Start = startHint
+	i.messageParser = tcp.NewMessageParser(i.CopyBufferSize, i.Expire, i.AllowIncomplete, Debug)
+
+	if i.Protocol == ProtocolHTTP {
+		i.messageParser.Start = http1StartHint
+		i.messageParser.End = http1EndHint
+	}
 	var ctx context.Context
 	ctx, i.cancelListener = context.WithCancel(context.Background())
-	errCh := i.listener.ListenBackground(ctx, pool.Handler)
-	select {
-	case err := <-errCh:
-		log.Fatal(err)
-	case <-i.listener.Reading:
-		Debug(1, i)
-	}
-}
-
-func (i *RAWInput) handler(m *tcp.Message) {
-	i.message <- m
+	errCh := i.listener.ListenBackground(ctx, i.messageParser.PacketHandler)
+	<-i.listener.Reading
+	Debug(1, i)
+	go func() {
+		<-errCh // the listener closed voluntarily
+		i.Close()
+	}()
 }
 
 func (i *RAWInput) String() string {
-	return fmt.Sprintf("Intercepting traffic from: %s:%d", i.host, i.port)
+	return fmt.Sprintf("Intercepting traffic from: %s:%s", i.host, strings.Join(strings.Fields(fmt.Sprint(i.ports)), ","))
 }
 
 // GetStats returns the stats so far and reset the stats
@@ -187,8 +191,14 @@ func (i *RAWInput) GetStats() []tcp.Stats {
 
 // Close closes the input raw listener
 func (i *RAWInput) Close() error {
+	i.Lock()
+	defer i.Unlock()
+	if i.closed {
+		return nil
+	}
 	i.cancelListener()
 	close(i.quit)
+	i.closed = true
 	return nil
 }
 
@@ -201,14 +211,23 @@ func (i *RAWInput) addStats(mStats tcp.Stats) {
 	i.Unlock()
 }
 
-func startHint(pckt *tcp.Packet) (isIncoming, isOutgoing bool) {
-	isIncoming = proto.HasRequestTitle(pckt.Payload)
-	if isIncoming {
-		return
+func http1StartHint(pckt *tcp.Packet) (isRequest, isResponse bool) {
+	if proto.HasRequestTitle(pckt.Payload) {
+		return true, false
 	}
-	return false, proto.HasResponseTitle(pckt.Payload)
+
+	if proto.HasResponseTitle(pckt.Payload) {
+		return false, true
+	}
+
+	// No request or response detected
+	return false, false
 }
 
-func endHint(m *tcp.Message) bool {
-	return proto.HasFullPayload(m.Data())
+func http1EndHint(m *tcp.Message) bool {
+	if m.MissingChunk() {
+		return false
+	}
+
+	return proto.HasFullPayload(m, m.PacketData()...)
 }

input_raw_test.go

@@ -2,7 +2,6 @@ package main
 
 import (
 	"bytes"
-	"io"
 	"io/ioutil"
 	"net"
 	"net/http"
@@ -23,9 +22,8 @@ const testRawExpire = time.Millisecond * 200
 
 func TestRAWInputIPv4(t *testing.T) {
 	wg := new(sync.WaitGroup)
-	quit := make(chan int)
 
-	listener, err := net.Listen("tcp", ":0")
+	listener, err := net.Listen("tcp4", "127.0.0.1:0")
 	if err != nil {
 		t.Error(err)
 		return
@@ -49,40 +47,43 @@ func TestRAWInputIPv4(t *testing.T) {
 		TrackResponse: true,
 		RealIPHeader:  "X-Real-IP",
 	}
-	input := NewRAWInput(":"+port, conf)
+	input := NewRAWInput(listener.Addr().String(), conf)
 
-	output := NewTestOutput(func(data []byte) {
-		if data[0] == '1' {
-			body := payloadBody(data)
-			if len(proto.Header(body, []byte("X-Real-IP"))) == 0 {
-				t.Error("Should have X-Real-IP header", string(body))
+	output := NewTestOutput(func(msg *Message) {
+		if msg.Meta[0] == '1' {
+			if len(proto.Header(msg.Data, []byte("X-Real-IP"))) == 0 {
+				t.Error("Should have X-Real-IP header")
 			}
-			atomic.AddInt64(&reqCounter, 1)
+			reqCounter++
 		} else {
-			atomic.AddInt64(&respCounter, 1)
+			respCounter++
 		}
+
 		wg.Done()
 	})
 
 	plugins := &InOutPlugins{
-		Inputs:  []io.Reader{input},
-		Outputs: []io.Writer{output},
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{output},
 	}
 	plugins.All = append(plugins.All, input, output)
 
-	client := NewHTTPClient("127.0.0.1:"+port, &HTTPClientConfig{})
-
-	emitter := NewEmitter(quit)
+	addr := "http://127.0.0.1:" + port
+	emitter := NewEmitter()
 	defer emitter.Close()
 	go emitter.Start(plugins, Settings.Middleware)
-	for i := 0; i < 10; i++ {
+
+	// time.Sleep(time.Second)
+	for i := 0; i < 1; i++ {
 		wg.Add(2)
-		_, err = client.Get("/")
+		_, err = http.Get(addr)
+
 		if err != nil {
 			t.Error(err)
 			return
 		}
 	}
+
 	wg.Wait()
 	const want = 10
 	if reqCounter != respCounter && reqCounter != want {
@@ -92,7 +93,6 @@ func TestRAWInputIPv4(t *testing.T) {
 
 func TestRAWInputNoKeepAlive(t *testing.T) {
 	wg := new(sync.WaitGroup)
-	quit := make(chan int)
 
 	listener, err := net.Listen("tcp", "127.0.0.1:0")
 	if err != nil {
@@ -118,8 +118,8 @@ func TestRAWInputNoKeepAlive(t *testing.T) {
 	}
 	input := NewRAWInput(":"+port, conf)
 	var respCounter, reqCounter int64
-	output := NewTestOutput(func(data []byte) {
-		if data[0] == '1' {
+	output := NewTestOutput(func(msg *Message) {
+		if msg.Meta[0] == '1' {
 			atomic.AddInt64(&reqCounter, 1)
 		} else {
 			atomic.AddInt64(&respCounter, 1)
@@ -128,20 +128,20 @@ func TestRAWInputNoKeepAlive(t *testing.T) {
 	})
 
 	plugins := &InOutPlugins{
-		Inputs:  []io.Reader{input},
-		Outputs: []io.Writer{output},
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{output},
 	}
 	plugins.All = append(plugins.All, input, output)
 
-	client := NewHTTPClient("127.0.0.1:"+port, &HTTPClientConfig{})
+	addr := "http://127.0.0.1:" + port
 
-	emitter := NewEmitter(quit)
+	emitter := NewEmitter()
 	go emitter.Start(plugins, Settings.Middleware)
 
 	for i := 0; i < 10; i++ {
 		// request + response
 		wg.Add(2)
-		_, err = client.Get("/")
+		_, err = http.Get(addr)
 		if err != nil {
 			t.Error(err)
 			return
@@ -158,7 +158,6 @@ func TestRAWInputNoKeepAlive(t *testing.T) {
 
 func TestRAWInputIPv6(t *testing.T) {
 	wg := new(sync.WaitGroup)
-	quit := make(chan int)
 
 	listener, err := net.Listen("tcp", "[::1]:0")
 	if err != nil {
@@ -184,8 +183,8 @@ func TestRAWInputIPv6(t *testing.T) {
 	}
 	input := NewRAWInput(originAddr, conf)
 
-	output := NewTestOutput(func(data []byte) {
-		if data[0] == '1' {
+	output := NewTestOutput(func(msg *Message) {
+		if msg.Meta[0] == '1' {
 			atomic.AddInt64(&reqCounter, 1)
 		} else {
 			atomic.AddInt64(&respCounter, 1)
@@ -194,18 +193,17 @@ func TestRAWInputIPv6(t *testing.T) {
 	})
 
 	plugins := &InOutPlugins{
-		Inputs:  []io.Reader{input},
-		Outputs: []io.Writer{output},
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{output},
 	}
 
-	client := NewHTTPClient(originAddr, &HTTPClientConfig{})
-
-	emitter := NewEmitter(quit)
+	emitter := NewEmitter()
+	addr := "http://" + originAddr
 	go emitter.Start(plugins, Settings.Middleware)
 	for i := 0; i < 10; i++ {
 		// request + response
 		wg.Add(2)
-		_, err = client.Get("/")
+		_, err = http.Get(addr)
 		if err != nil {
 			t.Error(err)
 			return
@@ -222,7 +220,6 @@ func TestRAWInputIPv6(t *testing.T) {
 
 func TestInputRAWChunkedEncoding(t *testing.T) {
 	wg := new(sync.WaitGroup)
-	quit := make(chan int)
 
 	fileContent, _ := ioutil.ReadFile("README.md")
 
@@ -256,15 +253,15 @@ func TestInputRAWChunkedEncoding(t *testing.T) {
 	}))
 	defer replay.Close()
 
-	httpOutput := NewHTTPOutput(replay.URL, &HTTPOutputConfig{Debug: true})
+	httpOutput := NewHTTPOutput(replay.URL, &HTTPOutputConfig{})
 
 	plugins := &InOutPlugins{
-		Inputs:  []io.Reader{input},
-		Outputs: []io.Writer{httpOutput},
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{httpOutput},
 	}
 	plugins.All = append(plugins.All, input, httpOutput)
 
-	emitter := NewEmitter(quit)
+	emitter := NewEmitter()
 	defer emitter.Close()
 	go emitter.Start(plugins, Settings.Middleware)
 	wg.Add(2)
@@ -280,17 +277,15 @@ func TestInputRAWChunkedEncoding(t *testing.T) {
 }
 
 func BenchmarkRAWInputWithReplay(b *testing.B) {
-	var respCounter, reqCounter, replayCounter, capturedBody uint64
+	var respCounter, reqCounter, replayCounter uint32
 	wg := &sync.WaitGroup{}
-	wg.Add(b.N * 3) // reqCounter + replayCounter + respCounter
 
-	quit := make(chan int)
-	listener, err := net.Listen("tcp", "127.0.0.1:0")
+	listener, err := net.Listen("tcp4", "127.0.0.1:0")
 	if err != nil {
 		b.Error(err)
 		return
 	}
-	listener0, err := net.Listen("tcp", "127.0.0.1:0")
+	listener0, err := net.Listen("tcp4", "127.0.0.1:0")
 	if err != nil {
 		b.Error(err)
 		return
@@ -307,15 +302,9 @@ func BenchmarkRAWInputWithReplay(b *testing.B) {
 
 	replay := http.Server{
 		Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-			defer wg.Done()
-			defer r.Body.Close()
-			w.Write([]byte("ab"))
-			atomic.AddUint64(&replayCounter, 1)
-			data, err := ioutil.ReadAll(r.Body)
-			if err != nil && err != io.EOF && err != io.ErrUnexpectedEOF {
-				b.Log(err)
-			}
-			atomic.AddUint64(&capturedBody, uint64(len(data)))
+			atomic.AddUint32(&replayCounter, 1)
+			w.Write(nil)
+			wg.Done()
 		}),
 	}
 	go replay.Serve(listener0)
@@ -330,41 +319,37 @@ func BenchmarkRAWInputWithReplay(b *testing.B) {
 	}
 	input := NewRAWInput(originAddr, conf)
 
-	testOutput := NewTestOutput(func(data []byte) {
-		if data[0] == '1' {
-			atomic.AddUint64(&reqCounter, 1)
+	testOutput := NewTestOutput(func(msg *Message) {
+		if msg.Meta[0] == '1' {
+			reqCounter++
 		} else {
-			atomic.AddUint64(&respCounter, 1)
+			respCounter++
 		}
-		atomic.AddUint64(&capturedBody, uint64(len(data)))
 		wg.Done()
 	})
-	httpOutput := NewHTTPOutput(replayAddr, &HTTPOutputConfig{Debug: false})
+	httpOutput := NewHTTPOutput("http://"+replayAddr, &HTTPOutputConfig{})
 
 	plugins := &InOutPlugins{
-		Inputs:  []io.Reader{input},
-		Outputs: []io.Writer{testOutput, httpOutput},
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{testOutput, httpOutput},
 	}
 
-	emitter := NewEmitter(quit)
+	emitter := NewEmitter()
 	go emitter.Start(plugins, Settings.Middleware)
-	now := time.Now()
-	var buf [1 << 20]byte
-	buf[1<<20-1] = 'a'
-	client := NewHTTPClient(originAddr, &HTTPClientConfig{ResponseBufferSize: 2 << 20, CompatibilityMode: true})
+	addr := "http://" + originAddr
+	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
-		if i&1 == 0 {
-			_, err = client.Get("/")
-		} else {
-			_, err = client.Post("/", buf[:])
-		}
+		wg.Add(3) // reqCounter + replayCounter + respCounter
+		resp, err := http.Get(addr)
 		if err != nil {
-			b.Log(err)
 			wg.Add(-3)
 		}
+		resp.Body.Close()
 	}
 
 	wg.Wait()
-	b.Logf("%d/%d Requests, %d/%d Responses, %d/%d Replayed, %d Bytes in %s\n", reqCounter, b.N, respCounter, b.N, replayCounter, b.N, capturedBody, time.Since(now))
+	b.ReportMetric(float64(reqCounter), "requests")
+	b.ReportMetric(float64(respCounter), "responses")
+	b.ReportMetric(float64(replayCounter), "replayed")
 	emitter.Close()
 }

input_tcp.go

@@ -8,18 +8,18 @@ import (
 	"io"
 	"log"
 	"net"
-	"os"
 )
 
 // TCPInput used for internal communication
 type TCPInput struct {
-	data     chan []byte
+	data     chan *Message
 	listener net.Listener
 	address  string
 	config   *TCPInputConfig
 	stop     chan bool // Channel used only to indicate goroutine should shutdown
 }
 
+// TCPInputConfig represents configuration of a TCP input plugin
 type TCPInputConfig struct {
 	Secure          bool   `json:"input-tcp-secure"`
 	CertificatePath string `json:"input-tcp-certificate"`
@@ -29,7 +29,7 @@ type TCPInputConfig struct {
 // NewTCPInput constructor for TCPInput, accepts address with port
 func NewTCPInput(address string, config *TCPInputConfig) (i *TCPInput) {
 	i = new(TCPInput)
-	i.data = make(chan []byte, 1000)
+	i.data = make(chan *Message, 1000)
 	i.address = address
 	i.config = config
 	i.stop = make(chan bool)
@@ -39,20 +39,21 @@ func NewTCPInput(address string, config *TCPInputConfig) (i *TCPInput) {
 	return
 }
 
-func (i *TCPInput) Read(data []byte) (int, error) {
-	var buf []byte
+// PluginRead returns data and details read from plugin
+func (i *TCPInput) PluginRead() (msg *Message, err error) {
 	select {
 	case <-i.stop:
-		return 0, ErrorStopped
-	case buf = <-i.data:
+		return nil, ErrorStopped
+	case msg = <-i.data:
+		return msg, nil
 	}
-	copy(data, buf)
 
-	return len(buf), nil
 }
 
+// Close closes the plugin
 func (i *TCPInput) Close() error {
 	close(i.stop)
+	i.listener.Close()
 	return nil
 }
 
@@ -60,64 +61,67 @@ func (i *TCPInput) listen(address string) {
 	if i.config.Secure {
 		cer, err := tls.LoadX509KeyPair(i.config.CertificatePath, i.config.KeyPath)
 		if err != nil {
-			log.Fatal("Error while loading --input-file certificate:", err)
+			log.Fatalln("error while loading --input-tcp TLS certificate:", err)
 		}
 
 		config := &tls.Config{Certificates: []tls.Certificate{cer}}
 		listener, err := tls.Listen("tcp", address, config)
 		if err != nil {
-			log.Fatal("Can't start --input-tcp with secure connection:", err)
+			log.Fatalln("[INPUT-TCP] failed to start INPUT-TCP listener:", err)
 		}
 		i.listener = listener
 	} else {
 		listener, err := net.Listen("tcp", address)
 		if err != nil {
-			log.Fatal("Can't start:", err)
+			log.Fatalln("failed to start INPUT-TCP listener:", err)
 		}
-
 		i.listener = listener
 	}
-
 	go func() {
 		for {
 			conn, err := i.listener.Accept()
-
-			if err != nil {
-				log.Println("Error while Accept()", err)
+			if err == nil {
+				go i.handleConnection(conn)
 				continue
 			}
-
-			go i.handleConnection(conn)
+			if isTemporaryNetworkError(err) {
+				continue
+			}
+			if operr, ok := err.(*net.OpError); ok && operr.Err.Error() != "use of closed network connection" {
+				Debug(0, fmt.Sprintf("[INPUT-TCP] listener closed, err: %q", err))
+			}
+			break
 		}
 	}()
 }
 
+var payloadSeparatorAsBytes = []byte(payloadSeparator)
+
 func (i *TCPInput) handleConnection(conn net.Conn) {
 	defer conn.Close()
 
-	payloadSeparatorAsBytes := []byte(payloadSeparator)
 	reader := bufio.NewReader(conn)
 	var buffer bytes.Buffer
 
 	for {
 		line, err := reader.ReadBytes('\n')
-
 		if err != nil {
+			if isTemporaryNetworkError(err) {
+				continue
+			}
 			if err != io.EOF {
-				fmt.Fprintln(os.Stderr, "Unexpected error in input tcp connection:", err)
+				Debug(0, fmt.Sprintf("[INPUT-TCP] connection error: %q", err))
 			}
 			break
-
 		}
 
 		if bytes.Equal(payloadSeparatorAsBytes[1:], line) {
-			asBytes := buffer.Bytes()
+			// unread the '\n' before monkeys
+			buffer.UnreadByte()
+			var msg Message
+			msg.Meta, msg.Data = payloadMetaWithBody(buffer.Bytes())
+			i.data <- &msg
 			buffer.Reset()
-
-			newBuf := make([]byte, len(asBytes)-1)
-			copy(newBuf, asBytes)
-
-			i.data <- newBuf
 		} else {
 			buffer.Write(line)
 		}
@@ -127,3 +131,13 @@ func (i *TCPInput) handleConnection(conn net.Conn) {
 func (i *TCPInput) String() string {
 	return "TCP input: " + i.address
 }
+
+func isTemporaryNetworkError(err error) bool {
+	if nerr, ok := err.(net.Error); ok && nerr.Temporary() {
+		return true
+	}
+	if operr, ok := err.(*net.OpError); ok && operr.Temporary() {
+		return true
+	}
+	return false
+}

input_tcp_test.go

@@ -7,7 +7,6 @@ import (
 	"crypto/tls"
 	"crypto/x509"
 	"encoding/pem"
-	"io"
 	"io/ioutil"
 	"log"
 	"math/big"
@@ -20,20 +19,19 @@ import (
 
 func TestTCPInput(t *testing.T) {
 	wg := new(sync.WaitGroup)
-	quit := make(chan int)
 
 	input := NewTCPInput("127.0.0.1:0", &TCPInputConfig{})
-	output := NewTestOutput(func(data []byte) {
+	output := NewTestOutput(func(*Message) {
 		wg.Done()
 	})
 
 	plugins := &InOutPlugins{
-		Inputs:  []io.Reader{input},
-		Outputs: []io.Writer{output},
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{output},
 	}
 	plugins.All = append(plugins.All, input, output)
 
-	emitter := NewEmitter(quit)
+	emitter := NewEmitter()
 	go emitter.Start(plugins, Settings.Middleware)
 
 	tcpAddr, err := net.ResolveTCPAddr("tcp", input.listener.Addr().String())
@@ -43,7 +41,6 @@ func TestTCPInput(t *testing.T) {
 	}
 
 	conn, err := net.DialTCP("tcp", nil, tcpAddr)
-
 	if err != nil {
 		log.Fatal(err)
 	}
@@ -52,10 +49,14 @@ func TestTCPInput(t *testing.T) {
 
 	for i := 0; i < 100; i++ {
 		wg.Add(1)
-		conn.Write(msg)
-		conn.Write([]byte(payloadSeparator))
+		if _, err = conn.Write(msg); err == nil {
+			_, err = conn.Write(payloadSeparatorAsBytes)
+		}
+		if err != nil {
+			t.Error(err)
+			return
+		}
 	}
-
 	wg.Wait()
 	emitter.Close()
 }
@@ -99,24 +100,23 @@ func TestTCPInputSecure(t *testing.T) {
 	}()
 
 	wg := new(sync.WaitGroup)
-	quit := make(chan int)
 
 	input := NewTCPInput("127.0.0.1:0", &TCPInputConfig{
 		Secure:          true,
 		CertificatePath: serverCertPemFile.Name(),
 		KeyPath:         serverPrivPemFile.Name(),
 	})
-	output := NewTestOutput(func(data []byte) {
+	output := NewTestOutput(func(*Message) {
 		wg.Done()
 	})
 
 	plugins := &InOutPlugins{
-		Inputs:  []io.Reader{input},
-		Outputs: []io.Writer{output},
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{output},
 	}
 	plugins.All = append(plugins.All, input, output)
 
-	emitter := NewEmitter(quit)
+	emitter := NewEmitter()
 	go emitter.Start(plugins, Settings.Middleware)
 
 	conf := &tls.Config{

kafka.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"crypto/tls"
 	"crypto/x509"
+	"errors"
 	"fmt"
 	"io/ioutil"
 	"log"
@@ -12,9 +13,8 @@ import (
 	"github.com/buger/goreplay/proto"
 )
 
-// KafkaConfig should contains required information to
+// InputKafkaConfig should contains required information to
 // build producers.
-
 type InputKafkaConfig struct {
 	producer sarama.AsyncProducer
 	consumer sarama.Consumer
@@ -23,6 +23,7 @@ type InputKafkaConfig struct {
 	UseJSON  bool   `json:"input-kafka-json-format"`
 }
 
+// OutputKafkaConfig is the representation of kfka output configuration
 type OutputKafkaConfig struct {
 	producer sarama.AsyncProducer
 	consumer sarama.Consumer
@@ -34,8 +35,8 @@ type OutputKafkaConfig struct {
 // KafkaTLSConfig should contains TLS certificates for connecting to secured Kafka clusters
 type KafkaTLSConfig struct {
 	CACert     string `json:"kafka-tls-ca-cert"`
-	clientCert string `json:"kafka-tls-client-cert"`
-	clientKey  string `json:"kafka-tls-client-key"`
+	ClientCert string `json:"kafka-tls-client-cert"`
+	ClientKey  string `json:"kafka-tls-client-key"`
 }
 
 // KafkaMessage should contains catched request information that should be
@@ -54,32 +55,40 @@ type KafkaMessage struct {
 func NewTLSConfig(clientCertFile, clientKeyFile, caCertFile string) (*tls.Config, error) {
 	tlsConfig := tls.Config{}
 
+	if clientCertFile != "" && clientKeyFile == "" {
+		return &tlsConfig, errors.New("Missing key of client certificate in kafka")
+	}
+	if clientCertFile == "" && clientKeyFile != "" {
+		return &tlsConfig, errors.New("missing TLS client certificate in kafka")
+	}
 	// Load client cert
-	cert, err := tls.LoadX509KeyPair(clientCertFile, clientKeyFile)
-	if err != nil {
-		return &tlsConfig, err
+	if (clientCertFile != "") && (clientKeyFile != "") {
+		cert, err := tls.LoadX509KeyPair(clientCertFile, clientKeyFile)
+		if err != nil {
+			return &tlsConfig, err
+		}
+		tlsConfig.Certificates = []tls.Certificate{cert}
 	}
-	tlsConfig.Certificates = []tls.Certificate{cert}
-
 	// Load CA cert
-	caCert, err := ioutil.ReadFile(caCertFile)
-	if err != nil {
-		return &tlsConfig, err
+	if caCertFile != "" {
+		caCert, err := ioutil.ReadFile(caCertFile)
+		if err != nil {
+			return &tlsConfig, err
+		}
+		caCertPool := x509.NewCertPool()
+		caCertPool.AppendCertsFromPEM(caCert)
+		tlsConfig.RootCAs = caCertPool
 	}
-	caCertPool := x509.NewCertPool()
-	caCertPool.AppendCertsFromPEM(caCert)
-	tlsConfig.RootCAs = caCertPool
-
-	return &tlsConfig, err
+	return &tlsConfig, nil
 }
 
 // NewKafkaConfig returns Kafka config with or without TLS
 func NewKafkaConfig(tlsConfig *KafkaTLSConfig) *sarama.Config {
 	config := sarama.NewConfig()
 	// Configuration options go here
-	if (tlsConfig != nil) && (tlsConfig.CACert != "") && (tlsConfig.clientCert != "") && (tlsConfig.clientKey != "") {
+	if tlsConfig != nil && (tlsConfig.ClientCert != "" || tlsConfig.CACert != "") {
 		config.Net.TLS.Enable = true
-		tlsConfig, err := NewTLSConfig(tlsConfig.clientCert, tlsConfig.clientKey, tlsConfig.CACert)
+		tlsConfig, err := NewTLSConfig(tlsConfig.ClientCert, tlsConfig.ClientKey, tlsConfig.CACert)
 		if err != nil {
 			log.Fatal(err)
 		}

limiter.go

@@ -20,8 +20,8 @@ type Limiter struct {
 }
 
 func parseLimitOptions(options string) (limit int, isPercent bool) {
-	if strings.Contains(options, "%") {
-		limit, _ = strconv.Atoi(strings.Split(options, "%")[0])
+	if n := strings.Index(options, "%"); n > 0 {
+		limit, _ = strconv.Atoi(options[:n])
 		isPercent = true
 	} else {
 		limit, _ = strconv.Atoi(options)
@@ -33,7 +33,7 @@ func parseLimitOptions(options string) (limit int, isPercent bool) {
 
 // NewLimiter constructor for Limiter, accepts plugin and options
 // `options` allow to sprcify relatve or absolute limiting
-func NewLimiter(plugin interface{}, options string) io.ReadWriter {
+func NewLimiter(plugin interface{}, options string) PluginReadWriter {
 	l := new(Limiter)
 	l.limit, l.isPercent = parseLimitOptions(options)
 	l.plugin = plugin
@@ -71,24 +71,29 @@ func (l *Limiter) isLimited() bool {
 	return false
 }
 
-func (l *Limiter) Write(data []byte) (n int, err error) {
+// PluginWrite writes message to this plugin
+func (l *Limiter) PluginWrite(msg *Message) (n int, err error) {
 	if l.isLimited() {
 		return 0, nil
 	}
-
-	n, err = l.plugin.(io.Writer).Write(data)
-	return
+	if w, ok := l.plugin.(PluginWriter); ok {
+		return w.PluginWrite(msg)
+	}
+	// avoid further writing
+	return 0, io.ErrClosedPipe
 }
 
-func (l *Limiter) Read(data []byte) (n int, err error) {
-	if r, ok := l.plugin.(io.Reader); ok {
-		n, err = r.Read(data)
+// PluginRead reads message from this plugin
+func (l *Limiter) PluginRead() (msg *Message, err error) {
+	if r, ok := l.plugin.(PluginReader); ok {
+		msg, err = r.PluginRead()
 	} else {
-		return 0, nil
+		// avoid further reading
+		return nil, io.ErrClosedPipe
 	}
 
 	if l.isLimited() {
-		return 0, nil
+		return nil, nil
 	}
 
 	return
@@ -100,7 +105,7 @@ func (l *Limiter) String() string {
 
 // Close closes the resources.
 func (l *Limiter) Close() error {
-	if fi, ok := l.plugin.(io.ReadCloser); ok {
+	if fi, ok := l.plugin.(io.Closer); ok {
 		fi.Close()
 	}
 	return nil

limiter_test.go

@@ -3,28 +3,26 @@
 package main
 
 import (
-	"io"
 	"sync"
 	"testing"
 )
 
 func TestOutputLimiter(t *testing.T) {
 	wg := new(sync.WaitGroup)
-	quit := make(chan int)
 
 	input := NewTestInput()
-	output := NewLimiter(NewTestOutput(func(data []byte) {
+	output := NewLimiter(NewTestOutput(func(*Message) {
 		wg.Done()
 	}), "10")
 	wg.Add(10)
 
 	plugins := &InOutPlugins{
-		Inputs:  []io.Reader{input},
-		Outputs: []io.Writer{output},
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{output},
 	}
 	plugins.All = append(plugins.All, input, output)
 
-	emitter := NewEmitter(quit)
+	emitter := NewEmitter()
 	go emitter.Start(plugins, Settings.Middleware)
 
 	for i := 0; i < 100; i++ {
@@ -37,21 +35,20 @@ func TestOutputLimiter(t *testing.T) {
 
 func TestInputLimiter(t *testing.T) {
 	wg := new(sync.WaitGroup)
-	quit := make(chan int)
 
 	input := NewLimiter(NewTestInput(), "10")
-	output := NewTestOutput(func(data []byte) {
+	output := NewTestOutput(func(*Message) {
 		wg.Done()
 	})
 	wg.Add(10)
 
 	plugins := &InOutPlugins{
-		Inputs:  []io.Reader{input},
-		Outputs: []io.Writer{output},
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{output},
 	}
 	plugins.All = append(plugins.All, input, output)
 
-	emitter := NewEmitter(quit)
+	emitter := NewEmitter()
 	go emitter.Start(plugins, Settings.Middleware)
 
 	for i := 0; i < 100; i++ {
@@ -65,20 +62,19 @@ func TestInputLimiter(t *testing.T) {
 // Should limit all requests
 func TestPercentLimiter1(t *testing.T) {
 	wg := new(sync.WaitGroup)
-	quit := make(chan int)
 
 	input := NewTestInput()
-	output := NewLimiter(NewTestOutput(func(data []byte) {
+	output := NewLimiter(NewTestOutput(func(*Message) {
 		wg.Done()
 	}), "0%")
 
 	plugins := &InOutPlugins{
-		Inputs:  []io.Reader{input},
-		Outputs: []io.Writer{output},
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{output},
 	}
 	plugins.All = append(plugins.All, input, output)
 
-	emitter := NewEmitter(quit)
+	emitter := NewEmitter()
 	go emitter.Start(plugins, Settings.Middleware)
 
 	for i := 0; i < 100; i++ {
@@ -91,21 +87,20 @@ func TestPercentLimiter1(t *testing.T) {
 // Should not limit at all
 func TestPercentLimiter2(t *testing.T) {
 	wg := new(sync.WaitGroup)
-	quit := make(chan int)
 
 	input := NewTestInput()
-	output := NewLimiter(NewTestOutput(func(data []byte) {
+	output := NewLimiter(NewTestOutput(func(*Message) {
 		wg.Done()
 	}), "100%")
 	wg.Add(100)
 
 	plugins := &InOutPlugins{
-		Inputs:  []io.Reader{input},
-		Outputs: []io.Writer{output},
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{output},
 	}
 	plugins.All = append(plugins.All, input, output)
 
-	emitter := NewEmitter(quit)
+	emitter := NewEmitter()
 	go emitter.Start(plugins, Settings.Middleware)
 
 	for i := 0; i < 100; i++ {

middleware.go

@@ -2,39 +2,40 @@ package main
 
 import (
 	"bufio"
+	"context"
 	"encoding/hex"
 	"fmt"
 	"io"
-	"log"
 	"os"
 	"os/exec"
 	"strings"
 	"sync"
+	"syscall"
 )
 
 // Middleware represents a middleware object
 type Middleware struct {
-	command string
-
-	data chan []byte
-
-	mu sync.Mutex
-
-	Stdin  io.Writer
-	Stdout io.Reader
-
-	stop chan bool // Channel used only to indicate goroutine should shutdown
+	command       string
+	data          chan *Message
+	Stdin         io.Writer
+	Stdout        io.Reader
+	commandCancel context.CancelFunc
+	stop          chan bool // Channel used only to indicate goroutine should shutdown
+	closed        bool
+	mu            sync.RWMutex
 }
 
 // NewMiddleware returns new middleware
 func NewMiddleware(command string) *Middleware {
 	m := new(Middleware)
 	m.command = command
-	m.data = make(chan []byte, 1000)
+	m.data = make(chan *Message, 1000)
 	m.stop = make(chan bool)
 
 	commands := strings.Split(command, " ")
-	cmd := exec.Command(commands[0], commands[1:]...)
+	ctx, cancl := context.WithCancel(context.Background())
+	m.commandCancel = cancl
+	cmd := exec.CommandContext(ctx, commands[0], commands[1:]...)
 
 	m.Stdout, _ = cmd.StdoutPipe()
 	m.Stdin, _ = cmd.StdinPipe()
@@ -44,16 +45,19 @@ func NewMiddleware(command string) *Middleware {
 	go m.read(m.Stdout)
 
 	go func() {
-		err := cmd.Start()
-
-		if err != nil {
-			log.Fatal(err)
+		defer m.Close()
+		var err error
+		if err = cmd.Start(); err == nil {
+			err = cmd.Wait()
 		}
-
-		err = cmd.Wait()
-
 		if err != nil {
-			log.Fatal(err)
+			if e, ok := err.(*exec.ExitError); ok {
+				status := e.Sys().(syscall.WaitStatus)
+				if status.Signal() == syscall.SIGKILL /*killed or context canceld */ {
+					return
+				}
+			}
+			Debug(0, fmt.Sprintf("[MIDDLEWARE] command[%q] error: %q", command, err.Error()))
 		}
 	}()
 
@@ -61,45 +65,42 @@ func NewMiddleware(command string) *Middleware {
 }
 
 // ReadFrom start a worker to read from this plugin
-func (m *Middleware) ReadFrom(plugin io.Reader) {
-	Debug(2, "[MIDDLEWARE-MASTER] Starting reading from", plugin)
+func (m *Middleware) ReadFrom(plugin PluginReader) {
+	Debug(2, fmt.Sprintf("[MIDDLEWARE] command[%q] Starting reading from %q", m.command, plugin))
 	go m.copy(m.Stdin, plugin)
 }
 
-func (m *Middleware) copy(to io.Writer, from io.Reader) {
-	buf := make([]byte, 5*1024*1024)
-	dst := make([]byte, len(buf)*4)
+func (m *Middleware) copy(to io.Writer, from PluginReader) {
+	var buf, dst []byte
 
 	for {
-		nr, _ := from.Read(buf)
-		if nr == 0 || nr > len(buf) {
+		msg, err := from.PluginRead()
+		if err != nil {
+			return
+		}
+		if msg == nil || len(msg.Data) == 0 {
 			continue
 		}
-
-		payload := buf[0:nr]
-
+		buf = msg.Data
 		if Settings.PrettifyHTTP {
-			payload = prettifyHTTP(payload)
-			nr = len(payload)
-
-			if nr*2 > len(dst) {
-				continue
-			}
+			buf = prettifyHTTP(msg.Data)
 		}
-
-		if Settings.PrettifyHTTP {
-			payload = prettifyHTTP(payload)
-			nr = len(payload)
+		dstLen := (len(buf)+len(msg.Meta))*2 + 1
+		// if enough space was previously allocated use it instead
+		if dstLen > len(dst) {
+			dst = make([]byte, dstLen)
 		}
+		n := hex.Encode(dst, msg.Meta)
+		n += hex.Encode(dst[n:], buf)
+		dst[n] = '\n'
 
-		hex.Encode(dst, payload)
-		dst[nr*2] = '\n'
-
-		m.mu.Lock()
-		to.Write(dst[0 : nr*2+1])
-		m.mu.Unlock()
-
-		Debug(3, "[MIDDLEWARE-MASTER] Sending:", string(buf[0:nr]), "From:", from)
+		n, err = to.Write(dst[:n+1])
+		if err == nil {
+			continue
+		}
+		if m.isClosed() {
+			return
+		}
 	}
 }
 
@@ -107,51 +108,59 @@ func (m *Middleware) read(from io.Reader) {
 	reader := bufio.NewReader(from)
 	var line []byte
 	var e error
-
 	for {
 		if line, e = reader.ReadBytes('\n'); e != nil {
-			if e == io.EOF {
-				continue
-			} else {
-				break
+			if m.isClosed() {
+				return
 			}
+			continue
 		}
-
-		buf := make([]byte, len(line)/2)
+		buf := make([]byte, (len(line)-1)/2)
 		if _, err := hex.Decode(buf, line[:len(line)-1]); err != nil {
-			fmt.Fprintln(os.Stderr, "Failed to decode input payload", err, len(line), string(line[:len(line)-1]))
+			Debug(0, fmt.Sprintf("[MIDDLEWARE] command[%q] failed to decode err: %q", m.command, err))
+			continue
 		}
-
-		Debug(3, "[MIDDLEWARE-MASTER] Received:", string(buf))
-
+		var msg Message
+		msg.Meta, msg.Data = payloadMetaWithBody(buf)
 		select {
 		case <-m.stop:
 			return
-		case m.data <- buf:
+		case m.data <- &msg:
 		}
 	}
 
+}
+
+// PluginRead reads message from this plugin
+func (m *Middleware) PluginRead() (msg *Message, err error) {
+	select {
+	case <-m.stop:
+		return nil, ErrorStopped
+	case msg = <-m.data:
+	}
+
 	return
 }
 
-func (m *Middleware) Read(data []byte) (int, error) {
-	var buf []byte
-	select {
-	case <-m.stop:
-		return 0, ErrorStopped
-	case buf = <-m.data:
-	}
-
-	n := copy(data, buf)
-	return n, nil
+func (m *Middleware) String() string {
+	return fmt.Sprintf("Modifying traffic using %q command", m.command)
 }
 
-func (m *Middleware) String() string {
-	return fmt.Sprintf("Modifying traffic using '%s' command", m.command)
+func (m *Middleware) isClosed() bool {
+	m.mu.RLock()
+	defer m.mu.RUnlock()
+	return m.closed
 }
 
 // Close closes this plugin
 func (m *Middleware) Close() error {
+	if m.isClosed() {
+		return nil
+	}
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	m.commandCancel()
 	close(m.stop)
+	m.closed = true
 	return nil
 }

middleware_test.go

@@ -1,225 +1,154 @@
 package main
 
-// import (
-// 	"bytes"
-// 	"crypto/rand"
-// 	"encoding/hex"
-// 	"io"
-// 	"net/http"
-// 	"net/http/httptest"
-// 	"strings"
-// 	"sync"
-// 	"testing"
-// 	"time"
+import (
+	"bytes"
+	"context"
+	"os"
+	"os/exec"
+	"strings"
+	"sync/atomic"
+	"syscall"
+	"testing"
 
-// 	"github.com/buger/goreplay/capture"
-// 	"github.com/buger/goreplay/proto"
-// )
+	"github.com/buger/goreplay/proto"
+)
 
-// type fakeServiceCb func(string, int, []byte)
+const echoSh = "./examples/middleware/echo.sh"
+const tokenModifier = "go run ./examples/middleware/token_modifier.go"
 
-// // Simple service that generate token on request, and require this token for accesing to secure area
-// func NewFakeSecureService(wg *sync.WaitGroup, cb fakeServiceCb) *httptest.Server {
-// 	active_tokens := make([]string, 0)
-// 	var mu sync.Mutex
+var noDebug = append(syscall.Environ(), "GOR_TEST=1")
 
-// 	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
-// 		mu.Lock()
-// 		defer mu.Unlock()
+func initMiddleware(cmd *exec.Cmd, cancl context.CancelFunc, l PluginReader, c func(error)) *Middleware {
+	var m Middleware
+	m.data = make(chan *Message, 1000)
+	m.stop = make(chan bool)
+	m.commandCancel = cancl
+	m.Stdout, _ = cmd.StdoutPipe()
+	m.Stdin, _ = cmd.StdinPipe()
+	cmd.Stderr = os.Stderr
+	go m.read(m.Stdout)
+	go func() {
+		defer m.Close()
+		var err error
+		if err = cmd.Start(); err == nil {
+			err = cmd.Wait()
+		}
+		if err != nil {
+			c(err)
+		}
+	}()
+	m.ReadFrom(l)
+	return &m
+}
 
-// 		switch req.URL.Path {
-// 		case "/token":
-// 			// Generate random token
-// 			token_length := 10
-// 			buf := make([]byte, token_length)
-// 			rand.Read(buf)
-// 			token := hex.EncodeToString(buf)
-// 			active_tokens = append(active_tokens, token)
+func initCmd(command string, env []string) (*exec.Cmd, context.CancelFunc) {
+	commands := strings.Split(command, " ")
+	ctx, cancl := context.WithCancel(context.Background())
+	cmd := exec.CommandContext(ctx, commands[0], commands[1:]...)
+	cmd.Env = env
+	return cmd, cancl
+}
 
-// 			w.Write([]byte(token))
+func TestMiddlewareEarlyClose(t *testing.T) {
+	quit := make(chan struct{})
+	in := NewTestInput()
+	cmd, cancl := initCmd(echoSh, noDebug)
+	midd := initMiddleware(cmd, cancl, in, func(err error) {
+		if err != nil {
+			if e, ok := err.(*exec.ExitError); ok {
+				status := e.Sys().(syscall.WaitStatus)
+				if status.Signal() != syscall.SIGKILL {
+					t.Errorf("expected error to be signal killed. got %s", status.Signal().String())
+				}
+			}
+		}
+		quit <- struct{}{}
+	})
+	var body = []byte("OPTIONS / HTTP/1.1\r\nHost: example.org\r\n\r\n")
+	count := uint32(0)
+	out := NewTestOutput(func(msg *Message) {
+		if !bytes.Equal(body, msg.Data) {
+			t.Errorf("expected %q to equal %q", body, msg.Data)
+		}
+		atomic.AddUint32(&count, 1)
+		if atomic.LoadUint32(&count) == 5 {
+			quit <- struct{}{}
+		}
+	})
+	pl := &InOutPlugins{}
+	pl.Inputs = []PluginReader{midd, in}
+	pl.Outputs = []PluginWriter{out}
+	pl.All = []interface{}{midd, out, in}
+	e := NewEmitter()
+	go e.Start(pl, "")
+	for i := 0; i < 5; i++ {
+		in.EmitBytes(body)
+	}
+	<-quit
+	midd.Close()
+	<-quit
+}
 
-// 			cb(req.URL.Path, 200, []byte(token))
-// 		case "/secure":
-// 			token := req.URL.Query().Get("token")
-// 			token_found := false
+func TestTokenMiddleware(t *testing.T) {
+	quit := make(chan struct{})
+	in := NewTestInput()
+	in.skipHeader = true
+	cmd, cancl := initCmd(tokenModifier, noDebug)
+	midd := initMiddleware(cmd, cancl, in, func(err error) {})
+	req := []byte("1 932079936fa4306fc308d67588178d17d823647c 1439818823587396305 200\nGET /token HTTP/1.1\r\nHost: example.org\r\n\r\n")
+	res := []byte("2 932079936fa4306fc308d67588178d17d823647c 1439818823587396305 200\nHTTP/1.1 200 OK\r\nContent-Length: 10\r\nContent-Type: text/plain; charset=utf-8\r\n\r\n17d823647c")
+	rep := []byte("3 932079936fa4306fc308d67588178d17d823647c 1439818823587396305 200\nHTTP/1.1 200 OK\r\nContent-Length: 15\r\nContent-Type: text/plain; charset=utf-8\r\n\r\n932079936fa4306")
+	count := uint32(0)
+	out := NewTestOutput(func(msg *Message) {
+		if msg.Meta[0] == '1' && !bytes.Equal(payloadID(msg.Meta), payloadID(req)) {
+			token, _, _ := proto.PathParam(msg.Data, []byte("token"))
+			if !bytes.Equal(token, proto.Body(rep)) {
+				t.Error("expected the token to be equal to the replayed responses's token")
+			}
+		}
+		atomic.AddUint32(&count, 1)
+		if atomic.LoadUint32(&count) == 2 {
+			quit <- struct{}{}
+		}
+	})
+	pl := &InOutPlugins{}
+	pl.Inputs = []PluginReader{midd, in}
+	pl.Outputs = []PluginWriter{out}
+	pl.All = []interface{}{midd, out, in}
+	e := NewEmitter()
+	go e.Start(pl, "")
+	in.EmitBytes(req) // emit original request
+	in.EmitBytes(res) // emit its response
+	in.EmitBytes(rep) // emit replayed response
+	// emit the request which should have modified token
+	token := []byte("1 8e091765ae902fef8a2b7d9dd96 14398188235873 100\nGET /?token=17d823647c HTTP/1.1\r\nHost: example.org\r\n\r\n")
+	in.EmitBytes(token)
+	<-quit
+	midd.Close()
+}
 
-// 			for _, t := range active_tokens {
-// 				if t == token {
-// 					token_found = true
-// 					break
-// 				}
-// 			}
-
-// 			if token_found {
-// 				w.WriteHeader(http.StatusAccepted)
-// 				cb(req.URL.Path, 202, []byte(nil))
-// 			} else {
-// 				w.WriteHeader(http.StatusForbidden)
-// 				cb(req.URL.Path, 403, []byte(nil))
-// 			}
-// 		}
-
-// 		wg.Done()
-// 	}))
-
-// 	return server
-// }
-
-// func TestFakeSecureService(t *testing.T) {
-// 	var resp, token []byte
-
-// 	wg := new(sync.WaitGroup)
-
-// 	server := NewFakeSecureService(wg, func(path string, status int, resp []byte) {
-// 	})
-// 	defer server.Close()
-
-// 	wg.Add(3)
-
-// 	client := NewHTTPClient(server.URL, &HTTPClientConfig{Debug: true})
-// 	resp, _ = client.Get("/token")
-// 	token = proto.Body(resp)
-
-// 	// Right token
-// 	resp, _ = client.Get("/secure?token=" + string(token))
-// 	if !bytes.Equal(proto.Status(resp), []byte("202")) {
-// 		t.Error("Valid token should return status 202:", string(proto.Status(resp)))
-// 	}
-
-// 	// Wrong tokens forbidden
-// 	resp, _ = client.Get("/secure?token=wrong")
-// 	if !bytes.Equal(proto.Status(resp), []byte("403")) {
-// 		t.Error("Wrong token should returns status 403:", string(proto.Status(resp)))
-// 	}
-
-// 	wg.Wait()
-// }
-
-// func TestEchoMiddleware(t *testing.T) {
-// 	wg := new(sync.WaitGroup)
-
-// 	from := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-// 		w.Header().Set("Env", "prod")
-// 		w.Header().Set("RequestPath", r.URL.Path)
-// 		wg.Done()
-// 	}))
-// 	defer from.Close()
-
-// 	to := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-// 		w.Header().Set("Env", "test")
-// 		w.Header().Set("RequestPath", r.URL.Path)
-// 		wg.Done()
-// 	}))
-// 	defer to.Close()
-
-// 	quit := make(chan int)
-
-// 	// Catch traffic from one service
-// 	fromAddr := strings.Replace(from.Listener.Addr().String(), "[::]", "127.0.0.1", -1)
-// 	conf := RAWInputConfig{
-// 		engine:        capture.EnginePcap,
-// 		expire:        testRawExpire,
-// 		protocol:      ProtocolHTTP,
-// 		trackResponse: true,
-// 	}
-// 	input := NewRAWInput(fromAddr, conf)
-
-// 	// And redirect to another
-// 	output := NewHTTPOutput(to.URL, &HTTPOutputConfig{Debug: false})
-
-// 	plugins := &InOutPlugins{
-// 		Inputs:  []io.Reader{input},
-// 		Outputs: []io.Writer{output},
-// 	}
-// 	plugins.All = append(plugins.All, input, output)
-
-// 	// Start Gor
-// 	emitter := NewEmitter(quit)
-// 	go emitter.Start(plugins, "echo -n && GOR_TEST=true && ./examples/middleware/echo.sh")
-
-// 	// Wait till middleware initialization
-// 	time.Sleep(100 * time.Millisecond)
-
-// 	// Should receive 2 requests from original + 2 from replayed
-// 	client := NewHTTPClient(from.URL, &HTTPClientConfig{Debug: false})
-
-// 	for i := 0; i < 10; i++ {
-// 		wg.Add(2)
-// 		// Request should be echoed
-// 		client.Get("/a")
-// 		time.Sleep(5 * time.Millisecond)
-// 		client.Get("/b")
-// 		time.Sleep(5 * time.Millisecond)
-// 	}
-
-// 	wg.Wait()
-// 	emitter.Close()
-// }
-
-// func TestTokenMiddleware(t *testing.T) {
-// 	var resp, token []byte
-
-// 	wg := new(sync.WaitGroup)
-
-// 	from := NewFakeSecureService(wg, func(path string, status int, tok []byte) {
-// 		time.Sleep(10 * time.Millisecond)
-// 	})
-// 	defer from.Close()
-
-// 	to := NewFakeSecureService(wg, func(path string, status int, tok []byte) {
-// 		switch path {
-// 		case "/secure":
-// 			if status != 202 {
-// 				t.Error("Server should receive valid rewritten token")
-// 			}
-// 		}
-
-// 		time.Sleep(10 * time.Millisecond)
-// 	})
-// 	defer to.Close()
-
-// 	quit := make(chan int)
-
-// 	Settings.middleware = "echo -n && GOR_TEST=true && go run ./examples/middleware/token_modifier.go"
-
-// 	fromAddr := strings.Replace(from.Listener.Addr().String(), "[::]", "127.0.0.1", -1)
-// 	conf := RAWInputConfig{
-// 		engine:        capture.EnginePcap,
-// 		expire:        testRawExpire,
-// 		protocol:      ProtocolHTTP,
-// 		trackResponse: true,
-// 	}
-// 	// Catch traffic from one service
-// 	input := NewRAWInput(fromAddr, conf)
-
-// 	// And redirect to another
-// 	output := NewHTTPOutput(to.URL, &HTTPOutputConfig{Debug: true})
-
-// 	plugins := &InOutPlugins{
-// 		Inputs:  []io.Reader{input},
-// 		Outputs: []io.Writer{output},
-// 	}
-// 	plugins.All = append(plugins.All, input, output)
-
-// 	// Start Gor
-// 	emitter := NewEmitter(quit)
-// 	go emitter.Start(plugins, Settings.middleware)
-
-// 	// Should receive 2 requests from original + 2 from replayed
-// 	wg.Add(2)
-
-// 	client := NewHTTPClient(from.URL, &HTTPClientConfig{Debug: true})
-
-// 	// Sending traffic to original service
-// 	resp, _ = client.Get("/token")
-// 	token = proto.Body(resp)
-
-// 	resp, _ = client.Get("/secure?token=" + string(token))
-// 	if !bytes.Equal(proto.Status(resp), []byte("202")) {
-// 		t.Error("Valid token should return 202:", proto.Status(resp))
-// 	}
-
-// 	wg.Wait()
-// 	emitter.Close()
-// 	Settings.middleware = ""
-// }
+func TestMiddlewareWithPrettify(t *testing.T) {
+	Settings.PrettifyHTTP = true
+	quit := make(chan struct{})
+	in := NewTestInput()
+	cmd, cancl := initCmd(echoSh, noDebug)
+	midd := initMiddleware(cmd, cancl, in, func(err error) {})
+	var b1 = []byte("POST / HTTP/1.1\r\nHost: example.org\r\nTransfer-Encoding: chunked\r\n\r\n4\r\nWiki\r\n5\r\npedia\r\nE\r\n in\r\n\r\nchunks.\r\n0\r\n\r\n")
+	var b2 = []byte("POST / HTTP/1.1\r\nHost: example.org\r\nContent-Length: 25\r\n\r\nWikipedia in\r\n\r\nchunks.")
+	out := NewTestOutput(func(msg *Message) {
+		if !bytes.Equal(proto.Body(b2), proto.Body(msg.Data)) {
+			t.Errorf("expected %q body to equal %q body", b2, msg.Data)
+		}
+		quit <- struct{}{}
+	})
+	pl := &InOutPlugins{}
+	pl.Inputs = []PluginReader{midd, in}
+	pl.Outputs = []PluginWriter{out}
+	pl.All = []interface{}{midd, out, in}
+	e := NewEmitter()
+	go e.Start(pl, "")
+	in.EmitBytes(b1)
+	<-quit
+	midd.Close()
+	Settings.PrettifyHTTP = false
+}

output_binary.go

@@ -1,7 +1,6 @@
 package main
 
 import (
-	"io"
 	"sync/atomic"
 	"time"
 
@@ -25,30 +24,27 @@ type BinaryOutput struct {
 	// alignment. atomic.* functions crash on 32bit machines if operand is not
 	// aligned at 64bit. See https://github.com/golang/go/issues/599
 	activeWorkers int64
-
-	address string
-	queue   chan []byte
-
-	responses chan response
-
-	needWorker chan int
-
-	config *BinaryOutputConfig
-
-	queueStats *GorStat
+	address       string
+	queue         chan *Message
+	responses     chan response
+	needWorker    chan int
+	quit          chan struct{}
+	config        *BinaryOutputConfig
+	queueStats    *GorStat
 }
 
 // NewBinaryOutput constructor for BinaryOutput
 // Initialize workers
-func NewBinaryOutput(address string, config *BinaryOutputConfig) io.Writer {
+func NewBinaryOutput(address string, config *BinaryOutputConfig) PluginReadWriter {
 	o := new(BinaryOutput)
 
 	o.address = address
 	o.config = config
 
-	o.queue = make(chan []byte, 1000)
+	o.queue = make(chan *Message, 1000)
 	o.responses = make(chan response, 1000)
 	o.needWorker = make(chan int, 1)
+	o.quit = make(chan struct{})
 
 	// Initial workers count
 	if o.config.Workers == 0 {
@@ -89,8 +85,8 @@ func (o *BinaryOutput) startWorker() {
 
 	for {
 		select {
-		case data := <-o.queue:
-			o.sendRequest(client, data)
+		case msg := <-o.queue:
+			o.sendRequest(client, msg)
 			deathCount = 0
 		case <-time.After(time.Millisecond * 100):
 			// When dynamic scaling enabled workers die after 2s of inactivity
@@ -113,15 +109,13 @@ func (o *BinaryOutput) startWorker() {
 	}
 }
 
-func (o *BinaryOutput) Write(data []byte) (n int, err error) {
-	if !isRequestPayload(data) {
-		return len(data), nil
+// PluginWrite writes a message tothis plugin
+func (o *BinaryOutput) PluginWrite(msg *Message) (n int, err error) {
+	if !isRequestPayload(msg.Meta) {
+		return len(msg.Data), nil
 	}
 
-	buf := make([]byte, len(data))
-	copy(buf, data)
-
-	o.queue <- buf
+	o.queue <- msg
 
 	if o.config.Workers == 0 {
 		workersCount := atomic.LoadInt64(&o.activeWorkers)
@@ -131,37 +125,33 @@ func (o *BinaryOutput) Write(data []byte) (n int, err error) {
 		}
 	}
 
-	return len(data), nil
+	return len(msg.Data) + len(msg.Meta), nil
 }
 
-func (o *BinaryOutput) Read(data []byte) (int, error) {
-	resp := <-o.responses
+// PluginRead reads a message from this plugin
+func (o *BinaryOutput) PluginRead() (*Message, error) {
+	var resp response
+	var msg Message
+	select {
+	case <-o.quit:
+		return nil, ErrorStopped
+	case resp = <-o.responses:
+	}
+	msg.Data = resp.payload
+	msg.Meta = payloadHeader(ReplayedResponsePayload, resp.uuid, resp.startedAt, resp.roundTripTime)
 
-	Debug(2, "[OUTPUT-TCP] Received response:", string(resp.payload))
-
-	header := payloadHeader(ReplayedResponsePayload, resp.uuid, resp.startedAt, resp.roundTripTime)
-	copy(data[0:len(header)], header)
-	copy(data[len(header):], resp.payload)
-
-	return len(resp.payload) + len(header), nil
+	return &msg, nil
 }
 
-func (o *BinaryOutput) sendRequest(client *TCPClient, request []byte) {
-	meta := payloadMeta(request)
-	if len(meta) < 2 {
+func (o *BinaryOutput) sendRequest(client *TCPClient, msg *Message) {
+	if !isRequestPayload(msg.Meta) {
 		return
 	}
 
-	if !isRequestPayload(request) {
-		return
-	}
-
-	uuid := meta[1]
-
-	body := payloadBody(request)
+	uuid := payloadID(msg.Meta)
 
 	start := time.Now()
-	resp, err := client.Send(body)
+	resp, err := client.Send(msg.Data)
 	stop := time.Now()
 
 	if err != nil {
@@ -176,3 +166,9 @@ func (o *BinaryOutput) sendRequest(client *TCPClient, request []byte) {
 func (o *BinaryOutput) String() string {
 	return "Binary output: " + o.address
 }
+
+// Close closes this plugin for reading
+func (o *BinaryOutput) Close() error {
+	close(o.quit)
+	return nil
+}

output_dummy.go

@@ -15,9 +15,15 @@ func NewDummyOutput() (di *DummyOutput) {
 	return
 }
 
-func (i *DummyOutput) Write(data []byte) (int, error) {
-	n, err := os.Stdout.Write(data)
-	os.Stdout.Write([]byte{'\n'})
+// PluginWrite writes message to this plugin
+func (i *DummyOutput) PluginWrite(msg *Message) (int, error) {
+	var n, nn int
+	var err error
+	n, err = os.Stdout.Write(msg.Meta)
+	nn, err = os.Stdout.Write(msg.Data)
+	n += nn
+	nn, err = os.Stdout.Write(payloadSeparatorAsBytes)
+	n += nn
 	return n, err
 }
 

output_file.go

@@ -195,10 +195,11 @@ func (o *FileOutput) updateName() {
 	o.Unlock()
 }
 
-func (o *FileOutput) Write(data []byte) (n int, err error) {
+// PluginWrite writes message to this plugin
+func (o *FileOutput) PluginWrite(msg *Message) (n int, err error) {
 	if o.requestPerFile {
 		o.Lock()
-		meta := payloadMeta(data)
+		meta := payloadMeta(msg.Meta)
 		o.currentID = meta[1]
 		o.payloadType = meta[0]
 		o.Unlock()
@@ -227,10 +228,12 @@ func (o *FileOutput) Write(data []byte) (n int, err error) {
 		o.QueueLength = 0
 	}
 
-	n, _ = o.writer.Write(data)
-	nSeparator, _ := o.writer.Write([]byte(payloadSeparator))
-
-	n += nSeparator
+	var nn int
+	n, err = o.writer.Write(msg.Meta)
+	nn, err = o.writer.Write(msg.Data)
+	n += nn
+	nn, err = o.writer.Write(payloadSeparatorAsBytes)
+	n += nn
 
 	o.totalFileSize += size.Size(n)
 	o.QueueLength++
@@ -239,14 +242,14 @@ func (o *FileOutput) Write(data []byte) (n int, err error) {
 		return n, errors.New("File output reached size limit")
 	}
 
-	return n, nil
+	return n, err
 }
 
 func (o *FileOutput) flush() {
 	// Don't exit on panic
 	defer func() {
 		if r := recover(); r != nil {
-			log.Println("PANIC while file flush: ", r, o, string(debug.Stack()))
+			Debug(0, "[OUTPUT-FILE] PANIC while file flush: ", r, o, string(debug.Stack()))
 		}
 	}()
 
@@ -263,7 +266,7 @@ func (o *FileOutput) flush() {
 		if stat, err := o.file.Stat(); err == nil {
 			o.chunkSize = int(stat.Size())
 		} else {
-			log.Println("Error accessing file sats", err)
+			Debug(0, "[OUTPUT-HTTP] error accessing file size", err)
 		}
 	}
 }

output_file_test.go

@@ -2,7 +2,6 @@ package main
 
 import (
 	"fmt"
-	"io"
 	"math/rand"
 	"os"
 	"reflect"
@@ -17,18 +16,17 @@ import (
 
 func TestFileOutput(t *testing.T) {
 	wg := new(sync.WaitGroup)
-	quit := make(chan int)
 
 	input := NewTestInput()
 	output := NewFileOutput("/tmp/test_requests.gor", &FileOutputConfig{FlushInterval: time.Minute, Append: true})
 
 	plugins := &InOutPlugins{
-		Inputs:  []io.Reader{input},
-		Outputs: []io.Writer{output},
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{output},
 	}
 	plugins.All = append(plugins.All, input, output)
 
-	emitter := NewEmitter(quit)
+	emitter := NewEmitter()
 	go emitter.Start(plugins, Settings.Middleware)
 
 	for i := 0; i < 100; i++ {
@@ -42,19 +40,18 @@ func TestFileOutput(t *testing.T) {
 
 	var counter int64
 	input2 := NewFileInput("/tmp/test_requests.gor", false)
-	output2 := NewTestOutput(func(data []byte) {
+	output2 := NewTestOutput(func(*Message) {
 		atomic.AddInt64(&counter, 1)
 		wg.Done()
 	})
 
 	plugins2 := &InOutPlugins{
-		Inputs:  []io.Reader{input2},
-		Outputs: []io.Writer{output2},
+		Inputs:  []PluginReader{input2},
+		Outputs: []PluginWriter{output2},
 	}
 	plugins2.All = append(plugins2.All, input2, output2)
 
-	quit2 := make(chan int)
-	emitter2 := NewEmitter(quit2)
+	emitter2 := NewEmitter()
 	go emitter2.Start(plugins2, Settings.Middleware)
 
 	wg.Wait()
@@ -91,16 +88,16 @@ func TestFileOutputMultipleFiles(t *testing.T) {
 		t.Error("Should not initialize file if no writes")
 	}
 
-	output.Write([]byte("1 1 1\r\ntest"))
+	output.PluginWrite(&Message{Meta: []byte("1 1 1\r\n"), Data: []byte("test")})
 	name1 := output.file.Name()
 
-	output.Write([]byte("1 1 1\r\ntest"))
+	output.PluginWrite(&Message{Meta: []byte("1 1 1\r\n"), Data: []byte("test")})
 	name2 := output.file.Name()
 
 	time.Sleep(time.Second)
 	output.updateName()
 
-	output.Write([]byte("1 1 1\r\ntest"))
+	output.PluginWrite(&Message{Meta: []byte("1 1 1\r\n"), Data: []byte("test")})
 	name3 := output.file.Name()
 
 	if name2 != name1 {
@@ -122,16 +119,16 @@ func TestFileOutputFilePerRequest(t *testing.T) {
 		t.Error("Should not initialize file if no writes")
 	}
 
-	output.Write([]byte("1 1 1\ntest"))
+	output.PluginWrite(&Message{Meta: []byte("1 1 1\r\n"), Data: []byte("test")})
 	name1 := output.file.Name()
 
-	output.Write([]byte("1 2 1\ntest"))
+	output.PluginWrite(&Message{Meta: []byte("1 2 1\r\n"), Data: []byte("test")})
 	name2 := output.file.Name()
 
 	time.Sleep(time.Second)
 	output.updateName()
 
-	output.Write([]byte("1 3 1\ntest"))
+	output.PluginWrite(&Message{Meta: []byte("1 3 1\r\n"), Data: []byte("test")})
 	name3 := output.file.Name()
 
 	if name3 == name2 || name2 == name1 || name3 == name1 {
@@ -151,7 +148,7 @@ func TestFileOutputCompression(t *testing.T) {
 	}
 
 	for i := 0; i < 1000; i++ {
-		output.Write([]byte("1 1 1\r\ntest"))
+		output.PluginWrite(&Message{Meta: []byte("1 1 1\r\n"), Data: []byte("test")})
 	}
 
 	name := output.file.Name()
@@ -210,15 +207,15 @@ func TestFileOutputAppendQueueLimitOverflow(t *testing.T) {
 
 	output := NewFileOutput(name, &FileOutputConfig{Append: false, FlushInterval: time.Minute, QueueLimit: 2})
 
-	output.Write([]byte("1 1 1\r\ntest"))
+	output.PluginWrite(&Message{Meta: []byte("1 1 1\r\n"), Data: []byte("test")})
 	name1 := output.file.Name()
 
-	output.Write([]byte("1 1 1\r\ntest"))
+	output.PluginWrite(&Message{Meta: []byte("1 1 1\r\n"), Data: []byte("test")})
 	name2 := output.file.Name()
 
 	output.updateName()
 
-	output.Write([]byte("1 1 1\r\ntest"))
+	output.PluginWrite(&Message{Meta: []byte("1 1 1\r\n"), Data: []byte("test")})
 	name3 := output.file.Name()
 
 	if name2 != name1 || name1 != fmt.Sprintf("/tmp/%d_0", rnd) {
@@ -239,15 +236,15 @@ func TestFileOutputAppendQueueLimitNoOverflow(t *testing.T) {
 
 	output := NewFileOutput(name, &FileOutputConfig{Append: false, FlushInterval: time.Minute, QueueLimit: 3})
 
-	output.Write([]byte("1 1 1\r\ntest"))
+	output.PluginWrite(&Message{Meta: []byte("1 1 1\r\n"), Data: []byte("test")})
 	name1 := output.file.Name()
 
-	output.Write([]byte("1 1 1\r\ntest"))
+	output.PluginWrite(&Message{Meta: []byte("1 1 1\r\n"), Data: []byte("test")})
 	name2 := output.file.Name()
 
 	output.updateName()
 
-	output.Write([]byte("1 1 1\r\ntest"))
+	output.PluginWrite(&Message{Meta: []byte("1 1 1\r\n"), Data: []byte("test")})
 	name3 := output.file.Name()
 
 	if name2 != name1 || name1 != fmt.Sprintf("/tmp/%d_0", rnd) {
@@ -268,15 +265,15 @@ func TestFileOutputAppendQueueLimitGzips(t *testing.T) {
 
 	output := NewFileOutput(name, &FileOutputConfig{Append: false, FlushInterval: time.Minute, QueueLimit: 2})
 
-	output.Write([]byte("1 1 1\r\ntest"))
+	output.PluginWrite(&Message{Meta: []byte("1 1 1\r\n"), Data: []byte("test")})
 	name1 := output.file.Name()
 
-	output.Write([]byte("1 1 1\r\ntest"))
+	output.PluginWrite(&Message{Meta: []byte("1 1 1\r\n"), Data: []byte("test")})
 	name2 := output.file.Name()
 
 	output.updateName()
 
-	output.Write([]byte("1 1 1\r\ntest"))
+	output.PluginWrite(&Message{Meta: []byte("1 1 1\r\n"), Data: []byte("test")})
 	name3 := output.file.Name()
 
 	if name2 != name1 || name1 != fmt.Sprintf("/tmp/%d_0.gz", rnd) {
@@ -311,16 +308,16 @@ func TestFileOutputAppendSizeLimitOverflow(t *testing.T) {
 
 	output := NewFileOutput(name, &FileOutputConfig{Append: false, FlushInterval: time.Minute, SizeLimit: size.Size(2 * messageSize)})
 
-	output.Write([]byte("1 1 1\r\ntest"))
+	output.PluginWrite(&Message{Meta: []byte("1 1 1\r\n"), Data: []byte("test")})
 	name1 := output.file.Name()
 
-	output.Write([]byte("1 1 1\r\ntest"))
+	output.PluginWrite(&Message{Meta: []byte("1 1 1\r\n"), Data: []byte("test")})
 	name2 := output.file.Name()
 
 	output.flush()
 	output.updateName()
 
-	output.Write([]byte("1 1 1\r\ntest"))
+	output.PluginWrite(&Message{Meta: []byte("1 1 1\r\n"), Data: []byte("test")})
 	name3 := output.file.Name()
 
 	if name2 != name1 || name1 != fmt.Sprintf("/tmp/%d_0", rnd) {

output_http.go

@@ -1,56 +1,26 @@
 package main
 
 import (
+	"bufio"
+	"bytes"
+	"crypto/tls"
 	"fmt"
-	"io"
 	"log"
+	"math"
+	"net/http"
+	"net/http/httputil"
+	"net/url"
 	"sync/atomic"
 	"time"
 
-	"github.com/buger/goreplay/proto"
 	"github.com/buger/goreplay/size"
 )
 
-const initialDynamicWorkers = 10
-
-type httpWorker struct {
-	output       *HTTPOutput
-	client       *HTTPClient
-	lastActivity time.Time
-	queue        chan []byte
-	stop         chan bool
-}
-
-func newHTTPWorker(output *HTTPOutput, queue chan []byte) *httpWorker {
-	client := NewHTTPClient(output.address, &HTTPClientConfig{
-		FollowRedirects:    output.config.RedirectLimit,
-		Debug:              output.config.Debug,
-		OriginalHost:       output.config.OriginalHost,
-		Timeout:            output.config.Timeout,
-		ResponseBufferSize: int(output.config.BufferSize),
-	})
-
-	w := &httpWorker{client: client}
-	if queue == nil {
-		w.queue = make(chan []byte, 100)
-	} else {
-		w.queue = queue
-	}
-	w.stop = make(chan bool)
-
-	go func() {
-		for {
-			select {
-			case payload := <-w.queue:
-				output.sendRequest(client, payload)
-			case <-w.stop:
-				return
-			}
-		}
-	}()
-
-	return w
-}
+const (
+	initialDynamicWorkers = 10
+	readChunkSize         = 64 * 1024
+	maxResponseSize       = 1073741824
+)
 
 type response struct {
 	payload       []byte
@@ -61,272 +31,296 @@ type response struct {
 
 // HTTPOutputConfig struct for holding http output configuration
 type HTTPOutputConfig struct {
-	RedirectLimit int `json:"output-http-redirect-limit"`
-
-	Stats      bool `json:"output-http-stats"`
-	WorkersMin int  `json:"output-http-workers-min"`
-	WorkersMax int  `json:"output-http-workers"`
-	StatsMs    int  `json:"output-http-stats-ms"`
-	Workers    int
-	QueueLen   int `json:"output-http-queue-len"`
-
-	ElasticSearch string `json:"output-http-elasticsearch"`
-
-	Timeout      time.Duration `json:"output-http-timeout"`
-	OriginalHost bool          `json:"output-http-original-host"`
-	BufferSize   size.Size     `json:"output-http-response-buffer"`
-
-	CompatibilityMode bool `json:"output-http-compatibility-mode"`
-
-	RequestGroup string
-
-	Debug bool `json:"output-http-debug"`
-
-	TrackResponses bool `json:"output-http-track-response"`
+	TrackResponses bool          `json:"output-http-track-response"`
+	Stats          bool          `json:"output-http-stats"`
+	OriginalHost   bool          `json:"output-http-original-host"`
+	RedirectLimit  int           `json:"output-http-redirect-limit"`
+	WorkersMin     int           `json:"output-http-workers-min"`
+	WorkersMax     int           `json:"output-http-workers"`
+	StatsMs        int           `json:"output-http-stats-ms"`
+	QueueLen       int           `json:"output-http-queue-len"`
+	ElasticSearch  string        `json:"output-http-elasticsearch"`
+	Timeout        time.Duration `json:"output-http-timeout"`
+	WorkerTimeout  time.Duration `json:"output-http-worker-timeout"`
+	BufferSize     size.Size     `json:"output-http-response-buffer"`
+	SkipVerify     bool          `json:"output-http-skip-verify"`
+	rawURL         string
+	url            *url.URL
 }
 
 // HTTPOutput plugin manage pool of workers which send request to replayed server
-// By default workers pool is dynamic and starts with 10 workers
-// You can specify fixed number of workers using `--output-http-workers`
+// By default workers pool is dynamic and starts with 1 worker or workerMin workers
+// You can specify maximum number of workers using `--output-http-workers`
 type HTTPOutput struct {
-	// Keep this as first element of struct because it guarantees 64bit
-	// alignment. atomic.* functions crash on 32bit machines if operand is not
-	// aligned at 64bit. See https://github.com/golang/go/issues/599
-	activeWorkers int64
-
-	workerSessions map[string]*httpWorker
-
-	address string
-	limit   int
-	queue   chan []byte
-
-	responses chan response
-
-	needWorker chan int
-
-	config *HTTPOutputConfig
-
-	queueStats *GorStat
-
+	activeWorkers int32
+	config        *HTTPOutputConfig
+	queueStats    *GorStat
 	elasticSearch *ESPlugin
-
-	stop chan bool // Channel used only to indicate goroutine should shutdown
+	client        *HTTPClient
+	stopWorker    chan struct{}
+	queue         chan *Message
+	responses     chan *response
+	stop          chan bool // Channel used only to indicate goroutine should shutdown
 }
 
 // NewHTTPOutput constructor for HTTPOutput
 // Initialize workers
-func NewHTTPOutput(address string, config *HTTPOutputConfig) io.Writer {
+func NewHTTPOutput(address string, config *HTTPOutputConfig) PluginReadWriter {
 	o := new(HTTPOutput)
-
-	o.address = address
+	var err error
+	config.url, err = url.Parse(address)
+	if err != nil {
+		log.Fatal(fmt.Sprintf("[OUTPUT-HTTP] parse HTTP output URL error[%q]", err))
+	}
+	if config.url.Scheme == "" {
+		config.url.Scheme = "http"
+	}
+	config.rawURL = config.url.String()
+	if config.Timeout < time.Millisecond*100 {
+		config.Timeout = time.Second
+	}
+	if config.BufferSize <= 0 {
+		config.BufferSize = 100 * 1024 // 100kb
+	}
+	if config.WorkersMin <= 0 {
+		config.WorkersMin = 1
+	}
+	if config.WorkersMin > 1000 {
+		config.WorkersMin = 1000
+	}
+	if config.WorkersMax <= 0 {
+		config.WorkersMax = math.MaxInt32 // idealy so large
+	}
+	if config.WorkersMax < config.WorkersMin {
+		config.WorkersMax = config.WorkersMin
+	}
+	if config.QueueLen <= 0 {
+		config.QueueLen = 1000
+	}
+	if config.RedirectLimit < 0 {
+		config.RedirectLimit = 0
+	}
+	if config.WorkerTimeout <= 0 {
+		config.WorkerTimeout = time.Second * 2
+	}
 	o.config = config
 	o.stop = make(chan bool)
-
 	if o.config.Stats {
 		o.queueStats = NewGorStat("output_http", o.config.StatsMs)
 	}
 
-	o.queue = make(chan []byte, o.config.QueueLen)
-	o.responses = make(chan response, o.config.QueueLen)
-	o.needWorker = make(chan int, 1)
-
-	// Initial workers count
-	if o.config.WorkersMax == 0 {
-		o.needWorker <- initialDynamicWorkers
-	} else {
-		o.needWorker <- o.config.WorkersMax
+	o.queue = make(chan *Message, o.config.QueueLen)
+	if o.config.TrackResponses {
+		o.responses = make(chan *response, o.config.QueueLen)
 	}
+	// it should not be buffered to avoid races
+	o.stopWorker = make(chan struct{})
 
 	if o.config.ElasticSearch != "" {
 		o.elasticSearch = new(ESPlugin)
 		o.elasticSearch.Init(o.config.ElasticSearch)
 	}
-
-	if Settings.RecognizeTCPSessions {
-		if !PRO {
-			log.Fatal("Detailed TCP sessions work only with PRO license")
-		}
-		o.workerSessions = make(map[string]*httpWorker, 100)
-		go o.sessionWorkerMaster()
-	} else {
-		go o.workerMaster()
+	o.client = NewHTTPClient(o.config)
+	o.activeWorkers += int32(o.config.WorkersMin)
+	for i := 0; i < o.config.WorkersMin; i++ {
+		go o.startWorker()
 	}
-
+	go o.workerMaster()
 	return o
 }
 
 func (o *HTTPOutput) workerMaster() {
-	for {
-		newWorkers := <-o.needWorker
-		atomic.AddInt64(&o.activeWorkers, int64(newWorkers))
-		for i := 0; i < newWorkers; i++ {
-			go o.startWorker()
-		}
-	}
-}
-
-func (o *HTTPOutput) sessionWorkerMaster() {
-	gc := time.Tick(time.Second)
-
-	for {
-		select {
-		case p := <-o.queue:
-			id := payloadID(p)
-			sessionID := string(id[0:20])
-			worker, ok := o.workerSessions[sessionID]
-
-			if !ok {
-				atomic.AddInt64(&o.activeWorkers, 1)
-				worker = newHTTPWorker(o, nil)
-				o.workerSessions[sessionID] = worker
-			}
-
-			worker.queue <- p
-			worker.lastActivity = time.Now()
-		case <-gc:
-			now := time.Now()
-
-			for id, w := range o.workerSessions {
-				if !w.lastActivity.IsZero() && now.Sub(w.lastActivity) >= 120*time.Second {
-					w.stop <- true
-					delete(o.workerSessions, id)
-					atomic.AddInt64(&o.activeWorkers, -1)
-				}
-			}
-		}
-	}
-}
-
-func (o *HTTPOutput) startWorker() {
-	client := NewHTTPClient(o.address, &HTTPClientConfig{
-		FollowRedirects:    o.config.RedirectLimit,
-		Debug:              o.config.Debug,
-		OriginalHost:       o.config.OriginalHost,
-		Timeout:            o.config.Timeout,
-		ResponseBufferSize: int(o.config.BufferSize),
-		CompatibilityMode:  o.config.CompatibilityMode,
-	})
-
+	var timer = time.NewTimer(o.config.WorkerTimeout)
+	defer func() {
+		// recover from panics caused by trying to send in
+		// a closed chan(o.stopWorker)
+		recover()
+	}()
+	defer timer.Stop()
 	for {
 		select {
 		case <-o.stop:
 			return
-		case data := <-o.queue:
-			o.sendRequest(client, data)
-		case <-time.After(2 * time.Second):
-			// When dynamic scaling enabled workers die after 2s of inactivity
-			if o.config.WorkersMin == o.config.WorkersMax {
-				continue
-			}
+		default:
+			<-timer.C
+		}
+		// rollback workers
+	rollback:
+		if atomic.LoadInt32(&o.activeWorkers) > int32(o.config.WorkersMin) && len(o.queue) < 1 {
+			// close one worker
+			o.stopWorker <- struct{}{}
+			atomic.AddInt32(&o.activeWorkers, -1)
+			goto rollback
+		}
+		timer.Reset(o.config.WorkerTimeout)
+	}
+}
 
-			workersCount := int(atomic.LoadInt64(&o.activeWorkers))
-
-			// At least 1 startWorker should be alive
-			if workersCount != 1 && workersCount > o.config.WorkersMin {
-				atomic.AddInt64(&o.activeWorkers, -1)
-				return
-			}
+func (o *HTTPOutput) startWorker() {
+	for {
+		select {
+		case <-o.stopWorker:
+			return
+		case msg := <-o.queue:
+			o.sendRequest(o.client, msg)
 		}
 	}
 }
 
-func (o *HTTPOutput) Write(data []byte) (n int, err error) {
-	if !isRequestPayload(data) {
-		return len(data), nil
+// PluginWrite writes message to this plugin
+func (o *HTTPOutput) PluginWrite(msg *Message) (n int, err error) {
+	if !isRequestPayload(msg.Meta) {
+		return len(msg.Data), nil
 	}
 
-	buf := make([]byte, len(data))
-	copy(buf, data)
-
 	select {
 	case <-o.stop:
 		return 0, ErrorStopped
-	case o.queue <- buf:
+	case o.queue <- msg:
 	}
 
 	if o.config.Stats {
 		o.queueStats.Write(len(o.queue))
 	}
-
-	if !Settings.RecognizeTCPSessions && o.config.WorkersMax != o.config.WorkersMin {
-		workersCount := int(atomic.LoadInt64(&o.activeWorkers))
-
-		if len(o.queue) > workersCount {
-			extraWorkersReq := len(o.queue) - workersCount + 1
-			maxWorkersAvailable := o.config.WorkersMax - workersCount
-			if extraWorkersReq > maxWorkersAvailable {
-				extraWorkersReq = maxWorkersAvailable
-			}
-			if extraWorkersReq > 0 {
-				o.needWorker <- extraWorkersReq
-			}
+	if len(o.queue) > 0 {
+		// try to start a new worker to serve
+		if atomic.LoadInt32(&o.activeWorkers) < int32(o.config.WorkersMax) {
+			go o.startWorker()
+			atomic.AddInt32(&o.activeWorkers, 1)
 		}
 	}
-
-	return len(data), nil
+	return len(msg.Data) + len(msg.Meta), nil
 }
 
-func (o *HTTPOutput) Read(data []byte) (int, error) {
-	var resp response
+// PluginRead reads message from this plugin
+func (o *HTTPOutput) PluginRead() (*Message, error) {
+	if !o.config.TrackResponses {
+		return nil, ErrorStopped
+	}
+	var resp *response
+	var msg Message
 	select {
 	case <-o.stop:
-		return 0, ErrorStopped
+		return nil, ErrorStopped
 	case resp = <-o.responses:
+		msg.Data = resp.payload
 	}
 
-	Debug(3, "[OUTPUT-HTTP] Received response:", string(resp.payload))
+	msg.Meta = payloadHeader(ReplayedResponsePayload, resp.uuid, resp.roundTripTime, resp.startedAt)
 
-	header := payloadHeader(ReplayedResponsePayload, resp.uuid, resp.roundTripTime, resp.startedAt)
-	n := copy(data, header)
-	if len(data) > len(header) {
-		n += copy(data[len(header):], resp.payload)
-	}
-	dis := len(header) + len(data) - n
-	if dis > 0 {
-		Debug(2, "[OUTPUT-HTTP] discarded", dis, "increase copy buffer size")
-	}
-
-	return n, nil
+	return &msg, nil
 }
 
-func (o *HTTPOutput) sendRequest(client *HTTPClient, request []byte) {
-	meta := payloadMeta(request)
-
-	Debug(2, fmt.Sprintf("[OUTPUT-HTTP] meta: %q", meta))
-
-	if len(meta) < 2 {
+func (o *HTTPOutput) sendRequest(client *HTTPClient, msg *Message) {
+	if !isRequestPayload(msg.Meta) {
 		return
 	}
-	uuid := meta[1]
-
-	body := payloadBody(request)
-	if !proto.HasRequestTitle(body) {
-		return
-	}
-
+	uuid := payloadID(msg.Meta)
 	start := time.Now()
-	resp, err := client.Send(body)
+	resp, err := client.Send(msg.Data)
 	stop := time.Now()
 
 	if err != nil {
-		Debug(1, "Error when sending ", err)
+		Debug(1, fmt.Sprintf("[HTTP-OUTPUT] error when sending: %q", err))
+		return
+	}
+	if resp == nil {
+		return
 	}
 
 	if o.config.TrackResponses {
-		o.responses <- response{resp, uuid, start.UnixNano(), stop.UnixNano() - start.UnixNano()}
+		o.responses <- &response{resp, uuid, start.UnixNano(), stop.UnixNano() - start.UnixNano()}
 	}
 
 	if o.elasticSearch != nil {
-		o.elasticSearch.ResponseAnalyze(request, resp, start, stop)
+		o.elasticSearch.ResponseAnalyze(msg.Data, resp, start, stop)
 	}
 }
 
 func (o *HTTPOutput) String() string {
-	return "HTTP output: " + o.address
+	return "HTTP output: " + o.config.rawURL
 }
 
 // Close closes the data channel so that data
 func (o *HTTPOutput) Close() error {
 	close(o.stop)
+	close(o.stopWorker)
 	return nil
 }
+
+// HTTPClient holds configurations for a single HTTP client
+type HTTPClient struct {
+	config *HTTPOutputConfig
+	Client *http.Client
+}
+
+// NewHTTPClient returns new http client with check redirects policy
+func NewHTTPClient(config *HTTPOutputConfig) *HTTPClient {
+	client := new(HTTPClient)
+	client.config = config
+	var transport *http.Transport
+	client.Client = &http.Client{
+		Timeout: client.config.Timeout,
+		CheckRedirect: func(req *http.Request, via []*http.Request) error {
+			if len(via) >= client.config.RedirectLimit {
+				Debug(1, fmt.Sprintf("[HTTPCLIENT] maximum output-http-redirects[%d] reached!", client.config.RedirectLimit))
+				return http.ErrUseLastResponse
+			}
+			lastReq := via[len(via)-1]
+			resp := req.Response
+			Debug(2, fmt.Sprintf("[HTTPCLIENT] HTTP redirects from %q to %q with %q", lastReq.Host, req.Host, resp.Status))
+			return nil
+		},
+	}
+	if config.SkipVerify {
+		// clone to avoid modying global default RoundTripper
+		transport = http.DefaultTransport.(*http.Transport).Clone()
+		transport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true}
+		client.Client.Transport = transport
+	}
+
+	return client
+}
+
+// Send sends an http request using client create by NewHTTPClient
+func (c *HTTPClient) Send(data []byte) ([]byte, error) {
+	var req *http.Request
+	var resp *http.Response
+	var err error
+
+	req, err = http.ReadRequest(bufio.NewReader(bytes.NewReader(data)))
+	if err != nil {
+		return nil, err
+	}
+	// we don't send CONNECT or OPTIONS request
+	if req.Method == http.MethodConnect {
+		return nil, nil
+	}
+
+	if !c.config.OriginalHost {
+		req.Host = c.config.url.Host
+	}
+
+	// fix #862
+	if c.config.url.Path == "" && c.config.url.RawQuery == "" {
+		req.URL.Scheme = c.config.url.Scheme
+		req.URL.Host = c.config.url.Host
+	} else {
+		req.URL = c.config.url
+	}
+
+	// force connection to not be closed, which can affect the global client
+	req.Close = false
+	// it's an error if this is not equal to empty string
+	req.RequestURI = ""
+
+	resp, err = c.Client.Do(req)
+	if err != nil {
+		return nil, err
+	}
+	if c.config.TrackResponses {
+		return httputil.DumpResponse(resp, true)
+	}
+	_ = resp.Body.Close()
+	return nil, nil
+}

output_http_test.go

@@ -1,19 +1,16 @@
 package main
 
 import (
-	"io"
 	"io/ioutil"
 	"net/http"
 	"net/http/httptest"
 	_ "net/http/httputil"
 	"sync"
 	"testing"
-	"time"
 )
 
 func TestHTTPOutput(t *testing.T) {
 	wg := new(sync.WaitGroup)
-	quit := make(chan int)
 
 	input := NewTestInput()
 
@@ -39,27 +36,27 @@ func TestHTTPOutput(t *testing.T) {
 	}))
 	defer server.Close()
 
-	headers := HTTPHeaders{HTTPHeader{"User-Agent", "Gor"}}
+	headers := HTTPHeaders{httpHeader{"User-Agent", "Gor"}}
 	methods := HTTPMethods{[]byte("GET"), []byte("PUT"), []byte("POST")}
 	Settings.ModifierConfig = HTTPModifierConfig{Headers: headers, Methods: methods}
 
-	http_output := NewHTTPOutput(server.URL, &HTTPOutputConfig{Debug: true, TrackResponses: true})
-	output := NewTestOutput(func(data []byte) {
+	httpOutput := NewHTTPOutput(server.URL, &HTTPOutputConfig{TrackResponses: false})
+	output := NewTestOutput(func(*Message) {
 		wg.Done()
 	})
 
 	plugins := &InOutPlugins{
-		Inputs:  []io.Reader{input},
-		Outputs: []io.Writer{http_output, output},
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{httpOutput, output},
 	}
-	plugins.All = append(plugins.All, input, output, http_output)
+	plugins.All = append(plugins.All, input, output, httpOutput)
 
-	emitter := NewEmitter(quit)
+	emitter := NewEmitter()
 	go emitter.Start(plugins, Settings.Middleware)
 
 	for i := 0; i < 10; i++ {
-		// 2 http-output, 2 - test output request, 2 - test output http response
-		wg.Add(6) // OPTIONS should be ignored
+		// 2 http-output, 2 - test output request
+		wg.Add(4) // OPTIONS should be ignored
 		input.EmitPOST()
 		input.EmitOPTIONS()
 		input.EmitGET()
@@ -73,7 +70,6 @@ func TestHTTPOutput(t *testing.T) {
 
 func TestHTTPOutputKeepOriginalHost(t *testing.T) {
 	wg := new(sync.WaitGroup)
-	quit := make(chan int)
 
 	input := NewTestInput()
 
@@ -86,18 +82,18 @@ func TestHTTPOutputKeepOriginalHost(t *testing.T) {
 	}))
 	defer server.Close()
 
-	headers := HTTPHeaders{HTTPHeader{"Host", "custom-host.com"}}
+	headers := HTTPHeaders{httpHeader{"Host", "custom-host.com"}}
 	Settings.ModifierConfig = HTTPModifierConfig{Headers: headers}
 
-	output := NewHTTPOutput(server.URL, &HTTPOutputConfig{Debug: false, OriginalHost: true})
+	output := NewHTTPOutput(server.URL, &HTTPOutputConfig{OriginalHost: true, SkipVerify: true})
 
 	plugins := &InOutPlugins{
-		Inputs:  []io.Reader{input},
-		Outputs: []io.Writer{output},
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{output},
 	}
 	plugins.All = append(plugins.All, input, output)
 
-	emitter := NewEmitter(quit)
+	emitter := NewEmitter()
 	go emitter.Start(plugins, Settings.Middleware)
 
 	wg.Add(1)
@@ -110,7 +106,6 @@ func TestHTTPOutputKeepOriginalHost(t *testing.T) {
 
 func TestHTTPOutputSSL(t *testing.T) {
 	wg := new(sync.WaitGroup)
-	quit := make(chan int)
 
 	// Origing and Replay server initialization
 	server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
@@ -118,15 +113,15 @@ func TestHTTPOutputSSL(t *testing.T) {
 	}))
 
 	input := NewTestInput()
-	output := NewHTTPOutput(server.URL, &HTTPOutputConfig{})
+	output := NewHTTPOutput(server.URL, &HTTPOutputConfig{SkipVerify: true})
 
 	plugins := &InOutPlugins{
-		Inputs:  []io.Reader{input},
-		Outputs: []io.Writer{output},
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{output},
 	}
 	plugins.All = append(plugins.All, input, output)
 
-	emitter := NewEmitter(quit)
+	emitter := NewEmitter()
 	go emitter.Start(plugins, Settings.Middleware)
 
 	wg.Add(2)
@@ -140,7 +135,6 @@ func TestHTTPOutputSSL(t *testing.T) {
 
 func TestHTTPOutputSessions(t *testing.T) {
 	wg := new(sync.WaitGroup)
-	quit := make(chan int)
 
 	input := NewTestInput()
 	input.skipHeader = true
@@ -151,25 +145,27 @@ func TestHTTPOutputSessions(t *testing.T) {
 	defer server.Close()
 
 	Settings.RecognizeTCPSessions = true
-	output := NewHTTPOutput(server.URL, &HTTPOutputConfig{Debug: true})
+	Settings.SplitOutput = true
+	output := NewHTTPOutput(server.URL, &HTTPOutputConfig{})
 
 	plugins := &InOutPlugins{
-		Inputs:  []io.Reader{input},
-		Outputs: []io.Writer{output},
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{output},
 	}
-	emitter := NewEmitter(quit)
+	plugins.All = append(plugins.All, input, output)
+	emitter := NewEmitter()
 	go emitter.Start(plugins, Settings.Middleware)
 
 	uuid1 := []byte("1234567890123456789a0000")
 	uuid2 := []byte("1234567890123456789d0000")
 
-	for i := 0; i < 100; i++ {
+	for i := 0; i < 10; i++ {
 		wg.Add(1) // OPTIONS should be ignored
 		copy(uuid1[20:], randByte(4))
 		input.EmitBytes([]byte("1 " + string(uuid1) + " 1\n" + "GET / HTTP/1.1\r\n\r\n"))
 	}
 
-	for i := 0; i < 100; i++ {
+	for i := 0; i < 10; i++ {
 		wg.Add(1) // OPTIONS should be ignored
 		copy(uuid2[20:], randByte(4))
 		input.EmitBytes([]byte("1 " + string(uuid2) + " 1\n" + "GET / HTTP/1.1\r\n\r\n"))
@@ -180,28 +176,56 @@ func TestHTTPOutputSessions(t *testing.T) {
 	emitter.Close()
 
 	Settings.RecognizeTCPSessions = false
+	Settings.SplitOutput = false
 }
 
 func BenchmarkHTTPOutput(b *testing.B) {
 	wg := new(sync.WaitGroup)
-	quit := make(chan int)
 
-	server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		time.Sleep(50 * time.Millisecond)
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		wg.Done()
 	}))
 	defer server.Close()
 
 	input := NewTestInput()
-	output := NewHTTPOutput(server.URL, &HTTPOutputConfig{})
+	output := NewHTTPOutput(server.URL, &HTTPOutputConfig{WorkersMax: 1})
 
 	plugins := &InOutPlugins{
-		Inputs:  []io.Reader{input},
-		Outputs: []io.Writer{output},
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{output},
 	}
 	plugins.All = append(plugins.All, input, output)
 
-	emitter := NewEmitter(quit)
+	emitter := NewEmitter()
+	go emitter.Start(plugins, Settings.Middleware)
+
+	for i := 0; i < b.N; i++ {
+		wg.Add(1)
+		input.EmitPOST()
+	}
+
+	wg.Wait()
+	emitter.Close()
+}
+
+func BenchmarkHTTPOutputTLS(b *testing.B) {
+	wg := new(sync.WaitGroup)
+
+	server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		wg.Done()
+	}))
+	defer server.Close()
+
+	input := NewTestInput()
+	output := NewHTTPOutput(server.URL, &HTTPOutputConfig{SkipVerify: true, WorkersMax: 1})
+
+	plugins := &InOutPlugins{
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{output},
+	}
+	plugins.All = append(plugins.All, input, output)
+
+	emitter := NewEmitter()
 	go emitter.Start(plugins, Settings.Middleware)
 
 	for i := 0; i < b.N; i++ {

output_kafka.go

@@ -2,11 +2,11 @@ package main
 
 import (
 	"encoding/json"
-	"io"
 	"log"
 	"strings"
 	"time"
 
+	"github.com/buger/goreplay/byteutils"
 	"github.com/buger/goreplay/proto"
 
 	"github.com/Shopify/sarama"
@@ -22,13 +22,8 @@ type KafkaOutput struct {
 // KafkaOutputFrequency in milliseconds
 const KafkaOutputFrequency = 500
 
-// NewKafkaOutput creates instance of kafka producer client.
-func NewKafkaOutput(address string, config *OutputKafkaConfig) io.Writer {
-	return NewKafkaOutputWithTLS(address, config, nil)
-}
-
-// NewKafkaOutputWithTLS creates instance of kafka producer client.
-func NewKafkaOutputWithTLS(address string, config *OutputKafkaConfig, tlsConfig *KafkaTLSConfig) io.Writer {
+// NewKafkaOutput creates instance of kafka producer client  with TLS config
+func NewKafkaOutput(address string, config *OutputKafkaConfig, tlsConfig *KafkaTLSConfig) PluginWriter {
 	c := NewKafkaConfig(tlsConfig)
 
 	var producer sarama.AsyncProducer
@@ -67,31 +62,33 @@ func (o *KafkaOutput) ErrorHandler() {
 	}
 }
 
-func (o *KafkaOutput) Write(data []byte) (n int, err error) {
+// PluginWrite writes a message to this plugin
+func (o *KafkaOutput) PluginWrite(msg *Message) (n int, err error) {
 	var message sarama.StringEncoder
 
 	if !o.config.UseJSON {
-		message = sarama.StringEncoder(data)
+		message = sarama.StringEncoder(byteutils.SliceToString(msg.Meta) + byteutils.SliceToString(msg.Data))
 	} else {
-		headers := make(map[string]string)
-		proto.ParseHeaders([][]byte{data}, func(header []byte, value []byte) {
-			headers[string(header)] = string(value)
-		})
+		mimeHeader := proto.ParseHeaders(msg.Data)
+		header := make(map[string]string)
+		for k, v := range mimeHeader {
+			header[k] = strings.Join(v, ", ")
+		}
 
-		meta := payloadMeta(data)
-		req := payloadBody(data)
+		meta := payloadMeta(msg.Meta)
+		req := msg.Data
 
 		kafkaMessage := KafkaMessage{
-			ReqURL:     string(proto.Path(req)),
-			ReqType:    string(meta[0]),
-			ReqID:      string(meta[1]),
-			ReqTs:      string(meta[2]),
-			ReqMethod:  string(proto.Method(req)),
-			ReqBody:    string(proto.Body(req)),
-			ReqHeaders: headers,
+			ReqURL:     byteutils.SliceToString(proto.Path(req)),
+			ReqType:    byteutils.SliceToString(meta[0]),
+			ReqID:      byteutils.SliceToString(meta[1]),
+			ReqTs:      byteutils.SliceToString(meta[2]),
+			ReqMethod:  byteutils.SliceToString(proto.Method(req)),
+			ReqBody:    byteutils.SliceToString(proto.Body(req)),
+			ReqHeaders: header,
 		}
 		jsonMessage, _ := json.Marshal(&kafkaMessage)
-		message = sarama.StringEncoder(jsonMessage)
+		message = sarama.StringEncoder(byteutils.SliceToString(jsonMessage))
 	}
 
 	o.producer.Input() <- &sarama.ProducerMessage{

output_kafka_test.go

@@ -17,16 +17,16 @@ func TestOutputKafkaRAW(t *testing.T) {
 		producer: producer,
 		Topic:    "test",
 		UseJSON:  false,
-	})
+	}, nil)
 
-	output.Write([]byte("1 2 3\nGET / HTTP1.1\r\nHeader: 1\r\n\r\n"))
+	output.PluginWrite(&Message{Meta: []byte("1 2 3\n"), Data: []byte("GET / HTTP1.1\r\nHeader: 1\r\n\r\n")})
 
 	resp := <-producer.Successes()
 
 	data, _ := resp.Value.Encode()
 
 	if string(data) != "1 2 3\nGET / HTTP1.1\r\nHeader: 1\r\n\r\n" {
-		t.Error("Message not properly encoded: ", string(data))
+		t.Errorf("Message not properly encoded: %q", data)
 	}
 }
 
@@ -40,9 +40,9 @@ func TestOutputKafkaJSON(t *testing.T) {
 		producer: producer,
 		Topic:    "test",
 		UseJSON:  true,
-	})
+	}, nil)
 
-	output.Write([]byte("1 2 3\nGET / HTTP1.1\r\nHeader: 1\r\n\r\n"))
+	output.PluginWrite(&Message{Meta: []byte("1 2 3\n"), Data: []byte("GET / HTTP1.1\r\nHeader: 1\r\n\r\n")})
 
 	resp := <-producer.Successes()
 

output_null.go

@@ -9,8 +9,9 @@ func NewNullOutput() (o *NullOutput) {
 	return new(NullOutput)
 }
 
-func (o *NullOutput) Write(data []byte) (int, error) {
-	return len(data), nil
+// PluginWrite writes message to this plugin
+func (o *NullOutput) PluginWrite(msg *Message) (int, error) {
+	return len(msg.Data) + len(msg.Meta), nil
 }
 
 func (o *NullOutput) String() string {

output_s3.go

@@ -63,12 +63,13 @@ func NewS3Output(pathTemplate string, config *FileOutputConfig) *S3Output {
 func (o *S3Output) connect() {
 	if o.session == nil {
 		o.session = session.Must(session.NewSession(awsConfig()))
-		log.Println("[S3 Output] S3 connection succesfully initialized")
+		log.Println("[S3 Output] S3 connection successfully initialized")
 	}
 }
 
-func (o *S3Output) Write(data []byte) (n int, err error) {
-	return o.buffer.Write(data)
+// PluginWrite writes message to this plugin
+func (o *S3Output) PluginWrite(msg *Message) (n int, err error) {
+	return o.buffer.PluginWrite(msg)
 }
 
 func (o *S3Output) String() string {
@@ -107,22 +108,23 @@ func (o *S3Output) onBufferUpdate(path string) {
 	idx := getFileIndex(path)
 	bucket, key := o.keyPath(idx)
 
-	file, _ := os.Open(path)
-	// reader := bufio.NewReader(file)
+	file, err := os.Open(path)
+	if err != nil {
+		Debug(0, fmt.Sprintf("[S3 Output] Failed to open file %q. err: %q", path, err))
+		return
+	}
+	defer os.Remove(path)
 
-	_, err := svc.PutObject(&s3.PutObjectInput{
+	_, err = svc.PutObject(&s3.PutObjectInput{
 		Body:   file,
 		Bucket: aws.String(bucket),
 		Key:    aws.String(key),
 	})
 	if err != nil {
-		log.Printf("[S3 Output] Failed to upload data to %s/%s, %s\n", bucket, key, err)
-		os.Remove(path)
+		Debug(0, fmt.Sprintf("[S3 Output] Failed to upload data to %q/%q, %q", bucket, key, err))
 		return
 	}
 
-	os.Remove(path)
-
 	if o.closeCh != nil {
 		o.closeCh <- struct{}{}
 	}

output_tcp.go

@@ -1,24 +1,26 @@
 package main
 
 import (
+	"context"
 	"crypto/tls"
 	"fmt"
 	"hash/fnv"
-	"io"
 	"net"
 	"time"
 )
 
 // TCPOutput used for sending raw tcp payloads
 // Currently used for internal communication between listener and replay server
-// Can be used for transfering binary payloads like protocol buffers
+// Can be used for transferring binary payloads like protocol buffers
 type TCPOutput struct {
 	address     string
 	limit       int
-	buf         []chan []byte
+	buf         []chan *Message
 	bufStats    *GorStat
 	config      *TCPOutputConfig
 	workerIndex uint32
+
+	close bool
 }
 
 // TCPOutputConfig tcp output configuration
@@ -31,7 +33,7 @@ type TCPOutputConfig struct {
 
 // NewTCPOutput constructor for TCPOutput
 // Initialize X workers which hold keep-alive connection
-func NewTCPOutput(address string, config *TCPOutputConfig) io.Writer {
+func NewTCPOutput(address string, config *TCPOutputConfig) PluginWriter {
 	o := new(TCPOutput)
 
 	o.address = address
@@ -42,9 +44,9 @@ func NewTCPOutput(address string, config *TCPOutputConfig) io.Writer {
 	}
 
 	// create X buffers and send the buffer index to the worker
-	o.buf = make([]chan []byte, o.config.Workers)
+	o.buf = make([]chan *Message, o.config.Workers)
 	for i := 0; i < o.config.Workers; i++ {
-		o.buf[i] = make(chan []byte, 100)
+		o.buf[i] = make(chan *Message, 100)
 		go o.worker(i)
 	}
 
@@ -55,6 +57,10 @@ func (o *TCPOutput) worker(bufferIndex int) {
 	retries := 0
 	conn, err := o.connect(o.address)
 	for {
+		if o.close {
+			return
+		}
+
 		if err == nil {
 			break
 		}
@@ -73,14 +79,16 @@ func (o *TCPOutput) worker(bufferIndex int) {
 	defer conn.Close()
 
 	for {
-		data := <-o.buf[bufferIndex]
-		if _, err = conn.Write(data); err == nil {
-			_, err = conn.Write([]byte(payloadSeparator))
+		msg := <-o.buf[bufferIndex]
+		if _, err = conn.Write(msg.Meta); err == nil {
+			if _, err = conn.Write(msg.Data); err == nil {
+				_, err = conn.Write(payloadSeparatorAsBytes)
+			}
 		}
 
 		if err != nil {
 			Debug(2, "INFO: TCP output connection closed, reconnecting")
-			o.buf[bufferIndex] <- data
+			o.buf[bufferIndex] <- msg
 			go o.worker(bufferIndex)
 			break
 		}
@@ -99,30 +107,34 @@ func (o *TCPOutput) getBufferIndex(data []byte) int {
 
 }
 
-func (o *TCPOutput) Write(data []byte) (n int, err error) {
-	if !isOriginPayload(data) {
-		return len(data), nil
+// PluginWrite writes message to this plugin
+func (o *TCPOutput) PluginWrite(msg *Message) (n int, err error) {
+	if !isOriginPayload(msg.Meta) {
+		return len(msg.Data), nil
 	}
 
-	// We have to copy, because sending data in multiple threads
-	newBuf := make([]byte, len(data))
-	copy(newBuf, data)
-
-	bufferIndex := o.getBufferIndex(data)
-	o.buf[bufferIndex] <- newBuf
+	bufferIndex := o.getBufferIndex(msg.Data)
+	o.buf[bufferIndex] <- msg
 
 	if Settings.OutputTCPStats {
 		o.bufStats.Write(len(o.buf[bufferIndex]))
 	}
 
-	return len(data), nil
+	return len(msg.Data) + len(msg.Meta), nil
 }
 
 func (o *TCPOutput) connect(address string) (conn net.Conn, err error) {
 	if o.config.Secure {
-		conn, err = tls.Dial("tcp", address, &tls.Config{InsecureSkipVerify: o.config.SkipVerify})
+		var d tls.Dialer
+		d.Config = &tls.Config{InsecureSkipVerify: o.config.SkipVerify}
+		ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+		defer cancel()
+		conn, err = d.DialContext(ctx, "tcp", address)
 	} else {
-		conn, err = net.Dial("tcp", address)
+		var d net.Dialer
+		ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+		defer cancel()
+		conn, err = d.DialContext(ctx, "tcp", address)
 	}
 
 	return
@@ -131,3 +143,7 @@ func (o *TCPOutput) connect(address string) (conn net.Conn, err error) {
 func (o *TCPOutput) String() string {
 	return fmt.Sprintf("TCP output %s, limit: %d", o.address, o.limit)
 }
+
+func (o *TCPOutput) Close() {
+	o.close = true
+}

output_tcp_test.go

@@ -2,7 +2,6 @@ package main
 
 import (
 	"bufio"
-	"io"
 	"log"
 	"net"
 	"sync"
@@ -11,9 +10,7 @@ import (
 )
 
 func TestTCPOutput(t *testing.T) {
-	Settings.Verbose = 2
 	wg := new(sync.WaitGroup)
-	quit := make(chan int)
 
 	listener := startTCP(func(data []byte) {
 		wg.Done()
@@ -22,12 +19,11 @@ func TestTCPOutput(t *testing.T) {
 	output := NewTCPOutput(listener.Addr().String(), &TCPOutputConfig{Workers: 10})
 
 	plugins := &InOutPlugins{
-		Inputs:  []io.Reader{input},
-		Outputs: []io.Writer{output},
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{output},
 	}
-	plugins.All = append(plugins.All, input, output)
 
-	emitter := NewEmitter(quit)
+	emitter := NewEmitter()
 	go emitter.Start(plugins, Settings.Middleware)
 
 	for i := 0; i < 10; i++ {
@@ -68,7 +64,6 @@ func startTCP(cb func([]byte)) net.Listener {
 
 func BenchmarkTCPOutput(b *testing.B) {
 	wg := new(sync.WaitGroup)
-	quit := make(chan int)
 
 	listener := startTCP(func(data []byte) {
 		wg.Done()
@@ -82,12 +77,11 @@ func BenchmarkTCPOutput(b *testing.B) {
 	output := NewTCPOutput(listener.Addr().String(), &TCPOutputConfig{Workers: 10})
 
 	plugins := &InOutPlugins{
-		Inputs:  []io.Reader{input},
-		Outputs: []io.Writer{output},
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{output},
 	}
-	plugins.All = append(plugins.All, input, output)
 
-	emitter := NewEmitter(quit)
+	emitter := NewEmitter()
 	// avoid counting above initialization
 	b.ResetTimer()
 	go emitter.Start(plugins, Settings.Middleware)
@@ -109,7 +103,7 @@ func TestStickyDisable(t *testing.T) {
 
 func TestBufferDistribution(t *testing.T) {
 	numberOfWorkers := 10
-	numberOfMessages := 1000000
+	numberOfMessages := 10000
 	percentDistributionErrorRange := 20
 
 	buffer := make([]int, numberOfWorkers)

plugins.go

@@ -1,15 +1,36 @@
 package main
 
 import (
-	"io"
 	"reflect"
 	"strings"
 )
 
+// Message represents data across plugins
+type Message struct {
+	Meta []byte // metadata
+	Data []byte // actual data
+}
+
+// PluginReader is an interface for input plugins
+type PluginReader interface {
+	PluginRead() (msg *Message, err error)
+}
+
+// PluginWriter is an interface for output plugins
+type PluginWriter interface {
+	PluginWrite(msg *Message) (n int, err error)
+}
+
+// PluginReadWriter is an interface for plugins that support reading and writing
+type PluginReadWriter interface {
+	PluginReader
+	PluginWriter
+}
+
 // InOutPlugins struct for holding references to plugins
 type InOutPlugins struct {
-	Inputs  []io.Reader
-	Outputs []io.Writer
+	Inputs  []PluginReader
+	Outputs []PluginWriter
 	All     []interface{}
 }
 
@@ -48,27 +69,21 @@ func (plugins *InOutPlugins) registerPlugin(constructor interface{}, options ...
 
 	// Calling our constructor with list of given options
 	plugin := vc.Call(vo)[0].Interface()
-	pluginWrapper := plugin
 
 	if limit != "" {
-		pluginWrapper = NewLimiter(plugin, limit)
-	} else {
-		pluginWrapper = plugin
+		plugin = NewLimiter(plugin, limit)
 	}
 
-	_, isR := plugin.(io.Reader)
-	_, isW := plugin.(io.Writer)
-
 	// Some of the output can be Readers as well because return responses
-	if isR && !isW {
-		plugins.Inputs = append(plugins.Inputs, pluginWrapper.(io.Reader))
+	if r, ok := plugin.(PluginReader); ok {
+		plugins.Inputs = append(plugins.Inputs, r)
 	}
 
-	if isW {
-		plugins.Outputs = append(plugins.Outputs, pluginWrapper.(io.Writer))
+	if w, ok := plugin.(PluginWriter); ok {
+		plugins.Outputs = append(plugins.Outputs, w)
 	}
-
 	plugins.All = append(plugins.All, plugin)
+
 }
 
 // NewPlugins specify and initialize all available plugins

plugins_test.go

@@ -12,8 +12,8 @@ func TestPluginsRegistration(t *testing.T) {
 
 	plugins := NewPlugins()
 
-	if len(plugins.Inputs) != 2 {
-		t.Errorf("Should be 2 inputs %d", len(plugins.Inputs))
+	if len(plugins.Inputs) != 3 {
+		t.Errorf("Should be 3 inputs got %d", len(plugins.Inputs))
 	}
 
 	if _, ok := plugins.Inputs[0].(*DummyInput); !ok {

proto/fuzz.go

@@ -4,9 +4,7 @@ package proto
 
 func Fuzz(data []byte) int {
 
-	ParseHeaders([][]byte{data}, func(header []byte, value []byte) bool {
-		return true
-	})
+	ParseHeaders(data)
 
 	return 1
 }

proto/proto.go

@@ -19,6 +19,7 @@ package proto
 import (
 	"bufio"
 	"bytes"
+	_ "fmt"
 	"net/http"
 	"net/textproto"
 	"strings"
@@ -58,10 +59,15 @@ func MIMEHeadersStartPos(payload []byte) int {
 // If not found, value will be blank, and headerStart will be -1
 // Do not support multi-line headers.
 func header(payload []byte, name []byte) (value []byte, headerStart, headerEnd, valueStart, valueEnd int) {
-	headerStart = MIMEHeadersStartPos(payload)
-	if headerStart < 0 {
-		return
+	if HasTitle(payload) {
+		headerStart = MIMEHeadersStartPos(payload)
+		if headerStart < 0 {
+			return
+		}
+	} else {
+		headerStart = 0
 	}
+
 	var colonIndex int
 	for headerStart < len(payload) {
 		headerEnd = bytes.IndexByte(payload[headerStart:], '\n')
@@ -111,14 +117,13 @@ func header(payload []byte, name []byte) (value []byte, headerStart, headerEnd,
 	return
 }
 
-// ParseHeaders Parsing headers from multiple payloads
-func ParseHeaders(payloads [][]byte, cb func(header []byte, value []byte)) {
-	p := bytes.Join(payloads, nil)
+// ParseHeaders Parsing headers from the payload
+func ParseHeaders(p []byte) textproto.MIMEHeader {
 	// trimming off the title of the request
-	if HasRequestTitle(p) || HasResponseTitle(p) {
+	if HasTitle(p) {
 		headerStart := MIMEHeadersStartPos(p)
 		if headerStart > len(p)-1 {
-			return
+			return nil
 		}
 		p = p[headerStart:]
 	}
@@ -126,17 +131,17 @@ func ParseHeaders(payloads [][]byte, cb func(header []byte, value []byte)) {
 	if headerEnd > 1 {
 		p = p[:headerEnd]
 	}
-	reader := textproto.NewReader(bufio.NewReader(bytes.NewBuffer(p)))
+	return GetHeaders(p)
+}
+
+// GetHeaders returns mime headers from the payload
+func GetHeaders(p []byte) textproto.MIMEHeader {
+	reader := textproto.NewReader(bufio.NewReader(bytes.NewReader(p)))
 	mime, err := reader.ReadMIMEHeader()
 	if err != nil {
-		return
+		return nil
 	}
-	for k, v := range mime {
-		for _, value := range v {
-			cb([]byte(k), []byte(value))
-		}
-	}
-	return
+	return mime
 }
 
 // Header returns header value, if header not found, value will be blank
@@ -188,15 +193,15 @@ func DeleteHeader(payload, name []byte) []byte {
 // Body returns request/response body
 func Body(payload []byte) []byte {
 	pos := MIMEHeadersEndPos(payload)
-	if pos == -1 {
+	if pos == -1 || len(payload) <= pos {
 		return nil
 	}
 	return payload[pos:]
 }
 
-// Path takes payload and retuns request path: Split(firstLine, ' ')[1]
+// Path takes payload and returns request path: Split(firstLine, ' ')[1]
 func Path(payload []byte) []byte {
-	if !HasTitle(payload) {
+	if !HasRequestTitle(payload) {
 		return nil
 	}
 	start := bytes.IndexByte(payload, ' ') + 1
@@ -310,7 +315,12 @@ func Method(payload []byte) []byte {
 // Status returns response status.
 // It happens to be in same position as request payload path
 func Status(payload []byte) []byte {
-	return Path(payload)
+	if !HasResponseTitle(payload) {
+		return nil
+	}
+	start := bytes.IndexByte(payload, ' ') + 1
+	// status code are in range 100-600
+	return payload[start : start+3]
 }
 
 // Methods holds the http methods ordered in ascending order
@@ -323,8 +333,8 @@ var Methods = [...]string{
 const (
 	//MinRequestCount GET / HTTP/1.1\r\n
 	MinRequestCount = 16
-	// MinResponseCount HTTP/1.1 200 OK\r\n
-	MinResponseCount = 17
+	// MinResponseCount HTTP/1.1 200\r\n
+	MinResponseCount = 14
 	// VersionLen HTTP/1.1
 	VersionLen = 8
 )
@@ -347,17 +357,15 @@ func HasResponseTitle(payload []byte) bool {
 		return false
 	}
 	status, ok := atoI(payload[VersionLen+1:VersionLen+4], 10)
-	if !ok || s[VersionLen+4] != ' ' {
+	if !ok {
 		return false
 	}
-	statusText := http.StatusText(status)
-	if statusText == "" {
+	// only validate status codes mentioned in rfc2616.
+	if http.StatusText(status) == "" {
 		return false
 	}
-	if titleLen+len(CRLF) > len(s) {
-		return false
-	}
-	return s[VersionLen+5:titleLen] == statusText
+	// handle cases from #875
+	return payload[VersionLen+4] == ' ' || payload[VersionLen+4] == '\r'
 }
 
 // HasRequestTitle reports whether this payload has an HTTP/1 request title
@@ -396,102 +404,172 @@ func HasTitle(payload []byte) bool {
 	return HasRequestTitle(payload) || HasResponseTitle(payload)
 }
 
-// CheckChunked checks HTTP/1 chunked data integrity and return the final index
-// of chunks(index after '0\r\n\r\n') or -1 if there is missing data
-// or there is bad format
-func CheckChunked(buf []byte) (chunkEnd int) {
-	var (
-		ok     bool
-		chkLen int
-		sz     int
-		ext    int
-	)
-	for {
-		sz = bytes.IndexByte(buf[chunkEnd:], '\r')
-		if sz < 1 {
-			return -1
-		}
-		// ignoring chunks extensions https://github.com/golang/go/issues/13135
-		// but chunks extensions are no longer a thing
-		ext = bytes.IndexByte(buf[chunkEnd:chunkEnd+sz], ';')
-		if ext < 0 {
-			ext = sz
-		}
-		chkLen, ok = atoI(buf[chunkEnd:chunkEnd+ext], 16)
-		if !ok {
-			return -1
-		}
-		chunkEnd += (sz + 2)
-		if chkLen == 0 {
-			if !bytes.Equal(buf[chunkEnd:chunkEnd+2], CRLF) {
-				return -1
-			}
-			return chunkEnd + 2
-		}
-		// ideally chunck length and at least len("\r\n0\r\n\r\n")
-		if len(buf[chunkEnd:]) < chkLen+7 {
-			return -1
-		}
-		chunkEnd += chkLen
-		// chunks must end with CRLF
-		if !bytes.Equal(buf[chunkEnd:chunkEnd+2], CRLF) {
-			return -1
-		}
-		chunkEnd += 2
+// CheckChunked checks HTTP/1 chunked data integrity(https://tools.ietf.org/html/rfc7230#section-4.1)
+// and returns the length of total valid scanned chunks(including chunk size, extensions and CRLFs) and
+// full is true if all chunks was scanned.
+func CheckChunked(bufs ...[]byte) (chunkEnd int, full bool) {
+	var buf []byte
+	if len(bufs) > 0 {
+		buf = bufs[0]
 	}
+	for chunkEnd < len(buf) {
+		sz := bytes.IndexByte(buf[chunkEnd:], '\r')
+		if sz < 1 {
+			break
+		}
+		// don't parse chunk extensions https://github.com/golang/go/issues/13135.
+		// chunks extensions are no longer a thing, but we do check if the byte
+		// following the parsed hex number is ';'
+		sz += chunkEnd
+		chkLen, ok := atoI(buf[chunkEnd:sz], 16)
+		if !ok && bytes.IndexByte(buf[chunkEnd:sz], ';') < 1 {
+			break
+		}
+		sz++ // + '\n'
+		// total length = SIZE + CRLF + OCTETS + CRLF
+		allChunk := sz + chkLen + 2
+		if allChunk >= len(buf) ||
+			buf[sz]&buf[allChunk] != '\n' ||
+			buf[allChunk-1] != '\r' {
+			break
+		}
+		chunkEnd = allChunk + 1
+		if chkLen == 0 {
+			full = true
+			break
+		}
+	}
+	return
 }
 
-// HasFullPayload reports if this http has full payloads
-func HasFullPayload(payload []byte) bool {
-	body := Body(payload)
+// ProtocolStateSetter is an interface used to provide protocol state for future use
+type ProtocolStateSetter interface {
+	SetProtocolState(interface{})
+	ProtocolState() interface{}
+}
 
-	// check for chunked transfer-encoding
-	header := Header(payload, []byte("Transfer-Encoding"))
-	if bytes.Contains(header, []byte("chunked")) {
+type httpProto struct {
+	body         int // body index
+	headerStart  int
+	headerParsed bool // we checked necessary headers
+	hasFullBody  bool // all chunks has been parsed
+	isChunked    bool // Transfer-Encoding: chunked
+	bodyLen      int  // Content-Length's value
+	hasTrailer   bool // Trailer header?
+}
 
-		// check chunks
-		if len(body) < 1 {
-			return false
+// HasFullPayload checks if this message has full or valid payloads and returns true.
+// Message param is optional but recommended on cases where 'data' is storing
+// partial-to-full stream of bytes(packets).
+func HasFullPayload(m ProtocolStateSetter, payloads ...[]byte) bool {
+	var state *httpProto
+	if m != nil {
+		state, _ = m.ProtocolState().(*httpProto)
+	}
+	if state == nil {
+		state = new(httpProto)
+		if m != nil {
+			m.SetProtocolState(state)
 		}
-		var chunkEnd int
-		if chunkEnd = CheckChunked(body); chunkEnd < 1 {
+	}
+	if state.headerStart < 1 {
+		for _, data := range payloads {
+			state.headerStart = MIMEHeadersStartPos(data)
+			if state.headerStart < 0 {
+				return false
+			} else {
+				break
+			}
+		}
+	}
+
+	if state.body < 1 {
+		var pos int
+		for _, data := range payloads {
+			endPos := MIMEHeadersEndPos(data)
+			if endPos < 0 {
+				pos += len(data)
+			} else {
+				pos += endPos
+			}
+
+			if endPos > 0 {
+				state.body = pos
+				break
+			}
+		}
+	}
+	if !state.headerParsed {
+		var pos int
+		for _, data := range payloads {
+			chunked := Header(data, []byte("Transfer-Encoding"))
+
+			if len(chunked) > 0 && bytes.Index(data, []byte("chunked")) > 0 {
+				state.isChunked = true
+				// trailers are generally not allowed in non-chunks body
+				state.hasTrailer = len(Header(data, []byte("Trailer"))) > 0
+			} else {
+				contentLen := Header(data, []byte("Content-Length"))
+				state.bodyLen, _ = atoI(contentLen, 10)
+			}
+
+			pos += len(data)
+
+			if state.bodyLen > 0 || pos >= state.body {
+				state.headerParsed = true
+				break
+			}
+		}
+	}
+
+	bodyLen := 0
+	for _, data := range payloads {
+		bodyLen += len(data)
+	}
+	bodyLen -= state.body
+
+	if state.isChunked {
+		// check chunks
+		if bodyLen < 1 {
 			return false
 		}
 
 		// check trailer headers
-		if len(Header(payload, []byte("Trailer"))) < 1 {
-			return true
+		if state.hasTrailer {
+			if bytes.HasSuffix(payloads[len(payloads)-1], []byte("\r\n\r\n")) {
+				return true
+			}
+		} else {
+			if bytes.HasSuffix(payloads[len(payloads)-1], []byte("0\r\n\r\n")) {
+				state.hasFullBody = true
+				return true
+			}
 		}
-		// trailer headers(whether chunked or plain) should end with empty line
-		return len(body) > chunkEnd && MIMEHeadersEndPos(body[chunkEnd:]) != -1
+
+		return false
 	}
 
 	// check for content-length header
-	// trailers are generally not allowed in non-chunks body
-	header = Header(payload, []byte("Content-Length"))
-	if len(header) > 1 {
-		num, ok := atoI(header, 10)
-		return ok && num == len(body)
-	}
-
-	// for empty body, check for emptyline
-	return MIMEHeadersEndPos(payload) != -1
+	return state.bodyLen == bodyLen
 }
 
 // this works with positive integers
 func atoI(s []byte, base int) (num int, ok bool) {
 	var v int
+	ok = true
 	for i := 0; i < len(s); i++ {
 		if s[i] > 127 {
-			return 0, false
+			ok = false
+			break
 		}
 		v = int(hexTable[s[i]])
 		if v >= base || (v == 0 && s[i] != '0') {
-			return 0, false
+			ok = false
+			break
 		}
 		num = (num * base) + v
 	}
-	return num, true
+	return
 }
 
 var hexTable = [128]byte{

proto/proto_test.go

@@ -2,6 +2,7 @@ package proto
 
 import (
 	"bytes"
+	"net/textproto"
 	"reflect"
 	"testing"
 )
@@ -124,21 +125,35 @@ func TestDeleteHeader(t *testing.T) {
 func TestParseHeaders(t *testing.T) {
 	payload := [][]byte{[]byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.or"), []byte("g\r\nUser-Ag"), []byte("ent:Chrome\r\n\r\n"), []byte("Fake-Header: asda")}
 
-	headers := make(map[string]string)
+	headers := ParseHeaders(bytes.Join(payload, nil))
 
-	ParseHeaders(payload, func(header []byte, value []byte) {
-		headers[string(header)] = string(value)
-	})
-
-	expected := map[string]string{
-		"Content-Length": "7",
-		"Host":           "www.w3.org",
-		"User-Agent":     "Chrome",
+	expected := textproto.MIMEHeader{
+		"Content-Length": []string{"7"},
+		"Host":           []string{"www.w3.org"},
+		"User-Agent":     []string{"Chrome"},
 	}
 
 	if !reflect.DeepEqual(headers, expected) {
 		t.Error("Headers do not properly parsed", headers)
 	}
+
+	// Response with Reason phrase
+	payload = [][]byte{[]byte("HTTP/1.1 200 OK\r\nContent-Length: 7\r\nHost: www.w3.org\r\nUser-Agent:Chrome\r\n\r\nbody")}
+
+	headers = ParseHeaders(bytes.Join(payload, nil))
+
+	if !reflect.DeepEqual(headers, expected) {
+		t.Error("Headers do not properly parsed", headers)
+	}
+
+	// Response without Reason phrase
+	payload = [][]byte{[]byte("HTTP/1.1 200\r\nContent-Length: 7\r\nHost: www.w3.org\r\nUser-Agent:Chrome\r\n\r\nbody")}
+
+	headers = ParseHeaders(bytes.Join(payload, nil))
+
+	if !reflect.DeepEqual(headers, expected) {
+		t.Error("Headers do not properly parsed", headers)
+	}
 }
 
 // See https://github.com/dvyukov/go-fuzz and fuzz.go
@@ -148,8 +163,7 @@ func TestFuzzCrashers(t *testing.T) {
 	}
 
 	for _, f := range crashers {
-		ParseHeaders([][]byte{[]byte(f)}, func(header []byte, value []byte) {
-		})
+		ParseHeaders([]byte(f))
 	}
 }
 
@@ -158,17 +172,13 @@ func TestParseHeadersWithComplexUserAgent(t *testing.T) {
 	// Parser should wait for \r\n
 	payload := [][]byte{[]byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.or"), []byte("g\r\nUser-Ag"), []byte("ent:Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0) like Gecko\r\n\r\n"), []byte("Fake-Header: asda")}
 
-	headers := make(map[string]string)
-
-	ParseHeaders(payload, func(header []byte, value []byte) {
-		headers[string(header)] = string(value)
-	})
+	headers := ParseHeaders(bytes.Join(payload, nil))
 
 	expected := map[string]string{
 		"User-Agent": "Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0) like Gecko",
 	}
 
-	if expected["User-Agent"] != headers["User-Agent"] {
+	if expected["User-Agent"] != headers["User-Agent"][0] {
 		t.Errorf("Header 'User-Agent' expected '%s' and parsed: '%s'", expected["User-Agent"], headers["User-Agent"])
 	}
 }
@@ -178,11 +188,7 @@ func TestParseHeadersWithOrigin(t *testing.T) {
 	// Parser should wait for \r\n
 	payload := [][]byte{[]byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.or"), []byte("g\r\nReferrer: http://127.0.0.1:3000\r\nOrigi"), []byte("n: https://www.example.com\r\nUser-Ag"), []byte("ent:Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0) like Gecko\r\n\r\n"), []byte("in:https://www.example.com\r\n\r\n"), []byte("Fake-Header: asda")}
 
-	headers := make(map[string]string)
-
-	ParseHeaders(payload, func(header []byte, value []byte) {
-		headers[string(header)] = string(value)
-	})
+	headers := ParseHeaders(bytes.Join(payload, nil))
 
 	expected := map[string]string{
 		"Origin":     "https://www.example.com",
@@ -190,15 +196,15 @@ func TestParseHeadersWithOrigin(t *testing.T) {
 		"Referrer":   "http://127.0.0.1:3000",
 	}
 
-	if expected["Referrer"] != headers["Referrer"] {
+	if expected["Referrer"] != headers["Referrer"][0] {
 		t.Errorf("Header 'Referrer' expected '%s' and parsed: '%s'", expected["Referrer"], headers["Referrer"])
 	}
 
-	if expected["Origin"] != headers["Origin"] {
+	if expected["Origin"] != headers["Origin"][0] {
 		t.Errorf("Header 'Origin' expected '%s' and parsed: '%s'", expected["Origin"], headers["Origin"])
 	}
 
-	if expected["User-Agent"] != headers["User-Agent"] {
+	if expected["User-Agent"] != headers["User-Agent"][0] {
 		t.Errorf("Header 'User-Agent' expected '%s' and parsed: '%s'", expected["User-Agent"], headers["User-Agent"])
 	}
 }
@@ -231,6 +237,25 @@ func TestPath(t *testing.T) {
 	}
 }
 
+func TestStatus(t *testing.T) {
+	var status, payload []byte
+
+	payload = []byte("HTTP/1.1 200 OK\r\n")
+	if status = Status(payload); !bytes.Equal(status, []byte("200")) {
+		t.Error("Should find status 200 but:", string(status))
+	}
+
+	payload = []byte("HTTP/1.1 200\r\n")
+	if status = Status(payload); !bytes.Equal(status, []byte("200")) {
+		t.Error("1Should find status 200 but:", string(status))
+	}
+
+	payload = []byte("HTTP/1.1 404 Not Found\r\n")
+	if status = Status(payload); !bytes.Equal(status, []byte("404")) {
+		t.Error("2Should find status 404 but:", string(status))
+	}
+}
+
 func TestSetPath(t *testing.T) {
 	var payload, payloadAfter []byte
 
@@ -330,9 +355,10 @@ func TestHasResponseTitle(t *testing.T) {
 		"HTTP/1.1 100 Continue\r\n": true,
 		"HTTP/1.1  \r\n":            false,
 		"HTTP/4.0 100Continue\r\n":  false,
+		"HTTP/1.0 100Continue\r\n":  false,
 		"HTTP/1.0 10r Continue\r\n": false,
-		"HTTP/1.1 200\r\n":          false,
-		"HTTP/1.1 200\r\nServer: Tengine\r\nContent-Length: 0\r\nConnection: close\r\n\r\n": false,
+		"HTTP/1.1 200\r\n":          true,
+		"HTTP/1.1 200\r\nServer: Tengine\r\nContent-Length: 0\r\nConnection: close\r\n\r\n": true,
 	}
 	for k, v := range m {
 		if HasResponseTitle([]byte(k)) != v {
@@ -362,63 +388,61 @@ func TestHasRequestTitle(t *testing.T) {
 
 func TestCheckChunks(t *testing.T) {
 	var m = "4\r\nWiki\r\n5\r\npedia\r\nE\r\n in\r\n\r\nchunks.\r\n0\r\n\r\n"
-	chunkEnd := CheckChunked([]byte(m))
-	expected := bytes.Index([]byte(m), []byte("0\r\n")) + 5
+	chunkEnd, _ := CheckChunked([]byte(m))
+	expected := len(m)
 	if chunkEnd != expected {
 		t.Errorf("expected %d to equal %d", chunkEnd, expected)
 	}
 
 	m = "7\r\nMozia\r\n9\r\nDeveloper\r\n7\r\nNetwork\r\n0\r\n\r\n"
-	chunkEnd = CheckChunked([]byte(m))
-	if chunkEnd != -1 {
-		t.Errorf("expected %d to equal %d", chunkEnd, -1)
+	chunkEnd, _ = CheckChunked([]byte(m))
+	if chunkEnd != 0 {
+		t.Errorf("expected %d to equal %d", chunkEnd, 0)
 	}
 
 	// with trailers
 	m = "4\r\nWiki\r\n5\r\npedia\r\nE\r\n in\r\n\r\nchunks.\r\n0\r\n\r\nEXpires"
-	chunkEnd = CheckChunked([]byte(m))
-	expected = bytes.Index([]byte(m), []byte("0\r\n")) + 5
+	chunkEnd, _ = CheckChunked([]byte(m))
+	expected = len(m) - 7
 	if chunkEnd != expected {
 		t.Errorf("expected %d to equal %d", chunkEnd, expected)
 	}
 
-	// last chunk inside the the body
+	// last chunk inside the body
 	// with trailers
 	m = "4\r\nWiki\r\n5\r\npedia\r\nE\r\n in\r\n\r\nchunks.\r\n3\r\n0\r\n\r\n0\r\n\r\nEXpires"
-	chunkEnd = CheckChunked([]byte(m))
-	expected = bytes.Index([]byte(m), []byte("0\r\n")) + 10
+	chunkEnd, _ = CheckChunked([]byte(m))
+	expected = len(m) - 7
 	if chunkEnd != expected {
 		t.Errorf("expected %d to equal %d", chunkEnd, expected)
 	}
 
 	// checks with chucks-extensions
 	m = "4\r\nWiki\r\n5\r\npedia\r\nE; name='quoted string'\r\n in\r\n\r\nchunks.\r\n3\r\n0\r\n\r\n0\r\n\r\nEXpires"
-	chunkEnd = CheckChunked([]byte(m))
-	expected = bytes.Index([]byte(m), []byte("0\r\n")) + 10
+	chunkEnd, _ = CheckChunked([]byte(m))
+	expected = len(m) - 7
 	if chunkEnd != expected {
 		t.Errorf("expected %d to equal %d", chunkEnd, expected)
 	}
 }
 
 func TestHasFullPayload(t *testing.T) {
-	var m = "HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nTransfer-Encoding: chunked\r\n\r\n7\r\nMozilla\r\n9\r\nDeveloper\r\n7\r\nNetwork\r\n0\r\n\r\n"
-	got := HasFullPayload([]byte(m))
-	expected := true
-	if got != expected {
-		t.Errorf("expected %v to equal %v", got, expected)
-	}
+	var m string
+	var got, expected bool
 
-	// check with invalid chunk format
-	m = "HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nTransfer-Encoding: chunked\r\n\r\n7\r\nMozia\r\n9\r\nDeveloper\r\n7\r\nNetwork\r\n0\r\n\r\n"
-	got = HasFullPayload([]byte(m))
-	expected = false
+	got = HasFullPayload(nil,
+		[]byte("HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\n"),
+		[]byte("Transfer-Encoding: chunked\r\n\r\n"),
+		[]byte("7\r\nMozilla\r\n9\r\nDeveloper\r\n"),
+		[]byte("7\r\nNetwork\r\n0\r\n\r\n"))
+	expected = true
 	if got != expected {
 		t.Errorf("expected %v to equal %v", got, expected)
 	}
 
 	// check chunks with trailers
 	m = "HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nTransfer-Encoding: chunked\r\nTrailer: Expires\r\n\r\n7\r\nMozilla\r\n9\r\nDeveloper\r\n7\r\nNetwork\r\n0\r\n\r\nExpires: Wed, 21 Oct 2015 07:28:00 GMT\r\n\r\n"
-	got = HasFullPayload([]byte(m))
+	got = HasFullPayload(nil, []byte(m))
 	expected = true
 	if got != expected {
 		t.Errorf("expected %v to equal %v", got, expected)
@@ -426,7 +450,7 @@ func TestHasFullPayload(t *testing.T) {
 
 	// check with missing trailers
 	m = "HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nTransfer-Encoding: chunked\r\nTrailer: Expires\r\n\r\n7\r\nMozilla\r\n9\r\nDeveloper\r\n7\r\nNetwork\r\n0\r\n\r\nExpires: Wed, 21 Oct 2015 07:28:00"
-	got = HasFullPayload([]byte(m))
+	got = HasFullPayload(nil, []byte(m))
 	expected = false
 	if got != expected {
 		t.Errorf("expected %v to equal %v", got, expected)
@@ -434,7 +458,7 @@ func TestHasFullPayload(t *testing.T) {
 
 	// check with content-length
 	m = "HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nContent-Length: 23\r\n\r\nMozillaDeveloperNetwork"
-	got = HasFullPayload([]byte(m))
+	got = HasFullPayload(nil, []byte(m))
 	expected = true
 	if got != expected {
 		t.Errorf("expected %v to equal %v", got, expected)
@@ -442,7 +466,7 @@ func TestHasFullPayload(t *testing.T) {
 
 	// check missing total length
 	m = "HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nContent-Length: 23\r\n\r\nMozillaDeveloperNet"
-	got = HasFullPayload([]byte(m))
+	got = HasFullPayload(nil, []byte(m))
 	expected = false
 	if got != expected {
 		t.Errorf("expected %v to equal %v", got, expected)
@@ -450,7 +474,7 @@ func TestHasFullPayload(t *testing.T) {
 
 	// check with no body
 	m = "HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\n\r\n"
-	got = HasFullPayload([]byte(m))
+	got = HasFullPayload(nil, []byte(m))
 	expected = true
 	if got != expected {
 		t.Errorf("expected %v to equal %v", got, expected)
@@ -458,21 +482,10 @@ func TestHasFullPayload(t *testing.T) {
 }
 
 func BenchmarkHasFullPayload(b *testing.B) {
-	var buf bytes.Buffer
-	buf.Write([]byte("HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nTransfer-Encoding: chunked\r\n\r\n"))
-	var chunk = []byte("1e\r\n111111111111111111111111111111\r\n")
-	for i := 0; i < 5000; i++ {
-		buf.Write(chunk)
-	}
-	buf.Write([]byte("0\r\n\r\n"))
-	data := buf.Bytes()
-	var ok bool
-	b.ResetTimer() // ignores the upper initialization
-	b.ReportMetric(float64(5000), "chunks/op")
+	data := []byte("HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nTransfer-Encoding: chunked\r\n\r\n1e\r\n111111111111111111111111111111\r\n0\r\n\r\n")
 	for i := 0; i < b.N; i++ {
-		ok = HasFullPayload(data)
-	}
-	if !ok {
-		b.Fail()
+		if !HasFullPayload(nil, data) {
+			b.Fail()
+		}
 	}
 }

protocol.go

@@ -66,25 +66,27 @@ func payloadMeta(payload []byte) [][]byte {
 	return bytes.Split(payload[:headerSize], []byte{' '})
 }
 
+func payloadMetaWithBody(payload []byte) (meta, body []byte) {
+	if i := bytes.IndexByte(payload, '\n'); i > 0 && len(payload) > i+1 {
+		meta = payload[:i+1]
+		body = payload[i+1:]
+		return
+	}
+	// we assume the message did not have meta data
+	return nil, payload
+}
+
 func payloadID(payload []byte) (id []byte) {
 	meta := payloadMeta(payload)
 
 	if len(meta) < 2 {
 		return
 	}
-	// id is encoded in hex, we need to revert to how it was
-	id = make([]byte, 20)
-	hex.Decode(id, meta[1])
-	return
+	return meta[1]
 }
 
 func isOriginPayload(payload []byte) bool {
-	switch payload[0] {
-	case RequestPayload, ResponsePayload:
-		return true
-	default:
-		return false
-	}
+	return payload[0] == RequestPayload || payload[0] == ResponsePayload
 }
 
 func isRequestPayload(payload []byte) bool {

ring/ring.go

@@ -0,0 +1,205 @@
+package ring
+
+import (
+	"errors"
+	"runtime"
+	"sync/atomic"
+	"time"
+)
+
+var (
+	// ErrDisposed is returned when an operation is performed on a disposed
+	// queue.
+	ErrDisposed = errors.New(`queue: disposed`)
+
+	// ErrTimeout is returned when an applicable queue operation times out.
+	ErrTimeout = errors.New(`queue: poll timed out`)
+
+	// ErrEmptyQueue is returned when an non-applicable queue operation was called
+	// due to the queue's empty item state
+	ErrEmptyQueue = errors.New(`queue: empty queue`)
+)
+
+// roundUp takes a uint64 greater than 0 and rounds it up to the next
+// power of 2.
+func roundUp(v uint64) uint64 {
+	v--
+	v |= v >> 1
+	v |= v >> 2
+	v |= v >> 4
+	v |= v >> 8
+	v |= v >> 16
+	v |= v >> 32
+	v++
+	return v
+}
+
+type node struct {
+	position uint64
+	data     interface{}
+}
+
+type nodes []node
+
+// RingBuffer is a MPMC buffer that achieves threadsafety with CAS operations
+// only.  A put on full or get on empty call will block until an item
+// is put or retrieved.  Calling Dispose on the RingBuffer will unblock
+// any blocked threads with an error.  This buffer is similar to the buffer
+// described here: http://www.1024cores.net/home/lock-free-algorithms/queues/bounded-mpmc-queue
+// with some minor additions.
+type RingBuffer struct {
+	_padding0      [8]uint64
+	queue          uint64
+	_padding1      [8]uint64
+	dequeue        uint64
+	_padding2      [8]uint64
+	mask, disposed uint64
+	_padding3      [8]uint64
+	nodes          nodes
+}
+
+func (rb *RingBuffer) init(size uint64) {
+	size = roundUp(size)
+	rb.nodes = make(nodes, size)
+	for i := uint64(0); i < size; i++ {
+		rb.nodes[i] = node{position: i}
+	}
+	rb.mask = size - 1 // so we don't have to do this with every put/get operation
+}
+
+// Put adds the provided item to the queue.  If the queue is full, this
+// call will block until an item is added to the queue or Dispose is called
+// on the queue.  An error will be returned if the queue is disposed.
+func (rb *RingBuffer) Put(item interface{}) error {
+	_, err := rb.put(item, false)
+	return err
+}
+
+// Offer adds the provided item to the queue if there is space.  If the queue
+// is full, this call will return false.  An error will be returned if the
+// queue is disposed.
+func (rb *RingBuffer) Offer(item interface{}) (bool, error) {
+	return rb.put(item, true)
+}
+
+func (rb *RingBuffer) put(item interface{}, offer bool) (bool, error) {
+	var n *node
+	pos := atomic.LoadUint64(&rb.queue)
+L:
+	for {
+		if atomic.LoadUint64(&rb.disposed) == 1 {
+			return false, ErrDisposed
+		}
+
+		n = &rb.nodes[pos&rb.mask]
+		seq := atomic.LoadUint64(&n.position)
+		switch dif := seq - pos; {
+		case dif == 0:
+			if atomic.CompareAndSwapUint64(&rb.queue, pos, pos+1) {
+				break L
+			}
+		case dif < 0:
+			panic(`Ring buffer in a compromised state during a put operation.`)
+		default:
+			pos = atomic.LoadUint64(&rb.queue)
+		}
+
+		if offer {
+			return false, nil
+		}
+
+		runtime.Gosched() // free up the cpu before the next iteration
+	}
+
+	n.data = item
+	atomic.StoreUint64(&n.position, pos+1)
+	return true, nil
+}
+
+// Get will return the next item in the queue.  This call will block
+// if the queue is empty.  This call will unblock when an item is added
+// to the queue or Dispose is called on the queue.  An error will be returned
+// if the queue is disposed.
+func (rb *RingBuffer) Get() (interface{}, error) {
+	return rb.Poll(0)
+}
+
+// Poll will return the next item in the queue.  This call will block
+// if the queue is empty.  This call will unblock when an item is added
+// to the queue, Dispose is called on the queue, or the timeout is reached. An
+// error will be returned if the queue is disposed or a timeout occurs. A
+// non-positive timeout will block indefinitely.
+func (rb *RingBuffer) Poll(timeout time.Duration) (interface{}, error) {
+	var (
+		n     *node
+		pos   = atomic.LoadUint64(&rb.dequeue)
+		start time.Time
+	)
+	if timeout > 0 {
+		start = time.Now()
+	}
+L:
+	for {
+		if atomic.LoadUint64(&rb.disposed) == 1 {
+			return nil, ErrDisposed
+		}
+
+		n = &rb.nodes[pos&rb.mask]
+		seq := atomic.LoadUint64(&n.position)
+		switch dif := seq - (pos + 1); {
+		case dif == 0:
+			if atomic.CompareAndSwapUint64(&rb.dequeue, pos, pos+1) {
+				break L
+			}
+		case dif < 0:
+			panic(`Ring buffer in compromised state during a get operation.`)
+		default:
+			pos = atomic.LoadUint64(&rb.dequeue)
+		}
+
+		if timeout > 0 && time.Since(start) >= timeout {
+			return nil, ErrTimeout
+		}
+
+		if timeout < 0 {
+			return nil, ErrTimeout
+		}
+
+		runtime.Gosched() // free up the cpu before the next iteration
+	}
+	data := n.data
+	n.data = nil
+	atomic.StoreUint64(&n.position, pos+rb.mask+1)
+	return data, nil
+}
+
+// Len returns the number of items in the queue.
+func (rb *RingBuffer) Len() uint64 {
+	return atomic.LoadUint64(&rb.queue) - atomic.LoadUint64(&rb.dequeue)
+}
+
+// Cap returns the capacity of this ring buffer.
+func (rb *RingBuffer) Cap() uint64 {
+	return uint64(len(rb.nodes))
+}
+
+// Dispose will dispose of this queue and free any blocked threads
+// in the Put and/or Get methods.  Calling those methods on a disposed
+// queue will return an error.
+func (rb *RingBuffer) Dispose() {
+	atomic.CompareAndSwapUint64(&rb.disposed, 0, 1)
+}
+
+// IsDisposed will return a bool indicating if this queue has been
+// disposed.
+func (rb *RingBuffer) IsDisposed() bool {
+	return atomic.LoadUint64(&rb.disposed) == 1
+}
+
+// NewRingBuffer will allocate, initialize, and return a ring buffer
+// with the specified size.
+func NewRingBuffer(size uint64) *RingBuffer {
+	rb := &RingBuffer{}
+	rb.init(size)
+	return rb
+}

s3_reader.go

@@ -60,7 +60,7 @@ func NewS3ReadCloser(path string) *S3ReadCloser {
 	bucket, key := parseS3Url(path)
 	sess := session.Must(session.NewSession(awsConfig()))
 
-	log.Println("[S3 Input] S3 connection succesfully initialized", path)
+	log.Println("[S3 Input] S3 connection successfully initialized", path)
 
 	return &S3ReadCloser{
 		bucket: bucket,

settings.go

@@ -93,14 +93,10 @@ func init() {
 	}
 
 	flag.BoolVar(&Settings.SplitOutput, "split-output", false, "By default each output gets same traffic. If set to `true` it splits traffic equally among all outputs.")
-
 	flag.BoolVar(&Settings.RecognizeTCPSessions, "recognize-tcp-sessions", false, "[PRO] If turned on http output will create separate worker for each TCP session. Splitting output will session based as well.")
 
 	flag.Var(&Settings.InputDummy, "input-dummy", "Used for testing outputs. Emits 'Get /' request every 1s")
-	flag.Var(&Settings.OutputDummy, "output-dummy", "DEPRECATED: use --output-stdout instead")
-
 	flag.BoolVar(&Settings.OutputStdout, "output-stdout", false, "Used for testing inputs. Just prints to console data coming from inputs.")
-
 	flag.BoolVar(&Settings.OutputNull, "output-null", false, "Used for testing inputs. Drops all requests.")
 
 	flag.Var(&Settings.InputTCP, "input-tcp", "Used for internal communication between Gor instances. Example: \n\t# Receive requests from other Gor instances on 28020 port, and redirect output to staging\n\tgor --input-tcp :28020 --output-http staging.com")
@@ -127,7 +123,7 @@ func init() {
 
 	flag.StringVar(&Settings.OutputFileConfig.BufferPath, "output-file-buffer", "/tmp", "The path for temporary storing current buffer: \n\tgor --input-raw :80 --output-file s3://mybucket/logs/%Y-%m-%d.gz --output-file-buffer /mnt/logs")
 
-	flag.BoolVar(&Settings.PrettifyHTTP, "prettify-http", false, "If enabled, will automatically decode requests and responses with: Content-Encoding: gzip and Transfer-Encoding: chunked. Useful for debugging, in conjuction with --output-stdout")
+	flag.BoolVar(&Settings.PrettifyHTTP, "prettify-http", false, "If enabled, will automatically decode requests and responses with: Content-Encoding: gzip and Transfer-Encoding: chunked. Useful for debugging, in conjunction with --output-stdout")
 
 	// input raw flags
 	flag.Var(&Settings.InputRAW, "input-raw", "Capture traffic from given port (use RAW sockets and require *sudo* access):\n\t# Capture traffic from 8080 port\n\tgor --input-raw :8080 --output-http staging.com")
@@ -145,6 +141,7 @@ func init() {
 	flag.BoolVar(&Settings.Promiscuous, "input-raw-promisc", false, "enable promiscuous mode")
 	flag.BoolVar(&Settings.Monitor, "input-raw-monitor", false, "enable RF monitor mode")
 	flag.BoolVar(&Settings.Stats, "input-raw-stats", false, "enable stats generator on raw TCP messages")
+	flag.BoolVar(&Settings.AllowIncomplete, "input-raw-allow-incomplete", false, "If turned on Gor will record HTTP messages with missing packets")
 
 	flag.StringVar(&Settings.Middleware, "middleware", "", "Used for modifying traffic using external command")
 
@@ -152,11 +149,11 @@ func init() {
 
 	/* outputHTTPConfig */
 	flag.Var(&Settings.OutputHTTPConfig.BufferSize, "output-http-response-buffer", "HTTP response buffer size, all data after this size will be discarded.")
-	flag.BoolVar(&Settings.OutputHTTPConfig.CompatibilityMode, "output-http-compatibility-mode", false, "Use standard Go client, instead of built-in implementation. Can be slower, but more compatible.")
-
 	flag.IntVar(&Settings.OutputHTTPConfig.WorkersMin, "output-http-workers-min", 0, "Gor uses dynamic worker scaling. Enter a number to set a minimum number of workers. default = 1.")
 	flag.IntVar(&Settings.OutputHTTPConfig.WorkersMax, "output-http-workers", 0, "Gor uses dynamic worker scaling. Enter a number to set a maximum number of workers. default = 0 = unlimited.")
 	flag.IntVar(&Settings.OutputHTTPConfig.QueueLen, "output-http-queue-len", 1000, "Number of requests that can be queued for output, if all workers are busy. default = 1000")
+	flag.BoolVar(&Settings.OutputHTTPConfig.SkipVerify, "output-http-skip-verify", false, "Don't verify hostname on TLS secure connection.")
+	flag.DurationVar(&Settings.OutputHTTPConfig.WorkerTimeout, "output-http-worker-timeout", 2*time.Second, "Duration to rollback idle workers.")
 
 	flag.IntVar(&Settings.OutputHTTPConfig.RedirectLimit, "output-http-redirects", 0, "Enable how often redirects should be followed.")
 	flag.DurationVar(&Settings.OutputHTTPConfig.Timeout, "output-http-timeout", 5*time.Second, "Specify HTTP request/response timeout. By default 5s. Example: --output-http-timeout 30s")
@@ -165,11 +162,11 @@ func init() {
 	flag.BoolVar(&Settings.OutputHTTPConfig.Stats, "output-http-stats", false, "Report http output queue stats to console every N milliseconds. See output-http-stats-ms")
 	flag.IntVar(&Settings.OutputHTTPConfig.StatsMs, "output-http-stats-ms", 5000, "Report http output queue stats to console every N milliseconds. default: 5000")
 	flag.BoolVar(&Settings.OutputHTTPConfig.OriginalHost, "http-original-host", false, "Normally gor replaces the Host http header with the host supplied with --output-http.  This option disables that behavior, preserving the original Host header.")
-	flag.BoolVar(&Settings.OutputHTTPConfig.Debug, "output-http-debug", false, "Enables http debug output.")
 	flag.StringVar(&Settings.OutputHTTPConfig.ElasticSearch, "output-http-elasticsearch", "", "Send request and response stats to ElasticSearch:\n\tgor --input-raw :8080 --output-http staging.com --output-http-elasticsearch 'es_host:api_port/index_name'")
 	/* outputHTTPConfig */
 
 	flag.Var(&Settings.OutputBinary, "output-binary", "Forwards incoming binary payloads to given address.\n\t# Redirect all incoming requests to staging.com address \n\tgor --input-raw :80 --input-raw-protocol binary --output-binary staging.com:80")
+
 	/* outputBinaryConfig */
 	flag.Var(&Settings.OutputBinaryConfig.BufferSize, "output-tcp-response-buffer", "TCP response buffer size, all data after this size will be discarded.")
 	flag.IntVar(&Settings.OutputBinaryConfig.Workers, "output-binary-workers", 0, "Gor uses dynamic worker scaling by default.  Enter a number to run a set number of workers.")
@@ -188,36 +185,20 @@ func init() {
 	flag.BoolVar(&Settings.InputKafkaConfig.UseJSON, "input-kafka-json-format", false, "If turned on, it will assume that messages coming in JSON format rather than  GoReplay text format.")
 
 	flag.StringVar(&Settings.KafkaTLSConfig.CACert, "kafka-tls-ca-cert", "", "CA certificate for Kafka TLS Config:\n\tgor  --input-raw :3000 --output-kafka-host '192.168.0.1:9092' --output-kafka-topic 'topic' --kafka-tls-ca-cert cacert.cer.pem --kafka-tls-client-cert client.cer.pem --kafka-tls-client-key client.key.pem")
-	flag.StringVar(&Settings.KafkaTLSConfig.clientCert, "kafka-tls-client-cert", "", "Client certificate for Kafka TLS Config (mandatory with to kafka-tls-ca-cert and kafka-tls-client-key)")
-	flag.StringVar(&Settings.KafkaTLSConfig.clientKey, "kafka-tls-client-key", "", "Client Key for Kafka TLS Config (mandatory with to kafka-tls-client-cert and kafka-tls-client-key)")
+	flag.StringVar(&Settings.KafkaTLSConfig.ClientCert, "kafka-tls-client-cert", "", "Client certificate for Kafka TLS Config (mandatory with to kafka-tls-ca-cert and kafka-tls-client-key)")
+	flag.StringVar(&Settings.KafkaTLSConfig.ClientKey, "kafka-tls-client-key", "", "Client Key for Kafka TLS Config (mandatory with to kafka-tls-client-cert and kafka-tls-client-key)")
 
 	flag.Var(&Settings.ModifierConfig.Headers, "http-set-header", "Inject additional headers to http reqest:\n\tgor --input-raw :8080 --output-http staging.com --http-set-header 'User-Agent: Gor'")
-	flag.Var(&Settings.ModifierConfig.Headers, "output-http-header", "WARNING: `--output-http-header` DEPRECATED, use `--http-set-header` instead")
-
 	flag.Var(&Settings.ModifierConfig.HeaderRewrite, "http-rewrite-header", "Rewrite the request header based on a mapping:\n\tgor --input-raw :8080 --output-http staging.com --http-rewrite-header Host: (.*).example.com,$1.beta.example.com")
-
 	flag.Var(&Settings.ModifierConfig.Params, "http-set-param", "Set request url param, if param already exists it will be overwritten:\n\tgor --input-raw :8080 --output-http staging.com --http-set-param api_key=1")
-
 	flag.Var(&Settings.ModifierConfig.Methods, "http-allow-method", "Whitelist of HTTP methods to replay. Anything else will be dropped:\n\tgor --input-raw :8080 --output-http staging.com --http-allow-method GET --http-allow-method OPTIONS")
-
 	flag.Var(&Settings.ModifierConfig.URLRegexp, "http-allow-url", "A regexp to match requests against. Filter get matched against full url with domain. Anything else will be dropped:\n\t gor --input-raw :8080 --output-http staging.com --http-allow-url ^www.")
-
 	flag.Var(&Settings.ModifierConfig.URLNegativeRegexp, "http-disallow-url", "A regexp to match requests against. Filter get matched against full url with domain. Anything else will be forwarded:\n\t gor --input-raw :8080 --output-http staging.com --http-disallow-url ^www.")
-
 	flag.Var(&Settings.ModifierConfig.URLRewrite, "http-rewrite-url", "Rewrite the request url based on a mapping:\n\tgor --input-raw :8080 --output-http staging.com --http-rewrite-url /v1/user/([^\\/]+)/ping:/v2/user/$1/ping")
-	flag.Var(&Settings.ModifierConfig.URLRewrite, "output-http-rewrite-url", "WARNING: `--output-http-rewrite-url` DEPRECATED, use `--http-rewrite-url` instead")
-
 	flag.Var(&Settings.ModifierConfig.HeaderFilters, "http-allow-header", "A regexp to match a specific header against. Requests with non-matching headers will be dropped:\n\t gor --input-raw :8080 --output-http staging.com --http-allow-header api-version:^v1")
-	flag.Var(&Settings.ModifierConfig.HeaderFilters, "output-http-header-filter", "WARNING: `--output-http-header-filter` DEPRECATED, use `--http-allow-header` instead")
-
 	flag.Var(&Settings.ModifierConfig.HeaderNegativeFilters, "http-disallow-header", "A regexp to match a specific header against. Requests with matching headers will be dropped:\n\t gor --input-raw :8080 --output-http staging.com --http-disallow-header \"User-Agent: Replayed by Gor\"")
-
 	flag.Var(&Settings.ModifierConfig.HeaderBasicAuthFilters, "http-basic-auth-filter", "A regexp to match the decoded basic auth string against. Requests with non-matching headers will be dropped:\n\t gor --input-raw :8080 --output-http staging.com --http-basic-auth-filter \"^customer[0-9].*\"")
-
 	flag.Var(&Settings.ModifierConfig.HeaderHashFilters, "http-header-limiter", "Takes a fraction of requests, consistently taking or rejecting a request based on the FNV32-1A hash of a specific header:\n\t gor --input-raw :8080 --output-http staging.com --http-header-limiter user-id:25%")
-
-	flag.Var(&Settings.ModifierConfig.HeaderHashFilters, "output-http-header-hash-filter", "WARNING: `output-http-header-hash-filter` DEPRECATED, use `--http-header-hash-limiter` instead")
-
 	flag.Var(&Settings.ModifierConfig.ParamHashFilters, "http-param-limiter", "Takes a fraction of requests, consistently taking or rejecting a request based on the FNV32-1A hash of a specific GET param:\n\t gor --input-raw :8080 --output-http staging.com --http-param-limiter user_id:25%")
 
 	// default values, using for tests

tcp/doc.go

@@ -15,7 +15,7 @@ maxSize := 5 << 20
 debugger := func(debugLevel int, data ...interface{}){} // debugger can also be nil
 messageHandler := func(mssg *tcp.Message){}
 
-mssgPool := tcp.NewMessagePool(maxMessageSize, messageExpire, debugger, messageHandler)
+mssgPool := tcp.NewMessageParser(maxMessageSize, messageExpire, debugger, messageHandler)
 listener.Listen(ctx, mssgPool.Handler)
 
 you can use pool.End or/and pool.Start to set custom session behaviors

tcp/tcp_message.go

@@ -1,90 +1,153 @@
 package tcp
 
 import (
-	"bytes"
-	"crypto/sha1"
+	"encoding/binary"
 	"encoding/hex"
-	"fmt"
+	_ "fmt"
+	"reflect"
 	"sort"
-	"sync"
 	"time"
+	"unsafe"
 
 	"github.com/buger/goreplay/size"
-	"github.com/google/gopacket"
 )
 
 // Stats every message carry its own stats object
 type Stats struct {
-	LostData   int
-	Length     int       // length of the data
-	Start      time.Time // first packet's timestamp
-	End        time.Time // last packet's timestamp
-	SrcAddr    string
-	DstAddr    string
-	IsIncoming bool
-	TimedOut   bool // timeout before getting the whole message
-	Truncated  bool // last packet truncated due to max message size
-	IPversion  byte
+	LostData  int
+	Length    int       // length of the data
+	Start     time.Time // first packet's timestamp
+	End       time.Time // last packet's timestamp
+	SrcAddr   string
+	DstAddr   string
+	IsRequest bool
+	TimedOut  bool // timeout before getting the whole message
+	Truncated bool // last packet truncated due to max message size
+	IPversion byte
 }
 
 // Message is the representation of a tcp message
 type Message struct {
-	packets []*Packet
-	done    chan bool
-	buf     bytes.Buffer
+	packets  []*Packet
+	parser   *MessageParser
+	feedback interface{}
 	Stats
 }
 
-// NewMessage ...
-func NewMessage(srcAddr, dstAddr string, ipVersion uint8) (m *Message) {
-	m = new(Message)
-	m.DstAddr = dstAddr
-	m.SrcAddr = srcAddr
-	m.IPversion = ipVersion
-	m.done = make(chan bool)
-	return
-}
-
-// UUID the unique id of a TCP session it is not granted to be unique!
+// UUID returns the UUID of a TCP request and its response.
 func (m *Message) UUID() []byte {
-	var src, dst string
-	if m.IsIncoming {
-		src = m.SrcAddr
-		dst = m.DstAddr
+	var streamID uint64
+	pckt := m.packets[0]
+
+	// check if response or request have generated the ID before.
+	if m.IsRequest {
+		streamID = uint64(pckt.SrcPort)<<48 | uint64(pckt.DstPort)<<32 |
+			uint64(ip2int(pckt.SrcIP))
 	} else {
-		src = m.DstAddr
-		dst = m.SrcAddr
+		streamID = uint64(pckt.DstPort)<<48 | uint64(pckt.SrcPort)<<32 |
+			uint64(ip2int(pckt.DstIP))
 	}
 
-	length := len(src) + len(dst)
-	uuid := make([]byte, length)
-	copy(uuid, src)
-	copy(uuid[len(src):], dst)
-	sha := sha1.Sum(uuid)
-	uuid = make([]byte, 40)
-	hex.Encode(uuid, sha[:])
+	id := make([]byte, 12)
+	binary.BigEndian.PutUint64(id, streamID)
 
-	return uuid
-}
-
-func (m *Message) add(pckt *Packet) {
-	m.Length += len(pckt.Payload)
-	m.LostData += int(pckt.Lost)
-	m.packets = append(m.packets, pckt)
-	if len(pckt.Payload) > 0 {
-		m.buf.Write(pckt.Payload)
+	if m.IsRequest {
+		binary.BigEndian.PutUint32(id[8:], pckt.Ack)
+	} else {
+		binary.BigEndian.PutUint32(id[8:], pckt.Seq)
 	}
-	m.End = pckt.Timestamp
+
+	uuidHex := make([]byte, 24)
+	hex.Encode(uuidHex[:], id[:])
+
+	return uuidHex
 }
 
-// Packets returns packets of this message
+func (m *Message) add(packet *Packet) {
+	// fmt.Println("SEQ:", packet.Seq, " - ", len(packet.Payload))
+
+	// Skip duplicates
+	for _, p := range m.packets {
+		if p.Seq == packet.Seq {
+			return
+		}
+	}
+
+	// Packets not always captured in same Seq order, and sometimes we need to prepend
+	if len(m.packets) == 0 || packet.Seq > m.packets[len(m.packets)-1].Seq {
+		m.packets = append(m.packets, packet)
+	} else if packet.Seq < m.packets[0].Seq {
+		m.packets = append([]*Packet{packet}, m.packets...)
+	} else { // insert somewhere in the middle...
+		for i, p := range m.packets {
+			if packet.Seq < p.Seq {
+				m.packets = append(m.packets[:i], append([]*Packet{packet}, m.packets[i:]...)...)
+				break
+			}
+		}
+	}
+
+	m.Length += len(packet.Payload)
+	m.LostData += int(packet.Lost)
+
+	if packet.Timestamp.After(m.End) || m.End.IsZero() {
+		m.End = packet.Timestamp
+	}
+}
+
+// Packets returns packets of the message
 func (m *Message) Packets() []*Packet {
 	return m.packets
 }
 
+func (m *Message) MissingChunk() bool {
+	nextSeq := m.packets[0].Seq
+
+	for _, p := range m.packets {
+		if p.Seq != nextSeq {
+			return true
+		}
+
+		nextSeq += uint32(len(p.Payload))
+	}
+
+	return false
+}
+
+func (m *Message) PacketData() [][]byte {
+	tmp := make([][]byte, len(m.packets))
+
+	for i, p := range m.packets {
+		tmp[i] = p.Payload
+	}
+
+	return tmp
+}
+
 // Data returns data in this message
 func (m *Message) Data() []byte {
-	return m.buf.Bytes()
+	var totalLen int
+	for _, p := range m.packets {
+		totalLen += len(p.Payload)
+	}
+	tmp := make([]byte, totalLen)
+
+	var i int
+	for _, p := range m.packets {
+		i += copy(tmp[i:], p.Payload)
+	}
+
+	return tmp
+}
+
+// SetProtocolState set feedback/data that can be used later, e.g with End or Start hint
+func (m *Message) SetProtocolState(feedback interface{}) {
+	m.feedback = feedback
+}
+
+// ProtocolState returns feedback associated to this message
+func (m *Message) ProtocolState() interface{} {
+	return m.feedback
 }
 
 // Sort a helper to sort packets
@@ -92,144 +155,202 @@ func (m *Message) Sort() {
 	sort.SliceStable(m.packets, func(i, j int) bool { return m.packets[i].Seq < m.packets[j].Seq })
 }
 
-// Handler message handler
-type Handler func(*Message)
+func (m *Message) Finalize() {
+	// Allow re-use memory
+	for _, p := range m.packets {
+		packetPool.Put(p)
+	}
+}
+
+// Emitter message handler
+type Emitter func(*Message)
 
 // Debugger is the debugger function. first params is the indicator of the issue's priority
 // the higher the number, the lower the priority. it can be 4 <= level <= 6.
 type Debugger func(int, ...interface{})
 
-// HintEnd hints the pool to stop the session, see MessagePool.End
+// HintEnd hints the parser to stop the session, see MessageParser.End
 // when set, it will be executed before checking FIN or RST flag
 type HintEnd func(*Message) bool
 
-// HintStart hints the pool to start the reassembling the message, see MessagePool.Start
-// when set, it will be used instead of checking SYN flag
-type HintStart func(*Packet) (IsIncoming, IsOutgoing bool)
+// HintStart hints the parser to start the reassembling the message, see MessageParser.Start
+// when set, it will be called after checking SYN flag
+type HintStart func(*Packet) (IsRequest, IsOutgoing bool)
 
-// MessagePool holds data of all tcp messages in progress(still receiving/sending packets).
+// MessageParser holds data of all tcp messages in progress(still receiving/sending packets).
 // message is identified by its source port and dst port, and last 4bytes of src IP.
-type MessagePool struct {
-	sync.Mutex
-	debug         Debugger
-	maxSize       size.Size // maximum message size, default 5mb
-	pool          map[uint64]*Message
-	handler       Handler
-	messageExpire time.Duration // the maximum time to wait for the final packet, minimum is 100ms
-	End           HintEnd
-	Start         HintStart
+type MessageParser struct {
+	debug   Debugger
+	maxSize size.Size // maximum message size, default 5mb
+	m       map[uint64]*Message
+
+	messageExpire  time.Duration // the maximum time to wait for the final packet, minimum is 100ms
+	allowIncompete bool
+	End            HintEnd
+	Start          HintStart
+	ticker         *time.Ticker
+	messages       chan *Message
+	packets        chan *Packet
+	close          chan struct{} // to signal that we are able to close
 }
 
-// NewMessagePool returns a new instance of message pool
-func NewMessagePool(maxSize size.Size, messageExpire time.Duration, debugger Debugger, handler Handler) (pool *MessagePool) {
-	pool = new(MessagePool)
-	pool.debug = debugger
-	pool.handler = handler
-	pool.messageExpire = time.Millisecond * 100
-	if pool.messageExpire < messageExpire {
-		pool.messageExpire = messageExpire
+// NewMessageParser returns a new instance of message parser
+func NewMessageParser(maxSize size.Size, messageExpire time.Duration, allowIncompete bool, debugger Debugger) (parser *MessageParser) {
+	parser = new(MessageParser)
+	parser.debug = debugger
+
+	parser.messageExpire = messageExpire
+	if parser.messageExpire == 0 {
+		parser.messageExpire = time.Millisecond * 500
 	}
-	pool.maxSize = maxSize
-	if pool.maxSize < 1 {
-		pool.maxSize = 5 << 20
+
+	parser.allowIncompete = allowIncompete
+	parser.maxSize = maxSize
+	if parser.maxSize < 1 {
+		parser.maxSize = 5 << 20
 	}
-	pool.pool = make(map[uint64]*Message)
-	return pool
+
+	parser.packets = make(chan *Packet, 10000)
+	parser.messages = make(chan *Message, 10000)
+
+	parser.m = make(map[uint64]*Message)
+	parser.ticker = time.NewTicker(time.Millisecond * 50)
+	parser.close = make(chan struct{}, 1)
+	go parser.wait()
+	return parser
 }
 
-// Handler returns packet handler
-func (pool *MessagePool) Handler(packet gopacket.Packet) {
+var packetLen int
+
+// Packet returns packet handler
+func (parser *MessageParser) PacketHandler(packet *Packet) {
+	packetLen++
+	parser.packets <- packet
+}
+
+func (parser *MessageParser) wait() {
+	var (
+		now time.Time
+	)
+	for {
+		select {
+		case pckt := <-parser.packets:
+			parser.processPacket(pckt)
+		case now = <-parser.ticker.C:
+			parser.timer(now)
+		case <-parser.close:
+			parser.ticker.Stop()
+			// parser.Close should wait for this function to return
+			parser.close <- struct{}{}
+			return
+			// default:
+		}
+	}
+}
+
+func (parser *MessageParser) processPacket(pckt *Packet) {
 	var in, out bool
-	pckt, err := ParsePacket(packet)
-	if err != nil || pckt == nil {
-		go pool.say(4, fmt.Sprintf("error decoding packet(%dBytes):%s\n", packet.Metadata().CaptureLength, err))
-		return
-	}
-	pool.Lock()
-	defer pool.Unlock()
-	lst := 3
-	if pckt.Version == 6 {
-		lst = 15
-	}
-	key := uint64(pckt.SrcPort)<<48 | uint64(pckt.DstPort)<<32 |
-		uint64(pckt.SrcIP[lst])<<24 | uint64(pckt.SrcIP[lst-1])<<16 |
-		uint64(pckt.SrcIP[lst-2])<<8 | uint64(pckt.SrcIP[lst-3])
-	m, ok := pool.pool[key]
-	if pckt.RST {
-		if ok {
-			m.done <- true
-			<-m.done
-		}
-		key = uint64(pckt.DstPort)<<48 | uint64(pckt.SrcPort)<<32 |
-			uint64(pckt.DstIP[lst])<<24 | uint64(pckt.DstIP[lst-1])<<16 |
-			uint64(pckt.DstIP[lst-2])<<8 | uint64(pckt.DstIP[lst-3])
-		m, ok = pool.pool[key]
-		if ok {
-			m.done <- true
-			<-m.done
-		}
-		go pool.say(4, fmt.Sprintf("RST flag from %s to %s at %s\n", pckt.Src(), pckt.Dst(), pckt.Timestamp))
-		return
-	}
+
+	// Trying to build unique hash, but there is small chance of collision
+	// No matter if it is request or response, all packets in the same message have same
+	m, ok := parser.m[pckt.MessageID()]
 	switch {
 	case ok:
-		pool.addPacket(m, pckt)
+		parser.addPacket(m, pckt)
 		return
-	case pckt.SYN:
-		in = !pckt.ACK
-	case pool.Start != nil:
-		if in, out = pool.Start(pckt); !(in || out) {
+	case parser.Start != nil:
+		if in, out = parser.Start(pckt); !(in || out) {
+			// Packet can be received out of order, so give it another chance
+			if pckt.Retry < 1 && len(pckt.Payload) > 0 {
+				// Requeue not known packets
+				pckt.Retry++
+
+				select {
+				case parser.packets <- pckt:
+				default:
+					packetPool.Put(pckt)
+				}
+			}
 			return
 		}
 	default:
-		return
+		in = pckt.Incoming
 	}
-	m = NewMessage(pckt.Src(), pckt.Dst(), pckt.Version)
-	m.IsIncoming = in
-	pool.pool[key] = m
+
+	m = new(Message)
+	m.IsRequest = in
+	parser.m[pckt.MessageID()] = m
 	m.Start = pckt.Timestamp
-	go pool.dispatch(key, m)
-	pool.addPacket(m, pckt)
+	m.parser = parser
+	parser.addPacket(m, pckt)
 }
 
-func (pool *MessagePool) dispatch(key uint64, m *Message) {
-	select {
-	case <-m.done:
-		defer func() { m.done <- true }() // signal that message was dispatched
-	case <-time.After(pool.messageExpire):
-		pool.Lock()
-		defer pool.Unlock()
-		// avoid dispathing message twice
-		if _, ok := pool.pool[key]; !ok {
-			return
-		}
-		m.TimedOut = true
-	}
-	delete(pool.pool, key)
-	pool.handler(m)
-}
-
-func (pool *MessagePool) addPacket(m *Message, pckt *Packet) {
-	trunc := m.Length + len(pckt.Payload) - int(pool.maxSize)
+func (parser *MessageParser) addPacket(m *Message, pckt *Packet) {
+	trunc := m.Length + len(pckt.Payload) - int(parser.maxSize)
 	if trunc > 0 {
 		m.Truncated = true
-		pckt.Payload = pckt.Payload[:int(pool.maxSize)-m.Length]
+		pckt.Payload = pckt.Payload[:int(parser.maxSize)-m.Length]
 	}
 	m.add(pckt)
-	switch {
-	case trunc >= 0:
-	case pckt.FIN:
-	case pool.End != nil && pool.End(m):
-	default:
+
+	if trunc > 0 {
 		return
 	}
-	m.done <- true
-	<-m.done
-}
 
-// this function should not block other pool operations
-func (pool *MessagePool) say(level int, args ...interface{}) {
-	if pool.debug != nil {
-		pool.debug(level, args...)
+	// If we are using protocol parsing, like HTTP, depend on its parsing func.
+	// For the binary procols wait for message to expire
+	if parser.End != nil {
+		if parser.End(m) {
+			parser.Emit(m)
+		}
 	}
 }
+
+func (parser *MessageParser) Read() *Message {
+	m := <-parser.messages
+	return m
+}
+
+func (parser *MessageParser) Messages() chan *Message {
+	return parser.messages
+}
+
+func (parser *MessageParser) Emit(m *Message) {
+	delete(parser.m, m.packets[0].MessageID())
+
+	parser.messages <- m
+}
+
+func GetUnexportedField(field reflect.Value) interface{} {
+	return reflect.NewAt(field.Type(), unsafe.Pointer(field.UnsafeAddr())).Elem().Interface()
+}
+
+func (parser *MessageParser) timer(now time.Time) {
+	// fmt.Println(parser.messages.Len(), parser.packets.Len(), packetLen, processedPackets)
+
+	for _, m := range parser.m {
+		if now.Sub(m.End) > parser.messageExpire {
+			m.TimedOut = true
+			if parser.End == nil || parser.allowIncompete {
+				parser.Emit(m)
+			} else {
+				// Just remove
+				delete(parser.m, m.packets[0].MessageID())
+				m.Finalize()
+			}
+		}
+	}
+}
+
+// this function should not block other parser operations
+func (parser *MessageParser) Debug(level int, args ...interface{}) {
+	if parser.debug != nil {
+		parser.debug(level, args...)
+	}
+}
+
+func (parser *MessageParser) Close() error {
+	parser.close <- struct{}{}
+	<-parser.close // wait for timer to be closed!
+	return nil
+}

tcp/tcp_packet.go

@@ -7,9 +7,50 @@ import (
 	"time"
 
 	"github.com/google/gopacket"
-	"github.com/google/gopacket/layers"
 )
 
+func copySlice(b, a []byte) []byte {
+	if cap(b) < len(a) {
+		b = make([]byte, len(a))
+	}
+	copy(b, a)
+	return b[:len(a)]
+}
+
+var packetPool = NewPool(10000)
+
+// Pool holds Clients.
+type Pool struct {
+	pool chan *Packet
+}
+
+// NewPool creates a new pool of Clients.
+func NewPool(max int) *Pool {
+	return &Pool{
+		pool: make(chan *Packet, max),
+	}
+}
+
+// Borrow a Client from the pool.
+func (p *Pool) Get() *Packet {
+	var c *Packet
+	select {
+	case c = <-p.pool:
+	default:
+		c = new(Packet)
+	}
+	return c
+}
+
+// Return returns a Client to the pool.
+func (p *Pool) Put(c *Packet) {
+	select {
+	case p.pool <- c:
+	default:
+		// let it go, let it go...
+	}
+}
+
 /*
 Packet represent data and layers of packet.
 parser extracts information from pcap Packet. functions of *Packet doesn't validate if packet is nil,
@@ -17,82 +58,148 @@ calllers must make sure that ParsePacket has'nt returned any error before callin
 function.
 */
 type Packet struct {
-	// Link layer
-	gopacket.LinkLayer
-
-	// IP Header
-	Version uint8 // Ip version
-	SrcIP   net.IP
-	DstIP   net.IP
-	IHL     uint8
-	Length  uint16
-
-	// TCP Segment Header
-	*layers.TCP
-
-	// Data info
-	Lost      uint16
-	Timestamp time.Time
+	Incoming           bool
+	messageID          uint64
+	SrcIP, DstIP       net.IP
+	Version            uint8
+	SrcPort, DstPort   uint16
+	Ack, Seq           uint32
+	ACK, SYN, FIN, RST bool
+	Lost               uint32
+	Retry              int
+	CaptureLength      int
+	Timestamp          time.Time
+	Payload            []byte
 }
 
 // ParsePacket parse raw packets
-func ParsePacket(packet gopacket.Packet) (pckt *Packet, err error) {
-	// early check of error
-	if packet == nil {
-		return
+func ParsePacket(data []byte, lType, lTypeLen int, cp *gopacket.CaptureInfo, allowEmpty bool) (pckt *Packet, err error) {
+	pckt = packetPool.Get()
+	if err := pckt.parse(data, lType, lTypeLen, cp, allowEmpty); err != nil {
+		packetPool.Put(pckt)
+		return nil, err
 	}
-	defer func() {
-		if packet.ErrorLayer() != nil {
-			err = packet.ErrorLayer().Error()
-			return
+
+	return pckt, nil
+}
+
+func (pckt *Packet) parse(data []byte, lType, lTypeLen int, cp *gopacket.CaptureInfo, allowEmpty bool) error {
+	pckt.Retry = 0
+	pckt.messageID = 0
+
+	// TODO: check resolution
+	pckt.Timestamp = cp.Timestamp
+
+	if len(data) < lTypeLen {
+		return ErrHdrLength("Link")
+	}
+	if len(data) <= lTypeLen {
+		return ErrHdrMissing("IPv4 or IPv6")
+	}
+
+	ldata := data[lTypeLen:]
+	var proto byte
+	var netLayer, transLayer []byte
+
+	if ldata[0]>>4 == 4 {
+		// IPv4 header
+		if len(ldata) < 20 {
+			return ErrHdrLength("IPv4")
 		}
-	}()
-
-	// initialization
-	pckt = new(Packet)
-	pckt.Timestamp = packet.Metadata().Timestamp
-	if pckt.Timestamp.IsZero() {
-		pckt.Timestamp = time.Now()
+		proto = ldata[9]
+		ihl := int(ldata[0]&0x0F) * 4
+		if ihl < 20 {
+			return ErrHdrInvalid("IPv4's IHL")
+		}
+		if len(ldata) < ihl {
+			return ErrHdrLength("IPv4 opts")
+		}
+		netLayer = ldata[:ihl]
+	} else if ldata[0]>>4 == 6 {
+		if len(ldata) < 40 {
+			return ErrHdrLength("IPv6")
+		}
+		proto = ldata[6]
+		totalLen := 40
+		for ipv6ExtensionHdr(proto) {
+			hdr := len(ldata) - totalLen
+			if hdr < 8 {
+				return ErrHdrExpected("IPv6 opts")
+			}
+			extLen := 8
+			if proto != 44 {
+				extLen = int(ldata[totalLen+1]+1) * 8
+			}
+			if hdr < extLen {
+				return ErrHdrLength("IPv6 opts")
+			}
+			proto = ldata[totalLen]
+			totalLen += extLen
+		}
+		netLayer = ldata[:totalLen]
+	} else {
+		return ErrHdrExpected("IPv4 or IPv6")
+	}
+	if proto != 6 {
+		return ErrHdrExpected("TCP")
+	}
+	if len(data) <= len(netLayer) {
+		return ErrHdrMissing("TCP")
+	}
+	ndata := ldata[len(netLayer):]
+	// TCP header
+	if len(ndata) < 20 {
+		return ErrHdrLength("TCP")
+	}
+	dOf := int(ndata[12]>>4) * 4
+	if dOf < 20 {
+		return ErrHdrInvalid("TCP's ndata offset")
+	}
+	if len(ndata) < dOf {
+		return ErrHdrLength("TCP opts")
 	}
 
-	// parsing link layer
-	pckt.LinkLayer = packet.LinkLayer()
+	if !allowEmpty && len(ndata[dOf:]) == 0 {
+		return EmptyPacket("")
+	}
 
-	// parsing network layer
-	if net4, ok := packet.NetworkLayer().(*layers.IPv4); ok {
+	if (netLayer[0] >> 4) == 4 {
+		// IPv4 header
 		pckt.Version = 4
-		pckt.SrcIP = net4.SrcIP
-		pckt.DstIP = net4.DstIP
-		pckt.IHL = net4.IHL * 4
-		pckt.Length = net4.Length
-	} else if net6, ok := packet.NetworkLayer().(*layers.IPv6); ok {
+		pckt.SrcIP = netLayer[12:16]
+		pckt.DstIP = netLayer[16:20]
+	} else {
+		// IPv6 header
 		pckt.Version = 6
-		pckt.SrcIP = net6.SrcIP
-		pckt.DstIP = net6.DstIP
-		pckt.IHL = 40
-		pckt.Length = net6.Length
-	} else {
-		pckt = nil
-		return
+		pckt.SrcIP = netLayer[8:24]
+		pckt.DstIP = netLayer[24:40]
 	}
 
-	// parsing tcp header(transportation layer)
-	if tcp, ok := packet.TransportLayer().(*layers.TCP); ok {
-		pckt.TCP = tcp
-	} else {
-		pckt = nil
-		return
-	}
-	pckt.DataOffset *= 4
+	transLayer = ndata[:dOf]
 
-	// calculating lost data
-	headerSize := int(uint32(pckt.DataOffset) + uint32(pckt.IHL))
-	if pckt.Version == 6 {
-		headerSize -= 40 // in ipv6 the length of payload doesn't include the IPheader size
-	}
-	pckt.Lost = pckt.Length - uint16(headerSize+len(pckt.Payload))
+	pckt.CaptureLength = cp.CaptureLength
+	pckt.SrcPort = binary.BigEndian.Uint16(transLayer[0:2])
+	pckt.DstPort = binary.BigEndian.Uint16(transLayer[2:4])
+	pckt.Seq = binary.BigEndian.Uint32(transLayer[4:8])
+	pckt.Ack = binary.BigEndian.Uint32(transLayer[8:12])
+	pckt.FIN = transLayer[13]&0x01 != 0
+	pckt.SYN = transLayer[13]&0x02 != 0
+	pckt.RST = transLayer[13]&0x04 != 0
+	pckt.ACK = transLayer[13]&0x10 != 0
+	pckt.Lost = uint32(cp.Length - cp.CaptureLength)
+	pckt.Payload = copySlice(pckt.Payload, ndata[dOf:])
 
-	return
+	return nil
+}
+
+func (pckt *Packet) MessageID() uint64 {
+	if pckt.messageID == 0 {
+		// All packets in the same message will share the same ID
+		pckt.messageID = uint64(pckt.SrcPort)<<48 | uint64(pckt.DstPort)<<32 |
+			(uint64(ip2int(pckt.SrcIP)) + uint64(ip2int(pckt.DstIP)) + uint64(pckt.Ack))
+	}
+
+	return pckt.messageID
 }
 
 // Src returns the source socket of a packet
@@ -105,93 +212,53 @@ func (pckt *Packet) Dst() string {
 	return fmt.Sprintf("%s:%d", pckt.DstIP, pckt.DstPort)
 }
 
-// SYNOptions returns MSS and windowscale of syn packets
-func (pckt *Packet) SYNOptions() (mss uint16, windowscale byte) {
-	if !pckt.SYN {
-		return
-	}
-	for _, v := range pckt.Options {
-		if v.OptionType == layers.TCPOptionKindMSS {
-			mss = binary.BigEndian.Uint16(v.OptionData)
-			continue
-		}
-		if v.OptionType == layers.TCPOptionKindWindowScale {
-			if v.OptionLength > 0 {
-				windowscale = v.OptionData[0]
-			}
-		}
-	}
-	return
+type EmptyPacket string
+
+func (err EmptyPacket) Error() string {
+	return "Empty packet"
 }
 
-// LinkInfo returns info about the link layer
-func (pckt *Packet) LinkInfo() string {
-	if l, ok := pckt.LinkLayer.(*layers.Ethernet); ok {
-		return fmt.Sprintf(
-			"Source Mac: %s\nDestination Mac: %s\nProtocol: %s",
-			l.SrcMAC,
-			l.DstMAC,
-			l.EthernetType,
-		)
-	}
-	return "<Not Ethernet>"
+// ErrHdrLength returned on short header length
+type ErrHdrLength string
+
+func (err ErrHdrLength) Error() string {
+	return "short " + string(err) + " length"
 }
 
-// Flag returns formatted tcp flags
-func (pckt *Packet) Flag() (flag string) {
-	if pckt.FIN {
-		flag += "FIN, "
-	}
-	if pckt.SYN {
-		flag += "SYN, "
-	}
-	if pckt.RST {
-		flag += "RST, "
-	}
-	if pckt.PSH {
-		flag += "PSH, "
-	}
-	if pckt.ACK {
-		flag += "ACK, "
-	}
-	if pckt.URG {
-		flag += "URG, "
-	}
-	if len(flag) != 0 {
-		return flag[:len(flag)-2]
-	}
-	return flag
+// ErrHdrMissing returned on missing header(s)
+type ErrHdrMissing string
+
+func (err ErrHdrMissing) Error() string {
+	return "missing " + string(err) + " header(s)"
 }
 
-// String output for a TCP Packet
-func (pckt *Packet) String() string {
-	return fmt.Sprintf(`Time: %s
-%s
-Source: %s
-Destination: %s
-IHL: %d
-Total Length: %d
-Sequence: %d
-Acknowledgment: %d
-DataOffset: %d
-Window: %d
-Flag: %s
-Options: %s
-Data Size: %d
-Lost Data: %d`,
-		pckt.Timestamp.Format(time.StampNano),
-		pckt.LinkInfo(),
-		pckt.Src(),
-		pckt.Dst(),
-		pckt.IHL,
-		pckt.Length,
-		pckt.Seq,
-		pckt.Ack,
-		pckt.DataOffset,
-		pckt.Window,
-		pckt.Flag(),
-		pckt.Options,
-		len(pckt.Payload),
-		pckt.Lost,
-	)
+// ErrHdrExpected returned when header(s) are different from the one expected
+type ErrHdrExpected string
+
+func (err ErrHdrExpected) Error() string {
+	return "expected " + string(err) + " header(s)"
+}
+
+// ErrHdrInvalid returned when header(s) are different from the one expected
+type ErrHdrInvalid string
+
+func (err ErrHdrInvalid) Error() string {
+	return "invalid " + string(err) + " value"
+}
+
+// https://en.wikipedia.org/wiki/IPv6_packet#Extension_headers
+func ipv6ExtensionHdr(b byte) bool {
+	// TODO: support all extension headers
+	return b == 0 || b == 43 || b == 44
+}
+
+func ip2int(ip net.IP) uint32 {
+	if len(ip) == 0 {
+		return 0
+	}
+
+	if len(ip) == 16 {
+		return binary.BigEndian.Uint32(ip[12:16])
+	}
+	return binary.BigEndian.Uint32(ip)
 }

tcp/tcp_test.go

@@ -3,154 +3,210 @@ package tcp
 import (
 	"bytes"
 	"encoding/binary"
+
+	// "runtime"
 	"testing"
 	"time"
 
+	"github.com/stretchr/testify/assert"
+
 	"github.com/buger/goreplay/proto"
 	"github.com/google/gopacket"
 	"github.com/google/gopacket/layers"
 )
 
-var decodeOpts = gopacket.DecodeOptions{Lazy: true, NoCopy: true}
+func generateHeader(request bool, seq uint32, length uint16) []byte {
+	hdr := make([]byte, 4+24+24)
+	binary.BigEndian.PutUint32(hdr, uint32(layers.ProtocolFamilyIPv4))
 
-func headersIP4(seq uint32, length uint16) (headers [54]byte) {
-	// set ethernet headers
-	binary.BigEndian.PutUint16(headers[12:14], uint16(layers.EthernetTypeIPv4))
-
-	// set ip header
-	ip := headers[14:]
-	copy(ip[0:2], []byte{4<<4 | 5, 0x28<<2 | 0x00})
-	binary.BigEndian.PutUint16(ip[2:4], length+40)
+	ip := hdr[4:]
+	ip[0] = 4<<4 | 6
+	binary.BigEndian.PutUint16(ip[2:4], length+24+24)
 	ip[9] = uint8(layers.IPProtocolTCP)
-	copy(ip[12:16], []byte{192, 168, 1, 2})
-	copy(ip[16:], []byte{192, 168, 1, 3})
+	copy(ip[12:16], []byte{127, 0, 0, 1})
+	copy(ip[16:], []byte{127, 0, 0, 1})
 
 	// set tcp header
-	tcp := ip[20:]
-	binary.BigEndian.PutUint16(tcp[0:2], 45678)
-	binary.BigEndian.PutUint16(tcp[2:4], 8001)
-	tcp[12] = 5 << 4
-	return
+	tcp := ip[24:]
+	tcp[12] = 6 << 4
+
+	if request {
+		binary.BigEndian.PutUint16(tcp, 5535)
+		binary.BigEndian.PutUint16(tcp[2:], 8000)
+	} else {
+		binary.BigEndian.PutUint16(tcp, 8000)
+		binary.BigEndian.PutUint16(tcp[2:], 5535)
+	}
+	binary.BigEndian.PutUint32(tcp[4:], seq)
+	return hdr
 }
 
-func GetPackets(start uint32, _len int, payload []byte) []gopacket.Packet {
-	var packets = make([]gopacket.Packet, _len)
+func GetPackets(request bool, start uint32, _len int, payload []byte) []*Packet {
+	var packets = make([]*Packet, _len)
+	var err error
 	for i := start; i < start+uint32(_len); i++ {
-		data := make([]byte, 54+len(payload))
-		h := headersIP4(i, uint16(len(payload)))
-		copy(data, h[:])
-		copy(data[len(h):], payload)
-		packets[i-start] = gopacket.NewPacket(data, layers.LinkTypeEthernet, decodeOpts)
+		d := append(generateHeader(request, i, uint16(len(payload))), payload...)
+		ci := &gopacket.CaptureInfo{Length: len(d), CaptureLength: len(d), Timestamp: time.Now()}
+
+		packets[i-start], err = ParsePacket(d, int(layers.LinkTypeLoop), 4, ci, true)
+		packets[i-start].Incoming = request
+		if err != nil {
+			panic(err)
+		}
 	}
 	return packets
 }
 
-func TestMessageParserWithHint(t *testing.T) {
-	var mssg = make(chan *Message, 3)
-	pool := NewMessagePool(1<<20, time.Second, nil, func(m *Message) { mssg <- m })
-	pool.Start = func(pckt *Packet) (bool, bool) {
+func TestRequestResponseMapping(t *testing.T) {
+	packets := []*Packet{
+		{SrcPort: 60000, DstPort: 80, Ack: 1, Seq: 1, Timestamp: time.Unix(1, 0), Payload: []byte("GET / HTTP/1.1\r\n")},
+		{SrcPort: 60000, DstPort: 80, Ack: 1, Seq: 17, Timestamp: time.Unix(2, 0), Payload: []byte("Host: localhost\r\n\r\n")},
+
+		// Seq of first response packet match Ack of first request packet
+		{SrcPort: 80, DstPort: 60000, Ack: 36, Seq: 1, Timestamp: time.Unix(3, 0), Payload: []byte("HTTP/1.1 200 OK\r\n")},
+		{SrcPort: 80, DstPort: 60000, Ack: 36, Seq: 18, Timestamp: time.Unix(4, 0), Payload: []byte("Content-Length: 0\r\n\r\n")},
+
+		// Same TCP stream
+		{SrcPort: 60000, DstPort: 80, Ack: 39, Seq: 36, Timestamp: time.Unix(5, 0), Payload: []byte("GET / HTTP/1.1\r\n")},
+		{SrcPort: 60000, DstPort: 80, Ack: 39, Seq: 52, Timestamp: time.Unix(6, 0), Payload: []byte("Host: localhost\r\n\r\n")},
+
+		// Seq of first response packet match Ack of first request packet
+		{SrcPort: 80, DstPort: 60000, Ack: 71, Seq: 39, Timestamp: time.Unix(7, 0), Payload: []byte("HTTP/1.1 200 OK\r\n")},
+		{SrcPort: 80, DstPort: 60000, Ack: 71, Seq: 56, Timestamp: time.Unix(8, 0), Payload: []byte("Content-Length: 0\r\n\r\n")},
+	}
+
+	parser := NewMessageParser(1<<20, time.Second, false, nil)
+	parser.Start = func(pckt *Packet) (bool, bool) {
 		return proto.HasRequestTitle(pckt.Payload), proto.HasResponseTitle(pckt.Payload)
 	}
-	pool.End = func(m *Message) bool {
-		return proto.HasFullPayload(m.Data())
+	parser.End = func(m *Message) bool {
+		return proto.HasFullPayload(m, m.Data())
 	}
-	packets := GetPackets(1, 30, nil)
-	packets[0].Data()[14:][20:][13] = 2  // SYN flag
-	packets[10].Data()[14:][20:][13] = 2 // SYN flag
-	packets[29].Data()[14:][20:][13] = 1 // FIN flag
-	packets[4] = GetPackets(5, 1, []byte("HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nTransfer-Encoding: chunked\r\n\r\n7"))[0]
-	packets[5] = GetPackets(6, 1, []byte("\r\nMozilla\r\n9\r\nDeveloper\r"))[0]
-	packets[6] = GetPackets(7, 1, []byte("\n7\r\nNetwork\r\n0\r\n\r\n"))[0]
-	packets[14] = GetPackets(5, 1, []byte("POST / HTTP/1.1\r\nContent-Type: text/plain\r\nContent-Length: 23\r\n\r\n"))[0]
-	packets[15] = GetPackets(6, 1, []byte("MozillaDeveloper"))[0]
-	packets[16] = GetPackets(7, 1, []byte("Network"))[0]
-	packets[24] = GetPackets(5, 1, []byte("HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nContent-Length: 0\r\n\r"))[0]
+
+	for _, packet := range packets {
+		parser.PacketHandler(packet)
+	}
+
+	messages := []*Message{}
+	for i := 0; i < 4; i++ {
+		m := parser.Read()
+		messages = append(messages, m)
+	}
+
+	assert.Equal(t, messages[0].IsRequest, true)
+	assert.Equal(t, messages[1].IsRequest, false)
+	assert.Equal(t, messages[2].IsRequest, true)
+	assert.Equal(t, messages[3].IsRequest, false)
+
+	assert.Equal(t, messages[0].UUID(), messages[1].UUID())
+	assert.Equal(t, messages[2].UUID(), messages[3].UUID())
+
+	assert.NotEqual(t, messages[0].UUID(), messages[2].UUID())
+}
+
+func TestMessageParserWithHint(t *testing.T) {
+	parser := NewMessageParser(1<<20, time.Second, false, nil)
+	parser.Start = func(pckt *Packet) (bool, bool) {
+		return proto.HasRequestTitle(pckt.Payload), proto.HasResponseTitle(pckt.Payload)
+	}
+	parser.End = func(m *Message) bool {
+		return proto.HasFullPayload(m, m.Data())
+	}
+	packets := GetPackets(true, 1, 30, nil)
+	packets[4] = GetPackets(false, 4, 1, []byte("HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nTransfer-Encoding: chunked\r\n\r\n7"))[0]
+	packets[5] = GetPackets(false, 5, 1, []byte("\r\nMozilla\r\n9\r\nDeveloper\r"))[0]
+	packets[6] = GetPackets(false, 6, 1, []byte("\n7\r\nNetwork\r\n0\r\n\r\n"))[0]
+	packets[14] = GetPackets(true, 14, 1, []byte("POST / HTTP/1.1\r\nContent-Type: text/plain\r\nContent-Length: 23\r\n\r\n"))[0]
+	packets[15] = GetPackets(true, 15, 1, []byte("MozillaDeveloper"))[0]
+	packets[16] = GetPackets(true, 16, 1, []byte("Network"))[0]
+	packets[24] = GetPackets(true, 24, 1, []byte("HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nContent-Length: 0\r\n\r\n"))[0]
+
 	for i := 0; i < 30; i++ {
-		pool.Handler(packets[i])
+		parser.PacketHandler(packets[i])
 	}
-	var m *Message
-	select {
-	case <-time.After(time.Second):
-		t.Errorf("can't parse packets fast enough")
-		return
-	case m = <-mssg:
+
+	messages := []*Message{}
+	for i := 0; i < 3; i++ {
+		m := parser.Read()
+		messages = append(messages, m)
 	}
-	if len(m.packets) != 7 {
-		t.Errorf("expected to have 7 packets got %d", len(m.packets))
+
+	if !bytes.HasSuffix(messages[0].Data(), []byte("\n7\r\nNetwork\r\n0\r\n\r\n")) {
+		t.Errorf("expected to %q to have suffix %q", messages[0].Data(), []byte("\n7\r\nNetwork\r\n0\r\n\r\n"))
 	}
+
+	if !bytes.HasSuffix(messages[1].Data(), []byte("Network")) {
+		t.Errorf("expected to %q to have suffix %q", messages[1].Data(), []byte("Network"))
+	}
+
+	if !bytes.HasSuffix(messages[2].Data(), []byte("Content-Length: 0\r\n\r\n")) {
+		t.Errorf("expected to %q to have suffix %q", messages[2].Data(), []byte("Content-Length: 0\r\n\r\n"))
+	}
+}
+
+func TestMessageParserWrongOrder(t *testing.T) {
+	parser := NewMessageParser(1<<20, time.Second, true, nil)
+	parser.Start = func(pckt *Packet) (bool, bool) {
+		return proto.HasRequestTitle(pckt.Payload), proto.HasResponseTitle(pckt.Payload)
+	}
+	parser.End = func(m *Message) bool {
+		return proto.HasFullPayload(m, m.Data())
+	}
+	packets := GetPackets(true, 1, 30, nil)
+	packets[6] = GetPackets(false, 4, 1, []byte("HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nTransfer-Encoding: chunked\r\n\r\n7"))[0]
+	packets[5] = GetPackets(false, 5, 1, []byte("\r\nMozilla\r\n9\r\nDeveloper\r"))[0]
+	packets[4] = GetPackets(false, 6, 1, []byte("\n7\r\nNetwork\r\n0\r\n\r\n"))[0]
+	// Duplicate with same seq
+	packets[7] = GetPackets(false, 6, 1, []byte("\n7\r\nNetwork\r\n0\r\n\r\n"))[0]
+
+	packets[16] = GetPackets(true, 14, 1, []byte("POST / HTTP/1.1\r\nContent-Type: text/plain\r\nContent-Length: 23\r\n\r\n"))[0]
+	packets[15] = GetPackets(true, 15, 1, []byte("MozillaDeveloper"))[0]
+	packets[14] = GetPackets(true, 16, 1, []byte("Network"))[0]
+
+	for i := 0; i < 30; i++ {
+		parser.PacketHandler(packets[i])
+	}
+
+	m := parser.Read()
+
 	if !bytes.HasSuffix(m.Data(), []byte("\n7\r\nNetwork\r\n0\r\n\r\n")) {
 		t.Errorf("expected to %q to have suffix %q", m.Data(), []byte("\n7\r\nNetwork\r\n0\r\n\r\n"))
 	}
 
-	select {
-	case <-time.After(time.Second):
-		t.Errorf("can't parse packets fast enough")
-		return
-	case m = <-mssg:
-	}
-	if len(m.packets) != 7 {
-		t.Errorf("expected to have 7 packets got %d", len(m.packets))
-	}
+	m = parser.Read()
+
 	if !bytes.HasSuffix(m.Data(), []byte("Network")) {
 		t.Errorf("expected to %q to have suffix %q", m.Data(), []byte("Network"))
 	}
-
-	select {
-	case <-time.After(time.Second):
-		t.Errorf("can't parse packets fast enough")
-		return
-	case m = <-mssg:
-	}
-	if len(m.packets) != 6 {
-		t.Errorf("expected to have 6 packets got %d", len(m.packets))
-	}
-	if !bytes.HasSuffix(m.Data(), []byte("Content-Length: 0\r\n\r")) {
-		t.Errorf("expected to %q to have suffix %q", m.Data(), []byte("Content-Length: 0\r\n\r"))
-	}
-
 }
 
 func TestMessageParserWithoutHint(t *testing.T) {
-	var mssg = make(chan *Message, 1)
 	var data [63 << 10]byte
-	packets := GetPackets(1, 10, data[:])
-	packets[0].Data()[14:][20:][13] = 2 // SYN flag
-	packets[9].Data()[14:][20:][13] = 1 // FIN flag
-	p := NewMessagePool(63<<10*10, time.Second, nil, func(m *Message) { mssg <- m })
+	packets := GetPackets(true, 1, 10, data[:])
+
+	p := NewMessageParser(63<<10*10, time.Second, false, nil)
 	for _, v := range packets {
-		p.Handler(v)
-	}
-	var m *Message
-	select {
-	case <-time.After(time.Second):
-		t.Errorf("can't parse packets fast enough")
-		return
-	case m = <-mssg:
+		p.PacketHandler(v)
 	}
+	m := p.Read()
+
 	if m.Length != 63<<10*10 {
 		t.Errorf("expected %d to equal %d", m.Length, 63<<10*10)
 	}
 }
 
 func TestMessageMaxSizeReached(t *testing.T) {
-	var mssg = make(chan *Message, 2)
 	var data [63 << 10]byte
-	packets := GetPackets(1, 2, data[:])
-	packets = append(packets, GetPackets(3, 1, make([]byte, 63<<10+10))...)
-	packets[0].Data()[14:][20:][13] = 2 // SYN flag
-	packets[2].Data()[14:][20:][13] = 2 // SYN flag
-	packets[2].Data()[14:][15] = 3      // changing address
-	p := NewMessagePool(63<<10+10, time.Second, nil, func(m *Message) { mssg <- m })
+	packets := GetPackets(true, 1, 2, data[:])
+	packets = append(packets, GetPackets(false, 1, 1, make([]byte, 63<<10+10))...)
+
+	p := NewMessageParser(63<<10+10, time.Millisecond, false, nil)
 	for _, v := range packets {
-		p.Handler(v)
-	}
-	var m *Message
-	select {
-	case <-time.After(time.Second):
-		t.Errorf("can't parse packets fast enough")
-		return
-	case m = <-mssg:
+		p.PacketHandler(v)
 	}
+	time.Sleep(10 * time.Millisecond)
+
+	m := p.Read()
 	if m.Length != 63<<10+10 {
 		t.Errorf("expected %d to equal %d", m.Length, 63<<10+10)
 	}
@@ -158,12 +214,8 @@ func TestMessageMaxSizeReached(t *testing.T) {
 		t.Error("expected message to be truncated")
 	}
 
-	select {
-	case <-time.After(time.Second):
-		t.Errorf("can't parse packets fast enough")
-		return
-	case m = <-mssg:
-	}
+	m = p.Read()
+
 	if m.Length != 63<<10+10 {
 		t.Errorf("expected %d to equal %d", m.Length, 63<<10+10)
 	}
@@ -173,15 +225,13 @@ func TestMessageMaxSizeReached(t *testing.T) {
 }
 
 func TestMessageTimeoutReached(t *testing.T) {
-	var mssg = make(chan *Message, 2)
 	var data [63 << 10]byte
-	packets := GetPackets(1, 2, data[:])
-	packets[0].Data()[14:][20:][13] = 2 // SYN flag
-	p := NewMessagePool(1<<20, 0, nil, func(m *Message) { mssg <- m })
-	p.Handler(packets[0])
-	time.Sleep(time.Millisecond * 200)
-	p.Handler(packets[1])
-	m := <-mssg
+	packets := GetPackets(true, 1, 2, data[:])
+	p := NewMessageParser(1<<20, 10*time.Millisecond, true, nil)
+	p.PacketHandler(packets[0])
+	time.Sleep(time.Millisecond * 50)
+	p.PacketHandler(packets[1])
+	m := p.Read()
 	if m.Length != 63<<10 {
 		t.Errorf("expected %d to equal %d", m.Length, 63<<10)
 	}
@@ -190,24 +240,29 @@ func TestMessageTimeoutReached(t *testing.T) {
 	}
 }
 
-func TestMessageUUID(t *testing.T) {
-	m1 := &Message{}
-	m1.IsIncoming = true
-	m1.SrcAddr = "src"
-	m1.DstAddr = "dst"
-	m2 := &Message{}
-	m2.SrcAddr = "dst"
-	m2.DstAddr = "src"
-	if string(m1.UUID()) != string(m2.UUID()) {
-		t.Errorf("expected %s, to equal %s", m1.UUID(), m2.UUID())
+func BenchmarkMessageUUID(b *testing.B) {
+	packets := GetPackets(true, 1, 5, nil)
+
+	var uuid []byte
+	parser := NewMessageParser(0, 0, false, nil)
+	for _, p := range packets {
+		parser.PacketHandler(p)
 	}
+
+	msg := parser.Read()
+
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		uuid = msg.UUID()
+	}
+	_ = uuid
 }
 
 func BenchmarkPacketParseAndSort(b *testing.B) {
 	m := new(Message)
 	m.packets = make([]*Packet, 100)
-	for i, v := range GetPackets(1, 100, nil) {
-		m.packets[i], _ = ParsePacket(v)
+	for i, v := range GetPackets(true, 1, 100, nil) {
+		m.packets[i] = v
 	}
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
@@ -216,21 +271,16 @@ func BenchmarkPacketParseAndSort(b *testing.B) {
 }
 
 func BenchmarkMessageParserWithoutHint(b *testing.B) {
-	var mssg = make(chan *Message, 1)
 	var chunk = []byte("111111111111111111111111111111")
-	packets := GetPackets(1, 1000, chunk)
-	packets[0].Data()[14:][20:][13] = 2      // SYN flag
-	packets[1000-1].Data()[14:][20:][13] = 1 // FIN flag
-	p := NewMessagePool(1<<20, time.Second*2, nil, func(m *Message) {
-		mssg <- m
-	})
+	packets := GetPackets(true, 1, 1000, chunk)
+	p := NewMessageParser(1<<20, time.Second*2, false, nil)
 	b.ResetTimer()
 	b.ReportMetric(float64(1000), "packets/op")
 	for i := 0; i < b.N; i++ {
 		for _, v := range packets {
-			p.Handler(v)
+			p.PacketHandler(v)
 		}
-		<-mssg
+		p.Read()
 	}
 }
 
@@ -242,25 +292,33 @@ func BenchmarkMessageParserWithHint(b *testing.B) {
 		buf[i] = chunk
 	}
 	buf[1001] = []byte("0\r\n\r\n")
-	packets := make([]gopacket.Packet, len(buf))
+	packets := make([]*Packet, len(buf))
 	for i := 0; i < len(buf); i++ {
-		packets[i] = GetPackets(uint32(i+10), 1, buf[i])[0]
+		packets[i] = GetPackets(false, 1, 1, buf[i])[0]
 	}
-	var mssg = make(chan *Message, 1)
-	pool := NewMessagePool(1<<30, time.Second*10, nil, func(m *Message) { mssg <- m })
-	pool.Start = func(pckt *Packet) (bool, bool) {
+
+	parser := NewMessageParser(1<<30, time.Second*10, false, nil)
+	parser.Start = func(pckt *Packet) (bool, bool) {
 		return false, proto.HasResponseTitle(pckt.Payload)
 	}
-	pool.End = func(m *Message) bool {
-		return proto.HasFullPayload(m.Data())
+	parser.End = func(m *Message) bool {
+		return proto.HasFullPayload(m, m.Data())
 	}
 	b.ResetTimer()
 	b.ReportMetric(float64(len(packets)), "packets/op")
 	b.ReportMetric(float64(1000), "chunks/op")
 	for i := 0; i < b.N; i++ {
 		for j := range packets {
-			pool.Handler(packets[j])
+			parser.PacketHandler(packets[j])
 		}
-		<-mssg
+		parser.Read()
+	}
+}
+
+func BenchmarkNewAndParsePacket(b *testing.B) {
+	data := append(generateHeader(true, 1024, 10), make([]byte, 10)...)
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		ParsePacket(data, int(layers.LinkTypeLoop), 4, &gopacket.CaptureInfo{}, true)
 	}
 }

test_input.go

@@ -25,22 +25,21 @@ func NewTestInput() (i *TestInput) {
 	return
 }
 
-func (i *TestInput) Read(data []byte) (int, error) {
+// PluginRead reads message from this plugin
+func (i *TestInput) PluginRead() (*Message, error) {
+	var msg Message
 	select {
 	case buf := <-i.data:
-		var header []byte
-
+		msg.Data = buf
 		if !i.skipHeader {
-			header = payloadHeader(RequestPayload, uuid(), time.Now().UnixNano(), -1)
-			copy(data[0:len(header)], header)
-			copy(data[len(header):], buf)
+			msg.Meta = payloadHeader(RequestPayload, uuid(), time.Now().UnixNano(), -1)
 		} else {
-			copy(data, buf)
+			msg.Meta, msg.Data = payloadMetaWithBody(msg.Data)
 		}
 
-		return len(buf) + len(header), nil
+		return &msg, nil
 	case <-i.stop:
-		return 0, ErrorStopped
+		return nil, ErrorStopped
 	}
 }
 

test_output.go

@@ -1,6 +1,6 @@
 package main
 
-type writeCallback func(data []byte)
+type writeCallback func(*Message)
 
 // TestOutput used in testing to intercept any output into callback
 type TestOutput struct {
@@ -8,19 +8,20 @@ type TestOutput struct {
 }
 
 // NewTestOutput constructor for TestOutput, accepts callback which get called on each incoming Write
-func NewTestOutput(cb writeCallback) (i *TestOutput) {
-	i = new(TestOutput)
+func NewTestOutput(cb writeCallback) PluginWriter {
+	i := new(TestOutput)
 	i.cb = cb
 
-	return
+	return i
 }
 
-func (i *TestOutput) Write(data []byte) (int, error) {
-	i.cb(data)
+// PluginWrite write message to this plugin
+func (i *TestOutput) PluginWrite(msg *Message) (int, error) {
+	i.cb(msg)
 
-	return len(data), nil
+	return len(msg.Data) + len(msg.Meta), nil
 }
 
 func (i *TestOutput) String() string {
-	return "Test Input"
+	return "Test Output"
 }

vendor/github.com/Shopify/sarama/.gitignore

@@ -1,27 +0,0 @@
-# Compiled Object files, Static and Dynamic libs (Shared Objects)
-*.o
-*.a
-*.so
-*.test
-
-# Folders
-_obj
-_test
-.vagrant
-
-# Architecture specific extensions/prefixes
-*.[568vq]
-[568vq].out
-
-*.cgo1.go
-*.cgo2.c
-_cgo_defun.c
-_cgo_gotypes.go
-_cgo_export.*
-
-_testmain.go
-
-*.exe
-
-coverage.txt
-profile.out

vendor/github.com/Shopify/sarama/.golangci.yml

@@ -1,74 +0,0 @@
-run:
-  timeout: 5m
-  deadline: 10m
-
-linters-settings:
-  govet:
-    check-shadowing: false
-  golint:
-    min-confidence: 0
-  gocyclo:
-    min-complexity: 99
-  maligned:
-    suggest-new: true
-  dupl:
-    threshold: 100
-  goconst:
-    min-len: 2
-    min-occurrences: 3
-  misspell:
-    locale: US
-  goimports:
-    local-prefixes: github.com/Shopify/sarama
-  gocritic:
-    enabled-tags:
-      - diagnostic
-      - experimental
-      - opinionated
-      - performance
-      - style
-    disabled-checks:
-      - wrapperFunc
-      - ifElseChain
-  funlen:
-    lines: 300
-    statements: 300
-
-linters:
-  disable-all: true
-  enable:
-    - bodyclose
-    - deadcode
-    - depguard
-    - dogsled
-    # - dupl
-    - errcheck
-    - funlen
-    # - gocritic
-    - gocyclo
-    - gofmt
-    - goimports
-    # - golint
-    - gosec
-    # - gosimple
-    - govet
-    # - ineffassign
-    - interfacer
-    # - misspell
-    # - nakedret
-    # - scopelint
-    # - staticcheck
-    - structcheck
-    # - stylecheck
-    - typecheck
-    - unconvert
-    - unused
-    - varcheck
-    - whitespace
-    # - goconst
-    # - gochecknoinits
-
-issues:
-  exclude:
-    - consider giving a name to these results
-    - include an explanation for nolint directive

vendor/github.com/Shopify/sarama/CHANGELOG.md

@@ -1,912 +0,0 @@
-# Changelog
-
-#### Unreleased
-
-#### Version 1.26.1 (2020-02-04)
-
-Improvements:
-- Add requests-in-flight metric ([1539](https://github.com/Shopify/sarama/pull/1539))
-- Fix misleading example for cluster admin ([1595](https://github.com/Shopify/sarama/pull/1595))
-- Replace Travis with GitHub Actions, linters housekeeping ([1573](https://github.com/Shopify/sarama/pull/1573))
-- Allow BalanceStrategy to provide custom assignment data ([1592](https://github.com/Shopify/sarama/pull/1592))
-
-Bug Fixes:
-- Adds back Consumer.Offsets.CommitInterval to fix API ([1590](https://github.com/Shopify/sarama/pull/1590))
-- Fix error message s/CommitInterval/AutoCommit.Interval ([1589](https://github.com/Shopify/sarama/pull/1589))
-
-#### Version 1.26.0 (2020-01-24)
-
-New Features:
-- Enable zstd compression
-  ([1574](https://github.com/Shopify/sarama/pull/1574),
-  [1582](https://github.com/Shopify/sarama/pull/1582))
-- Support headers in tools kafka-console-producer
-  ([1549](https://github.com/Shopify/sarama/pull/1549))
-
-Improvements:
-- Add SASL AuthIdentity to SASL frames (authzid)
-  ([1585](https://github.com/Shopify/sarama/pull/1585)).
-
-Bug Fixes:
-- Sending messages with ZStd compression enabled fails in multiple ways
-  ([1252](https://github.com/Shopify/sarama/issues/1252)).
-- Use the broker for any admin on BrokerConfig
-  ([1571](https://github.com/Shopify/sarama/pull/1571)).
-- Set DescribeConfigRequest Version field
-  ([1576](https://github.com/Shopify/sarama/pull/1576)).
-- ConsumerGroup flooding logs with client/metadata update req
-  ([1578](https://github.com/Shopify/sarama/pull/1578)).
-- MetadataRequest version in DescribeCluster
-  ([1580](https://github.com/Shopify/sarama/pull/1580)).
-- Fix deadlock in consumer group handleError
-  ([1581](https://github.com/Shopify/sarama/pull/1581))
-- Fill in the Fetch{Request,Response} protocol
-  ([1582](https://github.com/Shopify/sarama/pull/1582)).
-- Retry topic request on ControllerNotAvailable
-  ([1586](https://github.com/Shopify/sarama/pull/1586)).
-
-#### Version 1.25.0 (2020-01-13)
-
-New Features:
-- Support TLS protocol in kafka-producer-performance
-  ([1538](https://github.com/Shopify/sarama/pull/1538)).
-- Add support for kafka 2.4.0
-  ([1552](https://github.com/Shopify/sarama/pull/1552)).
-
-Improvements:
-- Allow the Consumer to disable auto-commit offsets
-  ([1164](https://github.com/Shopify/sarama/pull/1164)).
-- Produce records with consistent timestamps
-  ([1455](https://github.com/Shopify/sarama/pull/1455)).
-
-Bug Fixes:
-- Fix incorrect SetTopicMetadata name mentions
-  ([1534](https://github.com/Shopify/sarama/pull/1534)).
-- Fix client.tryRefreshMetadata Println
-  ([1535](https://github.com/Shopify/sarama/pull/1535)).
-- Fix panic on calling updateMetadata on closed client
-  ([1531](https://github.com/Shopify/sarama/pull/1531)).
-- Fix possible faulty metrics in TestFuncProducing
-  ([1545](https://github.com/Shopify/sarama/pull/1545)).
-
-#### Version 1.24.1 (2019-10-31)
-
-New Features:
-- Add DescribeLogDirs Request/Response pair
-  ([1520](https://github.com/Shopify/sarama/pull/1520)).
-
-Bug Fixes:
-- Fix ClusterAdmin returning invalid controller ID on DescribeCluster
-  ([1518](https://github.com/Shopify/sarama/pull/1518)).
-- Fix issue with consumergroup not rebalancing when new partition is added
-  ([1525](https://github.com/Shopify/sarama/pull/1525)).
-- Ensure consistent use of read/write deadlines
-  ([1529](https://github.com/Shopify/sarama/pull/1529)).
-
-#### Version 1.24.0 (2019-10-09)
-
-New Features:
-- Add sticky partition assignor
-  ([1416](https://github.com/Shopify/sarama/pull/1416)).
-- Switch from cgo zstd package to pure Go implementation
-  ([1477](https://github.com/Shopify/sarama/pull/1477)).
-
-Improvements:
-- Allow creating ClusterAdmin from client
-  ([1415](https://github.com/Shopify/sarama/pull/1415)).
-- Set KafkaVersion in ListAcls method
-  ([1452](https://github.com/Shopify/sarama/pull/1452)).
-- Set request version in CreateACL ClusterAdmin method
-  ([1458](https://github.com/Shopify/sarama/pull/1458)).
-- Set request version in DeleteACL ClusterAdmin method
-  ([1461](https://github.com/Shopify/sarama/pull/1461)).
-- Handle missed error codes on TopicMetaDataRequest and GroupCoordinatorRequest
-  ([1464](https://github.com/Shopify/sarama/pull/1464)).
-- Remove direct usage of gofork
-  ([1465](https://github.com/Shopify/sarama/pull/1465)).
-- Add support for Go 1.13
-  ([1478](https://github.com/Shopify/sarama/pull/1478)).
-- Improve behavior of NewMockListAclsResponse
-  ([1481](https://github.com/Shopify/sarama/pull/1481)).
-
-Bug Fixes:
-- Fix race condition in consumergroup example
-  ([1434](https://github.com/Shopify/sarama/pull/1434)).
-- Fix brokerProducer goroutine leak
-  ([1442](https://github.com/Shopify/sarama/pull/1442)).
-- Use released version of lz4 library
-  ([1469](https://github.com/Shopify/sarama/pull/1469)).
-- Set correct version in MockDeleteTopicsResponse
-  ([1484](https://github.com/Shopify/sarama/pull/1484)).
-- Fix CLI help message typo
-  ([1494](https://github.com/Shopify/sarama/pull/1494)).
-
-Known Issues:
-- Please **don't** use Zstd, as it doesn't work right now.
-  See https://github.com/Shopify/sarama/issues/1252
-
-#### Version 1.23.1 (2019-07-22)
-
-Bug Fixes:
-- Fix fetch delete bug record
-  ([1425](https://github.com/Shopify/sarama/pull/1425)).
-- Handle SASL/OAUTHBEARER token rejection
-  ([1428](https://github.com/Shopify/sarama/pull/1428)).
-
-#### Version 1.23.0 (2019-07-02)
-
-New Features:
-- Add support for Kafka 2.3.0
-  ([1418](https://github.com/Shopify/sarama/pull/1418)).
-- Add support for ListConsumerGroupOffsets v2
-  ([1374](https://github.com/Shopify/sarama/pull/1374)).
-- Add support for DeleteConsumerGroup
-  ([1417](https://github.com/Shopify/sarama/pull/1417)).
-- Add support for SASLVersion configuration
-  ([1410](https://github.com/Shopify/sarama/pull/1410)).
-- Add kerberos support
-  ([1366](https://github.com/Shopify/sarama/pull/1366)).
-
-Improvements:
-- Improve sasl_scram_client example
-  ([1406](https://github.com/Shopify/sarama/pull/1406)).
-- Fix shutdown and race-condition in consumer-group example
-  ([1404](https://github.com/Shopify/sarama/pull/1404)).
-- Add support for error codes 77—81
-  ([1397](https://github.com/Shopify/sarama/pull/1397)).
-- Pool internal objects allocated per message
-  ([1385](https://github.com/Shopify/sarama/pull/1385)).
-- Reduce packet decoder allocations
-  ([1373](https://github.com/Shopify/sarama/pull/1373)).
-- Support timeout when fetching metadata
-  ([1359](https://github.com/Shopify/sarama/pull/1359)).
-
-Bug Fixes:
-- Fix fetch size integer overflow
-  ([1376](https://github.com/Shopify/sarama/pull/1376)).
-- Handle and log throttled FetchResponses
-  ([1383](https://github.com/Shopify/sarama/pull/1383)).
-- Refactor misspelled word Resouce to Resource
-  ([1368](https://github.com/Shopify/sarama/pull/1368)).
-
-#### Version 1.22.1 (2019-04-29)
-
-Improvements:
-- Use zstd 1.3.8
-  ([1350](https://github.com/Shopify/sarama/pull/1350)).
-- Add support for SaslHandshakeRequest v1
-  ([1354](https://github.com/Shopify/sarama/pull/1354)).
-
-Bug Fixes:
-- Fix V5 MetadataRequest nullable topics array
-  ([1353](https://github.com/Shopify/sarama/pull/1353)).
-- Use a different SCRAM client for each broker connection
-  ([1349](https://github.com/Shopify/sarama/pull/1349)).
-- Fix AllowAutoTopicCreation for MetadataRequest greater than v3
-  ([1344](https://github.com/Shopify/sarama/pull/1344)).
-
-#### Version 1.22.0 (2019-04-09)
-
-New Features:
-- Add Offline Replicas Operation to Client
-  ([1318](https://github.com/Shopify/sarama/pull/1318)).
-- Allow using proxy when connecting to broker
-  ([1326](https://github.com/Shopify/sarama/pull/1326)).
-- Implement ReadCommitted
-  ([1307](https://github.com/Shopify/sarama/pull/1307)).
-- Add support for Kafka 2.2.0
-  ([1331](https://github.com/Shopify/sarama/pull/1331)).
-- Add SASL SCRAM-SHA-512 and SCRAM-SHA-256 mechanismes
-  ([1331](https://github.com/Shopify/sarama/pull/1295)).
-
-Improvements:
-- Unregister all broker metrics on broker stop
-  ([1232](https://github.com/Shopify/sarama/pull/1232)).
-- Add SCRAM authentication example
-  ([1303](https://github.com/Shopify/sarama/pull/1303)).
-- Add consumergroup examples
-  ([1304](https://github.com/Shopify/sarama/pull/1304)).
-- Expose consumer batch size metric
-  ([1296](https://github.com/Shopify/sarama/pull/1296)).
-- Add TLS options to console producer and consumer
-  ([1300](https://github.com/Shopify/sarama/pull/1300)).
-- Reduce client close bookkeeping
-  ([1297](https://github.com/Shopify/sarama/pull/1297)).
-- Satisfy error interface in create responses
-  ([1154](https://github.com/Shopify/sarama/pull/1154)).
-- Please lint gods
-  ([1346](https://github.com/Shopify/sarama/pull/1346)).
-
-Bug Fixes:
-- Fix multi consumer group instance crash
-  ([1338](https://github.com/Shopify/sarama/pull/1338)).
-- Update lz4 to latest version
-  ([1347](https://github.com/Shopify/sarama/pull/1347)).
-- Retry ErrNotCoordinatorForConsumer in new consumergroup session
-  ([1231](https://github.com/Shopify/sarama/pull/1231)).
-- Fix cleanup error handler
-  ([1332](https://github.com/Shopify/sarama/pull/1332)).
-- Fix rate condition in PartitionConsumer
-  ([1156](https://github.com/Shopify/sarama/pull/1156)).
-
-#### Version 1.21.0 (2019-02-24)
-
-New Features:
-- Add CreateAclRequest, DescribeAclRequest, DeleteAclRequest
-  ([1236](https://github.com/Shopify/sarama/pull/1236)).
-- Add DescribeTopic, DescribeConsumerGroup, ListConsumerGroups, ListConsumerGroupOffsets admin requests
-  ([1178](https://github.com/Shopify/sarama/pull/1178)).
-- Implement SASL/OAUTHBEARER
-  ([1240](https://github.com/Shopify/sarama/pull/1240)).
-
-Improvements:
-- Add Go mod support
-  ([1282](https://github.com/Shopify/sarama/pull/1282)).
-- Add error codes 73—76
-  ([1239](https://github.com/Shopify/sarama/pull/1239)).
-- Add retry backoff function
-  ([1160](https://github.com/Shopify/sarama/pull/1160)).
-- Maintain metadata in the producer even when retries are disabled
-  ([1189](https://github.com/Shopify/sarama/pull/1189)).
-- Include ReplicaAssignment in ListTopics
-  ([1274](https://github.com/Shopify/sarama/pull/1274)).
-- Add producer performance tool
-  ([1222](https://github.com/Shopify/sarama/pull/1222)).
-- Add support LogAppend timestamps
-  ([1258](https://github.com/Shopify/sarama/pull/1258)).
-
-Bug Fixes:
-- Fix potential deadlock when a heartbeat request fails
-  ([1286](https://github.com/Shopify/sarama/pull/1286)).
-- Fix consuming compacted topic
-  ([1227](https://github.com/Shopify/sarama/pull/1227)).
-- Set correct Kafka version for DescribeConfigsRequest v1
-  ([1277](https://github.com/Shopify/sarama/pull/1277)).
-- Update kafka test version
-  ([1273](https://github.com/Shopify/sarama/pull/1273)).
-
-#### Version 1.20.1 (2019-01-10)
-
-New Features:
-- Add optional replica id in offset request
-  ([1100](https://github.com/Shopify/sarama/pull/1100)).
-
-Improvements:
-- Implement DescribeConfigs Request + Response v1 & v2
-  ([1230](https://github.com/Shopify/sarama/pull/1230)).
-- Reuse compression objects
-  ([1185](https://github.com/Shopify/sarama/pull/1185)).
-- Switch from png to svg for GoDoc link in README
-  ([1243](https://github.com/Shopify/sarama/pull/1243)).
-- Fix typo in deprecation notice for FetchResponseBlock.Records
-  ([1242](https://github.com/Shopify/sarama/pull/1242)).
-- Fix typos in consumer metadata response file
-  ([1244](https://github.com/Shopify/sarama/pull/1244)).
-
-Bug Fixes:
-- Revert to individual msg retries for non-idempotent
-  ([1203](https://github.com/Shopify/sarama/pull/1203)).
-- Respect MaxMessageBytes limit for uncompressed messages
-  ([1141](https://github.com/Shopify/sarama/pull/1141)).
-
-#### Version 1.20.0 (2018-12-10)
-
-New Features:
- - Add support for zstd compression
-   ([#1170](https://github.com/Shopify/sarama/pull/1170)).
- - Add support for Idempotent Producer
-   ([#1152](https://github.com/Shopify/sarama/pull/1152)).
- - Add support support for Kafka 2.1.0
-   ([#1229](https://github.com/Shopify/sarama/pull/1229)).
- - Add support support for OffsetCommit request/response pairs versions v1 to v5
-   ([#1201](https://github.com/Shopify/sarama/pull/1201)).
- - Add support support for OffsetFetch request/response pair up to version v5
-   ([#1198](https://github.com/Shopify/sarama/pull/1198)).
-
-Improvements:
- - Export broker's Rack setting
-   ([#1173](https://github.com/Shopify/sarama/pull/1173)).
- - Always use latest patch version of Go on CI
-   ([#1202](https://github.com/Shopify/sarama/pull/1202)).
- - Add error codes 61 to 72
-   ([#1195](https://github.com/Shopify/sarama/pull/1195)).
-
-Bug Fixes:
- - Fix build without cgo
-   ([#1182](https://github.com/Shopify/sarama/pull/1182)).
- - Fix go vet suggestion in consumer group file
-   ([#1209](https://github.com/Shopify/sarama/pull/1209)).
- - Fix typos in code and comments
-   ([#1228](https://github.com/Shopify/sarama/pull/1228)).
-
-#### Version 1.19.0 (2018-09-27)
-
-New Features:
- - Implement a higher-level consumer group
-   ([#1099](https://github.com/Shopify/sarama/pull/1099)).
-
-Improvements:
- - Add support for Go 1.11
-   ([#1176](https://github.com/Shopify/sarama/pull/1176)).
-
-Bug Fixes:
- - Fix encoding of `MetadataResponse` with version 2 and higher
-   ([#1174](https://github.com/Shopify/sarama/pull/1174)).
- - Fix race condition in mock async producer
-   ([#1174](https://github.com/Shopify/sarama/pull/1174)).
-
-#### Version 1.18.0 (2018-09-07)
-
-New Features:
- - Make `Partitioner.RequiresConsistency` vary per-message
-   ([#1112](https://github.com/Shopify/sarama/pull/1112)).
- - Add customizable partitioner
-   ([#1118](https://github.com/Shopify/sarama/pull/1118)).
- - Add `ClusterAdmin` support for `CreateTopic`, `DeleteTopic`, `CreatePartitions`,
-   `DeleteRecords`, `DescribeConfig`, `AlterConfig`, `CreateACL`, `ListAcls`, `DeleteACL`
-   ([#1055](https://github.com/Shopify/sarama/pull/1055)).
-
-Improvements:
- - Add support for Kafka 2.0.0
-   ([#1149](https://github.com/Shopify/sarama/pull/1149)).
- - Allow setting `LocalAddr` when dialing an address to support multi-homed hosts
-   ([#1123](https://github.com/Shopify/sarama/pull/1123)).
- - Simpler offset management
-   ([#1127](https://github.com/Shopify/sarama/pull/1127)).
-
-Bug Fixes:
- - Fix mutation of `ProducerMessage.MetaData` when producing to Kafka
-   ([#1110](https://github.com/Shopify/sarama/pull/1110)).
- - Fix consumer block when response did not contain all the
-   expected topic/partition blocks
-   ([#1086](https://github.com/Shopify/sarama/pull/1086)).
- - Fix consumer block when response contains only constrol messages
-   ([#1115](https://github.com/Shopify/sarama/pull/1115)).
- - Add timeout config for ClusterAdmin requests
-   ([#1142](https://github.com/Shopify/sarama/pull/1142)).
- - Add version check when producing message with headers
-   ([#1117](https://github.com/Shopify/sarama/pull/1117)).
- - Fix `MetadataRequest` for empty list of topics
-   ([#1132](https://github.com/Shopify/sarama/pull/1132)).
- - Fix producer topic metadata on-demand fetch when topic error happens in metadata response
-   ([#1125](https://github.com/Shopify/sarama/pull/1125)).
-
-#### Version 1.17.0 (2018-05-30)
-
-New Features:
- - Add support for gzip compression levels
-   ([#1044](https://github.com/Shopify/sarama/pull/1044)).
- - Add support for Metadata request/response pairs versions v1 to v5
-   ([#1047](https://github.com/Shopify/sarama/pull/1047),
-    [#1069](https://github.com/Shopify/sarama/pull/1069)).
- - Add versioning to JoinGroup request/response pairs
-   ([#1098](https://github.com/Shopify/sarama/pull/1098))
- - Add support for CreatePartitions, DeleteGroups, DeleteRecords request/response pairs
-   ([#1065](https://github.com/Shopify/sarama/pull/1065),
-    [#1096](https://github.com/Shopify/sarama/pull/1096),
-    [#1027](https://github.com/Shopify/sarama/pull/1027)).
- - Add `Controller()` method to Client interface
-   ([#1063](https://github.com/Shopify/sarama/pull/1063)).
-
-Improvements:
- - ConsumerMetadataReq/Resp has been migrated to FindCoordinatorReq/Resp
-   ([#1010](https://github.com/Shopify/sarama/pull/1010)).
- - Expose missing protocol parts: `msgSet` and `recordBatch`
-   ([#1049](https://github.com/Shopify/sarama/pull/1049)).
- - Add support for v1 DeleteTopics Request
-   ([#1052](https://github.com/Shopify/sarama/pull/1052)).
- - Add support for Go 1.10
-   ([#1064](https://github.com/Shopify/sarama/pull/1064)).
- - Claim support for Kafka 1.1.0
-   ([#1073](https://github.com/Shopify/sarama/pull/1073)).
-
-Bug Fixes:
- - Fix FindCoordinatorResponse.encode to allow nil Coordinator
-   ([#1050](https://github.com/Shopify/sarama/pull/1050),
-    [#1051](https://github.com/Shopify/sarama/pull/1051)).
- - Clear all metadata when we have the latest topic info
-   ([#1033](https://github.com/Shopify/sarama/pull/1033)).
- - Make `PartitionConsumer.Close` idempotent
-   ([#1092](https://github.com/Shopify/sarama/pull/1092)).
-
-#### Version 1.16.0 (2018-02-12)
-
-New Features:
- - Add support for the Create/Delete Topics request/response pairs
-   ([#1007](https://github.com/Shopify/sarama/pull/1007),
-    [#1008](https://github.com/Shopify/sarama/pull/1008)).
- - Add support for the Describe/Create/Delete ACL request/response pairs
-   ([#1009](https://github.com/Shopify/sarama/pull/1009)).
- - Add support for the five transaction-related request/response pairs
-   ([#1016](https://github.com/Shopify/sarama/pull/1016)).
-
-Improvements:
- - Permit setting version on mock producer responses
-   ([#999](https://github.com/Shopify/sarama/pull/999)).
- - Add `NewMockBrokerListener` helper for testing TLS connections
-   ([#1019](https://github.com/Shopify/sarama/pull/1019)).
- - Changed the default value for `Consumer.Fetch.Default` from 32KiB to 1MiB
-   which results in much higher throughput in most cases
-   ([#1024](https://github.com/Shopify/sarama/pull/1024)).
- - Reuse the `time.Ticker` across fetch requests in the PartitionConsumer to
-   reduce CPU and memory usage when processing many partitions
-   ([#1028](https://github.com/Shopify/sarama/pull/1028)).
- - Assign relative offsets to messages in the producer to save the brokers a
-   recompression pass
-   ([#1002](https://github.com/Shopify/sarama/pull/1002),
-    [#1015](https://github.com/Shopify/sarama/pull/1015)).
-
-Bug Fixes:
- - Fix producing uncompressed batches with the new protocol format
-   ([#1032](https://github.com/Shopify/sarama/issues/1032)).
- - Fix consuming compacted topics with the new protocol format
-   ([#1005](https://github.com/Shopify/sarama/issues/1005)).
- - Fix consuming topics with a mix of protocol formats
-   ([#1021](https://github.com/Shopify/sarama/issues/1021)).
- - Fix consuming when the broker includes multiple batches in a single response
-   ([#1022](https://github.com/Shopify/sarama/issues/1022)).
- - Fix detection of `PartialTrailingMessage` when the partial message was
-   truncated before the magic value indicating its version
-   ([#1030](https://github.com/Shopify/sarama/pull/1030)).
- - Fix expectation-checking in the mock of `SyncProducer.SendMessages`
-   ([#1035](https://github.com/Shopify/sarama/pull/1035)).
-
-#### Version 1.15.0 (2017-12-08)
-
-New Features:
- - Claim official support for Kafka 1.0, though it did already work
-   ([#984](https://github.com/Shopify/sarama/pull/984)).
- - Helper methods for Kafka version numbers to/from strings
-   ([#989](https://github.com/Shopify/sarama/pull/989)).
- - Implement CreatePartitions request/response
-   ([#985](https://github.com/Shopify/sarama/pull/985)).
-
-Improvements:
- - Add error codes 45-60
-   ([#986](https://github.com/Shopify/sarama/issues/986)).
-
-Bug Fixes:
- - Fix slow consuming for certain Kafka 0.11/1.0 configurations
-   ([#982](https://github.com/Shopify/sarama/pull/982)).
- - Correctly determine when a FetchResponse contains the new message format
-   ([#990](https://github.com/Shopify/sarama/pull/990)).
- - Fix producing with multiple headers
-   ([#996](https://github.com/Shopify/sarama/pull/996)).
- - Fix handling of truncated record batches
-   ([#998](https://github.com/Shopify/sarama/pull/998)).
- - Fix leaking metrics when closing brokers
-   ([#991](https://github.com/Shopify/sarama/pull/991)).
-
-#### Version 1.14.0 (2017-11-13)
-
-New Features:
- - Add support for the new Kafka 0.11 record-batch format, including the wire
-   protocol and the necessary behavioural changes in the producer and consumer.
-   Transactions and idempotency are not yet supported, but producing and
-   consuming should work with all the existing bells and whistles (batching,
-   compression, etc) as well as the new custom headers. Thanks to Vlad Hanciuta
-   of Arista Networks for this work. Part of
-   ([#901](https://github.com/Shopify/sarama/issues/901)).
-
-Bug Fixes:
- - Fix encoding of ProduceResponse versions in test
-   ([#970](https://github.com/Shopify/sarama/pull/970)).
- - Return partial replicas list when we have it
-   ([#975](https://github.com/Shopify/sarama/pull/975)).
-
-#### Version 1.13.0 (2017-10-04)
-
-New Features:
- - Support for FetchRequest version 3
-   ([#905](https://github.com/Shopify/sarama/pull/905)).
- - Permit setting version on mock FetchResponses
-   ([#939](https://github.com/Shopify/sarama/pull/939)).
- - Add a configuration option to support storing only minimal metadata for
-   extremely large clusters
-   ([#937](https://github.com/Shopify/sarama/pull/937)).
- - Add `PartitionOffsetManager.ResetOffset` for backtracking tracked offsets
-   ([#932](https://github.com/Shopify/sarama/pull/932)).
-
-Improvements:
- - Provide the block-level timestamp when consuming compressed messages
-   ([#885](https://github.com/Shopify/sarama/issues/885)).
- - `Client.Replicas` and `Client.InSyncReplicas` now respect the order returned
-   by the broker, which can be meaningful
-   ([#930](https://github.com/Shopify/sarama/pull/930)).
- - Use a `Ticker` to reduce consumer timer overhead at the cost of higher
-   variance in the actual timeout
-   ([#933](https://github.com/Shopify/sarama/pull/933)).
-
-Bug Fixes:
- - Gracefully handle messages with negative timestamps
-   ([#907](https://github.com/Shopify/sarama/pull/907)).
- - Raise a proper error when encountering an unknown message version
-   ([#940](https://github.com/Shopify/sarama/pull/940)).
-
-#### Version 1.12.0 (2017-05-08)
-
-New Features:
- - Added support for the `ApiVersions` request and response pair, and Kafka
-   version 0.10.2 ([#867](https://github.com/Shopify/sarama/pull/867)). Note
-   that you still need to specify the Kafka version in the Sarama configuration
-   for the time being.
- - Added a `Brokers` method to the Client which returns the complete set of
-   active brokers ([#813](https://github.com/Shopify/sarama/pull/813)).
- - Added an `InSyncReplicas` method to the Client which returns the set of all
-   in-sync broker IDs for the given partition, now that the Kafka versions for
-   which this was misleading are no longer in our supported set
-   ([#872](https://github.com/Shopify/sarama/pull/872)).
- - Added a `NewCustomHashPartitioner` method which allows constructing a hash
-   partitioner with a custom hash method in case the default (FNV-1a) is not
-   suitable
-   ([#837](https://github.com/Shopify/sarama/pull/837),
-    [#841](https://github.com/Shopify/sarama/pull/841)).
-
-Improvements:
- - Recognize more Kafka error codes
-   ([#859](https://github.com/Shopify/sarama/pull/859)).
-
-Bug Fixes:
- - Fix an issue where decoding a malformed FetchRequest would not return the
-   correct error ([#818](https://github.com/Shopify/sarama/pull/818)).
- - Respect ordering of group protocols in JoinGroupRequests. This fix is
-   transparent if you're using the `AddGroupProtocol` or
-   `AddGroupProtocolMetadata` helpers; otherwise you will need to switch from
-   the `GroupProtocols` field (now deprecated) to use `OrderedGroupProtocols`
-   ([#812](https://github.com/Shopify/sarama/issues/812)).
- - Fix an alignment-related issue with atomics on 32-bit architectures
-   ([#859](https://github.com/Shopify/sarama/pull/859)).
-
-#### Version 1.11.0 (2016-12-20)
-
-_Important:_ As of Sarama 1.11 it is necessary to set the config value of
-`Producer.Return.Successes` to true in order to use the SyncProducer. Previous
-versions would silently override this value when instantiating a SyncProducer
-which led to unexpected values and data races.
-
-New Features:
- - Metrics! Thanks to Sébastien Launay for all his work on this feature
-   ([#701](https://github.com/Shopify/sarama/pull/701),
-    [#746](https://github.com/Shopify/sarama/pull/746),
-    [#766](https://github.com/Shopify/sarama/pull/766)).
- - Add support for LZ4 compression
-   ([#786](https://github.com/Shopify/sarama/pull/786)).
- - Add support for ListOffsetRequest v1 and Kafka 0.10.1
-   ([#775](https://github.com/Shopify/sarama/pull/775)).
- - Added a `HighWaterMarks` method to the Consumer which aggregates the
-   `HighWaterMarkOffset` values of its child topic/partitions
-   ([#769](https://github.com/Shopify/sarama/pull/769)).
-
-Bug Fixes:
- - Fixed producing when using timestamps, compression and Kafka 0.10
-   ([#759](https://github.com/Shopify/sarama/pull/759)).
- - Added missing decoder methods to DescribeGroups response
-   ([#756](https://github.com/Shopify/sarama/pull/756)).
- - Fix producer shutdown when `Return.Errors` is disabled
-   ([#787](https://github.com/Shopify/sarama/pull/787)).
- - Don't mutate configuration in SyncProducer
-   ([#790](https://github.com/Shopify/sarama/pull/790)).
- - Fix crash on SASL initialization failure
-   ([#795](https://github.com/Shopify/sarama/pull/795)).
-
-#### Version 1.10.1 (2016-08-30)
-
-Bug Fixes:
- - Fix the documentation for `HashPartitioner` which was incorrect
-   ([#717](https://github.com/Shopify/sarama/pull/717)).
- - Permit client creation even when it is limited by ACLs
-   ([#722](https://github.com/Shopify/sarama/pull/722)).
- - Several fixes to the consumer timer optimization code, regressions introduced
-   in v1.10.0. Go's timers are finicky
-   ([#730](https://github.com/Shopify/sarama/pull/730),
-    [#733](https://github.com/Shopify/sarama/pull/733),
-    [#734](https://github.com/Shopify/sarama/pull/734)).
- - Handle consuming compressed relative offsets with Kafka 0.10
-   ([#735](https://github.com/Shopify/sarama/pull/735)).
-
-#### Version 1.10.0 (2016-08-02)
-
-_Important:_ As of Sarama 1.10 it is necessary to tell Sarama the version of
-Kafka you are running against (via the `config.Version` value) in order to use
-features that may not be compatible with old Kafka versions. If you don't
-specify this value it will default to 0.8.2 (the minimum supported), and trying
-to use more recent features (like the offset manager) will fail with an error.
-
-_Also:_ The offset-manager's behaviour has been changed to match the upstream
-java consumer (see [#705](https://github.com/Shopify/sarama/pull/705) and
-[#713](https://github.com/Shopify/sarama/pull/713)). If you use the
-offset-manager, please ensure that you are committing one *greater* than the
-last consumed message offset or else you may end up consuming duplicate
-messages.
-
-New Features:
- - Support for Kafka 0.10
-   ([#672](https://github.com/Shopify/sarama/pull/672),
-    [#678](https://github.com/Shopify/sarama/pull/678),
-    [#681](https://github.com/Shopify/sarama/pull/681), and others).
- - Support for configuring the target Kafka version
-   ([#676](https://github.com/Shopify/sarama/pull/676)).
- - Batch producing support in the SyncProducer
-   ([#677](https://github.com/Shopify/sarama/pull/677)).
- - Extend producer mock to allow setting expectations on message contents
-   ([#667](https://github.com/Shopify/sarama/pull/667)).
-
-Improvements:
- - Support `nil` compressed messages for deleting in compacted topics
-   ([#634](https://github.com/Shopify/sarama/pull/634)).
- - Pre-allocate decoding errors, greatly reducing heap usage and GC time against
-   misbehaving brokers ([#690](https://github.com/Shopify/sarama/pull/690)).
- - Re-use consumer expiry timers, removing one allocation per consumed message
-   ([#707](https://github.com/Shopify/sarama/pull/707)).
-
-Bug Fixes:
- - Actually default the client ID to "sarama" like we say we do
-   ([#664](https://github.com/Shopify/sarama/pull/664)).
- - Fix a rare issue where `Client.Leader` could return the wrong error
-   ([#685](https://github.com/Shopify/sarama/pull/685)).
- - Fix a possible tight loop in the consumer
-   ([#693](https://github.com/Shopify/sarama/pull/693)).
- - Match upstream's offset-tracking behaviour
-   ([#705](https://github.com/Shopify/sarama/pull/705)).
- - Report UnknownTopicOrPartition errors from the offset manager
-   ([#706](https://github.com/Shopify/sarama/pull/706)).
- - Fix possible negative partition value from the HashPartitioner
-   ([#709](https://github.com/Shopify/sarama/pull/709)).
-
-#### Version 1.9.0 (2016-05-16)
-
-New Features:
- - Add support for custom offset manager retention durations
-   ([#602](https://github.com/Shopify/sarama/pull/602)).
- - Publish low-level mocks to enable testing of third-party producer/consumer
-   implementations ([#570](https://github.com/Shopify/sarama/pull/570)).
- - Declare support for Golang 1.6
-   ([#611](https://github.com/Shopify/sarama/pull/611)).
- - Support for SASL plain-text auth
-   ([#648](https://github.com/Shopify/sarama/pull/648)).
-
-Improvements:
- - Simplified broker locking scheme slightly
-   ([#604](https://github.com/Shopify/sarama/pull/604)).
- - Documentation cleanup
-   ([#605](https://github.com/Shopify/sarama/pull/605),
-    [#621](https://github.com/Shopify/sarama/pull/621),
-    [#654](https://github.com/Shopify/sarama/pull/654)).
-
-Bug Fixes:
- - Fix race condition shutting down the OffsetManager
-   ([#658](https://github.com/Shopify/sarama/pull/658)).
-
-#### Version 1.8.0 (2016-02-01)
-
-New Features:
- - Full support for Kafka 0.9:
-   - All protocol messages and fields
-   ([#586](https://github.com/Shopify/sarama/pull/586),
-   [#588](https://github.com/Shopify/sarama/pull/588),
-   [#590](https://github.com/Shopify/sarama/pull/590)).
-   - Verified that TLS support works
-   ([#581](https://github.com/Shopify/sarama/pull/581)).
-   - Fixed the OffsetManager compatibility
-   ([#585](https://github.com/Shopify/sarama/pull/585)).
-
-Improvements:
- - Optimize for fewer system calls when reading from the network
-   ([#584](https://github.com/Shopify/sarama/pull/584)).
- - Automatically retry `InvalidMessage` errors to match upstream behaviour
-   ([#589](https://github.com/Shopify/sarama/pull/589)).
-
-#### Version 1.7.0 (2015-12-11)
-
-New Features:
- - Preliminary support for Kafka 0.9
-   ([#572](https://github.com/Shopify/sarama/pull/572)). This comes with several
-   caveats:
-   - Protocol-layer support is mostly in place
-     ([#577](https://github.com/Shopify/sarama/pull/577)), however Kafka 0.9
-     renamed some messages and fields, which we did not in order to preserve API
-     compatibility.
-   - The producer and consumer work against 0.9, but the offset manager does
-     not ([#573](https://github.com/Shopify/sarama/pull/573)).
-   - TLS support may or may not work
-     ([#581](https://github.com/Shopify/sarama/pull/581)).
-
-Improvements:
- - Don't wait for request timeouts on dead brokers, greatly speeding recovery
-   when the TCP connection is left hanging
-   ([#548](https://github.com/Shopify/sarama/pull/548)).
- - Refactored part of the producer. The new version provides a much more elegant
-   solution to [#449](https://github.com/Shopify/sarama/pull/449). It is also
-   slightly more efficient, and much more precise in calculating batch sizes
-   when compression is used
-   ([#549](https://github.com/Shopify/sarama/pull/549),
-   [#550](https://github.com/Shopify/sarama/pull/550),
-   [#551](https://github.com/Shopify/sarama/pull/551)).
-
-Bug Fixes:
- - Fix race condition in consumer test mock
-   ([#553](https://github.com/Shopify/sarama/pull/553)).
-
-#### Version 1.6.1 (2015-09-25)
-
-Bug Fixes:
- - Fix panic that could occur if a user-supplied message value failed to encode
-   ([#449](https://github.com/Shopify/sarama/pull/449)).
-
-#### Version 1.6.0 (2015-09-04)
-
-New Features:
- - Implementation of a consumer offset manager using the APIs introduced in
-   Kafka 0.8.2. The API is designed mainly for integration into a future
-   high-level consumer, not for direct use, although it is *possible* to use it
-   directly.
-   ([#461](https://github.com/Shopify/sarama/pull/461)).
-
-Improvements:
- - CRC32 calculation is much faster on machines with SSE4.2 instructions,
-   removing a major hotspot from most profiles
-   ([#255](https://github.com/Shopify/sarama/pull/255)).
-
-Bug Fixes:
- - Make protocol decoding more robust against some malformed packets generated
-   by go-fuzz ([#523](https://github.com/Shopify/sarama/pull/523),
-   [#525](https://github.com/Shopify/sarama/pull/525)) or found in other ways
-   ([#528](https://github.com/Shopify/sarama/pull/528)).
- - Fix a potential race condition panic in the consumer on shutdown
-   ([#529](https://github.com/Shopify/sarama/pull/529)).
-
-#### Version 1.5.0 (2015-08-17)
-
-New Features:
- - TLS-encrypted network connections are now supported. This feature is subject
-   to change when Kafka releases built-in TLS support, but for now this is
-   enough to work with TLS-terminating proxies
-   ([#154](https://github.com/Shopify/sarama/pull/154)).
-
-Improvements:
- - The consumer will not block if a single partition is not drained by the user;
-   all other partitions will continue to consume normally
-   ([#485](https://github.com/Shopify/sarama/pull/485)).
- - Formatting of error strings has been much improved
-   ([#495](https://github.com/Shopify/sarama/pull/495)).
- - Internal refactoring of the producer for code cleanliness and to enable
-   future work ([#300](https://github.com/Shopify/sarama/pull/300)).
-
-Bug Fixes:
- - Fix a potential deadlock in the consumer on shutdown
-   ([#475](https://github.com/Shopify/sarama/pull/475)).
-
-#### Version 1.4.3 (2015-07-21)
-
-Bug Fixes:
- - Don't include the partitioner in the producer's "fetch partitions"
-   circuit-breaker ([#466](https://github.com/Shopify/sarama/pull/466)).
- - Don't retry messages until the broker is closed when abandoning a broker in
-   the producer ([#468](https://github.com/Shopify/sarama/pull/468)).
- - Update the import path for snappy-go, it has moved again and the API has
-   changed slightly ([#486](https://github.com/Shopify/sarama/pull/486)).
-
-#### Version 1.4.2 (2015-05-27)
-
-Bug Fixes:
- - Update the import path for snappy-go, it has moved from google code to github
-   ([#456](https://github.com/Shopify/sarama/pull/456)).
-
-#### Version 1.4.1 (2015-05-25)
-
-Improvements:
- - Optimizations when decoding snappy messages, thanks to John Potocny
-   ([#446](https://github.com/Shopify/sarama/pull/446)).
-
-Bug Fixes:
- - Fix hypothetical race conditions on producer shutdown
-   ([#450](https://github.com/Shopify/sarama/pull/450),
-   [#451](https://github.com/Shopify/sarama/pull/451)).
-
-#### Version 1.4.0 (2015-05-01)
-
-New Features:
- - The consumer now implements `Topics()` and `Partitions()` methods to enable
-   users to dynamically choose what topics/partitions to consume without
-   instantiating a full client
-   ([#431](https://github.com/Shopify/sarama/pull/431)).
- - The partition-consumer now exposes the high water mark offset value returned
-   by the broker via the `HighWaterMarkOffset()` method ([#339](https://github.com/Shopify/sarama/pull/339)).
- - Added a `kafka-console-consumer` tool capable of handling multiple
-   partitions, and deprecated the now-obsolete `kafka-console-partitionConsumer`
-   ([#439](https://github.com/Shopify/sarama/pull/439),
-   [#442](https://github.com/Shopify/sarama/pull/442)).
-
-Improvements:
- - The producer's logging during retry scenarios is more consistent, more
-   useful, and slightly less verbose
-   ([#429](https://github.com/Shopify/sarama/pull/429)).
- - The client now shuffles its initial list of seed brokers in order to prevent
-   thundering herd on the first broker in the list
-   ([#441](https://github.com/Shopify/sarama/pull/441)).
-
-Bug Fixes:
- - The producer now correctly manages its state if retries occur when it is
-   shutting down, fixing several instances of confusing behaviour and at least
-   one potential deadlock ([#419](https://github.com/Shopify/sarama/pull/419)).
- - The consumer now handles messages for different partitions asynchronously,
-   making it much more resilient to specific user code ordering
-   ([#325](https://github.com/Shopify/sarama/pull/325)).
-
-#### Version 1.3.0 (2015-04-16)
-
-New Features:
- - The client now tracks consumer group coordinators using
-   ConsumerMetadataRequests similar to how it tracks partition leadership using
-   regular MetadataRequests ([#411](https://github.com/Shopify/sarama/pull/411)).
-   This adds two methods to the client API:
-   - `Coordinator(consumerGroup string) (*Broker, error)`
-   - `RefreshCoordinator(consumerGroup string) error`
-
-Improvements:
- - ConsumerMetadataResponses now automatically create a Broker object out of the
-   ID/address/port combination for the Coordinator; accessing the fields
-   individually has been deprecated
-   ([#413](https://github.com/Shopify/sarama/pull/413)).
- - Much improved handling of `OffsetOutOfRange` errors in the consumer.
-   Consumers will fail to start if the provided offset is out of range
-   ([#418](https://github.com/Shopify/sarama/pull/418))
-   and they will automatically shut down if the offset falls out of range
-   ([#424](https://github.com/Shopify/sarama/pull/424)).
- - Small performance improvement in encoding and decoding protocol messages
-   ([#427](https://github.com/Shopify/sarama/pull/427)).
-
-Bug Fixes:
- - Fix a rare race condition in the client's background metadata refresher if
-   it happens to be activated while the client is being closed
-   ([#422](https://github.com/Shopify/sarama/pull/422)).
-
-#### Version 1.2.0 (2015-04-07)
-
-Improvements:
- - The producer's behaviour when `Flush.Frequency` is set is now more intuitive
-   ([#389](https://github.com/Shopify/sarama/pull/389)).
- - The producer is now somewhat more memory-efficient during and after retrying
-   messages due to an improved queue implementation
-   ([#396](https://github.com/Shopify/sarama/pull/396)).
- - The consumer produces much more useful logging output when leadership
-   changes ([#385](https://github.com/Shopify/sarama/pull/385)).
- - The client's `GetOffset` method will now automatically refresh metadata and
-   retry once in the event of stale information or similar
-   ([#394](https://github.com/Shopify/sarama/pull/394)).
- - Broker connections now have support for using TCP keepalives
-   ([#407](https://github.com/Shopify/sarama/issues/407)).
-
-Bug Fixes:
- - The OffsetCommitRequest message now correctly implements all three possible
-   API versions ([#390](https://github.com/Shopify/sarama/pull/390),
-   [#400](https://github.com/Shopify/sarama/pull/400)).
-
-#### Version 1.1.0 (2015-03-20)
-
-Improvements:
- - Wrap the producer's partitioner call in a circuit-breaker so that repeatedly
-   broken topics don't choke throughput
-   ([#373](https://github.com/Shopify/sarama/pull/373)).
-
-Bug Fixes:
- - Fix the producer's internal reference counting in certain unusual scenarios
-   ([#367](https://github.com/Shopify/sarama/pull/367)).
- - Fix the consumer's internal reference counting in certain unusual scenarios
-   ([#369](https://github.com/Shopify/sarama/pull/369)).
- - Fix a condition where the producer's internal control messages could have
-   gotten stuck ([#368](https://github.com/Shopify/sarama/pull/368)).
- - Fix an issue where invalid partition lists would be cached when asking for
-   metadata for a non-existant topic ([#372](https://github.com/Shopify/sarama/pull/372)).
-
-
-#### Version 1.0.0 (2015-03-17)
-
-Version 1.0.0 is the first tagged version, and is almost a complete rewrite. The primary differences with previous untagged versions are:
-
-- The producer has been rewritten; there is now a `SyncProducer` with a blocking API, and an `AsyncProducer` that is non-blocking.
-- The consumer has been rewritten to only open one connection per broker instead of one connection per partition.
-- The main types of Sarama are now interfaces to make depedency injection easy; mock implementations for `Consumer`, `SyncProducer` and `AsyncProducer` are provided in the `github.com/Shopify/sarama/mocks` package.
-- For most uses cases, it is no longer necessary to open a `Client`; this will be done for you.
-- All the configuration values have been unified in the `Config` struct.
-- Much improved test suite.

vendor/github.com/Shopify/sarama/LICENSE

@@ -1,20 +0,0 @@
-Copyright (c) 2013 Shopify
-
-Permission is hereby granted, free of charge, to any person obtaining
-a copy of this software and associated documentation files (the
-"Software"), to deal in the Software without restriction, including
-without limitation the rights to use, copy, modify, merge, publish,
-distribute, sublicense, and/or sell copies of the Software, and to
-permit persons to whom the Software is furnished to do so, subject to
-the following conditions:
-
-The above copyright notice and this permission notice shall be
-included in all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
-NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
-LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
-OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
-WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

vendor/github.com/Shopify/sarama/Makefile

@@ -1,27 +0,0 @@
-default: fmt get update test lint
-
-GO       := GO111MODULE=on GOPRIVATE=github.com/linkedin GOSUMDB=off go
-GOBUILD  := CGO_ENABLED=0 $(GO) build $(BUILD_FLAG)
-GOTEST   := $(GO) test -gcflags='-l' -p 3 -v -race -timeout 6m -coverprofile=profile.out -covermode=atomic
-
-FILES    := $(shell find . -name '*.go' -type f -not -name '*.pb.go' -not -name '*_generated.go' -not -name '*_test.go')
-TESTS    := $(shell find . -name '*.go' -type f -not -name '*.pb.go' -not -name '*_generated.go' -name '*_test.go')
-
-get:
-	$(GO) get ./...
-	$(GO) mod verify
-	$(GO) mod tidy
-
-update:
-	$(GO) get -u -v all
-	$(GO) mod verify
-	$(GO) mod tidy
-
-fmt:
-	gofmt -s -l -w $(FILES) $(TESTS)
-
-lint:
-	golangci-lint run
-
-test:
-	$(GOTEST) ./...

vendor/github.com/Shopify/sarama/README.md

@@ -1,36 +0,0 @@
-# sarama
-
-[![GoDoc](https://godoc.org/github.com/Shopify/sarama?status.svg)](https://godoc.org/github.com/Shopify/sarama)
-[![Build Status](https://travis-ci.org/Shopify/sarama.svg?branch=master)](https://travis-ci.org/Shopify/sarama)
-[![Coverage](https://codecov.io/gh/Shopify/sarama/branch/master/graph/badge.svg)](https://codecov.io/gh/Shopify/sarama)
-
-Sarama is an MIT-licensed Go client library for [Apache Kafka](https://kafka.apache.org/) version 0.8 (and later).
-
-## Getting started
-
-- API documentation and examples are available via [godoc](https://godoc.org/github.com/Shopify/sarama).
-- Mocks for testing are available in the [mocks](./mocks) subpackage.
-- The [examples](./examples) directory contains more elaborate example applications.
-- The [tools](./tools) directory contains command line tools that can be useful for testing, diagnostics, and instrumentation.
-
-You might also want to look at the [Frequently Asked Questions](https://github.com/Shopify/sarama/wiki/Frequently-Asked-Questions).
-
-## Compatibility and API stability
-
-Sarama provides a "2 releases + 2 months" compatibility guarantee: we support
-the two latest stable releases of Kafka and Go, and we provide a two month
-grace period for older releases. This means we currently officially support
-Go 1.12 through 1.14, and Kafka 2.1 through 2.4, although older releases are
-still likely to work.
-
-Sarama follows semantic versioning and provides API stability via the gopkg.in service.
-You can import a version with a guaranteed stable API via http://gopkg.in/Shopify/sarama.v1.
-A changelog is available [here](CHANGELOG.md).
-
-## Contributing
-
-- Get started by checking our [contribution guidelines](https://github.com/Shopify/sarama/blob/master/.github/CONTRIBUTING.md).
-- Read the [Sarama wiki](https://github.com/Shopify/sarama/wiki) for more technical and design details.
-- The [Kafka Protocol Specification](https://cwiki.apache.org/confluence/display/KAFKA/A+Guide+To+The+Kafka+Protocol) contains a wealth of useful information.
-- For more general issues, there is [a google group](https://groups.google.com/forum/#!forum/kafka-clients) for Kafka client developers.
-- If you have any questions, just ask!

vendor/github.com/Shopify/sarama/Vagrantfile

@@ -1,14 +0,0 @@
-# We have 5 * 192MB ZK processes and 5 * 320MB Kafka processes => 2560MB
-MEMORY = 3072
-
-Vagrant.configure("2") do |config|
-  config.vm.box = "ubuntu/bionic64"
-
-  config.vm.provision :shell, path: "vagrant/provision.sh"
-
-  config.vm.network "private_network", ip: "192.168.100.67"
-
-  config.vm.provider "virtualbox" do |v|
-    v.memory = MEMORY
-  end
-end

vendor/github.com/Shopify/sarama/acl_bindings.go

@@ -1,138 +0,0 @@
-package sarama
-
-//Resource holds information about acl resource type
-type Resource struct {
-	ResourceType        AclResourceType
-	ResourceName        string
-	ResourcePatternType AclResourcePatternType
-}
-
-func (r *Resource) encode(pe packetEncoder, version int16) error {
-	pe.putInt8(int8(r.ResourceType))
-
-	if err := pe.putString(r.ResourceName); err != nil {
-		return err
-	}
-
-	if version == 1 {
-		if r.ResourcePatternType == AclPatternUnknown {
-			Logger.Print("Cannot encode an unknown resource pattern type, using Literal instead")
-			r.ResourcePatternType = AclPatternLiteral
-		}
-		pe.putInt8(int8(r.ResourcePatternType))
-	}
-
-	return nil
-}
-
-func (r *Resource) decode(pd packetDecoder, version int16) (err error) {
-	resourceType, err := pd.getInt8()
-	if err != nil {
-		return err
-	}
-	r.ResourceType = AclResourceType(resourceType)
-
-	if r.ResourceName, err = pd.getString(); err != nil {
-		return err
-	}
-	if version == 1 {
-		pattern, err := pd.getInt8()
-		if err != nil {
-			return err
-		}
-		r.ResourcePatternType = AclResourcePatternType(pattern)
-	}
-
-	return nil
-}
-
-//Acl holds information about acl type
-type Acl struct {
-	Principal      string
-	Host           string
-	Operation      AclOperation
-	PermissionType AclPermissionType
-}
-
-func (a *Acl) encode(pe packetEncoder) error {
-	if err := pe.putString(a.Principal); err != nil {
-		return err
-	}
-
-	if err := pe.putString(a.Host); err != nil {
-		return err
-	}
-
-	pe.putInt8(int8(a.Operation))
-	pe.putInt8(int8(a.PermissionType))
-
-	return nil
-}
-
-func (a *Acl) decode(pd packetDecoder, version int16) (err error) {
-	if a.Principal, err = pd.getString(); err != nil {
-		return err
-	}
-
-	if a.Host, err = pd.getString(); err != nil {
-		return err
-	}
-
-	operation, err := pd.getInt8()
-	if err != nil {
-		return err
-	}
-	a.Operation = AclOperation(operation)
-
-	permissionType, err := pd.getInt8()
-	if err != nil {
-		return err
-	}
-	a.PermissionType = AclPermissionType(permissionType)
-
-	return nil
-}
-
-//ResourceAcls is an acl resource type
-type ResourceAcls struct {
-	Resource
-	Acls []*Acl
-}
-
-func (r *ResourceAcls) encode(pe packetEncoder, version int16) error {
-	if err := r.Resource.encode(pe, version); err != nil {
-		return err
-	}
-
-	if err := pe.putArrayLength(len(r.Acls)); err != nil {
-		return err
-	}
-	for _, acl := range r.Acls {
-		if err := acl.encode(pe); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-func (r *ResourceAcls) decode(pd packetDecoder, version int16) error {
-	if err := r.Resource.decode(pd, version); err != nil {
-		return err
-	}
-
-	n, err := pd.getArrayLength()
-	if err != nil {
-		return err
-	}
-
-	r.Acls = make([]*Acl, n)
-	for i := 0; i < n; i++ {
-		r.Acls[i] = new(Acl)
-		if err := r.Acls[i].decode(pd, version); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}

vendor/github.com/Shopify/sarama/acl_create_request.go

@@ -1,89 +0,0 @@
-package sarama
-
-//CreateAclsRequest is an acl creation request
-type CreateAclsRequest struct {
-	Version      int16
-	AclCreations []*AclCreation
-}
-
-func (c *CreateAclsRequest) encode(pe packetEncoder) error {
-	if err := pe.putArrayLength(len(c.AclCreations)); err != nil {
-		return err
-	}
-
-	for _, aclCreation := range c.AclCreations {
-		if err := aclCreation.encode(pe, c.Version); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-func (c *CreateAclsRequest) decode(pd packetDecoder, version int16) (err error) {
-	c.Version = version
-	n, err := pd.getArrayLength()
-	if err != nil {
-		return err
-	}
-
-	c.AclCreations = make([]*AclCreation, n)
-
-	for i := 0; i < n; i++ {
-		c.AclCreations[i] = new(AclCreation)
-		if err := c.AclCreations[i].decode(pd, version); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-func (c *CreateAclsRequest) key() int16 {
-	return 30
-}
-
-func (c *CreateAclsRequest) version() int16 {
-	return c.Version
-}
-
-func (c *CreateAclsRequest) headerVersion() int16 {
-	return 1
-}
-
-func (c *CreateAclsRequest) requiredVersion() KafkaVersion {
-	switch c.Version {
-	case 1:
-		return V2_0_0_0
-	default:
-		return V0_11_0_0
-	}
-}
-
-//AclCreation is a wrapper around Resource and Acl type
-type AclCreation struct {
-	Resource
-	Acl
-}
-
-func (a *AclCreation) encode(pe packetEncoder, version int16) error {
-	if err := a.Resource.encode(pe, version); err != nil {
-		return err
-	}
-	if err := a.Acl.encode(pe); err != nil {
-		return err
-	}
-
-	return nil
-}
-
-func (a *AclCreation) decode(pd packetDecoder, version int16) (err error) {
-	if err := a.Resource.decode(pd, version); err != nil {
-		return err
-	}
-	if err := a.Acl.decode(pd, version); err != nil {
-		return err
-	}
-
-	return nil
-}

vendor/github.com/Shopify/sarama/acl_create_response.go

@@ -1,94 +0,0 @@
-package sarama
-
-import "time"
-
-//CreateAclsResponse is a an acl response creation type
-type CreateAclsResponse struct {
-	ThrottleTime         time.Duration
-	AclCreationResponses []*AclCreationResponse
-}
-
-func (c *CreateAclsResponse) encode(pe packetEncoder) error {
-	pe.putInt32(int32(c.ThrottleTime / time.Millisecond))
-
-	if err := pe.putArrayLength(len(c.AclCreationResponses)); err != nil {
-		return err
-	}
-
-	for _, aclCreationResponse := range c.AclCreationResponses {
-		if err := aclCreationResponse.encode(pe); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-func (c *CreateAclsResponse) decode(pd packetDecoder, version int16) (err error) {
-	throttleTime, err := pd.getInt32()
-	if err != nil {
-		return err
-	}
-	c.ThrottleTime = time.Duration(throttleTime) * time.Millisecond
-
-	n, err := pd.getArrayLength()
-	if err != nil {
-		return err
-	}
-
-	c.AclCreationResponses = make([]*AclCreationResponse, n)
-	for i := 0; i < n; i++ {
-		c.AclCreationResponses[i] = new(AclCreationResponse)
-		if err := c.AclCreationResponses[i].decode(pd, version); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-func (c *CreateAclsResponse) key() int16 {
-	return 30
-}
-
-func (c *CreateAclsResponse) version() int16 {
-	return 0
-}
-
-func (c *CreateAclsResponse) headerVersion() int16 {
-	return 0
-}
-
-func (c *CreateAclsResponse) requiredVersion() KafkaVersion {
-	return V0_11_0_0
-}
-
-//AclCreationResponse is an acl creation response type
-type AclCreationResponse struct {
-	Err    KError
-	ErrMsg *string
-}
-
-func (a *AclCreationResponse) encode(pe packetEncoder) error {
-	pe.putInt16(int16(a.Err))
-
-	if err := pe.putNullableString(a.ErrMsg); err != nil {
-		return err
-	}
-
-	return nil
-}
-
-func (a *AclCreationResponse) decode(pd packetDecoder, version int16) (err error) {
-	kerr, err := pd.getInt16()
-	if err != nil {
-		return err
-	}
-	a.Err = KError(kerr)
-
-	if a.ErrMsg, err = pd.getNullableString(); err != nil {
-		return err
-	}
-
-	return nil
-}

vendor/github.com/Shopify/sarama/acl_delete_request.go

@@ -1,62 +0,0 @@
-package sarama
-
-//DeleteAclsRequest is a delete acl request
-type DeleteAclsRequest struct {
-	Version int
-	Filters []*AclFilter
-}
-
-func (d *DeleteAclsRequest) encode(pe packetEncoder) error {
-	if err := pe.putArrayLength(len(d.Filters)); err != nil {
-		return err
-	}
-
-	for _, filter := range d.Filters {
-		filter.Version = d.Version
-		if err := filter.encode(pe); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-func (d *DeleteAclsRequest) decode(pd packetDecoder, version int16) (err error) {
-	d.Version = int(version)
-	n, err := pd.getArrayLength()
-	if err != nil {
-		return err
-	}
-
-	d.Filters = make([]*AclFilter, n)
-	for i := 0; i < n; i++ {
-		d.Filters[i] = new(AclFilter)
-		d.Filters[i].Version = int(version)
-		if err := d.Filters[i].decode(pd, version); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-func (d *DeleteAclsRequest) key() int16 {
-	return 31
-}
-
-func (d *DeleteAclsRequest) version() int16 {
-	return int16(d.Version)
-}
-
-func (c *DeleteAclsRequest) headerVersion() int16 {
-	return 1
-}
-
-func (d *DeleteAclsRequest) requiredVersion() KafkaVersion {
-	switch d.Version {
-	case 1:
-		return V2_0_0_0
-	default:
-		return V0_11_0_0
-	}
-}

vendor/github.com/Shopify/sarama/acl_delete_response.go

@@ -1,163 +0,0 @@
-package sarama
-
-import "time"
-
-//DeleteAclsResponse is a delete acl response
-type DeleteAclsResponse struct {
-	Version         int16
-	ThrottleTime    time.Duration
-	FilterResponses []*FilterResponse
-}
-
-func (d *DeleteAclsResponse) encode(pe packetEncoder) error {
-	pe.putInt32(int32(d.ThrottleTime / time.Millisecond))
-
-	if err := pe.putArrayLength(len(d.FilterResponses)); err != nil {
-		return err
-	}
-
-	for _, filterResponse := range d.FilterResponses {
-		if err := filterResponse.encode(pe, d.Version); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-func (d *DeleteAclsResponse) decode(pd packetDecoder, version int16) (err error) {
-	throttleTime, err := pd.getInt32()
-	if err != nil {
-		return err
-	}
-	d.ThrottleTime = time.Duration(throttleTime) * time.Millisecond
-
-	n, err := pd.getArrayLength()
-	if err != nil {
-		return err
-	}
-	d.FilterResponses = make([]*FilterResponse, n)
-
-	for i := 0; i < n; i++ {
-		d.FilterResponses[i] = new(FilterResponse)
-		if err := d.FilterResponses[i].decode(pd, version); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-func (d *DeleteAclsResponse) key() int16 {
-	return 31
-}
-
-func (d *DeleteAclsResponse) version() int16 {
-	return d.Version
-}
-
-func (d *DeleteAclsResponse) headerVersion() int16 {
-	return 0
-}
-
-func (d *DeleteAclsResponse) requiredVersion() KafkaVersion {
-	return V0_11_0_0
-}
-
-//FilterResponse is a filter response type
-type FilterResponse struct {
-	Err          KError
-	ErrMsg       *string
-	MatchingAcls []*MatchingAcl
-}
-
-func (f *FilterResponse) encode(pe packetEncoder, version int16) error {
-	pe.putInt16(int16(f.Err))
-	if err := pe.putNullableString(f.ErrMsg); err != nil {
-		return err
-	}
-
-	if err := pe.putArrayLength(len(f.MatchingAcls)); err != nil {
-		return err
-	}
-	for _, matchingAcl := range f.MatchingAcls {
-		if err := matchingAcl.encode(pe, version); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-func (f *FilterResponse) decode(pd packetDecoder, version int16) (err error) {
-	kerr, err := pd.getInt16()
-	if err != nil {
-		return err
-	}
-	f.Err = KError(kerr)
-
-	if f.ErrMsg, err = pd.getNullableString(); err != nil {
-		return err
-	}
-
-	n, err := pd.getArrayLength()
-	if err != nil {
-		return err
-	}
-	f.MatchingAcls = make([]*MatchingAcl, n)
-	for i := 0; i < n; i++ {
-		f.MatchingAcls[i] = new(MatchingAcl)
-		if err := f.MatchingAcls[i].decode(pd, version); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-//MatchingAcl is a matching acl type
-type MatchingAcl struct {
-	Err    KError
-	ErrMsg *string
-	Resource
-	Acl
-}
-
-func (m *MatchingAcl) encode(pe packetEncoder, version int16) error {
-	pe.putInt16(int16(m.Err))
-	if err := pe.putNullableString(m.ErrMsg); err != nil {
-		return err
-	}
-
-	if err := m.Resource.encode(pe, version); err != nil {
-		return err
-	}
-
-	if err := m.Acl.encode(pe); err != nil {
-		return err
-	}
-
-	return nil
-}
-
-func (m *MatchingAcl) decode(pd packetDecoder, version int16) (err error) {
-	kerr, err := pd.getInt16()
-	if err != nil {
-		return err
-	}
-	m.Err = KError(kerr)
-
-	if m.ErrMsg, err = pd.getNullableString(); err != nil {
-		return err
-	}
-
-	if err := m.Resource.decode(pd, version); err != nil {
-		return err
-	}
-
-	if err := m.Acl.decode(pd, version); err != nil {
-		return err
-	}
-
-	return nil
-}

vendor/github.com/Shopify/sarama/acl_describe_request.go

@@ -1,39 +0,0 @@
-package sarama
-
-//DescribeAclsRequest is a secribe acl request type
-type DescribeAclsRequest struct {
-	Version int
-	AclFilter
-}
-
-func (d *DescribeAclsRequest) encode(pe packetEncoder) error {
-	d.AclFilter.Version = d.Version
-	return d.AclFilter.encode(pe)
-}
-
-func (d *DescribeAclsRequest) decode(pd packetDecoder, version int16) (err error) {
-	d.Version = int(version)
-	d.AclFilter.Version = int(version)
-	return d.AclFilter.decode(pd, version)
-}
-
-func (d *DescribeAclsRequest) key() int16 {
-	return 29
-}
-
-func (d *DescribeAclsRequest) version() int16 {
-	return int16(d.Version)
-}
-
-func (d *DescribeAclsRequest) headerVersion() int16 {
-	return 1
-}
-
-func (d *DescribeAclsRequest) requiredVersion() KafkaVersion {
-	switch d.Version {
-	case 1:
-		return V2_0_0_0
-	default:
-		return V0_11_0_0
-	}
-}

vendor/github.com/Shopify/sarama/acl_describe_response.go

@@ -1,91 +0,0 @@
-package sarama
-
-import "time"
-
-//DescribeAclsResponse is a describe acl response type
-type DescribeAclsResponse struct {
-	Version      int16
-	ThrottleTime time.Duration
-	Err          KError
-	ErrMsg       *string
-	ResourceAcls []*ResourceAcls
-}
-
-func (d *DescribeAclsResponse) encode(pe packetEncoder) error {
-	pe.putInt32(int32(d.ThrottleTime / time.Millisecond))
-	pe.putInt16(int16(d.Err))
-
-	if err := pe.putNullableString(d.ErrMsg); err != nil {
-		return err
-	}
-
-	if err := pe.putArrayLength(len(d.ResourceAcls)); err != nil {
-		return err
-	}
-
-	for _, resourceAcl := range d.ResourceAcls {
-		if err := resourceAcl.encode(pe, d.Version); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-func (d *DescribeAclsResponse) decode(pd packetDecoder, version int16) (err error) {
-	throttleTime, err := pd.getInt32()
-	if err != nil {
-		return err
-	}
-	d.ThrottleTime = time.Duration(throttleTime) * time.Millisecond
-
-	kerr, err := pd.getInt16()
-	if err != nil {
-		return err
-	}
-	d.Err = KError(kerr)
-
-	errmsg, err := pd.getString()
-	if err != nil {
-		return err
-	}
-	if errmsg != "" {
-		d.ErrMsg = &errmsg
-	}
-
-	n, err := pd.getArrayLength()
-	if err != nil {
-		return err
-	}
-	d.ResourceAcls = make([]*ResourceAcls, n)
-
-	for i := 0; i < n; i++ {
-		d.ResourceAcls[i] = new(ResourceAcls)
-		if err := d.ResourceAcls[i].decode(pd, version); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-func (d *DescribeAclsResponse) key() int16 {
-	return 29
-}
-
-func (d *DescribeAclsResponse) version() int16 {
-	return d.Version
-}
-
-func (d *DescribeAclsResponse) headerVersion() int16 {
-	return 0
-}
-
-func (d *DescribeAclsResponse) requiredVersion() KafkaVersion {
-	switch d.Version {
-	case 1:
-		return V2_0_0_0
-	default:
-		return V0_11_0_0
-	}
-}

vendor/github.com/Shopify/sarama/acl_filter.go

@@ -1,78 +0,0 @@
-package sarama
-
-type AclFilter struct {
-	Version                   int
-	ResourceType              AclResourceType
-	ResourceName              *string
-	ResourcePatternTypeFilter AclResourcePatternType
-	Principal                 *string
-	Host                      *string
-	Operation                 AclOperation
-	PermissionType            AclPermissionType
-}
-
-func (a *AclFilter) encode(pe packetEncoder) error {
-	pe.putInt8(int8(a.ResourceType))
-	if err := pe.putNullableString(a.ResourceName); err != nil {
-		return err
-	}
-
-	if a.Version == 1 {
-		pe.putInt8(int8(a.ResourcePatternTypeFilter))
-	}
-
-	if err := pe.putNullableString(a.Principal); err != nil {
-		return err
-	}
-	if err := pe.putNullableString(a.Host); err != nil {
-		return err
-	}
-	pe.putInt8(int8(a.Operation))
-	pe.putInt8(int8(a.PermissionType))
-
-	return nil
-}
-
-func (a *AclFilter) decode(pd packetDecoder, version int16) (err error) {
-	resourceType, err := pd.getInt8()
-	if err != nil {
-		return err
-	}
-	a.ResourceType = AclResourceType(resourceType)
-
-	if a.ResourceName, err = pd.getNullableString(); err != nil {
-		return err
-	}
-
-	if a.Version == 1 {
-		pattern, err := pd.getInt8()
-
-		if err != nil {
-			return err
-		}
-
-		a.ResourcePatternTypeFilter = AclResourcePatternType(pattern)
-	}
-
-	if a.Principal, err = pd.getNullableString(); err != nil {
-		return err
-	}
-
-	if a.Host, err = pd.getNullableString(); err != nil {
-		return err
-	}
-
-	operation, err := pd.getInt8()
-	if err != nil {
-		return err
-	}
-	a.Operation = AclOperation(operation)
-
-	permissionType, err := pd.getInt8()
-	if err != nil {
-		return err
-	}
-	a.PermissionType = AclPermissionType(permissionType)
-
-	return nil
-}

vendor/github.com/Shopify/sarama/acl_types.go

@@ -1,55 +0,0 @@
-package sarama
-
-type (
-	AclOperation int
-
-	AclPermissionType int
-
-	AclResourceType int
-
-	AclResourcePatternType int
-)
-
-// ref: https://github.com/apache/kafka/blob/trunk/clients/src/main/java/org/apache/kafka/common/acl/AclOperation.java
-const (
-	AclOperationUnknown AclOperation = iota
-	AclOperationAny
-	AclOperationAll
-	AclOperationRead
-	AclOperationWrite
-	AclOperationCreate
-	AclOperationDelete
-	AclOperationAlter
-	AclOperationDescribe
-	AclOperationClusterAction
-	AclOperationDescribeConfigs
-	AclOperationAlterConfigs
-	AclOperationIdempotentWrite
-)
-
-// ref: https://github.com/apache/kafka/blob/trunk/clients/src/main/java/org/apache/kafka/common/acl/AclPermissionType.java
-const (
-	AclPermissionUnknown AclPermissionType = iota
-	AclPermissionAny
-	AclPermissionDeny
-	AclPermissionAllow
-)
-
-// ref: https://github.com/apache/kafka/blob/trunk/clients/src/main/java/org/apache/kafka/common/resource/ResourceType.java
-const (
-	AclResourceUnknown AclResourceType = iota
-	AclResourceAny
-	AclResourceTopic
-	AclResourceGroup
-	AclResourceCluster
-	AclResourceTransactionalID
-)
-
-// ref: https://github.com/apache/kafka/blob/trunk/clients/src/main/java/org/apache/kafka/common/resource/PatternType.java
-const (
-	AclPatternUnknown AclResourcePatternType = iota
-	AclPatternAny
-	AclPatternMatch
-	AclPatternLiteral
-	AclPatternPrefixed
-)

vendor/github.com/Shopify/sarama/add_offsets_to_txn_request.go

@@ -1,57 +0,0 @@
-package sarama
-
-//AddOffsetsToTxnRequest adds offsets to a transaction request
-type AddOffsetsToTxnRequest struct {
-	TransactionalID string
-	ProducerID      int64
-	ProducerEpoch   int16
-	GroupID         string
-}
-
-func (a *AddOffsetsToTxnRequest) encode(pe packetEncoder) error {
-	if err := pe.putString(a.TransactionalID); err != nil {
-		return err
-	}
-
-	pe.putInt64(a.ProducerID)
-
-	pe.putInt16(a.ProducerEpoch)
-
-	if err := pe.putString(a.GroupID); err != nil {
-		return err
-	}
-
-	return nil
-}
-
-func (a *AddOffsetsToTxnRequest) decode(pd packetDecoder, version int16) (err error) {
-	if a.TransactionalID, err = pd.getString(); err != nil {
-		return err
-	}
-	if a.ProducerID, err = pd.getInt64(); err != nil {
-		return err
-	}
-	if a.ProducerEpoch, err = pd.getInt16(); err != nil {
-		return err
-	}
-	if a.GroupID, err = pd.getString(); err != nil {
-		return err
-	}
-	return nil
-}
-
-func (a *AddOffsetsToTxnRequest) key() int16 {
-	return 25
-}
-
-func (a *AddOffsetsToTxnRequest) version() int16 {
-	return 0
-}
-
-func (a *AddOffsetsToTxnRequest) headerVersion() int16 {
-	return 1
-}
-
-func (a *AddOffsetsToTxnRequest) requiredVersion() KafkaVersion {
-	return V0_11_0_0
-}

vendor/github.com/Shopify/sarama/add_offsets_to_txn_response.go

@@ -1,49 +0,0 @@
-package sarama
-
-import (
-	"time"
-)
-
-//AddOffsetsToTxnResponse is a response type for adding offsets to txns
-type AddOffsetsToTxnResponse struct {
-	ThrottleTime time.Duration
-	Err          KError
-}
-
-func (a *AddOffsetsToTxnResponse) encode(pe packetEncoder) error {
-	pe.putInt32(int32(a.ThrottleTime / time.Millisecond))
-	pe.putInt16(int16(a.Err))
-	return nil
-}
-
-func (a *AddOffsetsToTxnResponse) decode(pd packetDecoder, version int16) (err error) {
-	throttleTime, err := pd.getInt32()
-	if err != nil {
-		return err
-	}
-	a.ThrottleTime = time.Duration(throttleTime) * time.Millisecond
-
-	kerr, err := pd.getInt16()
-	if err != nil {
-		return err
-	}
-	a.Err = KError(kerr)
-
-	return nil
-}
-
-func (a *AddOffsetsToTxnResponse) key() int16 {
-	return 25
-}
-
-func (a *AddOffsetsToTxnResponse) version() int16 {
-	return 0
-}
-
-func (a *AddOffsetsToTxnResponse) headerVersion() int16 {
-	return 0
-}
-
-func (a *AddOffsetsToTxnResponse) requiredVersion() KafkaVersion {
-	return V0_11_0_0
-}

vendor/github.com/Shopify/sarama/add_partitions_to_txn_request.go

@@ -1,81 +0,0 @@
-package sarama
-
-//AddPartitionsToTxnRequest is a add paartition request
-type AddPartitionsToTxnRequest struct {
-	TransactionalID string
-	ProducerID      int64
-	ProducerEpoch   int16
-	TopicPartitions map[string][]int32
-}
-
-func (a *AddPartitionsToTxnRequest) encode(pe packetEncoder) error {
-	if err := pe.putString(a.TransactionalID); err != nil {
-		return err
-	}
-	pe.putInt64(a.ProducerID)
-	pe.putInt16(a.ProducerEpoch)
-
-	if err := pe.putArrayLength(len(a.TopicPartitions)); err != nil {
-		return err
-	}
-	for topic, partitions := range a.TopicPartitions {
-		if err := pe.putString(topic); err != nil {
-			return err
-		}
-		if err := pe.putInt32Array(partitions); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-func (a *AddPartitionsToTxnRequest) decode(pd packetDecoder, version int16) (err error) {
-	if a.TransactionalID, err = pd.getString(); err != nil {
-		return err
-	}
-	if a.ProducerID, err = pd.getInt64(); err != nil {
-		return err
-	}
-	if a.ProducerEpoch, err = pd.getInt16(); err != nil {
-		return err
-	}
-
-	n, err := pd.getArrayLength()
-	if err != nil {
-		return err
-	}
-
-	a.TopicPartitions = make(map[string][]int32)
-	for i := 0; i < n; i++ {
-		topic, err := pd.getString()
-		if err != nil {
-			return err
-		}
-
-		partitions, err := pd.getInt32Array()
-		if err != nil {
-			return err
-		}
-
-		a.TopicPartitions[topic] = partitions
-	}
-
-	return nil
-}
-
-func (a *AddPartitionsToTxnRequest) key() int16 {
-	return 24
-}
-
-func (a *AddPartitionsToTxnRequest) version() int16 {
-	return 0
-}
-
-func (a *AddPartitionsToTxnRequest) headerVersion() int16 {
-	return 1
-}
-
-func (a *AddPartitionsToTxnRequest) requiredVersion() KafkaVersion {
-	return V0_11_0_0
-}

vendor/github.com/Shopify/sarama/add_partitions_to_txn_response.go

@@ -1,114 +0,0 @@
-package sarama
-
-import (
-	"time"
-)
-
-//AddPartitionsToTxnResponse is a partition errors to transaction type
-type AddPartitionsToTxnResponse struct {
-	ThrottleTime time.Duration
-	Errors       map[string][]*PartitionError
-}
-
-func (a *AddPartitionsToTxnResponse) encode(pe packetEncoder) error {
-	pe.putInt32(int32(a.ThrottleTime / time.Millisecond))
-	if err := pe.putArrayLength(len(a.Errors)); err != nil {
-		return err
-	}
-
-	for topic, e := range a.Errors {
-		if err := pe.putString(topic); err != nil {
-			return err
-		}
-		if err := pe.putArrayLength(len(e)); err != nil {
-			return err
-		}
-		for _, partitionError := range e {
-			if err := partitionError.encode(pe); err != nil {
-				return err
-			}
-		}
-	}
-
-	return nil
-}
-
-func (a *AddPartitionsToTxnResponse) decode(pd packetDecoder, version int16) (err error) {
-	throttleTime, err := pd.getInt32()
-	if err != nil {
-		return err
-	}
-	a.ThrottleTime = time.Duration(throttleTime) * time.Millisecond
-
-	n, err := pd.getArrayLength()
-	if err != nil {
-		return err
-	}
-
-	a.Errors = make(map[string][]*PartitionError)
-
-	for i := 0; i < n; i++ {
-		topic, err := pd.getString()
-		if err != nil {
-			return err
-		}
-
-		m, err := pd.getArrayLength()
-		if err != nil {
-			return err
-		}
-
-		a.Errors[topic] = make([]*PartitionError, m)
-
-		for j := 0; j < m; j++ {
-			a.Errors[topic][j] = new(PartitionError)
-			if err := a.Errors[topic][j].decode(pd, version); err != nil {
-				return err
-			}
-		}
-	}
-
-	return nil
-}
-
-func (a *AddPartitionsToTxnResponse) key() int16 {
-	return 24
-}
-
-func (a *AddPartitionsToTxnResponse) version() int16 {
-	return 0
-}
-
-func (a *AddPartitionsToTxnResponse) headerVersion() int16 {
-	return 0
-}
-
-func (a *AddPartitionsToTxnResponse) requiredVersion() KafkaVersion {
-	return V0_11_0_0
-}
-
-//PartitionError is a partition error type
-type PartitionError struct {
-	Partition int32
-	Err       KError
-}
-
-func (p *PartitionError) encode(pe packetEncoder) error {
-	pe.putInt32(p.Partition)
-	pe.putInt16(int16(p.Err))
-	return nil
-}
-
-func (p *PartitionError) decode(pd packetDecoder, version int16) (err error) {
-	if p.Partition, err = pd.getInt32(); err != nil {
-		return err
-	}
-
-	kerr, err := pd.getInt16()
-	if err != nil {
-		return err
-	}
-	p.Err = KError(kerr)
-
-	return nil
-}

vendor/github.com/Shopify/sarama/admin.go

@@ -1,934 +0,0 @@
-package sarama
-
-import (
-	"errors"
-	"fmt"
-	"math/rand"
-	"strconv"
-	"sync"
-	"time"
-)
-
-// ClusterAdmin is the administrative client for Kafka, which supports managing and inspecting topics,
-// brokers, configurations and ACLs. The minimum broker version required is 0.10.0.0.
-// Methods with stricter requirements will specify the minimum broker version required.
-// You MUST call Close() on a client to avoid leaks
-type ClusterAdmin interface {
-	// Creates a new topic. This operation is supported by brokers with version 0.10.1.0 or higher.
-	// It may take several seconds after CreateTopic returns success for all the brokers
-	// to become aware that the topic has been created. During this time, listTopics
-	// may not return information about the new topic.The validateOnly option is supported from version 0.10.2.0.
-	CreateTopic(topic string, detail *TopicDetail, validateOnly bool) error
-
-	// List the topics available in the cluster with the default options.
-	ListTopics() (map[string]TopicDetail, error)
-
-	// Describe some topics in the cluster.
-	DescribeTopics(topics []string) (metadata []*TopicMetadata, err error)
-
-	// Delete a topic. It may take several seconds after the DeleteTopic to returns success
-	// and for all the brokers to become aware that the topics are gone.
-	// During this time, listTopics  may continue to return information about the deleted topic.
-	// If delete.topic.enable is false on the brokers, deleteTopic will mark
-	// the topic for deletion, but not actually delete them.
-	// This operation is supported by brokers with version 0.10.1.0 or higher.
-	DeleteTopic(topic string) error
-
-	// Increase the number of partitions of the topics  according to the corresponding values.
-	// If partitions are increased for a topic that has a key, the partition logic or ordering of
-	// the messages will be affected. It may take several seconds after this method returns
-	// success for all the brokers to become aware that the partitions have been created.
-	// During this time, ClusterAdmin#describeTopics may not return information about the
-	// new partitions. This operation is supported by brokers with version 1.0.0 or higher.
-	CreatePartitions(topic string, count int32, assignment [][]int32, validateOnly bool) error
-
-	// Alter the replica assignment for partitions.
-	// This operation is supported by brokers with version 2.4.0.0 or higher.
-	AlterPartitionReassignments(topic string, assignment [][]int32) error
-
-	// Provides info on ongoing partitions replica reassignments.
-	// This operation is supported by brokers with version 2.4.0.0 or higher.
-	ListPartitionReassignments(topics string, partitions []int32) (topicStatus map[string]map[int32]*PartitionReplicaReassignmentsStatus, err error)
-
-	// Delete records whose offset is smaller than the given offset of the corresponding partition.
-	// This operation is supported by brokers with version 0.11.0.0 or higher.
-	DeleteRecords(topic string, partitionOffsets map[int32]int64) error
-
-	// Get the configuration for the specified resources.
-	// The returned configuration includes default values and the Default is true
-	// can be used to distinguish them from user supplied values.
-	// Config entries where ReadOnly is true cannot be updated.
-	// The value of config entries where Sensitive is true is always nil so
-	// sensitive information is not disclosed.
-	// This operation is supported by brokers with version 0.11.0.0 or higher.
-	DescribeConfig(resource ConfigResource) ([]ConfigEntry, error)
-
-	// Update the configuration for the specified resources with the default options.
-	// This operation is supported by brokers with version 0.11.0.0 or higher.
-	// The resources with their configs (topic is the only resource type with configs
-	// that can be updated currently Updates are not transactional so they may succeed
-	// for some resources while fail for others. The configs for a particular resource are updated automatically.
-	AlterConfig(resourceType ConfigResourceType, name string, entries map[string]*string, validateOnly bool) error
-
-	// Creates access control lists (ACLs) which are bound to specific resources.
-	// This operation is not transactional so it may succeed for some ACLs while fail for others.
-	// If you attempt to add an ACL that duplicates an existing ACL, no error will be raised, but
-	// no changes will be made. This operation is supported by brokers with version 0.11.0.0 or higher.
-	CreateACL(resource Resource, acl Acl) error
-
-	// Lists access control lists (ACLs) according to the supplied filter.
-	// it may take some time for changes made by createAcls or deleteAcls to be reflected in the output of ListAcls
-	// This operation is supported by brokers with version 0.11.0.0 or higher.
-	ListAcls(filter AclFilter) ([]ResourceAcls, error)
-
-	// Deletes access control lists (ACLs) according to the supplied filters.
-	// This operation is not transactional so it may succeed for some ACLs while fail for others.
-	// This operation is supported by brokers with version 0.11.0.0 or higher.
-	DeleteACL(filter AclFilter, validateOnly bool) ([]MatchingAcl, error)
-
-	// List the consumer groups available in the cluster.
-	ListConsumerGroups() (map[string]string, error)
-
-	// Describe the given consumer groups.
-	DescribeConsumerGroups(groups []string) ([]*GroupDescription, error)
-
-	// List the consumer group offsets available in the cluster.
-	ListConsumerGroupOffsets(group string, topicPartitions map[string][]int32) (*OffsetFetchResponse, error)
-
-	// Delete a consumer group.
-	DeleteConsumerGroup(group string) error
-
-	// Get information about the nodes in the cluster
-	DescribeCluster() (brokers []*Broker, controllerID int32, err error)
-
-	// Get information about all log directories on the given set of brokers
-	DescribeLogDirs(brokers []int32) (map[int32][]DescribeLogDirsResponseDirMetadata, error)
-
-	// Close shuts down the admin and closes underlying client.
-	Close() error
-}
-
-type clusterAdmin struct {
-	client Client
-	conf   *Config
-}
-
-// NewClusterAdmin creates a new ClusterAdmin using the given broker addresses and configuration.
-func NewClusterAdmin(addrs []string, conf *Config) (ClusterAdmin, error) {
-	client, err := NewClient(addrs, conf)
-	if err != nil {
-		return nil, err
-	}
-	return NewClusterAdminFromClient(client)
-}
-
-// NewClusterAdminFromClient creates a new ClusterAdmin using the given client.
-// Note that underlying client will also be closed on admin's Close() call.
-func NewClusterAdminFromClient(client Client) (ClusterAdmin, error) {
-	//make sure we can retrieve the controller
-	_, err := client.Controller()
-	if err != nil {
-		return nil, err
-	}
-
-	ca := &clusterAdmin{
-		client: client,
-		conf:   client.Config(),
-	}
-	return ca, nil
-}
-
-func (ca *clusterAdmin) Close() error {
-	return ca.client.Close()
-}
-
-func (ca *clusterAdmin) Controller() (*Broker, error) {
-	return ca.client.Controller()
-}
-
-func (ca *clusterAdmin) refreshController() (*Broker, error) {
-	return ca.client.RefreshController()
-}
-
-// isErrNoController returns `true` if the given error type unwraps to an
-// `ErrNotController` response from Kafka
-func isErrNoController(err error) bool {
-	switch e := err.(type) {
-	case *TopicError:
-		return e.Err == ErrNotController
-	case *TopicPartitionError:
-		return e.Err == ErrNotController
-	case KError:
-		return e == ErrNotController
-	}
-	return false
-}
-
-// retryOnError will repeatedly call the given (error-returning) func in the
-// case that its response is non-nil and retriable (as determined by the
-// provided retriable func) up to the maximum number of tries permitted by
-// the admin client configuration
-func (ca *clusterAdmin) retryOnError(retriable func(error) bool, fn func() error) error {
-	var err error
-	for attempt := 0; attempt < ca.conf.Admin.Retry.Max; attempt++ {
-		err = fn()
-		if err == nil || !retriable(err) {
-			return err
-		}
-		Logger.Printf(
-			"admin/request retrying after %dms... (%d attempts remaining)\n",
-			ca.conf.Admin.Retry.Backoff/time.Millisecond, ca.conf.Admin.Retry.Max-attempt)
-		time.Sleep(ca.conf.Admin.Retry.Backoff)
-		continue
-	}
-	return err
-}
-
-func (ca *clusterAdmin) CreateTopic(topic string, detail *TopicDetail, validateOnly bool) error {
-	if topic == "" {
-		return ErrInvalidTopic
-	}
-
-	if detail == nil {
-		return errors.New("you must specify topic details")
-	}
-
-	topicDetails := make(map[string]*TopicDetail)
-	topicDetails[topic] = detail
-
-	request := &CreateTopicsRequest{
-		TopicDetails: topicDetails,
-		ValidateOnly: validateOnly,
-		Timeout:      ca.conf.Admin.Timeout,
-	}
-
-	if ca.conf.Version.IsAtLeast(V0_11_0_0) {
-		request.Version = 1
-	}
-	if ca.conf.Version.IsAtLeast(V1_0_0_0) {
-		request.Version = 2
-	}
-
-	return ca.retryOnError(isErrNoController, func() error {
-		b, err := ca.Controller()
-		if err != nil {
-			return err
-		}
-
-		rsp, err := b.CreateTopics(request)
-		if err != nil {
-			return err
-		}
-
-		topicErr, ok := rsp.TopicErrors[topic]
-		if !ok {
-			return ErrIncompleteResponse
-		}
-
-		if topicErr.Err != ErrNoError {
-			if topicErr.Err == ErrNotController {
-				_, _ = ca.refreshController()
-			}
-			return topicErr
-		}
-
-		return nil
-	})
-}
-
-func (ca *clusterAdmin) DescribeTopics(topics []string) (metadata []*TopicMetadata, err error) {
-	controller, err := ca.Controller()
-	if err != nil {
-		return nil, err
-	}
-
-	request := &MetadataRequest{
-		Topics:                 topics,
-		AllowAutoTopicCreation: false,
-	}
-
-	if ca.conf.Version.IsAtLeast(V1_0_0_0) {
-		request.Version = 5
-	} else if ca.conf.Version.IsAtLeast(V0_11_0_0) {
-		request.Version = 4
-	}
-
-	response, err := controller.GetMetadata(request)
-	if err != nil {
-		return nil, err
-	}
-	return response.Topics, nil
-}
-
-func (ca *clusterAdmin) DescribeCluster() (brokers []*Broker, controllerID int32, err error) {
-	controller, err := ca.Controller()
-	if err != nil {
-		return nil, int32(0), err
-	}
-
-	request := &MetadataRequest{
-		Topics: []string{},
-	}
-
-	if ca.conf.Version.IsAtLeast(V0_10_0_0) {
-		request.Version = 1
-	}
-
-	response, err := controller.GetMetadata(request)
-	if err != nil {
-		return nil, int32(0), err
-	}
-
-	return response.Brokers, response.ControllerID, nil
-}
-
-func (ca *clusterAdmin) findBroker(id int32) (*Broker, error) {
-	brokers := ca.client.Brokers()
-	for _, b := range brokers {
-		if b.ID() == id {
-			return b, nil
-		}
-	}
-	return nil, fmt.Errorf("could not find broker id %d", id)
-}
-
-func (ca *clusterAdmin) findAnyBroker() (*Broker, error) {
-	brokers := ca.client.Brokers()
-	if len(brokers) > 0 {
-		index := rand.Intn(len(brokers))
-		return brokers[index], nil
-	}
-	return nil, errors.New("no available broker")
-}
-
-func (ca *clusterAdmin) ListTopics() (map[string]TopicDetail, error) {
-	// In order to build TopicDetails we need to first get the list of all
-	// topics using a MetadataRequest and then get their configs using a
-	// DescribeConfigsRequest request. To avoid sending many requests to the
-	// broker, we use a single DescribeConfigsRequest.
-
-	// Send the all-topic MetadataRequest
-	b, err := ca.findAnyBroker()
-	if err != nil {
-		return nil, err
-	}
-	_ = b.Open(ca.client.Config())
-
-	metadataReq := &MetadataRequest{}
-	metadataResp, err := b.GetMetadata(metadataReq)
-	if err != nil {
-		return nil, err
-	}
-
-	topicsDetailsMap := make(map[string]TopicDetail)
-
-	var describeConfigsResources []*ConfigResource
-
-	for _, topic := range metadataResp.Topics {
-		topicDetails := TopicDetail{
-			NumPartitions: int32(len(topic.Partitions)),
-		}
-		if len(topic.Partitions) > 0 {
-			topicDetails.ReplicaAssignment = map[int32][]int32{}
-			for _, partition := range topic.Partitions {
-				topicDetails.ReplicaAssignment[partition.ID] = partition.Replicas
-			}
-			topicDetails.ReplicationFactor = int16(len(topic.Partitions[0].Replicas))
-		}
-		topicsDetailsMap[topic.Name] = topicDetails
-
-		// we populate the resources we want to describe from the MetadataResponse
-		topicResource := ConfigResource{
-			Type: TopicResource,
-			Name: topic.Name,
-		}
-		describeConfigsResources = append(describeConfigsResources, &topicResource)
-	}
-
-	// Send the DescribeConfigsRequest
-	describeConfigsReq := &DescribeConfigsRequest{
-		Resources: describeConfigsResources,
-	}
-
-	if ca.conf.Version.IsAtLeast(V1_1_0_0) {
-		describeConfigsReq.Version = 1
-	}
-
-	if ca.conf.Version.IsAtLeast(V2_0_0_0) {
-		describeConfigsReq.Version = 2
-	}
-
-	describeConfigsResp, err := b.DescribeConfigs(describeConfigsReq)
-	if err != nil {
-		return nil, err
-	}
-
-	for _, resource := range describeConfigsResp.Resources {
-		topicDetails := topicsDetailsMap[resource.Name]
-		topicDetails.ConfigEntries = make(map[string]*string)
-
-		for _, entry := range resource.Configs {
-			// only include non-default non-sensitive config
-			// (don't actually think topic config will ever be sensitive)
-			if entry.Default || entry.Sensitive {
-				continue
-			}
-			topicDetails.ConfigEntries[entry.Name] = &entry.Value
-		}
-
-		topicsDetailsMap[resource.Name] = topicDetails
-	}
-
-	return topicsDetailsMap, nil
-}
-
-func (ca *clusterAdmin) DeleteTopic(topic string) error {
-	if topic == "" {
-		return ErrInvalidTopic
-	}
-
-	request := &DeleteTopicsRequest{
-		Topics:  []string{topic},
-		Timeout: ca.conf.Admin.Timeout,
-	}
-
-	if ca.conf.Version.IsAtLeast(V0_11_0_0) {
-		request.Version = 1
-	}
-
-	return ca.retryOnError(isErrNoController, func() error {
-		b, err := ca.Controller()
-		if err != nil {
-			return err
-		}
-
-		rsp, err := b.DeleteTopics(request)
-		if err != nil {
-			return err
-		}
-
-		topicErr, ok := rsp.TopicErrorCodes[topic]
-		if !ok {
-			return ErrIncompleteResponse
-		}
-
-		if topicErr != ErrNoError {
-			if topicErr == ErrNotController {
-				_, _ = ca.refreshController()
-			}
-			return topicErr
-		}
-
-		return nil
-	})
-}
-
-func (ca *clusterAdmin) CreatePartitions(topic string, count int32, assignment [][]int32, validateOnly bool) error {
-	if topic == "" {
-		return ErrInvalidTopic
-	}
-
-	topicPartitions := make(map[string]*TopicPartition)
-	topicPartitions[topic] = &TopicPartition{Count: count, Assignment: assignment}
-
-	request := &CreatePartitionsRequest{
-		TopicPartitions: topicPartitions,
-		Timeout:         ca.conf.Admin.Timeout,
-	}
-
-	return ca.retryOnError(isErrNoController, func() error {
-		b, err := ca.Controller()
-		if err != nil {
-			return err
-		}
-
-		rsp, err := b.CreatePartitions(request)
-		if err != nil {
-			return err
-		}
-
-		topicErr, ok := rsp.TopicPartitionErrors[topic]
-		if !ok {
-			return ErrIncompleteResponse
-		}
-
-		if topicErr.Err != ErrNoError {
-			if topicErr.Err == ErrNotController {
-				_, _ = ca.refreshController()
-			}
-			return topicErr
-		}
-
-		return nil
-	})
-}
-
-func (ca *clusterAdmin) AlterPartitionReassignments(topic string, assignment [][]int32) error {
-	if topic == "" {
-		return ErrInvalidTopic
-	}
-
-	request := &AlterPartitionReassignmentsRequest{
-		TimeoutMs: int32(60000),
-		Version:   int16(0),
-	}
-
-	for i := 0; i < len(assignment); i++ {
-		request.AddBlock(topic, int32(i), assignment[i])
-	}
-
-	return ca.retryOnError(isErrNoController, func() error {
-		b, err := ca.Controller()
-		if err != nil {
-			return err
-		}
-
-		errs := make([]error, 0)
-
-		rsp, err := b.AlterPartitionReassignments(request)
-
-		if err != nil {
-			errs = append(errs, err)
-		} else {
-			if rsp.ErrorCode > 0 {
-				errs = append(errs, errors.New(rsp.ErrorCode.Error()))
-			}
-
-			for topic, topicErrors := range rsp.Errors {
-				for partition, partitionError := range topicErrors {
-					if partitionError.errorCode != ErrNoError {
-						errStr := fmt.Sprintf("[%s-%d]: %s", topic, partition, partitionError.errorCode.Error())
-						errs = append(errs, errors.New(errStr))
-					}
-				}
-			}
-		}
-
-		if len(errs) > 0 {
-			return ErrReassignPartitions{MultiError{&errs}}
-		}
-
-		return nil
-	})
-}
-
-func (ca *clusterAdmin) ListPartitionReassignments(topic string, partitions []int32) (topicStatus map[string]map[int32]*PartitionReplicaReassignmentsStatus, err error) {
-	if topic == "" {
-		return nil, ErrInvalidTopic
-	}
-
-	request := &ListPartitionReassignmentsRequest{
-		TimeoutMs: int32(60000),
-		Version:   int16(0),
-	}
-
-	request.AddBlock(topic, partitions)
-
-	b, err := ca.Controller()
-	if err != nil {
-		return nil, err
-	}
-	_ = b.Open(ca.client.Config())
-
-	rsp, err := b.ListPartitionReassignments(request)
-
-	if err == nil && rsp != nil {
-		return rsp.TopicStatus, nil
-	} else {
-		return nil, err
-	}
-}
-
-func (ca *clusterAdmin) DeleteRecords(topic string, partitionOffsets map[int32]int64) error {
-	if topic == "" {
-		return ErrInvalidTopic
-	}
-	partitionPerBroker := make(map[*Broker][]int32)
-	for partition := range partitionOffsets {
-		broker, err := ca.client.Leader(topic, partition)
-		if err != nil {
-			return err
-		}
-		if _, ok := partitionPerBroker[broker]; ok {
-			partitionPerBroker[broker] = append(partitionPerBroker[broker], partition)
-		} else {
-			partitionPerBroker[broker] = []int32{partition}
-		}
-	}
-	errs := make([]error, 0)
-	for broker, partitions := range partitionPerBroker {
-		topics := make(map[string]*DeleteRecordsRequestTopic)
-		recordsToDelete := make(map[int32]int64)
-		for _, p := range partitions {
-			recordsToDelete[p] = partitionOffsets[p]
-		}
-		topics[topic] = &DeleteRecordsRequestTopic{PartitionOffsets: recordsToDelete}
-		request := &DeleteRecordsRequest{
-			Topics:  topics,
-			Timeout: ca.conf.Admin.Timeout,
-		}
-
-		rsp, err := broker.DeleteRecords(request)
-		if err != nil {
-			errs = append(errs, err)
-		} else {
-			deleteRecordsResponseTopic, ok := rsp.Topics[topic]
-			if !ok {
-				errs = append(errs, ErrIncompleteResponse)
-			} else {
-				for _, deleteRecordsResponsePartition := range deleteRecordsResponseTopic.Partitions {
-					if deleteRecordsResponsePartition.Err != ErrNoError {
-						errs = append(errs, errors.New(deleteRecordsResponsePartition.Err.Error()))
-					}
-				}
-			}
-		}
-	}
-	if len(errs) > 0 {
-		return ErrDeleteRecords{MultiError{&errs}}
-	}
-	//todo since we are dealing with couple of partitions it would be good if we return slice of errors
-	//for each partition instead of one error
-	return nil
-}
-
-// Returns a bool indicating whether the resource request needs to go to a
-// specific broker
-func dependsOnSpecificNode(resource ConfigResource) bool {
-	return (resource.Type == BrokerResource && resource.Name != "") ||
-		resource.Type == BrokerLoggerResource
-}
-
-func (ca *clusterAdmin) DescribeConfig(resource ConfigResource) ([]ConfigEntry, error) {
-	var entries []ConfigEntry
-	var resources []*ConfigResource
-	resources = append(resources, &resource)
-
-	request := &DescribeConfigsRequest{
-		Resources: resources,
-	}
-
-	if ca.conf.Version.IsAtLeast(V1_1_0_0) {
-		request.Version = 1
-	}
-
-	if ca.conf.Version.IsAtLeast(V2_0_0_0) {
-		request.Version = 2
-	}
-
-	var (
-		b   *Broker
-		err error
-	)
-
-	// DescribeConfig of broker/broker logger must be sent to the broker in question
-	if dependsOnSpecificNode(resource) {
-		id, _ := strconv.Atoi(resource.Name)
-		b, err = ca.findBroker(int32(id))
-	} else {
-		b, err = ca.findAnyBroker()
-	}
-	if err != nil {
-		return nil, err
-	}
-
-	_ = b.Open(ca.client.Config())
-	rsp, err := b.DescribeConfigs(request)
-	if err != nil {
-		return nil, err
-	}
-
-	for _, rspResource := range rsp.Resources {
-		if rspResource.Name == resource.Name {
-			if rspResource.ErrorMsg != "" {
-				return nil, errors.New(rspResource.ErrorMsg)
-			}
-			if rspResource.ErrorCode != 0 {
-				return nil, KError(rspResource.ErrorCode)
-			}
-			for _, cfgEntry := range rspResource.Configs {
-				entries = append(entries, *cfgEntry)
-			}
-		}
-	}
-	return entries, nil
-}
-
-func (ca *clusterAdmin) AlterConfig(resourceType ConfigResourceType, name string, entries map[string]*string, validateOnly bool) error {
-	var resources []*AlterConfigsResource
-	resources = append(resources, &AlterConfigsResource{
-		Type:          resourceType,
-		Name:          name,
-		ConfigEntries: entries,
-	})
-
-	request := &AlterConfigsRequest{
-		Resources:    resources,
-		ValidateOnly: validateOnly,
-	}
-
-	var (
-		b   *Broker
-		err error
-	)
-
-	// AlterConfig of broker/broker logger must be sent to the broker in question
-	if dependsOnSpecificNode(ConfigResource{Name: name, Type: resourceType}) {
-		id, _ := strconv.Atoi(name)
-		b, err = ca.findBroker(int32(id))
-	} else {
-		b, err = ca.findAnyBroker()
-	}
-	if err != nil {
-		return err
-	}
-
-	_ = b.Open(ca.client.Config())
-	rsp, err := b.AlterConfigs(request)
-	if err != nil {
-		return err
-	}
-
-	for _, rspResource := range rsp.Resources {
-		if rspResource.Name == name {
-			if rspResource.ErrorMsg != "" {
-				return errors.New(rspResource.ErrorMsg)
-			}
-			if rspResource.ErrorCode != 0 {
-				return KError(rspResource.ErrorCode)
-			}
-		}
-	}
-	return nil
-}
-
-func (ca *clusterAdmin) CreateACL(resource Resource, acl Acl) error {
-	var acls []*AclCreation
-	acls = append(acls, &AclCreation{resource, acl})
-	request := &CreateAclsRequest{AclCreations: acls}
-
-	if ca.conf.Version.IsAtLeast(V2_0_0_0) {
-		request.Version = 1
-	}
-
-	b, err := ca.Controller()
-	if err != nil {
-		return err
-	}
-
-	_, err = b.CreateAcls(request)
-	return err
-}
-
-func (ca *clusterAdmin) ListAcls(filter AclFilter) ([]ResourceAcls, error) {
-	request := &DescribeAclsRequest{AclFilter: filter}
-
-	if ca.conf.Version.IsAtLeast(V2_0_0_0) {
-		request.Version = 1
-	}
-
-	b, err := ca.Controller()
-	if err != nil {
-		return nil, err
-	}
-
-	rsp, err := b.DescribeAcls(request)
-	if err != nil {
-		return nil, err
-	}
-
-	var lAcls []ResourceAcls
-	for _, rAcl := range rsp.ResourceAcls {
-		lAcls = append(lAcls, *rAcl)
-	}
-	return lAcls, nil
-}
-
-func (ca *clusterAdmin) DeleteACL(filter AclFilter, validateOnly bool) ([]MatchingAcl, error) {
-	var filters []*AclFilter
-	filters = append(filters, &filter)
-	request := &DeleteAclsRequest{Filters: filters}
-
-	if ca.conf.Version.IsAtLeast(V2_0_0_0) {
-		request.Version = 1
-	}
-
-	b, err := ca.Controller()
-	if err != nil {
-		return nil, err
-	}
-
-	rsp, err := b.DeleteAcls(request)
-	if err != nil {
-		return nil, err
-	}
-
-	var mAcls []MatchingAcl
-	for _, fr := range rsp.FilterResponses {
-		for _, mACL := range fr.MatchingAcls {
-			mAcls = append(mAcls, *mACL)
-		}
-	}
-	return mAcls, nil
-}
-
-func (ca *clusterAdmin) DescribeConsumerGroups(groups []string) (result []*GroupDescription, err error) {
-	groupsPerBroker := make(map[*Broker][]string)
-
-	for _, group := range groups {
-		controller, err := ca.client.Coordinator(group)
-		if err != nil {
-			return nil, err
-		}
-		groupsPerBroker[controller] = append(groupsPerBroker[controller], group)
-	}
-
-	for broker, brokerGroups := range groupsPerBroker {
-		response, err := broker.DescribeGroups(&DescribeGroupsRequest{
-			Groups: brokerGroups,
-		})
-		if err != nil {
-			return nil, err
-		}
-
-		result = append(result, response.Groups...)
-	}
-	return result, nil
-}
-
-func (ca *clusterAdmin) ListConsumerGroups() (allGroups map[string]string, err error) {
-	allGroups = make(map[string]string)
-
-	// Query brokers in parallel, since we have to query *all* brokers
-	brokers := ca.client.Brokers()
-	groupMaps := make(chan map[string]string, len(brokers))
-	errors := make(chan error, len(brokers))
-	wg := sync.WaitGroup{}
-
-	for _, b := range brokers {
-		wg.Add(1)
-		go func(b *Broker, conf *Config) {
-			defer wg.Done()
-			_ = b.Open(conf) // Ensure that broker is opened
-
-			response, err := b.ListGroups(&ListGroupsRequest{})
-			if err != nil {
-				errors <- err
-				return
-			}
-
-			groups := make(map[string]string)
-			for group, typ := range response.Groups {
-				groups[group] = typ
-			}
-
-			groupMaps <- groups
-		}(b, ca.conf)
-	}
-
-	wg.Wait()
-	close(groupMaps)
-	close(errors)
-
-	for groupMap := range groupMaps {
-		for group, protocolType := range groupMap {
-			allGroups[group] = protocolType
-		}
-	}
-
-	// Intentionally return only the first error for simplicity
-	err = <-errors
-	return
-}
-
-func (ca *clusterAdmin) ListConsumerGroupOffsets(group string, topicPartitions map[string][]int32) (*OffsetFetchResponse, error) {
-	coordinator, err := ca.client.Coordinator(group)
-	if err != nil {
-		return nil, err
-	}
-
-	request := &OffsetFetchRequest{
-		ConsumerGroup: group,
-		partitions:    topicPartitions,
-	}
-
-	if ca.conf.Version.IsAtLeast(V0_10_2_0) {
-		request.Version = 2
-	} else if ca.conf.Version.IsAtLeast(V0_8_2_2) {
-		request.Version = 1
-	}
-
-	return coordinator.FetchOffset(request)
-}
-
-func (ca *clusterAdmin) DeleteConsumerGroup(group string) error {
-	coordinator, err := ca.client.Coordinator(group)
-	if err != nil {
-		return err
-	}
-
-	request := &DeleteGroupsRequest{
-		Groups: []string{group},
-	}
-
-	resp, err := coordinator.DeleteGroups(request)
-	if err != nil {
-		return err
-	}
-
-	groupErr, ok := resp.GroupErrorCodes[group]
-	if !ok {
-		return ErrIncompleteResponse
-	}
-
-	if groupErr != ErrNoError {
-		return groupErr
-	}
-
-	return nil
-}
-
-func (ca *clusterAdmin) DescribeLogDirs(brokerIds []int32) (allLogDirs map[int32][]DescribeLogDirsResponseDirMetadata, err error) {
-	allLogDirs = make(map[int32][]DescribeLogDirsResponseDirMetadata)
-
-	// Query brokers in parallel, since we may have to query multiple brokers
-	logDirsMaps := make(chan map[int32][]DescribeLogDirsResponseDirMetadata, len(brokerIds))
-	errors := make(chan error, len(brokerIds))
-	wg := sync.WaitGroup{}
-
-	for _, b := range brokerIds {
-		wg.Add(1)
-		broker, err := ca.findBroker(b)
-		if err != nil {
-			Logger.Printf("Unable to find broker with ID = %v\n", b)
-			continue
-		}
-		go func(b *Broker, conf *Config) {
-			defer wg.Done()
-			_ = b.Open(conf) // Ensure that broker is opened
-
-			response, err := b.DescribeLogDirs(&DescribeLogDirsRequest{})
-			if err != nil {
-				errors <- err
-				return
-			}
-			logDirs := make(map[int32][]DescribeLogDirsResponseDirMetadata)
-			logDirs[b.ID()] = response.LogDirs
-			logDirsMaps <- logDirs
-		}(broker, ca.conf)
-	}
-
-	wg.Wait()
-	close(logDirsMaps)
-	close(errors)
-
-	for logDirsMap := range logDirsMaps {
-		for id, logDirs := range logDirsMap {
-			allLogDirs[id] = logDirs
-		}
-	}
-
-	// Intentionally return only the first error for simplicity
-	err = <-errors
-	return
-}

vendor/github.com/Shopify/sarama/alter_configs_request.go

@@ -1,126 +0,0 @@
-package sarama
-
-//AlterConfigsRequest is an alter config request type
-type AlterConfigsRequest struct {
-	Resources    []*AlterConfigsResource
-	ValidateOnly bool
-}
-
-//AlterConfigsResource is an alter config resource type
-type AlterConfigsResource struct {
-	Type          ConfigResourceType
-	Name          string
-	ConfigEntries map[string]*string
-}
-
-func (a *AlterConfigsRequest) encode(pe packetEncoder) error {
-	if err := pe.putArrayLength(len(a.Resources)); err != nil {
-		return err
-	}
-
-	for _, r := range a.Resources {
-		if err := r.encode(pe); err != nil {
-			return err
-		}
-	}
-
-	pe.putBool(a.ValidateOnly)
-	return nil
-}
-
-func (a *AlterConfigsRequest) decode(pd packetDecoder, version int16) error {
-	resourceCount, err := pd.getArrayLength()
-	if err != nil {
-		return err
-	}
-
-	a.Resources = make([]*AlterConfigsResource, resourceCount)
-	for i := range a.Resources {
-		r := &AlterConfigsResource{}
-		err = r.decode(pd, version)
-		if err != nil {
-			return err
-		}
-		a.Resources[i] = r
-	}
-
-	validateOnly, err := pd.getBool()
-	if err != nil {
-		return err
-	}
-
-	a.ValidateOnly = validateOnly
-
-	return nil
-}
-
-func (a *AlterConfigsResource) encode(pe packetEncoder) error {
-	pe.putInt8(int8(a.Type))
-
-	if err := pe.putString(a.Name); err != nil {
-		return err
-	}
-
-	if err := pe.putArrayLength(len(a.ConfigEntries)); err != nil {
-		return err
-	}
-	for configKey, configValue := range a.ConfigEntries {
-		if err := pe.putString(configKey); err != nil {
-			return err
-		}
-		if err := pe.putNullableString(configValue); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-func (a *AlterConfigsResource) decode(pd packetDecoder, version int16) error {
-	t, err := pd.getInt8()
-	if err != nil {
-		return err
-	}
-	a.Type = ConfigResourceType(t)
-
-	name, err := pd.getString()
-	if err != nil {
-		return err
-	}
-	a.Name = name
-
-	n, err := pd.getArrayLength()
-	if err != nil {
-		return err
-	}
-
-	if n > 0 {
-		a.ConfigEntries = make(map[string]*string, n)
-		for i := 0; i < n; i++ {
-			configKey, err := pd.getString()
-			if err != nil {
-				return err
-			}
-			if a.ConfigEntries[configKey], err = pd.getNullableString(); err != nil {
-				return err
-			}
-		}
-	}
-	return err
-}
-
-func (a *AlterConfigsRequest) key() int16 {
-	return 33
-}
-
-func (a *AlterConfigsRequest) version() int16 {
-	return 0
-}
-
-func (a *AlterConfigsRequest) headerVersion() int16 {
-	return 1
-}
-
-func (a *AlterConfigsRequest) requiredVersion() KafkaVersion {
-	return V0_11_0_0
-}

vendor/github.com/Shopify/sarama/alter_configs_response.go

@@ -1,101 +0,0 @@
-package sarama
-
-import "time"
-
-//AlterConfigsResponse is a response type for alter config
-type AlterConfigsResponse struct {
-	ThrottleTime time.Duration
-	Resources    []*AlterConfigsResourceResponse
-}
-
-//AlterConfigsResourceResponse is a response type for alter config resource
-type AlterConfigsResourceResponse struct {
-	ErrorCode int16
-	ErrorMsg  string
-	Type      ConfigResourceType
-	Name      string
-}
-
-func (a *AlterConfigsResponse) encode(pe packetEncoder) error {
-	pe.putInt32(int32(a.ThrottleTime / time.Millisecond))
-
-	if err := pe.putArrayLength(len(a.Resources)); err != nil {
-		return err
-	}
-
-	for i := range a.Resources {
-		pe.putInt16(a.Resources[i].ErrorCode)
-		err := pe.putString(a.Resources[i].ErrorMsg)
-		if err != nil {
-			return nil
-		}
-		pe.putInt8(int8(a.Resources[i].Type))
-		err = pe.putString(a.Resources[i].Name)
-		if err != nil {
-			return nil
-		}
-	}
-
-	return nil
-}
-
-func (a *AlterConfigsResponse) decode(pd packetDecoder, version int16) error {
-	throttleTime, err := pd.getInt32()
-	if err != nil {
-		return err
-	}
-	a.ThrottleTime = time.Duration(throttleTime) * time.Millisecond
-
-	responseCount, err := pd.getArrayLength()
-	if err != nil {
-		return err
-	}
-
-	a.Resources = make([]*AlterConfigsResourceResponse, responseCount)
-
-	for i := range a.Resources {
-		a.Resources[i] = new(AlterConfigsResourceResponse)
-
-		errCode, err := pd.getInt16()
-		if err != nil {
-			return err
-		}
-		a.Resources[i].ErrorCode = errCode
-
-		e, err := pd.getString()
-		if err != nil {
-			return err
-		}
-		a.Resources[i].ErrorMsg = e
-
-		t, err := pd.getInt8()
-		if err != nil {
-			return err
-		}
-		a.Resources[i].Type = ConfigResourceType(t)
-
-		name, err := pd.getString()
-		if err != nil {
-			return err
-		}
-		a.Resources[i].Name = name
-	}
-
-	return nil
-}
-
-func (a *AlterConfigsResponse) key() int16 {
-	return 32
-}
-
-func (a *AlterConfigsResponse) version() int16 {
-	return 0
-}
-
-func (a *AlterConfigsResponse) headerVersion() int16 {
-	return 0
-}
-
-func (a *AlterConfigsResponse) requiredVersion() KafkaVersion {
-	return V0_11_0_0
-}

vendor/github.com/Shopify/sarama/alter_partition_reassignments_request.go

@@ -1,130 +0,0 @@
-package sarama
-
-type alterPartitionReassignmentsBlock struct {
-	replicas []int32
-}
-
-func (b *alterPartitionReassignmentsBlock) encode(pe packetEncoder) error {
-	if err := pe.putNullableCompactInt32Array(b.replicas); err != nil {
-		return err
-	}
-
-	pe.putEmptyTaggedFieldArray()
-	return nil
-}
-
-func (b *alterPartitionReassignmentsBlock) decode(pd packetDecoder) (err error) {
-	if b.replicas, err = pd.getCompactInt32Array(); err != nil {
-		return err
-	}
-	return nil
-}
-
-type AlterPartitionReassignmentsRequest struct {
-	TimeoutMs int32
-	blocks    map[string]map[int32]*alterPartitionReassignmentsBlock
-	Version   int16
-}
-
-func (r *AlterPartitionReassignmentsRequest) encode(pe packetEncoder) error {
-	pe.putInt32(r.TimeoutMs)
-
-	pe.putCompactArrayLength(len(r.blocks))
-
-	for topic, partitions := range r.blocks {
-		if err := pe.putCompactString(topic); err != nil {
-			return err
-		}
-		pe.putCompactArrayLength(len(partitions))
-		for partition, block := range partitions {
-			pe.putInt32(partition)
-			if err := block.encode(pe); err != nil {
-				return err
-			}
-		}
-		pe.putEmptyTaggedFieldArray()
-	}
-
-	pe.putEmptyTaggedFieldArray()
-
-	return nil
-}
-
-func (r *AlterPartitionReassignmentsRequest) decode(pd packetDecoder, version int16) (err error) {
-	r.Version = version
-
-	if r.TimeoutMs, err = pd.getInt32(); err != nil {
-		return err
-	}
-
-	topicCount, err := pd.getCompactArrayLength()
-	if err != nil {
-		return err
-	}
-	if topicCount > 0 {
-		r.blocks = make(map[string]map[int32]*alterPartitionReassignmentsBlock)
-		for i := 0; i < topicCount; i++ {
-			topic, err := pd.getCompactString()
-			if err != nil {
-				return err
-			}
-			partitionCount, err := pd.getCompactArrayLength()
-			if err != nil {
-				return err
-			}
-			r.blocks[topic] = make(map[int32]*alterPartitionReassignmentsBlock)
-			for j := 0; j < partitionCount; j++ {
-				partition, err := pd.getInt32()
-				if err != nil {
-					return err
-				}
-				block := &alterPartitionReassignmentsBlock{}
-				if err := block.decode(pd); err != nil {
-					return err
-				}
-				r.blocks[topic][partition] = block
-
-				if _, err := pd.getEmptyTaggedFieldArray(); err != nil {
-					return err
-				}
-			}
-			if _, err := pd.getEmptyTaggedFieldArray(); err != nil {
-				return err
-			}
-		}
-	}
-
-	if _, err := pd.getEmptyTaggedFieldArray(); err != nil {
-		return err
-	}
-
-	return
-}
-
-func (r *AlterPartitionReassignmentsRequest) key() int16 {
-	return 45
-}
-
-func (r *AlterPartitionReassignmentsRequest) version() int16 {
-	return r.Version
-}
-
-func (r *AlterPartitionReassignmentsRequest) headerVersion() int16 {
-	return 2
-}
-
-func (r *AlterPartitionReassignmentsRequest) requiredVersion() KafkaVersion {
-	return V2_4_0_0
-}
-
-func (r *AlterPartitionReassignmentsRequest) AddBlock(topic string, partitionID int32, replicas []int32) {
-	if r.blocks == nil {
-		r.blocks = make(map[string]map[int32]*alterPartitionReassignmentsBlock)
-	}
-
-	if r.blocks[topic] == nil {
-		r.blocks[topic] = make(map[int32]*alterPartitionReassignmentsBlock)
-	}
-
-	r.blocks[topic][partitionID] = &alterPartitionReassignmentsBlock{replicas}
-}

vendor/github.com/Shopify/sarama/alter_partition_reassignments_response.go

@@ -1,157 +0,0 @@
-package sarama
-
-type alterPartitionReassignmentsErrorBlock struct {
-	errorCode    KError
-	errorMessage *string
-}
-
-func (b *alterPartitionReassignmentsErrorBlock) encode(pe packetEncoder) error {
-	pe.putInt16(int16(b.errorCode))
-	if err := pe.putNullableCompactString(b.errorMessage); err != nil {
-		return err
-	}
-	pe.putEmptyTaggedFieldArray()
-
-	return nil
-}
-
-func (b *alterPartitionReassignmentsErrorBlock) decode(pd packetDecoder) (err error) {
-	errorCode, err := pd.getInt16()
-	if err != nil {
-		return err
-	}
-	b.errorCode = KError(errorCode)
-	b.errorMessage, err = pd.getCompactNullableString()
-
-	if _, err := pd.getEmptyTaggedFieldArray(); err != nil {
-		return err
-	}
-	return err
-}
-
-type AlterPartitionReassignmentsResponse struct {
-	Version        int16
-	ThrottleTimeMs int32
-	ErrorCode      KError
-	ErrorMessage   *string
-	Errors         map[string]map[int32]*alterPartitionReassignmentsErrorBlock
-}
-
-func (r *AlterPartitionReassignmentsResponse) AddError(topic string, partition int32, kerror KError, message *string) {
-	if r.Errors == nil {
-		r.Errors = make(map[string]map[int32]*alterPartitionReassignmentsErrorBlock)
-	}
-	partitions := r.Errors[topic]
-	if partitions == nil {
-		partitions = make(map[int32]*alterPartitionReassignmentsErrorBlock)
-		r.Errors[topic] = partitions
-	}
-
-	partitions[partition] = &alterPartitionReassignmentsErrorBlock{errorCode: kerror, errorMessage: message}
-}
-
-func (r *AlterPartitionReassignmentsResponse) encode(pe packetEncoder) error {
-	pe.putInt32(r.ThrottleTimeMs)
-	pe.putInt16(int16(r.ErrorCode))
-	if err := pe.putNullableCompactString(r.ErrorMessage); err != nil {
-		return err
-	}
-
-	pe.putCompactArrayLength(len(r.Errors))
-	for topic, partitions := range r.Errors {
-		if err := pe.putCompactString(topic); err != nil {
-			return err
-		}
-		pe.putCompactArrayLength(len(partitions))
-		for partition, block := range partitions {
-			pe.putInt32(partition)
-
-			if err := block.encode(pe); err != nil {
-				return err
-			}
-		}
-		pe.putEmptyTaggedFieldArray()
-	}
-
-	pe.putEmptyTaggedFieldArray()
-	return nil
-}
-
-func (r *AlterPartitionReassignmentsResponse) decode(pd packetDecoder, version int16) (err error) {
-	r.Version = version
-
-	if r.ThrottleTimeMs, err = pd.getInt32(); err != nil {
-		return err
-	}
-
-	kerr, err := pd.getInt16()
-	if err != nil {
-		return err
-	}
-
-	r.ErrorCode = KError(kerr)
-
-	if r.ErrorMessage, err = pd.getCompactNullableString(); err != nil {
-		return err
-	}
-
-	numTopics, err := pd.getCompactArrayLength()
-	if err != nil {
-		return err
-	}
-
-	if numTopics > 0 {
-		r.Errors = make(map[string]map[int32]*alterPartitionReassignmentsErrorBlock, numTopics)
-		for i := 0; i < numTopics; i++ {
-			topic, err := pd.getCompactString()
-			if err != nil {
-				return err
-			}
-
-			ongoingPartitionReassignments, err := pd.getCompactArrayLength()
-			if err != nil {
-				return err
-			}
-
-			r.Errors[topic] = make(map[int32]*alterPartitionReassignmentsErrorBlock, ongoingPartitionReassignments)
-
-			for j := 0; j < ongoingPartitionReassignments; j++ {
-				partition, err := pd.getInt32()
-				if err != nil {
-					return err
-				}
-				block := &alterPartitionReassignmentsErrorBlock{}
-				if err := block.decode(pd); err != nil {
-					return err
-				}
-
-				r.Errors[topic][partition] = block
-			}
-			if _, err = pd.getEmptyTaggedFieldArray(); err != nil {
-				return err
-			}
-		}
-	}
-
-	if _, err = pd.getEmptyTaggedFieldArray(); err != nil {
-		return err
-	}
-
-	return nil
-}
-
-func (r *AlterPartitionReassignmentsResponse) key() int16 {
-	return 45
-}
-
-func (r *AlterPartitionReassignmentsResponse) version() int16 {
-	return r.Version
-}
-
-func (r *AlterPartitionReassignmentsResponse) headerVersion() int16 {
-	return 1
-}
-
-func (r *AlterPartitionReassignmentsResponse) requiredVersion() KafkaVersion {
-	return V2_4_0_0
-}

vendor/github.com/Shopify/sarama/api_versions_request.go

@@ -1,29 +0,0 @@
-package sarama
-
-//ApiVersionsRequest ...
-type ApiVersionsRequest struct {
-}
-
-func (a *ApiVersionsRequest) encode(pe packetEncoder) error {
-	return nil
-}
-
-func (a *ApiVersionsRequest) decode(pd packetDecoder, version int16) (err error) {
-	return nil
-}
-
-func (a *ApiVersionsRequest) key() int16 {
-	return 18
-}
-
-func (a *ApiVersionsRequest) version() int16 {
-	return 0
-}
-
-func (a *ApiVersionsRequest) headerVersion() int16 {
-	return 1
-}
-
-func (a *ApiVersionsRequest) requiredVersion() KafkaVersion {
-	return V0_10_0_0
-}