add id in raw message stats

the current UUID generator runs at:
```
BenchmarkMessageUUID 	13599342	        87.7 ns/op	      16 B/op	       0 allocs/op
```
 with the former version that was running on
```
BenchmarkMessageUUID 	 2818203	       427 ns/op	      96 B/op	       2 allocs/op
```

it fixes #842 #851

.travis.yml

@@ -1,6 +1,9 @@
+dist: focal
+
 language: go
 go: 1.14
-script: sudo -E bash -c "source /etc/profile && eval '$(gimme 1.14)' && export GOPATH=$HOME/gopath:$GOPATH && go get && GORACE='halt_on_error=1' go test ./... -v -timeout 120s -race"
+
+script: sudo -E bash -c "source /etc/profile && eval '$(gimme 1.14)' && go test ./... -v -timeout 120s -race"
 
 before_install:
   - sudo apt-get install libpcap-dev -y

Dockerfile

@@ -1,6 +1,6 @@
 FROM alpine:latest as builder
 RUN apk add --no-cache ca-certificates openssl
-RUN wget https://github.com/buger/goreplay/releases/download/v1.0.0/gor_1.0.0_x64.tar.gz -O gor.tar.gz
+RUN wget https://github.com/buger/goreplay/releases/download/v1.2.0/gor_v1.2.0_x64.tar.gz -O gor.tar.gz
 RUN tar xzf gor.tar.gz
 
 FROM scratch

Makefile

@@ -7,6 +7,7 @@ PREFIX=
 RUN = docker run -v `pwd`:$(SOURCE_PATH) -e AWS_ACCESS_KEY_ID=$(AWS_ACCESS_KEY_ID) -e AWS_SECRET_ACCESS_KEY=$(AWS_SECRET_ACCESS_KEY) -p 0.0.0.0:$(PORT):$(PORT) -t -i $(CONTAINER)
 BENCHMARK = BenchmarkRAWInput
 TEST = TestRawListenerBench
+BIN_NAME = gor
 VERSION = DEV-$(shell date +%s)
 LDFLAGS = -ldflags "-X main.VERSION=$(VERSION)$(PREFIX) -extldflags \"-static\" -X main.DEMO=$(DEMO)"
 MAC_LDFLAGS = -ldflags "-X main.VERSION=$(VERSION)$(PREFIX) -X main.DEMO=$(DEMO)"
@@ -25,29 +26,35 @@ FPMCOMMON= \
 release: release-x64 release-mac
 
 release-bin:
-	docker run -v `pwd`:$(SOURCE_PATH) -t --env GOOS=linux --env GOARCH=amd64  -i gor go build -o gor -tags netgo $(LDFLAGS)
+	docker run -v `pwd`:$(SOURCE_PATH) -t --env GOOS=linux --env GOARCH=amd64  -i $(CONTAINER) go build -o $(BIN_NAME) -tags netgo $(LDFLAGS)
+
+release-bin-mac:
+	GOOS=darwin go build -o $(BIN_NAME) $(MAC_LDFLAGS)
 
 release-x64:
-	docker run -v `pwd`:$(SOURCE_PATH) -t --env GOOS=linux --env GOARCH=amd64  -i $(CONTAINER) go build -o gor -tags netgo $(LDFLAGS)
-	tar -czf gor_$(VERSION)$(PREFIX)_x64.tar.gz gor
+	docker run -v `pwd`:$(SOURCE_PATH) -t --env GOOS=linux --env GOARCH=amd64  -i $(CONTAINER) go build -o $(BIN_NAME) -tags netgo $(LDFLAGS)
+	tar -czf gor_$(VERSION)$(PREFIX)_x64.tar.gz $(BIN_NAME)
 	mkdir -p /tmp/gor-build
-	mv ./gor /tmp/gor-build/gor
+	mv ./$(BIN_NAME) /tmp/gor-build/$(BIN_NAME)
 	cd /tmp/gor-build
+	rm -f goreplay_$(VERSION)_amd64.deb
+	rm -f goreplay-$(VERSION)-1.x86_64.rpm
 	fpm $(FPMCOMMON) -a amd64 -t deb ./=/usr/local/bin
 	fpm $(FPMCOMMON) -a amd64 -t rpm ./=/usr/local/bin
 	rm -rf /tmp/gor-build
 
 release-x86:
-	docker run -v `pwd`:$(SOURCE_PATH) -t --env GOOS=linux --env GOARCH=386 -i $(CONTAINER) go build -o gor -tags netgo $(LDFLAGS)
-	tar -czf gor_$(VERSION)$(PREFIX)_x86.tar.gz gor
-	rm gor
+	docker run -v `pwd`:$(SOURCE_PATH) -t --env GOOS=linux --env GOARCH=386 -i $(CONTAINER) go build -o $(BIN_NAME) -tags netgo $(LDFLAGS)
+	tar -czf gor_$(VERSION)$(PREFIX)_x86.tar.gz $(BIN_NAME)
+	rm $(BIN_NAME)
 
 release-mac:
-	go build -o gor $(MAC_LDFLAGS)
-	tar -czf gor_$(VERSION)$(PREFIX)_mac.tar.gz gor
+	go build -o $(BIN_NAME) $(MAC_LDFLAGS)
+	tar -czf gor_$(VERSION)$(PREFIX)_mac.tar.gz $(BIN_NAME)
 	mkdir -p /tmp/gor-build
-	mv ./gor /tmp/gor-build/gor
+	mv ./$(BIN_NAME) /tmp/gor-build/$(BIN_NAME)
 	cd /tmp/gor-build
+	rm -f goreplay-$(VERSION).pkg
 	fpm $(FPMCOMMON) -a amd64 -t osxpkg ./=/usr/local/bin
 	rm -rf /tmp/gor-build
 
@@ -59,7 +66,7 @@ build:
 	docker build -t $(CONTAINER) -f Dockerfile.dev .
 
 profile:
-	go build && ./gor --output-http="http://localhost:9000" --input-dummy 0 --input-raw :9000 --input-http :9000 --memprofile=./mem.out --cpuprofile=./cpu.out --stats --output-http-stats --output-http-timeout 100ms
+	go build && ./$(BIN_NAME) --output-http="http://localhost:9000" --input-dummy 0 --input-raw :9000 --input-http :9000 --memprofile=./mem.out --cpuprofile=./cpu.out --stats --output-http-stats --output-http-timeout 100ms
 
 lint:
 	$(RUN) golint $(PKG)
@@ -123,7 +130,7 @@ bash:
 
 
 FPMCOMMON= \
-    --name goreplay \
+    --name gor \
     --description "GoReplay is an open-source network monitoring tool which can record your live traffic, and use it for shadowing, load testing, monitoring and detailed analysis." \
     -v $(VERSION) \
     --vendor "Leonid Bugaev" \
@@ -134,6 +141,6 @@ FPMCOMMON= \
 
 build_packages:
 	mkdir -p /tmp/gor-build
-	go build -i -o /tmp/gor-build/gor
+	go build -i -o /tmp/gor-build/$(BIN_NAME)
 	fpm $(FPMCOMMON) -a amd64 -t deb ./=/usr/local/bin
 	fpm $(FPMCOMMON) -a amd64 -t rpm ./=/usr/local/bin

byteutils/byteutils.go

@@ -1,6 +1,10 @@
-// Package byteutils probvides helpers for working with byte slices
+// Package byteutils provides helpers for working with byte slices
 package byteutils
 
+import (
+	"unsafe"
+)
+
 // Cut elements from slice for a given range
 func Cut(a []byte, from, to int) []byte {
 	copy(a[from:], a[to:])
@@ -41,3 +45,8 @@ func Replace(a []byte, from, to int, new []byte) []byte {
 	copy(a[from:], new)
 	return a
 }
+
+// SliceToString preferred for large body payload (zero allocation and faster)
+func SliceToString(buf []byte) string {
+	return *(*string)(unsafe.Pointer(&buf))
+}

byteutils/byteutils_test.go

@@ -30,3 +30,12 @@ func TestReplace(t *testing.T) {
 		t.Error("Should replace when replacement length bigger")
 	}
 }
+
+func BenchmarkStringtoSlice(b *testing.B) {
+	var s string
+	var buf [1 << 20]byte
+	for i := 0; i < b.N; i++ {
+		s = SliceToString(buf[:])
+	}
+	_ = s // avoid gc to optimize away the loop body
+}

capture/capture.go

@@ -0,0 +1,500 @@
+package capture
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"net"
+	"os"
+	"runtime"
+	"sync"
+	"time"
+
+	"github.com/buger/goreplay/size"
+	"github.com/google/gopacket"
+	"github.com/google/gopacket/layers"
+	"github.com/google/gopacket/pcap"
+)
+
+// Handler is a function that is used to handle packets
+type Handler func(gopacket.Packet)
+
+// PcapOptions options that can be set on a pcap capture handle,
+// these options take effect on inactive pcap handles
+type PcapOptions struct {
+	BufferTimeout time.Duration `json:"input-raw-buffer-timeout"`
+	TimestampType string        `json:"input-raw-timestamp-type"`
+	BPFFilter     string        `json:"input-raw-bpf-filter"`
+	BufferSize    size.Size     `json:"input-raw-buffer-size"`
+	Promiscuous   bool          `json:"input-raw-promisc"`
+	Monitor       bool          `json:"input-raw-monitor"`
+	Snaplen       bool          `json:"input-raw-override-snaplen"`
+}
+
+// NetInterface represents network interface
+type NetInterface struct {
+	net.Interface
+	IPs []string
+}
+
+// Listener handle traffic capture, this is its representation.
+type Listener struct {
+	sync.Mutex
+	Transport  string       // transport layer default to tcp
+	Activate   func() error // function is used to activate the engine. it must be called before reading packets
+	Handles    map[string]gopacket.PacketDataSource
+	Interfaces []NetInterface
+	loopIndex  int
+	Reading    chan bool // this channel is closed when the listener has started reading packets
+	PcapOptions
+	Engine        EngineType
+	port          uint16 // src or/and dst port
+	trackResponse bool
+
+	host string // pcap file name or interface (name, hardware addr, index or ip address)
+
+	quit    chan bool
+	packets chan gopacket.Packet
+}
+
+// EngineType ...
+type EngineType uint8
+
+// Available engines for intercepting traffic
+const (
+	EnginePcap EngineType = 1 << iota
+	EnginePcapFile
+	EngineRawSocket
+)
+
+// Set is here so that EngineType can implement flag.Var
+func (eng *EngineType) Set(v string) error {
+	switch v {
+	case "", "libpcap":
+		*eng = EnginePcap
+	case "pcap_file":
+		*eng = EnginePcapFile
+	case "raw_socket", "af_packet":
+		*eng = EngineRawSocket
+	default:
+		return fmt.Errorf("invalid engine %s", v)
+	}
+	return nil
+}
+
+func (eng *EngineType) String() (e string) {
+	switch *eng {
+	case EnginePcapFile:
+		e = "pcap_file"
+	case EnginePcap:
+		e = "libpcap"
+	case EngineRawSocket:
+		e = "raw_socket"
+	default:
+		e = ""
+	}
+	return e
+}
+
+// NewListener creates and initialize a new Listener. if transport or/and engine are invalid/unsupported
+// is "tcp" and "pcap", are assumed. l.Engine and l.Transport can help to get the values used.
+// if there is an error it will be associated with getting network interfaces
+func NewListener(host string, port uint16, transport string, engine EngineType, trackResponse bool) (l *Listener, err error) {
+	l = &Listener{}
+
+	l.host = host
+	l.port = port
+	l.Transport = "tcp"
+	if transport != "" {
+		l.Transport = transport
+	}
+	l.Handles = make(map[string]gopacket.PacketDataSource)
+	l.trackResponse = trackResponse
+	l.packets = make(chan gopacket.Packet, 1000)
+	l.quit = make(chan bool, 1)
+	l.Reading = make(chan bool, 1)
+	switch engine {
+	default:
+		l.Engine = EnginePcap
+		l.Activate = l.activatePcap
+	case EngineRawSocket:
+		l.Engine = EngineRawSocket
+		l.Activate = l.activateRawSocket
+	case EnginePcapFile:
+		l.Engine = EnginePcapFile
+		l.Activate = l.activatePcapFile
+		return
+	}
+	err = l.setInterfaces()
+	if err != nil {
+		return nil, err
+	}
+	return
+}
+
+// SetPcapOptions set pcap options for all yet to be actived pcap handles
+// setting this on already activated handles will not have any effect
+func (l *Listener) SetPcapOptions(opts PcapOptions) {
+	l.PcapOptions = opts
+}
+
+// Listen listens for packets from the handles, and call handler on every packet received
+// until the context done signal is sent or EOF on handles.
+// this function should be called after activating pcap handles
+func (l *Listener) Listen(ctx context.Context, handler Handler) (err error) {
+	l.read()
+	done := ctx.Done()
+	var p gopacket.Packet
+	var ok bool
+	for {
+		select {
+		case <-done:
+			l.quit <- true
+			close(l.quit)
+			err = ctx.Err()
+			done = nil
+		case p, ok = <-l.packets:
+			if !ok {
+				return
+			}
+			if p == nil {
+				continue
+			}
+			handler(p)
+		}
+	}
+}
+
+// ListenBackground is like listen but can run concurrently and signal error through channel
+func (l *Listener) ListenBackground(ctx context.Context, handler Handler) chan error {
+	err := make(chan error, 1)
+	go func() {
+		defer close(err)
+		if e := l.Listen(ctx, handler); err != nil {
+			err <- e
+		}
+	}()
+	return err
+}
+
+// Filter returns automatic filter applied by goreplay
+// to a pcap handle of a specific interface
+func (l *Listener) Filter(ifi NetInterface) (filter string) {
+	// https://www.tcpdump.org/manpages/pcap-filter.7.html
+
+	port := fmt.Sprintf("portrange 0-%d", 1<<16-1)
+	if l.port != 0 {
+		port = fmt.Sprintf("port %d", l.port)
+	}
+	dir := " dst " // direction
+	if l.trackResponse {
+		dir = " "
+	}
+	filter = fmt.Sprintf("(%s%s%s)", l.Transport, dir, port)
+	if listenAll(l.host) || isDevice(l.host, ifi) {
+		return
+	}
+	filter = fmt.Sprintf("(%s%s%s and host %s)", l.Transport, dir, port, l.host)
+	return
+}
+
+// PcapDumpHandler returns a handler to write packet data in PCAP
+// format, See http://wiki.wireshark.org/Development/LibpcapFileFormathandler.
+// if link layer is invalid Ethernet is assumed
+func PcapDumpHandler(file *os.File, link layers.LinkType, debugger func(int, ...interface{})) (handler func(packet gopacket.Packet), err error) {
+	if link.String() == "" {
+		link = layers.LinkTypeEthernet
+	}
+	w := NewWriterNanos(file)
+	err = w.WriteFileHeader(64<<10, link)
+	if err != nil {
+		return nil, err
+	}
+	return func(packet gopacket.Packet) {
+		err = w.WritePacket(packet.Metadata().CaptureInfo, packet.Data())
+		if err != nil && debugger != nil {
+			go debugger(3, err)
+		}
+	}, nil
+}
+
+// PcapHandle returns new pcap Handle from dev on success.
+// this function should be called after setting all necessary options for this listener
+func (l *Listener) PcapHandle(ifi NetInterface) (handle *pcap.Handle, err error) {
+	var inactive *pcap.InactiveHandle
+	inactive, err = pcap.NewInactiveHandle(ifi.Name)
+	if inactive != nil && err != nil {
+		defer inactive.CleanUp()
+	}
+	if err != nil {
+		return nil, fmt.Errorf("inactive handle error: %q, interface: %q", err, ifi.Name)
+	}
+	if l.TimestampType != "" {
+		var ts pcap.TimestampSource
+		ts, err = pcap.TimestampSourceFromString(l.TimestampType)
+		err = inactive.SetTimestampSource(ts)
+		if err != nil {
+			return nil, fmt.Errorf("%q: supported timestamps: %q, interface: %q", err, inactive.SupportedTimestamps(), ifi.Name)
+		}
+	}
+	if l.Promiscuous {
+		if err = inactive.SetPromisc(l.Promiscuous); err != nil {
+			return nil, fmt.Errorf("promiscuous mode error: %q, interface: %q", err, ifi.Name)
+		}
+	}
+	if l.Monitor {
+		if err = inactive.SetRFMon(l.Monitor); err != nil && !errors.Is(err, pcap.CannotSetRFMon) {
+			return nil, fmt.Errorf("monitor mode error: %q, interface: %q", err, ifi.Name)
+		}
+	}
+	var snap int
+	if l.Snaplen {
+		snap = 64<<10 + 200
+	} else if ifi.MTU > 0 {
+		snap = ifi.MTU + 200
+	}
+	err = inactive.SetSnapLen(snap)
+	if err != nil {
+		return nil, fmt.Errorf("snapshot length error: %q, interface: %q", err, ifi.Name)
+	}
+	if l.BufferSize > 0 {
+		err = inactive.SetBufferSize(int(l.BufferSize))
+		if err != nil {
+			return nil, fmt.Errorf("handle buffer size error: %q, interface: %q", err, ifi.Name)
+		}
+	}
+	if l.BufferTimeout == 0 {
+		l.BufferTimeout = pcap.BlockForever
+	}
+	err = inactive.SetTimeout(l.BufferTimeout)
+	if err != nil {
+		return nil, fmt.Errorf("handle buffer timeout error: %q, interface: %q", err, ifi.Name)
+	}
+	handle, err = inactive.Activate()
+	if err != nil {
+		return nil, fmt.Errorf("PCAP Activate device error: %q, interface: %q", err, ifi.Name)
+	}
+	if l.BPFFilter != "" {
+		if l.BPFFilter[0] != '(' || l.BPFFilter[len(l.BPFFilter)-1] != ')' {
+			l.BPFFilter = "(" + l.BPFFilter + ")"
+		}
+	} else {
+		l.BPFFilter = l.Filter(ifi)
+	}
+	err = handle.SetBPFFilter(l.BPFFilter)
+	if err != nil {
+		handle.Close()
+		return nil, fmt.Errorf("BPF filter error: %q%s, interface: %q", err, l.BPFFilter, ifi.Name)
+	}
+	return
+}
+
+// SocketHandle returns new unix ethernet handle associated with this listener settings
+func (l *Listener) SocketHandle(ifi NetInterface) (handle Socket, err error) {
+	handle, err = NewSocket(ifi.Interface)
+	if err != nil {
+		return nil, fmt.Errorf("sock raw error: %q, interface: %q", err, ifi.Name)
+	}
+	if err = handle.SetPromiscuous(l.Promiscuous || l.Monitor); err != nil {
+		return nil, fmt.Errorf("promiscuous mode error: %q, interface: %q", err, ifi.Name)
+	}
+	if l.BPFFilter != "" {
+		if l.BPFFilter[0] != '(' || l.BPFFilter[len(l.BPFFilter)-1] != ')' {
+			l.BPFFilter = "(" + l.BPFFilter + ")"
+		}
+	} else {
+		l.BPFFilter = l.Filter(ifi)
+	}
+	if err = handle.SetBPFFilter(l.BPFFilter); err != nil {
+		handle.Close()
+		return nil, fmt.Errorf("BPF filter error: %q%s, interface: %q", err, l.BPFFilter, ifi.Name)
+	}
+	handle.SetLoopbackIndex(int32(l.loopIndex))
+	return
+}
+
+func (l *Listener) read() {
+	l.Lock()
+	defer l.Unlock()
+	for key, handle := range l.Handles {
+		var source *gopacket.PacketSource
+		linkType := layers.LinkTypeEthernet
+		if _, ok := handle.(*pcap.Handle); ok {
+			linkType = handle.(*pcap.Handle).LinkType()
+		}
+		source = gopacket.NewPacketSource(handle, linkType)
+		source.Lazy = true
+		source.NoCopy = true
+		ch := source.Packets()
+		go func(key string) {
+			defer l.closeHandles(key)
+			for {
+				select {
+				case <-l.quit:
+					return
+				case p, ok := <-ch:
+					if !ok {
+						return
+					}
+					l.packets <- p
+				}
+			}
+		}(key)
+	}
+	l.Reading <- true
+	close(l.Reading)
+}
+
+func (l *Listener) closeHandles(key string) {
+	l.Lock()
+	defer l.Unlock()
+	if handle, ok := l.Handles[key]; ok {
+		if _, ok = handle.(Socket); ok {
+			handle.(Socket).Close()
+		} else {
+			handle.(*pcap.Handle).Close()
+		}
+		delete(l.Handles, key)
+		if len(l.Handles) == 0 {
+			close(l.packets)
+		}
+	}
+}
+
+func (l *Listener) activatePcap() error {
+	var e error
+	var msg string
+	for _, ifi := range l.Interfaces {
+		var handle *pcap.Handle
+		handle, e = l.PcapHandle(ifi)
+		if e != nil {
+			msg += ("\n" + e.Error())
+			continue
+		}
+		l.Handles[ifi.Name] = handle
+	}
+	if len(l.Handles) == 0 {
+		return fmt.Errorf("pcap handles error:%s", msg)
+	}
+	return nil
+}
+
+func (l *Listener) activateRawSocket() error {
+	if runtime.GOOS != "linux" {
+		return fmt.Errorf("sock_raw is not stabilized on OS other than linux")
+	}
+	var msg string
+	var e error
+	for _, ifi := range l.Interfaces {
+		var handle Socket
+		handle, e = l.SocketHandle(ifi)
+		if e != nil {
+			msg += ("\n" + e.Error())
+			continue
+		}
+		l.Handles[ifi.Name] = handle
+	}
+	return e
+}
+
+func (l *Listener) activatePcapFile() (err error) {
+	var handle *pcap.Handle
+	var e error
+	if handle, e = pcap.OpenOffline(l.host); e != nil {
+		return fmt.Errorf("open pcap file error: %q", e)
+	}
+	if l.BPFFilter != "" {
+		if l.BPFFilter[0] != '(' || l.BPFFilter[len(l.BPFFilter)-1] != ')' {
+			l.BPFFilter = "(" + l.BPFFilter + ")"
+		}
+	} else {
+		addr := l.host
+		l.host = ""
+		l.BPFFilter = l.Filter(NetInterface{})
+		l.host = addr
+	}
+	if e = handle.SetBPFFilter(l.BPFFilter); e != nil {
+		handle.Close()
+		return fmt.Errorf("BPF filter error: %q, filter: %s", e, l.BPFFilter)
+	}
+	l.Handles["pcap_file"] = handle
+	return
+}
+
+func (l *Listener) setInterfaces() (err error) {
+	var Ifis []NetInterface
+	var ifis []net.Interface
+	ifis, err = net.Interfaces()
+	if err != nil {
+		return err
+	}
+
+	for i := 0; i < len(ifis); i++ {
+		if ifis[i].Flags&net.FlagLoopback != 0 {
+			l.loopIndex = ifis[i].Index
+		}
+		if ifis[i].Flags&net.FlagUp == 0 {
+			continue
+		}
+		var addrs []net.Addr
+		addrs, err = ifis[i].Addrs()
+		if err != nil {
+			return err
+		}
+		if len(addrs) == 0 {
+			continue
+		}
+		ifi := NetInterface{}
+		ifi.Interface = ifis[i]
+		ifi.IPs = make([]string, len(addrs))
+		for j, addr := range addrs {
+			ifi.IPs[j] = cutMask(addr)
+		}
+		Ifis = append(Ifis, ifi)
+	}
+
+	if listenAll(l.host) {
+		l.Interfaces = Ifis
+		return
+	}
+	found := false
+	for _, ifi := range Ifis {
+		if isDevice(l.host, ifi) {
+			found = true
+		}
+		for _, ip := range ifi.IPs {
+			if ip == l.host {
+				found = true
+				break
+			}
+		}
+		if found {
+			l.Interfaces = []NetInterface{ifi}
+			return
+		}
+	}
+	err = fmt.Errorf("can not find interface with addr, name or index %s", l.host)
+	return err
+}
+
+func cutMask(addr net.Addr) string {
+	mask := addr.String()
+	for i, v := range mask {
+		if v == '/' {
+			return mask[:i]
+		}
+	}
+	return mask
+}
+
+func isDevice(addr string, ifi NetInterface) bool {
+	return addr == ifi.Name || addr == fmt.Sprintf("%d", ifi.Index) || addr == ifi.HardwareAddr.String()
+}
+
+func listenAll(addr string) bool {
+	switch addr {
+	case "", "0.0.0.0", "[::]", "::":
+		return true
+	}
+	return false
+}

capture/capture_test.go

@@ -0,0 +1,360 @@
+package capture
+
+import (
+	"context"
+	"encoding/binary"
+	"io/ioutil"
+	"net"
+	"os"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/google/gopacket"
+	"github.com/google/gopacket/layers"
+	"github.com/google/gopacket/pcap"
+)
+
+var LoopBack = func() net.Interface {
+	ifis, _ := net.Interfaces()
+	for _, v := range ifis {
+		if v.Flags&net.FlagLoopback != 0 {
+			return v
+		}
+	}
+	return ifis[0]
+}()
+
+func TestSetInterfaces(t *testing.T) {
+	l := &Listener{}
+	l.host = "127.0.0.1"
+	l.setInterfaces()
+	if len(l.Interfaces) != 1 {
+		t.Error("expected a single interface")
+	}
+	l.host = LoopBack.HardwareAddr.String()
+	l.setInterfaces()
+	if l.Interfaces[0].Name != LoopBack.Name && len(l.Interfaces) != 1 {
+		t.Error("interface should be loop back interface")
+	}
+	l.host = ""
+	l.setInterfaces()
+	if len(l.Interfaces) < 1 {
+		t.Error("should get all interfaces")
+	}
+}
+
+func TestBPFFilter(t *testing.T) {
+	l := &Listener{}
+	l.host = "127.0.0.1"
+	l.Transport = "tcp"
+	l.setInterfaces()
+	filter := l.Filter(l.Interfaces[0])
+	if filter != "(tcp dst portrange 0-65535 and host 127.0.0.1)" {
+		t.Error("wrong filter", filter)
+	}
+	l.port = 8000
+	l.trackResponse = true
+	filter = l.Filter(l.Interfaces[0])
+	if filter != "(tcp port 8000 and host 127.0.0.1)" {
+		t.Error("wrong filter")
+	}
+}
+
+var decodeOpts = gopacket.DecodeOptions{Lazy: true, NoCopy: true}
+
+func generateHeaders(seq uint32, length uint16) (headers [44]byte) {
+	// set ethernet headers
+	binary.BigEndian.PutUint32(headers[0:4], uint32(layers.ProtocolFamilyIPv4))
+
+	// set ip header
+	ip := headers[4:]
+	copy(ip[0:2], []byte{4<<4 | 5, 0x28<<2 | 0x00})
+	binary.BigEndian.PutUint16(ip[2:4], length+54)
+	ip[9] = uint8(layers.IPProtocolTCP)
+	copy(ip[12:16], []byte{127, 0, 0, 1})
+	copy(ip[16:], []byte{127, 0, 0, 1})
+
+	// set tcp header
+	tcp := ip[20:]
+	binary.BigEndian.PutUint16(tcp[0:2], 45678)
+	binary.BigEndian.PutUint16(tcp[2:4], 8000)
+	tcp[12] = 5 << 4
+	return
+}
+
+func randomPackets(start uint32, _len int, length uint16) []gopacket.Packet {
+	var packets = make([]gopacket.Packet, _len)
+	for i := start; i < start+uint32(_len); i++ {
+		h := generateHeaders(i, length)
+		d := make([]byte, int(length)+len(h))
+		copy(d, h[0:])
+		packet := gopacket.NewPacket(d, layers.LinkTypeLoop, decodeOpts)
+		packets[i-start] = packet
+		inf := packets[i-start].Metadata()
+		_len := len(d)
+		inf.CaptureInfo = gopacket.CaptureInfo{CaptureLength: _len, Length: _len, Timestamp: time.Now()}
+	}
+	return packets
+}
+
+func TestPcapDump(t *testing.T) {
+	f, err := ioutil.TempFile("", "pcap_file")
+	if err != nil {
+		t.Error(err)
+	}
+	waiter := make(chan bool, 1)
+	h, _ := PcapDumpHandler(f, layers.LinkTypeLoop, func(level int, a ...interface{}) {
+		if level != 3 {
+			t.Errorf("expected debug level to be 3, got %d", level)
+		}
+		waiter <- true
+	})
+	packets := randomPackets(1, 5, 5)
+	for i := 0; i < len(packets); i++ {
+		if i == 1 {
+			tcp := packets[i].Data()[4:][20:]
+			// change dst port
+			binary.BigEndian.PutUint16(tcp[2:], 8001)
+		}
+		if i == 4 {
+			inf := packets[i].Metadata()
+			inf.CaptureLength = 40
+		}
+		h(packets[i])
+	}
+	<-waiter
+	name := f.Name()
+	f.Close()
+	testPcapDumpEngine(name, t)
+}
+
+func testPcapDumpEngine(f string, t *testing.T) {
+	defer os.Remove(f)
+	l, err := NewListener(f, 8000, "", EnginePcapFile, true)
+	err = l.Activate()
+	if err != nil {
+		t.Errorf("expected error to be nil, got %q", err)
+		return
+	}
+	pckts := 0
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	err = l.Listen(ctx, func(packet gopacket.Packet) {
+		if packet.Metadata().CaptureLength != 49 {
+			t.Errorf("expected packet length to be %d, got %d", 49, packet.Metadata().CaptureLength)
+		}
+		pckts++
+	})
+
+	if err != nil {
+		t.Errorf("expected error to be nil, got %q", err)
+	}
+	if pckts != 3 {
+		t.Errorf("expected %d packets, got %d packets", 3, pckts)
+	}
+}
+
+func TestPcapHandler(t *testing.T) {
+	l, err := NewListener(LoopBack.Name, 8000, "", EnginePcap, true)
+	if err != nil {
+		t.Errorf("expected error to be nil, got %v", err)
+		return
+	}
+	err = l.Activate()
+	if err != nil {
+		t.Errorf("expected error to be nil, got %v", err)
+		return
+	}
+	defer l.Handles[LoopBack.Name].(*pcap.Handle).Close()
+	if err != nil {
+		t.Errorf("expected error to be nil, got %v", err)
+		return
+	}
+	for i := 0; i < 5; i++ {
+		_, _ = net.Dial("tcp", "127.0.0.1:8000")
+	}
+	sts, _ := l.Handles[LoopBack.Name].(*pcap.Handle).Stats()
+	if sts.PacketsReceived < 5 {
+		t.Errorf("expected >=5 packets got %d", sts.PacketsReceived)
+	}
+}
+
+func TestSocketHandler(t *testing.T) {
+	l, err := NewListener(LoopBack.Name, 8000, "", EngineRawSocket, true)
+	err = l.Activate()
+	if err != nil {
+		return
+	}
+	defer l.Handles[LoopBack.Name].(*SockRaw).Close()
+	if err != nil {
+		t.Errorf("expected error to be nil, got %v", err)
+		return
+	}
+	for i := 0; i < 5; i++ {
+		_, _ = net.Dial("tcp", "127.0.0.1:8000")
+	}
+	sts, _ := l.Handles[LoopBack.Name].(*SockRaw).Stats()
+	if sts.Packets < 5 {
+		t.Errorf("expected >=5 packets got %d", sts.Packets)
+	}
+}
+
+func BenchmarkPcapDump(b *testing.B) {
+	f, err := ioutil.TempFile("", "pcap_file")
+	if err != nil {
+		b.Error(err)
+		return
+	}
+	now := time.Now()
+	defer os.Remove(f.Name())
+	h, _ := PcapDumpHandler(f, layers.LinkTypeLoop, nil)
+	packets := randomPackets(1, b.N, 5)
+	for i := 0; i < len(packets); i++ {
+		h(packets[i])
+	}
+	f.Close()
+	b.Logf("%d packets in %s", b.N, time.Since(now))
+}
+
+func BenchmarkPcapFile(b *testing.B) {
+	f, err := ioutil.TempFile("", "pcap_file")
+	if err != nil {
+		b.Error(err)
+		return
+	}
+	defer os.Remove(f.Name())
+	h, _ := PcapDumpHandler(f, layers.LinkTypeLoop, nil)
+	packets := randomPackets(1, b.N, 5)
+	for i := 0; i < len(packets); i++ {
+		h(packets[i])
+	}
+	name := f.Name()
+	f.Close()
+	b.ResetTimer()
+	var l *Listener
+	l, err = NewListener(name, 8000, "", EnginePcapFile, true)
+	if err != nil {
+		b.Error(err)
+		return
+	}
+	err = l.Activate()
+	if err != nil {
+		b.Error(err)
+		return
+	}
+	now := time.Now()
+	pckts := 0
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	if err = l.Listen(ctx, func(packet gopacket.Packet) {
+		if packet.Metadata().CaptureLength != 49 {
+			b.Errorf("expected packet length to be %d, got %d", 49, packet.Metadata().CaptureLength)
+		}
+		pckts++
+	}); err != nil {
+		b.Error(err)
+	}
+	b.Logf("%d/%d packets in %s", pckts, b.N, time.Since(now))
+}
+
+// used to benchmark sock engine
+var buf [1024]byte
+
+func init() {
+	for i := 0; i < len(buf); i++ {
+		buf[i] = 0xff
+	}
+}
+
+func handler(n, counter *int32) Handler {
+	return func(p gopacket.Packet) {
+		nn := int32(len(p.Data()))
+		atomic.AddInt32(n, nn)
+		atomic.AddInt32(counter, 1)
+	}
+}
+
+func BenchmarkPcap(b *testing.B) {
+	var err error
+	n := new(int32)
+	counter := new(int32)
+	l, err := NewListener(LoopBack.Name, 8000, "", EnginePcap, false)
+	if err != nil {
+		b.Error(err)
+		return
+	}
+	l.PcapOptions.BPFFilter = "udp dst port 8000 and host 127.0.0.1"
+	err = l.Activate()
+	if err != nil {
+		b.Error(err)
+		return
+	}
+	errCh := l.ListenBackground(context.Background(), handler(n, counter))
+	select {
+	case <-l.Reading:
+	case err = <-errCh:
+		b.Error(err)
+		return
+	}
+	var conn net.Conn
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		b.StopTimer()
+		conn, err = net.Dial("udp", "127.0.0.1:8000")
+		if err != nil {
+			b.Error(err)
+			return
+		}
+		b.StartTimer()
+		_, err = conn.Write(buf[:])
+		if err != nil {
+			b.Error(err)
+			return
+		}
+	}
+	b.ReportMetric(float64(atomic.LoadInt32(n)), "buf")
+	b.ReportMetric(float64(atomic.LoadInt32(counter)), "packets")
+}
+
+func BenchmarkRawSocket(b *testing.B) {
+	var err error
+	n := new(int32)
+	counter := new(int32)
+	l, err := NewListener(LoopBack.Name, 8000, "", EngineRawSocket, false)
+	if err != nil {
+		b.Error(err)
+		return
+	}
+	l.PcapOptions.BPFFilter = "udp dst port 8000 and host 127.0.0.1"
+	err = l.Activate()
+	if err != nil {
+		b.Error(err)
+		return
+	}
+	errCh := l.ListenBackground(context.Background(), handler(n, counter))
+	select {
+	case <-l.Reading:
+	case err = <-errCh:
+		b.Error(err)
+		return
+	}
+	var conn net.Conn
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		b.StopTimer()
+		conn, err = net.Dial("udp", "127.0.0.1:8000")
+		if err != nil {
+			b.Error(err)
+			return
+		}
+		b.StartTimer()
+		_, err = conn.Write(buf[:])
+		if err != nil {
+			b.Error(err)
+			return
+		}
+	}
+	b.ReportMetric(float64(atomic.LoadInt32(n)), "buf")
+	b.ReportMetric(float64(atomic.LoadInt32(counter)), "packets")
+}

capture/doc.go

@@ -0,0 +1,35 @@
+/*
+Package capture provides traffic sniffier using AF_PACKET, pcap or pcap file.
+it allows you to listen for traffic from any port (e.g. sniffing) because they operate on IP level.
+Ports is TCP/IP feature, same as flow control, reliable transmission and etc.
+Currently this package implements TCP layer: flow control is managed under tcp package.
+BPF filters can also be applied.
+
+example:
+
+// for the transport should be "tcp"
+listener, err := capture.NewListener(host, port, transport, engine, trackResponse)
+if err != nil {
+	// handle error
+}
+listener.SetPcapOptions(opts)
+err = listner.Activate()
+if err != nil {
+	// handle it
+}
+
+if err := listener.Listen(context.Background(), handler); err != nil {
+	 // handle error
+}
+// or
+errCh := listener.ListenBackground(context.Background(), handler) // runs in the backgorund
+select {
+case err := <- errCh:
+	// handle error
+case <-quit:
+	//
+case <- l.Reading: // if we have started reading
+}
+
+*/
+package capture // import github.com/buger/goreplay/capture

capture/dump.go

@@ -0,0 +1,126 @@
+// https://github.com/google/gopacket/blob/403ca653c4/pcapgo/read.go
+
+package capture
+
+import (
+	"encoding/binary"
+	"fmt"
+	"io"
+	"time"
+
+	"github.com/google/gopacket"
+	"github.com/google/gopacket/layers"
+)
+
+// Writer wraps an underlying io.Writer to write packet data in PCAP
+// format.  See http://wiki.wireshark.org/Development/LibpcapFileFormat
+// for information on the file format.
+//
+// For those that care, we currently write v2.4 files with nanosecond
+// or microsecond timestamp resolution and little-endian encoding.
+type Writer struct {
+	w        io.Writer
+	tsScaler int
+	// Moving this into the struct seems to save an allocation for each call to writePacketHeader
+	buf [16]byte
+}
+
+const magicNanoseconds = 0xA1B23C4D
+const magicMicroseconds = 0xA1B2C3D4
+const versionMajor = 2
+const versionMinor = 4
+
+// NewWriterNanos returns a new writer object, for writing packet data out
+// to the given writer.  If this is a new empty writer (as opposed to
+// an append), you must call WriteFileHeader before WritePacket.  Packet
+// timestamps are written with nanosecond precision.
+//
+//  // Write a new file:
+//  f, _ := os.Create("/tmp/file.pcap")
+//  w := pcapgo.NewWriterNanos(f)
+//  w.WriteFileHeader(65536, layers.LinkTypeEthernet)  // new file, must do this.
+//  w.WritePacket(gopacket.CaptureInfo{...}, data1)
+//  f.Close()
+//  // Append to existing file (must have same snaplen and linktype)
+//  f2, _ := os.OpenFile("/tmp/fileNano.pcap", os.O_APPEND, 0700)
+//  w2 := pcapgo.NewWriter(f2)
+//  // no need for file header, it's already written.
+//  w2.WritePacket(gopacket.CaptureInfo{...}, data2)
+//  f2.Close()
+func NewWriterNanos(w io.Writer) *Writer {
+	return &Writer{w: w, tsScaler: nanosPerNano}
+}
+
+// NewWriter returns a new writer object, for writing packet data out
+// to the given writer.  If this is a new empty writer (as opposed to
+// an append), you must call WriteFileHeader before WritePacket.
+// Packet timestamps are written witn microsecond precision.
+//
+//  // Write a new file:
+//  f, _ := os.Create("/tmp/file.pcap")
+//  w := pcapgo.NewWriter(f)
+//  w.WriteFileHeader(65536, layers.LinkTypeEthernet)  // new file, must do this.
+//  w.WritePacket(gopacket.CaptureInfo{...}, data1)
+//  f.Close()
+//  // Append to existing file (must have same snaplen and linktype)
+//  f2, _ := os.OpenFile("/tmp/file.pcap", os.O_APPEND, 0700)
+//  w2 := pcapgo.NewWriter(f2)
+//  // no need for file header, it's already written.
+//  w2.WritePacket(gopacket.CaptureInfo{...}, data2)
+//  f2.Close()
+func NewWriter(w io.Writer) *Writer {
+	return &Writer{w: w, tsScaler: nanosPerMicro}
+}
+
+// WriteFileHeader writes a file header out to the writer.
+// This must be called exactly once per output.
+func (w *Writer) WriteFileHeader(snaplen uint32, linktype layers.LinkType) error {
+	var buf [24]byte
+	if w.tsScaler == nanosPerMicro {
+		binary.LittleEndian.PutUint32(buf[0:4], magicMicroseconds)
+	} else {
+		binary.LittleEndian.PutUint32(buf[0:4], magicNanoseconds)
+	}
+	binary.LittleEndian.PutUint16(buf[4:6], versionMajor)
+	binary.LittleEndian.PutUint16(buf[6:8], versionMinor)
+	// bytes 8:12 stay 0 (timezone = UTC)
+	// bytes 12:16 stay 0 (sigfigs is always set to zero, according to
+	//   http://wiki.wireshark.org/Development/LibpcapFileFormat
+	binary.LittleEndian.PutUint32(buf[16:20], snaplen)
+	binary.LittleEndian.PutUint32(buf[20:24], uint32(linktype))
+	_, err := w.w.Write(buf[:])
+	return err
+}
+
+const nanosPerMicro = 1000
+const nanosPerNano = 1
+
+func (w *Writer) writePacketHeader(ci gopacket.CaptureInfo) error {
+	t := ci.Timestamp
+	if t.IsZero() {
+		t = time.Now()
+	}
+	secs := t.Unix()
+	usecs := t.Nanosecond() / w.tsScaler
+	binary.LittleEndian.PutUint32(w.buf[0:4], uint32(secs))
+	binary.LittleEndian.PutUint32(w.buf[4:8], uint32(usecs))
+	binary.LittleEndian.PutUint32(w.buf[8:12], uint32(ci.CaptureLength))
+	binary.LittleEndian.PutUint32(w.buf[12:16], uint32(ci.Length))
+	_, err := w.w.Write(w.buf[:])
+	return err
+}
+
+// WritePacket writes the given packet data out to the file.
+func (w *Writer) WritePacket(ci gopacket.CaptureInfo, data []byte) error {
+	if ci.CaptureLength != len(data) {
+		return fmt.Errorf("capture length %d does not match data length %d", ci.CaptureLength, len(data))
+	}
+	if ci.CaptureLength > ci.Length {
+		return fmt.Errorf("invalid capture info %+v:  capture length > length", ci)
+	}
+	if err := w.writePacketHeader(ci); err != nil {
+		return fmt.Errorf("error writing packet header: %v", err)
+	}
+	_, err := w.w.Write(data)
+	return err
+}

capture/listener.go

@@ -1,904 +0,0 @@
-/*
-Package capture provides traffic sniffier using RAW sockets.
-Capture traffic from socket using RAW_SOCKET's http://en.wikipedia.org/wiki/Raw_socket
-RAW_SOCKET allows you to listen for traffic from any port (e.g. sniffing) because they operate on IP level.
-Ports is TCP feature, same as flow control, reliable transmission and etc.
-This package implements own TCP layer: TCP packets is parsed using tcp_packet.go, and flow control is managed by tcp_message.go
-*/
-package capture
-
-import (
-	"bytes"
-	"encoding/binary"
-	"fmt"
-	"io"
-	"log"
-	"net"
-	"runtime"
-	"runtime/debug"
-	"strconv"
-	"strings"
-	"sync"
-	"time"
-
-	"github.com/buger/goreplay/proto"
-
-	"github.com/google/gopacket"
-	"github.com/google/gopacket/layers"
-	"github.com/google/gopacket/pcap"
-)
-
-type packet struct {
-	srcIP     []byte
-	data      []byte
-	timestamp time.Time
-}
-
-// Listener handle traffic capture
-type Listener struct {
-	sync.Mutex
-	// buffer of TCPMessages waiting to be send
-	// ID -> TCPMessage
-	messages map[tcpID]*TCPMessage
-
-	// Expect: 100-continue request is send in 2 tcp messages
-	// We store ACK aliases to merge this packets together
-	ackAliases map[uint32]uint32
-	// To get ACK of second message we need to compute its Seq and wait for them message
-	seqWithData map[uint32]uint32
-
-	// Ack -> Req
-	respAliases map[uint32]*TCPMessage
-
-	// Ack -> ID
-	respWithoutReq map[uint32]tcpID
-
-	// Messages ready to be send to client
-	packetsChan chan *packet
-
-	// Messages ready to be send to client
-	messagesChan chan *TCPMessage
-
-	addr string // IP to listen
-	port uint16 // Port to listen
-
-	trackResponse bool
-	messageExpire time.Duration
-
-	bpfFilter       string
-	timestampType   string
-	overrideSnapLen bool
-	immediateMode   bool
-
-	bufferSize int64
-
-	conn        net.PacketConn
-	pcapHandles []*pcap.Handle
-
-	quit  chan bool
-	ready bool
-
-	protocol TCPProtocol
-}
-
-type request struct {
-	id    tcpID
-	start time.Time
-	ack   uint32
-}
-
-// Available engines for intercepting traffic
-const (
-	EngineRawSocket = 1 << iota
-	EnginePcap
-	EnginePcapFile
-)
-
-// NewListener creates and initializes new Listener object
-func NewListener(addr string, port string, engine int, trackResponse bool, expire time.Duration, protocol TCPProtocol, bpfFilter string, timestampType string, bufferSize int64, overrideSnapLen bool, immediateMode bool) (l *Listener) {
-	l = &Listener{}
-
-	l.packetsChan = make(chan *packet, 10000)
-	l.messagesChan = make(chan *TCPMessage, 10000)
-	l.quit = make(chan bool)
-
-	l.messages = make(map[tcpID]*TCPMessage)
-	l.ackAliases = make(map[uint32]uint32)
-	l.seqWithData = make(map[uint32]uint32)
-	l.respAliases = make(map[uint32]*TCPMessage)
-	l.respWithoutReq = make(map[uint32]tcpID)
-	l.trackResponse = trackResponse
-	l.protocol = protocol
-	l.bpfFilter = bpfFilter
-	l.timestampType = timestampType
-	l.immediateMode = immediateMode
-	l.bufferSize = bufferSize
-	l.overrideSnapLen = overrideSnapLen
-
-	l.addr = addr
-	_port, _ := strconv.Atoi(port)
-	l.port = uint16(_port)
-
-	if expire.Nanoseconds() == 0 {
-		expire = 2000 * time.Millisecond
-	}
-
-	l.messageExpire = expire
-
-	go l.listen()
-
-	// Special case for testing
-	if l.port != 0 {
-		switch engine {
-		case EnginePcap:
-			go l.readPcap()
-		case EnginePcapFile:
-			go l.readPcapFile()
-		case EngineRawSocket:
-			go l.readRAWSocket()
-		default:
-			log.Fatal("Unknown traffic interception engine:", engine)
-		}
-	}
-
-	return
-}
-
-func (t *Listener) listen() {
-	gcTicker := time.Tick(t.messageExpire / 2)
-
-	for {
-		select {
-		case <-t.quit:
-			if t.conn != nil {
-				t.conn.Close()
-			}
-			return
-		case packet := <-t.packetsChan:
-			tcpPacket := ParseTCPPacket(packet.srcIP, packet.data, packet.timestamp)
-			t.processTCPPacket(tcpPacket)
-		case <-gcTicker:
-			now := time.Now()
-
-			// Dispatch requests before responses
-			for _, message := range t.messages {
-				if now.Sub(message.End) >= t.messageExpire {
-					t.dispatchMessage(message)
-				}
-			}
-		}
-	}
-}
-
-func (t *Listener) deleteMessage(message *TCPMessage) {
-	delete(t.messages, message.ID())
-	delete(t.ackAliases, message.Ack)
-	if message.DataAck != 0 {
-		delete(t.ackAliases, message.DataAck)
-	}
-	if message.DataSeq != 0 {
-		delete(t.seqWithData, message.DataSeq)
-	}
-
-	delete(t.respAliases, message.ResponseAck)
-}
-
-func (t *Listener) dispatchMessage(message *TCPMessage) {
-	// If already dispatched
-	if _, ok := t.messages[message.ID()]; !ok {
-		return
-	}
-
-	t.deleteMessage(message)
-
-	if t.protocol == ProtocolHTTP && !message.complete {
-		if !message.IsIncoming {
-			delete(t.respAliases, message.Ack)
-			delete(t.respWithoutReq, message.Ack)
-		}
-
-		return
-	}
-
-	if message.IsIncoming {
-		// If there were response before request
-		// log.Println("Looking for Response: ", t.respWithoutReq, message.ResponseAck)
-		if t.trackResponse {
-			if respID, ok := t.respWithoutReq[message.ResponseAck]; ok {
-				if resp, rok := t.messages[respID]; rok {
-					// if resp.AssocMessage == nil {
-					// log.Println("FOUND RESPONSE")
-					resp.setAssocMessage(message)
-					message.setAssocMessage(resp)
-
-					if resp.complete {
-						defer t.dispatchMessage(resp)
-					}
-					// }
-				}
-			}
-
-			if resp, ok := t.messages[message.ResponseID]; ok {
-				resp.setAssocMessage(message)
-			}
-		}
-	} else {
-		if message.AssocMessage == nil {
-			if responseRequest, ok := t.respAliases[message.Ack]; ok {
-				message.setAssocMessage(responseRequest)
-				responseRequest.setAssocMessage(message)
-			}
-		}
-
-		delete(t.respAliases, message.Ack)
-		delete(t.respWithoutReq, message.Ack)
-
-		// Do not track responses which have no associated requests
-		if message.AssocMessage == nil {
-			// log.Println("Can't dispatch resp", message.Seq, message.Ack, string(message.Bytes()))
-			return
-		}
-	}
-
-	t.messagesChan <- message
-}
-
-// DeviceNotFoundError raised if user specified wrong ip
-type DeviceNotFoundError struct {
-	addr string
-}
-
-func (e *DeviceNotFoundError) Error() string {
-	devices, _ := pcap.FindAllDevs()
-
-	if len(devices) == 0 {
-		return "Can't get list of network interfaces, ensure that you running Gor as root user or sudo.\nTo run as non-root users see this docs https://github.com/buger/goreplay/wiki/Running-as-non-root-user"
-	}
-
-	var msg string
-	msg += "Can't find interfaces with addr: " + e.addr + ". Provide available IP for intercepting traffic: \n"
-	for _, device := range devices {
-		msg += "Name: " + device.Name + "\n"
-		if device.Description != "" {
-			msg += "Description: " + device.Description + "\n"
-		}
-		for _, address := range device.Addresses {
-			msg += "- IP address: " + address.IP.String() + "\n"
-		}
-	}
-
-	return msg
-}
-
-func isLoopback(device pcap.Interface) bool {
-	if len(device.Addresses) == 0 {
-		return false
-	}
-
-	switch device.Addresses[0].IP.String() {
-	case "127.0.0.1", "::1":
-		return true
-	}
-
-	return false
-}
-
-func listenAllInterfaces(addr string) bool {
-	switch addr {
-	case "", "0.0.0.0", "[::]", "::":
-		return true
-	default:
-		return false
-	}
-}
-
-func findPcapDevices(addr string) (interfaces []pcap.Interface, err error) {
-	devices, err := pcap.FindAllDevs()
-	if err != nil {
-		log.Fatal(err)
-	}
-
-	for _, device := range devices {
-		if listenAllInterfaces(addr) && len(device.Addresses) > 0 || isLoopback(device) {
-			interfaces = append(interfaces, device)
-			continue
-		}
-
-		for _, address := range device.Addresses {
-			if device.Name == addr || address.IP.String() == addr {
-				interfaces = append(interfaces, device)
-				return interfaces, nil
-			}
-		}
-	}
-
-	if len(interfaces) == 0 {
-		return nil, &DeviceNotFoundError{addr}
-	}
-
-	return interfaces, nil
-}
-
-func (t *Listener) readPcap() {
-	devices, err := findPcapDevices(t.addr)
-	if err != nil {
-		log.Fatal(err)
-	}
-
-	const BPFSupported = runtime.GOOS != "darwin"
-
-	var wg sync.WaitGroup
-	wg.Add(len(devices))
-
-	for _, d := range devices {
-		go func(device pcap.Interface) {
-			inactive, err := pcap.NewInactiveHandle(device.Name)
-			if err != nil {
-				inactive.CleanUp()
-				log.Println("Pcap Error while opening device", device.Name, err)
-				wg.Done()
-				return
-			}
-
-			if t.timestampType != "" {
-				if tt, terr := pcap.TimestampSourceFromString(t.timestampType); terr != nil {
-					log.Println("Supported timestamp types: ", inactive.SupportedTimestamps(), device.Name)
-				} else if terr := inactive.SetTimestampSource(tt); terr != nil {
-					log.Println("Supported timestamp types: ", inactive.SupportedTimestamps(), device.Name)
-				}
-			}
-
-			if it, err := net.InterfaceByName(device.Name); err == nil && !t.overrideSnapLen {
-				// Auto-guess max length of packet to capture
-				inactive.SetSnapLen(it.MTU + 68*2)
-			} else {
-				inactive.SetSnapLen(65536)
-			}
-			inactive.SetTimeout(t.messageExpire)
-			inactive.SetPromisc(true)
-			inactive.SetImmediateMode(t.immediateMode)
-			if t.immediateMode {
-				log.Println("Setting immediate mode")
-			}
-			if t.bufferSize > 0 {
-				inactive.SetBufferSize(int(t.bufferSize))
-			}
-
-			handle, herr := inactive.Activate()
-			if herr != nil {
-				log.Printf("PCAP Activate device '%s' error: %s\n", device.Name, herr)
-				wg.Done()
-				return
-			}
-
-			defer handle.Close()
-
-			t.Lock()
-			t.pcapHandles = append(t.pcapHandles, handle)
-
-			var bpfDstHost, bpfSrcHost string
-			var loopback = isLoopback(device)
-
-			if loopback {
-				var allAddr []string
-				for _, dc := range devices {
-					for _, addr := range dc.Addresses {
-						allAddr = append(allAddr, "(dst host "+addr.IP.String()+" and src host "+addr.IP.String()+")")
-					}
-				}
-
-				bpfDstHost = strings.Join(allAddr, " or ")
-				bpfSrcHost = bpfDstHost
-			} else {
-				for i, addr := range device.Addresses {
-					bpfDstHost += "dst host " + addr.IP.String()
-					bpfSrcHost += "src host " + addr.IP.String()
-					if i != len(device.Addresses)-1 {
-						bpfDstHost += " or "
-						bpfSrcHost += " or "
-					}
-				}
-			}
-
-			if BPFSupported {
-				var bpf string
-
-				if t.bpfFilter != "" {
-					bpf = t.bpfFilter
-				} else {
-					if t.trackResponse {
-						bpf = fmt.Sprintf("(tcp dst port %d and (%s)) or (tcp src port %d and (%s))", t.port, bpfDstHost, t.port, bpfSrcHost)
-					} else {
-						bpf = fmt.Sprintf("(tcp dst port %d and (%s))", t.port, bpfDstHost)
-					}
-				}
-
-				if err := handle.SetBPFFilter(bpf); err != nil {
-					log.Println("BPF filter error:", err, "Device:", device.Name, bpf)
-					wg.Done()
-					return
-				}
-			}
-			t.Unlock()
-
-			var decoder gopacket.Decoder
-
-			// Special case for tunnel interface https://github.com/google/gopacket/issues/99
-			if handle.LinkType() == 12 {
-				decoder = layers.LayerTypeIPv4
-			} else {
-				decoder = handle.LinkType()
-			}
-
-			source := gopacket.NewPacketSource(handle, decoder)
-			source.Lazy = true
-			source.NoCopy = true
-
-			wg.Done()
-
-			var data, srcIP, dstIP []byte
-
-			for {
-				packet, err := source.NextPacket()
-
-				if err == io.EOF {
-					break
-				} else if err != nil {
-					continue
-				}
-
-				// We should remove network layer before parsing TCP/IP data
-				var of int
-				switch decoder {
-				case layers.LinkTypeEthernet:
-					of = 14
-				case layers.LinkTypePPP:
-					of = 1
-				case layers.LinkTypeFDDI:
-					of = 13
-				case layers.LinkTypeNull:
-					of = 4
-				case layers.LinkTypeLoop:
-					of = 4
-				case layers.LinkTypeRaw, layers.LayerTypeIPv4:
-					of = 0
-				case layers.LinkTypeLinuxSLL:
-					of = 16
-				default:
-					log.Println("Unknown packet layer", decoder, packet)
-					break
-				}
-				data = packet.Data()[of:]
-
-				version := uint8(data[0]) >> 4
-				ipLength := int(binary.BigEndian.Uint16(data[2:4]))
-
-				if version == 4 {
-					ihl := uint8(data[0]) & 0x0F
-
-					// Truncated IP info
-					if len(data) < int(ihl*4) {
-						continue
-					}
-
-					srcIP = data[12:16]
-					dstIP = data[16:20]
-
-					// Too small IP packet
-					if ipLength < 20 {
-						continue
-					}
-
-					// Invalid length
-					if int(ihl*4) > ipLength {
-						continue
-					}
-
-					if cmp := len(data) - ipLength; cmp > 0 {
-						data = data[:ipLength]
-					} else if cmp < 0 {
-						// Truncated packet
-						continue
-					}
-
-					data = data[ihl*4:]
-				} else {
-					// Truncated IP info
-					if len(data) < 40 {
-						continue
-					}
-
-					srcIP = data[8:24]
-					dstIP = data[24:40]
-
-					data = data[40:]
-				}
-
-				// Truncated TCP info
-				if len(data) <= 13 {
-					continue
-				}
-
-				dataOffset := data[12] >> 4
-				isFIN := data[13]&0x01 != 0
-
-				// We need only packets with data inside
-				// Check that the buffer is larger than the size of the TCP header
-				if len(data) > int(dataOffset*4) || isFIN {
-					if !BPFSupported {
-						destPort := binary.BigEndian.Uint16(data[2:4])
-						srcPort := binary.BigEndian.Uint16(data[0:2])
-
-						var addrCheck []byte
-
-						if destPort == t.port {
-							addrCheck = dstIP
-						}
-
-						if t.trackResponse && srcPort == t.port {
-							addrCheck = srcIP
-						}
-
-						if len(addrCheck) == 0 {
-							continue
-						}
-
-						addrMatched := false
-
-						if loopback {
-							for _, dc := range devices {
-								if addrMatched {
-									break
-								}
-								for _, a := range dc.Addresses {
-									if a.IP.Equal(net.IP(addrCheck)) {
-										addrMatched = true
-										break
-									}
-								}
-							}
-							addrMatched = true
-						} else {
-							for _, a := range device.Addresses {
-								if a.IP.Equal(net.IP(addrCheck)) {
-									addrMatched = true
-									break
-								}
-							}
-						}
-
-						if !addrMatched {
-							continue
-						}
-					}
-
-					t.packetsChan <- t.buildPacket(srcIP, data, packet.Metadata().Timestamp)
-				}
-			}
-		}(d)
-	}
-
-	wg.Wait()
-	t.Lock()
-	t.ready = true
-	t.Unlock()
-}
-
-func (t *Listener) readPcapFile() {
-	if handle, err := pcap.OpenOffline(t.addr); err != nil {
-		log.Fatal(err)
-	} else {
-		if t.bpfFilter != "" {
-			if err := handle.SetBPFFilter(t.bpfFilter); err != nil {
-				log.Println("BPF filter error:", err)
-				return
-			}
-		}
-
-		t.Lock()
-		t.ready = true
-		t.Unlock()
-		packetSource := gopacket.NewPacketSource(handle, handle.LinkType())
-
-		for {
-			packet, err := packetSource.NextPacket()
-			if err == io.EOF {
-				break
-			} else if err != nil {
-				log.Println("Error:", err)
-				continue
-			}
-
-			var addr, data []byte
-
-			if tcpLayer := packet.Layer(layers.LayerTypeTCP); tcpLayer != nil {
-				tcp, _ := tcpLayer.(*layers.TCP)
-				data = append(tcp.LayerContents(), tcp.LayerPayload()...)
-
-				if uint16(tcp.DstPort) == t.port {
-					copy(data[0:2], []byte{byte(tcp.SrcPort >> 8), byte(tcp.SrcPort)})
-					copy(data[2:4], []byte{byte(tcp.DstPort >> 8), byte(tcp.DstPort)})
-				} else {
-					copy(data[0:2], []byte{byte(tcp.DstPort >> 8), byte(tcp.DstPort)})
-					copy(data[2:4], []byte{byte(tcp.SrcPort >> 8), byte(tcp.SrcPort)})
-				}
-			} else {
-				continue
-			}
-
-			if ipLayer := packet.Layer(layers.LayerTypeIPv4); ipLayer != nil {
-				ip, _ := ipLayer.(*layers.IPv4)
-				addr = ip.SrcIP
-			} else if ipLayer = packet.Layer(layers.LayerTypeIPv6); ipLayer != nil {
-				ip, _ := ipLayer.(*layers.IPv6)
-				addr = ip.SrcIP
-			} else {
-				// log.Println("Can't find IP layer", packet)
-				continue
-			}
-
-			dataOffset := data[12] >> 4
-			isFIN := data[13]&0x01 != 0
-
-			// We need only packets with data inside
-			// Check that the buffer is larger than the size of the TCP header
-			if len(data) <= int(dataOffset*4) && !isFIN {
-				continue
-			}
-
-			t.packetsChan <- t.buildPacket(addr, data, packet.Metadata().Timestamp)
-		}
-	}
-}
-
-func (t *Listener) readRAWSocket() {
-	conn, e := net.ListenPacket("ip:tcp", t.addr)
-	t.conn = conn
-	if e != nil {
-		log.Fatal(e)
-	}
-	defer t.conn.Close()
-	type RSPacket struct {
-		buf  []byte
-		addr net.Addr
-		err  error
-		n    int
-	}
-	var bufChan = make(chan *RSPacket, 1000)
-	t.Lock()
-	t.ready = true
-	t.Unlock()
-	go func() {
-		for {
-			// Re-allocate data object to avoid data collision
-			var buf [64 * 104 * 1024]byte
-			// Note: ReadFrom receive messages without IP header
-			n, addr, err := t.conn.ReadFrom(buf[:])
-			bufChan <- &RSPacket{buf[:], addr, err, n}
-		}
-	}()
-	for {
-		packet := <-bufChan
-		if packet.err != nil {
-			if strings.HasSuffix(packet.err.Error(), "closed network connection") {
-				return
-			}
-			continue
-		}
-
-		if packet.n > 0 {
-			if t.isValidPacket(packet.buf[:packet.n]) {
-				t.packetsChan <- t.buildPacket([]byte(packet.addr.(*net.IPAddr).IP), packet.buf[:packet.n], time.Now())
-			}
-		}
-	}
-}
-
-func (t *Listener) buildPacket(packetSrcIP []byte, packetData []byte, timestamp time.Time) *packet {
-	return &packet{
-		srcIP:     packetSrcIP,
-		data:      packetData,
-		timestamp: timestamp,
-	}
-}
-
-func (t *Listener) isValidPacket(buf []byte) bool {
-	// To avoid full packet parsing every time, we manually parsing values needed for packet filtering
-	// http://en.wikipedia.org/wiki/Transmission_Control_Protocol
-	destPort := binary.BigEndian.Uint16(buf[2:4])
-	srcPort := binary.BigEndian.Uint16(buf[0:2])
-	// Because RAW_SOCKET can't be bound to port, we have to control it by ourself
-	if destPort == t.port || (t.trackResponse && srcPort == t.port) {
-		// Get the 'data offset' (size of the TCP header in 32-bit words)
-		dataOffset := buf[12] >> 4
-		// We need only packets with data inside
-		// Check that the buffer is larger than the size of the TCP header
-		if len(buf) > int(dataOffset*4) {
-			return true
-		}
-	}
-
-	return false
-}
-
-// Trying to add packet to existing message or creating new message
-//
-// For TCP message unique id is Acknowledgment number (see tcp_packet.go)
-func (t *Listener) processTCPPacket(packet *TCPPacket) {
-	// Don't exit on panic
-	defer func() {
-		if r := recover(); r != nil {
-			log.Println("PANIC: pkg:", r, packet, string(debug.Stack()))
-		}
-	}()
-
-	var responseRequest *TCPMessage
-	var message *TCPMessage
-
-	isIncoming := packet.DestPort == t.port
-
-	if t.protocol == ProtocolHTTP {
-		if !isIncoming {
-			responseRequest, _ = t.respAliases[packet.Ack]
-		}
-
-		// Seek for 100-expect chunks
-		// `packet.Ack != parentAck` is protection for clients who send data without ignoring server 100-continue response, e.g have data chunks have same Ack
-		if parentAck, ok := t.seqWithData[packet.Seq]; ok && packet.Ack != parentAck {
-			// Skip zero-length chunks https://github.com/buger/goreplay/issues/496
-			if len(packet.Data) == 0 {
-				return
-			}
-
-			// In case if non-first data chunks comes first
-			for _, m := range t.messages {
-				if m.Ack == packet.Ack && bytes.Equal(m.packets[0].Addr, packet.Addr) {
-					t.deleteMessage(m)
-
-					if m.AssocMessage != nil {
-						m.AssocMessage.setAssocMessage(nil)
-						m.setAssocMessage(nil)
-					}
-					for _, pkt := range m.packets {
-						// log.Println("Updating ack", parentAck, pkt.Ack)
-						pkt.UpdateAck(parentAck)
-						// Re-queue this packets
-						t.processTCPPacket(pkt)
-					}
-				}
-			}
-
-			t.ackAliases[packet.Ack] = parentAck
-			packet.UpdateAck(parentAck)
-		}
-	}
-
-	if isIncoming && packet.IsFIN {
-		if ma, ok := t.respAliases[packet.Seq]; ok {
-			if ma.packets[0].SrcPort == packet.SrcPort {
-				packet.UpdateAck(ma.Ack)
-			}
-		}
-	}
-
-	if alias, ok := t.ackAliases[packet.Ack]; ok {
-		packet.UpdateAck(alias)
-	}
-
-	message, ok := t.messages[packet.ID]
-
-	if !ok {
-		message = NewTCPMessage(packet.Seq, packet.Ack, isIncoming, t.protocol, packet.timestamp)
-		t.messages[packet.ID] = message
-
-		if !isIncoming {
-			if responseRequest != nil {
-				message.setAssocMessage(responseRequest)
-				responseRequest.setAssocMessage(message)
-			} else {
-				t.respWithoutReq[packet.Ack] = packet.ID
-			}
-		}
-	}
-
-	// Adding packet to message
-	message.AddPacket(packet)
-
-	// Handling Expect: 100-continue requests
-	if t.protocol == ProtocolHTTP && message.expectType == httpExpect100Continue && len(message.packets) == message.headerPacket+1 {
-		seq := packet.Seq + uint32(len(packet.Data))
-		t.seqWithData[seq] = packet.Ack
-
-		message.DataSeq = seq
-		message.complete = false
-
-		// In case if sequence packet came first
-		for _, m := range t.messages {
-			if m.Seq == seq {
-				t.deleteMessage(m)
-				if m.AssocMessage != nil {
-					message.setAssocMessage(m.AssocMessage)
-					m.AssocMessage.setAssocMessage(nil)
-				}
-
-				t.ackAliases[m.Ack] = packet.Ack
-
-				for _, pkt := range m.packets {
-					pkt.UpdateAck(packet.Ack)
-					message.AddPacket(pkt)
-				}
-			}
-		}
-
-		// Removing `Expect: 100-continue` header
-		packet.Data = proto.DeleteHeader(packet.Data, bExpectHeader)
-	}
-
-	// If client do sends Expect: 100-continue but do not respect server response
-	if message.expectType == httpExpect100Continue && (message.headerPacket != -1 && len(message.packets) > message.headerPacket+1) {
-		delete(t.seqWithData, message.DataSeq)
-		seq := packet.Seq + uint32(len(packet.Data))
-		t.seqWithData[seq] = packet.Ack
-		message.DataSeq = seq
-	}
-
-	if isIncoming {
-		// If message have multiple packets, delete previous alias
-		if len(message.packets) > 1 {
-			delete(t.respAliases, message.ResponseAck)
-		}
-
-		message.UpdateResponseAck()
-		t.respAliases[message.ResponseAck] = message
-	}
-
-	// If message contains only single packet immediately dispatch it
-	if message.complete {
-		// log.Println("COMPLETE!", isIncoming, message)
-		if isIncoming {
-			if t.trackResponse {
-				// log.Println("Found response!", message.ResponseID, t.messages)
-
-				if resp, ok := t.messages[message.ResponseID]; ok {
-					if resp.complete {
-						t.dispatchMessage(resp)
-					}
-
-					t.dispatchMessage(message)
-				}
-			} else {
-				t.dispatchMessage(message)
-			}
-		} else {
-			if message.AssocMessage == nil {
-				return
-			}
-
-			if req, ok := t.messages[message.AssocMessage.ID()]; ok {
-				if req.complete {
-					t.dispatchMessage(req)
-					t.dispatchMessage(message)
-				}
-			}
-		}
-	}
-}
-
-// Receiver TCP messages from the listener channel
-func (t *Listener) Receiver() chan *TCPMessage {
-	return t.messagesChan
-}
-
-// Close tcp listener
-func (t *Listener) Close() {
-	close(t.quit)
-	if t.conn != nil {
-		t.conn.Close()
-	}
-
-	for _, h := range t.pcapHandles {
-		h.Close()
-	}
-
-	return
-}

capture/listener_test.go

@@ -1,617 +0,0 @@
-package capture
-
-import (
-	"bytes"
-	"log"
-	"math/rand"
-	"sync/atomic"
-	"testing"
-	"time"
-)
-
-func TestRawListenerInput(t *testing.T) {
-	var req, resp *TCPMessage
-	listener := NewListener("", "0", EnginePcap, true, 10*time.Millisecond, ProtocolHTTP, "", "", 0, false, false)
-	defer listener.Close()
-
-	reqPacket := buildPacket(true, 1, 1, []byte("GET / HTTP/1.1\r\n\r\n"), time.Now())
-
-	respAck := reqPacket.Seq + uint32(len(reqPacket.Data))
-	respPacket := buildPacket(false, respAck, reqPacket.Seq+1, []byte("HTTP/1.1 200 OK\r\n\r\n"), time.Now())
-
-	listener.packetsChan <- reqPacket.dump()
-	listener.packetsChan <- respPacket.dump()
-
-	select {
-	case req = <-listener.messagesChan:
-	case <-time.After(time.Millisecond):
-		t.Error("Should return request immediately")
-		return
-	}
-
-	if !req.IsIncoming {
-		t.Error("Should be request")
-	}
-
-	select {
-	case resp = <-listener.messagesChan:
-	case <-time.After(20 * time.Millisecond):
-		t.Error("Should return response immediately")
-		return
-	}
-
-	if resp.IsIncoming {
-		t.Error("Should be response")
-	}
-}
-
-func firstPacket(payload []byte) *TCPPacket {
-	return buildPacket(
-		true,
-		1,
-		1,
-		payload,
-		time.Now(),
-	)
-}
-
-func nextPacket(prev *TCPPacket, payload []byte) *TCPPacket {
-	return buildPacket(
-		prev.SrcPort == 1,
-		prev.Ack,
-		prev.Seq+uint32(len(prev.Data)),
-		payload,
-		prev.timestamp.Add(time.Millisecond),
-	)
-}
-
-func responsePacket(prev *TCPPacket, payload []byte) *TCPPacket {
-	return buildPacket(
-		!(prev.SrcPort == 1),
-		prev.Seq+uint32(len(prev.Data)),
-		prev.Ack,
-		payload,
-		prev.timestamp.Add(time.Millisecond),
-	)
-}
-
-func TestHEADRequestNoBody(t *testing.T) {
-	listener := NewListener("", "0", EnginePcap, true, 10*time.Millisecond, ProtocolHTTP, "", "", 0, false, false)
-	defer listener.Close()
-
-	reqPacket := firstPacket([]byte("HEAD / HTTP/1.1\r\nContent-Length: 0\r\n\r\n"))
-	respPacket := responsePacket(reqPacket, []byte("HTTP/1.1 200 OK\r\nContent-Length: 100\r\n\r\n"))
-
-	listener.packetsChan <- reqPacket.dump()
-	listener.packetsChan <- respPacket.dump()
-
-	var req, resp *TCPMessage
-	select {
-	case req = <-listener.messagesChan:
-	case <-time.After(time.Millisecond):
-		t.Error("Should return request immediately")
-		return
-	}
-
-	if !req.IsIncoming {
-		t.Error("Should be request")
-	}
-
-	select {
-	case resp = <-listener.messagesChan:
-	case <-time.After(20 * time.Millisecond):
-		t.Error("Should return response immediately")
-		return
-	}
-
-	if resp.IsIncoming {
-		t.Error("Should be response")
-	}
-}
-
-func TestSingleAck100Continue(t *testing.T) {
-	listener := NewListener("", "0", EnginePcap, true, 10*time.Millisecond, ProtocolHTTP, "", "", 0, false, false)
-	defer listener.Close()
-
-	reqPacket1 := firstPacket([]byte("POST / HTTP/1.1\r\nExpect: 100-continue\r\nContent-Length: 4\r\n\r\n"))
-	respPacket1 := responsePacket(reqPacket1, []byte(""))
-	respPacket2 := responsePacket(reqPacket1, []byte("HTTP/1.1 100 Continue\r\n"))
-	reqPacket2 := responsePacket(respPacket2, []byte("DATA"))
-	respPacket3 := responsePacket(reqPacket2, []byte("HTTP/1.1 200 OK\r\n\r\n"))
-
-	result := []byte("POST / HTTP/1.1\r\nContent-Length: 4\r\n\r\nDATA")
-
-	testRawListener100Continue(t, listener, result,
-		reqPacket1,
-		respPacket1, respPacket2,
-		reqPacket2,
-		respPacket3)
-}
-
-func Test100ContinueWithoutWaiting(t *testing.T) {
-	listener := NewListener("", "0", EnginePcap, true, 10*time.Millisecond, ProtocolHTTP, "", "", 0, false, false)
-	defer listener.Close()
-
-	req1 := firstPacket([]byte("POST / HTTP/1.1\r\nExpect: 100-continue\r\nContent-Length: 4\r\n\r\n"))
-	req2 := nextPacket(req1, []byte("DATA"))
-	resp1 := responsePacket(req1, []byte("HTTP/1.1 100 Continue\r\n"))
-	resp2 := responsePacket(req2, []byte("HTTP/1.1 200 OK\r\n\r\n"))
-
-	result := []byte("POST / HTTP/1.1\r\nContent-Length: 4\r\n\r\nDATA")
-
-	testRawListener100Continue(t, listener, result,
-		req1, req2, resp1, resp2)
-}
-
-// Client first sends data without waiting 100-continue, but once response received, generate packets based on Ack payload
-func Test100ContinueMixed(t *testing.T) {
-	listener := NewListener("", "0", EnginePcap, true, 10*time.Millisecond, ProtocolHTTP, "", "", 0, false, false)
-	defer listener.Close()
-
-	req1 := firstPacket([]byte("POST / HTTP/1.1\r\nExpect: 100-continue\r\nContent-Length: 12\r\n\r\n"))
-	req2 := nextPacket(req1, []byte("DAT1"))
-	resp1 := responsePacket(req1, []byte("HTTP/1.1 100 Continue\r\n\r\n"))
-	req3 := responsePacket(resp1, []byte("DAT2"))
-	req3.Seq = req2.Seq + uint32(len(req2.Data))
-	req4 := nextPacket(req3, []byte("DAT3"))
-	resp2 := responsePacket(req4, []byte("HTTP/1.1 200 OK\r\n\r\n"))
-
-	result := []byte("POST / HTTP/1.1\r\nContent-Length: 12\r\n\r\nDAT1DAT2DAT3")
-
-	testRawListener100Continue(t, listener, result,
-		req1, req2, req3, req4, resp1, resp2)
-}
-
-func TestDoubleAck100Continue(t *testing.T) {
-	listener := NewListener("", "0", EnginePcap, true, 10*time.Millisecond, ProtocolHTTP, "", "", 0, false, false)
-	defer listener.Close()
-
-	reqPacket1 := firstPacket([]byte("POST / HTTP/1.1\r\nExpect: 100-continue\r\nContent-Length: 4\r\n\r\n"))
-
-	respPacket1 := responsePacket(reqPacket1, []byte(""))
-	respPacket2 := responsePacket(reqPacket1, []byte("HTTP/1.1 100 Continue\r\n"))
-	reqPacket2 := responsePacket(respPacket2, []byte(""))
-	reqPacket3 := responsePacket(respPacket2, []byte("DATA"))
-	respPacket3 := responsePacket(reqPacket3, []byte("HTTP/1.1 200 OK\r\n\r\n"))
-
-	result := []byte("POST / HTTP/1.1\r\nContent-Length: 4\r\n\r\nDATA")
-
-	testRawListener100Continue(t, listener, result,
-		reqPacket1,
-		respPacket1, respPacket2,
-		reqPacket2, reqPacket3,
-		respPacket3)
-}
-
-func TestRawListenerInputResponseByClose(t *testing.T) {
-	var req, resp *TCPMessage
-
-	listener := NewListener("", "0", EnginePcap, true, 10*time.Millisecond, ProtocolHTTP, "", "", 0, false, false)
-	defer listener.Close()
-
-	reqPacket := buildPacket(true, 1, 1, []byte("GET / HTTP/1.1\r\n\r\n"), time.Now())
-
-	respAck := reqPacket.Seq + uint32(len(reqPacket.Data))
-	respPacket := buildPacket(false, respAck, reqPacket.Seq+1, []byte("HTTP/1.1 200 OK\r\nConnection: close\r\n\r\nasd"), time.Now())
-	finPacket := buildPacket(false, respAck, reqPacket.Seq+2, []byte(""), time.Now())
-	finPacket.IsFIN = true
-
-	listener.packetsChan <- reqPacket.dump()
-	listener.packetsChan <- respPacket.dump()
-	listener.packetsChan <- finPacket.dump()
-
-	select {
-	case req = <-listener.messagesChan:
-	case <-time.After(time.Millisecond):
-		t.Error("Should return request immediately")
-		return
-	}
-
-	if !req.IsIncoming {
-		t.Error("Should be request")
-	}
-
-	select {
-	case resp = <-listener.messagesChan:
-	case <-time.After(20 * time.Millisecond):
-		t.Error("Should return response immediately")
-		return
-	}
-
-	if resp.IsIncoming {
-		t.Error("Should be response")
-	}
-}
-
-func TestRawListenerInputWithoutResponse(t *testing.T) {
-	var req *TCPMessage
-
-	listener := NewListener("", "0", EnginePcap, false, 10*time.Millisecond, ProtocolHTTP, "", "", 0, false, false)
-	defer listener.Close()
-
-	reqPacket := buildPacket(true, 1, 1, []byte("GET / HTTP/1.1\r\n\r\n"), time.Now())
-
-	listener.packetsChan <- reqPacket.dump()
-
-	select {
-	case req = <-listener.messagesChan:
-	case <-time.After(time.Millisecond):
-		t.Error("Should return request immediately")
-		return
-	}
-
-	if !req.IsIncoming {
-		t.Error("Should be request")
-	}
-}
-
-func TestRawListenerResponse(t *testing.T) {
-	var req, resp *TCPMessage
-
-	listener := NewListener("", "0", EnginePcap, true, 10*time.Millisecond, ProtocolHTTP, "", "", 0, false, false)
-	defer listener.Close()
-
-	reqPacket := firstPacket([]byte("GET / HTTP/1.1\r\n\r\n"))
-	respPacket := responsePacket(reqPacket, []byte("HTTP/1.1 200 OK\r\n\r\n"))
-
-	// If response packet comes before request
-	listener.packetsChan <- respPacket.dump()
-	listener.packetsChan <- reqPacket.dump()
-
-	select {
-	case req = <-listener.messagesChan:
-	case <-time.After(time.Millisecond):
-		t.Error("Should return request immediately")
-		return
-	}
-
-	if !req.IsIncoming {
-		t.Error("Should be request")
-	}
-
-	select {
-	case resp = <-listener.messagesChan:
-	case <-time.After(time.Millisecond):
-		t.Error("Should return response immediately")
-		return
-	}
-
-	if resp.IsIncoming {
-		t.Error("Should be response")
-	}
-
-	if !bytes.Equal(resp.UUID(), req.UUID()) {
-		t.Error("Resp and Req UUID should be equal")
-	}
-}
-
-func get100ContinuePackets() (req []*TCPPacket, resp []*TCPPacket) {
-	req1 := firstPacket([]byte("POST / HTTP/1.1\r\nExpect: 100-continue\r\nContent-Length: 2\r\n\r\n"))
-	resp1 := responsePacket(req1, []byte("HTTP/1.1 100 Continue\r\n"))
-	req2 := responsePacket(resp1, []byte("a"))
-	req3 := nextPacket(req2, []byte("b"))
-	resp2 := responsePacket(req3, []byte("HTTP/1.1 200 OK\r\n\r\n"))
-
-	return []*TCPPacket{req1, req2, req3}, []*TCPPacket{resp1, resp2}
-}
-
-func TestShort100Continue(t *testing.T) {
-	listener := NewListener("", "0", EnginePcap, true, 10*time.Millisecond, ProtocolHTTP, "", "", 0, false, false)
-	defer listener.Close()
-
-	req, resp := get100ContinuePackets()
-
-	result := []byte("POST / HTTP/1.1\r\nContent-Length: 2\r\n\r\nab")
-
-	testRawListener100Continue(t, listener, result, req[0], req[1], req[2], resp[0], resp[1])
-}
-
-// Response comes before Request
-func Test100ContinueWrongOrder(t *testing.T) {
-	listener := NewListener("", "0", EnginePcap, true, 10*time.Millisecond, ProtocolHTTP, "", "", 0, false, false)
-	defer listener.Close()
-
-	req, resp := get100ContinuePackets()
-
-	result := []byte("POST / HTTP/1.1\r\nContent-Length: 2\r\n\r\nab")
-
-	testRawListener100Continue(t, listener, result, resp[0], resp[1], req[0], req[1], req[2])
-}
-
-func testRawListener100Continue(t *testing.T, listener *Listener, result []byte, packets ...*TCPPacket) {
-	var req, resp *TCPMessage
-	for _, p := range packets {
-		listener.packetsChan <- p.dump()
-	}
-
-	select {
-	case req = <-listener.messagesChan:
-		break
-	case <-time.After(11 * time.Millisecond):
-		t.Error("Should return response after expire time")
-		return
-	}
-
-	if !bytes.Equal(req.Bytes(), result) {
-		t.Error("Should receive full message", string(req.Bytes()))
-	}
-
-	if !req.IsIncoming {
-		t.Error("Should be request")
-	}
-
-	select {
-	case resp = <-listener.messagesChan:
-		break
-	case <-time.After(21 * time.Millisecond):
-		t.Error("Should return response after expire time")
-		return
-	}
-
-	if resp.IsIncoming {
-		t.Error("Should be response")
-	}
-
-	if !bytes.Equal(resp.UUID(), req.UUID()) {
-		t.Error("Resp and Req UUID should be equal")
-	}
-}
-
-func testChunkedSequence(t *testing.T, listener *Listener, packets ...*TCPPacket) {
-	var r, req, resp *TCPMessage
-
-	for _, p := range packets {
-		listener.packetsChan <- p.dump()
-	}
-
-	select {
-	case r = <-listener.messagesChan:
-		if r.IsIncoming {
-			req = r
-		} else {
-			resp = r
-		}
-		break
-	case <-time.After(25 * time.Millisecond):
-		t.Error("Should return request after expire time")
-		return
-	}
-	select {
-	case r = <-listener.messagesChan:
-		if r.IsIncoming {
-			if req != nil {
-				t.Error("Request already received", r)
-				return
-			}
-			req = r
-		} else {
-			if resp != nil {
-				t.Error("Response already received", r)
-				return
-			}
-			resp = r
-		}
-		break
-	case <-time.After(25 * time.Millisecond):
-		t.Error("Should return request after expire time")
-		return
-	}
-
-	if !bytes.Equal(req.Bytes(), []byte("POST / HTTP/1.1\r\nTransfer-Encoding: chunked\r\n\r\n1\r\na\r\n1\r\nb\r\n0\r\n\r\n")) {
-		t.Error("Should receive full message", string(req.Bytes()))
-	}
-
-	if !req.IsIncoming {
-		t.Error("Should be request")
-	}
-
-	if resp.IsIncoming {
-		t.Error("Should be response")
-	}
-
-	if !bytes.Equal(resp.UUID(), req.UUID()) {
-		t.Error("Resp and Req UUID should be equal", string(resp.UUID()), string(req.UUID()))
-	}
-
-	time.Sleep(20 * time.Millisecond)
-
-	if len(listener.packetsChan) != 0 {
-		t.Fatal("packetsChan non empty:", listener.packetsChan)
-	}
-
-	if len(listener.ackAliases) != 0 {
-		t.Fatal("ackAliases non empty:", listener.ackAliases)
-	}
-
-	if len(listener.seqWithData) != 0 {
-		t.Fatal("seqWithData non empty:", listener.seqWithData)
-	}
-
-	if len(listener.respAliases) != 0 {
-		t.Fatal("respAliases non empty:", listener.respAliases)
-	}
-}
-
-// permutation using heap algorithm https://en.wikipedia.org/wiki/Heap%27s_algorithm
-func permutation(a []*TCPPacket, f func([]*TCPPacket)) {
-	n := len(a)
-	c := make([]int, n)
-	f(a)
-	i := 0
-	for i < n {
-		if c[i] < i {
-			if i&1 != 1 {
-				a[0], a[i] = a[i], a[0]
-			} else {
-				a[c[i]], a[i] = a[i], a[c[i]]
-			}
-			f(a)
-			c[i]++
-			i = 0
-		} else {
-			c[i] = 0
-			i++
-		}
-	}
-}
-
-// Response comes before Request
-func TestRawListenerChunkedWrongOrder(t *testing.T) {
-	listener := NewListener("", "0", EnginePcap, true, 10*time.Millisecond, ProtocolHTTP, "", "", 0, false, false)
-	defer listener.Close()
-
-	reqPacket1 := firstPacket([]byte("POST / HTTP/1.1\r\nTransfer-Encoding: chunked\r\nExpect: 100-continue\r\n\r\n"))
-
-	respPacket1 := responsePacket(reqPacket1, []byte("HTTP/1.1 100 Continue\r\n"))
-	reqPacket2 := responsePacket(respPacket1, []byte("1\r\na\r\n"))
-	reqPacket3 := nextPacket(reqPacket2, []byte("1\r\nb\r\n"))
-	reqPacket4 := nextPacket(reqPacket3, []byte("0\r\n\r\n"))
-
-	respPacket2 := responsePacket(reqPacket4, []byte("HTTP/1.1 200 OK\r\n\r\n"))
-
-	f := func(p []*TCPPacket) {
-		testChunkedSequence(t, listener, p...)
-	}
-	// Should re-construct message from all possible combinations
-	permutation([]*TCPPacket{reqPacket1, reqPacket2, reqPacket3, reqPacket4, respPacket1, respPacket2}, f)
-}
-
-func chunkedPostMessage() []*TCPPacket {
-	ack := uint32(rand.Int63())
-	seq := uint32(rand.Int63())
-
-	reqPacket1 := buildPacket(true, ack, seq, []byte("POST / HTTP/1.1\r\nTransfer-Encoding: chunked\r\n\r\n"), time.Now())
-	// Packet with data have different Seq
-	reqPacket2 := buildPacket(true, ack, seq+47, []byte("1\r\na\r\n"), time.Now())
-	reqPacket3 := buildPacket(true, ack, reqPacket2.Seq+5, []byte("1\r\nb\r\n"), time.Now())
-	reqPacket4 := buildPacket(true, ack, reqPacket3.Seq+5, []byte("0\r\n\r\n"), time.Now())
-
-	respPacket := buildPacket(false, reqPacket4.Seq+5 /* len of data */, ack, []byte("HTTP/1.1 200 OK\r\n\r\n"), time.Now())
-
-	return []*TCPPacket{
-		reqPacket1, reqPacket2, reqPacket3, reqPacket4, respPacket,
-	}
-}
-
-func postMessage() []*TCPPacket {
-	ack := uint32(rand.Int63())
-	seq2 := uint32(rand.Int63())
-	seq := uint32(rand.Int63())
-
-	c := 10000
-	data := make([]byte, c)
-	rand.Read(data)
-
-	head := []byte("POST / HTTP/1.1\r\nContent-Length: 9958\r\n\r\n")
-	for i := range head {
-		data[i] = head[i]
-	}
-
-	return []*TCPPacket{
-		buildPacket(true, ack, seq, data, time.Now()),
-		buildPacket(false, seq+uint32(len(data)), seq2, []byte("HTTP/1.1 200 OK\r\n\r\n"), time.Now()),
-	}
-}
-
-func getMessage() []*TCPPacket {
-	ack := uint32(rand.Int63())
-	seq2 := uint32(rand.Int63())
-	seq := uint32(rand.Int63())
-
-	return []*TCPPacket{
-		buildPacket(true, ack, seq, []byte("GET / HTTP/1.1\r\n\r\n"), time.Now()),
-		buildPacket(false, seq+18, seq2, []byte("HTTP/1.1 200 OK\r\n\r\n"), time.Now()),
-	}
-}
-
-// Response comes before Request
-func TestRawListenerBench(t *testing.T) {
-	l := NewListener("", "0", EnginePcap, true, 200*time.Millisecond, ProtocolHTTP, "", "", 0, false, false)
-	defer l.Close()
-
-	// Should re-construct message from all possible combinations
-	for i := 0; i < 1000; i++ {
-		go func(i int) {
-			for j := 0; j < 100; j++ {
-				var packets []*TCPPacket
-
-				if j%5 == 0 {
-					packets = chunkedPostMessage()
-				} else if j%3 == 0 {
-					packets = postMessage()
-				} else {
-					packets = getMessage()
-				}
-
-				for _, p := range packets {
-					// Randomly drop packets
-					if (i+j)%5 == 0 {
-						if rand.Int63()%3 == 0 {
-							continue
-						}
-					}
-
-					l.packetsChan <- p.dump()
-					time.Sleep(time.Millisecond)
-				}
-
-				time.Sleep(5 * time.Millisecond)
-			}
-		}(i)
-	}
-
-	ch := l.Receiver()
-
-	var count int32
-
-	for {
-		select {
-		case <-ch:
-			atomic.AddInt32(&count, 1)
-		case <-time.After(2000 * time.Millisecond):
-			log.Println("Emitted 200000 messages, captured: ", count, len(l.ackAliases), len(l.seqWithData), len(l.respAliases), len(l.respWithoutReq), len(l.packetsChan))
-			return
-		}
-	}
-}
-
-func TestResponseZeroContentLength(t *testing.T) {
-	var req, resp *TCPMessage
-	listener := NewListener("", "0", EnginePcap, true, 10*time.Millisecond, ProtocolHTTP, "", "", 0, false, false)
-	defer listener.Close()
-
-	reqPacket := firstPacket([]byte("POST /api/setup/install HTTP/1.1\r\nHost: localhost:22936\r\nUser-Agent: curl/7.57.0\r\nAccept: */*\r\nContent-Length: 0\r\nContent-Type: application/x-www-form-urlencoded\r\n\r\n"))
-	respPacket := responsePacket(reqPacket, []byte("HTTP/1.1 200\r\nDate: Fri, 11 May 2018 15:09:10 GMT\r\nServer: Kestrel\r\nCache-Control: no-cache\r\nTransfer-Encoding: chunked\r\n\r\n"))
-	respPacket2 := nextPacket(respPacket, []byte("0\r\n\r\n"))
-
-	// If response packet comes before request
-	listener.packetsChan <- reqPacket.dump()
-	listener.packetsChan <- respPacket.dump()
-	listener.packetsChan <- respPacket2.dump()
-
-	select {
-	case req = <-listener.messagesChan:
-	case <-time.After(time.Millisecond):
-		t.Error("Should return request immediately")
-		return
-	}
-
-	if !req.IsIncoming {
-		t.Error("Should be request")
-	}
-
-	select {
-	case resp = <-listener.messagesChan:
-	case <-time.After(time.Millisecond):
-		t.Error("Should return response immediately")
-		return
-	}
-
-	if resp.IsIncoming {
-		t.Error("Should be response")
-	}
-
-	if !bytes.Equal(resp.UUID(), req.UUID()) {
-		t.Error("Resp and Req UUID should be equal")
-	}
-}

capture/sock_linux.go

@@ -0,0 +1,260 @@
+package capture
+
+import (
+	"fmt"
+	"net"
+	"sync"
+	"time"
+	"unsafe"
+
+	"golang.org/x/sys/unix"
+
+	"github.com/google/gopacket"
+	"github.com/google/gopacket/layers"
+	"github.com/google/gopacket/pcap"
+)
+
+const (
+	// ETHALL htons(ETH_P_ALL)
+	ETHALL uint16 = unix.ETH_P_ALL<<8 | unix.ETH_P_ALL>>8
+	// BLOCKSIZE ring buffer block_size
+	BLOCKSIZE = 64 << 10
+	// BLOCKNR ring buffer block_nr
+	BLOCKNR = (2 << 20) / BLOCKSIZE // 2mb / 64kb
+	// FRAMESIZE ring buffer frame_size
+	FRAMESIZE = BLOCKSIZE
+	// FRAMENR ring buffer frame_nr
+	FRAMENR = BLOCKNR * BLOCKSIZE / FRAMESIZE
+	// MAPHUGE2MB 2mb huge map
+	MAPHUGE2MB = 21 << unix.MAP_HUGE_SHIFT
+)
+
+var tpacket2hdrlen = tpAlign(int(unsafe.Sizeof(unix.Tpacket2Hdr{})))
+
+// SockRaw is a linux M'maped af_packet socket
+type SockRaw struct {
+	mu          sync.Mutex
+	fd          int
+	ifindex     int
+	snaplen     int
+	pollTimeout uintptr
+	frame       uint32 // current frame
+	buf         []byte // points to the memory space of the ring buffer shared with the kernel.
+	loopIndex   int32  // this field must filled to avoid reading packet twice on a loopback device
+}
+
+// NewSocket returns new M'maped sock_raw on packet version 2.
+func NewSocket(ifi net.Interface) (*SockRaw, error) {
+	// sock create
+	fd, err := unix.Socket(unix.AF_PACKET, unix.SOCK_RAW, int(ETHALL))
+	if err != nil {
+		return nil, err
+	}
+	sock := &SockRaw{
+		fd:          fd,
+		ifindex:     ifi.Index,
+		snaplen:     FRAMESIZE,
+		pollTimeout: ^uintptr(0),
+	}
+
+	// set packet version
+	err = unix.SetsockoptInt(fd, unix.SOL_PACKET, unix.PACKET_VERSION, unix.TPACKET_V2)
+	if err != nil {
+		unix.Close(fd)
+		return nil, fmt.Errorf("setsockopt packet_version: %v", err)
+	}
+
+	// bind to interface
+	addr := unix.RawSockaddrLinklayer{
+		Family:   unix.AF_PACKET,
+		Protocol: ETHALL,
+		Ifindex:  int32(ifi.Index),
+	}
+	_, _, e := unix.Syscall(
+		unix.SYS_BIND,
+		uintptr(fd),
+		uintptr(unsafe.Pointer(&addr)),
+		uintptr(unix.SizeofSockaddrLinklayer),
+	)
+	if e != 0 {
+		unix.Close(fd)
+		return nil, e
+	}
+
+	// create shared-memory ring buffer
+	tp := &unix.TpacketReq{
+		Block_size: BLOCKSIZE,
+		Block_nr:   BLOCKNR,
+		Frame_size: FRAMESIZE,
+		Frame_nr:   FRAMENR,
+	}
+	err = unix.SetsockoptTpacketReq(sock.fd, unix.SOL_PACKET, unix.PACKET_RX_RING, tp)
+	if err != nil {
+		unix.Close(fd)
+		return nil, fmt.Errorf("setsockopt packet_rx_ring: %v", err)
+	}
+	sock.buf, err = unix.Mmap(
+		sock.fd,
+		0,
+		BLOCKSIZE*BLOCKNR,
+		unix.PROT_READ|unix.PROT_WRITE,
+		unix.MAP_SHARED|MAPHUGE2MB,
+	)
+	if err != nil {
+		unix.Close(fd)
+		return nil, fmt.Errorf("socket mmap error: %v", err)
+	}
+	return sock, nil
+}
+
+// ReadPacketData implements gopacket.PacketDataSource.
+func (sock *SockRaw) ReadPacketData() (buf []byte, ci gopacket.CaptureInfo, err error) {
+	sock.mu.Lock()
+	defer sock.mu.Unlock()
+	var tpHdr *unix.Tpacket2Hdr
+	poll := &unix.PollFd{
+		Fd:     int32(sock.fd),
+		Events: unix.POLLIN,
+	}
+	var i int
+read:
+	i = int(sock.frame * FRAMESIZE)
+	tpHdr = (*unix.Tpacket2Hdr)(unsafe.Pointer(&sock.buf[i]))
+	sock.frame = (sock.frame + 1) % FRAMENR
+
+	if tpHdr.Status&unix.TP_STATUS_USER == 0 {
+		_, _, e := unix.Syscall(unix.SYS_POLL, uintptr(unsafe.Pointer(poll)), 1, sock.pollTimeout)
+		if e != 0 && e != unix.EINTR {
+			return buf, ci, e
+		}
+		// it might be some other frame with data!
+		if tpHdr.Status&unix.TP_STATUS_USER == 0 {
+			goto read
+		}
+	}
+	tpHdr.Status = unix.TP_STATUS_KERNEL
+	sockAddr := (*unix.RawSockaddrLinklayer)(unsafe.Pointer(&sock.buf[i+tpacket2hdrlen]))
+
+	// parse out repeating packets on loopback
+	if sockAddr.Ifindex == sock.loopIndex && sock.frame%2 != 0 {
+		goto read
+	}
+
+	ci.Length = int(tpHdr.Len)
+	ci.Timestamp = time.Unix(int64(tpHdr.Sec), int64(tpHdr.Nsec))
+	ci.InterfaceIndex = int(sockAddr.Ifindex)
+	buf = make([]byte, tpHdr.Snaplen)
+	ci.CaptureLength = copy(buf, sock.buf[i+int(tpHdr.Mac):])
+
+	return
+}
+
+// Close closes the underlying socket
+func (sock *SockRaw) Close() (err error) {
+	sock.mu.Lock()
+	defer sock.mu.Unlock()
+	if sock.fd != -1 {
+		unix.Munmap(sock.buf)
+		sock.buf = nil
+		err = unix.Close(sock.fd)
+		sock.fd = -1
+	}
+	return
+}
+
+// SetSnapLen sets the maximum capture length to the given value.
+// for this to take effects on the kernel level SetBPFilter should be called too.
+func (sock *SockRaw) SetSnapLen(snap int) error {
+	sock.mu.Lock()
+	defer sock.mu.Unlock()
+	if snap < 0 {
+		return fmt.Errorf("expected %d snap length to be at least 0", snap)
+	}
+	if snap > FRAMESIZE {
+		snap = FRAMESIZE
+	}
+	sock.snaplen = snap
+	return nil
+}
+
+// SetTimeout sets poll wait timeout for the socket.
+// negative value will block forever
+func (sock *SockRaw) SetTimeout(t time.Duration) error {
+	sock.mu.Lock()
+	defer sock.mu.Unlock()
+	sock.pollTimeout = uintptr(t)
+	return nil
+}
+
+// GetSnapLen returns the maximum capture length
+func (sock *SockRaw) GetSnapLen() int {
+	sock.mu.Lock()
+	defer sock.mu.Unlock()
+	return sock.snaplen
+}
+
+// SetBPFFilter compiles and sets a BPF filter for the socket handle.
+func (sock *SockRaw) SetBPFFilter(expr string) error {
+	sock.mu.Lock()
+	defer sock.mu.Unlock()
+	if len(expr) == 0 {
+		return unix.SetsockoptInt(sock.fd, unix.SOL_SOCKET, unix.SO_DETACH_FILTER, 0)
+	}
+	filter, err := pcap.CompileBPFFilter(layers.LinkTypeEthernet, sock.snaplen, expr)
+	if err != nil {
+		return err
+	}
+	if len(filter) > int(^uint16(0)) {
+		return fmt.Errorf("filters out of range 0-%d", ^uint16(0))
+	}
+	if len(filter) == 0 {
+		return unix.SetsockoptInt(sock.fd, unix.SOL_SOCKET, unix.SO_DETACH_FILTER, 0)
+	}
+	fprog := &unix.SockFprog{
+		Len:    uint16(len(filter)),
+		Filter: &(*(*[]unix.SockFilter)(unsafe.Pointer(&filter)))[0],
+	}
+	return unix.SetsockoptSockFprog(sock.fd, unix.SOL_SOCKET, unix.SO_ATTACH_FILTER, fprog)
+}
+
+// SetPromiscuous sets promiscous mode to the required value. for better result capture on all interfaces instead.
+// If it is enabled, traffic not destined for the interface will also be captured.
+func (sock *SockRaw) SetPromiscuous(b bool) error {
+	sock.mu.Lock()
+	defer sock.mu.Unlock()
+	mreq := unix.PacketMreq{
+		Ifindex: int32(sock.ifindex),
+		Type:    unix.PACKET_MR_PROMISC,
+	}
+
+	opt := unix.PACKET_ADD_MEMBERSHIP
+	if !b {
+		opt = unix.PACKET_DROP_MEMBERSHIP
+	}
+
+	return unix.SetsockoptPacketMreq(sock.fd, unix.SOL_PACKET, opt, &mreq)
+}
+
+// Stats returns number of packets and dropped packets. This will be the number of packets/dropped packets since the last call to stats (not the cummulative sum!).
+func (sock *SockRaw) Stats() (*unix.TpacketStats, error) {
+	sock.mu.Lock()
+	defer sock.mu.Unlock()
+	return unix.GetsockoptTpacketStats(sock.fd, unix.SOL_PACKET, unix.PACKET_STATISTICS)
+}
+
+// SetLoopbackIndex necessary to avoid reading packet twice on a loopback device
+func (sock *SockRaw) SetLoopbackIndex(i int32) {
+	sock.mu.Lock()
+	defer sock.mu.Unlock()
+	sock.loopIndex = i
+}
+
+// WritePacketData transmits a raw packet.
+func (sock *SockRaw) WritePacketData(pkt []byte) error {
+	_, err := unix.Write(sock.fd, pkt)
+	return err
+}
+
+func tpAlign(x int) int {
+	return int((uint(x) + unix.TPACKET_ALIGNMENT - 1) &^ (unix.TPACKET_ALIGNMENT - 1))
+}

capture/sock_others.go

@@ -0,0 +1,13 @@
+// +build !linux
+
+package capture
+
+import (
+	"errors"
+	"net"
+)
+
+// NewSocket returns new M'maped sock_raw on packet version 2.
+func NewSocket(_ net.Interface) (Socket, error) {
+	return nil, errors.New("afpacket socket is only available on linux")
+}

capture/socket.go

@@ -0,0 +1,20 @@
+package capture
+
+import (
+	"time"
+
+	"github.com/google/gopacket"
+)
+
+// Socket is any interface that defines the behaviors of Socket
+type Socket interface {
+	ReadPacketData() ([]byte, gopacket.CaptureInfo, error)
+	WritePacketData([]byte) error
+	SetBPFFilter(string) error
+	SetPromiscuous(bool) error
+	SetSnapLen(int) error
+	GetSnapLen() int
+	SetTimeout(time.Duration) error
+	SetLoopbackIndex(i int32)
+	Close() error
+}

capture/tcp_message.go

@@ -1,522 +0,0 @@
-package capture
-
-import (
-	"bytes"
-	"crypto/sha1"
-	"encoding/binary"
-	"encoding/hex"
-	"net"
-	"strconv"
-	"strings"
-	"time"
-
-	"github.com/buger/goreplay/proto"
-)
-
-// TCPProtocol is a number to indicate type of protocol
-type TCPProtocol uint8
-
-const (
-	// ProtocolHTTP ...
-	ProtocolHTTP TCPProtocol = 0
-	// ProtocolBinary ...
-	ProtocolBinary TCPProtocol = 1
-)
-
-// TCPMessage ensure that all TCP packets for given request is received, and processed in right sequence
-// Its needed because all TCP message can be fragmented or re-transmitted
-//
-// Each TCP Packet have 2 ids: acknowledgment - message_id, and sequence - packet_id
-// Message can be compiled from unique packets with same message_id which sorted by sequence
-// Message is received if we didn't receive any packets for 2000ms
-type TCPMessage struct {
-	Seq         uint32
-	Ack         uint32
-	ResponseAck uint32
-	ResponseID  tcpID
-	DataAck     uint32
-	DataSeq     uint32
-
-	AssocMessage *TCPMessage
-	Start        time.Time
-	End          time.Time
-	IsIncoming   bool
-
-	packets []*TCPPacket
-
-	delChan chan *TCPMessage
-
-	protocol TCPProtocol
-
-	/* HTTP specific variables */
-	methodType    httpMethodType
-	bodyType      httpBodyType
-	expectType    httpExpectType
-	seqMissing    bool
-	headerPacket  int
-	contentLength int
-	complete      bool
-}
-
-// NewTCPMessage pointer created from a Acknowledgment number and a channel of messages readuy to be deleted
-func NewTCPMessage(Seq, Ack uint32, IsIncoming bool, protocol TCPProtocol, timestamp time.Time) (msg *TCPMessage) {
-	msg = &TCPMessage{Seq: Seq, Ack: Ack, IsIncoming: IsIncoming, protocol: protocol, Start: timestamp}
-	msg.Start = time.Now()
-
-	return
-}
-
-func (t *TCPMessage) packetsData() (d [][]byte) {
-	d = make([][]byte, len(t.packets))
-	for i, p := range t.packets {
-		d[i] = p.Data
-	}
-
-	return
-}
-
-// Bytes return message content
-func (t *TCPMessage) Bytes() (output []byte) {
-	for _, p := range t.packets {
-		output = append(output, p.Data...)
-	}
-
-	return output
-}
-
-// BodySize returns total body size
-func (t *TCPMessage) BodySize() (size int) {
-	if len(t.packets) == 0 || t.headerPacket == -1 {
-		return 0
-	}
-
-	size += len(proto.Body(t.packets[t.headerPacket].Data))
-
-	for _, p := range t.packets[t.headerPacket+1:] {
-		size += len(p.Data)
-	}
-
-	return
-}
-
-// Size returns total size of message
-func (t *TCPMessage) Size() (size int) {
-	if len(t.packets) == 0 {
-		return 0
-	}
-
-	for _, p := range t.packets {
-		size += len(p.Data)
-	}
-
-	return
-}
-
-// AddPacket to the message and ensure packet uniqueness
-// TCP allows that packet can be re-send multiple times
-func (t *TCPMessage) AddPacket(packet *TCPPacket) {
-	for _, pkt := range t.packets {
-		if packet.Seq == pkt.Seq {
-			return
-		}
-	}
-
-	// Packets not always captured in same Seq order, and sometimes we need to prepend
-	if len(t.packets) == 0 || packet.Seq > t.packets[len(t.packets)-1].Seq {
-		t.packets = append(t.packets, packet)
-	} else if packet.Seq < t.packets[0].Seq {
-		t.packets = append([]*TCPPacket{packet}, t.packets...)
-		t.Seq = packet.Seq // Message Seq should indicated starting seq
-	} else { // insert somewhere in the middle...
-		for i, p := range t.packets {
-			if packet.Seq < p.Seq {
-				t.packets = append(t.packets[:i], append([]*TCPPacket{packet}, t.packets[i:]...)...)
-				break
-			}
-		}
-
-		if packet.OrigAck != 0 {
-			t.DataAck = packet.OrigAck
-		}
-
-		if packet.timestamp.Before(t.Start) || t.Start.IsZero() {
-			t.Start = packet.timestamp
-		}
-
-		if packet.timestamp.After(t.End) || t.End.IsZero() {
-			t.End = packet.timestamp
-		}
-	}
-
-	t.checkSeqIntegrity()
-
-	if t.protocol == ProtocolHTTP {
-		t.updateHeadersPacket()
-		t.updateMethodType()
-		t.updateBodyType()
-		t.check100Continue()
-		t.checkIfComplete()
-	}
-}
-
-// Check if there is missing packet
-func (t *TCPMessage) checkSeqIntegrity() {
-	if len(t.packets) == 1 {
-		t.seqMissing = false
-	}
-
-	offset := len(t.packets) - 1
-
-	if t.packets[offset].IsFIN {
-		offset--
-
-		if offset < 0 {
-			return
-		}
-	}
-
-	for i, p := range t.packets[:offset] {
-		if p.IsFIN {
-			continue
-		}
-
-		// If final packet
-		if len(t.packets) == i+1 {
-			t.seqMissing = false
-			return
-		}
-		np := t.packets[i+1]
-
-		nextSeq := p.Seq + uint32(len(p.Data))
-
-		if np.Seq != nextSeq {
-			if t.protocol == ProtocolHTTP && t.expectType == httpExpect100Continue {
-				if np.Seq != nextSeq+22 {
-					t.seqMissing = true
-					return
-				}
-			} else {
-				t.seqMissing = true
-				return
-			}
-		}
-	}
-
-	t.seqMissing = false
-}
-
-var bEmptyLine = []byte("\r\n\r\n")
-var bBR = []byte("\r\n")
-
-// last-chunk always is 0\r\n\r\n\. More info https://tools.ietf.org/html/rfc2616#section-3.6.1
-var bChunkEnd = []byte("0\r\n\r\n")
-
-func (t *TCPMessage) updateHeadersPacket() {
-	if len(t.packets) == 1 {
-		t.headerPacket = -1
-	}
-
-	if t.headerPacket != -1 {
-		return
-	}
-
-	if t.seqMissing {
-		return
-	}
-
-	for i, p := range t.packets {
-		if len(p.Data) >= len(bEmptyLine) {
-			if bytes.LastIndex(p.Data, bEmptyLine) != -1 {
-				t.headerPacket = i
-				return
-			}
-		} else if i > 0 && bytes.Equal(p.Data, bBR) {
-			idx := bytes.LastIndex(t.packets[i-1].Data, bBR)
-			if idx != -1 && idx == len(t.packets[i-1].Data)-len(bBR) {
-				t.headerPacket = i
-				return
-			}
-		}
-	}
-
-	return
-}
-
-// checkIfComplete returns true if all of the packets that compse the message arrived.
-func (t *TCPMessage) checkIfComplete() {
-	if t.seqMissing || t.headerPacket == -1 {
-		// log.Println("Seq missing", t.seqMissing, t.packets)
-		return
-	}
-
-	if t.methodType == httpMethodNotFound {
-		// log.Println("Method missing", t.methodType, t.packets)
-		return
-	}
-
-	// Responses can be emitted only if we found request
-	if !t.IsIncoming && t.AssocMessage == nil {
-		// log.Println("Assoc not found", t)
-		return
-	}
-
-	// log.Println("Found?", t)
-
-	switch t.bodyType {
-	case httpBodyEmpty:
-		t.complete = true
-	case httpBodyContentLength:
-		if t.contentLength == 0 || t.contentLength == t.BodySize() {
-			t.complete = true
-		}
-	case httpBodyChunked:
-		lastPacket := t.packets[len(t.packets)-1]
-		if bytes.LastIndex(lastPacket.Data, bChunkEnd) != -1 {
-			t.complete = true
-		}
-	default:
-		if len(t.packets) == 0 {
-			return
-		}
-
-		last := t.packets[len(t.packets)-1]
-		if last.IsFIN {
-			t.complete = true
-		}
-	}
-}
-
-type httpMethodType uint8
-
-const (
-	httpMethodNotSet   httpMethodType = 0
-	httpMethodKnown    httpMethodType = 1
-	httpMethodNotFound httpMethodType = 2
-)
-
-func (t *TCPMessage) updateMethodType() {
-	// if there is cache
-	if t.methodType != httpMethodNotSet && t.methodType != httpMethodNotFound {
-		return
-	}
-
-	d := t.packets[0].Data
-
-	// Minimum length fo request: GET / HTTP/1.1\r\n
-
-	if len(d) < 16 {
-		t.methodType = httpMethodNotFound
-		return
-	}
-
-	if t.IsIncoming {
-		if mIdx := bytes.IndexByte(d[:8], ' '); mIdx != -1 {
-			// Check that after method we have absolute or relative path
-			switch d[mIdx+1] {
-			case '/', 'h', '*':
-			default:
-				t.methodType = httpMethodNotFound
-				return
-			}
-		} else {
-			t.methodType = httpMethodNotFound
-			return
-		}
-
-		t.methodType = httpMethodKnown
-	} else {
-		if !bytes.Equal(d[:6], []byte("HTTP/1")) {
-			t.methodType = httpMethodNotFound
-			return
-		}
-
-		t.methodType = httpMethodKnown
-	}
-}
-
-type httpBodyType uint8
-
-const (
-	httpBodyNotSet          httpBodyType = 0
-	httpBodyEmpty           httpBodyType = 1
-	httpBodyContentLength   httpBodyType = 2
-	httpBodyChunked         httpBodyType = 3
-	httpBodyConnectionClose httpBodyType = 4
-)
-
-func (t *TCPMessage) updateBodyType() {
-	// if there is cache
-	if t.bodyType != httpBodyNotSet {
-		return
-	}
-
-	// Headers not received
-	if t.headerPacket == -1 {
-		return
-	}
-
-	var lengthB, encB, connB []byte
-
-	proto.ParseHeaders(t.packetsData(), func(header, value []byte) bool {
-		if proto.HeadersEqual(header, []byte("Content-Length")) {
-			lengthB = value
-			return false
-		}
-
-		if proto.HeadersEqual(header, []byte("Transfer-Encoding")) {
-			encB = value
-			return false
-		}
-
-		if proto.HeadersEqual(header, []byte("Connection")) {
-			connB = value
-		}
-
-		return true
-	})
-
-	switch t.methodType {
-	case httpMethodNotFound:
-		return
-	case httpMethodKnown:
-
-		if !t.IsIncoming &&
-			t.AssocMessage != nil &&
-			bytes.IndexByte(t.AssocMessage.Bytes(), ' ') > -1 &&
-			bytes.Equal([]byte("HEAD"), proto.Method(t.AssocMessage.Bytes())) {
-			// Need to check if this is a response to a head request,
-			// in which case the body has to be empty regardless.
-			t.bodyType = httpBodyEmpty
-			return
-		}
-
-		if len(lengthB) > 0 {
-			t.contentLength, _ = strconv.Atoi(string(lengthB))
-			if t.contentLength == 0 {
-				t.bodyType = httpBodyEmpty
-			} else {
-				t.bodyType = httpBodyContentLength
-			}
-			return
-		}
-
-		if len(encB) > 0 {
-			t.bodyType = httpBodyChunked
-			return
-		}
-
-		if len(connB) > 0 && bytes.Equal(connB, []byte("close")) {
-			t.bodyType = httpBodyConnectionClose
-			return
-		}
-	}
-
-	t.bodyType = httpBodyEmpty
-}
-
-type httpExpectType uint8
-
-const (
-	httpExpectNotSet      httpExpectType = 0
-	httpExpectEmpty       httpExpectType = 1
-	httpExpect100Continue httpExpectType = 2
-)
-
-var bExpectHeader = []byte("Expect")
-var bExpect100Value = []byte("100-continue")
-
-func (t *TCPMessage) check100Continue() {
-	if t.expectType != httpExpectNotSet || len(t.packets[0].Data) < 25 {
-		return
-	}
-
-	if t.seqMissing || t.headerPacket == -1 {
-		return
-	}
-
-	last := t.packets[len(t.packets)-1]
-	// reading last 4 bytes for double CRLF
-	if !bytes.HasSuffix(last.Data, bEmptyLine) {
-		return
-	}
-
-	var expectB []byte
-	proto.ParseHeaders(t.packetsData(), func(header, value []byte) bool {
-		if proto.HeadersEqual(header, bExpectHeader) {
-			expectB = value
-			return false
-		}
-
-		return true
-	})
-
-	if len(expectB) > 0 && bytes.Equal(bExpect100Value, expectB) {
-		t.expectType = httpExpect100Continue
-		return
-	}
-
-	t.expectType = httpExpectEmpty
-}
-
-func (t *TCPMessage) setAssocMessage(m *TCPMessage) {
-	t.AssocMessage = m
-	t.checkIfComplete()
-}
-
-// UpdateResponseAck should be called after packet is added
-func (t *TCPMessage) UpdateResponseAck() uint32 {
-	lastPacket := t.packets[len(t.packets)-1]
-	if lastPacket.IsFIN && len(t.packets) > 1 {
-		lastPacket = t.packets[len(t.packets)-2]
-	}
-
-	respAck := lastPacket.Seq + uint32(len(lastPacket.Data))
-
-	if t.ResponseAck != respAck {
-		t.ResponseAck = lastPacket.Seq + uint32(len(lastPacket.Data))
-
-		// We swappwed src and dst port
-		copy(t.ResponseID[:16], lastPacket.Addr)
-		copy(t.ResponseID[16:], lastPacket.Raw[2:4]) // Src port
-		copy(t.ResponseID[18:], lastPacket.Raw[0:2]) // Dest port
-		binary.BigEndian.PutUint32(t.ResponseID[20:24], t.ResponseAck)
-	}
-
-	return t.ResponseAck
-}
-
-func (t *TCPMessage) UUID() []byte {
-	var key []byte
-
-	if t.IsIncoming {
-		// log.Println("UUID:", t.Ack, t.Start.UnixNano())
-		key = strconv.AppendInt(key, t.Start.UnixNano(), 10)
-		key = strconv.AppendUint(key, uint64(t.Ack), 10)
-	} else {
-		// log.Println("RequestMessage:", t.AssocMessage.Ack, t.AssocMessage.Start.UnixNano())
-		key = strconv.AppendInt(key, t.AssocMessage.Start.UnixNano(), 10)
-		key = strconv.AppendUint(key, uint64(t.AssocMessage.Ack), 10)
-	}
-
-	uuid := make([]byte, 40)
-	sha := sha1.Sum(key)
-	hex.Encode(uuid, sha[:20])
-
-	return uuid
-}
-
-func (t *TCPMessage) ID() tcpID {
-	return t.packets[0].ID
-}
-
-func (t *TCPMessage) IP() net.IP {
-	return net.IP(t.packets[0].Addr)
-}
-
-func (t *TCPMessage) String() string {
-	return strings.Join([]string{
-		"Len packets: " + strconv.Itoa(len(t.packets)),
-		"Data size:" + strconv.Itoa(len(t.Bytes())),
-		"Data:" + string(t.Bytes()),
-	}, "\n")
-}

capture/tcp_message_test.go

@@ -1,257 +0,0 @@
-package capture
-
-import (
-	"bytes"
-	"encoding/binary"
-	_ "log"
-	"testing"
-	"time"
-)
-
-func buildPacket(isIncoming bool, Ack, Seq uint32, Data []byte, timestamp time.Time) (packet *TCPPacket) {
-	var srcPort, destPort uint16
-
-	// For tests `listening` port is 0
-	if isIncoming {
-		srcPort = 1
-	} else {
-		destPort = 1
-	}
-
-	buf := make([]byte, 16)
-	binary.BigEndian.PutUint16(buf[2:4], destPort)
-	binary.BigEndian.PutUint16(buf[0:2], srcPort)
-	binary.BigEndian.PutUint32(buf[4:8], Seq)
-	binary.BigEndian.PutUint32(buf[8:12], Ack)
-	buf[12] = 64
-	buf = append(buf, Data...)
-
-	packet = ParseTCPPacket([]byte("123"), buf, timestamp)
-
-	return packet
-}
-
-func buildMessage(p *TCPPacket) *TCPMessage {
-	isIncoming := false
-	if p.SrcPort == 1 {
-		isIncoming = true
-	}
-	m := NewTCPMessage(p.Seq, p.Ack, isIncoming, ProtocolHTTP, p.timestamp)
-	m.AddPacket(p)
-
-	return m
-}
-
-func TestTCPMessagePacketsOrder(t *testing.T) {
-	msg := buildMessage(buildPacket(true, 1, 1, []byte("a"), time.Now()))
-	msg.AddPacket(buildPacket(true, 1, 2, []byte("b"), time.Now()))
-
-	if !bytes.Equal(msg.Bytes(), []byte("ab")) {
-		t.Error("Should contatenate packets in right order")
-	}
-
-	// When first packet have wrong order (Seq)
-	msg = buildMessage(buildPacket(true, 1, 2, []byte("b"), time.Now()))
-	msg.AddPacket(buildPacket(true, 1, 1, []byte("a"), time.Now()))
-
-	if !bytes.Equal(msg.Bytes(), []byte("ab")) {
-		t.Error("Should contatenate packets in right order")
-	}
-
-	// Should ignore packets with same sequence
-	msg = buildMessage(buildPacket(true, 1, 1, []byte("a"), time.Now()))
-	msg.AddPacket(buildPacket(true, 1, 1, []byte("a"), time.Now()))
-
-	if !bytes.Equal(msg.Bytes(), []byte("a")) {
-		t.Error("Should ignore packet with same Seq")
-	}
-}
-
-func TestTCPMessageSize(t *testing.T) {
-	msg := buildMessage(buildPacket(true, 1, 1, []byte("POST / HTTP/1.1\r\nContent-Length: 2\r\n\r\na"), time.Now()))
-	msg.AddPacket(buildPacket(true, 1, 2, []byte("b"), time.Now()))
-
-	if msg.BodySize() != 2 {
-		t.Error("Should count only body", msg.BodySize())
-	}
-
-	if msg.Size() != 40 {
-		t.Error("Should count all sizes", msg.Size())
-	}
-}
-
-func TestTCPMessageIsComplete(t *testing.T) {
-	testCases := []struct {
-		direction         bool
-		payload           string
-		assocMessage      bool
-		expectedCompleted bool
-	}{
-		{true, "GET / HTTP/1.1\r\n\r\n", false, true},
-		{true, "HEAD / HTTP/1.1\r\n\r\n", false, true},
-		{false, "HTTP/1.1 200 OK\r\n\r\n", true, true},
-		{true, "POST / HTTP/1.1\r\nContent-Length: 1\r\n\r\na", false, true},
-		{true, "PUT / HTTP/1.1\r\nContent-Length: 1\r\n\r\na", false, true},
-		{false, "HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n", true, true},
-		{false, "HTTP/1.1 200 OK\r\nContent-Length: 1\r\n\r\na", true, true},
-		{false, "HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n0\r\n\r\n", true, true},
-
-		// chunked not finished
-		{false, "HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n", true, false},
-
-		// content-length != actual length
-		{true, "POST / HTTP/1.1\r\nContent-Length: 2\r\n\r\na", false, false},
-		{false, "HTTP/1.1 200 OK\r\nContent-Length: 10\r\n\r\na", true, false},
-		// non-valid http request
-		{true, "UNKNOWN asd HTTP/1.1\r\n\r\n", false, false},
-
-		// response without associated request
-		{false, "HTTP/1.1 200 OK\r\n\r\n", false, false},
-	}
-
-	for _, tc := range testCases {
-		msg := buildMessage(buildPacket(tc.direction, 1, 1, []byte(tc.payload), time.Now()))
-		if tc.assocMessage {
-			msg.AssocMessage = &TCPMessage{}
-		}
-		msg.checkIfComplete()
-
-		if msg.complete != tc.expectedCompleted {
-			t.Errorf("Payload %s: Expected %t, got %t.", tc.payload, tc.expectedCompleted, msg.complete)
-		}
-	}
-}
-
-func TestTCPMessageIsSeqMissing(t *testing.T) {
-	p1 := buildPacket(false, 1, 1, []byte("HTTP/1.1 200 OK\r\n"), time.Now())
-	p2 := buildPacket(false, 1, p1.Seq+uint32(len(p1.Data)), []byte("Content-Length: 10\r\n\r\n"), time.Now())
-	p3 := buildPacket(false, 1, p2.Seq+uint32(len(p2.Data)), []byte("a"), time.Now())
-
-	msg := buildMessage(p1)
-	if msg.seqMissing {
-		t.Error("Should be complete if have only 1 packet")
-	}
-
-	msg.AddPacket(p3)
-	if !msg.seqMissing {
-		t.Error("Should be incomplete because missing middle component")
-	}
-
-	msg.AddPacket(p2)
-	if msg.seqMissing {
-		t.Error("Should be complete once missing packet added")
-	}
-}
-
-func TestTCPMessageIsHeadersReceived(t *testing.T) {
-	p1 := buildPacket(false, 1, 1, []byte("HTTP/1.1 200 OK\r\n\r\n"), time.Now())
-	p2 := buildPacket(false, 1, p1.Seq+uint32(len(p1.Data)), []byte("Content-Length: 10\r\n\r\n"), time.Now())
-
-	msg := buildMessage(p1)
-	if msg.headerPacket == -1 {
-		t.Error("Should be complete if have only 1 packet", msg.headerPacket)
-	}
-
-	msg.AddPacket(p2)
-	if msg.headerPacket == -1 {
-		t.Error("Should found double new line: headers received")
-	}
-
-	msg = buildMessage(buildPacket(true, 1, 1, []byte("GET / HTTP/1.1\r\nContent-Length: 1\r\n"), time.Now()))
-	if msg.headerPacket != -1 {
-		t.Error("Should not find headers end")
-	}
-}
-
-func TestTCPMessageMethodType(t *testing.T) {
-	testCases := []struct {
-		direction          bool
-		payload            string
-		expectedMethodType httpMethodType
-	}{
-		{true, "GET / HTTP/1.1\r\n\r\n", httpMethodKnown},
-		{true, "GET * HTTP/1.1\r\n\r\n", httpMethodKnown},
-		{true, "UNKNOWN / HTTP/1.1\r\n\r\n", httpMethodKnown},
-		{true, "GET http://example.com HTTP/1.1\r\n\r\n", httpMethodKnown},
-		{true, "POST / HTTP/1.1\r\n\r\n", httpMethodKnown},
-		{true, "PUT / HTTP/1.1\r\n\r\n", httpMethodKnown},
-		{true, "GET zxc HTTP/1.1\r\n\r\n", httpMethodNotFound},
-		{true, "GET / HTTP\r\n\r\n", httpMethodNotFound},
-		{true, "VERYLONGMETHOD / HTTP/1.1\r\n\r\n", httpMethodNotFound},
-		{false, "HTTP/1.1 200 OK\r\n\r\n", httpMethodKnown},
-		{false, "HTTP /1.1 200 OK\r\n\r\n", httpMethodNotFound},
-	}
-
-	for _, tc := range testCases {
-		msg := buildMessage(buildPacket(tc.direction, 1, 1, []byte(tc.payload), time.Now()))
-
-		if msg.methodType != tc.expectedMethodType {
-			t.Errorf("Expected %d, got %d", tc.expectedMethodType, msg.methodType)
-		}
-	}
-}
-
-func TestTCPMessageBodyType(t *testing.T) {
-	testCases := []struct {
-		direction        bool
-		payload          string
-		expectedBodyType httpBodyType
-	}{
-		{true, "GET / HTTP/1.1\r\n\r\n", httpBodyEmpty},
-		{true, "GET / HTTP/1.1\r\nContent-Length: 2\r\n\r\nab", httpBodyContentLength},
-		{true, "POST / HTTP/1.1\r\n\r\n", httpBodyEmpty},
-		{true, "POST / HTTP/1.1\r\nUser-Agent: zxc\r\n\r\n", httpBodyEmpty},
-		{false, "HTTP/1.1 200 OK\r\n\r\n", httpBodyEmpty},
-		{true, "POST / HTTP/1.1\r\nContent-Length: 2\r\n\r\nab", httpBodyContentLength},
-		{false, "HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nab", httpBodyContentLength},
-		{true, "POST / HTTP/1.1\r\nTransfer-Encoding: chunked\r\n\r\n2\r\nab\r\n0\r\n\r\n", httpBodyChunked},
-		{false, "HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n2\r\nab\r\n0\r\n\r\n", httpBodyChunked},
-	}
-
-	for _, tc := range testCases {
-		msg := buildMessage(buildPacket(tc.direction, 1, 1, []byte(tc.payload), time.Now()))
-
-		if msg.bodyType != tc.expectedBodyType {
-			t.Errorf("Expected %d, got %d", tc.expectedBodyType, msg.bodyType)
-		}
-	}
-}
-
-func TestTCPMessageBodySize(t *testing.T) {
-	testCases := []struct {
-		direction    bool
-		payloads     []string
-		expectedSize int
-	}{
-		{true, []string{"GET / HTTP/1.1\r\n\r\n"}, 0},
-		{true, []string{"POST / HTTP/1.1\r\nContent-Length: 2\r\n\r\nab"}, 2},
-		{true, []string{"GET / HTTP/1.1\r\n", "Content-Length: 2\r\n\r\nab"}, 2},
-		{true, []string{"GET / HTTP/1.1\r\n", "Content-Length: 2\r\n\r\n", "ab"}, 2},
-	}
-
-	for _, tc := range testCases {
-		msg := buildMessage(buildPacket(tc.direction, 1, 1, []byte(tc.payloads[0]), time.Now()))
-
-		if len(tc.payloads) > 1 {
-			for _, p := range tc.payloads[1:] {
-				seq := uint32(1 + msg.Size())
-				msg.AddPacket(buildPacket(tc.direction, 1, seq, []byte(p), time.Now()))
-			}
-		}
-
-		if msg.BodySize() != tc.expectedSize {
-			t.Errorf("Expected %d, got %d", tc.expectedSize, msg.BodySize())
-		}
-	}
-}
-
-func TestTcpMessageStart(t *testing.T) {
-	start := time.Now().Add(-1 * time.Second)
-
-	msg := buildMessage(buildPacket(true, 1, 2, []byte("b"), time.Now()))
-	msg.AddPacket(buildPacket(true, 1, 1, []byte("POST / HTTP/1.1\r\nContent-Length: 2\r\n\r\na"), start))
-
-	if msg.Start != start {
-		t.Error("Message timestamp should be equal to the lowest related packet timestamp", start, msg.Start)
-	}
-}

capture/tcp_packet.go

@@ -1,129 +0,0 @@
-package capture
-
-import (
-	"encoding/binary"
-	"strconv"
-	"strings"
-	"time"
-)
-
-// TCP Flags
-const (
-	fFIN = 1 << iota
-	fSYN
-	fRST
-	fPSH
-	fACK
-	fURG
-	fECE
-	fCWR
-	fNS
-)
-
-type tcpID [24]byte
-
-// TCPPacket provides tcp packet parser
-// Packet structure: http://en.wikipedia.org/wiki/Transmission_Control_Protocol
-type TCPPacket struct {
-	SrcPort    uint16
-	DestPort   uint16
-	Seq        uint32
-	Ack        uint32
-	OrigAck    uint32
-	DataOffset uint8
-	IsFIN      bool
-
-	Raw       []byte
-	Data      []byte
-	Addr      []byte
-	timestamp time.Time
-	ID        tcpID
-}
-
-// ParseTCPPacket takes address and tcp payload and returns parsed TCPPacket
-func ParseTCPPacket(addr []byte, data []byte, timestamp time.Time) (p *TCPPacket) {
-	p = &TCPPacket{Raw: data}
-	p.ParseBasic()
-	p.Addr = addr
-	p.timestamp = timestamp
-	p.GenID()
-
-	return
-}
-
-func (p *TCPPacket) GenID() {
-	copy(p.ID[:16], p.Addr)
-	copy(p.ID[16:], p.Raw[0:2])  // Src port
-	copy(p.ID[18:], p.Raw[2:4])  // Dest port
-	copy(p.ID[20:], p.Raw[8:12]) // Ack
-}
-
-func (p *TCPPacket) UpdateAck(ack uint32) {
-	p.OrigAck = p.Ack
-	p.Ack = ack
-	binary.BigEndian.PutUint32(p.Raw[8:12], ack)
-	p.GenID()
-}
-
-// ParseBasic set of fields
-func (t *TCPPacket) ParseBasic() {
-	t.DestPort = binary.BigEndian.Uint16(t.Raw[2:4])
-	t.SrcPort = binary.BigEndian.Uint16(t.Raw[0:2])
-	t.Seq = binary.BigEndian.Uint32(t.Raw[4:8])
-	t.Ack = binary.BigEndian.Uint32(t.Raw[8:12])
-	t.DataOffset = (t.Raw[12] & 0xF0) >> 4
-	t.IsFIN = t.Raw[13]&0x01 != 0
-
-	if len(t.Raw) >= int(t.DataOffset*4) {
-		t.Data = t.Raw[t.DataOffset*4:]
-	}
-}
-
-func (t *TCPPacket) dump() *packet {
-
-	packetSrcIP := make([]byte, 16)
-	packetData := make([]byte, len(t.Data)+16)
-
-	copy(packetSrcIP, t.Addr)
-
-	binary.BigEndian.PutUint16(packetData[0:2], t.SrcPort)
-	binary.BigEndian.PutUint16(packetData[2:4], t.DestPort)
-
-	binary.BigEndian.PutUint32(packetData[4:8], t.Seq)
-	binary.BigEndian.PutUint32(packetData[8:12], t.Ack)
-
-	packetData[12] = 64
-
-	if t.IsFIN {
-		packetData[13] = packetData[13] | 0x01
-	}
-
-	copy(packetData[16:], t.Data)
-	return &packet{
-		srcIP:     packetSrcIP,
-		data:      packetData,
-		timestamp: t.timestamp,
-	}
-
-}
-
-// String output for a TCP Packet
-func (t *TCPPacket) String() string {
-	maxLen := len(t.Data)
-	if maxLen > 200 {
-		maxLen = 200
-	}
-
-	return strings.Join([]string{
-		"Addr: " + string(t.Addr),
-		"Source port: " + strconv.Itoa(int(t.SrcPort)),
-		"Dest port:" + strconv.Itoa(int(t.DestPort)),
-		"Sequence:" + strconv.Itoa(int(t.Seq)),
-		"Acknowledgment:" + strconv.Itoa(int(t.Ack)),
-		"Header len:" + strconv.Itoa(int(t.DataOffset)),
-		"FIN:" + strconv.FormatBool(t.IsFIN),
-
-		"Data size:" + strconv.Itoa(len(t.Data)),
-		"Data:" + string(t.Data[:maxLen]),
-	}, "\n")
-}

elasticsearch.go

@@ -99,13 +99,9 @@ func (p *ESPlugin) Init(URI string) {
 	p.done = make(chan bool)
 	p.indexor.Start()
 
-	if Settings.Verbose {
-		// Only start the ErrorHandler goroutine when in verbose mode
-		// no need to burn ressources otherwise
-		go p.ErrorHandler()
-	}
+	go p.ErrorHandler()
 
-	log.Println("Initialized Elasticsearch Plugin")
+	Debug(1, "Initialized Elasticsearch Plugin")
 	return
 }
 
@@ -117,7 +113,7 @@ func (p *ESPlugin) IndexerShutdown() {
 func (p *ESPlugin) ErrorHandler() {
 	for {
 		errBuf := <-p.indexor.ErrorChannel
-		log.Println(errBuf.Err)
+		Debug(1, "[ELASTICSEARCH]", errBuf.Err)
 	}
 }
 
@@ -128,6 +124,7 @@ func (p *ESPlugin) RttDurationToMs(d time.Duration) int64 {
 	return int64(fl)
 }
 
+// ResponseAnalyze send req and resp to ES
 func (p *ESPlugin) ResponseAnalyze(req, resp []byte, start, stop time.Time) {
 	if len(resp) == 0 {
 		// nil http response - skipped elasticsearch export for this request
@@ -135,7 +132,6 @@ func (p *ESPlugin) ResponseAnalyze(req, resp []byte, start, stop time.Time) {
 	}
 	t := time.Now()
 	rtt := p.RttDurationToMs(stop.Sub(start))
-	req = payloadBody(req)
 
 	esResp := ESRequestResponse{
 		ReqURL:               string(proto.Path(req)),
@@ -163,7 +159,7 @@ func (p *ESPlugin) ResponseAnalyze(req, resp []byte, start, stop time.Time) {
 	}
 	j, err := json.Marshal(&esResp)
 	if err != nil {
-		log.Println(err)
+		Debug(0, "[ELASTIC-RESPONSE]", err)
 	} else {
 		p.indexor.Index(p.Index, "RequestResponse", "", "", "", &t, j)
 	}

emitter.go

@@ -1,30 +1,46 @@
 package main
 
 import (
-	"bytes"
+	"fmt"
 	"hash/fnv"
 	"io"
 	"log"
 	"sync"
 	"time"
+
+	"github.com/buger/goreplay/byteutils"
+	"github.com/buger/goreplay/stat"
 )
 
-type emitter struct {
-	sync.WaitGroup
-	quit chan int
+// EmitterConfig configurations associated with emitters
+type EmitterConfig struct {
+	Stats                bool
+	SplitOutput          bool
+	RecognizeTCPSessions bool
 }
 
-// NewEmitter creates and initializes new `emitter` object.
-func NewEmitter(quit chan int) *emitter {
-	return &emitter{
-		quit: quit,
-	}
+// Emitter represents an abject to manage plugins communication
+type Emitter struct {
+	sync.WaitGroup
+	plugins *InOutPlugins
+	st      *stat.Writer
+}
+
+// NewEmitter creates and initializes new Emitter object.
+func NewEmitter() *Emitter {
+	return &Emitter{}
 }
 
 // Start initialize loop for sending data from inputs to outputs
-func (e *emitter) Start(plugins *InOutPlugins, middlewareCmd string) {
-	e.Add(1)
-	defer e.Done()
+func (e *Emitter) Start(plugins *InOutPlugins, middlewareCmd string) {
+	if Settings.CopyBufferSize < 1 {
+		Settings.CopyBufferSize = 5 << 20
+	}
+	e.plugins = plugins
+	if Settings.EmitterConfig.Stats {
+		e.st = initializeStatWriter("meta.csv", "type: GOR_STAT\ntitle: Messages Meta")
+		statsWriteRecord(e.st, []string{"type", "id", "timestamp", "latency"}, "[EMITTER]")
+	}
 
 	if middlewareCmd != "" {
 		middleware := NewMiddleware(middlewareCmd)
@@ -33,159 +49,102 @@ func (e *emitter) Start(plugins *InOutPlugins, middlewareCmd string) {
 			middleware.ReadFrom(in)
 		}
 
-		// We are going only to read responses, so using same ReadFrom method
-		for _, out := range plugins.Outputs {
-			if r, ok := out.(io.Reader); ok {
-				middleware.ReadFrom(r)
-			}
-		}
+		e.plugins.Inputs = append(e.plugins.Inputs, middleware)
+		e.plugins.All = append(e.plugins.All, middleware)
 		e.Add(1)
 		go func() {
 			defer e.Done()
-			if err := CopyMulty(e.quit, middleware, plugins.Outputs...); err != nil {
-				log.Println("Error during copy: ", err)
-				e.close()
-			}
-		}()
-		go func() {
-			for {
-				select {
-				case <-e.quit:
-					middleware.Close()
-					return
-				}
+			if err := e.CopyMulty(middleware, plugins.Outputs...); err != nil {
+				Debug(1, fmt.Sprintf("[EMITTER] error during copy: %q", err))
 			}
 		}()
 	} else {
 		for _, in := range plugins.Inputs {
 			e.Add(1)
-			go func(in io.Reader) {
+			go func(in PluginReader) {
 				defer e.Done()
-				if err := CopyMulty(e.quit, in, plugins.Outputs...); err != nil {
-					log.Println("Error during copy: ", err)
-					e.close()
+				if err := e.CopyMulty(in, plugins.Outputs...); err != nil {
+					Debug(1, fmt.Sprintf("[EMITTER] error during copy: %q", err))
 				}
 			}(in)
 		}
-
-		for _, out := range plugins.Outputs {
-			if r, ok := out.(io.Reader); ok {
-				e.Add(1)
-				go func(r io.Reader) {
-					defer e.Done()
-					if err := CopyMulty(e.quit, r, plugins.Outputs...); err != nil {
-						log.Println("Error during copy: ", err)
-						e.close()
-					}
-				}(r)
-			}
-		}
-	}
-
-	for {
-		select {
-		case <-e.quit:
-			finalize(plugins)
-			return
-		case <-time.After(100 * time.Millisecond):
-		}
-	}
-}
-
-func (e *emitter) close() {
-	select {
-	case <-e.quit:
-	default:
-		close(e.quit)
 	}
 }
 
 // Close closes all the goroutine and waits for it to finish.
-func (e *emitter) Close() {
-	e.close()
-	e.Wait()
+func (e *Emitter) Close() {
+	for _, p := range e.plugins.All {
+		if cp, ok := p.(io.Closer); ok {
+			cp.Close()
+		}
+	}
+	if len(e.plugins.All) > 0 {
+		// wait for everything to stop
+		e.Wait()
+	}
+	e.plugins.All = nil // avoid Close to make changes again
+	if e.st != nil {
+		e.st.Close()
+	}
 }
 
 // CopyMulty copies from 1 reader to multiple writers
-func CopyMulty(stop chan int, src io.Reader, writers ...io.Writer) error {
-	buf := make([]byte, Settings.copyBufferSize)
+func (e *Emitter) CopyMulty(src PluginReader, writers ...PluginWriter) error {
 	wIndex := 0
 	modifier := NewHTTPModifier(&Settings.ModifierConfig)
-	filteredRequests := make(map[string]time.Time)
-	filteredRequestsLastCleanTime := time.Now()
+	filteredRequests := make(map[string]int64)
+	filteredRequestsLastCleanTime := time.Now().UnixNano()
+	filteredCount := 0
 
-	i := 0
 	for {
-		var nr int
-		nr, err := src.Read(buf)
-
-		select {
-		case <-stop:
-			return nil
-		default:
-		}
-
-		if err == io.EOF || err == ErrorStopped {
-			return nil
-		}
+		msg, err := src.PluginRead()
 		if err != nil {
+			if err == ErrorStopped || err == io.EOF {
+				return nil
+			}
 			return err
 		}
-
-		_maxN := nr
-		if nr > 500 {
-			_maxN = 500
-		}
-		if nr > 0 && len(buf) > nr {
-			payload := buf[:nr]
-			meta := payloadMeta(payload)
+		if msg != nil && len(msg.Data) > 0 {
+			if len(msg.Data) > int(Settings.CopyBufferSize) {
+				msg.Data = msg.Data[:Settings.CopyBufferSize]
+			}
+			meta := payloadMeta(msg.Meta)
 			if len(meta) < 3 {
-				if Settings.Debug {
-					Debug("[EMITTER] Found malformed record", string(payload[0:_maxN]), nr, "from:", src)
-				}
+				Debug(2, fmt.Sprintf("[EMITTER] Found malformed record %q from %q", msg.Meta, src))
 				continue
 			}
-			requestID := string(meta[1])
-
-			if nr >= 5*1024*1024 {
-				log.Println("INFO: Large packet... We received ", len(payload), " bytes from ", src)
+			if e.st != nil {
+				statsWriteRecord(e.st, splitUnsafe(meta), "[EMITTER]")
 			}
-
-			if Settings.Debug {
-				Debug("[EMITTER] input:", string(payload[0:_maxN]), nr, "from:", src)
+			requestID := byteutils.SliceToString(meta[1])
+			// start a subroutine only when necessary
+			if Settings.Verbose >= 3 {
+				Debug(3, "[EMITTER] input: ", byteutils.SliceToString(msg.Meta[:len(msg.Meta)-1]), " from: ", src)
 			}
-
 			if modifier != nil {
-				if isRequestPayload(payload) {
-					headSize := bytes.IndexByte(payload, '\n') + 1
-					body := payload[headSize:]
-					originalBodyLen := len(body)
-					body = modifier.Rewrite(body)
-
+				Debug(3, "[EMITTER] modifier: ", requestID, " from: ", src)
+				if isRequestPayload(msg.Meta) {
+					msg.Data = modifier.Rewrite(msg.Data)
 					// If modifier tells to skip request
-					if len(body) == 0 {
-						filteredRequests[requestID] = time.Now()
+					if len(msg.Data) == 0 {
+						filteredRequests[requestID] = time.Now().UnixNano()
+						filteredCount++
 						continue
 					}
+					Debug(3, "[EMITTER] Rewritten input:", requestID, "from:", src)
 
-					if originalBodyLen != len(body) {
-						payload = append(payload[:headSize], body...)
-					}
-
-					if Settings.Debug {
-						Debug("[EMITTER] Rewritten input:", len(payload), "First 500 bytes:", string(payload[0:_maxN]))
-					}
 				} else {
 					if _, ok := filteredRequests[requestID]; ok {
 						delete(filteredRequests, requestID)
+						filteredCount--
 						continue
 					}
 				}
 			}
 
 			if Settings.PrettifyHTTP {
-				payload = prettifyHTTP(payload)
-				if len(payload) == 0 {
+				msg.Data = prettifyHTTP(msg.Data)
+				if len(msg.Data) == 0 {
 					continue
 				}
 			}
@@ -196,49 +155,50 @@ func CopyMulty(stop chan int, src io.Reader, writers ...io.Writer) error {
 						log.Fatal("Detailed TCP sessions work only with PRO license")
 					}
 					hasher := fnv.New32a()
-					// First 20 bytes contain tcp session
-					id := payloadID(payload)
-					hasher.Write(id[:20])
+					hasher.Write(meta[1])
 
 					wIndex = int(hasher.Sum32()) % len(writers)
-					writers[wIndex].Write(payload)
+					if _, err := writers[wIndex].PluginWrite(msg); err != nil {
+						return err
+					}
 				} else {
 					// Simple round robin
-					if _, err := writers[wIndex].Write(payload); err != nil {
+					if _, err := writers[wIndex].PluginWrite(msg); err != nil {
 						return err
 					}
 
-					wIndex++
-
-					if wIndex >= len(writers) {
-						wIndex = 0
-					}
+					wIndex = (wIndex + 1) % len(writers)
 				}
 			} else {
 				for _, dst := range writers {
-					if _, err := dst.Write(payload); err != nil {
+					if _, err := dst.PluginWrite(msg); err != nil {
 						return err
 					}
 				}
 			}
-		} else if nr > 0 {
-			log.Println("WARN: Packet", nr, "bytes is too large to process. Consider increasing --copy-buffer-size")
 		}
 
 		// Run GC on each 1000 request
-		if i%1000 == 0 {
+		if filteredCount > 0 && filteredCount%1000 == 0 {
 			// Clean up filtered requests for which we didn't get a response to filter
-			now := time.Now()
-			if now.Sub(filteredRequestsLastCleanTime) > 60*time.Second {
+			now := time.Now().UnixNano()
+			if now-filteredRequestsLastCleanTime > int64(60*time.Second) {
 				for k, v := range filteredRequests {
-					if now.Sub(v) > 60*time.Second {
+					if now-v > int64(60*time.Second) {
 						delete(filteredRequests, k)
+						filteredCount--
 					}
 				}
-				filteredRequestsLastCleanTime = time.Now()
+				filteredRequestsLastCleanTime = time.Now().UnixNano()
 			}
 		}
-
-		i++
 	}
 }
+
+func splitUnsafe(data [][]byte) []string {
+	s := make([]string, len(data), len(data))
+	for i := range data {
+		s[i] = byteutils.SliceToString(data[i])
+	}
+	return s
+}

emitter_test.go

@@ -1,8 +1,7 @@
 package main
 
 import (
-	"bytes"
-	"io"
+	"fmt"
 	"os"
 	"sync"
 	"sync/atomic"
@@ -18,20 +17,19 @@ func TestMain(m *testing.M) {
 
 func TestEmitter(t *testing.T) {
 	wg := new(sync.WaitGroup)
-	quit := make(chan int)
 
 	input := NewTestInput()
-	output := NewTestOutput(func(data []byte) {
+	output := NewTestOutput(func(*Message) {
 		wg.Done()
 	})
 
 	plugins := &InOutPlugins{
-		Inputs:  []io.Reader{input},
-		Outputs: []io.Writer{output},
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{output},
 	}
 	plugins.All = append(plugins.All, input, output)
 
-	emitter := NewEmitter(quit)
+	emitter := NewEmitter()
 	go emitter.Start(plugins, Settings.Middleware)
 
 	for i := 0; i < 1000; i++ {
@@ -45,32 +43,31 @@ func TestEmitter(t *testing.T) {
 
 func TestEmitterFiltered(t *testing.T) {
 	wg := new(sync.WaitGroup)
-	quit := make(chan int)
 
 	input := NewTestInput()
 	input.skipHeader = true
 
-	output := NewTestOutput(func(data []byte) {
+	output := NewTestOutput(func(*Message) {
 		wg.Done()
 	})
 
 	plugins := &InOutPlugins{
-		Inputs:  []io.Reader{input},
-		Outputs: []io.Writer{output},
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{output},
 	}
 	plugins.All = append(plugins.All, input, output)
 
 	methods := HTTPMethods{[]byte("GET")}
 	Settings.ModifierConfig = HTTPModifierConfig{Methods: methods}
 
-	emitter := &emitter{quit: quit}
+	emitter := &Emitter{}
 	go emitter.Start(plugins, "")
 
 	wg.Add(2)
 
 	id := uuid()
 	reqh := payloadHeader(RequestPayload, id, time.Now().UnixNano(), -1)
-	reqb := append(reqh, []byte("GET / HTTP/1.1\r\nHost: www.w3.org\r\nUser-Agent: Go 1.1 package http\r\nAccept-Encoding: gzip\r\n\r\n")...)
+	reqb := append(reqh, []byte("POST / HTTP/1.1\r\nHost: www.w3.org\r\nUser-Agent: Go 1.1 package http\r\nAccept-Encoding: gzip\r\n\r\n")...)
 
 	resh := payloadHeader(ResponsePayload, id, time.Now().UnixNano()+1, 1)
 	respb := append(resh, []byte("HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n")...)
@@ -80,7 +77,7 @@ func TestEmitterFiltered(t *testing.T) {
 
 	id = uuid()
 	reqh = payloadHeader(RequestPayload, id, time.Now().UnixNano(), -1)
-	reqb = append(reqh, []byte("POST / HTTP/1.1\r\nHost: www.w3.org\r\nUser-Agent: Go 1.1 package http\r\nAccept-Encoding: gzip\r\n\r\n")...)
+	reqb = append(reqh, []byte("GET / HTTP/1.1\r\nHost: www.w3.org\r\nUser-Agent: Go 1.1 package http\r\nAccept-Encoding: gzip\r\n\r\n")...)
 
 	resh = payloadHeader(ResponsePayload, id, time.Now().UnixNano()+1, 1)
 	respb = append(resh, []byte("HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n")...)
@@ -96,30 +93,29 @@ func TestEmitterFiltered(t *testing.T) {
 
 func TestEmitterSplitRoundRobin(t *testing.T) {
 	wg := new(sync.WaitGroup)
-	quit := make(chan int)
 
 	input := NewTestInput()
 
 	var counter1, counter2 int32
 
-	output1 := NewTestOutput(func(data []byte) {
+	output1 := NewTestOutput(func(*Message) {
 		atomic.AddInt32(&counter1, 1)
 		wg.Done()
 	})
 
-	output2 := NewTestOutput(func(data []byte) {
+	output2 := NewTestOutput(func(*Message) {
 		atomic.AddInt32(&counter2, 1)
 		wg.Done()
 	})
 
 	plugins := &InOutPlugins{
-		Inputs:  []io.Reader{input},
-		Outputs: []io.Writer{output1, output2},
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{output1, output2},
 	}
 
 	Settings.SplitOutput = true
 
-	emitter := NewEmitter(quit)
+	emitter := NewEmitter()
 	go emitter.Start(plugins, Settings.Middleware)
 
 	for i := 0; i < 1000; i++ {
@@ -140,31 +136,30 @@ func TestEmitterSplitRoundRobin(t *testing.T) {
 
 func TestEmitterRoundRobin(t *testing.T) {
 	wg := new(sync.WaitGroup)
-	quit := make(chan int)
 
 	input := NewTestInput()
 
 	var counter1, counter2 int32
 
-	output1 := NewTestOutput(func(data []byte) {
-		atomic.AddInt32(&counter1, 1)
+	output1 := NewTestOutput(func(*Message) {
+		counter1++
 		wg.Done()
 	})
 
-	output2 := NewTestOutput(func(data []byte) {
-		atomic.AddInt32(&counter2, 1)
+	output2 := NewTestOutput(func(*Message) {
+		counter2++
 		wg.Done()
 	})
 
 	plugins := &InOutPlugins{
-		Inputs:  []io.Reader{input},
-		Outputs: []io.Writer{output1, output2},
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{output1, output2},
 	}
 	plugins.All = append(plugins.All, input, output1, output2)
 
 	Settings.SplitOutput = true
 
-	emitter := NewEmitter(quit)
+	emitter := NewEmitter()
 	go emitter.Start(plugins, Settings.Middleware)
 
 	for i := 0; i < 1000; i++ {
@@ -183,67 +178,53 @@ func TestEmitterRoundRobin(t *testing.T) {
 }
 
 func TestEmitterSplitSession(t *testing.T) {
-	wg1 := new(sync.WaitGroup)
-	wg2 := new(sync.WaitGroup)
-	wg1.Add(1000)
-	wg2.Add(1000)
-
-	// Base uuids, only 1 letter changed
-	uuid1 := []byte("1234567890123456789a0000")
-	uuid2 := []byte("1234567890123456789d0000")
-
-	quit := make(chan int)
+	wg := new(sync.WaitGroup)
+	wg.Add(200)
 
 	input := NewTestInput()
 	input.skipHeader = true
 
 	var counter1, counter2 int32
 
-	output1 := NewTestOutput(func(data []byte) {
-		atomic.AddInt32(&counter1, 1)
-		if !bytes.Equal(uuid1[:20], payloadID(data)[:20]) {
-			t.Errorf("All tcp sessions should have same id")
+	output1 := NewTestOutput(func(msg *Message) {
+		if payloadID(msg.Meta)[0] == 'a' {
+			counter1++
 		}
-
-		wg1.Done()
+		wg.Done()
 	})
 
-	output2 := NewTestOutput(func(data []byte) {
-		atomic.AddInt32(&counter2, 1)
-		if !bytes.Equal(uuid2[:20], payloadID(data)[:20]) {
-			t.Errorf("All tcp sessions should have same id")
+	output2 := NewTestOutput(func(msg *Message) {
+		if payloadID(msg.Meta)[0] == 'b' {
+			counter2++
 		}
-
-		wg2.Done()
+		wg.Done()
 	})
 
 	plugins := &InOutPlugins{
-		Inputs:  []io.Reader{input},
-		Outputs: []io.Writer{output1, output2},
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{output1, output2},
 	}
 
 	Settings.SplitOutput = true
 	Settings.RecognizeTCPSessions = true
 
-	emitter := NewEmitter(quit)
+	emitter := NewEmitter()
 	go emitter.Start(plugins, Settings.Middleware)
 
-	for i := 0; i < 1000; i++ {
-		// Keep session but randomize ACK
-		copy(uuid1[20:], randByte(4))
-		input.EmitBytes([]byte("1 " + string(uuid1) + " 1\n" + "GET / HTTP/1.1\r\n\r\n"))
+	for i := 0; i < 200; i++ {
+		// Keep session but randomize
+		id := make([]byte, 20)
+		if i&1 == 0 { // for recognizeTCPSessions one should be odd and other will be even number
+			id[0] = 'a'
+		} else {
+			id[0] = 'b'
+		}
+		input.EmitBytes([]byte(fmt.Sprintf("1 %s 1 1\nGET / HTTP/1.1\r\n\r\n", id[:20])))
 	}
 
-	for i := 0; i < 1000; i++ {
-		// Keep session but randomize ACK
-		copy(uuid2[20:], randByte(4))
-		input.EmitBytes([]byte("1 " + string(uuid2) + " 1\n" + "GET / HTTP/1.1\r\n\r\n"))
-	}
+	wg.Wait()
 
-	wg1.Wait()
-	wg2.Wait()
-
-	if counter1 != 1000 || counter2 != 1000 {
+	if counter1 != counter2 {
 		t.Errorf("Round robin should split traffic equally: %d vs %d", counter1, counter2)
 	}
 
@@ -254,21 +235,20 @@ func TestEmitterSplitSession(t *testing.T) {
 
 func BenchmarkEmitter(b *testing.B) {
 	wg := new(sync.WaitGroup)
-	quit := make(chan int)
 
 	input := NewTestInput()
 
-	output := NewTestOutput(func(data []byte) {
+	output := NewTestOutput(func(*Message) {
 		wg.Done()
 	})
 
 	plugins := &InOutPlugins{
-		Inputs:  []io.Reader{input},
-		Outputs: []io.Writer{output},
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{output},
 	}
 	plugins.All = append(plugins.All, input, output)
 
-	emitter := NewEmitter(quit)
+	emitter := NewEmitter()
 	go emitter.Start(plugins, Settings.Middleware)
 
 	b.ResetTimer()

examples/middleware/echo.sh

@@ -8,8 +8,10 @@
 # 
 
 function log {
+    if $GOR_TEST != ""; then # if we are not testing
     # Logging to stderr, because stdout/stdin used for data transfer
     >&2 echo "[DEBUG][ECHO] $1"
+    fi
 }
 
 while read line; do

examples/middleware/token_modifier.go

@@ -115,6 +115,8 @@ func encode(buf []byte) []byte {
 }
 
 func Debug(args ...interface{}) {
-	fmt.Fprint(os.Stderr, "[DEBUG][TOKEN-MOD] ")
-	fmt.Fprintln(os.Stderr, args...)
+	if os.Getenv("GOR_TEST") != "" { // if we are not testing
+		fmt.Fprint(os.Stderr, "[DEBUG][TOKEN-MOD] ")
+		fmt.Fprintln(os.Stderr, args...)
+	}
 }

go.mod

@@ -8,11 +8,12 @@ require (
 	github.com/aws/aws-sdk-go v1.33.2
 	github.com/bitly/go-hostpool v0.1.0 // indirect
 	github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 // indirect
-	github.com/google/gopacket v1.1.17
+	github.com/google/gopacket v1.1.18
 	github.com/klauspost/compress v1.10.10 // indirect
 	github.com/mattbaird/elastigo v0.0.0-20170123220020-2fe47fd29e4b
 	github.com/pierrec/lz4 v2.5.2+incompatible // indirect
 	github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0 // indirect
 	github.com/smartystreets/goconvey v1.6.4 // indirect
 	golang.org/x/net v0.0.0-20200707034311-ab3426394381 // indirect
+	golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd
 )

go.sum

@@ -30,8 +30,8 @@ github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEW
 github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4=
 github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/gopacket v1.1.17 h1:rMrlX2ZY2UbvT+sdz3+6J+pp2z+msCq9MxTU6ymxbBY=
-github.com/google/gopacket v1.1.17/go.mod h1:UdDNZ1OO62aGYVnPhxT1U6aI7ukYtA/kB8vaU0diBUM=
+github.com/google/gopacket v1.1.18 h1:lum7VRA9kdlvBi7/v2p7/zcbkduHaCH/SVVyurs7OpY=
+github.com/google/gopacket v1.1.18/go.mod h1:UdDNZ1OO62aGYVnPhxT1U6aI7ukYtA/kB8vaU0diBUM=
 github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8=
 github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
 github.com/hashicorp/go-uuid v1.0.2 h1:cfejS+Tpcp13yd5nYHWDI6qVCny6wyX2Mt5SGur2IGE=

gor.go

@@ -4,8 +4,6 @@ package main
 
 import (
 	"flag"
-	"fmt"
-	"io"
 	"log"
 	"net/http"
 	"net/http/httputil"
@@ -20,7 +18,6 @@ import (
 )
 
 var (
-	mode       string
 	cpuprofile = flag.String("cpuprofile", "", "write cpu profile to file")
 	memprofile = flag.String("memprofile", "", "write memory profile to this file")
 )
@@ -33,18 +30,7 @@ func loggingMiddleware(next http.Handler) http.Handler {
 	})
 }
 
-var closeCh chan int
-
 func main() {
-	closeCh = make(chan int)
-	// // Don't exit on panic
-	// defer func() {
-	// 	if r := recover(); r != nil {
-	// 		fmt.Printf("PANIC: pkg: %v %s \n", r, debug.Stack())
-	// 	}
-	// }()
-
-	// If not set via env cariable
 	if len(os.Getenv("GOMAXPROCS")) == 0 {
 		runtime.GOMAXPROCS(runtime.NumCPU() * 2)
 	}
@@ -57,16 +43,16 @@ func main() {
 		}
 		dir, _ := os.Getwd()
 
-		log.Println("Started example file server for current directory on address ", args[1])
+		Debug(0, "Started example file server for current directory on address ", args[1])
 
 		log.Fatal(http.ListenAndServe(args[1], loggingMiddleware(http.FileServer(http.Dir(dir)))))
 	} else {
 		flag.Parse()
 		checkSettings()
-		plugins = InitPlugins()
+		plugins = NewPlugins()
 	}
 
-	fmt.Println("Version:", VERSION)
+	log.Printf("[PPID %d and PID %d] Version:%s\n", os.Getppid(), os.Getpid(), VERSION)
 
 	if len(plugins.Inputs) == 0 || len(plugins.Outputs) == 0 {
 		log.Fatal("Required at least 1 input and 1 output")
@@ -86,35 +72,29 @@ func main() {
 		}()
 	}
 
-	emitter := NewEmitter(closeCh)
-	c := make(chan os.Signal, 1)
-	signal.Notify(c, os.Interrupt, syscall.SIGTERM)
-	go func() {
-		<-c
-		finalize(plugins)
-		os.Exit(1)
-	}()
-
+	closeCh := make(chan int)
+	emitter := NewEmitter()
+	go emitter.Start(plugins, Settings.Middleware)
 	if Settings.ExitAfter > 0 {
-		log.Println("Running gor for a duration of", Settings.ExitAfter)
+		log.Printf("Running gor for a duration of %s\n", Settings.ExitAfter)
 
 		time.AfterFunc(Settings.ExitAfter, func() {
-			log.Println("Stopping gor after", Settings.ExitAfter)
+			log.Printf("gor run timeout %s\n", Settings.ExitAfter)
 			close(closeCh)
 		})
 	}
-
-	emitter.Start(plugins, Settings.Middleware)
-}
-
-func finalize(plugins *InOutPlugins) {
-	for _, p := range plugins.All {
-		if cp, ok := p.(io.Closer); ok {
-			cp.Close()
-		}
+	c := make(chan os.Signal, 1)
+	signal.Notify(c, os.Interrupt, syscall.SIGTERM)
+	exit := 0
+	select {
+	case <-c:
+		exit = 1
+	case <-closeCh:
+		exit = 0
 	}
+	emitter.Close()
+	os.Exit(exit)
 
-	time.Sleep(100 * time.Millisecond)
 }
 
 func profileCPU(cpuprofile string) {
@@ -128,7 +108,6 @@ func profileCPU(cpuprofile string) {
 		time.AfterFunc(30*time.Second, func() {
 			pprof.StopCPUProfile()
 			f.Close()
-			log.Println("Stop profiling after 30 seconds")
 		})
 	}
 }

gor_stat.go

@@ -1,65 +0,0 @@
-package main
-
-import (
-	"log"
-	"runtime"
-	"strconv"
-	"time"
-)
-
-type GorStat struct {
-	statName string
-	rateMs   int
-	latest   int
-	mean     int
-	max      int
-	count    int
-}
-
-func NewGorStat(statName string, rateMs int) (s *GorStat) {
-	s = new(GorStat)
-	s.statName = statName
-	s.rateMs = rateMs
-	s.latest = 0
-	s.mean = 0
-	s.max = 0
-	s.count = 0
-
-	if Settings.Stats {
-		log.Println(s.statName + ":latest,mean,max,count,count/second,gcount")
-		go s.reportStats()
-	}
-	return
-}
-
-func (s *GorStat) Write(latest int) {
-	if Settings.Stats {
-		if latest > s.max {
-			s.max = latest
-		}
-		if latest != 0 {
-			s.mean = ((s.mean * s.count) + latest) / (s.count + 1)
-		}
-		s.latest = latest
-		s.count = s.count + 1
-	}
-}
-
-func (s *GorStat) Reset() {
-	s.latest = 0
-	s.max = 0
-	s.mean = 0
-	s.count = 0
-}
-
-func (s *GorStat) String() string {
-	return s.statName + ":" + strconv.Itoa(s.latest) + "," + strconv.Itoa(s.mean) + "," + strconv.Itoa(s.max) + "," + strconv.Itoa(s.count) + "," + strconv.Itoa(s.count/(s.rateMs/1000.0)) + "," + strconv.Itoa(runtime.NumGoroutine())
-}
-
-func (s *GorStat) reportStats() {
-	for {
-		log.Println(s)
-		s.Reset()
-		time.Sleep(time.Duration(s.rateMs) * time.Millisecond)
-	}
-}

http_client.go

@@ -1,523 +0,0 @@
-package main
-
-import (
-	"bufio"
-	"bytes"
-	"crypto/tls"
-	"encoding/base64"
-	"io"
-	"log"
-	"net"
-	"net/http"
-	"net/http/httputil"
-	"net/url"
-	"runtime/debug"
-	"strconv"
-	"strings"
-	"sync"
-	"syscall"
-	"time"
-
-	"github.com/buger/goreplay/proto"
-)
-
-var httpMu sync.Mutex
-
-const (
-	readChunkSize   = 64 * 1024
-	maxResponseSize = 1073741824
-)
-
-var chunkedSuffix = []byte("0\r\n\r\n")
-
-var defaultPorts = map[string]string{
-	"http":  "80",
-	"https": "443",
-}
-
-type HTTPClientConfig struct {
-	FollowRedirects    int
-	Debug              bool
-	OriginalHost       bool
-	ConnectionTimeout  time.Duration
-	Timeout            time.Duration
-	ResponseBufferSize int
-	CompatibilityMode  bool
-}
-
-type HTTPClient struct {
-	baseURL        string
-	scheme         string
-	host           string
-	auth           string
-	conn           net.Conn
-	proxy          *url.URL
-	proxyAuth      string
-	respBuf        []byte
-	config         *HTTPClientConfig
-	goClient       *http.Client
-	redirectsCount int
-}
-
-func NewHTTPClient(baseURL string, config *HTTPClientConfig) *HTTPClient {
-	if !strings.HasPrefix(baseURL, "http") {
-		baseURL = "http://" + baseURL
-	}
-
-	u, _ := url.Parse(baseURL)
-
-	if config.Timeout == 0 {
-		config.Timeout = time.Second
-	}
-
-	config.ConnectionTimeout = config.Timeout
-
-	if config.ResponseBufferSize == 0 {
-		config.ResponseBufferSize = 100 * 1024 // 100kb
-	}
-
-	client := new(HTTPClient)
-	client.baseURL = u.String()
-	client.host = u.Host
-	client.scheme = u.Scheme
-	client.respBuf = make([]byte, config.ResponseBufferSize)
-	client.config = config
-
-	if config.CompatibilityMode {
-		client.goClient = &http.Client{
-			// #TODO
-			// CheckRedirect: redirectPolicyFunc,
-		}
-	}
-
-	if u.User != nil {
-		client.auth = "Basic " + base64.StdEncoding.EncodeToString([]byte(u.User.String()))
-	}
-
-	client.proxy, _ = http.ProxyFromEnvironment(&http.Request{URL: u})
-
-	if client.isProxy() && client.proxy.User != nil {
-		client.proxyAuth = "Basic " + base64.StdEncoding.EncodeToString([]byte(client.proxy.User.String()))
-	}
-
-	return client
-}
-
-func (c *HTTPClient) Connect() (err error) {
-	c.Disconnect()
-
-	var toDial string
-	if !strings.Contains(c.host, ":") {
-		toDial = c.host + ":" + defaultPorts[c.scheme]
-	} else {
-		toDial = c.host
-	}
-
-	if c.isProxy() {
-		if c.proxy.Scheme != "http" {
-			panic("Unsupported HTTP Proxy method")
-		}
-		Debug("[HTTPClient] Connecting to proxy", c.proxy.String(), "<>", toDial)
-		c.conn, err = net.DialTimeout("tcp", c.proxy.Host, c.config.ConnectionTimeout)
-		if err != nil {
-			return
-		}
-		if c.scheme == "https" {
-			c.conn.Write([]byte("CONNECT " + toDial + " HTTP/1.1\r\n"))
-			if c.proxyAuth != "" {
-				c.conn.Write([]byte("Proxy-Authorization: " + c.proxyAuth + "\r\n"))
-			}
-			c.conn.Write([]byte("\r\n"))
-			br := bufio.NewReader(c.conn)
-			l, _, err := br.ReadLine()
-			if err != nil {
-				return err
-			}
-			if len(l) < 12 {
-				panic("HTTP proxy did not respond correctly")
-			}
-			status := l[9:12]
-			if !bytes.Equal(status, []byte("200")) {
-				panic("HTTP proxy did not respond correctly")
-			}
-			for {
-				// Read until we find the empty line
-				l, _, err := br.ReadLine()
-				if err != nil {
-					return err
-				}
-				if len(l) == 0 {
-					break
-				}
-			}
-		}
-		Debug("[HTTPClient] Proxy successfully connected")
-	} else {
-		c.conn, err = net.DialTimeout("tcp", toDial, c.config.ConnectionTimeout)
-		if err != nil {
-			return
-		}
-	}
-
-	if c.scheme == "https" {
-		// Wrap our socket in TLS
-		Debug("[HTTPClient] Wrapping socket in TLS", c.host)
-		tlsConn := tls.Client(c.conn, &tls.Config{InsecureSkipVerify: true, ServerName: c.host})
-
-		if err = tlsConn.Handshake(); err != nil {
-			return
-		}
-
-		c.conn = tlsConn
-		Debug("[HTTPClient] Successfully wrapped in TLS")
-	}
-
-	return
-}
-
-func (c *HTTPClient) Disconnect() {
-	if c.conn != nil {
-		c.conn.Close()
-		c.conn = nil
-		Debug("[HTTP] Disconnected: ", c.baseURL)
-	}
-}
-
-func (c *HTTPClient) isAlive(readBytes *int) bool {
-	// Ready 1 byte from socket without timeout to check if it not closed
-	c.conn.SetReadDeadline(time.Now().Add(time.Millisecond))
-	n, err := c.conn.Read(c.respBuf[:1])
-
-	if err == io.EOF {
-		Debug("[HTTPClient] connection closed, reconnecting")
-		return false
-	}
-
-	if err == syscall.EPIPE {
-		Debug("Detected broken pipe.", err)
-		return false
-	}
-	if n != 0 {
-		*readBytes += n
-		Debug("[HTTPClient] isAlive readBytes ", *readBytes)
-	}
-	return true
-}
-
-func (c *HTTPClient) SendGoClient(data []byte) ([]byte, error) {
-	var req *http.Request
-	var resp *http.Response
-	var err error
-
-	req, err = http.ReadRequest(bufio.NewReader(bytes.NewBuffer(data)))
-	if err != nil {
-		return nil, err
-	}
-
-	if !c.config.OriginalHost {
-		req.Host = c.host
-	}
-
-	if c.auth != "" {
-		req.Header.Add("Authorization", c.auth)
-	}
-
-	req.URL, _ = url.ParseRequestURI(c.scheme + "://" + c.host + req.RequestURI)
-	req.RequestURI = ""
-
-	resp, err = c.goClient.Do(req)
-	if err != nil {
-		return nil, err
-	}
-
-	return httputil.DumpResponse(resp, true)
-}
-
-func (c *HTTPClient) Send(data []byte) (response []byte, err error) {
-	// Don't exit on panic
-	defer func() {
-		if r := recover(); r != nil {
-			Debug("[HTTPClient]", r, string(data))
-
-			if _, ok := r.(error); ok {
-				log.Println("[HTTPClient] Failed to send request: ", string(data))
-				log.Println("[HTTPClient] Response: ", string(response))
-				log.Println("PANIC: pkg:", r, string(debug.Stack()))
-			}
-		}
-	}()
-
-	if c.config.CompatibilityMode {
-		return c.SendGoClient(data)
-	}
-
-	var readBytes int
-	if c.conn == nil || !c.isAlive(&readBytes) {
-		Debug("[HTTPClient] Connecting:", c.baseURL)
-		if err = c.Connect(); err != nil {
-			log.Println("[HTTPClient] Connection error:", err)
-			response = errorPayload(HTTP_CONNECTION_ERROR)
-			return
-		}
-	}
-
-	timeout := time.Now().Add(c.config.Timeout)
-
-	c.conn.SetWriteDeadline(timeout)
-
-	if !c.config.OriginalHost {
-		data = proto.SetHost(data, []byte(c.baseURL), []byte(c.host))
-	}
-
-	if c.isProxy() && c.scheme == "http" {
-		path := proto.Path(data)
-		if len(path) > 0 && path[0] == '/' {
-			data = proto.SetPath(data, c.proxyPath(path))
-			if c.proxyAuth != "" {
-				data = proto.SetHeader(data, []byte("Proxy-Authorization"), []byte(c.proxyAuth))
-			}
-		}
-	}
-
-	if c.auth != "" {
-		data = proto.SetHeader(data, []byte("Authorization"), []byte(c.auth))
-	}
-
-	if c.config.Debug {
-		Debug("[HTTPClient] Sending:", string(data))
-	}
-
-	return c.send(data, readBytes, timeout)
-}
-
-func (c *HTTPClient) send(data []byte, readBytes int, timeout time.Time) (response []byte, err error) {
-	var payload []byte
-	var n int
-	if _, err = c.conn.Write(data); err != nil {
-		Debug("[HTTPClient] Write error:", err, c.baseURL)
-		response = errorPayload(HTTP_TIMEOUT)
-		c.Disconnect()
-		return
-	}
-
-	var currentChunk []byte
-	timeout = time.Now().Add(c.config.Timeout)
-	chunked := false
-	contentLength := -1
-	currentContentLength := 0
-	chunks := 0
-
-	for {
-		c.conn.SetReadDeadline(timeout)
-
-		if readBytes < len(c.respBuf) {
-			n, err = c.conn.Read(c.respBuf[readBytes:])
-			readBytes += n
-			chunks++
-
-			// First chunk
-			if chunked || contentLength != -1 {
-				currentContentLength += n
-			} else {
-				// If headers are finished
-				var firstEmptyLine = bytes.Index(c.respBuf[:readBytes], proto.EmptyLine)
-				if firstEmptyLine != -1 {
-					if bytes.Equal(proto.Header(c.respBuf[:readBytes], []byte("Transfer-Encoding")), []byte("chunked")) {
-						chunked = true
-					} else {
-						status, _ := strconv.Atoi(string(proto.Status(c.respBuf[:readBytes])))
-						// We want to soak up all 100 Continues received to get the real result code
-						if status >= 100 && status < 200 {
-							timeout = time.Now().Add(c.config.Timeout)
-							var deleteLen = firstEmptyLine + len(proto.EmptyLine)
-							copy(c.respBuf, c.respBuf[deleteLen:readBytes])
-							readBytes -= deleteLen
-							chunks--
-							continue
-						} else if status == 204 || status == 304 {
-							contentLength = 0
-							break
-						} else {
-							l := proto.Header(c.respBuf[:readBytes], []byte("Content-Length"))
-							if len(l) > 0 {
-								contentLength, _ = strconv.Atoi(string(l))
-							}
-						}
-					}
-
-					currentContentLength += len(proto.Body(c.respBuf[:readBytes]))
-				}
-			}
-
-			if chunked {
-				// Check if chunked message finished
-				if bytes.HasSuffix(c.respBuf[:readBytes], chunkedSuffix) {
-					break
-				}
-			} else if contentLength != -1 {
-				if currentContentLength > contentLength {
-					Debug("[HTTPClient] disconnected, wrong length", currentContentLength, contentLength)
-					c.Disconnect()
-					break
-				} else if currentContentLength == contentLength {
-					break
-				}
-			}
-
-			if err != nil {
-				if err == io.EOF {
-					err = nil
-				}
-				break
-			}
-		} else {
-			if currentChunk == nil {
-				currentChunk = make([]byte, readChunkSize)
-			}
-
-			n, err = c.conn.Read(currentChunk)
-
-			readBytes += int(n)
-			chunks++
-			currentContentLength += n
-
-			if chunked {
-				// Check if chunked message finished
-				if bytes.HasSuffix(currentChunk[:n], chunkedSuffix) {
-					break
-				}
-			} else if contentLength != -1 {
-				if currentContentLength > contentLength {
-					Debug("[HTTPClient] disconnected, wrong length", currentContentLength, contentLength)
-					c.Disconnect()
-					break
-				} else if currentContentLength == contentLength {
-					break
-				}
-			} else {
-				Debug("[HTTPClient] disconnected, can't find Content-Length or Chunked")
-				c.Disconnect()
-				break
-			}
-
-			if err == io.EOF {
-				break
-			} else if err != nil {
-				Debug("[HTTPClient] Read the whole body error:", err, c.baseURL)
-				break
-			}
-
-		}
-
-		if readBytes >= maxResponseSize {
-			Debug("[HTTPClient] Body is more than the max size", maxResponseSize,
-				c.baseURL)
-			break
-		}
-
-		// For following chunks expect less timeout
-		timeout = time.Now().Add(c.config.Timeout / 5)
-	}
-
-	if err != nil && readBytes == 0 {
-		Debug("[HTTPClient] Response read timeout error", err, c.conn, readBytes, string(c.respBuf[:readBytes]))
-		response = errorPayload(HTTP_TIMEOUT)
-		c.Disconnect()
-		return
-	}
-
-	if readBytes < 4 || string(c.respBuf[:4]) != "HTTP" {
-		maxRead := 100
-		if readBytes < maxRead {
-			maxRead = readBytes
-		}
-		Debug("[HTTPClient] Response read unknown error", err, c.conn, readBytes, string(c.respBuf[:maxRead]))
-		response = errorPayload(HTTP_UNKNOWN_ERROR)
-		c.Disconnect()
-		return
-	}
-
-	if readBytes > len(c.respBuf) {
-		readBytes = len(c.respBuf)
-	}
-	payload = make([]byte, readBytes)
-	copy(payload, c.respBuf[:readBytes])
-
-	if c.config.Debug {
-		Debug("[HTTPClient] Received:", string(payload))
-	}
-
-	if c.config.FollowRedirects > 0 && c.redirectsCount < c.config.FollowRedirects {
-		status := payload[9:12]
-
-		// 3xx requests
-		if status[0] == '3' {
-			c.redirectsCount++
-
-			location := proto.Header(payload, []byte("Location"))
-			redirectPayload := proto.SetPath(data, location)
-
-			if c.config.Debug {
-				Debug("[HTTPClient] Redirecting to: " + string(location))
-			}
-
-			return c.Send(redirectPayload)
-		}
-	}
-
-	if bytes.Equal(proto.Status(payload), []byte("400")) {
-		Debug("[HTTPClient] Closed connection on 400 response")
-		c.Disconnect()
-	}
-
-	c.redirectsCount = 0
-
-	return payload, err
-}
-
-func (c *HTTPClient) Get(path string) (response []byte, err error) {
-	payload := "GET " + path + " HTTP/1.1\r\n\r\n"
-
-	return c.Send([]byte(payload))
-}
-
-func (c *HTTPClient) Post(path string, body []byte) (response []byte, err error) {
-	payload := "POST " + path + " HTTP/1.1\r\n"
-	payload += "Content-Length: " + strconv.Itoa(len(body)) + "\r\n\r\n"
-	payload += string(body)
-
-	return c.Send([]byte(payload))
-}
-
-func (c *HTTPClient) proxyPath(path []byte) []byte {
-	return append([]byte(c.scheme+"://"+c.host), path...)
-}
-
-func (c *HTTPClient) isProxy() bool {
-	return c.proxy != nil
-}
-
-const (
-	// https://support.cloudflare.com/hc/en-us/articles/200171936-Error-520-Web-server-is-returning-an-unknown-error
-	HTTP_UNKNOWN_ERROR = "520"
-	// https://support.cloudflare.com/hc/en-us/articles/200171916-Error-521-Web-server-is-down
-	HTTP_CONNECTION_ERROR = "521"
-	// https://support.cloudflare.com/hc/en-us/articles/200171906-Error-522-Connection-timed-out
-	HTTP_CONNECTION_TIMEOUT = "522"
-	// https://support.cloudflare.com/hc/en-us/articles/200171946-Error-523-Origin-is-unreachable
-	HTTP_UNREACHABLE = "523"
-	// https://support.cloudflare.com/hc/en-us/articles/200171926-Error-524-A-timeout-occurred
-	HTTP_TIMEOUT = "524"
-)
-
-var errorPayloadTemplate = "HTTP/1.1 202 Accepted\r\nDate: Mon, 17 Aug 2015 14:10:11 GMT\r\nContent-Length: 0\r\nContent-Type: text/plain; charset=utf-8\r\n\r\n"
-
-func errorPayload(errorCode string) []byte {
-	payload := make([]byte, len(errorPayloadTemplate))
-	copy(payload, errorPayloadTemplate)
-
-	copy(payload[29:58], []byte(time.Now().Format(time.RFC1123)))
-	copy(payload[9:12], errorCode)
-
-	return payload
-}

http_client_test.go

@@ -1,533 +0,0 @@
-package main
-
-import (
-	"bytes"
-	"crypto/rand"
-	"io/ioutil"
-	_ "log"
-	"net"
-	"net/http"
-	"net/http/httptest"
-	"net/http/httputil"
-	_ "reflect"
-	"strings"
-	"sync"
-	"testing"
-	"time"
-
-	"github.com/buger/goreplay/proto"
-)
-
-func TestHTTPClientURLPort(t *testing.T) {
-	c1 := NewHTTPClient("http://example.com", &HTTPClientConfig{})
-	if c1.baseURL != "http://example.com" {
-		t.Error("Sould not add 80 port for http:", c1.baseURL)
-	}
-
-	c2 := NewHTTPClient("https://example.com", &HTTPClientConfig{})
-	if c2.baseURL != "https://example.com" {
-		t.Error("Sould not add 443 port for https:", c2.baseURL)
-	}
-
-	c3 := NewHTTPClient("https://example.com:1", &HTTPClientConfig{})
-	if c3.baseURL != "https://example.com:1" {
-		t.Error("Sould use specified port:", c3.baseURL)
-	}
-
-	c4 := NewHTTPClient("example.com", &HTTPClientConfig{})
-	if c4.baseURL != "http://example.com" {
-		t.Error("Sould not add default protocol:", c4.baseURL)
-	}
-}
-
-func TestHTTPClientSend(t *testing.T) {
-	wg := new(sync.WaitGroup)
-
-	payload := func(reqType string) []byte {
-		switch reqType {
-		case "GET":
-			return []byte("GET / HTTP/1.1\r\n\r\n")
-		case "POST":
-			return []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
-		case "POST_CHUNKED":
-			return []byte("POST / HTTP/1.1\r\nHost: www.w3.org\r\nTransfer-Encoding: chunked\r\n\r\n4\r\nWiki\r\n5\r\npedia\r\ne\r\n in\r\n\r\nchunks.\r\n0\r\n\r\n")
-		}
-
-		return []byte("")
-	}
-
-	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-
-		if r.Method == "POST" {
-			defer r.Body.Close()
-			body, _ := ioutil.ReadAll(r.Body)
-
-			if len(r.TransferEncoding) > 0 && r.TransferEncoding[0] == "chunked" {
-				if string(body) != "Wikipedia in\r\n\r\nchunks." {
-					t.Error("Wrong POST body:", body, string(body))
-				}
-			} else {
-				if string(body) != "a=1&b=2" {
-					buf, _ := httputil.DumpRequest(r, true)
-					t.Error("Wrong POST body:", string(body), string(buf))
-				}
-			}
-		}
-
-		wg.Done()
-	}))
-	defer server.Close()
-
-	client := NewHTTPClient(server.URL, &HTTPClientConfig{Debug: true})
-
-	wg.Add(4)
-	client.Send(payload("POST"))
-	client.Send(payload("GET"))
-	client.Send(payload("POST_CHUNKED"))
-	client.Send(payload("POST"))
-
-	wg.Wait()
-
-	client = NewHTTPClient(server.URL, &HTTPClientConfig{Debug: true, CompatibilityMode: true})
-
-	wg.Add(4)
-
-	if _, err := client.Send(payload("POST")); err != nil {
-		t.Fatal(err.Error())
-	}
-	if _, err := client.Send(payload("GET")); err != nil {
-		t.Fatal(err.Error())
-	}
-	if _, err := client.Send(payload("POST_CHUNKED")); err != nil {
-		t.Fatal(err.Error())
-	}
-	if _, err := client.Send(payload("POST")); err != nil {
-		t.Fatal(err.Error())
-	}
-
-	wg.Wait()
-}
-
-func TestHTTPClientResonseByClose(t *testing.T) {
-	wg := new(sync.WaitGroup)
-
-	payload := []byte("GET / HTTP/1.1\r\n\r\n")
-	ln, _ := net.Listen("tcp", ":0")
-	go func() {
-		for {
-			conn, _ := ln.Accept()
-			buf := make([]byte, 4096)
-			conn.Read(buf)
-
-			conn.Write([]byte("HTTP/1.1 200 OK\r\n\r\n"))
-			conn.Write([]byte("ab"))
-			conn.Close()
-
-			wg.Done()
-		}
-	}()
-
-	client := NewHTTPClient(ln.Addr().String(), &HTTPClientConfig{Debug: true})
-
-	wg.Add(1)
-	resp, _ := client.Send(payload)
-
-	if !bytes.Equal(resp, []byte("HTTP/1.1 200 OK\r\n\r\nab")) {
-		t.Error("Should return valid response", string(resp))
-	}
-
-	wg.Wait()
-}
-
-// https://github.com/buger/gor/issues/184
-func TestHTTPClientResponseBuffer(t *testing.T) {
-	testCases := []struct {
-		name         string
-		responseSize int
-		buffserSize  int
-		expectedSize int
-		timeout      time.Duration
-	}{
-		{"Chunked, buffer overflow", 10 * 1024, 1024, 1024, 50 * time.Millisecond},
-
-		{"Chunked, fits buffer", 10 * 1024, 64 * 1024, 10*1024 + 145 /* headers length + chunked meta */, 50 * time.Millisecond},
-
-		{"Content-Length, buffer overflow", 1024, 1000, 1000, 50 * time.Millisecond},
-
-		{"Content-Length, fits buffer", 1024, 64 * 1024, 1024 + 118, 50 * time.Millisecond},
-	}
-
-	for _, tc := range testCases {
-		wg := new(sync.WaitGroup)
-
-		payload := []byte("GET / HTTP/1.1\r\n\r\n")
-
-		server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-
-			size := tc.responseSize // 1kb
-			rb := make([]byte, size)
-			rand.Read(rb)
-
-			w.Write(rb[:size/2])
-			w.Write(rb[size/2:])
-
-			wg.Done()
-		}))
-
-		client := NewHTTPClient(server.URL, &HTTPClientConfig{Debug: true, ResponseBufferSize: tc.buffserSize, Timeout: 100 * time.Millisecond})
-
-		wg.Add(2)
-
-		start := time.Now()
-		client.Send(payload)
-		resp, err := client.Send(payload)
-		stop := time.Now()
-
-		if err != nil {
-			t.Error("Request error", err)
-		}
-
-		if stop.Sub(start) > tc.timeout {
-			t.Error("Request took too long", stop.Sub(start), tc.timeout)
-		}
-
-		if len(resp) != tc.expectedSize {
-			t.Error(tc.name, " - Wrong response size:", tc.expectedSize, len(resp))
-		} else {
-			if !bytes.Equal(resp[0:8], []byte("HTTP/1.1")) {
-				t.Error(tc.name, " - Response buffer contains data from previous request", string(resp), len(resp))
-			}
-		}
-
-		wg.Wait()
-		server.Close()
-	}
-}
-
-func TestHTTPClientHTTPSSend(t *testing.T) {
-	wg := new(sync.WaitGroup)
-
-	payload := func(reqType string) []byte {
-		switch reqType {
-		case "GET":
-			return []byte("GET / HTTP/1.1\r\n\r\n")
-		case "POST":
-			return []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
-		case "POST_CHUNKED":
-			return []byte("POST / HTTP/1.1\r\nHost: www.w3.org\r\nTransfer-Encoding: chunked\r\n\r\n4\r\nWiki\r\n5\r\npedia\r\ne\r\n in\r\n\r\nchunks.\r\n0\r\n\r\n")
-		}
-
-		return []byte("")
-	}
-
-	server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-
-		if r.Method == "POST" {
-			defer r.Body.Close()
-			body, _ := ioutil.ReadAll(r.Body)
-
-			if len(r.TransferEncoding) > 0 && r.TransferEncoding[0] == "chunked" {
-				if string(body) != "Wikipedia in\r\n\r\nchunks." {
-					t.Error("Wrong POST body:", body, string(body))
-				}
-			} else {
-				if string(body) != "a=1&b=2" {
-					buf, _ := httputil.DumpRequest(r, true)
-					t.Error("Wrong POST body:", string(body), string(buf))
-				}
-			}
-		}
-
-		wg.Done()
-	}))
-	defer server.Close()
-
-	client := NewHTTPClient(server.URL, &HTTPClientConfig{})
-
-	wg.Add(4)
-	client.Send(payload("POST"))
-	client.Send(payload("GET"))
-	client.Send(payload("POST_CHUNKED"))
-	client.Send(payload("POST"))
-
-	wg.Wait()
-}
-
-func TestHTTPClientServerInstantDisconnect(t *testing.T) {
-	wg := new(sync.WaitGroup)
-
-	GETPayload := []byte("GET / HTTP/1.1\r\n\r\n")
-
-	ln, _ := net.Listen("tcp", "127.0.0.1:0")
-	defer ln.Close()
-
-	go func() {
-		for {
-			conn, err := ln.Accept()
-
-			if err != nil {
-				break
-			}
-			conn.Close()
-			wg.Done()
-		}
-	}()
-
-	client := NewHTTPClient(ln.Addr().String(), &HTTPClientConfig{})
-
-	wg.Add(2)
-	client.Send(GETPayload)
-	client.Send(GETPayload)
-
-	wg.Wait()
-}
-
-func TestHTTPClientServerNoKeepAlive(t *testing.T) {
-	wg := new(sync.WaitGroup)
-
-	GETPayload := []byte("GET / HTTP/1.1\r\n\r\n")
-
-	ln, _ := net.Listen("tcp", "127.0.0.1:0")
-	defer ln.Close()
-
-	go func() {
-		for {
-			conn, err := ln.Accept()
-			if err != nil {
-				break
-			}
-
-			buf := make([]byte, 4096)
-			reqLen, err := conn.Read(buf)
-			if err != nil {
-				t.Error("Error reading:", err.Error())
-			}
-			Debug("Received: ", string(buf[0:reqLen]))
-			conn.Write([]byte("OK"))
-
-			// No keep-alive connections
-			conn.Close()
-
-			wg.Done()
-		}
-	}()
-
-	client := NewHTTPClient(ln.Addr().String(), &HTTPClientConfig{})
-
-	wg.Add(2)
-	client.Send(GETPayload)
-	client.Send(GETPayload)
-
-	wg.Wait()
-}
-
-func TestHTTPClientRedirect(t *testing.T) {
-	wg := new(sync.WaitGroup)
-
-	GETPayload := []byte("GET / HTTP/1.1\r\n\r\n")
-
-	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-
-		if r.URL.Path == "/" {
-			http.Redirect(w, r, "/new", 301)
-		}
-
-		wg.Done()
-	}))
-	defer server.Close()
-
-	client := NewHTTPClient(server.URL, &HTTPClientConfig{FollowRedirects: 1, Debug: false})
-
-	// Should do 2 queries
-	wg.Add(2)
-	client.Send(GETPayload)
-
-	wg.Wait()
-}
-
-func TestHTTPClientRedirectLimit(t *testing.T) {
-	wg := new(sync.WaitGroup)
-
-	GETPayload := []byte("GET / HTTP/1.1\r\n\r\n")
-
-	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-
-		if r.URL.Path == "/" {
-			http.Redirect(w, r, "/r1", 301)
-		}
-
-		if r.URL.Path == "/r1" {
-			http.Redirect(w, r, "/r2", 301)
-		}
-
-		if r.URL.Path == "/r2" {
-			http.Redirect(w, r, "/new", 301)
-		}
-
-		wg.Done()
-	}))
-	defer server.Close()
-
-	client := NewHTTPClient(server.URL, &HTTPClientConfig{FollowRedirects: 2, Debug: false})
-
-	// Have 3 redirects + 1 GET, but should do only 2 redirects + GET
-	wg.Add(3)
-	client.Send(GETPayload)
-
-	wg.Wait()
-}
-
-func TestHTTPClientKeepHeadersRedirect(t *testing.T) {
-	wg := new(sync.WaitGroup)
-
-	GETPayload := []byte("GET / HTTP/1.1\r\n\r\n")
-
-	GETPayload = proto.AddHeader(GETPayload, []byte("keep-header"), []byte("true"))
-
-	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-
-		if r.URL.Path == "/" {
-			http.Redirect(w, r, "/new", 301)
-		}
-
-		if r.Header.Get("keep-header") != "true" {
-			t.Errorf("Header keep-header was incorrect, got: %s, want: %s.", r.Header.Get("keep-header"), "true")
-		}
-
-		wg.Done()
-	}))
-	defer server.Close()
-
-	client := NewHTTPClient(server.URL, &HTTPClientConfig{FollowRedirects: 1, Debug: false})
-
-	wg.Add(2)
-	client.Send(GETPayload)
-
-	wg.Wait()
-}
-
-func TestHTTPClientBasicAuth(t *testing.T) {
-	wg := new(sync.WaitGroup)
-	wg.Add(2)
-
-	GETPayload := []byte("GET / HTTP/1.1\r\n\r\n")
-
-	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		user, pass, _ := r.BasicAuth()
-
-		if user != "user" || pass != "pass" {
-			http.Error(w, "Unauthorized.", 401)
-			wg.Done()
-			return
-		}
-
-		wg.Done()
-	}))
-	defer server.Close()
-
-	client := NewHTTPClient(server.URL, &HTTPClientConfig{Debug: false})
-	resp, _ := client.Send(GETPayload)
-	client.Disconnect()
-
-	if !bytes.Equal(proto.Status(resp), []byte("401")) {
-		t.Error("Should return unauthorized error", string(resp))
-	}
-
-	authUrl := strings.Replace(server.URL, "http://", "http://user:pass@", -1)
-	client = NewHTTPClient(authUrl, &HTTPClientConfig{Debug: false})
-	resp, _ = client.Send(GETPayload)
-	client.Disconnect()
-
-	if !bytes.Equal(proto.Status(resp), []byte("200")) {
-		t.Error("Should return proper response", string(resp))
-	}
-
-	wg.Wait()
-}
-
-func TestHTTPClientHandleHTTP10(t *testing.T) {
-	wg := new(sync.WaitGroup)
-
-	GETPayload := []byte("GET http://foobar.com/path HTTP/1.0\r\n\r\n")
-
-	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-
-		if r.URL.Path != "/path" {
-			t.Error("Path not match:", r.URL.Path)
-		}
-
-		wg.Done()
-	}))
-	defer server.Close()
-
-	client := NewHTTPClient(server.URL, &HTTPClientConfig{Debug: true})
-
-	wg.Add(1)
-	client.Send(GETPayload)
-
-	wg.Wait()
-}
-
-func TestHTTPClientErrors(t *testing.T) {
-	req := []byte("GET http://foobar.com/path HTTP/1.0\r\n\r\n")
-
-	// Port not exists
-	client := NewHTTPClient("http://127.0.0.1:1", &HTTPClientConfig{Debug: true})
-	if resp, err := client.Send(req); err != nil {
-		if s := proto.Status(resp); !bytes.Equal(s, []byte("521")) {
-			t.Error("Should return status 521 for connection refused, instead:", string(s))
-		}
-	} else {
-		t.Error("Should throw error")
-	}
-
-	client = NewHTTPClient("http://not.existing", &HTTPClientConfig{Debug: true})
-	if resp, err := client.Send(req); err != nil {
-		if s := proto.Status(resp); !bytes.Equal(s, []byte("521")) {
-			t.Error("Should return status 521 for no such host, instead:", string(s))
-		}
-	} else {
-		t.Error("Should throw error")
-	}
-
-	// Non routable IP address to simulate connection timeout
-	client = NewHTTPClient("http://10.255.255.1", &HTTPClientConfig{Debug: true, ConnectionTimeout: 100 * time.Millisecond})
-
-	if resp, err := client.Send(req); err != nil {
-		if s := proto.Status(resp); !bytes.Equal(s, []byte("521")) {
-			t.Error("Should return status 521 for io/timeout:", string(s))
-		}
-	} else {
-		t.Error("Should throw error")
-	}
-
-	// Connecting but io timeout on read
-	ln, _ := net.Listen("tcp", "127.0.0.1:0")
-	client = NewHTTPClient("http://"+ln.Addr().String(), &HTTPClientConfig{Debug: true, Timeout: 10 * time.Millisecond})
-	defer ln.Close()
-
-	if resp, err := client.Send(req); err != nil {
-		if s := proto.Status(resp); !bytes.Equal(s, []byte("524")) {
-			t.Error("Should return status 524 for io read, instead:", string(s))
-		}
-	} else {
-		t.Error("Should throw error")
-	}
-
-	// Response read error read tcp [::1]:51128: connection reset by peer &{{0xc20802a000}}
-	ln1, _ := net.Listen("tcp", "127.0.0.1:0")
-	go func() {
-		ln1.Accept()
-	}()
-	defer ln1.Close()
-
-	client = NewHTTPClient("http://"+ln1.Addr().String(), &HTTPClientConfig{Debug: true, Timeout: 10 * time.Millisecond})
-
-	if resp, err := client.Send(req); err != nil {
-		if s := proto.Status(resp); !bytes.Equal(s, []byte("524")) {
-			t.Error("Should return status 524 for connection reset by peer, instead:", string(s))
-		}
-	} else {
-		t.Error("Should throw error")
-	}
-}

http_modifier.go

@@ -15,9 +15,9 @@ type HTTPModifier struct {
 
 func NewHTTPModifier(config *HTTPModifierConfig) *HTTPModifier {
 	// Optimization to skip modifier completely if we do not need it
-	if len(config.UrlRegexp) == 0 &&
-		len(config.UrlNegativeRegexp) == 0 &&
-		len(config.UrlRewrite) == 0 &&
+	if len(config.URLRegexp) == 0 &&
+		len(config.URLNegativeRegexp) == 0 &&
+		len(config.URLRewrite) == 0 &&
 		len(config.HeaderRewrite) == 0 &&
 		len(config.HeaderFilters) == 0 &&
 		len(config.HeaderNegativeFilters) == 0 &&
@@ -34,7 +34,7 @@ func NewHTTPModifier(config *HTTPModifierConfig) *HTTPModifier {
 }
 
 func (m *HTTPModifier) Rewrite(payload []byte) (response []byte) {
-	if !proto.IsHTTPPayload(payload) {
+	if !proto.HasRequestTitle(payload) {
 		return payload
 	}
 
@@ -67,12 +67,12 @@ func (m *HTTPModifier) Rewrite(payload []byte) (response []byte) {
 		}
 	}
 
-	if len(m.config.UrlRegexp) > 0 {
+	if len(m.config.URLRegexp) > 0 {
 		path := proto.Path(payload)
 
 		matched := false
 
-		for _, f := range m.config.UrlRegexp {
+		for _, f := range m.config.URLRegexp {
 			if f.regexp.Match(path) {
 				matched = true
 				break
@@ -84,10 +84,10 @@ func (m *HTTPModifier) Rewrite(payload []byte) (response []byte) {
 		}
 	}
 
-	if len(m.config.UrlNegativeRegexp) > 0 {
+	if len(m.config.URLNegativeRegexp) > 0 {
 		path := proto.Path(payload)
 
-		for _, f := range m.config.UrlNegativeRegexp {
+		for _, f := range m.config.URLNegativeRegexp {
 			if f.regexp.Match(path) {
 				return
 			}
@@ -165,10 +165,10 @@ func (m *HTTPModifier) Rewrite(payload []byte) (response []byte) {
 		}
 	}
 
-	if len(m.config.UrlRewrite) > 0 {
+	if len(m.config.URLRewrite) > 0 {
 		path := proto.Path(payload)
 
-		for _, f := range m.config.UrlRewrite {
+		for _, f := range m.config.URLRewrite {
 			if f.src.Match(path) {
 				path = f.src.ReplaceAll(path, f.target)
 				payload = proto.SetPath(payload, path)

http_modifier_settings.go

@@ -10,19 +10,18 @@ import (
 
 // HTTPModifierConfig holds configuration options for built-in traffic modifier
 type HTTPModifierConfig struct {
-	UrlNegativeRegexp      HTTPUrlRegexp              `json:"http-disallow-url"`
-	UrlRegexp              HTTPUrlRegexp              `json:"http-allow-url"`
-	UrlRewrite             UrlRewriteMap              `json:"http-rewrite-url"`
+	URLNegativeRegexp      HTTPURLRegexp              `json:"http-disallow-url"`
+	URLRegexp              HTTPURLRegexp              `json:"http-allow-url"`
+	URLRewrite             URLRewriteMap              `json:"http-rewrite-url"`
 	HeaderRewrite          HeaderRewriteMap           `json:"http-rewrite-header"`
 	HeaderFilters          HTTPHeaderFilters          `json:"http-allow-header"`
 	HeaderNegativeFilters  HTTPHeaderFilters          `json:"http-disallow-header"`
 	HeaderBasicAuthFilters HTTPHeaderBasicAuthFilters `json:"http-basic-auth-filter"`
 	HeaderHashFilters      HTTPHashFilters            `json:"http-header-limiter"`
 	ParamHashFilters       HTTPHashFilters            `json:"http-param-limiter"`
-
-	Params  HTTPParams  `json:"http-set-param"`
-	Headers HTTPHeaders `json:"http-set-header"`
-	Methods HTTPMethods `json:"http-allow-method"`
+	Params                 HTTPParams                 `json:"http-set-param"`
+	Headers                HTTPHeaders                `json:"http-set-header"`
+	Methods                HTTPMethods                `json:"http-allow-method"`
 }
 
 //
@@ -40,10 +39,11 @@ func (h *HTTPHeaderFilters) String() string {
 	return fmt.Sprint(*h)
 }
 
+// Set method to implement flags.Value
 func (h *HTTPHeaderFilters) Set(value string) error {
 	valArr := strings.SplitN(value, ":", 2)
 	if len(valArr) < 2 {
-		return errors.New("need both header and value, colon-delimited (ex. user_id:^169$).")
+		return errors.New("need both header and value, colon-delimited (ex. user_id:^169$)")
 	}
 	val := strings.TrimSpace(valArr[1])
 	r, err := regexp.Compile(val)
@@ -63,13 +63,14 @@ type basicAuthFilter struct {
 	regexp *regexp.Regexp
 }
 
-// HTTPHeaderFilters holds list of headers and their regexps
+// HTTPHeaderBasicAuthFilters holds list of regxp to match basic Auth header values
 type HTTPHeaderBasicAuthFilters []basicAuthFilter
 
 func (h *HTTPHeaderBasicAuthFilters) String() string {
 	return fmt.Sprint(*h)
 }
 
+// Set method to implement flags.Value
 func (h *HTTPHeaderBasicAuthFilters) Set(value string) error {
 	r, err := regexp.Compile(value)
 	if err != nil {
@@ -89,12 +90,14 @@ type hashFilter struct {
 	percent uint32
 }
 
+// HTTPHashFilters represents a slice of header hash filters
 type HTTPHashFilters []hashFilter
 
 func (h *HTTPHashFilters) String() string {
 	return fmt.Sprint(*h)
 }
 
+// Set method to implement flags.Value
 func (h *HTTPHashFilters) Set(value string) error {
 	valArr := strings.SplitN(value, ":", 2)
 	if len(valArr) < 2 {
@@ -129,23 +132,26 @@ func (h *HTTPHashFilters) Set(value string) error {
 //
 // Handling of --http-set-header option
 //
-type HTTPHeaders []HTTPHeader
-type HTTPHeader struct {
+type httpHeader struct {
 	Name  string
 	Value string
 }
 
+// HTTPHeaders is a slice of headers that must appended
+type HTTPHeaders []httpHeader
+
 func (h *HTTPHeaders) String() string {
 	return fmt.Sprint(*h)
 }
 
+// Set method to implement flags.Value
 func (h *HTTPHeaders) Set(value string) error {
 	v := strings.SplitN(value, ":", 2)
 	if len(v) != 2 {
 		return errors.New("Expected `Key: Value`")
 	}
 
-	header := HTTPHeader{
+	header := httpHeader{
 		strings.TrimSpace(v[0]),
 		strings.TrimSpace(v[1]),
 	}
@@ -157,23 +163,26 @@ func (h *HTTPHeaders) Set(value string) error {
 //
 // Handling of --http-set-param option
 //
-type HTTPParams []HTTPParam
-type HTTPParam struct {
+type httpParam struct {
 	Name  []byte
 	Value []byte
 }
 
+// HTTPParams filters for --http-set-param
+type HTTPParams []httpParam
+
 func (h *HTTPParams) String() string {
 	return fmt.Sprint(*h)
 }
 
+// Set method to implement flags.Value
 func (h *HTTPParams) Set(value string) error {
 	v := strings.SplitN(value, "=", 2)
 	if len(v) != 2 {
 		return errors.New("Expected `Key=Value`")
 	}
 
-	param := HTTPParam{
+	param := httpParam{
 		[]byte(strings.TrimSpace(v[0])),
 		[]byte(strings.TrimSpace(v[1])),
 	}
@@ -185,12 +194,15 @@ func (h *HTTPParams) Set(value string) error {
 //
 // Handling of --http-allow-method option
 //
+
+// HTTPMethods holds values for method allowed
 type HTTPMethods [][]byte
 
 func (h *HTTPMethods) String() string {
 	return fmt.Sprint(*h)
 }
 
+// Set method to implement flags.Value
 func (h *HTTPMethods) Set(value string) error {
 	*h = append(*h, []byte(value))
 	return nil
@@ -204,13 +216,15 @@ type urlRewrite struct {
 	target []byte
 }
 
-type UrlRewriteMap []urlRewrite
+// URLRewriteMap holds regexp and data to modify URL
+type URLRewriteMap []urlRewrite
 
-func (r *UrlRewriteMap) String() string {
+func (r *URLRewriteMap) String() string {
 	return fmt.Sprint(*r)
 }
 
-func (r *UrlRewriteMap) Set(value string) error {
+// Set method to implement flags.Value
+func (r *URLRewriteMap) Set(value string) error {
 	valArr := strings.SplitN(value, ":", 2)
 	if len(valArr) < 2 {
 		return errors.New("need both src and target, colon-delimited (ex. /a:/b)")
@@ -232,12 +246,14 @@ type headerRewrite struct {
 	target []byte
 }
 
+// HeaderRewriteMap holds regexp and data to rewrite headers
 type HeaderRewriteMap []headerRewrite
 
 func (r *HeaderRewriteMap) String() string {
 	return fmt.Sprint(*r)
 }
 
+// Set method to implement flags.Value
 func (r *HeaderRewriteMap) Set(value string) error {
 	headerArr := strings.SplitN(value, ":", 2)
 	if len(headerArr) < 2 {
@@ -266,13 +282,15 @@ type urlRegexp struct {
 	regexp *regexp.Regexp
 }
 
-type HTTPUrlRegexp []urlRegexp
+// HTTPURLRegexp a slice of regexp to match URLs
+type HTTPURLRegexp []urlRegexp
 
-func (r *HTTPUrlRegexp) String() string {
+func (r *HTTPURLRegexp) String() string {
 	return fmt.Sprint(*r)
 }
 
-func (r *HTTPUrlRegexp) Set(value string) error {
+// Set method to implement flags.Value
+func (r *HTTPURLRegexp) Set(value string) error {
 	regexp, err := regexp.Compile(value)
 
 	*r = append(*r, urlRegexp{regexp: regexp})

http_modifier_settings_test.go

@@ -53,7 +53,7 @@ func TestHTTPHashFilters(t *testing.T) {
 
 func TestUrlRewriteMap(t *testing.T) {
 	var err error
-	rewrites := UrlRewriteMap{}
+	rewrites := URLRewriteMap{}
 
 	if err = rewrites.Set("/v1/user/([^\\/]+)/ping:/v2/user/$1/ping"); err != nil {
 		t.Error("Should set mapping", err)

http_modifier_test.go

@@ -121,7 +121,7 @@ func TestHTTPHeaderBasicAuthFilters(t *testing.T) {
 func TestHTTPModifierURLRewrite(t *testing.T) {
 	var url, newURL []byte
 
-	rewrites := UrlRewriteMap{}
+	rewrites := URLRewriteMap{}
 
 	payload := func(url []byte) []byte {
 		return []byte("POST " + string(url) + " HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
@@ -133,7 +133,7 @@ func TestHTTPModifierURLRewrite(t *testing.T) {
 	}
 
 	modifier := NewHTTPModifier(&HTTPModifierConfig{
-		UrlRewrite: rewrites,
+		URLRewrite: rewrites,
 	})
 
 	url = []byte("/v1/user/joe/ping")
@@ -236,12 +236,12 @@ func TestHTTPModifierHeaders(t *testing.T) {
 }
 
 func TestHTTPModifierURLRegexp(t *testing.T) {
-	filters := HTTPUrlRegexp{}
+	filters := HTTPURLRegexp{}
 	filters.Set("/v1/app")
 	filters.Set("/v1/api")
 
 	modifier := NewHTTPModifier(&HTTPModifierConfig{
-		UrlRegexp: filters,
+		URLRegexp: filters,
 	})
 
 	payload := func(url string) []byte {
@@ -262,12 +262,12 @@ func TestHTTPModifierURLRegexp(t *testing.T) {
 }
 
 func TestHTTPModifierURLNegativeRegexp(t *testing.T) {
-	filters := HTTPUrlRegexp{}
+	filters := HTTPURLRegexp{}
 	filters.Set("/restricted1")
 	filters.Set("/some/restricted2")
 
 	modifier := NewHTTPModifier(&HTTPModifierConfig{
-		UrlNegativeRegexp: filters,
+		URLNegativeRegexp: filters,
 	})
 
 	payload := func(url string) []byte {

http_prettifier.go

@@ -3,6 +3,7 @@ package main
 import (
 	"bytes"
 	"compress/gzip"
+	"fmt"
 	"io/ioutil"
 	"net/http/httputil"
 	"strconv"
@@ -11,38 +12,25 @@ import (
 )
 
 func prettifyHTTP(p []byte) []byte {
-	headSize := bytes.IndexByte(p, '\n') + 1
-	head := p[:headSize]
-	body := p[headSize:]
 
-	headersPos := proto.MIMEHeadersEndPos(body)
+	tEnc := bytes.Equal(proto.Header(p, []byte("Transfer-Encoding")), []byte("chunked"))
+	cEnc := bytes.Equal(proto.Header(p, []byte("Content-Encoding")), []byte("gzip"))
 
-	if headersPos < 5 || headersPos > len(body) {
+	if !(tEnc || cEnc) {
 		return p
 	}
 
-	headers := body[:headersPos]
-	content := body[headersPos:]
+	headersPos := proto.MIMEHeadersEndPos(p)
 
-	var tEnc, cEnc []byte
-	proto.ParseHeaders([][]byte{headers}, func(header, value []byte) bool {
-		if proto.HeadersEqual(header, []byte("Transfer-Encoding")) {
-			tEnc = value
-		}
-
-		if proto.HeadersEqual(header, []byte("Content-Encoding")) {
-			cEnc = value
-		}
-
-		return true
-	})
-
-	if len(tEnc) == 0 && len(cEnc) == 0 {
+	if headersPos < 5 || headersPos > len(p) {
 		return p
 	}
 
-	if bytes.Equal(tEnc, []byte("chunked")) {
-		buf := bytes.NewBuffer(content)
+	headers := p[:headersPos]
+	content := p[headersPos:]
+
+	if tEnc {
+		buf := bytes.NewReader(content)
 		r := httputil.NewChunkedReader(buf)
 		content, _ = ioutil.ReadAll(r)
 
@@ -52,16 +40,20 @@ func prettifyHTTP(p []byte) []byte {
 		headers = proto.SetHeader(headers, []byte("Content-Length"), []byte(newLen))
 	}
 
-	if bytes.Equal(cEnc, []byte("gzip")) {
-		buf := bytes.NewBuffer(content)
+	if cEnc {
+		buf := bytes.NewReader(content)
 		g, err := gzip.NewReader(buf)
 
 		if err != nil {
-			Debug("[Prettifier] GZIP encoding error:", err)
+			Debug(1, "[Prettifier] GZIP encoding error:", err)
 			return []byte{}
 		}
 
-		content, _ = ioutil.ReadAll(g)
+		content, err = ioutil.ReadAll(g)
+		if err != nil {
+			Debug(1, fmt.Sprintf("[HTTP-PRETTIFIER] %q", err))
+			return p
+		}
 
 		headers = proto.DeleteHeader(headers, []byte("Content-Encoding"))
 
@@ -69,7 +61,7 @@ func prettifyHTTP(p []byte) []byte {
 		headers = proto.SetHeader(headers, []byte("Content-Length"), []byte(newLen))
 	}
 
-	newPayload := append(append(head, headers...), content...)
+	newPayload := append(headers, content...)
 
 	return newPayload
 }

http_prettifier_test.go

@@ -5,6 +5,8 @@ import (
 	"compress/gzip"
 	"strconv"
 	"testing"
+
+	"github.com/buger/goreplay/proto"
 )
 
 func TestHTTPPrettifierGzip(t *testing.T) {
@@ -15,22 +17,21 @@ func TestHTTPPrettifierGzip(t *testing.T) {
 
 	size := strconv.Itoa(len(b.Bytes()))
 
-	payload := []byte("2 1 1\nHTTP/1.1 200 OK\r\nContent-Length: " + size + "\r\nContent-Encoding: gzip\r\n\r\n")
+	payload := []byte("HTTP/1.1 200 OK\r\nContent-Length: " + size + "\r\nContent-Encoding: gzip\r\n\r\n")
 	payload = append(payload, b.Bytes()...)
 
 	newPayload := prettifyHTTP(payload)
 
-	if string(newPayload) != "2 1 1\nHTTP/1.1 200 OK\r\nContent-Length: 4\r\n\r\ntest" {
-		t.Error("Payload not match:", string(newPayload))
+	if string(newPayload) != "HTTP/1.1 200 OK\r\nContent-Length: 4\r\n\r\ntest" {
+		t.Errorf("Payload not match %q", string(newPayload))
 	}
 }
 
 func TestHTTPPrettifierChunked(t *testing.T) {
 	payload := []byte("POST / HTTP/1.1\r\nHost: www.w3.org\r\nTransfer-Encoding: chunked\r\n\r\n4\r\nWiki\r\n5\r\npedia\r\ne\r\n in\r\n\r\nchunks.\r\n0\r\n\r\n")
 
-	newPayload := prettifyHTTP(payload)
-
-	if string(newPayload) != "POST / HTTP/1.1\r\nHost: www.w3.org\r\nContent-Length: 23\r\n\r\nWikipedia in\r\n\r\nchunks." {
-		t.Error("Payload not match:", string(newPayload))
+	payload = prettifyHTTP(payload)
+	if string(proto.Header(payload, []byte("Content-Length"))) != "23" {
+		t.Errorf("payload should have content length of 23")
 	}
 }

input_dummy.go

@@ -7,24 +7,30 @@ import (
 // DummyInput used for debugging. It generate 1 "GET /"" request per second.
 type DummyInput struct {
 	data chan []byte
+	quit chan struct{}
 }
 
 // NewDummyInput constructor for DummyInput
 func NewDummyInput(options string) (di *DummyInput) {
 	di = new(DummyInput)
 	di.data = make(chan []byte)
+	di.quit = make(chan struct{})
 
 	go di.emit()
 
 	return
 }
 
-func (i *DummyInput) Read(data []byte) (int, error) {
-	buf := <-i.data
-
-	copy(data, buf)
-
-	return len(buf), nil
+// PluginRead reads message from this plugin
+func (i *DummyInput) PluginRead() (*Message, error) {
+	var msg Message
+	select {
+	case <-i.quit:
+		return nil, ErrorStopped
+	case buf := <-i.data:
+		msg.Meta, msg.Data = payloadMetaWithBody(buf)
+		return &msg, nil
+	}
 }
 
 func (i *DummyInput) emit() {
@@ -46,3 +52,9 @@ func (i *DummyInput) emit() {
 func (i *DummyInput) String() string {
 	return "Dummy Input"
 }
+
+// Close closes this plugins
+func (i *DummyInput) Close() error {
+	close(i.quit)
+	return nil
+}

input_file.go

@@ -5,8 +5,8 @@ import (
 	"bytes"
 	"compress/gzip"
 	"errors"
+	"fmt"
 	"io"
-	"log"
 	"os"
 	"path/filepath"
 	"strconv"
@@ -32,20 +32,16 @@ type fileInputReader struct {
 func (f *fileInputReader) parseNext() error {
 	payloadSeparatorAsBytes := []byte(payloadSeparator)
 	var buffer bytes.Buffer
-
 	for {
 		line, err := f.reader.ReadBytes('\n')
 
 		if err != nil {
 			if err != io.EOF {
-				log.Println(err)
-				return err
-			}
-
-			if err == io.EOF {
+				Debug(1, err)
+			} else {
 				f.Close()
-				return err
 			}
+			return err
 		}
 
 		if bytes.Equal(payloadSeparatorAsBytes[1:], line) {
@@ -61,7 +57,6 @@ func (f *fileInputReader) parseNext() error {
 		buffer.Write(line)
 	}
 
-	return nil
 }
 
 func (f *fileInputReader) ReadPayload() []byte {
@@ -69,6 +64,8 @@ func (f *fileInputReader) ReadPayload() []byte {
 
 	return f.data
 }
+
+// Close closes this plugin
 func (f *fileInputReader) Close() error {
 	if atomic.LoadInt32(&f.closed) == 0 {
 		atomic.StoreInt32(&f.closed, 1)
@@ -78,7 +75,7 @@ func (f *fileInputReader) Close() error {
 	return nil
 }
 
-func NewFileInputReader(path string) *fileInputReader {
+func newFileInputReader(path string) *fileInputReader {
 	var file io.ReadCloser
 	var err error
 
@@ -89,7 +86,7 @@ func NewFileInputReader(path string) *fileInputReader {
 	}
 
 	if err != nil {
-		log.Println(err)
+		Debug(0, fmt.Sprintf("[INPUT-FILE] err: %q", err))
 		return nil
 	}
 
@@ -97,7 +94,7 @@ func NewFileInputReader(path string) *fileInputReader {
 	if strings.HasSuffix(path, ".gz") {
 		gzReader, err := gzip.NewReader(file)
 		if err != nil {
-			log.Println(err)
+			Debug(0, fmt.Sprintf("[INPUT-FILE] err: %q", err))
 			return nil
 		}
 		r.reader = bufio.NewReader(gzReader)
@@ -158,7 +155,7 @@ func (i *FileInput) init() (err error) {
 
 		resp, err := svc.ListObjects(params)
 		if err != nil {
-			log.Println("Error while retreiving list of files from S3", i.path, err)
+			Debug(0, "[INPUT-FILE] Error while retreiving list of files from S3", i.path, err)
 			return err
 		}
 
@@ -167,34 +164,35 @@ func (i *FileInput) init() (err error) {
 		}
 	} else {
 		if matches, err = filepath.Glob(i.path); err != nil {
-			log.Println("Wrong file pattern", i.path, err)
+			Debug(0, "[INPUT-FILE] Wrong file pattern", i.path, err)
 			return
 		}
 	}
 
 	if len(matches) == 0 {
-		log.Println("No files match pattern: ", i.path)
+		Debug(0, "[INPUT-FILE] No files match pattern: ", i.path)
 		return errors.New("No matching files")
 	}
 
 	i.readers = make([]*fileInputReader, len(matches))
 
 	for idx, p := range matches {
-		i.readers[idx] = NewFileInputReader(p)
+		i.readers[idx] = newFileInputReader(p)
 	}
 
 	return nil
 }
 
-func (i *FileInput) Read(data []byte) (int, error) {
-	var buf []byte
+// PluginRead reads message from this plugin
+func (i *FileInput) PluginRead() (*Message, error) {
+	var msg Message
 	select {
 	case <-i.exit:
-		return 0, ErrorStopped
-	case buf = <-i.data:
+		return nil, ErrorStopped
+	case buf := <-i.data:
+		msg.Meta, msg.Data = payloadMetaWithBody(buf)
+		return &msg, nil
 	}
-	copy(data, buf)
-	return len(buf), nil
 }
 
 func (i *FileInput) String() string {
@@ -261,16 +259,11 @@ func (i *FileInput) emit() {
 		}
 	}
 
-	log.Printf("FileInput: end of file '%s'\n", i.path)
+	Debug(0, fmt.Sprintf("[INPUT-FILE] FileInput: end of file '%s'\n", i.path))
 
-	// For now having fixed timeout is temporary solution
-	// Further should be modified, so outputs can report if their queue empty or not
-	time.Sleep(time.Second)
-	if closeCh != nil {
-		close(closeCh)
-	}
 }
 
+// Close closes this plugin
 func (i *FileInput) Close() error {
 	defer i.mu.Unlock()
 	i.mu.Lock()

input_file_test.go

@@ -4,31 +4,26 @@ import (
 	"bytes"
 	"errors"
 	"fmt"
-	"io"
 	"io/ioutil"
-	"log"
 	"math/rand"
 	"os"
 	"sync"
-	"syscall"
 	"testing"
 	"time"
 )
 
-var _ = log.Println
-
 func TestInputFileWithGET(t *testing.T) {
 	input := NewTestInput()
-	rg := NewRequestGenerator([]io.Reader{input}, func() { input.EmitGET() }, 1)
-	readPayloads := [][]byte{}
+	rg := NewRequestGenerator([]PluginReader{input}, func() { input.EmitGET() }, 1)
+	readPayloads := []*Message{}
 
 	// Given a capture file with a GET request
 	expectedCaptureFile := CreateCaptureFile(rg)
 	defer expectedCaptureFile.TearDown()
 
 	// When the request is read from the capture file
-	err := ReadFromCaptureFile(expectedCaptureFile.file, 1, func(data []byte) {
-		readPayloads = append(readPayloads, Duplicate(data))
+	err := ReadFromCaptureFile(expectedCaptureFile.file, 1, func(msg *Message) {
+		readPayloads = append(readPayloads, msg)
 	})
 
 	// The read request should match the original request
@@ -42,18 +37,17 @@ func TestInputFileWithGET(t *testing.T) {
 }
 
 func TestInputFileWithPayloadLargerThan64Kb(t *testing.T) {
-
 	input := NewTestInput()
-	rg := NewRequestGenerator([]io.Reader{input}, func() { input.EmitSizedPOST(64 * 1024) }, 1)
-	readPayloads := [][]byte{}
+	rg := NewRequestGenerator([]PluginReader{input}, func() { input.EmitSizedPOST(64 * 1024) }, 1)
+	readPayloads := []*Message{}
 
 	// Given a capture file with a request over 64Kb
 	expectedCaptureFile := CreateCaptureFile(rg)
 	defer expectedCaptureFile.TearDown()
 
 	// When the request is read from the capture file
-	err := ReadFromCaptureFile(expectedCaptureFile.file, 1, func(data []byte) {
-		readPayloads = append(readPayloads, Duplicate(data))
+	err := ReadFromCaptureFile(expectedCaptureFile.file, 1, func(msg *Message) {
+		readPayloads = append(readPayloads, msg)
 	})
 
 	// The read request should match the original request
@@ -70,19 +64,19 @@ func TestInputFileWithPayloadLargerThan64Kb(t *testing.T) {
 func TestInputFileWithGETAndPOST(t *testing.T) {
 
 	input := NewTestInput()
-	rg := NewRequestGenerator([]io.Reader{input}, func() {
+	rg := NewRequestGenerator([]PluginReader{input}, func() {
 		input.EmitGET()
 		input.EmitPOST()
 	}, 2)
-	readPayloads := [][]byte{}
+	readPayloads := []*Message{}
 
 	// Given a capture file with a GET request
 	expectedCaptureFile := CreateCaptureFile(rg)
 	defer expectedCaptureFile.TearDown()
 
 	// When the requests are read from the capture file
-	err := ReadFromCaptureFile(expectedCaptureFile.file, 2, func(data []byte) {
-		readPayloads = append(readPayloads, Duplicate(data))
+	err := ReadFromCaptureFile(expectedCaptureFile.file, 2, func(msg *Message) {
+		readPayloads = append(readPayloads, msg)
 	})
 
 	// The read requests should match the original request
@@ -114,12 +108,11 @@ func TestInputFileMultipleFilesWithRequestsOnly(t *testing.T) {
 	file2.Close()
 
 	input := NewFileInput(fmt.Sprintf("/tmp/%d*", rnd), false)
-	buf := make([]byte, 1000)
 
 	for i := '1'; i <= '4'; i++ {
-		n, _ := input.Read(buf)
-		if buf[4] != byte(i) {
-			t.Error("Should emit requests in right order", string(buf[:n]))
+		msg, _ := input.PluginRead()
+		if msg.Meta[4] != byte(i) {
+			t.Error("Should emit requests in right order", string(msg.Meta))
 		}
 	}
 
@@ -141,24 +134,18 @@ func TestInputFileRequestsWithLatency(t *testing.T) {
 	file.Write([]byte(payloadSeparator))
 
 	input := NewFileInput(fmt.Sprintf("/tmp/%d", rnd), false)
-	buf := make([]byte, 1000)
 
 	start := time.Now().UnixNano()
 	for i := 0; i < 3; i++ {
-		input.Read(buf)
+		input.PluginRead()
 	}
 	end := time.Now().UnixNano()
 
-	var expectedLatency int64 = 250000000 - 100000000
+	var expectedLatency int64 = 300000000 - 100000000
 	realLatency := end - start
-	if realLatency < expectedLatency {
+	if realLatency > expectedLatency {
 		t.Errorf("Should emit requests respecting latency. Expected: %v, real: %v", expectedLatency, realLatency)
 	}
-
-	if realLatency > expectedLatency+10000000 {
-		t.Errorf("Should emit requests respecting latency. Expected: %v, real: %v", expectedLatency, realLatency)
-
-	}
 }
 
 func TestInputFileMultipleFilesWithRequestsAndResponses(t *testing.T) {
@@ -187,17 +174,16 @@ func TestInputFileMultipleFilesWithRequestsAndResponses(t *testing.T) {
 	file2.Close()
 
 	input := NewFileInput(fmt.Sprintf("/tmp/%d*", rnd), false)
-	buf := make([]byte, 1000)
 
 	for i := '1'; i <= '4'; i++ {
-		n, _ := input.Read(buf)
-		if buf[0] != '1' && buf[4] != byte(i) {
-			t.Error("Shound emit requests in right order", string(buf[:n]))
+		msg, _ := input.PluginRead()
+		if msg.Meta[0] != '1' && msg.Meta[4] != byte(i) {
+			t.Error("Shound emit requests in right order", string(msg.Meta))
 		}
 
-		n, _ = input.Read(buf)
-		if buf[0] != '2' && buf[4] != byte(i) {
-			t.Error("Shound emit responses in right order", string(buf[:n]))
+		msg, _ = input.PluginRead()
+		if msg.Meta[0] != '2' && msg.Meta[4] != byte(i) {
+			t.Error("Shound emit responses in right order", string(msg.Meta))
 		}
 	}
 
@@ -216,11 +202,10 @@ func TestInputFileLoop(t *testing.T) {
 	file.Close()
 
 	input := NewFileInput(fmt.Sprintf("/tmp/%d", rnd), true)
-	buf := make([]byte, 1000)
 
 	// Even if we have just 2 requests in file, it should indifinitly loop
 	for i := 0; i < 1000; i++ {
-		input.Read(buf)
+		input.PluginRead()
 	}
 
 	input.Close()
@@ -232,22 +217,21 @@ func TestInputFileCompressed(t *testing.T) {
 
 	output := NewFileOutput(fmt.Sprintf("/tmp/%d_0.gz", rnd), &FileOutputConfig{FlushInterval: time.Minute, Append: true})
 	for i := 0; i < 1000; i++ {
-		output.Write([]byte("1 1 1\r\ntest"))
+		output.PluginWrite(&Message{Meta: []byte("1 1 1\r\n"), Data: []byte("test")})
 	}
 	name1 := output.file.Name()
 	output.Close()
 
 	output2 := NewFileOutput(fmt.Sprintf("/tmp/%d_1.gz", rnd), &FileOutputConfig{FlushInterval: time.Minute, Append: true})
 	for i := 0; i < 1000; i++ {
-		output2.Write([]byte("1 1 1\r\ntest"))
+		output2.PluginWrite(&Message{Meta: []byte("1 1 1\r\n"), Data: []byte("test")})
 	}
 	name2 := output2.file.Name()
 	output2.Close()
 
 	input := NewFileInput(fmt.Sprintf("/tmp/%d*", rnd), false)
-	buf := make([]byte, 1000)
 	for i := 0; i < 2000; i++ {
-		input.Read(buf)
+		input.PluginRead()
 	}
 
 	os.Remove(name1)
@@ -255,30 +239,30 @@ func TestInputFileCompressed(t *testing.T) {
 }
 
 type CaptureFile struct {
-	data [][]byte
+	msgs []*Message
 	file *os.File
 }
 
-func NewExpectedCaptureFile(data [][]byte, file *os.File) *CaptureFile {
+func NewExpectedCaptureFile(msgs []*Message, file *os.File) *CaptureFile {
 	ecf := new(CaptureFile)
 	ecf.file = file
-	ecf.data = data
+	ecf.msgs = msgs
 	return ecf
 }
 
 func (expectedCaptureFile *CaptureFile) TearDown() {
 	if expectedCaptureFile.file != nil {
-		syscall.Unlink(expectedCaptureFile.file.Name())
+		os.Remove(expectedCaptureFile.file.Name())
 	}
 }
 
 type RequestGenerator struct {
-	inputs []io.Reader
+	inputs []PluginReader
 	emit   func()
 	wg     *sync.WaitGroup
 }
 
-func NewRequestGenerator(inputs []io.Reader, emit func(), count int) (rg *RequestGenerator) {
+func NewRequestGenerator(inputs []PluginReader, emit func(), count int) (rg *RequestGenerator) {
 	rg = new(RequestGenerator)
 	rg.inputs = inputs
 	rg.emit = emit
@@ -287,14 +271,17 @@ func NewRequestGenerator(inputs []io.Reader, emit func(), count int) (rg *Reques
 	return
 }
 
-func (expectedCaptureFile *CaptureFile) PayloadsEqual(other [][]byte) bool {
+func (expectedCaptureFile *CaptureFile) PayloadsEqual(other []*Message) bool {
 
-	if len(expectedCaptureFile.data) != len(other) {
+	if len(expectedCaptureFile.msgs) != len(other) {
 		return false
 	}
 
 	for i, payload := range other {
-		if !bytes.Equal(expectedCaptureFile.data[i], payload) {
+		if !bytes.Equal(expectedCaptureFile.msgs[i].Meta, payload.Meta) {
+			return false
+		}
+		if !bytes.Equal(expectedCaptureFile.msgs[i].Data, payload.Data) {
 			return false
 		}
 	}
@@ -309,26 +296,24 @@ func CreateCaptureFile(requestGenerator *RequestGenerator) *CaptureFile {
 		panic(err)
 	}
 
-	quit := make(chan int)
-
-	readPayloads := [][]byte{}
-	output := NewTestOutput(func(data []byte) {
-		readPayloads = append(readPayloads, Duplicate(data))
+	readPayloads := []*Message{}
+	output := NewTestOutput(func(msg *Message) {
+		readPayloads = append(readPayloads, msg)
 		requestGenerator.wg.Done()
 	})
 
-	outputFile := NewFileOutput(f.Name(), &FileOutputConfig{FlushInterval: time.Minute, Append: true})
+	outputFile := NewFileOutput(f.Name(), &FileOutputConfig{FlushInterval: time.Second, Append: true})
 
 	plugins := &InOutPlugins{
 		Inputs:  requestGenerator.inputs,
-		Outputs: []io.Writer{output, outputFile},
+		Outputs: []PluginWriter{output, outputFile},
 	}
 	for _, input := range requestGenerator.inputs {
 		plugins.All = append(plugins.All, input)
 	}
 	plugins.All = append(plugins.All, output, outputFile)
 
-	emitter := NewEmitter(quit)
+	emitter := NewEmitter()
 	go emitter.Start(plugins, Settings.Middleware)
 
 	requestGenerator.emit()
@@ -342,23 +327,22 @@ func CreateCaptureFile(requestGenerator *RequestGenerator) *CaptureFile {
 }
 
 func ReadFromCaptureFile(captureFile *os.File, count int, callback writeCallback) (err error) {
-	quit := make(chan int)
 	wg := new(sync.WaitGroup)
 
 	input := NewFileInput(captureFile.Name(), false)
-	output := NewTestOutput(func(data []byte) {
-		callback(data)
+	output := NewTestOutput(func(msg *Message) {
+		callback(msg)
 		wg.Done()
 	})
 
 	plugins := &InOutPlugins{
-		Inputs:  []io.Reader{input},
-		Outputs: []io.Writer{output},
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{output},
 	}
 	plugins.All = append(plugins.All, input, output)
 
 	wg.Add(count)
-	emitter := NewEmitter(quit)
+	emitter := NewEmitter()
 	go emitter.Start(plugins, Settings.Middleware)
 
 	done := make(chan int, 1)
@@ -373,14 +357,7 @@ func ReadFromCaptureFile(captureFile *os.File, count int, callback writeCallback
 	case <-time.After(2 * time.Second):
 		err = errors.New("Timed out")
 	}
-	emitter.close()
+	emitter.Close()
 	return
 
 }
-
-func Duplicate(data []byte) (duplicate []byte) {
-	duplicate = make([]byte, len(data))
-	copy(duplicate, data)
-
-	return
-}

input_http.go

@@ -16,11 +16,10 @@ type HTTPInput struct {
 	stop     chan bool // Channel used only to indicate goroutine should shutdown
 }
 
-// NewHTTPInput constructor for HTTPInput. Accepts address with port which he will listen on.
+// NewHTTPInput constructor for HTTPInput. Accepts address with port which it will listen on.
 func NewHTTPInput(address string) (i *HTTPInput) {
 	i = new(HTTPInput)
-	i.data = make(chan []byte, 10000)
-	i.address = address
+	i.data = make(chan []byte, 1000)
 	i.stop = make(chan bool)
 
 	i.listen(address)
@@ -28,22 +27,20 @@ func NewHTTPInput(address string) (i *HTTPInput) {
 	return
 }
 
-func (i *HTTPInput) Read(data []byte) (int, error) {
-	var buf []byte
+// PluginRead reads message from this plugin
+func (i *HTTPInput) PluginRead() (*Message, error) {
+	var msg Message
 	select {
 	case <-i.stop:
-		return 0, ErrorStopped
-	case buf = <-i.data:
+		return nil, ErrorStopped
+	case buf := <-i.data:
+		msg.Data = buf
+		msg.Meta = payloadHeader(RequestPayload, uuid(), time.Now().UnixNano(), -1)
+		return &msg, nil
 	}
-
-	header := payloadHeader(RequestPayload, uuid(), time.Now().UnixNano(), -1)
-
-	copy(data[0:len(header)], header)
-	copy(data[len(header):], buf)
-
-	return len(buf) + len(header), nil
 }
 
+// Close closes this plugin
 func (i *HTTPInput) Close() error {
 	close(i.stop)
 	return nil
@@ -51,16 +48,11 @@ func (i *HTTPInput) Close() error {
 
 func (i *HTTPInput) handler(w http.ResponseWriter, r *http.Request) {
 	r.URL.Scheme = "http"
-	r.URL.Host = i.listener.Addr().String()
+	r.URL.Host = i.address
 
 	buf, _ := httputil.DumpRequestOut(r, true)
 	http.Error(w, http.StatusText(200), 200)
-
-	select {
-	case i.data <- buf:
-	default:
-		Debug("[INPUT-HTTP] Dropping requests because output can't process them fast enough")
-	}
+	i.data <- buf
 }
 
 func (i *HTTPInput) listen(address string) {
@@ -74,11 +66,12 @@ func (i *HTTPInput) listen(address string) {
 	if err != nil {
 		log.Fatal("HTTP input listener failure:", err)
 	}
+	i.address = i.listener.Addr().String()
 
 	go func() {
 		err = http.Serve(i.listener, mux)
-		if err != nil {
-			log.Fatal("HTTP input serve failure:", err)
+		if err != nil && err != http.ErrServerClosed {
+			log.Fatal("HTTP input serve failure ", err)
 		}
 	}()
 }

input_http_test.go

@@ -1,36 +1,33 @@
 package main
 
 import (
-	"io"
-	"log"
+	"bytes"
 	"net/http"
-	"os/exec"
 	"strings"
 	"sync"
 	"testing"
-
-	"github.com/buger/goreplay/proto"
+	"time"
 )
 
 func TestHTTPInput(t *testing.T) {
 	wg := new(sync.WaitGroup)
-	quit := make(chan int)
 
 	input := NewHTTPInput("127.0.0.1:0")
-	output := NewTestOutput(func(data []byte) {
+	time.Sleep(time.Millisecond)
+	output := NewTestOutput(func(*Message) {
 		wg.Done()
 	})
 
 	plugins := &InOutPlugins{
-		Inputs:  []io.Reader{input},
-		Outputs: []io.Writer{output},
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{output},
 	}
 	plugins.All = append(plugins.All, input, output)
 
-	emitter := NewEmitter(quit)
+	emitter := NewEmitter()
 	go emitter.Start(plugins, Settings.Middleware)
 
-	address := strings.Replace(input.listener.Addr().String(), "[::]", "127.0.0.1", -1)
+	address := strings.Replace(input.address, "[::]", "127.0.0.1", -1)
 
 	for i := 0; i < 100; i++ {
 		wg.Add(1)
@@ -43,38 +40,41 @@ func TestHTTPInput(t *testing.T) {
 
 func TestInputHTTPLargePayload(t *testing.T) {
 	wg := new(sync.WaitGroup)
-	quit := make(chan int)
-
-	dd := exec.Command("dd", "if=/dev/urandom", "of=/tmp/large", "bs=1", "count=4000000")
-	err := dd.Run()
-	if err != nil {
-		log.Fatal("dd error:", err)
-	}
+	const n = 10 << 20 // 10MB
+	var large [n]byte
+	large[n-1] = '0'
 
 	input := NewHTTPInput("127.0.0.1:0")
-	output := NewTestOutput(func(data []byte) {
-		if len(proto.Body(payloadBody(data))) != 4000000 {
-			t.Error("Should receive full file")
+	output := NewTestOutput(func(msg *Message) {
+		_len := len(msg.Data)
+		if _len >= n { // considering http body CRLF
+			t.Errorf("expected body to be >= %d", n)
 		}
 		wg.Done()
 	})
 	plugins := &InOutPlugins{
-		Inputs:  []io.Reader{input},
-		Outputs: []io.Writer{output},
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{output},
 	}
 	plugins.All = append(plugins.All, input, output)
 
-	emitter := NewEmitter(quit)
+	emitter := NewEmitter()
+	defer emitter.Close()
 	go emitter.Start(plugins, Settings.Middleware)
 
-	wg.Add(1)
-	address := strings.Replace(input.listener.Addr().String(), "[::]", "127.0.0.1", -1)
-	curl := exec.Command("curl", "http://"+address, "--data-binary", "@/tmp/large")
-	err = curl.Run()
+	address := strings.Replace(input.address, "[::]", "127.0.0.1", -1)
+	var req *http.Request
+	var err error
+	req, err = http.NewRequest("POST", "http://"+address, bytes.NewBuffer(large[:]))
 	if err != nil {
-		log.Fatal("curl error:", err)
+		t.Error(err)
+		return
+	}
+	wg.Add(1)
+	_, err = http.DefaultClient.Do(req)
+	if err != nil {
+		t.Error(err)
+		return
 	}
-
 	wg.Wait()
-	emitter.Close()
 }

input_kafka.go

@@ -15,12 +15,12 @@ type KafkaInput struct {
 	config    *InputKafkaConfig
 	consumers []sarama.PartitionConsumer
 	messages  chan *sarama.ConsumerMessage
+	quit      chan struct{}
 }
 
-// NewKafkaInput creates instance of kafka consumer client.
-func NewKafkaInput(address string, config *InputKafkaConfig) *KafkaInput {
-	c := sarama.NewConfig()
-	// Configuration options go here
+// NewKafkaInput creates instance of kafka consumer client with TLS config
+func NewKafkaInput(address string, config *InputKafkaConfig, tlsConfig *KafkaTLSConfig) *KafkaInput {
+	c := NewKafkaConfig(tlsConfig)
 
 	var con sarama.Consumer
 
@@ -28,7 +28,6 @@ func NewKafkaInput(address string, config *InputKafkaConfig) *KafkaInput {
 		con = config.consumer
 	} else {
 		var err error
-		//con, err = sarama.NewConsumer([]string{config.Host}, c)
 		con, err = sarama.NewConsumer(strings.Split(config.Host, ","), c)
 
 		if err != nil {
@@ -45,6 +44,7 @@ func NewKafkaInput(address string, config *InputKafkaConfig) *KafkaInput {
 		config:    config,
 		consumers: make([]sarama.PartitionConsumer, len(partitions)),
 		messages:  make(chan *sarama.ConsumerMessage, 256),
+		quit:      make(chan struct{}),
 	}
 
 	for index, partition := range partitions {
@@ -61,10 +61,7 @@ func NewKafkaInput(address string, config *InputKafkaConfig) *KafkaInput {
 			}
 		}(consumer)
 
-		if Settings.Verbose {
-			// Start infinite loop for tracking errors for kafka producer.
-			go i.ErrorHandler(consumer)
-		}
+		go i.ErrorHandler(consumer)
 
 		i.consumers[index] = consumer
 	}
@@ -75,33 +72,49 @@ func NewKafkaInput(address string, config *InputKafkaConfig) *KafkaInput {
 // ErrorHandler should receive errors
 func (i *KafkaInput) ErrorHandler(consumer sarama.PartitionConsumer) {
 	for err := range consumer.Errors() {
-		log.Println("Failed to read access log entry:", err)
+		Debug(1, "Failed to read access log entry:", err)
 	}
 }
 
-func (i *KafkaInput) Read(data []byte) (int, error) {
-	message := <-i.messages
-
-	if !i.config.UseJSON {
-		copy(data, message.Value)
-		return len(message.Value), nil
+// PluginRead a reads message from this plugin
+func (i *KafkaInput) PluginRead() (*Message, error) {
+	var message *sarama.ConsumerMessage
+	var msg Message
+	select {
+	case <-i.quit:
+		return nil, ErrorStopped
+	case message = <-i.messages:
 	}
 
-	var kafkaMessage KafkaMessage
-	json.Unmarshal(message.Value, &kafkaMessage)
+	msg.Data = message.Value
+	if i.config.UseJSON {
 
-	buf, err := kafkaMessage.Dump()
-	if err != nil {
-		log.Println("Failed to decode access log entry:", err)
-		return 0, err
+		var kafkaMessage KafkaMessage
+		json.Unmarshal(message.Value, &kafkaMessage)
+
+		var err error
+		msg.Data, err = kafkaMessage.Dump()
+		if err != nil {
+			Debug(1, "[INPUT-KAFKA] failed to decode access log entry:", err)
+			return nil, err
+		}
 	}
 
-	copy(data, buf)
+	// does it have meta
+	if isOriginPayload(msg.Data) {
+		msg.Meta, msg.Data = payloadMetaWithBody(msg.Data)
+	}
 
-	return len(buf), nil
+	return &msg, nil
 
 }
 
 func (i *KafkaInput) String() string {
 	return "Kafka Input: " + i.config.Host + "/" + i.config.Topic
 }
+
+// Close closes this plugin
+func (i *KafkaInput) Close() error {
+	close(i.quit)
+	return nil
+}

input_kafka_test.go

@@ -20,17 +20,16 @@ func TestInputKafkaRAW(t *testing.T) {
 		consumer: consumer,
 		Topic:    "test",
 		UseJSON:  false,
-	})
+	}, nil)
 
-	buf := make([]byte, 1024)
-	n, err := input.Read(buf)
+	msg, err := input.PluginRead()
 
 	if err != nil {
 		t.Fatal(err)
 	}
 
-	if string(buf[:n]) != "1 2 3\nGET / HTTP1.1\r\nHeader: 1\r\n\r\n" {
-		t.Error("Message not properly decoded: ", string(buf[:n]), n)
+	if string(append(msg.Meta, msg.Data...)) != "1 2 3\nGET / HTTP1.1\r\nHeader: 1\r\n\r\n" {
+		t.Error("Message not properly decoded")
 	}
 }
 
@@ -47,16 +46,15 @@ func TestInputKafkaJSON(t *testing.T) {
 		consumer: consumer,
 		Topic:    "test",
 		UseJSON:  true,
-	})
+	}, nil)
 
-	buf := make([]byte, 1024)
-	n, err := input.Read(buf)
+	msg, err := input.PluginRead()
 
 	if err != nil {
 		t.Fatal(err)
 	}
 
-	if string(buf[:n]) != "1 2 3\nGET / HTTP/1.1\r\nHeader: 1\r\n\r\n" {
-		t.Error("Message not properly decoded: ", string(buf[:n]), n)
+	if string(append(msg.Meta, msg.Data...)) != "1 2 3\nGET / HTTP/1.1\r\nHeader: 1\r\n\r\n" {
+		t.Error("Message not properly decoded")
 	}
 }

input_raw.go

@@ -1,61 +1,108 @@
 package main
 
 import (
+	"context"
+	"fmt"
 	"log"
 	"net"
+	"strconv"
+	"sync"
 	"time"
 
-	raw "github.com/buger/goreplay/capture"
+	"github.com/buger/goreplay/capture"
 	"github.com/buger/goreplay/proto"
+	"github.com/buger/goreplay/size"
+	"github.com/buger/goreplay/stat"
+	"github.com/buger/goreplay/tcp"
 )
 
+// TCPProtocol is a number to indicate type of protocol
+type TCPProtocol uint8
+
+const (
+	// ProtocolHTTP ...
+	ProtocolHTTP TCPProtocol = iota
+	// ProtocolBinary ...
+	ProtocolBinary
+)
+
+// Set is here so that TCPProtocol can implement flag.Var
+func (protocol *TCPProtocol) Set(v string) error {
+	switch v {
+	case "", "http":
+		*protocol = ProtocolHTTP
+	case "binary":
+		*protocol = ProtocolBinary
+	default:
+		return fmt.Errorf("unsupported protocol %s", v)
+	}
+	return nil
+}
+
+func (protocol *TCPProtocol) String() string {
+	switch *protocol {
+	case ProtocolBinary:
+		return "binary"
+	case ProtocolHTTP:
+		return "http"
+	default:
+		return ""
+	}
+}
+
+// RAWInputConfig represents configuration that can be applied on raw input
+type RAWInputConfig struct {
+	capture.PcapOptions
+	Expire         time.Duration      `json:"input-raw-expire"`
+	CopyBufferSize size.Size          `json:"copy-buffer-size"`
+	Engine         capture.EngineType `json:"input-raw-engine"`
+	TrackResponse  bool               `json:"input-raw-track-response"`
+	Protocol       TCPProtocol        `json:"input-raw-protocol"`
+	RealIPHeader   string             `json:"input-raw-realip-header"`
+	Stats          bool               `json:"input-raw-stats"`
+	quit           chan bool          // Channel used only to indicate goroutine should shutdown
+	host           string
+	port           uint16
+}
+
 // RAWInput used for intercepting traffic for given address
 type RAWInput struct {
-	data          chan *raw.TCPMessage
-	address       string
-	expire        time.Duration
-	quit          chan bool // Channel used only to indicate goroutine should shutdown
-	engine        int
-	realIPHeader  []byte
-	trackResponse bool
-	listener      *raw.Listener
-	protocol      raw.TCPProtocol
-	bpfFilter     string
-	timestampType string
-	bufferSize    int64
+	sync.Mutex
+	RAWInputConfig
+	st             *stat.Writer
+	listener       *capture.Listener
+	message        chan *tcp.Message
+	cancelListener context.CancelFunc
 }
 
-// Available engines for intercepting traffic
-const (
-	EngineRawSocket = 1 << iota
-	EnginePcap
-	EnginePcapFile
-)
-
-// NewRAWInput constructor for RAWInput. Accepts address with port as argument.
-func NewRAWInput(address string, engine int, trackResponse bool, expire time.Duration, realIPHeader string, protocol string, bpfFilter string, timestampType string, bufferSize int64) (i *RAWInput) {
+// NewRAWInput constructor for RAWInput. Accepts raw input config as arguments.
+func NewRAWInput(address string, config RAWInputConfig) (i *RAWInput) {
 	i = new(RAWInput)
-	i.data = make(chan *raw.TCPMessage)
-	i.address = address
-	i.expire = expire
-	i.engine = engine
-	i.bpfFilter = bpfFilter
-	i.realIPHeader = []byte(realIPHeader)
+	i.RAWInputConfig = config
+	i.message = make(chan *tcp.Message, 1000)
 	i.quit = make(chan bool)
-	i.trackResponse = trackResponse
-	i.timestampType = timestampType
-	i.bufferSize = bufferSize
+	var host, _port string
+	var err error
+	var port int
+	host, _port, err = net.SplitHostPort(address)
+	if err != nil {
+		log.Fatalf("input-raw: error while parsing address: %s", err)
+	}
+	if _port != "" {
+		port, err = strconv.Atoi(_port)
+	}
 
-	switch protocol {
-	case "http":
-		i.protocol = raw.ProtocolHTTP
-	case "binary":
-		i.protocol = raw.ProtocolBinary
-		if !PRO {
-			log.Fatal("Binary protocols can be used only with PRO license")
-		}
-	default:
-		log.Fatal("Unsupported protocol:", protocol)
+	if err != nil {
+		log.Fatalf("parsing port error: %v", err)
+	}
+	i.host = host
+	i.port = uint16(port)
+	if i.Stats {
+		i.st = initializeStatWriter("input_raw_message.csv", "type: GOR_STAT\ntitle: raw messages info")
+		statsWriteRecord(i.st, []string{
+			"length", "lostdata", "source", "destination", "ipversion",
+			"timedout", "truncated", "id",
+		}, "[INPUT-RAW]")
 	}
 
 	i.listen(address)
@@ -63,63 +110,108 @@ func NewRAWInput(address string, engine int, trackResponse bool, expire time.Dur
 	return
 }
 
-func (i *RAWInput) Read(data []byte) (int, error) {
-	var msg *raw.TCPMessage
+// PluginRead reads meassage from this plugin
+func (i *RAWInput) PluginRead() (*Message, error) {
+	var msgTCP *tcp.Message
+	var msg Message
 	select {
 	case <-i.quit:
-		return 0, ErrorStopped
-	case msg = <-i.data:
-	}
-
-	buf := msg.Bytes()
-
-	var header []byte
-
-	if msg.IsIncoming {
-		header = payloadHeader(RequestPayload, msg.UUID(), msg.Start.UnixNano(), -1)
-		if len(i.realIPHeader) > 0 {
-			buf = proto.SetHeader(buf, i.realIPHeader, []byte(msg.IP().String()))
+		if i.st != nil {
+			i.st.Close()
+			i.st = nil
 		}
-	} else {
-		header = payloadHeader(ResponsePayload, msg.UUID(), msg.Start.UnixNano(), msg.End.UnixNano()-msg.AssocMessage.End.UnixNano())
+		return nil, ErrorStopped
+	case msgTCP = <-i.message:
+		msg.Data = msgTCP.Data()
 	}
+	var msgType byte = ResponsePayload
+	timestamp := msgTCP.Start.UnixNano()
+	latency := msgTCP.End.UnixNano() - timestamp
+	if msgTCP.IsIncoming {
+		msgType = RequestPayload
+		if i.RealIPHeader != "" {
+			msg.Data = proto.SetHeader(msg.Data, []byte(i.RealIPHeader), []byte(msgTCP.SrcAddr))
+		}
+	}
+	id := msgTCP.UUID()
+	msg.Meta = payloadHeader(msgType, id, timestamp, latency)
 
-	copy(data[0:len(header)], header)
-	copy(data[len(header):], buf)
-
-	return len(buf) + len(header), nil
+	if i.Stats && i.st != nil {
+		statsWriteRecord(i.st, getStats(msgTCP.Stats, id), "[INPUT-RAW]")
+	}
+	msgTCP = nil
+	return &msg, nil
 }
 
 func (i *RAWInput) listen(address string) {
-	Debug("Listening for traffic on: " + address)
-
-	host, port, err := net.SplitHostPort(address)
+	var err error
+	i.listener, err = capture.NewListener(i.host, i.port, "", i.Engine, i.TrackResponse)
 	if err != nil {
-		log.Fatalf("input-raw: error while parsing address: %s", err)
+		log.Fatal(err)
 	}
+	i.listener.SetPcapOptions(i.PcapOptions)
+	err = i.listener.Activate()
+	if err != nil {
+		log.Fatal(err)
+	}
+	pool := tcp.NewMessagePool(i.CopyBufferSize, i.Expire, Debug, i.handler)
+	pool.MatchUUID(i.TrackResponse)
+	if i.Protocol == ProtocolHTTP {
+		pool.Start = http1StartHint
+	}
+	var ctx context.Context
+	ctx, i.cancelListener = context.WithCancel(context.Background())
+	errCh := i.listener.ListenBackground(ctx, pool.Handler)
+	select {
+	case err := <-errCh:
+		log.Fatal(err)
+	case <-i.listener.Reading:
+		Debug(1, i)
+	}
+}
 
-	i.listener = raw.NewListener(host, port, i.engine, i.trackResponse, i.expire, i.protocol, i.bpfFilter, i.timestampType, i.bufferSize, Settings.InputRAWConfig.OverrideSnapLen, Settings.InputRAWConfig.ImmediateMode)
-
-	ch := i.listener.Receiver()
-
-	go func() {
-		for {
-			select {
-			case <-i.quit:
-				return
-			case i.data <- <-ch: // Receiving TCPMessage object
-			}
-		}
-	}()
+func (i *RAWInput) handler(m *tcp.Message) {
+	i.message <- m
 }
 
 func (i *RAWInput) String() string {
-	return "Intercepting traffic from: " + i.address
+	return fmt.Sprintf("Intercepting traffic from: %s:%d", i.host, i.port)
+}
+
+// GetStats returns the stats so far and reset the stats
+func getStats(stat tcp.Stats, id []byte) []string {
+	var s [8]string
+
+	s[0] = strconv.Itoa(stat.Length)    // length
+	s[1] = strconv.Itoa(stat.LostData)  // lostdata
+	s[2] = stat.SrcAddr                 // source
+	s[3] = stat.DstAddr                 // destination
+	s[4] = string(stat.IPversion + '0') // ipversion
+	if stat.TimedOut {                  // timedout
+		s[5] = "true"
+	} else {
+		s[5] = "false"
+	}
+	if stat.Truncated { // truncated
+		s[6] = "true"
+	} else {
+		s[6] = "false"
+	}
+	s[7] = string(id)
+	return s[:]
 }
 
 // Close closes the input raw listener
 func (i *RAWInput) Close() error {
-	i.listener.Close()
+	i.cancelListener()
 	close(i.quit)
 	return nil
 }
+
+func http1StartHint(pckt *tcp.Packet) (isIncoming, isOutgoing bool) {
+	isIncoming = proto.HasRequestTitle(pckt.Payload)
+	if isIncoming {
+		return
+	}
+	return false, proto.HasResponseTitle(pckt.Payload)
+}

input_raw_test.go

@@ -2,23 +2,19 @@ package main
 
 import (
 	"bytes"
-	"io"
 	"io/ioutil"
-	"log"
-	"math/rand"
 	"net"
 	"net/http"
 	"net/http/httptest"
 	"net/http/httputil"
-	"os"
 	"os/exec"
-	"strconv"
 	"strings"
 	"sync"
 	"sync/atomic"
 	"testing"
 	"time"
 
+	"github.com/buger/goreplay/capture"
 	"github.com/buger/goreplay/proto"
 )
 
@@ -26,79 +22,80 @@ const testRawExpire = time.Millisecond * 200
 
 func TestRAWInputIPv4(t *testing.T) {
 	wg := new(sync.WaitGroup)
-	quit := make(chan int)
 
-	listener, err := net.Listen("tcp", ":0")
+	listener, err := net.Listen("tcp4", "127.0.0.1:0")
 	if err != nil {
-		t.Fatal(err)
+		t.Error(err)
+		return
 	}
 	origin := &http.Server{
-		Handler:      http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {}),
+		Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			w.Write([]byte("ab"))
+		}),
 		ReadTimeout:  10 * time.Second,
 		WriteTimeout: 10 * time.Second,
 	}
 	go origin.Serve(listener)
 	defer listener.Close()
-
-	originAddr := listener.Addr().String()
+	_, port, _ := net.SplitHostPort(listener.Addr().String())
 
 	var respCounter, reqCounter int64
+	conf := RAWInputConfig{
+		Engine:        capture.EnginePcap,
+		Expire:        0,
+		Protocol:      ProtocolHTTP,
+		TrackResponse: true,
+		RealIPHeader:  "X-Real-IP",
+	}
+	input := NewRAWInput(listener.Addr().String(), conf)
 
-	input := NewRAWInput(originAddr, EnginePcap, true, testRawExpire, "X-Real-IP", "http", "", "", 0)
-	defer input.Close()
-
-	output := NewTestOutput(func(data []byte) {
-		if data[0] == '1' {
-			body := payloadBody(data)
-			if len(proto.Header(body, []byte("X-Real-IP"))) == 0 {
-				t.Error("Should have X-Real-IP header", string(body))
+	output := NewTestOutput(func(msg *Message) {
+		if msg.Meta[0] == '1' {
+			if len(proto.Header(msg.Data, []byte("X-Real-IP"))) == 0 {
+				t.Error("Should have X-Real-IP header")
 			}
-			atomic.AddInt64(&reqCounter, 1)
+			reqCounter++
 		} else {
-			atomic.AddInt64(&respCounter, 1)
+			respCounter++
 		}
-
-		if Settings.Debug {
-			log.Println(reqCounter, respCounter)
-		}
-
 		wg.Done()
 	})
 
 	plugins := &InOutPlugins{
-		Inputs:  []io.Reader{input},
-		Outputs: []io.Writer{output},
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{output},
 	}
 	plugins.All = append(plugins.All, input, output)
 
-	client := NewHTTPClient("http://"+listener.Addr().String(), &HTTPClientConfig{})
-
-	emitter := NewEmitter(quit)
+	addr := "http://127.0.0.1:" + port
+	emitter := NewEmitter()
+	defer emitter.Close()
 	go emitter.Start(plugins, Settings.Middleware)
-
-	for i := 0; i < 100; i++ {
-		// request + response
+	for i := 0; i < 1; i++ {
 		wg.Add(2)
-		client.Get("/")
-		time.Sleep(2 * time.Millisecond)
+		_, err = http.Get(addr)
+		if err != nil {
+			t.Error(err)
+			return
+		}
 	}
-
 	wg.Wait()
-	emitter.Close()
+	const want = 10
+	if reqCounter != respCounter && reqCounter != want {
+		t.Errorf("want %d requests and %d responses, got %d requests and %d responses", want, want, reqCounter, respCounter)
+	}
 }
 
 func TestRAWInputNoKeepAlive(t *testing.T) {
 	wg := new(sync.WaitGroup)
-	quit := make(chan int)
 
-	listener, err := net.Listen("tcp", ":0")
+	listener, err := net.Listen("tcp", "127.0.0.1:0")
 	if err != nil {
 		t.Fatal(err)
 	}
 	origin := &http.Server{
 		Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-			w.Write([]byte("a"))
-			w.Write([]byte("b"))
+			w.Write([]byte("ab"))
 		}),
 		ReadTimeout:  10 * time.Second,
 		WriteTimeout: 10 * time.Second,
@@ -106,128 +103,137 @@ func TestRAWInputNoKeepAlive(t *testing.T) {
 	origin.SetKeepAlivesEnabled(false)
 	go origin.Serve(listener)
 	defer listener.Close()
+	_, port, _ := net.SplitHostPort(listener.Addr().String())
 
-	originAddr := listener.Addr().String()
-
-	input := NewRAWInput(originAddr, EnginePcap, true, testRawExpire, "", "http", "", "", 0)
-	defer input.Close()
-
-	output := NewTestOutput(func(data []byte) {
+	conf := RAWInputConfig{
+		Engine:        capture.EnginePcap,
+		Expire:        testRawExpire,
+		Protocol:      ProtocolHTTP,
+		TrackResponse: true,
+	}
+	input := NewRAWInput(":"+port, conf)
+	var respCounter, reqCounter int64
+	output := NewTestOutput(func(msg *Message) {
+		if msg.Meta[0] == '1' {
+			atomic.AddInt64(&reqCounter, 1)
+		} else {
+			atomic.AddInt64(&respCounter, 1)
+		}
 		wg.Done()
 	})
 
 	plugins := &InOutPlugins{
-		Inputs:  []io.Reader{input},
-		Outputs: []io.Writer{output},
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{output},
 	}
 	plugins.All = append(plugins.All, input, output)
 
-	client := NewHTTPClient("http://"+listener.Addr().String(), &HTTPClientConfig{})
+	addr := "http://127.0.0.1:" + port
 
-	emitter := NewEmitter(quit)
+	emitter := NewEmitter()
 	go emitter.Start(plugins, Settings.Middleware)
 
-	for i := 0; i < 100; i++ {
+	for i := 0; i < 10; i++ {
 		// request + response
 		wg.Add(2)
-		client.Get("/")
-		time.Sleep(2 * time.Millisecond)
+		_, err = http.Get(addr)
+		if err != nil {
+			t.Error(err)
+			return
+		}
 	}
 
 	wg.Wait()
+	const want = 10
+	if reqCounter != respCounter && reqCounter != want {
+		t.Errorf("want %d requests and %d responses, got %d requests and %d responses", want, want, reqCounter, respCounter)
+	}
 	emitter.Close()
 }
 
 func TestRAWInputIPv6(t *testing.T) {
 	wg := new(sync.WaitGroup)
-	quit := make(chan int)
 
 	listener, err := net.Listen("tcp", "[::1]:0")
 	if err != nil {
-		t.Fatal(err)
+		return
 	}
 	origin := &http.Server{
-		Handler:      http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {}),
+		Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			w.Write([]byte("ab"))
+		}),
 		ReadTimeout:  10 * time.Second,
 		WriteTimeout: 10 * time.Second,
 	}
 	go origin.Serve(listener)
 	defer listener.Close()
-
-	originAddr := listener.Addr().String()
+	_, port, _ := net.SplitHostPort(listener.Addr().String())
+	originAddr := "[::1]:" + port
 
 	var respCounter, reqCounter int64
+	conf := RAWInputConfig{
+		Engine:        capture.EnginePcap,
+		Protocol:      ProtocolHTTP,
+		TrackResponse: true,
+	}
+	input := NewRAWInput(originAddr, conf)
 
-	input := NewRAWInput(originAddr, EnginePcap, true, testRawExpire, "", "http", "", "", 0)
-	defer input.Close()
-
-	output := NewTestOutput(func(data []byte) {
-		if data[0] == '1' {
+	output := NewTestOutput(func(msg *Message) {
+		if msg.Meta[0] == '1' {
 			atomic.AddInt64(&reqCounter, 1)
 		} else {
 			atomic.AddInt64(&respCounter, 1)
 		}
-
-		if Settings.Debug {
-			log.Println(reqCounter, respCounter)
-		}
-
 		wg.Done()
 	})
 
 	plugins := &InOutPlugins{
-		Inputs:  []io.Reader{input},
-		Outputs: []io.Writer{output},
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{output},
 	}
-	plugins.All = append(plugins.All, input, output)
 
-	client := NewHTTPClient("http://"+listener.Addr().String(), &HTTPClientConfig{})
-
-	emitter := NewEmitter(quit)
+	emitter := NewEmitter()
+	addr := "http://" + originAddr
 	go emitter.Start(plugins, Settings.Middleware)
-
-	for i := 0; i < 100; i++ {
+	for i := 0; i < 10; i++ {
 		// request + response
 		wg.Add(2)
-		client.Get("/")
-		time.Sleep(2 * time.Millisecond)
+		_, err = http.Get(addr)
+		if err != nil {
+			t.Error(err)
+			return
+		}
 	}
 
 	wg.Wait()
+	const want = 10
+	if reqCounter != respCounter && reqCounter != want {
+		t.Errorf("want %d requests and %d responses, got %d requests and %d responses", want, want, reqCounter, respCounter)
+	}
 	emitter.Close()
 }
 
-func TestInputRAW100Expect(t *testing.T) {
+func TestInputRAWChunkedEncoding(t *testing.T) {
 	wg := new(sync.WaitGroup)
-	quit := make(chan int)
 
-	fileContent, _ := ioutil.ReadFile("COMM-LICENSE")
+	fileContent, _ := ioutil.ReadFile("README.md")
 
 	// Origing and Replay server initialization
 	origin := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		defer r.Body.Close()
 		ioutil.ReadAll(r.Body)
+
 		wg.Done()
 	}))
-	defer origin.Close()
 
 	originAddr := strings.Replace(origin.Listener.Addr().String(), "[::]", "127.0.0.1", -1)
-
-	input := NewRAWInput(originAddr, EnginePcap, true, time.Second, "", "http", "", "", 0)
-	defer input.Close()
-
-	// We will use it to get content of raw HTTP request
-	testOutput := NewTestOutput(func(data []byte) {
-		switch data[0] {
-		case RequestPayload:
-			if strings.Contains(string(data), "Expect: 100-continue") {
-				t.Error("Should not contain 100-continue header")
-			}
-			wg.Done()
-		case ResponsePayload:
-			wg.Done()
-		}
-	})
+	conf := RAWInputConfig{
+		Engine:        capture.EnginePcap,
+		Expire:        time.Second,
+		Protocol:      ProtocolHTTP,
+		TrackResponse: true,
+	}
+	input := NewRAWInput(originAddr, conf)
 
 	replay := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		defer r.Body.Close()
@@ -245,212 +251,100 @@ func TestInputRAW100Expect(t *testing.T) {
 	httpOutput := NewHTTPOutput(replay.URL, &HTTPOutputConfig{})
 
 	plugins := &InOutPlugins{
-		Inputs:  []io.Reader{input},
-		Outputs: []io.Writer{testOutput, httpOutput},
-	}
-	plugins.All = append(plugins.All, input, testOutput, httpOutput)
-
-	emitter := NewEmitter(quit)
-	go emitter.Start(plugins, Settings.Middleware)
-
-	// Origin + Response/Request Test Output + Request Http Output
-	wg.Add(4)
-	curl := exec.Command("curl", "http://"+originAddr, "--data-binary", "@COMM-LICENSE")
-	err := curl.Run()
-	if err != nil {
-		log.Fatal(err)
-	}
-
-	wg.Wait()
-	emitter.Close()
-}
-
-func TestInputRAWChunkedEncoding(t *testing.T) {
-	wg := new(sync.WaitGroup)
-	quit := make(chan int)
-
-	fileContent, _ := ioutil.ReadFile("README.md")
-
-	// Origing and Replay server initialization
-	origin := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		defer r.Body.Close()
-		ioutil.ReadAll(r.Body)
-
-		wg.Done()
-	}))
-
-	originAddr := strings.Replace(origin.Listener.Addr().String(), "[::]", "127.0.0.1", -1)
-	input := NewRAWInput(originAddr, EnginePcap, true, time.Second, "", "http", "", "", 0)
-	defer input.Close()
-
-	replay := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		defer r.Body.Close()
-		body, _ := ioutil.ReadAll(r.Body)
-
-		if !bytes.Equal(body, fileContent) {
-			buf, _ := httputil.DumpRequest(r, true)
-			t.Error("Wrong POST body:", string(buf))
-		}
-
-		wg.Done()
-	}))
-	defer replay.Close()
-
-	httpOutput := NewHTTPOutput(replay.URL, &HTTPOutputConfig{Debug: true})
-
-	plugins := &InOutPlugins{
-		Inputs:  []io.Reader{input},
-		Outputs: []io.Writer{httpOutput},
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{httpOutput},
 	}
 	plugins.All = append(plugins.All, input, httpOutput)
 
-	emitter := NewEmitter(quit)
+	emitter := NewEmitter()
+	defer emitter.Close()
 	go emitter.Start(plugins, Settings.Middleware)
 	wg.Add(2)
 
 	curl := exec.Command("curl", "http://"+originAddr, "--header", "Transfer-Encoding: chunked", "--header", "Expect:", "--data-binary", "@README.md")
 	err := curl.Run()
 	if err != nil {
-		log.Fatal(err)
-	}
-
-	wg.Wait()
-	emitter.Close()
-}
-
-func TestInputRAWLargePayload(t *testing.T) {
-	// FIXME: Large payloads does not work for travis for some reason...
-	if os.Getenv("TRAVIS_BUILD_DIR") != "" {
+		t.Error(err)
 		return
 	}
-	wg := new(sync.WaitGroup)
-	quit := make(chan int)
-	sizeB := 100 * 1000
 
-	// Generate 100kb file
-	dd := exec.Command("dd", "if=/dev/urandom", "of=/tmp/large", "bs=1", "count="+strconv.Itoa(sizeB))
-	err := dd.Run()
+	wg.Wait()
+}
+
+func BenchmarkRAWInputWithReplay(b *testing.B) {
+	var respCounter, reqCounter, replayCounter uint32
+	wg := &sync.WaitGroup{}
+
+	listener, err := net.Listen("tcp4", "127.0.0.1:0")
 	if err != nil {
-		log.Fatal("dd error:", err)
+		b.Error(err)
+		return
+	}
+	listener0, err := net.Listen("tcp4", "127.0.0.1:0")
+	if err != nil {
+		b.Error(err)
+		return
 	}
 
-	origin := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
-		defer req.Body.Close()
-		body, _ := ioutil.ReadAll(req.Body)
+	origin := http.Server{
+		Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			w.Write([]byte("ab"))
+		}),
+	}
+	go origin.Serve(listener)
+	defer origin.Close()
+	originAddr := listener.Addr().String()
 
-		if len(body) != sizeB {
-			t.Error("File size should be 1mb:", len(body))
-		}
-
-		wg.Done()
-	}))
-	originAddr := strings.Replace(origin.Listener.Addr().String(), "[::]", "127.0.0.1", -1)
-
-	input := NewRAWInput(originAddr, EnginePcap, true, testRawExpire, "", "http", "", "", 0)
-	defer input.Close()
-
-	replay := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
-		body, _ := ioutil.ReadAll(req.Body)
-		// // req.Body = http.MaxBytesReader(w, req.Body, 1*1024*1024)
-		// // buf := make([]byte, 1*1024*1024)
-		// n, _ := req.Body.Read(buf)
-		// body := buf[0:n]
-
-		if len(body) != sizeB {
-			t.Errorf("File size should be %d bytes: %d", sizeB, len(body))
-		}
-
-		wg.Done()
-	}))
+	replay := http.Server{
+		Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			atomic.AddUint32(&replayCounter, 1)
+			w.Write(nil)
+			wg.Done()
+		}),
+	}
+	go replay.Serve(listener0)
 	defer replay.Close()
+	replayAddr := listener0.Addr().String()
 
-	httpOutput := NewHTTPOutput(replay.URL, &HTTPOutputConfig{Debug: false})
+	conf := RAWInputConfig{
+		Engine:        capture.EnginePcap,
+		Expire:        testRawExpire,
+		Protocol:      ProtocolHTTP,
+		TrackResponse: true,
+	}
+	input := NewRAWInput(originAddr, conf)
+
+	testOutput := NewTestOutput(func(msg *Message) {
+		if msg.Meta[0] == '1' {
+			reqCounter++
+		} else {
+			respCounter++
+		}
+		wg.Done()
+	})
+	httpOutput := NewHTTPOutput("http://"+replayAddr, &HTTPOutputConfig{})
 
 	plugins := &InOutPlugins{
-		Inputs:  []io.Reader{input},
-		Outputs: []io.Writer{httpOutput},
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{testOutput, httpOutput},
 	}
-	plugins.All = append(plugins.All, input, httpOutput)
 
-	emitter := NewEmitter(quit)
+	emitter := NewEmitter()
 	go emitter.Start(plugins, Settings.Middleware)
-
-	wg.Add(2)
-	curl := exec.Command("curl", "http://"+originAddr, "--header", "Transfer-Encoding: chunked", "--header", "Expect:", "--data-binary", "@/tmp/large")
-	err = curl.Run()
-	if err != nil {
-		log.Fatal("curl error:", err)
+	addr := "http://" + originAddr
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		wg.Add(3) // reqCounter + replayCounter + respCounter
+		resp, err := http.Get(addr)
+		if err != nil {
+			wg.Add(-3)
+		}
+		resp.Body.Close()
 	}
 
 	wg.Wait()
-	emitter.Close()
-}
-
-func BenchmarkRAWInput(b *testing.B) {
-	var respCounter, reqCounter, replayCounter int64
-
-	quit := make(chan int)
-
-	origin := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {}))
-	defer origin.Close()
-	originAddr := strings.Replace(origin.Listener.Addr().String(), "[::]", "127.0.0.1", -1)
-
-	upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		atomic.AddInt64(&replayCounter, 1)
-	}))
-	defer origin.Close()
-	upstreamAddr := strings.Replace(upstream.Listener.Addr().String(), "[::]", "127.0.0.1", -1)
-
-	input := NewRAWInput(originAddr, EnginePcap, true, testRawExpire, "", "http", "", "", 0)
-	defer input.Close()
-
-	output := NewTestOutput(func(data []byte) {
-		if data[0] == '1' {
-			atomic.AddInt64(&reqCounter, 1)
-		} else {
-			atomic.AddInt64(&respCounter, 1)
-		}
-	})
-
-	httpOutput := NewLimiter(NewHTTPOutput(upstreamAddr, &HTTPOutputConfig{}), "10%")
-
-	plugins := &InOutPlugins{
-		Inputs:  []io.Reader{input},
-		Outputs: []io.Writer{output, httpOutput},
-	}
-	plugins.All = append(plugins.All, input, output, httpOutput)
-
-	emitter := NewEmitter(quit)
-	go emitter.Start(plugins, Settings.Middleware)
-
-	emitted := 0
-	fileContent, _ := ioutil.ReadFile("LICENSE.txt")
-
-	time.Sleep(400 * time.Millisecond)
-
-	for i := 0; i < b.N; i++ {
-		wg := new(sync.WaitGroup)
-		wg.Add(100 * 1000)
-		emitted += 100 * 1000
-		for w := 0; w < 100; w++ {
-			go func() {
-				client := NewHTTPClient(origin.URL, &HTTPClientConfig{})
-				for i := 0; i < 1000; i++ {
-					if i%2 == 0 {
-						client.Post("/", fileContent)
-					} else {
-						client.Get("/")
-					}
-					time.Sleep(time.Duration(rand.Int63n(50)) * time.Millisecond)
-					wg.Done()
-				}
-			}()
-		}
-		wg.Wait()
-	}
-
-	time.Sleep(400 * time.Millisecond)
-	log.Println("Emitted ", emitted, ", Captured ", reqCounter, "requests and ", respCounter, " responses", "and replayed", replayCounter)
-
+	b.ReportMetric(float64(reqCounter), "requests")
+	b.ReportMetric(float64(respCounter), "responses")
+	b.ReportMetric(float64(replayCounter), "replayed")
 	emitter.Close()
 }

input_tcp.go

@@ -8,18 +8,18 @@ import (
 	"io"
 	"log"
 	"net"
-	"os"
 )
 
 // TCPInput used for internal communication
 type TCPInput struct {
-	data     chan []byte
+	data     chan *Message
 	listener net.Listener
 	address  string
 	config   *TCPInputConfig
 	stop     chan bool // Channel used only to indicate goroutine should shutdown
 }
 
+// TCPInputConfig represents configuration of a TCP input plugin
 type TCPInputConfig struct {
 	Secure          bool   `json:"input-tcp-secure"`
 	CertificatePath string `json:"input-tcp-certificate"`
@@ -29,7 +29,7 @@ type TCPInputConfig struct {
 // NewTCPInput constructor for TCPInput, accepts address with port
 func NewTCPInput(address string, config *TCPInputConfig) (i *TCPInput) {
 	i = new(TCPInput)
-	i.data = make(chan []byte, 1000)
+	i.data = make(chan *Message, 1000)
 	i.address = address
 	i.config = config
 	i.stop = make(chan bool)
@@ -39,20 +39,21 @@ func NewTCPInput(address string, config *TCPInputConfig) (i *TCPInput) {
 	return
 }
 
-func (i *TCPInput) Read(data []byte) (int, error) {
-	var buf []byte
+// PluginRead returns data and details read from plugin
+func (i *TCPInput) PluginRead() (msg *Message, err error) {
 	select {
 	case <-i.stop:
-		return 0, ErrorStopped
-	case buf = <-i.data:
+		return nil, ErrorStopped
+	case msg = <-i.data:
+		return msg, nil
 	}
-	copy(data, buf)
 
-	return len(buf), nil
 }
 
+// Close closes the plugin
 func (i *TCPInput) Close() error {
 	close(i.stop)
+	i.listener.Close()
 	return nil
 }
 
@@ -60,64 +61,67 @@ func (i *TCPInput) listen(address string) {
 	if i.config.Secure {
 		cer, err := tls.LoadX509KeyPair(i.config.CertificatePath, i.config.KeyPath)
 		if err != nil {
-			log.Fatal("Error while loading --input-file certificate:", err)
+			log.Fatalln("error while loading --input-tcp TLS certificate:", err)
 		}
 
 		config := &tls.Config{Certificates: []tls.Certificate{cer}}
 		listener, err := tls.Listen("tcp", address, config)
 		if err != nil {
-			log.Fatal("Can't start --input-tcp with secure connection:", err)
+			log.Fatalln("[INPUT-TCP] failed to start INPUT-TCP listener:", err)
 		}
 		i.listener = listener
 	} else {
 		listener, err := net.Listen("tcp", address)
 		if err != nil {
-			log.Fatal("Can't start:", err)
+			log.Fatalln("failed to start INPUT-TCP listener:", err)
 		}
-
 		i.listener = listener
 	}
-
 	go func() {
 		for {
 			conn, err := i.listener.Accept()
-
-			if err != nil {
-				log.Println("Error while Accept()", err)
+			if err == nil {
+				go i.handleConnection(conn)
 				continue
 			}
-
-			go i.handleConnection(conn)
+			if isTemporaryNetworkError(err) {
+				continue
+			}
+			if operr, ok := err.(*net.OpError); ok && operr.Err.Error() != "use of closed network connection" {
+				Debug(0, fmt.Sprintf("[INPUT-TCP] listener closed, err: %q", err))
+			}
+			break
 		}
 	}()
 }
 
+var payloadSeparatorAsBytes = []byte(payloadSeparator)
+
 func (i *TCPInput) handleConnection(conn net.Conn) {
 	defer conn.Close()
 
-	payloadSeparatorAsBytes := []byte(payloadSeparator)
 	reader := bufio.NewReader(conn)
 	var buffer bytes.Buffer
 
 	for {
 		line, err := reader.ReadBytes('\n')
-
 		if err != nil {
+			if isTemporaryNetworkError(err) {
+				continue
+			}
 			if err != io.EOF {
-				fmt.Fprintln(os.Stderr, "Unexpected error in input tcp connection:", err)
+				Debug(0, fmt.Sprintf("[INPUT-TCP] connection error: %q", err))
 			}
 			break
-
 		}
 
 		if bytes.Equal(payloadSeparatorAsBytes[1:], line) {
-			asBytes := buffer.Bytes()
+			// unread the '\n' before monkeys
+			buffer.UnreadByte()
+			var msg Message
+			msg.Meta, msg.Data = payloadMetaWithBody(buffer.Bytes())
+			i.data <- &msg
 			buffer.Reset()
-
-			newBuf := make([]byte, len(asBytes)-1)
-			copy(newBuf, asBytes)
-
-			i.data <- newBuf
 		} else {
 			buffer.Write(line)
 		}
@@ -127,3 +131,13 @@ func (i *TCPInput) handleConnection(conn net.Conn) {
 func (i *TCPInput) String() string {
 	return "TCP input: " + i.address
 }
+
+func isTemporaryNetworkError(err error) bool {
+	if nerr, ok := err.(net.Error); ok && nerr.Temporary() {
+		return true
+	}
+	if operr, ok := err.(*net.OpError); ok && operr.Temporary() {
+		return true
+	}
+	return false
+}

input_tcp_test.go

@@ -7,7 +7,6 @@ import (
 	"crypto/tls"
 	"crypto/x509"
 	"encoding/pem"
-	"io"
 	"io/ioutil"
 	"log"
 	"math/big"
@@ -20,20 +19,19 @@ import (
 
 func TestTCPInput(t *testing.T) {
 	wg := new(sync.WaitGroup)
-	quit := make(chan int)
 
 	input := NewTCPInput("127.0.0.1:0", &TCPInputConfig{})
-	output := NewTestOutput(func(data []byte) {
+	output := NewTestOutput(func(*Message) {
 		wg.Done()
 	})
 
 	plugins := &InOutPlugins{
-		Inputs:  []io.Reader{input},
-		Outputs: []io.Writer{output},
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{output},
 	}
 	plugins.All = append(plugins.All, input, output)
 
-	emitter := NewEmitter(quit)
+	emitter := NewEmitter()
 	go emitter.Start(plugins, Settings.Middleware)
 
 	tcpAddr, err := net.ResolveTCPAddr("tcp", input.listener.Addr().String())
@@ -43,7 +41,6 @@ func TestTCPInput(t *testing.T) {
 	}
 
 	conn, err := net.DialTCP("tcp", nil, tcpAddr)
-
 	if err != nil {
 		log.Fatal(err)
 	}
@@ -52,10 +49,14 @@ func TestTCPInput(t *testing.T) {
 
 	for i := 0; i < 100; i++ {
 		wg.Add(1)
-		conn.Write(msg)
-		conn.Write([]byte(payloadSeparator))
+		if _, err = conn.Write(msg); err == nil {
+			_, err = conn.Write(payloadSeparatorAsBytes)
+		}
+		if err != nil {
+			t.Error(err)
+			return
+		}
 	}
-
 	wg.Wait()
 	emitter.Close()
 }
@@ -99,24 +100,23 @@ func TestTCPInputSecure(t *testing.T) {
 	}()
 
 	wg := new(sync.WaitGroup)
-	quit := make(chan int)
 
 	input := NewTCPInput("127.0.0.1:0", &TCPInputConfig{
 		Secure:          true,
 		CertificatePath: serverCertPemFile.Name(),
 		KeyPath:         serverPrivPemFile.Name(),
 	})
-	output := NewTestOutput(func(data []byte) {
+	output := NewTestOutput(func(*Message) {
 		wg.Done()
 	})
 
 	plugins := &InOutPlugins{
-		Inputs:  []io.Reader{input},
-		Outputs: []io.Writer{output},
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{output},
 	}
 	plugins.All = append(plugins.All, input, output)
 
-	emitter := NewEmitter(quit)
+	emitter := NewEmitter()
 	go emitter.Start(plugins, Settings.Middleware)
 
 	conf := &tls.Config{

kafka.go

@@ -2,15 +2,19 @@ package main
 
 import (
 	"bytes"
+	"crypto/tls"
+	"crypto/x509"
+	"errors"
 	"fmt"
+	"io/ioutil"
+	"log"
 
 	"github.com/Shopify/sarama"
 	"github.com/buger/goreplay/proto"
 )
 
-// KafkaConfig should contains required information to
+// InputKafkaConfig should contains required information to
 // build producers.
-
 type InputKafkaConfig struct {
 	producer sarama.AsyncProducer
 	consumer sarama.Consumer
@@ -19,6 +23,7 @@ type InputKafkaConfig struct {
 	UseJSON  bool   `json:"input-kafka-json-format"`
 }
 
+// OutputKafkaConfig is the representation of kfka output configuration
 type OutputKafkaConfig struct {
 	producer sarama.AsyncProducer
 	consumer sarama.Consumer
@@ -27,6 +32,13 @@ type OutputKafkaConfig struct {
 	UseJSON  bool   `json:"output-kafka-json-format"`
 }
 
+// KafkaTLSConfig should contains TLS certificates for connecting to secured Kafka clusters
+type KafkaTLSConfig struct {
+	CACert     string `json:"kafka-tls-ca-cert"`
+	ClientCert string `json:"kafka-tls-client-cert"`
+	ClientKey  string `json:"kafka-tls-client-key"`
+}
+
 // KafkaMessage should contains catched request information that should be
 // passed as Json to Apache Kafka.
 type KafkaMessage struct {
@@ -39,6 +51,53 @@ type KafkaMessage struct {
 	ReqHeaders map[string]string `json:"Req_Headers,omitempty"`
 }
 
+
+// NewTLSConfig loads TLS certificates
+func NewTLSConfig(clientCertFile, clientKeyFile, caCertFile string) (*tls.Config, error) {
+	tlsConfig := tls.Config{}
+
+	if clientCertFile != "" && clientKeyFile == "" {
+		return &tlsConfig, errors.New("Missing key of client certificate in kafka")
+	}
+	if clientCertFile == "" && clientKeyFile != "" {
+		return &tlsConfig, errors.New("missing TLS client certificate in kafka")
+	}
+	// Load client cert
+	if  (clientCertFile != "") && (clientKeyFile != "") {
+		cert, err := tls.LoadX509KeyPair(clientCertFile, clientKeyFile)
+		if err != nil {
+			return &tlsConfig, err
+		}
+		tlsConfig.Certificates = []tls.Certificate{cert}
+	}
+	// Load CA cert
+	if caCertFile != "" {
+		caCert, err := ioutil.ReadFile(caCertFile)
+		if err != nil {
+			return &tlsConfig, err
+		}
+		caCertPool := x509.NewCertPool()
+		caCertPool.AppendCertsFromPEM(caCert)
+		tlsConfig.RootCAs = caCertPool
+	}
+	return &tlsConfig, nil
+}
+
+// NewKafkaConfig returns Kafka config with or without TLS
+func NewKafkaConfig(tlsConfig *KafkaTLSConfig) *sarama.Config {
+	config := sarama.NewConfig()
+	// Configuration options go here
+	if tlsConfig != nil && (tlsConfig.ClientCert != "" || tlsConfig.CACert != "") {
+		config.Net.TLS.Enable = true
+		tlsConfig, err := NewTLSConfig(tlsConfig.ClientCert, tlsConfig.ClientKey, tlsConfig.CACert)
+		if err != nil {
+			log.Fatal(err)
+		}
+		config.Net.TLS.Config = tlsConfig
+	}
+	return config
+}
+
 // Dump returns the given request in its HTTP/1.x wire
 // representation.
 func (m KafkaMessage) Dump() ([]byte, error) {
@@ -46,13 +105,13 @@ func (m KafkaMessage) Dump() ([]byte, error) {
 
 	b.WriteString(fmt.Sprintf("%s %s %s\n", m.ReqType, m.ReqID, m.ReqTs))
 	b.WriteString(fmt.Sprintf("%s %s HTTP/1.1", m.ReqMethod, m.ReqURL))
-	b.Write(proto.CLRF)
+	b.Write(proto.CRLF)
 	for key, value := range m.ReqHeaders {
 		b.WriteString(fmt.Sprintf("%s: %s", key, value))
-		b.Write(proto.CLRF)
+		b.Write(proto.CRLF)
 	}
 
-	b.Write(proto.CLRF)
+	b.Write(proto.CRLF)
 	b.WriteString(m.ReqBody)
 
 	return b.Bytes(), nil

limiter.go

@@ -20,8 +20,8 @@ type Limiter struct {
 }
 
 func parseLimitOptions(options string) (limit int, isPercent bool) {
-	if strings.Contains(options, "%") {
-		limit, _ = strconv.Atoi(strings.Split(options, "%")[0])
+	if n := strings.Index(options, "%"); n > 0 {
+		limit, _ = strconv.Atoi(options[:n])
 		isPercent = true
 	} else {
 		limit, _ = strconv.Atoi(options)
@@ -33,7 +33,7 @@ func parseLimitOptions(options string) (limit int, isPercent bool) {
 
 // NewLimiter constructor for Limiter, accepts plugin and options
 // `options` allow to sprcify relatve or absolute limiting
-func NewLimiter(plugin interface{}, options string) io.ReadWriter {
+func NewLimiter(plugin interface{}, options string) PluginReadWriter {
 	l := new(Limiter)
 	l.limit, l.isPercent = parseLimitOptions(options)
 	l.plugin = plugin
@@ -71,24 +71,29 @@ func (l *Limiter) isLimited() bool {
 	return false
 }
 
-func (l *Limiter) Write(data []byte) (n int, err error) {
+// PluginWrite writes message to this plugin
+func (l *Limiter) PluginWrite(msg *Message) (n int, err error) {
 	if l.isLimited() {
 		return 0, nil
 	}
-
-	n, err = l.plugin.(io.Writer).Write(data)
-	return
+	if w, ok := l.plugin.(PluginWriter); ok {
+		return w.PluginWrite(msg)
+	}
+	// avoid further writing
+	return 0, io.ErrClosedPipe
 }
 
-func (l *Limiter) Read(data []byte) (n int, err error) {
-	if r, ok := l.plugin.(io.Reader); ok {
-		n, err = r.Read(data)
+// PluginRead reads message from this plugin
+func (l *Limiter) PluginRead() (msg *Message, err error) {
+	if r, ok := l.plugin.(PluginReader); ok {
+		msg, err = r.PluginRead()
 	} else {
-		return 0, nil
+		// avoid further reading
+		return nil, io.ErrClosedPipe
 	}
 
 	if l.isLimited() {
-		return 0, nil
+		return nil, nil
 	}
 
 	return
@@ -100,7 +105,7 @@ func (l *Limiter) String() string {
 
 // Close closes the resources.
 func (l *Limiter) Close() error {
-	if fi, ok := l.plugin.(io.ReadCloser); ok {
+	if fi, ok := l.plugin.(io.Closer); ok {
 		fi.Close()
 	}
 	return nil

limiter_test.go

@@ -3,28 +3,26 @@
 package main
 
 import (
-	"io"
 	"sync"
 	"testing"
 )
 
 func TestOutputLimiter(t *testing.T) {
 	wg := new(sync.WaitGroup)
-	quit := make(chan int)
 
 	input := NewTestInput()
-	output := NewLimiter(NewTestOutput(func(data []byte) {
+	output := NewLimiter(NewTestOutput(func(*Message) {
 		wg.Done()
 	}), "10")
 	wg.Add(10)
 
 	plugins := &InOutPlugins{
-		Inputs:  []io.Reader{input},
-		Outputs: []io.Writer{output},
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{output},
 	}
 	plugins.All = append(plugins.All, input, output)
 
-	emitter := NewEmitter(quit)
+	emitter := NewEmitter()
 	go emitter.Start(plugins, Settings.Middleware)
 
 	for i := 0; i < 100; i++ {
@@ -37,21 +35,20 @@ func TestOutputLimiter(t *testing.T) {
 
 func TestInputLimiter(t *testing.T) {
 	wg := new(sync.WaitGroup)
-	quit := make(chan int)
 
 	input := NewLimiter(NewTestInput(), "10")
-	output := NewTestOutput(func(data []byte) {
+	output := NewTestOutput(func(*Message) {
 		wg.Done()
 	})
 	wg.Add(10)
 
 	plugins := &InOutPlugins{
-		Inputs:  []io.Reader{input},
-		Outputs: []io.Writer{output},
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{output},
 	}
 	plugins.All = append(plugins.All, input, output)
 
-	emitter := NewEmitter(quit)
+	emitter := NewEmitter()
 	go emitter.Start(plugins, Settings.Middleware)
 
 	for i := 0; i < 100; i++ {
@@ -65,20 +62,19 @@ func TestInputLimiter(t *testing.T) {
 // Should limit all requests
 func TestPercentLimiter1(t *testing.T) {
 	wg := new(sync.WaitGroup)
-	quit := make(chan int)
 
 	input := NewTestInput()
-	output := NewLimiter(NewTestOutput(func(data []byte) {
+	output := NewLimiter(NewTestOutput(func(*Message) {
 		wg.Done()
 	}), "0%")
 
 	plugins := &InOutPlugins{
-		Inputs:  []io.Reader{input},
-		Outputs: []io.Writer{output},
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{output},
 	}
 	plugins.All = append(plugins.All, input, output)
 
-	emitter := NewEmitter(quit)
+	emitter := NewEmitter()
 	go emitter.Start(plugins, Settings.Middleware)
 
 	for i := 0; i < 100; i++ {
@@ -86,27 +82,25 @@ func TestPercentLimiter1(t *testing.T) {
 	}
 
 	wg.Wait()
-	emitter.Close()
 }
 
 // Should not limit at all
 func TestPercentLimiter2(t *testing.T) {
 	wg := new(sync.WaitGroup)
-	quit := make(chan int)
 
 	input := NewTestInput()
-	output := NewLimiter(NewTestOutput(func(data []byte) {
+	output := NewLimiter(NewTestOutput(func(*Message) {
 		wg.Done()
 	}), "100%")
 	wg.Add(100)
 
 	plugins := &InOutPlugins{
-		Inputs:  []io.Reader{input},
-		Outputs: []io.Writer{output},
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{output},
 	}
 	plugins.All = append(plugins.All, input, output)
 
-	emitter := NewEmitter(quit)
+	emitter := NewEmitter()
 	go emitter.Start(plugins, Settings.Middleware)
 
 	for i := 0; i < 100; i++ {
@@ -114,5 +108,4 @@ func TestPercentLimiter2(t *testing.T) {
 	}
 
 	wg.Wait()
-	emitter.Close()
 }

middleware.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"bufio"
+	"context"
 	"encoding/hex"
 	"fmt"
 	"io"
@@ -9,30 +10,29 @@ import (
 	"os"
 	"os/exec"
 	"strings"
-	"sync"
 )
 
+// Middleware represents a middleware object
 type Middleware struct {
-	command string
-
-	data chan []byte
-
-	mu sync.Mutex
-
-	Stdin  io.Writer
-	Stdout io.Reader
-
-	stop chan bool // Channel used only to indicate goroutine should shutdown
+	command       string
+	data          chan *Message
+	Stdin         io.Writer
+	Stdout        io.Reader
+	commandCancel context.CancelFunc
+	stop          chan bool // Channel used only to indicate goroutine should shutdown
 }
 
+// NewMiddleware returns new middleware
 func NewMiddleware(command string) *Middleware {
 	m := new(Middleware)
 	m.command = command
-	m.data = make(chan []byte, 1000)
+	m.data = make(chan *Message, 1000)
 	m.stop = make(chan bool)
 
 	commands := strings.Split(command, " ")
-	cmd := exec.Command(commands[0], commands[1:]...)
+	ctx, cancl := context.WithCancel(context.Background())
+	m.commandCancel = cancl
+	cmd := exec.CommandContext(ctx, commands[0], commands[1:]...)
 
 	m.Stdout, _ = cmd.StdoutPipe()
 	m.Stdin, _ = cmd.StdinPipe()
@@ -58,47 +58,33 @@ func NewMiddleware(command string) *Middleware {
 	return m
 }
 
-func (m *Middleware) ReadFrom(plugin io.Reader) {
-	Debug("[MIDDLEWARE-MASTER] Starting reading from", plugin)
+// ReadFrom start a worker to read from this plugin
+func (m *Middleware) ReadFrom(plugin PluginReader) {
+	Debug(2, "[MIDDLEWARE-MASTER] Starting reading from", plugin)
 	go m.copy(m.Stdin, plugin)
 }
 
-func (m *Middleware) copy(to io.Writer, from io.Reader) {
-	buf := make([]byte, 5*1024*1024)
-	dst := make([]byte, len(buf)*4)
+func (m *Middleware) copy(to io.Writer, from PluginReader) {
+	var buf, dst []byte
 
 	for {
-		nr, _ := from.Read(buf)
-		if nr == 0 || nr > len(buf) {
+		msg, err := from.PluginRead()
+		if err != nil {
+			return
+		}
+		if msg == nil || len(msg.Data) == 0 {
 			continue
 		}
 
-		payload := buf[0:nr]
-
 		if Settings.PrettifyHTTP {
-			payload = prettifyHTTP(payload)
-			nr = len(payload)
-
-			if nr*2 > len(dst) {
-				continue
-			}
+			buf = prettifyHTTP(msg.Data)
 		}
+		dst = make([]byte, len(buf)*2+1)
+		hex.Encode(dst, buf)
+		dst[len(buf)*2] = '\n'
 
-		if Settings.PrettifyHTTP {
-			payload = prettifyHTTP(payload)
-			nr = len(payload)
-		}
+		to.Write(dst)
 
-		hex.Encode(dst, payload)
-		dst[nr*2] = '\n'
-
-		m.mu.Lock()
-		to.Write(dst[0 : nr*2+1])
-		m.mu.Unlock()
-
-		if Settings.Debug {
-			Debug("[MIDDLEWARE-MASTER] Sending:", string(buf[0:nr]), "From:", from)
-		}
 	}
 }
 
@@ -116,42 +102,41 @@ func (m *Middleware) read(from io.Reader) {
 			}
 		}
 
-		buf := make([]byte, len(line)/2)
+		buf := make([]byte, len(line)/2-1)
 		if _, err := hex.Decode(buf, line[:len(line)-1]); err != nil {
-			fmt.Fprintln(os.Stderr, "Failed to decode input payload", err, len(line), string(line[:len(line)-1]))
+			Debug(0, fmt.Sprintf("[MIDDLEWARE] failed to decode err: %q", err))
+			continue
 		}
-
-		if Settings.Debug {
-			Debug("[MIDDLEWARE-MASTER] Received:", string(buf))
-		}
-
+		var msg Message
+		msg.Meta, msg.Data = payloadMetaWithBody(buf)
 		select {
 		case <-m.stop:
 			return
-		case m.data <- buf:
+		case m.data <- &msg:
 		}
 	}
 
 	return
 }
 
-func (m *Middleware) Read(data []byte) (int, error) {
-	var buf []byte
+// PluginRead reads message from this plugin
+func (m *Middleware) PluginRead() (msg *Message, err error) {
 	select {
 	case <-m.stop:
-		return 0, ErrorStopped
-	case buf = <-m.data:
+		return nil, ErrorStopped
+	case msg = <-m.data:
 	}
 
-	copy(data, buf)
-	return len(buf), nil
+	return
 }
 
 func (m *Middleware) String() string {
-	return fmt.Sprintf("Modifying traffic using '%s' command", m.command)
+	return fmt.Sprintf("Modifying traffic using %q command", m.command)
 }
 
+// Close closes this plugin
 func (m *Middleware) Close() error {
+	m.commandCancel()
 	close(m.stop)
 	return nil
 }

middleware_test.go

@@ -4,14 +4,15 @@ import (
 	"bytes"
 	"crypto/rand"
 	"encoding/hex"
-	"io"
 	"net/http"
 	"net/http/httptest"
+	"net/http/httputil"
 	"strings"
 	"sync"
 	"testing"
 	"time"
 
+	"github.com/buger/goreplay/capture"
 	"github.com/buger/goreplay/proto"
 )
 
@@ -19,43 +20,42 @@ type fakeServiceCb func(string, int, []byte)
 
 // Simple service that generate token on request, and require this token for accesing to secure area
 func NewFakeSecureService(wg *sync.WaitGroup, cb fakeServiceCb) *httptest.Server {
-	active_tokens := make([]string, 0)
+	activeTokens := make([]string, 0)
 	var mu sync.Mutex
 
 	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
 		mu.Lock()
 		defer mu.Unlock()
-		Debug("Received request: " + req.URL.String())
 
 		switch req.URL.Path {
 		case "/token":
 			// Generate random token
-			token_length := 10
-			buf := make([]byte, token_length)
+			tokenLength := 10
+			buf := make([]byte, tokenLength)
 			rand.Read(buf)
 			token := hex.EncodeToString(buf)
-			active_tokens = append(active_tokens, token)
+			activeTokens = append(activeTokens, token)
 
 			w.Write([]byte(token))
 
 			cb(req.URL.Path, 200, []byte(token))
 		case "/secure":
 			token := req.URL.Query().Get("token")
-			token_found := false
+			tokenFound := false
 
-			for _, t := range active_tokens {
+			for _, t := range activeTokens {
 				if t == token {
-					token_found = true
+					tokenFound = true
 					break
 				}
 			}
 
-			if token_found {
+			if tokenFound {
 				w.WriteHeader(http.StatusAccepted)
-				cb(req.URL.Path, 202, []byte(nil))
+				cb(req.URL.Path, 202, nil)
 			} else {
 				w.WriteHeader(http.StatusForbidden)
-				cb(req.URL.Path, 403, []byte(nil))
+				cb(req.URL.Path, 403, nil)
 			}
 		}
 
@@ -76,18 +76,21 @@ func TestFakeSecureService(t *testing.T) {
 
 	wg.Add(3)
 
-	client := NewHTTPClient(server.URL, &HTTPClientConfig{Debug: true})
-	resp, _ = client.Get("/token")
+	client := NewHTTPClient(&HTTPOutputConfig{}).Client
+	rep, _ := client.Get(server.URL + "/token")
+	resp, _ = httputil.DumpResponse(rep, true)
 	token = proto.Body(resp)
 
 	// Right token
-	resp, _ = client.Get("/secure?token=" + string(token))
+	rep, _ = client.Get(server.URL + "/secure?token=" + string(token))
+	resp, _ = httputil.DumpResponse(rep, true)
 	if !bytes.Equal(proto.Status(resp), []byte("202")) {
 		t.Error("Valid token should return status 202:", string(proto.Status(resp)))
 	}
 
 	// Wrong tokens forbidden
-	resp, _ = client.Get("/secure?token=wrong")
+	rep, _ = client.Get(server.URL + "/secure?token=wrong")
+	resp, _ = httputil.DumpResponse(rep, true)
 	if !bytes.Equal(proto.Status(resp), []byte("403")) {
 		t.Error("Wrong token should returns status 403:", string(proto.Status(resp)))
 	}
@@ -112,48 +115,46 @@ func TestEchoMiddleware(t *testing.T) {
 	}))
 	defer to.Close()
 
-	quit := make(chan int)
-
-	Settings.Middleware = "./examples/middleware/echo.sh"
-
 	// Catch traffic from one service
 	fromAddr := strings.Replace(from.Listener.Addr().String(), "[::]", "127.0.0.1", -1)
-	input := NewRAWInput(fromAddr, EnginePcap, true, testRawExpire, "", "http", "", "", 0)
-	defer input.Close()
+	conf := RAWInputConfig{
+		Engine:        capture.EnginePcap,
+		Expire:        testRawExpire,
+		Protocol:      ProtocolHTTP,
+		TrackResponse: true,
+	}
+	input := NewRAWInput(fromAddr, conf)
 
 	// And redirect to another
-	output := NewHTTPOutput(to.URL, &HTTPOutputConfig{Debug: false})
+	output := NewHTTPOutput(to.URL, &HTTPOutputConfig{})
 
 	plugins := &InOutPlugins{
-		Inputs:  []io.Reader{input},
-		Outputs: []io.Writer{output},
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{output},
 	}
 	plugins.All = append(plugins.All, input, output)
 
 	// Start Gor
-	emitter := NewEmitter(quit)
-	go emitter.Start(plugins, Settings.Middleware)
+	emitter := NewEmitter()
+	emitter.Start(plugins, "echo -n && GOR_TEST=true && ./examples/middleware/echo.sh")
 
 	// Wait till middleware initialization
 	time.Sleep(100 * time.Millisecond)
 
 	// Should receive 2 requests from original + 2 from replayed
-	client := NewHTTPClient(from.URL, &HTTPClientConfig{Debug: false})
+	client := NewHTTPClient(output.(*HTTPOutput).config).Client
 
 	for i := 0; i < 10; i++ {
-		wg.Add(4)
+		wg.Add(2)
 		// Request should be echoed
-		client.Get("/a")
+		client.Get(to.URL + "/a")
 		time.Sleep(5 * time.Millisecond)
-		client.Get("/b")
+		client.Get(to.URL + "/b")
 		time.Sleep(5 * time.Millisecond)
 	}
 
 	wg.Wait()
 	emitter.Close()
-	time.Sleep(200 * time.Millisecond)
-
-	Settings.Middleware = ""
 }
 
 func TestTokenMiddleware(t *testing.T) {
@@ -178,52 +179,48 @@ func TestTokenMiddleware(t *testing.T) {
 	})
 	defer to.Close()
 
-	quit := make(chan int)
-
-	Settings.Middleware = "go run ./examples/middleware/token_modifier.go"
+	Settings.Middleware = "echo -n && GOR_TEST=true && go run ./examples/middleware/token_modifier.go"
 
 	fromAddr := strings.Replace(from.Listener.Addr().String(), "[::]", "127.0.0.1", -1)
+	conf := RAWInputConfig{
+		Engine:        capture.EnginePcap,
+		Expire:        testRawExpire,
+		Protocol:      ProtocolHTTP,
+		TrackResponse: true,
+	}
 	// Catch traffic from one service
-	input := NewRAWInput(fromAddr, EnginePcap, true, testRawExpire, "", "http", "", "", 0)
-	defer input.Close()
+	input := NewRAWInput(fromAddr, conf)
 
 	// And redirect to another
-	output := NewHTTPOutput(to.URL, &HTTPOutputConfig{Debug: true})
+	output := NewHTTPOutput(to.URL, &HTTPOutputConfig{})
 
 	plugins := &InOutPlugins{
-		Inputs:  []io.Reader{input},
-		Outputs: []io.Writer{output},
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{output},
 	}
 	plugins.All = append(plugins.All, input, output)
 
 	// Start Gor
-	emitter := NewEmitter(quit)
-	go emitter.Start(plugins, Settings.Middleware)
-
-	// Wait for middleware to initialize
-	// Give go compiller time to build programm
-	time.Sleep(500 * time.Millisecond)
+	emitter := NewEmitter()
+	emitter.Start(plugins, Settings.Middleware)
 
 	// Should receive 2 requests from original + 2 from replayed
-	wg.Add(4)
+	wg.Add(2)
 
-	client := NewHTTPClient(from.URL, &HTTPClientConfig{Debug: true})
+	client := NewHTTPClient(&HTTPOutputConfig{}).Client
 
 	// Sending traffic to original service
-	resp, _ = client.Get("/token")
+	rep, _ := client.Get(to.URL + "/token")
+	resp, _ = httputil.DumpResponse(rep, true)
 	token = proto.Body(resp)
 
-	// When delay is too smal, middleware does not always rewrite requests in time
-	// Hopefuly client will have delay more then 100ms :)
-	time.Sleep(100 * time.Millisecond)
-
-	resp, _ = client.Get("/secure?token=" + string(token))
+	rep, _ = client.Get(to.URL + "/secure?token=" + string(token))
+	resp, _ = httputil.DumpResponse(rep, true)
 	if !bytes.Equal(proto.Status(resp), []byte("202")) {
 		t.Error("Valid token should return 202:", proto.Status(resp))
 	}
 
 	wg.Wait()
 	emitter.Close()
-	time.Sleep(100 * time.Millisecond)
 	Settings.Middleware = ""
 }

output_binary.go

@@ -1,16 +1,17 @@
 package main
 
 import (
-	"io"
 	"sync/atomic"
 	"time"
+
+	"github.com/buger/goreplay/size"
 )
 
 // BinaryOutputConfig struct for holding binary output configuration
 type BinaryOutputConfig struct {
 	Workers        int           `json:"output-binary-workers"`
 	Timeout        time.Duration `json:"output-binary-timeout"`
-	BufferSize     int           `json:"output-tcp-response-buffer"`
+	BufferSize     size.Size     `json:"output-tcp-response-buffer"`
 	Debug          bool          `json:"output-binary-debug"`
 	TrackResponses bool          `json:"output-binary-track-response"`
 }
@@ -23,30 +24,26 @@ type BinaryOutput struct {
 	// alignment. atomic.* functions crash on 32bit machines if operand is not
 	// aligned at 64bit. See https://github.com/golang/go/issues/599
 	activeWorkers int64
-
-	address string
-	queue   chan []byte
-
-	responses chan response
-
-	needWorker chan int
-
-	config *BinaryOutputConfig
-
-	queueStats *GorStat
+	address       string
+	queue         chan *Message
+	responses     chan response
+	needWorker    chan int
+	quit          chan struct{}
+	config        *BinaryOutputConfig
 }
 
 // NewBinaryOutput constructor for BinaryOutput
 // Initialize workers
-func NewBinaryOutput(address string, config *BinaryOutputConfig) io.Writer {
+func NewBinaryOutput(address string, config *BinaryOutputConfig) PluginReadWriter {
 	o := new(BinaryOutput)
 
 	o.address = address
 	o.config = config
 
-	o.queue = make(chan []byte, 1000)
+	o.queue = make(chan *Message, 1000)
 	o.responses = make(chan response, 1000)
 	o.needWorker = make(chan int, 1)
+	o.quit = make(chan struct{})
 
 	// Initial workers count
 	if o.config.Workers == 0 {
@@ -78,7 +75,7 @@ func (o *BinaryOutput) startWorker() {
 	client := NewTCPClient(o.address, &TCPClientConfig{
 		Debug:              o.config.Debug,
 		Timeout:            o.config.Timeout,
-		ResponseBufferSize: o.config.BufferSize,
+		ResponseBufferSize: int(o.config.BufferSize),
 	})
 
 	deathCount := 0
@@ -87,8 +84,8 @@ func (o *BinaryOutput) startWorker() {
 
 	for {
 		select {
-		case data := <-o.queue:
-			o.sendRequest(client, data)
+		case msg := <-o.queue:
+			o.sendRequest(client, msg)
 			deathCount = 0
 		case <-time.After(time.Millisecond * 100):
 			// When dynamic scaling enabled workers die after 2s of inactivity
@@ -111,15 +108,13 @@ func (o *BinaryOutput) startWorker() {
 	}
 }
 
-func (o *BinaryOutput) Write(data []byte) (n int, err error) {
-	if !isRequestPayload(data) {
-		return len(data), nil
+// PluginWrite writes a message tothis plugin
+func (o *BinaryOutput) PluginWrite(msg *Message) (n int, err error) {
+	if !isRequestPayload(msg.Meta) {
+		return len(msg.Data), nil
 	}
 
-	buf := make([]byte, len(data))
-	copy(buf, data)
-
-	o.queue <- buf
+	o.queue <- msg
 
 	if o.config.Workers == 0 {
 		workersCount := atomic.LoadInt64(&o.activeWorkers)
@@ -129,41 +124,37 @@ func (o *BinaryOutput) Write(data []byte) (n int, err error) {
 		}
 	}
 
-	return len(data), nil
+	return len(msg.Data) + len(msg.Meta), nil
 }
 
-func (o *BinaryOutput) Read(data []byte) (int, error) {
-	resp := <-o.responses
+// PluginRead reads a message from this plugin
+func (o *BinaryOutput) PluginRead() (*Message, error) {
+	var resp response
+	var msg Message
+	select {
+	case <-o.quit:
+		return nil, ErrorStopped
+	case resp = <-o.responses:
+	}
+	msg.Data = resp.payload
+	msg.Meta = payloadHeader(ReplayedResponsePayload, resp.uuid, resp.startedAt, resp.roundTripTime)
 
-	Debug("[OUTPUT-TCP] Received response:", string(resp.payload))
-
-	header := payloadHeader(ReplayedResponsePayload, resp.uuid, resp.startedAt, resp.roundTripTime)
-	copy(data[0:len(header)], header)
-	copy(data[len(header):], resp.payload)
-
-	return len(resp.payload) + len(header), nil
+	return &msg, nil
 }
 
-func (o *BinaryOutput) sendRequest(client *TCPClient, request []byte) {
-	meta := payloadMeta(request)
-	if len(meta) < 2 {
+func (o *BinaryOutput) sendRequest(client *TCPClient, msg *Message) {
+	if !isRequestPayload(msg.Meta) {
 		return
 	}
 
-	if !isRequestPayload(request) {
-		return
-	}
-
-	uuid := meta[1]
-
-	body := payloadBody(request)
+	uuid := payloadID(msg.Meta)
 
 	start := time.Now()
-	resp, err := client.Send(body)
+	resp, err := client.Send(msg.Data)
 	stop := time.Now()
 
 	if err != nil {
-		Debug("Request error:", err)
+		Debug(1, "Request error:", err)
 	}
 
 	if o.config.TrackResponses {
@@ -174,3 +165,9 @@ func (o *BinaryOutput) sendRequest(client *TCPClient, request []byte) {
 func (o *BinaryOutput) String() string {
 	return "Binary output: " + o.address
 }
+
+// Close closes this plugin for reading
+func (o *BinaryOutput) Close() error {
+	close(o.quit)
+	return nil
+}

output_dummy.go

@@ -1,7 +1,7 @@
 package main
 
 import (
-	"fmt"
+	"os"
 )
 
 // DummyOutput used for debugging, prints all incoming requests
@@ -15,10 +15,16 @@ func NewDummyOutput() (di *DummyOutput) {
 	return
 }
 
-func (i *DummyOutput) Write(data []byte) (int, error) {
-	fmt.Println(string(data))
-
-	return len(data), nil
+// PluginWrite writes message to this plugin
+func (i *DummyOutput) PluginWrite(msg *Message) (int, error) {
+	var n, nn int
+	var err error
+	n, err = os.Stdout.Write(msg.Meta)
+	nn, err = os.Stdout.Write(msg.Data)
+	n += nn
+	nn, err = os.Stdout.Write(payloadSeparatorAsBytes)
+	n += nn
+	return n, err
 }
 
 func (i *DummyOutput) String() string {

output_file.go

@@ -15,6 +15,8 @@ import (
 	"strings"
 	"sync"
 	"time"
+
+	"github.com/buger/goreplay/size"
 )
 
 var dateFileNameFuncs = map[string]func(*FileOutput) string{
@@ -32,11 +34,11 @@ var dateFileNameFuncs = map[string]func(*FileOutput) string{
 // FileOutputConfig ...
 type FileOutputConfig struct {
 	FlushInterval     time.Duration `json:"output-file-flush-interval"`
-	sizeLimit         int64
-	outputFileMaxSize int64
-	QueueLimit        int64  `json:"output-file-queue-limit"`
-	Append            bool   `json:"output-file-append"`
-	BufferPath        string `json:"output-file-buffer"`
+	SizeLimit         size.Size     `json:"output-file-size-limit"`
+	OutputFileMaxSize size.Size     `json:"output-file-max-size-limit"`
+	QueueLimit        int           `json:"output-file-queue-limit"`
+	Append            bool          `json:"output-file-append"`
+	BufferPath        string        `json:"output-file-buffer"`
 	onClose           func(string)
 }
 
@@ -46,14 +48,14 @@ type FileOutput struct {
 	pathTemplate   string
 	currentName    string
 	file           *os.File
-	queueLength    int64
+	QueueLength    int
 	chunkSize      int
 	writer         io.Writer
 	requestPerFile bool
 	currentID      []byte
 	payloadType    []byte
 	closed         bool
-	totalFileSize  int64
+	totalFileSize  size.Size
 
 	config *FileOutputConfig
 }
@@ -154,8 +156,8 @@ func (o *FileOutput) filename() string {
 		nextChunk := false
 
 		if o.currentName == "" ||
-			((o.config.QueueLimit > 0 && o.queueLength >= o.config.QueueLimit) ||
-				(o.config.sizeLimit > 0 && o.chunkSize >= int(o.config.sizeLimit))) {
+			((o.config.QueueLimit > 0 && o.QueueLength >= o.config.QueueLimit) ||
+				(o.config.SizeLimit > 0 && o.chunkSize >= int(o.config.SizeLimit))) {
 			nextChunk = true
 		}
 
@@ -193,10 +195,11 @@ func (o *FileOutput) updateName() {
 	o.Unlock()
 }
 
-func (o *FileOutput) Write(data []byte) (n int, err error) {
+// PluginWrite writes message to this plugin
+func (o *FileOutput) PluginWrite(msg *Message) (n int, err error) {
 	if o.requestPerFile {
 		o.Lock()
-		meta := payloadMeta(data)
+		meta := payloadMeta(msg.Meta)
 		o.currentID = meta[1]
 		o.payloadType = meta[0]
 		o.Unlock()
@@ -222,29 +225,31 @@ func (o *FileOutput) Write(data []byte) (n int, err error) {
 			log.Fatal(o, "Cannot open file %q. Error: %s", o.currentName, err)
 		}
 
-		o.queueLength = 0
+		o.QueueLength = 0
 	}
 
-	n, _ = o.writer.Write(data)
-	nSeparator, _ := o.writer.Write([]byte(payloadSeparator))
+	var nn int
+	n, err = o.writer.Write(msg.Meta)
+	nn, err = o.writer.Write(msg.Data)
+	n += nn
+	nn, err = o.writer.Write(payloadSeparatorAsBytes)
+	n += nn
 
-	n += nSeparator
+	o.totalFileSize += size.Size(n)
+	o.QueueLength++
 
-	o.totalFileSize += int64(n)
-	o.queueLength++
-
-	if Settings.OutputFileConfig.outputFileMaxSize > 0 && o.totalFileSize >= Settings.OutputFileConfig.outputFileMaxSize {
+	if Settings.OutputFileConfig.OutputFileMaxSize > 0 && o.totalFileSize >= Settings.OutputFileConfig.OutputFileMaxSize {
 		return n, errors.New("File output reached size limit")
 	}
 
-	return n, nil
+	return n, err
 }
 
 func (o *FileOutput) flush() {
 	// Don't exit on panic
 	defer func() {
 		if r := recover(); r != nil {
-			log.Println("PANIC while file flush: ", r, o, string(debug.Stack()))
+			Debug(0, "[OUTPUT-FILE] PANIC while file flush: ", r, o, string(debug.Stack()))
 		}
 	}()
 
@@ -261,7 +266,7 @@ func (o *FileOutput) flush() {
 		if stat, err := o.file.Stat(); err == nil {
 			o.chunkSize = int(stat.Size())
 		} else {
-			log.Println("Error accessing file sats", err)
+			Debug(0, "[OUTPUT-HTTP] error accessing file size", err)
 		}
 	}
 }

output_file_test.go

@@ -2,7 +2,6 @@ package main
 
 import (
 	"fmt"
-	"io"
 	"math/rand"
 	"os"
 	"reflect"
@@ -11,22 +10,23 @@ import (
 	"sync/atomic"
 	"testing"
 	"time"
+
+	"github.com/buger/goreplay/size"
 )
 
 func TestFileOutput(t *testing.T) {
 	wg := new(sync.WaitGroup)
-	quit := make(chan int)
 
 	input := NewTestInput()
 	output := NewFileOutput("/tmp/test_requests.gor", &FileOutputConfig{FlushInterval: time.Minute, Append: true})
 
 	plugins := &InOutPlugins{
-		Inputs:  []io.Reader{input},
-		Outputs: []io.Writer{output},
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{output},
 	}
 	plugins.All = append(plugins.All, input, output)
 
-	emitter := NewEmitter(quit)
+	emitter := NewEmitter()
 	go emitter.Start(plugins, Settings.Middleware)
 
 	for i := 0; i < 100; i++ {
@@ -40,19 +40,18 @@ func TestFileOutput(t *testing.T) {
 
 	var counter int64
 	input2 := NewFileInput("/tmp/test_requests.gor", false)
-	output2 := NewTestOutput(func(data []byte) {
+	output2 := NewTestOutput(func(*Message) {
 		atomic.AddInt64(&counter, 1)
 		wg.Done()
 	})
 
 	plugins2 := &InOutPlugins{
-		Inputs:  []io.Reader{input2},
-		Outputs: []io.Writer{output2},
+		Inputs:  []PluginReader{input2},
+		Outputs: []PluginWriter{output2},
 	}
 	plugins2.All = append(plugins2.All, input2, output2)
 
-	quit2 := make(chan int)
-	emitter2 := NewEmitter(quit2)
+	emitter2 := NewEmitter()
 	go emitter2.Start(plugins2, Settings.Middleware)
 
 	wg.Wait()
@@ -89,16 +88,16 @@ func TestFileOutputMultipleFiles(t *testing.T) {
 		t.Error("Should not initialize file if no writes")
 	}
 
-	output.Write([]byte("1 1 1\r\ntest"))
+	output.PluginWrite(&Message{Meta: []byte("1 1 1\r\n"), Data: []byte("test")})
 	name1 := output.file.Name()
 
-	output.Write([]byte("1 1 1\r\ntest"))
+	output.PluginWrite(&Message{Meta: []byte("1 1 1\r\n"), Data: []byte("test")})
 	name2 := output.file.Name()
 
 	time.Sleep(time.Second)
 	output.updateName()
 
-	output.Write([]byte("1 1 1\r\ntest"))
+	output.PluginWrite(&Message{Meta: []byte("1 1 1\r\n"), Data: []byte("test")})
 	name3 := output.file.Name()
 
 	if name2 != name1 {
@@ -120,16 +119,16 @@ func TestFileOutputFilePerRequest(t *testing.T) {
 		t.Error("Should not initialize file if no writes")
 	}
 
-	output.Write([]byte("1 1 1\ntest"))
+	output.PluginWrite(&Message{Meta: []byte("1 1 1\r\n"), Data: []byte("test")})
 	name1 := output.file.Name()
 
-	output.Write([]byte("1 2 1\ntest"))
+	output.PluginWrite(&Message{Meta: []byte("1 2 1\r\n"), Data: []byte("test")})
 	name2 := output.file.Name()
 
 	time.Sleep(time.Second)
 	output.updateName()
 
-	output.Write([]byte("1 3 1\ntest"))
+	output.PluginWrite(&Message{Meta: []byte("1 3 1\r\n"), Data: []byte("test")})
 	name3 := output.file.Name()
 
 	if name3 == name2 || name2 == name1 || name3 == name1 {
@@ -149,7 +148,7 @@ func TestFileOutputCompression(t *testing.T) {
 	}
 
 	for i := 0; i < 1000; i++ {
-		output.Write([]byte("1 1 1\r\ntest"))
+		output.PluginWrite(&Message{Meta: []byte("1 1 1\r\n"), Data: []byte("test")})
 	}
 
 	name := output.file.Name()
@@ -163,30 +162,6 @@ func TestFileOutputCompression(t *testing.T) {
 	os.Remove(name)
 }
 
-func TestParseDataUnit(t *testing.T) {
-	var d = map[string]int64{
-		"42mb":                 42 << 20,
-		"4_2":                  42,
-		"00":                   0,
-		"\n\n 0.0\r\t\f":       0,
-		"0_600tb":              384 << 40,
-		"0600Tb":               384 << 40,
-		"0o12Mb":               10 << 20,
-		"0b_10010001111_1kb":   2335 << 10,
-		"1024":                 1 << 10,
-		"0b111":                7,
-		"0x12gB":               18 << 30,
-		"0x_67_7a_2f_cc_40_c6": 113774485586118,
-		"121562380192901":      121562380192901,
-	}
-	for k, v := range d {
-		n, err := bufferParser(k, "0")
-		if err != nil || n != v {
-			t.Errorf("Error parsing %s: %v", k, err)
-		}
-	}
-}
-
 func TestGetFileIndex(t *testing.T) {
 	var tests = []struct {
 		path  string
@@ -232,15 +207,15 @@ func TestFileOutputAppendQueueLimitOverflow(t *testing.T) {
 
 	output := NewFileOutput(name, &FileOutputConfig{Append: false, FlushInterval: time.Minute, QueueLimit: 2})
 
-	output.Write([]byte("1 1 1\r\ntest"))
+	output.PluginWrite(&Message{Meta: []byte("1 1 1\r\n"), Data: []byte("test")})
 	name1 := output.file.Name()
 
-	output.Write([]byte("1 1 1\r\ntest"))
+	output.PluginWrite(&Message{Meta: []byte("1 1 1\r\n"), Data: []byte("test")})
 	name2 := output.file.Name()
 
 	output.updateName()
 
-	output.Write([]byte("1 1 1\r\ntest"))
+	output.PluginWrite(&Message{Meta: []byte("1 1 1\r\n"), Data: []byte("test")})
 	name3 := output.file.Name()
 
 	if name2 != name1 || name1 != fmt.Sprintf("/tmp/%d_0", rnd) {
@@ -261,15 +236,15 @@ func TestFileOutputAppendQueueLimitNoOverflow(t *testing.T) {
 
 	output := NewFileOutput(name, &FileOutputConfig{Append: false, FlushInterval: time.Minute, QueueLimit: 3})
 
-	output.Write([]byte("1 1 1\r\ntest"))
+	output.PluginWrite(&Message{Meta: []byte("1 1 1\r\n"), Data: []byte("test")})
 	name1 := output.file.Name()
 
-	output.Write([]byte("1 1 1\r\ntest"))
+	output.PluginWrite(&Message{Meta: []byte("1 1 1\r\n"), Data: []byte("test")})
 	name2 := output.file.Name()
 
 	output.updateName()
 
-	output.Write([]byte("1 1 1\r\ntest"))
+	output.PluginWrite(&Message{Meta: []byte("1 1 1\r\n"), Data: []byte("test")})
 	name3 := output.file.Name()
 
 	if name2 != name1 || name1 != fmt.Sprintf("/tmp/%d_0", rnd) {
@@ -290,15 +265,15 @@ func TestFileOutputAppendQueueLimitGzips(t *testing.T) {
 
 	output := NewFileOutput(name, &FileOutputConfig{Append: false, FlushInterval: time.Minute, QueueLimit: 2})
 
-	output.Write([]byte("1 1 1\r\ntest"))
+	output.PluginWrite(&Message{Meta: []byte("1 1 1\r\n"), Data: []byte("test")})
 	name1 := output.file.Name()
 
-	output.Write([]byte("1 1 1\r\ntest"))
+	output.PluginWrite(&Message{Meta: []byte("1 1 1\r\n"), Data: []byte("test")})
 	name2 := output.file.Name()
 
 	output.updateName()
 
-	output.Write([]byte("1 1 1\r\ntest"))
+	output.PluginWrite(&Message{Meta: []byte("1 1 1\r\n"), Data: []byte("test")})
 	name3 := output.file.Name()
 
 	if name2 != name1 || name1 != fmt.Sprintf("/tmp/%d_0.gz", rnd) {
@@ -331,18 +306,18 @@ func TestFileOutputAppendSizeLimitOverflow(t *testing.T) {
 
 	messageSize := len(message) + len(payloadSeparator)
 
-	output := NewFileOutput(name, &FileOutputConfig{Append: false, FlushInterval: time.Minute, sizeLimit: 2 * int64(messageSize)})
+	output := NewFileOutput(name, &FileOutputConfig{Append: false, FlushInterval: time.Minute, SizeLimit: size.Size(2 * messageSize)})
 
-	output.Write([]byte("1 1 1\r\ntest"))
+	output.PluginWrite(&Message{Meta: []byte("1 1 1\r\n"), Data: []byte("test")})
 	name1 := output.file.Name()
 
-	output.Write([]byte("1 1 1\r\ntest"))
+	output.PluginWrite(&Message{Meta: []byte("1 1 1\r\n"), Data: []byte("test")})
 	name2 := output.file.Name()
 
 	output.flush()
 	output.updateName()
 
-	output.Write([]byte("1 1 1\r\ntest"))
+	output.PluginWrite(&Message{Meta: []byte("1 1 1\r\n"), Data: []byte("test")})
 	name3 := output.file.Name()
 
 	if name2 != name1 || name1 != fmt.Sprintf("/tmp/%d_0", rnd) {

output_http.go

@@ -1,57 +1,26 @@
 package main
 
 import (
+	"bufio"
+	"bytes"
+	"crypto/tls"
 	"fmt"
-	"io"
 	"log"
+	"math"
+	"net/http"
+	"net/http/httputil"
+	"net/url"
 	"sync/atomic"
 	"time"
 
-	"github.com/buger/goreplay/proto"
+	"github.com/buger/goreplay/size"
 )
 
-var _ = fmt.Println
-
-const initialDynamicWorkers = 10
-
-type httpWorker struct {
-	output       *HTTPOutput
-	client       *HTTPClient
-	lastActivity time.Time
-	queue        chan []byte
-	stop         chan bool
-}
-
-func newHTTPWorker(output *HTTPOutput, queue chan []byte) *httpWorker {
-	client := NewHTTPClient(output.address, &HTTPClientConfig{
-		FollowRedirects:    output.config.RedirectLimit,
-		Debug:              output.config.Debug,
-		OriginalHost:       output.config.OriginalHost,
-		Timeout:            output.config.Timeout,
-		ResponseBufferSize: output.config.BufferSize,
-	})
-
-	w := &httpWorker{client: client}
-	if queue == nil {
-		w.queue = make(chan []byte, 100)
-	} else {
-		w.queue = queue
-	}
-	w.stop = make(chan bool)
-
-	go func() {
-		for {
-			select {
-			case payload := <-w.queue:
-				output.sendRequest(client, payload)
-			case <-w.stop:
-				return
-			}
-		}
-	}()
-
-	return w
-}
+const (
+	initialDynamicWorkers = 10
+	readChunkSize         = 64 * 1024
+	maxResponseSize       = 1073741824
+)
 
 type response struct {
 	payload       []byte
@@ -62,271 +31,279 @@ type response struct {
 
 // HTTPOutputConfig struct for holding http output configuration
 type HTTPOutputConfig struct {
-	RedirectLimit int `json:"output-http-redirect-limit"`
-
-	Stats      bool `json:"output-http-stats"`
-	WorkersMin int  `json:"output-http-workers-min"`
-	WorkersMax int  `json:"output-http-workers"`
-	StatsMs    int  `json:"output-http-stats-ms"`
-	Workers    int
-	QueueLen   int `json:"output-http-queue-len"`
-
-	ElasticSearch string `json:"output-http-elasticsearch"`
-
-	Timeout      time.Duration `json:"output-http-timeout"`
-	OriginalHost bool          `json:"output-http-original-host"`
-	BufferSize   int           `json:"output-http-response-buffer"`
-
-	CompatibilityMode bool `json:"output-http-compatibility-mode"`
-
-	RequestGroup string
-
-	Debug bool `json:"output-http-debug"`
-
-	TrackResponses bool `json:"output-http-track-response"`
+	TrackResponses bool          `json:"output-http-track-response"`
+	OriginalHost   bool          `json:"output-http-original-host"`
+	RedirectLimit  int           `json:"output-http-redirect-limit"`
+	WorkersMin     int           `json:"output-http-workers-min"`
+	WorkersMax     int           `json:"output-http-workers"`
+	QueueLen       int           `json:"output-http-queue-len"`
+	ElasticSearch  string        `json:"output-http-elasticsearch"`
+	Timeout        time.Duration `json:"output-http-timeout"`
+	WorkerTimeout  time.Duration `json:"output-http-worker-timeout"`
+	BufferSize     size.Size     `json:"output-http-response-buffer"`
+	SkipVerify     bool          `json:"output-http-skip-verify"`
+	rawURL         string
+	url            *url.URL
 }
 
 // HTTPOutput plugin manage pool of workers which send request to replayed server
-// By default workers pool is dynamic and starts with 10 workers
-// You can specify fixed number of workers using `--output-http-workers`
+// By default workers pool is dynamic and starts with 1 worker or workerMin workers
+// You can specify maximum number of workers using `--output-http-workers`
 type HTTPOutput struct {
-	// Keep this as first element of struct because it guarantees 64bit
-	// alignment. atomic.* functions crash on 32bit machines if operand is not
-	// aligned at 64bit. See https://github.com/golang/go/issues/599
-	activeWorkers int64
-
-	workerSessions map[string]*httpWorker
-
-	address string
-	limit   int
-	queue   chan []byte
-
-	responses chan response
-
-	needWorker chan int
-
-	config *HTTPOutputConfig
-
-	queueStats *GorStat
-
+	activeWorkers int32
+	config        *HTTPOutputConfig
 	elasticSearch *ESPlugin
-
-	stop chan bool // Channel used only to indicate goroutine should shutdown
+	client        *HTTPClient
+	stopWorker    chan struct{}
+	queue         chan *Message
+	responses     chan *response
+	stop          chan bool // Channel used only to indicate goroutine should shutdown
 }
 
 // NewHTTPOutput constructor for HTTPOutput
 // Initialize workers
-func NewHTTPOutput(address string, config *HTTPOutputConfig) io.Writer {
+func NewHTTPOutput(address string, config *HTTPOutputConfig) PluginReadWriter {
 	o := new(HTTPOutput)
-
-	o.address = address
+	var err error
+	config.url, err = url.Parse(address)
+	if err != nil {
+		log.Fatal(fmt.Sprintf("[OUTPUT-HTTP] parse HTTP output URL error[%q]", err))
+	}
+	if config.url.Scheme == "" {
+		config.url.Scheme = "http"
+	}
+	config.rawURL = config.url.String()
+	if config.Timeout < time.Millisecond*100 {
+		config.Timeout = time.Second
+	}
+	if config.BufferSize <= 0 {
+		config.BufferSize = 100 * 1024 // 100kb
+	}
+	if config.WorkersMin <= 0 {
+		config.WorkersMin = 1
+	}
+	if config.WorkersMin > 1000 {
+		config.WorkersMin = 1000
+	}
+	if config.WorkersMax <= 0 {
+		config.WorkersMax = math.MaxInt32 // idealy so large
+	}
+	if config.WorkersMax < config.WorkersMin {
+		config.WorkersMax = config.WorkersMin
+	}
+	if config.QueueLen <= 0 {
+		config.QueueLen = 1000
+	}
+	if config.RedirectLimit < 0 {
+		config.RedirectLimit = 0
+	}
+	if config.WorkerTimeout <= 0 {
+		config.WorkerTimeout = time.Second * 2
+	}
 	o.config = config
 	o.stop = make(chan bool)
 
-	if o.config.Stats {
-		o.queueStats = NewGorStat("output_http", o.config.StatsMs)
-	}
-
-	o.queue = make(chan []byte, o.config.QueueLen)
-	o.responses = make(chan response, o.config.QueueLen)
-	o.needWorker = make(chan int, 1)
-
-	// Initial workers count
-	if o.config.WorkersMax == 0 {
-		o.needWorker <- initialDynamicWorkers
-	} else {
-		o.needWorker <- o.config.WorkersMax
+	o.queue = make(chan *Message, o.config.QueueLen)
+	if o.config.TrackResponses {
+		o.responses = make(chan *response, o.config.QueueLen)
 	}
+	// it should not be buffered to avoid races
+	o.stopWorker = make(chan struct{})
 
 	if o.config.ElasticSearch != "" {
 		o.elasticSearch = new(ESPlugin)
 		o.elasticSearch.Init(o.config.ElasticSearch)
 	}
-
-	if Settings.RecognizeTCPSessions {
-		if !PRO {
-			log.Fatal("Detailed TCP sessions work only with PRO license")
-		}
-		o.workerSessions = make(map[string]*httpWorker, 100)
-		go o.sessionWorkerMaster()
-	} else {
-		go o.workerMaster()
+	o.client = NewHTTPClient(o.config)
+	o.activeWorkers += int32(o.config.WorkersMin)
+	for i := 0; i < o.config.WorkersMin; i++ {
+		go o.startWorker()
 	}
-
+	go o.workerMaster()
 	return o
 }
 
 func (o *HTTPOutput) workerMaster() {
-	for {
-		newWorkers := <-o.needWorker
-		atomic.AddInt64(&o.activeWorkers, int64(newWorkers))
-		for i := 0; i < newWorkers; i++ {
-			go o.startWorker()
-		}
-	}
-}
-
-func (o *HTTPOutput) sessionWorkerMaster() {
-	gc := time.Tick(time.Second)
-
-	for {
-		select {
-		case p := <-o.queue:
-			id := payloadID(p)
-			sessionID := string(id[0:20])
-			worker, ok := o.workerSessions[sessionID]
-
-			if !ok {
-				atomic.AddInt64(&o.activeWorkers, 1)
-				worker = newHTTPWorker(o, nil)
-				o.workerSessions[sessionID] = worker
-			}
-
-			worker.queue <- p
-			worker.lastActivity = time.Now()
-		case <-gc:
-			now := time.Now()
-
-			for id, w := range o.workerSessions {
-				if !w.lastActivity.IsZero() && now.Sub(w.lastActivity) >= 120*time.Second {
-					w.stop <- true
-					delete(o.workerSessions, id)
-					atomic.AddInt64(&o.activeWorkers, -1)
-				}
-			}
-		}
-	}
-}
-
-func (o *HTTPOutput) startWorker() {
-	client := NewHTTPClient(o.address, &HTTPClientConfig{
-		FollowRedirects:    o.config.RedirectLimit,
-		Debug:              o.config.Debug,
-		OriginalHost:       o.config.OriginalHost,
-		Timeout:            o.config.Timeout,
-		ResponseBufferSize: o.config.BufferSize,
-		CompatibilityMode:  o.config.CompatibilityMode,
-	})
-
+	var timer = time.NewTimer(o.config.WorkerTimeout)
+	defer func() {
+		// recover from panics caused by trying to send in
+		// a closed chan(o.stopWorker)
+		recover()
+	}()
+	defer timer.Stop()
 	for {
 		select {
 		case <-o.stop:
 			return
-		case data := <-o.queue:
-			o.sendRequest(client, data)
-		case <-time.After(2 * time.Second):
-			// When dynamic scaling enabled workers die after 2s of inactivity
-			if o.config.WorkersMin == o.config.WorkersMax {
-				continue
-			}
+		default:
+			<-timer.C
+		}
+		// rollback workers
+	rollback:
+		if atomic.LoadInt32(&o.activeWorkers) > int32(o.config.WorkersMin) && len(o.queue) < 1 {
+			// close one worker
+			o.stopWorker <- struct{}{}
+			atomic.AddInt32(&o.activeWorkers, -1)
+			goto rollback
+		}
+		timer.Reset(o.config.WorkerTimeout)
+	}
+}
 
-			workersCount := int(atomic.LoadInt64(&o.activeWorkers))
-
-			// At least 1 startWorker should be alive
-			if workersCount != 1 && workersCount > o.config.WorkersMin {
-				atomic.AddInt64(&o.activeWorkers, -1)
-				return
-			}
+func (o *HTTPOutput) startWorker() {
+	for {
+		select {
+		case <-o.stopWorker:
+			return
+		case msg := <-o.queue:
+			o.sendRequest(o.client, msg)
 		}
 	}
 }
 
-func (o *HTTPOutput) Write(data []byte) (n int, err error) {
-	if !isRequestPayload(data) {
-		return len(data), nil
+// PluginWrite writes message to this plugin
+func (o *HTTPOutput) PluginWrite(msg *Message) (n int, err error) {
+	if !isRequestPayload(msg.Meta) {
+		return len(msg.Data), nil
 	}
 
-	buf := make([]byte, len(data))
-	copy(buf, data)
-
 	select {
 	case <-o.stop:
 		return 0, ErrorStopped
-	case o.queue <- buf:
+	case o.queue <- msg:
 	}
 
-	if o.config.Stats {
-		o.queueStats.Write(len(o.queue))
-	}
-
-	if !Settings.RecognizeTCPSessions && o.config.WorkersMax != o.config.WorkersMin {
-		workersCount := int(atomic.LoadInt64(&o.activeWorkers))
-
-		if len(o.queue) > workersCount {
-			extraWorkersReq := len(o.queue) - workersCount + 1
-			maxWorkersAvailable := o.config.WorkersMax - workersCount
-			if extraWorkersReq > maxWorkersAvailable {
-				extraWorkersReq = maxWorkersAvailable
-			}
-			if extraWorkersReq > 0 {
-				o.needWorker <- extraWorkersReq
-			}
+	if len(o.queue) > 0 {
+		// try to start a new worker to serve
+		if atomic.LoadInt32(&o.activeWorkers) < int32(o.config.WorkersMax) {
+			go o.startWorker()
+			atomic.AddInt32(&o.activeWorkers, 1)
 		}
 	}
-
-	return len(data), nil
+	return len(msg.Data) + len(msg.Meta), nil
 }
 
-func (o *HTTPOutput) Read(data []byte) (int, error) {
-	var resp response
+// PluginRead reads message from this plugin
+func (o *HTTPOutput) PluginRead() (*Message, error) {
+	if !o.config.TrackResponses {
+		return nil, ErrorStopped
+	}
+	var resp *response
+	var msg Message
 	select {
 	case <-o.stop:
-		return 0, ErrorStopped
+		return nil, ErrorStopped
 	case resp = <-o.responses:
+		msg.Data = resp.payload
 	}
 
-	if Settings.Debug {
-		Debug("[OUTPUT-HTTP] Received response:", string(resp.payload))
-	}
+	msg.Meta = payloadHeader(ReplayedResponsePayload, resp.uuid, resp.roundTripTime, resp.startedAt)
 
-	header := payloadHeader(ReplayedResponsePayload, resp.uuid, resp.roundTripTime, resp.startedAt)
-	copy(data[0:len(header)], header)
-	copy(data[len(header):], resp.payload)
-
-	return len(resp.payload) + len(header), nil
+	return &msg, nil
 }
 
-func (o *HTTPOutput) sendRequest(client *HTTPClient, request []byte) {
-	meta := payloadMeta(request)
-
-	if Settings.Debug {
-		Debug(meta)
-	}
-
-	if len(meta) < 2 {
+func (o *HTTPOutput) sendRequest(client *HTTPClient, msg *Message) {
+	if !isRequestPayload(msg.Meta) {
 		return
 	}
-	uuid := meta[1]
-
-	body := payloadBody(request)
-	if !proto.IsHTTPPayload(body) {
-		return
-	}
-
+	uuid := payloadID(msg.Meta)
 	start := time.Now()
-	resp, err := client.Send(body)
+	resp, err := client.Send(msg.Data)
 	stop := time.Now()
 
 	if err != nil {
-		log.Println("Error when sending ", err, time.Now())
-		Debug("Request error:", err)
+		Debug(1, fmt.Sprintf("[HTTP-OUTPUT] error when sending: %q", err))
+		return
+	}
+	if resp == nil {
+		return
 	}
 
 	if o.config.TrackResponses {
-		o.responses <- response{resp, uuid, start.UnixNano(), stop.UnixNano() - start.UnixNano()}
+		o.responses <- &response{resp, uuid, start.UnixNano(), stop.UnixNano() - start.UnixNano()}
 	}
 
 	if o.elasticSearch != nil {
-		o.elasticSearch.ResponseAnalyze(request, resp, start, stop)
+		o.elasticSearch.ResponseAnalyze(msg.Data, resp, start, stop)
 	}
 }
 
 func (o *HTTPOutput) String() string {
-	return "HTTP output: " + o.address
+	return "HTTP output: " + o.config.rawURL
 }
 
 // Close closes the data channel so that data
 func (o *HTTPOutput) Close() error {
 	close(o.stop)
+	close(o.stopWorker)
 	return nil
 }
+
+// HTTPClient holds configurations for a single HTTP client
+type HTTPClient struct {
+	config *HTTPOutputConfig
+	Client *http.Client
+}
+
+// NewHTTPClient returns new http client with check redirects policy
+func NewHTTPClient(config *HTTPOutputConfig) *HTTPClient {
+	client := new(HTTPClient)
+	client.config = config
+	var transport *http.Transport
+	client.Client = &http.Client{
+		Timeout: client.config.Timeout,
+		CheckRedirect: func(req *http.Request, via []*http.Request) error {
+			if len(via) >= client.config.RedirectLimit {
+				Debug(1, fmt.Sprintf("[HTTPCLIENT] maximum output-http-redirects[%d] reached!", client.config.RedirectLimit))
+				return http.ErrUseLastResponse
+			}
+			lastReq := via[len(via)-1]
+			resp := req.Response
+			Debug(2, fmt.Sprintf("[HTTPCLIENT] HTTP redirects from %q to %q with %q", lastReq.Host, req.Host, resp.Status))
+			return nil
+		},
+	}
+	if config.SkipVerify {
+		// clone to avoid modying global default RoundTripper
+		transport = http.DefaultTransport.(*http.Transport).Clone()
+		transport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true}
+		client.Client.Transport = transport
+	}
+
+	return client
+}
+
+// Send sends an http request using client create by NewHTTPClient
+func (c *HTTPClient) Send(data []byte) ([]byte, error) {
+	var req *http.Request
+	var resp *http.Response
+	var err error
+
+	req, err = http.ReadRequest(bufio.NewReader(bytes.NewReader(data)))
+	if err != nil {
+		return nil, err
+	}
+	// we don't send CONNECT or OPTIONS request
+	if req.Method == http.MethodConnect {
+		return nil, nil
+	}
+
+	if !c.config.OriginalHost {
+		req.Host = c.config.url.Host
+	}
+
+	req.URL = c.config.url
+	// force connection to not be closed, which can affect the global client
+	req.Close = false
+	// it's an error if this is not equal to empty string
+	req.RequestURI = ""
+
+	resp, err = c.Client.Do(req)
+	if err != nil {
+		return nil, err
+	}
+	if c.config.TrackResponses {
+		return httputil.DumpResponse(resp, true)
+	}
+	return nil, nil
+}

output_http_test.go

@@ -1,20 +1,16 @@
 package main
 
 import (
-	"io"
 	"io/ioutil"
 	"net/http"
 	"net/http/httptest"
 	_ "net/http/httputil"
 	"sync"
-	"sync/atomic"
 	"testing"
-	"time"
 )
 
 func TestHTTPOutput(t *testing.T) {
 	wg := new(sync.WaitGroup)
-	quit := make(chan int)
 
 	input := NewTestInput()
 
@@ -40,47 +36,40 @@ func TestHTTPOutput(t *testing.T) {
 	}))
 	defer server.Close()
 
-	headers := HTTPHeaders{HTTPHeader{"User-Agent", "Gor"}}
+	headers := HTTPHeaders{httpHeader{"User-Agent", "Gor"}}
 	methods := HTTPMethods{[]byte("GET"), []byte("PUT"), []byte("POST")}
 	Settings.ModifierConfig = HTTPModifierConfig{Headers: headers, Methods: methods}
 
-	http_output := NewHTTPOutput(server.URL, &HTTPOutputConfig{Debug: true, TrackResponses: true})
-	output := NewTestOutput(func(data []byte) {
+	httpOutput := NewHTTPOutput(server.URL, &HTTPOutputConfig{TrackResponses: false})
+	output := NewTestOutput(func(*Message) {
 		wg.Done()
 	})
 
 	plugins := &InOutPlugins{
-		Inputs:  []io.Reader{input},
-		Outputs: []io.Writer{http_output, output},
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{httpOutput, output},
 	}
-	plugins.All = append(plugins.All, input, output, http_output)
+	plugins.All = append(plugins.All, input, output, httpOutput)
 
-	emitter := NewEmitter(quit)
+	emitter := NewEmitter()
 	go emitter.Start(plugins, Settings.Middleware)
 
 	for i := 0; i < 10; i++ {
-		// 2 http-output, 2 - test output request, 2 - test output http response
-		wg.Add(6) // OPTIONS should be ignored
+		// 2 http-output, 2 - test output request
+		wg.Add(4) // OPTIONS should be ignored
 		input.EmitPOST()
 		input.EmitOPTIONS()
 		input.EmitGET()
 	}
 
 	wg.Wait()
-	close(quit)
-
-	activeWorkers := atomic.LoadInt64(&http_output.(*HTTPOutput).activeWorkers)
-
-	if activeWorkers < 50 {
-		t.Error("Should create workers for each request", activeWorkers)
-	}
+	emitter.Close()
 
 	Settings.ModifierConfig = HTTPModifierConfig{}
 }
 
 func TestHTTPOutputKeepOriginalHost(t *testing.T) {
 	wg := new(sync.WaitGroup)
-	quit := make(chan int)
 
 	input := NewTestInput()
 
@@ -93,18 +82,18 @@ func TestHTTPOutputKeepOriginalHost(t *testing.T) {
 	}))
 	defer server.Close()
 
-	headers := HTTPHeaders{HTTPHeader{"Host", "custom-host.com"}}
+	headers := HTTPHeaders{httpHeader{"Host", "custom-host.com"}}
 	Settings.ModifierConfig = HTTPModifierConfig{Headers: headers}
 
-	output := NewHTTPOutput(server.URL, &HTTPOutputConfig{Debug: false, OriginalHost: true})
+	output := NewHTTPOutput(server.URL, &HTTPOutputConfig{OriginalHost: true, SkipVerify: true})
 
 	plugins := &InOutPlugins{
-		Inputs:  []io.Reader{input},
-		Outputs: []io.Writer{output},
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{output},
 	}
 	plugins.All = append(plugins.All, input, output)
 
-	emitter := NewEmitter(quit)
+	emitter := NewEmitter()
 	go emitter.Start(plugins, Settings.Middleware)
 
 	wg.Add(1)
@@ -117,7 +106,6 @@ func TestHTTPOutputKeepOriginalHost(t *testing.T) {
 
 func TestHTTPOutputSSL(t *testing.T) {
 	wg := new(sync.WaitGroup)
-	quit := make(chan int)
 
 	// Origing and Replay server initialization
 	server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
@@ -125,15 +113,15 @@ func TestHTTPOutputSSL(t *testing.T) {
 	}))
 
 	input := NewTestInput()
-	output := NewHTTPOutput(server.URL, &HTTPOutputConfig{})
+	output := NewHTTPOutput(server.URL, &HTTPOutputConfig{SkipVerify: true})
 
 	plugins := &InOutPlugins{
-		Inputs:  []io.Reader{input},
-		Outputs: []io.Writer{output},
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{output},
 	}
 	plugins.All = append(plugins.All, input, output)
 
-	emitter := NewEmitter(quit)
+	emitter := NewEmitter()
 	go emitter.Start(plugins, Settings.Middleware)
 
 	wg.Add(2)
@@ -147,7 +135,6 @@ func TestHTTPOutputSSL(t *testing.T) {
 
 func TestHTTPOutputSessions(t *testing.T) {
 	wg := new(sync.WaitGroup)
-	quit := make(chan int)
 
 	input := NewTestInput()
 	input.skipHeader = true
@@ -158,25 +145,27 @@ func TestHTTPOutputSessions(t *testing.T) {
 	defer server.Close()
 
 	Settings.RecognizeTCPSessions = true
-	output := NewHTTPOutput(server.URL, &HTTPOutputConfig{Debug: true})
+	Settings.SplitOutput = true
+	output := NewHTTPOutput(server.URL, &HTTPOutputConfig{})
 
 	plugins := &InOutPlugins{
-		Inputs:  []io.Reader{input},
-		Outputs: []io.Writer{output},
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{output},
 	}
-	emitter := NewEmitter(quit)
+	plugins.All = append(plugins.All, input, output)
+	emitter := NewEmitter()
 	go emitter.Start(plugins, Settings.Middleware)
 
 	uuid1 := []byte("1234567890123456789a0000")
 	uuid2 := []byte("1234567890123456789d0000")
 
-	for i := 0; i < 100; i++ {
+	for i := 0; i < 10; i++ {
 		wg.Add(1) // OPTIONS should be ignored
 		copy(uuid1[20:], randByte(4))
 		input.EmitBytes([]byte("1 " + string(uuid1) + " 1\n" + "GET / HTTP/1.1\r\n\r\n"))
 	}
 
-	for i := 0; i < 100; i++ {
+	for i := 0; i < 10; i++ {
 		wg.Add(1) // OPTIONS should be ignored
 		copy(uuid2[20:], randByte(4))
 		input.EmitBytes([]byte("1 " + string(uuid2) + " 1\n" + "GET / HTTP/1.1\r\n\r\n"))
@@ -184,35 +173,59 @@ func TestHTTPOutputSessions(t *testing.T) {
 
 	wg.Wait()
 
-	if output.(*HTTPOutput).activeWorkers != 2 {
-		t.Error("Should have only 2 workers", output.(*HTTPOutput).activeWorkers)
-	}
-
 	emitter.Close()
 
 	Settings.RecognizeTCPSessions = false
+	Settings.SplitOutput = false
 }
 
 func BenchmarkHTTPOutput(b *testing.B) {
 	wg := new(sync.WaitGroup)
-	quit := make(chan int)
 
-	server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		time.Sleep(50 * time.Millisecond)
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		wg.Done()
 	}))
 	defer server.Close()
 
 	input := NewTestInput()
-	output := NewHTTPOutput(server.URL, &HTTPOutputConfig{})
+	output := NewHTTPOutput(server.URL, &HTTPOutputConfig{WorkersMax: 1})
 
 	plugins := &InOutPlugins{
-		Inputs:  []io.Reader{input},
-		Outputs: []io.Writer{output},
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{output},
 	}
 	plugins.All = append(plugins.All, input, output)
 
-	emitter := NewEmitter(quit)
+	emitter := NewEmitter()
+	go emitter.Start(plugins, Settings.Middleware)
+
+	for i := 0; i < b.N; i++ {
+		wg.Add(1)
+		input.EmitPOST()
+	}
+
+	wg.Wait()
+	emitter.Close()
+}
+
+func BenchmarkHTTPOutputTLS(b *testing.B) {
+	wg := new(sync.WaitGroup)
+
+	server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		wg.Done()
+	}))
+	defer server.Close()
+
+	input := NewTestInput()
+	output := NewHTTPOutput(server.URL, &HTTPOutputConfig{SkipVerify: true, WorkersMax: 1})
+
+	plugins := &InOutPlugins{
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{output},
+	}
+	plugins.All = append(plugins.All, input, output)
+
+	emitter := NewEmitter()
 	go emitter.Start(plugins, Settings.Middleware)
 
 	for i := 0; i < b.N; i++ {

output_kafka.go

@@ -2,11 +2,11 @@ package main
 
 import (
 	"encoding/json"
-	"io"
 	"log"
 	"strings"
 	"time"
 
+	"github.com/buger/goreplay/byteutils"
 	"github.com/buger/goreplay/proto"
 
 	"github.com/Shopify/sarama"
@@ -22,9 +22,9 @@ type KafkaOutput struct {
 // KafkaOutputFrequency in milliseconds
 const KafkaOutputFrequency = 500
 
-// NewKafkaOutput creates instance of kafka producer client.
-func NewKafkaOutput(address string, config *OutputKafkaConfig) io.Writer {
-	c := sarama.NewConfig()
+// NewKafkaOutput creates instance of kafka producer client  with TLS config
+func NewKafkaOutput(address string, config *OutputKafkaConfig, tlsConfig *KafkaTLSConfig) PluginWriter {
+	c := NewKafkaConfig(tlsConfig)
 
 	var producer sarama.AsyncProducer
 
@@ -49,10 +49,8 @@ func NewKafkaOutput(address string, config *OutputKafkaConfig) io.Writer {
 		producer: producer,
 	}
 
-	if Settings.Verbose {
-		// Start infinite loop for tracking errors for kafka producer.
-		go o.ErrorHandler()
-	}
+	// Start infinite loop for tracking errors for kafka producer.
+	go o.ErrorHandler()
 
 	return o
 }
@@ -60,36 +58,37 @@ func NewKafkaOutput(address string, config *OutputKafkaConfig) io.Writer {
 // ErrorHandler should receive errors
 func (o *KafkaOutput) ErrorHandler() {
 	for err := range o.producer.Errors() {
-		log.Println("Failed to write access log entry:", err)
+		Debug(1, "Failed to write access log entry:", err)
 	}
 }
 
-func (o *KafkaOutput) Write(data []byte) (n int, err error) {
+// PluginWrite writes a message to this plugin
+func (o *KafkaOutput) PluginWrite(msg *Message) (n int, err error) {
 	var message sarama.StringEncoder
 
 	if !o.config.UseJSON {
-		message = sarama.StringEncoder(data)
+		message = sarama.StringEncoder(byteutils.SliceToString(msg.Meta) + byteutils.SliceToString(msg.Data))
 	} else {
-		headers := make(map[string]string)
-		proto.ParseHeaders([][]byte{data}, func(header []byte, value []byte) bool {
-			headers[string(header)] = string(value)
-			return true
-		})
+		mimeHeader := proto.ParseHeaders(msg.Data)
+		var header map[string]string
+		for k, v := range mimeHeader {
+			header[k] = strings.Join(v, ", ")
+		}
 
-		meta := payloadMeta(data)
-		req := payloadBody(data)
+		meta := payloadMeta(msg.Meta)
+		req := msg.Data
 
 		kafkaMessage := KafkaMessage{
-			ReqURL:     string(proto.Path(req)),
-			ReqType:    string(meta[0]),
-			ReqID:      string(meta[1]),
-			ReqTs:      string(meta[2]),
-			ReqMethod:  string(proto.Method(req)),
-			ReqBody:    string(proto.Body(req)),
-			ReqHeaders: headers,
+			ReqURL:     byteutils.SliceToString(proto.Path(req)),
+			ReqType:    byteutils.SliceToString(meta[0]),
+			ReqID:      byteutils.SliceToString(meta[1]),
+			ReqTs:      byteutils.SliceToString(meta[2]),
+			ReqMethod:  byteutils.SliceToString(proto.Method(req)),
+			ReqBody:    byteutils.SliceToString(proto.Body(req)),
+			ReqHeaders: header,
 		}
 		jsonMessage, _ := json.Marshal(&kafkaMessage)
-		message = sarama.StringEncoder(jsonMessage)
+		message = sarama.StringEncoder(byteutils.SliceToString(jsonMessage))
 	}
 
 	o.producer.Input() <- &sarama.ProducerMessage{

output_kafka_test.go

@@ -17,16 +17,16 @@ func TestOutputKafkaRAW(t *testing.T) {
 		producer: producer,
 		Topic:    "test",
 		UseJSON:  false,
-	})
+	}, nil)
 
-	output.Write([]byte("1 2 3\nGET / HTTP1.1\r\nHeader: 1\r\n\r\n"))
+	output.PluginWrite(&Message{Meta: []byte("1 2 3\n"), Data: []byte("GET / HTTP1.1\r\nHeader: 1\r\n\r\n")})
 
 	resp := <-producer.Successes()
 
 	data, _ := resp.Value.Encode()
 
 	if string(data) != "1 2 3\nGET / HTTP1.1\r\nHeader: 1\r\n\r\n" {
-		t.Error("Message not properly encoded: ", string(data))
+		t.Errorf("Message not properly encoded: %q", data)
 	}
 }
 
@@ -40,15 +40,15 @@ func TestOutputKafkaJSON(t *testing.T) {
 		producer: producer,
 		Topic:    "test",
 		UseJSON:  true,
-	})
+	}, nil)
 
-	output.Write([]byte("1 2 3\nGET / HTTP1.1\r\nHeader: 1\r\n\r\n"))
+	output.PluginWrite(&Message{Meta: []byte("1 2 3\n"), Data: []byte("GET / HTTP1.1\r\nHeader: 1\r\n\r\n")})
 
 	resp := <-producer.Successes()
 
 	data, _ := resp.Value.Encode()
 
-	if string(data) != `{"Req_URL":"/","Req_Type":"1","Req_ID":"2","Req_Ts":"3","Req_Method":"GET","Req_Headers":{"Header":"1"}}` {
+	if string(data) != `{"Req_URL":"","Req_Type":"1","Req_ID":"2","Req_Ts":"3","Req_Method":"GET"}` {
 		t.Error("Message not properly encoded: ", string(data))
 	}
 }

output_null.go

@@ -9,8 +9,9 @@ func NewNullOutput() (o *NullOutput) {
 	return new(NullOutput)
 }
 
-func (o *NullOutput) Write(data []byte) (int, error) {
-	return len(data), nil
+// PluginWrite writes message to this plugin
+func (o *NullOutput) PluginWrite(msg *Message) (int, error) {
+	return len(msg.Data) + len(msg.Meta), nil
 }
 
 func (o *NullOutput) String() string {

output_s3.go

@@ -67,8 +67,9 @@ func (o *S3Output) connect() {
 	}
 }
 
-func (o *S3Output) Write(data []byte) (n int, err error) {
-	return o.buffer.Write(data)
+// PluginWrite writes message to this plugin
+func (o *S3Output) PluginWrite(msg *Message) (n int, err error) {
+	return o.buffer.PluginWrite(msg)
 }
 
 func (o *S3Output) String() string {
@@ -107,22 +108,23 @@ func (o *S3Output) onBufferUpdate(path string) {
 	idx := getFileIndex(path)
 	bucket, key := o.keyPath(idx)
 
-	file, _ := os.Open(path)
-	// reader := bufio.NewReader(file)
+	file, err := os.Open(path)
+	if err != nil {
+		Debug(0, fmt.Sprintf("[S3 Output] Failed to open file %q. err: %q", path, err))
+		return
+	}
+	defer os.Remove(path)
 
-	_, err := svc.PutObject(&s3.PutObjectInput{
+	_, err = svc.PutObject(&s3.PutObjectInput{
 		Body:   file,
 		Bucket: aws.String(bucket),
 		Key:    aws.String(key),
 	})
 	if err != nil {
-		log.Printf("[S3 Output] Failed to upload data to %s/%s, %s\n", bucket, key, err)
-		os.Remove(path)
+		Debug(0, fmt.Sprintf("[S3 Output] Failed to upload data to %q/%q, %q", bucket, key, err))
 		return
 	}
 
-	os.Remove(path)
-
 	if o.closeCh != nil {
 		o.closeCh <- struct{}{}
 	}

output_tcp.go

@@ -4,8 +4,6 @@ import (
 	"crypto/tls"
 	"fmt"
 	"hash/fnv"
-	"io"
-	"log"
 	"net"
 	"time"
 )
@@ -14,58 +12,50 @@ import (
 // Currently used for internal communication between listener and replay server
 // Can be used for transfering binary payloads like protocol buffers
 type TCPOutput struct {
-	address  string
-	limit    int
-	buf      []chan []byte
-	bufStats *GorStat
-	config   *TCPOutputConfig
+	address     string
+	limit       int
+	buf         []chan *Message
+	config      *TCPOutputConfig
+	workerIndex uint32
+	stop        chan struct{}
 }
 
+// TCPOutputConfig tcp output configuration
 type TCPOutputConfig struct {
-	Secure bool `json:"output-tcp-secure"`
-	Sticky bool `json:"output-tcp-sticky"`
+	Secure     bool `json:"output-tcp-secure"`
+	Sticky     bool `json:"output-tcp-sticky"`
+	SkipVerify bool `json:"output-tcp-skip-verify"`
+	Workers    int  `json:"output-tcp-workers"`
 }
 
 // NewTCPOutput constructor for TCPOutput
-// Initialize 10 workers which hold keep-alive connection
-func NewTCPOutput(address string, config *TCPOutputConfig) io.Writer {
+// Initialize X workers which hold keep-alive connection
+func NewTCPOutput(address string, config *TCPOutputConfig) PluginWriter {
 	o := new(TCPOutput)
 
 	o.address = address
 	o.config = config
+	o.stop = make(chan struct{})
 
-	if Settings.OutputTCPStats {
-		o.bufStats = NewGorStat("output_tcp", 5000)
-	}
-
-	if o.config.Sticky {
-		// create 10 buffers and send the buffer index to the worker
-		o.buf = make([]chan []byte, 10)
-		for i := 0; i < 10; i++ {
-			o.buf[i] = make(chan []byte, 100)
-			go o.worker(i)
-		}
-	} else {
-		// create 1 buffer and send its index (0) to all workers
-		o.buf = make([]chan []byte, 1)
-		o.buf[0] = make(chan []byte, 1000)
-		for i := 0; i < 10; i++ {
-			go o.worker(0)
-		}
+	// create X buffers and send the buffer index to the worker
+	o.buf = make([]chan *Message, o.config.Workers)
+	for i := 0; i < o.config.Workers; i++ {
+		o.buf[i] = make(chan *Message, 100)
+		go o.worker(i)
 	}
 
 	return o
 }
 
 func (o *TCPOutput) worker(bufferIndex int) {
-	retries := 1
+	retries := 0
 	conn, err := o.connect(o.address)
 	for {
 		if err == nil {
 			break
 		}
 
-		log.Println("Can't connect to aggregator instance, reconnecting in 1 second. Retries:", retries)
+		Debug(1, fmt.Sprintf("Can't connect to aggregator instance, reconnecting in 1 second. Retries:%d", retries))
 		time.Sleep(1 * time.Second)
 
 		conn, err = o.connect(o.address)
@@ -73,19 +63,27 @@ func (o *TCPOutput) worker(bufferIndex int) {
 	}
 
 	if retries > 0 {
-		log.Println("Connected to aggregator instance after ", retries, " retries")
+		Debug(2, fmt.Sprintf("Connected to aggregator instance after %d retries", retries))
 	}
 
 	defer conn.Close()
 
 	for {
-		data := <-o.buf[bufferIndex]
-		conn.Write(data)
-		_, err := conn.Write([]byte(payloadSeparator))
+		var msg *Message
+		select {
+		case <-o.stop:
+			return
+		case msg = <-o.buf[bufferIndex]:
+		}
+		if _, err = conn.Write(msg.Meta); err == nil {
+			if _, err = conn.Write(msg.Data); err == nil {
+				_, err = conn.Write(payloadSeparatorAsBytes)
+			}
+		}
 
 		if err != nil {
-			log.Println("INFO: TCP output connection closed, reconnecting")
-			o.buf[bufferIndex] <- data
+			Debug(2, "INFO: TCP output connection closed, reconnecting")
+			o.buf[bufferIndex] <- msg
 			go o.worker(bufferIndex)
 			break
 		}
@@ -94,36 +92,31 @@ func (o *TCPOutput) worker(bufferIndex int) {
 
 func (o *TCPOutput) getBufferIndex(data []byte) int {
 	if !o.config.Sticky {
-		return 0
+		o.workerIndex++
+		return int(o.workerIndex) % o.config.Workers
 	}
 
 	hasher := fnv.New32a()
 	hasher.Write(payloadMeta(data)[1])
-	return int(hasher.Sum32()) % 10
+	return int(hasher.Sum32()) % o.config.Workers
+
 }
 
-func (o *TCPOutput) Write(data []byte) (n int, err error) {
-	if !isOriginPayload(data) {
-		return len(data), nil
+// PluginWrite writes message to this plugin
+func (o *TCPOutput) PluginWrite(msg *Message) (n int, err error) {
+	if !isOriginPayload(msg.Meta) {
+		return len(msg.Data), nil
 	}
 
-	// We have to copy, because sending data in multiple threads
-	newBuf := make([]byte, len(data))
-	copy(newBuf, data)
+	bufferIndex := o.getBufferIndex(msg.Data)
+	o.buf[bufferIndex] <- msg
 
-	bufferIndex := o.getBufferIndex(data)
-	o.buf[bufferIndex] <- newBuf
-
-	if Settings.OutputTCPStats {
-		o.bufStats.Write(len(o.buf[bufferIndex]))
-	}
-
-	return len(data), nil
+	return len(msg.Data) + len(msg.Meta), nil
 }
 
 func (o *TCPOutput) connect(address string) (conn net.Conn, err error) {
 	if o.config.Secure {
-		conn, err = tls.Dial("tcp", address, &tls.Config{})
+		conn, err = tls.Dial("tcp", address, &tls.Config{InsecureSkipVerify: o.config.SkipVerify})
 	} else {
 		conn, err = net.Dial("tcp", address)
 	}
@@ -134,3 +127,9 @@ func (o *TCPOutput) connect(address string) (conn net.Conn, err error) {
 func (o *TCPOutput) String() string {
 	return fmt.Sprintf("TCP output %s, limit: %d", o.address, o.limit)
 }
+
+// Close closes the data channel so that data
+func (o *TCPOutput) Close() error {
+	close(o.stop)
+	return nil
+}

output_tcp_test.go

@@ -2,7 +2,6 @@ package main
 
 import (
 	"bufio"
-	"io"
 	"log"
 	"net"
 	"sync"
@@ -12,24 +11,22 @@ import (
 
 func TestTCPOutput(t *testing.T) {
 	wg := new(sync.WaitGroup)
-	quit := make(chan int)
 
 	listener := startTCP(func(data []byte) {
 		wg.Done()
 	})
 	input := NewTestInput()
-	output := NewTCPOutput(listener.Addr().String(), &TCPOutputConfig{})
+	output := NewTCPOutput(listener.Addr().String(), &TCPOutputConfig{Workers: 10})
 
 	plugins := &InOutPlugins{
-		Inputs:  []io.Reader{input},
-		Outputs: []io.Writer{output},
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{output},
 	}
-	plugins.All = append(plugins.All, input, output)
 
-	emitter := NewEmitter(quit)
+	emitter := NewEmitter()
 	go emitter.Start(plugins, Settings.Middleware)
 
-	for i := 0; i < 100; i++ {
+	for i := 0; i < 10; i++ {
 		wg.Add(1)
 		input.EmitGET()
 	}
@@ -48,9 +45,9 @@ func startTCP(cb func([]byte)) net.Listener {
 	go func() {
 		for {
 			conn, _ := listener.Accept()
-			defer conn.Close()
 
-			go func() {
+			go func(conn net.Conn) {
+				defer conn.Close()
 				reader := bufio.NewReader(conn)
 				scanner := bufio.NewScanner(reader)
 				scanner.Split(payloadScanner)
@@ -58,7 +55,7 @@ func startTCP(cb func([]byte)) net.Listener {
 				for scanner.Scan() {
 					cb(scanner.Bytes())
 				}
-			}()
+			}(conn)
 		}
 	}()
 
@@ -67,51 +64,50 @@ func startTCP(cb func([]byte)) net.Listener {
 
 func BenchmarkTCPOutput(b *testing.B) {
 	wg := new(sync.WaitGroup)
-	quit := make(chan int)
 
 	listener := startTCP(func(data []byte) {
 		wg.Done()
 	})
 	input := NewTestInput()
-	output := NewTCPOutput(listener.Addr().String(), &TCPOutputConfig{})
-
-	plugins := &InOutPlugins{
-		Inputs:  []io.Reader{input},
-		Outputs: []io.Writer{output},
-	}
-	plugins.All = append(plugins.All, input, output)
-
-	emitter := NewEmitter(quit)
-	go emitter.Start(plugins, Settings.Middleware)
-
-	b.ResetTimer()
+	input.data = make(chan []byte, b.N)
 	for i := 0; i < b.N; i++ {
-		wg.Add(1)
 		input.EmitGET()
 	}
+	wg.Add(b.N)
+	output := NewTCPOutput(listener.Addr().String(), &TCPOutputConfig{Workers: 10})
+
+	plugins := &InOutPlugins{
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{output},
+	}
+
+	emitter := NewEmitter()
+	// avoid counting above initialization
+	b.ResetTimer()
+	go emitter.Start(plugins, Settings.Middleware)
 
 	wg.Wait()
 	emitter.Close()
 }
 
 func TestStickyDisable(t *testing.T) {
-	tcpOutput := TCPOutput{config: &TCPOutputConfig{Sticky: false}}
+	tcpOutput := TCPOutput{config: &TCPOutputConfig{Sticky: false, Workers: 10}}
 
-	for i := 0; i < 1000; i++ {
+	for i := 0; i < 10; i++ {
 		index := tcpOutput.getBufferIndex(getTestBytes())
-		if index != 0 {
-			t.Errorf("Sticky is disable. Got: %d want 0", index)
+		if index != (i+1)%10 {
+			t.Errorf("Sticky is disable. Got: %d want %d", index, (i+1)%10)
 		}
 	}
 }
 
 func TestBufferDistribution(t *testing.T) {
 	numberOfWorkers := 10
-	numberOfMessages := 1000000
+	numberOfMessages := 10000
 	percentDistributionErrorRange := 20
 
 	buffer := make([]int, numberOfWorkers)
-	tcpOutput := TCPOutput{config: &TCPOutputConfig{Sticky: true}}
+	tcpOutput := TCPOutput{config: &TCPOutputConfig{Sticky: true, Workers: 10}}
 	for i := 0; i < numberOfMessages; i++ {
 		buffer[tcpOutput.getBufferIndex(getTestBytes())]++
 	}

plugins.go

@@ -1,24 +1,39 @@
 package main
 
 import (
-	"io"
 	"reflect"
 	"strings"
-	"sync"
 )
 
+// Message represents data accross plugins
+type Message struct {
+	Meta []byte // metadata
+	Data []byte // actual data
+}
+
+// PluginReader is an interface for input plugins
+type PluginReader interface {
+	PluginRead() (msg *Message, err error)
+}
+
+// PluginWriter is an interface for output plugins
+type PluginWriter interface {
+	PluginWrite(msg *Message) (n int, err error)
+}
+
+// PluginReadWriter is an interface for plugins that support reading and writing
+type PluginReadWriter interface {
+	PluginReader
+	PluginWriter
+}
+
 // InOutPlugins struct for holding references to plugins
 type InOutPlugins struct {
-	Inputs  []io.Reader
-	Outputs []io.Writer
+	Inputs  []PluginReader
+	Outputs []PluginWriter
 	All     []interface{}
 }
 
-var pluginMu sync.Mutex
-
-// Plugins holds all the plugin objects
-var plugins *InOutPlugins = new(InOutPlugins)
-
 // extractLimitOptions detects if plugin get called with limiter support
 // Returns address and limit
 func extractLimitOptions(options string) (string, string) {
@@ -33,8 +48,8 @@ func extractLimitOptions(options string) (string, string) {
 
 // Automatically detects type of plugin and initialize it
 //
-// See this article if curious about relfect stuff below: http://blog.burntsushi.net/type-parametric-functions-golang
-func registerPlugin(constructor interface{}, options ...interface{}) {
+// See this article if curious about reflect stuff below: http://blog.burntsushi.net/type-parametric-functions-golang
+func (plugins *InOutPlugins) registerPlugin(constructor interface{}, options ...interface{}) {
 	var path, limit string
 	vc := reflect.ValueOf(constructor)
 
@@ -54,83 +69,69 @@ func registerPlugin(constructor interface{}, options ...interface{}) {
 
 	// Calling our constructor with list of given options
 	plugin := vc.Call(vo)[0].Interface()
-	pluginWrapper := plugin
 
 	if limit != "" {
-		pluginWrapper = NewLimiter(plugin, limit)
-	} else {
-		pluginWrapper = plugin
+		plugin = NewLimiter(plugin, limit)
 	}
 
-	_, isR := plugin.(io.Reader)
-	_, isW := plugin.(io.Writer)
-
 	// Some of the output can be Readers as well because return responses
-	if isR && !isW {
-		plugins.Inputs = append(plugins.Inputs, pluginWrapper.(io.Reader))
+	if r, ok := plugin.(PluginReader); ok {
+		plugins.Inputs = append(plugins.Inputs, r)
 	}
 
-	if isW {
-		plugins.Outputs = append(plugins.Outputs, pluginWrapper.(io.Writer))
+	if w, ok := plugin.(PluginWriter); ok {
+		plugins.Outputs = append(plugins.Outputs, w)
 	}
-
 	plugins.All = append(plugins.All, plugin)
+
 }
 
-// InitPlugins specify and initialize all available plugins
-func InitPlugins() *InOutPlugins {
-	pluginMu.Lock()
-	defer pluginMu.Unlock()
+// NewPlugins specify and initialize all available plugins
+func NewPlugins() *InOutPlugins {
+	plugins := new(InOutPlugins)
 
 	for _, options := range Settings.InputDummy {
-		registerPlugin(NewDummyInput, options)
+		plugins.registerPlugin(NewDummyInput, options)
 	}
 
 	for range Settings.OutputDummy {
-		registerPlugin(NewDummyOutput)
+		plugins.registerPlugin(NewDummyOutput)
 	}
 
 	if Settings.OutputStdout {
-		registerPlugin(NewDummyOutput)
+		plugins.registerPlugin(NewDummyOutput)
 	}
 
 	if Settings.OutputNull {
-		registerPlugin(NewNullOutput)
-	}
-
-	engine := EnginePcap
-	if Settings.InputRAWConfig.Engine == "raw_socket" {
-		engine = EngineRawSocket
-	} else if Settings.InputRAWConfig.Engine == "pcap_file" {
-		engine = EnginePcapFile
+		plugins.registerPlugin(NewNullOutput)
 	}
 
 	for _, options := range Settings.InputRAW {
-		registerPlugin(NewRAWInput, options, engine, Settings.InputRAWConfig.TrackResponse, Settings.InputRAWConfig.Expire, Settings.InputRAWConfig.RealIPHeader, Settings.InputRAWConfig.Protocol, Settings.InputRAWConfig.BpfFilter, Settings.InputRAWConfig.TimestampType, Settings.InputRAWConfig.BufferSize)
+		plugins.registerPlugin(NewRAWInput, options, Settings.RAWInputConfig)
 	}
 
 	for _, options := range Settings.InputTCP {
-		registerPlugin(NewTCPInput, options, &Settings.InputTCPConfig)
+		plugins.registerPlugin(NewTCPInput, options, &Settings.InputTCPConfig)
 	}
 
 	for _, options := range Settings.OutputTCP {
-		registerPlugin(NewTCPOutput, options, &Settings.OutputTCPConfig)
+		plugins.registerPlugin(NewTCPOutput, options, &Settings.OutputTCPConfig)
 	}
 
 	for _, options := range Settings.InputFile {
-		registerPlugin(NewFileInput, options, Settings.InputFileLoop)
+		plugins.registerPlugin(NewFileInput, options, Settings.InputFileLoop)
 	}
 
 	for _, path := range Settings.OutputFile {
 		if strings.HasPrefix(path, "s3://") {
-			registerPlugin(NewS3Output, path, &Settings.OutputFileConfig)
+			plugins.registerPlugin(NewS3Output, path, &Settings.OutputFileConfig)
 		} else {
-			registerPlugin(NewFileOutput, path, &Settings.OutputFileConfig)
+			plugins.registerPlugin(NewFileOutput, path, &Settings.OutputFileConfig)
 		}
 	}
 
 	for _, options := range Settings.InputHTTP {
-		registerPlugin(NewHTTPInput, options)
+		plugins.registerPlugin(NewHTTPInput, options)
 	}
 
 	// If we explicitly set Host header http output should not rewrite it
@@ -143,19 +144,19 @@ func InitPlugins() *InOutPlugins {
 	}
 
 	for _, options := range Settings.OutputHTTP {
-		registerPlugin(NewHTTPOutput, options, &Settings.OutputHTTPConfig)
+		plugins.registerPlugin(NewHTTPOutput, options, &Settings.OutputHTTPConfig)
 	}
 
 	for _, options := range Settings.OutputBinary {
-		registerPlugin(NewBinaryOutput, options, &Settings.OutputBinaryConfig)
+		plugins.registerPlugin(NewBinaryOutput, options, &Settings.OutputBinaryConfig)
 	}
 
 	if Settings.OutputKafkaConfig.Host != "" && Settings.OutputKafkaConfig.Topic != "" {
-		registerPlugin(NewKafkaOutput, "", &Settings.OutputKafkaConfig)
+		plugins.registerPlugin(NewKafkaOutput, "", &Settings.OutputKafkaConfig, &Settings.KafkaTLSConfig)
 	}
 
 	if Settings.InputKafkaConfig.Host != "" && Settings.InputKafkaConfig.Topic != "" {
-		registerPlugin(NewKafkaInput, "", &Settings.InputKafkaConfig)
+		plugins.registerPlugin(NewKafkaInput, "", &Settings.InputKafkaConfig, &Settings.KafkaTLSConfig)
 	}
 
 	return plugins

plugins_test.go

@@ -10,10 +10,10 @@ func TestPluginsRegistration(t *testing.T) {
 	Settings.OutputHTTP = MultiOption{"www.example.com|10"}
 	Settings.InputFile = MultiOption{"/dev/null"}
 
-	plugins := InitPlugins()
+	plugins := NewPlugins()
 
-	if len(plugins.Inputs) != 2 {
-		t.Errorf("Should be 2 inputs %d", len(plugins.Inputs))
+	if len(plugins.Inputs) != 3 {
+		t.Errorf("Should be 3 inputs got %d", len(plugins.Inputs))
 	}
 
 	if _, ok := plugins.Inputs[0].(*DummyInput); !ok {

proto/fuzz.go

@@ -4,9 +4,7 @@ package proto
 
 func Fuzz(data []byte) int {
 
-	ParseHeaders([][]byte{data}, func(header []byte, value []byte) bool {
-		return true
-	})
+	ParseHeaders(data)
 
 	return 1
 }

proto/proto.go

@@ -17,216 +17,120 @@ Example of HTTP payload for future references, new line symbols escaped:
 package proto
 
 import (
+	"bufio"
 	"bytes"
+	"net/http"
+	"net/textproto"
+	"strings"
 
 	"github.com/buger/goreplay/byteutils"
 )
 
-// In HTTP newline defined by 2 bytes (for both windows and *nix support)
-var CLRF = []byte("\r\n")
+// CRLF In HTTP newline defined by 2 bytes (for both windows and *nix support)
+var CRLF = []byte("\r\n")
 
-// New line acts as separator: end of Headers or Body (in some cases)
+// EmptyLine acts as separator: end of Headers or Body (in some cases)
 var EmptyLine = []byte("\r\n\r\n")
 
-// Separator for Header line. Header looks like: `HeaderName: value`
+// HeaderDelim Separator for Header line. Header looks like: `HeaderName: value`
 var HeaderDelim = []byte(": ")
 
 // MIMEHeadersEndPos finds end of the Headers section, which should end with empty line.
 func MIMEHeadersEndPos(payload []byte) int {
-	return bytes.Index(payload, EmptyLine) + 4
+	pos := bytes.Index(payload, EmptyLine)
+	if pos < 0 {
+		return -1
+	}
+	return pos + 4
 }
 
 // MIMEHeadersStartPos finds start of Headers section
 // It just finds position of second line (first contains location and method).
 func MIMEHeadersStartPos(payload []byte) int {
-	return bytes.Index(payload, CLRF) + 2 // Find first line end
-}
-
-func headerIndex(payload []byte, name []byte) int {
-	i := 0
-	for {
-		// we need enough space for at least '\n' and the header name
-		if i >= (len(payload) - len(name) - 1) {
-			return -1
-		}
-
-		if payload[i] == '\n' {
-			i++
-			if bytes.EqualFold(name, payload[i:i+len(name)]) {
-				return i
-			}
-		}
-		i++
-
+	pos := bytes.Index(payload, CRLF)
+	if pos < 0 {
+		return -1
 	}
-	return -1
+	return pos + 2 // Find first line end
 }
 
 // header return value and positions of header/value start/end.
 // If not found, value will be blank, and headerStart will be -1
 // Do not support multi-line headers.
 func header(payload []byte, name []byte) (value []byte, headerStart, headerEnd, valueStart, valueEnd int) {
-	headerStart = headerIndex(payload, name)
-
-	if headerStart == -1 {
+	headerStart = MIMEHeadersStartPos(payload)
+	if headerStart < 0 {
+		return
+	}
+	var colonIndex int
+	for headerStart < len(payload) {
+		headerEnd = bytes.IndexByte(payload[headerStart:], '\n')
+		if headerEnd == -1 {
+			break
+		}
+		headerEnd += headerStart
+		colonIndex = bytes.IndexByte(payload[headerStart:headerEnd], ':')
+		if colonIndex == -1 {
+			break
+		}
+		colonIndex += headerStart
+		if bytes.EqualFold(payload[headerStart:colonIndex], name) {
+			valueStart = colonIndex + 1
+			valueEnd = headerEnd - 2
+			break
+		}
+		headerStart = headerEnd + 1 // move to the next header
+	}
+	if valueStart == 0 {
+		headerStart = -1
+		headerEnd = -1
+		valueEnd = -1
+		valueStart = -1
 		return
 	}
 
-	valueStart = headerStart + len(name) + 1 // Skip ":" after header name
-	headerEnd = valueStart + bytes.IndexByte(payload[valueStart:], '\n')
-
-	for valueStart < headerEnd { // Ignore empty space after ':'
-		if payload[valueStart] == ' ' {
+	// ignore empty space after ':'
+	for valueStart < valueEnd {
+		if payload[valueStart] < 0x21 {
 			valueStart++
 		} else {
 			break
 		}
 	}
 
-	valueEnd = valueStart + bytes.IndexByte(payload[valueStart:], '\n')
-
-	if payload[headerEnd-1] == '\r' {
-		valueEnd--
-	}
-
 	// ignore empty space at end of header value
-	for valueStart < valueEnd {
-		if payload[valueEnd-1] == ' ' {
+	for valueEnd > valueStart {
+		if payload[valueEnd] < 0x21 {
 			valueEnd--
 		} else {
 			break
 		}
 	}
-	value = payload[valueStart:valueEnd]
+	value = payload[valueStart : valueEnd+1]
 
 	return
 }
 
-// Works only with ASCII
-func HeadersEqual(h1 []byte, h2 []byte) bool {
-	if len(h1) != len(h2) {
-		return false
+// ParseHeaders Parsing headers from the payload
+func ParseHeaders(p []byte) textproto.MIMEHeader {
+	// trimming off the title of the request
+	if HasTitle(p) {
+		headerStart := MIMEHeadersStartPos(p)
+		if headerStart > len(p)-1 {
+			return nil
+		}
+		p = p[headerStart:]
 	}
-
-	for i, c1 := range h1 {
-		c2 := h2[i]
-
-		switch int(c1) - int(c2) {
-		case 0, 32, -32:
-		default:
-			return false
-		}
+	headerEnd := MIMEHeadersEndPos(p)
+	if headerEnd > 1 {
+		p = p[:headerEnd]
 	}
-
-	return true
-}
-
-// Parsing headers from multiple payloads
-func ParseHeaders(payloads [][]byte, cb func(header []byte, value []byte) bool) {
-	hS := [2]int{0, 0}   // header start
-	hE := [2]int{-1, -1} // header end
-	vS := [2]int{-1, -1} // value start
-	vE := [2]int{-1, -1} // value end
-
-	i := 0
-	pIdx := 0
-	lineBreaks := 0
-	newLineBreak := true
-
-	for {
-		if len(payloads)-1 < pIdx {
-			break
-		}
-
-		p := payloads[pIdx]
-
-		if len(p)-1 < i {
-			pIdx++
-			i = 0
-			continue
-		}
-
-		switch p[i] {
-		case '\r', '\n':
-			newLineBreak = true
-			lineBreaks++
-
-			// End of headers
-			if lineBreaks == 4 {
-				return
-			}
-
-			if lineBreaks > 1 {
-				break
-			}
-
-			vE = [2]int{pIdx, i}
-
-			if vS[1] != -1 && vE[1] != -1 &&
-				hS[1] != -1 && hE[1] != -1 {
-
-				var header, value []byte
-
-				phS, phE, pvS, pvE := payloads[hS[0]], payloads[hE[0]], payloads[vS[0]], payloads[vE[0]]
-
-				// If in same payload
-				if hS[0] == hE[0] {
-					header = phS[hS[1]:hE[1]]
-				} else {
-					header = make([]byte, len(phS)-hS[1]+hE[1])
-					copy(header, phS[hS[1]:])
-					copy(header[len(phS)-hS[1]:], phE[:hE[1]])
-				}
-
-				if vS[0] == vE[0] {
-					value = pvS[vS[1]:vE[1]]
-				} else {
-					value = make([]byte, len(pvS)-vS[1]+vE[1])
-					copy(value, pvS[vS[1]:])
-					copy(value[len(pvS)-vS[1]:], pvE[:vE[1]])
-				}
-
-				if !cb(header, value) {
-					return
-				}
-			}
-
-			// Header found, reset values
-			vS = [2]int{-1, -1}
-			vE = [2]int{-1, -1}
-			hS = [2]int{-1, -1}
-			hE = [2]int{-1, -1}
-		case ':':
-			if newLineBreak {
-				hE = [2]int{pIdx, i}
-				newLineBreak = false
-			}
-			lineBreaks = 0
-		default:
-			lineBreaks = 0
-
-			if hS[1] == -1 {
-				hS = [2]int{pIdx, i}
-				hE = [2]int{-1, -1}
-			} else {
-				if hE[1] == -1 {
-					break
-				}
-
-				if vS[1] == -1 {
-					if p[i] == ' ' {
-						break
-					}
-
-					vS = [2]int{pIdx, i}
-				}
-			}
-		}
-
-		i++
+	reader := textproto.NewReader(bufio.NewReader(bytes.NewReader(p)))
+	mime, err := reader.ReadMIMEHeader()
+	if err != nil {
+		return nil
 	}
-
-	return
+	return mime
 }
 
 // Header returns header value, if header not found, value will be blank
@@ -243,7 +147,7 @@ func SetHeader(payload, name, value []byte) []byte {
 
 	if hs != -1 {
 		// If header found we just replace its value
-		return byteutils.Replace(payload, vs, ve, value)
+		return byteutils.Replace(payload, vs, ve+1, value)
 	}
 
 	return AddHeader(payload, name, value)
@@ -252,67 +156,54 @@ func SetHeader(payload, name, value []byte) []byte {
 // AddHeader takes http payload and appends new header to the start of headers section
 // Returns modified request payload
 func AddHeader(payload, name, value []byte) []byte {
+	mimeStart := MIMEHeadersStartPos(payload)
+	if mimeStart < 1 {
+		return payload
+	}
 	header := make([]byte, len(name)+2+len(value)+2)
 	copy(header[0:], name)
 	copy(header[len(name):], HeaderDelim)
 	copy(header[len(name)+2:], value)
-	copy(header[len(header)-2:], CLRF)
-
-	mimeStart := MIMEHeadersStartPos(payload)
+	copy(header[len(header)-2:], CRLF)
 
 	return byteutils.Insert(payload, mimeStart, header)
 }
 
-// DelHeader takes http payload and removes header name from headers section
+// DeleteHeader takes http payload and removes header name from headers section
 // Returns modified request payload
 func DeleteHeader(payload, name []byte) []byte {
 	_, hs, he, _, _ := header(payload, name)
 	if hs != -1 {
-		newHeader := make([]byte, len(payload)-(he-hs)-1)
-		copy(newHeader[:hs], payload[:hs])
-		copy(newHeader[hs:], payload[he+1:])
-		return newHeader
+		return byteutils.Cut(payload, hs, he+1)
 	}
 	return payload
 }
 
 // Body returns request/response body
 func Body(payload []byte) []byte {
-	// 4 -> len(EMPTY_LINE)
-	if len(payload) < 4 {
-		return []byte{}
+	pos := MIMEHeadersEndPos(payload)
+	if pos == -1 {
+		return nil
 	}
-
-	return payload[MIMEHeadersEndPos(payload):]
+	return payload[pos:]
 }
 
 // Path takes payload and retuns request path: Split(firstLine, ' ')[1]
 func Path(payload []byte) []byte {
+	if !HasTitle(payload) {
+		return nil
+	}
 	start := bytes.IndexByte(payload, ' ') + 1
-	eol := bytes.IndexByte(payload[start:], '\r')
 	end := bytes.IndexByte(payload[start:], ' ')
 
-	if eol > 0 {
-		if end == -1 || eol < end {
-			return payload[start : start+eol]
-		}
-	} else { // support for legacy clients
-		eol = bytes.IndexByte(payload[start:], '\n')
-
-		if eol > 0 && (end == -1 || eol < end) {
-			return payload[start : start+eol]
-		}
-	}
-
-	if end < 0 {
-		return payload[start:]
-	}
-
 	return payload[start : start+end]
 }
 
 // SetPath takes payload, sets new path and returns modified payload
 func SetPath(payload, path []byte) []byte {
+	if !HasTitle(payload) {
+		return nil
+	}
 	start := bytes.IndexByte(payload, ' ') + 1
 	end := bytes.IndexByte(payload[start:], ' ')
 
@@ -403,31 +294,221 @@ func SetHost(payload, url, host []byte) []byte {
 // Method returns HTTP method
 func Method(payload []byte) []byte {
 	end := bytes.IndexByte(payload, ' ')
+	if end == -1 {
+		return nil
+	}
 
 	return payload[:end]
 }
 
 // Status returns response status.
-// It happend to be in same position as request payload path
+// It happens to be in same position as request payload path
 func Status(payload []byte) []byte {
 	return Path(payload)
 }
 
-var httpMethods []string = []string{
-	"GET ", "OPTI", "HEAD", "POST", "PUT ", "DELE", "TRAC", "CONN", "PATC" /* custom methods */, "BAN ", "PURG", "PROP", "MKCO", "COPY", "MOVE", "LOCK", "UNLO",
+// Methods holds the http methods ordered in ascending order
+var Methods = [...]string{
+	http.MethodConnect, http.MethodDelete, http.MethodGet,
+	http.MethodHead, http.MethodOptions, http.MethodPatch,
+	http.MethodPost, http.MethodPut, http.MethodTrace,
 }
 
-func IsHTTPPayload(payload []byte) bool {
-	if len(payload) < 4 {
+const (
+	//MinRequestCount GET / HTTP/1.1\r\n
+	MinRequestCount = 16
+	// MinResponseCount HTTP/1.1 200 OK\r\n
+	MinResponseCount = 17
+	// VersionLen HTTP/1.1
+	VersionLen = 8
+)
+
+// HasResponseTitle reports whether this payload has an HTTP/1 response title
+func HasResponseTitle(payload []byte) bool {
+	s := byteutils.SliceToString(payload)
+	if len(s) < MinResponseCount {
 		return false
 	}
+	titleLen := bytes.Index(payload, CRLF)
+	if titleLen == -1 {
+		return false
+	}
+	major, minor, ok := http.ParseHTTPVersion(s[0:VersionLen])
+	if !(ok && major == 1 && (minor == 0 || minor == 1)) {
+		return false
+	}
+	if s[VersionLen] != ' ' {
+		return false
+	}
+	status, ok := atoI(payload[VersionLen+1:VersionLen+4], 10)
+	if !ok || s[VersionLen+4] != ' ' {
+		return false
+	}
+	statusText := http.StatusText(status)
+	if statusText == "" {
+		return false
+	}
+	if titleLen+len(CRLF) > len(s) {
+		return false
+	}
+	return s[VersionLen+5:titleLen] == statusText
+}
 
-	method := string(payload[0:4])
-
-	for _, m := range httpMethods {
-		if method == m {
-			return true
+// HasRequestTitle reports whether this payload has an HTTP/1 request title
+func HasRequestTitle(payload []byte) bool {
+	s := byteutils.SliceToString(payload)
+	if len(s) < MinRequestCount {
+		return false
+	}
+	titleLen := bytes.Index(payload, CRLF)
+	if titleLen == -1 {
+		return false
+	}
+	if strings.Count(s[:titleLen], " ") != 2 {
+		return false
+	}
+	method := string(Method(payload))
+	var methodFound bool
+	for _, m := range Methods {
+		if methodFound = method == m; methodFound {
+			break
 		}
 	}
-	return false
+	if !methodFound {
+		return false
+	}
+	path := strings.Index(s[len(method)+1:], " ")
+	if path == -1 {
+		return false
+	}
+	major, minor, ok := http.ParseHTTPVersion(s[path+len(method)+2 : titleLen])
+	return ok && major == 1 && (minor == 0 || minor == 1)
+}
+
+// HasTitle reports if this payload has an http/1 title
+func HasTitle(payload []byte) bool {
+	return HasRequestTitle(payload) || HasResponseTitle(payload)
+}
+
+// CheckChunked checks HTTP/1 chunked data integrity and return the final index
+// of chunks(index after '0\r\n\r\n') or -1 if there is missing data
+// or there is bad format
+func CheckChunked(buf []byte) (chunkEnd int) {
+	var (
+		ok     bool
+		chkLen int
+		sz     int
+		ext    int
+	)
+	for {
+		sz = bytes.IndexByte(buf[chunkEnd:], '\r')
+		if sz < 1 {
+			return -1
+		}
+		// ignoring chunks extensions https://github.com/golang/go/issues/13135
+		// but chunks extensions are no longer a thing
+		ext = bytes.IndexByte(buf[chunkEnd:chunkEnd+sz], ';')
+		if ext < 0 {
+			ext = sz
+		}
+		chkLen, ok = atoI(buf[chunkEnd:chunkEnd+ext], 16)
+		if !ok {
+			return -1
+		}
+		chunkEnd += (sz + 2)
+		if chkLen == 0 {
+			if !bytes.Equal(buf[chunkEnd:chunkEnd+2], CRLF) {
+				return -1
+			}
+			return chunkEnd + 2
+		}
+		// ideally chunck length and at least len("\r\n0\r\n\r\n")
+		if len(buf[chunkEnd:]) < chkLen+7 {
+			return -1
+		}
+		chunkEnd += chkLen
+		// chunks must end with CRLF
+		if !bytes.Equal(buf[chunkEnd:chunkEnd+2], CRLF) {
+			return -1
+		}
+		chunkEnd += 2
+	}
+}
+
+// HasFullPayload reports if this http has full payloads
+func HasFullPayload(payload []byte) bool {
+	body := Body(payload)
+
+	// check for chunked transfer-encoding
+	header := Header(payload, []byte("Transfer-Encoding"))
+	if bytes.Contains(header, []byte("chunked")) {
+
+		// check chunks
+		if len(body) < 1 {
+			return false
+		}
+		var chunkEnd int
+		if chunkEnd = CheckChunked(body); chunkEnd < 1 {
+			return false
+		}
+
+		// check trailer headers
+		if len(Header(payload, []byte("Trailer"))) < 1 {
+			return true
+		}
+		// trailer headers(whether chunked or plain) should end with empty line
+		return len(body) > chunkEnd && MIMEHeadersEndPos(body[chunkEnd:]) != -1
+	}
+
+	// check for content-length header
+	// trailers are generally not allowed in non-chunks body
+	header = Header(payload, []byte("Content-Length"))
+	if len(header) > 1 {
+		num, ok := atoI(header, 10)
+		return ok && num == len(body)
+	}
+
+	// for empty body, check for emptyline
+	return MIMEHeadersEndPos(payload) != -1
+}
+
+// this works with positive integers
+func atoI(s []byte, base int) (num int, ok bool) {
+	var v int
+	for i := 0; i < len(s); i++ {
+		if s[i] > 127 {
+			return 0, false
+		}
+		v = int(hexTable[s[i]])
+		if v >= base || (v == 0 && s[i] != '0') {
+			return 0, false
+		}
+		num = (num * base) + v
+	}
+	return num, true
+}
+
+var hexTable = [128]byte{
+	'0': 0,
+	'1': 1,
+	'2': 2,
+	'3': 3,
+	'4': 4,
+	'5': 5,
+	'6': 6,
+	'7': 7,
+	'8': 8,
+	'9': 9,
+	'A': 10,
+	'a': 10,
+	'B': 11,
+	'b': 11,
+	'C': 12,
+	'c': 12,
+	'D': 13,
+	'd': 13,
+	'E': 14,
+	'e': 14,
+	'F': 15,
+	'f': 15,
 }

proto/proto_test.go

@@ -2,6 +2,7 @@ package proto
 
 import (
 	"bytes"
+	"net/textproto"
 	"reflect"
 	"testing"
 )
@@ -38,13 +39,6 @@ func TestHeader(t *testing.T) {
 		t.Error("Should return empty value")
 	}
 
-	// Wrong delimeter
-	payload = []byte("GET /p HTTP/1.1\r\nCookie: 123\nHost: www.w3.org\r\n\r\n")
-
-	if val = Header(payload, []byte("Cookie")); !bytes.Equal(val, []byte("123")) {
-		t.Error("Should handle wrong header delimeter")
-	}
-
 	// Header not found
 	if _, headerStart, _, _, _ = header(payload, []byte("Not-Found")); headerStart != -1 {
 		t.Error("Should not found header")
@@ -103,6 +97,10 @@ func TestSetHeader(t *testing.T) {
 	if payload = SetHeader(payload, []byte("User-Agent"), []byte("Gor")); !bytes.Equal(payload, payloadAfter) {
 		t.Error("Should add header if not found", string(payload))
 	}
+	invalidPayload := []byte("POST /post HTTP/1.1")
+	if invalidPayload = SetHeader(invalidPayload, []byte("User-Agent"), []byte("Gor")); !bytes.Equal(invalidPayload, []byte("POST /post HTTP/1.1")) {
+		t.Error("Should not modify payload if request is invalid", string(payload))
+	}
 }
 
 func TestDeleteHeader(t *testing.T) {
@@ -127,17 +125,12 @@ func TestDeleteHeader(t *testing.T) {
 func TestParseHeaders(t *testing.T) {
 	payload := [][]byte{[]byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.or"), []byte("g\r\nUser-Ag"), []byte("ent:Chrome\r\n\r\n"), []byte("Fake-Header: asda")}
 
-	headers := make(map[string]string)
+	headers := ParseHeaders(bytes.Join(payload, nil))
 
-	ParseHeaders(payload, func(header []byte, value []byte) bool {
-		headers[string(header)] = string(value)
-		return true
-	})
-
-	expected := map[string]string{
-		"Content-Length": "7",
-		"Host":           "www.w3.org",
-		"User-Agent":     "Chrome",
+	expected := textproto.MIMEHeader{
+		"Content-Length": []string{"7"},
+		"Host":           []string{"www.w3.org"},
+		"User-Agent":     []string{"Chrome"},
 	}
 
 	if !reflect.DeepEqual(headers, expected) {
@@ -152,9 +145,7 @@ func TestFuzzCrashers(t *testing.T) {
 	}
 
 	for _, f := range crashers {
-		ParseHeaders([][]byte{[]byte(f)}, func(header []byte, value []byte) bool {
-			return true
-		})
+		ParseHeaders([]byte(f))
 	}
 }
 
@@ -163,18 +154,13 @@ func TestParseHeadersWithComplexUserAgent(t *testing.T) {
 	// Parser should wait for \r\n
 	payload := [][]byte{[]byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.or"), []byte("g\r\nUser-Ag"), []byte("ent:Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0) like Gecko\r\n\r\n"), []byte("Fake-Header: asda")}
 
-	headers := make(map[string]string)
-
-	ParseHeaders(payload, func(header []byte, value []byte) bool {
-		headers[string(header)] = string(value)
-		return true
-	})
+	headers := ParseHeaders(bytes.Join(payload, nil))
 
 	expected := map[string]string{
 		"User-Agent": "Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0) like Gecko",
 	}
 
-	if expected["User-Agent"] != headers["User-Agent"] {
+	if expected["User-Agent"] != headers["User-Agent"][0] {
 		t.Errorf("Header 'User-Agent' expected '%s' and parsed: '%s'", expected["User-Agent"], headers["User-Agent"])
 	}
 }
@@ -184,12 +170,7 @@ func TestParseHeadersWithOrigin(t *testing.T) {
 	// Parser should wait for \r\n
 	payload := [][]byte{[]byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.or"), []byte("g\r\nReferrer: http://127.0.0.1:3000\r\nOrigi"), []byte("n: https://www.example.com\r\nUser-Ag"), []byte("ent:Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0) like Gecko\r\n\r\n"), []byte("in:https://www.example.com\r\n\r\n"), []byte("Fake-Header: asda")}
 
-	headers := make(map[string]string)
-
-	ParseHeaders(payload, func(header []byte, value []byte) bool {
-		headers[string(header)] = string(value)
-		return true
-	})
+	headers := ParseHeaders(bytes.Join(payload, nil))
 
 	expected := map[string]string{
 		"Origin":     "https://www.example.com",
@@ -197,38 +178,19 @@ func TestParseHeadersWithOrigin(t *testing.T) {
 		"Referrer":   "http://127.0.0.1:3000",
 	}
 
-	if expected["Referrer"] != headers["Referrer"] {
+	if expected["Referrer"] != headers["Referrer"][0] {
 		t.Errorf("Header 'Referrer' expected '%s' and parsed: '%s'", expected["Referrer"], headers["Referrer"])
 	}
 
-	if expected["Origin"] != headers["Origin"] {
+	if expected["Origin"] != headers["Origin"][0] {
 		t.Errorf("Header 'Origin' expected '%s' and parsed: '%s'", expected["Origin"], headers["Origin"])
 	}
 
-	if expected["User-Agent"] != headers["User-Agent"] {
+	if expected["User-Agent"] != headers["User-Agent"][0] {
 		t.Errorf("Header 'User-Agent' expected '%s' and parsed: '%s'", expected["User-Agent"], headers["User-Agent"])
 	}
 }
 
-func TestHeaderEquals(t *testing.T) {
-	tests := []struct {
-		h1     string
-		h2     string
-		equals bool
-	}{
-		{"Content-Length", "content-length", true},
-		{"content-length", "Content-Length", true},
-		{"content-Pength", "Content-Length", false},
-		{"Host", "Content-Length", false},
-	}
-
-	for _, tc := range tests {
-		if HeadersEqual([]byte(tc.h1), []byte(tc.h2)) != tc.equals {
-			t.Error(tc)
-		}
-	}
-}
-
 func TestPath(t *testing.T) {
 	var path, payload []byte
 
@@ -240,20 +202,20 @@ func TestPath(t *testing.T) {
 
 	payload = []byte("GET /get\r\n\r\nHost: www.w3.org\r\n\r\n")
 
-	if path = Path(payload); !bytes.Equal(path, []byte("/get")) {
-		t.Error("Should find path", string(path))
+	if path = Path(payload); !bytes.Equal(path, nil) {
+		t.Error("1Should not find path", string(path))
 	}
 
 	payload = []byte("GET /get\n")
 
-	if path = Path(payload); !bytes.Equal(path, []byte("/get")) {
-		t.Error("Should find path", string(path))
+	if path = Path(payload); !bytes.Equal(path, nil) {
+		t.Error("2Should not find path", string(path))
 	}
 
 	payload = []byte("GET /get")
 
-	if path = Path(payload); !bytes.Equal(path, []byte("/get")) {
-		t.Error("Should find path", string(path))
+	if path = Path(payload); !bytes.Equal(path, nil) {
+		t.Error("3Should not find path", string(path))
 	}
 }
 
@@ -266,6 +228,7 @@ func TestSetPath(t *testing.T) {
 	if payload = SetPath(payload, []byte("/new_path")); !bytes.Equal(payload, payloadAfter) {
 		t.Error("Should replace path", string(payload))
 	}
+
 }
 
 func TestPathParam(t *testing.T) {
@@ -332,4 +295,172 @@ func TestSetHostHTTP10(t *testing.T) {
 	if payload = SetHost(payload, []byte("http://new.com"), []byte("new.com")); !bytes.Equal(payload, payloadAfter) {
 		t.Error("Should replace host", string(payload))
 	}
+
+	payload = []byte("POST /post HTTP/1.0\r\nContent-Length: 7\r\nHost: example.com\r\n\r\na=1&b=2")
+	payloadAfter = []byte("POST /post HTTP/1.0\r\nContent-Length: 7\r\nHost: new.com\r\n\r\na=1&b=2")
+
+	if payload = SetHost(payload, nil, []byte("new.com")); !bytes.Equal(payload, payloadAfter) {
+		t.Error("Should replace host", string(payload))
+	}
+
+	payload = []byte("POST /post HTTP/1.0\r\nContent-Length: 7\r\n\r\na=1&b=2")
+
+	if payload = SetHost(payload, nil, []byte("new.com")); !bytes.Equal(payload, payload) {
+		t.Error("Should replace host", string(payload))
+	}
+}
+
+func TestHasResponseTitle(t *testing.T) {
+	var m = map[string]bool{
+		"HTTP":                      false,
+		"":                          false,
+		"HTTP/1.1 100 Continue":     false,
+		"HTTP/1.1 100 Continue\r\n": true,
+		"HTTP/1.1  \r\n":            false,
+		"HTTP/4.0 100Continue\r\n":  false,
+		"HTTP/1.0 10r Continue\r\n": false,
+		"HTTP/1.1 200\r\n":          false,
+		"HTTP/1.1 200\r\nServer: Tengine\r\nContent-Length: 0\r\nConnection: close\r\n\r\n": false,
+	}
+	for k, v := range m {
+		if HasResponseTitle([]byte(k)) != v {
+			t.Errorf("%q should yield %v", k, v)
+			break
+		}
+	}
+}
+
+func TestHasRequestTitle(t *testing.T) {
+	var m = map[string]bool{
+		"POST /post HTTP/1.0\r\n": true,
+		"":                        false,
+		"POST /post HTTP/1.\r\n":  false,
+		"POS /post HTTP/1.1\r\n":  false,
+		"GET / HTTP/1.1\r\n":      true,
+		"GET / HTTP/1.1\r":        false,
+		"GET / HTTP/1.400\r\n":    false,
+	}
+	for k, v := range m {
+		if HasRequestTitle([]byte(k)) != v {
+			t.Errorf("%q should yield %v", k, v)
+			break
+		}
+	}
+}
+
+func TestCheckChunks(t *testing.T) {
+	var m = "4\r\nWiki\r\n5\r\npedia\r\nE\r\n in\r\n\r\nchunks.\r\n0\r\n\r\n"
+	chunkEnd := CheckChunked([]byte(m))
+	expected := bytes.Index([]byte(m), []byte("0\r\n")) + 5
+	if chunkEnd != expected {
+		t.Errorf("expected %d to equal %d", chunkEnd, expected)
+	}
+
+	m = "7\r\nMozia\r\n9\r\nDeveloper\r\n7\r\nNetwork\r\n0\r\n\r\n"
+	chunkEnd = CheckChunked([]byte(m))
+	if chunkEnd != -1 {
+		t.Errorf("expected %d to equal %d", chunkEnd, -1)
+	}
+
+	// with trailers
+	m = "4\r\nWiki\r\n5\r\npedia\r\nE\r\n in\r\n\r\nchunks.\r\n0\r\n\r\nEXpires"
+	chunkEnd = CheckChunked([]byte(m))
+	expected = bytes.Index([]byte(m), []byte("0\r\n")) + 5
+	if chunkEnd != expected {
+		t.Errorf("expected %d to equal %d", chunkEnd, expected)
+	}
+
+	// last chunk inside the the body
+	// with trailers
+	m = "4\r\nWiki\r\n5\r\npedia\r\nE\r\n in\r\n\r\nchunks.\r\n3\r\n0\r\n\r\n0\r\n\r\nEXpires"
+	chunkEnd = CheckChunked([]byte(m))
+	expected = bytes.Index([]byte(m), []byte("0\r\n")) + 10
+	if chunkEnd != expected {
+		t.Errorf("expected %d to equal %d", chunkEnd, expected)
+	}
+
+	// checks with chucks-extensions
+	m = "4\r\nWiki\r\n5\r\npedia\r\nE; name='quoted string'\r\n in\r\n\r\nchunks.\r\n3\r\n0\r\n\r\n0\r\n\r\nEXpires"
+	chunkEnd = CheckChunked([]byte(m))
+	expected = bytes.Index([]byte(m), []byte("0\r\n")) + 10
+	if chunkEnd != expected {
+		t.Errorf("expected %d to equal %d", chunkEnd, expected)
+	}
+}
+
+func TestHasFullPayload(t *testing.T) {
+	var m = "HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nTransfer-Encoding: chunked\r\n\r\n7\r\nMozilla\r\n9\r\nDeveloper\r\n7\r\nNetwork\r\n0\r\n\r\n"
+	got := HasFullPayload([]byte(m))
+	expected := true
+	if got != expected {
+		t.Errorf("expected %v to equal %v", got, expected)
+	}
+
+	// check with invalid chunk format
+	m = "HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nTransfer-Encoding: chunked\r\n\r\n7\r\nMozia\r\n9\r\nDeveloper\r\n7\r\nNetwork\r\n0\r\n\r\n"
+	got = HasFullPayload([]byte(m))
+	expected = false
+	if got != expected {
+		t.Errorf("expected %v to equal %v", got, expected)
+	}
+
+	// check chunks with trailers
+	m = "HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nTransfer-Encoding: chunked\r\nTrailer: Expires\r\n\r\n7\r\nMozilla\r\n9\r\nDeveloper\r\n7\r\nNetwork\r\n0\r\n\r\nExpires: Wed, 21 Oct 2015 07:28:00 GMT\r\n\r\n"
+	got = HasFullPayload([]byte(m))
+	expected = true
+	if got != expected {
+		t.Errorf("expected %v to equal %v", got, expected)
+	}
+
+	// check with missing trailers
+	m = "HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nTransfer-Encoding: chunked\r\nTrailer: Expires\r\n\r\n7\r\nMozilla\r\n9\r\nDeveloper\r\n7\r\nNetwork\r\n0\r\n\r\nExpires: Wed, 21 Oct 2015 07:28:00"
+	got = HasFullPayload([]byte(m))
+	expected = false
+	if got != expected {
+		t.Errorf("expected %v to equal %v", got, expected)
+	}
+
+	// check with content-length
+	m = "HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nContent-Length: 23\r\n\r\nMozillaDeveloperNetwork"
+	got = HasFullPayload([]byte(m))
+	expected = true
+	if got != expected {
+		t.Errorf("expected %v to equal %v", got, expected)
+	}
+
+	// check missing total length
+	m = "HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nContent-Length: 23\r\n\r\nMozillaDeveloperNet"
+	got = HasFullPayload([]byte(m))
+	expected = false
+	if got != expected {
+		t.Errorf("expected %v to equal %v", got, expected)
+	}
+
+	// check with no body
+	m = "HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\n\r\n"
+	got = HasFullPayload([]byte(m))
+	expected = true
+	if got != expected {
+		t.Errorf("expected %v to equal %v", got, expected)
+	}
+}
+
+func BenchmarkHasFullPayload(b *testing.B) {
+	var buf bytes.Buffer
+	buf.Write([]byte("HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nTransfer-Encoding: chunked\r\n\r\n"))
+	var chunk = []byte("1e\r\n111111111111111111111111111111\r\n")
+	for i := 0; i < 5000; i++ {
+		buf.Write(chunk)
+	}
+	buf.Write([]byte("0\r\n\r\n"))
+	data := buf.Bytes()
+	var ok bool
+	b.ResetTimer() // ignores the upper initialization
+	b.ReportMetric(float64(5000), "chunks/op")
+	for i := 0; i < b.N; i++ {
+		ok = HasFullPayload(data)
+	}
+	if !ok {
+		b.Fail()
+	}
 }

protocol.go

@@ -4,7 +4,7 @@ import (
 	"bytes"
 	"crypto/rand"
 	"encoding/hex"
-	"strconv"
+	"fmt"
 )
 
 // These constants help to indicate the type of payload
@@ -48,37 +48,9 @@ func payloadScanner(data []byte, atEOF bool) (advance int, token []byte, err err
 
 // Timing is request start or round-trip time, depending on payloadType
 func payloadHeader(payloadType byte, uuid []byte, timing int64, latency int64) (header []byte) {
-	var sTime, sLatency string
-
-	sTime = strconv.FormatInt(timing, 10)
-	if latency != -1 {
-		sLatency = strconv.FormatInt(latency, 10)
-	}
-
 	//Example:
-	//  3 f45590522cd1838b4a0d5c5aab80b77929dea3b3 1231\n
-	// `+ 1` indicates space characters or end of line
-	headerLen := 1 + 1 + len(uuid) + 1 + len(sTime) + 1
-
-	if latency != -1 {
-		headerLen += len(sLatency) + 1
-	}
-
-	header = make([]byte, headerLen)
-	header[0] = payloadType
-	header[1] = ' '
-	header[2+len(uuid)] = ' '
-	header[len(header)-1] = '\n'
-
-	copy(header[2:], uuid)
-	copy(header[3+len(uuid):], sTime)
-
-	if latency != -1 {
-		header[3+len(uuid)+len(sTime)] = ' '
-		copy(header[4+len(uuid)+len(sTime):], sLatency)
-	}
-
-	return header
+	//  3 f45590522cd1838b4a0d5c5aab80b77929dea3b3 13923489726487326 1231\n
+	return []byte(fmt.Sprintf("%c %s %d %d\n", payloadType, uuid, timing, latency))
 }
 
 func payloadBody(payload []byte) []byte {
@@ -89,28 +61,32 @@ func payloadBody(payload []byte) []byte {
 func payloadMeta(payload []byte) [][]byte {
 	headerSize := bytes.IndexByte(payload, '\n')
 	if headerSize < 0 {
-		headerSize = 0
+		return nil
 	}
 	return bytes.Split(payload[:headerSize], []byte{' '})
 }
 
-func payloadID(payload []byte) []byte {
-	idx := bytes.IndexByte(payload[2:], ' ')
-
-	if idx == -1 {
-		return []byte{}
+func payloadMetaWithBody(payload []byte) (meta, body []byte) {
+	if i := bytes.IndexByte(payload, '\n'); i > 0 && len(payload) > i+1 {
+		meta = payload[:i+1]
+		body = payload[i+1:]
+		return
 	}
+	// we assume the message did not have meta data
+	return nil, payload
+}
 
-	return payload[2 : 2+idx]
+func payloadID(payload []byte) (id []byte) {
+	meta := payloadMeta(payload)
+
+	if len(meta) < 2 {
+		return
+	}
+	return meta[1]
 }
 
 func isOriginPayload(payload []byte) bool {
-	switch payload[0] {
-	case RequestPayload, ResponsePayload:
-		return true
-	default:
-		return false
-	}
+	return payload[0] == RequestPayload || payload[0] == ResponsePayload
 }
 
 func isRequestPayload(payload []byte) bool {

settings.go

@@ -5,13 +5,13 @@ import (
 	"fmt"
 	"log"
 	"os"
-	"regexp"
-	"runtime"
-	"strconv"
+	"path/filepath"
+	"strings"
 	"sync"
 	"time"
 )
 
+// DEMO indicates that goreplay is running in demo mode
 var DEMO string
 
 // MultiOption allows to specify multiple flags with same name and collects all values into array
@@ -27,31 +27,13 @@ func (h *MultiOption) Set(value string) error {
 	return nil
 }
 
-type InputRAWConfig struct {
-	Engine        string        `json:"input-raw-engine"`
-	TrackResponse bool          `json:"input-raw-track-response"`
-	RealIPHeader  string        `json:"input-raw-realip-header"`
-	Expire        time.Duration `json:"input-raw-expire"`
-	Protocol      string        `json:"input-raw-protocol"`
-	BpfFilter     string        `json:"input-raw-bpf-filter"`
-	TimestampType string        `json:"input-raw-timestamp-type"`
-
-	ImmediateMode   bool `json:"input-raw-immediate-mode"`
-	BufferSize      int64
-	OverrideSnapLen bool   `json:"input-raw-override-snaplen"`
-	BufferSizeFlag  string `json:"input-raw-buffer-size"`
-}
-
 // AppSettings is the struct of main configuration
 type AppSettings struct {
-	Verbose   bool          `json:"verbose"`
-	Debug     bool          `json:"debug"`
-	Stats     bool          `json:"stats"`
+	Verbose   int           `json:"verbose"`
 	ExitAfter time.Duration `json:"exit-after"`
 
-	SplitOutput          bool   `json:"split-output"`
-	RecognizeTCPSessions bool   `json:"recognize-tcp-sessions"`
-	Pprof                string `json:"http-pprof"`
+	EmitterConfig
+	Pprof string `json:"http-pprof"`
 
 	InputDummy   MultiOption `json:"input-dummy"`
 	OutputDummy  MultiOption
@@ -62,21 +44,14 @@ type AppSettings struct {
 	InputTCPConfig  TCPInputConfig
 	OutputTCP       MultiOption `json:"output-tcp"`
 	OutputTCPConfig TCPOutputConfig
-	OutputTCPStats  bool `json:"output-tcp-stats"`
 
 	InputFile        MultiOption `json:"input-file"`
 	InputFileLoop    bool        `json:"input-file-loop"`
 	OutputFile       MultiOption `json:"output-file"`
 	OutputFileConfig FileOutputConfig
 
-	InputRAW       MultiOption `json:"input_raw"`
-	InputRAWConfig InputRAWConfig
-
-	copyBufferSize int64
-
-	OutputFileSizeFlag    string `json:"output-file-size-limit"`
-	OutputFileMaxSizeFlag string `json:"output-file-max-size-limit"`
-	CopyBufferSizeFlag    string `json:"copy-buffer-size"`
+	InputRAW MultiOption `json:"input_raw"`
+	RAWInputConfig
 
 	Middleware string `json:"middleware"`
 
@@ -93,6 +68,7 @@ type AppSettings struct {
 
 	InputKafkaConfig  InputKafkaConfig
 	OutputKafkaConfig OutputKafkaConfig
+	KafkaTLSConfig    KafkaTLSConfig
 }
 
 // Settings holds Gor configuration
@@ -104,13 +80,10 @@ func usage() {
 	os.Exit(2)
 }
 
-func init() {
+func initFlags() {
 	flag.Usage = usage
-
 	flag.StringVar(&Settings.Pprof, "http-pprof", "", "Enable profiling. Starts  http server on specified port, exposing special /debug/pprof endpoint. Example: `:8181`")
-	flag.BoolVar(&Settings.Verbose, "verbose", false, "Turn on more verbose output")
-	flag.BoolVar(&Settings.Debug, "debug", false, "Turn on debug output, shows all intercepted traffic. Works only when with `verbose` flag")
-	flag.BoolVar(&Settings.Stats, "stats", false, "Turn on queue stats output")
+	flag.IntVar(&Settings.Verbose, "verbose", 0, "set the level of verbosity, if greater than zero then it will turn on debug output")
 
 	if DEMO == "" {
 		flag.DurationVar(&Settings.ExitAfter, "exit-after", 0, "exit after specified duration")
@@ -118,15 +91,12 @@ func init() {
 		Settings.ExitAfter = 5 * time.Minute
 	}
 
+	flag.BoolVar(&Settings.EmitterConfig.Stats, "meta-stats", false, "Turn on stats for meta data.")
 	flag.BoolVar(&Settings.SplitOutput, "split-output", false, "By default each output gets same traffic. If set to `true` it splits traffic equally among all outputs.")
-
 	flag.BoolVar(&Settings.RecognizeTCPSessions, "recognize-tcp-sessions", false, "[PRO] If turned on http output will create separate worker for each TCP session. Splitting output will session based as well.")
 
 	flag.Var(&Settings.InputDummy, "input-dummy", "Used for testing outputs. Emits 'Get /' request every 1s")
-	flag.Var(&Settings.OutputDummy, "output-dummy", "DEPRECATED: use --output-stdout instead")
-
 	flag.BoolVar(&Settings.OutputStdout, "output-stdout", false, "Used for testing inputs. Just prints to console data coming from inputs.")
-
 	flag.BoolVar(&Settings.OutputNull, "output-null", false, "Used for testing inputs. Drops all requests.")
 
 	flag.Var(&Settings.InputTCP, "input-tcp", "Used for internal communication between Gor instances. Example: \n\t# Receive requests from other Gor instances on 28020 port, and redirect output to staging\n\tgor --input-tcp :28020 --output-http staging.com")
@@ -136,8 +106,9 @@ func init() {
 
 	flag.Var(&Settings.OutputTCP, "output-tcp", "Used for internal communication between Gor instances. Example: \n\t# Listen for requests on 80 port and forward them to other Gor instance on 28020 port\n\tgor --input-raw :80 --output-tcp replay.local:28020")
 	flag.BoolVar(&Settings.OutputTCPConfig.Secure, "output-tcp-secure", false, "Use TLS secure connection. --input-file on another end should have TLS turned on as well.")
+	flag.BoolVar(&Settings.OutputTCPConfig.SkipVerify, "output-tcp-skip-verify", false, "Don't verify hostname on TLS secure connection.")
 	flag.BoolVar(&Settings.OutputTCPConfig.Sticky, "output-tcp-sticky", false, "Use Sticky connection. Request/Response with same ID will be sent to the same connection.")
-	flag.BoolVar(&Settings.OutputTCPStats, "output-tcp-stats", false, "Report TCP output queue stats to console every 5 seconds.")
+	flag.IntVar(&Settings.OutputTCPConfig.Workers, "output-tcp-workers", 10, "Number of parallel tcp connections, default is 10")
 
 	flag.Var(&Settings.InputFile, "input-file", "Read requests from file: \n\tgor --input-file ./requests.gor --output-http staging.com")
 	flag.BoolVar(&Settings.InputFileLoop, "input-file-loop", false, "Loop input files, useful for performance testing.")
@@ -145,62 +116,53 @@ func init() {
 	flag.Var(&Settings.OutputFile, "output-file", "Write incoming requests to file: \n\tgor --input-raw :80 --output-file ./requests.gor")
 	flag.DurationVar(&Settings.OutputFileConfig.FlushInterval, "output-file-flush-interval", time.Second, "Interval for forcing buffer flush to the file, default: 1s.")
 	flag.BoolVar(&Settings.OutputFileConfig.Append, "output-file-append", false, "The flushed chunk is appended to existence file or not. ")
-	flag.StringVar(&Settings.OutputFileSizeFlag, "output-file-size-limit", "32mb", "Size of each chunk. Default: 32mb")
-	flag.Int64Var(&Settings.OutputFileConfig.QueueLimit, "output-file-queue-limit", 256, "The length of the chunk queue. Default: 256")
-	flag.StringVar(&Settings.OutputFileMaxSizeFlag, "output-file-max-size-limit", "1TB", "Max size of output file, Default: 1TB")
+	flag.Var(&Settings.OutputFileConfig.SizeLimit, "output-file-size-limit", "Size of each chunk. Default: 32mb")
+	flag.IntVar(&Settings.OutputFileConfig.QueueLimit, "output-file-queue-limit", 256, "The length of the chunk queue. Default: 256")
+	flag.Var(&Settings.OutputFileConfig.OutputFileMaxSize, "output-file-max-size-limit", "Max size of output file, Default: 1TB")
 
 	flag.StringVar(&Settings.OutputFileConfig.BufferPath, "output-file-buffer", "/tmp", "The path for temporary storing current buffer: \n\tgor --input-raw :80 --output-file s3://mybucket/logs/%Y-%m-%d.gz --output-file-buffer /mnt/logs")
 
 	flag.BoolVar(&Settings.PrettifyHTTP, "prettify-http", false, "If enabled, will automatically decode requests and responses with: Content-Encoding: gzip and Transfer-Encoding: chunked. Useful for debugging, in conjuction with --output-stdout")
 
+	// input raw flags
 	flag.Var(&Settings.InputRAW, "input-raw", "Capture traffic from given port (use RAW sockets and require *sudo* access):\n\t# Capture traffic from 8080 port\n\tgor --input-raw :8080 --output-http staging.com")
-
-	flag.BoolVar(&Settings.InputRAWConfig.TrackResponse, "input-raw-track-response", false, "If turned on Gor will track responses in addition to requests, and they will be available to middleware and file output.")
-
-	flag.StringVar(&Settings.InputRAWConfig.Engine, "input-raw-engine", "libpcap", "Intercept traffic using `libpcap` (default), and `raw_socket`")
-
-	flag.StringVar(&Settings.InputRAWConfig.Protocol, "input-raw-protocol", "http", "Specify application protocol of intercepted traffic. Possible values: http, binary")
-
-	flag.StringVar(&Settings.InputRAWConfig.RealIPHeader, "input-raw-realip-header", "", "If not blank, injects header with given name and real IP value to the request payload. Usually this header should be named: X-Real-IP")
-
-	flag.DurationVar(&Settings.InputRAWConfig.Expire, "input-raw-expire", time.Second*2, "How much it should wait for the last TCP packet, till consider that TCP message complete.")
-
-	flag.StringVar(&Settings.InputRAWConfig.BpfFilter, "input-raw-bpf-filter", "", "BPF filter to write custom expressions. Can be useful in case of non standard network interfaces like tunneling or SPAN port. Example: --input-raw-bpf-filter 'dst port 80'")
-
-	flag.StringVar(&Settings.InputRAWConfig.TimestampType, "input-raw-timestamp-type", "", "Possible values: PCAP_TSTAMP_HOST, PCAP_TSTAMP_HOST_LOWPREC, PCAP_TSTAMP_HOST_HIPREC, PCAP_TSTAMP_ADAPTER, PCAP_TSTAMP_ADAPTER_UNSYNCED. This values not supported on all systems, GoReplay will tell you available values of you put wrong one.")
-	flag.StringVar(&Settings.CopyBufferSizeFlag, "copy-buffer-size", "5mb", "Set the buffer size for an individual request (default 5MB)")
-	flag.BoolVar(&Settings.InputRAWConfig.OverrideSnapLen, "input-raw-override-snaplen", false, "Override the capture snaplen to be 64k. Required for some Virtualized environments")
-	flag.BoolVar(&Settings.InputRAWConfig.ImmediateMode, "input-raw-immediate-mode", false, "Set pcap interface to immediate mode.")
-	flag.StringVar(&Settings.InputRAWConfig.BufferSizeFlag, "input-raw-buffer-size", "0", "Controls size of the OS buffer which holds packets until they dispatched. Default value depends by system: in Linux around 2MB. If you see big package drop, increase this value.")
+	flag.BoolVar(&Settings.TrackResponse, "input-raw-track-response", false, "If turned on Gor will track responses in addition to requests, and they will be available to middleware and file output.")
+	flag.Var(&Settings.Engine, "input-raw-engine", "Intercept traffic using `libpcap` (default), `raw_socket` or `pcap_file`")
+	flag.Var(&Settings.Protocol, "input-raw-protocol", "Specify application protocol of intercepted traffic. Possible values: http, binary")
+	flag.StringVar(&Settings.RealIPHeader, "input-raw-realip-header", "", "If not blank, injects header with given name and real IP value to the request payload. Usually this header should be named: X-Real-IP")
+	flag.DurationVar(&Settings.Expire, "input-raw-expire", time.Second*2, "How much it should wait for the last TCP packet, till consider that TCP message complete.")
+	flag.StringVar(&Settings.BPFFilter, "input-raw-bpf-filter", "", "BPF filter to write custom expressions. Can be useful in case of non standard network interfaces like tunneling or SPAN port. Example: --input-raw-bpf-filter 'dst port 80'")
+	flag.StringVar(&Settings.TimestampType, "input-raw-timestamp-type", "", "Possible values: PCAP_TSTAMP_HOST, PCAP_TSTAMP_HOST_LOWPREC, PCAP_TSTAMP_HOST_HIPREC, PCAP_TSTAMP_ADAPTER, PCAP_TSTAMP_ADAPTER_UNSYNCED. This values not supported on all systems, GoReplay will tell you available values of you put wrong one.")
+	flag.Var(&Settings.CopyBufferSize, "copy-buffer-size", "Set the buffer size for an individual request (default 5MB)")
+	flag.BoolVar(&Settings.Snaplen, "input-raw-override-snaplen", false, "Override the capture snaplen to be 64k. Required for some Virtualized environments")
+	flag.DurationVar(&Settings.BufferTimeout, "input-raw-buffer-timeout", 0, "set the pcap timeout. for immediate mode don't set this flag")
+	flag.Var(&Settings.BufferSize, "input-raw-buffer-size", "Controls size of the OS buffer which holds packets until they dispatched. Default value depends by system: in Linux around 2MB. If you see big package drop, increase this value.")
+	flag.BoolVar(&Settings.Promiscuous, "input-raw-promisc", false, "enable promiscuous mode")
+	flag.BoolVar(&Settings.Monitor, "input-raw-monitor", false, "enable RF monitor mode")
+	flag.BoolVar(&Settings.RAWInputConfig.Stats, "input-raw-stats", false, "enable stats generator on raw TCP messages")
 
 	flag.StringVar(&Settings.Middleware, "middleware", "", "Used for modifying traffic using external command")
 
-	// flag.Var(&Settings.inputHTTP, "input-http", "Read requests from HTTP, should be explicitly sent from your application:\n\t# Listen for http on 9000\n\tgor --input-http :9000 --output-http staging.com")
-
 	flag.Var(&Settings.OutputHTTP, "output-http", "Forwards incoming requests to given http address.\n\t# Redirect all incoming requests to staging.com address \n\tgor --input-raw :80 --output-http http://staging.com")
 
 	/* outputHTTPConfig */
-	flag.IntVar(&Settings.OutputHTTPConfig.BufferSize, "output-http-response-buffer", 0, "HTTP response buffer size, all data after this size will be discarded.")
-	flag.BoolVar(&Settings.OutputHTTPConfig.CompatibilityMode, "output-http-compatibility-mode", false, "Use standard Go client, instead of built-in implementation. Can be slower, but more compatible.")
-
+	flag.Var(&Settings.OutputHTTPConfig.BufferSize, "output-http-response-buffer", "HTTP response buffer size, all data after this size will be discarded.")
 	flag.IntVar(&Settings.OutputHTTPConfig.WorkersMin, "output-http-workers-min", 0, "Gor uses dynamic worker scaling. Enter a number to set a minimum number of workers. default = 1.")
 	flag.IntVar(&Settings.OutputHTTPConfig.WorkersMax, "output-http-workers", 0, "Gor uses dynamic worker scaling. Enter a number to set a maximum number of workers. default = 0 = unlimited.")
 	flag.IntVar(&Settings.OutputHTTPConfig.QueueLen, "output-http-queue-len", 1000, "Number of requests that can be queued for output, if all workers are busy. default = 1000")
-
-	flag.IntVar(&Settings.OutputHTTPConfig.RedirectLimit, "output-http-redirect-limit", 0, "Enable how often redirects should be followed.")
+	flag.BoolVar(&Settings.OutputHTTPConfig.SkipVerify, "output-http-skip-verify", false, "Don't verify hostname on TLS secure connection.")
+	flag.DurationVar(&Settings.OutputHTTPConfig.WorkerTimeout, "output-http-worker-timeout", 2*time.Second, "Duration to rollback idle workers.")
+	flag.IntVar(&Settings.OutputHTTPConfig.RedirectLimit, "output-http-redirects", 0, "Enable how often redirects should be followed.")
 	flag.DurationVar(&Settings.OutputHTTPConfig.Timeout, "output-http-timeout", 5*time.Second, "Specify HTTP request/response timeout. By default 5s. Example: --output-http-timeout 30s")
 	flag.BoolVar(&Settings.OutputHTTPConfig.TrackResponses, "output-http-track-response", false, "If turned on, HTTP output responses will be set to all outputs like stdout, file and etc.")
-
-	flag.BoolVar(&Settings.OutputHTTPConfig.Stats, "output-http-stats", false, "Report http output queue stats to console every N milliseconds. See output-http-stats-ms")
-	flag.IntVar(&Settings.OutputHTTPConfig.StatsMs, "output-http-stats-ms", 5000, "Report http output queue stats to console every N milliseconds. default: 5000")
-	flag.BoolVar(&Settings.OutputHTTPConfig.OriginalHost, "output-http-original-host", false, "Normally gor replaces the Host http header with the Host supplied with --output-http.  This option disables that behavior, preserving the original Host header.")
-	flag.BoolVar(&Settings.OutputHTTPConfig.Debug, "output-http-debug", false, "Enables http debug output.")
+	flag.BoolVar(&Settings.OutputHTTPConfig.OriginalHost, "http-original-host", false, "Normally gor replaces the Host http header with the host supplied with --output-http.  This option disables that behavior, preserving the original Host header.")
 	flag.StringVar(&Settings.OutputHTTPConfig.ElasticSearch, "output-http-elasticsearch", "", "Send request and response stats to ElasticSearch:\n\tgor --input-raw :8080 --output-http staging.com --output-http-elasticsearch 'es_host:api_port/index_name'")
 	/* outputHTTPConfig */
 
 	flag.Var(&Settings.OutputBinary, "output-binary", "Forwards incoming binary payloads to given address.\n\t# Redirect all incoming requests to staging.com address \n\tgor --input-raw :80 --input-raw-protocol binary --output-binary staging.com:80")
+
 	/* outputBinaryConfig */
-	flag.IntVar(&Settings.OutputBinaryConfig.BufferSize, "output-tcp-response-buffer", 0, "TCP response buffer size, all data after this size will be discarded.")
+	flag.Var(&Settings.OutputBinaryConfig.BufferSize, "output-tcp-response-buffer", "TCP response buffer size, all data after this size will be discarded.")
 	flag.IntVar(&Settings.OutputBinaryConfig.Workers, "output-binary-workers", 0, "Gor uses dynamic worker scaling by default.  Enter a number to run a set number of workers.")
 	flag.DurationVar(&Settings.OutputBinaryConfig.Timeout, "output-binary-timeout", 0, "Specify HTTP request/response timeout. By default 5s. Example: --output-binary-timeout 30s")
 	flag.BoolVar(&Settings.OutputBinaryConfig.TrackResponses, "output-binary-track-response", false, "If turned on, Binary output responses will be set to all outputs like stdout, file and etc.")
@@ -208,158 +170,91 @@ func init() {
 	flag.BoolVar(&Settings.OutputBinaryConfig.Debug, "output-binary-debug", false, "Enables binary debug output.")
 	/* outputBinaryConfig */
 
-	flag.StringVar(&Settings.OutputKafkaConfig.Host, "output-kafka-host", "", "Read request and response stats from Kafka:\n\tgor --input-raw :8080 --output-kafka-Host '192.168.0.1:9092,192.168.0.2:9092'")
-	flag.StringVar(&Settings.OutputKafkaConfig.Topic, "output-kafka-topic", "", "Read request and response stats from Kafka:\n\tgor --input-raw :8080 --output-kafka-Topic 'kafka-log'")
+	flag.StringVar(&Settings.OutputKafkaConfig.Host, "output-kafka-host", "", "Read request and response stats from Kafka:\n\tgor --input-raw :8080 --output-kafka-host '192.168.0.1:9092,192.168.0.2:9092'")
+	flag.StringVar(&Settings.OutputKafkaConfig.Topic, "output-kafka-topic", "", "Read request and response stats from Kafka:\n\tgor --input-raw :8080 --output-kafka-topic 'kafka-log'")
 	flag.BoolVar(&Settings.OutputKafkaConfig.UseJSON, "output-kafka-json-format", false, "If turned on, it will serialize messages from GoReplay text format to JSON.")
 
-	flag.StringVar(&Settings.InputKafkaConfig.Host, "input-kafka-host", "", "Send request and response stats to Kafka:\n\tgor --output-stdout --input-kafka-Host '192.168.0.1:9092,192.168.0.2:9092'")
-	flag.StringVar(&Settings.InputKafkaConfig.Topic, "input-kafka-topic", "", "Send request and response stats to Kafka:\n\tgor --output-stdout --input-kafka-Topic 'kafka-log'")
+	flag.StringVar(&Settings.InputKafkaConfig.Host, "input-kafka-host", "", "Send request and response stats to Kafka:\n\tgor --output-stdout --input-kafka-host '192.168.0.1:9092,192.168.0.2:9092'")
+	flag.StringVar(&Settings.InputKafkaConfig.Topic, "input-kafka-topic", "", "Send request and response stats to Kafka:\n\tgor --output-stdout --input-kafka-topic 'kafka-log'")
 	flag.BoolVar(&Settings.InputKafkaConfig.UseJSON, "input-kafka-json-format", false, "If turned on, it will assume that messages coming in JSON format rather than  GoReplay text format.")
 
+	flag.StringVar(&Settings.KafkaTLSConfig.CACert, "kafka-tls-ca-cert", "", "CA certificate for Kafka TLS Config:\n\tgor  --input-raw :3000 --output-kafka-host '192.168.0.1:9092' --output-kafka-topic 'topic' --kafka-tls-ca-cert cacert.cer.pem --kafka-tls-client-cert client.cer.pem --kafka-tls-client-key client.key.pem")
+	flag.StringVar(&Settings.KafkaTLSConfig.ClientCert, "kafka-tls-client-cert", "", "Client certificate for Kafka TLS Config (mandatory with to kafka-tls-ca-cert and kafka-tls-client-key)")
+	flag.StringVar(&Settings.KafkaTLSConfig.ClientKey, "kafka-tls-client-key", "", "Client Key for Kafka TLS Config (mandatory with to kafka-tls-client-cert and kafka-tls-client-key)")
+
 	flag.Var(&Settings.ModifierConfig.Headers, "http-set-header", "Inject additional headers to http reqest:\n\tgor --input-raw :8080 --output-http staging.com --http-set-header 'User-Agent: Gor'")
-	flag.Var(&Settings.ModifierConfig.Headers, "output-http-header", "WARNING: `--output-http-header` DEPRECATED, use `--http-set-header` instead")
-
 	flag.Var(&Settings.ModifierConfig.HeaderRewrite, "http-rewrite-header", "Rewrite the request header based on a mapping:\n\tgor --input-raw :8080 --output-http staging.com --http-rewrite-header Host: (.*).example.com,$1.beta.example.com")
-
 	flag.Var(&Settings.ModifierConfig.Params, "http-set-param", "Set request url param, if param already exists it will be overwritten:\n\tgor --input-raw :8080 --output-http staging.com --http-set-param api_key=1")
-
 	flag.Var(&Settings.ModifierConfig.Methods, "http-allow-method", "Whitelist of HTTP methods to replay. Anything else will be dropped:\n\tgor --input-raw :8080 --output-http staging.com --http-allow-method GET --http-allow-method OPTIONS")
-	flag.Var(&Settings.ModifierConfig.Methods, "output-http-method", "WARNING: `--output-http-method` DEPRECATED, use `--http-allow-method` instead")
-
-	flag.Var(&Settings.ModifierConfig.UrlRegexp, "http-allow-url", "A regexp to match requests against. Filter get matched against full url with domain. Anything else will be dropped:\n\t gor --input-raw :8080 --output-http staging.com --http-allow-url ^www.")
-	flag.Var(&Settings.ModifierConfig.UrlRegexp, "output-http-url-regexp", "WARNING: `--output-http-url-regexp` DEPRECATED, use `--http-allow-url` instead")
-
-	flag.Var(&Settings.ModifierConfig.UrlNegativeRegexp, "http-disallow-url", "A regexp to match requests against. Filter get matched against full url with domain. Anything else will be forwarded:\n\t gor --input-raw :8080 --output-http staging.com --http-disallow-url ^www.")
-
-	flag.Var(&Settings.ModifierConfig.UrlRewrite, "http-rewrite-url", "Rewrite the request url based on a mapping:\n\tgor --input-raw :8080 --output-http staging.com --http-rewrite-url /v1/user/([^\\/]+)/ping:/v2/user/$1/ping")
-	flag.Var(&Settings.ModifierConfig.UrlRewrite, "output-http-rewrite-url", "WARNING: `--output-http-rewrite-url` DEPRECATED, use `--http-rewrite-url` instead")
-
+	flag.Var(&Settings.ModifierConfig.URLRegexp, "http-allow-url", "A regexp to match requests against. Filter get matched against full url with domain. Anything else will be dropped:\n\t gor --input-raw :8080 --output-http staging.com --http-allow-url ^www.")
+	flag.Var(&Settings.ModifierConfig.URLNegativeRegexp, "http-disallow-url", "A regexp to match requests against. Filter get matched against full url with domain. Anything else will be forwarded:\n\t gor --input-raw :8080 --output-http staging.com --http-disallow-url ^www.")
+	flag.Var(&Settings.ModifierConfig.URLRewrite, "http-rewrite-url", "Rewrite the request url based on a mapping:\n\tgor --input-raw :8080 --output-http staging.com --http-rewrite-url /v1/user/([^\\/]+)/ping:/v2/user/$1/ping")
 	flag.Var(&Settings.ModifierConfig.HeaderFilters, "http-allow-header", "A regexp to match a specific header against. Requests with non-matching headers will be dropped:\n\t gor --input-raw :8080 --output-http staging.com --http-allow-header api-version:^v1")
-	flag.Var(&Settings.ModifierConfig.HeaderFilters, "output-http-header-filter", "WARNING: `--output-http-header-filter` DEPRECATED, use `--http-allow-header` instead")
-
 	flag.Var(&Settings.ModifierConfig.HeaderNegativeFilters, "http-disallow-header", "A regexp to match a specific header against. Requests with matching headers will be dropped:\n\t gor --input-raw :8080 --output-http staging.com --http-disallow-header \"User-Agent: Replayed by Gor\"")
-
 	flag.Var(&Settings.ModifierConfig.HeaderBasicAuthFilters, "http-basic-auth-filter", "A regexp to match the decoded basic auth string against. Requests with non-matching headers will be dropped:\n\t gor --input-raw :8080 --output-http staging.com --http-basic-auth-filter \"^customer[0-9].*\"")
-
 	flag.Var(&Settings.ModifierConfig.HeaderHashFilters, "http-header-limiter", "Takes a fraction of requests, consistently taking or rejecting a request based on the FNV32-1A hash of a specific header:\n\t gor --input-raw :8080 --output-http staging.com --http-header-limiter user-id:25%")
-
-	flag.Var(&Settings.ModifierConfig.HeaderHashFilters, "output-http-header-hash-filter", "WARNING: `output-http-header-hash-filter` DEPRECATED, use `--http-header-hash-limiter` instead")
-
 	flag.Var(&Settings.ModifierConfig.ParamHashFilters, "http-param-limiter", "Takes a fraction of requests, consistently taking or rejecting a request based on the FNV32-1A hash of a specific GET param:\n\t gor --input-raw :8080 --output-http staging.com --http-param-limiter user_id:25%")
 
 	// default values, using for tests
-	Settings.OutputFileConfig.sizeLimit = 33554432
-	Settings.OutputFileConfig.outputFileMaxSize = 1099511627776
-	Settings.copyBufferSize = 5242880
-	Settings.InputRAWConfig.BufferSize = 0
+	Settings.OutputFileConfig.SizeLimit = 33554432
+	Settings.OutputFileConfig.OutputFileMaxSize = 1099511627776
+	Settings.CopyBufferSize = 5242880
 
 }
 
 func checkSettings() {
-	outputFileSize, err := bufferParser(Settings.OutputFileSizeFlag, "32MB")
-	if err != nil {
-		log.Fatalf("output-file-size-limit error: %v\n", err)
+	if Settings.OutputFileConfig.SizeLimit < 1 {
+		Settings.OutputFileConfig.SizeLimit.Set("32mb")
 	}
-	Settings.OutputFileConfig.sizeLimit = outputFileSize
-
-	outputFileMaxSize, err := bufferParser(Settings.OutputFileMaxSizeFlag, "1TB")
-	if err != nil {
-		log.Fatalf("output-file-max-size-limit error: %v\n", err)
+	if Settings.OutputFileConfig.OutputFileMaxSize < 1 {
+		Settings.OutputFileConfig.OutputFileMaxSize.Set("1tb")
 	}
-	Settings.OutputFileConfig.outputFileMaxSize = outputFileMaxSize
-
-	copyBufferSize, err := bufferParser(Settings.CopyBufferSizeFlag, "5mb")
-	if err != nil {
-		log.Fatalf("copy-buffer-size error: %v\n", err)
-	}
-	Settings.copyBufferSize = copyBufferSize
-
-	inputRAWBufferSize, err := bufferParser(Settings.InputRAWConfig.BufferSizeFlag, "0")
-	if err != nil {
-		log.Fatalf("input-raw-buffer-size error: %v\n", err)
-	}
-	Settings.InputRAWConfig.BufferSize = inputRAWBufferSize
-
-	// libpcap has bug in mac os x. More info: https://github.com/buger/goreplay/issues/730
-	if Settings.InputRAWConfig.Expire == time.Second*2 && runtime.GOOS == "darwin" {
-		Settings.InputRAWConfig.Expire = time.Second
+	if Settings.CopyBufferSize < 1 {
+		Settings.CopyBufferSize.Set("5mb")
 	}
 }
 
 var previousDebugTime = time.Now()
 var debugMutex sync.Mutex
-var pID = os.Getpid()
 
-// Debug take an effect only if --verbose flag specified
-func Debug(args ...interface{}) {
-	if Settings.Verbose {
+// Debug take an effect only if --verbose greater than 0 is specified
+func Debug(level int, args ...interface{}) {
+	if Settings.Verbose >= level {
 		debugMutex.Lock()
 		defer debugMutex.Unlock()
 		now := time.Now()
-		diff := now.Sub(previousDebugTime).String()
+		diff := now.Sub(previousDebugTime)
 		previousDebugTime = now
-		fmt.Printf("[DEBUG][PID %d][%s][elapsed %s] ", pID, now.Format(time.StampNano), diff)
-		fmt.Println(args...)
+		fmt.Fprintf(os.Stderr, "[DEBUG][elapsed %s]: ", diff)
+		fmt.Fprintln(os.Stderr, args...)
 	}
 }
 
-// the following regexes follow Go semantics https://golang.org/ref/spec#Letters_and_digits
-var (
-	rB   = regexp.MustCompile(`(?i)^(?:0b|0x|0o)?[\da-f_]+$`)
-	rKB  = regexp.MustCompile(`(?i)^(?:0b|0x|0o)?[\da-f_]+kb$`)
-	rMB  = regexp.MustCompile(`(?i)^(?:0b|0x|0o)?[\da-f_]+mb$`)
-	rGB  = regexp.MustCompile(`(?i)^(?:0b|0x|0o)?[\da-f_]+gb$`)
-	rTB  = regexp.MustCompile(`(?i)^(?:0b|0x|0o)?[\da-f_]+tb$`)
-	empt = regexp.MustCompile(`^[\n\t\r 0.\f\a]*$`)
-)
-
-// bufferParser parses buffer to bytes from different bases and data units
-// size is the buffer in string, rpl act as a replacement for empty buffer.
-// e.g: (--output-file-size-limit "") may override default 32mb with empty buffer,
-// which can be solved by setting rpl by bufferParser(buffer, "32mb")
-func bufferParser(size, rpl string) (buffer int64, err error) {
-	const (
-		_ = 1 << (iota * 10)
-		KB
-		MB
-		GB
-		TB
-	)
-
-	var (
-		lmt = len(size) - 2
-		s   = []byte(size)
-	)
-
-	if empt.Match(s) {
-		size = rpl
-		s = []byte(size)
-	}
-
-	// recover, especially when buffer size overflows int64 i.e ~8019PBs
-	defer func() {
-		if e, ok := recover().(error); ok {
-			err = e.(error)
+func initGorDir() {
+	var err error
+	p := os.Getenv("GORDIR")
+	if p == "" {
+		if p, err = os.UserHomeDir(); err != nil {
+			goto errLog
 		}
-	}()
-
-	switch {
-	case rB.Match(s):
-		buffer, err = strconv.ParseInt(size, 0, 64)
-	case rKB.Match(s):
-		buffer, err = strconv.ParseInt(size[:lmt], 0, 64)
-		buffer *= KB
-	case rMB.Match(s):
-		buffer, err = strconv.ParseInt(size[:lmt], 0, 64)
-		buffer *= MB
-	case rGB.Match(s):
-		buffer, err = strconv.ParseInt(size[:lmt], 0, 64)
-		buffer *= GB
-	case rTB.Match(s):
-		buffer, err = strconv.ParseInt(size[:lmt], 0, 64)
-		buffer *= TB
-	default:
-		return 0, fmt.Errorf("invalid buffer %q", size)
+		if !strings.Contains(p, "root") {
+			p = filepath.Join(p, ".gor")
+		} else {
+			p = ".gor"
+		}
+	}
+	if err = os.MkdirAll(p, os.ModeDir|os.ModePerm); err != nil {
+		goto errLog
+	}
+	if err = os.Setenv("GORDIR", p); err != nil {
+		goto errLog
 	}
 	return
+errLog:
+	log.Fatalf("init GORDIR error:%q", err)
+}
+
+func init() {
+	initGorDir()
+	initFlags()
 }

settings_test.go

@@ -2,15 +2,13 @@ package main
 
 import (
 	"encoding/json"
-	"fmt"
 	"testing"
 )
 
 func TestAppSettings(t *testing.T) {
 	a := AppSettings{}
-	data, err := json.Marshal(&a)
+	_, err := json.Marshal(&a)
 	if err != nil {
-		panic(err)
+		t.Error(err)
 	}
-	fmt.Printf(string(data))
 }

size/size.go

@@ -0,0 +1,64 @@
+package size
+
+import (
+	"fmt"
+	"regexp"
+	"strconv"
+)
+
+// Size represents size that implements flag.Var
+type Size int64
+
+// the following regexes follow Go semantics https://golang.org/ref/spec#Letters_and_digits
+var (
+	rB  = regexp.MustCompile(`(?i)^(?:0b|0x|0o)?[\da-f_]+$`)
+	rKB = regexp.MustCompile(`(?i)^(?:0b|0x|0o)?[\da-f_]+kb$`)
+	rMB = regexp.MustCompile(`(?i)^(?:0b|0x|0o)?[\da-f_]+mb$`)
+	rGB = regexp.MustCompile(`(?i)^(?:0b|0x|0o)?[\da-f_]+gb$`)
+	rTB = regexp.MustCompile(`(?i)^(?:0b|0x|0o)?[\da-f_]+tb$`)
+)
+
+// Set parses size to integer from different bases and data units
+func (siz *Size) Set(size string) (err error) {
+	if size == "" {
+		return
+	}
+	const (
+		_ = 1 << (iota * 10)
+		KB
+		MB
+		GB
+		TB
+	)
+
+	var (
+		lmt = len(size) - 2
+		s   = []byte(size)
+	)
+
+	var _len int64
+	switch {
+	case rB.Match(s):
+		_len, err = strconv.ParseInt(size, 0, 64)
+	case rKB.Match(s):
+		_len, err = strconv.ParseInt(size[:lmt], 0, 64)
+		_len *= KB
+	case rMB.Match(s):
+		_len, err = strconv.ParseInt(size[:lmt], 0, 64)
+		_len *= MB
+	case rGB.Match(s):
+		_len, err = strconv.ParseInt(size[:lmt], 0, 64)
+		_len *= GB
+	case rTB.Match(s):
+		_len, err = strconv.ParseInt(size[:lmt], 0, 64)
+		_len *= TB
+	default:
+		return fmt.Errorf("invalid _len %q", size)
+	}
+	*siz = Size(_len)
+	return
+}
+
+func (siz *Size) String() string {
+	return fmt.Sprintf("%d", *siz)
+}

size/size_test.go

@@ -0,0 +1,29 @@
+package size
+
+import "testing"
+
+func TestParseDataUnit(t *testing.T) {
+	var d = map[string]int{
+		"42mb":                 42 << 20,
+		"4_2":                  42,
+		"00":                   0,
+		"0":                    0,
+		"0_600tb":              384 << 40,
+		"0600Tb":               384 << 40,
+		"0o12Mb":               10 << 20,
+		"0b_10010001111_1kb":   2335 << 10,
+		"1024":                 1 << 10,
+		"0b111":                7,
+		"0x12gB":               18 << 30,
+		"0x_67_7a_2f_cc_40_c6": 113774485586118,
+		"121562380192901":      121562380192901,
+	}
+	var buf Size
+	var err error
+	for k, v := range d {
+		err = buf.Set(k)
+		if err != nil || buf != Size(v) {
+			t.Errorf("Error parsing %s: %v", k, err)
+		}
+	}
+}

stat/csv/writer.go

@@ -0,0 +1,47 @@
+package csv
+
+import (
+	"encoding/csv"
+	"os"
+	"strings"
+)
+
+// Writer check encoding/csv.Writer
+type Writer struct {
+	*csv.Writer
+	f *os.File
+}
+
+// NewWriter returns an csv writer with file opened
+// error returned will be associated with opening the file
+func NewWriter(path string) (*Writer, error) {
+	var wr Writer
+	f, err := os.Create(path)
+	if err != nil {
+		return nil, err
+	}
+	wr.f = f
+	wr.Writer = csv.NewWriter(f)
+	return &wr, nil
+}
+
+// WriteHeader will write infos about this docs in form of comments
+// it uses "#" to indicate that this should be ignored by the reader
+func (wr *Writer) WriteHeader(hdr string) error {
+	_, err := wr.f.WriteString(formatHeader(hdr))
+	return err
+}
+
+// Close the underlying file
+func (wr *Writer) Close() error {
+	return wr.f.Close()
+}
+
+func formatHeader(hdr string) string {
+	if !strings.HasSuffix(hdr, "\n") {
+		hdr += "\n"
+	}
+	// add comments before every line
+	hdr = strings.ReplaceAll("#"+hdr, "\n", "\n#")
+	return hdr[:len(hdr)-1]
+}

stat/csv/writer_test.go

@@ -0,0 +1,23 @@
+package csv
+
+import (
+	"testing"
+)
+
+func TestFormatHeader(t *testing.T) {
+	m := map[string]string{
+		"hello":                       "#hello\n",
+		"":                            "#\n",
+		"hello\n":                     "#hello\n",
+		"hello\nhello\n":              "#hello\n#hello\n",
+		"hello\nhello":                "#hello\n#hello\n",
+		"hello\nhello\nhello":         "#hello\n#hello\n#hello\n",
+		"hello\nhello\nhello\n":       "#hello\n#hello\n#hello\n",
+		"hello\nhello\r\n\r\nhello\n": "#hello\n#hello\r\n#\r\n#hello\n",
+	}
+	for k, v := range m {
+		if vl := formatHeader(k); vl != v {
+			t.Errorf("expected %q to equal %q", vl, v)
+		}
+	}
+}

stat/writer.go

@@ -0,0 +1,124 @@
+package stat
+
+import (
+	"errors"
+	"sync"
+	"time"
+)
+
+// ErrNotImplemented returned when impls requested are not implemnted
+var ErrNotImplemented = errors.New("method not implemented")
+
+// RecordWriter represents a stat writer it should have all this method.
+// for rationale  behind this interface check encoding/csv or
+type RecordWriter interface {
+	Write(record []string) error
+	Flush()
+	Error() error
+}
+
+const flushSpan = 10
+
+// Writer represents a single instance of goreplay stats writer
+type Writer struct {
+	name    string
+	records [flushSpan][]string // a temporary stats holder
+	off     int
+	quit    chan struct{}
+	mu      sync.Mutex
+	w       RecordWriter
+	err     error
+}
+
+// NewWriter returns a new instance of stats associated with the writer "wr"
+// and "name" as the stats identity
+func NewWriter(w RecordWriter, name string) (wr *Writer) {
+	wr = new(Writer)
+	wr.name = name
+	wr.w = w
+	wr.quit = make(chan struct{})
+	return
+}
+
+// WriteHeader writes header of the documents to the underlying writer.
+// Header can be any kind of data like json unmarshal or file info.
+// underlying writer should sets a different way to represents the header.
+func (wr *Writer) WriteHeader(header string) error {
+	if wHdr, ok := wr.w.(interface{ WriteHeader(header string) error }); ok {
+		return wHdr.WriteHeader(header)
+	}
+	return ErrNotImplemented
+}
+
+// SetReader sets another source of records reader to use along with (*Writer).Write([]string)
+// rd function will be called every "rate" duration time
+func (wr *Writer) SetReader(rd func() []string, rate time.Duration) {
+	go func() {
+		timer := time.NewTicker(rate)
+		defer timer.Stop()
+		for {
+			select {
+			case <-wr.quit:
+				return
+			case <-timer.C:
+				err := wr.Write(rd())
+				if err != nil {
+					return
+				}
+			}
+		}
+	}()
+}
+
+func (wr *Writer) Write(record []string) error {
+	wr.mu.Lock()
+	defer wr.mu.Unlock()
+	if wr.err != nil {
+		return wr.err
+	}
+	if len(record) > 0 {
+		wr.records[wr.off] = record
+		wr.off++
+	}
+	return wr.flush(wr.off >= flushSpan)
+}
+
+func (wr *Writer) String() string {
+	return "stats: " + wr.name
+}
+
+// Error reports any error that has occurred during a previous Write or Flush.
+func (wr *Writer) Error() error {
+	wr.mu.Lock()
+	wr.mu.Unlock()
+	return wr.err
+}
+
+// Close closes and flush all data of the stat
+func (wr *Writer) Close() error {
+	wr.mu.Lock()
+	defer wr.mu.Unlock()
+	close(wr.quit)
+	wr.err = wr.flush(true)
+	if cl, ok := wr.w.(interface{ Close() error }); ok {
+		wr.err = cl.Close()
+	}
+	return wr.err
+}
+
+func (wr *Writer) flush(f bool) error {
+	if !f {
+		return nil
+	}
+	i := 0
+	for i < wr.off {
+		if wr.err = wr.w.Write(wr.records[i]); wr.err != nil {
+			break
+		}
+		i++
+	}
+	wr.off -= i
+	wr.w.Flush()
+	wr.err = wr.w.Error()
+	return wr.err
+}

stat/writer_test.go

@@ -0,0 +1,33 @@
+package stat
+
+import "testing"
+
+type testWriter struct {
+	f  func([]string)
+	fl func()
+}
+
+func (t testWriter) WriteHeader(_ string) error { return nil }
+
+func (t testWriter) Flush() { t.fl() }
+
+func (t testWriter) Error() error { return nil }
+
+func (t testWriter) Write(r []string) error {
+	t.f(r)
+	return nil
+}
+
+func TestTempSize(t *testing.T) {
+	count := 0
+	tW := testWriter{f: func(_ []string) { count++ }, fl: func() {
+		if count != flushSpan {
+			t.Errorf("should flush after %d records", flushSpan)
+		}
+	}}
+	s := NewWriter(tW, "")
+	defer s.Close()
+	for i := 0; i < flushSpan; i++ {
+		s.Write([]string{"1"})
+	}
+}

stats.go

@@ -0,0 +1,35 @@
+package main
+
+import (
+	"log"
+	"os"
+	"path/filepath"
+
+	"github.com/buger/goreplay/stat"
+	"github.com/buger/goreplay/stat/csv"
+)
+
+func statsWriteRecord(st *stat.Writer, record []string, plugin string) {
+	if st == nil {
+		return
+	}
+	if st.Error() != nil {
+		Debug(1, plugin, " stats closed with error: ", st.Error())
+		st.Close()
+		st = nil
+		return
+	}
+	st.Write(record)
+}
+
+func initializeStatWriter(file string, hdr string) *stat.Writer {
+	csvWr, err := csv.NewWriter(filepath.Join(os.Getenv("GORDIR"), file))
+	if err != nil {
+		log.Fatalf("init meta stat writer error: %q", err)
+	}
+	st := stat.NewWriter(csvWr, "meta")
+	if hdr != "" {
+		st.WriteHeader("type: GOR_STATS\ntitle: message meta")
+	}
+	return st
+}

tcp/doc.go

@@ -0,0 +1,27 @@
+/*
+Package tcp implements TCP transport layer protocol, it is responsible for
+parsing, reassembling tcp packets, handling communication with engine listeners(github.com/buger/goreplay/capture),
+and reporting errors and statistics of packets.
+the packets are parsed by following TCP way(https://en.wikipedia.org/wiki/Transmission_Control_Protocol#TCP_segment_structure).
+
+
+example:
+
+import "github.com/buger/goreplay/tcp"
+
+messageExpire := time.Second*5
+maxSize := 5 << 20
+
+debugger := func(debugLevel int, data ...interface{}){} // debugger can also be nil
+messageHandler := func(mssg *tcp.Message){}
+
+mssgPool := tcp.NewMessagePool(maxMessageSize, messageExpire, debugger, messageHandler)
+listener.Listen(ctx, mssgPool.Handler)
+
+you can use pool.End or/and pool.Start to set custom session behaviors
+
+debugLevel in debugger function indicates the priority of the logs, the bigger the number the lower
+the priority. errors are signified by debug level 4 for errors, 5 for discarded packets, and 6 for received packets.
+
+*/
+package tcp // import github.com/buger/goreplay/tcp

tcp/tcp_message.go

@@ -0,0 +1,291 @@
+package tcp
+
+import (
+	"encoding/binary"
+	"encoding/hex"
+	"fmt"
+	"sort"
+	"sync"
+	"time"
+
+	"github.com/buger/goreplay/size"
+	"github.com/google/gopacket"
+)
+
+// Stats every message carry its own stats object
+type Stats struct {
+	LostData   int
+	Length     int       // length of the data
+	Start      time.Time // first packet's timestamp
+	End        time.Time // last packet's timestamp
+	SrcAddr    string
+	DstAddr    string
+	IsIncoming bool
+	TimedOut   bool // timeout before getting the whole message
+	Truncated  bool // last packet truncated due to max message size
+	IPversion  byte
+}
+
+// Message is the representation of a tcp message
+type Message struct {
+	packets []*Packet
+	done    chan bool
+	pool    *MessagePool
+	Stats
+}
+
+// NewMessage ...
+func NewMessage(srcAddr, dstAddr string, ipVersion uint8) (m *Message) {
+	m = new(Message)
+	m.DstAddr = dstAddr
+	m.SrcAddr = srcAddr
+	m.IPversion = ipVersion
+	m.done = make(chan bool)
+	return
+}
+
+// UUID returns the UUID of a TCP request and its response.
+func (m *Message) UUID() []byte {
+	var key uint64
+	pckt := m.packets[0]
+
+	// check if response or request have generated the ID before.
+	if m.pool.uuids != nil {
+		lst := len(pckt.SrcIP) - 4
+		if m.IsIncoming {
+			key = uint64(pckt.SrcPort)<<48 | uint64(pckt.DstPort)<<32 |
+				uint64(_uint32(pckt.SrcIP[lst:]))
+		} else {
+			key = uint64(pckt.DstPort)<<48 | uint64(pckt.SrcPort)<<32 |
+				uint64(_uint32(pckt.DstIP[lst:]))
+		}
+		if uuidHex, ok := m.pool.uuids[key]; ok {
+			delete(m.pool.uuids, key)
+			return uuidHex
+		}
+	}
+
+	id := make([]byte, 12, 12)
+	binary.BigEndian.PutUint32(id, pckt.Seq)
+	tStamp := m.End.UnixNano()
+	binary.BigEndian.PutUint64(id[4:], uint64(tStamp))
+	uuidHex := make([]byte, 24, 24)
+	hex.Encode(uuidHex[:], id[:])
+	if m.pool.uuids != nil {
+		if len(m.pool.uuids) >= 1000 {
+			m.pool.cleanUUIDs()
+		}
+		m.pool.uuids[key] = uuidHex
+	}
+	return uuidHex
+}
+
+func (m *Message) add(pckt *Packet) {
+	m.Length += len(pckt.Payload)
+	m.LostData += int(pckt.Lost)
+	m.packets = append(m.packets, pckt)
+	m.End = pckt.Timestamp
+}
+
+// Packets returns packets of this message
+func (m *Message) Packets() []*Packet {
+	return m.packets
+}
+
+// Data returns data in this message
+func (m *Message) Data() []byte {
+	buf := make([]byte, m.Length, m.Length)
+	var n, nn int
+	for _, p := range m.packets {
+		if len(p.Payload) < 1 {
+			continue
+		}
+		nn = copy(buf[n:], p.Payload)
+		n += nn
+	}
+	return buf
+}
+
+// Sort a helper to sort packets
+func (m *Message) Sort() {
+	sort.SliceStable(m.packets, func(i, j int) bool { return m.packets[i].Seq < m.packets[j].Seq })
+}
+
+// Handler message handler
+type Handler func(*Message)
+
+// Debugger is the debugger function. first params is the indicator of the issue's priority
+// the higher the number, the lower the priority. it can be 4 <= level <= 6.
+type Debugger func(int, ...interface{})
+
+// HintEnd hints the pool to stop the session, see MessagePool.End
+// when set, it will be executed before checking FIN or RST flag
+type HintEnd func(*Message) bool
+
+// HintStart hints the pool to start the reassembling the message, see MessagePool.Start
+// when set, it will be called after checking SYN flag
+type HintStart func(*Packet) (IsIncoming, IsOutgoing bool)
+
+// MessagePool holds data of all tcp messages in progress(still receiving/sending packets).
+// message is identified by its source port and dst port, and last 4bytes of src IP.
+type MessagePool struct {
+	sync.Mutex
+	debug         Debugger
+	maxSize       size.Size // maximum message size, default 5mb
+	pool          map[uint64]*Message
+	uuids         map[uint64][]byte
+	handler       Handler
+	messageExpire time.Duration // the maximum time to wait for the final packet, minimum is 100ms
+	End           HintEnd
+	Start         HintStart
+}
+
+// NewMessagePool returns a new instance of message pool
+func NewMessagePool(maxSize size.Size, messageExpire time.Duration, debugger Debugger, handler Handler) (pool *MessagePool) {
+	pool = new(MessagePool)
+	pool.debug = debugger
+	pool.handler = handler
+	pool.messageExpire = time.Millisecond * 100
+	if pool.messageExpire < messageExpire {
+		pool.messageExpire = messageExpire
+	}
+	pool.maxSize = maxSize
+	if pool.maxSize < 1 {
+		pool.maxSize = 5 << 20
+	}
+	pool.pool = make(map[uint64]*Message)
+	return pool
+}
+
+// Handler returns packet handler
+func (pool *MessagePool) Handler(packet gopacket.Packet) {
+	var in, out bool
+	pckt, err := ParsePacket(packet)
+	if err != nil {
+		go pool.say(4, fmt.Sprintf("error decoding packet(%dBytes):%s\n", packet.Metadata().CaptureLength, err))
+		return
+	}
+	if pckt == nil {
+		return
+	}
+	pool.Lock()
+	defer pool.Unlock()
+	lst := len(pckt.SrcIP) - 4
+	key := uint64(pckt.SrcPort)<<48 | uint64(pckt.DstPort)<<32 |
+		uint64(_uint32(pckt.SrcIP[lst:]))
+	m, ok := pool.pool[key]
+	if pckt.RST {
+		if ok {
+			m.done <- true
+			<-m.done
+		}
+		key = uint64(pckt.DstPort)<<48 | uint64(pckt.SrcPort)<<32 |
+			uint64(_uint32(pckt.DstIP[lst:]))
+		m, ok = pool.pool[key]
+		if ok {
+			m.done <- true
+			<-m.done
+		}
+		go pool.say(4, fmt.Sprintf("RST flag from %s to %s at %s\n", pckt.Src(), pckt.Dst(), pckt.Timestamp))
+		return
+	}
+	switch {
+	case ok:
+		pool.addPacket(m, pckt)
+		return
+	case pckt.SYN:
+		in = !pckt.ACK
+	case pool.Start != nil:
+		if in, out = pool.Start(pckt); !(in || out) {
+			return
+		}
+	default:
+		return
+	}
+	m = NewMessage(pckt.Src(), pckt.Dst(), pckt.Version)
+	m.IsIncoming = in
+	pool.pool[key] = m
+	m.Start = pckt.Timestamp
+	m.pool = pool
+	go pool.dispatch(key, m)
+	pool.addPacket(m, pckt)
+}
+
+// MatchUUID instructs the pool to use same UUID for request and responses
+// this function should be called at initial stage of the pool
+func (pool *MessagePool) MatchUUID(match bool) {
+	if match {
+		pool.uuids = make(map[uint64][]byte)
+		return
+	}
+	pool.uuids = nil
+}
+
+// run GC on UUID map
+func (pool *MessagePool) cleanUUIDs() {
+	var tStamp int64
+	now := time.Now().UnixNano()
+	for k, v := range pool.uuids {
+		// there is a timestamp wrapped in every ID
+		tStamp = int64(binary.BigEndian.Uint64(v[4:]))
+		if time.Duration(now-tStamp) > pool.messageExpire {
+			delete(pool.uuids, k)
+		}
+	}
+}
+
+func (pool *MessagePool) dispatch(key uint64, m *Message) {
+	select {
+	case <-m.done:
+		defer func() { m.done <- true }() // signal that message was dispatched
+	case <-time.After(pool.messageExpire):
+		pool.Lock()
+		defer pool.Unlock()
+		// avoid dispathing message twice
+		if _, ok := pool.pool[key]; !ok {
+			return
+		}
+		m.TimedOut = true
+	}
+	delete(pool.pool, key)
+	pool.handler(m)
+}
+
+func (pool *MessagePool) addPacket(m *Message, pckt *Packet) {
+	trunc := m.Length + len(pckt.Payload) - int(pool.maxSize)
+	if trunc > 0 {
+		m.Truncated = true
+		pckt.Payload = pckt.Payload[:int(pool.maxSize)-m.Length]
+	}
+	m.add(pckt)
+	switch {
+
+	// if one of this cases matches, we dispatch the message
+	case trunc >= 0:
+	case pckt.FIN:
+	default:
+		if pool.End != nil {
+			if pool.End(m) {
+				break
+			}
+		} else if m.Length > 0 && len(pckt.Payload) < 1 {
+			// generally, data are chunked and sent sequentially in packets, if one packets is empty we assume
+			// that it's just an acknowledgement packet after the final chunk.
+			break
+		}
+		return
+	}
+	m.done <- true
+	<-m.done
+}
+
+// this function should not block other pool operations
+func (pool *MessagePool) say(level int, args ...interface{}) {
+	if pool.debug != nil {
+		pool.debug(level, args...)
+	}
+}
+
+func _uint32(b []byte) uint32 {
+	return binary.BigEndian.Uint32(b)
+}

tcp/tcp_packet.go

@@ -0,0 +1,197 @@
+package tcp
+
+import (
+	"encoding/binary"
+	"fmt"
+	"net"
+	"time"
+
+	"github.com/google/gopacket"
+	"github.com/google/gopacket/layers"
+)
+
+/*
+Packet represent data and layers of packet.
+parser extracts information from pcap Packet. functions of *Packet doesn't validate if packet is nil,
+calllers must make sure that ParsePacket has'nt returned any error before calling any other
+function.
+*/
+type Packet struct {
+	// Link layer
+	gopacket.LinkLayer
+
+	// IP Header
+	Version uint8 // Ip version
+	SrcIP   net.IP
+	DstIP   net.IP
+	IHL     uint8
+	Length  uint16
+
+	// TCP Segment Header
+	*layers.TCP
+
+	// Data info
+	Lost      uint16
+	Timestamp time.Time
+}
+
+// ParsePacket parse raw packets
+func ParsePacket(packet gopacket.Packet) (pckt *Packet, err error) {
+	// early check of error
+	if packet == nil {
+		return
+	}
+	defer func() {
+		if packet.ErrorLayer() != nil {
+			err = packet.ErrorLayer().Error()
+			return
+		}
+	}()
+
+	// initialization
+	pckt = new(Packet)
+	pckt.Timestamp = packet.Metadata().Timestamp
+	if pckt.Timestamp.IsZero() {
+		pckt.Timestamp = time.Now()
+	}
+
+	// parsing link layer
+	pckt.LinkLayer = packet.LinkLayer()
+
+	// parsing network layer
+	if net4, ok := packet.NetworkLayer().(*layers.IPv4); ok {
+		pckt.Version = 4
+		pckt.SrcIP = net4.SrcIP
+		pckt.DstIP = net4.DstIP
+		pckt.IHL = net4.IHL * 4
+		pckt.Length = net4.Length
+	} else if net6, ok := packet.NetworkLayer().(*layers.IPv6); ok {
+		pckt.Version = 6
+		pckt.SrcIP = net6.SrcIP
+		pckt.DstIP = net6.DstIP
+		pckt.IHL = 40
+		pckt.Length = net6.Length
+	} else {
+		pckt = nil
+		return
+	}
+
+	// parsing tcp header(transportation layer)
+	if tcp, ok := packet.TransportLayer().(*layers.TCP); ok {
+		pckt.TCP = tcp
+	} else {
+		pckt = nil
+		return
+	}
+	pckt.DataOffset *= 4
+
+	// calculating lost data
+	headerSize := int(uint32(pckt.DataOffset) + uint32(pckt.IHL))
+	if pckt.Version == 6 {
+		headerSize -= 40 // in ipv6 the length of payload doesn't include the IPheader size
+	}
+	pckt.Lost = pckt.Length - uint16(headerSize+len(pckt.Payload))
+
+	return
+}
+
+// Src returns the source socket of a packet
+func (pckt *Packet) Src() string {
+	return fmt.Sprintf("%s:%d", pckt.SrcIP, pckt.SrcPort)
+}
+
+// Dst returns destination socket
+func (pckt *Packet) Dst() string {
+	return fmt.Sprintf("%s:%d", pckt.DstIP, pckt.DstPort)
+}
+
+// SYNOptions returns MSS and windowscale of syn packets
+func (pckt *Packet) SYNOptions() (mss uint16, windowscale byte) {
+	if !pckt.SYN {
+		return
+	}
+	for _, v := range pckt.Options {
+		if v.OptionType == layers.TCPOptionKindMSS {
+			mss = binary.BigEndian.Uint16(v.OptionData)
+			continue
+		}
+		if v.OptionType == layers.TCPOptionKindWindowScale {
+			if v.OptionLength > 0 {
+				windowscale = v.OptionData[0]
+			}
+		}
+	}
+	return
+}
+
+// LinkInfo returns info about the link layer
+func (pckt *Packet) LinkInfo() string {
+	if l, ok := pckt.LinkLayer.(*layers.Ethernet); ok {
+		return fmt.Sprintf(
+			"Source Mac: %s\nDestination Mac: %s\nProtocol: %s",
+			l.SrcMAC,
+			l.DstMAC,
+			l.EthernetType,
+		)
+	}
+	return "<Not Ethernet>"
+}
+
+// Flag returns formatted tcp flags
+func (pckt *Packet) Flag() (flag string) {
+	if pckt.FIN {
+		flag += "FIN, "
+	}
+	if pckt.SYN {
+		flag += "SYN, "
+	}
+	if pckt.RST {
+		flag += "RST, "
+	}
+	if pckt.PSH {
+		flag += "PSH, "
+	}
+	if pckt.ACK {
+		flag += "ACK, "
+	}
+	if pckt.URG {
+		flag += "URG, "
+	}
+	if len(flag) != 0 {
+		return flag[:len(flag)-2]
+	}
+	return flag
+}
+
+// String output for a TCP Packet
+func (pckt *Packet) String() string {
+	return fmt.Sprintf(`Time: %s
+%s
+Source: %s
+Destination: %s
+IHL: %d
+Total Length: %d
+Sequence: %d
+Acknowledgment: %d
+DataOffset: %d
+Window: %d
+Flag: %s
+Options: %s
+Data Size: %d
+Lost Data: %d`,
+		pckt.Timestamp.Format(time.StampNano),
+		pckt.LinkInfo(),
+		pckt.Src(),
+		pckt.Dst(),
+		pckt.IHL,
+		pckt.Length,
+		pckt.Seq,
+		pckt.Ack,
+		pckt.DataOffset,
+		pckt.Window,
+		pckt.Flag(),
+		pckt.Options,
+		len(pckt.Payload),
+		pckt.Lost,
+	)
+}

tcp/tcp_test.go

@@ -0,0 +1,292 @@
+package tcp
+
+import (
+	"bytes"
+	"encoding/binary"
+	"testing"
+	"time"
+
+	"github.com/buger/goreplay/proto"
+	"github.com/google/gopacket"
+	"github.com/google/gopacket/layers"
+)
+
+var decodeOpts = gopacket.DecodeOptions{Lazy: true, NoCopy: true}
+
+func headersIP4(seq uint32, length uint16) (headers [54]byte) {
+	// set ethernet headers
+	binary.BigEndian.PutUint16(headers[12:14], uint16(layers.EthernetTypeIPv4))
+
+	// set ip header
+	ip := headers[14:]
+	copy(ip[0:2], []byte{4<<4 | 5, 0x28<<2 | 0x00})
+	binary.BigEndian.PutUint16(ip[2:4], length+40)
+	ip[9] = uint8(layers.IPProtocolTCP)
+	copy(ip[12:16], []byte{192, 168, 1, 2})
+	copy(ip[16:], []byte{192, 168, 1, 3})
+
+	// set tcp header
+	tcp := ip[20:]
+	binary.BigEndian.PutUint16(tcp[0:2], 45678)
+	binary.BigEndian.PutUint16(tcp[2:4], 8001)
+	tcp[12] = 5 << 4
+	return
+}
+
+func GetPackets(start uint32, _len int, payload []byte) []gopacket.Packet {
+	var packets = make([]gopacket.Packet, _len)
+	for i := start; i < start+uint32(_len); i++ {
+		data := make([]byte, 54+len(payload))
+		h := headersIP4(i, uint16(len(payload)))
+		copy(data, h[:])
+		copy(data[len(h):], payload)
+		packets[i-start] = gopacket.NewPacket(data, layers.LinkTypeEthernet, decodeOpts)
+	}
+	return packets
+}
+
+func TestMessageParserWithHint(t *testing.T) {
+	var mssg = make(chan *Message, 3)
+	pool := NewMessagePool(1<<20, time.Second, nil, func(m *Message) { mssg <- m })
+	pool.Start = func(pckt *Packet) (bool, bool) {
+		return proto.HasRequestTitle(pckt.Payload), proto.HasResponseTitle(pckt.Payload)
+	}
+	packets := GetPackets(1, 30, nil)
+	packets[0].Data()[14:][20:][13] = 2  // SYN flag
+	packets[10].Data()[14:][20:][13] = 2 // SYN flag
+	packets[29].Data()[14:][20:][13] = 1 // FIN flag
+	packets[4] = GetPackets(5, 1, []byte("HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nTransfer-Encoding: chunked\r\n\r\n7"))[0]
+	packets[5] = GetPackets(6, 1, []byte("\r\nMozilla\r\n9\r\nDeveloper\r"))[0]
+	packets[6] = GetPackets(7, 1, []byte("\n7\r\nNetwork\r\n0\r\n\r\n"))[0]
+	packets[14] = GetPackets(5, 1, []byte("POST / HTTP/1.1\r\nContent-Type: text/plain\r\nContent-Length: 23\r\n\r\n"))[0]
+	packets[15] = GetPackets(6, 1, []byte("MozillaDeveloper"))[0]
+	packets[16] = GetPackets(7, 1, []byte("Network"))[0]
+	packets[24] = GetPackets(5, 1, []byte("HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nContent-Length: 0\r\n\r"))[0]
+	for i := 0; i < 30; i++ {
+		pool.Handler(packets[i])
+	}
+	var m *Message
+	select {
+	case <-time.After(time.Second):
+		t.Errorf("can't parse packets fast enough")
+		return
+	case m = <-mssg:
+	}
+	if len(m.packets) != 8 {
+		t.Errorf("expected to have 8 packets got %d", len(m.packets))
+	}
+	if !bytes.HasSuffix(m.Data(), []byte("\n7\r\nNetwork\r\n0\r\n\r\n")) {
+		t.Errorf("expected to %q to have suffix %q", m.Data(), []byte("\n7\r\nNetwork\r\n0\r\n\r\n"))
+	}
+
+	select {
+	case <-time.After(time.Second):
+		t.Errorf("can't parse packets fast enough")
+		return
+	case m = <-mssg:
+	}
+	if len(m.packets) != 8 {
+		t.Errorf("expected to have 8 packets got %d", len(m.packets))
+	}
+	if !bytes.HasSuffix(m.Data(), []byte("Network")) {
+		t.Errorf("expected to %q to have suffix %q", m.Data(), []byte("Network"))
+	}
+
+	select {
+	case <-time.After(time.Second):
+		t.Errorf("can't parse packets fast enough")
+		return
+	case m = <-mssg:
+	}
+	if len(m.packets) != 2 {
+		t.Errorf("expected to have 2 packets got %d", len(m.packets))
+	}
+	if !bytes.HasSuffix(m.Data(), []byte("Content-Length: 0\r\n\r")) {
+		t.Errorf("expected to %q to have suffix %q", m.Data(), []byte("Content-Length: 0\r\n\r"))
+	}
+
+}
+
+func TestMessageParserWithoutHint(t *testing.T) {
+	var mssg = make(chan *Message, 1)
+	var data [63 << 10]byte
+	packets := GetPackets(1, 10, data[:])
+	packets[0].Data()[14:][20:][13] = 2 // SYN flag
+	packets[9].Data()[14:][20:][13] = 1 // FIN flag
+	p := NewMessagePool(63<<10*10, time.Second, nil, func(m *Message) { mssg <- m })
+	for _, v := range packets {
+		p.Handler(v)
+	}
+	var m *Message
+	select {
+	case <-time.After(time.Second):
+		t.Errorf("can't parse packets fast enough")
+		return
+	case m = <-mssg:
+	}
+	if m.Length != 63<<10*10 {
+		t.Errorf("expected %d to equal %d", m.Length, 63<<10*10)
+	}
+}
+
+func TestMessageMaxSizeReached(t *testing.T) {
+	var mssg = make(chan *Message, 2)
+	var data [63 << 10]byte
+	packets := GetPackets(1, 2, data[:])
+	packets = append(packets, GetPackets(3, 1, make([]byte, 63<<10+10))...)
+	packets[0].Data()[14:][20:][13] = 2 // SYN flag
+	packets[2].Data()[14:][20:][13] = 2 // SYN flag
+	packets[2].Data()[14:][15] = 3      // changing address
+	p := NewMessagePool(63<<10+10, time.Second, nil, func(m *Message) { mssg <- m })
+	for _, v := range packets {
+		p.Handler(v)
+	}
+	var m *Message
+	select {
+	case <-time.After(time.Second):
+		t.Errorf("can't parse packets fast enough")
+		return
+	case m = <-mssg:
+	}
+	if m.Length != 63<<10+10 {
+		t.Errorf("expected %d to equal %d", m.Length, 63<<10+10)
+	}
+	if !m.Truncated {
+		t.Error("expected message to be truncated")
+	}
+
+	select {
+	case <-time.After(time.Second):
+		t.Errorf("can't parse packets fast enough")
+		return
+	case m = <-mssg:
+	}
+	if m.Length != 63<<10+10 {
+		t.Errorf("expected %d to equal %d", m.Length, 63<<10+10)
+	}
+	if m.Truncated {
+		t.Error("expected message to not be truncated")
+	}
+}
+
+func TestMessageTimeoutReached(t *testing.T) {
+	var mssg = make(chan *Message, 2)
+	var data [63 << 10]byte
+	packets := GetPackets(1, 2, data[:])
+	packets[0].Data()[14:][20:][13] = 2 // SYN flag
+	p := NewMessagePool(1<<20, 0, nil, func(m *Message) { mssg <- m })
+	p.Handler(packets[0])
+	time.Sleep(time.Millisecond * 200)
+	p.Handler(packets[1])
+	m := <-mssg
+	if m.Length != 63<<10 {
+		t.Errorf("expected %d to equal %d", m.Length, 63<<10)
+	}
+	if !m.TimedOut {
+		t.Error("expected message to be timeout")
+	}
+}
+
+func TestMessageUUID(t *testing.T) {
+	packets := GetPackets(1, 10, nil)
+	packets[0].Data()[14:][20:][13] = 2 // SYN flag
+	packets[4].Data()[14:][20:][13] = 1 // FIN flag
+	packets[5].Data()[14:][20:][13] = 2 // SYN flag
+	packets[9].Data()[14:][20:][13] = 1 // FIN flag
+	var uuid, uuid1 []byte
+	pool := NewMessagePool(0, 0, nil, func(msg *Message) {
+		if len(uuid) == 0 {
+			uuid = msg.UUID()
+			return
+		}
+		uuid1 = msg.UUID()
+	})
+	pool.MatchUUID(true)
+	for _, p := range packets {
+		pool.Handler(p)
+	}
+
+	if string(uuid) != string(uuid1) {
+		t.Errorf("expected %s, to equal %s", uuid, uuid1)
+	}
+}
+
+func BenchmarkMessageUUID(b *testing.B) {
+	packets := GetPackets(1, 5, nil)
+	packets[0].Data()[14:][20:][13] = 2 // SYN flag
+	packets[4].Data()[14:][20:][13] = 1 // FIN flag
+	var uuid []byte
+	var msg *Message
+	pool := NewMessagePool(0, 0, nil, func(m *Message) {
+		msg = m
+	})
+	pool.MatchUUID(true)
+	for _, p := range packets {
+		pool.Handler(p)
+	}
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		uuid = msg.UUID()
+	}
+	_ = uuid
+}
+
+func BenchmarkPacketParseAndSort(b *testing.B) {
+	m := new(Message)
+	m.packets = make([]*Packet, 100)
+	for i, v := range GetPackets(1, 100, nil) {
+		m.packets[i], _ = ParsePacket(v)
+	}
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		m.Sort()
+	}
+}
+
+func BenchmarkMessageParserWithoutHint(b *testing.B) {
+	var mssg = make(chan *Message, 1)
+	var chunk = []byte("111111111111111111111111111111")
+	packets := GetPackets(1, 1000, chunk)
+	packets[0].Data()[14:][20:][13] = 2      // SYN flag
+	packets[1000-1].Data()[14:][20:][13] = 1 // FIN flag
+	p := NewMessagePool(1<<20, time.Second*2, nil, func(m *Message) {
+		mssg <- m
+	})
+	b.ResetTimer()
+	b.ReportMetric(float64(1000), "packets/op")
+	for i := 0; i < b.N; i++ {
+		for _, v := range packets {
+			p.Handler(v)
+		}
+		<-mssg
+	}
+}
+
+func BenchmarkMessageParserWithHint(b *testing.B) {
+	var buf [1003][]byte
+	var chunk = []byte("1e\r\n111111111111111111111111111111\r\n")
+	buf[0] = []byte("HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nTransfer-Encoding: chunked\r\n\r\n")
+	for i := 1; i < 1000; i++ {
+		buf[i] = chunk
+	}
+	buf[1001] = []byte("0\r\n\r\n")
+	packets := make([]gopacket.Packet, len(buf))
+	for i := 0; i < len(buf); i++ {
+		packets[i] = GetPackets(uint32(i+10), 1, buf[i])[0]
+	}
+	packets[1002] = GetPackets(1020, 1, nil)[0]
+	var mssg = make(chan *Message, 1)
+	pool := NewMessagePool(1<<30, time.Second*10, nil, func(m *Message) { mssg <- m })
+	pool.Start = func(pckt *Packet) (bool, bool) {
+		return false, proto.HasResponseTitle(pckt.Payload)
+	}
+	b.ResetTimer()
+	b.ReportMetric(float64(len(packets)), "packets/op")
+	b.ReportMetric(float64(1000), "chunks/op")
+	for i := 0; i < b.N; i++ {
+		for j := range packets {
+			pool.Handler(packets[j])
+		}
+		<-mssg
+	}
+}

tcp_client.go

@@ -3,7 +3,6 @@ package main
 import (
 	"crypto/tls"
 	"io"
-	"log"
 	"net"
 	"runtime/debug"
 	"syscall"
@@ -71,7 +70,7 @@ func (c *TCPClient) Disconnect() {
 	if c.conn != nil {
 		c.conn.Close()
 		c.conn = nil
-		Debug("[TCPClient] Disconnected: ", c.baseURL)
+		Debug(1, "[TCPClient] Disconnected: ", c.baseURL)
 	}
 }
 
@@ -85,12 +84,10 @@ func (c *TCPClient) isAlive() bool {
 	if err == nil {
 		return true
 	} else if err == io.EOF {
-		if c.config.Debug {
-			Debug("[TCPClient] connection closed, reconnecting")
-		}
+		Debug(1, "[TCPClient] connection closed, reconnecting")
 		return false
 	} else if err == syscall.EPIPE {
-		Debug("Detected broken pipe.", err)
+		Debug(1, "Detected broken pipe.", err)
 		return false
 	}
 
@@ -102,19 +99,19 @@ func (c *TCPClient) Send(data []byte) (response []byte, err error) {
 	// Don't exit on panic
 	defer func() {
 		if r := recover(); r != nil {
-			Debug("[TCPClient]", r, string(data))
+			Debug(1, "[TCPClient]", r, string(data))
 
 			if _, ok := r.(error); !ok {
-				log.Println("[TCPClient] Failed to send request: ", string(data))
-				log.Println("PANIC: pkg:", r, debug.Stack())
+				Debug(1, "[TCPClient] Failed to send request: ", string(data))
+				Debug(1, "PANIC: pkg:", r, debug.Stack())
 			}
 		}
 	}()
 
 	if c.conn == nil || !c.isAlive() {
-		Debug("[TCPClient] Connecting:", c.baseURL)
+		Debug(1, "[TCPClient] Connecting:", c.baseURL)
 		if err = c.Connect(); err != nil {
-			log.Println("[TCPClient] Connection error:", err)
+			Debug(1, "[TCPClient] Connection error:", err)
 			return
 		}
 	}
@@ -124,11 +121,11 @@ func (c *TCPClient) Send(data []byte) (response []byte, err error) {
 	c.conn.SetWriteDeadline(timeout)
 
 	if c.config.Debug {
-		Debug("[TCPClient] Sending:", string(data))
+		Debug(1, "[TCPClient] Sending:", string(data))
 	}
 
 	if _, err = c.conn.Write(data); err != nil {
-		Debug("[TCPClient] Write error:", err, c.baseURL)
+		Debug(1, "[TCPClient] Write error:", err, c.baseURL)
 		return
 	}
 
@@ -159,7 +156,7 @@ func (c *TCPClient) Send(data []byte) (response []byte, err error) {
 			if err == io.EOF {
 				break
 			} else if err != nil {
-				Debug("[TCPClient] Read the whole body error:", err, c.baseURL)
+				Debug(1, "[TCPClient] Read the whole body error:", err, c.baseURL)
 				break
 			}
 
@@ -167,7 +164,7 @@ func (c *TCPClient) Send(data []byte) (response []byte, err error) {
 		}
 
 		if readBytes >= maxResponseSize {
-			Debug("[TCPClient] Body is more than the max size", maxResponseSize,
+			Debug(1, "[TCPClient] Body is more than the max size", maxResponseSize,
 				c.baseURL)
 			break
 		}
@@ -177,7 +174,7 @@ func (c *TCPClient) Send(data []byte) (response []byte, err error) {
 	}
 
 	if err != nil {
-		Debug("[TCPClient] Response read error", err, c.conn, readBytes)
+		Debug(1, "[TCPClient] Response read error", err, c.conn, readBytes)
 		return
 	}
 
@@ -189,7 +186,7 @@ func (c *TCPClient) Send(data []byte) (response []byte, err error) {
 	copy(payload, c.respBuf[:readBytes])
 
 	if c.config.Debug {
-		Debug("[TCPClient] Received:", string(payload))
+		Debug(1, "[TCPClient] Received:", string(payload))
 	}
 
 	return payload, err

test_input.go

@@ -1,10 +1,9 @@
 package main
 
 import (
-	"crypto/rand"
 	"encoding/base64"
 	"errors"
-	"fmt"
+	"math/rand"
 	"time"
 )
 
@@ -26,30 +25,31 @@ func NewTestInput() (i *TestInput) {
 	return
 }
 
-func (i *TestInput) Read(data []byte) (int, error) {
+// PluginRead reads message from this plugin
+func (i *TestInput) PluginRead() (*Message, error) {
+	var msg Message
 	select {
 	case buf := <-i.data:
-		var header []byte
-
+		msg.Data = buf
 		if !i.skipHeader {
-			header = payloadHeader(RequestPayload, uuid(), time.Now().UnixNano(), -1)
-			copy(data[0:len(header)], header)
-			copy(data[len(header):], buf)
+			msg.Meta = payloadHeader(RequestPayload, uuid(), time.Now().UnixNano(), -1)
 		} else {
-			copy(data, buf)
+			msg.Meta, msg.Data = payloadMetaWithBody(msg.Data)
 		}
 
-		return len(buf) + len(header), nil
-	case <-time.After(10 * time.Second):
-		return 0, fmt.Errorf("timed out waiting for read")
+		return &msg, nil
+	case <-i.stop:
+		return nil, ErrorStopped
 	}
 }
 
+// Close closes this plugin
 func (i *TestInput) Close() error {
 	close(i.stop)
 	return nil
 }
 
+// EmitBytes sends data
 func (i *TestInput) EmitBytes(data []byte) {
 	i.data <- data
 }
@@ -77,8 +77,7 @@ func (i *TestInput) EmitLargePOST() {
 
 	rs := base64.URLEncoding.EncodeToString(rb)
 
-	i.data <- []byte("POST / HTTP/1.1\nHost: www.w3.org\nContent-Length:5242880\r\n\r\n" + rs)
-	Debug("Sent large POST")
+	i.data <- []byte("POST / HTTP/1.1\r\nHost: www.w3.org\nContent-Length:5242880\r\n\r\n" + rs)
 }
 
 // EmitSizedPOST emit a POST with a payload set to a supplied size
@@ -88,13 +87,12 @@ func (i *TestInput) EmitSizedPOST(payloadSize int) {
 
 	rs := base64.URLEncoding.EncodeToString(rb)
 
-	i.data <- []byte("POST / HTTP/1.1\nHost: www.w3.org\nContent-Length:5242880\r\n\r\n" + rs)
-	Debug("Sent large POST")
+	i.data <- []byte("POST / HTTP/1.1\r\nHost: www.w3.org\nContent-Length:5242880\r\n\r\n" + rs)
 }
 
 // EmitOPTIONS emits OPTIONS request, similar to GET
 func (i *TestInput) EmitOPTIONS() {
-	i.data <- []byte("OPTIONS / HTTP/1.1\nHost: www.w3.org\r\n\r\n")
+	i.data <- []byte("OPTIONS / HTTP/1.1\r\nHost: www.w3.org\r\n\r\n")
 }
 
 func (i *TestInput) String() string {

test_output.go

@@ -1,6 +1,6 @@
 package main
 
-type writeCallback func(data []byte)
+type writeCallback func(*Message)
 
 // TestOutput used in testing to intercept any output into callback
 type TestOutput struct {
@@ -8,19 +8,20 @@ type TestOutput struct {
 }
 
 // NewTestOutput constructor for TestOutput, accepts callback which get called on each incoming Write
-func NewTestOutput(cb writeCallback) (i *TestOutput) {
-	i = new(TestOutput)
+func NewTestOutput(cb writeCallback) PluginWriter {
+	i := new(TestOutput)
 	i.cb = cb
 
-	return
+	return i
 }
 
-func (i *TestOutput) Write(data []byte) (int, error) {
-	i.cb(data)
+// PluginWrite write message to this plugin
+func (i *TestOutput) PluginWrite(msg *Message) (int, error) {
+	i.cb(msg)
 
-	return len(data), nil
+	return len(msg.Data) + len(msg.Meta), nil
 }
 
 func (i *TestOutput) String() string {
-	return "Test Input"
+	return "Test Output"
 }

vendor/github.com/google/gopacket/.travis.yml

@@ -2,6 +2,7 @@ language: go
 go:
  - 1.11.x
  - 1.12.x
+ - 1.13.x
  - master
 
 addons:
@@ -38,10 +39,11 @@ jobs:
       install: ./.travis.install.sh
     - os: osx
       go: 1.x
-    - os: windows
-      go: 1.x
-      # winpcap does not work on travis ci - so install nmap to get libpcap
-      before_install: choco install nmap
+# windows doesn't work on travis (package installation just hangs and then errors out)
+#    - os: windows
+#      go: 1.x
+#      # We don't need nmap - but that's the only way to get npcap:
+#      before_install: choco install npcap --version 0.86 -y
     - stage: style
       name: "fmt/vet/lint"
       go: 1.x

vendor/github.com/google/gopacket/AUTHORS

@@ -17,6 +17,7 @@ Christian Mäder <christian.maeder@nine.ch>
 Gernot Vormayr <gvormayr@gmail.com>
 Vitor Garcia Graveto <victor.graveto@gmail.com>
 Elias Chavarria Reyes <elchavar@cisco.com>
+Daniel Rittweiler <ripx80@protonmail.com>
 
 CONTRIBUTORS:
 Attila Oláh <attila@attilaolah.eu>

vendor/github.com/google/gopacket/README.md

@@ -6,7 +6,7 @@ See [godoc](https://godoc.org/github.com/google/gopacket) for more details.
 [![Build Status](https://travis-ci.org/google/gopacket.svg?branch=master)](https://travis-ci.org/google/gopacket)
 [![GoDoc](https://godoc.org/github.com/google/gopacket?status.svg)](https://godoc.org/github.com/google/gopacket)
 
-Minimum Go version required is 1.5 except for pcapgo/EthernetHandle, afpacket, and bsdbpf which need at least 1.7 due to x/sys/unix dependencies.
+Minimum Go version required is 1.5 except for pcapgo/EthernetHandle, afpacket, and bsdbpf which need at least 1.9 due to x/sys/unix dependencies.
 
 Originally forked from the gopcap project written by Andreas
 Krennmair <ak@synflood.at> (http://github.com/akrennmair/gopcap).

vendor/github.com/google/gopacket/base.go

@@ -78,7 +78,7 @@ func (p Payload) SerializeTo(b SerializeBuffer, opts SerializeOptions) error {
 func decodePayload(data []byte, p PacketBuilder) error {
 	payload := &Payload{}
 	if err := payload.DecodeFromBytes(data, p); err != nil {
-		return nil
+		return err
 	}
 	p.AddLayer(payload)
 	p.SetApplicationLayer(payload)
@@ -132,7 +132,7 @@ func (p *Fragment) SerializeTo(b SerializeBuffer, opts SerializeOptions) error {
 func decodeFragment(data []byte, p PacketBuilder) error {
 	payload := &Fragment{}
 	if err := payload.DecodeFromBytes(data, p); err != nil {
-		return nil
+		return err
 	}
 	p.AddLayer(payload)
 	p.SetApplicationLayer(payload)

vendor/github.com/google/gopacket/doc.go

@@ -208,7 +208,7 @@ based on endpoint criteria:
    }
  }
  // Find all packets coming from UDP port 1000 to UDP port 500
- interestingFlow := gopacket.NewFlow(layers.NewUDPPortEndpoint(1000), layers.NewUDPPortEndpoint(500))
+ interestingFlow := gopacket.FlowFromEndpoints(layers.NewUDPPortEndpoint(1000), layers.NewUDPPortEndpoint(500))
  if t := packet.NetworkLayer(); t != nil && t.TransportFlow() == interestingFlow {
    fmt.Println("Found that UDP flow I was looking for!")
  }
@@ -320,6 +320,67 @@ implementing the DecodingLayer interface are usable.  Also, it's possible to
 create DecodingLayers that are not themselves Layers... see
 layers.IPv6ExtensionSkipper for an example of this.
 
+Faster And Customized Decoding with DecodingLayerContainer
+
+By default, DecodingLayerParser uses native map to store and search for a layer
+to decode. Though being versatile, in some cases this solution may be not so
+optimal. For example, if you have only few layers faster operations may be
+provided by sparse array indexing or linear array scan.
+
+To accomodate these scenarios, DecodingLayerContainer interface is introduced
+along with its implementations: DecodingLayerSparse, DecodingLayerArray and
+DecodingLayerMap. You can specify a container implementation to
+DecodingLayerParser with SetDecodingLayerContainer method. Example:
+
+ dlp := gopacket.NewDecodingLayerParser(LayerTypeEthernet)
+ dlp.SetDecodingLayerContainer(gopacket.DecodingLayerSparse(nil))
+ var eth layers.Ethernet
+ dlp.AddDecodingLayer(&eth)
+ // ... add layers and use DecodingLayerParser as usual...
+
+To skip one level of indirection (though sacrificing some capabilities) you may
+also use DecodingLayerContainer as a decoding tool as it is. In this case you have to
+handle unknown layer types and layer panics by yourself. Example:
+
+ func main() {
+   var eth layers.Ethernet
+   var ip4 layers.IPv4
+   var ip6 layers.IPv6
+   var tcp layers.TCP
+   dlc := gopacket.DecodingLayerContainer(gopacket.DecodingLayerArray(nil))
+   dlc = dlc.Put(&eth)
+   dlc = dlc.Put(&ip4)
+   dlc = dlc.Put(&ip6)
+   dlc = dlc.Put(&tcp)
+   // you may specify some meaningful DecodeFeedback
+   decoder := dlc.LayersDecoder(LayerTypeEthernet, gopacket.NilDecodeFeedback)
+   decoded := make([]gopacket.LayerType, 0, 20)
+   for packetData := range somehowGetPacketData() {
+     lt, err := decoder(packetData, &decoded)
+     if err != nil {
+       fmt.Fprintf(os.Stderr, "Could not decode layers: %v\n", err)
+       continue
+     }
+     if lt != gopacket.LayerTypeZero {
+       fmt.Fprintf(os.Stderr, "unknown layer type: %v\n", lt)
+       continue
+     }
+     for _, layerType := range decoded {
+       // examine decoded layertypes just as already shown above
+     }
+   }
+ }
+
+DecodingLayerSparse is the fastest but most effective when LayerType values
+that layers in use can decode are not large because otherwise that would lead
+to bigger memory footprint. DecodingLayerArray is very compact and primarily
+usable if the number of decoding layers is not big (up to ~10-15, but please do
+your own benchmarks). DecodingLayerMap is the most versatile one and used by
+DecodingLayerParser by default. Please refer to tests and benchmarks in layers
+subpackage to further examine usage examples and performance measurements.
+
+You may also choose to implement your own DecodingLayerContainer if you want to
+make use of your own internal packet decoding logic.
 
 Creating Packet Data
 

vendor/github.com/google/gopacket/layers/arp.go

@@ -10,6 +10,7 @@ package layers
 import (
 	"encoding/binary"
 	"errors"
+	"fmt"
 
 	"github.com/google/gopacket"
 )
@@ -39,17 +40,25 @@ func (arp *ARP) LayerType() gopacket.LayerType { return LayerTypeARP }
 
 // DecodeFromBytes decodes the given bytes into this layer.
 func (arp *ARP) DecodeFromBytes(data []byte, df gopacket.DecodeFeedback) error {
+	if len(data) < 8 {
+		df.SetTruncated()
+		return fmt.Errorf("ARP length %d too short", len(data))
+	}
 	arp.AddrType = LinkType(binary.BigEndian.Uint16(data[0:2]))
 	arp.Protocol = EthernetType(binary.BigEndian.Uint16(data[2:4]))
 	arp.HwAddressSize = data[4]
 	arp.ProtAddressSize = data[5]
 	arp.Operation = binary.BigEndian.Uint16(data[6:8])
+	arpLength := 8 + 2*arp.HwAddressSize + 2*arp.ProtAddressSize
+	if len(data) < int(arpLength) {
+		df.SetTruncated()
+		return fmt.Errorf("ARP length %d too short, %d expected", len(data), arpLength)
+	}
 	arp.SourceHwAddress = data[8 : 8+arp.HwAddressSize]
 	arp.SourceProtAddress = data[8+arp.HwAddressSize : 8+arp.HwAddressSize+arp.ProtAddressSize]
 	arp.DstHwAddress = data[8+arp.HwAddressSize+arp.ProtAddressSize : 8+2*arp.HwAddressSize+arp.ProtAddressSize]
 	arp.DstProtAddress = data[8+2*arp.HwAddressSize+arp.ProtAddressSize : 8+2*arp.HwAddressSize+2*arp.ProtAddressSize]
 
-	arpLength := 8 + 2*arp.HwAddressSize + 2*arp.ProtAddressSize
 	arp.Contents = data[:arpLength]
 	arp.Payload = data[arpLength:]
 	return nil

vendor/github.com/google/gopacket/layers/asf.go

@@ -0,0 +1,166 @@
+// Copyright 2019 The GoPacket Authors. All rights reserved.
+//
+// Use of this source code is governed by a BSD-style license that can be found
+// in the LICENSE file in the root of the source tree.
+
+package layers
+
+// This file implements the ASF RMCP payload specified in section 3.2.2.3 of
+// https://www.dmtf.org/sites/default/files/standards/documents/DSP0136.pdf
+
+import (
+	"encoding/binary"
+	"fmt"
+
+	"github.com/google/gopacket"
+)
+
+const (
+	// ASFRMCPEnterprise is the IANA-assigned Enterprise Number of the ASF-RMCP.
+	ASFRMCPEnterprise uint32 = 4542
+)
+
+// ASFDataIdentifier encapsulates fields used to uniquely identify the format of
+// the data block.
+//
+// While the enterprise number is almost always 4542 (ASF-RMCP), we support
+// registering layers using structs of this type as a key in case any users are
+// using OEM-extensions.
+type ASFDataIdentifier struct {
+
+	// Enterprise is the IANA Enterprise Number associated with the entity that
+	// defines the message type. A list can be found at
+	// https://www.iana.org/assignments/enterprise-numbers/enterprise-numbers.
+	// This can be thought of as the namespace for the message type.
+	Enterprise uint32
+
+	// Type is the message type, defined by the entity associated with the
+	// enterprise above. No pressure, but in the context of EN 4542, 1 byte is
+	// the difference between sending a ping and telling a machine to do an
+	// unconditional power down (0x80 and 0x12 respectively).
+	Type uint8
+}
+
+// LayerType returns the payload layer type corresponding to an ASF message
+// type.
+func (a ASFDataIdentifier) LayerType() gopacket.LayerType {
+	if lt := asfDataLayerTypes[a]; lt != 0 {
+		return lt
+	}
+
+	// some layer types don't have a payload, e.g. ASF-RMCP Presence Ping.
+	return gopacket.LayerTypePayload
+}
+
+// RegisterASFLayerType allows specifying that the data block of ASF packets
+// with a given enterprise number and type should be processed by a given layer
+// type. This overrides any existing registrations, including defaults.
+func RegisterASFLayerType(a ASFDataIdentifier, l gopacket.LayerType) {
+	asfDataLayerTypes[a] = l
+}
+
+var (
+	// ASFDataIdentifierPresencePong is the message type of the response to a
+	// Presence Ping message. It indicates the sender is ASF-RMCP-aware.
+	ASFDataIdentifierPresencePong = ASFDataIdentifier{
+		Enterprise: ASFRMCPEnterprise,
+		Type:       0x40,
+	}
+
+	// ASFDataIdentifierPresencePing is a message type sent to a managed client
+	// to solicit a Presence Pong response. Clients may ignore this if the RMCP
+	// version is unsupported. Sending this message with a sequence number <255
+	// is the recommended way of finding out whether an implementation sends
+	// RMCP ACKs (e.g. iDRAC does, Super Micro does not).
+	//
+	// Systems implementing IPMI must respond to this ping to conform to the
+	// spec, so it is a good substitute for an ICMP ping.
+	ASFDataIdentifierPresencePing = ASFDataIdentifier{
+		Enterprise: ASFRMCPEnterprise,
+		Type:       0x80,
+	}
+
+	// asfDataLayerTypes is used to find the next layer for a given ASF header.
+	asfDataLayerTypes = map[ASFDataIdentifier]gopacket.LayerType{
+		ASFDataIdentifierPresencePong: LayerTypeASFPresencePong,
+	}
+)
+
+// ASF defines ASF's generic RMCP message Data block format. See section
+// 3.2.2.3.
+type ASF struct {
+	BaseLayer
+	ASFDataIdentifier
+
+	// Tag is used to match request/response pairs. The tag of a response is set
+	// to that of the message it is responding to. If a message is
+	// unidirectional, i.e. not part of a request/response pair, this is set to
+	// 255.
+	Tag uint8
+
+	// 1 byte reserved, set to 0x00.
+
+	// Length is the length of this layer's payload in bytes.
+	Length uint8
+}
+
+// LayerType returns LayerTypeASF. It partially satisfies Layer and
+// SerializableLayer.
+func (*ASF) LayerType() gopacket.LayerType {
+	return LayerTypeASF
+}
+
+// CanDecode returns LayerTypeASF. It partially satisfies DecodingLayer.
+func (a *ASF) CanDecode() gopacket.LayerClass {
+	return a.LayerType()
+}
+
+// DecodeFromBytes makes the layer represent the provided bytes. It partially
+// satisfies DecodingLayer.
+func (a *ASF) DecodeFromBytes(data []byte, df gopacket.DecodeFeedback) error {
+	if len(data) < 8 {
+		df.SetTruncated()
+		return fmt.Errorf("invalid ASF data header, length %v less than 8",
+			len(data))
+	}
+
+	a.BaseLayer.Contents = data[:8]
+	a.BaseLayer.Payload = data[8:]
+
+	a.Enterprise = binary.BigEndian.Uint32(data[:4])
+	a.Type = uint8(data[4])
+	a.Tag = uint8(data[5])
+	// 1 byte reserved
+	a.Length = uint8(data[7])
+	return nil
+}
+
+// NextLayerType returns the layer type corresponding to the message type of
+// this ASF data layer. This partially satisfies DecodingLayer.
+func (a *ASF) NextLayerType() gopacket.LayerType {
+	return a.ASFDataIdentifier.LayerType()
+}
+
+// SerializeTo writes the serialized fom of this layer into the SerializeBuffer,
+// partially satisfying SerializableLayer.
+func (a *ASF) SerializeTo(b gopacket.SerializeBuffer, opts gopacket.SerializeOptions) error {
+	payload := b.Bytes()
+	bytes, err := b.PrependBytes(8)
+	if err != nil {
+		return err
+	}
+	binary.BigEndian.PutUint32(bytes[:4], a.Enterprise)
+	bytes[4] = uint8(a.Type)
+	bytes[5] = a.Tag
+	bytes[6] = 0x00
+	if opts.FixLengths {
+		a.Length = uint8(len(payload))
+	}
+	bytes[7] = a.Length
+	return nil
+}
+
+// decodeASF decodes the byte slice into an RMCP-ASF data struct.
+func decodeASF(data []byte, p gopacket.PacketBuilder) error {
+	return decodingLayerDecoder(&ASF{}, data, p)
+}

vendor/github.com/google/gopacket/layers/asf_presencepong.go

@@ -0,0 +1,194 @@
+// Copyright 2019 The GoPacket Authors. All rights reserved.
+//
+// Use of this source code is governed by a BSD-style license that can be found
+// in the LICENSE file in the root of the source tree.
+
+package layers
+
+// This file implements the RMCP ASF Presence Pong message, specified in section
+// 3.2.4.3 of
+// https://www.dmtf.org/sites/default/files/standards/documents/DSP0136.pdf. It
+// also contains non-competing elements from IPMI v2.0, specified in section
+// 13.2.4 of
+// https://www.intel.com/content/dam/www/public/us/en/documents/specification-updates/ipmi-intelligent-platform-mgt-interface-spec-2nd-gen-v2-0-spec-update.pdf.
+
+import (
+	"encoding/binary"
+	"fmt"
+
+	"github.com/google/gopacket"
+)
+
+type (
+	// ASFEntity is the type of individual entities that a Presence Pong
+	// response can indicate support of. The entities currently implemented by
+	// the spec are IPMI and ASFv1.
+	ASFEntity uint8
+
+	// ASFInteraction is the type of individual interactions that a Presence
+	// Pong response can indicate support for. The interactions currently
+	// implemented by the spec are RMCP security extensions. Although not
+	// specified, IPMI uses this field to indicate support for DASH, which is
+	// supported as well.
+	ASFInteraction uint8
+)
+
+const (
+	// ASFDCMIEnterprise is the IANA-assigned Enterprise Number of the Data
+	// Center Manageability Interface Forum. The Presence Pong response's
+	// Enterprise field being set to this value indicates support for DCMI. The
+	// DCMI spec regards the OEM field as reserved, so these should be null.
+	ASFDCMIEnterprise uint32 = 36465
+
+	// ASFPresencePongEntityIPMI ANDs with Presence Pong's supported entities
+	// field if the managed system supports IPMI.
+	ASFPresencePongEntityIPMI ASFEntity = 1 << 7
+
+	// ASFPresencePongEntityASFv1 ANDs with Presence Pong's supported entities
+	// field if the managed system supports ASF v1.0.
+	ASFPresencePongEntityASFv1 ASFEntity = 1
+
+	// ASFPresencePongInteractionSecurityExtensions ANDs with Presence Pong's
+	// supported interactions field if the managed system supports RMCP v2.0
+	// security extensions. See section 3.2.3.
+	ASFPresencePongInteractionSecurityExtensions ASFInteraction = 1 << 7
+
+	// ASFPresencePongInteractionDASH ANDs with Presence Pong's supported
+	// interactions field if the managed system supports DMTF DASH. See
+	// https://www.dmtf.org/standards/dash.
+	ASFPresencePongInteractionDASH ASFInteraction = 1 << 5
+)
+
+// ASFPresencePong defines the structure of a Presence Pong message's payload.
+// See section 3.2.4.3.
+type ASFPresencePong struct {
+	BaseLayer
+
+	// Enterprise is the IANA Enterprise Number of an entity that has defined
+	// OEM-specific capabilities for the managed client. If no such capabilities
+	// exist, this is set to ASF's IANA Enterprise Number.
+	Enterprise uint32
+
+	// OEM identifies OEM-specific capabilities. Its structure is defined by the
+	// OEM. This is set to 0s if no OEM-specific capabilities exist. This
+	// implementation does not change byte order from the wire for this field.
+	OEM [4]byte
+
+	// We break out entities and interactions into separate booleans as
+	// discovery is the entire point of this type of message, so we assume they
+	// are accessed. It also makes gopacket's default layer printing more
+	// useful.
+
+	// IPMI is true if IPMI is supported by the managed system. There is no
+	// explicit version in the specification, however given the dates, this is
+	// assumed to be IPMI v1.0.  Support for IPMI is contained in the "supported
+	// entities" field of the presence pong payload.
+	IPMI bool
+
+	// ASFv1 indicates support for ASF v1.0. This seems somewhat redundant as
+	// ASF must be supported in order to receive a response. This is contained
+	// in the "supported entities" field of the presence pong payload.
+	ASFv1 bool
+
+	// SecurityExtensions indicates support for RMCP Security Extensions,
+	// specified in ASF v2.0. This will always be false for v1.x
+	// implementations. This is contained in the "supported interactions" field
+	// of the presence pong payload. This field is defined in ASF v1.0, but has
+	// no useful value.
+	SecurityExtensions bool
+
+	// DASH is true if DMTF DASH is supported. This is not specified in ASF
+	// v2.0, but in IPMI v2.0, however the former does not preclude it, so we
+	// support it.
+	DASH bool
+
+	// 6 bytes reserved after the entities and interactions fields, set to 0s.
+}
+
+// SupportsDCMI returns whether the Presence Pong message indicates support for
+// the Data Center Management Interface, which is an extension of IPMI v2.0.
+func (a *ASFPresencePong) SupportsDCMI() bool {
+	return a.Enterprise == ASFDCMIEnterprise && a.IPMI && a.ASFv1
+}
+
+// LayerType returns LayerTypeASFPresencePong. It partially satisfies Layer and
+// SerializableLayer.
+func (*ASFPresencePong) LayerType() gopacket.LayerType {
+	return LayerTypeASFPresencePong
+}
+
+// CanDecode returns LayerTypeASFPresencePong. It partially satisfies
+// DecodingLayer.
+func (a *ASFPresencePong) CanDecode() gopacket.LayerClass {
+	return a.LayerType()
+}
+
+// DecodeFromBytes makes the layer represent the provided bytes. It partially
+// satisfies DecodingLayer.
+func (a *ASFPresencePong) DecodeFromBytes(data []byte, df gopacket.DecodeFeedback) error {
+	if len(data) < 16 {
+		df.SetTruncated()
+		return fmt.Errorf("invalid ASF presence pong payload, length %v less than 16",
+			len(data))
+	}
+
+	a.BaseLayer.Contents = data[:16]
+	a.BaseLayer.Payload = data[16:]
+
+	a.Enterprise = binary.BigEndian.Uint32(data[:4])
+	copy(a.OEM[:], data[4:8]) // N.B. no byte order change
+	a.IPMI = data[8]&uint8(ASFPresencePongEntityIPMI) != 0
+	a.ASFv1 = data[8]&uint8(ASFPresencePongEntityASFv1) != 0
+	a.SecurityExtensions = data[9]&uint8(ASFPresencePongInteractionSecurityExtensions) != 0
+	a.DASH = data[9]&uint8(ASFPresencePongInteractionDASH) != 0
+	// ignore remaining 6 bytes; should be set to 0s
+	return nil
+}
+
+// NextLayerType returns LayerTypePayload, as there are no further layers to
+// decode. This partially satisfies DecodingLayer.
+func (a *ASFPresencePong) NextLayerType() gopacket.LayerType {
+	return gopacket.LayerTypePayload
+}
+
+// SerializeTo writes the serialized fom of this layer into the SerializeBuffer,
+// partially satisfying SerializableLayer.
+func (a *ASFPresencePong) SerializeTo(b gopacket.SerializeBuffer, _ gopacket.SerializeOptions) error {
+	bytes, err := b.PrependBytes(16)
+	if err != nil {
+		return err
+	}
+
+	binary.BigEndian.PutUint32(bytes[:4], a.Enterprise)
+
+	copy(bytes[4:8], a.OEM[:])
+
+	bytes[8] = 0
+	if a.IPMI {
+		bytes[8] |= uint8(ASFPresencePongEntityIPMI)
+	}
+	if a.ASFv1 {
+		bytes[8] |= uint8(ASFPresencePongEntityASFv1)
+	}
+
+	bytes[9] = 0
+	if a.SecurityExtensions {
+		bytes[9] |= uint8(ASFPresencePongInteractionSecurityExtensions)
+	}
+	if a.DASH {
+		bytes[9] |= uint8(ASFPresencePongInteractionDASH)
+	}
+
+	// zero-out remaining 6 bytes
+	for i := 10; i < len(bytes); i++ {
+		bytes[i] = 0x00
+	}
+
+	return nil
+}
+
+// decodeASFPresencePong decodes the byte slice into an RMCP-ASF Presence Pong
+// struct.
+func decodeASFPresencePong(data []byte, p gopacket.PacketBuilder) error {
+	return decodingLayerDecoder(&ASFPresencePong{}, data, p)
+}

vendor/github.com/google/gopacket/layers/dhcpv4.go

@@ -124,6 +124,10 @@ func (d *DHCPv4) LayerType() gopacket.LayerType { return LayerTypeDHCPv4 }
 
 // DecodeFromBytes decodes the given bytes into this layer.
 func (d *DHCPv4) DecodeFromBytes(data []byte, df gopacket.DecodeFeedback) error {
+	if len(data) < 240 {
+		df.SetTruncated()
+		return fmt.Errorf("DHCPv4 length %d too short", len(data))
+	}
 	d.Options = d.Options[:0]
 	d.Operation = DHCPOp(data[0])
 	d.HardwareType = LinkType(data[1])
@@ -168,6 +172,9 @@ func (d *DHCPv4) DecodeFromBytes(data []byte, df gopacket.DecodeFeedback) error
 			start += int(o.Length) + 2
 		}
 	}
+
+	d.Contents = data
+
 	return nil
 }
 

vendor/github.com/google/gopacket/layers/dhcpv6.go

@@ -88,12 +88,20 @@ func (d *DHCPv6) LayerType() gopacket.LayerType { return LayerTypeDHCPv6 }
 
 // DecodeFromBytes decodes the given bytes into this layer.
 func (d *DHCPv6) DecodeFromBytes(data []byte, df gopacket.DecodeFeedback) error {
+	if len(data) < 4 {
+		df.SetTruncated()
+		return fmt.Errorf("DHCPv6 length %d too short", len(data))
+	}
 	d.BaseLayer = BaseLayer{Contents: data}
 	d.Options = d.Options[:0]
 	d.MsgType = DHCPv6MsgType(data[0])
 
 	offset := 0
 	if d.MsgType == DHCPv6MsgTypeRelayForward || d.MsgType == DHCPv6MsgTypeRelayReply {
+		if len(data) < 34 {
+			df.SetTruncated()
+			return fmt.Errorf("DHCPv6 length %d too short for message type %d", len(data), d.MsgType)
+		}
 		d.HopCount = data[1]
 		d.LinkAddr = net.IP(data[2:18])
 		d.PeerAddr = net.IP(data[18:34])

vendor/github.com/google/gopacket/layers/dns.go

@@ -52,25 +52,26 @@ type DNSType uint16
 
 // DNSType known values.
 const (
-	DNSTypeA     DNSType = 1  // a host address
-	DNSTypeNS    DNSType = 2  // an authoritative name server
-	DNSTypeMD    DNSType = 3  // a mail destination (Obsolete - use MX)
-	DNSTypeMF    DNSType = 4  // a mail forwarder (Obsolete - use MX)
-	DNSTypeCNAME DNSType = 5  // the canonical name for an alias
-	DNSTypeSOA   DNSType = 6  // marks the start of a zone of authority
-	DNSTypeMB    DNSType = 7  // a mailbox domain name (EXPERIMENTAL)
-	DNSTypeMG    DNSType = 8  // a mail group member (EXPERIMENTAL)
-	DNSTypeMR    DNSType = 9  // a mail rename domain name (EXPERIMENTAL)
-	DNSTypeNULL  DNSType = 10 // a null RR (EXPERIMENTAL)
-	DNSTypeWKS   DNSType = 11 // a well known service description
-	DNSTypePTR   DNSType = 12 // a domain name pointer
-	DNSTypeHINFO DNSType = 13 // host information
-	DNSTypeMINFO DNSType = 14 // mailbox or mail list information
-	DNSTypeMX    DNSType = 15 // mail exchange
-	DNSTypeTXT   DNSType = 16 // text strings
-	DNSTypeAAAA  DNSType = 28 // a IPv6 host address [RFC3596]
-	DNSTypeSRV   DNSType = 33 // server discovery [RFC2782] [RFC6195]
-	DNSTypeOPT   DNSType = 41 // OPT Pseudo-RR [RFC6891]
+	DNSTypeA     DNSType = 1   // a host address
+	DNSTypeNS    DNSType = 2   // an authoritative name server
+	DNSTypeMD    DNSType = 3   // a mail destination (Obsolete - use MX)
+	DNSTypeMF    DNSType = 4   // a mail forwarder (Obsolete - use MX)
+	DNSTypeCNAME DNSType = 5   // the canonical name for an alias
+	DNSTypeSOA   DNSType = 6   // marks the start of a zone of authority
+	DNSTypeMB    DNSType = 7   // a mailbox domain name (EXPERIMENTAL)
+	DNSTypeMG    DNSType = 8   // a mail group member (EXPERIMENTAL)
+	DNSTypeMR    DNSType = 9   // a mail rename domain name (EXPERIMENTAL)
+	DNSTypeNULL  DNSType = 10  // a null RR (EXPERIMENTAL)
+	DNSTypeWKS   DNSType = 11  // a well known service description
+	DNSTypePTR   DNSType = 12  // a domain name pointer
+	DNSTypeHINFO DNSType = 13  // host information
+	DNSTypeMINFO DNSType = 14  // mailbox or mail list information
+	DNSTypeMX    DNSType = 15  // mail exchange
+	DNSTypeTXT   DNSType = 16  // text strings
+	DNSTypeAAAA  DNSType = 28  // a IPv6 host address [RFC3596]
+	DNSTypeSRV   DNSType = 33  // server discovery [RFC2782] [RFC6195]
+	DNSTypeOPT   DNSType = 41  // OPT Pseudo-RR [RFC6891]
+	DNSTypeURI   DNSType = 256 // URI RR [RFC7553]
 )
 
 func (dt DNSType) String() string {
@@ -115,6 +116,8 @@ func (dt DNSType) String() string {
 		return "SRV"
 	case DNSTypeOPT:
 		return "OPT"
+	case DNSTypeURI:
+		return "URI"
 	}
 }
 
@@ -123,25 +126,26 @@ type DNSResponseCode uint8
 
 // DNSResponseCode known values.
 const (
-	DNSResponseCodeNoErr    DNSResponseCode = 0  // No error
-	DNSResponseCodeFormErr  DNSResponseCode = 1  // Format Error                       [RFC1035]
-	DNSResponseCodeServFail DNSResponseCode = 2  // Server Failure                     [RFC1035]
-	DNSResponseCodeNXDomain DNSResponseCode = 3  // Non-Existent Domain                [RFC1035]
-	DNSResponseCodeNotImp   DNSResponseCode = 4  // Not Implemented                    [RFC1035]
-	DNSResponseCodeRefused  DNSResponseCode = 5  // Query Refused                      [RFC1035]
-	DNSResponseCodeYXDomain DNSResponseCode = 6  // Name Exists when it should not     [RFC2136]
-	DNSResponseCodeYXRRSet  DNSResponseCode = 7  // RR Set Exists when it should not   [RFC2136]
-	DNSResponseCodeNXRRSet  DNSResponseCode = 8  // RR Set that should exist does not  [RFC2136]
-	DNSResponseCodeNotAuth  DNSResponseCode = 9  // Server Not Authoritative for zone  [RFC2136]
-	DNSResponseCodeNotZone  DNSResponseCode = 10 // Name not contained in zone         [RFC2136]
-	DNSResponseCodeBadVers  DNSResponseCode = 16 // Bad OPT Version                    [RFC2671]
-	DNSResponseCodeBadSig   DNSResponseCode = 16 // TSIG Signature Failure             [RFC2845]
-	DNSResponseCodeBadKey   DNSResponseCode = 17 // Key not recognized                 [RFC2845]
-	DNSResponseCodeBadTime  DNSResponseCode = 18 // Signature out of time window       [RFC2845]
-	DNSResponseCodeBadMode  DNSResponseCode = 19 // Bad TKEY Mode                      [RFC2930]
-	DNSResponseCodeBadName  DNSResponseCode = 20 // Duplicate key name                 [RFC2930]
-	DNSResponseCodeBadAlg   DNSResponseCode = 21 // Algorithm not supported            [RFC2930]
-	DNSResponseCodeBadTruc  DNSResponseCode = 22 // Bad Truncation                     [RFC4635]
+	DNSResponseCodeNoErr     DNSResponseCode = 0  // No error
+	DNSResponseCodeFormErr   DNSResponseCode = 1  // Format Error                       [RFC1035]
+	DNSResponseCodeServFail  DNSResponseCode = 2  // Server Failure                     [RFC1035]
+	DNSResponseCodeNXDomain  DNSResponseCode = 3  // Non-Existent Domain                [RFC1035]
+	DNSResponseCodeNotImp    DNSResponseCode = 4  // Not Implemented                    [RFC1035]
+	DNSResponseCodeRefused   DNSResponseCode = 5  // Query Refused                      [RFC1035]
+	DNSResponseCodeYXDomain  DNSResponseCode = 6  // Name Exists when it should not     [RFC2136]
+	DNSResponseCodeYXRRSet   DNSResponseCode = 7  // RR Set Exists when it should not   [RFC2136]
+	DNSResponseCodeNXRRSet   DNSResponseCode = 8  // RR Set that should exist does not  [RFC2136]
+	DNSResponseCodeNotAuth   DNSResponseCode = 9  // Server Not Authoritative for zone  [RFC2136]
+	DNSResponseCodeNotZone   DNSResponseCode = 10 // Name not contained in zone         [RFC2136]
+	DNSResponseCodeBadVers   DNSResponseCode = 16 // Bad OPT Version                    [RFC2671]
+	DNSResponseCodeBadSig    DNSResponseCode = 16 // TSIG Signature Failure             [RFC2845]
+	DNSResponseCodeBadKey    DNSResponseCode = 17 // Key not recognized                 [RFC2845]
+	DNSResponseCodeBadTime   DNSResponseCode = 18 // Signature out of time window       [RFC2845]
+	DNSResponseCodeBadMode   DNSResponseCode = 19 // Bad TKEY Mode                      [RFC2930]
+	DNSResponseCodeBadName   DNSResponseCode = 20 // Duplicate key name                 [RFC2930]
+	DNSResponseCodeBadAlg    DNSResponseCode = 21 // Algorithm not supported            [RFC2930]
+	DNSResponseCodeBadTruc   DNSResponseCode = 22 // Bad Truncation                     [RFC4635]
+	DNSResponseCodeBadCookie DNSResponseCode = 23 // Bad/missing Server Cookie          [RFC7873]
 )
 
 func (drc DNSResponseCode) String() string {
@@ -184,6 +188,8 @@ func (drc DNSResponseCode) String() string {
 		return "Algorithm not supported"
 	case DNSResponseCodeBadTruc:
 		return "Bad Truncation"
+	case DNSResponseCodeBadCookie:
+		return "Bad Cookie"
 	}
 }
 
@@ -367,6 +373,10 @@ func (d *DNS) DecodeFromBytes(data []byte, df gopacket.DecodeFeedback) error {
 			d.Additionals = d.Additionals[:i] // strip off erroneous value
 			return err
 		}
+		// extract extended RCODE from OPT RRs, RFC 6891 section 6.1.3
+		if d.Additionals[i].Type == DNSTypeOPT {
+			d.ResponseCode = DNSResponseCode(uint8(d.ResponseCode) | uint8(d.Additionals[i].TTL>>20&0xF0))
+		}
 	}
 
 	if uint16(len(d.Questions)) != d.QDCount {
@@ -427,6 +437,8 @@ func recSize(rr *DNSResourceRecord) int {
 		return l
 	case DNSTypeSRV:
 		return 6 + len(rr.SRV.Name) + 2
+	case DNSTypeURI:
+		return 4 + len(rr.URI.Target)
 	case DNSTypeOPT:
 		l := len(rr.OPT) * 4
 		for _, opt := range rr.OPT {
@@ -684,6 +696,7 @@ type DNSResourceRecord struct {
 	SRV            DNSSRV
 	MX             DNSMX
 	OPT            []DNSOPT // See RFC 6891, section 6.1.2
+	URI            DNSURI
 
 	// Undecoded TXT for backward compatibility
 	TXT []byte
@@ -781,6 +794,10 @@ func (rr *DNSResourceRecord) encode(data []byte, offset int, opts gopacket.Seria
 		binary.BigEndian.PutUint16(data[noff+12:], rr.SRV.Weight)
 		binary.BigEndian.PutUint16(data[noff+14:], rr.SRV.Port)
 		encodeName(rr.SRV.Name, data, noff+16)
+	case DNSTypeURI:
+		binary.BigEndian.PutUint16(data[noff+10:], rr.URI.Priority)
+		binary.BigEndian.PutUint16(data[noff+12:], rr.URI.Weight)
+		copy(data[noff+14:], rr.URI.Target)
 	case DNSTypeOPT:
 		noff2 := noff + 10
 		for _, opt := range rr.OPT {
@@ -813,6 +830,9 @@ func (rr *DNSResourceRecord) String() string {
 		}
 		return "OPT " + strings.Join(opts, ",")
 	}
+	if rr.Type == DNSTypeURI {
+		return fmt.Sprintf("URI %d %d %s", rr.URI.Priority, rr.URI.Weight, string(rr.URI.Target))
+	}
 	if rr.Class == DNSClassIN {
 		switch rr.Type {
 		case DNSTypeA, DNSTypeAAAA:
@@ -924,6 +944,10 @@ func (rr *DNSResourceRecord) decodeRData(data []byte, offset int, buffer *[]byte
 			return err
 		}
 		rr.MX.Name = name
+	case DNSTypeURI:
+		rr.URI.Priority = binary.BigEndian.Uint16(data[offset : offset+2])
+		rr.URI.Weight = binary.BigEndian.Uint16(data[offset+2 : offset+4])
+		rr.URI.Target = rr.Data[4:]
 	case DNSTypeSRV:
 		rr.SRV.Priority = binary.BigEndian.Uint16(data[offset : offset+2])
 		rr.SRV.Weight = binary.BigEndian.Uint16(data[offset+2 : offset+4])
@@ -964,6 +988,12 @@ type DNSMX struct {
 	Name       []byte
 }
 
+// DNSURI is a URI record, defining a target (URI) of a server/service
+type DNSURI struct {
+	Priority, Weight uint16
+	Target           []byte
+}
+
 // DNSOptionCode represents the code of a DNS Option, see RFC6891, section 6.1.2
 type DNSOptionCode uint16
 

vendor/github.com/google/gopacket/layers/dot11.go

@@ -922,6 +922,7 @@ func (m *Dot11) DecodeFromBytes(data []byte, df gopacket.DecodeFeedback) error {
 	}
 	m.Type = Dot11Type((data[0])&0xFC) >> 2
 
+	m.DataLayer = nil
 	m.Proto = uint8(data[0]) & 0x0003
 	m.Flags = Dot11Flags(data[1])
 	m.DurationID = binary.LittleEndian.Uint16(data[2:4])

vendor/github.com/google/gopacket/layers/dot1q.go

@@ -27,6 +27,10 @@ func (d *Dot1Q) LayerType() gopacket.LayerType { return LayerTypeDot1Q }
 
 // DecodeFromBytes decodes the given bytes into this layer.
 func (d *Dot1Q) DecodeFromBytes(data []byte, df gopacket.DecodeFeedback) error {
+	if len(data) < 4 {
+		df.SetTruncated()
+		return fmt.Errorf("802.1Q tag length %d too short", len(data))
+	}
 	d.Priority = (data[0] & 0xE0) >> 5
 	d.DropEligible = data[0]&0x10 != 0
 	d.VLANIdentifier = binary.BigEndian.Uint16(data[:2]) & 0x0FFF

vendor/github.com/google/gopacket/layers/eap.go

@@ -47,9 +47,17 @@ func (e *EAP) LayerType() gopacket.LayerType { return LayerTypeEAP }
 
 // DecodeFromBytes decodes the given bytes into this layer.
 func (e *EAP) DecodeFromBytes(data []byte, df gopacket.DecodeFeedback) error {
+	if len(data) < 4 {
+		df.SetTruncated()
+		return fmt.Errorf("EAP length %d too short", len(data))
+	}
 	e.Code = EAPCode(data[0])
 	e.Id = data[1]
 	e.Length = binary.BigEndian.Uint16(data[2:4])
+	if len(data) < int(e.Length) {
+		df.SetTruncated()
+		return fmt.Errorf("EAP length %d too short, %d expected", len(data), e.Length)
+	}
 	switch {
 	case e.Length > 4:
 		e.Type = EAPType(data[4])

vendor/github.com/google/gopacket/layers/eapol.go

@@ -25,6 +25,10 @@ func (e *EAPOL) LayerType() gopacket.LayerType { return LayerTypeEAPOL }
 
 // DecodeFromBytes decodes the given bytes into this layer.
 func (e *EAPOL) DecodeFromBytes(data []byte, df gopacket.DecodeFeedback) error {
+	if len(data) < 4 {
+		df.SetTruncated()
+		return fmt.Errorf("EAPOL length %d too short", len(data))
+	}
 	e.Version = data[0]
 	e.Type = EAPOLType(data[1])
 	e.Length = binary.BigEndian.Uint16(data[2:4])

vendor/github.com/google/gopacket/layers/enums.go

@@ -8,7 +8,6 @@
 package layers
 
 import (
-	"errors"
 	"fmt"
 	"runtime"
 
@@ -26,14 +25,6 @@ type EnumMetadata struct {
 	LayerType gopacket.LayerType
 }
 
-// errorFunc returns a decoder that spits out a specific error message.
-func errorFunc(msg string) gopacket.Decoder {
-	var e = errors.New(msg)
-	return gopacket.DecodeFunc(func([]byte, gopacket.PacketBuilder) error {
-		return e
-	})
-}
-
 // EthernetType is an enumeration of ethernet type values, and acts as a decoder
 // for any type it supports.
 type EthernetType uint16
@@ -130,6 +121,8 @@ const (
 	LinkTypeLinuxIRDA      LinkType = 144
 	LinkTypeLinuxLAPD      LinkType = 177
 	LinkTypeLinuxUSB       LinkType = 220
+	LinkTypeFC2            LinkType = 224
+	LinkTypeFC2Framed      LinkType = 225
 	LinkTypeIPv4           LinkType = 228
 	LinkTypeIPv6           LinkType = 229
 )