Add githubAction to build & push docker image on github release published event. (#881)

fixes #880 👍 

Update Dockerfile
- fix alpine image version
- add RELEASE_VERSION build arg

Add githubAction to build & push docker image on github release published event
- add github workflow file
- add dependabot file to automatically propose PRs on githubActions version update*

> ⚠️ *Note1: you will have to add `DOCKERHUB_USERNAME` and `DOCKERHUB_TOKEN` as github secrets in order for the 
> ⚠️ *Note2: github dependabot support must be enabled in order for this action to work
githubAction to be able to login to the dockerHub and push images on it 👍
> *Note3:  we can add other dependabot entries in order to update Go modules and Dockerfile base images 👍 (i.e `FROM` parts)

.github/dependabot.yaml

@@ -0,0 +1,10 @@
+version: 2
+updates:
+  # Maintain dependencies for GitHub Actions
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+    pull-request-branch-name:
+      # Separate sections of the branch name with a hyphen
+      separator: "-"

.github/workflows/ci-docker.yaml

@@ -0,0 +1,38 @@
+name: ci
+on:
+  release:
+    types: [published]
+jobs:
+  docker-build-and-push:
+    runs-on: ubuntu-latest
+    steps:
+      -
+        name: "Checkout repository"
+        uses: actions/checkout@v2
+      -
+        name: "Set up Docker Buildx"
+        uses: docker/setup-buildx-action@v1
+      -
+        name: "Cache Docker layers"
+        uses: actions/cache@v2
+        with:
+          path: /tmp/.buildx-cache
+          key: ${{ runner.os }}-buildx-${{ github.sha }}
+          restore-keys: |
+            ${{ runner.os }}-buildx-
+      -
+        name: "Login to Container Registry"
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      -
+        name: "Build and push docker image"
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          push: true
+          tags: ${{ github.repository }}:${{ github.event.release.tag_name }}
+          build-args: RELEASE_VERSION=${{ github.event.release.tag_name }}
+          cache-from: type=local,src=/tmp/.buildx-cache
+          cache-to: type=local,mode=max,dest=/tmp/.buildx-cache

.github/workflows/ci-test.yaml

@@ -0,0 +1,29 @@
+name: test
+on: [push, pull_request]
+jobs:
+  test:
+    strategy:
+      matrix:
+        go-version: [1.14.x, 1.15.x] # two latest minor versions
+    runs-on: ubuntu-latest
+    steps:
+    - name: update package index
+      run: sudo apt-get update
+    - name: install libpcap
+      run: sudo apt-get install libpcap-dev -y
+    - name: install Go
+      uses: actions/setup-go@v2
+      with:
+        go-version: ${{ matrix.go-version }}
+    - name: checkout code
+      uses: actions/checkout@v2
+    - uses: actions/cache@v2
+      with:
+        path: |
+          ~/go/pkg/mod              # Module download cache
+          ~/.cache/go-build         # Build cache (Linux)
+        key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
+        restore-keys: |
+          ${{ runner.os }}-go-
+    - name: test
+      run: sudo go test ./... -v -timeout 120s -race

.travis.yml

@@ -1,9 +0,0 @@
-dist: focal
-
-language: go
-go: 1.14
-
-script: sudo -E bash -c "source /etc/profile && eval '$(gimme 1.14)' && go test ./... -v -timeout 120s"
-
-before_install:
-  - sudo apt-get install libpcap-dev -y

Dockerfile

@@ -1,6 +1,9 @@
-FROM alpine:latest as builder
+FROM alpine:3.12 as builder
+
+ARG RELEASE_VERSION
+
 RUN apk add --no-cache ca-certificates openssl
-RUN wget https://github.com/buger/goreplay/releases/download/v1.2.0/gor_v1.2.0_x64.tar.gz -O gor.tar.gz
+RUN wget https://github.com/buger/goreplay/releases/download/${RELEASE_VERSION}/gor_${RELEASE_VERSION}_x64.tar.gz -O gor.tar.gz
 RUN tar xzf gor.tar.gz
 
 FROM scratch

Makefile

@@ -130,7 +130,7 @@ bash:
 
 
 FPMCOMMON= \
-    --name goreplay \
+    --name gor \
     --description "GoReplay is an open-source network monitoring tool which can record your live traffic, and use it for shadowing, load testing, monitoring and detailed analysis." \
     -v $(VERSION) \
     --vendor "Leonid Bugaev" \

capture/capture.go

@@ -4,6 +4,8 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"io"
+	"log"
 	"net"
 	"os"
 	"runtime"
@@ -14,10 +16,11 @@ import (
 	"github.com/google/gopacket"
 	"github.com/google/gopacket/layers"
 	"github.com/google/gopacket/pcap"
+	"golang.org/x/sys/unix"
 )
 
 // Handler is a function that is used to handle packets
-type Handler func(gopacket.Packet)
+type Handler func(*Packet)
 
 // PcapOptions options that can be set on a pcap capture handle,
 // these options take effect on inactive pcap handles
@@ -31,19 +34,13 @@ type PcapOptions struct {
 	Snaplen       bool          `json:"input-raw-override-snaplen"`
 }
 
-// NetInterface represents network interface
-type NetInterface struct {
-	net.Interface
-	IPs []string
-}
-
 // Listener handle traffic capture, this is its representation.
 type Listener struct {
 	sync.Mutex
 	Transport  string       // transport layer default to tcp
 	Activate   func() error // function is used to activate the engine. it must be called before reading packets
 	Handles    map[string]gopacket.PacketDataSource
-	Interfaces []NetInterface
+	Interfaces []net.Interface
 	loopIndex  int
 	Reading    chan bool // this channel is closed when the listener has started reading packets
 	PcapOptions
@@ -53,8 +50,8 @@ type Listener struct {
 
 	host string // pcap file name or interface (name, hardware addr, index or ip address)
 
-	quit    chan bool
-	packets chan gopacket.Packet
+	closeDone chan struct{}
+	quit      chan struct{}
 }
 
 // EngineType ...
@@ -110,9 +107,9 @@ func NewListener(host string, port uint16, transport string, engine EngineType,
 	}
 	l.Handles = make(map[string]gopacket.PacketDataSource)
 	l.trackResponse = trackResponse
-	l.packets = make(chan gopacket.Packet, 1000)
-	l.quit = make(chan bool, 1)
-	l.Reading = make(chan bool, 1)
+	l.closeDone = make(chan struct{})
+	l.quit = make(chan struct{})
+	l.Reading = make(chan bool)
 	switch engine {
 	default:
 		l.Engine = EnginePcap
@@ -139,30 +136,19 @@ func (l *Listener) SetPcapOptions(opts PcapOptions) {
 }
 
 // Listen listens for packets from the handles, and call handler on every packet received
-// until the context done signal is sent or EOF on handles.
-// this function should be called after activating pcap handles
+// until the context done signal is sent or there is unrecoverable error on all handles.
+// this function must be called after activating pcap handles
 func (l *Listener) Listen(ctx context.Context, handler Handler) (err error) {
-	l.read()
+	l.read(handler)
 	done := ctx.Done()
-	var p gopacket.Packet
-	var ok bool
-	for {
-		select {
-		case <-done:
-			l.quit <- true
-			close(l.quit)
-			err = ctx.Err()
-			done = nil
-		case p, ok = <-l.packets:
-			if !ok {
-				return
-			}
-			if p == nil {
-				continue
-			}
-			handler(p)
-		}
+	select {
+	case <-done:
+		close(l.quit) // signal close on all handles
+		<-l.closeDone // wait all handles to be closed
+		err = ctx.Err()
+	case <-l.closeDone: // all handles closed voluntarily
 	}
+	return
 }
 
 // ListenBackground is like listen but can run concurrently and signal error through channel
@@ -179,7 +165,7 @@ func (l *Listener) ListenBackground(ctx context.Context, handler Handler) chan e
 
 // Filter returns automatic filter applied by goreplay
 // to a pcap handle of a specific interface
-func (l *Listener) Filter(ifi NetInterface) (filter string) {
+func (l *Listener) Filter(ifi net.Interface) (filter string) {
 	// https://www.tcpdump.org/manpages/pcap-filter.7.html
 
 	port := fmt.Sprintf("portrange 0-%d", 1<<16-1)
@@ -201,7 +187,7 @@ func (l *Listener) Filter(ifi NetInterface) (filter string) {
 // PcapDumpHandler returns a handler to write packet data in PCAP
 // format, See http://wiki.wireshark.org/Development/LibpcapFileFormathandler.
 // if link layer is invalid Ethernet is assumed
-func PcapDumpHandler(file *os.File, link layers.LinkType, debugger func(int, ...interface{})) (handler func(packet gopacket.Packet), err error) {
+func PcapDumpHandler(file *os.File, link layers.LinkType) (handler func(packet *Packet) error, err error) {
 	if link.String() == "" {
 		link = layers.LinkTypeEthernet
 	}
@@ -210,25 +196,20 @@ func PcapDumpHandler(file *os.File, link layers.LinkType, debugger func(int, ...
 	if err != nil {
 		return nil, err
 	}
-	return func(packet gopacket.Packet) {
-		err = w.WritePacket(packet.Metadata().CaptureInfo, packet.Data())
-		if err != nil && debugger != nil {
-			go debugger(3, err)
-		}
+	return func(packet *Packet) error {
+		return w.WritePacket(*packet.Info, packet.Data)
 	}, nil
 }
 
 // PcapHandle returns new pcap Handle from dev on success.
 // this function should be called after setting all necessary options for this listener
-func (l *Listener) PcapHandle(ifi NetInterface) (handle *pcap.Handle, err error) {
+func (l *Listener) PcapHandle(ifi net.Interface) (handle *pcap.Handle, err error) {
 	var inactive *pcap.InactiveHandle
 	inactive, err = pcap.NewInactiveHandle(ifi.Name)
-	if inactive != nil && err != nil {
-		defer inactive.CleanUp()
-	}
 	if err != nil {
 		return nil, fmt.Errorf("inactive handle error: %q, interface: %q", err, ifi.Name)
 	}
+	defer inactive.CleanUp()
 	if l.TimestampType != "" {
 		var ts pcap.TimestampSource
 		ts, err = pcap.TimestampSourceFromString(l.TimestampType)
@@ -290,8 +271,8 @@ func (l *Listener) PcapHandle(ifi NetInterface) (handle *pcap.Handle, err error)
 }
 
 // SocketHandle returns new unix ethernet handle associated with this listener settings
-func (l *Listener) SocketHandle(ifi NetInterface) (handle Socket, err error) {
-	handle, err = NewSocket(ifi.Interface)
+func (l *Listener) SocketHandle(ifi net.Interface) (handle Socket, err error) {
+	handle, err = NewSocket(ifi)
 	if err != nil {
 		return nil, fmt.Errorf("sock raw error: %q, interface: %q", err, ifi.Name)
 	}
@@ -313,35 +294,54 @@ func (l *Listener) SocketHandle(ifi NetInterface) (handle Socket, err error) {
 	return
 }
 
-func (l *Listener) read() {
+func (l *Listener) read(handler Handler) {
 	l.Lock()
 	defer l.Unlock()
 	for key, handle := range l.Handles {
-		var source *gopacket.PacketSource
-		linkType := layers.LinkTypeEthernet
-		if _, ok := handle.(*pcap.Handle); ok {
-			linkType = handle.(*pcap.Handle).LinkType()
-		}
-		source = gopacket.NewPacketSource(handle, linkType)
-		source.Lazy = true
-		source.NoCopy = true
-		ch := source.Packets()
-		go func(key string) {
+		go func(key string, hndl gopacket.PacketDataSource) {
 			defer l.closeHandles(key)
+			linkSize := 14
+			linkType := int(layers.LinkTypeEthernet)
+			if _, ok := hndl.(*pcap.Handle); ok {
+				linkType = int(hndl.(*pcap.Handle).LinkType())
+				linkSize, ok = pcapLinkTypeLength(linkType)
+				if !ok {
+					if os.Getenv("GORDEBUG") != "0" {
+						log.Printf("can not identify link type of an interface '%s'\n", key)
+					}
+					return // can't find the linktype size
+				}
+			}
 			for {
 				select {
 				case <-l.quit:
 					return
-				case p, ok := <-ch:
-					if !ok {
+				default:
+					data, ci, err := hndl.ReadPacketData()
+					if err == nil {
+						handler(NewPacket(data, linkType, linkSize, &ci))
+						continue
+					}
+					if enext, ok := err.(pcap.NextError); ok && enext == pcap.NextErrorTimeoutExpired {
+						continue
+					}
+					if eno, ok := err.(unix.Errno); ok && eno.Temporary() {
+						continue
+					}
+					if enet, ok := err.(*net.OpError); ok && (enet.Temporary() || enet.Timeout()) {
+						continue
+					}
+					if err == io.EOF || err == io.ErrClosedPipe {
 						return
 					}
-					l.packets <- p
+					if os.Getenv("GORDEBUG") != "0" {
+						log.Printf("stopped reading from %s interface with error %s\n", key, err)
+					}
+					return
 				}
 			}
-		}(key)
+		}(key, handle)
 	}
-	l.Reading <- true
 	close(l.Reading)
 }
 
@@ -356,7 +356,7 @@ func (l *Listener) closeHandles(key string) {
 		}
 		delete(l.Handles, key)
 		if len(l.Handles) == 0 {
-			close(l.packets)
+			close(l.closeDone)
 		}
 	}
 }
@@ -394,7 +394,10 @@ func (l *Listener) activateRawSocket() error {
 		}
 		l.Handles[ifi.Name] = handle
 	}
-	return e
+	if len(l.Handles) == 0 {
+		return fmt.Errorf("raw socket handles error:%s", msg)
+	}
+	return nil
 }
 
 func (l *Listener) activatePcapFile() (err error) {
@@ -410,7 +413,7 @@ func (l *Listener) activatePcapFile() (err error) {
 	} else {
 		addr := l.host
 		l.host = ""
-		l.BPFFilter = l.Filter(NetInterface{})
+		l.BPFFilter = l.Filter(net.Interface{})
 		l.host = addr
 	}
 	if e = handle.SetBPFFilter(l.BPFFilter); e != nil {
@@ -422,59 +425,36 @@ func (l *Listener) activatePcapFile() (err error) {
 }
 
 func (l *Listener) setInterfaces() (err error) {
-	var Ifis []NetInterface
 	var ifis []net.Interface
 	ifis, err = net.Interfaces()
 	if err != nil {
-		return err
+		return
 	}
-
-	for i := 0; i < len(ifis); i++ {
+	for i := range ifis {
 		if ifis[i].Flags&net.FlagLoopback != 0 {
 			l.loopIndex = ifis[i].Index
 		}
 		if ifis[i].Flags&net.FlagUp == 0 {
 			continue
 		}
-		var addrs []net.Addr
-		addrs, err = ifis[i].Addrs()
-		if err != nil {
-			return err
-		}
-		if len(addrs) == 0 {
-			continue
-		}
-		ifi := NetInterface{}
-		ifi.Interface = ifis[i]
-		ifi.IPs = make([]string, len(addrs))
-		for j, addr := range addrs {
-			ifi.IPs[j] = cutMask(addr)
-		}
-		Ifis = append(Ifis, ifi)
-	}
-
-	if listenAll(l.host) {
-		l.Interfaces = Ifis
-		return
-	}
-	found := false
-	for _, ifi := range Ifis {
-		if isDevice(l.host, ifi) {
-			found = true
-		}
-		for _, ip := range ifi.IPs {
-			if ip == l.host {
-				found = true
-				break
-			}
-		}
-		if found {
-			l.Interfaces = []NetInterface{ifi}
+		if isDevice(l.host, ifis[i]) {
+			l.Interfaces = []net.Interface{ifis[i]}
 			return
 		}
+		addrs, e := ifis[i].Addrs()
+		if e != nil {
+			// don't give up on a failure from a single interface
+			continue
+		}
+		for _, addr := range addrs {
+			if cutMask(addr) == l.host {
+				l.Interfaces = []net.Interface{ifis[i]}
+				return
+			}
+		}
 	}
-	err = fmt.Errorf("can not find interface with addr, name or index %s", l.host)
-	return err
+	l.Interfaces = ifis
+	return
 }
 
 func cutMask(addr net.Addr) string {
@@ -487,8 +467,8 @@ func cutMask(addr net.Addr) string {
 	return mask
 }
 
-func isDevice(addr string, ifi NetInterface) bool {
-	return addr == ifi.Name || addr == fmt.Sprintf("%d", ifi.Index) || addr == ifi.HardwareAddr.String()
+func isDevice(addr string, ifi net.Interface) bool {
+	return addr == ifi.Name || addr == fmt.Sprintf("%d", ifi.Index) || (addr != "" && addr == ifi.HardwareAddr.String())
 }
 
 func listenAll(addr string) bool {

capture/capture_test.go

@@ -6,10 +6,10 @@ import (
 	"io/ioutil"
 	"net"
 	"os"
+	"sync/atomic"
 	"testing"
 	"time"
 
-	"github.com/google/gopacket"
 	"github.com/google/gopacket/layers"
 	"github.com/google/gopacket/pcap"
 )
@@ -60,69 +60,23 @@ func TestBPFFilter(t *testing.T) {
 	}
 }
 
-var decodeOpts = gopacket.DecodeOptions{Lazy: true, NoCopy: true}
-
-func generateHeaders(seq uint32, length uint16) (headers [44]byte) {
-	// set ethernet headers
-	binary.BigEndian.PutUint32(headers[0:4], uint32(layers.ProtocolFamilyIPv4))
-
-	// set ip header
-	ip := headers[4:]
-	copy(ip[0:2], []byte{4<<4 | 5, 0x28<<2 | 0x00})
-	binary.BigEndian.PutUint16(ip[2:4], length+54)
-	ip[9] = uint8(layers.IPProtocolTCP)
-	copy(ip[12:16], []byte{127, 0, 0, 1})
-	copy(ip[16:], []byte{127, 0, 0, 1})
-
-	// set tcp header
-	tcp := ip[20:]
-	binary.BigEndian.PutUint16(tcp[0:2], 45678)
-	binary.BigEndian.PutUint16(tcp[2:4], 8000)
-	tcp[12] = 5 << 4
-	return
-}
-
-func randomPackets(start uint32, _len int, length uint16) []gopacket.Packet {
-	var packets = make([]gopacket.Packet, _len)
-	for i := start; i < start+uint32(_len); i++ {
-		h := generateHeaders(i, length)
-		d := make([]byte, int(length)+len(h))
-		copy(d, h[0:])
-		packet := gopacket.NewPacket(d, layers.LinkTypeLoop, decodeOpts)
-		packets[i-start] = packet
-		inf := packets[i-start].Metadata()
-		_len := len(d)
-		inf.CaptureInfo = gopacket.CaptureInfo{CaptureLength: _len, Length: _len, Timestamp: time.Now()}
-	}
-	return packets
-}
-
 func TestPcapDump(t *testing.T) {
 	f, err := ioutil.TempFile("", "pcap_file")
 	if err != nil {
 		t.Error(err)
 	}
-	waiter := make(chan bool, 1)
-	h, _ := PcapDumpHandler(f, layers.LinkTypeLoop, func(level int, a ...interface{}) {
-		if level != 3 {
-			t.Errorf("expected debug level to be 3, got %d", level)
-		}
-		waiter <- true
-	})
-	packets := randomPackets(1, 5, 5)
+	h, _ := PcapDumpHandler(f, layers.LinkTypeLoop)
+	packets := Packets(1, 5, 5, 4)
 	for i := 0; i < len(packets); i++ {
 		if i == 1 {
-			tcp := packets[i].Data()[4:][20:]
 			// change dst port
-			binary.BigEndian.PutUint16(tcp[2:], 8001)
+			binary.BigEndian.PutUint16(packets[i].TransLayer[2:], 8001)
 		}
 		if i == 4 {
-			inf := packets[i].Metadata()
-			inf.CaptureLength = 40
+			packets[i].Info.CaptureLength = 40
 		}
 		h(packets[i])
 	}
-	<-waiter
 	name := f.Name()
 	f.Close()
 	testPcapDumpEngine(name, t)
@@ -139,9 +93,9 @@ func testPcapDumpEngine(f string, t *testing.T) {
 	pckts := 0
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
-	err = l.Listen(ctx, func(packet gopacket.Packet) {
-		if packet.Metadata().CaptureLength != 49 {
-			t.Errorf("expected packet length to be %d, got %d", 49, packet.Metadata().CaptureLength)
+	err = l.Listen(ctx, func(packet *Packet) {
+		if packet.Info.CaptureLength != 57 {
+			t.Errorf("expected packet length to be %d, got %d", 57, packet.Info.CaptureLength)
 		}
 		pckts++
 	})
@@ -207,8 +161,8 @@ func BenchmarkPcapDump(b *testing.B) {
 	}
 	now := time.Now()
 	defer os.Remove(f.Name())
-	h, _ := PcapDumpHandler(f, layers.LinkTypeLoop, nil)
-	packets := randomPackets(1, b.N, 5)
+	h, _ := PcapDumpHandler(f, layers.LinkTypeLoop)
+	packets := Packets(1, b.N, 5, 4)
 	for i := 0; i < len(packets); i++ {
 		h(packets[i])
 	}
@@ -223,8 +177,8 @@ func BenchmarkPcapFile(b *testing.B) {
 		return
 	}
 	defer os.Remove(f.Name())
-	h, _ := PcapDumpHandler(f, layers.LinkTypeLoop, nil)
-	packets := randomPackets(1, b.N, 5)
+	h, _ := PcapDumpHandler(f, layers.LinkTypeLoop)
+	packets := Packets(1, b.N, 5, 4)
 	for i := 0; i < len(packets); i++ {
 		h(packets[i])
 	}
@@ -246,9 +200,9 @@ func BenchmarkPcapFile(b *testing.B) {
 	pckts := 0
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
-	if err = l.Listen(ctx, func(packet gopacket.Packet) {
-		if packet.Metadata().CaptureLength != 49 {
-			b.Errorf("expected packet length to be %d, got %d", 49, packet.Metadata().CaptureLength)
+	if err = l.Listen(ctx, func(packet *Packet) {
+		if packet.Info.CaptureLength != 49 {
+			b.Errorf("expected packet length to be %d, got %d", 49, packet.Info.CaptureLength)
 		}
 		pckts++
 	}); err != nil {
@@ -257,41 +211,103 @@ func BenchmarkPcapFile(b *testing.B) {
 	b.Logf("%d/%d packets in %s", pckts, b.N, time.Since(now))
 }
 
-func BenchmarkPcap(b *testing.B) {
-	now := time.Now()
-	var err error
+// used to benchmark sock engine
+var buf [1024]byte
 
-	l, err := NewListener(LoopBack.Name, 8000, "", EnginePcap, true)
+func init() {
+	for i := 0; i < len(buf); i++ {
+		buf[i] = 0xff
+	}
+}
+
+func handler(n, counter *int32) Handler {
+	return func(p *Packet) {
+		nn := int32(len(p.Data))
+		atomic.AddInt32(n, nn)
+		atomic.AddInt32(counter, 1)
+	}
+}
+
+func BenchmarkPcap(b *testing.B) {
+	var err error
+	n := new(int32)
+	counter := new(int32)
+	l, err := NewListener(LoopBack.Name, 8000, "", EnginePcap, false)
 	if err != nil {
-		b.Errorf("expected error to be nil, got %v", err)
+		b.Error(err)
 		return
 	}
+	l.PcapOptions.BPFFilter = "udp dst port 8000 and host 127.0.0.1"
 	err = l.Activate()
 	if err != nil {
-		b.Errorf("expected error to be nil, got %v", err)
+		b.Error(err)
 		return
 	}
-	defer l.Handles[LoopBack.Name].(*pcap.Handle).Close()
-	for i := 0; i < b.N; i++ {
-		_, _ = net.Dial("tcp", "127.0.0.1:8000")
+	errCh := l.ListenBackground(context.Background(), handler(n, counter))
+	select {
+	case <-l.Reading:
+	case err = <-errCh:
+		b.Error(err)
+		return
 	}
-	sts, _ := l.Handles[LoopBack.Name].(*pcap.Handle).Stats()
-	b.Logf("%d packets in %s", sts.PacketsReceived, time.Since(now))
+	var conn net.Conn
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		b.StopTimer()
+		conn, err = net.Dial("udp", "127.0.0.1:8000")
+		if err != nil {
+			b.Error(err)
+			return
+		}
+		b.StartTimer()
+		_, err = conn.Write(buf[:])
+		if err != nil {
+			b.Error(err)
+			return
+		}
+	}
+	b.ReportMetric(float64(atomic.LoadInt32(n)), "buf")
+	b.ReportMetric(float64(atomic.LoadInt32(counter)), "packets")
 }
 
 func BenchmarkRawSocket(b *testing.B) {
-	now := time.Now()
 	var err error
-
-	l, err := NewListener(LoopBack.Name, 0, "", EngineRawSocket, true)
-	err = l.Activate()
+	n := new(int32)
+	counter := new(int32)
+	l, err := NewListener(LoopBack.Name, 8000, "", EngineRawSocket, false)
 	if err != nil {
+		b.Error(err)
 		return
 	}
-	defer l.Handles[LoopBack.Name].(*SockRaw).Close()
-	for i := 0; i < b.N; i++ {
-		_, _ = net.Dial("tcp", "127.0.0.1:8000")
+	l.PcapOptions.BPFFilter = "udp dst port 8000 and host 127.0.0.1"
+	err = l.Activate()
+	if err != nil {
+		b.Error(err)
+		return
 	}
-	sts, _ := l.Handles[LoopBack.Name].(*SockRaw).Stats()
-	b.Logf("%d packets in %s", sts.Packets, time.Since(now))
+	errCh := l.ListenBackground(context.Background(), handler(n, counter))
+	select {
+	case <-l.Reading:
+	case err = <-errCh:
+		b.Error(err)
+		return
+	}
+	var conn net.Conn
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		b.StopTimer()
+		conn, err = net.Dial("udp", "127.0.0.1:8000")
+		if err != nil {
+			b.Error(err)
+			return
+		}
+		b.StartTimer()
+		_, err = conn.Write(buf[:])
+		if err != nil {
+			b.Error(err)
+			return
+		}
+	}
+	b.ReportMetric(float64(atomic.LoadInt32(n)), "buf")
+	b.ReportMetric(float64(atomic.LoadInt32(counter)), "packets")
 }

capture/packet.go

@@ -0,0 +1,180 @@
+package capture
+
+import (
+	"github.com/google/gopacket"
+	"github.com/google/gopacket/layers"
+)
+
+// Packet properties of a pcaket
+type Packet struct {
+	Data         []byte
+	LinkLayer    []byte
+	NetLayer     []byte
+	NetOptsLen   int // length of extension headers(IPV6) or options(IPV4)
+	TransLayer   []byte
+	TransOptsLen int // length of tcp options
+	Payload      []byte
+
+	LinkType int
+	Info     *gopacket.CaptureInfo
+	Err      error
+}
+
+// NewPacket parses packet and stop at the first error encountered
+// pckt.Error will be nil if packet was parsed successfully
+func NewPacket(data []byte, lType, lTypeLen int, cp *gopacket.CaptureInfo) (p *Packet) {
+	p = new(Packet)
+	p.Info = cp
+	p.Data = data
+	if len(data) < lTypeLen {
+		p.Err = ErrHdrLength("Link")
+		return
+	}
+	p.LinkType = lType
+	p.LinkLayer = data[:lTypeLen]
+	if len(data) <= lTypeLen {
+		p.Err = ErrHdrMissing("IPv4 or IPv6")
+		return
+	}
+	data = data[lTypeLen:]
+	var proto byte
+	if data[0]>>4 == 4 {
+		// IPv4 header
+		if len(data) < 20 {
+			p.Err = ErrHdrLength("IPv4")
+			return
+		}
+		proto = data[9]
+		ihl := int(data[0]&0x0F) * 4
+		if ihl < 20 {
+			p.Err = ErrHdrInvalid("IPv4's IHL")
+			return
+		}
+		if len(data) < ihl {
+			p.Err = ErrHdrLength("IPv4 opts")
+			p.NetLayer = data[:20]
+			return
+		}
+		p.NetOptsLen = ihl - 20
+		p.NetLayer = data[:ihl]
+	} else if data[0]>>4 == 6 {
+		if len(data) < 40 {
+			p.Err = ErrHdrLength("IPv6")
+			return
+		}
+		proto = data[6]
+		totalLen := 40
+		for ipv6ExtensionHdr(proto) {
+			hdr := len(data) - totalLen
+			if hdr < 8 {
+				p.Err = ErrHdrExpected("IPv6 opts")
+				break
+			}
+			extLen := 8
+			if proto != 44 {
+				extLen = int(data[totalLen+1]+1) * 8
+			}
+			if hdr < extLen {
+				p.Err = ErrHdrLength("IPv6 opts")
+				break
+			}
+			p.NetOptsLen += extLen
+			proto = data[totalLen]
+			totalLen += extLen
+		}
+		p.NetLayer = data[:totalLen]
+		if p.Err != nil {
+			return
+		}
+	} else {
+		p.Err = ErrHdrExpected("IPv4 or IPv6")
+		return
+	}
+	if proto != 6 {
+		p.Err = ErrHdrExpected("TCP")
+		return
+	}
+	if len(data) <= len(p.NetLayer) {
+		p.Err = ErrHdrMissing("TCP")
+		return
+	}
+	data = data[len(p.NetLayer):]
+	// TCP header
+	if len(data) < 20 {
+		p.Err = ErrHdrLength("TCP")
+		return
+	}
+	dOf := int(data[12]>>4) * 4
+	if dOf < 20 {
+		p.Err = ErrHdrInvalid("TCP's data offset")
+		return
+	}
+	if len(data) < dOf {
+		p.Err = ErrHdrLength("TCP opts")
+		p.TransLayer = data[:20]
+		return
+	}
+	p.TransLayer = data[:dOf]
+	p.TransOptsLen = dOf - 20
+	if len(data) > dOf {
+		p.Payload = data[dOf:]
+	}
+	return
+}
+
+// ErrHdrLength returned on short header length
+type ErrHdrLength string
+
+func (err ErrHdrLength) Error() string {
+	return "short " + string(err) + " length"
+}
+
+// ErrHdrMissing returned on missing header(s)
+type ErrHdrMissing string
+
+func (err ErrHdrMissing) Error() string {
+	return "missing " + string(err) + " header(s)"
+}
+
+// ErrHdrExpected returned when header(s) are different from the one expected
+type ErrHdrExpected string
+
+func (err ErrHdrExpected) Error() string {
+	return "expected " + string(err) + " header(s)"
+}
+
+// ErrHdrInvalid returned when header(s) are different from the one expected
+type ErrHdrInvalid string
+
+func (err ErrHdrInvalid) Error() string {
+	return "invalid " + string(err) + " value"
+}
+
+func pcapLinkTypeLength(lType int) (int, bool) {
+	switch layers.LinkType(lType) {
+	case layers.LinkTypeEthernet:
+		return 14, true
+	case layers.LinkTypeNull, layers.LinkTypeLoop:
+		return 4, true
+	case layers.LinkTypeRaw, 12, 14:
+		return 0, true
+	case layers.LinkTypeIPv4, layers.LinkTypeIPv6:
+		// (TODO:) look out for IP encapsulation?
+		return 0, true
+	case layers.LinkTypeLinuxSLL:
+		return 16, true
+	case layers.LinkTypeFDDI:
+		return 13, true
+	case 226 /*DLT_IPNET*/ :
+		// https://www.tcpdump.org/linktypes/LINKTYPE_IPNET.html
+		return 24, true
+	default:
+		return 0, false
+	}
+}
+
+// https://en.wikipedia.org/wiki/IPv6_packet#Extension_headers
+func ipv6ExtensionHdr(b byte) bool {
+	// TODO: support all extension headers
+	return b == 0 || b == 43 || b == 44
+}

capture/packet_test.go

@@ -0,0 +1,197 @@
+package capture
+
+import (
+	"encoding/binary"
+	"errors"
+	"testing"
+	"time"
+
+	"github.com/google/gopacket"
+	"github.com/google/gopacket/layers"
+)
+
+func generateHeader4(seq uint32, length uint16) []byte {
+	hdr := make([]byte, 4+24+24, 4+24+24)
+	binary.BigEndian.PutUint32(hdr, uint32(layers.ProtocolFamilyIPv4))
+
+	ip := hdr[4:]
+	ip[0] = 4<<4 | 6
+	binary.BigEndian.PutUint16(ip[2:4], length+24+24)
+	ip[9] = uint8(layers.IPProtocolTCP)
+	copy(ip[12:16], []byte{127, 0, 0, 1})
+	copy(ip[16:], []byte{127, 0, 0, 1})
+
+	// set tcp header
+	tcp := ip[24:]
+	tcp[12] = 6 << 4
+	binary.BigEndian.PutUint16(tcp, 5535)
+	binary.BigEndian.PutUint16(tcp[2:], 8000)
+	binary.BigEndian.PutUint32(tcp[4:], seq)
+	return hdr
+}
+
+func generateHeader6(seq uint32, length uint16) []byte {
+	hdr := make([]byte, 4+40+32+24, 4+40+32+24)
+	binary.BigEndian.PutUint32(hdr, uint32(layers.ProtocolFamilyIPv6Linux))
+
+	ip := hdr[4:]
+	ip[0] = 6 << 4
+	binary.BigEndian.PutUint16(ip[4:], length+32+24)
+	var ipAddr [16]byte
+	ipAddr[15] = 0x01
+	copy(ip[8:], ip[:])
+	copy(ip[24:], ip[:])
+	copy(ip[40:], []byte{
+		// net-layer (IPv6-Opts)
+		0x2b, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+		0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+		uint8(layers.IPProtocolTCP), 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+		0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	})
+
+	// set tcp header
+	tcp := ip[40+32:]
+	tcp[12] = 6 << 4
+	binary.BigEndian.PutUint16(tcp, 5535)
+	binary.BigEndian.PutUint16(tcp[2:], 8000)
+	binary.BigEndian.PutUint32(tcp[4:], seq)
+	return hdr
+}
+
+func Packets(start uint32, _len int, length uint16, version byte) []*Packet {
+	var packets = make([]*Packet, _len)
+	for i := start; i < start+uint32(_len); i++ {
+		var h []byte
+		if version == 4 {
+			h = generateHeader4(i, length)
+		} else {
+			h = generateHeader6(i, length)
+		}
+		d := append(h, make([]byte, int(length))...)
+		ci := &gopacket.CaptureInfo{Length: len(d), CaptureLength: len(d), Timestamp: time.Now()}
+		packets[i-start] = NewPacket(d, int(layers.LinkTypeLoop), 4, ci)
+	}
+	return packets
+}
+
+func TestIPv4Packet(t *testing.T) {
+	pckt := packet(append(generateHeader4(1024, 10), make([]byte, 10)...))
+	if pckt.Err != nil {
+		t.Error(pckt)
+		return
+	}
+	if err := packet(pckt.Data[:2]).Err; !errors.Is(err, ErrHdrLength("Link")) {
+		t.Errorf("should fail with %q, got %q", ErrHdrLength("Link"), err)
+		return
+	}
+	if err := packet(pckt.Data[:20]).Err; !errors.Is(err, ErrHdrLength("IPv4")) {
+		t.Errorf("should fail with %q, got %q", ErrHdrLength("IPv4"), err)
+		return
+	}
+	if err := packet(pckt.Data[:27]).Err; !errors.Is(err, ErrHdrLength("IPv4 opts")) {
+		t.Errorf("should fail with %q, got %q", ErrHdrLength("IPv4 opts"), err)
+		return
+	}
+	if err := packet(pckt.Data[:40]).Err; !errors.Is(err, ErrHdrLength("TCP")) {
+		t.Errorf("should fail with %q, got %q", ErrHdrLength("TCP opts"), err)
+		return
+	}
+	if err := packet(pckt.Data[:50]).Err; !errors.Is(err, ErrHdrLength("TCP opts")) {
+		t.Errorf("should fail with %q, got %q", ErrHdrLength("TCP opts"), err)
+		return
+	}
+	pckt.TransLayer[12] = 0x10
+	if err := packet(pckt.Data[:50]).Err; !errors.Is(err, ErrHdrInvalid("TCP's data offset")) {
+		t.Errorf("should fail with %q, got %q", ErrHdrInvalid("TCP's data offset"), err)
+		return
+	}
+	pckt.TransLayer[12] = 0x60
+	if err := packet(pckt.Data[:28]).Err; !errors.Is(err, ErrHdrMissing("TCP")) {
+		t.Errorf("should fail with %q, got %q", ErrHdrMissing("TCP"), err)
+		return
+	}
+	pckt.NetLayer[9] = 0x02
+	if err := packet(pckt.Data).Err; !errors.Is(err, ErrHdrExpected("TCP")) {
+		t.Errorf("should fail with %q, got %q", ErrHdrExpected("TCP"), err)
+		return
+	}
+	pckt.NetLayer[9] = 0x06
+	pckt.NetLayer[0] = 0x44
+	if err := packet(pckt.Data).Err; !errors.Is(err, ErrHdrInvalid("IPv4's IHL")) {
+		t.Errorf("should fail with %q, got %q", ErrHdrInvalid("IPv4's IHL"), err)
+		return
+	}
+	pckt.NetLayer[0] = 0x56
+	if err := packet(pckt.Data).Err; !errors.Is(err, ErrHdrExpected("IPv4 or IPv6")) {
+		t.Errorf("should fail with %q, got %q", ErrHdrExpected("IPv4 or IPv6"), err)
+		return
+	}
+}
+
+func TestIPv6Packet(t *testing.T) {
+	pckt := packet(append(generateHeader6(1024, 10), make([]byte, 10)...))
+	if pckt.Err != nil {
+		t.Error(pckt)
+		return
+	}
+	if err := packet(pckt.Data[:4]).Err; !errors.Is(err, ErrHdrMissing("IPv4 or IPv6")) {
+		t.Errorf("should fail with %q, got %q", ErrHdrMissing("IPv4 or IPv6"), err)
+		return
+	}
+	if err := packet(pckt.Data[:40]).Err; !errors.Is(err, ErrHdrLength("IPv6")) {
+		t.Errorf("should fail with %q, got %q", ErrHdrLength("IPv6"), err)
+		return
+	}
+	if err := packet(pckt.Data[:52]).Err; !errors.Is(err, ErrHdrLength("IPv6 opts")) {
+		t.Errorf("should fail with %q, got %q", ErrHdrLength("IPv6 opts"), err)
+		return
+	}
+	if err := packet(pckt.Data[:80]).Err; !errors.Is(err, ErrHdrLength("TCP")) {
+		t.Errorf("should fail with %q, got %q", ErrHdrLength("TCP opts"), err)
+		return
+	}
+	if err := packet(pckt.Data[:98]).Err; !errors.Is(err, ErrHdrLength("TCP opts")) {
+		t.Errorf("should fail with %q, got %q", ErrHdrLength("TCP opts"), err)
+		return
+	}
+	pckt.TransLayer[12] = 0x10
+	if err := packet(pckt.Data).Err; !errors.Is(err, ErrHdrInvalid("TCP's data offset")) {
+		t.Errorf("should fail with %q, got %q", ErrHdrInvalid("TCP's data offset"), err)
+		return
+	}
+	pckt.TransLayer[12] = 0x60
+	if err := packet(pckt.Data[:76]).Err; !errors.Is(err, ErrHdrMissing("TCP")) {
+		t.Errorf("should fail with %q, got %q", ErrHdrMissing("TCP"), err)
+		return
+	}
+	pckt.NetLayer[56] = 0x02
+	if err := packet(pckt.Data).Err; !errors.Is(err, ErrHdrExpected("TCP")) {
+		t.Errorf("should fail with %q, got %q", ErrHdrExpected("TCP"), err)
+		return
+	}
+	pckt.NetLayer[56] = 0x06
+}
+
+func packet(data []byte) *Packet {
+	return NewPacket(data, int(layers.LinkTypeLoop), 4, &gopacket.CaptureInfo{})
+}
+
+func BenchmarkNewPacketIPv4(b *testing.B) {
+	data := append(generateHeader4(1204, 10), make([]byte, 10)...)
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		if err := packet(data).Err; err != nil {
+			b.Error(err)
+		}
+	}
+}
+
+func BenchmarkNewPacketIPv6(b *testing.B) {
+	data := append(generateHeader6(1024, 10), make([]byte, 10)...)
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		if err := packet(data).Err; err != nil {
+			b.Error(err)
+		}
+	}
+}

elasticsearch.go

@@ -124,6 +124,7 @@ func (p *ESPlugin) RttDurationToMs(d time.Duration) int64 {
 	return int64(fl)
 }
 
+// ResponseAnalyze send req and resp to ES
 func (p *ESPlugin) ResponseAnalyze(req, resp []byte, start, stop time.Time) {
 	if len(resp) == 0 {
 		// nil http response - skipped elasticsearch export for this request
@@ -131,7 +132,6 @@ func (p *ESPlugin) ResponseAnalyze(req, resp []byte, start, stop time.Time) {
 	}
 	t := time.Now()
 	rtt := p.RttDurationToMs(stop.Sub(start))
-	req = payloadBody(req)
 
 	esResp := ESRequestResponse{
 		ReqURL:               string(proto.Path(req)),

emitter.go

@@ -1,31 +1,29 @@
 package main
 
 import (
-	"bytes"
+	"fmt"
 	"hash/fnv"
 	"io"
 	"log"
 	"sync"
 	"time"
+
+	"github.com/buger/goreplay/byteutils"
 )
 
-type emitter struct {
-	sync.Mutex
+// Emitter represents an abject to manage plugins communication
+type Emitter struct {
 	sync.WaitGroup
-	quit    chan int
 	plugins *InOutPlugins
 }
 
-// NewEmitter creates and initializes new `emitter` object.
-func NewEmitter(quit chan int) *emitter {
-	return &emitter{
-		quit: quit,
-	}
+// NewEmitter creates and initializes new Emitter object.
+func NewEmitter() *Emitter {
+	return &Emitter{}
 }
 
 // Start initialize loop for sending data from inputs to outputs
-func (e *emitter) Start(plugins *InOutPlugins, middlewareCmd string) {
-	defer e.Wait()
+func (e *Emitter) Start(plugins *InOutPlugins, middlewareCmd string) {
 	if Settings.CopyBufferSize < 1 {
 		Settings.CopyBufferSize = 5 << 20
 	}
@@ -38,147 +36,96 @@ func (e *emitter) Start(plugins *InOutPlugins, middlewareCmd string) {
 			middleware.ReadFrom(in)
 		}
 
-		// We are going only to read responses, so using same ReadFrom method
-		for _, out := range plugins.Outputs {
-			if r, ok := out.(io.Reader); ok {
-				middleware.ReadFrom(r)
-			}
-		}
+		e.plugins.Inputs = append(e.plugins.Inputs, middleware)
+		e.plugins.All = append(e.plugins.All, middleware)
 		e.Add(1)
 		go func() {
 			defer e.Done()
-			if err := CopyMulty(e.quit, middleware, plugins.Outputs...); err != nil {
-				Debug(2, "Error during copy: ", err)
-				e.Close()
-			}
-		}()
-		go func() {
-			for {
-				select {
-				case <-e.quit:
-					middleware.Close()
-					return
-				}
+			if err := CopyMulty(middleware, plugins.Outputs...); err != nil {
+				Debug(2, fmt.Sprintf("[EMITTER] error during copy: %q", err))
 			}
 		}()
 	} else {
 		for _, in := range plugins.Inputs {
 			e.Add(1)
-			go func(in io.Reader) {
+			go func(in PluginReader) {
 				defer e.Done()
-				if err := CopyMulty(e.quit, in, plugins.Outputs...); err != nil {
-					Debug(2, "Error during copy: ", err)
-					e.Close()
+				if err := CopyMulty(in, plugins.Outputs...); err != nil {
+					Debug(2, fmt.Sprintf("[EMITTER] error during copy: %q", err))
 				}
 			}(in)
 		}
-
-		for _, out := range plugins.Outputs {
-			if r, ok := out.(io.Reader); ok {
-				e.Add(1)
-				go func(r io.Reader) {
-					defer e.Done()
-					if err := CopyMulty(e.quit, r, plugins.Outputs...); err != nil {
-						Debug(2, "Error during copy: ", err)
-						e.Close()
-					}
-				}(r)
-			}
-		}
-	}
-}
-
-func (e *emitter) close() {
-	select {
-	case <-e.quit:
-	default:
-		close(e.quit)
 	}
 }
 
 // Close closes all the goroutine and waits for it to finish.
-func (e *emitter) Close() {
-	e.close()
-	for _, p := range e.plugins.Inputs {
+func (e *Emitter) Close() {
+	for _, p := range e.plugins.All {
 		if cp, ok := p.(io.Closer); ok {
 			cp.Close()
 		}
 	}
-	for _, p := range e.plugins.Outputs {
-		if cp, ok := p.(io.Closer); ok {
-			cp.Close()
-		}
+	if len(e.plugins.All) > 0 {
+		// wait for everything to stop
+		e.Wait()
 	}
-	e.plugins = nil // avoid further accidental usage
+	e.plugins.All = nil // avoid Close to make changes again
 }
 
 // CopyMulty copies from 1 reader to multiple writers
-func CopyMulty(stop chan int, src io.Reader, writers ...io.Writer) error {
-	buf := make([]byte, Settings.CopyBufferSize)
+func CopyMulty(src PluginReader, writers ...PluginWriter) error {
 	wIndex := 0
 	modifier := NewHTTPModifier(&Settings.ModifierConfig)
-	filteredRequests := make(map[string]time.Time)
-	filteredRequestsLastCleanTime := time.Now()
+	filteredRequests := make(map[string]int64)
+	filteredRequestsLastCleanTime := time.Now().UnixNano()
+	filteredCount := 0
 
-	i := 0
 	for {
-		var nr int
-		nr, err := src.Read(buf)
-
-		select {
-		case <-stop:
-			return nil
-		default:
-		}
+		msg, err := src.PluginRead()
 		if err != nil {
+			if err == ErrorStopped || err == io.EOF {
+				return nil
+			}
 			return err
 		}
-
-		_maxN := nr
-		if nr > 500 {
-			_maxN = 500
-		}
-		if nr > 0 {
-			payload := buf[:nr]
-			meta := payloadMeta(payload)
+		if msg != nil && len(msg.Data) > 0 {
+			if len(msg.Data) > int(Settings.CopyBufferSize) {
+				msg.Data = msg.Data[:Settings.CopyBufferSize]
+			}
+			meta := payloadMeta(msg.Meta)
 			if len(meta) < 3 {
-				Debug(2, "[EMITTER] Found malformed record", string(payload[0:_maxN]), nr, "from:", src)
+				Debug(2, fmt.Sprintf("[EMITTER] Found malformed record %q from %q", msg.Meta, src))
 				continue
 			}
-			requestID := string(meta[1])
-
-			Debug(3, "[EMITTER] input:", string(payload[0:_maxN]), nr, "from:", src)
-
+			requestID := byteutils.SliceToString(meta[1])
+			// start a subroutine only when necessary
+			if Settings.Verbose >= 3 {
+				Debug(3, "[EMITTER] input: ", byteutils.SliceToString(msg.Meta[:len(msg.Meta)-1]), " from: ", src)
+			}
 			if modifier != nil {
-				if isRequestPayload(payload) {
-					headSize := bytes.IndexByte(payload, '\n') + 1
-					body := payload[headSize:]
-					originalBodyLen := len(body)
-					body = modifier.Rewrite(body)
-
+				Debug(3, "[EMITTER] modifier:", requestID, "from:", src)
+				if isRequestPayload(msg.Meta) {
+					msg.Data = modifier.Rewrite(msg.Data)
 					// If modifier tells to skip request
-					if len(body) == 0 {
-						filteredRequests[requestID] = time.Now()
+					if len(msg.Data) == 0 {
+						filteredRequests[requestID] = time.Now().UnixNano()
+						filteredCount++
 						continue
 					}
-
-					if originalBodyLen != len(body) {
-						payload = append(payload[:headSize], body...)
-					}
-
-					Debug(3, "[EMITTER] Rewritten input:", len(payload), "First %d bytes:", _maxN, string(payload[0:_maxN]))
+					Debug(3, "[EMITTER] Rewritten input:", requestID, "from:", src)
 
 				} else {
 					if _, ok := filteredRequests[requestID]; ok {
 						delete(filteredRequests, requestID)
+						filteredCount--
 						continue
 					}
 				}
 			}
 
 			if Settings.PrettifyHTTP {
-				payload = prettifyHTTP(payload)
-				if len(payload) == 0 {
+				msg.Data = prettifyHTTP(msg.Data)
+				if len(msg.Data) == 0 {
 					continue
 				}
 			}
@@ -189,27 +136,23 @@ func CopyMulty(stop chan int, src io.Reader, writers ...io.Writer) error {
 						log.Fatal("Detailed TCP sessions work only with PRO license")
 					}
 					hasher := fnv.New32a()
-					// First 20 bytes contain tcp session
-					id := payloadID(payload)
-					hasher.Write(id)
+					hasher.Write(meta[1])
 
 					wIndex = int(hasher.Sum32()) % len(writers)
-					writers[wIndex].Write(payload)
+					if _, err := writers[wIndex].PluginWrite(msg); err != nil {
+						return err
+					}
 				} else {
 					// Simple round robin
-					if _, err := writers[wIndex].Write(payload); err != nil {
+					if _, err := writers[wIndex].PluginWrite(msg); err != nil {
 						return err
 					}
 
-					wIndex++
-
-					if wIndex >= len(writers) {
-						wIndex = 0
-					}
+					wIndex = (wIndex + 1) % len(writers)
 				}
 			} else {
 				for _, dst := range writers {
-					if _, err := dst.Write(payload); err != nil {
+					if _, err := dst.PluginWrite(msg); err != nil {
 						return err
 					}
 				}
@@ -217,19 +160,18 @@ func CopyMulty(stop chan int, src io.Reader, writers ...io.Writer) error {
 		}
 
 		// Run GC on each 1000 request
-		if i%1000 == 0 {
+		if filteredCount > 0 && filteredCount%1000 == 0 {
 			// Clean up filtered requests for which we didn't get a response to filter
-			now := time.Now()
-			if now.Sub(filteredRequestsLastCleanTime) > 60*time.Second {
+			now := time.Now().UnixNano()
+			if now-filteredRequestsLastCleanTime > int64(60*time.Second) {
 				for k, v := range filteredRequests {
-					if now.Sub(v) > 60*time.Second {
+					if now-v > int64(60*time.Second) {
 						delete(filteredRequests, k)
+						filteredCount--
 					}
 				}
-				filteredRequestsLastCleanTime = time.Now()
+				filteredRequestsLastCleanTime = time.Now().UnixNano()
 			}
 		}
-
-		i++
 	}
 }

emitter_test.go

@@ -2,7 +2,6 @@ package main
 
 import (
 	"fmt"
-	"io"
 	"os"
 	"sync"
 	"sync/atomic"
@@ -18,20 +17,19 @@ func TestMain(m *testing.M) {
 
 func TestEmitter(t *testing.T) {
 	wg := new(sync.WaitGroup)
-	quit := make(chan int)
 
 	input := NewTestInput()
-	output := NewTestOutput(func(data []byte) {
+	output := NewTestOutput(func(*Message) {
 		wg.Done()
 	})
 
 	plugins := &InOutPlugins{
-		Inputs:  []io.Reader{input},
-		Outputs: []io.Writer{output},
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{output},
 	}
 	plugins.All = append(plugins.All, input, output)
 
-	emitter := NewEmitter(quit)
+	emitter := NewEmitter()
 	go emitter.Start(plugins, Settings.Middleware)
 
 	for i := 0; i < 1000; i++ {
@@ -45,32 +43,31 @@ func TestEmitter(t *testing.T) {
 
 func TestEmitterFiltered(t *testing.T) {
 	wg := new(sync.WaitGroup)
-	quit := make(chan int)
 
 	input := NewTestInput()
 	input.skipHeader = true
 
-	output := NewTestOutput(func(data []byte) {
+	output := NewTestOutput(func(*Message) {
 		wg.Done()
 	})
 
 	plugins := &InOutPlugins{
-		Inputs:  []io.Reader{input},
-		Outputs: []io.Writer{output},
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{output},
 	}
 	plugins.All = append(plugins.All, input, output)
 
 	methods := HTTPMethods{[]byte("GET")}
 	Settings.ModifierConfig = HTTPModifierConfig{Methods: methods}
 
-	emitter := &emitter{quit: quit}
+	emitter := &Emitter{}
 	go emitter.Start(plugins, "")
 
 	wg.Add(2)
 
 	id := uuid()
 	reqh := payloadHeader(RequestPayload, id, time.Now().UnixNano(), -1)
-	reqb := append(reqh, []byte("GET / HTTP/1.1\r\nHost: www.w3.org\r\nUser-Agent: Go 1.1 package http\r\nAccept-Encoding: gzip\r\n\r\n")...)
+	reqb := append(reqh, []byte("POST / HTTP/1.1\r\nHost: www.w3.org\r\nUser-Agent: Go 1.1 package http\r\nAccept-Encoding: gzip\r\n\r\n")...)
 
 	resh := payloadHeader(ResponsePayload, id, time.Now().UnixNano()+1, 1)
 	respb := append(resh, []byte("HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n")...)
@@ -80,7 +77,7 @@ func TestEmitterFiltered(t *testing.T) {
 
 	id = uuid()
 	reqh = payloadHeader(RequestPayload, id, time.Now().UnixNano(), -1)
-	reqb = append(reqh, []byte("POST / HTTP/1.1\r\nHost: www.w3.org\r\nUser-Agent: Go 1.1 package http\r\nAccept-Encoding: gzip\r\n\r\n")...)
+	reqb = append(reqh, []byte("GET / HTTP/1.1\r\nHost: www.w3.org\r\nUser-Agent: Go 1.1 package http\r\nAccept-Encoding: gzip\r\n\r\n")...)
 
 	resh = payloadHeader(ResponsePayload, id, time.Now().UnixNano()+1, 1)
 	respb = append(resh, []byte("HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n")...)
@@ -96,30 +93,29 @@ func TestEmitterFiltered(t *testing.T) {
 
 func TestEmitterSplitRoundRobin(t *testing.T) {
 	wg := new(sync.WaitGroup)
-	quit := make(chan int)
 
 	input := NewTestInput()
 
 	var counter1, counter2 int32
 
-	output1 := NewTestOutput(func(data []byte) {
+	output1 := NewTestOutput(func(*Message) {
 		atomic.AddInt32(&counter1, 1)
 		wg.Done()
 	})
 
-	output2 := NewTestOutput(func(data []byte) {
+	output2 := NewTestOutput(func(*Message) {
 		atomic.AddInt32(&counter2, 1)
 		wg.Done()
 	})
 
 	plugins := &InOutPlugins{
-		Inputs:  []io.Reader{input},
-		Outputs: []io.Writer{output1, output2},
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{output1, output2},
 	}
 
 	Settings.SplitOutput = true
 
-	emitter := NewEmitter(quit)
+	emitter := NewEmitter()
 	go emitter.Start(plugins, Settings.Middleware)
 
 	for i := 0; i < 1000; i++ {
@@ -140,31 +136,30 @@ func TestEmitterSplitRoundRobin(t *testing.T) {
 
 func TestEmitterRoundRobin(t *testing.T) {
 	wg := new(sync.WaitGroup)
-	quit := make(chan int)
 
 	input := NewTestInput()
 
 	var counter1, counter2 int32
 
-	output1 := NewTestOutput(func(data []byte) {
-		atomic.AddInt32(&counter1, 1)
+	output1 := NewTestOutput(func(*Message) {
+		counter1++
 		wg.Done()
 	})
 
-	output2 := NewTestOutput(func(data []byte) {
-		atomic.AddInt32(&counter2, 1)
+	output2 := NewTestOutput(func(*Message) {
+		counter2++
 		wg.Done()
 	})
 
 	plugins := &InOutPlugins{
-		Inputs:  []io.Reader{input},
-		Outputs: []io.Writer{output1, output2},
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{output1, output2},
 	}
 	plugins.All = append(plugins.All, input, output1, output2)
 
 	Settings.SplitOutput = true
 
-	emitter := NewEmitter(quit)
+	emitter := NewEmitter()
 	go emitter.Start(plugins, Settings.Middleware)
 
 	for i := 0; i < 1000; i++ {
@@ -186,36 +181,34 @@ func TestEmitterSplitSession(t *testing.T) {
 	wg := new(sync.WaitGroup)
 	wg.Add(200)
 
-	quit := make(chan int)
-
 	input := NewTestInput()
 	input.skipHeader = true
 
 	var counter1, counter2 int32
 
-	output1 := NewTestOutput(func(data []byte) {
-		if payloadID(data)[0] == 'a' {
-			atomic.AddInt32(&counter1, 1)
+	output1 := NewTestOutput(func(msg *Message) {
+		if payloadID(msg.Meta)[0] == 'a' {
+			counter1++
 		}
 		wg.Done()
 	})
 
-	output2 := NewTestOutput(func(data []byte) {
-		if payloadID(data)[0] == 'b' {
-			atomic.AddInt32(&counter2, 1)
+	output2 := NewTestOutput(func(msg *Message) {
+		if payloadID(msg.Meta)[0] == 'b' {
+			counter2++
 		}
 		wg.Done()
 	})
 
 	plugins := &InOutPlugins{
-		Inputs:  []io.Reader{input},
-		Outputs: []io.Writer{output1, output2},
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{output1, output2},
 	}
 
 	Settings.SplitOutput = true
 	Settings.RecognizeTCPSessions = true
 
-	emitter := NewEmitter(quit)
+	emitter := NewEmitter()
 	go emitter.Start(plugins, Settings.Middleware)
 
 	for i := 0; i < 200; i++ {
@@ -242,21 +235,20 @@ func TestEmitterSplitSession(t *testing.T) {
 
 func BenchmarkEmitter(b *testing.B) {
 	wg := new(sync.WaitGroup)
-	quit := make(chan int)
 
 	input := NewTestInput()
 
-	output := NewTestOutput(func(data []byte) {
+	output := NewTestOutput(func(*Message) {
 		wg.Done()
 	})
 
 	plugins := &InOutPlugins{
-		Inputs:  []io.Reader{input},
-		Outputs: []io.Writer{output},
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{output},
 	}
 	plugins.All = append(plugins.All, input, output)
 
-	emitter := NewEmitter(quit)
+	emitter := NewEmitter()
 	go emitter.Start(plugins, Settings.Middleware)
 
 	b.ResetTimer()

gor.go

@@ -4,7 +4,6 @@ package main
 
 import (
 	"flag"
-	"fmt"
 	"log"
 	"net/http"
 	"net/http/httputil"
@@ -74,13 +73,13 @@ func main() {
 	}
 
 	closeCh := make(chan int)
-	emitter := NewEmitter(closeCh)
+	emitter := NewEmitter()
 	go emitter.Start(plugins, Settings.Middleware)
 	if Settings.ExitAfter > 0 {
 		log.Printf("Running gor for a duration of %s\n", Settings.ExitAfter)
 
 		time.AfterFunc(Settings.ExitAfter, func() {
-			fmt.Printf("gor run timeout %s\n", Settings.ExitAfter)
+			log.Printf("gor run timeout %s\n", Settings.ExitAfter)
 			close(closeCh)
 		})
 	}

http_modifier_settings.go

@@ -19,10 +19,9 @@ type HTTPModifierConfig struct {
 	HeaderBasicAuthFilters HTTPHeaderBasicAuthFilters `json:"http-basic-auth-filter"`
 	HeaderHashFilters      HTTPHashFilters            `json:"http-header-limiter"`
 	ParamHashFilters       HTTPHashFilters            `json:"http-param-limiter"`
-
-	Params  HTTPParams  `json:"http-set-param"`
-	Headers HTTPHeaders `json:"http-set-header"`
-	Methods HTTPMethods `json:"http-allow-method"`
+	Params                 HTTPParams                 `json:"http-set-param"`
+	Headers                HTTPHeaders                `json:"http-set-header"`
+	Methods                HTTPMethods                `json:"http-allow-method"`
 }
 
 //
@@ -40,10 +39,11 @@ func (h *HTTPHeaderFilters) String() string {
 	return fmt.Sprint(*h)
 }
 
+// Set method to implement flags.Value
 func (h *HTTPHeaderFilters) Set(value string) error {
 	valArr := strings.SplitN(value, ":", 2)
 	if len(valArr) < 2 {
-		return errors.New("need both header and value, colon-delimited (ex. user_id:^169$).")
+		return errors.New("need both header and value, colon-delimited (ex. user_id:^169$)")
 	}
 	val := strings.TrimSpace(valArr[1])
 	r, err := regexp.Compile(val)
@@ -63,13 +63,14 @@ type basicAuthFilter struct {
 	regexp *regexp.Regexp
 }
 
-// HTTPHeaderFilters holds list of headers and their regexps
+// HTTPHeaderBasicAuthFilters holds list of regxp to match basic Auth header values
 type HTTPHeaderBasicAuthFilters []basicAuthFilter
 
 func (h *HTTPHeaderBasicAuthFilters) String() string {
 	return fmt.Sprint(*h)
 }
 
+// Set method to implement flags.Value
 func (h *HTTPHeaderBasicAuthFilters) Set(value string) error {
 	r, err := regexp.Compile(value)
 	if err != nil {
@@ -89,12 +90,14 @@ type hashFilter struct {
 	percent uint32
 }
 
+// HTTPHashFilters represents a slice of header hash filters
 type HTTPHashFilters []hashFilter
 
 func (h *HTTPHashFilters) String() string {
 	return fmt.Sprint(*h)
 }
 
+// Set method to implement flags.Value
 func (h *HTTPHashFilters) Set(value string) error {
 	valArr := strings.SplitN(value, ":", 2)
 	if len(valArr) < 2 {
@@ -129,23 +132,26 @@ func (h *HTTPHashFilters) Set(value string) error {
 //
 // Handling of --http-set-header option
 //
-type HTTPHeaders []HTTPHeader
-type HTTPHeader struct {
+type httpHeader struct {
 	Name  string
 	Value string
 }
 
+// HTTPHeaders is a slice of headers that must appended
+type HTTPHeaders []httpHeader
+
 func (h *HTTPHeaders) String() string {
 	return fmt.Sprint(*h)
 }
 
+// Set method to implement flags.Value
 func (h *HTTPHeaders) Set(value string) error {
 	v := strings.SplitN(value, ":", 2)
 	if len(v) != 2 {
 		return errors.New("Expected `Key: Value`")
 	}
 
-	header := HTTPHeader{
+	header := httpHeader{
 		strings.TrimSpace(v[0]),
 		strings.TrimSpace(v[1]),
 	}
@@ -157,23 +163,26 @@ func (h *HTTPHeaders) Set(value string) error {
 //
 // Handling of --http-set-param option
 //
-type HTTPParams []HTTPParam
-type HTTPParam struct {
+type httpParam struct {
 	Name  []byte
 	Value []byte
 }
 
+// HTTPParams filters for --http-set-param
+type HTTPParams []httpParam
+
 func (h *HTTPParams) String() string {
 	return fmt.Sprint(*h)
 }
 
+// Set method to implement flags.Value
 func (h *HTTPParams) Set(value string) error {
 	v := strings.SplitN(value, "=", 2)
 	if len(v) != 2 {
 		return errors.New("Expected `Key=Value`")
 	}
 
-	param := HTTPParam{
+	param := httpParam{
 		[]byte(strings.TrimSpace(v[0])),
 		[]byte(strings.TrimSpace(v[1])),
 	}
@@ -185,12 +194,15 @@ func (h *HTTPParams) Set(value string) error {
 //
 // Handling of --http-allow-method option
 //
+
+// HTTPMethods holds values for method allowed
 type HTTPMethods [][]byte
 
 func (h *HTTPMethods) String() string {
 	return fmt.Sprint(*h)
 }
 
+// Set method to implement flags.Value
 func (h *HTTPMethods) Set(value string) error {
 	*h = append(*h, []byte(value))
 	return nil
@@ -204,12 +216,14 @@ type urlRewrite struct {
 	target []byte
 }
 
+// URLRewriteMap holds regexp and data to modify URL
 type URLRewriteMap []urlRewrite
 
 func (r *URLRewriteMap) String() string {
 	return fmt.Sprint(*r)
 }
 
+// Set method to implement flags.Value
 func (r *URLRewriteMap) Set(value string) error {
 	valArr := strings.SplitN(value, ":", 2)
 	if len(valArr) < 2 {
@@ -232,12 +246,14 @@ type headerRewrite struct {
 	target []byte
 }
 
+// HeaderRewriteMap holds regexp and data to rewrite headers
 type HeaderRewriteMap []headerRewrite
 
 func (r *HeaderRewriteMap) String() string {
 	return fmt.Sprint(*r)
 }
 
+// Set method to implement flags.Value
 func (r *HeaderRewriteMap) Set(value string) error {
 	headerArr := strings.SplitN(value, ":", 2)
 	if len(headerArr) < 2 {
@@ -266,12 +282,14 @@ type urlRegexp struct {
 	regexp *regexp.Regexp
 }
 
+// HTTPURLRegexp a slice of regexp to match URLs
 type HTTPURLRegexp []urlRegexp
 
 func (r *HTTPURLRegexp) String() string {
 	return fmt.Sprint(*r)
 }
 
+// Set method to implement flags.Value
 func (r *HTTPURLRegexp) Set(value string) error {
 	regexp, err := regexp.Compile(value)
 

http_prettifier.go

@@ -12,25 +12,22 @@ import (
 )
 
 func prettifyHTTP(p []byte) []byte {
-	headSize := bytes.IndexByte(p, '\n') + 1
-	head := p[:headSize]
-	body := p[headSize:]
 
-	tEnc := bytes.Equal(proto.Header(body, []byte("Transfer-Encoding")), []byte("chunked"))
-	cEnc := bytes.Equal(proto.Header(body, []byte("Content-Encoding")), []byte("gzip"))
+	tEnc := bytes.Equal(proto.Header(p, []byte("Transfer-Encoding")), []byte("chunked"))
+	cEnc := bytes.Equal(proto.Header(p, []byte("Content-Encoding")), []byte("gzip"))
 
 	if !(tEnc || cEnc) {
 		return p
 	}
 
-	headersPos := proto.MIMEHeadersEndPos(body)
+	headersPos := proto.MIMEHeadersEndPos(p)
 
-	if headersPos < 5 || headersPos > len(body) {
+	if headersPos < 5 || headersPos > len(p) {
 		return p
 	}
 
-	headers := body[:headersPos]
-	content := body[headersPos:]
+	headers := p[:headersPos]
+	content := p[headersPos:]
 
 	if tEnc {
 		buf := bytes.NewReader(content)
@@ -64,7 +61,7 @@ func prettifyHTTP(p []byte) []byte {
 		headers = proto.SetHeader(headers, []byte("Content-Length"), []byte(newLen))
 	}
 
-	newPayload := append(append(head, headers...), content...)
+	newPayload := append(headers, content...)
 
 	return newPayload
 }

http_prettifier_test.go

@@ -5,6 +5,8 @@ import (
 	"compress/gzip"
 	"strconv"
 	"testing"
+
+	"github.com/buger/goreplay/proto"
 )
 
 func TestHTTPPrettifierGzip(t *testing.T) {
@@ -15,22 +17,21 @@ func TestHTTPPrettifierGzip(t *testing.T) {
 
 	size := strconv.Itoa(len(b.Bytes()))
 
-	payload := []byte("2 1 1\nHTTP/1.1 200 OK\r\nContent-Length: " + size + "\r\nContent-Encoding: gzip\r\n\r\n")
+	payload := []byte("HTTP/1.1 200 OK\r\nContent-Length: " + size + "\r\nContent-Encoding: gzip\r\n\r\n")
 	payload = append(payload, b.Bytes()...)
 
 	newPayload := prettifyHTTP(payload)
 
-	if string(newPayload) != "2 1 1\nHTTP/1.1 200 OK\r\nContent-Length: 4\r\n\r\ntest" {
-		t.Error("Payload not match:", string(newPayload))
+	if string(newPayload) != "HTTP/1.1 200 OK\r\nContent-Length: 4\r\n\r\ntest" {
+		t.Errorf("Payload not match %q", string(newPayload))
 	}
 }
 
 func TestHTTPPrettifierChunked(t *testing.T) {
 	payload := []byte("POST / HTTP/1.1\r\nHost: www.w3.org\r\nTransfer-Encoding: chunked\r\n\r\n4\r\nWiki\r\n5\r\npedia\r\ne\r\n in\r\n\r\nchunks.\r\n0\r\n\r\n")
 
-	newPayload := prettifyHTTP(payload)
-
-	if string(newPayload) != "POST / HTTP/1.1\r\nHost: www.w3.org\r\nContent-Length: 23\r\n\r\nWikipedia in\r\n\r\nchunks." {
-		t.Error("Payload not match:", string(newPayload))
+	payload = prettifyHTTP(payload)
+	if string(proto.Header(payload, []byte("Content-Length"))) != "23" {
+		t.Errorf("payload should have content length of 23")
 	}
 }

input_dummy.go

@@ -7,24 +7,30 @@ import (
 // DummyInput used for debugging. It generate 1 "GET /"" request per second.
 type DummyInput struct {
 	data chan []byte
+	quit chan struct{}
 }
 
 // NewDummyInput constructor for DummyInput
 func NewDummyInput(options string) (di *DummyInput) {
 	di = new(DummyInput)
 	di.data = make(chan []byte)
+	di.quit = make(chan struct{})
 
 	go di.emit()
 
 	return
 }
 
-func (i *DummyInput) Read(data []byte) (int, error) {
-	buf := <-i.data
-
-	copy(data, buf)
-
-	return len(buf), nil
+// PluginRead reads message from this plugin
+func (i *DummyInput) PluginRead() (*Message, error) {
+	var msg Message
+	select {
+	case <-i.quit:
+		return nil, ErrorStopped
+	case buf := <-i.data:
+		msg.Meta, msg.Data = payloadMetaWithBody(buf)
+		return &msg, nil
+	}
 }
 
 func (i *DummyInput) emit() {
@@ -46,3 +52,9 @@ func (i *DummyInput) emit() {
 func (i *DummyInput) String() string {
 	return "Dummy Input"
 }
+
+// Close closes this plugins
+func (i *DummyInput) Close() error {
+	close(i.quit)
+	return nil
+}

input_file.go

@@ -5,8 +5,8 @@ import (
 	"bytes"
 	"compress/gzip"
 	"errors"
+	"fmt"
 	"io"
-	"log"
 	"os"
 	"path/filepath"
 	"strconv"
@@ -64,6 +64,8 @@ func (f *fileInputReader) ReadPayload() []byte {
 
 	return f.data
 }
+
+// Close closes this plugin
 func (f *fileInputReader) Close() error {
 	if atomic.LoadInt32(&f.closed) == 0 {
 		atomic.StoreInt32(&f.closed, 1)
@@ -73,7 +75,7 @@ func (f *fileInputReader) Close() error {
 	return nil
 }
 
-func NewFileInputReader(path string) *fileInputReader {
+func newFileInputReader(path string) *fileInputReader {
 	var file io.ReadCloser
 	var err error
 
@@ -84,7 +86,7 @@ func NewFileInputReader(path string) *fileInputReader {
 	}
 
 	if err != nil {
-		log.Println(err)
+		Debug(0, fmt.Sprintf("[INPUT-FILE] err: %q", err))
 		return nil
 	}
 
@@ -92,7 +94,7 @@ func NewFileInputReader(path string) *fileInputReader {
 	if strings.HasSuffix(path, ".gz") {
 		gzReader, err := gzip.NewReader(file)
 		if err != nil {
-			log.Println(err)
+			Debug(0, fmt.Sprintf("[INPUT-FILE] err: %q", err))
 			return nil
 		}
 		r.reader = bufio.NewReader(gzReader)
@@ -153,7 +155,7 @@ func (i *FileInput) init() (err error) {
 
 		resp, err := svc.ListObjects(params)
 		if err != nil {
-			log.Println("Error while retreiving list of files from S3", i.path, err)
+			Debug(0, "[INPUT-FILE] Error while retreiving list of files from S3", i.path, err)
 			return err
 		}
 
@@ -162,34 +164,35 @@ func (i *FileInput) init() (err error) {
 		}
 	} else {
 		if matches, err = filepath.Glob(i.path); err != nil {
-			log.Println("Wrong file pattern", i.path, err)
+			Debug(0, "[INPUT-FILE] Wrong file pattern", i.path, err)
 			return
 		}
 	}
 
 	if len(matches) == 0 {
-		log.Println("No files match pattern: ", i.path)
+		Debug(0, "[INPUT-FILE] No files match pattern: ", i.path)
 		return errors.New("No matching files")
 	}
 
 	i.readers = make([]*fileInputReader, len(matches))
 
 	for idx, p := range matches {
-		i.readers[idx] = NewFileInputReader(p)
+		i.readers[idx] = newFileInputReader(p)
 	}
 
 	return nil
 }
 
-func (i *FileInput) Read(data []byte) (int, error) {
-	var buf []byte
+// PluginRead reads message from this plugin
+func (i *FileInput) PluginRead() (*Message, error) {
+	var msg Message
 	select {
 	case <-i.exit:
-		return 0, ErrorStopped
-	case buf = <-i.data:
+		return nil, ErrorStopped
+	case buf := <-i.data:
+		msg.Meta, msg.Data = payloadMetaWithBody(buf)
+		return &msg, nil
 	}
-	n := copy(data, buf)
-	return n, nil
 }
 
 func (i *FileInput) String() string {
@@ -256,7 +259,7 @@ func (i *FileInput) emit() {
 		}
 	}
 
-	log.Printf("FileInput: end of file '%s'\n", i.path)
+	Debug(0, fmt.Sprintf("[INPUT-FILE] FileInput: end of file '%s'\n", i.path))
 
 }
 

input_file_test.go

@@ -4,9 +4,7 @@ import (
 	"bytes"
 	"errors"
 	"fmt"
-	"io"
 	"io/ioutil"
-	"log"
 	"math/rand"
 	"os"
 	"sync"
@@ -14,20 +12,18 @@ import (
 	"time"
 )
 
-var _ = log.Println
-
 func TestInputFileWithGET(t *testing.T) {
 	input := NewTestInput()
-	rg := NewRequestGenerator([]io.Reader{input}, func() { input.EmitGET() }, 1)
-	readPayloads := [][]byte{}
+	rg := NewRequestGenerator([]PluginReader{input}, func() { input.EmitGET() }, 1)
+	readPayloads := []*Message{}
 
 	// Given a capture file with a GET request
 	expectedCaptureFile := CreateCaptureFile(rg)
 	defer expectedCaptureFile.TearDown()
 
 	// When the request is read from the capture file
-	err := ReadFromCaptureFile(expectedCaptureFile.file, 1, func(data []byte) {
-		readPayloads = append(readPayloads, Duplicate(data))
+	err := ReadFromCaptureFile(expectedCaptureFile.file, 1, func(msg *Message) {
+		readPayloads = append(readPayloads, msg)
 	})
 
 	// The read request should match the original request
@@ -41,18 +37,17 @@ func TestInputFileWithGET(t *testing.T) {
 }
 
 func TestInputFileWithPayloadLargerThan64Kb(t *testing.T) {
-
 	input := NewTestInput()
-	rg := NewRequestGenerator([]io.Reader{input}, func() { input.EmitSizedPOST(64 * 1024) }, 1)
-	readPayloads := [][]byte{}
+	rg := NewRequestGenerator([]PluginReader{input}, func() { input.EmitSizedPOST(64 * 1024) }, 1)
+	readPayloads := []*Message{}
 
 	// Given a capture file with a request over 64Kb
 	expectedCaptureFile := CreateCaptureFile(rg)
 	defer expectedCaptureFile.TearDown()
 
 	// When the request is read from the capture file
-	err := ReadFromCaptureFile(expectedCaptureFile.file, 1, func(data []byte) {
-		readPayloads = append(readPayloads, Duplicate(data))
+	err := ReadFromCaptureFile(expectedCaptureFile.file, 1, func(msg *Message) {
+		readPayloads = append(readPayloads, msg)
 	})
 
 	// The read request should match the original request
@@ -69,19 +64,19 @@ func TestInputFileWithPayloadLargerThan64Kb(t *testing.T) {
 func TestInputFileWithGETAndPOST(t *testing.T) {
 
 	input := NewTestInput()
-	rg := NewRequestGenerator([]io.Reader{input}, func() {
+	rg := NewRequestGenerator([]PluginReader{input}, func() {
 		input.EmitGET()
 		input.EmitPOST()
 	}, 2)
-	readPayloads := [][]byte{}
+	readPayloads := []*Message{}
 
 	// Given a capture file with a GET request
 	expectedCaptureFile := CreateCaptureFile(rg)
 	defer expectedCaptureFile.TearDown()
 
 	// When the requests are read from the capture file
-	err := ReadFromCaptureFile(expectedCaptureFile.file, 2, func(data []byte) {
-		readPayloads = append(readPayloads, Duplicate(data))
+	err := ReadFromCaptureFile(expectedCaptureFile.file, 2, func(msg *Message) {
+		readPayloads = append(readPayloads, msg)
 	})
 
 	// The read requests should match the original request
@@ -113,12 +108,11 @@ func TestInputFileMultipleFilesWithRequestsOnly(t *testing.T) {
 	file2.Close()
 
 	input := NewFileInput(fmt.Sprintf("/tmp/%d*", rnd), false)
-	buf := make([]byte, 1000)
 
 	for i := '1'; i <= '4'; i++ {
-		n, _ := input.Read(buf)
-		if buf[4] != byte(i) {
-			t.Error("Should emit requests in right order", string(buf[:n]))
+		msg, _ := input.PluginRead()
+		if msg.Meta[4] != byte(i) {
+			t.Error("Should emit requests in right order", string(msg.Meta))
 		}
 	}
 
@@ -140,11 +134,10 @@ func TestInputFileRequestsWithLatency(t *testing.T) {
 	file.Write([]byte(payloadSeparator))
 
 	input := NewFileInput(fmt.Sprintf("/tmp/%d", rnd), false)
-	buf := make([]byte, 1000)
 
 	start := time.Now().UnixNano()
 	for i := 0; i < 3; i++ {
-		input.Read(buf)
+		input.PluginRead()
 	}
 	end := time.Now().UnixNano()
 
@@ -181,17 +174,16 @@ func TestInputFileMultipleFilesWithRequestsAndResponses(t *testing.T) {
 	file2.Close()
 
 	input := NewFileInput(fmt.Sprintf("/tmp/%d*", rnd), false)
-	buf := make([]byte, 1000)
 
 	for i := '1'; i <= '4'; i++ {
-		n, _ := input.Read(buf)
-		if buf[0] != '1' && buf[4] != byte(i) {
-			t.Error("Shound emit requests in right order", string(buf[:n]))
+		msg, _ := input.PluginRead()
+		if msg.Meta[0] != '1' && msg.Meta[4] != byte(i) {
+			t.Error("Shound emit requests in right order", string(msg.Meta))
 		}
 
-		n, _ = input.Read(buf)
-		if buf[0] != '2' && buf[4] != byte(i) {
-			t.Error("Shound emit responses in right order", string(buf[:n]))
+		msg, _ = input.PluginRead()
+		if msg.Meta[0] != '2' && msg.Meta[4] != byte(i) {
+			t.Error("Shound emit responses in right order", string(msg.Meta))
 		}
 	}
 
@@ -210,11 +202,10 @@ func TestInputFileLoop(t *testing.T) {
 	file.Close()
 
 	input := NewFileInput(fmt.Sprintf("/tmp/%d", rnd), true)
-	buf := make([]byte, 1000)
 
 	// Even if we have just 2 requests in file, it should indifinitly loop
 	for i := 0; i < 1000; i++ {
-		input.Read(buf)
+		input.PluginRead()
 	}
 
 	input.Close()
@@ -226,22 +217,21 @@ func TestInputFileCompressed(t *testing.T) {
 
 	output := NewFileOutput(fmt.Sprintf("/tmp/%d_0.gz", rnd), &FileOutputConfig{FlushInterval: time.Minute, Append: true})
 	for i := 0; i < 1000; i++ {
-		output.Write([]byte("1 1 1\r\ntest"))
+		output.PluginWrite(&Message{Meta: []byte("1 1 1\r\n"), Data: []byte("test")})
 	}
 	name1 := output.file.Name()
 	output.Close()
 
 	output2 := NewFileOutput(fmt.Sprintf("/tmp/%d_1.gz", rnd), &FileOutputConfig{FlushInterval: time.Minute, Append: true})
 	for i := 0; i < 1000; i++ {
-		output2.Write([]byte("1 1 1\r\ntest"))
+		output2.PluginWrite(&Message{Meta: []byte("1 1 1\r\n"), Data: []byte("test")})
 	}
 	name2 := output2.file.Name()
 	output2.Close()
 
 	input := NewFileInput(fmt.Sprintf("/tmp/%d*", rnd), false)
-	buf := make([]byte, 1000)
 	for i := 0; i < 2000; i++ {
-		input.Read(buf)
+		input.PluginRead()
 	}
 
 	os.Remove(name1)
@@ -249,14 +239,14 @@ func TestInputFileCompressed(t *testing.T) {
 }
 
 type CaptureFile struct {
-	data [][]byte
+	msgs []*Message
 	file *os.File
 }
 
-func NewExpectedCaptureFile(data [][]byte, file *os.File) *CaptureFile {
+func NewExpectedCaptureFile(msgs []*Message, file *os.File) *CaptureFile {
 	ecf := new(CaptureFile)
 	ecf.file = file
-	ecf.data = data
+	ecf.msgs = msgs
 	return ecf
 }
 
@@ -267,12 +257,12 @@ func (expectedCaptureFile *CaptureFile) TearDown() {
 }
 
 type RequestGenerator struct {
-	inputs []io.Reader
+	inputs []PluginReader
 	emit   func()
 	wg     *sync.WaitGroup
 }
 
-func NewRequestGenerator(inputs []io.Reader, emit func(), count int) (rg *RequestGenerator) {
+func NewRequestGenerator(inputs []PluginReader, emit func(), count int) (rg *RequestGenerator) {
 	rg = new(RequestGenerator)
 	rg.inputs = inputs
 	rg.emit = emit
@@ -281,14 +271,17 @@ func NewRequestGenerator(inputs []io.Reader, emit func(), count int) (rg *Reques
 	return
 }
 
-func (expectedCaptureFile *CaptureFile) PayloadsEqual(other [][]byte) bool {
+func (expectedCaptureFile *CaptureFile) PayloadsEqual(other []*Message) bool {
 
-	if len(expectedCaptureFile.data) != len(other) {
+	if len(expectedCaptureFile.msgs) != len(other) {
 		return false
 	}
 
 	for i, payload := range other {
-		if !bytes.Equal(expectedCaptureFile.data[i], payload) {
+		if !bytes.Equal(expectedCaptureFile.msgs[i].Meta, payload.Meta) {
+			return false
+		}
+		if !bytes.Equal(expectedCaptureFile.msgs[i].Data, payload.Data) {
 			return false
 		}
 	}
@@ -303,26 +296,24 @@ func CreateCaptureFile(requestGenerator *RequestGenerator) *CaptureFile {
 		panic(err)
 	}
 
-	quit := make(chan int)
-
-	readPayloads := [][]byte{}
-	output := NewTestOutput(func(data []byte) {
-		readPayloads = append(readPayloads, Duplicate(data))
+	readPayloads := []*Message{}
+	output := NewTestOutput(func(msg *Message) {
+		readPayloads = append(readPayloads, msg)
 		requestGenerator.wg.Done()
 	})
 
-	outputFile := NewFileOutput(f.Name(), &FileOutputConfig{FlushInterval: time.Minute, Append: true})
+	outputFile := NewFileOutput(f.Name(), &FileOutputConfig{FlushInterval: time.Second, Append: true})
 
 	plugins := &InOutPlugins{
 		Inputs:  requestGenerator.inputs,
-		Outputs: []io.Writer{output, outputFile},
+		Outputs: []PluginWriter{output, outputFile},
 	}
 	for _, input := range requestGenerator.inputs {
 		plugins.All = append(plugins.All, input)
 	}
 	plugins.All = append(plugins.All, output, outputFile)
 
-	emitter := NewEmitter(quit)
+	emitter := NewEmitter()
 	go emitter.Start(plugins, Settings.Middleware)
 
 	requestGenerator.emit()
@@ -336,23 +327,22 @@ func CreateCaptureFile(requestGenerator *RequestGenerator) *CaptureFile {
 }
 
 func ReadFromCaptureFile(captureFile *os.File, count int, callback writeCallback) (err error) {
-	quit := make(chan int)
 	wg := new(sync.WaitGroup)
 
 	input := NewFileInput(captureFile.Name(), false)
-	output := NewTestOutput(func(data []byte) {
-		callback(data)
+	output := NewTestOutput(func(msg *Message) {
+		callback(msg)
 		wg.Done()
 	})
 
 	plugins := &InOutPlugins{
-		Inputs:  []io.Reader{input},
-		Outputs: []io.Writer{output},
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{output},
 	}
 	plugins.All = append(plugins.All, input, output)
 
 	wg.Add(count)
-	emitter := NewEmitter(quit)
+	emitter := NewEmitter()
 	go emitter.Start(plugins, Settings.Middleware)
 
 	done := make(chan int, 1)
@@ -371,10 +361,3 @@ func ReadFromCaptureFile(captureFile *os.File, count int, callback writeCallback
 	return
 
 }
-
-func Duplicate(data []byte) (duplicate []byte) {
-	duplicate = make([]byte, len(data))
-	copy(duplicate, data)
-
-	return
-}

input_http.go

@@ -27,25 +27,17 @@ func NewHTTPInput(address string) (i *HTTPInput) {
 	return
 }
 
-func (i *HTTPInput) Read(data []byte) (int, error) {
-	var buf []byte
+// PluginRead reads message from this plugin
+func (i *HTTPInput) PluginRead() (*Message, error) {
+	var msg Message
 	select {
 	case <-i.stop:
-		return 0, ErrorStopped
-	case buf = <-i.data:
+		return nil, ErrorStopped
+	case buf := <-i.data:
+		msg.Data = buf
+		msg.Meta = payloadHeader(RequestPayload, uuid(), time.Now().UnixNano(), -1)
+		return &msg, nil
 	}
-	header := payloadHeader(RequestPayload, uuid(), time.Now().UnixNano(), -1)
-
-	n := copy(data, header)
-	if len(data) > len(header) {
-		n += copy(data[len(header):], buf)
-	}
-	dis := len(header) + len(buf) - n
-	if dis > 0 {
-		Debug(2, "[INPUT-HTTP] discarded", dis, "increase copy buffer size")
-	}
-
-	return n, nil
 }
 
 // Close closes this plugin

input_http_test.go

@@ -2,33 +2,29 @@ package main
 
 import (
 	"bytes"
-	"io"
 	"net/http"
 	"strings"
 	"sync"
 	"testing"
 	"time"
-
-	"github.com/buger/goreplay/proto"
 )
 
 func TestHTTPInput(t *testing.T) {
 	wg := new(sync.WaitGroup)
-	quit := make(chan int)
 
 	input := NewHTTPInput("127.0.0.1:0")
 	time.Sleep(time.Millisecond)
-	output := NewTestOutput(func(data []byte) {
+	output := NewTestOutput(func(*Message) {
 		wg.Done()
 	})
 
 	plugins := &InOutPlugins{
-		Inputs:  []io.Reader{input},
-		Outputs: []io.Writer{output},
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{output},
 	}
 	plugins.All = append(plugins.All, input, output)
 
-	emitter := NewEmitter(quit)
+	emitter := NewEmitter()
 	go emitter.Start(plugins, Settings.Middleware)
 
 	address := strings.Replace(input.address, "[::]", "127.0.0.1", -1)
@@ -44,27 +40,25 @@ func TestHTTPInput(t *testing.T) {
 
 func TestInputHTTPLargePayload(t *testing.T) {
 	wg := new(sync.WaitGroup)
-	quit := make(chan int, 1)
 	const n = 10 << 20 // 10MB
 	var large [n]byte
 	large[n-1] = '0'
 
 	input := NewHTTPInput("127.0.0.1:0")
-	time.Sleep(time.Millisecond)
-	output := NewTestOutput(func(data []byte) {
-		_len := len(proto.Body(payloadBody(data)))
+	output := NewTestOutput(func(msg *Message) {
+		_len := len(msg.Data)
 		if _len >= n { // considering http body CRLF
 			t.Errorf("expected body to be >= %d", n)
 		}
 		wg.Done()
 	})
 	plugins := &InOutPlugins{
-		Inputs:  []io.Reader{input},
-		Outputs: []io.Writer{output},
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{output},
 	}
 	plugins.All = append(plugins.All, input, output)
 
-	emitter := NewEmitter(quit)
+	emitter := NewEmitter()
 	defer emitter.Close()
 	go emitter.Start(plugins, Settings.Middleware)
 

input_kafka.go

@@ -15,15 +15,11 @@ type KafkaInput struct {
 	config    *InputKafkaConfig
 	consumers []sarama.PartitionConsumer
 	messages  chan *sarama.ConsumerMessage
+	quit      chan struct{}
 }
 
-// NewKafkaInput creates instance of kafka consumer client.
-func NewKafkaInput(address string, config *InputKafkaConfig) *KafkaInput {
-	return NewKafkaInputWithTLS(address, config, nil)
-}
-
-// NewKafkaInputWithTLS creates instance of kafka consumer client with TLS
-func NewKafkaInputWithTLS(address string, config *InputKafkaConfig, tlsConfig *KafkaTLSConfig) *KafkaInput {
+// NewKafkaInput creates instance of kafka consumer client with TLS config
+func NewKafkaInput(address string, config *InputKafkaConfig, tlsConfig *KafkaTLSConfig) *KafkaInput {
 	c := NewKafkaConfig(tlsConfig)
 
 	var con sarama.Consumer
@@ -48,6 +44,7 @@ func NewKafkaInputWithTLS(address string, config *InputKafkaConfig, tlsConfig *K
 		config:    config,
 		consumers: make([]sarama.PartitionConsumer, len(partitions)),
 		messages:  make(chan *sarama.ConsumerMessage, 256),
+		quit:      make(chan struct{}),
 	}
 
 	for index, partition := range partitions {
@@ -79,29 +76,45 @@ func (i *KafkaInput) ErrorHandler(consumer sarama.PartitionConsumer) {
 	}
 }
 
-func (i *KafkaInput) Read(data []byte) (int, error) {
-	message := <-i.messages
-
-	if !i.config.UseJSON {
-		copy(data, message.Value)
-		return len(message.Value), nil
+// PluginRead a reads message from this plugin
+func (i *KafkaInput) PluginRead() (*Message, error) {
+	var message *sarama.ConsumerMessage
+	var msg Message
+	select {
+	case <-i.quit:
+		return nil, ErrorStopped
+	case message = <-i.messages:
 	}
 
-	var kafkaMessage KafkaMessage
-	json.Unmarshal(message.Value, &kafkaMessage)
+	msg.Data = message.Value
+	if i.config.UseJSON {
 
-	buf, err := kafkaMessage.Dump()
-	if err != nil {
-		Debug(1, "Failed to decode access log entry:", err)
-		return 0, err
+		var kafkaMessage KafkaMessage
+		json.Unmarshal(message.Value, &kafkaMessage)
+
+		var err error
+		msg.Data, err = kafkaMessage.Dump()
+		if err != nil {
+			Debug(1, "[INPUT-KAFKA] failed to decode access log entry:", err)
+			return nil, err
+		}
 	}
 
-	n := copy(data, buf)
+	// does it have meta
+	if isOriginPayload(msg.Data) {
+		msg.Meta, msg.Data = payloadMetaWithBody(msg.Data)
+	}
 
-	return n, nil
+	return &msg, nil
 
 }
 
 func (i *KafkaInput) String() string {
 	return "Kafka Input: " + i.config.Host + "/" + i.config.Topic
 }
+
+// Close closes this plugin
+func (i *KafkaInput) Close() error {
+	close(i.quit)
+	return nil
+}

input_kafka_test.go

@@ -20,17 +20,16 @@ func TestInputKafkaRAW(t *testing.T) {
 		consumer: consumer,
 		Topic:    "test",
 		UseJSON:  false,
-	})
+	}, nil)
 
-	buf := make([]byte, 1024)
-	n, err := input.Read(buf)
+	msg, err := input.PluginRead()
 
 	if err != nil {
 		t.Fatal(err)
 	}
 
-	if string(buf[:n]) != "1 2 3\nGET / HTTP1.1\r\nHeader: 1\r\n\r\n" {
-		t.Error("Message not properly decoded: ", string(buf[:n]), n)
+	if string(append(msg.Meta, msg.Data...)) != "1 2 3\nGET / HTTP1.1\r\nHeader: 1\r\n\r\n" {
+		t.Error("Message not properly decoded")
 	}
 }
 
@@ -47,16 +46,15 @@ func TestInputKafkaJSON(t *testing.T) {
 		consumer: consumer,
 		Topic:    "test",
 		UseJSON:  true,
-	})
+	}, nil)
 
-	buf := make([]byte, 1024)
-	n, err := input.Read(buf)
+	msg, err := input.PluginRead()
 
 	if err != nil {
 		t.Fatal(err)
 	}
 
-	if string(buf[:n]) != "1 2 3\nGET / HTTP/1.1\r\nHeader: 1\r\n\r\n" {
-		t.Error("Message not properly decoded: ", string(buf[:n]), n)
+	if string(append(msg.Meta, msg.Data...)) != "1 2 3\nGET / HTTP/1.1\r\nHeader: 1\r\n\r\n" {
+		t.Error("Message not properly decoded")
 	}
 }

input_raw.go

@@ -72,13 +72,14 @@ type RAWInput struct {
 	listener       *capture.Listener
 	message        chan *tcp.Message
 	cancelListener context.CancelFunc
+	closed         bool
 }
 
 // NewRAWInput constructor for RAWInput. Accepts raw input config as arguments.
 func NewRAWInput(address string, config RAWInputConfig) (i *RAWInput) {
 	i = new(RAWInput)
 	i.RAWInputConfig = config
-	i.message = make(chan *tcp.Message, 1000)
+	i.message = make(chan *tcp.Message, 10000)
 	i.quit = make(chan bool)
 	var host, _port string
 	var err error
@@ -102,44 +103,39 @@ func NewRAWInput(address string, config RAWInputConfig) (i *RAWInput) {
 	return
 }
 
-func (i *RAWInput) Read(data []byte) (n int, err error) {
-	var msg *tcp.Message
-	var buf []byte
+// PluginRead reads meassage from this plugin
+func (i *RAWInput) PluginRead() (*Message, error) {
+	var msgTCP *tcp.Message
+	var msg Message
 	select {
 	case <-i.quit:
-		return 0, ErrorStopped
-	case msg = <-i.message:
-		buf = msg.Data()
+		return nil, ErrorStopped
+	case msgTCP = <-i.message:
+		msg.Data = msgTCP.Data()
 	}
-	var header []byte
-
 	var msgType byte = ResponsePayload
-	if msg.IsIncoming {
+	if msgTCP.IsIncoming {
 		msgType = RequestPayload
 		if i.RealIPHeader != "" {
-			buf = proto.SetHeader(buf, []byte(i.RealIPHeader), []byte(msg.SrcAddr))
+			msg.Data = proto.SetHeader(msg.Data, []byte(i.RealIPHeader), []byte(msgTCP.SrcAddr))
 		}
 	}
-	header = payloadHeader(msgType, msg.UUID(), msg.Start.UnixNano(), msg.End.UnixNano()-msg.Start.UnixNano())
+	msg.Meta = payloadHeader(msgType, msgTCP.UUID(), msgTCP.Start.UnixNano(), msgTCP.End.UnixNano()-msgTCP.Start.UnixNano())
 
-	n = copy(data, header)
-	if len(data) > len(header) {
-		n += copy(data[len(header):], buf)
-	}
 	// to be removed....
-	if msg.Truncated || len(header)+len(buf)-n > 0 {
-		go Debug(2, "[INPUT-RAW] message truncated, increase copy-buffer-size")
+	if msgTCP.Truncated {
+		Debug(2, "[INPUT-RAW] message truncated, increase copy-buffer-size")
 	}
 	// to be removed...
-	if msg.TimedOut {
-		go Debug(2, "[INPUT-RAW] message timeout reached, increase input-raw-expire")
+	if msgTCP.TimedOut {
+		Debug(2, "[INPUT-RAW] message timeout reached, increase input-raw-expire")
 	}
 	if i.Stats {
-		stat := msg.Stats
+		stat := msgTCP.Stats
 		go i.addStats(stat)
 	}
-	msg = nil
-	return n, nil
+	msgTCP = nil
+	return &msg, nil
 }
 
 func (i *RAWInput) listen(address string) {
@@ -154,17 +150,20 @@ func (i *RAWInput) listen(address string) {
 		log.Fatal(err)
 	}
 	pool := tcp.NewMessagePool(i.CopyBufferSize, i.Expire, Debug, i.handler)
-	pool.End = endHint
-	pool.Start = startHint
+	pool.MatchUUID(i.TrackResponse)
+	if i.Protocol == ProtocolHTTP {
+		pool.Start = http1StartHint
+		pool.End = http1EndHint
+	}
 	var ctx context.Context
 	ctx, i.cancelListener = context.WithCancel(context.Background())
 	errCh := i.listener.ListenBackground(ctx, pool.Handler)
-	select {
-	case err := <-errCh:
-		log.Fatal(err)
-	case <-i.listener.Reading:
-		Debug(1, i)
-	}
+	<-i.listener.Reading
+	Debug(1, i)
+	go func() {
+		<-errCh // the listener closed voluntarily
+		i.Close()
+	}()
 }
 
 func (i *RAWInput) handler(m *tcp.Message) {
@@ -187,8 +186,14 @@ func (i *RAWInput) GetStats() []tcp.Stats {
 
 // Close closes the input raw listener
 func (i *RAWInput) Close() error {
+	i.Lock()
+	defer i.Unlock()
+	if i.closed {
+		return nil
+	}
 	i.cancelListener()
 	close(i.quit)
+	i.closed = true
 	return nil
 }
 
@@ -201,7 +206,7 @@ func (i *RAWInput) addStats(mStats tcp.Stats) {
 	i.Unlock()
 }
 
-func startHint(pckt *tcp.Packet) (isIncoming, isOutgoing bool) {
+func http1StartHint(pckt *tcp.Packet) (isIncoming, isOutgoing bool) {
 	isIncoming = proto.HasRequestTitle(pckt.Payload)
 	if isIncoming {
 		return
@@ -209,6 +214,6 @@ func startHint(pckt *tcp.Packet) (isIncoming, isOutgoing bool) {
 	return false, proto.HasResponseTitle(pckt.Payload)
 }
 
-func endHint(m *tcp.Message) bool {
-	return proto.HasFullPayload(m.Data())
+func http1EndHint(m *tcp.Message) bool {
+	return proto.HasFullPayload(m.Data(), m)
 }

input_raw_test.go

@@ -2,7 +2,6 @@ package main
 
 import (
 	"bytes"
-	"io"
 	"io/ioutil"
 	"net"
 	"net/http"
@@ -23,9 +22,8 @@ const testRawExpire = time.Millisecond * 200
 
 func TestRAWInputIPv4(t *testing.T) {
 	wg := new(sync.WaitGroup)
-	quit := make(chan int)
 
-	listener, err := net.Listen("tcp", ":0")
+	listener, err := net.Listen("tcp4", "127.0.0.1:0")
 	if err != nil {
 		t.Error(err)
 		return
@@ -49,32 +47,31 @@ func TestRAWInputIPv4(t *testing.T) {
 		TrackResponse: true,
 		RealIPHeader:  "X-Real-IP",
 	}
-	input := NewRAWInput(":"+port, conf)
+	input := NewRAWInput(listener.Addr().String(), conf)
 
-	output := NewTestOutput(func(data []byte) {
-		if data[0] == '1' {
-			body := payloadBody(data)
-			if len(proto.Header(body, []byte("X-Real-IP"))) == 0 {
-				t.Error("Should have X-Real-IP header", string(body))
+	output := NewTestOutput(func(msg *Message) {
+		if msg.Meta[0] == '1' {
+			if len(proto.Header(msg.Data, []byte("X-Real-IP"))) == 0 {
+				t.Error("Should have X-Real-IP header")
 			}
-			atomic.AddInt64(&reqCounter, 1)
+			reqCounter++
 		} else {
-			atomic.AddInt64(&respCounter, 1)
+			respCounter++
 		}
 		wg.Done()
 	})
 
 	plugins := &InOutPlugins{
-		Inputs:  []io.Reader{input},
-		Outputs: []io.Writer{output},
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{output},
 	}
 	plugins.All = append(plugins.All, input, output)
 
 	addr := "http://127.0.0.1:" + port
-	emitter := NewEmitter(quit)
+	emitter := NewEmitter()
 	defer emitter.Close()
 	go emitter.Start(plugins, Settings.Middleware)
-	for i := 0; i < 10; i++ {
+	for i := 0; i < 1; i++ {
 		wg.Add(2)
 		_, err = http.Get(addr)
 		if err != nil {
@@ -91,7 +88,6 @@ func TestRAWInputIPv4(t *testing.T) {
 
 func TestRAWInputNoKeepAlive(t *testing.T) {
 	wg := new(sync.WaitGroup)
-	quit := make(chan int)
 
 	listener, err := net.Listen("tcp", "127.0.0.1:0")
 	if err != nil {
@@ -117,8 +113,8 @@ func TestRAWInputNoKeepAlive(t *testing.T) {
 	}
 	input := NewRAWInput(":"+port, conf)
 	var respCounter, reqCounter int64
-	output := NewTestOutput(func(data []byte) {
-		if data[0] == '1' {
+	output := NewTestOutput(func(msg *Message) {
+		if msg.Meta[0] == '1' {
 			atomic.AddInt64(&reqCounter, 1)
 		} else {
 			atomic.AddInt64(&respCounter, 1)
@@ -127,14 +123,14 @@ func TestRAWInputNoKeepAlive(t *testing.T) {
 	})
 
 	plugins := &InOutPlugins{
-		Inputs:  []io.Reader{input},
-		Outputs: []io.Writer{output},
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{output},
 	}
 	plugins.All = append(plugins.All, input, output)
 
 	addr := "http://127.0.0.1:" + port
 
-	emitter := NewEmitter(quit)
+	emitter := NewEmitter()
 	go emitter.Start(plugins, Settings.Middleware)
 
 	for i := 0; i < 10; i++ {
@@ -157,7 +153,6 @@ func TestRAWInputNoKeepAlive(t *testing.T) {
 
 func TestRAWInputIPv6(t *testing.T) {
 	wg := new(sync.WaitGroup)
-	quit := make(chan int)
 
 	listener, err := net.Listen("tcp", "[::1]:0")
 	if err != nil {
@@ -183,8 +178,8 @@ func TestRAWInputIPv6(t *testing.T) {
 	}
 	input := NewRAWInput(originAddr, conf)
 
-	output := NewTestOutput(func(data []byte) {
-		if data[0] == '1' {
+	output := NewTestOutput(func(msg *Message) {
+		if msg.Meta[0] == '1' {
 			atomic.AddInt64(&reqCounter, 1)
 		} else {
 			atomic.AddInt64(&respCounter, 1)
@@ -193,11 +188,11 @@ func TestRAWInputIPv6(t *testing.T) {
 	})
 
 	plugins := &InOutPlugins{
-		Inputs:  []io.Reader{input},
-		Outputs: []io.Writer{output},
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{output},
 	}
 
-	emitter := NewEmitter(quit)
+	emitter := NewEmitter()
 	addr := "http://" + originAddr
 	go emitter.Start(plugins, Settings.Middleware)
 	for i := 0; i < 10; i++ {
@@ -220,7 +215,6 @@ func TestRAWInputIPv6(t *testing.T) {
 
 func TestInputRAWChunkedEncoding(t *testing.T) {
 	wg := new(sync.WaitGroup)
-	quit := make(chan int)
 
 	fileContent, _ := ioutil.ReadFile("README.md")
 
@@ -257,12 +251,12 @@ func TestInputRAWChunkedEncoding(t *testing.T) {
 	httpOutput := NewHTTPOutput(replay.URL, &HTTPOutputConfig{})
 
 	plugins := &InOutPlugins{
-		Inputs:  []io.Reader{input},
-		Outputs: []io.Writer{httpOutput},
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{httpOutput},
 	}
 	plugins.All = append(plugins.All, input, httpOutput)
 
-	emitter := NewEmitter(quit)
+	emitter := NewEmitter()
 	defer emitter.Close()
 	go emitter.Start(plugins, Settings.Middleware)
 	wg.Add(2)
@@ -278,17 +272,15 @@ func TestInputRAWChunkedEncoding(t *testing.T) {
 }
 
 func BenchmarkRAWInputWithReplay(b *testing.B) {
-	var respCounter, reqCounter, replayCounter uint64
+	var respCounter, reqCounter, replayCounter uint32
 	wg := &sync.WaitGroup{}
-	wg.Add(b.N * 3) // reqCounter + replayCounter + respCounter
 
-	quit := make(chan int)
-	listener, err := net.Listen("tcp", "127.0.0.1:0")
+	listener, err := net.Listen("tcp4", "127.0.0.1:0")
 	if err != nil {
 		b.Error(err)
 		return
 	}
-	listener0, err := net.Listen("tcp", "127.0.0.1:0")
+	listener0, err := net.Listen("tcp4", "127.0.0.1:0")
 	if err != nil {
 		b.Error(err)
 		return
@@ -305,6 +297,8 @@ func BenchmarkRAWInputWithReplay(b *testing.B) {
 
 	replay := http.Server{
 		Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			atomic.AddUint32(&replayCounter, 1)
+			w.Write(nil)
 			wg.Done()
 		}),
 	}
@@ -320,34 +314,37 @@ func BenchmarkRAWInputWithReplay(b *testing.B) {
 	}
 	input := NewRAWInput(originAddr, conf)
 
-	testOutput := NewTestOutput(func(data []byte) {
-		if data[0] == '1' {
-			atomic.AddUint64(&reqCounter, 1)
+	testOutput := NewTestOutput(func(msg *Message) {
+		if msg.Meta[0] == '1' {
+			reqCounter++
 		} else {
-			atomic.AddUint64(&respCounter, 1)
+			respCounter++
 		}
 		wg.Done()
 	})
-	httpOutput := NewHTTPOutput(replayAddr, &HTTPOutputConfig{})
+	httpOutput := NewHTTPOutput("http://"+replayAddr, &HTTPOutputConfig{})
 
 	plugins := &InOutPlugins{
-		Inputs:  []io.Reader{input},
-		Outputs: []io.Writer{testOutput, httpOutput},
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{testOutput, httpOutput},
 	}
 
-	emitter := NewEmitter(quit)
+	emitter := NewEmitter()
 	go emitter.Start(plugins, Settings.Middleware)
-	now := time.Now()
 	addr := "http://" + originAddr
+	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
-		_, err = http.Get(addr)
+		wg.Add(3) // reqCounter + replayCounter + respCounter
+		resp, err := http.Get(addr)
 		if err != nil {
-			b.Log(err)
 			wg.Add(-3)
 		}
+		resp.Body.Close()
 	}
 
 	wg.Wait()
-	b.Logf("%d/%d Requests, %d/%d Responses, %d/%d Replayed in %s\n", reqCounter, b.N, respCounter, b.N, replayCounter, b.N, time.Since(now))
+	b.ReportMetric(float64(reqCounter), "requests")
+	b.ReportMetric(float64(respCounter), "responses")
+	b.ReportMetric(float64(replayCounter), "replayed")
 	emitter.Close()
 }

input_tcp.go

@@ -8,18 +8,18 @@ import (
 	"io"
 	"log"
 	"net"
-	"os"
 )
 
 // TCPInput used for internal communication
 type TCPInput struct {
-	data     chan []byte
+	data     chan *Message
 	listener net.Listener
 	address  string
 	config   *TCPInputConfig
 	stop     chan bool // Channel used only to indicate goroutine should shutdown
 }
 
+// TCPInputConfig represents configuration of a TCP input plugin
 type TCPInputConfig struct {
 	Secure          bool   `json:"input-tcp-secure"`
 	CertificatePath string `json:"input-tcp-certificate"`
@@ -29,7 +29,7 @@ type TCPInputConfig struct {
 // NewTCPInput constructor for TCPInput, accepts address with port
 func NewTCPInput(address string, config *TCPInputConfig) (i *TCPInput) {
 	i = new(TCPInput)
-	i.data = make(chan []byte, 1000)
+	i.data = make(chan *Message, 1000)
 	i.address = address
 	i.config = config
 	i.stop = make(chan bool)
@@ -39,20 +39,21 @@ func NewTCPInput(address string, config *TCPInputConfig) (i *TCPInput) {
 	return
 }
 
-func (i *TCPInput) Read(data []byte) (int, error) {
-	var buf []byte
+// PluginRead returns data and details read from plugin
+func (i *TCPInput) PluginRead() (msg *Message, err error) {
 	select {
 	case <-i.stop:
-		return 0, ErrorStopped
-	case buf = <-i.data:
+		return nil, ErrorStopped
+	case msg = <-i.data:
+		return msg, nil
 	}
-	copy(data, buf)
 
-	return len(buf), nil
 }
 
+// Close closes the plugin
 func (i *TCPInput) Close() error {
 	close(i.stop)
+	i.listener.Close()
 	return nil
 }
 
@@ -60,64 +61,67 @@ func (i *TCPInput) listen(address string) {
 	if i.config.Secure {
 		cer, err := tls.LoadX509KeyPair(i.config.CertificatePath, i.config.KeyPath)
 		if err != nil {
-			log.Fatal("Error while loading --input-file certificate:", err)
+			log.Fatalln("error while loading --input-tcp TLS certificate:", err)
 		}
 
 		config := &tls.Config{Certificates: []tls.Certificate{cer}}
 		listener, err := tls.Listen("tcp", address, config)
 		if err != nil {
-			log.Fatal("Can't start --input-tcp with secure connection:", err)
+			log.Fatalln("[INPUT-TCP] failed to start INPUT-TCP listener:", err)
 		}
 		i.listener = listener
 	} else {
 		listener, err := net.Listen("tcp", address)
 		if err != nil {
-			log.Fatal("Can't start:", err)
+			log.Fatalln("failed to start INPUT-TCP listener:", err)
 		}
-
 		i.listener = listener
 	}
-
 	go func() {
 		for {
 			conn, err := i.listener.Accept()
-
-			if err != nil {
-				log.Println("Error while Accept()", err)
+			if err == nil {
+				go i.handleConnection(conn)
 				continue
 			}
-
-			go i.handleConnection(conn)
+			if isTemporaryNetworkError(err) {
+				continue
+			}
+			if operr, ok := err.(*net.OpError); ok && operr.Err.Error() != "use of closed network connection" {
+				Debug(0, fmt.Sprintf("[INPUT-TCP] listener closed, err: %q", err))
+			}
+			break
 		}
 	}()
 }
 
+var payloadSeparatorAsBytes = []byte(payloadSeparator)
+
 func (i *TCPInput) handleConnection(conn net.Conn) {
 	defer conn.Close()
 
-	payloadSeparatorAsBytes := []byte(payloadSeparator)
 	reader := bufio.NewReader(conn)
 	var buffer bytes.Buffer
 
 	for {
 		line, err := reader.ReadBytes('\n')
-
 		if err != nil {
+			if isTemporaryNetworkError(err) {
+				continue
+			}
 			if err != io.EOF {
-				fmt.Fprintln(os.Stderr, "Unexpected error in input tcp connection:", err)
+				Debug(0, fmt.Sprintf("[INPUT-TCP] connection error: %q", err))
 			}
 			break
-
 		}
 
 		if bytes.Equal(payloadSeparatorAsBytes[1:], line) {
-			asBytes := buffer.Bytes()
+			// unread the '\n' before monkeys
+			buffer.UnreadByte()
+			var msg Message
+			msg.Meta, msg.Data = payloadMetaWithBody(buffer.Bytes())
+			i.data <- &msg
 			buffer.Reset()
-
-			newBuf := make([]byte, len(asBytes)-1)
-			copy(newBuf, asBytes)
-
-			i.data <- newBuf
 		} else {
 			buffer.Write(line)
 		}
@@ -127,3 +131,13 @@ func (i *TCPInput) handleConnection(conn net.Conn) {
 func (i *TCPInput) String() string {
 	return "TCP input: " + i.address
 }
+
+func isTemporaryNetworkError(err error) bool {
+	if nerr, ok := err.(net.Error); ok && nerr.Temporary() {
+		return true
+	}
+	if operr, ok := err.(*net.OpError); ok && operr.Temporary() {
+		return true
+	}
+	return false
+}

input_tcp_test.go

@@ -7,7 +7,6 @@ import (
 	"crypto/tls"
 	"crypto/x509"
 	"encoding/pem"
-	"io"
 	"io/ioutil"
 	"log"
 	"math/big"
@@ -20,20 +19,19 @@ import (
 
 func TestTCPInput(t *testing.T) {
 	wg := new(sync.WaitGroup)
-	quit := make(chan int)
 
 	input := NewTCPInput("127.0.0.1:0", &TCPInputConfig{})
-	output := NewTestOutput(func(data []byte) {
+	output := NewTestOutput(func(*Message) {
 		wg.Done()
 	})
 
 	plugins := &InOutPlugins{
-		Inputs:  []io.Reader{input},
-		Outputs: []io.Writer{output},
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{output},
 	}
 	plugins.All = append(plugins.All, input, output)
 
-	emitter := NewEmitter(quit)
+	emitter := NewEmitter()
 	go emitter.Start(plugins, Settings.Middleware)
 
 	tcpAddr, err := net.ResolveTCPAddr("tcp", input.listener.Addr().String())
@@ -43,7 +41,6 @@ func TestTCPInput(t *testing.T) {
 	}
 
 	conn, err := net.DialTCP("tcp", nil, tcpAddr)
-
 	if err != nil {
 		log.Fatal(err)
 	}
@@ -52,10 +49,14 @@ func TestTCPInput(t *testing.T) {
 
 	for i := 0; i < 100; i++ {
 		wg.Add(1)
-		conn.Write(msg)
-		conn.Write([]byte(payloadSeparator))
+		if _, err = conn.Write(msg); err == nil {
+			_, err = conn.Write(payloadSeparatorAsBytes)
+		}
+		if err != nil {
+			t.Error(err)
+			return
+		}
 	}
-
 	wg.Wait()
 	emitter.Close()
 }
@@ -99,24 +100,23 @@ func TestTCPInputSecure(t *testing.T) {
 	}()
 
 	wg := new(sync.WaitGroup)
-	quit := make(chan int)
 
 	input := NewTCPInput("127.0.0.1:0", &TCPInputConfig{
 		Secure:          true,
 		CertificatePath: serverCertPemFile.Name(),
 		KeyPath:         serverPrivPemFile.Name(),
 	})
-	output := NewTestOutput(func(data []byte) {
+	output := NewTestOutput(func(*Message) {
 		wg.Done()
 	})
 
 	plugins := &InOutPlugins{
-		Inputs:  []io.Reader{input},
-		Outputs: []io.Writer{output},
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{output},
 	}
 	plugins.All = append(plugins.All, input, output)
 
-	emitter := NewEmitter(quit)
+	emitter := NewEmitter()
 	go emitter.Start(plugins, Settings.Middleware)
 
 	conf := &tls.Config{

kafka.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"crypto/tls"
 	"crypto/x509"
+	"errors"
 	"fmt"
 	"io/ioutil"
 	"log"
@@ -50,34 +51,43 @@ type KafkaMessage struct {
 	ReqHeaders map[string]string `json:"Req_Headers,omitempty"`
 }
 
+
 // NewTLSConfig loads TLS certificates
 func NewTLSConfig(clientCertFile, clientKeyFile, caCertFile string) (*tls.Config, error) {
 	tlsConfig := tls.Config{}
 
+	if clientCertFile != "" && clientKeyFile == "" {
+		return &tlsConfig, errors.New("Missing key of client certificate in kafka")
+	}
+	if clientCertFile == "" && clientKeyFile != "" {
+		return &tlsConfig, errors.New("missing TLS client certificate in kafka")
+	}
 	// Load client cert
-	cert, err := tls.LoadX509KeyPair(clientCertFile, clientKeyFile)
-	if err != nil {
-		return &tlsConfig, err
+	if  (clientCertFile != "") && (clientKeyFile != "") {
+		cert, err := tls.LoadX509KeyPair(clientCertFile, clientKeyFile)
+		if err != nil {
+			return &tlsConfig, err
+		}
+		tlsConfig.Certificates = []tls.Certificate{cert}
 	}
-	tlsConfig.Certificates = []tls.Certificate{cert}
-
 	// Load CA cert
-	caCert, err := ioutil.ReadFile(caCertFile)
-	if err != nil {
-		return &tlsConfig, err
+	if caCertFile != "" {
+		caCert, err := ioutil.ReadFile(caCertFile)
+		if err != nil {
+			return &tlsConfig, err
+		}
+		caCertPool := x509.NewCertPool()
+		caCertPool.AppendCertsFromPEM(caCert)
+		tlsConfig.RootCAs = caCertPool
 	}
-	caCertPool := x509.NewCertPool()
-	caCertPool.AppendCertsFromPEM(caCert)
-	tlsConfig.RootCAs = caCertPool
-
-	return &tlsConfig, err
+	return &tlsConfig, nil
 }
 
 // NewKafkaConfig returns Kafka config with or without TLS
 func NewKafkaConfig(tlsConfig *KafkaTLSConfig) *sarama.Config {
 	config := sarama.NewConfig()
 	// Configuration options go here
-	if (tlsConfig != nil) && (tlsConfig.CACert != "") && (tlsConfig.ClientCert != "") && (tlsConfig.ClientKey != "") {
+	if tlsConfig != nil && (tlsConfig.ClientCert != "" || tlsConfig.CACert != "") {
 		config.Net.TLS.Enable = true
 		tlsConfig, err := NewTLSConfig(tlsConfig.ClientCert, tlsConfig.ClientKey, tlsConfig.CACert)
 		if err != nil {

limiter.go

@@ -20,8 +20,8 @@ type Limiter struct {
 }
 
 func parseLimitOptions(options string) (limit int, isPercent bool) {
-	if strings.Contains(options, "%") {
-		limit, _ = strconv.Atoi(strings.Split(options, "%")[0])
+	if n := strings.Index(options, "%"); n > 0 {
+		limit, _ = strconv.Atoi(options[:n])
 		isPercent = true
 	} else {
 		limit, _ = strconv.Atoi(options)
@@ -33,7 +33,7 @@ func parseLimitOptions(options string) (limit int, isPercent bool) {
 
 // NewLimiter constructor for Limiter, accepts plugin and options
 // `options` allow to sprcify relatve or absolute limiting
-func NewLimiter(plugin interface{}, options string) io.ReadWriter {
+func NewLimiter(plugin interface{}, options string) PluginReadWriter {
 	l := new(Limiter)
 	l.limit, l.isPercent = parseLimitOptions(options)
 	l.plugin = plugin
@@ -71,24 +71,29 @@ func (l *Limiter) isLimited() bool {
 	return false
 }
 
-func (l *Limiter) Write(data []byte) (n int, err error) {
+// PluginWrite writes message to this plugin
+func (l *Limiter) PluginWrite(msg *Message) (n int, err error) {
 	if l.isLimited() {
 		return 0, nil
 	}
-
-	n, err = l.plugin.(io.Writer).Write(data)
-	return
+	if w, ok := l.plugin.(PluginWriter); ok {
+		return w.PluginWrite(msg)
+	}
+	// avoid further writing
+	return 0, io.ErrClosedPipe
 }
 
-func (l *Limiter) Read(data []byte) (n int, err error) {
-	if r, ok := l.plugin.(io.Reader); ok {
-		n, err = r.Read(data)
+// PluginRead reads message from this plugin
+func (l *Limiter) PluginRead() (msg *Message, err error) {
+	if r, ok := l.plugin.(PluginReader); ok {
+		msg, err = r.PluginRead()
 	} else {
-		return 0, nil
+		// avoid further reading
+		return nil, io.ErrClosedPipe
 	}
 
 	if l.isLimited() {
-		return 0, nil
+		return nil, nil
 	}
 
 	return
@@ -100,7 +105,7 @@ func (l *Limiter) String() string {
 
 // Close closes the resources.
 func (l *Limiter) Close() error {
-	if fi, ok := l.plugin.(io.ReadCloser); ok {
+	if fi, ok := l.plugin.(io.Closer); ok {
 		fi.Close()
 	}
 	return nil

limiter_test.go

@@ -3,28 +3,26 @@
 package main
 
 import (
-	"io"
 	"sync"
 	"testing"
 )
 
 func TestOutputLimiter(t *testing.T) {
 	wg := new(sync.WaitGroup)
-	quit := make(chan int)
 
 	input := NewTestInput()
-	output := NewLimiter(NewTestOutput(func(data []byte) {
+	output := NewLimiter(NewTestOutput(func(*Message) {
 		wg.Done()
 	}), "10")
 	wg.Add(10)
 
 	plugins := &InOutPlugins{
-		Inputs:  []io.Reader{input},
-		Outputs: []io.Writer{output},
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{output},
 	}
 	plugins.All = append(plugins.All, input, output)
 
-	emitter := NewEmitter(quit)
+	emitter := NewEmitter()
 	go emitter.Start(plugins, Settings.Middleware)
 
 	for i := 0; i < 100; i++ {
@@ -37,21 +35,20 @@ func TestOutputLimiter(t *testing.T) {
 
 func TestInputLimiter(t *testing.T) {
 	wg := new(sync.WaitGroup)
-	quit := make(chan int)
 
 	input := NewLimiter(NewTestInput(), "10")
-	output := NewTestOutput(func(data []byte) {
+	output := NewTestOutput(func(*Message) {
 		wg.Done()
 	})
 	wg.Add(10)
 
 	plugins := &InOutPlugins{
-		Inputs:  []io.Reader{input},
-		Outputs: []io.Writer{output},
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{output},
 	}
 	plugins.All = append(plugins.All, input, output)
 
-	emitter := NewEmitter(quit)
+	emitter := NewEmitter()
 	go emitter.Start(plugins, Settings.Middleware)
 
 	for i := 0; i < 100; i++ {
@@ -65,20 +62,19 @@ func TestInputLimiter(t *testing.T) {
 // Should limit all requests
 func TestPercentLimiter1(t *testing.T) {
 	wg := new(sync.WaitGroup)
-	quit := make(chan int)
 
 	input := NewTestInput()
-	output := NewLimiter(NewTestOutput(func(data []byte) {
+	output := NewLimiter(NewTestOutput(func(*Message) {
 		wg.Done()
 	}), "0%")
 
 	plugins := &InOutPlugins{
-		Inputs:  []io.Reader{input},
-		Outputs: []io.Writer{output},
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{output},
 	}
 	plugins.All = append(plugins.All, input, output)
 
-	emitter := NewEmitter(quit)
+	emitter := NewEmitter()
 	go emitter.Start(plugins, Settings.Middleware)
 
 	for i := 0; i < 100; i++ {
@@ -91,21 +87,20 @@ func TestPercentLimiter1(t *testing.T) {
 // Should not limit at all
 func TestPercentLimiter2(t *testing.T) {
 	wg := new(sync.WaitGroup)
-	quit := make(chan int)
 
 	input := NewTestInput()
-	output := NewLimiter(NewTestOutput(func(data []byte) {
+	output := NewLimiter(NewTestOutput(func(*Message) {
 		wg.Done()
 	}), "100%")
 	wg.Add(100)
 
 	plugins := &InOutPlugins{
-		Inputs:  []io.Reader{input},
-		Outputs: []io.Writer{output},
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{output},
 	}
 	plugins.All = append(plugins.All, input, output)
 
-	emitter := NewEmitter(quit)
+	emitter := NewEmitter()
 	go emitter.Start(plugins, Settings.Middleware)
 
 	for i := 0; i < 100; i++ {

middleware.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"bufio"
+	"context"
 	"encoding/hex"
 	"fmt"
 	"io"
@@ -9,32 +10,29 @@ import (
 	"os"
 	"os/exec"
 	"strings"
-	"sync"
 )
 
 // Middleware represents a middleware object
 type Middleware struct {
-	command string
-
-	data chan []byte
-
-	mu sync.Mutex
-
-	Stdin  io.Writer
-	Stdout io.Reader
-
-	stop chan bool // Channel used only to indicate goroutine should shutdown
+	command       string
+	data          chan *Message
+	Stdin         io.Writer
+	Stdout        io.Reader
+	commandCancel context.CancelFunc
+	stop          chan bool // Channel used only to indicate goroutine should shutdown
 }
 
 // NewMiddleware returns new middleware
 func NewMiddleware(command string) *Middleware {
 	m := new(Middleware)
 	m.command = command
-	m.data = make(chan []byte, 1000)
+	m.data = make(chan *Message, 1000)
 	m.stop = make(chan bool)
 
 	commands := strings.Split(command, " ")
-	cmd := exec.Command(commands[0], commands[1:]...)
+	ctx, cancl := context.WithCancel(context.Background())
+	m.commandCancel = cancl
+	cmd := exec.CommandContext(ctx, commands[0], commands[1:]...)
 
 	m.Stdout, _ = cmd.StdoutPipe()
 	m.Stdin, _ = cmd.StdinPipe()
@@ -61,45 +59,33 @@ func NewMiddleware(command string) *Middleware {
 }
 
 // ReadFrom start a worker to read from this plugin
-func (m *Middleware) ReadFrom(plugin io.Reader) {
+func (m *Middleware) ReadFrom(plugin PluginReader) {
 	Debug(2, "[MIDDLEWARE-MASTER] Starting reading from", plugin)
 	go m.copy(m.Stdin, plugin)
 }
 
-func (m *Middleware) copy(to io.Writer, from io.Reader) {
-	buf := make([]byte, 5*1024*1024)
-	dst := make([]byte, len(buf)*4)
+func (m *Middleware) copy(to io.Writer, from PluginReader) {
+	var buf, dst []byte
 
 	for {
-		nr, _ := from.Read(buf)
-		if nr == 0 || nr > len(buf) {
+		msg, err := from.PluginRead()
+		if err != nil {
+			return
+		}
+		if msg == nil || len(msg.Data) == 0 {
 			continue
 		}
 
-		payload := buf[0:nr]
-
+		buf = msg.Data
 		if Settings.PrettifyHTTP {
-			payload = prettifyHTTP(payload)
-			nr = len(payload)
-
-			if nr*2 > len(dst) {
-				continue
-			}
+			buf = prettifyHTTP(msg.Data)
 		}
+		dst = make([]byte, len(buf)*2+1)
+		hex.Encode(dst, buf)
+		dst[len(buf)*2] = '\n'
 
-		if Settings.PrettifyHTTP {
-			payload = prettifyHTTP(payload)
-			nr = len(payload)
-		}
+		to.Write(dst)
 
-		hex.Encode(dst, payload)
-		dst[nr*2] = '\n'
-
-		m.mu.Lock()
-		to.Write(dst[0 : nr*2+1])
-		m.mu.Unlock()
-
-		Debug(3, "[MIDDLEWARE-MASTER] Sending:", string(buf[0:nr]), "From:", from)
 	}
 }
 
@@ -117,41 +103,41 @@ func (m *Middleware) read(from io.Reader) {
 			}
 		}
 
-		buf := make([]byte, len(line)/2)
+		buf := make([]byte, len(line)/2-1)
 		if _, err := hex.Decode(buf, line[:len(line)-1]); err != nil {
-			fmt.Fprintln(os.Stderr, "Failed to decode input payload", err, len(line), string(line[:len(line)-1]))
+			Debug(0, fmt.Sprintf("[MIDDLEWARE] failed to decode err: %q", err))
+			continue
 		}
-
-		Debug(3, "[MIDDLEWARE-MASTER] Received:", string(buf))
-
+		var msg Message
+		msg.Meta, msg.Data = payloadMetaWithBody(buf)
 		select {
 		case <-m.stop:
 			return
-		case m.data <- buf:
+		case m.data <- &msg:
 		}
 	}
 
 	return
 }
 
-func (m *Middleware) Read(data []byte) (int, error) {
-	var buf []byte
+// PluginRead reads message from this plugin
+func (m *Middleware) PluginRead() (msg *Message, err error) {
 	select {
 	case <-m.stop:
-		return 0, ErrorStopped
-	case buf = <-m.data:
+		return nil, ErrorStopped
+	case msg = <-m.data:
 	}
 
-	n := copy(data, buf)
-	return n, nil
+	return
 }
 
 func (m *Middleware) String() string {
-	return fmt.Sprintf("Modifying traffic using '%s' command", m.command)
+	return fmt.Sprintf("Modifying traffic using %q command", m.command)
 }
 
 // Close closes this plugin
 func (m *Middleware) Close() error {
+	m.commandCancel()
 	close(m.stop)
 	return nil
 }

middleware_test.go

@@ -1,225 +1,226 @@
 package main
 
-// import (
-// 	"bytes"
-// 	"crypto/rand"
-// 	"encoding/hex"
-// 	"io"
-// 	"net/http"
-// 	"net/http/httptest"
-// 	"strings"
-// 	"sync"
-// 	"testing"
-// 	"time"
+import (
+	"bytes"
+	"crypto/rand"
+	"encoding/hex"
+	"net/http"
+	"net/http/httptest"
+	"net/http/httputil"
+	"strings"
+	"sync"
+	"testing"
+	"time"
 
-// 	"github.com/buger/goreplay/capture"
-// 	"github.com/buger/goreplay/proto"
-// )
+	"github.com/buger/goreplay/capture"
+	"github.com/buger/goreplay/proto"
+)
 
-// type fakeServiceCb func(string, int, []byte)
+type fakeServiceCb func(string, int, []byte)
 
-// // Simple service that generate token on request, and require this token for accesing to secure area
-// func NewFakeSecureService(wg *sync.WaitGroup, cb fakeServiceCb) *httptest.Server {
-// 	active_tokens := make([]string, 0)
-// 	var mu sync.Mutex
+// Simple service that generate token on request, and require this token for accesing to secure area
+func NewFakeSecureService(wg *sync.WaitGroup, cb fakeServiceCb) *httptest.Server {
+	activeTokens := make([]string, 0)
+	var mu sync.Mutex
 
-// 	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
-// 		mu.Lock()
-// 		defer mu.Unlock()
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
+		mu.Lock()
+		defer mu.Unlock()
 
-// 		switch req.URL.Path {
-// 		case "/token":
-// 			// Generate random token
-// 			token_length := 10
-// 			buf := make([]byte, token_length)
-// 			rand.Read(buf)
-// 			token := hex.EncodeToString(buf)
-// 			active_tokens = append(active_tokens, token)
+		switch req.URL.Path {
+		case "/token":
+			// Generate random token
+			tokenLength := 10
+			buf := make([]byte, tokenLength)
+			rand.Read(buf)
+			token := hex.EncodeToString(buf)
+			activeTokens = append(activeTokens, token)
 
-// 			w.Write([]byte(token))
+			w.Write([]byte(token))
 
-// 			cb(req.URL.Path, 200, []byte(token))
-// 		case "/secure":
-// 			token := req.URL.Query().Get("token")
-// 			token_found := false
+			cb(req.URL.Path, 200, []byte(token))
+		case "/secure":
+			token := req.URL.Query().Get("token")
+			tokenFound := false
 
-// 			for _, t := range active_tokens {
-// 				if t == token {
-// 					token_found = true
-// 					break
-// 				}
-// 			}
+			for _, t := range activeTokens {
+				if t == token {
+					tokenFound = true
+					break
+				}
+			}
 
-// 			if token_found {
-// 				w.WriteHeader(http.StatusAccepted)
-// 				cb(req.URL.Path, 202, []byte(nil))
-// 			} else {
-// 				w.WriteHeader(http.StatusForbidden)
-// 				cb(req.URL.Path, 403, []byte(nil))
-// 			}
-// 		}
+			if tokenFound {
+				w.WriteHeader(http.StatusAccepted)
+				cb(req.URL.Path, 202, nil)
+			} else {
+				w.WriteHeader(http.StatusForbidden)
+				cb(req.URL.Path, 403, nil)
+			}
+		}
 
-// 		wg.Done()
-// 	}))
+		wg.Done()
+	}))
 
-// 	return server
-// }
+	return server
+}
 
-// func TestFakeSecureService(t *testing.T) {
-// 	var resp, token []byte
+func TestFakeSecureService(t *testing.T) {
+	var resp, token []byte
 
-// 	wg := new(sync.WaitGroup)
+	wg := new(sync.WaitGroup)
 
-// 	server := NewFakeSecureService(wg, func(path string, status int, resp []byte) {
-// 	})
-// 	defer server.Close()
+	server := NewFakeSecureService(wg, func(path string, status int, resp []byte) {
+	})
+	defer server.Close()
 
-// 	wg.Add(3)
+	wg.Add(3)
 
-// 	client := NewHTTPClient(server.URL, &HTTPClientConfig{Debug: true})
-// 	resp, _ = client.Get("/token")
-// 	token = proto.Body(resp)
+	client := NewHTTPClient(&HTTPOutputConfig{}).Client
+	rep, _ := client.Get(server.URL + "/token")
+	resp, _ = httputil.DumpResponse(rep, true)
+	token = proto.Body(resp)
 
-// 	// Right token
-// 	resp, _ = client.Get("/secure?token=" + string(token))
-// 	if !bytes.Equal(proto.Status(resp), []byte("202")) {
-// 		t.Error("Valid token should return status 202:", string(proto.Status(resp)))
-// 	}
+	// Right token
+	rep, _ = client.Get(server.URL + "/secure?token=" + string(token))
+	resp, _ = httputil.DumpResponse(rep, true)
+	if !bytes.Equal(proto.Status(resp), []byte("202")) {
+		t.Error("Valid token should return status 202:", string(proto.Status(resp)))
+	}
 
-// 	// Wrong tokens forbidden
-// 	resp, _ = client.Get("/secure?token=wrong")
-// 	if !bytes.Equal(proto.Status(resp), []byte("403")) {
-// 		t.Error("Wrong token should returns status 403:", string(proto.Status(resp)))
-// 	}
+	// Wrong tokens forbidden
+	rep, _ = client.Get(server.URL + "/secure?token=wrong")
+	resp, _ = httputil.DumpResponse(rep, true)
+	if !bytes.Equal(proto.Status(resp), []byte("403")) {
+		t.Error("Wrong token should returns status 403:", string(proto.Status(resp)))
+	}
 
-// 	wg.Wait()
-// }
+	wg.Wait()
+}
 
-// func TestEchoMiddleware(t *testing.T) {
-// 	wg := new(sync.WaitGroup)
+func TestEchoMiddleware(t *testing.T) {
+	wg := new(sync.WaitGroup)
 
-// 	from := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-// 		w.Header().Set("Env", "prod")
-// 		w.Header().Set("RequestPath", r.URL.Path)
-// 		wg.Done()
-// 	}))
-// 	defer from.Close()
+	from := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Env", "prod")
+		w.Header().Set("RequestPath", r.URL.Path)
+		wg.Done()
+	}))
+	defer from.Close()
 
-// 	to := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-// 		w.Header().Set("Env", "test")
-// 		w.Header().Set("RequestPath", r.URL.Path)
-// 		wg.Done()
-// 	}))
-// 	defer to.Close()
+	to := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Env", "test")
+		w.Header().Set("RequestPath", r.URL.Path)
+		wg.Done()
+	}))
+	defer to.Close()
 
-// 	quit := make(chan int)
+	// Catch traffic from one service
+	fromAddr := strings.Replace(from.Listener.Addr().String(), "[::]", "127.0.0.1", -1)
+	conf := RAWInputConfig{
+		Engine:        capture.EnginePcap,
+		Expire:        testRawExpire,
+		Protocol:      ProtocolHTTP,
+		TrackResponse: true,
+	}
+	input := NewRAWInput(fromAddr, conf)
 
-// 	// Catch traffic from one service
-// 	fromAddr := strings.Replace(from.Listener.Addr().String(), "[::]", "127.0.0.1", -1)
-// 	conf := RAWInputConfig{
-// 		engine:        capture.EnginePcap,
-// 		expire:        testRawExpire,
-// 		protocol:      ProtocolHTTP,
-// 		trackResponse: true,
-// 	}
-// 	input := NewRAWInput(fromAddr, conf)
+	// And redirect to another
+	output := NewHTTPOutput(to.URL, &HTTPOutputConfig{})
 
-// 	// And redirect to another
-// 	output := NewHTTPOutput(to.URL, &HTTPOutputConfig{Debug: false})
+	plugins := &InOutPlugins{
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{output},
+	}
+	plugins.All = append(plugins.All, input, output)
 
-// 	plugins := &InOutPlugins{
-// 		Inputs:  []io.Reader{input},
-// 		Outputs: []io.Writer{output},
-// 	}
-// 	plugins.All = append(plugins.All, input, output)
+	// Start Gor
+	emitter := NewEmitter()
+	emitter.Start(plugins, "echo -n && GOR_TEST=true && ./examples/middleware/echo.sh")
 
-// 	// Start Gor
-// 	emitter := NewEmitter(quit)
-// 	go emitter.Start(plugins, "echo -n && GOR_TEST=true && ./examples/middleware/echo.sh")
+	// Wait till middleware initialization
+	time.Sleep(100 * time.Millisecond)
 
-// 	// Wait till middleware initialization
-// 	time.Sleep(100 * time.Millisecond)
+	// Should receive 2 requests from original + 2 from replayed
+	client := NewHTTPClient(output.(*HTTPOutput).config).Client
 
-// 	// Should receive 2 requests from original + 2 from replayed
-// 	client := NewHTTPClient(from.URL, &HTTPClientConfig{Debug: false})
+	for i := 0; i < 10; i++ {
+		wg.Add(2)
+		// Request should be echoed
+		client.Get(to.URL + "/a")
+		time.Sleep(5 * time.Millisecond)
+		client.Get(to.URL + "/b")
+		time.Sleep(5 * time.Millisecond)
+	}
 
-// 	for i := 0; i < 10; i++ {
-// 		wg.Add(2)
-// 		// Request should be echoed
-// 		client.Get("/a")
-// 		time.Sleep(5 * time.Millisecond)
-// 		client.Get("/b")
-// 		time.Sleep(5 * time.Millisecond)
-// 	}
+	wg.Wait()
+	emitter.Close()
+}
 
-// 	wg.Wait()
-// 	emitter.Close()
-// }
+func TestTokenMiddleware(t *testing.T) {
+	var resp, token []byte
 
-// func TestTokenMiddleware(t *testing.T) {
-// 	var resp, token []byte
+	wg := new(sync.WaitGroup)
 
-// 	wg := new(sync.WaitGroup)
+	from := NewFakeSecureService(wg, func(path string, status int, tok []byte) {
+		time.Sleep(10 * time.Millisecond)
+	})
+	defer from.Close()
 
-// 	from := NewFakeSecureService(wg, func(path string, status int, tok []byte) {
-// 		time.Sleep(10 * time.Millisecond)
-// 	})
-// 	defer from.Close()
+	to := NewFakeSecureService(wg, func(path string, status int, tok []byte) {
+		switch path {
+		case "/secure":
+			if status != 202 {
+				t.Error("Server should receive valid rewritten token")
+			}
+		}
 
-// 	to := NewFakeSecureService(wg, func(path string, status int, tok []byte) {
-// 		switch path {
-// 		case "/secure":
-// 			if status != 202 {
-// 				t.Error("Server should receive valid rewritten token")
-// 			}
-// 		}
+		time.Sleep(10 * time.Millisecond)
+	})
+	defer to.Close()
 
-// 		time.Sleep(10 * time.Millisecond)
-// 	})
-// 	defer to.Close()
+	Settings.Middleware = "echo -n && GOR_TEST=true && go run ./examples/middleware/token_modifier.go"
 
-// 	quit := make(chan int)
+	fromAddr := strings.Replace(from.Listener.Addr().String(), "[::]", "127.0.0.1", -1)
+	conf := RAWInputConfig{
+		Engine:        capture.EnginePcap,
+		Expire:        testRawExpire,
+		Protocol:      ProtocolHTTP,
+		TrackResponse: true,
+	}
+	// Catch traffic from one service
+	input := NewRAWInput(fromAddr, conf)
 
-// 	Settings.middleware = "echo -n && GOR_TEST=true && go run ./examples/middleware/token_modifier.go"
+	// And redirect to another
+	output := NewHTTPOutput(to.URL, &HTTPOutputConfig{})
 
-// 	fromAddr := strings.Replace(from.Listener.Addr().String(), "[::]", "127.0.0.1", -1)
-// 	conf := RAWInputConfig{
-// 		engine:        capture.EnginePcap,
-// 		expire:        testRawExpire,
-// 		protocol:      ProtocolHTTP,
-// 		trackResponse: true,
-// 	}
-// 	// Catch traffic from one service
-// 	input := NewRAWInput(fromAddr, conf)
+	plugins := &InOutPlugins{
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{output},
+	}
+	plugins.All = append(plugins.All, input, output)
 
-// 	// And redirect to another
-// 	output := NewHTTPOutput(to.URL, &HTTPOutputConfig{Debug: true})
+	// Start Gor
+	emitter := NewEmitter()
+	emitter.Start(plugins, Settings.Middleware)
 
-// 	plugins := &InOutPlugins{
-// 		Inputs:  []io.Reader{input},
-// 		Outputs: []io.Writer{output},
-// 	}
-// 	plugins.All = append(plugins.All, input, output)
+	// Should receive 2 requests from original + 2 from replayed
+	wg.Add(2)
 
-// 	// Start Gor
-// 	emitter := NewEmitter(quit)
-// 	go emitter.Start(plugins, Settings.middleware)
+	client := NewHTTPClient(&HTTPOutputConfig{}).Client
 
-// 	// Should receive 2 requests from original + 2 from replayed
-// 	wg.Add(2)
+	// Sending traffic to original service
+	rep, _ := client.Get(to.URL + "/token")
+	resp, _ = httputil.DumpResponse(rep, true)
+	token = proto.Body(resp)
 
-// 	client := NewHTTPClient(from.URL, &HTTPClientConfig{Debug: true})
+	rep, _ = client.Get(to.URL + "/secure?token=" + string(token))
+	resp, _ = httputil.DumpResponse(rep, true)
+	if !bytes.Equal(proto.Status(resp), []byte("202")) {
+		t.Error("Valid token should return 202:", proto.Status(resp))
+	}
 
-// 	// Sending traffic to original service
-// 	resp, _ = client.Get("/token")
-// 	token = proto.Body(resp)
-
-// 	resp, _ = client.Get("/secure?token=" + string(token))
-// 	if !bytes.Equal(proto.Status(resp), []byte("202")) {
-// 		t.Error("Valid token should return 202:", proto.Status(resp))
-// 	}
-
-// 	wg.Wait()
-// 	emitter.Close()
-// 	Settings.middleware = ""
-// }
+	wg.Wait()
+	emitter.Close()
+	Settings.Middleware = ""
+}

output_binary.go

@@ -1,7 +1,6 @@
 package main
 
 import (
-	"io"
 	"sync/atomic"
 	"time"
 
@@ -25,30 +24,27 @@ type BinaryOutput struct {
 	// alignment. atomic.* functions crash on 32bit machines if operand is not
 	// aligned at 64bit. See https://github.com/golang/go/issues/599
 	activeWorkers int64
-
-	address string
-	queue   chan []byte
-
-	responses chan response
-
-	needWorker chan int
-
-	config *BinaryOutputConfig
-
-	queueStats *GorStat
+	address       string
+	queue         chan *Message
+	responses     chan response
+	needWorker    chan int
+	quit          chan struct{}
+	config        *BinaryOutputConfig
+	queueStats    *GorStat
 }
 
 // NewBinaryOutput constructor for BinaryOutput
 // Initialize workers
-func NewBinaryOutput(address string, config *BinaryOutputConfig) io.Writer {
+func NewBinaryOutput(address string, config *BinaryOutputConfig) PluginReadWriter {
 	o := new(BinaryOutput)
 
 	o.address = address
 	o.config = config
 
-	o.queue = make(chan []byte, 1000)
+	o.queue = make(chan *Message, 1000)
 	o.responses = make(chan response, 1000)
 	o.needWorker = make(chan int, 1)
+	o.quit = make(chan struct{})
 
 	// Initial workers count
 	if o.config.Workers == 0 {
@@ -89,8 +85,8 @@ func (o *BinaryOutput) startWorker() {
 
 	for {
 		select {
-		case data := <-o.queue:
-			o.sendRequest(client, data)
+		case msg := <-o.queue:
+			o.sendRequest(client, msg)
 			deathCount = 0
 		case <-time.After(time.Millisecond * 100):
 			// When dynamic scaling enabled workers die after 2s of inactivity
@@ -113,15 +109,13 @@ func (o *BinaryOutput) startWorker() {
 	}
 }
 
-func (o *BinaryOutput) Write(data []byte) (n int, err error) {
-	if !isRequestPayload(data) {
-		return len(data), nil
+// PluginWrite writes a message tothis plugin
+func (o *BinaryOutput) PluginWrite(msg *Message) (n int, err error) {
+	if !isRequestPayload(msg.Meta) {
+		return len(msg.Data), nil
 	}
 
-	buf := make([]byte, len(data))
-	copy(buf, data)
-
-	o.queue <- buf
+	o.queue <- msg
 
 	if o.config.Workers == 0 {
 		workersCount := atomic.LoadInt64(&o.activeWorkers)
@@ -131,37 +125,33 @@ func (o *BinaryOutput) Write(data []byte) (n int, err error) {
 		}
 	}
 
-	return len(data), nil
+	return len(msg.Data) + len(msg.Meta), nil
 }
 
-func (o *BinaryOutput) Read(data []byte) (int, error) {
-	resp := <-o.responses
+// PluginRead reads a message from this plugin
+func (o *BinaryOutput) PluginRead() (*Message, error) {
+	var resp response
+	var msg Message
+	select {
+	case <-o.quit:
+		return nil, ErrorStopped
+	case resp = <-o.responses:
+	}
+	msg.Data = resp.payload
+	msg.Meta = payloadHeader(ReplayedResponsePayload, resp.uuid, resp.startedAt, resp.roundTripTime)
 
-	Debug(2, "[OUTPUT-TCP] Received response:", string(resp.payload))
-
-	header := payloadHeader(ReplayedResponsePayload, resp.uuid, resp.startedAt, resp.roundTripTime)
-	copy(data[0:len(header)], header)
-	copy(data[len(header):], resp.payload)
-
-	return len(resp.payload) + len(header), nil
+	return &msg, nil
 }
 
-func (o *BinaryOutput) sendRequest(client *TCPClient, request []byte) {
-	meta := payloadMeta(request)
-	if len(meta) < 2 {
+func (o *BinaryOutput) sendRequest(client *TCPClient, msg *Message) {
+	if !isRequestPayload(msg.Meta) {
 		return
 	}
 
-	if !isRequestPayload(request) {
-		return
-	}
-
-	uuid := meta[1]
-
-	body := payloadBody(request)
+	uuid := payloadID(msg.Meta)
 
 	start := time.Now()
-	resp, err := client.Send(body)
+	resp, err := client.Send(msg.Data)
 	stop := time.Now()
 
 	if err != nil {
@@ -176,3 +166,9 @@ func (o *BinaryOutput) sendRequest(client *TCPClient, request []byte) {
 func (o *BinaryOutput) String() string {
 	return "Binary output: " + o.address
 }
+
+// Close closes this plugin for reading
+func (o *BinaryOutput) Close() error {
+	close(o.quit)
+	return nil
+}

output_dummy.go

@@ -15,9 +15,15 @@ func NewDummyOutput() (di *DummyOutput) {
 	return
 }
 
-func (i *DummyOutput) Write(data []byte) (int, error) {
-	n, err := os.Stdout.Write(data)
-	os.Stdout.Write([]byte{'\n'})
+// PluginWrite writes message to this plugin
+func (i *DummyOutput) PluginWrite(msg *Message) (int, error) {
+	var n, nn int
+	var err error
+	n, err = os.Stdout.Write(msg.Meta)
+	nn, err = os.Stdout.Write(msg.Data)
+	n += nn
+	nn, err = os.Stdout.Write(payloadSeparatorAsBytes)
+	n += nn
 	return n, err
 }
 

output_file.go

@@ -195,10 +195,11 @@ func (o *FileOutput) updateName() {
 	o.Unlock()
 }
 
-func (o *FileOutput) Write(data []byte) (n int, err error) {
+// PluginWrite writes message to this plugin
+func (o *FileOutput) PluginWrite(msg *Message) (n int, err error) {
 	if o.requestPerFile {
 		o.Lock()
-		meta := payloadMeta(data)
+		meta := payloadMeta(msg.Meta)
 		o.currentID = meta[1]
 		o.payloadType = meta[0]
 		o.Unlock()
@@ -227,10 +228,12 @@ func (o *FileOutput) Write(data []byte) (n int, err error) {
 		o.QueueLength = 0
 	}
 
-	n, _ = o.writer.Write(data)
-	nSeparator, _ := o.writer.Write([]byte(payloadSeparator))
-
-	n += nSeparator
+	var nn int
+	n, err = o.writer.Write(msg.Meta)
+	nn, err = o.writer.Write(msg.Data)
+	n += nn
+	nn, err = o.writer.Write(payloadSeparatorAsBytes)
+	n += nn
 
 	o.totalFileSize += size.Size(n)
 	o.QueueLength++
@@ -239,14 +242,14 @@ func (o *FileOutput) Write(data []byte) (n int, err error) {
 		return n, errors.New("File output reached size limit")
 	}
 
-	return n, nil
+	return n, err
 }
 
 func (o *FileOutput) flush() {
 	// Don't exit on panic
 	defer func() {
 		if r := recover(); r != nil {
-			log.Println("PANIC while file flush: ", r, o, string(debug.Stack()))
+			Debug(0, "[OUTPUT-FILE] PANIC while file flush: ", r, o, string(debug.Stack()))
 		}
 	}()
 
@@ -263,7 +266,7 @@ func (o *FileOutput) flush() {
 		if stat, err := o.file.Stat(); err == nil {
 			o.chunkSize = int(stat.Size())
 		} else {
-			log.Println("Error accessing file sats", err)
+			Debug(0, "[OUTPUT-HTTP] error accessing file size", err)
 		}
 	}
 }

output_file_test.go

@@ -2,7 +2,6 @@ package main
 
 import (
 	"fmt"
-	"io"
 	"math/rand"
 	"os"
 	"reflect"
@@ -17,18 +16,17 @@ import (
 
 func TestFileOutput(t *testing.T) {
 	wg := new(sync.WaitGroup)
-	quit := make(chan int)
 
 	input := NewTestInput()
 	output := NewFileOutput("/tmp/test_requests.gor", &FileOutputConfig{FlushInterval: time.Minute, Append: true})
 
 	plugins := &InOutPlugins{
-		Inputs:  []io.Reader{input},
-		Outputs: []io.Writer{output},
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{output},
 	}
 	plugins.All = append(plugins.All, input, output)
 
-	emitter := NewEmitter(quit)
+	emitter := NewEmitter()
 	go emitter.Start(plugins, Settings.Middleware)
 
 	for i := 0; i < 100; i++ {
@@ -42,19 +40,18 @@ func TestFileOutput(t *testing.T) {
 
 	var counter int64
 	input2 := NewFileInput("/tmp/test_requests.gor", false)
-	output2 := NewTestOutput(func(data []byte) {
+	output2 := NewTestOutput(func(*Message) {
 		atomic.AddInt64(&counter, 1)
 		wg.Done()
 	})
 
 	plugins2 := &InOutPlugins{
-		Inputs:  []io.Reader{input2},
-		Outputs: []io.Writer{output2},
+		Inputs:  []PluginReader{input2},
+		Outputs: []PluginWriter{output2},
 	}
 	plugins2.All = append(plugins2.All, input2, output2)
 
-	quit2 := make(chan int)
-	emitter2 := NewEmitter(quit2)
+	emitter2 := NewEmitter()
 	go emitter2.Start(plugins2, Settings.Middleware)
 
 	wg.Wait()
@@ -91,16 +88,16 @@ func TestFileOutputMultipleFiles(t *testing.T) {
 		t.Error("Should not initialize file if no writes")
 	}
 
-	output.Write([]byte("1 1 1\r\ntest"))
+	output.PluginWrite(&Message{Meta: []byte("1 1 1\r\n"), Data: []byte("test")})
 	name1 := output.file.Name()
 
-	output.Write([]byte("1 1 1\r\ntest"))
+	output.PluginWrite(&Message{Meta: []byte("1 1 1\r\n"), Data: []byte("test")})
 	name2 := output.file.Name()
 
 	time.Sleep(time.Second)
 	output.updateName()
 
-	output.Write([]byte("1 1 1\r\ntest"))
+	output.PluginWrite(&Message{Meta: []byte("1 1 1\r\n"), Data: []byte("test")})
 	name3 := output.file.Name()
 
 	if name2 != name1 {
@@ -122,16 +119,16 @@ func TestFileOutputFilePerRequest(t *testing.T) {
 		t.Error("Should not initialize file if no writes")
 	}
 
-	output.Write([]byte("1 1 1\ntest"))
+	output.PluginWrite(&Message{Meta: []byte("1 1 1\r\n"), Data: []byte("test")})
 	name1 := output.file.Name()
 
-	output.Write([]byte("1 2 1\ntest"))
+	output.PluginWrite(&Message{Meta: []byte("1 2 1\r\n"), Data: []byte("test")})
 	name2 := output.file.Name()
 
 	time.Sleep(time.Second)
 	output.updateName()
 
-	output.Write([]byte("1 3 1\ntest"))
+	output.PluginWrite(&Message{Meta: []byte("1 3 1\r\n"), Data: []byte("test")})
 	name3 := output.file.Name()
 
 	if name3 == name2 || name2 == name1 || name3 == name1 {
@@ -151,7 +148,7 @@ func TestFileOutputCompression(t *testing.T) {
 	}
 
 	for i := 0; i < 1000; i++ {
-		output.Write([]byte("1 1 1\r\ntest"))
+		output.PluginWrite(&Message{Meta: []byte("1 1 1\r\n"), Data: []byte("test")})
 	}
 
 	name := output.file.Name()
@@ -210,15 +207,15 @@ func TestFileOutputAppendQueueLimitOverflow(t *testing.T) {
 
 	output := NewFileOutput(name, &FileOutputConfig{Append: false, FlushInterval: time.Minute, QueueLimit: 2})
 
-	output.Write([]byte("1 1 1\r\ntest"))
+	output.PluginWrite(&Message{Meta: []byte("1 1 1\r\n"), Data: []byte("test")})
 	name1 := output.file.Name()
 
-	output.Write([]byte("1 1 1\r\ntest"))
+	output.PluginWrite(&Message{Meta: []byte("1 1 1\r\n"), Data: []byte("test")})
 	name2 := output.file.Name()
 
 	output.updateName()
 
-	output.Write([]byte("1 1 1\r\ntest"))
+	output.PluginWrite(&Message{Meta: []byte("1 1 1\r\n"), Data: []byte("test")})
 	name3 := output.file.Name()
 
 	if name2 != name1 || name1 != fmt.Sprintf("/tmp/%d_0", rnd) {
@@ -239,15 +236,15 @@ func TestFileOutputAppendQueueLimitNoOverflow(t *testing.T) {
 
 	output := NewFileOutput(name, &FileOutputConfig{Append: false, FlushInterval: time.Minute, QueueLimit: 3})
 
-	output.Write([]byte("1 1 1\r\ntest"))
+	output.PluginWrite(&Message{Meta: []byte("1 1 1\r\n"), Data: []byte("test")})
 	name1 := output.file.Name()
 
-	output.Write([]byte("1 1 1\r\ntest"))
+	output.PluginWrite(&Message{Meta: []byte("1 1 1\r\n"), Data: []byte("test")})
 	name2 := output.file.Name()
 
 	output.updateName()
 
-	output.Write([]byte("1 1 1\r\ntest"))
+	output.PluginWrite(&Message{Meta: []byte("1 1 1\r\n"), Data: []byte("test")})
 	name3 := output.file.Name()
 
 	if name2 != name1 || name1 != fmt.Sprintf("/tmp/%d_0", rnd) {
@@ -268,15 +265,15 @@ func TestFileOutputAppendQueueLimitGzips(t *testing.T) {
 
 	output := NewFileOutput(name, &FileOutputConfig{Append: false, FlushInterval: time.Minute, QueueLimit: 2})
 
-	output.Write([]byte("1 1 1\r\ntest"))
+	output.PluginWrite(&Message{Meta: []byte("1 1 1\r\n"), Data: []byte("test")})
 	name1 := output.file.Name()
 
-	output.Write([]byte("1 1 1\r\ntest"))
+	output.PluginWrite(&Message{Meta: []byte("1 1 1\r\n"), Data: []byte("test")})
 	name2 := output.file.Name()
 
 	output.updateName()
 
-	output.Write([]byte("1 1 1\r\ntest"))
+	output.PluginWrite(&Message{Meta: []byte("1 1 1\r\n"), Data: []byte("test")})
 	name3 := output.file.Name()
 
 	if name2 != name1 || name1 != fmt.Sprintf("/tmp/%d_0.gz", rnd) {
@@ -311,16 +308,16 @@ func TestFileOutputAppendSizeLimitOverflow(t *testing.T) {
 
 	output := NewFileOutput(name, &FileOutputConfig{Append: false, FlushInterval: time.Minute, SizeLimit: size.Size(2 * messageSize)})
 
-	output.Write([]byte("1 1 1\r\ntest"))
+	output.PluginWrite(&Message{Meta: []byte("1 1 1\r\n"), Data: []byte("test")})
 	name1 := output.file.Name()
 
-	output.Write([]byte("1 1 1\r\ntest"))
+	output.PluginWrite(&Message{Meta: []byte("1 1 1\r\n"), Data: []byte("test")})
 	name2 := output.file.Name()
 
 	output.flush()
 	output.updateName()
 
-	output.Write([]byte("1 1 1\r\ntest"))
+	output.PluginWrite(&Message{Meta: []byte("1 1 1\r\n"), Data: []byte("test")})
 	name3 := output.file.Name()
 
 	if name2 != name1 || name1 != fmt.Sprintf("/tmp/%d_0", rnd) {

output_http.go

@@ -5,7 +5,6 @@ import (
 	"bytes"
 	"crypto/tls"
 	"fmt"
-	"io"
 	"log"
 	"math"
 	"net/http"
@@ -59,14 +58,14 @@ type HTTPOutput struct {
 	elasticSearch *ESPlugin
 	client        *HTTPClient
 	stopWorker    chan struct{}
-	queue         chan []byte
-	responses     chan response
+	queue         chan *Message
+	responses     chan *response
 	stop          chan bool // Channel used only to indicate goroutine should shutdown
 }
 
 // NewHTTPOutput constructor for HTTPOutput
 // Initialize workers
-func NewHTTPOutput(address string, config *HTTPOutputConfig) io.Writer {
+func NewHTTPOutput(address string, config *HTTPOutputConfig) PluginReadWriter {
 	o := new(HTTPOutput)
 	var err error
 	config.url, err = url.Parse(address)
@@ -110,8 +109,10 @@ func NewHTTPOutput(address string, config *HTTPOutputConfig) io.Writer {
 		o.queueStats = NewGorStat("output_http", o.config.StatsMs)
 	}
 
-	o.queue = make(chan []byte, o.config.QueueLen)
-	o.responses = make(chan response, o.config.QueueLen)
+	o.queue = make(chan *Message, o.config.QueueLen)
+	if o.config.TrackResponses {
+		o.responses = make(chan *response, o.config.QueueLen)
+	}
 	// it should not be buffered to avoid races
 	o.stopWorker = make(chan struct{})
 
@@ -160,23 +161,22 @@ func (o *HTTPOutput) startWorker() {
 		select {
 		case <-o.stopWorker:
 			return
-		case data := <-o.queue:
-			o.sendRequest(o.client, data)
+		case msg := <-o.queue:
+			o.sendRequest(o.client, msg)
 		}
 	}
 }
 
-func (o *HTTPOutput) Write(data []byte) (n int, err error) {
-	if !isRequestPayload(data) {
-		return len(data), nil
+// PluginWrite writes message to this plugin
+func (o *HTTPOutput) PluginWrite(msg *Message) (n int, err error) {
+	if !isRequestPayload(msg.Meta) {
+		return len(msg.Data), nil
 	}
 
-	buf := make([]byte, len(data))
-	copy(buf, data)
 	select {
 	case <-o.stop:
 		return 0, ErrorStopped
-	case o.queue <- buf:
+	case o.queue <- msg:
 	}
 
 	if o.config.Stats {
@@ -189,51 +189,51 @@ func (o *HTTPOutput) Write(data []byte) (n int, err error) {
 			atomic.AddInt32(&o.activeWorkers, 1)
 		}
 	}
-	return len(data), nil
+	return len(msg.Data) + len(msg.Meta), nil
 }
 
-func (o *HTTPOutput) Read(data []byte) (int, error) {
-	var resp response
+// PluginRead reads message from this plugin
+func (o *HTTPOutput) PluginRead() (*Message, error) {
+	if !o.config.TrackResponses {
+		return nil, ErrorStopped
+	}
+	var resp *response
+	var msg Message
 	select {
 	case <-o.stop:
-		return 0, ErrorStopped
+		return nil, ErrorStopped
 	case resp = <-o.responses:
+		msg.Data = resp.payload
 	}
 
-	header := payloadHeader(ReplayedResponsePayload, resp.uuid, resp.roundTripTime, resp.startedAt)
-	n := copy(data, header)
-	if len(data) > len(header) {
-		n += copy(data[len(header):], resp.payload)
-	}
-	dis := len(header) + len(data) - n
-	if dis > 0 {
-		Debug(2, fmt.Sprintf("[OUTPUT-HTTP] %dB discarded increase copy buffer size", dis))
-	}
+	msg.Meta = payloadHeader(ReplayedResponsePayload, resp.uuid, resp.roundTripTime, resp.startedAt)
 
-	return n, nil
+	return &msg, nil
 }
 
-func (o *HTTPOutput) sendRequest(client *HTTPClient, request []byte) {
-	if !isRequestPayload(request) {
+func (o *HTTPOutput) sendRequest(client *HTTPClient, msg *Message) {
+	if !isRequestPayload(msg.Meta) {
 		return
 	}
-	uuid := payloadID(request)
-	body := payloadBody(request)
+	uuid := payloadID(msg.Meta)
 	start := time.Now()
-	resp, err := client.Send(body)
+	resp, err := client.Send(msg.Data)
 	stop := time.Now()
 
 	if err != nil {
 		Debug(1, fmt.Sprintf("[HTTP-OUTPUT] error when sending: %q", err))
 		return
 	}
+	if resp == nil {
+		return
+	}
 
 	if o.config.TrackResponses {
-		o.responses <- response{resp, uuid, start.UnixNano(), stop.UnixNano() - start.UnixNano()}
+		o.responses <- &response{resp, uuid, start.UnixNano(), stop.UnixNano() - start.UnixNano()}
 	}
 
 	if o.elasticSearch != nil {
-		o.elasticSearch.ResponseAnalyze(request, resp, start, stop)
+		o.elasticSearch.ResponseAnalyze(msg.Data, resp, start, stop)
 	}
 }
 
@@ -300,8 +300,15 @@ func (c *HTTPClient) Send(data []byte) ([]byte, error) {
 	if !c.config.OriginalHost {
 		req.Host = c.config.url.Host
 	}
-
-	req.URL = c.config.url
+	
+	// fix #862
+	if c.config.url.Path == "" && c.config.url.RawQuery == "" {
+		req.URL.Scheme = c.config.url.Scheme
+		req.URL.Host = c.config.url.Host
+	} else {
+		req.URL = c.config.url
+	}
+	
 	// force connection to not be closed, which can affect the global client
 	req.Close = false
 	// it's an error if this is not equal to empty string

output_http_test.go

@@ -1,7 +1,6 @@
 package main
 
 import (
-	"io"
 	"io/ioutil"
 	"net/http"
 	"net/http/httptest"
@@ -12,7 +11,6 @@ import (
 
 func TestHTTPOutput(t *testing.T) {
 	wg := new(sync.WaitGroup)
-	quit := make(chan int)
 
 	input := NewTestInput()
 
@@ -38,27 +36,27 @@ func TestHTTPOutput(t *testing.T) {
 	}))
 	defer server.Close()
 
-	headers := HTTPHeaders{HTTPHeader{"User-Agent", "Gor"}}
+	headers := HTTPHeaders{httpHeader{"User-Agent", "Gor"}}
 	methods := HTTPMethods{[]byte("GET"), []byte("PUT"), []byte("POST")}
 	Settings.ModifierConfig = HTTPModifierConfig{Headers: headers, Methods: methods}
 
-	httpOutput := NewHTTPOutput(server.URL, &HTTPOutputConfig{TrackResponses: true})
-	output := NewTestOutput(func(data []byte) {
+	httpOutput := NewHTTPOutput(server.URL, &HTTPOutputConfig{TrackResponses: false})
+	output := NewTestOutput(func(*Message) {
 		wg.Done()
 	})
 
 	plugins := &InOutPlugins{
-		Inputs:  []io.Reader{input},
-		Outputs: []io.Writer{httpOutput, output},
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{httpOutput, output},
 	}
 	plugins.All = append(plugins.All, input, output, httpOutput)
 
-	emitter := NewEmitter(quit)
+	emitter := NewEmitter()
 	go emitter.Start(plugins, Settings.Middleware)
 
 	for i := 0; i < 10; i++ {
-		// 2 http-output, 2 - test output request, 2 - test output http response
-		wg.Add(6) // OPTIONS should be ignored
+		// 2 http-output, 2 - test output request
+		wg.Add(4) // OPTIONS should be ignored
 		input.EmitPOST()
 		input.EmitOPTIONS()
 		input.EmitGET()
@@ -72,7 +70,6 @@ func TestHTTPOutput(t *testing.T) {
 
 func TestHTTPOutputKeepOriginalHost(t *testing.T) {
 	wg := new(sync.WaitGroup)
-	quit := make(chan int)
 
 	input := NewTestInput()
 
@@ -85,18 +82,18 @@ func TestHTTPOutputKeepOriginalHost(t *testing.T) {
 	}))
 	defer server.Close()
 
-	headers := HTTPHeaders{HTTPHeader{"Host", "custom-host.com"}}
+	headers := HTTPHeaders{httpHeader{"Host", "custom-host.com"}}
 	Settings.ModifierConfig = HTTPModifierConfig{Headers: headers}
 
 	output := NewHTTPOutput(server.URL, &HTTPOutputConfig{OriginalHost: true, SkipVerify: true})
 
 	plugins := &InOutPlugins{
-		Inputs:  []io.Reader{input},
-		Outputs: []io.Writer{output},
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{output},
 	}
 	plugins.All = append(plugins.All, input, output)
 
-	emitter := NewEmitter(quit)
+	emitter := NewEmitter()
 	go emitter.Start(plugins, Settings.Middleware)
 
 	wg.Add(1)
@@ -109,7 +106,6 @@ func TestHTTPOutputKeepOriginalHost(t *testing.T) {
 
 func TestHTTPOutputSSL(t *testing.T) {
 	wg := new(sync.WaitGroup)
-	quit := make(chan int)
 
 	// Origing and Replay server initialization
 	server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
@@ -120,12 +116,12 @@ func TestHTTPOutputSSL(t *testing.T) {
 	output := NewHTTPOutput(server.URL, &HTTPOutputConfig{SkipVerify: true})
 
 	plugins := &InOutPlugins{
-		Inputs:  []io.Reader{input},
-		Outputs: []io.Writer{output},
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{output},
 	}
 	plugins.All = append(plugins.All, input, output)
 
-	emitter := NewEmitter(quit)
+	emitter := NewEmitter()
 	go emitter.Start(plugins, Settings.Middleware)
 
 	wg.Add(2)
@@ -139,7 +135,6 @@ func TestHTTPOutputSSL(t *testing.T) {
 
 func TestHTTPOutputSessions(t *testing.T) {
 	wg := new(sync.WaitGroup)
-	quit := make(chan int)
 
 	input := NewTestInput()
 	input.skipHeader = true
@@ -150,25 +145,27 @@ func TestHTTPOutputSessions(t *testing.T) {
 	defer server.Close()
 
 	Settings.RecognizeTCPSessions = true
+	Settings.SplitOutput = true
 	output := NewHTTPOutput(server.URL, &HTTPOutputConfig{})
 
 	plugins := &InOutPlugins{
-		Inputs:  []io.Reader{input},
-		Outputs: []io.Writer{output},
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{output},
 	}
-	emitter := NewEmitter(quit)
+	plugins.All = append(plugins.All, input, output)
+	emitter := NewEmitter()
 	go emitter.Start(plugins, Settings.Middleware)
 
 	uuid1 := []byte("1234567890123456789a0000")
 	uuid2 := []byte("1234567890123456789d0000")
 
-	for i := 0; i < 100; i++ {
+	for i := 0; i < 10; i++ {
 		wg.Add(1) // OPTIONS should be ignored
 		copy(uuid1[20:], randByte(4))
 		input.EmitBytes([]byte("1 " + string(uuid1) + " 1\n" + "GET / HTTP/1.1\r\n\r\n"))
 	}
 
-	for i := 0; i < 100; i++ {
+	for i := 0; i < 10; i++ {
 		wg.Add(1) // OPTIONS should be ignored
 		copy(uuid2[20:], randByte(4))
 		input.EmitBytes([]byte("1 " + string(uuid2) + " 1\n" + "GET / HTTP/1.1\r\n\r\n"))
@@ -179,11 +176,11 @@ func TestHTTPOutputSessions(t *testing.T) {
 	emitter.Close()
 
 	Settings.RecognizeTCPSessions = false
+	Settings.SplitOutput = false
 }
 
 func BenchmarkHTTPOutput(b *testing.B) {
 	wg := new(sync.WaitGroup)
-	quit := make(chan int)
 
 	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		wg.Done()
@@ -194,12 +191,12 @@ func BenchmarkHTTPOutput(b *testing.B) {
 	output := NewHTTPOutput(server.URL, &HTTPOutputConfig{WorkersMax: 1})
 
 	plugins := &InOutPlugins{
-		Inputs:  []io.Reader{input},
-		Outputs: []io.Writer{output},
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{output},
 	}
 	plugins.All = append(plugins.All, input, output)
 
-	emitter := NewEmitter(quit)
+	emitter := NewEmitter()
 	go emitter.Start(plugins, Settings.Middleware)
 
 	for i := 0; i < b.N; i++ {
@@ -213,7 +210,6 @@ func BenchmarkHTTPOutput(b *testing.B) {
 
 func BenchmarkHTTPOutputTLS(b *testing.B) {
 	wg := new(sync.WaitGroup)
-	quit := make(chan int)
 
 	server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		wg.Done()
@@ -224,12 +220,12 @@ func BenchmarkHTTPOutputTLS(b *testing.B) {
 	output := NewHTTPOutput(server.URL, &HTTPOutputConfig{SkipVerify: true, WorkersMax: 1})
 
 	plugins := &InOutPlugins{
-		Inputs:  []io.Reader{input},
-		Outputs: []io.Writer{output},
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{output},
 	}
 	plugins.All = append(plugins.All, input, output)
 
-	emitter := NewEmitter(quit)
+	emitter := NewEmitter()
 	go emitter.Start(plugins, Settings.Middleware)
 
 	for i := 0; i < b.N; i++ {

output_kafka.go

@@ -2,7 +2,6 @@ package main
 
 import (
 	"encoding/json"
-	"io"
 	"log"
 	"strings"
 	"time"
@@ -23,13 +22,8 @@ type KafkaOutput struct {
 // KafkaOutputFrequency in milliseconds
 const KafkaOutputFrequency = 500
 
-// NewKafkaOutput creates instance of kafka producer client.
-func NewKafkaOutput(address string, config *OutputKafkaConfig) io.Writer {
-	return NewKafkaOutputWithTLS(address, config, nil)
-}
-
-// NewKafkaOutputWithTLS creates instance of kafka producer client.
-func NewKafkaOutputWithTLS(address string, config *OutputKafkaConfig, tlsConfig *KafkaTLSConfig) io.Writer {
+// NewKafkaOutput creates instance of kafka producer client  with TLS config
+func NewKafkaOutput(address string, config *OutputKafkaConfig, tlsConfig *KafkaTLSConfig) PluginWriter {
 	c := NewKafkaConfig(tlsConfig)
 
 	var producer sarama.AsyncProducer
@@ -68,20 +62,21 @@ func (o *KafkaOutput) ErrorHandler() {
 	}
 }
 
-func (o *KafkaOutput) Write(data []byte) (n int, err error) {
+// PluginWrite writes a message to this plugin
+func (o *KafkaOutput) PluginWrite(msg *Message) (n int, err error) {
 	var message sarama.StringEncoder
 
 	if !o.config.UseJSON {
-		message = sarama.StringEncoder(byteutils.SliceToString(data))
+		message = sarama.StringEncoder(byteutils.SliceToString(msg.Meta) + byteutils.SliceToString(msg.Data))
 	} else {
-		mimeHeader := proto.ParseHeaders(data)
-		var header map[string]string
+		mimeHeader := proto.ParseHeaders(msg.Data)
+		header := make(map[string]string)
 		for k, v := range mimeHeader {
 			header[k] = strings.Join(v, ", ")
 		}
 
-		meta := payloadMeta(data)
-		req := payloadBody(data)
+		meta := payloadMeta(msg.Meta)
+		req := msg.Data
 
 		kafkaMessage := KafkaMessage{
 			ReqURL:     byteutils.SliceToString(proto.Path(req)),

output_kafka_test.go

@@ -17,16 +17,16 @@ func TestOutputKafkaRAW(t *testing.T) {
 		producer: producer,
 		Topic:    "test",
 		UseJSON:  false,
-	})
+	}, nil)
 
-	output.Write([]byte("1 2 3\nGET / HTTP1.1\r\nHeader: 1\r\n\r\n"))
+	output.PluginWrite(&Message{Meta: []byte("1 2 3\n"), Data: []byte("GET / HTTP1.1\r\nHeader: 1\r\n\r\n")})
 
 	resp := <-producer.Successes()
 
 	data, _ := resp.Value.Encode()
 
 	if string(data) != "1 2 3\nGET / HTTP1.1\r\nHeader: 1\r\n\r\n" {
-		t.Error("Message not properly encoded: ", string(data))
+		t.Errorf("Message not properly encoded: %q", data)
 	}
 }
 
@@ -40,9 +40,9 @@ func TestOutputKafkaJSON(t *testing.T) {
 		producer: producer,
 		Topic:    "test",
 		UseJSON:  true,
-	})
+	}, nil)
 
-	output.Write([]byte("1 2 3\nGET / HTTP1.1\r\nHeader: 1\r\n\r\n"))
+	output.PluginWrite(&Message{Meta: []byte("1 2 3\n"), Data: []byte("GET / HTTP1.1\r\nHeader: 1\r\n\r\n")})
 
 	resp := <-producer.Successes()
 

output_null.go

@@ -9,8 +9,9 @@ func NewNullOutput() (o *NullOutput) {
 	return new(NullOutput)
 }
 
-func (o *NullOutput) Write(data []byte) (int, error) {
-	return len(data), nil
+// PluginWrite writes message to this plugin
+func (o *NullOutput) PluginWrite(msg *Message) (int, error) {
+	return len(msg.Data) + len(msg.Meta), nil
 }
 
 func (o *NullOutput) String() string {

output_s3.go

@@ -67,8 +67,9 @@ func (o *S3Output) connect() {
 	}
 }
 
-func (o *S3Output) Write(data []byte) (n int, err error) {
-	return o.buffer.Write(data)
+// PluginWrite writes message to this plugin
+func (o *S3Output) PluginWrite(msg *Message) (n int, err error) {
+	return o.buffer.PluginWrite(msg)
 }
 
 func (o *S3Output) String() string {
@@ -107,22 +108,23 @@ func (o *S3Output) onBufferUpdate(path string) {
 	idx := getFileIndex(path)
 	bucket, key := o.keyPath(idx)
 
-	file, _ := os.Open(path)
-	// reader := bufio.NewReader(file)
+	file, err := os.Open(path)
+	if err != nil {
+		Debug(0, fmt.Sprintf("[S3 Output] Failed to open file %q. err: %q", path, err))
+		return
+	}
+	defer os.Remove(path)
 
-	_, err := svc.PutObject(&s3.PutObjectInput{
+	_, err = svc.PutObject(&s3.PutObjectInput{
 		Body:   file,
 		Bucket: aws.String(bucket),
 		Key:    aws.String(key),
 	})
 	if err != nil {
-		log.Printf("[S3 Output] Failed to upload data to %s/%s, %s\n", bucket, key, err)
-		os.Remove(path)
+		Debug(0, fmt.Sprintf("[S3 Output] Failed to upload data to %q/%q, %q", bucket, key, err))
 		return
 	}
 
-	os.Remove(path)
-
 	if o.closeCh != nil {
 		o.closeCh <- struct{}{}
 	}

output_tcp.go

@@ -4,7 +4,6 @@ import (
 	"crypto/tls"
 	"fmt"
 	"hash/fnv"
-	"io"
 	"net"
 	"time"
 )
@@ -15,7 +14,7 @@ import (
 type TCPOutput struct {
 	address     string
 	limit       int
-	buf         []chan []byte
+	buf         []chan *Message
 	bufStats    *GorStat
 	config      *TCPOutputConfig
 	workerIndex uint32
@@ -31,7 +30,7 @@ type TCPOutputConfig struct {
 
 // NewTCPOutput constructor for TCPOutput
 // Initialize X workers which hold keep-alive connection
-func NewTCPOutput(address string, config *TCPOutputConfig) io.Writer {
+func NewTCPOutput(address string, config *TCPOutputConfig) PluginWriter {
 	o := new(TCPOutput)
 
 	o.address = address
@@ -42,9 +41,9 @@ func NewTCPOutput(address string, config *TCPOutputConfig) io.Writer {
 	}
 
 	// create X buffers and send the buffer index to the worker
-	o.buf = make([]chan []byte, o.config.Workers)
+	o.buf = make([]chan *Message, o.config.Workers)
 	for i := 0; i < o.config.Workers; i++ {
-		o.buf[i] = make(chan []byte, 100)
+		o.buf[i] = make(chan *Message, 100)
 		go o.worker(i)
 	}
 
@@ -73,14 +72,16 @@ func (o *TCPOutput) worker(bufferIndex int) {
 	defer conn.Close()
 
 	for {
-		data := <-o.buf[bufferIndex]
-		if _, err = conn.Write(data); err == nil {
-			_, err = conn.Write([]byte(payloadSeparator))
+		msg := <-o.buf[bufferIndex]
+		if _, err = conn.Write(msg.Meta); err == nil {
+			if _, err = conn.Write(msg.Data); err == nil {
+				_, err = conn.Write(payloadSeparatorAsBytes)
+			}
 		}
 
 		if err != nil {
 			Debug(2, "INFO: TCP output connection closed, reconnecting")
-			o.buf[bufferIndex] <- data
+			o.buf[bufferIndex] <- msg
 			go o.worker(bufferIndex)
 			break
 		}
@@ -99,23 +100,20 @@ func (o *TCPOutput) getBufferIndex(data []byte) int {
 
 }
 
-func (o *TCPOutput) Write(data []byte) (n int, err error) {
-	if !isOriginPayload(data) {
-		return len(data), nil
+// PluginWrite writes message to this plugin
+func (o *TCPOutput) PluginWrite(msg *Message) (n int, err error) {
+	if !isOriginPayload(msg.Meta) {
+		return len(msg.Data), nil
 	}
 
-	// We have to copy, because sending data in multiple threads
-	newBuf := make([]byte, len(data))
-	copy(newBuf, data)
-
-	bufferIndex := o.getBufferIndex(data)
-	o.buf[bufferIndex] <- newBuf
+	bufferIndex := o.getBufferIndex(msg.Data)
+	o.buf[bufferIndex] <- msg
 
 	if Settings.OutputTCPStats {
 		o.bufStats.Write(len(o.buf[bufferIndex]))
 	}
 
-	return len(data), nil
+	return len(msg.Data) + len(msg.Meta), nil
 }
 
 func (o *TCPOutput) connect(address string) (conn net.Conn, err error) {

output_tcp_test.go

@@ -2,7 +2,6 @@ package main
 
 import (
 	"bufio"
-	"io"
 	"log"
 	"net"
 	"sync"
@@ -11,9 +10,7 @@ import (
 )
 
 func TestTCPOutput(t *testing.T) {
-	Settings.Verbose = 2
 	wg := new(sync.WaitGroup)
-	quit := make(chan int)
 
 	listener := startTCP(func(data []byte) {
 		wg.Done()
@@ -22,12 +19,11 @@ func TestTCPOutput(t *testing.T) {
 	output := NewTCPOutput(listener.Addr().String(), &TCPOutputConfig{Workers: 10})
 
 	plugins := &InOutPlugins{
-		Inputs:  []io.Reader{input},
-		Outputs: []io.Writer{output},
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{output},
 	}
-	plugins.All = append(plugins.All, input, output)
 
-	emitter := NewEmitter(quit)
+	emitter := NewEmitter()
 	go emitter.Start(plugins, Settings.Middleware)
 
 	for i := 0; i < 10; i++ {
@@ -68,7 +64,6 @@ func startTCP(cb func([]byte)) net.Listener {
 
 func BenchmarkTCPOutput(b *testing.B) {
 	wg := new(sync.WaitGroup)
-	quit := make(chan int)
 
 	listener := startTCP(func(data []byte) {
 		wg.Done()
@@ -82,12 +77,11 @@ func BenchmarkTCPOutput(b *testing.B) {
 	output := NewTCPOutput(listener.Addr().String(), &TCPOutputConfig{Workers: 10})
 
 	plugins := &InOutPlugins{
-		Inputs:  []io.Reader{input},
-		Outputs: []io.Writer{output},
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{output},
 	}
-	plugins.All = append(plugins.All, input, output)
 
-	emitter := NewEmitter(quit)
+	emitter := NewEmitter()
 	// avoid counting above initialization
 	b.ResetTimer()
 	go emitter.Start(plugins, Settings.Middleware)
@@ -109,7 +103,7 @@ func TestStickyDisable(t *testing.T) {
 
 func TestBufferDistribution(t *testing.T) {
 	numberOfWorkers := 10
-	numberOfMessages := 1000000
+	numberOfMessages := 10000
 	percentDistributionErrorRange := 20
 
 	buffer := make([]int, numberOfWorkers)

plugins.go

@@ -1,15 +1,36 @@
 package main
 
 import (
-	"io"
 	"reflect"
 	"strings"
 )
 
+// Message represents data accross plugins
+type Message struct {
+	Meta []byte // metadata
+	Data []byte // actual data
+}
+
+// PluginReader is an interface for input plugins
+type PluginReader interface {
+	PluginRead() (msg *Message, err error)
+}
+
+// PluginWriter is an interface for output plugins
+type PluginWriter interface {
+	PluginWrite(msg *Message) (n int, err error)
+}
+
+// PluginReadWriter is an interface for plugins that support reading and writing
+type PluginReadWriter interface {
+	PluginReader
+	PluginWriter
+}
+
 // InOutPlugins struct for holding references to plugins
 type InOutPlugins struct {
-	Inputs  []io.Reader
-	Outputs []io.Writer
+	Inputs  []PluginReader
+	Outputs []PluginWriter
 	All     []interface{}
 }
 
@@ -48,27 +69,21 @@ func (plugins *InOutPlugins) registerPlugin(constructor interface{}, options ...
 
 	// Calling our constructor with list of given options
 	plugin := vc.Call(vo)[0].Interface()
-	pluginWrapper := plugin
 
 	if limit != "" {
-		pluginWrapper = NewLimiter(plugin, limit)
-	} else {
-		pluginWrapper = plugin
+		plugin = NewLimiter(plugin, limit)
 	}
 
-	_, isR := plugin.(io.Reader)
-	_, isW := plugin.(io.Writer)
-
 	// Some of the output can be Readers as well because return responses
-	if isR && !isW {
-		plugins.Inputs = append(plugins.Inputs, pluginWrapper.(io.Reader))
+	if r, ok := plugin.(PluginReader); ok {
+		plugins.Inputs = append(plugins.Inputs, r)
 	}
 
-	if isW {
-		plugins.Outputs = append(plugins.Outputs, pluginWrapper.(io.Writer))
+	if w, ok := plugin.(PluginWriter); ok {
+		plugins.Outputs = append(plugins.Outputs, w)
 	}
-
 	plugins.All = append(plugins.All, plugin)
+
 }
 
 // NewPlugins specify and initialize all available plugins

plugins_test.go

@@ -12,8 +12,8 @@ func TestPluginsRegistration(t *testing.T) {
 
 	plugins := NewPlugins()
 
-	if len(plugins.Inputs) != 2 {
-		t.Errorf("Should be 2 inputs %d", len(plugins.Inputs))
+	if len(plugins.Inputs) != 3 {
+		t.Errorf("Should be 3 inputs got %d", len(plugins.Inputs))
 	}
 
 	if _, ok := plugins.Inputs[0].(*DummyInput); !ok {

proto/proto.go

@@ -125,6 +125,11 @@ func ParseHeaders(p []byte) textproto.MIMEHeader {
 	if headerEnd > 1 {
 		p = p[:headerEnd]
 	}
+	return GetHeaders(p)
+}
+
+// GetHeaders returns mime headers from the payload
+func GetHeaders(p []byte) textproto.MIMEHeader {
 	reader := textproto.NewReader(bufio.NewReader(bytes.NewReader(p)))
 	mime, err := reader.ReadMIMEHeader()
 	if err != nil {
@@ -182,7 +187,7 @@ func DeleteHeader(payload, name []byte) []byte {
 // Body returns request/response body
 func Body(payload []byte) []byte {
 	pos := MIMEHeadersEndPos(payload)
-	if pos == -1 {
+	if pos == -1 || len(payload) <= pos {
 		return nil
 	}
 	return payload[pos:]
@@ -190,7 +195,7 @@ func Body(payload []byte) []byte {
 
 // Path takes payload and retuns request path: Split(firstLine, ' ')[1]
 func Path(payload []byte) []byte {
-	if !HasTitle(payload) {
+	if !HasRequestTitle(payload) {
 		return nil
 	}
 	start := bytes.IndexByte(payload, ' ') + 1
@@ -304,7 +309,12 @@ func Method(payload []byte) []byte {
 // Status returns response status.
 // It happens to be in same position as request payload path
 func Status(payload []byte) []byte {
-	return Path(payload)
+	if !HasResponseTitle(payload) {
+		return nil
+	}
+	start := bytes.IndexByte(payload, ' ') + 1
+	// status code are in range 100-600
+	return payload[start : start+3]
 }
 
 // Methods holds the http methods ordered in ascending order
@@ -317,8 +327,8 @@ var Methods = [...]string{
 const (
 	//MinRequestCount GET / HTTP/1.1\r\n
 	MinRequestCount = 16
-	// MinResponseCount HTTP/1.1 200 OK\r\n
-	MinResponseCount = 17
+	// MinResponseCount HTTP/1.1 200\r\n
+	MinResponseCount = 14
 	// VersionLen HTTP/1.1
 	VersionLen = 8
 )
@@ -341,17 +351,15 @@ func HasResponseTitle(payload []byte) bool {
 		return false
 	}
 	status, ok := atoI(payload[VersionLen+1:VersionLen+4], 10)
-	if !ok || s[VersionLen+4] != ' ' {
+	if !ok {
 		return false
 	}
-	statusText := http.StatusText(status)
-	if statusText == "" {
+	// only validate status codes mentioned in rfc2616.
+	if len(http.StatusText(status)) == 0 {
 		return false
 	}
-	if titleLen+len(CRLF) > len(s) {
-		return false
-	}
-	return s[VersionLen+5:titleLen] == statusText
+	// handle cases from #875
+	return payload[VersionLen+4] == ' ' || payload[VersionLen+4] == '\r'
 }
 
 // HasRequestTitle reports whether this payload has an HTTP/1 request title
@@ -435,14 +443,55 @@ func CheckChunked(buf []byte) (chunkEnd int) {
 	}
 }
 
+// Feedback is an interface used to provide feedback or store dummy data for future use
+type Feedback interface {
+	SetFeedback(interface{})
+	Feedback() interface{}
+}
+
+type feedback struct {
+	body     int // body index
+	hdrStart int
+	headers  textproto.MIMEHeader
+}
+
 // HasFullPayload reports if this http has full payloads
-func HasFullPayload(payload []byte) bool {
-	body := Body(payload)
-
-	// check for chunked transfer-encoding
-	header := Header(payload, []byte("Transfer-Encoding"))
-	if bytes.Contains(header, []byte("chunked")) {
+func HasFullPayload(data []byte, f Feedback) bool {
+	var feed *feedback
+	var ok bool
+	var body []byte
+	if f != nil {
+		feed, ok = f.Feedback().(*feedback)
+	}
+	if !ok {
+		feed = new(feedback)
+	}
+	if f != nil {
+		f.SetFeedback(feed)
+	}
+	if feed.hdrStart < 1 {
+		feed.hdrStart = MIMEHeadersStartPos(data)
+		if feed.hdrStart < 0 {
+			return false
+		}
+	}
+	if feed.body < 1 {
+		feed.body = MIMEHeadersEndPos(data)
+		if feed.body < 0 {
+			return false
+		}
+	}
+	if feed.headers == nil {
+		feed.headers = GetHeaders(data[feed.hdrStart:feed.body])
+		if feed.headers == nil {
+			return false
+		}
+	}
+	if len(data) > feed.body {
+		body = data[feed.body:]
+	}
 
+	if feed.headers.Get("Transfer-Encoding") == "chunked" {
 		// check chunks
 		if len(body) < 1 {
 			return false
@@ -453,7 +502,7 @@ func HasFullPayload(payload []byte) bool {
 		}
 
 		// check trailer headers
-		if len(Header(payload, []byte("Trailer"))) < 1 {
+		if feed.headers.Get("Trailer") == "" {
 			return true
 		}
 		// trailer headers(whether chunked or plain) should end with empty line
@@ -461,15 +510,12 @@ func HasFullPayload(payload []byte) bool {
 	}
 
 	// check for content-length header
-	// trailers are generally not allowed in non-chunks body
-	header = Header(payload, []byte("Content-Length"))
-	if len(header) > 1 {
-		num, ok := atoI(header, 10)
+	if header := feed.headers.Get("Content-Length"); header != "" {
+		num, ok := atoI([]byte(header), 10)
+		// trailers are generally not allowed in non-chunks body
 		return ok && num == len(body)
 	}
-
-	// for empty body, check for emptyline
-	return MIMEHeadersEndPos(payload) != -1
+	return true
 }
 
 // this works with positive integers

proto/proto_test.go

@@ -136,6 +136,24 @@ func TestParseHeaders(t *testing.T) {
 	if !reflect.DeepEqual(headers, expected) {
 		t.Error("Headers do not properly parsed", headers)
 	}
+
+	// Response with Reason phrase
+	payload = [][]byte{[]byte("HTTP/1.1 200 OK\r\nContent-Length: 7\r\nHost: www.w3.org\r\nUser-Agent:Chrome\r\n\r\nbody")}
+
+	headers = ParseHeaders(bytes.Join(payload, nil))
+
+	if !reflect.DeepEqual(headers, expected) {
+		t.Error("Headers do not properly parsed", headers)
+	}
+
+	// Response without Reason phrase
+	payload = [][]byte{[]byte("HTTP/1.1 200\r\nContent-Length: 7\r\nHost: www.w3.org\r\nUser-Agent:Chrome\r\n\r\nbody")}
+
+	headers = ParseHeaders(bytes.Join(payload, nil))
+
+	if !reflect.DeepEqual(headers, expected) {
+		t.Error("Headers do not properly parsed", headers)
+	}
 }
 
 // See https://github.com/dvyukov/go-fuzz and fuzz.go
@@ -219,6 +237,25 @@ func TestPath(t *testing.T) {
 	}
 }
 
+func TestStatus(t *testing.T) {
+	var status, payload []byte
+
+	payload = []byte("HTTP/1.1 200 OK\r\n")
+	if status = Status(payload); !bytes.Equal(status, []byte("200")) {
+		t.Error("Should find status 200 but:", string(status))
+	}
+
+	payload = []byte("HTTP/1.1 200\r\n")
+	if status = Status(payload); !bytes.Equal(status, []byte("200")) {
+		t.Error("1Should find status 200 but:", string(status))
+	}
+
+	payload = []byte("HTTP/1.1 404 Not Found\r\n")
+	if status = Status(payload); !bytes.Equal(status, []byte("404")) {
+		t.Error("2Should find status 404 but:", string(status))
+	}
+}
+
 func TestSetPath(t *testing.T) {
 	var payload, payloadAfter []byte
 
@@ -318,9 +355,10 @@ func TestHasResponseTitle(t *testing.T) {
 		"HTTP/1.1 100 Continue\r\n": true,
 		"HTTP/1.1  \r\n":            false,
 		"HTTP/4.0 100Continue\r\n":  false,
+		"HTTP/1.0 100Continue\r\n":  false,
 		"HTTP/1.0 10r Continue\r\n": false,
-		"HTTP/1.1 200\r\n":          false,
-		"HTTP/1.1 200\r\nServer: Tengine\r\nContent-Length: 0\r\nConnection: close\r\n\r\n": false,
+		"HTTP/1.1 200\r\n":          true,
+		"HTTP/1.1 200\r\nServer: Tengine\r\nContent-Length: 0\r\nConnection: close\r\n\r\n": true,
 	}
 	for k, v := range m {
 		if HasResponseTitle([]byte(k)) != v {
@@ -390,7 +428,7 @@ func TestCheckChunks(t *testing.T) {
 
 func TestHasFullPayload(t *testing.T) {
 	var m = "HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nTransfer-Encoding: chunked\r\n\r\n7\r\nMozilla\r\n9\r\nDeveloper\r\n7\r\nNetwork\r\n0\r\n\r\n"
-	got := HasFullPayload([]byte(m))
+	got := HasFullPayload([]byte(m), nil)
 	expected := true
 	if got != expected {
 		t.Errorf("expected %v to equal %v", got, expected)
@@ -398,7 +436,7 @@ func TestHasFullPayload(t *testing.T) {
 
 	// check with invalid chunk format
 	m = "HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nTransfer-Encoding: chunked\r\n\r\n7\r\nMozia\r\n9\r\nDeveloper\r\n7\r\nNetwork\r\n0\r\n\r\n"
-	got = HasFullPayload([]byte(m))
+	got = HasFullPayload([]byte(m), nil)
 	expected = false
 	if got != expected {
 		t.Errorf("expected %v to equal %v", got, expected)
@@ -406,7 +444,7 @@ func TestHasFullPayload(t *testing.T) {
 
 	// check chunks with trailers
 	m = "HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nTransfer-Encoding: chunked\r\nTrailer: Expires\r\n\r\n7\r\nMozilla\r\n9\r\nDeveloper\r\n7\r\nNetwork\r\n0\r\n\r\nExpires: Wed, 21 Oct 2015 07:28:00 GMT\r\n\r\n"
-	got = HasFullPayload([]byte(m))
+	got = HasFullPayload([]byte(m), nil)
 	expected = true
 	if got != expected {
 		t.Errorf("expected %v to equal %v", got, expected)
@@ -414,7 +452,7 @@ func TestHasFullPayload(t *testing.T) {
 
 	// check with missing trailers
 	m = "HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nTransfer-Encoding: chunked\r\nTrailer: Expires\r\n\r\n7\r\nMozilla\r\n9\r\nDeveloper\r\n7\r\nNetwork\r\n0\r\n\r\nExpires: Wed, 21 Oct 2015 07:28:00"
-	got = HasFullPayload([]byte(m))
+	got = HasFullPayload([]byte(m), nil)
 	expected = false
 	if got != expected {
 		t.Errorf("expected %v to equal %v", got, expected)
@@ -422,7 +460,7 @@ func TestHasFullPayload(t *testing.T) {
 
 	// check with content-length
 	m = "HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nContent-Length: 23\r\n\r\nMozillaDeveloperNetwork"
-	got = HasFullPayload([]byte(m))
+	got = HasFullPayload([]byte(m), nil)
 	expected = true
 	if got != expected {
 		t.Errorf("expected %v to equal %v", got, expected)
@@ -430,7 +468,7 @@ func TestHasFullPayload(t *testing.T) {
 
 	// check missing total length
 	m = "HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nContent-Length: 23\r\n\r\nMozillaDeveloperNet"
-	got = HasFullPayload([]byte(m))
+	got = HasFullPayload([]byte(m), nil)
 	expected = false
 	if got != expected {
 		t.Errorf("expected %v to equal %v", got, expected)
@@ -438,7 +476,7 @@ func TestHasFullPayload(t *testing.T) {
 
 	// check with no body
 	m = "HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\n\r\n"
-	got = HasFullPayload([]byte(m))
+	got = HasFullPayload([]byte(m), nil)
 	expected = true
 	if got != expected {
 		t.Errorf("expected %v to equal %v", got, expected)
@@ -454,13 +492,11 @@ func BenchmarkHasFullPayload(b *testing.B) {
 	}
 	buf.Write([]byte("0\r\n\r\n"))
 	data := buf.Bytes()
-	var ok bool
 	b.ResetTimer() // ignores the upper initialization
 	b.ReportMetric(float64(5000), "chunks/op")
 	for i := 0; i < b.N; i++ {
-		ok = HasFullPayload(data)
-	}
-	if !ok {
-		b.Fail()
+		if !HasFullPayload(data, nil) {
+			b.Fail()
+		}
 	}
 }

protocol.go

@@ -66,6 +66,16 @@ func payloadMeta(payload []byte) [][]byte {
 	return bytes.Split(payload[:headerSize], []byte{' '})
 }
 
+func payloadMetaWithBody(payload []byte) (meta, body []byte) {
+	if i := bytes.IndexByte(payload, '\n'); i > 0 && len(payload) > i+1 {
+		meta = payload[:i+1]
+		body = payload[i+1:]
+		return
+	}
+	// we assume the message did not have meta data
+	return nil, payload
+}
+
 func payloadID(payload []byte) (id []byte) {
 	meta := payloadMeta(payload)
 

settings.go

@@ -93,14 +93,10 @@ func init() {
 	}
 
 	flag.BoolVar(&Settings.SplitOutput, "split-output", false, "By default each output gets same traffic. If set to `true` it splits traffic equally among all outputs.")
-
 	flag.BoolVar(&Settings.RecognizeTCPSessions, "recognize-tcp-sessions", false, "[PRO] If turned on http output will create separate worker for each TCP session. Splitting output will session based as well.")
 
 	flag.Var(&Settings.InputDummy, "input-dummy", "Used for testing outputs. Emits 'Get /' request every 1s")
-	flag.Var(&Settings.OutputDummy, "output-dummy", "DEPRECATED: use --output-stdout instead")
-
 	flag.BoolVar(&Settings.OutputStdout, "output-stdout", false, "Used for testing inputs. Just prints to console data coming from inputs.")
-
 	flag.BoolVar(&Settings.OutputNull, "output-null", false, "Used for testing inputs. Drops all requests.")
 
 	flag.Var(&Settings.InputTCP, "input-tcp", "Used for internal communication between Gor instances. Example: \n\t# Receive requests from other Gor instances on 28020 port, and redirect output to staging\n\tgor --input-tcp :28020 --output-http staging.com")
@@ -192,32 +188,16 @@ func init() {
 	flag.StringVar(&Settings.KafkaTLSConfig.ClientKey, "kafka-tls-client-key", "", "Client Key for Kafka TLS Config (mandatory with to kafka-tls-client-cert and kafka-tls-client-key)")
 
 	flag.Var(&Settings.ModifierConfig.Headers, "http-set-header", "Inject additional headers to http reqest:\n\tgor --input-raw :8080 --output-http staging.com --http-set-header 'User-Agent: Gor'")
-	flag.Var(&Settings.ModifierConfig.Headers, "output-http-header", "WARNING: `--output-http-header` DEPRECATED, use `--http-set-header` instead")
-
 	flag.Var(&Settings.ModifierConfig.HeaderRewrite, "http-rewrite-header", "Rewrite the request header based on a mapping:\n\tgor --input-raw :8080 --output-http staging.com --http-rewrite-header Host: (.*).example.com,$1.beta.example.com")
-
 	flag.Var(&Settings.ModifierConfig.Params, "http-set-param", "Set request url param, if param already exists it will be overwritten:\n\tgor --input-raw :8080 --output-http staging.com --http-set-param api_key=1")
-
 	flag.Var(&Settings.ModifierConfig.Methods, "http-allow-method", "Whitelist of HTTP methods to replay. Anything else will be dropped:\n\tgor --input-raw :8080 --output-http staging.com --http-allow-method GET --http-allow-method OPTIONS")
-
 	flag.Var(&Settings.ModifierConfig.URLRegexp, "http-allow-url", "A regexp to match requests against. Filter get matched against full url with domain. Anything else will be dropped:\n\t gor --input-raw :8080 --output-http staging.com --http-allow-url ^www.")
-
 	flag.Var(&Settings.ModifierConfig.URLNegativeRegexp, "http-disallow-url", "A regexp to match requests against. Filter get matched against full url with domain. Anything else will be forwarded:\n\t gor --input-raw :8080 --output-http staging.com --http-disallow-url ^www.")
-
 	flag.Var(&Settings.ModifierConfig.URLRewrite, "http-rewrite-url", "Rewrite the request url based on a mapping:\n\tgor --input-raw :8080 --output-http staging.com --http-rewrite-url /v1/user/([^\\/]+)/ping:/v2/user/$1/ping")
-	flag.Var(&Settings.ModifierConfig.URLRewrite, "output-http-rewrite-url", "WARNING: `--output-http-rewrite-url` DEPRECATED, use `--http-rewrite-url` instead")
-
 	flag.Var(&Settings.ModifierConfig.HeaderFilters, "http-allow-header", "A regexp to match a specific header against. Requests with non-matching headers will be dropped:\n\t gor --input-raw :8080 --output-http staging.com --http-allow-header api-version:^v1")
-	flag.Var(&Settings.ModifierConfig.HeaderFilters, "output-http-header-filter", "WARNING: `--output-http-header-filter` DEPRECATED, use `--http-allow-header` instead")
-
 	flag.Var(&Settings.ModifierConfig.HeaderNegativeFilters, "http-disallow-header", "A regexp to match a specific header against. Requests with matching headers will be dropped:\n\t gor --input-raw :8080 --output-http staging.com --http-disallow-header \"User-Agent: Replayed by Gor\"")
-
 	flag.Var(&Settings.ModifierConfig.HeaderBasicAuthFilters, "http-basic-auth-filter", "A regexp to match the decoded basic auth string against. Requests with non-matching headers will be dropped:\n\t gor --input-raw :8080 --output-http staging.com --http-basic-auth-filter \"^customer[0-9].*\"")
-
 	flag.Var(&Settings.ModifierConfig.HeaderHashFilters, "http-header-limiter", "Takes a fraction of requests, consistently taking or rejecting a request based on the FNV32-1A hash of a specific header:\n\t gor --input-raw :8080 --output-http staging.com --http-header-limiter user-id:25%")
-
-	flag.Var(&Settings.ModifierConfig.HeaderHashFilters, "output-http-header-hash-filter", "WARNING: `output-http-header-hash-filter` DEPRECATED, use `--http-header-hash-limiter` instead")
-
 	flag.Var(&Settings.ModifierConfig.ParamHashFilters, "http-param-limiter", "Takes a fraction of requests, consistently taking or rejecting a request based on the FNV32-1A hash of a specific GET param:\n\t gor --input-raw :8080 --output-http staging.com --http-param-limiter user_id:25%")
 
 	// default values, using for tests

tcp/tcp_message.go

@@ -2,15 +2,15 @@ package tcp
 
 import (
 	"bytes"
-	"crypto/sha1"
+	"encoding/binary"
 	"encoding/hex"
 	"fmt"
 	"sort"
 	"sync"
 	"time"
 
+	"github.com/buger/goreplay/capture"
 	"github.com/buger/goreplay/size"
-	"github.com/google/gopacket"
 )
 
 // Stats every message carry its own stats object
@@ -29,9 +29,11 @@ type Stats struct {
 
 // Message is the representation of a tcp message
 type Message struct {
-	packets []*Packet
-	done    chan bool
-	buf     bytes.Buffer
+	packets  []*Packet
+	done     chan bool
+	pool     *MessagePool
+	buf      *bytes.Buffer
+	feedback interface{}
 	Stats
 }
 
@@ -42,42 +44,55 @@ func NewMessage(srcAddr, dstAddr string, ipVersion uint8) (m *Message) {
 	m.SrcAddr = srcAddr
 	m.IPversion = ipVersion
 	m.done = make(chan bool)
+	m.buf = &bytes.Buffer{}
 	return
 }
 
-// UUID the unique id of a TCP session it is not granted to be unique!
+// UUID returns the UUID of a TCP request and its response.
 func (m *Message) UUID() []byte {
-	var src, dst string
-	if m.IsIncoming {
-		src = m.SrcAddr
-		dst = m.DstAddr
-	} else {
-		src = m.DstAddr
-		dst = m.SrcAddr
+	var key uint64
+	pckt := m.packets[0]
+
+	// check if response or request have generated the ID before.
+	if m.pool.uuids != nil {
+		lst := len(pckt.SrcIP) - 4
+		if m.IsIncoming {
+			key = uint64(pckt.SrcPort)<<48 | uint64(pckt.DstPort)<<32 |
+				uint64(_uint32(pckt.SrcIP[lst:]))
+		} else {
+			key = uint64(pckt.DstPort)<<48 | uint64(pckt.SrcPort)<<32 |
+				uint64(_uint32(pckt.DstIP[lst:]))
+		}
+		if uuidHex, ok := m.pool.uuids[key]; ok {
+			delete(m.pool.uuids, key)
+			return uuidHex
+		}
 	}
 
-	length := len(src) + len(dst)
-	uuid := make([]byte, length)
-	copy(uuid, src)
-	copy(uuid[len(src):], dst)
-	sha := sha1.Sum(uuid)
-	uuid = make([]byte, 40)
-	hex.Encode(uuid, sha[:])
-
-	return uuid
+	id := make([]byte, 12, 12)
+	binary.BigEndian.PutUint32(id, pckt.Seq)
+	tStamp := m.End.UnixNano()
+	binary.BigEndian.PutUint64(id[4:], uint64(tStamp))
+	uuidHex := make([]byte, 24, 24)
+	hex.Encode(uuidHex[:], id[:])
+	if m.pool.uuids != nil {
+		if len(m.pool.uuids) >= 1000 {
+			m.pool.cleanUUIDs()
+		}
+		m.pool.uuids[key] = uuidHex
+	}
+	return uuidHex
 }
 
 func (m *Message) add(pckt *Packet) {
 	m.Length += len(pckt.Payload)
 	m.LostData += int(pckt.Lost)
 	m.packets = append(m.packets, pckt)
-	if len(pckt.Payload) > 0 {
-		m.buf.Write(pckt.Payload)
-	}
 	m.End = pckt.Timestamp
+	m.buf.Write(pckt.Payload)
 }
 
-// Packets returns packets of this message
+// Packets returns packets of the message
 func (m *Message) Packets() []*Packet {
 	return m.packets
 }
@@ -87,6 +102,16 @@ func (m *Message) Data() []byte {
 	return m.buf.Bytes()
 }
 
+// SetFeedback set feedback/data that can be used later, e.g with End or Start hint
+func (m *Message) SetFeedback(feedback interface{}) {
+	m.feedback = feedback
+}
+
+// Feedback returns feedback associated to this message
+func (m *Message) Feedback() interface{} {
+	return m.feedback
+}
+
 // Sort a helper to sort packets
 func (m *Message) Sort() {
 	sort.SliceStable(m.packets, func(i, j int) bool { return m.packets[i].Seq < m.packets[j].Seq })
@@ -104,7 +129,7 @@ type Debugger func(int, ...interface{})
 type HintEnd func(*Message) bool
 
 // HintStart hints the pool to start the reassembling the message, see MessagePool.Start
-// when set, it will be used instead of checking SYN flag
+// when set, it will be called after checking SYN flag
 type HintStart func(*Packet) (IsIncoming, IsOutgoing bool)
 
 // MessagePool holds data of all tcp messages in progress(still receiving/sending packets).
@@ -114,6 +139,7 @@ type MessagePool struct {
 	debug         Debugger
 	maxSize       size.Size // maximum message size, default 5mb
 	pool          map[uint64]*Message
+	uuids         map[uint64][]byte
 	handler       Handler
 	messageExpire time.Duration // the maximum time to wait for the final packet, minimum is 100ms
 	End           HintEnd
@@ -138,22 +164,18 @@ func NewMessagePool(maxSize size.Size, messageExpire time.Duration, debugger Deb
 }
 
 // Handler returns packet handler
-func (pool *MessagePool) Handler(packet gopacket.Packet) {
+func (pool *MessagePool) Handler(packet *capture.Packet) {
 	var in, out bool
 	pckt, err := ParsePacket(packet)
-	if err != nil || pckt == nil {
-		go pool.say(4, fmt.Sprintf("error decoding packet(%dBytes):%s\n", packet.Metadata().CaptureLength, err))
+	if err != nil {
+		go pool.say(4, fmt.Sprintf("error decoding packet(%dBytes):%s\n", packet.Info.CaptureLength, err))
 		return
 	}
 	pool.Lock()
 	defer pool.Unlock()
-	lst := 3
-	if pckt.Version == 6 {
-		lst = 15
-	}
+	lst := len(pckt.SrcIP) - 4
 	key := uint64(pckt.SrcPort)<<48 | uint64(pckt.DstPort)<<32 |
-		uint64(pckt.SrcIP[lst])<<24 | uint64(pckt.SrcIP[lst-1])<<16 |
-		uint64(pckt.SrcIP[lst-2])<<8 | uint64(pckt.SrcIP[lst-3])
+		uint64(_uint32(pckt.SrcIP[lst:]))
 	m, ok := pool.pool[key]
 	if pckt.RST {
 		if ok {
@@ -161,8 +183,7 @@ func (pool *MessagePool) Handler(packet gopacket.Packet) {
 			<-m.done
 		}
 		key = uint64(pckt.DstPort)<<48 | uint64(pckt.SrcPort)<<32 |
-			uint64(pckt.DstIP[lst])<<24 | uint64(pckt.DstIP[lst-1])<<16 |
-			uint64(pckt.DstIP[lst-2])<<8 | uint64(pckt.DstIP[lst-3])
+			uint64(_uint32(pckt.DstIP[lst:]))
 		m, ok = pool.pool[key]
 		if ok {
 			m.done <- true
@@ -188,10 +209,34 @@ func (pool *MessagePool) Handler(packet gopacket.Packet) {
 	m.IsIncoming = in
 	pool.pool[key] = m
 	m.Start = pckt.Timestamp
+	m.pool = pool
 	go pool.dispatch(key, m)
 	pool.addPacket(m, pckt)
 }
 
+// MatchUUID instructs the pool to use same UUID for request and responses
+// this function should be called at initial stage of the pool
+func (pool *MessagePool) MatchUUID(match bool) {
+	if match {
+		pool.uuids = make(map[uint64][]byte)
+		return
+	}
+	pool.uuids = nil
+}
+
+// run GC on UUID map
+func (pool *MessagePool) cleanUUIDs() {
+	var tStamp int64
+	now := time.Now().UnixNano()
+	for k, v := range pool.uuids {
+		// there is a timestamp wrapped in every ID
+		tStamp = int64(binary.BigEndian.Uint64(v[4:]))
+		if time.Duration(now-tStamp) > pool.messageExpire {
+			delete(pool.uuids, k)
+		}
+	}
+}
+
 func (pool *MessagePool) dispatch(key uint64, m *Message) {
 	select {
 	case <-m.done:
@@ -217,10 +262,13 @@ func (pool *MessagePool) addPacket(m *Message, pckt *Packet) {
 	}
 	m.add(pckt)
 	switch {
+
+	// if one of this cases matches, we dispatch the message
 	case trunc >= 0:
 	case pckt.FIN:
 	case pool.End != nil && pool.End(m):
 	default:
+		// continue to receive packets
 		return
 	}
 	m.done <- true
@@ -233,3 +281,7 @@ func (pool *MessagePool) say(level int, args ...interface{}) {
 		pool.debug(level, args...)
 	}
 }
+
+func _uint32(b []byte) uint32 {
+	return binary.BigEndian.Uint32(b)
+}

tcp/tcp_packet.go

@@ -2,12 +2,12 @@ package tcp
 
 import (
 	"encoding/binary"
+	"errors"
 	"fmt"
 	"net"
 	"time"
 
-	"github.com/google/gopacket"
-	"github.com/google/gopacket/layers"
+	"github.com/buger/goreplay/capture"
 )
 
 /*
@@ -17,81 +17,51 @@ calllers must make sure that ParsePacket has'nt returned any error before callin
 function.
 */
 type Packet struct {
-	// Link layer
-	gopacket.LinkLayer
-
-	// IP Header
-	Version uint8 // Ip version
-	SrcIP   net.IP
-	DstIP   net.IP
-	IHL     uint8
-	Length  uint16
-
-	// TCP Segment Header
-	*layers.TCP
-
-	// Data info
-	Lost      uint16
-	Timestamp time.Time
+	SrcIP, DstIP       net.IP
+	Version            uint8
+	SrcPort, DstPort   uint16
+	Ack, Seq           uint32
+	ACK, SYN, FIN, RST bool
+	Lost               uint32
+	Timestamp          time.Time
+	Payload            []byte
 }
 
 // ParsePacket parse raw packets
-func ParsePacket(packet gopacket.Packet) (pckt *Packet, err error) {
+func ParsePacket(packet *capture.Packet) (pckt *Packet, err error) {
 	// early check of error
 	if packet == nil {
-		return
+		return nil, errors.New("empty packet")
 	}
-	defer func() {
-		if packet.ErrorLayer() != nil {
-			err = packet.ErrorLayer().Error()
-			return
-		}
-	}()
-
-	// initialization
-	pckt = new(Packet)
-	pckt.Timestamp = packet.Metadata().Timestamp
-	if pckt.Timestamp.IsZero() {
-		pckt.Timestamp = time.Now()
+	if packet.Err != nil {
+		return nil, packet.Err
 	}
 
-	// parsing link layer
-	pckt.LinkLayer = packet.LinkLayer()
-
-	// parsing network layer
-	if net4, ok := packet.NetworkLayer().(*layers.IPv4); ok {
+	var t Packet
+	pckt = &t
+	// TODO: check resolution
+	pckt.Timestamp = packet.Info.Timestamp
+	if (packet.NetLayer[0] >> 4) == 4 {
+		// IPv4 header
 		pckt.Version = 4
-		pckt.SrcIP = net4.SrcIP
-		pckt.DstIP = net4.DstIP
-		pckt.IHL = net4.IHL * 4
-		pckt.Length = net4.Length
-	} else if net6, ok := packet.NetworkLayer().(*layers.IPv6); ok {
+		pckt.SrcIP = packet.NetLayer[12:16]
+		pckt.DstIP = packet.NetLayer[16:20]
+	} else {
+		// IPv6 header
 		pckt.Version = 6
-		pckt.SrcIP = net6.SrcIP
-		pckt.DstIP = net6.DstIP
-		pckt.IHL = 40
-		pckt.Length = net6.Length
-	} else {
-		pckt = nil
-		return
+		pckt.SrcIP = packet.NetLayer[8:24]
+		pckt.DstIP = packet.NetLayer[24:40]
 	}
-
-	// parsing tcp header(transportation layer)
-	if tcp, ok := packet.TransportLayer().(*layers.TCP); ok {
-		pckt.TCP = tcp
-	} else {
-		pckt = nil
-		return
-	}
-	pckt.DataOffset *= 4
-
-	// calculating lost data
-	headerSize := int(uint32(pckt.DataOffset) + uint32(pckt.IHL))
-	if pckt.Version == 6 {
-		headerSize -= 40 // in ipv6 the length of payload doesn't include the IPheader size
-	}
-	pckt.Lost = pckt.Length - uint16(headerSize+len(pckt.Payload))
-
+	pckt.SrcPort = binary.BigEndian.Uint16(packet.TransLayer[0:2])
+	pckt.DstPort = binary.BigEndian.Uint16(packet.TransLayer[2:4])
+	pckt.Seq = binary.BigEndian.Uint32(packet.TransLayer[4:8])
+	pckt.Ack = binary.BigEndian.Uint32(packet.TransLayer[8:12])
+	pckt.FIN = packet.TransLayer[13]&0x01 != 0
+	pckt.SYN = packet.TransLayer[13]&0x02 != 0
+	pckt.RST = packet.TransLayer[13]&0x04 != 0
+	pckt.ACK = packet.TransLayer[13]&0x10 != 0
+	pckt.Lost = uint32(packet.Info.Length - packet.Info.CaptureLength)
+	pckt.Payload = packet.Payload
 	return
 }
 
@@ -104,94 +74,3 @@ func (pckt *Packet) Src() string {
 func (pckt *Packet) Dst() string {
 	return fmt.Sprintf("%s:%d", pckt.DstIP, pckt.DstPort)
 }
-
-// SYNOptions returns MSS and windowscale of syn packets
-func (pckt *Packet) SYNOptions() (mss uint16, windowscale byte) {
-	if !pckt.SYN {
-		return
-	}
-	for _, v := range pckt.Options {
-		if v.OptionType == layers.TCPOptionKindMSS {
-			mss = binary.BigEndian.Uint16(v.OptionData)
-			continue
-		}
-		if v.OptionType == layers.TCPOptionKindWindowScale {
-			if v.OptionLength > 0 {
-				windowscale = v.OptionData[0]
-			}
-		}
-	}
-	return
-}
-
-// LinkInfo returns info about the link layer
-func (pckt *Packet) LinkInfo() string {
-	if l, ok := pckt.LinkLayer.(*layers.Ethernet); ok {
-		return fmt.Sprintf(
-			"Source Mac: %s\nDestination Mac: %s\nProtocol: %s",
-			l.SrcMAC,
-			l.DstMAC,
-			l.EthernetType,
-		)
-	}
-	return "<Not Ethernet>"
-}
-
-// Flag returns formatted tcp flags
-func (pckt *Packet) Flag() (flag string) {
-	if pckt.FIN {
-		flag += "FIN, "
-	}
-	if pckt.SYN {
-		flag += "SYN, "
-	}
-	if pckt.RST {
-		flag += "RST, "
-	}
-	if pckt.PSH {
-		flag += "PSH, "
-	}
-	if pckt.ACK {
-		flag += "ACK, "
-	}
-	if pckt.URG {
-		flag += "URG, "
-	}
-	if len(flag) != 0 {
-		return flag[:len(flag)-2]
-	}
-	return flag
-}
-
-// String output for a TCP Packet
-func (pckt *Packet) String() string {
-	return fmt.Sprintf(`Time: %s
-%s
-Source: %s
-Destination: %s
-IHL: %d
-Total Length: %d
-Sequence: %d
-Acknowledgment: %d
-DataOffset: %d
-Window: %d
-Flag: %s
-Options: %s
-Data Size: %d
-Lost Data: %d`,
-		pckt.Timestamp.Format(time.StampNano),
-		pckt.LinkInfo(),
-		pckt.Src(),
-		pckt.Dst(),
-		pckt.IHL,
-		pckt.Length,
-		pckt.Seq,
-		pckt.Ack,
-		pckt.DataOffset,
-		pckt.Window,
-		pckt.Flag(),
-		pckt.Options,
-		len(pckt.Payload),
-		pckt.Lost,
-	)
-}

tcp/tcp_test.go

@@ -6,41 +6,38 @@ import (
 	"testing"
 	"time"
 
+	"github.com/buger/goreplay/capture"
 	"github.com/buger/goreplay/proto"
 	"github.com/google/gopacket"
 	"github.com/google/gopacket/layers"
 )
 
-var decodeOpts = gopacket.DecodeOptions{Lazy: true, NoCopy: true}
+func generateHeader(seq uint32, length uint16) []byte {
+	hdr := make([]byte, 4+24+24, 4+24+24)
+	binary.BigEndian.PutUint32(hdr, uint32(layers.ProtocolFamilyIPv4))
 
-func headersIP4(seq uint32, length uint16) (headers [54]byte) {
-	// set ethernet headers
-	binary.BigEndian.PutUint16(headers[12:14], uint16(layers.EthernetTypeIPv4))
-
-	// set ip header
-	ip := headers[14:]
-	copy(ip[0:2], []byte{4<<4 | 5, 0x28<<2 | 0x00})
-	binary.BigEndian.PutUint16(ip[2:4], length+40)
+	ip := hdr[4:]
+	ip[0] = 4<<4 | 6
+	binary.BigEndian.PutUint16(ip[2:4], length+24+24)
 	ip[9] = uint8(layers.IPProtocolTCP)
-	copy(ip[12:16], []byte{192, 168, 1, 2})
-	copy(ip[16:], []byte{192, 168, 1, 3})
+	copy(ip[12:16], []byte{127, 0, 0, 1})
+	copy(ip[16:], []byte{127, 0, 0, 1})
 
 	// set tcp header
-	tcp := ip[20:]
-	binary.BigEndian.PutUint16(tcp[0:2], 45678)
-	binary.BigEndian.PutUint16(tcp[2:4], 8001)
-	tcp[12] = 5 << 4
-	return
+	tcp := ip[24:]
+	tcp[12] = 6 << 4
+	binary.BigEndian.PutUint16(tcp, 5535)
+	binary.BigEndian.PutUint16(tcp[2:], 8000)
+	binary.BigEndian.PutUint32(tcp[4:], seq)
+	return hdr
 }
 
-func GetPackets(start uint32, _len int, payload []byte) []gopacket.Packet {
-	var packets = make([]gopacket.Packet, _len)
+func GetPackets(start uint32, _len int, payload []byte) []*capture.Packet {
+	var packets = make([]*capture.Packet, _len)
 	for i := start; i < start+uint32(_len); i++ {
-		data := make([]byte, 54+len(payload))
-		h := headersIP4(i, uint16(len(payload)))
-		copy(data, h[:])
-		copy(data[len(h):], payload)
-		packets[i-start] = gopacket.NewPacket(data, layers.LinkTypeEthernet, decodeOpts)
+		d := append(generateHeader(i, uint16(len(payload))), payload...)
+		ci := &gopacket.CaptureInfo{Length: len(d), CaptureLength: len(d), Timestamp: time.Now()}
+		packets[i-start] = capture.NewPacket(d, int(layers.LinkTypeLoop), 4, ci)
 	}
 	return packets
 }
@@ -52,12 +49,12 @@ func TestMessageParserWithHint(t *testing.T) {
 		return proto.HasRequestTitle(pckt.Payload), proto.HasResponseTitle(pckt.Payload)
 	}
 	pool.End = func(m *Message) bool {
-		return proto.HasFullPayload(m.Data())
+		return proto.HasFullPayload(m.Data(), m)
 	}
 	packets := GetPackets(1, 30, nil)
-	packets[0].Data()[14:][20:][13] = 2  // SYN flag
-	packets[10].Data()[14:][20:][13] = 2 // SYN flag
-	packets[29].Data()[14:][20:][13] = 1 // FIN flag
+	packets[0].TransLayer[13] = 2  // SYN flag
+	packets[10].TransLayer[13] = 2 // SYN flag
+	packets[29].TransLayer[13] = 1 // FIN flag
 	packets[4] = GetPackets(5, 1, []byte("HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nTransfer-Encoding: chunked\r\n\r\n7"))[0]
 	packets[5] = GetPackets(6, 1, []byte("\r\nMozilla\r\n9\r\nDeveloper\r"))[0]
 	packets[6] = GetPackets(7, 1, []byte("\n7\r\nNetwork\r\n0\r\n\r\n"))[0]
@@ -75,9 +72,6 @@ func TestMessageParserWithHint(t *testing.T) {
 		return
 	case m = <-mssg:
 	}
-	if len(m.packets) != 7 {
-		t.Errorf("expected to have 7 packets got %d", len(m.packets))
-	}
 	if !bytes.HasSuffix(m.Data(), []byte("\n7\r\nNetwork\r\n0\r\n\r\n")) {
 		t.Errorf("expected to %q to have suffix %q", m.Data(), []byte("\n7\r\nNetwork\r\n0\r\n\r\n"))
 	}
@@ -88,9 +82,6 @@ func TestMessageParserWithHint(t *testing.T) {
 		return
 	case m = <-mssg:
 	}
-	if len(m.packets) != 7 {
-		t.Errorf("expected to have 7 packets got %d", len(m.packets))
-	}
 	if !bytes.HasSuffix(m.Data(), []byte("Network")) {
 		t.Errorf("expected to %q to have suffix %q", m.Data(), []byte("Network"))
 	}
@@ -101,9 +92,6 @@ func TestMessageParserWithHint(t *testing.T) {
 		return
 	case m = <-mssg:
 	}
-	if len(m.packets) != 6 {
-		t.Errorf("expected to have 6 packets got %d", len(m.packets))
-	}
 	if !bytes.HasSuffix(m.Data(), []byte("Content-Length: 0\r\n\r")) {
 		t.Errorf("expected to %q to have suffix %q", m.Data(), []byte("Content-Length: 0\r\n\r"))
 	}
@@ -114,8 +102,8 @@ func TestMessageParserWithoutHint(t *testing.T) {
 	var mssg = make(chan *Message, 1)
 	var data [63 << 10]byte
 	packets := GetPackets(1, 10, data[:])
-	packets[0].Data()[14:][20:][13] = 2 // SYN flag
-	packets[9].Data()[14:][20:][13] = 1 // FIN flag
+	packets[0].TransLayer[13] = 2 // SYN flag
+	packets[9].TransLayer[13] = 1 // FIN flag
 	p := NewMessagePool(63<<10*10, time.Second, nil, func(m *Message) { mssg <- m })
 	for _, v := range packets {
 		p.Handler(v)
@@ -137,9 +125,9 @@ func TestMessageMaxSizeReached(t *testing.T) {
 	var data [63 << 10]byte
 	packets := GetPackets(1, 2, data[:])
 	packets = append(packets, GetPackets(3, 1, make([]byte, 63<<10+10))...)
-	packets[0].Data()[14:][20:][13] = 2 // SYN flag
-	packets[2].Data()[14:][20:][13] = 2 // SYN flag
-	packets[2].Data()[14:][15] = 3      // changing address
+	packets[0].TransLayer[13] = 2 // SYN flag
+	packets[2].TransLayer[13] = 2 // SYN flag
+	packets[2].NetLayer[15] = 3   // changing address
 	p := NewMessagePool(63<<10+10, time.Second, nil, func(m *Message) { mssg <- m })
 	for _, v := range packets {
 		p.Handler(v)
@@ -176,7 +164,7 @@ func TestMessageTimeoutReached(t *testing.T) {
 	var mssg = make(chan *Message, 2)
 	var data [63 << 10]byte
 	packets := GetPackets(1, 2, data[:])
-	packets[0].Data()[14:][20:][13] = 2 // SYN flag
+	packets[0].TransLayer[13] = 2 // SYN flag
 	p := NewMessagePool(1<<20, 0, nil, func(m *Message) { mssg <- m })
 	p.Handler(packets[0])
 	time.Sleep(time.Millisecond * 200)
@@ -191,16 +179,47 @@ func TestMessageTimeoutReached(t *testing.T) {
 }
 
 func TestMessageUUID(t *testing.T) {
-	m1 := &Message{}
-	m1.IsIncoming = true
-	m1.SrcAddr = "src"
-	m1.DstAddr = "dst"
-	m2 := &Message{}
-	m2.SrcAddr = "dst"
-	m2.DstAddr = "src"
-	if string(m1.UUID()) != string(m2.UUID()) {
-		t.Errorf("expected %s, to equal %s", m1.UUID(), m2.UUID())
+	packets := GetPackets(1, 10, nil)
+	packets[0].TransLayer[13] = 2 // SYN flag
+	packets[4].TransLayer[13] = 1 // FIN flag
+	packets[5].TransLayer[13] = 2 // SYN flag
+	packets[9].TransLayer[13] = 1 // FIN flag
+	var uuid, uuid1 []byte
+	pool := NewMessagePool(0, 0, nil, func(msg *Message) {
+		if len(uuid) == 0 {
+			uuid = msg.UUID()
+			return
+		}
+		uuid1 = msg.UUID()
+	})
+	pool.MatchUUID(true)
+	for _, p := range packets {
+		pool.Handler(p)
 	}
+
+	if string(uuid) != string(uuid1) {
+		t.Errorf("expected %s, to equal %s", uuid, uuid1)
+	}
+}
+
+func BenchmarkMessageUUID(b *testing.B) {
+	packets := GetPackets(1, 5, nil)
+	packets[0].TransLayer[13] = 2 // SYN flag
+	packets[4].TransLayer[13] = 1 // FIN flag
+	var uuid []byte
+	var msg *Message
+	pool := NewMessagePool(0, 0, nil, func(m *Message) {
+		msg = m
+	})
+	pool.MatchUUID(true)
+	for _, p := range packets {
+		pool.Handler(p)
+	}
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		uuid = msg.UUID()
+	}
+	_ = uuid
 }
 
 func BenchmarkPacketParseAndSort(b *testing.B) {
@@ -219,8 +238,8 @@ func BenchmarkMessageParserWithoutHint(b *testing.B) {
 	var mssg = make(chan *Message, 1)
 	var chunk = []byte("111111111111111111111111111111")
 	packets := GetPackets(1, 1000, chunk)
-	packets[0].Data()[14:][20:][13] = 2      // SYN flag
-	packets[1000-1].Data()[14:][20:][13] = 1 // FIN flag
+	packets[0].TransLayer[13] = 2      // SYN flag
+	packets[1000-1].TransLayer[13] = 1 // FIN flag
 	p := NewMessagePool(1<<20, time.Second*2, nil, func(m *Message) {
 		mssg <- m
 	})
@@ -242,7 +261,7 @@ func BenchmarkMessageParserWithHint(b *testing.B) {
 		buf[i] = chunk
 	}
 	buf[1001] = []byte("0\r\n\r\n")
-	packets := make([]gopacket.Packet, len(buf))
+	packets := make([]*capture.Packet, len(buf))
 	for i := 0; i < len(buf); i++ {
 		packets[i] = GetPackets(uint32(i+10), 1, buf[i])[0]
 	}
@@ -252,7 +271,7 @@ func BenchmarkMessageParserWithHint(b *testing.B) {
 		return false, proto.HasResponseTitle(pckt.Payload)
 	}
 	pool.End = func(m *Message) bool {
-		return proto.HasFullPayload(m.Data())
+		return proto.HasFullPayload(m.Data(), m)
 	}
 	b.ResetTimer()
 	b.ReportMetric(float64(len(packets)), "packets/op")
@@ -264,3 +283,24 @@ func BenchmarkMessageParserWithHint(b *testing.B) {
 		<-mssg
 	}
 }
+
+func BenchmarkNewAndParsePacket(b *testing.B) {
+	data := append(generateHeader(1024, 10), make([]byte, 10)...)
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		ParsePacket(capture.NewPacket(data, int(layers.LinkTypeLoop), 4, &gopacket.CaptureInfo{}))
+	}
+}
+
+func BenchmarkNewPacket(b *testing.B) {
+	packet := capture.NewPacket(
+		append(generateHeader(1024, 10), make([]byte, 10)...),
+		int(layers.LinkTypeLoop),
+		4,
+		&gopacket.CaptureInfo{},
+	)
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		ParsePacket(packet)
+	}
+}

test_input.go

@@ -25,22 +25,21 @@ func NewTestInput() (i *TestInput) {
 	return
 }
 
-func (i *TestInput) Read(data []byte) (int, error) {
+// PluginRead reads message from this plugin
+func (i *TestInput) PluginRead() (*Message, error) {
+	var msg Message
 	select {
 	case buf := <-i.data:
-		var header []byte
-
+		msg.Data = buf
 		if !i.skipHeader {
-			header = payloadHeader(RequestPayload, uuid(), time.Now().UnixNano(), -1)
-			copy(data[0:len(header)], header)
-			copy(data[len(header):], buf)
+			msg.Meta = payloadHeader(RequestPayload, uuid(), time.Now().UnixNano(), -1)
 		} else {
-			copy(data, buf)
+			msg.Meta, msg.Data = payloadMetaWithBody(msg.Data)
 		}
 
-		return len(buf) + len(header), nil
+		return &msg, nil
 	case <-i.stop:
-		return 0, ErrorStopped
+		return nil, ErrorStopped
 	}
 }
 

test_output.go

@@ -1,6 +1,6 @@
 package main
 
-type writeCallback func(data []byte)
+type writeCallback func(*Message)
 
 // TestOutput used in testing to intercept any output into callback
 type TestOutput struct {
@@ -8,19 +8,20 @@ type TestOutput struct {
 }
 
 // NewTestOutput constructor for TestOutput, accepts callback which get called on each incoming Write
-func NewTestOutput(cb writeCallback) (i *TestOutput) {
-	i = new(TestOutput)
+func NewTestOutput(cb writeCallback) PluginWriter {
+	i := new(TestOutput)
 	i.cb = cb
 
-	return
+	return i
 }
 
-func (i *TestOutput) Write(data []byte) (int, error) {
-	i.cb(data)
+// PluginWrite write message to this plugin
+func (i *TestOutput) PluginWrite(msg *Message) (int, error) {
+	i.cb(msg)
 
-	return len(data), nil
+	return len(msg.Data) + len(msg.Meta), nil
 }
 
 func (i *TestOutput) String() string {
-	return "Test Input"
+	return "Test Output"
 }