fix #825

The focus here was to **reduce allocation in TCP parser** but speed may have hopeful improved too!
pool no longer use map's key of **string** it uses **uint64**
**Benchmarks** was revamped to be more clear
if you want to compare these results copy the benchmark in tcp/bench_test.go@reduce-allocation to tcp/bench_test.go@master:

**before(master)**:
```
BenchmarkPacketParseAndSort-4         	 1000000	      1006 ns/op	      64 B/op	       2 allocs/op
BenchmarkMessageParserWithoutHint-4   	     625	   1772309 ns/op	      1000 packets/op	  419096 B/op	   10045 allocs/op
BenchmarkMessageParserWithHint-4      	      74	  14969926 ns/op	      1000 chunks/op	      1002 packets/op	  450992 B/op	   10126 allocs/op
```

**After(this branch)**:
```
BenchmarkPacketParseAndSort-4         	 1267662	       941 ns/op	      64 B/op	       2 allocs/op
BenchmarkMessageParserWithoutHint-4   	    2256	    523474 ns/op	      1000 packets/op	  243530 B/op	    1037 allocs/op
BenchmarkMessageParserWithHint-4      	      80	  13990955 ns/op	      1000 chunks/op	      1002 packets/op	  268609 B/op	    1099 allocs/op

```

.clabot

@@ -0,0 +1,3 @@
+{
+  "contributors": ["buger"]
+}

.gitignore

@@ -1,5 +1,8 @@
 *.swp
 *.gor
+*.rpm
+*.dep
+*.pkg
 
 *.out
 
@@ -21,6 +24,7 @@ gor
 
 .DS_Store
 
+goreplay
 corpus
 crashers
 suppressions

.gitmodules

@@ -1,15 +0,0 @@
-[submodule "vendor/github.com/mattbaird/elastigo"]
-	path = vendor/github.com/mattbaird/elastigo
-	url = https://github.com/mattbaird/elastigo
-[submodule "vendor/github.com/araddon/gou"]
-	path = vendor/github.com/araddon/gou
-	url = https://github.com/araddon/gou
-[submodule "vendor/github.com/bitly/go-hostpool"]
-	path = vendor/github.com/bitly/go-hostpool
-	url = https://github.com/bitly/go-hostpool
-[submodule "vendor/github.com/bmizerany/assert"]
-	path = vendor/github.com/bmizerany/assert
-	url = https://github.com/bmizerany/assert
-[submodule "vendor/github.com/google/gopacket"]
-	path = vendor/github.com/google/gopacket
-	url = https://github.com/google/gopacket

.travis.yml

@@ -1,6 +1,9 @@
+dist: focal
+
 language: go
-go: 1.6
-script: sudo -E bash -c "source /etc/profile && eval '$(gimme 1.6)' && export GOPATH=$HOME/gopath:$GOPATH && go get && GORACE='halt_on_error=1' go test ./... -v -timeout 120s -race"
+go: 1.14
+
+script: sudo -E bash -c "source /etc/profile && eval '$(gimme 1.14)' && go test ./... -v -timeout 120s"
 
 before_install:
-  - sudo apt-get install libpcap-dev -y
+  - sudo apt-get install libpcap-dev -y

Dockerfile

@@ -1,8 +1,8 @@
 FROM alpine:latest as builder
 RUN apk add --no-cache ca-certificates openssl
-RUN wget https://github.com/buger/goreplay/releases/download/v0.16.1/gor_0.16.1_x64.tar.gz -O gor.tar.gz
+RUN wget https://github.com/buger/goreplay/releases/download/v1.2.0/gor_v1.2.0_x64.tar.gz -O gor.tar.gz
 RUN tar xzf gor.tar.gz
 
 FROM scratch
-COPY --from=builder /goreplay .
-ENTRYPOINT ["./goreplay"]
+COPY --from=builder /gor .
+ENTRYPOINT ["./gor"]

Dockerfile.dev

@@ -1,26 +1,14 @@
-FROM golang:latest
+FROM golang:1.14
 
 RUN apt-get update && apt-get install ruby vim-common -y
 
-# Install Java for middleware testing
-RUN echo "deb http://ppa.launchpad.net/webupd8team/java/ubuntu trusty main" | tee /etc/apt/sources.list.d/webupd8team-java.list
-RUN echo "deb-src http://ppa.launchpad.net/webupd8team/java/ubuntu trusty main" | tee -a /etc/apt/sources.list.d/webupd8team-java.list
-RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys EEA14886
-RUN apt-get update -y
-RUN echo oracle-java7-installer shared/accepted-oracle-license-v1-1 select true | /usr/bin/debconf-set-selections
-RUN apt-get install oracle-java8-installer -y
-
 RUN apt-get install flex bison -y
-RUN wget http://www.tcpdump.org/release/libpcap-1.8.1.tar.gz && tar xzf libpcap-1.8.1.tar.gz && cd libpcap-1.8.1 && ./configure && make install
+RUN wget http://www.tcpdump.org/release/libpcap-1.9.1.tar.gz && tar xzf libpcap-1.9.1.tar.gz && cd libpcap-1.9.1 && ./configure && make install
+
 RUN go get github.com/google/gopacket
 RUN go get -u golang.org/x/lint/golint
 
 WORKDIR /go/src/github.com/buger/goreplay/
-
-RUN wget http://archive.apache.org/dist/commons/io/binaries/commons-io-2.4-bin.tar.gz && tar xzf commons-io-2.4-bin.tar.gz && cd commons-io-2.4 && mv commons-io-2.4.jar /tmp/
-RUN wget http://archive.apache.org/dist/commons/codec/binaries/commons-codec-1.9-bin.tar.gz && tar xzf commons-codec-1.9-bin.tar.gz
-
 ADD . /go/src/github.com/buger/goreplay/
 
-RUN javac -cp commons-io-2.4/commons-io-2.4.jar -cp commons-codec-1.9/commons-codec-1.9.jar ./examples/middleware/echo.java
 RUN go get

LICENSE.txt

@@ -1,3 +1,12 @@
+Copyright (c) 2011-present Leonid Bugaev
+
+Portions of this software are licensed as follows:
+
+* All content residing under the "doc/" directory of this repository is licensed under "Creative Commons: CC BY-SA 4.0 license".
+* Files with the following names "pro.go", "s3_reader.go", "output_s3.go" are released under the commercial license specified in "COMM-LICENSE" file.
+* Content outside of the above mentioned directories or restrictions above is available under the "LGPLv3" license as defined below.
+
+
 GoReplay is an Open Source project licensed under the terms of
 the LGPLv3 license.  Please see <http://www.gnu.org/licenses/lgpl-3.0.html>
 for license text.

Makefile

@@ -2,37 +2,71 @@ SOURCE = $(shell ls -1 *.go | grep -v _test.go)
 SOURCE_PATH = /go/src/github.com/buger/goreplay/
 PORT = 8000
 FADDR = :8000
-RUN = docker run -v `pwd`:$(SOURCE_PATH) -p 0.0.0.0:$(PORT):$(PORT) -i -t gor
+CONTAINER=gor
+PREFIX=
+RUN = docker run -v `pwd`:$(SOURCE_PATH) -e AWS_ACCESS_KEY_ID=$(AWS_ACCESS_KEY_ID) -e AWS_SECRET_ACCESS_KEY=$(AWS_SECRET_ACCESS_KEY) -p 0.0.0.0:$(PORT):$(PORT) -t -i $(CONTAINER)
 BENCHMARK = BenchmarkRAWInput
 TEST = TestRawListenerBench
+BIN_NAME = gor
 VERSION = DEV-$(shell date +%s)
-LDFLAGS = -ldflags "-X main.VERSION=$(VERSION) -extldflags \"-static\""
-MAC_LDFLAGS = -ldflags "-X main.VERSION=$(VERSION)"
+LDFLAGS = -ldflags "-X main.VERSION=$(VERSION)$(PREFIX) -extldflags \"-static\" -X main.DEMO=$(DEMO)"
+MAC_LDFLAGS = -ldflags "-X main.VERSION=$(VERSION)$(PREFIX) -X main.DEMO=$(DEMO)"
 FADDR = ":8000"
 
+FPMCOMMON= \
+    --name goreplay \
+    --description "GoReplay is an open-source network monitoring tool which can record your live traffic, and use it for shadowing, load testing, monitoring and detailed analysis." \
+    -v $(VERSION) \
+    --vendor "Leonid Bugaev" \
+    -m "<support@goreplay.org>" \
+    --url "https://goreplay.org" \
+    -s dir \
+    -C /tmp/gor-build \
+
 release: release-x64 release-mac
 
 release-bin:
-	docker run -v `pwd`:$(SOURCE_PATH) -t --env GOOS=linux --env GOARCH=amd64  -i gor go build -o gor -tags netgo $(LDFLAGS)
+	docker run -v `pwd`:$(SOURCE_PATH) -t --env GOOS=linux --env GOARCH=amd64  -i $(CONTAINER) go build -o $(BIN_NAME) -tags netgo $(LDFLAGS)
+
+release-bin-mac:
+	GOOS=darwin go build -o $(BIN_NAME) $(MAC_LDFLAGS)
 
 release-x64:
-	docker run -v `pwd`:$(SOURCE_PATH) -t --env GOOS=linux --env GOARCH=amd64  -i gor go build -o gor -tags netgo $(LDFLAGS) && tar -czf gor_$(VERSION)_x64.tar.gz gor && rm gor
+	docker run -v `pwd`:$(SOURCE_PATH) -t --env GOOS=linux --env GOARCH=amd64  -i $(CONTAINER) go build -o $(BIN_NAME) -tags netgo $(LDFLAGS)
+	tar -czf gor_$(VERSION)$(PREFIX)_x64.tar.gz $(BIN_NAME)
+	mkdir -p /tmp/gor-build
+	mv ./$(BIN_NAME) /tmp/gor-build/$(BIN_NAME)
+	cd /tmp/gor-build
+	rm -f goreplay_$(VERSION)_amd64.deb
+	rm -f goreplay-$(VERSION)-1.x86_64.rpm
+	fpm $(FPMCOMMON) -a amd64 -t deb ./=/usr/local/bin
+	fpm $(FPMCOMMON) -a amd64 -t rpm ./=/usr/local/bin
+	rm -rf /tmp/gor-build
 
 release-x86:
-	docker run -v `pwd`:$(SOURCE_PATH) -t --env GOOS=linux --env GOARCH=386 -i gor go build -o gor -tags netgo $(LDFLAGS) && tar -czf gor_$(VERSION)_x86.tar.gz gor && rm gor
+	docker run -v `pwd`:$(SOURCE_PATH) -t --env GOOS=linux --env GOARCH=386 -i $(CONTAINER) go build -o $(BIN_NAME) -tags netgo $(LDFLAGS)
+	tar -czf gor_$(VERSION)$(PREFIX)_x86.tar.gz $(BIN_NAME)
+	rm $(BIN_NAME)
 
 release-mac:
-	go build $(MAC_LDFLAGS) -o gor && tar -czf gor_$(VERSION)_mac.tar.gz gor && rm gor
+	go build -o $(BIN_NAME) $(MAC_LDFLAGS)
+	tar -czf gor_$(VERSION)$(PREFIX)_mac.tar.gz $(BIN_NAME)
+	mkdir -p /tmp/gor-build
+	mv ./$(BIN_NAME) /tmp/gor-build/$(BIN_NAME)
+	cd /tmp/gor-build
+	rm -f goreplay-$(VERSION).pkg
+	fpm $(FPMCOMMON) -a amd64 -t osxpkg ./=/usr/local/bin
+	rm -rf /tmp/gor-build
+
 
 install:
 	go install $(MAC_LDFLAGS)
 
 build:
-	docker build -t gor -f Dockerfile.dev .
-
+	docker build -t $(CONTAINER) -f Dockerfile.dev .
 
 profile:
-	go build && ./gor --output-http="http://localhost:9000" --input-dummy 0 --input-raw :9000 --input-http :9000 --memprofile=./mem.out --cpuprofile=./cpu.out --stats --output-http-stats --output-http-timeout 100ms
+	go build && ./$(BIN_NAME) --output-http="http://localhost:9000" --input-dummy 0 --input-raw :9000 --input-http :9000 --memprofile=./mem.out --cpuprofile=./cpu.out --stats --output-http-stats --output-http-timeout 100ms
 
 lint:
 	$(RUN) golint $(PKG)
@@ -41,13 +75,13 @@ race:
 	$(RUN) go test ./... $(ARGS) -v -race -timeout 15s
 
 test:
-	$(RUN) go test ./. -timeout 60s $(LDFLAGS) $(ARGS)  -v
+	$(RUN) go test ./. -timeout 120s $(LDFLAGS) $(ARGS)  -v
 
 test_all:
-	$(RUN) go test ./... -timeout 60s $(LDFLAGS) $(ARGS) -v
+	$(RUN) go test ./... -timeout 120s $(LDFLAGS) $(ARGS) -v
 
 testone:
-	$(RUN) go test ./... -timeout 4s $(LDFLAGS) -run $(TEST) $(ARGS) -v
+	$(RUN) go test ./. -timeout 60s $(LDFLAGS) -run $(TEST) $(ARGS) -v
 
 cover:
 	$(RUN) go test $(ARGS) -race -v -timeout 15s -coverprofile=coverage.out
@@ -63,8 +97,8 @@ bench:
 	$(RUN) go test $(LDFLAGS) -v -run NOT_EXISTING -bench $(BENCHMARK) -benchtime 5s
 
 profile_test:
-	$(RUN) go test $(LDFLAGS) -run $(TEST) ./raw_socket_listener/. $(ARGS) -memprofile mem.mprof -cpuprofile cpu.out
-	$(RUN) go test $(LDFLAGS) -run $(TEST) ./raw_socket_listener/. $(ARGS) -c
+	$(RUN) go test $(LDFLAGS) -run $(TEST) ./capture/. $(ARGS) -memprofile mem.mprof -cpuprofile cpu.out
+	$(RUN) go test $(LDFLAGS) -run $(TEST) ./capture/. $(ARGS) -c
 
 # Used mainly for debugging, because docker container do not have access to parent machine ports
 run:
@@ -93,3 +127,20 @@ replay:
 
 bash:
 	$(RUN) /bin/bash
+
+
+FPMCOMMON= \
+    --name goreplay \
+    --description "GoReplay is an open-source network monitoring tool which can record your live traffic, and use it for shadowing, load testing, monitoring and detailed analysis." \
+    -v $(VERSION) \
+    --vendor "Leonid Bugaev" \
+    -m "<support@goreplay.org>" \
+    --url "https://goreplay.org" \
+    -s dir \
+    -C /tmp/gor-build \
+
+build_packages:
+	mkdir -p /tmp/gor-build
+	go build -i -o /tmp/gor-build/$(BIN_NAME)
+	fpm $(FPMCOMMON) -a amd64 -t deb ./=/usr/local/bin
+	fpm $(FPMCOMMON) -a amd64 -t rpm ./=/usr/local/bin

README.md

@@ -4,21 +4,18 @@
 
 ## https://goreplay.org/
 
-## We just released a new version, see announcement https://leonsbox.com/goreplay-v0-16-and-4th-anniversary-5408b1fd72e0
+GoReplay is an open-source network monitoring tool which can record your live traffic, and use it for shadowing, load testing, monitoring and detailed analysis.
 
 ## About
 
-GoReplay is the simplest and safest way to test your app using real traffic before you put it into production.
-
 As your application grows, the effort required to test it also grows exponentially. GoReplay offers you the simple idea of reusing your existing traffic for testing, which makes it incredibly powerful. Our state of art technique allows you to analyze and record your application traffic without affecting it. This eliminates the risks that come with putting a third party component in the critical path. 
 
-GoReplay increases your confidence in code deployments, configuration changes and infrastructure changes. Did we mention that no coding is required?
+GoReplay increases your confidence in code deployments, configuration and infrastructure changes.
 
 
+GoReplay offers unique approach for shadowing. Instead of being a proxy, GoReplay in background listen for traffic on your network interface, requiring no changes in your production infrastructure, rather then running GoReplay daemon on the same machine as your service.
 
-Here is basic workflow: The listener server catches http traffic and sends it to the replay server or saves to file. The replay server forwards traffic to a given address.
-
-![Diagram](http://i.imgur.com/9mqj2SK.png)
+![Diagram](https://i.imgur.com/IN2xfDm.png)
 
 Check [latest documentation](http://github.com/buger/goreplay/wiki).
 
@@ -31,6 +28,7 @@ The most basic setup will be `sudo ./gor --input-raw :8000 --output-stdout` whic
 If you already have test environment you can start replaying: `sudo ./gor --input-raw :8000 --output-http http://staging.env`.
 
 See the our [documentation](https://github.com/buger/goreplay/wiki/) and [Getting started](https://github.com/buger/goreplay/wiki/Getting-Started) page for more info. 
+
 ## Newsletter
 Subscribe to our [newsletter](https://www.getdrip.com/forms/89690474/submissions/new) to stay informed about the latest features and changes to Gor project.
 

byteutils/byteutils.go

@@ -1,6 +1,10 @@
-// Package byteutils probvides helpers for working with byte slices
+// Package byteutils provides helpers for working with byte slices
 package byteutils
 
+import (
+	"unsafe"
+)
+
 // Cut elements from slice for a given range
 func Cut(a []byte, from, to int) []byte {
 	copy(a[from:], a[to:])
@@ -41,3 +45,8 @@ func Replace(a []byte, from, to int, new []byte) []byte {
 	copy(a[from:], new)
 	return a
 }
+
+// SliceToString preferred for large body payload (zero allocation and faster)
+func SliceToString(buf []byte) string {
+	return *(*string)(unsafe.Pointer(&buf))
+}

byteutils/byteutils_test.go

@@ -30,3 +30,12 @@ func TestReplace(t *testing.T) {
 		t.Error("Should replace when replacement length bigger")
 	}
 }
+
+func BenchmarkStringtoSlice(b *testing.B) {
+	var s string
+	var buf [1 << 20]byte
+	for i := 0; i < b.N; i++ {
+		s = SliceToString(buf[:])
+	}
+	_ = s // avoid gc to optimize away the loop body
+}

capture/capture.go

@@ -0,0 +1,500 @@
+package capture
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"net"
+	"os"
+	"runtime"
+	"sync"
+	"time"
+
+	"github.com/buger/goreplay/size"
+	"github.com/google/gopacket"
+	"github.com/google/gopacket/layers"
+	"github.com/google/gopacket/pcap"
+)
+
+// Handler is a function that is used to handle packets
+type Handler func(gopacket.Packet)
+
+// PcapOptions options that can be set on a pcap capture handle,
+// these options take effect on inactive pcap handles
+type PcapOptions struct {
+	BufferTimeout time.Duration `json:"input-raw-buffer-timeout"`
+	TimestampType string        `json:"input-raw-timestamp-type"`
+	BPFFilter     string        `json:"input-raw-bpf-filter"`
+	BufferSize    size.Size     `json:"input-raw-buffer-size"`
+	Promiscuous   bool          `json:"input-raw-promisc"`
+	Monitor       bool          `json:"input-raw-monitor"`
+	Snaplen       bool          `json:"input-raw-override-snaplen"`
+}
+
+// NetInterface represents network interface
+type NetInterface struct {
+	net.Interface
+	IPs []string
+}
+
+// Listener handle traffic capture, this is its representation.
+type Listener struct {
+	sync.Mutex
+	Transport  string       // transport layer default to tcp
+	Activate   func() error // function is used to activate the engine. it must be called before reading packets
+	Handles    map[string]gopacket.PacketDataSource
+	Interfaces []NetInterface
+	loopIndex  int
+	Reading    chan bool // this channel is closed when the listener has started reading packets
+	PcapOptions
+	Engine        EngineType
+	port          uint16 // src or/and dst port
+	trackResponse bool
+
+	host string // pcap file name or interface (name, hardware addr, index or ip address)
+
+	quit    chan bool
+	packets chan gopacket.Packet
+}
+
+// EngineType ...
+type EngineType uint8
+
+// Available engines for intercepting traffic
+const (
+	EnginePcap EngineType = 1 << iota
+	EnginePcapFile
+	EngineRawSocket
+)
+
+// Set is here so that EngineType can implement flag.Var
+func (eng *EngineType) Set(v string) error {
+	switch v {
+	case "", "libpcap":
+		*eng = EnginePcap
+	case "pcap_file":
+		*eng = EnginePcapFile
+	case "raw_socket", "af_packet":
+		*eng = EngineRawSocket
+	default:
+		return fmt.Errorf("invalid engine %s", v)
+	}
+	return nil
+}
+
+func (eng *EngineType) String() (e string) {
+	switch *eng {
+	case EnginePcapFile:
+		e = "pcap_file"
+	case EnginePcap:
+		e = "libpcap"
+	case EngineRawSocket:
+		e = "raw_socket"
+	default:
+		e = ""
+	}
+	return e
+}
+
+// NewListener creates and initialize a new Listener. if transport or/and engine are invalid/unsupported
+// is "tcp" and "pcap", are assumed. l.Engine and l.Transport can help to get the values used.
+// if there is an error it will be associated with getting network interfaces
+func NewListener(host string, port uint16, transport string, engine EngineType, trackResponse bool) (l *Listener, err error) {
+	l = &Listener{}
+
+	l.host = host
+	l.port = port
+	l.Transport = "tcp"
+	if transport != "" {
+		l.Transport = transport
+	}
+	l.Handles = make(map[string]gopacket.PacketDataSource)
+	l.trackResponse = trackResponse
+	l.packets = make(chan gopacket.Packet, 1000)
+	l.quit = make(chan bool, 1)
+	l.Reading = make(chan bool, 1)
+	switch engine {
+	default:
+		l.Engine = EnginePcap
+		l.Activate = l.activatePcap
+	case EngineRawSocket:
+		l.Engine = EngineRawSocket
+		l.Activate = l.activateRawSocket
+	case EnginePcapFile:
+		l.Engine = EnginePcapFile
+		l.Activate = l.activatePcapFile
+		return
+	}
+	err = l.setInterfaces()
+	if err != nil {
+		return nil, err
+	}
+	return
+}
+
+// SetPcapOptions set pcap options for all yet to be actived pcap handles
+// setting this on already activated handles will not have any effect
+func (l *Listener) SetPcapOptions(opts PcapOptions) {
+	l.PcapOptions = opts
+}
+
+// Listen listens for packets from the handles, and call handler on every packet received
+// until the context done signal is sent or EOF on handles.
+// this function should be called after activating pcap handles
+func (l *Listener) Listen(ctx context.Context, handler Handler) (err error) {
+	l.read()
+	done := ctx.Done()
+	var p gopacket.Packet
+	var ok bool
+	for {
+		select {
+		case <-done:
+			l.quit <- true
+			close(l.quit)
+			err = ctx.Err()
+			done = nil
+		case p, ok = <-l.packets:
+			if !ok {
+				return
+			}
+			if p == nil {
+				continue
+			}
+			handler(p)
+		}
+	}
+}
+
+// ListenBackground is like listen but can run concurrently and signal error through channel
+func (l *Listener) ListenBackground(ctx context.Context, handler Handler) chan error {
+	err := make(chan error, 1)
+	go func() {
+		defer close(err)
+		if e := l.Listen(ctx, handler); err != nil {
+			err <- e
+		}
+	}()
+	return err
+}
+
+// Filter returns automatic filter applied by goreplay
+// to a pcap handle of a specific interface
+func (l *Listener) Filter(ifi NetInterface) (filter string) {
+	// https://www.tcpdump.org/manpages/pcap-filter.7.html
+
+	port := fmt.Sprintf("portrange 0-%d", 1<<16-1)
+	if l.port != 0 {
+		port = fmt.Sprintf("port %d", l.port)
+	}
+	dir := " dst " // direction
+	if l.trackResponse {
+		dir = " "
+	}
+	filter = fmt.Sprintf("(%s%s%s)", l.Transport, dir, port)
+	if listenAll(l.host) || isDevice(l.host, ifi) {
+		return
+	}
+	filter = fmt.Sprintf("(%s%s%s and host %s)", l.Transport, dir, port, l.host)
+	return
+}
+
+// PcapDumpHandler returns a handler to write packet data in PCAP
+// format, See http://wiki.wireshark.org/Development/LibpcapFileFormathandler.
+// if link layer is invalid Ethernet is assumed
+func PcapDumpHandler(file *os.File, link layers.LinkType, debugger func(int, ...interface{})) (handler func(packet gopacket.Packet), err error) {
+	if link.String() == "" {
+		link = layers.LinkTypeEthernet
+	}
+	w := NewWriterNanos(file)
+	err = w.WriteFileHeader(64<<10, link)
+	if err != nil {
+		return nil, err
+	}
+	return func(packet gopacket.Packet) {
+		err = w.WritePacket(packet.Metadata().CaptureInfo, packet.Data())
+		if err != nil && debugger != nil {
+			go debugger(3, err)
+		}
+	}, nil
+}
+
+// PcapHandle returns new pcap Handle from dev on success.
+// this function should be called after setting all necessary options for this listener
+func (l *Listener) PcapHandle(ifi NetInterface) (handle *pcap.Handle, err error) {
+	var inactive *pcap.InactiveHandle
+	inactive, err = pcap.NewInactiveHandle(ifi.Name)
+	if inactive != nil && err != nil {
+		defer inactive.CleanUp()
+	}
+	if err != nil {
+		return nil, fmt.Errorf("inactive handle error: %q, interface: %q", err, ifi.Name)
+	}
+	if l.TimestampType != "" {
+		var ts pcap.TimestampSource
+		ts, err = pcap.TimestampSourceFromString(l.TimestampType)
+		err = inactive.SetTimestampSource(ts)
+		if err != nil {
+			return nil, fmt.Errorf("%q: supported timestamps: %q, interface: %q", err, inactive.SupportedTimestamps(), ifi.Name)
+		}
+	}
+	if l.Promiscuous {
+		if err = inactive.SetPromisc(l.Promiscuous); err != nil {
+			return nil, fmt.Errorf("promiscuous mode error: %q, interface: %q", err, ifi.Name)
+		}
+	}
+	if l.Monitor {
+		if err = inactive.SetRFMon(l.Monitor); err != nil && !errors.Is(err, pcap.CannotSetRFMon) {
+			return nil, fmt.Errorf("monitor mode error: %q, interface: %q", err, ifi.Name)
+		}
+	}
+	var snap int
+	if l.Snaplen {
+		snap = 64<<10 + 200
+	} else if ifi.MTU > 0 {
+		snap = ifi.MTU + 200
+	}
+	err = inactive.SetSnapLen(snap)
+	if err != nil {
+		return nil, fmt.Errorf("snapshot length error: %q, interface: %q", err, ifi.Name)
+	}
+	if l.BufferSize > 0 {
+		err = inactive.SetBufferSize(int(l.BufferSize))
+		if err != nil {
+			return nil, fmt.Errorf("handle buffer size error: %q, interface: %q", err, ifi.Name)
+		}
+	}
+	if l.BufferTimeout == 0 {
+		l.BufferTimeout = pcap.BlockForever
+	}
+	err = inactive.SetTimeout(l.BufferTimeout)
+	if err != nil {
+		return nil, fmt.Errorf("handle buffer timeout error: %q, interface: %q", err, ifi.Name)
+	}
+	handle, err = inactive.Activate()
+	if err != nil {
+		return nil, fmt.Errorf("PCAP Activate device error: %q, interface: %q", err, ifi.Name)
+	}
+	if l.BPFFilter != "" {
+		if l.BPFFilter[0] != '(' || l.BPFFilter[len(l.BPFFilter)-1] != ')' {
+			l.BPFFilter = "(" + l.BPFFilter + ")"
+		}
+	} else {
+		l.BPFFilter = l.Filter(ifi)
+	}
+	err = handle.SetBPFFilter(l.BPFFilter)
+	if err != nil {
+		handle.Close()
+		return nil, fmt.Errorf("BPF filter error: %q%s, interface: %q", err, l.BPFFilter, ifi.Name)
+	}
+	return
+}
+
+// SocketHandle returns new unix ethernet handle associated with this listener settings
+func (l *Listener) SocketHandle(ifi NetInterface) (handle Socket, err error) {
+	handle, err = NewSocket(ifi.Interface)
+	if err != nil {
+		return nil, fmt.Errorf("sock raw error: %q, interface: %q", err, ifi.Name)
+	}
+	if err = handle.SetPromiscuous(l.Promiscuous || l.Monitor); err != nil {
+		return nil, fmt.Errorf("promiscuous mode error: %q, interface: %q", err, ifi.Name)
+	}
+	if l.BPFFilter != "" {
+		if l.BPFFilter[0] != '(' || l.BPFFilter[len(l.BPFFilter)-1] != ')' {
+			l.BPFFilter = "(" + l.BPFFilter + ")"
+		}
+	} else {
+		l.BPFFilter = l.Filter(ifi)
+	}
+	if err = handle.SetBPFFilter(l.BPFFilter); err != nil {
+		handle.Close()
+		return nil, fmt.Errorf("BPF filter error: %q%s, interface: %q", err, l.BPFFilter, ifi.Name)
+	}
+	handle.SetLoopbackIndex(int32(l.loopIndex))
+	return
+}
+
+func (l *Listener) read() {
+	l.Lock()
+	defer l.Unlock()
+	for key, handle := range l.Handles {
+		var source *gopacket.PacketSource
+		linkType := layers.LinkTypeEthernet
+		if _, ok := handle.(*pcap.Handle); ok {
+			linkType = handle.(*pcap.Handle).LinkType()
+		}
+		source = gopacket.NewPacketSource(handle, linkType)
+		source.Lazy = true
+		source.NoCopy = true
+		ch := source.Packets()
+		go func(key string) {
+			defer l.closeHandles(key)
+			for {
+				select {
+				case <-l.quit:
+					return
+				case p, ok := <-ch:
+					if !ok {
+						return
+					}
+					l.packets <- p
+				}
+			}
+		}(key)
+	}
+	l.Reading <- true
+	close(l.Reading)
+}
+
+func (l *Listener) closeHandles(key string) {
+	l.Lock()
+	defer l.Unlock()
+	if handle, ok := l.Handles[key]; ok {
+		if _, ok = handle.(Socket); ok {
+			handle.(Socket).Close()
+		} else {
+			handle.(*pcap.Handle).Close()
+		}
+		delete(l.Handles, key)
+		if len(l.Handles) == 0 {
+			close(l.packets)
+		}
+	}
+}
+
+func (l *Listener) activatePcap() error {
+	var e error
+	var msg string
+	for _, ifi := range l.Interfaces {
+		var handle *pcap.Handle
+		handle, e = l.PcapHandle(ifi)
+		if e != nil {
+			msg += ("\n" + e.Error())
+			continue
+		}
+		l.Handles[ifi.Name] = handle
+	}
+	if len(l.Handles) == 0 {
+		return fmt.Errorf("pcap handles error:%s", msg)
+	}
+	return nil
+}
+
+func (l *Listener) activateRawSocket() error {
+	if runtime.GOOS != "linux" {
+		return fmt.Errorf("sock_raw is not stabilized on OS other than linux")
+	}
+	var msg string
+	var e error
+	for _, ifi := range l.Interfaces {
+		var handle Socket
+		handle, e = l.SocketHandle(ifi)
+		if e != nil {
+			msg += ("\n" + e.Error())
+			continue
+		}
+		l.Handles[ifi.Name] = handle
+	}
+	return e
+}
+
+func (l *Listener) activatePcapFile() (err error) {
+	var handle *pcap.Handle
+	var e error
+	if handle, e = pcap.OpenOffline(l.host); e != nil {
+		return fmt.Errorf("open pcap file error: %q", e)
+	}
+	if l.BPFFilter != "" {
+		if l.BPFFilter[0] != '(' || l.BPFFilter[len(l.BPFFilter)-1] != ')' {
+			l.BPFFilter = "(" + l.BPFFilter + ")"
+		}
+	} else {
+		addr := l.host
+		l.host = ""
+		l.BPFFilter = l.Filter(NetInterface{})
+		l.host = addr
+	}
+	if e = handle.SetBPFFilter(l.BPFFilter); e != nil {
+		handle.Close()
+		return fmt.Errorf("BPF filter error: %q, filter: %s", e, l.BPFFilter)
+	}
+	l.Handles["pcap_file"] = handle
+	return
+}
+
+func (l *Listener) setInterfaces() (err error) {
+	var Ifis []NetInterface
+	var ifis []net.Interface
+	ifis, err = net.Interfaces()
+	if err != nil {
+		return err
+	}
+
+	for i := 0; i < len(ifis); i++ {
+		if ifis[i].Flags&net.FlagLoopback != 0 {
+			l.loopIndex = ifis[i].Index
+		}
+		if ifis[i].Flags&net.FlagUp == 0 {
+			continue
+		}
+		var addrs []net.Addr
+		addrs, err = ifis[i].Addrs()
+		if err != nil {
+			return err
+		}
+		if len(addrs) == 0 {
+			continue
+		}
+		ifi := NetInterface{}
+		ifi.Interface = ifis[i]
+		ifi.IPs = make([]string, len(addrs))
+		for j, addr := range addrs {
+			ifi.IPs[j] = cutMask(addr)
+		}
+		Ifis = append(Ifis, ifi)
+	}
+
+	if listenAll(l.host) {
+		l.Interfaces = Ifis
+		return
+	}
+	found := false
+	for _, ifi := range Ifis {
+		if isDevice(l.host, ifi) {
+			found = true
+		}
+		for _, ip := range ifi.IPs {
+			if ip == l.host {
+				found = true
+				break
+			}
+		}
+		if found {
+			l.Interfaces = []NetInterface{ifi}
+			return
+		}
+	}
+	err = fmt.Errorf("can not find interface with addr, name or index %s", l.host)
+	return err
+}
+
+func cutMask(addr net.Addr) string {
+	mask := addr.String()
+	for i, v := range mask {
+		if v == '/' {
+			return mask[:i]
+		}
+	}
+	return mask
+}
+
+func isDevice(addr string, ifi NetInterface) bool {
+	return addr == ifi.Name || addr == fmt.Sprintf("%d", ifi.Index) || addr == ifi.HardwareAddr.String()
+}
+
+func listenAll(addr string) bool {
+	switch addr {
+	case "", "0.0.0.0", "[::]", "::":
+		return true
+	}
+	return false
+}

capture/capture_test.go

@@ -0,0 +1,297 @@
+package capture
+
+import (
+	"context"
+	"encoding/binary"
+	"io/ioutil"
+	"net"
+	"os"
+	"testing"
+	"time"
+
+	"github.com/google/gopacket"
+	"github.com/google/gopacket/layers"
+	"github.com/google/gopacket/pcap"
+)
+
+var LoopBack = func() net.Interface {
+	ifis, _ := net.Interfaces()
+	for _, v := range ifis {
+		if v.Flags&net.FlagLoopback != 0 {
+			return v
+		}
+	}
+	return ifis[0]
+}()
+
+func TestSetInterfaces(t *testing.T) {
+	l := &Listener{}
+	l.host = "127.0.0.1"
+	l.setInterfaces()
+	if len(l.Interfaces) != 1 {
+		t.Error("expected a single interface")
+	}
+	l.host = LoopBack.HardwareAddr.String()
+	l.setInterfaces()
+	if l.Interfaces[0].Name != LoopBack.Name && len(l.Interfaces) != 1 {
+		t.Error("interface should be loop back interface")
+	}
+	l.host = ""
+	l.setInterfaces()
+	if len(l.Interfaces) < 1 {
+		t.Error("should get all interfaces")
+	}
+}
+
+func TestBPFFilter(t *testing.T) {
+	l := &Listener{}
+	l.host = "127.0.0.1"
+	l.Transport = "tcp"
+	l.setInterfaces()
+	filter := l.Filter(l.Interfaces[0])
+	if filter != "(tcp dst portrange 0-65535 and host 127.0.0.1)" {
+		t.Error("wrong filter", filter)
+	}
+	l.port = 8000
+	l.trackResponse = true
+	filter = l.Filter(l.Interfaces[0])
+	if filter != "(tcp port 8000 and host 127.0.0.1)" {
+		t.Error("wrong filter")
+	}
+}
+
+var decodeOpts = gopacket.DecodeOptions{Lazy: true, NoCopy: true}
+
+func generateHeaders(seq uint32, length uint16) (headers [44]byte) {
+	// set ethernet headers
+	binary.BigEndian.PutUint32(headers[0:4], uint32(layers.ProtocolFamilyIPv4))
+
+	// set ip header
+	ip := headers[4:]
+	copy(ip[0:2], []byte{4<<4 | 5, 0x28<<2 | 0x00})
+	binary.BigEndian.PutUint16(ip[2:4], length+54)
+	ip[9] = uint8(layers.IPProtocolTCP)
+	copy(ip[12:16], []byte{127, 0, 0, 1})
+	copy(ip[16:], []byte{127, 0, 0, 1})
+
+	// set tcp header
+	tcp := ip[20:]
+	binary.BigEndian.PutUint16(tcp[0:2], 45678)
+	binary.BigEndian.PutUint16(tcp[2:4], 8000)
+	tcp[12] = 5 << 4
+	return
+}
+
+func randomPackets(start uint32, _len int, length uint16) []gopacket.Packet {
+	var packets = make([]gopacket.Packet, _len)
+	for i := start; i < start+uint32(_len); i++ {
+		h := generateHeaders(i, length)
+		d := make([]byte, int(length)+len(h))
+		copy(d, h[0:])
+		packet := gopacket.NewPacket(d, layers.LinkTypeLoop, decodeOpts)
+		packets[i-start] = packet
+		inf := packets[i-start].Metadata()
+		_len := len(d)
+		inf.CaptureInfo = gopacket.CaptureInfo{CaptureLength: _len, Length: _len, Timestamp: time.Now()}
+	}
+	return packets
+}
+
+func TestPcapDump(t *testing.T) {
+	f, err := ioutil.TempFile("", "pcap_file")
+	if err != nil {
+		t.Error(err)
+	}
+	waiter := make(chan bool, 1)
+	h, _ := PcapDumpHandler(f, layers.LinkTypeLoop, func(level int, a ...interface{}) {
+		if level != 3 {
+			t.Errorf("expected debug level to be 3, got %d", level)
+		}
+		waiter <- true
+	})
+	packets := randomPackets(1, 5, 5)
+	for i := 0; i < len(packets); i++ {
+		if i == 1 {
+			tcp := packets[i].Data()[4:][20:]
+			// change dst port
+			binary.BigEndian.PutUint16(tcp[2:], 8001)
+		}
+		if i == 4 {
+			inf := packets[i].Metadata()
+			inf.CaptureLength = 40
+		}
+		h(packets[i])
+	}
+	<-waiter
+	name := f.Name()
+	f.Close()
+	testPcapDumpEngine(name, t)
+}
+
+func testPcapDumpEngine(f string, t *testing.T) {
+	defer os.Remove(f)
+	l, err := NewListener(f, 8000, "", EnginePcapFile, true)
+	err = l.Activate()
+	if err != nil {
+		t.Errorf("expected error to be nil, got %q", err)
+		return
+	}
+	pckts := 0
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	err = l.Listen(ctx, func(packet gopacket.Packet) {
+		if packet.Metadata().CaptureLength != 49 {
+			t.Errorf("expected packet length to be %d, got %d", 49, packet.Metadata().CaptureLength)
+		}
+		pckts++
+	})
+
+	if err != nil {
+		t.Errorf("expected error to be nil, got %q", err)
+	}
+	if pckts != 3 {
+		t.Errorf("expected %d packets, got %d packets", 3, pckts)
+	}
+}
+
+func TestPcapHandler(t *testing.T) {
+	l, err := NewListener(LoopBack.Name, 8000, "", EnginePcap, true)
+	if err != nil {
+		t.Errorf("expected error to be nil, got %v", err)
+		return
+	}
+	err = l.Activate()
+	if err != nil {
+		t.Errorf("expected error to be nil, got %v", err)
+		return
+	}
+	defer l.Handles[LoopBack.Name].(*pcap.Handle).Close()
+	if err != nil {
+		t.Errorf("expected error to be nil, got %v", err)
+		return
+	}
+	for i := 0; i < 5; i++ {
+		_, _ = net.Dial("tcp", "127.0.0.1:8000")
+	}
+	sts, _ := l.Handles[LoopBack.Name].(*pcap.Handle).Stats()
+	if sts.PacketsReceived < 5 {
+		t.Errorf("expected >=5 packets got %d", sts.PacketsReceived)
+	}
+}
+
+func TestSocketHandler(t *testing.T) {
+	l, err := NewListener(LoopBack.Name, 8000, "", EngineRawSocket, true)
+	err = l.Activate()
+	if err != nil {
+		return
+	}
+	defer l.Handles[LoopBack.Name].(*SockRaw).Close()
+	if err != nil {
+		t.Errorf("expected error to be nil, got %v", err)
+		return
+	}
+	for i := 0; i < 5; i++ {
+		_, _ = net.Dial("tcp", "127.0.0.1:8000")
+	}
+	sts, _ := l.Handles[LoopBack.Name].(*SockRaw).Stats()
+	if sts.Packets < 5 {
+		t.Errorf("expected >=5 packets got %d", sts.Packets)
+	}
+}
+
+func BenchmarkPcapDump(b *testing.B) {
+	f, err := ioutil.TempFile("", "pcap_file")
+	if err != nil {
+		b.Error(err)
+		return
+	}
+	now := time.Now()
+	defer os.Remove(f.Name())
+	h, _ := PcapDumpHandler(f, layers.LinkTypeLoop, nil)
+	packets := randomPackets(1, b.N, 5)
+	for i := 0; i < len(packets); i++ {
+		h(packets[i])
+	}
+	f.Close()
+	b.Logf("%d packets in %s", b.N, time.Since(now))
+}
+
+func BenchmarkPcapFile(b *testing.B) {
+	f, err := ioutil.TempFile("", "pcap_file")
+	if err != nil {
+		b.Error(err)
+		return
+	}
+	defer os.Remove(f.Name())
+	h, _ := PcapDumpHandler(f, layers.LinkTypeLoop, nil)
+	packets := randomPackets(1, b.N, 5)
+	for i := 0; i < len(packets); i++ {
+		h(packets[i])
+	}
+	name := f.Name()
+	f.Close()
+	b.ResetTimer()
+	var l *Listener
+	l, err = NewListener(name, 8000, "", EnginePcapFile, true)
+	if err != nil {
+		b.Error(err)
+		return
+	}
+	err = l.Activate()
+	if err != nil {
+		b.Error(err)
+		return
+	}
+	now := time.Now()
+	pckts := 0
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	if err = l.Listen(ctx, func(packet gopacket.Packet) {
+		if packet.Metadata().CaptureLength != 49 {
+			b.Errorf("expected packet length to be %d, got %d", 49, packet.Metadata().CaptureLength)
+		}
+		pckts++
+	}); err != nil {
+		b.Error(err)
+	}
+	b.Logf("%d/%d packets in %s", pckts, b.N, time.Since(now))
+}
+
+func BenchmarkPcap(b *testing.B) {
+	now := time.Now()
+	var err error
+
+	l, err := NewListener(LoopBack.Name, 8000, "", EnginePcap, true)
+	if err != nil {
+		b.Errorf("expected error to be nil, got %v", err)
+		return
+	}
+	err = l.Activate()
+	if err != nil {
+		b.Errorf("expected error to be nil, got %v", err)
+		return
+	}
+	defer l.Handles[LoopBack.Name].(*pcap.Handle).Close()
+	for i := 0; i < b.N; i++ {
+		_, _ = net.Dial("tcp", "127.0.0.1:8000")
+	}
+	sts, _ := l.Handles[LoopBack.Name].(*pcap.Handle).Stats()
+	b.Logf("%d packets in %s", sts.PacketsReceived, time.Since(now))
+}
+
+func BenchmarkRawSocket(b *testing.B) {
+	now := time.Now()
+	var err error
+
+	l, err := NewListener(LoopBack.Name, 0, "", EngineRawSocket, true)
+	err = l.Activate()
+	if err != nil {
+		return
+	}
+	defer l.Handles[LoopBack.Name].(*SockRaw).Close()
+	for i := 0; i < b.N; i++ {
+		_, _ = net.Dial("tcp", "127.0.0.1:8000")
+	}
+	sts, _ := l.Handles[LoopBack.Name].(*SockRaw).Stats()
+	b.Logf("%d packets in %s", sts.Packets, time.Since(now))
+}

capture/doc.go

@@ -0,0 +1,35 @@
+/*
+Package capture provides traffic sniffier using AF_PACKET, pcap or pcap file.
+it allows you to listen for traffic from any port (e.g. sniffing) because they operate on IP level.
+Ports is TCP/IP feature, same as flow control, reliable transmission and etc.
+Currently this package implements TCP layer: flow control is managed under tcp package.
+BPF filters can also be applied.
+
+example:
+
+// for the transport should be "tcp"
+listener, err := capture.NewListener(host, port, transport, engine, trackResponse)
+if err != nil {
+	// handle error
+}
+listener.SetPcapOptions(opts)
+err = listner.Activate()
+if err != nil {
+	// handle it
+}
+
+if err := listener.Listen(context.Background(), handler); err != nil {
+	 // handle error
+}
+// or
+errCh := listener.ListenBackground(context.Background(), handler) // runs in the backgorund
+select {
+case err := <- errCh:
+	// handle error
+case <-quit:
+	//
+case <- l.Reading: // if we have started reading
+}
+
+*/
+package capture // import github.com/buger/goreplay/capture

capture/dump.go

@@ -0,0 +1,126 @@
+// https://github.com/google/gopacket/blob/403ca653c4/pcapgo/read.go
+
+package capture
+
+import (
+	"encoding/binary"
+	"fmt"
+	"io"
+	"time"
+
+	"github.com/google/gopacket"
+	"github.com/google/gopacket/layers"
+)
+
+// Writer wraps an underlying io.Writer to write packet data in PCAP
+// format.  See http://wiki.wireshark.org/Development/LibpcapFileFormat
+// for information on the file format.
+//
+// For those that care, we currently write v2.4 files with nanosecond
+// or microsecond timestamp resolution and little-endian encoding.
+type Writer struct {
+	w        io.Writer
+	tsScaler int
+	// Moving this into the struct seems to save an allocation for each call to writePacketHeader
+	buf [16]byte
+}
+
+const magicNanoseconds = 0xA1B23C4D
+const magicMicroseconds = 0xA1B2C3D4
+const versionMajor = 2
+const versionMinor = 4
+
+// NewWriterNanos returns a new writer object, for writing packet data out
+// to the given writer.  If this is a new empty writer (as opposed to
+// an append), you must call WriteFileHeader before WritePacket.  Packet
+// timestamps are written with nanosecond precision.
+//
+//  // Write a new file:
+//  f, _ := os.Create("/tmp/file.pcap")
+//  w := pcapgo.NewWriterNanos(f)
+//  w.WriteFileHeader(65536, layers.LinkTypeEthernet)  // new file, must do this.
+//  w.WritePacket(gopacket.CaptureInfo{...}, data1)
+//  f.Close()
+//  // Append to existing file (must have same snaplen and linktype)
+//  f2, _ := os.OpenFile("/tmp/fileNano.pcap", os.O_APPEND, 0700)
+//  w2 := pcapgo.NewWriter(f2)
+//  // no need for file header, it's already written.
+//  w2.WritePacket(gopacket.CaptureInfo{...}, data2)
+//  f2.Close()
+func NewWriterNanos(w io.Writer) *Writer {
+	return &Writer{w: w, tsScaler: nanosPerNano}
+}
+
+// NewWriter returns a new writer object, for writing packet data out
+// to the given writer.  If this is a new empty writer (as opposed to
+// an append), you must call WriteFileHeader before WritePacket.
+// Packet timestamps are written witn microsecond precision.
+//
+//  // Write a new file:
+//  f, _ := os.Create("/tmp/file.pcap")
+//  w := pcapgo.NewWriter(f)
+//  w.WriteFileHeader(65536, layers.LinkTypeEthernet)  // new file, must do this.
+//  w.WritePacket(gopacket.CaptureInfo{...}, data1)
+//  f.Close()
+//  // Append to existing file (must have same snaplen and linktype)
+//  f2, _ := os.OpenFile("/tmp/file.pcap", os.O_APPEND, 0700)
+//  w2 := pcapgo.NewWriter(f2)
+//  // no need for file header, it's already written.
+//  w2.WritePacket(gopacket.CaptureInfo{...}, data2)
+//  f2.Close()
+func NewWriter(w io.Writer) *Writer {
+	return &Writer{w: w, tsScaler: nanosPerMicro}
+}
+
+// WriteFileHeader writes a file header out to the writer.
+// This must be called exactly once per output.
+func (w *Writer) WriteFileHeader(snaplen uint32, linktype layers.LinkType) error {
+	var buf [24]byte
+	if w.tsScaler == nanosPerMicro {
+		binary.LittleEndian.PutUint32(buf[0:4], magicMicroseconds)
+	} else {
+		binary.LittleEndian.PutUint32(buf[0:4], magicNanoseconds)
+	}
+	binary.LittleEndian.PutUint16(buf[4:6], versionMajor)
+	binary.LittleEndian.PutUint16(buf[6:8], versionMinor)
+	// bytes 8:12 stay 0 (timezone = UTC)
+	// bytes 12:16 stay 0 (sigfigs is always set to zero, according to
+	//   http://wiki.wireshark.org/Development/LibpcapFileFormat
+	binary.LittleEndian.PutUint32(buf[16:20], snaplen)
+	binary.LittleEndian.PutUint32(buf[20:24], uint32(linktype))
+	_, err := w.w.Write(buf[:])
+	return err
+}
+
+const nanosPerMicro = 1000
+const nanosPerNano = 1
+
+func (w *Writer) writePacketHeader(ci gopacket.CaptureInfo) error {
+	t := ci.Timestamp
+	if t.IsZero() {
+		t = time.Now()
+	}
+	secs := t.Unix()
+	usecs := t.Nanosecond() / w.tsScaler
+	binary.LittleEndian.PutUint32(w.buf[0:4], uint32(secs))
+	binary.LittleEndian.PutUint32(w.buf[4:8], uint32(usecs))
+	binary.LittleEndian.PutUint32(w.buf[8:12], uint32(ci.CaptureLength))
+	binary.LittleEndian.PutUint32(w.buf[12:16], uint32(ci.Length))
+	_, err := w.w.Write(w.buf[:])
+	return err
+}
+
+// WritePacket writes the given packet data out to the file.
+func (w *Writer) WritePacket(ci gopacket.CaptureInfo, data []byte) error {
+	if ci.CaptureLength != len(data) {
+		return fmt.Errorf("capture length %d does not match data length %d", ci.CaptureLength, len(data))
+	}
+	if ci.CaptureLength > ci.Length {
+		return fmt.Errorf("invalid capture info %+v:  capture length > length", ci)
+	}
+	if err := w.writePacketHeader(ci); err != nil {
+		return fmt.Errorf("error writing packet header: %v", err)
+	}
+	_, err := w.w.Write(data)
+	return err
+}

capture/sock_linux.go

@@ -0,0 +1,260 @@
+package capture
+
+import (
+	"fmt"
+	"net"
+	"sync"
+	"time"
+	"unsafe"
+
+	"golang.org/x/sys/unix"
+
+	"github.com/google/gopacket"
+	"github.com/google/gopacket/layers"
+	"github.com/google/gopacket/pcap"
+)
+
+const (
+	// ETHALL htons(ETH_P_ALL)
+	ETHALL uint16 = unix.ETH_P_ALL<<8 | unix.ETH_P_ALL>>8
+	// BLOCKSIZE ring buffer block_size
+	BLOCKSIZE = 64 << 10
+	// BLOCKNR ring buffer block_nr
+	BLOCKNR = (2 << 20) / BLOCKSIZE // 2mb / 64kb
+	// FRAMESIZE ring buffer frame_size
+	FRAMESIZE = BLOCKSIZE
+	// FRAMENR ring buffer frame_nr
+	FRAMENR = BLOCKNR * BLOCKSIZE / FRAMESIZE
+	// MAPHUGE2MB 2mb huge map
+	MAPHUGE2MB = 21 << unix.MAP_HUGE_SHIFT
+)
+
+var tpacket2hdrlen = tpAlign(int(unsafe.Sizeof(unix.Tpacket2Hdr{})))
+
+// SockRaw is a linux M'maped af_packet socket
+type SockRaw struct {
+	mu          sync.Mutex
+	fd          int
+	ifindex     int
+	snaplen     int
+	pollTimeout uintptr
+	frame       uint32 // current frame
+	buf         []byte // points to the memory space of the ring buffer shared with the kernel.
+	loopIndex   int32  // this field must filled to avoid reading packet twice on a loopback device
+}
+
+// NewSocket returns new M'maped sock_raw on packet version 2.
+func NewSocket(ifi net.Interface) (*SockRaw, error) {
+	// sock create
+	fd, err := unix.Socket(unix.AF_PACKET, unix.SOCK_RAW, int(ETHALL))
+	if err != nil {
+		return nil, err
+	}
+	sock := &SockRaw{
+		fd:          fd,
+		ifindex:     ifi.Index,
+		snaplen:     FRAMESIZE,
+		pollTimeout: ^uintptr(0),
+	}
+
+	// set packet version
+	err = unix.SetsockoptInt(fd, unix.SOL_PACKET, unix.PACKET_VERSION, unix.TPACKET_V2)
+	if err != nil {
+		unix.Close(fd)
+		return nil, fmt.Errorf("setsockopt packet_version: %v", err)
+	}
+
+	// bind to interface
+	addr := unix.RawSockaddrLinklayer{
+		Family:   unix.AF_PACKET,
+		Protocol: ETHALL,
+		Ifindex:  int32(ifi.Index),
+	}
+	_, _, e := unix.Syscall(
+		unix.SYS_BIND,
+		uintptr(fd),
+		uintptr(unsafe.Pointer(&addr)),
+		uintptr(unix.SizeofSockaddrLinklayer),
+	)
+	if e != 0 {
+		unix.Close(fd)
+		return nil, e
+	}
+
+	// create shared-memory ring buffer
+	tp := &unix.TpacketReq{
+		Block_size: BLOCKSIZE,
+		Block_nr:   BLOCKNR,
+		Frame_size: FRAMESIZE,
+		Frame_nr:   FRAMENR,
+	}
+	err = unix.SetsockoptTpacketReq(sock.fd, unix.SOL_PACKET, unix.PACKET_RX_RING, tp)
+	if err != nil {
+		unix.Close(fd)
+		return nil, fmt.Errorf("setsockopt packet_rx_ring: %v", err)
+	}
+	sock.buf, err = unix.Mmap(
+		sock.fd,
+		0,
+		BLOCKSIZE*BLOCKNR,
+		unix.PROT_READ|unix.PROT_WRITE,
+		unix.MAP_SHARED|MAPHUGE2MB,
+	)
+	if err != nil {
+		unix.Close(fd)
+		return nil, fmt.Errorf("socket mmap error: %v", err)
+	}
+	return sock, nil
+}
+
+// ReadPacketData implements gopacket.PacketDataSource.
+func (sock *SockRaw) ReadPacketData() (buf []byte, ci gopacket.CaptureInfo, err error) {
+	sock.mu.Lock()
+	defer sock.mu.Unlock()
+	var tpHdr *unix.Tpacket2Hdr
+	poll := &unix.PollFd{
+		Fd:     int32(sock.fd),
+		Events: unix.POLLIN,
+	}
+	var i int
+read:
+	i = int(sock.frame * FRAMESIZE)
+	tpHdr = (*unix.Tpacket2Hdr)(unsafe.Pointer(&sock.buf[i]))
+	sock.frame = (sock.frame + 1) % FRAMENR
+
+	if tpHdr.Status&unix.TP_STATUS_USER == 0 {
+		_, _, e := unix.Syscall(unix.SYS_POLL, uintptr(unsafe.Pointer(poll)), 1, sock.pollTimeout)
+		if e != 0 && e != unix.EINTR {
+			return buf, ci, e
+		}
+		// it might be some other frame with data!
+		if tpHdr.Status&unix.TP_STATUS_USER == 0 {
+			goto read
+		}
+	}
+	tpHdr.Status = unix.TP_STATUS_KERNEL
+	sockAddr := (*unix.RawSockaddrLinklayer)(unsafe.Pointer(&sock.buf[i+tpacket2hdrlen]))
+
+	// parse out repeating packets on loopback
+	if sockAddr.Ifindex == sock.loopIndex && sock.frame%2 != 0 {
+		goto read
+	}
+
+	ci.Length = int(tpHdr.Len)
+	ci.Timestamp = time.Unix(int64(tpHdr.Sec), int64(tpHdr.Nsec))
+	ci.InterfaceIndex = int(sockAddr.Ifindex)
+	buf = make([]byte, tpHdr.Snaplen)
+	ci.CaptureLength = copy(buf, sock.buf[i+int(tpHdr.Mac):])
+
+	return
+}
+
+// Close closes the underlying socket
+func (sock *SockRaw) Close() (err error) {
+	sock.mu.Lock()
+	defer sock.mu.Unlock()
+	if sock.fd != -1 {
+		unix.Munmap(sock.buf)
+		sock.buf = nil
+		err = unix.Close(sock.fd)
+		sock.fd = -1
+	}
+	return
+}
+
+// SetSnapLen sets the maximum capture length to the given value.
+// for this to take effects on the kernel level SetBPFilter should be called too.
+func (sock *SockRaw) SetSnapLen(snap int) error {
+	sock.mu.Lock()
+	defer sock.mu.Unlock()
+	if snap < 0 {
+		return fmt.Errorf("expected %d snap length to be at least 0", snap)
+	}
+	if snap > FRAMESIZE {
+		snap = FRAMESIZE
+	}
+	sock.snaplen = snap
+	return nil
+}
+
+// SetTimeout sets poll wait timeout for the socket.
+// negative value will block forever
+func (sock *SockRaw) SetTimeout(t time.Duration) error {
+	sock.mu.Lock()
+	defer sock.mu.Unlock()
+	sock.pollTimeout = uintptr(t)
+	return nil
+}
+
+// GetSnapLen returns the maximum capture length
+func (sock *SockRaw) GetSnapLen() int {
+	sock.mu.Lock()
+	defer sock.mu.Unlock()
+	return sock.snaplen
+}
+
+// SetBPFFilter compiles and sets a BPF filter for the socket handle.
+func (sock *SockRaw) SetBPFFilter(expr string) error {
+	sock.mu.Lock()
+	defer sock.mu.Unlock()
+	if len(expr) == 0 {
+		return unix.SetsockoptInt(sock.fd, unix.SOL_SOCKET, unix.SO_DETACH_FILTER, 0)
+	}
+	filter, err := pcap.CompileBPFFilter(layers.LinkTypeEthernet, sock.snaplen, expr)
+	if err != nil {
+		return err
+	}
+	if len(filter) > int(^uint16(0)) {
+		return fmt.Errorf("filters out of range 0-%d", ^uint16(0))
+	}
+	if len(filter) == 0 {
+		return unix.SetsockoptInt(sock.fd, unix.SOL_SOCKET, unix.SO_DETACH_FILTER, 0)
+	}
+	fprog := &unix.SockFprog{
+		Len:    uint16(len(filter)),
+		Filter: &(*(*[]unix.SockFilter)(unsafe.Pointer(&filter)))[0],
+	}
+	return unix.SetsockoptSockFprog(sock.fd, unix.SOL_SOCKET, unix.SO_ATTACH_FILTER, fprog)
+}
+
+// SetPromiscuous sets promiscous mode to the required value. for better result capture on all interfaces instead.
+// If it is enabled, traffic not destined for the interface will also be captured.
+func (sock *SockRaw) SetPromiscuous(b bool) error {
+	sock.mu.Lock()
+	defer sock.mu.Unlock()
+	mreq := unix.PacketMreq{
+		Ifindex: int32(sock.ifindex),
+		Type:    unix.PACKET_MR_PROMISC,
+	}
+
+	opt := unix.PACKET_ADD_MEMBERSHIP
+	if !b {
+		opt = unix.PACKET_DROP_MEMBERSHIP
+	}
+
+	return unix.SetsockoptPacketMreq(sock.fd, unix.SOL_PACKET, opt, &mreq)
+}
+
+// Stats returns number of packets and dropped packets. This will be the number of packets/dropped packets since the last call to stats (not the cummulative sum!).
+func (sock *SockRaw) Stats() (*unix.TpacketStats, error) {
+	sock.mu.Lock()
+	defer sock.mu.Unlock()
+	return unix.GetsockoptTpacketStats(sock.fd, unix.SOL_PACKET, unix.PACKET_STATISTICS)
+}
+
+// SetLoopbackIndex necessary to avoid reading packet twice on a loopback device
+func (sock *SockRaw) SetLoopbackIndex(i int32) {
+	sock.mu.Lock()
+	defer sock.mu.Unlock()
+	sock.loopIndex = i
+}
+
+// WritePacketData transmits a raw packet.
+func (sock *SockRaw) WritePacketData(pkt []byte) error {
+	_, err := unix.Write(sock.fd, pkt)
+	return err
+}
+
+func tpAlign(x int) int {
+	return int((uint(x) + unix.TPACKET_ALIGNMENT - 1) &^ (unix.TPACKET_ALIGNMENT - 1))
+}

capture/sock_others.go

@@ -0,0 +1,13 @@
+// +build !linux
+
+package capture
+
+import (
+	"errors"
+	"net"
+)
+
+// NewSocket returns new M'maped sock_raw on packet version 2.
+func NewSocket(_ net.Interface) (Socket, error) {
+	return nil, errors.New("afpacket socket is only available on linux")
+}

capture/socket.go

@@ -0,0 +1,20 @@
+package capture
+
+import (
+	"time"
+
+	"github.com/google/gopacket"
+)
+
+// Socket is any interface that defines the behaviors of Socket
+type Socket interface {
+	ReadPacketData() ([]byte, gopacket.CaptureInfo, error)
+	WritePacketData([]byte) error
+	SetBPFFilter(string) error
+	SetPromiscuous(bool) error
+	SetSnapLen(int) error
+	GetSnapLen() int
+	SetTimeout(time.Duration) error
+	SetLoopbackIndex(i int32)
+	Close() error
+}

elasticsearch.go

@@ -5,12 +5,11 @@ import (
 	"log"
 	"net/url"
 	"strings"
-	//"regexp"
 	"time"
 
 	"github.com/buger/goreplay/proto"
 
-	"github.com/mattbaird/elastigo/lib"
+	elastigo "github.com/mattbaird/elastigo/lib"
 )
 
 type ESUriErorr struct{}
@@ -67,9 +66,10 @@ func parseURI(URI string) (err error, index string) {
 
 	if parseErr != nil {
 		err = new(ESUriErorr)
+		return
 	}
 
-	//	check URL validity by extracting host and undex values.
+	//	check URL validity by extracting host and index values.
 	host := parsedUrl.Host
 	urlPathParts := strings.Split(parsedUrl.Path, "/")
 	index = urlPathParts[len(urlPathParts)-1]
@@ -99,13 +99,9 @@ func (p *ESPlugin) Init(URI string) {
 	p.done = make(chan bool)
 	p.indexor.Start()
 
-	if Settings.verbose {
-		// Only start the ErrorHandler goroutine when in verbose mode
-		// no need to burn ressources otherwise
-		go p.ErrorHandler()
-	}
+	go p.ErrorHandler()
 
-	log.Println("Initialized Elasticsearch Plugin")
+	Debug(1, "Initialized Elasticsearch Plugin")
 	return
 }
 
@@ -117,7 +113,7 @@ func (p *ESPlugin) IndexerShutdown() {
 func (p *ESPlugin) ErrorHandler() {
 	for {
 		errBuf := <-p.indexor.ErrorChannel
-		log.Println(errBuf.Err)
+		Debug(1, "[ELASTICSEARCH]", errBuf.Err)
 	}
 }
 
@@ -163,7 +159,7 @@ func (p *ESPlugin) ResponseAnalyze(req, resp []byte, start, stop time.Time) {
 	}
 	j, err := json.Marshal(&esResp)
 	if err != nil {
-		log.Println(err)
+		Debug(0, "[ELASTIC-RESPONSE]", err)
 	} else {
 		p.indexor.Index(p.Index, "RequestResponse", "", "", "", &t, j)
 	}

elasticsearch_test.go

@@ -42,7 +42,7 @@ func TestElasticConnectionBuildFailWithoutScheme(t *testing.T) {
 	assertExpectedError(err, t)
 }
 
-// Argument scheme://host:port
+// Argument scheme://Host:port
 // i.e : http://localhost:9200
 // Fail : explicit index is required
 func TestElasticConnectionBuildFailWithoutIndex(t *testing.T) {
@@ -55,7 +55,7 @@ func TestElasticConnectionBuildFailWithoutIndex(t *testing.T) {
 	assertExpectedError(err, t)
 }
 
-// Argument scheme://host/index_name
+// Argument scheme://Host/index_name
 // i.e : http://localhost/gor
 func TestElasticConnectionBuildFailWithoutPort(t *testing.T) {
 	uri := "http://localhost/" + expectedIndex
@@ -67,7 +67,7 @@ func TestElasticConnectionBuildFailWithoutPort(t *testing.T) {
 	assertExpectedGorIndex(index, t)
 }
 
-// Argument scheme://host:port/index_name
+// Argument scheme://Host:port/index_name
 // i.e : http://localhost:9200/gor
 func TestElasticLocalConnectionBuild(t *testing.T) {
 	uri := "http://localhost:9200/" + expectedIndex
@@ -79,7 +79,7 @@ func TestElasticLocalConnectionBuild(t *testing.T) {
 	assertExpectedGorIndex(index, t)
 }
 
-// Argument scheme://host:port/index_name
+// Argument scheme://Host:port/index_name
 // i.e : http://localhost.local:9200/gor or https://localhost.local:9200/gor
 func TestElasticSimpleLocalWithSchemeConnectionBuild(t *testing.T) {
 	uri := "http://localhost.local:9200/" + expectedIndex
@@ -91,7 +91,7 @@ func TestElasticSimpleLocalWithSchemeConnectionBuild(t *testing.T) {
 	assertExpectedGorIndex(index, t)
 }
 
-// Argument scheme://host:port/index_name
+// Argument scheme://Host:port/index_name
 // i.e : http://localhost.local:9200/gor or https://localhost.local:9200/gor
 func TestElasticSimpleLocalWithHTTPSConnectionBuild(t *testing.T) {
 	uri := "https://localhost.local:9200/" + expectedIndex
@@ -103,7 +103,7 @@ func TestElasticSimpleLocalWithHTTPSConnectionBuild(t *testing.T) {
 	assertExpectedGorIndex(index, t)
 }
 
-// Argument scheme://host:port/index_name
+// Argument scheme://Host:port/index_name
 // i.e : localhost.local:9200/pathtoElastic/gor
 func TestElasticLongPathConnectionBuild(t *testing.T) {
 	uri := "http://localhost.local:9200/pathtoElastic/" + expectedIndex
@@ -115,7 +115,7 @@ func TestElasticLongPathConnectionBuild(t *testing.T) {
 	assertExpectedGorIndex(index, t)
 }
 
-// Argument scheme://host:userinfo@port/index_name
+// Argument scheme://Host:userinfo@port/index_name
 // i.e : http://user:password@localhost.local:9200/gor
 func TestElasticBasicAuthConnectionBuild(t *testing.T) {
 	uri := "http://user:password@localhost.local:9200/" + expectedIndex
@@ -127,7 +127,7 @@ func TestElasticBasicAuthConnectionBuild(t *testing.T) {
 	assertExpectedGorIndex(index, t)
 }
 
-// Argument scheme://host:port/path/index_name
+// Argument scheme://Host:port/path/index_name
 // i.e : http://localhost.local:9200/path/gor or https://localhost.local:9200/path/gor
 func TestElasticComplexPathConnectionBuild(t *testing.T) {
 	uri := "http://localhost.local:9200/path/" + expectedIndex

emitter.go

@@ -2,82 +2,135 @@ package main
 
 import (
 	"bytes"
+	"hash/fnv"
 	"io"
 	"log"
+	"sync"
 	"time"
 )
 
-// Start initialize loop for sending data from inputs to outputs
-func Start(stop chan int) {
-	if Settings.middleware != "" {
-		middleware := NewMiddleware(Settings.middleware)
+type emitter struct {
+	sync.Mutex
+	sync.WaitGroup
+	quit    chan int
+	plugins *InOutPlugins
+}
 
-		for _, in := range Plugins.Inputs {
+// NewEmitter creates and initializes new `emitter` object.
+func NewEmitter(quit chan int) *emitter {
+	return &emitter{
+		quit: quit,
+	}
+}
+
+// Start initialize loop for sending data from inputs to outputs
+func (e *emitter) Start(plugins *InOutPlugins, middlewareCmd string) {
+	defer e.Wait()
+	if Settings.CopyBufferSize < 1 {
+		Settings.CopyBufferSize = 5 << 20
+	}
+	e.plugins = plugins
+
+	if middlewareCmd != "" {
+		middleware := NewMiddleware(middlewareCmd)
+
+		for _, in := range plugins.Inputs {
 			middleware.ReadFrom(in)
 		}
 
 		// We are going only to read responses, so using same ReadFrom method
-		for _, out := range Plugins.Outputs {
+		for _, out := range plugins.Outputs {
 			if r, ok := out.(io.Reader); ok {
 				middleware.ReadFrom(r)
 			}
 		}
-
+		e.Add(1)
 		go func() {
-			if err := CopyMulty(middleware, Plugins.Outputs...); err != nil {
-				log.Println("Error during copy: ", err)
-				close(stop)
+			defer e.Done()
+			if err := CopyMulty(e.quit, middleware, plugins.Outputs...); err != nil {
+				Debug(2, "Error during copy: ", err)
+				e.Close()
+			}
+		}()
+		go func() {
+			for {
+				select {
+				case <-e.quit:
+					middleware.Close()
+					return
+				}
 			}
 		}()
 	} else {
-		for _, in := range Plugins.Inputs {
-			go func() {
-				if err := CopyMulty(in, Plugins.Outputs...); err != nil {
-					log.Println("Error during copy: ", err)
-					close(stop)
+		for _, in := range plugins.Inputs {
+			e.Add(1)
+			go func(in io.Reader) {
+				defer e.Done()
+				if err := CopyMulty(e.quit, in, plugins.Outputs...); err != nil {
+					Debug(2, "Error during copy: ", err)
+					e.Close()
 				}
-			}()
+			}(in)
 		}
 
-		for _, out := range Plugins.Outputs {
+		for _, out := range plugins.Outputs {
 			if r, ok := out.(io.Reader); ok {
-				go func() {
-					if err := CopyMulty(r, Plugins.Outputs...); err != nil {
-						log.Println("Error during copy: ", err)
-						close(stop)
+				e.Add(1)
+				go func(r io.Reader) {
+					defer e.Done()
+					if err := CopyMulty(e.quit, r, plugins.Outputs...); err != nil {
+						Debug(2, "Error during copy: ", err)
+						e.Close()
 					}
-				}()
+				}(r)
 			}
 		}
 	}
+}
 
-	for {
-		select {
-		case <-stop:
-			finalize()
-			return
-		case <-time.After(100 * time.Millisecond):
+func (e *emitter) close() {
+	select {
+	case <-e.quit:
+	default:
+		close(e.quit)
+	}
+}
+
+// Close closes all the goroutine and waits for it to finish.
+func (e *emitter) Close() {
+	e.close()
+	for _, p := range e.plugins.Inputs {
+		if cp, ok := p.(io.Closer); ok {
+			cp.Close()
+		}
+	}
+	for _, p := range e.plugins.Outputs {
+		if cp, ok := p.(io.Closer); ok {
+			cp.Close()
 		}
 	}
+	e.plugins = nil // avoid further accidental usage
 }
 
 // CopyMulty copies from 1 reader to multiple writers
-func CopyMulty(src io.Reader, writers ...io.Writer) (err error) {
-	buf := make([]byte, Settings.copyBufferSize)
+func CopyMulty(stop chan int, src io.Reader, writers ...io.Writer) error {
+	buf := make([]byte, Settings.CopyBufferSize)
 	wIndex := 0
-	modifier := NewHTTPModifier(&Settings.modifierConfig)
+	modifier := NewHTTPModifier(&Settings.ModifierConfig)
 	filteredRequests := make(map[string]time.Time)
 	filteredRequestsLastCleanTime := time.Now()
 
 	i := 0
-
 	for {
-		nr, er := src.Read(buf)
+		var nr int
+		nr, err := src.Read(buf)
 
-		if er == io.EOF {
+		select {
+		case <-stop:
 			return nil
+		default:
 		}
-		if er != nil {
+		if err != nil {
 			return err
 		}
 
@@ -85,24 +138,16 @@ func CopyMulty(src io.Reader, writers ...io.Writer) (err error) {
 		if nr > 500 {
 			_maxN = 500
 		}
-		if nr > 0 && len(buf) > nr {
+		if nr > 0 {
 			payload := buf[:nr]
 			meta := payloadMeta(payload)
 			if len(meta) < 3 {
-				if Settings.debug {
-					Debug("[EMITTER] Found malformed record", string(payload[0:_maxN]), nr, "from:", src)
-				}
+				Debug(2, "[EMITTER] Found malformed record", string(payload[0:_maxN]), nr, "from:", src)
 				continue
 			}
 			requestID := string(meta[1])
 
-			if nr >= 5*1024*1024 {
-				log.Println("INFO: Large packet... We received ", len(payload), " bytes from ", src)
-			}
-
-			if Settings.debug {
-				Debug("[EMITTER] input:", string(payload[0:_maxN]), nr, "from:", src)
-			}
+			Debug(3, "[EMITTER] input:", string(payload[0:_maxN]), nr, "from:", src)
 
 			if modifier != nil {
 				if isRequestPayload(payload) {
@@ -121,9 +166,8 @@ func CopyMulty(src io.Reader, writers ...io.Writer) (err error) {
 						payload = append(payload[:headSize], body...)
 					}
 
-					if Settings.debug {
-						Debug("[EMITTER] Rewritten input:", len(payload), "First 500 bytes:", string(payload[0:_maxN]))
-					}
+					Debug(3, "[EMITTER] Rewritten input:", len(payload), "First %d bytes:", _maxN, string(payload[0:_maxN]))
+
 				} else {
 					if _, ok := filteredRequests[requestID]; ok {
 						delete(filteredRequests, requestID)
@@ -132,23 +176,36 @@ func CopyMulty(src io.Reader, writers ...io.Writer) (err error) {
 				}
 			}
 
-			if Settings.prettifyHTTP {
+			if Settings.PrettifyHTTP {
 				payload = prettifyHTTP(payload)
 				if len(payload) == 0 {
 					continue
 				}
 			}
 
-			if Settings.splitOutput {
-				// Simple round robin
-				if _, err := writers[wIndex].Write(payload); err != nil {
-					return err
-				}
+			if Settings.SplitOutput {
+				if Settings.RecognizeTCPSessions {
+					if !PRO {
+						log.Fatal("Detailed TCP sessions work only with PRO license")
+					}
+					hasher := fnv.New32a()
+					// First 20 bytes contain tcp session
+					id := payloadID(payload)
+					hasher.Write(id)
 
-				wIndex++
+					wIndex = int(hasher.Sum32()) % len(writers)
+					writers[wIndex].Write(payload)
+				} else {
+					// Simple round robin
+					if _, err := writers[wIndex].Write(payload); err != nil {
+						return err
+					}
 
-				if wIndex >= len(writers) {
-					wIndex = 0
+					wIndex++
+
+					if wIndex >= len(writers) {
+						wIndex = 0
+					}
 				}
 			} else {
 				for _, dst := range writers {
@@ -157,8 +214,6 @@ func CopyMulty(src io.Reader, writers ...io.Writer) (err error) {
 					}
 				}
 			}
-		} else if nr > 0 {
-			log.Println("WARN: Packet", nr, "bytes is too large to process. Consider increasing --copy-buffer-size")
 		}
 
 		// Run GC on each 1000 request
@@ -177,6 +232,4 @@ func CopyMulty(src io.Reader, writers ...io.Writer) (err error) {
 
 		i++
 	}
-
-	return err
 }

emitter_test.go

@@ -1,13 +1,21 @@
 package main
 
 import (
+	"fmt"
 	"io"
+	"os"
 	"sync"
 	"sync/atomic"
 	"testing"
 	"time"
 )
 
+func TestMain(m *testing.M) {
+	PRO = true
+	code := m.Run()
+	os.Exit(code)
+}
+
 func TestEmitter(t *testing.T) {
 	wg := new(sync.WaitGroup)
 	quit := make(chan int)
@@ -17,10 +25,14 @@ func TestEmitter(t *testing.T) {
 		wg.Done()
 	})
 
-	Plugins.Inputs = []io.Reader{input}
-	Plugins.Outputs = []io.Writer{output}
+	plugins := &InOutPlugins{
+		Inputs:  []io.Reader{input},
+		Outputs: []io.Writer{output},
+	}
+	plugins.All = append(plugins.All, input, output)
 
-	go Start(quit)
+	emitter := NewEmitter(quit)
+	go emitter.Start(plugins, Settings.Middleware)
 
 	for i := 0; i < 1000; i++ {
 		wg.Add(1)
@@ -28,8 +40,7 @@ func TestEmitter(t *testing.T) {
 	}
 
 	wg.Wait()
-
-	close(quit)
+	emitter.Close()
 }
 
 func TestEmitterFiltered(t *testing.T) {
@@ -43,12 +54,17 @@ func TestEmitterFiltered(t *testing.T) {
 		wg.Done()
 	})
 
-	Plugins.Inputs = []io.Reader{input}
-	Plugins.Outputs = []io.Writer{output}
-	methods := HTTPMethods{[]byte("GET")}
-	Settings.modifierConfig = HTTPModifierConfig{methods: methods}
+	plugins := &InOutPlugins{
+		Inputs:  []io.Reader{input},
+		Outputs: []io.Writer{output},
+	}
+	plugins.All = append(plugins.All, input, output)
 
-	go Start(quit)
+	methods := HTTPMethods{[]byte("GET")}
+	Settings.ModifierConfig = HTTPModifierConfig{Methods: methods}
+
+	emitter := &emitter{quit: quit}
+	go emitter.Start(plugins, "")
 
 	wg.Add(2)
 
@@ -73,10 +89,53 @@ func TestEmitterFiltered(t *testing.T) {
 	input.EmitBytes(respb)
 
 	wg.Wait()
+	emitter.Close()
 
-	close(quit)
+	Settings.ModifierConfig = HTTPModifierConfig{}
+}
 
-	Settings.modifierConfig = HTTPModifierConfig{}
+func TestEmitterSplitRoundRobin(t *testing.T) {
+	wg := new(sync.WaitGroup)
+	quit := make(chan int)
+
+	input := NewTestInput()
+
+	var counter1, counter2 int32
+
+	output1 := NewTestOutput(func(data []byte) {
+		atomic.AddInt32(&counter1, 1)
+		wg.Done()
+	})
+
+	output2 := NewTestOutput(func(data []byte) {
+		atomic.AddInt32(&counter2, 1)
+		wg.Done()
+	})
+
+	plugins := &InOutPlugins{
+		Inputs:  []io.Reader{input},
+		Outputs: []io.Writer{output1, output2},
+	}
+
+	Settings.SplitOutput = true
+
+	emitter := NewEmitter(quit)
+	go emitter.Start(plugins, Settings.Middleware)
+
+	for i := 0; i < 1000; i++ {
+		wg.Add(1)
+		input.EmitGET()
+	}
+
+	wg.Wait()
+
+	emitter.Close()
+
+	if counter1 == 0 || counter2 == 0 || counter1 != counter2 {
+		t.Errorf("Round robin should split traffic equally: %d vs %d", counter1, counter2)
+	}
+
+	Settings.SplitOutput = false
 }
 
 func TestEmitterRoundRobin(t *testing.T) {
@@ -97,12 +156,16 @@ func TestEmitterRoundRobin(t *testing.T) {
 		wg.Done()
 	})
 
-	Plugins.Inputs = []io.Reader{input}
-	Plugins.Outputs = []io.Writer{output1, output2}
+	plugins := &InOutPlugins{
+		Inputs:  []io.Reader{input},
+		Outputs: []io.Writer{output1, output2},
+	}
+	plugins.All = append(plugins.All, input, output1, output2)
 
-	Settings.splitOutput = true
+	Settings.SplitOutput = true
 
-	go Start(quit)
+	emitter := NewEmitter(quit)
+	go emitter.Start(plugins, Settings.Middleware)
 
 	for i := 0; i < 1000; i++ {
 		wg.Add(1)
@@ -110,14 +173,71 @@ func TestEmitterRoundRobin(t *testing.T) {
 	}
 
 	wg.Wait()
-
-	close(quit)
+	emitter.Close()
 
 	if counter1 == 0 || counter2 == 0 {
 		t.Errorf("Round robin should split traffic equally: %d vs %d", counter1, counter2)
 	}
 
-	Settings.splitOutput = false
+	Settings.SplitOutput = false
+}
+
+func TestEmitterSplitSession(t *testing.T) {
+	wg := new(sync.WaitGroup)
+	wg.Add(200)
+
+	quit := make(chan int)
+
+	input := NewTestInput()
+	input.skipHeader = true
+
+	var counter1, counter2 int32
+
+	output1 := NewTestOutput(func(data []byte) {
+		if payloadID(data)[0] == 'a' {
+			atomic.AddInt32(&counter1, 1)
+		}
+		wg.Done()
+	})
+
+	output2 := NewTestOutput(func(data []byte) {
+		if payloadID(data)[0] == 'b' {
+			atomic.AddInt32(&counter2, 1)
+		}
+		wg.Done()
+	})
+
+	plugins := &InOutPlugins{
+		Inputs:  []io.Reader{input},
+		Outputs: []io.Writer{output1, output2},
+	}
+
+	Settings.SplitOutput = true
+	Settings.RecognizeTCPSessions = true
+
+	emitter := NewEmitter(quit)
+	go emitter.Start(plugins, Settings.Middleware)
+
+	for i := 0; i < 200; i++ {
+		// Keep session but randomize
+		id := make([]byte, 20)
+		if i&1 == 0 { // for recognizeTCPSessions one should be odd and other will be even number
+			id[0] = 'a'
+		} else {
+			id[0] = 'b'
+		}
+		input.EmitBytes([]byte(fmt.Sprintf("1 %s 1 1\nGET / HTTP/1.1\r\n\r\n", id[:20])))
+	}
+
+	wg.Wait()
+
+	if counter1 != counter2 {
+		t.Errorf("Round robin should split traffic equally: %d vs %d", counter1, counter2)
+	}
+
+	Settings.SplitOutput = false
+	Settings.RecognizeTCPSessions = false
+	emitter.Close()
 }
 
 func BenchmarkEmitter(b *testing.B) {
@@ -130,10 +250,14 @@ func BenchmarkEmitter(b *testing.B) {
 		wg.Done()
 	})
 
-	Plugins.Inputs = []io.Reader{input}
-	Plugins.Outputs = []io.Writer{output}
+	plugins := &InOutPlugins{
+		Inputs:  []io.Reader{input},
+		Outputs: []io.Writer{output},
+	}
+	plugins.All = append(plugins.All, input, output)
 
-	go Start(quit)
+	emitter := NewEmitter(quit)
+	go emitter.Start(plugins, Settings.Middleware)
 
 	b.ResetTimer()
 
@@ -143,5 +267,5 @@ func BenchmarkEmitter(b *testing.B) {
 	}
 
 	wg.Wait()
-	close(quit)
+	emitter.Close()
 }

examples/middleware/echo.sh

@@ -8,8 +8,10 @@
 # 
 
 function log {
+    if $GOR_TEST != ""; then # if we are not testing
     # Logging to stderr, because stdout/stdin used for data transfer
     >&2 echo "[DEBUG][ECHO] $1"
+    fi
 }
 
 while read line; do

examples/middleware/token_modifier.go

@@ -115,6 +115,8 @@ func encode(buf []byte) []byte {
 }
 
 func Debug(args ...interface{}) {
-	fmt.Fprint(os.Stderr, "[DEBUG][TOKEN-MOD] ")
-	fmt.Fprintln(os.Stderr, args...)
+	if os.Getenv("GOR_TEST") != "" { // if we are not testing
+		fmt.Fprint(os.Stderr, "[DEBUG][TOKEN-MOD] ")
+		fmt.Fprintln(os.Stderr, args...)
+	}
 }

go.mod

@@ -0,0 +1,19 @@
+module github.com/buger/goreplay
+
+go 1.14
+
+require (
+	github.com/Shopify/sarama v1.26.4
+	github.com/araddon/gou v0.0.0-20190110011759-c797efecbb61 // indirect
+	github.com/aws/aws-sdk-go v1.33.2
+	github.com/bitly/go-hostpool v0.1.0 // indirect
+	github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 // indirect
+	github.com/google/gopacket v1.1.18
+	github.com/klauspost/compress v1.10.10 // indirect
+	github.com/mattbaird/elastigo v0.0.0-20170123220020-2fe47fd29e4b
+	github.com/pierrec/lz4 v2.5.2+incompatible // indirect
+	github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0 // indirect
+	github.com/smartystreets/goconvey v1.6.4 // indirect
+	golang.org/x/net v0.0.0-20200707034311-ab3426394381 // indirect
+	golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd
+)

go.sum

@@ -0,0 +1,117 @@
+github.com/Shopify/sarama v1.26.4 h1:+17TxUq/PJEAfZAll0T7XJjSgQWCpaQSoki/x5yN8o8=
+github.com/Shopify/sarama v1.26.4/go.mod h1:NbSGBSSndYaIhRcBtY9V0U7AyH+x71bG668AuWys/yU=
+github.com/Shopify/toxiproxy v2.1.4+incompatible h1:TKdv8HiTLgE5wdJuEML90aBgNWsokNbMijUGhmcoBJc=
+github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI=
+github.com/araddon/gou v0.0.0-20190110011759-c797efecbb61 h1:Xz25cuW4REGC5W5UtpMU3QItMIImag615HiQcRbxqKQ=
+github.com/araddon/gou v0.0.0-20190110011759-c797efecbb61/go.mod h1:ikc1XA58M+Rx7SEbf0bLJCfBkwayZ8T5jBo5FXK8Uz8=
+github.com/aws/aws-sdk-go v1.33.2 h1:8TVrnPnSD7I+AmDp66xBUvS3K0J+jH09YXdrkJ34ey0=
+github.com/aws/aws-sdk-go v1.33.2/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0=
+github.com/bitly/go-hostpool v0.1.0 h1:XKmsF6k5el6xHG3WPJ8U0Ku/ye7njX7W81Ng7O2ioR0=
+github.com/bitly/go-hostpool v0.1.0/go.mod h1:4gOCgp6+NZnVqlKyZ/iBZFTAJKembaVENUpMkpg42fw=
+github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 h1:DDGfHa7BWjL4YnC6+E63dPcxHo2sUxDIu8g3QgEJdRY=
+github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4=
+github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/eapache/go-resiliency v1.2.0 h1:v7g92e/KSN71Rq7vSThKaWIq68fL4YHvWyiUKorFR1Q=
+github.com/eapache/go-resiliency v1.2.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs=
+github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 h1:YEetp8/yCZMuEPMUDHG0CW/brkkEp8mzqk2+ODEitlw=
+github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU=
+github.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc=
+github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I=
+github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw=
+github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g=
+github.com/frankban/quicktest v1.7.2 h1:2QxQoC1TS09S7fhCPsrvqYdvP1H5M1P1ih5ABm3BTYk=
+github.com/frankban/quicktest v1.7.2/go.mod h1:jaStnuzAqU1AJdCO0l53JDCJrVDKcS03DbaAcR7Ks/o=
+github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
+github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4=
+github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4=
+github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/gopacket v1.1.18 h1:lum7VRA9kdlvBi7/v2p7/zcbkduHaCH/SVVyurs7OpY=
+github.com/google/gopacket v1.1.18/go.mod h1:UdDNZ1OO62aGYVnPhxT1U6aI7ukYtA/kB8vaU0diBUM=
+github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8=
+github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
+github.com/hashicorp/go-uuid v1.0.2 h1:cfejS+Tpcp13yd5nYHWDI6qVCny6wyX2Mt5SGur2IGE=
+github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
+github.com/jcmturner/gofork v1.0.0 h1:J7uCkflzTEhUZ64xqKnkDxq3kzc96ajM1Gli5ktUem8=
+github.com/jcmturner/gofork v1.0.0/go.mod h1:MK8+TM0La+2rjBD4jE12Kj1pCCxK7d2LK/UM3ncEo0o=
+github.com/jmespath/go-jmespath v0.3.0 h1:OS12ieG61fsCg5+qLJ+SsW9NicxNkg3b25OyT2yCeUc=
+github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik=
+github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
+github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
+github.com/klauspost/compress v1.9.8 h1:VMAMUUOh+gaxKTMk+zqbjsSjsIcUcL/LF4o63i82QyA=
+github.com/klauspost/compress v1.9.8/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
+github.com/klauspost/compress v1.10.10 h1:a/y8CglcM7gLGYmlbP/stPE5sR3hbhFRUjCBfd/0B3I=
+github.com/klauspost/compress v1.10.10/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
+github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs=
+github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/mattbaird/elastigo v0.0.0-20170123220020-2fe47fd29e4b h1:v29yPGHhOqw7VHEnTeQFAth3SsBrmwc8JfuhNY0G34k=
+github.com/mattbaird/elastigo v0.0.0-20170123220020-2fe47fd29e4b/go.mod h1:5MWrJXKRQyhQdUCF+vu6U5c4nQpg70vW3eHaU0/AYbU=
+github.com/pierrec/lz4 v2.4.1+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
+github.com/pierrec/lz4 v2.5.2+incompatible h1:WCjObylUIOlKy/+7Abdn34TLIkXiA4UWUMhxq9m9ZXI=
+github.com/pierrec/lz4 v2.5.2+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
+github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0 h1:MkV+77GLUNo5oJ0jf870itWm3D0Sjh7+Za9gazKc5LQ=
+github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
+github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM=
+github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
+github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s=
+github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4=
+github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
+github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I=
+github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20200204104054-c9f3fb736b72 h1:+ELyKg6m8UBf0nPFSqD0mi7zUfwPyXo23HNjMnXPz7w=
+golang.org/x/crypto v0.0.0-20200204104054-c9f3fb736b72/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI=
+golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20200202094626-16171245cfb2 h1:CCH4IOTTfewWjGOlSp+zGcjutRKlBEZQ6wTn8ozI/nI=
+golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200707034311-ab3426394381 h1:VXak5I6aEWmAXeQjA+QSZzlgNrpq9mjcfDemuexIKsU=
+golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190405154228-4b34438f7a67/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884=
+golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
+golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
+gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/jcmturner/aescts.v1 v1.0.1 h1:cVVZBK2b1zY26haWB4vbBiZrfFQnfbTVrE3xZq6hrEw=
+gopkg.in/jcmturner/aescts.v1 v1.0.1/go.mod h1:nsR8qBOg+OucoIW+WMhB3GspUQXq9XorLnQb9XtvcOo=
+gopkg.in/jcmturner/dnsutils.v1 v1.0.1 h1:cIuC1OLRGZrld+16ZJvvZxVJeKPsvd5eUIvxfoN5hSM=
+gopkg.in/jcmturner/dnsutils.v1 v1.0.1/go.mod h1:m3v+5svpVOhtFAP/wSz+yzh4Mc0Fg7eRhxkJMWSIz9Q=
+gopkg.in/jcmturner/goidentity.v3 v3.0.0 h1:1duIyWiTaYvVx3YX2CYtpJbUFd7/UuPYCfgXtQ3VTbI=
+gopkg.in/jcmturner/goidentity.v3 v3.0.0/go.mod h1:oG2kH0IvSYNIu80dVAyu/yoefjq1mNfM5bm88whjWx4=
+gopkg.in/jcmturner/gokrb5.v7 v7.5.0 h1:a9tsXlIDD9SKxotJMK3niV7rPZAJeX2aD/0yg3qlIrg=
+gopkg.in/jcmturner/gokrb5.v7 v7.5.0/go.mod h1:l8VISx+WGYp+Fp7KRbsiUuXTTOnxIc3Tuvyavf11/WM=
+gopkg.in/jcmturner/rpc.v1 v1.1.0 h1:QHIUxTX1ISuAv9dD2wJ9HWQVuWDX/Zc0PfeC2tjc4rU=
+gopkg.in/jcmturner/rpc.v1 v1.1.0/go.mod h1:YIdkC4XfD6GXbzje11McwsDuOlZQSb9W4vfLvuNnlv8=
+gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=
+gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

gor.go

@@ -5,7 +5,6 @@ package main
 import (
 	"flag"
 	"fmt"
-	"io"
 	"log"
 	"net/http"
 	"net/http/httputil"
@@ -20,7 +19,6 @@ import (
 )
 
 var (
-	mode       string
 	cpuprofile = flag.String("cpuprofile", "", "write cpu profile to file")
 	memprofile = flag.String("memprofile", "", "write memory profile to this file")
 )
@@ -33,40 +31,31 @@ func loggingMiddleware(next http.Handler) http.Handler {
 	})
 }
 
-var closeCh chan int
-
 func main() {
-	closeCh = make(chan int)
-	// // Don't exit on panic
-	// defer func() {
-	// 	if r := recover(); r != nil {
-	// 		fmt.Printf("PANIC: pkg: %v %s \n", r, debug.Stack())
-	// 	}
-	// }()
-
-	// If not set via env cariable
 	if len(os.Getenv("GOMAXPROCS")) == 0 {
 		runtime.GOMAXPROCS(runtime.NumCPU() * 2)
 	}
 
 	args := os.Args[1:]
+	var plugins *InOutPlugins
 	if len(args) > 0 && args[0] == "file-server" {
 		if len(args) != 2 {
 			log.Fatal("You should specify port and IP (optional) for the file server. Example: `gor file-server :80`")
 		}
 		dir, _ := os.Getwd()
 
-		log.Println("Started example file server for current directory on address ", args[1])
+		Debug(0, "Started example file server for current directory on address ", args[1])
 
 		log.Fatal(http.ListenAndServe(args[1], loggingMiddleware(http.FileServer(http.Dir(dir)))))
 	} else {
 		flag.Parse()
-		InitPlugins()
+		checkSettings()
+		plugins = NewPlugins()
 	}
 
-	fmt.Println("Version:", VERSION)
+	log.Printf("[PPID %d and PID %d] Version:%s\n", os.Getppid(), os.Getpid(), VERSION)
 
-	if len(Plugins.Inputs) == 0 || len(Plugins.Outputs) == 0 {
+	if len(plugins.Inputs) == 0 || len(plugins.Outputs) == 0 {
 		log.Fatal("Required at least 1 input and 1 output")
 	}
 
@@ -78,38 +67,35 @@ func main() {
 		profileCPU(*cpuprofile)
 	}
 
-	if Settings.pprof != "" {
+	if Settings.Pprof != "" {
 		go func() {
-			log.Println(http.ListenAndServe(Settings.pprof, nil))
+			log.Println(http.ListenAndServe(Settings.Pprof, nil))
 		}()
 	}
 
-	c := make(chan os.Signal, 1)
-	signal.Notify(c, os.Interrupt, syscall.SIGTERM)
-	go func() {
-		<-c
-		finalize()
-		os.Exit(1)
-	}()
+	closeCh := make(chan int)
+	emitter := NewEmitter(closeCh)
+	go emitter.Start(plugins, Settings.Middleware)
+	if Settings.ExitAfter > 0 {
+		log.Printf("Running gor for a duration of %s\n", Settings.ExitAfter)
 
-	if Settings.exitAfter > 0 {
-		log.Println("Running gor for a duration of", Settings.exitAfter)
-
-		time.AfterFunc(Settings.exitAfter, func() {
-			log.Println("Stopping gor after", Settings.exitAfter)
+		time.AfterFunc(Settings.ExitAfter, func() {
+			fmt.Printf("gor run timeout %s\n", Settings.ExitAfter)
 			close(closeCh)
 		})
 	}
-
-	Start(closeCh)
-}
-
-func finalize() {
-	for _, p := range Plugins.All {
-		if cp, ok := p.(io.Closer); ok {
-			cp.Close()
-		}
+	c := make(chan os.Signal, 1)
+	signal.Notify(c, os.Interrupt, syscall.SIGTERM)
+	exit := 0
+	select {
+	case <-c:
+		exit = 1
+	case <-closeCh:
+		exit = 0
 	}
+	emitter.Close()
+	os.Exit(exit)
+
 }
 
 func profileCPU(cpuprofile string) {
@@ -123,7 +109,6 @@ func profileCPU(cpuprofile string) {
 		time.AfterFunc(30*time.Second, func() {
 			pprof.StopCPUProfile()
 			f.Close()
-			log.Println("Stop profiling after 30 seconds")
 		})
 	}
 }

gor_stat.go

@@ -1,7 +1,6 @@
 package main
 
 import (
-	"log"
 	"runtime"
 	"strconv"
 	"time"
@@ -25,15 +24,14 @@ func NewGorStat(statName string, rateMs int) (s *GorStat) {
 	s.max = 0
 	s.count = 0
 
-	if Settings.stats {
-		log.Println(s.statName + ":latest,mean,max,count,count/second,gcount")
+	if Settings.Stats {
 		go s.reportStats()
 	}
 	return
 }
 
 func (s *GorStat) Write(latest int) {
-	if Settings.stats {
+	if Settings.Stats {
 		if latest > s.max {
 			s.max = latest
 		}
@@ -57,8 +55,9 @@ func (s *GorStat) String() string {
 }
 
 func (s *GorStat) reportStats() {
+	Debug(0, "\n", s.statName+":latest,mean,max,count,count/second,gcount")
 	for {
-		log.Println(s)
+		Debug(0, "\n", s)
 		s.Reset()
 		time.Sleep(time.Duration(s.rateMs) * time.Millisecond)
 	}

http_client.go

@@ -117,7 +117,7 @@ func (c *HTTPClient) Connect() (err error) {
 		if c.proxy.Scheme != "http" {
 			panic("Unsupported HTTP Proxy method")
 		}
-		Debug("[HTTPClient] Connecting to proxy", c.proxy.String(), "<>", toDial)
+		Debug(3, "[HTTPClient] Connecting to proxy", c.proxy.String(), "<>", toDial)
 		c.conn, err = net.DialTimeout("tcp", c.proxy.Host, c.config.ConnectionTimeout)
 		if err != nil {
 			return
@@ -151,7 +151,7 @@ func (c *HTTPClient) Connect() (err error) {
 				}
 			}
 		}
-		Debug("[HTTPClient] Proxy successfully connected")
+		Debug(3, "[HTTPClient] Proxy successfully connected")
 	} else {
 		c.conn, err = net.DialTimeout("tcp", toDial, c.config.ConnectionTimeout)
 		if err != nil {
@@ -161,7 +161,7 @@ func (c *HTTPClient) Connect() (err error) {
 
 	if c.scheme == "https" {
 		// Wrap our socket in TLS
-		Debug("[HTTPClient] Wrapping socket in TLS", c.host)
+		Debug(3, "[HTTPClient] Wrapping socket in TLS", c.host)
 		tlsConn := tls.Client(c.conn, &tls.Config{InsecureSkipVerify: true, ServerName: c.host})
 
 		if err = tlsConn.Handshake(); err != nil {
@@ -169,7 +169,7 @@ func (c *HTTPClient) Connect() (err error) {
 		}
 
 		c.conn = tlsConn
-		Debug("[HTTPClient] Successfully wrapped in TLS")
+		Debug(3, "[HTTPClient] Successfully wrapped in TLS")
 	}
 
 	return
@@ -179,7 +179,7 @@ func (c *HTTPClient) Disconnect() {
 	if c.conn != nil {
 		c.conn.Close()
 		c.conn = nil
-		Debug("[HTTP] Disconnected: ", c.baseURL)
+		Debug(3, "[HTTP] Disconnected: ", c.baseURL)
 	}
 }
 
@@ -189,17 +189,17 @@ func (c *HTTPClient) isAlive(readBytes *int) bool {
 	n, err := c.conn.Read(c.respBuf[:1])
 
 	if err == io.EOF {
-		Debug("[HTTPClient] connection closed, reconnecting")
+		Debug(3, "[HTTPClient] connection closed, reconnecting")
 		return false
 	}
 
 	if err == syscall.EPIPE {
-		Debug("Detected broken pipe.", err)
+		Debug(3, "Detected broken pipe.", err)
 		return false
 	}
 	if n != 0 {
 		*readBytes += n
-		Debug("[HTTPClient] isAlive readBytes ", *readBytes)
+		Debug(3, "[HTTPClient] isAlive readBytes ", *readBytes)
 	}
 	return true
 }
@@ -237,7 +237,7 @@ func (c *HTTPClient) Send(data []byte) (response []byte, err error) {
 	// Don't exit on panic
 	defer func() {
 		if r := recover(); r != nil {
-			Debug("[HTTPClient]", r, string(data))
+			Debug(3, "[HTTPClient]", r, string(data))
 
 			if _, ok := r.(error); ok {
 				log.Println("[HTTPClient] Failed to send request: ", string(data))
@@ -253,9 +253,9 @@ func (c *HTTPClient) Send(data []byte) (response []byte, err error) {
 
 	var readBytes int
 	if c.conn == nil || !c.isAlive(&readBytes) {
-		Debug("[HTTPClient] Connecting:", c.baseURL)
+		Debug(3, "[HTTPClient] Connecting:", c.baseURL)
 		if err = c.Connect(); err != nil {
-			log.Println("[HTTPClient] Connection error:", err)
+			Debug(1, "[HTTPClient] Connection error:", err)
 			response = errorPayload(HTTP_CONNECTION_ERROR)
 			return
 		}
@@ -284,7 +284,7 @@ func (c *HTTPClient) Send(data []byte) (response []byte, err error) {
 	}
 
 	if c.config.Debug {
-		Debug("[HTTPClient] Sending:", string(data))
+		Debug(3, "[HTTPClient] Sending:", string(data))
 	}
 
 	return c.send(data, readBytes, timeout)
@@ -294,7 +294,7 @@ func (c *HTTPClient) send(data []byte, readBytes int, timeout time.Time) (respon
 	var payload []byte
 	var n int
 	if _, err = c.conn.Write(data); err != nil {
-		Debug("[HTTPClient] Write error:", err, c.baseURL)
+		Debug(1, "[HTTPClient] Write error:", err, c.baseURL)
 		response = errorPayload(HTTP_TIMEOUT)
 		c.Disconnect()
 		return
@@ -356,7 +356,7 @@ func (c *HTTPClient) send(data []byte, readBytes int, timeout time.Time) (respon
 				}
 			} else if contentLength != -1 {
 				if currentContentLength > contentLength {
-					Debug("[HTTPClient] disconnected, wrong length", currentContentLength, contentLength)
+					Debug(3, "[HTTPClient] disconnected, wrong length", currentContentLength, contentLength)
 					c.Disconnect()
 					break
 				} else if currentContentLength == contentLength {
@@ -388,14 +388,14 @@ func (c *HTTPClient) send(data []byte, readBytes int, timeout time.Time) (respon
 				}
 			} else if contentLength != -1 {
 				if currentContentLength > contentLength {
-					Debug("[HTTPClient] disconnected, wrong length", currentContentLength, contentLength)
+					Debug(3, "[HTTPClient] disconnected, wrong length", currentContentLength, contentLength)
 					c.Disconnect()
 					break
 				} else if currentContentLength == contentLength {
 					break
 				}
 			} else {
-				Debug("[HTTPClient] disconnected, can't find Content-Length or Chunked")
+				Debug(3, "[HTTPClient] disconnected, can't find Content-Length or Chunked")
 				c.Disconnect()
 				break
 			}
@@ -403,14 +403,14 @@ func (c *HTTPClient) send(data []byte, readBytes int, timeout time.Time) (respon
 			if err == io.EOF {
 				break
 			} else if err != nil {
-				Debug("[HTTPClient] Read the whole body error:", err, c.baseURL)
+				Debug(3, "[HTTPClient] Read the whole body error:", err, c.baseURL)
 				break
 			}
 
 		}
 
 		if readBytes >= maxResponseSize {
-			Debug("[HTTPClient] Body is more than the max size", maxResponseSize,
+			Debug(3, "[HTTPClient] Body is more than the max size", maxResponseSize,
 				c.baseURL)
 			break
 		}
@@ -420,11 +420,7 @@ func (c *HTTPClient) send(data []byte, readBytes int, timeout time.Time) (respon
 	}
 
 	if err != nil && readBytes == 0 {
-		maxRead := 100
-		if readBytes < maxRead {
-			maxRead = readBytes
-		}
-		Debug("[HTTPClient] Response read timeout error", err, c.conn, readBytes, string(c.respBuf[:maxRead]))
+		Debug(3, "[HTTPClient] Response read timeout error", err, c.conn, readBytes, string(c.respBuf[:readBytes]))
 		response = errorPayload(HTTP_TIMEOUT)
 		c.Disconnect()
 		return
@@ -435,7 +431,7 @@ func (c *HTTPClient) send(data []byte, readBytes int, timeout time.Time) (respon
 		if readBytes < maxRead {
 			maxRead = readBytes
 		}
-		Debug("[HTTPClient] Response read unknown error", err, c.conn, readBytes, string(c.respBuf[:maxRead]))
+		Debug(3, "[HTTPClient] Response read unknown error", err, c.conn, readBytes, string(c.respBuf[:maxRead]))
 		response = errorPayload(HTTP_UNKNOWN_ERROR)
 		c.Disconnect()
 		return
@@ -448,7 +444,7 @@ func (c *HTTPClient) send(data []byte, readBytes int, timeout time.Time) (respon
 	copy(payload, c.respBuf[:readBytes])
 
 	if c.config.Debug {
-		Debug("[HTTPClient] Received:", string(payload))
+		Debug(3, "[HTTPClient] Received:", string(payload))
 	}
 
 	if c.config.FollowRedirects > 0 && c.redirectsCount < c.config.FollowRedirects {
@@ -461,16 +457,14 @@ func (c *HTTPClient) send(data []byte, readBytes int, timeout time.Time) (respon
 			location := proto.Header(payload, []byte("Location"))
 			redirectPayload := proto.SetPath(data, location)
 
-			if c.config.Debug {
-				Debug("[HTTPClient] Redirecting to: " + string(location))
-			}
+			Debug(3, "[HTTPClient] Redirecting to: "+string(location))
 
 			return c.Send(redirectPayload)
 		}
 	}
 
 	if bytes.Equal(proto.Status(payload), []byte("400")) {
-		Debug("[HTTPClient] Closed connection on 400 response")
+		Debug(3, "[HTTPClient] Closed connection on 400 response")
 		c.Disconnect()
 	}
 

http_client_test.go

@@ -298,11 +298,10 @@ func TestHTTPClientServerNoKeepAlive(t *testing.T) {
 			}
 
 			buf := make([]byte, 4096)
-			reqLen, err := conn.Read(buf)
+			_, err = conn.Read(buf)
 			if err != nil {
 				t.Error("Error reading:", err.Error())
 			}
-			Debug("Received: ", string(buf[0:reqLen]))
 			conn.Write([]byte("OK"))
 
 			// No keep-alive connections
@@ -468,66 +467,66 @@ func TestHTTPClientHandleHTTP10(t *testing.T) {
 	wg.Wait()
 }
 
-func TestHTTPClientErrors(t *testing.T) {
-	req := []byte("GET http://foobar.com/path HTTP/1.0\r\n\r\n")
+// func TestHTTPClientErrors(t *testing.T) {
+// 	req := []byte("GET http://foobar.com/path HTTP/1.0\r\n\r\n")
 
-	// Port not exists
-	client := NewHTTPClient("http://127.0.0.1:1", &HTTPClientConfig{Debug: true})
-	if resp, err := client.Send(req); err != nil {
-		if s := proto.Status(resp); !bytes.Equal(s, []byte("521")) {
-			t.Error("Should return status 521 for connection refused, instead:", string(s))
-		}
-	} else {
-		t.Error("Should throw error")
-	}
+// 	// Port not exists
+// 	client := NewHTTPClient("http://127.0.0.1:1", &HTTPClientConfig{Debug: true})
+// 	if resp, err := client.Send(req); err != nil {
+// 		if s := proto.Status(resp); !bytes.Equal(s, []byte("521")) {
+// 			t.Error("Should return status 521 for connection refused, instead:", string(s), err)
+// 		}
+// 	} else {
+// 		t.Error("Should throw error")
+// 	}
 
-	client = NewHTTPClient("http://not.existing", &HTTPClientConfig{Debug: true})
-	if resp, err := client.Send(req); err != nil {
-		if s := proto.Status(resp); !bytes.Equal(s, []byte("521")) {
-			t.Error("Should return status 521 for no such host, instead:", string(s))
-		}
-	} else {
-		t.Error("Should throw error")
-	}
+// 	client = NewHTTPClient("http://not.existing", &HTTPClientConfig{Debug: true})
+// 	if resp, err := client.Send(req); err != nil {
+// 		if s := proto.Status(resp); !bytes.Equal(s, []byte("521")) {
+// 			t.Error("Should return status 521 for no such host, instead:", string(s))
+// 		}
+// 	} else {
+// 		t.Error("Should throw error")
+// 	}
 
-	// Non routable IP address to simulate connection timeout
-	client = NewHTTPClient("http://10.255.255.1", &HTTPClientConfig{Debug: true, ConnectionTimeout: 100 * time.Millisecond})
+// 	// Non routable IP address to simulate connection timeout
+// 	client = NewHTTPClient("http://10.255.255.1", &HTTPClientConfig{Debug: true, ConnectionTimeout: 100 * time.Millisecond})
 
-	if resp, err := client.Send(req); err != nil {
-		if s := proto.Status(resp); !bytes.Equal(s, []byte("521")) {
-			t.Error("Should return status 521 for io/timeout:", string(s))
-		}
-	} else {
-		t.Error("Should throw error")
-	}
+// 	if resp, err := client.Send(req); err != nil {
+// 		if s := proto.Status(resp); !bytes.Equal(s, []byte("521")) {
+// 			t.Error("Should return status 521 for io/timeout:", string(s))
+// 		}
+// 	} else {
+// 		t.Error("Should throw error")
+// 	}
 
-	// Connecting but io timeout on read
-	ln, _ := net.Listen("tcp", "127.0.0.1:0")
-	client = NewHTTPClient("http://"+ln.Addr().String(), &HTTPClientConfig{Debug: true, Timeout: 10 * time.Millisecond})
-	defer ln.Close()
+// 	// Connecting but io timeout on read
+// 	ln, _ := net.Listen("tcp", "127.0.0.1:0")
+// 	client = NewHTTPClient("http://"+ln.Addr().String(), &HTTPClientConfig{Debug: true, Timeout: 10 * time.Millisecond})
+// 	defer ln.Close()
 
-	if resp, err := client.Send(req); err != nil {
-		if s := proto.Status(resp); !bytes.Equal(s, []byte("524")) {
-			t.Error("Should return status 524 for io read, instead:", string(s))
-		}
-	} else {
-		t.Error("Should throw error")
-	}
+// 	if resp, err := client.Send(req); err != nil {
+// 		if s := proto.Status(resp); !bytes.Equal(s, []byte("524")) {
+// 			t.Error("Should return status 524 for io read, instead:", string(s))
+// 		}
+// 	} else {
+// 		t.Error("Should throw error")
+// 	}
 
-	// Response read error read tcp [::1]:51128: connection reset by peer &{{0xc20802a000}}
-	ln1, _ := net.Listen("tcp", "127.0.0.1:0")
-	go func() {
-		ln1.Accept()
-	}()
-	defer ln1.Close()
+// 	// Response read error read tcp [::1]:51128: connection reset by peer &{{0xc20802a000}}
+// 	ln1, _ := net.Listen("tcp", "127.0.0.1:0")
+// 	go func() {
+// 		ln1.Accept()
+// 	}()
+// 	defer ln1.Close()
 
-	client = NewHTTPClient("http://"+ln1.Addr().String(), &HTTPClientConfig{Debug: true, Timeout: 10 * time.Millisecond})
+// 	client = NewHTTPClient("http://"+ln1.Addr().String(), &HTTPClientConfig{Debug: true, Timeout: 10 * time.Millisecond})
 
-	if resp, err := client.Send(req); err != nil {
-		if s := proto.Status(resp); !bytes.Equal(s, []byte("524")) {
-			t.Error("Should return status 524 for connection reset by peer, instead:", string(s))
-		}
-	} else {
-		t.Error("Should throw error")
-	}
-}
+// 	if resp, err := client.Send(req); err != nil {
+// 		if s := proto.Status(resp); !bytes.Equal(s, []byte("524")) {
+// 			t.Error("Should return status 524 for connection reset by peer, instead:", string(s))
+// 		}
+// 	} else {
+// 		t.Error("Should throw error")
+// 	}
+// }

http_modifier.go

@@ -15,18 +15,18 @@ type HTTPModifier struct {
 
 func NewHTTPModifier(config *HTTPModifierConfig) *HTTPModifier {
 	// Optimization to skip modifier completely if we do not need it
-	if len(config.urlRegexp) == 0 &&
-		len(config.urlNegativeRegexp) == 0 &&
-		len(config.urlRewrite) == 0 &&
-		len(config.headerRewrite) == 0 &&
-		len(config.headerFilters) == 0 &&
-		len(config.headerNegativeFilters) == 0 &&
-		len(config.headerBasicAuthFilters) == 0 &&
-		len(config.headerHashFilters) == 0 &&
-		len(config.paramHashFilters) == 0 &&
-		len(config.params) == 0 &&
-		len(config.headers) == 0 &&
-		len(config.methods) == 0 {
+	if len(config.URLRegexp) == 0 &&
+		len(config.URLNegativeRegexp) == 0 &&
+		len(config.URLRewrite) == 0 &&
+		len(config.HeaderRewrite) == 0 &&
+		len(config.HeaderFilters) == 0 &&
+		len(config.HeaderNegativeFilters) == 0 &&
+		len(config.HeaderBasicAuthFilters) == 0 &&
+		len(config.HeaderHashFilters) == 0 &&
+		len(config.ParamHashFilters) == 0 &&
+		len(config.Params) == 0 &&
+		len(config.Headers) == 0 &&
+		len(config.Methods) == 0 {
 		return nil
 	}
 
@@ -34,16 +34,16 @@ func NewHTTPModifier(config *HTTPModifierConfig) *HTTPModifier {
 }
 
 func (m *HTTPModifier) Rewrite(payload []byte) (response []byte) {
-	if !proto.IsHTTPPayload(payload) {
+	if !proto.HasRequestTitle(payload) {
 		return payload
 	}
 
-	if len(m.config.methods) > 0 {
+	if len(m.config.Methods) > 0 {
 		method := proto.Method(payload)
 
 		matched := false
 
-		for _, m := range m.config.methods {
+		for _, m := range m.config.Methods {
 			if bytes.Equal(method, m) {
 				matched = true
 				break
@@ -55,24 +55,24 @@ func (m *HTTPModifier) Rewrite(payload []byte) (response []byte) {
 		}
 	}
 
-	if len(m.config.headers) > 0 {
-		for _, header := range m.config.headers {
+	if len(m.config.Headers) > 0 {
+		for _, header := range m.config.Headers {
 			payload = proto.SetHeader(payload, []byte(header.Name), []byte(header.Value))
 		}
 	}
 
-	if len(m.config.params) > 0 {
-		for _, param := range m.config.params {
+	if len(m.config.Params) > 0 {
+		for _, param := range m.config.Params {
 			payload = proto.SetPathParam(payload, param.Name, param.Value)
 		}
 	}
 
-	if len(m.config.urlRegexp) > 0 {
+	if len(m.config.URLRegexp) > 0 {
 		path := proto.Path(payload)
 
 		matched := false
 
-		for _, f := range m.config.urlRegexp {
+		for _, f := range m.config.URLRegexp {
 			if f.regexp.Match(path) {
 				matched = true
 				break
@@ -84,18 +84,18 @@ func (m *HTTPModifier) Rewrite(payload []byte) (response []byte) {
 		}
 	}
 
-	if len(m.config.urlNegativeRegexp) > 0 {
+	if len(m.config.URLNegativeRegexp) > 0 {
 		path := proto.Path(payload)
 
-		for _, f := range m.config.urlNegativeRegexp {
+		for _, f := range m.config.URLNegativeRegexp {
 			if f.regexp.Match(path) {
 				return
 			}
 		}
 	}
 
-	if len(m.config.headerFilters) > 0 {
-		for _, f := range m.config.headerFilters {
+	if len(m.config.HeaderFilters) > 0 {
+		for _, f := range m.config.HeaderFilters {
 			value := proto.Header(payload, f.name)
 
 			if len(value) == 0 {
@@ -108,8 +108,8 @@ func (m *HTTPModifier) Rewrite(payload []byte) (response []byte) {
 		}
 	}
 
-	if len(m.config.headerNegativeFilters) > 0 {
-		for _, f := range m.config.headerNegativeFilters {
+	if len(m.config.HeaderNegativeFilters) > 0 {
+		for _, f := range m.config.HeaderNegativeFilters {
 			value := proto.Header(payload, f.name)
 
 			if len(value) > 0 && f.regexp.Match(value) {
@@ -118,8 +118,8 @@ func (m *HTTPModifier) Rewrite(payload []byte) (response []byte) {
 		}
 	}
 
-	if len(m.config.headerBasicAuthFilters) > 0 {
-		for _, f := range m.config.headerBasicAuthFilters {
+	if len(m.config.HeaderBasicAuthFilters) > 0 {
+		for _, f := range m.config.HeaderBasicAuthFilters {
 			value := proto.Header(payload, []byte("Authorization"))
 
 			if len(value) > 0 {
@@ -135,8 +135,8 @@ func (m *HTTPModifier) Rewrite(payload []byte) (response []byte) {
 		}
 	}
 
-	if len(m.config.headerHashFilters) > 0 {
-		for _, f := range m.config.headerHashFilters {
+	if len(m.config.HeaderHashFilters) > 0 {
+		for _, f := range m.config.HeaderHashFilters {
 			value := proto.Header(payload, f.name)
 
 			if len(value) > 0 {
@@ -150,8 +150,8 @@ func (m *HTTPModifier) Rewrite(payload []byte) (response []byte) {
 		}
 	}
 
-	if len(m.config.paramHashFilters) > 0 {
-		for _, f := range m.config.paramHashFilters {
+	if len(m.config.ParamHashFilters) > 0 {
+		for _, f := range m.config.ParamHashFilters {
 			value, s, _ := proto.PathParam(payload, f.name)
 
 			if s != -1 {
@@ -165,10 +165,10 @@ func (m *HTTPModifier) Rewrite(payload []byte) (response []byte) {
 		}
 	}
 
-	if len(m.config.urlRewrite) > 0 {
+	if len(m.config.URLRewrite) > 0 {
 		path := proto.Path(payload)
 
-		for _, f := range m.config.urlRewrite {
+		for _, f := range m.config.URLRewrite {
 			if f.src.Match(path) {
 				path = f.src.ReplaceAll(path, f.target)
 				payload = proto.SetPath(payload, path)
@@ -178,8 +178,8 @@ func (m *HTTPModifier) Rewrite(payload []byte) (response []byte) {
 		}
 	}
 
-	if len(m.config.headerRewrite) > 0 {
-		for _, f := range m.config.headerRewrite {
+	if len(m.config.HeaderRewrite) > 0 {
+		for _, f := range m.config.HeaderRewrite {
 			value := proto.Header(payload, f.header)
 			if len(value) == 0 {
 				break

http_modifier_settings.go

@@ -10,19 +10,19 @@ import (
 
 // HTTPModifierConfig holds configuration options for built-in traffic modifier
 type HTTPModifierConfig struct {
-	urlNegativeRegexp      HTTPUrlRegexp
-	urlRegexp              HTTPUrlRegexp
-	urlRewrite             UrlRewriteMap
-	headerRewrite          HeaderRewriteMap
-	headerFilters          HTTPHeaderFilters
-	headerNegativeFilters  HTTPHeaderFilters
-	headerBasicAuthFilters HTTPHeaderBasicAuthFilters
-	headerHashFilters      HTTPHashFilters
-	paramHashFilters       HTTPHashFilters
+	URLNegativeRegexp      HTTPURLRegexp              `json:"http-disallow-url"`
+	URLRegexp              HTTPURLRegexp              `json:"http-allow-url"`
+	URLRewrite             URLRewriteMap              `json:"http-rewrite-url"`
+	HeaderRewrite          HeaderRewriteMap           `json:"http-rewrite-header"`
+	HeaderFilters          HTTPHeaderFilters          `json:"http-allow-header"`
+	HeaderNegativeFilters  HTTPHeaderFilters          `json:"http-disallow-header"`
+	HeaderBasicAuthFilters HTTPHeaderBasicAuthFilters `json:"http-basic-auth-filter"`
+	HeaderHashFilters      HTTPHashFilters            `json:"http-header-limiter"`
+	ParamHashFilters       HTTPHashFilters            `json:"http-param-limiter"`
 
-	params  HTTPParams
-	headers HTTPHeaders
-	methods HTTPMethods
+	Params  HTTPParams  `json:"http-set-param"`
+	Headers HTTPHeaders `json:"http-set-header"`
+	Methods HTTPMethods `json:"http-allow-method"`
 }
 
 //
@@ -204,13 +204,13 @@ type urlRewrite struct {
 	target []byte
 }
 
-type UrlRewriteMap []urlRewrite
+type URLRewriteMap []urlRewrite
 
-func (r *UrlRewriteMap) String() string {
+func (r *URLRewriteMap) String() string {
 	return fmt.Sprint(*r)
 }
 
-func (r *UrlRewriteMap) Set(value string) error {
+func (r *URLRewriteMap) Set(value string) error {
 	valArr := strings.SplitN(value, ":", 2)
 	if len(valArr) < 2 {
 		return errors.New("need both src and target, colon-delimited (ex. /a:/b)")
@@ -266,13 +266,13 @@ type urlRegexp struct {
 	regexp *regexp.Regexp
 }
 
-type HTTPUrlRegexp []urlRegexp
+type HTTPURLRegexp []urlRegexp
 
-func (r *HTTPUrlRegexp) String() string {
+func (r *HTTPURLRegexp) String() string {
 	return fmt.Sprint(*r)
 }
 
-func (r *HTTPUrlRegexp) Set(value string) error {
+func (r *HTTPURLRegexp) Set(value string) error {
 	regexp, err := regexp.Compile(value)
 
 	*r = append(*r, urlRegexp{regexp: regexp})

http_modifier_settings_test.go

@@ -53,7 +53,7 @@ func TestHTTPHashFilters(t *testing.T) {
 
 func TestUrlRewriteMap(t *testing.T) {
 	var err error
-	rewrites := UrlRewriteMap{}
+	rewrites := URLRewriteMap{}
 
 	if err = rewrites.Set("/v1/user/([^\\/]+)/ping:/v2/user/$1/ping"); err != nil {
 		t.Error("Should set mapping", err)

http_modifier_test.go

@@ -18,7 +18,7 @@ func TestHTTPModifierHeaderFilters(t *testing.T) {
 	filters.Set("Host:^www.w3.org$")
 
 	modifier := NewHTTPModifier(&HTTPModifierConfig{
-		headerFilters: filters,
+		HeaderFilters: filters,
 	})
 
 	payload := []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
@@ -32,7 +32,7 @@ func TestHTTPModifierHeaderFilters(t *testing.T) {
 	filters.Set("Host:^www.w4.org$")
 
 	modifier = NewHTTPModifier(&HTTPModifierConfig{
-		headerFilters: filters,
+		HeaderFilters: filters,
 	})
 
 	if len(modifier.Rewrite(payload)) != 0 {
@@ -45,7 +45,7 @@ func TestHTTPModifierHeaderNegativeFilters(t *testing.T) {
 	filters.Set("Host:^www.w3.org$")
 
 	modifier := NewHTTPModifier(&HTTPModifierConfig{
-		headerNegativeFilters: filters,
+		HeaderNegativeFilters: filters,
 	})
 
 	payload := []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w4.org\r\n\r\na=1&b=2")
@@ -59,7 +59,7 @@ func TestHTTPModifierHeaderNegativeFilters(t *testing.T) {
 	filters.Set("Host:^www.w4.org$")
 
 	modifier = NewHTTPModifier(&HTTPModifierConfig{
-		headerNegativeFilters: filters,
+		HeaderNegativeFilters: filters,
 	})
 
 	if len(modifier.Rewrite(payload)) != 0 {
@@ -71,7 +71,7 @@ func TestHTTPModifierHeaderNegativeFilters(t *testing.T) {
 	filters.Set("Host: www*")
 
 	modifier = NewHTTPModifier(&HTTPModifierConfig{
-		headerNegativeFilters: filters,
+		HeaderNegativeFilters: filters,
 	})
 
 	if len(modifier.Rewrite(payload)) != 0 {
@@ -84,7 +84,7 @@ func TestHTTPHeaderBasicAuthFilters(t *testing.T) {
 	filters.Set("^customer[0-9].*")
 
 	modifier := NewHTTPModifier(&HTTPModifierConfig{
-		headerBasicAuthFilters: filters,
+		HeaderBasicAuthFilters: filters,
 	})
 
 	//Encoded UserId:Password = customer3:welcome
@@ -104,7 +104,7 @@ func TestHTTPHeaderBasicAuthFilters(t *testing.T) {
 	filters.Set("^(homer simpson|mickey mouse).*")
 
 	modifier = NewHTTPModifier(&HTTPModifierConfig{
-		headerBasicAuthFilters: filters,
+		HeaderBasicAuthFilters: filters,
 	})
 
 	if len(modifier.Rewrite(payload)) != 0 {
@@ -121,7 +121,7 @@ func TestHTTPHeaderBasicAuthFilters(t *testing.T) {
 func TestHTTPModifierURLRewrite(t *testing.T) {
 	var url, newURL []byte
 
-	rewrites := UrlRewriteMap{}
+	rewrites := URLRewriteMap{}
 
 	payload := func(url []byte) []byte {
 		return []byte("POST " + string(url) + " HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
@@ -133,7 +133,7 @@ func TestHTTPModifierURLRewrite(t *testing.T) {
 	}
 
 	modifier := NewHTTPModifier(&HTTPModifierConfig{
-		urlRewrite: rewrites,
+		URLRewrite: rewrites,
 	})
 
 	url = []byte("/v1/user/joe/ping")
@@ -159,7 +159,7 @@ func TestHTTPModifierHeaderRewrite(t *testing.T) {
 	}
 
 	modifier := NewHTTPModifier(&HTTPModifierConfig{
-		headerRewrite: rewrites,
+		HeaderRewrite: rewrites,
 	})
 
 	header = []byte("www.beta.w3.org")
@@ -173,7 +173,7 @@ func TestHTTPModifierHeaderHashFilters(t *testing.T) {
 	filters.Set("Header2:1/2")
 
 	modifier := NewHTTPModifier(&HTTPModifierConfig{
-		headerHashFilters: filters,
+		HeaderHashFilters: filters,
 	})
 
 	payload := func(header []byte) []byte {
@@ -198,7 +198,7 @@ func TestHTTPModifierParamHashFilters(t *testing.T) {
 	filters.Set("user_id:1/2")
 
 	modifier := NewHTTPModifier(&HTTPModifierConfig{
-		paramHashFilters: filters,
+		ParamHashFilters: filters,
 	})
 
 	payload := func(value []byte) []byte {
@@ -224,7 +224,7 @@ func TestHTTPModifierHeaders(t *testing.T) {
 	headers.Set("Host:localhost")
 
 	modifier := NewHTTPModifier(&HTTPModifierConfig{
-		headers: headers,
+		Headers: headers,
 	})
 
 	payload := []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
@@ -236,12 +236,12 @@ func TestHTTPModifierHeaders(t *testing.T) {
 }
 
 func TestHTTPModifierURLRegexp(t *testing.T) {
-	filters := HTTPUrlRegexp{}
+	filters := HTTPURLRegexp{}
 	filters.Set("/v1/app")
 	filters.Set("/v1/api")
 
 	modifier := NewHTTPModifier(&HTTPModifierConfig{
-		urlRegexp: filters,
+		URLRegexp: filters,
 	})
 
 	payload := func(url string) []byte {
@@ -262,12 +262,12 @@ func TestHTTPModifierURLRegexp(t *testing.T) {
 }
 
 func TestHTTPModifierURLNegativeRegexp(t *testing.T) {
-	filters := HTTPUrlRegexp{}
+	filters := HTTPURLRegexp{}
 	filters.Set("/restricted1")
 	filters.Set("/some/restricted2")
 
 	modifier := NewHTTPModifier(&HTTPModifierConfig{
-		urlNegativeRegexp: filters,
+		URLNegativeRegexp: filters,
 	})
 
 	payload := func(url string) []byte {
@@ -292,7 +292,7 @@ func TestHTTPModifierSetHeader(t *testing.T) {
 	filters.Set("User-Agent:Gor")
 
 	modifier := NewHTTPModifier(&HTTPModifierConfig{
-		headers: filters,
+		Headers: filters,
 	})
 
 	payload := []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
@@ -308,7 +308,7 @@ func TestHTTPModifierSetParam(t *testing.T) {
 	filters.Set("api_key=1")
 
 	modifier := NewHTTPModifier(&HTTPModifierConfig{
-		params: filters,
+		Params: filters,
 	})
 
 	payload := []byte("POST /post?api_key=1234 HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")

http_prettifier.go

@@ -25,16 +25,14 @@ func prettifyHTTP(p []byte) []byte {
 	content := body[headersPos:]
 
 	var tEnc, cEnc []byte
-	proto.ParseHeaders([][]byte{headers}, func(header, value []byte) bool {
-		if proto.HeadersEqual(header, []byte("Transfer-Encoding")) {
+	proto.ParseHeaders([][]byte{headers}, func(header, value []byte) {
+		if bytes.EqualFold(header, []byte("Transfer-Encoding")) {
 			tEnc = value
 		}
 
-		if proto.HeadersEqual(header, []byte("Content-Encoding")) {
+		if bytes.EqualFold(header, []byte("Content-Encoding")) {
 			cEnc = value
 		}
-
-		return true
 	})
 
 	if len(tEnc) == 0 && len(cEnc) == 0 {
@@ -57,7 +55,7 @@ func prettifyHTTP(p []byte) []byte {
 		g, err := gzip.NewReader(buf)
 
 		if err != nil {
-			Debug("[Prettifier] GZIP encoding error:", err)
+			Debug(1, "[Prettifier] GZIP encoding error:", err)
 			return []byte{}
 		}
 

input_file.go

@@ -12,34 +12,36 @@ import (
 	"strconv"
 	"strings"
 	"sync"
+	"sync/atomic"
 	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/s3"
 )
 
 type fileInputReader struct {
 	reader    *bufio.Reader
 	data      []byte
-	file      *os.File
+	file      io.ReadCloser
 	timestamp int64
+	closed    int32 // Value of 0 indicates that the file is still open.
+	s3        bool
 }
 
 func (f *fileInputReader) parseNext() error {
 	payloadSeparatorAsBytes := []byte(payloadSeparator)
 	var buffer bytes.Buffer
-
 	for {
 		line, err := f.reader.ReadBytes('\n')
 
 		if err != nil {
 			if err != io.EOF {
-				log.Println(err)
-				return err
-			}
-
-			if err == io.EOF {
-				f.file.Close()
-				f.file = nil
-				return err
+				Debug(1, err)
+			} else {
+				f.Close()
 			}
+			return err
 		}
 
 		if bytes.Equal(payloadSeparatorAsBytes[1:], line) {
@@ -55,7 +57,6 @@ func (f *fileInputReader) parseNext() error {
 		buffer.Write(line)
 	}
 
-	return nil
 }
 
 func (f *fileInputReader) ReadPayload() []byte {
@@ -64,7 +65,8 @@ func (f *fileInputReader) ReadPayload() []byte {
 	return f.data
 }
 func (f *fileInputReader) Close() error {
-	if f.file != nil {
+	if atomic.LoadInt32(&f.closed) == 0 {
+		atomic.StoreInt32(&f.closed, 1)
 		f.file.Close()
 	}
 
@@ -72,14 +74,21 @@ func (f *fileInputReader) Close() error {
 }
 
 func NewFileInputReader(path string) *fileInputReader {
-	file, err := os.Open(path)
+	var file io.ReadCloser
+	var err error
+
+	if strings.HasPrefix(path, "s3://") {
+		file = NewS3ReadCloser(path)
+	} else {
+		file, err = os.Open(path)
+	}
 
 	if err != nil {
 		log.Println(err)
 		return nil
 	}
 
-	r := &fileInputReader{file: file}
+	r := &fileInputReader{file: file, closed: 0}
 	if strings.HasSuffix(path, ".gz") {
 		gzReader, err := gzip.NewReader(file)
 		if err != nil {
@@ -111,7 +120,7 @@ type FileInput struct {
 func NewFileInput(path string, loop bool) (i *FileInput) {
 	i = new(FileInput)
 	i.data = make(chan []byte, 1000)
-	i.exit = make(chan bool, 1)
+	i.exit = make(chan bool)
 	i.path = path
 	i.speedFactor = 1
 	i.loop = loop
@@ -125,21 +134,37 @@ func NewFileInput(path string, loop bool) (i *FileInput) {
 	return
 }
 
-type NextFileNotFound struct{}
-
-func (_ *NextFileNotFound) Error() string {
-	return "There is no new files"
-}
-
 func (i *FileInput) init() (err error) {
 	defer i.mu.Unlock()
 	i.mu.Lock()
 
 	var matches []string
 
-	if matches, err = filepath.Glob(i.path); err != nil {
-		log.Println("Wrong file pattern", i.path, err)
-		return
+	if strings.HasPrefix(i.path, "s3://") {
+		sess := session.Must(session.NewSession(awsConfig()))
+		svc := s3.New(sess)
+
+		bucket, key := parseS3Url(i.path)
+
+		params := &s3.ListObjectsInput{
+			Bucket: aws.String(bucket),
+			Prefix: aws.String(key),
+		}
+
+		resp, err := svc.ListObjects(params)
+		if err != nil {
+			log.Println("Error while retreiving list of files from S3", i.path, err)
+			return err
+		}
+
+		for _, c := range resp.Contents {
+			matches = append(matches, "s3://"+bucket+"/"+(*c.Key))
+		}
+	} else {
+		if matches, err = filepath.Glob(i.path); err != nil {
+			log.Println("Wrong file pattern", i.path, err)
+			return
+		}
 	}
 
 	if len(matches) == 0 {
@@ -157,10 +182,14 @@ func (i *FileInput) init() (err error) {
 }
 
 func (i *FileInput) Read(data []byte) (int, error) {
-	buf := <-i.data
-	copy(data, buf)
-
-	return len(buf), nil
+	var buf []byte
+	select {
+	case <-i.exit:
+		return 0, ErrorStopped
+	case buf = <-i.data:
+	}
+	n := copy(data, buf)
+	return n, nil
 }
 
 func (i *FileInput) String() string {
@@ -170,7 +199,7 @@ func (i *FileInput) String() string {
 // Find reader with smallest timestamp e.g next payload in row
 func (i *FileInput) nextReader() (next *fileInputReader) {
 	for _, r := range i.readers {
-		if r == nil || r.file == nil {
+		if r == nil || atomic.LoadInt32(&r.closed) != 0 {
 			continue
 		}
 
@@ -218,25 +247,25 @@ func (i *FileInput) emit() {
 			lastTime = reader.timestamp
 		}
 
-		i.data <- reader.ReadPayload()
+		// Recheck if we have exited since last check.
+		select {
+		case <-i.exit:
+			return
+		default:
+			i.data <- reader.ReadPayload()
+		}
 	}
 
 	log.Printf("FileInput: end of file '%s'\n", i.path)
 
-	// For now having fixed timeout is temporary solution
-	// Further should be modified, so outputs can report if their queue empty or not
-	time.Sleep(time.Second)
-	if closeCh != nil {
-		close(closeCh)
-	}
 }
 
+// Close closes this plugin
 func (i *FileInput) Close() error {
 	defer i.mu.Unlock()
 	i.mu.Lock()
 
-	i.exit <- true
-
+	close(i.exit)
 	for _, r := range i.readers {
 		r.Close()
 	}

input_file_test.go

@@ -10,7 +10,6 @@ import (
 	"math/rand"
 	"os"
 	"sync"
-	"syscall"
 	"testing"
 	"time"
 )
@@ -18,7 +17,6 @@ import (
 var _ = log.Println
 
 func TestInputFileWithGET(t *testing.T) {
-
 	input := NewTestInput()
 	rg := NewRequestGenerator([]io.Reader{input}, func() { input.EmitGET() }, 1)
 	readPayloads := [][]byte{}
@@ -231,14 +229,14 @@ func TestInputFileLoop(t *testing.T) {
 func TestInputFileCompressed(t *testing.T) {
 	rnd := rand.Int63()
 
-	output := NewFileOutput(fmt.Sprintf("/tmp/%d_0.gz", rnd), &FileOutputConfig{flushInterval: time.Minute, append: true})
+	output := NewFileOutput(fmt.Sprintf("/tmp/%d_0.gz", rnd), &FileOutputConfig{FlushInterval: time.Minute, Append: true})
 	for i := 0; i < 1000; i++ {
 		output.Write([]byte("1 1 1\r\ntest"))
 	}
 	name1 := output.file.Name()
 	output.Close()
 
-	output2 := NewFileOutput(fmt.Sprintf("/tmp/%d_1.gz", rnd), &FileOutputConfig{flushInterval: time.Minute, append: true})
+	output2 := NewFileOutput(fmt.Sprintf("/tmp/%d_1.gz", rnd), &FileOutputConfig{FlushInterval: time.Minute, Append: true})
 	for i := 0; i < 1000; i++ {
 		output2.Write([]byte("1 1 1\r\ntest"))
 	}
@@ -269,7 +267,7 @@ func NewExpectedCaptureFile(data [][]byte, file *os.File) *CaptureFile {
 
 func (expectedCaptureFile *CaptureFile) TearDown() {
 	if expectedCaptureFile.file != nil {
-		syscall.Unlink(expectedCaptureFile.file.Name())
+		os.Remove(expectedCaptureFile.file.Name())
 	}
 }
 
@@ -305,7 +303,6 @@ func (expectedCaptureFile *CaptureFile) PayloadsEqual(other [][]byte) bool {
 }
 
 func CreateCaptureFile(requestGenerator *RequestGenerator) *CaptureFile {
-
 	f, err := ioutil.TempFile("", "testmainconf")
 	if err != nil {
 		panic(err)
@@ -316,31 +313,34 @@ func CreateCaptureFile(requestGenerator *RequestGenerator) *CaptureFile {
 	readPayloads := [][]byte{}
 	output := NewTestOutput(func(data []byte) {
 		readPayloads = append(readPayloads, Duplicate(data))
-
 		requestGenerator.wg.Done()
 	})
 
-	outputFile := NewFileOutput(f.Name(), &FileOutputConfig{flushInterval: time.Minute, append: true})
+	outputFile := NewFileOutput(f.Name(), &FileOutputConfig{FlushInterval: time.Minute, Append: true})
 
-	Plugins.Inputs = requestGenerator.inputs
-	Plugins.Outputs = []io.Writer{output, outputFile}
+	plugins := &InOutPlugins{
+		Inputs:  requestGenerator.inputs,
+		Outputs: []io.Writer{output, outputFile},
+	}
+	for _, input := range requestGenerator.inputs {
+		plugins.All = append(plugins.All, input)
+	}
+	plugins.All = append(plugins.All, output, outputFile)
 
-	go Start(quit)
+	emitter := NewEmitter(quit)
+	go emitter.Start(plugins, Settings.Middleware)
 
 	requestGenerator.emit()
 	requestGenerator.wg.Wait()
 
 	time.Sleep(100 * time.Millisecond)
-	outputFile.Close()
-
-	close(quit)
+	emitter.Close()
 
 	return NewExpectedCaptureFile(readPayloads, f)
 
 }
 
 func ReadFromCaptureFile(captureFile *os.File, count int, callback writeCallback) (err error) {
-
 	quit := make(chan int)
 	wg := new(sync.WaitGroup)
 
@@ -350,11 +350,15 @@ func ReadFromCaptureFile(captureFile *os.File, count int, callback writeCallback
 		wg.Done()
 	})
 
-	Plugins.Inputs = []io.Reader{input}
-	Plugins.Outputs = []io.Writer{output}
+	plugins := &InOutPlugins{
+		Inputs:  []io.Reader{input},
+		Outputs: []io.Writer{output},
+	}
+	plugins.All = append(plugins.All, input, output)
 
 	wg.Add(count)
-	go Start(quit)
+	emitter := NewEmitter(quit)
+	go emitter.Start(plugins, Settings.Middleware)
 
 	done := make(chan int, 1)
 	go func() {
@@ -368,8 +372,7 @@ func ReadFromCaptureFile(captureFile *os.File, count int, callback writeCallback
 	case <-time.After(2 * time.Second):
 		err = errors.New("Timed out")
 	}
-	close(quit)
-
+	emitter.Close()
 	return
 
 }

input_http.go

@@ -13,13 +13,14 @@ type HTTPInput struct {
 	data     chan []byte
 	address  string
 	listener net.Listener
+	stop     chan bool // Channel used only to indicate goroutine should shutdown
 }
 
-// NewHTTPInput constructor for HTTPInput. Accepts address with port which he will listen on.
+// NewHTTPInput constructor for HTTPInput. Accepts address with port which it will listen on.
 func NewHTTPInput(address string) (i *HTTPInput) {
 	i = new(HTTPInput)
-	i.data = make(chan []byte, 10000)
-	i.address = address
+	i.data = make(chan []byte, 1000)
+	i.stop = make(chan bool)
 
 	i.listen(address)
 
@@ -27,28 +28,39 @@ func NewHTTPInput(address string) (i *HTTPInput) {
 }
 
 func (i *HTTPInput) Read(data []byte) (int, error) {
-	buf := <-i.data
-
+	var buf []byte
+	select {
+	case <-i.stop:
+		return 0, ErrorStopped
+	case buf = <-i.data:
+	}
 	header := payloadHeader(RequestPayload, uuid(), time.Now().UnixNano(), -1)
 
-	copy(data[0:len(header)], header)
-	copy(data[len(header):], buf)
+	n := copy(data, header)
+	if len(data) > len(header) {
+		n += copy(data[len(header):], buf)
+	}
+	dis := len(header) + len(buf) - n
+	if dis > 0 {
+		Debug(2, "[INPUT-HTTP] discarded", dis, "increase copy buffer size")
+	}
 
-	return len(buf) + len(header), nil
+	return n, nil
+}
+
+// Close closes this plugin
+func (i *HTTPInput) Close() error {
+	close(i.stop)
+	return nil
 }
 
 func (i *HTTPInput) handler(w http.ResponseWriter, r *http.Request) {
 	r.URL.Scheme = "http"
-	r.URL.Host = i.listener.Addr().String()
+	r.URL.Host = i.address
 
 	buf, _ := httputil.DumpRequestOut(r, true)
 	http.Error(w, http.StatusText(200), 200)
-
-	select {
-	case i.data <- buf:
-	default:
-		Debug("[INPUT-HTTP] Dropping requests because output can't process them fast enough")
-	}
+	i.data <- buf
 }
 
 func (i *HTTPInput) listen(address string) {
@@ -62,11 +74,12 @@ func (i *HTTPInput) listen(address string) {
 	if err != nil {
 		log.Fatal("HTTP input listener failure:", err)
 	}
+	i.address = i.listener.Addr().String()
 
 	go func() {
 		err = http.Serve(i.listener, mux)
-		if err != nil {
-			log.Fatal("HTTP input serve failure:", err)
+		if err != nil && err != http.ErrServerClosed {
+			log.Fatal("HTTP input serve failure ", err)
 		}
 	}()
 }

input_http_test.go

@@ -1,13 +1,13 @@
 package main
 
 import (
+	"bytes"
 	"io"
-	"log"
 	"net/http"
-	"os/exec"
 	"strings"
 	"sync"
 	"testing"
+	"time"
 
 	"github.com/buger/goreplay/proto"
 )
@@ -17,16 +17,21 @@ func TestHTTPInput(t *testing.T) {
 	quit := make(chan int)
 
 	input := NewHTTPInput("127.0.0.1:0")
+	time.Sleep(time.Millisecond)
 	output := NewTestOutput(func(data []byte) {
 		wg.Done()
 	})
 
-	Plugins.Inputs = []io.Reader{input}
-	Plugins.Outputs = []io.Writer{output}
+	plugins := &InOutPlugins{
+		Inputs:  []io.Reader{input},
+		Outputs: []io.Writer{output},
+	}
+	plugins.All = append(plugins.All, input, output)
 
-	go Start(quit)
+	emitter := NewEmitter(quit)
+	go emitter.Start(plugins, Settings.Middleware)
 
-	address := strings.Replace(input.listener.Addr().String(), "[::]", "127.0.0.1", -1)
+	address := strings.Replace(input.address, "[::]", "127.0.0.1", -1)
 
 	for i := 0; i < 100; i++ {
 		wg.Add(1)
@@ -34,40 +39,48 @@ func TestHTTPInput(t *testing.T) {
 	}
 
 	wg.Wait()
-
-	close(quit)
+	emitter.Close()
 }
 
 func TestInputHTTPLargePayload(t *testing.T) {
 	wg := new(sync.WaitGroup)
-	quit := make(chan int)
-
-	dd := exec.Command("dd", "if=/dev/urandom", "of=/tmp/large", "bs=1", "count=4000000")
-	err := dd.Run()
-	if err != nil {
-		log.Fatal("dd error:", err)
-	}
+	quit := make(chan int, 1)
+	const n = 10 << 20 // 10MB
+	var large [n]byte
+	large[n-1] = '0'
 
 	input := NewHTTPInput("127.0.0.1:0")
+	time.Sleep(time.Millisecond)
 	output := NewTestOutput(func(data []byte) {
-		if len(proto.Body(payloadBody(data))) != 4000000 {
-			t.Error("Should receive full file")
+		_len := len(proto.Body(payloadBody(data)))
+		if _len >= n { // considering http body CRLF
+			t.Errorf("expected body to be >= %d", n)
 		}
 		wg.Done()
 	})
-	Plugins.Inputs = []io.Reader{input}
-	Plugins.Outputs = []io.Writer{output}
-
-	go Start(quit)
-
-	wg.Add(1)
-	address := strings.Replace(input.listener.Addr().String(), "[::]", "127.0.0.1", -1)
-	curl := exec.Command("curl", "http://"+address, "--data-binary", "@/tmp/large")
-	err = curl.Run()
-	if err != nil {
-		log.Fatal("curl error:", err)
+	plugins := &InOutPlugins{
+		Inputs:  []io.Reader{input},
+		Outputs: []io.Writer{output},
 	}
+	plugins.All = append(plugins.All, input, output)
 
+	emitter := NewEmitter(quit)
+	defer emitter.Close()
+	go emitter.Start(plugins, Settings.Middleware)
+
+	address := strings.Replace(input.address, "[::]", "127.0.0.1", -1)
+	var req *http.Request
+	var err error
+	req, err = http.NewRequest("POST", "http://"+address, bytes.NewBuffer(large[:]))
+	if err != nil {
+		t.Error(err)
+		return
+	}
+	wg.Add(1)
+	_, err = http.DefaultClient.Do(req)
+	if err != nil {
+		t.Error(err)
+		return
+	}
 	wg.Wait()
-	close(quit)
 }

input_kafka.go

@@ -12,15 +12,19 @@ import (
 // KafkaInput is used for recieving Kafka messages and
 // transforming them into HTTP payloads.
 type KafkaInput struct {
-	config    *KafkaConfig
+	config    *InputKafkaConfig
 	consumers []sarama.PartitionConsumer
 	messages  chan *sarama.ConsumerMessage
 }
 
 // NewKafkaInput creates instance of kafka consumer client.
-func NewKafkaInput(address string, config *KafkaConfig) *KafkaInput {
-	c := sarama.NewConfig()
-	// Configuration options go here
+func NewKafkaInput(address string, config *InputKafkaConfig) *KafkaInput {
+	return NewKafkaInputWithTLS(address, config, nil)
+}
+
+// NewKafkaInputWithTLS creates instance of kafka consumer client with TLS
+func NewKafkaInputWithTLS(address string, config *InputKafkaConfig, tlsConfig *KafkaTLSConfig) *KafkaInput {
+	c := NewKafkaConfig(tlsConfig)
 
 	var con sarama.Consumer
 
@@ -28,15 +32,15 @@ func NewKafkaInput(address string, config *KafkaConfig) *KafkaInput {
 		con = config.consumer
 	} else {
 		var err error
-		//con, err = sarama.NewConsumer([]string{config.host}, c)
-		con, err = sarama.NewConsumer(strings.Split(config.host, ","), c)
+		//con, err = sarama.NewConsumer([]string{config.Host}, c)
+		con, err = sarama.NewConsumer(strings.Split(config.Host, ","), c)
 
 		if err != nil {
 			log.Fatalln("Failed to start Sarama(Kafka) consumer:", err)
 		}
 	}
 
-	partitions, err := con.Partitions(config.topic)
+	partitions, err := con.Partitions(config.Topic)
 	if err != nil {
 		log.Fatalln("Failed to collect Sarama(Kafka) partitions:", err)
 	}
@@ -48,7 +52,7 @@ func NewKafkaInput(address string, config *KafkaConfig) *KafkaInput {
 	}
 
 	for index, partition := range partitions {
-		consumer, err := con.ConsumePartition(config.topic, partition, sarama.OffsetNewest)
+		consumer, err := con.ConsumePartition(config.Topic, partition, sarama.OffsetNewest)
 		if err != nil {
 			log.Fatalln("Failed to start Sarama(Kafka) partition consumer:", err)
 		}
@@ -61,10 +65,7 @@ func NewKafkaInput(address string, config *KafkaConfig) *KafkaInput {
 			}
 		}(consumer)
 
-		if Settings.verbose {
-			// Start infinite loop for tracking errors for kafka producer.
-			go i.ErrorHandler(consumer)
-		}
+		go i.ErrorHandler(consumer)
 
 		i.consumers[index] = consumer
 	}
@@ -75,14 +76,14 @@ func NewKafkaInput(address string, config *KafkaConfig) *KafkaInput {
 // ErrorHandler should receive errors
 func (i *KafkaInput) ErrorHandler(consumer sarama.PartitionConsumer) {
 	for err := range consumer.Errors() {
-		log.Println("Failed to read access log entry:", err)
+		Debug(1, "Failed to read access log entry:", err)
 	}
 }
 
 func (i *KafkaInput) Read(data []byte) (int, error) {
 	message := <-i.messages
 
-	if !i.config.useJSON {
+	if !i.config.UseJSON {
 		copy(data, message.Value)
 		return len(message.Value), nil
 	}
@@ -92,7 +93,7 @@ func (i *KafkaInput) Read(data []byte) (int, error) {
 
 	buf, err := kafkaMessage.Dump()
 	if err != nil {
-		log.Println("Failed to decode access log entry:", err)
+		Debug(1, "Failed to decode access log entry:", err)
 		return 0, err
 	}
 
@@ -103,5 +104,5 @@ func (i *KafkaInput) Read(data []byte) (int, error) {
 }
 
 func (i *KafkaInput) String() string {
-	return "Kafka Input: " + i.config.host + "/" + i.config.topic
+	return "Kafka Input: " + i.config.Host + "/" + i.config.Topic
 }

input_kafka_test.go

@@ -16,10 +16,10 @@ func TestInputKafkaRAW(t *testing.T) {
 		map[string][]int32{"test": {0}},
 	)
 
-	input := NewKafkaInput("", &KafkaConfig{
+	input := NewKafkaInput("", &InputKafkaConfig{
 		consumer: consumer,
-		topic:    "test",
-		useJSON:  false,
+		Topic:    "test",
+		UseJSON:  false,
 	})
 
 	buf := make([]byte, 1024)
@@ -43,10 +43,10 @@ func TestInputKafkaJSON(t *testing.T) {
 		map[string][]int32{"test": {0}},
 	)
 
-	input := NewKafkaInput("", &KafkaConfig{
+	input := NewKafkaInput("", &InputKafkaConfig{
 		consumer: consumer,
-		topic:    "test",
-		useJSON:  true,
+		Topic:    "test",
+		UseJSON:  true,
 	})
 
 	buf := make([]byte, 1024)

input_raw.go

@@ -1,112 +1,214 @@
 package main
 
 import (
+	"context"
+	"fmt"
 	"log"
 	"net"
+	"strconv"
+	"sync"
 	"time"
 
+	"github.com/buger/goreplay/capture"
 	"github.com/buger/goreplay/proto"
-	raw "github.com/buger/goreplay/raw_socket_listener"
+	"github.com/buger/goreplay/size"
+	"github.com/buger/goreplay/tcp"
 )
 
+// TCPProtocol is a number to indicate type of protocol
+type TCPProtocol uint8
+
+const (
+	// ProtocolHTTP ...
+	ProtocolHTTP TCPProtocol = iota
+	// ProtocolBinary ...
+	ProtocolBinary
+)
+
+// Set is here so that TCPProtocol can implement flag.Var
+func (protocol *TCPProtocol) Set(v string) error {
+	switch v {
+	case "", "http":
+		*protocol = ProtocolHTTP
+	case "binary":
+		*protocol = ProtocolBinary
+	default:
+		return fmt.Errorf("unsupported protocol %s", v)
+	}
+	return nil
+}
+
+func (protocol *TCPProtocol) String() string {
+	switch *protocol {
+	case ProtocolBinary:
+		return "binary"
+	case ProtocolHTTP:
+		return "http"
+	default:
+		return ""
+	}
+}
+
+// RAWInputConfig represents configuration that can be applied on raw input
+type RAWInputConfig struct {
+	capture.PcapOptions
+	Expire         time.Duration      `json:"input-raw-expire"`
+	CopyBufferSize size.Size          `json:"copy-buffer-size"`
+	Engine         capture.EngineType `json:"input-raw-engine"`
+	TrackResponse  bool               `json:"input-raw-track-response"`
+	Protocol       TCPProtocol        `json:"input-raw-protocol"`
+	RealIPHeader   string             `json:"input-raw-realip-header"`
+	Stats          bool               `json:"input-raw-stats"`
+	quit           chan bool          // Channel used only to indicate goroutine should shutdown
+	host           string
+	port           uint16
+}
+
 // RAWInput used for intercepting traffic for given address
 type RAWInput struct {
-	data          chan *raw.TCPMessage
-	address       string
-	expire        time.Duration
-	quit          chan bool
-	engine        int
-	realIPHeader  []byte
-	trackResponse bool
-	listener      *raw.Listener
-	bpfFilter     string
-	timestampType string
-	bufferSize    int
+	sync.Mutex
+	RAWInputConfig
+	messageStats   []tcp.Stats
+	listener       *capture.Listener
+	message        chan *tcp.Message
+	cancelListener context.CancelFunc
 }
 
-// Available engines for intercepting traffic
-const (
-	EngineRawSocket = 1 << iota
-	EnginePcap
-	EnginePcapFile
-)
-
-// NewRAWInput constructor for RAWInput. Accepts address with port as argument.
-func NewRAWInput(address string, engine int, trackResponse bool, expire time.Duration, realIPHeader string, bpfFilter string, timestampType string, bufferSize int) (i *RAWInput) {
+// NewRAWInput constructor for RAWInput. Accepts raw input config as arguments.
+func NewRAWInput(address string, config RAWInputConfig) (i *RAWInput) {
 	i = new(RAWInput)
-	i.data = make(chan *raw.TCPMessage)
-	i.address = address
-	i.expire = expire
-	i.engine = engine
-	i.bpfFilter = bpfFilter
-	i.realIPHeader = []byte(realIPHeader)
+	i.RAWInputConfig = config
+	i.message = make(chan *tcp.Message, 1000)
 	i.quit = make(chan bool)
-	i.trackResponse = trackResponse
-	i.timestampType = timestampType
-	i.bufferSize = bufferSize
+	var host, _port string
+	var err error
+	var port int
+	host, _port, err = net.SplitHostPort(address)
+	if err != nil {
+		log.Fatalf("input-raw: error while parsing address: %s", err)
+	}
+	if _port != "" {
+		port, err = strconv.Atoi(_port)
+	}
+
+	if err != nil {
+		log.Fatalf("parsing port error: %v", err)
+	}
+	i.host = host
+	i.port = uint16(port)
 
 	i.listen(address)
-	i.listener.IsReady()
 
 	return
 }
 
-func (i *RAWInput) Read(data []byte) (int, error) {
-	msg := <-i.data
-	buf := msg.Bytes()
-
+func (i *RAWInput) Read(data []byte) (n int, err error) {
+	var msg *tcp.Message
+	var buf []byte
+	select {
+	case <-i.quit:
+		return 0, ErrorStopped
+	case msg = <-i.message:
+		buf = msg.Data()
+	}
 	var header []byte
 
+	var msgType byte = ResponsePayload
 	if msg.IsIncoming {
-		header = payloadHeader(RequestPayload, msg.UUID(), msg.Start.UnixNano(), -1)
-		if len(i.realIPHeader) > 0 {
-			buf = proto.SetHeader(buf, i.realIPHeader, []byte(msg.IP().String()))
+		msgType = RequestPayload
+		if i.RealIPHeader != "" {
+			buf = proto.SetHeader(buf, []byte(i.RealIPHeader), []byte(msg.SrcAddr))
 		}
-	} else {
-		header = payloadHeader(ResponsePayload, msg.UUID(), msg.Start.UnixNano(), msg.End.UnixNano()-msg.AssocMessage.End.UnixNano())
 	}
+	header = payloadHeader(msgType, msg.UUID(), msg.Start.UnixNano(), msg.End.UnixNano()-msg.Start.UnixNano())
 
-	copy(data[0:len(header)], header)
-	copy(data[len(header):], buf)
-
-	return len(buf) + len(header), nil
+	n = copy(data, header)
+	if len(data) > len(header) {
+		n += copy(data[len(header):], buf)
+	}
+	// to be removed....
+	if msg.Truncated || len(header)+len(buf)-n > 0 {
+		go Debug(2, "[INPUT-RAW] message truncated, increase copy-buffer-size")
+	}
+	// to be removed...
+	if msg.TimedOut {
+		go Debug(2, "[INPUT-RAW] message timeout reached, increase input-raw-expire")
+	}
+	if i.Stats {
+		stat := msg.Stats
+		go i.addStats(stat)
+	}
+	msg = nil
+	return n, nil
 }
 
 func (i *RAWInput) listen(address string) {
-	Debug("Listening for traffic on: " + address)
-
-	host, port, err := net.SplitHostPort(address)
-
+	var err error
+	i.listener, err = capture.NewListener(i.host, i.port, "", i.Engine, i.TrackResponse)
 	if err != nil {
-		log.Fatal("input-raw: error while parsing address", err)
+		log.Fatal(err)
 	}
+	i.listener.SetPcapOptions(i.PcapOptions)
+	err = i.listener.Activate()
+	if err != nil {
+		log.Fatal(err)
+	}
+	pool := tcp.NewMessagePool(i.CopyBufferSize, i.Expire, Debug, i.handler)
+	pool.End = endHint
+	pool.Start = startHint
+	var ctx context.Context
+	ctx, i.cancelListener = context.WithCancel(context.Background())
+	errCh := i.listener.ListenBackground(ctx, pool.Handler)
+	select {
+	case err := <-errCh:
+		log.Fatal(err)
+	case <-i.listener.Reading:
+		Debug(1, i)
+	}
+}
 
-	i.listener = raw.NewListener(host, port, i.engine, i.trackResponse, i.expire, i.bpfFilter, i.timestampType, i.bufferSize, Settings.inputRAWOverrideSnapLen, Settings.inputRAWImmediateMode)
-
-	ch := i.listener.Receiver()
-
-	go func() {
-		for {
-			select {
-			case <-i.quit:
-				return
-			default:
-			}
-
-			// Receiving TCPMessage object
-			m := <-ch
-
-			i.data <- m
-		}
-	}()
+func (i *RAWInput) handler(m *tcp.Message) {
+	i.message <- m
 }
 
 func (i *RAWInput) String() string {
-	return "Intercepting traffic from: " + i.address
+	return fmt.Sprintf("Intercepting traffic from: %s:%d", i.host, i.port)
 }
 
+// GetStats returns the stats so far and reset the stats
+func (i *RAWInput) GetStats() []tcp.Stats {
+	i.Lock()
+	defer func() {
+		i.messageStats = []tcp.Stats{}
+		i.Unlock()
+	}()
+	return i.messageStats
+}
+
+// Close closes the input raw listener
 func (i *RAWInput) Close() error {
-	i.listener.Close()
+	i.cancelListener()
 	close(i.quit)
 	return nil
 }
+
+func (i *RAWInput) addStats(mStats tcp.Stats) {
+	i.Lock()
+	if len(i.messageStats) >= 10000 {
+		i.messageStats = []tcp.Stats{}
+	}
+	i.messageStats = append(i.messageStats, mStats)
+	i.Unlock()
+}
+
+func startHint(pckt *tcp.Packet) (isIncoming, isOutgoing bool) {
+	isIncoming = proto.HasRequestTitle(pckt.Payload)
+	if isIncoming {
+		return
+	}
+	return false, proto.HasResponseTitle(pckt.Payload)
+}
+
+func endHint(m *tcp.Message) bool {
+	return proto.HasFullPayload(m.Data())
+}

input_raw_test.go

@@ -4,21 +4,18 @@ import (
 	"bytes"
 	"io"
 	"io/ioutil"
-	"log"
-	"math/rand"
 	"net"
 	"net/http"
 	"net/http/httptest"
 	"net/http/httputil"
-	"os"
 	"os/exec"
-	"strconv"
 	"strings"
 	"sync"
 	"sync/atomic"
 	"testing"
 	"time"
 
+	"github.com/buger/goreplay/capture"
 	"github.com/buger/goreplay/proto"
 )
 
@@ -30,22 +27,29 @@ func TestRAWInputIPv4(t *testing.T) {
 
 	listener, err := net.Listen("tcp", ":0")
 	if err != nil {
-		t.Fatal(err)
+		t.Error(err)
+		return
 	}
 	origin := &http.Server{
-		Handler:      http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {}),
+		Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			w.Write([]byte("ab"))
+		}),
 		ReadTimeout:  10 * time.Second,
 		WriteTimeout: 10 * time.Second,
 	}
 	go origin.Serve(listener)
 	defer listener.Close()
-
-	originAddr := listener.Addr().String()
+	_, port, _ := net.SplitHostPort(listener.Addr().String())
 
 	var respCounter, reqCounter int64
-
-	input := NewRAWInput(originAddr, EnginePcap, true, testRawExpire, "X-Real-IP", "", "", 0)
-	defer input.Close()
+	conf := RAWInputConfig{
+		Engine:        capture.EnginePcap,
+		Expire:        0,
+		Protocol:      ProtocolHTTP,
+		TrackResponse: true,
+		RealIPHeader:  "X-Real-IP",
+	}
+	input := NewRAWInput(":"+port, conf)
 
 	output := NewTestOutput(func(data []byte) {
 		if data[0] == '1' {
@@ -57,45 +61,46 @@ func TestRAWInputIPv4(t *testing.T) {
 		} else {
 			atomic.AddInt64(&respCounter, 1)
 		}
-
-		if Settings.debug {
-			log.Println(reqCounter, respCounter)
-		}
-
 		wg.Done()
 	})
 
-	Plugins.Inputs = []io.Reader{input}
-	Plugins.Outputs = []io.Writer{output}
-
-	client := NewHTTPClient("http://"+listener.Addr().String(), &HTTPClientConfig{})
-
-	go Start(quit)
-
-	for i := 0; i < 100; i++ {
-		// request + response
-		wg.Add(2)
-		client.Get("/")
-		time.Sleep(2 * time.Millisecond)
+	plugins := &InOutPlugins{
+		Inputs:  []io.Reader{input},
+		Outputs: []io.Writer{output},
 	}
+	plugins.All = append(plugins.All, input, output)
 
+	client := NewHTTPClient("127.0.0.1:"+port, &HTTPClientConfig{})
+
+	emitter := NewEmitter(quit)
+	defer emitter.Close()
+	go emitter.Start(plugins, Settings.Middleware)
+	for i := 0; i < 10; i++ {
+		wg.Add(2)
+		_, err = client.Get("/")
+		if err != nil {
+			t.Error(err)
+			return
+		}
+	}
 	wg.Wait()
-
-	close(quit)
+	const want = 10
+	if reqCounter != respCounter && reqCounter != want {
+		t.Errorf("want %d requests and %d responses, got %d requests and %d responses", want, want, reqCounter, respCounter)
+	}
 }
 
 func TestRAWInputNoKeepAlive(t *testing.T) {
 	wg := new(sync.WaitGroup)
 	quit := make(chan int)
 
-	listener, err := net.Listen("tcp", ":0")
+	listener, err := net.Listen("tcp", "127.0.0.1:0")
 	if err != nil {
 		t.Fatal(err)
 	}
 	origin := &http.Server{
 		Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-			w.Write([]byte("a"))
-			w.Write([]byte("b"))
+			w.Write([]byte("ab"))
 		}),
 		ReadTimeout:  10 * time.Second,
 		WriteTimeout: 10 * time.Second,
@@ -103,33 +108,52 @@ func TestRAWInputNoKeepAlive(t *testing.T) {
 	origin.SetKeepAlivesEnabled(false)
 	go origin.Serve(listener)
 	defer listener.Close()
+	_, port, _ := net.SplitHostPort(listener.Addr().String())
 
-	originAddr := listener.Addr().String()
-
-	input := NewRAWInput(originAddr, EnginePcap, true, testRawExpire, "", "", "", 0)
-	defer input.Close()
-
+	conf := RAWInputConfig{
+		Engine:        capture.EnginePcap,
+		Expire:        testRawExpire,
+		Protocol:      ProtocolHTTP,
+		TrackResponse: true,
+	}
+	input := NewRAWInput(":"+port, conf)
+	var respCounter, reqCounter int64
 	output := NewTestOutput(func(data []byte) {
+		if data[0] == '1' {
+			atomic.AddInt64(&reqCounter, 1)
+		} else {
+			atomic.AddInt64(&respCounter, 1)
+		}
 		wg.Done()
 	})
 
-	Plugins.Inputs = []io.Reader{input}
-	Plugins.Outputs = []io.Writer{output}
+	plugins := &InOutPlugins{
+		Inputs:  []io.Reader{input},
+		Outputs: []io.Writer{output},
+	}
+	plugins.All = append(plugins.All, input, output)
 
-	client := NewHTTPClient("http://"+listener.Addr().String(), &HTTPClientConfig{})
+	client := NewHTTPClient("127.0.0.1:"+port, &HTTPClientConfig{})
 
-	go Start(quit)
+	emitter := NewEmitter(quit)
+	go emitter.Start(plugins, Settings.Middleware)
 
-	for i := 0; i < 100; i++ {
+	for i := 0; i < 10; i++ {
 		// request + response
 		wg.Add(2)
-		client.Get("/")
-		time.Sleep(2 * time.Millisecond)
+		_, err = client.Get("/")
+		if err != nil {
+			t.Error(err)
+			return
+		}
 	}
 
 	wg.Wait()
-
-	close(quit)
+	const want = 10
+	if reqCounter != respCounter && reqCounter != want {
+		t.Errorf("want %d requests and %d responses, got %d requests and %d responses", want, want, reqCounter, respCounter)
+	}
+	emitter.Close()
 }
 
 func TestRAWInputIPv6(t *testing.T) {
@@ -138,22 +162,27 @@ func TestRAWInputIPv6(t *testing.T) {
 
 	listener, err := net.Listen("tcp", "[::1]:0")
 	if err != nil {
-		t.Fatal(err)
+		return
 	}
 	origin := &http.Server{
-		Handler:      http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {}),
+		Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			w.Write([]byte("ab"))
+		}),
 		ReadTimeout:  10 * time.Second,
 		WriteTimeout: 10 * time.Second,
 	}
 	go origin.Serve(listener)
 	defer listener.Close()
-
-	originAddr := listener.Addr().String()
+	_, port, _ := net.SplitHostPort(listener.Addr().String())
+	originAddr := "[::1]:" + port
 
 	var respCounter, reqCounter int64
-
-	input := NewRAWInput(originAddr, EnginePcap, true, testRawExpire, "", "", "", 0)
-	defer input.Close()
+	conf := RAWInputConfig{
+		Engine:        capture.EnginePcap,
+		Protocol:      ProtocolHTTP,
+		TrackResponse: true,
+	}
+	input := NewRAWInput(originAddr, conf)
 
 	output := NewTestOutput(func(data []byte) {
 		if data[0] == '1' {
@@ -161,94 +190,34 @@ func TestRAWInputIPv6(t *testing.T) {
 		} else {
 			atomic.AddInt64(&respCounter, 1)
 		}
-
-		if Settings.debug {
-			log.Println(reqCounter, respCounter)
-		}
-
 		wg.Done()
 	})
 
-	Plugins.Inputs = []io.Reader{input}
-	Plugins.Outputs = []io.Writer{output}
+	plugins := &InOutPlugins{
+		Inputs:  []io.Reader{input},
+		Outputs: []io.Writer{output},
+	}
 
-	client := NewHTTPClient("http://"+listener.Addr().String(), &HTTPClientConfig{})
+	client := NewHTTPClient(originAddr, &HTTPClientConfig{})
 
-	go Start(quit)
-
-	for i := 0; i < 100; i++ {
+	emitter := NewEmitter(quit)
+	go emitter.Start(plugins, Settings.Middleware)
+	for i := 0; i < 10; i++ {
 		// request + response
 		wg.Add(2)
-		client.Get("/")
-		time.Sleep(2 * time.Millisecond)
+		_, err = client.Get("/")
+		if err != nil {
+			t.Error(err)
+			return
+		}
 	}
 
 	wg.Wait()
-	close(quit)
-}
-
-func TestInputRAW100Expect(t *testing.T) {
-	wg := new(sync.WaitGroup)
-	quit := make(chan int)
-
-	fileContent, _ := ioutil.ReadFile("COMM-LICENSE")
-
-	// Origing and Replay server initialization
-	origin := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		defer r.Body.Close()
-		ioutil.ReadAll(r.Body)
-		wg.Done()
-	}))
-	defer origin.Close()
-
-	originAddr := strings.Replace(origin.Listener.Addr().String(), "[::]", "127.0.0.1", -1)
-
-	input := NewRAWInput(originAddr, EnginePcap, true, time.Second, "", "", "", 0)
-	defer input.Close()
-
-	// We will use it to get content of raw HTTP request
-	testOutput := NewTestOutput(func(data []byte) {
-		switch data[0] {
-		case RequestPayload:
-			if strings.Contains(string(data), "Expect: 100-continue") {
-				t.Error("Should not contain 100-continue header")
-			}
-			wg.Done()
-		case ResponsePayload:
-			wg.Done()
-		}
-	})
-
-	replay := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		defer r.Body.Close()
-		body, _ := ioutil.ReadAll(r.Body)
-
-		if !bytes.Equal(body, fileContent) {
-			buf, _ := httputil.DumpRequest(r, true)
-			t.Error("Wrong POST body:", string(buf))
-		}
-
-		wg.Done()
-	}))
-	defer replay.Close()
-
-	httpOutput := NewHTTPOutput(replay.URL, &HTTPOutputConfig{})
-
-	Plugins.Inputs = []io.Reader{input}
-	Plugins.Outputs = []io.Writer{testOutput, httpOutput}
-
-	go Start(quit)
-
-	// Origin + Response/Request Test Output + Request Http Output
-	wg.Add(4)
-	curl := exec.Command("curl", "http://"+originAddr, "--data-binary", "@COMM-LICENSE")
-	err := curl.Run()
-	if err != nil {
-		log.Fatal(err)
+	const want = 10
+	if reqCounter != respCounter && reqCounter != want {
+		t.Errorf("want %d requests and %d responses, got %d requests and %d responses", want, want, reqCounter, respCounter)
 	}
-
-	wg.Wait()
-	close(quit)
+	emitter.Close()
 }
 
 func TestInputRAWChunkedEncoding(t *testing.T) {
@@ -266,8 +235,13 @@ func TestInputRAWChunkedEncoding(t *testing.T) {
 	}))
 
 	originAddr := strings.Replace(origin.Listener.Addr().String(), "[::]", "127.0.0.1", -1)
-	input := NewRAWInput(originAddr, EnginePcap, true, time.Second, "", "", "", 0)
-	defer input.Close()
+	conf := RAWInputConfig{
+		Engine:        capture.EnginePcap,
+		Expire:        time.Second,
+		Protocol:      ProtocolHTTP,
+		TrackResponse: true,
+	}
+	input := NewRAWInput(originAddr, conf)
 
 	replay := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		defer r.Body.Close()
@@ -284,141 +258,113 @@ func TestInputRAWChunkedEncoding(t *testing.T) {
 
 	httpOutput := NewHTTPOutput(replay.URL, &HTTPOutputConfig{Debug: true})
 
-	Plugins.Inputs = []io.Reader{input}
-	Plugins.Outputs = []io.Writer{httpOutput}
-
-	go Start(quit)
+	plugins := &InOutPlugins{
+		Inputs:  []io.Reader{input},
+		Outputs: []io.Writer{httpOutput},
+	}
+	plugins.All = append(plugins.All, input, httpOutput)
 
+	emitter := NewEmitter(quit)
+	defer emitter.Close()
+	go emitter.Start(plugins, Settings.Middleware)
 	wg.Add(2)
 
 	curl := exec.Command("curl", "http://"+originAddr, "--header", "Transfer-Encoding: chunked", "--header", "Expect:", "--data-binary", "@README.md")
 	err := curl.Run()
 	if err != nil {
-		log.Fatal(err)
-	}
-
-	wg.Wait()
-
-	close(quit)
-}
-
-func TestInputRAWLargePayload(t *testing.T) {
-	// FIXME: Large payloads does not work for travis for some reason...
-	if os.Getenv("TRAVIS_BUILD_DIR") != "" {
+		t.Error(err)
 		return
 	}
-	wg := new(sync.WaitGroup)
-	quit := make(chan int)
-	sizeB := 100 * 1000
 
-	// Generate 100kb file
-	dd := exec.Command("dd", "if=/dev/urandom", "of=/tmp/large", "bs=1", "count="+strconv.Itoa(sizeB))
-	err := dd.Run()
+	wg.Wait()
+}
+
+func BenchmarkRAWInputWithReplay(b *testing.B) {
+	var respCounter, reqCounter, replayCounter, capturedBody uint64
+	wg := &sync.WaitGroup{}
+	wg.Add(b.N * 3) // reqCounter + replayCounter + respCounter
+
+	quit := make(chan int)
+	listener, err := net.Listen("tcp", "127.0.0.1:0")
 	if err != nil {
-		log.Fatal("dd error:", err)
+		b.Error(err)
+		return
+	}
+	listener0, err := net.Listen("tcp", "127.0.0.1:0")
+	if err != nil {
+		b.Error(err)
+		return
 	}
 
-	origin := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
-		defer req.Body.Close()
-		body, _ := ioutil.ReadAll(req.Body)
+	origin := http.Server{
+		Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			w.Write([]byte("ab"))
+		}),
+	}
+	go origin.Serve(listener)
+	defer origin.Close()
+	originAddr := listener.Addr().String()
 
-		if len(body) != sizeB {
-			t.Error("File size should be 1mb:", len(body))
-		}
-
-		wg.Done()
-	}))
-	originAddr := strings.Replace(origin.Listener.Addr().String(), "[::]", "127.0.0.1", -1)
-
-	input := NewRAWInput(originAddr, EnginePcap, true, testRawExpire, "", "", "", 0)
-	defer input.Close()
-
-	replay := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
-		body, _ := ioutil.ReadAll(req.Body)
-		// // req.Body = http.MaxBytesReader(w, req.Body, 1*1024*1024)
-		// // buf := make([]byte, 1*1024*1024)
-		// n, _ := req.Body.Read(buf)
-		// body := buf[0:n]
-
-		if len(body) != sizeB {
-			t.Errorf("File size should be %d bytes: %d", sizeB, len(body))
-		}
-
-		wg.Done()
-	}))
+	replay := http.Server{
+		Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			defer wg.Done()
+			defer r.Body.Close()
+			w.Write([]byte("ab"))
+			atomic.AddUint64(&replayCounter, 1)
+			data, err := ioutil.ReadAll(r.Body)
+			if err != nil && err != io.EOF && err != io.ErrUnexpectedEOF {
+				b.Log(err)
+			}
+			atomic.AddUint64(&capturedBody, uint64(len(data)))
+		}),
+	}
+	go replay.Serve(listener0)
 	defer replay.Close()
+	replayAddr := listener0.Addr().String()
 
-	httpOutput := NewHTTPOutput(replay.URL, &HTTPOutputConfig{Debug: false})
+	conf := RAWInputConfig{
+		Engine:        capture.EnginePcap,
+		Expire:        testRawExpire,
+		Protocol:      ProtocolHTTP,
+		TrackResponse: true,
+	}
+	input := NewRAWInput(originAddr, conf)
 
-	Plugins.Inputs = []io.Reader{input}
-	Plugins.Outputs = []io.Writer{httpOutput}
+	testOutput := NewTestOutput(func(data []byte) {
+		if data[0] == '1' {
+			atomic.AddUint64(&reqCounter, 1)
+		} else {
+			atomic.AddUint64(&respCounter, 1)
+		}
+		atomic.AddUint64(&capturedBody, uint64(len(data)))
+		wg.Done()
+	})
+	httpOutput := NewHTTPOutput(replayAddr, &HTTPOutputConfig{Debug: false})
 
-	go Start(quit)
+	plugins := &InOutPlugins{
+		Inputs:  []io.Reader{input},
+		Outputs: []io.Writer{testOutput, httpOutput},
+	}
 
-	wg.Add(2)
-	curl := exec.Command("curl", "http://"+originAddr, "--header", "Transfer-Encoding: chunked", "--header", "Expect:", "--data-binary", "@/tmp/large")
-	err = curl.Run()
-	if err != nil {
-		log.Fatal("curl error:", err)
+	emitter := NewEmitter(quit)
+	go emitter.Start(plugins, Settings.Middleware)
+	now := time.Now()
+	var buf [1 << 20]byte
+	buf[1<<20-1] = 'a'
+	client := NewHTTPClient(originAddr, &HTTPClientConfig{ResponseBufferSize: 2 << 20, CompatibilityMode: true})
+	for i := 0; i < b.N; i++ {
+		if i&1 == 0 {
+			_, err = client.Get("/")
+		} else {
+			_, err = client.Post("/", buf[:])
+		}
+		if err != nil {
+			b.Log(err)
+			wg.Add(-3)
+		}
 	}
 
 	wg.Wait()
-	close(quit)
-}
-
-func BenchmarkRAWInput(b *testing.B) {
-	quit := make(chan int)
-
-	origin := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {}))
-	defer origin.Close()
-	originAddr := strings.Replace(origin.Listener.Addr().String(), "[::]", "127.0.0.1", -1)
-
-	var respCounter, reqCounter int64
-
-	input := NewRAWInput(originAddr, EnginePcap, true, testRawExpire, "", "", "", 0)
-	defer input.Close()
-
-	output := NewTestOutput(func(data []byte) {
-		if data[0] == '1' {
-			atomic.AddInt64(&reqCounter, 1)
-		} else {
-			atomic.AddInt64(&respCounter, 1)
-		}
-
-		// log.Println("Captured ", reqCounter, "requests and ", respCounter, " responses")
-	})
-
-	Plugins.Inputs = []io.Reader{input}
-	Plugins.Outputs = []io.Writer{output}
-
-	go Start(quit)
-
-	emitted := 0
-	fileContent, _ := ioutil.ReadFile("LICENSE.txt")
-
-	for i := 0; i < b.N; i++ {
-		wg := new(sync.WaitGroup)
-		wg.Add(10 * 100)
-		emitted += 10 * 100
-		for w := 0; w < 100; w++ {
-			go func() {
-				client := NewHTTPClient(origin.URL, &HTTPClientConfig{})
-				for i := 0; i < 10; i++ {
-					if rand.Int63n(2) == 0 {
-						client.Post("/", fileContent)
-					} else {
-						client.Get("/")
-					}
-					time.Sleep(time.Duration(rand.Int63n(50)) * time.Millisecond)
-					wg.Done()
-				}
-			}()
-		}
-		wg.Wait()
-	}
-
-	time.Sleep(400 * time.Millisecond)
-	log.Println("Emitted ", emitted, ", Captured ", reqCounter, "requests and ", respCounter, " responses")
-
-	close(quit)
+	b.Logf("%d/%d Requests, %d/%d Responses, %d/%d Replayed, %d Bytes in %s\n", reqCounter, b.N, respCounter, b.N, replayCounter, b.N, capturedBody, time.Since(now))
+	emitter.Close()
 }

input_tcp.go

@@ -17,12 +17,13 @@ type TCPInput struct {
 	listener net.Listener
 	address  string
 	config   *TCPInputConfig
+	stop     chan bool // Channel used only to indicate goroutine should shutdown
 }
 
 type TCPInputConfig struct {
-	secure          bool
-	certificatePath string
-	keyPath         string
+	Secure          bool   `json:"input-tcp-secure"`
+	CertificatePath string `json:"input-tcp-certificate"`
+	KeyPath         string `json:"input-tcp-certificate-key"`
 }
 
 // NewTCPInput constructor for TCPInput, accepts address with port
@@ -31,6 +32,7 @@ func NewTCPInput(address string, config *TCPInputConfig) (i *TCPInput) {
 	i.data = make(chan []byte, 1000)
 	i.address = address
 	i.config = config
+	i.stop = make(chan bool)
 
 	i.listen(address)
 
@@ -38,15 +40,25 @@ func NewTCPInput(address string, config *TCPInputConfig) (i *TCPInput) {
 }
 
 func (i *TCPInput) Read(data []byte) (int, error) {
-	buf := <-i.data
+	var buf []byte
+	select {
+	case <-i.stop:
+		return 0, ErrorStopped
+	case buf = <-i.data:
+	}
 	copy(data, buf)
 
 	return len(buf), nil
 }
 
+func (i *TCPInput) Close() error {
+	close(i.stop)
+	return nil
+}
+
 func (i *TCPInput) listen(address string) {
-	if i.config.secure {
-		cer, err := tls.LoadX509KeyPair(i.config.certificatePath, i.config.keyPath)
+	if i.config.Secure {
+		cer, err := tls.LoadX509KeyPair(i.config.CertificatePath, i.config.KeyPath)
 		if err != nil {
 			log.Fatal("Error while loading --input-file certificate:", err)
 		}

input_tcp_test.go

@@ -27,10 +27,14 @@ func TestTCPInput(t *testing.T) {
 		wg.Done()
 	})
 
-	Plugins.Inputs = []io.Reader{input}
-	Plugins.Outputs = []io.Writer{output}
+	plugins := &InOutPlugins{
+		Inputs:  []io.Reader{input},
+		Outputs: []io.Writer{output},
+	}
+	plugins.All = append(plugins.All, input, output)
 
-	go Start(quit)
+	emitter := NewEmitter(quit)
+	go emitter.Start(plugins, Settings.Middleware)
 
 	tcpAddr, err := net.ResolveTCPAddr("tcp", input.listener.Addr().String())
 
@@ -53,8 +57,7 @@ func TestTCPInput(t *testing.T) {
 	}
 
 	wg.Wait()
-
-	close(quit)
+	emitter.Close()
 }
 
 func genCertificate(template *x509.Certificate) ([]byte, []byte) {
@@ -99,18 +102,22 @@ func TestTCPInputSecure(t *testing.T) {
 	quit := make(chan int)
 
 	input := NewTCPInput("127.0.0.1:0", &TCPInputConfig{
-		secure:          true,
-		certificatePath: serverCertPemFile.Name(),
-		keyPath:         serverPrivPemFile.Name(),
+		Secure:          true,
+		CertificatePath: serverCertPemFile.Name(),
+		KeyPath:         serverPrivPemFile.Name(),
 	})
 	output := NewTestOutput(func(data []byte) {
 		wg.Done()
 	})
 
-	Plugins.Inputs = []io.Reader{input}
-	Plugins.Outputs = []io.Writer{output}
+	plugins := &InOutPlugins{
+		Inputs:  []io.Reader{input},
+		Outputs: []io.Writer{output},
+	}
+	plugins.All = append(plugins.All, input, output)
 
-	go Start(quit)
+	emitter := NewEmitter(quit)
+	go emitter.Start(plugins, Settings.Middleware)
 
 	conf := &tls.Config{
 		InsecureSkipVerify: true,
@@ -131,6 +138,5 @@ func TestTCPInputSecure(t *testing.T) {
 	}
 
 	wg.Wait()
-
-	close(quit)
+	emitter.Close()
 }

kafka.go

@@ -2,21 +2,40 @@ package main
 
 import (
 	"bytes"
+	"crypto/tls"
+	"crypto/x509"
 	"fmt"
-
-	"github.com/buger/goreplay/proto"
+	"io/ioutil"
+	"log"
 
 	"github.com/Shopify/sarama"
+	"github.com/buger/goreplay/proto"
 )
 
 // KafkaConfig should contains required information to
 // build producers.
-type KafkaConfig struct {
-	host     string
-	topic    string
+
+type InputKafkaConfig struct {
 	producer sarama.AsyncProducer
 	consumer sarama.Consumer
-	useJSON  bool
+	Host     string `json:"input-kafka-host"`
+	Topic    string `json:"input-kafka-topic"`
+	UseJSON  bool   `json:"input-kafka-json-format"`
+}
+
+type OutputKafkaConfig struct {
+	producer sarama.AsyncProducer
+	consumer sarama.Consumer
+	Host     string `json:"output-kafka-host"`
+	Topic    string `json:"output-kafka-topic"`
+	UseJSON  bool   `json:"output-kafka-json-format"`
+}
+
+// KafkaTLSConfig should contains TLS certificates for connecting to secured Kafka clusters
+type KafkaTLSConfig struct {
+	CACert     string `json:"kafka-tls-ca-cert"`
+	clientCert string `json:"kafka-tls-client-cert"`
+	clientKey  string `json:"kafka-tls-client-key"`
 }
 
 // KafkaMessage should contains catched request information that should be
@@ -31,6 +50,44 @@ type KafkaMessage struct {
 	ReqHeaders map[string]string `json:"Req_Headers,omitempty"`
 }
 
+// NewTLSConfig loads TLS certificates
+func NewTLSConfig(clientCertFile, clientKeyFile, caCertFile string) (*tls.Config, error) {
+	tlsConfig := tls.Config{}
+
+	// Load client cert
+	cert, err := tls.LoadX509KeyPair(clientCertFile, clientKeyFile)
+	if err != nil {
+		return &tlsConfig, err
+	}
+	tlsConfig.Certificates = []tls.Certificate{cert}
+
+	// Load CA cert
+	caCert, err := ioutil.ReadFile(caCertFile)
+	if err != nil {
+		return &tlsConfig, err
+	}
+	caCertPool := x509.NewCertPool()
+	caCertPool.AppendCertsFromPEM(caCert)
+	tlsConfig.RootCAs = caCertPool
+
+	return &tlsConfig, err
+}
+
+// NewKafkaConfig returns Kafka config with or without TLS
+func NewKafkaConfig(tlsConfig *KafkaTLSConfig) *sarama.Config {
+	config := sarama.NewConfig()
+	// Configuration options go here
+	if (tlsConfig != nil) && (tlsConfig.CACert != "") && (tlsConfig.clientCert != "") && (tlsConfig.clientKey != "") {
+		config.Net.TLS.Enable = true
+		tlsConfig, err := NewTLSConfig(tlsConfig.clientCert, tlsConfig.clientKey, tlsConfig.CACert)
+		if err != nil {
+			log.Fatal(err)
+		}
+		config.Net.TLS.Config = tlsConfig
+	}
+	return config
+}
+
 // Dump returns the given request in its HTTP/1.x wire
 // representation.
 func (m KafkaMessage) Dump() ([]byte, error) {
@@ -38,13 +95,13 @@ func (m KafkaMessage) Dump() ([]byte, error) {
 
 	b.WriteString(fmt.Sprintf("%s %s %s\n", m.ReqType, m.ReqID, m.ReqTs))
 	b.WriteString(fmt.Sprintf("%s %s HTTP/1.1", m.ReqMethod, m.ReqURL))
-	b.Write(proto.CLRF)
+	b.Write(proto.CRLF)
 	for key, value := range m.ReqHeaders {
 		b.WriteString(fmt.Sprintf("%s: %s", key, value))
-		b.Write(proto.CLRF)
+		b.Write(proto.CRLF)
 	}
 
-	b.Write(proto.CLRF)
+	b.Write(proto.CRLF)
 	b.WriteString(m.ReqBody)
 
 	return b.Bytes(), nil

limiter.go

@@ -77,7 +77,6 @@ func (l *Limiter) Write(data []byte) (n int, err error) {
 	}
 
 	n, err = l.plugin.(io.Writer).Write(data)
-
 	return
 }
 
@@ -98,3 +97,11 @@ func (l *Limiter) Read(data []byte) (n int, err error) {
 func (l *Limiter) String() string {
 	return fmt.Sprintf("Limiting %s to: %d (isPercent: %v)", l.plugin, l.limit, l.isPercent)
 }
+
+// Close closes the resources.
+func (l *Limiter) Close() error {
+	if fi, ok := l.plugin.(io.ReadCloser); ok {
+		fi.Close()
+	}
+	return nil
+}

limiter_test.go

@@ -18,18 +18,21 @@ func TestOutputLimiter(t *testing.T) {
 	}), "10")
 	wg.Add(10)
 
-	Plugins.Inputs = []io.Reader{input}
-	Plugins.Outputs = []io.Writer{output}
+	plugins := &InOutPlugins{
+		Inputs:  []io.Reader{input},
+		Outputs: []io.Writer{output},
+	}
+	plugins.All = append(plugins.All, input, output)
 
-	go Start(quit)
+	emitter := NewEmitter(quit)
+	go emitter.Start(plugins, Settings.Middleware)
 
 	for i := 0; i < 100; i++ {
 		input.EmitGET()
 	}
 
 	wg.Wait()
-
-	close(quit)
+	emitter.Close()
 }
 
 func TestInputLimiter(t *testing.T) {
@@ -42,18 +45,21 @@ func TestInputLimiter(t *testing.T) {
 	})
 	wg.Add(10)
 
-	Plugins.Inputs = []io.Reader{input}
-	Plugins.Outputs = []io.Writer{output}
+	plugins := &InOutPlugins{
+		Inputs:  []io.Reader{input},
+		Outputs: []io.Writer{output},
+	}
+	plugins.All = append(plugins.All, input, output)
 
-	go Start(quit)
+	emitter := NewEmitter(quit)
+	go emitter.Start(plugins, Settings.Middleware)
 
 	for i := 0; i < 100; i++ {
 		input.(*Limiter).plugin.(*TestInput).EmitGET()
 	}
 
 	wg.Wait()
-
-	close(quit)
+	emitter.Close()
 }
 
 // Should limit all requests
@@ -66,18 +72,20 @@ func TestPercentLimiter1(t *testing.T) {
 		wg.Done()
 	}), "0%")
 
-	Plugins.Inputs = []io.Reader{input}
-	Plugins.Outputs = []io.Writer{output}
+	plugins := &InOutPlugins{
+		Inputs:  []io.Reader{input},
+		Outputs: []io.Writer{output},
+	}
+	plugins.All = append(plugins.All, input, output)
 
-	go Start(quit)
+	emitter := NewEmitter(quit)
+	go emitter.Start(plugins, Settings.Middleware)
 
 	for i := 0; i < 100; i++ {
 		input.EmitGET()
 	}
 
 	wg.Wait()
-
-	close(quit)
 }
 
 // Should not limit at all
@@ -91,16 +99,18 @@ func TestPercentLimiter2(t *testing.T) {
 	}), "100%")
 	wg.Add(100)
 
-	Plugins.Inputs = []io.Reader{input}
-	Plugins.Outputs = []io.Writer{output}
+	plugins := &InOutPlugins{
+		Inputs:  []io.Reader{input},
+		Outputs: []io.Writer{output},
+	}
+	plugins.All = append(plugins.All, input, output)
 
-	go Start(quit)
+	emitter := NewEmitter(quit)
+	go emitter.Start(plugins, Settings.Middleware)
 
 	for i := 0; i < 100; i++ {
 		input.EmitGET()
 	}
 
 	wg.Wait()
-
-	close(quit)
 }

middleware.go

@@ -12,6 +12,7 @@ import (
 	"sync"
 )
 
+// Middleware represents a middleware object
 type Middleware struct {
 	command string
 
@@ -21,12 +22,16 @@ type Middleware struct {
 
 	Stdin  io.Writer
 	Stdout io.Reader
+
+	stop chan bool // Channel used only to indicate goroutine should shutdown
 }
 
+// NewMiddleware returns new middleware
 func NewMiddleware(command string) *Middleware {
 	m := new(Middleware)
 	m.command = command
 	m.data = make(chan []byte, 1000)
+	m.stop = make(chan bool)
 
 	commands := strings.Split(command, " ")
 	cmd := exec.Command(commands[0], commands[1:]...)
@@ -55,8 +60,9 @@ func NewMiddleware(command string) *Middleware {
 	return m
 }
 
+// ReadFrom start a worker to read from this plugin
 func (m *Middleware) ReadFrom(plugin io.Reader) {
-	Debug("[MIDDLEWARE-MASTER] Starting reading from", plugin)
+	Debug(2, "[MIDDLEWARE-MASTER] Starting reading from", plugin)
 	go m.copy(m.Stdin, plugin)
 }
 
@@ -72,7 +78,7 @@ func (m *Middleware) copy(to io.Writer, from io.Reader) {
 
 		payload := buf[0:nr]
 
-		if Settings.prettifyHTTP {
+		if Settings.PrettifyHTTP {
 			payload = prettifyHTTP(payload)
 			nr = len(payload)
 
@@ -81,7 +87,7 @@ func (m *Middleware) copy(to io.Writer, from io.Reader) {
 			}
 		}
 
-		if Settings.prettifyHTTP {
+		if Settings.PrettifyHTTP {
 			payload = prettifyHTTP(payload)
 			nr = len(payload)
 		}
@@ -93,9 +99,7 @@ func (m *Middleware) copy(to io.Writer, from io.Reader) {
 		to.Write(dst[0 : nr*2+1])
 		m.mu.Unlock()
 
-		if Settings.debug {
-			Debug("[MIDDLEWARE-MASTER] Sending:", string(buf[0:nr]), "From:", from)
-		}
+		Debug(3, "[MIDDLEWARE-MASTER] Sending:", string(buf[0:nr]), "From:", from)
 	}
 }
 
@@ -115,26 +119,39 @@ func (m *Middleware) read(from io.Reader) {
 
 		buf := make([]byte, len(line)/2)
 		if _, err := hex.Decode(buf, line[:len(line)-1]); err != nil {
-			fmt.Fprintln(os.Stderr, "Failed to decode input payload", err, len(line))
+			fmt.Fprintln(os.Stderr, "Failed to decode input payload", err, len(line), string(line[:len(line)-1]))
 		}
 
-		if Settings.debug {
-			Debug("[MIDDLEWARE-MASTER] Received:", string(buf))
-		}
+		Debug(3, "[MIDDLEWARE-MASTER] Received:", string(buf))
 
-		m.data <- buf
+		select {
+		case <-m.stop:
+			return
+		case m.data <- buf:
+		}
 	}
 
 	return
 }
 
 func (m *Middleware) Read(data []byte) (int, error) {
-	buf := <-m.data
-	copy(data, buf)
+	var buf []byte
+	select {
+	case <-m.stop:
+		return 0, ErrorStopped
+	case buf = <-m.data:
+	}
 
-	return len(buf), nil
+	n := copy(data, buf)
+	return n, nil
 }
 
 func (m *Middleware) String() string {
 	return fmt.Sprintf("Modifying traffic using '%s' command", m.command)
 }
+
+// Close closes this plugin
+func (m *Middleware) Close() error {
+	close(m.stop)
+	return nil
+}

middleware/README.md

@@ -4,7 +4,7 @@ GoReplay support protocol for writing middleware in any language, which allows y
 
 To simplify middleware creation we provide packages for NodeJS and Go (upcoming).
 
-If you want to get access to original and replayed responses, do not forget adding `--output-http-track-respose` and `--input-raw-track-response` options.
+If you want to get access to original and replayed responses, do not forget adding `--output-http-track-response` and `--input-raw-track-response` options.
 
 ## NodeJS
 

middleware/middleware.js

@@ -335,7 +335,12 @@ function deleteHttpHeader(payload, name) {
 }
 
 function httpBody(payload) {
-    return payload.slice(payload.indexOf("\r\n\r\n") + 4, payload.length);
+    let bodyIndex = payload.indexOf("\r\n\r\n");
+    if (-1 != bodyIndex){
+        return payload.slice(bodyIndex + 4, payload.length);   
+    } else {
+        return null;
+    }
 }
 
 function setHttpBody(payload, newBody) {
@@ -710,6 +715,12 @@ function TEST_httpBody() {
     if (body != "hello") {
         fail(`'${body}' != 'hello'`)
     }
+
+    const exampleInvalidPayload = "Invalid HTTP Response by Network issue";
+    let invalidBody = httpBody(Buffer.from(exampleInvalidPayload));
+    if (invalidBody != null) {
+        fail(`'${invalidBody}' != 'null'`)
+    }
 }
 
 function TEST_setHttpBody() {

middleware_test.go

@@ -1,221 +1,225 @@
 package main
 
-import (
-	"bytes"
-	"crypto/rand"
-	"encoding/hex"
-	"io"
-	"net/http"
-	"net/http/httptest"
-	"strings"
-	"sync"
-	"testing"
-	"time"
+// import (
+// 	"bytes"
+// 	"crypto/rand"
+// 	"encoding/hex"
+// 	"io"
+// 	"net/http"
+// 	"net/http/httptest"
+// 	"strings"
+// 	"sync"
+// 	"testing"
+// 	"time"
 
-	"github.com/buger/goreplay/proto"
-)
+// 	"github.com/buger/goreplay/capture"
+// 	"github.com/buger/goreplay/proto"
+// )
 
-type fakeServiceCb func(string, int, []byte)
+// type fakeServiceCb func(string, int, []byte)
 
-// Simple service that generate token on request, and require this token for accesing to secure area
-func NewFakeSecureService(wg *sync.WaitGroup, cb fakeServiceCb) *httptest.Server {
-	active_tokens := make([]string, 0)
-	var mu sync.Mutex
+// // Simple service that generate token on request, and require this token for accesing to secure area
+// func NewFakeSecureService(wg *sync.WaitGroup, cb fakeServiceCb) *httptest.Server {
+// 	active_tokens := make([]string, 0)
+// 	var mu sync.Mutex
 
-	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
-		mu.Lock()
-		defer mu.Unlock()
-		Debug("Received request: " + req.URL.String())
+// 	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
+// 		mu.Lock()
+// 		defer mu.Unlock()
 
-		switch req.URL.Path {
-		case "/token":
-			// Generate random token
-			token_length := 10
-			buf := make([]byte, token_length)
-			rand.Read(buf)
-			token := hex.EncodeToString(buf)
-			active_tokens = append(active_tokens, token)
+// 		switch req.URL.Path {
+// 		case "/token":
+// 			// Generate random token
+// 			token_length := 10
+// 			buf := make([]byte, token_length)
+// 			rand.Read(buf)
+// 			token := hex.EncodeToString(buf)
+// 			active_tokens = append(active_tokens, token)
 
-			w.Write([]byte(token))
+// 			w.Write([]byte(token))
 
-			cb(req.URL.Path, 200, []byte(token))
-		case "/secure":
-			token := req.URL.Query().Get("token")
-			token_found := false
+// 			cb(req.URL.Path, 200, []byte(token))
+// 		case "/secure":
+// 			token := req.URL.Query().Get("token")
+// 			token_found := false
 
-			for _, t := range active_tokens {
-				if t == token {
-					token_found = true
-					break
-				}
-			}
+// 			for _, t := range active_tokens {
+// 				if t == token {
+// 					token_found = true
+// 					break
+// 				}
+// 			}
 
-			if token_found {
-				w.WriteHeader(http.StatusAccepted)
-				cb(req.URL.Path, 202, []byte(nil))
-			} else {
-				w.WriteHeader(http.StatusForbidden)
-				cb(req.URL.Path, 403, []byte(nil))
-			}
-		}
+// 			if token_found {
+// 				w.WriteHeader(http.StatusAccepted)
+// 				cb(req.URL.Path, 202, []byte(nil))
+// 			} else {
+// 				w.WriteHeader(http.StatusForbidden)
+// 				cb(req.URL.Path, 403, []byte(nil))
+// 			}
+// 		}
 
-		wg.Done()
-	}))
+// 		wg.Done()
+// 	}))
 
-	return server
-}
+// 	return server
+// }
 
-func TestFakeSecureService(t *testing.T) {
-	var resp, token []byte
+// func TestFakeSecureService(t *testing.T) {
+// 	var resp, token []byte
 
-	wg := new(sync.WaitGroup)
+// 	wg := new(sync.WaitGroup)
 
-	server := NewFakeSecureService(wg, func(path string, status int, resp []byte) {
-	})
-	defer server.Close()
+// 	server := NewFakeSecureService(wg, func(path string, status int, resp []byte) {
+// 	})
+// 	defer server.Close()
 
-	wg.Add(3)
+// 	wg.Add(3)
 
-	client := NewHTTPClient(server.URL, &HTTPClientConfig{Debug: true})
-	resp, _ = client.Get("/token")
-	token = proto.Body(resp)
+// 	client := NewHTTPClient(server.URL, &HTTPClientConfig{Debug: true})
+// 	resp, _ = client.Get("/token")
+// 	token = proto.Body(resp)
 
-	// Right token
-	resp, _ = client.Get("/secure?token=" + string(token))
-	if !bytes.Equal(proto.Status(resp), []byte("202")) {
-		t.Error("Valid token should return status 202:", string(proto.Status(resp)))
-	}
+// 	// Right token
+// 	resp, _ = client.Get("/secure?token=" + string(token))
+// 	if !bytes.Equal(proto.Status(resp), []byte("202")) {
+// 		t.Error("Valid token should return status 202:", string(proto.Status(resp)))
+// 	}
 
-	// Wrong tokens forbidden
-	resp, _ = client.Get("/secure?token=wrong")
-	if !bytes.Equal(proto.Status(resp), []byte("403")) {
-		t.Error("Wrong token should returns status 403:", string(proto.Status(resp)))
-	}
+// 	// Wrong tokens forbidden
+// 	resp, _ = client.Get("/secure?token=wrong")
+// 	if !bytes.Equal(proto.Status(resp), []byte("403")) {
+// 		t.Error("Wrong token should returns status 403:", string(proto.Status(resp)))
+// 	}
 
-	wg.Wait()
-}
+// 	wg.Wait()
+// }
 
-func TestEchoMiddleware(t *testing.T) {
-	wg := new(sync.WaitGroup)
+// func TestEchoMiddleware(t *testing.T) {
+// 	wg := new(sync.WaitGroup)
 
-	from := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		w.Header().Set("Env", "prod")
-		w.Header().Set("RequestPath", r.URL.Path)
-		wg.Done()
-	}))
-	defer from.Close()
+// 	from := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+// 		w.Header().Set("Env", "prod")
+// 		w.Header().Set("RequestPath", r.URL.Path)
+// 		wg.Done()
+// 	}))
+// 	defer from.Close()
 
-	to := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		w.Header().Set("Env", "test")
-		w.Header().Set("RequestPath", r.URL.Path)
-		wg.Done()
-	}))
-	defer to.Close()
+// 	to := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+// 		w.Header().Set("Env", "test")
+// 		w.Header().Set("RequestPath", r.URL.Path)
+// 		wg.Done()
+// 	}))
+// 	defer to.Close()
 
-	quit := make(chan int)
+// 	quit := make(chan int)
 
-	Settings.middleware = "./examples/middleware/echo.sh"
+// 	// Catch traffic from one service
+// 	fromAddr := strings.Replace(from.Listener.Addr().String(), "[::]", "127.0.0.1", -1)
+// 	conf := RAWInputConfig{
+// 		engine:        capture.EnginePcap,
+// 		expire:        testRawExpire,
+// 		protocol:      ProtocolHTTP,
+// 		trackResponse: true,
+// 	}
+// 	input := NewRAWInput(fromAddr, conf)
 
-	// Catch traffic from one service
-	fromAddr := strings.Replace(from.Listener.Addr().String(), "[::]", "127.0.0.1", -1)
-	input := NewRAWInput(fromAddr, EnginePcap, true, testRawExpire, "", "", "", 0)
-	defer input.Close()
+// 	// And redirect to another
+// 	output := NewHTTPOutput(to.URL, &HTTPOutputConfig{Debug: false})
 
-	// And redirect to another
-	output := NewHTTPOutput(to.URL, &HTTPOutputConfig{Debug: false})
+// 	plugins := &InOutPlugins{
+// 		Inputs:  []io.Reader{input},
+// 		Outputs: []io.Writer{output},
+// 	}
+// 	plugins.All = append(plugins.All, input, output)
 
-	Plugins.Inputs = []io.Reader{input}
-	Plugins.Outputs = []io.Writer{output}
+// 	// Start Gor
+// 	emitter := NewEmitter(quit)
+// 	go emitter.Start(plugins, "echo -n && GOR_TEST=true && ./examples/middleware/echo.sh")
 
-	// Start Gor
-	go Start(quit)
+// 	// Wait till middleware initialization
+// 	time.Sleep(100 * time.Millisecond)
 
-	// Wait till middleware initialization
-	time.Sleep(100 * time.Millisecond)
+// 	// Should receive 2 requests from original + 2 from replayed
+// 	client := NewHTTPClient(from.URL, &HTTPClientConfig{Debug: false})
 
-	// Should receive 2 requests from original + 2 from replayed
-	client := NewHTTPClient(from.URL, &HTTPClientConfig{Debug: false})
+// 	for i := 0; i < 10; i++ {
+// 		wg.Add(2)
+// 		// Request should be echoed
+// 		client.Get("/a")
+// 		time.Sleep(5 * time.Millisecond)
+// 		client.Get("/b")
+// 		time.Sleep(5 * time.Millisecond)
+// 	}
 
-	for i := 0; i < 10; i++ {
-		wg.Add(4)
-		// Request should be echoed
-		client.Get("/a")
-		time.Sleep(5 * time.Millisecond)
-		client.Get("/b")
-		time.Sleep(5 * time.Millisecond)
-	}
+// 	wg.Wait()
+// 	emitter.Close()
+// }
 
-	wg.Wait()
-	close(quit)
-	time.Sleep(200 * time.Millisecond)
+// func TestTokenMiddleware(t *testing.T) {
+// 	var resp, token []byte
 
-	Settings.middleware = ""
-}
+// 	wg := new(sync.WaitGroup)
 
-func TestTokenMiddleware(t *testing.T) {
-	var resp, token []byte
+// 	from := NewFakeSecureService(wg, func(path string, status int, tok []byte) {
+// 		time.Sleep(10 * time.Millisecond)
+// 	})
+// 	defer from.Close()
 
-	wg := new(sync.WaitGroup)
+// 	to := NewFakeSecureService(wg, func(path string, status int, tok []byte) {
+// 		switch path {
+// 		case "/secure":
+// 			if status != 202 {
+// 				t.Error("Server should receive valid rewritten token")
+// 			}
+// 		}
 
-	from := NewFakeSecureService(wg, func(path string, status int, tok []byte) {
-		time.Sleep(10 * time.Millisecond)
-	})
-	defer from.Close()
+// 		time.Sleep(10 * time.Millisecond)
+// 	})
+// 	defer to.Close()
 
-	to := NewFakeSecureService(wg, func(path string, status int, tok []byte) {
-		switch path {
-		case "/secure":
-			if status != 202 {
-				t.Error("Server should receive valid rewritten token")
-			}
-		}
+// 	quit := make(chan int)
 
-		time.Sleep(10 * time.Millisecond)
-	})
-	defer to.Close()
+// 	Settings.middleware = "echo -n && GOR_TEST=true && go run ./examples/middleware/token_modifier.go"
 
-	quit := make(chan int)
+// 	fromAddr := strings.Replace(from.Listener.Addr().String(), "[::]", "127.0.0.1", -1)
+// 	conf := RAWInputConfig{
+// 		engine:        capture.EnginePcap,
+// 		expire:        testRawExpire,
+// 		protocol:      ProtocolHTTP,
+// 		trackResponse: true,
+// 	}
+// 	// Catch traffic from one service
+// 	input := NewRAWInput(fromAddr, conf)
 
-	Settings.middleware = "go run ./examples/middleware/token_modifier.go"
+// 	// And redirect to another
+// 	output := NewHTTPOutput(to.URL, &HTTPOutputConfig{Debug: true})
 
-	fromAddr := strings.Replace(from.Listener.Addr().String(), "[::]", "127.0.0.1", -1)
-	// Catch traffic from one service
-	input := NewRAWInput(fromAddr, EnginePcap, true, testRawExpire, "", "", "", 0)
-	defer input.Close()
+// 	plugins := &InOutPlugins{
+// 		Inputs:  []io.Reader{input},
+// 		Outputs: []io.Writer{output},
+// 	}
+// 	plugins.All = append(plugins.All, input, output)
 
-	// And redirect to another
-	output := NewHTTPOutput(to.URL, &HTTPOutputConfig{Debug: true})
+// 	// Start Gor
+// 	emitter := NewEmitter(quit)
+// 	go emitter.Start(plugins, Settings.middleware)
 
-	Plugins.Inputs = []io.Reader{input}
-	Plugins.Outputs = []io.Writer{output}
+// 	// Should receive 2 requests from original + 2 from replayed
+// 	wg.Add(2)
 
-	// Start Gor
-	go Start(quit)
+// 	client := NewHTTPClient(from.URL, &HTTPClientConfig{Debug: true})
 
-	// Wait for middleware to initialize
-	// Give go compiller time to build programm
-	time.Sleep(500 * time.Millisecond)
+// 	// Sending traffic to original service
+// 	resp, _ = client.Get("/token")
+// 	token = proto.Body(resp)
 
-	// Should receive 2 requests from original + 2 from replayed
-	wg.Add(4)
+// 	resp, _ = client.Get("/secure?token=" + string(token))
+// 	if !bytes.Equal(proto.Status(resp), []byte("202")) {
+// 		t.Error("Valid token should return 202:", proto.Status(resp))
+// 	}
 
-	client := NewHTTPClient(from.URL, &HTTPClientConfig{Debug: true})
-
-	// Sending traffic to original service
-	resp, _ = client.Get("/token")
-	token = proto.Body(resp)
-
-	// When delay is too smal, middleware does not always rewrite requests in time
-	// Hopefuly client will have delay more then 100ms :)
-	time.Sleep(100 * time.Millisecond)
-
-	resp, _ = client.Get("/secure?token=" + string(token))
-	if !bytes.Equal(proto.Status(resp), []byte("202")) {
-		t.Error("Valid token should return 202:", proto.Status(resp))
-	}
-
-	wg.Wait()
-	close(quit)
-	time.Sleep(100 * time.Millisecond)
-	Settings.middleware = ""
-}
+// 	wg.Wait()
+// 	emitter.Close()
+// 	Settings.middleware = ""
+// }

output_binary.go

@@ -0,0 +1,178 @@
+package main
+
+import (
+	"io"
+	"sync/atomic"
+	"time"
+
+	"github.com/buger/goreplay/size"
+)
+
+// BinaryOutputConfig struct for holding binary output configuration
+type BinaryOutputConfig struct {
+	Workers        int           `json:"output-binary-workers"`
+	Timeout        time.Duration `json:"output-binary-timeout"`
+	BufferSize     size.Size     `json:"output-tcp-response-buffer"`
+	Debug          bool          `json:"output-binary-debug"`
+	TrackResponses bool          `json:"output-binary-track-response"`
+}
+
+// BinaryOutput plugin manage pool of workers which send request to replayed server
+// By default workers pool is dynamic and starts with 10 workers
+// You can specify fixed number of workers using `--output-tcp-workers`
+type BinaryOutput struct {
+	// Keep this as first element of struct because it guarantees 64bit
+	// alignment. atomic.* functions crash on 32bit machines if operand is not
+	// aligned at 64bit. See https://github.com/golang/go/issues/599
+	activeWorkers int64
+
+	address string
+	queue   chan []byte
+
+	responses chan response
+
+	needWorker chan int
+
+	config *BinaryOutputConfig
+
+	queueStats *GorStat
+}
+
+// NewBinaryOutput constructor for BinaryOutput
+// Initialize workers
+func NewBinaryOutput(address string, config *BinaryOutputConfig) io.Writer {
+	o := new(BinaryOutput)
+
+	o.address = address
+	o.config = config
+
+	o.queue = make(chan []byte, 1000)
+	o.responses = make(chan response, 1000)
+	o.needWorker = make(chan int, 1)
+
+	// Initial workers count
+	if o.config.Workers == 0 {
+		o.needWorker <- initialDynamicWorkers
+	} else {
+		o.needWorker <- o.config.Workers
+	}
+
+	go o.workerMaster()
+
+	return o
+}
+
+func (o *BinaryOutput) workerMaster() {
+	for {
+		newWorkers := <-o.needWorker
+		for i := 0; i < newWorkers; i++ {
+			go o.startWorker()
+		}
+
+		// Disable dynamic scaling if workers poll fixed size
+		if o.config.Workers != 0 {
+			return
+		}
+	}
+}
+
+func (o *BinaryOutput) startWorker() {
+	client := NewTCPClient(o.address, &TCPClientConfig{
+		Debug:              o.config.Debug,
+		Timeout:            o.config.Timeout,
+		ResponseBufferSize: int(o.config.BufferSize),
+	})
+
+	deathCount := 0
+
+	atomic.AddInt64(&o.activeWorkers, 1)
+
+	for {
+		select {
+		case data := <-o.queue:
+			o.sendRequest(client, data)
+			deathCount = 0
+		case <-time.After(time.Millisecond * 100):
+			// When dynamic scaling enabled workers die after 2s of inactivity
+			if o.config.Workers == 0 {
+				deathCount++
+			} else {
+				continue
+			}
+
+			if deathCount > 20 {
+				workersCount := atomic.LoadInt64(&o.activeWorkers)
+
+				// At least 1 startWorker should be alive
+				if workersCount != 1 {
+					atomic.AddInt64(&o.activeWorkers, -1)
+					return
+				}
+			}
+		}
+	}
+}
+
+func (o *BinaryOutput) Write(data []byte) (n int, err error) {
+	if !isRequestPayload(data) {
+		return len(data), nil
+	}
+
+	buf := make([]byte, len(data))
+	copy(buf, data)
+
+	o.queue <- buf
+
+	if o.config.Workers == 0 {
+		workersCount := atomic.LoadInt64(&o.activeWorkers)
+
+		if len(o.queue) > int(workersCount) {
+			o.needWorker <- len(o.queue)
+		}
+	}
+
+	return len(data), nil
+}
+
+func (o *BinaryOutput) Read(data []byte) (int, error) {
+	resp := <-o.responses
+
+	Debug(2, "[OUTPUT-TCP] Received response:", string(resp.payload))
+
+	header := payloadHeader(ReplayedResponsePayload, resp.uuid, resp.startedAt, resp.roundTripTime)
+	copy(data[0:len(header)], header)
+	copy(data[len(header):], resp.payload)
+
+	return len(resp.payload) + len(header), nil
+}
+
+func (o *BinaryOutput) sendRequest(client *TCPClient, request []byte) {
+	meta := payloadMeta(request)
+	if len(meta) < 2 {
+		return
+	}
+
+	if !isRequestPayload(request) {
+		return
+	}
+
+	uuid := meta[1]
+
+	body := payloadBody(request)
+
+	start := time.Now()
+	resp, err := client.Send(body)
+	stop := time.Now()
+
+	if err != nil {
+		Debug(1, "Request error:", err)
+	}
+
+	if o.config.TrackResponses {
+		o.responses <- response{resp, uuid, start.UnixNano(), stop.UnixNano() - start.UnixNano()}
+	}
+}
+
+func (o *BinaryOutput) String() string {
+	return "Binary output: " + o.address
+}

output_dummy.go

@@ -1,7 +1,7 @@
 package main
 
 import (
-	"fmt"
+	"os"
 )
 
 // DummyOutput used for debugging, prints all incoming requests
@@ -16,9 +16,9 @@ func NewDummyOutput() (di *DummyOutput) {
 }
 
 func (i *DummyOutput) Write(data []byte) (int, error) {
-	fmt.Println(string(data))
-
-	return len(data), nil
+	n, err := os.Stdout.Write(data)
+	os.Stdout.Write([]byte{'\n'})
+	return n, err
 }
 
 func (i *DummyOutput) String() string {

output_file.go

@@ -15,6 +15,8 @@ import (
 	"strings"
 	"sync"
 	"time"
+
+	"github.com/buger/goreplay/size"
 )
 
 var dateFileNameFuncs = map[string]func(*FileOutput) string{
@@ -29,28 +31,31 @@ var dateFileNameFuncs = map[string]func(*FileOutput) string{
 	"%t":  func(o *FileOutput) string { return string(o.payloadType) },
 }
 
+// FileOutputConfig ...
 type FileOutputConfig struct {
-	flushInterval     time.Duration
-	sizeLimit         unitSizeVar
-	outputFileMaxSize unitSizeVar
-	queueLimit        int
-	append            bool
+	FlushInterval     time.Duration `json:"output-file-flush-interval"`
+	SizeLimit         size.Size     `json:"output-file-size-limit"`
+	OutputFileMaxSize size.Size     `json:"output-file-max-size-limit"`
+	QueueLimit        int           `json:"output-file-queue-limit"`
+	Append            bool          `json:"output-file-append"`
+	BufferPath        string        `json:"output-file-buffer"`
+	onClose           func(string)
 }
 
 // FileOutput output plugin
 type FileOutput struct {
-	mu             sync.Mutex
+	sync.RWMutex
 	pathTemplate   string
 	currentName    string
 	file           *os.File
-	queueLength    int
+	QueueLength    int
 	chunkSize      int
 	writer         io.Writer
 	requestPerFile bool
 	currentID      []byte
 	payloadType    []byte
 	closed         bool
-	totalFileSize  int64
+	totalFileSize  size.Size
 
 	config *FileOutputConfig
 }
@@ -66,10 +71,14 @@ func NewFileOutput(pathTemplate string, config *FileOutputConfig) *FileOutput {
 		o.requestPerFile = true
 	}
 
+	if config.FlushInterval == 0 {
+		config.FlushInterval = 100 * time.Millisecond
+	}
+
 	go func() {
 		for {
-			time.Sleep(config.flushInterval)
-			if o.closed {
+			time.Sleep(config.FlushInterval)
+			if o.IsClosed() {
 				break
 			}
 			o.updateName()
@@ -134,8 +143,8 @@ func (s sortByFileIndex) Less(i, j int) bool {
 }
 
 func (o *FileOutput) filename() string {
-	defer o.mu.Unlock()
-	o.mu.Lock()
+	o.RLock()
+	defer o.RUnlock()
 
 	path := o.pathTemplate
 
@@ -143,12 +152,12 @@ func (o *FileOutput) filename() string {
 		path = strings.Replace(path, name, fn(o), -1)
 	}
 
-	if !o.config.append {
+	if !o.config.Append {
 		nextChunk := false
 
 		if o.currentName == "" ||
-			((o.config.queueLimit > 0 && o.queueLength >= o.config.queueLimit) ||
-				(o.config.sizeLimit > 0 && o.chunkSize >= int(o.config.sizeLimit))) {
+			((o.config.QueueLimit > 0 && o.QueueLength >= o.config.QueueLimit) ||
+				(o.config.SizeLimit > 0 && o.chunkSize >= int(o.config.SizeLimit))) {
 			nextChunk = true
 		}
 
@@ -180,21 +189,27 @@ func (o *FileOutput) filename() string {
 }
 
 func (o *FileOutput) updateName() {
-	o.currentName = filepath.Clean(o.filename())
+	name := filepath.Clean(o.filename())
+	o.Lock()
+	o.currentName = name
+	o.Unlock()
 }
 
 func (o *FileOutput) Write(data []byte) (n int, err error) {
 	if o.requestPerFile {
+		o.Lock()
 		meta := payloadMeta(data)
 		o.currentID = meta[1]
 		o.payloadType = meta[0]
+		o.Unlock()
 	}
 
 	o.updateName()
+	o.Lock()
+	defer o.Unlock()
 
 	if o.file == nil || o.currentName != o.file.Name() {
-		o.mu.Lock()
-		o.Close()
+		o.closeLocked()
 
 		o.file, err = os.OpenFile(o.currentName, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0660)
 		o.file.Sync()
@@ -209,21 +224,22 @@ func (o *FileOutput) Write(data []byte) (n int, err error) {
 			log.Fatal(o, "Cannot open file %q. Error: %s", o.currentName, err)
 		}
 
-		o.queueLength = 0
-		o.mu.Unlock()
+		o.QueueLength = 0
 	}
 
-	o.writer.Write(data)
-	o.writer.Write([]byte(payloadSeparator))
+	n, _ = o.writer.Write(data)
+	nSeparator, _ := o.writer.Write([]byte(payloadSeparator))
 
-	o.totalFileSize += int64(len(data) + len(payloadSeparator))
-	o.queueLength++
+	n += nSeparator
 
-	if Settings.outputFileConfig.outputFileMaxSize > 0 && o.totalFileSize >= int64(Settings.outputFileConfig.outputFileMaxSize) {
-		return len(data), errors.New("File output reached size limit")
+	o.totalFileSize += size.Size(n)
+	o.QueueLength++
+
+	if Settings.OutputFileConfig.OutputFileMaxSize > 0 && o.totalFileSize >= Settings.OutputFileConfig.OutputFileMaxSize {
+		return n, errors.New("File output reached size limit")
 	}
 
-	return len(data), nil
+	return n, nil
 }
 
 func (o *FileOutput) flush() {
@@ -234,8 +250,8 @@ func (o *FileOutput) flush() {
 		}
 	}()
 
-	defer o.mu.Unlock()
-	o.mu.Lock()
+	o.Lock()
+	defer o.Unlock()
 
 	if o.file != nil {
 		if strings.HasSuffix(o.currentName, ".gz") {
@@ -256,7 +272,7 @@ func (o *FileOutput) String() string {
 	return "File output: " + o.file.Name()
 }
 
-func (o *FileOutput) Close() error {
+func (o *FileOutput) closeLocked() error {
 	if o.file != nil {
 		if strings.HasSuffix(o.currentName, ".gz") {
 			o.writer.(*gzip.Writer).Close()
@@ -264,8 +280,26 @@ func (o *FileOutput) Close() error {
 			o.writer.(*bufio.Writer).Flush()
 		}
 		o.file.Close()
+
+		if o.config.onClose != nil {
+			o.config.onClose(o.file.Name())
+		}
 	}
 
 	o.closed = true
 	return nil
 }
+
+// Close closes the output file that is being written to.
+func (o *FileOutput) Close() error {
+	o.Lock()
+	defer o.Unlock()
+	return o.closeLocked()
+}
+
+// IsClosed returns if the output file is closed or not.
+func (o *FileOutput) IsClosed() bool {
+	o.Lock()
+	defer o.Unlock()
+	return o.closed
+}

output_file_settings.go

@@ -1,40 +0,0 @@
-package main
-
-import (
-	"strconv"
-	"strings"
-)
-
-var dataUnitMap = map[byte]int64{
-	'k': 1024,
-	'm': 1024 * 1024,
-	'g': 1024 * 1024 * 1024,
-}
-
-func parseDataUnit(s string) int64 {
-	// Allow kb, mb, gb
-	if strings.HasSuffix(s, "b") {
-		s = s[:len(s)-1]
-	}
-
-	if unit, ok := dataUnitMap[s[len(s)-1]]; ok {
-		size, _ := strconv.ParseInt(s[:len(s)-1], 10, 64)
-		return unit * size
-	}
-
-	// If no unit specified use bytes
-	size, _ := strconv.ParseInt(s, 10, 64)
-	return size
-
-}
-
-type unitSizeVar int64
-
-func (u unitSizeVar) String() string {
-	return strconv.Itoa(int(u))
-}
-
-func (u *unitSizeVar) Set(s string) error {
-	*u = unitSizeVar(parseDataUnit(s))
-	return nil
-}

output_file_test.go

@@ -11,6 +11,8 @@ import (
 	"sync/atomic"
 	"testing"
 	"time"
+
+	"github.com/buger/goreplay/size"
 )
 
 func TestFileOutput(t *testing.T) {
@@ -18,12 +20,16 @@ func TestFileOutput(t *testing.T) {
 	quit := make(chan int)
 
 	input := NewTestInput()
-	output := NewFileOutput("/tmp/test_requests.gor", &FileOutputConfig{flushInterval: time.Minute, append: true})
+	output := NewFileOutput("/tmp/test_requests.gor", &FileOutputConfig{FlushInterval: time.Minute, Append: true})
 
-	Plugins.Inputs = []io.Reader{input}
-	Plugins.Outputs = []io.Writer{output}
+	plugins := &InOutPlugins{
+		Inputs:  []io.Reader{input},
+		Outputs: []io.Writer{output},
+	}
+	plugins.All = append(plugins.All, input, output)
 
-	go Start(quit)
+	emitter := NewEmitter(quit)
+	go emitter.Start(plugins, Settings.Middleware)
 
 	for i := 0; i < 100; i++ {
 		wg.Add(2)
@@ -32,10 +38,7 @@ func TestFileOutput(t *testing.T) {
 	}
 	time.Sleep(100 * time.Millisecond)
 	output.flush()
-
-	close(quit)
-
-	quit = make(chan int)
+	emitter.Close()
 
 	var counter int64
 	input2 := NewFileInput("/tmp/test_requests.gor", false)
@@ -44,17 +47,22 @@ func TestFileOutput(t *testing.T) {
 		wg.Done()
 	})
 
-	Plugins.Inputs = []io.Reader{input2}
-	Plugins.Outputs = []io.Writer{output2}
+	plugins2 := &InOutPlugins{
+		Inputs:  []io.Reader{input2},
+		Outputs: []io.Writer{output2},
+	}
+	plugins2.All = append(plugins2.All, input2, output2)
 
-	go Start(quit)
+	quit2 := make(chan int)
+	emitter2 := NewEmitter(quit2)
+	go emitter2.Start(plugins2, Settings.Middleware)
 
 	wg.Wait()
-	close(quit)
+	emitter2.Close()
 }
 
 func TestFileOutputWithNameCleaning(t *testing.T) {
-	output := &FileOutput{pathTemplate: "./test_requests.gor", config: &FileOutputConfig{flushInterval: time.Minute, append: false}}
+	output := &FileOutput{pathTemplate: "./test_requests.gor", config: &FileOutputConfig{FlushInterval: time.Minute, Append: false}}
 	expectedFileName := "test_requests_0.gor"
 	output.updateName()
 
@@ -65,7 +73,7 @@ func TestFileOutputWithNameCleaning(t *testing.T) {
 }
 
 func TestFileOutputPathTemplate(t *testing.T) {
-	output := &FileOutput{pathTemplate: "/tmp/log-%Y-%m-%d-%S-%t", config: &FileOutputConfig{flushInterval: time.Minute, append: true}}
+	output := &FileOutput{pathTemplate: "/tmp/log-%Y-%m-%d-%S-%t", config: &FileOutputConfig{FlushInterval: time.Minute, Append: true}}
 	now := time.Now()
 	output.payloadType = []byte("3")
 	expectedPath := fmt.Sprintf("/tmp/log-%s-%s-%s-%s-3", now.Format("2006"), now.Format("01"), now.Format("02"), now.Format("05"))
@@ -77,7 +85,7 @@ func TestFileOutputPathTemplate(t *testing.T) {
 }
 
 func TestFileOutputMultipleFiles(t *testing.T) {
-	output := NewFileOutput("/tmp/log-%Y-%m-%d-%S", &FileOutputConfig{append: true, flushInterval: time.Minute})
+	output := NewFileOutput("/tmp/log-%Y-%m-%d-%S", &FileOutputConfig{Append: true, FlushInterval: time.Minute})
 
 	if output.file != nil {
 		t.Error("Should not initialize file if no writes")
@@ -108,7 +116,7 @@ func TestFileOutputMultipleFiles(t *testing.T) {
 }
 
 func TestFileOutputFilePerRequest(t *testing.T) {
-	output := NewFileOutput("/tmp/log-%Y-%m-%d-%S-%r", &FileOutputConfig{append: true})
+	output := NewFileOutput("/tmp/log-%Y-%m-%d-%S-%r", &FileOutputConfig{Append: true})
 
 	if output.file != nil {
 		t.Error("Should not initialize file if no writes")
@@ -136,7 +144,7 @@ func TestFileOutputFilePerRequest(t *testing.T) {
 }
 
 func TestFileOutputCompression(t *testing.T) {
-	output := NewFileOutput("/tmp/log-%Y-%m-%d-%S.gz", &FileOutputConfig{append: true, flushInterval: time.Minute})
+	output := NewFileOutput("/tmp/log-%Y-%m-%d-%S.gz", &FileOutputConfig{Append: true, FlushInterval: time.Minute})
 
 	if output.file != nil {
 		t.Error("Should not initialize file if no writes")
@@ -157,26 +165,6 @@ func TestFileOutputCompression(t *testing.T) {
 	os.Remove(name)
 }
 
-func TestParseDataUnit(t *testing.T) {
-	var tests = []struct {
-		value string
-		size  int64
-	}{
-		{"100kb", dataUnitMap['k'] * 100},
-		{"100k", dataUnitMap['k'] * 100},
-		{"1kb", dataUnitMap['k']},
-		{"1g", dataUnitMap['g']},
-		{"10m", dataUnitMap['m'] * 10},
-		{"zsaa312", 0},
-	}
-
-	for _, c := range tests {
-		if parseDataUnit(c.value) != c.size {
-			t.Error(c.value, "should be", c.size, "instead", parseDataUnit(c.value))
-		}
-	}
-}
-
 func TestGetFileIndex(t *testing.T) {
 	var tests = []struct {
 		path  string
@@ -220,7 +208,7 @@ func TestFileOutputAppendQueueLimitOverflow(t *testing.T) {
 	rnd := rand.Int63()
 	name := fmt.Sprintf("/tmp/%d", rnd)
 
-	output := NewFileOutput(name, &FileOutputConfig{append: false, flushInterval: time.Minute, queueLimit: 2})
+	output := NewFileOutput(name, &FileOutputConfig{Append: false, FlushInterval: time.Minute, QueueLimit: 2})
 
 	output.Write([]byte("1 1 1\r\ntest"))
 	name1 := output.file.Name()
@@ -249,7 +237,7 @@ func TestFileOutputAppendQueueLimitNoOverflow(t *testing.T) {
 	rnd := rand.Int63()
 	name := fmt.Sprintf("/tmp/%d", rnd)
 
-	output := NewFileOutput(name, &FileOutputConfig{append: false, flushInterval: time.Minute, queueLimit: 3})
+	output := NewFileOutput(name, &FileOutputConfig{Append: false, FlushInterval: time.Minute, QueueLimit: 3})
 
 	output.Write([]byte("1 1 1\r\ntest"))
 	name1 := output.file.Name()
@@ -278,7 +266,7 @@ func TestFileOutputAppendQueueLimitGzips(t *testing.T) {
 	rnd := rand.Int63()
 	name := fmt.Sprintf("/tmp/%d.gz", rnd)
 
-	output := NewFileOutput(name, &FileOutputConfig{append: false, flushInterval: time.Minute, queueLimit: 2})
+	output := NewFileOutput(name, &FileOutputConfig{Append: false, FlushInterval: time.Minute, QueueLimit: 2})
 
 	output.Write([]byte("1 1 1\r\ntest"))
 	name1 := output.file.Name()
@@ -321,7 +309,7 @@ func TestFileOutputAppendSizeLimitOverflow(t *testing.T) {
 
 	messageSize := len(message) + len(payloadSeparator)
 
-	output := NewFileOutput(name, &FileOutputConfig{append: false, flushInterval: time.Minute, sizeLimit: unitSizeVar(2 * messageSize)})
+	output := NewFileOutput(name, &FileOutputConfig{Append: false, FlushInterval: time.Minute, SizeLimit: size.Size(2 * messageSize)})
 
 	output.Write([]byte("1 1 1\r\ntest"))
 	name1 := output.file.Name()

output_http.go

@@ -1,16 +1,57 @@
 package main
 
 import (
+	"fmt"
 	"io"
 	"log"
 	"sync/atomic"
 	"time"
 
 	"github.com/buger/goreplay/proto"
+	"github.com/buger/goreplay/size"
 )
 
 const initialDynamicWorkers = 10
 
+type httpWorker struct {
+	output       *HTTPOutput
+	client       *HTTPClient
+	lastActivity time.Time
+	queue        chan []byte
+	stop         chan bool
+}
+
+func newHTTPWorker(output *HTTPOutput, queue chan []byte) *httpWorker {
+	client := NewHTTPClient(output.address, &HTTPClientConfig{
+		FollowRedirects:    output.config.RedirectLimit,
+		Debug:              output.config.Debug,
+		OriginalHost:       output.config.OriginalHost,
+		Timeout:            output.config.Timeout,
+		ResponseBufferSize: int(output.config.BufferSize),
+	})
+
+	w := &httpWorker{client: client}
+	if queue == nil {
+		w.queue = make(chan []byte, 100)
+	} else {
+		w.queue = queue
+	}
+	w.stop = make(chan bool)
+
+	go func() {
+		for {
+			select {
+			case payload := <-w.queue:
+				output.sendRequest(client, payload)
+			case <-w.stop:
+				return
+			}
+		}
+	}()
+
+	return w
+}
+
 type response struct {
 	payload       []byte
 	uuid          []byte
@@ -20,26 +61,28 @@ type response struct {
 
 // HTTPOutputConfig struct for holding http output configuration
 type HTTPOutputConfig struct {
-	redirectLimit int
+	RedirectLimit int `json:"output-http-redirect-limit"`
 
-	stats      bool
-	workersMin int
-	workersMax int
-	statsMs    int
-	workers    int
-	queueLen   int
+	Stats      bool `json:"output-http-stats"`
+	WorkersMin int  `json:"output-http-workers-min"`
+	WorkersMax int  `json:"output-http-workers"`
+	StatsMs    int  `json:"output-http-stats-ms"`
+	Workers    int
+	QueueLen   int `json:"output-http-queue-len"`
 
-	elasticSearch string
+	ElasticSearch string `json:"output-http-elasticsearch"`
 
-	Timeout      time.Duration
-	OriginalHost bool
-	BufferSize   int
+	Timeout      time.Duration `json:"output-http-timeout"`
+	OriginalHost bool          `json:"output-http-original-host"`
+	BufferSize   size.Size     `json:"output-http-response-buffer"`
 
-	CompatibilityMode bool
+	CompatibilityMode bool `json:"output-http-compatibility-mode"`
 
-	Debug bool
+	RequestGroup string
 
-	TrackResponses bool
+	Debug bool `json:"output-http-debug"`
+
+	TrackResponses bool `json:"output-http-track-response"`
 }
 
 // HTTPOutput plugin manage pool of workers which send request to replayed server
@@ -51,6 +94,8 @@ type HTTPOutput struct {
 	// aligned at 64bit. See https://github.com/golang/go/issues/599
 	activeWorkers int64
 
+	workerSessions map[string]*httpWorker
+
 	address string
 	limit   int
 	queue   chan []byte
@@ -64,6 +109,8 @@ type HTTPOutput struct {
 	queueStats *GorStat
 
 	elasticSearch *ESPlugin
+
+	stop chan bool // Channel used only to indicate goroutine should shutdown
 }
 
 // NewHTTPOutput constructor for HTTPOutput
@@ -73,28 +120,37 @@ func NewHTTPOutput(address string, config *HTTPOutputConfig) io.Writer {
 
 	o.address = address
 	o.config = config
+	o.stop = make(chan bool)
 
-	if o.config.stats {
-		o.queueStats = NewGorStat("output_http", o.config.statsMs)
+	if o.config.Stats {
+		o.queueStats = NewGorStat("output_http", o.config.StatsMs)
 	}
 
-	o.queue = make(chan []byte, o.config.queueLen)
-	o.responses = make(chan response, o.config.queueLen)
+	o.queue = make(chan []byte, o.config.QueueLen)
+	o.responses = make(chan response, o.config.QueueLen)
 	o.needWorker = make(chan int, 1)
 
 	// Initial workers count
-	if o.config.workersMax == 0 {
+	if o.config.WorkersMax == 0 {
 		o.needWorker <- initialDynamicWorkers
 	} else {
-		o.needWorker <- o.config.workersMax
+		o.needWorker <- o.config.WorkersMax
 	}
 
-	if o.config.elasticSearch != "" {
+	if o.config.ElasticSearch != "" {
 		o.elasticSearch = new(ESPlugin)
-		o.elasticSearch.Init(o.config.elasticSearch)
+		o.elasticSearch.Init(o.config.ElasticSearch)
 	}
 
-	go o.workerMaster()
+	if Settings.RecognizeTCPSessions {
+		if !PRO {
+			log.Fatal("Detailed TCP sessions work only with PRO license")
+		}
+		o.workerSessions = make(map[string]*httpWorker, 100)
+		go o.sessionWorkerMaster()
+	} else {
+		go o.workerMaster()
+	}
 
 	return o
 }
@@ -102,46 +158,73 @@ func NewHTTPOutput(address string, config *HTTPOutputConfig) io.Writer {
 func (o *HTTPOutput) workerMaster() {
 	for {
 		newWorkers := <-o.needWorker
+		atomic.AddInt64(&o.activeWorkers, int64(newWorkers))
 		for i := 0; i < newWorkers; i++ {
 			go o.startWorker()
 		}
 	}
 }
 
-func (o *HTTPOutput) startWorker() {
-	client := NewHTTPClient(o.address, &HTTPClientConfig{
-		FollowRedirects:    o.config.redirectLimit,
-		Debug:              o.config.Debug,
-		OriginalHost:       o.config.OriginalHost,
-		Timeout:            o.config.Timeout,
-		ResponseBufferSize: o.config.BufferSize,
-		CompatibilityMode:  o.config.CompatibilityMode,
-	})
-
-	deathCount := 0
-
-	atomic.AddInt64(&o.activeWorkers, 1)
+func (o *HTTPOutput) sessionWorkerMaster() {
+	gc := time.Tick(time.Second)
 
 	for {
 		select {
+		case p := <-o.queue:
+			id := payloadID(p)
+			sessionID := string(id[0:20])
+			worker, ok := o.workerSessions[sessionID]
+
+			if !ok {
+				atomic.AddInt64(&o.activeWorkers, 1)
+				worker = newHTTPWorker(o, nil)
+				o.workerSessions[sessionID] = worker
+			}
+
+			worker.queue <- p
+			worker.lastActivity = time.Now()
+		case <-gc:
+			now := time.Now()
+
+			for id, w := range o.workerSessions {
+				if !w.lastActivity.IsZero() && now.Sub(w.lastActivity) >= 120*time.Second {
+					w.stop <- true
+					delete(o.workerSessions, id)
+					atomic.AddInt64(&o.activeWorkers, -1)
+				}
+			}
+		}
+	}
+}
+
+func (o *HTTPOutput) startWorker() {
+	client := NewHTTPClient(o.address, &HTTPClientConfig{
+		FollowRedirects:    o.config.RedirectLimit,
+		Debug:              o.config.Debug,
+		OriginalHost:       o.config.OriginalHost,
+		Timeout:            o.config.Timeout,
+		ResponseBufferSize: int(o.config.BufferSize),
+		CompatibilityMode:  o.config.CompatibilityMode,
+	})
+
+	for {
+		select {
+		case <-o.stop:
+			return
 		case data := <-o.queue:
 			o.sendRequest(client, data)
-			deathCount = 0
-		case <-time.After(time.Millisecond * 100):
+		case <-time.After(2 * time.Second):
 			// When dynamic scaling enabled workers die after 2s of inactivity
-			if o.config.workersMin == o.config.workersMax {
+			if o.config.WorkersMin == o.config.WorkersMax {
 				continue
 			}
 
-			deathCount++
-			if deathCount > 20 {
-				workersCount := int(atomic.LoadInt64(&o.activeWorkers))
+			workersCount := int(atomic.LoadInt64(&o.activeWorkers))
 
-				// At least 1 startWorker should be alive
-				if workersCount != 1 && workersCount > o.config.workersMin {
-					atomic.AddInt64(&o.activeWorkers, -1)
-					return
-				}
+			// At least 1 startWorker should be alive
+			if workersCount != 1 && workersCount > o.config.WorkersMin {
+				atomic.AddInt64(&o.activeWorkers, -1)
+				return
 			}
 		}
 	}
@@ -155,18 +238,22 @@ func (o *HTTPOutput) Write(data []byte) (n int, err error) {
 	buf := make([]byte, len(data))
 	copy(buf, data)
 
-	o.queue <- buf
+	select {
+	case <-o.stop:
+		return 0, ErrorStopped
+	case o.queue <- buf:
+	}
 
-	if o.config.stats {
+	if o.config.Stats {
 		o.queueStats.Write(len(o.queue))
 	}
 
-	if o.config.workersMax != o.config.workersMin {
+	if !Settings.RecognizeTCPSessions && o.config.WorkersMax != o.config.WorkersMin {
 		workersCount := int(atomic.LoadInt64(&o.activeWorkers))
 
 		if len(o.queue) > workersCount {
 			extraWorkersReq := len(o.queue) - workersCount + 1
-			maxWorkersAvailable := o.config.workersMax - workersCount
+			maxWorkersAvailable := o.config.WorkersMax - workersCount
 			if extraWorkersReq > maxWorkersAvailable {
 				extraWorkersReq = maxWorkersAvailable
 			}
@@ -180,25 +267,32 @@ func (o *HTTPOutput) Write(data []byte) (n int, err error) {
 }
 
 func (o *HTTPOutput) Read(data []byte) (int, error) {
-	resp := <-o.responses
-
-	if Settings.debug {
-		Debug("[OUTPUT-HTTP] Received response:", string(resp.payload))
+	var resp response
+	select {
+	case <-o.stop:
+		return 0, ErrorStopped
+	case resp = <-o.responses:
 	}
 
-	header := payloadHeader(ReplayedResponsePayload, resp.uuid, resp.roundTripTime, resp.startedAt)
-	copy(data[0:len(header)], header)
-	copy(data[len(header):], resp.payload)
+	Debug(3, "[OUTPUT-HTTP] Received response:", string(resp.payload))
 
-	return len(resp.payload) + len(header), nil
+	header := payloadHeader(ReplayedResponsePayload, resp.uuid, resp.roundTripTime, resp.startedAt)
+	n := copy(data, header)
+	if len(data) > len(header) {
+		n += copy(data[len(header):], resp.payload)
+	}
+	dis := len(header) + len(data) - n
+	if dis > 0 {
+		Debug(2, "[OUTPUT-HTTP] discarded", dis, "increase copy buffer size")
+	}
+
+	return n, nil
 }
 
 func (o *HTTPOutput) sendRequest(client *HTTPClient, request []byte) {
 	meta := payloadMeta(request)
 
-	if Settings.debug {
-		Debug(meta)
-	}
+	Debug(2, fmt.Sprintf("[OUTPUT-HTTP] meta: %q", meta))
 
 	if len(meta) < 2 {
 		return
@@ -206,7 +300,7 @@ func (o *HTTPOutput) sendRequest(client *HTTPClient, request []byte) {
 	uuid := meta[1]
 
 	body := payloadBody(request)
-	if !proto.IsHTTPPayload(body) {
+	if !proto.HasRequestTitle(body) {
 		return
 	}
 
@@ -215,8 +309,7 @@ func (o *HTTPOutput) sendRequest(client *HTTPClient, request []byte) {
 	stop := time.Now()
 
 	if err != nil {
-		log.Println("Error when sending ", err, time.Now())
-		Debug("Request error:", err)
+		Debug(1, "Error when sending ", err)
 	}
 
 	if o.config.TrackResponses {
@@ -231,3 +324,9 @@ func (o *HTTPOutput) sendRequest(client *HTTPClient, request []byte) {
 func (o *HTTPOutput) String() string {
 	return "HTTP output: " + o.address
 }
+
+// Close closes the data channel so that data
+func (o *HTTPOutput) Close() error {
+	close(o.stop)
+	return nil
+}

output_http_test.go

@@ -41,19 +41,23 @@ func TestHTTPOutput(t *testing.T) {
 
 	headers := HTTPHeaders{HTTPHeader{"User-Agent", "Gor"}}
 	methods := HTTPMethods{[]byte("GET"), []byte("PUT"), []byte("POST")}
-	Settings.modifierConfig = HTTPModifierConfig{headers: headers, methods: methods}
+	Settings.ModifierConfig = HTTPModifierConfig{Headers: headers, Methods: methods}
 
 	http_output := NewHTTPOutput(server.URL, &HTTPOutputConfig{Debug: true, TrackResponses: true})
 	output := NewTestOutput(func(data []byte) {
 		wg.Done()
 	})
 
-	Plugins.Inputs = []io.Reader{input}
-	Plugins.Outputs = []io.Writer{http_output, output}
+	plugins := &InOutPlugins{
+		Inputs:  []io.Reader{input},
+		Outputs: []io.Writer{http_output, output},
+	}
+	plugins.All = append(plugins.All, input, output, http_output)
 
-	go Start(quit)
+	emitter := NewEmitter(quit)
+	go emitter.Start(plugins, Settings.Middleware)
 
-	for i := 0; i < 1; i++ {
+	for i := 0; i < 10; i++ {
 		// 2 http-output, 2 - test output request, 2 - test output http response
 		wg.Add(6) // OPTIONS should be ignored
 		input.EmitPOST()
@@ -62,10 +66,9 @@ func TestHTTPOutput(t *testing.T) {
 	}
 
 	wg.Wait()
+	emitter.Close()
 
-	close(quit)
-
-	Settings.modifierConfig = HTTPModifierConfig{}
+	Settings.ModifierConfig = HTTPModifierConfig{}
 }
 
 func TestHTTPOutputKeepOriginalHost(t *testing.T) {
@@ -84,26 +87,28 @@ func TestHTTPOutputKeepOriginalHost(t *testing.T) {
 	defer server.Close()
 
 	headers := HTTPHeaders{HTTPHeader{"Host", "custom-host.com"}}
-	Settings.modifierConfig = HTTPModifierConfig{headers: headers}
+	Settings.ModifierConfig = HTTPModifierConfig{Headers: headers}
 
 	output := NewHTTPOutput(server.URL, &HTTPOutputConfig{Debug: false, OriginalHost: true})
 
-	Plugins.Inputs = []io.Reader{input}
-	Plugins.Outputs = []io.Writer{output}
+	plugins := &InOutPlugins{
+		Inputs:  []io.Reader{input},
+		Outputs: []io.Writer{output},
+	}
+	plugins.All = append(plugins.All, input, output)
 
-	go Start(quit)
+	emitter := NewEmitter(quit)
+	go emitter.Start(plugins, Settings.Middleware)
 
 	wg.Add(1)
 	input.EmitGET()
 
 	wg.Wait()
-
-	close(quit)
-
-	Settings.modifierConfig = HTTPModifierConfig{}
+	emitter.Close()
+	Settings.ModifierConfig = HTTPModifierConfig{}
 }
 
-func TestOutputHTTPSSL(t *testing.T) {
+func TestHTTPOutputSSL(t *testing.T) {
 	wg := new(sync.WaitGroup)
 	quit := make(chan int)
 
@@ -115,10 +120,14 @@ func TestOutputHTTPSSL(t *testing.T) {
 	input := NewTestInput()
 	output := NewHTTPOutput(server.URL, &HTTPOutputConfig{})
 
-	Plugins.Inputs = []io.Reader{input}
-	Plugins.Outputs = []io.Writer{output}
+	plugins := &InOutPlugins{
+		Inputs:  []io.Reader{input},
+		Outputs: []io.Writer{output},
+	}
+	plugins.All = append(plugins.All, input, output)
 
-	go Start(quit)
+	emitter := NewEmitter(quit)
+	go emitter.Start(plugins, Settings.Middleware)
 
 	wg.Add(2)
 
@@ -126,7 +135,51 @@ func TestOutputHTTPSSL(t *testing.T) {
 	input.EmitGET()
 
 	wg.Wait()
-	close(quit)
+	emitter.Close()
+}
+
+func TestHTTPOutputSessions(t *testing.T) {
+	wg := new(sync.WaitGroup)
+	quit := make(chan int)
+
+	input := NewTestInput()
+	input.skipHeader = true
+
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
+		wg.Done()
+	}))
+	defer server.Close()
+
+	Settings.RecognizeTCPSessions = true
+	output := NewHTTPOutput(server.URL, &HTTPOutputConfig{Debug: true})
+
+	plugins := &InOutPlugins{
+		Inputs:  []io.Reader{input},
+		Outputs: []io.Writer{output},
+	}
+	emitter := NewEmitter(quit)
+	go emitter.Start(plugins, Settings.Middleware)
+
+	uuid1 := []byte("1234567890123456789a0000")
+	uuid2 := []byte("1234567890123456789d0000")
+
+	for i := 0; i < 100; i++ {
+		wg.Add(1) // OPTIONS should be ignored
+		copy(uuid1[20:], randByte(4))
+		input.EmitBytes([]byte("1 " + string(uuid1) + " 1\n" + "GET / HTTP/1.1\r\n\r\n"))
+	}
+
+	for i := 0; i < 100; i++ {
+		wg.Add(1) // OPTIONS should be ignored
+		copy(uuid2[20:], randByte(4))
+		input.EmitBytes([]byte("1 " + string(uuid2) + " 1\n" + "GET / HTTP/1.1\r\n\r\n"))
+	}
+
+	wg.Wait()
+
+	emitter.Close()
+
+	Settings.RecognizeTCPSessions = false
 }
 
 func BenchmarkHTTPOutput(b *testing.B) {
@@ -142,10 +195,14 @@ func BenchmarkHTTPOutput(b *testing.B) {
 	input := NewTestInput()
 	output := NewHTTPOutput(server.URL, &HTTPOutputConfig{})
 
-	Plugins.Inputs = []io.Reader{input}
-	Plugins.Outputs = []io.Writer{output}
+	plugins := &InOutPlugins{
+		Inputs:  []io.Reader{input},
+		Outputs: []io.Writer{output},
+	}
+	plugins.All = append(plugins.All, input, output)
 
-	go Start(quit)
+	emitter := NewEmitter(quit)
+	go emitter.Start(plugins, Settings.Middleware)
 
 	for i := 0; i < b.N; i++ {
 		wg.Add(1)
@@ -153,6 +210,5 @@ func BenchmarkHTTPOutput(b *testing.B) {
 	}
 
 	wg.Wait()
-
-	close(quit)
+	emitter.Close()
 }

output_kafka.go

@@ -15,7 +15,7 @@ import (
 
 // KafkaOutput is used for sending payloads to kafka in JSON format.
 type KafkaOutput struct {
-	config   *KafkaConfig
+	config   *OutputKafkaConfig
 	producer sarama.AsyncProducer
 }
 
@@ -23,8 +23,13 @@ type KafkaOutput struct {
 const KafkaOutputFrequency = 500
 
 // NewKafkaOutput creates instance of kafka producer client.
-func NewKafkaOutput(address string, config *KafkaConfig) io.Writer {
-	c := sarama.NewConfig()
+func NewKafkaOutput(address string, config *OutputKafkaConfig) io.Writer {
+	return NewKafkaOutputWithTLS(address, config, nil)
+}
+
+// NewKafkaOutputWithTLS creates instance of kafka producer client.
+func NewKafkaOutputWithTLS(address string, config *OutputKafkaConfig, tlsConfig *KafkaTLSConfig) io.Writer {
+	c := NewKafkaConfig(tlsConfig)
 
 	var producer sarama.AsyncProducer
 
@@ -35,7 +40,7 @@ func NewKafkaOutput(address string, config *KafkaConfig) io.Writer {
 		c.Producer.Compression = sarama.CompressionSnappy
 		c.Producer.Flush.Frequency = KafkaOutputFrequency * time.Millisecond
 
-		brokerList := strings.Split(config.host, ",")
+		brokerList := strings.Split(config.Host, ",")
 
 		var err error
 		producer, err = sarama.NewAsyncProducer(brokerList, c)
@@ -49,10 +54,8 @@ func NewKafkaOutput(address string, config *KafkaConfig) io.Writer {
 		producer: producer,
 	}
 
-	if Settings.verbose {
-		// Start infinite loop for tracking errors for kafka producer.
-		go o.ErrorHandler()
-	}
+	// Start infinite loop for tracking errors for kafka producer.
+	go o.ErrorHandler()
 
 	return o
 }
@@ -60,20 +63,19 @@ func NewKafkaOutput(address string, config *KafkaConfig) io.Writer {
 // ErrorHandler should receive errors
 func (o *KafkaOutput) ErrorHandler() {
 	for err := range o.producer.Errors() {
-		log.Println("Failed to write access log entry:", err)
+		Debug(1, "Failed to write access log entry:", err)
 	}
 }
 
 func (o *KafkaOutput) Write(data []byte) (n int, err error) {
 	var message sarama.StringEncoder
 
-	if !o.config.useJSON {
+	if !o.config.UseJSON {
 		message = sarama.StringEncoder(data)
 	} else {
 		headers := make(map[string]string)
-		proto.ParseHeaders([][]byte{data}, func(header []byte, value []byte) bool {
+		proto.ParseHeaders([][]byte{data}, func(header []byte, value []byte) {
 			headers[string(header)] = string(value)
-			return true
 		})
 
 		meta := payloadMeta(data)
@@ -93,7 +95,7 @@ func (o *KafkaOutput) Write(data []byte) (n int, err error) {
 	}
 
 	o.producer.Input() <- &sarama.ProducerMessage{
-		Topic: o.config.topic,
+		Topic: o.config.Topic,
 		Value: message,
 	}
 

output_kafka_test.go

@@ -13,10 +13,10 @@ func TestOutputKafkaRAW(t *testing.T) {
 	producer := mocks.NewAsyncProducer(t, config)
 	producer.ExpectInputAndSucceed()
 
-	output := NewKafkaOutput("", &KafkaConfig{
+	output := NewKafkaOutput("", &OutputKafkaConfig{
 		producer: producer,
-		topic:    "test",
-		useJSON:  false,
+		Topic:    "test",
+		UseJSON:  false,
 	})
 
 	output.Write([]byte("1 2 3\nGET / HTTP1.1\r\nHeader: 1\r\n\r\n"))
@@ -36,10 +36,10 @@ func TestOutputKafkaJSON(t *testing.T) {
 	producer := mocks.NewAsyncProducer(t, config)
 	producer.ExpectInputAndSucceed()
 
-	output := NewKafkaOutput("", &KafkaConfig{
+	output := NewKafkaOutput("", &OutputKafkaConfig{
 		producer: producer,
-		topic:    "test",
-		useJSON:  true,
+		Topic:    "test",
+		UseJSON:  true,
 	})
 
 	output.Write([]byte("1 2 3\nGET / HTTP1.1\r\nHeader: 1\r\n\r\n"))
@@ -48,7 +48,7 @@ func TestOutputKafkaJSON(t *testing.T) {
 
 	data, _ := resp.Value.Encode()
 
-	if string(data) != `{"Req_URL":"/","Req_Type":"1","Req_ID":"2","Req_Ts":"3","Req_Method":"GET","Req_Headers":{"Header":"1"}}` {
+	if string(data) != `{"Req_URL":"","Req_Type":"1","Req_ID":"2","Req_Ts":"3","Req_Method":"GET"}` {
 		t.Error("Message not properly encoded: ", string(data))
 	}
 }

output_null.go

@@ -4,7 +4,7 @@ package main
 type NullOutput struct {
 }
 
-// NullOutput constructor for NullOutput
+// NewNullOutput constructor for NullOutput
 func NewNullOutput() (o *NullOutput) {
 	return new(NullOutput)
 }

output_s3.go

@@ -0,0 +1,129 @@
+package main
+
+import (
+	_ "bufio"
+	"fmt"
+	_ "io"
+	"log"
+	"math/rand"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/s3"
+	_ "github.com/aws/aws-sdk-go/service/s3/s3manager"
+)
+
+// S3Output output plugin
+type S3Output struct {
+	pathTemplate string
+
+	buffer  *FileOutput
+	session *session.Session
+	config  *FileOutputConfig
+	closeCh chan struct{}
+}
+
+// NewS3Output constructor for FileOutput, accepts path
+func NewS3Output(pathTemplate string, config *FileOutputConfig) *S3Output {
+	if !PRO {
+		log.Fatal("Using S3 output and input requires PRO license")
+		return nil
+	}
+
+	o := new(S3Output)
+	o.pathTemplate = pathTemplate
+	o.config = config
+	o.config.onClose = o.onBufferUpdate
+
+	if config.BufferPath == "" {
+		config.BufferPath = "/tmp"
+	}
+
+	rnd := rand.Int63()
+	bufferName := fmt.Sprintf("gor_output_s3_%d_buf_", rnd)
+
+	pathParts := strings.Split(pathTemplate, "/")
+	bufferName += pathParts[len(pathParts)-1]
+
+	if strings.HasSuffix(o.pathTemplate, ".gz") {
+		bufferName += ".gz"
+	}
+
+	bufferPath := filepath.Join(config.BufferPath, bufferName)
+
+	o.buffer = NewFileOutput(bufferPath, config)
+	o.connect()
+
+	return o
+}
+
+func (o *S3Output) connect() {
+	if o.session == nil {
+		o.session = session.Must(session.NewSession(awsConfig()))
+		log.Println("[S3 Output] S3 connection succesfully initialized")
+	}
+}
+
+func (o *S3Output) Write(data []byte) (n int, err error) {
+	return o.buffer.Write(data)
+}
+
+func (o *S3Output) String() string {
+	return "S3 output: " + o.pathTemplate
+}
+
+// Close close the buffer of the S3 connection
+func (o *S3Output) Close() error {
+	return o.buffer.Close()
+}
+
+func parseS3Url(path string) (bucket, key string) {
+	path = path[5:] // stripping `s3://`
+	sep := strings.IndexByte(path, '/')
+
+	bucket = path[:sep]
+	key = path[sep+1:]
+
+	return bucket, key
+}
+
+func (o *S3Output) keyPath(idx int) (bucket, key string) {
+	bucket, key = parseS3Url(o.pathTemplate)
+
+	for name, fn := range dateFileNameFuncs {
+		key = strings.Replace(key, name, fn(o.buffer), -1)
+	}
+
+	key = setFileIndex(key, idx)
+
+	return
+}
+
+func (o *S3Output) onBufferUpdate(path string) {
+	svc := s3.New(o.session)
+	idx := getFileIndex(path)
+	bucket, key := o.keyPath(idx)
+
+	file, _ := os.Open(path)
+	// reader := bufio.NewReader(file)
+
+	_, err := svc.PutObject(&s3.PutObjectInput{
+		Body:   file,
+		Bucket: aws.String(bucket),
+		Key:    aws.String(key),
+	})
+	if err != nil {
+		log.Printf("[S3 Output] Failed to upload data to %s/%s, %s\n", bucket, key, err)
+		os.Remove(path)
+		return
+	}
+
+	os.Remove(path)
+
+	if o.closeCh != nil {
+		o.closeCh <- struct{}{}
+	}
+}

output_tcp.go

@@ -3,6 +3,7 @@ package main
 import (
 	"crypto/tls"
 	"fmt"
+	"hash/fnv"
 	"io"
 	"log"
 	"net"
@@ -15,37 +16,52 @@ import (
 type TCPOutput struct {
 	address  string
 	limit    int
-	buf      chan []byte
+	buf      []chan []byte
 	bufStats *GorStat
 	config   *TCPOutputConfig
 }
 
+// TCPOutputConfig tcp output configuration
 type TCPOutputConfig struct {
-	secure bool
+	Secure     bool `json:"output-tcp-secure"`
+	Sticky     bool `json:"output-tcp-sticky"`
+	SkipVerify bool `json:"output-tcp-skip-verify"`
+	Workers    int  `json:"output-tcp-workers"`
 }
 
 // NewTCPOutput constructor for TCPOutput
-// Initialize 10 workers which hold keep-alive connection
+// Initialize X workers which hold keep-alive connection
 func NewTCPOutput(address string, config *TCPOutputConfig) io.Writer {
 	o := new(TCPOutput)
 
 	o.address = address
 	o.config = config
 
-	o.buf = make(chan []byte, 1000)
-	if Settings.outputTCPStats {
+	if Settings.OutputTCPStats {
 		o.bufStats = NewGorStat("output_tcp", 5000)
 	}
 
-	for i := 0; i < 10; i++ {
-		go o.worker()
+	if o.config.Sticky {
+		// create X buffers and send the buffer index to the worker
+		o.buf = make([]chan []byte, o.config.Workers)
+		for i := 0; i < o.config.Workers; i++ {
+			o.buf[i] = make(chan []byte, 100)
+			go o.worker(i)
+		}
+	} else {
+		// create 1 buffer and send its index (0) to all workers
+		o.buf = make([]chan []byte, 1)
+		o.buf[0] = make(chan []byte, 1000)
+		for i := 0; i < o.config.Workers; i++ {
+			go o.worker(0)
+		}
 	}
 
 	return o
 }
 
-func (o *TCPOutput) worker() {
-	retries := 1
+func (o *TCPOutput) worker(bufferIndex int) {
+	retries := 0
 	conn, err := o.connect(o.address)
 	for {
 		if err == nil {
@@ -66,19 +82,29 @@ func (o *TCPOutput) worker() {
 	defer conn.Close()
 
 	for {
-		data := <-o.buf
+		data := <-o.buf[bufferIndex]
 		conn.Write(data)
 		_, err := conn.Write([]byte(payloadSeparator))
 
 		if err != nil {
 			log.Println("INFO: TCP output connection closed, reconnecting")
-			o.buf <- data
-			go o.worker()
+			o.buf[bufferIndex] <- data
+			go o.worker(bufferIndex)
 			break
 		}
 	}
 }
 
+func (o *TCPOutput) getBufferIndex(data []byte) int {
+	if !o.config.Sticky {
+		return 0
+	}
+
+	hasher := fnv.New32a()
+	hasher.Write(payloadMeta(data)[1])
+	return int(hasher.Sum32()) % o.config.Workers
+}
+
 func (o *TCPOutput) Write(data []byte) (n int, err error) {
 	if !isOriginPayload(data) {
 		return len(data), nil
@@ -88,18 +114,19 @@ func (o *TCPOutput) Write(data []byte) (n int, err error) {
 	newBuf := make([]byte, len(data))
 	copy(newBuf, data)
 
-	o.buf <- newBuf
+	bufferIndex := o.getBufferIndex(data)
+	o.buf[bufferIndex] <- newBuf
 
-	if Settings.outputTCPStats {
-		o.bufStats.Write(len(o.buf))
+	if Settings.OutputTCPStats {
+		o.bufStats.Write(len(o.buf[bufferIndex]))
 	}
 
 	return len(data), nil
 }
 
 func (o *TCPOutput) connect(address string) (conn net.Conn, err error) {
-	if o.config.secure {
-		conn, err = tls.Dial("tcp", address, &tls.Config{})
+	if o.config.Secure {
+		conn, err = tls.Dial("tcp", address, &tls.Config{InsecureSkipVerify: o.config.SkipVerify})
 	} else {
 		conn, err = net.Dial("tcp", address)
 	}

output_tcp_test.go

@@ -7,6 +7,7 @@ import (
 	"net"
 	"sync"
 	"testing"
+	"time"
 )
 
 func TestTCPOutput(t *testing.T) {
@@ -19,10 +20,14 @@ func TestTCPOutput(t *testing.T) {
 	input := NewTestInput()
 	output := NewTCPOutput(listener.Addr().String(), &TCPOutputConfig{})
 
-	Plugins.Inputs = []io.Reader{input}
-	Plugins.Outputs = []io.Writer{output}
+	plugins := &InOutPlugins{
+		Inputs:  []io.Reader{input},
+		Outputs: []io.Writer{output},
+	}
+	plugins.All = append(plugins.All, input, output)
 
-	go Start(quit)
+	emitter := NewEmitter(quit)
+	go emitter.Start(plugins, Settings.Middleware)
 
 	for i := 0; i < 100; i++ {
 		wg.Add(1)
@@ -30,8 +35,7 @@ func TestTCPOutput(t *testing.T) {
 	}
 
 	wg.Wait()
-
-	close(quit)
+	emitter.Close()
 }
 
 func startTCP(cb func([]byte)) net.Listener {
@@ -71,10 +75,14 @@ func BenchmarkTCPOutput(b *testing.B) {
 	input := NewTestInput()
 	output := NewTCPOutput(listener.Addr().String(), &TCPOutputConfig{})
 
-	Plugins.Inputs = []io.Reader{input}
-	Plugins.Outputs = []io.Writer{output}
+	plugins := &InOutPlugins{
+		Inputs:  []io.Reader{input},
+		Outputs: []io.Writer{output},
+	}
+	plugins.All = append(plugins.All, input, output)
 
-	go Start(quit)
+	emitter := NewEmitter(quit)
+	go emitter.Start(plugins, Settings.Middleware)
 
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
@@ -83,6 +91,46 @@ func BenchmarkTCPOutput(b *testing.B) {
 	}
 
 	wg.Wait()
+	emitter.Close()
+}
 
-	close(quit)
+func TestStickyDisable(t *testing.T) {
+	tcpOutput := TCPOutput{config: &TCPOutputConfig{Sticky: false}}
+
+	for i := 0; i < 1000; i++ {
+		index := tcpOutput.getBufferIndex(getTestBytes())
+		if index != 0 {
+			t.Errorf("Sticky is disable. Got: %d want 0", index)
+		}
+	}
+}
+
+func TestBufferDistribution(t *testing.T) {
+	numberOfWorkers := 10
+	numberOfMessages := 1000000
+	percentDistributionErrorRange := 20
+
+	buffer := make([]int, numberOfWorkers)
+	tcpOutput := TCPOutput{config: &TCPOutputConfig{Sticky: true}}
+	for i := 0; i < numberOfMessages; i++ {
+		buffer[tcpOutput.getBufferIndex(getTestBytes())]++
+	}
+
+	expectedDistribution := numberOfMessages / numberOfWorkers
+	lowerDistribution := expectedDistribution - (expectedDistribution * percentDistributionErrorRange / 100)
+	upperDistribution := expectedDistribution + (expectedDistribution * percentDistributionErrorRange / 100)
+	for i := 0; i < numberOfWorkers; i++ {
+		if buffer[i] < lowerDistribution {
+			t.Errorf("Under expected distribution. Got %d expected lower distribution %d", buffer[i], lowerDistribution)
+		}
+		if buffer[i] > upperDistribution {
+			t.Errorf("Under expected distribution. Got %d expected upper distribution %d", buffer[i], upperDistribution)
+		}
+	}
+}
+
+func getTestBytes() []byte {
+	reqh := payloadHeader(RequestPayload, uuid(), time.Now().UnixNano(), -1)
+	reqb := append(reqh, []byte("GET / HTTP/1.1\r\nHost: www.w3.org\r\nUser-Agent: Go 1.1 package http\r\nAccept-Encoding: gzip\r\n\r\n")...)
+	return reqb
 }

plugins.go

@@ -4,7 +4,6 @@ import (
 	"io"
 	"reflect"
 	"strings"
-	"sync"
 )
 
 // InOutPlugins struct for holding references to plugins
@@ -14,11 +13,6 @@ type InOutPlugins struct {
 	All     []interface{}
 }
 
-var pluginMu sync.Mutex
-
-// Plugins holds all the plugin objects
-var Plugins *InOutPlugins = new(InOutPlugins)
-
 // extractLimitOptions detects if plugin get called with limiter support
 // Returns address and limit
 func extractLimitOptions(options string) (string, string) {
@@ -33,8 +27,8 @@ func extractLimitOptions(options string) (string, string) {
 
 // Automatically detects type of plugin and initialize it
 //
-// See this article if curious about relfect stuff below: http://blog.burntsushi.net/type-parametric-functions-golang
-func registerPlugin(constructor interface{}, options ...interface{}) {
+// See this article if curious about reflect stuff below: http://blog.burntsushi.net/type-parametric-functions-golang
+func (plugins *InOutPlugins) registerPlugin(constructor interface{}, options ...interface{}) {
 	var path, limit string
 	vc := reflect.ValueOf(constructor)
 
@@ -67,86 +61,88 @@ func registerPlugin(constructor interface{}, options ...interface{}) {
 
 	// Some of the output can be Readers as well because return responses
 	if isR && !isW {
-		Plugins.Inputs = append(Plugins.Inputs, pluginWrapper.(io.Reader))
+		plugins.Inputs = append(plugins.Inputs, pluginWrapper.(io.Reader))
 	}
 
 	if isW {
-		Plugins.Outputs = append(Plugins.Outputs, pluginWrapper.(io.Writer))
+		plugins.Outputs = append(plugins.Outputs, pluginWrapper.(io.Writer))
 	}
 
-	Plugins.All = append(Plugins.All, plugin)
+	plugins.All = append(plugins.All, plugin)
 }
 
-// InitPlugins specify and initialize all available plugins
-func InitPlugins() {
-	pluginMu.Lock()
-	defer pluginMu.Unlock()
+// NewPlugins specify and initialize all available plugins
+func NewPlugins() *InOutPlugins {
+	plugins := new(InOutPlugins)
 
-	for _, options := range Settings.inputDummy {
-		registerPlugin(NewDummyInput, options)
+	for _, options := range Settings.InputDummy {
+		plugins.registerPlugin(NewDummyInput, options)
 	}
 
-	for range Settings.outputDummy {
-		registerPlugin(NewDummyOutput)
+	for range Settings.OutputDummy {
+		plugins.registerPlugin(NewDummyOutput)
 	}
 
-	if Settings.outputStdout {
-		registerPlugin(NewDummyOutput)
+	if Settings.OutputStdout {
+		plugins.registerPlugin(NewDummyOutput)
 	}
 
-	if Settings.outputNull {
-		registerPlugin(NewNullOutput)
+	if Settings.OutputNull {
+		plugins.registerPlugin(NewNullOutput)
 	}
 
-	engine := EnginePcap
-	if Settings.inputRAWEngine == "raw_socket" {
-		engine = EngineRawSocket
-	} else if Settings.inputRAWEngine == "pcap_file" {
-		engine = EnginePcapFile
+	for _, options := range Settings.InputRAW {
+		plugins.registerPlugin(NewRAWInput, options, Settings.RAWInputConfig)
 	}
 
-	for _, options := range Settings.inputRAW {
-		registerPlugin(NewRAWInput, options, engine, Settings.inputRAWTrackResponse, Settings.inputRAWExpire, Settings.inputRAWRealIPHeader, Settings.inputRAWBpfFilter, Settings.inputRAWTimestampType, Settings.inputRawBufferSize)
+	for _, options := range Settings.InputTCP {
+		plugins.registerPlugin(NewTCPInput, options, &Settings.InputTCPConfig)
 	}
 
-	for _, options := range Settings.inputTCP {
-		registerPlugin(NewTCPInput, options, &Settings.inputTCPConfig)
+	for _, options := range Settings.OutputTCP {
+		plugins.registerPlugin(NewTCPOutput, options, &Settings.OutputTCPConfig)
 	}
 
-	for _, options := range Settings.outputTCP {
-		registerPlugin(NewTCPOutput, options, &Settings.outputTCPConfig)
+	for _, options := range Settings.InputFile {
+		plugins.registerPlugin(NewFileInput, options, Settings.InputFileLoop)
 	}
 
-	for _, options := range Settings.inputFile {
-		registerPlugin(NewFileInput, options, Settings.inputFileLoop)
+	for _, path := range Settings.OutputFile {
+		if strings.HasPrefix(path, "s3://") {
+			plugins.registerPlugin(NewS3Output, path, &Settings.OutputFileConfig)
+		} else {
+			plugins.registerPlugin(NewFileOutput, path, &Settings.OutputFileConfig)
+		}
 	}
 
-	for _, options := range Settings.outputFile {
-		registerPlugin(NewFileOutput, options, &Settings.outputFileConfig)
-	}
-
-	for _, options := range Settings.inputHTTP {
-		registerPlugin(NewHTTPInput, options)
+	for _, options := range Settings.InputHTTP {
+		plugins.registerPlugin(NewHTTPInput, options)
 	}
 
 	// If we explicitly set Host header http output should not rewrite it
 	// Fix: https://github.com/buger/gor/issues/174
-	for _, header := range Settings.modifierConfig.headers {
+	for _, header := range Settings.ModifierConfig.Headers {
 		if header.Name == "Host" {
-			Settings.outputHTTPConfig.OriginalHost = true
+			Settings.OutputHTTPConfig.OriginalHost = true
 			break
 		}
 	}
 
-	for _, options := range Settings.outputHTTP {
-		registerPlugin(NewHTTPOutput, options, &Settings.outputHTTPConfig)
+	for _, options := range Settings.OutputHTTP {
+		plugins.registerPlugin(NewHTTPOutput, options, &Settings.OutputHTTPConfig)
 	}
 
-	if Settings.outputKafkaConfig.host != "" && Settings.outputKafkaConfig.topic != "" {
-		registerPlugin(NewKafkaOutput, "", &Settings.outputKafkaConfig)
+	for _, options := range Settings.OutputBinary {
+		plugins.registerPlugin(NewBinaryOutput, options, &Settings.OutputBinaryConfig)
 	}
 
-	if Settings.inputKafkaConfig.host != "" && Settings.inputKafkaConfig.topic != "" {
-		registerPlugin(NewKafkaInput, "", &Settings.inputKafkaConfig)
+	if Settings.OutputKafkaConfig.Host != "" && Settings.OutputKafkaConfig.Topic != "" {
+		plugins.registerPlugin(NewKafkaOutput, "", &Settings.OutputKafkaConfig, &Settings.KafkaTLSConfig)
 	}
+
+	if Settings.InputKafkaConfig.Host != "" && Settings.InputKafkaConfig.Topic != "" {
+		plugins.registerPlugin(NewKafkaInput, "", &Settings.InputKafkaConfig, &Settings.KafkaTLSConfig)
+	}
+
+	return plugins
 }

plugins_test.go

@@ -1,42 +1,38 @@
 package main
 
 import (
-	"io"
 	"testing"
 )
 
 func TestPluginsRegistration(t *testing.T) {
-	Plugins.Inputs = []io.Reader{}
-	Plugins.Outputs = []io.Writer{}
+	Settings.InputDummy = MultiOption{"[]"}
+	Settings.OutputDummy = MultiOption{"[]"}
+	Settings.OutputHTTP = MultiOption{"www.example.com|10"}
+	Settings.InputFile = MultiOption{"/dev/null"}
 
-	Settings.inputDummy = MultiOption{"[]"}
-	Settings.outputDummy = MultiOption{"[]"}
-	Settings.outputHTTP = MultiOption{"www.example.com|10"}
-	Settings.inputFile = MultiOption{"/dev/null"}
+	plugins := NewPlugins()
 
-	InitPlugins()
-
-	if len(Plugins.Inputs) != 2 {
-		t.Errorf("Should be 2 inputs %d", len(Plugins.Inputs))
+	if len(plugins.Inputs) != 2 {
+		t.Errorf("Should be 2 inputs %d", len(plugins.Inputs))
 	}
 
-	if _, ok := Plugins.Inputs[0].(*DummyInput); !ok {
+	if _, ok := plugins.Inputs[0].(*DummyInput); !ok {
 		t.Errorf("First input should be DummyInput")
 	}
 
-	if _, ok := Plugins.Inputs[1].(*FileInput); !ok {
+	if _, ok := plugins.Inputs[1].(*FileInput); !ok {
 		t.Errorf("Second input should be FileInput")
 	}
 
-	if len(Plugins.Outputs) != 2 {
-		t.Errorf("Should be 2 output %d", len(Plugins.Outputs))
+	if len(plugins.Outputs) != 2 {
+		t.Errorf("Should be 2 output %d", len(plugins.Outputs))
 	}
 
-	if _, ok := Plugins.Outputs[0].(*DummyOutput); !ok {
+	if _, ok := plugins.Outputs[0].(*DummyOutput); !ok {
 		t.Errorf("First output should be DummyOutput")
 	}
 
-	if l, ok := Plugins.Outputs[1].(*Limiter); ok {
+	if l, ok := plugins.Outputs[1].(*Limiter); ok {
 		if _, ok := l.plugin.(*HTTPOutput); !ok {
 			t.Errorf("HTTPOutput should be wrapped in limiter")
 		}

pro.go

@@ -0,0 +1,5 @@
+package main
+
+// PRO this value indicates if goreplay is running in PRO mode..
+// it must not be modified explicitly in production
+var PRO = false

proto/proto.go

@@ -17,215 +17,125 @@ Example of HTTP payload for future references, new line symbols escaped:
 package proto
 
 import (
+	"bufio"
 	"bytes"
+	"net/http"
+	"net/textproto"
+	"strings"
 
 	"github.com/buger/goreplay/byteutils"
 )
 
-// In HTTP newline defined by 2 bytes (for both windows and *nix support)
-var CLRF = []byte("\r\n")
+// CRLF In HTTP newline defined by 2 bytes (for both windows and *nix support)
+var CRLF = []byte("\r\n")
 
-// New line acts as separator: end of Headers or Body (in some cases)
+// EmptyLine acts as separator: end of Headers or Body (in some cases)
 var EmptyLine = []byte("\r\n\r\n")
 
-// Separator for Header line. Header looks like: `HeaderName: value`
+// HeaderDelim Separator for Header line. Header looks like: `HeaderName: value`
 var HeaderDelim = []byte(": ")
 
 // MIMEHeadersEndPos finds end of the Headers section, which should end with empty line.
 func MIMEHeadersEndPos(payload []byte) int {
-	return bytes.Index(payload, EmptyLine) + 4
+	pos := bytes.Index(payload, EmptyLine)
+	if pos < 0 {
+		return -1
+	}
+	return pos + 4
 }
 
 // MIMEHeadersStartPos finds start of Headers section
 // It just finds position of second line (first contains location and method).
 func MIMEHeadersStartPos(payload []byte) int {
-	return bytes.Index(payload, CLRF) + 2 // Find first line end
-}
-
-func headerIndex(payload []byte, name []byte) int {
-	i := 0
-	for {
-		// we need enough space for at least '\n' and the header name
-		if i >= (len(payload) - len(name) - 1) {
-			return -1
-		}
-
-		if payload[i] == '\n' {
-			i++
-			if bytes.EqualFold(name, payload[i:i+len(name)]) {
-				return i
-			}
-		}
-		i++
-
+	pos := bytes.Index(payload, CRLF)
+	if pos < 0 {
+		return -1
 	}
-	return -1
+	return pos + 2 // Find first line end
 }
 
 // header return value and positions of header/value start/end.
 // If not found, value will be blank, and headerStart will be -1
 // Do not support multi-line headers.
 func header(payload []byte, name []byte) (value []byte, headerStart, headerEnd, valueStart, valueEnd int) {
-	headerStart = headerIndex(payload, name)
-
-	if headerStart == -1 {
+	headerStart = MIMEHeadersStartPos(payload)
+	if headerStart < 0 {
+		return
+	}
+	var colonIndex int
+	for headerStart < len(payload) {
+		headerEnd = bytes.IndexByte(payload[headerStart:], '\n')
+		if headerEnd == -1 {
+			break
+		}
+		headerEnd += headerStart
+		colonIndex = bytes.IndexByte(payload[headerStart:headerEnd], ':')
+		if colonIndex == -1 {
+			break
+		}
+		colonIndex += headerStart
+		if bytes.EqualFold(payload[headerStart:colonIndex], name) {
+			valueStart = colonIndex + 1
+			valueEnd = headerEnd - 2
+			break
+		}
+		headerStart = headerEnd + 1 // move to the next header
+	}
+	if valueStart == 0 {
+		headerStart = -1
+		headerEnd = -1
+		valueEnd = -1
+		valueStart = -1
 		return
 	}
 
-	valueStart = headerStart + len(name) + 1 // Skip ":" after header name
-	headerEnd = valueStart + bytes.IndexByte(payload[valueStart:], '\n')
-
-	for valueStart < headerEnd { // Ignore empty space after ':'
-		if payload[valueStart] == ' ' {
+	// ignore empty space after ':'
+	for valueStart < valueEnd {
+		if payload[valueStart] < 0x21 {
 			valueStart++
 		} else {
 			break
 		}
 	}
 
-	valueEnd = valueStart + bytes.IndexByte(payload[valueStart:], '\n')
-
-	if payload[headerEnd-1] == '\r' {
-		valueEnd--
-	}
-
 	// ignore empty space at end of header value
-	for valueStart < valueEnd {
-		if payload[valueEnd-1] == ' ' {
+	for valueEnd > valueStart {
+		if payload[valueEnd] < 0x21 {
 			valueEnd--
 		} else {
 			break
 		}
 	}
-	value = payload[valueStart:valueEnd]
+	value = payload[valueStart : valueEnd+1]
 
 	return
 }
 
-// Works only with ASCII
-func HeadersEqual(h1 []byte, h2 []byte) bool {
-	if len(h1) != len(h2) {
-		return false
+// ParseHeaders Parsing headers from multiple payloads
+func ParseHeaders(payloads [][]byte, cb func(header []byte, value []byte)) {
+	p := bytes.Join(payloads, nil)
+	// trimming off the title of the request
+	if HasRequestTitle(p) || HasResponseTitle(p) {
+		headerStart := MIMEHeadersStartPos(p)
+		if headerStart > len(p)-1 {
+			return
+		}
+		p = p[headerStart:]
 	}
-
-	for i, c1 := range h1 {
-		c2 := h2[i]
-
-		switch int(c1) - int(c2) {
-		case 0, 32, -32:
-		default:
-			return false
+	headerEnd := MIMEHeadersEndPos(p)
+	if headerEnd > 1 {
+		p = p[:headerEnd]
+	}
+	reader := textproto.NewReader(bufio.NewReader(bytes.NewBuffer(p)))
+	mime, err := reader.ReadMIMEHeader()
+	if err != nil {
+		return
+	}
+	for k, v := range mime {
+		for _, value := range v {
+			cb([]byte(k), []byte(value))
 		}
 	}
-
-	return true
-}
-
-// Parsing headers from multiple payloads
-func ParseHeaders(payloads [][]byte, cb func(header []byte, value []byte) bool) {
-	hS := [2]int{0, 0}   // header start
-	hE := [2]int{-1, -1} // header end
-	vS := [2]int{-1, -1} // value start
-	vE := [2]int{-1, -1} // value end
-
-	i := 0
-	pIdx := 0
-	lineBreaks := 0
-	newLineBreak := true
-
-	for {
-		if len(payloads)-1 < pIdx {
-			break
-		}
-
-		p := payloads[pIdx]
-
-		if len(p)-1 < i {
-			pIdx++
-			i = 0
-			continue
-		}
-
-		switch p[i] {
-		case '\r', '\n':
-			newLineBreak = true
-			lineBreaks++
-
-			// End of headers
-			if lineBreaks == 4 {
-				return
-			}
-
-			if lineBreaks > 1 {
-				break
-			}
-
-			vE = [2]int{pIdx, i}
-
-			if vS[1] != -1 && vE[1] != -1 &&
-				hS[1] != -1 && hE[1] != -1 {
-
-				var header, value []byte
-
-				phS, phE, pvS, pvE := payloads[hS[0]], payloads[hE[0]], payloads[vS[0]], payloads[vE[0]]
-
-				// If in same payload
-				if hS[0] == hE[0] {
-					header = phS[hS[1]:hE[1]]
-				} else {
-					header = make([]byte, len(phS)-hS[1]+hE[1])
-					copy(header, phS[hS[1]:])
-					copy(header[len(phS)-hS[1]:], phE[:hE[1]])
-				}
-
-				if vS[0] == vE[0] {
-					value = pvS[vS[1]:vE[1]]
-				} else {
-					value = make([]byte, len(pvS)-vS[1]+vE[1])
-					copy(value, pvS[vS[1]:])
-					copy(value[len(pvS)-vS[1]:], pvE[:vE[1]])
-				}
-
-				if !cb(header, value) {
-					return
-				}
-			}
-
-			// Header found, reset values
-			vS = [2]int{-1, -1}
-			vE = [2]int{-1, -1}
-			hS = [2]int{-1, -1}
-			hE = [2]int{-1, -1}
-		case ':':
-			if newLineBreak {
-				hE = [2]int{pIdx, i}
-				newLineBreak = false
-			}
-			lineBreaks = 0
-		default:
-			lineBreaks = 0
-
-			if hS[1] == -1 {
-				hS = [2]int{pIdx, i}
-				hE = [2]int{-1, -1}
-			} else {
-				if hE[1] == -1 {
-					break
-				}
-
-				if vS[1] == -1 {
-					if p[i] == ' ' {
-						break
-					}
-
-					vS = [2]int{pIdx, i}
-				}
-			}
-		}
-
-		i++
-	}
-
 	return
 }
 
@@ -243,7 +153,7 @@ func SetHeader(payload, name, value []byte) []byte {
 
 	if hs != -1 {
 		// If header found we just replace its value
-		return byteutils.Replace(payload, vs, ve, value)
+		return byteutils.Replace(payload, vs, ve+1, value)
 	}
 
 	return AddHeader(payload, name, value)
@@ -252,67 +162,54 @@ func SetHeader(payload, name, value []byte) []byte {
 // AddHeader takes http payload and appends new header to the start of headers section
 // Returns modified request payload
 func AddHeader(payload, name, value []byte) []byte {
+	mimeStart := MIMEHeadersStartPos(payload)
+	if mimeStart < 1 {
+		return payload
+	}
 	header := make([]byte, len(name)+2+len(value)+2)
 	copy(header[0:], name)
 	copy(header[len(name):], HeaderDelim)
 	copy(header[len(name)+2:], value)
-	copy(header[len(header)-2:], CLRF)
-
-	mimeStart := MIMEHeadersStartPos(payload)
+	copy(header[len(header)-2:], CRLF)
 
 	return byteutils.Insert(payload, mimeStart, header)
 }
 
-// DelHeader takes http payload and removes header name from headers section
+// DeleteHeader takes http payload and removes header name from headers section
 // Returns modified request payload
 func DeleteHeader(payload, name []byte) []byte {
 	_, hs, he, _, _ := header(payload, name)
 	if hs != -1 {
-		newHeader := make([]byte, len(payload)-(he-hs)-1)
-		copy(newHeader[:hs], payload[:hs])
-		copy(newHeader[hs:], payload[he+1:])
-		return newHeader
+		return byteutils.Cut(payload, hs, he+1)
 	}
 	return payload
 }
 
 // Body returns request/response body
 func Body(payload []byte) []byte {
-	// 4 -> len(EMPTY_LINE)
-	if len(payload) < 4 {
-		return []byte{}
+	pos := MIMEHeadersEndPos(payload)
+	if pos == -1 {
+		return nil
 	}
-
-	return payload[MIMEHeadersEndPos(payload):]
+	return payload[pos:]
 }
 
 // Path takes payload and retuns request path: Split(firstLine, ' ')[1]
 func Path(payload []byte) []byte {
+	if !HasTitle(payload) {
+		return nil
+	}
 	start := bytes.IndexByte(payload, ' ') + 1
-	eol := bytes.IndexByte(payload[start:], '\r')
 	end := bytes.IndexByte(payload[start:], ' ')
 
-	if eol > 0 {
-		if end == -1 || eol < end {
-			return payload[start : start+eol]
-		}
-	} else { // support for legacy clients
-		eol = bytes.IndexByte(payload[start:], '\n')
-
-		if eol > 0 && (end == -1 || eol < end) {
-			return payload[start : start+eol]
-		}
-	}
-
-	if end < 0 {
-		return payload[start:len(payload)]
-	}
-
 	return payload[start : start+end]
 }
 
 // SetPath takes payload, sets new path and returns modified payload
 func SetPath(payload, path []byte) []byte {
+	if !HasTitle(payload) {
+		return nil
+	}
 	start := bytes.IndexByte(payload, ' ') + 1
 	end := bytes.IndexByte(payload[start:], ' ')
 
@@ -323,21 +220,23 @@ func SetPath(payload, path []byte) []byte {
 func PathParam(payload, name []byte) (value []byte, valueStart, valueEnd int) {
 	path := Path(payload)
 
-	if paramStart := bytes.Index(path, append(name, '=')); paramStart != -1 {
-		valueStart := paramStart + len(name) + 1
-		paramEnd := bytes.IndexByte(path[valueStart:], '&')
-
-		// Param can end with '&' (another param), or end of line
-		if paramEnd == -1 { // It is final param
-			paramEnd = len(path)
-		} else {
-			paramEnd += valueStart
+	paramStart := -1
+	if paramStart = bytes.Index(path, append([]byte{'&'}, append(name, '=')...)); paramStart == -1 {
+		if paramStart = bytes.Index(path, append([]byte{'?'}, append(name, '=')...)); paramStart == -1 {
+			return []byte(""), -1, -1
 		}
-
-		return path[valueStart:paramEnd], valueStart, paramEnd
 	}
 
-	return []byte(""), -1, -1
+	valueStart = paramStart + len(name) + 2
+	paramEnd := bytes.IndexByte(path[valueStart:], '&')
+
+	// Param can end with '&' (another param), or end of line
+	if paramEnd == -1 { // It is final param
+		paramEnd = len(path)
+	} else {
+		paramEnd += valueStart
+	}
+	return path[valueStart:paramEnd], valueStart, paramEnd
 }
 
 // SetPathParam takes payload and updates path Query attribute
@@ -401,31 +300,221 @@ func SetHost(payload, url, host []byte) []byte {
 // Method returns HTTP method
 func Method(payload []byte) []byte {
 	end := bytes.IndexByte(payload, ' ')
+	if end == -1 {
+		return nil
+	}
 
 	return payload[:end]
 }
 
 // Status returns response status.
-// It happend to be in same position as request payload path
+// It happens to be in same position as request payload path
 func Status(payload []byte) []byte {
 	return Path(payload)
 }
 
-var httpMethods []string = []string{
-	"GET ", "OPTI", "HEAD", "POST", "PUT ", "DELE", "TRAC", "CONN", "PATC" /* custom methods */, "BAN ", "PURG", "PROP", "MKCO", "COPY", "MOVE", "LOCK", "UNLO",
+// Methods holds the http methods ordered in ascending order
+var Methods = [...]string{
+	http.MethodConnect, http.MethodDelete, http.MethodGet,
+	http.MethodHead, http.MethodOptions, http.MethodPatch,
+	http.MethodPost, http.MethodPut, http.MethodTrace,
 }
 
-func IsHTTPPayload(payload []byte) bool {
-	if len(payload) < 4 {
+const (
+	//MinRequestCount GET / HTTP/1.1\r\n
+	MinRequestCount = 16
+	// MinResponseCount HTTP/1.1 200 OK\r\n
+	MinResponseCount = 17
+	// VersionLen HTTP/1.1
+	VersionLen = 8
+)
+
+// HasResponseTitle reports whether this payload has an HTTP/1 response title
+func HasResponseTitle(payload []byte) bool {
+	s := byteutils.SliceToString(payload)
+	if len(s) < MinResponseCount {
 		return false
 	}
+	titleLen := bytes.Index(payload, CRLF)
+	if titleLen == -1 {
+		return false
+	}
+	major, minor, ok := http.ParseHTTPVersion(s[0:VersionLen])
+	if !(ok && major == 1 && (minor == 0 || minor == 1)) {
+		return false
+	}
+	if s[VersionLen] != ' ' {
+		return false
+	}
+	status, ok := atoI(payload[VersionLen+1:VersionLen+4], 10)
+	if !ok || s[VersionLen+4] != ' ' {
+		return false
+	}
+	statusText := http.StatusText(status)
+	if statusText == "" {
+		return false
+	}
+	if titleLen+len(CRLF) > len(s) {
+		return false
+	}
+	return s[VersionLen+5:titleLen] == statusText
+}
 
-	method := string(payload[0:4])
-
-	for _, m := range httpMethods {
-		if method == m {
-			return true
+// HasRequestTitle reports whether this payload has an HTTP/1 request title
+func HasRequestTitle(payload []byte) bool {
+	s := byteutils.SliceToString(payload)
+	if len(s) < MinRequestCount {
+		return false
+	}
+	titleLen := bytes.Index(payload, CRLF)
+	if titleLen == -1 {
+		return false
+	}
+	if strings.Count(s[:titleLen], " ") != 2 {
+		return false
+	}
+	method := string(Method(payload))
+	var methodFound bool
+	for _, m := range Methods {
+		if methodFound = method == m; methodFound {
+			break
 		}
 	}
-	return false
+	if !methodFound {
+		return false
+	}
+	path := strings.Index(s[len(method)+1:], " ")
+	if path == -1 {
+		return false
+	}
+	major, minor, ok := http.ParseHTTPVersion(s[path+len(method)+2 : titleLen])
+	return ok && major == 1 && (minor == 0 || minor == 1)
+}
+
+// HasTitle reports if this payload has an http/1 title
+func HasTitle(payload []byte) bool {
+	return HasRequestTitle(payload) || HasResponseTitle(payload)
+}
+
+// CheckChunked checks HTTP/1 chunked data integrity and return the final index
+// of chunks(index after '0\r\n\r\n') or -1 if there is missing data
+// or there is bad format
+func CheckChunked(buf []byte) (chunkEnd int) {
+	var (
+		ok     bool
+		chkLen int
+		sz     int
+		ext    int
+	)
+	for {
+		sz = bytes.IndexByte(buf[chunkEnd:], '\r')
+		if sz < 1 {
+			return -1
+		}
+		// ignoring chunks extensions https://github.com/golang/go/issues/13135
+		// but chunks extensions are no longer a thing
+		ext = bytes.IndexByte(buf[chunkEnd:chunkEnd+sz], ';')
+		if ext < 0 {
+			ext = sz
+		}
+		chkLen, ok = atoI(buf[chunkEnd:chunkEnd+ext], 16)
+		if !ok {
+			return -1
+		}
+		chunkEnd += (sz + 2)
+		if chkLen == 0 {
+			if !bytes.Equal(buf[chunkEnd:chunkEnd+2], CRLF) {
+				return -1
+			}
+			return chunkEnd + 2
+		}
+		// ideally chunck length and at least len("\r\n0\r\n\r\n")
+		if len(buf[chunkEnd:]) < chkLen+7 {
+			return -1
+		}
+		chunkEnd += chkLen
+		// chunks must end with CRLF
+		if !bytes.Equal(buf[chunkEnd:chunkEnd+2], CRLF) {
+			return -1
+		}
+		chunkEnd += 2
+	}
+}
+
+// HasFullPayload reports if this http has full payloads
+func HasFullPayload(payload []byte) bool {
+	body := Body(payload)
+
+	// check for chunked transfer-encoding
+	header := Header(payload, []byte("Transfer-Encoding"))
+	if bytes.Contains(header, []byte("chunked")) {
+
+		// check chunks
+		if len(body) < 1 {
+			return false
+		}
+		var chunkEnd int
+		if chunkEnd = CheckChunked(body); chunkEnd < 1 {
+			return false
+		}
+
+		// check trailer headers
+		if len(Header(payload, []byte("Trailer"))) < 1 {
+			return true
+		}
+		// trailer headers(whether chunked or plain) should end with empty line
+		return len(body) > chunkEnd && MIMEHeadersEndPos(body[chunkEnd:]) != -1
+	}
+
+	// check for content-length header
+	// trailers are generally not allowed in non-chunks body
+	header = Header(payload, []byte("Content-Length"))
+	if len(header) > 1 {
+		num, ok := atoI(header, 10)
+		return ok && num == len(body)
+	}
+
+	// for empty body, check for emptyline
+	return MIMEHeadersEndPos(payload) != -1
+}
+
+// this works with positive integers
+func atoI(s []byte, base int) (num int, ok bool) {
+	var v int
+	for i := 0; i < len(s); i++ {
+		if s[i] > 127 {
+			return 0, false
+		}
+		v = int(hexTable[s[i]])
+		if v >= base || (v == 0 && s[i] != '0') {
+			return 0, false
+		}
+		num = (num * base) + v
+	}
+	return num, true
+}
+
+var hexTable = [128]byte{
+	'0': 0,
+	'1': 1,
+	'2': 2,
+	'3': 3,
+	'4': 4,
+	'5': 5,
+	'6': 6,
+	'7': 7,
+	'8': 8,
+	'9': 9,
+	'A': 10,
+	'a': 10,
+	'B': 11,
+	'b': 11,
+	'C': 12,
+	'c': 12,
+	'D': 13,
+	'd': 13,
+	'E': 14,
+	'e': 14,
+	'F': 15,
+	'f': 15,
 }

proto/proto_test.go

@@ -38,13 +38,6 @@ func TestHeader(t *testing.T) {
 		t.Error("Should return empty value")
 	}
 
-	// Wrong delimeter
-	payload = []byte("GET /p HTTP/1.1\r\nCookie: 123\nHost: www.w3.org\r\n\r\n")
-
-	if val = Header(payload, []byte("Cookie")); !bytes.Equal(val, []byte("123")) {
-		t.Error("Should handle wrong header delimeter")
-	}
-
 	// Header not found
 	if _, headerStart, _, _, _ = header(payload, []byte("Not-Found")); headerStart != -1 {
 		t.Error("Should not found header")
@@ -103,6 +96,10 @@ func TestSetHeader(t *testing.T) {
 	if payload = SetHeader(payload, []byte("User-Agent"), []byte("Gor")); !bytes.Equal(payload, payloadAfter) {
 		t.Error("Should add header if not found", string(payload))
 	}
+	invalidPayload := []byte("POST /post HTTP/1.1")
+	if invalidPayload = SetHeader(invalidPayload, []byte("User-Agent"), []byte("Gor")); !bytes.Equal(invalidPayload, []byte("POST /post HTTP/1.1")) {
+		t.Error("Should not modify payload if request is invalid", string(payload))
+	}
 }
 
 func TestDeleteHeader(t *testing.T) {
@@ -129,9 +126,8 @@ func TestParseHeaders(t *testing.T) {
 
 	headers := make(map[string]string)
 
-	ParseHeaders(payload, func(header []byte, value []byte) bool {
+	ParseHeaders(payload, func(header []byte, value []byte) {
 		headers[string(header)] = string(value)
-		return true
 	})
 
 	expected := map[string]string{
@@ -152,8 +148,7 @@ func TestFuzzCrashers(t *testing.T) {
 	}
 
 	for _, f := range crashers {
-		ParseHeaders([][]byte{[]byte(f)}, func(header []byte, value []byte) bool {
-			return true
+		ParseHeaders([][]byte{[]byte(f)}, func(header []byte, value []byte) {
 		})
 	}
 }
@@ -165,9 +160,8 @@ func TestParseHeadersWithComplexUserAgent(t *testing.T) {
 
 	headers := make(map[string]string)
 
-	ParseHeaders(payload, func(header []byte, value []byte) bool {
+	ParseHeaders(payload, func(header []byte, value []byte) {
 		headers[string(header)] = string(value)
-		return true
 	})
 
 	expected := map[string]string{
@@ -186,9 +180,8 @@ func TestParseHeadersWithOrigin(t *testing.T) {
 
 	headers := make(map[string]string)
 
-	ParseHeaders(payload, func(header []byte, value []byte) bool {
+	ParseHeaders(payload, func(header []byte, value []byte) {
 		headers[string(header)] = string(value)
-		return true
 	})
 
 	expected := map[string]string{
@@ -210,25 +203,6 @@ func TestParseHeadersWithOrigin(t *testing.T) {
 	}
 }
 
-func TestHeaderEquals(t *testing.T) {
-	tests := []struct {
-		h1     string
-		h2     string
-		equals bool
-	}{
-		{"Content-Length", "content-length", true},
-		{"content-length", "Content-Length", true},
-		{"content-Pength", "Content-Length", false},
-		{"Host", "Content-Length", false},
-	}
-
-	for _, tc := range tests {
-		if HeadersEqual([]byte(tc.h1), []byte(tc.h2)) != tc.equals {
-			t.Error(tc)
-		}
-	}
-}
-
 func TestPath(t *testing.T) {
 	var path, payload []byte
 
@@ -240,20 +214,20 @@ func TestPath(t *testing.T) {
 
 	payload = []byte("GET /get\r\n\r\nHost: www.w3.org\r\n\r\n")
 
-	if path = Path(payload); !bytes.Equal(path, []byte("/get")) {
-		t.Error("Should find path", string(path))
+	if path = Path(payload); !bytes.Equal(path, nil) {
+		t.Error("1Should not find path", string(path))
 	}
 
 	payload = []byte("GET /get\n")
 
-	if path = Path(payload); !bytes.Equal(path, []byte("/get")) {
-		t.Error("Should find path", string(path))
+	if path = Path(payload); !bytes.Equal(path, nil) {
+		t.Error("2Should not find path", string(path))
 	}
 
 	payload = []byte("GET /get")
 
-	if path = Path(payload); !bytes.Equal(path, []byte("/get")) {
-		t.Error("Should find path", string(path))
+	if path = Path(payload); !bytes.Equal(path, nil) {
+		t.Error("3Should not find path", string(path))
 	}
 }
 
@@ -266,12 +240,13 @@ func TestSetPath(t *testing.T) {
 	if payload = SetPath(payload, []byte("/new_path")); !bytes.Equal(payload, payloadAfter) {
 		t.Error("Should replace path", string(payload))
 	}
+
 }
 
 func TestPathParam(t *testing.T) {
 	var payload []byte
 
-	payload = []byte("POST /post?param=test&user_id=1 HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
+	payload = []byte("POST /post?param=test&user_id=1&d_type=1&type=2&d_type=3 HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
 
 	if val, _, _ := PathParam(payload, []byte("param")); !bytes.Equal(val, []byte("test")) {
 		t.Error("Should detect attribute", string(val))
@@ -280,6 +255,15 @@ func TestPathParam(t *testing.T) {
 	if val, _, _ := PathParam(payload, []byte("user_id")); !bytes.Equal(val, []byte("1")) {
 		t.Error("Should detect attribute", string(val))
 	}
+
+	if val, _, _ := PathParam(payload, []byte("type")); !bytes.Equal(val, []byte("2")) {
+		t.Error("Should detect attribute", string(val))
+	}
+
+	if val, _, _ := PathParam(payload, []byte("d_type")); !bytes.Equal(val, []byte("1")) {
+		// this function is not designed for cases with duplicate param keys
+		t.Error("Should detect attribute", string(val))
+	}
 }
 
 func TestSetPathParam(t *testing.T) {
@@ -323,4 +307,172 @@ func TestSetHostHTTP10(t *testing.T) {
 	if payload = SetHost(payload, []byte("http://new.com"), []byte("new.com")); !bytes.Equal(payload, payloadAfter) {
 		t.Error("Should replace host", string(payload))
 	}
+
+	payload = []byte("POST /post HTTP/1.0\r\nContent-Length: 7\r\nHost: example.com\r\n\r\na=1&b=2")
+	payloadAfter = []byte("POST /post HTTP/1.0\r\nContent-Length: 7\r\nHost: new.com\r\n\r\na=1&b=2")
+
+	if payload = SetHost(payload, nil, []byte("new.com")); !bytes.Equal(payload, payloadAfter) {
+		t.Error("Should replace host", string(payload))
+	}
+
+	payload = []byte("POST /post HTTP/1.0\r\nContent-Length: 7\r\n\r\na=1&b=2")
+
+	if payload = SetHost(payload, nil, []byte("new.com")); !bytes.Equal(payload, payload) {
+		t.Error("Should replace host", string(payload))
+	}
+}
+
+func TestHasResponseTitle(t *testing.T) {
+	var m = map[string]bool{
+		"HTTP":                      false,
+		"":                          false,
+		"HTTP/1.1 100 Continue":     false,
+		"HTTP/1.1 100 Continue\r\n": true,
+		"HTTP/1.1  \r\n":            false,
+		"HTTP/4.0 100Continue\r\n":  false,
+		"HTTP/1.0 10r Continue\r\n": false,
+		"HTTP/1.1 200\r\n":          false,
+		"HTTP/1.1 200\r\nServer: Tengine\r\nContent-Length: 0\r\nConnection: close\r\n\r\n": false,
+	}
+	for k, v := range m {
+		if HasResponseTitle([]byte(k)) != v {
+			t.Errorf("%q should yield %v", k, v)
+			break
+		}
+	}
+}
+
+func TestHasRequestTitle(t *testing.T) {
+	var m = map[string]bool{
+		"POST /post HTTP/1.0\r\n": true,
+		"":                        false,
+		"POST /post HTTP/1.\r\n":  false,
+		"POS /post HTTP/1.1\r\n":  false,
+		"GET / HTTP/1.1\r\n":      true,
+		"GET / HTTP/1.1\r":        false,
+		"GET / HTTP/1.400\r\n":    false,
+	}
+	for k, v := range m {
+		if HasRequestTitle([]byte(k)) != v {
+			t.Errorf("%q should yield %v", k, v)
+			break
+		}
+	}
+}
+
+func TestCheckChunks(t *testing.T) {
+	var m = "4\r\nWiki\r\n5\r\npedia\r\nE\r\n in\r\n\r\nchunks.\r\n0\r\n\r\n"
+	chunkEnd := CheckChunked([]byte(m))
+	expected := bytes.Index([]byte(m), []byte("0\r\n")) + 5
+	if chunkEnd != expected {
+		t.Errorf("expected %d to equal %d", chunkEnd, expected)
+	}
+
+	m = "7\r\nMozia\r\n9\r\nDeveloper\r\n7\r\nNetwork\r\n0\r\n\r\n"
+	chunkEnd = CheckChunked([]byte(m))
+	if chunkEnd != -1 {
+		t.Errorf("expected %d to equal %d", chunkEnd, -1)
+	}
+
+	// with trailers
+	m = "4\r\nWiki\r\n5\r\npedia\r\nE\r\n in\r\n\r\nchunks.\r\n0\r\n\r\nEXpires"
+	chunkEnd = CheckChunked([]byte(m))
+	expected = bytes.Index([]byte(m), []byte("0\r\n")) + 5
+	if chunkEnd != expected {
+		t.Errorf("expected %d to equal %d", chunkEnd, expected)
+	}
+
+	// last chunk inside the the body
+	// with trailers
+	m = "4\r\nWiki\r\n5\r\npedia\r\nE\r\n in\r\n\r\nchunks.\r\n3\r\n0\r\n\r\n0\r\n\r\nEXpires"
+	chunkEnd = CheckChunked([]byte(m))
+	expected = bytes.Index([]byte(m), []byte("0\r\n")) + 10
+	if chunkEnd != expected {
+		t.Errorf("expected %d to equal %d", chunkEnd, expected)
+	}
+
+	// checks with chucks-extensions
+	m = "4\r\nWiki\r\n5\r\npedia\r\nE; name='quoted string'\r\n in\r\n\r\nchunks.\r\n3\r\n0\r\n\r\n0\r\n\r\nEXpires"
+	chunkEnd = CheckChunked([]byte(m))
+	expected = bytes.Index([]byte(m), []byte("0\r\n")) + 10
+	if chunkEnd != expected {
+		t.Errorf("expected %d to equal %d", chunkEnd, expected)
+	}
+}
+
+func TestHasFullPayload(t *testing.T) {
+	var m = "HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nTransfer-Encoding: chunked\r\n\r\n7\r\nMozilla\r\n9\r\nDeveloper\r\n7\r\nNetwork\r\n0\r\n\r\n"
+	got := HasFullPayload([]byte(m))
+	expected := true
+	if got != expected {
+		t.Errorf("expected %v to equal %v", got, expected)
+	}
+
+	// check with invalid chunk format
+	m = "HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nTransfer-Encoding: chunked\r\n\r\n7\r\nMozia\r\n9\r\nDeveloper\r\n7\r\nNetwork\r\n0\r\n\r\n"
+	got = HasFullPayload([]byte(m))
+	expected = false
+	if got != expected {
+		t.Errorf("expected %v to equal %v", got, expected)
+	}
+
+	// check chunks with trailers
+	m = "HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nTransfer-Encoding: chunked\r\nTrailer: Expires\r\n\r\n7\r\nMozilla\r\n9\r\nDeveloper\r\n7\r\nNetwork\r\n0\r\n\r\nExpires: Wed, 21 Oct 2015 07:28:00 GMT\r\n\r\n"
+	got = HasFullPayload([]byte(m))
+	expected = true
+	if got != expected {
+		t.Errorf("expected %v to equal %v", got, expected)
+	}
+
+	// check with missing trailers
+	m = "HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nTransfer-Encoding: chunked\r\nTrailer: Expires\r\n\r\n7\r\nMozilla\r\n9\r\nDeveloper\r\n7\r\nNetwork\r\n0\r\n\r\nExpires: Wed, 21 Oct 2015 07:28:00"
+	got = HasFullPayload([]byte(m))
+	expected = false
+	if got != expected {
+		t.Errorf("expected %v to equal %v", got, expected)
+	}
+
+	// check with content-length
+	m = "HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nContent-Length: 23\r\n\r\nMozillaDeveloperNetwork"
+	got = HasFullPayload([]byte(m))
+	expected = true
+	if got != expected {
+		t.Errorf("expected %v to equal %v", got, expected)
+	}
+
+	// check missing total length
+	m = "HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nContent-Length: 23\r\n\r\nMozillaDeveloperNet"
+	got = HasFullPayload([]byte(m))
+	expected = false
+	if got != expected {
+		t.Errorf("expected %v to equal %v", got, expected)
+	}
+
+	// check with no body
+	m = "HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\n\r\n"
+	got = HasFullPayload([]byte(m))
+	expected = true
+	if got != expected {
+		t.Errorf("expected %v to equal %v", got, expected)
+	}
+}
+
+func BenchmarkHasFullPayload(b *testing.B) {
+	var buf bytes.Buffer
+	buf.Write([]byte("HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nTransfer-Encoding: chunked\r\n\r\n"))
+	var chunk = []byte("1e\r\n111111111111111111111111111111\r\n")
+	for i := 0; i < 5000; i++ {
+		buf.Write(chunk)
+	}
+	buf.Write([]byte("0\r\n\r\n"))
+	data := buf.Bytes()
+	var ok bool
+	b.ResetTimer() // ignores the upper initialization
+	b.ReportMetric(float64(5000), "chunks/op")
+	for i := 0; i < b.N; i++ {
+		ok = HasFullPayload(data)
+	}
+	if !ok {
+		b.Fail()
+	}
 }

protocol.go

@@ -4,23 +4,28 @@ import (
 	"bytes"
 	"crypto/rand"
 	"encoding/hex"
-	"strconv"
+	"fmt"
 )
 
+// These constants help to indicate the type of payload
 const (
 	RequestPayload          = '1'
 	ResponsePayload         = '2'
 	ReplayedResponsePayload = '3'
 )
 
-func uuid() []byte {
-	b := make([]byte, 20)
+func randByte(len int) []byte {
+	b := make([]byte, len/2)
 	rand.Read(b)
 
-	uuid := make([]byte, 40)
-	hex.Encode(uuid, b)
+	h := make([]byte, len)
+	hex.Encode(h, b)
 
-	return uuid
+	return h
+}
+
+func uuid() []byte {
+	return randByte(24)
 }
 
 var payloadSeparator = "\n🐵🙈🙉\n"
@@ -43,37 +48,9 @@ func payloadScanner(data []byte, atEOF bool) (advance int, token []byte, err err
 
 // Timing is request start or round-trip time, depending on payloadType
 func payloadHeader(payloadType byte, uuid []byte, timing int64, latency int64) (header []byte) {
-	var sTime, sLatency string
-
-	sTime = strconv.FormatInt(timing, 10)
-	if latency != -1 {
-		sLatency = strconv.FormatInt(latency, 10)
-	}
-
 	//Example:
-	//  3 f45590522cd1838b4a0d5c5aab80b77929dea3b3 1231\n
-	// `+ 1` indicates space characters or end of line
-	headerLen := 1 + 1 + len(uuid) + 1 + len(sTime) + 1
-
-	if latency != -1 {
-		headerLen += len(sLatency) + 1
-	}
-
-	header = make([]byte, headerLen)
-	header[0] = payloadType
-	header[1] = ' '
-	header[2+len(uuid)] = ' '
-	header[len(header)-1] = '\n'
-
-	copy(header[2:], uuid)
-	copy(header[3+len(uuid):], sTime)
-
-	if latency != -1 {
-		header[3+len(uuid)+len(sTime)] = ' '
-		copy(header[4+len(uuid)+len(sTime):], sLatency)
-	}
-
-	return header
+	//  3 f45590522cd1838b4a0d5c5aab80b77929dea3b3 13923489726487326 1231\n
+	return []byte(fmt.Sprintf("%c %s %d %d\n", payloadType, uuid, timing, latency))
 }
 
 func payloadBody(payload []byte) []byte {
@@ -84,11 +61,23 @@ func payloadBody(payload []byte) []byte {
 func payloadMeta(payload []byte) [][]byte {
 	headerSize := bytes.IndexByte(payload, '\n')
 	if headerSize < 0 {
-		headerSize = 0
+		return nil
 	}
 	return bytes.Split(payload[:headerSize], []byte{' '})
 }
 
+func payloadID(payload []byte) (id []byte) {
+	meta := payloadMeta(payload)
+
+	if len(meta) < 2 {
+		return
+	}
+	// id is encoded in hex, we need to revert to how it was
+	id = make([]byte, 20)
+	hex.Decode(id, meta[1])
+	return
+}
+
 func isOriginPayload(payload []byte) bool {
 	switch payload[0] {
 	case RequestPayload, ResponsePayload:

raw_socket_listener/listener.go

@@ -1,908 +0,0 @@
-/*
-Package rawSocket provides traffic sniffier using RAW sockets.
-
-Capture traffic from socket using RAW_SOCKET's
-http://en.wikipedia.org/wiki/Raw_socket
-
-RAW_SOCKET allow you listen for traffic on any port (e.g. sniffing) because they operate on IP level.
-
-Ports is TCP feature, same as flow control, reliable transmission and etc.
-
-This package implements own TCP layer: TCP packets is parsed using tcp_packet.go, and flow control is managed by tcp_message.go
-*/
-package rawSocket
-
-import (
-	"bytes"
-	"encoding/binary"
-	"fmt"
-	"io"
-	"log"
-	"net"
-	"runtime"
-	"runtime/debug"
-	"strconv"
-	"strings"
-	"sync"
-	"time"
-
-	"github.com/buger/goreplay/proto"
-
-	"github.com/google/gopacket"
-	"github.com/google/gopacket/layers"
-	"github.com/google/gopacket/pcap"
-)
-
-var _ = fmt.Println
-
-type packet struct {
-	srcIP     []byte
-	data      []byte
-	timestamp time.Time
-}
-
-// Listener handle traffic capture
-type Listener struct {
-	mu sync.Mutex
-	// buffer of TCPMessages waiting to be send
-	// ID -> TCPMessage
-	messages map[tcpID]*TCPMessage
-
-	// Expect: 100-continue request is send in 2 tcp messages
-	// We store ACK aliases to merge this packets together
-	ackAliases map[uint32]uint32
-	// To get ACK of second message we need to compute its Seq and wait for them message
-	seqWithData map[uint32]uint32
-
-	// Ack -> Req
-	respAliases map[uint32]*TCPMessage
-
-	// Ack -> ID
-	respWithoutReq map[uint32]tcpID
-
-	// Messages ready to be send to client
-	packetsChan chan *packet
-
-	// Messages ready to be send to client
-	messagesChan chan *TCPMessage
-
-	addr string // IP to listen
-	port uint16 // Port to listen
-
-	trackResponse bool
-	messageExpire time.Duration
-
-	bpfFilter     string
-	timestampType string
-	overrideSnapLen bool
-	immediateMode bool
-
-	bufferSize int
-
-	conn        net.PacketConn
-	pcapHandles []*pcap.Handle
-
-	quit    chan bool
-	readyCh chan bool
-}
-
-type request struct {
-	id    tcpID
-	start time.Time
-	ack   uint32
-}
-
-// Available engines for intercepting traffic
-const (
-	EngineRawSocket = 1 << iota
-	EnginePcap
-	EnginePcapFile
-)
-
-// NewListener creates and initializes new Listener object
-func NewListener(addr string, port string, engine int, trackResponse bool, expire time.Duration, bpfFilter string, timestampType string, bufferSize int, overrideSnapLen bool, immediateMode bool) (l *Listener) {
-	l = &Listener{}
-
-	l.packetsChan = make(chan *packet, 10000)
-	l.messagesChan = make(chan *TCPMessage, 10000)
-	l.quit = make(chan bool)
-	l.readyCh = make(chan bool, 1)
-
-	l.messages = make(map[tcpID]*TCPMessage)
-	l.ackAliases = make(map[uint32]uint32)
-	l.seqWithData = make(map[uint32]uint32)
-	l.respAliases = make(map[uint32]*TCPMessage)
-	l.respWithoutReq = make(map[uint32]tcpID)
-	l.trackResponse = trackResponse
-	l.bpfFilter = bpfFilter
-	l.timestampType = timestampType
-	l.immediateMode = immediateMode
-	l.bufferSize = bufferSize
-	l.overrideSnapLen = overrideSnapLen
-
-	l.addr = addr
-	_port, _ := strconv.Atoi(port)
-	l.port = uint16(_port)
-
-	if expire.Nanoseconds() == 0 {
-		expire = 2000 * time.Millisecond
-	}
-
-	l.messageExpire = expire
-
-	go l.listen()
-
-	// Special case for testing
-	if l.port != 0 {
-		switch engine {
-		case EnginePcap:
-			go l.readPcap()
-		case EnginePcapFile:
-			go l.readPcapFile()
-		default:
-			log.Fatal("Unknown traffic interception engine:", engine)
-		}
-	}
-
-	return
-}
-
-func (t *Listener) listen() {
-	gcTicker := time.Tick(t.messageExpire / 2)
-
-	for {
-		select {
-		case <-t.quit:
-			if t.conn != nil {
-				t.conn.Close()
-			}
-			return
-		case packet := <-t.packetsChan:
-			tcpPacket := ParseTCPPacket(packet.srcIP, packet.data, packet.timestamp)
-			t.processTCPPacket(tcpPacket)
-		case <-gcTicker:
-			now := time.Now()
-
-			// Dispatch requests before responses
-			for _, message := range t.messages {
-				if now.Sub(message.End) >= t.messageExpire {
-					t.dispatchMessage(message)
-				}
-			}
-		}
-	}
-}
-
-func (t *Listener) deleteMessage(message *TCPMessage) {
-	delete(t.messages, message.ID())
-	delete(t.ackAliases, message.Ack)
-	if message.DataAck != 0 {
-		delete(t.ackAliases, message.DataAck)
-	}
-	if message.DataSeq != 0 {
-		delete(t.seqWithData, message.DataSeq)
-	}
-
-	delete(t.respAliases, message.ResponseAck)
-}
-
-func (t *Listener) dispatchMessage(message *TCPMessage) {
-	// If already dispatched
-	if _, ok := t.messages[message.ID()]; !ok {
-		return
-	}
-
-	t.deleteMessage(message)
-
-	if !message.complete {
-		if !message.IsIncoming {
-			delete(t.respAliases, message.Ack)
-			delete(t.respWithoutReq, message.Ack)
-		}
-
-		return
-	}
-
-	if message.IsIncoming {
-		// If there were response before request
-		// log.Println("Looking for Response: ", t.respWithoutReq, message.ResponseAck)
-		if t.trackResponse {
-			if respID, ok := t.respWithoutReq[message.ResponseAck]; ok {
-				if resp, rok := t.messages[respID]; rok {
-					// if resp.AssocMessage == nil {
-					// log.Println("FOUND RESPONSE")
-					resp.setAssocMessage(message)
-					message.setAssocMessage(resp)
-
-					if resp.complete {
-						defer t.dispatchMessage(resp)
-					}
-					// }
-				}
-			}
-
-			if resp, ok := t.messages[message.ResponseID]; ok {
-				resp.setAssocMessage(message)
-			}
-		}
-	} else {
-		if message.AssocMessage == nil {
-			if responseRequest, ok := t.respAliases[message.Ack]; ok {
-				message.setAssocMessage(responseRequest)
-				responseRequest.setAssocMessage(message)
-			}
-		}
-
-		delete(t.respAliases, message.Ack)
-		delete(t.respWithoutReq, message.Ack)
-
-		// Do not track responses which have no associated requests
-		if message.AssocMessage == nil {
-			// log.Println("Can't dispatch resp", message.Seq, message.Ack, string(message.Bytes()))
-			return
-		}
-	}
-
-	t.messagesChan <- message
-}
-
-// DeviceNotFoundError raised if user specified wrong ip
-type DeviceNotFoundError struct {
-	addr string
-}
-
-func (e *DeviceNotFoundError) Error() string {
-	devices, _ := pcap.FindAllDevs()
-
-	if len(devices) == 0 {
-		return "Can't get list of network interfaces, ensure that you running Gor as root user or sudo.\nTo run as non-root users see this docs https://github.com/buger/goreplay/wiki/Running-as-non-root-user"
-	}
-
-	var msg string
-	msg += "Can't find interfaces with addr: " + e.addr + ". Provide available IP for intercepting traffic: \n"
-	for _, device := range devices {
-		msg += "Name: " + device.Name + "\n"
-		if device.Description != "" {
-			msg += "Description: " + device.Description + "\n"
-		}
-		for _, address := range device.Addresses {
-			msg += "- IP address: " + address.IP.String() + "\n"
-		}
-	}
-
-	return msg
-}
-
-func isLoopback(device pcap.Interface) bool {
-	if len(device.Addresses) == 0 {
-		return false
-	}
-
-	switch device.Addresses[0].IP.String() {
-	case "127.0.0.1", "::1":
-		return true
-	}
-
-	return false
-}
-
-func listenAllInterfaces(addr string) bool {
-	switch addr {
-	case "", "0.0.0.0", "[::]", "::":
-		return true
-	default:
-		return false
-	}
-}
-
-func findPcapDevices(addr string) (interfaces []pcap.Interface, err error) {
-	devices, err := pcap.FindAllDevs()
-	if err != nil {
-		log.Fatal(err)
-	}
-
-	for _, device := range devices {
-		if listenAllInterfaces(addr) && len(device.Addresses) > 0 || isLoopback(device) {
-			interfaces = append(interfaces, device)
-			continue
-		}
-
-		for _, address := range device.Addresses {
-			if device.Name == addr || address.IP.String() == addr {
-				interfaces = append(interfaces, device)
-				return interfaces, nil
-			}
-		}
-	}
-
-	if len(interfaces) == 0 {
-		return nil, &DeviceNotFoundError{addr}
-	}
-
-	return interfaces, nil
-}
-
-func (t *Listener) readPcap() {
-	devices, err := findPcapDevices(t.addr)
-	if err != nil {
-		log.Fatal(err)
-	}
-
-	bpfSupported := true
-	if runtime.GOOS == "darwin" {
-		bpfSupported = false
-	}
-
-	var wg sync.WaitGroup
-	wg.Add(len(devices))
-
-	for _, d := range devices {
-		go func(device pcap.Interface) {
-			inactive, err := pcap.NewInactiveHandle(device.Name)
-			if err != nil {
-				log.Println("Pcap Error while opening device", device.Name, err)
-				wg.Done()
-				return
-			}
-
-			if t.timestampType != "" {
-				if tt, terr := pcap.TimestampSourceFromString(t.timestampType); terr != nil {
-					log.Println("Supported timestamp types: ", inactive.SupportedTimestamps(), device.Name)
-				} else if terr := inactive.SetTimestampSource(tt); terr != nil {
-					log.Println("Supported timestamp types: ", inactive.SupportedTimestamps(), device.Name)
-				}
-			}
-
-			if it, err := net.InterfaceByName(device.Name); err == nil && !t.overrideSnapLen {
-				// Auto-guess max length of packet to capture
-				inactive.SetSnapLen(it.MTU + 68*2)
-			} else {
-				inactive.SetSnapLen(65536)
-			}
-
-			inactive.SetTimeout(t.messageExpire)
-			inactive.SetPromisc(true)
-			inactive.SetImmediateMode(t.immediateMode)
-			if t.immediateMode {
-				log.Println("Setting immediate mode")
-			}
-			if t.bufferSize > 0 {
-				inactive.SetBufferSize(t.bufferSize)
-			}
-
-			handle, herr := inactive.Activate()
-			if herr != nil {
-				log.Println("PCAP Activate error:", herr)
-				wg.Done()
-				return
-			}
-
-			defer handle.Close()
-
-			t.mu.Lock()
-			t.pcapHandles = append(t.pcapHandles, handle)
-
-			var bpfDstHost, bpfSrcHost string
-			var loopback = isLoopback(device)
-
-			if loopback {
-				var allAddr []string
-				for _, dc := range devices {
-					for _, addr := range dc.Addresses {
-						allAddr = append(allAddr, "(dst host "+addr.IP.String()+" and src host "+addr.IP.String()+")")
-					}
-				}
-
-				bpfDstHost = strings.Join(allAddr, " or ")
-				bpfSrcHost = bpfDstHost
-			} else {
-				for i, addr := range device.Addresses {
-					bpfDstHost += "dst host " + addr.IP.String()
-					bpfSrcHost += "src host " + addr.IP.String()
-					if i != len(device.Addresses)-1 {
-						bpfDstHost += " or "
-						bpfSrcHost += " or "
-					}
-				}
-			}
-
-			if bpfSupported {
-				var bpf string
-
-				if t.trackResponse {
-					bpf = "(tcp dst port " + strconv.Itoa(int(t.port)) + " and (" + bpfDstHost + ")) or (" + "tcp src port " + strconv.Itoa(int(t.port)) + " and (" + bpfSrcHost + "))"
-				} else {
-					bpf = "tcp dst port " + strconv.Itoa(int(t.port)) + " and (" + bpfDstHost + ")"
-				}
-
-				if t.bpfFilter != "" {
-					bpf = t.bpfFilter
-				}
-
-				if err := handle.SetBPFFilter(bpf); err != nil {
-					log.Println("BPF filter error:", err, "Device:", device.Name, bpf)
-					wg.Done()
-					return
-				}
-			}
-			t.mu.Unlock()
-
-			var decoder gopacket.Decoder
-
-			// Special case for tunnel interface https://github.com/google/gopacket/issues/99
-			if handle.LinkType() == 12 {
-				decoder = layers.LayerTypeIPv4
-			} else {
-				decoder = handle.LinkType()
-			}
-
-			source := gopacket.NewPacketSource(handle, decoder)
-			source.Lazy = true
-			source.NoCopy = true
-
-			wg.Done()
-
-			var data, srcIP, dstIP []byte
-
-			for {
-				packet, err := source.NextPacket()
-
-				if err == io.EOF {
-					break
-				} else if err != nil {
-					continue
-				}
-
-				// We should remove network layer before parsing TCP/IP data
-				var of int
-				switch decoder {
-				case layers.LinkTypeEthernet:
-					of = 14
-				case layers.LinkTypePPP:
-					of = 1
-				case layers.LinkTypeFDDI:
-					of = 13
-				case layers.LinkTypeNull:
-					of = 4
-				case layers.LinkTypeLoop:
-					of = 4
-				case layers.LinkTypeRaw, layers.LayerTypeIPv4:
-					of = 0
-				case layers.LinkTypeLinuxSLL:
-					of = 16
-				default:
-					log.Println("Unknown packet layer", decoder, packet)
-					break
-				}
-
-				data = packet.Data()[of:]
-
-				version := uint8(data[0]) >> 4
-				ipLength := int(binary.BigEndian.Uint16(data[2:4]))
-
-				if version == 4 {
-					ihl := uint8(data[0]) & 0x0F
-
-					// Truncated IP info
-					if len(data) < int(ihl*4) {
-						continue
-					}
-
-					srcIP = data[12:16]
-					dstIP = data[16:20]
-
-					// Too small IP packet
-					if ipLength < 20 {
-						continue
-					}
-
-					// Invalid length
-					if int(ihl*4) > ipLength {
-						continue
-					}
-
-					if cmp := len(data) - ipLength; cmp > 0 {
-						data = data[:ipLength]
-					} else if cmp < 0 {
-						// Truncated packet
-						continue
-					}
-
-					data = data[ihl*4:]
-				} else {
-					// Truncated IP info
-					if len(data) < 40 {
-						continue
-					}
-
-					srcIP = data[8:24]
-					dstIP = data[24:40]
-
-					data = data[40:]
-				}
-
-				// Truncated TCP info
-				if len(data) <= 13 {
-					continue
-				}
-
-				dataOffset := (data[12] & 0xF0) >> 4
-				isFIN := data[13]&0x01 != 0
-
-				// We need only packets with data inside
-				// Check that the buffer is larger than the size of the TCP header
-				if len(data) > int(dataOffset*4) || isFIN {
-					if !bpfSupported {
-						destPort := binary.BigEndian.Uint16(data[2:4])
-						srcPort := binary.BigEndian.Uint16(data[0:2])
-
-						var addrCheck []byte
-
-						if destPort == t.port {
-							addrCheck = dstIP
-						}
-
-						if t.trackResponse && srcPort == t.port {
-							addrCheck = srcIP
-						}
-
-						if len(addrCheck) == 0 {
-							continue
-						}
-
-						addrMatched := false
-
-						if loopback {
-							for _, dc := range devices {
-								if addrMatched {
-									break
-								}
-								for _, a := range dc.Addresses {
-									if a.IP.Equal(net.IP(addrCheck)) {
-										addrMatched = true
-										break
-									}
-								}
-							}
-							addrMatched = true
-						} else {
-							for _, a := range device.Addresses {
-								if a.IP.Equal(net.IP(addrCheck)) {
-									addrMatched = true
-									break
-								}
-							}
-						}
-
-						if !addrMatched {
-							continue
-						}
-					}
-
-					t.packetsChan <- t.buildPacket(srcIP, data, packet.Metadata().Timestamp)
-				}
-			}
-		}(d)
-	}
-
-	wg.Wait()
-	t.readyCh <- true
-}
-
-func (t *Listener) readPcapFile() {
-	if handle, err := pcap.OpenOffline(t.addr); err != nil {
-		log.Fatal(err)
-	} else {
-		if t.bpfFilter != "" {
-			if err := handle.SetBPFFilter(t.bpfFilter); err != nil {
-				log.Println("BPF filter error:", err)
-				return
-			}
-		}
-
-		t.readyCh <- true
-		packetSource := gopacket.NewPacketSource(handle, handle.LinkType())
-
-		for {
-			packet, err := packetSource.NextPacket()
-			if err == io.EOF {
-				break
-			} else if err != nil {
-				log.Println("Error:", err)
-				continue
-			}
-
-			var addr, data []byte
-
-			if tcpLayer := packet.Layer(layers.LayerTypeTCP); tcpLayer != nil {
-				tcp, _ := tcpLayer.(*layers.TCP)
-				data = append(tcp.LayerContents(), tcp.LayerPayload()...)
-
-				if uint16(tcp.DstPort) == t.port {
-					copy(data[0:2], []byte{byte(tcp.SrcPort >> 8), byte(tcp.SrcPort)})
-					copy(data[2:4], []byte{byte(tcp.DstPort >> 8), byte(tcp.DstPort)})
-				} else {
-					copy(data[0:2], []byte{byte(tcp.DstPort >> 8), byte(tcp.DstPort)})
-					copy(data[2:4], []byte{byte(tcp.SrcPort >> 8), byte(tcp.SrcPort)})
-				}
-			} else {
-				continue
-			}
-
-			if ipLayer := packet.Layer(layers.LayerTypeIPv4); ipLayer != nil {
-				ip, _ := ipLayer.(*layers.IPv4)
-				addr = ip.SrcIP
-			} else if ipLayer = packet.Layer(layers.LayerTypeIPv6); ipLayer != nil {
-				ip, _ := ipLayer.(*layers.IPv6)
-				addr = ip.SrcIP
-			} else {
-				// log.Println("Can't find IP layer", packet)
-				continue
-			}
-
-			dataOffset := (data[12] & 0xF0) >> 4
-			isFIN := data[13]&0x01 != 0
-
-			// We need only packets with data inside
-			// Check that the buffer is larger than the size of the TCP header
-			if len(data) <= int(dataOffset*4) && !isFIN {
-				continue
-			}
-
-			t.packetsChan <- t.buildPacket(addr, data, packet.Metadata().Timestamp)
-		}
-	}
-}
-
-func (t *Listener) readRAWSocket() {
-	conn, e := net.ListenPacket("ip:tcp", t.addr)
-	t.conn = conn
-
-	if e != nil {
-		log.Fatal(e)
-	}
-
-	defer t.conn.Close()
-
-	buf := make([]byte, 64*1024) // 64kb
-
-	t.readyCh <- true
-
-	for {
-		// Note: ReadFrom receive messages without IP header
-		n, addr, err := t.conn.ReadFrom(buf)
-
-		if err != nil {
-			if strings.HasSuffix(err.Error(), "closed network connection") {
-				return
-			} else {
-				continue
-			}
-		}
-
-		if n > 0 {
-			if t.isValidPacket(buf[:n]) {
-				t.packetsChan <- t.buildPacket([]byte(addr.(*net.IPAddr).IP), buf[:n], time.Now())
-			}
-		}
-	}
-}
-
-func (t *Listener) buildPacket(packetSrcIP []byte, packetData []byte, timestamp time.Time) *packet {
-	return &packet{
-		srcIP:     packetSrcIP,
-		data:      packetData,
-		timestamp: timestamp,
-	}
-}
-
-func (t *Listener) isValidPacket(buf []byte) bool {
-	// To avoid full packet parsing every time, we manually parsing values needed for packet filtering
-	// http://en.wikipedia.org/wiki/Transmission_Control_Protocol
-	destPort := binary.BigEndian.Uint16(buf[2:4])
-	srcPort := binary.BigEndian.Uint16(buf[0:2])
-
-	// Because RAW_SOCKET can't be bound to port, we have to control it by ourself
-	if destPort == t.port || (t.trackResponse && srcPort == t.port) {
-		// Get the 'data offset' (size of the TCP header in 32-bit words)
-		dataOffset := (buf[12] & 0xF0) >> 4
-
-		// We need only packets with data inside
-		// Check that the buffer is larger than the size of the TCP header
-		if len(buf) > int(dataOffset*4) {
-			// We should create new buffer because go slices is pointers. So buffer data shoud be immutable.
-			return true
-		}
-	}
-
-	return false
-}
-
-// Trying to add packet to existing message or creating new message
-//
-// For TCP message unique id is Acknowledgment number (see tcp_packet.go)
-func (t *Listener) processTCPPacket(packet *TCPPacket) {
-	// Don't exit on panic
-	defer func() {
-		if r := recover(); r != nil {
-			log.Println("PANIC: pkg:", r, packet, string(debug.Stack()))
-		}
-	}()
-
-	var responseRequest *TCPMessage
-	var message *TCPMessage
-
-	isIncoming := packet.DestPort == t.port
-
-	if !isIncoming {
-		responseRequest, _ = t.respAliases[packet.Ack]
-	}
-
-	// Seek for 100-expect chunks
-	// `packet.Ack != parentAck` is protection for clients who send data without ignoring server 100-continue response, e.g have data chunks have same Ack
-	if parentAck, ok := t.seqWithData[packet.Seq]; ok && packet.Ack != parentAck {
-		// Skip zero-length chunks https://github.com/buger/goreplay/issues/496
-		if len(packet.Data) == 0 {
-			return
-		}
-
-		// In case if non-first data chunks comes first
-		for _, m := range t.messages {
-			if m.Ack == packet.Ack && bytes.Equal(m.packets[0].Addr, packet.Addr) {
-				t.deleteMessage(m)
-
-				if m.AssocMessage != nil {
-					m.AssocMessage.setAssocMessage(nil)
-					m.setAssocMessage(nil)
-				}
-
-				for _, pkt := range m.packets {
-					// log.Println("Updating ack", parentAck, pkt.Ack)
-					pkt.UpdateAck(parentAck)
-					// Re-queue this packets
-					t.processTCPPacket(pkt)
-				}
-			}
-		}
-
-		t.ackAliases[packet.Ack] = parentAck
-		packet.UpdateAck(parentAck)
-	}
-
-	if isIncoming && packet.IsFIN {
-		if ma, ok := t.respAliases[packet.Seq]; ok {
-			if ma.packets[0].SrcPort == packet.SrcPort {
-				packet.UpdateAck(ma.Ack)
-			}
-		}
-	}
-
-	if alias, ok := t.ackAliases[packet.Ack]; ok {
-		packet.UpdateAck(alias)
-	}
-
-	message, ok := t.messages[packet.ID]
-
-	if !ok {
-		message = NewTCPMessage(packet.Seq, packet.Ack, isIncoming, packet.timestamp)
-		t.messages[packet.ID] = message
-
-		if !isIncoming {
-			if responseRequest != nil {
-				message.setAssocMessage(responseRequest)
-				responseRequest.setAssocMessage(message)
-			} else {
-				t.respWithoutReq[packet.Ack] = packet.ID
-			}
-		}
-	}
-
-	// Adding packet to message
-	message.AddPacket(packet)
-
-	// Handling Expect: 100-continue requests
-	if message.expectType == httpExpect100Continue && len(message.packets) == message.headerPacket+1 {
-		seq := packet.Seq + uint32(len(packet.Data))
-		t.seqWithData[seq] = packet.Ack
-
-		message.DataSeq = seq
-		message.complete = false
-
-		// In case if sequence packet came first
-		for _, m := range t.messages {
-			if m.Seq == seq {
-				t.deleteMessage(m)
-				if m.AssocMessage != nil {
-					message.setAssocMessage(m.AssocMessage)
-					m.AssocMessage.setAssocMessage(nil)
-				}
-
-				t.ackAliases[m.Ack] = packet.Ack
-
-				for _, pkt := range m.packets {
-					pkt.UpdateAck(packet.Ack)
-					message.AddPacket(pkt)
-				}
-			}
-		}
-
-		// Removing `Expect: 100-continue` header
-		packet.Data = proto.DeleteHeader(packet.Data, bExpectHeader)
-	}
-
-	// If client do sends Expect: 100-continue but do not respect server response
-	if message.expectType == httpExpect100Continue && (message.headerPacket != -1 && len(message.packets) > message.headerPacket+1) {
-		delete(t.seqWithData, message.DataSeq)
-		seq := packet.Seq + uint32(len(packet.Data))
-		t.seqWithData[seq] = packet.Ack
-		message.DataSeq = seq
-	}
-
-	if isIncoming {
-		// If message have multiple packets, delete previous alias
-		if len(message.packets) > 1 {
-			delete(t.respAliases, message.ResponseAck)
-		}
-
-		message.UpdateResponseAck()
-		t.respAliases[message.ResponseAck] = message
-	}
-
-	// If message contains only single packet immediately dispatch it
-	if message.complete {
-		// log.Println("COMPLETE!", isIncoming, message)
-		if isIncoming {
-			if t.trackResponse {
-				// log.Println("Found response!", message.ResponseID, t.messages)
-
-				if resp, ok := t.messages[message.ResponseID]; ok {
-					if resp.complete {
-						t.dispatchMessage(resp)
-					}
-
-					t.dispatchMessage(message)
-				}
-			} else {
-				t.dispatchMessage(message)
-			}
-		} else {
-			if message.AssocMessage == nil {
-				return
-			}
-
-			if req, ok := t.messages[message.AssocMessage.ID()]; ok {
-				if req.complete {
-					t.dispatchMessage(req)
-					t.dispatchMessage(message)
-				}
-			}
-		}
-	}
-}
-
-func (t *Listener) IsReady() bool {
-	select {
-	case <-t.readyCh:
-		return true
-	case <-time.After(5 * time.Second):
-		return false
-	}
-}
-
-// Receive TCP messages from the listener channel
-func (t *Listener) Receiver() chan *TCPMessage {
-	return t.messagesChan
-}
-
-func (t *Listener) Close() {
-	close(t.quit)
-	if t.conn != nil {
-		t.conn.Close()
-	}
-
-	for _, h := range t.pcapHandles {
-		h.Close()
-	}
-
-	return
-}

raw_socket_listener/listener_test.go

@@ -1,623 +0,0 @@
-package rawSocket
-
-import (
-	"bytes"
-	"log"
-	"math/rand"
-	"sync/atomic"
-	"testing"
-	"time"
-)
-
-func TestRawListenerInput(t *testing.T) {
-	var req, resp *TCPMessage
-
-	listener := NewListener("", "0", EnginePcap, true, 10*time.Millisecond, "", "", 0, false)
-	defer listener.Close()
-
-	reqPacket := buildPacket(true, 1, 1, []byte("GET / HTTP/1.1\r\n\r\n"), time.Now())
-
-	respAck := reqPacket.Seq + uint32(len(reqPacket.Data))
-	respPacket := buildPacket(false, respAck, reqPacket.Seq+1, []byte("HTTP/1.1 200 OK\r\n\r\n"), time.Now())
-
-	listener.packetsChan <- reqPacket.dump()
-	listener.packetsChan <- respPacket.dump()
-
-	select {
-	case req = <-listener.messagesChan:
-	case <-time.After(time.Millisecond):
-		t.Error("Should return request immediately")
-		return
-	}
-
-	if !req.IsIncoming {
-		t.Error("Should be request")
-	}
-
-	select {
-	case resp = <-listener.messagesChan:
-	case <-time.After(20 * time.Millisecond):
-		t.Error("Should return response immediately")
-		return
-	}
-
-	if resp.IsIncoming {
-		t.Error("Should be response")
-	}
-}
-
-func firstPacket(payload []byte) *TCPPacket {
-	return buildPacket(
-		true,
-		1,
-		1,
-		payload,
-		time.Now(),
-	)
-}
-
-func nextPacket(prev *TCPPacket, payload []byte) *TCPPacket {
-	return buildPacket(
-		prev.SrcPort == 1,
-		prev.Ack,
-		prev.Seq+uint32(len(prev.Data)),
-		payload,
-		prev.timestamp.Add(time.Millisecond),
-	)
-}
-
-func responsePacket(prev *TCPPacket, payload []byte) *TCPPacket {
-	return buildPacket(
-		!(prev.SrcPort == 1),
-		prev.Seq+uint32(len(prev.Data)),
-		prev.Ack,
-		payload,
-		prev.timestamp.Add(time.Millisecond),
-	)
-}
-
-func TestHEADRequestNoBody(t *testing.T) {
-	listener := NewListener("", "0", EnginePcap, true, 10*time.Millisecond, "", "", 0, false)
-	defer listener.Close()
-
-	reqPacket := firstPacket([]byte("HEAD / HTTP/1.1\r\nContent-Length: 0\r\n\r\n"))
-	respPacket := responsePacket(reqPacket, []byte("HTTP/1.1 200 OK\r\nContent-Length: 100\r\n\r\n"))
-
-	listener.packetsChan <- reqPacket.dump()
-	listener.packetsChan <- respPacket.dump()
-
-	var req, resp *TCPMessage
-	select {
-	case req = <-listener.messagesChan:
-	case <-time.After(time.Millisecond):
-		t.Error("Should return request immediately")
-		return
-	}
-
-	if !req.IsIncoming {
-		t.Error("Should be request")
-	}
-
-	select {
-	case resp = <-listener.messagesChan:
-	case <-time.After(20 * time.Millisecond):
-		t.Error("Should return response immediately")
-		return
-	}
-
-	if resp.IsIncoming {
-		t.Error("Should be response")
-	}
-}
-
-func TestSingleAck100Continue(t *testing.T) {
-	listener := NewListener("", "0", EnginePcap, true, 10*time.Millisecond, "", "", 0, false)
-	defer listener.Close()
-
-	reqPacket1 := firstPacket([]byte("POST / HTTP/1.1\r\nExpect: 100-continue\r\nContent-Length: 4\r\n\r\n"))
-	respPacket1 := responsePacket(reqPacket1, []byte(""))
-	respPacket2 := responsePacket(reqPacket1, []byte("HTTP/1.1 100 Continue\r\n"))
-	reqPacket2 := responsePacket(respPacket2, []byte("DATA"))
-	respPacket3 := responsePacket(reqPacket2, []byte("HTTP/1.1 200 OK\r\n\r\n"))
-
-	result := []byte("POST / HTTP/1.1\r\nContent-Length: 4\r\n\r\nDATA")
-
-	testRawListener100Continue(t, listener, result,
-		reqPacket1,
-		respPacket1, respPacket2,
-		reqPacket2,
-		respPacket3)
-}
-
-func Test100ContinueWithoutWaiting(t *testing.T) {
-	listener := NewListener("", "0", EnginePcap, true, 10*time.Millisecond, "", "", 0, false)
-	defer listener.Close()
-
-	req1 := firstPacket([]byte("POST / HTTP/1.1\r\nExpect: 100-continue\r\nContent-Length: 4\r\n\r\n"))
-	req2 := nextPacket(req1, []byte("DATA"))
-	resp1 := responsePacket(req1, []byte("HTTP/1.1 100 Continue\r\n"))
-	resp2 := responsePacket(req2, []byte("HTTP/1.1 200 OK\r\n\r\n"))
-
-	result := []byte("POST / HTTP/1.1\r\nContent-Length: 4\r\n\r\nDATA")
-
-	testRawListener100Continue(t, listener, result,
-		req1, req2, resp1, resp2)
-}
-
-// Client first sends data without waiting 100-continue, but once response received, generate packets based on Ack payload
-func Test100ContinueMixed(t *testing.T) {
-	listener := NewListener("", "0", EnginePcap, true, 10*time.Millisecond, "", "", 0, false)
-	defer listener.Close()
-
-	req1 := firstPacket([]byte("POST / HTTP/1.1\r\nExpect: 100-continue\r\nContent-Length: 12\r\n\r\n"))
-	req2 := nextPacket(req1, []byte("DAT1"))
-	resp1 := responsePacket(req1, []byte("HTTP/1.1 100 Continue\r\n\r\n"))
-	req3 := responsePacket(resp1, []byte("DAT2"))
-	req3.Seq = req2.Seq + uint32(len(req2.Data))
-	req4 := nextPacket(req3, []byte("DAT3"))
-	resp2 := responsePacket(req4, []byte("HTTP/1.1 200 OK\r\n\r\n"))
-
-	result := []byte("POST / HTTP/1.1\r\nContent-Length: 12\r\n\r\nDAT1DAT2DAT3")
-
-	testRawListener100Continue(t, listener, result,
-		req1, req2, req3, req4, resp1, resp2)
-}
-
-func TestDoubleAck100Continue(t *testing.T) {
-	listener := NewListener("", "0", EnginePcap, true, 10*time.Millisecond, "", "", 0, false)
-	defer listener.Close()
-
-	reqPacket1 := firstPacket([]byte("POST / HTTP/1.1\r\nExpect: 100-continue\r\nContent-Length: 4\r\n\r\n"))
-
-	respPacket1 := responsePacket(reqPacket1, []byte(""))
-	respPacket2 := responsePacket(reqPacket1, []byte("HTTP/1.1 100 Continue\r\n"))
-	reqPacket2 := responsePacket(respPacket2, []byte(""))
-	reqPacket3 := responsePacket(respPacket2, []byte("DATA"))
-	respPacket3 := responsePacket(reqPacket3, []byte("HTTP/1.1 200 OK\r\n\r\n"))
-
-	result := []byte("POST / HTTP/1.1\r\nContent-Length: 4\r\n\r\nDATA")
-
-	testRawListener100Continue(t, listener, result,
-		reqPacket1,
-		respPacket1, respPacket2,
-		reqPacket2, reqPacket3,
-		respPacket3)
-}
-
-func TestRawListenerInputResponseByClose(t *testing.T) {
-	var req, resp *TCPMessage
-
-	listener := NewListener("", "0", EnginePcap, true, 10*time.Millisecond, "", "", 0, false)
-	defer listener.Close()
-
-	reqPacket := buildPacket(true, 1, 1, []byte("GET / HTTP/1.1\r\n\r\n"), time.Now())
-
-	respAck := reqPacket.Seq + uint32(len(reqPacket.Data))
-	respPacket := buildPacket(false, respAck, reqPacket.Seq+1, []byte("HTTP/1.1 200 OK\r\nConnection: close\r\n\r\nasd"), time.Now())
-	finPacket := buildPacket(false, respAck, reqPacket.Seq+2, []byte(""), time.Now())
-	finPacket.IsFIN = true
-
-	listener.packetsChan <- reqPacket.dump()
-	listener.packetsChan <- respPacket.dump()
-	listener.packetsChan <- finPacket.dump()
-
-	select {
-	case req = <-listener.messagesChan:
-	case <-time.After(time.Millisecond):
-		t.Error("Should return request immediately")
-		return
-	}
-
-	if !req.IsIncoming {
-		t.Error("Should be request")
-	}
-
-	select {
-	case resp = <-listener.messagesChan:
-	case <-time.After(20 * time.Millisecond):
-		t.Error("Should return response immediately")
-		return
-	}
-
-	if resp.IsIncoming {
-		t.Error("Should be response")
-	}
-}
-
-func TestRawListenerInputWithoutResponse(t *testing.T) {
-	var req *TCPMessage
-
-	listener := NewListener("", "0", EnginePcap, false, 10*time.Millisecond, "", "", 0, false)
-	defer listener.Close()
-
-	reqPacket := buildPacket(true, 1, 1, []byte("GET / HTTP/1.1\r\n\r\n"), time.Now())
-
-	listener.packetsChan <- reqPacket.dump()
-
-	select {
-	case req = <-listener.messagesChan:
-	case <-time.After(time.Millisecond):
-		t.Error("Should return request immediately")
-		return
-	}
-
-	if !req.IsIncoming {
-		t.Error("Should be request")
-	}
-}
-
-func TestRawListenerResponse(t *testing.T) {
-	var req, resp *TCPMessage
-
-	listener := NewListener("", "0", EnginePcap, true, 10*time.Millisecond, "", "", 0, false)
-	defer listener.Close()
-
-	reqPacket := firstPacket([]byte("GET / HTTP/1.1\r\n\r\n"))
-	respPacket := responsePacket(reqPacket, []byte("HTTP/1.1 200 OK\r\n\r\n"))
-
-	// If response packet comes before request
-	listener.packetsChan <- respPacket.dump()
-	listener.packetsChan <- reqPacket.dump()
-
-	select {
-	case req = <-listener.messagesChan:
-	case <-time.After(time.Millisecond):
-		t.Error("Should return respose immediately")
-		return
-	}
-
-	if !req.IsIncoming {
-		t.Error("Should be request")
-	}
-
-	select {
-	case resp = <-listener.messagesChan:
-	case <-time.After(time.Millisecond):
-		t.Error("Should return response immediately")
-		return
-	}
-
-	if resp.IsIncoming {
-		t.Error("Should be response")
-	}
-
-	if !bytes.Equal(resp.UUID(), req.UUID()) {
-		t.Error("Resp and Req UUID should be equal")
-	}
-}
-
-func get100ContinuePackets() (req []*TCPPacket, resp []*TCPPacket) {
-	req1 := firstPacket([]byte("POST / HTTP/1.1\r\nExpect: 100-continue\r\nContent-Length: 2\r\n\r\n"))
-	resp1 := responsePacket(req1, []byte("HTTP/1.1 100 Continue\r\n"))
-	req2 := responsePacket(resp1, []byte("a"))
-	req3 := nextPacket(req2, []byte("b"))
-	resp2 := responsePacket(req3, []byte("HTTP/1.1 200 OK\r\n\r\n"))
-
-	return []*TCPPacket{req1, req2, req3}, []*TCPPacket{resp1, resp2}
-}
-
-func TestShort100Continue(t *testing.T) {
-	listener := NewListener("", "0", EnginePcap, true, 10*time.Millisecond, "", "", 0, false)
-	defer listener.Close()
-
-	req, resp := get100ContinuePackets()
-
-	result := []byte("POST / HTTP/1.1\r\nContent-Length: 2\r\n\r\nab")
-
-	testRawListener100Continue(t, listener, result, req[0], req[1], req[2], resp[0], resp[1])
-}
-
-// Response comes before Request
-func Test100ContinueWrongOrder(t *testing.T) {
-	listener := NewListener("", "0", EnginePcap, true, 10*time.Millisecond, "", "", 0, false)
-	defer listener.Close()
-
-	req, resp := get100ContinuePackets()
-
-	result := []byte("POST / HTTP/1.1\r\nContent-Length: 2\r\n\r\nab")
-
-	testRawListener100Continue(t, listener, result, resp[0], resp[1], req[0], req[1], req[2])
-}
-
-func testRawListener100Continue(t *testing.T, listener *Listener, result []byte, packets ...*TCPPacket) {
-	var req, resp *TCPMessage
-	for _, p := range packets {
-		listener.packetsChan <- p.dump()
-	}
-
-	select {
-	case req = <-listener.messagesChan:
-		break
-	case <-time.After(11 * time.Millisecond):
-		t.Error("Should return response after expire time")
-		return
-	}
-
-	if !bytes.Equal(req.Bytes(), result) {
-		t.Error("Should receive full message", string(req.Bytes()))
-	}
-
-	if !req.IsIncoming {
-		t.Error("Should be request")
-	}
-
-	select {
-	case resp = <-listener.messagesChan:
-		break
-	case <-time.After(21 * time.Millisecond):
-		t.Error("Should return response after expire time")
-		return
-	}
-
-	if resp.IsIncoming {
-		t.Error("Should be response")
-	}
-
-	if !bytes.Equal(resp.UUID(), req.UUID()) {
-		t.Error("Resp and Req UUID should be equal")
-	}
-}
-
-func testChunkedSequence(t *testing.T, listener *Listener, packets ...*TCPPacket) {
-	var r, req, resp *TCPMessage
-
-	for _, p := range packets {
-		listener.packetsChan <- p.dump()
-	}
-
-	select {
-	case r = <-listener.messagesChan:
-		if r.IsIncoming {
-			req = r
-		} else {
-			resp = r
-		}
-		break
-	case <-time.After(25 * time.Millisecond):
-		t.Error("Should return request after expire time")
-		return
-	}
-	select {
-	case r = <-listener.messagesChan:
-		if r.IsIncoming {
-			if req != nil {
-				t.Error("Request already received", r)
-				return
-			}
-			req = r
-		} else {
-			if resp != nil {
-				t.Error("Response already received", r)
-				return
-			}
-			resp = r
-		}
-		break
-	case <-time.After(25 * time.Millisecond):
-		t.Error("Should return request after expire time")
-		return
-	}
-
-	if !bytes.Equal(req.Bytes(), []byte("POST / HTTP/1.1\r\nTransfer-Encoding: chunked\r\n\r\n1\r\na\r\n1\r\nb\r\n0\r\n\r\n")) {
-		t.Error("Should receive full message", string(req.Bytes()))
-	}
-
-	if !req.IsIncoming {
-		t.Error("Should be request")
-	}
-
-	if resp.IsIncoming {
-		t.Error("Should be response")
-	}
-
-	if !bytes.Equal(resp.UUID(), req.UUID()) {
-		t.Error("Resp and Req UUID should be equal", string(resp.UUID()), string(req.UUID()))
-	}
-
-	time.Sleep(20 * time.Millisecond)
-
-	if len(listener.packetsChan) != 0 {
-		t.Fatal("packetsChan non empty:", listener.packetsChan)
-	}
-
-	if len(listener.messagesChan) != 0 {
-		t.Fatal("messagesChan non empty:", <-listener.messagesChan)
-	}
-
-	if len(listener.messages) != 0 {
-		t.Fatal("Messages non empty:", listener.messages)
-	}
-
-	if len(listener.ackAliases) != 0 {
-		t.Fatal("ackAliases non empty:", listener.ackAliases)
-	}
-
-	if len(listener.seqWithData) != 0 {
-		t.Fatal("seqWithData non empty:", listener.seqWithData)
-	}
-
-	if len(listener.respAliases) != 0 {
-		t.Fatal("respAliases non empty:", listener.respAliases)
-	}
-
-	if len(listener.respWithoutReq) != 0 {
-		t.Fatal("respWithoutReq non empty:", listener.respWithoutReq)
-	}
-}
-
-func permutation(n int, list []*TCPPacket) []*TCPPacket {
-	if len(list) == 1 {
-		return list
-	}
-
-	k := n % len(list)
-
-	first := []*TCPPacket{list[k]}
-	next := make([]*TCPPacket, len(list)-1)
-
-	copy(next, append(list[:k], list[k+1:]...))
-
-	return append(first, permutation(n/len(list), next)...)
-}
-
-// Response comes before Request
-func TestRawListenerChunkedWrongOrder(t *testing.T) {
-	listener := NewListener("", "0", EnginePcap, true, 10*time.Millisecond, "", "", 0, false)
-	defer listener.Close()
-
-	reqPacket1 := firstPacket([]byte("POST / HTTP/1.1\r\nTransfer-Encoding: chunked\r\nExpect: 100-continue\r\n\r\n"))
-
-	respPacket1 := responsePacket(reqPacket1, []byte("HTTP/1.1 100 Continue\r\n"))
-	reqPacket2 := responsePacket(respPacket1, []byte("1\r\na\r\n"))
-	reqPacket3 := nextPacket(reqPacket2, []byte("1\r\nb\r\n"))
-	reqPacket4 := nextPacket(reqPacket3, []byte("0\r\n\r\n"))
-
-	respPacket2 := responsePacket(reqPacket4, []byte("HTTP/1.1 200 OK\r\n\r\n"))
-
-	// Should re-construct message from all possible combinations
-	for i := 0; i < 6*5*4*3*2*1; i++ {
-		packets := permutation(i, []*TCPPacket{reqPacket1, reqPacket2, reqPacket3, reqPacket4, respPacket1, respPacket2})
-
-		testChunkedSequence(t, listener, packets...)
-	}
-}
-
-func chunkedPostMessage() []*TCPPacket {
-	ack := uint32(rand.Int63())
-	seq := uint32(rand.Int63())
-
-	reqPacket1 := buildPacket(true, ack, seq, []byte("POST / HTTP/1.1\r\nTransfer-Encoding: chunked\r\n\r\n"), time.Now())
-	// Packet with data have different Seq
-	reqPacket2 := buildPacket(true, ack, seq+47, []byte("1\r\na\r\n"), time.Now())
-	reqPacket3 := buildPacket(true, ack, reqPacket2.Seq+5, []byte("1\r\nb\r\n"), time.Now())
-	reqPacket4 := buildPacket(true, ack, reqPacket3.Seq+5, []byte("0\r\n\r\n"), time.Now())
-
-	respPacket := buildPacket(false, reqPacket4.Seq+5 /* len of data */, ack, []byte("HTTP/1.1 200 OK\r\n\r\n"), time.Now())
-
-	return []*TCPPacket{
-		reqPacket1, reqPacket2, reqPacket3, reqPacket4, respPacket,
-	}
-}
-
-func postMessage() []*TCPPacket {
-	ack := uint32(rand.Int63())
-	seq2 := uint32(rand.Int63())
-	seq := uint32(rand.Int63())
-
-	c := 10000
-	data := make([]byte, c)
-	rand.Read(data)
-
-	head := []byte("POST / HTTP/1.1\r\nContent-Length: 9958\r\n\r\n")
-	for i := range head {
-		data[i] = head[i]
-	}
-
-	return []*TCPPacket{
-		buildPacket(true, ack, seq, data, time.Now()),
-		buildPacket(false, seq+uint32(len(data)), seq2, []byte("HTTP/1.1 200 OK\r\n\r\n"), time.Now()),
-	}
-}
-
-func getMessage() []*TCPPacket {
-	ack := uint32(rand.Int63())
-	seq2 := uint32(rand.Int63())
-	seq := uint32(rand.Int63())
-
-	return []*TCPPacket{
-		buildPacket(true, ack, seq, []byte("GET / HTTP/1.1\r\n\r\n"), time.Now()),
-		buildPacket(false, seq+18, seq2, []byte("HTTP/1.1 200 OK\r\n\r\n"), time.Now()),
-	}
-}
-
-// Response comes before Request
-func TestRawListenerBench(t *testing.T) {
-	l := NewListener("", "0", EnginePcap, true, 200*time.Millisecond, "", "", 0, false)
-	defer l.Close()
-
-	// Should re-construct message from all possible combinations
-	for i := 0; i < 1000; i++ {
-		go func(i int) {
-			for j := 0; j < 100; j++ {
-				var packets []*TCPPacket
-
-				if j%5 == 0 {
-					packets = chunkedPostMessage()
-				} else if j%3 == 0 {
-					packets = postMessage()
-				} else {
-					packets = getMessage()
-				}
-
-				for _, p := range packets {
-					// Randomly drop packets
-					if (i+j)%5 == 0 {
-						if rand.Int63()%3 == 0 {
-							continue
-						}
-					}
-
-					l.packetsChan <- p.dump()
-					time.Sleep(time.Millisecond)
-				}
-
-				time.Sleep(5 * time.Millisecond)
-			}
-		}(i)
-	}
-
-	ch := l.Receiver()
-
-	var count int32
-
-	for {
-		select {
-		case <-ch:
-			atomic.AddInt32(&count, 1)
-		case <-time.After(2000 * time.Millisecond):
-			log.Println("Emitted 200000 messages, captured: ", count, len(l.ackAliases), len(l.seqWithData), len(l.respAliases), len(l.respWithoutReq), len(l.packetsChan))
-			return
-		}
-	}
-}
-
-func TestResponseZeroContentLength(t *testing.T) {
-	var req, resp *TCPMessage
-	listener := NewListener("", "0", EnginePcap, true, 10*time.Millisecond, "", "", 0, false)
-	defer listener.Close()
-
-	reqPacket := firstPacket([]byte("POST /api/setup/install HTTP/1.1\r\nHost: localhost:22936\r\nUser-Agent: curl/7.57.0\r\nAccept: */*\r\nContent-Length: 0\r\nContent-Type: application/x-www-form-urlencoded\r\n\r\n"))
-	respPacket := responsePacket(reqPacket, []byte("HTTP/1.1 200\r\nDate: Fri, 11 May 2018 15:09:10 GMT\r\nServer: Kestrel\r\nCache-Control: no-cache\r\nTransfer-Encoding: chunked\r\n\r\n"))
-	respPacket2 := nextPacket(respPacket, []byte("0\r\n\r\n"))
-
-	// If response packet comes before request
-	listener.packetsChan <- reqPacket.dump()
-	listener.packetsChan <- respPacket.dump()
-	listener.packetsChan <- respPacket2.dump()
-
-	select {
-	case req = <-listener.messagesChan:
-	case <-time.After(time.Millisecond):
-		t.Error("Should return respose immediately")
-		return
-	}
-
-	if !req.IsIncoming {
-		t.Error("Should be request")
-	}
-
-	select {
-	case resp = <-listener.messagesChan:
-	case <-time.After(time.Millisecond):
-		t.Error("Should return response immediately")
-		return
-	}
-
-	if resp.IsIncoming {
-		t.Error("Should be response")
-	}
-
-	if !bytes.Equal(resp.UUID(), req.UUID()) {
-		t.Error("Resp and Req UUID should be equal")
-	}
-}

raw_socket_listener/tcp_message.go

@@ -1,512 +0,0 @@
-package rawSocket
-
-import (
-	"bytes"
-	"crypto/sha1"
-	"encoding/binary"
-	"encoding/hex"
-	"log"
-	"net"
-	"strconv"
-	"strings"
-	"time"
-
-	"github.com/buger/goreplay/proto"
-)
-
-var _ = log.Println
-
-// TCPMessage ensure that all TCP packets for given request is received, and processed in right sequence
-// Its needed because all TCP message can be fragmented or re-transmitted
-//
-// Each TCP Packet have 2 ids: acknowledgment - message_id, and sequence - packet_id
-// Message can be compiled from unique packets with same message_id which sorted by sequence
-// Message is received if we didn't receive any packets for 2000ms
-type TCPMessage struct {
-	Seq         uint32
-	Ack         uint32
-	ResponseAck uint32
-	ResponseID  tcpID
-	DataAck     uint32
-	DataSeq     uint32
-
-	AssocMessage *TCPMessage
-	Start        time.Time
-	End          time.Time
-	IsIncoming   bool
-
-	packets []*TCPPacket
-
-	delChan chan *TCPMessage
-
-	/* HTTP specific variables */
-	methodType    httpMethodType
-	bodyType      httpBodyType
-	expectType    httpExpectType
-	seqMissing    bool
-	headerPacket  int
-	contentLength int
-	complete      bool
-}
-
-// NewTCPMessage pointer created from a sequence and acknowledgment numbers, whether the message is incoming and a timestamp
-// that indicates when the packet was captrued.
-func NewTCPMessage(Seq, Ack uint32, IsIncoming bool, timestamp time.Time) (msg *TCPMessage) {
-	msg = &TCPMessage{Seq: Seq, Ack: Ack, IsIncoming: IsIncoming, Start: timestamp}
-
-	return
-}
-
-func (t *TCPMessage) packetsData() (d [][]byte) {
-	d = make([][]byte, len(t.packets))
-	for i, p := range t.packets {
-		d[i] = p.Data
-	}
-
-	return
-}
-
-// Bytes return message content
-func (t *TCPMessage) Bytes() (output []byte) {
-	for _, p := range t.packets {
-		output = append(output, p.Data...)
-	}
-
-	return output
-}
-
-// BodySize returns total body size
-func (t *TCPMessage) BodySize() (size int) {
-	if len(t.packets) == 0 || t.headerPacket == -1 {
-		return 0
-	}
-
-	size += len(proto.Body(t.packets[t.headerPacket].Data))
-
-	for _, p := range t.packets[t.headerPacket+1:] {
-		size += len(p.Data)
-	}
-
-	return
-}
-
-// Size returns total size of message
-func (t *TCPMessage) Size() (size int) {
-	if len(t.packets) == 0 {
-		return 0
-	}
-
-	for _, p := range t.packets {
-		size += len(p.Data)
-	}
-
-	return
-}
-
-// AddPacket to the message and ensure packet uniqueness
-// TCP allows that packet can be re-send multiple times
-func (t *TCPMessage) AddPacket(packet *TCPPacket) {
-	packetFound := false
-
-	for _, pkt := range t.packets {
-		if packet.Seq == pkt.Seq {
-			packetFound = true
-			break
-		}
-	}
-
-	if !packetFound {
-		// Packets not always captured in same Seq order, and sometimes we need to prepend
-		if len(t.packets) == 0 || packet.Seq > t.packets[len(t.packets)-1].Seq {
-			t.packets = append(t.packets, packet)
-		} else if packet.Seq < t.packets[0].Seq {
-			t.packets = append([]*TCPPacket{packet}, t.packets...)
-			t.Seq = packet.Seq // Message Seq should indicated starting seq
-		} else { // insert somewhere in the middle...
-			for i, p := range t.packets {
-				if packet.Seq < p.Seq {
-					t.packets = append(t.packets[:i], append([]*TCPPacket{packet}, t.packets[i:]...)...)
-					break
-				}
-			}
-		}
-		if packet.OrigAck != 0 {
-			t.DataAck = packet.OrigAck
-		}
-
-		if packet.timestamp.Before(t.Start) || t.Start.IsZero() {
-			t.Start = packet.timestamp
-		}
-
-		if packet.timestamp.After(t.End) || t.End.IsZero() {
-			t.End = packet.timestamp
-		}
-	}
-
-	t.checkSeqIntegrity()
-	t.updateHeadersPacket()
-	t.updateMethodType()
-	t.updateBodyType()
-	t.checkIfComplete()
-	t.check100Continue()
-}
-
-// Check if there is missing packet
-func (t *TCPMessage) checkSeqIntegrity() {
-	if len(t.packets) == 1 {
-		t.seqMissing = false
-	}
-
-	offset := len(t.packets) - 1
-
-	if t.packets[offset].IsFIN {
-		offset--
-
-		if offset < 0 {
-			return
-		}
-	}
-
-	for i, p := range t.packets[:offset] {
-		if p.IsFIN {
-			continue
-		}
-
-		// If final packet
-		if len(t.packets) == i+1 {
-			t.seqMissing = false
-			return
-		}
-		np := t.packets[i+1]
-
-		nextSeq := p.Seq + uint32(len(p.Data))
-
-		if np.Seq != nextSeq {
-			if t.expectType == httpExpect100Continue {
-				if np.Seq != nextSeq+22 {
-					t.seqMissing = true
-					return
-				}
-			} else {
-				t.seqMissing = true
-				return
-			}
-		}
-	}
-
-	t.seqMissing = false
-}
-
-var bEmptyLine = []byte("\r\n\r\n")
-var bBR = []byte("\r\n")
-var bChunkEnd = []byte("\r\n0\r\n\r\n")
-
-func (t *TCPMessage) updateHeadersPacket() {
-	if len(t.packets) == 1 {
-		t.headerPacket = -1
-	}
-
-	if t.headerPacket != -1 {
-		return
-	}
-
-	if t.seqMissing {
-		return
-	}
-
-	for i, p := range t.packets {
-		if len(p.Data) >= len(bEmptyLine) {
-			if bytes.LastIndex(p.Data, bEmptyLine) != -1 {
-				t.headerPacket = i
-				return
-			}
-		} else if i > 0 && bytes.Equal(p.Data, bBR) {
-			idx := bytes.LastIndex(t.packets[i-1].Data, bBR)
-			if idx != -1 && idx == len(t.packets[i-1].Data)-len(bBR) {
-				t.headerPacket = i
-				return
-			}
-		}
-	}
-
-	return
-}
-
-// checkIfComplete returns true if all of the packets that compse the message arrived.
-func (t *TCPMessage) checkIfComplete() {
-	if t.seqMissing || t.headerPacket == -1 {
-		// log.Println("Seq missing", t.seqMissing, t.packets)
-		return
-	}
-
-	if t.methodType == httpMethodNotFound {
-		// log.Println("Method missing", t.methodType, t.packets)
-		return
-	}
-
-	// Responses can be emitted only if we found request
-	if !t.IsIncoming && t.AssocMessage == nil {
-		// log.Println("Assoc not found", t)
-		return
-	}
-
-	// log.Println("Found?", t)
-
-	switch t.bodyType {
-	case httpBodyEmpty:
-		t.complete = true
-	case httpBodyContentLength:
-		if t.contentLength == 0 || t.contentLength == t.BodySize() {
-			t.complete = true
-		}
-	case httpBodyChunked:
-		lastPacket := t.packets[len(t.packets)-1]
-		if bytes.LastIndex(lastPacket.Data, bChunkEnd) != -1 {
-			t.complete = true
-		}
-	default:
-		if len(t.packets) == 0 {
-			return
-		}
-
-		last := t.packets[len(t.packets)-1]
-		if last.IsFIN {
-			t.complete = true
-		}
-	}
-}
-
-type httpMethodType uint8
-
-const (
-	httpMethodNotSet   httpMethodType = 0
-	httpMethodKnown    httpMethodType = 1
-	httpMethodNotFound httpMethodType = 2
-)
-
-func (t *TCPMessage) updateMethodType() {
-	// if there is cache
-	if t.methodType != httpMethodNotSet && t.methodType != httpMethodNotFound {
-		return
-	}
-
-	d := t.packets[0].Data
-
-	// Minimum length fo request: GET / HTTP/1.1\r\n
-
-	if len(d) < 16 {
-		t.methodType = httpMethodNotFound
-		return
-	}
-
-	if t.IsIncoming {
-		if mIdx := bytes.IndexByte(d[:8], ' '); mIdx != -1 {
-			// Check that after method we have absolute or relative path
-			switch d[mIdx+1] {
-			case '/', 'h', '*':
-			default:
-				t.methodType = httpMethodNotFound
-				return
-			}
-		} else {
-			t.methodType = httpMethodNotFound
-			return
-		}
-
-		t.methodType = httpMethodKnown
-	} else {
-		if !bytes.Equal(d[:6], []byte("HTTP/1")) {
-			t.methodType = httpMethodNotFound
-			return
-		}
-
-		t.methodType = httpMethodKnown
-	}
-}
-
-type httpBodyType uint8
-
-const (
-	httpBodyNotSet          httpBodyType = 0
-	httpBodyEmpty           httpBodyType = 1
-	httpBodyContentLength   httpBodyType = 2
-	httpBodyChunked         httpBodyType = 3
-	httpBodyConnectionClose httpBodyType = 4
-)
-
-func (t *TCPMessage) updateBodyType() {
-	// if there is cache
-	if t.bodyType != httpBodyNotSet {
-		return
-	}
-
-	// Headers not received
-	if t.headerPacket == -1 {
-		return
-	}
-
-	var lengthB, encB, connB []byte
-
-	proto.ParseHeaders(t.packetsData(), func(header, value []byte) bool {
-		if proto.HeadersEqual(header, []byte("Content-Length")) {
-			lengthB = value
-			return false
-		}
-
-		if proto.HeadersEqual(header, []byte("Transfer-Encoding")) {
-			encB = value
-			return false
-		}
-
-		if proto.HeadersEqual(header, []byte("Connection")) {
-			connB = value
-		}
-
-		return true
-	})
-
-	switch t.methodType {
-	case httpMethodNotFound:
-		return
-	case httpMethodKnown:
-
-		if !t.IsIncoming &&
-			t.AssocMessage != nil &&
-			bytes.IndexByte(t.AssocMessage.Bytes(), ' ') > -1 &&
-			bytes.Equal([]byte("HEAD"), proto.Method(t.AssocMessage.Bytes())) {
-			// Need to check if this is a response to a head request,
-			// in which case the body has to be empty regardless.
-			t.bodyType = httpBodyEmpty
-			return
-		}
-
-		if len(lengthB) > 0 {
-			t.contentLength, _ = strconv.Atoi(string(lengthB))
-			if t.contentLength == 0 {
-				t.bodyType = httpBodyEmpty
-			} else {
-				t.bodyType = httpBodyContentLength
-			}
-			return
-		}
-
-		if len(encB) > 0 {
-			t.bodyType = httpBodyChunked
-			return
-		}
-
-		if len(connB) > 0 && bytes.Equal(connB, []byte("close")) {
-			t.bodyType = httpBodyConnectionClose
-			return
-		}
-	}
-
-	t.bodyType = httpBodyEmpty
-}
-
-type httpExpectType uint8
-
-const (
-	httpExpectNotSet      httpExpectType = 0
-	httpExpectEmpty       httpExpectType = 1
-	httpExpect100Continue httpExpectType = 2
-)
-
-var bExpectHeader = []byte("Expect")
-var bExpect100Value = []byte("100-continue")
-
-func (t *TCPMessage) check100Continue() {
-	if t.expectType != httpExpectNotSet || len(t.packets[0].Data) < 25 {
-		return
-	}
-
-	if t.seqMissing || t.headerPacket == -1 {
-		return
-	}
-
-	last := t.packets[len(t.packets)-1]
-	// reading last 4 bytes for double CRLF
-	if !bytes.HasSuffix(last.Data, bEmptyLine) {
-		return
-	}
-
-	var expectB []byte
-	proto.ParseHeaders(t.packetsData(), func(header, value []byte) bool {
-		if proto.HeadersEqual(header, bExpectHeader) {
-			expectB = value
-			return false
-		}
-
-		return true
-	})
-
-	if len(expectB) > 0 && bytes.Equal(bExpect100Value, expectB) {
-		t.expectType = httpExpect100Continue
-		return
-	}
-
-	t.expectType = httpExpectEmpty
-}
-
-func (t *TCPMessage) setAssocMessage(m *TCPMessage) {
-	t.AssocMessage = m
-	t.checkIfComplete()
-}
-
-// UpdateResponseAck should be called after packet is added
-func (t *TCPMessage) UpdateResponseAck() uint32 {
-	lastPacket := t.packets[len(t.packets)-1]
-	if lastPacket.IsFIN && len(t.packets) > 1 {
-		lastPacket = t.packets[len(t.packets)-2]
-	}
-
-	respAck := lastPacket.Seq + uint32(len(lastPacket.Data))
-
-	if t.ResponseAck != respAck {
-		t.ResponseAck = lastPacket.Seq + uint32(len(lastPacket.Data))
-
-		// We swappwed src and dst port
-		copy(t.ResponseID[:16], lastPacket.Addr)
-		copy(t.ResponseID[16:], lastPacket.Raw[2:4]) // Src port
-		copy(t.ResponseID[18:], lastPacket.Raw[0:2]) // Dest port
-		binary.BigEndian.PutUint32(t.ResponseID[20:24], t.ResponseAck)
-	}
-
-	return t.ResponseAck
-}
-
-func (t *TCPMessage) UUID() []byte {
-	var key []byte
-
-	if t.IsIncoming {
-		// log.Println("UUID:", t.Ack, t.Start.UnixNano())
-		key = strconv.AppendInt(key, t.Start.UnixNano(), 10)
-		key = strconv.AppendUint(key, uint64(t.Ack), 10)
-	} else {
-		// log.Println("RequestMessage:", t.AssocMessage.Ack, t.AssocMessage.Start.UnixNano())
-		key = strconv.AppendInt(key, t.AssocMessage.Start.UnixNano(), 10)
-		key = strconv.AppendUint(key, uint64(t.AssocMessage.Ack), 10)
-	}
-
-	uuid := make([]byte, 40)
-	sha := sha1.Sum(key)
-	hex.Encode(uuid, sha[:20])
-
-	return uuid
-}
-
-func (t *TCPMessage) ID() tcpID {
-	return t.packets[0].ID
-}
-
-func (t *TCPMessage) IP() net.IP {
-	return net.IP(t.packets[0].Addr)
-}
-
-func (t *TCPMessage) String() string {
-	return strings.Join([]string{
-		"Len packets: " + strconv.Itoa(len(t.packets)),
-		"Data size:" + strconv.Itoa(len(t.Bytes())),
-		"Data:" + string(t.Bytes()),
-	}, "\n")
-}

raw_socket_listener/tcp_message_test.go

@@ -1,258 +0,0 @@
-package rawSocket
-
-import (
-	"bytes"
-	"encoding/binary"
-	_ "log"
-	"testing"
-	"time"
-)
-
-func buildPacket(isIncoming bool, Ack, Seq uint32, Data []byte, timestamp time.Time) (packet *TCPPacket) {
-	var srcPort, destPort uint16
-
-	// For tests `listening` port is 0
-	if isIncoming {
-		srcPort = 1
-	} else {
-		destPort = 1
-	}
-
-	buf := make([]byte, 16)
-	binary.BigEndian.PutUint16(buf[2:4], destPort)
-	binary.BigEndian.PutUint16(buf[0:2], srcPort)
-	binary.BigEndian.PutUint32(buf[4:8], Seq)
-	binary.BigEndian.PutUint32(buf[8:12], Ack)
-	buf[12] = 64
-	buf = append(buf, Data...)
-
-	packet = ParseTCPPacket([]byte("123"), buf, timestamp)
-
-	return packet
-}
-
-func buildMessage(p *TCPPacket) *TCPMessage {
-	isIncoming := false
-	if p.SrcPort == 1 {
-		isIncoming = true
-	}
-
-	m := NewTCPMessage(p.Seq, p.Ack, isIncoming, p.timestamp)
-	m.AddPacket(p)
-
-	return m
-}
-
-func TestTCPMessagePacketsOrder(t *testing.T) {
-	msg := buildMessage(buildPacket(true, 1, 1, []byte("a"), time.Now()))
-	msg.AddPacket(buildPacket(true, 1, 2, []byte("b"), time.Now()))
-
-	if !bytes.Equal(msg.Bytes(), []byte("ab")) {
-		t.Error("Should contatenate packets in right order")
-	}
-
-	// When first packet have wrong order (Seq)
-	msg = buildMessage(buildPacket(true, 1, 2, []byte("b"), time.Now()))
-	msg.AddPacket(buildPacket(true, 1, 1, []byte("a"), time.Now()))
-
-	if !bytes.Equal(msg.Bytes(), []byte("ab")) {
-		t.Error("Should contatenate packets in right order")
-	}
-
-	// Should ignore packets with same sequence
-	msg = buildMessage(buildPacket(true, 1, 1, []byte("a"), time.Now()))
-	msg.AddPacket(buildPacket(true, 1, 1, []byte("a"), time.Now()))
-
-	if !bytes.Equal(msg.Bytes(), []byte("a")) {
-		t.Error("Should ignore packet with same Seq")
-	}
-}
-
-func TestTCPMessageSize(t *testing.T) {
-	msg := buildMessage(buildPacket(true, 1, 1, []byte("POST / HTTP/1.1\r\nContent-Length: 2\r\n\r\na"), time.Now()))
-	msg.AddPacket(buildPacket(true, 1, 2, []byte("b"), time.Now()))
-
-	if msg.BodySize() != 2 {
-		t.Error("Should count only body", msg.BodySize())
-	}
-
-	if msg.Size() != 40 {
-		t.Error("Should count all sizes", msg.Size())
-	}
-}
-
-func TestTCPMessageIsComplete(t *testing.T) {
-	testCases := []struct {
-		direction         bool
-		payload           string
-		assocMessage      bool
-		expectedCompleted bool
-	}{
-		{true, "GET / HTTP/1.1\r\n\r\n", false, true},
-		{true, "HEAD / HTTP/1.1\r\n\r\n", false, true},
-		{false, "HTTP/1.1 200 OK\r\n\r\n", true, true},
-		{true, "POST / HTTP/1.1\r\nContent-Length: 1\r\n\r\na", false, true},
-		{true, "PUT / HTTP/1.1\r\nContent-Length: 1\r\n\r\na", false, true},
-		{false, "HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n", true, true},
-		{false, "HTTP/1.1 200 OK\r\nContent-Length: 1\r\n\r\na", true, true},
-		{false, "HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n0\r\n\r\n", true, true},
-
-		// chunked not finished
-		{false, "HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n", true, false},
-
-		// content-length != actual length
-		{true, "POST / HTTP/1.1\r\nContent-Length: 2\r\n\r\na", false, false},
-		{false, "HTTP/1.1 200 OK\r\nContent-Length: 10\r\n\r\na", true, false},
-		// non-valid http request
-		{true, "UNKNOWN asd HTTP/1.1\r\n\r\n", false, false},
-
-		// response without associated request
-		{false, "HTTP/1.1 200 OK\r\n\r\n", false, false},
-	}
-
-	for _, tc := range testCases {
-		msg := buildMessage(buildPacket(tc.direction, 1, 1, []byte(tc.payload), time.Now()))
-		if tc.assocMessage {
-			msg.AssocMessage = &TCPMessage{}
-		}
-		msg.checkIfComplete()
-
-		if msg.complete != tc.expectedCompleted {
-			t.Errorf("Payload %s: Expected %t, got %t.", tc.payload, tc.expectedCompleted, msg.complete)
-		}
-	}
-}
-
-func TestTCPMessageIsSeqMissing(t *testing.T) {
-	p1 := buildPacket(false, 1, 1, []byte("HTTP/1.1 200 OK\r\n"), time.Now())
-	p2 := buildPacket(false, 1, p1.Seq+uint32(len(p1.Data)), []byte("Content-Length: 10\r\n\r\n"), time.Now())
-	p3 := buildPacket(false, 1, p2.Seq+uint32(len(p2.Data)), []byte("a"), time.Now())
-
-	msg := buildMessage(p1)
-	if msg.seqMissing {
-		t.Error("Should be complete if have only 1 packet")
-	}
-
-	msg.AddPacket(p3)
-	if !msg.seqMissing {
-		t.Error("Should be incomplete because missing middle component")
-	}
-
-	msg.AddPacket(p2)
-	if msg.seqMissing {
-		t.Error("Should be complete once missing packet added")
-	}
-}
-
-func TestTCPMessageIsHeadersReceived(t *testing.T) {
-	p1 := buildPacket(false, 1, 1, []byte("HTTP/1.1 200 OK\r\n\r\n"), time.Now())
-	p2 := buildPacket(false, 1, p1.Seq+uint32(len(p1.Data)), []byte("Content-Length: 10\r\n\r\n"), time.Now())
-
-	msg := buildMessage(p1)
-	if msg.headerPacket == -1 {
-		t.Error("Should be complete if have only 1 packet", msg.headerPacket)
-	}
-
-	msg.AddPacket(p2)
-	if msg.headerPacket == -1 {
-		t.Error("Should found double new line: headers received")
-	}
-
-	msg = buildMessage(buildPacket(true, 1, 1, []byte("GET / HTTP/1.1\r\nContent-Length: 1\r\n"), time.Now()))
-	if msg.headerPacket != -1 {
-		t.Error("Should not find headers end")
-	}
-}
-
-func TestTCPMessageMethodType(t *testing.T) {
-	testCases := []struct {
-		direction          bool
-		payload            string
-		expectedMethodType httpMethodType
-	}{
-		{true, "GET / HTTP/1.1\r\n\r\n", httpMethodKnown},
-		{true, "GET * HTTP/1.1\r\n\r\n", httpMethodKnown},
-		{true, "UNKNOWN / HTTP/1.1\r\n\r\n", httpMethodKnown},
-		{true, "GET http://example.com HTTP/1.1\r\n\r\n", httpMethodKnown},
-		{true, "POST / HTTP/1.1\r\n\r\n", httpMethodKnown},
-		{true, "PUT / HTTP/1.1\r\n\r\n", httpMethodKnown},
-		{true, "GET zxc HTTP/1.1\r\n\r\n", httpMethodNotFound},
-		{true, "GET / HTTP\r\n\r\n", httpMethodNotFound},
-		{true, "VERYLONGMETHOD / HTTP/1.1\r\n\r\n", httpMethodNotFound},
-		{false, "HTTP/1.1 200 OK\r\n\r\n", httpMethodKnown},
-		{false, "HTTP /1.1 200 OK\r\n\r\n", httpMethodNotFound},
-	}
-
-	for _, tc := range testCases {
-		msg := buildMessage(buildPacket(tc.direction, 1, 1, []byte(tc.payload), time.Now()))
-
-		if msg.methodType != tc.expectedMethodType {
-			t.Errorf("Expected %d, got %d", tc.expectedMethodType, msg.methodType)
-		}
-	}
-}
-
-func TestTCPMessageBodyType(t *testing.T) {
-	testCases := []struct {
-		direction        bool
-		payload          string
-		expectedBodyType httpBodyType
-	}{
-		{true, "GET / HTTP/1.1\r\n\r\n", httpBodyEmpty},
-		{true, "GET / HTTP/1.1\r\nContent-Length: 2\r\n\r\nab", httpBodyContentLength},
-		{true, "POST / HTTP/1.1\r\n\r\n", httpBodyEmpty},
-		{true, "POST / HTTP/1.1\r\nUser-Agent: zxc\r\n\r\n", httpBodyEmpty},
-		{false, "HTTP/1.1 200 OK\r\n\r\n", httpBodyEmpty},
-		{true, "POST / HTTP/1.1\r\nContent-Length: 2\r\n\r\nab", httpBodyContentLength},
-		{false, "HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nab", httpBodyContentLength},
-		{true, "POST / HTTP/1.1\r\nTransfer-Encoding: chunked\r\n\r\n2\r\nab\r\n0\r\n\r\n", httpBodyChunked},
-		{false, "HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n2\r\nab\r\n0\r\n\r\n", httpBodyChunked},
-	}
-
-	for _, tc := range testCases {
-		msg := buildMessage(buildPacket(tc.direction, 1, 1, []byte(tc.payload), time.Now()))
-
-		if msg.bodyType != tc.expectedBodyType {
-			t.Errorf("Expected %d, got %d", tc.expectedBodyType, msg.bodyType)
-		}
-	}
-}
-
-func TestTCPMessageBodySize(t *testing.T) {
-	testCases := []struct {
-		direction    bool
-		payloads     []string
-		expectedSize int
-	}{
-		{true, []string{"GET / HTTP/1.1\r\n\r\n"}, 0},
-		{true, []string{"POST / HTTP/1.1\r\nContent-Length: 2\r\n\r\nab"}, 2},
-		{true, []string{"GET / HTTP/1.1\r\n", "Content-Length: 2\r\n\r\nab"}, 2},
-		{true, []string{"GET / HTTP/1.1\r\n", "Content-Length: 2\r\n\r\n", "ab"}, 2},
-	}
-
-	for _, tc := range testCases {
-		msg := buildMessage(buildPacket(tc.direction, 1, 1, []byte(tc.payloads[0]), time.Now()))
-
-		if len(tc.payloads) > 1 {
-			for _, p := range tc.payloads[1:] {
-				seq := uint32(1 + msg.Size())
-				msg.AddPacket(buildPacket(tc.direction, 1, seq, []byte(p), time.Now()))
-			}
-		}
-
-		if msg.BodySize() != tc.expectedSize {
-			t.Errorf("Expected %d, got %d", tc.expectedSize, msg.BodySize())
-		}
-	}
-}
-
-func TestTcpMessageStart(t *testing.T) {
-	start := time.Now().Add(-1 * time.Second)
-
-	msg := buildMessage(buildPacket(true, 1, 2, []byte("b"), time.Now()))
-	msg.AddPacket(buildPacket(true, 1, 1, []byte("POST / HTTP/1.1\r\nContent-Length: 2\r\n\r\na"), start))
-
-	if msg.Start != start {
-		t.Error("Message timestamp should be equal to the lowest related packet timestamp", start, msg.Start)
-	}
-}

raw_socket_listener/tcp_packet.go

@@ -1,133 +0,0 @@
-package rawSocket
-
-import (
-	"encoding/binary"
-	"log"
-	"strconv"
-	"strings"
-	"time"
-)
-
-var _ = log.Println
-
-// TCP Flags
-const (
-	fFIN = 1 << iota
-	fSYN
-	fRST
-	fPSH
-	fACK
-	fURG
-	fECE
-	fCWR
-	fNS
-)
-
-type tcpID [24]byte
-
-// TCPPacket provides tcp packet parser
-// Packet structure: http://en.wikipedia.org/wiki/Transmission_Control_Protocol
-type TCPPacket struct {
-	SrcPort    uint16
-	DestPort   uint16
-	Seq        uint32
-	Ack        uint32
-	OrigAck    uint32
-	DataOffset uint8
-	IsFIN      bool
-
-	Raw       []byte
-	Data      []byte
-	Addr      []byte
-	timestamp time.Time
-	ID        tcpID
-}
-
-// ParseTCPPacket takes address and tcp payload and returns parsed TCPPacket
-func ParseTCPPacket(addr []byte, data []byte, timestamp time.Time) (p *TCPPacket) {
-	p = &TCPPacket{Raw: data}
-	p.ParseBasic()
-	p.Addr = addr
-	p.timestamp = timestamp
-	p.GenID()
-
-	return
-}
-
-func (p *TCPPacket) GenID() {
-	copy(p.ID[:16], p.Addr)
-	copy(p.ID[16:], p.Raw[0:2])  // Src port
-	copy(p.ID[18:], p.Raw[2:4])  // Dest port
-	copy(p.ID[20:], p.Raw[8:12]) // Ack
-}
-
-func (p *TCPPacket) UpdateAck(ack uint32) {
-	p.OrigAck = p.Ack
-	p.Ack = ack
-	binary.BigEndian.PutUint32(p.Raw[8:12], ack)
-	p.GenID()
-}
-
-// ParseBasic set of fields
-func (t *TCPPacket) ParseBasic() {
-	t.DestPort = binary.BigEndian.Uint16(t.Raw[2:4])
-	t.SrcPort = binary.BigEndian.Uint16(t.Raw[0:2])
-	t.Seq = binary.BigEndian.Uint32(t.Raw[4:8])
-	t.Ack = binary.BigEndian.Uint32(t.Raw[8:12])
-	t.DataOffset = (t.Raw[12] & 0xF0) >> 4
-	t.IsFIN = t.Raw[13]&0x01 != 0
-
-	if len(t.Raw) >= int(t.DataOffset*4) {
-		t.Data = t.Raw[t.DataOffset*4:]
-	}
-}
-
-func (t *TCPPacket) dump() *packet {
-
-	packetSrcIP := make([]byte, 16)
-	packetData := make([]byte, len(t.Data)+16)
-
-	copy(packetSrcIP, t.Addr)
-
-	binary.BigEndian.PutUint16(packetData[0:2], t.SrcPort)
-	binary.BigEndian.PutUint16(packetData[2:4], t.DestPort)
-
-	binary.BigEndian.PutUint32(packetData[4:8], t.Seq)
-	binary.BigEndian.PutUint32(packetData[8:12], t.Ack)
-
-	packetData[12] = 64
-
-	if t.IsFIN {
-		packetData[13] = packetData[13] | 0x01
-	}
-
-	copy(packetData[16:], t.Data)
-
-	return &packet{
-		srcIP:     packetSrcIP,
-		data:      packetData,
-		timestamp: t.timestamp,
-	}
-
-}
-
-// String output for a TCP Packet
-func (t *TCPPacket) String() string {
-	maxLen := len(t.Data)
-	if maxLen > 200 {
-		maxLen = 200
-	}
-
-	return strings.Join([]string{
-		"Addr: " + string(t.Addr),
-		"Source port: " + strconv.Itoa(int(t.SrcPort)),
-		"Dest port:" + strconv.Itoa(int(t.DestPort)),
-		"Sequence:" + strconv.Itoa(int(t.Seq)),
-		"Acknowledgment:" + strconv.Itoa(int(t.Ack)),
-		"Header len:" + strconv.Itoa(int(t.DataOffset)),
-		"FIN:" + strconv.FormatBool(t.IsFIN),
-
-		"Data size:" + strconv.Itoa(len(t.Data)),
-		"Data:" + string(t.Data[:maxLen]),
-	}, "\n")
-}

s3_reader.go

@@ -0,0 +1,105 @@
+package main
+
+import (
+	"bytes"
+	"log"
+	"os"
+	"strconv"
+	"strings"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/s3"
+)
+
+// S3ReadCloser ...
+type S3ReadCloser struct {
+	bucket    string
+	key       string
+	offset    int
+	totalSize int
+	readBytes int
+	sess      *session.Session
+	buf       *bytes.Buffer
+}
+
+func awsConfig() *aws.Config {
+	region := os.Getenv("AWS_DEFAULT_REGION")
+	if region == "" {
+		region = os.Getenv("AWS_REGION")
+		if region == "" {
+			region = "us-east-1"
+		}
+	}
+
+	config := &aws.Config{Region: aws.String(region)}
+
+	if endpoint := os.Getenv("AWS_ENDPOINT_URL"); endpoint != "" {
+		config.Endpoint = aws.String(endpoint)
+		log.Println("Custom endpoint:", endpoint)
+	}
+
+	log.Println("Connecting to S3. Region: " + region)
+
+	config.CredentialsChainVerboseErrors = aws.Bool(true)
+
+	if os.Getenv("AWS_DEBUG") != "" {
+		config.LogLevel = aws.LogLevel(aws.LogDebugWithHTTPBody)
+	}
+
+	return config
+}
+
+// NewS3ReadCloser returns new instance of S3 read closer
+func NewS3ReadCloser(path string) *S3ReadCloser {
+	if !PRO {
+		log.Fatal("Using S3 input and output require PRO license")
+		return nil
+	}
+
+	bucket, key := parseS3Url(path)
+	sess := session.Must(session.NewSession(awsConfig()))
+
+	log.Println("[S3 Input] S3 connection succesfully initialized", path)
+
+	return &S3ReadCloser{
+		bucket: bucket,
+		key:    key,
+		sess:   sess,
+		buf:    &bytes.Buffer{},
+	}
+}
+
+// Read reads buffer from s3 session
+func (s *S3ReadCloser) Read(b []byte) (n int, e error) {
+	if s.readBytes == 0 || s.readBytes+len(b) > s.offset {
+		svc := s3.New(s.sess)
+
+		objectRange := "bytes=" + strconv.Itoa(s.offset)
+		s.offset += 1000000 // Reading in chunks of 1 mb
+		objectRange += "-" + strconv.Itoa(s.offset-1)
+
+		params := &s3.GetObjectInput{
+			Bucket: aws.String(s.bucket),
+			Key:    aws.String(s.key),
+			Range:  aws.String(objectRange),
+		}
+		resp, err := svc.GetObject(params)
+
+		if err != nil {
+			log.Println("[S3 Input] Error during getting file", s.bucket, s.key, err)
+		} else {
+			s.totalSize, _ = strconv.Atoi(strings.Split(*resp.ContentRange, "/")[1])
+			s.buf.ReadFrom(resp.Body)
+		}
+	}
+
+	s.readBytes += len(b)
+
+	return s.buf.Read(b)
+}
+
+// Close is here to make S3ReadCloser satisfy ReadCloser interface
+func (s *S3ReadCloser) Close() error {
+	return nil
+}

s3_test.go

@@ -0,0 +1,150 @@
+// +build pro
+
+package main
+
+import (
+	"fmt"
+	"math/rand"
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/s3"
+)
+
+func TestS3Output(t *testing.T) {
+	bucket := aws.String("test-gor")
+	rnd := rand.Int63()
+	path := fmt.Sprintf("s3://test-gor/%d/requests.gz", rnd)
+
+	output := NewS3Output(path, &FileOutputConfig{queueLimit: 2})
+
+	svc := s3.New(output.session)
+
+	output.Write([]byte("1 1 1\ntest"))
+	output.Write([]byte("1 1 1\ntest"))
+	output.buffer.updateName()
+	output.Write([]byte("1 1 1\ntest"))
+	output.Write([]byte("1 1 1\ntest"))
+	output.buffer.updateName()
+	output.Write([]byte("1 1 1\ntest"))
+
+	time.Sleep(time.Second)
+
+	params := &s3.ListObjectsInput{
+		Bucket: bucket,
+		Prefix: aws.String(fmt.Sprintf("%d", rnd)),
+	}
+
+	resp, _ := svc.ListObjects(params)
+	if len(resp.Contents) != 2 {
+		t.Error("Should create 2 objects", len(resp.Contents))
+	} else {
+		if *resp.Contents[0].Key != fmt.Sprintf("%d/requests_0.gz", rnd) ||
+			*resp.Contents[1].Key != fmt.Sprintf("%d/requests_1.gz", rnd) {
+			t.Error("Should assign proper names", resp.Contents)
+		}
+	}
+
+	for _, c := range resp.Contents {
+		svc.DeleteObject(&s3.DeleteObjectInput{Bucket: bucket, Key: c.Key})
+	}
+
+	matches, _ := filepath.Glob(fmt.Sprintf("/tmp/gor_output_s3_*"))
+	for _, m := range matches {
+		os.Remove(m)
+	}
+}
+
+func TestS3OutputQueueLimit(t *testing.T) {
+	bucket := aws.String("test-gor")
+	rnd := rand.Int63()
+	path := fmt.Sprintf("s3://test-gor/%d/requests.gz", rnd)
+
+	output := NewS3Output(path, &FileOutputConfig{queueLimit: 100})
+	output.closeCh = make(chan struct{}, 3)
+
+	svc := s3.New(output.session)
+
+	for i := 0; i < 3; i++ {
+		for i := 0; i < 100; i++ {
+			output.Write([]byte("1 1 1\ntest"))
+		}
+		output.buffer.updateName()
+	}
+	output.buffer.updateName()
+	output.Write([]byte("1 1 1\ntest"))
+
+	for i := 0; i < 3; i++ {
+		<-output.closeCh
+	}
+
+	params := &s3.ListObjectsInput{
+		Bucket: bucket,
+		Prefix: aws.String(fmt.Sprintf("%d", rnd)),
+	}
+
+	resp, _ := svc.ListObjects(params)
+	if len(resp.Contents) != 3 {
+		t.Error("Should create 3 object", len(resp.Contents))
+	} else {
+		if *resp.Contents[0].Key != fmt.Sprintf("%d/requests_0.gz", rnd) ||
+			*resp.Contents[1].Key != fmt.Sprintf("%d/requests_1.gz", rnd) {
+			t.Error("Should assign proper names", resp.Contents)
+		}
+	}
+
+	for _, c := range resp.Contents {
+		svc.DeleteObject(&s3.DeleteObjectInput{Bucket: bucket, Key: c.Key})
+	}
+
+	matches, _ := filepath.Glob(fmt.Sprintf("/tmp/gor_output_s3_*"))
+	for _, m := range matches {
+		os.Remove(m)
+	}
+}
+
+func TestInputFileFromS3(t *testing.T) {
+	rnd := rand.Int63()
+	path := fmt.Sprintf("s3://test-gor-eu/%d/requests.gz", rnd)
+
+	output := NewS3Output(path, &FileOutputConfig{queueLimit: 5000})
+	output.closeCh = make(chan struct{}, 10)
+
+	for i := 0; i <= 20000; i++ {
+		output.Write([]byte("1 1 1\ntest"))
+
+		if i%5000 == 0 {
+			output.buffer.updateName()
+		}
+	}
+
+	output.Write([]byte("1 1 1\ntest"))
+
+	for i := 0; i < 2; i++ {
+		<-output.closeCh
+	}
+
+	input := NewFileInput(fmt.Sprintf("s3://test-gor-eu/%d", rnd), false)
+
+	buf := make([]byte, 1000)
+	for i := 0; i <= 19999; i++ {
+		input.Read(buf)
+	}
+
+	// Cleanup artifacts
+	bucket := aws.String("test-gor")
+	svc := s3.New(output.session)
+	params := &s3.ListObjectsInput{
+		Bucket: bucket,
+		Prefix: aws.String(fmt.Sprintf("%d", rnd)),
+	}
+
+	resp, _ := svc.ListObjects(params)
+
+	for _, c := range resp.Contents {
+		svc.DeleteObject(&s3.DeleteObjectInput{Bucket: bucket, Key: c.Key})
+	}
+}

settings.go

@@ -4,11 +4,13 @@ import (
 	"flag"
 	"fmt"
 	"os"
+	"runtime"
 	"sync"
 	"time"
 )
 
-var VERSION string
+// DEMO indicates that goreplay is running in demo mode
+var DEMO string
 
 // MultiOption allows to specify multiple flags with same name and collects all values into array
 type MultiOption []string
@@ -25,204 +27,235 @@ func (h *MultiOption) Set(value string) error {
 
 // AppSettings is the struct of main configuration
 type AppSettings struct {
-	verbose   bool
-	debug     bool
-	stats     bool
-	exitAfter time.Duration
+	Verbose   int           `json:"verbose"`
+	Stats     bool          `json:"stats"`
+	ExitAfter time.Duration `json:"exit-after"`
 
-	pprof string
+	SplitOutput          bool   `json:"split-output"`
+	RecognizeTCPSessions bool   `json:"recognize-tcp-sessions"`
+	Pprof                string `json:"http-pprof"`
 
-	splitOutput bool
+	InputDummy   MultiOption `json:"input-dummy"`
+	OutputDummy  MultiOption
+	OutputStdout bool `json:"output-stdout"`
+	OutputNull   bool `json:"output-null"`
 
-	inputDummy   MultiOption
-	outputDummy  MultiOption
-	outputStdout bool
-	outputNull   bool
+	InputTCP        MultiOption `json:"input-tcp"`
+	InputTCPConfig  TCPInputConfig
+	OutputTCP       MultiOption `json:"output-tcp"`
+	OutputTCPConfig TCPOutputConfig
+	OutputTCPStats  bool `json:"output-tcp-stats"`
 
-	inputTCP        MultiOption
-	inputTCPConfig  TCPInputConfig
-	outputTCP       MultiOption
-	outputTCPConfig TCPOutputConfig
-	outputTCPStats  bool
+	InputFile        MultiOption `json:"input-file"`
+	InputFileLoop    bool        `json:"input-file-loop"`
+	OutputFile       MultiOption `json:"output-file"`
+	OutputFileConfig FileOutputConfig
 
-	inputFile        MultiOption
-	inputFileLoop    bool
-	outputFile       MultiOption
-	outputFileConfig FileOutputConfig
+	InputRAW MultiOption `json:"input_raw"`
+	RAWInputConfig
 
-	inputRAW                MultiOption
-	inputRAWEngine          string
-	inputRAWTrackResponse   bool
-	inputRAWRealIPHeader    string
-	inputRAWExpire          time.Duration
-	inputRAWBpfFilter       string
-	inputRAWTimestampType   string
-	copyBufferSize          int
-	inputRAWImmediateMode   bool
-	inputRawBufferSize      int
-	inputRAWOverrideSnapLen bool
+	Middleware string `json:"middleware"`
 
-	middleware string
+	InputHTTP    MultiOption
+	OutputHTTP   MultiOption `json:"output-http"`
+	PrettifyHTTP bool        `json:"prettify-http"`
 
-	inputHTTP  MultiOption
-	outputHTTP MultiOption
+	OutputHTTPConfig HTTPOutputConfig
 
-	prettifyHTTP bool
+	OutputBinary       MultiOption `json:"output-binary"`
+	OutputBinaryConfig BinaryOutputConfig
 
-	outputHTTPConfig HTTPOutputConfig
-	modifierConfig   HTTPModifierConfig
+	ModifierConfig HTTPModifierConfig
 
-	inputKafkaConfig  KafkaConfig
-	outputKafkaConfig KafkaConfig
+	InputKafkaConfig  InputKafkaConfig
+	OutputKafkaConfig OutputKafkaConfig
+	KafkaTLSConfig    KafkaTLSConfig
 }
 
 // Settings holds Gor configuration
 var Settings AppSettings
 
 func usage() {
-	fmt.Printf("Gor is a simple http traffic replication tool written in Go. Its main goal is to replay traffic from production servers to staging and dev environments.\nProject page: https://github.com/buger/gor\nAuthor: <Leonid Bugaev> leonsbox@gmail.com\nCurrent Version: %s\n\n", VERSION)
+	fmt.Printf("Gor is a simple http traffic replication tool written in Go. Its main goal is to replay traffic from production servers to staging and dev environments.\nProject page: https://github.com/buger/gor\nAuthor: <Leonid Bugaev> leonsbox@gmail.com\nCurrent Version: v%s\n\n", VERSION)
 	flag.PrintDefaults()
 	os.Exit(2)
 }
 
 func init() {
 	flag.Usage = usage
+	flag.StringVar(&Settings.Pprof, "http-pprof", "", "Enable profiling. Starts  http server on specified port, exposing special /debug/pprof endpoint. Example: `:8181`")
+	flag.IntVar(&Settings.Verbose, "verbose", 0, "set the level of verbosity, if greater than zero then it will turn on debug output")
+	flag.BoolVar(&Settings.Stats, "stats", false, "Turn on queue stats output")
 
-	flag.StringVar(&Settings.pprof, "http-pprof", "", "Enable profiling. Starts  http server on specified port, exposing special /debug/pprof endpoint. Example: `:8181`")
-	flag.BoolVar(&Settings.verbose, "verbose", false, "Turn on more verbose output")
-	flag.BoolVar(&Settings.debug, "debug", false, "Turn on debug output, shows all intercepted traffic. Works only when with `verbose` flag")
-	flag.BoolVar(&Settings.stats, "stats", false, "Turn on queue stats output")
-	flag.DurationVar(&Settings.exitAfter, "exit-after", 0, "exit after specified duration")
+	if DEMO == "" {
+		flag.DurationVar(&Settings.ExitAfter, "exit-after", 0, "exit after specified duration")
+	} else {
+		Settings.ExitAfter = 5 * time.Minute
+	}
 
-	flag.BoolVar(&Settings.splitOutput, "split-output", false, "By default each output gets same traffic. If set to `true` it splits traffic equally among all outputs.")
+	flag.BoolVar(&Settings.SplitOutput, "split-output", false, "By default each output gets same traffic. If set to `true` it splits traffic equally among all outputs.")
 
-	flag.Var(&Settings.inputDummy, "input-dummy", "Used for testing outputs. Emits 'Get /' request every 1s")
-	flag.Var(&Settings.outputDummy, "output-dummy", "DEPRECATED: use --output-stdout instead")
+	flag.BoolVar(&Settings.RecognizeTCPSessions, "recognize-tcp-sessions", false, "[PRO] If turned on http output will create separate worker for each TCP session. Splitting output will session based as well.")
 
-	flag.BoolVar(&Settings.outputStdout, "output-stdout", false, "Used for testing inputs. Just prints to console data coming from inputs.")
+	flag.Var(&Settings.InputDummy, "input-dummy", "Used for testing outputs. Emits 'Get /' request every 1s")
+	flag.Var(&Settings.OutputDummy, "output-dummy", "DEPRECATED: use --output-stdout instead")
 
-	flag.BoolVar(&Settings.outputNull, "output-null", false, "Used for testing inputs. Drops all requests.")
+	flag.BoolVar(&Settings.OutputStdout, "output-stdout", false, "Used for testing inputs. Just prints to console data coming from inputs.")
 
-	flag.Var(&Settings.inputTCP, "input-tcp", "Used for internal communication between Gor instances. Example: \n\t# Receive requests from other Gor instances on 28020 port, and redirect output to staging\n\tgor --input-tcp :28020 --output-http staging.com")
-	flag.BoolVar(&Settings.inputTCPConfig.secure, "input-tcp-secure", false, "Turn on TLS security. Do not forget to specify certificate and key files.")
-	flag.StringVar(&Settings.inputTCPConfig.certificatePath, "input-tcp-certificate", "", "Path to PEM encoded certificate file. Used when TLS turned on.")
-	flag.StringVar(&Settings.inputTCPConfig.keyPath, "input-tcp-certificate-key", "", "Path to PEM encoded certificate key file. Used when TLS turned on.")
+	flag.BoolVar(&Settings.OutputNull, "output-null", false, "Used for testing inputs. Drops all requests.")
 
-	flag.Var(&Settings.outputTCP, "output-tcp", "Used for internal communication between Gor instances. Example: \n\t# Listen for requests on 80 port and forward them to other Gor instance on 28020 port\n\tgor --input-raw :80 --output-tcp replay.local:28020")
-	flag.BoolVar(&Settings.outputTCPConfig.secure, "output-tcp-secure", false, "Use TLS secure connection. --input-file on another end should have TLS turned on as well.")
-	flag.BoolVar(&Settings.outputTCPStats, "output-tcp-stats", false, "Report TCP output queue stats to console every 5 seconds.")
+	flag.Var(&Settings.InputTCP, "input-tcp", "Used for internal communication between Gor instances. Example: \n\t# Receive requests from other Gor instances on 28020 port, and redirect output to staging\n\tgor --input-tcp :28020 --output-http staging.com")
+	flag.BoolVar(&Settings.InputTCPConfig.Secure, "input-tcp-secure", false, "Turn on TLS security. Do not forget to specify certificate and key files.")
+	flag.StringVar(&Settings.InputTCPConfig.CertificatePath, "input-tcp-certificate", "", "Path to PEM encoded certificate file. Used when TLS turned on.")
+	flag.StringVar(&Settings.InputTCPConfig.KeyPath, "input-tcp-certificate-key", "", "Path to PEM encoded certificate key file. Used when TLS turned on.")
 
-	flag.Var(&Settings.inputFile, "input-file", "Read requests from file: \n\tgor --input-file ./requests.gor --output-http staging.com")
-	flag.BoolVar(&Settings.inputFileLoop, "input-file-loop", false, "Loop input files, useful for performance testing.")
+	flag.Var(&Settings.OutputTCP, "output-tcp", "Used for internal communication between Gor instances. Example: \n\t# Listen for requests on 80 port and forward them to other Gor instance on 28020 port\n\tgor --input-raw :80 --output-tcp replay.local:28020")
+	flag.BoolVar(&Settings.OutputTCPConfig.Secure, "output-tcp-secure", false, "Use TLS secure connection. --input-file on another end should have TLS turned on as well.")
+	flag.BoolVar(&Settings.OutputTCPConfig.SkipVerify, "output-tcp-skip-verify", false, "Don't verify hostname on TLS secure connection.")
+	flag.BoolVar(&Settings.OutputTCPConfig.Sticky, "output-tcp-sticky", false, "Use Sticky connection. Request/Response with same ID will be sent to the same connection.")
+	flag.IntVar(&Settings.OutputTCPConfig.Workers, "output-tcp-workers", 10, "Number of parallel tcp connections, default is 10")
+	flag.BoolVar(&Settings.OutputTCPStats, "output-tcp-stats", false, "Report TCP output queue stats to console every 5 seconds.")
 
-	flag.Var(&Settings.outputFile, "output-file", "Write incoming requests to file: \n\tgor --input-raw :80 --output-file ./requests.gor")
-	flag.DurationVar(&Settings.outputFileConfig.flushInterval, "output-file-flush-interval", time.Second, "Interval for forcing buffer flush to the file, default: 1s.")
-	flag.BoolVar(&Settings.outputFileConfig.append, "output-file-append", false, "The flushed chunk is appended to existence file or not. ")
+	flag.Var(&Settings.InputFile, "input-file", "Read requests from file: \n\tgor --input-file ./requests.gor --output-http staging.com")
+	flag.BoolVar(&Settings.InputFileLoop, "input-file-loop", false, "Loop input files, useful for performance testing.")
 
-	// Set default
-	Settings.outputFileConfig.sizeLimit.Set("32mb")
-	flag.Var(&Settings.outputFileConfig.sizeLimit, "output-file-size-limit", "Size of each chunk. Default: 32mb")
-	flag.IntVar(&Settings.outputFileConfig.queueLimit, "output-file-queue-limit", 256, "The length of the chunk queue. Default: 256")
-	Settings.outputFileConfig.outputFileMaxSize.Set("-1")
-	flag.Var(&Settings.outputFileConfig.outputFileMaxSize, "output-file-max-size-limit", "Max size of output file, Default: 1TB")
+	flag.Var(&Settings.OutputFile, "output-file", "Write incoming requests to file: \n\tgor --input-raw :80 --output-file ./requests.gor")
+	flag.DurationVar(&Settings.OutputFileConfig.FlushInterval, "output-file-flush-interval", time.Second, "Interval for forcing buffer flush to the file, default: 1s.")
+	flag.BoolVar(&Settings.OutputFileConfig.Append, "output-file-append", false, "The flushed chunk is appended to existence file or not. ")
+	flag.Var(&Settings.OutputFileConfig.SizeLimit, "output-file-size-limit", "Size of each chunk. Default: 32mb")
+	flag.IntVar(&Settings.OutputFileConfig.QueueLimit, "output-file-queue-limit", 256, "The length of the chunk queue. Default: 256")
+	flag.Var(&Settings.OutputFileConfig.OutputFileMaxSize, "output-file-max-size-limit", "Max size of output file, Default: 1TB")
 
-	flag.BoolVar(&Settings.prettifyHTTP, "prettify-http", false, "If enabled, will automatically decode requests and responses with: Content-Encodning: gzip and Transfer-Encoding: chunked. Useful for debugging, in conjuction with --output-stdout")
+	flag.StringVar(&Settings.OutputFileConfig.BufferPath, "output-file-buffer", "/tmp", "The path for temporary storing current buffer: \n\tgor --input-raw :80 --output-file s3://mybucket/logs/%Y-%m-%d.gz --output-file-buffer /mnt/logs")
 
-	flag.Var(&Settings.inputRAW, "input-raw", "Capture traffic from given port (use RAW sockets and require *sudo* access):\n\t# Capture traffic from 8080 port\n\tgor --input-raw :8080 --output-http staging.com")
+	flag.BoolVar(&Settings.PrettifyHTTP, "prettify-http", false, "If enabled, will automatically decode requests and responses with: Content-Encoding: gzip and Transfer-Encoding: chunked. Useful for debugging, in conjuction with --output-stdout")
 
-	flag.BoolVar(&Settings.inputRAWTrackResponse, "input-raw-track-response", false, "If turned on Gor will track responses in addition to requests, and they will be available to middleware and file output.")
+	// input raw flags
+	flag.Var(&Settings.InputRAW, "input-raw", "Capture traffic from given port (use RAW sockets and require *sudo* access):\n\t# Capture traffic from 8080 port\n\tgor --input-raw :8080 --output-http staging.com")
+	flag.BoolVar(&Settings.TrackResponse, "input-raw-track-response", false, "If turned on Gor will track responses in addition to requests, and they will be available to middleware and file output.")
+	flag.Var(&Settings.Engine, "input-raw-engine", "Intercept traffic using `libpcap` (default), `raw_socket` or `pcap_file`")
+	flag.Var(&Settings.Protocol, "input-raw-protocol", "Specify application protocol of intercepted traffic. Possible values: http, binary")
+	flag.StringVar(&Settings.RealIPHeader, "input-raw-realip-header", "", "If not blank, injects header with given name and real IP value to the request payload. Usually this header should be named: X-Real-IP")
+	flag.DurationVar(&Settings.Expire, "input-raw-expire", time.Second*2, "How much it should wait for the last TCP packet, till consider that TCP message complete.")
+	flag.StringVar(&Settings.BPFFilter, "input-raw-bpf-filter", "", "BPF filter to write custom expressions. Can be useful in case of non standard network interfaces like tunneling or SPAN port. Example: --input-raw-bpf-filter 'dst port 80'")
+	flag.StringVar(&Settings.TimestampType, "input-raw-timestamp-type", "", "Possible values: PCAP_TSTAMP_HOST, PCAP_TSTAMP_HOST_LOWPREC, PCAP_TSTAMP_HOST_HIPREC, PCAP_TSTAMP_ADAPTER, PCAP_TSTAMP_ADAPTER_UNSYNCED. This values not supported on all systems, GoReplay will tell you available values of you put wrong one.")
+	flag.Var(&Settings.CopyBufferSize, "copy-buffer-size", "Set the buffer size for an individual request (default 5MB)")
+	flag.BoolVar(&Settings.Snaplen, "input-raw-override-snaplen", false, "Override the capture snaplen to be 64k. Required for some Virtualized environments")
+	flag.DurationVar(&Settings.BufferTimeout, "input-raw-buffer-timeout", 0, "set the pcap timeout. for immediate mode don't set this flag")
+	flag.Var(&Settings.BufferSize, "input-raw-buffer-size", "Controls size of the OS buffer which holds packets until they dispatched. Default value depends by system: in Linux around 2MB. If you see big package drop, increase this value.")
+	flag.BoolVar(&Settings.Promiscuous, "input-raw-promisc", false, "enable promiscuous mode")
+	flag.BoolVar(&Settings.Monitor, "input-raw-monitor", false, "enable RF monitor mode")
+	flag.BoolVar(&Settings.Stats, "input-raw-stats", false, "enable stats generator on raw TCP messages")
 
-	flag.StringVar(&Settings.inputRAWEngine, "input-raw-engine", "libpcap", "Intercept traffic using `libpcap` (default), and `raw_socket`")
+	flag.StringVar(&Settings.Middleware, "middleware", "", "Used for modifying traffic using external command")
 
-	flag.StringVar(&Settings.inputRAWRealIPHeader, "input-raw-realip-header", "", "If not blank, injects header with given name and real IP value to the request payload. Usually this header should be named: X-Real-IP")
+	flag.Var(&Settings.OutputHTTP, "output-http", "Forwards incoming requests to given http address.\n\t# Redirect all incoming requests to staging.com address \n\tgor --input-raw :80 --output-http http://staging.com")
 
-	flag.DurationVar(&Settings.inputRAWExpire, "input-raw-expire", time.Second*2, "How much it should wait for the last TCP packet, till consider that TCP message complete.")
+	/* outputHTTPConfig */
+	flag.Var(&Settings.OutputHTTPConfig.BufferSize, "output-http-response-buffer", "HTTP response buffer size, all data after this size will be discarded.")
+	flag.BoolVar(&Settings.OutputHTTPConfig.CompatibilityMode, "output-http-compatibility-mode", false, "Use standard Go client, instead of built-in implementation. Can be slower, but more compatible.")
 
-	flag.StringVar(&Settings.inputRAWBpfFilter, "input-raw-bpf-filter", "", "BPF filter to write custom expressions. Can be useful in case of non standard network interfaces like tunneling or SPAN port. Example: --input-raw-bpf-filter 'dst port 80'")
+	flag.IntVar(&Settings.OutputHTTPConfig.WorkersMin, "output-http-workers-min", 0, "Gor uses dynamic worker scaling. Enter a number to set a minimum number of workers. default = 1.")
+	flag.IntVar(&Settings.OutputHTTPConfig.WorkersMax, "output-http-workers", 0, "Gor uses dynamic worker scaling. Enter a number to set a maximum number of workers. default = 0 = unlimited.")
+	flag.IntVar(&Settings.OutputHTTPConfig.QueueLen, "output-http-queue-len", 1000, "Number of requests that can be queued for output, if all workers are busy. default = 1000")
 
-	flag.StringVar(&Settings.inputRAWTimestampType, "input-raw-timestamp-type", "", "Possible values: PCAP_TSTAMP_HOST, PCAP_TSTAMP_HOST_LOWPREC, PCAP_TSTAMP_HOST_HIPREC, PCAP_TSTAMP_ADAPTER, PCAP_TSTAMP_ADAPTER_UNSYNCED. This values not supported on all systems, GoReplay will tell you available values of you put wrong one.")
-	flag.IntVar(&Settings.copyBufferSize, "copy-buffer-size", 5*1024*1024, "Set the buffer size for an individual request (default 5M)")
-	flag.BoolVar(&Settings.inputRAWOverrideSnapLen, "input-raw-override-snaplen", false, "Override the capture snaplen to be 64k. Required for some Virtualized environments")
-	flag.BoolVar(&Settings.inputRAWImmediateMode, "input-raw-immediate-mode", false, "Set pcap interface to immediate mode.")
+	flag.IntVar(&Settings.OutputHTTPConfig.RedirectLimit, "output-http-redirects", 0, "Enable how often redirects should be followed.")
+	flag.DurationVar(&Settings.OutputHTTPConfig.Timeout, "output-http-timeout", 5*time.Second, "Specify HTTP request/response timeout. By default 5s. Example: --output-http-timeout 30s")
+	flag.BoolVar(&Settings.OutputHTTPConfig.TrackResponses, "output-http-track-response", false, "If turned on, HTTP output responses will be set to all outputs like stdout, file and etc.")
 
-	flag.IntVar(&Settings.inputRawBufferSize, "input-raw-buffer-size", 0, "Controls size of the OS buffer (in bytes) which holds packets until they dispatched. Default value depends by system: in Linux around 2MB. If you see big package drop, increase this value.")
+	flag.BoolVar(&Settings.OutputHTTPConfig.Stats, "output-http-stats", false, "Report http output queue stats to console every N milliseconds. See output-http-stats-ms")
+	flag.IntVar(&Settings.OutputHTTPConfig.StatsMs, "output-http-stats-ms", 5000, "Report http output queue stats to console every N milliseconds. default: 5000")
+	flag.BoolVar(&Settings.OutputHTTPConfig.OriginalHost, "http-original-host", false, "Normally gor replaces the Host http header with the host supplied with --output-http.  This option disables that behavior, preserving the original Host header.")
+	flag.BoolVar(&Settings.OutputHTTPConfig.Debug, "output-http-debug", false, "Enables http debug output.")
+	flag.StringVar(&Settings.OutputHTTPConfig.ElasticSearch, "output-http-elasticsearch", "", "Send request and response stats to ElasticSearch:\n\tgor --input-raw :8080 --output-http staging.com --output-http-elasticsearch 'es_host:api_port/index_name'")
+	/* outputHTTPConfig */
 
-	flag.StringVar(&Settings.middleware, "middleware", "", "Used for modifying traffic using external command")
+	flag.Var(&Settings.OutputBinary, "output-binary", "Forwards incoming binary payloads to given address.\n\t# Redirect all incoming requests to staging.com address \n\tgor --input-raw :80 --input-raw-protocol binary --output-binary staging.com:80")
+	/* outputBinaryConfig */
+	flag.Var(&Settings.OutputBinaryConfig.BufferSize, "output-tcp-response-buffer", "TCP response buffer size, all data after this size will be discarded.")
+	flag.IntVar(&Settings.OutputBinaryConfig.Workers, "output-binary-workers", 0, "Gor uses dynamic worker scaling by default.  Enter a number to run a set number of workers.")
+	flag.DurationVar(&Settings.OutputBinaryConfig.Timeout, "output-binary-timeout", 0, "Specify HTTP request/response timeout. By default 5s. Example: --output-binary-timeout 30s")
+	flag.BoolVar(&Settings.OutputBinaryConfig.TrackResponses, "output-binary-track-response", false, "If turned on, Binary output responses will be set to all outputs like stdout, file and etc.")
 
-	// flag.Var(&Settings.inputHTTP, "input-http", "Read requests from HTTP, should be explicitly sent from your application:\n\t# Listen for http on 9000\n\tgor --input-http :9000 --output-http staging.com")
+	flag.BoolVar(&Settings.OutputBinaryConfig.Debug, "output-binary-debug", false, "Enables binary debug output.")
+	/* outputBinaryConfig */
 
-	flag.Var(&Settings.outputHTTP, "output-http", "Forwards incoming requests to given http address.\n\t# Redirect all incoming requests to staging.com address \n\tgor --input-raw :80 --output-http http://staging.com")
-	flag.IntVar(&Settings.outputHTTPConfig.BufferSize, "output-http-response-buffer", 0, "HTTP response buffer size, all data after this size will be discarded.")
-	flag.BoolVar(&Settings.outputHTTPConfig.CompatibilityMode, "output-http-compatibility-mode", false, "Use standard Go client, instead of built-in implementation. Can be slower, but more compatible.")
+	flag.StringVar(&Settings.OutputKafkaConfig.Host, "output-kafka-host", "", "Read request and response stats from Kafka:\n\tgor --input-raw :8080 --output-kafka-host '192.168.0.1:9092,192.168.0.2:9092'")
+	flag.StringVar(&Settings.OutputKafkaConfig.Topic, "output-kafka-topic", "", "Read request and response stats from Kafka:\n\tgor --input-raw :8080 --output-kafka-topic 'kafka-log'")
+	flag.BoolVar(&Settings.OutputKafkaConfig.UseJSON, "output-kafka-json-format", false, "If turned on, it will serialize messages from GoReplay text format to JSON.")
 
-	flag.IntVar(&Settings.outputHTTPConfig.workersMin, "output-http-workers-min", 0, "Gor uses dynamic worker scaling. Enter a number to set a minimum number of workers. default = 1.")
-	flag.IntVar(&Settings.outputHTTPConfig.workersMax, "output-http-workers", 0, "Gor uses dynamic worker scaling. Enter a number to set a maximum number of workers. default = 0 = unlimited.")
-	flag.IntVar(&Settings.outputHTTPConfig.queueLen, "output-http-queue-len", 1000, "Number of requests that can be queued for output, if all workers are busy. default = 1000")
+	flag.StringVar(&Settings.InputKafkaConfig.Host, "input-kafka-host", "", "Send request and response stats to Kafka:\n\tgor --output-stdout --input-kafka-host '192.168.0.1:9092,192.168.0.2:9092'")
+	flag.StringVar(&Settings.InputKafkaConfig.Topic, "input-kafka-topic", "", "Send request and response stats to Kafka:\n\tgor --output-stdout --input-kafka-topic 'kafka-log'")
+	flag.BoolVar(&Settings.InputKafkaConfig.UseJSON, "input-kafka-json-format", false, "If turned on, it will assume that messages coming in JSON format rather than  GoReplay text format.")
 
-	flag.IntVar(&Settings.outputHTTPConfig.redirectLimit, "output-http-redirects", 0, "Enable how often redirects should be followed.")
-	flag.DurationVar(&Settings.outputHTTPConfig.Timeout, "output-http-timeout", 5*time.Second, "Specify HTTP request/response timeout. By default 5s. Example: --output-http-timeout 30s")
-	flag.BoolVar(&Settings.outputHTTPConfig.TrackResponses, "output-http-track-response", false, "If turned on, HTTP output responses will be set to all outputs like stdout, file and etc.")
+	flag.StringVar(&Settings.KafkaTLSConfig.CACert, "kafka-tls-ca-cert", "", "CA certificate for Kafka TLS Config:\n\tgor  --input-raw :3000 --output-kafka-host '192.168.0.1:9092' --output-kafka-topic 'topic' --kafka-tls-ca-cert cacert.cer.pem --kafka-tls-client-cert client.cer.pem --kafka-tls-client-key client.key.pem")
+	flag.StringVar(&Settings.KafkaTLSConfig.clientCert, "kafka-tls-client-cert", "", "Client certificate for Kafka TLS Config (mandatory with to kafka-tls-ca-cert and kafka-tls-client-key)")
+	flag.StringVar(&Settings.KafkaTLSConfig.clientKey, "kafka-tls-client-key", "", "Client Key for Kafka TLS Config (mandatory with to kafka-tls-client-cert and kafka-tls-client-key)")
 
-	flag.BoolVar(&Settings.outputHTTPConfig.stats, "output-http-stats", false, "Report http output queue stats to console every N milliseconds. See output-http-stats-ms")
-	flag.IntVar(&Settings.outputHTTPConfig.statsMs, "output-http-stats-ms", 5000, "Report http output queue stats to console every N milliseconds. default: 5000")
-	flag.BoolVar(&Settings.outputHTTPConfig.OriginalHost, "http-original-host", false, "Normally gor replaces the Host http header with the host supplied with --output-http.  This option disables that behavior, preserving the original Host header.")
-	flag.BoolVar(&Settings.outputHTTPConfig.Debug, "output-http-debug", false, "Enables http debug output.")
+	flag.Var(&Settings.ModifierConfig.Headers, "http-set-header", "Inject additional headers to http reqest:\n\tgor --input-raw :8080 --output-http staging.com --http-set-header 'User-Agent: Gor'")
+	flag.Var(&Settings.ModifierConfig.Headers, "output-http-header", "WARNING: `--output-http-header` DEPRECATED, use `--http-set-header` instead")
 
-	flag.StringVar(&Settings.outputHTTPConfig.elasticSearch, "output-http-elasticsearch", "", "Send request and response stats to ElasticSearch:\n\tgor --input-raw :8080 --output-http staging.com --output-http-elasticsearch 'es_host:api_port/index_name'")
+	flag.Var(&Settings.ModifierConfig.HeaderRewrite, "http-rewrite-header", "Rewrite the request header based on a mapping:\n\tgor --input-raw :8080 --output-http staging.com --http-rewrite-header Host: (.*).example.com,$1.beta.example.com")
 
-	flag.StringVar(&Settings.outputKafkaConfig.host, "output-kafka-host", "", "Read request and response stats from Kafka:\n\tgor --input-raw :8080 --output-kafka-host '192.168.0.1:9092,192.168.0.2:9092'")
-	flag.StringVar(&Settings.outputKafkaConfig.topic, "output-kafka-topic", "", "Read request and response stats from Kafka:\n\tgor --input-raw :8080 --output-kafka-topic 'kafka-log'")
-	flag.BoolVar(&Settings.outputKafkaConfig.useJSON, "output-kafka-json-format", false, "If turned on, it will serialize messages from GoReplay text format to JSON.")
+	flag.Var(&Settings.ModifierConfig.Params, "http-set-param", "Set request url param, if param already exists it will be overwritten:\n\tgor --input-raw :8080 --output-http staging.com --http-set-param api_key=1")
 
-	flag.StringVar(&Settings.inputKafkaConfig.host, "input-kafka-host", "", "Send request and response stats to Kafka:\n\tgor --output-stdout --input-kafka-host '192.168.0.1:9092,192.168.0.2:9092'")
-	flag.StringVar(&Settings.inputKafkaConfig.topic, "input-kafka-topic", "", "Send request and response stats to Kafka:\n\tgor --output-stdout --input-kafka-topic 'kafka-log'")
-	flag.BoolVar(&Settings.inputKafkaConfig.useJSON, "input-kafka-json-format", false, "If turned on, it will assume that messages coming in JSON format rather than  GoReplay text format.")
+	flag.Var(&Settings.ModifierConfig.Methods, "http-allow-method", "Whitelist of HTTP methods to replay. Anything else will be dropped:\n\tgor --input-raw :8080 --output-http staging.com --http-allow-method GET --http-allow-method OPTIONS")
 
-	flag.Var(&Settings.modifierConfig.headers, "http-set-header", "Inject additional headers to http reqest:\n\tgor --input-raw :8080 --output-http staging.com --http-set-header 'User-Agent: Gor'")
-	flag.Var(&Settings.modifierConfig.headers, "output-http-header", "WARNING: `--output-http-header` DEPRECATED, use `--http-set-header` instead")
+	flag.Var(&Settings.ModifierConfig.URLRegexp, "http-allow-url", "A regexp to match requests against. Filter get matched against full url with domain. Anything else will be dropped:\n\t gor --input-raw :8080 --output-http staging.com --http-allow-url ^www.")
 
-	flag.Var(&Settings.modifierConfig.headerRewrite, "http-rewrite-header", "Rewrite the request header based on a mapping:\n\tgor --input-raw :8080 --output-http staging.com --http-rewrite-header Host: (.*).example.com,$1.beta.example.com")
+	flag.Var(&Settings.ModifierConfig.URLNegativeRegexp, "http-disallow-url", "A regexp to match requests against. Filter get matched against full url with domain. Anything else will be forwarded:\n\t gor --input-raw :8080 --output-http staging.com --http-disallow-url ^www.")
 
-	flag.Var(&Settings.modifierConfig.params, "http-set-param", "Set request url param, if param already exists it will be overwritten:\n\tgor --input-raw :8080 --output-http staging.com --http-set-param api_key=1")
+	flag.Var(&Settings.ModifierConfig.URLRewrite, "http-rewrite-url", "Rewrite the request url based on a mapping:\n\tgor --input-raw :8080 --output-http staging.com --http-rewrite-url /v1/user/([^\\/]+)/ping:/v2/user/$1/ping")
+	flag.Var(&Settings.ModifierConfig.URLRewrite, "output-http-rewrite-url", "WARNING: `--output-http-rewrite-url` DEPRECATED, use `--http-rewrite-url` instead")
 
-	flag.Var(&Settings.modifierConfig.methods, "http-allow-method", "Whitelist of HTTP methods to replay. Anything else will be dropped:\n\tgor --input-raw :8080 --output-http staging.com --http-allow-method GET --http-allow-method OPTIONS")
-	flag.Var(&Settings.modifierConfig.methods, "output-http-method", "WARNING: `--output-http-method` DEPRECATED, use `--http-allow-method` instead")
+	flag.Var(&Settings.ModifierConfig.HeaderFilters, "http-allow-header", "A regexp to match a specific header against. Requests with non-matching headers will be dropped:\n\t gor --input-raw :8080 --output-http staging.com --http-allow-header api-version:^v1")
+	flag.Var(&Settings.ModifierConfig.HeaderFilters, "output-http-header-filter", "WARNING: `--output-http-header-filter` DEPRECATED, use `--http-allow-header` instead")
 
-	flag.Var(&Settings.modifierConfig.urlRegexp, "http-allow-url", "A regexp to match requests against. Filter get matched against full url with domain. Anything else will be dropped:\n\t gor --input-raw :8080 --output-http staging.com --http-allow-url ^www.")
-	flag.Var(&Settings.modifierConfig.urlRegexp, "output-http-url-regexp", "WARNING: `--output-http-url-regexp` DEPRECATED, use `--http-allow-url` instead")
+	flag.Var(&Settings.ModifierConfig.HeaderNegativeFilters, "http-disallow-header", "A regexp to match a specific header against. Requests with matching headers will be dropped:\n\t gor --input-raw :8080 --output-http staging.com --http-disallow-header \"User-Agent: Replayed by Gor\"")
 
-	flag.Var(&Settings.modifierConfig.urlNegativeRegexp, "http-disallow-url", "A regexp to match requests against. Filter get matched against full url with domain. Anything else will be forwarded:\n\t gor --input-raw :8080 --output-http staging.com --http-disallow-url ^www.")
+	flag.Var(&Settings.ModifierConfig.HeaderBasicAuthFilters, "http-basic-auth-filter", "A regexp to match the decoded basic auth string against. Requests with non-matching headers will be dropped:\n\t gor --input-raw :8080 --output-http staging.com --http-basic-auth-filter \"^customer[0-9].*\"")
 
-	flag.Var(&Settings.modifierConfig.urlRewrite, "http-rewrite-url", "Rewrite the request url based on a mapping:\n\tgor --input-raw :8080 --output-http staging.com --http-rewrite-url /v1/user/([^\\/]+)/ping:/v2/user/$1/ping")
-	flag.Var(&Settings.modifierConfig.urlRewrite, "output-http-rewrite-url", "WARNING: `--output-http-rewrite-url` DEPRECATED, use `--http-rewrite-url` instead")
+	flag.Var(&Settings.ModifierConfig.HeaderHashFilters, "http-header-limiter", "Takes a fraction of requests, consistently taking or rejecting a request based on the FNV32-1A hash of a specific header:\n\t gor --input-raw :8080 --output-http staging.com --http-header-limiter user-id:25%")
 
-	flag.Var(&Settings.modifierConfig.headerFilters, "http-allow-header", "A regexp to match a specific header against. Requests with non-matching headers will be dropped:\n\t gor --input-raw :8080 --output-http staging.com --http-allow-header api-version:^v1")
-	flag.Var(&Settings.modifierConfig.headerFilters, "output-http-header-filter", "WARNING: `--output-http-header-filter` DEPRECATED, use `--http-allow-header` instead")
+	flag.Var(&Settings.ModifierConfig.HeaderHashFilters, "output-http-header-hash-filter", "WARNING: `output-http-header-hash-filter` DEPRECATED, use `--http-header-hash-limiter` instead")
 
-	flag.Var(&Settings.modifierConfig.headerNegativeFilters, "http-disallow-header", "A regexp to match a specific header against. Requests with matching headers will be dropped:\n\t gor --input-raw :8080 --output-http staging.com --http-disallow-header \"User-Agent: Replayed by Gor\"")
+	flag.Var(&Settings.ModifierConfig.ParamHashFilters, "http-param-limiter", "Takes a fraction of requests, consistently taking or rejecting a request based on the FNV32-1A hash of a specific GET param:\n\t gor --input-raw :8080 --output-http staging.com --http-param-limiter user_id:25%")
 
-	flag.Var(&Settings.modifierConfig.headerBasicAuthFilters, "http-basic-auth-filter", "A regexp to match the decoded basic auth string against. Requests with non-matching headers will be dropped:\n\t gor --input-raw :8080 --output-http staging.com --http-basic-auth-filter \"^customer[0-9].*\"")
+	// default values, using for tests
+	Settings.OutputFileConfig.SizeLimit = 33554432
+	Settings.OutputFileConfig.OutputFileMaxSize = 1099511627776
+	Settings.CopyBufferSize = 5242880
 
-	flag.Var(&Settings.modifierConfig.headerHashFilters, "http-header-limiter", "Takes a fraction of requests, consistently taking or rejecting a request based on the FNV32-1A hash of a specific header:\n\t gor --input-raw :8080 --output-http staging.com --http-header-limiter user-id:25%")
-
-	flag.Var(&Settings.modifierConfig.headerHashFilters, "output-http-header-hash-filter", "WARNING: `output-http-header-hash-filter` DEPRECATED, use `--http-header-hash-limiter` instead")
-
-	flag.Var(&Settings.modifierConfig.paramHashFilters, "http-param-limiter", "Takes a fraction of requests, consistently taking or rejecting a request based on the FNV32-1A hash of a specific GET param:\n\t gor --input-raw :8080 --output-http staging.com --http-param-limiter user_id:25%")
 }
 
-var previousDebugTime int64
+func checkSettings() {
+	if Settings.OutputFileConfig.SizeLimit < 1 {
+		Settings.OutputFileConfig.SizeLimit.Set("32mb")
+	}
+	if Settings.OutputFileConfig.OutputFileMaxSize < 1 {
+		Settings.OutputFileConfig.OutputFileMaxSize.Set("1tb")
+	}
+	if Settings.CopyBufferSize < 1 {
+		Settings.CopyBufferSize.Set("5mb")
+	}
+	// libpcap has bug in mac os x. More info: https://github.com/buger/goreplay/issues/730
+	if Settings.Expire == time.Second*2 && runtime.GOOS == "darwin" {
+		Settings.Expire = time.Second
+	}
+}
+
+var previousDebugTime = time.Now()
 var debugMutex sync.Mutex
 
-// Debug gets called only if --verbose flag specified
-func Debug(args ...interface{}) {
-	if Settings.verbose {
+// Debug take an effect only if --verbose is greater than 0 specified
+func Debug(level int, args ...interface{}) {
+	if Settings.Verbose >= level {
 		debugMutex.Lock()
+		defer debugMutex.Unlock()
 		now := time.Now()
-		diff := float64(now.UnixNano()-previousDebugTime) / 1000000
-		previousDebugTime = now.UnixNano()
-		debugMutex.Unlock()
-
-		fmt.Printf("[DEBUG][PID %d][%d][%fms] ", os.Getpid(), now.UnixNano(), diff)
+		diff := now.Sub(previousDebugTime)
+		previousDebugTime = now
+		fmt.Printf("[DEBUG][elapsed %s]: ", diff)
 		fmt.Println(args...)
 	}
 }

settings_test.go

@@ -0,0 +1,14 @@
+package main
+
+import (
+	"encoding/json"
+	"testing"
+)
+
+func TestAppSettings(t *testing.T) {
+	a := AppSettings{}
+	_, err := json.Marshal(&a)
+	if err != nil {
+		t.Error(err)
+	}
+}

size/size.go

@@ -0,0 +1,64 @@
+package size
+
+import (
+	"fmt"
+	"regexp"
+	"strconv"
+)
+
+// Size represents size that implements flag.Var
+type Size int64
+
+// the following regexes follow Go semantics https://golang.org/ref/spec#Letters_and_digits
+var (
+	rB  = regexp.MustCompile(`(?i)^(?:0b|0x|0o)?[\da-f_]+$`)
+	rKB = regexp.MustCompile(`(?i)^(?:0b|0x|0o)?[\da-f_]+kb$`)
+	rMB = regexp.MustCompile(`(?i)^(?:0b|0x|0o)?[\da-f_]+mb$`)
+	rGB = regexp.MustCompile(`(?i)^(?:0b|0x|0o)?[\da-f_]+gb$`)
+	rTB = regexp.MustCompile(`(?i)^(?:0b|0x|0o)?[\da-f_]+tb$`)
+)
+
+// Set parses size to integer from different bases and data units
+func (siz *Size) Set(size string) (err error) {
+	if size == "" {
+		return
+	}
+	const (
+		_ = 1 << (iota * 10)
+		KB
+		MB
+		GB
+		TB
+	)
+
+	var (
+		lmt = len(size) - 2
+		s   = []byte(size)
+	)
+
+	var _len int64
+	switch {
+	case rB.Match(s):
+		_len, err = strconv.ParseInt(size, 0, 64)
+	case rKB.Match(s):
+		_len, err = strconv.ParseInt(size[:lmt], 0, 64)
+		_len *= KB
+	case rMB.Match(s):
+		_len, err = strconv.ParseInt(size[:lmt], 0, 64)
+		_len *= MB
+	case rGB.Match(s):
+		_len, err = strconv.ParseInt(size[:lmt], 0, 64)
+		_len *= GB
+	case rTB.Match(s):
+		_len, err = strconv.ParseInt(size[:lmt], 0, 64)
+		_len *= TB
+	default:
+		return fmt.Errorf("invalid _len %q", size)
+	}
+	*siz = Size(_len)
+	return
+}
+
+func (siz *Size) String() string {
+	return fmt.Sprintf("%d", *siz)
+}

size/size_test.go

@@ -0,0 +1,29 @@
+package size
+
+import "testing"
+
+func TestParseDataUnit(t *testing.T) {
+	var d = map[string]int{
+		"42mb":                 42 << 20,
+		"4_2":                  42,
+		"00":                   0,
+		"0":                    0,
+		"0_600tb":              384 << 40,
+		"0600Tb":               384 << 40,
+		"0o12Mb":               10 << 20,
+		"0b_10010001111_1kb":   2335 << 10,
+		"1024":                 1 << 10,
+		"0b111":                7,
+		"0x12gB":               18 << 30,
+		"0x_67_7a_2f_cc_40_c6": 113774485586118,
+		"121562380192901":      121562380192901,
+	}
+	var buf Size
+	var err error
+	for k, v := range d {
+		err = buf.Set(k)
+		if err != nil || buf != Size(v) {
+			t.Errorf("Error parsing %s: %v", k, err)
+		}
+	}
+}

tcp/doc.go

@@ -0,0 +1,27 @@
+/*
+Package tcp implements TCP transport layer protocol, it is responsible for
+parsing, reassembling tcp packets, handling communication with engine listeners(github.com/buger/goreplay/capture),
+and reporting errors and statistics of packets.
+the packets are parsed by following TCP way(https://en.wikipedia.org/wiki/Transmission_Control_Protocol#TCP_segment_structure).
+
+
+example:
+
+import "github.com/buger/goreplay/tcp"
+
+messageExpire := time.Second*5
+maxSize := 5 << 20
+
+debugger := func(debugLevel int, data ...interface{}){} // debugger can also be nil
+messageHandler := func(mssg *tcp.Message){}
+
+mssgPool := tcp.NewMessagePool(maxMessageSize, messageExpire, debugger, messageHandler)
+listener.Listen(ctx, mssgPool.Handler)
+
+you can use pool.End or/and pool.Start to set custom session behaviors
+
+debugLevel in debugger function indicates the priority of the logs, the bigger the number the lower
+the priority. errors are signified by debug level 4 for errors, 5 for discarded packets, and 6 for received packets.
+
+*/
+package tcp // import github.com/buger/goreplay/tcp

tcp/tcp_message.go

@@ -0,0 +1,235 @@
+package tcp
+
+import (
+	"bytes"
+	"crypto/sha1"
+	"encoding/hex"
+	"fmt"
+	"sort"
+	"sync"
+	"time"
+
+	"github.com/buger/goreplay/size"
+	"github.com/google/gopacket"
+)
+
+// Stats every message carry its own stats object
+type Stats struct {
+	LostData   int
+	Length     int       // length of the data
+	Start      time.Time // first packet's timestamp
+	End        time.Time // last packet's timestamp
+	SrcAddr    string
+	DstAddr    string
+	IsIncoming bool
+	TimedOut   bool // timeout before getting the whole message
+	Truncated  bool // last packet truncated due to max message size
+	IPversion  byte
+}
+
+// Message is the representation of a tcp message
+type Message struct {
+	packets []*Packet
+	done    chan bool
+	buf     bytes.Buffer
+	Stats
+}
+
+// NewMessage ...
+func NewMessage(srcAddr, dstAddr string, ipVersion uint8) (m *Message) {
+	m = new(Message)
+	m.DstAddr = dstAddr
+	m.SrcAddr = srcAddr
+	m.IPversion = ipVersion
+	m.done = make(chan bool)
+	return
+}
+
+// UUID the unique id of a TCP session it is not granted to be unique!
+func (m *Message) UUID() []byte {
+	var src, dst string
+	if m.IsIncoming {
+		src = m.SrcAddr
+		dst = m.DstAddr
+	} else {
+		src = m.DstAddr
+		dst = m.SrcAddr
+	}
+
+	length := len(src) + len(dst)
+	uuid := make([]byte, length)
+	copy(uuid, src)
+	copy(uuid[len(src):], dst)
+	sha := sha1.Sum(uuid)
+	uuid = make([]byte, 40)
+	hex.Encode(uuid, sha[:])
+
+	return uuid
+}
+
+func (m *Message) add(pckt *Packet) {
+	m.Length += len(pckt.Payload)
+	m.LostData += int(pckt.Lost)
+	m.packets = append(m.packets, pckt)
+	if len(pckt.Payload) > 0 {
+		m.buf.Write(pckt.Payload)
+	}
+	m.End = pckt.Timestamp
+}
+
+// Packets returns packets of this message
+func (m *Message) Packets() []*Packet {
+	return m.packets
+}
+
+// Data returns data in this message
+func (m *Message) Data() []byte {
+	return m.buf.Bytes()
+}
+
+// Sort a helper to sort packets
+func (m *Message) Sort() {
+	sort.SliceStable(m.packets, func(i, j int) bool { return m.packets[i].Seq < m.packets[j].Seq })
+}
+
+// Handler message handler
+type Handler func(*Message)
+
+// Debugger is the debugger function. first params is the indicator of the issue's priority
+// the higher the number, the lower the priority. it can be 4 <= level <= 6.
+type Debugger func(int, ...interface{})
+
+// HintEnd hints the pool to stop the session, see MessagePool.End
+// when set, it will be executed before checking FIN or RST flag
+type HintEnd func(*Message) bool
+
+// HintStart hints the pool to start the reassembling the message, see MessagePool.Start
+// when set, it will be used instead of checking SYN flag
+type HintStart func(*Packet) (IsIncoming, IsOutgoing bool)
+
+// MessagePool holds data of all tcp messages in progress(still receiving/sending packets).
+// message is identified by its source port and dst port, and last 4bytes of src IP.
+type MessagePool struct {
+	sync.Mutex
+	debug         Debugger
+	maxSize       size.Size // maximum message size, default 5mb
+	pool          map[uint64]*Message
+	handler       Handler
+	messageExpire time.Duration // the maximum time to wait for the final packet, minimum is 100ms
+	End           HintEnd
+	Start         HintStart
+}
+
+// NewMessagePool returns a new instance of message pool
+func NewMessagePool(maxSize size.Size, messageExpire time.Duration, debugger Debugger, handler Handler) (pool *MessagePool) {
+	pool = new(MessagePool)
+	pool.debug = debugger
+	pool.handler = handler
+	pool.messageExpire = time.Millisecond * 100
+	if pool.messageExpire < messageExpire {
+		pool.messageExpire = messageExpire
+	}
+	pool.maxSize = maxSize
+	if pool.maxSize < 1 {
+		pool.maxSize = 5 << 20
+	}
+	pool.pool = make(map[uint64]*Message)
+	return pool
+}
+
+// Handler returns packet handler
+func (pool *MessagePool) Handler(packet gopacket.Packet) {
+	var in, out bool
+	pckt, err := ParsePacket(packet)
+	if err != nil || pckt == nil {
+		go pool.say(4, fmt.Sprintf("error decoding packet(%dBytes):%s\n", packet.Metadata().CaptureLength, err))
+		return
+	}
+	pool.Lock()
+	defer pool.Unlock()
+	lst := 3
+	if pckt.Version == 6 {
+		lst = 15
+	}
+	key := uint64(pckt.SrcPort)<<48 | uint64(pckt.DstPort)<<32 |
+		uint64(pckt.SrcIP[lst])<<24 | uint64(pckt.SrcIP[lst-1])<<16 |
+		uint64(pckt.SrcIP[lst-2])<<8 | uint64(pckt.SrcIP[lst-3])
+	m, ok := pool.pool[key]
+	if pckt.RST {
+		if ok {
+			m.done <- true
+			<-m.done
+		}
+		key = uint64(pckt.DstPort)<<48 | uint64(pckt.SrcPort)<<32 |
+			uint64(pckt.DstIP[lst])<<24 | uint64(pckt.DstIP[lst-1])<<16 |
+			uint64(pckt.DstIP[lst-2])<<8 | uint64(pckt.DstIP[lst-3])
+		m, ok = pool.pool[key]
+		if ok {
+			m.done <- true
+			<-m.done
+		}
+		go pool.say(4, fmt.Sprintf("RST flag from %s to %s at %s\n", pckt.Src(), pckt.Dst(), pckt.Timestamp))
+		return
+	}
+	switch {
+	case ok:
+		pool.addPacket(m, pckt)
+		return
+	case pckt.SYN:
+		in = !pckt.ACK
+	case pool.Start != nil:
+		if in, out = pool.Start(pckt); !(in || out) {
+			return
+		}
+	default:
+		return
+	}
+	m = NewMessage(pckt.Src(), pckt.Dst(), pckt.Version)
+	m.IsIncoming = in
+	pool.pool[key] = m
+	m.Start = pckt.Timestamp
+	go pool.dispatch(key, m)
+	pool.addPacket(m, pckt)
+}
+
+func (pool *MessagePool) dispatch(key uint64, m *Message) {
+	select {
+	case <-m.done:
+		defer func() { m.done <- true }() // signal that message was dispatched
+	case <-time.After(pool.messageExpire):
+		pool.Lock()
+		defer pool.Unlock()
+		// avoid dispathing message twice
+		if _, ok := pool.pool[key]; !ok {
+			return
+		}
+		m.TimedOut = true
+	}
+	delete(pool.pool, key)
+	pool.handler(m)
+}
+
+func (pool *MessagePool) addPacket(m *Message, pckt *Packet) {
+	trunc := m.Length + len(pckt.Payload) - int(pool.maxSize)
+	if trunc > 0 {
+		m.Truncated = true
+		pckt.Payload = pckt.Payload[:int(pool.maxSize)-m.Length]
+	}
+	m.add(pckt)
+	switch {
+	case trunc >= 0:
+	case pckt.FIN:
+	case pool.End != nil && pool.End(m):
+	default:
+		return
+	}
+	m.done <- true
+	<-m.done
+}
+
+// this function should not block other pool operations
+func (pool *MessagePool) say(level int, args ...interface{}) {
+	if pool.debug != nil {
+		pool.debug(level, args...)
+	}
+}

tcp/tcp_packet.go

@@ -0,0 +1,197 @@
+package tcp
+
+import (
+	"encoding/binary"
+	"fmt"
+	"net"
+	"time"
+
+	"github.com/google/gopacket"
+	"github.com/google/gopacket/layers"
+)
+
+/*
+Packet represent data and layers of packet.
+parser extracts information from pcap Packet. functions of *Packet doesn't validate if packet is nil,
+calllers must make sure that ParsePacket has'nt returned any error before calling any other
+function.
+*/
+type Packet struct {
+	// Link layer
+	gopacket.LinkLayer
+
+	// IP Header
+	Version uint8 // Ip version
+	SrcIP   net.IP
+	DstIP   net.IP
+	IHL     uint8
+	Length  uint16
+
+	// TCP Segment Header
+	*layers.TCP
+
+	// Data info
+	Lost      uint16
+	Timestamp time.Time
+}
+
+// ParsePacket parse raw packets
+func ParsePacket(packet gopacket.Packet) (pckt *Packet, err error) {
+	// early check of error
+	if packet == nil {
+		return
+	}
+	defer func() {
+		if packet.ErrorLayer() != nil {
+			err = packet.ErrorLayer().Error()
+			return
+		}
+	}()
+
+	// initialization
+	pckt = new(Packet)
+	pckt.Timestamp = packet.Metadata().Timestamp
+	if pckt.Timestamp.IsZero() {
+		pckt.Timestamp = time.Now()
+	}
+
+	// parsing link layer
+	pckt.LinkLayer = packet.LinkLayer()
+
+	// parsing network layer
+	if net4, ok := packet.NetworkLayer().(*layers.IPv4); ok {
+		pckt.Version = 4
+		pckt.SrcIP = net4.SrcIP
+		pckt.DstIP = net4.DstIP
+		pckt.IHL = net4.IHL * 4
+		pckt.Length = net4.Length
+	} else if net6, ok := packet.NetworkLayer().(*layers.IPv6); ok {
+		pckt.Version = 6
+		pckt.SrcIP = net6.SrcIP
+		pckt.DstIP = net6.DstIP
+		pckt.IHL = 40
+		pckt.Length = net6.Length
+	} else {
+		pckt = nil
+		return
+	}
+
+	// parsing tcp header(transportation layer)
+	if tcp, ok := packet.TransportLayer().(*layers.TCP); ok {
+		pckt.TCP = tcp
+	} else {
+		pckt = nil
+		return
+	}
+	pckt.DataOffset *= 4
+
+	// calculating lost data
+	headerSize := int(uint32(pckt.DataOffset) + uint32(pckt.IHL))
+	if pckt.Version == 6 {
+		headerSize -= 40 // in ipv6 the length of payload doesn't include the IPheader size
+	}
+	pckt.Lost = pckt.Length - uint16(headerSize+len(pckt.Payload))
+
+	return
+}
+
+// Src returns the source socket of a packet
+func (pckt *Packet) Src() string {
+	return fmt.Sprintf("%s:%d", pckt.SrcIP, pckt.SrcPort)
+}
+
+// Dst returns destination socket
+func (pckt *Packet) Dst() string {
+	return fmt.Sprintf("%s:%d", pckt.DstIP, pckt.DstPort)
+}
+
+// SYNOptions returns MSS and windowscale of syn packets
+func (pckt *Packet) SYNOptions() (mss uint16, windowscale byte) {
+	if !pckt.SYN {
+		return
+	}
+	for _, v := range pckt.Options {
+		if v.OptionType == layers.TCPOptionKindMSS {
+			mss = binary.BigEndian.Uint16(v.OptionData)
+			continue
+		}
+		if v.OptionType == layers.TCPOptionKindWindowScale {
+			if v.OptionLength > 0 {
+				windowscale = v.OptionData[0]
+			}
+		}
+	}
+	return
+}
+
+// LinkInfo returns info about the link layer
+func (pckt *Packet) LinkInfo() string {
+	if l, ok := pckt.LinkLayer.(*layers.Ethernet); ok {
+		return fmt.Sprintf(
+			"Source Mac: %s\nDestination Mac: %s\nProtocol: %s",
+			l.SrcMAC,
+			l.DstMAC,
+			l.EthernetType,
+		)
+	}
+	return "<Not Ethernet>"
+}
+
+// Flag returns formatted tcp flags
+func (pckt *Packet) Flag() (flag string) {
+	if pckt.FIN {
+		flag += "FIN, "
+	}
+	if pckt.SYN {
+		flag += "SYN, "
+	}
+	if pckt.RST {
+		flag += "RST, "
+	}
+	if pckt.PSH {
+		flag += "PSH, "
+	}
+	if pckt.ACK {
+		flag += "ACK, "
+	}
+	if pckt.URG {
+		flag += "URG, "
+	}
+	if len(flag) != 0 {
+		return flag[:len(flag)-2]
+	}
+	return flag
+}
+
+// String output for a TCP Packet
+func (pckt *Packet) String() string {
+	return fmt.Sprintf(`Time: %s
+%s
+Source: %s
+Destination: %s
+IHL: %d
+Total Length: %d
+Sequence: %d
+Acknowledgment: %d
+DataOffset: %d
+Window: %d
+Flag: %s
+Options: %s
+Data Size: %d
+Lost Data: %d`,
+		pckt.Timestamp.Format(time.StampNano),
+		pckt.LinkInfo(),
+		pckt.Src(),
+		pckt.Dst(),
+		pckt.IHL,
+		pckt.Length,
+		pckt.Seq,
+		pckt.Ack,
+		pckt.DataOffset,
+		pckt.Window,
+		pckt.Flag(),
+		pckt.Options,
+		len(pckt.Payload),
+		pckt.Lost,
+	)
+}

tcp/tcp_test.go

@@ -0,0 +1,266 @@
+package tcp
+
+import (
+	"bytes"
+	"encoding/binary"
+	"testing"
+	"time"
+
+	"github.com/buger/goreplay/proto"
+	"github.com/google/gopacket"
+	"github.com/google/gopacket/layers"
+)
+
+var decodeOpts = gopacket.DecodeOptions{Lazy: true, NoCopy: true}
+
+func headersIP4(seq uint32, length uint16) (headers [54]byte) {
+	// set ethernet headers
+	binary.BigEndian.PutUint16(headers[12:14], uint16(layers.EthernetTypeIPv4))
+
+	// set ip header
+	ip := headers[14:]
+	copy(ip[0:2], []byte{4<<4 | 5, 0x28<<2 | 0x00})
+	binary.BigEndian.PutUint16(ip[2:4], length+40)
+	ip[9] = uint8(layers.IPProtocolTCP)
+	copy(ip[12:16], []byte{192, 168, 1, 2})
+	copy(ip[16:], []byte{192, 168, 1, 3})
+
+	// set tcp header
+	tcp := ip[20:]
+	binary.BigEndian.PutUint16(tcp[0:2], 45678)
+	binary.BigEndian.PutUint16(tcp[2:4], 8001)
+	tcp[12] = 5 << 4
+	return
+}
+
+func GetPackets(start uint32, _len int, payload []byte) []gopacket.Packet {
+	var packets = make([]gopacket.Packet, _len)
+	for i := start; i < start+uint32(_len); i++ {
+		data := make([]byte, 54+len(payload))
+		h := headersIP4(i, uint16(len(payload)))
+		copy(data, h[:])
+		copy(data[len(h):], payload)
+		packets[i-start] = gopacket.NewPacket(data, layers.LinkTypeEthernet, decodeOpts)
+	}
+	return packets
+}
+
+func TestMessageParserWithHint(t *testing.T) {
+	var mssg = make(chan *Message, 3)
+	pool := NewMessagePool(1<<20, time.Second, nil, func(m *Message) { mssg <- m })
+	pool.Start = func(pckt *Packet) (bool, bool) {
+		return proto.HasRequestTitle(pckt.Payload), proto.HasResponseTitle(pckt.Payload)
+	}
+	pool.End = func(m *Message) bool {
+		return proto.HasFullPayload(m.Data())
+	}
+	packets := GetPackets(1, 30, nil)
+	packets[0].Data()[14:][20:][13] = 2  // SYN flag
+	packets[10].Data()[14:][20:][13] = 2 // SYN flag
+	packets[29].Data()[14:][20:][13] = 1 // FIN flag
+	packets[4] = GetPackets(5, 1, []byte("HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nTransfer-Encoding: chunked\r\n\r\n7"))[0]
+	packets[5] = GetPackets(6, 1, []byte("\r\nMozilla\r\n9\r\nDeveloper\r"))[0]
+	packets[6] = GetPackets(7, 1, []byte("\n7\r\nNetwork\r\n0\r\n\r\n"))[0]
+	packets[14] = GetPackets(5, 1, []byte("POST / HTTP/1.1\r\nContent-Type: text/plain\r\nContent-Length: 23\r\n\r\n"))[0]
+	packets[15] = GetPackets(6, 1, []byte("MozillaDeveloper"))[0]
+	packets[16] = GetPackets(7, 1, []byte("Network"))[0]
+	packets[24] = GetPackets(5, 1, []byte("HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nContent-Length: 0\r\n\r"))[0]
+	for i := 0; i < 30; i++ {
+		pool.Handler(packets[i])
+	}
+	var m *Message
+	select {
+	case <-time.After(time.Second):
+		t.Errorf("can't parse packets fast enough")
+		return
+	case m = <-mssg:
+	}
+	if len(m.packets) != 7 {
+		t.Errorf("expected to have 7 packets got %d", len(m.packets))
+	}
+	if !bytes.HasSuffix(m.Data(), []byte("\n7\r\nNetwork\r\n0\r\n\r\n")) {
+		t.Errorf("expected to %q to have suffix %q", m.Data(), []byte("\n7\r\nNetwork\r\n0\r\n\r\n"))
+	}
+
+	select {
+	case <-time.After(time.Second):
+		t.Errorf("can't parse packets fast enough")
+		return
+	case m = <-mssg:
+	}
+	if len(m.packets) != 7 {
+		t.Errorf("expected to have 7 packets got %d", len(m.packets))
+	}
+	if !bytes.HasSuffix(m.Data(), []byte("Network")) {
+		t.Errorf("expected to %q to have suffix %q", m.Data(), []byte("Network"))
+	}
+
+	select {
+	case <-time.After(time.Second):
+		t.Errorf("can't parse packets fast enough")
+		return
+	case m = <-mssg:
+	}
+	if len(m.packets) != 6 {
+		t.Errorf("expected to have 6 packets got %d", len(m.packets))
+	}
+	if !bytes.HasSuffix(m.Data(), []byte("Content-Length: 0\r\n\r")) {
+		t.Errorf("expected to %q to have suffix %q", m.Data(), []byte("Content-Length: 0\r\n\r"))
+	}
+
+}
+
+func TestMessageParserWithoutHint(t *testing.T) {
+	var mssg = make(chan *Message, 1)
+	var data [63 << 10]byte
+	packets := GetPackets(1, 10, data[:])
+	packets[0].Data()[14:][20:][13] = 2 // SYN flag
+	packets[9].Data()[14:][20:][13] = 1 // FIN flag
+	p := NewMessagePool(63<<10*10, time.Second, nil, func(m *Message) { mssg <- m })
+	for _, v := range packets {
+		p.Handler(v)
+	}
+	var m *Message
+	select {
+	case <-time.After(time.Second):
+		t.Errorf("can't parse packets fast enough")
+		return
+	case m = <-mssg:
+	}
+	if m.Length != 63<<10*10 {
+		t.Errorf("expected %d to equal %d", m.Length, 63<<10*10)
+	}
+}
+
+func TestMessageMaxSizeReached(t *testing.T) {
+	var mssg = make(chan *Message, 2)
+	var data [63 << 10]byte
+	packets := GetPackets(1, 2, data[:])
+	packets = append(packets, GetPackets(3, 1, make([]byte, 63<<10+10))...)
+	packets[0].Data()[14:][20:][13] = 2 // SYN flag
+	packets[2].Data()[14:][20:][13] = 2 // SYN flag
+	packets[2].Data()[14:][15] = 3      // changing address
+	p := NewMessagePool(63<<10+10, time.Second, nil, func(m *Message) { mssg <- m })
+	for _, v := range packets {
+		p.Handler(v)
+	}
+	var m *Message
+	select {
+	case <-time.After(time.Second):
+		t.Errorf("can't parse packets fast enough")
+		return
+	case m = <-mssg:
+	}
+	if m.Length != 63<<10+10 {
+		t.Errorf("expected %d to equal %d", m.Length, 63<<10+10)
+	}
+	if !m.Truncated {
+		t.Error("expected message to be truncated")
+	}
+
+	select {
+	case <-time.After(time.Second):
+		t.Errorf("can't parse packets fast enough")
+		return
+	case m = <-mssg:
+	}
+	if m.Length != 63<<10+10 {
+		t.Errorf("expected %d to equal %d", m.Length, 63<<10+10)
+	}
+	if m.Truncated {
+		t.Error("expected message to not be truncated")
+	}
+}
+
+func TestMessageTimeoutReached(t *testing.T) {
+	var mssg = make(chan *Message, 2)
+	var data [63 << 10]byte
+	packets := GetPackets(1, 2, data[:])
+	packets[0].Data()[14:][20:][13] = 2 // SYN flag
+	p := NewMessagePool(1<<20, 0, nil, func(m *Message) { mssg <- m })
+	p.Handler(packets[0])
+	time.Sleep(time.Millisecond * 200)
+	p.Handler(packets[1])
+	m := <-mssg
+	if m.Length != 63<<10 {
+		t.Errorf("expected %d to equal %d", m.Length, 63<<10)
+	}
+	if !m.TimedOut {
+		t.Error("expected message to be timeout")
+	}
+}
+
+func TestMessageUUID(t *testing.T) {
+	m1 := &Message{}
+	m1.IsIncoming = true
+	m1.SrcAddr = "src"
+	m1.DstAddr = "dst"
+	m2 := &Message{}
+	m2.SrcAddr = "dst"
+	m2.DstAddr = "src"
+	if string(m1.UUID()) != string(m2.UUID()) {
+		t.Errorf("expected %s, to equal %s", m1.UUID(), m2.UUID())
+	}
+}
+
+func BenchmarkPacketParseAndSort(b *testing.B) {
+	m := new(Message)
+	m.packets = make([]*Packet, 100)
+	for i, v := range GetPackets(1, 100, nil) {
+		m.packets[i], _ = ParsePacket(v)
+	}
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		m.Sort()
+	}
+}
+
+func BenchmarkMessageParserWithoutHint(b *testing.B) {
+	var mssg = make(chan *Message, 1)
+	var chunk = []byte("111111111111111111111111111111")
+	packets := GetPackets(1, 1000, chunk)
+	packets[0].Data()[14:][20:][13] = 2      // SYN flag
+	packets[1000-1].Data()[14:][20:][13] = 1 // FIN flag
+	p := NewMessagePool(1<<20, time.Second*2, nil, func(m *Message) {
+		mssg <- m
+	})
+	b.ResetTimer()
+	b.ReportMetric(float64(1000), "packets/op")
+	for i := 0; i < b.N; i++ {
+		for _, v := range packets {
+			p.Handler(v)
+		}
+		<-mssg
+	}
+}
+
+func BenchmarkMessageParserWithHint(b *testing.B) {
+	var buf [1002][]byte
+	var chunk = []byte("1e\r\n111111111111111111111111111111\r\n")
+	buf[0] = []byte("HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nTransfer-Encoding: chunked\r\n\r\n")
+	for i := 1; i < 1000; i++ {
+		buf[i] = chunk
+	}
+	buf[1001] = []byte("0\r\n\r\n")
+	packets := make([]gopacket.Packet, len(buf))
+	for i := 0; i < len(buf); i++ {
+		packets[i] = GetPackets(uint32(i+10), 1, buf[i])[0]
+	}
+	var mssg = make(chan *Message, 1)
+	pool := NewMessagePool(1<<30, time.Second*10, nil, func(m *Message) { mssg <- m })
+	pool.Start = func(pckt *Packet) (bool, bool) {
+		return false, proto.HasResponseTitle(pckt.Payload)
+	}
+	pool.End = func(m *Message) bool {
+		return proto.HasFullPayload(m.Data())
+	}
+	b.ResetTimer()
+	b.ReportMetric(float64(len(packets)), "packets/op")
+	b.ReportMetric(float64(1000), "chunks/op")
+	for i := 0; i < b.N; i++ {
+		for j := range packets {
+			pool.Handler(packets[j])
+		}
+		<-mssg
+	}
+}

tcp_client.go

@@ -0,0 +1,193 @@
+package main
+
+import (
+	"crypto/tls"
+	"io"
+	"net"
+	"runtime/debug"
+	"syscall"
+	"time"
+)
+
+// TCPClientConfig client configuration
+type TCPClientConfig struct {
+	Debug              bool
+	ConnectionTimeout  time.Duration
+	Timeout            time.Duration
+	ResponseBufferSize int
+	Secure             bool
+}
+
+// TCPClient client connection properties
+type TCPClient struct {
+	baseURL        string
+	addr           string
+	conn           net.Conn
+	respBuf        []byte
+	config         *TCPClientConfig
+	redirectsCount int
+}
+
+// NewTCPClient returns new TCPClient
+func NewTCPClient(addr string, config *TCPClientConfig) *TCPClient {
+	if config.Timeout.Nanoseconds() == 0 {
+		config.Timeout = 5 * time.Second
+	}
+
+	config.ConnectionTimeout = config.Timeout
+
+	if config.ResponseBufferSize == 0 {
+		config.ResponseBufferSize = 100 * 1024 // 100kb
+	}
+
+	client := &TCPClient{config: config, addr: addr}
+	client.respBuf = make([]byte, config.ResponseBufferSize)
+
+	return client
+}
+
+// Connect creates a tcp connection of the client
+func (c *TCPClient) Connect() (err error) {
+	c.Disconnect()
+
+	c.conn, err = net.DialTimeout("tcp", c.addr, c.config.ConnectionTimeout)
+
+	if c.config.Secure {
+		tlsConn := tls.Client(c.conn, &tls.Config{InsecureSkipVerify: true})
+
+		if err = tlsConn.Handshake(); err != nil {
+			return
+		}
+
+		c.conn = tlsConn
+	}
+
+	return
+}
+
+// Disconnect closes the client connection
+func (c *TCPClient) Disconnect() {
+	if c.conn != nil {
+		c.conn.Close()
+		c.conn = nil
+		Debug(1, "[TCPClient] Disconnected: ", c.baseURL)
+	}
+}
+
+func (c *TCPClient) isAlive() bool {
+	one := make([]byte, 1)
+
+	// Ready 1 byte from socket without timeout to check if it not closed
+	c.conn.SetReadDeadline(time.Now().Add(time.Millisecond))
+	_, err := c.conn.Read(one)
+
+	if err == nil {
+		return true
+	} else if err == io.EOF {
+		Debug(1, "[TCPClient] connection closed, reconnecting")
+		return false
+	} else if err == syscall.EPIPE {
+		Debug(1, "Detected broken pipe.", err)
+		return false
+	}
+
+	return true
+}
+
+// Send sends data over created tcp connection
+func (c *TCPClient) Send(data []byte) (response []byte, err error) {
+	// Don't exit on panic
+	defer func() {
+		if r := recover(); r != nil {
+			Debug(1, "[TCPClient]", r, string(data))
+
+			if _, ok := r.(error); !ok {
+				Debug(1, "[TCPClient] Failed to send request: ", string(data))
+				Debug(1, "PANIC: pkg:", r, debug.Stack())
+			}
+		}
+	}()
+
+	if c.conn == nil || !c.isAlive() {
+		Debug(1, "[TCPClient] Connecting:", c.baseURL)
+		if err = c.Connect(); err != nil {
+			Debug(1, "[TCPClient] Connection error:", err)
+			return
+		}
+	}
+
+	timeout := time.Now().Add(c.config.Timeout)
+
+	c.conn.SetWriteDeadline(timeout)
+
+	if c.config.Debug {
+		Debug(1, "[TCPClient] Sending:", string(data))
+	}
+
+	if _, err = c.conn.Write(data); err != nil {
+		Debug(1, "[TCPClient] Write error:", err, c.baseURL)
+		return
+	}
+
+	var readBytes, n int
+	var currentChunk []byte
+	timeout = time.Now().Add(c.config.Timeout)
+
+	for {
+		c.conn.SetReadDeadline(timeout)
+
+		if readBytes < len(c.respBuf) {
+			n, err = c.conn.Read(c.respBuf[readBytes:])
+			readBytes += n
+
+			if err != nil {
+				if err == io.EOF {
+					err = nil
+				}
+				break
+			}
+		} else {
+			if currentChunk == nil {
+				currentChunk = make([]byte, readChunkSize)
+			}
+
+			n, err = c.conn.Read(currentChunk)
+
+			if err == io.EOF {
+				break
+			} else if err != nil {
+				Debug(1, "[TCPClient] Read the whole body error:", err, c.baseURL)
+				break
+			}
+
+			readBytes += int(n)
+		}
+
+		if readBytes >= maxResponseSize {
+			Debug(1, "[TCPClient] Body is more than the max size", maxResponseSize,
+				c.baseURL)
+			break
+		}
+
+		// For following chunks expect less timeout
+		timeout = time.Now().Add(c.config.Timeout / 5)
+	}
+
+	if err != nil {
+		Debug(1, "[TCPClient] Response read error", err, c.conn, readBytes)
+		return
+	}
+
+	if readBytes > len(c.respBuf) {
+		readBytes = len(c.respBuf)
+	}
+
+	payload := make([]byte, readBytes)
+	copy(payload, c.respBuf[:readBytes])
+
+	if c.config.Debug {
+		Debug(1, "[TCPClient] Received:", string(payload))
+	}
+
+	return payload, err
+}

test_input.go

@@ -1,41 +1,56 @@
 package main
 
 import (
-	"crypto/rand"
 	"encoding/base64"
+	"errors"
+	"math/rand"
 	"time"
 )
 
+// ErrorStopped is the error returned when the go routines reading the input is stopped.
+var ErrorStopped = errors.New("reading stopped")
+
 // TestInput used for testing purpose, it allows emitting requests on demand
 type TestInput struct {
 	data       chan []byte
 	skipHeader bool
+	stop       chan bool // Channel used only to indicate goroutine should shutdown
 }
 
 // NewTestInput constructor for TestInput
 func NewTestInput() (i *TestInput) {
 	i = new(TestInput)
 	i.data = make(chan []byte, 100)
-
+	i.stop = make(chan bool)
 	return
 }
 
 func (i *TestInput) Read(data []byte) (int, error) {
-	buf := <-i.data
+	select {
+	case buf := <-i.data:
+		var header []byte
 
-	var header []byte
+		if !i.skipHeader {
+			header = payloadHeader(RequestPayload, uuid(), time.Now().UnixNano(), -1)
+			copy(data[0:len(header)], header)
+			copy(data[len(header):], buf)
+		} else {
+			copy(data, buf)
+		}
 
-	if !i.skipHeader {
-		header = payloadHeader(RequestPayload, uuid(), time.Now().UnixNano(), -1)
-		copy(data[0:len(header)], header)
-		copy(data[len(header):], buf)
-	} else {
-		copy(data, buf)
+		return len(buf) + len(header), nil
+	case <-i.stop:
+		return 0, ErrorStopped
 	}
-
-	return len(buf) + len(header), nil
 }
 
+// Close closes this plugin
+func (i *TestInput) Close() error {
+	close(i.stop)
+	return nil
+}
+
+// EmitBytes sends data
 func (i *TestInput) EmitBytes(data []byte) {
 	i.data <- data
 }
@@ -63,8 +78,7 @@ func (i *TestInput) EmitLargePOST() {
 
 	rs := base64.URLEncoding.EncodeToString(rb)
 
-	i.data <- []byte("POST / HTTP/1.1\nHost: www.w3.org\nContent-Length:5242880\r\n\r\n" + rs)
-	Debug("Sent large POST")
+	i.data <- []byte("POST / HTTP/1.1\r\nHost: www.w3.org\nContent-Length:5242880\r\n\r\n" + rs)
 }
 
 // EmitSizedPOST emit a POST with a payload set to a supplied size
@@ -74,13 +88,12 @@ func (i *TestInput) EmitSizedPOST(payloadSize int) {
 
 	rs := base64.URLEncoding.EncodeToString(rb)
 
-	i.data <- []byte("POST / HTTP/1.1\nHost: www.w3.org\nContent-Length:5242880\r\n\r\n" + rs)
-	Debug("Sent large POST")
+	i.data <- []byte("POST / HTTP/1.1\r\nHost: www.w3.org\nContent-Length:5242880\r\n\r\n" + rs)
 }
 
 // EmitOPTIONS emits OPTIONS request, similar to GET
 func (i *TestInput) EmitOPTIONS() {
-	i.data <- []byte("OPTIONS / HTTP/1.1\nHost: www.w3.org\r\n\r\n")
+	i.data <- []byte("OPTIONS / HTTP/1.1\r\nHost: www.w3.org\r\n\r\n")
 }
 
 func (i *TestInput) String() string {

vendor/github.com/Shopify/sarama/.gitignore

@@ -0,0 +1,27 @@
+# Compiled Object files, Static and Dynamic libs (Shared Objects)
+*.o
+*.a
+*.so
+*.test
+
+# Folders
+_obj
+_test
+.vagrant
+
+# Architecture specific extensions/prefixes
+*.[568vq]
+[568vq].out
+
+*.cgo1.go
+*.cgo2.c
+_cgo_defun.c
+_cgo_gotypes.go
+_cgo_export.*
+
+_testmain.go
+
+*.exe
+
+coverage.txt
+profile.out

vendor/github.com/Shopify/sarama/.golangci.yml

@@ -0,0 +1,74 @@
+run:
+  timeout: 5m
+  deadline: 10m
+
+linters-settings:
+  govet:
+    check-shadowing: false
+  golint:
+    min-confidence: 0
+  gocyclo:
+    min-complexity: 99
+  maligned:
+    suggest-new: true
+  dupl:
+    threshold: 100
+  goconst:
+    min-len: 2
+    min-occurrences: 3
+  misspell:
+    locale: US
+  goimports:
+    local-prefixes: github.com/Shopify/sarama
+  gocritic:
+    enabled-tags:
+      - diagnostic
+      - experimental
+      - opinionated
+      - performance
+      - style
+    disabled-checks:
+      - wrapperFunc
+      - ifElseChain
+  funlen:
+    lines: 300
+    statements: 300
+
+linters:
+  disable-all: true
+  enable:
+    - bodyclose
+    - deadcode
+    - depguard
+    - dogsled
+    # - dupl
+    - errcheck
+    - funlen
+    # - gocritic
+    - gocyclo
+    - gofmt
+    - goimports
+    # - golint
+    - gosec
+    # - gosimple
+    - govet
+    # - ineffassign
+    - interfacer
+    # - misspell
+    # - nakedret
+    # - scopelint
+    # - staticcheck
+    - structcheck
+    # - stylecheck
+    - typecheck
+    - unconvert
+    - unused
+    - varcheck
+    - whitespace
+    # - goconst
+    # - gochecknoinits
+
+issues:
+  exclude:
+    - consider giving a name to these results
+    - include an explanation for nolint directive

vendor/github.com/Shopify/sarama/CHANGELOG.md

@@ -1,7 +1,569 @@
 # Changelog
 
+#### Unreleased
+
+#### Version 1.26.1 (2020-02-04)
+
+Improvements:
+- Add requests-in-flight metric ([1539](https://github.com/Shopify/sarama/pull/1539))
+- Fix misleading example for cluster admin ([1595](https://github.com/Shopify/sarama/pull/1595))
+- Replace Travis with GitHub Actions, linters housekeeping ([1573](https://github.com/Shopify/sarama/pull/1573))
+- Allow BalanceStrategy to provide custom assignment data ([1592](https://github.com/Shopify/sarama/pull/1592))
+
+Bug Fixes:
+- Adds back Consumer.Offsets.CommitInterval to fix API ([1590](https://github.com/Shopify/sarama/pull/1590))
+- Fix error message s/CommitInterval/AutoCommit.Interval ([1589](https://github.com/Shopify/sarama/pull/1589))
+
+#### Version 1.26.0 (2020-01-24)
+
+New Features:
+- Enable zstd compression
+  ([1574](https://github.com/Shopify/sarama/pull/1574),
+  [1582](https://github.com/Shopify/sarama/pull/1582))
+- Support headers in tools kafka-console-producer
+  ([1549](https://github.com/Shopify/sarama/pull/1549))
+
+Improvements:
+- Add SASL AuthIdentity to SASL frames (authzid)
+  ([1585](https://github.com/Shopify/sarama/pull/1585)).
+
+Bug Fixes:
+- Sending messages with ZStd compression enabled fails in multiple ways
+  ([1252](https://github.com/Shopify/sarama/issues/1252)).
+- Use the broker for any admin on BrokerConfig
+  ([1571](https://github.com/Shopify/sarama/pull/1571)).
+- Set DescribeConfigRequest Version field
+  ([1576](https://github.com/Shopify/sarama/pull/1576)).
+- ConsumerGroup flooding logs with client/metadata update req
+  ([1578](https://github.com/Shopify/sarama/pull/1578)).
+- MetadataRequest version in DescribeCluster
+  ([1580](https://github.com/Shopify/sarama/pull/1580)).
+- Fix deadlock in consumer group handleError
+  ([1581](https://github.com/Shopify/sarama/pull/1581))
+- Fill in the Fetch{Request,Response} protocol
+  ([1582](https://github.com/Shopify/sarama/pull/1582)).
+- Retry topic request on ControllerNotAvailable
+  ([1586](https://github.com/Shopify/sarama/pull/1586)).
+
+#### Version 1.25.0 (2020-01-13)
+
+New Features:
+- Support TLS protocol in kafka-producer-performance
+  ([1538](https://github.com/Shopify/sarama/pull/1538)).
+- Add support for kafka 2.4.0
+  ([1552](https://github.com/Shopify/sarama/pull/1552)).
+
+Improvements:
+- Allow the Consumer to disable auto-commit offsets
+  ([1164](https://github.com/Shopify/sarama/pull/1164)).
+- Produce records with consistent timestamps
+  ([1455](https://github.com/Shopify/sarama/pull/1455)).
+
+Bug Fixes:
+- Fix incorrect SetTopicMetadata name mentions
+  ([1534](https://github.com/Shopify/sarama/pull/1534)).
+- Fix client.tryRefreshMetadata Println
+  ([1535](https://github.com/Shopify/sarama/pull/1535)).
+- Fix panic on calling updateMetadata on closed client
+  ([1531](https://github.com/Shopify/sarama/pull/1531)).
+- Fix possible faulty metrics in TestFuncProducing
+  ([1545](https://github.com/Shopify/sarama/pull/1545)).
+
+#### Version 1.24.1 (2019-10-31)
+
+New Features:
+- Add DescribeLogDirs Request/Response pair
+  ([1520](https://github.com/Shopify/sarama/pull/1520)).
+
+Bug Fixes:
+- Fix ClusterAdmin returning invalid controller ID on DescribeCluster
+  ([1518](https://github.com/Shopify/sarama/pull/1518)).
+- Fix issue with consumergroup not rebalancing when new partition is added
+  ([1525](https://github.com/Shopify/sarama/pull/1525)).
+- Ensure consistent use of read/write deadlines
+  ([1529](https://github.com/Shopify/sarama/pull/1529)).
+
+#### Version 1.24.0 (2019-10-09)
+
+New Features:
+- Add sticky partition assignor
+  ([1416](https://github.com/Shopify/sarama/pull/1416)).
+- Switch from cgo zstd package to pure Go implementation
+  ([1477](https://github.com/Shopify/sarama/pull/1477)).
+
+Improvements:
+- Allow creating ClusterAdmin from client
+  ([1415](https://github.com/Shopify/sarama/pull/1415)).
+- Set KafkaVersion in ListAcls method
+  ([1452](https://github.com/Shopify/sarama/pull/1452)).
+- Set request version in CreateACL ClusterAdmin method
+  ([1458](https://github.com/Shopify/sarama/pull/1458)).
+- Set request version in DeleteACL ClusterAdmin method
+  ([1461](https://github.com/Shopify/sarama/pull/1461)).
+- Handle missed error codes on TopicMetaDataRequest and GroupCoordinatorRequest
+  ([1464](https://github.com/Shopify/sarama/pull/1464)).
+- Remove direct usage of gofork
+  ([1465](https://github.com/Shopify/sarama/pull/1465)).
+- Add support for Go 1.13
+  ([1478](https://github.com/Shopify/sarama/pull/1478)).
+- Improve behavior of NewMockListAclsResponse
+  ([1481](https://github.com/Shopify/sarama/pull/1481)).
+
+Bug Fixes:
+- Fix race condition in consumergroup example
+  ([1434](https://github.com/Shopify/sarama/pull/1434)).
+- Fix brokerProducer goroutine leak
+  ([1442](https://github.com/Shopify/sarama/pull/1442)).
+- Use released version of lz4 library
+  ([1469](https://github.com/Shopify/sarama/pull/1469)).
+- Set correct version in MockDeleteTopicsResponse
+  ([1484](https://github.com/Shopify/sarama/pull/1484)).
+- Fix CLI help message typo
+  ([1494](https://github.com/Shopify/sarama/pull/1494)).
+
+Known Issues:
+- Please **don't** use Zstd, as it doesn't work right now.
+  See https://github.com/Shopify/sarama/issues/1252
+
+#### Version 1.23.1 (2019-07-22)
+
+Bug Fixes:
+- Fix fetch delete bug record
+  ([1425](https://github.com/Shopify/sarama/pull/1425)).
+- Handle SASL/OAUTHBEARER token rejection
+  ([1428](https://github.com/Shopify/sarama/pull/1428)).
+
+#### Version 1.23.0 (2019-07-02)
+
+New Features:
+- Add support for Kafka 2.3.0
+  ([1418](https://github.com/Shopify/sarama/pull/1418)).
+- Add support for ListConsumerGroupOffsets v2
+  ([1374](https://github.com/Shopify/sarama/pull/1374)).
+- Add support for DeleteConsumerGroup
+  ([1417](https://github.com/Shopify/sarama/pull/1417)).
+- Add support for SASLVersion configuration
+  ([1410](https://github.com/Shopify/sarama/pull/1410)).
+- Add kerberos support
+  ([1366](https://github.com/Shopify/sarama/pull/1366)).
+
+Improvements:
+- Improve sasl_scram_client example
+  ([1406](https://github.com/Shopify/sarama/pull/1406)).
+- Fix shutdown and race-condition in consumer-group example
+  ([1404](https://github.com/Shopify/sarama/pull/1404)).
+- Add support for error codes 77—81
+  ([1397](https://github.com/Shopify/sarama/pull/1397)).
+- Pool internal objects allocated per message
+  ([1385](https://github.com/Shopify/sarama/pull/1385)).
+- Reduce packet decoder allocations
+  ([1373](https://github.com/Shopify/sarama/pull/1373)).
+- Support timeout when fetching metadata
+  ([1359](https://github.com/Shopify/sarama/pull/1359)).
+
+Bug Fixes:
+- Fix fetch size integer overflow
+  ([1376](https://github.com/Shopify/sarama/pull/1376)).
+- Handle and log throttled FetchResponses
+  ([1383](https://github.com/Shopify/sarama/pull/1383)).
+- Refactor misspelled word Resouce to Resource
+  ([1368](https://github.com/Shopify/sarama/pull/1368)).
+
+#### Version 1.22.1 (2019-04-29)
+
+Improvements:
+- Use zstd 1.3.8
+  ([1350](https://github.com/Shopify/sarama/pull/1350)).
+- Add support for SaslHandshakeRequest v1
+  ([1354](https://github.com/Shopify/sarama/pull/1354)).
+
+Bug Fixes:
+- Fix V5 MetadataRequest nullable topics array
+  ([1353](https://github.com/Shopify/sarama/pull/1353)).
+- Use a different SCRAM client for each broker connection
+  ([1349](https://github.com/Shopify/sarama/pull/1349)).
+- Fix AllowAutoTopicCreation for MetadataRequest greater than v3
+  ([1344](https://github.com/Shopify/sarama/pull/1344)).
+
+#### Version 1.22.0 (2019-04-09)
+
+New Features:
+- Add Offline Replicas Operation to Client
+  ([1318](https://github.com/Shopify/sarama/pull/1318)).
+- Allow using proxy when connecting to broker
+  ([1326](https://github.com/Shopify/sarama/pull/1326)).
+- Implement ReadCommitted
+  ([1307](https://github.com/Shopify/sarama/pull/1307)).
+- Add support for Kafka 2.2.0
+  ([1331](https://github.com/Shopify/sarama/pull/1331)).
+- Add SASL SCRAM-SHA-512 and SCRAM-SHA-256 mechanismes
+  ([1331](https://github.com/Shopify/sarama/pull/1295)).
+
+Improvements:
+- Unregister all broker metrics on broker stop
+  ([1232](https://github.com/Shopify/sarama/pull/1232)).
+- Add SCRAM authentication example
+  ([1303](https://github.com/Shopify/sarama/pull/1303)).
+- Add consumergroup examples
+  ([1304](https://github.com/Shopify/sarama/pull/1304)).
+- Expose consumer batch size metric
+  ([1296](https://github.com/Shopify/sarama/pull/1296)).
+- Add TLS options to console producer and consumer
+  ([1300](https://github.com/Shopify/sarama/pull/1300)).
+- Reduce client close bookkeeping
+  ([1297](https://github.com/Shopify/sarama/pull/1297)).
+- Satisfy error interface in create responses
+  ([1154](https://github.com/Shopify/sarama/pull/1154)).
+- Please lint gods
+  ([1346](https://github.com/Shopify/sarama/pull/1346)).
+
+Bug Fixes:
+- Fix multi consumer group instance crash
+  ([1338](https://github.com/Shopify/sarama/pull/1338)).
+- Update lz4 to latest version
+  ([1347](https://github.com/Shopify/sarama/pull/1347)).
+- Retry ErrNotCoordinatorForConsumer in new consumergroup session
+  ([1231](https://github.com/Shopify/sarama/pull/1231)).
+- Fix cleanup error handler
+  ([1332](https://github.com/Shopify/sarama/pull/1332)).
+- Fix rate condition in PartitionConsumer
+  ([1156](https://github.com/Shopify/sarama/pull/1156)).
+
+#### Version 1.21.0 (2019-02-24)
+
+New Features:
+- Add CreateAclRequest, DescribeAclRequest, DeleteAclRequest
+  ([1236](https://github.com/Shopify/sarama/pull/1236)).
+- Add DescribeTopic, DescribeConsumerGroup, ListConsumerGroups, ListConsumerGroupOffsets admin requests
+  ([1178](https://github.com/Shopify/sarama/pull/1178)).
+- Implement SASL/OAUTHBEARER
+  ([1240](https://github.com/Shopify/sarama/pull/1240)).
+
+Improvements:
+- Add Go mod support
+  ([1282](https://github.com/Shopify/sarama/pull/1282)).
+- Add error codes 73—76
+  ([1239](https://github.com/Shopify/sarama/pull/1239)).
+- Add retry backoff function
+  ([1160](https://github.com/Shopify/sarama/pull/1160)).
+- Maintain metadata in the producer even when retries are disabled
+  ([1189](https://github.com/Shopify/sarama/pull/1189)).
+- Include ReplicaAssignment in ListTopics
+  ([1274](https://github.com/Shopify/sarama/pull/1274)).
+- Add producer performance tool
+  ([1222](https://github.com/Shopify/sarama/pull/1222)).
+- Add support LogAppend timestamps
+  ([1258](https://github.com/Shopify/sarama/pull/1258)).
+
+Bug Fixes:
+- Fix potential deadlock when a heartbeat request fails
+  ([1286](https://github.com/Shopify/sarama/pull/1286)).
+- Fix consuming compacted topic
+  ([1227](https://github.com/Shopify/sarama/pull/1227)).
+- Set correct Kafka version for DescribeConfigsRequest v1
+  ([1277](https://github.com/Shopify/sarama/pull/1277)).
+- Update kafka test version
+  ([1273](https://github.com/Shopify/sarama/pull/1273)).
+
+#### Version 1.20.1 (2019-01-10)
+
+New Features:
+- Add optional replica id in offset request
+  ([1100](https://github.com/Shopify/sarama/pull/1100)).
+
+Improvements:
+- Implement DescribeConfigs Request + Response v1 & v2
+  ([1230](https://github.com/Shopify/sarama/pull/1230)).
+- Reuse compression objects
+  ([1185](https://github.com/Shopify/sarama/pull/1185)).
+- Switch from png to svg for GoDoc link in README
+  ([1243](https://github.com/Shopify/sarama/pull/1243)).
+- Fix typo in deprecation notice for FetchResponseBlock.Records
+  ([1242](https://github.com/Shopify/sarama/pull/1242)).
+- Fix typos in consumer metadata response file
+  ([1244](https://github.com/Shopify/sarama/pull/1244)).
+
+Bug Fixes:
+- Revert to individual msg retries for non-idempotent
+  ([1203](https://github.com/Shopify/sarama/pull/1203)).
+- Respect MaxMessageBytes limit for uncompressed messages
+  ([1141](https://github.com/Shopify/sarama/pull/1141)).
+
+#### Version 1.20.0 (2018-12-10)
+
+New Features:
+ - Add support for zstd compression
+   ([#1170](https://github.com/Shopify/sarama/pull/1170)).
+ - Add support for Idempotent Producer
+   ([#1152](https://github.com/Shopify/sarama/pull/1152)).
+ - Add support support for Kafka 2.1.0
+   ([#1229](https://github.com/Shopify/sarama/pull/1229)).
+ - Add support support for OffsetCommit request/response pairs versions v1 to v5
+   ([#1201](https://github.com/Shopify/sarama/pull/1201)).
+ - Add support support for OffsetFetch request/response pair up to version v5
+   ([#1198](https://github.com/Shopify/sarama/pull/1198)).
+
+Improvements:
+ - Export broker's Rack setting
+   ([#1173](https://github.com/Shopify/sarama/pull/1173)).
+ - Always use latest patch version of Go on CI
+   ([#1202](https://github.com/Shopify/sarama/pull/1202)).
+ - Add error codes 61 to 72
+   ([#1195](https://github.com/Shopify/sarama/pull/1195)).
+
+Bug Fixes:
+ - Fix build without cgo
+   ([#1182](https://github.com/Shopify/sarama/pull/1182)).
+ - Fix go vet suggestion in consumer group file
+   ([#1209](https://github.com/Shopify/sarama/pull/1209)).
+ - Fix typos in code and comments
+   ([#1228](https://github.com/Shopify/sarama/pull/1228)).
+
+#### Version 1.19.0 (2018-09-27)
+
+New Features:
+ - Implement a higher-level consumer group
+   ([#1099](https://github.com/Shopify/sarama/pull/1099)).
+
+Improvements:
+ - Add support for Go 1.11
+   ([#1176](https://github.com/Shopify/sarama/pull/1176)).
+
+Bug Fixes:
+ - Fix encoding of `MetadataResponse` with version 2 and higher
+   ([#1174](https://github.com/Shopify/sarama/pull/1174)).
+ - Fix race condition in mock async producer
+   ([#1174](https://github.com/Shopify/sarama/pull/1174)).
+
+#### Version 1.18.0 (2018-09-07)
+
+New Features:
+ - Make `Partitioner.RequiresConsistency` vary per-message
+   ([#1112](https://github.com/Shopify/sarama/pull/1112)).
+ - Add customizable partitioner
+   ([#1118](https://github.com/Shopify/sarama/pull/1118)).
+ - Add `ClusterAdmin` support for `CreateTopic`, `DeleteTopic`, `CreatePartitions`,
+   `DeleteRecords`, `DescribeConfig`, `AlterConfig`, `CreateACL`, `ListAcls`, `DeleteACL`
+   ([#1055](https://github.com/Shopify/sarama/pull/1055)).
+
+Improvements:
+ - Add support for Kafka 2.0.0
+   ([#1149](https://github.com/Shopify/sarama/pull/1149)).
+ - Allow setting `LocalAddr` when dialing an address to support multi-homed hosts
+   ([#1123](https://github.com/Shopify/sarama/pull/1123)).
+ - Simpler offset management
+   ([#1127](https://github.com/Shopify/sarama/pull/1127)).
+
+Bug Fixes:
+ - Fix mutation of `ProducerMessage.MetaData` when producing to Kafka
+   ([#1110](https://github.com/Shopify/sarama/pull/1110)).
+ - Fix consumer block when response did not contain all the
+   expected topic/partition blocks
+   ([#1086](https://github.com/Shopify/sarama/pull/1086)).
+ - Fix consumer block when response contains only constrol messages
+   ([#1115](https://github.com/Shopify/sarama/pull/1115)).
+ - Add timeout config for ClusterAdmin requests
+   ([#1142](https://github.com/Shopify/sarama/pull/1142)).
+ - Add version check when producing message with headers
+   ([#1117](https://github.com/Shopify/sarama/pull/1117)).
+ - Fix `MetadataRequest` for empty list of topics
+   ([#1132](https://github.com/Shopify/sarama/pull/1132)).
+ - Fix producer topic metadata on-demand fetch when topic error happens in metadata response
+   ([#1125](https://github.com/Shopify/sarama/pull/1125)).
+
+#### Version 1.17.0 (2018-05-30)
+
+New Features:
+ - Add support for gzip compression levels
+   ([#1044](https://github.com/Shopify/sarama/pull/1044)).
+ - Add support for Metadata request/response pairs versions v1 to v5
+   ([#1047](https://github.com/Shopify/sarama/pull/1047),
+    [#1069](https://github.com/Shopify/sarama/pull/1069)).
+ - Add versioning to JoinGroup request/response pairs
+   ([#1098](https://github.com/Shopify/sarama/pull/1098))
+ - Add support for CreatePartitions, DeleteGroups, DeleteRecords request/response pairs
+   ([#1065](https://github.com/Shopify/sarama/pull/1065),
+    [#1096](https://github.com/Shopify/sarama/pull/1096),
+    [#1027](https://github.com/Shopify/sarama/pull/1027)).
+ - Add `Controller()` method to Client interface
+   ([#1063](https://github.com/Shopify/sarama/pull/1063)).
+
+Improvements:
+ - ConsumerMetadataReq/Resp has been migrated to FindCoordinatorReq/Resp
+   ([#1010](https://github.com/Shopify/sarama/pull/1010)).
+ - Expose missing protocol parts: `msgSet` and `recordBatch`
+   ([#1049](https://github.com/Shopify/sarama/pull/1049)).
+ - Add support for v1 DeleteTopics Request
+   ([#1052](https://github.com/Shopify/sarama/pull/1052)).
+ - Add support for Go 1.10
+   ([#1064](https://github.com/Shopify/sarama/pull/1064)).
+ - Claim support for Kafka 1.1.0
+   ([#1073](https://github.com/Shopify/sarama/pull/1073)).
+
+Bug Fixes:
+ - Fix FindCoordinatorResponse.encode to allow nil Coordinator
+   ([#1050](https://github.com/Shopify/sarama/pull/1050),
+    [#1051](https://github.com/Shopify/sarama/pull/1051)).
+ - Clear all metadata when we have the latest topic info
+   ([#1033](https://github.com/Shopify/sarama/pull/1033)).
+ - Make `PartitionConsumer.Close` idempotent
+   ([#1092](https://github.com/Shopify/sarama/pull/1092)).
+
+#### Version 1.16.0 (2018-02-12)
+
+New Features:
+ - Add support for the Create/Delete Topics request/response pairs
+   ([#1007](https://github.com/Shopify/sarama/pull/1007),
+    [#1008](https://github.com/Shopify/sarama/pull/1008)).
+ - Add support for the Describe/Create/Delete ACL request/response pairs
+   ([#1009](https://github.com/Shopify/sarama/pull/1009)).
+ - Add support for the five transaction-related request/response pairs
+   ([#1016](https://github.com/Shopify/sarama/pull/1016)).
+
+Improvements:
+ - Permit setting version on mock producer responses
+   ([#999](https://github.com/Shopify/sarama/pull/999)).
+ - Add `NewMockBrokerListener` helper for testing TLS connections
+   ([#1019](https://github.com/Shopify/sarama/pull/1019)).
+ - Changed the default value for `Consumer.Fetch.Default` from 32KiB to 1MiB
+   which results in much higher throughput in most cases
+   ([#1024](https://github.com/Shopify/sarama/pull/1024)).
+ - Reuse the `time.Ticker` across fetch requests in the PartitionConsumer to
+   reduce CPU and memory usage when processing many partitions
+   ([#1028](https://github.com/Shopify/sarama/pull/1028)).
+ - Assign relative offsets to messages in the producer to save the brokers a
+   recompression pass
+   ([#1002](https://github.com/Shopify/sarama/pull/1002),
+    [#1015](https://github.com/Shopify/sarama/pull/1015)).
+
+Bug Fixes:
+ - Fix producing uncompressed batches with the new protocol format
+   ([#1032](https://github.com/Shopify/sarama/issues/1032)).
+ - Fix consuming compacted topics with the new protocol format
+   ([#1005](https://github.com/Shopify/sarama/issues/1005)).
+ - Fix consuming topics with a mix of protocol formats
+   ([#1021](https://github.com/Shopify/sarama/issues/1021)).
+ - Fix consuming when the broker includes multiple batches in a single response
+   ([#1022](https://github.com/Shopify/sarama/issues/1022)).
+ - Fix detection of `PartialTrailingMessage` when the partial message was
+   truncated before the magic value indicating its version
+   ([#1030](https://github.com/Shopify/sarama/pull/1030)).
+ - Fix expectation-checking in the mock of `SyncProducer.SendMessages`
+   ([#1035](https://github.com/Shopify/sarama/pull/1035)).
+
+#### Version 1.15.0 (2017-12-08)
+
+New Features:
+ - Claim official support for Kafka 1.0, though it did already work
+   ([#984](https://github.com/Shopify/sarama/pull/984)).
+ - Helper methods for Kafka version numbers to/from strings
+   ([#989](https://github.com/Shopify/sarama/pull/989)).
+ - Implement CreatePartitions request/response
+   ([#985](https://github.com/Shopify/sarama/pull/985)).
+
+Improvements:
+ - Add error codes 45-60
+   ([#986](https://github.com/Shopify/sarama/issues/986)).
+
+Bug Fixes:
+ - Fix slow consuming for certain Kafka 0.11/1.0 configurations
+   ([#982](https://github.com/Shopify/sarama/pull/982)).
+ - Correctly determine when a FetchResponse contains the new message format
+   ([#990](https://github.com/Shopify/sarama/pull/990)).
+ - Fix producing with multiple headers
+   ([#996](https://github.com/Shopify/sarama/pull/996)).
+ - Fix handling of truncated record batches
+   ([#998](https://github.com/Shopify/sarama/pull/998)).
+ - Fix leaking metrics when closing brokers
+   ([#991](https://github.com/Shopify/sarama/pull/991)).
+
+#### Version 1.14.0 (2017-11-13)
+
+New Features:
+ - Add support for the new Kafka 0.11 record-batch format, including the wire
+   protocol and the necessary behavioural changes in the producer and consumer.
+   Transactions and idempotency are not yet supported, but producing and
+   consuming should work with all the existing bells and whistles (batching,
+   compression, etc) as well as the new custom headers. Thanks to Vlad Hanciuta
+   of Arista Networks for this work. Part of
+   ([#901](https://github.com/Shopify/sarama/issues/901)).
+
+Bug Fixes:
+ - Fix encoding of ProduceResponse versions in test
+   ([#970](https://github.com/Shopify/sarama/pull/970)).
+ - Return partial replicas list when we have it
+   ([#975](https://github.com/Shopify/sarama/pull/975)).
+
+#### Version 1.13.0 (2017-10-04)
+
+New Features:
+ - Support for FetchRequest version 3
+   ([#905](https://github.com/Shopify/sarama/pull/905)).
+ - Permit setting version on mock FetchResponses
+   ([#939](https://github.com/Shopify/sarama/pull/939)).
+ - Add a configuration option to support storing only minimal metadata for
+   extremely large clusters
+   ([#937](https://github.com/Shopify/sarama/pull/937)).
+ - Add `PartitionOffsetManager.ResetOffset` for backtracking tracked offsets
+   ([#932](https://github.com/Shopify/sarama/pull/932)).
+
+Improvements:
+ - Provide the block-level timestamp when consuming compressed messages
+   ([#885](https://github.com/Shopify/sarama/issues/885)).
+ - `Client.Replicas` and `Client.InSyncReplicas` now respect the order returned
+   by the broker, which can be meaningful
+   ([#930](https://github.com/Shopify/sarama/pull/930)).
+ - Use a `Ticker` to reduce consumer timer overhead at the cost of higher
+   variance in the actual timeout
+   ([#933](https://github.com/Shopify/sarama/pull/933)).
+
+Bug Fixes:
+ - Gracefully handle messages with negative timestamps
+   ([#907](https://github.com/Shopify/sarama/pull/907)).
+ - Raise a proper error when encountering an unknown message version
+   ([#940](https://github.com/Shopify/sarama/pull/940)).
+
+#### Version 1.12.0 (2017-05-08)
+
+New Features:
+ - Added support for the `ApiVersions` request and response pair, and Kafka
+   version 0.10.2 ([#867](https://github.com/Shopify/sarama/pull/867)). Note
+   that you still need to specify the Kafka version in the Sarama configuration
+   for the time being.
+ - Added a `Brokers` method to the Client which returns the complete set of
+   active brokers ([#813](https://github.com/Shopify/sarama/pull/813)).
+ - Added an `InSyncReplicas` method to the Client which returns the set of all
+   in-sync broker IDs for the given partition, now that the Kafka versions for
+   which this was misleading are no longer in our supported set
+   ([#872](https://github.com/Shopify/sarama/pull/872)).
+ - Added a `NewCustomHashPartitioner` method which allows constructing a hash
+   partitioner with a custom hash method in case the default (FNV-1a) is not
+   suitable
+   ([#837](https://github.com/Shopify/sarama/pull/837),
+    [#841](https://github.com/Shopify/sarama/pull/841)).
+
+Improvements:
+ - Recognize more Kafka error codes
+   ([#859](https://github.com/Shopify/sarama/pull/859)).
+
+Bug Fixes:
+ - Fix an issue where decoding a malformed FetchRequest would not return the
+   correct error ([#818](https://github.com/Shopify/sarama/pull/818)).
+ - Respect ordering of group protocols in JoinGroupRequests. This fix is
+   transparent if you're using the `AddGroupProtocol` or
+   `AddGroupProtocolMetadata` helpers; otherwise you will need to switch from
+   the `GroupProtocols` field (now deprecated) to use `OrderedGroupProtocols`
+   ([#812](https://github.com/Shopify/sarama/issues/812)).
+ - Fix an alignment-related issue with atomics on 32-bit architectures
+   ([#859](https://github.com/Shopify/sarama/pull/859)).
+
 #### Version 1.11.0 (2016-12-20)
 
+_Important:_ As of Sarama 1.11 it is necessary to set the config value of
+`Producer.Return.Successes` to true in order to use the SyncProducer. Previous
+versions would silently override this value when instantiating a SyncProducer
+which led to unexpected values and data races.
+
 New Features:
  - Metrics! Thanks to Sébastien Launay for all his work on this feature
    ([#701](https://github.com/Shopify/sarama/pull/701),

vendor/github.com/Shopify/sarama/LICENSE

@@ -1,4 +1,4 @@
-Copyright (c) 2013 Evan Huus
+Copyright (c) 2013 Shopify
 
 Permission is hereby granted, free of charge, to any person obtaining
 a copy of this software and associated documentation files (the

vendor/github.com/Shopify/sarama/Makefile

@@ -1,21 +1,27 @@
-default: fmt vet errcheck test
+default: fmt get update test lint
 
-test:
-	go test -v -timeout 60s -race ./...
+GO       := GO111MODULE=on GOPRIVATE=github.com/linkedin GOSUMDB=off go
+GOBUILD  := CGO_ENABLED=0 $(GO) build $(BUILD_FLAG)
+GOTEST   := $(GO) test -gcflags='-l' -p 3 -v -race -timeout 6m -coverprofile=profile.out -covermode=atomic
 
-vet:
-	go vet ./...
-
-errcheck:
-	errcheck github.com/Shopify/sarama/...
-
-fmt:
-	@if [ -n "$$(go fmt ./...)" ]; then echo 'Please run go fmt on your code.' && exit 1; fi
-
-install_dependencies: install_errcheck get
-
-install_errcheck:
-	go get github.com/kisielk/errcheck
+FILES    := $(shell find . -name '*.go' -type f -not -name '*.pb.go' -not -name '*_generated.go' -not -name '*_test.go')
+TESTS    := $(shell find . -name '*.go' -type f -not -name '*.pb.go' -not -name '*_generated.go' -name '*_test.go')
 
 get:
-	go get -t
+	$(GO) get ./...
+	$(GO) mod verify
+	$(GO) mod tidy
+
+update:
+	$(GO) get -u -v all
+	$(GO) mod verify
+	$(GO) mod tidy
+
+fmt:
+	gofmt -s -l -w $(FILES) $(TESTS)
+
+lint:
+	golangci-lint run
+
+test:
+	$(GOTEST) ./...

vendor/github.com/Shopify/sarama/README.md

@@ -1,36 +1,36 @@
-sarama
-======
+# sarama
 
-[![GoDoc](https://godoc.org/github.com/Shopify/sarama?status.png)](https://godoc.org/github.com/Shopify/sarama)
+[![GoDoc](https://godoc.org/github.com/Shopify/sarama?status.svg)](https://godoc.org/github.com/Shopify/sarama)
 [![Build Status](https://travis-ci.org/Shopify/sarama.svg?branch=master)](https://travis-ci.org/Shopify/sarama)
+[![Coverage](https://codecov.io/gh/Shopify/sarama/branch/master/graph/badge.svg)](https://codecov.io/gh/Shopify/sarama)
 
 Sarama is an MIT-licensed Go client library for [Apache Kafka](https://kafka.apache.org/) version 0.8 (and later).
 
-### Getting started
+## Getting started
 
 - API documentation and examples are available via [godoc](https://godoc.org/github.com/Shopify/sarama).
 - Mocks for testing are available in the [mocks](./mocks) subpackage.
 - The [examples](./examples) directory contains more elaborate example applications.
 - The [tools](./tools) directory contains command line tools that can be useful for testing, diagnostics, and instrumentation.
 
-### Compatibility and API stability
+You might also want to look at the [Frequently Asked Questions](https://github.com/Shopify/sarama/wiki/Frequently-Asked-Questions).
+
+## Compatibility and API stability
 
 Sarama provides a "2 releases + 2 months" compatibility guarantee: we support
 the two latest stable releases of Kafka and Go, and we provide a two month
 grace period for older releases. This means we currently officially support
-Go 1.7 and 1.6, and Kafka 0.10.0 and 0.9.0, although older releases are
+Go 1.12 through 1.14, and Kafka 2.1 through 2.4, although older releases are
 still likely to work.
 
 Sarama follows semantic versioning and provides API stability via the gopkg.in service.
 You can import a version with a guaranteed stable API via http://gopkg.in/Shopify/sarama.v1.
 A changelog is available [here](CHANGELOG.md).
 
-### Contributing
+## Contributing
 
-* Get started by checking our [contribution guidelines](https://github.com/Shopify/sarama/blob/master/CONTRIBUTING.md).
-* Read the [Sarama wiki](https://github.com/Shopify/sarama/wiki) for more
-  technical and design details.
-* The [Kafka Protocol Specification](https://cwiki.apache.org/confluence/display/KAFKA/A+Guide+To+The+Kafka+Protocol)
-  contains a wealth of useful information.
-* For more general issues, there is [a google group](https://groups.google.com/forum/#!forum/kafka-clients) for Kafka client developers.
-* If you have any questions, just ask!
+- Get started by checking our [contribution guidelines](https://github.com/Shopify/sarama/blob/master/.github/CONTRIBUTING.md).
+- Read the [Sarama wiki](https://github.com/Shopify/sarama/wiki) for more technical and design details.
+- The [Kafka Protocol Specification](https://cwiki.apache.org/confluence/display/KAFKA/A+Guide+To+The+Kafka+Protocol) contains a wealth of useful information.
+- For more general issues, there is [a google group](https://groups.google.com/forum/#!forum/kafka-clients) for Kafka client developers.
+- If you have any questions, just ask!

vendor/github.com/Shopify/sarama/Vagrantfile

@@ -1,14 +1,8 @@
-# -*- mode: ruby -*-
-# vi: set ft=ruby :
-
-# Vagrantfile API/syntax version. Don't touch unless you know what you're doing!
-VAGRANTFILE_API_VERSION = "2"
-
 # We have 5 * 192MB ZK processes and 5 * 320MB Kafka processes => 2560MB
 MEMORY = 3072
 
-Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
-  config.vm.box = "ubuntu/trusty64"
+Vagrant.configure("2") do |config|
+  config.vm.box = "ubuntu/bionic64"
 
   config.vm.provision :shell, path: "vagrant/provision.sh"
 

vendor/github.com/Shopify/sarama/acl_bindings.go

@@ -0,0 +1,138 @@
+package sarama
+
+//Resource holds information about acl resource type
+type Resource struct {
+	ResourceType        AclResourceType
+	ResourceName        string
+	ResourcePatternType AclResourcePatternType
+}
+
+func (r *Resource) encode(pe packetEncoder, version int16) error {
+	pe.putInt8(int8(r.ResourceType))
+
+	if err := pe.putString(r.ResourceName); err != nil {
+		return err
+	}
+
+	if version == 1 {
+		if r.ResourcePatternType == AclPatternUnknown {
+			Logger.Print("Cannot encode an unknown resource pattern type, using Literal instead")
+			r.ResourcePatternType = AclPatternLiteral
+		}
+		pe.putInt8(int8(r.ResourcePatternType))
+	}
+
+	return nil
+}
+
+func (r *Resource) decode(pd packetDecoder, version int16) (err error) {
+	resourceType, err := pd.getInt8()
+	if err != nil {
+		return err
+	}
+	r.ResourceType = AclResourceType(resourceType)
+
+	if r.ResourceName, err = pd.getString(); err != nil {
+		return err
+	}
+	if version == 1 {
+		pattern, err := pd.getInt8()
+		if err != nil {
+			return err
+		}
+		r.ResourcePatternType = AclResourcePatternType(pattern)
+	}
+
+	return nil
+}
+
+//Acl holds information about acl type
+type Acl struct {
+	Principal      string
+	Host           string
+	Operation      AclOperation
+	PermissionType AclPermissionType
+}
+
+func (a *Acl) encode(pe packetEncoder) error {
+	if err := pe.putString(a.Principal); err != nil {
+		return err
+	}
+
+	if err := pe.putString(a.Host); err != nil {
+		return err
+	}
+
+	pe.putInt8(int8(a.Operation))
+	pe.putInt8(int8(a.PermissionType))
+
+	return nil
+}
+
+func (a *Acl) decode(pd packetDecoder, version int16) (err error) {
+	if a.Principal, err = pd.getString(); err != nil {
+		return err
+	}
+
+	if a.Host, err = pd.getString(); err != nil {
+		return err
+	}
+
+	operation, err := pd.getInt8()
+	if err != nil {
+		return err
+	}
+	a.Operation = AclOperation(operation)
+
+	permissionType, err := pd.getInt8()
+	if err != nil {
+		return err
+	}
+	a.PermissionType = AclPermissionType(permissionType)
+
+	return nil
+}
+
+//ResourceAcls is an acl resource type
+type ResourceAcls struct {
+	Resource
+	Acls []*Acl
+}
+
+func (r *ResourceAcls) encode(pe packetEncoder, version int16) error {
+	if err := r.Resource.encode(pe, version); err != nil {
+		return err
+	}
+
+	if err := pe.putArrayLength(len(r.Acls)); err != nil {
+		return err
+	}
+	for _, acl := range r.Acls {
+		if err := acl.encode(pe); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (r *ResourceAcls) decode(pd packetDecoder, version int16) error {
+	if err := r.Resource.decode(pd, version); err != nil {
+		return err
+	}
+
+	n, err := pd.getArrayLength()
+	if err != nil {
+		return err
+	}
+
+	r.Acls = make([]*Acl, n)
+	for i := 0; i < n; i++ {
+		r.Acls[i] = new(Acl)
+		if err := r.Acls[i].decode(pd, version); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

vendor/github.com/Shopify/sarama/acl_create_request.go

@@ -0,0 +1,89 @@
+package sarama
+
+//CreateAclsRequest is an acl creation request
+type CreateAclsRequest struct {
+	Version      int16
+	AclCreations []*AclCreation
+}
+
+func (c *CreateAclsRequest) encode(pe packetEncoder) error {
+	if err := pe.putArrayLength(len(c.AclCreations)); err != nil {
+		return err
+	}
+
+	for _, aclCreation := range c.AclCreations {
+		if err := aclCreation.encode(pe, c.Version); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (c *CreateAclsRequest) decode(pd packetDecoder, version int16) (err error) {
+	c.Version = version
+	n, err := pd.getArrayLength()
+	if err != nil {
+		return err
+	}
+
+	c.AclCreations = make([]*AclCreation, n)
+
+	for i := 0; i < n; i++ {
+		c.AclCreations[i] = new(AclCreation)
+		if err := c.AclCreations[i].decode(pd, version); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (c *CreateAclsRequest) key() int16 {
+	return 30
+}
+
+func (c *CreateAclsRequest) version() int16 {
+	return c.Version
+}
+
+func (c *CreateAclsRequest) headerVersion() int16 {
+	return 1
+}
+
+func (c *CreateAclsRequest) requiredVersion() KafkaVersion {
+	switch c.Version {
+	case 1:
+		return V2_0_0_0
+	default:
+		return V0_11_0_0
+	}
+}
+
+//AclCreation is a wrapper around Resource and Acl type
+type AclCreation struct {
+	Resource
+	Acl
+}
+
+func (a *AclCreation) encode(pe packetEncoder, version int16) error {
+	if err := a.Resource.encode(pe, version); err != nil {
+		return err
+	}
+	if err := a.Acl.encode(pe); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (a *AclCreation) decode(pd packetDecoder, version int16) (err error) {
+	if err := a.Resource.decode(pd, version); err != nil {
+		return err
+	}
+	if err := a.Acl.decode(pd, version); err != nil {
+		return err
+	}
+
+	return nil
+}

vendor/github.com/Shopify/sarama/acl_create_response.go

@@ -0,0 +1,94 @@
+package sarama
+
+import "time"
+
+//CreateAclsResponse is a an acl response creation type
+type CreateAclsResponse struct {
+	ThrottleTime         time.Duration
+	AclCreationResponses []*AclCreationResponse
+}
+
+func (c *CreateAclsResponse) encode(pe packetEncoder) error {
+	pe.putInt32(int32(c.ThrottleTime / time.Millisecond))
+
+	if err := pe.putArrayLength(len(c.AclCreationResponses)); err != nil {
+		return err
+	}
+
+	for _, aclCreationResponse := range c.AclCreationResponses {
+		if err := aclCreationResponse.encode(pe); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (c *CreateAclsResponse) decode(pd packetDecoder, version int16) (err error) {
+	throttleTime, err := pd.getInt32()
+	if err != nil {
+		return err
+	}
+	c.ThrottleTime = time.Duration(throttleTime) * time.Millisecond
+
+	n, err := pd.getArrayLength()
+	if err != nil {
+		return err
+	}
+
+	c.AclCreationResponses = make([]*AclCreationResponse, n)
+	for i := 0; i < n; i++ {
+		c.AclCreationResponses[i] = new(AclCreationResponse)
+		if err := c.AclCreationResponses[i].decode(pd, version); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (c *CreateAclsResponse) key() int16 {
+	return 30
+}
+
+func (c *CreateAclsResponse) version() int16 {
+	return 0
+}
+
+func (c *CreateAclsResponse) headerVersion() int16 {
+	return 0
+}
+
+func (c *CreateAclsResponse) requiredVersion() KafkaVersion {
+	return V0_11_0_0
+}
+
+//AclCreationResponse is an acl creation response type
+type AclCreationResponse struct {
+	Err    KError
+	ErrMsg *string
+}
+
+func (a *AclCreationResponse) encode(pe packetEncoder) error {
+	pe.putInt16(int16(a.Err))
+
+	if err := pe.putNullableString(a.ErrMsg); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (a *AclCreationResponse) decode(pd packetDecoder, version int16) (err error) {
+	kerr, err := pd.getInt16()
+	if err != nil {
+		return err
+	}
+	a.Err = KError(kerr)
+
+	if a.ErrMsg, err = pd.getNullableString(); err != nil {
+		return err
+	}
+
+	return nil
+}

vendor/github.com/Shopify/sarama/acl_delete_request.go

@@ -0,0 +1,62 @@
+package sarama
+
+//DeleteAclsRequest is a delete acl request
+type DeleteAclsRequest struct {
+	Version int
+	Filters []*AclFilter
+}
+
+func (d *DeleteAclsRequest) encode(pe packetEncoder) error {
+	if err := pe.putArrayLength(len(d.Filters)); err != nil {
+		return err
+	}
+
+	for _, filter := range d.Filters {
+		filter.Version = d.Version
+		if err := filter.encode(pe); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (d *DeleteAclsRequest) decode(pd packetDecoder, version int16) (err error) {
+	d.Version = int(version)
+	n, err := pd.getArrayLength()
+	if err != nil {
+		return err
+	}
+
+	d.Filters = make([]*AclFilter, n)
+	for i := 0; i < n; i++ {
+		d.Filters[i] = new(AclFilter)
+		d.Filters[i].Version = int(version)
+		if err := d.Filters[i].decode(pd, version); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (d *DeleteAclsRequest) key() int16 {
+	return 31
+}
+
+func (d *DeleteAclsRequest) version() int16 {
+	return int16(d.Version)
+}
+
+func (c *DeleteAclsRequest) headerVersion() int16 {
+	return 1
+}
+
+func (d *DeleteAclsRequest) requiredVersion() KafkaVersion {
+	switch d.Version {
+	case 1:
+		return V2_0_0_0
+	default:
+		return V0_11_0_0
+	}
+}