Add AF_PACKET engine

Co-authored-by: Leonid Bugaev <leonsbox@gmail.com>

Added a way to capture multiple ports at the same time, with a single listener. 
Ports should be separated by comma like this: --input-raw :80,8080,3000

.clabot

@@ -0,0 +1,3 @@
+{
+  "contributors": ["buger"]
+}

.deepsource.toml

@@ -0,0 +1,28 @@
+version = 1
+
+exclude_patterns = [
+  "vendor/**"
+]
+
+[[analyzers]]
+name = "go"
+enabled = true
+
+  [analyzers.meta]
+  import_paths = ["github.com/ankitdobhal/goreplay"]
+
+[[analyzers]]
+name = "docker"
+enabled = true
+
+[[analyzers]]
+name = "ruby"
+enabled = true
+
+[[analyzers]]
+name = "javascript"
+enabled = true
+
+  [analyzers.meta]
+  environment = ["nodejs"]
+

.github/dependabot.yaml

@@ -0,0 +1,10 @@
+version: 2
+updates:
+  # Maintain dependencies for GitHub Actions
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+    pull-request-branch-name:
+      # Separate sections of the branch name with a hyphen
+      separator: "-"

.github/workflows/ci-docker.yaml

@@ -0,0 +1,38 @@
+name: ci
+on:
+  release:
+    types: [published]
+jobs:
+  docker-build-and-push:
+    runs-on: ubuntu-latest
+    steps:
+      -
+        name: "Checkout repository"
+        uses: actions/checkout@v2
+      -
+        name: "Set up Docker Buildx"
+        uses: docker/setup-buildx-action@v1
+      -
+        name: "Cache Docker layers"
+        uses: actions/cache@v2
+        with:
+          path: /tmp/.buildx-cache
+          key: ${{ runner.os }}-buildx-${{ github.sha }}
+          restore-keys: |
+            ${{ runner.os }}-buildx-
+      -
+        name: "Login to Container Registry"
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      -
+        name: "Build and push docker image"
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          push: true
+          tags: ${{ github.repository }}:${{ github.event.release.tag_name }}
+          build-args: RELEASE_VERSION=${{ github.event.release.tag_name }}
+          cache-from: type=local,src=/tmp/.buildx-cache
+          cache-to: type=local,mode=max,dest=/tmp/.buildx-cache

.github/workflows/ci-test.yaml

@@ -0,0 +1,29 @@
+name: test
+on: [push, pull_request]
+jobs:
+  test:
+    strategy:
+      matrix:
+        go-version: [1.15.x, 1.16.x] # two latest minor versions
+    runs-on: ubuntu-latest
+    steps:
+    - name: update package index
+      run: sudo apt-get update
+    - name: install libpcap
+      run: sudo apt-get install libpcap-dev -y
+    - name: install Go
+      uses: actions/setup-go@v2
+      with:
+        go-version: ${{ matrix.go-version }}
+    - name: checkout code
+      uses: actions/checkout@v2
+    - uses: actions/cache@v2
+      with:
+        path: |
+          ~/go/pkg/mod              # Module download cache
+          ~/.cache/go-build         # Build cache (Linux)
+        key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
+        restore-keys: |
+          ${{ runner.os }}-go-
+    - name: test
+      run: sudo go test ./... -v -timeout 120s

.gitignore

@@ -1,5 +1,10 @@
 *.swp
 *.gor
+*.rpm
+*.dep
+*.deb
+*.pkg
+*.exe
 
 *.out
 
@@ -21,6 +26,7 @@ gor
 
 .DS_Store
 
+goreplay
 corpus
 crashers
 suppressions

.gitmodules

@@ -1,15 +0,0 @@
-[submodule "vendor/github.com/mattbaird/elastigo"]
-	path = vendor/github.com/mattbaird/elastigo
-	url = https://github.com/mattbaird/elastigo
-[submodule "vendor/github.com/araddon/gou"]
-	path = vendor/github.com/araddon/gou
-	url = https://github.com/araddon/gou
-[submodule "vendor/github.com/bitly/go-hostpool"]
-	path = vendor/github.com/bitly/go-hostpool
-	url = https://github.com/bitly/go-hostpool
-[submodule "vendor/github.com/bmizerany/assert"]
-	path = vendor/github.com/bmizerany/assert
-	url = https://github.com/bmizerany/assert
-[submodule "vendor/github.com/google/gopacket"]
-	path = vendor/github.com/google/gopacket
-	url = https://github.com/google/gopacket

.travis.yml

@@ -1,6 +0,0 @@
-language: go
-go: 1.6
-script: sudo -E bash -c "source /etc/profile && eval '$(gimme 1.6)' && export GOPATH=$HOME/gopath:$GOPATH && go get && GORACE='halt_on_error=1' go test ./... -v -timeout 120s -race"
-
-before_install:
-  - sudo apt-get install libpcap-dev -y

COMM-LICENSE

@@ -36,7 +36,7 @@ under the terms of the GNU Lesser General Public License versions 3.0
 
 5. Fees and Payment. The Software license fees will be due and payable in full as set forth in the applicable invoice or at the time of purchase. If the Software does not function properly within two weeks of purchase, please contact us within those two weeks for a refund. You shall be responsible for all taxes, withholdings, duties and levies arising from the order (excluding taxes based on the net income of GoReplay LLC). 
 
-6. Support, Maintenance and Services. Subject to the terms and conditions of this Agreement, as set forth in your invoice, and as set forth on the GoReplay Pro support page (https://github.com/buger/gor/wiki/Pro-Support), support and maintenance services may be included with the purchase of your license subscription.
+6. Support, Maintenance and Services. Subject to the terms and conditions of this Agreement, as set forth in your invoice, and as set forth on the GoReplay Pro support page (https://github.com/buger/goreplay/wiki/Pro-Support), support and maintenance services may be included with the purchase of your license subscription.
 
 7. Term of Agreement. 
 
@@ -48,10 +48,11 @@ under the terms of the GNU Lesser General Public License versions 3.0
 
 8. Disclaimer of Warranties. The Software is provided "as is," with all faults, defects and errors, and without warranty of any kind. GoReplay LLC does not warrant that the Software will be free of bugs, errors, viruses or other defects, and GoReplay LLC shall have no liability of any kind for the use of or inability to use the Software, the Software content or any associated service, and you acknowledge that it is not technically practicable for GoReplay LLC to do so. 
 To the maximum extent permitted by applicable law, GoReplay LLC disclaims all warranties, express, implied, arising by law or otherwise, regarding the Software, the Software content and their respective performance or suitability for your intended use, including without limitation any implied warranty of merchantability, fitness for a particular purpose.
+Notwithstanding the foregoing, GoReplay LLC represents and warrants that it either owns the entire right to, title to, interest in, or has the right to license, the Software, and your proper use of the same will not violate any intellectual property or proprietary rights of another.
 
 9. Limitation of Liability. 
 
-In no event will GoReplay LLC be liable for any direct, indirect, consequential, incidental, special, exemplary, or punitive damages or liabilities whatsoever arising from or relating to the Software, the Software content or this Agreement, whether based on contract, tort (including negligence), strict liability or other theory, even if GoReplay LLC has been advised of the possibility of such damages.
+In no event will either party be liable for any direct, indirect, consequential, incidental, special, exemplary, or punitive damages or liabilities whatsoever arising from or relating to the Software, the Software content or this Agreement, whether based on contract, tort (including negligence), strict liability or other theory, even if such party has been advised of the possibility of such damages.
 
 In no event will GoReplay LLC liability exceed the Software license price as indicated in the invoice. The existence of more than one claim will not enlarge or extend this limit.
 
@@ -59,9 +60,9 @@ In no event will GoReplay LLC liability exceed the Software license price as ind
 
 11. Acknowledgements.
 
-11.1 Consent to the Use of Data. You agree that GoReplay LLC and its affiliates may collect and use technical information gathered as part of the product support services. GoReplay LLC may use this information solely to improve products and services and will not disclose this information in a form that personally identifies you.
+11.1 Consent to the Use of Data. You agree that GoReplay LLC and its affiliates may collect and use technical information gathered as part of the product support services to be used in an anonymized manner. GoReplay LLC may use this information solely to improve products and services and will not disclose this information in a form that personally identifies you.
 
-11.2 Verification. We or a certified auditor acting on our behalf, may, upon its reasonable request and at its expense, audit you with respect to the use of the Software. Such audit may be conducted by mail, electronic means or through an in-person visit to your place of business. Any such in-person audit shall be conducted during regular business hours at your facilities and shall not unreasonably interfere with your business activities. We shall not remove, copy, or redistribute any electronic material during the course of an audit. If an audit reveals that you are using the Software in a way that is in material violation of the terms of the EULA, then you shall pay our reasonable costs of conducting the audit. In the case of a material violation, you agree to pay Us any amounts owing that are attributable to the unauthorized use. In the alternative, We reserve the right, at our sole option, to terminate the licenses for the Software.
+11.2 Verification. We or a certified auditor acting on our behalf, may, upon its reasonable request and at its expense, audit you with respect to the use of the Software, subject to your reasonable information security and confidentiality protections. Such audit may be conducted by mail, electronic means or through an in-person visit to your place of business. Any such in-person audit shall be conducted during regular business hours at your facilities and shall not unreasonably interfere with your business activities. We shall not remove, copy, or redistribute any electronic material during the course of an audit. If an audit reveals that you are using the Software in a way that is in material violation of the terms of the EULA, then you shall pay our reasonable costs of conducting the audit. In the case of a material violation, you agree to pay Us any amounts owing that are attributable to the unauthorized use. In the alternative, We reserve the right, at our sole option, to terminate the licenses for the Software.
 
 11.3 Government End Users. If the Software and related documentation are supplied to or purchased by or on behalf of the United States Government, then the Software is deemed to be "commercial software" as that term is used in the Federal Acquisition Regulation system. Rights of the United States shall not exceed the minimum rights set forth in FAR 52.227-19 for "restricted computer software". All other terms and conditions of this Agreement apply.
 
@@ -72,11 +73,11 @@ In no event will GoReplay LLC liability exceed the Software license price as ind
 
 13.1 Entire Agreement. This Agreement sets forth our entire agreement with respect to the Software and the subject matter hereof and supersedes all prior and contemporaneous understandings and agreements whether written or oral.
 
-13.2 Amendment. GoReplay LLC reserves the right, in its sole discretion, to amend this Agreement from time. Amendments to this Agreement can be located at: https://github.com/buger/gor/blob/master/COMM-LICENSE.
+13.2 Amendment. GoReplay LLC reserves the right, in its sole discretion, to amend this Agreement from time. Amendments to this Agreement can be located at: https://github.com/buger/goreplay/blob/master/COMM-LICENSE.
 
 13.3 Assignment. You may not assign this Agreement or any of its rights under this Agreement without the prior written consent of GoReplay LLC and any attempted assignment without such consent shall be void.
 
-13.4 Export Compliance. You agree to comply with all applicable laws and regulations, including laws, regulations, orders or other restrictions on export, re-export or redistribution of software.
+13.4 Export Compliance. Each party agrees to comply with all applicable laws and regulations, including laws, regulations, orders or other restrictions on export, re-export or redistribution of software.
 
 13.5 Indemnification. You agree to defend, indemnify, and hold harmless GoReplay LLC from and against any lawsuits, claims, losses, damages, fines and expenses (including attorneys' fees and costs) arising out of your use of the Software or breach of this Agreement.
 

Dockerfile

@@ -1,24 +1,11 @@
-FROM golang:latest
+FROM alpine:3.12 as builder
 
-RUN apt-get update && apt-get install ruby vim-common -y
+ARG RELEASE_VERSION
 
-# Install Java for middleware testing
-RUN echo "deb http://ppa.launchpad.net/webupd8team/java/ubuntu trusty main" | tee /etc/apt/sources.list.d/webupd8team-java.list
-RUN echo "deb-src http://ppa.launchpad.net/webupd8team/java/ubuntu trusty main" | tee -a /etc/apt/sources.list.d/webupd8team-java.list
-RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys EEA14886
-RUN apt-get update -y
-RUN echo oracle-java7-installer shared/accepted-oracle-license-v1-1 select true | /usr/bin/debconf-set-selections
-RUN apt-get install oracle-java8-installer -y
+RUN apk add --no-cache ca-certificates openssl
+RUN wget https://github.com/buger/goreplay/releases/download/${RELEASE_VERSION}/gor_${RELEASE_VERSION}_x64.tar.gz -O gor.tar.gz
+RUN tar xzf gor.tar.gz
 
-RUN apt-get install flex bison -y
-RUN wget http://www.tcpdump.org/release/libpcap-1.7.4.tar.gz && tar xzf libpcap-1.7.4.tar.gz && cd libpcap-1.7.4 && ./configure && make install
-RUN go get github.com/google/gopacket
-RUN go get -u github.com/golang/lint/golint
-
-WORKDIR /go/src/github.com/buger/gor/
-ADD . /go/src/github.com/buger/gor/
-
-RUN wget http://archive.apache.org/dist/commons/io/binaries/commons-io-2.4-bin.tar.gz && tar xzf commons-io-2.4-bin.tar.gz && cd commons-io-2.4 && mv commons-io-2.4.jar /tmp/
-RUN wget http://archive.apache.org/dist/commons/codec/binaries/commons-codec-1.9-bin.tar.gz && tar xzf commons-codec-1.9-bin.tar.gz
-RUN javac -cp commons-io-2.4/commons-io-2.4.jar -cp commons-codec-1.9/commons-codec-1.9.jar ./examples/middleware/echo.java
-RUN go get
+FROM scratch
+COPY --from=builder /gor .
+ENTRYPOINT ["./gor"]

Dockerfile.dev

@@ -0,0 +1,14 @@
+FROM golang:1.15
+
+RUN apt-get update && apt-get install ruby vim-common -y
+
+RUN apt-get install flex bison -y
+RUN wget http://www.tcpdump.org/release/libpcap-1.10.0.tar.gz && tar xzf libpcap-1.10.0.tar.gz && cd libpcap-1.10.0 && ./configure && make install
+
+RUN go get github.com/google/gopacket
+RUN go get -u golang.org/x/lint/golint
+
+WORKDIR /go/src/github.com/buger/goreplay/
+ADD . /go/src/github.com/buger/goreplay/
+
+RUN go get

LICENSE.txt

@@ -1,3 +1,12 @@
+Copyright (c) 2011-present Leonid Bugaev
+
+Portions of this software are licensed as follows:
+
+* All content residing under the "doc/" directory of this repository is licensed under "Creative Commons: CC BY-SA 4.0 license".
+* Files with the following names "pro.go", "s3_reader.go", "output_s3.go" are released under the commercial license specified in "COMM-LICENSE" file.
+* Content outside of the above mentioned directories or restrictions above is available under the "LGPLv3" license as defined below.
+
+
 GoReplay is an Open Source project licensed under the terms of
 the LGPLv3 license.  Please see <http://www.gnu.org/licenses/lgpl-3.0.html>
 for license text.

Makefile

@@ -1,35 +1,86 @@
-SOURCE = emitter.go gor.go gor_stat.go input_dummy.go input_file.go input_raw.go input_tcp.go limiter.go output_dummy.go output_null.go output_file.go input_http.go output_http.go output_tcp.go plugins.go settings.go test_input.go elasticsearch.go http_modifier.go http_modifier_settings.go http_client.go middleware.go protocol.go output_file_settings.go output_kafka.go
-SOURCE_PATH = /go/src/github.com/buger/gor/
+SOURCE = $(shell ls -1 *.go | grep -v _test.go)
+SOURCE_PATH = /go/src/github.com/buger/goreplay/
 PORT = 8000
 FADDR = :8000
-RUN = docker run -v `pwd`:$(SOURCE_PATH) -p 0.0.0.0:$(PORT):$(PORT) -i -t gor
+CONTAINER=gor
+PREFIX=
+RUN = docker run -v `pwd`:$(SOURCE_PATH) -e AWS_ACCESS_KEY_ID=$(AWS_ACCESS_KEY_ID) -e AWS_SECRET_ACCESS_KEY=$(AWS_SECRET_ACCESS_KEY) -p 0.0.0.0:$(PORT):$(PORT) -t -i $(CONTAINER)
 BENCHMARK = BenchmarkRAWInput
 TEST = TestRawListenerBench
+BIN_NAME = gor
 VERSION = DEV-$(shell date +%s)
-LDFLAGS = -ldflags "-X main.VERSION=$(VERSION) -extldflags \"-static\""
-MAC_LDFLAGS = -ldflags "-X main.VERSION=$(VERSION)"
+LDFLAGS = -ldflags "-X main.VERSION=$(VERSION)$(PREFIX) -extldflags \"-static\" -X main.DEMO=$(DEMO)"
+MAC_LDFLAGS = -ldflags "-X main.VERSION=$(VERSION)$(PREFIX) -X main.DEMO=$(DEMO)"
 FADDR = ":8000"
 
-release: release-x64 release-mac
+FPMCOMMON= \
+    --name goreplay \
+    --description "GoReplay is an open-source network monitoring tool which can record your live traffic, and use it for shadowing, load testing, monitoring and detailed analysis." \
+    -v $(VERSION) \
+    --vendor "Leonid Bugaev" \
+    -m "<support@goreplay.org>" \
+    --url "https://goreplay.org" \
+    -s dir \
+    -C /tmp/gor-build \
+
+release: release-x64 release-mac release-windows
 
 release-bin:
-	docker run -v `pwd`:$(SOURCE_PATH) -t --env GOOS=linux --env GOARCH=amd64  -i gor go build -tags netgo $(LDFLAGS)
+	docker run -v `pwd`:$(SOURCE_PATH) -t --env GOOS=linux --env GOARCH=amd64  -i $(CONTAINER) go build -o $(BIN_NAME) -tags netgo $(LDFLAGS)
+
+release-bin-mac:
+	GOOS=darwin go build -o $(BIN_NAME) $(MAC_LDFLAGS)
 
 release-x64:
-	docker run -v `pwd`:$(SOURCE_PATH) -t --env GOOS=linux --env GOARCH=amd64  -i gor go build -tags netgo $(LDFLAGS) && tar -czf gor_$(VERSION)_x64.tar.gz gor && rm gor
+	docker run -v `pwd`:$(SOURCE_PATH) -t --env GOOS=linux --env GOARCH=amd64  -i $(CONTAINER) go build -o $(BIN_NAME) -tags netgo $(LDFLAGS)
+	tar -czf gor_$(VERSION)$(PREFIX)_x64.tar.gz $(BIN_NAME)
+	mkdir -p /tmp/gor-build
+	mv ./$(BIN_NAME) /tmp/gor-build/$(BIN_NAME)
+	cd /tmp/gor-build
+	rm -f goreplay_$(VERSION)_amd64.deb
+	rm -f goreplay-$(VERSION)-1.x86_64.rpm
+	fpm $(FPMCOMMON) -a amd64 -t deb ./=/usr/local/bin
+	fpm $(FPMCOMMON) -a amd64 -t rpm ./=/usr/local/bin
+	rm -rf /tmp/gor-build
 
 release-x86:
-	docker run -v `pwd`:$(SOURCE_PATH) -t --env GOOS=linux --env GOARCH=386 -i gor go build -tags netgo $(LDFLAGS) && tar -czf gor_$(VERSION)_x86.tar.gz gor && rm gor
+	docker run -v `pwd`:$(SOURCE_PATH) -t --env GOOS=linux --env GOARCH=386 -i $(CONTAINER) go build -o $(BIN_NAME) -tags netgo $(LDFLAGS)
+	tar -czf gor_$(VERSION)$(PREFIX)_x86.tar.gz $(BIN_NAME)
+	rm $(BIN_NAME)
 
 release-mac:
-	go build $(MAC_LDFLAGS) && tar -czf gor_$(VERSION)_mac.tar.gz gor && rm gor
+	go build -o $(BIN_NAME) $(MAC_LDFLAGS)
+	tar -czf gor_$(VERSION)$(PREFIX)_mac.tar.gz $(BIN_NAME)
+	mkdir -p /tmp/gor-build
+	mv ./$(BIN_NAME) /tmp/gor-build/$(BIN_NAME)
+	cd /tmp/gor-build
+	rm -f goreplay-$(VERSION).pkg
+	fpm $(FPMCOMMON) -a amd64 -t osxpkg ./=/usr/local/bin
+	rm -rf /tmp/gor-build
+
+release-windows:
+	docker run -it --rm \
+	  -v `pwd`:/go/src/github.com/buger/goreplay \
+	  -w /go/src/github.com/buger/goreplay \
+	  -e CGO_ENABLED=1 \
+	  docker.elastic.co/beats-dev/golang-crossbuild:1.16.4-main \
+	  --build-cmd "make VERSION=$(VERSION) build" \
+	  -p "windows/amd64"
+	mv ./gor ./gor.exe
+	zip gor-$(VERSION)$(PREFIX)_windows.zip ./gor.exe
+	rm -rf ./gor.exe
 
 build:
-	docker build -t gor .
+	go build -o $(BIN_NAME) $(LDFLAGS)
 
+install:
+	go install $(MAC_LDFLAGS)
+
+build-env:
+	docker build -t $(CONTAINER) -f Dockerfile.dev .
 
 profile:
-	go build && ./gor --output-http="http://localhost:9000" --input-dummy 0 --input-raw :9000 --input-http :9000 --memprofile=./mem.out --cpuprofile=./cpu.out --stats --output-http-stats --output-http-timeout 100ms
+	go build && ./$(BIN_NAME) --output-http="http://localhost:9000" --input-dummy 0 --input-raw :9000 --input-http :9000 --memprofile=./mem.out --cpuprofile=./cpu.out --stats --output-http-stats --output-http-timeout 100ms
 
 lint:
 	$(RUN) golint $(PKG)
@@ -38,13 +89,13 @@ race:
 	$(RUN) go test ./... $(ARGS) -v -race -timeout 15s
 
 test:
-	$(RUN) go test ./. -timeout 60s $(LDFLAGS) $(ARGS)  -v
+	$(RUN) go test ./. -timeout 120s $(LDFLAGS) $(ARGS)  -v
 
 test_all:
-	$(RUN) go test ./... -timeout 60s $(LDFLAGS) $(ARGS) -v
+	$(RUN) go test ./... -timeout 120s $(LDFLAGS) $(ARGS) -v
 
 testone:
-	$(RUN) go test ./... -timeout 4s $(LDFLAGS) -run $(TEST) $(ARGS) -v
+	$(RUN) go test ./. -timeout 60s $(LDFLAGS) -run $(TEST) $(ARGS) -v
 
 cover:
 	$(RUN) go test $(ARGS) -race -v -timeout 15s -coverprofile=coverage.out
@@ -60,15 +111,15 @@ bench:
 	$(RUN) go test $(LDFLAGS) -v -run NOT_EXISTING -bench $(BENCHMARK) -benchtime 5s
 
 profile_test:
-	$(RUN) go test $(LDFLAGS) -run $(TEST) ./raw_socket_listener/. $(ARGS) -memprofile mem.mprof -cpuprofile cpu.out
-	$(RUN) go test $(LDFLAGS) -run $(TEST) ./raw_socket_listener/. $(ARGS) -c
+	$(RUN) go test $(LDFLAGS) -run $(TEST) ./capture/. $(ARGS) -memprofile mem.mprof -cpuprofile cpu.out
+	$(RUN) go test $(LDFLAGS) -run $(TEST) ./capture/. $(ARGS) -c
 
 # Used mainly for debugging, because docker container do not have access to parent machine ports
 run:
-	$(RUN) go run $(LDFLAGS) $(SOURCE) --input-dummy=0 --output-http="http://localhost:9000" --input-raw-track-response --input-raw 127.0.0.1:9000 --verbose --debug --middleware "./examples/middleware/echo.sh" --output-file requests.gor
+	$(RUN) go run $(LDFLAGS) $(SOURCE) --input-dummy=0 --output-http="http://localhost:9000" --input-raw-track-response --input-raw 127.0.0.1:9000 --verbose 0 --middleware "./examples/middleware/echo.sh" --output-file requests.gor
 
 run-2:
-	sudo -E go run $(SOURCE) --input-dummy="" --output-tcp localhost:27001 --verbose --debug
+	$(RUN) go run $(LDFLAGS) $(SOURCE) --input-raw :8000 --input-raw-bpf-filter "dst port 8000" --output-stdout --output-http "http://localhost:8000" --input-dummy=0
 
 run-3:
 	sudo -E go run $(SOURCE) --input-tcp :27001 --output-stdout
@@ -80,7 +131,7 @@ file-server:
 	go run $(SOURCE) file-server $(FADDR)
 
 readpcap:
-	go run $(SOURCE) --input-raw $(FILE) --input-raw-engine pcap_file --output-null
+	go run $(SOURCE) --input-raw $(FILE) --input-raw-track-response --input-raw-engine pcap_file --output-stdout
 
 record:
 	$(RUN) go run $(SOURCE) --input-dummy=0 --output-file=requests.gor --verbose --debug
@@ -90,3 +141,20 @@ replay:
 
 bash:
 	$(RUN) /bin/bash
+
+
+FPMCOMMON= \
+    --name gor \
+    --description "GoReplay is an open-source network monitoring tool which can record your live traffic, and use it for shadowing, load testing, monitoring and detailed analysis." \
+    -v $(VERSION) \
+    --vendor "Leonid Bugaev" \
+    -m "<support@goreplay.org>" \
+    --url "https://goreplay.org" \
+    -s dir \
+    -C /tmp/gor-build \
+
+build_packages:
+	mkdir -p /tmp/gor-build
+	go build -i -o /tmp/gor-build/$(BIN_NAME)
+	fpm $(FPMCOMMON) -a amd64 -t deb ./=/usr/local/bin
+	fpm $(FPMCOMMON) -a amd64 -t rpm ./=/usr/local/bin

README.md

@@ -1,34 +1,34 @@
-[![GitHub release](https://img.shields.io/github/release/buger/gor.svg?maxAge=3600)](https://github.com/buger/gor/releases) [![codebeat](https://codebeat.co/badges/6427d589-a78e-416c-a546-d299b4089893)](https://codebeat.co/projects/github-com-buger-gor) [![Go Report Card](https://goreportcard.com/badge/github.com/buger/gor)](https://goreportcard.com/report/github.com/buger/gor) [![Join the chat at https://gitter.im/buger/gor](https://badges.gitter.im/buger/gor.svg)](https://gitter.im/buger/gor?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
+[![GitHub release](https://img.shields.io/github/release/buger/gor.svg?maxAge=3600)](https://github.com/buger/goreplay/releases) [![codebeat](https://codebeat.co/badges/6427d589-a78e-416c-a546-d299b4089893)](https://codebeat.co/projects/github-com-buger-gor) [![Go Report Card](https://goreportcard.com/badge/github.com/buger/gor)](https://goreportcard.com/report/github.com/buger/gor) [![Join the chat at https://gitter.im/buger/gor](https://badges.gitter.im/buger/gor.svg)](https://gitter.im/buger/gor?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) [![Reviewed by Hound](https://img.shields.io/badge/Reviewed_by-Hound-8E64B0.svg)](https://houndci.com)
 
 ![Go Replay](http://i.imgur.com/ZG2ki5n.png)
 
 ## https://goreplay.org/
 
+GoReplay is an open-source network monitoring tool which can record your live traffic, and use it for shadowing, load testing, monitoring and detailed analysis.
+
 ## About
 
-GoReplay is the simplest and safest way to test your app using real traffic before you put it into production.
+As your application grows, the effort required to test it also grows exponentially. GoReplay offers you the simple idea of reusing your existing traffic for testing, which makes it incredibly powerful. Our state of art technique allows you to analyze and record your application traffic without affecting it. This eliminates the risks that come with putting a third party component in the critical path. 
 
-As your application grows, the effort required to test it also grows exponentially. GoReplay offers you the simple idea of reusing your existing traffic for testing, which makes it incredibly powerful. Our state of art technique allows to analyze and record your application traffic without affecting it. This eliminates the risks that come with putting a third party component in the critical path. 
-
-GoReplay increases your confidence in code deployments, configuration changes and infrastructure changes. Did we mention that no coding is required?
+GoReplay increases your confidence in code deployments, configuration and infrastructure changes.
 
 
+GoReplay offers unique approach for shadowing. Instead of being a proxy, GoReplay in background listen for traffic on your network interface, requiring no changes in your production infrastructure, rather then running GoReplay daemon on the same machine as your service.
 
-Here is basic workflow: The listener server catches http traffic and sends it to the replay server or saves to file. The replay server forwards traffic to a given address.
+![Diagram](https://i.imgur.com/IN2xfDm.png)
 
-![Diagram](http://i.imgur.com/9mqj2SK.png)
-
-Check [latest documentation](http://github.com/buger/gor/wiki).
+Check [latest documentation](http://github.com/buger/goreplay/wiki).
 
 ## Installation
-Download latest binary from https://github.com/buger/gor/releases or [compile by yourself](https://github.com/buger/gor/wiki/Compilation).
+Download latest binary from https://github.com/buger/goreplay/releases or [compile by yourself](https://github.com/buger/goreplay/wiki/Compilation).
 
 ## Getting started
 
 The most basic setup will be `sudo ./gor --input-raw :8000 --output-stdout` which acts like tcpdump.
 If you already have test environment you can start replaying: `sudo ./gor --input-raw :8000 --output-http http://staging.env`.
 
-See the our [documentation](https://github.com/buger/gor/wiki/) and [Getting started](https://github.com/buger/gor/wiki/Getting-Started) page for more info. 
+See the our [documentation](https://github.com/buger/goreplay/wiki/) and [Getting started](https://github.com/buger/goreplay/wiki/Getting-Started) page for more info. 
+
 ## Newsletter
 Subscribe to our [newsletter](https://www.getdrip.com/forms/89690474/submissions/new) to stay informed about the latest features and changes to Gor project.
 
@@ -39,9 +39,9 @@ We have created a [GoReplay PRO](https://goreplay.org/pro.html) extension which
 
 
 ## Problems?
-If you have a problem, please review the [FAQ](https://github.com/buger/gor/wiki/FAQ) and [Troubleshooting](https://github.com/buger/gor/wiki/Troubleshooting) wiki pages. Searching the [issues](https://github.com/buger/gor/issues) for your problem is also a good idea.
+If you have a problem, please review the [FAQ](https://github.com/buger/goreplay/wiki/FAQ) and [Troubleshooting](https://github.com/buger/goreplay/wiki/Troubleshooting) wiki pages. Searching the [issues](https://github.com/buger/goreplay/issues) for your problem is also a good idea.
 
-All bug-reports and suggestions should go though Github Issues or our [Google Group](https://groups.google.com/forum/#!forum/gor-users) (you can just send email to gor-users@googlegroups.com).
+All bug-reports and suggestions should go through Github Issues or our [Google Group](https://groups.google.com/forum/#!forum/gor-users) (you can just send email to gor-users@googlegroups.com).
 If you have a private question feel free to send email to support@gortool.com.
 
 
@@ -66,7 +66,7 @@ If you have a private question feel free to send email to support@gortool.com.
 * [Granify](http://granify.com) - AI backed SaaS solution that enables online retailers to maximise their sales
 * And many more!
 
-If you are using Gor we are happy add you to the list and share your story, just write to: hello@goreplay.org
+If you are using Gor, we are happy to add you to the list and share your story, just write to: hello@goreplay.org
 
 ## Author
 

byteutils/byteutils.go

@@ -1,6 +1,10 @@
-// Package byteutils probvides helpers for working with byte slices
+// Package byteutils provides helpers for working with byte slices
 package byteutils
 
+import (
+	"unsafe"
+)
+
 // Cut elements from slice for a given range
 func Cut(a []byte, from, to int) []byte {
 	copy(a[from:], a[to:])
@@ -29,12 +33,20 @@ func Replace(a []byte, from, to int, new []byte) []byte {
 		copy(a[from:from+len(new)], new)
 
 		return a
-	} else if lenDiff < 0 {
+	}
+
+	if lenDiff < 0 {
 		copy(a[from:], new)
 		copy(a[from+len(new):], a[to:])
 		return a[:len(a)+lenDiff]
-	} else { // same size
-		copy(a[from:], new)
-		return a
 	}
+
+	// same size
+	copy(a[from:], new)
+	return a
+}
+
+// SliceToString preferred for large body payload (zero allocation and faster)
+func SliceToString(buf []byte) string {
+	return *(*string)(unsafe.Pointer(&buf))
 }

byteutils/byteutils_test.go

@@ -30,3 +30,12 @@ func TestReplace(t *testing.T) {
 		t.Error("Should replace when replacement length bigger")
 	}
 }
+
+func BenchmarkStringtoSlice(b *testing.B) {
+	var s string
+	var buf [1 << 20]byte
+	for i := 0; i < b.N; i++ {
+		s = SliceToString(buf[:])
+	}
+	_ = s // avoid gc to optimize away the loop body
+}

capture/af_packet.go

@@ -0,0 +1,180 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+	"log"
+	"os"
+	"runtime/pprof"
+	"time"
+
+	"github.com/google/gopacket"
+	"github.com/google/gopacket/afpacket"
+	"github.com/google/gopacket/layers"
+	"github.com/google/gopacket/pcap"
+	"golang.org/x/net/bpf"
+
+	_ "github.com/google/gopacket/layers"
+)
+
+var (
+	iface      = flag.String("i", "any", "Interface to read from")
+	cpuprofile = flag.String("cpuprofile", "", "If non-empty, write CPU profile here")
+	snaplen    = flag.Int("s", 0, "Snaplen, if <= 0, use 65535")
+	bufferSize = flag.Int("b", 8, "Interface buffersize (MB)")
+	filter     = flag.String("f", "port not 22", "BPF filter")
+	count      = flag.Int64("c", -1, "If >= 0, # of packets to capture before returning")
+	verbose    = flag.Int64("log_every", 1, "Write a log every X packets")
+	addVLAN    = flag.Bool("add_vlan", false, "If true, add VLAN header")
+)
+
+type afpacketHandle struct {
+	TPacket *afpacket.TPacket
+}
+
+func newAfpacketHandle(device string, snaplen int, block_size int, num_blocks int,
+	useVLAN bool, timeout time.Duration) (*afpacketHandle, error) {
+
+	h := &afpacketHandle{}
+	var err error
+
+	if device == "any" {
+		h.TPacket, err = afpacket.NewTPacket(
+			afpacket.OptFrameSize(snaplen),
+			afpacket.OptBlockSize(block_size),
+			afpacket.OptNumBlocks(num_blocks),
+			afpacket.OptAddVLANHeader(useVLAN),
+			afpacket.OptPollTimeout(timeout),
+			afpacket.SocketRaw,
+			afpacket.TPacketVersion3)
+	} else {
+		h.TPacket, err = afpacket.NewTPacket(
+			afpacket.OptInterface(device),
+			afpacket.OptFrameSize(snaplen),
+			afpacket.OptBlockSize(block_size),
+			afpacket.OptNumBlocks(num_blocks),
+			afpacket.OptAddVLANHeader(useVLAN),
+			afpacket.OptPollTimeout(timeout),
+			afpacket.SocketRaw,
+			afpacket.TPacketVersion3)
+	}
+	return h, err
+}
+
+// ZeroCopyReadPacketData satisfies ZeroCopyPacketDataSource interface
+func (h *afpacketHandle) ZeroCopyReadPacketData() (data []byte, ci gopacket.CaptureInfo, err error) {
+	return h.TPacket.ZeroCopyReadPacketData()
+}
+
+// SetBPFFilter translates a BPF filter string into BPF RawInstruction and applies them.
+func (h *afpacketHandle) SetBPFFilter(filter string, snaplen int) (err error) {
+	pcapBPF, err := pcap.CompileBPFFilter(layers.LinkTypeEthernet, snaplen, filter)
+	if err != nil {
+		return err
+	}
+	bpfIns := []bpf.RawInstruction{}
+	for _, ins := range pcapBPF {
+		bpfIns2 := bpf.RawInstruction{
+			Op: ins.Code,
+			Jt: ins.Jt,
+			Jf: ins.Jf,
+			K:  ins.K,
+		}
+		bpfIns = append(bpfIns, bpfIns2)
+	}
+	if h.TPacket.SetBPF(bpfIns); err != nil {
+		return err
+	}
+	return h.TPacket.SetBPF(bpfIns)
+}
+
+// LinkType returns ethernet link type.
+func (h *afpacketHandle) LinkType() layers.LinkType {
+	return layers.LinkTypeEthernet
+}
+
+// Close will close afpacket source.
+func (h *afpacketHandle) Close() {
+	h.TPacket.Close()
+}
+
+// SocketStats prints received, dropped, queue-freeze packet stats.
+func (h *afpacketHandle) SocketStats() (as afpacket.SocketStats, asv afpacket.SocketStatsV3, err error) {
+	return h.TPacket.SocketStats()
+}
+
+// afpacketComputeSize computes the block_size and the num_blocks in such a way that the
+// allocated mmap buffer is close to but smaller than target_size_mb.
+// The restriction is that the block_size must be divisible by both the
+// frame size and page size.
+func afpacketComputeSize(targetSizeMb int, snaplen int, pageSize int) (
+	frameSize int, blockSize int, numBlocks int, err error) {
+
+	if snaplen < pageSize {
+		frameSize = pageSize / (pageSize / snaplen)
+	} else {
+		frameSize = (snaplen/pageSize + 1) * pageSize
+	}
+
+	// 128 is the default from the gopacket library so just use that
+	blockSize = frameSize * 128
+	numBlocks = (targetSizeMb * 1024 * 1024) / blockSize
+
+	if numBlocks == 0 {
+		return 0, 0, 0, fmt.Errorf("Interface buffersize is too small")
+	}
+
+	return frameSize, blockSize, numBlocks, nil
+}
+
+func main() {
+	flag.Parse()
+	if *cpuprofile != "" {
+		log.Printf("Writing CPU profile to %q", *cpuprofile)
+		f, err := os.Create(*cpuprofile)
+		if err != nil {
+			log.Fatal(err)
+		}
+		if err := pprof.StartCPUProfile(f); err != nil {
+			log.Fatal(err)
+		}
+		defer pprof.StopCPUProfile()
+	}
+	log.Printf("Starting on interface %q", *iface)
+	if *snaplen <= 0 {
+		*snaplen = 65535
+	}
+	szFrame, szBlock, numBlocks, err := afpacketComputeSize(*bufferSize, *snaplen, os.Getpagesize())
+	if err != nil {
+		log.Fatal(err)
+	}
+	afpacketHandle, err := newAfpacketHandle(*iface, szFrame, szBlock, numBlocks, *addVLAN, pcap.BlockForever)
+	if err != nil {
+		log.Fatal(err)
+	}
+	err = afpacketHandle.SetBPFFilter(*filter, *snaplen)
+	if err != nil {
+		log.Fatal(err)
+	}
+	source := gopacket.ZeroCopyPacketDataSource(afpacketHandle)
+	defer afpacketHandle.Close()
+
+	bytes := uint64(0)
+	packets := uint64(0)
+	for ; *count != 0; *count-- {
+		data, _, err := source.ZeroCopyReadPacketData()
+		if err != nil {
+			log.Fatal(err)
+		}
+		bytes += uint64(len(data))
+		packets++
+		if *count%*verbose == 0 {
+			_, afpacketStats, err := afpacketHandle.SocketStats()
+			if err != nil {
+				log.Println(err)
+			}
+			log.Printf("Read in %d bytes in %d packets", bytes, packets)
+			log.Printf("Stats {received dropped queue-freeze}: %d", afpacketStats)
+		}
+	}
+}

capture/capture.go

@@ -0,0 +1,584 @@
+package capture
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"log"
+	"net"
+	"os"
+	"runtime"
+	"strings"
+	"sync"
+	"syscall"
+	"time"
+
+	"github.com/buger/goreplay/size"
+	"github.com/buger/goreplay/tcp"
+
+	"github.com/google/gopacket"
+	"github.com/google/gopacket/layers"
+	"github.com/google/gopacket/pcap"
+)
+
+// PacketHandler is a function that is used to handle packets
+type PacketHandler func(*tcp.Packet)
+
+// PcapOptions options that can be set on a pcap capture handle,
+// these options take effect on inactive pcap handles
+type PcapOptions struct {
+	BufferTimeout time.Duration `json:"input-raw-buffer-timeout"`
+	TimestampType string        `json:"input-raw-timestamp-type"`
+	BPFFilter     string        `json:"input-raw-bpf-filter"`
+	BufferSize    size.Size     `json:"input-raw-buffer-size"`
+	Promiscuous   bool          `json:"input-raw-promisc"`
+	Monitor       bool          `json:"input-raw-monitor"`
+	Snaplen       bool          `json:"input-raw-override-snaplen"`
+}
+
+// Listener handle traffic capture, this is its representation.
+type Listener struct {
+	sync.Mutex
+	Transport  string       // transport layer default to tcp
+	Activate   func() error // function is used to activate the engine. it must be called before reading packets
+	Handles    map[string]gopacket.ZeroCopyPacketDataSource
+	Interfaces []pcap.Interface
+	loopIndex  int
+	Reading    chan bool // this channel is closed when the listener has started reading packets
+	PcapOptions
+	Engine        EngineType
+	ports         []uint16 // src or/and dst ports
+	trackResponse bool
+
+	host string // pcap file name or interface (name, hardware addr, index or ip address)
+
+	closeDone chan struct{}
+	quit      chan struct{}
+}
+
+// EngineType ...
+type EngineType uint8
+
+// Available engines for intercepting traffic
+const (
+	EnginePcap EngineType = 1 << iota
+	EnginePcapFile
+	EngineRawSocket
+	EngineAFPacket
+)
+
+// Set is here so that EngineType can implement flag.Var
+func (eng *EngineType) Set(v string) error {
+	switch v {
+	case "", "libpcap":
+		*eng = EnginePcap
+	case "pcap_file":
+		*eng = EnginePcapFile
+	case "raw_socket":
+		*eng = EngineRawSocket
+	case "af_packet":
+		*eng = EngineAFPacket
+	default:
+		return fmt.Errorf("invalid engine %s", v)
+	}
+	return nil
+}
+
+func (eng *EngineType) String() (e string) {
+	switch *eng {
+	case EnginePcapFile:
+		e = "pcap_file"
+	case EnginePcap:
+		e = "libpcap"
+	case EngineRawSocket:
+		e = "raw_socket"
+	case EngineAFPacket:
+		e = "af_packet"
+	default:
+		e = ""
+	}
+	return e
+}
+
+// NewListener creates and initialize a new Listener. if transport or/and engine are invalid/unsupported
+// is "tcp" and "pcap", are assumed. l.Engine and l.Transport can help to get the values used.
+// if there is an error it will be associated with getting network interfaces
+func NewListener(host string, ports []uint16, transport string, engine EngineType, trackResponse bool) (l *Listener, err error) {
+	l = &Listener{}
+
+	l.host = host
+	if l.host == "localhost" {
+		l.host = "127.0.0.1"
+	}
+	l.ports = ports
+
+	l.Transport = "tcp"
+	if transport != "" {
+		l.Transport = transport
+	}
+	l.Handles = make(map[string]gopacket.ZeroCopyPacketDataSource)
+	l.trackResponse = trackResponse
+	l.closeDone = make(chan struct{})
+	l.quit = make(chan struct{})
+	l.Reading = make(chan bool)
+	switch engine {
+	default:
+		l.Engine = EnginePcap
+		l.Activate = l.activatePcap
+	case EngineRawSocket:
+		l.Engine = EngineRawSocket
+		l.Activate = l.activateRawSocket
+	case EngineAFPacket:
+		l.Engine = EngineAFPacket
+		l.Activate = l.activateAFPacket
+	case EnginePcapFile:
+		l.Engine = EnginePcapFile
+		l.Activate = l.activatePcapFile
+		return
+	}
+
+	err = l.setInterfaces()
+	if err != nil {
+		return nil, err
+	}
+	return
+}
+
+// SetPcapOptions set pcap options for all yet to be actived pcap handles
+// setting this on already activated handles will not have any effect
+func (l *Listener) SetPcapOptions(opts PcapOptions) {
+	l.PcapOptions = opts
+}
+
+// Listen listens for packets from the handles, and call handler on every packet received
+// until the context done signal is sent or there is unrecoverable error on all handles.
+// this function must be called after activating pcap handles
+func (l *Listener) Listen(ctx context.Context, handler PacketHandler) (err error) {
+	l.read(handler)
+	done := ctx.Done()
+	select {
+	case <-done:
+		close(l.quit) // signal close on all handles
+		<-l.closeDone // wait all handles to be closed
+		err = ctx.Err()
+	case <-l.closeDone: // all handles closed voluntarily
+	}
+	return
+}
+
+// ListenBackground is like listen but can run concurrently and signal error through channel
+func (l *Listener) ListenBackground(ctx context.Context, handler PacketHandler) chan error {
+	err := make(chan error, 1)
+	go func() {
+		defer close(err)
+		if e := l.Listen(ctx, handler); err != nil {
+			err <- e
+		}
+	}()
+	return err
+}
+
+// Filter returns automatic filter applied by goreplay
+// to a pcap handle of a specific interface
+func (l *Listener) Filter(ifi pcap.Interface) (filter string) {
+	// https://www.tcpdump.org/manpages/pcap-filter.7.html
+
+	hosts := []string{l.host}
+	if listenAll(l.host) || isDevice(l.host, ifi) {
+		hosts = interfaceAddresses(ifi)
+	}
+
+	filter = portsFilter(l.Transport, "dst", l.ports)
+
+	if len(hosts) != 0 {
+		filter = fmt.Sprintf("((%s) and (%s))", filter, hostsFilter("dst", hosts))
+	} else {
+		filter = fmt.Sprintf("(%s)", filter)
+	}
+
+	if l.trackResponse {
+		responseFilter := portsFilter(l.Transport, "src", l.ports)
+
+		if len(hosts) != 0 {
+			responseFilter = fmt.Sprintf("((%s) and (%s))", responseFilter, hostsFilter("src", hosts))
+		} else {
+			responseFilter = fmt.Sprintf("(%s)", responseFilter)
+		}
+
+		filter = fmt.Sprintf("%s or %s", filter, responseFilter)
+	}
+
+	return
+}
+
+// PcapDumpHandler returns a handler to write packet data in PCAP
+// format, See http://wiki.wireshark.org/Development/LibpcapFileFormathandler.
+// if link layer is invalid Ethernet is assumed
+// func PcapDumpHandler(file *os.File, link layers.LinkType) (handler func(packet *tcp.Packet) error, err error) {
+// 	if link.String() == "" {
+// 		link = layers.LinkTypeEthernet
+// 	}
+// 	w := NewWriterNanos(file)
+// 	err = w.WriteFileHeader(64<<10, link)
+// 	if err != nil {
+// 		return nil, err
+// 	}
+// 	return func(packet *tcp.Packet) error {
+// 		return w.WritePacket(*packet.Info, packet.Data)
+// 	}, nil
+// }
+
+// PcapHandle returns new pcap Handle from dev on success.
+// this function should be called after setting all necessary options for this listener
+func (l *Listener) PcapHandle(ifi pcap.Interface) (handle *pcap.Handle, err error) {
+	var inactive *pcap.InactiveHandle
+	inactive, err = pcap.NewInactiveHandle(ifi.Name)
+	if err != nil {
+		return nil, fmt.Errorf("inactive handle error: %q, interface: %q", err, ifi.Name)
+	}
+	defer inactive.CleanUp()
+	if l.TimestampType != "" {
+		var ts pcap.TimestampSource
+		ts, err = pcap.TimestampSourceFromString(l.TimestampType)
+		err = inactive.SetTimestampSource(ts)
+		if err != nil {
+			return nil, fmt.Errorf("%q: supported timestamps: %q, interface: %q", err, inactive.SupportedTimestamps(), ifi.Name)
+		}
+	}
+	if l.Promiscuous {
+		if err = inactive.SetPromisc(l.Promiscuous); err != nil {
+			return nil, fmt.Errorf("promiscuous mode error: %q, interface: %q", err, ifi.Name)
+		}
+	}
+	if l.Monitor {
+		if err = inactive.SetRFMon(l.Monitor); err != nil && !errors.Is(err, pcap.CannotSetRFMon) {
+			return nil, fmt.Errorf("monitor mode error: %q, interface: %q", err, ifi.Name)
+		}
+	}
+
+	var snap int
+
+	if !l.Snaplen {
+		infs, _ := net.Interfaces()
+		for _, i := range infs {
+			if i.Name == ifi.Name {
+				snap = i.MTU + 200
+			}
+		}
+	}
+
+	if snap == 0 {
+		snap = 64<<10 + 200
+	}
+
+	err = inactive.SetSnapLen(snap)
+	if err != nil {
+		return nil, fmt.Errorf("snapshot length error: %q, interface: %q", err, ifi.Name)
+	}
+	if l.BufferSize > 0 {
+		err = inactive.SetBufferSize(int(l.BufferSize))
+		if err != nil {
+			return nil, fmt.Errorf("handle buffer size error: %q, interface: %q", err, ifi.Name)
+		}
+	}
+	if l.BufferTimeout == 0 {
+		l.BufferTimeout = pcap.BlockForever
+	}
+	err = inactive.SetTimeout(l.BufferTimeout)
+	if err != nil {
+		return nil, fmt.Errorf("handle buffer timeout error: %q, interface: %q", err, ifi.Name)
+	}
+	handle, err = inactive.Activate()
+	if err != nil {
+		return nil, fmt.Errorf("PCAP Activate device error: %q, interface: %q", err, ifi.Name)
+	}
+	l.BPFFilter = l.Filter(ifi)
+	fmt.Println("Interface:", ifi.Name, ". BPF Filter:", l.BPFFilter)
+	err = handle.SetBPFFilter(l.BPFFilter)
+	if err != nil {
+		handle.Close()
+		return nil, fmt.Errorf("BPF filter error: %q%s, interface: %q", err, l.BPFFilter, ifi.Name)
+	}
+	return
+}
+
+// SocketHandle returns new unix ethernet handle associated with this listener settings
+func (l *Listener) SocketHandle(ifi pcap.Interface) (handle Socket, err error) {
+	handle, err = NewSocket(ifi)
+	if err != nil {
+		return nil, fmt.Errorf("sock raw error: %q, interface: %q", err, ifi.Name)
+	}
+	if err = handle.SetPromiscuous(l.Promiscuous || l.Monitor); err != nil {
+		return nil, fmt.Errorf("promiscuous mode error: %q, interface: %q", err, ifi.Name)
+	}
+	l.BPFFilter = l.Filter(ifi)
+	fmt.Println("BPF Filter: ", l.BPFFilter)
+	if err = handle.SetBPFFilter(l.BPFFilter); err != nil {
+		handle.Close()
+		return nil, fmt.Errorf("BPF filter error: %q%s, interface: %q", err, l.BPFFilter, ifi.Name)
+	}
+	handle.SetLoopbackIndex(int32(l.loopIndex))
+	return
+}
+
+func (l *Listener) read(handler PacketHandler) {
+	l.Lock()
+	defer l.Unlock()
+	for key, handle := range l.Handles {
+		go func(key string, hndl gopacket.ZeroCopyPacketDataSource) {
+			defer l.closeHandles(key)
+			linkSize := 14
+			linkType := int(layers.LinkTypeEthernet)
+			if _, ok := hndl.(*pcap.Handle); ok {
+				linkType = int(hndl.(*pcap.Handle).LinkType())
+				linkSize, ok = pcapLinkTypeLength(linkType)
+				if !ok {
+					if os.Getenv("GORDEBUG") != "0" {
+						log.Printf("can not identify link type of an interface '%s'\n", key)
+					}
+					return // can't find the linktype size
+				}
+			}
+
+			for {
+				select {
+				case <-l.quit:
+					return
+				default:
+					data, ci, err := hndl.ZeroCopyReadPacketData()
+					if err == nil {
+						pckt, err := tcp.ParsePacket(data, linkType, linkSize, &ci)
+						if err == nil {
+							handler(pckt)
+						}
+						continue
+					}
+					if enext, ok := err.(pcap.NextError); ok && enext == pcap.NextErrorTimeoutExpired {
+						continue
+					}
+					if eno, ok := err.(syscall.Errno); ok && eno.Temporary() {
+						continue
+					}
+					if enet, ok := err.(*net.OpError); ok && (enet.Temporary() || enet.Timeout()) {
+						continue
+					}
+					if err == io.EOF || err == io.ErrClosedPipe {
+						log.Printf("stopped reading from %s interface with error %s\n", key, err)
+						return
+					}
+
+					log.Printf("stopped reading from %s interface with error %s\n", key, err)
+					return
+				}
+			}
+		}(key, handle)
+	}
+	close(l.Reading)
+}
+
+func (l *Listener) closeHandles(key string) {
+	l.Lock()
+	defer l.Unlock()
+	if handle, ok := l.Handles[key]; ok {
+		if _, ok = handle.(Socket); ok {
+			handle.(Socket).Close()
+		} else {
+			handle.(*pcap.Handle).Close()
+		}
+		delete(l.Handles, key)
+		if len(l.Handles) == 0 {
+			close(l.closeDone)
+		}
+	}
+}
+
+func (l *Listener) activatePcap() error {
+	var e error
+	var msg string
+	for _, ifi := range l.Interfaces {
+		var handle *pcap.Handle
+		handle, e = l.PcapHandle(ifi)
+		if e != nil {
+			msg += ("\n" + e.Error())
+			continue
+		}
+		l.Handles[ifi.Name] = handle
+	}
+	if len(l.Handles) == 0 {
+		return fmt.Errorf("pcap handles error:%s", msg)
+	}
+	return nil
+}
+
+func (l *Listener) activateAFPacket() error {
+	var e error
+	var msg string
+
+	szFrame, szBlock, numBlocks, err := afpacketComputeSize(*bufferSize, *snaplen, os.Getpagesize())
+	if l.host == "" || l.host == "any" {
+		handle, err := newAfpacketHandle("any", szFrame, szBlock, numBlocks, false, pcap.BlockForever)
+		if err != nil {
+			return err
+		}
+		return nil
+	}
+
+	for _, ifi := range l.Interfaces {
+		var handle *pcap.Handle
+
+		handle, err := newAfpacketHandle(ifi.Name, szFrame, szBlock, numBlocks, false, pcap.BlockForever)
+
+		if err != nil {
+			msg += ("\n" + err.Error())
+			continue
+		}
+		l.Handles[ifi.Name] = handle
+	}
+	if len(l.Handles) == 0 {
+		return fmt.Errorf("pcap handles error:%s", msg)
+	}
+	return nil
+}
+
+func (l *Listener) activateRawSocket() error {
+	if runtime.GOOS != "linux" {
+		return fmt.Errorf("sock_raw is not stabilized on OS other than linux")
+	}
+	var msg string
+	var e error
+	for _, ifi := range l.Interfaces {
+		var handle Socket
+		handle, e = l.SocketHandle(ifi)
+		if e != nil {
+			msg += ("\n" + e.Error())
+			continue
+		}
+		l.Handles[ifi.Name] = handle
+	}
+	if len(l.Handles) == 0 {
+		return fmt.Errorf("raw socket handles error:%s", msg)
+	}
+	return nil
+}
+
+func (l *Listener) activatePcapFile() (err error) {
+	var handle *pcap.Handle
+	var e error
+	if handle, e = pcap.OpenOffline(l.host); e != nil {
+		return fmt.Errorf("open pcap file error: %q", e)
+	}
+
+	tmp := l.host
+	l.host = ""
+	l.BPFFilter = l.Filter(pcap.Interface{})
+	l.host = tmp
+
+	if e = handle.SetBPFFilter(l.BPFFilter); e != nil {
+		handle.Close()
+		return fmt.Errorf("BPF filter error: %q, filter: %s", e, l.BPFFilter)
+	}
+	l.Handles["pcap_file"] = handle
+	return
+}
+
+func (l *Listener) setInterfaces() (err error) {
+	var pifis []pcap.Interface
+	pifis, err = pcap.FindAllDevs()
+	ifis, _ := net.Interfaces()
+	if err != nil {
+		return
+	}
+	for _, pi := range pifis {
+		var ni net.Interface
+		for _, i := range ifis {
+			if i.Name == pi.Name {
+				ni = i
+				break
+			}
+		}
+
+		if net.Flags(pi.Flags)&net.FlagLoopback != 0 {
+			l.loopIndex = ni.Index
+		}
+		if net.Flags(pi.Flags)&net.FlagUp == 0 {
+			continue
+		}
+		if isDevice(l.host, pi) {
+			l.Interfaces = []pcap.Interface{pi}
+			return
+		}
+		for _, addr := range pi.Addresses {
+			if addr.IP.String() == l.host {
+				l.Interfaces = []pcap.Interface{pi}
+				return
+			}
+		}
+	}
+	l.Interfaces = pifis
+	return
+}
+
+func isDevice(addr string, ifi pcap.Interface) bool {
+	return addr == ifi.Name
+}
+
+func interfaceAddresses(ifi pcap.Interface) []string {
+	var hosts []string
+	for _, addr := range ifi.Addresses {
+		hosts = append(hosts, addr.IP.String())
+	}
+	return hosts
+}
+
+func listenAll(addr string) bool {
+	switch addr {
+	case "", "0.0.0.0", "[::]", "::":
+		return true
+	}
+	return false
+}
+
+func portsFilter(transport string, direction string, ports []uint16) string {
+	if len(ports) == 0 || ports[0] == 0 {
+		return fmt.Sprintf("%s %s portrange 0-%d", transport, direction, 1<<16-1)
+	}
+
+	var filters []string
+	for _, port := range ports {
+		filters = append(filters, fmt.Sprintf("%s %s port %d", transport, direction, port))
+	}
+	return strings.Join(filters, " or ")
+}
+
+func hostsFilter(direction string, hosts []string) string {
+	var hostsFilters []string
+	for _, host := range hosts {
+		hostsFilters = append(hostsFilters, fmt.Sprintf("%s host %s", direction, host))
+	}
+
+	return strings.Join(hostsFilters, " or ")
+}
+
+func pcapLinkTypeLength(lType int) (int, bool) {
+	switch layers.LinkType(lType) {
+	case layers.LinkTypeEthernet:
+		return 14, true
+	case layers.LinkTypeNull, layers.LinkTypeLoop:
+		return 4, true
+	case layers.LinkTypeRaw, 12, 14:
+		return 0, true
+	case layers.LinkTypeIPv4, layers.LinkTypeIPv6:
+		// (TODO:) look out for IP encapsulation?
+		return 0, true
+	case layers.LinkTypeLinuxSLL:
+		return 16, true
+	case layers.LinkTypeFDDI:
+		return 13, true
+	case 226 /*DLT_IPNET*/ :
+		// https://www.tcpdump.org/linktypes/LINKTYPE_IPNET.html
+		return 24, true
+	default:
+		return 0, false
+	}
+}

capture/capture_test.go

@@ -0,0 +1,313 @@
+package capture
+
+import (
+	"context"
+	"encoding/binary"
+	"io/ioutil"
+	"net"
+	"os"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/google/gopacket/layers"
+	"github.com/google/gopacket/pcap"
+)
+
+var LoopBack = func() net.Interface {
+	ifis, _ := net.Interfaces()
+	for _, v := range ifis {
+		if v.Flags&net.FlagLoopback != 0 {
+			return v
+		}
+	}
+	return ifis[0]
+}()
+
+func TestSetInterfaces(t *testing.T) {
+	l := &Listener{}
+	l.host = "127.0.0.1"
+	l.setInterfaces()
+	if len(l.Interfaces) != 1 {
+		t.Error("expected a single interface")
+	}
+	l.host = LoopBack.HardwareAddr.String()
+	l.setInterfaces()
+	if l.Interfaces[0].Name != LoopBack.Name && len(l.Interfaces) != 1 {
+		t.Error("interface should be loop back interface")
+	}
+	l.host = ""
+	l.setInterfaces()
+	if len(l.Interfaces) < 1 {
+		t.Error("should get all interfaces")
+	}
+}
+
+func TestBPFFilter(t *testing.T) {
+	l := &Listener{}
+	l.host = "127.0.0.1"
+	l.Transport = "tcp"
+	l.setInterfaces()
+	filter := l.Filter(l.Interfaces[0])
+	if filter != "(tcp dst portrange 0-65535 and host 127.0.0.1)" {
+		t.Error("wrong filter", filter)
+	}
+	l.port = 8000
+	l.trackResponse = true
+	filter = l.Filter(l.Interfaces[0])
+	if filter != "(tcp port 8000 and host 127.0.0.1)" {
+		t.Error("wrong filter")
+	}
+}
+
+func TestPcapDump(t *testing.T) {
+	f, err := ioutil.TempFile("", "pcap_file")
+	if err != nil {
+		t.Error(err)
+	}
+	h, _ := PcapDumpHandler(f, layers.LinkTypeLoop)
+	packets := Packets(1, 5, 5, 4)
+	for i := 0; i < len(packets); i++ {
+		if i == 1 {
+			// change dst port
+			binary.BigEndian.PutUint16(packets[i].TransLayer[2:], 8001)
+		}
+		if i == 4 {
+			packets[i].Info.CaptureLength = 40
+		}
+		h(packets[i])
+	}
+	name := f.Name()
+	f.Close()
+	testPcapDumpEngine(name, t)
+}
+
+func testPcapDumpEngine(f string, t *testing.T) {
+	defer os.Remove(f)
+	l, err := NewListener(f, 8000, "", EnginePcapFile, true)
+	err = l.Activate()
+	if err != nil {
+		t.Errorf("expected error to be nil, got %q", err)
+		return
+	}
+	pckts := 0
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	err = l.Listen(ctx, func(packet *Packet) {
+		if packet.Info.CaptureLength != 57 {
+			t.Errorf("expected packet length to be %d, got %d", 57, packet.Info.CaptureLength)
+		}
+		pckts++
+	})
+
+	if err != nil {
+		t.Errorf("expected error to be nil, got %q", err)
+	}
+	if pckts != 3 {
+		t.Errorf("expected %d packets, got %d packets", 3, pckts)
+	}
+}
+
+func TestPcapHandler(t *testing.T) {
+	l, err := NewListener(LoopBack.Name, 8000, "", EnginePcap, true)
+	if err != nil {
+		t.Errorf("expected error to be nil, got %v", err)
+		return
+	}
+	err = l.Activate()
+	if err != nil {
+		t.Errorf("expected error to be nil, got %v", err)
+		return
+	}
+	defer l.Handles[LoopBack.Name].(*pcap.Handle).Close()
+	if err != nil {
+		t.Errorf("expected error to be nil, got %v", err)
+		return
+	}
+	for i := 0; i < 5; i++ {
+		_, _ = net.Dial("tcp", "127.0.0.1:8000")
+	}
+	sts, _ := l.Handles[LoopBack.Name].(*pcap.Handle).Stats()
+	if sts.PacketsReceived < 5 {
+		t.Errorf("expected >=5 packets got %d", sts.PacketsReceived)
+	}
+}
+
+// func TestSocketHandler(t *testing.T) {
+// 	l, err := NewListener(LoopBack.Name, 8000, "", EngineRawSocket, true)
+// 	err = l.Activate()
+// 	if err != nil {
+// 		return
+// 	}
+// 	defer l.Handles[LoopBack.Name].(*SockRaw).Close()
+// 	if err != nil {
+// 		t.Errorf("expected error to be nil, got %v", err)
+// 		return
+// 	}
+// 	for i := 0; i < 5; i++ {
+// 		_, _ = net.Dial("tcp", "127.0.0.1:8000")
+// 	}
+// 	sts, _ := l.Handles[LoopBack.Name].(*SockRaw).Stats()
+// 	if sts.Packets < 5 {
+// 		t.Errorf("expected >=5 packets got %d", sts.Packets)
+// 	}
+// }
+
+func BenchmarkPcapDump(b *testing.B) {
+	f, err := ioutil.TempFile("", "pcap_file")
+	if err != nil {
+		b.Error(err)
+		return
+	}
+	now := time.Now()
+	defer os.Remove(f.Name())
+	h, _ := PcapDumpHandler(f, layers.LinkTypeLoop)
+	packets := Packets(1, b.N, 5, 4)
+	for i := 0; i < len(packets); i++ {
+		h(packets[i])
+	}
+	f.Close()
+	b.Logf("%d packets in %s", b.N, time.Since(now))
+}
+
+func BenchmarkPcapFile(b *testing.B) {
+	f, err := ioutil.TempFile("", "pcap_file")
+	if err != nil {
+		b.Error(err)
+		return
+	}
+	defer os.Remove(f.Name())
+	h, _ := PcapDumpHandler(f, layers.LinkTypeLoop)
+	packets := Packets(1, b.N, 5, 4)
+	for i := 0; i < len(packets); i++ {
+		h(packets[i])
+	}
+	name := f.Name()
+	f.Close()
+	b.ResetTimer()
+	var l *Listener
+	l, err = NewListener(name, 8000, "", EnginePcapFile, true)
+	if err != nil {
+		b.Error(err)
+		return
+	}
+	err = l.Activate()
+	if err != nil {
+		b.Error(err)
+		return
+	}
+	now := time.Now()
+	pckts := 0
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	if err = l.Listen(ctx, func(packet *Packet) {
+		if packet.Info.CaptureLength != 49 {
+			b.Errorf("expected packet length to be %d, got %d", 49, packet.Info.CaptureLength)
+		}
+		pckts++
+	}); err != nil {
+		b.Error(err)
+	}
+	b.Logf("%d/%d packets in %s", pckts, b.N, time.Since(now))
+}
+
+// used to benchmark sock engine
+var buf [1024]byte
+
+func init() {
+	for i := 0; i < len(buf); i++ {
+		buf[i] = 0xff
+	}
+}
+
+func handler(n, counter *int32) PacketHandler {
+	return func(p *Packet) {
+		nn := int32(len(p.Data))
+		atomic.AddInt32(n, nn)
+		atomic.AddInt32(counter, 1)
+	}
+}
+
+func BenchmarkPcap(b *testing.B) {
+	var err error
+	n := new(int32)
+	counter := new(int32)
+	l, err := NewListener(LoopBack.Name, 8000, "", EnginePcap, false)
+	if err != nil {
+		b.Error(err)
+		return
+	}
+	l.PcapOptions.BPFFilter = "udp dst port 8000 and host 127.0.0.1"
+	err = l.Activate()
+	if err != nil {
+		b.Error(err)
+		return
+	}
+	errCh := l.ListenBackground(context.Background(), handler(n, counter))
+	select {
+	case <-l.Reading:
+	case err = <-errCh:
+		b.Error(err)
+		return
+	}
+	var conn net.Conn
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		b.StopTimer()
+		conn, err = net.Dial("udp", "127.0.0.1:8000")
+		if err != nil {
+			b.Error(err)
+			return
+		}
+		b.StartTimer()
+		_, err = conn.Write(buf[:])
+		if err != nil {
+			b.Error(err)
+			return
+		}
+	}
+	b.ReportMetric(float64(atomic.LoadInt32(n)), "buf")
+	b.ReportMetric(float64(atomic.LoadInt32(counter)), "packets")
+}
+
+func BenchmarkRawSocket(b *testing.B) {
+	var err error
+	n := new(int32)
+	counter := new(int32)
+	l, err := NewListener(LoopBack.Name, 8000, "", EngineRawSocket, false)
+	if err != nil {
+		b.Error(err)
+		return
+	}
+	l.PcapOptions.BPFFilter = "udp dst port 8000 and host 127.0.0.1"
+	err = l.Activate()
+	if err != nil {
+		b.Error(err)
+		return
+	}
+	errCh := l.ListenBackground(context.Background(), handler(n, counter))
+	select {
+	case <-l.Reading:
+	case err = <-errCh:
+		b.Error(err)
+		return
+	}
+	var conn net.Conn
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		b.StopTimer()
+		conn, err = net.Dial("udp", "127.0.0.1:8000")
+		if err != nil {
+			b.Error(err)
+			return
+		}
+		b.StartTimer()
+		_, err = conn.Write(buf[:])
+		if err != nil {
+			b.Error(err)
+			return
+		}
+	}
+	b.ReportMetric(float64(atomic.LoadInt32(n)), "buf")
+	b.ReportMetric(float64(atomic.LoadInt32(counter)), "packets")
+}

capture/doc.go

@@ -0,0 +1,35 @@
+/*
+Package capture provides traffic sniffier using AF_PACKET, pcap or pcap file.
+it allows you to listen for traffic from any port (e.g. sniffing) because they operate on IP level.
+Ports is TCP/IP feature, same as flow control, reliable transmission and etc.
+Currently this package implements TCP layer: flow control is managed under tcp package.
+BPF filters can also be applied.
+
+example:
+
+// for the transport should be "tcp"
+listener, err := capture.NewListener(host, port, transport, engine, trackResponse)
+if err != nil {
+	// handle error
+}
+listener.SetPcapOptions(opts)
+err = listner.Activate()
+if err != nil {
+	// handle it
+}
+
+if err := listener.Listen(context.Background(), handler); err != nil {
+	 // handle error
+}
+// or
+errCh := listener.ListenBackground(context.Background(), handler) // runs in the background
+select {
+case err := <- errCh:
+	// handle error
+case <-quit:
+	//
+case <- l.Reading: // if we have started reading
+}
+
+*/
+package capture // import github.com/buger/goreplay/capture

capture/dump.go

@@ -0,0 +1,126 @@
+// https://github.com/google/gopacket/blob/403ca653c4/pcapgo/read.go
+
+package capture
+
+import (
+	"encoding/binary"
+	"fmt"
+	"io"
+	"time"
+
+	"github.com/google/gopacket"
+	"github.com/google/gopacket/layers"
+)
+
+// Writer wraps an underlying io.Writer to write packet data in PCAP
+// format.  See http://wiki.wireshark.org/Development/LibpcapFileFormat
+// for information on the file format.
+//
+// For those that care, we currently write v2.4 files with nanosecond
+// or microsecond timestamp resolution and little-endian encoding.
+type Writer struct {
+	w        io.Writer
+	tsScaler int
+	// Moving this into the struct seems to save an allocation for each call to writePacketHeader
+	buf [16]byte
+}
+
+const magicNanoseconds = 0xA1B23C4D
+const magicMicroseconds = 0xA1B2C3D4
+const versionMajor = 2
+const versionMinor = 4
+
+// NewWriterNanos returns a new writer object, for writing packet data out
+// to the given writer.  If this is a new empty writer (as opposed to
+// an append), you must call WriteFileHeader before WritePacket.  Packet
+// timestamps are written with nanosecond precision.
+//
+//  // Write a new file:
+//  f, _ := os.Create("/tmp/file.pcap")
+//  w := pcapgo.NewWriterNanos(f)
+//  w.WriteFileHeader(65536, layers.LinkTypeEthernet)  // new file, must do this.
+//  w.WritePacket(gopacket.CaptureInfo{...}, data1)
+//  f.Close()
+//  // Append to existing file (must have same snaplen and linktype)
+//  f2, _ := os.OpenFile("/tmp/fileNano.pcap", os.O_APPEND, 0700)
+//  w2 := pcapgo.NewWriter(f2)
+//  // no need for file header, it's already written.
+//  w2.WritePacket(gopacket.CaptureInfo{...}, data2)
+//  f2.Close()
+func NewWriterNanos(w io.Writer) *Writer {
+	return &Writer{w: w, tsScaler: nanosPerNano}
+}
+
+// NewWriter returns a new writer object, for writing packet data out
+// to the given writer.  If this is a new empty writer (as opposed to
+// an append), you must call WriteFileHeader before WritePacket.
+// Packet timestamps are written with microsecond precision.
+//
+//  // Write a new file:
+//  f, _ := os.Create("/tmp/file.pcap")
+//  w := pcapgo.NewWriter(f)
+//  w.WriteFileHeader(65536, layers.LinkTypeEthernet)  // new file, must do this.
+//  w.WritePacket(gopacket.CaptureInfo{...}, data1)
+//  f.Close()
+//  // Append to existing file (must have same snaplen and linktype)
+//  f2, _ := os.OpenFile("/tmp/file.pcap", os.O_APPEND, 0700)
+//  w2 := pcapgo.NewWriter(f2)
+//  // no need for file header, it's already written.
+//  w2.WritePacket(gopacket.CaptureInfo{...}, data2)
+//  f2.Close()
+func NewWriter(w io.Writer) *Writer {
+	return &Writer{w: w, tsScaler: nanosPerMicro}
+}
+
+// WriteFileHeader writes a file header out to the writer.
+// This must be called exactly once per output.
+func (w *Writer) WriteFileHeader(snaplen uint32, linktype layers.LinkType) error {
+	var buf [24]byte
+	if w.tsScaler == nanosPerMicro {
+		binary.LittleEndian.PutUint32(buf[0:4], magicMicroseconds)
+	} else {
+		binary.LittleEndian.PutUint32(buf[0:4], magicNanoseconds)
+	}
+	binary.LittleEndian.PutUint16(buf[4:6], versionMajor)
+	binary.LittleEndian.PutUint16(buf[6:8], versionMinor)
+	// bytes 8:12 stay 0 (timezone = UTC)
+	// bytes 12:16 stay 0 (sigfigs is always set to zero, according to
+	//   http://wiki.wireshark.org/Development/LibpcapFileFormat
+	binary.LittleEndian.PutUint32(buf[16:20], snaplen)
+	binary.LittleEndian.PutUint32(buf[20:24], uint32(linktype))
+	_, err := w.w.Write(buf[:])
+	return err
+}
+
+const nanosPerMicro = 1000
+const nanosPerNano = 1
+
+func (w *Writer) writePacketHeader(ci gopacket.CaptureInfo) error {
+	t := ci.Timestamp
+	if t.IsZero() {
+		t = time.Now()
+	}
+	secs := t.Unix()
+	usecs := t.Nanosecond() / w.tsScaler
+	binary.LittleEndian.PutUint32(w.buf[0:4], uint32(secs))
+	binary.LittleEndian.PutUint32(w.buf[4:8], uint32(usecs))
+	binary.LittleEndian.PutUint32(w.buf[8:12], uint32(ci.CaptureLength))
+	binary.LittleEndian.PutUint32(w.buf[12:16], uint32(ci.Length))
+	_, err := w.w.Write(w.buf[:])
+	return err
+}
+
+// WritePacket writes the given packet data out to the file.
+func (w *Writer) WritePacket(ci gopacket.CaptureInfo, data []byte) error {
+	if ci.CaptureLength != len(data) {
+		return fmt.Errorf("capture length %d does not match data length %d", ci.CaptureLength, len(data))
+	}
+	if ci.CaptureLength > ci.Length {
+		return fmt.Errorf("invalid capture info %+v:  capture length > length", ci)
+	}
+	if err := w.writePacketHeader(ci); err != nil {
+		return fmt.Errorf("error writing packet header: %v", err)
+	}
+	_, err := w.w.Write(data)
+	return err
+}

capture/packet_test.go

@@ -0,0 +1,197 @@
+package capture
+
+import (
+	"encoding/binary"
+	"errors"
+	"testing"
+	"time"
+
+	"github.com/google/gopacket"
+	"github.com/google/gopacket/layers"
+)
+
+func generateHeader4(seq uint32, length uint16) []byte {
+	hdr := make([]byte, 4+24+24, 4+24+24)
+	binary.BigEndian.PutUint32(hdr, uint32(layers.ProtocolFamilyIPv4))
+
+	ip := hdr[4:]
+	ip[0] = 4<<4 | 6
+	binary.BigEndian.PutUint16(ip[2:4], length+24+24)
+	ip[9] = uint8(layers.IPProtocolTCP)
+	copy(ip[12:16], []byte{127, 0, 0, 1})
+	copy(ip[16:], []byte{127, 0, 0, 1})
+
+	// set tcp header
+	tcp := ip[24:]
+	tcp[12] = 6 << 4
+	binary.BigEndian.PutUint16(tcp, 5535)
+	binary.BigEndian.PutUint16(tcp[2:], 8000)
+	binary.BigEndian.PutUint32(tcp[4:], seq)
+	return hdr
+}
+
+func generateHeader6(seq uint32, length uint16) []byte {
+	hdr := make([]byte, 4+40+32+24, 4+40+32+24)
+	binary.BigEndian.PutUint32(hdr, uint32(layers.ProtocolFamilyIPv6Linux))
+
+	ip := hdr[4:]
+	ip[0] = 6 << 4
+	binary.BigEndian.PutUint16(ip[4:], length+32+24)
+	var ipAddr [16]byte
+	ipAddr[15] = 0x01
+	copy(ip[8:], ip[:])
+	copy(ip[24:], ip[:])
+	copy(ip[40:], []byte{
+		// net-layer (IPv6-Opts)
+		0x2b, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+		0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+		uint8(layers.IPProtocolTCP), 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+		0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	})
+
+	// set tcp header
+	tcp := ip[40+32:]
+	tcp[12] = 6 << 4
+	binary.BigEndian.PutUint16(tcp, 5535)
+	binary.BigEndian.PutUint16(tcp[2:], 8000)
+	binary.BigEndian.PutUint32(tcp[4:], seq)
+	return hdr
+}
+
+func Packets(start uint32, _len int, length uint16, version byte) []*Packet {
+	var packets = make([]*Packet, _len)
+	for i := start; i < start+uint32(_len); i++ {
+		var h []byte
+		if version == 4 {
+			h = generateHeader4(i, length)
+		} else {
+			h = generateHeader6(i, length)
+		}
+		d := append(h, make([]byte, int(length))...)
+		ci := &gopacket.CaptureInfo{Length: len(d), CaptureLength: len(d), Timestamp: time.Now()}
+		packets[i-start] = NewPacket(d, int(layers.LinkTypeLoop), 4, ci)
+	}
+	return packets
+}
+
+func TestIPv4Packet(t *testing.T) {
+	pckt := packet(append(generateHeader4(1024, 10), make([]byte, 10)...))
+	if pckt.Err != nil {
+		t.Error(pckt)
+		return
+	}
+	if err := packet(pckt.Data[:2]).Err; !errors.Is(err, ErrHdrLength("Link")) {
+		t.Errorf("should fail with %q, got %q", ErrHdrLength("Link"), err)
+		return
+	}
+	if err := packet(pckt.Data[:20]).Err; !errors.Is(err, ErrHdrLength("IPv4")) {
+		t.Errorf("should fail with %q, got %q", ErrHdrLength("IPv4"), err)
+		return
+	}
+	if err := packet(pckt.Data[:27]).Err; !errors.Is(err, ErrHdrLength("IPv4 opts")) {
+		t.Errorf("should fail with %q, got %q", ErrHdrLength("IPv4 opts"), err)
+		return
+	}
+	if err := packet(pckt.Data[:40]).Err; !errors.Is(err, ErrHdrLength("TCP")) {
+		t.Errorf("should fail with %q, got %q", ErrHdrLength("TCP opts"), err)
+		return
+	}
+	if err := packet(pckt.Data[:50]).Err; !errors.Is(err, ErrHdrLength("TCP opts")) {
+		t.Errorf("should fail with %q, got %q", ErrHdrLength("TCP opts"), err)
+		return
+	}
+	pckt.TransLayer[12] = 0x10
+	if err := packet(pckt.Data[:50]).Err; !errors.Is(err, ErrHdrInvalid("TCP's data offset")) {
+		t.Errorf("should fail with %q, got %q", ErrHdrInvalid("TCP's data offset"), err)
+		return
+	}
+	pckt.TransLayer[12] = 0x60
+	if err := packet(pckt.Data[:28]).Err; !errors.Is(err, ErrHdrMissing("TCP")) {
+		t.Errorf("should fail with %q, got %q", ErrHdrMissing("TCP"), err)
+		return
+	}
+	pckt.NetLayer[9] = 0x02
+	if err := packet(pckt.Data).Err; !errors.Is(err, ErrHdrExpected("TCP")) {
+		t.Errorf("should fail with %q, got %q", ErrHdrExpected("TCP"), err)
+		return
+	}
+	pckt.NetLayer[9] = 0x06
+	pckt.NetLayer[0] = 0x44
+	if err := packet(pckt.Data).Err; !errors.Is(err, ErrHdrInvalid("IPv4's IHL")) {
+		t.Errorf("should fail with %q, got %q", ErrHdrInvalid("IPv4's IHL"), err)
+		return
+	}
+	pckt.NetLayer[0] = 0x56
+	if err := packet(pckt.Data).Err; !errors.Is(err, ErrHdrExpected("IPv4 or IPv6")) {
+		t.Errorf("should fail with %q, got %q", ErrHdrExpected("IPv4 or IPv6"), err)
+		return
+	}
+}
+
+func TestIPv6Packet(t *testing.T) {
+	pckt := packet(append(generateHeader6(1024, 10), make([]byte, 10)...))
+	if pckt.Err != nil {
+		t.Error(pckt)
+		return
+	}
+	if err := packet(pckt.Data[:4]).Err; !errors.Is(err, ErrHdrMissing("IPv4 or IPv6")) {
+		t.Errorf("should fail with %q, got %q", ErrHdrMissing("IPv4 or IPv6"), err)
+		return
+	}
+	if err := packet(pckt.Data[:40]).Err; !errors.Is(err, ErrHdrLength("IPv6")) {
+		t.Errorf("should fail with %q, got %q", ErrHdrLength("IPv6"), err)
+		return
+	}
+	if err := packet(pckt.Data[:52]).Err; !errors.Is(err, ErrHdrLength("IPv6 opts")) {
+		t.Errorf("should fail with %q, got %q", ErrHdrLength("IPv6 opts"), err)
+		return
+	}
+	if err := packet(pckt.Data[:80]).Err; !errors.Is(err, ErrHdrLength("TCP")) {
+		t.Errorf("should fail with %q, got %q", ErrHdrLength("TCP opts"), err)
+		return
+	}
+	if err := packet(pckt.Data[:98]).Err; !errors.Is(err, ErrHdrLength("TCP opts")) {
+		t.Errorf("should fail with %q, got %q", ErrHdrLength("TCP opts"), err)
+		return
+	}
+	pckt.TransLayer[12] = 0x10
+	if err := packet(pckt.Data).Err; !errors.Is(err, ErrHdrInvalid("TCP's data offset")) {
+		t.Errorf("should fail with %q, got %q", ErrHdrInvalid("TCP's data offset"), err)
+		return
+	}
+	pckt.TransLayer[12] = 0x60
+	if err := packet(pckt.Data[:76]).Err; !errors.Is(err, ErrHdrMissing("TCP")) {
+		t.Errorf("should fail with %q, got %q", ErrHdrMissing("TCP"), err)
+		return
+	}
+	pckt.NetLayer[56] = 0x02
+	if err := packet(pckt.Data).Err; !errors.Is(err, ErrHdrExpected("TCP")) {
+		t.Errorf("should fail with %q, got %q", ErrHdrExpected("TCP"), err)
+		return
+	}
+	pckt.NetLayer[56] = 0x06
+}
+
+func packet(data []byte) *Packet {
+	return NewPacket(data, int(layers.LinkTypeLoop), 4, &gopacket.CaptureInfo{})
+}
+
+func BenchmarkNewPacketIPv4(b *testing.B) {
+	data := append(generateHeader4(1204, 10), make([]byte, 10)...)
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		if err := packet(data).Err; err != nil {
+			b.Error(err)
+		}
+	}
+}
+
+func BenchmarkNewPacketIPv6(b *testing.B) {
+	data := append(generateHeader6(1024, 10), make([]byte, 10)...)
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		if err := packet(data).Err; err != nil {
+			b.Error(err)
+		}
+	}
+}

capture/sock_linux.go

@@ -0,0 +1,276 @@
+package capture
+
+import (
+	"fmt"
+	"net"
+	"sync"
+	"time"
+	"unsafe"
+
+	"golang.org/x/sys/unix"
+
+	"github.com/google/gopacket"
+	"github.com/google/gopacket/layers"
+	"github.com/google/gopacket/pcap"
+)
+
+const (
+	// ETHALL htons(ETH_P_ALL)
+	ETHALL uint16 = unix.ETH_P_ALL<<8 | unix.ETH_P_ALL>>8
+	// BLOCKSIZE ring buffer block_size
+	BLOCKSIZE = 64 << 10
+	// BLOCKNR ring buffer block_nr
+	BLOCKNR = (2 << 20) / BLOCKSIZE // 2mb / 64kb
+	// FRAMESIZE ring buffer frame_size
+	FRAMESIZE = BLOCKSIZE
+	// FRAMENR ring buffer frame_nr
+	FRAMENR = BLOCKNR * BLOCKSIZE / FRAMESIZE
+	// MAPHUGE2MB 2mb huge map
+	MAPHUGE2MB = 21 << unix.MAP_HUGE_SHIFT
+)
+
+var tpacket2hdrlen = tpAlign(int(unsafe.Sizeof(unix.Tpacket2Hdr{})))
+
+// SockRaw is a linux M'maped af_packet socket
+type SockRaw struct {
+	mu          sync.Mutex
+	fd          int
+	ifindex     int
+	snaplen     int
+	pollTimeout uintptr
+	frame       uint32 // current frame
+	buf         []byte // points to the memory space of the ring buffer shared with the kernel.
+	loopIndex   int32  // this field must filled to avoid reading packet twice on a loopback device
+}
+
+// NewSocket returns new M'maped sock_raw on packet version 2.
+func NewSocket(pifi pcap.Interface) (*SockRaw, error) {
+	var ifi net.Interface
+
+	infs, _ := net.Interfaces()
+	found := false
+	for _, i := range infs {
+		if i.Name == pifi.Name {
+			ifi = i
+			found = true
+			break
+		}
+	}
+
+	if !found {
+		return nil, fmt.Errorf("Can't find matching interface")
+	}
+
+	// sock create
+	fd, err := unix.Socket(unix.AF_PACKET, unix.SOCK_RAW, int(ETHALL))
+	if err != nil {
+		return nil, err
+	}
+	sock := &SockRaw{
+		fd:          fd,
+		ifindex:     ifi.Index,
+		snaplen:     FRAMESIZE,
+		pollTimeout: ^uintptr(0),
+	}
+
+	// set packet version
+	err = unix.SetsockoptInt(fd, unix.SOL_PACKET, unix.PACKET_VERSION, unix.TPACKET_V2)
+	if err != nil {
+		unix.Close(fd)
+		return nil, fmt.Errorf("setsockopt packet_version: %v", err)
+	}
+
+	// bind to interface
+	addr := unix.RawSockaddrLinklayer{
+		Family:   unix.AF_PACKET,
+		Protocol: ETHALL,
+		Ifindex:  int32(ifi.Index),
+	}
+	_, _, e := unix.Syscall(
+		unix.SYS_BIND,
+		uintptr(fd),
+		uintptr(unsafe.Pointer(&addr)),
+		uintptr(unix.SizeofSockaddrLinklayer),
+	)
+	if e != 0 {
+		unix.Close(fd)
+		return nil, e
+	}
+
+	// create shared-memory ring buffer
+	tp := &unix.TpacketReq{
+		Block_size: BLOCKSIZE,
+		Block_nr:   BLOCKNR,
+		Frame_size: FRAMESIZE,
+		Frame_nr:   FRAMENR,
+	}
+	err = unix.SetsockoptTpacketReq(sock.fd, unix.SOL_PACKET, unix.PACKET_RX_RING, tp)
+	if err != nil {
+		unix.Close(fd)
+		return nil, fmt.Errorf("setsockopt packet_rx_ring: %v", err)
+	}
+	sock.buf, err = unix.Mmap(
+		sock.fd,
+		0,
+		BLOCKSIZE*BLOCKNR,
+		unix.PROT_READ|unix.PROT_WRITE,
+		unix.MAP_SHARED|MAPHUGE2MB,
+	)
+	if err != nil {
+		unix.Close(fd)
+		return nil, fmt.Errorf("socket mmap error: %v", err)
+	}
+	return sock, nil
+}
+
+// ReadPacketData implements gopacket.PacketDataSource.
+func (sock *SockRaw) ZeroCopyReadPacketData() (buf []byte, ci gopacket.CaptureInfo, err error) {
+	sock.mu.Lock()
+	defer sock.mu.Unlock()
+	var tpHdr *unix.Tpacket2Hdr
+	poll := &unix.PollFd{
+		Fd:     int32(sock.fd),
+		Events: unix.POLLIN,
+	}
+	var i int
+read:
+	i = int(sock.frame * FRAMESIZE)
+	tpHdr = (*unix.Tpacket2Hdr)(unsafe.Pointer(&sock.buf[i]))
+	sock.frame = (sock.frame + 1) % FRAMENR
+
+	if tpHdr.Status&unix.TP_STATUS_USER == 0 {
+		_, _, e := unix.Syscall(unix.SYS_POLL, uintptr(unsafe.Pointer(poll)), 1, sock.pollTimeout)
+		if e != 0 && e != unix.EINTR {
+			return buf, ci, e
+		}
+		// it might be some other frame with data!
+		if tpHdr.Status&unix.TP_STATUS_USER == 0 {
+			goto read
+		}
+	}
+	tpHdr.Status = unix.TP_STATUS_KERNEL
+	sockAddr := (*unix.RawSockaddrLinklayer)(unsafe.Pointer(&sock.buf[i+tpacket2hdrlen]))
+
+	// parse out repeating packets on loopback
+	if sockAddr.Ifindex == sock.loopIndex && sock.frame%2 != 0 {
+		goto read
+	}
+
+	ci.Length = int(tpHdr.Len)
+	ci.Timestamp = time.Unix(int64(tpHdr.Sec), int64(tpHdr.Nsec))
+	ci.InterfaceIndex = int(sockAddr.Ifindex)
+	buf = make([]byte, tpHdr.Snaplen)
+	ci.CaptureLength = copy(buf, sock.buf[i+int(tpHdr.Mac):])
+
+	return
+}
+
+// Close closes the underlying socket
+func (sock *SockRaw) Close() (err error) {
+	sock.mu.Lock()
+	defer sock.mu.Unlock()
+	if sock.fd != -1 {
+		unix.Munmap(sock.buf)
+		sock.buf = nil
+		err = unix.Close(sock.fd)
+		sock.fd = -1
+	}
+	return
+}
+
+// SetSnapLen sets the maximum capture length to the given value.
+// for this to take effects on the kernel level SetBPFilter should be called too.
+func (sock *SockRaw) SetSnapLen(snap int) error {
+	sock.mu.Lock()
+	defer sock.mu.Unlock()
+	if snap < 0 {
+		return fmt.Errorf("expected %d snap length to be at least 0", snap)
+	}
+	if snap > FRAMESIZE {
+		snap = FRAMESIZE
+	}
+	sock.snaplen = snap
+	return nil
+}
+
+// SetTimeout sets poll wait timeout for the socket.
+// negative value will block forever
+func (sock *SockRaw) SetTimeout(t time.Duration) error {
+	sock.mu.Lock()
+	defer sock.mu.Unlock()
+	sock.pollTimeout = uintptr(t)
+	return nil
+}
+
+// GetSnapLen returns the maximum capture length
+func (sock *SockRaw) GetSnapLen() int {
+	sock.mu.Lock()
+	defer sock.mu.Unlock()
+	return sock.snaplen
+}
+
+// SetBPFFilter compiles and sets a BPF filter for the socket handle.
+func (sock *SockRaw) SetBPFFilter(expr string) error {
+	sock.mu.Lock()
+	defer sock.mu.Unlock()
+	if expr == "" {
+		return unix.SetsockoptInt(sock.fd, unix.SOL_SOCKET, unix.SO_DETACH_FILTER, 0)
+	}
+	filter, err := pcap.CompileBPFFilter(layers.LinkTypeEthernet, sock.snaplen, expr)
+	if err != nil {
+		return err
+	}
+	if len(filter) > int(^uint16(0)) {
+		return fmt.Errorf("filters out of range 0-%d", ^uint16(0))
+	}
+	if len(filter) == 0 {
+		return unix.SetsockoptInt(sock.fd, unix.SOL_SOCKET, unix.SO_DETACH_FILTER, 0)
+	}
+	fprog := &unix.SockFprog{
+		Len:    uint16(len(filter)),
+		Filter: &(*(*[]unix.SockFilter)(unsafe.Pointer(&filter)))[0],
+	}
+	return unix.SetsockoptSockFprog(sock.fd, unix.SOL_SOCKET, unix.SO_ATTACH_FILTER, fprog)
+}
+
+// SetPromiscuous sets promiscuous mode to the required value. for better result capture on all interfaces instead.
+// If it is enabled, traffic not destined for the interface will also be captured.
+func (sock *SockRaw) SetPromiscuous(b bool) error {
+	sock.mu.Lock()
+	defer sock.mu.Unlock()
+	mreq := unix.PacketMreq{
+		Ifindex: int32(sock.ifindex),
+		Type:    unix.PACKET_MR_PROMISC,
+	}
+
+	opt := unix.PACKET_ADD_MEMBERSHIP
+	if !b {
+		opt = unix.PACKET_DROP_MEMBERSHIP
+	}
+
+	return unix.SetsockoptPacketMreq(sock.fd, unix.SOL_PACKET, opt, &mreq)
+}
+
+// Stats returns number of packets and dropped packets. This will be the number of packets/dropped packets since the last call to stats (not the cummulative sum!).
+func (sock *SockRaw) Stats() (*unix.TpacketStats, error) {
+	sock.mu.Lock()
+	defer sock.mu.Unlock()
+	return unix.GetsockoptTpacketStats(sock.fd, unix.SOL_PACKET, unix.PACKET_STATISTICS)
+}
+
+// SetLoopbackIndex necessary to avoid reading packet twice on a loopback device
+func (sock *SockRaw) SetLoopbackIndex(i int32) {
+	sock.mu.Lock()
+	defer sock.mu.Unlock()
+	sock.loopIndex = i
+}
+
+// WritePacketData transmits a raw packet.
+func (sock *SockRaw) WritePacketData(pkt []byte) error {
+	_, err := unix.Write(sock.fd, pkt)
+	return err
+}
+
+func tpAlign(x int) int {
+	return int((uint(x) + unix.TPACKET_ALIGNMENT - 1) &^ (unix.TPACKET_ALIGNMENT - 1))
+}

capture/sock_others.go

@@ -0,0 +1,14 @@
+// +build !linux
+
+package capture
+
+import (
+	"errors"
+
+	"github.com/google/gopacket/pcap"
+)
+
+// NewSocket returns new M'maped sock_raw on packet version 2.
+func NewSocket(_ pcap.Interface) (Socket, error) {
+	return nil, errors.New("afpacket socket is only available on linux")
+}

capture/socket.go

@@ -0,0 +1,20 @@
+package capture
+
+import (
+	"time"
+
+	"github.com/google/gopacket"
+)
+
+// Socket is any interface that defines the behaviors of Socket
+type Socket interface {
+	ZeroCopyReadPacketData() ([]byte, gopacket.CaptureInfo, error)
+	WritePacketData([]byte) error
+	SetBPFFilter(string) error
+	SetPromiscuous(bool) error
+	SetSnapLen(int) error
+	GetSnapLen() int
+	SetTimeout(time.Duration) error
+	SetLoopbackIndex(i int32)
+	Close() error
+}

docs/Distributed-configuration.md

@@ -13,7 +13,7 @@ If you have multiple replay machines you can split traffic among them using `--s
 gor --input-raw :80 --split-output --output-tcp replay1.local:28020 --output-tcp replay2.local:28020
 ```
 
-[GoReplay PRO](https://goreplay.com/pro.html) support accurate recording and replaying of tcp sessions, and when `--recognize-tcp-sessions` option is passed, instead of round-robin it will use a smarter algorithm which ensures that same sessions will be sent to the same replay instance.
+[GoReplay PRO](https://goreplay.org/pro.html) support accurate recording and replaying of tcp sessions, and when `--recognize-tcp-sessions` option is passed, instead of round-robin it will use a smarter algorithm which ensures that same sessions will be sent to the same replay instance.
 
 
 In case if you are planning a large load testing, you may consider use separate master instance which will control Gor slaves which actually replay traffic. For example:

docs/Middleware.md

@@ -26,7 +26,7 @@ Simple bash echo middleware (returns same request) will look like this:
 ```bash
 while read line; do
   echo $line
-end
+done
 ```
 
 Middleware can be enabled using `--middleware` option, by specifying path to executable file:
@@ -71,4 +71,4 @@ Imagine that you have auth system that randomly generate access tokens, which us
 
 ***
 
-You may also read about [[Request filtering]], [[Rate limiting]] and [[Request rewriting]].
+You may also read about [[Request filtering]], [[Rate limiting]] and [[Request rewriting]].

docs/Request-rewriting.md

@@ -23,16 +23,16 @@ Set request header, if header already exists it will be overwritten. May be usef
 
 ```
 gor --input-raw :80 --output-http "http://staging.server" \
-    --http-header "User-Agent: Replayed by Gor" \
-    --http-header "Enable-Feature-X: true"
+    --http-set-header "User-Agent: Replayed by Gor" \
+    --http-set-header "Enable-Feature-X: true"
 ```
 
 #### Host header
-Host header gets special treatment. By default Host get set to the value specified in --output-http. If you manually set --http-header "Host: anonther.com", Gor will not override Host value.
+Host header gets special treatment. By default Host get set to the value specified in --output-http. If you manually set --http-set-header "Host: anonther.com", Gor will not override Host value.
 
 If you app accepts traffic from multiple domains, and you want to keep original headers, there is specific `--http-original-host` with tells Gor do not touch Host header at all.
 
 
 ***
 
-You may also read about [[Request filtering]], [[Rate limiting]] and [[Middleware]]
+You may also read about [[Request filtering]], [[Rate limiting]] and [[Middleware]]

elasticsearch.go

@@ -2,17 +2,20 @@ package main
 
 import (
 	"encoding/json"
-	"github.com/buger/gor/proto"
-	"github.com/mattbaird/elastigo/lib"
 	"log"
-	"regexp"
+	"net/url"
+	"strings"
 	"time"
+
+	"github.com/buger/goreplay/proto"
+
+	elastigo "github.com/mattbaird/elastigo/lib"
 )
 
 type ESUriErorr struct{}
 
 func (e *ESUriErorr) Error() string {
-	return "Wrong ElasticSearch URL format. Expected to be: host:port/index_name"
+	return "Wrong ElasticSearch URL format. Expected to be: scheme://host/index_name"
 }
 
 type ESPlugin struct {
@@ -52,17 +55,28 @@ type ESRequestResponse struct {
 
 // Parse ElasticSearch URI
 //
-// Proper format is: host:port/index_name
-func parseURI(URI string) (err error, host string, port string, index string) {
-	rURI := regexp.MustCompile("(.+):([0-9]+)/(.+)")
-	match := rURI.FindAllStringSubmatch(URI, -1)
+// Proper format is: scheme://[userinfo@]host/index_name
+// userinfo is: user[:password]
+// net/url.Parse() does not fail if scheme is not provided but actually does not
+// handle URI properly.
+// So we must 'validate' URI format to match requirements to use net/url.Parse()
+func parseURI(URI string) (err error, index string) {
 
-	if len(match) == 0 {
+	parsedUrl, parseErr := url.Parse(URI)
+
+	if parseErr != nil {
+		err = new(ESUriErorr)
+		return
+	}
+
+	//	check URL validity by extracting host and index values.
+	host := parsedUrl.Host
+	urlPathParts := strings.Split(parsedUrl.Path, "/")
+	index = urlPathParts[len(urlPathParts)-1]
+
+	// force index specification in uri : ie no implicit index
+	if host == "" || index == "" {
 		err = new(ESUriErorr)
-	} else {
-		host = match[0][1]
-		port = match[0][2]
-		index = match[0][3]
 	}
 
 	return
@@ -71,26 +85,23 @@ func parseURI(URI string) (err error, host string, port string, index string) {
 func (p *ESPlugin) Init(URI string) {
 	var err error
 
-	err, p.Host, p.ApiPort, p.Index = parseURI(URI)
+	err, p.Index = parseURI(URI)
 
 	if err != nil {
 		log.Fatal("Can't initialize ElasticSearch plugin.", err)
 	}
+
 	p.eConn = elastigo.NewConn()
-	p.eConn.SetPort(p.ApiPort)
-	p.eConn.SetHosts([]string{p.Host})
+
+	p.eConn.SetFromUrl(URI)
 
 	p.indexor = p.eConn.NewBulkIndexerErrors(50, 60)
 	p.done = make(chan bool)
 	p.indexor.Start()
 
-	if Settings.verbose {
-		// Only start the ErrorHandler goroutine when in verbose mode
-		// no need to burn ressources otherwise
-		go p.ErrorHandler()
-	}
+	go p.ErrorHandler()
 
-	log.Println("Initialized Elasticsearch Plugin")
+	Debug(1, "Initialized Elasticsearch Plugin")
 	return
 }
 
@@ -102,7 +113,7 @@ func (p *ESPlugin) IndexerShutdown() {
 func (p *ESPlugin) ErrorHandler() {
 	for {
 		errBuf := <-p.indexor.ErrorChannel
-		log.Println(errBuf.Err)
+		Debug(1, "[ELASTICSEARCH]", errBuf.Err)
 	}
 }
 
@@ -113,6 +124,7 @@ func (p *ESPlugin) RttDurationToMs(d time.Duration) int64 {
 	return int64(fl)
 }
 
+// ResponseAnalyze send req and resp to ES
 func (p *ESPlugin) ResponseAnalyze(req, resp []byte, start, stop time.Time) {
 	if len(resp) == 0 {
 		// nil http response - skipped elasticsearch export for this request
@@ -120,7 +132,6 @@ func (p *ESPlugin) ResponseAnalyze(req, resp []byte, start, stop time.Time) {
 	}
 	t := time.Now()
 	rtt := p.RttDurationToMs(stop.Sub(start))
-	req = payloadBody(req)
 
 	esResp := ESRequestResponse{
 		ReqURL:               string(proto.Path(req)),
@@ -148,7 +159,7 @@ func (p *ESPlugin) ResponseAnalyze(req, resp []byte, start, stop time.Time) {
 	}
 	j, err := json.Marshal(&esResp)
 	if err != nil {
-		log.Println(err)
+		Debug(0, "[ELASTIC-RESPONSE]", err)
 	} else {
 		p.indexor.Index(p.Index, "RequestResponse", "", "", "", &t, j)
 	}

elasticsearch_test.go

@@ -0,0 +1,140 @@
+package main
+
+import (
+	"testing"
+)
+
+const expectedIndex = "gor"
+
+func assertExpectedGorIndex(index string, t *testing.T) {
+	if expectedIndex != index {
+		t.Fatalf("Expected index %s but got %s", expectedIndex, index)
+	}
+}
+
+func assertExpectedIndex(expectedIndex string, index string, t *testing.T) {
+	if expectedIndex != index {
+		t.Fatalf("Expected index %s but got %s", expectedIndex, index)
+	}
+}
+
+func assertExpectedError(returnedError error, t *testing.T) {
+	expectedError := new(ESUriErorr)
+
+	if expectedError != returnedError {
+		t.Errorf("Expected err %s but got %s", expectedError, returnedError)
+	}
+}
+
+func assertNoError(returnedError error, t *testing.T) {
+	if nil != returnedError {
+		t.Errorf("Expected no err but got %s", returnedError)
+	}
+}
+
+// Argument host:port/index_name
+// i.e : localhost:9200/gor
+// Fail because scheme is mandatory
+func TestElasticConnectionBuildFailWithoutScheme(t *testing.T) {
+	uri := "localhost:9200/" + expectedIndex
+
+	err, _ := parseURI(uri)
+	assertExpectedError(err, t)
+}
+
+// Argument scheme://Host:port
+// i.e : http://localhost:9200
+// Fail : explicit index is required
+func TestElasticConnectionBuildFailWithoutIndex(t *testing.T) {
+	uri := "http://localhost:9200"
+
+	err, index := parseURI(uri)
+
+	assertExpectedIndex("", index, t)
+
+	assertExpectedError(err, t)
+}
+
+// Argument scheme://Host/index_name
+// i.e : http://localhost/gor
+func TestElasticConnectionBuildFailWithoutPort(t *testing.T) {
+	uri := "http://localhost/" + expectedIndex
+
+	err, index := parseURI(uri)
+
+	assertNoError(err, t)
+
+	assertExpectedGorIndex(index, t)
+}
+
+// Argument scheme://Host:port/index_name
+// i.e : http://localhost:9200/gor
+func TestElasticLocalConnectionBuild(t *testing.T) {
+	uri := "http://localhost:9200/" + expectedIndex
+
+	err, index := parseURI(uri)
+
+	assertNoError(err, t)
+
+	assertExpectedGorIndex(index, t)
+}
+
+// Argument scheme://Host:port/index_name
+// i.e : http://localhost.local:9200/gor or https://localhost.local:9200/gor
+func TestElasticSimpleLocalWithSchemeConnectionBuild(t *testing.T) {
+	uri := "http://localhost.local:9200/" + expectedIndex
+
+	err, index := parseURI(uri)
+
+	assertNoError(err, t)
+
+	assertExpectedGorIndex(index, t)
+}
+
+// Argument scheme://Host:port/index_name
+// i.e : http://localhost.local:9200/gor or https://localhost.local:9200/gor
+func TestElasticSimpleLocalWithHTTPSConnectionBuild(t *testing.T) {
+	uri := "https://localhost.local:9200/" + expectedIndex
+
+	err, index := parseURI(uri)
+
+	assertNoError(err, t)
+
+	assertExpectedGorIndex(index, t)
+}
+
+// Argument scheme://Host:port/index_name
+// i.e : localhost.local:9200/pathtoElastic/gor
+func TestElasticLongPathConnectionBuild(t *testing.T) {
+	uri := "http://localhost.local:9200/pathtoElastic/" + expectedIndex
+
+	err, index := parseURI(uri)
+
+	assertNoError(err, t)
+
+	assertExpectedGorIndex(index, t)
+}
+
+// Argument scheme://Host:userinfo@port/index_name
+// i.e : http://user:password@localhost.local:9200/gor
+func TestElasticBasicAuthConnectionBuild(t *testing.T) {
+	uri := "http://user:password@localhost.local:9200/" + expectedIndex
+
+	err, index := parseURI(uri)
+
+	assertNoError(err, t)
+
+	assertExpectedGorIndex(index, t)
+}
+
+// Argument scheme://Host:port/path/index_name
+// i.e : http://localhost.local:9200/path/gor or https://localhost.local:9200/path/gor
+func TestElasticComplexPathConnectionBuild(t *testing.T) {
+	uri := "http://localhost.local:9200/path/" + expectedIndex
+
+	err, index := parseURI(uri)
+
+	assertNoError(err, t)
+
+	assertExpectedGorIndex(index, t)
+}

emitter.go

@@ -1,139 +1,177 @@
 package main
 
 import (
-	"bytes"
+	"fmt"
+	"hash/fnv"
 	"io"
+	"log"
+	"sync"
 	"time"
+
+	"github.com/buger/goreplay/byteutils"
 )
 
-// Start initialize loop for sending data from inputs to outputs
-func Start(stop chan int) {
-	if Settings.middleware != "" {
-		middleware := NewMiddleware(Settings.middleware)
+// Emitter represents an abject to manage plugins communication
+type Emitter struct {
+	sync.WaitGroup
+	plugins *InOutPlugins
+}
 
-		for _, in := range Plugins.Inputs {
+// NewEmitter creates and initializes new Emitter object.
+func NewEmitter() *Emitter {
+	return &Emitter{}
+}
+
+// Start initialize loop for sending data from inputs to outputs
+func (e *Emitter) Start(plugins *InOutPlugins, middlewareCmd string) {
+	if Settings.CopyBufferSize < 1 {
+		Settings.CopyBufferSize = 5 << 20
+	}
+	e.plugins = plugins
+
+	if middlewareCmd != "" {
+		middleware := NewMiddleware(middlewareCmd)
+
+		for _, in := range plugins.Inputs {
 			middleware.ReadFrom(in)
 		}
 
-		// We are going only to read responses, so using same ReadFrom method
-		for _, out := range Plugins.Outputs {
-			if r, ok := out.(io.Reader); ok {
-				middleware.ReadFrom(r)
+		e.plugins.Inputs = append(e.plugins.Inputs, middleware)
+		e.plugins.All = append(e.plugins.All, middleware)
+		e.Add(1)
+		go func() {
+			defer e.Done()
+			if err := CopyMulty(middleware, plugins.Outputs...); err != nil {
+				Debug(2, fmt.Sprintf("[EMITTER] error during copy: %q", err))
 			}
-		}
-
-		go CopyMulty(middleware, Plugins.Outputs...)
+		}()
 	} else {
-		for _, in := range Plugins.Inputs {
-			go CopyMulty(in, Plugins.Outputs...)
+		for _, in := range plugins.Inputs {
+			e.Add(1)
+			go func(in PluginReader) {
+				defer e.Done()
+				if err := CopyMulty(in, plugins.Outputs...); err != nil {
+					Debug(2, fmt.Sprintf("[EMITTER] error during copy: %q", err))
+				}
+			}(in)
 		}
 	}
+}
 
-	for {
-		select {
-		case <-stop:
-			finalize()
-			return
-		case <-time.After(100 * time.Millisecond):
+// Close closes all the goroutine and waits for it to finish.
+func (e *Emitter) Close() {
+	for _, p := range e.plugins.All {
+		if cp, ok := p.(io.Closer); ok {
+			cp.Close()
 		}
 	}
+	if len(e.plugins.All) > 0 {
+		// wait for everything to stop
+		e.Wait()
+	}
+	e.plugins.All = nil // avoid Close to make changes again
 }
 
 // CopyMulty copies from 1 reader to multiple writers
-func CopyMulty(src io.Reader, writers ...io.Writer) (err error) {
-	buf := make([]byte, 5*1024*1024)
+func CopyMulty(src PluginReader, writers ...PluginWriter) error {
 	wIndex := 0
-	modifier := NewHTTPModifier(&Settings.modifierConfig)
-	filteredRequests := make(map[string]time.Time)
-	filteredRequestsLastCleanTime := time.Now()
-
-	i := 0
+	modifier := NewHTTPModifier(&Settings.ModifierConfig)
+	filteredRequests := make(map[string]int64)
+	filteredRequestsLastCleanTime := time.Now().UnixNano()
+	filteredCount := 0
 
 	for {
-		nr, er := src.Read(buf)
-
-		if nr > 0 && len(buf) > nr {
-			payload := buf[:nr]
-			meta := payloadMeta(payload)
-			requestID := string(meta[1])
-
-			_maxN := nr
-			if nr > 500 {
-				_maxN = 500
+		msg, err := src.PluginRead()
+		if err != nil {
+			if err == ErrorStopped || err == io.EOF {
+				return nil
 			}
-
-			if Settings.debug {
-				Debug("[EMITTER] input:", string(payload[0:_maxN]), nr, "from:", src)
+			return err
+		}
+		if msg != nil && len(msg.Data) > 0 {
+			if len(msg.Data) > int(Settings.CopyBufferSize) {
+				msg.Data = msg.Data[:Settings.CopyBufferSize]
+			}
+			meta := payloadMeta(msg.Meta)
+			if len(meta) < 3 {
+				Debug(2, fmt.Sprintf("[EMITTER] Found malformed record %q from %q", msg.Meta, src))
+				continue
+			}
+			requestID := byteutils.SliceToString(meta[1])
+			// start a subroutine only when necessary
+			if Settings.Verbose >= 3 {
+				Debug(3, "[EMITTER] input: ", byteutils.SliceToString(msg.Meta[:len(msg.Meta)-1]), " from: ", src)
 			}
-
 			if modifier != nil {
-				if isRequestPayload(payload) {
-					headSize := bytes.IndexByte(payload, '\n') + 1
-					body := payload[headSize:]
-					originalBodyLen := len(body)
-					body = modifier.Rewrite(body)
-
+				Debug(3, "[EMITTER] modifier:", requestID, "from:", src)
+				if isRequestPayload(msg.Meta) {
+					msg.Data = modifier.Rewrite(msg.Data)
 					// If modifier tells to skip request
-					if len(body) == 0 {
-						filteredRequests[requestID] = time.Now()
+					if len(msg.Data) == 0 {
+						filteredRequests[requestID] = time.Now().UnixNano()
+						filteredCount++
 						continue
 					}
+					Debug(3, "[EMITTER] Rewritten input:", requestID, "from:", src)
 
-					if originalBodyLen != len(body) {
-						payload = append(payload[:headSize], body...)
-					}
-
-					if Settings.debug {
-						Debug("[EMITTER] Rewritten input:", len(payload), "First 500 bytes:", string(payload[0:_maxN]))
-					}
 				} else {
 					if _, ok := filteredRequests[requestID]; ok {
-						delete(filteredRequests, requestID);
+						delete(filteredRequests, requestID)
+						filteredCount--
 						continue
 					}
 				}
 			}
 
-			if Settings.splitOutput {
-				// Simple round robin
-				writers[wIndex].Write(payload)
+			if Settings.PrettifyHTTP {
+				msg.Data = prettifyHTTP(msg.Data)
+				if len(msg.Data) == 0 {
+					continue
+				}
+			}
 
-				wIndex++
+			if Settings.SplitOutput {
+				if Settings.RecognizeTCPSessions {
+					if !PRO {
+						log.Fatal("Detailed TCP sessions work only with PRO license")
+					}
+					hasher := fnv.New32a()
+					hasher.Write(meta[1])
 
-				if wIndex >= len(writers) {
-					wIndex = 0
+					wIndex = int(hasher.Sum32()) % len(writers)
+					if _, err := writers[wIndex].PluginWrite(msg); err != nil {
+						return err
+					}
+				} else {
+					// Simple round robin
+					if _, err := writers[wIndex].PluginWrite(msg); err != nil {
+						return err
+					}
+
+					wIndex = (wIndex + 1) % len(writers)
 				}
 			} else {
 				for _, dst := range writers {
-					dst.Write(payload)
+					if _, err := dst.PluginWrite(msg); err != nil {
+						return err
+					}
 				}
 			}
-
-		}
-		if er == io.EOF {
-			break
-		}
-		if er != nil {
-			err = er
-			break
 		}
 
 		// Run GC on each 1000 request
-		if i % 1000 == 0 {
+		if filteredCount > 0 && filteredCount%1000 == 0 {
 			// Clean up filtered requests for which we didn't get a response to filter
-			now := time.Now()
-			if now.Sub(filteredRequestsLastCleanTime) > 60 * time.Second {
+			now := time.Now().UnixNano()
+			if now-filteredRequestsLastCleanTime > int64(60*time.Second) {
 				for k, v := range filteredRequests {
-					if now.Sub(v) > 60 * time.Second {
+					if now-v > int64(60*time.Second) {
 						delete(filteredRequests, k)
+						filteredCount--
 					}
 				}
-				filteredRequestsLastCleanTime = time.Now()
+				filteredRequestsLastCleanTime = time.Now().UnixNano()
 			}
 		}
-
-		i++
 	}
-
-	return err
 }

emitter_test.go

@@ -1,26 +1,36 @@
 package main
 
 import (
-	"io"
+	"fmt"
+	"os"
 	"sync"
 	"sync/atomic"
 	"testing"
 	"time"
 )
 
+func TestMain(m *testing.M) {
+	PRO = true
+	code := m.Run()
+	os.Exit(code)
+}
+
 func TestEmitter(t *testing.T) {
 	wg := new(sync.WaitGroup)
-	quit := make(chan int)
 
 	input := NewTestInput()
-	output := NewTestOutput(func(data []byte) {
+	output := NewTestOutput(func(*Message) {
 		wg.Done()
 	})
 
-	Plugins.Inputs = []io.Reader{input}
-	Plugins.Outputs = []io.Writer{output}
+	plugins := &InOutPlugins{
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{output},
+	}
+	plugins.All = append(plugins.All, input, output)
 
-	go Start(quit)
+	emitter := NewEmitter()
+	go emitter.Start(plugins, Settings.Middleware)
 
 	for i := 0; i < 1000; i++ {
 		wg.Add(1)
@@ -28,33 +38,36 @@ func TestEmitter(t *testing.T) {
 	}
 
 	wg.Wait()
-
-	close(quit)
+	emitter.Close()
 }
 
 func TestEmitterFiltered(t *testing.T) {
 	wg := new(sync.WaitGroup)
-	quit := make(chan int)
 
 	input := NewTestInput()
 	input.skipHeader = true
 
-	output := NewTestOutput(func(data []byte) {
+	output := NewTestOutput(func(*Message) {
 		wg.Done()
 	})
 
-	Plugins.Inputs = []io.Reader{input}
-	Plugins.Outputs = []io.Writer{output}
-	methods := HTTPMethods{[]byte("GET")}
-	Settings.modifierConfig = HTTPModifierConfig{methods: methods}
+	plugins := &InOutPlugins{
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{output},
+	}
+	plugins.All = append(plugins.All, input, output)
 
-	go Start(quit)
+	methods := HTTPMethods{[]byte("GET")}
+	Settings.ModifierConfig = HTTPModifierConfig{Methods: methods}
+
+	emitter := &Emitter{}
+	go emitter.Start(plugins, "")
 
 	wg.Add(2)
 
 	id := uuid()
 	reqh := payloadHeader(RequestPayload, id, time.Now().UnixNano(), -1)
-	reqb := append(reqh, []byte("GET / HTTP/1.1\r\nHost: www.w3.org\r\nUser-Agent: Go 1.1 package http\r\nAccept-Encoding: gzip\r\n\r\n")...)
+	reqb := append(reqh, []byte("POST / HTTP/1.1\r\nHost: www.w3.org\r\nUser-Agent: Go 1.1 package http\r\nAccept-Encoding: gzip\r\n\r\n")...)
 
 	resh := payloadHeader(ResponsePayload, id, time.Now().UnixNano()+1, 1)
 	respb := append(resh, []byte("HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n")...)
@@ -64,7 +77,7 @@ func TestEmitterFiltered(t *testing.T) {
 
 	id = uuid()
 	reqh = payloadHeader(RequestPayload, id, time.Now().UnixNano(), -1)
-	reqb = append(reqh, []byte("POST / HTTP/1.1\r\nHost: www.w3.org\r\nUser-Agent: Go 1.1 package http\r\nAccept-Encoding: gzip\r\n\r\n")...)
+	reqb = append(reqh, []byte("GET / HTTP/1.1\r\nHost: www.w3.org\r\nUser-Agent: Go 1.1 package http\r\nAccept-Encoding: gzip\r\n\r\n")...)
 
 	resh = payloadHeader(ResponsePayload, id, time.Now().UnixNano()+1, 1)
 	respb = append(resh, []byte("HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n")...)
@@ -73,34 +86,37 @@ func TestEmitterFiltered(t *testing.T) {
 	input.EmitBytes(respb)
 
 	wg.Wait()
+	emitter.Close()
 
-	close(quit)
+	Settings.ModifierConfig = HTTPModifierConfig{}
 }
 
-func TestEmitterRoundRobin(t *testing.T) {
+func TestEmitterSplitRoundRobin(t *testing.T) {
 	wg := new(sync.WaitGroup)
-	quit := make(chan int)
 
 	input := NewTestInput()
 
 	var counter1, counter2 int32
 
-	output1 := NewTestOutput(func(data []byte) {
+	output1 := NewTestOutput(func(*Message) {
 		atomic.AddInt32(&counter1, 1)
 		wg.Done()
 	})
 
-	output2 := NewTestOutput(func(data []byte) {
+	output2 := NewTestOutput(func(*Message) {
 		atomic.AddInt32(&counter2, 1)
 		wg.Done()
 	})
 
-	Plugins.Inputs = []io.Reader{input}
-	Plugins.Outputs = []io.Writer{output1, output2}
+	plugins := &InOutPlugins{
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{output1, output2},
+	}
 
-	Settings.splitOutput = true
+	Settings.SplitOutput = true
 
-	go Start(quit)
+	emitter := NewEmitter()
+	go emitter.Start(plugins, Settings.Middleware)
 
 	for i := 0; i < 1000; i++ {
 		wg.Add(1)
@@ -109,29 +125,131 @@ func TestEmitterRoundRobin(t *testing.T) {
 
 	wg.Wait()
 
-	close(quit)
+	emitter.Close()
+
+	if counter1 == 0 || counter2 == 0 || counter1 != counter2 {
+		t.Errorf("Round robin should split traffic equally: %d vs %d", counter1, counter2)
+	}
+
+	Settings.SplitOutput = false
+}
+
+func TestEmitterRoundRobin(t *testing.T) {
+	wg := new(sync.WaitGroup)
+
+	input := NewTestInput()
+
+	var counter1, counter2 int32
+
+	output1 := NewTestOutput(func(*Message) {
+		counter1++
+		wg.Done()
+	})
+
+	output2 := NewTestOutput(func(*Message) {
+		counter2++
+		wg.Done()
+	})
+
+	plugins := &InOutPlugins{
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{output1, output2},
+	}
+	plugins.All = append(plugins.All, input, output1, output2)
+
+	Settings.SplitOutput = true
+
+	emitter := NewEmitter()
+	go emitter.Start(plugins, Settings.Middleware)
+
+	for i := 0; i < 1000; i++ {
+		wg.Add(1)
+		input.EmitGET()
+	}
+
+	wg.Wait()
+	emitter.Close()
 
 	if counter1 == 0 || counter2 == 0 {
 		t.Errorf("Round robin should split traffic equally: %d vs %d", counter1, counter2)
 	}
 
-	Settings.splitOutput = false
+	Settings.SplitOutput = false
+}
+
+func TestEmitterSplitSession(t *testing.T) {
+	wg := new(sync.WaitGroup)
+	wg.Add(200)
+
+	input := NewTestInput()
+	input.skipHeader = true
+
+	var counter1, counter2 int32
+
+	output1 := NewTestOutput(func(msg *Message) {
+		if payloadID(msg.Meta)[0] == 'a' {
+			counter1++
+		}
+		wg.Done()
+	})
+
+	output2 := NewTestOutput(func(msg *Message) {
+		if payloadID(msg.Meta)[0] == 'b' {
+			counter2++
+		}
+		wg.Done()
+	})
+
+	plugins := &InOutPlugins{
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{output1, output2},
+	}
+
+	Settings.SplitOutput = true
+	Settings.RecognizeTCPSessions = true
+
+	emitter := NewEmitter()
+	go emitter.Start(plugins, Settings.Middleware)
+
+	for i := 0; i < 200; i++ {
+		// Keep session but randomize
+		id := make([]byte, 20)
+		if i&1 == 0 { // for recognizeTCPSessions one should be odd and other will be even number
+			id[0] = 'a'
+		} else {
+			id[0] = 'b'
+		}
+		input.EmitBytes([]byte(fmt.Sprintf("1 %s 1 1\nGET / HTTP/1.1\r\n\r\n", id[:20])))
+	}
+
+	wg.Wait()
+
+	if counter1 != counter2 {
+		t.Errorf("Round robin should split traffic equally: %d vs %d", counter1, counter2)
+	}
+
+	Settings.SplitOutput = false
+	Settings.RecognizeTCPSessions = false
+	emitter.Close()
 }
 
 func BenchmarkEmitter(b *testing.B) {
 	wg := new(sync.WaitGroup)
-	quit := make(chan int)
 
 	input := NewTestInput()
 
-	output := NewTestOutput(func(data []byte) {
+	output := NewTestOutput(func(*Message) {
 		wg.Done()
 	})
 
-	Plugins.Inputs = []io.Reader{input}
-	Plugins.Outputs = []io.Writer{output}
+	plugins := &InOutPlugins{
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{output},
+	}
+	plugins.All = append(plugins.All, input, output)
 
-	go Start(quit)
+	emitter := NewEmitter()
+	go emitter.Start(plugins, Settings.Middleware)
 
 	b.ResetTimer()
 
@@ -141,5 +259,5 @@ func BenchmarkEmitter(b *testing.B) {
 	}
 
 	wg.Wait()
-	close(quit)
+	emitter.Close()
 }

examples/middleware/echo.js

@@ -0,0 +1,57 @@
+#!/usr/bin/env node
+const readline = require("readline");
+const StringDecoder = require("string_decoder").StringDecoder
+
+const rl = readline.createInterface({
+  input: process.stdin
+});
+
+var ignoreIds = new Set();
+var ignoreAddresses = "/api";
+const decoder = new StringDecoder("utf8");
+
+function convertHexString(hex) {
+  var bytes = [];
+  for (var i = 0; i < hex.length - 1; i += 2) {
+    bytes.push(parseInt(hex.substr(i, 2), 16));
+  }
+  return decoder.write(Buffer.from(bytes));
+}
+
+function log(output) {
+	console.error("===================");
+	console.error(output);
+}
+
+function shouldOutputLine(request) {
+  const components = request.split("\n");
+  const header = components[0].split(" ");
+  const type = parseInt(header[0]);
+  const tag = header[1];
+
+  if (type === 3) {
+    return true;
+  }
+  if (type === 1) {
+    // Check if it's oauth
+    const endpoint = components[1].split(" ")[1];
+    if (!endpoint.startsWith(ignoreAddresses)) {
+      ignoreIds.add(tag);
+      return false;
+    }
+  } else if (type === 2) {
+    if (ignoreIds.has(tag)) {
+      ignoreIds.delete(tag);
+      return false;
+    }
+  }
+  return true;
+}
+
+rl.on("line", (input) => {
+  const str = convertHexString(input);
+  console.log(input);
+  if (shouldOutputLine(str)) {
+    log(str);
+	}
+});

examples/middleware/echo.py

@@ -0,0 +1,76 @@
+#! /usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+import sys
+import fileinput
+import binascii
+
+# Used to find end of the Headers section
+EMPTY_LINE = b'\r\n\r\n'
+
+
+def log(msg):
+    """
+    Logging to STDERR as STDOUT and STDIN used for data transfer
+    @type msg: str or byte string
+    @param msg: Message to log to STDERR
+    """
+    try:
+        msg = str(msg) + '\n'
+    except:
+        pass
+    sys.stderr.write(msg)
+    sys.stderr.flush()
+
+
+def find_end_of_headers(byte_data):
+    """
+    Finds where the header portion ends and the content portion begins.
+    @type byte_data: str or byte string
+    @param byte_data: Hex decoded req or resp string
+    """
+    return byte_data.index(EMPTY_LINE) + 4
+
+
+def process_stdin():
+    """
+    Process STDIN and output to STDOUT
+    """
+    for raw_line in fileinput.input():
+
+        line = raw_line.rstrip()
+
+        # Decode base64 encoded line
+        decoded = bytes.fromhex(line)
+
+        # Split into metadata and payload, the payload is headers + body
+        (raw_metadata, payload) = decoded.split(b'\n', 1)
+
+        # Split into headers and payload
+        headers_pos = find_end_of_headers(payload)
+        raw_headers = payload[:headers_pos]
+        raw_content = payload[headers_pos:]
+
+        log('===================================')
+        request_type_id = int(raw_metadata.split(b' ')[0])
+        log('Request type: {}'.format({
+          1: 'Request',
+          2: 'Original Response',
+          3: 'Replayed Response'
+        }[request_type_id]))
+        log('===================================')
+
+        log('Original data:')
+        log(line)
+
+        log('Decoded request:')
+        log(decoded)
+
+        encoded = binascii.hexlify(raw_metadata + b'\n' + raw_headers + raw_content).decode('ascii')
+        log('Encoded data:')
+        log(encoded)
+
+        sys.stdout.write(encoded + '\n')
+
+if __name__ == '__main__':
+    process_stdin()

examples/middleware/echo.rb

@@ -1,12 +1,12 @@
 #!/usr/bin/env ruby
 # encoding: utf-8
-while data = STDIN.gets # continiously read line from STDIN
+while data = STDIN.gets # continuously read line from STDIN
   next unless data
   data = data.chomp # remove end of line symbol
   
   decoded = [data].pack("H*") # decode base64 encoded request
   
-  # dedoded value is raw HTTP payload, example:
+  # decoded value is raw HTTP payload, example:
   #   
   #   POST /post HTTP/1.1
   #   Content-Length: 7

examples/middleware/echo.sh

@@ -8,8 +8,10 @@
 # 
 
 function log {
+    if [[ ! -v GOR_TEST ]]; then # if we are not testing
     # Logging to stderr, because stdout/stdin used for data transfer
     >&2 echo "[DEBUG][ECHO] $1"
+    fi
 }
 
 while read line; do

examples/middleware/token_modifier.go

@@ -24,8 +24,9 @@ import (
 	"bytes"
 	"encoding/hex"
 	"fmt"
-	"github.com/buger/gor/proto"
 	"os"
+
+	"github.com/buger/goreplay/proto"
 )
 
 // requestID -> originalToken
@@ -114,6 +115,8 @@ func encode(buf []byte) []byte {
 }
 
 func Debug(args ...interface{}) {
-	fmt.Fprint(os.Stderr, "[DEBUG][TOKEN-MOD] ")
-	fmt.Fprintln(os.Stderr, args...)
+	if os.Getenv("GOR_TEST") == "" { // if we are not testing
+		fmt.Fprint(os.Stderr, "[DEBUG][TOKEN-MOD] ")
+		fmt.Fprintln(os.Stderr, args...)
+	}
 }

go.mod

@@ -0,0 +1,20 @@
+module github.com/buger/goreplay
+
+go 1.15
+
+require (
+	github.com/Shopify/sarama v1.26.4
+	github.com/araddon/gou v0.0.0-20190110011759-c797efecbb61 // indirect
+	github.com/aws/aws-sdk-go v1.33.2
+	github.com/bitly/go-hostpool v0.1.0 // indirect
+	github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 // indirect
+	github.com/google/gopacket v1.1.18
+	github.com/klauspost/compress v1.10.10 // indirect
+	github.com/mattbaird/elastigo v0.0.0-20170123220020-2fe47fd29e4b
+	github.com/pierrec/lz4 v2.5.2+incompatible // indirect
+	github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0 // indirect
+	github.com/smartystreets/goconvey v1.6.4 // indirect
+	github.com/stretchr/testify v1.5.1
+	golang.org/x/net v0.0.0-20200707034311-ab3426394381 // indirect
+	golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd
+)

go.sum

@@ -0,0 +1,110 @@
+github.com/Shopify/sarama v1.26.4 h1:+17TxUq/PJEAfZAll0T7XJjSgQWCpaQSoki/x5yN8o8=
+github.com/Shopify/sarama v1.26.4/go.mod h1:NbSGBSSndYaIhRcBtY9V0U7AyH+x71bG668AuWys/yU=
+github.com/Shopify/toxiproxy v2.1.4+incompatible h1:TKdv8HiTLgE5wdJuEML90aBgNWsokNbMijUGhmcoBJc=
+github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI=
+github.com/araddon/gou v0.0.0-20190110011759-c797efecbb61 h1:Xz25cuW4REGC5W5UtpMU3QItMIImag615HiQcRbxqKQ=
+github.com/araddon/gou v0.0.0-20190110011759-c797efecbb61/go.mod h1:ikc1XA58M+Rx7SEbf0bLJCfBkwayZ8T5jBo5FXK8Uz8=
+github.com/aws/aws-sdk-go v1.33.2 h1:8TVrnPnSD7I+AmDp66xBUvS3K0J+jH09YXdrkJ34ey0=
+github.com/aws/aws-sdk-go v1.33.2/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0=
+github.com/bitly/go-hostpool v0.1.0 h1:XKmsF6k5el6xHG3WPJ8U0Ku/ye7njX7W81Ng7O2ioR0=
+github.com/bitly/go-hostpool v0.1.0/go.mod h1:4gOCgp6+NZnVqlKyZ/iBZFTAJKembaVENUpMkpg42fw=
+github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 h1:DDGfHa7BWjL4YnC6+E63dPcxHo2sUxDIu8g3QgEJdRY=
+github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/eapache/go-resiliency v1.2.0 h1:v7g92e/KSN71Rq7vSThKaWIq68fL4YHvWyiUKorFR1Q=
+github.com/eapache/go-resiliency v1.2.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs=
+github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 h1:YEetp8/yCZMuEPMUDHG0CW/brkkEp8mzqk2+ODEitlw=
+github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU=
+github.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc=
+github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I=
+github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw=
+github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g=
+github.com/frankban/quicktest v1.7.2 h1:2QxQoC1TS09S7fhCPsrvqYdvP1H5M1P1ih5ABm3BTYk=
+github.com/frankban/quicktest v1.7.2/go.mod h1:jaStnuzAqU1AJdCO0l53JDCJrVDKcS03DbaAcR7Ks/o=
+github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
+github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4=
+github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4=
+github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/gopacket v1.1.18 h1:lum7VRA9kdlvBi7/v2p7/zcbkduHaCH/SVVyurs7OpY=
+github.com/google/gopacket v1.1.18/go.mod h1:UdDNZ1OO62aGYVnPhxT1U6aI7ukYtA/kB8vaU0diBUM=
+github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8=
+github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
+github.com/hashicorp/go-uuid v1.0.2 h1:cfejS+Tpcp13yd5nYHWDI6qVCny6wyX2Mt5SGur2IGE=
+github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
+github.com/jcmturner/gofork v1.0.0 h1:J7uCkflzTEhUZ64xqKnkDxq3kzc96ajM1Gli5ktUem8=
+github.com/jcmturner/gofork v1.0.0/go.mod h1:MK8+TM0La+2rjBD4jE12Kj1pCCxK7d2LK/UM3ncEo0o=
+github.com/jmespath/go-jmespath v0.3.0 h1:OS12ieG61fsCg5+qLJ+SsW9NicxNkg3b25OyT2yCeUc=
+github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik=
+github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
+github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
+github.com/klauspost/compress v1.9.8/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
+github.com/klauspost/compress v1.10.10 h1:a/y8CglcM7gLGYmlbP/stPE5sR3hbhFRUjCBfd/0B3I=
+github.com/klauspost/compress v1.10.10/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
+github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs=
+github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/mattbaird/elastigo v0.0.0-20170123220020-2fe47fd29e4b h1:v29yPGHhOqw7VHEnTeQFAth3SsBrmwc8JfuhNY0G34k=
+github.com/mattbaird/elastigo v0.0.0-20170123220020-2fe47fd29e4b/go.mod h1:5MWrJXKRQyhQdUCF+vu6U5c4nQpg70vW3eHaU0/AYbU=
+github.com/pierrec/lz4 v2.4.1+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
+github.com/pierrec/lz4 v2.5.2+incompatible h1:WCjObylUIOlKy/+7Abdn34TLIkXiA4UWUMhxq9m9ZXI=
+github.com/pierrec/lz4 v2.5.2+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
+github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0 h1:MkV+77GLUNo5oJ0jf870itWm3D0Sjh7+Za9gazKc5LQ=
+github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
+github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM=
+github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
+github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s=
+github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4=
+github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
+github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I=
+github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20200204104054-c9f3fb736b72/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI=
+golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200707034311-ab3426394381 h1:VXak5I6aEWmAXeQjA+QSZzlgNrpq9mjcfDemuexIKsU=
+golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190405154228-4b34438f7a67/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884=
+golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
+golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
+gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/jcmturner/aescts.v1 v1.0.1 h1:cVVZBK2b1zY26haWB4vbBiZrfFQnfbTVrE3xZq6hrEw=
+gopkg.in/jcmturner/aescts.v1 v1.0.1/go.mod h1:nsR8qBOg+OucoIW+WMhB3GspUQXq9XorLnQb9XtvcOo=
+gopkg.in/jcmturner/dnsutils.v1 v1.0.1 h1:cIuC1OLRGZrld+16ZJvvZxVJeKPsvd5eUIvxfoN5hSM=
+gopkg.in/jcmturner/dnsutils.v1 v1.0.1/go.mod h1:m3v+5svpVOhtFAP/wSz+yzh4Mc0Fg7eRhxkJMWSIz9Q=
+gopkg.in/jcmturner/goidentity.v3 v3.0.0 h1:1duIyWiTaYvVx3YX2CYtpJbUFd7/UuPYCfgXtQ3VTbI=
+gopkg.in/jcmturner/goidentity.v3 v3.0.0/go.mod h1:oG2kH0IvSYNIu80dVAyu/yoefjq1mNfM5bm88whjWx4=
+gopkg.in/jcmturner/gokrb5.v7 v7.5.0 h1:a9tsXlIDD9SKxotJMK3niV7rPZAJeX2aD/0yg3qlIrg=
+gopkg.in/jcmturner/gokrb5.v7 v7.5.0/go.mod h1:l8VISx+WGYp+Fp7KRbsiUuXTTOnxIc3Tuvyavf11/WM=
+gopkg.in/jcmturner/rpc.v1 v1.1.0 h1:QHIUxTX1ISuAv9dD2wJ9HWQVuWDX/Zc0PfeC2tjc4rU=
+gopkg.in/jcmturner/rpc.v1 v1.1.0/go.mod h1:YIdkC4XfD6GXbzje11McwsDuOlZQSb9W4vfLvuNnlv8=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=
+gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

gor.go

@@ -4,11 +4,10 @@ package main
 
 import (
 	"flag"
-	"fmt"
-	"io"
 	"log"
 	"net/http"
 	"net/http/httputil"
+	_ "net/http/pprof"
 	"os"
 	"os/signal"
 	"runtime"
@@ -19,13 +18,17 @@ import (
 )
 
 var (
-	mode       string
 	cpuprofile = flag.String("cpuprofile", "", "write cpu profile to file")
 	memprofile = flag.String("memprofile", "", "write memory profile to this file")
 )
 
-func loggingMiddleware(next http.Handler) http.Handler {
+func loggingMiddleware(addr string, next http.Handler) http.Handler {
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path == "/loop" {
+			_, err := http.Get("http://" + addr)
+			log.Println(err)
+		}
+
 		rb, _ := httputil.DumpRequest(r, false)
 		log.Println(string(rb))
 		next.ServeHTTP(w, r)
@@ -33,36 +36,30 @@ func loggingMiddleware(next http.Handler) http.Handler {
 }
 
 func main() {
-	// // Don't exit on panic
-	// defer func() {
-	// 	if r := recover(); r != nil {
-	// 		fmt.Printf("PANIC: pkg: %v %s \n", r, debug.Stack())
-	// 	}
-	// }()
-
-	// If not set via env cariable
-	if len(os.Getenv("GOMAXPROCS")) == 0 {
+	if os.Getenv("GOMAXPROCS") == "" {
 		runtime.GOMAXPROCS(runtime.NumCPU() * 2)
 	}
 
 	args := os.Args[1:]
+	var plugins *InOutPlugins
 	if len(args) > 0 && args[0] == "file-server" {
 		if len(args) != 2 {
 			log.Fatal("You should specify port and IP (optional) for the file server. Example: `gor file-server :80`")
 		}
 		dir, _ := os.Getwd()
 
-		log.Println("Started example file server for current dirrectory on address ", args[1])
+		Debug(0, "Started example file server for current directory on address ", args[1])
 
-		log.Fatal(http.ListenAndServe(args[1], loggingMiddleware(http.FileServer(http.Dir(dir)))))
+		log.Fatal(http.ListenAndServe(args[1], loggingMiddleware(args[1], http.FileServer(http.Dir(dir)))))
 	} else {
 		flag.Parse()
-		InitPlugins()
+		checkSettings()
+		plugins = NewPlugins()
 	}
 
-	fmt.Println("Version:", VERSION)
+	log.Printf("[PPID %d and PID %d] Version:%s\n", os.Getppid(), os.Getpid(), VERSION)
 
-	if len(Plugins.Inputs) == 0 || len(Plugins.Outputs) == 0 {
+	if len(plugins.Inputs) == 0 || len(plugins.Outputs) == 0 {
 		log.Fatal("Required at least 1 input and 1 output")
 	}
 
@@ -74,35 +71,34 @@ func main() {
 		profileCPU(*cpuprofile)
 	}
 
-	c := make(chan os.Signal, 1)
-	signal.Notify(c, os.Interrupt, syscall.SIGTERM)
-	go func() {
-		<-c
-		finalize()
-		os.Exit(1)
-	}()
+	if Settings.Pprof != "" {
+		go func() {
+			log.Println(http.ListenAndServe(Settings.Pprof, nil))
+		}()
+	}
 
-	if Settings.exitAfter > 0 {
-		log.Println("Running gor for a duration of", Settings.exitAfter)
-		closeCh := make(chan int)
+	closeCh := make(chan int)
+	emitter := NewEmitter()
+	go emitter.Start(plugins, Settings.Middleware)
+	if Settings.ExitAfter > 0 {
+		log.Printf("Running gor for a duration of %s\n", Settings.ExitAfter)
 
-		time.AfterFunc(Settings.exitAfter, func() {
-			log.Println("Stopping gor after", Settings.exitAfter)
+		time.AfterFunc(Settings.ExitAfter, func() {
+			log.Printf("gor run timeout %s\n", Settings.ExitAfter)
 			close(closeCh)
 		})
-
-		Start(closeCh)
-	} else {
-		Start(nil)
 	}
-}
-
-func finalize() {
-	for _, p := range Plugins.All {
-		if cp, ok := p.(io.Closer); ok {
-			cp.Close()
-		}
+	c := make(chan os.Signal, 1)
+	signal.Notify(c, os.Interrupt, syscall.SIGTERM)
+	exit := 0
+	select {
+	case <-c:
+		exit = 1
+	case <-closeCh:
+		exit = 0
 	}
+	emitter.Close()
+	os.Exit(exit)
 }
 
 func profileCPU(cpuprofile string) {
@@ -116,7 +112,6 @@ func profileCPU(cpuprofile string) {
 		time.AfterFunc(30*time.Second, func() {
 			pprof.StopCPUProfile()
 			f.Close()
-			log.Println("Stop profiling after 30 seconds")
 		})
 	}
 }

gor_stat.go

@@ -1,46 +1,42 @@
 package main
 
 import (
-	"log"
 	"runtime"
 	"strconv"
 	"time"
 )
 
-const (
-	rate = 5
-)
-
 type GorStat struct {
 	statName string
+	rateMs   int
 	latest   int
 	mean     int
 	max      int
 	count    int
 }
 
-func NewGorStat(statName string) (s *GorStat) {
+func NewGorStat(statName string, rateMs int) (s *GorStat) {
 	s = new(GorStat)
 	s.statName = statName
+	s.rateMs = rateMs
 	s.latest = 0
 	s.mean = 0
 	s.max = 0
 	s.count = 0
 
-	if Settings.stats {
-		log.Println(s.statName + ":latest,mean,max,count,count/second,gcount")
+	if Settings.Stats {
 		go s.reportStats()
 	}
 	return
 }
 
 func (s *GorStat) Write(latest int) {
-	if Settings.stats {
+	if Settings.Stats {
 		if latest > s.max {
 			s.max = latest
 		}
 		if latest != 0 {
-			s.mean = (s.mean + latest) / 2
+			s.mean = ((s.mean * s.count) + latest) / (s.count + 1)
 		}
 		s.latest = latest
 		s.count = s.count + 1
@@ -55,13 +51,14 @@ func (s *GorStat) Reset() {
 }
 
 func (s *GorStat) String() string {
-	return s.statName + ":" + strconv.Itoa(s.latest) + "," + strconv.Itoa(s.mean) + "," + strconv.Itoa(s.max) + "," + strconv.Itoa(s.count) + "," + strconv.Itoa(s.count/rate) + "," + strconv.Itoa(runtime.NumGoroutine())
+	return s.statName + ":" + strconv.Itoa(s.latest) + "," + strconv.Itoa(s.mean) + "," + strconv.Itoa(s.max) + "," + strconv.Itoa(s.count) + "," + strconv.Itoa(s.count/(s.rateMs/1000.0)) + "," + strconv.Itoa(runtime.NumGoroutine())
 }
 
 func (s *GorStat) reportStats() {
+	Debug(0, "\n", s.statName+":latest,mean,max,count,count/second,gcount")
 	for {
-		log.Println(s)
+		Debug(0, "\n", s)
 		s.Reset()
-		time.Sleep(rate * time.Second)
+		time.Sleep(time.Duration(s.rateMs) * time.Millisecond)
 	}
 }

http_client.go

@@ -1,372 +0,0 @@
-package main
-
-import (
-	"bytes"
-	"crypto/tls"
-	"encoding/base64"
-	"io"
-	"log"
-	"net"
-	"net/url"
-	"runtime/debug"
-	"strconv"
-	"strings"
-	"sync"
-	"syscall"
-	"time"
-
-	"github.com/buger/gor/proto"
-)
-
-var httpMu sync.Mutex
-
-const (
-	readChunkSize   = 64 * 1024
-	maxResponseSize = 1073741824
-)
-
-var chunkedSuffix = []byte("0\r\n\r\n")
-
-var defaultPorts = map[string]string{
-	"http":  "80",
-	"https": "443",
-}
-
-type HTTPClientConfig struct {
-	FollowRedirects    int
-	Debug              bool
-	OriginalHost       bool
-	ConnectionTimeout  time.Duration
-	Timeout            time.Duration
-	ResponseBufferSize int
-}
-
-type HTTPClient struct {
-	baseURL        string
-	scheme         string
-	host           string
-	auth           string
-	conn           net.Conn
-	respBuf        []byte
-	config         *HTTPClientConfig
-	redirectsCount int
-}
-
-func NewHTTPClient(baseURL string, config *HTTPClientConfig) *HTTPClient {
-	if !strings.HasPrefix(baseURL, "http") {
-		baseURL = "http://" + baseURL
-	}
-
-	u, _ := url.Parse(baseURL)
-
-	if config.Timeout == 0 {
-		config.Timeout = time.Second
-    }
-
-	config.ConnectionTimeout = config.Timeout
-
-	if config.ResponseBufferSize == 0 {
-		config.ResponseBufferSize = 100 * 1024 // 100kb
-	}
-
-	client := new(HTTPClient)
-	client.baseURL = u.String()
-	client.host = u.Host
-	client.scheme = u.Scheme
-	client.respBuf = make([]byte, config.ResponseBufferSize)
-	client.config = config
-
-	if u.User != nil {
-		client.auth = "Basic " + base64.StdEncoding.EncodeToString([]byte(u.User.String()))
-	}
-
-	return client
-}
-
-func (c *HTTPClient) Connect() (err error) {
-	c.Disconnect()
-
-	if !strings.Contains(c.host, ":") {
-		c.conn, err = net.DialTimeout("tcp", c.host + ":" + defaultPorts[c.scheme], c.config.ConnectionTimeout)
-	} else {
-		c.conn, err = net.DialTimeout("tcp", c.host, c.config.ConnectionTimeout)
-	}
-
-	if c.scheme == "https" {
-		tlsConn := tls.Client(c.conn, &tls.Config{InsecureSkipVerify: true, ServerName: c.host})
-
-		if err = tlsConn.Handshake(); err != nil {
-			return
-		}
-
-		c.conn = tlsConn
-	}
-
-	return
-}
-
-func (c *HTTPClient) Disconnect() {
-	if c.conn != nil {
-		c.conn.Close()
-		c.conn = nil
-		Debug("[HTTP] Disconnected: ", c.baseURL)
-	}
-}
-
-func (c *HTTPClient) isAlive() bool {
-	one := make([]byte, 1)
-
-	// Ready 1 byte from socket without timeout to check if it not closed
-	c.conn.SetReadDeadline(time.Now().Add(time.Millisecond))
-	_, err := c.conn.Read(one)
-
-	if err == nil {
-		return true
-	} else if err == io.EOF {
-		if c.config.Debug {
-			Debug("[HTTPClient] connection closed, reconnecting")
-		}
-		return false
-	} else if err == syscall.EPIPE {
-		Debug("Detected broken pipe.", err)
-		return false
-	}
-
-	return true
-}
-
-func (c *HTTPClient) Send(data []byte) (response []byte, err error) {
-	// Don't exit on panic
-	defer func() {
-		if r := recover(); r != nil {
-			Debug("[HTTPClient]", r, string(data))
-
-			if _, ok := r.(error); !ok {
-				log.Println("[HTTPClient] Failed to send request: ", string(data))
-				log.Println("PANIC: pkg:", r, debug.Stack())
-			}
-		}
-	}()
-
-	if c.conn == nil || !c.isAlive() {
-		Debug("[HTTPClient] Connecting:", c.baseURL)
-		if err = c.Connect(); err != nil {
-			log.Println("[HTTPClient] Connection error:", err)
-			response = errorPayload(HTTP_CONNECTION_ERROR)
-			return
-		}
-	}
-
-	timeout := time.Now().Add(c.config.Timeout)
-
-	c.conn.SetWriteDeadline(timeout)
-
-	if !c.config.OriginalHost {
-		data = proto.SetHost(data, []byte(c.baseURL), []byte(c.host))
-	}
-
-	if c.auth != "" {
-		data = proto.SetHeader(data, []byte("Authorization"), []byte(c.auth))
-	}
-
-	if c.config.Debug {
-		Debug("[HTTPClient] Sending:", string(data))
-	}
-
-	if _, err = c.conn.Write(data); err != nil {
-		Debug("[HTTPClient] Write error:", err, c.baseURL)
-		response = errorPayload(HTTP_TIMEOUT)
-		return
-	}
-
-	var readBytes, n int
-	var currentChunk []byte
-	timeout = time.Now().Add(c.config.Timeout)
-	chunked := false
-	contentLength := -1
-	currentContentLength := 0
-	chunks := 0
-
-	for {
-		c.conn.SetReadDeadline(timeout)
-
-		if readBytes < len(c.respBuf) {
-			n, err = c.conn.Read(c.respBuf[readBytes:])
-			readBytes += n
-			chunks++
-
-			if err != nil {
-				if err == io.EOF {
-					err = nil
-				}
-				break
-			}
-
-			// First chunk
-			if chunked || contentLength != -1 {
-				currentContentLength += n
-			} else {
-				// If headers are finished
-				if bytes.Contains(c.respBuf[:readBytes], proto.EmptyLine) {
-					if bytes.Equal(proto.Header(c.respBuf, []byte("Transfer-Encoding")), []byte("chunked")) {
-						chunked = true
-					} else {
-						status, _ := strconv.Atoi(string(proto.Status(c.respBuf)))
-						if (status >= 100 && status < 200) || status == 204 || status == 304 {
-							contentLength = 0
-						} else {
-							l := proto.Header(c.respBuf, []byte("Content-Length"))
-							if len(l) > 0 {
-								contentLength, _ = strconv.Atoi(string(l))
-							}
-						}
-					}
-
-					currentContentLength += len(proto.Body(c.respBuf[:readBytes]))
-				}
-			}
-
-			if chunked {
-				// Check if chunked message finished
-				if bytes.HasSuffix(c.respBuf[:readBytes], chunkedSuffix) {
-					break
-				}
-			} else if contentLength != -1 {
-				if currentContentLength > contentLength {
-					Debug("[HTTPClient] disconnected, wrong length", currentContentLength, contentLength)
-					c.Disconnect()
-					break
-				} else if currentContentLength == contentLength {
-					break
-				}
-			}
-		} else {
-			if currentChunk == nil {
-				currentChunk = make([]byte, readChunkSize)
-			}
-
-			n, err = c.conn.Read(currentChunk)
-
-			if err == io.EOF {
-				break
-			} else if err != nil {
-				Debug("[HTTPClient] Read the whole body error:", err, c.baseURL)
-				break
-			}
-
-			readBytes += int(n)
-			chunks++
-			currentContentLength += n
-
-			if chunked {
-				// Check if chunked message finished
-				if bytes.HasSuffix(currentChunk[:n], chunkedSuffix) {
-					break
-				}
-			} else if contentLength != -1 {
-				if currentContentLength > contentLength {
-					Debug("[HTTPClient] disconnected, wrong length", currentContentLength, contentLength)
-					c.Disconnect()
-					break
-				} else if currentContentLength == contentLength {
-					break
-				}
-			} else {
-				Debug("[HTTPClient] disconnected, can't find Content-Length or Chunked")
-				c.Disconnect()
-				break
-			}
-		}
-
-		if readBytes >= maxResponseSize {
-			Debug("[HTTPClient] Body is more than the max size", maxResponseSize,
-				c.baseURL)
-			break
-		}
-
-		// For following chunks expect less timeout
-		timeout = time.Now().Add(c.config.Timeout / 5)
-	}
-
-	if err != nil {
-		Debug("[HTTPClient] Response read error", err, c.conn, readBytes)
-		response = errorPayload(HTTP_TIMEOUT)
-		return
-	}
-
-	if readBytes > len(c.respBuf) {
-		readBytes = len(c.respBuf)
-	}
-	payload := make([]byte, readBytes)
-	copy(payload, c.respBuf[:readBytes])
-
-	if c.config.Debug {
-		Debug("[HTTPClient] Received:", string(payload))
-	}
-
-	if c.config.FollowRedirects > 0 && c.redirectsCount < c.config.FollowRedirects {
-		status := payload[9:12]
-
-		// 3xx requests
-		if status[0] == '3' {
-			c.redirectsCount++
-
-			location := proto.Header(payload, []byte("Location"))
-			redirectPayload := []byte("GET " + string(location) + " HTTP/1.1\r\n\r\n")
-
-			if c.config.Debug {
-				Debug("[HTTPClient] Redirecting to: " + string(location))
-			}
-
-			return c.Send(redirectPayload)
-		}
-	}
-
-	if bytes.Equal(proto.Status(payload), []byte("400")) {
-		c.Disconnect()
-		Debug("[HTTPClient] Closed connection on 400 response")
-	}
-
-	c.redirectsCount = 0
-
-	return payload, err
-}
-
-func (c *HTTPClient) Get(path string) (response []byte, err error) {
-	payload := "GET " + path + " HTTP/1.1\r\n\r\n"
-
-	return c.Send([]byte(payload))
-}
-
-func (c *HTTPClient) Post(path string, body []byte) (response []byte, err error) {
-	payload := "POST " + path + " HTTP/1.1\r\n"
-	payload += "Content-Length: " + strconv.Itoa(len(body)) + "\r\n\r\n"
-	payload += string(body)
-
-	return c.Send([]byte(payload))
-}
-
-const (
-	// https://support.cloudflare.com/hc/en-us/articles/200171936-Error-520-Web-server-is-returning-an-unknown-error
-	HTTP_UNKNOWN_ERROR = "520"
-	// https://support.cloudflare.com/hc/en-us/articles/200171916-Error-521-Web-server-is-down
-	HTTP_CONNECTION_ERROR = "521"
-	// https://support.cloudflare.com/hc/en-us/articles/200171906-Error-522-Connection-timed-out
-	HTTP_CONNECTION_TIMEOUT = "522"
-	// https://support.cloudflare.com/hc/en-us/articles/200171946-Error-523-Origin-is-unreachable
-	HTTP_UNREACHABLE = "523"
-	// https://support.cloudflare.com/hc/en-us/articles/200171926-Error-524-A-timeout-occurred
-	HTTP_TIMEOUT = "524"
-)
-
-var errorPayloadTemplate = "HTTP/1.1 202 Accepted\r\nDate: Mon, 17 Aug 2015 14:10:11 GMT\r\nContent-Length: 0\r\nContent-Type: text/plain; charset=utf-8\r\n\r\n"
-
-func errorPayload(errorCode string) []byte {
-	payload := make([]byte, len(errorPayloadTemplate))
-	copy(payload, errorPayloadTemplate)
-
-	copy(payload[29:58], []byte(time.Now().Format(time.RFC1123)))
-	copy(payload[9:12], errorCode)
-
-	return payload
-}

http_client_test.go

@@ -1,484 +0,0 @@
-package main
-
-import (
-	"bytes"
-	"crypto/rand"
-	"github.com/buger/gor/proto"
-	"io/ioutil"
-	_ "log"
-	"net"
-	"net/http"
-	"net/http/httptest"
-	"net/http/httputil"
-	_ "reflect"
-	"strings"
-	"sync"
-	"testing"
-	"time"
-)
-
-func TestHTTPClientURLPort(t *testing.T) {
-	c1 := NewHTTPClient("http://example.com", &HTTPClientConfig{})
-	if c1.baseURL != "http://example.com" {
-		t.Error("Sould not add 80 port for http:", c1.baseURL)
-	}
-
-	c2 := NewHTTPClient("https://example.com", &HTTPClientConfig{})
-	if c2.baseURL != "https://example.com" {
-		t.Error("Sould not add 443 port for https:", c2.baseURL)
-	}
-
-	c3 := NewHTTPClient("https://example.com:1", &HTTPClientConfig{})
-	if c3.baseURL != "https://example.com:1" {
-		t.Error("Sould use specified port:", c3.baseURL)
-	}
-
-	c4 := NewHTTPClient("example.com", &HTTPClientConfig{})
-	if c4.baseURL != "http://example.com" {
-		t.Error("Sould not add default protocol:", c4.baseURL)
-	}
-}
-
-func TestHTTPClientSend(t *testing.T) {
-	wg := new(sync.WaitGroup)
-
-	payload := func(reqType string) []byte {
-		switch reqType {
-		case "GET":
-			return []byte("GET / HTTP/1.1\r\n\r\n")
-		case "POST":
-			return []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
-		case "POST_CHUNKED":
-			return []byte("POST / HTTP/1.1\r\nHost: www.w3.org\r\nTransfer-Encoding: chunked\r\n\r\n4\r\nWiki\r\n5\r\npedia\r\ne\r\n in\r\n\r\nchunks.\r\n0\r\n\r\n")
-		}
-
-		return []byte("")
-	}
-
-	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-
-		if r.Method == "POST" {
-			defer r.Body.Close()
-			body, _ := ioutil.ReadAll(r.Body)
-
-			if len(r.TransferEncoding) > 0 && r.TransferEncoding[0] == "chunked" {
-				if string(body) != "Wikipedia in\r\n\r\nchunks." {
-					t.Error("Wrong POST body:", body, string(body))
-				}
-			} else {
-				if string(body) != "a=1&b=2" {
-					buf, _ := httputil.DumpRequest(r, true)
-					t.Error("Wrong POST body:", string(body), string(buf))
-				}
-			}
-		}
-
-		wg.Done()
-	}))
-	defer server.Close()
-
-	client := NewHTTPClient(server.URL, &HTTPClientConfig{Debug: true})
-
-	wg.Add(4)
-	client.Send(payload("POST"))
-	client.Send(payload("GET"))
-	client.Send(payload("POST_CHUNKED"))
-	client.Send(payload("POST"))
-
-	wg.Wait()
-}
-
-func TestHTTPClientResonseByClose(t *testing.T) {
-	wg := new(sync.WaitGroup)
-
-	payload := []byte("GET / HTTP/1.1\r\n\r\n")
-	ln, _ := net.Listen("tcp", ":0")
-	go func() {
-		for {
-			conn, _ := ln.Accept()
-			buf := make([]byte, 4096)
-			conn.Read(buf)
-
-			conn.Write([]byte("HTTP/1.1 200 OK\r\n\r\n"))
-			conn.Write([]byte("ab"))
-			conn.Close()
-
-			wg.Done()
-		}
-	}()
-
-	client := NewHTTPClient(ln.Addr().String(), &HTTPClientConfig{Debug: true})
-
-	wg.Add(1)
-	resp, _ := client.Send(payload)
-
-	if !bytes.Equal(resp, []byte("HTTP/1.1 200 OK\r\n\r\nab")) {
-		t.Error("Should return valid response", string(resp))
-	}
-
-	wg.Wait()
-}
-
-// https://github.com/buger/gor/issues/184
-func TestHTTPClientResponseBuffer(t *testing.T) {
-	testCases := []struct {
-		name         string
-		responseSize int
-		buffserSize  int
-		expectedSize int
-		timeout      time.Duration
-	}{
-		{"Chunked, buffer overflow", 10 * 1024, 1024, 1024, 50 * time.Millisecond},
-
-		{"Chunked, fits buffer", 10 * 1024, 64 * 1024, 10*1024 + 145 /* headers length + chunked meta */, 50 * time.Millisecond},
-
-		{"Content-Length, buffer overflow", 1024, 1000, 1000, 50 * time.Millisecond},
-
-		{"Content-Length, fits buffer", 1024, 64 * 1024, 1024 + 118, 50 * time.Millisecond},
-	}
-
-	for _, tc := range testCases {
-		wg := new(sync.WaitGroup)
-
-		payload := []byte("GET / HTTP/1.1\r\n\r\n")
-
-		server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-
-			size := tc.responseSize // 1kb
-			rb := make([]byte, size)
-			rand.Read(rb)
-
-			w.Write(rb[:size/2])
-			w.Write(rb[size/2:])
-
-			wg.Done()
-		}))
-
-		client := NewHTTPClient(server.URL, &HTTPClientConfig{Debug: true, ResponseBufferSize: tc.buffserSize, Timeout: 100 * time.Millisecond})
-
-		wg.Add(2)
-
-		start := time.Now()
-		client.Send(payload)
-		resp, err := client.Send(payload)
-		stop := time.Now()
-
-		if err != nil {
-			t.Error("Request error", err)
-		}
-
-		if stop.Sub(start) > tc.timeout {
-			t.Error("Request took too long", stop.Sub(start), tc.timeout)
-		}
-
-		if len(resp) != tc.expectedSize {
-			t.Error(tc.name, " - Wrong response size:", tc.expectedSize, len(resp))
-		} else {
-			if !bytes.Equal(resp[0:8], []byte("HTTP/1.1")) {
-				t.Error(tc.name, " - Response buffer contains data from previous request", string(resp), len(resp))
-			}
-		}
-
-		wg.Wait()
-		server.Close()
-	}
-}
-
-func TestHTTPClientHTTPSSend(t *testing.T) {
-	wg := new(sync.WaitGroup)
-
-	payload := func(reqType string) []byte {
-		switch reqType {
-		case "GET":
-			return []byte("GET / HTTP/1.1\r\n\r\n")
-		case "POST":
-			return []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
-		case "POST_CHUNKED":
-			return []byte("POST / HTTP/1.1\r\nHost: www.w3.org\r\nTransfer-Encoding: chunked\r\n\r\n4\r\nWiki\r\n5\r\npedia\r\ne\r\n in\r\n\r\nchunks.\r\n0\r\n\r\n")
-		}
-
-		return []byte("")
-	}
-
-	server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-
-		if r.Method == "POST" {
-			defer r.Body.Close()
-			body, _ := ioutil.ReadAll(r.Body)
-
-			if len(r.TransferEncoding) > 0 && r.TransferEncoding[0] == "chunked" {
-				if string(body) != "Wikipedia in\r\n\r\nchunks." {
-					t.Error("Wrong POST body:", body, string(body))
-				}
-			} else {
-				if string(body) != "a=1&b=2" {
-					buf, _ := httputil.DumpRequest(r, true)
-					t.Error("Wrong POST body:", string(body), string(buf))
-				}
-			}
-		}
-
-		wg.Done()
-	}))
-	defer server.Close()
-
-	client := NewHTTPClient(server.URL, &HTTPClientConfig{})
-
-	wg.Add(4)
-	client.Send(payload("POST"))
-	client.Send(payload("GET"))
-	client.Send(payload("POST_CHUNKED"))
-	client.Send(payload("POST"))
-
-	wg.Wait()
-}
-
-func TestHTTPClientServerInstantDisconnect(t *testing.T) {
-	wg := new(sync.WaitGroup)
-
-	GETPayload := []byte("GET / HTTP/1.1\r\n\r\n")
-
-	ln, _ := net.Listen("tcp", "127.0.0.1:0")
-	defer ln.Close()
-
-	go func() {
-		for {
-			conn, err := ln.Accept()
-
-			if err != nil {
-				break
-			}
-			conn.Close()
-			wg.Done()
-		}
-	}()
-
-	client := NewHTTPClient(ln.Addr().String(), &HTTPClientConfig{})
-
-	wg.Add(2)
-	client.Send(GETPayload)
-	client.Send(GETPayload)
-
-	wg.Wait()
-}
-
-func TestHTTPClientServerNoKeepAlive(t *testing.T) {
-	wg := new(sync.WaitGroup)
-
-	GETPayload := []byte("GET / HTTP/1.1\r\n\r\n")
-
-	ln, _ := net.Listen("tcp", "127.0.0.1:0")
-	defer ln.Close()
-
-	go func() {
-		for {
-			conn, err := ln.Accept()
-			if err != nil {
-				break
-			}
-
-			buf := make([]byte, 4096)
-			reqLen, err := conn.Read(buf)
-			if err != nil {
-				t.Error("Error reading:", err.Error())
-			}
-			Debug("Received: ", string(buf[0:reqLen]))
-			conn.Write([]byte("OK"))
-
-			// No keep-alive connections
-			conn.Close()
-
-			wg.Done()
-		}
-	}()
-
-	client := NewHTTPClient(ln.Addr().String(), &HTTPClientConfig{})
-
-	wg.Add(2)
-	client.Send(GETPayload)
-	client.Send(GETPayload)
-
-	wg.Wait()
-}
-
-func TestHTTPClientRedirect(t *testing.T) {
-	wg := new(sync.WaitGroup)
-
-	GETPayload := []byte("GET / HTTP/1.1\r\n\r\n")
-
-	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-
-		if r.URL.Path == "/" {
-			http.Redirect(w, r, "/new", 301)
-		}
-
-		wg.Done()
-	}))
-	defer server.Close()
-
-	client := NewHTTPClient(server.URL, &HTTPClientConfig{FollowRedirects: 1, Debug: false})
-
-	// Should do 2 queries
-	wg.Add(2)
-	client.Send(GETPayload)
-
-	wg.Wait()
-}
-
-func TestHTTPClientRedirectLimit(t *testing.T) {
-	wg := new(sync.WaitGroup)
-
-	GETPayload := []byte("GET / HTTP/1.1\r\n\r\n")
-
-	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-
-		if r.URL.Path == "/" {
-			http.Redirect(w, r, "/r1", 301)
-		}
-
-		if r.URL.Path == "/r1" {
-			http.Redirect(w, r, "/r2", 301)
-		}
-
-		if r.URL.Path == "/r2" {
-			http.Redirect(w, r, "/new", 301)
-		}
-
-		wg.Done()
-	}))
-	defer server.Close()
-
-	client := NewHTTPClient(server.URL, &HTTPClientConfig{FollowRedirects: 2, Debug: false})
-
-	// Have 3 redirects + 1 GET, but should do only 2 redirects + GET
-	wg.Add(3)
-	client.Send(GETPayload)
-
-	wg.Wait()
-}
-
-func TestHTTPClientBasicAuth(t *testing.T) {
-	wg := new(sync.WaitGroup)
-	wg.Add(2)
-
-	GETPayload := []byte("GET / HTTP/1.1\r\n\r\n")
-
-	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		user, pass, _ := r.BasicAuth()
-
-		if user != "user" || pass != "pass" {
-			http.Error(w, "Unauthorized.", 401)
-			wg.Done()
-			return
-		}
-
-		wg.Done()
-	}))
-	defer server.Close()
-
-	client := NewHTTPClient(server.URL, &HTTPClientConfig{Debug: false})
-	resp, _ := client.Send(GETPayload)
-	client.Disconnect()
-
-	if !bytes.Equal(proto.Status(resp), []byte("401")) {
-		t.Error("Should return unauthorized error", string(resp))
-	}
-
-	authUrl := strings.Replace(server.URL, "http://", "http://user:pass@", -1)
-	client = NewHTTPClient(authUrl, &HTTPClientConfig{Debug: false})
-	resp, _ = client.Send(GETPayload)
-	client.Disconnect()
-
-	if !bytes.Equal(proto.Status(resp), []byte("200")) {
-		t.Error("Should return proper response", string(resp))
-	}
-
-	wg.Wait()
-}
-
-func TestHTTPClientHandleHTTP10(t *testing.T) {
-	wg := new(sync.WaitGroup)
-
-	GETPayload := []byte("GET http://foobar.com/path HTTP/1.0\r\n\r\n")
-
-	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-
-		if r.URL.Path != "/path" {
-			t.Error("Path not match:", r.URL.Path)
-		}
-
-		wg.Done()
-	}))
-	defer server.Close()
-
-	client := NewHTTPClient(server.URL, &HTTPClientConfig{Debug: true})
-
-	wg.Add(1)
-	client.Send(GETPayload)
-
-	wg.Wait()
-}
-
-func TestHTTPClientErrors(t *testing.T) {
-	req := []byte("GET http://foobar.com/path HTTP/1.0\r\n\r\n")
-
-	// Port not exists
-	client := NewHTTPClient("http://127.0.0.1:1", &HTTPClientConfig{Debug: true})
-	if resp, err := client.Send(req); err != nil {
-		if s := proto.Status(resp); !bytes.Equal(s, []byte("521")) {
-			t.Error("Should return status 521 for connection refused, instead:", string(s))
-		}
-	} else {
-		t.Error("Should throw error")
-	}
-
-	client = NewHTTPClient("http://not.existing", &HTTPClientConfig{Debug: true})
-	if resp, err := client.Send(req); err != nil {
-		if s := proto.Status(resp); !bytes.Equal(s, []byte("521")) {
-			t.Error("Should return status 521 for no such host, instead:", string(s))
-		}
-	} else {
-		t.Error("Should throw error")
-	}
-
-	// Non routable IP address to simulate connection timeout
-	client = NewHTTPClient("http://10.255.255.1", &HTTPClientConfig{Debug: true, ConnectionTimeout: 100 * time.Millisecond})
-
-	if resp, err := client.Send(req); err != nil {
-		if s := proto.Status(resp); !bytes.Equal(s, []byte("521")) {
-			t.Error("Should return status 521 for io/timeout:", string(s))
-		}
-	} else {
-		t.Error("Should throw error")
-	}
-
-	// Connecting but io timeout on read
-	ln, _ := net.Listen("tcp", "127.0.0.1:0")
-	client = NewHTTPClient("http://"+ln.Addr().String(), &HTTPClientConfig{Debug: true, Timeout: 10 * time.Millisecond})
-	defer ln.Close()
-
-	if resp, err := client.Send(req); err != nil {
-		if s := proto.Status(resp); !bytes.Equal(s, []byte("524")) {
-			t.Error("Should return status 524 for io read, instead:", string(s))
-		}
-	} else {
-		t.Error("Should throw error")
-	}
-
-	// Response read error read tcp [::1]:51128: connection reset by peer &{{0xc20802a000}}
-	ln1, _ := net.Listen("tcp", "127.0.0.1:0")
-	go func() {
-		ln1.Accept()
-	}()
-	defer ln1.Close()
-
-	client = NewHTTPClient("http://"+ln1.Addr().String(), &HTTPClientConfig{Debug: true, Timeout: 10 * time.Millisecond})
-
-	if resp, err := client.Send(req); err != nil {
-		if s := proto.Status(resp); !bytes.Equal(s, []byte("524")) {
-			t.Error("Should return status 524 for connection reset by peer, instead:", string(s))
-		}
-	} else {
-		t.Error("Should throw error")
-	}
-}

http_modifier.go

@@ -2,9 +2,11 @@ package main
 
 import (
 	"bytes"
+	"encoding/base64"
 	"hash/fnv"
+	"strings"
 
-	"github.com/buger/gor/proto"
+	"github.com/buger/goreplay/proto"
 )
 
 type HTTPModifier struct {
@@ -13,16 +15,18 @@ type HTTPModifier struct {
 
 func NewHTTPModifier(config *HTTPModifierConfig) *HTTPModifier {
 	// Optimization to skip modifier completely if we do not need it
-	if len(config.urlRegexp) == 0 &&
-		len(config.urlNegativeRegexp) == 0 &&
-		len(config.urlRewrite) == 0 &&
-		len(config.headerFilters) == 0 &&
-		len(config.headerNegativeFilters) == 0 &&
-		len(config.headerHashFilters) == 0 &&
-		len(config.paramHashFilters) == 0 &&
-		len(config.params) == 0 &&
-		len(config.headers) == 0 &&
-		len(config.methods) == 0 {
+	if len(config.URLRegexp) == 0 &&
+		len(config.URLNegativeRegexp) == 0 &&
+		len(config.URLRewrite) == 0 &&
+		len(config.HeaderRewrite) == 0 &&
+		len(config.HeaderFilters) == 0 &&
+		len(config.HeaderNegativeFilters) == 0 &&
+		len(config.HeaderBasicAuthFilters) == 0 &&
+		len(config.HeaderHashFilters) == 0 &&
+		len(config.ParamHashFilters) == 0 &&
+		len(config.Params) == 0 &&
+		len(config.Headers) == 0 &&
+		len(config.Methods) == 0 {
 		return nil
 	}
 
@@ -30,16 +34,16 @@ func NewHTTPModifier(config *HTTPModifierConfig) *HTTPModifier {
 }
 
 func (m *HTTPModifier) Rewrite(payload []byte) (response []byte) {
-	if !proto.IsHTTPPayload(payload) {
+	if !proto.HasRequestTitle(payload) {
 		return payload
 	}
 
-	if len(m.config.methods) > 0 {
+	if len(m.config.Methods) > 0 {
 		method := proto.Method(payload)
 
 		matched := false
 
-		for _, m := range m.config.methods {
+		for _, m := range m.config.Methods {
 			if bytes.Equal(method, m) {
 				matched = true
 				break
@@ -51,24 +55,24 @@ func (m *HTTPModifier) Rewrite(payload []byte) (response []byte) {
 		}
 	}
 
-	if len(m.config.headers) > 0 {
-		for _, header := range m.config.headers {
+	if len(m.config.Headers) > 0 {
+		for _, header := range m.config.Headers {
 			payload = proto.SetHeader(payload, []byte(header.Name), []byte(header.Value))
 		}
 	}
 
-	if len(m.config.params) > 0 {
-		for _, param := range m.config.params {
+	if len(m.config.Params) > 0 {
+		for _, param := range m.config.Params {
 			payload = proto.SetPathParam(payload, param.Name, param.Value)
 		}
 	}
 
-	if len(m.config.urlRegexp) > 0 {
+	if len(m.config.URLRegexp) > 0 {
 		path := proto.Path(payload)
 
 		matched := false
 
-		for _, f := range m.config.urlRegexp {
+		for _, f := range m.config.URLRegexp {
 			if f.regexp.Match(path) {
 				matched = true
 				break
@@ -80,28 +84,32 @@ func (m *HTTPModifier) Rewrite(payload []byte) (response []byte) {
 		}
 	}
 
-	if len(m.config.urlNegativeRegexp) > 0 {
+	if len(m.config.URLNegativeRegexp) > 0 {
 		path := proto.Path(payload)
 
-		for _, f := range m.config.urlNegativeRegexp {
+		for _, f := range m.config.URLNegativeRegexp {
 			if f.regexp.Match(path) {
 				return
 			}
 		}
 	}
 
-	if len(m.config.headerFilters) > 0 {
-		for _, f := range m.config.headerFilters {
+	if len(m.config.HeaderFilters) > 0 {
+		for _, f := range m.config.HeaderFilters {
 			value := proto.Header(payload, f.name)
 
-			if len(value) > 0 && !f.regexp.Match(value) {
+			if len(value) == 0 {
+				return
+			}
+
+			if !f.regexp.Match(value) {
 				return
 			}
 		}
 	}
 
-	if len(m.config.headerNegativeFilters) > 0 {
-		for _, f := range m.config.headerNegativeFilters {
+	if len(m.config.HeaderNegativeFilters) > 0 {
+		for _, f := range m.config.HeaderNegativeFilters {
 			value := proto.Header(payload, f.name)
 
 			if len(value) > 0 && f.regexp.Match(value) {
@@ -110,8 +118,25 @@ func (m *HTTPModifier) Rewrite(payload []byte) (response []byte) {
 		}
 	}
 
-	if len(m.config.headerHashFilters) > 0 {
-		for _, f := range m.config.headerHashFilters {
+	if len(m.config.HeaderBasicAuthFilters) > 0 {
+		for _, f := range m.config.HeaderBasicAuthFilters {
+			value := proto.Header(payload, []byte("Authorization"))
+
+			if len(value) > 0 {
+				valueString := string(value)
+				trimmedBasicAuthEncoded := strings.TrimPrefix(valueString, "Basic ")
+				if strings.Compare(valueString, trimmedBasicAuthEncoded) != 0 {
+					decodedAuth, _ := base64.StdEncoding.DecodeString(trimmedBasicAuthEncoded)
+					if !f.regexp.Match(decodedAuth) {
+						return
+					}
+				}
+			}
+		}
+	}
+
+	if len(m.config.HeaderHashFilters) > 0 {
+		for _, f := range m.config.HeaderHashFilters {
 			value := proto.Header(payload, f.name)
 
 			if len(value) > 0 {
@@ -125,8 +150,8 @@ func (m *HTTPModifier) Rewrite(payload []byte) (response []byte) {
 		}
 	}
 
-	if len(m.config.paramHashFilters) > 0 {
-		for _, f := range m.config.paramHashFilters {
+	if len(m.config.ParamHashFilters) > 0 {
+		for _, f := range m.config.ParamHashFilters {
 			value, s, _ := proto.PathParam(payload, f.name)
 
 			if s != -1 {
@@ -140,10 +165,10 @@ func (m *HTTPModifier) Rewrite(payload []byte) (response []byte) {
 		}
 	}
 
-	if len(m.config.urlRewrite) > 0 {
+	if len(m.config.URLRewrite) > 0 {
 		path := proto.Path(payload)
 
-		for _, f := range m.config.urlRewrite {
+		for _, f := range m.config.URLRewrite {
 			if f.src.Match(path) {
 				path = f.src.ReplaceAll(path, f.target)
 				payload = proto.SetPath(payload, path)
@@ -153,5 +178,19 @@ func (m *HTTPModifier) Rewrite(payload []byte) (response []byte) {
 		}
 	}
 
+	if len(m.config.HeaderRewrite) > 0 {
+		for _, f := range m.config.HeaderRewrite {
+			value := proto.Header(payload, f.header)
+			if len(value) == 0 {
+				break
+			}
+
+			if f.src.Match(value) {
+				newValue := f.src.ReplaceAll(value, f.target)
+				payload = proto.SetHeader(payload, f.header, newValue)
+			}
+		}
+	}
+
 	return payload
 }

http_modifier_settings.go

@@ -10,17 +10,18 @@ import (
 
 // HTTPModifierConfig holds configuration options for built-in traffic modifier
 type HTTPModifierConfig struct {
-	urlNegativeRegexp     HTTPUrlRegexp
-	urlRegexp             HTTPUrlRegexp
-	urlRewrite            UrlRewriteMap
-	headerFilters         HTTPHeaderFilters
-	headerNegativeFilters HTTPHeaderFilters
-	headerHashFilters     HTTPHashFilters
-	paramHashFilters      HTTPHashFilters
-
-	params  HTTPParams
-	headers HTTPHeaders
-	methods HTTPMethods
+	URLNegativeRegexp      HTTPURLRegexp              `json:"http-disallow-url"`
+	URLRegexp              HTTPURLRegexp              `json:"http-allow-url"`
+	URLRewrite             URLRewriteMap              `json:"http-rewrite-url"`
+	HeaderRewrite          HeaderRewriteMap           `json:"http-rewrite-header"`
+	HeaderFilters          HTTPHeaderFilters          `json:"http-allow-header"`
+	HeaderNegativeFilters  HTTPHeaderFilters          `json:"http-disallow-header"`
+	HeaderBasicAuthFilters HTTPHeaderBasicAuthFilters `json:"http-basic-auth-filter"`
+	HeaderHashFilters      HTTPHashFilters            `json:"http-header-limiter"`
+	ParamHashFilters       HTTPHashFilters            `json:"http-param-limiter"`
+	Params                 HTTPParams                 `json:"http-set-param"`
+	Headers                HTTPHeaders                `json:"http-set-header"`
+	Methods                HTTPMethods                `json:"http-allow-method"`
 }
 
 //
@@ -38,10 +39,11 @@ func (h *HTTPHeaderFilters) String() string {
 	return fmt.Sprint(*h)
 }
 
+// Set method to implement flags.Value
 func (h *HTTPHeaderFilters) Set(value string) error {
 	valArr := strings.SplitN(value, ":", 2)
 	if len(valArr) < 2 {
-		return errors.New("need both header and value, colon-delimited (ex. user_id:^169$).")
+		return errors.New("need both header and value, colon-delimited (ex. user_id:^169$)")
 	}
 	val := strings.TrimSpace(valArr[1])
 	r, err := regexp.Compile(val)
@@ -54,6 +56,32 @@ func (h *HTTPHeaderFilters) Set(value string) error {
 	return nil
 }
 
+//
+// Handling of --http-basic-auth-filter option
+//
+type basicAuthFilter struct {
+	regexp *regexp.Regexp
+}
+
+// HTTPHeaderBasicAuthFilters holds list of regxp to match basic Auth header values
+type HTTPHeaderBasicAuthFilters []basicAuthFilter
+
+func (h *HTTPHeaderBasicAuthFilters) String() string {
+	return fmt.Sprint(*h)
+}
+
+// Set method to implement flags.Value
+func (h *HTTPHeaderBasicAuthFilters) Set(value string) error {
+	r, err := regexp.Compile(value)
+	if err != nil {
+		return err
+	}
+
+	*h = append(*h, basicAuthFilter{regexp: r})
+
+	return nil
+}
+
 //
 // Handling of --http-allow-header-hash and --http-allow-param-hash options
 //
@@ -62,12 +90,14 @@ type hashFilter struct {
 	percent uint32
 }
 
+// HTTPHashFilters represents a slice of header hash filters
 type HTTPHashFilters []hashFilter
 
 func (h *HTTPHashFilters) String() string {
 	return fmt.Sprint(*h)
 }
 
+// Set method to implement flags.Value
 func (h *HTTPHashFilters) Set(value string) error {
 	valArr := strings.SplitN(value, ":", 2)
 	if len(valArr) < 2 {
@@ -102,23 +132,26 @@ func (h *HTTPHashFilters) Set(value string) error {
 //
 // Handling of --http-set-header option
 //
-type HTTPHeaders []HTTPHeader
-type HTTPHeader struct {
+type httpHeader struct {
 	Name  string
 	Value string
 }
 
+// HTTPHeaders is a slice of headers that must appended
+type HTTPHeaders []httpHeader
+
 func (h *HTTPHeaders) String() string {
 	return fmt.Sprint(*h)
 }
 
+// Set method to implement flags.Value
 func (h *HTTPHeaders) Set(value string) error {
 	v := strings.SplitN(value, ":", 2)
 	if len(v) != 2 {
 		return errors.New("Expected `Key: Value`")
 	}
 
-	header := HTTPHeader{
+	header := httpHeader{
 		strings.TrimSpace(v[0]),
 		strings.TrimSpace(v[1]),
 	}
@@ -130,23 +163,26 @@ func (h *HTTPHeaders) Set(value string) error {
 //
 // Handling of --http-set-param option
 //
-type HTTPParams []HTTPParam
-type HTTPParam struct {
+type httpParam struct {
 	Name  []byte
 	Value []byte
 }
 
+// HTTPParams filters for --http-set-param
+type HTTPParams []httpParam
+
 func (h *HTTPParams) String() string {
 	return fmt.Sprint(*h)
 }
 
+// Set method to implement flags.Value
 func (h *HTTPParams) Set(value string) error {
 	v := strings.SplitN(value, "=", 2)
 	if len(v) != 2 {
 		return errors.New("Expected `Key=Value`")
 	}
 
-	param := HTTPParam{
+	param := httpParam{
 		[]byte(strings.TrimSpace(v[0])),
 		[]byte(strings.TrimSpace(v[1])),
 	}
@@ -158,12 +194,15 @@ func (h *HTTPParams) Set(value string) error {
 //
 // Handling of --http-allow-method option
 //
+
+// HTTPMethods holds values for method allowed
 type HTTPMethods [][]byte
 
 func (h *HTTPMethods) String() string {
 	return fmt.Sprint(*h)
 }
 
+// Set method to implement flags.Value
 func (h *HTTPMethods) Set(value string) error {
 	*h = append(*h, []byte(value))
 	return nil
@@ -177,13 +216,15 @@ type urlRewrite struct {
 	target []byte
 }
 
-type UrlRewriteMap []urlRewrite
+// URLRewriteMap holds regexp and data to modify URL
+type URLRewriteMap []urlRewrite
 
-func (r *UrlRewriteMap) String() string {
+func (r *URLRewriteMap) String() string {
 	return fmt.Sprint(*r)
 }
 
-func (r *UrlRewriteMap) Set(value string) error {
+// Set method to implement flags.Value
+func (r *URLRewriteMap) Set(value string) error {
 	valArr := strings.SplitN(value, ":", 2)
 	if len(valArr) < 2 {
 		return errors.New("need both src and target, colon-delimited (ex. /a:/b)")
@@ -196,6 +237,44 @@ func (r *UrlRewriteMap) Set(value string) error {
 	return nil
 }
 
+//
+// Handling of --http-rewrite-header option
+//
+type headerRewrite struct {
+	header []byte
+	src    *regexp.Regexp
+	target []byte
+}
+
+// HeaderRewriteMap holds regexp and data to rewrite headers
+type HeaderRewriteMap []headerRewrite
+
+func (r *HeaderRewriteMap) String() string {
+	return fmt.Sprint(*r)
+}
+
+// Set method to implement flags.Value
+func (r *HeaderRewriteMap) Set(value string) error {
+	headerArr := strings.SplitN(value, ":", 2)
+	if len(headerArr) < 2 {
+		return errors.New("need both header, regexp and rewrite target, colon-delimited (ex. Header: regexp,target)")
+	}
+
+	header := headerArr[0]
+	valArr := strings.SplitN(strings.TrimSpace(headerArr[1]), ",", 2)
+
+	if len(valArr) < 2 {
+		return errors.New("need both header, regexp and rewrite target, colon-delimited (ex. Header: regexp,target)")
+	}
+
+	regexp, err := regexp.Compile(valArr[0])
+	if err != nil {
+		return err
+	}
+	*r = append(*r, headerRewrite{header: []byte(header), src: regexp, target: []byte(valArr[1])})
+	return nil
+}
+
 //
 // Handling of --http-allow-url option
 //
@@ -203,13 +282,15 @@ type urlRegexp struct {
 	regexp *regexp.Regexp
 }
 
-type HTTPUrlRegexp []urlRegexp
+// HTTPURLRegexp a slice of regexp to match URLs
+type HTTPURLRegexp []urlRegexp
 
-func (r *HTTPUrlRegexp) String() string {
+func (r *HTTPURLRegexp) String() string {
 	return fmt.Sprint(*r)
 }
 
-func (r *HTTPUrlRegexp) Set(value string) error {
+// Set method to implement flags.Value
+func (r *HTTPURLRegexp) Set(value string) error {
 	regexp, err := regexp.Compile(value)
 
 	*r = append(*r, urlRegexp{regexp: regexp})

http_modifier_settings_test.go

@@ -53,7 +53,7 @@ func TestHTTPHashFilters(t *testing.T) {
 
 func TestUrlRewriteMap(t *testing.T) {
 	var err error
-	rewrites := UrlRewriteMap{}
+	rewrites := URLRewriteMap{}
 
 	if err = rewrites.Set("/v1/user/([^\\/]+)/ping:/v2/user/$1/ping"); err != nil {
 		t.Error("Should set mapping", err)

http_modifier_test.go

@@ -2,8 +2,9 @@ package main
 
 import (
 	"bytes"
-	"github.com/buger/gor/proto"
 	"testing"
+
+	"github.com/buger/goreplay/proto"
 )
 
 func TestHTTPModifierWithoutConfig(t *testing.T) {
@@ -17,7 +18,7 @@ func TestHTTPModifierHeaderFilters(t *testing.T) {
 	filters.Set("Host:^www.w3.org$")
 
 	modifier := NewHTTPModifier(&HTTPModifierConfig{
-		headerFilters: filters,
+		HeaderFilters: filters,
 	})
 
 	payload := []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
@@ -31,7 +32,7 @@ func TestHTTPModifierHeaderFilters(t *testing.T) {
 	filters.Set("Host:^www.w4.org$")
 
 	modifier = NewHTTPModifier(&HTTPModifierConfig{
-		headerFilters: filters,
+		HeaderFilters: filters,
 	})
 
 	if len(modifier.Rewrite(payload)) != 0 {
@@ -44,7 +45,7 @@ func TestHTTPModifierHeaderNegativeFilters(t *testing.T) {
 	filters.Set("Host:^www.w3.org$")
 
 	modifier := NewHTTPModifier(&HTTPModifierConfig{
-		headerNegativeFilters: filters,
+		HeaderNegativeFilters: filters,
 	})
 
 	payload := []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w4.org\r\n\r\na=1&b=2")
@@ -58,7 +59,7 @@ func TestHTTPModifierHeaderNegativeFilters(t *testing.T) {
 	filters.Set("Host:^www.w4.org$")
 
 	modifier = NewHTTPModifier(&HTTPModifierConfig{
-		headerNegativeFilters: filters,
+		HeaderNegativeFilters: filters,
 	})
 
 	if len(modifier.Rewrite(payload)) != 0 {
@@ -70,7 +71,7 @@ func TestHTTPModifierHeaderNegativeFilters(t *testing.T) {
 	filters.Set("Host: www*")
 
 	modifier = NewHTTPModifier(&HTTPModifierConfig{
-		headerNegativeFilters: filters,
+		HeaderNegativeFilters: filters,
 	})
 
 	if len(modifier.Rewrite(payload)) != 0 {
@@ -78,10 +79,49 @@ func TestHTTPModifierHeaderNegativeFilters(t *testing.T) {
 	}
 }
 
+func TestHTTPHeaderBasicAuthFilters(t *testing.T) {
+	filters := HTTPHeaderBasicAuthFilters{}
+	filters.Set("^customer[0-9].*")
+
+	modifier := NewHTTPModifier(&HTTPModifierConfig{
+		HeaderBasicAuthFilters: filters,
+	})
+
+	//Encoded UserId:Password = customer3:welcome
+	payload := []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nAuthorization: Basic Y3VzdG9tZXIzOndlbGNvbWU=\r\n\r\na=1&b=2")
+	if len(modifier.Rewrite(payload)) == 0 {
+		t.Error("Request should pass filters")
+	}
+
+	//customer6:rest@123^TEST
+	payload = []byte("POST /post HTTP/1.1\r\nContent-Length: 88\r\nAuthorization: Basic Y3VzdG9tZXI2OnJlc3RAMTIzXlRFU1Q==\r\n\r\na=1&b=2")
+	if len(modifier.Rewrite(payload)) == 0 {
+		t.Error("Request should pass filters")
+	}
+
+	filters = HTTPHeaderBasicAuthFilters{}
+	// Setting filter that not match our header
+	filters.Set("^(homer simpson|mickey mouse).*")
+
+	modifier = NewHTTPModifier(&HTTPModifierConfig{
+		HeaderBasicAuthFilters: filters,
+	})
+
+	if len(modifier.Rewrite(payload)) != 0 {
+		t.Error("Request should not pass filters")
+	}
+
+	//mickey mouse:happy123
+	payload = []byte("POST /post HTTP/1.1\r\nContent-Length: 88\r\nAuthorization: Basic bWlja2V5IG1vdXNlOmhhcHB5MTIz\r\n\r\na=1&b=2")
+	if len(modifier.Rewrite(payload)) == 0 {
+		t.Error("Request should pass filters")
+	}
+}
+
 func TestHTTPModifierURLRewrite(t *testing.T) {
 	var url, newURL []byte
 
-	rewrites := UrlRewriteMap{}
+	rewrites := URLRewriteMap{}
 
 	payload := func(url []byte) []byte {
 		return []byte("POST " + string(url) + " HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
@@ -93,7 +133,7 @@ func TestHTTPModifierURLRewrite(t *testing.T) {
 	}
 
 	modifier := NewHTTPModifier(&HTTPModifierConfig{
-		urlRewrite: rewrites,
+		URLRewrite: rewrites,
 	})
 
 	url = []byte("/v1/user/joe/ping")
@@ -107,12 +147,33 @@ func TestHTTPModifierURLRewrite(t *testing.T) {
 	}
 }
 
+func TestHTTPModifierHeaderRewrite(t *testing.T) {
+	var header, newHeader []byte
+
+	rewrites := HeaderRewriteMap{}
+	payload := []byte("GET / HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
+
+	err := rewrites.Set("Host: (.*).w3.org,$1.beta.w3.org")
+	if err != nil {
+		t.Error("Should not error", err)
+	}
+
+	modifier := NewHTTPModifier(&HTTPModifierConfig{
+		HeaderRewrite: rewrites,
+	})
+
+	header = []byte("www.beta.w3.org")
+	if newHeader = proto.Header(modifier.Rewrite(payload), []byte("Host")); !bytes.Equal(newHeader, header) {
+		t.Error("Request header should have been rewritten, wasn't", string(newHeader), string(header))
+	}
+}
+
 func TestHTTPModifierHeaderHashFilters(t *testing.T) {
 	filters := HTTPHashFilters{}
 	filters.Set("Header2:1/2")
 
 	modifier := NewHTTPModifier(&HTTPModifierConfig{
-		headerHashFilters: filters,
+		HeaderHashFilters: filters,
 	})
 
 	payload := func(header []byte) []byte {
@@ -137,7 +198,7 @@ func TestHTTPModifierParamHashFilters(t *testing.T) {
 	filters.Set("user_id:1/2")
 
 	modifier := NewHTTPModifier(&HTTPModifierConfig{
-		paramHashFilters: filters,
+		ParamHashFilters: filters,
 	})
 
 	payload := func(value []byte) []byte {
@@ -163,7 +224,7 @@ func TestHTTPModifierHeaders(t *testing.T) {
 	headers.Set("Host:localhost")
 
 	modifier := NewHTTPModifier(&HTTPModifierConfig{
-		headers: headers,
+		Headers: headers,
 	})
 
 	payload := []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
@@ -175,12 +236,12 @@ func TestHTTPModifierHeaders(t *testing.T) {
 }
 
 func TestHTTPModifierURLRegexp(t *testing.T) {
-	filters := HTTPUrlRegexp{}
+	filters := HTTPURLRegexp{}
 	filters.Set("/v1/app")
 	filters.Set("/v1/api")
 
 	modifier := NewHTTPModifier(&HTTPModifierConfig{
-		urlRegexp: filters,
+		URLRegexp: filters,
 	})
 
 	payload := func(url string) []byte {
@@ -201,12 +262,12 @@ func TestHTTPModifierURLRegexp(t *testing.T) {
 }
 
 func TestHTTPModifierURLNegativeRegexp(t *testing.T) {
-	filters := HTTPUrlRegexp{}
+	filters := HTTPURLRegexp{}
 	filters.Set("/restricted1")
 	filters.Set("/some/restricted2")
 
 	modifier := NewHTTPModifier(&HTTPModifierConfig{
-		urlNegativeRegexp: filters,
+		URLNegativeRegexp: filters,
 	})
 
 	payload := func(url string) []byte {
@@ -231,7 +292,7 @@ func TestHTTPModifierSetHeader(t *testing.T) {
 	filters.Set("User-Agent:Gor")
 
 	modifier := NewHTTPModifier(&HTTPModifierConfig{
-		headers: filters,
+		Headers: filters,
 	})
 
 	payload := []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
@@ -247,7 +308,7 @@ func TestHTTPModifierSetParam(t *testing.T) {
 	filters.Set("api_key=1")
 
 	modifier := NewHTTPModifier(&HTTPModifierConfig{
-		params: filters,
+		Params: filters,
 	})
 
 	payload := []byte("POST /post?api_key=1234 HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")

http_prettifier.go

@@ -0,0 +1,67 @@
+package main
+
+import (
+	"bytes"
+	"compress/gzip"
+	"fmt"
+	"io/ioutil"
+	"net/http/httputil"
+	"strconv"
+
+	"github.com/buger/goreplay/proto"
+)
+
+func prettifyHTTP(p []byte) []byte {
+
+	tEnc := bytes.Equal(proto.Header(p, []byte("Transfer-Encoding")), []byte("chunked"))
+	cEnc := bytes.Equal(proto.Header(p, []byte("Content-Encoding")), []byte("gzip"))
+
+	if !(tEnc || cEnc) {
+		return p
+	}
+
+	headersPos := proto.MIMEHeadersEndPos(p)
+
+	if headersPos < 5 || headersPos > len(p) {
+		return p
+	}
+
+	headers := p[:headersPos]
+	content := p[headersPos:]
+
+	if tEnc {
+		buf := bytes.NewReader(content)
+		r := httputil.NewChunkedReader(buf)
+		content, _ = ioutil.ReadAll(r)
+
+		headers = proto.DeleteHeader(headers, []byte("Transfer-Encoding"))
+
+		newLen := strconv.Itoa(len(content))
+		headers = proto.SetHeader(headers, []byte("Content-Length"), []byte(newLen))
+	}
+
+	if cEnc {
+		buf := bytes.NewReader(content)
+		g, err := gzip.NewReader(buf)
+
+		if err != nil {
+			Debug(1, "[Prettifier] GZIP encoding error:", err)
+			return []byte{}
+		}
+
+		content, err = ioutil.ReadAll(g)
+		if err != nil {
+			Debug(1, fmt.Sprintf("[HTTP-PRETTIFIER] %q", err))
+			return p
+		}
+
+		headers = proto.DeleteHeader(headers, []byte("Content-Encoding"))
+
+		newLen := strconv.Itoa(len(content))
+		headers = proto.SetHeader(headers, []byte("Content-Length"), []byte(newLen))
+	}
+
+	newPayload := append(headers, content...)
+
+	return newPayload
+}

http_prettifier_test.go

@@ -0,0 +1,37 @@
+package main
+
+import (
+	"bytes"
+	"compress/gzip"
+	"strconv"
+	"testing"
+
+	"github.com/buger/goreplay/proto"
+)
+
+func TestHTTPPrettifierGzip(t *testing.T) {
+	b := bytes.NewBufferString("")
+	w := gzip.NewWriter(b)
+	w.Write([]byte("test"))
+	w.Close()
+
+	size := strconv.Itoa(len(b.Bytes()))
+
+	payload := []byte("HTTP/1.1 200 OK\r\nContent-Length: " + size + "\r\nContent-Encoding: gzip\r\n\r\n")
+	payload = append(payload, b.Bytes()...)
+
+	newPayload := prettifyHTTP(payload)
+
+	if string(newPayload) != "HTTP/1.1 200 OK\r\nContent-Length: 4\r\n\r\ntest" {
+		t.Errorf("Payload not match %q", string(newPayload))
+	}
+}
+
+func TestHTTPPrettifierChunked(t *testing.T) {
+	payload := []byte("POST / HTTP/1.1\r\nHost: www.w3.org\r\nTransfer-Encoding: chunked\r\n\r\n4\r\nWiki\r\n5\r\npedia\r\ne\r\n in\r\n\r\nchunks.\r\n0\r\n\r\n")
+
+	payload = prettifyHTTP(payload)
+	if string(proto.Header(payload, []byte("Content-Length"))) != "23" {
+		t.Errorf("payload should have content length of 23")
+	}
+}

input_dummy.go

@@ -7,24 +7,30 @@ import (
 // DummyInput used for debugging. It generate 1 "GET /"" request per second.
 type DummyInput struct {
 	data chan []byte
+	quit chan struct{}
 }
 
 // NewDummyInput constructor for DummyInput
 func NewDummyInput(options string) (di *DummyInput) {
 	di = new(DummyInput)
 	di.data = make(chan []byte)
+	di.quit = make(chan struct{})
 
 	go di.emit()
 
 	return
 }
 
-func (i *DummyInput) Read(data []byte) (int, error) {
-	buf := <-i.data
-
-	copy(data, buf)
-
-	return len(buf), nil
+// PluginRead reads message from this plugin
+func (i *DummyInput) PluginRead() (*Message, error) {
+	var msg Message
+	select {
+	case <-i.quit:
+		return nil, ErrorStopped
+	case buf := <-i.data:
+		msg.Meta, msg.Data = payloadMetaWithBody(buf)
+		return &msg, nil
+	}
 }
 
 func (i *DummyInput) emit() {
@@ -46,3 +52,9 @@ func (i *DummyInput) emit() {
 func (i *DummyInput) String() string {
 	return "Dummy Input"
 }
+
+// Close closes this plugins
+func (i *DummyInput) Close() error {
+	close(i.quit)
+	return nil
+}

input_file.go

@@ -5,41 +5,43 @@ import (
 	"bytes"
 	"compress/gzip"
 	"errors"
+	"fmt"
 	"io"
-	"log"
 	"os"
 	"path/filepath"
 	"strconv"
 	"strings"
 	"sync"
+	"sync/atomic"
 	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/s3"
 )
 
 type fileInputReader struct {
 	reader    *bufio.Reader
 	data      []byte
-	file      *os.File
+	file      io.ReadCloser
 	timestamp int64
+	closed    int32 // Value of 0 indicates that the file is still open.
+	s3        bool
 }
 
 func (f *fileInputReader) parseNext() error {
 	payloadSeparatorAsBytes := []byte(payloadSeparator)
 	var buffer bytes.Buffer
-
 	for {
 		line, err := f.reader.ReadBytes('\n')
 
 		if err != nil {
 			if err != io.EOF {
-				log.Println(err)
-				return err
-			}
-
-			if err == io.EOF {
-				f.file.Close()
-				f.file = nil
-				return err
+				Debug(1, err)
+			} else {
+				f.Close()
 			}
+			return err
 		}
 
 		if bytes.Equal(payloadSeparatorAsBytes[1:], line) {
@@ -55,7 +57,6 @@ func (f *fileInputReader) parseNext() error {
 		buffer.Write(line)
 	}
 
-	return nil
 }
 
 func (f *fileInputReader) ReadPayload() []byte {
@@ -63,27 +64,37 @@ func (f *fileInputReader) ReadPayload() []byte {
 
 	return f.data
 }
+
+// Close closes this plugin
 func (f *fileInputReader) Close() error {
-	if f.file != nil {
+	if atomic.LoadInt32(&f.closed) == 0 {
+		atomic.StoreInt32(&f.closed, 1)
 		f.file.Close()
 	}
 
 	return nil
 }
 
-func NewFileInputReader(path string) *fileInputReader {
-	file, err := os.Open(path)
+func newFileInputReader(path string) *fileInputReader {
+	var file io.ReadCloser
+	var err error
+
+	if strings.HasPrefix(path, "s3://") {
+		file = NewS3ReadCloser(path)
+	} else {
+		file, err = os.Open(path)
+	}
 
 	if err != nil {
-		log.Println(err)
+		Debug(0, fmt.Sprintf("[INPUT-FILE] err: %q", err))
 		return nil
 	}
 
-	r := &fileInputReader{file: file}
+	r := &fileInputReader{file: file, closed: 0}
 	if strings.HasSuffix(path, ".gz") {
 		gzReader, err := gzip.NewReader(file)
 		if err != nil {
-			log.Println(err)
+			Debug(0, fmt.Sprintf("[INPUT-FILE] err: %q", err))
 			return nil
 		}
 		r.reader = bufio.NewReader(gzReader)
@@ -111,7 +122,7 @@ type FileInput struct {
 func NewFileInput(path string, loop bool) (i *FileInput) {
 	i = new(FileInput)
 	i.data = make(chan []byte, 1000)
-	i.exit = make(chan bool, 1)
+	i.exit = make(chan bool)
 	i.path = path
 	i.speedFactor = 1
 	i.loop = loop
@@ -125,42 +136,62 @@ func NewFileInput(path string, loop bool) (i *FileInput) {
 	return
 }
 
-type NextFileNotFound struct{}
-
-func (_ *NextFileNotFound) Error() string {
-	return "There is no new files"
-}
-
 func (i *FileInput) init() (err error) {
 	defer i.mu.Unlock()
 	i.mu.Lock()
 
 	var matches []string
 
-	if matches, err = filepath.Glob(i.path); err != nil {
-		log.Println("Wrong file pattern", i.path, err)
+	if strings.HasPrefix(i.path, "s3://") {
+		sess := session.Must(session.NewSession(awsConfig()))
+		svc := s3.New(sess)
+
+		bucket, key := parseS3Url(i.path)
+
+		params := &s3.ListObjectsInput{
+			Bucket: aws.String(bucket),
+			Prefix: aws.String(key),
+		}
+
+		resp, err := svc.ListObjects(params)
+		if err != nil {
+			Debug(0, "[INPUT-FILE] Error while retreiving list of files from S3", i.path, err)
+			return err
+		}
+
+		for _, c := range resp.Contents {
+			matches = append(matches, "s3://"+bucket+"/"+(*c.Key))
+		}
+	} else if matches, err = filepath.Glob(i.path); err != nil {
+		Debug(0, "[INPUT-FILE] Wrong file pattern", i.path, err)
 		return
+
 	}
 
 	if len(matches) == 0 {
-		log.Println("No files match pattern: ", i.path)
+		Debug(0, "[INPUT-FILE] No files match pattern: ", i.path)
 		return errors.New("No matching files")
 	}
 
 	i.readers = make([]*fileInputReader, len(matches))
 
 	for idx, p := range matches {
-		i.readers[idx] = NewFileInputReader(p)
+		i.readers[idx] = newFileInputReader(p)
 	}
 
 	return nil
 }
 
-func (i *FileInput) Read(data []byte) (int, error) {
-	buf := <-i.data
-	copy(data, buf)
-
-	return len(buf), nil
+// PluginRead reads message from this plugin
+func (i *FileInput) PluginRead() (*Message, error) {
+	var msg Message
+	select {
+	case <-i.exit:
+		return nil, ErrorStopped
+	case buf := <-i.data:
+		msg.Meta, msg.Data = payloadMetaWithBody(buf)
+		return &msg, nil
+	}
 }
 
 func (i *FileInput) String() string {
@@ -170,7 +201,7 @@ func (i *FileInput) String() string {
 // Find reader with smallest timestamp e.g next payload in row
 func (i *FileInput) nextReader() (next *fileInputReader) {
 	for _, r := range i.readers {
-		if r == nil || r.file == nil {
+		if r == nil || atomic.LoadInt32(&r.closed) != 0 {
 			continue
 		}
 
@@ -218,18 +249,25 @@ func (i *FileInput) emit() {
 			lastTime = reader.timestamp
 		}
 
-		i.data <- reader.ReadPayload()
+		// Recheck if we have exited since last check.
+		select {
+		case <-i.exit:
+			return
+		default:
+			i.data <- reader.ReadPayload()
+		}
 	}
 
-	log.Printf("FileInput: end of file '%s'\n", i.path)
+	Debug(0, fmt.Sprintf("[INPUT-FILE] FileInput: end of file '%s'\n", i.path))
+
 }
 
+// Close closes this plugin
 func (i *FileInput) Close() error {
 	defer i.mu.Unlock()
 	i.mu.Lock()
 
-	i.exit <- true
-
+	close(i.exit)
 	for _, r := range i.readers {
 		r.Close()
 	}

input_file_test.go

@@ -4,66 +4,57 @@ import (
 	"bytes"
 	"errors"
 	"fmt"
-	"io"
 	"io/ioutil"
-	"log"
 	"math/rand"
 	"os"
 	"sync"
-	"syscall"
 	"testing"
 	"time"
 )
 
-var _ = log.Println
-
 func TestInputFileWithGET(t *testing.T) {
-
 	input := NewTestInput()
-	rg := NewRequestGenerator([]io.Reader{input}, func() { input.EmitGET() }, 1)
-	readPayloads := [][]byte{}
+	rg := NewRequestGenerator([]PluginReader{input}, func() { input.EmitGET() }, 1)
+	readPayloads := []*Message{}
 
 	// Given a capture file with a GET request
 	expectedCaptureFile := CreateCaptureFile(rg)
 	defer expectedCaptureFile.TearDown()
 
 	// When the request is read from the capture file
-	err := ReadFromCaptureFile(expectedCaptureFile.file, 1, func(data []byte) {
-		readPayloads = append(readPayloads, Duplicate(data))
+	err := ReadFromCaptureFile(expectedCaptureFile.file, 1, func(msg *Message) {
+		readPayloads = append(readPayloads, msg)
 	})
 
 	// The read request should match the original request
 	if err != nil {
 		t.Error(err)
-	} else {
-		if !expectedCaptureFile.PayloadsEqual(readPayloads) {
-			t.Error("Request read back from file should match")
-		}
+	} else if !expectedCaptureFile.PayloadsEqual(readPayloads) {
+		t.Error("Request read back from file should match")
+
 	}
 }
 
 func TestInputFileWithPayloadLargerThan64Kb(t *testing.T) {
-
 	input := NewTestInput()
-	rg := NewRequestGenerator([]io.Reader{input}, func() { input.EmitSizedPOST(64 * 1024) }, 1)
-	readPayloads := [][]byte{}
+	rg := NewRequestGenerator([]PluginReader{input}, func() { input.EmitSizedPOST(64 * 1024) }, 1)
+	readPayloads := []*Message{}
 
 	// Given a capture file with a request over 64Kb
 	expectedCaptureFile := CreateCaptureFile(rg)
 	defer expectedCaptureFile.TearDown()
 
 	// When the request is read from the capture file
-	err := ReadFromCaptureFile(expectedCaptureFile.file, 1, func(data []byte) {
-		readPayloads = append(readPayloads, Duplicate(data))
+	err := ReadFromCaptureFile(expectedCaptureFile.file, 1, func(msg *Message) {
+		readPayloads = append(readPayloads, msg)
 	})
 
 	// The read request should match the original request
 	if err != nil {
 		t.Error(err)
-	} else {
-		if !expectedCaptureFile.PayloadsEqual(readPayloads) {
-			t.Error("Request read back from file should match")
-		}
+	} else if !expectedCaptureFile.PayloadsEqual(readPayloads) {
+		t.Error("Request read back from file should match")
+
 	}
 
 }
@@ -71,28 +62,27 @@ func TestInputFileWithPayloadLargerThan64Kb(t *testing.T) {
 func TestInputFileWithGETAndPOST(t *testing.T) {
 
 	input := NewTestInput()
-	rg := NewRequestGenerator([]io.Reader{input}, func() {
+	rg := NewRequestGenerator([]PluginReader{input}, func() {
 		input.EmitGET()
 		input.EmitPOST()
 	}, 2)
-	readPayloads := [][]byte{}
+	readPayloads := []*Message{}
 
 	// Given a capture file with a GET request
 	expectedCaptureFile := CreateCaptureFile(rg)
 	defer expectedCaptureFile.TearDown()
 
 	// When the requests are read from the capture file
-	err := ReadFromCaptureFile(expectedCaptureFile.file, 2, func(data []byte) {
-		readPayloads = append(readPayloads, Duplicate(data))
+	err := ReadFromCaptureFile(expectedCaptureFile.file, 2, func(msg *Message) {
+		readPayloads = append(readPayloads, msg)
 	})
 
 	// The read requests should match the original request
 	if err != nil {
 		t.Error(err)
-	} else {
-		if !expectedCaptureFile.PayloadsEqual(readPayloads) {
-			t.Error("Request read back from file should match")
-		}
+	} else if !expectedCaptureFile.PayloadsEqual(readPayloads) {
+		t.Error("Request read back from file should match")
+
 	}
 
 }
@@ -115,12 +105,11 @@ func TestInputFileMultipleFilesWithRequestsOnly(t *testing.T) {
 	file2.Close()
 
 	input := NewFileInput(fmt.Sprintf("/tmp/%d*", rnd), false)
-	buf := make([]byte, 1000)
 
 	for i := '1'; i <= '4'; i++ {
-		n, _ := input.Read(buf)
-		if buf[4] != byte(i) {
-			t.Error("Should emit requests in right order", string(buf[:n]))
+		msg, _ := input.PluginRead()
+		if msg.Meta[4] != byte(i) {
+			t.Error("Should emit requests in right order", string(msg.Meta))
 		}
 	}
 
@@ -142,24 +131,18 @@ func TestInputFileRequestsWithLatency(t *testing.T) {
 	file.Write([]byte(payloadSeparator))
 
 	input := NewFileInput(fmt.Sprintf("/tmp/%d", rnd), false)
-	buf := make([]byte, 1000)
 
 	start := time.Now().UnixNano()
 	for i := 0; i < 3; i++ {
-		input.Read(buf)
+		input.PluginRead()
 	}
 	end := time.Now().UnixNano()
 
-	var expectedLatency int64 = 250000000 - 100000000
+	var expectedLatency int64 = 300000000 - 100000000
 	realLatency := end - start
-	if realLatency < expectedLatency {
+	if realLatency > expectedLatency {
 		t.Errorf("Should emit requests respecting latency. Expected: %v, real: %v", expectedLatency, realLatency)
 	}
-
-	if realLatency > expectedLatency+10000000 {
-		t.Errorf("Should emit requests respecting latency. Expected: %v, real: %v", expectedLatency, realLatency)
-
-	}
 }
 
 func TestInputFileMultipleFilesWithRequestsAndResponses(t *testing.T) {
@@ -188,17 +171,16 @@ func TestInputFileMultipleFilesWithRequestsAndResponses(t *testing.T) {
 	file2.Close()
 
 	input := NewFileInput(fmt.Sprintf("/tmp/%d*", rnd), false)
-	buf := make([]byte, 1000)
 
 	for i := '1'; i <= '4'; i++ {
-		n, _ := input.Read(buf)
-		if buf[0] != '1' && buf[4] != byte(i) {
-			t.Error("Shound emit requests in right order", string(buf[:n]))
+		msg, _ := input.PluginRead()
+		if msg.Meta[0] != '1' && msg.Meta[4] != byte(i) {
+			t.Error("Shound emit requests in right order", string(msg.Meta))
 		}
 
-		n, _ = input.Read(buf)
-		if buf[0] != '2' && buf[4] != byte(i) {
-			t.Error("Shound emit responses in right order", string(buf[:n]))
+		msg, _ = input.PluginRead()
+		if msg.Meta[0] != '2' && msg.Meta[4] != byte(i) {
+			t.Error("Shound emit responses in right order", string(msg.Meta))
 		}
 	}
 
@@ -217,11 +199,10 @@ func TestInputFileLoop(t *testing.T) {
 	file.Close()
 
 	input := NewFileInput(fmt.Sprintf("/tmp/%d", rnd), true)
-	buf := make([]byte, 1000)
 
 	// Even if we have just 2 requests in file, it should indifinitly loop
 	for i := 0; i < 1000; i++ {
-		input.Read(buf)
+		input.PluginRead()
 	}
 
 	input.Close()
@@ -231,24 +212,23 @@ func TestInputFileLoop(t *testing.T) {
 func TestInputFileCompressed(t *testing.T) {
 	rnd := rand.Int63()
 
-	output := NewFileOutput(fmt.Sprintf("/tmp/%d_0.gz", rnd), &FileOutputConfig{flushInterval: time.Minute, append: true})
+	output := NewFileOutput(fmt.Sprintf("/tmp/%d_0.gz", rnd), &FileOutputConfig{FlushInterval: time.Minute, Append: true})
 	for i := 0; i < 1000; i++ {
-		output.Write([]byte("1 1 1\r\ntest"))
+		output.PluginWrite(&Message{Meta: []byte("1 1 1\r\n"), Data: []byte("test")})
 	}
 	name1 := output.file.Name()
 	output.Close()
 
-	output2 := NewFileOutput(fmt.Sprintf("/tmp/%d_1.gz", rnd), &FileOutputConfig{flushInterval: time.Minute, append: true})
+	output2 := NewFileOutput(fmt.Sprintf("/tmp/%d_1.gz", rnd), &FileOutputConfig{FlushInterval: time.Minute, Append: true})
 	for i := 0; i < 1000; i++ {
-		output2.Write([]byte("1 1 1\r\ntest"))
+		output2.PluginWrite(&Message{Meta: []byte("1 1 1\r\n"), Data: []byte("test")})
 	}
 	name2 := output2.file.Name()
 	output2.Close()
 
 	input := NewFileInput(fmt.Sprintf("/tmp/%d*", rnd), false)
-	buf := make([]byte, 1000)
 	for i := 0; i < 2000; i++ {
-		input.Read(buf)
+		input.PluginRead()
 	}
 
 	os.Remove(name1)
@@ -256,30 +236,30 @@ func TestInputFileCompressed(t *testing.T) {
 }
 
 type CaptureFile struct {
-	data [][]byte
+	msgs []*Message
 	file *os.File
 }
 
-func NewExpectedCaptureFile(data [][]byte, file *os.File) *CaptureFile {
+func NewExpectedCaptureFile(msgs []*Message, file *os.File) *CaptureFile {
 	ecf := new(CaptureFile)
 	ecf.file = file
-	ecf.data = data
+	ecf.msgs = msgs
 	return ecf
 }
 
 func (expectedCaptureFile *CaptureFile) TearDown() {
 	if expectedCaptureFile.file != nil {
-		syscall.Unlink(expectedCaptureFile.file.Name())
+		os.Remove(expectedCaptureFile.file.Name())
 	}
 }
 
 type RequestGenerator struct {
-	inputs []io.Reader
+	inputs []PluginReader
 	emit   func()
 	wg     *sync.WaitGroup
 }
 
-func NewRequestGenerator(inputs []io.Reader, emit func(), count int) (rg *RequestGenerator) {
+func NewRequestGenerator(inputs []PluginReader, emit func(), count int) (rg *RequestGenerator) {
 	rg = new(RequestGenerator)
 	rg.inputs = inputs
 	rg.emit = emit
@@ -288,14 +268,17 @@ func NewRequestGenerator(inputs []io.Reader, emit func(), count int) (rg *Reques
 	return
 }
 
-func (expectedCaptureFile *CaptureFile) PayloadsEqual(other [][]byte) bool {
+func (expectedCaptureFile *CaptureFile) PayloadsEqual(other []*Message) bool {
 
-	if len(expectedCaptureFile.data) != len(other) {
+	if len(expectedCaptureFile.msgs) != len(other) {
 		return false
 	}
 
 	for i, payload := range other {
-		if !bytes.Equal(expectedCaptureFile.data[i], payload) {
+		if !bytes.Equal(expectedCaptureFile.msgs[i].Meta, payload.Meta) {
+			return false
+		}
+		if !bytes.Equal(expectedCaptureFile.msgs[i].Data, payload.Data) {
 			return false
 		}
 	}
@@ -305,56 +288,59 @@ func (expectedCaptureFile *CaptureFile) PayloadsEqual(other [][]byte) bool {
 }
 
 func CreateCaptureFile(requestGenerator *RequestGenerator) *CaptureFile {
-
 	f, err := ioutil.TempFile("", "testmainconf")
 	if err != nil {
 		panic(err)
 	}
 
-	quit := make(chan int)
-
-	readPayloads := [][]byte{}
-	output := NewTestOutput(func(data []byte) {
-		readPayloads = append(readPayloads, Duplicate(data))
-
+	readPayloads := []*Message{}
+	output := NewTestOutput(func(msg *Message) {
+		readPayloads = append(readPayloads, msg)
 		requestGenerator.wg.Done()
 	})
 
-	outputFile := NewFileOutput(f.Name(), &FileOutputConfig{flushInterval: time.Minute, append: true})
+	outputFile := NewFileOutput(f.Name(), &FileOutputConfig{FlushInterval: time.Second, Append: true})
 
-	Plugins.Inputs = requestGenerator.inputs
-	Plugins.Outputs = []io.Writer{output, outputFile}
+	plugins := &InOutPlugins{
+		Inputs:  requestGenerator.inputs,
+		Outputs: []PluginWriter{output, outputFile},
+	}
+	for _, input := range requestGenerator.inputs {
+		plugins.All = append(plugins.All, input)
+	}
+	plugins.All = append(plugins.All, output, outputFile)
 
-	go Start(quit)
+	emitter := NewEmitter()
+	go emitter.Start(plugins, Settings.Middleware)
 
 	requestGenerator.emit()
 	requestGenerator.wg.Wait()
 
 	time.Sleep(100 * time.Millisecond)
-	outputFile.Close()
-
-	close(quit)
+	emitter.Close()
 
 	return NewExpectedCaptureFile(readPayloads, f)
 
 }
 
 func ReadFromCaptureFile(captureFile *os.File, count int, callback writeCallback) (err error) {
-
-	quit := make(chan int)
 	wg := new(sync.WaitGroup)
 
 	input := NewFileInput(captureFile.Name(), false)
-	output := NewTestOutput(func(data []byte) {
-		callback(data)
+	output := NewTestOutput(func(msg *Message) {
+		callback(msg)
 		wg.Done()
 	})
 
-	Plugins.Inputs = []io.Reader{input}
-	Plugins.Outputs = []io.Writer{output}
+	plugins := &InOutPlugins{
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{output},
+	}
+	plugins.All = append(plugins.All, input, output)
 
 	wg.Add(count)
-	go Start(quit)
+	emitter := NewEmitter()
+	go emitter.Start(plugins, Settings.Middleware)
 
 	done := make(chan int, 1)
 	go func() {
@@ -368,15 +354,7 @@ func ReadFromCaptureFile(captureFile *os.File, count int, callback writeCallback
 	case <-time.After(2 * time.Second):
 		err = errors.New("Timed out")
 	}
-	close(quit)
-
+	emitter.Close()
 	return
 
 }
-
-func Duplicate(data []byte) (duplicate []byte) {
-	duplicate = make([]byte, len(data))
-	copy(duplicate, data)
-
-	return
-}

input_http.go

@@ -13,42 +13,46 @@ type HTTPInput struct {
 	data     chan []byte
 	address  string
 	listener net.Listener
+	stop     chan bool // Channel used only to indicate goroutine should shutdown
 }
 
-// NewHTTPInput constructor for HTTPInput. Accepts address with port which he will listen on.
+// NewHTTPInput constructor for HTTPInput. Accepts address with port which it will listen on.
 func NewHTTPInput(address string) (i *HTTPInput) {
 	i = new(HTTPInput)
-	i.data = make(chan []byte, 10000)
-	i.address = address
+	i.data = make(chan []byte, 1000)
+	i.stop = make(chan bool)
 
 	i.listen(address)
 
 	return
 }
 
-func (i *HTTPInput) Read(data []byte) (int, error) {
-	buf := <-i.data
+// PluginRead reads message from this plugin
+func (i *HTTPInput) PluginRead() (*Message, error) {
+	var msg Message
+	select {
+	case <-i.stop:
+		return nil, ErrorStopped
+	case buf := <-i.data:
+		msg.Data = buf
+		msg.Meta = payloadHeader(RequestPayload, uuid(), time.Now().UnixNano(), -1)
+		return &msg, nil
+	}
+}
 
-	header := payloadHeader(RequestPayload, uuid(), time.Now().UnixNano(), -1)
-
-	copy(data[0:len(header)], header)
-	copy(data[len(header):], buf)
-
-	return len(buf) + len(header), nil
+// Close closes this plugin
+func (i *HTTPInput) Close() error {
+	close(i.stop)
+	return nil
 }
 
 func (i *HTTPInput) handler(w http.ResponseWriter, r *http.Request) {
 	r.URL.Scheme = "http"
-	r.URL.Host = i.listener.Addr().String()
+	r.URL.Host = i.address
 
 	buf, _ := httputil.DumpRequestOut(r, true)
 	http.Error(w, http.StatusText(200), 200)
-
-	select {
-	case i.data <- buf:
-	default:
-		Debug("[INPUT-HTTP] Dropping requests because output can't process them fast enough")
-	}
+	i.data <- buf
 }
 
 func (i *HTTPInput) listen(address string) {
@@ -62,11 +66,12 @@ func (i *HTTPInput) listen(address string) {
 	if err != nil {
 		log.Fatal("HTTP input listener failure:", err)
 	}
+	i.address = i.listener.Addr().String()
 
 	go func() {
 		err = http.Serve(i.listener, mux)
-		if err != nil {
-			log.Fatal("HTTP input serve failure:", err)
+		if err != nil && err != http.ErrServerClosed {
+			log.Fatal("HTTP input serve failure ", err)
 		}
 	}()
 }

input_http_test.go

@@ -1,31 +1,33 @@
 package main
 
 import (
-	"github.com/buger/gor/proto"
-	"io"
-	"log"
+	"bytes"
 	"net/http"
-	"os/exec"
 	"strings"
 	"sync"
 	"testing"
+	"time"
 )
 
 func TestHTTPInput(t *testing.T) {
 	wg := new(sync.WaitGroup)
-	quit := make(chan int)
 
 	input := NewHTTPInput("127.0.0.1:0")
-	output := NewTestOutput(func(data []byte) {
+	time.Sleep(time.Millisecond)
+	output := NewTestOutput(func(*Message) {
 		wg.Done()
 	})
 
-	Plugins.Inputs = []io.Reader{input}
-	Plugins.Outputs = []io.Writer{output}
+	plugins := &InOutPlugins{
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{output},
+	}
+	plugins.All = append(plugins.All, input, output)
 
-	go Start(quit)
+	emitter := NewEmitter()
+	go emitter.Start(plugins, Settings.Middleware)
 
-	address := strings.Replace(input.listener.Addr().String(), "[::]", "127.0.0.1", -1)
+	address := strings.Replace(input.address, "[::]", "127.0.0.1", -1)
 
 	for i := 0; i < 100; i++ {
 		wg.Add(1)
@@ -33,40 +35,46 @@ func TestHTTPInput(t *testing.T) {
 	}
 
 	wg.Wait()
-
-	close(quit)
+	emitter.Close()
 }
 
 func TestInputHTTPLargePayload(t *testing.T) {
 	wg := new(sync.WaitGroup)
-	quit := make(chan int)
-
-	dd := exec.Command("dd", "if=/dev/urandom", "of=/tmp/large", "bs=1", "count=4000000")
-	err := dd.Run()
-	if err != nil {
-		log.Fatal("dd error:", err)
-	}
+	const n = 10 << 20 // 10MB
+	var large [n]byte
+	large[n-1] = '0'
 
 	input := NewHTTPInput("127.0.0.1:0")
-	output := NewTestOutput(func(data []byte) {
-		if len(proto.Body(payloadBody(data))) != 4000000 {
-			t.Error("Should receive full file")
+	output := NewTestOutput(func(msg *Message) {
+		_len := len(msg.Data)
+		if _len >= n { // considering http body CRLF
+			t.Errorf("expected body to be >= %d", n)
 		}
 		wg.Done()
 	})
-	Plugins.Inputs = []io.Reader{input}
-	Plugins.Outputs = []io.Writer{output}
-
-	go Start(quit)
-
-	wg.Add(1)
-	address := strings.Replace(input.listener.Addr().String(), "[::]", "127.0.0.1", -1)
-	curl := exec.Command("curl", "http://"+address, "--data-binary", "@/tmp/large")
-	err = curl.Run()
-	if err != nil {
-		log.Fatal("curl error:", err)
+	plugins := &InOutPlugins{
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{output},
 	}
+	plugins.All = append(plugins.All, input, output)
 
+	emitter := NewEmitter()
+	defer emitter.Close()
+	go emitter.Start(plugins, Settings.Middleware)
+
+	address := strings.Replace(input.address, "[::]", "127.0.0.1", -1)
+	var req *http.Request
+	var err error
+	req, err = http.NewRequest("POST", "http://"+address, bytes.NewBuffer(large[:]))
+	if err != nil {
+		t.Error(err)
+		return
+	}
+	wg.Add(1)
+	_, err = http.DefaultClient.Do(req)
+	if err != nil {
+		t.Error(err)
+		return
+	}
 	wg.Wait()
-	close(quit)
 }

input_kafka.go

@@ -2,38 +2,40 @@ package main
 
 import (
 	"encoding/json"
+	"log"
+	"strings"
+
 	"github.com/Shopify/sarama"
 	"github.com/Shopify/sarama/mocks"
-	"log"
 )
 
-// KafkaInput is used for recieving Kafka messages and
+// KafkaInput is used for receiving Kafka messages and
 // transforming them into HTTP payloads.
 type KafkaInput struct {
-	config    *KafkaConfig
+	config    *InputKafkaConfig
 	consumers []sarama.PartitionConsumer
 	messages  chan *sarama.ConsumerMessage
+	quit      chan struct{}
 }
 
-// NewKafkaInput creates instance of kafka consumer client.
-func NewKafkaInput(address string, config *KafkaConfig) *KafkaInput {
-	c := sarama.NewConfig()
-	// Configuration options go here
+// NewKafkaInput creates instance of kafka consumer client with TLS config
+func NewKafkaInput(address string, config *InputKafkaConfig, tlsConfig *KafkaTLSConfig) *KafkaInput {
+	c := NewKafkaConfig(tlsConfig)
 
 	var con sarama.Consumer
 
-	if config.consumer.(*mocks.Consumer) != nil {
+	if mock, ok := config.consumer.(*mocks.Consumer); ok && mock != nil {
 		con = config.consumer
 	} else {
 		var err error
-		con, err = sarama.NewConsumer([]string{config.host}, c)
+		con, err = sarama.NewConsumer(strings.Split(config.Host, ","), c)
 
 		if err != nil {
 			log.Fatalln("Failed to start Sarama(Kafka) consumer:", err)
 		}
 	}
 
-	partitions, err := con.Partitions(config.topic)
+	partitions, err := con.Partitions(config.Topic)
 	if err != nil {
 		log.Fatalln("Failed to collect Sarama(Kafka) partitions:", err)
 	}
@@ -42,10 +44,11 @@ func NewKafkaInput(address string, config *KafkaConfig) *KafkaInput {
 		config:    config,
 		consumers: make([]sarama.PartitionConsumer, len(partitions)),
 		messages:  make(chan *sarama.ConsumerMessage, 256),
+		quit:      make(chan struct{}),
 	}
 
 	for index, partition := range partitions {
-		consumer, err := con.ConsumePartition(config.topic, partition, sarama.OffsetNewest)
+		consumer, err := con.ConsumePartition(config.Topic, partition, sarama.OffsetNewest)
 		if err != nil {
 			log.Fatalln("Failed to start Sarama(Kafka) partition consumer:", err)
 		}
@@ -58,10 +61,7 @@ func NewKafkaInput(address string, config *KafkaConfig) *KafkaInput {
 			}
 		}(consumer)
 
-		if Settings.verbose {
-			// Start infinite loop for tracking errors for kafka producer.
-			go i.ErrorHandler(consumer)
-		}
+		go i.ErrorHandler(consumer)
 
 		i.consumers[index] = consumer
 	}
@@ -72,32 +72,49 @@ func NewKafkaInput(address string, config *KafkaConfig) *KafkaInput {
 // ErrorHandler should receive errors
 func (i *KafkaInput) ErrorHandler(consumer sarama.PartitionConsumer) {
 	for err := range consumer.Errors() {
-		log.Println("Failed to read access log entry:", err)
+		Debug(1, "Failed to read access log entry:", err)
 	}
 }
 
-func (i *KafkaInput) Read(data []byte) (int, error) {
-	message := <-i.messages
+// PluginRead a reads message from this plugin
+func (i *KafkaInput) PluginRead() (*Message, error) {
+	var message *sarama.ConsumerMessage
+	var msg Message
+	select {
+	case <-i.quit:
+		return nil, ErrorStopped
+	case message = <-i.messages:
+	}
+
+	msg.Data = message.Value
+	if i.config.UseJSON {
 
-	if !i.config.useJSON {
-		copy(data, message.Value)
-		return len(message.Value), nil
-	} else {
 		var kafkaMessage KafkaMessage
 		json.Unmarshal(message.Value, &kafkaMessage)
 
-		buf, err := kafkaMessage.Dump()
+		var err error
+		msg.Data, err = kafkaMessage.Dump()
 		if err != nil {
-			log.Println("Failed to decode access log entry:", err)
-			return 0, err
+			Debug(1, "[INPUT-KAFKA] failed to decode access log entry:", err)
+			return nil, err
 		}
-
-		copy(data, buf)
-
-		return len(buf), nil
 	}
+
+	// does it have meta
+	if isOriginPayload(msg.Data) {
+		msg.Meta, msg.Data = payloadMetaWithBody(msg.Data)
+	}
+
+	return &msg, nil
+
 }
 
 func (i *KafkaInput) String() string {
-	return "Kafka Input: " + i.config.host + "/" + i.config.topic
+	return "Kafka Input: " + i.config.Host + "/" + i.config.Topic
+}
+
+// Close closes this plugin
+func (i *KafkaInput) Close() error {
+	close(i.quit)
+	return nil
 }

input_kafka_test.go

@@ -1,9 +1,10 @@
 package main
 
 import (
+	"testing"
+
 	"github.com/Shopify/sarama"
 	"github.com/Shopify/sarama/mocks"
-	"testing"
 )
 
 func TestInputKafkaRAW(t *testing.T) {
@@ -12,24 +13,23 @@ func TestInputKafkaRAW(t *testing.T) {
 
 	consumer.ExpectConsumePartition("test", 0, mocks.AnyOffset).YieldMessage(&sarama.ConsumerMessage{Value: []byte("1 2 3\nGET / HTTP1.1\r\nHeader: 1\r\n\r\n")})
 	consumer.SetTopicMetadata(
-		map[string][]int32{"test": []int32{0}},
+		map[string][]int32{"test": {0}},
 	)
 
-	input := NewKafkaInput("", &KafkaConfig{
+	input := NewKafkaInput("", &InputKafkaConfig{
 		consumer: consumer,
-		topic:    "test",
-		useJSON:  false,
-	})
+		Topic:    "test",
+		UseJSON:  false,
+	}, nil)
 
-	buf := make([]byte, 1024)
-	n, err := input.Read(buf)
+	msg, err := input.PluginRead()
 
 	if err != nil {
 		t.Fatal(err)
 	}
 
-	if string(buf[:n]) != "1 2 3\nGET / HTTP1.1\r\nHeader: 1\r\n\r\n" {
-		t.Error("Message not properly decoded: ", string(buf[:n]), n)
+	if string(append(msg.Meta, msg.Data...)) != "1 2 3\nGET / HTTP1.1\r\nHeader: 1\r\n\r\n" {
+		t.Error("Message not properly decoded")
 	}
 }
 
@@ -39,23 +39,22 @@ func TestInputKafkaJSON(t *testing.T) {
 
 	consumer.ExpectConsumePartition("test", 0, mocks.AnyOffset).YieldMessage(&sarama.ConsumerMessage{Value: []byte(`{"Req_URL":"/","Req_Type":"1","Req_ID":"2","Req_Ts":"3","Req_Method":"GET","Req_Headers":{"Header":"1"}}`)})
 	consumer.SetTopicMetadata(
-		map[string][]int32{"test": []int32{0}},
+		map[string][]int32{"test": {0}},
 	)
 
-	input := NewKafkaInput("", &KafkaConfig{
+	input := NewKafkaInput("", &InputKafkaConfig{
 		consumer: consumer,
-		topic:    "test",
-		useJSON:  true,
-	})
+		Topic:    "test",
+		UseJSON:  true,
+	}, nil)
 
-	buf := make([]byte, 1024)
-	n, err := input.Read(buf)
+	msg, err := input.PluginRead()
 
 	if err != nil {
 		t.Fatal(err)
 	}
 
-	if string(buf[:n]) != "1 2 3\nGET / HTTP/1.1\r\nHeader: 1\r\n\r\n" {
-		t.Error("Message not properly decoded: ", string(buf[:n]), n)
+	if string(append(msg.Meta, msg.Data...)) != "1 2 3\nGET / HTTP/1.1\r\nHeader: 1\r\n\r\n" {
+		t.Error("Message not properly decoded")
 	}
 }

input_raw.go

@@ -1,111 +1,236 @@
 package main
 
 import (
-	"github.com/buger/gor/proto"
-	raw "github.com/buger/gor/raw_socket_listener"
+	"context"
+	"fmt"
 	"log"
 	"net"
+	"strconv"
+	"strings"
+	"sync"
 	"time"
+
+	"github.com/buger/goreplay/capture"
+	"github.com/buger/goreplay/proto"
+	"github.com/buger/goreplay/size"
+	"github.com/buger/goreplay/tcp"
 )
 
+// TCPProtocol is a number to indicate type of protocol
+type TCPProtocol uint8
+
+const (
+	// ProtocolHTTP ...
+	ProtocolHTTP TCPProtocol = iota
+	// ProtocolBinary ...
+	ProtocolBinary
+)
+
+// Set is here so that TCPProtocol can implement flag.Var
+func (protocol *TCPProtocol) Set(v string) error {
+	switch v {
+	case "", "http":
+		*protocol = ProtocolHTTP
+	case "binary":
+		*protocol = ProtocolBinary
+	default:
+		return fmt.Errorf("unsupported protocol %s", v)
+	}
+	return nil
+}
+
+func (protocol *TCPProtocol) String() string {
+	switch *protocol {
+	case ProtocolBinary:
+		return "binary"
+	case ProtocolHTTP:
+		return "http"
+	default:
+		return ""
+	}
+}
+
+// RAWInputConfig represents configuration that can be applied on raw input
+type RAWInputConfig struct {
+	capture.PcapOptions
+	Expire         time.Duration      `json:"input-raw-expire"`
+	CopyBufferSize size.Size          `json:"copy-buffer-size"`
+	Engine         capture.EngineType `json:"input-raw-engine"`
+	TrackResponse  bool               `json:"input-raw-track-response"`
+	Protocol       TCPProtocol        `json:"input-raw-protocol"`
+	RealIPHeader   string             `json:"input-raw-realip-header"`
+	Stats          bool               `json:"input-raw-stats"`
+	quit           chan bool          // Channel used only to indicate goroutine should shutdown
+	host           string
+	ports          []uint16
+}
+
 // RAWInput used for intercepting traffic for given address
 type RAWInput struct {
-	data          chan *raw.TCPMessage
-	address       string
-	expire        time.Duration
-	quit          chan bool
-	engine        int
-	realIPHeader  []byte
-	trackResponse bool
-	listener      *raw.Listener
+	sync.Mutex
+	RAWInputConfig
+	messageStats   []tcp.Stats
+	listener       *capture.Listener
+	message        chan *tcp.Message
+	cancelListener context.CancelFunc
+	closed         bool
 }
 
-// Available engines for intercepting traffic
-const (
-	EngineRawSocket = 1 << iota
-	EnginePcap
-	EnginePcapFile
-)
-
-// NewRAWInput constructor for RAWInput. Accepts address with port as argument.
-func NewRAWInput(address string, engine int, trackResponse bool, expire time.Duration, realIPHeader string) (i *RAWInput) {
+// NewRAWInput constructor for RAWInput. Accepts raw input config as arguments.
+func NewRAWInput(address string, config RAWInputConfig) (i *RAWInput) {
 	i = new(RAWInput)
-	i.data = make(chan *raw.TCPMessage)
-	i.address = address
-	i.expire = expire
-	i.engine = engine
-	i.realIPHeader = []byte(realIPHeader)
+	i.RAWInputConfig = config
+	i.message = make(chan *tcp.Message, 10000)
 	i.quit = make(chan bool)
-	i.trackResponse = trackResponse
+
+	host, _ports, err := net.SplitHostPort(address)
+	if err != nil {
+		log.Fatalf("input-raw: error while parsing address: %s", err)
+	}
+
+	var ports []uint16
+	if _ports != "" {
+		portsStr := strings.Split(_ports, ",")
+
+		for _, portStr := range portsStr {
+			port, err := strconv.Atoi(strings.TrimSpace(portStr))
+			if err != nil {
+				log.Fatalf("parsing port error: %v", err)
+			}
+			ports = append(ports, uint16(port))
+
+		}
+	}
+
+	i.host = host
+	i.ports = ports
 
 	i.listen(address)
-	i.listener.IsReady()
 
 	return
 }
 
-func (i *RAWInput) Read(data []byte) (int, error) {
-	msg := <-i.data
-	buf := msg.Bytes()
-
-	var header []byte
-
-	if msg.IsIncoming {
-		header = payloadHeader(RequestPayload, msg.UUID(), msg.Start.UnixNano(), -1)
-		if len(i.realIPHeader) > 0 {
-			buf = proto.SetHeader(buf, i.realIPHeader, []byte(msg.IP().String()))
-		}
-	} else {
-		header = payloadHeader(ResponsePayload, msg.UUID(), msg.AssocMessage.Start.UnixNano(), msg.End.UnixNano()-msg.AssocMessage.Start.UnixNano())
+// PluginRead reads meassage from this plugin
+func (i *RAWInput) PluginRead() (*Message, error) {
+	var msgTCP *tcp.Message
+	var msg Message
+	select {
+	case <-i.quit:
+		return nil, ErrorStopped
+	case msgTCP = <-i.message:
+		msg.Data = msgTCP.Data()
 	}
+	var msgType byte = ResponsePayload
+	if msgTCP.IsRequest {
+		msgType = RequestPayload
+		if i.RealIPHeader != "" {
+			msg.Data = proto.SetHeader(msg.Data, []byte(i.RealIPHeader), []byte(msgTCP.SrcAddr))
+		}
+	}
+	msg.Meta = payloadHeader(msgType, msgTCP.UUID(), msgTCP.Start.UnixNano(), msgTCP.End.UnixNano()-msgTCP.Start.UnixNano())
 
-	copy(data[0:len(header)], header)
-	copy(data[len(header):], buf)
-
-	return len(buf) + len(header), nil
+	// to be removed....
+	if msgTCP.Truncated {
+		Debug(2, "[INPUT-RAW] message truncated, increase copy-buffer-size")
+	}
+	// to be removed...
+	if msgTCP.TimedOut {
+		Debug(2, "[INPUT-RAW] message timeout reached, increase input-raw-expire")
+	}
+	if i.Stats {
+		stat := msgTCP.Stats
+		go i.addStats(stat)
+	}
+	msgTCP.Finalize()
+	msgTCP = nil
+	return &msg, nil
 }
 
 func (i *RAWInput) listen(address string) {
-	Debug("Listening for traffic on: " + address)
-
-	host, port, err := net.SplitHostPort(address)
-
-	if i.engine == EnginePcapFile {
-		host = address
-		port = "1"
-		err = nil
-	}
-
+	var err error
+	i.listener, err = capture.NewListener(i.host, i.ports, "", i.Engine, i.TrackResponse)
 	if err != nil {
-		log.Fatal("input-raw: error while parsing address", err)
+		log.Fatal(err)
 	}
+	i.listener.SetPcapOptions(i.PcapOptions)
+	err = i.listener.Activate()
+	if err != nil {
+		log.Fatal(err)
+	}
+	parser := tcp.NewMessageParser(i.CopyBufferSize, i.Expire, Debug, i.messageEmitter)
 
-	i.listener = raw.NewListener(host, port, i.engine, i.trackResponse, i.expire)
-
-	ch := i.listener.Receiver()
-
+	if i.Protocol == ProtocolHTTP {
+		parser.Start = http1StartHint
+		parser.End = http1EndHint
+	}
+	var ctx context.Context
+	ctx, i.cancelListener = context.WithCancel(context.Background())
+	errCh := i.listener.ListenBackground(ctx, parser.PacketHandler)
+	<-i.listener.Reading
+	Debug(1, i)
 	go func() {
-		for {
-			select {
-			case <-i.quit:
-				return
-			default:
-			}
-
-			// Receiving TCPMessage object
-			m := <-ch
-
-			i.data <- m
-		}
+		<-errCh // the listener closed voluntarily
+		i.Close()
 	}()
 }
 
-func (i *RAWInput) String() string {
-	return "Intercepting traffic from: " + i.address
+func (i *RAWInput) messageEmitter(m *tcp.Message) {
+	i.message <- m
 }
 
+func (i *RAWInput) String() string {
+	return fmt.Sprintf("Intercepting traffic from: %s:%s", i.host, strings.Join(strings.Fields(fmt.Sprint(i.ports)), ","))
+}
+
+// GetStats returns the stats so far and reset the stats
+func (i *RAWInput) GetStats() []tcp.Stats {
+	i.Lock()
+	defer func() {
+		i.messageStats = []tcp.Stats{}
+		i.Unlock()
+	}()
+	return i.messageStats
+}
+
+// Close closes the input raw listener
 func (i *RAWInput) Close() error {
-	i.listener.Close()
+	i.Lock()
+	defer i.Unlock()
+	if i.closed {
+		return nil
+	}
+	i.cancelListener()
 	close(i.quit)
+	i.closed = true
 	return nil
 }
+
+func (i *RAWInput) addStats(mStats tcp.Stats) {
+	i.Lock()
+	if len(i.messageStats) >= 10000 {
+		i.messageStats = []tcp.Stats{}
+	}
+	i.messageStats = append(i.messageStats, mStats)
+	i.Unlock()
+}
+
+func http1StartHint(pckt *tcp.Packet) (isRequest, isResponse bool) {
+	if proto.HasRequestTitle(pckt.Payload) {
+		return true, false
+	}
+
+	if proto.HasResponseTitle(pckt.Payload) {
+		return false, true
+	}
+
+	// No request or response detected
+	return false, false
+}
+
+func http1EndHint(m *tcp.Message) bool {
+	if m.MissingChunk() {
+		return false
+	}
+
+	return proto.HasFullPayload(m, m.PacketData()...)
+}

input_raw_test.go

@@ -2,99 +2,100 @@ package main
 
 import (
 	"bytes"
-	"github.com/buger/gor/proto"
-	"io"
 	"io/ioutil"
-	"log"
-	"math/rand"
 	"net"
 	"net/http"
 	"net/http/httptest"
 	"net/http/httputil"
-	"os"
 	"os/exec"
-	"strconv"
 	"strings"
 	"sync"
 	"sync/atomic"
 	"testing"
 	"time"
+
+	"github.com/buger/goreplay/capture"
+	"github.com/buger/goreplay/proto"
 )
 
 const testRawExpire = time.Millisecond * 200
 
 func TestRAWInputIPv4(t *testing.T) {
 	wg := new(sync.WaitGroup)
-	quit := make(chan int)
 
-	listener, err := net.Listen("tcp", ":0")
+	listener, err := net.Listen("tcp4", "127.0.0.1:0")
 	if err != nil {
-		t.Fatal(err)
+		t.Error(err)
+		return
 	}
 	origin := &http.Server{
-		Handler:      http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {}),
+		Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			w.Write([]byte("ab"))
+		}),
 		ReadTimeout:  10 * time.Second,
 		WriteTimeout: 10 * time.Second,
 	}
 	go origin.Serve(listener)
 	defer listener.Close()
-
-	originAddr := listener.Addr().String()
+	_, port, _ := net.SplitHostPort(listener.Addr().String())
 
 	var respCounter, reqCounter int64
+	conf := RAWInputConfig{
+		Engine:        capture.EnginePcap,
+		Expire:        0,
+		Protocol:      ProtocolHTTP,
+		TrackResponse: true,
+		RealIPHeader:  "X-Real-IP",
+	}
+	input := NewRAWInput(listener.Addr().String(), conf)
 
-	input := NewRAWInput(originAddr, EnginePcap, true, testRawExpire, "X-Real-IP")
-	defer input.Close()
-
-	output := NewTestOutput(func(data []byte) {
-		if data[0] == '1' {
-			body := payloadBody(data)
-			if len(proto.Header(body, []byte("X-Real-IP"))) == 0 {
-				t.Error("Should have X-Real-IP header", string(body))
+	output := NewTestOutput(func(msg *Message) {
+		if msg.Meta[0] == '1' {
+			if len(proto.Header(msg.Data, []byte("X-Real-IP"))) == 0 {
+				t.Error("Should have X-Real-IP header")
 			}
-			atomic.AddInt64(&reqCounter, 1)
+			reqCounter++
 		} else {
-			atomic.AddInt64(&respCounter, 1)
+			respCounter++
 		}
-
-		if Settings.debug {
-			log.Println(reqCounter, respCounter)
-		}
-
 		wg.Done()
 	})
 
-	Plugins.Inputs = []io.Reader{input}
-	Plugins.Outputs = []io.Writer{output}
-
-	client := NewHTTPClient("http://"+listener.Addr().String(), &HTTPClientConfig{})
-
-	go Start(quit)
-
-	for i := 0; i < 100; i++ {
-		// request + response
-		wg.Add(2)
-		client.Get("/")
-		time.Sleep(2 * time.Millisecond)
+	plugins := &InOutPlugins{
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{output},
 	}
+	plugins.All = append(plugins.All, input, output)
 
+	addr := "http://127.0.0.1:" + port
+	emitter := NewEmitter()
+	defer emitter.Close()
+	go emitter.Start(plugins, Settings.Middleware)
+	for i := 0; i < 1; i++ {
+		wg.Add(2)
+		_, err = http.Get(addr)
+		if err != nil {
+			t.Error(err)
+			return
+		}
+	}
 	wg.Wait()
-
-	close(quit)
+	const want = 10
+	if reqCounter != respCounter && reqCounter != want {
+		t.Errorf("want %d requests and %d responses, got %d requests and %d responses", want, want, reqCounter, respCounter)
+	}
 }
 
 func TestRAWInputNoKeepAlive(t *testing.T) {
 	wg := new(sync.WaitGroup)
-	quit := make(chan int)
 
-	listener, err := net.Listen("tcp", ":0")
+	listener, err := net.Listen("tcp", "127.0.0.1:0")
 	if err != nil {
 		t.Fatal(err)
 	}
 	origin := &http.Server{
 		Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-			w.Write([]byte("a"))
-			w.Write([]byte("b"))
+			w.Write([]byte("ab"))
 		}),
 		ReadTimeout:  10 * time.Second,
 		WriteTimeout: 10 * time.Second,
@@ -102,121 +103,137 @@ func TestRAWInputNoKeepAlive(t *testing.T) {
 	origin.SetKeepAlivesEnabled(false)
 	go origin.Serve(listener)
 	defer listener.Close()
+	_, port, _ := net.SplitHostPort(listener.Addr().String())
 
-	originAddr := listener.Addr().String()
-
-	input := NewRAWInput(originAddr, EnginePcap, true, testRawExpire, "")
-	defer input.Close()
-
-	output := NewTestOutput(func(data []byte) {
+	conf := RAWInputConfig{
+		Engine:        capture.EnginePcap,
+		Expire:        testRawExpire,
+		Protocol:      ProtocolHTTP,
+		TrackResponse: true,
+	}
+	input := NewRAWInput(":"+port, conf)
+	var respCounter, reqCounter int64
+	output := NewTestOutput(func(msg *Message) {
+		if msg.Meta[0] == '1' {
+			atomic.AddInt64(&reqCounter, 1)
+		} else {
+			atomic.AddInt64(&respCounter, 1)
+		}
 		wg.Done()
 	})
 
-	Plugins.Inputs = []io.Reader{input}
-	Plugins.Outputs = []io.Writer{output}
+	plugins := &InOutPlugins{
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{output},
+	}
+	plugins.All = append(plugins.All, input, output)
 
-	client := NewHTTPClient("http://"+listener.Addr().String(), &HTTPClientConfig{})
+	addr := "http://127.0.0.1:" + port
 
-	go Start(quit)
+	emitter := NewEmitter()
+	go emitter.Start(plugins, Settings.Middleware)
 
-	for i := 0; i < 100; i++ {
+	for i := 0; i < 10; i++ {
 		// request + response
 		wg.Add(2)
-		client.Get("/")
-		time.Sleep(2 * time.Millisecond)
+		_, err = http.Get(addr)
+		if err != nil {
+			t.Error(err)
+			return
+		}
 	}
 
 	wg.Wait()
-
-	close(quit)
+	const want = 10
+	if reqCounter != respCounter && reqCounter != want {
+		t.Errorf("want %d requests and %d responses, got %d requests and %d responses", want, want, reqCounter, respCounter)
+	}
+	emitter.Close()
 }
 
 func TestRAWInputIPv6(t *testing.T) {
 	wg := new(sync.WaitGroup)
-	quit := make(chan int)
 
 	listener, err := net.Listen("tcp", "[::1]:0")
 	if err != nil {
-		t.Fatal(err)
+		return
 	}
 	origin := &http.Server{
-		Handler:      http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {}),
+		Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			w.Write([]byte("ab"))
+		}),
 		ReadTimeout:  10 * time.Second,
 		WriteTimeout: 10 * time.Second,
 	}
 	go origin.Serve(listener)
 	defer listener.Close()
-
-	originAddr := listener.Addr().String()
+	_, port, _ := net.SplitHostPort(listener.Addr().String())
+	originAddr := "[::1]:" + port
 
 	var respCounter, reqCounter int64
+	conf := RAWInputConfig{
+		Engine:        capture.EnginePcap,
+		Protocol:      ProtocolHTTP,
+		TrackResponse: true,
+	}
+	input := NewRAWInput(originAddr, conf)
 
-	input := NewRAWInput(originAddr, EnginePcap, true, testRawExpire, "")
-	defer input.Close()
-
-	output := NewTestOutput(func(data []byte) {
-		if data[0] == '1' {
+	output := NewTestOutput(func(msg *Message) {
+		if msg.Meta[0] == '1' {
 			atomic.AddInt64(&reqCounter, 1)
 		} else {
 			atomic.AddInt64(&respCounter, 1)
 		}
-
-		if Settings.debug {
-			log.Println(reqCounter, respCounter)
-		}
-
 		wg.Done()
 	})
 
-	Plugins.Inputs = []io.Reader{input}
-	Plugins.Outputs = []io.Writer{output}
+	plugins := &InOutPlugins{
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{output},
+	}
 
-	client := NewHTTPClient("http://"+listener.Addr().String(), &HTTPClientConfig{})
-
-	go Start(quit)
-
-	for i := 0; i < 100; i++ {
+	emitter := NewEmitter()
+	addr := "http://" + originAddr
+	go emitter.Start(plugins, Settings.Middleware)
+	for i := 0; i < 10; i++ {
 		// request + response
 		wg.Add(2)
-		client.Get("/")
-		time.Sleep(2 * time.Millisecond)
+		_, err = http.Get(addr)
+		if err != nil {
+			t.Error(err)
+			return
+		}
 	}
 
 	wg.Wait()
-	close(quit)
+	const want = 10
+	if reqCounter != respCounter && reqCounter != want {
+		t.Errorf("want %d requests and %d responses, got %d requests and %d responses", want, want, reqCounter, respCounter)
+	}
+	emitter.Close()
 }
 
-func TestInputRAW100Expect(t *testing.T) {
+func TestInputRAWChunkedEncoding(t *testing.T) {
 	wg := new(sync.WaitGroup)
-	quit := make(chan int)
 
-	fileContent, _ := ioutil.ReadFile("COMM-LICENSE")
+	fileContent, _ := ioutil.ReadFile("README.md")
 
 	// Origing and Replay server initialization
 	origin := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		defer r.Body.Close()
 		ioutil.ReadAll(r.Body)
+
 		wg.Done()
 	}))
-	defer origin.Close()
 
 	originAddr := strings.Replace(origin.Listener.Addr().String(), "[::]", "127.0.0.1", -1)
-
-	input := NewRAWInput(originAddr, EnginePcap, true, time.Second, "")
-	defer input.Close()
-
-	// We will use it to get content of raw HTTP request
-	testOutput := NewTestOutput(func(data []byte) {
-		switch data[0] {
-		case RequestPayload:
-			if strings.Contains(string(data), "Expect: 100-continue") {
-				t.Error("Should not contain 100-continue header")
-			}
-			wg.Done()
-		case ResponsePayload:
-			wg.Done()
-		}
-	})
+	conf := RAWInputConfig{
+		Engine:        capture.EnginePcap,
+		Expire:        time.Second,
+		Protocol:      ProtocolHTTP,
+		TrackResponse: true,
+	}
+	input := NewRAWInput(originAddr, conf)
 
 	replay := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		defer r.Body.Close()
@@ -233,191 +250,101 @@ func TestInputRAW100Expect(t *testing.T) {
 
 	httpOutput := NewHTTPOutput(replay.URL, &HTTPOutputConfig{})
 
-	Plugins.Inputs = []io.Reader{input}
-	Plugins.Outputs = []io.Writer{testOutput, httpOutput}
-
-	go Start(quit)
-
-	// Origin + Response/Request Test Output + Request Http Output
-	wg.Add(4)
-	curl := exec.Command("curl", "http://"+originAddr, "--data-binary", "@COMM-LICENSE")
-	err := curl.Run()
-	if err != nil {
-		log.Fatal(err)
+	plugins := &InOutPlugins{
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{httpOutput},
 	}
+	plugins.All = append(plugins.All, input, httpOutput)
 
-	wg.Wait()
-	close(quit)
-}
-
-func TestInputRAWChunkedEncoding(t *testing.T) {
-	wg := new(sync.WaitGroup)
-	quit := make(chan int)
-
-	fileContent, _ := ioutil.ReadFile("README.md")
-
-	// Origing and Replay server initialization
-	origin := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		defer r.Body.Close()
-		ioutil.ReadAll(r.Body)
-
-		wg.Done()
-	}))
-
-	originAddr := strings.Replace(origin.Listener.Addr().String(), "[::]", "127.0.0.1", -1)
-	input := NewRAWInput(originAddr, EnginePcap, true, time.Second, "")
-	defer input.Close()
-
-	replay := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		defer r.Body.Close()
-		body, _ := ioutil.ReadAll(r.Body)
-
-		if !bytes.Equal(body, fileContent) {
-			buf, _ := httputil.DumpRequest(r, true)
-			t.Error("Wrong POST body:", string(buf))
-		}
-
-		wg.Done()
-	}))
-	defer replay.Close()
-
-	httpOutput := NewHTTPOutput(replay.URL, &HTTPOutputConfig{Debug: true})
-
-	Plugins.Inputs = []io.Reader{input}
-	Plugins.Outputs = []io.Writer{httpOutput}
-
-	go Start(quit)
-
+	emitter := NewEmitter()
+	defer emitter.Close()
+	go emitter.Start(plugins, Settings.Middleware)
 	wg.Add(2)
 
 	curl := exec.Command("curl", "http://"+originAddr, "--header", "Transfer-Encoding: chunked", "--header", "Expect:", "--data-binary", "@README.md")
 	err := curl.Run()
 	if err != nil {
-		log.Fatal(err)
-	}
-
-	wg.Wait()
-
-	close(quit)
-}
-
-func TestInputRAWLargePayload(t *testing.T) {
-	// FIXME: Large payloads does not work for travis for some reason...
-	if os.Getenv("TRAVIS_BUILD_DIR") != "" {
+		t.Error(err)
 		return
 	}
-	wg := new(sync.WaitGroup)
-	quit := make(chan int)
-	sizeB := 100 * 1000
 
-	// Generate 100kb file
-	dd := exec.Command("dd", "if=/dev/urandom", "of=/tmp/large", "bs=1", "count="+strconv.Itoa(sizeB))
-	err := dd.Run()
+	wg.Wait()
+}
+
+func BenchmarkRAWInputWithReplay(b *testing.B) {
+	var respCounter, reqCounter, replayCounter uint32
+	wg := &sync.WaitGroup{}
+
+	listener, err := net.Listen("tcp4", "127.0.0.1:0")
 	if err != nil {
-		log.Fatal("dd error:", err)
+		b.Error(err)
+		return
+	}
+	listener0, err := net.Listen("tcp4", "127.0.0.1:0")
+	if err != nil {
+		b.Error(err)
+		return
 	}
 
-	origin := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
-		defer req.Body.Close()
-		body, _ := ioutil.ReadAll(req.Body)
+	origin := http.Server{
+		Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			w.Write([]byte("ab"))
+		}),
+	}
+	go origin.Serve(listener)
+	defer origin.Close()
+	originAddr := listener.Addr().String()
 
-		if len(body) != sizeB {
-			t.Error("File size should be 1mb:", len(body))
-		}
-
-		wg.Done()
-	}))
-	originAddr := strings.Replace(origin.Listener.Addr().String(), "[::]", "127.0.0.1", -1)
-
-	input := NewRAWInput(originAddr, EnginePcap, true, testRawExpire, "")
-	defer input.Close()
-
-	replay := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
-		body, _ := ioutil.ReadAll(req.Body)
-		// // req.Body = http.MaxBytesReader(w, req.Body, 1*1024*1024)
-		// // buf := make([]byte, 1*1024*1024)
-		// n, _ := req.Body.Read(buf)
-		// body := buf[0:n]
-
-		if len(body) != sizeB {
-			t.Errorf("File size should be %d bytes: %d", sizeB, len(body))
-		}
-
-		wg.Done()
-	}))
+	replay := http.Server{
+		Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			atomic.AddUint32(&replayCounter, 1)
+			w.Write(nil)
+			wg.Done()
+		}),
+	}
+	go replay.Serve(listener0)
 	defer replay.Close()
+	replayAddr := listener0.Addr().String()
 
-	httpOutput := NewHTTPOutput(replay.URL, &HTTPOutputConfig{Debug: false})
+	conf := RAWInputConfig{
+		Engine:        capture.EnginePcap,
+		Expire:        testRawExpire,
+		Protocol:      ProtocolHTTP,
+		TrackResponse: true,
+	}
+	input := NewRAWInput(originAddr, conf)
 
-	Plugins.Inputs = []io.Reader{input}
-	Plugins.Outputs = []io.Writer{httpOutput}
+	testOutput := NewTestOutput(func(msg *Message) {
+		if msg.Meta[0] == '1' {
+			reqCounter++
+		} else {
+			respCounter++
+		}
+		wg.Done()
+	})
+	httpOutput := NewHTTPOutput("http://"+replayAddr, &HTTPOutputConfig{})
 
-	go Start(quit)
+	plugins := &InOutPlugins{
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{testOutput, httpOutput},
+	}
 
-	wg.Add(2)
-	curl := exec.Command("curl", "http://"+originAddr, "--header", "Transfer-Encoding: chunked", "--header", "Expect:", "--data-binary", "@/tmp/large")
-	err = curl.Run()
-	if err != nil {
-		log.Fatal("curl error:", err)
+	emitter := NewEmitter()
+	go emitter.Start(plugins, Settings.Middleware)
+	addr := "http://" + originAddr
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		wg.Add(3) // reqCounter + replayCounter + respCounter
+		resp, err := http.Get(addr)
+		if err != nil {
+			wg.Add(-3)
+		}
+		resp.Body.Close()
 	}
 
 	wg.Wait()
-	close(quit)
-}
-
-func BenchmarkRAWInput(b *testing.B) {
-	quit := make(chan int)
-
-	origin := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {}))
-	defer origin.Close()
-	originAddr := strings.Replace(origin.Listener.Addr().String(), "[::]", "127.0.0.1", -1)
-
-	var respCounter, reqCounter int64
-
-	input := NewRAWInput(originAddr, EnginePcap, true, testRawExpire, "")
-	defer input.Close()
-
-	output := NewTestOutput(func(data []byte) {
-		if data[0] == '1' {
-			atomic.AddInt64(&reqCounter, 1)
-		} else {
-			atomic.AddInt64(&respCounter, 1)
-		}
-
-		// log.Println("Captured ", reqCounter, "requests and ", respCounter, " responses")
-	})
-
-	Plugins.Inputs = []io.Reader{input}
-	Plugins.Outputs = []io.Writer{output}
-
-	go Start(quit)
-
-	emitted := 0
-	fileContent, _ := ioutil.ReadFile("LICENSE.txt")
-
-	for i := 0; i < b.N; i++ {
-		wg := new(sync.WaitGroup)
-		wg.Add(10 * 100)
-		emitted += 10 * 100
-		for w := 0; w < 100; w++ {
-			go func() {
-				client := NewHTTPClient(origin.URL, &HTTPClientConfig{})
-				for i := 0; i < 10; i++ {
-					if rand.Int63n(2) == 0 {
-						client.Post("/", fileContent)
-					} else {
-						client.Get("/")
-					}
-					time.Sleep(time.Duration(rand.Int63n(50)) * time.Millisecond)
-					wg.Done()
-				}
-			}()
-		}
-		wg.Wait()
-	}
-
-	time.Sleep(400 * time.Millisecond)
-	log.Println("Emitted ", emitted, ", Captured ", reqCounter, "requests and ", respCounter, " responses")
-
-	close(quit)
+	b.ReportMetric(float64(reqCounter), "requests")
+	b.ReportMetric(float64(respCounter), "responses")
+	b.ReportMetric(float64(replayCounter), "replayed")
+	emitter.Close()
 }

input_tcp.go

@@ -3,86 +3,125 @@ package main
 import (
 	"bufio"
 	"bytes"
+	"crypto/tls"
 	"fmt"
 	"io"
 	"log"
 	"net"
-	"os"
 )
 
 // TCPInput used for internal communication
 type TCPInput struct {
-	data     chan []byte
-	address  string
+	data     chan *Message
 	listener net.Listener
+	address  string
+	config   *TCPInputConfig
+	stop     chan bool // Channel used only to indicate goroutine should shutdown
+}
+
+// TCPInputConfig represents configuration of a TCP input plugin
+type TCPInputConfig struct {
+	Secure          bool   `json:"input-tcp-secure"`
+	CertificatePath string `json:"input-tcp-certificate"`
+	KeyPath         string `json:"input-tcp-certificate-key"`
 }
 
 // NewTCPInput constructor for TCPInput, accepts address with port
-func NewTCPInput(address string) (i *TCPInput) {
+func NewTCPInput(address string, config *TCPInputConfig) (i *TCPInput) {
 	i = new(TCPInput)
-	i.data = make(chan []byte, 1000)
+	i.data = make(chan *Message, 1000)
 	i.address = address
+	i.config = config
+	i.stop = make(chan bool)
 
 	i.listen(address)
 
 	return
 }
 
-func (i *TCPInput) Read(data []byte) (int, error) {
-	buf := <-i.data
-	copy(data, buf)
+// PluginRead returns data and details read from plugin
+func (i *TCPInput) PluginRead() (msg *Message, err error) {
+	select {
+	case <-i.stop:
+		return nil, ErrorStopped
+	case msg = <-i.data:
+		return msg, nil
+	}
 
-	return len(buf), nil
+}
+
+// Close closes the plugin
+func (i *TCPInput) Close() error {
+	close(i.stop)
+	i.listener.Close()
+	return nil
 }
 
 func (i *TCPInput) listen(address string) {
-	listener, err := net.Listen("tcp", address)
-	i.listener = listener
+	if i.config.Secure {
+		cer, err := tls.LoadX509KeyPair(i.config.CertificatePath, i.config.KeyPath)
+		if err != nil {
+			log.Fatalln("error while loading --input-tcp TLS certificate:", err)
+		}
 
-	if err != nil {
-		log.Fatal("Can't start:", err)
+		config := &tls.Config{Certificates: []tls.Certificate{cer}}
+		listener, err := tls.Listen("tcp", address, config)
+		if err != nil {
+			log.Fatalln("[INPUT-TCP] failed to start INPUT-TCP listener:", err)
+		}
+		i.listener = listener
+	} else {
+		listener, err := net.Listen("tcp", address)
+		if err != nil {
+			log.Fatalln("failed to start INPUT-TCP listener:", err)
+		}
+		i.listener = listener
 	}
-
 	go func() {
 		for {
-			conn, err := listener.Accept()
-
-			if err != nil {
-				log.Println("Error while Accept()", err)
+			conn, err := i.listener.Accept()
+			if err == nil {
+				go i.handleConnection(conn)
 				continue
 			}
-
-			go i.handleConnection(conn)
+			if isTemporaryNetworkError(err) {
+				continue
+			}
+			if operr, ok := err.(*net.OpError); ok && operr.Err.Error() != "use of closed network connection" {
+				Debug(0, fmt.Sprintf("[INPUT-TCP] listener closed, err: %q", err))
+			}
+			break
 		}
 	}()
 }
 
+var payloadSeparatorAsBytes = []byte(payloadSeparator)
+
 func (i *TCPInput) handleConnection(conn net.Conn) {
 	defer conn.Close()
 
-	payloadSeparatorAsBytes := []byte(payloadSeparator)
 	reader := bufio.NewReader(conn)
 	var buffer bytes.Buffer
 
 	for {
 		line, err := reader.ReadBytes('\n')
-
 		if err != nil {
+			if isTemporaryNetworkError(err) {
+				continue
+			}
 			if err != io.EOF {
-				fmt.Fprintln(os.Stderr, "Unexpected error in input tcp connection:", err)
+				Debug(0, fmt.Sprintf("[INPUT-TCP] connection error: %q", err))
 			}
 			break
-
 		}
 
 		if bytes.Equal(payloadSeparatorAsBytes[1:], line) {
-			asBytes := buffer.Bytes()
+			// unread the '\n' before monkeys
+			buffer.UnreadByte()
+			var msg Message
+			msg.Meta, msg.Data = payloadMetaWithBody(buffer.Bytes())
+			i.data <- &msg
 			buffer.Reset()
-
-			newBuf := make([]byte, len(asBytes)-1)
-			copy(newBuf, asBytes)
-
-			i.data <- newBuf
 		} else {
 			buffer.Write(line)
 		}
@@ -92,3 +131,13 @@ func (i *TCPInput) handleConnection(conn net.Conn) {
 func (i *TCPInput) String() string {
 	return "TCP input: " + i.address
 }
+
+func isTemporaryNetworkError(err error) bool {
+	if nerr, ok := err.(net.Error); ok && nerr.Temporary() {
+		return true
+	}
+	if operr, ok := err.(*net.OpError); ok && operr.Temporary() {
+		return true
+	}
+	return false
+}

input_tcp_test.go

@@ -1,26 +1,38 @@
 package main
 
 import (
-	"io"
+	"bytes"
+	"crypto/rand"
+	"crypto/rsa"
+	"crypto/tls"
+	"crypto/x509"
+	"encoding/pem"
+	"io/ioutil"
 	"log"
+	"math/big"
 	"net"
+	"os"
 	"sync"
 	"testing"
+	"time"
 )
 
 func TestTCPInput(t *testing.T) {
 	wg := new(sync.WaitGroup)
-	quit := make(chan int)
 
-	input := NewTCPInput("127.0.0.1:0")
-	output := NewTestOutput(func(data []byte) {
+	input := NewTCPInput("127.0.0.1:0", &TCPInputConfig{})
+	output := NewTestOutput(func(*Message) {
 		wg.Done()
 	})
 
-	Plugins.Inputs = []io.Reader{input}
-	Plugins.Outputs = []io.Writer{output}
+	plugins := &InOutPlugins{
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{output},
+	}
+	plugins.All = append(plugins.All, input, output)
 
-	go Start(quit)
+	emitter := NewEmitter()
+	go emitter.Start(plugins, Settings.Middleware)
 
 	tcpAddr, err := net.ResolveTCPAddr("tcp", input.listener.Addr().String())
 
@@ -29,13 +41,96 @@ func TestTCPInput(t *testing.T) {
 	}
 
 	conn, err := net.DialTCP("tcp", nil, tcpAddr)
-
 	if err != nil {
 		log.Fatal(err)
 	}
 
 	msg := []byte("1 1 1\nGET / HTTP/1.1\r\n\r\n")
 
+	for i := 0; i < 100; i++ {
+		wg.Add(1)
+		if _, err = conn.Write(msg); err == nil {
+			_, err = conn.Write(payloadSeparatorAsBytes)
+		}
+		if err != nil {
+			t.Error(err)
+			return
+		}
+	}
+	wg.Wait()
+	emitter.Close()
+}
+
+func genCertificate(template *x509.Certificate) ([]byte, []byte) {
+	priv, _ := rsa.GenerateKey(rand.Reader, 2048)
+
+	serialNumberLimit := new(big.Int).Lsh(big.NewInt(1), 128)
+	serialNumber, _ := rand.Int(rand.Reader, serialNumberLimit)
+	template.SerialNumber = serialNumber
+	template.BasicConstraintsValid = true
+	template.NotBefore = time.Now()
+	template.NotAfter = time.Now().Add(time.Hour)
+
+	derBytes, _ := x509.CreateCertificate(rand.Reader, template, template, &priv.PublicKey, priv)
+
+	var certPem, keyPem bytes.Buffer
+	pem.Encode(&certPem, &pem.Block{Type: "CERTIFICATE", Bytes: derBytes})
+	pem.Encode(&keyPem, &pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(priv)})
+
+	return certPem.Bytes(), keyPem.Bytes()
+}
+
+func TestTCPInputSecure(t *testing.T) {
+	serverCertPem, serverPrivPem := genCertificate(&x509.Certificate{
+		DNSNames:    []string{"localhost"},
+		IPAddresses: []net.IP{net.ParseIP("127.0.0.1"), net.ParseIP("::")},
+	})
+
+	serverCertPemFile, _ := ioutil.TempFile("", "server.crt")
+	serverCertPemFile.Write(serverCertPem)
+	serverCertPemFile.Close()
+
+	serverPrivPemFile, _ := ioutil.TempFile("", "server.key")
+	serverPrivPemFile.Write(serverPrivPem)
+	serverPrivPemFile.Close()
+
+	defer func() {
+		os.Remove(serverPrivPemFile.Name())
+		os.Remove(serverCertPemFile.Name())
+	}()
+
+	wg := new(sync.WaitGroup)
+
+	input := NewTCPInput("127.0.0.1:0", &TCPInputConfig{
+		Secure:          true,
+		CertificatePath: serverCertPemFile.Name(),
+		KeyPath:         serverPrivPemFile.Name(),
+	})
+	output := NewTestOutput(func(*Message) {
+		wg.Done()
+	})
+
+	plugins := &InOutPlugins{
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{output},
+	}
+	plugins.All = append(plugins.All, input, output)
+
+	emitter := NewEmitter()
+	go emitter.Start(plugins, Settings.Middleware)
+
+	conf := &tls.Config{
+		InsecureSkipVerify: true,
+	}
+
+	conn, err := tls.Dial("tcp", input.listener.Addr().String(), conf)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer conn.Close()
+
+	msg := []byte("1 1 1\nGET / HTTP/1.1\r\n\r\n")
+
 	for i := 0; i < 100; i++ {
 		wg.Add(1)
 		conn.Write(msg)
@@ -43,6 +138,5 @@ func TestTCPInput(t *testing.T) {
 	}
 
 	wg.Wait()
-
-	close(quit)
+	emitter.Close()
 }

kafka.go

@@ -2,19 +2,41 @@ package main
 
 import (
 	"bytes"
+	"crypto/tls"
+	"crypto/x509"
+	"errors"
 	"fmt"
+	"io/ioutil"
+	"log"
+
 	"github.com/Shopify/sarama"
-	"github.com/buger/gor/proto"
+	"github.com/buger/goreplay/proto"
 )
 
-// KafkaConfig should contains required information to
+// InputKafkaConfig should contains required information to
 // build producers.
-type KafkaConfig struct {
-	host     string
-	topic    string
+type InputKafkaConfig struct {
 	producer sarama.AsyncProducer
 	consumer sarama.Consumer
-	useJSON  bool
+	Host     string `json:"input-kafka-host"`
+	Topic    string `json:"input-kafka-topic"`
+	UseJSON  bool   `json:"input-kafka-json-format"`
+}
+
+// OutputKafkaConfig is the representation of kfka output configuration
+type OutputKafkaConfig struct {
+	producer sarama.AsyncProducer
+	consumer sarama.Consumer
+	Host     string `json:"output-kafka-host"`
+	Topic    string `json:"output-kafka-topic"`
+	UseJSON  bool   `json:"output-kafka-json-format"`
+}
+
+// KafkaTLSConfig should contains TLS certificates for connecting to secured Kafka clusters
+type KafkaTLSConfig struct {
+	CACert     string `json:"kafka-tls-ca-cert"`
+	ClientCert string `json:"kafka-tls-client-cert"`
+	ClientKey  string `json:"kafka-tls-client-key"`
 }
 
 // KafkaMessage should contains catched request information that should be
@@ -29,6 +51,52 @@ type KafkaMessage struct {
 	ReqHeaders map[string]string `json:"Req_Headers,omitempty"`
 }
 
+// NewTLSConfig loads TLS certificates
+func NewTLSConfig(clientCertFile, clientKeyFile, caCertFile string) (*tls.Config, error) {
+	tlsConfig := tls.Config{}
+
+	if clientCertFile != "" && clientKeyFile == "" {
+		return &tlsConfig, errors.New("Missing key of client certificate in kafka")
+	}
+	if clientCertFile == "" && clientKeyFile != "" {
+		return &tlsConfig, errors.New("missing TLS client certificate in kafka")
+	}
+	// Load client cert
+	if (clientCertFile != "") && (clientKeyFile != "") {
+		cert, err := tls.LoadX509KeyPair(clientCertFile, clientKeyFile)
+		if err != nil {
+			return &tlsConfig, err
+		}
+		tlsConfig.Certificates = []tls.Certificate{cert}
+	}
+	// Load CA cert
+	if caCertFile != "" {
+		caCert, err := ioutil.ReadFile(caCertFile)
+		if err != nil {
+			return &tlsConfig, err
+		}
+		caCertPool := x509.NewCertPool()
+		caCertPool.AppendCertsFromPEM(caCert)
+		tlsConfig.RootCAs = caCertPool
+	}
+	return &tlsConfig, nil
+}
+
+// NewKafkaConfig returns Kafka config with or without TLS
+func NewKafkaConfig(tlsConfig *KafkaTLSConfig) *sarama.Config {
+	config := sarama.NewConfig()
+	// Configuration options go here
+	if tlsConfig != nil && (tlsConfig.ClientCert != "" || tlsConfig.CACert != "") {
+		config.Net.TLS.Enable = true
+		tlsConfig, err := NewTLSConfig(tlsConfig.ClientCert, tlsConfig.ClientKey, tlsConfig.CACert)
+		if err != nil {
+			log.Fatal(err)
+		}
+		config.Net.TLS.Config = tlsConfig
+	}
+	return config
+}
+
 // Dump returns the given request in its HTTP/1.x wire
 // representation.
 func (m KafkaMessage) Dump() ([]byte, error) {
@@ -36,13 +104,13 @@ func (m KafkaMessage) Dump() ([]byte, error) {
 
 	b.WriteString(fmt.Sprintf("%s %s %s\n", m.ReqType, m.ReqID, m.ReqTs))
 	b.WriteString(fmt.Sprintf("%s %s HTTP/1.1", m.ReqMethod, m.ReqURL))
-	b.Write(proto.CLRF)
+	b.Write(proto.CRLF)
 	for key, value := range m.ReqHeaders {
 		b.WriteString(fmt.Sprintf("%s: %s", key, value))
-		b.Write(proto.CLRF)
+		b.Write(proto.CRLF)
 	}
 
-	b.Write(proto.CLRF)
+	b.Write(proto.CRLF)
 	b.WriteString(m.ReqBody)
 
 	return b.Bytes(), nil

limiter.go

@@ -20,8 +20,8 @@ type Limiter struct {
 }
 
 func parseLimitOptions(options string) (limit int, isPercent bool) {
-	if strings.Contains(options, "%") {
-		limit, _ = strconv.Atoi(strings.Split(options, "%")[0])
+	if n := strings.Index(options, "%"); n > 0 {
+		limit, _ = strconv.Atoi(options[:n])
 		isPercent = true
 	} else {
 		limit, _ = strconv.Atoi(options)
@@ -33,7 +33,7 @@ func parseLimitOptions(options string) (limit int, isPercent bool) {
 
 // NewLimiter constructor for Limiter, accepts plugin and options
 // `options` allow to sprcify relatve or absolute limiting
-func NewLimiter(plugin interface{}, options string) io.ReadWriter {
+func NewLimiter(plugin interface{}, options string) PluginReadWriter {
 	l := new(Limiter)
 	l.limit, l.isPercent = parseLimitOptions(options)
 	l.plugin = plugin
@@ -71,26 +71,42 @@ func (l *Limiter) isLimited() bool {
 	return false
 }
 
-func (l *Limiter) Write(data []byte) (n int, err error) {
+// PluginWrite writes message to this plugin
+func (l *Limiter) PluginWrite(msg *Message) (n int, err error) {
 	if l.isLimited() {
 		return 0, nil
 	}
-
-	n, err = l.plugin.(io.Writer).Write(data)
-
-	return
+	if w, ok := l.plugin.(PluginWriter); ok {
+		return w.PluginWrite(msg)
+	}
+	// avoid further writing
+	return 0, io.ErrClosedPipe
 }
 
-func (l *Limiter) Read(data []byte) (n int, err error) {
-	n, err = l.plugin.(io.Reader).Read(data)
+// PluginRead reads message from this plugin
+func (l *Limiter) PluginRead() (msg *Message, err error) {
+	if r, ok := l.plugin.(PluginReader); ok {
+		msg, err = r.PluginRead()
+	} else {
+		// avoid further reading
+		return nil, io.ErrClosedPipe
+	}
 
 	if l.isLimited() {
-		return 0, nil
+		return nil, nil
 	}
 
 	return
 }
 
 func (l *Limiter) String() string {
-	return fmt.Sprintf("Limiting %s to: %d (isPercent: %b)", l.plugin, l.limit, l.isPercent)
+	return fmt.Sprintf("Limiting %s to: %d (isPercent: %v)", l.plugin, l.limit, l.isPercent)
+}
+
+// Close closes the resources.
+func (l *Limiter) Close() error {
+	if fi, ok := l.plugin.(io.Closer); ok {
+		fi.Close()
+	}
+	return nil
 }

limiter_test.go

@@ -3,104 +3,109 @@
 package main
 
 import (
-	"io"
 	"sync"
 	"testing"
 )
 
 func TestOutputLimiter(t *testing.T) {
 	wg := new(sync.WaitGroup)
-	quit := make(chan int)
 
 	input := NewTestInput()
-	output := NewLimiter(NewTestOutput(func(data []byte) {
+	output := NewLimiter(NewTestOutput(func(*Message) {
 		wg.Done()
 	}), "10")
 	wg.Add(10)
 
-	Plugins.Inputs = []io.Reader{input}
-	Plugins.Outputs = []io.Writer{output}
+	plugins := &InOutPlugins{
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{output},
+	}
+	plugins.All = append(plugins.All, input, output)
 
-	go Start(quit)
+	emitter := NewEmitter()
+	go emitter.Start(plugins, Settings.Middleware)
 
 	for i := 0; i < 100; i++ {
 		input.EmitGET()
 	}
 
 	wg.Wait()
-
-	close(quit)
+	emitter.Close()
 }
 
 func TestInputLimiter(t *testing.T) {
 	wg := new(sync.WaitGroup)
-	quit := make(chan int)
 
 	input := NewLimiter(NewTestInput(), "10")
-	output := NewTestOutput(func(data []byte) {
+	output := NewTestOutput(func(*Message) {
 		wg.Done()
 	})
 	wg.Add(10)
 
-	Plugins.Inputs = []io.Reader{input}
-	Plugins.Outputs = []io.Writer{output}
+	plugins := &InOutPlugins{
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{output},
+	}
+	plugins.All = append(plugins.All, input, output)
 
-	go Start(quit)
+	emitter := NewEmitter()
+	go emitter.Start(plugins, Settings.Middleware)
 
 	for i := 0; i < 100; i++ {
 		input.(*Limiter).plugin.(*TestInput).EmitGET()
 	}
 
 	wg.Wait()
-
-	close(quit)
+	emitter.Close()
 }
 
 // Should limit all requests
 func TestPercentLimiter1(t *testing.T) {
 	wg := new(sync.WaitGroup)
-	quit := make(chan int)
 
 	input := NewTestInput()
-	output := NewLimiter(NewTestOutput(func(data []byte) {
+	output := NewLimiter(NewTestOutput(func(*Message) {
 		wg.Done()
 	}), "0%")
 
-	Plugins.Inputs = []io.Reader{input}
-	Plugins.Outputs = []io.Writer{output}
+	plugins := &InOutPlugins{
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{output},
+	}
+	plugins.All = append(plugins.All, input, output)
 
-	go Start(quit)
+	emitter := NewEmitter()
+	go emitter.Start(plugins, Settings.Middleware)
 
 	for i := 0; i < 100; i++ {
 		input.EmitGET()
 	}
 
 	wg.Wait()
-
-	close(quit)
 }
 
 // Should not limit at all
 func TestPercentLimiter2(t *testing.T) {
 	wg := new(sync.WaitGroup)
-	quit := make(chan int)
 
 	input := NewTestInput()
-	output := NewLimiter(NewTestOutput(func(data []byte) {
+	output := NewLimiter(NewTestOutput(func(*Message) {
 		wg.Done()
 	}), "100%")
 	wg.Add(100)
 
-	Plugins.Inputs = []io.Reader{input}
-	Plugins.Outputs = []io.Writer{output}
+	plugins := &InOutPlugins{
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{output},
+	}
+	plugins.All = append(plugins.All, input, output)
 
-	go Start(quit)
+	emitter := NewEmitter()
+	go emitter.Start(plugins, Settings.Middleware)
 
 	for i := 0; i < 100; i++ {
 		input.EmitGET()
 	}
 
 	wg.Wait()
-
-	close(quit)
 }

middleware.go

@@ -2,84 +2,104 @@ package main
 
 import (
 	"bufio"
+	"context"
 	"encoding/hex"
 	"fmt"
 	"io"
-	"log"
 	"os"
 	"os/exec"
 	"strings"
 	"sync"
+	"syscall"
 )
 
+// Middleware represents a middleware object
 type Middleware struct {
-	command string
-
-	data chan []byte
-
-	mu sync.Mutex
-
-	Stdin  io.Writer
-	Stdout io.Reader
+	command       string
+	data          chan *Message
+	Stdin         io.Writer
+	Stdout        io.Reader
+	commandCancel context.CancelFunc
+	stop          chan bool // Channel used only to indicate goroutine should shutdown
+	closed        bool
+	mu            sync.RWMutex
 }
 
+// NewMiddleware returns new middleware
 func NewMiddleware(command string) *Middleware {
 	m := new(Middleware)
 	m.command = command
-	m.data = make(chan []byte, 1000)
+	m.data = make(chan *Message, 1000)
+	m.stop = make(chan bool)
 
 	commands := strings.Split(command, " ")
-	cmd := exec.Command(commands[0], commands[1:]...)
+	ctx, cancl := context.WithCancel(context.Background())
+	m.commandCancel = cancl
+	cmd := exec.CommandContext(ctx, commands[0], commands[1:]...)
 
 	m.Stdout, _ = cmd.StdoutPipe()
 	m.Stdin, _ = cmd.StdinPipe()
 
-	if Settings.verbose {
-		cmd.Stderr = os.Stderr
-	}
+	cmd.Stderr = os.Stderr
 
 	go m.read(m.Stdout)
 
 	go func() {
-		err := cmd.Start()
-
-		if err != nil {
-			log.Fatal(err)
+		defer m.Close()
+		var err error
+		if err = cmd.Start(); err == nil {
+			err = cmd.Wait()
 		}
-
-		err = cmd.Wait()
-
 		if err != nil {
-			log.Fatal(err)
+			if e, ok := err.(*exec.ExitError); ok {
+				status := e.Sys().(syscall.WaitStatus)
+				if status.Signal() == syscall.SIGKILL /*killed or context canceld */ {
+					return
+				}
+			}
+			Debug(0, fmt.Sprintf("[MIDDLEWARE] command[%q] error: %q", command, err.Error()))
 		}
 	}()
 
 	return m
 }
 
-func (m *Middleware) ReadFrom(plugin io.Reader) {
-	Debug("[MIDDLEWARE-MASTER] Starting reading from", plugin)
+// ReadFrom start a worker to read from this plugin
+func (m *Middleware) ReadFrom(plugin PluginReader) {
+	Debug(2, fmt.Sprintf("[MIDDLEWARE] command[%q] Starting reading from %q", m.command, plugin))
 	go m.copy(m.Stdin, plugin)
 }
 
-func (m *Middleware) copy(to io.Writer, from io.Reader) {
-	buf := make([]byte, 5*1024*1024)
-	dst := make([]byte, len(buf)*2)
+func (m *Middleware) copy(to io.Writer, from PluginReader) {
+	var buf, dst []byte
 
 	for {
-		nr, _ := from.Read(buf)
-		if nr > 0 && len(buf) > nr {
+		msg, err := from.PluginRead()
+		if err != nil {
+			return
+		}
+		if msg == nil || len(msg.Data) == 0 {
+			continue
+		}
+		buf = msg.Data
+		if Settings.PrettifyHTTP {
+			buf = prettifyHTTP(msg.Data)
+		}
+		dstLen := (len(buf)+len(msg.Meta))*2 + 1
+		// if enough space was previously allocated use it instead
+		if dstLen > len(dst) {
+			dst = make([]byte, dstLen)
+		}
+		n := hex.Encode(dst, msg.Meta)
+		n += hex.Encode(dst[n:], buf)
+		dst[n] = '\n'
 
-			hex.Encode(dst, buf[0:nr])
-			dst[nr*2] = '\n'
-
-			m.mu.Lock()
-			to.Write(dst[0 : nr*2+1])
-			m.mu.Unlock()
-
-			if Settings.debug {
-				Debug("[MIDDLEWARE-MASTER] Sending:", string(buf[0:nr]), "From:", from)
-			}
+		n, err = to.Write(dst[:n+1])
+		if err == nil {
+			continue
+		}
+		if m.isClosed() {
+			return
 		}
 	}
 }
@@ -88,38 +108,59 @@ func (m *Middleware) read(from io.Reader) {
 	reader := bufio.NewReader(from)
 	var line []byte
 	var e error
-
 	for {
 		if line, e = reader.ReadBytes('\n'); e != nil {
-			if e == io.EOF {
-				continue
-			} else {
-				break
+			if m.isClosed() {
+				return
 			}
+			continue
 		}
-
-		buf := make([]byte, len(line)/2)
+		buf := make([]byte, (len(line)-1)/2)
 		if _, err := hex.Decode(buf, line[:len(line)-1]); err != nil {
-			fmt.Fprintln(os.Stderr, "Failed to decode input payload", err, len(line))
+			Debug(0, fmt.Sprintf("[MIDDLEWARE] command[%q] failed to decode err: %q", m.command, err))
+			continue
 		}
-
-		if Settings.debug {
-			Debug("[MIDDLEWARE-MASTER] Received:", string(buf))
+		var msg Message
+		msg.Meta, msg.Data = payloadMetaWithBody(buf)
+		select {
+		case <-m.stop:
+			return
+		case m.data <- &msg:
 		}
+	}
 
-		m.data <- buf
+}
+
+// PluginRead reads message from this plugin
+func (m *Middleware) PluginRead() (msg *Message, err error) {
+	select {
+	case <-m.stop:
+		return nil, ErrorStopped
+	case msg = <-m.data:
 	}
 
 	return
 }
 
-func (m *Middleware) Read(data []byte) (int, error) {
-	buf := <-m.data
-	copy(data, buf)
-
-	return len(buf), nil
-}
-
 func (m *Middleware) String() string {
-	return fmt.Sprintf("Modifying traffic using '%s' command", m.command)
+	return fmt.Sprintf("Modifying traffic using %q command", m.command)
+}
+
+func (m *Middleware) isClosed() bool {
+	m.mu.RLock()
+	defer m.mu.RUnlock()
+	return m.closed
+}
+
+// Close closes this plugin
+func (m *Middleware) Close() error {
+	if m.isClosed() {
+		return nil
+	}
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	m.commandCancel()
+	close(m.stop)
+	m.closed = true
+	return nil
 }

middleware/README.md

@@ -0,0 +1,122 @@
+# GoReplay middleware
+
+GoReplay support protocol for writing middleware in any language, which allows you to implement custom logic like authentification or complex rewriting and filterting. See protocol description here: https://github.com/buger/goreplay/wiki/Middleware, but the basic idea that middleware process receive hex encoded data via STDIN and emits it back via STDOUT. STDERR for loggin inside middleware. Yes, that's simple.
+
+To simplify middleware creation we provide packages for NodeJS and Go (upcoming).
+
+If you want to get access to original and replayed responses, do not forget adding `--output-http-track-response` and `--input-raw-track-response` options.
+
+## NodeJS
+
+Before starting, you should install the package via npm: `npm install goreplay_middleware`.
+And initialize middleware the following way:
+```javascript
+var gor = require("goreplay_middleware");
+// `init` will initialize STDIN listener
+gor.init();
+```
+
+Basic idea is that you write callbacks which respond to `request`, `response`, `replay`, or `message` events, which contain request meta information and actuall http paylod. Depending on your needs you may compare, override or filter incoming requests and responses.
+
+You can respond to the incoming events using `on` function, by providing callbacks:
+```javascript
+// valid events are `request`, `response` (original response), `replay` (replayed response), and `message` (all events)
+gor.on('request', function(data) {
+    // `data` contains incoming message its meta information.
+    data
+
+    // Raw HTTP payload of `Buffer` type
+    // Example (hidden character for line endings shown on purpose):
+    //   GET / HTTP/1.1\r\n
+    //   User-Agent: Golang\r\n
+    //   \r\n
+    data.http
+
+    // Meta is an array size of 4, containing:
+    //   1. request type - 1, 2 or 3 (which maps to `request`, `respose` and `replay`)
+    //   2. uuid - request unique identifier. Request responses have the same ID as their request.
+    //   3. timestamp of when request was made (for responses it is time of request start too)
+    //   4. latency - time difference between request start and finish. For `request` is zero.
+    data.meta
+
+    // Unique request ID. It should be same for `request`, `response` and `replay` events of the same request.
+    data.ID
+
+    // You should return data at the end of function, even if you not changed request, if you do not want to filter it out.
+    // If you just `return` nothing, request will be filtered
+    return data
+})
+```
+### Mapping requests and responses
+You can provide request ID as additional argument to `on` function, which allow you to map related requests and responses. Below is example of middleware which checks that original and replayed response have same HTTP status code.
+
+```javascript
+// Example of very basic way to compare if replayed traffic have no errors
+gor.on("request", function(req) {
+    gor.on("response", req.ID, function(resp) {
+        gor.on("replay", req.ID, function(repl) {
+            if (gor.httpStatus(resp.http) != gor.httpStatus(repl.http)) {
+                // Note that STDERR is used for logging, and it actually will be send to `Gor` STDOUT.
+                // This trick is used because SDTIN and STDOUT already used for process communication.
+                // You can write logger that writes to files insead.
+                console.error(`${gor.httpPath(req.http)} STATUS NOT MATCH: 'Expected ${gor.httpStatus(resp.http)}' got '${gor.httpStatus(repl.http)}'`)
+            }
+            return repl;
+        })
+        return resp;
+    })
+    return req
+})
+```
+
+This middleware includes `searchResponses` helper which is used to compare value of original response with the replayed response. If authentication system or xsrf protection returns unique tokens in headers or the response, it will be helpful to rewrite your requests based on them. Because tokens are unique, and the value contained in original and replayed responses will be different. So, you need to extract value from both responses, and rewrite requests based on those mappings.
+
+`searchResponses` accepts request id, regexp pattern for searching the compared value (should include capture group), and callback which returns both original and replayed matched value.
+
+Example: 
+```javascript
+   // Compare HTTP headers for response and replayed response, and map values
+let tokMap = {};
+
+gor.on("request", function(req) {
+    let tok = gor.httpHeader(req.http, "Auth-Token");
+    if (tok && tokMap[tok]) {
+        req.http = gor.setHttpHeader(req.http, "Auth-Token", tokMap[tok]) 
+    }
+    
+    gor.searchResponses(req.ID, "X-Set-Token: (\w+)$", function(respTok, replTok) {
+        if (respTok && replTok) tokMap[respTok] = replTok;
+    })
+
+    return req;
+})
+```
+
+
+### API documentation
+
+Package expose following functions to process raw HTTP payloads:
+* `init` - initialize middleware object, start reading from STDIN.
+* `httpPath` - URL path of the request: `gor.httpPath(req.http)`
+* `httpMethod` - Http method: 'GET', 'POST', etc. `gor.httpMethod(req.http)`. 
+* `setHttpPath` - update URL path: `req.http = gor.setHttpPath(req.http, newPath)`
+* `httpPathParam` - get param from URL path: `gor.httpPathParam(req.http, queryParam)`
+* `setHttpPathParam` - set URL param: `req.http = gor.setHttpPathParam(req.http, queryParam, value)` 
+* `httpStatus` - response status code
+* `httpHeaders` - get all headers: `gor.httpHeaders(req.http)`
+* `httpHeader` - get HTTP header: `gor.httpHeader(req.http, "Content-Length")`
+* `setHttpHeader` - Set HTTP header, returns modified payload: `req.http = gor.setHttpHeader(req.http, "X-Replayed", "1")`
+* `httpBody` - get HTTP Body: `gor.httpBody(req.http)`
+* `setHttpBody` - Set HTTP Body and ensures that `Content-Length` header have proper value. Returns modified payload: `req.http = gor.setHttpBody(req.http, Buffer.from('hello!'))`.
+* `httpBodyParam` - get POST body param: `gor.httpBodyParam(req.http, param)`
+* `setHttpBodyParam` - set POST body param: `req.http = gor.setHttpBodyParam(req.http, param, value)`
+* `httpCookie` - get HTTP cookie: `gor.httpCookie(req.http, "SESSSION_ID")`
+* `setHttpCookie` - set HTTP cookie, returns modified payload: `req.http = gor.setHttpCookie(req.http, "iam", "cuckoo")`
+* `deleteHttpCookie` - delete HTTP cookie, returns modified payload: `req.http = gor.deleteHttpCookie(req.http, "iam")`
+
+Also it is totally legit to use standard `Buffer` functions like `indexOf` for processing the HTTP payload. Just do not forget that if you modify the body, update the `Content-Length` header with a new value. And if you modify any of the headers, line endings should be `\r\n`. Rest is up to your imagination.
+
+
+## Support
+
+Feel free to ask questions here and by sending email to [support@goreplay.org](mailto:support@goreplay.org). Commercial support is available and welcomed 🙈.

middleware/middleware.js

@@ -0,0 +1,790 @@
+// ======= GoReplay Middleware helper =============
+// Created by Leonid Bugaev in 2017
+//
+// For questions use GitHub or support@goreplay.org
+//
+// GoReplay: https://github.com/buger/goreplay
+// Middleware package: https://github.com/buger/goreplay/middleware
+
+var middleware;
+
+function init() {
+    var proxy = {
+        ch: {},
+        on: function(chan, id, cb) {
+            if (!cb && id) {
+                cb = id;
+            } else if (cb && id) {
+                chan = chan + "#" + id;
+            }
+
+            if (!proxy.ch[chan]) {
+                proxy.ch[chan] = [];
+            }
+
+            proxy.ch[chan].push({
+                created: new Date(),
+                cb: cb
+            });
+
+            return proxy;
+        },
+
+        emit: function(msg, raw) {
+            var chanPrefix;
+
+            switch(msg.type) {
+                case "1": chanPrefix = "request"; break;
+                case "2": chanPrefix = "response"; break;
+                case "3": chanPrefix = "replay"; break;
+            }
+
+            let resp = msg;
+
+            ["message", chanPrefix, chanPrefix + "#" + msg.ID].forEach(function(chanID, idx){
+                if (proxy.ch[chanID]) {
+                    proxy.ch[chanID].forEach(function(ch){
+                        let r = ch.cb(msg);
+                        if (resp) resp = r; // If one of callback decided not to send response back, do not override it in global callbacks
+                    })
+                    
+                    // Cleanup Individual message channels to avoid memory leaks
+                    if (idx == 2) {
+                        delete proxy.ch[chanID]
+                    }
+                }
+            })
+
+            if (resp) {
+              process.stdout.write(`${resp.rawMeta.toString('hex')}${Buffer.from("\n").toString("hex")}${resp.http.toString('hex')}\n`)
+            }
+
+            return resp
+        }
+    }
+
+    // Clean up old messaged ID specific channels if they are older then 60s
+    let gc = function(gcTime){
+        let now = new Date();
+        for (k in proxy.ch) {
+            if (k.indexOf("#") == -1) continue;
+
+            proxy.ch[k] = proxy.ch[k].filter(function(ch){
+                return (now - ch.created) < gcTime
+            })
+
+            if (proxy.ch[k].length == 0) {
+                delete proxy.ch[k]
+            }
+        }
+    }
+    proxy.gc = gc
+
+    setInterval(function(){
+        gc(10 * 1000)
+    }, 1000);
+
+    const readline = require('readline');
+    const rl = readline.createInterface({
+          input: process.stdin
+    });
+
+    rl.on('line', function(line) {
+        let msg = parseMessage(line)
+        if (msg) {
+            proxy.emit(msg, line)
+        }
+    });
+
+    middleware = proxy;
+
+    return proxy;
+}
+
+
+function parseMessage(msg) {
+    try {
+        let payload = Buffer.from(msg, "hex");
+        let metaPos = payload.indexOf("\n");
+        let meta = payload.slice(0, metaPos);
+        let metaArr = meta.toString("ascii").split(" ");
+        let pType = metaArr[0];
+        let pID = metaArr[1];
+        let raw = payload.slice(metaPos + 1, payload.length);
+
+        return {
+            type: pType,
+            ID: pID,
+            rawMeta: meta,
+            meta: metaArr,
+            http: raw
+        }
+    } catch(e) {
+        fail(`Error while parsing incoming request: ${msg}`)
+    }
+}
+
+// Used to compare values from original and replayed responses
+// Accepts request id, regexp pattern for searching the compared value (should include capture group), and callback which returns both original and replayed matched value.
+// Example: 
+//
+//   // Compare HTTP headers for response and replayed response, and map values
+//   let tokMap = {};
+//
+//   gor.on("request", function(req) {
+//     let tok = gor.httpHeader(req.http, "Auth-Token");
+//     if (tok && tokMap[tok]) {
+//       req.http = gor.setHttpHeader(req.http, "Auth-Token", tokMap[tok]) 
+//     }
+//
+//     gor.searchResponses(req.ID, "X-Set-Token: (\w+)$", function(respTok, replTok) {
+//       tokMap[respTok] = replTok;
+//     })
+//
+//     return req;
+//   })
+//
+function searchResponses(id, searchPattern, callback) {
+    let re = new RegExp(searchPattern);
+
+    // Using regexp require converting buffer to string
+    // Before converting to string we can use initial `Buffer.indexOf` check
+    let indexPattern = searchPattern.split("(")[0];
+
+    if (!indexPattern) {
+        console.error("Search regexp should include capture group, pointing to the value: `prefix-(.*)`")
+        return
+    }
+
+    middleware.on("response", id, function(resp){
+        if (resp.http.indexOf(indexPattern) == -1) {
+            callback()
+            return resp
+        }
+
+        let respMatch = resp.http.toString('utf-8').match(re);
+        if (!respMatch) {
+            callback()
+            return resp
+        }
+
+        middleware.on("replay", id, function(repl) {
+            if (repl.http.indexOf(indexPattern) == -1) {
+                callback(respMatch[1]);
+                return repl;
+            }
+
+            let replMatch = repl.http.toString('utf-8').match(re);
+
+            if (!replMatch) {
+                callback(respMatch[1]);
+                return repl;
+            }
+        
+            callback(respMatch[1], replMatch[1]);
+            
+            return repl;
+        })
+
+        return resp;
+    })
+}
+
+
+// =========== HTTP parsing =================
+
+// Example HTTP payload record (including hidden characters):
+//
+//  POST / HTTP/1.1\r\n
+//  User-Agent: Node\r\n
+//  Content-Length: 5\r\n
+//  \r\n
+//  hello
+
+function httpMethod(payload) {
+    var pEnd = payload.indexOf(' ');
+    return payload.slice(0, pEnd).toString("ascii");
+}
+
+function httpPath(payload) {
+    var pStart = payload.indexOf(' ') + 1;
+    var pEnd = payload.indexOf(' ', pStart);
+    return payload.slice(pStart, pEnd).toString("ascii");
+}
+
+function setHttpPath(payload, newPath) {
+    var pStart = payload.indexOf(' ') + 1;
+    var pEnd = payload.indexOf(' ', pStart);
+
+    return Buffer.concat([payload.slice(0, pStart), Buffer.from(newPath), payload.slice(pEnd, payload.length)])
+}
+
+function httpPathParam(payload, name) {
+    let path = httpPath(payload);
+    let re = new RegExp(name + "=([^&$]+)");
+    let match = path.match(re);
+
+    if (match) return decodeURI(match[1]);
+}
+
+function setHttpPathParam(payload, name, value) {
+    let path = httpPath(payload);
+    let re = new RegExp(name + "=([^&$]+)");
+    let newPath = path.replace(re, name + "=" + encodeURI(value));
+    
+    // If we should add new param instead
+    if (newPath == path) {
+        if (newPath.indexOf("?") == -1) {
+            newPath += "?"
+        } else {
+            newPath += "&"
+        }
+
+        newPath += name + "=" + encodeURI(value);
+    }
+
+    return setHttpPath(payload, newPath)
+}
+
+// HTTP response have status code in same position as `path` for requests
+function httpStatus(payload) {
+    return httpPath(payload);
+}
+
+function setHttpStatus(payload, newStatus) {
+    return setHttpPath(payload, newStatus);
+}
+
+function httpHeaders(payload) {
+    var httpHeaderString = payload.slice(0,payload.indexOf("\r\n\r\n") + 4).toString().split("\n").slice(1);
+    var headers = {};
+
+    for (var item in httpHeaderString) {
+        var parts = httpHeaderString[item].split(":");
+
+        if (parts.length > 1) {
+            headers[parts[0]] = parts.slice(1).join(":").trim();    
+        }
+    }
+
+    return headers;
+}
+
+function httpHeader(payload, name) {
+    var currentLine = 0;
+    var i = 0;
+    var header = { start: -1, end: -1, valueStart: -1 }
+    var nameBuf = Buffer.from(name);
+    var nameBufLower = Buffer.from(name.toLowerCase());
+
+    while(c = payload[i]) {
+        if (c == 13) { // new line "\n"
+            currentLine++;
+            i++
+            header.end = i
+
+            if (currentLine > 0 && header.start > 0 && header.valueStart > 0) {
+                if (nameBuf.compare(payload, header.start, header.valueStart - 1) == 0 ||
+                    nameBufLower.compare(payload, header.start, header.valueStart - 1) == 0) { // ensure that headers are not case sensitive
+                    header.value = payload.slice(header.valueStart, header.end - 1).toString("utf-8").trim();
+                    header.name = payload.slice(header.start, header.valueStart - 1).toString("utf-8");
+                    return header
+                }
+            }
+
+            header.start = -1
+            header.valueStart = -1
+            continue;
+        } else if (c == 10) { // "\r"
+            i++
+            continue;
+        } else if (c == 58) { // ":" Header/value separator symbol
+            if (header.valueStart == -1) {
+                header.valueStart = i + 1;
+                i++
+                continue;
+            }
+        }
+
+        if (header.start == -1) header.start = i;
+
+        i++
+    }
+
+    return
+}
+
+function setHttpHeader(payload, name, value) {
+    let header = httpHeader(payload, name);
+    if (!header) {
+        let headerStart = payload.indexOf(13) + 1;
+        return Buffer.concat([payload.slice(0, headerStart + 1), Buffer.from(name + ": " + value + "\r\n"), payload.slice(headerStart + 1, payload.length)])
+    } else {
+        return Buffer.concat([payload.slice(0, header.valueStart), Buffer.from(" " + value + "\r\n"), payload.slice(header.end + 1, payload.length)])
+    }
+}
+
+function deleteHttpHeader(payload, name) {
+	let header = httpHeader(payload, name);
+
+    if (header) {
+        return Buffer.concat([payload.slice(0, header.start), payload.slice(header.end+1, payload.length)])
+    }
+
+	return payload
+}
+
+function httpBody(payload) {
+    let bodyIndex = payload.indexOf("\r\n\r\n");
+    if (-1 != bodyIndex){
+        return payload.slice(bodyIndex + 4, payload.length);   
+    } else {
+        return null;
+    }
+}
+
+function setHttpBody(payload, newBody) {
+    let p = setHttpHeader(payload, "Content-Length", newBody.length)
+    let headerEnd = p.indexOf("\r\n\r\n") + 4;
+    return Buffer.concat([p.slice(0, headerEnd), newBody])
+}
+
+function httpBodyParam(payload, name) {
+    let body = httpBody(payload);
+    let re = new RegExp(name + "=([^&$]+)");
+    if (body.indexOf(name + "=") != -1) {
+        let param = body.toString('utf-8').match(re);
+        if (param) {
+            return decodeURI(param[1]);
+        }
+    }
+}
+
+function setHttpBodyParam(payload, name, value) {
+    let body = httpBody(payload);
+    let re = new RegExp(name + "=([^&$]+)");
+
+    let newBody = body.toString('utf-8');
+
+    if (newBody.indexOf(name + "=") != -1 ) {
+        newBody = newBody.replace(re, name + "=" + encodeURI(value));
+    } else {
+        if (newBody.indexOf("=") != -1) {
+            newBody += "&";
+        }
+        newBody += name + "=" + value;
+    }
+    
+    return setHttpBody(payload, Buffer.from(newBody));
+}
+
+function setHttpCookie(payload, name, value) {
+    let h = httpHeader(payload, "Cookie");
+    let cookie = h ? h.value : "";
+    let cookies = cookie.split("; ").filter(function(v){ return v.indexOf(name + "=") != 0 })
+    cookies.push(name + "=" + value)
+    return setHttpHeader(payload, "Cookie", cookies.join("; "))
+}
+
+function deleteHttpCookie(payload, name) {
+    let h = httpHeader(payload, "Cookie");
+    let cookie = h ? h.value : "";
+    let cookies = cookie.split("; ").filter(function(v){ return v.indexOf(name + "=") != 0 })
+    return setHttpHeader(payload, "Cookie", cookies.join("; "))
+}
+
+function httpCookie(payload, name) {
+    let h = httpHeader(payload, "Cookie");
+    let cookie = h ? h.value : "";
+    let value;
+    let cookies = cookie.split("; ").forEach(function(v){
+        if (v.indexOf(name + "=") == 0) {
+            value = v.substr(name.length + 1);
+        }
+    })
+    return value;
+}
+
+module.exports = {
+    init: init,
+    on: function(){ return middleware.on.apply(this, arguments) },
+    parseMessage: parseMessage,
+    searchResponses: searchResponses,
+    httpPath: httpPath,
+    httpMethod: httpMethod,
+    setHttpPath: setHttpPath,
+    httpPathParam: httpPathParam,
+    setHttpPathParam: setHttpPathParam,
+    httpStatus: httpStatus,
+    setHttpStatus: setHttpStatus,
+    httpHeader: httpHeader,
+    setHttpHeader: setHttpHeader,
+    deleteHttpHeader: deleteHttpHeader,
+    httpBody: httpBody,
+    setHttpBody: setHttpBody,
+    httpBodyParam: httpBodyParam,
+    setHttpBodyParam: setHttpBodyParam,
+    httpCookie: httpCookie,
+    setHttpCookie: setHttpCookie,
+    deleteHttpCookie: deleteHttpCookie,
+    test: testRunner,
+    benchmark: testBenchmark,
+    httpHeaders: httpHeaders
+}
+
+
+// =========== Tests ==============
+
+function testRunner(){
+    ["init", "filter", "parseMessage", "httpMethod", "httpPath", "setHttpHeader", "deleteHttpHeader", "httpPathParam", "httpHeader", "httpBody", "setHttpBody", "httpBodyParam", "httpCookie", "setHttpCookie", "deleteHttpCookie", "httpHeaders"].forEach(function(t){
+        console.log(`====== Start ${t} =======`)
+        eval(`TEST_${t}()`)
+        console.log(`====== End ${t} =======`)
+    })
+}
+
+function testBenchmark(){
+    const child_process = require('child_process');
+
+    let gor = init();
+    gor.on("message", function(){
+    });
+
+    gor.on("request", function(){
+    });
+
+    for (var i = 0; i<256; i++) {
+        let req = parseMessage(Buffer.from("1 2 3\nGET / HTTP/1.1\r\n\r\n").toString('hex'));
+        req.ID = +Date.now()
+        gor.emit(req);
+
+        gor.on("request", req.ID+"", function(){
+            gor.on("response", req.ID+"", function(){
+            })
+        })
+
+        if ( i % 3 == 0 ) {
+            let resp = parseMessage(Buffer.from("2 2 3\nHTTP/1.1 200 OK\r\n\r\n").toString('hex'));
+            resp.ID = req.ID
+            gor.emit(resp);
+        }
+    }
+    
+    child_process.execSync("sleep 0.01");
+
+    gor.gc(1) 
+    
+    fail(JSON.stringify(gor.ch))
+}
+
+// Just print in red color
+function fail(message) {
+    console.error("\x1b[31m[MIDDLEWARE] %s\x1b[0m", message)
+}
+
+function log(message) {
+    console.error(message)
+}
+
+function TEST_init() {
+    const child_process = require('child_process');
+
+    let received = 0;
+    let gor = init();
+    gor.on("message", function(){
+        received++; // should be called 3 times for for every request
+    });
+
+    gor.on("request", function(){
+        received++; // should be called 1 time only for request
+    });
+
+    gor.on("response", "2", function(){
+        received++; // should be called 1 time only for specific response
+    })
+
+    if (Object.keys(gor.ch).length != 3) {
+        return fail("Should create 3 channels");
+    }
+
+    let req = parseMessage(Buffer.from("1 2 3\nGET / HTTP/1.1\r\n\r\n").toString('hex'));
+    let resp = parseMessage(Buffer.from("2 2 3\nHTTP/1.1 200 OK\r\n\r\n").toString('hex'));
+    let resp2 = parseMessage(Buffer.from("2 3 3\nHTTP/1.1 200 OK\r\n\r\n").toString('hex'));
+    
+    gor.emit(req);
+    gor.emit(resp);
+    gor.emit(resp2);
+
+    child_process.execSync("sleep 0.01");
+
+    if (received != 5) {
+        fail(`Should receive 5 messages: ${received}`);
+    }
+}
+
+function TEST_filter() {
+    const child_process = require('child_process');
+
+    let gor = init();
+    gor.on("request", function(req){
+        if (httpPath(req.http) != "/filter") {
+            return req
+        }
+    });
+
+    gor.on("request", function(req){
+        return req
+    });
+
+
+    let reqPass = parseMessage(Buffer.from("1 2 3\nGET / HTTP/1.1\r\n\r\n").toString('hex'));
+    let reqFilter = parseMessage(Buffer.from("1 2 3\nGET /filter HTTP/1.1\r\n\r\n").toString('hex'));
+    
+    if (!gor.emit(reqPass)) {
+        return fail("Should not filter request")
+    }
+
+    if (gor.emit(reqFilter)) {
+        return fail("Should filter request even if one middleware rejected it")
+    }
+
+}
+
+function TEST_parseMessage() {
+    const exampleMessage = Buffer.from("1 2 3\nGET / HTTP/1.1\r\n\r\n").toString('hex')
+    let msg = parseMessage(exampleMessage)
+    let expected = { type: '1', ID: '2', meta: ["1", "2", "3"], http: Buffer.from("GET / HTTP/1.1\r\n\r\n") }
+
+    Object.keys(expected).forEach(function(k){
+        if (msg[k].toString() != expected[k].toString()) {
+            fail(`${k}: '${expected[k]}' != '${msg[k]}'`)
+        }
+    })
+}
+
+function TEST_httpPath() {
+    const examplePayload = "GET /test HTTP/1.1\r\n\r\n";
+
+    let payload = Buffer.from(examplePayload);
+    let path = httpPath(payload);
+
+    if (path != "/test") {
+        return fail(`Path '${patj}' != '/test'`)
+    }
+
+    let newPayload = setHttpPath(payload, '/')
+    if (newPayload.toString() != "GET / HTTP/1.1\r\n\r\n") {
+        return fail(`Malformed payload '${newPayload}'`)
+    }
+
+    newPayload = setHttpPath(payload, '/bigger')
+    if (newPayload.toString() != "GET /bigger HTTP/1.1\r\n\r\n") {
+        return fail(`Malformed payload '${newPayload}'`)
+    }
+}
+
+function TEST_httpMethod() {
+    const examplePayload = "GET /test HTTP/1.1\r\n\r\n";
+
+    let payload = Buffer.from(examplePayload);
+    let method = httpMethod(payload);
+
+    if (method != "GET") {
+        return fail(`Path '${method}' != 'GET'`)
+    }
+}
+
+
+function TEST_httpPathParam() {
+    let p = Buffer.from("GET / HTTP/1.1\r\n\r\n");
+
+    if (httpPathParam(p, "test")) {
+        return fail("Should not found param")
+    }
+
+    p = setHttpPathParam(p, "test", "123");
+    if (httpPath(p) != "/?test=123") {
+        return fail("Should set first param: " + httpPath(p));
+    }
+
+    if (httpPathParam(p, "test") != "123") {
+        return fail("Should get first param: " + httpPathParam(p, "test"));
+    }
+
+    p = setHttpPathParam(p, "qwer", "ty");
+    if (httpPath(p) != "/?test=123&qwer=ty") {
+        return fail("Should set second param: " + httpPath(p));
+    }
+
+    p = setHttpPathParam(p, "test", "4321");
+    if (httpPath(p) != "/?test=4321&qwer=ty") {
+        return fail("Should update first param: " + httpPath(p));
+    }
+
+    if (httpPathParam(p, "test") != "4321") {
+        return fail("Should update first param: " + httpPath(p));
+    }
+}
+
+function TEST_httpBodyParam() {
+    let p = Buffer.from("POST / HTTP/1.1\r\n\r\n");
+
+    if (httpBodyParam(p, "test")) {
+        return fail("Should not found param")
+    }
+
+    p = setHttpBodyParam(p, "test", "123");
+    if (httpBody(p).toString() != "test=123") {
+        return fail("Should set first param: " + httpBody(p).toString());
+    }
+
+    if (httpBodyParam(p, "test") != "123") {
+        return fail("Should get first param: " + httpBodyParam(p, "test"));
+    }
+
+    p = setHttpBodyParam(p, "qwer", "ty");
+    if (httpBody(p).toString() != "test=123&qwer=ty") {
+        return fail("Should set second param: " + httpBody(p).toString());
+    }
+
+    p = setHttpBodyParam(p, "test", "4321");
+    if (httpBody(p).toString() != "test=4321&qwer=ty") {
+        return fail("Should update first param: " + httpBody(p).toString());
+    }
+
+    if (httpBodyParam(p, "test") != "4321") {
+        return fail("Should update first param: " + httpBody(p).toString());
+    }
+}
+
+function TEST_httpHeader() {
+    const examplePayload = "GET / HTTP/1.1\r\nHost: localhost:3000\r\nUser-Agent: Node\r\nContent-Length:5\r\n\r\nhello";
+
+    let expected = {"Host": "localhost:3000", "User-Agent": "Node", "Content-Length": "5"}
+
+    Object.keys(expected).forEach(function(name){
+        let payload = Buffer.from(examplePayload);
+        let header = httpHeader(payload, name);
+        if (!header) {
+            fail(`Header not found. Was looking for: ${name}`)
+        }
+        if (header && header.value != expected[name]) {
+            fail(`${name}: '${expected[name]}' != '${header.value}'`)
+        }
+    })
+}
+
+
+function TEST_setHttpHeader() {
+    const examplePayload = "GET / HTTP/1.1\r\nUser-Agent: Node\r\nContent-Length: 5\r\n\r\nhello";
+
+    // Modify existing header
+    ["", "1", "Long test header"].forEach(function(ua){
+        let expected = `GET / HTTP/1.1\r\nUser-Agent: ${ua}\r\nContent-Length: 5\r\n\r\nhello`;
+        let p = Buffer.from(examplePayload);
+        p = setHttpHeader(p, "User-Agent", ua);
+        if (p != expected) {
+            console.error(`setHeader failed, expected User-Agent value: ${ua}.\n${p}`)
+        }
+    })
+
+    // Adding new header
+    let expected = `GET / HTTP/1.1\r\nX-Test: test\r\nUser-Agent: Node\r\nContent-Length: 5\r\n\r\nhello`;
+    let p = Buffer.from(examplePayload);
+    p = setHttpHeader(p, "X-Test", "test");
+    if (p != expected) {
+        console.error(`setHeader failed, expected new header 'X-Test' header: ${p}`)
+    }
+}
+
+function TEST_deleteHttpHeader() {
+    const examplePayload = "GET / HTTP/1.1\r\nUser-Agent: Node\r\nContent-Length: 5\r\n\r\nhello";
+
+    // Adding new header
+    let expected = `GET / HTTP/1.1\r\nContent-Length: 5\r\n\r\nhello`;
+    let p = Buffer.from(examplePayload);
+    p = deleteHttpHeader(p, "User-Agent", "test");
+    if (p != expected) {
+        console.error(`setHeader failed, expected delete header 'User-Agent' header: ${p}`)
+    }
+}
+
+function TEST_httpBody() {
+    const examplePayload = "GET / HTTP/1.1\r\nUser-Agent: Node\r\nContent-Length: 5\r\n\r\nhello";
+    let body = httpBody(Buffer.from(examplePayload));
+    if (body != "hello") {
+        fail(`'${body}' != 'hello'`)
+    }
+
+    const exampleInvalidPayload = "Invalid HTTP Response by Network issue";
+    let invalidBody = httpBody(Buffer.from(exampleInvalidPayload));
+    if (invalidBody != null) {
+        fail(`'${invalidBody}' != 'null'`)
+    }
+}
+
+function TEST_setHttpBody() {
+    const examplePayload = "GET / HTTP/1.1\r\nUser-Agent: Node\r\nContent-Length: 5\r\n\r\nhello";
+    let p = setHttpBody(Buffer.from(examplePayload), Buffer.from("hello, world!"));
+
+    if (p != "GET / HTTP/1.1\r\nUser-Agent: Node\r\nContent-Length: 13\r\n\r\nhello, world!") {
+        fail(`Wrong body: '${p}'`)
+    }
+}
+
+function TEST_httpCookie() {
+    const examplePayload = "GET / HTTP/1.1\r\nCookie: a=b; test=zxc\r\n\r\n";
+    let c = httpCookie(Buffer.from(examplePayload), "test");
+    if (c != "zxc") {
+        return fail(`Should get cookie: ${c}`);
+    }
+
+    c = httpCookie(Buffer.from(examplePayload), "nope");
+    if (c != null) {
+        return fail(`Should not find cookie: ${c}`);
+    }
+}
+
+function TEST_setHttpCookie() {
+    const examplePayload = "GET / HTTP/1.1\r\nCookie: a=b; test=zxc\r\n\r\n";
+    let p = setHttpCookie(Buffer.from(examplePayload), "test", "1");
+    if (p != "GET / HTTP/1.1\r\nCookie: a=b; test=1\r\n\r\n") {
+        return fail(`Should update cookie: ${p}`)
+    }
+
+    p = setHttpCookie(Buffer.from(examplePayload), "new", "one");
+    if (p != "GET / HTTP/1.1\r\nCookie: a=b; test=zxc; new=one\r\n\r\n") {
+        return fail(`Should add new cookie: ${p}`)
+    }
+}
+
+function TEST_deleteHttpCookie() {
+    const examplePayload = "GET / HTTP/1.1\r\nCookie: a=b; test=zxc\r\n\r\n";
+    let p = deleteHttpCookie(Buffer.from(examplePayload), "a");
+    if (p != "GET / HTTP/1.1\r\nCookie: test=zxc\r\n\r\n") {
+        return fail(`Should delete cookie: ${p}`)
+    }
+}
+
+
+function TEST_httpHeaders() {
+    const examplePayload = "GET / HTTP/1.1\r\nHost: localhost:3000\r\nUser-Agent: Node\r\nContent-Length:5\r\n\r\nhello";
+
+    let expectedHeaders = {"Host": "localhost:3000", "User-Agent": "Node", "Content-Length": "5"}
+    let payload = Buffer.from(examplePayload);
+    let headers = httpHeaders(payload);
+
+    ["Host", "User-Agent", "Content-Length"].forEach(function(header){
+        let actual = headers[header];
+        let expected = expectedHeaders[header];
+
+        if (!actual) {
+            fail(`${header} Header was not found`);
+        }
+
+        if (actual != expected) {
+            fail(`${header} Header not Equal to Expected: ${expected} was ${actual}`);
+        }
+
+    })
+}

middleware/package.json

@@ -0,0 +1,16 @@
+{
+  "name": "goreplay_middleware",
+  "version": "1.0.0",
+  "description": "Package for writing middleware for GoReplay https://goreplay.org",
+  "main": "middleware.js",
+  "scripts": {
+    "test": "node -e \"var gor = require('./middleware.js'); gor.test(); process.exit()\"",
+    "benchmark": "node -e \"var gor = require('./middleware.js'); gor.benchmark(); process.exit()\""
+  },
+  "keywords": [
+    "middleware",
+    "goreplay"
+  ],
+  "author": "Leonid Bugaev",
+  "license": "LGPL-3.0"
+}

middleware_test.go

@@ -2,219 +2,153 @@ package main
 
 import (
 	"bytes"
-	"crypto/rand"
-	"encoding/hex"
-	"github.com/buger/gor/proto"
-	"io"
-	"net/http"
-	"net/http/httptest"
+	"context"
+	"os"
+	"os/exec"
 	"strings"
-	"sync"
+	"sync/atomic"
+	"syscall"
 	"testing"
-	"time"
+
+	"github.com/buger/goreplay/proto"
 )
 
-type fakeServiceCb func(string, int, []byte)
+const echoSh = "./examples/middleware/echo.sh"
+const tokenModifier = "go run ./examples/middleware/token_modifier.go"
 
-// Simple service that generate token on request, and require this token for accesing to secure area
-func NewFakeSecureService(wg *sync.WaitGroup, cb fakeServiceCb) *httptest.Server {
-	active_tokens := make([]string, 0)
-	var mu sync.Mutex
+var noDebug = append(syscall.Environ(), "GOR_TEST=1")
 
-	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
-		mu.Lock()
-		defer mu.Unlock()
-		Debug("Received request: " + req.URL.String())
+func initMiddleware(cmd *exec.Cmd, cancl context.CancelFunc, l PluginReader, c func(error)) *Middleware {
+	var m Middleware
+	m.data = make(chan *Message, 1000)
+	m.stop = make(chan bool)
+	m.commandCancel = cancl
+	m.Stdout, _ = cmd.StdoutPipe()
+	m.Stdin, _ = cmd.StdinPipe()
+	cmd.Stderr = os.Stderr
+	go m.read(m.Stdout)
+	go func() {
+		defer m.Close()
+		var err error
+		if err = cmd.Start(); err == nil {
+			err = cmd.Wait()
+		}
+		if err != nil {
+			c(err)
+		}
+	}()
+	m.ReadFrom(l)
+	return &m
+}
 
-		switch req.URL.Path {
-		case "/token":
-			// Generate random token
-			token_length := 10
-			buf := make([]byte, token_length)
-			rand.Read(buf)
-			token := hex.EncodeToString(buf)
-			active_tokens = append(active_tokens, token)
+func initCmd(command string, env []string) (*exec.Cmd, context.CancelFunc) {
+	commands := strings.Split(command, " ")
+	ctx, cancl := context.WithCancel(context.Background())
+	cmd := exec.CommandContext(ctx, commands[0], commands[1:]...)
+	cmd.Env = env
+	return cmd, cancl
+}
 
-			w.Write([]byte(token))
-
-			cb(req.URL.Path, 200, []byte(token))
-		case "/secure":
-			token := req.URL.Query().Get("token")
-			token_found := false
-
-			for _, t := range active_tokens {
-				if t == token {
-					token_found = true
-					break
+func TestMiddlewareEarlyClose(t *testing.T) {
+	quit := make(chan struct{})
+	in := NewTestInput()
+	cmd, cancl := initCmd(echoSh, noDebug)
+	midd := initMiddleware(cmd, cancl, in, func(err error) {
+		if err != nil {
+			if e, ok := err.(*exec.ExitError); ok {
+				status := e.Sys().(syscall.WaitStatus)
+				if status.Signal() != syscall.SIGKILL {
+					t.Errorf("expected error to be signal killed. got %s", status.Signal().String())
 				}
 			}
-
-			if token_found {
-				w.WriteHeader(http.StatusAccepted)
-				cb(req.URL.Path, 202, []byte(nil))
-			} else {
-				w.WriteHeader(http.StatusForbidden)
-				cb(req.URL.Path, 403, []byte(nil))
-			}
 		}
-
-		wg.Done()
-	}))
-
-	return server
-}
-
-func TestFakeSecureService(t *testing.T) {
-	var resp, token []byte
-
-	wg := new(sync.WaitGroup)
-
-	server := NewFakeSecureService(wg, func(path string, status int, resp []byte) {
+		quit <- struct{}{}
 	})
-	defer server.Close()
-
-	wg.Add(3)
-
-	client := NewHTTPClient(server.URL, &HTTPClientConfig{Debug: true})
-	resp, _ = client.Get("/token")
-	token = proto.Body(resp)
-
-	// Right token
-	resp, _ = client.Get("/secure?token=" + string(token))
-	if !bytes.Equal(proto.Status(resp), []byte("202")) {
-		t.Error("Valid token should return status 202:", string(proto.Status(resp)))
+	var body = []byte("OPTIONS / HTTP/1.1\r\nHost: example.org\r\n\r\n")
+	count := uint32(0)
+	out := NewTestOutput(func(msg *Message) {
+		if !bytes.Equal(body, msg.Data) {
+			t.Errorf("expected %q to equal %q", body, msg.Data)
+		}
+		atomic.AddUint32(&count, 1)
+		if atomic.LoadUint32(&count) == 5 {
+			quit <- struct{}{}
+		}
+	})
+	pl := &InOutPlugins{}
+	pl.Inputs = []PluginReader{midd, in}
+	pl.Outputs = []PluginWriter{out}
+	pl.All = []interface{}{midd, out, in}
+	e := NewEmitter()
+	go e.Start(pl, "")
+	for i := 0; i < 5; i++ {
+		in.EmitBytes(body)
 	}
-
-	// Wrong tokens forbidden
-	resp, _ = client.Get("/secure?token=wrong")
-	if !bytes.Equal(proto.Status(resp), []byte("403")) {
-		t.Error("Wrong token should returns status 403:", string(proto.Status(resp)))
-	}
-
-	wg.Wait()
-}
-
-func TestEchoMiddleware(t *testing.T) {
-	wg := new(sync.WaitGroup)
-
-	from := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		w.Header().Set("Env", "prod")
-		w.Header().Set("RequestPath", r.URL.Path)
-		wg.Done()
-	}))
-	defer from.Close()
-
-	to := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		w.Header().Set("Env", "test")
-		w.Header().Set("RequestPath", r.URL.Path)
-		wg.Done()
-	}))
-	defer to.Close()
-
-	quit := make(chan int)
-
-	Settings.middleware = "./examples/middleware/echo.sh"
-
-	// Catch traffic from one service
-	fromAddr := strings.Replace(from.Listener.Addr().String(), "[::]", "127.0.0.1", -1)
-	input := NewRAWInput(fromAddr, EnginePcap, true, testRawExpire, "")
-	defer input.Close()
-
-	// And redirect to another
-	output := NewHTTPOutput(to.URL, &HTTPOutputConfig{Debug: false})
-
-	Plugins.Inputs = []io.Reader{input}
-	Plugins.Outputs = []io.Writer{output}
-
-	// Start Gor
-	go Start(quit)
-
-	// Wait till middleware initialization
-	time.Sleep(100 * time.Millisecond)
-
-	// Should receive 2 requests from original + 2 from replayed
-	client := NewHTTPClient(from.URL, &HTTPClientConfig{Debug: false})
-
-	for i := 0; i < 10; i++ {
-		wg.Add(4)
-		// Request should be echoed
-		client.Get("/a")
-		time.Sleep(5 * time.Millisecond)
-		client.Get("/b")
-		time.Sleep(5 * time.Millisecond)
-	}
-
-	wg.Wait()
-	close(quit)
-	time.Sleep(200 * time.Millisecond)
-
-	Settings.middleware = ""
+	<-quit
+	midd.Close()
+	<-quit
 }
 
 func TestTokenMiddleware(t *testing.T) {
-	var resp, token []byte
-
-	wg := new(sync.WaitGroup)
-
-	from := NewFakeSecureService(wg, func(path string, status int, tok []byte) {
-		time.Sleep(10 * time.Millisecond)
-	})
-	defer from.Close()
-
-	to := NewFakeSecureService(wg, func(path string, status int, tok []byte) {
-		switch path {
-		case "/secure":
-			if status != 202 {
-				t.Error("Server should receive valid rewritten token")
+	quit := make(chan struct{})
+	in := NewTestInput()
+	in.skipHeader = true
+	cmd, cancl := initCmd(tokenModifier, noDebug)
+	midd := initMiddleware(cmd, cancl, in, func(err error) {})
+	req := []byte("1 932079936fa4306fc308d67588178d17d823647c 1439818823587396305 200\nGET /token HTTP/1.1\r\nHost: example.org\r\n\r\n")
+	res := []byte("2 932079936fa4306fc308d67588178d17d823647c 1439818823587396305 200\nHTTP/1.1 200 OK\r\nContent-Length: 10\r\nContent-Type: text/plain; charset=utf-8\r\n\r\n17d823647c")
+	rep := []byte("3 932079936fa4306fc308d67588178d17d823647c 1439818823587396305 200\nHTTP/1.1 200 OK\r\nContent-Length: 15\r\nContent-Type: text/plain; charset=utf-8\r\n\r\n932079936fa4306")
+	count := uint32(0)
+	out := NewTestOutput(func(msg *Message) {
+		if msg.Meta[0] == '1' && !bytes.Equal(payloadID(msg.Meta), payloadID(req)) {
+			token, _, _ := proto.PathParam(msg.Data, []byte("token"))
+			if !bytes.Equal(token, proto.Body(rep)) {
+				t.Error("expected the token to be equal to the replayed responses's token")
 			}
 		}
-
-		time.Sleep(10 * time.Millisecond)
+		atomic.AddUint32(&count, 1)
+		if atomic.LoadUint32(&count) == 2 {
+			quit <- struct{}{}
+		}
 	})
-	defer to.Close()
-
-	quit := make(chan int)
-
-	Settings.middleware = "go run ./examples/middleware/token_modifier.go"
-
-	fromAddr := strings.Replace(from.Listener.Addr().String(), "[::]", "127.0.0.1", -1)
-	// Catch traffic from one service
-	input := NewRAWInput(fromAddr, EnginePcap, true, testRawExpire, "")
-	defer input.Close()
-
-	// And redirect to another
-	output := NewHTTPOutput(to.URL, &HTTPOutputConfig{Debug: true})
-
-	Plugins.Inputs = []io.Reader{input}
-	Plugins.Outputs = []io.Writer{output}
-
-	// Start Gor
-	go Start(quit)
-
-	// Wait for middleware to initialize
-	// Give go compiller time to build programm
-	time.Sleep(500 * time.Millisecond)
-
-	// Should receive 2 requests from original + 2 from replayed
-	wg.Add(4)
-
-	client := NewHTTPClient(from.URL, &HTTPClientConfig{Debug: true})
-
-	// Sending traffic to original service
-	resp, _ = client.Get("/token")
-	token = proto.Body(resp)
-
-	// When delay is too smal, middleware does not always rewrite requests in time
-	// Hopefuly client will have delay more then 100ms :)
-	time.Sleep(100 * time.Millisecond)
-
-	resp, _ = client.Get("/secure?token=" + string(token))
-	if !bytes.Equal(proto.Status(resp), []byte("202")) {
-		t.Error("Valid token should return 202:", proto.Status(resp))
-	}
-
-	wg.Wait()
-	close(quit)
-	time.Sleep(100 * time.Millisecond)
-	Settings.middleware = ""
+	pl := &InOutPlugins{}
+	pl.Inputs = []PluginReader{midd, in}
+	pl.Outputs = []PluginWriter{out}
+	pl.All = []interface{}{midd, out, in}
+	e := NewEmitter()
+	go e.Start(pl, "")
+	in.EmitBytes(req) // emit original request
+	in.EmitBytes(res) // emit its response
+	in.EmitBytes(rep) // emit replayed response
+	// emit the request which should have modified token
+	token := []byte("1 8e091765ae902fef8a2b7d9dd96 14398188235873 100\nGET /?token=17d823647c HTTP/1.1\r\nHost: example.org\r\n\r\n")
+	in.EmitBytes(token)
+	<-quit
+	midd.Close()
+}
+
+func TestMiddlewareWithPrettify(t *testing.T) {
+	Settings.PrettifyHTTP = true
+	quit := make(chan struct{})
+	in := NewTestInput()
+	cmd, cancl := initCmd(echoSh, noDebug)
+	midd := initMiddleware(cmd, cancl, in, func(err error) {})
+	var b1 = []byte("POST / HTTP/1.1\r\nHost: example.org\r\nTransfer-Encoding: chunked\r\n\r\n4\r\nWiki\r\n5\r\npedia\r\nE\r\n in\r\n\r\nchunks.\r\n0\r\n\r\n")
+	var b2 = []byte("POST / HTTP/1.1\r\nHost: example.org\r\nContent-Length: 25\r\n\r\nWikipedia in\r\n\r\nchunks.")
+	out := NewTestOutput(func(msg *Message) {
+		if !bytes.Equal(proto.Body(b2), proto.Body(msg.Data)) {
+			t.Errorf("expected %q body to equal %q body", b2, msg.Data)
+		}
+		quit <- struct{}{}
+	})
+	pl := &InOutPlugins{}
+	pl.Inputs = []PluginReader{midd, in}
+	pl.Outputs = []PluginWriter{out}
+	pl.All = []interface{}{midd, out, in}
+	e := NewEmitter()
+	go e.Start(pl, "")
+	in.EmitBytes(b1)
+	<-quit
+	midd.Close()
+	Settings.PrettifyHTTP = false
 }

output_binary.go

@@ -0,0 +1,174 @@
+package main
+
+import (
+	"sync/atomic"
+	"time"
+
+	"github.com/buger/goreplay/size"
+)
+
+// BinaryOutputConfig struct for holding binary output configuration
+type BinaryOutputConfig struct {
+	Workers        int           `json:"output-binary-workers"`
+	Timeout        time.Duration `json:"output-binary-timeout"`
+	BufferSize     size.Size     `json:"output-tcp-response-buffer"`
+	Debug          bool          `json:"output-binary-debug"`
+	TrackResponses bool          `json:"output-binary-track-response"`
+}
+
+// BinaryOutput plugin manage pool of workers which send request to replayed server
+// By default workers pool is dynamic and starts with 10 workers
+// You can specify fixed number of workers using `--output-tcp-workers`
+type BinaryOutput struct {
+	// Keep this as first element of struct because it guarantees 64bit
+	// alignment. atomic.* functions crash on 32bit machines if operand is not
+	// aligned at 64bit. See https://github.com/golang/go/issues/599
+	activeWorkers int64
+	address       string
+	queue         chan *Message
+	responses     chan response
+	needWorker    chan int
+	quit          chan struct{}
+	config        *BinaryOutputConfig
+	queueStats    *GorStat
+}
+
+// NewBinaryOutput constructor for BinaryOutput
+// Initialize workers
+func NewBinaryOutput(address string, config *BinaryOutputConfig) PluginReadWriter {
+	o := new(BinaryOutput)
+
+	o.address = address
+	o.config = config
+
+	o.queue = make(chan *Message, 1000)
+	o.responses = make(chan response, 1000)
+	o.needWorker = make(chan int, 1)
+	o.quit = make(chan struct{})
+
+	// Initial workers count
+	if o.config.Workers == 0 {
+		o.needWorker <- initialDynamicWorkers
+	} else {
+		o.needWorker <- o.config.Workers
+	}
+
+	go o.workerMaster()
+
+	return o
+}
+
+func (o *BinaryOutput) workerMaster() {
+	for {
+		newWorkers := <-o.needWorker
+		for i := 0; i < newWorkers; i++ {
+			go o.startWorker()
+		}
+
+		// Disable dynamic scaling if workers poll fixed size
+		if o.config.Workers != 0 {
+			return
+		}
+	}
+}
+
+func (o *BinaryOutput) startWorker() {
+	client := NewTCPClient(o.address, &TCPClientConfig{
+		Debug:              o.config.Debug,
+		Timeout:            o.config.Timeout,
+		ResponseBufferSize: int(o.config.BufferSize),
+	})
+
+	deathCount := 0
+
+	atomic.AddInt64(&o.activeWorkers, 1)
+
+	for {
+		select {
+		case msg := <-o.queue:
+			o.sendRequest(client, msg)
+			deathCount = 0
+		case <-time.After(time.Millisecond * 100):
+			// When dynamic scaling enabled workers die after 2s of inactivity
+			if o.config.Workers == 0 {
+				deathCount++
+			} else {
+				continue
+			}
+
+			if deathCount > 20 {
+				workersCount := atomic.LoadInt64(&o.activeWorkers)
+
+				// At least 1 startWorker should be alive
+				if workersCount != 1 {
+					atomic.AddInt64(&o.activeWorkers, -1)
+					return
+				}
+			}
+		}
+	}
+}
+
+// PluginWrite writes a message tothis plugin
+func (o *BinaryOutput) PluginWrite(msg *Message) (n int, err error) {
+	if !isRequestPayload(msg.Meta) {
+		return len(msg.Data), nil
+	}
+
+	o.queue <- msg
+
+	if o.config.Workers == 0 {
+		workersCount := atomic.LoadInt64(&o.activeWorkers)
+
+		if len(o.queue) > int(workersCount) {
+			o.needWorker <- len(o.queue)
+		}
+	}
+
+	return len(msg.Data) + len(msg.Meta), nil
+}
+
+// PluginRead reads a message from this plugin
+func (o *BinaryOutput) PluginRead() (*Message, error) {
+	var resp response
+	var msg Message
+	select {
+	case <-o.quit:
+		return nil, ErrorStopped
+	case resp = <-o.responses:
+	}
+	msg.Data = resp.payload
+	msg.Meta = payloadHeader(ReplayedResponsePayload, resp.uuid, resp.startedAt, resp.roundTripTime)
+
+	return &msg, nil
+}
+
+func (o *BinaryOutput) sendRequest(client *TCPClient, msg *Message) {
+	if !isRequestPayload(msg.Meta) {
+		return
+	}
+
+	uuid := payloadID(msg.Meta)
+
+	start := time.Now()
+	resp, err := client.Send(msg.Data)
+	stop := time.Now()
+
+	if err != nil {
+		Debug(1, "Request error:", err)
+	}
+
+	if o.config.TrackResponses {
+		o.responses <- response{resp, uuid, start.UnixNano(), stop.UnixNano() - start.UnixNano()}
+	}
+}
+
+func (o *BinaryOutput) String() string {
+	return "Binary output: " + o.address
+}
+
+// Close closes this plugin for reading
+func (o *BinaryOutput) Close() error {
+	close(o.quit)
+	return nil
+}

output_dummy.go

@@ -1,7 +1,7 @@
 package main
 
 import (
-	"fmt"
+	"os"
 )
 
 // DummyOutput used for debugging, prints all incoming requests
@@ -15,10 +15,16 @@ func NewDummyOutput() (di *DummyOutput) {
 	return
 }
 
-func (i *DummyOutput) Write(data []byte) (int, error) {
-	fmt.Println(string(data))
-
-	return len(data), nil
+// PluginWrite writes message to this plugin
+func (i *DummyOutput) PluginWrite(msg *Message) (int, error) {
+	var n, nn int
+	var err error
+	n, err = os.Stdout.Write(msg.Meta)
+	nn, err = os.Stdout.Write(msg.Data)
+	n += nn
+	nn, err = os.Stdout.Write(payloadSeparatorAsBytes)
+	n += nn
+	return n, err
 }
 
 func (i *DummyOutput) String() string {

output_file.go

@@ -3,6 +3,7 @@ package main
 import (
 	"bufio"
 	"compress/gzip"
+	"errors"
 	"fmt"
 	"io"
 	"log"
@@ -14,6 +15,8 @@ import (
 	"strings"
 	"sync"
 	"time"
+
+	"github.com/buger/goreplay/size"
 )
 
 var dateFileNameFuncs = map[string]func(*FileOutput) string{
@@ -24,27 +27,35 @@ var dateFileNameFuncs = map[string]func(*FileOutput) string{
 	"%M":  func(o *FileOutput) string { return time.Now().Format("04") },
 	"%S":  func(o *FileOutput) string { return time.Now().Format("05") },
 	"%NS": func(o *FileOutput) string { return fmt.Sprint(time.Now().Nanosecond()) },
-	"%r":  func(o *FileOutput) string { return o.currentID },
+	"%r":  func(o *FileOutput) string { return string(o.currentID) },
+	"%t":  func(o *FileOutput) string { return string(o.payloadType) },
 }
 
+// FileOutputConfig ...
 type FileOutputConfig struct {
-	flushInterval time.Duration
-	sizeLimit     unitSizeVar
-	queueLimit    int
-	append        bool
+	FlushInterval     time.Duration `json:"output-file-flush-interval"`
+	SizeLimit         size.Size     `json:"output-file-size-limit"`
+	OutputFileMaxSize size.Size     `json:"output-file-max-size-limit"`
+	QueueLimit        int           `json:"output-file-queue-limit"`
+	Append            bool          `json:"output-file-append"`
+	BufferPath        string        `json:"output-file-buffer"`
+	onClose           func(string)
 }
 
 // FileOutput output plugin
 type FileOutput struct {
-	mu             sync.Mutex
+	sync.RWMutex
 	pathTemplate   string
 	currentName    string
 	file           *os.File
-	queueLength    int
+	QueueLength    int
 	chunkSize      int
 	writer         io.Writer
 	requestPerFile bool
-	currentID      string
+	currentID      []byte
+	payloadType    []byte
+	closed         bool
+	totalFileSize  size.Size
 
 	config *FileOutputConfig
 }
@@ -60,9 +71,16 @@ func NewFileOutput(pathTemplate string, config *FileOutputConfig) *FileOutput {
 		o.requestPerFile = true
 	}
 
+	if config.FlushInterval == 0 {
+		config.FlushInterval = 100 * time.Millisecond
+	}
+
 	go func() {
 		for {
-			time.Sleep(config.flushInterval)
+			time.Sleep(config.FlushInterval)
+			if o.IsClosed() {
+				break
+			}
 			o.updateName()
 			o.flush()
 		}
@@ -125,8 +143,8 @@ func (s sortByFileIndex) Less(i, j int) bool {
 }
 
 func (o *FileOutput) filename() string {
-	defer o.mu.Unlock()
-	o.mu.Lock()
+	o.RLock()
+	defer o.RUnlock()
 
 	path := o.pathTemplate
 
@@ -134,12 +152,12 @@ func (o *FileOutput) filename() string {
 		path = strings.Replace(path, name, fn(o), -1)
 	}
 
-	if !o.config.append {
+	if !o.config.Append {
 		nextChunk := false
 
 		if o.currentName == "" ||
-			((o.config.queueLimit > 0 && o.queueLength >= o.config.queueLimit) ||
-				(o.config.sizeLimit > 0 && o.chunkSize >= int(o.config.sizeLimit))) {
+			((o.config.QueueLimit > 0 && o.QueueLength >= o.config.QueueLimit) ||
+				(o.config.SizeLimit > 0 && o.chunkSize >= int(o.config.SizeLimit))) {
 			nextChunk = true
 		}
 
@@ -171,22 +189,28 @@ func (o *FileOutput) filename() string {
 }
 
 func (o *FileOutput) updateName() {
-	o.currentName = filepath.Clean(o.filename())
+	name := filepath.Clean(o.filename())
+	o.Lock()
+	o.currentName = name
+	o.Unlock()
 }
 
-func (o *FileOutput) Write(data []byte) (n int, err error) {
+// PluginWrite writes message to this plugin
+func (o *FileOutput) PluginWrite(msg *Message) (n int, err error) {
 	if o.requestPerFile {
-		o.currentID = string(payloadMeta(data)[1])
-		o.updateName()
+		o.Lock()
+		meta := payloadMeta(msg.Meta)
+		o.currentID = meta[1]
+		o.payloadType = meta[0]
+		o.Unlock()
 	}
 
-	if !isOriginPayload(data) {
-		return len(data), nil
-	}
+	o.updateName()
+	o.Lock()
+	defer o.Unlock()
 
 	if o.file == nil || o.currentName != o.file.Name() {
-		o.mu.Lock()
-		o.Close()
+		o.closeLocked()
 
 		o.file, err = os.OpenFile(o.currentName, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0660)
 		o.file.Sync()
@@ -201,28 +225,36 @@ func (o *FileOutput) Write(data []byte) (n int, err error) {
 			log.Fatal(o, "Cannot open file %q. Error: %s", o.currentName, err)
 		}
 
-		o.queueLength = 0
-		o.mu.Unlock()
+		o.QueueLength = 0
 	}
 
-	o.writer.Write(data)
-	o.writer.Write([]byte(payloadSeparator))
+	var nn int
+	n, err = o.writer.Write(msg.Meta)
+	nn, err = o.writer.Write(msg.Data)
+	n += nn
+	nn, err = o.writer.Write(payloadSeparatorAsBytes)
+	n += nn
 
-	o.queueLength++
+	o.totalFileSize += size.Size(n)
+	o.QueueLength++
 
-	return len(data), nil
+	if Settings.OutputFileConfig.OutputFileMaxSize > 0 && o.totalFileSize >= Settings.OutputFileConfig.OutputFileMaxSize {
+		return n, errors.New("File output reached size limit")
+	}
+
+	return n, err
 }
 
 func (o *FileOutput) flush() {
 	// Don't exit on panic
 	defer func() {
 		if r := recover(); r != nil {
-			log.Println("PANIC while file flush: ", r, o, string(debug.Stack()))
+			Debug(0, "[OUTPUT-FILE] PANIC while file flush: ", r, o, string(debug.Stack()))
 		}
 	}()
 
-	defer o.mu.Unlock()
-	o.mu.Lock()
+	o.Lock()
+	defer o.Unlock()
 
 	if o.file != nil {
 		if strings.HasSuffix(o.currentName, ".gz") {
@@ -231,8 +263,10 @@ func (o *FileOutput) flush() {
 			o.writer.(*bufio.Writer).Flush()
 		}
 
-		if stat, err := o.file.Stat(); err != nil {
+		if stat, err := o.file.Stat(); err == nil {
 			o.chunkSize = int(stat.Size())
+		} else {
+			Debug(0, "[OUTPUT-HTTP] error accessing file size", err)
 		}
 	}
 }
@@ -241,7 +275,7 @@ func (o *FileOutput) String() string {
 	return "File output: " + o.file.Name()
 }
 
-func (o *FileOutput) Close() error {
+func (o *FileOutput) closeLocked() error {
 	if o.file != nil {
 		if strings.HasSuffix(o.currentName, ".gz") {
 			o.writer.(*gzip.Writer).Close()
@@ -249,6 +283,26 @@ func (o *FileOutput) Close() error {
 			o.writer.(*bufio.Writer).Flush()
 		}
 		o.file.Close()
+
+		if o.config.onClose != nil {
+			o.config.onClose(o.file.Name())
+		}
 	}
+
+	o.closed = true
 	return nil
 }
+
+// Close closes the output file that is being written to.
+func (o *FileOutput) Close() error {
+	o.Lock()
+	defer o.Unlock()
+	return o.closeLocked()
+}
+
+// IsClosed returns if the output file is closed or not.
+func (o *FileOutput) IsClosed() bool {
+	o.Lock()
+	defer o.Unlock()
+	return o.closed
+}

output_file_settings.go

@@ -1,39 +0,0 @@
-package main
-
-import (
-	"strconv"
-	"strings"
-)
-
-var dataUnitMap = map[byte]int64{
-	'k': 1024,
-	'm': 1024 * 1024,
-	'g': 1024 * 1024 * 1024,
-}
-
-func parseDataUnit(s string) int64 {
-	// Allow kb, mb, gb
-	if strings.HasSuffix(s, "b") {
-		s = s[:len(s)-1]
-	}
-
-	if unit, ok := dataUnitMap[s[len(s)-1]]; ok {
-		size, _ := strconv.ParseInt(s[:len(s)-1], 10, 64)
-		return unit * size
-	} else {
-		// If no unit specified use bytes
-		size, _ := strconv.ParseInt(s, 10, 64)
-		return size
-	}
-}
-
-type unitSizeVar int64
-
-func (u unitSizeVar) String() string {
-	return strconv.Itoa(int(u))
-}
-
-func (u unitSizeVar) Set(s string) error {
-	u = unitSizeVar(parseDataUnit(s))
-	return nil
-}

output_file_test.go

@@ -2,7 +2,6 @@ package main
 
 import (
 	"fmt"
-	"io"
 	"math/rand"
 	"os"
 	"reflect"
@@ -11,19 +10,24 @@ import (
 	"sync/atomic"
 	"testing"
 	"time"
+
+	"github.com/buger/goreplay/size"
 )
 
 func TestFileOutput(t *testing.T) {
 	wg := new(sync.WaitGroup)
-	quit := make(chan int)
 
 	input := NewTestInput()
-	output := NewFileOutput("/tmp/test_requests.gor", &FileOutputConfig{flushInterval: time.Minute, append: true})
+	output := NewFileOutput("/tmp/test_requests.gor", &FileOutputConfig{FlushInterval: time.Minute, Append: true})
 
-	Plugins.Inputs = []io.Reader{input}
-	Plugins.Outputs = []io.Writer{output}
+	plugins := &InOutPlugins{
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{output},
+	}
+	plugins.All = append(plugins.All, input, output)
 
-	go Start(quit)
+	emitter := NewEmitter()
+	go emitter.Start(plugins, Settings.Middleware)
 
 	for i := 0; i < 100; i++ {
 		wg.Add(2)
@@ -32,29 +36,30 @@ func TestFileOutput(t *testing.T) {
 	}
 	time.Sleep(100 * time.Millisecond)
 	output.flush()
-
-	close(quit)
-
-	quit = make(chan int)
+	emitter.Close()
 
 	var counter int64
 	input2 := NewFileInput("/tmp/test_requests.gor", false)
-	output2 := NewTestOutput(func(data []byte) {
+	output2 := NewTestOutput(func(*Message) {
 		atomic.AddInt64(&counter, 1)
 		wg.Done()
 	})
 
-	Plugins.Inputs = []io.Reader{input2}
-	Plugins.Outputs = []io.Writer{output2}
+	plugins2 := &InOutPlugins{
+		Inputs:  []PluginReader{input2},
+		Outputs: []PluginWriter{output2},
+	}
+	plugins2.All = append(plugins2.All, input2, output2)
 
-	go Start(quit)
+	emitter2 := NewEmitter()
+	go emitter2.Start(plugins2, Settings.Middleware)
 
 	wg.Wait()
-	close(quit)
+	emitter2.Close()
 }
 
 func TestFileOutputWithNameCleaning(t *testing.T) {
-	output := &FileOutput{pathTemplate: "./test_requests.gor", config: &FileOutputConfig{flushInterval: time.Minute, append: false}}
+	output := &FileOutput{pathTemplate: "./test_requests.gor", config: &FileOutputConfig{FlushInterval: time.Minute, Append: false}}
 	expectedFileName := "test_requests_0.gor"
 	output.updateName()
 
@@ -65,9 +70,10 @@ func TestFileOutputWithNameCleaning(t *testing.T) {
 }
 
 func TestFileOutputPathTemplate(t *testing.T) {
-	output := &FileOutput{pathTemplate: "/tmp/log-%Y-%m-%d-%S", config: &FileOutputConfig{flushInterval: time.Minute, append: true}}
+	output := &FileOutput{pathTemplate: "/tmp/log-%Y-%m-%d-%S-%t", config: &FileOutputConfig{FlushInterval: time.Minute, Append: true}}
 	now := time.Now()
-	expectedPath := fmt.Sprintf("/tmp/log-%s-%s-%s-%s", now.Format("2006"), now.Format("01"), now.Format("02"), now.Format("05"))
+	output.payloadType = []byte("3")
+	expectedPath := fmt.Sprintf("/tmp/log-%s-%s-%s-%s-3", now.Format("2006"), now.Format("01"), now.Format("02"), now.Format("05"))
 	path := output.filename()
 
 	if expectedPath != path {
@@ -76,22 +82,22 @@ func TestFileOutputPathTemplate(t *testing.T) {
 }
 
 func TestFileOutputMultipleFiles(t *testing.T) {
-	output := NewFileOutput("/tmp/log-%Y-%m-%d-%S", &FileOutputConfig{append: true, flushInterval: time.Minute})
+	output := NewFileOutput("/tmp/log-%Y-%m-%d-%S", &FileOutputConfig{Append: true, FlushInterval: time.Minute})
 
 	if output.file != nil {
 		t.Error("Should not initialize file if no writes")
 	}
 
-	output.Write([]byte("1 1 1\r\ntest"))
+	output.PluginWrite(&Message{Meta: []byte("1 1 1\r\n"), Data: []byte("test")})
 	name1 := output.file.Name()
 
-	output.Write([]byte("1 1 1\r\ntest"))
+	output.PluginWrite(&Message{Meta: []byte("1 1 1\r\n"), Data: []byte("test")})
 	name2 := output.file.Name()
 
 	time.Sleep(time.Second)
 	output.updateName()
 
-	output.Write([]byte("1 1 1\r\ntest"))
+	output.PluginWrite(&Message{Meta: []byte("1 1 1\r\n"), Data: []byte("test")})
 	name3 := output.file.Name()
 
 	if name2 != name1 {
@@ -107,22 +113,22 @@ func TestFileOutputMultipleFiles(t *testing.T) {
 }
 
 func TestFileOutputFilePerRequest(t *testing.T) {
-	output := NewFileOutput("/tmp/log-%Y-%m-%d-%S-%r", &FileOutputConfig{append: true})
+	output := NewFileOutput("/tmp/log-%Y-%m-%d-%S-%r", &FileOutputConfig{Append: true})
 
 	if output.file != nil {
 		t.Error("Should not initialize file if no writes")
 	}
 
-	output.Write([]byte("1 1 1\ntest"))
+	output.PluginWrite(&Message{Meta: []byte("1 1 1\r\n"), Data: []byte("test")})
 	name1 := output.file.Name()
 
-	output.Write([]byte("1 2 1\ntest"))
+	output.PluginWrite(&Message{Meta: []byte("1 2 1\r\n"), Data: []byte("test")})
 	name2 := output.file.Name()
 
 	time.Sleep(time.Second)
 	output.updateName()
 
-	output.Write([]byte("1 3 1\ntest"))
+	output.PluginWrite(&Message{Meta: []byte("1 3 1\r\n"), Data: []byte("test")})
 	name3 := output.file.Name()
 
 	if name3 == name2 || name2 == name1 || name3 == name1 {
@@ -135,14 +141,14 @@ func TestFileOutputFilePerRequest(t *testing.T) {
 }
 
 func TestFileOutputCompression(t *testing.T) {
-	output := NewFileOutput("/tmp/log-%Y-%m-%d-%S.gz", &FileOutputConfig{append: true, flushInterval: time.Minute})
+	output := NewFileOutput("/tmp/log-%Y-%m-%d-%S.gz", &FileOutputConfig{Append: true, FlushInterval: time.Minute})
 
 	if output.file != nil {
 		t.Error("Should not initialize file if no writes")
 	}
 
 	for i := 0; i < 1000; i++ {
-		output.Write([]byte("1 1 1\r\ntest"))
+		output.PluginWrite(&Message{Meta: []byte("1 1 1\r\n"), Data: []byte("test")})
 	}
 
 	name := output.file.Name()
@@ -156,26 +162,6 @@ func TestFileOutputCompression(t *testing.T) {
 	os.Remove(name)
 }
 
-func TestParseDataUnit(t *testing.T) {
-	var tests = []struct {
-		value string
-		size  int64
-	}{
-		{"100kb", dataUnitMap['k'] * 100},
-		{"100k", dataUnitMap['k'] * 100},
-		{"1kb", dataUnitMap['k']},
-		{"1g", dataUnitMap['g']},
-		{"10m", dataUnitMap['m'] * 10},
-		{"zsaa312", 0},
-	}
-
-	for _, c := range tests {
-		if parseDataUnit(c.value) != c.size {
-			t.Error(c.value, "should be", c.size, "instead", parseDataUnit(c.value))
-		}
-	}
-}
-
 func TestGetFileIndex(t *testing.T) {
 	var tests = []struct {
 		path  string
@@ -219,17 +205,17 @@ func TestFileOutputAppendQueueLimitOverflow(t *testing.T) {
 	rnd := rand.Int63()
 	name := fmt.Sprintf("/tmp/%d", rnd)
 
-	output := NewFileOutput(name, &FileOutputConfig{append: false, flushInterval: time.Minute, queueLimit: 2})
+	output := NewFileOutput(name, &FileOutputConfig{Append: false, FlushInterval: time.Minute, QueueLimit: 2})
 
-	output.Write([]byte("1 1 1\r\ntest"))
+	output.PluginWrite(&Message{Meta: []byte("1 1 1\r\n"), Data: []byte("test")})
 	name1 := output.file.Name()
 
-	output.Write([]byte("1 1 1\r\ntest"))
+	output.PluginWrite(&Message{Meta: []byte("1 1 1\r\n"), Data: []byte("test")})
 	name2 := output.file.Name()
 
 	output.updateName()
 
-	output.Write([]byte("1 1 1\r\ntest"))
+	output.PluginWrite(&Message{Meta: []byte("1 1 1\r\n"), Data: []byte("test")})
 	name3 := output.file.Name()
 
 	if name2 != name1 || name1 != fmt.Sprintf("/tmp/%d_0", rnd) {
@@ -248,17 +234,17 @@ func TestFileOutputAppendQueueLimitNoOverflow(t *testing.T) {
 	rnd := rand.Int63()
 	name := fmt.Sprintf("/tmp/%d", rnd)
 
-	output := NewFileOutput(name, &FileOutputConfig{append: false, flushInterval: time.Minute, queueLimit: 3})
+	output := NewFileOutput(name, &FileOutputConfig{Append: false, FlushInterval: time.Minute, QueueLimit: 3})
 
-	output.Write([]byte("1 1 1\r\ntest"))
+	output.PluginWrite(&Message{Meta: []byte("1 1 1\r\n"), Data: []byte("test")})
 	name1 := output.file.Name()
 
-	output.Write([]byte("1 1 1\r\ntest"))
+	output.PluginWrite(&Message{Meta: []byte("1 1 1\r\n"), Data: []byte("test")})
 	name2 := output.file.Name()
 
 	output.updateName()
 
-	output.Write([]byte("1 1 1\r\ntest"))
+	output.PluginWrite(&Message{Meta: []byte("1 1 1\r\n"), Data: []byte("test")})
 	name3 := output.file.Name()
 
 	if name2 != name1 || name1 != fmt.Sprintf("/tmp/%d_0", rnd) {
@@ -277,17 +263,17 @@ func TestFileOutputAppendQueueLimitGzips(t *testing.T) {
 	rnd := rand.Int63()
 	name := fmt.Sprintf("/tmp/%d.gz", rnd)
 
-	output := NewFileOutput(name, &FileOutputConfig{append: false, flushInterval: time.Minute, queueLimit: 2})
+	output := NewFileOutput(name, &FileOutputConfig{Append: false, FlushInterval: time.Minute, QueueLimit: 2})
 
-	output.Write([]byte("1 1 1\r\ntest"))
+	output.PluginWrite(&Message{Meta: []byte("1 1 1\r\n"), Data: []byte("test")})
 	name1 := output.file.Name()
 
-	output.Write([]byte("1 1 1\r\ntest"))
+	output.PluginWrite(&Message{Meta: []byte("1 1 1\r\n"), Data: []byte("test")})
 	name2 := output.file.Name()
 
 	output.updateName()
 
-	output.Write([]byte("1 1 1\r\ntest"))
+	output.PluginWrite(&Message{Meta: []byte("1 1 1\r\n"), Data: []byte("test")})
 	name3 := output.file.Name()
 
 	if name2 != name1 || name1 != fmt.Sprintf("/tmp/%d_0.gz", rnd) {
@@ -311,3 +297,37 @@ func TestFileOutputSort(t *testing.T) {
 		t.Error("Should properly sort file names using indexes", files, expected)
 	}
 }
+
+func TestFileOutputAppendSizeLimitOverflow(t *testing.T) {
+	rnd := rand.Int63()
+	name := fmt.Sprintf("/tmp/%d", rnd)
+
+	message := []byte("1 1 1\r\ntest")
+
+	messageSize := len(message) + len(payloadSeparator)
+
+	output := NewFileOutput(name, &FileOutputConfig{Append: false, FlushInterval: time.Minute, SizeLimit: size.Size(2 * messageSize)})
+
+	output.PluginWrite(&Message{Meta: []byte("1 1 1\r\n"), Data: []byte("test")})
+	name1 := output.file.Name()
+
+	output.PluginWrite(&Message{Meta: []byte("1 1 1\r\n"), Data: []byte("test")})
+	name2 := output.file.Name()
+
+	output.flush()
+	output.updateName()
+
+	output.PluginWrite(&Message{Meta: []byte("1 1 1\r\n"), Data: []byte("test")})
+	name3 := output.file.Name()
+
+	if name2 != name1 || name1 != fmt.Sprintf("/tmp/%d_0", rnd) {
+		t.Error("Fast changes should happen in same file:", name1, name2, name3)
+	}
+
+	if name3 == name1 || name3 != fmt.Sprintf("/tmp/%d_1", rnd) {
+		t.Error("File name should change:", name1, name2, name3)
+	}
+
+	os.Remove(name1)
+	os.Remove(name3)
+}

output_http.go

@@ -1,14 +1,26 @@
 package main
 
 import (
-	"io"
+	"bufio"
+	"bytes"
+	"crypto/tls"
+	"fmt"
+	"log"
+	"math"
+	"net/http"
+	"net/http/httputil"
+	"net/url"
 	"sync/atomic"
 	"time"
 
-	"github.com/buger/gor/proto"
+	"github.com/buger/goreplay/size"
 )
 
-const initialDynamicWorkers = 10
+const (
+	initialDynamicWorkers = 10
+	readChunkSize         = 64 * 1024
+	maxResponseSize       = 1073741824
+)
 
 type response struct {
 	payload       []byte
@@ -19,202 +31,296 @@ type response struct {
 
 // HTTPOutputConfig struct for holding http output configuration
 type HTTPOutputConfig struct {
-	redirectLimit int
-
-	stats   bool
-	workers int
-
-	elasticSearch string
-
-	Timeout      time.Duration
-	OriginalHost bool
-	BufferSize   int
-
-	Debug bool
-
-	TrackResponses bool
+	TrackResponses bool          `json:"output-http-track-response"`
+	Stats          bool          `json:"output-http-stats"`
+	OriginalHost   bool          `json:"output-http-original-host"`
+	RedirectLimit  int           `json:"output-http-redirect-limit"`
+	WorkersMin     int           `json:"output-http-workers-min"`
+	WorkersMax     int           `json:"output-http-workers"`
+	StatsMs        int           `json:"output-http-stats-ms"`
+	QueueLen       int           `json:"output-http-queue-len"`
+	ElasticSearch  string        `json:"output-http-elasticsearch"`
+	Timeout        time.Duration `json:"output-http-timeout"`
+	WorkerTimeout  time.Duration `json:"output-http-worker-timeout"`
+	BufferSize     size.Size     `json:"output-http-response-buffer"`
+	SkipVerify     bool          `json:"output-http-skip-verify"`
+	rawURL         string
+	url            *url.URL
 }
 
 // HTTPOutput plugin manage pool of workers which send request to replayed server
-// By default workers pool is dynamic and starts with 10 workers
-// You can specify fixed number of workers using `--output-http-workers`
+// By default workers pool is dynamic and starts with 1 worker or workerMin workers
+// You can specify maximum number of workers using `--output-http-workers`
 type HTTPOutput struct {
-	// Keep this as first element of struct because it guarantees 64bit
-	// alignment. atomic.* functions crash on 32bit machines if operand is not
-	// aligned at 64bit. See https://github.com/golang/go/issues/599
-	activeWorkers int64
-
-	address string
-	limit   int
-	queue   chan []byte
-
-	responses chan response
-
-	needWorker chan int
-
-	config *HTTPOutputConfig
-
-	queueStats *GorStat
-
+	activeWorkers int32
+	config        *HTTPOutputConfig
+	queueStats    *GorStat
 	elasticSearch *ESPlugin
+	client        *HTTPClient
+	stopWorker    chan struct{}
+	queue         chan *Message
+	responses     chan *response
+	stop          chan bool // Channel used only to indicate goroutine should shutdown
 }
 
 // NewHTTPOutput constructor for HTTPOutput
 // Initialize workers
-func NewHTTPOutput(address string, config *HTTPOutputConfig) io.Writer {
+func NewHTTPOutput(address string, config *HTTPOutputConfig) PluginReadWriter {
 	o := new(HTTPOutput)
-
-	o.address = address
+	var err error
+	config.url, err = url.Parse(address)
+	if err != nil {
+		log.Fatal(fmt.Sprintf("[OUTPUT-HTTP] parse HTTP output URL error[%q]", err))
+	}
+	if config.url.Scheme == "" {
+		config.url.Scheme = "http"
+	}
+	config.rawURL = config.url.String()
+	if config.Timeout < time.Millisecond*100 {
+		config.Timeout = time.Second
+	}
+	if config.BufferSize <= 0 {
+		config.BufferSize = 100 * 1024 // 100kb
+	}
+	if config.WorkersMin <= 0 {
+		config.WorkersMin = 1
+	}
+	if config.WorkersMin > 1000 {
+		config.WorkersMin = 1000
+	}
+	if config.WorkersMax <= 0 {
+		config.WorkersMax = math.MaxInt32 // idealy so large
+	}
+	if config.WorkersMax < config.WorkersMin {
+		config.WorkersMax = config.WorkersMin
+	}
+	if config.QueueLen <= 0 {
+		config.QueueLen = 1000
+	}
+	if config.RedirectLimit < 0 {
+		config.RedirectLimit = 0
+	}
+	if config.WorkerTimeout <= 0 {
+		config.WorkerTimeout = time.Second * 2
+	}
 	o.config = config
-
-	if o.config.stats {
-		o.queueStats = NewGorStat("output_http")
+	o.stop = make(chan bool)
+	if o.config.Stats {
+		o.queueStats = NewGorStat("output_http", o.config.StatsMs)
 	}
 
-	o.queue = make(chan []byte, 1000)
-	o.responses = make(chan response, 1000)
-	o.needWorker = make(chan int, 1)
-
-	// Initial workers count
-	if o.config.workers == 0 {
-		o.needWorker <- initialDynamicWorkers
-	} else {
-		o.needWorker <- o.config.workers
+	o.queue = make(chan *Message, o.config.QueueLen)
+	if o.config.TrackResponses {
+		o.responses = make(chan *response, o.config.QueueLen)
 	}
+	// it should not be buffered to avoid races
+	o.stopWorker = make(chan struct{})
 
-	if o.config.elasticSearch != "" {
+	if o.config.ElasticSearch != "" {
 		o.elasticSearch = new(ESPlugin)
-		o.elasticSearch.Init(o.config.elasticSearch)
+		o.elasticSearch.Init(o.config.ElasticSearch)
 	}
-
-	if len(Settings.middleware) > 0 {
-		o.config.TrackResponses = true
+	o.client = NewHTTPClient(o.config)
+	o.activeWorkers += int32(o.config.WorkersMin)
+	for i := 0; i < o.config.WorkersMin; i++ {
+		go o.startWorker()
 	}
-
 	go o.workerMaster()
-
 	return o
 }
 
 func (o *HTTPOutput) workerMaster() {
+	var timer = time.NewTimer(o.config.WorkerTimeout)
+	defer func() {
+		// recover from panics caused by trying to send in
+		// a closed chan(o.stopWorker)
+		recover()
+	}()
+	defer timer.Stop()
 	for {
-		newWorkers := <-o.needWorker
-		for i := 0; i < newWorkers; i++ {
-			go o.startWorker()
-		}
-
-		// Disable dynamic scaling if workers poll fixed size
-		if o.config.workers != 0 {
+		select {
+		case <-o.stop:
 			return
+		default:
+			<-timer.C
 		}
+		// rollback workers
+	rollback:
+		if atomic.LoadInt32(&o.activeWorkers) > int32(o.config.WorkersMin) && len(o.queue) < 1 {
+			// close one worker
+			o.stopWorker <- struct{}{}
+			atomic.AddInt32(&o.activeWorkers, -1)
+			goto rollback
+		}
+		timer.Reset(o.config.WorkerTimeout)
 	}
 }
 
 func (o *HTTPOutput) startWorker() {
-	client := NewHTTPClient(o.address, &HTTPClientConfig{
-		FollowRedirects:    o.config.redirectLimit,
-		Debug:              o.config.Debug,
-		OriginalHost:       o.config.OriginalHost,
-		Timeout:            o.config.Timeout,
-		ResponseBufferSize: o.config.BufferSize,
-	})
-
-	deathCount := 0
-
-	atomic.AddInt64(&o.activeWorkers, 1)
-
 	for {
 		select {
-		case data := <-o.queue:
-			o.sendRequest(client, data)
-			deathCount = 0
-		case <-time.After(time.Millisecond * 100):
-			// When dynamic scaling enabled workers die after 2s of inactivity
-			if o.config.workers == 0 {
-				deathCount++
-			} else {
-				continue
-			}
-
-			if deathCount > 20 {
-				workersCount := atomic.LoadInt64(&o.activeWorkers)
-
-				// At least 1 startWorker should be alive
-				if workersCount != 1 {
-					atomic.AddInt64(&o.activeWorkers, -1)
-					return
-				}
-			}
+		case <-o.stopWorker:
+			return
+		case msg := <-o.queue:
+			o.sendRequest(o.client, msg)
 		}
 	}
 }
 
-func (o *HTTPOutput) Write(data []byte) (n int, err error) {
-	if !isRequestPayload(data) {
-		return len(data), nil
+// PluginWrite writes message to this plugin
+func (o *HTTPOutput) PluginWrite(msg *Message) (n int, err error) {
+	if !isRequestPayload(msg.Meta) {
+		return len(msg.Data), nil
 	}
 
-	buf := make([]byte, len(data))
-	copy(buf, data)
+	select {
+	case <-o.stop:
+		return 0, ErrorStopped
+	case o.queue <- msg:
+	}
 
-	o.queue <- buf
-
-	if o.config.stats {
+	if o.config.Stats {
 		o.queueStats.Write(len(o.queue))
 	}
-
-	if o.config.workers == 0 {
-		workersCount := atomic.LoadInt64(&o.activeWorkers)
-
-		if len(o.queue) > int(workersCount) {
-			o.needWorker <- len(o.queue)
+	if len(o.queue) > 0 {
+		// try to start a new worker to serve
+		if atomic.LoadInt32(&o.activeWorkers) < int32(o.config.WorkersMax) {
+			go o.startWorker()
+			atomic.AddInt32(&o.activeWorkers, 1)
 		}
 	}
-
-	return len(data), nil
+	return len(msg.Data) + len(msg.Meta), nil
 }
 
-func (o *HTTPOutput) Read(data []byte) (int, error) {
-	resp := <-o.responses
+// PluginRead reads message from this plugin
+func (o *HTTPOutput) PluginRead() (*Message, error) {
+	if !o.config.TrackResponses {
+		return nil, ErrorStopped
+	}
+	var resp *response
+	var msg Message
+	select {
+	case <-o.stop:
+		return nil, ErrorStopped
+	case resp = <-o.responses:
+		msg.Data = resp.payload
+	}
 
-	Debug("[OUTPUT-HTTP] Received response:", string(resp.payload))
+	msg.Meta = payloadHeader(ReplayedResponsePayload, resp.uuid, resp.roundTripTime, resp.startedAt)
 
-	header := payloadHeader(ReplayedResponsePayload, resp.uuid, resp.roundTripTime, resp.startedAt)
-	copy(data[0:len(header)], header)
-	copy(data[len(header):], resp.payload)
-
-	return len(resp.payload) + len(header), nil
+	return &msg, nil
 }
 
-func (o *HTTPOutput) sendRequest(client *HTTPClient, request []byte) {
-	meta := payloadMeta(request)
-	if len(meta) < 2 {
+func (o *HTTPOutput) sendRequest(client *HTTPClient, msg *Message) {
+	if !isRequestPayload(msg.Meta) {
 		return
 	}
-	uuid := meta[1]
-
-	body := payloadBody(request)
-	if !proto.IsHTTPPayload(body) {
-		return
-	}
-
+	uuid := payloadID(msg.Meta)
 	start := time.Now()
-	resp, err := client.Send(body)
+	resp, err := client.Send(msg.Data)
 	stop := time.Now()
 
 	if err != nil {
-		Debug("Request error:", err)
+		Debug(1, fmt.Sprintf("[HTTP-OUTPUT] error when sending: %q", err))
+		return
+	}
+	if resp == nil {
+		return
 	}
 
 	if o.config.TrackResponses {
-		o.responses <- response{resp, uuid, start.UnixNano(), stop.UnixNano() - start.UnixNano()}
+		o.responses <- &response{resp, uuid, start.UnixNano(), stop.UnixNano() - start.UnixNano()}
 	}
 
 	if o.elasticSearch != nil {
-		o.elasticSearch.ResponseAnalyze(request, resp, start, stop)
+		o.elasticSearch.ResponseAnalyze(msg.Data, resp, start, stop)
 	}
 }
 
 func (o *HTTPOutput) String() string {
-	return "HTTP output: " + o.address
+	return "HTTP output: " + o.config.rawURL
+}
+
+// Close closes the data channel so that data
+func (o *HTTPOutput) Close() error {
+	close(o.stop)
+	close(o.stopWorker)
+	return nil
+}
+
+// HTTPClient holds configurations for a single HTTP client
+type HTTPClient struct {
+	config *HTTPOutputConfig
+	Client *http.Client
+}
+
+// NewHTTPClient returns new http client with check redirects policy
+func NewHTTPClient(config *HTTPOutputConfig) *HTTPClient {
+	client := new(HTTPClient)
+	client.config = config
+	var transport *http.Transport
+	client.Client = &http.Client{
+		Timeout: client.config.Timeout,
+		CheckRedirect: func(req *http.Request, via []*http.Request) error {
+			if len(via) >= client.config.RedirectLimit {
+				Debug(1, fmt.Sprintf("[HTTPCLIENT] maximum output-http-redirects[%d] reached!", client.config.RedirectLimit))
+				return http.ErrUseLastResponse
+			}
+			lastReq := via[len(via)-1]
+			resp := req.Response
+			Debug(2, fmt.Sprintf("[HTTPCLIENT] HTTP redirects from %q to %q with %q", lastReq.Host, req.Host, resp.Status))
+			return nil
+		},
+	}
+	if config.SkipVerify {
+		// clone to avoid modying global default RoundTripper
+		transport = http.DefaultTransport.(*http.Transport).Clone()
+		transport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true}
+		client.Client.Transport = transport
+	}
+
+	return client
+}
+
+// Send sends an http request using client create by NewHTTPClient
+func (c *HTTPClient) Send(data []byte) ([]byte, error) {
+	var req *http.Request
+	var resp *http.Response
+	var err error
+
+	req, err = http.ReadRequest(bufio.NewReader(bytes.NewReader(data)))
+	if err != nil {
+		return nil, err
+	}
+	// we don't send CONNECT or OPTIONS request
+	if req.Method == http.MethodConnect {
+		return nil, nil
+	}
+
+	if !c.config.OriginalHost {
+		req.Host = c.config.url.Host
+	}
+
+	// fix #862
+	if c.config.url.Path == "" && c.config.url.RawQuery == "" {
+		req.URL.Scheme = c.config.url.Scheme
+		req.URL.Host = c.config.url.Host
+	} else {
+		req.URL = c.config.url
+	}
+
+	// force connection to not be closed, which can affect the global client
+	req.Close = false
+	// it's an error if this is not equal to empty string
+	req.RequestURI = ""
+
+	resp, err = c.Client.Do(req)
+	if err != nil {
+		return nil, err
+	}
+	if c.config.TrackResponses {
+		return httputil.DumpResponse(resp, true)
+	}
+	_ = resp.Body.Close()
+	return nil, nil
 }

output_http_test.go

@@ -1,19 +1,16 @@
 package main
 
 import (
-	"io"
 	"io/ioutil"
 	"net/http"
 	"net/http/httptest"
 	_ "net/http/httputil"
 	"sync"
 	"testing"
-	"time"
 )
 
 func TestHTTPOutput(t *testing.T) {
 	wg := new(sync.WaitGroup)
-	quit := make(chan int)
 
 	input := NewTestInput()
 
@@ -39,34 +36,40 @@ func TestHTTPOutput(t *testing.T) {
 	}))
 	defer server.Close()
 
-	headers := HTTPHeaders{HTTPHeader{"User-Agent", "Gor"}}
+	headers := HTTPHeaders{httpHeader{"User-Agent", "Gor"}}
 	methods := HTTPMethods{[]byte("GET"), []byte("PUT"), []byte("POST")}
-	Settings.modifierConfig = HTTPModifierConfig{headers: headers, methods: methods}
+	Settings.ModifierConfig = HTTPModifierConfig{Headers: headers, Methods: methods}
 
-	output := NewHTTPOutput(server.URL, &HTTPOutputConfig{Debug: true})
+	httpOutput := NewHTTPOutput(server.URL, &HTTPOutputConfig{TrackResponses: false})
+	output := NewTestOutput(func(*Message) {
+		wg.Done()
+	})
 
-	Plugins.Inputs = []io.Reader{input}
-	Plugins.Outputs = []io.Writer{output}
+	plugins := &InOutPlugins{
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{httpOutput, output},
+	}
+	plugins.All = append(plugins.All, input, output, httpOutput)
 
-	go Start(quit)
+	emitter := NewEmitter()
+	go emitter.Start(plugins, Settings.Middleware)
 
-	for i := 0; i < 100; i++ {
-		wg.Add(2) // OPTIONS should be ignored
+	for i := 0; i < 10; i++ {
+		// 2 http-output, 2 - test output request
+		wg.Add(4) // OPTIONS should be ignored
 		input.EmitPOST()
 		input.EmitOPTIONS()
 		input.EmitGET()
 	}
 
 	wg.Wait()
+	emitter.Close()
 
-	close(quit)
-
-	Settings.modifierConfig = HTTPModifierConfig{}
+	Settings.ModifierConfig = HTTPModifierConfig{}
 }
 
 func TestHTTPOutputKeepOriginalHost(t *testing.T) {
 	wg := new(sync.WaitGroup)
-	quit := make(chan int)
 
 	input := NewTestInput()
 
@@ -79,29 +82,30 @@ func TestHTTPOutputKeepOriginalHost(t *testing.T) {
 	}))
 	defer server.Close()
 
-	headers := HTTPHeaders{HTTPHeader{"Host", "custom-host.com"}}
-	Settings.modifierConfig = HTTPModifierConfig{headers: headers}
+	headers := HTTPHeaders{httpHeader{"Host", "custom-host.com"}}
+	Settings.ModifierConfig = HTTPModifierConfig{Headers: headers}
 
-	output := NewHTTPOutput(server.URL, &HTTPOutputConfig{Debug: false, OriginalHost: true})
+	output := NewHTTPOutput(server.URL, &HTTPOutputConfig{OriginalHost: true, SkipVerify: true})
 
-	Plugins.Inputs = []io.Reader{input}
-	Plugins.Outputs = []io.Writer{output}
+	plugins := &InOutPlugins{
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{output},
+	}
+	plugins.All = append(plugins.All, input, output)
 
-	go Start(quit)
+	emitter := NewEmitter()
+	go emitter.Start(plugins, Settings.Middleware)
 
 	wg.Add(1)
 	input.EmitGET()
 
 	wg.Wait()
-
-	close(quit)
-
-	Settings.modifierConfig = HTTPModifierConfig{}
+	emitter.Close()
+	Settings.ModifierConfig = HTTPModifierConfig{}
 }
 
-func TestOutputHTTPSSL(t *testing.T) {
+func TestHTTPOutputSSL(t *testing.T) {
 	wg := new(sync.WaitGroup)
-	quit := make(chan int)
 
 	// Origing and Replay server initialization
 	server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
@@ -109,12 +113,16 @@ func TestOutputHTTPSSL(t *testing.T) {
 	}))
 
 	input := NewTestInput()
-	output := NewHTTPOutput(server.URL, &HTTPOutputConfig{})
+	output := NewHTTPOutput(server.URL, &HTTPOutputConfig{SkipVerify: true})
 
-	Plugins.Inputs = []io.Reader{input}
-	Plugins.Outputs = []io.Writer{output}
+	plugins := &InOutPlugins{
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{output},
+	}
+	plugins.All = append(plugins.All, input, output)
 
-	go Start(quit)
+	emitter := NewEmitter()
+	go emitter.Start(plugins, Settings.Middleware)
 
 	wg.Add(2)
 
@@ -122,26 +130,74 @@ func TestOutputHTTPSSL(t *testing.T) {
 	input.EmitGET()
 
 	wg.Wait()
-	close(quit)
+	emitter.Close()
+}
+
+func TestHTTPOutputSessions(t *testing.T) {
+	wg := new(sync.WaitGroup)
+
+	input := NewTestInput()
+	input.skipHeader = true
+
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
+		wg.Done()
+	}))
+	defer server.Close()
+
+	Settings.RecognizeTCPSessions = true
+	Settings.SplitOutput = true
+	output := NewHTTPOutput(server.URL, &HTTPOutputConfig{})
+
+	plugins := &InOutPlugins{
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{output},
+	}
+	plugins.All = append(plugins.All, input, output)
+	emitter := NewEmitter()
+	go emitter.Start(plugins, Settings.Middleware)
+
+	uuid1 := []byte("1234567890123456789a0000")
+	uuid2 := []byte("1234567890123456789d0000")
+
+	for i := 0; i < 10; i++ {
+		wg.Add(1) // OPTIONS should be ignored
+		copy(uuid1[20:], randByte(4))
+		input.EmitBytes([]byte("1 " + string(uuid1) + " 1\n" + "GET / HTTP/1.1\r\n\r\n"))
+	}
+
+	for i := 0; i < 10; i++ {
+		wg.Add(1) // OPTIONS should be ignored
+		copy(uuid2[20:], randByte(4))
+		input.EmitBytes([]byte("1 " + string(uuid2) + " 1\n" + "GET / HTTP/1.1\r\n\r\n"))
+	}
+
+	wg.Wait()
+
+	emitter.Close()
+
+	Settings.RecognizeTCPSessions = false
+	Settings.SplitOutput = false
 }
 
 func BenchmarkHTTPOutput(b *testing.B) {
 	wg := new(sync.WaitGroup)
-	quit := make(chan int)
 
-	server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		time.Sleep(50 * time.Millisecond)
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		wg.Done()
 	}))
 	defer server.Close()
 
 	input := NewTestInput()
-	output := NewHTTPOutput(server.URL, &HTTPOutputConfig{})
+	output := NewHTTPOutput(server.URL, &HTTPOutputConfig{WorkersMax: 1})
 
-	Plugins.Inputs = []io.Reader{input}
-	Plugins.Outputs = []io.Writer{output}
+	plugins := &InOutPlugins{
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{output},
+	}
+	plugins.All = append(plugins.All, input, output)
 
-	go Start(quit)
+	emitter := NewEmitter()
+	go emitter.Start(plugins, Settings.Middleware)
 
 	for i := 0; i < b.N; i++ {
 		wg.Add(1)
@@ -149,6 +205,34 @@ func BenchmarkHTTPOutput(b *testing.B) {
 	}
 
 	wg.Wait()
+	emitter.Close()
+}
 
-	close(quit)
+func BenchmarkHTTPOutputTLS(b *testing.B) {
+	wg := new(sync.WaitGroup)
+
+	server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		wg.Done()
+	}))
+	defer server.Close()
+
+	input := NewTestInput()
+	output := NewHTTPOutput(server.URL, &HTTPOutputConfig{SkipVerify: true, WorkersMax: 1})
+
+	plugins := &InOutPlugins{
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{output},
+	}
+	plugins.All = append(plugins.All, input, output)
+
+	emitter := NewEmitter()
+	go emitter.Start(plugins, Settings.Middleware)
+
+	for i := 0; i < b.N; i++ {
+		wg.Add(1)
+		input.EmitPOST()
+	}
+
+	wg.Wait()
+	emitter.Close()
 }

output_kafka.go

@@ -2,38 +2,40 @@ package main
 
 import (
 	"encoding/json"
-	"github.com/Shopify/sarama"
-	"github.com/Shopify/sarama/mocks"
-	"github.com/buger/gor/proto"
-	"io"
 	"log"
 	"strings"
 	"time"
+
+	"github.com/buger/goreplay/byteutils"
+	"github.com/buger/goreplay/proto"
+
+	"github.com/Shopify/sarama"
+	"github.com/Shopify/sarama/mocks"
 )
 
 // KafkaOutput is used for sending payloads to kafka in JSON format.
 type KafkaOutput struct {
-	config   *KafkaConfig
+	config   *OutputKafkaConfig
 	producer sarama.AsyncProducer
 }
 
 // KafkaOutputFrequency in milliseconds
 const KafkaOutputFrequency = 500
 
-// NewKafkaOutput creates instance of kafka producer client.
-func NewKafkaOutput(address string, config *KafkaConfig) io.Writer {
-	c := sarama.NewConfig()
+// NewKafkaOutput creates instance of kafka producer client  with TLS config
+func NewKafkaOutput(address string, config *OutputKafkaConfig, tlsConfig *KafkaTLSConfig) PluginWriter {
+	c := NewKafkaConfig(tlsConfig)
 
 	var producer sarama.AsyncProducer
 
-	if config.producer.(*mocks.AsyncProducer) != nil {
+	if mock, ok := config.producer.(*mocks.AsyncProducer); ok && mock != nil {
 		producer = config.producer
 	} else {
 		c.Producer.RequiredAcks = sarama.WaitForLocal
 		c.Producer.Compression = sarama.CompressionSnappy
 		c.Producer.Flush.Frequency = KafkaOutputFrequency * time.Millisecond
 
-		brokerList := strings.Split(config.host, ",")
+		brokerList := strings.Split(config.Host, ",")
 
 		var err error
 		producer, err = sarama.NewAsyncProducer(brokerList, c)
@@ -47,10 +49,8 @@ func NewKafkaOutput(address string, config *KafkaConfig) io.Writer {
 		producer: producer,
 	}
 
-	if Settings.verbose {
-		// Start infinite loop for tracking errors for kafka producer.
-		go o.ErrorHandler()
-	}
+	// Start infinite loop for tracking errors for kafka producer.
+	go o.ErrorHandler()
 
 	return o
 }
@@ -58,40 +58,41 @@ func NewKafkaOutput(address string, config *KafkaConfig) io.Writer {
 // ErrorHandler should receive errors
 func (o *KafkaOutput) ErrorHandler() {
 	for err := range o.producer.Errors() {
-		log.Println("Failed to write access log entry:", err)
+		Debug(1, "Failed to write access log entry:", err)
 	}
 }
 
-func (o *KafkaOutput) Write(data []byte) (n int, err error) {
+// PluginWrite writes a message to this plugin
+func (o *KafkaOutput) PluginWrite(msg *Message) (n int, err error) {
 	var message sarama.StringEncoder
 
-	if !o.config.useJSON {
-		message = sarama.StringEncoder(data)
+	if !o.config.UseJSON {
+		message = sarama.StringEncoder(byteutils.SliceToString(msg.Meta) + byteutils.SliceToString(msg.Data))
 	} else {
-		headers := make(map[string]string)
-		proto.ParseHeaders([][]byte{data}, func(header []byte, value []byte) bool {
-			headers[string(header)] = string(value)
-			return true
-		})
+		mimeHeader := proto.ParseHeaders(msg.Data)
+		header := make(map[string]string)
+		for k, v := range mimeHeader {
+			header[k] = strings.Join(v, ", ")
+		}
 
-		meta := payloadMeta(data)
-		req := payloadBody(data)
+		meta := payloadMeta(msg.Meta)
+		req := msg.Data
 
 		kafkaMessage := KafkaMessage{
-			ReqURL:     string(proto.Path(req)),
-			ReqType:    string(meta[0]),
-			ReqID:      string(meta[1]),
-			ReqTs:      string(meta[2]),
-			ReqMethod:  string(proto.Method(req)),
-			ReqBody:    string(proto.Body(req)),
-			ReqHeaders: headers,
+			ReqURL:     byteutils.SliceToString(proto.Path(req)),
+			ReqType:    byteutils.SliceToString(meta[0]),
+			ReqID:      byteutils.SliceToString(meta[1]),
+			ReqTs:      byteutils.SliceToString(meta[2]),
+			ReqMethod:  byteutils.SliceToString(proto.Method(req)),
+			ReqBody:    byteutils.SliceToString(proto.Body(req)),
+			ReqHeaders: header,
 		}
 		jsonMessage, _ := json.Marshal(&kafkaMessage)
-		message = sarama.StringEncoder(jsonMessage)
+		message = sarama.StringEncoder(byteutils.SliceToString(jsonMessage))
 	}
 
 	o.producer.Input() <- &sarama.ProducerMessage{
-		Topic: o.config.topic,
+		Topic: o.config.Topic,
 		Value: message,
 	}
 

output_kafka_test.go

@@ -1,9 +1,10 @@
 package main
 
 import (
+	"testing"
+
 	"github.com/Shopify/sarama"
 	"github.com/Shopify/sarama/mocks"
-	"testing"
 )
 
 func TestOutputKafkaRAW(t *testing.T) {
@@ -12,20 +13,20 @@ func TestOutputKafkaRAW(t *testing.T) {
 	producer := mocks.NewAsyncProducer(t, config)
 	producer.ExpectInputAndSucceed()
 
-	output := NewKafkaOutput("", &KafkaConfig{
+	output := NewKafkaOutput("", &OutputKafkaConfig{
 		producer: producer,
-		topic:    "test",
-		useJSON:  false,
-	})
+		Topic:    "test",
+		UseJSON:  false,
+	}, nil)
 
-	output.Write([]byte("1 2 3\nGET / HTTP1.1\r\nHeader: 1\r\n\r\n"))
+	output.PluginWrite(&Message{Meta: []byte("1 2 3\n"), Data: []byte("GET / HTTP1.1\r\nHeader: 1\r\n\r\n")})
 
 	resp := <-producer.Successes()
 
 	data, _ := resp.Value.Encode()
 
 	if string(data) != "1 2 3\nGET / HTTP1.1\r\nHeader: 1\r\n\r\n" {
-		t.Error("Message not properly encoded: ", string(data))
+		t.Errorf("Message not properly encoded: %q", data)
 	}
 }
 
@@ -35,19 +36,19 @@ func TestOutputKafkaJSON(t *testing.T) {
 	producer := mocks.NewAsyncProducer(t, config)
 	producer.ExpectInputAndSucceed()
 
-	output := NewKafkaOutput("", &KafkaConfig{
+	output := NewKafkaOutput("", &OutputKafkaConfig{
 		producer: producer,
-		topic:    "test",
-		useJSON:  true,
-	})
+		Topic:    "test",
+		UseJSON:  true,
+	}, nil)
 
-	output.Write([]byte("1 2 3\nGET / HTTP1.1\r\nHeader: 1\r\n\r\n"))
+	output.PluginWrite(&Message{Meta: []byte("1 2 3\n"), Data: []byte("GET / HTTP1.1\r\nHeader: 1\r\n\r\n")})
 
 	resp := <-producer.Successes()
 
 	data, _ := resp.Value.Encode()
 
-	if string(data) != `{"Req_URL":"/","Req_Type":"1","Req_ID":"2","Req_Ts":"3","Req_Method":"GET","Req_Headers":{"Header":"1"}}` {
+	if string(data) != `{"Req_URL":"","Req_Type":"1","Req_ID":"2","Req_Ts":"3","Req_Method":"GET"}` {
 		t.Error("Message not properly encoded: ", string(data))
 	}
 }

output_null.go

@@ -4,13 +4,14 @@ package main
 type NullOutput struct {
 }
 
-// NullOutput constructor for NullOutput
+// NewNullOutput constructor for NullOutput
 func NewNullOutput() (o *NullOutput) {
 	return new(NullOutput)
 }
 
-func (o *NullOutput) Write(data []byte) (int, error) {
-	return len(data), nil
+// PluginWrite writes message to this plugin
+func (o *NullOutput) PluginWrite(msg *Message) (int, error) {
+	return len(msg.Data) + len(msg.Meta), nil
 }
 
 func (o *NullOutput) String() string {

output_s3.go

@@ -0,0 +1,131 @@
+package main
+
+import (
+	_ "bufio"
+	"fmt"
+	_ "io"
+	"log"
+	"math/rand"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/s3"
+	_ "github.com/aws/aws-sdk-go/service/s3/s3manager"
+)
+
+// S3Output output plugin
+type S3Output struct {
+	pathTemplate string
+
+	buffer  *FileOutput
+	session *session.Session
+	config  *FileOutputConfig
+	closeCh chan struct{}
+}
+
+// NewS3Output constructor for FileOutput, accepts path
+func NewS3Output(pathTemplate string, config *FileOutputConfig) *S3Output {
+	if !PRO {
+		log.Fatal("Using S3 output and input requires PRO license")
+		return nil
+	}
+
+	o := new(S3Output)
+	o.pathTemplate = pathTemplate
+	o.config = config
+	o.config.onClose = o.onBufferUpdate
+
+	if config.BufferPath == "" {
+		config.BufferPath = "/tmp"
+	}
+
+	rnd := rand.Int63()
+	bufferName := fmt.Sprintf("gor_output_s3_%d_buf_", rnd)
+
+	pathParts := strings.Split(pathTemplate, "/")
+	bufferName += pathParts[len(pathParts)-1]
+
+	if strings.HasSuffix(o.pathTemplate, ".gz") {
+		bufferName += ".gz"
+	}
+
+	bufferPath := filepath.Join(config.BufferPath, bufferName)
+
+	o.buffer = NewFileOutput(bufferPath, config)
+	o.connect()
+
+	return o
+}
+
+func (o *S3Output) connect() {
+	if o.session == nil {
+		o.session = session.Must(session.NewSession(awsConfig()))
+		log.Println("[S3 Output] S3 connection successfully initialized")
+	}
+}
+
+// PluginWrite writes message to this plugin
+func (o *S3Output) PluginWrite(msg *Message) (n int, err error) {
+	return o.buffer.PluginWrite(msg)
+}
+
+func (o *S3Output) String() string {
+	return "S3 output: " + o.pathTemplate
+}
+
+// Close close the buffer of the S3 connection
+func (o *S3Output) Close() error {
+	return o.buffer.Close()
+}
+
+func parseS3Url(path string) (bucket, key string) {
+	path = path[5:] // stripping `s3://`
+	sep := strings.IndexByte(path, '/')
+
+	bucket = path[:sep]
+	key = path[sep+1:]
+
+	return bucket, key
+}
+
+func (o *S3Output) keyPath(idx int) (bucket, key string) {
+	bucket, key = parseS3Url(o.pathTemplate)
+
+	for name, fn := range dateFileNameFuncs {
+		key = strings.Replace(key, name, fn(o.buffer), -1)
+	}
+
+	key = setFileIndex(key, idx)
+
+	return
+}
+
+func (o *S3Output) onBufferUpdate(path string) {
+	svc := s3.New(o.session)
+	idx := getFileIndex(path)
+	bucket, key := o.keyPath(idx)
+
+	file, err := os.Open(path)
+	if err != nil {
+		Debug(0, fmt.Sprintf("[S3 Output] Failed to open file %q. err: %q", path, err))
+		return
+	}
+	defer os.Remove(path)
+
+	_, err = svc.PutObject(&s3.PutObjectInput{
+		Body:   file,
+		Bucket: aws.String(bucket),
+		Key:    aws.String(key),
+	})
+	if err != nil {
+		Debug(0, fmt.Sprintf("[S3 Output] Failed to upload data to %q/%q, %q", bucket, key, err))
+		return
+	}
+
+	if o.closeCh != nil {
+		o.closeCh <- struct{}{}
+	}
+}

output_tcp.go

@@ -1,51 +1,71 @@
 package main
 
 import (
+	"context"
+	"crypto/tls"
 	"fmt"
-	"io"
-	"log"
+	"hash/fnv"
 	"net"
 	"time"
 )
 
 // TCPOutput used for sending raw tcp payloads
 // Currently used for internal communication between listener and replay server
-// Can be used for transfering binary payloads like protocol buffers
+// Can be used for transferring binary payloads like protocol buffers
 type TCPOutput struct {
-	address  string
-	limit    int
-	buf      chan []byte
-	bufStats *GorStat
+	address     string
+	limit       int
+	buf         []chan *Message
+	bufStats    *GorStat
+	config      *TCPOutputConfig
+	workerIndex uint32
+
+	close bool
+}
+
+// TCPOutputConfig tcp output configuration
+type TCPOutputConfig struct {
+	Secure     bool `json:"output-tcp-secure"`
+	Sticky     bool `json:"output-tcp-sticky"`
+	SkipVerify bool `json:"output-tcp-skip-verify"`
+	Workers    int  `json:"output-tcp-workers"`
 }
 
 // NewTCPOutput constructor for TCPOutput
-// Initialize 10 workers which hold keep-alive connection
-func NewTCPOutput(address string) io.Writer {
+// Initialize X workers which hold keep-alive connection
+func NewTCPOutput(address string, config *TCPOutputConfig) PluginWriter {
 	o := new(TCPOutput)
 
 	o.address = address
+	o.config = config
 
-	o.buf = make(chan []byte, 100)
-	if Settings.outputTCPStats {
-		o.bufStats = NewGorStat("output_tcp")
+	if Settings.OutputTCPStats {
+		o.bufStats = NewGorStat("output_tcp", 5000)
 	}
 
-	for i := 0; i < 10; i++ {
-		go o.worker()
+	// create X buffers and send the buffer index to the worker
+	o.buf = make([]chan *Message, o.config.Workers)
+	for i := 0; i < o.config.Workers; i++ {
+		o.buf[i] = make(chan *Message, 100)
+		go o.worker(i)
 	}
 
 	return o
 }
 
-func (o *TCPOutput) worker() {
-	retries := 1
+func (o *TCPOutput) worker(bufferIndex int) {
+	retries := 0
 	conn, err := o.connect(o.address)
 	for {
+		if o.close {
+			return
+		}
+
 		if err == nil {
 			break
 		}
 
-		log.Println("Can't connect to aggregator instance, reconnecting in 1 second. Retries:", retries)
+		Debug(1, fmt.Sprintf("Can't connect to aggregator instance, reconnecting in 1 second. Retries:%d", retries))
 		time.Sleep(1 * time.Second)
 
 		conn, err = o.connect(o.address)
@@ -53,43 +73,69 @@ func (o *TCPOutput) worker() {
 	}
 
 	if retries > 0 {
-		log.Println("Connected to aggregator instance after ", retries, " retries")
+		Debug(2, fmt.Sprintf("Connected to aggregator instance after %d retries", retries))
 	}
 
 	defer conn.Close()
 
 	for {
-		conn.Write(<-o.buf)
-		_, err := conn.Write([]byte(payloadSeparator))
+		msg := <-o.buf[bufferIndex]
+		if _, err = conn.Write(msg.Meta); err == nil {
+			if _, err = conn.Write(msg.Data); err == nil {
+				_, err = conn.Write(payloadSeparatorAsBytes)
+			}
+		}
 
 		if err != nil {
-			log.Println("Lost connection with aggregator instance, reconnecting")
-			go o.worker()
+			Debug(2, "INFO: TCP output connection closed, reconnecting")
+			o.buf[bufferIndex] <- msg
+			go o.worker(bufferIndex)
 			break
 		}
 	}
 }
 
-func (o *TCPOutput) Write(data []byte) (n int, err error) {
-	if !isOriginPayload(data) {
-		return len(data), nil
+func (o *TCPOutput) getBufferIndex(data []byte) int {
+	if !o.config.Sticky {
+		o.workerIndex++
+		return int(o.workerIndex) % o.config.Workers
 	}
 
-	// We have to copy, because sending data in multiple threads
-	newBuf := make([]byte, len(data))
-	copy(newBuf, data)
+	hasher := fnv.New32a()
+	hasher.Write(payloadMeta(data)[1])
+	return int(hasher.Sum32()) % o.config.Workers
 
-	o.buf <- newBuf
+}
 
-	if Settings.outputTCPStats {
-		o.bufStats.Write(len(o.buf))
+// PluginWrite writes message to this plugin
+func (o *TCPOutput) PluginWrite(msg *Message) (n int, err error) {
+	if !isOriginPayload(msg.Meta) {
+		return len(msg.Data), nil
 	}
 
-	return len(data), nil
+	bufferIndex := o.getBufferIndex(msg.Data)
+	o.buf[bufferIndex] <- msg
+
+	if Settings.OutputTCPStats {
+		o.bufStats.Write(len(o.buf[bufferIndex]))
+	}
+
+	return len(msg.Data) + len(msg.Meta), nil
 }
 
 func (o *TCPOutput) connect(address string) (conn net.Conn, err error) {
-	conn, err = net.Dial("tcp", address)
+	if o.config.Secure {
+		var d tls.Dialer
+		d.Config = &tls.Config{InsecureSkipVerify: o.config.SkipVerify}
+		ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+		defer cancel()
+		conn, err = d.DialContext(ctx, "tcp", address)
+	} else {
+		var d net.Dialer
+		ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+		defer cancel()
+		conn, err = d.DialContext(ctx, "tcp", address)
+	}
 
 	return
 }
@@ -97,3 +143,7 @@ func (o *TCPOutput) connect(address string) (conn net.Conn, err error) {
 func (o *TCPOutput) String() string {
 	return fmt.Sprintf("TCP output %s, limit: %d", o.address, o.limit)
 }
+
+func (o *TCPOutput) Close() {
+	o.close = true
+}

output_tcp_test.go

@@ -2,36 +2,37 @@ package main
 
 import (
 	"bufio"
-	"io"
 	"log"
 	"net"
 	"sync"
 	"testing"
+	"time"
 )
 
 func TestTCPOutput(t *testing.T) {
 	wg := new(sync.WaitGroup)
-	quit := make(chan int)
 
 	listener := startTCP(func(data []byte) {
 		wg.Done()
 	})
 	input := NewTestInput()
-	output := NewTCPOutput(listener.Addr().String())
+	output := NewTCPOutput(listener.Addr().String(), &TCPOutputConfig{Workers: 10})
 
-	Plugins.Inputs = []io.Reader{input}
-	Plugins.Outputs = []io.Writer{output}
+	plugins := &InOutPlugins{
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{output},
+	}
 
-	go Start(quit)
+	emitter := NewEmitter()
+	go emitter.Start(plugins, Settings.Middleware)
 
-	for i := 0; i < 100; i++ {
+	for i := 0; i < 10; i++ {
 		wg.Add(1)
 		input.EmitGET()
 	}
 
 	wg.Wait()
-
-	close(quit)
+	emitter.Close()
 }
 
 func startTCP(cb func([]byte)) net.Listener {
@@ -44,9 +45,9 @@ func startTCP(cb func([]byte)) net.Listener {
 	go func() {
 		for {
 			conn, _ := listener.Accept()
-			defer conn.Close()
 
-			go func() {
+			go func(conn net.Conn) {
+				defer conn.Close()
 				reader := bufio.NewReader(conn)
 				scanner := bufio.NewScanner(reader)
 				scanner.Split(payloadScanner)
@@ -54,7 +55,7 @@ func startTCP(cb func([]byte)) net.Listener {
 				for scanner.Scan() {
 					cb(scanner.Bytes())
 				}
-			}()
+			}(conn)
 		}
 	}()
 
@@ -63,26 +64,69 @@ func startTCP(cb func([]byte)) net.Listener {
 
 func BenchmarkTCPOutput(b *testing.B) {
 	wg := new(sync.WaitGroup)
-	quit := make(chan int)
 
 	listener := startTCP(func(data []byte) {
 		wg.Done()
 	})
 	input := NewTestInput()
-	output := NewTCPOutput(listener.Addr().String())
-
-	Plugins.Inputs = []io.Reader{input}
-	Plugins.Outputs = []io.Writer{output}
-
-	go Start(quit)
-
-	b.ResetTimer()
+	input.data = make(chan []byte, b.N)
 	for i := 0; i < b.N; i++ {
-		wg.Add(1)
 		input.EmitGET()
 	}
+	wg.Add(b.N)
+	output := NewTCPOutput(listener.Addr().String(), &TCPOutputConfig{Workers: 10})
+
+	plugins := &InOutPlugins{
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{output},
+	}
+
+	emitter := NewEmitter()
+	// avoid counting above initialization
+	b.ResetTimer()
+	go emitter.Start(plugins, Settings.Middleware)
 
 	wg.Wait()
+	emitter.Close()
+}
 
-	close(quit)
+func TestStickyDisable(t *testing.T) {
+	tcpOutput := TCPOutput{config: &TCPOutputConfig{Sticky: false, Workers: 10}}
+
+	for i := 0; i < 10; i++ {
+		index := tcpOutput.getBufferIndex(getTestBytes())
+		if index != (i+1)%10 {
+			t.Errorf("Sticky is disable. Got: %d want %d", index, (i+1)%10)
+		}
+	}
+}
+
+func TestBufferDistribution(t *testing.T) {
+	numberOfWorkers := 10
+	numberOfMessages := 10000
+	percentDistributionErrorRange := 20
+
+	buffer := make([]int, numberOfWorkers)
+	tcpOutput := TCPOutput{config: &TCPOutputConfig{Sticky: true, Workers: 10}}
+	for i := 0; i < numberOfMessages; i++ {
+		buffer[tcpOutput.getBufferIndex(getTestBytes())]++
+	}
+
+	expectedDistribution := numberOfMessages / numberOfWorkers
+	lowerDistribution := expectedDistribution - (expectedDistribution * percentDistributionErrorRange / 100)
+	upperDistribution := expectedDistribution + (expectedDistribution * percentDistributionErrorRange / 100)
+	for i := 0; i < numberOfWorkers; i++ {
+		if buffer[i] < lowerDistribution {
+			t.Errorf("Under expected distribution. Got %d expected lower distribution %d", buffer[i], lowerDistribution)
+		}
+		if buffer[i] > upperDistribution {
+			t.Errorf("Under expected distribution. Got %d expected upper distribution %d", buffer[i], upperDistribution)
+		}
+	}
+}
+
+func getTestBytes() []byte {
+	reqh := payloadHeader(RequestPayload, uuid(), time.Now().UnixNano(), -1)
+	reqb := append(reqh, []byte("GET / HTTP/1.1\r\nHost: www.w3.org\r\nUser-Agent: Go 1.1 package http\r\nAccept-Encoding: gzip\r\n\r\n")...)
+	return reqb
 }

plugins.go

@@ -1,25 +1,39 @@
 package main
 
 import (
-	"io"
 	"reflect"
 	"strings"
-	"sync"
-	"time"
 )
 
+// Message represents data across plugins
+type Message struct {
+	Meta []byte // metadata
+	Data []byte // actual data
+}
+
+// PluginReader is an interface for input plugins
+type PluginReader interface {
+	PluginRead() (msg *Message, err error)
+}
+
+// PluginWriter is an interface for output plugins
+type PluginWriter interface {
+	PluginWrite(msg *Message) (n int, err error)
+}
+
+// PluginReadWriter is an interface for plugins that support reading and writing
+type PluginReadWriter interface {
+	PluginReader
+	PluginWriter
+}
+
 // InOutPlugins struct for holding references to plugins
 type InOutPlugins struct {
-	Inputs  []io.Reader
-	Outputs []io.Writer
+	Inputs  []PluginReader
+	Outputs []PluginWriter
 	All     []interface{}
 }
 
-var pluginMu sync.Mutex
-
-// Plugins holds all the plugin objects
-var Plugins *InOutPlugins = new(InOutPlugins)
-
 // extractLimitOptions detects if plugin get called with limiter support
 // Returns address and limit
 func extractLimitOptions(options string) (string, string) {
@@ -34,8 +48,8 @@ func extractLimitOptions(options string) (string, string) {
 
 // Automatically detects type of plugin and initialize it
 //
-// See this article if curious about relfect stuff below: http://blog.burntsushi.net/type-parametric-functions-golang
-func registerPlugin(constructor interface{}, options ...interface{}) {
+// See this article if curious about reflect stuff below: http://blog.burntsushi.net/type-parametric-functions-golang
+func (plugins *InOutPlugins) registerPlugin(constructor interface{}, options ...interface{}) {
 	var path, limit string
 	vc := reflect.ValueOf(constructor)
 
@@ -55,99 +69,95 @@ func registerPlugin(constructor interface{}, options ...interface{}) {
 
 	// Calling our constructor with list of given options
 	plugin := vc.Call(vo)[0].Interface()
-	pluginWrapper := plugin
 
 	if limit != "" {
-		pluginWrapper = NewLimiter(plugin, limit)
-	} else {
-		pluginWrapper = plugin
+		plugin = NewLimiter(plugin, limit)
 	}
 
-	_, isR := plugin.(io.Reader)
-	_, isW := plugin.(io.Writer)
-
 	// Some of the output can be Readers as well because return responses
-	if isR && !isW {
-		Plugins.Inputs = append(Plugins.Inputs, pluginWrapper.(io.Reader))
+	if r, ok := plugin.(PluginReader); ok {
+		plugins.Inputs = append(plugins.Inputs, r)
 	}
 
-	if isW {
-		Plugins.Outputs = append(Plugins.Outputs, pluginWrapper.(io.Writer))
+	if w, ok := plugin.(PluginWriter); ok {
+		plugins.Outputs = append(plugins.Outputs, w)
 	}
+	plugins.All = append(plugins.All, plugin)
 
-	Plugins.All = append(Plugins.All, plugin)
 }
 
-// InitPlugins specify and initialize all available plugins
-func InitPlugins() {
-	pluginMu.Lock()
-	defer pluginMu.Unlock()
+// NewPlugins specify and initialize all available plugins
+func NewPlugins() *InOutPlugins {
+	plugins := new(InOutPlugins)
 
-	for _, options := range Settings.inputDummy {
-		registerPlugin(NewDummyInput, options)
+	for _, options := range Settings.InputDummy {
+		plugins.registerPlugin(NewDummyInput, options)
 	}
 
-	for range Settings.outputDummy {
-		registerPlugin(NewDummyOutput)
+	for range Settings.OutputDummy {
+		plugins.registerPlugin(NewDummyOutput)
 	}
 
-	if Settings.outputStdout {
-		registerPlugin(NewDummyOutput)
+	if Settings.OutputStdout {
+		plugins.registerPlugin(NewDummyOutput)
 	}
 
-	if Settings.outputNull {
-		registerPlugin(NewNullOutput)
+	if Settings.OutputNull {
+		plugins.registerPlugin(NewNullOutput)
 	}
 
-	engine := EnginePcap
-	if Settings.inputRAWEngine == "raw_socket" {
-		engine = EngineRawSocket
-	} else if Settings.inputRAWEngine == "pcap_file" {
-		engine = EnginePcapFile
+	for _, options := range Settings.InputRAW {
+		plugins.registerPlugin(NewRAWInput, options, Settings.RAWInputConfig)
 	}
 
-	for _, options := range Settings.inputRAW {
-		registerPlugin(NewRAWInput, options, engine, Settings.inputRAWTrackResponse, time.Duration(0), Settings.inputRAWRealIPHeader)
+	for _, options := range Settings.InputTCP {
+		plugins.registerPlugin(NewTCPInput, options, &Settings.InputTCPConfig)
 	}
 
-	for _, options := range Settings.inputTCP {
-		registerPlugin(NewTCPInput, options)
+	for _, options := range Settings.OutputTCP {
+		plugins.registerPlugin(NewTCPOutput, options, &Settings.OutputTCPConfig)
 	}
 
-	for _, options := range Settings.outputTCP {
-		registerPlugin(NewTCPOutput, options)
+	for _, options := range Settings.InputFile {
+		plugins.registerPlugin(NewFileInput, options, Settings.InputFileLoop)
 	}
 
-	for _, options := range Settings.inputFile {
-		registerPlugin(NewFileInput, options, Settings.inputFileLoop)
+	for _, path := range Settings.OutputFile {
+		if strings.HasPrefix(path, "s3://") {
+			plugins.registerPlugin(NewS3Output, path, &Settings.OutputFileConfig)
+		} else {
+			plugins.registerPlugin(NewFileOutput, path, &Settings.OutputFileConfig)
+		}
 	}
 
-	for _, options := range Settings.outputFile {
-		registerPlugin(NewFileOutput, options, &Settings.outputFileConfig)
-	}
-
-	for _, options := range Settings.inputHTTP {
-		registerPlugin(NewHTTPInput, options)
+	for _, options := range Settings.InputHTTP {
+		plugins.registerPlugin(NewHTTPInput, options)
 	}
 
 	// If we explicitly set Host header http output should not rewrite it
 	// Fix: https://github.com/buger/gor/issues/174
-	for _, header := range Settings.modifierConfig.headers {
+	for _, header := range Settings.ModifierConfig.Headers {
 		if header.Name == "Host" {
-			Settings.outputHTTPConfig.OriginalHost = true
+			Settings.OutputHTTPConfig.OriginalHost = true
 			break
 		}
 	}
 
-	for _, options := range Settings.outputHTTP {
-		registerPlugin(NewHTTPOutput, options, &Settings.outputHTTPConfig)
+	for _, options := range Settings.OutputHTTP {
+		plugins.registerPlugin(NewHTTPOutput, options, &Settings.OutputHTTPConfig)
 	}
 
-	if Settings.outputKafkaConfig.host != "" && Settings.outputKafkaConfig.topic != "" {
-		registerPlugin(NewKafkaOutput, "", &Settings.outputKafkaConfig)
+	for _, options := range Settings.OutputBinary {
+		plugins.registerPlugin(NewBinaryOutput, options, &Settings.OutputBinaryConfig)
 	}
 
-	if Settings.inputKafkaConfig.host != "" && Settings.inputKafkaConfig.topic != "" {
-		registerPlugin(NewKafkaInput, "", &Settings.inputKafkaConfig)
+	if Settings.OutputKafkaConfig.Host != "" && Settings.OutputKafkaConfig.Topic != "" {
+		plugins.registerPlugin(NewKafkaOutput, "", &Settings.OutputKafkaConfig, &Settings.KafkaTLSConfig)
 	}
+
+	if Settings.InputKafkaConfig.Host != "" && Settings.InputKafkaConfig.Topic != "" {
+		plugins.registerPlugin(NewKafkaInput, "", &Settings.InputKafkaConfig, &Settings.KafkaTLSConfig)
+	}
+
+	return plugins
 }

plugins_test.go

@@ -1,42 +1,38 @@
 package main
 
 import (
-	"io"
 	"testing"
 )
 
 func TestPluginsRegistration(t *testing.T) {
-	Plugins.Inputs = []io.Reader{}
-	Plugins.Outputs = []io.Writer{}
+	Settings.InputDummy = MultiOption{"[]"}
+	Settings.OutputDummy = MultiOption{"[]"}
+	Settings.OutputHTTP = MultiOption{"www.example.com|10"}
+	Settings.InputFile = MultiOption{"/dev/null"}
 
-	Settings.inputDummy = MultiOption{"[]"}
-	Settings.outputDummy = MultiOption{"[]"}
-	Settings.outputHTTP = MultiOption{"www.example.com|10"}
-	Settings.inputFile = MultiOption{"/dev/null"}
+	plugins := NewPlugins()
 
-	InitPlugins()
-
-	if len(Plugins.Inputs) != 2 {
-		t.Errorf("Should be 2 inputs %d", len(Plugins.Inputs))
+	if len(plugins.Inputs) != 3 {
+		t.Errorf("Should be 3 inputs got %d", len(plugins.Inputs))
 	}
 
-	if _, ok := Plugins.Inputs[0].(*DummyInput); !ok {
+	if _, ok := plugins.Inputs[0].(*DummyInput); !ok {
 		t.Errorf("First input should be DummyInput")
 	}
 
-	if _, ok := Plugins.Inputs[1].(*FileInput); !ok {
+	if _, ok := plugins.Inputs[1].(*FileInput); !ok {
 		t.Errorf("Second input should be FileInput")
 	}
 
-	if len(Plugins.Outputs) != 2 {
-		t.Errorf("Should be 2 output %d", len(Plugins.Outputs))
+	if len(plugins.Outputs) != 2 {
+		t.Errorf("Should be 2 output %d", len(plugins.Outputs))
 	}
 
-	if _, ok := Plugins.Outputs[0].(*DummyOutput); !ok {
+	if _, ok := plugins.Outputs[0].(*DummyOutput); !ok {
 		t.Errorf("First output should be DummyOutput")
 	}
 
-	if l, ok := Plugins.Outputs[1].(*Limiter); ok {
+	if l, ok := plugins.Outputs[1].(*Limiter); ok {
 		if _, ok := l.plugin.(*HTTPOutput); !ok {
 			t.Errorf("HTTPOutput should be wrapped in limiter")
 		}

pro.go

@@ -0,0 +1,5 @@
+package main
+
+// PRO this value indicates if goreplay is running in PRO mode..
+// it must not be modified explicitly in production
+var PRO = false

proto/fuzz.go

@@ -4,9 +4,7 @@ package proto
 
 func Fuzz(data []byte) int {
 
-    ParseHeaders([][]byte{data}, func(header []byte, value []byte) bool {
-      return true
-    })
+	ParseHeaders(data)
 
-    return 1
+	return 1
 }

proto/proto.go

@@ -17,274 +17,131 @@ Example of HTTP payload for future references, new line symbols escaped:
 package proto
 
 import (
+	"bufio"
 	"bytes"
-	"github.com/buger/gor/byteutils"
+	_ "fmt"
+	"net/http"
+	"net/textproto"
+	"strings"
+
+	"github.com/buger/goreplay/byteutils"
 )
 
-// In HTTP newline defined by 2 bytes (for both windows and *nix support)
-var CLRF = []byte("\r\n")
+// CRLF In HTTP newline defined by 2 bytes (for both windows and *nix support)
+var CRLF = []byte("\r\n")
 
-// New line acts as separator: end of Headers or Body (in some cases)
+// EmptyLine acts as separator: end of Headers or Body (in some cases)
 var EmptyLine = []byte("\r\n\r\n")
 
-// Separator for Header line. Header looks like: `HeaderName: value`
+// HeaderDelim Separator for Header line. Header looks like: `HeaderName: value`
 var HeaderDelim = []byte(": ")
 
 // MIMEHeadersEndPos finds end of the Headers section, which should end with empty line.
 func MIMEHeadersEndPos(payload []byte) int {
-	return bytes.Index(payload, EmptyLine)
+	pos := bytes.Index(payload, EmptyLine)
+	if pos < 0 {
+		return -1
+	}
+	return pos + 4
 }
 
 // MIMEHeadersStartPos finds start of Headers section
 // It just finds position of second line (first contains location and method).
 func MIMEHeadersStartPos(payload []byte) int {
-	return bytes.Index(payload, CLRF) + 2 // Find first line end
-}
-
-func isLower(b byte) bool {
-	if 'a' <= b && b <= 'z' {
-		return true
+	pos := bytes.Index(payload, CRLF)
+	if pos < 0 {
+		return -1
 	}
-
-	return false
-}
-
-func toUpper(b byte) byte {
-	if 'a' <= b && b <= 'z' {
-		b -= 'a' - 'A'
-	}
-	return b
-}
-
-func toLower(b byte) byte {
-	if 'A' <= b && b <= 'Z' {
-		b += 'a' - 'A'
-	}
-	return b
-}
-
-func headerIndex(payload []byte, name []byte) int {
-	isLower := isLower(name[0])
-	i := 0
-
-	for {
-		if i >= len(payload) {
-			return -1
-		}
-
-		if payload[i] == '\n' {
-			i++
-
-			// We are at the end
-			if i == len(payload) {
-				return -1
-			}
-
-			if payload[i] == name[0] ||
-				(!isLower && payload[i] == toLower(name[0])) ||
-				(isLower && payload[i] == toUpper(name[0])) {
-
-				i++
-				j := 1
-				for {
-					if j == len(name) {
-						// Matched, and return start of the header
-						return i - len(name)
-					}
-
-					if payload[i] != name[j] {
-						break
-					}
-
-					// If compound header name do one more case check: Content-Length or Transfer-Encoding
-					if name[j] == '-' {
-						i++
-						j++
-
-						if !(payload[i] == name[j] ||
-							(!isLower && payload[i] == toLower(name[j])) ||
-							(isLower && payload[i] == toUpper(name[j]))) {
-							break
-						}
-					}
-
-					j++
-					i++
-				}
-			}
-		}
-
-		i++
-	}
-
-	return -1
+	return pos + 2 // Find first line end
 }
 
 // header return value and positions of header/value start/end.
 // If not found, value will be blank, and headerStart will be -1
 // Do not support multi-line headers.
 func header(payload []byte, name []byte) (value []byte, headerStart, headerEnd, valueStart, valueEnd int) {
-	headerStart = headerIndex(payload, name)
+	if HasTitle(payload) {
+		headerStart = MIMEHeadersStartPos(payload)
+		if headerStart < 0 {
+			return
+		}
+	} else {
+		headerStart = 0
+	}
 
-	if headerStart == -1 {
+	var colonIndex int
+	for headerStart < len(payload) {
+		headerEnd = bytes.IndexByte(payload[headerStart:], '\n')
+		if headerEnd == -1 {
+			break
+		}
+		headerEnd += headerStart
+		colonIndex = bytes.IndexByte(payload[headerStart:headerEnd], ':')
+		if colonIndex == -1 {
+			break
+		}
+		colonIndex += headerStart
+		if bytes.EqualFold(payload[headerStart:colonIndex], name) {
+			valueStart = colonIndex + 1
+			valueEnd = headerEnd - 2
+			break
+		}
+		headerStart = headerEnd + 1 // move to the next header
+	}
+	if valueStart == 0 {
+		headerStart = -1
+		headerEnd = -1
+		valueEnd = -1
+		valueStart = -1
 		return
 	}
 
-	valueStart = headerStart + len(name) + 1 // Skip ":" after header name
-	headerEnd = valueStart + bytes.IndexByte(payload[valueStart:], '\n')
-
-	for valueStart < headerEnd { // Ignore empty space after ':'
-		if payload[valueStart] == ' ' {
+	// ignore empty space after ':'
+	for valueStart < valueEnd {
+		if payload[valueStart] < 0x21 {
 			valueStart++
 		} else {
 			break
 		}
 	}
 
-	valueEnd = valueStart + bytes.IndexByte(payload[valueStart:], '\n')
-
-	if payload[headerEnd-1] == '\r' {
-		valueEnd--
-	}
-
 	// ignore empty space at end of header value
-	for valueStart < valueEnd {
-		if payload[valueEnd-1] == ' ' {
+	for valueEnd > valueStart {
+		if payload[valueEnd] < 0x21 {
 			valueEnd--
 		} else {
 			break
 		}
 	}
-	value = payload[valueStart:valueEnd]
+	value = payload[valueStart : valueEnd+1]
 
 	return
 }
 
-// Works only with ASCII
-func HeadersEqual(h1 []byte, h2 []byte) bool {
-	if len(h1) != len(h2) {
-		return false
-	}
-
-	for i, c1 := range h1 {
-		c2 := h2[i]
-
-		switch int(c1) - int(c2) {
-		case 0, 32, -32:
-		default:
-			return false
+// ParseHeaders Parsing headers from the payload
+func ParseHeaders(p []byte) textproto.MIMEHeader {
+	// trimming off the title of the request
+	if HasTitle(p) {
+		headerStart := MIMEHeadersStartPos(p)
+		if headerStart > len(p)-1 {
+			return nil
 		}
+		p = p[headerStart:]
 	}
-
-	return true
+	headerEnd := MIMEHeadersEndPos(p)
+	if headerEnd > 1 {
+		p = p[:headerEnd]
+	}
+	return GetHeaders(p)
 }
 
-// Parsing headers from multiple payloads
-func ParseHeaders(payloads [][]byte, cb func(header []byte, value []byte) bool) {
-	hS := [2]int{0, 0} // header start
-	hE := [2]int{-1, -1} // header end
-	vS := [2]int{-1, -1} // value start
-	vE := [2]int{-1, -1} // value end
-
-	i := 0
-	pIdx := 0
-	lineBreaks := 0
-	newLineBreak := true
-
-	for {
-		if len(payloads)-1 < pIdx {
-			break
-		}
-
-		p := payloads[pIdx]
-
-		if len(p)-1 < i {
-			pIdx++
-			i = 0
-			continue
-		}
-
-		switch p[i] {
-		case '\r', '\n':
-			newLineBreak = true
-			lineBreaks++
-
-			// End of headers
-			if lineBreaks == 4 {
-				return
-			}
-
-			if lineBreaks > 1 {
-				break
-			}
-
-			vE = [2]int{pIdx, i}
-
-			if vS[1] != -1 && vE[1] != -1 &&
-				hS[1] != -1 && hE[1] != -1 {
-
-				var header, value []byte
-
-				phS, phE, pvS, pvE := payloads[hS[0]], payloads[hE[0]], payloads[vS[0]], payloads[vE[0]]
-
-				// If in same payload
-				if hS[0] == hE[0] {
-					header = phS[hS[1]:hE[1]]
-				} else {
-					header = make([]byte, len(phS)-hS[1]+hE[1])
-					copy(header, phS[hS[1]:])
-					copy(header[len(phS)-hS[1]:], phE[:hE[1]])
-				}
-
-				if vS[0] == vE[0] {
-					value = pvS[vS[1]:vE[1]]
-				} else {
-					value = make([]byte, len(pvS)-vS[1]+vE[1])
-					copy(value, pvS[vS[1]:])
-					copy(value[len(pvS)-vS[1]:], pvE[:vE[1]])
-				}
-
-				if !cb(header, value) {
-					return
-				}
-			}
-
-			// Header found, reset values
-			vS = [2]int{-1, -1}
-			vE = [2]int{-1, -1}
-			hS = [2]int{-1, -1}
-			hE = [2]int{-1, -1}
-		case ':':
-			if newLineBreak {
-				hE = [2]int{pIdx, i}
-				newLineBreak = false
-			}
-			lineBreaks = 0
-		default:
-			lineBreaks = 0
-
-			if hS[1] == -1 {
-				hS = [2]int{pIdx, i}
-				hE = [2]int{-1, -1}
-			} else {
-				if hE[1] == -1 {
-					break
-				}
-
-				if vS[1] == -1 {
-					if p[i] == ' ' {
-						break
-					}
-
-					vS = [2]int{pIdx, i}
-				}
-			}
-		}
-
-		i++
+// GetHeaders returns mime headers from the payload
+func GetHeaders(p []byte) textproto.MIMEHeader {
+	reader := textproto.NewReader(bufio.NewReader(bytes.NewReader(p)))
+	mime, err := reader.ReadMIMEHeader()
+	if err != nil {
+		return nil
 	}
-
-	return
+	return mime
 }
 
 // Header returns header value, if header not found, value will be blank
@@ -301,7 +158,7 @@ func SetHeader(payload, name, value []byte) []byte {
 
 	if hs != -1 {
 		// If header found we just replace its value
-		return byteutils.Replace(payload, vs, ve, value)
+		return byteutils.Replace(payload, vs, ve+1, value)
 	}
 
 	return AddHeader(payload, name, value)
@@ -310,38 +167,43 @@ func SetHeader(payload, name, value []byte) []byte {
 // AddHeader takes http payload and appends new header to the start of headers section
 // Returns modified request payload
 func AddHeader(payload, name, value []byte) []byte {
+	mimeStart := MIMEHeadersStartPos(payload)
+	if mimeStart < 1 {
+		return payload
+	}
 	header := make([]byte, len(name)+2+len(value)+2)
 	copy(header[0:], name)
 	copy(header[len(name):], HeaderDelim)
 	copy(header[len(name)+2:], value)
-	copy(header[len(header)-2:], CLRF)
-
-	mimeStart := MIMEHeadersStartPos(payload)
+	copy(header[len(header)-2:], CRLF)
 
 	return byteutils.Insert(payload, mimeStart, header)
 }
 
-// DelHeader takes http payload and removes header name from headers section
+// DeleteHeader takes http payload and removes header name from headers section
 // Returns modified request payload
 func DeleteHeader(payload, name []byte) []byte {
 	_, hs, he, _, _ := header(payload, name)
 	if hs != -1 {
-		newHeader := make([]byte, len(payload)-(he-hs)-1)
-		copy(newHeader[:hs], payload[:hs])
-		copy(newHeader[hs:], payload[he+1:])
-		return newHeader
+		return byteutils.Cut(payload, hs, he+1)
 	}
 	return payload
 }
 
 // Body returns request/response body
 func Body(payload []byte) []byte {
-	// 4 -> len(EMPTY_LINE)
-	return payload[MIMEHeadersEndPos(payload)+4:]
+	pos := MIMEHeadersEndPos(payload)
+	if pos == -1 || len(payload) <= pos {
+		return nil
+	}
+	return payload[pos:]
 }
 
-// Path takes payload and retuns request path: Split(firstLine, ' ')[1]
+// Path takes payload and returns request path: Split(firstLine, ' ')[1]
 func Path(payload []byte) []byte {
+	if !HasRequestTitle(payload) {
+		return nil
+	}
 	start := bytes.IndexByte(payload, ' ') + 1
 	end := bytes.IndexByte(payload[start:], ' ')
 
@@ -350,6 +212,9 @@ func Path(payload []byte) []byte {
 
 // SetPath takes payload, sets new path and returns modified payload
 func SetPath(payload, path []byte) []byte {
+	if !HasTitle(payload) {
+		return nil
+	}
 	start := bytes.IndexByte(payload, ' ') + 1
 	end := bytes.IndexByte(payload[start:], ' ')
 
@@ -360,21 +225,23 @@ func SetPath(payload, path []byte) []byte {
 func PathParam(payload, name []byte) (value []byte, valueStart, valueEnd int) {
 	path := Path(payload)
 
-	if paramStart := bytes.Index(path, append(name, '=')); paramStart != -1 {
-		valueStart := paramStart + len(name) + 1
-		paramEnd := bytes.IndexByte(path[valueStart:], '&')
-
-		// Param can end with '&' (another param), or end of line
-		if paramEnd == -1 { // It is final param
-			paramEnd = len(path)
-		} else {
-			paramEnd += valueStart
+	paramStart := -1
+	if paramStart = bytes.Index(path, append([]byte{'&'}, append(name, '=')...)); paramStart == -1 {
+		if paramStart = bytes.Index(path, append([]byte{'?'}, append(name, '=')...)); paramStart == -1 {
+			return []byte(""), -1, -1
 		}
-
-		return path[valueStart:paramEnd], valueStart, paramEnd
 	}
 
-	return []byte(""), -1, -1
+	valueStart = paramStart + len(name) + 2
+	paramEnd := bytes.IndexByte(path[valueStart:], '&')
+
+	// Param can end with '&' (another param), or end of line
+	if paramEnd == -1 { // It is final param
+		paramEnd = len(path)
+	} else {
+		paramEnd += valueStart
+	}
+	return path[valueStart:paramEnd], valueStart, paramEnd
 }
 
 // SetPathParam takes payload and updates path Query attribute
@@ -438,31 +305,294 @@ func SetHost(payload, url, host []byte) []byte {
 // Method returns HTTP method
 func Method(payload []byte) []byte {
 	end := bytes.IndexByte(payload, ' ')
+	if end == -1 {
+		return nil
+	}
 
 	return payload[:end]
 }
 
 // Status returns response status.
-// It happend to be in same position as request payload path
+// It happens to be in same position as request payload path
 func Status(payload []byte) []byte {
-	return Path(payload)
+	if !HasResponseTitle(payload) {
+		return nil
+	}
+	start := bytes.IndexByte(payload, ' ') + 1
+	// status code are in range 100-600
+	return payload[start : start+3]
 }
 
-var httpMethods []string = []string{
-	"GET ", "OPTI", "HEAD", "POST", "PUT ", "DELE", "TRAC", "CONN", "PATC" /* custom methods */, "BAN", "PURG",
+// Methods holds the http methods ordered in ascending order
+var Methods = [...]string{
+	http.MethodConnect, http.MethodDelete, http.MethodGet,
+	http.MethodHead, http.MethodOptions, http.MethodPatch,
+	http.MethodPost, http.MethodPut, http.MethodTrace,
 }
 
-func IsHTTPPayload(payload []byte) bool {
-	if len(payload) < 4 {
+const (
+	//MinRequestCount GET / HTTP/1.1\r\n
+	MinRequestCount = 16
+	// MinResponseCount HTTP/1.1 200\r\n
+	MinResponseCount = 14
+	// VersionLen HTTP/1.1
+	VersionLen = 8
+)
+
+// HasResponseTitle reports whether this payload has an HTTP/1 response title
+func HasResponseTitle(payload []byte) bool {
+	s := byteutils.SliceToString(payload)
+	if len(s) < MinResponseCount {
+		return false
+	}
+	titleLen := bytes.Index(payload, CRLF)
+	if titleLen == -1 {
+		return false
+	}
+	major, minor, ok := http.ParseHTTPVersion(s[0:VersionLen])
+	if !(ok && major == 1 && (minor == 0 || minor == 1)) {
+		return false
+	}
+	if s[VersionLen] != ' ' {
+		return false
+	}
+	status, ok := atoI(payload[VersionLen+1:VersionLen+4], 10)
+	if !ok {
+		return false
+	}
+	// only validate status codes mentioned in rfc2616.
+	if http.StatusText(status) == "" {
+		return false
+	}
+	// handle cases from #875
+	return payload[VersionLen+4] == ' ' || payload[VersionLen+4] == '\r'
+}
+
+// HasRequestTitle reports whether this payload has an HTTP/1 request title
+func HasRequestTitle(payload []byte) bool {
+	s := byteutils.SliceToString(payload)
+	if len(s) < MinRequestCount {
+		return false
+	}
+	titleLen := bytes.Index(payload, CRLF)
+	if titleLen == -1 {
+		return false
+	}
+	if strings.Count(s[:titleLen], " ") != 2 {
+		return false
+	}
+	method := string(Method(payload))
+	var methodFound bool
+	for _, m := range Methods {
+		if methodFound = method == m; methodFound {
+			break
+		}
+	}
+	if !methodFound {
+		return false
+	}
+	path := strings.Index(s[len(method)+1:], " ")
+	if path == -1 {
+		return false
+	}
+	major, minor, ok := http.ParseHTTPVersion(s[path+len(method)+2 : titleLen])
+	return ok && major == 1 && (minor == 0 || minor == 1)
+}
+
+// HasTitle reports if this payload has an http/1 title
+func HasTitle(payload []byte) bool {
+	return HasRequestTitle(payload) || HasResponseTitle(payload)
+}
+
+// CheckChunked checks HTTP/1 chunked data integrity(https://tools.ietf.org/html/rfc7230#section-4.1)
+// and returns the length of total valid scanned chunks(including chunk size, extensions and CRLFs) and
+// full is true if all chunks was scanned.
+func CheckChunked(bufs ...[]byte) (chunkEnd int, full bool) {
+	var buf []byte
+	if len(bufs) > 0 {
+		buf = bufs[0]
+	}
+	for chunkEnd < len(buf) {
+		sz := bytes.IndexByte(buf[chunkEnd:], '\r')
+		if sz < 1 {
+			break
+		}
+		// don't parse chunk extensions https://github.com/golang/go/issues/13135.
+		// chunks extensions are no longer a thing, but we do check if the byte
+		// following the parsed hex number is ';'
+		sz += chunkEnd
+		chkLen, ok := atoI(buf[chunkEnd:sz], 16)
+		if !ok && bytes.IndexByte(buf[chunkEnd:sz], ';') < 1 {
+			break
+		}
+		sz++ // + '\n'
+		// total length = SIZE + CRLF + OCTETS + CRLF
+		allChunk := sz + chkLen + 2
+		if allChunk >= len(buf) ||
+			buf[sz]&buf[allChunk] != '\n' ||
+			buf[allChunk-1] != '\r' {
+			break
+		}
+		chunkEnd = allChunk + 1
+		if chkLen == 0 {
+			full = true
+			break
+		}
+	}
+	return
+}
+
+// ProtocolStateSetter is an interface used to provide protocol state for future use
+type ProtocolStateSetter interface {
+	SetProtocolState(interface{})
+	ProtocolState() interface{}
+}
+
+type httpProto struct {
+	body         int // body index
+	headerStart  int
+	headerParsed bool // we checked necessary headers
+	hasFullBody  bool // all chunks has been parsed
+	isChunked    bool // Transfer-Encoding: chunked
+	bodyLen      int  // Content-Length's value
+	hasTrailer   bool // Trailer header?
+}
+
+// HasFullPayload checks if this message has full or valid payloads and returns true.
+// Message param is optional but recommended on cases where 'data' is storing
+// partial-to-full stream of bytes(packets).
+func HasFullPayload(m ProtocolStateSetter, payloads ...[]byte) bool {
+	var state *httpProto
+	if m != nil {
+		state, _ = m.ProtocolState().(*httpProto)
+	}
+	if state == nil {
+		state = new(httpProto)
+		if m != nil {
+			m.SetProtocolState(state)
+		}
+	}
+	if state.headerStart < 1 {
+		for _, data := range payloads {
+			state.headerStart = MIMEHeadersStartPos(data)
+			if state.headerStart < 0 {
+				return false
+			} else {
+				break
+			}
+		}
+	}
+
+	if state.body < 1 {
+		var pos int
+		for _, data := range payloads {
+			endPos := MIMEHeadersEndPos(data)
+			if endPos < 0 {
+				pos += len(data)
+			} else {
+				pos += endPos
+			}
+
+			if endPos > 0 {
+				state.body = pos
+				break
+			}
+		}
+	}
+	if !state.headerParsed {
+		var pos int
+		for _, data := range payloads {
+			chunked := Header(data, []byte("Transfer-Encoding"))
+
+			if len(chunked) > 0 && bytes.Index(data, []byte("chunked")) > 0 {
+				state.isChunked = true
+				// trailers are generally not allowed in non-chunks body
+				state.hasTrailer = len(Header(data, []byte("Trailer"))) > 0
+			} else {
+				contentLen := Header(data, []byte("Content-Length"))
+				state.bodyLen, _ = atoI(contentLen, 10)
+			}
+
+			pos += len(data)
+
+			if state.bodyLen > 0 || pos >= state.body {
+				state.headerParsed = true
+				break
+			}
+		}
+	}
+
+	bodyLen := 0
+	for _, data := range payloads {
+		bodyLen += len(data)
+	}
+	bodyLen -= state.body
+
+	if state.isChunked {
+		// check chunks
+		if bodyLen < 1 {
+			return false
+		}
+
+		// check trailer headers
+		if state.hasTrailer {
+			if bytes.HasSuffix(payloads[len(payloads)-1], []byte("\r\n\r\n")) {
+				return true
+			}
+		} else {
+			if bytes.HasSuffix(payloads[len(payloads)-1], []byte("0\r\n\r\n")) {
+				state.hasFullBody = true
+				return true
+			}
+		}
+
 		return false
 	}
 
-	method := string(payload[0:4])
+	// check for content-length header
+	return state.bodyLen == bodyLen
+}
 
-	for _, m := range httpMethods {
-		if method == m {
-			return true
+// this works with positive integers
+func atoI(s []byte, base int) (num int, ok bool) {
+	var v int
+	ok = true
+	for i := 0; i < len(s); i++ {
+		if s[i] > 127 {
+			ok = false
+			break
 		}
+		v = int(hexTable[s[i]])
+		if v >= base || (v == 0 && s[i] != '0') {
+			ok = false
+			break
+		}
+		num = (num * base) + v
 	}
-	return false
+	return
+}
+
+var hexTable = [128]byte{
+	'0': 0,
+	'1': 1,
+	'2': 2,
+	'3': 3,
+	'4': 4,
+	'5': 5,
+	'6': 6,
+	'7': 7,
+	'8': 8,
+	'9': 9,
+	'A': 10,
+	'a': 10,
+	'B': 11,
+	'b': 11,
+	'C': 12,
+	'c': 12,
+	'D': 13,
+	'd': 13,
+	'E': 14,
+	'e': 14,
+	'F': 15,
+	'f': 15,
 }

proto/proto_test.go

@@ -2,6 +2,7 @@ package proto
 
 import (
 	"bytes"
+	"net/textproto"
 	"reflect"
 	"testing"
 )
@@ -38,13 +39,6 @@ func TestHeader(t *testing.T) {
 		t.Error("Should return empty value")
 	}
 
-	// Wrong delimeter
-	payload = []byte("GET /p HTTP/1.1\r\nCookie: 123\nHost: www.w3.org\r\n\r\n")
-
-	if val = Header(payload, []byte("Cookie")); !bytes.Equal(val, []byte("123")) {
-		t.Error("Should handle wrong header delimeter")
-	}
-
 	// Header not found
 	if _, headerStart, _, _, _ = header(payload, []byte("Not-Found")); headerStart != -1 {
 		t.Error("Should not found header")
@@ -65,13 +59,13 @@ func TestHeader(t *testing.T) {
 }
 
 func TestMIMEHeadersEndPos(t *testing.T) {
-	head := []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org")
+	head := []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\n")
 	payload := []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
 
 	end := MIMEHeadersEndPos(payload)
 
 	if !bytes.Equal(payload[:end], head) {
-		t.Error("Wrong headers end position:", end)
+		t.Error("Wrong headers end position:", end, head, payload[:end])
 	}
 }
 
@@ -80,10 +74,10 @@ func TestMIMEHeadersStartPos(t *testing.T) {
 	payload := []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
 
 	start := MIMEHeadersStartPos(payload)
-	end := MIMEHeadersEndPos(payload)
+	end := MIMEHeadersEndPos(payload) - 4
 
 	if !bytes.Equal(payload[start:end], headers) {
-		t.Error("Wrong headers end position:", start, end)
+		t.Error("Wrong headers end position:", start, end, payload[start:end])
 	}
 }
 
@@ -103,6 +97,10 @@ func TestSetHeader(t *testing.T) {
 	if payload = SetHeader(payload, []byte("User-Agent"), []byte("Gor")); !bytes.Equal(payload, payloadAfter) {
 		t.Error("Should add header if not found", string(payload))
 	}
+	invalidPayload := []byte("POST /post HTTP/1.1")
+	if invalidPayload = SetHeader(invalidPayload, []byte("User-Agent"), []byte("Gor")); !bytes.Equal(invalidPayload, []byte("POST /post HTTP/1.1")) {
+		t.Error("Should not modify payload if request is invalid", string(payload))
+	}
 }
 
 func TestDeleteHeader(t *testing.T) {
@@ -127,22 +125,35 @@ func TestDeleteHeader(t *testing.T) {
 func TestParseHeaders(t *testing.T) {
 	payload := [][]byte{[]byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.or"), []byte("g\r\nUser-Ag"), []byte("ent:Chrome\r\n\r\n"), []byte("Fake-Header: asda")}
 
-	headers := make(map[string]string)
+	headers := ParseHeaders(bytes.Join(payload, nil))
 
-	ParseHeaders(payload, func(header []byte, value []byte) bool {
-		headers[string(header)] = string(value)
-		return true
-	})
-
-	expected := map[string]string{
-		"Content-Length": "7",
-		"Host":           "www.w3.org",
-		"User-Agent":     "Chrome",
+	expected := textproto.MIMEHeader{
+		"Content-Length": []string{"7"},
+		"Host":           []string{"www.w3.org"},
+		"User-Agent":     []string{"Chrome"},
 	}
 
 	if !reflect.DeepEqual(headers, expected) {
 		t.Error("Headers do not properly parsed", headers)
 	}
+
+	// Response with Reason phrase
+	payload = [][]byte{[]byte("HTTP/1.1 200 OK\r\nContent-Length: 7\r\nHost: www.w3.org\r\nUser-Agent:Chrome\r\n\r\nbody")}
+
+	headers = ParseHeaders(bytes.Join(payload, nil))
+
+	if !reflect.DeepEqual(headers, expected) {
+		t.Error("Headers do not properly parsed", headers)
+	}
+
+	// Response without Reason phrase
+	payload = [][]byte{[]byte("HTTP/1.1 200\r\nContent-Length: 7\r\nHost: www.w3.org\r\nUser-Agent:Chrome\r\n\r\nbody")}
+
+	headers = ParseHeaders(bytes.Join(payload, nil))
+
+	if !reflect.DeepEqual(headers, expected) {
+		t.Error("Headers do not properly parsed", headers)
+	}
 }
 
 // See https://github.com/dvyukov/go-fuzz and fuzz.go
@@ -152,9 +163,7 @@ func TestFuzzCrashers(t *testing.T) {
 	}
 
 	for _, f := range crashers {
-		ParseHeaders([][]byte{[]byte(f)}, func(header []byte, value []byte) bool {
-	      return true
-	    })
+		ParseHeaders([]byte(f))
 	}
 }
 
@@ -163,18 +172,13 @@ func TestParseHeadersWithComplexUserAgent(t *testing.T) {
 	// Parser should wait for \r\n
 	payload := [][]byte{[]byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.or"), []byte("g\r\nUser-Ag"), []byte("ent:Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0) like Gecko\r\n\r\n"), []byte("Fake-Header: asda")}
 
-	headers := make(map[string]string)
-
-	ParseHeaders(payload, func(header []byte, value []byte) bool {
-		headers[string(header)] = string(value)
-		return true
-	})
+	headers := ParseHeaders(bytes.Join(payload, nil))
 
 	expected := map[string]string{
 		"User-Agent": "Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0) like Gecko",
 	}
 
-	if expected["User-Agent"] != headers["User-Agent"] {
+	if expected["User-Agent"] != headers["User-Agent"][0] {
 		t.Errorf("Header 'User-Agent' expected '%s' and parsed: '%s'", expected["User-Agent"], headers["User-Agent"])
 	}
 }
@@ -184,12 +188,7 @@ func TestParseHeadersWithOrigin(t *testing.T) {
 	// Parser should wait for \r\n
 	payload := [][]byte{[]byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.or"), []byte("g\r\nReferrer: http://127.0.0.1:3000\r\nOrigi"), []byte("n: https://www.example.com\r\nUser-Ag"), []byte("ent:Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0) like Gecko\r\n\r\n"), []byte("in:https://www.example.com\r\n\r\n"), []byte("Fake-Header: asda")}
 
-	headers := make(map[string]string)
-
-	ParseHeaders(payload, func(header []byte, value []byte) bool {
-		headers[string(header)] = string(value)
-		return true
-	})
+	headers := ParseHeaders(bytes.Join(payload, nil))
 
 	expected := map[string]string{
 		"Origin":     "https://www.example.com",
@@ -197,38 +196,19 @@ func TestParseHeadersWithOrigin(t *testing.T) {
 		"Referrer":   "http://127.0.0.1:3000",
 	}
 
-	if expected["Referrer"] != headers["Referrer"] {
+	if expected["Referrer"] != headers["Referrer"][0] {
 		t.Errorf("Header 'Referrer' expected '%s' and parsed: '%s'", expected["Referrer"], headers["Referrer"])
 	}
 
-	if expected["Origin"] != headers["Origin"] {
+	if expected["Origin"] != headers["Origin"][0] {
 		t.Errorf("Header 'Origin' expected '%s' and parsed: '%s'", expected["Origin"], headers["Origin"])
 	}
 
-	if expected["User-Agent"] != headers["User-Agent"] {
+	if expected["User-Agent"] != headers["User-Agent"][0] {
 		t.Errorf("Header 'User-Agent' expected '%s' and parsed: '%s'", expected["User-Agent"], headers["User-Agent"])
 	}
 }
 
-func TestHeaderEquals(t *testing.T) {
-	tests := []struct {
-		h1     string
-		h2     string
-		equals bool
-	}{
-		{"Content-Length", "content-length", true},
-		{"content-length", "Content-Length", true},
-		{"content-Pength", "Content-Length", false},
-		{"Host", "Content-Length", false},
-	}
-
-	for _, tc := range tests {
-		if HeadersEqual([]byte(tc.h1), []byte(tc.h2)) != tc.equals {
-			t.Error(tc)
-		}
-	}
-}
-
 func TestPath(t *testing.T) {
 	var path, payload []byte
 
@@ -237,6 +217,43 @@ func TestPath(t *testing.T) {
 	if path = Path(payload); !bytes.Equal(path, []byte("/post")) {
 		t.Error("Should find path", string(path))
 	}
+
+	payload = []byte("GET /get\r\n\r\nHost: www.w3.org\r\n\r\n")
+
+	if path = Path(payload); !bytes.Equal(path, nil) {
+		t.Error("1Should not find path", string(path))
+	}
+
+	payload = []byte("GET /get\n")
+
+	if path = Path(payload); !bytes.Equal(path, nil) {
+		t.Error("2Should not find path", string(path))
+	}
+
+	payload = []byte("GET /get")
+
+	if path = Path(payload); !bytes.Equal(path, nil) {
+		t.Error("3Should not find path", string(path))
+	}
+}
+
+func TestStatus(t *testing.T) {
+	var status, payload []byte
+
+	payload = []byte("HTTP/1.1 200 OK\r\n")
+	if status = Status(payload); !bytes.Equal(status, []byte("200")) {
+		t.Error("Should find status 200 but:", string(status))
+	}
+
+	payload = []byte("HTTP/1.1 200\r\n")
+	if status = Status(payload); !bytes.Equal(status, []byte("200")) {
+		t.Error("1Should find status 200 but:", string(status))
+	}
+
+	payload = []byte("HTTP/1.1 404 Not Found\r\n")
+	if status = Status(payload); !bytes.Equal(status, []byte("404")) {
+		t.Error("2Should find status 404 but:", string(status))
+	}
 }
 
 func TestSetPath(t *testing.T) {
@@ -248,12 +265,13 @@ func TestSetPath(t *testing.T) {
 	if payload = SetPath(payload, []byte("/new_path")); !bytes.Equal(payload, payloadAfter) {
 		t.Error("Should replace path", string(payload))
 	}
+
 }
 
 func TestPathParam(t *testing.T) {
 	var payload []byte
 
-	payload = []byte("POST /post?param=test&user_id=1 HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
+	payload = []byte("POST /post?param=test&user_id=1&d_type=1&type=2&d_type=3 HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
 
 	if val, _, _ := PathParam(payload, []byte("param")); !bytes.Equal(val, []byte("test")) {
 		t.Error("Should detect attribute", string(val))
@@ -262,6 +280,15 @@ func TestPathParam(t *testing.T) {
 	if val, _, _ := PathParam(payload, []byte("user_id")); !bytes.Equal(val, []byte("1")) {
 		t.Error("Should detect attribute", string(val))
 	}
+
+	if val, _, _ := PathParam(payload, []byte("type")); !bytes.Equal(val, []byte("2")) {
+		t.Error("Should detect attribute", string(val))
+	}
+
+	if val, _, _ := PathParam(payload, []byte("d_type")); !bytes.Equal(val, []byte("1")) {
+		// this function is not designed for cases with duplicate param keys
+		t.Error("Should detect attribute", string(val))
+	}
 }
 
 func TestSetPathParam(t *testing.T) {
@@ -305,4 +332,160 @@ func TestSetHostHTTP10(t *testing.T) {
 	if payload = SetHost(payload, []byte("http://new.com"), []byte("new.com")); !bytes.Equal(payload, payloadAfter) {
 		t.Error("Should replace host", string(payload))
 	}
+
+	payload = []byte("POST /post HTTP/1.0\r\nContent-Length: 7\r\nHost: example.com\r\n\r\na=1&b=2")
+	payloadAfter = []byte("POST /post HTTP/1.0\r\nContent-Length: 7\r\nHost: new.com\r\n\r\na=1&b=2")
+
+	if payload = SetHost(payload, nil, []byte("new.com")); !bytes.Equal(payload, payloadAfter) {
+		t.Error("Should replace host", string(payload))
+	}
+
+	payload = []byte("POST /post HTTP/1.0\r\nContent-Length: 7\r\n\r\na=1&b=2")
+
+	if payload = SetHost(payload, nil, []byte("new.com")); !bytes.Equal(payload, payload) {
+		t.Error("Should replace host", string(payload))
+	}
+}
+
+func TestHasResponseTitle(t *testing.T) {
+	var m = map[string]bool{
+		"HTTP":                      false,
+		"":                          false,
+		"HTTP/1.1 100 Continue":     false,
+		"HTTP/1.1 100 Continue\r\n": true,
+		"HTTP/1.1  \r\n":            false,
+		"HTTP/4.0 100Continue\r\n":  false,
+		"HTTP/1.0 100Continue\r\n":  false,
+		"HTTP/1.0 10r Continue\r\n": false,
+		"HTTP/1.1 200\r\n":          true,
+		"HTTP/1.1 200\r\nServer: Tengine\r\nContent-Length: 0\r\nConnection: close\r\n\r\n": true,
+	}
+	for k, v := range m {
+		if HasResponseTitle([]byte(k)) != v {
+			t.Errorf("%q should yield %v", k, v)
+			break
+		}
+	}
+}
+
+func TestHasRequestTitle(t *testing.T) {
+	var m = map[string]bool{
+		"POST /post HTTP/1.0\r\n": true,
+		"":                        false,
+		"POST /post HTTP/1.\r\n":  false,
+		"POS /post HTTP/1.1\r\n":  false,
+		"GET / HTTP/1.1\r\n":      true,
+		"GET / HTTP/1.1\r":        false,
+		"GET / HTTP/1.400\r\n":    false,
+	}
+	for k, v := range m {
+		if HasRequestTitle([]byte(k)) != v {
+			t.Errorf("%q should yield %v", k, v)
+			break
+		}
+	}
+}
+
+func TestCheckChunks(t *testing.T) {
+	var m = "4\r\nWiki\r\n5\r\npedia\r\nE\r\n in\r\n\r\nchunks.\r\n0\r\n\r\n"
+	chunkEnd, _ := CheckChunked([]byte(m))
+	expected := len(m)
+	if chunkEnd != expected {
+		t.Errorf("expected %d to equal %d", chunkEnd, expected)
+	}
+
+	m = "7\r\nMozia\r\n9\r\nDeveloper\r\n7\r\nNetwork\r\n0\r\n\r\n"
+	chunkEnd, _ = CheckChunked([]byte(m))
+	if chunkEnd != 0 {
+		t.Errorf("expected %d to equal %d", chunkEnd, 0)
+	}
+
+	// with trailers
+	m = "4\r\nWiki\r\n5\r\npedia\r\nE\r\n in\r\n\r\nchunks.\r\n0\r\n\r\nEXpires"
+	chunkEnd, _ = CheckChunked([]byte(m))
+	expected = len(m) - 7
+	if chunkEnd != expected {
+		t.Errorf("expected %d to equal %d", chunkEnd, expected)
+	}
+
+	// last chunk inside the body
+	// with trailers
+	m = "4\r\nWiki\r\n5\r\npedia\r\nE\r\n in\r\n\r\nchunks.\r\n3\r\n0\r\n\r\n0\r\n\r\nEXpires"
+	chunkEnd, _ = CheckChunked([]byte(m))
+	expected = len(m) - 7
+	if chunkEnd != expected {
+		t.Errorf("expected %d to equal %d", chunkEnd, expected)
+	}
+
+	// checks with chucks-extensions
+	m = "4\r\nWiki\r\n5\r\npedia\r\nE; name='quoted string'\r\n in\r\n\r\nchunks.\r\n3\r\n0\r\n\r\n0\r\n\r\nEXpires"
+	chunkEnd, _ = CheckChunked([]byte(m))
+	expected = len(m) - 7
+	if chunkEnd != expected {
+		t.Errorf("expected %d to equal %d", chunkEnd, expected)
+	}
+}
+
+func TestHasFullPayload(t *testing.T) {
+	var m string
+	var got, expected bool
+
+	got = HasFullPayload(nil,
+		[]byte("HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\n"),
+		[]byte("Transfer-Encoding: chunked\r\n\r\n"),
+		[]byte("7\r\nMozilla\r\n9\r\nDeveloper\r\n"),
+		[]byte("7\r\nNetwork\r\n0\r\n\r\n"))
+	expected = true
+	if got != expected {
+		t.Errorf("expected %v to equal %v", got, expected)
+	}
+
+	// check chunks with trailers
+	m = "HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nTransfer-Encoding: chunked\r\nTrailer: Expires\r\n\r\n7\r\nMozilla\r\n9\r\nDeveloper\r\n7\r\nNetwork\r\n0\r\n\r\nExpires: Wed, 21 Oct 2015 07:28:00 GMT\r\n\r\n"
+	got = HasFullPayload(nil, []byte(m))
+	expected = true
+	if got != expected {
+		t.Errorf("expected %v to equal %v", got, expected)
+	}
+
+	// check with missing trailers
+	m = "HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nTransfer-Encoding: chunked\r\nTrailer: Expires\r\n\r\n7\r\nMozilla\r\n9\r\nDeveloper\r\n7\r\nNetwork\r\n0\r\n\r\nExpires: Wed, 21 Oct 2015 07:28:00"
+	got = HasFullPayload(nil, []byte(m))
+	expected = false
+	if got != expected {
+		t.Errorf("expected %v to equal %v", got, expected)
+	}
+
+	// check with content-length
+	m = "HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nContent-Length: 23\r\n\r\nMozillaDeveloperNetwork"
+	got = HasFullPayload(nil, []byte(m))
+	expected = true
+	if got != expected {
+		t.Errorf("expected %v to equal %v", got, expected)
+	}
+
+	// check missing total length
+	m = "HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nContent-Length: 23\r\n\r\nMozillaDeveloperNet"
+	got = HasFullPayload(nil, []byte(m))
+	expected = false
+	if got != expected {
+		t.Errorf("expected %v to equal %v", got, expected)
+	}
+
+	// check with no body
+	m = "HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\n\r\n"
+	got = HasFullPayload(nil, []byte(m))
+	expected = true
+	if got != expected {
+		t.Errorf("expected %v to equal %v", got, expected)
+	}
+}
+
+func BenchmarkHasFullPayload(b *testing.B) {
+	data := []byte("HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nTransfer-Encoding: chunked\r\n\r\n1e\r\n111111111111111111111111111111\r\n0\r\n\r\n")
+	for i := 0; i < b.N; i++ {
+		if !HasFullPayload(nil, data) {
+			b.Fail()
+		}
+	}
 }

protocol.go

@@ -4,23 +4,28 @@ import (
 	"bytes"
 	"crypto/rand"
 	"encoding/hex"
-	"strconv"
+	"fmt"
 )
 
+// These constants help to indicate the type of payload
 const (
 	RequestPayload          = '1'
 	ResponsePayload         = '2'
 	ReplayedResponsePayload = '3'
 )
 
-func uuid() []byte {
-	b := make([]byte, 20)
+func randByte(len int) []byte {
+	b := make([]byte, len/2)
 	rand.Read(b)
 
-	uuid := make([]byte, 40)
-	hex.Encode(uuid, b)
+	h := make([]byte, len)
+	hex.Encode(h, b)
 
-	return uuid
+	return h
+}
+
+func uuid() []byte {
+	return randByte(24)
 }
 
 var payloadSeparator = "\n🐵🙈🙉\n"
@@ -43,37 +48,9 @@ func payloadScanner(data []byte, atEOF bool) (advance int, token []byte, err err
 
 // Timing is request start or round-trip time, depending on payloadType
 func payloadHeader(payloadType byte, uuid []byte, timing int64, latency int64) (header []byte) {
-	var sTime, sLatency string
-
-	sTime = strconv.FormatInt(timing, 10)
-	if latency != -1 {
-		sLatency = strconv.FormatInt(latency, 10)
-	}
-
 	//Example:
-	//  3 f45590522cd1838b4a0d5c5aab80b77929dea3b3 1231\n
-	// `+ 1` indicates space characters or end of line
-	headerLen := 1 + 1 + len(uuid) + 1 + len(sTime) + 1
-
-	if latency != -1 {
-		headerLen += len(sLatency) + 1
-	}
-
-	header = make([]byte, headerLen)
-	header[0] = payloadType
-	header[1] = ' '
-	header[2+len(uuid)] = ' '
-	header[len(header)-1] = '\n'
-
-	copy(header[2:], uuid)
-	copy(header[3+len(uuid):], sTime)
-
-	if latency != -1 {
-		header[3+len(uuid)+len(sTime)] = ' '
-		copy(header[4+len(uuid)+len(sTime):], sLatency)
-	}
-
-	return header
+	//  3 f45590522cd1838b4a0d5c5aab80b77929dea3b3 13923489726487326 1231\n
+	return []byte(fmt.Sprintf("%c %s %d %d\n", payloadType, uuid, timing, latency))
 }
 
 func payloadBody(payload []byte) []byte {
@@ -84,18 +61,32 @@ func payloadBody(payload []byte) []byte {
 func payloadMeta(payload []byte) [][]byte {
 	headerSize := bytes.IndexByte(payload, '\n')
 	if headerSize < 0 {
-		headerSize = 0
+		return nil
 	}
 	return bytes.Split(payload[:headerSize], []byte{' '})
 }
 
-func isOriginPayload(payload []byte) bool {
-	switch payload[0] {
-	case RequestPayload, ResponsePayload:
-		return true
-	default:
-		return false
+func payloadMetaWithBody(payload []byte) (meta, body []byte) {
+	if i := bytes.IndexByte(payload, '\n'); i > 0 && len(payload) > i+1 {
+		meta = payload[:i+1]
+		body = payload[i+1:]
+		return
 	}
+	// we assume the message did not have meta data
+	return nil, payload
+}
+
+func payloadID(payload []byte) (id []byte) {
+	meta := payloadMeta(payload)
+
+	if len(meta) < 2 {
+		return
+	}
+	return meta[1]
+}
+
+func isOriginPayload(payload []byte) bool {
+	return payload[0] == RequestPayload || payload[0] == ResponsePayload
 }
 
 func isRequestPayload(payload []byte) bool {

raw_socket_listener/listener.go

@@ -1,842 +0,0 @@
-/*
-Package rawSocket provides traffic sniffier using RAW sockets.
-
-Capture traffic from socket using RAW_SOCKET's
-http://en.wikipedia.org/wiki/Raw_socket
-
-RAW_SOCKET allow you listen for traffic on any port (e.g. sniffing) because they operate on IP level.
-
-Ports is TCP feature, same as flow control, reliable transmission and etc.
-
-This package implements own TCP layer: TCP packets is parsed using tcp_packet.go, and flow control is managed by tcp_message.go
-*/
-package rawSocket
-
-import (
-	"bytes"
-	"encoding/binary"
-	"fmt"
-	"github.com/buger/gor/proto"
-	"github.com/google/gopacket"
-	"github.com/google/gopacket/layers"
-	"github.com/google/gopacket/pcap"
-	"io"
-	"log"
-	"net"
-	"runtime"
-	"runtime/debug"
-	"strconv"
-	"strings"
-	"sync"
-	"time"
-)
-
-var _ = fmt.Println
-
-type packet struct {
-	srcIP		[]byte
-	data		[]byte
-	timestamp	time.Time
-}
-
-// Listener handle traffic capture
-type Listener struct {
-	mu sync.Mutex
-	// buffer of TCPMessages waiting to be send
-	// ID -> TCPMessage
-	messages map[tcpID]*TCPMessage
-
-	// Expect: 100-continue request is send in 2 tcp messages
-	// We store ACK aliases to merge this packets together
-	ackAliases map[uint32]uint32
-	// To get ACK of second message we need to compute its Seq and wait for them message
-	seqWithData map[uint32]uint32
-
-	// Ack -> Req
-	respAliases map[uint32]*TCPMessage
-
-	// Ack -> ID
-	respWithoutReq map[uint32]tcpID
-
-	// Messages ready to be send to client
-	packetsChan chan *packet
-
-	// Messages ready to be send to client
-	messagesChan chan *TCPMessage
-
-	addr string // IP to listen
-	port uint16 // Port to listen
-
-	trackResponse bool
-	messageExpire time.Duration
-
-	conn        net.PacketConn
-	pcapHandles []*pcap.Handle
-
-	quit    chan bool
-	readyCh chan bool
-}
-
-type request struct {
-	id    tcpID
-	start time.Time
-	ack   uint32
-}
-
-// Available engines for intercepting traffic
-const (
-	EngineRawSocket = 1 << iota
-	EnginePcap
-	EnginePcapFile
-)
-
-// NewListener creates and initializes new Listener object
-func NewListener(addr string, port string, engine int, trackResponse bool, expire time.Duration) (l *Listener) {
-	l = &Listener{}
-
-	l.packetsChan = make(chan *packet, 10000)
-	l.messagesChan = make(chan *TCPMessage, 10000)
-	l.quit = make(chan bool)
-	l.readyCh = make(chan bool, 1)
-
-	l.messages = make(map[tcpID]*TCPMessage)
-	l.ackAliases = make(map[uint32]uint32)
-	l.seqWithData = make(map[uint32]uint32)
-	l.respAliases = make(map[uint32]*TCPMessage)
-	l.respWithoutReq = make(map[uint32]tcpID)
-	l.trackResponse = trackResponse
-
-	l.addr = addr
-	_port, _ := strconv.Atoi(port)
-	l.port = uint16(_port)
-
-	if expire.Nanoseconds() == 0 {
-		expire = 2000 * time.Millisecond
-	}
-
-	l.messageExpire = expire
-
-	go l.listen()
-
-	// Special case for testing
-	if l.port != 0 {
-		switch engine {
-		case EnginePcap:
-			go l.readPcap()
-		case EnginePcapFile:
-			go l.readPcapFile()
-		default:
-			log.Fatal("Unknown traffic interception engine:", engine)
-		}
-	}
-
-	return
-}
-
-func (t *Listener) listen() {
-	gcTicker := time.Tick(t.messageExpire / 2)
-
-	for {
-		select {
-		case <-t.quit:
-			if t.conn != nil {
-				t.conn.Close()
-			}
-			return
-		case packet := <-t.packetsChan:
-			tcpPacket := ParseTCPPacket(packet.srcIP, packet.data, packet.timestamp)
-			t.processTCPPacket(tcpPacket)
-		case <-gcTicker:
-			now := time.Now()
-
-			// Dispatch requests before responses
-			for _, message := range t.messages {
-				if now.Sub(message.End) >= t.messageExpire {
-					t.dispatchMessage(message)
-				}
-			}
-		}
-	}
-}
-
-func (t *Listener) deleteMessage(message *TCPMessage) {
-	delete(t.messages, message.ID())
-	delete(t.ackAliases, message.Ack)
-	if message.DataAck != 0 {
-		delete(t.ackAliases, message.DataAck)
-	}
-	if message.DataSeq != 0 {
-		delete(t.seqWithData, message.DataSeq)
-	}
-
-	delete(t.respAliases, message.ResponseAck)
-}
-
-func (t *Listener) dispatchMessage(message *TCPMessage) {
-	// If already dispatched
-	if _, ok := t.messages[message.ID()]; !ok {
-		return
-	}
-
-	t.deleteMessage(message)
-
-	if !message.complete {
-		if !message.IsIncoming {
-			delete(t.respAliases, message.Ack)
-			delete(t.respWithoutReq, message.Ack)
-		}
-
-		return
-	}
-
-	if message.IsIncoming {
-		// If there were response before request
-		// log.Println("Looking for Response: ", t.respWithoutReq, message.ResponseAck)
-		if t.trackResponse {
-			if respID, ok := t.respWithoutReq[message.ResponseAck]; ok {
-				if resp, rok := t.messages[respID]; rok {
-					// if resp.AssocMessage == nil {
-					// log.Println("FOUND RESPONSE")
-					resp.setAssocMessage(message)
-					message.setAssocMessage(resp)
-
-					if resp.complete {
-						defer t.dispatchMessage(resp)
-					}
-					// }
-				}
-			}
-
-			if resp, ok := t.messages[message.ResponseID]; ok {
-				resp.setAssocMessage(message)
-			}
-		}
-	} else {
-		if message.AssocMessage == nil {
-			if responseRequest, ok := t.respAliases[message.Ack]; ok {
-				message.setAssocMessage(responseRequest)
-				responseRequest.setAssocMessage(message)
-			}
-		}
-
-		delete(t.respAliases, message.Ack)
-		delete(t.respWithoutReq, message.Ack)
-
-		// Do not track responses which have no associated requests
-		if message.AssocMessage == nil {
-			// log.Println("Can't dispatch resp", message.Seq, message.Ack, string(message.Bytes()))
-			return
-		}
-	}
-
-	t.messagesChan <- message
-}
-
-// DeviceNotFoundError raised if user specified wrong ip
-type DeviceNotFoundError struct {
-	addr string
-}
-
-func (e *DeviceNotFoundError) Error() string {
-	devices, _ := pcap.FindAllDevs()
-
-	if len(devices) == 0 {
-		return "Can't get list of network interfaces, ensure that you running Gor as root user or sudo.\nTo run as non-root users see this docs https://github.com/buger/gor/wiki/Running-as-non-root-user"
-	}
-
-	var msg string
-	msg += "Can't find interfaces with addr: " + e.addr + ". Provide available IP for intercepting traffic: \n"
-	for _, device := range devices {
-		msg += "Name: " + device.Name + "\n"
-		if device.Description != "" {
-			msg += "Description: " + device.Description + "\n"
-		}
-		for _, address := range device.Addresses {
-			msg += "- IP address: " + address.IP.String() + "\n"
-		}
-	}
-
-	return msg
-}
-
-func isLoopback(device pcap.Interface) bool {
-	if len(device.Addresses) == 0 {
-		return false
-	}
-
-	switch device.Addresses[0].IP.String() {
-	case "127.0.0.1", "::1":
-		return true
-	}
-
-	return false
-}
-
-func listenAllInterfaces(addr string) bool {
-	switch addr {
-	case "", "0.0.0.0", "[::]", "::":
-		return true
-	default:
-		return false
-	}
-}
-
-func findPcapDevices(addr string) (interfaces []pcap.Interface, err error) {
-	devices, err := pcap.FindAllDevs()
-	if err != nil {
-		log.Fatal(err)
-	}
-
-	for _, device := range devices {
-		if listenAllInterfaces(addr) && len(device.Addresses) > 0 || isLoopback(device) {
-			interfaces = append(interfaces, device)
-			continue
-		}
-
-		for _, address := range device.Addresses {
-			if device.Name == addr || address.IP.String() == addr {
-				interfaces = append(interfaces, device)
-				return interfaces, nil
-			}
-		}
-	}
-
-	if len(interfaces) == 0 {
-		return nil, &DeviceNotFoundError{addr}
-	} else {
-		return interfaces, nil
-	}
-}
-
-func (t *Listener) readPcap() {
-	devices, err := findPcapDevices(t.addr)
-	if err != nil {
-		log.Fatal(err)
-	}
-
-	bpfSupported := true
-	if runtime.GOOS == "darwin" {
-		bpfSupported = false
-	}
-
-	var wg sync.WaitGroup
-	wg.Add(len(devices))
-
-	for _, d := range devices {
-		go func(device pcap.Interface) {
-			handle, err := pcap.OpenLive(device.Name, 65536, true, t.messageExpire)
-			if err != nil {
-				log.Println("Pcap Error while opening device", device.Name, err)
-				wg.Done()
-				return
-			}
-			defer handle.Close()
-
-			t.mu.Lock()
-			t.pcapHandles = append(t.pcapHandles, handle)
-
-			var bpfDstHost, bpfSrcHost string
-			var loopback = isLoopback(device)
-
-			if loopback {
-				var allAddr []string
-				for _, dc := range devices {
-					for _, addr := range dc.Addresses {
-						allAddr = append(allAddr, "(dst host "+addr.IP.String()+" and src host "+addr.IP.String()+")")
-					}
-				}
-
-				bpfDstHost = strings.Join(allAddr, " or ")
-				bpfSrcHost = bpfDstHost
-			} else {
-				for i, addr := range device.Addresses {
-					bpfDstHost += "dst host " + addr.IP.String()
-					bpfSrcHost += "src host " + addr.IP.String()
-					if i != len(device.Addresses)-1 {
-						bpfDstHost += " or "
-						bpfSrcHost += " or "
-					}
-				}
-			}
-
-			if bpfSupported {
-				var bpf string
-
-				if t.trackResponse {
-					bpf = "(tcp dst port " + strconv.Itoa(int(t.port)) + " and (" + bpfDstHost + ")) or (" + "tcp src port " + strconv.Itoa(int(t.port)) + " and (" + bpfSrcHost + "))"
-				} else {
-					bpf = "tcp dst port " + strconv.Itoa(int(t.port)) + " and (" + bpfDstHost + ")"
-				}
-
-				if err := handle.SetBPFFilter(bpf); err != nil {
-					log.Println("BPF filter error:", err, "Device:", device.Name, bpf)
-					wg.Done()
-					return
-				}
-			}
-			t.mu.Unlock()
-
-			var decoder gopacket.Decoder
-
-			// Special case for tunnel interface https://github.com/google/gopacket/issues/99
-			if handle.LinkType() == 12 {
-				decoder = layers.LayerTypeIPv4
-			} else {
-				decoder = handle.LinkType()
-			}
-
-			source := gopacket.NewPacketSource(handle, decoder)
-			source.Lazy = true
-			source.NoCopy = true
-
-			wg.Done()
-
-			var data, srcIP, dstIP []byte
-
-			for {
-				packet, err := source.NextPacket()
-
-				if err == io.EOF {
-					break
-				} else if err != nil {
-					continue
-				}
-
-				// We should remove network layer before parsing TCP/IP data
-				var of int
-				switch decoder {
-				case layers.LinkTypeEthernet:
-					of = 14
-				case layers.LinkTypePPP:
-					of = 1
-				case layers.LinkTypeFDDI:
-					of = 13
-				case layers.LinkTypeNull:
-					of = 4
-				case layers.LinkTypeLoop:
-					of = 4
-				case layers.LinkTypeRaw:
-					of = 0
-				case layers.LinkTypeLinuxSLL:
-					of = 16
-				default:
-					log.Println("Unknown packet layer", packet)
-					break
-				}
-
-				data = packet.Data()[of:]
-
-				version := uint8(data[0]) >> 4
-				ipLength := int(binary.BigEndian.Uint16(data[2:4]))
-
-				if version == 4 {
-					ihl := uint8(data[0]) & 0x0F
-
-					// Truncated IP info
-					if len(data) < int(ihl*4) {
-						continue
-					}
-
-					srcIP = data[12:16]
-					dstIP = data[16:20]
-
-					// Too small IP packet
-					if ipLength < 20 {
-						continue
-					}
-
-					// Invalid length
-					if int(ihl*4) > ipLength {
-						continue
-					}
-
-					if cmp := len(data) - ipLength; cmp > 0 {
-						data = data[:ipLength]
-					} else if cmp < 0 {
-						// Truncated packet
-						continue
-					}
-
-					data = data[ihl*4:]
-				} else {
-					// Truncated IP info
-					if len(data) < 40 {
-						continue
-					}
-
-					srcIP = data[8:24]
-					dstIP = data[24:40]
-
-					data = data[40:]
-				}
-
-				// Truncated TCP info
-				if len(data) <= 13 {
-					continue
-				}
-
-				dataOffset := (data[12] & 0xF0) >> 4
-				isFIN := data[13]&0x01 != 0
-
-				// We need only packets with data inside
-				// Check that the buffer is larger than the size of the TCP header
-				if len(data) > int(dataOffset*4) || isFIN {
-					if !bpfSupported {
-						destPort := binary.BigEndian.Uint16(data[2:4])
-						srcPort := binary.BigEndian.Uint16(data[0:2])
-
-						var addrCheck []byte
-
-						if destPort == t.port {
-							addrCheck = dstIP
-						}
-
-						if t.trackResponse && srcPort == t.port {
-							addrCheck = srcIP
-						}
-
-						if len(addrCheck) == 0 {
-							continue
-						}
-
-						addrMatched := false
-
-						if loopback {
-							for _, dc := range devices {
-								if addrMatched {
-									break
-								}
-								for _, a := range dc.Addresses {
-									if a.IP.Equal(net.IP(addrCheck)) {
-										addrMatched = true
-										break
-									}
-								}
-							}
-							addrMatched = true
-						} else {
-							for _, a := range device.Addresses {
-								if a.IP.Equal(net.IP(addrCheck)) {
-									addrMatched = true
-									break
-								}
-							}
-						}
-
-						if !addrMatched {
-							continue
-						}
-					}
-
-					t.packetsChan <- t.buildPacket(srcIP, data, packet.Metadata().Timestamp)
-				}
-			}
-		}(d)
-	}
-
-	wg.Wait()
-	t.readyCh <- true
-}
-
-func (t *Listener) readPcapFile() {
-	if handle, err := pcap.OpenOffline(t.addr); err != nil {
-		log.Fatal(err)
-	} else {
-		t.readyCh <- true
-		packetSource := gopacket.NewPacketSource(handle, handle.LinkType())
-
-		for {
-			packet, err := packetSource.NextPacket()
-			if err == io.EOF {
-				break
-			} else if err != nil {
-				log.Println("Error:", err)
-				continue
-			}
-
-			var addr, data []byte
-
-			if tcpLayer := packet.Layer(layers.LayerTypeTCP); tcpLayer != nil {
-				tcp, _ := tcpLayer.(*layers.TCP)
-				data = append(tcp.LayerContents(), tcp.LayerPayload()...)
-
-				if tcp.SrcPort >= 32768 && tcp.SrcPort <= 61000 {
-					copy(data[0:2], []byte{0, 0})
-					copy(data[2:4], []byte{0, 1})
-				} else {
-					copy(data[0:2], []byte{0, 1})
-					copy(data[2:4], []byte{0, 0})
-				}
-			} else {
-				continue
-			}
-
-			if ipLayer := packet.Layer(layers.LayerTypeIPv4); ipLayer != nil {
-				ip, _ := ipLayer.(*layers.IPv4)
-				addr = ip.SrcIP
-			} else if ipLayer = packet.Layer(layers.LayerTypeIPv6); ipLayer != nil {
-				ip, _ := ipLayer.(*layers.IPv6)
-				addr = ip.SrcIP
-			} else {
-				// log.Println("Can't find IP layer", packet)
-				continue
-			}
-
-			dataOffset := (data[12] & 0xF0) >> 4
-			isFIN := data[13]&0x01 != 0
-
-			// We need only packets with data inside
-			// Check that the buffer is larger than the size of the TCP header
-			if len(data) <= int(dataOffset*4) && !isFIN {
-				continue
-			}
-
-			t.packetsChan <- t.buildPacket(addr, data, packet.Metadata().Timestamp)
-		}
-	}
-}
-
-func (t *Listener) readRAWSocket() {
-	conn, e := net.ListenPacket("ip:tcp", t.addr)
-	t.conn = conn
-
-	if e != nil {
-		log.Fatal(e)
-	}
-
-	defer t.conn.Close()
-
-	buf := make([]byte, 64*1024) // 64kb
-
-	t.readyCh <- true
-
-	for {
-		// Note: ReadFrom receive messages without IP header
-		n, addr, err := t.conn.ReadFrom(buf)
-
-		if err != nil {
-			if strings.HasSuffix(err.Error(), "closed network connection") {
-				return
-			} else {
-				continue
-			}
-		}
-
-		if n > 0 {
-			if t.isValidPacket(buf[:n]) {
-				t.packetsChan <- t.buildPacket([]byte(addr.(*net.IPAddr).IP), buf[:n], time.Now())
-			}
-		}
-	}
-}
-
-func (t *Listener) buildPacket(packetSrcIP []byte, packetData []byte, timestamp time.Time) *packet {
-	copyPacketSrcIP := make([]byte, 16)
-	copyPacketData := make([]byte, len(packetData))
-
-	copy(copyPacketSrcIP, packetSrcIP)
-	copy(copyPacketData, packetSrcIP)
-
-	return &packet{
-		srcIP: packetSrcIP,
-		data: packetData,
-		timestamp:timestamp,
-	}
-}
-
-func (t *Listener) isValidPacket(buf []byte) bool {
-	// To avoid full packet parsing every time, we manually parsing values needed for packet filtering
-	// http://en.wikipedia.org/wiki/Transmission_Control_Protocol
-	destPort := binary.BigEndian.Uint16(buf[2:4])
-	srcPort := binary.BigEndian.Uint16(buf[0:2])
-
-	// Because RAW_SOCKET can't be bound to port, we have to control it by ourself
-	if destPort == t.port || (t.trackResponse && srcPort == t.port) {
-		// Get the 'data offset' (size of the TCP header in 32-bit words)
-		dataOffset := (buf[12] & 0xF0) >> 4
-
-		// We need only packets with data inside
-		// Check that the buffer is larger than the size of the TCP header
-		if len(buf) > int(dataOffset*4) {
-			// We should create new buffer because go slices is pointers. So buffer data shoud be immutable.
-			return true
-		}
-	}
-
-	return false
-}
-
-// Trying to add packet to existing message or creating new message
-//
-// For TCP message unique id is Acknowledgment number (see tcp_packet.go)
-func (t *Listener) processTCPPacket(packet *TCPPacket) {
-	// Don't exit on panic
-	defer func() {
-		if r := recover(); r != nil {
-			log.Println("PANIC: pkg:", r, packet, string(debug.Stack()))
-		}
-	}()
-
-	var message *TCPMessage
-
-	isIncoming := packet.DestPort == t.port
-
-	// Seek for 100-expect chunks
-	if parentAck, ok := t.seqWithData[packet.Seq]; ok {
-		// In case if non-first data chunks comes first
-		for _, m := range t.messages {
-			if m.Ack == packet.Ack && bytes.Equal(m.packets[0].Addr, packet.Addr) {
-				t.deleteMessage(m)
-
-				if m.AssocMessage != nil {
-					m.setAssocMessage(nil)
-				}
-
-				for _, pkt := range m.packets {
-					// log.Println("Updating ack", parentAck, pkt.Ack)
-					pkt.UpdateAck(parentAck)
-					// Re-queue this packets
-					t.processTCPPacket(pkt)
-				}
-			}
-		}
-
-		t.ackAliases[packet.Ack] = parentAck
-		packet.UpdateAck(parentAck)
-	}
-
-	if isIncoming && packet.IsFIN {
-		if ma, ok := t.respAliases[packet.Seq]; ok {
-			if ma.packets[0].SrcPort == packet.SrcPort {
-				packet.UpdateAck(ma.Ack)
-			}
-		}
-	}
-
-	if alias, ok := t.ackAliases[packet.Ack]; ok {
-		packet.UpdateAck(alias)
-	}
-
-	var responseRequest *TCPMessage
-
-	if !isIncoming {
-		responseRequest, _ = t.respAliases[packet.Ack]
-	}
-
-	message, ok := t.messages[packet.ID]
-
-	if !ok {
-		message = NewTCPMessage(packet.Seq, packet.Ack, isIncoming, packet.timestamp)
-		t.messages[packet.ID] = message
-
-		if !isIncoming {
-			if responseRequest != nil {
-				message.setAssocMessage(responseRequest)
-				responseRequest.setAssocMessage(message)
-			} else {
-				t.respWithoutReq[packet.Ack] = packet.ID
-			}
-		}
-	}
-
-	// Adding packet to message
-	message.AddPacket(packet)
-
-	// Handling Expect: 100-continue requests
-	if message.expectType == httpExpect100Continue && len(message.packets) == message.headerPacket+1 {
-		seq := packet.Seq + uint32(message.Size())
-		t.seqWithData[seq] = packet.Ack
-		message.DataSeq = seq
-		message.complete = false
-
-		// In case if sequence packet came first
-		for _, m := range t.messages {
-			if m.Seq == seq {
-				t.deleteMessage(m)
-				if m.AssocMessage != nil {
-					message.setAssocMessage(m.AssocMessage)
-				}
-				// log.Println("2: Adding ack alias:", m.Ack, packet.Ack)
-				t.ackAliases[m.Ack] = packet.Ack
-
-				for _, pkt := range m.packets {
-					pkt.UpdateAck(packet.Ack)
-					message.AddPacket(pkt)
-				}
-			}
-		}
-
-		// Removing `Expect: 100-continue` header
-		packet.Data = proto.DeleteHeader(packet.Data, bExpectHeader)
-	}
-
-	// log.Println("Received message:", string(message.Bytes()), message.ID(), t.messages)
-
-	if isIncoming {
-		// If message have multiple packets, delete previous alias
-		if len(message.packets) > 1 {
-			delete(t.respAliases, message.ResponseAck)
-		}
-
-		message.UpdateResponseAck()
-		t.respAliases[message.ResponseAck] = message
-	}
-
-	// If message contains only single packet immediately dispatch it
-	if message.complete {
-		// log.Println("COMPLETE!", isIncoming, message)
-		if isIncoming {
-			if t.trackResponse {
-				// log.Println("Found response!", message.ResponseID, t.messages)
-
-				if resp, ok := t.messages[message.ResponseID]; ok {
-					if resp.complete {
-						t.dispatchMessage(resp)
-					}
-
-					t.dispatchMessage(message)
-				}
-			} else {
-				t.dispatchMessage(message)
-			}
-		} else {
-			if message.AssocMessage == nil {
-				return
-			}
-
-			if req, ok := t.messages[message.AssocMessage.ID()]; ok {
-				if req.complete {
-					t.dispatchMessage(req)
-					t.dispatchMessage(message)
-				}
-			}
-		}
-	}
-}
-
-func (t *Listener) IsReady() bool {
-	select {
-	case <-t.readyCh:
-		return true
-	case <-time.After(5 * time.Second):
-		return false
-	}
-}
-
-// Receive TCP messages from the listener channel
-func (t *Listener) Receiver() chan *TCPMessage {
-	return t.messagesChan
-}
-
-func (t *Listener) Close() {
-	close(t.quit)
-	if t.conn != nil {
-		t.conn.Close()
-	}
-
-	for _, h := range t.pcapHandles {
-		h.Close()
-	}
-
-	return
-}

raw_socket_listener/listener_test.go

@@ -1,477 +0,0 @@
-package rawSocket
-
-import (
-	"bytes"
-	"log"
-	"math/rand"
-	"sync/atomic"
-	"testing"
-	"time"
-)
-
-func TestRawListenerInput(t *testing.T) {
-	var req, resp *TCPMessage
-
-	listener := NewListener("", "0", EnginePcap, true, 10*time.Millisecond)
-	defer listener.Close()
-
-	reqPacket := buildPacket(true, 1, 1, []byte("GET / HTTP/1.1\r\n\r\n"), time.Now())
-
-	respAck := reqPacket.Seq + uint32(len(reqPacket.Data))
-	respPacket := buildPacket(false, respAck, reqPacket.Seq+1, []byte("HTTP/1.1 200 OK\r\n\r\n"), time.Now())
-
-	listener.packetsChan <- reqPacket.dump()
-	listener.packetsChan <- respPacket.dump()
-
-	select {
-	case req = <-listener.messagesChan:
-	case <-time.After(time.Millisecond):
-		t.Error("Should return request immediately")
-		return
-	}
-
-	if !req.IsIncoming {
-		t.Error("Should be request")
-	}
-
-	select {
-	case resp = <-listener.messagesChan:
-	case <-time.After(20 * time.Millisecond):
-		t.Error("Should return response immediately")
-		return
-	}
-
-	if resp.IsIncoming {
-		t.Error("Should be response")
-	}
-}
-
-func TestRawListenerInputResponseByClose(t *testing.T) {
-	var req, resp *TCPMessage
-
-	listener := NewListener("", "0", EnginePcap, true, 10*time.Millisecond)
-	defer listener.Close()
-
-	reqPacket := buildPacket(true, 1, 1, []byte("GET / HTTP/1.1\r\n\r\n"), time.Now())
-
-	respAck := reqPacket.Seq + uint32(len(reqPacket.Data))
-	respPacket := buildPacket(false, respAck, reqPacket.Seq+1, []byte("HTTP/1.1 200 OK\r\nConnection: close\r\n\r\nasd"), time.Now())
-	finPacket := buildPacket(false, respAck, reqPacket.Seq+2, []byte(""), time.Now())
-	finPacket.IsFIN = true
-
-	listener.packetsChan <- reqPacket.dump()
-	listener.packetsChan <- respPacket.dump()
-	listener.packetsChan <- finPacket.dump()
-
-	select {
-	case req = <-listener.messagesChan:
-	case <-time.After(time.Millisecond):
-		t.Error("Should return request immediately")
-		return
-	}
-
-	if !req.IsIncoming {
-		t.Error("Should be request")
-	}
-
-	select {
-	case resp = <-listener.messagesChan:
-	case <-time.After(20 * time.Millisecond):
-		t.Error("Should return response immediately")
-		return
-	}
-
-	if resp.IsIncoming {
-		t.Error("Should be response")
-	}
-}
-
-func TestRawListenerInputWithoutResponse(t *testing.T) {
-	var req *TCPMessage
-
-	listener := NewListener("", "0", EnginePcap, false, 10*time.Millisecond)
-	defer listener.Close()
-
-	reqPacket := buildPacket(true, 1, 1, []byte("GET / HTTP/1.1\r\n\r\n"), time.Now())
-
-	listener.packetsChan <- reqPacket.dump()
-
-	select {
-	case req = <-listener.messagesChan:
-	case <-time.After(time.Millisecond):
-		t.Error("Should return request immediately")
-		return
-	}
-
-	if !req.IsIncoming {
-		t.Error("Should be request")
-	}
-}
-
-func TestRawListenerResponse(t *testing.T) {
-	var req, resp *TCPMessage
-
-	listener := NewListener("", "0", EnginePcap, true, 10*time.Millisecond)
-	defer listener.Close()
-
-	reqPacket := buildPacket(true, 1, 1, []byte("GET / HTTP/1.1\r\n\r\n"), time.Now())
-	respPacket := buildPacket(false, 1+uint32(len(reqPacket.Data)), 2, []byte("HTTP/1.1 200 OK\r\n\r\n"), time.Now())
-
-	// If response packet comes before request
-	listener.packetsChan <- respPacket.dump()
-	listener.packetsChan <- reqPacket.dump()
-
-	select {
-	case req = <-listener.messagesChan:
-	case <-time.After(time.Millisecond):
-		t.Error("Should return respose immediately")
-		return
-	}
-
-	if !req.IsIncoming {
-		t.Error("Should be request")
-	}
-
-	select {
-	case resp = <-listener.messagesChan:
-	case <-time.After(time.Millisecond):
-		t.Error("Should return response immediately")
-		return
-	}
-
-	if resp.IsIncoming {
-		t.Error("Should be response")
-	}
-
-	if !bytes.Equal(resp.UUID(), req.UUID()) {
-		t.Error("Resp and Req UUID should be equal")
-	}
-}
-
-func TestShort100Continue(t *testing.T) {
-	listener := NewListener("", "0", EnginePcap, true, 10*time.Millisecond)
-	defer listener.Close()
-
-	reqPacket1 := buildPacket(true, 1, 1, []byte("POST / HTTP/1.1\r\nContent-Length: 2\r\nExpect: 100-continue\r\n\r\n"), time.Now())
-	// Packet with data have different Seq
-	reqPacket2 := buildPacket(true, 2, reqPacket1.Seq+uint32(len(reqPacket1.Data)), []byte("a"), time.Now())
-	reqPacket3 := buildPacket(true, 2, reqPacket2.Seq+1, []byte("b"), time.Now())
-
-	respPacket1 := buildPacket(false, 10, 3, []byte("HTTP/1.1 100 Continue\r\n\r\n"), time.Now())
-
-	// panic(int(uint32(len(reqPacket1.Data)) + uint32(len(reqPacket2.Data)) + uint32(len(reqPacket3.Data))))
-	respPacket2 := buildPacket(false, reqPacket3.Seq+1 /* len of data */, 2, []byte("HTTP/1.1 200 OK\r\n\r\n"), time.Now())
-
-	result := []byte("POST / HTTP/1.1\r\nContent-Length: 2\r\n\r\nab")
-
-	testRawListener100Continue(t, listener, result, reqPacket1, reqPacket2, reqPacket3, respPacket1, respPacket2)
-}
-
-// Response comes before Request
-func Test100ContinueWrongOrder(t *testing.T) {
-	listener := NewListener("", "0", EnginePcap, true, 10*time.Millisecond)
-	defer listener.Close()
-
-	reqPacket1 := buildPacket(true, 1, 1, []byte("POST / HTTP/1.1\r\nContent-Length: 2\r\nExpect: 100-continue\r\n\r\n"), time.Now())
-	// Packet with data have different Seq
-	reqPacket2 := buildPacket(true, 2, reqPacket1.Seq+uint32(len(reqPacket1.Data)), []byte("a"), time.Now())
-	reqPacket3 := buildPacket(true, 2, reqPacket2.Seq+1, []byte("b"), time.Now())
-
-	respPacket1 := buildPacket(false, 10, 3, []byte("HTTP/1.1 100 Continue\r\n"), time.Now())
-
-	// panic(int(uint32(len(reqPacket1.Data)) + uint32(len(reqPacket2.Data)) + uint32(len(reqPacket3.Data))))
-	respPacket2 := buildPacket(false, reqPacket3.Seq+1 /* len of data */, 2, []byte("HTTP/1.1 200 OK\r\n\r\n"), time.Now())
-
-	result := []byte("POST / HTTP/1.1\r\nContent-Length: 2\r\n\r\nab")
-
-	testRawListener100Continue(t, listener, result, respPacket1, respPacket2, reqPacket1, reqPacket2, reqPacket3)
-}
-
-func TestAlt100ContinueHeaderOrder(t *testing.T) {
-	listener := NewListener("", "0", EnginePcap, true, 10*time.Millisecond)
-	defer listener.Close()
-
-	reqPacket1 := buildPacket(true, 1, 1, []byte("POST / HTTP/1.1\r\nExpect: 100-continue\r\nContent-Length: 2\r\n\r\n"), time.Now())
-	// Packet with data have different Seq
-	reqPacket2 := buildPacket(true, 2, reqPacket1.Seq+uint32(len(reqPacket1.Data)), []byte("a"), time.Now())
-	reqPacket3 := buildPacket(true, 2, reqPacket2.Seq+1, []byte("b"), time.Now())
-
-	respPacket1 := buildPacket(false, 10, 3, []byte("HTTP/1.1 100 Continue\r\n"), time.Now())
-
-	// panic(int(uint32(len(reqPacket1.Data)) + uint32(len(reqPacket2.Data)) + uint32(len(reqPacket3.Data))))
-	respPacket2 := buildPacket(false, reqPacket3.Seq+1 /* len of data */, 2, []byte("HTTP/1.1 200 OK\r\n\r\n"), time.Now())
-
-	result := []byte("POST / HTTP/1.1\r\nContent-Length: 2\r\n\r\nab")
-
-	testRawListener100Continue(t, listener, result, reqPacket1, reqPacket2, reqPacket3, respPacket1, respPacket2)
-}
-
-func testRawListener100Continue(t *testing.T, listener *Listener, result []byte, packets ...*TCPPacket) {
-	var req, resp *TCPMessage
-	for _, p := range packets {
-		listener.packetsChan <- p.dump()
-	}
-
-	select {
-	case req = <-listener.messagesChan:
-		break
-	case <-time.After(11 * time.Millisecond):
-		t.Error("Should return response after expire time")
-		return
-	}
-
-	if !bytes.Equal(req.Bytes(), result) {
-		t.Error("Should receive full message", string(req.Bytes()))
-	}
-
-	if !req.IsIncoming {
-		t.Error("Should be request")
-	}
-
-	select {
-	case resp = <-listener.messagesChan:
-		break
-	case <-time.After(21 * time.Millisecond):
-		t.Error("Should return response after expire time")
-		return
-	}
-
-	if resp.IsIncoming {
-		t.Error("Should be response")
-	}
-
-	if !bytes.Equal(resp.UUID(), req.UUID()) {
-		t.Error("Resp and Req UUID should be equal")
-	}
-}
-
-func testChunkedSequence(t *testing.T, listener *Listener, packets ...*TCPPacket) {
-	var r, req, resp *TCPMessage
-
-	for _, p := range packets {
-		listener.packetsChan <- p.dump()
-	}
-
-	select {
-	case r = <-listener.messagesChan:
-		if r.IsIncoming {
-			req = r
-		} else {
-			resp = r
-		}
-		break
-	case <-time.After(25 * time.Millisecond):
-		t.Error("Should return request after expire time")
-		return
-	}
-	select {
-	case r = <-listener.messagesChan:
-		if r.IsIncoming {
-			if req != nil {
-				t.Error("Request already received", r)
-				return
-			}
-			req = r
-		} else {
-			if resp != nil {
-				t.Error("Response already received", r)
-				return
-			}
-			resp = r
-		}
-		break
-	case <-time.After(25 * time.Millisecond):
-		t.Error("Should return request after expire time")
-		return
-	}
-
-	if !bytes.Equal(req.Bytes(), []byte("POST / HTTP/1.1\r\nTransfer-Encoding: chunked\r\n\r\n1\r\na\r\n1\r\nb\r\n0\r\n\r\n")) {
-		t.Error("Should receive full message", string(req.Bytes()))
-	}
-
-	if !req.IsIncoming {
-		t.Error("Should be request")
-	}
-
-	if resp.IsIncoming {
-		t.Error("Should be response")
-	}
-
-	if !bytes.Equal(resp.UUID(), req.UUID()) {
-		t.Error("Resp and Req UUID should be equal", string(resp.UUID()), string(req.UUID()))
-	}
-
-	time.Sleep(20 * time.Millisecond)
-
-	if len(listener.packetsChan) != 0 {
-		t.Fatal("packetsChan non empty:", listener.packetsChan)
-	}
-
-	if len(listener.messagesChan) != 0 {
-		t.Fatal("messagesChan non empty:", <-listener.messagesChan)
-	}
-
-	if len(listener.messages) != 0 {
-		t.Fatal("Messages non empty:", listener.messages)
-	}
-
-	if len(listener.ackAliases) != 0 {
-		t.Fatal("ackAliases non empty:", listener.ackAliases)
-	}
-
-	if len(listener.seqWithData) != 0 {
-		t.Fatal("seqWithData non empty:", listener.seqWithData)
-	}
-
-	if len(listener.respAliases) != 0 {
-		t.Fatal("respAliases non empty:", listener.respAliases)
-	}
-
-	if len(listener.respWithoutReq) != 0 {
-		t.Fatal("respWithoutReq non empty:", listener.respWithoutReq)
-	}
-}
-
-func permutation(n int, list []*TCPPacket) []*TCPPacket {
-	if len(list) == 1 {
-		return list
-	}
-
-	k := n % len(list)
-
-	first := []*TCPPacket{list[k]}
-	next := make([]*TCPPacket, len(list)-1)
-
-	copy(next, append(list[:k], list[k+1:]...))
-
-	return append(first, permutation(n/len(list), next)...)
-}
-
-// Response comes before Request
-func TestRawListenerChunkedWrongOrder(t *testing.T) {
-	listener := NewListener("", "0", EnginePcap, true, 10*time.Millisecond)
-	defer listener.Close()
-
-	reqPacket1 := buildPacket(true, 1, 1, []byte("POST / HTTP/1.1\r\nTransfer-Encoding: chunked\r\nExpect: 100-continue\r\n\r\n"), time.Now())
-	// Packet with data have different Seq
-	reqPacket2 := buildPacket(true, 2, reqPacket1.Seq+uint32(len(reqPacket1.Data)), []byte("1\r\na\r\n"), time.Now())
-	reqPacket3 := buildPacket(true, 2, reqPacket2.Seq+uint32(len(reqPacket2.Data)), []byte("1\r\nb\r\n"), time.Now())
-	reqPacket4 := buildPacket(true, 2, reqPacket3.Seq+uint32(len(reqPacket3.Data)), []byte("0\r\n\r\n"), time.Now())
-
-	respPacket1 := buildPacket(false, 10, 3, []byte("HTTP/1.1 100 Continue\r\n\r\n"), time.Now())
-
-	// panic(int(uint32(len(reqPacket1.Data)) + uint32(len(reqPacket2.Data)) + uint32(len(reqPacket3.Data))))
-	respPacket2 := buildPacket(false, reqPacket4.Seq+5 /* len of data */, 2, []byte("HTTP/1.1 200 OK\r\n\r\n"), time.Now())
-
-	// Should re-construct message from all possible combinations
-	for i := 0; i < 6*5*4*3*2*1; i++ {
-
-		// if i < 54 || i > 57 {
-		// 	continue
-		// }
-
-		packets := permutation(i, []*TCPPacket{reqPacket1, reqPacket2, reqPacket3, reqPacket4, respPacket1, respPacket2})
-
-		t.Log("permutation:", i)
-		testChunkedSequence(t, listener, packets...)
-	}
-}
-
-func chunkedPostMessage() []*TCPPacket {
-	ack := uint32(rand.Int63())
-	seq := uint32(rand.Int63())
-
-	reqPacket1 := buildPacket(true, ack, seq, []byte("POST / HTTP/1.1\r\nTransfer-Encoding: chunked\r\n\r\n"), time.Now())
-	// Packet with data have different Seq
-	reqPacket2 := buildPacket(true, ack, seq+47, []byte("1\r\na\r\n"), time.Now())
-	reqPacket3 := buildPacket(true, ack, reqPacket2.Seq+5, []byte("1\r\nb\r\n"), time.Now())
-	reqPacket4 := buildPacket(true, ack, reqPacket3.Seq+5, []byte("0\r\n\r\n"), time.Now())
-
-	respPacket := buildPacket(false, reqPacket4.Seq+5 /* len of data */, ack, []byte("HTTP/1.1 200 OK\r\n\r\n"), time.Now())
-
-	return []*TCPPacket{
-		reqPacket1, reqPacket2, reqPacket3, reqPacket4, respPacket,
-	}
-}
-
-func postMessage() []*TCPPacket {
-	ack := uint32(rand.Int63())
-	seq2 := uint32(rand.Int63())
-	seq := uint32(rand.Int63())
-
-	c := 10000
-	data := make([]byte, c)
-	rand.Read(data)
-
-	head := []byte("POST / HTTP/1.1\r\nContent-Length: 9958\r\n\r\n")
-	for i := range head {
-		data[i] = head[i]
-	}
-
-	return []*TCPPacket{
-		buildPacket(true, ack, seq, data, time.Now()),
-		buildPacket(false, seq+uint32(len(data)), seq2, []byte("HTTP/1.1 200 OK\r\n\r\n"), time.Now()),
-	}
-}
-
-func getMessage() []*TCPPacket {
-	ack := uint32(rand.Int63())
-	seq2 := uint32(rand.Int63())
-	seq := uint32(rand.Int63())
-
-	return []*TCPPacket{
-		buildPacket(true, ack, seq, []byte("GET / HTTP/1.1\r\n\r\n"), time.Now()),
-		buildPacket(false, seq+18, seq2, []byte("HTTP/1.1 200 OK\r\n\r\n"), time.Now()),
-	}
-}
-
-// Response comes before Request
-func TestRawListenerBench(t *testing.T) {
-	l := NewListener("", "0", EnginePcap, true, 200*time.Millisecond)
-	defer l.Close()
-
-	// Should re-construct message from all possible combinations
-	for i := 0; i < 1000; i++ {
-		go func(i int) {
-			for j := 0; j < 100; j++ {
-				var packets []*TCPPacket
-
-				if j%5 == 0 {
-					packets = chunkedPostMessage()
-				} else if j%3 == 0 {
-					packets = postMessage()
-				} else {
-					packets = getMessage()
-				}
-
-				for _, p := range packets {
-					// Randomly drop packets
-					if (i+j)%5 == 0 {
-						if rand.Int63()%3 == 0 {
-							continue
-						}
-					}
-
-					l.packetsChan <- p.dump()
-					time.Sleep(time.Millisecond)
-				}
-
-				time.Sleep(5 * time.Millisecond)
-			}
-		}(i)
-	}
-
-	ch := l.Receiver()
-
-	var count int32
-
-	for {
-		select {
-		case <-ch:
-			atomic.AddInt32(&count, 1)
-		case <-time.After(2000 * time.Millisecond):
-			log.Println("Emitted 200000 messages, captured: ", count, len(l.ackAliases), len(l.seqWithData), len(l.respAliases), len(l.respWithoutReq), len(l.packetsChan))
-			return
-		}
-	}
-}

raw_socket_listener/tcp_message.go

@@ -1,511 +0,0 @@
-package rawSocket
-
-import (
-	"bytes"
-	"crypto/sha1"
-	"encoding/binary"
-	"encoding/hex"
-	"github.com/buger/gor/proto"
-	"log"
-	"net"
-	"strconv"
-	"time"
-)
-
-var _ = log.Println
-
-// TCPMessage ensure that all TCP packets for given request is received, and processed in right sequence
-// Its needed because all TCP message can be fragmented or re-transmitted
-//
-// Each TCP Packet have 2 ids: acknowledgment - message_id, and sequence - packet_id
-// Message can be compiled from unique packets with same message_id which sorted by sequence
-// Message is received if we didn't receive any packets for 2000ms
-type TCPMessage struct {
-	Seq         uint32
-	Ack         uint32
-	ResponseAck uint32
-	ResponseID  tcpID
-	DataAck     uint32
-	DataSeq     uint32
-
-	AssocMessage *TCPMessage
-	Start        time.Time
-	End          time.Time
-	IsIncoming   bool
-
-	packets []*TCPPacket
-
-	delChan chan *TCPMessage
-
-	/* HTTP specific variables */
-	methodType    httpMethodType
-	bodyType      httpBodyType
-	expectType    httpExpectType
-	seqMissing    bool
-	headerPacket  int
-	contentLength int
-	complete      bool
-}
-
-// NewTCPMessage pointer created from a Acknowledgment number and a channel of messages readuy to be deleted
-func NewTCPMessage(Seq, Ack uint32, IsIncoming bool, timestamp time.Time) (msg *TCPMessage) {
-	msg = &TCPMessage{Seq: Seq, Ack: Ack, IsIncoming: IsIncoming, Start: timestamp}
-
-	return
-}
-
-func (t *TCPMessage) packetsData() (d [][]byte) {
-	d = make([][]byte, len(t.packets))
-	for i, p := range t.packets {
-		d[i] = p.Data
-	}
-
-	return
-}
-
-// Bytes return message content
-func (t *TCPMessage) Bytes() (output []byte) {
-	for _, p := range t.packets {
-		output = append(output, p.Data...)
-	}
-
-	return output
-}
-
-// Size returns total body size
-func (t *TCPMessage) BodySize() (size int) {
-	if len(t.packets) == 0 || t.headerPacket == -1 {
-		return 0
-	}
-
-	size += len(proto.Body(t.packets[t.headerPacket].Data))
-
-	for _, p := range t.packets[t.headerPacket+1:] {
-		size += len(p.Data)
-	}
-
-	return
-}
-
-// Size returns total size of message
-func (t *TCPMessage) Size() (size int) {
-	if len(t.packets) == 0 {
-		return 0
-	}
-
-	for _, p := range t.packets {
-		size += len(p.Data)
-	}
-
-	return
-}
-
-// AddPacket to the message and ensure packet uniqueness
-// TCP allows that packet can be re-send multiple times
-func (t *TCPMessage) AddPacket(packet *TCPPacket) {
-	packetFound := false
-
-	for _, pkt := range t.packets {
-		if packet.Seq == pkt.Seq {
-			packetFound = true
-			break
-		}
-	}
-
-	if !packetFound {
-		// Packets not always captured in same Seq order, and sometimes we need to prepend
-		if len(t.packets) == 0 || packet.Seq > t.packets[len(t.packets)-1].Seq {
-			t.packets = append(t.packets, packet)
-		} else if packet.Seq < t.packets[0].Seq {
-			t.packets = append([]*TCPPacket{packet}, t.packets...)
-			t.Seq = packet.Seq // Message Seq should indicated starting seq
-		} else { // insert somewhere in the middle...
-			for i, p := range t.packets {
-				if packet.Seq < p.Seq {
-					t.packets = append(t.packets[:i], append([]*TCPPacket{packet}, t.packets[i:]...)...)
-					break
-				}
-			}
-		}
-
-		if t.IsIncoming {
-			t.End = time.Now()
-		} else {
-			t.End = time.Now().Add(time.Millisecond)
-		}
-
-		if packet.OrigAck != 0 {
-			t.DataAck = packet.OrigAck
-		}
-
-		if packet.timestamp.Before(t.Start) {
-			t.Start = packet.timestamp
-		}
-	}
-
-	t.checkSeqIntegrity()
-	t.updateHeadersPacket()
-	t.updateMethodType()
-	t.updateBodyType()
-	t.checkIfComplete()
-	t.check100Continue()
-}
-
-// Check if there is missing packet
-func (t *TCPMessage) checkSeqIntegrity() {
-	if len(t.packets) == 1 {
-		t.seqMissing = false
-	}
-
-	offset := len(t.packets) - 1
-
-	if t.packets[offset].IsFIN {
-		offset--
-
-		if offset < 0 {
-			return
-		}
-	}
-
-	for i, p := range t.packets[:offset] {
-		if p.IsFIN {
-			continue
-		}
-
-		// If final packet
-		if len(t.packets) == i+1 {
-			t.seqMissing = false
-			return
-		}
-		np := t.packets[i+1]
-
-		nextSeq := p.Seq + uint32(len(p.Data))
-
-		if np.Seq != nextSeq {
-			if t.expectType == httpExpect100Continue {
-				if np.Seq != nextSeq+22 {
-					t.seqMissing = true
-					return
-				}
-			} else {
-				t.seqMissing = true
-				return
-			}
-		}
-	}
-
-	t.seqMissing = false
-}
-
-var bEmptyLine = []byte("\r\n\r\n")
-var bChunkEnd = []byte("0\r\n\r\n")
-
-func (t *TCPMessage) updateHeadersPacket() {
-	if len(t.packets) == 1 {
-		t.headerPacket = -1
-	}
-
-	if t.headerPacket != -1 {
-		return
-	}
-
-	if t.seqMissing {
-		return
-	}
-
-	for i, p := range t.packets {
-		if bytes.LastIndex(p.Data, bEmptyLine) != -1 {
-			t.headerPacket = i
-			return
-		}
-	}
-
-	return
-}
-
-// isMultipart returns true if message contains from multiple tcp packets
-func (t *TCPMessage) checkIfComplete() {
-	if t.seqMissing || t.headerPacket == -1 {
-		return
-	}
-
-	if t.methodType == httpMethodNotFound {
-		return
-	}
-
-	// Responses can be emitted only if we found request
-	if !t.IsIncoming && t.AssocMessage == nil {
-		return
-	}
-
-	// If one GET, OPTIONS, or HEAD request
-	if t.methodType == httpMethodWithoutBody {
-		t.complete = true
-	} else {
-		switch t.bodyType {
-		case httpBodyEmpty:
-			t.complete = true
-		case httpBodyContentLength:
-			if t.contentLength == 0 || t.contentLength == t.BodySize() {
-				t.complete = true
-			}
-		case httpBodyChunked:
-			lastPacket := t.packets[len(t.packets)-1]
-			if bytes.LastIndex(lastPacket.Data, bChunkEnd) != -1 {
-				t.complete = true
-			}
-		default:
-			if len(t.packets) == 0 {
-				return
-			}
-
-			last := t.packets[len(t.packets)-1]
-			if last.IsFIN {
-				t.complete = true
-			}
-		}
-	}
-}
-
-type httpMethodType uint8
-
-const (
-	httpMethodNotSet      httpMethodType = 0
-	httpMethodWithBody    httpMethodType = 1
-	httpMethodWithoutBody httpMethodType = 2
-	httpMethodNotFound    httpMethodType = 3
-)
-
-var methodsWithBody = [][]byte{
-	[]byte("POST"),
-	[]byte("PUT"),
-	[]byte("PATCH"),
-	[]byte("CONNECT"),
-}
-
-func (t *TCPMessage) updateMethodType() {
-	// if there is cache
-	if t.methodType != httpMethodNotSet && t.methodType != httpMethodNotFound {
-		return
-	}
-
-	d := t.packets[0].Data
-
-	// Minimum length fo request: GET / HTTP/1.1\r\n
-
-	if len(d) < 16 {
-		t.methodType = httpMethodNotFound
-		return
-	}
-
-	if t.IsIncoming {
-		var method []byte
-
-		if mIdx := bytes.IndexByte(d[:8], ' '); mIdx != -1 {
-			method = d[:mIdx]
-
-			// Check that after method we have absolute or relative path
-			switch d[mIdx+1] {
-			case '/', 'h', '*':
-			default:
-				t.methodType = httpMethodNotFound
-				return
-			}
-		} else {
-			t.methodType = httpMethodNotFound
-			return
-		}
-
-		for _, m := range methodsWithBody {
-			if len(m) == len(method) && bytes.Equal(m, method) {
-				t.methodType = httpMethodWithBody
-				return
-			}
-		}
-
-		t.methodType = httpMethodWithoutBody
-	} else {
-		if !bytes.Equal(d[:6], []byte("HTTP/1")) {
-			t.methodType = httpMethodNotFound
-			return
-		}
-
-		t.methodType = httpMethodWithBody
-	}
-}
-
-type httpBodyType uint8
-
-const (
-	httpBodyNotSet          httpBodyType = 0
-	httpBodyEmpty           httpBodyType = 1
-	httpBodyContentLength   httpBodyType = 2
-	httpBodyChunked         httpBodyType = 3
-	httpBodyConnectionClose httpBodyType = 4
-)
-
-func (t *TCPMessage) updateBodyType() {
-	// if there is cache
-	if t.bodyType != httpBodyNotSet {
-		return
-	}
-
-	// Headers not received
-	if t.headerPacket == -1 {
-		return
-	}
-
-	var lengthB, encB, connB []byte
-
-	proto.ParseHeaders(t.packetsData(), func(header, value []byte)bool{
-		if proto.HeadersEqual(header, []byte("Content-Length")) {
-			lengthB = value
-			return false
-		}
-
-		if proto.HeadersEqual(header, []byte("Transfer-Encoding")) {
-			encB = value
-			return false
-		}
-
-		if proto.HeadersEqual(header, []byte("Connection")) {
-			connB = value
-		}
-
-		return true
-	})
-
-	switch t.methodType {
-	case httpMethodNotFound:
-		return
-	case httpMethodWithoutBody:
-		t.bodyType = httpBodyEmpty
-		return
-	case httpMethodWithBody:
-		if len(lengthB) > 0 {
-			t.contentLength, _ = strconv.Atoi(string(lengthB))
-
-			if t.contentLength == 0 {
-				t.bodyType = httpBodyEmpty
-			} else {
-				t.bodyType = httpBodyContentLength
-			}
-			return
-		}
-
-		if len(encB) > 0 {
-			t.bodyType = httpBodyChunked
-			return
-		}
-
-		if len(connB) > 0 && bytes.Equal(connB, []byte("close")) {
-			t.bodyType = httpBodyConnectionClose
-			return
-		}
-	}
-
-	t.bodyType = httpBodyEmpty
-}
-
-type httpExpectType uint8
-
-const (
-	httpExpectNotSet      httpExpectType = 0
-	httpExpectEmpty       httpExpectType = 1
-	httpExpect100Continue httpExpectType = 2
-)
-
-var bExpectHeader = []byte("Expect")
-var bExpect100Value = []byte("100-continue")
-
-func (t *TCPMessage) check100Continue() {
-	if t.expectType != httpExpectNotSet || len(t.packets[0].Data) < 25 {
-		return
-	}
-
-	if t.methodType != httpMethodWithBody {
-		return
-	}
-
-	if t.seqMissing || t.headerPacket == -1 {
-		return
-	}
-
-	last := t.packets[len(t.packets)-1]
-	// reading last 4 bytes for double CRLF
-	if !bytes.HasSuffix(last.Data, bEmptyLine) {
-		return
-	}
-
-	var expectB []byte
-	proto.ParseHeaders(t.packetsData(), func(header, value []byte)bool{
-		if proto.HeadersEqual(header, bExpectHeader) {
-			expectB = value
-			return false
-		}
-
-		return true
-	})
-
-	if len(expectB) > 0 && bytes.Equal(bExpect100Value, expectB) {
-		t.expectType = httpExpect100Continue
-		return
-	}
-
-	t.expectType = httpExpectEmpty
-}
-
-func (t *TCPMessage) setAssocMessage(m *TCPMessage) {
-	t.AssocMessage = m
-	t.checkIfComplete()
-}
-
-// UpdateResponseAck should be called after packet is added
-func (t *TCPMessage) UpdateResponseAck() uint32 {
-	lastPacket := t.packets[len(t.packets)-1]
-	if lastPacket.IsFIN && len(t.packets) > 1 {
-		lastPacket = t.packets[len(t.packets)-2]
-	}
-
-	respAck := lastPacket.Seq + uint32(len(lastPacket.Data))
-
-	if t.ResponseAck != respAck {
-		t.ResponseAck = lastPacket.Seq + uint32(len(lastPacket.Data))
-
-		// We swappwed src and dst port
-		copy(t.ResponseID[:16], lastPacket.Addr)
-		copy(t.ResponseID[16:], lastPacket.Raw[2:4]) // Src port
-		copy(t.ResponseID[18:], lastPacket.Raw[0:2]) // Dest port
-		binary.BigEndian.PutUint32(t.ResponseID[20:24], t.ResponseAck)
-	}
-
-	return t.ResponseAck
-}
-
-func (t *TCPMessage) UUID() []byte {
-	var key []byte
-
-	if t.IsIncoming {
-		// log.Println("UUID:", t.Ack, t.Start.UnixNano())
-		key = strconv.AppendInt(key, t.Start.UnixNano(), 10)
-		key = strconv.AppendUint(key, uint64(t.Ack), 10)
-	} else {
-		// log.Println("RequestMessage:", t.AssocMessage.Ack, t.AssocMessage.Start.UnixNano())
-		key = strconv.AppendInt(key, t.AssocMessage.Start.UnixNano(), 10)
-		key = strconv.AppendUint(key, uint64(t.AssocMessage.Ack), 10)
-	}
-
-	uuid := make([]byte, 40)
-	sha := sha1.Sum(key)
-	hex.Encode(uuid, sha[:20])
-
-	return uuid
-}
-
-func (t *TCPMessage) ID() tcpID {
-	return t.packets[0].ID
-}
-
-func (t *TCPMessage) IP() net.IP {
-	return net.IP(t.packets[0].Addr)
-}

raw_socket_listener/tcp_message_test.go

@@ -1,258 +0,0 @@
-package rawSocket
-
-import (
-	"bytes"
-	"encoding/binary"
-	_ "log"
-	"testing"
-	"time"
-)
-
-func buildPacket(isIncoming bool, Ack, Seq uint32, Data []byte, timestamp time.Time) (packet *TCPPacket) {
-	var srcPort, destPort uint16
-
-	// For tests `listening` port is 0
-	if isIncoming {
-		srcPort = 1
-	} else {
-		destPort = 1
-	}
-
-	buf := make([]byte, 16)
-	binary.BigEndian.PutUint16(buf[2:4], destPort)
-	binary.BigEndian.PutUint16(buf[0:2], srcPort)
-	binary.BigEndian.PutUint32(buf[4:8], Seq)
-	binary.BigEndian.PutUint32(buf[8:12], Ack)
-	buf[12] = 64
-	buf = append(buf, Data...)
-
-	packet = ParseTCPPacket([]byte("123"), buf, timestamp)
-
-	return packet
-}
-
-func buildMessage(p *TCPPacket) *TCPMessage {
-	isIncoming := false
-	if p.SrcPort == 1 {
-		isIncoming = true
-	}
-
-	m := NewTCPMessage(p.Seq, p.Ack, isIncoming, p.timestamp)
-	m.AddPacket(p)
-
-	return m
-}
-
-func TestTCPMessagePacketsOrder(t *testing.T) {
-	msg := buildMessage(buildPacket(true, 1, 1, []byte("a"), time.Now()))
-	msg.AddPacket(buildPacket(true, 1, 2, []byte("b"), time.Now()))
-
-	if !bytes.Equal(msg.Bytes(), []byte("ab")) {
-		t.Error("Should contatenate packets in right order")
-	}
-
-	// When first packet have wrong order (Seq)
-	msg = buildMessage(buildPacket(true, 1, 2, []byte("b"), time.Now()))
-	msg.AddPacket(buildPacket(true, 1, 1, []byte("a"), time.Now()))
-
-	if !bytes.Equal(msg.Bytes(), []byte("ab")) {
-		t.Error("Should contatenate packets in right order")
-	}
-
-	// Should ignore packets with same sequence
-	msg = buildMessage(buildPacket(true, 1, 1, []byte("a"), time.Now()))
-	msg.AddPacket(buildPacket(true, 1, 1, []byte("a"), time.Now()))
-
-	if !bytes.Equal(msg.Bytes(), []byte("a")) {
-		t.Error("Should ignore packet with same Seq")
-	}
-}
-
-func TestTCPMessageSize(t *testing.T) {
-	msg := buildMessage(buildPacket(true, 1, 1, []byte("POST / HTTP/1.1\r\nContent-Length: 2\r\n\r\na"), time.Now()))
-	msg.AddPacket(buildPacket(true, 1, 2, []byte("b"), time.Now()))
-
-	if msg.BodySize() != 2 {
-		t.Error("Should count only body", msg.BodySize())
-	}
-
-	if msg.Size() != 40 {
-		t.Error("Should count all sizes", msg.Size())
-	}
-}
-
-func TestTCPMessageIsComplete(t *testing.T) {
-	testCases := []struct {
-		direction         bool
-		payload           string
-		assocMessage      bool
-		expectedCompleted bool
-	}{
-		{true, "GET / HTTP/1.1\r\n\r\n", false, true},
-		{true, "HEAD / HTTP/1.1\r\n\r\n", false, true},
-		{false, "HTTP/1.1 200 OK\r\n\r\n", true, true},
-		{true, "POST / HTTP/1.1\r\nContent-Length: 1\r\n\r\na", false, true},
-		{true, "PUT / HTTP/1.1\r\nContent-Length: 1\r\n\r\na", false, true},
-		{false, "HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n", true, true},
-		{false, "HTTP/1.1 200 OK\r\nContent-Length: 1\r\n\r\na", true, true},
-		{false, "HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n0\r\n\r\n", true, true},
-
-		// chunked not finished
-		{false, "HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n", true, false},
-
-		// content-length != actual length
-		{true, "POST / HTTP/1.1\r\nContent-Length: 2\r\n\r\na", false, false},
-		{false, "HTTP/1.1 200 OK\r\nContent-Length: 10\r\n\r\na", true, false},
-		// non-valid http request
-		{true, "UNKNOWN asd HTTP/1.1\r\n\r\n", false, false},
-
-		// response without associated request
-		{false, "HTTP/1.1 200 OK\r\n\r\n", false, false},
-	}
-
-	for _, tc := range testCases {
-		msg := buildMessage(buildPacket(tc.direction, 1, 1, []byte(tc.payload), time.Now()))
-		if tc.assocMessage {
-			msg.AssocMessage = &TCPMessage{}
-		}
-		msg.checkIfComplete()
-
-		if msg.complete != tc.expectedCompleted {
-			t.Errorf("Payload %s: Expected %t, got %t.", tc.payload, tc.expectedCompleted, msg.complete)
-		}
-	}
-}
-
-func TestTCPMessageIsSeqMissing(t *testing.T) {
-	p1 := buildPacket(false, 1, 1, []byte("HTTP/1.1 200 OK\r\n"), time.Now())
-	p2 := buildPacket(false, 1, p1.Seq+uint32(len(p1.Data)), []byte("Content-Length: 10\r\n\r\n"), time.Now())
-	p3 := buildPacket(false, 1, p2.Seq+uint32(len(p2.Data)), []byte("a"), time.Now())
-
-	msg := buildMessage(p1)
-	if msg.seqMissing {
-		t.Error("Should be complete if have only 1 packet")
-	}
-
-	msg.AddPacket(p3)
-	if !msg.seqMissing {
-		t.Error("Should be incomplete because missing middle component")
-	}
-
-	msg.AddPacket(p2)
-	if msg.seqMissing {
-		t.Error("Should be complete once missing packet added")
-	}
-}
-
-func TestTCPMessageIsHeadersReceived(t *testing.T) {
-	p1 := buildPacket(false, 1, 1, []byte("HTTP/1.1 200 OK\r\n\r\n"), time.Now())
-	p2 := buildPacket(false, 1, p1.Seq+uint32(len(p1.Data)), []byte("Content-Length: 10\r\n\r\n"), time.Now())
-
-	msg := buildMessage(p1)
-	if msg.headerPacket == -1 {
-		t.Error("Should be complete if have only 1 packet", msg.headerPacket)
-	}
-
-	msg.AddPacket(p2)
-	if msg.headerPacket == -1 {
-		t.Error("Should found double new line: headers received")
-	}
-
-	msg = buildMessage(buildPacket(true, 1, 1, []byte("GET / HTTP/1.1\r\nContent-Length: 1\r\n"), time.Now()))
-	if msg.headerPacket != -1 {
-		t.Error("Should not find headers end")
-	}
-}
-
-func TestTCPMessageMethodType(t *testing.T) {
-	testCases := []struct {
-		direction          bool
-		payload            string
-		expectedMethodType httpMethodType
-	}{
-		{true, "GET / HTTP/1.1\r\n\r\n", httpMethodWithoutBody},
-		{true, "GET * HTTP/1.1\r\n\r\n", httpMethodWithoutBody},
-		{true, "UNKNOWN / HTTP/1.1\r\n\r\n", httpMethodWithoutBody},
-		{true, "GET http://example.com HTTP/1.1\r\n\r\n", httpMethodWithoutBody},
-		{true, "POST / HTTP/1.1\r\n\r\n", httpMethodWithBody},
-		{true, "PUT / HTTP/1.1\r\n\r\n", httpMethodWithBody},
-		{true, "GET zxc HTTP/1.1\r\n\r\n", httpMethodNotFound},
-		{true, "GET / HTTP\r\n\r\n", httpMethodNotFound},
-		{true, "VERYLONGMETHOD / HTTP/1.1\r\n\r\n", httpMethodNotFound},
-		{false, "HTTP/1.1 200 OK\r\n\r\n", httpMethodWithBody},
-		{false, "HTTP /1.1 200 OK\r\n\r\n", httpMethodNotFound},
-	}
-
-	for _, tc := range testCases {
-		msg := buildMessage(buildPacket(tc.direction, 1, 1, []byte(tc.payload), time.Now()))
-
-		if msg.methodType != tc.expectedMethodType {
-			t.Errorf("Expected %d, got %d", tc.expectedMethodType, msg.methodType)
-		}
-	}
-}
-
-func TestTCPMessageBodyType(t *testing.T) {
-	testCases := []struct {
-		direction        bool
-		payload          string
-		expectedBodyType httpBodyType
-	}{
-		{true, "GET / HTTP/1.1\r\n\r\n", httpBodyEmpty},
-		{true, "POST / HTTP/1.1\r\n\r\n", httpBodyEmpty},
-		{true, "POST / HTTP/1.1\r\nUser-Agent: zxc\r\n\r\n", httpBodyEmpty},
-		{false, "HTTP/1.1 200 OK\r\n\r\n", httpBodyEmpty},
-		{true, "POST / HTTP/1.1\r\nContent-Length: 2\r\n\r\nab", httpBodyContentLength},
-		{false, "HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nab", httpBodyContentLength},
-		{true, "POST / HTTP/1.1\r\nTransfer-Encoding: chunked\r\n\r\n2\r\nab\r\n0\r\n\r\n", httpBodyChunked},
-		{false, "HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n2\r\nab\r\n0\r\n\r\n", httpBodyChunked},
-	}
-
-	for _, tc := range testCases {
-		msg := buildMessage(buildPacket(tc.direction, 1, 1, []byte(tc.payload), time.Now()))
-
-		if msg.bodyType != tc.expectedBodyType {
-			t.Errorf("Expected %d, got %d", tc.expectedBodyType, msg.bodyType)
-		}
-	}
-}
-
-func TestTCPMessageBodySize(t *testing.T) {
-	testCases := []struct {
-		direction    bool
-		payloads     []string
-		expectedSize int
-	}{
-		{true, []string{"GET / HTTP/1.1\r\n\r\n"}, 0},
-		{true, []string{"POST / HTTP/1.1\r\nContent-Length: 2\r\n\r\nab"}, 2},
-		{true, []string{"GET / HTTP/1.1\r\n", "Content-Length: 2\r\n\r\nab"}, 2},
-		{true, []string{"GET / HTTP/1.1\r\n", "Content-Length: 2\r\n\r\n", "ab"}, 2},
-	}
-
-	for _, tc := range testCases {
-		msg := buildMessage(buildPacket(tc.direction, 1, 1, []byte(tc.payloads[0]), time.Now()))
-
-		if len(tc.payloads) > 1 {
-			for _, p := range tc.payloads[1:] {
-				seq := uint32(1 + msg.Size())
-				msg.AddPacket(buildPacket(tc.direction, 1, seq, []byte(p), time.Now()))
-			}
-		}
-
-		if msg.BodySize() != tc.expectedSize {
-			t.Errorf("Expected %d, got %d", tc.expectedSize, msg.BodySize())
-		}
-	}
-}
-
-func TestTcpMessageStart(t *testing.T) {
-	start := time.Now().Add(-1 * time.Second)
-
-	msg := buildMessage(buildPacket(true, 1, 2, []byte("b"), time.Now()))
-	msg.AddPacket(buildPacket(true, 1, 1, []byte("POST / HTTP/1.1\r\nContent-Length: 2\r\n\r\na"), start))
-
-	if msg.Start != start {
-		t.Error("Message timestamp should be equal to the lowest related packet timestamp", start, msg.Start)
-	}
-}
-

raw_socket_listener/tcp_packet.go

@@ -1,133 +0,0 @@
-package rawSocket
-
-import (
-	"encoding/binary"
-	"log"
-	"strconv"
-	"strings"
-	"time"
-)
-
-var _ = log.Println
-
-// TCP Flags
-const (
-	fFIN = 1 << iota
-	fSYN
-	fRST
-	fPSH
-	fACK
-	fURG
-	fECE
-	fCWR
-	fNS
-)
-
-type tcpID [24]byte
-
-// TCPPacket provides tcp packet parser
-// Packet structure: http://en.wikipedia.org/wiki/Transmission_Control_Protocol
-type TCPPacket struct {
-	SrcPort    uint16
-	DestPort   uint16
-	Seq        uint32
-	Ack        uint32
-	OrigAck    uint32
-	DataOffset uint8
-	IsFIN      bool
-
-	Raw  []byte
-	Data []byte
-	Addr []byte
-	timestamp time.Time
-	ID   tcpID
-}
-
-// ParseTCPPacket takes address and tcp payload and returns parsed TCPPacket
-func ParseTCPPacket(addr []byte, data []byte, timestamp time.Time) (p *TCPPacket) {
-	p = &TCPPacket{Raw: data}
-	p.ParseBasic()
-	p.Addr = addr
-	p.timestamp = timestamp
-	p.GenID()
-
-	return
-}
-
-func (p *TCPPacket) GenID() {
-	copy(p.ID[:16], p.Addr)
-	copy(p.ID[16:], p.Raw[0:2])  // Src port
-	copy(p.ID[18:], p.Raw[2:4])  // Dest port
-	copy(p.ID[20:], p.Raw[8:12]) // Ack
-}
-
-func (p *TCPPacket) UpdateAck(ack uint32) {
-	p.OrigAck = p.Ack
-	p.Ack = ack
-	binary.BigEndian.PutUint32(p.Raw[8:12], ack)
-	p.GenID()
-}
-
-// ParseBasic set of fields
-func (t *TCPPacket) ParseBasic() {
-	t.DestPort = binary.BigEndian.Uint16(t.Raw[2:4])
-	t.SrcPort = binary.BigEndian.Uint16(t.Raw[0:2])
-	t.Seq = binary.BigEndian.Uint32(t.Raw[4:8])
-	t.Ack = binary.BigEndian.Uint32(t.Raw[8:12])
-	t.DataOffset = (t.Raw[12] & 0xF0) >> 4
-	t.IsFIN = t.Raw[13]&0x01 != 0
-
-	// log.Println("DataOffset:", t.DataOffset, t.DestPort, t.SrcPort, t.Seq, t.Ack)
-
-	t.Data = t.Raw[t.DataOffset*4:]
-}
-
-func (t *TCPPacket) dump() *packet {
-
-	packetSrcIP := make([]byte, 16)
-	packetData := make([]byte, len(t.Data) + 16)
-
-	copy(packetSrcIP, t.Addr)
-
-	binary.BigEndian.PutUint16(packetData[0:2], t.SrcPort)
-	binary.BigEndian.PutUint16(packetData[2:4], t.DestPort)
-
-	binary.BigEndian.PutUint32(packetData[4:8], t.Seq)
-	binary.BigEndian.PutUint32(packetData[8:12], t.Ack)
-
-	packetData[12] = 64
-
-	if t.IsFIN {
-		packetData[13] = packetData[13] | 0x01
-	}
-
-	copy(packetData[16:], t.Data)
-
-	return &packet{
-		srcIP: packetSrcIP,
-		data:packetData,
-		timestamp:t.timestamp,
-	}
-
-}
-
-// String output for a TCP Packet
-func (t *TCPPacket) String() string {
-	maxLen := len(t.Data)
-	if maxLen > 200 {
-		maxLen = 200
-	}
-
-	return strings.Join([]string{
-		"Addr: " + string(t.Addr),
-		"Source port: " + strconv.Itoa(int(t.SrcPort)),
-		"Dest port:" + strconv.Itoa(int(t.DestPort)),
-		"Sequence:" + strconv.Itoa(int(t.Seq)),
-		"Acknowledgment:" + strconv.Itoa(int(t.Ack)),
-		"Header len:" + strconv.Itoa(int(t.DataOffset)),
-		"FIN:" + strconv.FormatBool(t.IsFIN),
-
-		"Data size:" + strconv.Itoa(len(t.Data)),
-		"Data:" + string(t.Data[:maxLen]),
-	}, "\n")
-}

s3_reader.go

@@ -0,0 +1,105 @@
+package main
+
+import (
+	"bytes"
+	"log"
+	"os"
+	"strconv"
+	"strings"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/s3"
+)
+
+// S3ReadCloser ...
+type S3ReadCloser struct {
+	bucket    string
+	key       string
+	offset    int
+	totalSize int
+	readBytes int
+	sess      *session.Session
+	buf       *bytes.Buffer
+}
+
+func awsConfig() *aws.Config {
+	region := os.Getenv("AWS_DEFAULT_REGION")
+	if region == "" {
+		region = os.Getenv("AWS_REGION")
+		if region == "" {
+			region = "us-east-1"
+		}
+	}
+
+	config := &aws.Config{Region: aws.String(region)}
+
+	if endpoint := os.Getenv("AWS_ENDPOINT_URL"); endpoint != "" {
+		config.Endpoint = aws.String(endpoint)
+		log.Println("Custom endpoint:", endpoint)
+	}
+
+	log.Println("Connecting to S3. Region: " + region)
+
+	config.CredentialsChainVerboseErrors = aws.Bool(true)
+
+	if os.Getenv("AWS_DEBUG") != "" {
+		config.LogLevel = aws.LogLevel(aws.LogDebugWithHTTPBody)
+	}
+
+	return config
+}
+
+// NewS3ReadCloser returns new instance of S3 read closer
+func NewS3ReadCloser(path string) *S3ReadCloser {
+	if !PRO {
+		log.Fatal("Using S3 input and output require PRO license")
+		return nil
+	}
+
+	bucket, key := parseS3Url(path)
+	sess := session.Must(session.NewSession(awsConfig()))
+
+	log.Println("[S3 Input] S3 connection successfully initialized", path)
+
+	return &S3ReadCloser{
+		bucket: bucket,
+		key:    key,
+		sess:   sess,
+		buf:    &bytes.Buffer{},
+	}
+}
+
+// Read reads buffer from s3 session
+func (s *S3ReadCloser) Read(b []byte) (n int, e error) {
+	if s.readBytes == 0 || s.readBytes+len(b) > s.offset {
+		svc := s3.New(s.sess)
+
+		objectRange := "bytes=" + strconv.Itoa(s.offset)
+		s.offset += 1000000 // Reading in chunks of 1 mb
+		objectRange += "-" + strconv.Itoa(s.offset-1)
+
+		params := &s3.GetObjectInput{
+			Bucket: aws.String(s.bucket),
+			Key:    aws.String(s.key),
+			Range:  aws.String(objectRange),
+		}
+		resp, err := svc.GetObject(params)
+
+		if err != nil {
+			log.Println("[S3 Input] Error during getting file", s.bucket, s.key, err)
+		} else {
+			s.totalSize, _ = strconv.Atoi(strings.Split(*resp.ContentRange, "/")[1])
+			s.buf.ReadFrom(resp.Body)
+		}
+	}
+
+	s.readBytes += len(b)
+
+	return s.buf.Read(b)
+}
+
+// Close is here to make S3ReadCloser satisfy ReadCloser interface
+func (s *S3ReadCloser) Close() error {
+	return nil
+}

s3_test.go

@@ -0,0 +1,150 @@
+// +build pro
+
+package main
+
+import (
+	"fmt"
+	"math/rand"
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/s3"
+)
+
+func TestS3Output(t *testing.T) {
+	bucket := aws.String("test-gor")
+	rnd := rand.Int63()
+	path := fmt.Sprintf("s3://test-gor/%d/requests.gz", rnd)
+
+	output := NewS3Output(path, &FileOutputConfig{queueLimit: 2})
+
+	svc := s3.New(output.session)
+
+	output.Write([]byte("1 1 1\ntest"))
+	output.Write([]byte("1 1 1\ntest"))
+	output.buffer.updateName()
+	output.Write([]byte("1 1 1\ntest"))
+	output.Write([]byte("1 1 1\ntest"))
+	output.buffer.updateName()
+	output.Write([]byte("1 1 1\ntest"))
+
+	time.Sleep(time.Second)
+
+	params := &s3.ListObjectsInput{
+		Bucket: bucket,
+		Prefix: aws.String(fmt.Sprintf("%d", rnd)),
+	}
+
+	resp, _ := svc.ListObjects(params)
+	if len(resp.Contents) != 2 {
+		t.Error("Should create 2 objects", len(resp.Contents))
+	} else {
+		if *resp.Contents[0].Key != fmt.Sprintf("%d/requests_0.gz", rnd) ||
+			*resp.Contents[1].Key != fmt.Sprintf("%d/requests_1.gz", rnd) {
+			t.Error("Should assign proper names", resp.Contents)
+		}
+	}
+
+	for _, c := range resp.Contents {
+		svc.DeleteObject(&s3.DeleteObjectInput{Bucket: bucket, Key: c.Key})
+	}
+
+	matches, _ := filepath.Glob(fmt.Sprintf("/tmp/gor_output_s3_*"))
+	for _, m := range matches {
+		os.Remove(m)
+	}
+}
+
+func TestS3OutputQueueLimit(t *testing.T) {
+	bucket := aws.String("test-gor")
+	rnd := rand.Int63()
+	path := fmt.Sprintf("s3://test-gor/%d/requests.gz", rnd)
+
+	output := NewS3Output(path, &FileOutputConfig{queueLimit: 100})
+	output.closeCh = make(chan struct{}, 3)
+
+	svc := s3.New(output.session)
+
+	for i := 0; i < 3; i++ {
+		for i := 0; i < 100; i++ {
+			output.Write([]byte("1 1 1\ntest"))
+		}
+		output.buffer.updateName()
+	}
+	output.buffer.updateName()
+	output.Write([]byte("1 1 1\ntest"))
+
+	for i := 0; i < 3; i++ {
+		<-output.closeCh
+	}
+
+	params := &s3.ListObjectsInput{
+		Bucket: bucket,
+		Prefix: aws.String(fmt.Sprintf("%d", rnd)),
+	}
+
+	resp, _ := svc.ListObjects(params)
+	if len(resp.Contents) != 3 {
+		t.Error("Should create 3 object", len(resp.Contents))
+	} else {
+		if *resp.Contents[0].Key != fmt.Sprintf("%d/requests_0.gz", rnd) ||
+			*resp.Contents[1].Key != fmt.Sprintf("%d/requests_1.gz", rnd) {
+			t.Error("Should assign proper names", resp.Contents)
+		}
+	}
+
+	for _, c := range resp.Contents {
+		svc.DeleteObject(&s3.DeleteObjectInput{Bucket: bucket, Key: c.Key})
+	}
+
+	matches, _ := filepath.Glob(fmt.Sprintf("/tmp/gor_output_s3_*"))
+	for _, m := range matches {
+		os.Remove(m)
+	}
+}
+
+func TestInputFileFromS3(t *testing.T) {
+	rnd := rand.Int63()
+	path := fmt.Sprintf("s3://test-gor-eu/%d/requests.gz", rnd)
+
+	output := NewS3Output(path, &FileOutputConfig{queueLimit: 5000})
+	output.closeCh = make(chan struct{}, 10)
+
+	for i := 0; i <= 20000; i++ {
+		output.Write([]byte("1 1 1\ntest"))
+
+		if i%5000 == 0 {
+			output.buffer.updateName()
+		}
+	}
+
+	output.Write([]byte("1 1 1\ntest"))
+
+	for i := 0; i < 2; i++ {
+		<-output.closeCh
+	}
+
+	input := NewFileInput(fmt.Sprintf("s3://test-gor-eu/%d", rnd), false)
+
+	buf := make([]byte, 1000)
+	for i := 0; i <= 19999; i++ {
+		input.Read(buf)
+	}
+
+	// Cleanup artifacts
+	bucket := aws.String("test-gor")
+	svc := s3.New(output.session)
+	params := &s3.ListObjectsInput{
+		Bucket: bucket,
+		Prefix: aws.String(fmt.Sprintf("%d", rnd)),
+	}
+
+	resp, _ := svc.ListObjects(params)
+
+	for _, c := range resp.Contents {
+		svc.DeleteObject(&s3.DeleteObjectInput{Bucket: bucket, Key: c.Key})
+	}
+}

settings.go

@@ -8,7 +8,8 @@ import (
 	"time"
 )
 
-var VERSION string
+// DEMO indicates that goreplay is running in demo mode
+var DEMO string
 
 // MultiOption allows to specify multiple flags with same name and collects all values into array
 type MultiOption []string
@@ -25,158 +26,211 @@ func (h *MultiOption) Set(value string) error {
 
 // AppSettings is the struct of main configuration
 type AppSettings struct {
-	verbose   bool
-	debug     bool
-	stats     bool
-	exitAfter time.Duration
+	Verbose   int           `json:"verbose"`
+	Stats     bool          `json:"stats"`
+	ExitAfter time.Duration `json:"exit-after"`
 
-	splitOutput bool
+	SplitOutput          bool   `json:"split-output"`
+	RecognizeTCPSessions bool   `json:"recognize-tcp-sessions"`
+	Pprof                string `json:"http-pprof"`
 
-	inputDummy   MultiOption
-	outputDummy  MultiOption
-	outputStdout bool
-	outputNull   bool
+	InputDummy   MultiOption `json:"input-dummy"`
+	OutputDummy  MultiOption
+	OutputStdout bool `json:"output-stdout"`
+	OutputNull   bool `json:"output-null"`
 
-	inputTCP       MultiOption
-	outputTCP      MultiOption
-	outputTCPStats bool
+	InputTCP        MultiOption `json:"input-tcp"`
+	InputTCPConfig  TCPInputConfig
+	OutputTCP       MultiOption `json:"output-tcp"`
+	OutputTCPConfig TCPOutputConfig
+	OutputTCPStats  bool `json:"output-tcp-stats"`
 
-	inputFile        MultiOption
-	inputFileLoop    bool
-	outputFile       MultiOption
-	outputFileConfig FileOutputConfig
+	InputFile        MultiOption `json:"input-file"`
+	InputFileLoop    bool        `json:"input-file-loop"`
+	OutputFile       MultiOption `json:"output-file"`
+	OutputFileConfig FileOutputConfig
 
-	inputRAW              MultiOption
-	inputRAWEngine        string
-	inputRAWTrackResponse bool
-	inputRAWRealIPHeader  string
+	InputRAW MultiOption `json:"input_raw"`
+	RAWInputConfig
 
-	middleware string
+	Middleware string `json:"middleware"`
 
-	inputHTTP  MultiOption
-	outputHTTP MultiOption
+	InputHTTP    MultiOption
+	OutputHTTP   MultiOption `json:"output-http"`
+	PrettifyHTTP bool        `json:"prettify-http"`
 
-	outputHTTPConfig HTTPOutputConfig
-	modifierConfig   HTTPModifierConfig
+	OutputHTTPConfig HTTPOutputConfig
 
-	inputKafkaConfig  KafkaConfig
-	outputKafkaConfig KafkaConfig
+	OutputBinary       MultiOption `json:"output-binary"`
+	OutputBinaryConfig BinaryOutputConfig
+
+	ModifierConfig HTTPModifierConfig
+
+	InputKafkaConfig  InputKafkaConfig
+	OutputKafkaConfig OutputKafkaConfig
+	KafkaTLSConfig    KafkaTLSConfig
 }
 
 // Settings holds Gor configuration
 var Settings AppSettings
 
 func usage() {
-	fmt.Printf("Gor is a simple http traffic replication tool written in Go. Its main goal is to replay traffic from production servers to staging and dev environments.\nProject page: https://github.com/buger/gor\nAuthor: <Leonid Bugaev> leonsbox@gmail.com\nCurrent Version: %s\n\n", VERSION)
+	fmt.Printf("Gor is a simple http traffic replication tool written in Go. Its main goal is to replay traffic from production servers to staging and dev environments.\nProject page: https://github.com/buger/gor\nAuthor: <Leonid Bugaev> leonsbox@gmail.com\nCurrent Version: v%s\n\n", VERSION)
 	flag.PrintDefaults()
 	os.Exit(2)
 }
 
 func init() {
 	flag.Usage = usage
+	flag.StringVar(&Settings.Pprof, "http-pprof", "", "Enable profiling. Starts  http server on specified port, exposing special /debug/pprof endpoint. Example: `:8181`")
+	flag.IntVar(&Settings.Verbose, "verbose", 0, "set the level of verbosity, if greater than zero then it will turn on debug output")
+	flag.BoolVar(&Settings.Stats, "stats", false, "Turn on queue stats output")
 
-	flag.BoolVar(&Settings.verbose, "verbose", false, "Turn on more verbose output")
-	flag.BoolVar(&Settings.debug, "debug", false, "Turn on debug output, shows all intercepted traffic. Works only when with `verbose` flag")
-	flag.BoolVar(&Settings.stats, "stats", false, "Turn on queue stats output")
-	flag.DurationVar(&Settings.exitAfter, "exit-after", 0, "exit after specified duration")
+	if DEMO == "" {
+		flag.DurationVar(&Settings.ExitAfter, "exit-after", 0, "exit after specified duration")
+	} else {
+		Settings.ExitAfter = 5 * time.Minute
+	}
 
-	flag.BoolVar(&Settings.splitOutput, "split-output", false, "By default each output gets same traffic. If set to `true` it splits traffic equally among all outputs.")
+	flag.BoolVar(&Settings.SplitOutput, "split-output", false, "By default each output gets same traffic. If set to `true` it splits traffic equally among all outputs.")
+	flag.BoolVar(&Settings.RecognizeTCPSessions, "recognize-tcp-sessions", false, "[PRO] If turned on http output will create separate worker for each TCP session. Splitting output will session based as well.")
 
-	flag.Var(&Settings.inputDummy, "input-dummy", "Used for testing outputs. Emits 'Get /' request every 1s")
-	flag.Var(&Settings.outputDummy, "output-dummy", "DEPRECATED: use --output-stdout instead")
+	flag.Var(&Settings.InputDummy, "input-dummy", "Used for testing outputs. Emits 'Get /' request every 1s")
+	flag.BoolVar(&Settings.OutputStdout, "output-stdout", false, "Used for testing inputs. Just prints to console data coming from inputs.")
+	flag.BoolVar(&Settings.OutputNull, "output-null", false, "Used for testing inputs. Drops all requests.")
 
-	flag.BoolVar(&Settings.outputStdout, "output-stdout", false, "Used for testing inputs. Just prints to console data coming from inputs.")
+	flag.Var(&Settings.InputTCP, "input-tcp", "Used for internal communication between Gor instances. Example: \n\t# Receive requests from other Gor instances on 28020 port, and redirect output to staging\n\tgor --input-tcp :28020 --output-http staging.com")
+	flag.BoolVar(&Settings.InputTCPConfig.Secure, "input-tcp-secure", false, "Turn on TLS security. Do not forget to specify certificate and key files.")
+	flag.StringVar(&Settings.InputTCPConfig.CertificatePath, "input-tcp-certificate", "", "Path to PEM encoded certificate file. Used when TLS turned on.")
+	flag.StringVar(&Settings.InputTCPConfig.KeyPath, "input-tcp-certificate-key", "", "Path to PEM encoded certificate key file. Used when TLS turned on.")
 
-	flag.BoolVar(&Settings.outputNull, "output-null", false, "Used for testing inputs. Drops all requests.")
+	flag.Var(&Settings.OutputTCP, "output-tcp", "Used for internal communication between Gor instances. Example: \n\t# Listen for requests on 80 port and forward them to other Gor instance on 28020 port\n\tgor --input-raw :80 --output-tcp replay.local:28020")
+	flag.BoolVar(&Settings.OutputTCPConfig.Secure, "output-tcp-secure", false, "Use TLS secure connection. --input-file on another end should have TLS turned on as well.")
+	flag.BoolVar(&Settings.OutputTCPConfig.SkipVerify, "output-tcp-skip-verify", false, "Don't verify hostname on TLS secure connection.")
+	flag.BoolVar(&Settings.OutputTCPConfig.Sticky, "output-tcp-sticky", false, "Use Sticky connection. Request/Response with same ID will be sent to the same connection.")
+	flag.IntVar(&Settings.OutputTCPConfig.Workers, "output-tcp-workers", 10, "Number of parallel tcp connections, default is 10")
+	flag.BoolVar(&Settings.OutputTCPStats, "output-tcp-stats", false, "Report TCP output queue stats to console every 5 seconds.")
 
-	flag.Var(&Settings.inputTCP, "input-tcp", "Used for internal communication between Gor instances. Example: \n\t# Receive requests from other Gor instances on 28020 port, and redirect output to staging\n\tgor --input-tcp :28020 --output-http staging.com")
-	flag.Var(&Settings.outputTCP, "output-tcp", "Used for internal communication between Gor instances. Example: \n\t# Listen for requests on 80 port and forward them to other Gor instance on 28020 port\n\tgor --input-raw :80 --output-tcp replay.local:28020")
-	flag.BoolVar(&Settings.outputTCPStats, "output-tcp-stats", false, "Report TCP output queue stats to console every 5 seconds.")
+	flag.Var(&Settings.InputFile, "input-file", "Read requests from file: \n\tgor --input-file ./requests.gor --output-http staging.com")
+	flag.BoolVar(&Settings.InputFileLoop, "input-file-loop", false, "Loop input files, useful for performance testing.")
 
-	flag.Var(&Settings.inputFile, "input-file", "Read requests from file: \n\tgor --input-file ./requests.gor --output-http staging.com")
-	flag.BoolVar(&Settings.inputFileLoop, "input-file-loop", false, "Loop input files, useful for performance testing.")
+	flag.Var(&Settings.OutputFile, "output-file", "Write incoming requests to file: \n\tgor --input-raw :80 --output-file ./requests.gor")
+	flag.DurationVar(&Settings.OutputFileConfig.FlushInterval, "output-file-flush-interval", time.Second, "Interval for forcing buffer flush to the file, default: 1s.")
+	flag.BoolVar(&Settings.OutputFileConfig.Append, "output-file-append", false, "The flushed chunk is appended to existence file or not. ")
+	flag.Var(&Settings.OutputFileConfig.SizeLimit, "output-file-size-limit", "Size of each chunk. Default: 32mb")
+	flag.IntVar(&Settings.OutputFileConfig.QueueLimit, "output-file-queue-limit", 256, "The length of the chunk queue. Default: 256")
+	flag.Var(&Settings.OutputFileConfig.OutputFileMaxSize, "output-file-max-size-limit", "Max size of output file, Default: 1TB")
 
-	flag.Var(&Settings.outputFile, "output-file", "Write incoming requests to file: \n\tgor --input-raw :80 --output-file ./requests.gor")
-	flag.DurationVar(&Settings.outputFileConfig.flushInterval, "output-file-flush-interval", time.Second, "Interval for forcing buffer flush to the file, default: 1s.")
-	flag.BoolVar(&Settings.outputFileConfig.append, "output-file-append", false, "The flushed chunk is appended to existence file or not. ")
+	flag.StringVar(&Settings.OutputFileConfig.BufferPath, "output-file-buffer", "/tmp", "The path for temporary storing current buffer: \n\tgor --input-raw :80 --output-file s3://mybucket/logs/%Y-%m-%d.gz --output-file-buffer /mnt/logs")
 
-	// Set default
-	Settings.outputFileConfig.sizeLimit.Set("32mb")
-	flag.Var(&Settings.outputFileConfig.sizeLimit, "output-file-size-limit", "Size of each chunk. Default: 32mb")
-	flag.IntVar(&Settings.outputFileConfig.queueLimit, "output-file-queue-limit", 256, "The length of the chunk queue. Default: 256")
+	flag.BoolVar(&Settings.PrettifyHTTP, "prettify-http", false, "If enabled, will automatically decode requests and responses with: Content-Encoding: gzip and Transfer-Encoding: chunked. Useful for debugging, in conjunction with --output-stdout")
 
-	flag.Var(&Settings.inputRAW, "input-raw", "Capture traffic from given port (use RAW sockets and require *sudo* access):\n\t# Capture traffic from 8080 port\n\tgor --input-raw :8080 --output-http staging.com")
+	// input raw flags
+	flag.Var(&Settings.InputRAW, "input-raw", "Capture traffic from given port (use RAW sockets and require *sudo* access):\n\t# Capture traffic from 8080 port\n\tgor --input-raw :8080 --output-http staging.com")
+	flag.BoolVar(&Settings.TrackResponse, "input-raw-track-response", false, "If turned on Gor will track responses in addition to requests, and they will be available to middleware and file output.")
+	flag.Var(&Settings.Engine, "input-raw-engine", "Intercept traffic using `libpcap` (default), `raw_socket` or `pcap_file`")
+	flag.Var(&Settings.Protocol, "input-raw-protocol", "Specify application protocol of intercepted traffic. Possible values: http, binary")
+	flag.StringVar(&Settings.RealIPHeader, "input-raw-realip-header", "", "If not blank, injects header with given name and real IP value to the request payload. Usually this header should be named: X-Real-IP")
+	flag.DurationVar(&Settings.Expire, "input-raw-expire", time.Second*2, "How much it should wait for the last TCP packet, till consider that TCP message complete.")
+	flag.StringVar(&Settings.BPFFilter, "input-raw-bpf-filter", "", "BPF filter to write custom expressions. Can be useful in case of non standard network interfaces like tunneling or SPAN port. Example: --input-raw-bpf-filter 'dst port 80'")
+	flag.StringVar(&Settings.TimestampType, "input-raw-timestamp-type", "", "Possible values: PCAP_TSTAMP_HOST, PCAP_TSTAMP_HOST_LOWPREC, PCAP_TSTAMP_HOST_HIPREC, PCAP_TSTAMP_ADAPTER, PCAP_TSTAMP_ADAPTER_UNSYNCED. This values not supported on all systems, GoReplay will tell you available values of you put wrong one.")
+	flag.Var(&Settings.CopyBufferSize, "copy-buffer-size", "Set the buffer size for an individual request (default 5MB)")
+	flag.BoolVar(&Settings.Snaplen, "input-raw-override-snaplen", false, "Override the capture snaplen to be 64k. Required for some Virtualized environments")
+	flag.DurationVar(&Settings.BufferTimeout, "input-raw-buffer-timeout", 0, "set the pcap timeout. for immediate mode don't set this flag")
+	flag.Var(&Settings.BufferSize, "input-raw-buffer-size", "Controls size of the OS buffer which holds packets until they dispatched. Default value depends by system: in Linux around 2MB. If you see big package drop, increase this value.")
+	flag.BoolVar(&Settings.Promiscuous, "input-raw-promisc", false, "enable promiscuous mode")
+	flag.BoolVar(&Settings.Monitor, "input-raw-monitor", false, "enable RF monitor mode")
+	flag.BoolVar(&Settings.Stats, "input-raw-stats", false, "enable stats generator on raw TCP messages")
 
-	flag.BoolVar(&Settings.inputRAWTrackResponse, "input-raw-track-response", false, "If turned on Gor will track responses in addition to requests, and they will be available to middleware and file output.")
+	flag.StringVar(&Settings.Middleware, "middleware", "", "Used for modifying traffic using external command")
 
-	flag.StringVar(&Settings.inputRAWEngine, "input-raw-engine", "libpcap", "Intercept traffic using `libpcap` (default), and `raw_socket`")
+	flag.Var(&Settings.OutputHTTP, "output-http", "Forwards incoming requests to given http address.\n\t# Redirect all incoming requests to staging.com address \n\tgor --input-raw :80 --output-http http://staging.com")
 
-	flag.StringVar(&Settings.inputRAWRealIPHeader, "input-raw-realip-header", "", "If not blank, injects header with given name and real IP value to the request payload. Usually this header should be named: X-Real-IP")
+	/* outputHTTPConfig */
+	flag.Var(&Settings.OutputHTTPConfig.BufferSize, "output-http-response-buffer", "HTTP response buffer size, all data after this size will be discarded.")
+	flag.IntVar(&Settings.OutputHTTPConfig.WorkersMin, "output-http-workers-min", 0, "Gor uses dynamic worker scaling. Enter a number to set a minimum number of workers. default = 1.")
+	flag.IntVar(&Settings.OutputHTTPConfig.WorkersMax, "output-http-workers", 0, "Gor uses dynamic worker scaling. Enter a number to set a maximum number of workers. default = 0 = unlimited.")
+	flag.IntVar(&Settings.OutputHTTPConfig.QueueLen, "output-http-queue-len", 1000, "Number of requests that can be queued for output, if all workers are busy. default = 1000")
+	flag.BoolVar(&Settings.OutputHTTPConfig.SkipVerify, "output-http-skip-verify", false, "Don't verify hostname on TLS secure connection.")
+	flag.DurationVar(&Settings.OutputHTTPConfig.WorkerTimeout, "output-http-worker-timeout", 2*time.Second, "Duration to rollback idle workers.")
 
-	flag.StringVar(&Settings.middleware, "middleware", "", "Used for modifying traffic using external command")
+	flag.IntVar(&Settings.OutputHTTPConfig.RedirectLimit, "output-http-redirects", 0, "Enable how often redirects should be followed.")
+	flag.DurationVar(&Settings.OutputHTTPConfig.Timeout, "output-http-timeout", 5*time.Second, "Specify HTTP request/response timeout. By default 5s. Example: --output-http-timeout 30s")
+	flag.BoolVar(&Settings.OutputHTTPConfig.TrackResponses, "output-http-track-response", false, "If turned on, HTTP output responses will be set to all outputs like stdout, file and etc.")
 
-	// flag.Var(&Settings.inputHTTP, "input-http", "Read requests from HTTP, should be explicitly sent from your application:\n\t# Listen for http on 9000\n\tgor --input-http :9000 --output-http staging.com")
+	flag.BoolVar(&Settings.OutputHTTPConfig.Stats, "output-http-stats", false, "Report http output queue stats to console every N milliseconds. See output-http-stats-ms")
+	flag.IntVar(&Settings.OutputHTTPConfig.StatsMs, "output-http-stats-ms", 5000, "Report http output queue stats to console every N milliseconds. default: 5000")
+	flag.BoolVar(&Settings.OutputHTTPConfig.OriginalHost, "http-original-host", false, "Normally gor replaces the Host http header with the host supplied with --output-http.  This option disables that behavior, preserving the original Host header.")
+	flag.StringVar(&Settings.OutputHTTPConfig.ElasticSearch, "output-http-elasticsearch", "", "Send request and response stats to ElasticSearch:\n\tgor --input-raw :8080 --output-http staging.com --output-http-elasticsearch 'es_host:api_port/index_name'")
+	/* outputHTTPConfig */
 
-	flag.Var(&Settings.outputHTTP, "output-http", "Forwards incoming requests to given http address.\n\t# Redirect all incoming requests to staging.com address \n\tgor --input-raw :80 --output-http http://staging.com")
-	flag.IntVar(&Settings.outputHTTPConfig.BufferSize, "output-http-response-buffer", 0, "HTTP response buffer size, all data after this size will be discarded.")
-	flag.IntVar(&Settings.outputHTTPConfig.workers, "output-http-workers", 0, "Gor uses dynamic worker scaling by default.  Enter a number to run a set number of workers.")
-	flag.IntVar(&Settings.outputHTTPConfig.redirectLimit, "output-http-redirects", 0, "Enable how often redirects should be followed.")
-	flag.DurationVar(&Settings.outputHTTPConfig.Timeout, "output-http-timeout", 5*time.Second, "Specify HTTP request/response timeout. By default 5s. Example: --output-http-timeout 30s")
+	flag.Var(&Settings.OutputBinary, "output-binary", "Forwards incoming binary payloads to given address.\n\t# Redirect all incoming requests to staging.com address \n\tgor --input-raw :80 --input-raw-protocol binary --output-binary staging.com:80")
 
-	flag.BoolVar(&Settings.outputHTTPConfig.stats, "output-http-stats", false, "Report http output queue stats to console every 5 seconds.")
-	flag.BoolVar(&Settings.outputHTTPConfig.OriginalHost, "http-original-host", false, "Normally gor replaces the Host http header with the host supplied with --output-http.  This option disables that behavior, preserving the original Host header.")
-	flag.BoolVar(&Settings.outputHTTPConfig.Debug, "output-http-debug", false, "Enables http debug output.")
+	/* outputBinaryConfig */
+	flag.Var(&Settings.OutputBinaryConfig.BufferSize, "output-tcp-response-buffer", "TCP response buffer size, all data after this size will be discarded.")
+	flag.IntVar(&Settings.OutputBinaryConfig.Workers, "output-binary-workers", 0, "Gor uses dynamic worker scaling by default.  Enter a number to run a set number of workers.")
+	flag.DurationVar(&Settings.OutputBinaryConfig.Timeout, "output-binary-timeout", 0, "Specify HTTP request/response timeout. By default 5s. Example: --output-binary-timeout 30s")
+	flag.BoolVar(&Settings.OutputBinaryConfig.TrackResponses, "output-binary-track-response", false, "If turned on, Binary output responses will be set to all outputs like stdout, file and etc.")
 
-	flag.StringVar(&Settings.outputHTTPConfig.elasticSearch, "output-http-elasticsearch", "", "Send request and response stats to ElasticSearch:\n\tgor --input-raw :8080 --output-http staging.com --output-http-elasticsearch 'es_host:api_port/index_name'")
+	flag.BoolVar(&Settings.OutputBinaryConfig.Debug, "output-binary-debug", false, "Enables binary debug output.")
+	/* outputBinaryConfig */
 
-	flag.StringVar(&Settings.outputKafkaConfig.host, "output-kafka-host", "", "Read request and response stats from Kafka:\n\tgor --input-raw :8080 --output-kafka-host '192.168.0.1:9092,192.168.0.2:9092'")
-	flag.StringVar(&Settings.outputKafkaConfig.topic, "output-kafka-topic", "", "Read request and response stats from Kafka:\n\tgor --input-raw :8080 --output-kafka-topic 'kafka-log'")
-	flag.BoolVar(&Settings.outputKafkaConfig.useJSON, "output-kafka-json-format", false, "If turned on, it will serialize messages from GoReplay text format to JSON.")
+	flag.StringVar(&Settings.OutputKafkaConfig.Host, "output-kafka-host", "", "Read request and response stats from Kafka:\n\tgor --input-raw :8080 --output-kafka-host '192.168.0.1:9092,192.168.0.2:9092'")
+	flag.StringVar(&Settings.OutputKafkaConfig.Topic, "output-kafka-topic", "", "Read request and response stats from Kafka:\n\tgor --input-raw :8080 --output-kafka-topic 'kafka-log'")
+	flag.BoolVar(&Settings.OutputKafkaConfig.UseJSON, "output-kafka-json-format", false, "If turned on, it will serialize messages from GoReplay text format to JSON.")
 
-	flag.StringVar(&Settings.inputKafkaConfig.host, "input-kafka-host", "", "Send request and response stats to Kafka:\n\tgor --output-stdout --input-kafka-host '192.168.0.1:9092,192.168.0.2:9092'")
-	flag.StringVar(&Settings.inputKafkaConfig.topic, "input-kafka-topic", "", "Send request and response stats to Kafka:\n\tgor --output-stdout --input-kafka-topic 'kafka-log'")
-	flag.BoolVar(&Settings.inputKafkaConfig.useJSON, "input-kafka-json-format", false, "If turned on, it will assume that messages coming in JSON format rather than  GoReplay text format.")
+	flag.StringVar(&Settings.InputKafkaConfig.Host, "input-kafka-host", "", "Send request and response stats to Kafka:\n\tgor --output-stdout --input-kafka-host '192.168.0.1:9092,192.168.0.2:9092'")
+	flag.StringVar(&Settings.InputKafkaConfig.Topic, "input-kafka-topic", "", "Send request and response stats to Kafka:\n\tgor --output-stdout --input-kafka-topic 'kafka-log'")
+	flag.BoolVar(&Settings.InputKafkaConfig.UseJSON, "input-kafka-json-format", false, "If turned on, it will assume that messages coming in JSON format rather than  GoReplay text format.")
 
-	flag.Var(&Settings.modifierConfig.headers, "http-set-header", "Inject additional headers to http reqest:\n\tgor --input-raw :8080 --output-http staging.com --http-set-header 'User-Agent: Gor'")
-	flag.Var(&Settings.modifierConfig.headers, "output-http-header", "WARNING: `--output-http-header` DEPRECATED, use `--http-set-header` instead")
+	flag.StringVar(&Settings.KafkaTLSConfig.CACert, "kafka-tls-ca-cert", "", "CA certificate for Kafka TLS Config:\n\tgor  --input-raw :3000 --output-kafka-host '192.168.0.1:9092' --output-kafka-topic 'topic' --kafka-tls-ca-cert cacert.cer.pem --kafka-tls-client-cert client.cer.pem --kafka-tls-client-key client.key.pem")
+	flag.StringVar(&Settings.KafkaTLSConfig.ClientCert, "kafka-tls-client-cert", "", "Client certificate for Kafka TLS Config (mandatory with to kafka-tls-ca-cert and kafka-tls-client-key)")
+	flag.StringVar(&Settings.KafkaTLSConfig.ClientKey, "kafka-tls-client-key", "", "Client Key for Kafka TLS Config (mandatory with to kafka-tls-client-cert and kafka-tls-client-key)")
 
-	flag.Var(&Settings.modifierConfig.params, "http-set-param", "Set request url param, if param already exists it will be overwritten:\n\tgor --input-raw :8080 --output-http staging.com --http-set-param api_key=1")
+	flag.Var(&Settings.ModifierConfig.Headers, "http-set-header", "Inject additional headers to http reqest:\n\tgor --input-raw :8080 --output-http staging.com --http-set-header 'User-Agent: Gor'")
+	flag.Var(&Settings.ModifierConfig.HeaderRewrite, "http-rewrite-header", "Rewrite the request header based on a mapping:\n\tgor --input-raw :8080 --output-http staging.com --http-rewrite-header Host: (.*).example.com,$1.beta.example.com")
+	flag.Var(&Settings.ModifierConfig.Params, "http-set-param", "Set request url param, if param already exists it will be overwritten:\n\tgor --input-raw :8080 --output-http staging.com --http-set-param api_key=1")
+	flag.Var(&Settings.ModifierConfig.Methods, "http-allow-method", "Whitelist of HTTP methods to replay. Anything else will be dropped:\n\tgor --input-raw :8080 --output-http staging.com --http-allow-method GET --http-allow-method OPTIONS")
+	flag.Var(&Settings.ModifierConfig.URLRegexp, "http-allow-url", "A regexp to match requests against. Filter get matched against full url with domain. Anything else will be dropped:\n\t gor --input-raw :8080 --output-http staging.com --http-allow-url ^www.")
+	flag.Var(&Settings.ModifierConfig.URLNegativeRegexp, "http-disallow-url", "A regexp to match requests against. Filter get matched against full url with domain. Anything else will be forwarded:\n\t gor --input-raw :8080 --output-http staging.com --http-disallow-url ^www.")
+	flag.Var(&Settings.ModifierConfig.URLRewrite, "http-rewrite-url", "Rewrite the request url based on a mapping:\n\tgor --input-raw :8080 --output-http staging.com --http-rewrite-url /v1/user/([^\\/]+)/ping:/v2/user/$1/ping")
+	flag.Var(&Settings.ModifierConfig.HeaderFilters, "http-allow-header", "A regexp to match a specific header against. Requests with non-matching headers will be dropped:\n\t gor --input-raw :8080 --output-http staging.com --http-allow-header api-version:^v1")
+	flag.Var(&Settings.ModifierConfig.HeaderNegativeFilters, "http-disallow-header", "A regexp to match a specific header against. Requests with matching headers will be dropped:\n\t gor --input-raw :8080 --output-http staging.com --http-disallow-header \"User-Agent: Replayed by Gor\"")
+	flag.Var(&Settings.ModifierConfig.HeaderBasicAuthFilters, "http-basic-auth-filter", "A regexp to match the decoded basic auth string against. Requests with non-matching headers will be dropped:\n\t gor --input-raw :8080 --output-http staging.com --http-basic-auth-filter \"^customer[0-9].*\"")
+	flag.Var(&Settings.ModifierConfig.HeaderHashFilters, "http-header-limiter", "Takes a fraction of requests, consistently taking or rejecting a request based on the FNV32-1A hash of a specific header:\n\t gor --input-raw :8080 --output-http staging.com --http-header-limiter user-id:25%")
+	flag.Var(&Settings.ModifierConfig.ParamHashFilters, "http-param-limiter", "Takes a fraction of requests, consistently taking or rejecting a request based on the FNV32-1A hash of a specific GET param:\n\t gor --input-raw :8080 --output-http staging.com --http-param-limiter user_id:25%")
 
-	flag.Var(&Settings.modifierConfig.methods, "http-allow-method", "Whitelist of HTTP methods to replay. Anything else will be dropped:\n\tgor --input-raw :8080 --output-http staging.com --http-allow-method GET --http-allow-method OPTIONS")
-	flag.Var(&Settings.modifierConfig.methods, "output-http-method", "WARNING: `--output-http-method` DEPRECATED, use `--http-allow-method` instead")
+	// default values, using for tests
+	Settings.OutputFileConfig.SizeLimit = 33554432
+	Settings.OutputFileConfig.OutputFileMaxSize = 1099511627776
+	Settings.CopyBufferSize = 5242880
 
-	flag.Var(&Settings.modifierConfig.urlRegexp, "http-allow-url", "A regexp to match requests against. Filter get matched against full url with domain. Anything else will be dropped:\n\t gor --input-raw :8080 --output-http staging.com --http-allow-url ^www.")
-	flag.Var(&Settings.modifierConfig.urlRegexp, "output-http-url-regexp", "WARNING: `--output-http-url-regexp` DEPRECATED, use `--http-allow-url` instead")
-
-	flag.Var(&Settings.modifierConfig.urlNegativeRegexp, "http-disallow-url", "A regexp to match requests against. Filter get matched against full url with domain. Anything else will be forwarded:\n\t gor --input-raw :8080 --output-http staging.com --http-disallow-url ^www.")
-
-	flag.Var(&Settings.modifierConfig.urlRewrite, "http-rewrite-url", "Rewrite the request url based on a mapping:\n\tgor --input-raw :8080 --output-http staging.com --http-rewrite-url /v1/user/([^\\/]+)/ping:/v2/user/$1/ping")
-	flag.Var(&Settings.modifierConfig.urlRewrite, "output-http-rewrite-url", "WARNING: `--output-http-rewrite-url` DEPRECATED, use `--http-rewrite-url` instead")
-
-	flag.Var(&Settings.modifierConfig.headerFilters, "http-allow-header", "A regexp to match a specific header against. Requests with non-matching headers will be dropped:\n\t gor --input-raw :8080 --output-http staging.com --http-allow-header api-version:^v1")
-	flag.Var(&Settings.modifierConfig.headerFilters, "output-http-header-filter", "WARNING: `--output-http-header-filter` DEPRECATED, use `--http-allow-header` instead")
-
-	flag.Var(&Settings.modifierConfig.headerNegativeFilters, "http-disallow-header", "A regexp to match a specific header against. Requests with matching headers will be dropped:\n\t gor --input-raw :8080 --output-http staging.com --http-disallow-header \"User-Agent: Replayed by Gor\"")
-
-	flag.Var(&Settings.modifierConfig.headerHashFilters, "http-header-limiter", "Takes a fraction of requests, consistently taking or rejecting a request based on the FNV32-1A hash of a specific header:\n\t gor --input-raw :8080 --output-http staging.com --http-header-imiter user-id:25%")
-	flag.Var(&Settings.modifierConfig.headerHashFilters, "output-http-header-hash-filter", "WARNING: `output-http-header-hash-filter` DEPRECATED, use `--http-header-hash-limiter` instead")
-
-	flag.Var(&Settings.modifierConfig.paramHashFilters, "http-param-limiter", "Takes a fraction of requests, consistently taking or rejecting a request based on the FNV32-1A hash of a specific GET param:\n\t gor --input-raw :8080 --output-http staging.com --http-param-limiter user_id:25%")
 }
 
-var previousDebugTime int64
+func checkSettings() {
+	if Settings.OutputFileConfig.SizeLimit < 1 {
+		Settings.OutputFileConfig.SizeLimit.Set("32mb")
+	}
+	if Settings.OutputFileConfig.OutputFileMaxSize < 1 {
+		Settings.OutputFileConfig.OutputFileMaxSize.Set("1tb")
+	}
+	if Settings.CopyBufferSize < 1 {
+		Settings.CopyBufferSize.Set("5mb")
+	}
+}
+
+var previousDebugTime = time.Now()
 var debugMutex sync.Mutex
 
-// Debug gets called only if --verbose flag specified
-func Debug(args ...interface{}) {
-	if Settings.verbose {
+// Debug take an effect only if --verbose greater than 0 is specified
+func Debug(level int, args ...interface{}) {
+	if Settings.Verbose >= level {
 		debugMutex.Lock()
+		defer debugMutex.Unlock()
 		now := time.Now()
-		diff := float64(now.UnixNano()-previousDebugTime) / 1000000
-		previousDebugTime = now.UnixNano()
-		debugMutex.Unlock()
-
-		fmt.Printf("[DEBUG][PID %d][%d][%fms] ", os.Getpid(), now.UnixNano(), diff)
-		fmt.Println(args...)
+		diff := now.Sub(previousDebugTime)
+		previousDebugTime = now
+		fmt.Fprintf(os.Stderr, "[DEBUG][elapsed %s]: ", diff)
+		fmt.Fprintln(os.Stderr, args...)
 	}
 }

settings_test.go

@@ -0,0 +1,14 @@
+package main
+
+import (
+	"encoding/json"
+	"testing"
+)
+
+func TestAppSettings(t *testing.T) {
+	a := AppSettings{}
+	_, err := json.Marshal(&a)
+	if err != nil {
+		t.Error(err)
+	}
+}

size/size.go

@@ -0,0 +1,64 @@
+package size
+
+import (
+	"fmt"
+	"regexp"
+	"strconv"
+)
+
+// Size represents size that implements flag.Var
+type Size int64
+
+// the following regexes follow Go semantics https://golang.org/ref/spec#Letters_and_digits
+var (
+	rB  = regexp.MustCompile(`(?i)^(?:0b|0x|0o)?[\da-f_]+$`)
+	rKB = regexp.MustCompile(`(?i)^(?:0b|0x|0o)?[\da-f_]+kb$`)
+	rMB = regexp.MustCompile(`(?i)^(?:0b|0x|0o)?[\da-f_]+mb$`)
+	rGB = regexp.MustCompile(`(?i)^(?:0b|0x|0o)?[\da-f_]+gb$`)
+	rTB = regexp.MustCompile(`(?i)^(?:0b|0x|0o)?[\da-f_]+tb$`)
+)
+
+// Set parses size to integer from different bases and data units
+func (siz *Size) Set(size string) (err error) {
+	if size == "" {
+		return
+	}
+	const (
+		_ = 1 << (iota * 10)
+		KB
+		MB
+		GB
+		TB
+	)
+
+	var (
+		lmt = len(size) - 2
+		s   = []byte(size)
+	)
+
+	var _len int64
+	switch {
+	case rB.Match(s):
+		_len, err = strconv.ParseInt(size, 0, 64)
+	case rKB.Match(s):
+		_len, err = strconv.ParseInt(size[:lmt], 0, 64)
+		_len *= KB
+	case rMB.Match(s):
+		_len, err = strconv.ParseInt(size[:lmt], 0, 64)
+		_len *= MB
+	case rGB.Match(s):
+		_len, err = strconv.ParseInt(size[:lmt], 0, 64)
+		_len *= GB
+	case rTB.Match(s):
+		_len, err = strconv.ParseInt(size[:lmt], 0, 64)
+		_len *= TB
+	default:
+		return fmt.Errorf("invalid _len %q", size)
+	}
+	*siz = Size(_len)
+	return
+}
+
+func (siz *Size) String() string {
+	return fmt.Sprintf("%d", *siz)
+}

size/size_test.go

@@ -0,0 +1,29 @@
+package size
+
+import "testing"
+
+func TestParseDataUnit(t *testing.T) {
+	var d = map[string]int{
+		"42mb":                 42 << 20,
+		"4_2":                  42,
+		"00":                   0,
+		"0":                    0,
+		"0_600tb":              384 << 40,
+		"0600Tb":               384 << 40,
+		"0o12Mb":               10 << 20,
+		"0b_10010001111_1kb":   2335 << 10,
+		"1024":                 1 << 10,
+		"0b111":                7,
+		"0x12gB":               18 << 30,
+		"0x_67_7a_2f_cc_40_c6": 113774485586118,
+		"121562380192901":      121562380192901,
+	}
+	var buf Size
+	var err error
+	for k, v := range d {
+		err = buf.Set(k)
+		if err != nil || buf != Size(v) {
+			t.Errorf("Error parsing %s: %v", k, err)
+		}
+	}
+}