Merge pull request #162 from buger/input-modifier

Middleware for custom request rewrite logic

.dockerignore

@@ -0,0 +1 @@
+*.tar.gz

.gitignore

@@ -1,2 +1,10 @@
 *.swp
 *.gor
+
+*.out
+
+*.bin
+
+*.gz
+
+*.class

.travis.yml

@@ -1,3 +1,3 @@
-language: go 
-go: 1.3.3
-script: sudo -E bash -c "source /etc/profile && gvm use go1.3.3 && export GOPATH=$HOME/gopath:$GOPATH && go get && go test -v"
+language: go
+go: 1.4.2
+script: sudo -E bash -c "source /etc/profile && eval '$(gimme 1.4.2)' && export GOPATH=$HOME/gopath:$GOPATH && go get && GORACE='halt_on_error=1' go test ./... -v -timeout 60s -race"

CHANGELOG.md

@@ -1,25 +0,0 @@
-v0.7.0 - 31 Oct 2013
-* New modular architecture. Listener and Replay functionality merged.
-* Added option to equally split traffic between multiple outputs: --split-output true
-* Saving requests to file and replaying from it
-* Injecting custom headers to http requests
-* Advanced stats using ElasticSearch
-
-v0.3.5 - 15 Sep 2013
-* Significantly improved test coverage
-* Fixed bug with redirect replay https://github.com/buger/gor/pull/15
-* Added limit on listener side
-* Improved stability (catch and log panic, instead of exiting)
-* Added License file
-
-v0.3.3 - 22 Jun 2013
-* Using TCP instead of UDP for communication between Listener and Replay
-* Significantly improved performance
-* Fixed bugs causing locking and message dropping (concurrency issues)
-* Rewrote concurrency model to use more channels
-
-v0.3 - 10 Jun 2013
-* Use RAW_SOCKETS instead of tcpdump
-* Own TCP stack
-* All HTTP request types support
-* Simplified request parsing

Dockerfile

@@ -1,7 +1,24 @@
-FROM google/golang
+FROM google/golang:1.4
 
-WORKDIR /gopath/src/gor
+RUN cd /goroot/src/ && GOOS=linux GOARCH=386 ./make.bash --no-clean
 
-ADD . /gopath/src/gor
+RUN apt-get update && apt-get install ruby vim-common -y
 
+# Install Java for middleware testing
+RUN echo "deb http://ppa.launchpad.net/webupd8team/java/ubuntu trusty main" | tee /etc/apt/sources.list.d/webupd8team-java.list
+RUN echo "deb-src http://ppa.launchpad.net/webupd8team/java/ubuntu trusty main" | tee -a /etc/apt/sources.list.d/webupd8team-java.list
+RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys EEA14886
+RUN apt-get update -y
+RUN echo oracle-java7-installer shared/accepted-oracle-license-v1-1 select true | /usr/bin/debconf-set-selections
+RUN apt-get install oracle-java8-installer -y
+
+RUN wget http://apache-mirror.rbc.ru/pub/apache//commons/io/binaries/commons-io-2.4-bin.tar.gz -P /tmp
+RUN tar xzf /tmp/commons-io-2.4-bin.tar.gz -C /tmp
+
+WORKDIR /gopath/src/github.com/buger/gor/
+ADD . /gopath/src/github.com/buger/gor/
+
+RUN javac -cp /tmp/commons-io-2.4/commons-io-2.4.jar ./examples/middleware/echo.java
+
+RUN go get -u github.com/golang/lint/golint
 RUN go get

Makefile

@@ -1,25 +1,52 @@
-SOURCE = emitter.go gor.go gor_stat.go input_dummy.go input_file.go input_raw.go input_tcp.go limiter.go output_dummy.go output_file.go output_http.go output_tcp.go plugins.go settings.go settings_header_filters.go settings_header_hash_filters.go settings_headers.go settings_methods.go settings_option.go settings_url_regexp.go test_input.go elasticsearch.go settings_url_map.go
+SOURCE = emitter.go gor.go gor_stat.go input_dummy.go input_file.go input_raw.go input_tcp.go limiter.go output_dummy.go output_file.go input_http.go output_http.go output_tcp.go plugins.go settings.go test_input.go elasticsearch.go http_modifier.go http_modifier_settings.go http_client.go middleware.go protocol.go
 
-all: build-x86 build-x64
+SOURCE_PATH = /gopath/src/github.com/buger/gor/
 
-build-x64:
-	GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build
-	tar -czf gor_x64.tar.gz gor
-	rm gor
+release: release-x86 release-x64
 
-build-x86:
-	GOOS=linux GOARCH=386 CGO_ENABLED=0 go build
-	tar -czf gor_x86.tar.gz gor
-	rm gor
+release-x64:
+	docker run -v `pwd`:$(SOURCE_PATH) -t --env GOOS=linux --env GOARCH=amd64 --env CGO_ENABLED=0 -i gor go build -ldflags "-X main.VERSION $(VERSION)"&& tar -czf gor_$(VERSION)_x64.tar.gz gor && rm gor
+
+release-x86:
+	docker run -v `pwd`:$(SOURCE_PATH) -t --env GOOS=linux --env GOARCH=386 --env CGO_ENABLED=0 -i gor go build -ldflags "-X main.VERSION $(VERSION)" && tar -czf gor_$(VERSION)_x86.tar.gz gor && rm gor
 
 dbuild:
 	docker build -t gor .
 
+dlint:
+	docker run -v `pwd`:$(SOURCE_PATH) -t -i --env GORACE="halt_on_error=1" gor golint $(PKG)
+
+drace:
+	docker run -v `pwd`:$(SOURCE_PATH) -t -i --env GORACE="halt_on_error=1" gor go test ./... $(ARGS) -v -race -timeout 15s
+
 dtest:
-	docker run -v `pwd`:/gopath/src/gor -t -i gor go test -v
+	docker run -v `pwd`:$(SOURCE_PATH) -t -i gor go test ./... $(ARGS) -v -timeout 60s
 
+dcover:
+	docker run -v `pwd`:$(SOURCE_PATH) -t -i --env GORACE="halt_on_error=1" gor go test $(ARGS) -race -v -timeout 15s -coverprofile=coverage.out
+	go tool cover -html=coverage.out
+
+dfmt:
+	docker run -v `pwd`:$(SOURCE_PATH) -t -i gor go fmt ./...
+
+dvet:
+	docker run -v `pwd`:$(SOURCE_PATH) -t -i gor go vet
+
+dbench:
+	docker run -v `pwd`:$(SOURCE_PATH) -t -i gor go test -v -run NOT_EXISTING -bench HTTP
+
+# Used mainly for debugging, because docker container do not have access to parent machine ports
 drun:
-	docker run -v `pwd`:/gopath/src/gor -t -i gor go run $(SOURCE) --input-dummy=0 --output-dummy=0 --verbose
+	docker run -v `pwd`:$(SOURCE_PATH) -t -i gor go run $(SOURCE) --input-dummy=0 --output-http="http://localhost:9000" --input-raw :9000 --input-http :9000 --verbose --debug --middleware "./examples/middleware/echo.sh"
 
-dbash: 
-	docker run -v `pwd`:/gopath/src/gor -t -i gor /bin/bash
+drun-2:
+	docker run -v `pwd`:$(SOURCE_PATH) -t -i gor go run $(SOURCE) --input-file="./fixtures/requests.gor" --output-dummy=0 --verbose --debug --middleware "java -cp ./examples/middleware echo"
+
+drecord:
+	docker run -v `pwd`:$(SOURCE_PATH) -t -i gor go run $(SOURCE) --input-dummy=0 --output-file=requests.gor --verbose --debug
+
+dreplay:
+	docker run -v `pwd`:$(SOURCE_PATH) -t -i gor go run $(SOURCE) --input-file=requests.bin --output-tcp=:9000 --verbose -h
+
+dbash:
+	docker run -v `pwd`:$(SOURCE_PATH) -t -i gor /bin/bash

README.md

@@ -1,357 +1,488 @@
-[![Stories in Ready](https://badge.waffle.io/buger/gor.png?label=ready)](https://waffle.io/buger/gor)
-[![Build Status](https://travis-ci.org/buger/gor.png?branch=master)](https://travis-ci.org/buger/gor)
-
-## About
-
-Gor is a simple http traffic replication tool written in Go.
-Its main goal is to replay traffic from production servers to staging and dev environments.
-
-
-Now you can test your code on real user sessions in an automated and repeatable fashion.
-**No more falling down in production!**
-
-Here is basic workflow: The listener server catches http traffic and sends it to the replay server or saves to file.The replay server forwards traffic to a given address.
-
-
-![Diagram](http://i.imgur.com/9mqj2SK.png)
-
-
-## Examples
-
-### Capture traffic from port
-```bash
-# Run on servers where you want to catch traffic. You can run it on each `web` machine.
-sudo gor --input-raw :80 --output-tcp replay.local:28020
-
-# Replay server (replay.local).
-gor --input-tcp replay.local:28020 --output-http http://staging.com
-```
-
-### Using 1 Gor instance for both listening and replaying
-It's recommended to use separate server for replaying traffic, but if you have enough CPU resources you can use single Gor instance.
-
-```
-sudo gor --input-raw :80 --output-http "http://staging.com"
-```
-
-## Advanced use
-
-### Rate limiting
-Both replay and listener support rate limiting. It can be useful if you want
-forward only part of production traffic and not overload your staging
-environment. 
-Current throttling works like this: If for current second it reached specified requests limit - disregard the rest, on next second counter reseted.
-
-You can specify your desired requests per second using the
-"|" operator after the server address:
-
-#### Limiting replay
-```
-# staging.server will not get more than 10 requests per second
-gor --input-tcp :28020 --output-http "http://staging.com|10"
-```
-
-#### Limiting listener
-```
-# replay server will not get more than 10 requests per second
-# useful for high-load environments
-gor --input-raw :80 --output-tcp "replay.local:28020|10"
-```
-
-#### Match on regexp of url
-```
-# only forward requests being sent to the api... domains
-gor --input-raw :8080 --output-http staging.com --output-http-url-regexp ^www.
-```
-
-#### Filter based on regexp of header
-```
-# only forward requests with an api version of 1.0x
-gor --input-raw :8080 --output-http staging.com --output-http-header-filter api-version:^1\.0\d
-```
-
-#### Filter based on hash of header
-```
-# send 1/32 of all users consistently to staging
-gor --input-raw :8080 --output-http staging.com --output-http-header-hash-filter user-id:1/32
-```
-
-### Forward to multiple addresses
-
-You can forward traffic to multiple endpoints. Just add multiple --output-* arguments.
-```
-gor --input-tcp :28020 --output-http "http://staging.com"  --output-http "http://dev.com"
-```
-
-#### Splitting traffic
-By default it will send same traffic to all outputs, but you have options to equally split it:
-
-```
-gor --input-tcp :28020 --output-http "http://staging.com"  --output-http "http://dev.com" --split-output true
-```
-
-### Saving requests to file
-You can save requests to file, and replay them later:
-```
-# write to file
-gor --input-raw :80 --output-file requests.gor
-
-# read from file
-gor --input-file requests.gor --output-http "http://staging.com"
-```
-
-**Note:** Replay will preserve the original time differences between requests.
-
-### Injecting headers
-
-Additional headers can be injected/overwritten into requests during replay. This may be useful if you need to identify requests generated by Gor or enable feature flagged functionality in an application:
-
-```
-gor --input-raw :80 --output-http "http://staging.server" \
-    --output-http-header "User-Agent: Replayed by Gor" \
-    --output-http-header "Enable-Feature-X: true"
-```
-
-## Filtering HTTP methods
-
-Requests not matching a specified whitelist can be filtered out. For example to strip non-nullipotent requests:
-
-```
-gor --input-raw :80 --output-http "http://staging.server" \
-    --output-http-method GET \
-    --output-http-method OPTIONS
-```
-
-### Basic Auth
-
-If your development or staging environment is protected by Basic Authentication then those credentials can be injected in during the replay:
-
-```
-gor --input-raw :80 --output-http "http://user:pass@staging .com"
-```
-
-Note: This will overwrite any Authorization headers in the original request.
-
-#### Rewrite the target urls based on a mapping
-```
-# rewrite url to match the following
-gor --input-raw :8080 --output-http staging.com --output-http-rewrite-url /xml_test/interface.php:/api/service.do
-```
-
-
-## Stats 
-### ElasticSearch 
-For deep response analyze based on url, cookie, user-agent and etc. you can export response metadata to ElasticSearch. See [ELASTICSEARCH.md](ELASTICSEARCH.md) for more details.
-
-```
-gor --input-tcp :80 --output-http "http://staging.com" --output-http-elasticsearch "es_host:api_port/index_name"
-```
-
-## Additional help
-
-Feel free to ask question directly by email or by creating github issue.
-
-## Latest releases (including binaries)
-
-https://github.com/buger/gor/releases
-
-## Command line reference
-`gor -h` output:
-```
-  -cpuprofile="": write cpu profile to file
-  -memprofile="": write memory profile to this file
-  
-  -input-dummy=[]: Used for testing outputs. Emits 'Get /' request every 1s
-
-  -input-file=[]: Read requests from file: 
-    gor --input-file ./requests.gor --output-http staging.com
-
-  -input-raw=[]: Capture traffic from given port (use RAW sockets and require *sudo* access):
-    # Capture traffic from 8080 port
-    gor --input-raw :8080 --output-http staging.com
-
-  -input-tcp=[]: Used for internal communication between Gor instances. Example: 
-    # Receive requests from other Gor instances on 28020 port, and redirect output to staging
-    gor --input-tcp :28020 --output-http staging.com
-
-  -output-dummy=[]: Used for testing inputs. Just prints data coming from inputs.
-
-  -output-file=[]: Write incoming requests to file: 
-    gor --input-raw :80 --output-file ./requests.gor
-
-  -output-http=[]: Forwards incoming requests to given http address.
-    # Redirect all incoming requests to staging.com address 
-    gor --input-raw :80 --output-http http://staging.com
-
-  -output-http-elasticsearch="": Send request and response stats to ElasticSearch:
-    gor --input-raw :8080 --output-http staging.com --output-http-elasticsearch 'es_host:api_port/index_name'
-
-  -output-http-header=[]: Inject additional headers to http reqest:
-    gor --input-raw :8080 --output-http staging.com --output-http-header 'User-Agent: Gor'
-
-  -output-http-header-filter=[]: A regexp to match a specific header against. Requests with non-matching headers will be dropped:
-    gor --input-raw :8080 --output-http staging.com --output-http-header-filter api-version:^v1
-
-  -output-http-header-hash-filter=[]: Takes a fraction of requests, consistently taking or rejecting a request based on the FNV32-1A hash of a specific header. The fraction must have a denominator that is a power of two:
-    gor --input-raw :8080 --output-http staging.com --output-http-header-hash-filter user-id:1/4
-
-  -output-http-url-regexp=: A regexp to match requests against. Anything else will be dropped:
-    gor --input-raw :8080 --output-http staging.com --output-http-url-regexp ^www.
-    
-  -output-http-workers=-1: Number of http output workers desired. Use default -1 for dynamic worker scaling.  Gor will add http workers if its work queue starts getting too full and kill them .
-    
-  -output-http-stats=false: If set to `true` it gives out queuing stats for the HTTP output every 5 seconds in the form latest,mean,max,count,count/second.
-    
-  -output-tcp=[]: Used for internal communication between Gor instances. Example: 
-    # Listen for requests on 80 port and forward them to other Gor instance on 28020 port
-    gor --input-raw :80 --output-tcp replay.local:28020
-    
-  -output-tcp-stats=false: If set to `true` it gives out queuing stats for the TCP output every 5 seconds in the form latest,mean,max,count,count/second.
-    
-  -split-output=false: By default each output gets same traffic. If set to `true` it splits traffic equally among all outputs.
-
-  -output-http-rewrite-url=[]: Rewrites the url in the request based on a mapping
-    gor --input-raw :8080 --output-http staging.com --output-http-rewrite-url /xml_test/interface.php:/api/service.do
-```
-
-## Building from source
-
-1. Setup standard Go environment http://golang.org/doc/code.html and ensure that $GOPATH environment variable properly set.
-2. `go get github.com/buger/gor`.
-3. `cd $GOPATH/src/github.com/buger/gor`
-4. `go build` to get binary, or `go test` to run tests
-
-## Development
-Project contains Docker environment.
-
-1. Build container: `make dbuild`
-2. Run tests: `make dtest`
-3. Bash access to container: `make dbash`. Inside container you have python to run simple web server `python -m SimpleHTTPServer 8080` and `curl` to make http requests. 
-
-
-## Questions and support 
-
-All bug-reports and suggestions should go though Github Issues or our [Google Group](https://groups.google.com/forum/#!forum/gor-users). Or you can just send email to gor-users@googlegroups.com
-
-If you have some private questions you can send direct mail to leonsbox@gmail.com
-
-## FAQ
-
-### What OS are supported?
-For now only Linux based. *BSD (including MacOS is not supported yet, check https://github.com/buger/gor/issues/22 for details)
-
-### Why does the `--input-raw` requires sudo or root access?
-Listener works by sniffing traffic from a given port. It's accessible
-only by using sudo or root access.
-
-### I'm getting 'too many open files' error
-Typical linux shell has a small open files soft limit at 1024. You can easily raise that when you do this before starting your gor replay process:
-  
-  ulimit -n 64000
-
-More about ulimit: http://blog.thecodingmachine.com/content/solving-too-many-open-files-exception-red5-or-any-other-application
-
-### What do the stats commands do?
-Gor can report stats on the output-tcp and output-http request queues. Stats are reported to the console every 5 seconds in the form `latest,mean,max,count,count/second` by using the `-output-http-stats` and `-output-tcp-stats` options.
-
-Examples:
-
-```
-2014/04/23 21:17:50 output_tcp:latest,mean,max,count,count/second
-2014/04/23 21:17:50 output_tcp:0,0,0,0,0
-2014/04/23 21:17:55 output_tcp:1,1,2,68,13
-2014/04/23 21:18:00 output_tcp:1,1,2,92,18
-2014/04/23 21:18:05 output_tcp:1,1,2,119,23
-2014/04/23 21:18:10 output_tcp:1,0,1,95,19
-2014/04/23 21:18:15 output_tcp:1,1,2,92,18
-2014/04/23 21:18:20 output_tcp:1,1,2,108,21
-2014/04/23 21:18:25 output_tcp:1,1,2,117,23
-2014/04/23 21:18:30 output_tcp:1,1,2,113,22
-2014/04/23 21:18:35 output_tcp:21,20,21,132,26
-2014/04/23 21:18:40 output_tcp:100,99,100,99,19
-```
-
-```
-Version: 0.8
-2014/04/23 21:19:46 output_http:latest,mean,max,count,count/second
-2014/04/23 21:19:46 output_http:0,0,0,0,0
-2014/04/23 21:19:51 output_http:0,0,0,0,0
-2014/04/23 21:19:56 output_http:0,0,0,0,0
-2014/04/23 21:20:01 output_http:1,0,1,50,10
-2014/04/23 21:20:06 output_http:1,1,4,72,14
-2014/04/23 21:20:11 output_http:1,0,1,179,35
-2014/04/23 21:20:16 output_http:1,0,1,148,29
-2014/04/23 21:20:21 output_http:1,1,2,91,18
-2014/04/23 21:20:26 output_http:1,1,2,150,30
-2014/04/23 21:18:15 output_http:100,99,100,70,14
-2014/04/23 21:18:21 output_http:100,99,100,55,11
-2014/04/23 21:18:28 output_http:100,99,100,55,11
-2014/04/23 21:18:34 output_http:100,99,100,57,11
-2014/04/23 21:18:41 output_http:100,99,100,61,12
-2014/04/23 21:18:48 output_http:100,99,100,56,11
-2014/04/23 21:18:56 output_http:100,99,100,58,11
-2014/04/23 21:19:01 output_http:100,99,100,31,6
-2014/04/23 21:19:08 output_http:100,99,100,61,12
-2014/04/23 21:19:15 output_http:100,99,100,64,12
-2014/04/23 21:19:21 output_http:100,99,100,70,14
-2014/04/23 21:19:28 output_http:100,99,100,61,12
-2014/04/23 21:19:35 output_http:100,99,100,56,11
-```
-
-### How can I tell if I have bottlenecks?
-Key areas that sometimes experience bottlenecks are the output-tcp and output-http functions which have internal queues for requests. Each queue has an upper limit of 100. Enable stats reporting to see if any queues are experiencing bottleneck behavior.
- 
-#### output-http bottlenecks
-When running a Gor replay the output-http feature may bottleneck if:
-
-  * the replay has inadequate bandwidth. If the replay is receiving or sending more messages than its network adapter can handle the output-http-stats  may report that the output-http queue is filling up. See if there is a way to upgrade the replay's bandwidth.
-  * with `--output-http-workers` set to anything other than `-1` the `-output-http` target is unable to respond to messages in a timely manner. The http output workers which take messages off the output-http queue, process the request, and ensure that the request did not result in an error may not be able to keep up with the number of incoming requests. If the replay is not using dynamic worker scaling (`--output-http-workers=-1`)  The optimal number of output-http-workers can be determined with the formula `output-workers = (Average number of requests per second)/(Average target response time per second)`.
-
-#### output-tcp bottlenecks
-When using the Gor listener the output-tcp feature may bottleneck if:
-
-  * the replay is unable to accept and process more requests than the listener is able generate. Prior to troubleshooting the output-tcp bottleneck, ensure that the replay target is not experiencing any bottlenecks. 
-  * the replay target has inadequate bandwidth to handle all its incoming requests.  If a replay target's incoming bandwidth is maxed out the output-tcp-stats may report that the output-tcp queue is filling up. See if there is a way to upgrade the replay's bandwidth.
-
-### The CPU average across my load-balanced targets is higher than the source
-If you are replaying traffic from multiple listeners to a load-balanced target and you use sticky sessions, you may observe that the target servers have a higher CPU load than the listener servers. This may be because the sticky session cookie of the original load balancer is not honored by the target load balancer thus resulting in requests that would normally hit the same target server hitting different servers on the backend thus reducing some caching benefits gained via the load balancing.  Try running just one listener against one replay target and see if the CPU utilization comparison is more accurate.
- 
-### How does dynamic http worker scaling work?
-By using the Gor setting `--output-http-workers=-1` Gor will create more http output workers when the http output queue length is greater than 10.  The number of workers created (N) is equal to the queue length at the time which it is checked and found to have a length greater than 10. The queue length is checked every time a message is written to the http output queue.  No more workers will be spawned until that request to spawn N workers is satisfied.  If a dynamic worker cannot process a message at that time, it will sleep for 100 milliseconds. If a dynamic worker cannot process a message for 2 seconds it dies.  
-
-## Tuning
-
-To achieve the top most performance you should tune the source server system limits:
-
-    net.ipv4.tcp_max_tw_buckets = 65536
-    net.ipv4.tcp_tw_recycle = 1
-    net.ipv4.tcp_tw_reuse = 0
-    net.ipv4.tcp_max_syn_backlog = 131072
-    net.ipv4.tcp_syn_retries = 3
-    net.ipv4.tcp_synack_retries = 3
-    net.ipv4.tcp_retries1 = 3
-    net.ipv4.tcp_retries2 = 8
-    net.ipv4.tcp_rmem = 16384 174760 349520
-    net.ipv4.tcp_wmem = 16384 131072 262144
-    net.ipv4.tcp_mem = 262144 524288 1048576
-    net.ipv4.tcp_max_orphans = 65536
-    net.ipv4.tcp_fin_timeout = 10
-    net.ipv4.tcp_low_latency = 1
-    net.ipv4.tcp_syncookies = 0
-
-
-## Contributing
-
-1. Fork it
-2. Create your feature branch (git checkout -b my-new-feature)
-3. Commit your changes (git commit -am 'Added some feature')
-4. Push to the branch (git push origin my-new-feature)
-5. Create new Pull Request
-
-## Companies using Gor
-
-* [Granify](http://granify.com)
-* [GOV.UK](https://www.gov.uk) ([Government Digital Service](http://digital.cabinetoffice.gov.uk/))
-* [theguardian.com](http://theguardian.com)
-* [TomTom](http://www.tomtom.com/)
-* To add your company drop me a line to github.com/buger or leonsbox@gmail.com
+[![Build Status](https://travis-ci.org/buger/gor.png?branch=master)](https://travis-ci.org/buger/gor)
+
+## About
+
+Gor is a simple http traffic replication tool written in Go.
+Its main goal is to replay traffic from production servers to staging and dev environments.
+
+
+Now you can test your code on real user sessions in an automated and repeatable fashion.
+**No more falling down in production!**
+
+Here is basic workflow: The listener server catches http traffic and sends it to the replay server or saves to file. The replay server forwards traffic to a given address.
+
+
+![Diagram](http://i.imgur.com/9mqj2SK.png)
+
+
+## Examples
+
+### Capture traffic from port
+```bash
+# Run on servers where you want to catch traffic. You can run it on each `web` machine.
+sudo gor --input-raw :80 --output-tcp replay.local:28020
+
+# Replay server (replay.local).
+gor --input-tcp replay.local:28020 --output-http http://staging.com
+```
+
+Since Gor use raw sockets to capture traffic it require `sudo` access. Alternatively you can allow access to raw sockets like this: `sudo setcap CAP_NET_RAW=ep gor`
+
+### Using 1 Gor instance for both listening and replaying
+It's recommended to use separate server for replaying traffic, but if you have enough CPU resources you can use single Gor instance.
+
+```
+sudo gor --input-raw :80 --output-http "http://staging.com"
+```
+
+### Guarantee of replay and HTTP input
+Due to how traffic interception works, there is chance of missing requests. If you want guarantee that requests will be replayed you can use http input, but it will require changes in your app as well. 
+
+```
+sudo gor --input-http :28019 --output-http "http://staging.com"
+```
+
+Then in your application you should send copy (e.g. like reverse proxy) all incoming requests to Gor http input. 
+
+## Configuration
+
+### Forward to multiple addresses
+
+You can forward traffic to multiple endpoints. Just add multiple --output-* arguments.
+```
+gor --input-tcp :28020 --output-http "http://staging.com"  --output-http "http://dev.com"
+```
+
+#### Splitting traffic
+By default it will send same traffic to all outputs, but you have options to equally split it:
+
+```
+gor --input-tcp :28020 --output-http "http://staging.com"  --output-http "http://dev.com" --split-output true
+```
+
+### HTTP output workers
+By default Gor creates dynamic pull of workers: it starts with 10 and create more http output workers when the http output queue length is greater than 10.  The number of workers created (N) is equal to the queue length at the time which it is checked and found to have a length greater than 10. The queue length is checked every time a message is written to the http output queue.  No more workers will be spawned until that request to spawn N workers is satisfied.  If a dynamic worker cannot process a message at that time, it will sleep for 100 milliseconds. If a dynamic worker cannot process a message for 2 seconds it dies.
+You may specify fixed number of workers using  `--output-http-workers=20` option.
+
+### Follow redirects
+By default Gor will ignore all redirects since they are handled by clients using your app, but in scenarios when your replayed environment introduce new redirects, you can enable them like this: 
+```
+gor --input-tcp replay.local:28020 --output-http http://staging.com --output-http-redirects 2
+```
+The given example will follow up to 2 redirects per request.
+
+### HTTP timeouts
+By default http timeout for both request and response is 5 seconds. You can override it like this:
+```
+gor --input-tcp replay.local:28020 --output-http http://staging.com --output-http-timeout 30s
+```
+
+### Rate limiting
+Rate limiting can be useful if you want forward only part of production traffic and not overload your staging environment. There is 2 strategies: dropping random requests or dropping fraction of requests based on Header or URL param value. 
+
+#### Dropping random requests
+Every input and output support random rate limiting.
+There are 2 limiting algorithms: absolute or percentage based. 
+
+Absolute: If for current second it reached specified requests limit - disregard the rest, on next second counter reseted.
+
+Percentage: For input-file it will slowdown or speedup request execution, for the rest it will use random generator to decide if request pass or not based on chance you specified. 
+
+You can specify your desired limit using the
+"|" operator after the server address:
+
+#### Limiting replay using absolute number
+```
+# staging.server will not get more than 10 requests per second
+gor --input-tcp :28020 --output-http "http://staging.com|10"
+```
+
+#### Limiting listener using percentage based limiter
+```
+# replay server will not get more than 10% of requests 
+# useful for high-load environments
+gor --input-raw :80 --output-tcp "replay.local:28020|10%"
+```
+
+#### Limiting based on Header or URL param value
+If you have unique user id (like API key) stored in header or URL you can consistently forward specified percent of traffic only for fraction of this users. 
+Basic formula looks like this: `FNV32-1A_hashing(value) % 100 >= chance`. Examples:
+```
+# Limit based on header value
+gor --input-raw :80 --output-tcp "replay.local:28020|10%" --http-header-limiter "X-API-KEY: 10%"
+
+# Limit based on header value
+gor --input-raw :80 --output-tcp "replay.local:28020|10%" --http-param-limiter "api_key: 10%"
+```
+Only percentage based limiting supported.
+
+### Filtering 
+
+#### Allow url regexp
+```
+# only forward requests being sent to the /api endpoint
+gor --input-raw :8080 --output-http staging.com --http-allow-url /api
+```
+
+#### Disallow url regexp
+```
+# only forward requests NOT being sent to the /api... endpoint
+gor --input-raw :8080 --output-http staging.com --http-disallow-url /api
+```
+#### Filter based on regexp of header
+
+```
+# only forward requests with an api version of 1.0x
+gor --input-raw :8080 --output-http staging.com --http-allow-header api-version:^1\.0\d
+
+# only forward requests NOT containing User-Agent header value "Replayed by Gor"
+gor --input-raw :8080 --output-http staging.com --http-disallow-header "User-Agent: Replayed by Gor"
+```
+
+#### Filter based on http method
+Requests not matching a specified whitelist can be filtered out. For example to strip non-nullipotent requests:
+
+```
+gor --input-raw :80 --output-http "http://staging.server" \
+    --http-allow-method GET \
+    --http-allow-method OPTIONS
+```
+
+### Rewriting original request
+Gor supports some basic request rewriting support. For complex logic you can use middleware, see below.
+
+#### Rewrite URL based on a mapping
+```
+# rewrite url to match the following
+gor --input-raw :8080 --output-http staging.com --http-rewrite-url /v1/user/([^\\/]+)/ping:/v2/user/$1/ping
+```
+
+#### Set URL param
+Set request url param, if param already exists it will be overwritten
+```
+gor --input-raw :8080 --output-http staging.com --http-set-param api_key=1
+```
+
+#### Set Header
+Set request header, if header already exists it will be overwritten. This may be useful if you need to identify requests generated by Gor or enable feature flagged functionality in an application:
+
+```
+gor --input-raw :80 --output-http "http://staging.server" \
+    --http-header "User-Agent: Replayed by Gor" \
+    --http-header "Enable-Feature-X: true"
+```
+
+#### Host header
+Host header gets special treatment. By default Host get set to the value specified in --output-http. If you manually set --http-header "Host: anonther.com", Gor will not override Host value.
+
+If you app accepts traffic from multiple domain, and you want to keep original headers, there is specific `--http-original-host` with tells Gor do not touch Host header at all.
+
+### Middleware
+Middleware is a program that accepts request and response payload at STDIN and emits modified requests at STDOUT. You can implement any custom logic like stripping private data, advanced rewriting, support for oAuth and etc.
+
+```
+                   Original request      +--------------+
++-------------+----------STDIN---------->+              |
+|  Gor input  |                          |  Middleware  |
++-------------+----------STDIN---------->+              |
+                   Original response     +------+---+---+
+                                                |   ^
++-------------+    Modified request             v   |
+| Gor output  +<---------STDOUT-----------------+   |
++-----+-------+                                     |
+      |                                             |
+      |            Replayed response                |
+      +------------------STDIN----------------->----+
+```
+
+Middleware can be written in any language, see `examples/middleware` folder for examples.
+Middleware program should accept the fact that all communication with Gor is asynchronous, there is no guarantee that original request and response messages will come one after each other. Your app should take care of the state if logic depends on original or replayed response, see `examples/middleware/token_modifier.go` as example.
+
+Simple bash echo middleware (returns same request) will look like this:
+```bash
+while read line; do
+  echo $line
+end
+```
+
+Middleware can be enabled using `--middleware` option, by specifying path to executable file:
+```
+gor --input-raw :80 --middleware "/opt/middleware_executable" --output-http "http://staging.server"
+```
+
+#### Communication protocol
+All messages should be hex encoded, new line character specifieds the end of the message, eg. new message per line.
+
+Decoded payload consist of 2 parts: header and HTTP payload, separated by new line character.  
+
+Example request payload:
+
+```
+1 932079936fa4306fc308d67588178d17d823647c 1439818823587396305
+GET /a HTTP/1.1
+Host: 127.0.0.1
+
+```
+
+Example response payload:
+
+```
+2 8e091765ae902fef8a2b7d9dd960e9d52222bd8c 2782013
+HTTP/1.1 200 OK
+Date: Mon, 17 Aug 2015 13:40:23 GMT
+Content-Length: 0
+Content-Type: text/plain; charset=utf-8
+
+```
+
+Header contains request meta information separated by spaces. First value is payload type, possible values: `1` - request, `2` - original response, `3` - replayed response.
+Next goes request id: unique among all requests (sha1 of time and Ack), but remain same for original and replayed response, so you can create associations between request and responses. Third argument varies depending on payload type: for request - start time, for responses - round-trip time.
+
+HTTP payload is unmodified HTTP requests/responses intercepted from network. You can read more about request format [here](http://www.jmarshall.com/easy/http/), [here](https://en.wikipedia.org/wiki/Hypertext_Transfer_Protocol) and [here](http://www.w3.org/Protocols/rfc2616/rfc2616.html). You can operate with payload as you want, add headers, change path, and etc. Basically you just editing a string, just ensure that it is RCF compliant.
+
+At the end modified (or untouched) request should be emitted back to STDOUT, keeping original header, and hex-encoded. If you want to filter request, just not send it. Emitting responses back is required, even if you did not touch them.
+
+#### Advanced example
+Imagine that you have auth system that randomly generate access tokens, which used later for accessing secure content. Since there is no pre-defined token value, naive approach without middleware (or if middleware use only request payloads) will fail, because replayed server have own tokens, not synced with origin. To fix this, our middleware should take in account responses of replayed and origin server, store `originalToken -> replayedToken` aliases and rewrite all requests using this token to use replayed alias. See `examples/middleware/token_modifier.go` and `middleware_test.go#TestTokenMiddleware` as example of described scheme.
+
+### Saving requests to file and replaying them
+You can save requests to file, and replay them later:
+```
+# write to file
+gor --input-raw :80 --output-file requests.gor
+
+# read from file
+gor --input-file requests.gor --output-http "http://staging.com"
+```
+
+**Note:** Replay will preserve the original time differences between requests.
+
+### Load testing
+
+Currently it supported only by `input-file` and only when using percentage based limiter. Unlike default limiter for `input-file` instead of dropping requests it will slowdown or speedup request emitting. Note that unlike examples above limiter is applied to input:
+
+```
+# Replay from file on 2x speed 
+gor --input-file "requests.gor|200%" --output-http "staging.com"
+```
+
+### Basic Auth
+
+If your development or staging environment is protected by Basic Authentication then those credentials can be injected in during the replay:
+
+```
+gor --input-raw :80 --output-http "http://user:pass@staging .com"
+```
+
+Note: This will overwrite any Authorization headers in the original request.
+
+
+## Stats 
+
+Gor can report stats on the `output-tcp` and `output-http` request queues. Stats are reported to the console every 5 seconds in the form `latest,mean,max,count,count/second` by using the `--output-http-stats` and `--output-tcp-stats` options.
+
+Examples:
+
+```
+2014/04/23 21:17:50 output_tcp:latest,mean,max,count,count/second
+2014/04/23 21:17:50 output_tcp:0,0,0,0,0
+2014/04/23 21:17:55 output_tcp:1,1,2,68,13
+2014/04/23 21:18:00 output_tcp:1,1,2,92,18
+2014/04/23 21:18:05 output_tcp:1,1,2,119,23
+```
+
+```
+Version: 0.8
+2014/04/23 21:19:46 output_http:latest,mean,max,count,count/second
+2014/04/23 21:19:46 output_http:0,0,0,0,0
+2014/04/23 21:19:51 output_http:0,0,0,0,0
+2014/04/23 21:19:56 output_http:0,0,0,0,0
+2014/04/23 21:20:01 output_http:1,0,1,50,10
+2014/04/23 21:20:06 output_http:1,1,4,72,14
+2014/04/23 21:20:11 output_http:1,0,1,179,35
+2014/04/23 21:20:16 output_http:1,0,1,148,29
+2014/04/23 21:20:21 output_http:1,1,2,91,18
+2014/04/23 21:20:26 output_http:1,1,2,150,30
+2014/04/23 21:18:15 output_http:100,99,100,70,14
+2014/04/23 21:18:21 output_http:100,99,100,55,11
+```
+
+### How can I tell if I have bottlenecks?
+Key areas that sometimes experience bottlenecks are the output-tcp and output-http functions which have internal queues for requests. Each queue has an upper limit of 100. Enable stats reporting to see if any queues are experiencing bottleneck behavior.
+ 
+#### output-http bottlenecks
+When running a Gor replay the output-http feature may bottleneck if:
+
+  * the replay has inadequate bandwidth. If the replay is receiving or sending more messages than its network adapter can handle the output-http-stats  may report that the output-http queue is filling up. See if there is a way to upgrade the replay's bandwidth.
+  * with `--output-http-workers` set to anything other than `-1` the `-output-http` target is unable to respond to messages in a timely manner. The http output workers which take messages off the output-http queue, process the request, and ensure that the request did not result in an error may not be able to keep up with the number of incoming requests. If the replay is not using dynamic worker scaling (`--output-http-workers=-1`)  The optimal number of output-http-workers can be determined with the formula `output-workers = (Average number of requests per second)/(Average target response time per second)`.
+
+#### output-tcp bottlenecks
+When using the Gor listener the output-tcp feature may bottleneck if:
+
+  * the replay is unable to accept and process more requests than the listener is able generate. Prior to troubleshooting the output-tcp bottleneck, ensure that the replay target is not experiencing any bottlenecks. 
+  * the replay target has inadequate bandwidth to handle all its incoming requests.  If a replay target's incoming bandwidth is maxed out the output-tcp-stats may report that the output-tcp queue is filling up. See if there is a way to upgrade the replay's bandwidth.
+
+### ElasticSearch 
+For deep response analyze based on url, cookie, user-agent and etc. you can export response metadata to ElasticSearch. See [ELASTICSEARCH.md](ELASTICSEARCH.md) for more details.
+
+```
+gor --input-tcp :80 --output-http "http://staging.com" --output-http-elasticsearch "es_host:api_port/index_name"
+```
+
+## Additional help
+
+Feel free to ask question directly by email or by creating github issue.
+
+## Latest releases (including binaries)
+
+https://github.com/buger/gor/releases
+
+## Command line reference
+`gor -h` output:
+```
+  -http-allow-header=[]: A regexp to match a specific header against. Requests with non-matching headers will be dropped:
+   gor --input-raw :8080 --output-http staging.com --http-allow-header api-version:^v1
+  -http-disallow-header=[]: A regexp to match a specific header against. Requests with matching headers will be dropped:
+   gor --input-raw :8080 --output-http staging.com --http-disallow-header "User-Agent: Replayed by Gor"
+  -http-allow-method=[]: Whitelist of HTTP methods to replay. Anything else will be dropped:
+  gor --input-raw :8080 --output-http staging.com --http-allow-method GET --http-allow-method OPTIONS
+  -http-allow-url=[]: A regexp to match requests against. Filter get matched agains full url with domain. Anything else will be dropped:
+   gor --input-raw :8080 --output-http staging.com --http-allow-url ^www.
+  -http-diallow-url=[]: A regexp to match requests against. Filter get matched agains full url with domain. Anything else will be dropped:
+   gor --input-raw :8080 --output-http staging.com --http-disallow-url ^www.
+  -http-header-limiter=[]: Takes a fraction of requests, consistently taking or rejecting a request based on the FNV32-1A hash of a specific header:
+   gor --input-raw :8080 --output-http staging.com --http-header-imiter user-id:25%
+  -http-param-limiter=[]: Takes a fraction of requests, consistently taking or rejecting a request based on the FNV32-1A hash of a specific GET param:
+   gor --input-raw :8080 --output-http staging.com --http-param-limiter user_id:25%
+  -http-rewrite-url=[]: Rewrite the request url based on a mapping:
+  gor --input-raw :8080 --output-http staging.com --http-rewrite-url /v1/user/([^\/]+)/ping:/v2/user/$1/ping
+  -http-set-header=[]: Inject additional headers to http reqest:
+  gor --input-raw :8080 --output-http staging.com --http-set-header 'User-Agent: Gor'
+  -http-set-param=[]: Set request url param, if param already exists it will be overwritten:
+  gor --input-raw :8080 --output-http staging.com --http-set-param api_key=1
+  -input-dummy=[]: Used for testing outputs. Emits 'Get /' request every 1s
+  -input-file=[]: Read requests from file:
+  gor --input-file ./requests.gor --output-http staging.com
+  -input-http=[]: Read requests from HTTP, should be explicitly sent from your application:
+  # Listen for http on 9000
+  gor --input-http :9000 --output-http staging.com
+  -input-raw=[]: Capture traffic from given port (use RAW sockets and require *sudo* access):
+  # Capture traffic from 8080 port
+  gor --input-raw :8080 --output-http staging.com
+  -input-tcp=[]: Used for internal communication between Gor instances. Example:
+  # Receive requests from other Gor instances on 28020 port, and redirect output to staging
+  gor --input-tcp :28020 --output-http staging.com
+  -memprofile="": write memory profile to this file
+  -output-dummy=[]: Used for testing inputs. Just prints data coming from inputs.
+  -output-file=[]: Write incoming requests to file:
+  gor --input-raw :80 --output-file ./requests.gor
+  -output-http=[]: Forwards incoming requests to given http address.
+  # Redirect all incoming requests to staging.com address
+  gor --input-raw :80 --output-http http://staging.com
+  -output-http-elasticsearch="": Send request and response stats to ElasticSearch:
+  gor --input-raw :8080 --output-http staging.com --output-http-elasticsearch 'es_host:api_port/index_name'
+  -output-http-header-filter=[]: WARNING: `--output-http-header-filter` DEPRECATED, use `--http-allow-header` instead  -output-http-redirects=0: Enable how often redirects should be followed.
+  -output-http-stats=false: Report http output queue stats to console every 5 seconds. Remember to include also `--stats`
+  -output-http-workers=0: Gor uses dynamic worker scaling by default.  Enter a number to run a set number of workers.
+  -output-tcp=[]: Used for internal communication between Gor instances. Example:
+  # Listen for requests on 80 port and forward them to other Gor instance on 28020 port
+  gor --input-raw :80 --output-tcp replay.local:28020
+  -output-tcp-stats=false: Report TCP output queue stats to console every 5 seconds. Remember to include also `--stats`
+  -split-output=false: By default each output gets same traffic. If set to `true` it splits traffic equally among all outputs.
+  -stats=false: Turn on queue stats output. Use in combination with the other *-stats flags.
+  -verbose=false: Turn on verbose/debug output
+```
+
+## Building from source
+
+1. Setup standard Go environment http://golang.org/doc/code.html and ensure that $GOPATH environment variable properly set.
+2. `go get github.com/buger/gor`.
+3. `cd $GOPATH/src/github.com/buger/gor`
+4. `go build` to get binary, or `go test` to run tests
+
+## Development
+Project contains Docker environment.
+
+1. Build container: `make dbuild`
+2. Run all tests: `make dtest`. Run specific test: `make dtest ARGS=-test.run=**regexp**`
+3. Bash access to container: `make dbash`. Inside container you have python to run simple web server `python -m SimpleHTTPServer 8080` and `curl` to make http requests. 
+
+
+## Questions and support 
+
+All bug-reports and suggestions should go though Github Issues or our [Google Group](https://groups.google.com/forum/#!forum/gor-users). Or you can just send email to gor-users@googlegroups.com
+
+If you have some private questions you can send direct mail to leonsbox@gmail.com
+
+## FAQ
+
+### What OS are supported?
+For now only Linux based. *BSD (including MacOS is not supported yet, check https://github.com/buger/gor/issues/22 for details)
+
+### Why does the `--input-raw` requires sudo or root access?
+Listener works by sniffing traffic from a given port. It's accessible
+only by using sudo or root access.
+
+### How do you deal with user session to replay the traffic correctly?
+You can rewrite session related headers/params to match your staging environment. If you require custom logic (e.g random token based auth) follow this discussion: https://github.com/buger/gor/issues/154
+
+### Can i use Gor to intercept SSL traffic?
+Basic idea is that SSL was made to protect itself from traffic interception. There 2 options: 
+1. Move SSL handling to proxy like Nginx or Amazon ELB. And allow Gor to listen on upstreams. 
+2. Use `--input-http` so you can duplicate request payload directly from your app to Gor, but it will require your app modifications.
+
+More can be find here: https://github.com/buger/gor/issues/85
+
+### Is there a limit for size of HTTP request when using output-http?
+Due to the fact that Gor can't guarantee interception of all packets, for large payloads > 200kb there is chance of missing some packets and corrupting body. Treat it as a feature and chance to test broken bodies handling :)
+The only way to guarantee delivery is using `--input-http`, but you will miss some features.
+
+### I'm getting 'too many open files' error
+Typical linux shell has a small open files soft limit at 1024. You can easily raise that when you do this before starting your gor replay process:
+  
+  ulimit -n 64000
+
+More about ulimit: http://blog.thecodingmachine.com/content/solving-too-many-open-files-exception-red5-or-any-other-application
+
+### The CPU average across my load-balanced targets is higher than the source
+If you are replaying traffic from multiple listeners to a load-balanced target and you use sticky sessions, you may observe that the target servers have a higher CPU load than the listener servers. This may be because the sticky session cookie of the original load balancer is not honored by the target load balancer thus resulting in requests that would normally hit the same target server hitting different servers on the backend thus reducing some caching benefits gained via the load balancing.  Try running just one listener against one replay target and see if the CPU utilization comparison is more accurate.
+
+## Tuning
+
+To achieve the top most performance you should tune the source server system limits:
+
+    net.ipv4.tcp_max_tw_buckets = 65536
+    net.ipv4.tcp_tw_recycle = 1
+    net.ipv4.tcp_tw_reuse = 0
+    net.ipv4.tcp_max_syn_backlog = 131072
+    net.ipv4.tcp_syn_retries = 3
+    net.ipv4.tcp_synack_retries = 3
+    net.ipv4.tcp_retries1 = 3
+    net.ipv4.tcp_retries2 = 8
+    net.ipv4.tcp_rmem = 16384 174760 349520
+    net.ipv4.tcp_wmem = 16384 131072 262144
+    net.ipv4.tcp_mem = 262144 524288 1048576
+    net.ipv4.tcp_max_orphans = 65536
+    net.ipv4.tcp_fin_timeout = 10
+    net.ipv4.tcp_low_latency = 1
+    net.ipv4.tcp_syncookies = 0
+
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (git checkout -b my-new-feature)
+3. Commit your changes (git commit -am 'Added some feature')
+4. Push to the branch (git push origin my-new-feature)
+5. Create new Pull Request
+
+## Companies using Gor
+
+* [Granify](http://granify.com)
+* [GOV.UK](https://www.gov.uk) ([Government Digital Service](http://digital.cabinetoffice.gov.uk/))
+* [theguardian.com](http://theguardian.com)
+* [TomTom](http://www.tomtom.com/)
+* [3SCALE](http://www.3scale.net/)
+* [Optionlab](http://www.opinionlab.com)
+* [TubeMogul] (http://tubemogul.com)
+* To add your company drop me a line to github.com/buger or leonsbox@gmail.com

byteutils/byteutils.go

@@ -0,0 +1,40 @@
+// Package byteutils probvides helpers for working with byte slices
+package byteutils
+
+// Cut elements from slice for a given range
+func Cut(a []byte, from, to int) []byte {
+	copy(a[from:], a[to:])
+	a = a[:len(a)-to+from]
+
+	return a
+}
+
+// Insert new slice at specified position
+func Insert(a []byte, i int, b []byte) []byte {
+	a = append(a, make([]byte, len(b))...)
+	copy(a[i+len(b):], a[i:])
+	copy(a[i:i+len(b)], b)
+
+	return a
+}
+
+// Replace function unlike bytes.Replace allows you to specify range
+func Replace(a []byte, from, to int, new []byte) []byte {
+	lenDiff := len(new) - (to - from)
+
+	if lenDiff > 0 {
+		// Extend if new segment bigger
+		a = append(a, make([]byte, lenDiff)...)
+		copy(a[to+lenDiff:], a[to:])
+		copy(a[from:from+len(new)], new)
+
+		return a
+	} else if lenDiff < 0 {
+		copy(a[from:], new)
+		copy(a[from+len(new):], a[to:])
+		return a[:len(a)+lenDiff]
+	} else { // same size
+		copy(a[from:], new)
+		return a
+	}
+}

byteutils/byteutils_test.go

@@ -0,0 +1,32 @@
+package byteutils
+
+import (
+	"bytes"
+	"testing"
+)
+
+func TestCut(t *testing.T) {
+	if !bytes.Equal(Cut([]byte("123456"), 2, 4), []byte("1256")) {
+		t.Error("Should properly cut")
+	}
+}
+
+func TestInsert(t *testing.T) {
+	if !bytes.Equal(Insert([]byte("123456"), 2, []byte("abcd")), []byte("12abcd3456")) {
+		t.Error("Should insert into middle of slice")
+	}
+}
+
+func TestReplace(t *testing.T) {
+	if !bytes.Equal(Replace([]byte("123456"), 2, 4, []byte("ab")), []byte("12ab56")) {
+		t.Error("Should replace when same length")
+	}
+
+	if !bytes.Equal(Replace([]byte("123456"), 2, 4, []byte("abcd")), []byte("12abcd56")) {
+		t.Error("Should replace when replacement length bigger")
+	}
+
+	if !bytes.Equal(Replace([]byte("123456"), 2, 5, []byte("ab")), []byte("12ab6")) {
+		t.Error("Should replace when replacement length bigger")
+	}
+}

elasticsearch.go

@@ -4,8 +4,8 @@ import (
 	"encoding/json"
 	"github.com/buger/elastigo/api"
 	"github.com/buger/elastigo/core"
+	"github.com/buger/gor/proto"
 	"log"
-	"net/http"
 	"regexp"
 	"time"
 )
@@ -26,27 +26,27 @@ type ESPlugin struct {
 }
 
 type ESRequestResponse struct {
-	ReqUrl               string         `json:"Req_URL"`
-	ReqMethod            string         `json:"Req_Method"`
-	ReqUserAgent         string         `json:"Req_User-Agent"`
-	ReqAcceptLanguage    string         `json:"Req_Accept-Language,omitempty"`
-	ReqAccept            string         `json:"Req_Accept,omitempty"`
-	ReqAcceptEncoding    string         `json:"Req_Accept-Encoding,omitempty"`
-	ReqIfModifiedSince   string         `json:"Req_If-Modified-Since,omitempty"`
-	ReqConnection        string         `json:"Req_Connection,omitempty"`
-	ReqCookies           []*http.Cookie `json:"Req_Cookies,omitempty"`
-	RespStatus           string         `json:"Resp_Status"`
-	RespStatusCode       int            `json:"Resp_Status-Code"`
-	RespProto            string         `json:"Resp_Proto,omitempty"`
-	RespContentLength    int64          `json:"Resp_Content-Length,omitempty"`
-	RespContentType      string         `json:"Resp_Content-Type,omitempty"`
-	RespTransferEncoding []string       `json:"Resp_Transfer-Encoding,omitempty"`
-	RespContentEncoding  string         `json:"Resp_Content-Encoding,omitempty"`
-	RespExpires          string         `json:"Resp_Expires,omitempty"`
-	RespCacheControl     string         `json:"Resp_Cache-Control,omitempty"`
-	RespVary             string         `json:"Resp_Vary,omitempty"`
-	RespSetCookie        string         `json:"Resp_Set-Cookie,omitempty"`
-	Rtt                  int64          `json:"RTT"`
+	ReqUrl               []byte `json:"Req_URL"`
+	ReqMethod            []byte `json:"Req_Method"`
+	ReqUserAgent         []byte `json:"Req_User-Agent"`
+	ReqAcceptLanguage    []byte `json:"Req_Accept-Language,omitempty"`
+	ReqAccept            []byte `json:"Req_Accept,omitempty"`
+	ReqAcceptEncoding    []byte `json:"Req_Accept-Encoding,omitempty"`
+	ReqIfModifiedSince   []byte `json:"Req_If-Modified-Since,omitempty"`
+	ReqConnection        []byte `json:"Req_Connection,omitempty"`
+	ReqCookies           []byte `json:"Req_Cookies,omitempty"`
+	RespStatus           []byte `json:"Resp_Status"`
+	RespStatusCode       []byte `json:"Resp_Status-Code"`
+	RespProto            []byte `json:"Resp_Proto,omitempty"`
+	RespContentLength    []byte `json:"Resp_Content-Length,omitempty"`
+	RespContentType      []byte `json:"Resp_Content-Type,omitempty"`
+	RespTransferEncoding []byte `json:"Resp_Transfer-Encoding,omitempty"`
+	RespContentEncoding  []byte `json:"Resp_Content-Encoding,omitempty"`
+	RespExpires          []byte `json:"Resp_Expires,omitempty"`
+	RespCacheControl     []byte `json:"Resp_Cache-Control,omitempty"`
+	RespVary             []byte `json:"Resp_Vary,omitempty"`
+	RespSetCookie        []byte `json:"Resp_Set-Cookie,omitempty"`
+	Rtt                  int64  `json:"RTT"`
 	Timestamp            time.Time
 }
 
@@ -111,8 +111,8 @@ func (p *ESPlugin) RttDurationToMs(d time.Duration) int64 {
 	return int64(fl)
 }
 
-func (p *ESPlugin) ResponseAnalyze(req *http.Request, resp *http.Response, start, stop time.Time) {
-	if resp == nil {
+func (p *ESPlugin) ResponseAnalyze(req, resp []byte, start, stop time.Time) {
+	if len(resp) == 0 {
 		// nil http response - skipped elasticsearch export for this request
 		return
 	}
@@ -120,26 +120,26 @@ func (p *ESPlugin) ResponseAnalyze(req *http.Request, resp *http.Response, start
 	rtt := p.RttDurationToMs(stop.Sub(start))
 
 	esResp := ESRequestResponse{
-		ReqUrl:               req.URL.String(),
-		ReqMethod:            req.Method,
-		ReqUserAgent:         req.UserAgent(),
-		ReqAcceptLanguage:    req.Header.Get("Accept-Language"),
-		ReqAccept:            req.Header.Get("Accept"),
-		ReqAcceptEncoding:    req.Header.Get("Accept-Encoding"),
-		ReqIfModifiedSince:   req.Header.Get("If-Modified-Since"),
-		ReqConnection:        req.Header.Get("Connection"),
-		ReqCookies:           req.Cookies(),
-		RespStatus:           resp.Status,
-		RespStatusCode:       resp.StatusCode,
-		RespProto:            resp.Proto,
-		RespContentLength:    resp.ContentLength,
-		RespContentType:      resp.Header.Get("Content-Type"),
-		RespTransferEncoding: resp.TransferEncoding,
-		RespContentEncoding:  resp.Header.Get("Content-Encoding"),
-		RespExpires:          resp.Header.Get("Expires"),
-		RespCacheControl:     resp.Header.Get("Cache-Control"),
-		RespVary:             resp.Header.Get("Vary"),
-		RespSetCookie:        resp.Header.Get("Set-Cookie"),
+		ReqUrl:               proto.Path(req),
+		ReqMethod:            proto.Method(req),
+		ReqUserAgent:         proto.Header(req, []byte("User-Agent")),
+		ReqAcceptLanguage:    proto.Header(req, []byte("Accept-Language")),
+		ReqAccept:            proto.Header(req, []byte("Accept")),
+		ReqAcceptEncoding:    proto.Header(req, []byte("Accept-Encoding")),
+		ReqIfModifiedSince:   proto.Header(req, []byte("If-Modified-Since")),
+		ReqConnection:        proto.Header(req, []byte("Connection")),
+		ReqCookies:           proto.Header(req, []byte("Cookie")),
+		RespStatus:           proto.Status(resp),
+		RespStatusCode:       proto.Status(resp),
+		RespProto:            proto.Method(resp),
+		RespContentLength:    proto.Header(resp, []byte("Content-Length")),
+		RespContentType:      proto.Header(resp, []byte("Content-Type")),
+		RespTransferEncoding: proto.Header(resp, []byte("Transfer-Encoding")),
+		RespContentEncoding:  proto.Header(resp, []byte("Content-Encoding")),
+		RespExpires:          proto.Header(resp, []byte("Expires")),
+		RespCacheControl:     proto.Header(resp, []byte("Cache-Control")),
+		RespVary:             proto.Header(resp, []byte("Vary")),
+		RespSetCookie:        proto.Header(resp, []byte("Set-Cookie")),
 		Rtt:                  rtt,
 		Timestamp:            t,
 	}

emitter.go

@@ -1,33 +1,87 @@
 package main
 
 import (
+	"bytes"
 	"io"
+	"time"
 )
 
+// Start initialize loop for sending data from inputs to outputs
 func Start(stop chan int) {
-	for _, in := range Plugins.Inputs {
-		go CopyMulty(in, Plugins.Outputs...)
+	if Settings.middleware != "" {
+		middleware := NewMiddleware(Settings.middleware)
+
+		for _, in := range Plugins.Inputs {
+			middleware.ReadFrom(in)
+		}
+
+		// We going only to read responses, so using same ReadFrom method
+		for _, out := range Plugins.Outputs {
+			if r, ok := out.(io.Reader); ok {
+				middleware.ReadFrom(r)
+			}
+		}
+
+		go CopyMulty(middleware, Plugins.Outputs...)
+	} else {
+		for _, in := range Plugins.Inputs {
+			go CopyMulty(in, Plugins.Outputs...)
+		}
 	}
 
-	select {
-	case <-stop:
-		return
+	for {
+		select {
+		case <-stop:
+			return
+		case <-time.After(time.Second):
+		}
 	}
 }
 
-// Copy from 1 reader to multiple writers
+// CopyMulty copies from 1 reader to multiple writers
 func CopyMulty(src io.Reader, writers ...io.Writer) (err error) {
-	buf := make([]byte, 32*1024)
+	buf := make([]byte, 5*1024*1024)
 	wIndex := 0
+	modifier := NewHTTPModifier(&Settings.modifierConfig)
 
 	for {
 		nr, er := src.Read(buf)
-		if nr > 0 && len(buf) > nr{
-			Debug("Sending", src, ": ", string(buf[0:nr]))
+
+		if nr > 0 && len(buf) > nr {
+			payload := buf[0:nr]
+
+			_maxN := nr
+			if nr > 500 {
+				_maxN = 500
+			}
+
+			if Settings.debug {
+				Debug("[EMITTER] input:", string(payload[0:_maxN]))
+			}
+
+			if modifier != nil && isRequestPayload(payload) {
+				headSize := bytes.IndexByte(payload, '\n') + 1
+				body := payload[headSize:]
+				originalBodyLen := len(body)
+				body = modifier.Rewrite(body)
+
+				// If modifier tells to skip request
+				if len(body) == 0 {
+					continue
+				}
+
+				if originalBodyLen != len(body) {
+					payload = append(payload[:headSize], body...)
+				}
+
+				if Settings.debug {
+					Debug("[EMITTER] Rewrittern input:", len(payload), "First 500 bytes:", string(payload[0:_maxN]))
+				}
+			}
 
 			if Settings.splitOutput {
 				// Simple round robin
-				writers[wIndex].Write(buf[0:nr])
+				writers[wIndex].Write(payload)
 
 				wIndex++
 
@@ -36,7 +90,7 @@ func CopyMulty(src io.Reader, writers ...io.Writer) (err error) {
 				}
 			} else {
 				for _, dst := range writers {
-					dst.Write(buf[0:nr])
+					dst.Write(payload)
 				}
 			}
 

examples/middleware/echo.java

@@ -0,0 +1,30 @@
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+
+import org.apache.commons.io.IOUtils;
+
+public class echo {
+    public static void main(String[] args) {
+        if(args != null){
+            for(String arg : args){
+                System.out.println(arg);
+            }
+
+        }
+
+        BufferedReader stdin = new BufferedReader(new InputStreamReader(
+                System.in));
+        String line = null;
+
+        try {
+            while ((line = stdin.readLine()) != null) {
+
+                System.out.println(line);
+
+            }
+        } catch (IOException e) {
+            IOUtils.closeQuietly(stdin);
+        }
+    }
+}

examples/middleware/echo.rb

@@ -0,0 +1,15 @@
+#!/usr/bin/env ruby
+# encoding: utf-8
+while data = STDIN.gets
+  next unless data
+  data = data.chomp
+
+  decoded = [data].pack("H*")
+  encoded = decoded.unpack("H*").first
+
+  STDOUT.puts encoded
+
+  STDERR.puts "[DEBUG][MIDDLEWARE] Original data: #{data}"
+  STDERR.puts "[DEBUG][MIDDLEWARE] Decoded request: #{decoded}"
+  STDERR.puts "[DEBUG][MIDDLEWARE] Encoded data: #{encoded}"
+end

examples/middleware/echo.sh

@@ -0,0 +1,42 @@
+#!/usr/bin/env bash
+#
+# `xxd` utility included into vim-common package
+# It allow hex decoding/encoding
+
+function log {
+    # Logging to stderr, because stdout/stdin used for data transfer
+    >&2 echo "[DEBUG][ECHO] $1"
+}
+
+while read line; do
+    decoded=$(echo -e "$line" | xxd -r -p)
+
+    header=$(echo -e "$decoded" | head -n +1)
+    payload=$(echo -e "$decoded" | tail -n +2)
+
+    encoded=$(echo -e "$header\n$payload" | xxd -p | tr -d "\\n")
+
+    log ""
+    log "==================================="
+
+    case ${header:0:1} in
+    "1")
+        log "Request type: Request"
+        ;;
+    "2")
+        log "Request type: Original Response"
+        ;;
+    "3")
+        log "Request type: Replayed Response"
+        ;;
+    *)
+        log "Unknown request type $header"
+    esac
+    echo "$encoded"
+
+    log "==================================="
+
+    log "Original data: $line"
+    log "Decoded request: $decoded"
+    log "Encoded data: $encoded"
+done;

examples/middleware/token_modifier.go

@@ -0,0 +1,119 @@
+/*
+This middleware made for auth system that randomly generate access tokens, which used later for accessing secure content. Since there is no pre-defined token value, naive approach without middleware (or if middleware use only request payloads) will fail, because replayed server have own tokens, not synced with origin. To fix this, our middleware should take in account responses of replayed and origin server, store `originalToken -> replayedToken` aliases and rewrite all requests using this token to use replayed alias. See `middleware_test.go#TestTokenMiddleware` test for examples of using this middleware.
+
+How middleware works:
+
+                   Original request      +--------------+
++-------------+----------STDIN---------->+              |
+|  Gor input  |                          |  Middleware  |
++-------------+----------STDIN---------->+              |
+                   Original response     +------+---+---+
+                                                |   ^
++-------------+    Modified request             v   |
+| Gor output  +<---------STDOUT-----------------+   |
++-----+-------+                                     |
+      |                                             |
+      |            Replayed response                |
+      +------------------STDIN----------------->----+
+*/
+
+package main
+
+import (
+	"bufio"
+	"bytes"
+	"encoding/hex"
+	"fmt"
+	"github.com/buger/gor/proto"
+	"os"
+)
+
+// requestID -> originalToken
+var originalTokens map[string][]byte
+
+// originalToken -> replayedToken
+var tokenAliases map[string][]byte
+
+func main() {
+	originalTokens = make(map[string][]byte)
+	tokenAliases = make(map[string][]byte)
+
+	scanner := bufio.NewScanner(os.Stdin)
+
+	for scanner.Scan() {
+		encoded := scanner.Bytes()
+		buf := make([]byte, len(encoded)/2)
+		hex.Decode(buf, encoded)
+
+		go process(buf)
+	}
+}
+
+func process(buf []byte) {
+	// First byte indicate payload type, possible values:
+	//  1 - Request
+	//  2 - Response
+	//  3 - ReplayedResponse
+	payloadType := buf[0]
+	headerSize := bytes.IndexByte(buf, '\n') + 1
+	header := buf[:headerSize-1]
+
+	// Header contains space separated values of: request type, request id, and request start time (or round-trip time for responses)
+	meta := bytes.Split(header, []byte(" "))
+	// For each request you should receive 3 payloads (request, response, replayed response) with same request id
+	reqID := string(meta[1])
+	payload := buf[headerSize:]
+
+	Debug("Received payload:", string(buf))
+
+	switch payloadType {
+	case '1': // Request
+		if bytes.Equal(proto.Path(payload), []byte("/token")) {
+			originalTokens[reqID] = []byte{}
+			Debug("Found token request:", reqID)
+		} else {
+			token, vs, _ := proto.PathParam(payload, []byte("token"))
+
+			if vs != -1 { // If there is GET token param
+				if alias, ok := tokenAliases[string(token)]; ok {
+					// Rewrite original token to alias
+					payload = proto.SetPathParam(payload, []byte("token"), alias)
+
+					// Copy modified payload to our buffer
+					buf = append(buf[:headerSize], payload...)
+				}
+			}
+		}
+
+		// Emitting data back
+		os.Stdout.Write(encode(buf))
+	case '2': // Original response
+		if _, ok := originalTokens[reqID]; ok {
+			// Token is inside response body
+			secureToken := proto.Body(payload)
+			originalTokens[reqID] = secureToken
+			Debug("Remember origial token:", string(secureToken))
+		}
+	case '3': // Replayed response
+		if originalToken, ok := originalTokens[reqID]; ok {
+			delete(originalTokens, reqID)
+			secureToken := proto.Body(payload)
+			tokenAliases[string(originalToken)] = secureToken
+
+			Debug("Create alias for new token token, was:", string(originalToken), "now:", string(secureToken))
+		}
+	}
+}
+
+func encode(buf []byte) []byte {
+	dst := make([]byte, len(buf)*2+1)
+	hex.Encode(dst, buf)
+	dst[len(dst)-1] = '\n'
+
+	return dst
+}
+
+func Debug(args ...interface{}) {
+	fmt.Fprint(os.Stderr, "[DEBUG][TOKEN-MOD] ")
+	fmt.Fprintln(os.Stderr, args...)
+}

gor.go

@@ -7,7 +7,8 @@ import (
 	"fmt"
 	"log"
 	"os"
-	"runtime/debug"
+	"runtime"
+	_ "runtime/debug"
 	"runtime/pprof"
 	"time"
 )
@@ -19,14 +20,17 @@ var (
 )
 
 func main() {
-	// Don't exit on panic
-	defer func() {
-		if r := recover(); r != nil {
-			if _, ok := r.(error); !ok {
-				fmt.Printf("PANIC: pkg: %v %s \n", r, debug.Stack())
-			}
-		}
-	}()
+	// // Don't exit on panic
+	// defer func() {
+	// 	if r := recover(); r != nil {
+	// 		fmt.Printf("PANIC: pkg: %v %s \n", r, debug.Stack())
+	// 	}
+	// }()
+
+	// If not set via env cariable
+	if len(os.Getenv("GOMAXPROCS")) == 0 {
+		runtime.GOMAXPROCS(runtime.NumCPU() * 2)
+	}
 
 	fmt.Println("Version:", VERSION)
 

gor_stat.go

@@ -1,9 +1,9 @@
 package main
 
 import (
-	"time"
 	"log"
 	"strconv"
+	"time"
 )
 
 const (
@@ -11,11 +11,11 @@ const (
 )
 
 type GorStat struct {
-	statName	string
-	latest		int
-	mean	 	int
-	max			int
-	count		int
+	statName string
+	latest   int
+	mean     int
+	max      int
+	count    int
 }
 
 func NewGorStat(statName string) (s *GorStat) {
@@ -54,14 +54,13 @@ func (s *GorStat) Reset() {
 }
 
 func (s *GorStat) String() string {
-	return s.statName + ":" + strconv.Itoa(s.latest) + "," + strconv.Itoa(s.mean) + "," + strconv.Itoa(s.max) + "," + strconv.Itoa(s.count) + "," + strconv.Itoa(s.count / rate)
+	return s.statName + ":" + strconv.Itoa(s.latest) + "," + strconv.Itoa(s.mean) + "," + strconv.Itoa(s.max) + "," + strconv.Itoa(s.count) + "," + strconv.Itoa(s.count/rate)
 }
 
 func (s *GorStat) reportStats() {
 	for {
-			log.Println(s)
-			s.Reset()
-			time.Sleep(rate * time.Second)
+		log.Println(s)
+		s.Reset()
+		time.Sleep(rate * time.Second)
 	}
 }
-

http_client.go

@@ -0,0 +1,228 @@
+package main
+
+import (
+	"crypto/tls"
+	"github.com/buger/gor/proto"
+	"io"
+	"log"
+	"net"
+	"net/url"
+	"runtime/debug"
+	"strings"
+	"time"
+)
+
+var defaultPorts = map[string]string{
+	"http":  "80",
+	"https": "443",
+}
+
+type HTTPClientConfig struct {
+	FollowRedirects    int
+	Debug              bool
+	OriginalHost       bool
+	ConnectionTimeout  time.Duration
+	Timeout            time.Duration
+	ResponseBufferSize int
+}
+
+type HTTPClient struct {
+	baseURL        string
+	scheme         string
+	host           string
+	conn           net.Conn
+	respBuf        []byte
+	config         *HTTPClientConfig
+	redirectsCount int
+}
+
+func NewHTTPClient(baseURL string, config *HTTPClientConfig) *HTTPClient {
+	if !strings.HasPrefix(baseURL, "http") {
+		baseURL = "http://" + baseURL
+	}
+
+	u, _ := url.Parse(baseURL)
+	if !strings.Contains(u.Host, ":") {
+		if u.Scheme != "http" {
+			u.Host += ":" + defaultPorts[u.Scheme]
+		}
+	}
+
+	if config.Timeout.Nanoseconds() == 0 {
+		config.Timeout = 5 * time.Second
+	}
+
+	if config.ResponseBufferSize == 0 {
+		config.ResponseBufferSize = 512 * 1024 // 500kb
+	}
+
+	client := new(HTTPClient)
+	client.baseURL = u.String()
+	client.host = u.Host
+	client.scheme = u.Scheme
+	client.respBuf = make([]byte, config.ResponseBufferSize)
+	client.config = config
+
+	return client
+}
+
+func (c *HTTPClient) Connect() (err error) {
+	c.Disconnect()
+
+	if !strings.Contains(c.host, ":") {
+		c.conn, err = net.DialTimeout("tcp", c.host+":80", c.config.ConnectionTimeout)
+	} else {
+		c.conn, err = net.DialTimeout("tcp", c.host, c.config.ConnectionTimeout)
+	}
+
+	if c.scheme == "https" {
+		tlsConn := tls.Client(c.conn, &tls.Config{InsecureSkipVerify: true})
+
+		if err = tlsConn.Handshake(); err != nil {
+			return
+		}
+
+		c.conn = tlsConn
+	}
+
+	return
+}
+
+func (c *HTTPClient) Disconnect() {
+	if c.conn != nil {
+		c.conn.Close()
+		c.conn = nil
+		Debug("[HTTP] Disconnected: ", c.baseURL)
+	}
+}
+
+func (c *HTTPClient) isAlive() bool {
+	one := make([]byte, 1)
+
+	// Ready 1 byte from socket without timeout to check if it not closed
+	c.conn.SetReadDeadline(time.Now().Add(time.Millisecond))
+	if _, err := c.conn.Read(one); err == io.EOF {
+		return false
+	}
+
+	return true
+}
+
+func (c *HTTPClient) Send(data []byte) (response []byte, err error) {
+	// Don't exit on panic
+	defer func() {
+		if r := recover(); r != nil {
+			Debug("[HTTPClient]", r, string(data))
+
+			if _, ok := r.(error); !ok {
+				log.Println("[HTTPClient] Failed to send request: ", string(data))
+				log.Println("PANIC: pkg:", r, debug.Stack())
+			}
+		}
+	}()
+
+	if c.conn == nil || !c.isAlive() {
+		Debug("[HTTPClient] Connecting:", c.baseURL)
+		if err = c.Connect(); err != nil {
+			log.Println("[HTTPClient] Connection error:", err)
+			response = errorPayload(HTTP_CONNECTION_ERROR)
+			return
+		}
+	}
+
+	timeout := time.Now().Add(c.config.Timeout)
+
+	c.conn.SetWriteDeadline(timeout)
+
+	if !c.config.OriginalHost {
+		data = proto.SetHost(data, []byte(c.baseURL), []byte(c.host))
+	}
+
+	if c.config.Debug {
+		Debug("[HTTPClient] Sending:", string(data))
+	}
+
+	if _, err = c.conn.Write(data); err != nil {
+		Debug("[HTTPClient] Write error:", err, c.baseURL)
+		response = errorPayload(HTTP_TIMEOUT)
+		return
+	}
+
+	c.conn.SetReadDeadline(timeout)
+	n, err := c.conn.Read(c.respBuf)
+
+	// If response large then our buffer, we need to read all response buffer
+	// Otherwise it will corrupt response of next request
+	// Parsing response body is non trivial thing, especially with keep-alive
+	// Simples case is to to close connection if response too large
+	//
+	// See https://github.com/buger/gor/issues/184
+	if n == len(c.respBuf) {
+		c.Disconnect()
+	}
+
+	if err != nil {
+		Debug("[HTTPClient] Response read error", err, c.conn)
+		response = errorPayload(HTTP_TIMEOUT)
+		return
+	}
+
+	payload := c.respBuf[:n]
+
+	if c.config.Debug {
+		Debug("[HTTPClient] Received:", string(payload))
+	}
+
+	if c.config.FollowRedirects > 0 && c.redirectsCount < c.config.FollowRedirects {
+		status := payload[9:12]
+
+		// 3xx requests
+		if status[0] == '3' {
+			c.redirectsCount++
+
+			location := proto.Header(payload, []byte("Location"))
+			redirectPayload := []byte("GET " + string(location) + " HTTP/1.1\r\n\r\n")
+
+			if c.config.Debug {
+				Debug("[HTTPClient] Redirecting to: " + string(location))
+			}
+
+			return c.Send(redirectPayload)
+		}
+	}
+
+	c.redirectsCount = 0
+
+	return payload, err
+}
+
+func (c *HTTPClient) Get(path string) (response []byte, err error) {
+	payload := "GET " + path + " HTTP/1.1\r\n\r\n"
+
+	return c.Send([]byte(payload))
+}
+
+const (
+	// https://support.cloudflare.com/hc/en-us/articles/200171936-Error-520-Web-server-is-returning-an-unknown-error
+	HTTP_UNKNOWN_ERROR = "520"
+	// https://support.cloudflare.com/hc/en-us/articles/200171916-Error-521-Web-server-is-down
+	HTTP_CONNECTION_ERROR = "521"
+	// https://support.cloudflare.com/hc/en-us/articles/200171906-Error-522-Connection-timed-out
+	HTTP_CONNECTION_TIMEOUT = "522"
+	// https://support.cloudflare.com/hc/en-us/articles/200171946-Error-523-Origin-is-unreachable
+	HTTP_UNREACHABLE = "523"
+	// https://support.cloudflare.com/hc/en-us/articles/200171926-Error-524-A-timeout-occurred
+	HTTP_TIMEOUT = "524"
+)
+
+var errorPayloadTemplate = "HTTP/1.1 202 Accepted\r\nDate: Mon, 17 Aug 2015 14:10:11 GMT\r\nContent-Length: 0\r\nContent-Type: text/plain; charset=utf-8\r\n\r\n"
+
+func errorPayload(errorCode string) []byte {
+	payload := make([]byte, len(errorPayloadTemplate))
+	copy(payload, errorPayloadTemplate)
+
+	copy(payload[29:58], []byte(time.Now().Format(time.RFC1123)))
+	copy(payload[9:12], errorCode)
+
+	return payload
+}

http_client_test.go

@@ -0,0 +1,400 @@
+package main
+
+import (
+	"bytes"
+	"crypto/rand"
+	"github.com/buger/gor/proto"
+	"io/ioutil"
+	_ "log"
+	"net"
+	"net/http"
+	"net/http/httptest"
+	"net/http/httputil"
+	_ "reflect"
+	"sync"
+	"testing"
+	"time"
+	_ "time"
+)
+
+func TestHTTPClientURLPort(t *testing.T) {
+	c1 := NewHTTPClient("http://example.com", &HTTPClientConfig{})
+	if c1.baseURL != "http://example.com" {
+		t.Error("Sould not add 80 port for http:", c1.baseURL)
+	}
+
+	c2 := NewHTTPClient("https://example.com", &HTTPClientConfig{})
+	if c2.baseURL != "https://example.com:443" {
+		t.Error("Sould add 443 port for https:", c2.baseURL)
+	}
+
+	c3 := NewHTTPClient("https://example.com:1", &HTTPClientConfig{})
+	if c3.baseURL != "https://example.com:1" {
+		t.Error("Sould use specified port:", c3.baseURL)
+	}
+
+	c4 := NewHTTPClient("example.com", &HTTPClientConfig{})
+	if c4.baseURL != "http://example.com" {
+		t.Error("Sould not add default protocol:", c4.baseURL)
+	}
+}
+
+func TestHTTPClientSend(t *testing.T) {
+	wg := new(sync.WaitGroup)
+
+	payload := func(reqType string) []byte {
+		switch reqType {
+		case "GET":
+			return []byte("GET / HTTP/1.1\r\n\r\n")
+		case "POST":
+			return []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
+		case "POST_CHUNKED":
+			return []byte("POST / HTTP/1.1\r\nHost: www.w3.org\r\nTransfer-Encoding: chunked\r\n\r\n4\r\nWiki\r\n5\r\npedia\r\ne\r\n in\r\n\r\nchunks.\r\n0\r\n\r\n")
+		}
+
+		return []byte("")
+	}
+
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+
+		if r.Method == "POST" {
+			defer r.Body.Close()
+			body, _ := ioutil.ReadAll(r.Body)
+
+			if len(r.TransferEncoding) > 0 && r.TransferEncoding[0] == "chunked" {
+				if string(body) != "Wikipedia in\r\n\r\nchunks." {
+					t.Error("Wrong POST body:", body, string(body))
+				}
+			} else {
+				if string(body) != "a=1&b=2" {
+					buf, _ := httputil.DumpRequest(r, true)
+					t.Error("Wrong POST body:", string(body), string(buf))
+				}
+			}
+		}
+
+		wg.Done()
+	}))
+	defer server.Close()
+
+	client := NewHTTPClient(server.URL, &HTTPClientConfig{Debug: true})
+
+	wg.Add(4)
+	client.Send(payload("POST"))
+	client.Send(payload("GET"))
+	client.Send(payload("POST_CHUNKED"))
+	client.Send(payload("POST"))
+
+	wg.Wait()
+}
+
+// https://github.com/buger/gor/issues/184
+func TestHTTPClientResponseBuffer(t *testing.T) {
+	wg := new(sync.WaitGroup)
+
+	payload := []byte("GET / HTTP/1.1\r\n\r\n")
+
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+
+		size := 10 * 1024 // 10kb
+		rb := make([]byte, size)
+		rand.Read(rb)
+
+		w.Write(rb)
+
+		wg.Done()
+	}))
+	defer server.Close()
+
+	client := NewHTTPClient(server.URL, &HTTPClientConfig{Debug: false, ResponseBufferSize: 1024})
+
+	wg.Add(2)
+	client.Send(payload)
+	resp, _ := client.Send(payload)
+
+	if !bytes.Equal(resp[0:8], []byte("HTTP/1.1")) {
+		t.Error("Response buffer contains data from previous request", string(resp[0:5]))
+	}
+
+	wg.Wait()
+}
+
+func TestHTTPClientHTTPSSend(t *testing.T) {
+	wg := new(sync.WaitGroup)
+
+	payload := func(reqType string) []byte {
+		switch reqType {
+		case "GET":
+			return []byte("GET / HTTP/1.1\r\n\r\n")
+		case "POST":
+			return []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
+		case "POST_CHUNKED":
+			return []byte("POST / HTTP/1.1\r\nHost: www.w3.org\r\nTransfer-Encoding: chunked\r\n\r\n4\r\nWiki\r\n5\r\npedia\r\ne\r\n in\r\n\r\nchunks.\r\n0\r\n\r\n")
+		}
+
+		return []byte("")
+	}
+
+	server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+
+		if r.Method == "POST" {
+			defer r.Body.Close()
+			body, _ := ioutil.ReadAll(r.Body)
+
+			if len(r.TransferEncoding) > 0 && r.TransferEncoding[0] == "chunked" {
+				if string(body) != "Wikipedia in\r\n\r\nchunks." {
+					t.Error("Wrong POST body:", body, string(body))
+				}
+			} else {
+				if string(body) != "a=1&b=2" {
+					buf, _ := httputil.DumpRequest(r, true)
+					t.Error("Wrong POST body:", string(body), string(buf))
+				}
+			}
+		}
+
+		wg.Done()
+	}))
+	defer server.Close()
+
+	client := NewHTTPClient(server.URL, &HTTPClientConfig{})
+
+	wg.Add(4)
+	client.Send(payload("POST"))
+	client.Send(payload("GET"))
+	client.Send(payload("POST_CHUNKED"))
+	client.Send(payload("POST"))
+
+	wg.Wait()
+}
+
+func TestHTTPClientServerInstantDisconnect(t *testing.T) {
+	wg := new(sync.WaitGroup)
+
+	GETPayload := []byte("GET / HTTP/1.1\r\n\r\n")
+
+	ln, _ := net.Listen("tcp", ":0")
+	defer ln.Close()
+
+	go func() {
+		for {
+			conn, err := ln.Accept()
+
+			if err != nil {
+				break
+			}
+			conn.Close()
+			wg.Done()
+		}
+	}()
+
+	client := NewHTTPClient(ln.Addr().String(), &HTTPClientConfig{})
+
+	wg.Add(2)
+	client.Send(GETPayload)
+	client.Send(GETPayload)
+
+	wg.Wait()
+}
+
+func TestHTTPClientServerNoKeepAlive(t *testing.T) {
+	wg := new(sync.WaitGroup)
+
+	GETPayload := []byte("GET / HTTP/1.1\r\n\r\n")
+
+	ln, _ := net.Listen("tcp", ":0")
+	defer ln.Close()
+
+	go func() {
+		for {
+			conn, err := ln.Accept()
+			if err != nil {
+				break
+			}
+
+			buf := make([]byte, 4096)
+			reqLen, err := conn.Read(buf)
+			if err != nil {
+				t.Error("Error reading:", err.Error())
+			}
+			Debug("Received: ", string(buf[0:reqLen]))
+			conn.Write([]byte("OK"))
+
+			// No keep-alive connections
+			conn.Close()
+
+			wg.Done()
+		}
+	}()
+
+	client := NewHTTPClient(ln.Addr().String(), &HTTPClientConfig{})
+
+	wg.Add(2)
+	client.Send(GETPayload)
+	client.Send(GETPayload)
+
+	wg.Wait()
+}
+
+func TestHTTPClientRedirect(t *testing.T) {
+	wg := new(sync.WaitGroup)
+
+	GETPayload := []byte("GET / HTTP/1.1\r\n\r\n")
+
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+
+		if r.URL.Path == "/" {
+			http.Redirect(w, r, "/new", 301)
+		}
+
+		wg.Done()
+	}))
+	defer server.Close()
+
+	client := NewHTTPClient(server.URL, &HTTPClientConfig{FollowRedirects: 1, Debug: false})
+
+	// Should do 2 queries
+	wg.Add(2)
+	client.Send(GETPayload)
+
+	wg.Wait()
+}
+
+func TestHTTPClientRedirectLimit(t *testing.T) {
+	wg := new(sync.WaitGroup)
+
+	GETPayload := []byte("GET / HTTP/1.1\r\n\r\n")
+
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+
+		if r.URL.Path == "/" {
+			http.Redirect(w, r, "/r1", 301)
+		}
+
+		if r.URL.Path == "/r1" {
+			http.Redirect(w, r, "/r2", 301)
+		}
+
+		if r.URL.Path == "/r2" {
+			http.Redirect(w, r, "/new", 301)
+		}
+
+		wg.Done()
+	}))
+	defer server.Close()
+
+	client := NewHTTPClient(server.URL, &HTTPClientConfig{FollowRedirects: 2, Debug: false})
+
+	// Have 3 redirects + 1 GET, but should do only 2 redirects + GET
+	wg.Add(3)
+	client.Send(GETPayload)
+
+	wg.Wait()
+}
+
+func TestHTTPClientHandleHTTP10(t *testing.T) {
+	wg := new(sync.WaitGroup)
+
+	GETPayload := []byte("GET http://foobar.com/path HTTP/1.0\r\n\r\n")
+
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+
+		if r.URL.Path != "/path" {
+			t.Error("Path not match:", r.URL.Path)
+		}
+
+		wg.Done()
+	}))
+	defer server.Close()
+
+	client := NewHTTPClient(server.URL, &HTTPClientConfig{Debug: true})
+
+	wg.Add(1)
+	client.Send(GETPayload)
+
+	wg.Wait()
+}
+
+func TestHTTPClientErrors(t *testing.T) {
+	req := []byte("GET http://foobar.com/path HTTP/1.0\r\n\r\n")
+
+	// Port not exists
+	client := NewHTTPClient("http://127.0.0.1:1", &HTTPClientConfig{Debug: true})
+	if resp, err := client.Send(req); err != nil {
+		if s := proto.Status(resp); !bytes.Equal(s, []byte("521")) {
+			t.Error("Should return status 521 for connection refused, instead:", string(s))
+		}
+	} else {
+		t.Error("Should throw error")
+	}
+
+	client = NewHTTPClient("http://not.existing", &HTTPClientConfig{Debug: true})
+	if resp, err := client.Send(req); err != nil {
+		if s := proto.Status(resp); !bytes.Equal(s, []byte("521")) {
+			t.Error("Should return status 521 for no such host, instead:", string(s))
+		}
+	} else {
+		t.Error("Should throw error")
+	}
+
+	// Non routable IP address to simulate connection timeout
+	client = NewHTTPClient("http://10.255.255.1", &HTTPClientConfig{Debug: true, ConnectionTimeout: 100 * time.Millisecond})
+
+	if resp, err := client.Send(req); err != nil {
+		if s := proto.Status(resp); !bytes.Equal(s, []byte("521")) {
+			t.Error("Should return status 521 for io/timeout:", string(s))
+		}
+	} else {
+		t.Error("Should throw error")
+	}
+
+	// Connecting but io timeout on read
+	ln, _ := net.Listen("tcp", ":0")
+	client = NewHTTPClient("http://"+ln.Addr().String(), &HTTPClientConfig{Debug: true, Timeout: 10 * time.Millisecond})
+	defer ln.Close()
+
+	if resp, err := client.Send(req); err != nil {
+		if s := proto.Status(resp); !bytes.Equal(s, []byte("524")) {
+			t.Error("Should return status 524 for io read, instead:", string(s))
+		}
+	} else {
+		t.Error("Should throw error")
+	}
+
+	// Response read error read tcp [::1]:51128: connection reset by peer &{{0xc20802a000}}
+	ln1, _ := net.Listen("tcp", ":0")
+	go func() {
+		ln1.Accept()
+	}()
+	defer ln1.Close()
+
+	client = NewHTTPClient("http://"+ln1.Addr().String(), &HTTPClientConfig{Debug: true, Timeout: 10 * time.Millisecond})
+
+	if resp, err := client.Send(req); err != nil {
+		if s := proto.Status(resp); !bytes.Equal(s, []byte("524")) {
+			t.Error("Should return status 524 for connection reset by peer, instead:", string(s))
+		}
+	} else {
+		t.Error("Should throw error")
+	}
+
+	ln2, _ := net.Listen("tcp", ":0")
+	go func() {
+		buf := make([]byte, 64*1024)
+		conn, _ := ln2.Accept()
+
+		conn.Read(buf)
+		defer conn.Close()
+	}()
+	defer ln2.Close()
+
+	client = NewHTTPClient("http://"+ln2.Addr().String(), &HTTPClientConfig{Debug: true, Timeout: 10 * time.Millisecond})
+
+	if resp, err := client.Send(req); err != nil {
+		if s := proto.Status(resp); !bytes.Equal(s, []byte("524")) {
+			t.Error("Should return status 524 for connection reset by peer, instead:", string(s))
+		}
+	} else {
+		t.Error("Should throw error")
+	}
+}

http_modifier.go

@@ -0,0 +1,157 @@
+package main
+
+import (
+	"bytes"
+	"hash/fnv"
+
+	"github.com/buger/gor/proto"
+)
+
+type HTTPModifier struct {
+	config *HTTPModifierConfig
+}
+
+func NewHTTPModifier(config *HTTPModifierConfig) *HTTPModifier {
+	// Optimization to skip modifier completely if we do not need it
+	if len(config.urlRegexp) == 0 &&
+		len(config.urlNegativeRegexp) == 0 &&
+		len(config.urlRewrite) == 0 &&
+		len(config.headerFilters) == 0 &&
+		len(config.headerNegativeFilters) == 0 &&
+		len(config.headerHashFilters) == 0 &&
+		len(config.paramHashFilters) == 0 &&
+		len(config.params) == 0 &&
+		len(config.headers) == 0 &&
+		len(config.methods) == 0 {
+		return nil
+	}
+
+	return &HTTPModifier{config: config}
+}
+
+func (m *HTTPModifier) Rewrite(payload []byte) (response []byte) {
+	if !proto.IsHTTPPayload(payload) {
+		return payload
+	}
+
+	if len(m.config.methods) > 0 {
+		method := proto.Method(payload)
+
+		matched := false
+
+		for _, m := range m.config.methods {
+			if bytes.Equal(method, m) {
+				matched = true
+				break
+			}
+		}
+
+		if !matched {
+			return
+		}
+	}
+
+	if len(m.config.headers) > 0 {
+		for _, header := range m.config.headers {
+			payload = proto.SetHeader(payload, []byte(header.Name), []byte(header.Value))
+		}
+	}
+
+	if len(m.config.params) > 0 {
+		for _, param := range m.config.params {
+			payload = proto.SetPathParam(payload, param.Name, param.Value)
+		}
+	}
+
+	if len(m.config.urlRegexp) > 0 {
+		path := proto.Path(payload)
+
+		matched := false
+
+		for _, f := range m.config.urlRegexp {
+			if f.regexp.Match(path) {
+				matched = true
+				break
+			}
+		}
+
+		if !matched {
+			return
+		}
+	}
+
+	if len(m.config.urlNegativeRegexp) > 0 {
+		path := proto.Path(payload)
+
+		for _, f := range m.config.urlNegativeRegexp {
+			if f.regexp.Match(path) {
+				return
+			}
+		}
+	}
+
+	if len(m.config.headerFilters) > 0 {
+		for _, f := range m.config.headerFilters {
+			value := proto.Header(payload, f.name)
+
+			if len(value) > 0 && !f.regexp.Match(value) {
+				return
+			}
+		}
+	}
+
+	if len(m.config.headerNegativeFilters) > 0 {
+		for _, f := range m.config.headerNegativeFilters {
+			value := proto.Header(payload, f.name)
+
+			if len(value) > 0 && f.regexp.Match(value) {
+				return
+			}
+		}
+	}
+
+	if len(m.config.headerHashFilters) > 0 {
+		for _, f := range m.config.headerHashFilters {
+			value := proto.Header(payload, f.name)
+
+			if len(value) > 0 {
+				hasher := fnv.New32a()
+				hasher.Write(value)
+
+				if (hasher.Sum32() % 100) >= f.percent {
+					return
+				}
+			}
+		}
+	}
+
+	if len(m.config.paramHashFilters) > 0 {
+		for _, f := range m.config.paramHashFilters {
+			value, s, _ := proto.PathParam(payload, f.name)
+
+			if s != -1 {
+				hasher := fnv.New32a()
+				hasher.Write(value)
+
+				if (hasher.Sum32() % 100) >= f.percent {
+					return
+				}
+			}
+		}
+	}
+
+	if len(m.config.urlRewrite) > 0 {
+		path := proto.Path(payload)
+
+		for _, f := range m.config.urlRewrite {
+			if f.src.Match(path) {
+				path = f.src.ReplaceAll(path, f.target)
+				payload = proto.SetPath(payload, path)
+
+				break
+			}
+		}
+	}
+
+	return payload
+}

http_modifier_settings.go

@@ -0,0 +1,215 @@
+package main
+
+import (
+	"errors"
+	"fmt"
+	"regexp"
+	"strconv"
+	"strings"
+)
+
+// HTTPModifierConfig holds configuration options for built-in traffic modifier
+type HTTPModifierConfig struct {
+	urlNegativeRegexp     HTTPUrlRegexp
+	urlRegexp             HTTPUrlRegexp
+	urlRewrite            UrlRewriteMap
+	headerFilters         HTTPHeaderFilters
+	headerNegativeFilters HTTPHeaderFilters
+	headerHashFilters     HTTPHashFilters
+	paramHashFilters      HTTPHashFilters
+
+	params  HTTPParams
+	headers HTTPHeaders
+	methods HTTPMethods
+}
+
+//
+// Handling of --http-allow-header, --http-disallow-header options
+//
+type headerFilter struct {
+	name   []byte
+	regexp *regexp.Regexp
+}
+
+// HTTPHeaderFilters holds list of headers and their regexps
+type HTTPHeaderFilters []headerFilter
+
+func (h *HTTPHeaderFilters) String() string {
+	return fmt.Sprint(*h)
+}
+
+func (h *HTTPHeaderFilters) Set(value string) error {
+	valArr := strings.SplitN(value, ":", 2)
+	if len(valArr) < 2 {
+		return errors.New("need both header and value, colon-delimited (ex. user_id:^169$).")
+	}
+	r, err := regexp.Compile(valArr[1])
+	if err != nil {
+		return err
+	}
+
+	*h = append(*h, headerFilter{name: []byte(valArr[0]), regexp: r})
+
+	return nil
+}
+
+//
+// Handling of --http-allow-header-hash and --http-allow-param-hash options
+//
+type hashFilter struct {
+	name    []byte
+	percent uint32
+}
+
+type HTTPHashFilters []hashFilter
+
+func (h *HTTPHashFilters) String() string {
+	return fmt.Sprint(*h)
+}
+
+func (h *HTTPHashFilters) Set(value string) error {
+	valArr := strings.SplitN(value, ":", 2)
+	if len(valArr) < 2 {
+		return errors.New("need both header and value, colon-delimited (ex. user_id:50%)")
+	}
+
+	f := hashFilter{name: []byte(valArr[0])}
+
+	if strings.Contains(valArr[1], "%") {
+		p, _ := strconv.ParseInt(valArr[1][:len(valArr[1])-1], 0, 0)
+		f.percent = uint32(p)
+	} else if strings.Contains(valArr[1], "/") {
+		// DEPRECATED format
+		var num, den uint64
+
+		fracArr := strings.Split(valArr[1], "/")
+		num, _ = strconv.ParseUint(fracArr[0], 10, 64)
+		den, _ = strconv.ParseUint(fracArr[1], 10, 64)
+
+		f.percent = uint32((float64(num) / float64(den)) * 100)
+	} else {
+		return errors.New("Value should be percent and contain '%'")
+	}
+
+	*h = append(*h, f)
+
+	return nil
+}
+
+//
+// Handling of --http-set-header option
+//
+type HTTPHeaders []HTTPHeader
+type HTTPHeader struct {
+	Name  string
+	Value string
+}
+
+func (h *HTTPHeaders) String() string {
+	return fmt.Sprint(*h)
+}
+
+func (h *HTTPHeaders) Set(value string) error {
+	v := strings.SplitN(value, ":", 2)
+	if len(v) != 2 {
+		return errors.New("Expected `Key: Value`")
+	}
+
+	header := HTTPHeader{
+		strings.TrimSpace(v[0]),
+		strings.TrimSpace(v[1]),
+	}
+
+	*h = append(*h, header)
+	return nil
+}
+
+//
+// Handling of --http-set-param option
+//
+type HTTPParams []HTTPParam
+type HTTPParam struct {
+	Name  []byte
+	Value []byte
+}
+
+func (h *HTTPParams) String() string {
+	return fmt.Sprint(*h)
+}
+
+func (h *HTTPParams) Set(value string) error {
+	v := strings.SplitN(value, "=", 2)
+	if len(v) != 2 {
+		return errors.New("Expected `Key=Value`")
+	}
+
+	param := HTTPParam{
+		[]byte(strings.TrimSpace(v[0])),
+		[]byte(strings.TrimSpace(v[1])),
+	}
+
+	*h = append(*h, param)
+	return nil
+}
+
+//
+// Handling of --http-allow-method option
+//
+type HTTPMethods [][]byte
+
+func (h *HTTPMethods) String() string {
+	return fmt.Sprint(*h)
+}
+
+func (h *HTTPMethods) Set(value string) error {
+	*h = append(*h, []byte(value))
+	return nil
+}
+
+//
+// Handling of --http-rewrite-url option
+//
+type urlRewrite struct {
+	src    *regexp.Regexp
+	target []byte
+}
+
+type UrlRewriteMap []urlRewrite
+
+func (r *UrlRewriteMap) String() string {
+	return fmt.Sprint(*r)
+}
+
+func (r *UrlRewriteMap) Set(value string) error {
+	valArr := strings.SplitN(value, ":", 2)
+	if len(valArr) < 2 {
+		return errors.New("need both src and target, colon-delimited (ex. /a:/b)")
+	}
+	regexp, err := regexp.Compile(valArr[0])
+	if err != nil {
+		return err
+	}
+	*r = append(*r, urlRewrite{src: regexp, target: []byte(valArr[1])})
+	return nil
+}
+
+//
+// Handling of --http-allow-url option
+//
+type urlRegexp struct {
+	regexp *regexp.Regexp
+}
+
+type HTTPUrlRegexp []urlRegexp
+
+func (r *HTTPUrlRegexp) String() string {
+	return fmt.Sprint(*r)
+}
+
+func (r *HTTPUrlRegexp) Set(value string) error {
+	regexp, err := regexp.Compile(value)
+
+	*r = append(*r, urlRegexp{regexp: regexp})
+
+	return err
+}

http_modifier_settings_test.go

@@ -0,0 +1,65 @@
+package main
+
+import (
+	"testing"
+)
+
+func TestHTTPHeaderFilters(t *testing.T) {
+	filters := HTTPHeaderFilters{}
+
+	err := filters.Set("Header1:^$")
+	if err != nil {
+		t.Error("Should not error on Header1:^$")
+	}
+
+	err = filters.Set("Header2:^:$")
+	if err != nil {
+		t.Error("Should not error on Header2:^:$")
+	}
+
+	// Missing colon
+	err = filters.Set("Header3-^$")
+	if err == nil {
+		t.Error("Should error on Header2:^:$")
+	}
+}
+
+func TestHTTPHashFilters(t *testing.T) {
+	filters := HTTPHashFilters{}
+
+	err := filters.Set("Header1:1/2")
+	if err != nil {
+		t.Error("Should support old syntax")
+	}
+
+	if filters[0].percent != 50 {
+		t.Error("Wrong percentage", filters[0].percent)
+	}
+
+	err = filters.Set("Header2:1")
+	if err == nil {
+		t.Error("Should error on Header2 because no % symbol")
+	}
+
+	err = filters.Set("Header2:10%")
+	if err != nil {
+		t.Error("Should pass")
+	}
+
+	if filters[1].percent != 10 {
+		t.Error("Wrong percentage", filters[1].percent)
+	}
+}
+
+func TestUrlRewriteMap(t *testing.T) {
+	var err error
+	rewrites := UrlRewriteMap{}
+
+	if err = rewrites.Set("/v1/user/([^\\/]+)/ping:/v2/user/$1/ping"); err != nil {
+		t.Error("Should set mapping", err)
+	}
+
+	if err = rewrites.Set("/v1/user/([^\\/]+)/ping"); err == nil {
+		t.Error("Should not set mapping without :")
+	}
+}

http_modifier_test.go

@@ -0,0 +1,247 @@
+package main
+
+import (
+	"bytes"
+	"github.com/buger/gor/proto"
+	"testing"
+)
+
+func TestHTTPModifierWithoutConfig(t *testing.T) {
+	if NewHTTPModifier(&HTTPModifierConfig{}) != nil {
+		t.Error("If no config specified should not be initialized")
+	}
+}
+
+func TestHTTPModifierHeaderFilters(t *testing.T) {
+	filters := HTTPHeaderFilters{}
+	filters.Set("Host:^www.w3.org$")
+
+	modifier := NewHTTPModifier(&HTTPModifierConfig{
+		headerFilters: filters,
+	})
+
+	payload := []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
+
+	if len(modifier.Rewrite(payload)) == 0 {
+		t.Error("Request should pass filters")
+	}
+
+	filters = HTTPHeaderFilters{}
+	// Setting filter that not match our header
+	filters.Set("Host:^www.w4.org$")
+
+	modifier = NewHTTPModifier(&HTTPModifierConfig{
+		headerFilters: filters,
+	})
+
+	if len(modifier.Rewrite(payload)) != 0 {
+		t.Error("Request should not pass filters")
+	}
+}
+
+func TestHTTPModifierHeaderNegativeFilters(t *testing.T) {
+	filters := HTTPHeaderFilters{}
+	filters.Set("Host:^www.w3.org$")
+
+	modifier := NewHTTPModifier(&HTTPModifierConfig{
+		headerNegativeFilters: filters,
+	})
+
+	payload := []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w4.org\r\n\r\na=1&b=2")
+
+	if len(modifier.Rewrite(payload)) == 0 {
+		t.Error("Request should pass filters")
+	}
+
+	filters = HTTPHeaderFilters{}
+	// Setting filter that not match our header
+	filters.Set("Host:^www.w4.org$")
+
+	modifier = NewHTTPModifier(&HTTPModifierConfig{
+		headerNegativeFilters: filters,
+	})
+
+	if len(modifier.Rewrite(payload)) != 0 {
+		t.Error("Request should not pass filters")
+	}
+}
+
+func TestHTTPModifierURLRewrite(t *testing.T) {
+	var url, newURL []byte
+
+	rewrites := UrlRewriteMap{}
+
+	payload := func(url []byte) []byte {
+		return []byte("POST " + string(url) + " HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
+	}
+
+	err := rewrites.Set("/v1/user/([^\\/]+)/ping:/v2/user/$1/ping")
+	if err != nil {
+		t.Error("Should not error on /v1/user/([^\\/]+)/ping:/v2/user/$1/ping")
+	}
+
+	modifier := NewHTTPModifier(&HTTPModifierConfig{
+		urlRewrite: rewrites,
+	})
+
+	url = []byte("/v1/user/joe/ping")
+	if newURL = proto.Path(modifier.Rewrite(payload(url))); bytes.Equal(newURL, url) {
+		t.Error("Request url should have been rewritten, wasn't", string(newURL))
+	}
+
+	url = []byte("/v1/user/ping")
+	if newURL = proto.Path(modifier.Rewrite(payload(url))); !bytes.Equal(newURL, url) {
+		t.Error("Request url should have been rewritten, wasn't", string(newURL))
+	}
+}
+
+func TestHTTPModifierHeaderHashFilters(t *testing.T) {
+	filters := HTTPHashFilters{}
+	filters.Set("Header2:1/2")
+
+	modifier := NewHTTPModifier(&HTTPModifierConfig{
+		headerHashFilters: filters,
+	})
+
+	payload := func(header []byte) []byte {
+		return []byte("POST / HTTP/1.1\r\n" + string(header) + "Content-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
+	}
+
+	if p := modifier.Rewrite(payload([]byte(""))); len(p) == 0 {
+		t.Error("Request should pass filters if Header does not exist")
+	}
+
+	if p := modifier.Rewrite(payload([]byte("Header2: 3\r\n"))); len(p) > 0 {
+		t.Error("Request should not pass filters, Header2 hash too high")
+	}
+
+	if p := modifier.Rewrite(payload([]byte("Header2: 1\r\n"))); len(p) == 0 {
+		t.Error("Request should pass filters")
+	}
+}
+
+func TestHTTPModifierParamHashFilters(t *testing.T) {
+	filters := HTTPHashFilters{}
+	filters.Set("user_id:1/2")
+
+	modifier := NewHTTPModifier(&HTTPModifierConfig{
+		paramHashFilters: filters,
+	})
+
+	payload := func(value []byte) []byte {
+		return []byte("POST /" + string(value) + " HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
+	}
+
+	if p := modifier.Rewrite(payload([]byte(""))); len(p) == 0 {
+		t.Error("Request should pass filters if param does not exist")
+	}
+
+	if p := modifier.Rewrite(payload([]byte("?user_id=3"))); len(p) > 0 {
+		t.Error("Request should not pass filters", string(p))
+	}
+
+	if p := modifier.Rewrite(payload([]byte("?user_id=1"))); len(p) == 0 {
+		t.Error("Request should pass filters")
+	}
+}
+
+func TestHTTPModifierHeaders(t *testing.T) {
+	headers := HTTPHeaders{}
+	headers.Set("Header1:1")
+	headers.Set("Host:localhost")
+
+	modifier := NewHTTPModifier(&HTTPModifierConfig{
+		headers: headers,
+	})
+
+	payload := []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
+	newPayload := []byte("POST /post HTTP/1.1\r\nHeader1: 1\r\nContent-Length: 7\r\nHost: localhost\r\n\r\na=1&b=2")
+
+	if payload = modifier.Rewrite(payload); !bytes.Equal(payload, newPayload) {
+		t.Error("Should update request headers", string(payload))
+	}
+}
+
+func TestHTTPModifierURLRegexp(t *testing.T) {
+	filters := HTTPUrlRegexp{}
+	filters.Set("/v1/app")
+	filters.Set("/v1/api")
+
+	modifier := NewHTTPModifier(&HTTPModifierConfig{
+		urlRegexp: filters,
+	})
+
+	payload := func(url string) []byte {
+		return []byte("POST " + url + " HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
+	}
+
+	if len(modifier.Rewrite(payload("/v1/app/test"))) == 0 {
+		t.Error("Should pass url")
+	}
+
+	if len(modifier.Rewrite(payload("/v1/api/test"))) == 0 {
+		t.Error("Should pass url")
+	}
+
+	if len(modifier.Rewrite(payload("/other"))) > 0 {
+		t.Error("Should not pass url")
+	}
+}
+
+func TestHTTPModifierURLNegativeRegexp(t *testing.T) {
+	filters := HTTPUrlRegexp{}
+	filters.Set("/restricted1")
+	filters.Set("/some/restricted2")
+
+	modifier := NewHTTPModifier(&HTTPModifierConfig{
+		urlNegativeRegexp: filters,
+	})
+
+	payload := func(url string) []byte {
+		return []byte("POST " + url + " HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
+	}
+
+	if len(modifier.Rewrite(payload("/v1/app/test"))) == 0 {
+		t.Error("Should pass url")
+	}
+
+	if len(modifier.Rewrite(payload("/restricted1"))) > 0 {
+		t.Error("Should not pass url")
+	}
+
+	if len(modifier.Rewrite(payload("/some/restricted2"))) > 0 {
+		t.Error("Should not pass url")
+	}
+}
+
+func TestHTTPModifierSetHeader(t *testing.T) {
+	filters := HTTPHeaders{}
+	filters.Set("User-Agent:Gor")
+
+	modifier := NewHTTPModifier(&HTTPModifierConfig{
+		headers: filters,
+	})
+
+	payload := []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
+	payloadAfter := []byte("POST /post HTTP/1.1\r\nUser-Agent: Gor\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
+
+	if payload = modifier.Rewrite(payload); !bytes.Equal(payloadAfter, payload) {
+		t.Error("Should add new header", string(payload))
+	}
+}
+
+func TestHTTPModifierSetParam(t *testing.T) {
+	filters := HTTPParams{}
+	filters.Set("api_key=1")
+
+	modifier := NewHTTPModifier(&HTTPModifierConfig{
+		params: filters,
+	})
+
+	payload := []byte("POST /post?api_key=1234 HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
+	payloadAfter := []byte("POST /post?api_key=1 HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
+
+	if payload = modifier.Rewrite(payload); !bytes.Equal(payloadAfter, payload) {
+		t.Error("Should override param", string(payload))
+	}
+}

input_dummy.go

@@ -4,10 +4,12 @@ import (
 	"time"
 )
 
+// DummyInput used for debugging. It generate 1 "GET /"" request per second.
 type DummyInput struct {
 	data chan []byte
 }
 
+// NewDummyInput constructor for DummyInput
 func NewDummyInput(options string) (di *DummyInput) {
 	di = new(DummyInput)
 	di.data = make(chan []byte)
@@ -19,6 +21,7 @@ func NewDummyInput(options string) (di *DummyInput) {
 
 func (i *DummyInput) Read(data []byte) (int, error) {
 	buf := <-i.data
+
 	copy(data, buf)
 
 	return len(buf), nil
@@ -30,7 +33,12 @@ func (i *DummyInput) emit() {
 	for {
 		select {
 		case <-ticker.C:
-			i.data <- []byte("GET / HTTP/1.1\r\n\r\n")
+			uuid := uuid()
+			reqh := payloadHeader(RequestPayload, uuid, time.Now().UnixNano())
+			i.data <- append(reqh, []byte("POST /pub/WWW/å HTTP/1.1\nHost: www.w3.org\r\nContent-Length: 7\r\n\r\na=1&b=2")...)
+
+			resh := payloadHeader(ResponsePayload, uuid, 1)
+			i.data <- append(resh, []byte("HTTP/1.1 200 OK\r\n\r\n")...)
 		}
 	}
 }

input_file.go

@@ -1,37 +1,42 @@
 package main
 
 import (
-	"encoding/gob"
+	"bufio"
 	"log"
 	"os"
+	"strconv"
 	"time"
 )
 
+// FileInput can read requests generated by FileOutput
 type FileInput struct {
-	data    chan []byte
-	path    string
-	decoder *gob.Decoder
+	data        chan []byte
+	path        string
+	file        *os.File
+	speedFactor float64
 }
 
+// NewFileInput constructor for FileInput. Accepts file path as argument.
 func NewFileInput(path string) (i *FileInput) {
 	i = new(FileInput)
 	i.data = make(chan []byte)
 	i.path = path
-	i.Init(path)
+	i.speedFactor = 1
+	i.init(path)
 
 	go i.emit()
 
 	return
 }
 
-func (i *FileInput) Init(path string) {
+func (i *FileInput) init(path string) {
 	file, err := os.Open(path)
 
 	if err != nil {
 		log.Fatal(i, "Cannot open file %q. Error: %s", path, err)
 	}
 
-	i.decoder = gob.NewDecoder(file)
+	i.file = file
 }
 
 func (i *FileInput) Read(data []byte) (int, error) {
@@ -48,19 +53,33 @@ func (i *FileInput) String() string {
 func (i *FileInput) emit() {
 	var lastTime int64
 
-	for {
-		raw := new(RawRequest)
-		err := i.decoder.Decode(raw)
+	// reader := bufio.NewReader(conn)
+	scanner := bufio.NewScanner(i.file)
+	scanner.Split(payloadScanner)
 
-		if err != nil {
-			return
+	for scanner.Scan() {
+		buf := scanner.Bytes()
+		meta := payloadMeta(buf)
+
+		if meta[0][0] == RequestPayload && lastTime != 0 {
+			ts, _ := strconv.ParseInt(string(meta[2]), 10, 64)
+			timeDiff := ts - lastTime
+
+			if i.speedFactor != 1 {
+				timeDiff = int64(float64(timeDiff) / i.speedFactor)
+			}
+
+			time.Sleep(time.Duration(timeDiff))
+
+			lastTime = ts
 		}
 
-		if lastTime != 0 {
-			time.Sleep(time.Duration(raw.Timestamp - lastTime))
-		}
-		lastTime = raw.Timestamp
+		// scanner returs only pointer, so to remove data-race we have to allocate new array
+		newBuf := make([]byte, len(buf))
+		copy(newBuf, buf)
 
-		i.data <- raw.Request
+		i.data <- newBuf
 	}
+
+	log.Println("FileInput: end of file")
 }

input_http.go

@@ -0,0 +1,70 @@
+package main
+
+import (
+	"log"
+	"net"
+	"net/http"
+	"net/http/httputil"
+	"time"
+)
+
+// HTTPInput used for sending requests to Gor via http
+type HTTPInput struct {
+	data     chan []byte
+	address  string
+	listener net.Listener
+}
+
+// NewHTTPInput constructor for HTTPInput. Accepts address with port which he will listen on.
+func NewHTTPInput(address string) (i *HTTPInput) {
+	i = new(HTTPInput)
+	i.data = make(chan []byte)
+	i.address = address
+
+	i.listen(address)
+
+	return
+}
+
+func (i *HTTPInput) Read(data []byte) (int, error) {
+	buf := <-i.data
+
+	header := payloadHeader(RequestPayload, uuid(), time.Now().UnixNano())
+
+	copy(data[0:len(header)], header)
+	copy(data[len(header):], buf)
+
+	return len(buf) + len(header), nil
+}
+
+func (i *HTTPInput) handler(w http.ResponseWriter, r *http.Request) {
+	buf, _ := httputil.DumpRequest(r, true)
+
+	i.data <- buf
+
+	http.Error(w, http.StatusText(200), 200)
+}
+
+func (i *HTTPInput) listen(address string) {
+	var err error
+
+	mux := http.NewServeMux()
+
+	mux.HandleFunc("/", i.handler)
+
+	i.listener, err = net.Listen("tcp", address)
+	if err != nil {
+		log.Fatal("HTTP input listener failure:", err)
+	}
+
+	go func() {
+		err = http.Serve(i.listener, mux)
+		if err != nil {
+			log.Fatal("HTTP input serve failure:", err)
+		}
+	}()
+}
+
+func (i *HTTPInput) String() string {
+	return "HTTP input: " + i.address
+}

input_http_test.go

@@ -0,0 +1,35 @@
+package main
+
+import (
+	"io"
+	"net/http"
+	"strings"
+	"sync"
+	"testing"
+)
+
+func TestHTTPInput(t *testing.T) {
+	wg := new(sync.WaitGroup)
+	quit := make(chan int)
+
+	input := NewHTTPInput(":0")
+	output := NewTestOutput(func(data []byte) {
+		wg.Done()
+	})
+
+	Plugins.Inputs = []io.Reader{input}
+	Plugins.Outputs = []io.Writer{output}
+
+	go Start(quit)
+
+	address := strings.Replace(input.listener.Addr().String(), "[::]", "127.0.0.1", -1)
+
+	for i := 0; i < 100; i++ {
+		wg.Add(1)
+		http.Get("http://" + address + "/")
+	}
+
+	wg.Wait()
+
+	close(quit)
+}

input_raw.go

@@ -5,17 +5,25 @@ import (
 	"log"
 	"net"
 	"strings"
+	"time"
 )
 
+// RAWInput used for intercepting traffic for given address
 type RAWInput struct {
-	data    chan []byte
-	address string
+	data     chan *raw.TCPMessage
+	address  string
+	expire   time.Duration
+	quit     chan bool
+	listener *raw.Listener
 }
 
-func NewRAWInput(address string) (i *RAWInput) {
+// NewRAWInput constructor for RAWInput. Accepts address with port as argument.
+func NewRAWInput(address string, expire time.Duration) (i *RAWInput) {
 	i = new(RAWInput)
-	i.data = make(chan []byte)
+	i.data = make(chan *raw.TCPMessage)
 	i.address = address
+	i.expire = expire
+	i.quit = make(chan bool)
 
 	go i.listen(address)
 
@@ -23,31 +31,55 @@ func NewRAWInput(address string) (i *RAWInput) {
 }
 
 func (i *RAWInput) Read(data []byte) (int, error) {
-	buf := <-i.data
-	copy(data, buf)
+	msg := <-i.data
+	buf := msg.Bytes()
 
-	return len(buf), nil
+	var header []byte
+
+	if msg.IsIncoming {
+		header = payloadHeader(RequestPayload, msg.UUID(), msg.Start)
+	} else {
+		header = payloadHeader(ResponsePayload, msg.UUID(), msg.End-msg.RequestStart)
+	}
+
+	copy(data[0:len(header)], header)
+	copy(data[len(header):], buf)
+
+	return len(buf) + len(header), nil
 }
 
 func (i *RAWInput) listen(address string) {
 	address = strings.Replace(address, "[::]", "127.0.0.1", -1)
 
+	Debug("Listening for traffic on: " + address)
+
 	host, port, err := net.SplitHostPort(address)
 
 	if err != nil {
 		log.Fatal("input-raw: error while parsing address", err)
 	}
 
-	listener := raw.NewListener(host, port)
+	i.listener = raw.NewListener(host, port, i.expire, true)
 
 	for {
-		// Receiving TCPMessage object
-		m := listener.Receive()
+		select {
+		case <-i.quit:
+			return
+		default:
+		}
 
-		i.data <- m.Bytes()
+		// Receiving TCPMessage object
+		m := i.listener.Receive()
+
+		i.data <- m
 	}
 }
 
 func (i *RAWInput) String() string {
 	return "RAW Socket input: " + i.address
 }
+
+func (i *RAWInput) Close() {
+	i.listener.Close()
+	close(i.quit)
+}

input_raw_test.go

@@ -1,39 +1,239 @@
 package main
 
 import (
+	"bytes"
 	"io"
+	"io/ioutil"
+	"log"
 	"net/http"
+	"net/http/httptest"
+	"net/http/httputil"
+	"os/exec"
 	"strings"
 	"sync"
+	"sync/atomic"
 	"testing"
+	"time"
 )
 
-func TestRAWInput(t *testing.T) {
+const testRawExpire = time.Millisecond * 200
 
+func TestRAWInput(t *testing.T) {
 	wg := new(sync.WaitGroup)
 	quit := make(chan int)
 
-	listener := startHTTP(func(req *http.Request) {})
+	origin := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {}))
+	defer origin.Close()
+	originAddr := strings.Replace(origin.Listener.Addr().String(), "[::]", "127.0.0.1", -1)
+
+	var respCounter, reqCounter int64
+
+	input := NewRAWInput(originAddr, testRawExpire)
+	defer input.Close()
 
-	input := NewRAWInput(listener.Addr().String())
 	output := NewTestOutput(func(data []byte) {
+		if data[0] == '1' {
+			atomic.AddInt64(&reqCounter, 1)
+		} else {
+			atomic.AddInt64(&respCounter, 1)
+		}
+
+		if Settings.debug {
+			log.Println(reqCounter, respCounter)
+		}
+
 		wg.Done()
 	})
 
 	Plugins.Inputs = []io.Reader{input}
 	Plugins.Outputs = []io.Writer{output}
 
-	address := strings.Replace(listener.Addr().String(), "[::]", "127.0.0.1", -1)
+	client := NewHTTPClient(origin.URL, &HTTPClientConfig{})
+
+	time.Sleep(time.Millisecond)
 
 	go Start(quit)
 
 	for i := 0; i < 100; i++ {
-		wg.Add(1)
-		res, _ := http.Get("http://" + address)
-		res.Body.Close()
+		// request + response
+		wg.Add(2)
+		client.Get("/")
+	}
+
+	wg.Wait()
+	close(quit)
+}
+
+func TestInputRAW100Expect(t *testing.T) {
+	wg := new(sync.WaitGroup)
+	quit := make(chan int)
+
+	fileContent, _ := ioutil.ReadFile("README.md")
+
+	// Origing and Replay server initialization
+	origin := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		defer r.Body.Close()
+		ioutil.ReadAll(r.Body)
+
+		wg.Done()
+	}))
+	defer origin.Close()
+
+	originAddr := strings.Replace(origin.Listener.Addr().String(), "[::]", "127.0.0.1", -1)
+
+	input := NewRAWInput(originAddr, time.Second)
+	defer input.Close()
+
+	// We will use it to get content of raw HTTP request
+	testOutput := NewTestOutput(func(data []byte) {
+		switch data[0] {
+		case RequestPayload:
+			if strings.Contains(string(data), "Expect: 100-continue") {
+				t.Error("Should not contain 100-continue header")
+			}
+			wg.Done()
+		case ResponsePayload:
+			wg.Done()
+		}
+	})
+
+	replay := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		defer r.Body.Close()
+		body, _ := ioutil.ReadAll(r.Body)
+
+		if !bytes.Equal(body, fileContent) {
+			buf, _ := httputil.DumpRequest(r, true)
+			t.Error("Wrong POST body:", string(buf))
+		}
+
+		wg.Done()
+	}))
+	defer replay.Close()
+
+	httpOutput := NewHTTPOutput(replay.URL, &HTTPOutputConfig{})
+
+	Plugins.Inputs = []io.Reader{input}
+	Plugins.Outputs = []io.Writer{testOutput, httpOutput}
+
+	go Start(quit)
+
+	// Origin + Response/Request Test Output + Request Http Output
+	wg.Add(4)
+	curl := exec.Command("curl", "http://"+originAddr, "--data-binary", "@README.md")
+	err := curl.Run()
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	wg.Wait()
+	close(quit)
+}
+
+func TestInputRAWChunkedEncoding(t *testing.T) {
+	wg := new(sync.WaitGroup)
+	quit := make(chan int)
+
+	fileContent, _ := ioutil.ReadFile("README.md")
+
+	// Origing and Replay server initialization
+	origin := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		defer r.Body.Close()
+		ioutil.ReadAll(r.Body)
+
+		wg.Done()
+	}))
+
+	originAddr := strings.Replace(origin.Listener.Addr().String(), "[::]", "127.0.0.1", -1)
+	input := NewRAWInput(originAddr, time.Second)
+	defer input.Close()
+
+	replay := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		defer r.Body.Close()
+		body, _ := ioutil.ReadAll(r.Body)
+
+		if !bytes.Equal(body, fileContent) {
+			buf, _ := httputil.DumpRequest(r, true)
+			t.Error("Wrong POST body:", string(buf))
+		}
+
+		wg.Done()
+	}))
+	defer replay.Close()
+
+	httpOutput := NewHTTPOutput(replay.URL, &HTTPOutputConfig{Debug: true})
+
+	Plugins.Inputs = []io.Reader{input}
+	Plugins.Outputs = []io.Writer{httpOutput}
+
+	go Start(quit)
+
+	wg.Add(2)
+
+	curl := exec.Command("curl", "http://"+originAddr, "--header", "Transfer-Encoding: chunked", "--data-binary", "@README.md")
+	err := curl.Run()
+	if err != nil {
+		log.Fatal(err)
 	}
 
 	wg.Wait()
 
 	close(quit)
 }
+
+func TestInputRAWLargePayload(t *testing.T) {
+	wg := new(sync.WaitGroup)
+	quit := make(chan int)
+
+	// Generate 200kb file
+	dd := exec.Command("dd", "if=/dev/urandom", "of=/tmp/large", "bs=1KB", "count=100")
+	err := dd.Run()
+	if err != nil {
+		log.Fatal("dd error:", err)
+	}
+
+	origin := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
+		defer req.Body.Close()
+		body, _ := ioutil.ReadAll(req.Body)
+
+		if len(body) != 100*1000 {
+			t.Error("File size should be 1mb:", len(body))
+		}
+
+		wg.Done()
+	}))
+	originAddr := strings.Replace(origin.Listener.Addr().String(), "[::]", "127.0.0.1", -1)
+
+	input := NewRAWInput(originAddr, time.Second)
+	defer input.Close()
+
+	replay := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
+		req.Body = http.MaxBytesReader(w, req.Body, 1*1024*1024)
+		buf := make([]byte, 1*1024*1024)
+		n, _ := req.Body.Read(buf)
+		body := buf[0:n]
+
+		if len(body) != 100*1000 {
+			t.Error("File size should be 100000 bytes:", len(body))
+		}
+
+		wg.Done()
+	}))
+	defer replay.Close()
+
+	httpOutput := NewHTTPOutput(replay.URL, &HTTPOutputConfig{Debug: false})
+
+	Plugins.Inputs = []io.Reader{input}
+	Plugins.Outputs = []io.Writer{httpOutput}
+
+	go Start(quit)
+
+	wg.Add(2)
+	curl := exec.Command("curl", "http://"+originAddr, "--header", "Transfer-Encoding: chunked", "--data-binary", "@/tmp/large")
+	err = curl.Run()
+	if err != nil {
+		log.Fatal("curl error:", err)
+	}
+
+	wg.Wait()
+	close(quit)
+}

input_tcp.go

@@ -2,20 +2,20 @@ package main
 
 import (
 	"bufio"
-	"io"
+	"fmt"
 	"log"
 	"net"
+	"os"
 )
 
-// Can be tested using nc tool:
-//    echo "asdad" | nc 127.0.0.1 27017
-//
+// TCPInput used for internal communication
 type TCPInput struct {
 	data     chan []byte
 	address  string
 	listener net.Listener
 }
 
+// NewTCPInput constructor for TCPInput, accepts address with port
 func NewTCPInput(address string) (i *TCPInput) {
 	i = new(TCPInput)
 	i.data = make(chan []byte)
@@ -59,23 +59,15 @@ func (i *TCPInput) handleConnection(conn net.Conn) {
 	defer conn.Close()
 
 	reader := bufio.NewReader(conn)
+	scanner := bufio.NewScanner(reader)
+	scanner.Split(payloadScanner)
 
-	for {
-		buf,err := reader.ReadBytes('¶')
-		buf_len := len(buf)
-		if buf_len > 0 {
-			new_buf_len := len(buf) - 2
-			if new_buf_len > 0 {
-				new_buf := make([]byte, new_buf_len)
-				copy(new_buf, buf[:new_buf_len])
-				i.data <- new_buf
-				if err != nil {
-					if err != io.EOF {
-						log.Printf("error: %s\n", err)
-					}
-				}
-			}
-		}
+	for scanner.Scan() {
+		i.data <- scanner.Bytes()
+	}
+
+	if err := scanner.Err(); err != nil {
+		fmt.Fprintln(os.Stderr, "Unexpected error in input tcp connection:", err)
 	}
 }
 

input_tcp_test.go

@@ -34,14 +34,12 @@ func TestTCPInput(t *testing.T) {
 		log.Fatal(err)
 	}
 
-	msg := []byte("GET / HTTP/1.1\r\n\r\n")
+	msg := []byte("1 1 1\nGET / HTTP/1.1\r\n\r\n")
 
 	for i := 0; i < 100; i++ {
 		wg.Add(1)
-		new_buf := make([]byte, len(msg) + 2)
-		msg = append(msg,[]byte("¶")...)
-		copy(new_buf, msg)
-		conn.Write(new_buf)
+		conn.Write(msg)
+		conn.Write([]byte(payloadSeparator))
 	}
 
 	wg.Wait()
@@ -82,10 +80,10 @@ func BenchmarkTCPInput(b *testing.B) {
 			for {
 				data := <-dataChan
 
-				new_buf := make([]byte, len(data) + 2)
-				data = append(data,[]byte("¶")...)
-				copy(new_buf, data)
-				conn.Write(new_buf)
+				buf := make([]byte, len(data)+2)
+				data = append(data, []byte("¶")...)
+				copy(buf, data)
+				conn.Write(buf)
 			}
 		}(conn)
 	}

limiter.go

@@ -3,43 +3,94 @@ package main
 import (
 	"fmt"
 	"io"
+	"math/rand"
+	"strconv"
+	"strings"
 	"time"
 )
 
+// Limiter is a wrapper for input or output plugin which adds rate limiting
 type Limiter struct {
-	writer io.Writer
-	limit  int
+	plugin    interface{}
+	limit     int
+	isPercent bool
 
 	currentRPS  int
 	currentTime int64
 }
 
-func NewLimiter(writer io.Writer, limit int) (l *Limiter) {
-	l = new(Limiter)
-	l.limit = limit
-	l.writer = writer
-	l.currentTime = time.Now().UnixNano()
+func parseLimitOptions(options string) (limit int, isPercent bool) {
+	if strings.Contains(options, "%") {
+		limit, _ = strconv.Atoi(strings.Split(options, "%")[0])
+		isPercent = true
+	} else {
+		limit, _ = strconv.Atoi(options)
+		isPercent = false
+	}
 
 	return
 }
 
-func (l *Limiter) Write(data []byte) (n int, err error) {
+// NewLimiter constructor for Limiter, accepts plugin and options
+// `options` allow to sprcify relatve or absolute limiting
+func NewLimiter(plugin interface{}, options string) io.ReadWriter {
+	l := new(Limiter)
+	l.limit, l.isPercent = parseLimitOptions(options)
+	l.plugin = plugin
+	l.currentTime = time.Now().UnixNano()
+
+	// FileInput have its own rate limiting. Unlike other inputs we not just dropping requests, we can slow down or speed up request emittion.
+	if fi, ok := l.plugin.(*FileInput); ok && l.isPercent {
+		fi.speedFactor = float64(l.limit) / float64(100)
+	}
+
+	return l
+}
+
+func (l *Limiter) isLimited() bool {
+	// File input have its own limiting algorithm
+	if _, ok := l.plugin.(*FileInput); ok && l.isPercent {
+		return false
+	}
+
+	if l.isPercent {
+		return l.limit <= rand.Intn(100)
+	}
+
 	if (time.Now().UnixNano() - l.currentTime) > time.Second.Nanoseconds() {
 		l.currentTime = time.Now().UnixNano()
 		l.currentRPS = 0
 	}
 
 	if l.currentRPS >= l.limit {
+		return true
+	}
+
+	l.currentRPS++
+
+	return false
+}
+
+func (l *Limiter) Write(data []byte) (n int, err error) {
+	if l.isLimited() {
 		return 0, nil
 	}
 
-	n, err = l.writer.Write(data)
+	n, err = l.plugin.(io.Writer).Write(data)
 
-	l.currentRPS++
+	return
+}
+
+func (l *Limiter) Read(data []byte) (n int, err error) {
+	n, err = l.plugin.(io.Reader).Read(data)
+
+	if l.isLimited() {
+		return 0, nil
+	}
 
 	return
 }
 
 func (l *Limiter) String() string {
-	return fmt.Sprintf("Limiting %s to: %d", l.writer, l.limit)
+	return fmt.Sprintf("Limiting %s to: %d (isPercent: %b)", l.plugin, l.limit, l.isPercent)
 }

limiter_test.go

@@ -1,3 +1,5 @@
+// +build !race
+
 package main
 
 import (
@@ -6,14 +8,14 @@ import (
 	"testing"
 )
 
-func TestLimiter(t *testing.T) {
+func TestOutputLimiter(t *testing.T) {
 	wg := new(sync.WaitGroup)
 	quit := make(chan int)
 
 	input := NewTestInput()
 	output := NewLimiter(NewTestOutput(func(data []byte) {
 		wg.Done()
-	}), 10)
+	}), "10")
 	wg.Add(10)
 
 	Plugins.Inputs = []io.Reader{input}
@@ -29,3 +31,76 @@ func TestLimiter(t *testing.T) {
 
 	close(quit)
 }
+
+func TestInputLimiter(t *testing.T) {
+	wg := new(sync.WaitGroup)
+	quit := make(chan int)
+
+	input := NewLimiter(NewTestInput(), "10")
+	output := NewTestOutput(func(data []byte) {
+		wg.Done()
+	})
+	wg.Add(10)
+
+	Plugins.Inputs = []io.Reader{input}
+	Plugins.Outputs = []io.Writer{output}
+
+	go Start(quit)
+
+	for i := 0; i < 100; i++ {
+		input.(*Limiter).plugin.(*TestInput).EmitGET()
+	}
+
+	wg.Wait()
+
+	close(quit)
+}
+
+// Should limit all requests
+func TestPercentLimiter1(t *testing.T) {
+	wg := new(sync.WaitGroup)
+	quit := make(chan int)
+
+	input := NewTestInput()
+	output := NewLimiter(NewTestOutput(func(data []byte) {
+		wg.Done()
+	}), "0%")
+
+	Plugins.Inputs = []io.Reader{input}
+	Plugins.Outputs = []io.Writer{output}
+
+	go Start(quit)
+
+	for i := 0; i < 100; i++ {
+		input.EmitGET()
+	}
+
+	wg.Wait()
+
+	close(quit)
+}
+
+// Should not limit at all
+func TestPercentLimiter2(t *testing.T) {
+	wg := new(sync.WaitGroup)
+	quit := make(chan int)
+
+	input := NewTestInput()
+	output := NewLimiter(NewTestOutput(func(data []byte) {
+		wg.Done()
+	}), "100%")
+	wg.Add(100)
+
+	Plugins.Inputs = []io.Reader{input}
+	Plugins.Outputs = []io.Writer{output}
+
+	go Start(quit)
+
+	for i := 0; i < 100; i++ {
+		input.EmitGET()
+	}
+
+	wg.Wait()
+
+	close(quit)
+}

middleware.go

@@ -0,0 +1,116 @@
+package main
+
+import (
+	"bufio"
+	"encoding/hex"
+	"fmt"
+	"io"
+	"log"
+	"os"
+	"os/exec"
+	"strings"
+	"sync"
+)
+
+type Middleware struct {
+	command string
+
+	data chan []byte
+
+	mu sync.Mutex
+
+	Stdin  io.Writer
+	Stdout io.Reader
+}
+
+func NewMiddleware(command string) *Middleware {
+	m := new(Middleware)
+	m.command = command
+	m.data = make(chan []byte, 1000)
+
+	commands := strings.Split(command, " ")
+	cmd := exec.Command(commands[0], commands[1:]...)
+
+	m.Stdout, _ = cmd.StdoutPipe()
+	m.Stdin, _ = cmd.StdinPipe()
+
+	if Settings.verbose {
+		cmd.Stderr = os.Stderr
+	}
+
+	go m.read(m.Stdout)
+
+	go func() {
+		err := cmd.Start()
+
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		cmd.Wait()
+	}()
+
+	return m
+}
+
+func (m *Middleware) ReadFrom(plugin io.Reader) {
+	Debug("[MIDDLEWARE-MASTER] Starting reading from", plugin)
+	go m.copy(m.Stdin, plugin)
+}
+
+func (m *Middleware) copy(to io.Writer, from io.Reader) {
+	buf := make([]byte, 5*1024*1024)
+	dst := make([]byte, len(buf)*2)
+
+	for {
+		nr, _ := from.Read(buf)
+		if nr > 0 && len(buf) > nr {
+
+			hex.Encode(dst, buf[0:nr])
+			dst[nr*2] = '\n'
+
+			m.mu.Lock()
+			to.Write(dst[0 : nr*2+1])
+			m.mu.Unlock()
+
+			if Settings.debug {
+				Debug("[MIDDLEWARE-MASTER] Sending:", string(buf[0:nr]), "From:", from)
+			}
+		}
+	}
+}
+
+func (m *Middleware) read(from io.Reader) {
+	scanner := bufio.NewScanner(from)
+
+	for scanner.Scan() {
+		bytes := scanner.Bytes()
+		buf := make([]byte, len(bytes)/2)
+		if _, err := hex.Decode(buf, bytes); err != nil {
+			fmt.Fprintln(os.Stderr, "Failed to decode input payload", err, len(bytes))
+		}
+
+		if Settings.debug {
+			Debug("[MIDDLEWARE-MASTER] Received:", string(buf))
+		}
+
+		m.data <- buf
+	}
+
+	if err := scanner.Err(); err != nil {
+		fmt.Fprintln(os.Stderr, "Traffic modifier command failed:", err)
+	}
+
+	return
+}
+
+func (m *Middleware) Read(data []byte) (int, error) {
+	buf := <-m.data
+	copy(data, buf)
+
+	return len(buf), nil
+}
+
+func (m *Middleware) String() string {
+	return fmt.Sprintf("Modifying traffic using '%s' command", m.command)
+}

middleware_test.go

@@ -0,0 +1,215 @@
+package main
+
+import (
+	"bytes"
+	"crypto/rand"
+	"encoding/hex"
+	"github.com/buger/gor/proto"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+)
+
+type fakeServiceCb func(string, int, []byte)
+
+// Simple service that generate token on request, and require this token for accesing to secure area
+func NewFakeSecureService(wg *sync.WaitGroup, cb fakeServiceCb) *httptest.Server {
+	active_tokens := make([]string, 0)
+	var mu sync.Mutex
+
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
+		mu.Lock()
+		defer mu.Unlock()
+		Debug("Received request: " + req.URL.String())
+
+		switch req.URL.Path {
+		case "/token":
+			// Generate random token
+			token_length := 10
+			buf := make([]byte, token_length)
+			rand.Read(buf)
+			token := hex.EncodeToString(buf)
+			active_tokens = append(active_tokens, token)
+
+			w.Write([]byte(token))
+
+			cb(req.URL.Path, 200, []byte(token))
+		case "/secure":
+			token := req.URL.Query().Get("token")
+			token_found := false
+
+			for _, t := range active_tokens {
+				if t == token {
+					token_found = true
+					break
+				}
+			}
+
+			if token_found {
+				w.WriteHeader(http.StatusAccepted)
+				cb(req.URL.Path, 202, []byte(nil))
+			} else {
+				w.WriteHeader(http.StatusForbidden)
+				cb(req.URL.Path, 403, []byte(nil))
+			}
+		}
+
+		wg.Done()
+	}))
+
+	return server
+}
+
+func TestFakeSecureService(t *testing.T) {
+	var resp, token []byte
+
+	wg := new(sync.WaitGroup)
+
+	server := NewFakeSecureService(wg, func(path string, status int, resp []byte) {
+	})
+	defer server.Close()
+
+	wg.Add(3)
+
+	client := NewHTTPClient(server.URL, &HTTPClientConfig{Debug: true})
+	resp, _ = client.Get("/token")
+	token = proto.Body(resp)
+
+	// Right token
+	resp, _ = client.Get("/secure?token=" + string(token))
+	if !bytes.Equal(proto.Status(resp), []byte("202")) {
+		t.Error("Valid token should return status 202:", string(proto.Status(resp)))
+	}
+
+	// Wrong tokens forbidden
+	resp, _ = client.Get("/secure?token=wrong")
+	if !bytes.Equal(proto.Status(resp), []byte("403")) {
+		t.Error("Wrong token should returns status 403:", string(proto.Status(resp)))
+	}
+
+	wg.Wait()
+}
+
+func TestEchoMiddleware(t *testing.T) {
+	wg := new(sync.WaitGroup)
+
+	from := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Env", "prod")
+		w.Header().Set("RequestPath", r.URL.Path)
+		wg.Done()
+	}))
+	defer from.Close()
+
+	to := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Env", "test")
+		w.Header().Set("RequestPath", r.URL.Path)
+		wg.Done()
+	}))
+	defer to.Close()
+
+	quit := make(chan int)
+
+	Settings.middleware = "./examples/middleware/echo.sh"
+
+	// Catch traffic from one service
+	input := NewRAWInput(from.Listener.Addr().String(), testRawExpire)
+	defer input.Close()
+
+	// And redirect to another
+	output := NewHTTPOutput(to.URL, &HTTPOutputConfig{Debug: false})
+
+	Plugins.Inputs = []io.Reader{input}
+	Plugins.Outputs = []io.Writer{output}
+
+	// Start Gor
+	go Start(quit)
+
+	time.Sleep(time.Millisecond)
+
+	// Should receive 2 requests from original + 2 from replayed
+	wg.Add(4)
+
+	client := NewHTTPClient(from.URL, &HTTPClientConfig{Debug: false})
+
+	// Request should be echoed
+	client.Get("/a")
+	client.Get("/b")
+
+	wg.Wait()
+	close(quit)
+	time.Sleep(100 * time.Millisecond)
+
+	Settings.middleware = ""
+}
+
+func TestTokenMiddleware(t *testing.T) {
+	var resp, token []byte
+
+	wg := new(sync.WaitGroup)
+
+	from := NewFakeSecureService(wg, func(path string, status int, tok []byte) {
+		time.Sleep(10 * time.Millisecond)
+	})
+	defer from.Close()
+
+	to := NewFakeSecureService(wg, func(path string, status int, tok []byte) {
+		switch path {
+		case "/secure":
+			if status != 202 {
+				t.Error("Server should receive valid rewritten token")
+			}
+		}
+
+		time.Sleep(10 * time.Millisecond)
+	})
+	defer to.Close()
+
+	quit := make(chan int)
+
+	Settings.middleware = "go run ./examples/middleware/token_modifier.go"
+
+	fromAddr := strings.Replace(from.Listener.Addr().String(), "[::]", "127.0.0.1", -1)
+	// Catch traffic from one service
+	input := NewRAWInput(fromAddr, testRawExpire)
+	defer input.Close()
+
+	// And redirect to another
+	output := NewHTTPOutput(to.URL, &HTTPOutputConfig{Debug: true})
+
+	Plugins.Inputs = []io.Reader{input}
+	Plugins.Outputs = []io.Writer{output}
+
+	// Start Gor
+	go Start(quit)
+
+	// Wait for middleware to initialize
+	// Give go compiller time to build programm
+	time.Sleep(500 * time.Millisecond)
+
+	// Should receive 2 requests from original + 2 from replayed
+	wg.Add(4)
+
+	client := NewHTTPClient(from.URL, &HTTPClientConfig{Debug: false})
+
+	// Sending traffic to original service
+	resp, _ = client.Get("/token")
+	token = proto.Body(resp)
+
+	// When delay is too smal, middleware does not always rewrite requests in time
+	// Hopefuly client will have delay more then 10ms :)
+	time.Sleep(10 * time.Millisecond)
+
+	resp, _ = client.Get("/secure?token=" + string(token))
+	if !bytes.Equal(proto.Status(resp), []byte("202")) {
+		t.Error("Valid token should return 202:", proto.Status(resp))
+	}
+
+	wg.Wait()
+	close(quit)
+	time.Sleep(100 * time.Millisecond)
+	Settings.middleware = ""
+}

output_dummy.go

@@ -4,9 +4,11 @@ import (
 	"fmt"
 )
 
+// DummyOutput used for debugging, prints all incoming requests
 type DummyOutput struct {
 }
 
+// NewDummyOutput constructor for DummyOutput
 func NewDummyOutput(options string) (di *DummyOutput) {
 	di = new(DummyOutput)
 
@@ -14,7 +16,7 @@ func NewDummyOutput(options string) (di *DummyOutput) {
 }
 
 func (i *DummyOutput) Write(data []byte) (int, error) {
-	fmt.Println("Writing message: ", data)
+	fmt.Println("Writing message: ", string(data))
 
 	return len(data), nil
 }

output_file.go

@@ -1,32 +1,27 @@
 package main
 
 import (
-	"encoding/gob"
+	"io"
 	"log"
 	"os"
-	"time"
 )
 
-type RawRequest struct {
-	Timestamp int64
-	Request   []byte
-}
-
+// FileOutput output plugin
 type FileOutput struct {
-	path    string
-	encoder *gob.Encoder
-	file    *os.File
+	path string
+	file *os.File
 }
 
-func NewFileOutput(path string) (o *FileOutput) {
-	o = new(FileOutput)
+// NewFileOutput constructor for FileOutput, accepts path
+func NewFileOutput(path string) io.Writer {
+	o := new(FileOutput)
 	o.path = path
-	o.Init(path)
+	o.init(path)
 
-	return
+	return o
 }
 
-func (o *FileOutput) Init(path string) {
+func (o *FileOutput) init(path string) {
 	var err error
 
 	o.file, err = os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0660)
@@ -34,14 +29,15 @@ func (o *FileOutput) Init(path string) {
 	if err != nil {
 		log.Fatal(o, "Cannot open file %q. Error: %s", path, err)
 	}
-
-	o.encoder = gob.NewEncoder(o.file)
 }
 
 func (o *FileOutput) Write(data []byte) (n int, err error) {
-	raw := RawRequest{time.Now().UnixNano(), data}
+	if !isOriginPayload(data) {
+		return len(data), nil
+	}
 
-	o.encoder.Encode(raw)
+	o.file.Write(data)
+	o.file.Write([]byte(payloadSeparator))
 
 	return len(data), nil
 }

output_http.go

@@ -1,210 +1,207 @@
 package main
 
 import (
-	"bufio"
-	"bytes"	
+	"github.com/buger/gor/proto"
 	"io"
 	"log"
-	"net/http"
-	"net/url"
-	"strconv"
-	"strings"
+	"sync/atomic"
 	"time"
 )
 
-type RedirectNotAllowed struct{}
+const initialDynamicWorkers = 10
 
-func (e *RedirectNotAllowed) Error() string {
-	return "Redirects not allowed"
+type response struct {
+	payload       []byte
+	uuid          []byte
+	roundTripTime int64
 }
 
-// customCheckRedirect disables redirects https://github.com/buger/gor/pull/15
-func customCheckRedirect(req *http.Request, via []*http.Request) error {
-	if len(via) >= 0 {
-		return new(RedirectNotAllowed)
-	}
-	return nil
-}
+// HTTPOutputConfig struct for holding http output configuration
+type HTTPOutputConfig struct {
+	redirectLimit int
 
-// ParseRequest in []byte returns a http request or an error
-func ParseRequest(data []byte) (request *http.Request, err error) {
-	buf := bytes.NewBuffer(data)
-	reader := bufio.NewReader(buf)
+	stats   bool
+	workers int
 
-	request, err = http.ReadRequest(reader)
+	elasticSearch string
 
-	return
+	Timeout      time.Duration
+	OriginalHost bool
+
+	Debug bool
+
+	TrackResponses bool
 }
 
+// HTTPOutput plugin manage pool of workers which send request to replayed server
+// By default workers pool is dynamic and starts with 10 workers
+// You can specify fixed number of workers using `--output-http-workers`
 type HTTPOutput struct {
+	// Keep this as first element of struct because it guarantees 64bit
+	// alignment. atomic.* functions crash on 32bit machines if operand is not
+	// aligned at 64bit. See https://github.com/golang/go/issues/599
+	activeWorkers int64
+
 	address string
 	limit   int
-	buf     chan []byte
+	queue   chan []byte
+
+	responses chan response
+
 	needWorker chan int
 
-	urlRegexp             HTTPUrlRegexp
-	headerFilters         HTTPHeaderFilters
-	headerHashFilters     HTTPHeaderHashFilters
-    outputHTTPUrlRewrite  UrlRewriteMap
+	config *HTTPOutputConfig
 
-	headers HTTPHeaders
-	methods HTTPMethods
+	queueStats *GorStat
 
 	elasticSearch *ESPlugin
-
-	bufStats *GorStat
 }
 
-func NewHTTPOutput(options string, headers HTTPHeaders, methods HTTPMethods, urlRegexp HTTPUrlRegexp, headerFilters HTTPHeaderFilters, headerHashFilters HTTPHeaderHashFilters, elasticSearchAddr string, outputHTTPUrlRewrite UrlRewriteMap) io.Writer {
-
+// NewHTTPOutput constructor for HTTPOutput
+// Initialize workers
+func NewHTTPOutput(address string, config *HTTPOutputConfig) io.Writer {
 	o := new(HTTPOutput)
 
-	optionsArr := strings.Split(options, "|")
-	address := optionsArr[0]
-
-	if !strings.HasPrefix(address, "http") {
-		address = "http://" + address
-	}
-
 	o.address = address
-	o.headers = headers
-	o.methods = methods
+	o.config = config
 
-	o.urlRegexp = urlRegexp
-	o.headerFilters = headerFilters
-	o.headerHashFilters = headerHashFilters
-	o.outputHTTPUrlRewrite = outputHTTPUrlRewrite
-
-	o.buf = make(chan []byte, 100)
-	if Settings.outputHTTPStats {
-		o.bufStats = NewGorStat("output_http")
-	}
-	if Settings.outputHTTPWorkers == -1 {
-		o.needWorker = make(chan int)
+	if o.config.stats {
+		o.queueStats = NewGorStat("output_http")
 	}
 
-	if elasticSearchAddr != "" {
-		o.elasticSearch = new(ESPlugin)
-		o.elasticSearch.Init(elasticSearchAddr)
-	}
+	o.queue = make(chan []byte, 100)
+	o.responses = make(chan response, 100)
+	o.needWorker = make(chan int, 1)
 
-	if len(optionsArr) > 1 {
-		o.limit, _ = strconv.Atoi(optionsArr[1])
-	}
-
-	go o.WorkerMaster(Settings.outputHTTPWorkers)
-
-	if o.limit > 0 {
-		return NewLimiter(o, o.limit)
+	// Initial workers count
+	if o.config.workers == 0 {
+		o.needWorker <- initialDynamicWorkers
 	} else {
-		return o
+		o.needWorker <- o.config.workers
 	}
+
+	if o.config.elasticSearch != "" {
+		o.elasticSearch = new(ESPlugin)
+		o.elasticSearch.Init(o.config.elasticSearch)
+	}
+
+	if len(Settings.middleware) > 0 {
+		o.config.TrackResponses = true
+	}
+
+	go o.workerMaster()
+
+	return o
 }
 
-func (o *HTTPOutput) WorkerMaster(n int) {
-	for i := 0; i < n; i++ {
-		go o.Worker()
-	}
+func (o *HTTPOutput) workerMaster() {
+	for {
+		newWorkers := <-o.needWorker
+		for i := 0; i < newWorkers; i++ {
+			go o.startWorker()
+		}
 
-	if Settings.outputHTTPWorkers == -1 {
-		for {
-			new_workers := <-o.needWorker
-			for i := 0; i < new_workers; i++ {
-				go o.Worker()
-			}
+		// Disable dynamic scaling if workers poll fixed size
+		if o.config.workers != 0 {
+			return
 		}
 	}
 }
 
-func (o *HTTPOutput) Worker() {
-	client := &http.Client{
-		CheckRedirect: customCheckRedirect,
-	}
-	death_count := 0
-	Loop:
-		for {
-			select {
-				case data := <-o.buf:
-				o.sendRequest(client, data)
-				death_count = 0
-			default:
-				if Settings.outputHTTPWorkers == -1 {
-					death_count += 1
-				}
-				if death_count > 20 {
-					break Loop
-				} else {
-					time.Sleep(time.Millisecond * 100)
-				}
+func (o *HTTPOutput) startWorker() {
+	client := NewHTTPClient(o.address, &HTTPClientConfig{
+		FollowRedirects: o.config.redirectLimit,
+		Debug:           o.config.Debug,
+		OriginalHost:    o.config.OriginalHost,
+		Timeout:         o.config.Timeout,
+	})
 
+	deathCount := 0
+
+	atomic.AddInt64(&o.activeWorkers, 1)
+
+	for {
+		select {
+		case data := <-o.queue:
+			o.sendRequest(client, data)
+			deathCount = 0
+		case <-time.After(time.Millisecond * 100):
+			// When dynamic scaling enabled workers die after 2s of inactivity
+			if o.config.workers == 0 {
+				deathCount++
+			} else {
+				continue
+			}
+
+			if deathCount > 20 {
+				workersCount := atomic.LoadInt64(&o.activeWorkers)
+
+				// At least 1 startWorker should be alive
+				if workersCount != 1 {
+					atomic.AddInt64(&o.activeWorkers, -1)
+					return
+				}
 			}
 		}
+	}
 }
 
 func (o *HTTPOutput) Write(data []byte) (n int, err error) {
+	if !isRequestPayload(data) {
+		return len(data), nil
+	}
+
 	buf := make([]byte, len(data))
 	copy(buf, data)
 
-	o.buf <- buf
-	buf_len := len(o.buf)
-	if Settings.outputHTTPStats {
-		o.bufStats.Write(len(o.buf))
+	o.queue <- buf
+
+	if o.config.stats {
+		o.queueStats.Write(len(o.queue))
 	}
 
-	if Settings.outputHTTPWorkers == -1 {
-		if buf_len > 10 {
-			if len(o.needWorker) == 0 {
-				o.needWorker <- buf_len
-			}
+	if o.config.workers == 0 {
+		workersCount := atomic.LoadInt64(&o.activeWorkers)
+
+		if len(o.queue) > int(workersCount) {
+			o.needWorker <- len(o.queue)
 		}
 	}
+
 	return len(data), nil
 }
 
-func (o *HTTPOutput) sendRequest(client *http.Client, data []byte) {
-	request, err := ParseRequest(data)
+func (o *HTTPOutput) Read(data []byte) (int, error) {
+	resp := <-o.responses
 
-	if err != nil {
-		log.Println("Cannot parse request", string(data), err)
+	Debug("[OUTPUT-HTTP] Received response:", string(resp.payload))
+
+	header := payloadHeader(ReplayedResponsePayload, resp.uuid, resp.roundTripTime)
+	copy(data[0:len(header)], header)
+	copy(data[len(header):], resp.payload)
+
+	return len(resp.payload) + len(header), nil
+}
+
+func (o *HTTPOutput) sendRequest(client *HTTPClient, request []byte) {
+	meta := payloadMeta(request)
+	uuid := meta[1]
+
+	body := payloadBody(request)
+	if !proto.IsHTTPPayload(body) {
 		return
 	}
 
-	if len(o.methods) > 0 && !o.methods.Contains(request.Method) {
-		return
-	}
-
-	if !(o.urlRegexp.Good(request) && o.headerFilters.Good(request) && o.headerHashFilters.Good(request)) {
-		return
-	}
-
-    // Rewrite the path as necessary
-    request.URL.Path = o.outputHTTPUrlRewrite.Rewrite(request.URL.Path)
-
-	// Change HOST of original request
-	URL := o.address + request.URL.Path + "?" + request.URL.RawQuery
-
-	request.RequestURI = ""
-	request.URL, _ = url.ParseRequestURI(URL)
-
-	for _, header := range o.headers {
-		SetHeader(request, header.Name, header.Value)
-	}
-
 	start := time.Now()
-	resp, err := client.Do(request)
+	resp, err := client.Send(body)
 	stop := time.Now()
 
-	// We should not count Redirect as errors
-	if urlErr, ok := err.(*url.Error); ok {
-		if _, ok := urlErr.Err.(*RedirectNotAllowed); ok {
-			err = nil
-		}
+	if err != nil {
+		log.Println("Request error:", err)
 	}
 
-	if err == nil {
-		defer resp.Body.Close()
-	} else {
-		log.Println("Request error:", err)
+	if o.config.TrackResponses {
+		o.responses <- response{resp, uuid, stop.UnixNano() - start.UnixNano()}
 	}
 
 	if o.elasticSearch != nil {
@@ -212,20 +209,6 @@ func (o *HTTPOutput) sendRequest(client *http.Client, data []byte) {
 	}
 }
 
-func SetHeader(request *http.Request, name string, value string) {
-
-	// Need to check here for the Host header as it needs to be set on the request and not as a separate header
-	// http.ReadRequest sets it by default to the URL Host of the request being read
-	if name == "Host" {
-		request.Host = value
-	} else {
-		request.Header.Set(name, value)
-	}
-
-	return
-
-}
-
 func (o *HTTPOutput) String() string {
 	return "HTTP output: " + o.address
 }

output_http_test.go

@@ -2,56 +2,22 @@ package main
 
 import (
 	"io"
-	"net"
+	"io/ioutil"
 	"net/http"
+	"net/http/httptest"
+	_ "net/http/httputil"
 	"sync"
 	"testing"
 	"time"
 )
 
-func startHTTP(cb func(*http.Request)) net.Listener {
-	handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		go cb(r)
-	})
-
-	listener, _ := net.Listen("tcp", ":0")
-
-	go http.Serve(listener, handler)
-
-	return listener
-}
-
-func TestSetHeader(t *testing.T) {
-
-	req := &http.Request{
-		Header: make(map[string][]string),
-	}
-	req.Host = "test.com"
-
-	SetHeader(req, "Host", "test2.com")
-
-	if req.Host != "test2.com" {
-		t.Error("Expected test2.com - got ", req.Host)
-	}
-
-	SetHeader(req, "test_header", "test_value")
-
-	if req.Header.Get("test_header") != "test_value" {
-		t.Error("Wrong header value found")
-	}
-
-}
-
 func TestHTTPOutput(t *testing.T) {
 	wg := new(sync.WaitGroup)
 	quit := make(chan int)
 
 	input := NewTestInput()
 
-	headers := HTTPHeaders{HTTPHeader{"User-Agent", "Gor"}}
-	methods := HTTPMethods{"GET", "PUT", "POST"}
-
-	listener := startHTTP(func(req *http.Request) {
+	server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
 		if req.Header.Get("User-Agent") != "Gor" {
 			t.Error("Wrong header")
 		}
@@ -60,10 +26,24 @@ func TestHTTPOutput(t *testing.T) {
 			t.Error("Wrong method")
 		}
 
-		wg.Done()
-	})
+		if req.Method == "POST" {
+			defer req.Body.Close()
+			body, _ := ioutil.ReadAll(req.Body)
 
-	output := NewHTTPOutput(listener.Addr().String(), headers, methods, HTTPUrlRegexp{}, HTTPHeaderFilters{}, HTTPHeaderHashFilters{}, "", UrlRewriteMap{})
+			if string(body) != "a=1&b=2" {
+				t.Error("Wrong POST body:", string(body))
+			}
+		}
+
+		wg.Done()
+	}))
+	defer server.Close()
+
+	headers := HTTPHeaders{HTTPHeader{"User-Agent", "Gor"}}
+	methods := HTTPMethods{[]byte("GET"), []byte("PUT"), []byte("POST")}
+	Settings.modifierConfig = HTTPModifierConfig{headers: headers, methods: methods}
+
+	output := NewHTTPOutput(server.URL, &HTTPOutputConfig{Debug: true})
 
 	Plugins.Inputs = []io.Reader{input}
 	Plugins.Outputs = []io.Writer{output}
@@ -71,7 +51,7 @@ func TestHTTPOutput(t *testing.T) {
 	go Start(quit)
 
 	for i := 0; i < 100; i++ {
-		wg.Add(2)
+		wg.Add(2) // OPTIONS should be ignored
 		input.EmitPOST()
 		input.EmitOPTIONS()
 		input.EmitGET()
@@ -80,23 +60,83 @@ func TestHTTPOutput(t *testing.T) {
 	wg.Wait()
 
 	close(quit)
+
+	Settings.modifierConfig = HTTPModifierConfig{}
+}
+
+func TestHTTPOutputKeepOriginalHost(t *testing.T) {
+	wg := new(sync.WaitGroup)
+	quit := make(chan int)
+
+	input := NewTestInput()
+
+	server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
+		if req.Host != "custom-host.com" {
+			t.Error("Wrong header", req.Host)
+		}
+
+		wg.Done()
+	}))
+	defer server.Close()
+
+	headers := HTTPHeaders{HTTPHeader{"Host", "custom-host.com"}}
+	Settings.modifierConfig = HTTPModifierConfig{headers: headers}
+
+	output := NewHTTPOutput(server.URL, &HTTPOutputConfig{Debug: false, OriginalHost: true})
+
+	Plugins.Inputs = []io.Reader{input}
+	Plugins.Outputs = []io.Writer{output}
+
+	go Start(quit)
+
+	wg.Add(1)
+	input.EmitGET()
+
+	wg.Wait()
+
+	close(quit)
+
+	Settings.modifierConfig = HTTPModifierConfig{}
+}
+
+func TestOutputHTTPSSL(t *testing.T) {
+	wg := new(sync.WaitGroup)
+	quit := make(chan int)
+
+	// Origing and Replay server initialization
+	server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		wg.Done()
+	}))
+
+	input := NewTestInput()
+	output := NewHTTPOutput(server.URL, &HTTPOutputConfig{})
+
+	Plugins.Inputs = []io.Reader{input}
+	Plugins.Outputs = []io.Writer{output}
+
+	go Start(quit)
+
+	wg.Add(2)
+
+	input.EmitPOST()
+	input.EmitGET()
+
+	wg.Wait()
+	close(quit)
 }
 
 func BenchmarkHTTPOutput(b *testing.B) {
 	wg := new(sync.WaitGroup)
 	quit := make(chan int)
 
-	input := NewTestInput()
-
-	headers := HTTPHeaders{HTTPHeader{"User-Agent", "Gor"}}
-	methods := HTTPMethods{"GET", "PUT", "POST"}
-
-	listener := startHTTP(func(req *http.Request) {
+	server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		time.Sleep(50 * time.Millisecond)
 		wg.Done()
-	})
+	}))
+	defer server.Close()
 
-	output := NewHTTPOutput(listener.Addr().String(), headers, methods, HTTPUrlRegexp{}, HTTPHeaderFilters{}, HTTPHeaderHashFilters{}, "",UrlRewriteMap{})
+	input := NewTestInput()
+	output := NewHTTPOutput(server.URL, &HTTPOutputConfig{})
 
 	Plugins.Inputs = []io.Reader{input}
 	Plugins.Outputs = []io.Writer{output}

output_tcp.go

@@ -5,57 +5,69 @@ import (
 	"io"
 	"log"
 	"net"
-	"strconv"
-	"strings"
+	"time"
 )
 
+// TCPOutput used for sending raw tcp payloads
+// Currently used for internal communication between listener and replay server
+// Can be used for transfering binary payloads like protocol buffers
 type TCPOutput struct {
-	address string
-	limit   int
-	buf     chan []byte
+	address  string
+	limit    int
+	buf      chan []byte
 	bufStats *GorStat
 }
 
-func NewTCPOutput(options string) io.Writer {
+// NewTCPOutput constructor for TCPOutput
+// Initialize 10 workers which hold keep-alive connection
+func NewTCPOutput(address string) io.Writer {
 	o := new(TCPOutput)
 
-	optionsArr := strings.Split(options, "|")
-	o.address = optionsArr[0]
+	o.address = address
 
 	o.buf = make(chan []byte, 100)
 	if Settings.outputTCPStats {
 		o.bufStats = NewGorStat("output_tcp")
 	}
 
-	if len(optionsArr) > 1 {
-		o.limit, _ = strconv.Atoi(optionsArr[1])
-	}
-
 	for i := 0; i < 10; i++ {
 		go o.worker()
 	}
 
-	if o.limit > 0 {
-		return NewLimiter(o, o.limit)
-	} else {
-		return o
-	}
+	return o
 }
 
 func (o *TCPOutput) worker() {
-	conn, _ := o.connect(o.address)
+	conn, err := o.connect(o.address)
+	for ; err != nil; conn, err = o.connect(o.address) {
+		time.Sleep(2 * time.Second)
+	}
+
 	defer conn.Close()
 
 	for {
 		conn.Write(<-o.buf)
+		_, err := conn.Write([]byte(payloadSeparator))
+
+		if err != nil {
+			log.Println("Worker failed on write, exitings and starting new worker")
+			go o.worker()
+			break
+		}
 	}
 }
 
 func (o *TCPOutput) Write(data []byte) (n int, err error) {
-	new_buf := make([]byte, len(data) + 2)
-	data = append(data,[]byte("¶")...)
-	copy(new_buf, data)
-	o.buf <- new_buf
+	if !isOriginPayload(data) {
+		return len(data), nil
+	}
+
+	// We have to copy, because sending data in multiple threads
+	newBuf := make([]byte, len(data))
+	copy(newBuf, data)
+
+	o.buf <- newBuf
+
 	if Settings.outputTCPStats {
 		o.bufStats.Write(len(o.buf))
 	}

output_tcp_test.go

@@ -44,22 +44,16 @@ func startTCP(cb func([]byte)) net.Listener {
 	go func() {
 		for {
 			conn, _ := listener.Accept()
-			
+			defer conn.Close()
+
 			go func() {
 				reader := bufio.NewReader(conn)
-				for {
-					buf,err := reader.ReadBytes('¶')
-					new_buf_len := len(buf) - 2
-					new_buf := make([]byte, new_buf_len)
-					copy(new_buf, buf[:new_buf_len])
-					if err != nil {
-						if err != io.EOF {
-							log.Printf("error: %s\n", err)
-						}
-					}
-					cb(new_buf)
+				scanner := bufio.NewScanner(reader)
+				scanner.Split(payloadScanner)
+
+				for scanner.Scan() {
+					cb(scanner.Bytes())
 				}
-				conn.Close()
 			}()
 		}
 	}()

plugins.go

@@ -2,45 +2,117 @@ package main
 
 import (
 	"io"
+	"reflect"
+	"strings"
+	"time"
 )
 
+// InOutPlugins struct for holding references to plugins
 type InOutPlugins struct {
 	Inputs  []io.Reader
 	Outputs []io.Writer
 }
 
+// Plugins holds all the plugin objects
 var Plugins *InOutPlugins = new(InOutPlugins)
 
+// extractLimitOptions detects if plugin get called with limiter support
+// Returns address and limit
+func extractLimitOptions(options string) (string, string) {
+	split := strings.Split(options, "|")
+
+	if len(split) > 1 {
+		return split[0], split[1]
+	}
+
+	return split[0], ""
+}
+
+// Automatically detects type of plugin and initialize it
+//
+// See this article if curious about relfect stuff below: http://blog.burntsushi.net/type-parametric-functions-golang
+func registerPlugin(constructor interface{}, options ...interface{}) {
+	vc := reflect.ValueOf(constructor)
+
+	// Pre-processing options to make it work with reflect
+	vo := []reflect.Value{}
+	for _, oi := range options {
+		vo = append(vo, reflect.ValueOf(oi))
+	}
+
+	// Removing limit options from path
+	path, limit := extractLimitOptions(vo[0].String())
+
+	// Writing value back without limiter "|" options
+	vo[0] = reflect.ValueOf(path)
+
+	// Calling our constructor with list of given options
+	plugin := vc.Call(vo)[0].Interface()
+	pluginWrapper := plugin
+
+	if limit != "" {
+		pluginWrapper = NewLimiter(plugin, limit)
+	} else {
+		pluginWrapper = plugin
+	}
+
+	_, isR := plugin.(io.Reader)
+	_, isW := plugin.(io.Writer)
+
+	// Some of the output can be Readers as well because return responses
+	if isR && !isW {
+		Plugins.Inputs = append(Plugins.Inputs, pluginWrapper.(io.Reader))
+	}
+
+	if isW {
+		Plugins.Outputs = append(Plugins.Outputs, pluginWrapper.(io.Writer))
+	}
+}
+
+// InitPlugins specify and initialize all available plugins
 func InitPlugins() {
 	for _, options := range Settings.inputDummy {
-		Plugins.Inputs = append(Plugins.Inputs, NewDummyInput(options))
+		registerPlugin(NewDummyInput, options)
 	}
 
 	for _, options := range Settings.outputDummy {
-		Plugins.Outputs = append(Plugins.Outputs, NewDummyOutput(options))
+		registerPlugin(NewDummyOutput, options)
 	}
 
 	for _, options := range Settings.inputRAW {
-		Plugins.Inputs = append(Plugins.Inputs, NewRAWInput(options))
+		registerPlugin(NewRAWInput, options, time.Duration(0))
 	}
 
 	for _, options := range Settings.inputTCP {
-		Plugins.Inputs = append(Plugins.Inputs, NewTCPInput(options))
+		registerPlugin(NewTCPInput, options)
 	}
 
 	for _, options := range Settings.outputTCP {
-		Plugins.Outputs = append(Plugins.Outputs, NewTCPOutput(options))
+		registerPlugin(NewTCPOutput, options)
 	}
 
 	for _, options := range Settings.inputFile {
-		Plugins.Inputs = append(Plugins.Inputs, NewFileInput(options))
+		registerPlugin(NewFileInput, options)
 	}
 
 	for _, options := range Settings.outputFile {
-		Plugins.Outputs = append(Plugins.Outputs, NewFileOutput(options))
+		registerPlugin(NewFileOutput, options)
+	}
+
+	for _, options := range Settings.inputHTTP {
+		registerPlugin(NewHTTPInput, options)
+	}
+
+	// If we explicitly set Host header http output should not rewrite it
+	// Fix: https://github.com/buger/gor/issues/174
+	for _, header := range Settings.modifierConfig.headers {
+		if header.Name == "Host" {
+			Settings.outputHTTPConfig.OriginalHost = true
+			break
+		}
 	}
 
 	for _, options := range Settings.outputHTTP {
-		Plugins.Outputs = append(Plugins.Outputs, NewHTTPOutput(options, Settings.outputHTTPHeaders, Settings.outputHTTPMethods, Settings.outputHTTPUrlRegexp, Settings.outputHTTPHeaderFilters, Settings.outputHTTPHeaderHashFilters, Settings.outputHTTPElasticSearch, Settings.outputHTTPUrlRewrite))
+		registerPlugin(NewHTTPOutput, options, &Settings.outputHTTPConfig)
 	}
 }

plugins_test.go

@@ -0,0 +1,47 @@
+package main
+
+import (
+	"io"
+	"testing"
+)
+
+func TestPluginsRegistration(t *testing.T) {
+	Plugins.Inputs = []io.Reader{}
+	Plugins.Outputs = []io.Writer{}
+
+	Settings.inputDummy = MultiOption{"[]"}
+	Settings.outputDummy = MultiOption{"[]"}
+	Settings.outputHTTP = MultiOption{"www.example.com|10"}
+	Settings.inputFile = MultiOption{"/dev/null"}
+
+	InitPlugins()
+
+	if len(Plugins.Inputs) != 2 {
+		t.Errorf("Should be 2 inputs %d", len(Plugins.Inputs))
+	}
+
+	if _, ok := Plugins.Inputs[0].(*DummyInput); !ok {
+		t.Errorf("First input should be DummyInput")
+	}
+
+	if _, ok := Plugins.Inputs[1].(*FileInput); !ok {
+		t.Errorf("Second input should be FileInput")
+	}
+
+	if len(Plugins.Outputs) != 2 {
+		t.Errorf("Should be 2 output %d", len(Plugins.Outputs))
+	}
+
+	if _, ok := Plugins.Outputs[0].(*DummyOutput); !ok {
+		t.Errorf("First output should be DummyOutput")
+	}
+
+	if l, ok := Plugins.Outputs[1].(*Limiter); ok {
+		if _, ok := l.plugin.(*HTTPOutput); !ok {
+			t.Errorf("HTTPOutput should be wrapped in limiter")
+		}
+	} else {
+		t.Errorf("Second output should be Limiter")
+	}
+
+}

proto/proto.go

@@ -0,0 +1,225 @@
+/*
+Package proto provides byte-level interaction with HTTP request payload.
+
+Example of HTTP payload for future references, new line symbols escaped:
+
+	POST /upload HTTP/1.1\r\n
+	User-Agent: Gor\r\n
+	Content-Length: 11\r\n
+	\r\n
+	Hello world
+
+	GET /index.html HTTP/1.1\r\n
+	User-Agent: Gor\r\n
+	\r\n
+	\r\n
+*/
+package proto
+
+import (
+	"bytes"
+	"github.com/buger/gor/byteutils"
+)
+
+// In HTTP newline defined by 2 bytes (for both windows and *nix support)
+var CLRF = []byte("\r\n")
+
+// New line acts as separator: end of Headers or Body (in some cases)
+var EmptyLine = []byte("\r\n\r\n")
+
+// Separator for Header line. Header looks like: `HeaderName: value`
+var HeaderDelim = []byte(": ")
+
+// MIMEHeadersEndPos finds end of the Headers section, which should end with empty line.
+func MIMEHeadersEndPos(payload []byte) int {
+	return bytes.Index(payload, EmptyLine)
+}
+
+// MIMEHeadersStartPos finds start of Headers section
+// It just finds position of second line (first contains location and method).
+func MIMEHeadersStartPos(payload []byte) int {
+	return bytes.Index(payload, CLRF) + 2 // Find first line end
+}
+
+// header return value and positions of header/value start/end.
+// If not found, value will be blank, and headerStart will be -1
+// Do not support multi-line headers.
+func header(payload []byte, name []byte) (value []byte, headerStart, valueStart, headerEnd int) {
+	headerStart = bytes.Index(payload, name)
+
+	if headerStart == -1 {
+		return
+	}
+
+	valueStart = headerStart + len(name) + 1 // Skip ":" after header name
+	if payload[valueStart] == ' ' {          // Ignore empty space after ':'
+		valueStart++
+	}
+	headerEnd = valueStart + bytes.IndexByte(payload[valueStart:], '\r')
+	value = payload[valueStart:headerEnd]
+
+	return
+}
+
+// Header returns header value, if header not found, value will be blank
+func Header(payload, name []byte) []byte {
+	val, _, _, _ := header(payload, name)
+
+	return val
+}
+
+// SetHeader sets header value. If header not found it creates new one.
+// Returns modified request payload
+func SetHeader(payload, name, value []byte) []byte {
+	_, hs, vs, he := header(payload, name)
+
+	if hs != -1 {
+		// If header found we just repace its value
+		return byteutils.Replace(payload, vs, he, value)
+	}
+
+	return AddHeader(payload, name, value)
+}
+
+// AddHeader takes http payload and appends new header to the start of headers section
+// Returns modified request payload
+func AddHeader(payload, name, value []byte) []byte {
+	header := make([]byte, len(name)+2+len(value)+2)
+	copy(header[0:], name)
+	copy(header[len(name):], HeaderDelim)
+	copy(header[len(name)+2:], value)
+	copy(header[len(header)-2:], CLRF)
+
+	mimeStart := MIMEHeadersStartPos(payload)
+
+	return byteutils.Insert(payload, mimeStart, header)
+}
+
+// Body returns request/response body
+func Body(payload []byte) []byte {
+	// 4 -> len(EMPTY_LINE)
+	return payload[MIMEHeadersEndPos(payload)+4:]
+}
+
+// Path takes payload and retuns request path: Split(firstLine, ' ')[1]
+func Path(payload []byte) []byte {
+	start := bytes.IndexByte(payload, ' ') + 1
+	end := bytes.IndexByte(payload[start:], ' ')
+
+	return payload[start : start+end]
+}
+
+// SetPath takes payload, sets new path and returns modified payload
+func SetPath(payload, path []byte) []byte {
+	start := bytes.IndexByte(payload, ' ') + 1
+	end := bytes.IndexByte(payload[start:], ' ')
+
+	return byteutils.Replace(payload, start, start+end, path)
+}
+
+// PathParam returns URL query attribute by given name, if no found: valueStart will be -1
+func PathParam(payload, name []byte) (value []byte, valueStart, valueEnd int) {
+	path := Path(payload)
+
+	if paramStart := bytes.Index(path, append(name, '=')); paramStart != -1 {
+		valueStart := paramStart + len(name) + 1
+		paramEnd := bytes.IndexByte(path[valueStart:], '&')
+
+		// Param can end with '&' (another param), or end of line
+		if paramEnd == -1 { // It is final param
+			paramEnd = len(path)
+		} else {
+			paramEnd += valueStart
+		}
+
+		return path[valueStart:paramEnd], valueStart, paramEnd
+	}
+
+	return []byte(""), -1, -1
+}
+
+// SetPathParam takes payload and updates path Query attribute
+// If query param not found, it will append new
+// Returns modified payload
+func SetPathParam(payload, name, value []byte) []byte {
+	path := Path(payload)
+	_, vs, ve := PathParam(payload, name)
+
+	if vs != -1 { // If param found, replace its value and set new Path
+		newPath := make([]byte, len(path))
+		copy(newPath, path)
+		newPath = byteutils.Replace(newPath, vs, ve, value)
+
+		return SetPath(payload, newPath)
+	}
+
+	// if param not found append to end of url
+	// Adding 2 because of '?' or '&' at start, and '=' in middle
+	newParam := make([]byte, len(name)+len(value)+2)
+
+	if bytes.IndexByte(path, '?') == -1 {
+		newParam[0] = '?'
+	} else {
+		newParam[0] = '&'
+	}
+
+	// Copy "param=value" into buffer, after it looks like "?param=value"
+	copy(newParam[1:], name)
+	newParam[1+len(name)] = '='
+	copy(newParam[2+len(name):], value)
+
+	// Append param to the end of path
+	newPath := make([]byte, len(path)+len(newParam))
+	copy(newPath, path)
+	copy(newPath[len(path):], newParam)
+
+	return SetPath(payload, newPath)
+}
+
+// SetHost updates Host header for HTTP/1.1 or updates host in path for HTTP/1.0 or Proxy requests
+// Returns modified payload
+func SetHost(payload, url, host []byte) []byte {
+	// If this is HTTP 1.0 traffic or proxy traffic it may include host right into path variable, so instead of setting Host header we rewrite Path
+	// Fix for https://github.com/buger/gor/issues/156
+	if path := Path(payload); bytes.HasPrefix(path, []byte("http")) {
+		hostStart := bytes.IndexByte(path, ':') // : position "https?:"
+		hostStart += 3                          // Skip 1 ':' and 2 '\'
+		hostEnd := hostStart + bytes.IndexByte(path[hostStart:], '/')
+
+		newPath := make([]byte, len(path))
+		copy(newPath, path)
+		newPath = byteutils.Replace(newPath, 0, hostEnd, url)
+
+		return SetPath(payload, newPath)
+	}
+
+	return SetHeader(payload, []byte("Host"), host)
+}
+
+// Method returns HTTP method
+func Method(payload []byte) []byte {
+	end := bytes.IndexByte(payload, ' ')
+
+	return payload[:end]
+}
+
+// Status returns response status.
+// It happend to be in same position as request payload path
+func Status(payload []byte) []byte {
+	return Path(payload)
+}
+
+var httpMethods []string = []string{
+	"GET ", "OPTI", "HEAD", "POST", "PUT ", "DELE", "TRAC", "CONN",
+}
+
+func IsHTTPPayload(payload []byte) bool {
+	method := string(payload[0:4])
+
+	for _, m := range httpMethods {
+		if method == m {
+			return true
+		}
+	}
+	return false
+}

proto/proto_test.go

@@ -0,0 +1,146 @@
+package proto
+
+import (
+	"bytes"
+	"testing"
+)
+
+func TestHeader(t *testing.T) {
+	var payload, val []byte
+	var headerStart int
+
+	payload = []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
+
+	if val = Header(payload, []byte("Content-Length")); !bytes.Equal(val, []byte("7")) {
+		t.Error("Should find header value")
+	}
+
+	payload = []byte("POST /post HTTP/1.1\r\nContent-Length:7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
+
+	if val = Header(payload, []byte("Content-Length")); !bytes.Equal(val, []byte("7")) {
+		t.Error("Should find header value without space after :")
+	}
+
+	if _, headerStart, _, _ = header(payload, []byte("Not-Found")); headerStart != -1 {
+		t.Error("Should not found header")
+	}
+}
+
+func TestMIMEHeadersEndPos(t *testing.T) {
+	head := []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org")
+	payload := []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
+
+	end := MIMEHeadersEndPos(payload)
+
+	if !bytes.Equal(payload[:end], head) {
+		t.Error("Wrong headers end position:", end)
+	}
+}
+
+func TestMIMEHeadersStartPos(t *testing.T) {
+	headers := []byte("Content-Length: 7\r\nHost: www.w3.org")
+	payload := []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
+
+	start := MIMEHeadersStartPos(payload)
+	end := MIMEHeadersEndPos(payload)
+
+	if !bytes.Equal(payload[start:end], headers) {
+		t.Error("Wrong headers end position:", start, end)
+	}
+}
+
+func TestSetHeader(t *testing.T) {
+	var payload, payloadAfter []byte
+
+	payload = []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
+	payloadAfter = []byte("POST /post HTTP/1.1\r\nContent-Length: 14\r\nHost: www.w3.org\r\n\r\na=1&b=2")
+
+	if payload = SetHeader(payload, []byte("Content-Length"), []byte("14")); !bytes.Equal(payload, payloadAfter) {
+		t.Error("Should update header if it exists", string(payload))
+	}
+
+	payload = []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
+	payloadAfter = []byte("POST /post HTTP/1.1\r\nUser-Agent: Gor\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
+
+	if payload = SetHeader(payload, []byte("User-Agent"), []byte("Gor")); !bytes.Equal(payload, payloadAfter) {
+		t.Error("Should add header if not found", string(payload))
+	}
+}
+
+func TestPath(t *testing.T) {
+	var path, payload []byte
+
+	payload = []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
+
+	if path = Path(payload); !bytes.Equal(path, []byte("/post")) {
+		t.Error("Should find path", string(path))
+	}
+}
+
+func TestSetPath(t *testing.T) {
+	var payload, payloadAfter []byte
+
+	payload = []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
+	payloadAfter = []byte("POST /new_path HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
+
+	if payload = SetPath(payload, []byte("/new_path")); !bytes.Equal(payload, payloadAfter) {
+		t.Error("Should replace path", string(payload))
+	}
+}
+
+func TestPathParam(t *testing.T) {
+	var payload []byte
+
+	payload = []byte("POST /post?param=test&user_id=1 HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
+
+	if val, _, _ := PathParam(payload, []byte("param")); !bytes.Equal(val, []byte("test")) {
+		t.Error("Should detect attribute", string(val))
+	}
+
+	if val, _, _ := PathParam(payload, []byte("user_id")); !bytes.Equal(val, []byte("1")) {
+		t.Error("Should detect attribute", string(val))
+	}
+}
+
+func TestSetPathParam(t *testing.T) {
+	var payload, payloadAfter []byte
+
+	payload = []byte("POST /post?param=test&user_id=1 HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
+	payloadAfter = []byte("POST /post?param=new&user_id=1 HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
+
+	if payload = SetPathParam(payload, []byte("param"), []byte("new")); !bytes.Equal(payload, payloadAfter) {
+		t.Error("Should replace existing value", string(payload))
+	}
+
+	payload = []byte("POST /post?param=test&user_id=1 HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
+	payloadAfter = []byte("POST /post?param=test&user_id=2 HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
+
+	if payload = SetPathParam(payload, []byte("user_id"), []byte("2")); !bytes.Equal(payload, payloadAfter) {
+		t.Error("Should replace existing value", string(payload))
+	}
+
+	payload = []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
+	payloadAfter = []byte("POST /post?param=test HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
+
+	if payload = SetPathParam(payload, []byte("param"), []byte("test")); !bytes.Equal(payload, payloadAfter) {
+		t.Error("Should set param if url have no params", string(payload))
+	}
+
+	payload = []byte("POST /post?param=test HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
+	payloadAfter = []byte("POST /post?param=test&user_id=1 HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
+
+	if payload = SetPathParam(payload, []byte("user_id"), []byte("1")); !bytes.Equal(payload, payloadAfter) {
+		t.Error("Should set param at the end if url params", string(payload))
+	}
+}
+
+func TestSetHostHTTP10(t *testing.T) {
+	var payload, payloadAfter []byte
+
+	payload = []byte("POST http://example.com/post HTTP/1.0\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
+	payloadAfter = []byte("POST http://new.com/post HTTP/1.0\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
+
+	if payload = SetHost(payload, []byte("http://new.com"), []byte("new.com")); !bytes.Equal(payload, payloadAfter) {
+		t.Error("Should replace host", string(payload))
+	}
+}

protocol.go

@@ -0,0 +1,84 @@
+package main
+
+import (
+	"bytes"
+	"crypto/rand"
+	"encoding/hex"
+	"strconv"
+)
+
+const (
+	RequestPayload          = '1'
+	ResponsePayload         = '2'
+	ReplayedResponsePayload = '3'
+)
+
+func uuid() []byte {
+	b := make([]byte, 20)
+	rand.Read(b)
+
+	uuid := make([]byte, 40)
+	hex.Encode(uuid, b)
+
+	return uuid
+}
+
+var payloadSeparator = "\n🐵🙈🙉\n"
+
+func payloadScanner(data []byte, atEOF bool) (advance int, token []byte, err error) {
+	if atEOF && len(data) == 0 {
+		return 0, nil, nil
+	}
+
+	if i := bytes.Index(data, []byte(payloadSeparator)); i >= 0 {
+		// We have a full newline-terminated line.
+		return i + len([]byte(payloadSeparator)), data[0:i], nil
+	}
+
+	if atEOF {
+		return len(data), data, nil
+	}
+	return 0, nil, nil
+}
+
+// Timing is request start or round-trip time, depending on payloadType
+func payloadHeader(payloadType byte, uuid []byte, timing int64) (header []byte) {
+	sTime := strconv.FormatInt(timing, 10)
+
+	//Example:
+	//  3 f45590522cd1838b4a0d5c5aab80b77929dea3b3 1231\n
+	// `+ 1` indicates space characters or end of line
+	header = make([]byte, 1+1+len(uuid)+1+len(sTime)+1)
+	header[0] = payloadType
+	header[1] = ' '
+	header[2+len(uuid)] = ' '
+	header[len(header)-1] = '\n'
+
+	copy(header[2:], uuid)
+	copy(header[3+len(uuid):], sTime)
+
+	return header
+}
+
+func payloadBody(payload []byte) []byte {
+	headerSize := bytes.IndexByte(payload, '\n')
+	return payload[headerSize+1:]
+}
+
+func payloadMeta(payload []byte) [][]byte {
+	headerSize := bytes.IndexByte(payload, '\n')
+	return bytes.Split(payload[:headerSize], []byte{' '})
+}
+
+func isOriginPayload(payload []byte) bool {
+	switch payload[0] {
+	case RequestPayload, ResponsePayload:
+		return true
+	default:
+		return false
+	}
+}
+
+func isRequestPayload(payload []byte) bool {
+	return payload[0] == RequestPayload
+}

raw_socket_listener/listener.go

@@ -1,45 +1,92 @@
-package raw_socket
+/*
+Package rawSocket provides traffic sniffier using RAW sockets.
+
+Capture traffic from socket using RAW_SOCKET's
+http://en.wikipedia.org/wiki/Raw_socket
+
+RAW_SOCKET allow you listen for traffic on any port (e.g. sniffing) because they operate on IP level.
+
+Ports is TCP feature, same as flow control, reliable transmission and etc.
+
+This package implements own TCP layer: TCP packets is parsed using tcp_packet.go, and flow control is managed by tcp_message.go
+*/
+package rawSocket
 
 import (
+	"bytes"
 	"encoding/binary"
 	"log"
 	"net"
+	"runtime/debug"
 	"strconv"
+	"strings"
+	"time"
 )
 
-// Capture traffic from socket using RAW_SOCKET's
-// http://en.wikipedia.org/wiki/Raw_socket
-//
-// RAW_SOCKET allow you listen for traffic on any port (e.g. sniffing) because they operate on IP level.
-// Ports is TCP feature, same as flow control, reliable transmission and etc.
-// Since we can't use default TCP libraries RAWTCPLitener implements own TCP layer
-// TCP packets is parsed using tcp_packet.go, and flow control is managed by tcp_message.go
+// Listener handle traffic capture
 type Listener struct {
-	messages map[string]*TCPMessage // buffer of TCPMessages waiting to be send
+	// buffer of TCPMessages waiting to be send
+	messages map[string]*TCPMessage
 
-	c_packets  chan *TCPPacket
-	c_messages chan *TCPMessage // Messages ready to be send to client
+	// Expect: 100-continue request is send in 2 tcp messages
+	// We store ACK aliases to merge this packets together
+	ackAliases map[uint32]uint32
+	// To get ACK of second message we need to compute its Seq and wait for them message
+	seqWithData map[uint32]uint32
 
-	c_del_message chan *TCPMessage // Used for notifications about completed or expired messages
+	respAliases map[uint32]*request
+
+	// Messages ready to be send to client
+	packetsChan chan *TCPPacket
+
+	// Messages ready to be send to client
+	messagesChan chan *TCPMessage
+
+	// Used for notifications about completed or expired messages
+	messageDelChan chan *TCPMessage
 
 	addr string // IP to listen
-	port int    // Port to listen
+	port uint16 // Port to listen
+
+	messageExpire time.Duration
+
+	captureResponse bool
+
+	conn net.PacketConn
+	quit chan bool
 }
 
-// RAWTCPListen creates a listener to capture traffic from RAW_SOCKET
-func NewListener(addr string, port string) (rawListener *Listener) {
-	rawListener = &Listener{}
+type request struct {
+	start int64
+	ack   uint32
+}
 
-	rawListener.c_packets = make(chan *TCPPacket, 100)
-	rawListener.c_messages = make(chan *TCPMessage, 100)
-	rawListener.c_del_message = make(chan *TCPMessage, 100)
-	rawListener.messages = make(map[string]*TCPMessage)
+// NewListener creates and initializes new Listener object
+func NewListener(addr string, port string, expire time.Duration, captureResponse bool) (l *Listener) {
+	l = &Listener{captureResponse: captureResponse}
 
-	rawListener.addr = addr
-	rawListener.port, _ = strconv.Atoi(port)
+	l.packetsChan = make(chan *TCPPacket, 10000)
+	l.messagesChan = make(chan *TCPMessage, 10000)
+	l.messageDelChan = make(chan *TCPMessage, 10000)
+	l.quit = make(chan bool)
 
-	go rawListener.listen()
-	go rawListener.readRAWSocket()
+	l.messages = make(map[string]*TCPMessage)
+	l.ackAliases = make(map[uint32]uint32)
+	l.seqWithData = make(map[uint32]uint32)
+	l.respAliases = make(map[uint32]*request)
+
+	l.addr = addr
+	_port, _ := strconv.Atoi(port)
+	l.port = uint16(_port)
+
+	if expire.Nanoseconds() == 0 {
+		expire = 2000 * time.Millisecond
+	}
+
+	l.messageExpire = expire
+
+	go l.listen()
+	go l.readRAWSocket()
 
 	return
 }
@@ -47,59 +94,70 @@ func NewListener(addr string, port string) (rawListener *Listener) {
 func (t *Listener) listen() {
 	for {
 		select {
+		case <-t.quit:
+			t.conn.Close()
+			return
 		// If message ready for deletion it means that its also complete or expired by timeout
-		case message := <-t.c_del_message:
-			t.c_messages <- message
+		case message := <-t.messageDelChan:
+			delete(t.ackAliases, message.Ack)
 			delete(t.messages, message.ID)
 
+			if !message.IsIncoming {
+				delete(t.respAliases, message.Ack)
+			}
+
+			t.messagesChan <- message
+
 		// We need to use channels to process each packet to avoid data races
-		case packet := <-t.c_packets:
+		case packet := <-t.packetsChan:
 			t.processTCPPacket(packet)
 		}
 	}
 }
-
 func (t *Listener) readRAWSocket() {
 	conn, e := net.ListenPacket("ip4:tcp", t.addr)
-	defer conn.Close()
+	t.conn = conn
 
 	if e != nil {
 		log.Fatal(e)
 	}
 
-	buf := make([]byte, 4096*2)
+	defer t.conn.Close()
 
 	for {
+		buf := make([]byte, 64*1024) // 64kb
 		// Note: ReadFrom receive messages without IP header
-		n, addr, err := conn.ReadFrom(buf)
+		n, addr, err := t.conn.ReadFrom(buf)
 
 		if err != nil {
-			log.Println("Error:", err)
-			continue
+			if strings.HasSuffix(err.Error(), "closed network connection") {
+				return
+			} else {
+				log.Println("Raw listener error:", err)
+				continue
+			}
 		}
 
 		if n > 0 {
-			t.parsePacket(addr, buf[:n])
+			go t.parsePacket(addr, buf[:n])
 		}
 	}
 }
 
 func (t *Listener) parsePacket(addr net.Addr, buf []byte) {
-	if t.isIncomingDataPacket(buf) {
-		new_buf := make([]byte, len(buf))
-		copy(new_buf, buf)
-
-		t.c_packets <- ParseTCPPacket(addr, new_buf)
+	if t.isValidPacket(buf) {
+		t.packetsChan <- ParseTCPPacket(addr, buf)
 	}
 }
 
-func (t *Listener) isIncomingDataPacket(buf []byte) bool {
+func (t *Listener) isValidPacket(buf []byte) bool {
 	// To avoid full packet parsing every time, we manually parsing values needed for packet filtering
 	// http://en.wikipedia.org/wiki/Transmission_Control_Protocol
-	dest_port := binary.BigEndian.Uint16(buf[2:4])
+	destPort := binary.BigEndian.Uint16(buf[2:4])
+	srcPort := binary.BigEndian.Uint16(buf[0:2])
 
 	// Because RAW_SOCKET can't be bound to port, we have to control it by ourself
-	if int(dest_port) == t.port {
+	if destPort == t.port || (t.captureResponse && srcPort == t.port) {
 		// Get the 'data offset' (size of the TCP header in 32-bit words)
 		dataOffset := (buf[12] & 0xF0) >> 4
 
@@ -114,28 +172,89 @@ func (t *Listener) isIncomingDataPacket(buf []byte) bool {
 	return false
 }
 
+var bExpect100ContinueCheck = []byte("Expect: 100-continue")
+var bPOST = []byte("POST")
+
 // Trying to add packet to existing message or creating new message
 //
 // For TCP message unique id is Acknowledgment number (see tcp_packet.go)
 func (t *Listener) processTCPPacket(packet *TCPPacket) {
-	defer func() { recover() }()
+	// Don't exit on panic
+	defer func() {
+		if r := recover(); r != nil {
+			log.Println("PANIC: pkg:", r, string(debug.Stack()))
+		}
+	}()
 
 	var message *TCPMessage
-	m_id := packet.Addr.String() + strconv.Itoa(int(packet.Ack))
 
-	message, ok := t.messages[m_id]
+	isIncoming := packet.DestPort == t.port
+
+	if parentAck, ok := t.seqWithData[packet.Seq]; ok {
+		t.ackAliases[packet.Ack] = parentAck
+		delete(t.seqWithData, packet.Seq)
+	}
+
+	if alias, ok := t.ackAliases[packet.Ack]; ok {
+		packet.Ack = alias
+	}
+
+	var responseRequest *request
+
+	if t.captureResponse && !isIncoming {
+		responseRequest, _ = t.respAliases[packet.Ack]
+	}
+
+	mID := packet.Addr.String() + strconv.Itoa(int(packet.DestPort)) + strconv.Itoa(int(packet.Ack))
+
+	message, ok := t.messages[mID]
 
 	if !ok {
-		// We sending c_del_message channel, so message object can communicate with Listener and notify it if message completed
-		message = NewTCPMessage(m_id, t.c_del_message)
-		t.messages[m_id] = message
+		// We sending messageDelChan channel, so message object can communicate with Listener and notify it if message completed
+		message = NewTCPMessage(mID, t.messageDelChan, packet.Ack, &t.messageExpire, isIncoming)
+		t.messages[mID] = message
+
+		if !isIncoming && responseRequest != nil {
+			message.RequestStart = responseRequest.start
+			message.RequestAck = responseRequest.ack
+		}
+	}
+
+	// Handling Expect: 100-continue requests
+	if bytes.Equal(packet.Data[0:4], bPOST) {
+		// reading last 20 bytes (not counting CRLF): last header value (if no body presented)
+		if bytes.Equal(packet.Data[len(packet.Data)-24:len(packet.Data)-4], bExpect100ContinueCheck) {
+			t.seqWithData[packet.Seq+uint32(len(packet.Data))] = packet.Ack
+
+			// Removing `Expect: 100-continue` header
+			packet.Data = append(packet.Data[:len(packet.Data)-24], packet.Data[len(packet.Data)-2:]...)
+		}
+	}
+
+	if t.captureResponse && isIncoming {
+		message.mu.Lock()
+		// If message have multiple packets, delete previous alias
+		if len(message.packets) > 0 {
+			delete(t.respAliases, message.ResponseAck)
+		}
+		message.mu.Unlock()
+
+		responseAck := packet.Seq + uint32(len(packet.Data))
+		t.respAliases[responseAck] = &request{message.Start, message.Ack}
+		message.ResponseAck = responseAck
 	}
 
 	// Adding packet to message
-	message.c_packets <- packet
+	message.packetsChan <- packet
 }
 
 // Receive TCP messages from the listener channel
 func (t *Listener) Receive() *TCPMessage {
-	return <-t.c_messages
+	return <-t.messagesChan
+}
+
+func (t *Listener) Close() {
+	close(t.quit)
+	t.conn.Close()
+	return
 }

raw_socket_listener/tcp_message.go

@@ -1,13 +1,16 @@
-package raw_socket
+package rawSocket
 
 import (
+	"bytes"
+	"crypto/sha1"
+	"encoding/hex"
+	"github.com/buger/gor/proto"
 	"log"
-	"sort"
+	"strconv"
+	"sync"
 	"time"
 )
 
-const MSG_EXPIRE = 2000 * time.Millisecond
-
 // TCPMessage ensure that all TCP packets for given request is received, and processed in right sequence
 // Its needed because all TCP message can be fragmented or re-transmitted
 //
@@ -15,25 +18,35 @@ const MSG_EXPIRE = 2000 * time.Millisecond
 // Message can be compiled from unique packets with same message_id which sorted by sequence
 // Message is received if we didn't receive any packets for 2000ms
 type TCPMessage struct {
-	ID      string // Message ID
+	ID           string // Message ID
+	Ack          uint32
+	ResponseAck  uint32
+	RequestStart int64
+	RequestAck   uint32
+	Start        int64
+	End          int64
+	IsIncoming   bool
+
 	packets []*TCPPacket
 
 	timer *time.Timer // Used for expire check
 
-	c_packets chan *TCPPacket
+	packetsChan chan *TCPPacket
 
-	c_del_message chan *TCPMessage
+	delChan chan *TCPMessage
+
+	expire *time.Duration
+
+	mu sync.Mutex
 }
 
 // NewTCPMessage pointer created from a Acknowledgment number and a channel of messages readuy to be deleted
-func NewTCPMessage(ID string, c_del chan *TCPMessage) (msg *TCPMessage) {
-	msg = &TCPMessage{ID: ID}
-
-	msg.c_packets = make(chan *TCPPacket)
-	msg.c_del_message = c_del // used for notifying that message completed or expired
-
-	// Every time we receive packet we reset this timer
-	msg.timer = time.AfterFunc(MSG_EXPIRE, msg.Timeout)
+func NewTCPMessage(ID string, delChan chan *TCPMessage, Ack uint32, expire *time.Duration, IsIncoming bool) (msg *TCPMessage) {
+	msg = &TCPMessage{ID: ID, Ack: Ack, expire: expire, IsIncoming: IsIncoming}
+	msg.Start = time.Now().UnixNano()
+	msg.packetsChan = make(chan *TCPPacket)
+	msg.delChan = delChan // used for notifying that message completed or expired
+	msg.timer = time.NewTimer(0)
 
 	go msg.listen()
 
@@ -43,7 +56,7 @@ func NewTCPMessage(ID string, c_del chan *TCPMessage) (msg *TCPMessage) {
 func (t *TCPMessage) listen() {
 	for {
 		select {
-		case packet, more := <-t.c_packets:
+		case packet, more := <-t.packetsChan:
 			if more {
 				t.AddPacket(packet)
 			} else {
@@ -56,16 +69,39 @@ func (t *TCPMessage) listen() {
 
 // Timeout notifies message to stop listening, close channel and message ready to be sent
 func (t *TCPMessage) Timeout() {
-	close(t.c_packets)   // Notify to stop listen loop and close channel
-	t.c_del_message <- t // Notify RAWListener that message is ready to be send to replay server
+	select {
+	// In some cases Timeout can be called multiple times (do not know how yet)
+	// Ensure that we did not close channel 2 times
+	case packet, ok := <-t.packetsChan:
+		if ok {
+			t.AddPacket(packet)
+			t.Timeout()
+		} else {
+			return
+		}
+	default:
+		close(t.packetsChan)
+		// Notify RAWListener that message is ready to be send to replay server
+		// Responses without requests gets discarded
+		if t.IsIncoming || t.RequestStart != 0 {
+			t.delChan <- t
+		}
+	}
 }
 
-// Bytes sorts packets in right orders and return message content
+// Bytes return message content
 func (t *TCPMessage) Bytes() (output []byte) {
-	sort.Sort(BySeq(t.packets))
+	for _, p := range t.packets {
+		output = append(output, p.Data...)
+	}
 
-	for _, v := range t.packets {
-		output = append(output, v.Data...)
+	return output
+}
+
+// Size returns total size of message
+func (t *TCPMessage) Size() (size int) {
+	for _, p := range t.packets {
+		size += len(p.Data)
 	}
 
 	return
@@ -74,6 +110,9 @@ func (t *TCPMessage) Bytes() (output []byte) {
 // AddPacket to the message and ensure packet uniqueness
 // TCP allows that packet can be re-send multiple times
 func (t *TCPMessage) AddPacket(packet *TCPPacket) {
+	t.mu.Lock()
+	defer t.mu.Unlock()
+
 	packetFound := false
 
 	for _, pkt := range t.packets {
@@ -86,9 +125,88 @@ func (t *TCPMessage) AddPacket(packet *TCPPacket) {
 	if packetFound {
 		log.Println("Received packet with same sequence")
 	} else {
-		t.packets = append(t.packets, packet)
+		// Packets not always captured in same Seq order, and sometimes we need to prepend
+		if len(t.packets) == 0 || packet.Seq > t.packets[len(t.packets)-1].Seq {
+			t.packets = append(t.packets, packet)
+		} else {
+			t.packets = append([]*TCPPacket{packet}, t.packets...)
+		}
+
+		t.End = time.Now().UnixNano()
 	}
 
-	// Reset message timeout timer
-	t.timer.Reset(MSG_EXPIRE)
+	if !t.isMultipart() {
+		t.Timeout()
+	} else {
+		// If more then 1 packet, wait for more, and set expiration
+		if len(t.packets) == 1 {
+			// Every time we receive packet we reset this timer
+			t.timer = time.AfterFunc(*t.expire, t.Timeout)
+		} else {
+			// Reset message timeout timer
+			t.timer.Reset(*t.expire)
+		}
+	}
+}
+
+// isMultipart returns true if message contains from multiple tcp packets
+func (t *TCPMessage) isMultipart() bool {
+	if len(t.packets) > 1 {
+		return true
+	}
+
+	payload := t.packets[0].Data
+	m := payload[:4]
+
+	if t.IsIncoming {
+		// If one GET, OPTIONS, or HEAD request
+		if bytes.Equal(m, []byte("GET ")) || bytes.Equal(m, []byte("OPTI")) || bytes.Equal(m, []byte("HEAD")) {
+			return false
+		} else {
+			// Sometimes header comes after the body :(
+			if bytes.Equal(m, []byte("POST")) || bytes.Equal(m, []byte("PUT ")) || bytes.Equal(m, []byte("PATC")) {
+				if length := proto.Header(payload, []byte("Content-Length")); len(length) > 0 {
+					l, _ := strconv.Atoi(string(length))
+
+					// If content-length equal current body length
+					if l > 0 && l == len(proto.Body(payload)) {
+						return false
+					}
+				}
+			}
+		}
+	} else {
+		if length := proto.Header(payload, []byte("Content-Length")); len(length) > 0 {
+			if length[0] == '0' {
+				return false
+			}
+
+			l, _ := strconv.Atoi(string(length))
+
+			// If content-length equal current body length
+			if l > 0 && l == len(proto.Body(payload)) {
+				return false
+			}
+		}
+	}
+
+	return true
+}
+
+func (t *TCPMessage) UUID() []byte {
+	var key []byte
+
+	if t.IsIncoming {
+		key = strconv.AppendInt(key, t.Start, 10)
+		key = strconv.AppendUint(key, uint64(t.Ack), 10)
+	} else {
+		key = strconv.AppendInt(key, t.RequestStart, 10)
+		key = strconv.AppendUint(key, uint64(t.RequestAck), 10)
+	}
+
+	uuid := make([]byte, 40)
+	sha := sha1.Sum(key)
+	hex.Encode(uuid, sha[:20])
+
+	return uuid
 }

raw_socket_listener/tcp_packet.go

@@ -1,4 +1,4 @@
-package raw_socket
+package rawSocket
 
 import (
 	"encoding/binary"
@@ -9,19 +9,18 @@ import (
 
 // TCP Flags
 const (
-	TCP_FIN = 1 << iota
-	TCP_SYN
-	TCP_RST
-	TCP_PSH
-	TCP_ACK
-	TCP_URG
-	TCP_ECE
-	TCP_CWR
-	TCP_NS
+	fFIN = 1 << iota
+	fSYN
+	fRST
+	fPSH
+	fACK
+	fURG
+	fECE
+	fCWR
+	fNS
 )
 
-// Simple TCP packet parser
-//
+// TCPPacket provides tcp packet parser
 // Packet structure: http://en.wikipedia.org/wiki/Transmission_Control_Protocol
 type TCPPacket struct {
 	SrcPort    uint16
@@ -39,6 +38,7 @@ type TCPPacket struct {
 	Addr net.Addr
 }
 
+// ParseTCPPacket takes address and tcp payload and returns parsed TCPPacket
 func ParseTCPPacket(addr net.Addr, b []byte) (p *TCPPacket) {
 	p = &TCPPacket{Data: b}
 	p.ParseBasic()
@@ -50,8 +50,6 @@ func ParseTCPPacket(addr net.Addr, b []byte) (p *TCPPacket) {
 // Parse TCP Packet, inspired by: https://github.com/miekg/pcap/blob/master/packet.go
 func (t *TCPPacket) Parse() {
 	t.ParseBasic()
-	t.SrcPort = binary.BigEndian.Uint16(t.Data[0:2])
-	t.DestPort = binary.BigEndian.Uint16(t.Data[2:4])
 	t.Flags = binary.BigEndian.Uint16(t.Data[12:14]) & 0x1FF
 	t.Window = binary.BigEndian.Uint16(t.Data[14:16])
 	t.Checksum = binary.BigEndian.Uint16(t.Data[16:18])
@@ -60,6 +58,8 @@ func (t *TCPPacket) Parse() {
 
 // ParseBasic set of fields
 func (t *TCPPacket) ParseBasic() {
+	t.DestPort = binary.BigEndian.Uint16(t.Data[2:4])
+	t.SrcPort = binary.BigEndian.Uint16(t.Data[0:2])
 	t.Seq = binary.BigEndian.Uint32(t.Data[4:8])
 	t.Ack = binary.BigEndian.Uint32(t.Data[8:12])
 	t.DataOffset = (t.Data[12] & 0xF0) >> 4
@@ -69,6 +69,11 @@ func (t *TCPPacket) ParseBasic() {
 
 // String output for a TCP Packet
 func (t *TCPPacket) String() string {
+	maxLen := len(t.Data)
+	if maxLen > 500 {
+		maxLen = 500
+	}
+
 	return strings.Join([]string{
 		"Source port: " + strconv.Itoa(int(t.SrcPort)),
 		"Dest port:" + strconv.Itoa(int(t.DestPort)),
@@ -76,25 +81,26 @@ func (t *TCPPacket) String() string {
 		"Acknowledgment:" + strconv.Itoa(int(t.Ack)),
 		"Header len:" + strconv.Itoa(int(t.DataOffset)),
 
-		"Flag ns:" + strconv.FormatBool(t.Flags&TCP_NS != 0),
-		"Flag crw:" + strconv.FormatBool(t.Flags&TCP_CWR != 0),
-		"Flag ece:" + strconv.FormatBool(t.Flags&TCP_ECE != 0),
-		"Flag urg:" + strconv.FormatBool(t.Flags&TCP_URG != 0),
-		"Flag ack:" + strconv.FormatBool(t.Flags&TCP_ACK != 0),
-		"Flag psh:" + strconv.FormatBool(t.Flags&TCP_PSH != 0),
-		"Flag rst:" + strconv.FormatBool(t.Flags&TCP_RST != 0),
-		"Flag syn:" + strconv.FormatBool(t.Flags&TCP_SYN != 0),
-		"Flag fin:" + strconv.FormatBool(t.Flags&TCP_FIN != 0),
+		"Flag ns:" + strconv.FormatBool(t.Flags&fNS != 0),
+		"Flag crw:" + strconv.FormatBool(t.Flags&fCWR != 0),
+		"Flag ece:" + strconv.FormatBool(t.Flags&fECE != 0),
+		"Flag urg:" + strconv.FormatBool(t.Flags&fURG != 0),
+		"Flag ack:" + strconv.FormatBool(t.Flags&fACK != 0),
+		"Flag psh:" + strconv.FormatBool(t.Flags&fPSH != 0),
+		"Flag rst:" + strconv.FormatBool(t.Flags&fRST != 0),
+		"Flag syn:" + strconv.FormatBool(t.Flags&fSYN != 0),
+		"Flag fin:" + strconv.FormatBool(t.Flags&fFIN != 0),
 
 		"Window size:" + strconv.Itoa(int(t.Window)),
 		"Checksum:" + strconv.Itoa(int(t.Checksum)),
 
-		"Data:" + string(t.Data),
+		"Data size:" + strconv.Itoa(len(t.Data)),
+		"Data:" + string(t.Data[:maxLen]),
 	}, "\n")
 }
 
-type BySeq []*TCPPacket
+type sortBySeq []*TCPPacket
 
-func (a BySeq) Len() int			{ return len(a) }
-func (a BySeq) Swap(i, j int)		{ a[i], a[j] = a[j], a[i] }
-func (a BySeq) Less(i, j int) bool	{ return a[i].Seq < a[j].Seq }
+func (a sortBySeq) Len() int           { return len(a) }
+func (a sortBySeq) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
+func (a sortBySeq) Less(i, j int) bool { return a[i].Seq < a[j].Seq }

settings.go

@@ -3,16 +3,30 @@ package main
 import (
 	"flag"
 	"fmt"
-	"log"
 	"os"
+	"sync"
+	"time"
 )
 
-const (
-	VERSION = "0.9.0"
-)
+var VERSION string
 
+// MultiOption allows to specify multiple flags with same name and collects all values into array
+type MultiOption []string
+
+func (h *MultiOption) String() string {
+	return fmt.Sprint(*h)
+}
+
+// Set gets called multiple times for each flag with same name
+func (h *MultiOption) Set(value string) error {
+	*h = append(*h, value)
+	return nil
+}
+
+// AppSettings is the struct of main configuration
 type AppSettings struct {
 	verbose bool
+	debug   bool
 	stats   bool
 
 	splitOutput bool
@@ -20,8 +34,8 @@ type AppSettings struct {
 	inputDummy  MultiOption
 	outputDummy MultiOption
 
-	inputTCP  MultiOption
-	outputTCP MultiOption
+	inputTCP       MultiOption
+	outputTCP      MultiOption
 	outputTCPStats bool
 
 	inputFile  MultiOption
@@ -29,19 +43,17 @@ type AppSettings struct {
 
 	inputRAW MultiOption
 
-	outputHTTP                  MultiOption
-	outputHTTPHeaders           HTTPHeaders
-	outputHTTPMethods           HTTPMethods
-	outputHTTPUrlRegexp         HTTPUrlRegexp
-	outputHTTPUrlRewrite        UrlRewriteMap
-	outputHTTPHeaderFilters     HTTPHeaderFilters
-	outputHTTPHeaderHashFilters HTTPHeaderHashFilters
-	outputHTTPElasticSearch     string
-	outputHTTPWorkers           int
-	outputHTTPStats             bool
+	middleware string
+
+	inputHTTP  MultiOption
+	outputHTTP MultiOption
+
+	outputHTTPConfig HTTPOutputConfig
+	modifierConfig   HTTPModifierConfig
 }
 
-var Settings AppSettings = AppSettings{}
+// Settings holds Gor configuration
+var Settings AppSettings
 
 func usage() {
 	fmt.Printf("Gor is a simple http traffic replication tool written in Go. Its main goal is to replay traffic from production servers to staging and dev environments.\nProject page: https://github.com/buger/gor\nAuthor: <Leonid Bugaev> leonsbox@gmail.com\nCurrent Version: %s\n\n", VERSION)
@@ -52,7 +64,8 @@ func usage() {
 func init() {
 	flag.Usage = usage
 
-	flag.BoolVar(&Settings.verbose, "verbose", false, "Turn on verbose/debug output")
+	flag.BoolVar(&Settings.verbose, "verbose", false, "Turn on more verbose output")
+	flag.BoolVar(&Settings.debug, "debug", false, "Turn on debug output, shows all itercepted traffic. Works only when with `verbose` flag")
 	flag.BoolVar(&Settings.stats, "stats", false, "Turn on queue stats output")
 
 	flag.BoolVar(&Settings.splitOutput, "split-output", false, "By default each output gets same traffic. If set to `true` it splits traffic equally among all outputs.")
@@ -69,21 +82,60 @@ func init() {
 
 	flag.Var(&Settings.inputRAW, "input-raw", "Capture traffic from given port (use RAW sockets and require *sudo* access):\n\t# Capture traffic from 8080 port\n\tgor --input-raw :8080 --output-http staging.com")
 
-	flag.Var(&Settings.outputHTTP, "output-http", "Forwards incoming requests to given http address.\n\t# Redirect all incoming requests to staging.com address \n\tgor --input-raw :80 --output-http http://staging.com")
-	flag.Var(&Settings.outputHTTPHeaders, "output-http-header", "Inject additional headers to http reqest:\n\tgor --input-raw :8080 --output-http staging.com --output-http-header 'User-Agent: Gor'")
-	flag.Var(&Settings.outputHTTPMethods, "output-http-method", "Whitelist of HTTP methods to replay. Anything else will be dropped:\n\tgor --input-raw :8080 --output-http staging.com --output-http-method GET --output-http-method OPTIONS")
-	flag.Var(&Settings.outputHTTPUrlRegexp, "output-http-url-regexp", "A regexp to match requests against. Anything else will be dropped:\n\t gor --input-raw :8080 --output-http staging.com --output-http-url-regexp ^www.")
-	flag.Var(&Settings.outputHTTPHeaderFilters, "output-http-header-filter", "A regexp to match a specific header against. Requests with non-matching headers will be dropped:\n\t gor --input-raw :8080 --output-http staging.com --output-http-header-filter api-version:^v1")
-	flag.Var(&Settings.outputHTTPHeaderHashFilters, "output-http-header-hash-filter", "Takes a fraction of requests, consistently taking or rejecting a request based on the FNV32-1A hash of a specific header. The fraction must have a denominator that is a power of two:\n\t gor --input-raw :8080 --output-http staging.com --output-http-header-hash-filter user-id:1/4")
-	flag.IntVar(&Settings.outputHTTPWorkers, "output-http-workers", -1, "Gor uses dynamic worker scaling by default.  Enter a number to run a set number of workers.")
-	flag.BoolVar(&Settings.outputHTTPStats, "output-http-stats", false, "Report http output queue stats to console every 5 seconds.")
+	flag.StringVar(&Settings.middleware, "middleware", "", "Used for modifying traffic using external command")
 
-	flag.StringVar(&Settings.outputHTTPElasticSearch, "output-http-elasticsearch", "", "Send request and response stats to ElasticSearch:\n\tgor --input-raw :8080 --output-http staging.com --output-http-elasticsearch 'es_host:api_port/index_name'")
-	flag.Var(&Settings.outputHTTPUrlRewrite, "output-http-rewrite-url", "Rewrite the requst url based on a mapping:\n\tgor --input-raw :8080 --output-http staging.com --output-http-rewrite-url /xml_test/interface.php:/api/service.do")
+	flag.Var(&Settings.inputHTTP, "input-http", "Read requests from HTTP, should be explicitly sent from your application:\n\t# Listen for http on 9000\n\tgor --input-http :9000 --output-http staging.com")
+
+	flag.Var(&Settings.outputHTTP, "output-http", "Forwards incoming requests to given http address.\n\t# Redirect all incoming requests to staging.com address \n\tgor --input-raw :80 --output-http http://staging.com")
+	flag.IntVar(&Settings.outputHTTPConfig.workers, "output-http-workers", 0, "Gor uses dynamic worker scaling by default.  Enter a number to run a set number of workers.")
+	flag.IntVar(&Settings.outputHTTPConfig.redirectLimit, "output-http-redirects", 0, "Enable how often redirects should be followed.")
+	flag.DurationVar(&Settings.outputHTTPConfig.Timeout, "output-http-timeout", 0, "Specify HTTP request/response timeout. By default 5s. Example: --output-http-timeout 30s")
+
+	flag.BoolVar(&Settings.outputHTTPConfig.stats, "output-http-stats", false, "Report http output queue stats to console every 5 seconds.")
+	flag.BoolVar(&Settings.outputHTTPConfig.OriginalHost, "http-original-host", false, "Normally gor replaces the Host http header with the host supplied with --output-http.  This option disables that behavior, preserving the original Host header.")
+
+	flag.StringVar(&Settings.outputHTTPConfig.elasticSearch, "output-http-elasticsearch", "", "Send request and response stats to ElasticSearch:\n\tgor --input-raw :8080 --output-http staging.com --output-http-elasticsearch 'es_host:api_port/index_name'")
+
+	flag.Var(&Settings.modifierConfig.headers, "http-set-header", "Inject additional headers to http reqest:\n\tgor --input-raw :8080 --output-http staging.com --http-set-header 'User-Agent: Gor'")
+	flag.Var(&Settings.modifierConfig.headers, "output-http-header", "WARNING: `--output-http-header` DEPRECATED, use `--http-set-header` instead")
+
+	flag.Var(&Settings.modifierConfig.params, "http-set-param", "Set request url param, if param already exists it will be overwritten:\n\tgor --input-raw :8080 --output-http staging.com --http-set-param api_key=1")
+
+	flag.Var(&Settings.modifierConfig.methods, "http-allow-method", "Whitelist of HTTP methods to replay. Anything else will be dropped:\n\tgor --input-raw :8080 --output-http staging.com --http-allow-method GET --http-allow-method OPTIONS")
+	flag.Var(&Settings.modifierConfig.methods, "output-http-method", "WARNING: `--output-http-method` DEPRECATED, use `--http-allow-method` instead")
+
+	flag.Var(&Settings.modifierConfig.urlRegexp, "http-allow-url", "A regexp to match requests against. Filter get matched agains full url with domain. Anything else will be dropped:\n\t gor --input-raw :8080 --output-http staging.com --http-allow-url ^www.")
+	flag.Var(&Settings.modifierConfig.urlRegexp, "output-http-url-regexp", "WARNING: `--output-http-url-regexp` DEPRECATED, use `--http-allow-url` instead")
+
+	flag.Var(&Settings.modifierConfig.urlNegativeRegexp, "http-diallow-url", "A regexp to match requests against. Filter get matched agains full url with domain. Anything else will be dropped:\n\t gor --input-raw :8080 --output-http staging.com --http-disallow-url ^www.")
+
+	flag.Var(&Settings.modifierConfig.urlRewrite, "http-rewrite-url", "Rewrite the request url based on a mapping:\n\tgor --input-raw :8080 --output-http staging.com --http-rewrite-url /v1/user/([^\\/]+)/ping:/v2/user/$1/ping")
+	flag.Var(&Settings.modifierConfig.urlRewrite, "output-http-rewrite-url", "WARNING: `--output-http-rewrite-url` DEPRECATED, use `--http-rewrite-url` instead")
+
+	flag.Var(&Settings.modifierConfig.headerFilters, "http-allow-header", "A regexp to match a specific header against. Requests with non-matching headers will be dropped:\n\t gor --input-raw :8080 --output-http staging.com --http-allow-header api-version:^v1")
+	flag.Var(&Settings.modifierConfig.headerFilters, "output-http-header-filter", "WARNING: `--output-http-header-filter` DEPRECATED, use `--http-allow-header` instead")
+
+	flag.Var(&Settings.modifierConfig.headerFilters, "http-disallow-header", "A regexp to match a specific header against. Requests with matching headers will be dropped:\n\t gor --input-raw :8080 --output-http staging.com --http-disallow-header \"User-Agent: Replayed by Gor\"")
+
+	flag.Var(&Settings.modifierConfig.headerHashFilters, "http-header-limiter", "Takes a fraction of requests, consistently taking or rejecting a request based on the FNV32-1A hash of a specific header:\n\t gor --input-raw :8080 --output-http staging.com --http-header-imiter user-id:25%")
+	flag.Var(&Settings.modifierConfig.headerHashFilters, "output-http-header-hash-filter", "WARNING: `output-http-header-hash-filter` DEPRECATED, use `--http-header-hash-limiter` instead")
+
+	flag.Var(&Settings.modifierConfig.paramHashFilters, "http-param-limiter", "Takes a fraction of requests, consistently taking or rejecting a request based on the FNV32-1A hash of a specific GET param:\n\t gor --input-raw :8080 --output-http staging.com --http-param-limiter user_id:25%")
 }
 
+var previousDebugTime int64
+var debugMutex sync.Mutex
+
+// Debug gets called only if --verbose flag specified
 func Debug(args ...interface{}) {
 	if Settings.verbose {
-		log.Println(args...)
+		debugMutex.Lock()
+		now := time.Now()
+		diff := float64(now.UnixNano()-previousDebugTime) / 1000000
+		previousDebugTime = now.UnixNano()
+		debugMutex.Unlock()
+
+		fmt.Printf("[DEBUG][PID %d][%d][%fms] ", os.Getpid(), now.UnixNano(), diff)
+		fmt.Println(args...)
 	}
 }

settings_header_filters.go

@@ -1,44 +0,0 @@
-package main
-
-import (
-	"errors"
-	"fmt"
-	"net/http"
-	"regexp"
-	"strings"
-)
-
-type headerFilter struct {
-	name   string
-	regexp *regexp.Regexp
-}
-
-type HTTPHeaderFilters []headerFilter
-
-func (h *HTTPHeaderFilters) String() string {
-	return fmt.Sprint(*h)
-}
-
-func (h *HTTPHeaderFilters) Set(value string) error {
-	valArr := strings.SplitN(value, ":", 2)
-	if len(valArr) < 2 {
-		return errors.New("need both header and value, colon-delimited (ex. user_id:^169$).")
-	}
-	r, err := regexp.Compile(valArr[1])
-	if err != nil {
-		return err
-	}
-
-	*h = append(*h, headerFilter{name: valArr[0], regexp: r})
-
-	return nil
-}
-
-func (h *HTTPHeaderFilters) Good(req *http.Request) bool {
-	for _, f := range *h {
-		if !f.regexp.MatchString(req.Header.Get(f.name)) {
-			return false
-		}
-	}
-	return true
-}

settings_header_filters_test.go

@@ -1,35 +0,0 @@
-package main
-
-import (
-	"testing"
-	"net/http"
-)
-
-func TestHTTPHeaderFilters(t *testing.T) {
-	filters := HTTPHeaderFilters{}
-
-	err := filters.Set("Header1:^$")
-	if err != nil {
-		t.Error("Should not error on Header1:^$")
-	}
-
-	err = filters.Set("Header2:^:$")
-	if err != nil {
-		t.Error("Should not error on Header2:^:$")
-	}
-
-	err = filters.Set("Header3-^$")
-	if err == nil {
-		t.Error("Should error on Header2:^:$")
-	}
-
-	req := http.Request{}
-	req.Header = make(map[string][]string)
-	req.Header.Add("Header1", "")
-	req.Header.Add("Header2", ":")
-	req.Header.Add("Header3", "Irrelevant")
-
-	if(!filters.Good(&req)) {
-		t.Error("Request should pass filters")
-	}
-}

settings_header_hash_filters.go

@@ -1,68 +0,0 @@
-package main
-
-import (
-	"errors"
-	"fmt"
-	"hash/fnv"
-	"net/http"
-	"strconv"
-	"strings"
-)
-
-type headerHashFilter struct {
-	name    string
-	maxHash uint32
-}
-
-type HTTPHeaderHashFilters []headerHashFilter
-
-func (h *HTTPHeaderHashFilters) String() string {
-	return fmt.Sprint(*h)
-}
-
-func (h *HTTPHeaderHashFilters) Set(value string) error {
-	valArr := strings.SplitN(value, ":", 2)
-	if len(valArr) < 2 {
-		return errors.New("need both header and value, colon-delimited (ex. user_id:1/2).")
-	}
-
-	fracArr := strings.Split(valArr[1], "/")
-	if len(fracArr) < 2 {
-		return errors.New("need both a numerator and denominator specified, slash-delimited (ex. user_id:1/4).")
-	}
-
-	var num, den uint64
-	num, _ = strconv.ParseUint(fracArr[0], 10, 64)
-	den, _ = strconv.ParseUint(fracArr[1], 10, 64)
-
-	if num < 1 || den < 1 || num > den {
-		panic("need positive numerators and denominators, with the former less than the latter.")
-	}
-
-	for test := den; test != 1; test /= 2 {
-		if test%2 == 1 {
-			return errors.New("must have a denominator which is a power of two.")
-		}
-	}
-
-	var f headerHashFilter
-	f.name = valArr[0]
-	f.maxHash = (uint32)(num * (((uint64)(2 << 31)) / den))
-	*h = append(*h, f)
-
-	return nil
-}
-
-func (h *HTTPHeaderHashFilters) Good(req *http.Request) bool {
-	for _, f := range *h {
-		if req.Header.Get(f.name) == "" {
-			return false
-		}
-		hasher := fnv.New32a()
-		hasher.Write([]byte(req.Header.Get(f.name)))
-		if hasher.Sum32() > f.maxHash {
-			return false
-		}
-	}
-	return true
-}

settings_header_hash_filters_test.go

@@ -1,43 +0,0 @@
-package main
-
-import (
-	"testing"
-	"net/http"
-)
-
-func TestHTTPHeaderHashFilters(t *testing.T) {
-	filters := HTTPHeaderHashFilters{}
-
-	err := filters.Set("Header1:1/2")
-	if err != nil {
-		t.Error("Should not error on Header1:^$")
-	}
-
-	err = filters.Set("Header2:1/2")
-	if err != nil {
-		t.Error("Should not error on Header2:^:$")
-	}
-
-	err = filters.Set("HeaderIrrelevant:1/3")
-	if err == nil {
-		t.Error("Should error on HeaderIrrelevant:1/3")
-	}
-
-	req := http.Request{}
-	req.Header = make(map[string][]string)
-	req.Header.Add("Header1", "test3414")
-
-	if(filters.Good(&req)) {
-		t.Error("Request should not pass filters, Header2 does not exist")
-	}
-
-	req.Header.Add("Header2", "test2")
-	if(filters.Good(&req)) {
-		t.Error("Request should not pass filters, Header2 hash too high")
-	}
-
-	req.Header.Set("Header2", "test3414")
-	if(!filters.Good(&req)) {
-		t.Error("Request should pass filters")
-	}
-}

settings_headers.go

@@ -1,32 +0,0 @@
-package main
-
-import (
-	"errors"
-	"fmt"
-	"strings"
-)
-
-type HTTPHeaders []HTTPHeader
-type HTTPHeader struct {
-	Name  string
-	Value string
-}
-
-func (h *HTTPHeaders) String() string {
-	return fmt.Sprint(*h)
-}
-
-func (h *HTTPHeaders) Set(value string) error {
-	v := strings.SplitN(value, ":", 2)
-	if len(v) != 2 {
-		return errors.New("Expected `Key: Value`")
-	}
-
-	header := HTTPHeader{
-		strings.TrimSpace(v[0]),
-		strings.TrimSpace(v[1]),
-	}
-
-	*h = append(*h, header)
-	return nil
-}

settings_methods.go

@@ -1,26 +0,0 @@
-package main
-
-import (
-	"fmt"
-	"strings"
-)
-
-type HTTPMethods []string
-
-func (h *HTTPMethods) String() string {
-	return fmt.Sprint(*h)
-}
-
-func (h *HTTPMethods) Set(value string) error {
-	*h = append(*h, strings.ToUpper(value))
-	return nil
-}
-
-func (h *HTTPMethods) Contains(value string) bool {
-	for _, method := range *h {
-		if value == method {
-			return true
-		}
-	}
-	return false
-}

settings_methods_test.go

@@ -1,24 +0,0 @@
-package main
-
-import (
-	"testing"
-)
-
-func TestHTTPMethods(t *testing.T) {
-	methods := HTTPMethods{}
-
-	methods.Set("lower")
-	methods.Set("UPPER")
-
-	if !methods.Contains("LOWER") {
-		t.Error("Does not contain LOWER")
-	}
-
-	if !methods.Contains("UPPER") {
-		t.Error("Does not contain UPPER")
-	}
-
-	if methods.Contains("ABSENT") {
-		t.Error("Does contain ABSENT")
-	}
-}

settings_option.go

@@ -1,16 +0,0 @@
-package main
-
-import (
-	"fmt"
-)
-
-type MultiOption []string
-
-func (h *MultiOption) String() string {
-	return fmt.Sprint(*h)
-}
-
-func (h *MultiOption) Set(value string) error {
-	*h = append(*h, value)
-	return nil
-}

settings_url_map.go

@@ -1,36 +0,0 @@
-package main
-
-import (
-	"errors"
-	"fmt"
-	"strings"
-)
-
-type urlRewrite struct {
-	src    string
-	target string
-}
-
-type UrlRewriteMap []urlRewrite
-
-func (r *UrlRewriteMap) String() string {
-	return fmt.Sprint(*r)
-}
-
-func (r *UrlRewriteMap) Set(value string) error {
-	valArr := strings.SplitN(value, ":", 2)
-	if len(valArr) < 2 {
-		return errors.New("need both src and target, colon-delimited (ex. /a:/b).")
-	}
-	*r = append(*r, urlRewrite{src: valArr[0], target: valArr[1]})
-	return nil
-}
-
-func (r *UrlRewriteMap) Rewrite(path string) string {
-        for _, f := range *r {
-                if f.src == path {
-                        return f.target
-                }
-        }
-        return path
-}

settings_url_map_test.go

@@ -1,27 +0,0 @@
-package main
-
-import (
-	"testing"
-)
-
-func TestUrlRewriteMap(t *testing.T) {
-    var url string;
-
-	rewrites := UrlRewriteMap{}
-
-	err := rewrites.Set("/abc:/123")
-	if err != nil {
-		t.Error("Should not error on /abc:/123")
-	}
-
-	url = "/abc"
-
-    if(rewrites.Rewrite(url) == url) {
-        t.Error("Request url should have been rewritten, wasn't")
-    }
-
-    url = "/wibble"
-    if(rewrites.Rewrite(url) != url) {
-        t.Error("Request url should not have been rewritten, was")
-    }
-}

settings_url_regexp.go

@@ -1,30 +0,0 @@
-package main
-
-import (
-	"net/http"
-	"regexp"
-)
-
-type HTTPUrlRegexp struct {
-	regexp *regexp.Regexp
-}
-
-func (r *HTTPUrlRegexp) String() string {
-	if r.regexp == nil {
-		return ""
-	}
-	return r.regexp.String()
-}
-
-func (r *HTTPUrlRegexp) Set(value string) error {
-	regexp, err := regexp.Compile(value)
-	r.regexp = regexp
-	return err
-}
-
-func (r *HTTPUrlRegexp) Good(req *http.Request) bool {
-	if r.regexp == nil {
-		return true
-	}
-	return r.regexp.MatchString(req.Host + req.URL.String())
-}

settings_url_regexp_test.go

@@ -1,26 +0,0 @@
-package main
-
-import (
-	"testing"
-	"net/http"
-	"net/url"
-)
-
-func TestHTTPUrlRegexp(t *testing.T) {
-	filter := HTTPUrlRegexp{}
-
-	filter.Set("^www.google.com/admin/")
-
-	req := http.Request{}
-	req.Host = "www.google.com"
-	var err error
-	req.URL, err = url.Parse("/admin/testpage1")
-	if(!filter.Good(&req) || err != nil) {
-		t.Error("Request should pass filters")
-	}
-
-	req.URL, err = url.Parse("/user/testpage2")
-	if(filter.Good(&req) || err != nil) {
-		t.Error("Request should not pass filters")
-	}
-}

test_input.go

@@ -1,9 +1,17 @@
 package main
 
+import (
+	"crypto/rand"
+	"encoding/base64"
+	"time"
+)
+
+// TestInput used for testing purpose, it allows emitting requests on demand
 type TestInput struct {
 	data chan []byte
 }
 
+// NewTestInput constructor for TestInput
 func NewTestInput() (i *TestInput) {
 	i = new(TestInput)
 	i.data = make(chan []byte, 100)
@@ -13,19 +21,41 @@ func NewTestInput() (i *TestInput) {
 
 func (i *TestInput) Read(data []byte) (int, error) {
 	buf := <-i.data
-	copy(data, buf)
 
-	return len(buf), nil
+	header := payloadHeader(RequestPayload, uuid(), time.Now().UnixNano())
+	copy(data[0:len(header)], header)
+	copy(data[len(header):], buf)
+
+	return len(buf) + len(header), nil
 }
 
+// EmitGET emits GET request without headers
 func (i *TestInput) EmitGET() {
 	i.data <- []byte("GET / HTTP/1.1\r\n\r\n")
 }
 
+// EmitPOST emits POST request with Content-Length
 func (i *TestInput) EmitPOST() {
-	i.data <- []byte("POST /pub/WWW/ HTTP/1.1\nHost: www.w3.org\r\n\r\na=1&b=2\r\n\r\n")
+	i.data <- []byte("POST /pub/WWW/ HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
 }
 
+// EmitChunkedPOST emits POST request with `Transfer-Encoding: chunked` and chunked body
+func (i *TestInput) EmitChunkedPOST() {
+	i.data <- []byte("POST /pub/WWW/ HTTP/1.1\r\nHost: www.w3.org\r\nTransfer-Encoding: chunked\r\n\r\n4\r\nWiki\r\n5\r\npedia\r\ne\r\n in\r\n\r\nchunks.\r\n0\r\n\r\n")
+}
+
+// EmitLargePOST emits POST request with large payload (5mb)
+func (i *TestInput) EmitLargePOST() {
+	size := 5 * 1024 * 1024 // 5 MB
+	rb := make([]byte, size)
+	rand.Read(rb)
+
+	rs := base64.URLEncoding.EncodeToString(rb)
+
+	i.data <- []byte("POST / HTTP/1.1\nHost: www.w3.org\nContent-Length:5242880\r\n\r\n" + rs)
+}
+
+// EmitOPTIONS emits OPTIONS request, similar to GET
 func (i *TestInput) EmitOPTIONS() {
 	i.data <- []byte("OPTIONS / HTTP/1.1\nHost: www.w3.org\r\n\r\n")
 }

test_output.go

@@ -2,10 +2,12 @@ package main
 
 type writeCallback func(data []byte)
 
+// TestOutput used in testing to intercept any output into callback
 type TestOutput struct {
 	cb writeCallback
 }
 
+// NewTestOutput constructor for TestOutput, accepts callback which get called on each incoming Write
 func NewTestOutput(cb writeCallback) (i *TestOutput) {
 	i = new(TestOutput)
 	i.cb = cb