Merge pull request #169 from buger/http-client-bugfixes

Improve http-client reliability and improve debug

.dockerignore

@@ -0,0 +1 @@
+*.tar.gz

.gitignore

@@ -0,0 +1,4 @@
+*.swp
+*.gor
+
+*.out

.travis.yml

@@ -0,0 +1,3 @@
+language: go 
+go: 1.3.3
+script: sudo -E bash -c "source /etc/profile && eval '$(gimme 1.4)' && export GOPATH=$HOME/gopath:$GOPATH && go get && GORACE='halt_on_error=1' go test -race -v"

CHANGELOG.md

@@ -1,5 +1,25 @@
+v0.7.0 - 31 Oct 2013
+* New modular architecture. Listener and Replay functionality merged.
+* Added option to equally split traffic between multiple outputs: --split-output true
+* Saving requests to file and replaying from it
+* Injecting custom headers to http requests
+* Advanced stats using ElasticSearch
+
+v0.3.5 - 15 Sep 2013
+* Significantly improved test coverage
+* Fixed bug with redirect replay https://github.com/buger/gor/pull/15
+* Added limit on listener side
+* Improved stability (catch and log panic, instead of exiting)
+* Added License file
+
+v0.3.3 - 22 Jun 2013
+* Using TCP instead of UDP for communication between Listener and Replay
+* Significantly improved performance
+* Fixed bugs causing locking and message dropping (concurrency issues)
+* Rewrote concurrency model to use more channels
+
 v0.3 - 10 Jun 2013
 * Use RAW_SOCKETS instead of tcpdump
 * Own TCP stack
 * All HTTP request types support
-* Simplified request parsing
+* Simplified request parsing

Dockerfile

@@ -0,0 +1,9 @@
+FROM google/golang
+
+RUN cd /goroot/src/ && GOOS=linux GOARCH=386 ./make.bash --no-clean
+
+WORKDIR /gopath/src/github.com/buger/gor/
+
+ADD . /gopath/src/github.com/buger/gor/
+
+RUN go get

ELASTICSEARCH.md

@@ -0,0 +1,65 @@
+gor & elasticsearch
+===================
+
+Prerequisites
+-------------
+
+- elasticsearch
+- kibana (Get it here: http://www.elasticsearch.org/overview/kibana/)
+- gor
+
+
+elasticsearch
+-------------
+
+The default elasticsearch configuration is just fine for most workloads. You won't need clustering, sharding or something like that.
+
+In this example we're installing it on our gor replay server which gives us the elasticsearch listener on _http://localhost:9200_
+
+
+kibana
+------
+
+Kibana (elasticsearch analytics web-ui) is just as simple. 
+Download it, extract it and serve it via a simple webserver.
+(Could be nginx or apache)
+
+You could also use a shell, ```cd``` into the kibana directory and start a little quick and dirty python webserver with:
+
+```
+python -m SimpleHTTPServer 8000
+```
+
+In this example we're also choosing the gor replay server as our kibana host. If you choose a different server you'll have to point kibana to your elasticsearch host.
+
+
+gor
+---
+
+Start your gor replay server with elasticsearch option:
+
+```
+./gor --input-raw :8000 --output-http http://staging.com  --output-http-elasticsearch localhost:9200/gor
+```
+
+
+(You don't have to create the index upfront. That will be done for you automatically)
+
+
+Now visit your kibana url, load the predefined dashboard from the gist https://gist.github.com/gottwald/b2c875037f24719a9616 and watch the data rush in.
+
+
+Troubleshooting
+---------------
+
+The replay process may complain about __too many open files__.
+That's because your typical linux shell has a small open files soft limit at 1024.
+You can easily raise that when you do this before starting your _gor replay_ process:
+
+```
+ulimit -n 64000
+```
+
+Please be aware, this is not a permanent setting. It's just valid for the following jobs you start from that shell.
+
+We reached the 1024 limit in our tests with a ubuntu box replaying about 9000 requests per minute. (We had very slow responses there, should be way more with fast responses)

LICENSE.txt

@@ -0,0 +1,191 @@
+Apache License
+Version 2.0, January 2004
+http://www.apache.org/licenses/
+
+TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+1. Definitions.
+
+"License" shall mean the terms and conditions for use, reproduction, and
+distribution as defined by Sections 1 through 9 of this document.
+
+"Licensor" shall mean the copyright owner or entity authorized by the copyright
+owner that is granting the License.
+
+"Legal Entity" shall mean the union of the acting entity and all other entities
+that control, are controlled by, or are under common control with that entity.
+For the purposes of this definition, "control" means (i) the power, direct or
+indirect, to cause the direction or management of such entity, whether by
+contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the
+outstanding shares, or (iii) beneficial ownership of such entity.
+
+"You" (or "Your") shall mean an individual or Legal Entity exercising
+permissions granted by this License.
+
+"Source" form shall mean the preferred form for making modifications, including
+but not limited to software source code, documentation source, and configuration
+files.
+
+"Object" form shall mean any form resulting from mechanical transformation or
+translation of a Source form, including but not limited to compiled object code,
+generated documentation, and conversions to other media types.
+
+"Work" shall mean the work of authorship, whether in Source or Object form, made
+available under the License, as indicated by a copyright notice that is included
+in or attached to the work (an example is provided in the Appendix below).
+
+"Derivative Works" shall mean any work, whether in Source or Object form, that
+is based on (or derived from) the Work and for which the editorial revisions,
+annotations, elaborations, or other modifications represent, as a whole, an
+original work of authorship. For the purposes of this License, Derivative Works
+shall not include works that remain separable from, or merely link (or bind by
+name) to the interfaces of, the Work and Derivative Works thereof.
+
+"Contribution" shall mean any work of authorship, including the original version
+of the Work and any modifications or additions to that Work or Derivative Works
+thereof, that is intentionally submitted to Licensor for inclusion in the Work
+by the copyright owner or by an individual or Legal Entity authorized to submit
+on behalf of the copyright owner. For the purposes of this definition,
+"submitted" means any form of electronic, verbal, or written communication sent
+to the Licensor or its representatives, including but not limited to
+communication on electronic mailing lists, source code control systems, and
+issue tracking systems that are managed by, or on behalf of, the Licensor for
+the purpose of discussing and improving the Work, but excluding communication
+that is conspicuously marked or otherwise designated in writing by the copyright
+owner as "Not a Contribution."
+
+"Contributor" shall mean Licensor and any individual or Legal Entity on behalf
+of whom a Contribution has been received by Licensor and subsequently
+incorporated within the Work.
+
+2. Grant of Copyright License.
+
+Subject to the terms and conditions of this License, each Contributor hereby
+grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free,
+irrevocable copyright license to reproduce, prepare Derivative Works of,
+publicly display, publicly perform, sublicense, and distribute the Work and such
+Derivative Works in Source or Object form.
+
+3. Grant of Patent License.
+
+Subject to the terms and conditions of this License, each Contributor hereby
+grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free,
+irrevocable (except as stated in this section) patent license to make, have
+made, use, offer to sell, sell, import, and otherwise transfer the Work, where
+such license applies only to those patent claims licensable by such Contributor
+that are necessarily infringed by their Contribution(s) alone or by combination
+of their Contribution(s) with the Work to which such Contribution(s) was
+submitted. If You institute patent litigation against any entity (including a
+cross-claim or counterclaim in a lawsuit) alleging that the Work or a
+Contribution incorporated within the Work constitutes direct or contributory
+patent infringement, then any patent licenses granted to You under this License
+for that Work shall terminate as of the date such litigation is filed.
+
+4. Redistribution.
+
+You may reproduce and distribute copies of the Work or Derivative Works thereof
+in any medium, with or without modifications, and in Source or Object form,
+provided that You meet the following conditions:
+
+You must give any other recipients of the Work or Derivative Works a copy of
+this License; and
+You must cause any modified files to carry prominent notices stating that You
+changed the files; and
+You must retain, in the Source form of any Derivative Works that You distribute,
+all copyright, patent, trademark, and attribution notices from the Source form
+of the Work, excluding those notices that do not pertain to any part of the
+Derivative Works; and
+If the Work includes a "NOTICE" text file as part of its distribution, then any
+Derivative Works that You distribute must include a readable copy of the
+attribution notices contained within such NOTICE file, excluding those notices
+that do not pertain to any part of the Derivative Works, in at least one of the
+following places: within a NOTICE text file distributed as part of the
+Derivative Works; within the Source form or documentation, if provided along
+with the Derivative Works; or, within a display generated by the Derivative
+Works, if and wherever such third-party notices normally appear. The contents of
+the NOTICE file are for informational purposes only and do not modify the
+License. You may add Your own attribution notices within Derivative Works that
+You distribute, alongside or as an addendum to the NOTICE text from the Work,
+provided that such additional attribution notices cannot be construed as
+modifying the License.
+You may add Your own copyright statement to Your modifications and may provide
+additional or different license terms and conditions for use, reproduction, or
+distribution of Your modifications, or for any such Derivative Works as a whole,
+provided Your use, reproduction, and distribution of the Work otherwise complies
+with the conditions stated in this License.
+
+5. Submission of Contributions.
+
+Unless You explicitly state otherwise, any Contribution intentionally submitted
+for inclusion in the Work by You to the Licensor shall be under the terms and
+conditions of this License, without any additional terms or conditions.
+Notwithstanding the above, nothing herein shall supersede or modify the terms of
+any separate license agreement you may have executed with Licensor regarding
+such Contributions.
+
+6. Trademarks.
+
+This License does not grant permission to use the trade names, trademarks,
+service marks, or product names of the Licensor, except as required for
+reasonable and customary use in describing the origin of the Work and
+reproducing the content of the NOTICE file.
+
+7. Disclaimer of Warranty.
+
+Unless required by applicable law or agreed to in writing, Licensor provides the
+Work (and each Contributor provides its Contributions) on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied,
+including, without limitation, any warranties or conditions of TITLE,
+NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are
+solely responsible for determining the appropriateness of using or
+redistributing the Work and assume any risks associated with Your exercise of
+permissions under this License.
+
+8. Limitation of Liability.
+
+In no event and under no legal theory, whether in tort (including negligence),
+contract, or otherwise, unless required by applicable law (such as deliberate
+and grossly negligent acts) or agreed to in writing, shall any Contributor be
+liable to You for damages, including any direct, indirect, special, incidental,
+or consequential damages of any character arising as a result of this License or
+out of the use or inability to use the Work (including but not limited to
+damages for loss of goodwill, work stoppage, computer failure or malfunction, or
+any and all other commercial damages or losses), even if such Contributor has
+been advised of the possibility of such damages.
+
+9. Accepting Warranty or Additional Liability.
+
+While redistributing the Work or Derivative Works thereof, You may choose to
+offer, and charge a fee for, acceptance of support, warranty, indemnity, or
+other liability obligations and/or rights consistent with this License. However,
+in accepting such obligations, You may act only on Your own behalf and on Your
+sole responsibility, not on behalf of any other Contributor, and only if You
+agree to indemnify, defend, and hold each Contributor harmless for any liability
+incurred by, or claims asserted against, such Contributor by reason of your
+accepting any such warranty or additional liability.
+
+END OF TERMS AND CONDITIONS
+
+APPENDIX: How to apply the Apache License to your work
+
+To apply the Apache License to your work, attach the following boilerplate
+notice, with the fields enclosed by brackets "[]" replaced with your own
+identifying information. (Don't include the brackets!) The text should be
+enclosed in the appropriate comment syntax for the file format. We also
+recommend that a file or class name and description of purpose be included on
+the same "printed page" as the copyright notice for easier identification within
+third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

Makefile

@@ -0,0 +1,37 @@
+SOURCE = emitter.go gor.go gor_stat.go input_dummy.go input_file.go input_raw.go input_tcp.go limiter.go output_dummy.go output_file.go input_http.go output_http.go output_tcp.go plugins.go settings.go test_input.go elasticsearch.go http_modifier.go http_modifier_settings.go http_client.go
+
+SOURCE_PATH = /gopath/src/github.com/buger/gor/
+
+release: release-x86 release-x64
+
+release-x64:
+	docker run -v `pwd`:$(SOURCE_PATH) -t --env GOOS=linux --env GOARCH=amd64 --env CGO_ENABLED=0 -i gor go build && tar -czf gor_x64.tar.gz gor && rm gor
+
+release-x86:
+	docker run -v `pwd`:$(SOURCE_PATH) -t --env GOOS=linux --env GOARCH=386 --env CGO_ENABLED=0 -i gor go build && tar -czf gor_x86.tar.gz gor && rm gor
+
+dbuild:
+	docker build -t gor .
+
+dtest:
+	docker run -v `pwd`:$(SOURCE_PATH) -t -i --env GORACE="halt_on_error=1" gor go test ./... $(ARGS) -race -v -timeout 15s
+
+dcover:
+	docker run -v `pwd`:$(SOURCE_PATH) -t -i --env GORACE="halt_on_error=1" gor go test $(ARGS) -race -v -timeout 15s -coverprofile=coverage.out
+	go tool cover -html=coverage.out
+
+dfmt:
+	docker run -v `pwd`:$(SOURCE_PATH) -t -i gor go fmt
+
+dvet:
+	docker run -v `pwd`:$(SOURCE_PATH) -t -i gor go vet
+
+dbench:
+	docker run -v `pwd`:$(SOURCE_PATH) -t -i gor go test -v -run NOT_EXISTING -bench HTTP
+
+# Used mainly for debugging, because docker container do not have access to parent machine ports
+drun:
+	docker run -v `pwd`:$(SOURCE_PATH) -t -i gor go run $(SOURCE) --input-dummy=0 --output-http="http://localhost:9000"  --verbose -h
+
+dbash: 
+	docker run -v `pwd`:$(SOURCE_PATH) -t -i gor /bin/bash

Procfile

@@ -0,0 +1,4 @@
+web: python -m SimpleHTTPServer 8000
+replayed_web: python -m SimpleHTTPServer 8001
+listener: sudo -E go run ./bin/gor.go --input-raw :8000 --output-tcp :8002 --verbose
+replay: go run ./bin/gor.go --input-tcp :8002 --output-http localhost:8001 --verbose

README.md

@@ -1,88 +1,400 @@
-## About
-
-Gor is a simple http traffic replication tool written in Go. 
-Its main goal is to replay traffic from production servers to staging and dev environments.
-
-
-Now you can test your code on real user sessions in an automated and repeatable fashion.  
-**No more falling down in production!**
-
-Gor consists of 2 parts: listener and replay servers.
-
-The listener server catches http traffic from a given port in real-time
-and sends it to the replay server via UDP. 
-The replay server forwards traffic to a given address.
-
-
-![Diagram](http://i.imgur.com/zZCFPCY.png)
-
-
-## Basic example
-
-```bash
-# Run on servers where you want to catch traffic. You can run it on each `web` machine.
-sudo gor listen -p 80 -r replay.server.local:28020 
-
-# Replay server (replay.server.local). 
-gor replay -f http://staging.server
-```
-
-## Advanced use
-
-### Rate limiting
-The replay server supports rate limiting. It can be useful if you want
-forward only part of production traffic and not overload your staging
-environment. You can specify your desired requests per second using the
-"|" operator after the server address:
-
-```
-# staging.server will not get more than 10 requests per second
-gor replay -f "http://staging.server|10"
-```
-
-### Forward to multiple addresses
-
-You can forward traffic to multiple endpoints. Just separate the addresses by coma.
-```
-gor replay -f "http://staging.server|10,http://dev.server|5"
-```
-
-## Additional help
-```
-$ gor listen -h
-Usage of ./bin/gor-linux:
-  -i="any": By default it try to listen on all network interfaces.To get list of interfaces run `ifconfig`
-  -p=80: Specify the http server port whose traffic you want to capture
-  -r="localhost:28020": Address of replay server.
-```
-
-```
-$ gor replay -h
-Usage of ./bin/gor-linux:
-  -f="http://localhost:8080": http address to forward traffic.
-	You can limit requests per second by adding `|#{num}` after address.
-	If you have multiple addresses with different limits. For example: http://staging.example.com|100,http://dev.example.com|10
-  -ip="0.0.0.0": ip addresses to listen on
-  -p=28020: specify port number
-```
-
-## Pre-build binaries
-
-[Download binaries (linux 32/64, darwin)](https://drive.google.com/folderview?id=0B46uay48NwcfWFowc1E4a1BISVU&usp=sharing)
-
-## FAQ
-
-### Why does the `gor listener` requires sudo or root access?
-Listener works by sniffing traffic from a given port. It's accessible
-only by using sudo or root access.
-
-### Do you support all http request types?
-Yes. ~~Right now it supports only "GET" requests.~~
-
-## TODO
-
-Use buffering for request throttling instead of simple rate limiting. 
-
-Better stats
-
-Optimize for load testing cases
+[![Build Status](https://travis-ci.org/buger/gor.png?branch=master)](https://travis-ci.org/buger/gor)
+
+## About
+
+Gor is a simple http traffic replication tool written in Go.
+Its main goal is to replay traffic from production servers to staging and dev environments.
+
+
+Now you can test your code on real user sessions in an automated and repeatable fashion.
+**No more falling down in production!**
+
+Here is basic workflow: The listener server catches http traffic and sends it to the replay server or saves to file. The replay server forwards traffic to a given address.
+
+
+![Diagram](http://i.imgur.com/9mqj2SK.png)
+
+
+## Examples
+
+### Capture traffic from port
+```bash
+# Run on servers where you want to catch traffic. You can run it on each `web` machine.
+sudo gor --input-raw :80 --output-tcp replay.local:28020
+
+# Replay server (replay.local).
+gor --input-tcp replay.local:28020 --output-http http://staging.com
+```
+
+Since Gor use raw sockets to capture traffic it require `sudo` access. Alternatively you can allow access to raw sockets like this: `sudo setcap CAP_NET_RAW=ep gor`
+
+### Using 1 Gor instance for both listening and replaying
+It's recommended to use separate server for replaying traffic, but if you have enough CPU resources you can use single Gor instance.
+
+```
+sudo gor --input-raw :80 --output-http "http://staging.com"
+```
+
+### Guarantee of replay and HTTP input
+Due to how traffic interception works, there is chance of missing requests. If you want guarantee that requests will be replayed you can use http input, but it will require changes in your app as well. 
+
+```
+sudo gor --input-http :28019 --output-http "http://staging.com"
+```
+
+Then in your application you should send copy (e.g. like reverse proxy) all incoming requests to Gor http input. 
+
+## Configuration
+
+### Forward to multiple addresses
+
+You can forward traffic to multiple endpoints. Just add multiple --output-* arguments.
+```
+gor --input-tcp :28020 --output-http "http://staging.com"  --output-http "http://dev.com"
+```
+
+#### Splitting traffic
+By default it will send same traffic to all outputs, but you have options to equally split it:
+
+```
+gor --input-tcp :28020 --output-http "http://staging.com"  --output-http "http://dev.com" --split-output true
+```
+
+### HTTP output workers
+By default Gor creates dynamic pull of workers: it starts with 10 and create more http output workers when the http output queue length is greater than 10.  The number of workers created (N) is equal to the queue length at the time which it is checked and found to have a length greater than 10. The queue length is checked every time a message is written to the http output queue.  No more workers will be spawned until that request to spawn N workers is satisfied.  If a dynamic worker cannot process a message at that time, it will sleep for 100 milliseconds. If a dynamic worker cannot process a message for 2 seconds it dies.  
+You may specify fixed number of workers using  `--output-http-workers=20` option.
+
+### Follow redirects
+By default Gor will ignore all redirects since they are handled by clients using your app, but in scenarios when your replayed environment introduce new redirects, you can enable them like this: 
+```
+gor --input-tcp replay.local:28020 --output-http http://staging.com --output-http-redirects 2
+```
+The given example will follow up to 2 redirects per request.
+
+### Rate limiting
+Rate limiting can be useful if you want forward only part of production traffic and not overload your staging environment. There is 2 strategies: dropping random requests or dropping fraction of requests based on Header or URL param value. 
+
+#### Dropping random requests
+Every input and output support random rate limiting.
+There are 2 limiting algorithms: absolute or percentage based. 
+
+Absolute: If for current second it reached specified requests limit - disregard the rest, on next second counter reseted.
+
+Percentage: For input-file it will slowdown or speedup request execution, for the rest it will use random generator to decide if request pass or not based on chance you specified. 
+
+You can specify your desired limit using the
+"|" operator after the server address:
+
+#### Limiting replay using absolute number
+```
+# staging.server will not get more than 10 requests per second
+gor --input-tcp :28020 --output-http "http://staging.com|10"
+```
+
+#### Limiting listener using percentage based limiter
+```
+# replay server will not get more than 10% of requests 
+# useful for high-load environments
+gor --input-raw :80 --output-tcp "replay.local:28020|10%"
+```
+
+#### Limiting based on Header or URL param value
+If you have unique user id (like API key) stored in header or URL you can consistently forward specified percent of traffic only for fraction of this users. 
+Basic formula looks like this: `FNV32-1A_hashing(value) % 100 >= chance`. Examples:
+```
+# Limit based on header value
+gor --input-raw :80 --output-tcp "replay.local:28020|10%" --http-header-limiter "X-API-KEY: 10%"
+
+# Limit based on header value
+gor --input-raw :80 --output-tcp "replay.local:28020|10%" --http-param-limiter "api_key: 10%"
+```
+Only percentage based limiting supported.
+
+### Filtering 
+
+#### Allow url regexp
+```
+# only forward requests being sent to the /api endpoint
+gor --input-raw :8080 --output-http staging.com --http-allow-url /api
+```
+
+#### Disallow url regexp
+```
+# only forward requests NOT being sent to the /api... endpoint
+gor --input-raw :8080 --output-http staging.com --http-disallow-url /api
+```
+#### Filter based on regexp of header
+
+```
+# only forward requests with an api version of 1.0x
+gor --input-raw :8080 --output-http staging.com --http-allow-header api-version:^1\.0\d
+```
+
+#### Filter based on http method
+Requests not matching a specified whitelist can be filtered out. For example to strip non-nullipotent requests:
+
+```
+gor --input-raw :80 --output-http "http://staging.server" \
+    --http-allow-method GET \
+    --http-allow-method OPTIONS
+```
+
+### Rewriting original request
+Gor supports built-in basic rewriting support, for complex logic see https://github.com/buger/gor/pull/162
+
+#### Rewrite URL based on a mapping
+```
+# rewrite url to match the following
+gor --input-raw :8080 --output-http staging.com --http-rewrite-url /v1/user/([^\\/]+)/ping:/v2/user/$1/ping
+```
+
+#### Set URL param
+Set request url param, if param already exists it will be overwritten
+```
+gor --input-raw :8080 --output-http staging.com --http-set-param api_key=1
+```
+
+#### Set Header
+Set request header, if header already exists it will be overwritten. This may be useful if you need to identify requests generated by Gor or enable feature flagged functionality in an application:
+
+```
+gor --input-raw :80 --output-http "http://staging.server" \
+    --http-header "User-Agent: Replayed by Gor" \
+    --http-header "Enable-Feature-X: true"
+```
+
+### Saving requests to file and replaying them
+You can save requests to file, and replay them later:
+```
+# write to file
+gor --input-raw :80 --output-file requests.gor
+
+# read from file
+gor --input-file requests.gor --output-http "http://staging.com"
+```
+
+**Note:** Replay will preserve the original time differences between requests.
+
+### Load testing
+
+Currently it supported only by `input-file` and only when using percentage based limiter. Unlike default limiter for `input-file` instead of dropping requests it will slowdown or speedup request emitting. Note that unlike examples above limiter is applied to input:
+
+```
+# Replay from file on 2x speed 
+gor --input-file "requests.gor|200%" --output-http "staging.com"
+```
+
+### Basic Auth
+
+If your development or staging environment is protected by Basic Authentication then those credentials can be injected in during the replay:
+
+```
+gor --input-raw :80 --output-http "http://user:pass@staging .com"
+```
+
+Note: This will overwrite any Authorization headers in the original request.
+
+
+## Stats 
+
+Gor can report stats on the `output-tcp` and `output-http` request queues. Stats are reported to the console every 5 seconds in the form `latest,mean,max,count,count/second` by using the `--output-http-stats` and `--output-tcp-stats` options.
+
+Examples:
+
+```
+2014/04/23 21:17:50 output_tcp:latest,mean,max,count,count/second
+2014/04/23 21:17:50 output_tcp:0,0,0,0,0
+2014/04/23 21:17:55 output_tcp:1,1,2,68,13
+2014/04/23 21:18:00 output_tcp:1,1,2,92,18
+2014/04/23 21:18:05 output_tcp:1,1,2,119,23
+```
+
+```
+Version: 0.8
+2014/04/23 21:19:46 output_http:latest,mean,max,count,count/second
+2014/04/23 21:19:46 output_http:0,0,0,0,0
+2014/04/23 21:19:51 output_http:0,0,0,0,0
+2014/04/23 21:19:56 output_http:0,0,0,0,0
+2014/04/23 21:20:01 output_http:1,0,1,50,10
+2014/04/23 21:20:06 output_http:1,1,4,72,14
+2014/04/23 21:20:11 output_http:1,0,1,179,35
+2014/04/23 21:20:16 output_http:1,0,1,148,29
+2014/04/23 21:20:21 output_http:1,1,2,91,18
+2014/04/23 21:20:26 output_http:1,1,2,150,30
+2014/04/23 21:18:15 output_http:100,99,100,70,14
+2014/04/23 21:18:21 output_http:100,99,100,55,11
+```
+
+### How can I tell if I have bottlenecks?
+Key areas that sometimes experience bottlenecks are the output-tcp and output-http functions which have internal queues for requests. Each queue has an upper limit of 100. Enable stats reporting to see if any queues are experiencing bottleneck behavior.
+ 
+#### output-http bottlenecks
+When running a Gor replay the output-http feature may bottleneck if:
+
+  * the replay has inadequate bandwidth. If the replay is receiving or sending more messages than its network adapter can handle the output-http-stats  may report that the output-http queue is filling up. See if there is a way to upgrade the replay's bandwidth.
+  * with `--output-http-workers` set to anything other than `-1` the `-output-http` target is unable to respond to messages in a timely manner. The http output workers which take messages off the output-http queue, process the request, and ensure that the request did not result in an error may not be able to keep up with the number of incoming requests. If the replay is not using dynamic worker scaling (`--output-http-workers=-1`)  The optimal number of output-http-workers can be determined with the formula `output-workers = (Average number of requests per second)/(Average target response time per second)`.
+
+#### output-tcp bottlenecks
+When using the Gor listener the output-tcp feature may bottleneck if:
+
+  * the replay is unable to accept and process more requests than the listener is able generate. Prior to troubleshooting the output-tcp bottleneck, ensure that the replay target is not experiencing any bottlenecks. 
+  * the replay target has inadequate bandwidth to handle all its incoming requests.  If a replay target's incoming bandwidth is maxed out the output-tcp-stats may report that the output-tcp queue is filling up. See if there is a way to upgrade the replay's bandwidth.
+
+### ElasticSearch 
+For deep response analyze based on url, cookie, user-agent and etc. you can export response metadata to ElasticSearch. See [ELASTICSEARCH.md](ELASTICSEARCH.md) for more details.
+
+```
+gor --input-tcp :80 --output-http "http://staging.com" --output-http-elasticsearch "es_host:api_port/index_name"
+```
+
+## Additional help
+
+Feel free to ask question directly by email or by creating github issue.
+
+## Latest releases (including binaries)
+
+https://github.com/buger/gor/releases
+
+## Command line reference
+`gor -h` output:
+```
+  -http-allow-header=[]: A regexp to match a specific header against. Requests with non-matching headers will be dropped:
+   gor --input-raw :8080 --output-http staging.com --http-allow-header api-version:^v1
+  -http-allow-method=[]: Whitelist of HTTP methods to replay. Anything else will be dropped:
+  gor --input-raw :8080 --output-http staging.com --http-allow-method GET --http-allow-method OPTIONS
+  -http-allow-url=[]: A regexp to match requests against. Filter get matched agains full url with domain. Anything else will be dropped:
+   gor --input-raw :8080 --output-http staging.com --http-allow-url ^www.
+  -http-diallow-url=[]: A regexp to match requests against. Filter get matched agains full url with domain. Anything else will be dropped:
+   gor --input-raw :8080 --output-http staging.com --http-disallow-url ^www.
+  -http-header-limiter=[]: Takes a fraction of requests, consistently taking or rejecting a request based on the FNV32-1A hash of a specific header:
+   gor --input-raw :8080 --output-http staging.com --http-header-imiter user-id:25%
+  -http-param-limiter=[]: Takes a fraction of requests, consistently taking or rejecting a request based on the FNV32-1A hash of a specific GET param:
+   gor --input-raw :8080 --output-http staging.com --http-param-limiter user_id:25%
+  -http-rewrite-url=[]: Rewrite the request url based on a mapping:
+  gor --input-raw :8080 --output-http staging.com --http-rewrite-url /v1/user/([^\/]+)/ping:/v2/user/$1/ping
+  -http-set-header=[]: Inject additional headers to http reqest:
+  gor --input-raw :8080 --output-http staging.com --http-set-header 'User-Agent: Gor'
+  -http-set-param=[]: Set request url param, if param already exists it will be overwritten:
+  gor --input-raw :8080 --output-http staging.com --http-set-param api_key=1
+  -input-dummy=[]: Used for testing outputs. Emits 'Get /' request every 1s
+  -input-file=[]: Read requests from file:
+  gor --input-file ./requests.gor --output-http staging.com
+  -input-http=[]: Read requests from HTTP, should be explicitly sent from your application:
+  # Listen for http on 9000
+  gor --input-http :9000 --output-http staging.com
+  -input-raw=[]: Capture traffic from given port (use RAW sockets and require *sudo* access):
+  # Capture traffic from 8080 port
+  gor --input-raw :8080 --output-http staging.com
+  -input-tcp=[]: Used for internal communication between Gor instances. Example:
+  # Receive requests from other Gor instances on 28020 port, and redirect output to staging
+  gor --input-tcp :28020 --output-http staging.com
+  -memprofile="": write memory profile to this file
+  -output-dummy=[]: Used for testing inputs. Just prints data coming from inputs.
+  -output-file=[]: Write incoming requests to file:
+  gor --input-raw :80 --output-file ./requests.gor
+  -output-http=[]: Forwards incoming requests to given http address.
+  # Redirect all incoming requests to staging.com address
+  gor --input-raw :80 --output-http http://staging.com
+  -output-http-elasticsearch="": Send request and response stats to ElasticSearch:
+  gor --input-raw :8080 --output-http staging.com --output-http-elasticsearch 'es_host:api_port/index_name'
+  -output-http-header-filter=[]: WARNING: `--output-http-header-filter` DEPRECATED, use `--http-allow-header` instead  -output-http-redirects=0: Enable how often redirects should be followed.
+  -output-http-stats=false: Report http output queue stats to console every 5 seconds.
+  -output-http-workers=0: Gor uses dynamic worker scaling by default.  Enter a number to run a set number of workers.
+  -output-tcp=[]: Used for internal communication between Gor instances. Example:
+  # Listen for requests on 80 port and forward them to other Gor instance on 28020 port
+  gor --input-raw :80 --output-tcp replay.local:28020
+  -output-tcp-stats=false: Report TCP output queue stats to console every 5 seconds.
+  -split-output=false: By default each output gets same traffic. If set to `true` it splits traffic equally among all outputs.
+  -stats=false: Turn on queue stats output
+  -verbose=false: Turn on verbose/debug output
+```
+
+## Building from source
+
+1. Setup standard Go environment http://golang.org/doc/code.html and ensure that $GOPATH environment variable properly set.
+2. `go get github.com/buger/gor`.
+3. `cd $GOPATH/src/github.com/buger/gor`
+4. `go build` to get binary, or `go test` to run tests
+
+## Development
+Project contains Docker environment.
+
+1. Build container: `make dbuild`
+2. Run all tests: `make dtest`. Run specific test: `make dtest ARGS=-test.run=**regexp**`
+3. Bash access to container: `make dbash`. Inside container you have python to run simple web server `python -m SimpleHTTPServer 8080` and `curl` to make http requests. 
+
+
+## Questions and support 
+
+All bug-reports and suggestions should go though Github Issues or our [Google Group](https://groups.google.com/forum/#!forum/gor-users). Or you can just send email to gor-users@googlegroups.com
+
+If you have some private questions you can send direct mail to leonsbox@gmail.com
+
+## FAQ
+
+### What OS are supported?
+For now only Linux based. *BSD (including MacOS is not supported yet, check https://github.com/buger/gor/issues/22 for details)
+
+### Why does the `--input-raw` requires sudo or root access?
+Listener works by sniffing traffic from a given port. It's accessible
+only by using sudo or root access.
+
+### How do you deal with user session to replay the traffic correctly?
+You can rewrite session related headers/params to match your staging environment. If you require custom logic (e.g random token based auth) follow this discussion: https://github.com/buger/gor/issues/154
+
+### Can i use Gor to intercept SSL traffic?
+Basic idea is that SSL was made to protect itself from traffic interception. There 2 options: 
+1. Move SSL handling to proxy like Nginx or Amazon ELB. And allow Gor to listen on upstreams. 
+2. Use `--input-http` so you can duplicate request payload directly from your app to Gor, but it will require your app modifications.
+
+More can be find here: https://github.com/buger/gor/issues/85
+
+### I'm getting 'too many open files' error
+Typical linux shell has a small open files soft limit at 1024. You can easily raise that when you do this before starting your gor replay process:
+  
+  ulimit -n 64000
+
+More about ulimit: http://blog.thecodingmachine.com/content/solving-too-many-open-files-exception-red5-or-any-other-application
+
+### The CPU average across my load-balanced targets is higher than the source
+If you are replaying traffic from multiple listeners to a load-balanced target and you use sticky sessions, you may observe that the target servers have a higher CPU load than the listener servers. This may be because the sticky session cookie of the original load balancer is not honored by the target load balancer thus resulting in requests that would normally hit the same target server hitting different servers on the backend thus reducing some caching benefits gained via the load balancing.  Try running just one listener against one replay target and see if the CPU utilization comparison is more accurate.
+
+## Tuning
+
+To achieve the top most performance you should tune the source server system limits:
+
+    net.ipv4.tcp_max_tw_buckets = 65536
+    net.ipv4.tcp_tw_recycle = 1
+    net.ipv4.tcp_tw_reuse = 0
+    net.ipv4.tcp_max_syn_backlog = 131072
+    net.ipv4.tcp_syn_retries = 3
+    net.ipv4.tcp_synack_retries = 3
+    net.ipv4.tcp_retries1 = 3
+    net.ipv4.tcp_retries2 = 8
+    net.ipv4.tcp_rmem = 16384 174760 349520
+    net.ipv4.tcp_wmem = 16384 131072 262144
+    net.ipv4.tcp_mem = 262144 524288 1048576
+    net.ipv4.tcp_max_orphans = 65536
+    net.ipv4.tcp_fin_timeout = 10
+    net.ipv4.tcp_low_latency = 1
+    net.ipv4.tcp_syncookies = 0
+
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (git checkout -b my-new-feature)
+3. Commit your changes (git commit -am 'Added some feature')
+4. Push to the branch (git push origin my-new-feature)
+5. Create new Pull Request
+
+## Companies using Gor
+
+* [Granify](http://granify.com)
+* [GOV.UK](https://www.gov.uk) ([Government Digital Service](http://digital.cabinetoffice.gov.uk/))
+* [theguardian.com](http://theguardian.com)
+* [TomTom](http://www.tomtom.com/)
+* [3SCALE](http://www.3scale.net/)
+* [Optionlab](http://www.opinionlab.com)
+* [TubeMogul] (http://tubemogul.com)
+* To add your company drop me a line to github.com/buger or leonsbox@gmail.com

byteutils/byteutils.go

@@ -0,0 +1,37 @@
+package byteutils
+
+func Cut(a []byte, from, to int) []byte {
+    copy(a[from:], a[to:])
+    a = a[:len(a)-to+from]
+
+    return a
+}
+
+func Insert(a []byte, i int, b []byte) []byte {
+    a = append(a, make([]byte, len(b))...)
+    copy(a[i+len(b):], a[i:])
+    copy(a[i:i+len(b)], b)
+
+    return a
+}
+
+// Unlike bytes.Replace it allows you to specify range
+func Replace(a []byte, from, to int, new []byte) []byte {
+    lenDiff := len(new) - (to - from)
+
+    if lenDiff > 0 {
+        // Extend if new segment bigger
+        a = append(a, make([]byte, lenDiff)...)
+        copy(a[to+lenDiff:], a[to:])
+        copy(a[from:from+len(new)], new)
+
+        return a
+    } else if lenDiff < 0 {
+        copy(a[from:], new)
+        copy(a[from+len(new):],a[to:])
+        return a[:len(a) + lenDiff]
+    } else { // same size
+        copy(a[from:], new)
+        return a
+    }
+}

byteutils/byteutils_test.go

@@ -0,0 +1,32 @@
+package byteutils
+
+import (
+    "testing"
+    "bytes"
+)
+
+func TestCut(t *testing.T) {
+    if !bytes.Equal(Cut([]byte("123456"), 2, 4), []byte("1256")) {
+        t.Error("Should properly cut")
+    }
+}
+
+func TestInsert(t *testing.T) {
+    if !bytes.Equal(Insert([]byte("123456"), 2, []byte("abcd")), []byte("12abcd3456")) {
+        t.Error("Should insert into middle of slice")
+    }
+}
+
+func TestReplace(t *testing.T) {
+    if !bytes.Equal(Replace([]byte("123456"), 2, 4, []byte("ab")), []byte("12ab56")) {
+        t.Error("Should replace when same length")
+    }
+
+    if !bytes.Equal(Replace([]byte("123456"), 2, 4, []byte("abcd")), []byte("12abcd56")) {
+        t.Error("Should replace when replacement length bigger")
+    }
+
+    if !bytes.Equal(Replace([]byte("123456"), 2, 5, []byte("ab")), []byte("12ab6")) {
+        t.Error("Should replace when replacement length bigger")
+    }
+}

elasticsearch.go

@@ -0,0 +1,153 @@
+package main
+
+import (
+	"encoding/json"
+	"github.com/buger/elastigo/api"
+	"github.com/buger/elastigo/core"
+	"github.com/buger/gor/proto"
+	"log"
+	"regexp"
+	"time"
+)
+
+type ESUriErorr struct{}
+
+func (e *ESUriErorr) Error() string {
+	return "Wrong ElasticSearch URL format. Expected to be: host:port/index_name"
+}
+
+type ESPlugin struct {
+	Active  bool
+	ApiPort string
+	Host    string
+	Index   string
+	indexor *core.BulkIndexer
+	done    chan bool
+}
+
+type ESRequestResponse struct {
+	ReqUrl               []byte `json:"Req_URL"`
+	ReqMethod            []byte `json:"Req_Method"`
+	ReqUserAgent         []byte `json:"Req_User-Agent"`
+	ReqAcceptLanguage    []byte `json:"Req_Accept-Language,omitempty"`
+	ReqAccept            []byte `json:"Req_Accept,omitempty"`
+	ReqAcceptEncoding    []byte `json:"Req_Accept-Encoding,omitempty"`
+	ReqIfModifiedSince   []byte `json:"Req_If-Modified-Since,omitempty"`
+	ReqConnection        []byte `json:"Req_Connection,omitempty"`
+	ReqCookies           []byte `json:"Req_Cookies,omitempty"`
+	RespStatus           []byte `json:"Resp_Status"`
+	RespStatusCode       []byte `json:"Resp_Status-Code"`
+	RespProto            []byte `json:"Resp_Proto,omitempty"`
+	RespContentLength    []byte `json:"Resp_Content-Length,omitempty"`
+	RespContentType      []byte `json:"Resp_Content-Type,omitempty"`
+	RespTransferEncoding []byte `json:"Resp_Transfer-Encoding,omitempty"`
+	RespContentEncoding  []byte `json:"Resp_Content-Encoding,omitempty"`
+	RespExpires          []byte `json:"Resp_Expires,omitempty"`
+	RespCacheControl     []byte `json:"Resp_Cache-Control,omitempty"`
+	RespVary             []byte `json:"Resp_Vary,omitempty"`
+	RespSetCookie        []byte `json:"Resp_Set-Cookie,omitempty"`
+	Rtt                  int64  `json:"RTT"`
+	Timestamp            time.Time
+}
+
+// Parse ElasticSearch URI
+//
+// Proper format is: host:port/index_name
+func parseURI(URI string) (err error, host string, port string, index string) {
+	rURI := regexp.MustCompile("(.+):([0-9]+)/(.+)")
+	match := rURI.FindAllStringSubmatch(URI, -1)
+
+	if len(match) == 0 {
+		err = new(ESUriErorr)
+	} else {
+		host = match[0][1]
+		port = match[0][2]
+		index = match[0][3]
+	}
+
+	return
+}
+
+func (p *ESPlugin) Init(URI string) {
+	var err error
+
+	err, p.Host, p.ApiPort, p.Index = parseURI(URI)
+
+	if err != nil {
+		log.Fatal("Can't initialize ElasticSearch plugin.", err)
+	}
+
+	api.Domain = p.Host
+	api.Port = p.ApiPort
+
+	p.indexor = core.NewBulkIndexerErrors(50, 60)
+	p.done = make(chan bool)
+	p.indexor.Run(p.done)
+
+	// Only start the ErrorHandler goroutine when in verbose mode
+	// no need to burn ressources otherwise
+	// go p.ErrorHandler()
+
+	log.Println("Initialized Elasticsearch Plugin")
+	return
+}
+
+func (p *ESPlugin) IndexerShutdown() {
+	p.done <- true
+	return
+}
+
+func (p *ESPlugin) ErrorHandler() {
+	for {
+		errBuf := <-p.indexor.ErrorChannel
+		log.Println(errBuf.Err)
+	}
+}
+
+func (p *ESPlugin) RttDurationToMs(d time.Duration) int64 {
+	sec := d / time.Second
+	nsec := d % time.Second
+	fl := float64(sec) + float64(nsec)*1e-6
+	return int64(fl)
+}
+
+func (p *ESPlugin) ResponseAnalyze(req, resp []byte, start, stop time.Time) {
+	if len(resp) == 0 {
+		// nil http response - skipped elasticsearch export for this request
+		return
+	}
+	t := time.Now()
+	rtt := p.RttDurationToMs(stop.Sub(start))
+
+	esResp := ESRequestResponse{
+		ReqUrl:               proto.Path(req),
+		ReqMethod:            proto.Method(req),
+		ReqUserAgent:         proto.GetHeader(req, "User-Agent"),
+		ReqAcceptLanguage:    proto.GetHeader(req, "Accept-Language"),
+		ReqAccept:            proto.GetHeader(req, "Accept"),
+		ReqAcceptEncoding:    proto.GetHeader(req, "Accept-Encoding"),
+		ReqIfModifiedSince:   proto.GetHeader(req, "If-Modified-Since"),
+		ReqConnection:        proto.GetHeader(req, "Connection"),
+		ReqCookies:           proto.GetHeader(req, "Cookie"),
+		RespStatus:           proto.Status(resp),
+		RespStatusCode:       proto.Status(resp),
+		RespProto:            proto.Method(resp),
+		RespContentLength:    proto.GetHeader(resp, "Content-Length"),
+		RespContentType:      proto.GetHeader(resp, "Content-Type"),
+		RespTransferEncoding: proto.GetHeader(resp, "Transfer-Encoding"),
+		RespContentEncoding:  proto.GetHeader(resp, "Content-Encoding"),
+		RespExpires:          proto.GetHeader(resp, "Expires"),
+		RespCacheControl:     proto.GetHeader(resp, "Cache-Control"),
+		RespVary:             proto.GetHeader(resp, "Vary"),
+		RespSetCookie:        proto.GetHeader(resp, "Set-Cookie"),
+		Rtt:                  rtt,
+		Timestamp:            t,
+	}
+	j, err := json.Marshal(&esResp)
+	if err != nil {
+		log.Println(err)
+	} else {
+		p.indexor.Index(p.Index, "RequestResponse", "", "", &t, j, true)
+	}
+	return
+}

emitter.go

@@ -0,0 +1,71 @@
+package main
+
+import (
+	"io"
+	"time"
+)
+
+func Start(stop chan int) {
+	for _, in := range Plugins.Inputs {
+		go CopyMulty(in, Plugins.Outputs...)
+	}
+
+	for {
+		select {
+		case <-stop:
+			return
+		case <-time.After(1 * time.Second):
+		}
+	}
+}
+
+// Copy from 1 reader to multiple writers
+func CopyMulty(src io.Reader, writers ...io.Writer) (err error) {
+	buf := make([]byte, 5*1024*1024)
+	wIndex := 0
+	modifier := NewHTTPModifier(&Settings.modifierConfig)
+
+	for {
+		nr, er := src.Read(buf)
+		if nr > 0 && len(buf) > nr {
+			payload := buf[0:nr]
+
+			if modifier != nil {
+				payload = modifier.Rewrite(payload)
+
+				// If modifier tells to skip request
+				if len(payload) == 0 {
+					continue
+				}
+			}
+
+			if Settings.debug {
+				Debug("[EMITTER] Sending paylod:", string(payload))
+			}
+
+			if Settings.splitOutput {
+				// Simple round robin
+				writers[wIndex].Write(payload)
+
+				wIndex++
+
+				if wIndex >= len(writers) {
+					wIndex = 0
+				}
+			} else {
+				for _, dst := range writers {
+					dst.Write(payload)
+				}
+			}
+
+		}
+		if er == io.EOF {
+			break
+		}
+		if er != nil {
+			err = er
+			break
+		}
+	}
+	return err
+}

emitter_test.go

@@ -0,0 +1,99 @@
+package main
+
+import (
+	"io"
+	"sync"
+	"sync/atomic"
+	"testing"
+)
+
+func TestEmitter(t *testing.T) {
+	wg := new(sync.WaitGroup)
+	quit := make(chan int)
+
+	input := NewTestInput()
+	output := NewTestOutput(func(data []byte) {
+		wg.Done()
+	})
+
+	Plugins.Inputs = []io.Reader{input}
+	Plugins.Outputs = []io.Writer{output}
+
+	go Start(quit)
+
+	for i := 0; i < 1000; i++ {
+		wg.Add(1)
+		input.EmitGET()
+	}
+
+	wg.Wait()
+
+	close(quit)
+}
+
+func TestEmitterRoundRobin(t *testing.T) {
+	wg := new(sync.WaitGroup)
+	quit := make(chan int)
+
+	input := NewTestInput()
+
+	var counter1, counter2 int32
+
+	output1 := NewTestOutput(func(data []byte) {
+		atomic.AddInt32(&counter1, 1)
+		wg.Done()
+	})
+
+	output2 := NewTestOutput(func(data []byte) {
+		atomic.AddInt32(&counter2, 1)
+		wg.Done()
+	})
+
+	Plugins.Inputs = []io.Reader{input}
+	Plugins.Outputs = []io.Writer{output1, output2}
+
+	Settings.splitOutput = true
+
+	go Start(quit)
+
+	for i := 0; i < 1000; i++ {
+		wg.Add(1)
+		input.EmitGET()
+	}
+
+	wg.Wait()
+
+	close(quit)
+
+	if counter1 == 0 || counter2 == 0 {
+		t.Errorf("Round robin should split traffic equally: %d vs %d", counter1, counter2)
+	}
+
+	Settings.splitOutput = false
+}
+
+func BenchmarkEmitter(b *testing.B) {
+	wg := new(sync.WaitGroup)
+	quit := make(chan int)
+
+	input := NewTestInput()
+
+	output := NewTestOutput(func(data []byte) {
+		wg.Done()
+	})
+
+	Plugins.Inputs = []io.Reader{input}
+	Plugins.Outputs = []io.Writer{output}
+
+	go Start(quit)
+
+	b.ResetTimer()
+
+	for i := 0; i < b.N; i++ {
+		wg.Add(1)
+		input.EmitGET()
+	}
+
+	wg.Wait()
+	close(quit)
+}

gor.go

@@ -1,46 +1,78 @@
 // Gor is simple http traffic replication tool written in Go. Its main goal to replay traffic from production servers to staging and dev environments.
 // Now you can test your code on real user sessions in an automated and repeatable fashion.
-//
-// Gor consists of 2 parts: listener and replay servers.
-// Listener catch http traffic from given port in real-time and send it to replay server via UDP. Replay server forwards traffic to given address.
 package main
 
 import (
 	"flag"
 	"fmt"
-	"github.com/buger/gor/listener"
-	"github.com/buger/gor/replay"
+	"log"
 	"os"
+	"runtime/debug"
+	"runtime/pprof"
+	"time"
 )
 
-const (
-	VERSION = "0.3"
+var (
+	mode       string
+	cpuprofile = flag.String("cpuprofile", "", "write cpu profile to file")
+	memprofile = flag.String("memprofile", "", "write memory profile to this file")
 )
 
 func main() {
+	// Don't exit on panic
+	defer func() {
+		if r := recover(); r != nil {
+			if _, ok := r.(error); !ok {
+				fmt.Printf("PANIC: pkg: %v %s \n", r, debug.Stack())
+			}
+		}
+	}()
+
 	fmt.Println("Version:", VERSION)
 
-	mode := "unknown"
-
-	if len(os.Args) > 1 {
-		mode = os.Args[1]
-	}
-
-	if mode != "listen" && mode != "replay" {
-		fmt.Println("Usage: \n\tgor listen -h\n\tgor replay -h")
-		return
-	}
-
-	// Remove mode attr
-	os.Args = append(os.Args[:1], os.Args[2:]...)
-
 	flag.Parse()
+	InitPlugins()
 
-	switch mode {
-	case "listen":
-		listener.Run()
-	case "replay":
-		replay.Run()
+	if len(Plugins.Inputs) == 0 || len(Plugins.Outputs) == 0 {
+		log.Fatal("Required at least 1 input and 1 output")
 	}
 
+	if *memprofile != "" {
+		profileMEM(*memprofile)
+	}
+
+	if *cpuprofile != "" {
+		profileCPU(*cpuprofile)
+	}
+
+	Start(nil)
+}
+
+func profileCPU(cpuprofile string) {
+	if cpuprofile != "" {
+		f, err := os.Create(cpuprofile)
+		if err != nil {
+			log.Fatal(err)
+		}
+		pprof.StartCPUProfile(f)
+
+		time.AfterFunc(60*time.Second, func() {
+			pprof.StopCPUProfile()
+			f.Close()
+			log.Println("Stop profiling after 60 seconds")
+		})
+	}
+}
+
+func profileMEM(memprofile string) {
+	if memprofile != "" {
+		f, err := os.Create(memprofile)
+		if err != nil {
+			log.Fatal(err)
+		}
+		time.AfterFunc(60*time.Second, func() {
+			pprof.WriteHeapProfile(f)
+			f.Close()
+		})
+	}
 }

gor_stat.go

@@ -0,0 +1,66 @@
+package main
+
+import (
+	"log"
+	"strconv"
+	"time"
+)
+
+const (
+	rate = 5
+)
+
+type GorStat struct {
+	statName string
+	latest   int
+	mean     int
+	max      int
+	count    int
+}
+
+func NewGorStat(statName string) (s *GorStat) {
+	s = new(GorStat)
+	s.statName = statName
+	s.latest = 0
+	s.mean = 0
+	s.max = 0
+	s.count = 0
+
+	if Settings.stats {
+		log.Println(s.statName + ":latest,mean,max,count,count/second")
+		go s.reportStats()
+	}
+	return
+}
+
+func (s *GorStat) Write(latest int) {
+	if Settings.stats {
+		if latest > s.max {
+			s.max = latest
+		}
+		if latest != 0 {
+			s.mean = (s.mean + latest) / 2
+		}
+		s.latest = latest
+		s.count = s.count + 1
+	}
+}
+
+func (s *GorStat) Reset() {
+	s.latest = 0
+	s.max = 0
+	s.mean = 0
+	s.count = 0
+}
+
+func (s *GorStat) String() string {
+	return s.statName + ":" + strconv.Itoa(s.latest) + "," + strconv.Itoa(s.mean) + "," + strconv.Itoa(s.max) + "," + strconv.Itoa(s.count) + "," + strconv.Itoa(s.count/rate)
+}
+
+func (s *GorStat) reportStats() {
+	for {
+		log.Println(s)
+		s.Reset()
+		time.Sleep(rate * time.Second)
+	}
+}

http_client.go

@@ -0,0 +1,165 @@
+package main
+
+import (
+	"crypto/tls"
+	"github.com/buger/gor/proto"
+	"io"
+	"net"
+	"net/url"
+	"strings"
+	"time"
+	"runtime/debug"
+	"log"
+)
+
+var defaultPorts = map[string]string{
+	"http":  "80",
+	"https": "443",
+}
+
+type HTTPClientConfig struct {
+	FollowRedirects int
+	Debug           bool
+}
+
+type HTTPClient struct {
+	baseURL        string
+	scheme         string
+	host           string
+	conn           net.Conn
+	respBuf        []byte
+	config         *HTTPClientConfig
+	redirectsCount int
+}
+
+func NewHTTPClient(baseURL string, config *HTTPClientConfig) *HTTPClient {
+	if !strings.HasPrefix(baseURL, "http") {
+		baseURL = "http://" + baseURL
+	}
+
+	u, _ := url.Parse(baseURL)
+	if !strings.Contains(u.Host, ":") {
+		u.Host += ":" + defaultPorts[u.Scheme]
+	}
+
+	client := new(HTTPClient)
+	client.baseURL = u.String()
+	client.host = u.Host
+	client.scheme = u.Scheme
+	client.respBuf = make([]byte, 4096*10)
+	client.config = config
+
+	return client
+}
+
+func (c *HTTPClient) Connect() (err error) {
+	c.Disconnect()
+
+	c.conn, err = net.Dial("tcp", c.host)
+
+	if c.scheme == "https" {
+		tlsConn := tls.Client(c.conn, &tls.Config{InsecureSkipVerify: true})
+
+		if err = tlsConn.Handshake(); err != nil {
+			return
+		}
+
+		c.conn = tlsConn
+	}
+
+	return
+}
+
+func (c *HTTPClient) Disconnect() {
+	if c.conn != nil {
+		c.conn.Close()
+		c.conn = nil
+		Debug("Disconnected: ", c.baseURL)
+	}
+}
+
+func (c *HTTPClient) isAlive() bool {
+	one := make([]byte, 1)
+
+	// Ready 1 byte from socket without timeout to check if it not closed
+	c.conn.SetReadDeadline(time.Now().Add(time.Millisecond))
+	if _, err := c.conn.Read(one); err == io.EOF {
+		return false
+	}
+
+	return true
+}
+
+func (c *HTTPClient) Send(data []byte) (response []byte, err error) {
+	// Don't exit on panic
+	defer func() {
+		if r := recover(); r != nil {
+			Debug("[HTTPClient]", r, string(data))
+
+			if _, ok := r.(error); !ok {
+				log.Println("[HTTPClient] Failed to send request: ", string(data))
+				log.Println("PANIC: pkg:", r, debug.Stack())
+			}
+		}
+	}()
+
+
+	if c.conn == nil || !c.isAlive() {
+		Debug("[HTTPClient] Connecting:", c.baseURL)
+		if err = c.Connect(); err != nil {
+			log.Println("[HTTPClient] Connection error:", err)
+			return
+		}
+	}
+
+	timeout := time.Now().Add(5 * time.Second)
+
+	c.conn.SetWriteDeadline(timeout)
+
+	data = proto.SetHost(data, []byte(c.baseURL), []byte(c.host))
+
+	if c.config.Debug {
+		Debug("[HTTPClient] Sending:", string(data))
+	}
+
+	if _, err = c.conn.Write(data); err != nil {
+		Debug("[HTTPClient] Write error:", err, c.baseURL)
+		return
+	}
+
+	c.conn.SetReadDeadline(timeout)
+	n, err := c.conn.Read(c.respBuf)
+
+	if err != nil {
+		Debug("[HTTPClient] Response read error", err, c.conn)
+		return
+	}
+
+	payload := c.respBuf[:n]
+
+	if c.config.Debug {
+		Debug("[HTTPClient] Received:", string(payload))
+	}
+
+	if c.config.FollowRedirects > 0 && c.redirectsCount < c.config.FollowRedirects {
+		status := payload[9:12]
+
+		// 3xx requests
+		if status[0] == '3' {
+			c.redirectsCount += 1
+
+			location, _, _, _ := proto.Header(payload, []byte("Location"))
+			redirectPayload := []byte("GET " + string(location) + " HTTP/1.1\r\n\r\n")
+
+			if c.config.Debug {
+				Debug("[HTTPClient] Redirecting to: " + string(location))
+			}
+
+			return c.Send(redirectPayload)
+		}
+	}
+
+	c.redirectsCount = 0
+
+	return payload, err
+}

http_client_test.go

@@ -0,0 +1,269 @@
+package main
+
+import (
+	"io/ioutil"
+	"net"
+	"net/http"
+	"net/http/httptest"
+	"net/http/httputil"
+	"sync"
+	"testing"
+	_ "time"
+)
+
+func TestHTTPClientURLPort(t *testing.T) {
+	c1 := NewHTTPClient("http://example.com", &HTTPClientConfig{})
+	if c1.baseURL != "http://example.com:80" {
+		t.Error("Sould add 80 port for http:", c1.baseURL)
+	}
+
+	c2 := NewHTTPClient("https://example.com", &HTTPClientConfig{})
+	if c2.baseURL != "https://example.com:443" {
+		t.Error("Sould add 443 port for https:", c2.baseURL)
+	}
+
+	c3 := NewHTTPClient("https://example.com:1", &HTTPClientConfig{})
+	if c3.baseURL != "https://example.com:1" {
+		t.Error("Sould use specified port:", c3.baseURL)
+	}
+
+	c4 := NewHTTPClient("example.com", &HTTPClientConfig{})
+	if c4.baseURL != "http://example.com:80" {
+		t.Error("Sould add default protocol:", c4.baseURL)
+	}
+}
+
+func TestHTTPClientSend(t *testing.T) {
+	wg := new(sync.WaitGroup)
+
+	payload := func(reqType string) []byte {
+		switch reqType {
+		case "GET":
+			return []byte("GET / HTTP/1.1\r\n\r\n")
+		case "POST":
+			return []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
+		case "POST_CHUNKED":
+			return []byte("POST / HTTP/1.1\r\nHost: www.w3.org\r\nTransfer-Encoding: chunked\r\n\r\n4\r\nWiki\r\n5\r\npedia\r\ne\r\n in\r\n\r\nchunks.\r\n0\r\n\r\n")
+		}
+
+		return []byte("")
+	}
+
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+
+		if r.Method == "POST" {
+			defer r.Body.Close()
+			body, _ := ioutil.ReadAll(r.Body)
+
+			if len(r.TransferEncoding) > 0 && r.TransferEncoding[0] == "chunked" {
+				if string(body) != "Wikipedia in\r\n\r\nchunks." {
+					t.Error("Wrong POST body:", body, string(body))
+				}
+			} else {
+				if string(body) != "a=1&b=2" {
+					buf, _ := httputil.DumpRequest(r, true)
+					t.Error("Wrong POST body:", string(body), string(buf))
+				}
+			}
+		}
+
+		wg.Done()
+	}))
+
+	client := NewHTTPClient(server.URL, &HTTPClientConfig{Debug: true})
+
+	wg.Add(4)
+	client.Send(payload("POST"))
+	client.Send(payload("GET"))
+	client.Send(payload("POST_CHUNKED"))
+	client.Send(payload("POST"))
+
+	wg.Wait()
+}
+
+func TestHTTPClientHTTPSSend(t *testing.T) {
+	wg := new(sync.WaitGroup)
+
+	payload := func(reqType string) []byte {
+		switch reqType {
+		case "GET":
+			return []byte("GET / HTTP/1.1\r\n\r\n")
+		case "POST":
+			return []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
+		case "POST_CHUNKED":
+			return []byte("POST / HTTP/1.1\r\nHost: www.w3.org\r\nTransfer-Encoding: chunked\r\n\r\n4\r\nWiki\r\n5\r\npedia\r\ne\r\n in\r\n\r\nchunks.\r\n0\r\n\r\n")
+		}
+
+		return []byte("")
+	}
+
+	server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+
+		if r.Method == "POST" {
+			defer r.Body.Close()
+			body, _ := ioutil.ReadAll(r.Body)
+
+			if len(r.TransferEncoding) > 0 && r.TransferEncoding[0] == "chunked" {
+				if string(body) != "Wikipedia in\r\n\r\nchunks." {
+					t.Error("Wrong POST body:", body, string(body))
+				}
+			} else {
+				if string(body) != "a=1&b=2" {
+					buf, _ := httputil.DumpRequest(r, true)
+					t.Error("Wrong POST body:", string(body), string(buf))
+				}
+			}
+		}
+
+		wg.Done()
+	}))
+
+	client := NewHTTPClient(server.URL, &HTTPClientConfig{})
+
+	wg.Add(4)
+	client.Send(payload("POST"))
+	client.Send(payload("GET"))
+	client.Send(payload("POST_CHUNKED"))
+	client.Send(payload("POST"))
+
+	wg.Wait()
+}
+
+func TestHTTPClientServerInstantDisconnect(t *testing.T) {
+	wg := new(sync.WaitGroup)
+
+	GET_payload := []byte("GET / HTTP/1.1\r\n\r\n")
+
+	ln, _ := net.Listen("tcp", ":0")
+
+	go func() {
+		for {
+			conn, _ := ln.Accept()
+			conn.Close()
+
+			wg.Done()
+		}
+	}()
+
+	client := NewHTTPClient(ln.Addr().String(), &HTTPClientConfig{})
+
+	wg.Add(2)
+	client.Send(GET_payload)
+	client.Send(GET_payload)
+
+	wg.Wait()
+}
+
+func TestHTTPClientServerNoKeepAlive(t *testing.T) {
+	wg := new(sync.WaitGroup)
+
+	GET_payload := []byte("GET / HTTP/1.1\r\n\r\n")
+
+	ln, _ := net.Listen("tcp", ":0")
+
+	go func() {
+		for {
+			conn, err := ln.Accept()
+			if err != nil {
+				// handle error
+			}
+
+			buf := make([]byte, 4096)
+			reqLen, err := conn.Read(buf)
+			if err != nil {
+				t.Error("Error reading:", err.Error())
+			}
+			Debug("Received: ", string(buf[0:reqLen]))
+			conn.Write([]byte("OK"))
+
+			// No keep-alive connections
+			conn.Close()
+
+			wg.Done()
+		}
+	}()
+
+	client := NewHTTPClient(ln.Addr().String(), &HTTPClientConfig{})
+
+	wg.Add(2)
+	client.Send(GET_payload)
+	client.Send(GET_payload)
+
+	wg.Wait()
+}
+
+func TestHTTPClientRedirect(t *testing.T) {
+	wg := new(sync.WaitGroup)
+
+	GET_payload := []byte("GET / HTTP/1.1\r\n\r\n")
+
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+
+		if r.URL.Path == "/" {
+			http.Redirect(w, r, "/new", 301)
+		}
+
+		wg.Done()
+	}))
+
+	client := NewHTTPClient(server.URL, &HTTPClientConfig{FollowRedirects: 1, Debug: false})
+
+	// Should do 2 queries
+	wg.Add(2)
+	client.Send(GET_payload)
+
+	wg.Wait()
+}
+
+func TestHTTPClientRedirectLimit(t *testing.T) {
+	wg := new(sync.WaitGroup)
+
+	GET_payload := []byte("GET / HTTP/1.1\r\n\r\n")
+
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+
+		if r.URL.Path == "/" {
+			http.Redirect(w, r, "/r1", 301)
+		}
+
+		if r.URL.Path == "/r1" {
+			http.Redirect(w, r, "/r2", 301)
+		}
+
+		if r.URL.Path == "/r2" {
+			http.Redirect(w, r, "/new", 301)
+		}
+
+		wg.Done()
+	}))
+
+	client := NewHTTPClient(server.URL, &HTTPClientConfig{FollowRedirects: 2, Debug: false})
+
+	// Have 3 redirects + 1 GET, but should do only 2 redirects + GET
+	wg.Add(3)
+	client.Send(GET_payload)
+
+	wg.Wait()
+}
+
+func TestHTTPClientHandleHTTP10(t *testing.T) {
+	wg := new(sync.WaitGroup)
+
+	GET_payload := []byte("GET http://foobar.com/path HTTP/1.0\r\n\r\n")
+
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+
+		if r.URL.Path != "/path" {
+			t.Error("Path not match:", r.URL.Path)
+		}
+
+		wg.Done()
+	}))
+
+	client := NewHTTPClient(server.URL, &HTTPClientConfig{Debug: true})
+
+	wg.Add(1)
+	client.Send(GET_payload)
+
+	wg.Wait()
+}

http_modifier.go

@@ -0,0 +1,141 @@
+package main
+
+import (
+	"bytes"
+	"github.com/buger/gor/proto"
+	"hash/fnv"
+)
+
+type HTTPModifier struct {
+	config *HTTPModifierConfig
+}
+
+func NewHTTPModifier(config *HTTPModifierConfig) *HTTPModifier {
+	// Optimization to skip modifier completely if we do not need it
+	if len(config.urlRegexp) == 0 &&
+		len(config.urlNegativeRegexp) == 0 &&
+		len(config.urlRewrite) == 0 &&
+		len(config.headerFilters) == 0 &&
+		len(config.headerHashFilters) == 0 &&
+		len(config.paramHashFilters) == 0 &&
+		len(config.params) == 0 &&
+		len(config.headers) == 0 &&
+		len(config.methods) == 0 {
+		return nil
+	}
+
+	return &HTTPModifier{config: config}
+}
+
+func (m *HTTPModifier) Rewrite(payload []byte) (response []byte) {
+	if len(m.config.methods) > 0 {
+		method := proto.Method(payload)
+
+		matched := false
+
+		for _, m := range m.config.methods {
+			if bytes.Equal(method, m) {
+				matched = true
+				break
+			}
+		}
+
+		if !matched {
+			return
+		}
+	}
+
+	if len(m.config.headers) > 0 {
+		for _, header := range m.config.headers {
+			payload = proto.SetHeader(payload, []byte(header.Name), []byte(header.Value))
+		}
+	}
+
+	if len(m.config.params) > 0 {
+		for _, param := range m.config.params {
+			payload = proto.SetPathParam(payload, param.Name, param.Value)
+		}
+	}
+
+	if len(m.config.urlRegexp) > 0 {
+		path := proto.Path(payload)
+
+		matched := false
+
+		for _, f := range m.config.urlRegexp {
+			if f.regexp.Match(path) {
+				matched = true
+				break
+			}
+		}
+
+		if !matched {
+			return
+		}
+	}
+
+	if len(m.config.urlNegativeRegexp) > 0 {
+		path := proto.Path(payload)
+
+		for _, f := range m.config.urlNegativeRegexp {
+			if f.regexp.Match(path) {
+				return
+			}
+		}
+	}
+
+	if len(m.config.headerFilters) > 0 {
+		for _, f := range m.config.headerFilters {
+			value, s, _, _ := proto.Header(payload, f.name)
+
+			if s != -1 && !f.regexp.Match(value) {
+				return
+			}
+		}
+	}
+
+	if len(m.config.headerHashFilters) > 0 {
+		for _, f := range m.config.headerHashFilters {
+			value, s, _, _ := proto.Header(payload, f.name)
+
+			if s != -1 {
+				hasher := fnv.New32a()
+				hasher.Write(value)
+
+				if (hasher.Sum32() % 100) >= f.percent {
+					return
+				}
+			}
+		}
+	}
+
+	if len(m.config.paramHashFilters) > 0 {
+		for _, f := range m.config.paramHashFilters {
+			value, s, _ := proto.PathParam(payload, f.name)
+
+			if s != -1 {
+				hasher := fnv.New32a()
+				hasher.Write(value)
+
+				if (hasher.Sum32() % 100) >= f.percent {
+					return
+				}
+			}
+		}
+	}
+
+	if len(m.config.urlRewrite) > 0 {
+		path := proto.Path(payload)
+
+		for _, f := range m.config.urlRewrite {
+			if f.src.Match(path) {
+				path = f.src.ReplaceAll(path, f.target)
+				payload = proto.SetPath(payload, path)
+
+				break
+			}
+		}
+	}
+
+	return payload
+}

http_modifier_settings.go

@@ -0,0 +1,212 @@
+package main
+
+import (
+	"errors"
+	"fmt"
+	"regexp"
+	"strconv"
+	"strings"
+)
+
+type HTTPModifierConfig struct {
+	urlNegativeRegexp HTTPUrlRegexp
+	urlRegexp         HTTPUrlRegexp
+	urlRewrite        UrlRewriteMap
+	headerFilters     HTTPHeaderFilters
+	headerHashFilters HTTPHashFilters
+	paramHashFilters  HTTPHashFilters
+
+	params  HTTPParams
+	headers HTTPHeaders
+	methods HTTPMethods
+}
+
+//
+// Handling of --http-allow-header options
+//
+type headerFilter struct {
+	name   []byte
+	regexp *regexp.Regexp
+}
+
+type HTTPHeaderFilters []headerFilter
+
+func (h *HTTPHeaderFilters) String() string {
+	return fmt.Sprint(*h)
+}
+
+func (h *HTTPHeaderFilters) Set(value string) error {
+	valArr := strings.SplitN(value, ":", 2)
+	if len(valArr) < 2 {
+		return errors.New("need both header and value, colon-delimited (ex. user_id:^169$).")
+	}
+	r, err := regexp.Compile(valArr[1])
+	if err != nil {
+		return err
+	}
+
+	*h = append(*h, headerFilter{name: []byte(valArr[0]), regexp: r})
+
+	return nil
+}
+
+//
+// Handling of --http-allow-header-hash and --http-allow-param-hash options
+//
+type hashFilter struct {
+	name    []byte
+	percent uint32
+}
+
+type HTTPHashFilters []hashFilter
+
+func (h *HTTPHashFilters) String() string {
+	return fmt.Sprint(*h)
+}
+
+func (h *HTTPHashFilters) Set(value string) error {
+	valArr := strings.SplitN(value, ":", 2)
+	if len(valArr) < 2 {
+		return errors.New("need both header and value, colon-delimited (ex. user_id:50%).")
+	}
+
+	f := hashFilter{name: []byte(valArr[0])}
+
+	if strings.Contains(valArr[1], "%") {
+		p, _ := strconv.ParseInt(valArr[1][:len(valArr[1])-1], 0, 0)
+		f.percent = uint32(p)
+	} else if strings.Contains(valArr[1], "/") {
+		// DEPRECATED format
+		var num, den uint64
+
+		fracArr := strings.Split(valArr[1], "/")
+		num, _ = strconv.ParseUint(fracArr[0], 10, 64)
+		den, _ = strconv.ParseUint(fracArr[1], 10, 64)
+
+		f.percent = uint32((float64(num) / float64(den)) * 100)
+	} else {
+		return errors.New("Value should be percent and contain '%'")
+	}
+
+	*h = append(*h, f)
+
+	return nil
+}
+
+//
+// Handling of --http-set-header option
+//
+type HTTPHeaders []HTTPHeader
+type HTTPHeader struct {
+	Name  string
+	Value string
+}
+
+func (h *HTTPHeaders) String() string {
+	return fmt.Sprint(*h)
+}
+
+func (h *HTTPHeaders) Set(value string) error {
+	v := strings.SplitN(value, ":", 2)
+	if len(v) != 2 {
+		return errors.New("Expected `Key: Value`")
+	}
+
+	header := HTTPHeader{
+		strings.TrimSpace(v[0]),
+		strings.TrimSpace(v[1]),
+	}
+
+	*h = append(*h, header)
+	return nil
+}
+
+//
+// Handling of --http-set-param option
+//
+type HTTPParams []HTTPParam
+type HTTPParam struct {
+	Name  []byte
+	Value []byte
+}
+
+func (h *HTTPParams) String() string {
+	return fmt.Sprint(*h)
+}
+
+func (h *HTTPParams) Set(value string) error {
+	v := strings.SplitN(value, "=", 2)
+	if len(v) != 2 {
+		return errors.New("Expected `Key=Value`")
+	}
+
+	param := HTTPParam{
+		[]byte(strings.TrimSpace(v[0])),
+		[]byte(strings.TrimSpace(v[1])),
+	}
+
+	*h = append(*h, param)
+	return nil
+}
+
+//
+// Handling of --http-allow-method option
+//
+type HTTPMethods [][]byte
+
+func (h *HTTPMethods) String() string {
+	return fmt.Sprint(*h)
+}
+
+func (h *HTTPMethods) Set(value string) error {
+	*h = append(*h, []byte(value))
+	return nil
+}
+
+//
+// Handling of --http-rewrite-url option
+//
+type urlRewrite struct {
+	src    *regexp.Regexp
+	target []byte
+}
+
+type UrlRewriteMap []urlRewrite
+
+func (r *UrlRewriteMap) String() string {
+	return fmt.Sprint(*r)
+}
+
+func (r *UrlRewriteMap) Set(value string) error {
+	valArr := strings.SplitN(value, ":", 2)
+	if len(valArr) < 2 {
+		return errors.New("need both src and target, colon-delimited (ex. /a:/b).")
+	}
+	regexp, err := regexp.Compile(valArr[0])
+	if err != nil {
+		return err
+	}
+	*r = append(*r, urlRewrite{src: regexp, target: []byte(valArr[1])})
+	return nil
+}
+
+//
+// Handling of --http-allow-url option
+//
+type urlRegexp struct {
+	regexp *regexp.Regexp
+}
+
+type HTTPUrlRegexp []urlRegexp
+
+func (r *HTTPUrlRegexp) String() string {
+	return fmt.Sprint(*r)
+}
+
+func (r *HTTPUrlRegexp) Set(value string) error {
+	regexp, err := regexp.Compile(value)
+
+	*r = append(*r, urlRegexp{regexp: regexp})
+
+	return err
+}

http_modifier_settings_test.go

@@ -0,0 +1,65 @@
+package main
+
+import (
+	"testing"
+)
+
+func TestHTTPHeaderFilters(t *testing.T) {
+	filters := HTTPHeaderFilters{}
+
+	err := filters.Set("Header1:^$")
+	if err != nil {
+		t.Error("Should not error on Header1:^$")
+	}
+
+	err = filters.Set("Header2:^:$")
+	if err != nil {
+		t.Error("Should not error on Header2:^:$")
+	}
+
+	// Missing colon
+	err = filters.Set("Header3-^$")
+	if err == nil {
+		t.Error("Should error on Header2:^:$")
+	}
+}
+
+func TestHTTPHashFilters(t *testing.T) {
+	filters := HTTPHashFilters{}
+
+	err := filters.Set("Header1:1/2")
+	if err != nil {
+		t.Error("Should support old syntax")
+	}
+
+	if filters[0].percent != 50 {
+		t.Error("Wrong percentage", filters[0].percent)
+	}
+
+	err = filters.Set("Header2:1")
+	if err == nil {
+		t.Error("Should error on Header2 because no % symbol")
+	}
+
+	err = filters.Set("Header2:10%")
+	if err != nil {
+		t.Error("Should pass")
+	}
+
+	if filters[1].percent != 10 {
+		t.Error("Wrong percentage", filters[1].percent)
+	}
+}
+
+func TestUrlRewriteMap(t *testing.T) {
+	var err error
+	rewrites := UrlRewriteMap{}
+
+	if err = rewrites.Set("/v1/user/([^\\/]+)/ping:/v2/user/$1/ping"); err != nil {
+		t.Error("Should set mapping", err)
+	}
+
+	if err = rewrites.Set("/v1/user/([^\\/]+)/ping"); err == nil {
+		t.Error("Should not set mapping without :")
+	}
+}

http_modifier_test.go

@@ -0,0 +1,220 @@
+package main
+
+import (
+	"bytes"
+	"github.com/buger/gor/proto"
+	"testing"
+)
+
+func TestHTTPModifierWithoutConfig(t *testing.T) {
+	if NewHTTPModifier(&HTTPModifierConfig{}) != nil {
+		t.Error("If no config specified should not be initialized")
+	}
+}
+
+func TestHTTPModifierHeaderFilters(t *testing.T) {
+	filters := HTTPHeaderFilters{}
+	filters.Set("Host:^www.w3.org$")
+
+	modifier := NewHTTPModifier(&HTTPModifierConfig{
+		headerFilters: filters,
+	})
+
+	payload := []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
+
+	if len(modifier.Rewrite(payload)) == 0 {
+		t.Error("Request should pass filters")
+	}
+
+	filters = HTTPHeaderFilters{}
+	// Setting filter that not match our header
+	filters.Set("Host:^www.w4.org$")
+
+	modifier = NewHTTPModifier(&HTTPModifierConfig{
+		headerFilters: filters,
+	})
+
+	if len(modifier.Rewrite(payload)) != 0 {
+		t.Error("Request should not pass filters")
+	}
+}
+
+func TestHTTPModifierURLRewrite(t *testing.T) {
+	var url, new_url []byte
+
+	rewrites := UrlRewriteMap{}
+
+	payload := func(url []byte) []byte {
+		return []byte("POST " + string(url) + " HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
+	}
+
+	err := rewrites.Set("/v1/user/([^\\/]+)/ping:/v2/user/$1/ping")
+	if err != nil {
+		t.Error("Should not error on /v1/user/([^\\/]+)/ping:/v2/user/$1/ping")
+	}
+
+	modifier := NewHTTPModifier(&HTTPModifierConfig{
+		urlRewrite: rewrites,
+	})
+
+	url = []byte("/v1/user/joe/ping")
+	if new_url = proto.Path(modifier.Rewrite(payload(url))); bytes.Equal(new_url, url) {
+		t.Error("Request url should have been rewritten, wasn't", string(new_url))
+	}
+
+	url = []byte("/v1/user/ping")
+	if new_url = proto.Path(modifier.Rewrite(payload(url))); !bytes.Equal(new_url, url) {
+		t.Error("Request url should have been rewritten, wasn't", string(new_url))
+	}
+}
+
+func TestHTTPModifierHeaderHashFilters(t *testing.T) {
+	filters := HTTPHashFilters{}
+	filters.Set("Header2:1/2")
+
+	modifier := NewHTTPModifier(&HTTPModifierConfig{
+		headerHashFilters: filters,
+	})
+
+	payload := func(header []byte) []byte {
+		return []byte("POST / HTTP/1.1\r\n" + string(header) + "Content-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
+	}
+
+	if p := modifier.Rewrite(payload([]byte(""))); len(p) == 0 {
+		t.Error("Request should pass filters if Header does not exist")
+	}
+
+	if p := modifier.Rewrite(payload([]byte("Header2: 3\r\n"))); len(p) > 0 {
+		t.Error("Request should not pass filters, Header2 hash too high")
+	}
+
+	if p := modifier.Rewrite(payload([]byte("Header2: 1\r\n"))); len(p) == 0 {
+		t.Error("Request should pass filters")
+	}
+}
+
+func TestHTTPModifierParamHashFilters(t *testing.T) {
+	filters := HTTPHashFilters{}
+	filters.Set("user_id:1/2")
+
+	modifier := NewHTTPModifier(&HTTPModifierConfig{
+		paramHashFilters: filters,
+	})
+
+	payload := func(value []byte) []byte {
+		return []byte("POST /" + string(value) + " HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
+	}
+
+	if p := modifier.Rewrite(payload([]byte(""))); len(p) == 0 {
+		t.Error("Request should pass filters if param does not exist")
+	}
+
+	if p := modifier.Rewrite(payload([]byte("?user_id=3"))); len(p) > 0 {
+		t.Error("Request should not pass filters", string(p))
+	}
+
+	if p := modifier.Rewrite(payload([]byte("?user_id=1"))); len(p) == 0 {
+		t.Error("Request should pass filters")
+	}
+}
+
+func TestHTTPModifierHeaders(t *testing.T) {
+	headers := HTTPHeaders{}
+	headers.Set("Header1:1")
+	headers.Set("Host:localhost")
+
+	modifier := NewHTTPModifier(&HTTPModifierConfig{
+		headers: headers,
+	})
+
+	payload := []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
+	new_payload := []byte("POST /post HTTP/1.1\r\nHeader1: 1\r\nContent-Length: 7\r\nHost: localhost\r\n\r\na=1&b=2")
+
+	if payload = modifier.Rewrite(payload); !bytes.Equal(payload, new_payload) {
+		t.Error("Should update request headers", string(payload))
+	}
+}
+
+func TestHTTPModifierURLRegexp(t *testing.T) {
+	filters := HTTPUrlRegexp{}
+	filters.Set("/v1/app")
+	filters.Set("/v1/api")
+
+	modifier := NewHTTPModifier(&HTTPModifierConfig{
+		urlRegexp: filters,
+	})
+
+	payload := func(url string) []byte {
+		return []byte("POST " + url + " HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
+	}
+
+	if len(modifier.Rewrite(payload("/v1/app/test"))) == 0 {
+		t.Error("Should pass url")
+	}
+
+	if len(modifier.Rewrite(payload("/v1/api/test"))) == 0 {
+		t.Error("Should pass url")
+	}
+
+	if len(modifier.Rewrite(payload("/other"))) > 0 {
+		t.Error("Should not pass url")
+	}
+}
+
+func TestHTTPModifierURLNegativeRegexp(t *testing.T) {
+	filters := HTTPUrlRegexp{}
+	filters.Set("/restricted1")
+	filters.Set("/some/restricted2")
+
+	modifier := NewHTTPModifier(&HTTPModifierConfig{
+		urlNegativeRegexp: filters,
+	})
+
+	payload := func(url string) []byte {
+		return []byte("POST " + url + " HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
+	}
+
+	if len(modifier.Rewrite(payload("/v1/app/test"))) == 0 {
+		t.Error("Should pass url")
+	}
+
+	if len(modifier.Rewrite(payload("/restricted1"))) > 0 {
+		t.Error("Should not pass url")
+	}
+
+	if len(modifier.Rewrite(payload("/some/restricted2"))) > 0 {
+		t.Error("Should not pass url")
+	}
+}
+
+func TestHTTPModifierSetHeader(t *testing.T) {
+	filters := HTTPHeaders{}
+	filters.Set("User-Agent:Gor")
+
+	modifier := NewHTTPModifier(&HTTPModifierConfig{
+		headers: filters,
+	})
+
+	payload := []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
+	payload_after := []byte("POST /post HTTP/1.1\r\nUser-Agent: Gor\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
+
+	if payload = modifier.Rewrite(payload); !bytes.Equal(payload_after, payload) {
+		t.Error("Should add new header", string(payload))
+	}
+}
+
+func TestHTTPModifierSetParam(t *testing.T) {
+	filters := HTTPParams{}
+	filters.Set("api_key=1")
+
+	modifier := NewHTTPModifier(&HTTPModifierConfig{
+		params: filters,
+	})
+
+	payload := []byte("POST /post?api_key=1234 HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
+	payload_after := []byte("POST /post?api_key=1 HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
+
+	if payload = modifier.Rewrite(payload); !bytes.Equal(payload_after, payload) {
+		t.Error("Should override param", string(payload))
+	}
+}

input_dummy.go

@@ -0,0 +1,40 @@
+package main
+
+import (
+	"time"
+)
+
+type DummyInput struct {
+	data chan []byte
+}
+
+func NewDummyInput(options string) (di *DummyInput) {
+	di = new(DummyInput)
+	di.data = make(chan []byte)
+
+	go di.emit()
+
+	return
+}
+
+func (i *DummyInput) Read(data []byte) (int, error) {
+	buf := <-i.data
+	copy(data, buf)
+
+	return len(buf), nil
+}
+
+func (i *DummyInput) emit() {
+	ticker := time.NewTicker(time.Second)
+
+	for {
+		select {
+		case <-ticker.C:
+			i.data <- []byte("GET / HTTP/1.1\r\n\r\n")
+		}
+	}
+}
+
+func (i *DummyInput) String() string {
+	return "Dummy Input"
+}

input_file.go

@@ -0,0 +1,76 @@
+package main
+
+import (
+	"encoding/gob"
+	"log"
+	"os"
+	"time"
+)
+
+type FileInput struct {
+	data        chan []byte
+	path        string
+	decoder     *gob.Decoder
+	speedFactor float64
+}
+
+func NewFileInput(path string) (i *FileInput) {
+	i = new(FileInput)
+	i.data = make(chan []byte)
+	i.path = path
+	i.speedFactor = 1
+	i.Init(path)
+
+	go i.emit()
+
+	return
+}
+
+func (i *FileInput) Init(path string) {
+	file, err := os.Open(path)
+
+	if err != nil {
+		log.Fatal(i, "Cannot open file %q. Error: %s", path, err)
+	}
+
+	i.decoder = gob.NewDecoder(file)
+}
+
+func (i *FileInput) Read(data []byte) (int, error) {
+	buf := <-i.data
+	copy(data, buf)
+
+	return len(buf), nil
+}
+
+func (i *FileInput) String() string {
+	return "File input: " + i.path
+}
+
+func (i *FileInput) emit() {
+	var lastTime int64
+
+	for {
+		raw := new(RawRequest)
+		err := i.decoder.Decode(raw)
+
+		if err != nil {
+			return
+		}
+
+		if lastTime != 0 {
+			timeDiff := raw.Timestamp - lastTime
+
+			// We can speedup or slowdown execution based on speedFactor
+			if i.speedFactor != 1 {
+				timeDiff = int64(float64(raw.Timestamp-lastTime) / i.speedFactor)
+			}
+
+			time.Sleep(time.Duration(timeDiff))
+		}
+
+		lastTime = raw.Timestamp
+
+		i.data <- raw.Request
+	}
+}

input_http.go

@@ -0,0 +1,63 @@
+package main
+
+import (
+	"log"
+	"net"
+	"net/http"
+	"net/http/httputil"
+)
+
+type HTTPInput struct {
+	data     chan []byte
+	address  string
+	listener net.Listener
+}
+
+func NewHTTPInput(address string) (i *HTTPInput) {
+	i = new(HTTPInput)
+	i.data = make(chan []byte)
+	i.address = address
+
+	i.listen(address)
+
+	return
+}
+
+func (i *HTTPInput) Read(data []byte) (int, error) {
+	buf := <-i.data
+	copy(data, buf)
+
+	return len(buf), nil
+}
+
+func (i *HTTPInput) handler(w http.ResponseWriter, r *http.Request) {
+	buf, _ := httputil.DumpRequest(r, true)
+
+	i.data <- buf
+
+	http.Error(w, http.StatusText(200), 200)
+}
+
+func (i *HTTPInput) listen(address string) {
+	var err error
+
+	mux := http.NewServeMux()
+
+	mux.HandleFunc("/", i.handler)
+
+	i.listener, err = net.Listen("tcp", address)
+	if err != nil {
+		log.Fatal("HTTP input listener failure:", err)
+	}
+
+	go func() {
+		err = http.Serve(i.listener, mux)
+		if err != nil {
+			log.Fatal("HTTP input serve failure:", err)
+		}
+	}()
+}
+
+func (i *HTTPInput) String() string {
+	return "HTTP input: " + i.address
+}

input_http_test.go

@@ -0,0 +1,35 @@
+package main
+
+import (
+	"io"
+	"net/http"
+	"strings"
+	"sync"
+	"testing"
+)
+
+func TestHTTPInput(t *testing.T) {
+	wg := new(sync.WaitGroup)
+	quit := make(chan int)
+
+	input := NewHTTPInput(":0")
+	output := NewTestOutput(func(data []byte) {
+		wg.Done()
+	})
+
+	Plugins.Inputs = []io.Reader{input}
+	Plugins.Outputs = []io.Writer{output}
+
+	go Start(quit)
+
+	address := strings.Replace(input.listener.Addr().String(), "[::]", "127.0.0.1", -1)
+
+	for i := 0; i < 100; i++ {
+		wg.Add(1)
+		http.Get("http://" + address + "/")
+	}
+
+	wg.Wait()
+
+	close(quit)
+}

input_raw.go

@@ -0,0 +1,53 @@
+package main
+
+import (
+	raw "github.com/buger/gor/raw_socket_listener"
+	"log"
+	"net"
+	"strings"
+)
+
+type RAWInput struct {
+	data    chan []byte
+	address string
+}
+
+func NewRAWInput(address string) (i *RAWInput) {
+	i = new(RAWInput)
+	i.data = make(chan []byte)
+	i.address = address
+
+	go i.listen(address)
+
+	return
+}
+
+func (i *RAWInput) Read(data []byte) (int, error) {
+	buf := <-i.data
+	copy(data, buf)
+
+	return len(buf), nil
+}
+
+func (i *RAWInput) listen(address string) {
+	address = strings.Replace(address, "[::]", "127.0.0.1", -1)
+
+	host, port, err := net.SplitHostPort(address)
+
+	if err != nil {
+		log.Fatal("input-raw: error while parsing address", err)
+	}
+
+	listener := raw.NewListener(host, port)
+
+	for {
+		// Receiving TCPMessage object
+		m := listener.Receive()
+
+		i.data <- m.Bytes()
+	}
+}
+
+func (i *RAWInput) String() string {
+	return "RAW Socket input: " + i.address
+}

input_raw_test.go

@@ -0,0 +1,152 @@
+package main
+
+import (
+	"bytes"
+	"io"
+	"io/ioutil"
+	"log"
+	"net/http"
+	"net/http/httputil"
+	"os/exec"
+	"strings"
+	"sync"
+	"testing"
+)
+
+func TestRAWInput(t *testing.T) {
+
+	wg := new(sync.WaitGroup)
+	quit := make(chan int)
+
+	listener := startHTTP(func(req *http.Request) {})
+
+	input := NewRAWInput(listener.Addr().String())
+	output := NewTestOutput(func(data []byte) {
+		wg.Done()
+	})
+
+	Plugins.Inputs = []io.Reader{input}
+	Plugins.Outputs = []io.Writer{output}
+
+	address := strings.Replace(listener.Addr().String(), "[::]", "127.0.0.1", -1)
+
+	go Start(quit)
+
+	for i := 0; i < 100; i++ {
+		wg.Add(1)
+		res, _ := http.Get("http://" + address)
+		res.Body.Close()
+	}
+
+	wg.Wait()
+
+	close(quit)
+}
+
+func TestInputRAW100Expect(t *testing.T) {
+	wg := new(sync.WaitGroup)
+	quit := make(chan int)
+
+	file_content, _ := ioutil.ReadFile("README.md")
+
+	// Origing and Replay server initialization
+	origin := startHTTP(func(req *http.Request) {
+		defer req.Body.Close()
+		ioutil.ReadAll(req.Body)
+
+		wg.Done()
+	})
+
+	origin_address := strings.Replace(origin.Addr().String(), "[::]", "127.0.0.1", -1)
+
+	input := NewRAWInput(origin_address)
+
+	// We will use it to get content of raw HTTP request
+	test_output := NewTestOutput(func(data []byte) {
+		if strings.Contains(string(data), "Expect: 100-continue") {
+			t.Error("Should not contain 100-continue header")
+		}
+		wg.Done()
+	})
+
+	listener := startHTTP(func(req *http.Request) {
+		defer req.Body.Close()
+		body, _ := ioutil.ReadAll(req.Body)
+
+		if !bytes.Equal(body, file_content) {
+			buf, _ := httputil.DumpRequest(req, true)
+			t.Error("Wrong POST body:", string(buf))
+		}
+
+		wg.Done()
+	})
+	replay_address := listener.Addr().String()
+
+	http_output := NewHTTPOutput(replay_address, &HTTPOutputConfig{})
+
+	Plugins.Inputs = []io.Reader{input}
+	Plugins.Outputs = []io.Writer{test_output, http_output}
+
+	go Start(quit)
+
+	wg.Add(3)
+	curl := exec.Command("curl", "http://"+origin_address, "--data-binary", "@README.md")
+	err := curl.Run()
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	wg.Wait()
+	close(quit)
+}
+
+func TestInputRAWChunkedEncoding(t *testing.T) {
+	wg := new(sync.WaitGroup)
+	quit := make(chan int)
+
+	file_content, _ := ioutil.ReadFile("README.md")
+
+	// Origing and Replay server initialization
+	origin := startHTTP(func(req *http.Request) {
+		defer req.Body.Close()
+		ioutil.ReadAll(req.Body)
+
+		wg.Done()
+	})
+
+	origin_address := strings.Replace(origin.Addr().String(), "[::]", "127.0.0.1", -1)
+
+	input := NewRAWInput(origin_address)
+
+	listener := startHTTP(func(req *http.Request) {
+		defer req.Body.Close()
+		body, _ := ioutil.ReadAll(req.Body)
+
+		if !bytes.Equal(body, file_content) {
+			buf, _ := httputil.DumpRequest(req, true)
+			t.Error("Wrong POST body:", string(buf))
+		}
+
+		wg.Done()
+	})
+	replay_address := listener.Addr().String()
+
+	http_output := NewHTTPOutput(replay_address, &HTTPOutputConfig{Debug: true})
+
+	Plugins.Inputs = []io.Reader{input}
+	Plugins.Outputs = []io.Writer{http_output}
+
+	go Start(quit)
+
+	wg.Add(2)
+
+	curl := exec.Command("curl", "http://"+origin_address, "--header", "Transfer-Encoding: chunked", "--data-binary", "@README.md")
+	err := curl.Run()
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	wg.Wait()
+
+	close(quit)
+}

input_tcp.go

@@ -0,0 +1,81 @@
+package main
+
+import (
+	"bufio"
+	"encoding/hex"
+	"fmt"
+	"log"
+	"net"
+	"os"
+)
+
+// Can be tested using nc tool:
+//    echo "asdad" | nc 127.0.0.1 27017
+//
+type TCPInput struct {
+	data     chan []byte
+	address  string
+	listener net.Listener
+}
+
+func NewTCPInput(address string) (i *TCPInput) {
+	i = new(TCPInput)
+	i.data = make(chan []byte)
+	i.address = address
+
+	i.listen(address)
+
+	return
+}
+
+func (i *TCPInput) Read(data []byte) (int, error) {
+	buf := <-i.data
+	copy(data, buf)
+
+	return len(buf), nil
+}
+
+func (i *TCPInput) listen(address string) {
+	listener, err := net.Listen("tcp", address)
+	i.listener = listener
+
+	if err != nil {
+		log.Fatal("Can't start:", err)
+	}
+
+	go func() {
+		for {
+			conn, err := listener.Accept()
+
+			if err != nil {
+				log.Println("Error while Accept()", err)
+				continue
+			}
+
+			go i.handleConnection(conn)
+		}
+	}()
+}
+
+func (i *TCPInput) handleConnection(conn net.Conn) {
+	defer conn.Close()
+
+	reader := bufio.NewReader(conn)
+	scanner := bufio.NewScanner(reader)
+
+	for scanner.Scan() {
+		encodedPayload := scanner.Bytes()
+		// Hex encoding always 2x number of bytes
+		decoded := make([]byte, len(encodedPayload)/2)
+		hex.Decode(decoded, encodedPayload)
+		i.data <- decoded
+	}
+
+	if err := scanner.Err(); err != nil {
+		fmt.Fprintln(os.Stderr, "Unexpected error in input tcp connection:", err)
+	}
+}
+
+func (i *TCPInput) String() string {
+	return "TCP input: " + i.address
+}

input_tcp_test.go

@@ -0,0 +1,113 @@
+package main
+
+import (
+	"encoding/hex"
+	"io"
+	"log"
+	"net"
+	"sync"
+	"testing"
+)
+
+func TestTCPInput(t *testing.T) {
+	wg := new(sync.WaitGroup)
+	quit := make(chan int)
+
+	input := NewTCPInput(":0")
+	output := NewTestOutput(func(data []byte) {
+		wg.Done()
+	})
+
+	Plugins.Inputs = []io.Reader{input}
+	Plugins.Outputs = []io.Writer{output}
+
+	go Start(quit)
+
+	tcpAddr, err := net.ResolveTCPAddr("tcp", input.listener.Addr().String())
+
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	conn, err := net.DialTCP("tcp", nil, tcpAddr)
+
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	msg := []byte("GET / HTTP/1.1\r\n\r\n")
+
+	for i := 0; i < 100; i++ {
+		wg.Add(1)
+
+		encoded := make([]byte, len(msg)*2+1)
+		hex.Encode(encoded, msg)
+		conn.Write(append(encoded, '\n'))
+	}
+
+	wg.Wait()
+
+	close(quit)
+}
+
+func BenchmarkTCPInput(b *testing.B) {
+	wg := new(sync.WaitGroup)
+	quit := make(chan int)
+
+	input := NewTCPInput(":0")
+	output := NewTestOutput(func(data []byte) {
+		wg.Done()
+	})
+
+	Plugins.Inputs = []io.Reader{input}
+	Plugins.Outputs = []io.Writer{output}
+
+	go Start(quit)
+
+	tcpAddr, err := net.ResolveTCPAddr("tcp", input.listener.Addr().String())
+
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	var connections []net.Conn
+
+	// Creating simple pool of workers, same as output_tcp have
+	dataChan := make(chan []byte, 1000)
+
+	for i := 0; i < 10; i++ {
+		conn, _ := net.DialTCP("tcp", nil, tcpAddr)
+		connections = append(connections, conn)
+
+		go func(conn net.Conn) {
+			for {
+				data := <-dataChan
+
+				new_buf := make([]byte, len(data)+2)
+				data = append(data, []byte("¶")...)
+				copy(new_buf, data)
+				conn.Write(new_buf)
+			}
+		}(conn)
+	}
+
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	msg := []byte("GET / HTTP/1.1\r\n\r\n")
+
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		wg.Add(1)
+		dataChan <- msg
+	}
+
+	wg.Wait()
+
+	for _, conn := range connections {
+		conn.Close()
+	}
+
+	close(quit)
+}

limiter.go

@@ -0,0 +1,93 @@
+package main
+
+import (
+	"fmt"
+	"io"
+	"math/rand"
+	"strconv"
+	"strings"
+	"time"
+)
+
+type Limiter struct {
+	plugin    interface{}
+	limit     int
+	isPercent bool
+
+	currentRPS  int
+	currentTime int64
+}
+
+func parseLimitOptions(options string) (limit int, isPercent bool) {
+	if strings.Contains(options, "%") {
+		limit, _ = strconv.Atoi(strings.Split(options, "%")[0])
+		isPercent = true
+	} else {
+		limit, _ = strconv.Atoi(options)
+		isPercent = false
+	}
+
+	return
+}
+
+func NewLimiter(plugin interface{}, options string) io.ReadWriter {
+	l := new(Limiter)
+	l.limit, l.isPercent = parseLimitOptions(options)
+	l.plugin = plugin
+	l.currentTime = time.Now().UnixNano()
+
+	// FileInput have its own rate limiting. Unlike other inputs we not just dropping requests, we can slow down or speed up request emittion.
+	if fi, ok := l.plugin.(*FileInput); ok && l.isPercent {
+		fi.speedFactor = float64(l.limit) / float64(100)
+	}
+
+	return l
+}
+
+func (l *Limiter) isLimited() bool {
+	// File input have its own limiting algorithm
+	if _, ok := l.plugin.(*FileInput); ok && l.isPercent {
+		return false
+	}
+
+	if l.isPercent {
+		return l.limit <= rand.Intn(100)
+	}
+
+	if (time.Now().UnixNano() - l.currentTime) > time.Second.Nanoseconds() {
+		l.currentTime = time.Now().UnixNano()
+		l.currentRPS = 0
+	}
+
+	if l.currentRPS >= l.limit {
+		return true
+	}
+
+	l.currentRPS++
+
+	return false
+}
+
+func (l *Limiter) Write(data []byte) (n int, err error) {
+	if l.isLimited() {
+		return 0, nil
+	}
+
+	n, err = l.plugin.(io.Writer).Write(data)
+
+	return
+}
+
+func (l *Limiter) Read(data []byte) (n int, err error) {
+	n, err = l.plugin.(io.Reader).Read(data)
+
+	if l.isLimited() {
+		return 0, nil
+	}
+
+	return
+}
+
+func (l *Limiter) String() string {
+	return fmt.Sprintf("Limiting %s to: %d (isPercent: %b)", l.plugin, l.limit, l.isPercent)
+}

limiter_test.go

@@ -0,0 +1,106 @@
+// +build !race
+
+package main
+
+import (
+	"io"
+	"sync"
+	"testing"
+)
+
+func TestOutputLimiter(t *testing.T) {
+	wg := new(sync.WaitGroup)
+	quit := make(chan int)
+
+	input := NewTestInput()
+	output := NewLimiter(NewTestOutput(func(data []byte) {
+		wg.Done()
+	}), "10")
+	wg.Add(10)
+
+	Plugins.Inputs = []io.Reader{input}
+	Plugins.Outputs = []io.Writer{output}
+
+	go Start(quit)
+
+	for i := 0; i < 100; i++ {
+		input.EmitGET()
+	}
+
+	wg.Wait()
+
+	close(quit)
+}
+
+func TestInputLimiter(t *testing.T) {
+	wg := new(sync.WaitGroup)
+	quit := make(chan int)
+
+	input := NewLimiter(NewTestInput(), "10")
+	output := NewTestOutput(func(data []byte) {
+		wg.Done()
+	})
+	wg.Add(10)
+
+	Plugins.Inputs = []io.Reader{input}
+	Plugins.Outputs = []io.Writer{output}
+
+	go Start(quit)
+
+	for i := 0; i < 100; i++ {
+		input.(*Limiter).plugin.(*TestInput).EmitGET()
+	}
+
+	wg.Wait()
+
+	close(quit)
+}
+
+// Should limit all requests
+func TestPercentLimiter1(t *testing.T) {
+	wg := new(sync.WaitGroup)
+	quit := make(chan int)
+
+	input := NewTestInput()
+	output := NewLimiter(NewTestOutput(func(data []byte) {
+		wg.Done()
+	}), "0%")
+
+	Plugins.Inputs = []io.Reader{input}
+	Plugins.Outputs = []io.Writer{output}
+
+	go Start(quit)
+
+	for i := 0; i < 100; i++ {
+		input.EmitGET()
+	}
+
+	wg.Wait()
+
+	close(quit)
+}
+
+// Should not limit at all
+func TestPercentLimiter2(t *testing.T) {
+	wg := new(sync.WaitGroup)
+	quit := make(chan int)
+
+	input := NewTestInput()
+	output := NewLimiter(NewTestOutput(func(data []byte) {
+		wg.Done()
+	}), "100%")
+	wg.Add(100)
+
+	Plugins.Inputs = []io.Reader{input}
+	Plugins.Outputs = []io.Writer{output}
+
+	go Start(quit)
+
+	for i := 0; i < 100; i++ {
+		input.EmitGET()
+	}
+
+	wg.Wait()
+
+	close(quit)
+}

listener/listener.go

@@ -1,80 +0,0 @@
-// Listener capture TCP traffic using RAW SOCKETS.
-// Note: it requires sudo or root access.
-//
-// Rigt now it suport only HTTP, and only GET requests.
-package listener
-
-import (
-	"bufio"
-	"bytes"
-	"fmt"
-	"log"
-	"net"
-	"net/http"
-	"os"
-)
-
-type HttpRequest struct {
-	Tag     string            // Not used yet
-	Method  string            // Right now only 'GET'
-	Url     string            // Request URL
-	Headers map[string]string // Request Headers
-}
-
-// Enable debug logging only if "--verbose" flag passed
-func Debug(v ...interface{}) {
-	if Settings.verbose {
-		log.Println(v...)
-	}
-}
-
-func greeting() {
-
-}
-
-// Because its sub-program, Run acts as `main`
-func Run() {
-	if os.Getuid() != 0 {
-		fmt.Println("Please start the listener as root or sudo!")
-		fmt.Println("This is required since listener sniff traffic on given port.")
-		os.Exit(1)
-	}
-
-	fmt.Println("Listening for HTTP traffic on", Settings.port, "port")
-	fmt.Println("Forwarding requests to replay server:", Settings.ReplayServer())
-
-	// Connection to reaplay server
-	serverAddr, err := net.ResolveUDPAddr("udp4", Settings.ReplayServer())
-	conn, err := net.DialUDP("udp", nil, serverAddr)
-
-	if err != nil {
-		log.Fatal("Connection error", err)
-	}
-
-	// Sniffing traffic from given port
-	listener := RAWTCPListen("0.0.0.0", Settings.port)
-
-	for {
-		message := listener.Receive()
-
-		go func(m *TCPMessage) {
-			if Settings.verbose {
-				buf := bytes.NewBuffer(m.Bytes())
-				reader := bufio.NewReader(buf)
-
-				request, err := http.ReadRequest(reader)
-
-				if err != nil {
-					Debug("Error while parsing request:", string(m.Bytes()))
-				} else {
-					request.ParseMultipartForm(32 << 20)
-					Debug("Forwarding request:", request)
-				}
-			}
-
-			conn.Write(m.Bytes())
-		}(message)
-	}
-
-	conn.Close()
-}

listener/raw_tcp_listener.go

@@ -1,105 +0,0 @@
-package listener
-
-import (
-	"log"
-	"net"
-	"time"
-)
-
-type RAWTCPListener struct {
-	messages map[uint32]*TCPMessage // buffer of TCPMessages waiting to be send
-
-	c_packets  chan *TCPPacket
-	c_messages chan *TCPMessage
-
-	addr string
-	port int
-}
-
-func RAWTCPListen(addr string, port int) (listener *RAWTCPListener) {
-	listener = &RAWTCPListener{}
-	listener.messages = make(map[uint32]*TCPMessage)
-
-	listener.c_packets = make(chan *TCPPacket)
-	listener.c_messages = make(chan *TCPMessage)
-
-	listener.addr = addr
-	listener.port = port
-
-	go listener.listen()
-	go listener.readTCPPackets()
-
-	return
-}
-
-func (t *RAWTCPListener) listen() {
-
-	for {
-		var messages chan *TCPMessage
-		var message *TCPMessage
-
-		for _, msg := range t.messages {
-			if msg.Complete() {
-				messages = t.c_messages
-				message = msg
-			}
-		}
-
-		select {
-		case messages <- message:
-			delete(t.messages, message.ask)
-		case packet := <-t.c_packets:
-			t.processTCPPacket(packet)
-
-		// Ensure that this will be run at least each 200 ms, to ensure that all messages will be send
-		// Without it last message may not be send (it will be send only on next TCP packets)
-		case <-time.After(200 * time.Millisecond):
-		}
-	}
-}
-
-func (t *RAWTCPListener) readTCPPackets() {
-	conn, e := net.ListenPacket("ip4:tcp", t.addr)
-	defer conn.Close()
-
-	if e != nil {
-		log.Fatal(e)
-	}
-
-	for {
-		buf := make([]byte, 65565)
-		n, _, err := conn.ReadFrom(buf)
-
-		if err != nil {
-			Debug("Error:", err)
-		}
-
-		if n > 0 {
-			packet := NewTCPPacket(buf[:n])
-
-			if int(packet.dest_port) == t.port {
-				t.c_packets <- packet
-			}
-		}
-	}
-}
-
-//
-func (t *RAWTCPListener) processTCPPacket(packet *TCPPacket) {
-	// We interested only in packets that contain some data
-	if !(packet.f_ask && packet.f_psh) {
-		return
-	}
-
-	ask := packet.asknowledgement
-
-	if _, ok := t.messages[ask]; !ok {
-		t.messages[ask] = NewTCPMessage(ask)
-	}
-
-	t.messages[ask].AddPacket(packet)
-}
-
-func (t *RAWTCPListener) Receive() *TCPMessage {
-	return <-t.c_messages
-}

listener/settings.go

@@ -1,47 +0,0 @@
-package listener
-
-import (
-	"flag"
-	"os"
-	"strings"
-)
-
-const (
-	defaultPort             = 80
-	defaultNetworkInterface = "any"
-
-	defaultReplayAddress = "localhost:28020"
-)
-
-type ListenerSettings struct {
-	networkInterface string
-	port             int
-
-	replayAddress string
-
-	verbose bool
-}
-
-var Settings ListenerSettings = ListenerSettings{}
-
-func (s *ListenerSettings) ReplayServer() string {
-	if !strings.Contains(s.replayAddress, ":") {
-		return s.replayAddress + ":28020"
-	}
-
-	return s.replayAddress
-}
-
-func init() {
-	if len(os.Args) < 2 || os.Args[1] != "listen" {
-		return
-	}
-
-	flag.IntVar(&Settings.port, "p", defaultPort, "Specify the http server port whose traffic you want to capture")
-
-	flag.StringVar(&Settings.networkInterface, "i", defaultNetworkInterface, "By default it try to listen on all network interfaces.To get list of interfaces run `ifconfig`")
-
-	flag.StringVar(&Settings.replayAddress, "r", defaultReplayAddress, "Address of replay server.")
-
-	flag.BoolVar(&Settings.verbose, "verbose", false, "Log requests")
-}

listener/tcp_message.go

@@ -1,64 +0,0 @@
-package listener
-
-import (
-	"sort"
-	"time"
-)
-
-// TCPMessage ensure that all TCP packets for given request is received, and processed in right sequence
-// Its needed because all TCP message can be fragmented or re-transmitted
-//
-// Each TCP Packet have 2 ids: asknowledgement - message_id, and sequence - packet_id
-// Message can be compiled from unique packets with same message_id which sorted by sequence
-// Message is received if we did't receive any packets for 200ms OR if we received packet with "fin" flag
-type TCPMessage struct {
-	ask     uint32             // Message ID
-	packets map[int]*TCPPacket // map[packet.sequence]*TCPPacket
-	updated int64              // time of last packet
-}
-
-func NewTCPMessage(ask uint32) (msg *TCPMessage) {
-	msg = &TCPMessage{}
-	msg.packets = make(map[int]*TCPPacket)
-	msg.updated = time.Now().UnixNano()
-	msg.ask = ask
-	return
-}
-
-// Sort packets in right orders and return message content
-func (t *TCPMessage) Bytes() (output []byte) {
-	mk := make([]int, len(t.packets))
-
-	i := 0
-	for k, _ := range t.packets {
-		mk[i] = k
-		i++
-	}
-
-	sort.Ints(mk)
-
-	for _, k := range mk {
-		output = append(output, t.packets[k].data...)
-	}
-
-	return
-}
-
-// Add packet to the message
-func (t *TCPMessage) AddPacket(packet *TCPPacket) {
-	seq := int(packet.sequence)
-
-	if _, ok := t.packets[seq]; !ok {
-		t.packets[seq] = packet
-	} else {
-		Debug("Received packet with same sequence")
-	}
-
-	t.updated = time.Now().UnixNano()
-}
-
-// TCP message is complete if we not received any packets for 200ms since last packet
-func (t *TCPMessage) Complete() bool {
-	ns := time.Now().UnixNano()
-	return (ns - t.updated) > int64(200*time.Millisecond)
-}

listener/tcp_packet.go

@@ -1,106 +0,0 @@
-package listener
-
-import (
-	"bytes"
-	"encoding/binary"
-	"strconv"
-	"strings"
-)
-
-// Simple TCP packet parser
-//
-// Packet structure: http://en.wikipedia.org/wiki/Transmission_Control_Protocol
-type TCPPacket struct {
-	packet []byte
-
-	buf *bytes.Buffer
-
-	source_port uint16
-	dest_port   uint16
-
-	sequence        uint32
-	asknowledgement uint32
-
-	doff_reserved uint16
-	tcph_length   uint16
-
-	f_ns  bool
-	f_crw bool
-	f_ece bool
-	f_urg bool
-	f_ask bool
-	f_psh bool
-	f_rst bool
-	f_syn bool
-	f_fin bool
-
-	window_size uint16
-	checksum    uint16
-
-	data []byte
-}
-
-func NewTCPPacket(b []byte) (p *TCPPacket) {
-	buf := bytes.NewBuffer(b)
-	p = &TCPPacket{packet: b, buf: buf}
-	p.Parse()
-
-	return p
-}
-
-// Helper for binary.Read
-func (t *TCPPacket) read(data interface{}) {
-	binary.Read(t.buf, binary.BigEndian, data)
-}
-
-// Inspired by: https://gist.github.com/clicube/4978853
-func (t *TCPPacket) Parse() {
-	t.read(&t.source_port)
-	t.read(&t.dest_port)
-
-	t.read(&t.sequence)
-	t.read(&t.asknowledgement)
-	t.read(&t.doff_reserved)
-
-	t.tcph_length = t.doff_reserved >> 12 * 4
-
-	t.f_ns = (t.doff_reserved & 256) != 0
-	t.f_crw = (t.doff_reserved & 128) != 0
-	t.f_ece = (t.doff_reserved & 64) != 0
-	t.f_urg = (t.doff_reserved & 32) != 0
-	t.f_ask = (t.doff_reserved & 16) != 0
-	t.f_psh = (t.doff_reserved & 8) != 0
-	t.f_rst = (t.doff_reserved & 4) != 0
-	t.f_syn = (t.doff_reserved & 2) != 0
-	t.f_fin = (t.doff_reserved & 1) != 0
-
-	t.read(&t.window_size)
-	t.read(&t.checksum)
-
-	t.data = t.packet[t.tcph_length:]
-}
-
-func (t *TCPPacket) String() string {
-	return strings.Join([]string{
-		"Source port: " + strconv.Itoa(int(t.source_port)),
-		"Dest port:" + strconv.Itoa(int(t.dest_port)),
-		"Sequence:" + strconv.Itoa(int(t.sequence)),
-		"Acknowledgement:" + strconv.Itoa(int(t.asknowledgement)),
-		"Header len:" + strconv.Itoa(int(t.tcph_length)),
-
-		"Flag ns:" + strconv.FormatBool(t.f_ns),
-		"Flag crw:" + strconv.FormatBool(t.f_crw),
-		"Flag ece:" + strconv.FormatBool(t.f_ece),
-		"Flag urg:" + strconv.FormatBool(t.f_urg),
-		"Flag ask:" + strconv.FormatBool(t.f_ask),
-		"Flag psh:" + strconv.FormatBool(t.f_psh),
-		"Flag rst:" + strconv.FormatBool(t.f_rst),
-		"Flag syn:" + strconv.FormatBool(t.f_syn),
-		"Flag fin:" + strconv.FormatBool(t.f_fin),
-
-		"Window size:" + strconv.Itoa(int(t.window_size)),
-		"Checksum:" + strconv.Itoa(int(t.checksum)),
-
-		"Data:" + string(t.data),
-	}, "\n")
-}

output_dummy.go

@@ -0,0 +1,24 @@
+package main
+
+import (
+	"fmt"
+)
+
+type DummyOutput struct {
+}
+
+func NewDummyOutput(options string) (di *DummyOutput) {
+	di = new(DummyOutput)
+
+	return
+}
+
+func (i *DummyOutput) Write(data []byte) (int, error) {
+	fmt.Println("Writing message: ", string(data))
+
+	return len(data), nil
+}
+
+func (i *DummyOutput) String() string {
+	return "Dummy Output"
+}

output_file.go

@@ -0,0 +1,52 @@
+package main
+
+import (
+	"encoding/gob"
+	"io"
+	"log"
+	"os"
+	"time"
+)
+
+type RawRequest struct {
+	Timestamp int64
+	Request   []byte
+}
+
+type FileOutput struct {
+	path    string
+	encoder *gob.Encoder
+	file    *os.File
+}
+
+func NewFileOutput(path string) io.Writer {
+	o := new(FileOutput)
+	o.path = path
+	o.Init(path)
+
+	return o
+}
+
+func (o *FileOutput) Init(path string) {
+	var err error
+
+	o.file, err = os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0660)
+
+	if err != nil {
+		log.Fatal(o, "Cannot open file %q. Error: %s", path, err)
+	}
+
+	o.encoder = gob.NewEncoder(o.file)
+}
+
+func (o *FileOutput) Write(data []byte) (n int, err error) {
+	raw := RawRequest{time.Now().UnixNano(), data}
+
+	o.encoder.Encode(raw)
+
+	return len(data), nil
+}
+
+func (o *FileOutput) String() string {
+	return "File output: " + o.path
+}

output_file_test.go

@@ -0,0 +1,42 @@
+package main
+
+import (
+	"io"
+	"sync"
+	"testing"
+)
+
+func TestFileOutput(t *testing.T) {
+	wg := new(sync.WaitGroup)
+	quit := make(chan int)
+
+	input := NewTestInput()
+	output := NewFileOutput("/tmp/test_requests.gor")
+
+	Plugins.Inputs = []io.Reader{input}
+	Plugins.Outputs = []io.Writer{output}
+
+	go Start(quit)
+
+	for i := 0; i < 100; i++ {
+		wg.Add(2)
+		input.EmitGET()
+		input.EmitPOST()
+	}
+	close(quit)
+
+	quit = make(chan int)
+
+	input2 := NewFileInput("/tmp/test_requests.gor")
+	output2 := NewTestOutput(func(data []byte) {
+		wg.Done()
+	})
+
+	Plugins.Inputs = []io.Reader{input2}
+	Plugins.Outputs = []io.Writer{output2}
+
+	go Start(quit)
+
+	wg.Wait()
+	close(quit)
+}

output_http.go

@@ -0,0 +1,160 @@
+package main
+
+import (
+	"io"
+	"log"
+	"sync/atomic"
+	"time"
+)
+
+const InitialDynamicWorkers = 10
+
+type HTTPOutputConfig struct {
+	redirectLimit int
+
+	stats   bool
+	workers int
+
+	elasticSearch string
+
+	Debug bool
+}
+
+type HTTPOutput struct {
+	// Keep this as first element of struct because it guarantees 64bit
+	// alignment. atomic.* functions crash on 32bit machines if operand is not
+	// aligned at 64bit. See https://github.com/golang/go/issues/599
+	activeWorkers int64
+
+	address string
+	limit   int
+	queue   chan []byte
+
+	needWorker chan int
+
+	config *HTTPOutputConfig
+
+	queueStats *GorStat
+
+	elasticSearch *ESPlugin
+}
+
+func NewHTTPOutput(address string, config *HTTPOutputConfig) io.Writer {
+
+	o := new(HTTPOutput)
+
+	o.address = address
+	o.config = config
+
+	if o.config.stats {
+		o.queueStats = NewGorStat("output_http")
+	}
+
+	o.queue = make(chan []byte, 100)
+	o.needWorker = make(chan int, 1)
+
+	// Initial workers count
+	if o.config.workers == 0 {
+		o.needWorker <- InitialDynamicWorkers
+	} else {
+		o.needWorker <- o.config.workers
+	}
+
+	if o.config.elasticSearch != "" {
+		o.elasticSearch = new(ESPlugin)
+		o.elasticSearch.Init(o.config.elasticSearch)
+	}
+
+	go o.WorkerMaster()
+
+	return o
+}
+
+func (o *HTTPOutput) WorkerMaster() {
+	for {
+		new_workers := <-o.needWorker
+		for i := 0; i < new_workers; i++ {
+			go o.Worker()
+		}
+
+		// Disable dynamic scaling if workers poll fixed size
+		if o.config.workers != 0 {
+			return
+		}
+	}
+}
+
+func (o *HTTPOutput) Worker() {
+	client := NewHTTPClient(o.address, &HTTPClientConfig{
+		FollowRedirects: o.config.redirectLimit,
+		Debug:           o.config.Debug,
+	})
+
+	death_count := 0
+
+	atomic.AddInt64(&o.activeWorkers, 1)
+
+	for {
+		select {
+		case data := <-o.queue:
+			o.sendRequest(client, data)
+			death_count = 0
+		case <-time.After(time.Millisecond * 100):
+			// When dynamic scaling enabled workers die after 2s of inactivity
+			if o.config.workers == 0 {
+				death_count += 1
+			} else {
+				continue
+			}
+
+			if death_count > 20 {
+				workersCount := atomic.LoadInt64(&o.activeWorkers)
+
+				// At least 1 worker should be alive
+				if workersCount != 1 {
+					atomic.AddInt64(&o.activeWorkers, -1)
+					return
+				}
+			}
+		}
+	}
+}
+
+func (o *HTTPOutput) Write(data []byte) (n int, err error) {
+	buf := make([]byte, len(data))
+	copy(buf, data)
+
+	o.queue <- buf
+
+	if o.config.stats {
+		o.queueStats.Write(len(o.queue))
+	}
+
+	if o.config.workers == 0 {
+		workersCount := atomic.LoadInt64(&o.activeWorkers)
+
+		if len(o.queue) > int(workersCount) {
+			o.needWorker <- len(o.queue)
+		}
+	}
+
+	return len(data), nil
+}
+
+func (o *HTTPOutput) sendRequest(client *HTTPClient, request []byte) {
+	start := time.Now()
+	resp, err := client.Send(request)
+	stop := time.Now()
+
+	if err != nil {
+		log.Println("Request error:", err)
+	}
+
+	if o.elasticSearch != nil {
+		o.elasticSearch.ResponseAnalyze(request, resp, start, stop)
+	}
+}
+
+func (o *HTTPOutput) String() string {
+	return "HTTP output: " + o.address
+}

output_http_test.go

@@ -0,0 +1,130 @@
+package main
+
+import (
+	"io"
+	"io/ioutil"
+	"net"
+	"net/http"
+	"net/http/httptest"
+	_ "net/http/httputil"
+	"sync"
+	"testing"
+	"time"
+)
+
+func startHTTP(cb func(*http.Request)) net.Listener {
+	handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		cb(r)
+	})
+
+	listener, _ := net.Listen("tcp", ":0")
+
+	go http.Serve(listener, handler)
+
+	return listener
+}
+
+func TestHTTPOutput(t *testing.T) {
+	wg := new(sync.WaitGroup)
+	quit := make(chan int)
+
+	input := NewTestInput()
+
+	listener := startHTTP(func(req *http.Request) {
+		if req.Header.Get("User-Agent") != "Gor" {
+			t.Error("Wrong header")
+		}
+
+		if req.Method == "OPTIONS" {
+			t.Error("Wrong method")
+		}
+
+		if req.Method == "POST" {
+			defer req.Body.Close()
+			body, _ := ioutil.ReadAll(req.Body)
+
+			if string(body) != "a=1&b=2" {
+				t.Error("Wrong POST body:", string(body))
+			}
+		}
+
+		wg.Done()
+	})
+
+	headers := HTTPHeaders{HTTPHeader{"User-Agent", "Gor"}}
+	methods := HTTPMethods{[]byte("GET"), []byte("PUT"), []byte("POST")}
+	Settings.modifierConfig = HTTPModifierConfig{headers: headers, methods: methods}
+
+	output := NewHTTPOutput(listener.Addr().String(), &HTTPOutputConfig{})
+
+	Plugins.Inputs = []io.Reader{input}
+	Plugins.Outputs = []io.Writer{output}
+
+	go Start(quit)
+
+	for i := 0; i < 100; i++ {
+		wg.Add(2) // OPTIONS should be ignored
+		input.EmitPOST()
+		input.EmitOPTIONS()
+		input.EmitGET()
+	}
+
+	wg.Wait()
+
+	close(quit)
+
+	Settings.modifierConfig = HTTPModifierConfig{}
+}
+
+func TestOutputHTTPSSL(t *testing.T) {
+	wg := new(sync.WaitGroup)
+	quit := make(chan int)
+
+	// Origing and Replay server initialization
+	server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		wg.Done()
+	}))
+
+	input := NewTestInput()
+	http_output := NewHTTPOutput(server.URL, &HTTPOutputConfig{})
+
+	Plugins.Inputs = []io.Reader{input}
+	Plugins.Outputs = []io.Writer{http_output}
+
+	go Start(quit)
+
+	wg.Add(2)
+
+	input.EmitPOST()
+	input.EmitGET()
+
+	wg.Wait()
+	close(quit)
+}
+
+func BenchmarkHTTPOutput(b *testing.B) {
+	wg := new(sync.WaitGroup)
+	quit := make(chan int)
+
+	listener := startHTTP(func(req *http.Request) {
+		time.Sleep(50 * time.Millisecond)
+		wg.Done()
+	})
+
+	input := NewTestInput()
+	output := NewHTTPOutput(listener.Addr().String(), &HTTPOutputConfig{})
+
+	Plugins.Inputs = []io.Reader{input}
+	Plugins.Outputs = []io.Writer{output}
+
+	go Start(quit)
+
+	for i := 0; i < b.N; i++ {
+		wg.Add(1)
+		input.EmitPOST()
+	}
+
+	wg.Wait()
+
+	close(quit)
+}

output_tcp.go

@@ -0,0 +1,79 @@
+package main
+
+import (
+	"encoding/hex"
+	"fmt"
+	"io"
+	"log"
+	"net"
+	"time"
+)
+
+type TCPOutput struct {
+	address  string
+	limit    int
+	buf      chan []byte
+	bufStats *GorStat
+}
+
+func NewTCPOutput(address string) io.Writer {
+	o := new(TCPOutput)
+
+	o.address = address
+
+	o.buf = make(chan []byte, 100)
+	if Settings.outputTCPStats {
+		o.bufStats = NewGorStat("output_tcp")
+	}
+
+	for i := 0; i < 10; i++ {
+		go o.worker()
+	}
+
+	return o
+}
+
+func (o *TCPOutput) worker() {
+	conn, err := o.connect(o.address)
+	for ; err != nil; conn, err = o.connect(o.address) {
+		time.Sleep(2 * time.Second)
+	}
+
+	defer conn.Close()
+
+	for {
+		_, err := conn.Write(<-o.buf)
+		if err != nil {
+			log.Println("Worker failed on write, exitings and starting new worker")
+			go o.worker()
+			break
+		}
+	}
+}
+
+func (o *TCPOutput) Write(data []byte) (n int, err error) {
+	// Hex encoding always 2x number of bytes
+	encoded := make([]byte, len(data)*2+1)
+	hex.Encode(encoded, data)
+	o.buf <- append(encoded, '\n')
+
+	if Settings.outputTCPStats {
+		o.bufStats.Write(len(o.buf))
+	}
+
+	return len(data), nil
+}
+
+func (o *TCPOutput) connect(address string) (conn net.Conn, err error) {
+	conn, err = net.Dial("tcp", address)
+
+	if err != nil {
+		log.Println("Connection error ", err, o.address)
+	}
+
+	return
+}
+
+func (o *TCPOutput) String() string {
+	return fmt.Sprintf("TCP output %s, limit: %d", o.address, o.limit)
+}

output_tcp_test.go

@@ -0,0 +1,92 @@
+package main
+
+import (
+	"bufio"
+	"encoding/hex"
+	"io"
+	"log"
+	"net"
+	"sync"
+	"testing"
+)
+
+func TestTCPOutput(t *testing.T) {
+	wg := new(sync.WaitGroup)
+	quit := make(chan int)
+
+	listener := startTCP(func(data []byte) {
+		wg.Done()
+	})
+	input := NewTestInput()
+	output := NewTCPOutput(listener.Addr().String())
+
+	Plugins.Inputs = []io.Reader{input}
+	Plugins.Outputs = []io.Writer{output}
+
+	go Start(quit)
+
+	for i := 0; i < 100; i++ {
+		wg.Add(1)
+		input.EmitGET()
+	}
+
+	wg.Wait()
+
+	close(quit)
+}
+
+func startTCP(cb func([]byte)) net.Listener {
+	listener, err := net.Listen("tcp", ":0")
+
+	if err != nil {
+		log.Fatal("Can't start:", err)
+	}
+
+	go func() {
+		for {
+			conn, _ := listener.Accept()
+			defer conn.Close()
+
+			go func() {
+				reader := bufio.NewReader(conn)
+				scanner := bufio.NewScanner(reader)
+
+				for scanner.Scan() {
+					encodedPayload := scanner.Bytes()
+					// Hex encoding always 2x number of bytes
+					decoded := make([]byte, len(encodedPayload)/2)
+					hex.Decode(decoded, encodedPayload)
+					cb(decoded)
+				}
+			}()
+		}
+	}()
+
+	return listener
+}
+
+func BenchmarkTCPOutput(b *testing.B) {
+	wg := new(sync.WaitGroup)
+	quit := make(chan int)
+
+	listener := startTCP(func(data []byte) {
+		wg.Done()
+	})
+	input := NewTestInput()
+	output := NewTCPOutput(listener.Addr().String())
+
+	Plugins.Inputs = []io.Reader{input}
+	Plugins.Outputs = []io.Writer{output}
+
+	go Start(quit)
+
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		wg.Add(1)
+		input.EmitGET()
+	}
+
+	wg.Wait()
+
+	close(quit)
+}

plugins.go

@@ -0,0 +1,101 @@
+package main
+
+import (
+	"io"
+	"reflect"
+	"strings"
+)
+
+type InOutPlugins struct {
+	Inputs  []io.Reader
+	Outputs []io.Writer
+}
+
+type ReaderOrWriter interface{}
+
+var Plugins *InOutPlugins = new(InOutPlugins)
+
+func extractLimitOptions(options string) (string, string) {
+	split := strings.Split(options, "|")
+
+	if len(split) > 1 {
+		return split[0], split[1]
+	} else {
+		return split[0], ""
+	}
+}
+
+// Automatically detects type of plugin and initialize it
+//
+// See this article if curious about relfect stuff below: http://blog.burntsushi.net/type-parametric-functions-golang
+func registerPlugin(constructor interface{}, options ...interface{}) {
+	vc := reflect.ValueOf(constructor)
+
+	// Pre-processing options to make it work with reflect
+	vo := []reflect.Value{}
+	for _, oi := range options {
+		vo = append(vo, reflect.ValueOf(oi))
+	}
+
+	// Removing limit options from path
+	path, limit := extractLimitOptions(vo[0].String())
+
+	// Writing value back without limiter "|" options
+	vo[0] = reflect.ValueOf(path)
+
+	// Calling our constructor with list of given options
+	plugin := vc.Call(vo)[0].Interface()
+	plugin_wrapper := plugin
+
+	if limit != "" {
+		plugin_wrapper = NewLimiter(plugin, limit)
+	} else {
+		plugin_wrapper = plugin
+	}
+
+	if _, ok := plugin.(io.Reader); ok {
+		Plugins.Inputs = append(Plugins.Inputs, plugin_wrapper.(io.Reader))
+	}
+
+	if _, ok := plugin.(io.Writer); ok {
+		Plugins.Outputs = append(Plugins.Outputs, plugin_wrapper.(io.Writer))
+	}
+}
+
+func InitPlugins() {
+	for _, options := range Settings.inputDummy {
+		registerPlugin(NewDummyInput, options)
+	}
+
+	for _, options := range Settings.outputDummy {
+		registerPlugin(NewDummyOutput, options)
+	}
+
+	for _, options := range Settings.inputRAW {
+		registerPlugin(NewRAWInput, options)
+	}
+
+	for _, options := range Settings.inputTCP {
+		registerPlugin(NewTCPInput, options)
+	}
+
+	for _, options := range Settings.outputTCP {
+		registerPlugin(NewTCPOutput, options)
+	}
+
+	for _, options := range Settings.inputFile {
+		registerPlugin(NewFileInput, options)
+	}
+
+	for _, options := range Settings.outputFile {
+		registerPlugin(NewFileOutput, options)
+	}
+
+	for _, options := range Settings.inputHTTP {
+		registerPlugin(NewHTTPInput, options)
+	}
+
+	for _, options := range Settings.outputHTTP {
+		registerPlugin(NewHTTPOutput, options, &Settings.outputHTTPConfig)
+	}
+}

plugins_test.go

@@ -0,0 +1,47 @@
+package main
+
+import (
+	"io"
+	"testing"
+)
+
+func TestPluginsRegistration(t *testing.T) {
+	Plugins.Inputs = []io.Reader{}
+	Plugins.Outputs = []io.Writer{}
+
+	Settings.inputDummy = MultiOption{"[]"}
+	Settings.outputDummy = MultiOption{"[]"}
+	Settings.outputHTTP = MultiOption{"www.example.com|10"}
+	Settings.inputFile = MultiOption{"/dev/null"}
+
+	InitPlugins()
+
+	if len(Plugins.Inputs) != 2 {
+		t.Errorf("Should be 2 inputs %d", len(Plugins.Inputs))
+	}
+
+	if _, ok := Plugins.Inputs[0].(*DummyInput); !ok {
+		t.Errorf("First input should be DummyInput")
+	}
+
+	if _, ok := Plugins.Inputs[1].(*FileInput); !ok {
+		t.Errorf("Second input should be FileInput")
+	}
+
+	if len(Plugins.Outputs) != 2 {
+		t.Errorf("Should be 2 output %d", len(Plugins.Outputs))
+	}
+
+	if _, ok := Plugins.Outputs[0].(*DummyOutput); !ok {
+		t.Errorf("First output should be DummyOutput")
+	}
+
+	if l, ok := Plugins.Outputs[1].(*Limiter); ok {
+		if _, ok := l.plugin.(*HTTPOutput); !ok {
+			t.Errorf("HTTPOutput should be wrapped in limiter")
+		}
+	} else {
+		t.Errorf("Second output should be Limiter")
+	}
+
+}

proto/proto.go

@@ -0,0 +1,166 @@
+// Low-level interaction with HTTP request payload
+package proto
+
+import (
+    "bytes"
+    "github.com/buger/gor/byteutils"
+    _ "log"
+)
+
+var CLRF = []byte("\r\n")
+var EMPTY_LINE = []byte("\r\n\r\n")
+var HEADER_DELIM = []byte(": ")
+
+// Headers should end with empty line
+func MIMEHeadersEndPos(payload []byte) int {
+    return bytes.Index(payload, EMPTY_LINE)
+}
+
+func MIMEHeadersStartPos(payload []byte) int {
+    return bytes.Index(payload, CLRF) + 2 // Find first line end
+}
+
+// Find header value or return error
+// Do not support multi-line headers
+func Header(payload []byte, name []byte) (value []byte, headerStart, valueStart, headerEnd int) {
+    headerStart = bytes.Index(payload, name)
+
+    if headerStart == -1 {
+        return
+    }
+
+    valueStart = headerStart + len(name) + 1 // Skip ":" after header name
+    if payload[valueStart] == ' ' { // Ignore empty space after ':'
+        valueStart += 1
+    }
+    headerEnd = valueStart + bytes.IndexByte(payload[valueStart:], '\r')
+    value = payload[valueStart:headerEnd]
+
+    return
+}
+
+func GetHeader(payload []byte, name string) []byte {
+    val, _, _, _ := Header(payload, []byte(name))
+
+    return val
+}
+
+func SetHeader(payload, name, value []byte) []byte {
+    _, hs, vs, he := Header(payload, name)
+
+    // If header found
+    if hs != -1 {
+        return byteutils.Replace(payload, vs, he, value)
+    } else {
+        return AddHeader(payload, name, value)
+    }
+}
+
+func AddHeader(payload, name, value []byte) []byte {
+    header := make([]byte, len(name) + 2 + len(value) + 2)
+    copy(header[0:], name)
+    copy(header[len(name):], HEADER_DELIM)
+    copy(header[len(name)+2:], value)
+    copy(header[len(header)-2:], CLRF)
+
+    mimeStart := MIMEHeadersStartPos(payload)
+
+    return byteutils.Insert(payload, mimeStart, header)
+}
+
+func Path(payload []byte) []byte {
+    start := bytes.IndexByte(payload, ' ')
+    start += 1
+
+    end := bytes.IndexByte(payload[start:], ' ')
+
+    return payload[start:start+end]
+}
+
+func SetPath(payload, path []byte) []byte {
+    start := bytes.IndexByte(payload, ' ')
+    start += 1
+
+    end := bytes.IndexByte(payload[start:], ' ')
+
+    return byteutils.Replace(payload, start, start+end, path)
+}
+
+func PathParam(payload, name []byte) (value []byte, valueStart, valueEnd int) {
+    path := Path(payload)
+
+    if paramStart := bytes.Index(path, append(name, '=')); paramStart != -1 {
+        valueStart := paramStart + len(name) + 1
+        paramEnd := bytes.IndexByte(path[valueStart:], '&')
+        if paramEnd == -1 { // It is final param
+            paramEnd = len(path)
+        } else {
+            paramEnd += valueStart
+        }
+
+        return path[valueStart:paramEnd], valueStart, paramEnd
+    } else {
+        return []byte(""), -1, -1
+    }
+}
+
+func SetPathParam(payload, name, value []byte) []byte {
+    path := Path(payload)
+    _, vs, ve := PathParam(payload, name)
+
+    if vs != -1 {
+        newPath := make([]byte, len(path))
+        copy(newPath, path)
+        newPath = byteutils.Replace(newPath, vs, ve, value)
+
+        return SetPath(payload, newPath)
+    } else { // if param not found append to end of url
+        // Adding 2 because of '?' or '&' at start, and '=' in middle
+        newParam := make([]byte, len(name) + len(value) + 2)
+
+        if bytes.IndexByte(path, '?') == -1 {
+            newParam[0] = '?'
+        } else {
+            newParam[0] = '&'
+        }
+
+        copy(newParam[1:], name)
+        newParam[1+len(name)] = '='
+        copy(newParam[2+len(name):], value)
+
+        newPath := make([]byte, len(path) + len(newParam))
+        copy(newPath, path)
+        copy(newPath[len(path):], newParam)
+
+        return SetPath(payload, newPath)
+    }
+}
+
+func SetHost(payload, url, host []byte) []byte {
+    // If this is HTTP 1.0 traffic or proxy traffic it may include host right into path variable, so instead of setting Host header we rewrite Path
+    // Fix for https://github.com/buger/gor/issues/156
+    if path := Path(payload); bytes.HasPrefix(path, []byte("http")) {
+        hostStart := bytes.IndexByte(path, ':') // : position "https?:"
+        hostStart += 3 // Skip 1 ':' and 2 '\'
+        hostEnd := hostStart + bytes.IndexByte(path[hostStart:], '/')
+
+        newPath := make([]byte, len(path))
+        copy(newPath, path)
+        newPath = byteutils.Replace(newPath, 0, hostEnd, url)
+
+        return SetPath(payload, newPath)
+    } else {
+        return SetHeader(payload, []byte("Host"), host)
+    }
+}
+
+func Method(payload []byte) []byte {
+    end := bytes.IndexByte(payload, ' ')
+
+    return payload[:end]
+}
+
+// Status in response have same position as Path in request
+func Status(payload []byte) []byte {
+    return Path(payload)
+}

proto/proto_test.go

@@ -0,0 +1,152 @@
+package proto
+
+import (
+    "testing"
+    "bytes"
+)
+
+func TestHeader(t *testing.T) {
+    var payload, val []byte
+    var headerStart int
+
+    payload = []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
+
+    if val, _, _, _ = Header(payload, []byte("Content-Length")); !bytes.Equal(val, []byte("7")) {
+        t.Error("Should find header value")
+    }
+
+    payload = []byte("POST /post HTTP/1.1\r\nContent-Length:7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
+
+    if val, _, _, _ = Header(payload, []byte("Content-Length")); !bytes.Equal(val, []byte("7")) {
+        t.Error("Should find header value without space after :")
+    }
+
+    if _, headerStart, _, _ = Header(payload, []byte("Not-Found")); headerStart != -1  {
+        t.Error("Should not found header")
+    }
+}
+
+func TestMIMEHeadersEndPos(t *testing.T) {
+    head := []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org")
+    payload := []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
+
+    end := MIMEHeadersEndPos(payload)
+
+    if !bytes.Equal(payload[:end], head) {
+        t.Error("Wrong headers end position:", end)
+    }
+}
+
+func TestMIMEHeadersStartPos(t *testing.T) {
+    headers := []byte("Content-Length: 7\r\nHost: www.w3.org")
+    payload := []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
+
+    start := MIMEHeadersStartPos(payload)
+    end := MIMEHeadersEndPos(payload)
+
+    if !bytes.Equal(payload[start:end], headers) {
+        t.Error("Wrong headers end position:", start, end)
+    }
+}
+
+func TestSetHeader(t *testing.T) {
+    var payload, payload_after []byte
+
+    payload = []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
+    payload_after = []byte("POST /post HTTP/1.1\r\nContent-Length: 14\r\nHost: www.w3.org\r\n\r\na=1&b=2")
+
+    if payload = SetHeader(payload, []byte("Content-Length"), []byte("14")); !bytes.Equal(payload, payload_after) {
+        t.Error("Should update header if it exists", string(payload))
+    }
+
+
+    payload = []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
+    payload_after = []byte("POST /post HTTP/1.1\r\nUser-Agent: Gor\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
+
+    if payload = SetHeader(payload, []byte("User-Agent"), []byte("Gor")); !bytes.Equal(payload, payload_after) {
+        t.Error("Should add header if not found", string(payload))
+    }
+}
+
+func TestPath(t *testing.T) {
+    var path, payload []byte
+
+    payload = []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
+
+    if path = Path(payload); !bytes.Equal(path, []byte("/post")) {
+        t.Error("Should find path", string(path))
+    }
+}
+
+func TestSetPath(t *testing.T) {
+    var payload, payload_after []byte
+
+    payload = []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
+    payload_after = []byte("POST /new_path HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
+
+    if payload = SetPath(payload, []byte("/new_path")); !bytes.Equal(payload, payload_after) {
+        t.Error("Should replace path", string(payload))
+    }
+}
+
+func TestPathParam(t *testing.T) {
+    var payload []byte
+
+    payload = []byte("POST /post?param=test&user_id=1 HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
+
+    if val, _, _ := PathParam(payload, []byte("param")); !bytes.Equal(val, []byte("test")) {
+        t.Error("Should detect attribute", string(val))
+    }
+
+    if val, _, _ := PathParam(payload, []byte("user_id")); !bytes.Equal(val, []byte("1")) {
+        t.Error("Should detect attribute", string(val))
+    }
+}
+
+
+func TestSetPathParam(t *testing.T) {
+    var payload, payload_after []byte
+
+    payload = []byte("POST /post?param=test&user_id=1 HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
+    payload_after = []byte("POST /post?param=new&user_id=1 HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
+
+    if payload = SetPathParam(payload, []byte("param"), []byte("new")); !bytes.Equal(payload, payload_after) {
+        t.Error("Should replace existing value", string(payload))
+    }
+
+    payload = []byte("POST /post?param=test&user_id=1 HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
+    payload_after = []byte("POST /post?param=test&user_id=2 HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
+
+    if payload = SetPathParam(payload, []byte("user_id"), []byte("2")); !bytes.Equal(payload, payload_after) {
+        t.Error("Should replace existing value", string(payload))
+    }
+
+
+    payload = []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
+    payload_after = []byte("POST /post?param=test HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
+
+    if payload = SetPathParam(payload, []byte("param"), []byte("test")); !bytes.Equal(payload, payload_after) {
+        t.Error("Should set param if url have no params", string(payload))
+    }
+
+
+    payload = []byte("POST /post?param=test HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
+    payload_after = []byte("POST /post?param=test&user_id=1 HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
+
+    if payload = SetPathParam(payload, []byte("user_id"), []byte("1")); !bytes.Equal(payload, payload_after) {
+        t.Error("Should set param at the end if url params", string(payload))
+    }
+}
+
+
+
+func TestSetHostHTTP10(t *testing.T) {
+    var payload, payload_after []byte
+
+    payload = []byte("POST http://example.com/post HTTP/1.0\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
+    payload_after = []byte("POST http://new.com/post HTTP/1.0\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
+
+    if payload = SetHost(payload, []byte("http://new.com"), []byte("new.com")); !bytes.Equal(payload, payload_after) {
+        t.Error("Should replace host", string(payload))
+    }
+}

raw_socket_listener/listener.go

@@ -0,0 +1,175 @@
+package raw_socket
+
+import (
+    "encoding/binary"
+    "log"
+    "net"
+    "strconv"
+    "bytes"
+)
+
+// Capture traffic from socket using RAW_SOCKET's
+// http://en.wikipedia.org/wiki/Raw_socket
+//
+// RAW_SOCKET allow you listen for traffic on any port (e.g. sniffing) because they operate on IP level.
+// Ports is TCP feature, same as flow control, reliable transmission and etc.
+// Since we can't use default TCP libraries RAWTCPLitener implements own TCP layer
+// TCP packets is parsed using tcp_packet.go, and flow control is managed by tcp_message.go
+type Listener struct {
+    messages map[string]*TCPMessage // buffer of TCPMessages waiting to be send
+
+    // Expect: 100-continue request is send in 2 tcp messages
+    // We store ACK aliases to merge this packets together
+    ack_aliases map[uint32]uint32
+    seq_with_data map[uint32]uint32
+
+    c_packets  chan *TCPPacket
+    c_messages chan *TCPMessage // Messages ready to be send to client
+
+    c_del_message chan *TCPMessage // Used for notifications about completed or expired messages
+
+    addr string // IP to listen
+    port int    // Port to listen
+}
+
+// RAWTCPListen creates a listener to capture traffic from RAW_SOCKET
+func NewListener(addr string, port string) (rawListener *Listener) {
+    rawListener = &Listener{}
+
+    rawListener.c_packets = make(chan *TCPPacket, 10000)
+    rawListener.c_messages = make(chan *TCPMessage, 10000)
+    rawListener.c_del_message = make(chan *TCPMessage, 10000)
+
+    rawListener.messages = make(map[string]*TCPMessage)
+    rawListener.ack_aliases = make(map[uint32]uint32)
+    rawListener.seq_with_data = make(map[uint32]uint32)
+
+    rawListener.addr = addr
+    rawListener.port, _ = strconv.Atoi(port)
+
+    go rawListener.listen()
+    go rawListener.readRAWSocket()
+
+    return
+}
+
+func (t *Listener) listen() {
+    for {
+        select {
+        // If message ready for deletion it means that its also complete or expired by timeout
+        case message := <-t.c_del_message:
+            t.c_messages <- message
+            delete(t.ack_aliases, message.packets[0].Ack)
+            delete(t.messages, message.ID)
+
+        // We need to use channels to process each packet to avoid data races
+        case packet := <-t.c_packets:
+            t.processTCPPacket(packet)
+        }
+    }
+}
+
+func (t *Listener) readRAWSocket() {
+    conn, e := net.ListenPacket("ip4:tcp", t.addr)
+
+    if e != nil {
+        log.Fatal(e)
+    }
+
+    defer conn.Close()
+
+    buf := make([]byte, 4096*10)
+
+    for {
+        // Note: ReadFrom receive messages without IP header
+        n, addr, err := conn.ReadFrom(buf)
+
+        if err != nil {
+            log.Println("Error:", err)
+            continue
+        }
+
+        if n > 0 {
+            t.parsePacket(addr, buf[:n])
+        }
+    }
+}
+
+func (t *Listener) parsePacket(addr net.Addr, buf []byte) {
+    if t.isIncomingDataPacket(buf) {
+        new_buf := make([]byte, len(buf))
+        copy(new_buf, buf)
+
+        t.c_packets <- ParseTCPPacket(addr, new_buf)
+    }
+}
+
+func (t *Listener) isIncomingDataPacket(buf []byte) bool {
+    // To avoid full packet parsing every time, we manually parsing values needed for packet filtering
+    // http://en.wikipedia.org/wiki/Transmission_Control_Protocol
+    dest_port := binary.BigEndian.Uint16(buf[2:4])
+
+    // Because RAW_SOCKET can't be bound to port, we have to control it by ourself
+    if int(dest_port) == t.port {
+        // Get the 'data offset' (size of the TCP header in 32-bit words)
+        dataOffset := (buf[12] & 0xF0) >> 4
+
+        // We need only packets with data inside
+        // Check that the buffer is larger than the size of the TCP header
+        if len(buf) > int(dataOffset*4) {
+            // We should create new buffer because go slices is pointers. So buffer data shoud be immutable.
+            return true
+        }
+    }
+
+    return false
+}
+
+var bExpect100ContinueCheck = []byte("Expect: 100-continue")
+var bPOST = []byte("POST")
+
+// Trying to add packet to existing message or creating new message
+//
+// For TCP message unique id is Acknowledgment number (see tcp_packet.go)
+func (t *Listener) processTCPPacket(packet *TCPPacket) {
+    defer func() { recover() }()
+
+    var message *TCPMessage
+
+    parent_message_ack, parent_ok := t.seq_with_data[packet.Seq]
+    if parent_ok {
+        t.ack_aliases[packet.Ack] = parent_message_ack
+        delete(t.seq_with_data, packet.Seq)
+    }
+
+    ack_alias, alias_ok := t.ack_aliases[packet.Ack]
+    if alias_ok {
+        packet.Ack = ack_alias
+    }
+
+    m_id := packet.Addr.String() + strconv.Itoa(int(packet.Ack))
+    message, ok := t.messages[m_id]
+
+    if !ok {
+        // We sending c_del_message channel, so message object can communicate with Listener and notify it if message completed
+        message = NewTCPMessage(m_id, t.c_del_message)
+        t.messages[m_id] = message
+    }
+
+    if bytes.Equal(packet.Data[0:4], bPOST) {
+        if bytes.Equal(packet.Data[len(packet.Data)-24:len(packet.Data)-4], bExpect100ContinueCheck) {
+            t.seq_with_data[packet.Seq + uint32(len(packet.Data))] = packet.Ack
+
+            // Removing `Expect: 100-continue` header
+            packet.Data = append(packet.Data[:len(packet.Data)-24], packet.Data[len(packet.Data)-2:]...)
+        }
+    }
+
+    // Adding packet to message
+    message.c_packets <- packet
+}
+
+// Receive TCP messages from the listener channel
+func (t *Listener) Receive() *TCPMessage {
+    return <-t.c_messages
+}

raw_socket_listener/tcp_message.go

@@ -0,0 +1,105 @@
+package raw_socket
+
+import (
+    "log"
+    "sort"
+    "time"
+)
+
+const MSG_EXPIRE = 2000 * time.Millisecond
+
+// TCPMessage ensure that all TCP packets for given request is received, and processed in right sequence
+// Its needed because all TCP message can be fragmented or re-transmitted
+//
+// Each TCP Packet have 2 ids: acknowledgment - message_id, and sequence - packet_id
+// Message can be compiled from unique packets with same message_id which sorted by sequence
+// Message is received if we didn't receive any packets for 2000ms
+type TCPMessage struct {
+    ID      string // Message ID
+    packets []*TCPPacket
+
+    timer *time.Timer // Used for expire check
+
+    c_packets chan *TCPPacket
+
+    c_del_message chan *TCPMessage
+}
+
+// NewTCPMessage pointer created from a Acknowledgment number and a channel of messages readuy to be deleted
+func NewTCPMessage(ID string, c_del chan *TCPMessage) (msg *TCPMessage) {
+    msg = &TCPMessage{ID: ID}
+
+    msg.c_packets = make(chan *TCPPacket)
+    msg.c_del_message = c_del // used for notifying that message completed or expired
+
+    // Every time we receive packet we reset this timer
+    msg.timer = time.AfterFunc(MSG_EXPIRE, msg.Timeout)
+
+    go msg.listen()
+
+    return
+}
+
+func (t *TCPMessage) listen() {
+    for {
+        select {
+        case packet, more := <-t.c_packets:
+            if more {
+                t.AddPacket(packet)
+            } else {
+                // Stop loop if channel closed
+                return
+            }
+        }
+    }
+}
+
+// Timeout notifies message to stop listening, close channel and message ready to be sent
+func (t *TCPMessage) Timeout() {
+    select {
+        // In some cases Timeout can be called multiple times (do not know how yet)
+        // Ensure that we did not close channel 2 times
+        case packet, ok := <- t.c_packets:
+            if ok {
+                t.AddPacket(packet)
+            } else {
+                return
+            }
+        default:
+            close(t.c_packets)
+            t.c_del_message <- t // Notify RAWListener that message is ready to be send to replay server
+    }
+}
+
+// Bytes sorts packets in right orders and return message content
+func (t *TCPMessage) Bytes() (output []byte) {
+    sort.Sort(BySeq(t.packets))
+
+    for _, v := range t.packets {
+        output = append(output, v.Data...)
+    }
+
+    return output
+}
+
+// AddPacket to the message and ensure packet uniqueness
+// TCP allows that packet can be re-send multiple times
+func (t *TCPMessage) AddPacket(packet *TCPPacket) {
+    packetFound := false
+
+    for _, pkt := range t.packets {
+        if packet.Seq == pkt.Seq {
+            packetFound = true
+            break
+        }
+    }
+
+    if packetFound {
+        log.Println("Received packet with same sequence")
+    } else {
+        t.packets = append(t.packets, packet)
+    }
+
+    // Reset message timeout timer
+    t.timer.Reset(MSG_EXPIRE)
+}

raw_socket_listener/tcp_packet.go

@@ -0,0 +1,101 @@
+package raw_socket
+
+import (
+    "encoding/binary"
+    "net"
+    "strconv"
+    "strings"
+)
+
+// TCP Flags
+const (
+    TCP_FIN = 1 << iota
+    TCP_SYN
+    TCP_RST
+    TCP_PSH
+    TCP_ACK
+    TCP_URG
+    TCP_ECE
+    TCP_CWR
+    TCP_NS
+)
+
+// Simple TCP packet parser
+//
+// Packet structure: http://en.wikipedia.org/wiki/Transmission_Control_Protocol
+type TCPPacket struct {
+    SrcPort    uint16
+    DestPort   uint16
+    Seq        uint32
+    Ack        uint32
+    DataOffset uint8
+    Flags      uint16
+    Window     uint16
+    Checksum   uint16
+    Urgent     uint16
+
+    Data []byte
+
+    Addr net.Addr
+}
+
+func ParseTCPPacket(addr net.Addr, b []byte) (p *TCPPacket) {
+    p = &TCPPacket{Data: b}
+    p.ParseBasic()
+    p.Addr = addr
+
+    return p
+}
+
+// Parse TCP Packet, inspired by: https://github.com/miekg/pcap/blob/master/packet.go
+func (t *TCPPacket) Parse() {
+    t.ParseBasic()
+    t.SrcPort = binary.BigEndian.Uint16(t.Data[0:2])
+    t.DestPort = binary.BigEndian.Uint16(t.Data[2:4])
+    t.Flags = binary.BigEndian.Uint16(t.Data[12:14]) & 0x1FF
+    t.Window = binary.BigEndian.Uint16(t.Data[14:16])
+    t.Checksum = binary.BigEndian.Uint16(t.Data[16:18])
+    t.Urgent = binary.BigEndian.Uint16(t.Data[18:20])
+}
+
+// ParseBasic set of fields
+func (t *TCPPacket) ParseBasic() {
+    t.Seq = binary.BigEndian.Uint32(t.Data[4:8])
+    t.Ack = binary.BigEndian.Uint32(t.Data[8:12])
+    t.DataOffset = (t.Data[12] & 0xF0) >> 4
+
+    t.Data = t.Data[t.DataOffset*4:]
+}
+
+// String output for a TCP Packet
+func (t *TCPPacket) String() string {
+    return strings.Join([]string{
+        "Source port: " + strconv.Itoa(int(t.SrcPort)),
+        "Dest port:" + strconv.Itoa(int(t.DestPort)),
+        "Sequence:" + strconv.Itoa(int(t.Seq)),
+        "Acknowledgment:" + strconv.Itoa(int(t.Ack)),
+        "Header len:" + strconv.Itoa(int(t.DataOffset)),
+
+        "Flag ns:" + strconv.FormatBool(t.Flags&TCP_NS != 0),
+        "Flag crw:" + strconv.FormatBool(t.Flags&TCP_CWR != 0),
+        "Flag ece:" + strconv.FormatBool(t.Flags&TCP_ECE != 0),
+        "Flag urg:" + strconv.FormatBool(t.Flags&TCP_URG != 0),
+        "Flag ack:" + strconv.FormatBool(t.Flags&TCP_ACK != 0),
+        "Flag psh:" + strconv.FormatBool(t.Flags&TCP_PSH != 0),
+        "Flag rst:" + strconv.FormatBool(t.Flags&TCP_RST != 0),
+        "Flag syn:" + strconv.FormatBool(t.Flags&TCP_SYN != 0),
+        "Flag fin:" + strconv.FormatBool(t.Flags&TCP_FIN != 0),
+
+        "Window size:" + strconv.Itoa(int(t.Window)),
+        "Checksum:" + strconv.Itoa(int(t.Checksum)),
+
+        "Data size:" + strconv.Itoa(len(t.Data)),
+        "Data:" + string(t.Data),
+    }, "\n")
+}
+
+type BySeq []*TCPPacket
+
+func (a BySeq) Len() int            { return len(a) }
+func (a BySeq) Swap(i, j int)       { a[i], a[j] = a[j], a[i] }
+func (a BySeq) Less(i, j int) bool  { return a[i].Seq < a[j].Seq }

replay/replay.go

@@ -1,98 +0,0 @@
-// Replay server receive requests objects from Listeners and forward it to given address.
-// Basic usage:
-//
-//     gor replay -f http://staging.server
-//
-//
-// Rate limiting
-//
-// It can be useful if you want forward only part of production traffic, not to overload staging environment. You can specify desired request per second using "|" operator after server address:
-//
-//     # staging.server not get more than 10 requests per second
-//     gor replay -f "http://staging.server|10"
-//
-//
-// Forward to multiple addresses
-//
-// Just separate addresses by coma:
-//    gor replay -f "http://staging.server|10,http://dev.server|20"
-//
-//
-//  For more help run:
-//
-//     gor replay -h
-//
-package replay
-
-import (
-	"bufio"
-	"bytes"
-	"log"
-	"net"
-	"net/http"
-)
-
-const bufSize = 1024 * 10
-
-// Enable debug logging only if "--verbose" flag passed
-func Debug(v ...interface{}) {
-	if Settings.verbose {
-		log.Println(v...)
-	}
-}
-
-func ParseRequest(data []byte) (request *http.Request, err error) {
-	buf := bytes.NewBuffer(data)
-	reader := bufio.NewReader(buf)
-
-	request, err = http.ReadRequest(reader)
-	return
-}
-
-// Because its sub-program, Run acts as `main`
-// Replay server listen to UDP traffic from Listeners
-// Each request processed by RequestFactory
-func Run() {
-	var buf [bufSize]byte
-
-	addr, err := net.ResolveUDPAddr("udp", Settings.Address())
-	if err != nil {
-		log.Fatal("Can't start:", err)
-	}
-
-	conn, err := net.ListenUDP("udp", addr)
-	log.Println("Starting replay server at:", Settings.Address())
-
-	if err != nil {
-		log.Fatal("Can't start:", err)
-	}
-
-	defer conn.Close()
-
-	for _, host := range Settings.ForwardedHosts() {
-		log.Println("Forwarding requests to:", host.Url, "limit:", host.Limit)
-	}
-
-	requestFactory := NewRequestFactory()
-
-	for {
-		n, _, err := conn.ReadFromUDP(buf[0:])
-
-		if err != nil {
-			continue
-		}
-
-		if n > 0 {
-			if n > bufSize {
-				Debug("Too large udp packet", bufSize)
-			}
-
-			if request, err := ParseRequest(buf[0:n]); err != nil {
-				Debug("Error while parsing request", err, buf[0:n])
-			} else {
-				requestFactory.Add(request)
-			}
-		}
-	}
-
-}

replay/request_factory.go

@@ -1,92 +0,0 @@
-package replay
-
-import (
-	"net/http"
-	"net/url"
-)
-
-type HttpResponse struct {
-	host *ForwardHost
-	req  *http.Request
-	resp *http.Response
-	err  error
-}
-
-// Class for processing requests
-//
-// Basic workflow:
-//
-// 1. When request added via Add() it get pushed to `responses` chan
-// 2. handleRequest() listen for `responses` chan and decide where request should be forwarded, and apply rate-limit if needed
-// 3. sendRequest() forwards request and returns response info to `responses` chan
-// 4. handleRequest() listen for `response` channel and updates stats
-type RequestFactory struct {
-	responses chan *HttpResponse
-	requests  chan *http.Request
-}
-
-// RequestFactory contstuctor
-// One created, it starts listening for incoming requests: requests channel
-func NewRequestFactory() (factory *RequestFactory) {
-	factory = &RequestFactory{}
-	factory.responses = make(chan *HttpResponse)
-	factory.requests = make(chan *http.Request)
-
-	go factory.handleRequests()
-
-	return
-}
-
-// Forward http request to given host
-func (f *RequestFactory) sendRequest(host *ForwardHost, request *http.Request) {
-	client := &http.Client{}
-
-	URL := host.Url + request.URL.String()
-
-	request.RequestURI = ""
-	request.URL, _ = url.ParseRequestURI(URL)
-
-	if Settings.verbose {
-		Debug("Sending request:", request)
-	}
-
-	resp, err := client.Do(request)
-
-	if err == nil {
-		defer resp.Body.Close()
-	} else {
-		Debug("Request error:", err)
-	}
-
-	f.responses <- &HttpResponse{host, request, resp, err}
-}
-
-// Handle incoming requests, and they responses
-func (f *RequestFactory) handleRequests() {
-	hosts := Settings.ForwardedHosts()
-
-	for {
-		select {
-		case req := <-f.requests:
-			for _, host := range hosts {
-				// Ensure that we have actual stats for given timestamp
-				host.Stat.Touch()
-
-				if host.Limit == 0 || host.Stat.Count < host.Limit {
-					// Increment Stat.Count
-					host.Stat.IncReq()
-
-					go f.sendRequest(host, req)
-				}
-			}
-		case resp := <-f.responses:
-			// Increment returned http code stats, and elapsed time
-			resp.host.Stat.IncResp(resp)
-		}
-	}
-}
-
-// Add request to channel for further processing
-func (f *RequestFactory) Add(request *http.Request) {
-	f.requests <- request
-}

replay/request_stats.go

@@ -1,63 +0,0 @@
-package replay
-
-import (
-	"time"
-)
-
-// Stats stores in context of current timestamp
-type RequestStat struct {
-	timestamp int64
-
-	Codes map[int]int // { 200: 10, 404:2, 500:1 }
-
-	Count  int // All requests including errors
-	Errors int // Rquests with errors (timeout or host not reachable). Not include 50x errors.
-
-	host *ForwardHost
-}
-
-// Ensure that current stats is actual (for current timestamp)
-func (s *RequestStat) Touch() {
-	if s.timestamp != time.Now().Unix() {
-		s.reset()
-	}
-}
-
-// Called on request start
-func (s *RequestStat) IncReq() {
-	s.Count++
-}
-
-// Called after response
-func (s *RequestStat) IncResp(resp *HttpResponse) {
-	s.Touch()
-
-	if resp.err != nil {
-		s.Errors++
-		return
-	}
-
-	s.Codes[resp.resp.StatusCode]++
-}
-
-// Updated stats timestamp to current time and reset to zero all stats values
-// TODO: Further on reset it should write stats to file
-func (s *RequestStat) reset() {
-	if s.timestamp != 0 {        
-	   Debug("Host:", s.host.Url, "Requests:", s.Count, "Errors:", s.Errors, "Status codes:", s.Codes)
-	}
-
-	s.timestamp = time.Now().Unix()
-
-	s.Codes = make(map[int]int)
-	s.Count = 0
-	s.Errors = 0
-}
-
-// RequestStat constructor
-func NewRequestStats(host *ForwardHost) (stat *RequestStat) {
-	stat = &RequestStat{host: host}
-	stat.reset()
-
-	return
-}

replay/settings.go

@@ -1,80 +0,0 @@
-package replay
-
-import (
-	"strconv"
-	"strings"
-	"os"
-	"flag"
-)
-
-type ForwardHost struct {
-	Url   string
-	Limit int
-
-	Stat *RequestStat
-}
-
-type ReplaySettings struct {
-	port int
-	host string
-
-	forwardAddress string
-
-	verbose bool
-}
-
-// ForwardedHosts implements forwardAddress syntax support for multiple hosts (coma separated), and rate limiting by specifing "|maxRps" after host name.
-//
-//    -f "host1,http://host2|10,host3"
-//
-func (r *ReplaySettings) ForwardedHosts() (hosts []*ForwardHost) {
-	hosts = make([]*ForwardHost, 0, 10)
-
-	for _, address := range strings.Split(r.forwardAddress, ",") {
-		host_info := strings.Split(address, "|")
-
-		if strings.Index(host_info[0], "http") == -1 {
-			host_info[0] = "http://" + host_info[0]
-		}
-
-		host := &ForwardHost{Url: host_info[0]}
-		host.Stat = NewRequestStats(host)
-
-		if len(host_info) > 1 {
-			host.Limit, _ = strconv.Atoi(host_info[1])
-		}
-
-		hosts = append(hosts, host)
-	}
-
-	return
-}
-
-// Helper to return address with port, e.g.: 127.0.0.1:28020
-func (r *ReplaySettings) Address() string {
-	return r.host + ":" + strconv.Itoa(r.port)
-}
-
-var Settings ReplaySettings = ReplaySettings{}
-
-
-func init() {
-	if len(os.Args) < 2 || os.Args[1] != "replay" {
-		return
-	}
-
-	const (
-		defaultPort = 28020
-		defaultHost = "0.0.0.0"
-
-		defaultAddress = "http://localhost:8080"
-	)
-
-	flag.IntVar(&Settings.port, "p", defaultPort, "specify port number")
-
-	flag.StringVar(&Settings.host, "ip", defaultHost, "ip addresses to listen on")
-
-	flag.StringVar(&Settings.forwardAddress, "f", defaultAddress, "http address to forward traffic.\n\tYou can limit requests per second by adding `|num` after address.\n\tIf you have multiple addresses with different limits. For example: http://staging.example.com|100,http://dev.example.com|10")
-
-	flag.BoolVar(&Settings.verbose, "verbose", false, "Log requests")
-}

settings.go

@@ -0,0 +1,121 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+	"log"
+	"os"
+)
+
+const (
+	VERSION = "0.9.7"
+)
+
+// Allows to specify multiple flags with same name and collects all values to array
+type MultiOption []string
+
+func (h *MultiOption) String() string {
+	return fmt.Sprint(*h)
+}
+
+func (h *MultiOption) Set(value string) error {
+	*h = append(*h, value)
+	return nil
+}
+
+type AppSettings struct {
+	verbose bool
+	debug bool
+	stats   bool
+
+	splitOutput bool
+
+	inputDummy  MultiOption
+	outputDummy MultiOption
+
+	inputTCP       MultiOption
+	outputTCP      MultiOption
+	outputTCPStats bool
+
+	inputFile  MultiOption
+	outputFile MultiOption
+
+	inputRAW MultiOption
+
+	inputHTTP  MultiOption
+	outputHTTP MultiOption
+
+	outputHTTPConfig HTTPOutputConfig
+	modifierConfig   HTTPModifierConfig
+}
+
+var Settings AppSettings = AppSettings{}
+
+func usage() {
+	fmt.Printf("Gor is a simple http traffic replication tool written in Go. Its main goal is to replay traffic from production servers to staging and dev environments.\nProject page: https://github.com/buger/gor\nAuthor: <Leonid Bugaev> leonsbox@gmail.com\nCurrent Version: %s\n\n", VERSION)
+	flag.PrintDefaults()
+	os.Exit(2)
+}
+
+func init() {
+	flag.Usage = usage
+
+	flag.BoolVar(&Settings.verbose, "verbose", false, "Turn on more verbose output")
+	flag.BoolVar(&Settings.debug, "debug", false, "Turn on debug output, shows all itercepted traffic. Works only when with `verbose` flag")
+	flag.BoolVar(&Settings.stats, "stats", false, "Turn on queue stats output")
+
+	flag.BoolVar(&Settings.splitOutput, "split-output", false, "By default each output gets same traffic. If set to `true` it splits traffic equally among all outputs.")
+
+	flag.Var(&Settings.inputDummy, "input-dummy", "Used for testing outputs. Emits 'Get /' request every 1s")
+	flag.Var(&Settings.outputDummy, "output-dummy", "Used for testing inputs. Just prints data coming from inputs.")
+
+	flag.Var(&Settings.inputTCP, "input-tcp", "Used for internal communication between Gor instances. Example: \n\t# Receive requests from other Gor instances on 28020 port, and redirect output to staging\n\tgor --input-tcp :28020 --output-http staging.com")
+	flag.Var(&Settings.outputTCP, "output-tcp", "Used for internal communication between Gor instances. Example: \n\t# Listen for requests on 80 port and forward them to other Gor instance on 28020 port\n\tgor --input-raw :80 --output-tcp replay.local:28020")
+	flag.BoolVar(&Settings.outputTCPStats, "output-tcp-stats", false, "Report TCP output queue stats to console every 5 seconds.")
+
+	flag.Var(&Settings.inputFile, "input-file", "Read requests from file: \n\tgor --input-file ./requests.gor --output-http staging.com")
+	flag.Var(&Settings.outputFile, "output-file", "Write incoming requests to file: \n\tgor --input-raw :80 --output-file ./requests.gor")
+
+	flag.Var(&Settings.inputRAW, "input-raw", "Capture traffic from given port (use RAW sockets and require *sudo* access):\n\t# Capture traffic from 8080 port\n\tgor --input-raw :8080 --output-http staging.com")
+
+	flag.Var(&Settings.inputHTTP, "input-http", "Read requests from HTTP, should be explicitly sent from your application:\n\t# Listen for http on 9000\n\tgor --input-http :9000 --output-http staging.com")
+
+	flag.Var(&Settings.outputHTTP, "output-http", "Forwards incoming requests to given http address.\n\t# Redirect all incoming requests to staging.com address \n\tgor --input-raw :80 --output-http http://staging.com")
+	flag.IntVar(&Settings.outputHTTPConfig.workers, "output-http-workers", 0, "Gor uses dynamic worker scaling by default.  Enter a number to run a set number of workers.")
+	flag.IntVar(&Settings.outputHTTPConfig.redirectLimit, "output-http-redirects", 0, "Enable how often redirects should be followed.")
+
+	flag.BoolVar(&Settings.outputHTTPConfig.stats, "output-http-stats", false, "Report http output queue stats to console every 5 seconds.")
+
+	flag.StringVar(&Settings.outputHTTPConfig.elasticSearch, "output-http-elasticsearch", "", "Send request and response stats to ElasticSearch:\n\tgor --input-raw :8080 --output-http staging.com --output-http-elasticsearch 'es_host:api_port/index_name'")
+
+	flag.Var(&Settings.modifierConfig.headers, "http-set-header", "Inject additional headers to http reqest:\n\tgor --input-raw :8080 --output-http staging.com --http-set-header 'User-Agent: Gor'")
+	flag.Var(&Settings.modifierConfig.headers, "output-http-header", "WARNING: `--output-http-header` DEPRECATED, use `--http-set-header` instead")
+
+	flag.Var(&Settings.modifierConfig.params, "http-set-param", "Set request url param, if param already exists it will be overwritten:\n\tgor --input-raw :8080 --output-http staging.com --http-set-param api_key=1")
+
+	flag.Var(&Settings.modifierConfig.methods, "http-allow-method", "Whitelist of HTTP methods to replay. Anything else will be dropped:\n\tgor --input-raw :8080 --output-http staging.com --http-allow-method GET --http-allow-method OPTIONS")
+	flag.Var(&Settings.modifierConfig.methods, "output-http-method", "WARNING: `--output-http-method` DEPRECATED, use `--http-allow-method` instead")
+
+	flag.Var(&Settings.modifierConfig.urlRegexp, "http-allow-url", "A regexp to match requests against. Filter get matched agains full url with domain. Anything else will be dropped:\n\t gor --input-raw :8080 --output-http staging.com --http-allow-url ^www.")
+	flag.Var(&Settings.modifierConfig.urlRegexp, "output-http-url-regexp", "WARNING: `--output-http-url-regexp` DEPRECATED, use `--http-allow-url` instead")
+
+	flag.Var(&Settings.modifierConfig.urlNegativeRegexp, "http-diallow-url", "A regexp to match requests against. Filter get matched agains full url with domain. Anything else will be dropped:\n\t gor --input-raw :8080 --output-http staging.com --http-disallow-url ^www.")
+
+	flag.Var(&Settings.modifierConfig.urlRewrite, "http-rewrite-url", "Rewrite the request url based on a mapping:\n\tgor --input-raw :8080 --output-http staging.com --http-rewrite-url /v1/user/([^\\/]+)/ping:/v2/user/$1/ping")
+	flag.Var(&Settings.modifierConfig.urlRewrite, "output-http-rewrite-url", "WARNING: `--output-http-rewrite-url` DEPRECATED, use `--http-rewrite-url` instead")
+
+	flag.Var(&Settings.modifierConfig.headerFilters, "http-allow-header", "A regexp to match a specific header against. Requests with non-matching headers will be dropped:\n\t gor --input-raw :8080 --output-http staging.com --http-allow-header api-version:^v1")
+	flag.Var(&Settings.modifierConfig.headerFilters, "output-http-header-filter", "WARNING: `--output-http-header-filter` DEPRECATED, use `--http-allow-header` instead")
+
+	flag.Var(&Settings.modifierConfig.headerHashFilters, "http-header-limiter", "Takes a fraction of requests, consistently taking or rejecting a request based on the FNV32-1A hash of a specific header:\n\t gor --input-raw :8080 --output-http staging.com --http-header-imiter user-id:25%")
+	flag.Var(&Settings.modifierConfig.headerHashFilters, "output-http-header-hash-filter", "WARNING: `output-http-header-hash-filter` DEPRECATED, use `--http-header-hash-limiter` instead")
+
+	flag.Var(&Settings.modifierConfig.paramHashFilters, "http-param-limiter", "Takes a fraction of requests, consistently taking or rejecting a request based on the FNV32-1A hash of a specific GET param:\n\t gor --input-raw :8080 --output-http staging.com --http-param-limiter user_id:25%")
+}
+
+func Debug(args ...interface{}) {
+	if Settings.verbose {
+		fmt.Print("[DEBUG] ")
+		log.Println(args...)
+	}
+}

test_input.go

@@ -0,0 +1,54 @@
+package main
+
+import (
+	"crypto/rand"
+	"encoding/base64"
+)
+
+type TestInput struct {
+	data chan []byte
+}
+
+func NewTestInput() (i *TestInput) {
+	i = new(TestInput)
+	i.data = make(chan []byte, 100)
+
+	return
+}
+
+func (i *TestInput) Read(data []byte) (int, error) {
+	buf := <-i.data
+	copy(data, buf)
+
+	return len(buf), nil
+}
+
+func (i *TestInput) EmitGET() {
+	i.data <- []byte("GET / HTTP/1.1\r\n\r\n")
+}
+
+func (i *TestInput) EmitPOST() {
+	i.data <- []byte("POST /pub/WWW/ HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
+}
+
+func (i *TestInput) EmitChunkedPOST() {
+	i.data <- []byte("POST /pub/WWW/ HTTP/1.1\r\nHost: www.w3.org\r\nTransfer-Encoding: chunked\r\n\r\n4\r\nWiki\r\n5\r\npedia\r\ne\r\n in\r\n\r\nchunks.\r\n0\r\n\r\n")
+}
+
+func (i *TestInput) EmitFile() {
+	size := 1024 * 1024 // 1 MB
+	rb := make([]byte, size)
+	rand.Read(rb)
+
+	rs := base64.URLEncoding.EncodeToString(rb)
+
+	i.data <- []byte("POST /pub/WWW/ HTTP/1.1\nHost: www.w3.org\r\n\r\n------WebKitFormBoundaryePkpFF7tjBAqx29L\nContent-Disposition: form-data; name=\"uploadedfile\"; filename=\"hello.o\"\nContent-Type: application/x-object\n" + rs + "\r\n\r\n")
+}
+
+func (i *TestInput) EmitOPTIONS() {
+	i.data <- []byte("OPTIONS / HTTP/1.1\nHost: www.w3.org\r\n\r\n")
+}
+
+func (i *TestInput) String() string {
+	return "Test Input"
+}

test_output.go

@@ -0,0 +1,24 @@
+package main
+
+type writeCallback func(data []byte)
+
+type TestOutput struct {
+	cb writeCallback
+}
+
+func NewTestOutput(cb writeCallback) (i *TestOutput) {
+	i = new(TestOutput)
+	i.cb = cb
+
+	return
+}
+
+func (i *TestOutput) Write(data []byte) (int, error) {
+	i.cb(data)
+
+	return len(data), nil
+}
+
+func (i *TestOutput) String() string {
+	return "Test Input"
+}