Update version

Settings improvements

.dockerignore

@@ -0,0 +1 @@
+*.tar.gz

.gitignore

@@ -1,2 +1,4 @@
 *.swp
 *.gor
+
+*.out

.travis.yml

@@ -1,3 +1,3 @@
 language: go 
 go: 1.3.3
-script: sudo -E bash -c "source /etc/profile && gvm use go1.3.3 && export GOPATH=$HOME/gopath:$GOPATH && go get && GORACE='halt_on_error=1' go test -race -v"
+script: sudo -E bash -c "source /etc/profile && eval '$(gimme 1.4)' && export GOPATH=$HOME/gopath:$GOPATH && go get && GORACE='halt_on_error=1' go test -race -v"

Dockerfile

@@ -2,8 +2,8 @@ FROM google/golang
 
 RUN cd /goroot/src/ && GOOS=linux GOARCH=386 ./make.bash --no-clean
 
-WORKDIR /gopath/src/gor
+WORKDIR /gopath/src/github.com/buger/gor/
 
-ADD . /gopath/src/gor
+ADD . /gopath/src/github.com/buger/gor/
 
 RUN go get

Makefile

@@ -1,28 +1,37 @@
-SOURCE = emitter.go gor.go gor_stat.go input_dummy.go input_file.go input_raw.go input_tcp.go limiter.go output_dummy.go output_file.go input_http.go output_http.go output_tcp.go plugins.go settings.go settings_header_filters.go settings_header_hash_filters.go settings_headers.go settings_methods.go settings_option.go settings_url_regexp.go test_input.go elasticsearch.go settings_url_map.go
+SOURCE = emitter.go gor.go gor_stat.go input_dummy.go input_file.go input_raw.go input_tcp.go limiter.go output_dummy.go output_file.go input_http.go output_http.go output_tcp.go plugins.go settings.go test_input.go elasticsearch.go http_modifier.go http_modifier_settings.go http_client.go
+
+SOURCE_PATH = /gopath/src/github.com/buger/gor/
 
 release: release-x86 release-x64
 
 release-x64:
-	docker run -v `pwd`:/gopath/src/gor -t --env GOOS=linux --env GOARCH=amd64 --env CGO_ENABLED=0 -i gor go build && tar -czf gor_x64.tar.gz gor && rm gor
-	
+	docker run -v `pwd`:$(SOURCE_PATH) -t --env GOOS=linux --env GOARCH=amd64 --env CGO_ENABLED=0 -i gor go build && tar -czf gor_x64.tar.gz gor && rm gor
+
 release-x86:
-	docker run -v `pwd`:/gopath/src/gor -t --env GOOS=linux --env GOARCH=386 --env CGO_ENABLED=0 -i gor go build && tar -czf gor_x86.tar.gz gor && rm gor
+	docker run -v `pwd`:$(SOURCE_PATH) -t --env GOOS=linux --env GOARCH=386 --env CGO_ENABLED=0 -i gor go build && tar -czf gor_x86.tar.gz gor && rm gor
 
 dbuild:
 	docker build -t gor .
 
 dtest:
-	docker run -v `pwd`:/gopath/src/gor -t -i --env GORACE="halt_on_error=1" gor go test $(ARGS) -race -v
+	docker run -v `pwd`:$(SOURCE_PATH) -t -i --env GORACE="halt_on_error=1" gor go test ./... $(ARGS) -race -v -timeout 15s
+
+dcover:
+	docker run -v `pwd`:$(SOURCE_PATH) -t -i --env GORACE="halt_on_error=1" gor go test $(ARGS) -race -v -timeout 15s -coverprofile=coverage.out
+	go tool cover -html=coverage.out
 
 dfmt:
-	docker run -v `pwd`:/gopath/src/gor -t -i gor go fmt
+	docker run -v `pwd`:$(SOURCE_PATH) -t -i gor go fmt
+
+dvet:
+	docker run -v `pwd`:$(SOURCE_PATH) -t -i gor go vet
 
 dbench:
-	docker run -v `pwd`:/gopath/src/gor -t -i gor go test -v -run NOT_EXISTING -bench HTTP
+	docker run -v `pwd`:$(SOURCE_PATH) -t -i gor go test -v -run NOT_EXISTING -bench HTTP
 
 # Used mainly for debugging, because docker container do not have access to parent machine ports
 drun:
-	docker run -v `pwd`:/gopath/src/gor -t -i gor go run $(SOURCE) --input-dummy=0 --input-http=:9000 --output-http="http://localhost:9000"  --verbose
+	docker run -v `pwd`:$(SOURCE_PATH) -t -i gor go run $(SOURCE) --input-dummy=0 --output-http="http://localhost:9000"  --verbose -h
 
 dbash: 
-	docker run -v `pwd`:/gopath/src/gor -t -i gor /bin/bash
+	docker run -v `pwd`:$(SOURCE_PATH) -t -i gor /bin/bash

README.md

@@ -1,4 +1,3 @@
-[![Stories in Ready](https://badge.waffle.io/buger/gor.png?label=ready)](https://waffle.io/buger/gor)
 [![Build Status](https://travis-ci.org/buger/gor.png?branch=master)](https://travis-ci.org/buger/gor)
 
 ## About
@@ -10,7 +9,7 @@ Its main goal is to replay traffic from production servers to staging and dev en
 Now you can test your code on real user sessions in an automated and repeatable fashion.
 **No more falling down in production!**
 
-Here is basic workflow: The listener server catches http traffic and sends it to the replay server or saves to file.The replay server forwards traffic to a given address.
+Here is basic workflow: The listener server catches http traffic and sends it to the replay server or saves to file. The replay server forwards traffic to a given address.
 
 
 ![Diagram](http://i.imgur.com/9mqj2SK.png)
@@ -27,6 +26,8 @@ sudo gor --input-raw :80 --output-tcp replay.local:28020
 gor --input-tcp replay.local:28020 --output-http http://staging.com
 ```
 
+Since Gor use raw sockets to capture traffic it require `sudo` access. Alternatively you can allow access to raw sockets like this: `sudo setcap CAP_NET_RAW=ep gor`
+
 ### Using 1 Gor instance for both listening and replaying
 It's recommended to use separate server for replaying traffic, but if you have enough CPU resources you can use single Gor instance.
 
@@ -34,7 +35,7 @@ It's recommended to use separate server for replaying traffic, but if you have e
 sudo gor --input-raw :80 --output-http "http://staging.com"
 ```
 
-### Guarante of replay and HTTP input
+### Guarantee of replay and HTTP input
 Due to how traffic interception works, there is chance of missing requests. If you want guarantee that requests will be replayed you can use http input, but it will require changes in your app as well. 
 
 ```
@@ -43,19 +44,43 @@ sudo gor --input-http :28019 --output-http "http://staging.com"
 
 Then in your application you should send copy (e.g. like reverse proxy) all incoming requests to Gor http input. 
 
+## Configuration
 
-## Advanced use
+### Forward to multiple addresses
+
+You can forward traffic to multiple endpoints. Just add multiple --output-* arguments.
+```
+gor --input-tcp :28020 --output-http "http://staging.com"  --output-http "http://dev.com"
+```
+
+#### Splitting traffic
+By default it will send same traffic to all outputs, but you have options to equally split it:
+
+```
+gor --input-tcp :28020 --output-http "http://staging.com"  --output-http "http://dev.com" --split-output true
+```
+
+### HTTP output workers
+By default Gor creates dynamic pull of workers: it starts with 10 and create more http output workers when the http output queue length is greater than 10.  The number of workers created (N) is equal to the queue length at the time which it is checked and found to have a length greater than 10. The queue length is checked every time a message is written to the http output queue.  No more workers will be spawned until that request to spawn N workers is satisfied.  If a dynamic worker cannot process a message at that time, it will sleep for 100 milliseconds. If a dynamic worker cannot process a message for 2 seconds it dies.  
+You may specify fixed number of workers using  `--output-http-workers=20` option.
+
+### Follow redirects
+By default Gor will ignore all redirects since they are handled by clients using your app, but in scenarios when your replayed environment introduce new redirects, you can enable them like this: 
+```
+gor --input-tcp replay.local:28020 --output-http http://staging.com --output-http-redirects 2
+```
+The given example will follow up to 2 redirects per request.
 
 ### Rate limiting
-Every input and output support rate limiting. It can be useful if you want
-forward only part of production traffic and not overload your staging
-environment. 
+Rate limiting can be useful if you want forward only part of production traffic and not overload your staging environment. There is 2 strategies: dropping random requests or dropping fraction of requests based on Header or URL param value. 
 
+#### Dropping random requests
+Every input and output support random rate limiting.
 There are 2 limiting algorithms: absolute or percentage based. 
 
 Absolute: If for current second it reached specified requests limit - disregard the rest, on next second counter reseted.
 
-Percentage: For input-file it will slowdown or speedup request execution, for the rest it will use random generator to decide if request pass or not based on weight you specified. 
+Percentage: For input-file it will slowdown or speedup request execution, for the rest it will use random generator to decide if request pass or not based on chance you specified. 
 
 You can specify your desired limit using the
 "|" operator after the server address:
@@ -73,51 +98,72 @@ gor --input-tcp :28020 --output-http "http://staging.com|10"
 gor --input-raw :80 --output-tcp "replay.local:28020|10%"
 ```
 
-### Load testing
-
-Currently it supported only by `input-file` and only when using percentage based limiter. Unlike default limiter for `input-file` instead of dropping requests it will slowdown or speedup request emitting.
-
-```
-# Replay from file on 2x speed 
-gor --input-file "requests.gor|200%" --output-http "staging.com"
+#### Limiting based on Header or URL param value
+If you have unique user id (like API key) stored in header or URL you can consistently forward specified percent of traffic only for fraction of this users. 
+Basic formula looks like this: `FNV32-1A_hashing(value) % 100 >= chance`. Examples:
 ```
+# Limit based on header value
+gor --input-raw :80 --output-tcp "replay.local:28020|10%" --http-header-limiter "X-API-KEY: 10%"
 
+# Limit based on header value
+gor --input-raw :80 --output-tcp "replay.local:28020|10%" --http-param-limiter "api_key: 10%"
+```
+Only percentage based limiting supported.
 
 ### Filtering 
 
-#### Match on regexp of url
+#### Allow url regexp
 ```
-# only forward requests being sent to the api... domains
-gor --input-raw :8080 --output-http staging.com --output-http-url-regexp ^www.
+# only forward requests being sent to the /api endpoint
+gor --input-raw :8080 --output-http staging.com --http-allow-url /api
 ```
 
+#### Disallow url regexp
+```
+# only forward requests NOT being sent to the /api... endpoint
+gor --input-raw :8080 --output-http staging.com --http-disallow-url /api
+```
 #### Filter based on regexp of header
+
 ```
 # only forward requests with an api version of 1.0x
-gor --input-raw :8080 --output-http staging.com --output-http-header-filter api-version:^1\.0\d
+gor --input-raw :8080 --output-http staging.com --http-allow-header api-version:^1\.0\d
 ```
 
-#### Filter based on hash of header
-```
-# send 1/32 of all users consistently to staging
-gor --input-raw :8080 --output-http staging.com --output-http-header-hash-filter user-id:1/32
-```
-
-### Forward to multiple addresses
-
-You can forward traffic to multiple endpoints. Just add multiple --output-* arguments.
-```
-gor --input-tcp :28020 --output-http "http://staging.com"  --output-http "http://dev.com"
-```
-
-#### Splitting traffic
-By default it will send same traffic to all outputs, but you have options to equally split it:
+#### Filter based on http method
+Requests not matching a specified whitelist can be filtered out. For example to strip non-nullipotent requests:
 
 ```
-gor --input-tcp :28020 --output-http "http://staging.com"  --output-http "http://dev.com" --split-output true
+gor --input-raw :80 --output-http "http://staging.server" \
+    --http-allow-method GET \
+    --http-allow-method OPTIONS
 ```
 
-### Saving requests to file
+### Rewriting original request
+Gor supports built-in basic rewriting support, for complex logic see https://github.com/buger/gor/pull/162
+
+#### Rewrite URL based on a mapping
+```
+# rewrite url to match the following
+gor --input-raw :8080 --output-http staging.com --http-rewrite-url /v1/user/([^\\/]+)/ping:/v2/user/$1/ping
+```
+
+#### Set URL param
+Set request url param, if param already exists it will be overwritten
+```
+gor --input-raw :8080 --output-http staging.com --http-set-param api_key=1
+```
+
+#### Set Header
+Set request header, if header already exists it will be overwritten. This may be useful if you need to identify requests generated by Gor or enable feature flagged functionality in an application:
+
+```
+gor --input-raw :80 --output-http "http://staging.server" \
+    --http-header "User-Agent: Replayed by Gor" \
+    --http-header "Enable-Feature-X: true"
+```
+
+### Saving requests to file and replaying them
 You can save requests to file, and replay them later:
 ```
 # write to file
@@ -129,24 +175,13 @@ gor --input-file requests.gor --output-http "http://staging.com"
 
 **Note:** Replay will preserve the original time differences between requests.
 
-### Injecting headers
+### Load testing
 
-Additional headers can be injected/overwritten into requests during replay. This may be useful if you need to identify requests generated by Gor or enable feature flagged functionality in an application:
+Currently it supported only by `input-file` and only when using percentage based limiter. Unlike default limiter for `input-file` instead of dropping requests it will slowdown or speedup request emitting. Note that unlike examples above limiter is applied to input:
 
 ```
-gor --input-raw :80 --output-http "http://staging.server" \
-    --output-http-header "User-Agent: Replayed by Gor" \
-    --output-http-header "Enable-Feature-X: true"
-```
-
-## Filtering HTTP methods
-
-Requests not matching a specified whitelist can be filtered out. For example to strip non-nullipotent requests:
-
-```
-gor --input-raw :80 --output-http "http://staging.server" \
-    --output-http-method GET \
-    --output-http-method OPTIONS
+# Replay from file on 2x speed 
+gor --input-file "requests.gor|200%" --output-http "staging.com"
 ```
 
 ### Basic Auth
@@ -159,14 +194,52 @@ gor --input-raw :80 --output-http "http://user:pass@staging .com"
 
 Note: This will overwrite any Authorization headers in the original request.
 
-#### Rewrite the target urls based on a mapping
-```
-# rewrite url to match the following
-gor --input-raw :8080 --output-http staging.com --output-http-rewrite-url /xml_test/interface.php:/api/service.do
-```
-
 
 ## Stats 
+
+Gor can report stats on the `output-tcp` and `output-http` request queues. Stats are reported to the console every 5 seconds in the form `latest,mean,max,count,count/second` by using the `--output-http-stats` and `--output-tcp-stats` options.
+
+Examples:
+
+```
+2014/04/23 21:17:50 output_tcp:latest,mean,max,count,count/second
+2014/04/23 21:17:50 output_tcp:0,0,0,0,0
+2014/04/23 21:17:55 output_tcp:1,1,2,68,13
+2014/04/23 21:18:00 output_tcp:1,1,2,92,18
+2014/04/23 21:18:05 output_tcp:1,1,2,119,23
+```
+
+```
+Version: 0.8
+2014/04/23 21:19:46 output_http:latest,mean,max,count,count/second
+2014/04/23 21:19:46 output_http:0,0,0,0,0
+2014/04/23 21:19:51 output_http:0,0,0,0,0
+2014/04/23 21:19:56 output_http:0,0,0,0,0
+2014/04/23 21:20:01 output_http:1,0,1,50,10
+2014/04/23 21:20:06 output_http:1,1,4,72,14
+2014/04/23 21:20:11 output_http:1,0,1,179,35
+2014/04/23 21:20:16 output_http:1,0,1,148,29
+2014/04/23 21:20:21 output_http:1,1,2,91,18
+2014/04/23 21:20:26 output_http:1,1,2,150,30
+2014/04/23 21:18:15 output_http:100,99,100,70,14
+2014/04/23 21:18:21 output_http:100,99,100,55,11
+```
+
+### How can I tell if I have bottlenecks?
+Key areas that sometimes experience bottlenecks are the output-tcp and output-http functions which have internal queues for requests. Each queue has an upper limit of 100. Enable stats reporting to see if any queues are experiencing bottleneck behavior.
+ 
+#### output-http bottlenecks
+When running a Gor replay the output-http feature may bottleneck if:
+
+  * the replay has inadequate bandwidth. If the replay is receiving or sending more messages than its network adapter can handle the output-http-stats  may report that the output-http queue is filling up. See if there is a way to upgrade the replay's bandwidth.
+  * with `--output-http-workers` set to anything other than `-1` the `-output-http` target is unable to respond to messages in a timely manner. The http output workers which take messages off the output-http queue, process the request, and ensure that the request did not result in an error may not be able to keep up with the number of incoming requests. If the replay is not using dynamic worker scaling (`--output-http-workers=-1`)  The optimal number of output-http-workers can be determined with the formula `output-workers = (Average number of requests per second)/(Average target response time per second)`.
+
+#### output-tcp bottlenecks
+When using the Gor listener the output-tcp feature may bottleneck if:
+
+  * the replay is unable to accept and process more requests than the listener is able generate. Prior to troubleshooting the output-tcp bottleneck, ensure that the replay target is not experiencing any bottlenecks. 
+  * the replay target has inadequate bandwidth to handle all its incoming requests.  If a replay target's incoming bandwidth is maxed out the output-tcp-stats may report that the output-tcp queue is filling up. See if there is a way to upgrade the replay's bandwidth.
+
 ### ElasticSearch 
 For deep response analyze based on url, cookie, user-agent and etc. you can export response metadata to ElasticSearch. See [ELASTICSEARCH.md](ELASTICSEARCH.md) for more details.
 
@@ -185,60 +258,55 @@ https://github.com/buger/gor/releases
 ## Command line reference
 `gor -h` output:
 ```
-  -cpuprofile="": write cpu profile to file
-  -memprofile="": write memory profile to this file
-  
+  -http-allow-header=[]: A regexp to match a specific header against. Requests with non-matching headers will be dropped:
+   gor --input-raw :8080 --output-http staging.com --http-allow-header api-version:^v1
+  -http-allow-method=[]: Whitelist of HTTP methods to replay. Anything else will be dropped:
+  gor --input-raw :8080 --output-http staging.com --http-allow-method GET --http-allow-method OPTIONS
+  -http-allow-url=[]: A regexp to match requests against. Filter get matched agains full url with domain. Anything else will be dropped:
+   gor --input-raw :8080 --output-http staging.com --http-allow-url ^www.
+  -http-diallow-url=[]: A regexp to match requests against. Filter get matched agains full url with domain. Anything else will be dropped:
+   gor --input-raw :8080 --output-http staging.com --http-disallow-url ^www.
+  -http-header-limiter=[]: Takes a fraction of requests, consistently taking or rejecting a request based on the FNV32-1A hash of a specific header:
+   gor --input-raw :8080 --output-http staging.com --http-header-imiter user-id:25%
+  -http-param-limiter=[]: Takes a fraction of requests, consistently taking or rejecting a request based on the FNV32-1A hash of a specific GET param:
+   gor --input-raw :8080 --output-http staging.com --http-param-limiter user_id:25%
+  -http-rewrite-url=[]: Rewrite the request url based on a mapping:
+  gor --input-raw :8080 --output-http staging.com --http-rewrite-url /v1/user/([^\/]+)/ping:/v2/user/$1/ping
+  -http-set-header=[]: Inject additional headers to http reqest:
+  gor --input-raw :8080 --output-http staging.com --http-set-header 'User-Agent: Gor'
+  -http-set-param=[]: Set request url param, if param already exists it will be overwritten:
+  gor --input-raw :8080 --output-http staging.com --http-set-param api_key=1
   -input-dummy=[]: Used for testing outputs. Emits 'Get /' request every 1s
-
-  -input-file=[]: Read requests from file: 
-    gor --input-file ./requests.gor --output-http staging.com
-
+  -input-file=[]: Read requests from file:
+  gor --input-file ./requests.gor --output-http staging.com
+  -input-http=[]: Read requests from HTTP, should be explicitly sent from your application:
+  # Listen for http on 9000
+  gor --input-http :9000 --output-http staging.com
   -input-raw=[]: Capture traffic from given port (use RAW sockets and require *sudo* access):
-    # Capture traffic from 8080 port
-    gor --input-raw :8080 --output-http staging.com
-
-  -input-tcp=[]: Used for internal communication between Gor instances. Example: 
-    # Receive requests from other Gor instances on 28020 port, and redirect output to staging
-    gor --input-tcp :28020 --output-http staging.com
-
+  # Capture traffic from 8080 port
+  gor --input-raw :8080 --output-http staging.com
+  -input-tcp=[]: Used for internal communication between Gor instances. Example:
+  # Receive requests from other Gor instances on 28020 port, and redirect output to staging
+  gor --input-tcp :28020 --output-http staging.com
+  -memprofile="": write memory profile to this file
   -output-dummy=[]: Used for testing inputs. Just prints data coming from inputs.
-
-  -output-file=[]: Write incoming requests to file: 
-    gor --input-raw :80 --output-file ./requests.gor
-
+  -output-file=[]: Write incoming requests to file:
+  gor --input-raw :80 --output-file ./requests.gor
   -output-http=[]: Forwards incoming requests to given http address.
-    # Redirect all incoming requests to staging.com address 
-    gor --input-raw :80 --output-http http://staging.com
-
+  # Redirect all incoming requests to staging.com address
+  gor --input-raw :80 --output-http http://staging.com
   -output-http-elasticsearch="": Send request and response stats to ElasticSearch:
-    gor --input-raw :8080 --output-http staging.com --output-http-elasticsearch 'es_host:api_port/index_name'
-
-  -output-http-header=[]: Inject additional headers to http reqest:
-    gor --input-raw :8080 --output-http staging.com --output-http-header 'User-Agent: Gor'
-
-  -output-http-header-filter=[]: A regexp to match a specific header against. Requests with non-matching headers will be dropped:
-    gor --input-raw :8080 --output-http staging.com --output-http-header-filter api-version:^v1
-
-  -output-http-header-hash-filter=[]: Takes a fraction of requests, consistently taking or rejecting a request based on the FNV32-1A hash of a specific header. The fraction must have a denominator that is a power of two:
-    gor --input-raw :8080 --output-http staging.com --output-http-header-hash-filter user-id:1/4
-
-  -output-http-url-regexp=: A regexp to match requests against. Anything else will be dropped:
-    gor --input-raw :8080 --output-http staging.com --output-http-url-regexp ^www.
-    
-  -output-http-workers=-1: Number of http output workers desired. Use default -1 for dynamic worker scaling.  Gor will add http workers if its work queue starts getting too full and kill them .
-    
-  -output-http-stats=false: If set to `true` it gives out queuing stats for the HTTP output every 5 seconds in the form latest,mean,max,count,count/second.
-    
-  -output-tcp=[]: Used for internal communication between Gor instances. Example: 
-    # Listen for requests on 80 port and forward them to other Gor instance on 28020 port
-    gor --input-raw :80 --output-tcp replay.local:28020
-    
-  -output-tcp-stats=false: If set to `true` it gives out queuing stats for the TCP output every 5 seconds in the form latest,mean,max,count,count/second.
-    
+  gor --input-raw :8080 --output-http staging.com --output-http-elasticsearch 'es_host:api_port/index_name'
+  -output-http-header-filter=[]: WARNING: `--output-http-header-filter` DEPRECATED, use `--http-allow-header` instead  -output-http-redirects=0: Enable how often redirects should be followed.
+  -output-http-stats=false: Report http output queue stats to console every 5 seconds.
+  -output-http-workers=0: Gor uses dynamic worker scaling by default.  Enter a number to run a set number of workers.
+  -output-tcp=[]: Used for internal communication between Gor instances. Example:
+  # Listen for requests on 80 port and forward them to other Gor instance on 28020 port
+  gor --input-raw :80 --output-tcp replay.local:28020
+  -output-tcp-stats=false: Report TCP output queue stats to console every 5 seconds.
   -split-output=false: By default each output gets same traffic. If set to `true` it splits traffic equally among all outputs.
-
-  -output-http-rewrite-url=[]: Rewrites the url in the request based on a mapping
-    gor --input-raw :8080 --output-http staging.com --output-http-rewrite-url /xml_test/interface.php:/api/service.do
+  -stats=false: Turn on queue stats output
+  -verbose=false: Turn on verbose/debug output
 ```
 
 ## Building from source
@@ -271,6 +339,16 @@ For now only Linux based. *BSD (including MacOS is not supported yet, check http
 Listener works by sniffing traffic from a given port. It's accessible
 only by using sudo or root access.
 
+### How do you deal with user session to replay the traffic correctly?
+You can rewrite session related headers/params to match your staging environment. If you require custom logic (e.g random token based auth) follow this discussion: https://github.com/buger/gor/issues/154
+
+### Can i use Gor to intercept SSL traffic?
+Basic idea is that SSL was made to protect itself from traffic interception. There 2 options: 
+1. Move SSL handling to proxy like Nginx or Amazon ELB. And allow Gor to listen on upstreams. 
+2. Use `--input-http` so you can duplicate request payload directly from your app to Gor, but it will require your app modifications.
+
+More can be find here: https://github.com/buger/gor/issues/85
+
 ### I'm getting 'too many open files' error
 Typical linux shell has a small open files soft limit at 1024. You can easily raise that when you do this before starting your gor replay process:
   
@@ -278,73 +356,8 @@ Typical linux shell has a small open files soft limit at 1024. You can easily ra
 
 More about ulimit: http://blog.thecodingmachine.com/content/solving-too-many-open-files-exception-red5-or-any-other-application
 
-### What do the stats commands do?
-Gor can report stats on the output-tcp and output-http request queues. Stats are reported to the console every 5 seconds in the form `latest,mean,max,count,count/second` by using the `-output-http-stats` and `-output-tcp-stats` options.
-
-Examples:
-
-```
-2014/04/23 21:17:50 output_tcp:latest,mean,max,count,count/second
-2014/04/23 21:17:50 output_tcp:0,0,0,0,0
-2014/04/23 21:17:55 output_tcp:1,1,2,68,13
-2014/04/23 21:18:00 output_tcp:1,1,2,92,18
-2014/04/23 21:18:05 output_tcp:1,1,2,119,23
-2014/04/23 21:18:10 output_tcp:1,0,1,95,19
-2014/04/23 21:18:15 output_tcp:1,1,2,92,18
-2014/04/23 21:18:20 output_tcp:1,1,2,108,21
-2014/04/23 21:18:25 output_tcp:1,1,2,117,23
-2014/04/23 21:18:30 output_tcp:1,1,2,113,22
-2014/04/23 21:18:35 output_tcp:21,20,21,132,26
-2014/04/23 21:18:40 output_tcp:100,99,100,99,19
-```
-
-```
-Version: 0.8
-2014/04/23 21:19:46 output_http:latest,mean,max,count,count/second
-2014/04/23 21:19:46 output_http:0,0,0,0,0
-2014/04/23 21:19:51 output_http:0,0,0,0,0
-2014/04/23 21:19:56 output_http:0,0,0,0,0
-2014/04/23 21:20:01 output_http:1,0,1,50,10
-2014/04/23 21:20:06 output_http:1,1,4,72,14
-2014/04/23 21:20:11 output_http:1,0,1,179,35
-2014/04/23 21:20:16 output_http:1,0,1,148,29
-2014/04/23 21:20:21 output_http:1,1,2,91,18
-2014/04/23 21:20:26 output_http:1,1,2,150,30
-2014/04/23 21:18:15 output_http:100,99,100,70,14
-2014/04/23 21:18:21 output_http:100,99,100,55,11
-2014/04/23 21:18:28 output_http:100,99,100,55,11
-2014/04/23 21:18:34 output_http:100,99,100,57,11
-2014/04/23 21:18:41 output_http:100,99,100,61,12
-2014/04/23 21:18:48 output_http:100,99,100,56,11
-2014/04/23 21:18:56 output_http:100,99,100,58,11
-2014/04/23 21:19:01 output_http:100,99,100,31,6
-2014/04/23 21:19:08 output_http:100,99,100,61,12
-2014/04/23 21:19:15 output_http:100,99,100,64,12
-2014/04/23 21:19:21 output_http:100,99,100,70,14
-2014/04/23 21:19:28 output_http:100,99,100,61,12
-2014/04/23 21:19:35 output_http:100,99,100,56,11
-```
-
-### How can I tell if I have bottlenecks?
-Key areas that sometimes experience bottlenecks are the output-tcp and output-http functions which have internal queues for requests. Each queue has an upper limit of 100. Enable stats reporting to see if any queues are experiencing bottleneck behavior.
- 
-#### output-http bottlenecks
-When running a Gor replay the output-http feature may bottleneck if:
-
-  * the replay has inadequate bandwidth. If the replay is receiving or sending more messages than its network adapter can handle the output-http-stats  may report that the output-http queue is filling up. See if there is a way to upgrade the replay's bandwidth.
-  * with `--output-http-workers` set to anything other than `-1` the `-output-http` target is unable to respond to messages in a timely manner. The http output workers which take messages off the output-http queue, process the request, and ensure that the request did not result in an error may not be able to keep up with the number of incoming requests. If the replay is not using dynamic worker scaling (`--output-http-workers=-1`)  The optimal number of output-http-workers can be determined with the formula `output-workers = (Average number of requests per second)/(Average target response time per second)`.
-
-#### output-tcp bottlenecks
-When using the Gor listener the output-tcp feature may bottleneck if:
-
-  * the replay is unable to accept and process more requests than the listener is able generate. Prior to troubleshooting the output-tcp bottleneck, ensure that the replay target is not experiencing any bottlenecks. 
-  * the replay target has inadequate bandwidth to handle all its incoming requests.  If a replay target's incoming bandwidth is maxed out the output-tcp-stats may report that the output-tcp queue is filling up. See if there is a way to upgrade the replay's bandwidth.
-
 ### The CPU average across my load-balanced targets is higher than the source
 If you are replaying traffic from multiple listeners to a load-balanced target and you use sticky sessions, you may observe that the target servers have a higher CPU load than the listener servers. This may be because the sticky session cookie of the original load balancer is not honored by the target load balancer thus resulting in requests that would normally hit the same target server hitting different servers on the backend thus reducing some caching benefits gained via the load balancing.  Try running just one listener against one replay target and see if the CPU utilization comparison is more accurate.
- 
-### How does dynamic http worker scaling work?
-By using the Gor setting `--output-http-workers=-1` Gor will create more http output workers when the http output queue length is greater than 10.  The number of workers created (N) is equal to the queue length at the time which it is checked and found to have a length greater than 10. The queue length is checked every time a message is written to the http output queue.  No more workers will be spawned until that request to spawn N workers is satisfied.  If a dynamic worker cannot process a message at that time, it will sleep for 100 milliseconds. If a dynamic worker cannot process a message for 2 seconds it dies.  
 
 ## Tuning
 
@@ -383,4 +396,5 @@ To achieve the top most performance you should tune the source server system lim
 * [TomTom](http://www.tomtom.com/)
 * [3SCALE](http://www.3scale.net/)
 * [Optionlab](http://www.opinionlab.com)
-* To add your company drop me a line to github.com/buger or leonsbox@gmail.com
+* [TubeMogul] (http://tubemogul.com)
+* To add your company drop me a line to github.com/buger or leonsbox@gmail.com

byteutils/byteutils.go

@@ -0,0 +1,37 @@
+package byteutils
+
+func Cut(a []byte, from, to int) []byte {
+    copy(a[from:], a[to:])
+    a = a[:len(a)-to+from]
+
+    return a
+}
+
+func Insert(a []byte, i int, b []byte) []byte {
+    a = append(a, make([]byte, len(b))...)
+    copy(a[i+len(b):], a[i:])
+    copy(a[i:i+len(b)], b)
+
+    return a
+}
+
+// Unlike bytes.Replace it allows you to specify range
+func Replace(a []byte, from, to int, new []byte) []byte {
+    lenDiff := len(new) - (to - from)
+
+    if lenDiff > 0 {
+        // Extend if new segment bigger
+        a = append(a, make([]byte, lenDiff)...)
+        copy(a[to+lenDiff:], a[to:])
+        copy(a[from:from+len(new)], new)
+
+        return a
+    } else if lenDiff < 0 {
+        copy(a[from:], new)
+        copy(a[from+len(new):],a[to:])
+        return a[:len(a) + lenDiff]
+    } else { // same size
+        copy(a[from:], new)
+        return a
+    }
+}

byteutils/byteutils_test.go

@@ -0,0 +1,32 @@
+package byteutils
+
+import (
+    "testing"
+    "bytes"
+)
+
+func TestCut(t *testing.T) {
+    if !bytes.Equal(Cut([]byte("123456"), 2, 4), []byte("1256")) {
+        t.Error("Should properly cut")
+    }
+}
+
+func TestInsert(t *testing.T) {
+    if !bytes.Equal(Insert([]byte("123456"), 2, []byte("abcd")), []byte("12abcd3456")) {
+        t.Error("Should insert into middle of slice")
+    }
+}
+
+func TestReplace(t *testing.T) {
+    if !bytes.Equal(Replace([]byte("123456"), 2, 4, []byte("ab")), []byte("12ab56")) {
+        t.Error("Should replace when same length")
+    }
+
+    if !bytes.Equal(Replace([]byte("123456"), 2, 4, []byte("abcd")), []byte("12abcd56")) {
+        t.Error("Should replace when replacement length bigger")
+    }
+
+    if !bytes.Equal(Replace([]byte("123456"), 2, 5, []byte("ab")), []byte("12ab6")) {
+        t.Error("Should replace when replacement length bigger")
+    }
+}

elasticsearch.go

@@ -4,8 +4,8 @@ import (
 	"encoding/json"
 	"github.com/buger/elastigo/api"
 	"github.com/buger/elastigo/core"
+	"github.com/buger/gor/proto"
 	"log"
-	"net/http"
 	"regexp"
 	"time"
 )
@@ -26,27 +26,27 @@ type ESPlugin struct {
 }
 
 type ESRequestResponse struct {
-	ReqUrl               string         `json:"Req_URL"`
-	ReqMethod            string         `json:"Req_Method"`
-	ReqUserAgent         string         `json:"Req_User-Agent"`
-	ReqAcceptLanguage    string         `json:"Req_Accept-Language,omitempty"`
-	ReqAccept            string         `json:"Req_Accept,omitempty"`
-	ReqAcceptEncoding    string         `json:"Req_Accept-Encoding,omitempty"`
-	ReqIfModifiedSince   string         `json:"Req_If-Modified-Since,omitempty"`
-	ReqConnection        string         `json:"Req_Connection,omitempty"`
-	ReqCookies           []*http.Cookie `json:"Req_Cookies,omitempty"`
-	RespStatus           string         `json:"Resp_Status"`
-	RespStatusCode       int            `json:"Resp_Status-Code"`
-	RespProto            string         `json:"Resp_Proto,omitempty"`
-	RespContentLength    int64          `json:"Resp_Content-Length,omitempty"`
-	RespContentType      string         `json:"Resp_Content-Type,omitempty"`
-	RespTransferEncoding []string       `json:"Resp_Transfer-Encoding,omitempty"`
-	RespContentEncoding  string         `json:"Resp_Content-Encoding,omitempty"`
-	RespExpires          string         `json:"Resp_Expires,omitempty"`
-	RespCacheControl     string         `json:"Resp_Cache-Control,omitempty"`
-	RespVary             string         `json:"Resp_Vary,omitempty"`
-	RespSetCookie        string         `json:"Resp_Set-Cookie,omitempty"`
-	Rtt                  int64          `json:"RTT"`
+	ReqUrl               []byte `json:"Req_URL"`
+	ReqMethod            []byte `json:"Req_Method"`
+	ReqUserAgent         []byte `json:"Req_User-Agent"`
+	ReqAcceptLanguage    []byte `json:"Req_Accept-Language,omitempty"`
+	ReqAccept            []byte `json:"Req_Accept,omitempty"`
+	ReqAcceptEncoding    []byte `json:"Req_Accept-Encoding,omitempty"`
+	ReqIfModifiedSince   []byte `json:"Req_If-Modified-Since,omitempty"`
+	ReqConnection        []byte `json:"Req_Connection,omitempty"`
+	ReqCookies           []byte `json:"Req_Cookies,omitempty"`
+	RespStatus           []byte `json:"Resp_Status"`
+	RespStatusCode       []byte `json:"Resp_Status-Code"`
+	RespProto            []byte `json:"Resp_Proto,omitempty"`
+	RespContentLength    []byte `json:"Resp_Content-Length,omitempty"`
+	RespContentType      []byte `json:"Resp_Content-Type,omitempty"`
+	RespTransferEncoding []byte `json:"Resp_Transfer-Encoding,omitempty"`
+	RespContentEncoding  []byte `json:"Resp_Content-Encoding,omitempty"`
+	RespExpires          []byte `json:"Resp_Expires,omitempty"`
+	RespCacheControl     []byte `json:"Resp_Cache-Control,omitempty"`
+	RespVary             []byte `json:"Resp_Vary,omitempty"`
+	RespSetCookie        []byte `json:"Resp_Set-Cookie,omitempty"`
+	Rtt                  int64  `json:"RTT"`
 	Timestamp            time.Time
 }
 
@@ -111,8 +111,8 @@ func (p *ESPlugin) RttDurationToMs(d time.Duration) int64 {
 	return int64(fl)
 }
 
-func (p *ESPlugin) ResponseAnalyze(req *http.Request, resp *http.Response, start, stop time.Time) {
-	if resp == nil {
+func (p *ESPlugin) ResponseAnalyze(req, resp []byte, start, stop time.Time) {
+	if len(resp) == 0 {
 		// nil http response - skipped elasticsearch export for this request
 		return
 	}
@@ -120,26 +120,26 @@ func (p *ESPlugin) ResponseAnalyze(req *http.Request, resp *http.Response, start
 	rtt := p.RttDurationToMs(stop.Sub(start))
 
 	esResp := ESRequestResponse{
-		ReqUrl:               req.URL.String(),
-		ReqMethod:            req.Method,
-		ReqUserAgent:         req.UserAgent(),
-		ReqAcceptLanguage:    req.Header.Get("Accept-Language"),
-		ReqAccept:            req.Header.Get("Accept"),
-		ReqAcceptEncoding:    req.Header.Get("Accept-Encoding"),
-		ReqIfModifiedSince:   req.Header.Get("If-Modified-Since"),
-		ReqConnection:        req.Header.Get("Connection"),
-		ReqCookies:           req.Cookies(),
-		RespStatus:           resp.Status,
-		RespStatusCode:       resp.StatusCode,
-		RespProto:            resp.Proto,
-		RespContentLength:    resp.ContentLength,
-		RespContentType:      resp.Header.Get("Content-Type"),
-		RespTransferEncoding: resp.TransferEncoding,
-		RespContentEncoding:  resp.Header.Get("Content-Encoding"),
-		RespExpires:          resp.Header.Get("Expires"),
-		RespCacheControl:     resp.Header.Get("Cache-Control"),
-		RespVary:             resp.Header.Get("Vary"),
-		RespSetCookie:        resp.Header.Get("Set-Cookie"),
+		ReqUrl:               proto.Path(req),
+		ReqMethod:            proto.Method(req),
+		ReqUserAgent:         proto.GetHeader(req, "User-Agent"),
+		ReqAcceptLanguage:    proto.GetHeader(req, "Accept-Language"),
+		ReqAccept:            proto.GetHeader(req, "Accept"),
+		ReqAcceptEncoding:    proto.GetHeader(req, "Accept-Encoding"),
+		ReqIfModifiedSince:   proto.GetHeader(req, "If-Modified-Since"),
+		ReqConnection:        proto.GetHeader(req, "Connection"),
+		ReqCookies:           proto.GetHeader(req, "Cookie"),
+		RespStatus:           proto.Status(resp),
+		RespStatusCode:       proto.Status(resp),
+		RespProto:            proto.Method(resp),
+		RespContentLength:    proto.GetHeader(resp, "Content-Length"),
+		RespContentType:      proto.GetHeader(resp, "Content-Type"),
+		RespTransferEncoding: proto.GetHeader(resp, "Transfer-Encoding"),
+		RespContentEncoding:  proto.GetHeader(resp, "Content-Encoding"),
+		RespExpires:          proto.GetHeader(resp, "Expires"),
+		RespCacheControl:     proto.GetHeader(resp, "Cache-Control"),
+		RespVary:             proto.GetHeader(resp, "Vary"),
+		RespSetCookie:        proto.GetHeader(resp, "Set-Cookie"),
 		Rtt:                  rtt,
 		Timestamp:            t,
 	}

emitter.go

@@ -23,15 +23,25 @@ func Start(stop chan int) {
 func CopyMulty(src io.Reader, writers ...io.Writer) (err error) {
 	buf := make([]byte, 5*1024*1024)
 	wIndex := 0
+	modifier := NewHTTPModifier(&Settings.modifierConfig)
 
 	for {
 		nr, er := src.Read(buf)
 		if nr > 0 && len(buf) > nr {
-			Debug("Sending", src, ": ", string(buf[0:nr]))
+			payload := buf[0:nr]
+
+			if modifier != nil {
+				payload = modifier.Rewrite(payload)
+
+				// If modifier tells to skip request
+				if len(payload) == 0 {
+					continue
+				}
+			}
 
 			if Settings.splitOutput {
 				// Simple round robin
-				writers[wIndex].Write(buf[0:nr])
+				writers[wIndex].Write(payload)
 
 				wIndex++
 
@@ -40,7 +50,7 @@ func CopyMulty(src io.Reader, writers ...io.Writer) (err error) {
 				}
 			} else {
 				for _, dst := range writers {
-					dst.Write(buf[0:nr])
+					dst.Write(payload)
 				}
 			}
 

http_client.go

@@ -0,0 +1,147 @@
+package main
+
+import (
+	"crypto/tls"
+	"github.com/buger/gor/proto"
+	"io"
+	"net"
+	"net/url"
+	"strings"
+	"time"
+)
+
+var defaultPorts = map[string]string{
+	"http":  "80",
+	"https": "443",
+}
+
+type HTTPClientConfig struct {
+	FollowRedirects int
+	Debug           bool
+}
+
+type HTTPClient struct {
+	baseURL        string
+	scheme         string
+	host           string
+	conn           net.Conn
+	respBuf        []byte
+	config         *HTTPClientConfig
+	redirectsCount int
+}
+
+func NewHTTPClient(baseURL string, config *HTTPClientConfig) *HTTPClient {
+	if !strings.HasPrefix(baseURL, "http") {
+		baseURL = "http://" + baseURL
+	}
+
+	u, _ := url.Parse(baseURL)
+	if !strings.Contains(u.Host, ":") {
+		u.Host += ":" + defaultPorts[u.Scheme]
+	}
+
+	client := new(HTTPClient)
+	client.baseURL = u.String()
+	client.host = u.Host
+	client.scheme = u.Scheme
+	client.respBuf = make([]byte, 4096*10)
+	client.config = config
+
+	return client
+}
+
+func (c *HTTPClient) Connect() (err error) {
+	c.Disconnect()
+
+	c.conn, err = net.Dial("tcp", c.host)
+
+	if c.scheme == "https" {
+		tlsConn := tls.Client(c.conn, &tls.Config{InsecureSkipVerify: true})
+
+		if err = tlsConn.Handshake(); err != nil {
+			return
+		}
+
+		c.conn = tlsConn
+	}
+
+	return
+}
+
+func (c *HTTPClient) Disconnect() {
+	if c.conn != nil {
+		c.conn.Close()
+		c.conn = nil
+		Debug("Disconnected: ", c.baseURL)
+	}
+}
+
+func (c *HTTPClient) isAlive() bool {
+	one := make([]byte, 1)
+
+	// Ready 1 byte from socket without timeout to check if it not closed
+	c.conn.SetReadDeadline(time.Now().Add(time.Millisecond))
+	if _, err := c.conn.Read(one); err == io.EOF {
+		return false
+	}
+
+	return true
+}
+
+func (c *HTTPClient) Send(data []byte) (response []byte, err error) {
+	if c.conn == nil || !c.isAlive() {
+		Debug("Connecting:", c.baseURL)
+		c.Connect()
+	}
+
+	timeout := time.Now().Add(5 * time.Second)
+
+	c.conn.SetWriteDeadline(timeout)
+
+	data = proto.SetHost(data, []byte(c.baseURL), []byte(c.host))
+
+	if c.config.Debug {
+		Debug("Sending:", string(data))
+	}
+
+	if _, err = c.conn.Write(data); err != nil {
+		Debug("Write error:", err, c.baseURL)
+		return
+	}
+
+	c.conn.SetReadDeadline(timeout)
+	n, err := c.conn.Read(c.respBuf)
+
+	if err != nil {
+		Debug("READ ERRORR!", err, c.conn)
+		return
+	}
+
+	payload := c.respBuf[:n]
+
+	if c.config.Debug {
+		Debug("Received:", string(payload))
+	}
+
+	if c.config.FollowRedirects > 0 && c.redirectsCount < c.config.FollowRedirects {
+		status := payload[9:12]
+
+		// 3xx requests
+		if status[0] == '3' {
+			c.redirectsCount += 1
+
+			location, _, _, _ := proto.Header(payload, []byte("Location"))
+			redirectPayload := []byte("GET " + string(location) + " HTTP/1.1\r\n\r\n")
+
+			if c.config.Debug {
+				Debug("Redirecting to: " + string(location))
+			}
+
+			return c.Send(redirectPayload)
+		}
+	}
+
+	c.redirectsCount = 0
+
+	return payload, err
+}

http_client_test.go

@@ -0,0 +1,269 @@
+package main
+
+import (
+	"io/ioutil"
+	"net"
+	"net/http"
+	"net/http/httptest"
+	"net/http/httputil"
+	"sync"
+	"testing"
+	_ "time"
+)
+
+func TestHTTPClientURLPort(t *testing.T) {
+	c1 := NewHTTPClient("http://example.com", &HTTPClientConfig{})
+	if c1.baseURL != "http://example.com:80" {
+		t.Error("Sould add 80 port for http:", c1.baseURL)
+	}
+
+	c2 := NewHTTPClient("https://example.com", &HTTPClientConfig{})
+	if c2.baseURL != "https://example.com:443" {
+		t.Error("Sould add 443 port for https:", c2.baseURL)
+	}
+
+	c3 := NewHTTPClient("https://example.com:1", &HTTPClientConfig{})
+	if c3.baseURL != "https://example.com:1" {
+		t.Error("Sould use specified port:", c3.baseURL)
+	}
+
+	c4 := NewHTTPClient("example.com", &HTTPClientConfig{})
+	if c4.baseURL != "http://example.com:80" {
+		t.Error("Sould add default protocol:", c4.baseURL)
+	}
+}
+
+func TestHTTPClientSend(t *testing.T) {
+	wg := new(sync.WaitGroup)
+
+	payload := func(reqType string) []byte {
+		switch reqType {
+		case "GET":
+			return []byte("GET / HTTP/1.1\r\n\r\n")
+		case "POST":
+			return []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
+		case "POST_CHUNKED":
+			return []byte("POST / HTTP/1.1\r\nHost: www.w3.org\r\nTransfer-Encoding: chunked\r\n\r\n4\r\nWiki\r\n5\r\npedia\r\ne\r\n in\r\n\r\nchunks.\r\n0\r\n\r\n")
+		}
+
+		return []byte("")
+	}
+
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+
+		if r.Method == "POST" {
+			defer r.Body.Close()
+			body, _ := ioutil.ReadAll(r.Body)
+
+			if len(r.TransferEncoding) > 0 && r.TransferEncoding[0] == "chunked" {
+				if string(body) != "Wikipedia in\r\n\r\nchunks." {
+					t.Error("Wrong POST body:", body, string(body))
+				}
+			} else {
+				if string(body) != "a=1&b=2" {
+					buf, _ := httputil.DumpRequest(r, true)
+					t.Error("Wrong POST body:", string(body), string(buf))
+				}
+			}
+		}
+
+		wg.Done()
+	}))
+
+	client := NewHTTPClient(server.URL, &HTTPClientConfig{Debug: true})
+
+	wg.Add(4)
+	client.Send(payload("POST"))
+	client.Send(payload("GET"))
+	client.Send(payload("POST_CHUNKED"))
+	client.Send(payload("POST"))
+
+	wg.Wait()
+}
+
+func TestHTTPClientHTTPSSend(t *testing.T) {
+	wg := new(sync.WaitGroup)
+
+	payload := func(reqType string) []byte {
+		switch reqType {
+		case "GET":
+			return []byte("GET / HTTP/1.1\r\n\r\n")
+		case "POST":
+			return []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
+		case "POST_CHUNKED":
+			return []byte("POST / HTTP/1.1\r\nHost: www.w3.org\r\nTransfer-Encoding: chunked\r\n\r\n4\r\nWiki\r\n5\r\npedia\r\ne\r\n in\r\n\r\nchunks.\r\n0\r\n\r\n")
+		}
+
+		return []byte("")
+	}
+
+	server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+
+		if r.Method == "POST" {
+			defer r.Body.Close()
+			body, _ := ioutil.ReadAll(r.Body)
+
+			if len(r.TransferEncoding) > 0 && r.TransferEncoding[0] == "chunked" {
+				if string(body) != "Wikipedia in\r\n\r\nchunks." {
+					t.Error("Wrong POST body:", body, string(body))
+				}
+			} else {
+				if string(body) != "a=1&b=2" {
+					buf, _ := httputil.DumpRequest(r, true)
+					t.Error("Wrong POST body:", string(body), string(buf))
+				}
+			}
+		}
+
+		wg.Done()
+	}))
+
+	client := NewHTTPClient(server.URL, &HTTPClientConfig{})
+
+	wg.Add(4)
+	client.Send(payload("POST"))
+	client.Send(payload("GET"))
+	client.Send(payload("POST_CHUNKED"))
+	client.Send(payload("POST"))
+
+	wg.Wait()
+}
+
+func TestHTTPClientServerInstantDisconnect(t *testing.T) {
+	wg := new(sync.WaitGroup)
+
+	GET_payload := []byte("GET / HTTP/1.1\r\n\r\n")
+
+	ln, _ := net.Listen("tcp", ":0")
+
+	go func() {
+		for {
+			conn, _ := ln.Accept()
+			conn.Close()
+
+			wg.Done()
+		}
+	}()
+
+	client := NewHTTPClient(ln.Addr().String(), &HTTPClientConfig{})
+
+	wg.Add(2)
+	client.Send(GET_payload)
+	client.Send(GET_payload)
+
+	wg.Wait()
+}
+
+func TestHTTPClientServerNoKeepAlive(t *testing.T) {
+	wg := new(sync.WaitGroup)
+
+	GET_payload := []byte("GET / HTTP/1.1\r\n\r\n")
+
+	ln, _ := net.Listen("tcp", ":0")
+
+	go func() {
+		for {
+			conn, err := ln.Accept()
+			if err != nil {
+				// handle error
+			}
+
+			buf := make([]byte, 4096)
+			reqLen, err := conn.Read(buf)
+			if err != nil {
+				t.Error("Error reading:", err.Error())
+			}
+			Debug("Received: ", string(buf[0:reqLen]))
+			conn.Write([]byte("OK"))
+
+			// No keep-alive connections
+			conn.Close()
+
+			wg.Done()
+		}
+	}()
+
+	client := NewHTTPClient(ln.Addr().String(), &HTTPClientConfig{})
+
+	wg.Add(2)
+	client.Send(GET_payload)
+	client.Send(GET_payload)
+
+	wg.Wait()
+}
+
+func TestHTTPClientRedirect(t *testing.T) {
+	wg := new(sync.WaitGroup)
+
+	GET_payload := []byte("GET / HTTP/1.1\r\n\r\n")
+
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+
+		if r.URL.Path == "/" {
+			http.Redirect(w, r, "/new", 301)
+		}
+
+		wg.Done()
+	}))
+
+	client := NewHTTPClient(server.URL, &HTTPClientConfig{FollowRedirects: 1, Debug: false})
+
+	// Should do 2 queries
+	wg.Add(2)
+	client.Send(GET_payload)
+
+	wg.Wait()
+}
+
+func TestHTTPClientRedirectLimit(t *testing.T) {
+	wg := new(sync.WaitGroup)
+
+	GET_payload := []byte("GET / HTTP/1.1\r\n\r\n")
+
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+
+		if r.URL.Path == "/" {
+			http.Redirect(w, r, "/r1", 301)
+		}
+
+		if r.URL.Path == "/r1" {
+			http.Redirect(w, r, "/r2", 301)
+		}
+
+		if r.URL.Path == "/r2" {
+			http.Redirect(w, r, "/new", 301)
+		}
+
+		wg.Done()
+	}))
+
+	client := NewHTTPClient(server.URL, &HTTPClientConfig{FollowRedirects: 2, Debug: false})
+
+	// Have 3 redirects + 1 GET, but should do only 2 redirects + GET
+	wg.Add(3)
+	client.Send(GET_payload)
+
+	wg.Wait()
+}
+
+func TestHTTPClientHandleHTTP10(t *testing.T) {
+	wg := new(sync.WaitGroup)
+
+	GET_payload := []byte("GET http://foobar.com/path HTTP/1.0\r\n\r\n")
+
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+
+		if r.URL.Path != "/path" {
+			t.Error("Path not match:", r.URL.Path)
+		}
+
+		wg.Done()
+	}))
+
+	client := NewHTTPClient(server.URL, &HTTPClientConfig{Debug: true})
+
+	wg.Add(1)
+	client.Send(GET_payload)
+
+	wg.Wait()
+}

http_modifier.go

@@ -0,0 +1,141 @@
+package main
+
+import (
+	"bytes"
+	"github.com/buger/gor/proto"
+	"hash/fnv"
+)
+
+type HTTPModifier struct {
+	config *HTTPModifierConfig
+}
+
+func NewHTTPModifier(config *HTTPModifierConfig) *HTTPModifier {
+	// Optimization to skip modifier completely if we do not need it
+	if len(config.urlRegexp) == 0 &&
+		len(config.urlNegativeRegexp) == 0 &&
+		len(config.urlRewrite) == 0 &&
+		len(config.headerFilters) == 0 &&
+		len(config.headerHashFilters) == 0 &&
+		len(config.paramHashFilters) == 0 &&
+		len(config.params) == 0 &&
+		len(config.headers) == 0 &&
+		len(config.methods) == 0 {
+		return nil
+	}
+
+	return &HTTPModifier{config: config}
+}
+
+func (m *HTTPModifier) Rewrite(payload []byte) (response []byte) {
+	if len(m.config.methods) > 0 {
+		method := proto.Method(payload)
+
+		matched := false
+
+		for _, m := range m.config.methods {
+			if bytes.Equal(method, m) {
+				matched = true
+				break
+			}
+		}
+
+		if !matched {
+			return
+		}
+	}
+
+	if len(m.config.headers) > 0 {
+		for _, header := range m.config.headers {
+			payload = proto.SetHeader(payload, []byte(header.Name), []byte(header.Value))
+		}
+	}
+
+	if len(m.config.params) > 0 {
+		for _, param := range m.config.params {
+			payload = proto.SetPathParam(payload, param.Name, param.Value)
+		}
+	}
+
+	if len(m.config.urlRegexp) > 0 {
+		path := proto.Path(payload)
+
+		matched := false
+
+		for _, f := range m.config.urlRegexp {
+			if f.regexp.Match(path) {
+				matched = true
+				break
+			}
+		}
+
+		if !matched {
+			return
+		}
+	}
+
+	if len(m.config.urlNegativeRegexp) > 0 {
+		path := proto.Path(payload)
+
+		for _, f := range m.config.urlNegativeRegexp {
+			if f.regexp.Match(path) {
+				return
+			}
+		}
+	}
+
+	if len(m.config.headerFilters) > 0 {
+		for _, f := range m.config.headerFilters {
+			value, s, _, _ := proto.Header(payload, f.name)
+
+			if s != -1 && !f.regexp.Match(value) {
+				return
+			}
+		}
+	}
+
+	if len(m.config.headerHashFilters) > 0 {
+		for _, f := range m.config.headerHashFilters {
+			value, s, _, _ := proto.Header(payload, f.name)
+
+			if s != -1 {
+				hasher := fnv.New32a()
+				hasher.Write(value)
+
+				if (hasher.Sum32() % 100) >= f.percent {
+					return
+				}
+			}
+		}
+	}
+
+	if len(m.config.paramHashFilters) > 0 {
+		for _, f := range m.config.paramHashFilters {
+			value, s, _ := proto.PathParam(payload, f.name)
+
+			if s != -1 {
+				hasher := fnv.New32a()
+				hasher.Write(value)
+
+				if (hasher.Sum32() % 100) >= f.percent {
+					return
+				}
+			}
+		}
+	}
+
+	if len(m.config.urlRewrite) > 0 {
+		path := proto.Path(payload)
+
+		for _, f := range m.config.urlRewrite {
+			if f.src.Match(path) {
+				path = f.src.ReplaceAll(path, f.target)
+				payload = proto.SetPath(payload, path)
+
+				break
+			}
+		}
+	}
+
+	return payload
+}

http_modifier_settings.go

@@ -0,0 +1,212 @@
+package main
+
+import (
+	"errors"
+	"fmt"
+	"regexp"
+	"strconv"
+	"strings"
+)
+
+type HTTPModifierConfig struct {
+	urlNegativeRegexp HTTPUrlRegexp
+	urlRegexp         HTTPUrlRegexp
+	urlRewrite        UrlRewriteMap
+	headerFilters     HTTPHeaderFilters
+	headerHashFilters HTTPHashFilters
+	paramHashFilters  HTTPHashFilters
+
+	params  HTTPParams
+	headers HTTPHeaders
+	methods HTTPMethods
+}
+
+//
+// Handling of --http-allow-header options
+//
+type headerFilter struct {
+	name   []byte
+	regexp *regexp.Regexp
+}
+
+type HTTPHeaderFilters []headerFilter
+
+func (h *HTTPHeaderFilters) String() string {
+	return fmt.Sprint(*h)
+}
+
+func (h *HTTPHeaderFilters) Set(value string) error {
+	valArr := strings.SplitN(value, ":", 2)
+	if len(valArr) < 2 {
+		return errors.New("need both header and value, colon-delimited (ex. user_id:^169$).")
+	}
+	r, err := regexp.Compile(valArr[1])
+	if err != nil {
+		return err
+	}
+
+	*h = append(*h, headerFilter{name: []byte(valArr[0]), regexp: r})
+
+	return nil
+}
+
+//
+// Handling of --http-allow-header-hash and --http-allow-param-hash options
+//
+type hashFilter struct {
+	name    []byte
+	percent uint32
+}
+
+type HTTPHashFilters []hashFilter
+
+func (h *HTTPHashFilters) String() string {
+	return fmt.Sprint(*h)
+}
+
+func (h *HTTPHashFilters) Set(value string) error {
+	valArr := strings.SplitN(value, ":", 2)
+	if len(valArr) < 2 {
+		return errors.New("need both header and value, colon-delimited (ex. user_id:50%).")
+	}
+
+	f := hashFilter{name: []byte(valArr[0])}
+
+	if strings.Contains(valArr[1], "%") {
+		p, _ := strconv.ParseInt(valArr[1][:len(valArr[1])-1], 0, 0)
+		f.percent = uint32(p)
+	} else if strings.Contains(valArr[1], "/") {
+		// DEPRECATED format
+		var num, den uint64
+
+		fracArr := strings.Split(valArr[1], "/")
+		num, _ = strconv.ParseUint(fracArr[0], 10, 64)
+		den, _ = strconv.ParseUint(fracArr[1], 10, 64)
+
+		f.percent = uint32((float64(num) / float64(den)) * 100)
+	} else {
+		return errors.New("Value should be percent and contain '%'")
+	}
+
+	*h = append(*h, f)
+
+	return nil
+}
+
+//
+// Handling of --http-set-header option
+//
+type HTTPHeaders []HTTPHeader
+type HTTPHeader struct {
+	Name  string
+	Value string
+}
+
+func (h *HTTPHeaders) String() string {
+	return fmt.Sprint(*h)
+}
+
+func (h *HTTPHeaders) Set(value string) error {
+	v := strings.SplitN(value, ":", 2)
+	if len(v) != 2 {
+		return errors.New("Expected `Key: Value`")
+	}
+
+	header := HTTPHeader{
+		strings.TrimSpace(v[0]),
+		strings.TrimSpace(v[1]),
+	}
+
+	*h = append(*h, header)
+	return nil
+}
+
+//
+// Handling of --http-set-param option
+//
+type HTTPParams []HTTPParam
+type HTTPParam struct {
+	Name  []byte
+	Value []byte
+}
+
+func (h *HTTPParams) String() string {
+	return fmt.Sprint(*h)
+}
+
+func (h *HTTPParams) Set(value string) error {
+	v := strings.SplitN(value, "=", 2)
+	if len(v) != 2 {
+		return errors.New("Expected `Key=Value`")
+	}
+
+	param := HTTPParam{
+		[]byte(strings.TrimSpace(v[0])),
+		[]byte(strings.TrimSpace(v[1])),
+	}
+
+	*h = append(*h, param)
+	return nil
+}
+
+//
+// Handling of --http-allow-method option
+//
+type HTTPMethods [][]byte
+
+func (h *HTTPMethods) String() string {
+	return fmt.Sprint(*h)
+}
+
+func (h *HTTPMethods) Set(value string) error {
+	*h = append(*h, []byte(value))
+	return nil
+}
+
+//
+// Handling of --http-rewrite-url option
+//
+type urlRewrite struct {
+	src    *regexp.Regexp
+	target []byte
+}
+
+type UrlRewriteMap []urlRewrite
+
+func (r *UrlRewriteMap) String() string {
+	return fmt.Sprint(*r)
+}
+
+func (r *UrlRewriteMap) Set(value string) error {
+	valArr := strings.SplitN(value, ":", 2)
+	if len(valArr) < 2 {
+		return errors.New("need both src and target, colon-delimited (ex. /a:/b).")
+	}
+	regexp, err := regexp.Compile(valArr[0])
+	if err != nil {
+		return err
+	}
+	*r = append(*r, urlRewrite{src: regexp, target: []byte(valArr[1])})
+	return nil
+}
+
+//
+// Handling of --http-allow-url option
+//
+type urlRegexp struct {
+	regexp *regexp.Regexp
+}
+
+type HTTPUrlRegexp []urlRegexp
+
+func (r *HTTPUrlRegexp) String() string {
+	return fmt.Sprint(*r)
+}
+
+func (r *HTTPUrlRegexp) Set(value string) error {
+	regexp, err := regexp.Compile(value)
+
+	*r = append(*r, urlRegexp{regexp: regexp})
+
+	return err
+}

http_modifier_settings_test.go

@@ -0,0 +1,65 @@
+package main
+
+import (
+	"testing"
+)
+
+func TestHTTPHeaderFilters(t *testing.T) {
+	filters := HTTPHeaderFilters{}
+
+	err := filters.Set("Header1:^$")
+	if err != nil {
+		t.Error("Should not error on Header1:^$")
+	}
+
+	err = filters.Set("Header2:^:$")
+	if err != nil {
+		t.Error("Should not error on Header2:^:$")
+	}
+
+	// Missing colon
+	err = filters.Set("Header3-^$")
+	if err == nil {
+		t.Error("Should error on Header2:^:$")
+	}
+}
+
+func TestHTTPHashFilters(t *testing.T) {
+	filters := HTTPHashFilters{}
+
+	err := filters.Set("Header1:1/2")
+	if err != nil {
+		t.Error("Should support old syntax")
+	}
+
+	if filters[0].percent != 50 {
+		t.Error("Wrong percentage", filters[0].percent)
+	}
+
+	err = filters.Set("Header2:1")
+	if err == nil {
+		t.Error("Should error on Header2 because no % symbol")
+	}
+
+	err = filters.Set("Header2:10%")
+	if err != nil {
+		t.Error("Should pass")
+	}
+
+	if filters[1].percent != 10 {
+		t.Error("Wrong percentage", filters[1].percent)
+	}
+}
+
+func TestUrlRewriteMap(t *testing.T) {
+	var err error
+	rewrites := UrlRewriteMap{}
+
+	if err = rewrites.Set("/v1/user/([^\\/]+)/ping:/v2/user/$1/ping"); err != nil {
+		t.Error("Should set mapping", err)
+	}
+
+	if err = rewrites.Set("/v1/user/([^\\/]+)/ping"); err == nil {
+		t.Error("Should not set mapping without :")
+	}
+}

http_modifier_test.go

@@ -0,0 +1,220 @@
+package main
+
+import (
+	"bytes"
+	"github.com/buger/gor/proto"
+	"testing"
+)
+
+func TestHTTPModifierWithoutConfig(t *testing.T) {
+	if NewHTTPModifier(&HTTPModifierConfig{}) != nil {
+		t.Error("If no config specified should not be initialized")
+	}
+}
+
+func TestHTTPModifierHeaderFilters(t *testing.T) {
+	filters := HTTPHeaderFilters{}
+	filters.Set("Host:^www.w3.org$")
+
+	modifier := NewHTTPModifier(&HTTPModifierConfig{
+		headerFilters: filters,
+	})
+
+	payload := []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
+
+	if len(modifier.Rewrite(payload)) == 0 {
+		t.Error("Request should pass filters")
+	}
+
+	filters = HTTPHeaderFilters{}
+	// Setting filter that not match our header
+	filters.Set("Host:^www.w4.org$")
+
+	modifier = NewHTTPModifier(&HTTPModifierConfig{
+		headerFilters: filters,
+	})
+
+	if len(modifier.Rewrite(payload)) != 0 {
+		t.Error("Request should not pass filters")
+	}
+}
+
+func TestHTTPModifierURLRewrite(t *testing.T) {
+	var url, new_url []byte
+
+	rewrites := UrlRewriteMap{}
+
+	payload := func(url []byte) []byte {
+		return []byte("POST " + string(url) + " HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
+	}
+
+	err := rewrites.Set("/v1/user/([^\\/]+)/ping:/v2/user/$1/ping")
+	if err != nil {
+		t.Error("Should not error on /v1/user/([^\\/]+)/ping:/v2/user/$1/ping")
+	}
+
+	modifier := NewHTTPModifier(&HTTPModifierConfig{
+		urlRewrite: rewrites,
+	})
+
+	url = []byte("/v1/user/joe/ping")
+	if new_url = proto.Path(modifier.Rewrite(payload(url))); bytes.Equal(new_url, url) {
+		t.Error("Request url should have been rewritten, wasn't", string(new_url))
+	}
+
+	url = []byte("/v1/user/ping")
+	if new_url = proto.Path(modifier.Rewrite(payload(url))); !bytes.Equal(new_url, url) {
+		t.Error("Request url should have been rewritten, wasn't", string(new_url))
+	}
+}
+
+func TestHTTPModifierHeaderHashFilters(t *testing.T) {
+	filters := HTTPHashFilters{}
+	filters.Set("Header2:1/2")
+
+	modifier := NewHTTPModifier(&HTTPModifierConfig{
+		headerHashFilters: filters,
+	})
+
+	payload := func(header []byte) []byte {
+		return []byte("POST / HTTP/1.1\r\n" + string(header) + "Content-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
+	}
+
+	if p := modifier.Rewrite(payload([]byte(""))); len(p) == 0 {
+		t.Error("Request should pass filters if Header does not exist")
+	}
+
+	if p := modifier.Rewrite(payload([]byte("Header2: 3\r\n"))); len(p) > 0 {
+		t.Error("Request should not pass filters, Header2 hash too high")
+	}
+
+	if p := modifier.Rewrite(payload([]byte("Header2: 1\r\n"))); len(p) == 0 {
+		t.Error("Request should pass filters")
+	}
+}
+
+func TestHTTPModifierParamHashFilters(t *testing.T) {
+	filters := HTTPHashFilters{}
+	filters.Set("user_id:1/2")
+
+	modifier := NewHTTPModifier(&HTTPModifierConfig{
+		paramHashFilters: filters,
+	})
+
+	payload := func(value []byte) []byte {
+		return []byte("POST /" + string(value) + " HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
+	}
+
+	if p := modifier.Rewrite(payload([]byte(""))); len(p) == 0 {
+		t.Error("Request should pass filters if param does not exist")
+	}
+
+	if p := modifier.Rewrite(payload([]byte("?user_id=3"))); len(p) > 0 {
+		t.Error("Request should not pass filters", string(p))
+	}
+
+	if p := modifier.Rewrite(payload([]byte("?user_id=1"))); len(p) == 0 {
+		t.Error("Request should pass filters")
+	}
+}
+
+func TestHTTPModifierHeaders(t *testing.T) {
+	headers := HTTPHeaders{}
+	headers.Set("Header1:1")
+	headers.Set("Host:localhost")
+
+	modifier := NewHTTPModifier(&HTTPModifierConfig{
+		headers: headers,
+	})
+
+	payload := []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
+	new_payload := []byte("POST /post HTTP/1.1\r\nHeader1: 1\r\nContent-Length: 7\r\nHost: localhost\r\n\r\na=1&b=2")
+
+	if payload = modifier.Rewrite(payload); !bytes.Equal(payload, new_payload) {
+		t.Error("Should update request headers", string(payload))
+	}
+}
+
+func TestHTTPModifierURLRegexp(t *testing.T) {
+	filters := HTTPUrlRegexp{}
+	filters.Set("/v1/app")
+	filters.Set("/v1/api")
+
+	modifier := NewHTTPModifier(&HTTPModifierConfig{
+		urlRegexp: filters,
+	})
+
+	payload := func(url string) []byte {
+		return []byte("POST " + url + " HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
+	}
+
+	if len(modifier.Rewrite(payload("/v1/app/test"))) == 0 {
+		t.Error("Should pass url")
+	}
+
+	if len(modifier.Rewrite(payload("/v1/api/test"))) == 0 {
+		t.Error("Should pass url")
+	}
+
+	if len(modifier.Rewrite(payload("/other"))) > 0 {
+		t.Error("Should not pass url")
+	}
+}
+
+func TestHTTPModifierURLNegativeRegexp(t *testing.T) {
+	filters := HTTPUrlRegexp{}
+	filters.Set("/restricted1")
+	filters.Set("/some/restricted2")
+
+	modifier := NewHTTPModifier(&HTTPModifierConfig{
+		urlNegativeRegexp: filters,
+	})
+
+	payload := func(url string) []byte {
+		return []byte("POST " + url + " HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
+	}
+
+	if len(modifier.Rewrite(payload("/v1/app/test"))) == 0 {
+		t.Error("Should pass url")
+	}
+
+	if len(modifier.Rewrite(payload("/restricted1"))) > 0 {
+		t.Error("Should not pass url")
+	}
+
+	if len(modifier.Rewrite(payload("/some/restricted2"))) > 0 {
+		t.Error("Should not pass url")
+	}
+}
+
+func TestHTTPModifierSetHeader(t *testing.T) {
+	filters := HTTPHeaders{}
+	filters.Set("User-Agent:Gor")
+
+	modifier := NewHTTPModifier(&HTTPModifierConfig{
+		headers: filters,
+	})
+
+	payload := []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
+	payload_after := []byte("POST /post HTTP/1.1\r\nUser-Agent: Gor\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
+
+	if payload = modifier.Rewrite(payload); !bytes.Equal(payload_after, payload) {
+		t.Error("Should add new header", string(payload))
+	}
+}
+
+func TestHTTPModifierSetParam(t *testing.T) {
+	filters := HTTPParams{}
+	filters.Set("api_key=1")
+
+	modifier := NewHTTPModifier(&HTTPModifierConfig{
+		params: filters,
+	})
+
+	payload := []byte("POST /post?api_key=1234 HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
+	payload_after := []byte("POST /post?api_key=1 HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
+
+	if payload = modifier.Rewrite(payload); !bytes.Equal(payload_after, payload) {
+		t.Error("Should override param", string(payload))
+	}
+}

input_raw_test.go

@@ -1,8 +1,13 @@
 package main
 
 import (
+	"bytes"
 	"io"
+	"io/ioutil"
+	"log"
 	"net/http"
+	"net/http/httputil"
+	"os/exec"
 	"strings"
 	"sync"
 	"testing"
@@ -37,3 +42,111 @@ func TestRAWInput(t *testing.T) {
 
 	close(quit)
 }
+
+func TestInputRAW100Expect(t *testing.T) {
+	wg := new(sync.WaitGroup)
+	quit := make(chan int)
+
+	file_content, _ := ioutil.ReadFile("README.md")
+
+	// Origing and Replay server initialization
+	origin := startHTTP(func(req *http.Request) {
+		defer req.Body.Close()
+		ioutil.ReadAll(req.Body)
+
+		wg.Done()
+	})
+
+	origin_address := strings.Replace(origin.Addr().String(), "[::]", "127.0.0.1", -1)
+
+	input := NewRAWInput(origin_address)
+
+	// We will use it to get content of raw HTTP request
+	test_output := NewTestOutput(func(data []byte) {
+		if strings.Contains(string(data), "Expect: 100-continue") {
+			t.Error("Should not contain 100-continue header")
+		}
+		wg.Done()
+	})
+
+	listener := startHTTP(func(req *http.Request) {
+		defer req.Body.Close()
+		body, _ := ioutil.ReadAll(req.Body)
+
+		if !bytes.Equal(body, file_content) {
+			buf, _ := httputil.DumpRequest(req, true)
+			t.Error("Wrong POST body:", string(buf))
+		}
+
+		wg.Done()
+	})
+	replay_address := listener.Addr().String()
+
+	http_output := NewHTTPOutput(replay_address, &HTTPOutputConfig{})
+
+	Plugins.Inputs = []io.Reader{input}
+	Plugins.Outputs = []io.Writer{test_output, http_output}
+
+	go Start(quit)
+
+	wg.Add(3)
+	curl := exec.Command("curl", "http://"+origin_address, "--data-binary", "@README.md")
+	err := curl.Run()
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	wg.Wait()
+	close(quit)
+}
+
+func TestInputRAWChunkedEncoding(t *testing.T) {
+	wg := new(sync.WaitGroup)
+	quit := make(chan int)
+
+	file_content, _ := ioutil.ReadFile("README.md")
+
+	// Origing and Replay server initialization
+	origin := startHTTP(func(req *http.Request) {
+		defer req.Body.Close()
+		ioutil.ReadAll(req.Body)
+
+		wg.Done()
+	})
+
+	origin_address := strings.Replace(origin.Addr().String(), "[::]", "127.0.0.1", -1)
+
+	input := NewRAWInput(origin_address)
+
+	listener := startHTTP(func(req *http.Request) {
+		defer req.Body.Close()
+		body, _ := ioutil.ReadAll(req.Body)
+
+		if !bytes.Equal(body, file_content) {
+			buf, _ := httputil.DumpRequest(req, true)
+			t.Error("Wrong POST body:", string(buf))
+		}
+
+		wg.Done()
+	})
+	replay_address := listener.Addr().String()
+
+	http_output := NewHTTPOutput(replay_address, &HTTPOutputConfig{Debug: true})
+
+	Plugins.Inputs = []io.Reader{input}
+	Plugins.Outputs = []io.Writer{http_output}
+
+	go Start(quit)
+
+	wg.Add(2)
+
+	curl := exec.Command("curl", "http://"+origin_address, "--header", "Transfer-Encoding: chunked", "--data-binary", "@README.md")
+	err := curl.Run()
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	wg.Wait()
+
+	close(quit)
+}

input_tcp.go

@@ -2,9 +2,11 @@ package main
 
 import (
 	"bufio"
-	"io"
+	"encoding/hex"
+	"fmt"
 	"log"
 	"net"
+	"os"
 )
 
 // Can be tested using nc tool:
@@ -59,23 +61,18 @@ func (i *TCPInput) handleConnection(conn net.Conn) {
 	defer conn.Close()
 
 	reader := bufio.NewReader(conn)
+	scanner := bufio.NewScanner(reader)
 
-	for {
-		buf, err := reader.ReadBytes('¶')
-		buf_len := len(buf)
-		if buf_len > 0 {
-			new_buf_len := len(buf) - 2
-			if new_buf_len > 0 {
-				new_buf := make([]byte, new_buf_len)
-				copy(new_buf, buf[:new_buf_len])
-				i.data <- new_buf
-				if err != nil {
-					if err != io.EOF {
-						log.Printf("error: %s\n", err)
-					}
-				}
-			}
-		}
+	for scanner.Scan() {
+		encodedPayload := scanner.Bytes()
+		// Hex encoding always 2x number of bytes
+		decoded := make([]byte, len(encodedPayload)/2)
+		hex.Decode(decoded, encodedPayload)
+		i.data <- decoded
+	}
+
+	if err := scanner.Err(); err != nil {
+		fmt.Fprintln(os.Stderr, "Unexpected error in input tcp connection:", err)
 	}
 }
 

input_tcp_test.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"encoding/hex"
 	"io"
 	"log"
 	"net"
@@ -38,10 +39,10 @@ func TestTCPInput(t *testing.T) {
 
 	for i := 0; i < 100; i++ {
 		wg.Add(1)
-		new_buf := make([]byte, len(msg)+2)
-		msg = append(msg, []byte("¶")...)
-		copy(new_buf, msg)
-		conn.Write(new_buf)
+
+		encoded := make([]byte, len(msg)*2+1)
+		hex.Encode(encoded, msg)
+		conn.Write(append(encoded, '\n'))
 	}
 
 	wg.Wait()

output_http.go

@@ -1,98 +1,68 @@
 package main
 
 import (
-	"bufio"
-	"bytes"
 	"io"
 	"log"
-	"net/http"
-	"net/url"
-	"strings"
 	"sync/atomic"
 	"time"
 )
 
-type RedirectNotAllowed struct{}
-
-func (e *RedirectNotAllowed) Error() string {
-	return "Redirects not allowed"
-}
-
-// customCheckRedirect disables redirects https://github.com/buger/gor/pull/15
-func customCheckRedirect(req *http.Request, via []*http.Request) error {
-	if len(via) >= 0 {
-		return new(RedirectNotAllowed)
-	}
-	return nil
-}
-
-// ParseRequest in []byte returns a http request or an error
-func ParseRequest(data []byte) (request *http.Request, err error) {
-	buf := bytes.NewBuffer(data)
-	reader := bufio.NewReader(buf)
-
-	request, err = http.ReadRequest(reader)
-
-	return
-}
-
 const InitialDynamicWorkers = 10
 
+type HTTPOutputConfig struct {
+	redirectLimit int
+
+	stats   bool
+	workers int
+
+	elasticSearch string
+
+	Debug bool
+}
+
 type HTTPOutput struct {
+	// Keep this as first element of struct because it guarantees 64bit
+	// alignment. atomic.* functions crash on 32bit machines if operand is not
+	// aligned at 64bit. See https://github.com/golang/go/issues/599
+	activeWorkers int64
+
 	address string
 	limit   int
 	queue   chan []byte
 
-	activeWorkers int64
-	needWorker    chan int
+	needWorker chan int
 
-	urlRegexp            HTTPUrlRegexp
-	headerFilters        HTTPHeaderFilters
-	headerHashFilters    HTTPHeaderHashFilters
-	outputHTTPUrlRewrite UrlRewriteMap
-
-	headers HTTPHeaders
-	methods HTTPMethods
-
-	elasticSearch *ESPlugin
+	config *HTTPOutputConfig
 
 	queueStats *GorStat
+
+	elasticSearch *ESPlugin
 }
 
-func NewHTTPOutput(address string, headers HTTPHeaders, methods HTTPMethods, urlRegexp HTTPUrlRegexp, headerFilters HTTPHeaderFilters, headerHashFilters HTTPHeaderHashFilters, elasticSearchAddr string, outputHTTPUrlRewrite UrlRewriteMap) io.Writer {
+func NewHTTPOutput(address string, config *HTTPOutputConfig) io.Writer {
 
 	o := new(HTTPOutput)
 
-	if !strings.HasPrefix(address, "http") {
-		address = "http://" + address
-	}
-
 	o.address = address
-	o.headers = headers
-	o.methods = methods
+	o.config = config
 
-	o.urlRegexp = urlRegexp
-	o.headerFilters = headerFilters
-	o.headerHashFilters = headerHashFilters
-	o.outputHTTPUrlRewrite = outputHTTPUrlRewrite
-
-	o.queue = make(chan []byte, 100)
-	if Settings.outputHTTPStats {
+	if o.config.stats {
 		o.queueStats = NewGorStat("output_http")
 	}
 
+	o.queue = make(chan []byte, 100)
 	o.needWorker = make(chan int, 1)
 
 	// Initial workers count
-	if Settings.outputHTTPWorkers == -1 {
+	if o.config.workers == 0 {
 		o.needWorker <- InitialDynamicWorkers
 	} else {
-		o.needWorker <- Settings.outputHTTPWorkers
+		o.needWorker <- o.config.workers
 	}
 
-	if elasticSearchAddr != "" {
+	if o.config.elasticSearch != "" {
 		o.elasticSearch = new(ESPlugin)
-		o.elasticSearch.Init(elasticSearchAddr)
+		o.elasticSearch.Init(o.config.elasticSearch)
 	}
 
 	go o.WorkerMaster()
@@ -108,16 +78,17 @@ func (o *HTTPOutput) WorkerMaster() {
 		}
 
 		// Disable dynamic scaling if workers poll fixed size
-		if Settings.outputHTTPWorkers != -1 {
+		if o.config.workers != 0 {
 			return
 		}
 	}
 }
 
 func (o *HTTPOutput) Worker() {
-	client := &http.Client{
-		CheckRedirect: customCheckRedirect,
-	}
+	client := NewHTTPClient(o.address, &HTTPClientConfig{
+		FollowRedirects: o.config.redirectLimit,
+		Debug:           o.config.Debug,
+	})
 
 	death_count := 0
 
@@ -130,7 +101,7 @@ func (o *HTTPOutput) Worker() {
 			death_count = 0
 		case <-time.After(time.Millisecond * 100):
 			// When dynamic scaling enabled workers die after 2s of inactivity
-			if Settings.outputHTTPWorkers == -1 {
+			if o.config.workers == 0 {
 				death_count += 1
 			} else {
 				continue
@@ -155,11 +126,11 @@ func (o *HTTPOutput) Write(data []byte) (n int, err error) {
 
 	o.queue <- buf
 
-	if Settings.outputHTTPStats {
+	if o.config.stats {
 		o.queueStats.Write(len(o.queue))
 	}
 
-	if Settings.outputHTTPWorkers == -1 {
+	if o.config.workers == 0 {
 		workersCount := atomic.LoadInt64(&o.activeWorkers)
 
 		if len(o.queue) > int(workersCount) {
@@ -170,49 +141,12 @@ func (o *HTTPOutput) Write(data []byte) (n int, err error) {
 	return len(data), nil
 }
 
-func (o *HTTPOutput) sendRequest(client *http.Client, data []byte) {
-	request, err := ParseRequest(data)
-
-	if err != nil {
-		log.Println("Cannot parse request", string(data), err)
-		return
-	}
-
-	if len(o.methods) > 0 && !o.methods.Contains(request.Method) {
-		return
-	}
-
-	if !(o.urlRegexp.Good(request) && o.headerFilters.Good(request) && o.headerHashFilters.Good(request)) {
-		return
-	}
-
-	// Rewrite the path as necessary
-	request.URL.Path = o.outputHTTPUrlRewrite.Rewrite(request.URL.Path)
-
-	// Change HOST of original request
-	URL := o.address + request.URL.Path + "?" + request.URL.RawQuery
-
-	request.RequestURI = ""
-	request.URL, _ = url.ParseRequestURI(URL)
-
-	for _, header := range o.headers {
-		SetHeader(request, header.Name, header.Value)
-	}
-
+func (o *HTTPOutput) sendRequest(client *HTTPClient, request []byte) {
 	start := time.Now()
-	resp, err := client.Do(request)
+	resp, err := client.Send(request)
 	stop := time.Now()
 
-	// We should not count Redirect as errors
-	if urlErr, ok := err.(*url.Error); ok {
-		if _, ok := urlErr.Err.(*RedirectNotAllowed); ok {
-			err = nil
-		}
-	}
-
-	if err == nil {
-		defer resp.Body.Close()
-	} else {
+	if err != nil {
 		log.Println("Request error:", err)
 	}
 
@@ -221,19 +155,6 @@ func (o *HTTPOutput) sendRequest(client *http.Client, data []byte) {
 	}
 }
 
-func SetHeader(request *http.Request, name string, value string) {
-	// Need to check here for the Host header as it needs to be set on the request and not as a separate header
-	// http.ReadRequest sets it by default to the URL Host of the request being read
-	if name == "Host" {
-		request.Host = value
-	} else {
-		request.Header.Set(name, value)
-	}
-
-	return
-
-}
-
 func (o *HTTPOutput) String() string {
 	return "HTTP output: " + o.address
 }

output_http_test.go

@@ -2,8 +2,11 @@ package main
 
 import (
 	"io"
+	"io/ioutil"
 	"net"
 	"net/http"
+	"net/http/httptest"
+	_ "net/http/httputil"
 	"sync"
 	"testing"
 	"time"
@@ -11,7 +14,7 @@ import (
 
 func startHTTP(cb func(*http.Request)) net.Listener {
 	handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		go cb(r)
+		cb(r)
 	})
 
 	listener, _ := net.Listen("tcp", ":0")
@@ -21,36 +24,12 @@ func startHTTP(cb func(*http.Request)) net.Listener {
 	return listener
 }
 
-func TestSetHeader(t *testing.T) {
-
-	req := &http.Request{
-		Header: make(map[string][]string),
-	}
-	req.Host = "test.com"
-
-	SetHeader(req, "Host", "test2.com")
-
-	if req.Host != "test2.com" {
-		t.Error("Expected test2.com - got ", req.Host)
-	}
-
-	SetHeader(req, "test_header", "test_value")
-
-	if req.Header.Get("test_header") != "test_value" {
-		t.Error("Wrong header value found")
-	}
-
-}
-
 func TestHTTPOutput(t *testing.T) {
 	wg := new(sync.WaitGroup)
 	quit := make(chan int)
 
 	input := NewTestInput()
 
-	headers := HTTPHeaders{HTTPHeader{"User-Agent", "Gor"}}
-	methods := HTTPMethods{"GET", "PUT", "POST"}
-
 	listener := startHTTP(func(req *http.Request) {
 		if req.Header.Get("User-Agent") != "Gor" {
 			t.Error("Wrong header")
@@ -60,10 +39,23 @@ func TestHTTPOutput(t *testing.T) {
 			t.Error("Wrong method")
 		}
 
+		if req.Method == "POST" {
+			defer req.Body.Close()
+			body, _ := ioutil.ReadAll(req.Body)
+
+			if string(body) != "a=1&b=2" {
+				t.Error("Wrong POST body:", string(body))
+			}
+		}
+
 		wg.Done()
 	})
 
-	output := NewHTTPOutput(listener.Addr().String(), headers, methods, HTTPUrlRegexp{}, HTTPHeaderFilters{}, HTTPHeaderHashFilters{}, "", UrlRewriteMap{})
+	headers := HTTPHeaders{HTTPHeader{"User-Agent", "Gor"}}
+	methods := HTTPMethods{[]byte("GET"), []byte("PUT"), []byte("POST")}
+	Settings.modifierConfig = HTTPModifierConfig{headers: headers, methods: methods}
+
+	output := NewHTTPOutput(listener.Addr().String(), &HTTPOutputConfig{})
 
 	Plugins.Inputs = []io.Reader{input}
 	Plugins.Outputs = []io.Writer{output}
@@ -71,7 +63,7 @@ func TestHTTPOutput(t *testing.T) {
 	go Start(quit)
 
 	for i := 0; i < 100; i++ {
-		wg.Add(2)
+		wg.Add(2) // OPTIONS should be ignored
 		input.EmitPOST()
 		input.EmitOPTIONS()
 		input.EmitGET()
@@ -80,23 +72,47 @@ func TestHTTPOutput(t *testing.T) {
 	wg.Wait()
 
 	close(quit)
+
+	Settings.modifierConfig = HTTPModifierConfig{}
+}
+
+func TestOutputHTTPSSL(t *testing.T) {
+	wg := new(sync.WaitGroup)
+	quit := make(chan int)
+
+	// Origing and Replay server initialization
+	server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		wg.Done()
+	}))
+
+	input := NewTestInput()
+	http_output := NewHTTPOutput(server.URL, &HTTPOutputConfig{})
+
+	Plugins.Inputs = []io.Reader{input}
+	Plugins.Outputs = []io.Writer{http_output}
+
+	go Start(quit)
+
+	wg.Add(2)
+
+	input.EmitPOST()
+	input.EmitGET()
+
+	wg.Wait()
+	close(quit)
 }
 
 func BenchmarkHTTPOutput(b *testing.B) {
 	wg := new(sync.WaitGroup)
 	quit := make(chan int)
 
-	input := NewTestInput()
-
-	headers := HTTPHeaders{HTTPHeader{"User-Agent", "Gor"}}
-	methods := HTTPMethods{"GET", "PUT", "POST"}
-
 	listener := startHTTP(func(req *http.Request) {
 		time.Sleep(50 * time.Millisecond)
 		wg.Done()
 	})
 
-	output := NewHTTPOutput(listener.Addr().String(), headers, methods, HTTPUrlRegexp{}, HTTPHeaderFilters{}, HTTPHeaderHashFilters{}, "", UrlRewriteMap{})
+	input := NewTestInput()
+	output := NewHTTPOutput(listener.Addr().String(), &HTTPOutputConfig{})
 
 	Plugins.Inputs = []io.Reader{input}
 	Plugins.Outputs = []io.Writer{output}

output_tcp.go

@@ -1,10 +1,12 @@
 package main
 
 import (
+	"encoding/hex"
 	"fmt"
 	"io"
 	"log"
 	"net"
+	"time"
 )
 
 type TCPOutput struct {
@@ -32,19 +34,29 @@ func NewTCPOutput(address string) io.Writer {
 }
 
 func (o *TCPOutput) worker() {
-	conn, _ := o.connect(o.address)
+	conn, err := o.connect(o.address)
+	for ; err != nil; conn, err = o.connect(o.address) {
+		time.Sleep(2 * time.Second)
+	}
+
 	defer conn.Close()
 
 	for {
-		conn.Write(<-o.buf)
+		_, err := conn.Write(<-o.buf)
+		if err != nil {
+			log.Println("Worker failed on write, exitings and starting new worker")
+			go o.worker()
+			break
+		}
 	}
 }
 
 func (o *TCPOutput) Write(data []byte) (n int, err error) {
-	new_buf := make([]byte, len(data)+2)
-	data = append(data, []byte("¶")...)
-	copy(new_buf, data)
-	o.buf <- new_buf
+	// Hex encoding always 2x number of bytes
+	encoded := make([]byte, len(data)*2+1)
+	hex.Encode(encoded, data)
+	o.buf <- append(encoded, '\n')
+
 	if Settings.outputTCPStats {
 		o.bufStats.Write(len(o.buf))
 	}

output_tcp_test.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"bufio"
+	"encoding/hex"
 	"io"
 	"log"
 	"net"
@@ -44,22 +45,19 @@ func startTCP(cb func([]byte)) net.Listener {
 	go func() {
 		for {
 			conn, _ := listener.Accept()
+			defer conn.Close()
 
 			go func() {
 				reader := bufio.NewReader(conn)
-				for {
-					buf, err := reader.ReadBytes('¶')
-					new_buf_len := len(buf) - 2
-					new_buf := make([]byte, new_buf_len)
-					copy(new_buf, buf[:new_buf_len])
-					if err != nil {
-						if err != io.EOF {
-							log.Printf("error: %s\n", err)
-						}
-					}
-					cb(new_buf)
+				scanner := bufio.NewScanner(reader)
+
+				for scanner.Scan() {
+					encodedPayload := scanner.Bytes()
+					// Hex encoding always 2x number of bytes
+					decoded := make([]byte, len(encodedPayload)/2)
+					hex.Decode(decoded, encodedPayload)
+					cb(decoded)
 				}
-				conn.Close()
 			}()
 		}
 	}()

plugins.go

@@ -96,6 +96,6 @@ func InitPlugins() {
 	}
 
 	for _, options := range Settings.outputHTTP {
-		registerPlugin(NewHTTPOutput, options, Settings.outputHTTPHeaders, Settings.outputHTTPMethods, Settings.outputHTTPUrlRegexp, Settings.outputHTTPHeaderFilters, Settings.outputHTTPHeaderHashFilters, Settings.outputHTTPElasticSearch, Settings.outputHTTPUrlRewrite)
+		registerPlugin(NewHTTPOutput, options, &Settings.outputHTTPConfig)
 	}
 }

proto/proto.go

@@ -0,0 +1,166 @@
+// Low-level interaction with HTTP request payload
+package proto
+
+import (
+    "bytes"
+    "github.com/buger/gor/byteutils"
+    _ "log"
+)
+
+var CLRF = []byte("\r\n")
+var EMPTY_LINE = []byte("\r\n\r\n")
+var HEADER_DELIM = []byte(": ")
+
+// Headers should end with empty line
+func MIMEHeadersEndPos(payload []byte) int {
+    return bytes.Index(payload, EMPTY_LINE)
+}
+
+func MIMEHeadersStartPos(payload []byte) int {
+    return bytes.Index(payload, CLRF) + 2 // Find first line end
+}
+
+// Find header value or return error
+// Do not support multi-line headers
+func Header(payload []byte, name []byte) (value []byte, headerStart, valueStart, headerEnd int) {
+    headerStart = bytes.Index(payload, name)
+
+    if headerStart == -1 {
+        return
+    }
+
+    valueStart = headerStart + len(name) + 1 // Skip ":" after header name
+    if payload[valueStart] == ' ' { // Ignore empty space after ':'
+        valueStart += 1
+    }
+    headerEnd = valueStart + bytes.IndexByte(payload[valueStart:], '\r')
+    value = payload[valueStart:headerEnd]
+
+    return
+}
+
+func GetHeader(payload []byte, name string) []byte {
+    val, _, _, _ := Header(payload, []byte(name))
+
+    return val
+}
+
+func SetHeader(payload, name, value []byte) []byte {
+    _, hs, vs, he := Header(payload, name)
+
+    // If header found
+    if hs != -1 {
+        return byteutils.Replace(payload, vs, he, value)
+    } else {
+        return AddHeader(payload, name, value)
+    }
+}
+
+func AddHeader(payload, name, value []byte) []byte {
+    header := make([]byte, len(name) + 2 + len(value) + 2)
+    copy(header[0:], name)
+    copy(header[len(name):], HEADER_DELIM)
+    copy(header[len(name)+2:], value)
+    copy(header[len(header)-2:], CLRF)
+
+    mimeStart := MIMEHeadersStartPos(payload)
+
+    return byteutils.Insert(payload, mimeStart, header)
+}
+
+func Path(payload []byte) []byte {
+    start := bytes.IndexByte(payload, ' ')
+    start += 1
+
+    end := bytes.IndexByte(payload[start:], ' ')
+
+    return payload[start:start+end]
+}
+
+func SetPath(payload, path []byte) []byte {
+    start := bytes.IndexByte(payload, ' ')
+    start += 1
+
+    end := bytes.IndexByte(payload[start:], ' ')
+
+    return byteutils.Replace(payload, start, start+end, path)
+}
+
+func PathParam(payload, name []byte) (value []byte, valueStart, valueEnd int) {
+    path := Path(payload)
+
+    if paramStart := bytes.Index(path, append(name, '=')); paramStart != -1 {
+        valueStart := paramStart + len(name) + 1
+        paramEnd := bytes.IndexByte(path[valueStart:], '&')
+        if paramEnd == -1 { // It is final param
+            paramEnd = len(path)
+        } else {
+            paramEnd += valueStart
+        }
+
+        return path[valueStart:paramEnd], valueStart, paramEnd
+    } else {
+        return []byte(""), -1, -1
+    }
+}
+
+func SetPathParam(payload, name, value []byte) []byte {
+    path := Path(payload)
+    _, vs, ve := PathParam(payload, name)
+
+    if vs != -1 {
+        newPath := make([]byte, len(path))
+        copy(newPath, path)
+        newPath = byteutils.Replace(newPath, vs, ve, value)
+
+        return SetPath(payload, newPath)
+    } else { // if param not found append to end of url
+        // Adding 2 because of '?' or '&' at start, and '=' in middle
+        newParam := make([]byte, len(name) + len(value) + 2)
+
+        if bytes.IndexByte(path, '?') == -1 {
+            newParam[0] = '?'
+        } else {
+            newParam[0] = '&'
+        }
+
+        copy(newParam[1:], name)
+        newParam[1+len(name)] = '='
+        copy(newParam[2+len(name):], value)
+
+        newPath := make([]byte, len(path) + len(newParam))
+        copy(newPath, path)
+        copy(newPath[len(path):], newParam)
+
+        return SetPath(payload, newPath)
+    }
+}
+
+func SetHost(payload, url, host []byte) []byte {
+    // If this is HTTP 1.0 traffic or proxy traffic it may include host right into path variable, so instead of setting Host header we rewrite Path
+    // Fix for https://github.com/buger/gor/issues/156
+    if path := Path(payload); bytes.HasPrefix(path, []byte("http")) {
+        hostStart := bytes.IndexByte(path, ':') // : position "https?:"
+        hostStart += 3 // Skip 1 ':' and 2 '\'
+        hostEnd := hostStart + bytes.IndexByte(path[hostStart:], '/')
+
+        newPath := make([]byte, len(path))
+        copy(newPath, path)
+        newPath = byteutils.Replace(newPath, 0, hostEnd, url)
+
+        return SetPath(payload, newPath)
+    } else {
+        return SetHeader(payload, []byte("Host"), host)
+    }
+}
+
+func Method(payload []byte) []byte {
+    end := bytes.IndexByte(payload, ' ')
+
+    return payload[:end]
+}
+
+// Status in response have same position as Path in request
+func Status(payload []byte) []byte {
+    return Path(payload)
+}

proto/proto_test.go

@@ -0,0 +1,152 @@
+package proto
+
+import (
+    "testing"
+    "bytes"
+)
+
+func TestHeader(t *testing.T) {
+    var payload, val []byte
+    var headerStart int
+
+    payload = []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
+
+    if val, _, _, _ = Header(payload, []byte("Content-Length")); !bytes.Equal(val, []byte("7")) {
+        t.Error("Should find header value")
+    }
+
+    payload = []byte("POST /post HTTP/1.1\r\nContent-Length:7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
+
+    if val, _, _, _ = Header(payload, []byte("Content-Length")); !bytes.Equal(val, []byte("7")) {
+        t.Error("Should find header value without space after :")
+    }
+
+    if _, headerStart, _, _ = Header(payload, []byte("Not-Found")); headerStart != -1  {
+        t.Error("Should not found header")
+    }
+}
+
+func TestMIMEHeadersEndPos(t *testing.T) {
+    head := []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org")
+    payload := []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
+
+    end := MIMEHeadersEndPos(payload)
+
+    if !bytes.Equal(payload[:end], head) {
+        t.Error("Wrong headers end position:", end)
+    }
+}
+
+func TestMIMEHeadersStartPos(t *testing.T) {
+    headers := []byte("Content-Length: 7\r\nHost: www.w3.org")
+    payload := []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
+
+    start := MIMEHeadersStartPos(payload)
+    end := MIMEHeadersEndPos(payload)
+
+    if !bytes.Equal(payload[start:end], headers) {
+        t.Error("Wrong headers end position:", start, end)
+    }
+}
+
+func TestSetHeader(t *testing.T) {
+    var payload, payload_after []byte
+
+    payload = []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
+    payload_after = []byte("POST /post HTTP/1.1\r\nContent-Length: 14\r\nHost: www.w3.org\r\n\r\na=1&b=2")
+
+    if payload = SetHeader(payload, []byte("Content-Length"), []byte("14")); !bytes.Equal(payload, payload_after) {
+        t.Error("Should update header if it exists", string(payload))
+    }
+
+
+    payload = []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
+    payload_after = []byte("POST /post HTTP/1.1\r\nUser-Agent: Gor\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
+
+    if payload = SetHeader(payload, []byte("User-Agent"), []byte("Gor")); !bytes.Equal(payload, payload_after) {
+        t.Error("Should add header if not found", string(payload))
+    }
+}
+
+func TestPath(t *testing.T) {
+    var path, payload []byte
+
+    payload = []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
+
+    if path = Path(payload); !bytes.Equal(path, []byte("/post")) {
+        t.Error("Should find path", string(path))
+    }
+}
+
+func TestSetPath(t *testing.T) {
+    var payload, payload_after []byte
+
+    payload = []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
+    payload_after = []byte("POST /new_path HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
+
+    if payload = SetPath(payload, []byte("/new_path")); !bytes.Equal(payload, payload_after) {
+        t.Error("Should replace path", string(payload))
+    }
+}
+
+func TestPathParam(t *testing.T) {
+    var payload []byte
+
+    payload = []byte("POST /post?param=test&user_id=1 HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
+
+    if val, _, _ := PathParam(payload, []byte("param")); !bytes.Equal(val, []byte("test")) {
+        t.Error("Should detect attribute", string(val))
+    }
+
+    if val, _, _ := PathParam(payload, []byte("user_id")); !bytes.Equal(val, []byte("1")) {
+        t.Error("Should detect attribute", string(val))
+    }
+}
+
+
+func TestSetPathParam(t *testing.T) {
+    var payload, payload_after []byte
+
+    payload = []byte("POST /post?param=test&user_id=1 HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
+    payload_after = []byte("POST /post?param=new&user_id=1 HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
+
+    if payload = SetPathParam(payload, []byte("param"), []byte("new")); !bytes.Equal(payload, payload_after) {
+        t.Error("Should replace existing value", string(payload))
+    }
+
+    payload = []byte("POST /post?param=test&user_id=1 HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
+    payload_after = []byte("POST /post?param=test&user_id=2 HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
+
+    if payload = SetPathParam(payload, []byte("user_id"), []byte("2")); !bytes.Equal(payload, payload_after) {
+        t.Error("Should replace existing value", string(payload))
+    }
+
+
+    payload = []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
+    payload_after = []byte("POST /post?param=test HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
+
+    if payload = SetPathParam(payload, []byte("param"), []byte("test")); !bytes.Equal(payload, payload_after) {
+        t.Error("Should set param if url have no params", string(payload))
+    }
+
+
+    payload = []byte("POST /post?param=test HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
+    payload_after = []byte("POST /post?param=test&user_id=1 HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
+
+    if payload = SetPathParam(payload, []byte("user_id"), []byte("1")); !bytes.Equal(payload, payload_after) {
+        t.Error("Should set param at the end if url params", string(payload))
+    }
+}
+
+
+
+func TestSetHostHTTP10(t *testing.T) {
+    var payload, payload_after []byte
+
+    payload = []byte("POST http://example.com/post HTTP/1.0\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
+    payload_after = []byte("POST http://new.com/post HTTP/1.0\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
+
+    if payload = SetHost(payload, []byte("http://new.com"), []byte("new.com")); !bytes.Equal(payload, payload_after) {
+        t.Error("Should replace host", string(payload))
+    }
+}

raw_socket_listener/listener.go

@@ -5,6 +5,7 @@ import (
     "log"
     "net"
     "strconv"
+    "bytes"
 )
 
 // Capture traffic from socket using RAW_SOCKET's
@@ -17,6 +18,11 @@ import (
 type Listener struct {
     messages map[string]*TCPMessage // buffer of TCPMessages waiting to be send
 
+    // Expect: 100-continue request is send in 2 tcp messages
+    // We store ACK aliases to merge this packets together
+    ack_aliases map[uint32]uint32
+    seq_with_data map[uint32]uint32
+
     c_packets  chan *TCPPacket
     c_messages chan *TCPMessage // Messages ready to be send to client
 
@@ -30,10 +36,13 @@ type Listener struct {
 func NewListener(addr string, port string) (rawListener *Listener) {
     rawListener = &Listener{}
 
-    rawListener.c_packets = make(chan *TCPPacket, 100)
-    rawListener.c_messages = make(chan *TCPMessage, 100)
-    rawListener.c_del_message = make(chan *TCPMessage, 100)
+    rawListener.c_packets = make(chan *TCPPacket, 10000)
+    rawListener.c_messages = make(chan *TCPMessage, 10000)
+    rawListener.c_del_message = make(chan *TCPMessage, 10000)
+
     rawListener.messages = make(map[string]*TCPMessage)
+    rawListener.ack_aliases = make(map[uint32]uint32)
+    rawListener.seq_with_data = make(map[uint32]uint32)
 
     rawListener.addr = addr
     rawListener.port, _ = strconv.Atoi(port)
@@ -50,6 +59,7 @@ func (t *Listener) listen() {
         // If message ready for deletion it means that its also complete or expired by timeout
         case message := <-t.c_del_message:
             t.c_messages <- message
+            delete(t.ack_aliases, message.packets[0].Ack)
             delete(t.messages, message.ID)
 
         // We need to use channels to process each packet to avoid data races
@@ -61,13 +71,14 @@ func (t *Listener) listen() {
 
 func (t *Listener) readRAWSocket() {
     conn, e := net.ListenPacket("ip4:tcp", t.addr)
-    defer conn.Close()
 
     if e != nil {
         log.Fatal(e)
     }
 
-    buf := make([]byte, 4096*2)
+    defer conn.Close()
+
+    buf := make([]byte, 4096*10)
 
     for {
         // Note: ReadFrom receive messages without IP header
@@ -114,6 +125,9 @@ func (t *Listener) isIncomingDataPacket(buf []byte) bool {
     return false
 }
 
+var bExpect100ContinueCheck = []byte("Expect: 100-continue")
+var bPOST = []byte("POST")
+
 // Trying to add packet to existing message or creating new message
 //
 // For TCP message unique id is Acknowledgment number (see tcp_packet.go)
@@ -121,8 +135,19 @@ func (t *Listener) processTCPPacket(packet *TCPPacket) {
     defer func() { recover() }()
 
     var message *TCPMessage
-    m_id := packet.Addr.String() + strconv.Itoa(int(packet.Ack))
 
+    parent_message_ack, parent_ok := t.seq_with_data[packet.Seq]
+    if parent_ok {
+        t.ack_aliases[packet.Ack] = parent_message_ack
+        delete(t.seq_with_data, packet.Seq)
+    }
+
+    ack_alias, alias_ok := t.ack_aliases[packet.Ack]
+    if alias_ok {
+        packet.Ack = ack_alias
+    }
+
+    m_id := packet.Addr.String() + strconv.Itoa(int(packet.Ack))
     message, ok := t.messages[m_id]
 
     if !ok {
@@ -131,6 +156,15 @@ func (t *Listener) processTCPPacket(packet *TCPPacket) {
         t.messages[m_id] = message
     }
 
+    if bytes.Equal(packet.Data[0:4], bPOST) {
+        if bytes.Equal(packet.Data[len(packet.Data)-24:len(packet.Data)-4], bExpect100ContinueCheck) {
+            t.seq_with_data[packet.Seq + uint32(len(packet.Data))] = packet.Ack
+
+            // Removing `Expect: 100-continue` header
+            packet.Data = append(packet.Data[:len(packet.Data)-24], packet.Data[len(packet.Data)-2:]...)
+        }
+    }
+
     // Adding packet to message
     message.c_packets <- packet
 }

raw_socket_listener/tcp_message.go

@@ -79,7 +79,7 @@ func (t *TCPMessage) Bytes() (output []byte) {
         output = append(output, v.Data...)
     }
 
-    return
+    return output
 }
 
 // AddPacket to the message and ensure packet uniqueness

raw_socket_listener/tcp_packet.go

@@ -89,6 +89,7 @@ func (t *TCPPacket) String() string {
         "Window size:" + strconv.Itoa(int(t.Window)),
         "Checksum:" + strconv.Itoa(int(t.Checksum)),
 
+        "Data size:" + strconv.Itoa(len(t.Data)),
         "Data:" + string(t.Data),
     }, "\n")
 }

settings.go

@@ -8,9 +8,21 @@ import (
 )
 
 const (
-	VERSION = "0.9.2"
+	VERSION = "0.9.6"
 )
 
+// Allows to specify multiple flags with same name and collects all values to array
+type MultiOption []string
+
+func (h *MultiOption) String() string {
+	return fmt.Sprint(*h)
+}
+
+func (h *MultiOption) Set(value string) error {
+	*h = append(*h, value)
+	return nil
+}
+
 type AppSettings struct {
 	verbose bool
 	stats   bool
@@ -29,17 +41,11 @@ type AppSettings struct {
 
 	inputRAW MultiOption
 
-	inputHTTP                   MultiOption
-	outputHTTP                  MultiOption
-	outputHTTPHeaders           HTTPHeaders
-	outputHTTPMethods           HTTPMethods
-	outputHTTPUrlRegexp         HTTPUrlRegexp
-	outputHTTPUrlRewrite        UrlRewriteMap
-	outputHTTPHeaderFilters     HTTPHeaderFilters
-	outputHTTPHeaderHashFilters HTTPHeaderHashFilters
-	outputHTTPElasticSearch     string
-	outputHTTPWorkers           int
-	outputHTTPStats             bool
+	inputHTTP  MultiOption
+	outputHTTP MultiOption
+
+	outputHTTPConfig HTTPOutputConfig
+	modifierConfig   HTTPModifierConfig
 }
 
 var Settings AppSettings = AppSettings{}
@@ -73,20 +79,41 @@ func init() {
 	flag.Var(&Settings.inputHTTP, "input-http", "Read requests from HTTP, should be explicitly sent from your application:\n\t# Listen for http on 9000\n\tgor --input-http :9000 --output-http staging.com")
 
 	flag.Var(&Settings.outputHTTP, "output-http", "Forwards incoming requests to given http address.\n\t# Redirect all incoming requests to staging.com address \n\tgor --input-raw :80 --output-http http://staging.com")
-	flag.Var(&Settings.outputHTTPHeaders, "output-http-header", "Inject additional headers to http reqest:\n\tgor --input-raw :8080 --output-http staging.com --output-http-header 'User-Agent: Gor'")
-	flag.Var(&Settings.outputHTTPMethods, "output-http-method", "Whitelist of HTTP methods to replay. Anything else will be dropped:\n\tgor --input-raw :8080 --output-http staging.com --output-http-method GET --output-http-method OPTIONS")
-	flag.Var(&Settings.outputHTTPUrlRegexp, "output-http-url-regexp", "A regexp to match requests against. Anything else will be dropped:\n\t gor --input-raw :8080 --output-http staging.com --output-http-url-regexp ^www.")
-	flag.Var(&Settings.outputHTTPHeaderFilters, "output-http-header-filter", "A regexp to match a specific header against. Requests with non-matching headers will be dropped:\n\t gor --input-raw :8080 --output-http staging.com --output-http-header-filter api-version:^v1")
-	flag.Var(&Settings.outputHTTPHeaderHashFilters, "output-http-header-hash-filter", "Takes a fraction of requests, consistently taking or rejecting a request based on the FNV32-1A hash of a specific header. The fraction must have a denominator that is a power of two:\n\t gor --input-raw :8080 --output-http staging.com --output-http-header-hash-filter user-id:1/4")
-	flag.IntVar(&Settings.outputHTTPWorkers, "output-http-workers", -1, "Gor uses dynamic worker scaling by default.  Enter a number to run a set number of workers.")
-	flag.BoolVar(&Settings.outputHTTPStats, "output-http-stats", false, "Report http output queue stats to console every 5 seconds.")
+	flag.IntVar(&Settings.outputHTTPConfig.workers, "output-http-workers", 0, "Gor uses dynamic worker scaling by default.  Enter a number to run a set number of workers.")
+	flag.IntVar(&Settings.outputHTTPConfig.redirectLimit, "output-http-redirects", 0, "Enable how often redirects should be followed.")
 
-	flag.StringVar(&Settings.outputHTTPElasticSearch, "output-http-elasticsearch", "", "Send request and response stats to ElasticSearch:\n\tgor --input-raw :8080 --output-http staging.com --output-http-elasticsearch 'es_host:api_port/index_name'")
-	flag.Var(&Settings.outputHTTPUrlRewrite, "output-http-rewrite-url", "Rewrite the requst url based on a mapping:\n\tgor --input-raw :8080 --output-http staging.com --output-http-rewrite-url /xml_test/interface.php:/api/service.do")
+	flag.BoolVar(&Settings.outputHTTPConfig.stats, "output-http-stats", false, "Report http output queue stats to console every 5 seconds.")
+
+	flag.StringVar(&Settings.outputHTTPConfig.elasticSearch, "output-http-elasticsearch", "", "Send request and response stats to ElasticSearch:\n\tgor --input-raw :8080 --output-http staging.com --output-http-elasticsearch 'es_host:api_port/index_name'")
+
+	flag.Var(&Settings.modifierConfig.headers, "http-set-header", "Inject additional headers to http reqest:\n\tgor --input-raw :8080 --output-http staging.com --http-set-header 'User-Agent: Gor'")
+	flag.Var(&Settings.modifierConfig.headers, "output-http-header", "WARNING: `--output-http-header` DEPRECATED, use `--http-set-header` instead")
+
+	flag.Var(&Settings.modifierConfig.params, "http-set-param", "Set request url param, if param already exists it will be overwritten:\n\tgor --input-raw :8080 --output-http staging.com --http-set-param api_key=1")
+
+	flag.Var(&Settings.modifierConfig.methods, "http-allow-method", "Whitelist of HTTP methods to replay. Anything else will be dropped:\n\tgor --input-raw :8080 --output-http staging.com --http-allow-method GET --http-allow-method OPTIONS")
+	flag.Var(&Settings.modifierConfig.methods, "output-http-method", "WARNING: `--output-http-method` DEPRECATED, use `--http-allow-method` instead")
+
+	flag.Var(&Settings.modifierConfig.urlRegexp, "http-allow-url", "A regexp to match requests against. Filter get matched agains full url with domain. Anything else will be dropped:\n\t gor --input-raw :8080 --output-http staging.com --http-allow-url ^www.")
+	flag.Var(&Settings.modifierConfig.urlRegexp, "output-http-url-regexp", "WARNING: `--output-http-url-regexp` DEPRECATED, use `--http-allow-url` instead")
+
+	flag.Var(&Settings.modifierConfig.urlNegativeRegexp, "http-diallow-url", "A regexp to match requests against. Filter get matched agains full url with domain. Anything else will be dropped:\n\t gor --input-raw :8080 --output-http staging.com --http-disallow-url ^www.")
+
+	flag.Var(&Settings.modifierConfig.urlRewrite, "http-rewrite-url", "Rewrite the request url based on a mapping:\n\tgor --input-raw :8080 --output-http staging.com --http-rewrite-url /v1/user/([^\\/]+)/ping:/v2/user/$1/ping")
+	flag.Var(&Settings.modifierConfig.urlRewrite, "output-http-rewrite-url", "WARNING: `--output-http-rewrite-url` DEPRECATED, use `--http-rewrite-url` instead")
+
+	flag.Var(&Settings.modifierConfig.headerFilters, "http-allow-header", "A regexp to match a specific header against. Requests with non-matching headers will be dropped:\n\t gor --input-raw :8080 --output-http staging.com --http-allow-header api-version:^v1")
+	flag.Var(&Settings.modifierConfig.headerFilters, "output-http-header-filter", "WARNING: `--output-http-header-filter` DEPRECATED, use `--http-allow-header` instead")
+
+	flag.Var(&Settings.modifierConfig.headerHashFilters, "http-header-limiter", "Takes a fraction of requests, consistently taking or rejecting a request based on the FNV32-1A hash of a specific header:\n\t gor --input-raw :8080 --output-http staging.com --http-header-imiter user-id:25%")
+	flag.Var(&Settings.modifierConfig.headerHashFilters, "output-http-header-hash-filter", "WARNING: `output-http-header-hash-filter` DEPRECATED, use `--http-header-hash-limiter` instead")
+
+	flag.Var(&Settings.modifierConfig.paramHashFilters, "http-param-limiter", "Takes a fraction of requests, consistently taking or rejecting a request based on the FNV32-1A hash of a specific GET param:\n\t gor --input-raw :8080 --output-http staging.com --http-param-limiter user_id:25%")
 }
 
 func Debug(args ...interface{}) {
 	if Settings.verbose {
+		log.Print("[DEBUG] ")
 		log.Println(args...)
 	}
 }

settings_header_filters.go

@@ -1,44 +0,0 @@
-package main
-
-import (
-	"errors"
-	"fmt"
-	"net/http"
-	"regexp"
-	"strings"
-)
-
-type headerFilter struct {
-	name   string
-	regexp *regexp.Regexp
-}
-
-type HTTPHeaderFilters []headerFilter
-
-func (h *HTTPHeaderFilters) String() string {
-	return fmt.Sprint(*h)
-}
-
-func (h *HTTPHeaderFilters) Set(value string) error {
-	valArr := strings.SplitN(value, ":", 2)
-	if len(valArr) < 2 {
-		return errors.New("need both header and value, colon-delimited (ex. user_id:^169$).")
-	}
-	r, err := regexp.Compile(valArr[1])
-	if err != nil {
-		return err
-	}
-
-	*h = append(*h, headerFilter{name: valArr[0], regexp: r})
-
-	return nil
-}
-
-func (h *HTTPHeaderFilters) Good(req *http.Request) bool {
-	for _, f := range *h {
-		if !f.regexp.MatchString(req.Header.Get(f.name)) {
-			return false
-		}
-	}
-	return true
-}

settings_header_filters_test.go

@@ -1,35 +0,0 @@
-package main
-
-import (
-	"net/http"
-	"testing"
-)
-
-func TestHTTPHeaderFilters(t *testing.T) {
-	filters := HTTPHeaderFilters{}
-
-	err := filters.Set("Header1:^$")
-	if err != nil {
-		t.Error("Should not error on Header1:^$")
-	}
-
-	err = filters.Set("Header2:^:$")
-	if err != nil {
-		t.Error("Should not error on Header2:^:$")
-	}
-
-	err = filters.Set("Header3-^$")
-	if err == nil {
-		t.Error("Should error on Header2:^:$")
-	}
-
-	req := http.Request{}
-	req.Header = make(map[string][]string)
-	req.Header.Add("Header1", "")
-	req.Header.Add("Header2", ":")
-	req.Header.Add("Header3", "Irrelevant")
-
-	if !filters.Good(&req) {
-		t.Error("Request should pass filters")
-	}
-}

settings_header_hash_filters.go

@@ -1,68 +0,0 @@
-package main
-
-import (
-	"errors"
-	"fmt"
-	"hash/fnv"
-	"net/http"
-	"strconv"
-	"strings"
-)
-
-type headerHashFilter struct {
-	name    string
-	maxHash uint32
-}
-
-type HTTPHeaderHashFilters []headerHashFilter
-
-func (h *HTTPHeaderHashFilters) String() string {
-	return fmt.Sprint(*h)
-}
-
-func (h *HTTPHeaderHashFilters) Set(value string) error {
-	valArr := strings.SplitN(value, ":", 2)
-	if len(valArr) < 2 {
-		return errors.New("need both header and value, colon-delimited (ex. user_id:1/2).")
-	}
-
-	fracArr := strings.Split(valArr[1], "/")
-	if len(fracArr) < 2 {
-		return errors.New("need both a numerator and denominator specified, slash-delimited (ex. user_id:1/4).")
-	}
-
-	var num, den uint64
-	num, _ = strconv.ParseUint(fracArr[0], 10, 64)
-	den, _ = strconv.ParseUint(fracArr[1], 10, 64)
-
-	if num < 1 || den < 1 || num > den {
-		panic("need positive numerators and denominators, with the former less than the latter.")
-	}
-
-	for test := den; test != 1; test /= 2 {
-		if test%2 == 1 {
-			return errors.New("must have a denominator which is a power of two.")
-		}
-	}
-
-	var f headerHashFilter
-	f.name = valArr[0]
-	f.maxHash = (uint32)(num * (((uint64)(2 << 31)) / den))
-	*h = append(*h, f)
-
-	return nil
-}
-
-func (h *HTTPHeaderHashFilters) Good(req *http.Request) bool {
-	for _, f := range *h {
-		if req.Header.Get(f.name) == "" {
-			return false
-		}
-		hasher := fnv.New32a()
-		hasher.Write([]byte(req.Header.Get(f.name)))
-		if hasher.Sum32() > f.maxHash {
-			return false
-		}
-	}
-	return true
-}

settings_header_hash_filters_test.go

@@ -1,43 +0,0 @@
-package main
-
-import (
-	"net/http"
-	"testing"
-)
-
-func TestHTTPHeaderHashFilters(t *testing.T) {
-	filters := HTTPHeaderHashFilters{}
-
-	err := filters.Set("Header1:1/2")
-	if err != nil {
-		t.Error("Should not error on Header1:^$")
-	}
-
-	err = filters.Set("Header2:1/2")
-	if err != nil {
-		t.Error("Should not error on Header2:^:$")
-	}
-
-	err = filters.Set("HeaderIrrelevant:1/3")
-	if err == nil {
-		t.Error("Should error on HeaderIrrelevant:1/3")
-	}
-
-	req := http.Request{}
-	req.Header = make(map[string][]string)
-	req.Header.Add("Header1", "test3414")
-
-	if filters.Good(&req) {
-		t.Error("Request should not pass filters, Header2 does not exist")
-	}
-
-	req.Header.Add("Header2", "test2")
-	if filters.Good(&req) {
-		t.Error("Request should not pass filters, Header2 hash too high")
-	}
-
-	req.Header.Set("Header2", "test3414")
-	if !filters.Good(&req) {
-		t.Error("Request should pass filters")
-	}
-}

settings_headers.go

@@ -1,32 +0,0 @@
-package main
-
-import (
-	"errors"
-	"fmt"
-	"strings"
-)
-
-type HTTPHeaders []HTTPHeader
-type HTTPHeader struct {
-	Name  string
-	Value string
-}
-
-func (h *HTTPHeaders) String() string {
-	return fmt.Sprint(*h)
-}
-
-func (h *HTTPHeaders) Set(value string) error {
-	v := strings.SplitN(value, ":", 2)
-	if len(v) != 2 {
-		return errors.New("Expected `Key: Value`")
-	}
-
-	header := HTTPHeader{
-		strings.TrimSpace(v[0]),
-		strings.TrimSpace(v[1]),
-	}
-
-	*h = append(*h, header)
-	return nil
-}

settings_methods.go

@@ -1,26 +0,0 @@
-package main
-
-import (
-	"fmt"
-	"strings"
-)
-
-type HTTPMethods []string
-
-func (h *HTTPMethods) String() string {
-	return fmt.Sprint(*h)
-}
-
-func (h *HTTPMethods) Set(value string) error {
-	*h = append(*h, strings.ToUpper(value))
-	return nil
-}
-
-func (h *HTTPMethods) Contains(value string) bool {
-	for _, method := range *h {
-		if value == method {
-			return true
-		}
-	}
-	return false
-}

settings_methods_test.go

@@ -1,24 +0,0 @@
-package main
-
-import (
-	"testing"
-)
-
-func TestHTTPMethods(t *testing.T) {
-	methods := HTTPMethods{}
-
-	methods.Set("lower")
-	methods.Set("UPPER")
-
-	if !methods.Contains("LOWER") {
-		t.Error("Does not contain LOWER")
-	}
-
-	if !methods.Contains("UPPER") {
-		t.Error("Does not contain UPPER")
-	}
-
-	if methods.Contains("ABSENT") {
-		t.Error("Does contain ABSENT")
-	}
-}

settings_option.go

@@ -1,16 +0,0 @@
-package main
-
-import (
-	"fmt"
-)
-
-type MultiOption []string
-
-func (h *MultiOption) String() string {
-	return fmt.Sprint(*h)
-}
-
-func (h *MultiOption) Set(value string) error {
-	*h = append(*h, value)
-	return nil
-}

settings_url_map.go

@@ -1,36 +0,0 @@
-package main
-
-import (
-	"errors"
-	"fmt"
-	"strings"
-)
-
-type urlRewrite struct {
-	src    string
-	target string
-}
-
-type UrlRewriteMap []urlRewrite
-
-func (r *UrlRewriteMap) String() string {
-	return fmt.Sprint(*r)
-}
-
-func (r *UrlRewriteMap) Set(value string) error {
-	valArr := strings.SplitN(value, ":", 2)
-	if len(valArr) < 2 {
-		return errors.New("need both src and target, colon-delimited (ex. /a:/b).")
-	}
-	*r = append(*r, urlRewrite{src: valArr[0], target: valArr[1]})
-	return nil
-}
-
-func (r *UrlRewriteMap) Rewrite(path string) string {
-	for _, f := range *r {
-		if f.src == path {
-			return f.target
-		}
-	}
-	return path
-}

settings_url_map_test.go

@@ -1,27 +0,0 @@
-package main
-
-import (
-	"testing"
-)
-
-func TestUrlRewriteMap(t *testing.T) {
-	var url string
-
-	rewrites := UrlRewriteMap{}
-
-	err := rewrites.Set("/abc:/123")
-	if err != nil {
-		t.Error("Should not error on /abc:/123")
-	}
-
-	url = "/abc"
-
-	if rewrites.Rewrite(url) == url {
-		t.Error("Request url should have been rewritten, wasn't")
-	}
-
-	url = "/wibble"
-	if rewrites.Rewrite(url) != url {
-		t.Error("Request url should not have been rewritten, was")
-	}
-}

settings_url_regexp.go

@@ -1,30 +0,0 @@
-package main
-
-import (
-	"net/http"
-	"regexp"
-)
-
-type HTTPUrlRegexp struct {
-	regexp *regexp.Regexp
-}
-
-func (r *HTTPUrlRegexp) String() string {
-	if r.regexp == nil {
-		return ""
-	}
-	return r.regexp.String()
-}
-
-func (r *HTTPUrlRegexp) Set(value string) error {
-	regexp, err := regexp.Compile(value)
-	r.regexp = regexp
-	return err
-}
-
-func (r *HTTPUrlRegexp) Good(req *http.Request) bool {
-	if r.regexp == nil {
-		return true
-	}
-	return r.regexp.MatchString(req.Host + req.URL.String())
-}

settings_url_regexp_test.go

@@ -1,26 +0,0 @@
-package main
-
-import (
-	"net/http"
-	"net/url"
-	"testing"
-)
-
-func TestHTTPUrlRegexp(t *testing.T) {
-	filter := HTTPUrlRegexp{}
-
-	filter.Set("^www.google.com/admin/")
-
-	req := http.Request{}
-	req.Host = "www.google.com"
-	var err error
-	req.URL, err = url.Parse("/admin/testpage1")
-	if !filter.Good(&req) || err != nil {
-		t.Error("Request should pass filters")
-	}
-
-	req.URL, err = url.Parse("/user/testpage2")
-	if filter.Good(&req) || err != nil {
-		t.Error("Request should not pass filters")
-	}
-}

test_input.go

@@ -28,7 +28,11 @@ func (i *TestInput) EmitGET() {
 }
 
 func (i *TestInput) EmitPOST() {
-	i.data <- []byte("POST /pub/WWW/ HTTP/1.1\nHost: www.w3.org\r\n\r\na=1&b=2\r\n\r\n")
+	i.data <- []byte("POST /pub/WWW/ HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
+}
+
+func (i *TestInput) EmitChunkedPOST() {
+	i.data <- []byte("POST /pub/WWW/ HTTP/1.1\r\nHost: www.w3.org\r\nTransfer-Encoding: chunked\r\n\r\n4\r\nWiki\r\n5\r\npedia\r\ne\r\n in\r\n\r\nchunks.\r\n0\r\n\r\n")
 }
 
 func (i *TestInput) EmitFile() {