From 6e125a7b7b1294e418bef6ac6032fefd83a17625 Mon Sep 17 00:00:00 2001 From: Joseph Lawson Date: Wed, 13 Aug 2014 09:44:30 -0400 Subject: [PATCH 1/4] allow dynamic http workers with a signal channel for full queue --- output_http.go | 52 +++++++++++++++++++++++++++++++++++++++----------- settings.go | 2 +- 2 files changed, 42 insertions(+), 12 deletions(-) diff --git a/output_http.go b/output_http.go index fc7aa75..fe0f40a 100644 --- a/output_http.go +++ b/output_http.go @@ -9,6 +9,7 @@ import ( "net/url" "strconv" "strings" + "time" ) type RedirectNotAllowed struct{} @@ -38,13 +39,13 @@ func ParseRequest(data []byte) (request *http.Request, err error) { type HTTPOutput struct { address string limit int + buf chan []byte + need_worker chan int urlRegexp HTTPUrlRegexp headerFilters HTTPHeaderFilters headerHashFilters HTTPHeaderHashFilters - buf chan []byte - headers HTTPHeaders methods HTTPMethods @@ -71,14 +72,13 @@ func NewHTTPOutput(options string, headers HTTPHeaders, methods HTTPMethods, url o.buf = make(chan []byte, 100) o.bufStats = NewGorStat("output_http") + o.need_worker = make(chan int) if len(optionsArr) > 1 { o.limit, _ = strconv.Atoi(optionsArr[1]) } - for i := 0; i < 10; i++ { - go o.worker(i) - } + go o.worker_master(10) if o.limit > 0 { return NewLimiter(o, o.limit) @@ -87,24 +87,54 @@ func NewHTTPOutput(options string, headers HTTPHeaders, methods HTTPMethods, url } } -func (o *HTTPOutput) worker(n int) { - client := &http.Client{ - CheckRedirect: customCheckRedirect, +func (o *HTTPOutput) worker_master(n int) { + for i := 0; i < n; i++ { + go o.worker() } for { - data := <-o.buf - o.sendRequest(client, data) + new_workers := <- o.need_worker + for i := 0; i < new_workers; i++ { + go o.worker() + } } } +func (o *HTTPOutput) worker() { + client := &http.Client{ + CheckRedirect: customCheckRedirect, + } + death_count := 0 + Loop: + for { + select { + case data := <-o.buf: + o.sendRequest(client, data) + death_count = 0 + default: + death_count += 1 + if death_count > 20 { + break Loop + } else { + time.Sleep(time.Millisecond * 100) + } + + } + } +} + func (o *HTTPOutput) Write(data []byte) (n int, err error) { buf := make([]byte, len(data)) copy(buf, data) o.buf <- buf + buf_len := len(o.buf) o.bufStats.Write(len(o.buf)) - + if buf_len > 20 { + if len(o.need_worker) == 0 { + o.need_worker <- buf_len + } + } return len(data), nil } diff --git a/settings.go b/settings.go index d9b971e..209857e 100644 --- a/settings.go +++ b/settings.go @@ -8,7 +8,7 @@ import ( ) const ( - VERSION = "0.8" + VERSION = "0.8.4.1" ) type AppSettings struct { From 58944fd42dd04f1584562ef3cc72911a983b3e77 Mon Sep 17 00:00:00 2001 From: Joseph Lawson Date: Wed, 24 Sep 2014 14:57:49 -0400 Subject: [PATCH 2/4] calling this version 0.9.0 --- README.md | 2 ++ settings.go | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 293eddd..62cb457 100644 --- a/README.md +++ b/README.md @@ -182,6 +182,8 @@ https://github.com/buger/gor/releases gor --input-raw :80 --output-tcp replay.local:28020 -split-output=false: By default each output gets same traffic. If set to `true` it splits traffic equally among all outputs. + + -stats=false: If set to `true` it gives out queuing stats for the HTTP output and TCP listener every 5 seconds in the form latest,mean,max,count,count/second. ``` ## Building from source diff --git a/settings.go b/settings.go index 209857e..40b3738 100644 --- a/settings.go +++ b/settings.go @@ -8,7 +8,7 @@ import ( ) const ( - VERSION = "0.8.4.1" + VERSION = "0.9.0" ) type AppSettings struct { From 988c6abdaded162f329469982d1aaf95a629b417 Mon Sep 17 00:00:00 2001 From: Joseph Lawson Date: Sun, 5 Oct 2014 15:53:50 -0400 Subject: [PATCH 3/4] Dynamic workers toggle. Separate stats toggles. README and FAQs * Created separate toggles for output-http-stats and output-tcp-stats * Created HTTPWorkers option (defaults to 10) to allow larger number of HTTPWorkers to be spawned. * Refactored output_http to allow dynamic http worker scaling when httpworkers equals -1 * Updated README.me to include updates and some common FAQs. --- README.md | 76 ++++++++++++++++++++++++++++++++++++++++++++++++-- output_http.go | 45 +++++++++++++++++++----------- output_tcp.go | 8 ++++-- settings.go | 6 ++++ 4 files changed, 115 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index 62cb457..208f3ec 100644 --- a/README.md +++ b/README.md @@ -177,13 +177,17 @@ https://github.com/buger/gor/releases -output-http-url-regexp=: A regexp to match requests against. Anything else will be dropped: gor --input-raw :8080 --output-http staging.com --output-http-url-regexp ^www. + -output-http-workers=10: Number of http output workers desired. Enter -1 for dynamic worker scaling. Gor will add http workers if its work queue starts getting too full and kill them . + + -output-http-stats=false: If set to `true` it gives out queuing stats for the HTTP output every 5 seconds in the form latest,mean,max,count,count/second. + -output-tcp=[]: Used for internal communication between Gor instances. Example: # Listen for requests on 80 port and forward them to other Gor instance on 28020 port gor --input-raw :80 --output-tcp replay.local:28020 + -output-tcp-stats=false: If set to `true` it gives out queuing stats for the TCP output every 5 seconds in the form latest,mean,max,count,count/second. + -split-output=false: By default each output gets same traffic. If set to `true` it splits traffic equally among all outputs. - - -stats=false: If set to `true` it gives out queuing stats for the HTTP output and TCP listener every 5 seconds in the form latest,mean,max,count,count/second. ``` ## Building from source @@ -215,6 +219,74 @@ Typical linux shell has a small open files soft limit at 1024. You can easily ra More about ulimit: http://blog.thecodingmachine.com/content/solving-too-many-open-files-exception-red5-or-any-other-application +### What do the stats commands do? +Gor can report stats on the output-tcp and output-http request queues. Stats are reported to the console every 5 seconds in the form `latest,mean,max,count,count/second` by using the `-output-http-stats` and `-output-tcp-stats` options. + +Examples: + +``` +2014/04/23 21:17:50 output_tcp:latest,mean,max,count,count/second +2014/04/23 21:17:50 output_tcp:0,0,0,0,0 +2014/04/23 21:17:55 output_tcp:1,1,2,68,13 +2014/04/23 21:18:00 output_tcp:1,1,2,92,18 +2014/04/23 21:18:05 output_tcp:1,1,2,119,23 +2014/04/23 21:18:10 output_tcp:1,0,1,95,19 +2014/04/23 21:18:15 output_tcp:1,1,2,92,18 +2014/04/23 21:18:20 output_tcp:1,1,2,108,21 +2014/04/23 21:18:25 output_tcp:1,1,2,117,23 +2014/04/23 21:18:30 output_tcp:1,1,2,113,22 +2014/04/23 21:18:35 output_tcp:21,20,21,132,26 +2014/04/23 21:18:40 output_tcp:100,99,100,99,19 +``` + +``` +Version: 0.8 +2014/04/23 21:19:46 output_http:latest,mean,max,count,count/second +2014/04/23 21:19:46 output_http:0,0,0,0,0 +2014/04/23 21:19:51 output_http:0,0,0,0,0 +2014/04/23 21:19:56 output_http:0,0,0,0,0 +2014/04/23 21:20:01 output_http:1,0,1,50,10 +2014/04/23 21:20:06 output_http:1,1,4,72,14 +2014/04/23 21:20:11 output_http:1,0,1,179,35 +2014/04/23 21:20:16 output_http:1,0,1,148,29 +2014/04/23 21:20:21 output_http:1,1,2,91,18 +2014/04/23 21:20:26 output_http:1,1,2,150,30 +2014/04/23 21:18:15 output_http:100,99,100,70,14 +2014/04/23 21:18:21 output_http:100,99,100,55,11 +2014/04/23 21:18:28 output_http:100,99,100,55,11 +2014/04/23 21:18:34 output_http:100,99,100,57,11 +2014/04/23 21:18:41 output_http:100,99,100,61,12 +2014/04/23 21:18:48 output_http:100,99,100,56,11 +2014/04/23 21:18:56 output_http:100,99,100,58,11 +2014/04/23 21:19:01 output_http:100,99,100,31,6 +2014/04/23 21:19:08 output_http:100,99,100,61,12 +2014/04/23 21:19:15 output_http:100,99,100,64,12 +2014/04/23 21:19:21 output_http:100,99,100,70,14 +2014/04/23 21:19:28 output_http:100,99,100,61,12 +2014/04/23 21:19:35 output_http:100,99,100,56,11 +``` + +### How can I tell if I have bottlenecks? +Key areas that sometimes experience bottlenecks are the output-tcp and output-http functions which have internal queues for requests. Each queue has an upper limit of 100. Enable stats reporting to see if any queues are experiencing bottleneck behavior. + +#### output-http bottlenecks +When running a Gor replay the output-http feature may bottleneck if: + + * the replay has inadequate bandwidth. If the replay is receiving or sending more messages than its network adapter can handle the output-http-stats may report that the output-http queue is filling up. See if there is a way to upgrade the replay's bandwidth. + * the `-output-http` target is unable to respond to messages in a timely manner. By default the replay has 10 http output workers which take messages off the output-http queue, process the request, and ensure that the request did not result in an error. The optimal number of output-http-workers can be determined with the formula `output-workers = (Average number of requests per second)/(Average target response time per second)`. Gor also has the setting `--output-http-workers=-1` which enables dynamic worker scaling. + +#### output-tcp bottlenecks +When using the Gor listener the output-tcp feature may bottleneck if: + + * the replay is unable to accept and process more requests than the listener is able generate. Prior to troubleshooting the output-tcp bottleneck, ensure that the replay target is not experiencing any bottlenecks. + * the replay target has inadequate bandwidth to handle all its incoming requests. If a replay target's incoming bandwidth is maxed out the output-tcp-stats may report that the output-tcp queue is filling up. See if there is a way to upgrade the replay's bandwidth. + +### The CPU average across my load-balanced targets is higher than the source +If you are replaying traffic from multiple listeners to a load-balanced target and you use sticky sessions, you may observe that the target servers have a higher CPU load than the listener servers. This may be because the sticky session cookie of the original load balancer is not honored by the target load balancer thus resulting in requests that would normally hit the same target server hitting different servers on the backend thus reducing some caching benefits gained via the load balancing. Try running just one listener against one replay target and see if the CPU utilization comparison is more accurate. + +### How does dynamic http worker scaling work? +By using the Gor setting `--output-http-workers=-1` Gor will create more http output workers when the http output queue length is greater than 10. The number of workers created (N) is equal to the queue length at the time which it is checked and found to have a length greater than 10. The queue length is checked every time a message is written to the http output queue. No more workers will be spawned until that request to spawn N workers is satisfied. If a dynamic worker cannot process a message at that time, it will sleep for 100 milliseconds. If a dynamic worker cannot process a message for 2 seconds it dies. + ## Tuning To achieve the top most performance you should tune the source server system limits: diff --git a/output_http.go b/output_http.go index fe0f40a..57d2d82 100644 --- a/output_http.go +++ b/output_http.go @@ -40,7 +40,7 @@ type HTTPOutput struct { address string limit int buf chan []byte - need_worker chan int + needWorker chan int urlRegexp HTTPUrlRegexp headerFilters HTTPHeaderFilters @@ -71,14 +71,18 @@ func NewHTTPOutput(options string, headers HTTPHeaders, methods HTTPMethods, url o.headerHashFilters = headerHashFilters o.buf = make(chan []byte, 100) - o.bufStats = NewGorStat("output_http") - o.need_worker = make(chan int) + if Settings.outputHTTPStats { + o.bufStats = NewGorStat("output_http") + } + if Settings.outputHTTPWorkers == -1 { + o.needWorker = make(chan int) + } if len(optionsArr) > 1 { o.limit, _ = strconv.Atoi(optionsArr[1]) } - go o.worker_master(10) + go o.WorkerMaster(Settings.outputHTTPWorkers) if o.limit > 0 { return NewLimiter(o, o.limit) @@ -87,20 +91,22 @@ func NewHTTPOutput(options string, headers HTTPHeaders, methods HTTPMethods, url } } -func (o *HTTPOutput) worker_master(n int) { +func (o *HTTPOutput) WorkerMaster(n int) { for i := 0; i < n; i++ { - go o.worker() + go o.Worker() } - for { - new_workers := <- o.need_worker - for i := 0; i < new_workers; i++ { - go o.worker() + if Settings.outputHTTPWorkers == -1 { + for { + new_workers := <-o.needWorker + for i := 0; i < new_workers; i++ { + go o.Worker() + } } } } -func (o *HTTPOutput) worker() { +func (o *HTTPOutput) Worker() { client := &http.Client{ CheckRedirect: customCheckRedirect, } @@ -112,7 +118,9 @@ func (o *HTTPOutput) worker() { o.sendRequest(client, data) death_count = 0 default: - death_count += 1 + if Settings.outputHTTPWorkers == -1 { + death_count += 1 + } if death_count > 20 { break Loop } else { @@ -129,10 +137,15 @@ func (o *HTTPOutput) Write(data []byte) (n int, err error) { o.buf <- buf buf_len := len(o.buf) - o.bufStats.Write(len(o.buf)) - if buf_len > 20 { - if len(o.need_worker) == 0 { - o.need_worker <- buf_len + if Settings.outputHTTPStats { + o.bufStats.Write(len(o.buf)) + } + + if Settings.outputHTTPWorkers == -1 { + if buf_len > 10 { + if len(o.needWorker) == 0 { + o.needWorker <- buf_len + } } } return len(data), nil diff --git a/output_tcp.go b/output_tcp.go index 48e2a87..d67466c 100644 --- a/output_tcp.go +++ b/output_tcp.go @@ -23,7 +23,9 @@ func NewTCPOutput(options string) io.Writer { o.address = optionsArr[0] o.buf = make(chan []byte, 100) - o.bufStats = NewGorStat("output_tcp") + if Settings.outputTCPStats { + o.bufStats = NewGorStat("output_tcp") + } if len(optionsArr) > 1 { o.limit, _ = strconv.Atoi(optionsArr[1]) @@ -54,7 +56,9 @@ func (o *TCPOutput) Write(data []byte) (n int, err error) { data = append(data,[]byte("ΒΆ")...) copy(new_buf, data) o.buf <- new_buf - o.bufStats.Write(len(o.buf)) + if Settings.outputTCPStats { + o.bufStats.Write(len(o.buf)) + } return len(data), nil } diff --git a/settings.go b/settings.go index 40b3738..a837f68 100644 --- a/settings.go +++ b/settings.go @@ -22,6 +22,7 @@ type AppSettings struct { inputTCP MultiOption outputTCP MultiOption + outputTCPStats bool inputFile MultiOption outputFile MultiOption @@ -35,6 +36,8 @@ type AppSettings struct { outputHTTPHeaderFilters HTTPHeaderFilters outputHTTPHeaderHashFilters HTTPHeaderHashFilters outputHTTPElasticSearch string + outputHTTPWorkers int + outputHTTPStats bool } var Settings AppSettings = AppSettings{} @@ -58,6 +61,7 @@ func init() { flag.Var(&Settings.inputTCP, "input-tcp", "Used for internal communication between Gor instances. Example: \n\t# Receive requests from other Gor instances on 28020 port, and redirect output to staging\n\tgor --input-tcp :28020 --output-http staging.com") flag.Var(&Settings.outputTCP, "output-tcp", "Used for internal communication between Gor instances. Example: \n\t# Listen for requests on 80 port and forward them to other Gor instance on 28020 port\n\tgor --input-raw :80 --output-tcp replay.local:28020") + flag.BoolVar(&Settings.outputTCPStats, "output-tcp-stats", false, "Report TCP output queue stats to console every 5 seconds.") flag.Var(&Settings.inputFile, "input-file", "Read requests from file: \n\tgor --input-file ./requests.gor --output-http staging.com") flag.Var(&Settings.outputFile, "output-file", "Write incoming requests to file: \n\tgor --input-raw :80 --output-file ./requests.gor") @@ -70,6 +74,8 @@ func init() { flag.Var(&Settings.outputHTTPUrlRegexp, "output-http-url-regexp", "A regexp to match requests against. Anything else will be dropped:\n\t gor --input-raw :8080 --output-http staging.com --output-http-url-regexp ^www.") flag.Var(&Settings.outputHTTPHeaderFilters, "output-http-header-filter", "A regexp to match a specific header against. Requests with non-matching headers will be dropped:\n\t gor --input-raw :8080 --output-http staging.com --output-http-header-filter api-version:^v1") flag.Var(&Settings.outputHTTPHeaderHashFilters, "output-http-header-hash-filter", "Takes a fraction of requests, consistently taking or rejecting a request based on the FNV32-1A hash of a specific header. The fraction must have a denominator that is a power of two:\n\t gor --input-raw :8080 --output-http staging.com --output-http-header-hash-filter user-id:1/4") + flag.IntVar(&Settings.outputHTTPWorkers, "output-http-workers", 10, "Gor runs 10 workers by default. Enter -1 to enable dynamic worker scaling.") + flag.BoolVar(&Settings.outputHTTPStats, "output-http-stats", false, "Report http output queue stats to console every 5 seconds.") flag.StringVar(&Settings.outputHTTPElasticSearch, "output-http-elasticsearch", "", "Send request and response stats to ElasticSearch:\n\tgor --input-raw :8080 --output-http staging.com --output-http-elasticsearch 'es_host:api_port/index_name'") } From 4f9c983bc75973e5e96641b9ad334b6e81f896f4 Mon Sep 17 00:00:00 2001 From: Joseph Lawson Date: Sun, 5 Oct 2014 20:03:50 -0400 Subject: [PATCH 4/4] Dynamic scaling by default --- README.md | 4 ++-- settings.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 208f3ec..0e8b9e6 100644 --- a/README.md +++ b/README.md @@ -177,7 +177,7 @@ https://github.com/buger/gor/releases -output-http-url-regexp=: A regexp to match requests against. Anything else will be dropped: gor --input-raw :8080 --output-http staging.com --output-http-url-regexp ^www. - -output-http-workers=10: Number of http output workers desired. Enter -1 for dynamic worker scaling. Gor will add http workers if its work queue starts getting too full and kill them . + -output-http-workers=-1: Number of http output workers desired. Use default -1 for dynamic worker scaling. Gor will add http workers if its work queue starts getting too full and kill them . -output-http-stats=false: If set to `true` it gives out queuing stats for the HTTP output every 5 seconds in the form latest,mean,max,count,count/second. @@ -273,7 +273,7 @@ Key areas that sometimes experience bottlenecks are the output-tcp and output-ht When running a Gor replay the output-http feature may bottleneck if: * the replay has inadequate bandwidth. If the replay is receiving or sending more messages than its network adapter can handle the output-http-stats may report that the output-http queue is filling up. See if there is a way to upgrade the replay's bandwidth. - * the `-output-http` target is unable to respond to messages in a timely manner. By default the replay has 10 http output workers which take messages off the output-http queue, process the request, and ensure that the request did not result in an error. The optimal number of output-http-workers can be determined with the formula `output-workers = (Average number of requests per second)/(Average target response time per second)`. Gor also has the setting `--output-http-workers=-1` which enables dynamic worker scaling. + * with `--output-http-workers` set to anything other than `-1` the `-output-http` target is unable to respond to messages in a timely manner. The http output workers which take messages off the output-http queue, process the request, and ensure that the request did not result in an error may not be able to keep up with the number of incoming requests. If the replay is not using dynamic worker scaling (`--output-http-workers=-1`) The optimal number of output-http-workers can be determined with the formula `output-workers = (Average number of requests per second)/(Average target response time per second)`. #### output-tcp bottlenecks When using the Gor listener the output-tcp feature may bottleneck if: diff --git a/settings.go b/settings.go index a837f68..3186a06 100644 --- a/settings.go +++ b/settings.go @@ -74,7 +74,7 @@ func init() { flag.Var(&Settings.outputHTTPUrlRegexp, "output-http-url-regexp", "A regexp to match requests against. Anything else will be dropped:\n\t gor --input-raw :8080 --output-http staging.com --output-http-url-regexp ^www.") flag.Var(&Settings.outputHTTPHeaderFilters, "output-http-header-filter", "A regexp to match a specific header against. Requests with non-matching headers will be dropped:\n\t gor --input-raw :8080 --output-http staging.com --output-http-header-filter api-version:^v1") flag.Var(&Settings.outputHTTPHeaderHashFilters, "output-http-header-hash-filter", "Takes a fraction of requests, consistently taking or rejecting a request based on the FNV32-1A hash of a specific header. The fraction must have a denominator that is a power of two:\n\t gor --input-raw :8080 --output-http staging.com --output-http-header-hash-filter user-id:1/4") - flag.IntVar(&Settings.outputHTTPWorkers, "output-http-workers", 10, "Gor runs 10 workers by default. Enter -1 to enable dynamic worker scaling.") + flag.IntVar(&Settings.outputHTTPWorkers, "output-http-workers", -1, "Gor uses dynamic worker scaling by default. Enter a number to run a set number of workers.") flag.BoolVar(&Settings.outputHTTPStats, "output-http-stats", false, "Report http output queue stats to console every 5 seconds.") flag.StringVar(&Settings.outputHTTPElasticSearch, "output-http-elasticsearch", "", "Send request and response stats to ElasticSearch:\n\tgor --input-raw :8080 --output-http staging.com --output-http-elasticsearch 'es_host:api_port/index_name'")