github/goreplay
4
0
Fork 0
mirror of https://github.com/buger/goreplay.git synced 2026-10-19 16:15:42 +00:00
Code Issues Releases Wiki Activity

Allow listening on ALL interfaces

Browse Source
This commit is contained in:
Leonid Bugaev
2016-05-11 16:52:52 +05:00
parent fb9cf68ff9
commit 47bb312b73
3 changed files with 178 additions and 102 deletions
Download Patch File Download Diff File Expand all files Collapse all files
README.md
+33 -33
View File
@@ -26,7 +26,7 @@ Subscribe to the [quarterly newsletter](https://tinyletter.com/gor) to stay info
### Capture traffic from port
```bash
# Run on servers where you want to catch traffic. You can run it on each `web` machine.
sudo gor --input-raw 127.0.0.1:80 --output-tcp replay.local:28020
sudo gor --input-raw :80 --output-tcp replay.local:28020
# Replay server (replay.local).
gor --input-tcp replay.local:28020 --output-http http://staging.com
@@ -38,7 +38,7 @@ Since Gor use raw sockets to capture traffic it require `sudo` access. Alternati
It's recommended to use separate server for replaying traffic, but if you have enough CPU resources you can use single Gor instance.
```
sudo gor --input-raw 127.0.0.1:80 --output-http "http://staging.com"
sudo gor --input-raw :80 --output-http "http://staging.com"
```
## Configuration
@@ -98,7 +98,7 @@ gor --input-tcp :28020 --output-http "http://staging.com|10"
```
# replay server will not get more than 10% of requests
# useful for high-load environments
gor --input-raw 127.0.0.1:80 --output-tcp "replay.local:28020|10%"
gor --input-raw :80 --output-tcp "replay.local:28020|10%"
```
#### Limiting based on Header or URL param value
@@ -106,10 +106,10 @@ If you have unique user id (like API key) stored in header or URL you can consis
Basic formula looks like this: `FNV32-1A_hashing(value) % 100 >= chance`. Examples:
```
# Limit based on header value
gor --input-raw 127.0.0.1:80 --output-tcp "replay.local:28020|10%" --http-header-limiter "X-API-KEY: 10%"
gor --input-raw :80 --output-tcp "replay.local:28020|10%" --http-header-limiter "X-API-KEY: 10%"
# Limit based on header value
gor --input-raw 127.0.0.1:80 --output-tcp "replay.local:28020|10%" --http-param-limiter "api_key: 10%"
gor --input-raw :80 --output-tcp "replay.local:28020|10%" --http-param-limiter "api_key: 10%"
```
Only percentage based limiting supported.
@@ -118,29 +118,29 @@ Only percentage based limiting supported.
#### Allow url regexp
```
# only forward requests being sent to the /api endpoint
gor --input-raw 127.0.0.1:8080 --output-http staging.com --http-allow-url /api
gor --input-raw :8080 --output-http staging.com --http-allow-url /api
```
#### Disallow url regexp
```
# only forward requests NOT being sent to the /api... endpoint
gor --input-raw 127.0.0.1:8080 --output-http staging.com --http-disallow-url /api
gor --input-raw :8080 --output-http staging.com --http-disallow-url /api
```
#### Filter based on regexp of header
```
# only forward requests with an api version of 1.0x
gor --input-raw 127.0.0.1:8080 --output-http staging.com --http-allow-header api-version:^1\.0\d
gor --input-raw :8080 --output-http staging.com --http-allow-header api-version:^1\.0\d
# only forward requests NOT containing User-Agent header value "Replayed by Gor"
gor --input-raw 127.0.0.1:8080 --output-http staging.com --http-disallow-header "User-Agent: Replayed by Gor"
gor --input-raw :8080 --output-http staging.com --http-disallow-header "User-Agent: Replayed by Gor"
```
#### Filter based on http method
Requests not matching a specified whitelist can be filtered out. For example to strip non-nullipotent requests:
```
gor --input-raw 127.0.0.1:80 --output-http "http://staging.server" \
gor --input-raw :80 --output-http "http://staging.server" \
--http-allow-method GET \
--http-allow-method OPTIONS
```
@@ -151,20 +151,20 @@ Gor supports some basic request rewriting support. For complex logic you can use
#### Rewrite URL based on a mapping
```
# rewrite url to match the following
gor --input-raw 127.0.0.1:8080 --output-http staging.com --http-rewrite-url /v1/user/([^\\/]+)/ping:/v2/user/$1/ping
gor --input-raw :8080 --output-http staging.com --http-rewrite-url /v1/user/([^\\/]+)/ping:/v2/user/$1/ping
```
#### Set URL param
Set request url param, if param already exists it will be overwritten
```
gor --input-raw 127.0.0.1:8080 --output-http staging.com --http-set-param api_key=1
gor --input-raw :8080 --output-http staging.com --http-set-param api_key=1
```
#### Set Header
Set request header, if header already exists it will be overwritten. This may be useful if you need to identify requests generated by Gor or enable feature flagged functionality in an application:
```
gor --input-raw 127.0.0.1:80 --output-http "http://staging.server" \
gor --input-raw :80 --output-http "http://staging.server" \
--http-header "User-Agent: Replayed by Gor" \
--http-header "Enable-Feature-X: true"
```
@@ -204,7 +204,7 @@ end
Middleware can be enabled using `--middleware` option, by specifying path to executable file:
```
gor --input-raw 127.0.0.1:80 --middleware "/opt/middleware_executable" --output-http "http://staging.server"
gor --input-raw :80 --middleware "/opt/middleware_executable" --output-http "http://staging.server"
```
#### Communication protocol
@@ -246,7 +246,7 @@ Imagine that you have auth system that randomly generate access tokens, which us
You can save requests to file, and replay them later:
```
# write to file
gor --input-raw 127.0.0.1:80 --output-file requests.gor
gor --input-raw :80 --output-file requests.gor
# read from file
gor --input-file requests.gor --output-http "http://staging.com"
@@ -268,7 +268,7 @@ gor --input-file "requests.gor|200%" --output-http "staging.com"
If your development or staging environment is protected by Basic Authentication then those credentials can be injected in during the replay:
```
gor --input-raw 127.0.0.1:80 --output-http "http://user:pass@staging .com"
gor --input-raw :80 --output-http "http://user:pass@staging .com"
```
Note: This will overwrite any Authorization headers in the original request.
@@ -277,7 +277,7 @@ Note: This will overwrite any Authorization headers in the original request.
By default Gor use `libpcap` for intercepting traffic. If you have any troubles with it, you may try alternative engine: `raw_socket`.
```
sudo gor --input-raw 127.0.0.1:80 --input-raw-engine "libpcap" --output-http "http://staging.com"
sudo gor --input-raw :80 --input-raw-engine "libpcap" --output-http "http://staging.com"
```
@@ -345,53 +345,53 @@ https://github.com/buger/gor/releases
`gor -h` output:
```
-http-allow-header=[]: A regexp to match a specific header against. Requests with non-matching headers will be dropped:
gor --input-raw 127.0.0.1:8080 --output-http staging.com --http-allow-header api-version:^v1
gor --input-raw :8080 --output-http staging.com --http-allow-header api-version:^v1
-http-disallow-header=[]: A regexp to match a specific header against. Requests with matching headers will be dropped:
gor --input-raw 127.0.0.1:8080 --output-http staging.com --http-disallow-header "User-Agent: Replayed by Gor"
gor --input-raw :8080 --output-http staging.com --http-disallow-header "User-Agent: Replayed by Gor"
-http-allow-method=[]: Whitelist of HTTP methods to replay. Anything else will be dropped:
gor --input-raw 127.0.0.1:8080 --output-http staging.com --http-allow-method GET --http-allow-method OPTIONS
gor --input-raw :8080 --output-http staging.com --http-allow-method GET --http-allow-method OPTIONS
-http-allow-url=[]: A regexp to match requests against. Filter get matched agains full url with domain. Anything else will be dropped:
gor --input-raw 127.0.0.1:8080 --output-http staging.com --http-allow-url ^www.
gor --input-raw :8080 --output-http staging.com --http-allow-url ^www.
-http-disallow-url=[]: A regexp to match requests against. Filter get matched agains full url with domain. Anything else will be forwarded:
gor --input-raw 127.0.0.1:8080 --output-http staging.com --http-disallow-url ^www.
gor --input-raw :8080 --output-http staging.com --http-disallow-url ^www.
-http-header-limiter=[]: Takes a fraction of requests, consistently taking or rejecting a request based on the FNV32-1A hash of a specific header:
gor --input-raw 127.0.0.1:8080 --output-http staging.com --http-header-imiter user-id:25%
gor --input-raw :8080 --output-http staging.com --http-header-imiter user-id:25%
-http-param-limiter=[]: Takes a fraction of requests, consistently taking or rejecting a request based on the FNV32-1A hash of a specific GET param:
gor --input-raw 127.0.0.1:8080 --output-http staging.com --http-param-limiter user_id:25%
gor --input-raw :8080 --output-http staging.com --http-param-limiter user_id:25%
-http-rewrite-url=[]: Rewrite the request url based on a mapping:
gor --input-raw 127.0.0.1:8080 --output-http staging.com --http-rewrite-url /v1/user/([^\/]+)/ping:/v2/user/$1/ping
gor --input-raw :8080 --output-http staging.com --http-rewrite-url /v1/user/([^\/]+)/ping:/v2/user/$1/ping
-http-set-header=[]: Inject additional headers to http reqest:
gor --input-raw 127.0.0.1:8080 --output-http staging.com --http-set-header 'User-Agent: Gor'
gor --input-raw :8080 --output-http staging.com --http-set-header 'User-Agent: Gor'
-http-set-param=[]: Set request url param, if param already exists it will be overwritten:
gor --input-raw 127.0.0.1:8080 --output-http staging.com --http-set-param api_key=1
gor --input-raw :8080 --output-http staging.com --http-set-param api_key=1
-input-dummy=[]: Used for testing outputs. Emits 'Get /' request every 1s
-input-file=[]: Read requests from file:
gor --input-file ./requests.gor --output-http staging.com
-input-http=[]: Read requests from HTTP, should be explicitly sent from your application:
# Listen for http on 9000
gor --input-http 127.0.0.1:9000 --output-http staging.com
gor --input-http :9000 --output-http staging.com
-input-raw=[]: Capture traffic from given port (use RAW sockets and require *sudo* access):
# Capture traffic from 8080 port
gor --input-raw 127.0.0.1:8080 --output-http staging.com
gor --input-raw :8080 --output-http staging.com
-input-tcp=[]: Used for internal communication between Gor instances. Example:
# Receive requests from other Gor instances on 28020 port, and redirect output to staging
gor --input-tcp :28020 --output-http staging.com
-memprofile="": write memory profile to this file
-output-dummy=[]: Used for testing inputs. Just prints data coming from inputs.
-output-file=[]: Write incoming requests to file:
gor --input-raw 127.0.0.1:80 --output-file ./requests.gor
gor --input-raw :80 --output-file ./requests.gor
-output-http=[]: Forwards incoming requests to given http address.
# Redirect all incoming requests to staging.com address
gor --input-raw 127.0.0.1:80 --output-http http://staging.com
gor --input-raw :80 --output-http http://staging.com
-output-http-elasticsearch="": Send request and response stats to ElasticSearch:
gor --input-raw 127.0.0.1:8080 --output-http staging.com --output-http-elasticsearch 'es_host:api_port/index_name'
gor --input-raw :8080 --output-http staging.com --output-http-elasticsearch 'es_host:api_port/index_name'
-output-http-header-filter=[]: WARNING: `--output-http-header-filter` DEPRECATED, use `--http-allow-header` instead
-output-http-redirects=0: Enable how often redirects should be followed.
-output-http-stats=false: Report http output queue stats to console every 5 seconds. Remember to include also `--stats`
-output-http-workers=0: Gor uses dynamic worker scaling by default. Enter a number to run a set number of workers.
-output-tcp=[]: Used for internal communication between Gor instances. Example:
# Listen for requests on 80 port and forward them to other Gor instance on 28020 port
gor --input-raw 127.0.0.1:80 --output-tcp replay.local:28020
gor --input-raw :80 --output-tcp replay.local:28020
-output-tcp-stats=false: Report TCP output queue stats to console every 5 seconds. Remember to include also `--stats`
-split-output=false: By default each output gets same traffic. If set to `true` it splits traffic equally among all outputs.
-stats=false: Turn on queue stats output. Use in combination with the other *-stats flags.
input_raw_test.go
+56
View File
@@ -26,6 +26,62 @@ func TestRAWInput(t *testing.T) {
wg := new(sync.WaitGroup)
quit := make(chan int)
listener, err := net.Listen("tcp", ":0")
if err != nil {
t.Fatal(err)
}
origin := &http.Server{
Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {}),
ReadTimeout: 10 * time.Second,
WriteTimeout: 10 * time.Second,
}
go origin.Serve(listener)
defer listener.Close()
originAddr := listener.Addr().String()
var respCounter, reqCounter int64
input := NewRAWInput(originAddr, EnginePcap, testRawExpire)
defer input.Close()
output := NewTestOutput(func(data []byte) {
if data[0] == '1' {
atomic.AddInt64(&reqCounter, 1)
} else {
atomic.AddInt64(&respCounter, 1)
}
if Settings.debug {
log.Println(reqCounter, respCounter)
}
wg.Done()
})
Plugins.Inputs = []io.Reader{input}
Plugins.Outputs = []io.Writer{output}
client := NewHTTPClient("http://" + listener.Addr().String(), &HTTPClientConfig{})
go Start(quit)
for i := 0; i < 100; i++ {
// request + response
wg.Add(2)
client.Get("/")
time.Sleep(2 * time.Millisecond)
}
wg.Wait()
close(quit)
}
func TestRAWInputIPv6(t *testing.T) {
wg := new(sync.WaitGroup)
quit := make(chan int)
listener, err := net.Listen("tcp", "[::1]:0")
if err != nil {
t.Fatal(err)
raw_socket_listener/listener.go
+89 -69
View File
@@ -233,98 +233,118 @@ func (e *DeviceNotFoundError) Error() string {
return msg
}
func findPcapDevice(addr string) (*pcap.Interface, error) {
func findPcapDevices(addr string) (interfaces []pcap.Interface, err error) {
devices, err := pcap.FindAllDevs()
if err != nil {
log.Fatal(err)
}
for _, device := range devices {
if (addr == "" || addr == "0.0.0.0" || addr == "[::]" || addr == "::") && len(device.Addresses) > 0 {
interfaces = append(interfaces, device)
continue
}
for _, address := range device.Addresses {
if device.Name == addr || address.IP.String() == addr {
return &device, nil
interfaces = append(interfaces, device)
return interfaces, nil
}
}
}
return nil, &DeviceNotFoundError{addr}
if len(interfaces) == 0 {
return nil, &DeviceNotFoundError{addr}
} else {
return interfaces, nil
}
}
func (t *Listener) readPcap() {
device, err := findPcapDevice(t.addr)
devices, err := findPcapDevices(t.addr)
if err != nil {
log.Fatal(err)
}
handle, err := pcap.OpenLive(device.Name, 65536, true, t.messageExpire)
if err != nil {
log.Fatal(err)
}
defer handle.Close()
var wg sync.WaitGroup
wg.Add(len(devices))
if err := handle.SetBPFFilter("tcp and port " + strconv.Itoa(int(t.port))); err != nil {
log.Fatal(err)
for _, d := range devices {
go func(device pcap.Interface) {
handle, err := pcap.OpenLive(device.Name, 65536, true, t.messageExpire)
if err != nil {
log.Fatal(err)
}
defer handle.Close()
if err := handle.SetBPFFilter("tcp and port " + strconv.Itoa(int(t.port))); err != nil {
log.Fatal(err)
}
source := gopacket.NewPacketSource(handle, handle.LinkType())
source.Lazy = true
source.NoCopy = true
wg.Done()
for {
packet, err := source.NextPacket()
if err == io.EOF {
break
} else if err != nil {
continue
}
// Skip ethernet layer, 14 bytes
data := packet.Data()[14:]
version := uint8(data[0]) >> 4
var srcIP []byte
if version == 4 {
ihl := uint8(data[0]) & 0x0F
// Truncated IP info
if len(data) < int(ihl*4) {
continue
}
srcIP = data[12:16]
data = data[ihl*4:]
} else {
// Truncated IP info
if len(data) < 40 {
continue
}
srcIP = data[8:24]
data = data[40:]
}
// Truncated TCP info
if len(data) < 13 {
continue
}
dataOffset := (data[12] & 0xF0) >> 4
// We need only packets with data inside
// Check that the buffer is larger than the size of the TCP header
if len(data) > int(dataOffset*4) {
newBuf := make([]byte, len(data) + 16)
copy(newBuf[:16], srcIP)
copy(newBuf[16:], data)
t.packetsChan <- newBuf
}
}
}(d)
}
source := gopacket.NewPacketSource(handle, handle.LinkType())
source.Lazy = true
source.NoCopy = true
wg.Wait()
t.readyCh <- true
for {
packet, err := source.NextPacket()
if err == io.EOF {
break
} else if err != nil {
continue
}
// Skip ethernet layer, 14 bytes
data := packet.Data()[14:]
version := uint8(data[0]) >> 4
var srcIP []byte
if version == 4 {
ihl := uint8(data[0]) & 0x0F
// Truncated IP info
if len(data) < int(ihl*4) {
continue
}
srcIP = data[12:16]
data = data[ihl*4:]
} else {
// Truncated IP info
if len(data) < 40 {
continue
}
srcIP = data[8:24]
data = data[40:]
}
// Truncated TCP info
if len(data) < 13 {
continue
}
dataOffset := (data[12] & 0xF0) >> 4
// We need only packets with data inside
// Check that the buffer is larger than the size of the TCP header
if len(data) > int(dataOffset*4) {
newBuf := make([]byte, len(data) + 16)
copy(newBuf[:16], srcIP)
copy(newBuf[16:], data)
t.packetsChan <- newBuf
}
}
}
func (t *Listener) readRAWSocket() {