mirror of
https://github.com/buger/goreplay.git
synced 2026-10-20 00:16:42 +00:00
Compare commits
3
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Leonid Bugaev
|
7e5d71758e | ||
|
Leonid Bugaev
|
be033f1f1e | ||
|
Leonid Bugaev
|
2d370bd767 |
+15
@@ -0,0 +1,15 @@
|
||||
[submodule "vendor/github.com/mattbaird/elastigo"]
|
||||
path = vendor/github.com/mattbaird/elastigo
|
||||
url = https://github.com/mattbaird/elastigo
|
||||
[submodule "vendor/github.com/araddon/gou"]
|
||||
path = vendor/github.com/araddon/gou
|
||||
url = https://github.com/araddon/gou
|
||||
[submodule "vendor/github.com/bitly/go-hostpool"]
|
||||
path = vendor/github.com/bitly/go-hostpool
|
||||
url = https://github.com/bitly/go-hostpool
|
||||
[submodule "vendor/github.com/bmizerany/assert"]
|
||||
path = vendor/github.com/bmizerany/assert
|
||||
url = https://github.com/bmizerany/assert
|
||||
[submodule "vendor/github.com/google/gopacket"]
|
||||
path = vendor/github.com/google/gopacket
|
||||
url = https://github.com/google/gopacket
|
||||
|
||||
+3
-3
@@ -1,6 +1,6 @@
|
||||
language: go
|
||||
go: 1.14
|
||||
script: sudo -E bash -c "source /etc/profile && eval '$(gimme 1.14)' && export GOPATH=$HOME/gopath:$GOPATH && go get && GORACE='halt_on_error=1' go test ./... -v -timeout 120s -race"
|
||||
go: 1.6
|
||||
script: sudo -E bash -c "source /etc/profile && eval '$(gimme 1.6)' && export GOPATH=$HOME/gopath:$GOPATH && go get && GORACE='halt_on_error=1' go test ./... -v -timeout 120s -race"
|
||||
|
||||
before_install:
|
||||
- sudo apt-get install libpcap-dev -y
|
||||
- sudo apt-get install libpcap-dev -y
|
||||
+6
-7
@@ -36,7 +36,7 @@ under the terms of the GNU Lesser General Public License versions 3.0
|
||||
|
||||
5. Fees and Payment. The Software license fees will be due and payable in full as set forth in the applicable invoice or at the time of purchase. If the Software does not function properly within two weeks of purchase, please contact us within those two weeks for a refund. You shall be responsible for all taxes, withholdings, duties and levies arising from the order (excluding taxes based on the net income of GoReplay LLC).
|
||||
|
||||
6. Support, Maintenance and Services. Subject to the terms and conditions of this Agreement, as set forth in your invoice, and as set forth on the GoReplay Pro support page (https://github.com/buger/goreplay/wiki/Pro-Support), support and maintenance services may be included with the purchase of your license subscription.
|
||||
6. Support, Maintenance and Services. Subject to the terms and conditions of this Agreement, as set forth in your invoice, and as set forth on the GoReplay Pro support page (https://github.com/buger/gor/wiki/Pro-Support), support and maintenance services may be included with the purchase of your license subscription.
|
||||
|
||||
7. Term of Agreement.
|
||||
|
||||
@@ -48,11 +48,10 @@ under the terms of the GNU Lesser General Public License versions 3.0
|
||||
|
||||
8. Disclaimer of Warranties. The Software is provided "as is," with all faults, defects and errors, and without warranty of any kind. GoReplay LLC does not warrant that the Software will be free of bugs, errors, viruses or other defects, and GoReplay LLC shall have no liability of any kind for the use of or inability to use the Software, the Software content or any associated service, and you acknowledge that it is not technically practicable for GoReplay LLC to do so.
|
||||
To the maximum extent permitted by applicable law, GoReplay LLC disclaims all warranties, express, implied, arising by law or otherwise, regarding the Software, the Software content and their respective performance or suitability for your intended use, including without limitation any implied warranty of merchantability, fitness for a particular purpose.
|
||||
Notwithstanding the foregoing, GoReplay LLC represents and warrants that it either owns the entire right to, title to, interest in, or has the right to license, the Software, and your proper use of the same will not violate any intellectual property or proprietary rights of another.
|
||||
|
||||
9. Limitation of Liability.
|
||||
|
||||
In no event will either party be liable for any direct, indirect, consequential, incidental, special, exemplary, or punitive damages or liabilities whatsoever arising from or relating to the Software, the Software content or this Agreement, whether based on contract, tort (including negligence), strict liability or other theory, even if such party has been advised of the possibility of such damages.
|
||||
In no event will GoReplay LLC be liable for any direct, indirect, consequential, incidental, special, exemplary, or punitive damages or liabilities whatsoever arising from or relating to the Software, the Software content or this Agreement, whether based on contract, tort (including negligence), strict liability or other theory, even if GoReplay LLC has been advised of the possibility of such damages.
|
||||
|
||||
In no event will GoReplay LLC liability exceed the Software license price as indicated in the invoice. The existence of more than one claim will not enlarge or extend this limit.
|
||||
|
||||
@@ -60,9 +59,9 @@ In no event will GoReplay LLC liability exceed the Software license price as ind
|
||||
|
||||
11. Acknowledgements.
|
||||
|
||||
11.1 Consent to the Use of Data. You agree that GoReplay LLC and its affiliates may collect and use technical information gathered as part of the product support services to be used in an anonymized manner. GoReplay LLC may use this information solely to improve products and services and will not disclose this information in a form that personally identifies you.
|
||||
11.1 Consent to the Use of Data. You agree that GoReplay LLC and its affiliates may collect and use technical information gathered as part of the product support services. GoReplay LLC may use this information solely to improve products and services and will not disclose this information in a form that personally identifies you.
|
||||
|
||||
11.2 Verification. We or a certified auditor acting on our behalf, may, upon its reasonable request and at its expense, audit you with respect to the use of the Software, subject to your reasonable information security and confidentiality protections. Such audit may be conducted by mail, electronic means or through an in-person visit to your place of business. Any such in-person audit shall be conducted during regular business hours at your facilities and shall not unreasonably interfere with your business activities. We shall not remove, copy, or redistribute any electronic material during the course of an audit. If an audit reveals that you are using the Software in a way that is in material violation of the terms of the EULA, then you shall pay our reasonable costs of conducting the audit. In the case of a material violation, you agree to pay Us any amounts owing that are attributable to the unauthorized use. In the alternative, We reserve the right, at our sole option, to terminate the licenses for the Software.
|
||||
11.2 Verification. We or a certified auditor acting on our behalf, may, upon its reasonable request and at its expense, audit you with respect to the use of the Software. Such audit may be conducted by mail, electronic means or through an in-person visit to your place of business. Any such in-person audit shall be conducted during regular business hours at your facilities and shall not unreasonably interfere with your business activities. We shall not remove, copy, or redistribute any electronic material during the course of an audit. If an audit reveals that you are using the Software in a way that is in material violation of the terms of the EULA, then you shall pay our reasonable costs of conducting the audit. In the case of a material violation, you agree to pay Us any amounts owing that are attributable to the unauthorized use. In the alternative, We reserve the right, at our sole option, to terminate the licenses for the Software.
|
||||
|
||||
11.3 Government End Users. If the Software and related documentation are supplied to or purchased by or on behalf of the United States Government, then the Software is deemed to be "commercial software" as that term is used in the Federal Acquisition Regulation system. Rights of the United States shall not exceed the minimum rights set forth in FAR 52.227-19 for "restricted computer software". All other terms and conditions of this Agreement apply.
|
||||
|
||||
@@ -73,11 +72,11 @@ In no event will GoReplay LLC liability exceed the Software license price as ind
|
||||
|
||||
13.1 Entire Agreement. This Agreement sets forth our entire agreement with respect to the Software and the subject matter hereof and supersedes all prior and contemporaneous understandings and agreements whether written or oral.
|
||||
|
||||
13.2 Amendment. GoReplay LLC reserves the right, in its sole discretion, to amend this Agreement from time. Amendments to this Agreement can be located at: https://github.com/buger/goreplay/blob/master/COMM-LICENSE.
|
||||
13.2 Amendment. GoReplay LLC reserves the right, in its sole discretion, to amend this Agreement from time. Amendments to this Agreement can be located at: https://github.com/buger/gor/blob/master/COMM-LICENSE.
|
||||
|
||||
13.3 Assignment. You may not assign this Agreement or any of its rights under this Agreement without the prior written consent of GoReplay LLC and any attempted assignment without such consent shall be void.
|
||||
|
||||
13.4 Export Compliance. Each party agrees to comply with all applicable laws and regulations, including laws, regulations, orders or other restrictions on export, re-export or redistribution of software.
|
||||
13.4 Export Compliance. You agree to comply with all applicable laws and regulations, including laws, regulations, orders or other restrictions on export, re-export or redistribution of software.
|
||||
|
||||
13.5 Indemnification. You agree to defend, indemnify, and hold harmless GoReplay LLC from and against any lawsuits, claims, losses, damages, fines and expenses (including attorneys' fees and costs) arising out of your use of the Software or breach of this Agreement.
|
||||
|
||||
|
||||
+23
-7
@@ -1,8 +1,24 @@
|
||||
FROM alpine:latest as builder
|
||||
RUN apk add --no-cache ca-certificates openssl
|
||||
RUN wget https://github.com/buger/goreplay/releases/download/v1.0.0/gor_1.0.0_x64.tar.gz -O gor.tar.gz
|
||||
RUN tar xzf gor.tar.gz
|
||||
FROM golang:latest
|
||||
|
||||
FROM scratch
|
||||
COPY --from=builder /gor .
|
||||
ENTRYPOINT ["./gor"]
|
||||
RUN apt-get update && apt-get install ruby vim-common -y
|
||||
|
||||
# Install Java for middleware testing
|
||||
RUN echo "deb http://ppa.launchpad.net/webupd8team/java/ubuntu trusty main" | tee /etc/apt/sources.list.d/webupd8team-java.list
|
||||
RUN echo "deb-src http://ppa.launchpad.net/webupd8team/java/ubuntu trusty main" | tee -a /etc/apt/sources.list.d/webupd8team-java.list
|
||||
RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys EEA14886
|
||||
RUN apt-get update -y
|
||||
RUN echo oracle-java7-installer shared/accepted-oracle-license-v1-1 select true | /usr/bin/debconf-set-selections
|
||||
RUN apt-get install oracle-java8-installer -y
|
||||
|
||||
RUN apt-get install flex bison -y
|
||||
RUN wget http://www.tcpdump.org/release/libpcap-1.7.4.tar.gz && tar xzf libpcap-1.7.4.tar.gz && cd libpcap-1.7.4 && ./configure && make install
|
||||
RUN go get github.com/google/gopacket
|
||||
RUN go get -u github.com/golang/lint/golint
|
||||
|
||||
WORKDIR /go/src/github.com/buger/gor/
|
||||
ADD . /go/src/github.com/buger/gor/
|
||||
|
||||
RUN wget http://archive.apache.org/dist/commons/io/binaries/commons-io-2.4-bin.tar.gz && tar xzf commons-io-2.4-bin.tar.gz && cd commons-io-2.4 && mv commons-io-2.4.jar /tmp/
|
||||
RUN wget http://archive.apache.org/dist/commons/codec/binaries/commons-codec-1.9-bin.tar.gz && tar xzf commons-codec-1.9-bin.tar.gz
|
||||
RUN javac -cp commons-io-2.4/commons-io-2.4.jar -cp commons-codec-1.9/commons-codec-1.9.jar ./examples/middleware/echo.java
|
||||
RUN go get
|
||||
@@ -1,14 +0,0 @@
|
||||
FROM golang:1.14
|
||||
|
||||
RUN apt-get update && apt-get install ruby vim-common -y
|
||||
|
||||
RUN apt-get install flex bison -y
|
||||
RUN wget http://www.tcpdump.org/release/libpcap-1.8.1.tar.gz && tar xzf libpcap-1.8.1.tar.gz && cd libpcap-1.8.1 && ./configure && make install
|
||||
RUN apt-get install default-jre default-jdk -y
|
||||
RUN go get github.com/google/gopacket
|
||||
RUN go get -u golang.org/x/lint/golint
|
||||
|
||||
WORKDIR /go/src/github.com/buger/goreplay/
|
||||
ADD . /go/src/github.com/buger/goreplay/
|
||||
|
||||
RUN go get
|
||||
@@ -1,12 +1,3 @@
|
||||
Copyright (c) 2011-present Leonid Bugaev
|
||||
|
||||
Portions of this software are licensed as follows:
|
||||
|
||||
* All content residing under the "doc/" directory of this repository is licensed under "Creative Commons: CC BY-SA 4.0 license".
|
||||
* Files with the following names "pro.go", "s3_reader.go", "output_s3.go" are released under the commercial license specified in "COMM-LICENSE" file.
|
||||
* Content outside of the above mentioned directories or restrictions above is available under the "LGPLv3" license as defined below.
|
||||
|
||||
|
||||
GoReplay is an Open Source project licensed under the terms of
|
||||
the LGPLv3 license. Please see <http://www.gnu.org/licenses/lgpl-3.0.html>
|
||||
for license text.
|
||||
|
||||
@@ -1,36 +1,32 @@
|
||||
SOURCE = $(shell ls -1 *.go | grep -v _test.go)
|
||||
SOURCE_PATH = /go/src/github.com/buger/goreplay/
|
||||
SOURCE = emitter.go gor.go gor_stat.go input_dummy.go input_file.go input_raw.go input_tcp.go limiter.go output_dummy.go output_null.go output_file.go input_http.go output_http.go output_tcp.go plugins.go settings.go test_input.go elasticsearch.go http_modifier.go http_modifier_settings.go http_client.go middleware.go protocol.go output_file_settings.go output_kafka.go
|
||||
SOURCE_PATH = /go/src/github.com/buger/gor/
|
||||
PORT = 8000
|
||||
FADDR = :8000
|
||||
CONTAINER=gor
|
||||
PREFIX=
|
||||
RUN = docker run -v `pwd`:$(SOURCE_PATH) -e AWS_ACCESS_KEY_ID=$(AWS_ACCESS_KEY_ID) -e AWS_SECRET_ACCESS_KEY=$(AWS_SECRET_ACCESS_KEY) -p 0.0.0.0:$(PORT):$(PORT) -t -i $(CONTAINER)
|
||||
RUN = docker run -v `pwd`:$(SOURCE_PATH) -p 0.0.0.0:$(PORT):$(PORT) -i -t gor
|
||||
BENCHMARK = BenchmarkRAWInput
|
||||
TEST = TestRawListenerBench
|
||||
VERSION = DEV-$(shell date +%s)
|
||||
LDFLAGS = -ldflags "-X main.VERSION=$(VERSION)$(PREFIX) -extldflags \"-static\" -X main.DEMO=$(DEMO)"
|
||||
MAC_LDFLAGS = -ldflags "-X main.VERSION=$(VERSION)$(PREFIX) -X main.DEMO=$(DEMO)"
|
||||
LDFLAGS = -ldflags "-X main.VERSION=$(VERSION) -extldflags \"-static\""
|
||||
MAC_LDFLAGS = -ldflags "-X main.VERSION=$(VERSION)"
|
||||
FADDR = ":8000"
|
||||
|
||||
release: release-x64 release-mac
|
||||
|
||||
release-bin:
|
||||
docker run -v `pwd`:$(SOURCE_PATH) -t --env GOOS=linux --env GOARCH=amd64 -i gor go build -o gor -tags netgo $(LDFLAGS)
|
||||
docker run -v `pwd`:$(SOURCE_PATH) -t --env GOOS=linux --env GOARCH=amd64 -i gor go build -tags netgo $(LDFLAGS)
|
||||
|
||||
release-x64:
|
||||
docker run -v `pwd`:$(SOURCE_PATH) -t --env GOOS=linux --env GOARCH=amd64 -i $(CONTAINER) go build -o gor -tags netgo $(LDFLAGS) && tar -czf gor_$(VERSION)$(PREFIX)_x64.tar.gz gor && rm gor
|
||||
docker run -v `pwd`:$(SOURCE_PATH) -t --env GOOS=linux --env GOARCH=amd64 -i gor go build -tags netgo $(LDFLAGS) && tar -czf gor_$(VERSION)_x64.tar.gz gor && rm gor
|
||||
|
||||
release-x86:
|
||||
docker run -v `pwd`:$(SOURCE_PATH) -t --env GOOS=linux --env GOARCH=386 -i $(CONTAINER) go build -o gor -tags netgo $(LDFLAGS) && tar -czf gor_$(VERSION)$(PREFIX)_x86.tar.gz gor && rm gor
|
||||
docker run -v `pwd`:$(SOURCE_PATH) -t --env GOOS=linux --env GOARCH=386 -i gor go build -tags netgo $(LDFLAGS) && tar -czf gor_$(VERSION)_x86.tar.gz gor && rm gor
|
||||
|
||||
release-mac:
|
||||
go build -o gor $(MAC_LDFLAGS) && tar -czf gor_$(VERSION)$(PREFIX)_mac.tar.gz gor && rm gor
|
||||
|
||||
install:
|
||||
go install $(MAC_LDFLAGS)
|
||||
go build $(MAC_LDFLAGS) && tar -czf gor_$(VERSION)_mac.tar.gz gor && rm gor
|
||||
|
||||
build:
|
||||
docker build -t $(CONTAINER) -f Dockerfile.dev .
|
||||
docker build -t gor .
|
||||
|
||||
|
||||
profile:
|
||||
go build && ./gor --output-http="http://localhost:9000" --input-dummy 0 --input-raw :9000 --input-http :9000 --memprofile=./mem.out --cpuprofile=./cpu.out --stats --output-http-stats --output-http-timeout 100ms
|
||||
@@ -42,13 +38,13 @@ race:
|
||||
$(RUN) go test ./... $(ARGS) -v -race -timeout 15s
|
||||
|
||||
test:
|
||||
$(RUN) go test ./. -timeout 120s $(LDFLAGS) $(ARGS) -v
|
||||
$(RUN) go test ./. -timeout 60s $(LDFLAGS) $(ARGS) -v
|
||||
|
||||
test_all:
|
||||
$(RUN) go test ./... -timeout 120s $(LDFLAGS) $(ARGS) -v
|
||||
$(RUN) go test ./... -timeout 60s $(LDFLAGS) $(ARGS) -v
|
||||
|
||||
testone:
|
||||
$(RUN) go test ./. -timeout 60s $(LDFLAGS) -run $(TEST) $(ARGS) -v
|
||||
$(RUN) go test ./... -timeout 4s $(LDFLAGS) -run $(TEST) $(ARGS) -v
|
||||
|
||||
cover:
|
||||
$(RUN) go test $(ARGS) -race -v -timeout 15s -coverprofile=coverage.out
|
||||
@@ -64,15 +60,15 @@ bench:
|
||||
$(RUN) go test $(LDFLAGS) -v -run NOT_EXISTING -bench $(BENCHMARK) -benchtime 5s
|
||||
|
||||
profile_test:
|
||||
$(RUN) go test $(LDFLAGS) -run $(TEST) ./capture/. $(ARGS) -memprofile mem.mprof -cpuprofile cpu.out
|
||||
$(RUN) go test $(LDFLAGS) -run $(TEST) ./capture/. $(ARGS) -c
|
||||
$(RUN) go test $(LDFLAGS) -run $(TEST) ./raw_socket_listener/. $(ARGS) -memprofile mem.mprof -cpuprofile cpu.out
|
||||
$(RUN) go test $(LDFLAGS) -run $(TEST) ./raw_socket_listener/. $(ARGS) -c
|
||||
|
||||
# Used mainly for debugging, because docker container do not have access to parent machine ports
|
||||
run:
|
||||
$(RUN) go run $(LDFLAGS) $(SOURCE) --input-dummy=0 --output-http="http://localhost:9000" --input-raw-track-response --input-raw 127.0.0.1:9000 --verbose --debug --middleware "./examples/middleware/echo.sh" --output-file requests.gor
|
||||
|
||||
run-2:
|
||||
$(RUN) go run $(LDFLAGS) $(SOURCE) --input-raw :8000 --input-raw-bpf-filter "dst port 8000" --output-stdout --output-http "http://localhost:8000" --input-dummy=0
|
||||
sudo -E go run $(SOURCE) --input-dummy="" --output-tcp localhost:27001 --verbose --debug
|
||||
|
||||
run-3:
|
||||
sudo -E go run $(SOURCE) --input-tcp :27001 --output-stdout
|
||||
@@ -84,7 +80,7 @@ file-server:
|
||||
go run $(SOURCE) file-server $(FADDR)
|
||||
|
||||
readpcap:
|
||||
go run $(SOURCE) --input-raw $(FILE) --input-raw-track-response --input-raw-engine pcap_file --output-stdout
|
||||
go run $(SOURCE) --input-raw $(FILE) --input-raw-engine pcap_file --output-null
|
||||
|
||||
record:
|
||||
$(RUN) go run $(SOURCE) --input-dummy=0 --output-file=requests.gor --verbose --debug
|
||||
@@ -94,20 +90,3 @@ replay:
|
||||
|
||||
bash:
|
||||
$(RUN) /bin/bash
|
||||
|
||||
|
||||
FPMCOMMON= \
|
||||
--name goreplay \
|
||||
--description "GoReplay is an open-source network monitoring tool which can record your live traffic, and use it for shadowing, load testing, monitoring and detailed analysis." \
|
||||
-v $(VERSION) \
|
||||
--vendor "Leonid Bugaev" \
|
||||
-m "<support@goreplay.org>" \
|
||||
--url "https://goreplay.org" \
|
||||
-s dir \
|
||||
-C /tmp/gor-build \
|
||||
|
||||
build_packages:
|
||||
mkdir -p /tmp/gor-build
|
||||
go build -i -o /tmp/gor-build/gor
|
||||
fpm $(FPMCOMMON) -a amd64 -t deb ./=/usr/local/bin
|
||||
fpm $(FPMCOMMON) -a amd64 -t rpm ./=/usr/local/bin
|
||||
|
||||
@@ -1,34 +1,34 @@
|
||||
[](https://github.com/buger/goreplay/releases) [](https://codebeat.co/projects/github-com-buger-gor) [](https://goreportcard.com/report/github.com/buger/gor) [](https://gitter.im/buger/gor?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) [](https://houndci.com)
|
||||
[](https://github.com/buger/gor/releases) [](https://codebeat.co/projects/github-com-buger-gor) [](https://goreportcard.com/report/github.com/buger/gor) [](https://gitter.im/buger/gor?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
|
||||
|
||||
|
||||
|
||||
## https://goreplay.org/
|
||||
|
||||
GoReplay is an open-source network monitoring tool which can record your live traffic, and use it for shadowing, load testing, monitoring and detailed analysis.
|
||||
|
||||
## About
|
||||
|
||||
As your application grows, the effort required to test it also grows exponentially. GoReplay offers you the simple idea of reusing your existing traffic for testing, which makes it incredibly powerful. Our state of art technique allows you to analyze and record your application traffic without affecting it. This eliminates the risks that come with putting a third party component in the critical path.
|
||||
GoReplay is the simplest and safest way to test your app using real traffic before you put it into production.
|
||||
|
||||
GoReplay increases your confidence in code deployments, configuration and infrastructure changes.
|
||||
As your application grows, the effort required to test it also grows exponentially. GoReplay offers you the simple idea of reusing your existing traffic for testing, which makes it incredibly powerful. Our state of art technique allows to analyze and record your application traffic without affecting it. This eliminates the risks that come with putting a third party component in the critical path.
|
||||
|
||||
GoReplay increases your confidence in code deployments, configuration changes and infrastructure changes. Did we mention that no coding is required?
|
||||
|
||||
|
||||
GoReplay offers unique approach for shadowing. Instead of being a proxy, GoReplay in background listen for traffic on your network interface, requiring no changes in your production infrastructure, rather then running GoReplay daemon on the same machine as your service.
|
||||
|
||||
|
||||
Here is basic workflow: The listener server catches http traffic and sends it to the replay server or saves to file. The replay server forwards traffic to a given address.
|
||||
|
||||
Check [latest documentation](http://github.com/buger/goreplay/wiki).
|
||||
|
||||
|
||||
Check [latest documentation](http://github.com/buger/gor/wiki).
|
||||
|
||||
## Installation
|
||||
Download latest binary from https://github.com/buger/goreplay/releases or [compile by yourself](https://github.com/buger/goreplay/wiki/Compilation).
|
||||
Download latest binary from https://github.com/buger/gor/releases or [compile by yourself](https://github.com/buger/gor/wiki/Compilation).
|
||||
|
||||
## Getting started
|
||||
|
||||
The most basic setup will be `sudo ./gor --input-raw :8000 --output-stdout` which acts like tcpdump.
|
||||
If you already have test environment you can start replaying: `sudo ./gor --input-raw :8000 --output-http http://staging.env`.
|
||||
|
||||
See the our [documentation](https://github.com/buger/goreplay/wiki/) and [Getting started](https://github.com/buger/goreplay/wiki/Getting-Started) page for more info.
|
||||
|
||||
See the our [documentation](https://github.com/buger/gor/wiki/) and [Getting started](https://github.com/buger/gor/wiki/Getting-Started) page for more info.
|
||||
## Newsletter
|
||||
Subscribe to our [newsletter](https://www.getdrip.com/forms/89690474/submissions/new) to stay informed about the latest features and changes to Gor project.
|
||||
|
||||
@@ -39,9 +39,9 @@ We have created a [GoReplay PRO](https://goreplay.org/pro.html) extension which
|
||||
|
||||
|
||||
## Problems?
|
||||
If you have a problem, please review the [FAQ](https://github.com/buger/goreplay/wiki/FAQ) and [Troubleshooting](https://github.com/buger/goreplay/wiki/Troubleshooting) wiki pages. Searching the [issues](https://github.com/buger/goreplay/issues) for your problem is also a good idea.
|
||||
If you have a problem, please review the [FAQ](https://github.com/buger/gor/wiki/FAQ) and [Troubleshooting](https://github.com/buger/gor/wiki/Troubleshooting) wiki pages. Searching the [issues](https://github.com/buger/gor/issues) for your problem is also a good idea.
|
||||
|
||||
All bug-reports and suggestions should go through Github Issues or our [Google Group](https://groups.google.com/forum/#!forum/gor-users) (you can just send email to gor-users@googlegroups.com).
|
||||
All bug-reports and suggestions should go though Github Issues or our [Google Group](https://groups.google.com/forum/#!forum/gor-users) (you can just send email to gor-users@googlegroups.com).
|
||||
If you have a private question feel free to send email to support@gortool.com.
|
||||
|
||||
|
||||
@@ -66,7 +66,7 @@ If you have a private question feel free to send email to support@gortool.com.
|
||||
* [Granify](http://granify.com) - AI backed SaaS solution that enables online retailers to maximise their sales
|
||||
* And many more!
|
||||
|
||||
If you are using Gor, we are happy to add you to the list and share your story, just write to: hello@goreplay.org
|
||||
If you are using Gor we are happy add you to the list and share your story, just write to: hello@goreplay.org
|
||||
|
||||
## Author
|
||||
|
||||
|
||||
@@ -29,15 +29,12 @@ func Replace(a []byte, from, to int, new []byte) []byte {
|
||||
copy(a[from:from+len(new)], new)
|
||||
|
||||
return a
|
||||
}
|
||||
|
||||
if lenDiff < 0 {
|
||||
} else if lenDiff < 0 {
|
||||
copy(a[from:], new)
|
||||
copy(a[from+len(new):], a[to:])
|
||||
return a[:len(a)+lenDiff]
|
||||
} else { // same size
|
||||
copy(a[from:], new)
|
||||
return a
|
||||
}
|
||||
|
||||
// same size
|
||||
copy(a[from:], new)
|
||||
return a
|
||||
}
|
||||
|
||||
@@ -13,7 +13,7 @@ If you have multiple replay machines you can split traffic among them using `--s
|
||||
gor --input-raw :80 --split-output --output-tcp replay1.local:28020 --output-tcp replay2.local:28020
|
||||
```
|
||||
|
||||
[GoReplay PRO](https://goreplay.org/pro.html) support accurate recording and replaying of tcp sessions, and when `--recognize-tcp-sessions` option is passed, instead of round-robin it will use a smarter algorithm which ensures that same sessions will be sent to the same replay instance.
|
||||
[GoReplay PRO](https://goreplay.com/pro.html) support accurate recording and replaying of tcp sessions, and when `--recognize-tcp-sessions` option is passed, instead of round-robin it will use a smarter algorithm which ensures that same sessions will be sent to the same replay instance.
|
||||
|
||||
|
||||
In case if you are planning a large load testing, you may consider use separate master instance which will control Gor slaves which actually replay traffic. For example:
|
||||
|
||||
+2
-2
@@ -26,7 +26,7 @@ Simple bash echo middleware (returns same request) will look like this:
|
||||
```bash
|
||||
while read line; do
|
||||
echo $line
|
||||
done
|
||||
end
|
||||
```
|
||||
|
||||
Middleware can be enabled using `--middleware` option, by specifying path to executable file:
|
||||
@@ -71,4 +71,4 @@ Imagine that you have auth system that randomly generate access tokens, which us
|
||||
|
||||
***
|
||||
|
||||
You may also read about [[Request filtering]], [[Rate limiting]] and [[Request rewriting]].
|
||||
You may also read about [[Request filtering]], [[Rate limiting]] and [[Request rewriting]].
|
||||
@@ -23,16 +23,16 @@ Set request header, if header already exists it will be overwritten. May be usef
|
||||
|
||||
```
|
||||
gor --input-raw :80 --output-http "http://staging.server" \
|
||||
--http-set-header "User-Agent: Replayed by Gor" \
|
||||
--http-set-header "Enable-Feature-X: true"
|
||||
--http-header "User-Agent: Replayed by Gor" \
|
||||
--http-header "Enable-Feature-X: true"
|
||||
```
|
||||
|
||||
#### Host header
|
||||
Host header gets special treatment. By default Host get set to the value specified in --output-http. If you manually set --http-set-header "Host: anonther.com", Gor will not override Host value.
|
||||
Host header gets special treatment. By default Host get set to the value specified in --output-http. If you manually set --http-header "Host: anonther.com", Gor will not override Host value.
|
||||
|
||||
If you app accepts traffic from multiple domains, and you want to keep original headers, there is specific `--http-original-host` with tells Gor do not touch Host header at all.
|
||||
|
||||
|
||||
***
|
||||
|
||||
You may also read about [[Request filtering]], [[Rate limiting]] and [[Middleware]]
|
||||
You may also read about [[Request filtering]], [[Rate limiting]] and [[Middleware]]
|
||||
+18
-33
@@ -2,20 +2,17 @@ package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"github.com/buger/gor/proto"
|
||||
"github.com/mattbaird/elastigo/lib"
|
||||
"log"
|
||||
"net/url"
|
||||
"strings"
|
||||
"regexp"
|
||||
"time"
|
||||
|
||||
"github.com/buger/goreplay/proto"
|
||||
|
||||
elastigo "github.com/mattbaird/elastigo/lib"
|
||||
)
|
||||
|
||||
type ESUriErorr struct{}
|
||||
|
||||
func (e *ESUriErorr) Error() string {
|
||||
return "Wrong ElasticSearch URL format. Expected to be: scheme://host/index_name"
|
||||
return "Wrong ElasticSearch URL format. Expected to be: host:port/index_name"
|
||||
}
|
||||
|
||||
type ESPlugin struct {
|
||||
@@ -55,28 +52,17 @@ type ESRequestResponse struct {
|
||||
|
||||
// Parse ElasticSearch URI
|
||||
//
|
||||
// Proper format is: scheme://[userinfo@]host/index_name
|
||||
// userinfo is: user[:password]
|
||||
// net/url.Parse() does not fail if scheme is not provided but actualy does not
|
||||
// handle URI properly.
|
||||
// So we must 'validate' URI format to match requirements to use net/url.Parse()
|
||||
func parseURI(URI string) (err error, index string) {
|
||||
// Proper format is: host:port/index_name
|
||||
func parseURI(URI string) (err error, host string, port string, index string) {
|
||||
rURI := regexp.MustCompile("(.+):([0-9]+)/(.+)")
|
||||
match := rURI.FindAllStringSubmatch(URI, -1)
|
||||
|
||||
parsedUrl, parseErr := url.Parse(URI)
|
||||
|
||||
if parseErr != nil {
|
||||
err = new(ESUriErorr)
|
||||
return
|
||||
}
|
||||
|
||||
// check URL validity by extracting host and undex values.
|
||||
host := parsedUrl.Host
|
||||
urlPathParts := strings.Split(parsedUrl.Path, "/")
|
||||
index = urlPathParts[len(urlPathParts)-1]
|
||||
|
||||
// force index specification in uri : ie no implicit index
|
||||
if host == "" || index == "" {
|
||||
if len(match) == 0 {
|
||||
err = new(ESUriErorr)
|
||||
} else {
|
||||
host = match[0][1]
|
||||
port = match[0][2]
|
||||
index = match[0][3]
|
||||
}
|
||||
|
||||
return
|
||||
@@ -85,22 +71,21 @@ func parseURI(URI string) (err error, index string) {
|
||||
func (p *ESPlugin) Init(URI string) {
|
||||
var err error
|
||||
|
||||
err, p.Index = parseURI(URI)
|
||||
err, p.Host, p.ApiPort, p.Index = parseURI(URI)
|
||||
|
||||
if err != nil {
|
||||
log.Fatal("Can't initialize ElasticSearch plugin.", err)
|
||||
}
|
||||
|
||||
p.eConn = elastigo.NewConn()
|
||||
|
||||
p.eConn.SetFromUrl(URI)
|
||||
p.eConn.SetPort(p.ApiPort)
|
||||
p.eConn.SetHosts([]string{p.Host})
|
||||
|
||||
p.indexor = p.eConn.NewBulkIndexerErrors(50, 60)
|
||||
p.done = make(chan bool)
|
||||
p.indexor.Start()
|
||||
|
||||
if Settings.Verbose {
|
||||
// Only start the ErrorHandler goroutine when in Verbose mode
|
||||
if Settings.verbose {
|
||||
// Only start the ErrorHandler goroutine when in verbose mode
|
||||
// no need to burn ressources otherwise
|
||||
go p.ErrorHandler()
|
||||
}
|
||||
|
||||
@@ -1,140 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"testing"
|
||||
)
|
||||
|
||||
const expectedIndex = "gor"
|
||||
|
||||
func assertExpectedGorIndex(index string, t *testing.T) {
|
||||
if expectedIndex != index {
|
||||
t.Fatalf("Expected index %s but got %s", expectedIndex, index)
|
||||
}
|
||||
}
|
||||
|
||||
func assertExpectedIndex(expectedIndex string, index string, t *testing.T) {
|
||||
if expectedIndex != index {
|
||||
t.Fatalf("Expected index %s but got %s", expectedIndex, index)
|
||||
}
|
||||
}
|
||||
|
||||
func assertExpectedError(returnedError error, t *testing.T) {
|
||||
expectedError := new(ESUriErorr)
|
||||
|
||||
if expectedError != returnedError {
|
||||
t.Errorf("Expected err %s but got %s", expectedError, returnedError)
|
||||
}
|
||||
}
|
||||
|
||||
func assertNoError(returnedError error, t *testing.T) {
|
||||
if nil != returnedError {
|
||||
t.Errorf("Expected no err but got %s", returnedError)
|
||||
}
|
||||
}
|
||||
|
||||
// Argument host:port/index_name
|
||||
// i.e : localhost:9200/gor
|
||||
// Fail because scheme is mandatory
|
||||
func TestElasticConnectionBuildFailWithoutScheme(t *testing.T) {
|
||||
uri := "localhost:9200/" + expectedIndex
|
||||
|
||||
err, _ := parseURI(uri)
|
||||
assertExpectedError(err, t)
|
||||
}
|
||||
|
||||
// Argument scheme://host:port
|
||||
// i.e : http://localhost:9200
|
||||
// Fail : explicit index is required
|
||||
func TestElasticConnectionBuildFailWithoutIndex(t *testing.T) {
|
||||
uri := "http://localhost:9200"
|
||||
|
||||
err, index := parseURI(uri)
|
||||
|
||||
assertExpectedIndex("", index, t)
|
||||
|
||||
assertExpectedError(err, t)
|
||||
}
|
||||
|
||||
// Argument scheme://host/index_name
|
||||
// i.e : http://localhost/gor
|
||||
func TestElasticConnectionBuildFailWithoutPort(t *testing.T) {
|
||||
uri := "http://localhost/" + expectedIndex
|
||||
|
||||
err, index := parseURI(uri)
|
||||
|
||||
assertNoError(err, t)
|
||||
|
||||
assertExpectedGorIndex(index, t)
|
||||
}
|
||||
|
||||
// Argument scheme://host:port/index_name
|
||||
// i.e : http://localhost:9200/gor
|
||||
func TestElasticLocalConnectionBuild(t *testing.T) {
|
||||
uri := "http://localhost:9200/" + expectedIndex
|
||||
|
||||
err, index := parseURI(uri)
|
||||
|
||||
assertNoError(err, t)
|
||||
|
||||
assertExpectedGorIndex(index, t)
|
||||
}
|
||||
|
||||
// Argument scheme://host:port/index_name
|
||||
// i.e : http://localhost.local:9200/gor or https://localhost.local:9200/gor
|
||||
func TestElasticSimpleLocalWithSchemeConnectionBuild(t *testing.T) {
|
||||
uri := "http://localhost.local:9200/" + expectedIndex
|
||||
|
||||
err, index := parseURI(uri)
|
||||
|
||||
assertNoError(err, t)
|
||||
|
||||
assertExpectedGorIndex(index, t)
|
||||
}
|
||||
|
||||
// Argument scheme://host:port/index_name
|
||||
// i.e : http://localhost.local:9200/gor or https://localhost.local:9200/gor
|
||||
func TestElasticSimpleLocalWithHTTPSConnectionBuild(t *testing.T) {
|
||||
uri := "https://localhost.local:9200/" + expectedIndex
|
||||
|
||||
err, index := parseURI(uri)
|
||||
|
||||
assertNoError(err, t)
|
||||
|
||||
assertExpectedGorIndex(index, t)
|
||||
}
|
||||
|
||||
// Argument scheme://host:port/index_name
|
||||
// i.e : localhost.local:9200/pathtoElastic/gor
|
||||
func TestElasticLongPathConnectionBuild(t *testing.T) {
|
||||
uri := "http://localhost.local:9200/pathtoElastic/" + expectedIndex
|
||||
|
||||
err, index := parseURI(uri)
|
||||
|
||||
assertNoError(err, t)
|
||||
|
||||
assertExpectedGorIndex(index, t)
|
||||
}
|
||||
|
||||
// Argument scheme://host:userinfo@port/index_name
|
||||
// i.e : http://user:password@localhost.local:9200/gor
|
||||
func TestElasticBasicAuthConnectionBuild(t *testing.T) {
|
||||
uri := "http://user:password@localhost.local:9200/" + expectedIndex
|
||||
|
||||
err, index := parseURI(uri)
|
||||
|
||||
assertNoError(err, t)
|
||||
|
||||
assertExpectedGorIndex(index, t)
|
||||
}
|
||||
|
||||
// Argument scheme://host:port/path/index_name
|
||||
// i.e : http://localhost.local:9200/path/gor or https://localhost.local:9200/path/gor
|
||||
func TestElasticComplexPathConnectionBuild(t *testing.T) {
|
||||
uri := "http://localhost.local:9200/path/" + expectedIndex
|
||||
|
||||
err, index := parseURI(uri)
|
||||
|
||||
assertNoError(err, t)
|
||||
|
||||
assertExpectedGorIndex(index, t)
|
||||
}
|
||||
+40
-135
@@ -2,156 +2,73 @@ package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"hash/fnv"
|
||||
"io"
|
||||
"log"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
type emitter struct {
|
||||
sync.WaitGroup
|
||||
quit chan int
|
||||
}
|
||||
|
||||
// NewEmitter creates and initializes new `emitter` object.
|
||||
func NewEmitter(quit chan int) *emitter {
|
||||
return &emitter{
|
||||
quit: quit,
|
||||
}
|
||||
}
|
||||
|
||||
// Start initialize loop for sending data from inputs to outputs
|
||||
func (e *emitter) Start(plugins *InOutPlugins, middlewareCmd string) {
|
||||
e.Add(1)
|
||||
defer e.Done()
|
||||
func Start(stop chan int) {
|
||||
if Settings.middleware != "" {
|
||||
middleware := NewMiddleware(Settings.middleware)
|
||||
|
||||
if middlewareCmd != "" {
|
||||
middleware := NewMiddleware(middlewareCmd)
|
||||
|
||||
for _, in := range plugins.Inputs {
|
||||
for _, in := range Plugins.Inputs {
|
||||
middleware.ReadFrom(in)
|
||||
}
|
||||
|
||||
// We are going only to read responses, so using same ReadFrom method
|
||||
for _, out := range plugins.Outputs {
|
||||
for _, out := range Plugins.Outputs {
|
||||
if r, ok := out.(io.Reader); ok {
|
||||
middleware.ReadFrom(r)
|
||||
}
|
||||
}
|
||||
e.Add(1)
|
||||
go func() {
|
||||
defer e.Done()
|
||||
if err := CopyMulty(e.quit, middleware, plugins.Outputs...); err != nil {
|
||||
log.Println("Error during copy: ", err)
|
||||
e.close()
|
||||
}
|
||||
}()
|
||||
go func() {
|
||||
for {
|
||||
select {
|
||||
case <-e.quit:
|
||||
middleware.Close()
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
go CopyMulty(middleware, Plugins.Outputs...)
|
||||
} else {
|
||||
for _, in := range plugins.Inputs {
|
||||
e.Add(1)
|
||||
go func(in io.Reader) {
|
||||
defer e.Done()
|
||||
if err := CopyMulty(e.quit, in, plugins.Outputs...); err != nil {
|
||||
log.Println("Error during copy: ", err)
|
||||
e.close()
|
||||
}
|
||||
}(in)
|
||||
for _, in := range Plugins.Inputs {
|
||||
go CopyMulty(in, Plugins.Outputs...)
|
||||
}
|
||||
|
||||
for _, out := range plugins.Outputs {
|
||||
for _, out := range Plugins.Outputs {
|
||||
if r, ok := out.(io.Reader); ok {
|
||||
e.Add(1)
|
||||
go func(r io.Reader) {
|
||||
defer e.Done()
|
||||
if err := CopyMulty(e.quit, r, plugins.Outputs...); err != nil {
|
||||
log.Println("Error during copy: ", err)
|
||||
e.close()
|
||||
}
|
||||
}(r)
|
||||
go CopyMulty(r, Plugins.Outputs...)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-e.quit:
|
||||
finalize(plugins)
|
||||
case <-stop:
|
||||
finalize()
|
||||
return
|
||||
case <-time.After(100 * time.Millisecond):
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (e *emitter) close() {
|
||||
select {
|
||||
case <-e.quit:
|
||||
default:
|
||||
close(e.quit)
|
||||
}
|
||||
}
|
||||
|
||||
// Close closes all the goroutine and waits for it to finish.
|
||||
func (e *emitter) Close() {
|
||||
e.close()
|
||||
e.Wait()
|
||||
}
|
||||
|
||||
// CopyMulty copies from 1 reader to multiple writers
|
||||
func CopyMulty(stop chan int, src io.Reader, writers ...io.Writer) error {
|
||||
buf := make([]byte, Settings.CopyBufferSize)
|
||||
func CopyMulty(src io.Reader, writers ...io.Writer) (err error) {
|
||||
buf := make([]byte, 5*1024*1024)
|
||||
wIndex := 0
|
||||
modifier := NewHTTPModifier(&Settings.modifierConfig)
|
||||
filteredRequests := make(map[string]time.Time)
|
||||
filteredRequestsLastCleanTime := time.Now()
|
||||
|
||||
i := 0
|
||||
|
||||
for {
|
||||
var nr int
|
||||
nr, err := src.Read(buf)
|
||||
nr, er := src.Read(buf)
|
||||
|
||||
select {
|
||||
case <-stop:
|
||||
return nil
|
||||
default:
|
||||
}
|
||||
|
||||
if err == io.EOF || err == ErrorStopped {
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
_maxN := nr
|
||||
if nr > 500 {
|
||||
_maxN = 500
|
||||
}
|
||||
if nr > 0 && len(buf) > nr {
|
||||
payload := buf[:nr]
|
||||
meta := payloadMeta(payload)
|
||||
if len(meta) < 3 {
|
||||
if Settings.Debug {
|
||||
Debug("[EMITTER] Found malformed record", string(payload[0:_maxN]), nr, "from:", src)
|
||||
}
|
||||
continue
|
||||
}
|
||||
requestID := string(meta[1])
|
||||
|
||||
if nr >= 5*1024*1024 {
|
||||
log.Println("INFO: Large packet... We received ", len(payload), " bytes from ", src)
|
||||
_maxN := nr
|
||||
if nr > 500 {
|
||||
_maxN = 500
|
||||
}
|
||||
|
||||
if Settings.Debug {
|
||||
if Settings.debug {
|
||||
Debug("[EMITTER] input:", string(payload[0:_maxN]), nr, "from:", src)
|
||||
}
|
||||
|
||||
@@ -172,7 +89,7 @@ func CopyMulty(stop chan int, src io.Reader, writers ...io.Writer) error {
|
||||
payload = append(payload[:headSize], body...)
|
||||
}
|
||||
|
||||
if Settings.Debug {
|
||||
if Settings.debug {
|
||||
Debug("[EMITTER] Rewritten input:", len(payload), "First 500 bytes:", string(payload[0:_maxN]))
|
||||
}
|
||||
} else {
|
||||
@@ -183,46 +100,32 @@ func CopyMulty(stop chan int, src io.Reader, writers ...io.Writer) error {
|
||||
}
|
||||
}
|
||||
|
||||
if Settings.PrettifyHTTP {
|
||||
if Settings.prettifyHTTP {
|
||||
payload = prettifyHTTP(payload)
|
||||
if len(payload) == 0 {
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
if Settings.SplitOutput {
|
||||
if Settings.RecognizeTCPSessions {
|
||||
if !PRO {
|
||||
log.Fatal("Detailed TCP sessions work only with PRO license")
|
||||
}
|
||||
hasher := fnv.New32a()
|
||||
// First 20 bytes contain tcp session
|
||||
id := payloadID(payload)
|
||||
hasher.Write(id[:20])
|
||||
if Settings.splitOutput {
|
||||
// Simple round robin
|
||||
writers[wIndex].Write(payload)
|
||||
|
||||
wIndex = int(hasher.Sum32()) % len(writers)
|
||||
writers[wIndex].Write(payload)
|
||||
} else {
|
||||
// Simple round robin
|
||||
if _, err := writers[wIndex].Write(payload); err != nil {
|
||||
return err
|
||||
}
|
||||
wIndex++
|
||||
|
||||
wIndex++
|
||||
|
||||
if wIndex >= len(writers) {
|
||||
wIndex = 0
|
||||
}
|
||||
if wIndex >= len(writers) {
|
||||
wIndex = 0
|
||||
}
|
||||
} else {
|
||||
for _, dst := range writers {
|
||||
if _, err := dst.Write(payload); err != nil {
|
||||
return err
|
||||
}
|
||||
dst.Write(payload)
|
||||
}
|
||||
}
|
||||
} else if nr > 0 {
|
||||
log.Println("WARN: Packet", nr, "bytes is too large to process. Consider increasing --copy-buffer-size")
|
||||
|
||||
}
|
||||
if er == io.EOF {
|
||||
break
|
||||
}
|
||||
if er != nil {
|
||||
err = er
|
||||
break
|
||||
}
|
||||
|
||||
// Run GC on each 1000 request
|
||||
@@ -241,4 +144,6 @@ func CopyMulty(stop chan int, src io.Reader, writers ...io.Writer) error {
|
||||
|
||||
i++
|
||||
}
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
+21
-157
@@ -1,21 +1,13 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"io"
|
||||
"os"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestMain(m *testing.M) {
|
||||
PRO = true
|
||||
code := m.Run()
|
||||
os.Exit(code)
|
||||
}
|
||||
|
||||
func TestEmitter(t *testing.T) {
|
||||
wg := new(sync.WaitGroup)
|
||||
quit := make(chan int)
|
||||
@@ -25,14 +17,10 @@ func TestEmitter(t *testing.T) {
|
||||
wg.Done()
|
||||
})
|
||||
|
||||
plugins := &InOutPlugins{
|
||||
Inputs: []io.Reader{input},
|
||||
Outputs: []io.Writer{output},
|
||||
}
|
||||
plugins.All = append(plugins.All, input, output)
|
||||
Plugins.Inputs = []io.Reader{input}
|
||||
Plugins.Outputs = []io.Writer{output}
|
||||
|
||||
emitter := NewEmitter(quit)
|
||||
go emitter.Start(plugins, Settings.Middleware)
|
||||
go Start(quit)
|
||||
|
||||
for i := 0; i < 1000; i++ {
|
||||
wg.Add(1)
|
||||
@@ -40,7 +28,8 @@ func TestEmitter(t *testing.T) {
|
||||
}
|
||||
|
||||
wg.Wait()
|
||||
emitter.Close()
|
||||
|
||||
close(quit)
|
||||
}
|
||||
|
||||
func TestEmitterFiltered(t *testing.T) {
|
||||
@@ -54,17 +43,12 @@ func TestEmitterFiltered(t *testing.T) {
|
||||
wg.Done()
|
||||
})
|
||||
|
||||
plugins := &InOutPlugins{
|
||||
Inputs: []io.Reader{input},
|
||||
Outputs: []io.Writer{output},
|
||||
}
|
||||
plugins.All = append(plugins.All, input, output)
|
||||
|
||||
Plugins.Inputs = []io.Reader{input}
|
||||
Plugins.Outputs = []io.Writer{output}
|
||||
methods := HTTPMethods{[]byte("GET")}
|
||||
Settings.modifierConfig = HTTPModifierConfig{methods: methods}
|
||||
|
||||
emitter := &emitter{quit: quit}
|
||||
go emitter.Start(plugins, "")
|
||||
go Start(quit)
|
||||
|
||||
wg.Add(2)
|
||||
|
||||
@@ -89,55 +73,12 @@ func TestEmitterFiltered(t *testing.T) {
|
||||
input.EmitBytes(respb)
|
||||
|
||||
wg.Wait()
|
||||
emitter.Close()
|
||||
|
||||
close(quit)
|
||||
|
||||
Settings.modifierConfig = HTTPModifierConfig{}
|
||||
}
|
||||
|
||||
func TestEmitterSplitRoundRobin(t *testing.T) {
|
||||
wg := new(sync.WaitGroup)
|
||||
quit := make(chan int)
|
||||
|
||||
input := NewTestInput()
|
||||
|
||||
var counter1, counter2 int32
|
||||
|
||||
output1 := NewTestOutput(func(data []byte) {
|
||||
atomic.AddInt32(&counter1, 1)
|
||||
wg.Done()
|
||||
})
|
||||
|
||||
output2 := NewTestOutput(func(data []byte) {
|
||||
atomic.AddInt32(&counter2, 1)
|
||||
wg.Done()
|
||||
})
|
||||
|
||||
plugins := &InOutPlugins{
|
||||
Inputs: []io.Reader{input},
|
||||
Outputs: []io.Writer{output1, output2},
|
||||
}
|
||||
|
||||
Settings.SplitOutput = true
|
||||
|
||||
emitter := NewEmitter(quit)
|
||||
go emitter.Start(plugins, Settings.Middleware)
|
||||
|
||||
for i := 0; i < 1000; i++ {
|
||||
wg.Add(1)
|
||||
input.EmitGET()
|
||||
}
|
||||
|
||||
wg.Wait()
|
||||
|
||||
emitter.Close()
|
||||
|
||||
if counter1 == 0 || counter2 == 0 || counter1 != counter2 {
|
||||
t.Errorf("Round robin should split traffic equally: %d vs %d", counter1, counter2)
|
||||
}
|
||||
|
||||
Settings.SplitOutput = false
|
||||
}
|
||||
|
||||
func TestEmitterRoundRobin(t *testing.T) {
|
||||
wg := new(sync.WaitGroup)
|
||||
quit := make(chan int)
|
||||
@@ -156,16 +97,12 @@ func TestEmitterRoundRobin(t *testing.T) {
|
||||
wg.Done()
|
||||
})
|
||||
|
||||
plugins := &InOutPlugins{
|
||||
Inputs: []io.Reader{input},
|
||||
Outputs: []io.Writer{output1, output2},
|
||||
}
|
||||
plugins.All = append(plugins.All, input, output1, output2)
|
||||
Plugins.Inputs = []io.Reader{input}
|
||||
Plugins.Outputs = []io.Writer{output1, output2}
|
||||
|
||||
Settings.SplitOutput = true
|
||||
Settings.splitOutput = true
|
||||
|
||||
emitter := NewEmitter(quit)
|
||||
go emitter.Start(plugins, Settings.Middleware)
|
||||
go Start(quit)
|
||||
|
||||
for i := 0; i < 1000; i++ {
|
||||
wg.Add(1)
|
||||
@@ -173,83 +110,14 @@ func TestEmitterRoundRobin(t *testing.T) {
|
||||
}
|
||||
|
||||
wg.Wait()
|
||||
emitter.Close()
|
||||
|
||||
close(quit)
|
||||
|
||||
if counter1 == 0 || counter2 == 0 {
|
||||
t.Errorf("Round robin should split traffic equally: %d vs %d", counter1, counter2)
|
||||
}
|
||||
|
||||
Settings.SplitOutput = false
|
||||
}
|
||||
|
||||
func TestEmitterSplitSession(t *testing.T) {
|
||||
wg1 := new(sync.WaitGroup)
|
||||
wg2 := new(sync.WaitGroup)
|
||||
wg1.Add(1000)
|
||||
wg2.Add(1000)
|
||||
|
||||
// Base uuids, only 1 letter changed
|
||||
uuid1 := []byte("1234567890123456789a0000")
|
||||
uuid2 := []byte("1234567890123456789d0000")
|
||||
|
||||
quit := make(chan int)
|
||||
|
||||
input := NewTestInput()
|
||||
input.skipHeader = true
|
||||
|
||||
var counter1, counter2 int32
|
||||
|
||||
output1 := NewTestOutput(func(data []byte) {
|
||||
atomic.AddInt32(&counter1, 1)
|
||||
if !bytes.Equal(uuid1[:20], payloadID(data)[:20]) {
|
||||
t.Errorf("All tcp sessions should have same id")
|
||||
}
|
||||
|
||||
wg1.Done()
|
||||
})
|
||||
|
||||
output2 := NewTestOutput(func(data []byte) {
|
||||
atomic.AddInt32(&counter2, 1)
|
||||
if !bytes.Equal(uuid2[:20], payloadID(data)[:20]) {
|
||||
t.Errorf("All tcp sessions should have same id")
|
||||
}
|
||||
|
||||
wg2.Done()
|
||||
})
|
||||
|
||||
plugins := &InOutPlugins{
|
||||
Inputs: []io.Reader{input},
|
||||
Outputs: []io.Writer{output1, output2},
|
||||
}
|
||||
|
||||
Settings.SplitOutput = true
|
||||
Settings.RecognizeTCPSessions = true
|
||||
|
||||
emitter := NewEmitter(quit)
|
||||
go emitter.Start(plugins, Settings.Middleware)
|
||||
|
||||
for i := 0; i < 1000; i++ {
|
||||
// Keep session but randomize ACK
|
||||
copy(uuid1[20:], randByte(4))
|
||||
input.EmitBytes([]byte("1 " + string(uuid1) + " 1\n" + "GET / HTTP/1.1\r\n\r\n"))
|
||||
}
|
||||
|
||||
for i := 0; i < 1000; i++ {
|
||||
// Keep session but randomize ACK
|
||||
copy(uuid2[20:], randByte(4))
|
||||
input.EmitBytes([]byte("1 " + string(uuid2) + " 1\n" + "GET / HTTP/1.1\r\n\r\n"))
|
||||
}
|
||||
|
||||
wg1.Wait()
|
||||
wg2.Wait()
|
||||
|
||||
if counter1 != 1000 || counter2 != 1000 {
|
||||
t.Errorf("Round robin should split traffic equally: %d vs %d", counter1, counter2)
|
||||
}
|
||||
|
||||
Settings.SplitOutput = false
|
||||
Settings.RecognizeTCPSessions = false
|
||||
emitter.Close()
|
||||
Settings.splitOutput = false
|
||||
}
|
||||
|
||||
func BenchmarkEmitter(b *testing.B) {
|
||||
@@ -262,14 +130,10 @@ func BenchmarkEmitter(b *testing.B) {
|
||||
wg.Done()
|
||||
})
|
||||
|
||||
plugins := &InOutPlugins{
|
||||
Inputs: []io.Reader{input},
|
||||
Outputs: []io.Writer{output},
|
||||
}
|
||||
plugins.All = append(plugins.All, input, output)
|
||||
Plugins.Inputs = []io.Reader{input}
|
||||
Plugins.Outputs = []io.Writer{output}
|
||||
|
||||
emitter := NewEmitter(quit)
|
||||
go emitter.Start(plugins, Settings.Middleware)
|
||||
go Start(quit)
|
||||
|
||||
b.ResetTimer()
|
||||
|
||||
@@ -279,5 +143,5 @@ func BenchmarkEmitter(b *testing.B) {
|
||||
}
|
||||
|
||||
wg.Wait()
|
||||
emitter.Close()
|
||||
close(quit)
|
||||
}
|
||||
|
||||
@@ -1,57 +0,0 @@
|
||||
#!/usr/bin/env node
|
||||
const readline = require("readline");
|
||||
const StringDecoder = require("string_decoder").StringDecoder
|
||||
|
||||
const rl = readline.createInterface({
|
||||
input: process.stdin
|
||||
});
|
||||
|
||||
var ignoreIds = new Set();
|
||||
var ignoreAddresses = "/api";
|
||||
const decoder = new StringDecoder("utf8");
|
||||
|
||||
function convertHexString(hex) {
|
||||
var bytes = [];
|
||||
for (var i = 0; i < hex.length - 1; i += 2) {
|
||||
bytes.push(parseInt(hex.substr(i, 2), 16));
|
||||
}
|
||||
return decoder.write(Buffer.from(bytes));
|
||||
}
|
||||
|
||||
function log(output) {
|
||||
console.error("===================");
|
||||
console.error(output);
|
||||
}
|
||||
|
||||
function shouldOutputLine(request) {
|
||||
const components = request.split("\n");
|
||||
const header = components[0].split(" ");
|
||||
const type = parseInt(header[0]);
|
||||
const tag = header[1];
|
||||
|
||||
if (type === 3) {
|
||||
return true;
|
||||
}
|
||||
if (type === 1) {
|
||||
// Check if it's oauth
|
||||
const endpoint = components[1].split(" ")[1];
|
||||
if (!endpoint.startsWith(ignoreAddresses)) {
|
||||
ignoreIds.add(tag);
|
||||
return false;
|
||||
}
|
||||
} else if (type === 2) {
|
||||
if (ignoreIds.has(tag)) {
|
||||
ignoreIds.delete(tag);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
rl.on("line", (input) => {
|
||||
const str = convertHexString(input);
|
||||
console.log(input);
|
||||
if (shouldOutputLine(str)) {
|
||||
log(str);
|
||||
}
|
||||
});
|
||||
@@ -1,76 +0,0 @@
|
||||
#! /usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
import sys
|
||||
import fileinput
|
||||
import binascii
|
||||
|
||||
# Used to find end of the Headers section
|
||||
EMPTY_LINE = b'\r\n\r\n'
|
||||
|
||||
|
||||
def log(msg):
|
||||
"""
|
||||
Logging to STDERR as STDOUT and STDIN used for data transfer
|
||||
@type msg: str or byte string
|
||||
@param msg: Message to log to STDERR
|
||||
"""
|
||||
try:
|
||||
msg = str(msg) + '\n'
|
||||
except:
|
||||
pass
|
||||
sys.stderr.write(msg)
|
||||
sys.stderr.flush()
|
||||
|
||||
|
||||
def find_end_of_headers(byte_data):
|
||||
"""
|
||||
Finds where the header portion ends and the content portion begins.
|
||||
@type byte_data: str or byte string
|
||||
@param byte_data: Hex decoded req or resp string
|
||||
"""
|
||||
return byte_data.index(EMPTY_LINE) + 4
|
||||
|
||||
|
||||
def process_stdin():
|
||||
"""
|
||||
Process STDIN and output to STDOUT
|
||||
"""
|
||||
for raw_line in fileinput.input():
|
||||
|
||||
line = raw_line.rstrip()
|
||||
|
||||
# Decode base64 encoded line
|
||||
decoded = bytes.fromhex(line)
|
||||
|
||||
# Split into metadata and payload, the payload is headers + body
|
||||
(raw_metadata, payload) = decoded.split(b'\n', 1)
|
||||
|
||||
# Split into headers and payload
|
||||
headers_pos = find_end_of_headers(payload)
|
||||
raw_headers = payload[:headers_pos]
|
||||
raw_content = payload[headers_pos:]
|
||||
|
||||
log('===================================')
|
||||
request_type_id = int(raw_metadata.split(b' ')[0])
|
||||
log('Request type: {}'.format({
|
||||
1: 'Request',
|
||||
2: 'Original Response',
|
||||
3: 'Replayed Response'
|
||||
}[request_type_id]))
|
||||
log('===================================')
|
||||
|
||||
log('Original data:')
|
||||
log(line)
|
||||
|
||||
log('Decoded request:')
|
||||
log(decoded)
|
||||
|
||||
encoded = binascii.hexlify(raw_metadata + b'\n' + raw_headers + raw_content).decode('ascii')
|
||||
log('Encoded data:')
|
||||
log(encoded)
|
||||
|
||||
sys.stdout.write(encoded + '\n')
|
||||
|
||||
if __name__ == '__main__':
|
||||
process_stdin()
|
||||
@@ -1,12 +1,12 @@
|
||||
#!/usr/bin/env ruby
|
||||
# encoding: utf-8
|
||||
while data = STDIN.gets # continuously read line from STDIN
|
||||
while data = STDIN.gets # continiously read line from STDIN
|
||||
next unless data
|
||||
data = data.chomp # remove end of line symbol
|
||||
|
||||
decoded = [data].pack("H*") # decode base64 encoded request
|
||||
|
||||
# decoded value is raw HTTP payload, example:
|
||||
# dedoded value is raw HTTP payload, example:
|
||||
#
|
||||
# POST /post HTTP/1.1
|
||||
# Content-Length: 7
|
||||
|
||||
@@ -24,9 +24,8 @@ import (
|
||||
"bytes"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"github.com/buger/gor/proto"
|
||||
"os"
|
||||
|
||||
"github.com/buger/goreplay/proto"
|
||||
)
|
||||
|
||||
// requestID -> originalToken
|
||||
|
||||
@@ -1,22 +0,0 @@
|
||||
module github.com/buger/goreplay
|
||||
|
||||
go 1.14
|
||||
|
||||
require (
|
||||
github.com/Shopify/sarama v1.26.4
|
||||
github.com/araddon/gou v0.0.0-20190110011759-c797efecbb61 // indirect
|
||||
github.com/aws/aws-sdk-go v1.32.7
|
||||
github.com/bitly/go-hostpool v0.1.0 // indirect
|
||||
github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 // indirect
|
||||
github.com/filecoin-project/go-fil-markets v0.3.2 // indirect
|
||||
github.com/fsnotify/fsnotify v1.4.7
|
||||
github.com/google/gopacket v1.1.17
|
||||
github.com/klauspost/compress v1.10.10 // indirect
|
||||
github.com/mattbaird/elastigo v0.0.0-20170123220020-2fe47fd29e4b
|
||||
github.com/pierrec/lz4 v2.5.2+incompatible // indirect
|
||||
github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0 // indirect
|
||||
github.com/spf13/pflag v1.0.3
|
||||
github.com/spf13/viper v1.7.0
|
||||
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 // indirect
|
||||
golang.org/x/net v0.0.0-20200602114024-627f9648deb9 // indirect
|
||||
)
|
||||
@@ -1,839 +0,0 @@
|
||||
cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
|
||||
cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
|
||||
cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=
|
||||
cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU=
|
||||
cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY=
|
||||
cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc=
|
||||
cloud.google.com/go v0.46.3 h1:AVXDdKsrtX33oR9fbCMu/+c1o8Ofjq6Ku/MInaLVg5Y=
|
||||
cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0=
|
||||
cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
|
||||
cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
|
||||
cloud.google.com/go/firestore v1.1.0 h1:9x7Bx0A9R5/M9jibeJeZWqjeVEIxYW9fZYqB9a70/bY=
|
||||
cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk=
|
||||
cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I=
|
||||
cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw=
|
||||
dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
|
||||
github.com/AndreasBriese/bbloom v0.0.0-20180913140656-343706a395b7/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8=
|
||||
github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8=
|
||||
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
|
||||
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
|
||||
github.com/Kubuxu/go-os-helper v0.0.1/go.mod h1:N8B+I7vPCT80IcP58r50u4+gEEcsZETFUpAzWW2ep1Y=
|
||||
github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
|
||||
github.com/Shopify/sarama v1.26.4 h1:+17TxUq/PJEAfZAll0T7XJjSgQWCpaQSoki/x5yN8o8=
|
||||
github.com/Shopify/sarama v1.26.4/go.mod h1:NbSGBSSndYaIhRcBtY9V0U7AyH+x71bG668AuWys/yU=
|
||||
github.com/Shopify/toxiproxy v2.1.4+incompatible h1:TKdv8HiTLgE5wdJuEML90aBgNWsokNbMijUGhmcoBJc=
|
||||
github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI=
|
||||
github.com/Stebalien/go-bitfield v0.0.1/go.mod h1:GNjFpasyUVkHMsfEOk8EFLJ9syQ6SI+XWrX9Wf2XH0s=
|
||||
github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII=
|
||||
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
|
||||
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
|
||||
github.com/araddon/gou v0.0.0-20190110011759-c797efecbb61 h1:Xz25cuW4REGC5W5UtpMU3QItMIImag615HiQcRbxqKQ=
|
||||
github.com/araddon/gou v0.0.0-20190110011759-c797efecbb61/go.mod h1:ikc1XA58M+Rx7SEbf0bLJCfBkwayZ8T5jBo5FXK8Uz8=
|
||||
github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o=
|
||||
github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
|
||||
github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da h1:8GUt8eRujhVEGZFFEjBj46YV4rDjvGrNxb0KMWYkL2I=
|
||||
github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY=
|
||||
github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
|
||||
github.com/aws/aws-sdk-go v1.32.7 h1:H4VgdCSF1cHw0VD8zGc98T1bGdACoLkh/vK2L6wgOUU=
|
||||
github.com/aws/aws-sdk-go v1.32.7/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0=
|
||||
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
|
||||
github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
|
||||
github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
|
||||
github.com/bitly/go-hostpool v0.1.0 h1:XKmsF6k5el6xHG3WPJ8U0Ku/ye7njX7W81Ng7O2ioR0=
|
||||
github.com/bitly/go-hostpool v0.1.0/go.mod h1:4gOCgp6+NZnVqlKyZ/iBZFTAJKembaVENUpMkpg42fw=
|
||||
github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c h1:+0HFd5KSZ/mm3JmhmrDukiId5iR6w4+BdFtfSy4yWIc=
|
||||
github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84=
|
||||
github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 h1:DDGfHa7BWjL4YnC6+E63dPcxHo2sUxDIu8g3QgEJdRY=
|
||||
github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4=
|
||||
github.com/btcsuite/btcd v0.0.0-20190213025234-306aecffea32/go.mod h1:DrZx5ec/dmnfpw9KyYoQyYo7d0KEvTkk/5M/vbZjAr8=
|
||||
github.com/btcsuite/btcd v0.0.0-20190523000118-16327141da8c/go.mod h1:3J08xEfcugPacsc34/LKRU2yO7YmuT8yt28J8k2+rrI=
|
||||
github.com/btcsuite/btcd v0.0.0-20190605094302-a0d1e3e36d50/go.mod h1:3J08xEfcugPacsc34/LKRU2yO7YmuT8yt28J8k2+rrI=
|
||||
github.com/btcsuite/btcd v0.0.0-20190824003749-130ea5bddde3/go.mod h1:3J08xEfcugPacsc34/LKRU2yO7YmuT8yt28J8k2+rrI=
|
||||
github.com/btcsuite/btcd v0.20.1-beta/go.mod h1:wVuoA8VJLEcwgqHBwHmzLRazpKxTv13Px/pDuV7OomQ=
|
||||
github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f/go.mod h1:TdznJufoqS23FtqVCzL0ZqgP5MqXbb4fg/WgDys70nA=
|
||||
github.com/btcsuite/btcutil v0.0.0-20190207003914-4c204d697803/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg=
|
||||
github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg=
|
||||
github.com/btcsuite/go-socks v0.0.0-20170105172521-4720035b7bfd/go.mod h1:HHNXQzUsZCxOoE+CPiyCTO6x34Zs86zZUiwtpXoGdtg=
|
||||
github.com/btcsuite/goleveldb v0.0.0-20160330041536-7834afc9e8cd/go.mod h1:F+uVaaLLH7j4eDXPRvw78tMflu7Ie2bzYOH4Y8rRKBY=
|
||||
github.com/btcsuite/snappy-go v0.0.0-20151229074030-0bdef8d06723/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg3lh6TiUghc=
|
||||
github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtEyQwv5/p4Mg4C0fgbePVuGr935/5ddU9Z3TmDRY=
|
||||
github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs=
|
||||
github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
|
||||
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
|
||||
github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
|
||||
github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
|
||||
github.com/coreos/etcd v3.3.13+incompatible h1:8F3hqu9fGYLBifCmRCJsicFqDx/D68Rt3q1JMazcgBQ=
|
||||
github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
|
||||
github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk=
|
||||
github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
|
||||
github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM=
|
||||
github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
|
||||
github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
|
||||
github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
|
||||
github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE=
|
||||
github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
|
||||
github.com/crackcomm/go-gitignore v0.0.0-20170627025303-887ab5e44cc3/go.mod h1:p1d6YEZWvFzEh4KLyvBcVSnrfNDDvK2zfK/4x2v/4pE=
|
||||
github.com/cskr/pubsub v1.0.2/go.mod h1:/8MzYXk/NJAz782G8RPkFzXTZVu63VotefPnR9TIRis=
|
||||
github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davidlazar/go-crypto v0.0.0-20170701192655-dcfb0a7ac018/go.mod h1:rQYf4tfk5sSwFsnDg3qYaBxSjsD9S8+59vW0dKUgme4=
|
||||
github.com/detailyang/go-fallocate v0.0.0-20180908115635-432fa640bd2e/go.mod h1:3ZQK6DMPSz/QZ73jlWxBtUhNA8xZx7LzUFSq/OfP8vk=
|
||||
github.com/dgraph-io/badger v1.5.5-0.20190226225317-8115aed38f8f/go.mod h1:VZxzAIRPHRVNRKRo6AXrX9BJegn6il06VMTZVJYCIjQ=
|
||||
github.com/dgraph-io/badger v1.6.0-rc1/go.mod h1:zwt7syl517jmP8s94KqSxTlM6IMsdhYy6psNgSztDR4=
|
||||
github.com/dgraph-io/badger v1.6.0/go.mod h1:zwt7syl517jmP8s94KqSxTlM6IMsdhYy6psNgSztDR4=
|
||||
github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
|
||||
github.com/dgryski/go-farm v0.0.0-20190104051053-3adb47b1fb0f/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
|
||||
github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
|
||||
github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
|
||||
github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
|
||||
github.com/eapache/go-resiliency v1.2.0 h1:v7g92e/KSN71Rq7vSThKaWIq68fL4YHvWyiUKorFR1Q=
|
||||
github.com/eapache/go-resiliency v1.2.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs=
|
||||
github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 h1:YEetp8/yCZMuEPMUDHG0CW/brkkEp8mzqk2+ODEitlw=
|
||||
github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU=
|
||||
github.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc=
|
||||
github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I=
|
||||
github.com/elastic/go-sysinfo v1.3.0/go.mod h1:i1ZYdU10oLNfRzq4vq62BEwD2fH8KaWh6eh0ikPT9F0=
|
||||
github.com/elastic/go-windows v1.0.0/go.mod h1:TsU0Nrp7/y3+VwE82FoZF8gC/XFg/Elz6CcloAxnPgU=
|
||||
github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
|
||||
github.com/fatih/color v1.8.0/go.mod h1:3l45GVGkyrnYNl9HoIjnp2NnNWvh6hLAqD8yTfGjnw8=
|
||||
github.com/filecoin-project/filecoin-ffi v0.0.0-20200326153646-e899cc1dd072/go.mod h1:PtH9YP0rURHUKHrKeEBeWg/BqIBMQOz8wtlXlVGREBE=
|
||||
github.com/filecoin-project/go-address v0.0.0-20200107215422-da8eea2842b5/go.mod h1:SAOwJoakQ8EPjwNIsiakIQKsoKdkcbx8U3IapgCg9R0=
|
||||
github.com/filecoin-project/go-address v0.0.2-0.20200218010043-eb9bb40ed5be/go.mod h1:SAOwJoakQ8EPjwNIsiakIQKsoKdkcbx8U3IapgCg9R0=
|
||||
github.com/filecoin-project/go-amt-ipld/v2 v2.0.1-0.20200131012142-05d80eeccc5e/go.mod h1:boRtQhzmxNocrMxOXo1NYn4oUc1NGvR8tEa79wApNXg=
|
||||
github.com/filecoin-project/go-amt-ipld/v2 v2.0.1-0.20200424220931-6263827e49f2/go.mod h1:boRtQhzmxNocrMxOXo1NYn4oUc1NGvR8tEa79wApNXg=
|
||||
github.com/filecoin-project/go-bitfield v0.0.0-20200416002808-b3ee67ec9060/go.mod h1:iodsLxOFZnqKtjj2zkgqzoGNrv6vUqj69AT/J8DKXEw=
|
||||
github.com/filecoin-project/go-bitfield v0.0.1/go.mod h1:Ry9/iUlWSyjPUzlAvdnfy4Gtvrq4kWmWDztCU1yEgJY=
|
||||
github.com/filecoin-project/go-bitfield v0.0.2-0.20200518150651-562fdb554b6e/go.mod h1:Ry9/iUlWSyjPUzlAvdnfy4Gtvrq4kWmWDztCU1yEgJY=
|
||||
github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2/go.mod h1:pqTiPHobNkOVM5thSRsHYjyQfq7O5QSCMhvuu9JoDlg=
|
||||
github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03/go.mod h1:+viYnvGtUTgJRdy6oaeF4MTFKAfatX071MPDPBL11EQ=
|
||||
github.com/filecoin-project/go-data-transfer v0.3.0/go.mod h1:cONglGP4s/d+IUQw5mWZrQK+FQATQxr3AXzi4dRh0l4=
|
||||
github.com/filecoin-project/go-fil-commcid v0.0.0-20200208005934-2b8bd03caca5/go.mod h1:JbkIgFF/Z9BDlvrJO1FuKkaWsH673/UdFaiVS6uIHlA=
|
||||
github.com/filecoin-project/go-fil-markets v0.3.2 h1:fvNgdTTIVtckBu61wxbKYSMJzedoFFIKYJagiCDFCiM=
|
||||
github.com/filecoin-project/go-fil-markets v0.3.2/go.mod h1:e/IofcotbwH7ftgeK+TjjdjFsrCDWrh5vvnr7k1OSH8=
|
||||
github.com/filecoin-project/go-padreader v0.0.0-20200210211231-548257017ca6/go.mod h1:0HgYnrkeSU4lu1p+LEOeDpFsNBssa0OGGriWdA4hvaE=
|
||||
github.com/filecoin-project/go-paramfetch v0.0.1/go.mod h1:fZzmf4tftbwf9S37XRifoJlz7nCjRdIrMGLR07dKLCc=
|
||||
github.com/filecoin-project/go-statemachine v0.0.0-20200619205156-c7bf525c06ef/go.mod h1:FGwQgZAt2Gh5mjlwJUlVB62JeYdo+if0xWxSEfBD9ig=
|
||||
github.com/filecoin-project/go-statestore v0.1.0/go.mod h1:LFc9hD+fRxPqiHiaqUEZOinUJB4WARkRfNl10O7kTnI=
|
||||
github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b/go.mod h1:Q0GQOBtKf1oE10eSXSlhN45kDBdGvEcVOqMiffqX+N8=
|
||||
github.com/filecoin-project/sector-storage v0.0.0-20200615154852-728a47ab99d6/go.mod h1:M59QnAeA/oV+Z8oHFLoNpGMv0LZ8Rll+vHVXX7GirPM=
|
||||
github.com/filecoin-project/specs-actors v0.0.0-20200210130641-2d1fbd8672cf/go.mod h1:xtDZUB6pe4Pksa/bAJbJ693OilaC5Wbot9jMhLm3cZA=
|
||||
github.com/filecoin-project/specs-actors v0.0.0-20200226200336-94c9b92b2775/go.mod h1:0HAWYrvajFHDgRaKbF0rl+IybVLZL5z4gQ8koCMPhoU=
|
||||
github.com/filecoin-project/specs-actors v0.3.0/go.mod h1:nQYnFbQ7Y0bHZyq6HDEuVlCPR+U3z5Q3wMOQ+2aiV+Y=
|
||||
github.com/filecoin-project/specs-actors v0.6.0/go.mod h1:dRdy3cURykh2R8O/DKqy8olScl70rmIS7GrB4hB1IDY=
|
||||
github.com/filecoin-project/specs-storage v0.1.0/go.mod h1:Pr5ntAaxsh+sLG/LYiL4tKzvA83Vk5vLODYhfNwOg7k=
|
||||
github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw=
|
||||
github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g=
|
||||
github.com/frankban/quicktest v1.7.2 h1:2QxQoC1TS09S7fhCPsrvqYdvP1H5M1P1ih5ABm3BTYk=
|
||||
github.com/frankban/quicktest v1.7.2/go.mod h1:jaStnuzAqU1AJdCO0l53JDCJrVDKcS03DbaAcR7Ks/o=
|
||||
github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
|
||||
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
|
||||
github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
|
||||
github.com/go-check/check v0.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclKsC9YodN5RgxqK/VD9HM9JsCSh7rNhMZE98=
|
||||
github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
|
||||
github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
|
||||
github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
|
||||
github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
|
||||
github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
|
||||
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
|
||||
github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
|
||||
github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
|
||||
github.com/gogo/protobuf v1.3.0/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
|
||||
github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
|
||||
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
|
||||
github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
|
||||
github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
|
||||
github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
|
||||
github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
|
||||
github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y=
|
||||
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
|
||||
github.com/golang/protobuf v1.3.0/go.mod h1:Qd/q+1AKNOZr9uGQzbzCmRO6sUih6GTPZv6a1/R87v0=
|
||||
github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
|
||||
github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs=
|
||||
github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
|
||||
github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
|
||||
github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4=
|
||||
github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
|
||||
github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
|
||||
github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
|
||||
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
|
||||
github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
|
||||
github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
|
||||
github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4=
|
||||
github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/gopacket v1.1.17 h1:rMrlX2ZY2UbvT+sdz3+6J+pp2z+msCq9MxTU6ymxbBY=
|
||||
github.com/google/gopacket v1.1.17/go.mod h1:UdDNZ1OO62aGYVnPhxT1U6aI7ukYtA/kB8vaU0diBUM=
|
||||
github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
|
||||
github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
|
||||
github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
|
||||
github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
|
||||
github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
|
||||
github.com/googleapis/gax-go/v2 v2.0.5 h1:sjZBwGj9Jlw33ImPtvFviGYvseOtDM7hkSKB7+Tv3SM=
|
||||
github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
|
||||
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8=
|
||||
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
|
||||
github.com/gopherjs/gopherjs v0.0.0-20190430165422-3e4dfb77656c/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
|
||||
github.com/gopherjs/gopherjs v0.0.0-20190812055157-5d271430af9f/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
|
||||
github.com/gorilla/mux v1.7.4/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
|
||||
github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
|
||||
github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
|
||||
github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
|
||||
github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
|
||||
github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
|
||||
github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
|
||||
github.com/gxed/hashland/keccakpg v0.0.1/go.mod h1:kRzw3HkwxFU1mpmPP8v1WyQzwdGfmKFJ6tItnhQ67kU=
|
||||
github.com/gxed/hashland/murmur3 v0.0.1/go.mod h1:KjXop02n4/ckmZSnY2+HKcLud/tcmvhST0bie/0lS48=
|
||||
github.com/hannahhoward/cbor-gen-for v0.0.0-20191218204337-9ab7b1bcc099/go.mod h1:WVPCl0HO/0RAL5+vBH2GMxBomlxBF70MAS78+Lu1//k=
|
||||
github.com/hannahhoward/go-pubsub v0.0.0-20200423002714-8d62886cc36e/go.mod h1:I8h3MITA53gN9OnWGCgaMa0JWVRdXthWw4M3CPM54OY=
|
||||
github.com/hashicorp/consul/api v1.1.0 h1:BNQPM9ytxj6jbjjdRPioQ94T6YXriSopn0i8COv6SRA=
|
||||
github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q=
|
||||
github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8=
|
||||
github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
|
||||
github.com/hashicorp/go-cleanhttp v0.5.1 h1:dH3aiDG9Jvb5r5+bYHsikaOUIpcM0xvgMXVoDkXMzJM=
|
||||
github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
|
||||
github.com/hashicorp/go-immutable-radix v1.0.0 h1:AKDB1HM5PWEA7i4nhcpwOrO2byshxBjXVn/J/3+z5/0=
|
||||
github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60=
|
||||
github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM=
|
||||
github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk=
|
||||
github.com/hashicorp/go-rootcerts v1.0.0 h1:Rqb66Oo1X/eSV1x66xbDccZjhJigjg0+e82kpwzSwCI=
|
||||
github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU=
|
||||
github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU=
|
||||
github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4=
|
||||
github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
|
||||
github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
|
||||
github.com/hashicorp/go-uuid v1.0.2 h1:cfejS+Tpcp13yd5nYHWDI6qVCny6wyX2Mt5SGur2IGE=
|
||||
github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
|
||||
github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90=
|
||||
github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
|
||||
github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU=
|
||||
github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
|
||||
github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
|
||||
github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
|
||||
github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
|
||||
github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64=
|
||||
github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ=
|
||||
github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I=
|
||||
github.com/hashicorp/serf v0.8.2 h1:YZ7UKsJv+hKjqGVUUbtE3HNj79Eln2oQ75tniF6iPt0=
|
||||
github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc=
|
||||
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
|
||||
github.com/huin/goupnp v1.0.0/go.mod h1:n9v9KO1tAxYH82qOn+UTIFQDmx5n1Zxd/ClZDMX7Bnc=
|
||||
github.com/huin/goutil v0.0.0-20170803182201-1ca381bf3150/go.mod h1:PpLOETDnJ0o3iZrZfqZzyLl6l7F3c6L1oWn7OICBi6o=
|
||||
github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
|
||||
github.com/ipfs/bbloom v0.0.1/go.mod h1:oqo8CVWsJFMOZqTglBG4wydCE4IQA/G2/SEofB0rjUI=
|
||||
github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0=
|
||||
github.com/ipfs/go-bitswap v0.1.0/go.mod h1:FFJEf18E9izuCqUtHxbWEvq+reg7o4CW5wSAE1wsxj0=
|
||||
github.com/ipfs/go-bitswap v0.1.2/go.mod h1:qxSWS4NXGs7jQ6zQvoPY3+NmOfHHG47mhkiLzBpJQIs=
|
||||
github.com/ipfs/go-bitswap v0.1.8/go.mod h1:TOWoxllhccevbWFUR2N7B1MTSVVge1s6XSMiCSA4MzM=
|
||||
github.com/ipfs/go-block-format v0.0.1/go.mod h1:DK/YYcsSUIVAFNwo/KZCdIIbpN0ROH/baNLgayt4pFc=
|
||||
github.com/ipfs/go-block-format v0.0.2/go.mod h1:AWR46JfpcObNfg3ok2JHDUfdiHRgWhJgCQF+KIgOPJY=
|
||||
github.com/ipfs/go-blockservice v0.1.0/go.mod h1:hzmMScl1kXHg3M2BjTymbVPjv627N7sYcvYaKbop39M=
|
||||
github.com/ipfs/go-blockservice v0.1.3/go.mod h1:OTZhFpkgY48kNzbgyvcexW9cHrpjBYIjSR0KoDOFOLU=
|
||||
github.com/ipfs/go-cid v0.0.1/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM=
|
||||
github.com/ipfs/go-cid v0.0.2/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM=
|
||||
github.com/ipfs/go-cid v0.0.3/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM=
|
||||
github.com/ipfs/go-cid v0.0.4-0.20191112011718-79e75dffeb10/go.mod h1:/BYOuUoxkE+0f6tGzlzMvycuN+5l35VOR4Bpg2sCmds=
|
||||
github.com/ipfs/go-cid v0.0.4/go.mod h1:4LLaPOQwmk5z9LBgQnpkivrx8BJjUyGwTXCd5Xfj6+M=
|
||||
github.com/ipfs/go-cid v0.0.5/go.mod h1:plgt+Y5MnOey4vO4UlUazGqdbEXuFYitED67FexhXog=
|
||||
github.com/ipfs/go-datastore v0.0.1/go.mod h1:d4KVXhMt913cLBEI/PXAy6ko+W7e9AhyAKBGh803qeE=
|
||||
github.com/ipfs/go-datastore v0.0.5/go.mod h1:d4KVXhMt913cLBEI/PXAy6ko+W7e9AhyAKBGh803qeE=
|
||||
github.com/ipfs/go-datastore v0.1.0/go.mod h1:d4KVXhMt913cLBEI/PXAy6ko+W7e9AhyAKBGh803qeE=
|
||||
github.com/ipfs/go-datastore v0.1.1/go.mod h1:w38XXW9kVFNp57Zj5knbKWM2T+KOZCGDRVNdgPHtbHw=
|
||||
github.com/ipfs/go-datastore v0.3.1/go.mod h1:w38XXW9kVFNp57Zj5knbKWM2T+KOZCGDRVNdgPHtbHw=
|
||||
github.com/ipfs/go-datastore v0.4.0/go.mod h1:SX/xMIKoCszPqp+z9JhPYCmoOoXTvaa13XEbGtsFUhA=
|
||||
github.com/ipfs/go-datastore v0.4.1/go.mod h1:SX/xMIKoCszPqp+z9JhPYCmoOoXTvaa13XEbGtsFUhA=
|
||||
github.com/ipfs/go-datastore v0.4.4/go.mod h1:SX/xMIKoCszPqp+z9JhPYCmoOoXTvaa13XEbGtsFUhA=
|
||||
github.com/ipfs/go-detect-race v0.0.1/go.mod h1:8BNT7shDZPo99Q74BpGMK+4D8Mn4j46UU0LZ723meps=
|
||||
github.com/ipfs/go-ds-badger v0.0.2/go.mod h1:Y3QpeSFWQf6MopLTiZD+VT6IC1yZqaGmjvRcKeSGij8=
|
||||
github.com/ipfs/go-ds-badger v0.0.5/go.mod h1:g5AuuCGmr7efyzQhLL8MzwqcauPojGPUaHzfGTzuE3s=
|
||||
github.com/ipfs/go-ds-badger v0.2.1/go.mod h1:Tx7l3aTph3FMFrRS838dcSJh+jjA7cX9DrGVwx/NOwE=
|
||||
github.com/ipfs/go-ds-leveldb v0.0.1/go.mod h1:feO8V3kubwsEF22n0YRQCffeb79OOYIykR4L04tMOYc=
|
||||
github.com/ipfs/go-ds-leveldb v0.4.1/go.mod h1:jpbku/YqBSsBc1qgME8BkWS4AxzF2cEu1Ii2r79Hh9s=
|
||||
github.com/ipfs/go-graphsync v0.0.6-0.20200504202014-9d5f2c26a103/go.mod h1:jMXfqIEDFukLPZHqDPp8tJMbHO9Rmeb9CEGevngQbmE=
|
||||
github.com/ipfs/go-hamt-ipld v0.0.15-0.20200131012125-dd88a59d3f2e/go.mod h1:9aQJu/i/TaRDW6jqB5U217dLIDopn50wxLdHXM2CTfE=
|
||||
github.com/ipfs/go-ipfs-blockstore v0.0.1/go.mod h1:d3WClOmRQKFnJ0Jz/jj/zmksX0ma1gROTlovZKBmN08=
|
||||
github.com/ipfs/go-ipfs-blockstore v0.1.0/go.mod h1:5aD0AvHPi7mZc6Ci1WCAhiBQu2IsfTduLl+422H6Rqw=
|
||||
github.com/ipfs/go-ipfs-blockstore v0.1.4/go.mod h1:Jxm3XMVjh6R17WvxFEiyKBLUGr86HgIYJW/D/MwqeYQ=
|
||||
github.com/ipfs/go-ipfs-blockstore v1.0.0/go.mod h1:knLVdhVU9L7CC4T+T4nvGdeUIPAXlnd9zmXfp+9MIjU=
|
||||
github.com/ipfs/go-ipfs-blocksutil v0.0.1/go.mod h1:Yq4M86uIOmxmGPUHv/uI7uKqZNtLb449gwKqXjIsnRk=
|
||||
github.com/ipfs/go-ipfs-chunker v0.0.1/go.mod h1:tWewYK0we3+rMbOh7pPFGDyypCtvGcBFymgY4rSDLAw=
|
||||
github.com/ipfs/go-ipfs-chunker v0.0.5/go.mod h1:jhgdF8vxRHycr00k13FM8Y0E+6BoalYeobXmUyTreP8=
|
||||
github.com/ipfs/go-ipfs-delay v0.0.0-20181109222059-70721b86a9a8/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw=
|
||||
github.com/ipfs/go-ipfs-delay v0.0.1/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw=
|
||||
github.com/ipfs/go-ipfs-ds-help v0.0.1/go.mod h1:gtP9xRaZXqIQRh1HRpp595KbBEdgqWFxefeVKOV8sxo=
|
||||
github.com/ipfs/go-ipfs-ds-help v0.1.1/go.mod h1:SbBafGJuGsPI/QL3j9Fc5YPLeAu+SzOkI0gFwAg+mOs=
|
||||
github.com/ipfs/go-ipfs-ds-help v1.0.0/go.mod h1:ujAbkeIgkKAWtxxNkoZHWLCyk5JpPoKnGyCcsoF6ueE=
|
||||
github.com/ipfs/go-ipfs-exchange-interface v0.0.1/go.mod h1:c8MwfHjtQjPoDyiy9cFquVtVHkO9b9Ob3FG91qJnWCM=
|
||||
github.com/ipfs/go-ipfs-exchange-offline v0.0.1/go.mod h1:WhHSFCVYX36H/anEKQboAzpUws3x7UeEGkzQc3iNkM0=
|
||||
github.com/ipfs/go-ipfs-files v0.0.3/go.mod h1:INEFm0LL2LWXBhNJ2PMIIb2w45hpXgPjNoE7yA8Y1d4=
|
||||
github.com/ipfs/go-ipfs-files v0.0.7/go.mod h1:wiN/jSG8FKyk7N0WyctKSvq3ljIa2NNTiZB55kpTdOs=
|
||||
github.com/ipfs/go-ipfs-files v0.0.8/go.mod h1:wiN/jSG8FKyk7N0WyctKSvq3ljIa2NNTiZB55kpTdOs=
|
||||
github.com/ipfs/go-ipfs-posinfo v0.0.1/go.mod h1:SwyeVP+jCwiDu0C313l/8jg6ZxM0qqtlt2a0vILTc1A=
|
||||
github.com/ipfs/go-ipfs-pq v0.0.1/go.mod h1:LWIqQpqfRG3fNc5XsnIhz/wQ2XXGyugQwls7BgUmUfY=
|
||||
github.com/ipfs/go-ipfs-pq v0.0.2/go.mod h1:LWIqQpqfRG3fNc5XsnIhz/wQ2XXGyugQwls7BgUmUfY=
|
||||
github.com/ipfs/go-ipfs-routing v0.1.0/go.mod h1:hYoUkJLyAUKhF58tysKpids8RNDPO42BVMgK5dNsoqY=
|
||||
github.com/ipfs/go-ipfs-util v0.0.1/go.mod h1:spsl5z8KUnrve+73pOhSVZND1SIxPW5RyBCNzQxlJBc=
|
||||
github.com/ipfs/go-ipld-cbor v0.0.2/go.mod h1:wTBtrQZA3SoFKMVkp6cn6HMRteIB1VsmHA0AQFOn7Nc=
|
||||
github.com/ipfs/go-ipld-cbor v0.0.3/go.mod h1:wTBtrQZA3SoFKMVkp6cn6HMRteIB1VsmHA0AQFOn7Nc=
|
||||
github.com/ipfs/go-ipld-cbor v0.0.4/go.mod h1:BkCduEx3XBCO6t2Sfo5BaHzuok7hbhdMm9Oh8B2Ftq4=
|
||||
github.com/ipfs/go-ipld-format v0.0.1/go.mod h1:kyJtbkDALmFHv3QR6et67i35QzO3S0dCDnkOJhcZkms=
|
||||
github.com/ipfs/go-ipld-format v0.0.2/go.mod h1:4B6+FM2u9OJ9zCV+kSbgFAZlOrv1Hqbf0INGQgiKf9k=
|
||||
github.com/ipfs/go-ipld-format v0.2.0/go.mod h1:3l3C1uKoadTPbeNfrDi+xMInYKlx2Cvg1BuydPSdzQs=
|
||||
github.com/ipfs/go-log v0.0.1/go.mod h1:kL1d2/hzSpI0thNYjiKfjanbVNU+IIGA/WnNESY9leM=
|
||||
github.com/ipfs/go-log v1.0.0/go.mod h1:JO7RzlMK6rA+CIxFMLOuB6Wf5b81GDiKElL7UPSIKjA=
|
||||
github.com/ipfs/go-log v1.0.1/go.mod h1:HuWlQttfN6FWNHRhlY5yMk/lW7evQC0HHGOxEwMRR8I=
|
||||
github.com/ipfs/go-log v1.0.2/go.mod h1:1MNjMxe0u6xvJZgeqbJ8vdo2TKaGwZ1a0Bpza+sr2Sk=
|
||||
github.com/ipfs/go-log v1.0.3/go.mod h1:OsLySYkwIbiSUR/yBTdv1qPtcE4FW3WPWk/ewz9Ru+A=
|
||||
github.com/ipfs/go-log/v2 v2.0.1/go.mod h1:O7P1lJt27vWHhOwQmcFEvlmo49ry2VY2+JfBWFaa9+0=
|
||||
github.com/ipfs/go-log/v2 v2.0.2/go.mod h1:O7P1lJt27vWHhOwQmcFEvlmo49ry2VY2+JfBWFaa9+0=
|
||||
github.com/ipfs/go-log/v2 v2.0.3/go.mod h1:O7P1lJt27vWHhOwQmcFEvlmo49ry2VY2+JfBWFaa9+0=
|
||||
github.com/ipfs/go-merkledag v0.2.3/go.mod h1:SQiXrtSts3KGNmgOzMICy5c0POOpUNQLvB3ClKnBAlk=
|
||||
github.com/ipfs/go-merkledag v0.3.1/go.mod h1:fvkZNNZixVW6cKSZ/JfLlON5OlgTXNdRLz0p6QG/I2M=
|
||||
github.com/ipfs/go-metrics-interface v0.0.1/go.mod h1:6s6euYU4zowdslK0GKHmqaIZ3j/b/tL7HTWtJ4VPgWY=
|
||||
github.com/ipfs/go-peertaskqueue v0.1.0/go.mod h1:Jmk3IyCcfl1W3jTW3YpghSwSEC6IJ3Vzz/jUmWw8Z0U=
|
||||
github.com/ipfs/go-peertaskqueue v0.1.1/go.mod h1:Jmk3IyCcfl1W3jTW3YpghSwSEC6IJ3Vzz/jUmWw8Z0U=
|
||||
github.com/ipfs/go-peertaskqueue v0.2.0/go.mod h1:5/eNrBEbtSKWCG+kQK8K8fGNixoYUnr+P7jivavs9lY=
|
||||
github.com/ipfs/go-unixfs v0.2.4/go.mod h1:SUdisfUjNoSDzzhGVxvCL9QO/nKdwXdr+gbMUdqcbYw=
|
||||
github.com/ipfs/go-verifcid v0.0.1/go.mod h1:5Hrva5KBeIog4A+UpqlaIU+DEstipcJYQQZc0g37pY0=
|
||||
github.com/ipld/go-car v0.1.1-0.20200429200904-c222d793c339/go.mod h1:eajxljm6I8o3LitnFeVEmucwZmz7+yLSiKce9yYMefg=
|
||||
github.com/ipld/go-ipld-prime v0.0.2-0.20200428162820-8b59dc292b8e/go.mod h1:uVIwe/u0H4VdKv3kaN1ck7uCb6yD9cFLS9/ELyXbsw8=
|
||||
github.com/ipld/go-ipld-prime-proto v0.0.0-20200428191222-c1ffdadc01e1/go.mod h1:OAV6xBmuTLsPZ+epzKkPB1e25FHk/vCtyatkdHcArLs=
|
||||
github.com/ipsn/go-secp256k1 v0.0.0-20180726113642-9d62b9f0bc52/go.mod h1:fdg+/X9Gg4AsAIzWpEHwnqd+QY3b7lajxyjE1m4hkq4=
|
||||
github.com/jackpal/gateway v1.0.5/go.mod h1:lTpwd4ACLXmpyiCTRtfiNyVnUmqT9RivzCDQetPfnjA=
|
||||
github.com/jackpal/go-nat-pmp v1.0.1/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc=
|
||||
github.com/jbenet/go-cienv v0.0.0-20150120210510-1bb1476777ec/go.mod h1:rGaEvXB4uRSZMmzKNLoXvTu1sfx+1kv/DojUlPrSZGs=
|
||||
github.com/jbenet/go-cienv v0.1.0/go.mod h1:TqNnHUmJgXau0nCzC7kXWeotg3J9W34CUv5Djy1+FlA=
|
||||
github.com/jbenet/go-random v0.0.0-20190219211222-123a90aedc0c/go.mod h1:sdx1xVM9UuLw1tXnhJWN3piypTUO3vCIHYmG15KE/dU=
|
||||
github.com/jbenet/go-temp-err-catcher v0.0.0-20150120210811-aac704a3f4f2/go.mod h1:8GXXJV31xl8whumTzdZsTt3RnUIiPqzkyf7mxToRCMs=
|
||||
github.com/jbenet/goprocess v0.0.0-20160826012719-b497e2f366b8/go.mod h1:Ly/wlsjFq/qrU3Rar62tu1gASgGw6chQbSh/XgIIXCY=
|
||||
github.com/jbenet/goprocess v0.1.3/go.mod h1:5yspPrukOVuOLORacaBi858NqyClJPQxYZlqdZVfqY4=
|
||||
github.com/jcmturner/gofork v1.0.0 h1:J7uCkflzTEhUZ64xqKnkDxq3kzc96ajM1Gli5ktUem8=
|
||||
github.com/jcmturner/gofork v1.0.0/go.mod h1:MK8+TM0La+2rjBD4jE12Kj1pCCxK7d2LK/UM3ncEo0o=
|
||||
github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
|
||||
github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
|
||||
github.com/jmespath/go-jmespath v0.3.0 h1:OS12ieG61fsCg5+qLJ+SsW9NicxNkg3b25OyT2yCeUc=
|
||||
github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik=
|
||||
github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901/go.mod h1:Z86h9688Y0wesXCyonoVr47MasHilkuLMqGhRZ4Hpak=
|
||||
github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
|
||||
github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ=
|
||||
github.com/json-iterator/go v1.1.6 h1:MrUvLMLTMxbqFJ9kzlvat/rYZqZnW3u4wkLzWTaFwKs=
|
||||
github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
|
||||
github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
|
||||
github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
|
||||
github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
|
||||
github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
|
||||
github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
|
||||
github.com/kami-zh/go-capturer v0.0.0-20171211120116-e492ea43421d/go.mod h1:P2viExyCEfeWGU259JnaQ34Inuec4R38JCyBx2edgD0=
|
||||
github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
|
||||
github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=
|
||||
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
|
||||
github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4=
|
||||
github.com/klauspost/compress v1.9.8 h1:VMAMUUOh+gaxKTMk+zqbjsSjsIcUcL/LF4o63i82QyA=
|
||||
github.com/klauspost/compress v1.9.8/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
|
||||
github.com/klauspost/compress v1.10.10 h1:a/y8CglcM7gLGYmlbP/stPE5sR3hbhFRUjCBfd/0B3I=
|
||||
github.com/klauspost/compress v1.10.10/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
|
||||
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
|
||||
github.com/koron/go-ssdp v0.0.0-20180514024734-4a0ed625a78b/go.mod h1:5Ky9EC2xfoUKUor0Hjgi2BJhCSXJfMOFlmyYrVKGQMk=
|
||||
github.com/koron/go-ssdp v0.0.0-20191105050749-2e1c40ed0b5d/go.mod h1:5Ky9EC2xfoUKUor0Hjgi2BJhCSXJfMOFlmyYrVKGQMk=
|
||||
github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
|
||||
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
|
||||
github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs=
|
||||
github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
|
||||
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
|
||||
github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
|
||||
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
|
||||
github.com/libp2p/go-addr-util v0.0.1/go.mod h1:4ac6O7n9rIAKB1dnd+s8IbbMXkt+oBpzX4/+RACcnlQ=
|
||||
github.com/libp2p/go-buffer-pool v0.0.1/go.mod h1:xtyIz9PMobb13WaxR6Zo1Pd1zXJKYg0a8KiIvDp3TzQ=
|
||||
github.com/libp2p/go-buffer-pool v0.0.2/go.mod h1:MvaB6xw5vOrDl8rYZGLFdKAuk/hRoRZd1Vi32+RXyFM=
|
||||
github.com/libp2p/go-conn-security-multistream v0.1.0/go.mod h1:aw6eD7LOsHEX7+2hJkDxw1MteijaVcI+/eP2/x3J1xc=
|
||||
github.com/libp2p/go-eventbus v0.1.0/go.mod h1:vROgu5cs5T7cv7POWlWxBaVLxfSegC5UGQf8A2eEmx4=
|
||||
github.com/libp2p/go-flow-metrics v0.0.1/go.mod h1:Iv1GH0sG8DtYN3SVJ2eG221wMiNpZxBdp967ls1g+k8=
|
||||
github.com/libp2p/go-flow-metrics v0.0.3/go.mod h1:HeoSNUrOJVK1jEpDqVEiUOIXqhbnS27omG0uWU5slZs=
|
||||
github.com/libp2p/go-libp2p v0.1.0/go.mod h1:6D/2OBauqLUoqcADOJpn9WbKqvaM07tDw68qHM0BxUM=
|
||||
github.com/libp2p/go-libp2p v0.1.1/go.mod h1:I00BRo1UuUSdpuc8Q2mN7yDF/oTUTRAX6JWpTiK9Rp8=
|
||||
github.com/libp2p/go-libp2p v0.6.0/go.mod h1:mfKWI7Soz3ABX+XEBR61lGbg+ewyMtJHVt043oWeqwg=
|
||||
github.com/libp2p/go-libp2p-autonat v0.1.0/go.mod h1:1tLf2yXxiE/oKGtDwPYWTSYG3PtvYlJmg7NeVtPRqH8=
|
||||
github.com/libp2p/go-libp2p-autonat v0.1.1/go.mod h1:OXqkeGOY2xJVWKAGV2inNF5aKN/djNA3fdpCWloIudE=
|
||||
github.com/libp2p/go-libp2p-blankhost v0.1.1/go.mod h1:pf2fvdLJPsC1FsVrNP3DUUvMzUts2dsLLBEpo1vW1ro=
|
||||
github.com/libp2p/go-libp2p-blankhost v0.1.4/go.mod h1:oJF0saYsAXQCSfDq254GMNmLNz6ZTHTOvtF4ZydUvwU=
|
||||
github.com/libp2p/go-libp2p-circuit v0.1.0/go.mod h1:Ahq4cY3V9VJcHcn1SBXjr78AbFkZeIRmfunbA7pmFh8=
|
||||
github.com/libp2p/go-libp2p-circuit v0.1.4/go.mod h1:CY67BrEjKNDhdTk8UgBX1Y/H5c3xkAcs3gnksxY7osU=
|
||||
github.com/libp2p/go-libp2p-core v0.0.1/go.mod h1:g/VxnTZ/1ygHxH3dKok7Vno1VfpvGcGip57wjTU4fco=
|
||||
github.com/libp2p/go-libp2p-core v0.0.2/go.mod h1:9dAcntw/n46XycV4RnlBq3BpgrmyUi9LuoTNdPrbUco=
|
||||
github.com/libp2p/go-libp2p-core v0.0.3/go.mod h1:j+YQMNz9WNSkNezXOsahp9kwZBKBvxLpKD316QWSJXE=
|
||||
github.com/libp2p/go-libp2p-core v0.0.4/go.mod h1:jyuCQP356gzfCFtRKyvAbNkyeuxb7OlyhWZ3nls5d2I=
|
||||
github.com/libp2p/go-libp2p-core v0.2.0/go.mod h1:X0eyB0Gy93v0DZtSYbEM7RnMChm9Uv3j7yRXjO77xSI=
|
||||
github.com/libp2p/go-libp2p-core v0.2.2/go.mod h1:8fcwTbsG2B+lTgRJ1ICZtiM5GWCWZVoVrLaDRvIRng0=
|
||||
github.com/libp2p/go-libp2p-core v0.2.4/go.mod h1:STh4fdfa5vDYr0/SzYYeqnt+E6KfEV5VxfIrm0bcI0g=
|
||||
github.com/libp2p/go-libp2p-core v0.3.0/go.mod h1:ACp3DmS3/N64c2jDzcV429ukDpicbL6+TrrxANBjPGw=
|
||||
github.com/libp2p/go-libp2p-core v0.3.1/go.mod h1:thvWy0hvaSBhnVBaW37BvzgVV68OUhgJJLAa6almrII=
|
||||
github.com/libp2p/go-libp2p-core v0.4.0/go.mod h1:49XGI+kc38oGVwqSBhDEwytaAxgZasHhFfQKibzTls0=
|
||||
github.com/libp2p/go-libp2p-core v0.5.0/go.mod h1:49XGI+kc38oGVwqSBhDEwytaAxgZasHhFfQKibzTls0=
|
||||
github.com/libp2p/go-libp2p-crypto v0.1.0/go.mod h1:sPUokVISZiy+nNuTTH/TY+leRSxnFj/2GLjtOTW90hI=
|
||||
github.com/libp2p/go-libp2p-discovery v0.1.0/go.mod h1:4F/x+aldVHjHDHuX85x1zWoFTGElt8HnoDzwkFZm29g=
|
||||
github.com/libp2p/go-libp2p-discovery v0.2.0/go.mod h1:s4VGaxYMbw4+4+tsoQTqh7wfxg97AEdo4GYBt6BadWg=
|
||||
github.com/libp2p/go-libp2p-loggables v0.1.0/go.mod h1:EyumB2Y6PrYjr55Q3/tiJ/o3xoDasoRYM7nOzEpoa90=
|
||||
github.com/libp2p/go-libp2p-mplex v0.2.0/go.mod h1:Ejl9IyjvXJ0T9iqUTE1jpYATQ9NM3g+OtR+EMMODbKo=
|
||||
github.com/libp2p/go-libp2p-mplex v0.2.1/go.mod h1:SC99Rxs8Vuzrf/6WhmH41kNn13TiYdAWNYHrwImKLnE=
|
||||
github.com/libp2p/go-libp2p-mplex v0.2.2/go.mod h1:74S9eum0tVQdAfFiKxAyKzNdSuLqw5oadDq7+L/FELo=
|
||||
github.com/libp2p/go-libp2p-nat v0.0.4/go.mod h1:N9Js/zVtAXqaeT99cXgTV9e75KpnWCvVOiGzlcHmBbY=
|
||||
github.com/libp2p/go-libp2p-nat v0.0.5/go.mod h1:1qubaE5bTZMJE+E/uu2URroMbzdubFz1ChgiN79yKPE=
|
||||
github.com/libp2p/go-libp2p-netutil v0.1.0/go.mod h1:3Qv/aDqtMLTUyQeundkKsA+YCThNdbQD54k3TqjpbFU=
|
||||
github.com/libp2p/go-libp2p-peer v0.2.0/go.mod h1:RCffaCvUyW2CJmG2gAWVqwePwW7JMgxjsHm7+J5kjWY=
|
||||
github.com/libp2p/go-libp2p-peerstore v0.1.0/go.mod h1:2CeHkQsr8svp4fZ+Oi9ykN1HBb6u0MOvdJ7YIsmcwtY=
|
||||
github.com/libp2p/go-libp2p-peerstore v0.1.3/go.mod h1:BJ9sHlm59/80oSkpWgr1MyY1ciXAXV397W6h1GH/uKI=
|
||||
github.com/libp2p/go-libp2p-peerstore v0.2.0/go.mod h1:N2l3eVIeAitSg3Pi2ipSrJYnqhVnMNQZo9nkSCuAbnQ=
|
||||
github.com/libp2p/go-libp2p-pnet v0.2.0/go.mod h1:Qqvq6JH/oMZGwqs3N1Fqhv8NVhrdYcO0BW4wssv21LA=
|
||||
github.com/libp2p/go-libp2p-record v0.1.0/go.mod h1:ujNc8iuE5dlKWVy6wuL6dd58t0n7xI4hAIl8pE6wu5Q=
|
||||
github.com/libp2p/go-libp2p-record v0.1.1/go.mod h1:VRgKajOyMVgP/F0L5g3kH7SVskp17vFi2xheb5uMJtg=
|
||||
github.com/libp2p/go-libp2p-secio v0.1.0/go.mod h1:tMJo2w7h3+wN4pgU2LSYeiKPrfqBgkOsdiKK77hE7c8=
|
||||
github.com/libp2p/go-libp2p-secio v0.2.0/go.mod h1:2JdZepB8J5V9mBp79BmwsaPQhRPNN2NrnB2lKQcdy6g=
|
||||
github.com/libp2p/go-libp2p-secio v0.2.1/go.mod h1:cWtZpILJqkqrSkiYcDBh5lA3wbT2Q+hz3rJQq3iftD8=
|
||||
github.com/libp2p/go-libp2p-swarm v0.1.0/go.mod h1:wQVsCdjsuZoc730CgOvh5ox6K8evllckjebkdiY5ta4=
|
||||
github.com/libp2p/go-libp2p-swarm v0.2.2/go.mod h1:fvmtQ0T1nErXym1/aa1uJEyN7JzaTNyBcHImCxRpPKU=
|
||||
github.com/libp2p/go-libp2p-testing v0.0.2/go.mod h1:gvchhf3FQOtBdr+eFUABet5a4MBLK8jM3V4Zghvmi+E=
|
||||
github.com/libp2p/go-libp2p-testing v0.0.3/go.mod h1:gvchhf3FQOtBdr+eFUABet5a4MBLK8jM3V4Zghvmi+E=
|
||||
github.com/libp2p/go-libp2p-testing v0.0.4/go.mod h1:gvchhf3FQOtBdr+eFUABet5a4MBLK8jM3V4Zghvmi+E=
|
||||
github.com/libp2p/go-libp2p-testing v0.1.0/go.mod h1:xaZWMJrPUM5GlDBxCeGUi7kI4eqnjVyavGroI2nxEM0=
|
||||
github.com/libp2p/go-libp2p-testing v0.1.1/go.mod h1:xaZWMJrPUM5GlDBxCeGUi7kI4eqnjVyavGroI2nxEM0=
|
||||
github.com/libp2p/go-libp2p-transport-upgrader v0.1.1/go.mod h1:IEtA6or8JUbsV07qPW4r01GnTenLW4oi3lOPbUMGJJA=
|
||||
github.com/libp2p/go-libp2p-transport-upgrader v0.2.0/go.mod h1:mQcrHj4asu6ArfSoMuyojOdjx73Q47cYD7s5+gZOlns=
|
||||
github.com/libp2p/go-libp2p-yamux v0.2.0/go.mod h1:Db2gU+XfLpm6E4rG5uGCFX6uXA8MEXOxFcRoXUODaK8=
|
||||
github.com/libp2p/go-libp2p-yamux v0.2.1/go.mod h1:1FBXiHDk1VyRM1C0aez2bCfHQ4vMZKkAQzZbkSQt5fI=
|
||||
github.com/libp2p/go-libp2p-yamux v0.2.2/go.mod h1:lIohaR0pT6mOt0AZ0L2dFze9hds9Req3OfS+B+dv4qw=
|
||||
github.com/libp2p/go-maddr-filter v0.0.4/go.mod h1:6eT12kSQMA9x2pvFQa+xesMKUBlj9VImZbj3B9FBH/Q=
|
||||
github.com/libp2p/go-maddr-filter v0.0.5/go.mod h1:Jk+36PMfIqCJhAnaASRH83bdAvfDRp/w6ENFaC9bG+M=
|
||||
github.com/libp2p/go-mplex v0.0.3/go.mod h1:pK5yMLmOoBR1pNCqDlA2GQrdAVTMkqFalaTWe7l4Yd0=
|
||||
github.com/libp2p/go-mplex v0.1.0/go.mod h1:SXgmdki2kwCUlCCbfGLEgHjC4pFqhTp0ZoV6aiKgxDU=
|
||||
github.com/libp2p/go-mplex v0.1.1/go.mod h1:Xgz2RDCi3co0LeZfgjm4OgUF15+sVR8SRcu3SFXI1lk=
|
||||
github.com/libp2p/go-msgio v0.0.2/go.mod h1:63lBBgOTDKQL6EWazRMCwXsEeEeK9O2Cd+0+6OOuipQ=
|
||||
github.com/libp2p/go-msgio v0.0.3/go.mod h1:63lBBgOTDKQL6EWazRMCwXsEeEeK9O2Cd+0+6OOuipQ=
|
||||
github.com/libp2p/go-msgio v0.0.4/go.mod h1:63lBBgOTDKQL6EWazRMCwXsEeEeK9O2Cd+0+6OOuipQ=
|
||||
github.com/libp2p/go-nat v0.0.3/go.mod h1:88nUEt0k0JD45Bk93NIwDqjlhiOwOoV36GchpcVc1yI=
|
||||
github.com/libp2p/go-nat v0.0.4/go.mod h1:Nmw50VAvKuk38jUBcmNh6p9lUJLoODbJRvYAa/+KSDo=
|
||||
github.com/libp2p/go-openssl v0.0.2/go.mod h1:v8Zw2ijCSWBQi8Pq5GAixw6DbFfa9u6VIYDXnvOXkc0=
|
||||
github.com/libp2p/go-openssl v0.0.3/go.mod h1:unDrJpgy3oFr+rqXsarWifmJuNnJR4chtO1HmaZjggc=
|
||||
github.com/libp2p/go-openssl v0.0.4/go.mod h1:unDrJpgy3oFr+rqXsarWifmJuNnJR4chtO1HmaZjggc=
|
||||
github.com/libp2p/go-reuseport v0.0.1/go.mod h1:jn6RmB1ufnQwl0Q1f+YxAj8isJgDCQzaaxIFYDhcYEA=
|
||||
github.com/libp2p/go-reuseport-transport v0.0.2/go.mod h1:YkbSDrvjUVDL6b8XqriyA20obEtsW9BLkuOUyQAOCbs=
|
||||
github.com/libp2p/go-stream-muxer v0.0.1/go.mod h1:bAo8x7YkSpadMTbtTaxGVHWUQsR/l5MEaHbKaliuT14=
|
||||
github.com/libp2p/go-stream-muxer-multistream v0.2.0/go.mod h1:j9eyPol/LLRqT+GPLSxvimPhNph4sfYfMoDPd7HkzIc=
|
||||
github.com/libp2p/go-tcp-transport v0.1.0/go.mod h1:oJ8I5VXryj493DEJ7OsBieu8fcg2nHGctwtInJVpipc=
|
||||
github.com/libp2p/go-tcp-transport v0.1.1/go.mod h1:3HzGvLbx6etZjnFlERyakbaYPdfjg2pWP97dFZworkY=
|
||||
github.com/libp2p/go-testutil v0.1.0/go.mod h1:81b2n5HypcVyrCg/MJx4Wgfp/VHojytjVe/gLzZ2Ehc=
|
||||
github.com/libp2p/go-ws-transport v0.1.0/go.mod h1:rjw1MG1LU9YDC6gzmwObkPd/Sqwhw7yT74kj3raBFuo=
|
||||
github.com/libp2p/go-ws-transport v0.2.0/go.mod h1:9BHJz/4Q5A9ludYWKoGCFC5gUElzlHoKzu0yY9p/klM=
|
||||
github.com/libp2p/go-yamux v1.2.2/go.mod h1:FGTiPvoV/3DVdgWpX+tM0OW3tsM+W5bSE3gZwqQTcow=
|
||||
github.com/libp2p/go-yamux v1.2.3/go.mod h1:FGTiPvoV/3DVdgWpX+tM0OW3tsM+W5bSE3gZwqQTcow=
|
||||
github.com/libp2p/go-yamux v1.3.0/go.mod h1:FGTiPvoV/3DVdgWpX+tM0OW3tsM+W5bSE3gZwqQTcow=
|
||||
github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
|
||||
github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4=
|
||||
github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
|
||||
github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
|
||||
github.com/mattbaird/elastigo v0.0.0-20170123220020-2fe47fd29e4b h1:v29yPGHhOqw7VHEnTeQFAth3SsBrmwc8JfuhNY0G34k=
|
||||
github.com/mattbaird/elastigo v0.0.0-20170123220020-2fe47fd29e4b/go.mod h1:5MWrJXKRQyhQdUCF+vu6U5c4nQpg70vW3eHaU0/AYbU=
|
||||
github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
|
||||
github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ=
|
||||
github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
|
||||
github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
|
||||
github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
|
||||
github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
|
||||
github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
|
||||
github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
|
||||
github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ=
|
||||
github.com/mattn/go-runewidth v0.0.7/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
|
||||
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
|
||||
github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE=
|
||||
github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
|
||||
github.com/miekg/dns v1.1.12/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
|
||||
github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1/go.mod h1:pD8RvIylQ358TN4wwqatJ8rNavkEINozVn9DtGI3dfQ=
|
||||
github.com/minio/sha256-simd v0.0.0-20190131020904-2d45a736cd16/go.mod h1:2FMWW+8GMoPweT6+pI63m9YE3Lmw4J71hV56Chs1E/U=
|
||||
github.com/minio/sha256-simd v0.0.0-20190328051042-05b4dd3047e5/go.mod h1:2FMWW+8GMoPweT6+pI63m9YE3Lmw4J71hV56Chs1E/U=
|
||||
github.com/minio/sha256-simd v0.1.0/go.mod h1:2FMWW+8GMoPweT6+pI63m9YE3Lmw4J71hV56Chs1E/U=
|
||||
github.com/minio/sha256-simd v0.1.1-0.20190913151208-6de447530771/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM=
|
||||
github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM=
|
||||
github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc=
|
||||
github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
|
||||
github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
|
||||
github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI=
|
||||
github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg=
|
||||
github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY=
|
||||
github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
|
||||
github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE=
|
||||
github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
|
||||
github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI=
|
||||
github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
|
||||
github.com/mr-tron/base58 v1.1.0/go.mod h1:xcD2VGqlgYjBdcBLw+TuYLr8afG+Hj8g2eTVqeSzSU8=
|
||||
github.com/mr-tron/base58 v1.1.1/go.mod h1:xcD2VGqlgYjBdcBLw+TuYLr8afG+Hj8g2eTVqeSzSU8=
|
||||
github.com/mr-tron/base58 v1.1.2/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc=
|
||||
github.com/mr-tron/base58 v1.1.3/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc=
|
||||
github.com/multiformats/go-base32 v0.0.3/go.mod h1:pLiuGC8y0QR3Ue4Zug5UzK9LjgbkL8NSQj0zQ5Nz/AA=
|
||||
github.com/multiformats/go-multiaddr v0.0.1/go.mod h1:xKVEak1K9cS1VdmPZW3LSIb6lgmoS58qz/pzqmAxV44=
|
||||
github.com/multiformats/go-multiaddr v0.0.2/go.mod h1:xKVEak1K9cS1VdmPZW3LSIb6lgmoS58qz/pzqmAxV44=
|
||||
github.com/multiformats/go-multiaddr v0.0.4/go.mod h1:xKVEak1K9cS1VdmPZW3LSIb6lgmoS58qz/pzqmAxV44=
|
||||
github.com/multiformats/go-multiaddr v0.1.0/go.mod h1:xKVEak1K9cS1VdmPZW3LSIb6lgmoS58qz/pzqmAxV44=
|
||||
github.com/multiformats/go-multiaddr v0.1.1/go.mod h1:aMKBKNEYmzmDmxfX88/vz+J5IU55txyt0p4aiWVohjo=
|
||||
github.com/multiformats/go-multiaddr v0.2.0/go.mod h1:0nO36NvPpyV4QzvTLi/lafl2y95ncPj0vFwVF6k6wJ4=
|
||||
github.com/multiformats/go-multiaddr v0.2.1/go.mod h1:s/Apk6IyxfvMjDafnhJgJ3/46z7tZ04iMk5wP4QMGGE=
|
||||
github.com/multiformats/go-multiaddr-dns v0.0.1/go.mod h1:9kWcqw/Pj6FwxAwW38n/9403szc57zJPs45fmnznu3Q=
|
||||
github.com/multiformats/go-multiaddr-dns v0.0.2/go.mod h1:9kWcqw/Pj6FwxAwW38n/9403szc57zJPs45fmnznu3Q=
|
||||
github.com/multiformats/go-multiaddr-dns v0.2.0/go.mod h1:TJ5pr5bBO7Y1B18djPuRsVkduhQH2YqYSbxWJzYGdK0=
|
||||
github.com/multiformats/go-multiaddr-fmt v0.0.1/go.mod h1:aBYjqL4T/7j4Qx+R73XSv/8JsgnRFlf0w2KGLCmXl3Q=
|
||||
github.com/multiformats/go-multiaddr-fmt v0.1.0/go.mod h1:hGtDIW4PU4BqJ50gW2quDuPVjyWNZxToGUh/HwTZYJo=
|
||||
github.com/multiformats/go-multiaddr-net v0.0.1/go.mod h1:nw6HSxNmCIQH27XPGBuX+d1tnvM7ihcFwHMSstNAVUU=
|
||||
github.com/multiformats/go-multiaddr-net v0.1.0/go.mod h1:5JNbcfBOP4dnhoZOv10JJVkJO0pCCEf8mTnipAo2UZQ=
|
||||
github.com/multiformats/go-multiaddr-net v0.1.1/go.mod h1:5JNbcfBOP4dnhoZOv10JJVkJO0pCCEf8mTnipAo2UZQ=
|
||||
github.com/multiformats/go-multiaddr-net v0.1.2/go.mod h1:QsWt3XK/3hwvNxZJp92iMQKME1qHfpYmyIjFVsSOY6Y=
|
||||
github.com/multiformats/go-multibase v0.0.1/go.mod h1:bja2MqRZ3ggyXtZSEDKpl0uO/gviWFaSteVbWT51qgs=
|
||||
github.com/multiformats/go-multihash v0.0.1/go.mod h1:w/5tugSrLEbWqlcgJabL3oHFKTwfvkofsjW2Qa1ct4U=
|
||||
github.com/multiformats/go-multihash v0.0.5/go.mod h1:lt/HCbqlQwlPBz7lv0sQCdtfcMtlJvakRUn/0Ual8po=
|
||||
github.com/multiformats/go-multihash v0.0.8/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew=
|
||||
github.com/multiformats/go-multihash v0.0.9/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew=
|
||||
github.com/multiformats/go-multihash v0.0.10/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew=
|
||||
github.com/multiformats/go-multihash v0.0.13/go.mod h1:VdAWLKTwram9oKAatUcLxBNUjdtcVwxObEQBtRfuyjc=
|
||||
github.com/multiformats/go-multistream v0.1.0/go.mod h1:fJTiDfXJVmItycydCnNx4+wSzZ5NwG2FEVAI30fiovg=
|
||||
github.com/multiformats/go-multistream v0.1.1/go.mod h1:KmHZ40hzVxiaiwlj3MEbYgK9JFk2/9UktWZAF54Du38=
|
||||
github.com/multiformats/go-varint v0.0.1/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE=
|
||||
github.com/multiformats/go-varint v0.0.2/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE=
|
||||
github.com/multiformats/go-varint v0.0.5/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE=
|
||||
github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
|
||||
github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
|
||||
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
|
||||
github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
|
||||
github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
|
||||
github.com/onsi/ginkgo v1.12.0/go.mod h1:oUhWkIvk5aDxtKvDDuw8gItl8pKl42LzjC9KZE0HfGg=
|
||||
github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
|
||||
github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
|
||||
github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
|
||||
github.com/onsi/gomega v1.9.0/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoTdcA=
|
||||
github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
|
||||
github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
|
||||
github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
|
||||
github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc=
|
||||
github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
|
||||
github.com/pierrec/lz4 v2.4.1+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
|
||||
github.com/pierrec/lz4 v2.5.2+incompatible h1:WCjObylUIOlKy/+7Abdn34TLIkXiA4UWUMhxq9m9ZXI=
|
||||
github.com/pierrec/lz4 v2.5.2+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
|
||||
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/polydawn/refmt v0.0.0-20190221155625-df39d6c2d992/go.mod h1:uIp+gprXxxrWSjjklXD+mN4wed/tMfjMMmN/9+JsA9o=
|
||||
github.com/polydawn/refmt v0.0.0-20190408063855-01bf1e26dd14/go.mod h1:uIp+gprXxxrWSjjklXD+mN4wed/tMfjMMmN/9+JsA9o=
|
||||
github.com/polydawn/refmt v0.0.0-20190807091052-3d65705ee9f1/go.mod h1:uIp+gprXxxrWSjjklXD+mN4wed/tMfjMMmN/9+JsA9o=
|
||||
github.com/polydawn/refmt v0.0.0-20190809202753-05966cbd336a/go.mod h1:uIp+gprXxxrWSjjklXD+mN4wed/tMfjMMmN/9+JsA9o=
|
||||
github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI=
|
||||
github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
|
||||
github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso=
|
||||
github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
|
||||
github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
|
||||
github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
|
||||
github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
|
||||
github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
|
||||
github.com/prometheus/procfs v0.0.0-20190425082905-87a4384529e0/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
|
||||
github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
|
||||
github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
|
||||
github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
|
||||
github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0 h1:MkV+77GLUNo5oJ0jf870itWm3D0Sjh7+Za9gazKc5LQ=
|
||||
github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
|
||||
github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
|
||||
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
|
||||
github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
|
||||
github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
|
||||
github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
|
||||
github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
|
||||
github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
|
||||
github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
|
||||
github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM=
|
||||
github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
|
||||
github.com/smartystreets/assertions v1.0.0/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM=
|
||||
github.com/smartystreets/assertions v1.0.1/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM=
|
||||
github.com/smartystreets/goconvey v0.0.0-20190222223459-a17d461953aa/go.mod h1:2RVY1rIf+2J2o/IM9+vPq9RzmHDSseB7FoXiSNIUsoU=
|
||||
github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
|
||||
github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
|
||||
github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s=
|
||||
github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
|
||||
github.com/smola/gocompat v0.2.0/go.mod h1:1B0MlxbmoZNo3h8guHp8HztB3BSYR5itql9qtVc0ypY=
|
||||
github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=
|
||||
github.com/spacemonkeygo/openssl v0.0.0-20181017203307-c2dcc5cca94a/go.mod h1:7AyxJNCJ7SBZ1MfVQCWD6Uqo2oubI2Eq2y2eqf+A5r0=
|
||||
github.com/spacemonkeygo/spacelog v0.0.0-20180420211403-2296661a0572/go.mod h1:w0SWMsp6j9O/dk4/ZpIhL+3CkG8ofA2vuv7k+ltqUMc=
|
||||
github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
|
||||
github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
|
||||
github.com/spf13/afero v1.1.2 h1:m8/z1t7/fwjysjQRYbP0RD+bUIF/8tJwPdEZsI83ACI=
|
||||
github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
|
||||
github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8=
|
||||
github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
|
||||
github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU=
|
||||
github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9Gc1vn7yk=
|
||||
github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
|
||||
github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg=
|
||||
github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
|
||||
github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s=
|
||||
github.com/spf13/viper v1.7.0 h1:xVKxvI7ouOI5I+U9s2eeiUfMaWBVoXA3AWskkrqK0VM=
|
||||
github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg=
|
||||
github.com/src-d/envconfig v1.0.0/go.mod h1:Q9YQZ7BKITldTBnoxsE5gOeB5y66RyPXeue/R4aaNBc=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
|
||||
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
||||
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
|
||||
github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4=
|
||||
github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
|
||||
github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s=
|
||||
github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
|
||||
github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ=
|
||||
github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
|
||||
github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
|
||||
github.com/urfave/cli/v2 v2.0.0/go.mod h1:SE9GqnLQmjVa0iPEY0f1w3ygNIYcIJ0OKPMoW2caLfQ=
|
||||
github.com/warpfork/go-wish v0.0.0-20180510122957-5ad1f5abf436/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw=
|
||||
github.com/warpfork/go-wish v0.0.0-20190328234359-8b3e70f8e830/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw=
|
||||
github.com/warpfork/go-wish v0.0.0-20200122115046-b9ea61034e4a/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw=
|
||||
github.com/whyrusleeping/cbor-gen v0.0.0-20191212224538-d370462a7e8a/go.mod h1:xdlJQaiqipF0HW+Mzpg7XRM3fWbGvfgFlcppuvlkIvY=
|
||||
github.com/whyrusleeping/cbor-gen v0.0.0-20191216205031-b047b6acb3c0/go.mod h1:xdlJQaiqipF0HW+Mzpg7XRM3fWbGvfgFlcppuvlkIvY=
|
||||
github.com/whyrusleeping/cbor-gen v0.0.0-20200123233031-1cdf64d27158/go.mod h1:Xj/M2wWU+QdTdRbu/L/1dIZY8/Wb2K9pAhtroQuxJJI=
|
||||
github.com/whyrusleeping/cbor-gen v0.0.0-20200206220010-03c9665e2a66/go.mod h1:Xj/M2wWU+QdTdRbu/L/1dIZY8/Wb2K9pAhtroQuxJJI=
|
||||
github.com/whyrusleeping/cbor-gen v0.0.0-20200402171437-3d27c146c105/go.mod h1:Xj/M2wWU+QdTdRbu/L/1dIZY8/Wb2K9pAhtroQuxJJI=
|
||||
github.com/whyrusleeping/cbor-gen v0.0.0-20200414195334-429a0b5e922e/go.mod h1:Xj/M2wWU+QdTdRbu/L/1dIZY8/Wb2K9pAhtroQuxJJI=
|
||||
github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f/go.mod h1:p9UJB6dDgdPgMJZs7UjUOdulKyRr9fqkS+6JKAInPy8=
|
||||
github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1/go.mod h1:8UvriyWtv5Q5EOgjHaSseUEdkQfvwFv1I/In/O2M9gc=
|
||||
github.com/whyrusleeping/go-logging v0.0.0-20170515211332-0457bb6b88fc/go.mod h1:bopw91TMyo8J3tvftk8xmU2kPmlrt4nScJQZU2hE5EM=
|
||||
github.com/whyrusleeping/go-logging v0.0.1/go.mod h1:lDPYj54zutzG1XYfHAhcc7oNXEburHQBn+Iqd4yS4vE=
|
||||
github.com/whyrusleeping/go-notifier v0.0.0-20170827234753-097c5d47330f/go.mod h1:cZNvX9cFybI01GriPRMXDtczuvUhgbcYr9iCGaNlRv8=
|
||||
github.com/whyrusleeping/mafmt v1.2.8/go.mod h1:faQJFPbLSxzD9xpA02ttW/tS9vZykNvXwGvqIpk20FA=
|
||||
github.com/whyrusleeping/mdns v0.0.0-20180901202407-ef14215e6b30/go.mod h1:j4l84WPFclQPj320J9gp0XwNKBb3U0zt5CBqjPp22G4=
|
||||
github.com/whyrusleeping/mdns v0.0.0-20190826153040-b9b60ed33aa9/go.mod h1:j4l84WPFclQPj320J9gp0XwNKBb3U0zt5CBqjPp22G4=
|
||||
github.com/whyrusleeping/multiaddr-filter v0.0.0-20160516205228-e903e4adabd7/go.mod h1:X2c0RVCI1eSUFI8eLcY3c0423ykwiUdxLJtkDvruhjI=
|
||||
github.com/x-cray/logrus-prefixed-formatter v0.5.2/go.mod h1:2duySbKsL6M18s5GU7VPsoEPHyzalCE06qoARUCeBBE=
|
||||
github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I=
|
||||
github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y=
|
||||
github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
|
||||
github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
|
||||
go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
|
||||
go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
|
||||
go.opencensus.io v0.22.0 h1:C9hSCOW830chIVkdja34wa6Ky+IzWllkUinR+BtRZd4=
|
||||
go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
|
||||
go.opencensus.io v0.22.1/go.mod h1:Ap50jQcDJrx6rB6VgeeFPtuPIf3wMRvRfrfYDO6+BmA=
|
||||
go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
|
||||
go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
|
||||
go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
|
||||
go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
|
||||
go.uber.org/atomic v1.5.1/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
|
||||
go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
|
||||
go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4=
|
||||
go.uber.org/multierr v1.4.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4=
|
||||
go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA=
|
||||
go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
|
||||
go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM=
|
||||
golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
|
||||
golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
|
||||
golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
|
||||
golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
|
||||
golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
|
||||
golang.org/x/crypto v0.0.0-20190225124518-7f87c0fbb88b/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.0.0-20190530122614-20be4c3c3ed5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.0.0-20190618222545-ea8f1a30c443/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
|
||||
golang.org/x/crypto v0.0.0-20200117160349-530e935923ad/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
|
||||
golang.org/x/crypto v0.0.0-20200204104054-c9f3fb736b72 h1:+ELyKg6m8UBf0nPFSqD0mi7zUfwPyXo23HNjMnXPz7w=
|
||||
golang.org/x/crypto v0.0.0-20200204104054-c9f3fb736b72/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
|
||||
golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
|
||||
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI=
|
||||
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
|
||||
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||
golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||
golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
|
||||
golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek=
|
||||
golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY=
|
||||
golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
|
||||
golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
|
||||
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
|
||||
golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
|
||||
golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
|
||||
golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
|
||||
golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
|
||||
golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
|
||||
golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
|
||||
golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs=
|
||||
golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=
|
||||
golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=
|
||||
golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
|
||||
golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY=
|
||||
golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
|
||||
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20181011144130-49bb7cea24b1/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20190227160552-c95aed5357e7/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
|
||||
golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
|
||||
golang.org/x/net v0.0.0-20190611141213-3f473d35a33a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20200202094626-16171245cfb2 h1:CCH4IOTTfewWjGOlSp+zGcjutRKlBEZQ6wTn8ozI/nI=
|
||||
golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20200602114024-627f9648deb9 h1:pNX+40auqi2JqRfOP1akLGtYcn15TUbkhwuCO3foqqM=
|
||||
golang.org/x/net v0.0.0-20200602114024-627f9648deb9/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
|
||||
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
|
||||
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 h1:SVwTIAaPC2U/AvvLNZ2a7OVsmBpC8L5BlwK1whH3hm0=
|
||||
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sys v0.0.0-20180202135801-37707fdb30a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190219092855-153ac476189d/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190228124157-a34e9553db1e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190302025703-b6889370fb10/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190405154228-4b34438f7a67/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190524122548-abf6ff778158/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190610200419-93c9922d18ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20191025021431-6c3a3bfe00ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200107162124-548cf772de50/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884=
|
||||
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
|
||||
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
|
||||
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20181130052023-1c3d964395ce/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
|
||||
golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
|
||||
golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
|
||||
golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
|
||||
golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
|
||||
golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
|
||||
golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
|
||||
golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
|
||||
golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
|
||||
golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
|
||||
golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
|
||||
golang.org/x/tools v0.0.0-20200108195415-316d2f248479/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
|
||||
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
|
||||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
|
||||
google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=
|
||||
google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
|
||||
google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
|
||||
google.golang.org/api v0.13.0 h1:Q3Ui3V3/CVinFWFiW39Iw0kMuVrRzYX0wN6OPFp0lTA=
|
||||
google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
|
||||
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
|
||||
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
|
||||
google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
|
||||
google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
|
||||
google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
|
||||
google.golang.org/genproto v0.0.0-20180831171423-11092d34479b/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
|
||||
google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
|
||||
google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
|
||||
google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
|
||||
google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
|
||||
google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
|
||||
google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
|
||||
google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8=
|
||||
google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a h1:Ob5/580gVHBJZgXnff1cZDbG+xLtMVE5mDRTe+nIsX4=
|
||||
google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
|
||||
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
|
||||
google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
|
||||
google.golang.org/grpc v1.21.1 h1:j6XxA85m/6txkUCHvzlV5f+HBNl/1r5cZ2A/3IEFOO8=
|
||||
google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
|
||||
gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
|
||||
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/cheggaaa/pb.v1 v1.0.28/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw=
|
||||
gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
|
||||
gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
|
||||
gopkg.in/ini.v1 v1.51.0 h1:AQvPpx3LzTDM0AjnIRlVFwFFGC+npRopjZxLJj6gdno=
|
||||
gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
|
||||
gopkg.in/jcmturner/aescts.v1 v1.0.1 h1:cVVZBK2b1zY26haWB4vbBiZrfFQnfbTVrE3xZq6hrEw=
|
||||
gopkg.in/jcmturner/aescts.v1 v1.0.1/go.mod h1:nsR8qBOg+OucoIW+WMhB3GspUQXq9XorLnQb9XtvcOo=
|
||||
gopkg.in/jcmturner/dnsutils.v1 v1.0.1 h1:cIuC1OLRGZrld+16ZJvvZxVJeKPsvd5eUIvxfoN5hSM=
|
||||
gopkg.in/jcmturner/dnsutils.v1 v1.0.1/go.mod h1:m3v+5svpVOhtFAP/wSz+yzh4Mc0Fg7eRhxkJMWSIz9Q=
|
||||
gopkg.in/jcmturner/goidentity.v3 v3.0.0 h1:1duIyWiTaYvVx3YX2CYtpJbUFd7/UuPYCfgXtQ3VTbI=
|
||||
gopkg.in/jcmturner/goidentity.v3 v3.0.0/go.mod h1:oG2kH0IvSYNIu80dVAyu/yoefjq1mNfM5bm88whjWx4=
|
||||
gopkg.in/jcmturner/gokrb5.v7 v7.5.0 h1:a9tsXlIDD9SKxotJMK3niV7rPZAJeX2aD/0yg3qlIrg=
|
||||
gopkg.in/jcmturner/gokrb5.v7 v7.5.0/go.mod h1:l8VISx+WGYp+Fp7KRbsiUuXTTOnxIc3Tuvyavf11/WM=
|
||||
gopkg.in/jcmturner/rpc.v1 v1.1.0 h1:QHIUxTX1ISuAv9dD2wJ9HWQVuWDX/Zc0PfeC2tjc4rU=
|
||||
gopkg.in/jcmturner/rpc.v1 v1.1.0/go.mod h1:YIdkC4XfD6GXbzje11McwsDuOlZQSb9W4vfLvuNnlv8=
|
||||
gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo=
|
||||
gopkg.in/src-d/go-cli.v0 v0.0.0-20181105080154-d492247bbc0d/go.mod h1:z+K8VcOYVYcSwSjGebuDL6176A1XskgbtNl64NSg+n8=
|
||||
gopkg.in/src-d/go-log.v1 v1.0.1/go.mod h1:GN34hKP0g305ysm2/hctJ0Y8nWP3zxXXJ8GFabTyABE=
|
||||
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
|
||||
gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74=
|
||||
gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
|
||||
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=
|
||||
gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw=
|
||||
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
||||
honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
||||
honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
||||
honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
|
||||
howett.net/plist v0.0.0-20181124034731-591f970eefbb/go.mod h1:vMygbs4qMhSZSc4lCUl2OEE+rDiIIJAIdR4m7MiMcm0=
|
||||
rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
|
||||
@@ -9,7 +9,6 @@ import (
|
||||
"log"
|
||||
"net/http"
|
||||
"net/http/httputil"
|
||||
_ "net/http/pprof"
|
||||
"os"
|
||||
"os/signal"
|
||||
"runtime"
|
||||
@@ -33,14 +32,11 @@ func loggingMiddleware(next http.Handler) http.Handler {
|
||||
})
|
||||
}
|
||||
|
||||
var closeCh chan int
|
||||
|
||||
func main() {
|
||||
closeCh = make(chan int)
|
||||
// // Don't exit on panic
|
||||
// defer func() {
|
||||
// if r := recover(); r != nil {
|
||||
// fmt.Printf("PANIC: pkg: %v %s \n", r, Debug.Stack())
|
||||
// fmt.Printf("PANIC: pkg: %v %s \n", r, debug.Stack())
|
||||
// }
|
||||
// }()
|
||||
|
||||
@@ -50,26 +46,23 @@ func main() {
|
||||
}
|
||||
|
||||
args := os.Args[1:]
|
||||
var plugins *InOutPlugins
|
||||
if len(args) > 0 && args[0] == "file-server" {
|
||||
if len(args) != 2 {
|
||||
log.Fatal("You should specify port and IP (optional) for the file server. Example: `gor file-server :80`")
|
||||
}
|
||||
dir, _ := os.Getwd()
|
||||
|
||||
log.Println("Started example file server for current directory on address ", args[1])
|
||||
log.Println("Started example file server for current dirrectory on address ", args[1])
|
||||
|
||||
log.Fatal(http.ListenAndServe(args[1], loggingMiddleware(http.FileServer(http.Dir(dir)))))
|
||||
} else {
|
||||
flag.Parse()
|
||||
checkSettings()
|
||||
time.Sleep(60*time.Second)
|
||||
plugins = InitPlugins()
|
||||
InitPlugins()
|
||||
}
|
||||
|
||||
fmt.Println("Version:", VERSION)
|
||||
|
||||
if len(plugins.Inputs) == 0 || len(plugins.Outputs) == 0 {
|
||||
if len(Plugins.Inputs) == 0 || len(Plugins.Outputs) == 0 {
|
||||
log.Fatal("Required at least 1 input and 1 output")
|
||||
}
|
||||
|
||||
@@ -81,41 +74,35 @@ func main() {
|
||||
profileCPU(*cpuprofile)
|
||||
}
|
||||
|
||||
if Settings.Pprof != "" {
|
||||
go func() {
|
||||
log.Println(http.ListenAndServe(Settings.Pprof, nil))
|
||||
}()
|
||||
}
|
||||
|
||||
emitter := NewEmitter(closeCh)
|
||||
c := make(chan os.Signal, 1)
|
||||
signal.Notify(c, os.Interrupt, syscall.SIGTERM)
|
||||
go func() {
|
||||
<-c
|
||||
finalize(plugins)
|
||||
finalize()
|
||||
os.Exit(1)
|
||||
}()
|
||||
|
||||
if Settings.ExitAfter > 0 {
|
||||
log.Println("Running gor for a duration of", Settings.ExitAfter)
|
||||
if Settings.exitAfter > 0 {
|
||||
log.Println("Running gor for a duration of", Settings.exitAfter)
|
||||
closeCh := make(chan int)
|
||||
|
||||
time.AfterFunc(Settings.ExitAfter, func() {
|
||||
log.Println("Stopping gor after", Settings.ExitAfter)
|
||||
time.AfterFunc(Settings.exitAfter, func() {
|
||||
log.Println("Stopping gor after", Settings.exitAfter)
|
||||
close(closeCh)
|
||||
})
|
||||
}
|
||||
|
||||
emitter.Start(plugins, Settings.Middleware)
|
||||
Start(closeCh)
|
||||
} else {
|
||||
Start(nil)
|
||||
}
|
||||
}
|
||||
|
||||
func finalize(plugins *InOutPlugins) {
|
||||
for _, p := range plugins.All {
|
||||
func finalize() {
|
||||
for _, p := range Plugins.All {
|
||||
if cp, ok := p.(io.Closer); ok {
|
||||
cp.Close()
|
||||
}
|
||||
}
|
||||
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
}
|
||||
|
||||
func profileCPU(cpuprofile string) {
|
||||
|
||||
+10
-8
@@ -7,25 +7,27 @@ import (
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
rate = 5
|
||||
)
|
||||
|
||||
type GorStat struct {
|
||||
statName string
|
||||
rateMs int
|
||||
latest int
|
||||
mean int
|
||||
max int
|
||||
count int
|
||||
}
|
||||
|
||||
func NewGorStat(statName string, rateMs int) (s *GorStat) {
|
||||
func NewGorStat(statName string) (s *GorStat) {
|
||||
s = new(GorStat)
|
||||
s.statName = statName
|
||||
s.rateMs = rateMs
|
||||
s.latest = 0
|
||||
s.mean = 0
|
||||
s.max = 0
|
||||
s.count = 0
|
||||
|
||||
if Settings.Stats {
|
||||
if Settings.stats {
|
||||
log.Println(s.statName + ":latest,mean,max,count,count/second,gcount")
|
||||
go s.reportStats()
|
||||
}
|
||||
@@ -33,12 +35,12 @@ func NewGorStat(statName string, rateMs int) (s *GorStat) {
|
||||
}
|
||||
|
||||
func (s *GorStat) Write(latest int) {
|
||||
if Settings.Stats {
|
||||
if Settings.stats {
|
||||
if latest > s.max {
|
||||
s.max = latest
|
||||
}
|
||||
if latest != 0 {
|
||||
s.mean = ((s.mean * s.count) + latest) / (s.count + 1)
|
||||
s.mean = (s.mean + latest) / 2
|
||||
}
|
||||
s.latest = latest
|
||||
s.count = s.count + 1
|
||||
@@ -53,13 +55,13 @@ func (s *GorStat) Reset() {
|
||||
}
|
||||
|
||||
func (s *GorStat) String() string {
|
||||
return s.statName + ":" + strconv.Itoa(s.latest) + "," + strconv.Itoa(s.mean) + "," + strconv.Itoa(s.max) + "," + strconv.Itoa(s.count) + "," + strconv.Itoa(s.count/(s.rateMs/1000.0)) + "," + strconv.Itoa(runtime.NumGoroutine())
|
||||
return s.statName + ":" + strconv.Itoa(s.latest) + "," + strconv.Itoa(s.mean) + "," + strconv.Itoa(s.max) + "," + strconv.Itoa(s.count) + "," + strconv.Itoa(s.count/rate) + "," + strconv.Itoa(runtime.NumGoroutine())
|
||||
}
|
||||
|
||||
func (s *GorStat) reportStats() {
|
||||
for {
|
||||
log.Println(s)
|
||||
s.Reset()
|
||||
time.Sleep(time.Duration(s.rateMs) * time.Millisecond)
|
||||
time.Sleep(rate * time.Second)
|
||||
}
|
||||
}
|
||||
|
||||
+43
-194
@@ -1,15 +1,12 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"crypto/tls"
|
||||
"encoding/base64"
|
||||
"io"
|
||||
"log"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/http/httputil"
|
||||
"net/url"
|
||||
"runtime/debug"
|
||||
"strconv"
|
||||
@@ -18,7 +15,7 @@ import (
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/buger/goreplay/proto"
|
||||
"github.com/buger/gor/proto"
|
||||
)
|
||||
|
||||
var httpMu sync.Mutex
|
||||
@@ -42,7 +39,6 @@ type HTTPClientConfig struct {
|
||||
ConnectionTimeout time.Duration
|
||||
Timeout time.Duration
|
||||
ResponseBufferSize int
|
||||
CompatibilityMode bool
|
||||
}
|
||||
|
||||
type HTTPClient struct {
|
||||
@@ -51,11 +47,8 @@ type HTTPClient struct {
|
||||
host string
|
||||
auth string
|
||||
conn net.Conn
|
||||
proxy *url.URL
|
||||
proxyAuth string
|
||||
respBuf []byte
|
||||
config *HTTPClientConfig
|
||||
goClient *http.Client
|
||||
redirectsCount int
|
||||
}
|
||||
|
||||
@@ -83,85 +76,23 @@ func NewHTTPClient(baseURL string, config *HTTPClientConfig) *HTTPClient {
|
||||
client.respBuf = make([]byte, config.ResponseBufferSize)
|
||||
client.config = config
|
||||
|
||||
if config.CompatibilityMode {
|
||||
client.goClient = &http.Client{
|
||||
// #TODO
|
||||
// CheckRedirect: redirectPolicyFunc,
|
||||
}
|
||||
}
|
||||
|
||||
if u.User != nil {
|
||||
client.auth = "Basic " + base64.StdEncoding.EncodeToString([]byte(u.User.String()))
|
||||
}
|
||||
|
||||
client.proxy, _ = http.ProxyFromEnvironment(&http.Request{URL: u})
|
||||
|
||||
if client.isProxy() && client.proxy.User != nil {
|
||||
client.proxyAuth = "Basic " + base64.StdEncoding.EncodeToString([]byte(client.proxy.User.String()))
|
||||
}
|
||||
|
||||
return client
|
||||
}
|
||||
|
||||
func (c *HTTPClient) Connect() (err error) {
|
||||
c.Disconnect()
|
||||
|
||||
var toDial string
|
||||
if !strings.Contains(c.host, ":") {
|
||||
toDial = c.host + ":" + defaultPorts[c.scheme]
|
||||
c.conn, err = net.DialTimeout("tcp", c.host+":"+defaultPorts[c.scheme], c.config.ConnectionTimeout)
|
||||
} else {
|
||||
toDial = c.host
|
||||
}
|
||||
|
||||
if c.isProxy() {
|
||||
if c.proxy.Scheme != "http" {
|
||||
panic("Unsupported HTTP Proxy method")
|
||||
}
|
||||
Debug("[HTTPClient] Connecting to proxy", c.proxy.String(), "<>", toDial)
|
||||
c.conn, err = net.DialTimeout("tcp", c.proxy.Host, c.config.ConnectionTimeout)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
if c.scheme == "https" {
|
||||
c.conn.Write([]byte("CONNECT " + toDial + " HTTP/1.1\r\n"))
|
||||
if c.proxyAuth != "" {
|
||||
c.conn.Write([]byte("Proxy-Authorization: " + c.proxyAuth + "\r\n"))
|
||||
}
|
||||
c.conn.Write([]byte("\r\n"))
|
||||
br := bufio.NewReader(c.conn)
|
||||
l, _, err := br.ReadLine()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(l) < 12 {
|
||||
panic("HTTP proxy did not respond correctly")
|
||||
}
|
||||
status := l[9:12]
|
||||
if !bytes.Equal(status, []byte("200")) {
|
||||
panic("HTTP proxy did not respond correctly")
|
||||
}
|
||||
for {
|
||||
// Read until we find the empty line
|
||||
l, _, err := br.ReadLine()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(l) == 0 {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
Debug("[HTTPClient] Proxy successfully connected")
|
||||
} else {
|
||||
c.conn, err = net.DialTimeout("tcp", toDial, c.config.ConnectionTimeout)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
c.conn, err = net.DialTimeout("tcp", c.host, c.config.ConnectionTimeout)
|
||||
}
|
||||
|
||||
if c.scheme == "https" {
|
||||
// Wrap our socket in TLS
|
||||
Debug("[HTTPClient] Wrapping socket in TLS", c.host)
|
||||
tlsConn := tls.Client(c.conn, &tls.Config{InsecureSkipVerify: true, ServerName: c.host})
|
||||
|
||||
if err = tlsConn.Handshake(); err != nil {
|
||||
@@ -169,7 +100,6 @@ func (c *HTTPClient) Connect() (err error) {
|
||||
}
|
||||
|
||||
c.conn = tlsConn
|
||||
Debug("[HTTPClient] Successfully wrapped in TLS")
|
||||
}
|
||||
|
||||
return
|
||||
@@ -183,76 +113,42 @@ func (c *HTTPClient) Disconnect() {
|
||||
}
|
||||
}
|
||||
|
||||
func (c *HTTPClient) isAlive(readBytes *int) bool {
|
||||
func (c *HTTPClient) isAlive() bool {
|
||||
one := make([]byte, 1)
|
||||
|
||||
// Ready 1 byte from socket without timeout to check if it not closed
|
||||
c.conn.SetReadDeadline(time.Now().Add(time.Millisecond))
|
||||
n, err := c.conn.Read(c.respBuf[:1])
|
||||
_, err := c.conn.Read(one)
|
||||
|
||||
if err == io.EOF {
|
||||
Debug("[HTTPClient] connection closed, reconnecting")
|
||||
if err == nil {
|
||||
return true
|
||||
} else if err == io.EOF {
|
||||
if c.config.Debug {
|
||||
Debug("[HTTPClient] connection closed, reconnecting")
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
if err == syscall.EPIPE {
|
||||
} else if err == syscall.EPIPE {
|
||||
Debug("Detected broken pipe.", err)
|
||||
return false
|
||||
}
|
||||
if n != 0 {
|
||||
*readBytes += n
|
||||
Debug("[HTTPClient] isAlive readBytes ", *readBytes)
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
func (c *HTTPClient) SendGoClient(data []byte) ([]byte, error) {
|
||||
var req *http.Request
|
||||
var resp *http.Response
|
||||
var err error
|
||||
|
||||
req, err = http.ReadRequest(bufio.NewReader(bytes.NewBuffer(data)))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if !c.config.OriginalHost {
|
||||
req.Host = c.host
|
||||
}
|
||||
|
||||
if c.auth != "" {
|
||||
req.Header.Add("Authorization", c.auth)
|
||||
}
|
||||
|
||||
req.URL, _ = url.ParseRequestURI(c.scheme + "://" + c.host + req.RequestURI)
|
||||
req.RequestURI = ""
|
||||
|
||||
resp, err = c.goClient.Do(req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return httputil.DumpResponse(resp, true)
|
||||
}
|
||||
|
||||
func (c *HTTPClient) Send(data []byte) (response []byte, err error) {
|
||||
// Don't exit on panic
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
Debug("[HTTPClient]", r, string(data))
|
||||
|
||||
if _, ok := r.(error); ok {
|
||||
if _, ok := r.(error); !ok {
|
||||
log.Println("[HTTPClient] Failed to send request: ", string(data))
|
||||
log.Println("[HTTPClient] Response: ", string(response))
|
||||
log.Println("PANIC: pkg:", r, string(debug.Stack()))
|
||||
log.Println("PANIC: pkg:", r, debug.Stack())
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
if c.config.CompatibilityMode {
|
||||
return c.SendGoClient(data)
|
||||
}
|
||||
|
||||
var readBytes int
|
||||
if c.conn == nil || !c.isAlive(&readBytes) {
|
||||
if c.conn == nil || !c.isAlive() {
|
||||
Debug("[HTTPClient] Connecting:", c.baseURL)
|
||||
if err = c.Connect(); err != nil {
|
||||
log.Println("[HTTPClient] Connection error:", err)
|
||||
@@ -269,16 +165,6 @@ func (c *HTTPClient) Send(data []byte) (response []byte, err error) {
|
||||
data = proto.SetHost(data, []byte(c.baseURL), []byte(c.host))
|
||||
}
|
||||
|
||||
if c.isProxy() && c.scheme == "http" {
|
||||
path := proto.Path(data)
|
||||
if len(path) > 0 && path[0] == '/' {
|
||||
data = proto.SetPath(data, c.proxyPath(path))
|
||||
if c.proxyAuth != "" {
|
||||
data = proto.SetHeader(data, []byte("Proxy-Authorization"), []byte(c.proxyAuth))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if c.auth != "" {
|
||||
data = proto.SetHeader(data, []byte("Authorization"), []byte(c.auth))
|
||||
}
|
||||
@@ -287,19 +173,13 @@ func (c *HTTPClient) Send(data []byte) (response []byte, err error) {
|
||||
Debug("[HTTPClient] Sending:", string(data))
|
||||
}
|
||||
|
||||
return c.send(data, readBytes, timeout)
|
||||
}
|
||||
|
||||
func (c *HTTPClient) send(data []byte, readBytes int, timeout time.Time) (response []byte, err error) {
|
||||
var payload []byte
|
||||
var n int
|
||||
if _, err = c.conn.Write(data); err != nil {
|
||||
Debug("[HTTPClient] Write error:", err, c.baseURL)
|
||||
response = errorPayload(HTTP_TIMEOUT)
|
||||
c.Disconnect()
|
||||
return
|
||||
}
|
||||
|
||||
var readBytes, n int
|
||||
var currentChunk []byte
|
||||
timeout = time.Now().Add(c.config.Timeout)
|
||||
chunked := false
|
||||
@@ -315,30 +195,27 @@ func (c *HTTPClient) send(data []byte, readBytes int, timeout time.Time) (respon
|
||||
readBytes += n
|
||||
chunks++
|
||||
|
||||
if err != nil {
|
||||
if err == io.EOF {
|
||||
err = nil
|
||||
}
|
||||
break
|
||||
}
|
||||
|
||||
// First chunk
|
||||
if chunked || contentLength != -1 {
|
||||
currentContentLength += n
|
||||
} else {
|
||||
// If headers are finished
|
||||
var firstEmptyLine = bytes.Index(c.respBuf[:readBytes], proto.EmptyLine)
|
||||
if firstEmptyLine != -1 {
|
||||
if bytes.Equal(proto.Header(c.respBuf[:readBytes], []byte("Transfer-Encoding")), []byte("chunked")) {
|
||||
if bytes.Contains(c.respBuf[:readBytes], proto.EmptyLine) {
|
||||
if bytes.Equal(proto.Header(c.respBuf, []byte("Transfer-Encoding")), []byte("chunked")) {
|
||||
chunked = true
|
||||
} else {
|
||||
status, _ := strconv.Atoi(string(proto.Status(c.respBuf[:readBytes])))
|
||||
// We want to soak up all 100 Continues received to get the real result code
|
||||
if status >= 100 && status < 200 {
|
||||
timeout = time.Now().Add(c.config.Timeout)
|
||||
var deleteLen = firstEmptyLine + len(proto.EmptyLine)
|
||||
copy(c.respBuf, c.respBuf[deleteLen:readBytes])
|
||||
readBytes -= deleteLen
|
||||
chunks--
|
||||
continue
|
||||
} else if status == 204 || status == 304 {
|
||||
status, _ := strconv.Atoi(string(proto.Status(c.respBuf)))
|
||||
if (status >= 100 && status < 200) || status == 204 || status == 304 {
|
||||
contentLength = 0
|
||||
break
|
||||
} else {
|
||||
l := proto.Header(c.respBuf[:readBytes], []byte("Content-Length"))
|
||||
l := proto.Header(c.respBuf, []byte("Content-Length"))
|
||||
if len(l) > 0 {
|
||||
contentLength, _ = strconv.Atoi(string(l))
|
||||
}
|
||||
@@ -363,13 +240,6 @@ func (c *HTTPClient) send(data []byte, readBytes int, timeout time.Time) (respon
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
if err == io.EOF {
|
||||
err = nil
|
||||
}
|
||||
break
|
||||
}
|
||||
} else {
|
||||
if currentChunk == nil {
|
||||
currentChunk = make([]byte, readChunkSize)
|
||||
@@ -377,6 +247,13 @@ func (c *HTTPClient) send(data []byte, readBytes int, timeout time.Time) (respon
|
||||
|
||||
n, err = c.conn.Read(currentChunk)
|
||||
|
||||
if err == io.EOF {
|
||||
break
|
||||
} else if err != nil {
|
||||
Debug("[HTTPClient] Read the whole body error:", err, c.baseURL)
|
||||
break
|
||||
}
|
||||
|
||||
readBytes += int(n)
|
||||
chunks++
|
||||
currentContentLength += n
|
||||
@@ -399,14 +276,6 @@ func (c *HTTPClient) send(data []byte, readBytes int, timeout time.Time) (respon
|
||||
c.Disconnect()
|
||||
break
|
||||
}
|
||||
|
||||
if err == io.EOF {
|
||||
break
|
||||
} else if err != nil {
|
||||
Debug("[HTTPClient] Read the whole body error:", err, c.baseURL)
|
||||
break
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
if readBytes >= maxResponseSize {
|
||||
@@ -419,28 +288,16 @@ func (c *HTTPClient) send(data []byte, readBytes int, timeout time.Time) (respon
|
||||
timeout = time.Now().Add(c.config.Timeout / 5)
|
||||
}
|
||||
|
||||
if err != nil && readBytes == 0 {
|
||||
Debug("[HTTPClient] Response read timeout error", err, c.conn, readBytes, string(c.respBuf[:readBytes]))
|
||||
if err != nil {
|
||||
Debug("[HTTPClient] Response read error", err, c.conn, readBytes)
|
||||
response = errorPayload(HTTP_TIMEOUT)
|
||||
c.Disconnect()
|
||||
return
|
||||
}
|
||||
|
||||
if readBytes < 4 || string(c.respBuf[:4]) != "HTTP" {
|
||||
maxRead := 100
|
||||
if readBytes < maxRead {
|
||||
maxRead = readBytes
|
||||
}
|
||||
Debug("[HTTPClient] Response read unknown error", err, c.conn, readBytes, string(c.respBuf[:maxRead]))
|
||||
response = errorPayload(HTTP_UNKNOWN_ERROR)
|
||||
c.Disconnect()
|
||||
return
|
||||
}
|
||||
|
||||
if readBytes > len(c.respBuf) {
|
||||
readBytes = len(c.respBuf)
|
||||
}
|
||||
payload = make([]byte, readBytes)
|
||||
payload := make([]byte, readBytes)
|
||||
copy(payload, c.respBuf[:readBytes])
|
||||
|
||||
if c.config.Debug {
|
||||
@@ -455,7 +312,7 @@ func (c *HTTPClient) send(data []byte, readBytes int, timeout time.Time) (respon
|
||||
c.redirectsCount++
|
||||
|
||||
location := proto.Header(payload, []byte("Location"))
|
||||
redirectPayload := proto.SetPath(data, location)
|
||||
redirectPayload := []byte("GET " + string(location) + " HTTP/1.1\r\n\r\n")
|
||||
|
||||
if c.config.Debug {
|
||||
Debug("[HTTPClient] Redirecting to: " + string(location))
|
||||
@@ -466,8 +323,8 @@ func (c *HTTPClient) send(data []byte, readBytes int, timeout time.Time) (respon
|
||||
}
|
||||
|
||||
if bytes.Equal(proto.Status(payload), []byte("400")) {
|
||||
Debug("[HTTPClient] Closed connection on 400 response")
|
||||
c.Disconnect()
|
||||
Debug("[HTTPClient] Closed connection on 400 response")
|
||||
}
|
||||
|
||||
c.redirectsCount = 0
|
||||
@@ -489,14 +346,6 @@ func (c *HTTPClient) Post(path string, body []byte) (response []byte, err error)
|
||||
return c.Send([]byte(payload))
|
||||
}
|
||||
|
||||
func (c *HTTPClient) proxyPath(path []byte) []byte {
|
||||
return append([]byte(c.scheme+"://"+c.host), path...)
|
||||
}
|
||||
|
||||
func (c *HTTPClient) isProxy() bool {
|
||||
return c.proxy != nil
|
||||
}
|
||||
|
||||
const (
|
||||
// https://support.cloudflare.com/hc/en-us/articles/200171936-Error-520-Web-server-is-returning-an-unknown-error
|
||||
HTTP_UNKNOWN_ERROR = "520"
|
||||
|
||||
+1
-50
@@ -3,6 +3,7 @@ package main
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/rand"
|
||||
"github.com/buger/gor/proto"
|
||||
"io/ioutil"
|
||||
_ "log"
|
||||
"net"
|
||||
@@ -14,8 +15,6 @@ import (
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/buger/goreplay/proto"
|
||||
)
|
||||
|
||||
func TestHTTPClientURLPort(t *testing.T) {
|
||||
@@ -87,25 +86,6 @@ func TestHTTPClientSend(t *testing.T) {
|
||||
client.Send(payload("POST"))
|
||||
|
||||
wg.Wait()
|
||||
|
||||
client = NewHTTPClient(server.URL, &HTTPClientConfig{Debug: true, CompatibilityMode: true})
|
||||
|
||||
wg.Add(4)
|
||||
|
||||
if _, err := client.Send(payload("POST")); err != nil {
|
||||
t.Fatal(err.Error())
|
||||
}
|
||||
if _, err := client.Send(payload("GET")); err != nil {
|
||||
t.Fatal(err.Error())
|
||||
}
|
||||
if _, err := client.Send(payload("POST_CHUNKED")); err != nil {
|
||||
t.Fatal(err.Error())
|
||||
}
|
||||
if _, err := client.Send(payload("POST")); err != nil {
|
||||
t.Fatal(err.Error())
|
||||
}
|
||||
|
||||
wg.Wait()
|
||||
}
|
||||
|
||||
func TestHTTPClientResonseByClose(t *testing.T) {
|
||||
@@ -377,35 +357,6 @@ func TestHTTPClientRedirectLimit(t *testing.T) {
|
||||
wg.Wait()
|
||||
}
|
||||
|
||||
func TestHTTPClientKeepHeadersRedirect(t *testing.T) {
|
||||
wg := new(sync.WaitGroup)
|
||||
|
||||
GETPayload := []byte("GET / HTTP/1.1\r\n\r\n")
|
||||
|
||||
GETPayload = proto.AddHeader(GETPayload, []byte("keep-header"), []byte("true"))
|
||||
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
if r.URL.Path == "/" {
|
||||
http.Redirect(w, r, "/new", 301)
|
||||
}
|
||||
|
||||
if r.Header.Get("keep-header") != "true" {
|
||||
t.Errorf("Header keep-header was incorrect, got: %s, want: %s.", r.Header.Get("keep-header"), "true")
|
||||
}
|
||||
|
||||
wg.Done()
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
client := NewHTTPClient(server.URL, &HTTPClientConfig{FollowRedirects: 1, Debug: false})
|
||||
|
||||
wg.Add(2)
|
||||
client.Send(GETPayload)
|
||||
|
||||
wg.Wait()
|
||||
}
|
||||
|
||||
func TestHTTPClientBasicAuth(t *testing.T) {
|
||||
wg := new(sync.WaitGroup)
|
||||
wg.Add(2)
|
||||
|
||||
+2
-41
@@ -2,11 +2,9 @@ package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/base64"
|
||||
"hash/fnv"
|
||||
"strings"
|
||||
|
||||
"github.com/buger/goreplay/proto"
|
||||
"github.com/buger/gor/proto"
|
||||
)
|
||||
|
||||
type HTTPModifier struct {
|
||||
@@ -18,10 +16,8 @@ func NewHTTPModifier(config *HTTPModifierConfig) *HTTPModifier {
|
||||
if len(config.urlRegexp) == 0 &&
|
||||
len(config.urlNegativeRegexp) == 0 &&
|
||||
len(config.urlRewrite) == 0 &&
|
||||
len(config.headerRewrite) == 0 &&
|
||||
len(config.headerFilters) == 0 &&
|
||||
len(config.headerNegativeFilters) == 0 &&
|
||||
len(config.headerBasicAuthFilters) == 0 &&
|
||||
len(config.headerHashFilters) == 0 &&
|
||||
len(config.paramHashFilters) == 0 &&
|
||||
len(config.params) == 0 &&
|
||||
@@ -98,11 +94,7 @@ func (m *HTTPModifier) Rewrite(payload []byte) (response []byte) {
|
||||
for _, f := range m.config.headerFilters {
|
||||
value := proto.Header(payload, f.name)
|
||||
|
||||
if len(value) == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
if !f.regexp.Match(value) {
|
||||
if len(value) > 0 && !f.regexp.Match(value) {
|
||||
return
|
||||
}
|
||||
}
|
||||
@@ -118,23 +110,6 @@ func (m *HTTPModifier) Rewrite(payload []byte) (response []byte) {
|
||||
}
|
||||
}
|
||||
|
||||
if len(m.config.headerBasicAuthFilters) > 0 {
|
||||
for _, f := range m.config.headerBasicAuthFilters {
|
||||
value := proto.Header(payload, []byte("Authorization"))
|
||||
|
||||
if len(value) > 0 {
|
||||
valueString := string(value)
|
||||
trimmedBasicAuthEncoded := strings.TrimPrefix(valueString, "Basic ")
|
||||
if strings.Compare(valueString, trimmedBasicAuthEncoded) != 0 {
|
||||
decodedAuth, _ := base64.StdEncoding.DecodeString(trimmedBasicAuthEncoded)
|
||||
if !f.regexp.Match(decodedAuth) {
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if len(m.config.headerHashFilters) > 0 {
|
||||
for _, f := range m.config.headerHashFilters {
|
||||
value := proto.Header(payload, f.name)
|
||||
@@ -178,19 +153,5 @@ func (m *HTTPModifier) Rewrite(payload []byte) (response []byte) {
|
||||
}
|
||||
}
|
||||
|
||||
if len(m.config.headerRewrite) > 0 {
|
||||
for _, f := range m.config.headerRewrite {
|
||||
value := proto.Header(payload, f.header)
|
||||
if len(value) == 0 {
|
||||
break
|
||||
}
|
||||
|
||||
if f.src.Match(value) {
|
||||
newValue := f.src.ReplaceAll(value, f.target)
|
||||
payload = proto.SetHeader(payload, f.header, newValue)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return payload
|
||||
}
|
||||
|
||||
@@ -10,15 +10,13 @@ import (
|
||||
|
||||
// HTTPModifierConfig holds configuration options for built-in traffic modifier
|
||||
type HTTPModifierConfig struct {
|
||||
urlNegativeRegexp HTTPUrlRegexp
|
||||
urlRegexp HTTPUrlRegexp
|
||||
urlRewrite UrlRewriteMap
|
||||
headerRewrite HeaderRewriteMap
|
||||
headerFilters HTTPHeaderFilters
|
||||
headerNegativeFilters HTTPHeaderFilters
|
||||
headerBasicAuthFilters HTTPHeaderBasicAuthFilters
|
||||
headerHashFilters HTTPHashFilters
|
||||
paramHashFilters HTTPHashFilters
|
||||
urlNegativeRegexp HTTPUrlRegexp
|
||||
urlRegexp HTTPUrlRegexp
|
||||
urlRewrite UrlRewriteMap
|
||||
headerFilters HTTPHeaderFilters
|
||||
headerNegativeFilters HTTPHeaderFilters
|
||||
headerHashFilters HTTPHashFilters
|
||||
paramHashFilters HTTPHashFilters
|
||||
|
||||
params HTTPParams
|
||||
headers HTTPHeaders
|
||||
@@ -56,31 +54,6 @@ func (h *HTTPHeaderFilters) Set(value string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
//
|
||||
// Handling of --http-basic-auth-filter option
|
||||
//
|
||||
type basicAuthFilter struct {
|
||||
regexp *regexp.Regexp
|
||||
}
|
||||
|
||||
// HTTPHeaderFilters holds list of headers and their regexps
|
||||
type HTTPHeaderBasicAuthFilters []basicAuthFilter
|
||||
|
||||
func (h *HTTPHeaderBasicAuthFilters) String() string {
|
||||
return fmt.Sprint(*h)
|
||||
}
|
||||
|
||||
func (h *HTTPHeaderBasicAuthFilters) Set(value string) error {
|
||||
r, err := regexp.Compile(value)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
*h = append(*h, basicAuthFilter{regexp: r})
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
//
|
||||
// Handling of --http-allow-header-hash and --http-allow-param-hash options
|
||||
//
|
||||
@@ -223,42 +196,6 @@ func (r *UrlRewriteMap) Set(value string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
//
|
||||
// Handling of --http-rewrite-header option
|
||||
//
|
||||
type headerRewrite struct {
|
||||
header []byte
|
||||
src *regexp.Regexp
|
||||
target []byte
|
||||
}
|
||||
|
||||
type HeaderRewriteMap []headerRewrite
|
||||
|
||||
func (r *HeaderRewriteMap) String() string {
|
||||
return fmt.Sprint(*r)
|
||||
}
|
||||
|
||||
func (r *HeaderRewriteMap) Set(value string) error {
|
||||
headerArr := strings.SplitN(value, ":", 2)
|
||||
if len(headerArr) < 2 {
|
||||
return errors.New("need both header, regexp and rewrite target, colon-delimited (ex. Header: regexp,target)")
|
||||
}
|
||||
|
||||
header := headerArr[0]
|
||||
valArr := strings.SplitN(strings.TrimSpace(headerArr[1]), ",", 2)
|
||||
|
||||
if len(valArr) < 2 {
|
||||
return errors.New("need both header, regexp and rewrite target, colon-delimited (ex. Header: regexp,target)")
|
||||
}
|
||||
|
||||
regexp, err := regexp.Compile(valArr[0])
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
*r = append(*r, headerRewrite{header: []byte(header), src: regexp, target: []byte(valArr[1])})
|
||||
return nil
|
||||
}
|
||||
|
||||
//
|
||||
// Handling of --http-allow-url option
|
||||
//
|
||||
|
||||
+1
-62
@@ -2,9 +2,8 @@ package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"github.com/buger/gor/proto"
|
||||
"testing"
|
||||
|
||||
"github.com/buger/goreplay/proto"
|
||||
)
|
||||
|
||||
func TestHTTPModifierWithoutConfig(t *testing.T) {
|
||||
@@ -79,45 +78,6 @@ func TestHTTPModifierHeaderNegativeFilters(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestHTTPHeaderBasicAuthFilters(t *testing.T) {
|
||||
filters := HTTPHeaderBasicAuthFilters{}
|
||||
filters.Set("^customer[0-9].*")
|
||||
|
||||
modifier := NewHTTPModifier(&HTTPModifierConfig{
|
||||
headerBasicAuthFilters: filters,
|
||||
})
|
||||
|
||||
//Encoded UserId:Password = customer3:welcome
|
||||
payload := []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nAuthorization: Basic Y3VzdG9tZXIzOndlbGNvbWU=\r\n\r\na=1&b=2")
|
||||
if len(modifier.Rewrite(payload)) == 0 {
|
||||
t.Error("Request should pass filters")
|
||||
}
|
||||
|
||||
//customer6:rest@123^TEST
|
||||
payload = []byte("POST /post HTTP/1.1\r\nContent-Length: 88\r\nAuthorization: Basic Y3VzdG9tZXI2OnJlc3RAMTIzXlRFU1Q==\r\n\r\na=1&b=2")
|
||||
if len(modifier.Rewrite(payload)) == 0 {
|
||||
t.Error("Request should pass filters")
|
||||
}
|
||||
|
||||
filters = HTTPHeaderBasicAuthFilters{}
|
||||
// Setting filter that not match our header
|
||||
filters.Set("^(homer simpson|mickey mouse).*")
|
||||
|
||||
modifier = NewHTTPModifier(&HTTPModifierConfig{
|
||||
headerBasicAuthFilters: filters,
|
||||
})
|
||||
|
||||
if len(modifier.Rewrite(payload)) != 0 {
|
||||
t.Error("Request should not pass filters")
|
||||
}
|
||||
|
||||
//mickey mouse:happy123
|
||||
payload = []byte("POST /post HTTP/1.1\r\nContent-Length: 88\r\nAuthorization: Basic bWlja2V5IG1vdXNlOmhhcHB5MTIz\r\n\r\na=1&b=2")
|
||||
if len(modifier.Rewrite(payload)) == 0 {
|
||||
t.Error("Request should pass filters")
|
||||
}
|
||||
}
|
||||
|
||||
func TestHTTPModifierURLRewrite(t *testing.T) {
|
||||
var url, newURL []byte
|
||||
|
||||
@@ -147,27 +107,6 @@ func TestHTTPModifierURLRewrite(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestHTTPModifierHeaderRewrite(t *testing.T) {
|
||||
var header, newHeader []byte
|
||||
|
||||
rewrites := HeaderRewriteMap{}
|
||||
payload := []byte("GET / HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
|
||||
|
||||
err := rewrites.Set("Host: (.*).w3.org,$1.beta.w3.org")
|
||||
if err != nil {
|
||||
t.Error("Should not error", err)
|
||||
}
|
||||
|
||||
modifier := NewHTTPModifier(&HTTPModifierConfig{
|
||||
headerRewrite: rewrites,
|
||||
})
|
||||
|
||||
header = []byte("www.beta.w3.org")
|
||||
if newHeader = proto.Header(modifier.Rewrite(payload), []byte("Host")); !bytes.Equal(newHeader, header) {
|
||||
t.Error("Request header should have been rewritten, wasn't", string(newHeader), string(header))
|
||||
}
|
||||
}
|
||||
|
||||
func TestHTTPModifierHeaderHashFilters(t *testing.T) {
|
||||
filters := HTTPHashFilters{}
|
||||
filters.Set("Header2:1/2")
|
||||
|
||||
+47
-54
@@ -1,75 +1,68 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"io/ioutil"
|
||||
"net/http/httputil"
|
||||
"strconv"
|
||||
|
||||
"github.com/buger/goreplay/proto"
|
||||
"github.com/buger/gor/proto"
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"strconv"
|
||||
"io/ioutil"
|
||||
"net/http/httputil"
|
||||
)
|
||||
|
||||
func prettifyHTTP(p []byte) []byte {
|
||||
headSize := bytes.IndexByte(p, '\n') + 1
|
||||
head := p[:headSize]
|
||||
body := p[headSize:]
|
||||
headSize := bytes.IndexByte(p, '\n') + 1
|
||||
head := p[:headSize]
|
||||
body := p[headSize:]
|
||||
|
||||
headersPos := proto.MIMEHeadersEndPos(body)
|
||||
headersPos := proto.MIMEHeadersEndPos(body)
|
||||
headers := body[:headersPos]
|
||||
content := body[headersPos:]
|
||||
|
||||
if headersPos < 5 || headersPos > len(body) {
|
||||
return p
|
||||
}
|
||||
var tEnc, cEnc []byte
|
||||
proto.ParseHeaders([][]byte{headers}, func(header, value []byte) bool {
|
||||
if proto.HeadersEqual(header, []byte("Transfer-Encoding")) {
|
||||
tEnc = value
|
||||
}
|
||||
|
||||
headers := body[:headersPos]
|
||||
content := body[headersPos:]
|
||||
if proto.HeadersEqual(header, []byte("Content-Encoding")) {
|
||||
cEnc = value
|
||||
}
|
||||
|
||||
var tEnc, cEnc []byte
|
||||
proto.ParseHeaders([][]byte{headers}, func(header, value []byte) bool {
|
||||
if proto.HeadersEqual(header, []byte("Transfer-Encoding")) {
|
||||
tEnc = value
|
||||
}
|
||||
return true
|
||||
})
|
||||
|
||||
if proto.HeadersEqual(header, []byte("Content-Encoding")) {
|
||||
cEnc = value
|
||||
}
|
||||
if len(tEnc) == 0 && len(cEnc) == 0 {
|
||||
return p
|
||||
}
|
||||
|
||||
return true
|
||||
})
|
||||
if bytes.Equal(tEnc, []byte("chunked")) {
|
||||
buf := bytes.NewBuffer(content)
|
||||
r := httputil.NewChunkedReader(buf)
|
||||
content, _ = ioutil.ReadAll(r)
|
||||
|
||||
if len(tEnc) == 0 && len(cEnc) == 0 {
|
||||
return p
|
||||
}
|
||||
headers = proto.DeleteHeader(headers, []byte("Transfer-Encoding"))
|
||||
|
||||
if bytes.Equal(tEnc, []byte("chunked")) {
|
||||
buf := bytes.NewBuffer(content)
|
||||
r := httputil.NewChunkedReader(buf)
|
||||
content, _ = ioutil.ReadAll(r)
|
||||
newLen := strconv.Itoa(len(content))
|
||||
headers = proto.SetHeader(headers, []byte("Content-Length"), []byte(newLen))
|
||||
}
|
||||
|
||||
headers = proto.DeleteHeader(headers, []byte("Transfer-Encoding"))
|
||||
if bytes.Equal(cEnc, []byte("gzip")) {
|
||||
buf := bytes.NewBuffer(content)
|
||||
g, err := gzip.NewReader(buf)
|
||||
|
||||
newLen := strconv.Itoa(len(content))
|
||||
headers = proto.SetHeader(headers, []byte("Content-Length"), []byte(newLen))
|
||||
}
|
||||
if err != nil {
|
||||
Debug("[Prettifier] GZIP encoding error:", err)
|
||||
}
|
||||
|
||||
if bytes.Equal(cEnc, []byte("gzip")) {
|
||||
buf := bytes.NewBuffer(content)
|
||||
g, err := gzip.NewReader(buf)
|
||||
content, _ = ioutil.ReadAll(g)
|
||||
|
||||
if err != nil {
|
||||
Debug("[Prettifier] GZIP encoding error:", err)
|
||||
return []byte{}
|
||||
}
|
||||
headers = proto.DeleteHeader(headers, []byte("Content-Encoding"))
|
||||
|
||||
content, _ = ioutil.ReadAll(g)
|
||||
newLen := strconv.Itoa(len(content))
|
||||
headers = proto.SetHeader(headers, []byte("Content-Length"), []byte(newLen))
|
||||
}
|
||||
|
||||
headers = proto.DeleteHeader(headers, []byte("Content-Encoding"))
|
||||
newPayload := append(append(head, headers...), content...)
|
||||
|
||||
newLen := strconv.Itoa(len(content))
|
||||
headers = proto.SetHeader(headers, []byte("Content-Length"), []byte(newLen))
|
||||
}
|
||||
|
||||
newPayload := append(append(head, headers...), content...)
|
||||
|
||||
return newPayload
|
||||
}
|
||||
return newPayload
|
||||
}
|
||||
+21
-21
@@ -1,36 +1,36 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"strconv"
|
||||
"testing"
|
||||
"compress/gzip"
|
||||
"testing"
|
||||
"strconv"
|
||||
"bytes"
|
||||
)
|
||||
|
||||
func TestHTTPPrettifierGzip(t *testing.T) {
|
||||
b := bytes.NewBufferString("")
|
||||
w := gzip.NewWriter(b)
|
||||
w.Write([]byte("test"))
|
||||
w.Close()
|
||||
b := bytes.NewBufferString("")
|
||||
w := gzip.NewWriter(b)
|
||||
w.Write([]byte("test"))
|
||||
w.Close()
|
||||
|
||||
size := strconv.Itoa(len(b.Bytes()))
|
||||
size := strconv.Itoa(len(b.Bytes()))
|
||||
|
||||
payload := []byte("2 1 1\nHTTP/1.1 200 OK\r\nContent-Length: " + size + "\r\nContent-Encoding: gzip\r\n\r\n")
|
||||
payload = append(payload, b.Bytes()...)
|
||||
payload := []byte("2 1 1\nHTTP/1.1 200 OK\r\nContent-Length: " + size + "\r\nContent-Encoding: gzip\r\n\r\n")
|
||||
payload = append(payload, b.Bytes()...)
|
||||
|
||||
newPayload := prettifyHTTP(payload)
|
||||
newPayload := prettifyHTTP(payload)
|
||||
|
||||
if string(newPayload) != "2 1 1\nHTTP/1.1 200 OK\r\nContent-Length: 4\r\n\r\ntest" {
|
||||
t.Error("Payload not match:", string(newPayload))
|
||||
}
|
||||
if string(newPayload) != "2 1 1\nHTTP/1.1 200 OK\r\nContent-Length: 4\r\n\r\ntest" {
|
||||
t.Error("Payload not match:", string(newPayload))
|
||||
}
|
||||
}
|
||||
|
||||
func TestHTTPPrettifierChunked(t *testing.T) {
|
||||
payload := []byte("POST / HTTP/1.1\r\nHost: www.w3.org\r\nTransfer-Encoding: chunked\r\n\r\n4\r\nWiki\r\n5\r\npedia\r\ne\r\n in\r\n\r\nchunks.\r\n0\r\n\r\n")
|
||||
payload := []byte("POST / HTTP/1.1\r\nHost: www.w3.org\r\nTransfer-Encoding: chunked\r\n\r\n4\r\nWiki\r\n5\r\npedia\r\ne\r\n in\r\n\r\nchunks.\r\n0\r\n\r\n")
|
||||
|
||||
newPayload := prettifyHTTP(payload)
|
||||
newPayload := prettifyHTTP(payload)
|
||||
|
||||
if string(newPayload) != "POST / HTTP/1.1\r\nHost: www.w3.org\r\nContent-Length: 23\r\n\r\nWikipedia in\r\n\r\nchunks." {
|
||||
t.Error("Payload not match:", string(newPayload))
|
||||
}
|
||||
}
|
||||
if string(newPayload) != "POST / HTTP/1.1\r\nHost: www.w3.org\r\nContent-Length: 23\r\n\r\nWikipedia in\r\n\r\nchunks." {
|
||||
t.Error("Payload not match:", string(newPayload))
|
||||
}
|
||||
}
|
||||
+22
-68
@@ -12,21 +12,14 @@ import (
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/aws/session"
|
||||
"github.com/aws/aws-sdk-go/service/s3"
|
||||
)
|
||||
|
||||
type fileInputReader struct {
|
||||
reader *bufio.Reader
|
||||
data []byte
|
||||
file io.ReadCloser
|
||||
file *os.File
|
||||
timestamp int64
|
||||
closed int32 // Value of 0 indicates that the file is still open.
|
||||
s3 bool
|
||||
}
|
||||
|
||||
func (f *fileInputReader) parseNext() error {
|
||||
@@ -43,7 +36,8 @@ func (f *fileInputReader) parseNext() error {
|
||||
}
|
||||
|
||||
if err == io.EOF {
|
||||
f.Close()
|
||||
f.file.Close()
|
||||
f.file = nil
|
||||
return err
|
||||
}
|
||||
}
|
||||
@@ -70,8 +64,7 @@ func (f *fileInputReader) ReadPayload() []byte {
|
||||
return f.data
|
||||
}
|
||||
func (f *fileInputReader) Close() error {
|
||||
if atomic.LoadInt32(&f.closed) == 0 {
|
||||
atomic.StoreInt32(&f.closed, 1)
|
||||
if f.file != nil {
|
||||
f.file.Close()
|
||||
}
|
||||
|
||||
@@ -79,21 +72,14 @@ func (f *fileInputReader) Close() error {
|
||||
}
|
||||
|
||||
func NewFileInputReader(path string) *fileInputReader {
|
||||
var file io.ReadCloser
|
||||
var err error
|
||||
|
||||
if strings.HasPrefix(path, "s3://") {
|
||||
file = NewS3ReadCloser(path)
|
||||
} else {
|
||||
file, err = os.Open(path)
|
||||
}
|
||||
file, err := os.Open(path)
|
||||
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
return nil
|
||||
}
|
||||
|
||||
r := &fileInputReader{file: file, closed: 0}
|
||||
r := &fileInputReader{file: file}
|
||||
if strings.HasSuffix(path, ".gz") {
|
||||
gzReader, err := gzip.NewReader(file)
|
||||
if err != nil {
|
||||
@@ -125,7 +111,7 @@ type FileInput struct {
|
||||
func NewFileInput(path string, loop bool) (i *FileInput) {
|
||||
i = new(FileInput)
|
||||
i.data = make(chan []byte, 1000)
|
||||
i.exit = make(chan bool)
|
||||
i.exit = make(chan bool, 1)
|
||||
i.path = path
|
||||
i.speedFactor = 1
|
||||
i.loop = loop
|
||||
@@ -139,37 +125,21 @@ func NewFileInput(path string, loop bool) (i *FileInput) {
|
||||
return
|
||||
}
|
||||
|
||||
type NextFileNotFound struct{}
|
||||
|
||||
func (_ *NextFileNotFound) Error() string {
|
||||
return "There is no new files"
|
||||
}
|
||||
|
||||
func (i *FileInput) init() (err error) {
|
||||
defer i.mu.Unlock()
|
||||
i.mu.Lock()
|
||||
|
||||
var matches []string
|
||||
|
||||
if strings.HasPrefix(i.path, "s3://") {
|
||||
sess := session.Must(session.NewSession(awsConfig()))
|
||||
svc := s3.New(sess)
|
||||
|
||||
bucket, key := parseS3Url(i.path)
|
||||
|
||||
params := &s3.ListObjectsInput{
|
||||
Bucket: aws.String(bucket),
|
||||
Prefix: aws.String(key),
|
||||
}
|
||||
|
||||
resp, err := svc.ListObjects(params)
|
||||
if err != nil {
|
||||
log.Println("Error while retreiving list of files from S3", i.path, err)
|
||||
return err
|
||||
}
|
||||
|
||||
for _, c := range resp.Contents {
|
||||
matches = append(matches, "s3://"+bucket+"/"+(*c.Key))
|
||||
}
|
||||
} else {
|
||||
if matches, err = filepath.Glob(i.path); err != nil {
|
||||
log.Println("Wrong file pattern", i.path, err)
|
||||
return
|
||||
}
|
||||
if matches, err = filepath.Glob(i.path); err != nil {
|
||||
log.Println("Wrong file pattern", i.path, err)
|
||||
return
|
||||
}
|
||||
|
||||
if len(matches) == 0 {
|
||||
@@ -187,13 +157,9 @@ func (i *FileInput) init() (err error) {
|
||||
}
|
||||
|
||||
func (i *FileInput) Read(data []byte) (int, error) {
|
||||
var buf []byte
|
||||
select {
|
||||
case <-i.exit:
|
||||
return 0, ErrorStopped
|
||||
case buf = <-i.data:
|
||||
}
|
||||
buf := <-i.data
|
||||
copy(data, buf)
|
||||
|
||||
return len(buf), nil
|
||||
}
|
||||
|
||||
@@ -204,7 +170,7 @@ func (i *FileInput) String() string {
|
||||
// Find reader with smallest timestamp e.g next payload in row
|
||||
func (i *FileInput) nextReader() (next *fileInputReader) {
|
||||
for _, r := range i.readers {
|
||||
if r == nil || atomic.LoadInt32(&r.closed) != 0 {
|
||||
if r == nil || r.file == nil {
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -252,30 +218,18 @@ func (i *FileInput) emit() {
|
||||
lastTime = reader.timestamp
|
||||
}
|
||||
|
||||
// Recheck if we have exited since last check.
|
||||
select {
|
||||
case <-i.exit:
|
||||
return
|
||||
default:
|
||||
i.data <- reader.ReadPayload()
|
||||
}
|
||||
i.data <- reader.ReadPayload()
|
||||
}
|
||||
|
||||
log.Printf("FileInput: end of file '%s'\n", i.path)
|
||||
|
||||
// For now having fixed timeout is temporary solution
|
||||
// Further should be modified, so outputs can report if their queue empty or not
|
||||
time.Sleep(time.Second)
|
||||
if closeCh != nil {
|
||||
close(closeCh)
|
||||
}
|
||||
}
|
||||
|
||||
func (i *FileInput) Close() error {
|
||||
defer i.mu.Unlock()
|
||||
i.mu.Lock()
|
||||
|
||||
close(i.exit)
|
||||
i.exit <- true
|
||||
|
||||
for _, r := range i.readers {
|
||||
r.Close()
|
||||
}
|
||||
|
||||
+15
-19
@@ -18,6 +18,7 @@ import (
|
||||
var _ = log.Println
|
||||
|
||||
func TestInputFileWithGET(t *testing.T) {
|
||||
|
||||
input := NewTestInput()
|
||||
rg := NewRequestGenerator([]io.Reader{input}, func() { input.EmitGET() }, 1)
|
||||
readPayloads := [][]byte{}
|
||||
@@ -304,6 +305,7 @@ func (expectedCaptureFile *CaptureFile) PayloadsEqual(other [][]byte) bool {
|
||||
}
|
||||
|
||||
func CreateCaptureFile(requestGenerator *RequestGenerator) *CaptureFile {
|
||||
|
||||
f, err := ioutil.TempFile("", "testmainconf")
|
||||
if err != nil {
|
||||
panic(err)
|
||||
@@ -314,34 +316,31 @@ func CreateCaptureFile(requestGenerator *RequestGenerator) *CaptureFile {
|
||||
readPayloads := [][]byte{}
|
||||
output := NewTestOutput(func(data []byte) {
|
||||
readPayloads = append(readPayloads, Duplicate(data))
|
||||
|
||||
requestGenerator.wg.Done()
|
||||
})
|
||||
|
||||
outputFile := NewFileOutput(f.Name(), &FileOutputConfig{flushInterval: time.Minute, append: true})
|
||||
|
||||
plugins := &InOutPlugins{
|
||||
Inputs: requestGenerator.inputs,
|
||||
Outputs: []io.Writer{output, outputFile},
|
||||
}
|
||||
for _, input := range requestGenerator.inputs {
|
||||
plugins.All = append(plugins.All, input)
|
||||
}
|
||||
plugins.All = append(plugins.All, output, outputFile)
|
||||
Plugins.Inputs = requestGenerator.inputs
|
||||
Plugins.Outputs = []io.Writer{output, outputFile}
|
||||
|
||||
emitter := NewEmitter(quit)
|
||||
go emitter.Start(plugins, Settings.Middleware)
|
||||
go Start(quit)
|
||||
|
||||
requestGenerator.emit()
|
||||
requestGenerator.wg.Wait()
|
||||
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
emitter.Close()
|
||||
outputFile.Close()
|
||||
|
||||
close(quit)
|
||||
|
||||
return NewExpectedCaptureFile(readPayloads, f)
|
||||
|
||||
}
|
||||
|
||||
func ReadFromCaptureFile(captureFile *os.File, count int, callback writeCallback) (err error) {
|
||||
|
||||
quit := make(chan int)
|
||||
wg := new(sync.WaitGroup)
|
||||
|
||||
@@ -351,15 +350,11 @@ func ReadFromCaptureFile(captureFile *os.File, count int, callback writeCallback
|
||||
wg.Done()
|
||||
})
|
||||
|
||||
plugins := &InOutPlugins{
|
||||
Inputs: []io.Reader{input},
|
||||
Outputs: []io.Writer{output},
|
||||
}
|
||||
plugins.All = append(plugins.All, input, output)
|
||||
Plugins.Inputs = []io.Reader{input}
|
||||
Plugins.Outputs = []io.Writer{output}
|
||||
|
||||
wg.Add(count)
|
||||
emitter := NewEmitter(quit)
|
||||
go emitter.Start(plugins, Settings.Middleware)
|
||||
go Start(quit)
|
||||
|
||||
done := make(chan int, 1)
|
||||
go func() {
|
||||
@@ -373,7 +368,8 @@ func ReadFromCaptureFile(captureFile *os.File, count int, callback writeCallback
|
||||
case <-time.After(2 * time.Second):
|
||||
err = errors.New("Timed out")
|
||||
}
|
||||
emitter.close()
|
||||
close(quit)
|
||||
|
||||
return
|
||||
|
||||
}
|
||||
|
||||
+1
-13
@@ -13,7 +13,6 @@ type HTTPInput struct {
|
||||
data chan []byte
|
||||
address string
|
||||
listener net.Listener
|
||||
stop chan bool // Channel used only to indicate goroutine should shutdown
|
||||
}
|
||||
|
||||
// NewHTTPInput constructor for HTTPInput. Accepts address with port which he will listen on.
|
||||
@@ -21,7 +20,6 @@ func NewHTTPInput(address string) (i *HTTPInput) {
|
||||
i = new(HTTPInput)
|
||||
i.data = make(chan []byte, 10000)
|
||||
i.address = address
|
||||
i.stop = make(chan bool)
|
||||
|
||||
i.listen(address)
|
||||
|
||||
@@ -29,12 +27,7 @@ func NewHTTPInput(address string) (i *HTTPInput) {
|
||||
}
|
||||
|
||||
func (i *HTTPInput) Read(data []byte) (int, error) {
|
||||
var buf []byte
|
||||
select {
|
||||
case <-i.stop:
|
||||
return 0, ErrorStopped
|
||||
case buf = <-i.data:
|
||||
}
|
||||
buf := <-i.data
|
||||
|
||||
header := payloadHeader(RequestPayload, uuid(), time.Now().UnixNano(), -1)
|
||||
|
||||
@@ -44,11 +37,6 @@ func (i *HTTPInput) Read(data []byte) (int, error) {
|
||||
return len(buf) + len(header), nil
|
||||
}
|
||||
|
||||
func (i *HTTPInput) Close() error {
|
||||
close(i.stop)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (i *HTTPInput) handler(w http.ResponseWriter, r *http.Request) {
|
||||
r.URL.Scheme = "http"
|
||||
r.URL.Host = i.listener.Addr().String()
|
||||
|
||||
+10
-18
@@ -1,6 +1,7 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"github.com/buger/gor/proto"
|
||||
"io"
|
||||
"log"
|
||||
"net/http"
|
||||
@@ -8,8 +9,6 @@ import (
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"github.com/buger/goreplay/proto"
|
||||
)
|
||||
|
||||
func TestHTTPInput(t *testing.T) {
|
||||
@@ -21,14 +20,10 @@ func TestHTTPInput(t *testing.T) {
|
||||
wg.Done()
|
||||
})
|
||||
|
||||
plugins := &InOutPlugins{
|
||||
Inputs: []io.Reader{input},
|
||||
Outputs: []io.Writer{output},
|
||||
}
|
||||
plugins.All = append(plugins.All, input, output)
|
||||
Plugins.Inputs = []io.Reader{input}
|
||||
Plugins.Outputs = []io.Writer{output}
|
||||
|
||||
emitter := NewEmitter(quit)
|
||||
go emitter.Start(plugins, Settings.Middleware)
|
||||
go Start(quit)
|
||||
|
||||
address := strings.Replace(input.listener.Addr().String(), "[::]", "127.0.0.1", -1)
|
||||
|
||||
@@ -38,7 +33,8 @@ func TestHTTPInput(t *testing.T) {
|
||||
}
|
||||
|
||||
wg.Wait()
|
||||
emitter.Close()
|
||||
|
||||
close(quit)
|
||||
}
|
||||
|
||||
func TestInputHTTPLargePayload(t *testing.T) {
|
||||
@@ -58,14 +54,10 @@ func TestInputHTTPLargePayload(t *testing.T) {
|
||||
}
|
||||
wg.Done()
|
||||
})
|
||||
plugins := &InOutPlugins{
|
||||
Inputs: []io.Reader{input},
|
||||
Outputs: []io.Writer{output},
|
||||
}
|
||||
plugins.All = append(plugins.All, input, output)
|
||||
Plugins.Inputs = []io.Reader{input}
|
||||
Plugins.Outputs = []io.Writer{output}
|
||||
|
||||
emitter := NewEmitter(quit)
|
||||
go emitter.Start(plugins, Settings.Middleware)
|
||||
go Start(quit)
|
||||
|
||||
wg.Add(1)
|
||||
address := strings.Replace(input.listener.Addr().String(), "[::]", "127.0.0.1", -1)
|
||||
@@ -76,5 +68,5 @@ func TestInputHTTPLargePayload(t *testing.T) {
|
||||
}
|
||||
|
||||
wg.Wait()
|
||||
emitter.Close()
|
||||
close(quit)
|
||||
}
|
||||
|
||||
+17
-21
@@ -2,11 +2,9 @@ package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"log"
|
||||
"strings"
|
||||
|
||||
"github.com/Shopify/sarama"
|
||||
"github.com/Shopify/sarama/mocks"
|
||||
"log"
|
||||
)
|
||||
|
||||
// KafkaInput is used for recieving Kafka messages and
|
||||
@@ -24,12 +22,11 @@ func NewKafkaInput(address string, config *KafkaConfig) *KafkaInput {
|
||||
|
||||
var con sarama.Consumer
|
||||
|
||||
if mock, ok := config.consumer.(*mocks.Consumer); ok && mock != nil {
|
||||
if config.consumer.(*mocks.Consumer) != nil {
|
||||
con = config.consumer
|
||||
} else {
|
||||
var err error
|
||||
//con, err = sarama.NewConsumer([]string{config.host}, c)
|
||||
con, err = sarama.NewConsumer(strings.Split(config.host, ","), c)
|
||||
con, err = sarama.NewConsumer([]string{config.host}, c)
|
||||
|
||||
if err != nil {
|
||||
log.Fatalln("Failed to start Sarama(Kafka) consumer:", err)
|
||||
@@ -61,7 +58,7 @@ func NewKafkaInput(address string, config *KafkaConfig) *KafkaInput {
|
||||
}
|
||||
}(consumer)
|
||||
|
||||
if Settings.Verbose {
|
||||
if Settings.verbose {
|
||||
// Start infinite loop for tracking errors for kafka producer.
|
||||
go i.ErrorHandler(consumer)
|
||||
}
|
||||
@@ -85,21 +82,20 @@ func (i *KafkaInput) Read(data []byte) (int, error) {
|
||||
if !i.config.useJSON {
|
||||
copy(data, message.Value)
|
||||
return len(message.Value), nil
|
||||
} else {
|
||||
var kafkaMessage KafkaMessage
|
||||
json.Unmarshal(message.Value, &kafkaMessage)
|
||||
|
||||
buf, err := kafkaMessage.Dump()
|
||||
if err != nil {
|
||||
log.Println("Failed to decode access log entry:", err)
|
||||
return 0, err
|
||||
}
|
||||
|
||||
copy(data, buf)
|
||||
|
||||
return len(buf), nil
|
||||
}
|
||||
|
||||
var kafkaMessage KafkaMessage
|
||||
json.Unmarshal(message.Value, &kafkaMessage)
|
||||
|
||||
buf, err := kafkaMessage.Dump()
|
||||
if err != nil {
|
||||
log.Println("Failed to decode access log entry:", err)
|
||||
return 0, err
|
||||
}
|
||||
|
||||
copy(data, buf)
|
||||
|
||||
return len(buf), nil
|
||||
|
||||
}
|
||||
|
||||
func (i *KafkaInput) String() string {
|
||||
|
||||
+1
-2
@@ -1,10 +1,9 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/Shopify/sarama"
|
||||
"github.com/Shopify/sarama/mocks"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestInputKafkaRAW(t *testing.T) {
|
||||
|
||||
+23
-37
@@ -1,12 +1,11 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"github.com/buger/gor/proto"
|
||||
raw "github.com/buger/gor/raw_socket_listener"
|
||||
"log"
|
||||
"net"
|
||||
"time"
|
||||
|
||||
raw "github.com/buger/goreplay/capture"
|
||||
"github.com/buger/goreplay/proto"
|
||||
)
|
||||
|
||||
// RAWInput used for intercepting traffic for given address
|
||||
@@ -14,15 +13,11 @@ type RAWInput struct {
|
||||
data chan *raw.TCPMessage
|
||||
address string
|
||||
expire time.Duration
|
||||
quit chan bool // Channel used only to indicate goroutine should shutdown
|
||||
quit chan bool
|
||||
engine int
|
||||
realIPHeader []byte
|
||||
trackResponse bool
|
||||
listener *raw.Listener
|
||||
protocol raw.TCPProtocol
|
||||
bpfFilter string
|
||||
timestampType string
|
||||
bufferSize int64
|
||||
}
|
||||
|
||||
// Available engines for intercepting traffic
|
||||
@@ -33,44 +28,24 @@ const (
|
||||
)
|
||||
|
||||
// NewRAWInput constructor for RAWInput. Accepts address with port as argument.
|
||||
func NewRAWInput(address string, engine int, trackResponse bool, expire time.Duration, realIPHeader string, protocol string, bpfFilter string, timestampType string, bufferSize int64) (i *RAWInput) {
|
||||
func NewRAWInput(address string, engine int, trackResponse bool, expire time.Duration, realIPHeader string) (i *RAWInput) {
|
||||
i = new(RAWInput)
|
||||
i.data = make(chan *raw.TCPMessage)
|
||||
i.address = address
|
||||
i.expire = expire
|
||||
i.engine = engine
|
||||
i.bpfFilter = bpfFilter
|
||||
i.realIPHeader = []byte(realIPHeader)
|
||||
i.quit = make(chan bool)
|
||||
i.trackResponse = trackResponse
|
||||
i.timestampType = timestampType
|
||||
i.bufferSize = bufferSize
|
||||
|
||||
switch protocol {
|
||||
case "http":
|
||||
i.protocol = raw.ProtocolHTTP
|
||||
case "binary":
|
||||
i.protocol = raw.ProtocolBinary
|
||||
if !PRO {
|
||||
log.Fatal("Binary protocols can be used only with PRO license")
|
||||
}
|
||||
default:
|
||||
log.Fatal("Unsupported protocol:", protocol)
|
||||
}
|
||||
|
||||
i.listen(address)
|
||||
i.listener.IsReady()
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
func (i *RAWInput) Read(data []byte) (int, error) {
|
||||
var msg *raw.TCPMessage
|
||||
select {
|
||||
case <-i.quit:
|
||||
return 0, ErrorStopped
|
||||
case msg = <-i.data:
|
||||
}
|
||||
|
||||
msg := <-i.data
|
||||
buf := msg.Bytes()
|
||||
|
||||
var header []byte
|
||||
@@ -81,7 +56,7 @@ func (i *RAWInput) Read(data []byte) (int, error) {
|
||||
buf = proto.SetHeader(buf, i.realIPHeader, []byte(msg.IP().String()))
|
||||
}
|
||||
} else {
|
||||
header = payloadHeader(ResponsePayload, msg.UUID(), msg.Start.UnixNano(), msg.End.UnixNano()-msg.AssocMessage.End.UnixNano())
|
||||
header = payloadHeader(ResponsePayload, msg.UUID(), msg.AssocMessage.Start.UnixNano(), msg.End.UnixNano()-msg.AssocMessage.Start.UnixNano())
|
||||
}
|
||||
|
||||
copy(data[0:len(header)], header)
|
||||
@@ -94,11 +69,18 @@ func (i *RAWInput) listen(address string) {
|
||||
Debug("Listening for traffic on: " + address)
|
||||
|
||||
host, port, err := net.SplitHostPort(address)
|
||||
if err != nil {
|
||||
log.Fatalf("input-raw: error while parsing address: %s", err)
|
||||
|
||||
if i.engine == EnginePcapFile {
|
||||
host = address
|
||||
port = "1"
|
||||
err = nil
|
||||
}
|
||||
|
||||
i.listener = raw.NewListener(host, port, i.engine, i.trackResponse, i.expire, i.protocol, i.bpfFilter, i.timestampType, i.bufferSize, Settings.InputRAWOverrideSnapLen, Settings.InputRAWImmediateMode)
|
||||
if err != nil {
|
||||
log.Fatal("input-raw: error while parsing address", err)
|
||||
}
|
||||
|
||||
i.listener = raw.NewListener(host, port, i.engine, i.trackResponse, i.expire)
|
||||
|
||||
ch := i.listener.Receiver()
|
||||
|
||||
@@ -107,8 +89,13 @@ func (i *RAWInput) listen(address string) {
|
||||
select {
|
||||
case <-i.quit:
|
||||
return
|
||||
case i.data <- <-ch: // Receiving TCPMessage object
|
||||
default:
|
||||
}
|
||||
|
||||
// Receiving TCPMessage object
|
||||
m := <-ch
|
||||
|
||||
i.data <- m
|
||||
}
|
||||
}()
|
||||
}
|
||||
@@ -117,7 +104,6 @@ func (i *RAWInput) String() string {
|
||||
return "Intercepting traffic from: " + i.address
|
||||
}
|
||||
|
||||
// Close closes the input raw listener
|
||||
func (i *RAWInput) Close() error {
|
||||
i.listener.Close()
|
||||
close(i.quit)
|
||||
|
||||
+50
-83
@@ -2,6 +2,7 @@ package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"github.com/buger/gor/proto"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"log"
|
||||
@@ -18,8 +19,6 @@ import (
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/buger/goreplay/proto"
|
||||
)
|
||||
|
||||
const testRawExpire = time.Millisecond * 200
|
||||
@@ -44,7 +43,7 @@ func TestRAWInputIPv4(t *testing.T) {
|
||||
|
||||
var respCounter, reqCounter int64
|
||||
|
||||
input := NewRAWInput(originAddr, EnginePcap, true, testRawExpire, "X-Real-IP", "http", "", "", 0)
|
||||
input := NewRAWInput(originAddr, EnginePcap, true, testRawExpire, "X-Real-IP")
|
||||
defer input.Close()
|
||||
|
||||
output := NewTestOutput(func(data []byte) {
|
||||
@@ -58,23 +57,19 @@ func TestRAWInputIPv4(t *testing.T) {
|
||||
atomic.AddInt64(&respCounter, 1)
|
||||
}
|
||||
|
||||
if Settings.Debug {
|
||||
if Settings.debug {
|
||||
log.Println(reqCounter, respCounter)
|
||||
}
|
||||
|
||||
wg.Done()
|
||||
})
|
||||
|
||||
plugins := &InOutPlugins{
|
||||
Inputs: []io.Reader{input},
|
||||
Outputs: []io.Writer{output},
|
||||
}
|
||||
plugins.All = append(plugins.All, input, output)
|
||||
Plugins.Inputs = []io.Reader{input}
|
||||
Plugins.Outputs = []io.Writer{output}
|
||||
|
||||
client := NewHTTPClient("http://"+listener.Addr().String(), &HTTPClientConfig{})
|
||||
|
||||
emitter := NewEmitter(quit)
|
||||
go emitter.Start(plugins, Settings.Middleware)
|
||||
go Start(quit)
|
||||
|
||||
for i := 0; i < 100; i++ {
|
||||
// request + response
|
||||
@@ -84,7 +79,8 @@ func TestRAWInputIPv4(t *testing.T) {
|
||||
}
|
||||
|
||||
wg.Wait()
|
||||
emitter.Close()
|
||||
|
||||
close(quit)
|
||||
}
|
||||
|
||||
func TestRAWInputNoKeepAlive(t *testing.T) {
|
||||
@@ -109,23 +105,19 @@ func TestRAWInputNoKeepAlive(t *testing.T) {
|
||||
|
||||
originAddr := listener.Addr().String()
|
||||
|
||||
input := NewRAWInput(originAddr, EnginePcap, true, testRawExpire, "", "http", "", "", 0)
|
||||
input := NewRAWInput(originAddr, EnginePcap, true, testRawExpire, "")
|
||||
defer input.Close()
|
||||
|
||||
output := NewTestOutput(func(data []byte) {
|
||||
wg.Done()
|
||||
})
|
||||
|
||||
plugins := &InOutPlugins{
|
||||
Inputs: []io.Reader{input},
|
||||
Outputs: []io.Writer{output},
|
||||
}
|
||||
plugins.All = append(plugins.All, input, output)
|
||||
Plugins.Inputs = []io.Reader{input}
|
||||
Plugins.Outputs = []io.Writer{output}
|
||||
|
||||
client := NewHTTPClient("http://"+listener.Addr().String(), &HTTPClientConfig{})
|
||||
|
||||
emitter := NewEmitter(quit)
|
||||
go emitter.Start(plugins, Settings.Middleware)
|
||||
go Start(quit)
|
||||
|
||||
for i := 0; i < 100; i++ {
|
||||
// request + response
|
||||
@@ -135,7 +127,8 @@ func TestRAWInputNoKeepAlive(t *testing.T) {
|
||||
}
|
||||
|
||||
wg.Wait()
|
||||
emitter.Close()
|
||||
|
||||
close(quit)
|
||||
}
|
||||
|
||||
func TestRAWInputIPv6(t *testing.T) {
|
||||
@@ -158,7 +151,7 @@ func TestRAWInputIPv6(t *testing.T) {
|
||||
|
||||
var respCounter, reqCounter int64
|
||||
|
||||
input := NewRAWInput(originAddr, EnginePcap, true, testRawExpire, "", "http", "", "", 0)
|
||||
input := NewRAWInput(originAddr, EnginePcap, true, testRawExpire, "")
|
||||
defer input.Close()
|
||||
|
||||
output := NewTestOutput(func(data []byte) {
|
||||
@@ -168,23 +161,19 @@ func TestRAWInputIPv6(t *testing.T) {
|
||||
atomic.AddInt64(&respCounter, 1)
|
||||
}
|
||||
|
||||
if Settings.Debug {
|
||||
if Settings.debug {
|
||||
log.Println(reqCounter, respCounter)
|
||||
}
|
||||
|
||||
wg.Done()
|
||||
})
|
||||
|
||||
plugins := &InOutPlugins{
|
||||
Inputs: []io.Reader{input},
|
||||
Outputs: []io.Writer{output},
|
||||
}
|
||||
plugins.All = append(plugins.All, input, output)
|
||||
Plugins.Inputs = []io.Reader{input}
|
||||
Plugins.Outputs = []io.Writer{output}
|
||||
|
||||
client := NewHTTPClient("http://"+listener.Addr().String(), &HTTPClientConfig{})
|
||||
|
||||
emitter := NewEmitter(quit)
|
||||
go emitter.Start(plugins, Settings.Middleware)
|
||||
go Start(quit)
|
||||
|
||||
for i := 0; i < 100; i++ {
|
||||
// request + response
|
||||
@@ -194,7 +183,7 @@ func TestRAWInputIPv6(t *testing.T) {
|
||||
}
|
||||
|
||||
wg.Wait()
|
||||
emitter.Close()
|
||||
close(quit)
|
||||
}
|
||||
|
||||
func TestInputRAW100Expect(t *testing.T) {
|
||||
@@ -213,7 +202,7 @@ func TestInputRAW100Expect(t *testing.T) {
|
||||
|
||||
originAddr := strings.Replace(origin.Listener.Addr().String(), "[::]", "127.0.0.1", -1)
|
||||
|
||||
input := NewRAWInput(originAddr, EnginePcap, true, time.Second, "", "http", "", "", 0)
|
||||
input := NewRAWInput(originAddr, EnginePcap, true, time.Second, "")
|
||||
defer input.Close()
|
||||
|
||||
// We will use it to get content of raw HTTP request
|
||||
@@ -244,14 +233,10 @@ func TestInputRAW100Expect(t *testing.T) {
|
||||
|
||||
httpOutput := NewHTTPOutput(replay.URL, &HTTPOutputConfig{})
|
||||
|
||||
plugins := &InOutPlugins{
|
||||
Inputs: []io.Reader{input},
|
||||
Outputs: []io.Writer{testOutput, httpOutput},
|
||||
}
|
||||
plugins.All = append(plugins.All, input, testOutput, httpOutput)
|
||||
Plugins.Inputs = []io.Reader{input}
|
||||
Plugins.Outputs = []io.Writer{testOutput, httpOutput}
|
||||
|
||||
emitter := NewEmitter(quit)
|
||||
go emitter.Start(plugins, Settings.Middleware)
|
||||
go Start(quit)
|
||||
|
||||
// Origin + Response/Request Test Output + Request Http Output
|
||||
wg.Add(4)
|
||||
@@ -262,7 +247,7 @@ func TestInputRAW100Expect(t *testing.T) {
|
||||
}
|
||||
|
||||
wg.Wait()
|
||||
emitter.Close()
|
||||
close(quit)
|
||||
}
|
||||
|
||||
func TestInputRAWChunkedEncoding(t *testing.T) {
|
||||
@@ -280,7 +265,7 @@ func TestInputRAWChunkedEncoding(t *testing.T) {
|
||||
}))
|
||||
|
||||
originAddr := strings.Replace(origin.Listener.Addr().String(), "[::]", "127.0.0.1", -1)
|
||||
input := NewRAWInput(originAddr, EnginePcap, true, time.Second, "", "http", "", "", 0)
|
||||
input := NewRAWInput(originAddr, EnginePcap, true, time.Second, "")
|
||||
defer input.Close()
|
||||
|
||||
replay := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
@@ -298,14 +283,11 @@ func TestInputRAWChunkedEncoding(t *testing.T) {
|
||||
|
||||
httpOutput := NewHTTPOutput(replay.URL, &HTTPOutputConfig{Debug: true})
|
||||
|
||||
plugins := &InOutPlugins{
|
||||
Inputs: []io.Reader{input},
|
||||
Outputs: []io.Writer{httpOutput},
|
||||
}
|
||||
plugins.All = append(plugins.All, input, httpOutput)
|
||||
Plugins.Inputs = []io.Reader{input}
|
||||
Plugins.Outputs = []io.Writer{httpOutput}
|
||||
|
||||
go Start(quit)
|
||||
|
||||
emitter := NewEmitter(quit)
|
||||
go emitter.Start(plugins, Settings.Middleware)
|
||||
wg.Add(2)
|
||||
|
||||
curl := exec.Command("curl", "http://"+originAddr, "--header", "Transfer-Encoding: chunked", "--header", "Expect:", "--data-binary", "@README.md")
|
||||
@@ -315,7 +297,8 @@ func TestInputRAWChunkedEncoding(t *testing.T) {
|
||||
}
|
||||
|
||||
wg.Wait()
|
||||
emitter.Close()
|
||||
|
||||
close(quit)
|
||||
}
|
||||
|
||||
func TestInputRAWLargePayload(t *testing.T) {
|
||||
@@ -346,7 +329,7 @@ func TestInputRAWLargePayload(t *testing.T) {
|
||||
}))
|
||||
originAddr := strings.Replace(origin.Listener.Addr().String(), "[::]", "127.0.0.1", -1)
|
||||
|
||||
input := NewRAWInput(originAddr, EnginePcap, true, testRawExpire, "", "http", "", "", 0)
|
||||
input := NewRAWInput(originAddr, EnginePcap, true, testRawExpire, "")
|
||||
defer input.Close()
|
||||
|
||||
replay := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
|
||||
@@ -366,14 +349,10 @@ func TestInputRAWLargePayload(t *testing.T) {
|
||||
|
||||
httpOutput := NewHTTPOutput(replay.URL, &HTTPOutputConfig{Debug: false})
|
||||
|
||||
plugins := &InOutPlugins{
|
||||
Inputs: []io.Reader{input},
|
||||
Outputs: []io.Writer{httpOutput},
|
||||
}
|
||||
plugins.All = append(plugins.All, input, httpOutput)
|
||||
Plugins.Inputs = []io.Reader{input}
|
||||
Plugins.Outputs = []io.Writer{httpOutput}
|
||||
|
||||
emitter := NewEmitter(quit)
|
||||
go emitter.Start(plugins, Settings.Middleware)
|
||||
go Start(quit)
|
||||
|
||||
wg.Add(2)
|
||||
curl := exec.Command("curl", "http://"+originAddr, "--header", "Transfer-Encoding: chunked", "--header", "Expect:", "--data-binary", "@/tmp/large")
|
||||
@@ -383,25 +362,19 @@ func TestInputRAWLargePayload(t *testing.T) {
|
||||
}
|
||||
|
||||
wg.Wait()
|
||||
emitter.Close()
|
||||
close(quit)
|
||||
}
|
||||
|
||||
func BenchmarkRAWInput(b *testing.B) {
|
||||
var respCounter, reqCounter, replayCounter int64
|
||||
|
||||
quit := make(chan int)
|
||||
|
||||
origin := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {}))
|
||||
defer origin.Close()
|
||||
originAddr := strings.Replace(origin.Listener.Addr().String(), "[::]", "127.0.0.1", -1)
|
||||
|
||||
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
atomic.AddInt64(&replayCounter, 1)
|
||||
}))
|
||||
defer origin.Close()
|
||||
upstreamAddr := strings.Replace(upstream.Listener.Addr().String(), "[::]", "127.0.0.1", -1)
|
||||
var respCounter, reqCounter int64
|
||||
|
||||
input := NewRAWInput(originAddr, EnginePcap, true, testRawExpire, "", "http", "", "", 0)
|
||||
input := NewRAWInput(originAddr, EnginePcap, true, testRawExpire, "")
|
||||
defer input.Close()
|
||||
|
||||
output := NewTestOutput(func(data []byte) {
|
||||
@@ -410,33 +383,27 @@ func BenchmarkRAWInput(b *testing.B) {
|
||||
} else {
|
||||
atomic.AddInt64(&respCounter, 1)
|
||||
}
|
||||
|
||||
// log.Println("Captured ", reqCounter, "requests and ", respCounter, " responses")
|
||||
})
|
||||
|
||||
httpOutput := NewLimiter(NewHTTPOutput(upstreamAddr, &HTTPOutputConfig{}), "10%")
|
||||
Plugins.Inputs = []io.Reader{input}
|
||||
Plugins.Outputs = []io.Writer{output}
|
||||
|
||||
plugins := &InOutPlugins{
|
||||
Inputs: []io.Reader{input},
|
||||
Outputs: []io.Writer{output, httpOutput},
|
||||
}
|
||||
plugins.All = append(plugins.All, input, output, httpOutput)
|
||||
|
||||
emitter := NewEmitter(quit)
|
||||
go emitter.Start(plugins, Settings.Middleware)
|
||||
go Start(quit)
|
||||
|
||||
emitted := 0
|
||||
fileContent, _ := ioutil.ReadFile("LICENSE.txt")
|
||||
|
||||
time.Sleep(400 * time.Millisecond)
|
||||
|
||||
for i := 0; i < b.N; i++ {
|
||||
wg := new(sync.WaitGroup)
|
||||
wg.Add(100 * 1000)
|
||||
emitted += 100 * 1000
|
||||
wg.Add(10 * 100)
|
||||
emitted += 10 * 100
|
||||
for w := 0; w < 100; w++ {
|
||||
go func() {
|
||||
client := NewHTTPClient(origin.URL, &HTTPClientConfig{})
|
||||
for i := 0; i < 1000; i++ {
|
||||
if i%2 == 0 {
|
||||
for i := 0; i < 10; i++ {
|
||||
if rand.Int63n(2) == 0 {
|
||||
client.Post("/", fileContent)
|
||||
} else {
|
||||
client.Get("/")
|
||||
@@ -450,7 +417,7 @@ func BenchmarkRAWInput(b *testing.B) {
|
||||
}
|
||||
|
||||
time.Sleep(400 * time.Millisecond)
|
||||
log.Println("Emitted ", emitted, ", Captured ", reqCounter, "requests and ", respCounter, " responses", "and replayed", replayCounter)
|
||||
log.Println("Emitted ", emitted, ", Captured ", reqCounter, "requests and ", respCounter, " responses")
|
||||
|
||||
emitter.Close()
|
||||
close(quit)
|
||||
}
|
||||
|
||||
+8
-43
@@ -3,7 +3,6 @@ package main
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"crypto/tls"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
@@ -14,25 +13,15 @@ import (
|
||||
// TCPInput used for internal communication
|
||||
type TCPInput struct {
|
||||
data chan []byte
|
||||
listener net.Listener
|
||||
address string
|
||||
config *TCPInputConfig
|
||||
stop chan bool // Channel used only to indicate goroutine should shutdown
|
||||
}
|
||||
|
||||
type TCPInputConfig struct {
|
||||
Secure bool
|
||||
CertificatePath string
|
||||
KeyPath string
|
||||
listener net.Listener
|
||||
}
|
||||
|
||||
// NewTCPInput constructor for TCPInput, accepts address with port
|
||||
func NewTCPInput(address string, config *TCPInputConfig) (i *TCPInput) {
|
||||
func NewTCPInput(address string) (i *TCPInput) {
|
||||
i = new(TCPInput)
|
||||
i.data = make(chan []byte, 1000)
|
||||
i.address = address
|
||||
i.config = config
|
||||
i.stop = make(chan bool)
|
||||
|
||||
i.listen(address)
|
||||
|
||||
@@ -40,47 +29,23 @@ func NewTCPInput(address string, config *TCPInputConfig) (i *TCPInput) {
|
||||
}
|
||||
|
||||
func (i *TCPInput) Read(data []byte) (int, error) {
|
||||
var buf []byte
|
||||
select {
|
||||
case <-i.stop:
|
||||
return 0, ErrorStopped
|
||||
case buf = <-i.data:
|
||||
}
|
||||
buf := <-i.data
|
||||
copy(data, buf)
|
||||
|
||||
return len(buf), nil
|
||||
}
|
||||
|
||||
func (i *TCPInput) Close() error {
|
||||
close(i.stop)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (i *TCPInput) listen(address string) {
|
||||
if i.config.Secure {
|
||||
cer, err := tls.LoadX509KeyPair(i.config.CertificatePath, i.config.KeyPath)
|
||||
if err != nil {
|
||||
log.Fatal("Error while loading --input-file certificate:", err)
|
||||
}
|
||||
listener, err := net.Listen("tcp", address)
|
||||
i.listener = listener
|
||||
|
||||
config := &tls.Config{Certificates: []tls.Certificate{cer}}
|
||||
listener, err := tls.Listen("tcp", address, config)
|
||||
if err != nil {
|
||||
log.Fatal("Can't start --input-tcp with Secure connection:", err)
|
||||
}
|
||||
i.listener = listener
|
||||
} else {
|
||||
listener, err := net.Listen("tcp", address)
|
||||
if err != nil {
|
||||
log.Fatal("Can't start:", err)
|
||||
}
|
||||
|
||||
i.listener = listener
|
||||
if err != nil {
|
||||
log.Fatal("Can't start:", err)
|
||||
}
|
||||
|
||||
go func() {
|
||||
for {
|
||||
conn, err := i.listener.Accept()
|
||||
conn, err := listener.Accept()
|
||||
|
||||
if err != nil {
|
||||
log.Println("Error while Accept()", err)
|
||||
|
||||
+5
-99
@@ -1,40 +1,26 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/rand"
|
||||
"crypto/rsa"
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"encoding/pem"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"log"
|
||||
"math/big"
|
||||
"net"
|
||||
"os"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestTCPInput(t *testing.T) {
|
||||
wg := new(sync.WaitGroup)
|
||||
quit := make(chan int)
|
||||
|
||||
input := NewTCPInput("127.0.0.1:0", &TCPInputConfig{})
|
||||
input := NewTCPInput("127.0.0.1:0")
|
||||
output := NewTestOutput(func(data []byte) {
|
||||
wg.Done()
|
||||
})
|
||||
|
||||
plugins := &InOutPlugins{
|
||||
Inputs: []io.Reader{input},
|
||||
Outputs: []io.Writer{output},
|
||||
}
|
||||
plugins.All = append(plugins.All, input, output)
|
||||
Plugins.Inputs = []io.Reader{input}
|
||||
Plugins.Outputs = []io.Writer{output}
|
||||
|
||||
emitter := NewEmitter(quit)
|
||||
go emitter.Start(plugins, Settings.Middleware)
|
||||
go Start(quit)
|
||||
|
||||
tcpAddr, err := net.ResolveTCPAddr("tcp", input.listener.Addr().String())
|
||||
|
||||
@@ -57,86 +43,6 @@ func TestTCPInput(t *testing.T) {
|
||||
}
|
||||
|
||||
wg.Wait()
|
||||
emitter.Close()
|
||||
}
|
||||
|
||||
func genCertificate(template *x509.Certificate) ([]byte, []byte) {
|
||||
priv, _ := rsa.GenerateKey(rand.Reader, 2048)
|
||||
|
||||
serialNumberLimit := new(big.Int).Lsh(big.NewInt(1), 128)
|
||||
serialNumber, _ := rand.Int(rand.Reader, serialNumberLimit)
|
||||
template.SerialNumber = serialNumber
|
||||
template.BasicConstraintsValid = true
|
||||
template.NotBefore = time.Now()
|
||||
template.NotAfter = time.Now().Add(time.Hour)
|
||||
|
||||
derBytes, _ := x509.CreateCertificate(rand.Reader, template, template, &priv.PublicKey, priv)
|
||||
|
||||
var certPem, keyPem bytes.Buffer
|
||||
pem.Encode(&certPem, &pem.Block{Type: "CERTIFICATE", Bytes: derBytes})
|
||||
pem.Encode(&keyPem, &pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(priv)})
|
||||
|
||||
return certPem.Bytes(), keyPem.Bytes()
|
||||
}
|
||||
|
||||
func TestTCPInputSecure(t *testing.T) {
|
||||
serverCertPem, serverPrivPem := genCertificate(&x509.Certificate{
|
||||
DNSNames: []string{"localhost"},
|
||||
IPAddresses: []net.IP{net.ParseIP("127.0.0.1"), net.ParseIP("::")},
|
||||
})
|
||||
|
||||
serverCertPemFile, _ := ioutil.TempFile("", "server.crt")
|
||||
serverCertPemFile.Write(serverCertPem)
|
||||
serverCertPemFile.Close()
|
||||
|
||||
serverPrivPemFile, _ := ioutil.TempFile("", "server.key")
|
||||
serverPrivPemFile.Write(serverPrivPem)
|
||||
serverPrivPemFile.Close()
|
||||
|
||||
defer func() {
|
||||
os.Remove(serverPrivPemFile.Name())
|
||||
os.Remove(serverCertPemFile.Name())
|
||||
}()
|
||||
|
||||
wg := new(sync.WaitGroup)
|
||||
quit := make(chan int)
|
||||
|
||||
input := NewTCPInput("127.0.0.1:0", &TCPInputConfig{
|
||||
Secure: true,
|
||||
CertificatePath: serverCertPemFile.Name(),
|
||||
KeyPath: serverPrivPemFile.Name(),
|
||||
})
|
||||
output := NewTestOutput(func(data []byte) {
|
||||
wg.Done()
|
||||
})
|
||||
|
||||
plugins := &InOutPlugins{
|
||||
Inputs: []io.Reader{input},
|
||||
Outputs: []io.Writer{output},
|
||||
}
|
||||
plugins.All = append(plugins.All, input, output)
|
||||
|
||||
emitter := NewEmitter(quit)
|
||||
go emitter.Start(plugins, Settings.Middleware)
|
||||
|
||||
conf := &tls.Config{
|
||||
InsecureSkipVerify: true,
|
||||
}
|
||||
|
||||
conn, err := tls.Dial("tcp", input.listener.Addr().String(), conf)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer conn.Close()
|
||||
|
||||
msg := []byte("1 1 1\nGET / HTTP/1.1\r\n\r\n")
|
||||
|
||||
for i := 0; i < 100; i++ {
|
||||
wg.Add(1)
|
||||
conn.Write(msg)
|
||||
conn.Write([]byte(payloadSeparator))
|
||||
}
|
||||
|
||||
wg.Wait()
|
||||
emitter.Close()
|
||||
close(quit)
|
||||
}
|
||||
|
||||
@@ -3,9 +3,8 @@ package main
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
|
||||
"github.com/Shopify/sarama"
|
||||
"github.com/buger/goreplay/proto"
|
||||
"github.com/buger/gor/proto"
|
||||
)
|
||||
|
||||
// KafkaConfig should contains required information to
|
||||
|
||||
+2
-9
@@ -77,6 +77,7 @@ func (l *Limiter) Write(data []byte) (n int, err error) {
|
||||
}
|
||||
|
||||
n, err = l.plugin.(io.Writer).Write(data)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
@@ -95,13 +96,5 @@ func (l *Limiter) Read(data []byte) (n int, err error) {
|
||||
}
|
||||
|
||||
func (l *Limiter) String() string {
|
||||
return fmt.Sprintf("Limiting %s to: %d (isPercent: %v)", l.plugin, l.limit, l.isPercent)
|
||||
}
|
||||
|
||||
// Close closes the resources.
|
||||
func (l *Limiter) Close() error {
|
||||
if fi, ok := l.plugin.(io.ReadCloser); ok {
|
||||
fi.Close()
|
||||
}
|
||||
return nil
|
||||
return fmt.Sprintf("Limiting %s to: %d (isPercent: %b)", l.plugin, l.limit, l.isPercent)
|
||||
}
|
||||
|
||||
+20
-32
@@ -18,21 +18,18 @@ func TestOutputLimiter(t *testing.T) {
|
||||
}), "10")
|
||||
wg.Add(10)
|
||||
|
||||
plugins := &InOutPlugins{
|
||||
Inputs: []io.Reader{input},
|
||||
Outputs: []io.Writer{output},
|
||||
}
|
||||
plugins.All = append(plugins.All, input, output)
|
||||
Plugins.Inputs = []io.Reader{input}
|
||||
Plugins.Outputs = []io.Writer{output}
|
||||
|
||||
emitter := NewEmitter(quit)
|
||||
go emitter.Start(plugins, Settings.Middleware)
|
||||
go Start(quit)
|
||||
|
||||
for i := 0; i < 100; i++ {
|
||||
input.EmitGET()
|
||||
}
|
||||
|
||||
wg.Wait()
|
||||
emitter.Close()
|
||||
|
||||
close(quit)
|
||||
}
|
||||
|
||||
func TestInputLimiter(t *testing.T) {
|
||||
@@ -45,21 +42,18 @@ func TestInputLimiter(t *testing.T) {
|
||||
})
|
||||
wg.Add(10)
|
||||
|
||||
plugins := &InOutPlugins{
|
||||
Inputs: []io.Reader{input},
|
||||
Outputs: []io.Writer{output},
|
||||
}
|
||||
plugins.All = append(plugins.All, input, output)
|
||||
Plugins.Inputs = []io.Reader{input}
|
||||
Plugins.Outputs = []io.Writer{output}
|
||||
|
||||
emitter := NewEmitter(quit)
|
||||
go emitter.Start(plugins, Settings.Middleware)
|
||||
go Start(quit)
|
||||
|
||||
for i := 0; i < 100; i++ {
|
||||
input.(*Limiter).plugin.(*TestInput).EmitGET()
|
||||
}
|
||||
|
||||
wg.Wait()
|
||||
emitter.Close()
|
||||
|
||||
close(quit)
|
||||
}
|
||||
|
||||
// Should limit all requests
|
||||
@@ -72,21 +66,18 @@ func TestPercentLimiter1(t *testing.T) {
|
||||
wg.Done()
|
||||
}), "0%")
|
||||
|
||||
plugins := &InOutPlugins{
|
||||
Inputs: []io.Reader{input},
|
||||
Outputs: []io.Writer{output},
|
||||
}
|
||||
plugins.All = append(plugins.All, input, output)
|
||||
Plugins.Inputs = []io.Reader{input}
|
||||
Plugins.Outputs = []io.Writer{output}
|
||||
|
||||
emitter := NewEmitter(quit)
|
||||
go emitter.Start(plugins, Settings.Middleware)
|
||||
go Start(quit)
|
||||
|
||||
for i := 0; i < 100; i++ {
|
||||
input.EmitGET()
|
||||
}
|
||||
|
||||
wg.Wait()
|
||||
emitter.Close()
|
||||
|
||||
close(quit)
|
||||
}
|
||||
|
||||
// Should not limit at all
|
||||
@@ -100,19 +91,16 @@ func TestPercentLimiter2(t *testing.T) {
|
||||
}), "100%")
|
||||
wg.Add(100)
|
||||
|
||||
plugins := &InOutPlugins{
|
||||
Inputs: []io.Reader{input},
|
||||
Outputs: []io.Writer{output},
|
||||
}
|
||||
plugins.All = append(plugins.All, input, output)
|
||||
Plugins.Inputs = []io.Reader{input}
|
||||
Plugins.Outputs = []io.Writer{output}
|
||||
|
||||
emitter := NewEmitter(quit)
|
||||
go emitter.Start(plugins, Settings.Middleware)
|
||||
go Start(quit)
|
||||
|
||||
for i := 0; i < 100; i++ {
|
||||
input.EmitGET()
|
||||
}
|
||||
|
||||
wg.Wait()
|
||||
emitter.Close()
|
||||
|
||||
close(quit)
|
||||
}
|
||||
|
||||
+17
-49
@@ -21,15 +21,12 @@ type Middleware struct {
|
||||
|
||||
Stdin io.Writer
|
||||
Stdout io.Reader
|
||||
|
||||
stop chan bool // Channel used only to indicate goroutine should shutdown
|
||||
}
|
||||
|
||||
func NewMiddleware(command string) *Middleware {
|
||||
m := new(Middleware)
|
||||
m.command = command
|
||||
m.data = make(chan []byte, 1000)
|
||||
m.stop = make(chan bool)
|
||||
|
||||
commands := strings.Split(command, " ")
|
||||
cmd := exec.Command(commands[0], commands[1:]...)
|
||||
@@ -37,7 +34,9 @@ func NewMiddleware(command string) *Middleware {
|
||||
m.Stdout, _ = cmd.StdoutPipe()
|
||||
m.Stdin, _ = cmd.StdinPipe()
|
||||
|
||||
cmd.Stderr = os.Stderr
|
||||
if Settings.verbose {
|
||||
cmd.Stderr = os.Stderr
|
||||
}
|
||||
|
||||
go m.read(m.Stdout)
|
||||
|
||||
@@ -65,40 +64,23 @@ func (m *Middleware) ReadFrom(plugin io.Reader) {
|
||||
|
||||
func (m *Middleware) copy(to io.Writer, from io.Reader) {
|
||||
buf := make([]byte, 5*1024*1024)
|
||||
dst := make([]byte, len(buf)*4)
|
||||
dst := make([]byte, len(buf)*2)
|
||||
|
||||
for {
|
||||
nr, _ := from.Read(buf)
|
||||
if nr == 0 || nr > len(buf) {
|
||||
continue
|
||||
}
|
||||
if nr > 0 && len(buf) > nr {
|
||||
|
||||
payload := buf[0:nr]
|
||||
hex.Encode(dst, buf[0:nr])
|
||||
dst[nr*2] = '\n'
|
||||
|
||||
if Settings.PrettifyHTTP {
|
||||
payload = prettifyHTTP(payload)
|
||||
nr = len(payload)
|
||||
m.mu.Lock()
|
||||
to.Write(dst[0 : nr*2+1])
|
||||
m.mu.Unlock()
|
||||
|
||||
if nr*2 > len(dst) {
|
||||
continue
|
||||
if Settings.debug {
|
||||
Debug("[MIDDLEWARE-MASTER] Sending:", string(buf[0:nr]), "From:", from)
|
||||
}
|
||||
}
|
||||
|
||||
if Settings.PrettifyHTTP {
|
||||
payload = prettifyHTTP(payload)
|
||||
nr = len(payload)
|
||||
}
|
||||
|
||||
hex.Encode(dst, payload)
|
||||
dst[nr*2] = '\n'
|
||||
|
||||
m.mu.Lock()
|
||||
to.Write(dst[0 : nr*2+1])
|
||||
m.mu.Unlock()
|
||||
|
||||
if Settings.Debug {
|
||||
Debug("[MIDDLEWARE-MASTER] Sending:", string(buf[0:nr]), "From:", from)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -118,40 +100,26 @@ func (m *Middleware) read(from io.Reader) {
|
||||
|
||||
buf := make([]byte, len(line)/2)
|
||||
if _, err := hex.Decode(buf, line[:len(line)-1]); err != nil {
|
||||
fmt.Fprintln(os.Stderr, "Failed to decode input payload", err, len(line), string(line[:len(line)-1]))
|
||||
fmt.Fprintln(os.Stderr, "Failed to decode input payload", err, len(line))
|
||||
}
|
||||
|
||||
if Settings.Debug {
|
||||
if Settings.debug {
|
||||
Debug("[MIDDLEWARE-MASTER] Received:", string(buf))
|
||||
}
|
||||
|
||||
select {
|
||||
case <-m.stop:
|
||||
return
|
||||
case m.data <- buf:
|
||||
}
|
||||
m.data <- buf
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
func (m *Middleware) Read(data []byte) (int, error) {
|
||||
var buf []byte
|
||||
select {
|
||||
case <-m.stop:
|
||||
return 0, ErrorStopped
|
||||
case buf = <-m.data:
|
||||
}
|
||||
|
||||
buf := <-m.data
|
||||
copy(data, buf)
|
||||
|
||||
return len(buf), nil
|
||||
}
|
||||
|
||||
func (m *Middleware) String() string {
|
||||
return fmt.Sprintf("Modifying traffic using '%s' command", m.command)
|
||||
}
|
||||
|
||||
func (m *Middleware) Close() error {
|
||||
close(m.stop)
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -1,122 +0,0 @@
|
||||
# GoReplay middleware
|
||||
|
||||
GoReplay support protocol for writing middleware in any language, which allows you to implement custom logic like authentification or complex rewriting and filterting. See protocol description here: https://github.com/buger/goreplay/wiki/Middleware, but the basic idea that middleware process receive hex encoded data via STDIN and emits it back via STDOUT. STDERR for loggin inside middleware. Yes, that's simple.
|
||||
|
||||
To simplify middleware creation we provide packages for NodeJS and Go (upcoming).
|
||||
|
||||
If you want to get access to original and replayed responses, do not forget adding `--output-http-track-response` and `--input-raw-track-response` options.
|
||||
|
||||
## NodeJS
|
||||
|
||||
Before starting, you should install the package via npm: `npm install goreplay_middleware`.
|
||||
And initialize middleware the following way:
|
||||
```javascript
|
||||
var gor = require("goreplay_middleware");
|
||||
// `init` will initialize STDIN listener
|
||||
gor.init();
|
||||
```
|
||||
|
||||
Basic idea is that you write callbacks which respond to `request`, `response`, `replay`, or `message` events, which contain request meta information and actuall http paylod. Depending on your needs you may compare, override or filter incoming requests and responses.
|
||||
|
||||
You can respond to the incoming events using `on` function, by providing callbacks:
|
||||
```javascript
|
||||
// valid events are `request`, `response` (original response), `replay` (replayed response), and `message` (all events)
|
||||
gor.on('request', function(data) {
|
||||
// `data` contains incoming message its meta information.
|
||||
data
|
||||
|
||||
// Raw HTTP payload of `Buffer` type
|
||||
// Example (hidden character for line endings shown on purpose):
|
||||
// GET / HTTP/1.1\r\n
|
||||
// User-Agent: Golang\r\n
|
||||
// \r\n
|
||||
data.http
|
||||
|
||||
// Meta is an array size of 4, containing:
|
||||
// 1. request type - 1, 2 or 3 (which maps to `request`, `respose` and `replay`)
|
||||
// 2. uuid - request unique identifier. Request responses have the same ID as their request.
|
||||
// 3. timestamp of when request was made (for responses it is time of request start too)
|
||||
// 4. latency - time difference between request start and finish. For `request` is zero.
|
||||
data.meta
|
||||
|
||||
// Unique request ID. It should be same for `request`, `response` and `replay` events of the same request.
|
||||
data.ID
|
||||
|
||||
// You should return data at the end of function, even if you not changed request, if you do not want to filter it out.
|
||||
// If you just `return` nothing, request will be filtered
|
||||
return data
|
||||
})
|
||||
```
|
||||
### Mapping requests and responses
|
||||
You can provide request ID as additional argument to `on` function, which allow you to map related requests and responses. Below is example of middleware which checks that original and replayed response have same HTTP status code.
|
||||
|
||||
```javascript
|
||||
// Example of very basic way to compare if replayed traffic have no errors
|
||||
gor.on("request", function(req) {
|
||||
gor.on("response", req.ID, function(resp) {
|
||||
gor.on("replay", req.ID, function(repl) {
|
||||
if (gor.httpStatus(resp.http) != gor.httpStatus(repl.http)) {
|
||||
// Note that STDERR is used for logging, and it actually will be send to `Gor` STDOUT.
|
||||
// This trick is used because SDTIN and STDOUT already used for process communication.
|
||||
// You can write logger that writes to files insead.
|
||||
console.error(`${gor.httpPath(req.http)} STATUS NOT MATCH: 'Expected ${gor.httpStatus(resp.http)}' got '${gor.httpStatus(repl.http)}'`)
|
||||
}
|
||||
return repl;
|
||||
})
|
||||
return resp;
|
||||
})
|
||||
return req
|
||||
})
|
||||
```
|
||||
|
||||
This middleware includes `searchResponses` helper which is used to compare value of original response with the replayed response. If authentication system or xsrf protection returns unique tokens in headers or the response, it will be helpful to rewrite your requests based on them. Because tokens are unique, and the value contained in original and replayed responses will be different. So, you need to extract value from both responses, and rewrite requests based on those mappings.
|
||||
|
||||
`searchResponses` accepts request id, regexp pattern for searching the compared value (should include capture group), and callback which returns both original and replayed matched value.
|
||||
|
||||
Example:
|
||||
```javascript
|
||||
// Compare HTTP headers for response and replayed response, and map values
|
||||
let tokMap = {};
|
||||
|
||||
gor.on("request", function(req) {
|
||||
let tok = gor.httpHeader(req.http, "Auth-Token");
|
||||
if (tok && tokMap[tok]) {
|
||||
req.http = gor.setHttpHeader(req.http, "Auth-Token", tokMap[tok])
|
||||
}
|
||||
|
||||
gor.searchResponses(req.ID, "X-Set-Token: (\w+)$", function(respTok, replTok) {
|
||||
if (respTok && replTok) tokMap[respTok] = replTok;
|
||||
})
|
||||
|
||||
return req;
|
||||
})
|
||||
```
|
||||
|
||||
|
||||
### API documentation
|
||||
|
||||
Package expose following functions to process raw HTTP payloads:
|
||||
* `init` - initialize middleware object, start reading from STDIN.
|
||||
* `httpPath` - URL path of the request: `gor.httpPath(req.http)`
|
||||
* `httpMethod` - Http method: 'GET', 'POST', etc. `gor.httpMethod(req.http)`.
|
||||
* `setHttpPath` - update URL path: `req.http = gor.setHttpPath(req.http, newPath)`
|
||||
* `httpPathParam` - get param from URL path: `gor.httpPathParam(req.http, queryParam)`
|
||||
* `setHttpPathParam` - set URL param: `req.http = gor.setHttpPathParam(req.http, queryParam, value)`
|
||||
* `httpStatus` - response status code
|
||||
* `httpHeaders` - get all headers: `gor.httpHeaders(req.http)`
|
||||
* `httpHeader` - get HTTP header: `gor.httpHeader(req.http, "Content-Length")`
|
||||
* `setHttpHeader` - Set HTTP header, returns modified payload: `req.http = gor.setHttpHeader(req.http, "X-Replayed", "1")`
|
||||
* `httpBody` - get HTTP Body: `gor.httpBody(req.http)`
|
||||
* `setHttpBody` - Set HTTP Body and ensures that `Content-Length` header have proper value. Returns modified payload: `req.http = gor.setHttpBody(req.http, Buffer.from('hello!'))`.
|
||||
* `httpBodyParam` - get POST body param: `gor.httpBodyParam(req.http, param)`
|
||||
* `setHttpBodyParam` - set POST body param: `req.http = gor.setHttpBodyParam(req.http, param, value)`
|
||||
* `httpCookie` - get HTTP cookie: `gor.httpCookie(req.http, "SESSSION_ID")`
|
||||
* `setHttpCookie` - set HTTP cookie, returns modified payload: `req.http = gor.setHttpCookie(req.http, "iam", "cuckoo")`
|
||||
* `deleteHttpCookie` - delete HTTP cookie, returns modified payload: `req.http = gor.deleteHttpCookie(req.http, "iam")`
|
||||
|
||||
Also it is totally legit to use standard `Buffer` functions like `indexOf` for processing the HTTP payload. Just do not forget that if you modify the body, update the `Content-Length` header with a new value. And if you modify any of the headers, line endings should be `\r\n`. Rest is up to your imagination.
|
||||
|
||||
|
||||
## Support
|
||||
|
||||
Feel free to ask questions here and by sending email to [support@goreplay.org](mailto:support@goreplay.org). Commercial support is available and welcomed 🙈.
|
||||
@@ -1,790 +0,0 @@
|
||||
// ======= GoReplay Middleware helper =============
|
||||
// Created by Leonid Bugaev in 2017
|
||||
//
|
||||
// For questions use GitHub or support@goreplay.org
|
||||
//
|
||||
// GoReplay: https://github.com/buger/goreplay
|
||||
// Middleware package: https://github.com/buger/goreplay/middleware
|
||||
|
||||
var middleware;
|
||||
|
||||
function init() {
|
||||
var proxy = {
|
||||
ch: {},
|
||||
on: function(chan, id, cb) {
|
||||
if (!cb && id) {
|
||||
cb = id;
|
||||
} else if (cb && id) {
|
||||
chan = chan + "#" + id;
|
||||
}
|
||||
|
||||
if (!proxy.ch[chan]) {
|
||||
proxy.ch[chan] = [];
|
||||
}
|
||||
|
||||
proxy.ch[chan].push({
|
||||
created: new Date(),
|
||||
cb: cb
|
||||
});
|
||||
|
||||
return proxy;
|
||||
},
|
||||
|
||||
emit: function(msg, raw) {
|
||||
var chanPrefix;
|
||||
|
||||
switch(msg.type) {
|
||||
case "1": chanPrefix = "request"; break;
|
||||
case "2": chanPrefix = "response"; break;
|
||||
case "3": chanPrefix = "replay"; break;
|
||||
}
|
||||
|
||||
let resp = msg;
|
||||
|
||||
["message", chanPrefix, chanPrefix + "#" + msg.ID].forEach(function(chanID, idx){
|
||||
if (proxy.ch[chanID]) {
|
||||
proxy.ch[chanID].forEach(function(ch){
|
||||
let r = ch.cb(msg);
|
||||
if (resp) resp = r; // If one of callback decided not to send response back, do not override it in global callbacks
|
||||
})
|
||||
|
||||
// Cleanup Individual message channels to avoid memory leaks
|
||||
if (idx == 2) {
|
||||
delete proxy.ch[chanID]
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
if (resp) {
|
||||
process.stdout.write(`${resp.rawMeta.toString('hex')}${Buffer.from("\n").toString("hex")}${resp.http.toString('hex')}\n`)
|
||||
}
|
||||
|
||||
return resp
|
||||
}
|
||||
}
|
||||
|
||||
// Clean up old messaged ID specific channels if they are older then 60s
|
||||
let gc = function(gcTime){
|
||||
let now = new Date();
|
||||
for (k in proxy.ch) {
|
||||
if (k.indexOf("#") == -1) continue;
|
||||
|
||||
proxy.ch[k] = proxy.ch[k].filter(function(ch){
|
||||
return (now - ch.created) < gcTime
|
||||
})
|
||||
|
||||
if (proxy.ch[k].length == 0) {
|
||||
delete proxy.ch[k]
|
||||
}
|
||||
}
|
||||
}
|
||||
proxy.gc = gc
|
||||
|
||||
setInterval(function(){
|
||||
gc(10 * 1000)
|
||||
}, 1000);
|
||||
|
||||
const readline = require('readline');
|
||||
const rl = readline.createInterface({
|
||||
input: process.stdin
|
||||
});
|
||||
|
||||
rl.on('line', function(line) {
|
||||
let msg = parseMessage(line)
|
||||
if (msg) {
|
||||
proxy.emit(msg, line)
|
||||
}
|
||||
});
|
||||
|
||||
middleware = proxy;
|
||||
|
||||
return proxy;
|
||||
}
|
||||
|
||||
|
||||
function parseMessage(msg) {
|
||||
try {
|
||||
let payload = Buffer.from(msg, "hex");
|
||||
let metaPos = payload.indexOf("\n");
|
||||
let meta = payload.slice(0, metaPos);
|
||||
let metaArr = meta.toString("ascii").split(" ");
|
||||
let pType = metaArr[0];
|
||||
let pID = metaArr[1];
|
||||
let raw = payload.slice(metaPos + 1, payload.length);
|
||||
|
||||
return {
|
||||
type: pType,
|
||||
ID: pID,
|
||||
rawMeta: meta,
|
||||
meta: metaArr,
|
||||
http: raw
|
||||
}
|
||||
} catch(e) {
|
||||
fail(`Error while parsing incoming request: ${msg}`)
|
||||
}
|
||||
}
|
||||
|
||||
// Used to compare values from original and replayed responses
|
||||
// Accepts request id, regexp pattern for searching the compared value (should include capture group), and callback which returns both original and replayed matched value.
|
||||
// Example:
|
||||
//
|
||||
// // Compare HTTP headers for response and replayed response, and map values
|
||||
// let tokMap = {};
|
||||
//
|
||||
// gor.on("request", function(req) {
|
||||
// let tok = gor.httpHeader(req.http, "Auth-Token");
|
||||
// if (tok && tokMap[tok]) {
|
||||
// req.http = gor.setHttpHeader(req.http, "Auth-Token", tokMap[tok])
|
||||
// }
|
||||
//
|
||||
// gor.searchResponses(req.ID, "X-Set-Token: (\w+)$", function(respTok, replTok) {
|
||||
// tokMap[respTok] = replTok;
|
||||
// })
|
||||
//
|
||||
// return req;
|
||||
// })
|
||||
//
|
||||
function searchResponses(id, searchPattern, callback) {
|
||||
let re = new RegExp(searchPattern);
|
||||
|
||||
// Using regexp require converting buffer to string
|
||||
// Before converting to string we can use initial `Buffer.indexOf` check
|
||||
let indexPattern = searchPattern.split("(")[0];
|
||||
|
||||
if (!indexPattern) {
|
||||
console.error("Search regexp should include capture group, pointing to the value: `prefix-(.*)`")
|
||||
return
|
||||
}
|
||||
|
||||
middleware.on("response", id, function(resp){
|
||||
if (resp.http.indexOf(indexPattern) == -1) {
|
||||
callback()
|
||||
return resp
|
||||
}
|
||||
|
||||
let respMatch = resp.http.toString('utf-8').match(re);
|
||||
if (!respMatch) {
|
||||
callback()
|
||||
return resp
|
||||
}
|
||||
|
||||
middleware.on("replay", id, function(repl) {
|
||||
if (repl.http.indexOf(indexPattern) == -1) {
|
||||
callback(respMatch[1]);
|
||||
return repl;
|
||||
}
|
||||
|
||||
let replMatch = repl.http.toString('utf-8').match(re);
|
||||
|
||||
if (!replMatch) {
|
||||
callback(respMatch[1]);
|
||||
return repl;
|
||||
}
|
||||
|
||||
callback(respMatch[1], replMatch[1]);
|
||||
|
||||
return repl;
|
||||
})
|
||||
|
||||
return resp;
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
// =========== HTTP parsing =================
|
||||
|
||||
// Example HTTP payload record (including hidden characters):
|
||||
//
|
||||
// POST / HTTP/1.1\r\n
|
||||
// User-Agent: Node\r\n
|
||||
// Content-Length: 5\r\n
|
||||
// \r\n
|
||||
// hello
|
||||
|
||||
function httpMethod(payload) {
|
||||
var pEnd = payload.indexOf(' ');
|
||||
return payload.slice(0, pEnd).toString("ascii");
|
||||
}
|
||||
|
||||
function httpPath(payload) {
|
||||
var pStart = payload.indexOf(' ') + 1;
|
||||
var pEnd = payload.indexOf(' ', pStart);
|
||||
return payload.slice(pStart, pEnd).toString("ascii");
|
||||
}
|
||||
|
||||
function setHttpPath(payload, newPath) {
|
||||
var pStart = payload.indexOf(' ') + 1;
|
||||
var pEnd = payload.indexOf(' ', pStart);
|
||||
|
||||
return Buffer.concat([payload.slice(0, pStart), Buffer.from(newPath), payload.slice(pEnd, payload.length)])
|
||||
}
|
||||
|
||||
function httpPathParam(payload, name) {
|
||||
let path = httpPath(payload);
|
||||
let re = new RegExp(name + "=([^&$]+)");
|
||||
let match = path.match(re);
|
||||
|
||||
if (match) return decodeURI(match[1]);
|
||||
}
|
||||
|
||||
function setHttpPathParam(payload, name, value) {
|
||||
let path = httpPath(payload);
|
||||
let re = new RegExp(name + "=([^&$]+)");
|
||||
let newPath = path.replace(re, name + "=" + encodeURI(value));
|
||||
|
||||
// If we should add new param instead
|
||||
if (newPath == path) {
|
||||
if (newPath.indexOf("?") == -1) {
|
||||
newPath += "?"
|
||||
} else {
|
||||
newPath += "&"
|
||||
}
|
||||
|
||||
newPath += name + "=" + encodeURI(value);
|
||||
}
|
||||
|
||||
return setHttpPath(payload, newPath)
|
||||
}
|
||||
|
||||
// HTTP response have status code in same position as `path` for requests
|
||||
function httpStatus(payload) {
|
||||
return httpPath(payload);
|
||||
}
|
||||
|
||||
function setHttpStatus(payload, newStatus) {
|
||||
return setHttpPath(payload, newStatus);
|
||||
}
|
||||
|
||||
function httpHeaders(payload) {
|
||||
var httpHeaderString = payload.slice(0,payload.indexOf("\r\n\r\n") + 4).toString().split("\n").slice(1);
|
||||
var headers = {};
|
||||
|
||||
for (var item in httpHeaderString) {
|
||||
var parts = httpHeaderString[item].split(":");
|
||||
|
||||
if (parts.length > 1) {
|
||||
headers[parts[0]] = parts.slice(1).join(":").trim();
|
||||
}
|
||||
}
|
||||
|
||||
return headers;
|
||||
}
|
||||
|
||||
function httpHeader(payload, name) {
|
||||
var currentLine = 0;
|
||||
var i = 0;
|
||||
var header = { start: -1, end: -1, valueStart: -1 }
|
||||
var nameBuf = Buffer.from(name);
|
||||
var nameBufLower = Buffer.from(name.toLowerCase());
|
||||
|
||||
while(c = payload[i]) {
|
||||
if (c == 13) { // new line "\n"
|
||||
currentLine++;
|
||||
i++
|
||||
header.end = i
|
||||
|
||||
if (currentLine > 0 && header.start > 0 && header.valueStart > 0) {
|
||||
if (nameBuf.compare(payload, header.start, header.valueStart - 1) == 0 ||
|
||||
nameBufLower.compare(payload, header.start, header.valueStart - 1) == 0) { // ensure that headers are not case sensitive
|
||||
header.value = payload.slice(header.valueStart, header.end - 1).toString("utf-8").trim();
|
||||
header.name = payload.slice(header.start, header.valueStart - 1).toString("utf-8");
|
||||
return header
|
||||
}
|
||||
}
|
||||
|
||||
header.start = -1
|
||||
header.valueStart = -1
|
||||
continue;
|
||||
} else if (c == 10) { // "\r"
|
||||
i++
|
||||
continue;
|
||||
} else if (c == 58) { // ":" Header/value separator symbol
|
||||
if (header.valueStart == -1) {
|
||||
header.valueStart = i + 1;
|
||||
i++
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
if (header.start == -1) header.start = i;
|
||||
|
||||
i++
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
function setHttpHeader(payload, name, value) {
|
||||
let header = httpHeader(payload, name);
|
||||
if (!header) {
|
||||
let headerStart = payload.indexOf(13) + 1;
|
||||
return Buffer.concat([payload.slice(0, headerStart + 1), Buffer.from(name + ": " + value + "\r\n"), payload.slice(headerStart + 1, payload.length)])
|
||||
} else {
|
||||
return Buffer.concat([payload.slice(0, header.valueStart), Buffer.from(" " + value + "\r\n"), payload.slice(header.end + 1, payload.length)])
|
||||
}
|
||||
}
|
||||
|
||||
function deleteHttpHeader(payload, name) {
|
||||
let header = httpHeader(payload, name);
|
||||
|
||||
if (header) {
|
||||
return Buffer.concat([payload.slice(0, header.start), payload.slice(header.end+1, payload.length)])
|
||||
}
|
||||
|
||||
return payload
|
||||
}
|
||||
|
||||
function httpBody(payload) {
|
||||
let bodyIndex = payload.indexOf("\r\n\r\n");
|
||||
if (-1 != bodyIndex){
|
||||
return payload.slice(bodyIndex + 4, payload.length);
|
||||
} else {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function setHttpBody(payload, newBody) {
|
||||
let p = setHttpHeader(payload, "Content-Length", newBody.length)
|
||||
let headerEnd = p.indexOf("\r\n\r\n") + 4;
|
||||
return Buffer.concat([p.slice(0, headerEnd), newBody])
|
||||
}
|
||||
|
||||
function httpBodyParam(payload, name) {
|
||||
let body = httpBody(payload);
|
||||
let re = new RegExp(name + "=([^&$]+)");
|
||||
if (body.indexOf(name + "=") != -1) {
|
||||
let param = body.toString('utf-8').match(re);
|
||||
if (param) {
|
||||
return decodeURI(param[1]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function setHttpBodyParam(payload, name, value) {
|
||||
let body = httpBody(payload);
|
||||
let re = new RegExp(name + "=([^&$]+)");
|
||||
|
||||
let newBody = body.toString('utf-8');
|
||||
|
||||
if (newBody.indexOf(name + "=") != -1 ) {
|
||||
newBody = newBody.replace(re, name + "=" + encodeURI(value));
|
||||
} else {
|
||||
if (newBody.indexOf("=") != -1) {
|
||||
newBody += "&";
|
||||
}
|
||||
newBody += name + "=" + value;
|
||||
}
|
||||
|
||||
return setHttpBody(payload, Buffer.from(newBody));
|
||||
}
|
||||
|
||||
function setHttpCookie(payload, name, value) {
|
||||
let h = httpHeader(payload, "Cookie");
|
||||
let cookie = h ? h.value : "";
|
||||
let cookies = cookie.split("; ").filter(function(v){ return v.indexOf(name + "=") != 0 })
|
||||
cookies.push(name + "=" + value)
|
||||
return setHttpHeader(payload, "Cookie", cookies.join("; "))
|
||||
}
|
||||
|
||||
function deleteHttpCookie(payload, name) {
|
||||
let h = httpHeader(payload, "Cookie");
|
||||
let cookie = h ? h.value : "";
|
||||
let cookies = cookie.split("; ").filter(function(v){ return v.indexOf(name + "=") != 0 })
|
||||
return setHttpHeader(payload, "Cookie", cookies.join("; "))
|
||||
}
|
||||
|
||||
function httpCookie(payload, name) {
|
||||
let h = httpHeader(payload, "Cookie");
|
||||
let cookie = h ? h.value : "";
|
||||
let value;
|
||||
let cookies = cookie.split("; ").forEach(function(v){
|
||||
if (v.indexOf(name + "=") == 0) {
|
||||
value = v.substr(name.length + 1);
|
||||
}
|
||||
})
|
||||
return value;
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
init: init,
|
||||
on: function(){ return middleware.on.apply(this, arguments) },
|
||||
parseMessage: parseMessage,
|
||||
searchResponses: searchResponses,
|
||||
httpPath: httpPath,
|
||||
httpMethod: httpMethod,
|
||||
setHttpPath: setHttpPath,
|
||||
httpPathParam: httpPathParam,
|
||||
setHttpPathParam: setHttpPathParam,
|
||||
httpStatus: httpStatus,
|
||||
setHttpStatus: setHttpStatus,
|
||||
httpHeader: httpHeader,
|
||||
setHttpHeader: setHttpHeader,
|
||||
deleteHttpHeader: deleteHttpHeader,
|
||||
httpBody: httpBody,
|
||||
setHttpBody: setHttpBody,
|
||||
httpBodyParam: httpBodyParam,
|
||||
setHttpBodyParam: setHttpBodyParam,
|
||||
httpCookie: httpCookie,
|
||||
setHttpCookie: setHttpCookie,
|
||||
deleteHttpCookie: deleteHttpCookie,
|
||||
test: testRunner,
|
||||
benchmark: testBenchmark,
|
||||
httpHeaders: httpHeaders
|
||||
}
|
||||
|
||||
|
||||
// =========== Tests ==============
|
||||
|
||||
function testRunner(){
|
||||
["init", "filter", "parseMessage", "httpMethod", "httpPath", "setHttpHeader", "deleteHttpHeader", "httpPathParam", "httpHeader", "httpBody", "setHttpBody", "httpBodyParam", "httpCookie", "setHttpCookie", "deleteHttpCookie", "httpHeaders"].forEach(function(t){
|
||||
console.log(`====== Start ${t} =======`)
|
||||
eval(`TEST_${t}()`)
|
||||
console.log(`====== End ${t} =======`)
|
||||
})
|
||||
}
|
||||
|
||||
function testBenchmark(){
|
||||
const child_process = require('child_process');
|
||||
|
||||
let gor = init();
|
||||
gor.on("message", function(){
|
||||
});
|
||||
|
||||
gor.on("request", function(){
|
||||
});
|
||||
|
||||
for (var i = 0; i<256; i++) {
|
||||
let req = parseMessage(Buffer.from("1 2 3\nGET / HTTP/1.1\r\n\r\n").toString('hex'));
|
||||
req.ID = +Date.now()
|
||||
gor.emit(req);
|
||||
|
||||
gor.on("request", req.ID+"", function(){
|
||||
gor.on("response", req.ID+"", function(){
|
||||
})
|
||||
})
|
||||
|
||||
if ( i % 3 == 0 ) {
|
||||
let resp = parseMessage(Buffer.from("2 2 3\nHTTP/1.1 200 OK\r\n\r\n").toString('hex'));
|
||||
resp.ID = req.ID
|
||||
gor.emit(resp);
|
||||
}
|
||||
}
|
||||
|
||||
child_process.execSync("sleep 0.01");
|
||||
|
||||
gor.gc(1)
|
||||
|
||||
fail(JSON.stringify(gor.ch))
|
||||
}
|
||||
|
||||
// Just print in red color
|
||||
function fail(message) {
|
||||
console.error("\x1b[31m[MIDDLEWARE] %s\x1b[0m", message)
|
||||
}
|
||||
|
||||
function log(message) {
|
||||
console.error(message)
|
||||
}
|
||||
|
||||
function TEST_init() {
|
||||
const child_process = require('child_process');
|
||||
|
||||
let received = 0;
|
||||
let gor = init();
|
||||
gor.on("message", function(){
|
||||
received++; // should be called 3 times for for every request
|
||||
});
|
||||
|
||||
gor.on("request", function(){
|
||||
received++; // should be called 1 time only for request
|
||||
});
|
||||
|
||||
gor.on("response", "2", function(){
|
||||
received++; // should be called 1 time only for specific response
|
||||
})
|
||||
|
||||
if (Object.keys(gor.ch).length != 3) {
|
||||
return fail("Should create 3 channels");
|
||||
}
|
||||
|
||||
let req = parseMessage(Buffer.from("1 2 3\nGET / HTTP/1.1\r\n\r\n").toString('hex'));
|
||||
let resp = parseMessage(Buffer.from("2 2 3\nHTTP/1.1 200 OK\r\n\r\n").toString('hex'));
|
||||
let resp2 = parseMessage(Buffer.from("2 3 3\nHTTP/1.1 200 OK\r\n\r\n").toString('hex'));
|
||||
|
||||
gor.emit(req);
|
||||
gor.emit(resp);
|
||||
gor.emit(resp2);
|
||||
|
||||
child_process.execSync("sleep 0.01");
|
||||
|
||||
if (received != 5) {
|
||||
fail(`Should receive 5 messages: ${received}`);
|
||||
}
|
||||
}
|
||||
|
||||
function TEST_filter() {
|
||||
const child_process = require('child_process');
|
||||
|
||||
let gor = init();
|
||||
gor.on("request", function(req){
|
||||
if (httpPath(req.http) != "/filter") {
|
||||
return req
|
||||
}
|
||||
});
|
||||
|
||||
gor.on("request", function(req){
|
||||
return req
|
||||
});
|
||||
|
||||
|
||||
let reqPass = parseMessage(Buffer.from("1 2 3\nGET / HTTP/1.1\r\n\r\n").toString('hex'));
|
||||
let reqFilter = parseMessage(Buffer.from("1 2 3\nGET /filter HTTP/1.1\r\n\r\n").toString('hex'));
|
||||
|
||||
if (!gor.emit(reqPass)) {
|
||||
return fail("Should not filter request")
|
||||
}
|
||||
|
||||
if (gor.emit(reqFilter)) {
|
||||
return fail("Should filter request even if one middleware rejected it")
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
function TEST_parseMessage() {
|
||||
const exampleMessage = Buffer.from("1 2 3\nGET / HTTP/1.1\r\n\r\n").toString('hex')
|
||||
let msg = parseMessage(exampleMessage)
|
||||
let expected = { type: '1', ID: '2', meta: ["1", "2", "3"], http: Buffer.from("GET / HTTP/1.1\r\n\r\n") }
|
||||
|
||||
Object.keys(expected).forEach(function(k){
|
||||
if (msg[k].toString() != expected[k].toString()) {
|
||||
fail(`${k}: '${expected[k]}' != '${msg[k]}'`)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
function TEST_httpPath() {
|
||||
const examplePayload = "GET /test HTTP/1.1\r\n\r\n";
|
||||
|
||||
let payload = Buffer.from(examplePayload);
|
||||
let path = httpPath(payload);
|
||||
|
||||
if (path != "/test") {
|
||||
return fail(`Path '${patj}' != '/test'`)
|
||||
}
|
||||
|
||||
let newPayload = setHttpPath(payload, '/')
|
||||
if (newPayload.toString() != "GET / HTTP/1.1\r\n\r\n") {
|
||||
return fail(`Malformed payload '${newPayload}'`)
|
||||
}
|
||||
|
||||
newPayload = setHttpPath(payload, '/bigger')
|
||||
if (newPayload.toString() != "GET /bigger HTTP/1.1\r\n\r\n") {
|
||||
return fail(`Malformed payload '${newPayload}'`)
|
||||
}
|
||||
}
|
||||
|
||||
function TEST_httpMethod() {
|
||||
const examplePayload = "GET /test HTTP/1.1\r\n\r\n";
|
||||
|
||||
let payload = Buffer.from(examplePayload);
|
||||
let method = httpMethod(payload);
|
||||
|
||||
if (method != "GET") {
|
||||
return fail(`Path '${method}' != 'GET'`)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
function TEST_httpPathParam() {
|
||||
let p = Buffer.from("GET / HTTP/1.1\r\n\r\n");
|
||||
|
||||
if (httpPathParam(p, "test")) {
|
||||
return fail("Should not found param")
|
||||
}
|
||||
|
||||
p = setHttpPathParam(p, "test", "123");
|
||||
if (httpPath(p) != "/?test=123") {
|
||||
return fail("Should set first param: " + httpPath(p));
|
||||
}
|
||||
|
||||
if (httpPathParam(p, "test") != "123") {
|
||||
return fail("Should get first param: " + httpPathParam(p, "test"));
|
||||
}
|
||||
|
||||
p = setHttpPathParam(p, "qwer", "ty");
|
||||
if (httpPath(p) != "/?test=123&qwer=ty") {
|
||||
return fail("Should set second param: " + httpPath(p));
|
||||
}
|
||||
|
||||
p = setHttpPathParam(p, "test", "4321");
|
||||
if (httpPath(p) != "/?test=4321&qwer=ty") {
|
||||
return fail("Should update first param: " + httpPath(p));
|
||||
}
|
||||
|
||||
if (httpPathParam(p, "test") != "4321") {
|
||||
return fail("Should update first param: " + httpPath(p));
|
||||
}
|
||||
}
|
||||
|
||||
function TEST_httpBodyParam() {
|
||||
let p = Buffer.from("POST / HTTP/1.1\r\n\r\n");
|
||||
|
||||
if (httpBodyParam(p, "test")) {
|
||||
return fail("Should not found param")
|
||||
}
|
||||
|
||||
p = setHttpBodyParam(p, "test", "123");
|
||||
if (httpBody(p).toString() != "test=123") {
|
||||
return fail("Should set first param: " + httpBody(p).toString());
|
||||
}
|
||||
|
||||
if (httpBodyParam(p, "test") != "123") {
|
||||
return fail("Should get first param: " + httpBodyParam(p, "test"));
|
||||
}
|
||||
|
||||
p = setHttpBodyParam(p, "qwer", "ty");
|
||||
if (httpBody(p).toString() != "test=123&qwer=ty") {
|
||||
return fail("Should set second param: " + httpBody(p).toString());
|
||||
}
|
||||
|
||||
p = setHttpBodyParam(p, "test", "4321");
|
||||
if (httpBody(p).toString() != "test=4321&qwer=ty") {
|
||||
return fail("Should update first param: " + httpBody(p).toString());
|
||||
}
|
||||
|
||||
if (httpBodyParam(p, "test") != "4321") {
|
||||
return fail("Should update first param: " + httpBody(p).toString());
|
||||
}
|
||||
}
|
||||
|
||||
function TEST_httpHeader() {
|
||||
const examplePayload = "GET / HTTP/1.1\r\nHost: localhost:3000\r\nUser-Agent: Node\r\nContent-Length:5\r\n\r\nhello";
|
||||
|
||||
let expected = {"Host": "localhost:3000", "User-Agent": "Node", "Content-Length": "5"}
|
||||
|
||||
Object.keys(expected).forEach(function(name){
|
||||
let payload = Buffer.from(examplePayload);
|
||||
let header = httpHeader(payload, name);
|
||||
if (!header) {
|
||||
fail(`Header not found. Was looking for: ${name}`)
|
||||
}
|
||||
if (header && header.value != expected[name]) {
|
||||
fail(`${name}: '${expected[name]}' != '${header.value}'`)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
function TEST_setHttpHeader() {
|
||||
const examplePayload = "GET / HTTP/1.1\r\nUser-Agent: Node\r\nContent-Length: 5\r\n\r\nhello";
|
||||
|
||||
// Modify existing header
|
||||
["", "1", "Long test header"].forEach(function(ua){
|
||||
let expected = `GET / HTTP/1.1\r\nUser-Agent: ${ua}\r\nContent-Length: 5\r\n\r\nhello`;
|
||||
let p = Buffer.from(examplePayload);
|
||||
p = setHttpHeader(p, "User-Agent", ua);
|
||||
if (p != expected) {
|
||||
console.error(`setHeader failed, expected User-Agent value: ${ua}.\n${p}`)
|
||||
}
|
||||
})
|
||||
|
||||
// Adding new header
|
||||
let expected = `GET / HTTP/1.1\r\nX-Test: test\r\nUser-Agent: Node\r\nContent-Length: 5\r\n\r\nhello`;
|
||||
let p = Buffer.from(examplePayload);
|
||||
p = setHttpHeader(p, "X-Test", "test");
|
||||
if (p != expected) {
|
||||
console.error(`setHeader failed, expected new header 'X-Test' header: ${p}`)
|
||||
}
|
||||
}
|
||||
|
||||
function TEST_deleteHttpHeader() {
|
||||
const examplePayload = "GET / HTTP/1.1\r\nUser-Agent: Node\r\nContent-Length: 5\r\n\r\nhello";
|
||||
|
||||
// Adding new header
|
||||
let expected = `GET / HTTP/1.1\r\nContent-Length: 5\r\n\r\nhello`;
|
||||
let p = Buffer.from(examplePayload);
|
||||
p = deleteHttpHeader(p, "User-Agent", "test");
|
||||
if (p != expected) {
|
||||
console.error(`setHeader failed, expected delete header 'User-Agent' header: ${p}`)
|
||||
}
|
||||
}
|
||||
|
||||
function TEST_httpBody() {
|
||||
const examplePayload = "GET / HTTP/1.1\r\nUser-Agent: Node\r\nContent-Length: 5\r\n\r\nhello";
|
||||
let body = httpBody(Buffer.from(examplePayload));
|
||||
if (body != "hello") {
|
||||
fail(`'${body}' != 'hello'`)
|
||||
}
|
||||
|
||||
const exampleInvalidPayload = "Invalid HTTP Response by Network issue";
|
||||
let invalidBody = httpBody(Buffer.from(exampleInvalidPayload));
|
||||
if (invalidBody != null) {
|
||||
fail(`'${invalidBody}' != 'null'`)
|
||||
}
|
||||
}
|
||||
|
||||
function TEST_setHttpBody() {
|
||||
const examplePayload = "GET / HTTP/1.1\r\nUser-Agent: Node\r\nContent-Length: 5\r\n\r\nhello";
|
||||
let p = setHttpBody(Buffer.from(examplePayload), Buffer.from("hello, world!"));
|
||||
|
||||
if (p != "GET / HTTP/1.1\r\nUser-Agent: Node\r\nContent-Length: 13\r\n\r\nhello, world!") {
|
||||
fail(`Wrong body: '${p}'`)
|
||||
}
|
||||
}
|
||||
|
||||
function TEST_httpCookie() {
|
||||
const examplePayload = "GET / HTTP/1.1\r\nCookie: a=b; test=zxc\r\n\r\n";
|
||||
let c = httpCookie(Buffer.from(examplePayload), "test");
|
||||
if (c != "zxc") {
|
||||
return fail(`Should get cookie: ${c}`);
|
||||
}
|
||||
|
||||
c = httpCookie(Buffer.from(examplePayload), "nope");
|
||||
if (c != null) {
|
||||
return fail(`Should not find cookie: ${c}`);
|
||||
}
|
||||
}
|
||||
|
||||
function TEST_setHttpCookie() {
|
||||
const examplePayload = "GET / HTTP/1.1\r\nCookie: a=b; test=zxc\r\n\r\n";
|
||||
let p = setHttpCookie(Buffer.from(examplePayload), "test", "1");
|
||||
if (p != "GET / HTTP/1.1\r\nCookie: a=b; test=1\r\n\r\n") {
|
||||
return fail(`Should update cookie: ${p}`)
|
||||
}
|
||||
|
||||
p = setHttpCookie(Buffer.from(examplePayload), "new", "one");
|
||||
if (p != "GET / HTTP/1.1\r\nCookie: a=b; test=zxc; new=one\r\n\r\n") {
|
||||
return fail(`Should add new cookie: ${p}`)
|
||||
}
|
||||
}
|
||||
|
||||
function TEST_deleteHttpCookie() {
|
||||
const examplePayload = "GET / HTTP/1.1\r\nCookie: a=b; test=zxc\r\n\r\n";
|
||||
let p = deleteHttpCookie(Buffer.from(examplePayload), "a");
|
||||
if (p != "GET / HTTP/1.1\r\nCookie: test=zxc\r\n\r\n") {
|
||||
return fail(`Should delete cookie: ${p}`)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
function TEST_httpHeaders() {
|
||||
const examplePayload = "GET / HTTP/1.1\r\nHost: localhost:3000\r\nUser-Agent: Node\r\nContent-Length:5\r\n\r\nhello";
|
||||
|
||||
let expectedHeaders = {"Host": "localhost:3000", "User-Agent": "Node", "Content-Length": "5"}
|
||||
let payload = Buffer.from(examplePayload);
|
||||
let headers = httpHeaders(payload);
|
||||
|
||||
["Host", "User-Agent", "Content-Length"].forEach(function(header){
|
||||
let actual = headers[header];
|
||||
let expected = expectedHeaders[header];
|
||||
|
||||
if (!actual) {
|
||||
fail(`${header} Header was not found`);
|
||||
}
|
||||
|
||||
if (actual != expected) {
|
||||
fail(`${header} Header not Equal to Expected: ${expected} was ${actual}`);
|
||||
}
|
||||
|
||||
})
|
||||
}
|
||||
@@ -1,16 +0,0 @@
|
||||
{
|
||||
"name": "goreplay_middleware",
|
||||
"version": "1.0.0",
|
||||
"description": "Package for writing middleware for GoReplay https://goreplay.org",
|
||||
"main": "middleware.js",
|
||||
"scripts": {
|
||||
"test": "node -e \"var gor = require('./middleware.js'); gor.test(); process.exit()\"",
|
||||
"benchmark": "node -e \"var gor = require('./middleware.js'); gor.benchmark(); process.exit()\""
|
||||
},
|
||||
"keywords": [
|
||||
"middleware",
|
||||
"goreplay"
|
||||
],
|
||||
"author": "Leonid Bugaev",
|
||||
"license": "LGPL-3.0"
|
||||
}
|
||||
+24
-33
@@ -4,6 +4,7 @@ import (
|
||||
"bytes"
|
||||
"crypto/rand"
|
||||
"encoding/hex"
|
||||
"github.com/buger/gor/proto"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
@@ -11,13 +12,11 @@ import (
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/buger/goreplay/proto"
|
||||
)
|
||||
|
||||
type fakeServiceCb func(string, int, []byte)
|
||||
|
||||
// Simple service that generate token on request, and require this token for accesing to Secure area
|
||||
// Simple service that generate token on request, and require this token for accesing to secure area
|
||||
func NewFakeSecureService(wg *sync.WaitGroup, cb fakeServiceCb) *httptest.Server {
|
||||
active_tokens := make([]string, 0)
|
||||
var mu sync.Mutex
|
||||
@@ -39,7 +38,7 @@ func NewFakeSecureService(wg *sync.WaitGroup, cb fakeServiceCb) *httptest.Server
|
||||
w.Write([]byte(token))
|
||||
|
||||
cb(req.URL.Path, 200, []byte(token))
|
||||
case "/Secure":
|
||||
case "/secure":
|
||||
token := req.URL.Query().Get("token")
|
||||
token_found := false
|
||||
|
||||
@@ -81,13 +80,13 @@ func TestFakeSecureService(t *testing.T) {
|
||||
token = proto.Body(resp)
|
||||
|
||||
// Right token
|
||||
resp, _ = client.Get("/Secure?token=" + string(token))
|
||||
resp, _ = client.Get("/secure?token=" + string(token))
|
||||
if !bytes.Equal(proto.Status(resp), []byte("202")) {
|
||||
t.Error("Valid token should return status 202:", string(proto.Status(resp)))
|
||||
}
|
||||
|
||||
// Wrong tokens forbidden
|
||||
resp, _ = client.Get("/Secure?token=wrong")
|
||||
resp, _ = client.Get("/secure?token=wrong")
|
||||
if !bytes.Equal(proto.Status(resp), []byte("403")) {
|
||||
t.Error("Wrong token should returns status 403:", string(proto.Status(resp)))
|
||||
}
|
||||
@@ -114,27 +113,23 @@ func TestEchoMiddleware(t *testing.T) {
|
||||
|
||||
quit := make(chan int)
|
||||
|
||||
Settings.Middleware = "./examples/Middleware/echo.sh"
|
||||
Settings.middleware = "./examples/middleware/echo.sh"
|
||||
|
||||
// Catch traffic from one service
|
||||
fromAddr := strings.Replace(from.Listener.Addr().String(), "[::]", "127.0.0.1", -1)
|
||||
input := NewRAWInput(fromAddr, EnginePcap, true, testRawExpire, "", "http", "", "", 0)
|
||||
input := NewRAWInput(fromAddr, EnginePcap, true, testRawExpire, "")
|
||||
defer input.Close()
|
||||
|
||||
// And redirect to another
|
||||
output := NewHTTPOutput(to.URL, &HTTPOutputConfig{Debug: false})
|
||||
|
||||
plugins := &InOutPlugins{
|
||||
Inputs: []io.Reader{input},
|
||||
Outputs: []io.Writer{output},
|
||||
}
|
||||
plugins.All = append(plugins.All, input, output)
|
||||
Plugins.Inputs = []io.Reader{input}
|
||||
Plugins.Outputs = []io.Writer{output}
|
||||
|
||||
// Start Gor
|
||||
emitter := NewEmitter(quit)
|
||||
go emitter.Start(plugins, Settings.Middleware)
|
||||
go Start(quit)
|
||||
|
||||
// Wait till Middleware initialization
|
||||
// Wait till middleware initialization
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
|
||||
// Should receive 2 requests from original + 2 from replayed
|
||||
@@ -150,10 +145,10 @@ func TestEchoMiddleware(t *testing.T) {
|
||||
}
|
||||
|
||||
wg.Wait()
|
||||
emitter.Close()
|
||||
close(quit)
|
||||
time.Sleep(200 * time.Millisecond)
|
||||
|
||||
Settings.Middleware = ""
|
||||
Settings.middleware = ""
|
||||
}
|
||||
|
||||
func TestTokenMiddleware(t *testing.T) {
|
||||
@@ -168,7 +163,7 @@ func TestTokenMiddleware(t *testing.T) {
|
||||
|
||||
to := NewFakeSecureService(wg, func(path string, status int, tok []byte) {
|
||||
switch path {
|
||||
case "/Secure":
|
||||
case "/secure":
|
||||
if status != 202 {
|
||||
t.Error("Server should receive valid rewritten token")
|
||||
}
|
||||
@@ -180,27 +175,23 @@ func TestTokenMiddleware(t *testing.T) {
|
||||
|
||||
quit := make(chan int)
|
||||
|
||||
Settings.Middleware = "go run ./examples/Middleware/token_modifier.go"
|
||||
Settings.middleware = "go run ./examples/middleware/token_modifier.go"
|
||||
|
||||
fromAddr := strings.Replace(from.Listener.Addr().String(), "[::]", "127.0.0.1", -1)
|
||||
// Catch traffic from one service
|
||||
input := NewRAWInput(fromAddr, EnginePcap, true, testRawExpire, "", "http", "", "", 0)
|
||||
input := NewRAWInput(fromAddr, EnginePcap, true, testRawExpire, "")
|
||||
defer input.Close()
|
||||
|
||||
// And redirect to another
|
||||
output := NewHTTPOutput(to.URL, &HTTPOutputConfig{Debug: true})
|
||||
|
||||
plugins := &InOutPlugins{
|
||||
Inputs: []io.Reader{input},
|
||||
Outputs: []io.Writer{output},
|
||||
}
|
||||
plugins.All = append(plugins.All, input, output)
|
||||
Plugins.Inputs = []io.Reader{input}
|
||||
Plugins.Outputs = []io.Writer{output}
|
||||
|
||||
// Start Gor
|
||||
emitter := NewEmitter(quit)
|
||||
go emitter.Start(plugins, Settings.Middleware)
|
||||
go Start(quit)
|
||||
|
||||
// Wait for Middleware to initialize
|
||||
// Wait for middleware to initialize
|
||||
// Give go compiller time to build programm
|
||||
time.Sleep(500 * time.Millisecond)
|
||||
|
||||
@@ -213,17 +204,17 @@ func TestTokenMiddleware(t *testing.T) {
|
||||
resp, _ = client.Get("/token")
|
||||
token = proto.Body(resp)
|
||||
|
||||
// When delay is too smal, Middleware does not always rewrite requests in time
|
||||
// When delay is too smal, middleware does not always rewrite requests in time
|
||||
// Hopefuly client will have delay more then 100ms :)
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
|
||||
resp, _ = client.Get("/Secure?token=" + string(token))
|
||||
resp, _ = client.Get("/secure?token=" + string(token))
|
||||
if !bytes.Equal(proto.Status(resp), []byte("202")) {
|
||||
t.Error("Valid token should return 202:", proto.Status(resp))
|
||||
}
|
||||
|
||||
wg.Wait()
|
||||
emitter.Close()
|
||||
close(quit)
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
Settings.Middleware = ""
|
||||
Settings.middleware = ""
|
||||
}
|
||||
|
||||
@@ -1,176 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"io"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
)
|
||||
|
||||
// BinaryOutputConfig struct for holding binary output configuration
|
||||
type BinaryOutputConfig struct {
|
||||
workers int
|
||||
Timeout time.Duration
|
||||
BufferSize int
|
||||
Debug bool
|
||||
TrackResponses bool
|
||||
}
|
||||
|
||||
// BinaryOutput plugin manage pool of workers which send request to replayed server
|
||||
// By default workers pool is dynamic and starts with 10 workers
|
||||
// You can specify fixed number of workers using `--output-tcp-workers`
|
||||
type BinaryOutput struct {
|
||||
// Keep this as first element of struct because it guarantees 64bit
|
||||
// alignment. atomic.* functions crash on 32bit machines if operand is not
|
||||
// aligned at 64bit. See https://github.com/golang/go/issues/599
|
||||
activeWorkers int64
|
||||
|
||||
address string
|
||||
queue chan []byte
|
||||
|
||||
responses chan response
|
||||
|
||||
needWorker chan int
|
||||
|
||||
config *BinaryOutputConfig
|
||||
|
||||
queueStats *GorStat
|
||||
}
|
||||
|
||||
// NewBinaryOutput constructor for BinaryOutput
|
||||
// Initialize workers
|
||||
func NewBinaryOutput(address string, config *BinaryOutputConfig) io.Writer {
|
||||
o := new(BinaryOutput)
|
||||
|
||||
o.address = address
|
||||
o.config = config
|
||||
|
||||
o.queue = make(chan []byte, 1000)
|
||||
o.responses = make(chan response, 1000)
|
||||
o.needWorker = make(chan int, 1)
|
||||
|
||||
// Initial workers count
|
||||
if o.config.workers == 0 {
|
||||
o.needWorker <- initialDynamicWorkers
|
||||
} else {
|
||||
o.needWorker <- o.config.workers
|
||||
}
|
||||
|
||||
go o.workerMaster()
|
||||
|
||||
return o
|
||||
}
|
||||
|
||||
func (o *BinaryOutput) workerMaster() {
|
||||
for {
|
||||
newWorkers := <-o.needWorker
|
||||
for i := 0; i < newWorkers; i++ {
|
||||
go o.startWorker()
|
||||
}
|
||||
|
||||
// Disable dynamic scaling if workers poll fixed size
|
||||
if o.config.workers != 0 {
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (o *BinaryOutput) startWorker() {
|
||||
client := NewTCPClient(o.address, &TCPClientConfig{
|
||||
Debug: o.config.Debug,
|
||||
Timeout: o.config.Timeout,
|
||||
ResponseBufferSize: o.config.BufferSize,
|
||||
})
|
||||
|
||||
deathCount := 0
|
||||
|
||||
atomic.AddInt64(&o.activeWorkers, 1)
|
||||
|
||||
for {
|
||||
select {
|
||||
case data := <-o.queue:
|
||||
o.sendRequest(client, data)
|
||||
deathCount = 0
|
||||
case <-time.After(time.Millisecond * 100):
|
||||
// When dynamic scaling enabled workers die after 2s of inactivity
|
||||
if o.config.workers == 0 {
|
||||
deathCount++
|
||||
} else {
|
||||
continue
|
||||
}
|
||||
|
||||
if deathCount > 20 {
|
||||
workersCount := atomic.LoadInt64(&o.activeWorkers)
|
||||
|
||||
// At least 1 startWorker should be alive
|
||||
if workersCount != 1 {
|
||||
atomic.AddInt64(&o.activeWorkers, -1)
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (o *BinaryOutput) Write(data []byte) (n int, err error) {
|
||||
if !isRequestPayload(data) {
|
||||
return len(data), nil
|
||||
}
|
||||
|
||||
buf := make([]byte, len(data))
|
||||
copy(buf, data)
|
||||
|
||||
o.queue <- buf
|
||||
|
||||
if o.config.workers == 0 {
|
||||
workersCount := atomic.LoadInt64(&o.activeWorkers)
|
||||
|
||||
if len(o.queue) > int(workersCount) {
|
||||
o.needWorker <- len(o.queue)
|
||||
}
|
||||
}
|
||||
|
||||
return len(data), nil
|
||||
}
|
||||
|
||||
func (o *BinaryOutput) Read(data []byte) (int, error) {
|
||||
resp := <-o.responses
|
||||
|
||||
Debug("[OUTPUT-TCP] Received response:", string(resp.payload))
|
||||
|
||||
header := payloadHeader(ReplayedResponsePayload, resp.uuid, resp.startedAt, resp.roundTripTime)
|
||||
copy(data[0:len(header)], header)
|
||||
copy(data[len(header):], resp.payload)
|
||||
|
||||
return len(resp.payload) + len(header), nil
|
||||
}
|
||||
|
||||
func (o *BinaryOutput) sendRequest(client *TCPClient, request []byte) {
|
||||
meta := payloadMeta(request)
|
||||
if len(meta) < 2 {
|
||||
return
|
||||
}
|
||||
|
||||
if !isRequestPayload(request) {
|
||||
return
|
||||
}
|
||||
|
||||
uuid := meta[1]
|
||||
|
||||
body := payloadBody(request)
|
||||
|
||||
start := time.Now()
|
||||
resp, err := client.Send(body)
|
||||
stop := time.Now()
|
||||
|
||||
if err != nil {
|
||||
Debug("Request error:", err)
|
||||
}
|
||||
|
||||
if o.config.TrackResponses {
|
||||
o.responses <- response{resp, uuid, start.UnixNano(), stop.UnixNano() - start.UnixNano()}
|
||||
}
|
||||
}
|
||||
|
||||
func (o *BinaryOutput) String() string {
|
||||
return "Binary output: " + o.address
|
||||
}
|
||||
+23
-64
@@ -3,7 +3,6 @@ package main
|
||||
import (
|
||||
"bufio"
|
||||
"compress/gzip"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
@@ -29,31 +28,26 @@ var dateFileNameFuncs = map[string]func(*FileOutput) string{
|
||||
"%t": func(o *FileOutput) string { return string(o.payloadType) },
|
||||
}
|
||||
|
||||
// FileOutputConfig ...
|
||||
type FileOutputConfig struct {
|
||||
flushInterval time.Duration
|
||||
sizeLimit int64
|
||||
outputFileMaxSize int64
|
||||
queueLimit int64
|
||||
append bool
|
||||
bufferPath string
|
||||
onClose func(string)
|
||||
flushInterval time.Duration
|
||||
sizeLimit unitSizeVar
|
||||
queueLimit int
|
||||
append bool
|
||||
}
|
||||
|
||||
// FileOutput output plugin
|
||||
type FileOutput struct {
|
||||
sync.RWMutex
|
||||
mu sync.Mutex
|
||||
pathTemplate string
|
||||
currentName string
|
||||
file *os.File
|
||||
queueLength int64
|
||||
queueLength int
|
||||
chunkSize int
|
||||
writer io.Writer
|
||||
requestPerFile bool
|
||||
currentID []byte
|
||||
payloadType []byte
|
||||
closed bool
|
||||
totalFileSize int64
|
||||
|
||||
config *FileOutputConfig
|
||||
}
|
||||
@@ -69,18 +63,16 @@ func NewFileOutput(pathTemplate string, config *FileOutputConfig) *FileOutput {
|
||||
o.requestPerFile = true
|
||||
}
|
||||
|
||||
if config.flushInterval == 0 {
|
||||
config.flushInterval = 100 * time.Millisecond
|
||||
}
|
||||
|
||||
go func() {
|
||||
for {
|
||||
time.Sleep(config.flushInterval)
|
||||
if o.IsClosed() {
|
||||
if o.closed {
|
||||
break
|
||||
}
|
||||
o.mu.Lock()
|
||||
o.updateName()
|
||||
o.flush()
|
||||
o.mu.Unlock()
|
||||
}
|
||||
}()
|
||||
|
||||
@@ -141,8 +133,8 @@ func (s sortByFileIndex) Less(i, j int) bool {
|
||||
}
|
||||
|
||||
func (o *FileOutput) filename() string {
|
||||
o.RLock()
|
||||
defer o.RUnlock()
|
||||
defer o.mu.Unlock()
|
||||
o.mu.Lock()
|
||||
|
||||
path := o.pathTemplate
|
||||
|
||||
@@ -187,27 +179,20 @@ func (o *FileOutput) filename() string {
|
||||
}
|
||||
|
||||
func (o *FileOutput) updateName() {
|
||||
name := filepath.Clean(o.filename())
|
||||
o.Lock()
|
||||
o.currentName = name
|
||||
o.Unlock()
|
||||
o.currentName = filepath.Clean(o.filename())
|
||||
}
|
||||
|
||||
func (o *FileOutput) Write(data []byte) (n int, err error) {
|
||||
if o.requestPerFile {
|
||||
o.Lock()
|
||||
meta := payloadMeta(data)
|
||||
o.currentID = meta[1]
|
||||
o.payloadType = meta[0]
|
||||
o.Unlock()
|
||||
o.updateName()
|
||||
}
|
||||
|
||||
o.updateName()
|
||||
o.Lock()
|
||||
defer o.Unlock()
|
||||
|
||||
if o.file == nil || o.currentName != o.file.Name() {
|
||||
o.closeLocked()
|
||||
o.mu.Lock()
|
||||
o.Close()
|
||||
|
||||
o.file, err = os.OpenFile(o.currentName, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0660)
|
||||
o.file.Sync()
|
||||
@@ -223,21 +208,15 @@ func (o *FileOutput) Write(data []byte) (n int, err error) {
|
||||
}
|
||||
|
||||
o.queueLength = 0
|
||||
o.mu.Unlock()
|
||||
}
|
||||
|
||||
n, _ = o.writer.Write(data)
|
||||
nSeparator, _ := o.writer.Write([]byte(payloadSeparator))
|
||||
o.writer.Write(data)
|
||||
o.writer.Write([]byte(payloadSeparator))
|
||||
|
||||
n += nSeparator
|
||||
|
||||
o.totalFileSize += int64(n)
|
||||
o.queueLength++
|
||||
|
||||
if Settings.OutputFileConfig.outputFileMaxSize > 0 && o.totalFileSize >= Settings.OutputFileConfig.outputFileMaxSize {
|
||||
return n, errors.New("File output reached size limit")
|
||||
}
|
||||
|
||||
return n, nil
|
||||
return len(data), nil
|
||||
}
|
||||
|
||||
func (o *FileOutput) flush() {
|
||||
@@ -248,8 +227,8 @@ func (o *FileOutput) flush() {
|
||||
}
|
||||
}()
|
||||
|
||||
o.Lock()
|
||||
defer o.Unlock()
|
||||
defer o.mu.Unlock()
|
||||
o.mu.Lock()
|
||||
|
||||
if o.file != nil {
|
||||
if strings.HasSuffix(o.currentName, ".gz") {
|
||||
@@ -258,10 +237,8 @@ func (o *FileOutput) flush() {
|
||||
o.writer.(*bufio.Writer).Flush()
|
||||
}
|
||||
|
||||
if stat, err := o.file.Stat(); err == nil {
|
||||
if stat, err := o.file.Stat(); err != nil {
|
||||
o.chunkSize = int(stat.Size())
|
||||
} else {
|
||||
log.Println("Error accessing file sats", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -270,7 +247,7 @@ func (o *FileOutput) String() string {
|
||||
return "File output: " + o.file.Name()
|
||||
}
|
||||
|
||||
func (o *FileOutput) closeLocked() error {
|
||||
func (o *FileOutput) Close() error {
|
||||
if o.file != nil {
|
||||
if strings.HasSuffix(o.currentName, ".gz") {
|
||||
o.writer.(*gzip.Writer).Close()
|
||||
@@ -278,26 +255,8 @@ func (o *FileOutput) closeLocked() error {
|
||||
o.writer.(*bufio.Writer).Flush()
|
||||
}
|
||||
o.file.Close()
|
||||
|
||||
if o.config.onClose != nil {
|
||||
o.config.onClose(o.file.Name())
|
||||
}
|
||||
}
|
||||
|
||||
o.closed = true
|
||||
return nil
|
||||
}
|
||||
|
||||
// Close closes the output file that is being written to.
|
||||
func (o *FileOutput) Close() error {
|
||||
o.Lock()
|
||||
defer o.Unlock()
|
||||
return o.closeLocked()
|
||||
}
|
||||
|
||||
// IsClosed returns if the output file is closed or not.
|
||||
func (o *FileOutput) IsClosed() bool {
|
||||
o.Lock()
|
||||
defer o.Unlock()
|
||||
return o.closed
|
||||
}
|
||||
|
||||
@@ -0,0 +1,39 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
var dataUnitMap = map[byte]int64{
|
||||
'k': 1024,
|
||||
'm': 1024 * 1024,
|
||||
'g': 1024 * 1024 * 1024,
|
||||
}
|
||||
|
||||
func parseDataUnit(s string) int64 {
|
||||
// Allow kb, mb, gb
|
||||
if strings.HasSuffix(s, "b") {
|
||||
s = s[:len(s)-1]
|
||||
}
|
||||
|
||||
if unit, ok := dataUnitMap[s[len(s)-1]]; ok {
|
||||
size, _ := strconv.ParseInt(s[:len(s)-1], 10, 64)
|
||||
return unit * size
|
||||
} else {
|
||||
// If no unit specified use bytes
|
||||
size, _ := strconv.ParseInt(s, 10, 64)
|
||||
return size
|
||||
}
|
||||
}
|
||||
|
||||
type unitSizeVar int64
|
||||
|
||||
func (u unitSizeVar) String() string {
|
||||
return strconv.Itoa(int(u))
|
||||
}
|
||||
|
||||
func (u unitSizeVar) Set(s string) error {
|
||||
u = unitSizeVar(parseDataUnit(s))
|
||||
return nil
|
||||
}
|
||||
+25
-69
@@ -20,14 +20,10 @@ func TestFileOutput(t *testing.T) {
|
||||
input := NewTestInput()
|
||||
output := NewFileOutput("/tmp/test_requests.gor", &FileOutputConfig{flushInterval: time.Minute, append: true})
|
||||
|
||||
plugins := &InOutPlugins{
|
||||
Inputs: []io.Reader{input},
|
||||
Outputs: []io.Writer{output},
|
||||
}
|
||||
plugins.All = append(plugins.All, input, output)
|
||||
Plugins.Inputs = []io.Reader{input}
|
||||
Plugins.Outputs = []io.Writer{output}
|
||||
|
||||
emitter := NewEmitter(quit)
|
||||
go emitter.Start(plugins, Settings.Middleware)
|
||||
go Start(quit)
|
||||
|
||||
for i := 0; i < 100; i++ {
|
||||
wg.Add(2)
|
||||
@@ -36,7 +32,10 @@ func TestFileOutput(t *testing.T) {
|
||||
}
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
output.flush()
|
||||
emitter.Close()
|
||||
|
||||
close(quit)
|
||||
|
||||
quit = make(chan int)
|
||||
|
||||
var counter int64
|
||||
input2 := NewFileInput("/tmp/test_requests.gor", false)
|
||||
@@ -45,18 +44,13 @@ func TestFileOutput(t *testing.T) {
|
||||
wg.Done()
|
||||
})
|
||||
|
||||
plugins2 := &InOutPlugins{
|
||||
Inputs: []io.Reader{input2},
|
||||
Outputs: []io.Writer{output2},
|
||||
}
|
||||
plugins2.All = append(plugins2.All, input2, output2)
|
||||
Plugins.Inputs = []io.Reader{input2}
|
||||
Plugins.Outputs = []io.Writer{output2}
|
||||
|
||||
quit2 := make(chan int)
|
||||
emitter2 := NewEmitter(quit2)
|
||||
go emitter2.Start(plugins2, Settings.Middleware)
|
||||
go Start(quit)
|
||||
|
||||
wg.Wait()
|
||||
emitter2.Close()
|
||||
close(quit)
|
||||
}
|
||||
|
||||
func TestFileOutputWithNameCleaning(t *testing.T) {
|
||||
@@ -164,25 +158,21 @@ func TestFileOutputCompression(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestParseDataUnit(t *testing.T) {
|
||||
var d = map[string]int64{
|
||||
"42mb": 42 << 20,
|
||||
"4_2": 42,
|
||||
"00": 0,
|
||||
"\n\n 0.0\r\t\f": 0,
|
||||
"0_600tb": 384 << 40,
|
||||
"0600Tb": 384 << 40,
|
||||
"0o12Mb": 10 << 20,
|
||||
"0b_10010001111_1kb": 2335 << 10,
|
||||
"1024": 1 << 10,
|
||||
"0b111": 7,
|
||||
"0x12gB": 18 << 30,
|
||||
"0x_67_7a_2f_cc_40_c6": 113774485586118,
|
||||
"121562380192901": 121562380192901,
|
||||
var tests = []struct {
|
||||
value string
|
||||
size int64
|
||||
}{
|
||||
{"100kb", dataUnitMap['k'] * 100},
|
||||
{"100k", dataUnitMap['k'] * 100},
|
||||
{"1kb", dataUnitMap['k']},
|
||||
{"1g", dataUnitMap['g']},
|
||||
{"10m", dataUnitMap['m'] * 10},
|
||||
{"zsaa312", 0},
|
||||
}
|
||||
for k, v := range d {
|
||||
n, err := bufferParser(k, "0")
|
||||
if err != nil || n != v {
|
||||
t.Errorf("Error parsing %s: %v", k, err)
|
||||
|
||||
for _, c := range tests {
|
||||
if parseDataUnit(c.value) != c.size {
|
||||
t.Error(c.value, "should be", c.size, "instead", parseDataUnit(c.value))
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -322,37 +312,3 @@ func TestFileOutputSort(t *testing.T) {
|
||||
t.Error("Should properly sort file names using indexes", files, expected)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFileOutputAppendSizeLimitOverflow(t *testing.T) {
|
||||
rnd := rand.Int63()
|
||||
name := fmt.Sprintf("/tmp/%d", rnd)
|
||||
|
||||
message := []byte("1 1 1\r\ntest")
|
||||
|
||||
messageSize := len(message) + len(payloadSeparator)
|
||||
|
||||
output := NewFileOutput(name, &FileOutputConfig{append: false, flushInterval: time.Minute, sizeLimit: 2 * int64(messageSize)})
|
||||
|
||||
output.Write([]byte("1 1 1\r\ntest"))
|
||||
name1 := output.file.Name()
|
||||
|
||||
output.Write([]byte("1 1 1\r\ntest"))
|
||||
name2 := output.file.Name()
|
||||
|
||||
output.flush()
|
||||
output.updateName()
|
||||
|
||||
output.Write([]byte("1 1 1\r\ntest"))
|
||||
name3 := output.file.Name()
|
||||
|
||||
if name2 != name1 || name1 != fmt.Sprintf("/tmp/%d_0", rnd) {
|
||||
t.Error("Fast changes should happen in same file:", name1, name2, name3)
|
||||
}
|
||||
|
||||
if name3 == name1 || name3 != fmt.Sprintf("/tmp/%d_1", rnd) {
|
||||
t.Error("File name should change:", name1, name2, name3)
|
||||
}
|
||||
|
||||
os.Remove(name1)
|
||||
os.Remove(name3)
|
||||
}
|
||||
|
||||
+35
-151
@@ -1,58 +1,15 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/buger/goreplay/proto"
|
||||
"github.com/buger/gor/proto"
|
||||
)
|
||||
|
||||
var _ = fmt.Println
|
||||
|
||||
const initialDynamicWorkers = 10
|
||||
|
||||
type httpWorker struct {
|
||||
output *HTTPOutput
|
||||
client *HTTPClient
|
||||
lastActivity time.Time
|
||||
queue chan []byte
|
||||
stop chan bool
|
||||
}
|
||||
|
||||
func newHTTPWorker(output *HTTPOutput, queue chan []byte) *httpWorker {
|
||||
client := NewHTTPClient(output.address, &HTTPClientConfig{
|
||||
FollowRedirects: output.config.redirectLimit,
|
||||
Debug: output.config.Debug,
|
||||
OriginalHost: output.config.OriginalHost,
|
||||
Timeout: output.config.Timeout,
|
||||
ResponseBufferSize: output.config.BufferSize,
|
||||
})
|
||||
|
||||
w := &httpWorker{client: client}
|
||||
if queue == nil {
|
||||
w.queue = make(chan []byte, 100)
|
||||
} else {
|
||||
w.queue = queue
|
||||
}
|
||||
w.stop = make(chan bool)
|
||||
|
||||
go func() {
|
||||
for {
|
||||
select {
|
||||
case payload := <-w.queue:
|
||||
output.sendRequest(client, payload)
|
||||
case <-w.stop:
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
return w
|
||||
}
|
||||
|
||||
type response struct {
|
||||
payload []byte
|
||||
uuid []byte
|
||||
@@ -64,12 +21,8 @@ type response struct {
|
||||
type HTTPOutputConfig struct {
|
||||
redirectLimit int
|
||||
|
||||
stats bool
|
||||
workersMin int
|
||||
workersMax int
|
||||
statsMs int
|
||||
workers int
|
||||
queueLen int
|
||||
stats bool
|
||||
workers int
|
||||
|
||||
elasticSearch string
|
||||
|
||||
@@ -77,10 +30,6 @@ type HTTPOutputConfig struct {
|
||||
OriginalHost bool
|
||||
BufferSize int
|
||||
|
||||
CompatibilityMode bool
|
||||
|
||||
RequestGroup string
|
||||
|
||||
Debug bool
|
||||
|
||||
TrackResponses bool
|
||||
@@ -95,8 +44,6 @@ type HTTPOutput struct {
|
||||
// aligned at 64bit. See https://github.com/golang/go/issues/599
|
||||
activeWorkers int64
|
||||
|
||||
workerSessions map[string]*httpWorker
|
||||
|
||||
address string
|
||||
limit int
|
||||
queue chan []byte
|
||||
@@ -110,8 +57,6 @@ type HTTPOutput struct {
|
||||
queueStats *GorStat
|
||||
|
||||
elasticSearch *ESPlugin
|
||||
|
||||
stop chan bool // Channel used only to indicate goroutine should shutdown
|
||||
}
|
||||
|
||||
// NewHTTPOutput constructor for HTTPOutput
|
||||
@@ -121,21 +66,20 @@ func NewHTTPOutput(address string, config *HTTPOutputConfig) io.Writer {
|
||||
|
||||
o.address = address
|
||||
o.config = config
|
||||
o.stop = make(chan bool)
|
||||
|
||||
if o.config.stats {
|
||||
o.queueStats = NewGorStat("output_http", o.config.statsMs)
|
||||
o.queueStats = NewGorStat("output_http")
|
||||
}
|
||||
|
||||
o.queue = make(chan []byte, o.config.queueLen)
|
||||
o.responses = make(chan response, o.config.queueLen)
|
||||
o.queue = make(chan []byte, 1000)
|
||||
o.responses = make(chan response, 1000)
|
||||
o.needWorker = make(chan int, 1)
|
||||
|
||||
// Initial workers count
|
||||
if o.config.workersMax == 0 {
|
||||
if o.config.workers == 0 {
|
||||
o.needWorker <- initialDynamicWorkers
|
||||
} else {
|
||||
o.needWorker <- o.config.workersMax
|
||||
o.needWorker <- o.config.workers
|
||||
}
|
||||
|
||||
if o.config.elasticSearch != "" {
|
||||
@@ -143,15 +87,7 @@ func NewHTTPOutput(address string, config *HTTPOutputConfig) io.Writer {
|
||||
o.elasticSearch.Init(o.config.elasticSearch)
|
||||
}
|
||||
|
||||
if Settings.RecognizeTCPSessions {
|
||||
if !PRO {
|
||||
log.Fatal("Detailed TCP sessions work only with PRO license")
|
||||
}
|
||||
o.workerSessions = make(map[string]*httpWorker, 100)
|
||||
go o.sessionWorkerMaster()
|
||||
} else {
|
||||
go o.workerMaster()
|
||||
}
|
||||
go o.workerMaster()
|
||||
|
||||
return o
|
||||
}
|
||||
@@ -159,41 +95,13 @@ func NewHTTPOutput(address string, config *HTTPOutputConfig) io.Writer {
|
||||
func (o *HTTPOutput) workerMaster() {
|
||||
for {
|
||||
newWorkers := <-o.needWorker
|
||||
atomic.AddInt64(&o.activeWorkers, int64(newWorkers))
|
||||
for i := 0; i < newWorkers; i++ {
|
||||
go o.startWorker()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (o *HTTPOutput) sessionWorkerMaster() {
|
||||
gc := time.Tick(time.Second)
|
||||
|
||||
for {
|
||||
select {
|
||||
case p := <-o.queue:
|
||||
id := payloadID(p)
|
||||
sessionID := string(id[0:20])
|
||||
worker, ok := o.workerSessions[sessionID]
|
||||
|
||||
if !ok {
|
||||
atomic.AddInt64(&o.activeWorkers, 1)
|
||||
worker = newHTTPWorker(o, nil)
|
||||
o.workerSessions[sessionID] = worker
|
||||
}
|
||||
|
||||
worker.queue <- p
|
||||
worker.lastActivity = time.Now()
|
||||
case <-gc:
|
||||
now := time.Now()
|
||||
|
||||
for id, w := range o.workerSessions {
|
||||
if !w.lastActivity.IsZero() && now.Sub(w.lastActivity) >= 120*time.Second {
|
||||
w.stop <- true
|
||||
delete(o.workerSessions, id)
|
||||
atomic.AddInt64(&o.activeWorkers, -1)
|
||||
}
|
||||
}
|
||||
// Disable dynamic scaling if workers poll fixed size
|
||||
if o.config.workers != 0 {
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -205,27 +113,33 @@ func (o *HTTPOutput) startWorker() {
|
||||
OriginalHost: o.config.OriginalHost,
|
||||
Timeout: o.config.Timeout,
|
||||
ResponseBufferSize: o.config.BufferSize,
|
||||
CompatibilityMode: o.config.CompatibilityMode,
|
||||
})
|
||||
|
||||
deathCount := 0
|
||||
|
||||
atomic.AddInt64(&o.activeWorkers, 1)
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-o.stop:
|
||||
return
|
||||
case data := <-o.queue:
|
||||
o.sendRequest(client, data)
|
||||
case <-time.After(2 * time.Second):
|
||||
deathCount = 0
|
||||
case <-time.After(time.Millisecond * 100):
|
||||
// When dynamic scaling enabled workers die after 2s of inactivity
|
||||
if o.config.workersMin == o.config.workersMax {
|
||||
if o.config.workers == 0 {
|
||||
deathCount++
|
||||
} else {
|
||||
continue
|
||||
}
|
||||
|
||||
workersCount := int(atomic.LoadInt64(&o.activeWorkers))
|
||||
if deathCount > 20 {
|
||||
workersCount := atomic.LoadInt64(&o.activeWorkers)
|
||||
|
||||
// At least 1 startWorker should be alive
|
||||
if workersCount != 1 && workersCount > o.config.workersMin {
|
||||
atomic.AddInt64(&o.activeWorkers, -1)
|
||||
return
|
||||
// At least 1 startWorker should be alive
|
||||
if workersCount != 1 {
|
||||
atomic.AddInt64(&o.activeWorkers, -1)
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -239,28 +153,17 @@ func (o *HTTPOutput) Write(data []byte) (n int, err error) {
|
||||
buf := make([]byte, len(data))
|
||||
copy(buf, data)
|
||||
|
||||
select {
|
||||
case <-o.stop:
|
||||
return 0, ErrorStopped
|
||||
case o.queue <- buf:
|
||||
}
|
||||
o.queue <- buf
|
||||
|
||||
if o.config.stats {
|
||||
o.queueStats.Write(len(o.queue))
|
||||
}
|
||||
|
||||
if !Settings.RecognizeTCPSessions && o.config.workersMax != o.config.workersMin {
|
||||
workersCount := int(atomic.LoadInt64(&o.activeWorkers))
|
||||
if o.config.workers == 0 {
|
||||
workersCount := atomic.LoadInt64(&o.activeWorkers)
|
||||
|
||||
if len(o.queue) > workersCount {
|
||||
extraWorkersReq := len(o.queue) - workersCount + 1
|
||||
maxWorkersAvailable := o.config.workersMax - workersCount
|
||||
if extraWorkersReq > maxWorkersAvailable {
|
||||
extraWorkersReq = maxWorkersAvailable
|
||||
}
|
||||
if extraWorkersReq > 0 {
|
||||
o.needWorker <- extraWorkersReq
|
||||
}
|
||||
if len(o.queue) > int(workersCount) {
|
||||
o.needWorker <- len(o.queue)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -268,16 +171,9 @@ func (o *HTTPOutput) Write(data []byte) (n int, err error) {
|
||||
}
|
||||
|
||||
func (o *HTTPOutput) Read(data []byte) (int, error) {
|
||||
var resp response
|
||||
select {
|
||||
case <-o.stop:
|
||||
return 0, ErrorStopped
|
||||
case resp = <-o.responses:
|
||||
}
|
||||
resp := <-o.responses
|
||||
|
||||
if Settings.Debug {
|
||||
Debug("[OUTPUT-HTTP] Received response:", string(resp.payload))
|
||||
}
|
||||
Debug("[OUTPUT-HTTP] Received response:", string(resp.payload))
|
||||
|
||||
header := payloadHeader(ReplayedResponsePayload, resp.uuid, resp.roundTripTime, resp.startedAt)
|
||||
copy(data[0:len(header)], header)
|
||||
@@ -288,11 +184,6 @@ func (o *HTTPOutput) Read(data []byte) (int, error) {
|
||||
|
||||
func (o *HTTPOutput) sendRequest(client *HTTPClient, request []byte) {
|
||||
meta := payloadMeta(request)
|
||||
|
||||
if Settings.Debug {
|
||||
Debug(meta)
|
||||
}
|
||||
|
||||
if len(meta) < 2 {
|
||||
return
|
||||
}
|
||||
@@ -308,7 +199,6 @@ func (o *HTTPOutput) sendRequest(client *HTTPClient, request []byte) {
|
||||
stop := time.Now()
|
||||
|
||||
if err != nil {
|
||||
log.Println("Error when sending ", err, time.Now())
|
||||
Debug("Request error:", err)
|
||||
}
|
||||
|
||||
@@ -324,9 +214,3 @@ func (o *HTTPOutput) sendRequest(client *HTTPClient, request []byte) {
|
||||
func (o *HTTPOutput) String() string {
|
||||
return "HTTP output: " + o.address
|
||||
}
|
||||
|
||||
// Close closes the data channel so that data
|
||||
func (o *HTTPOutput) Close() error {
|
||||
close(o.stop)
|
||||
return nil
|
||||
}
|
||||
|
||||
+21
-88
@@ -7,7 +7,6 @@ import (
|
||||
"net/http/httptest"
|
||||
_ "net/http/httputil"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
@@ -49,16 +48,12 @@ func TestHTTPOutput(t *testing.T) {
|
||||
wg.Done()
|
||||
})
|
||||
|
||||
plugins := &InOutPlugins{
|
||||
Inputs: []io.Reader{input},
|
||||
Outputs: []io.Writer{http_output, output},
|
||||
}
|
||||
plugins.All = append(plugins.All, input, output, http_output)
|
||||
Plugins.Inputs = []io.Reader{input}
|
||||
Plugins.Outputs = []io.Writer{http_output, output}
|
||||
|
||||
emitter := NewEmitter(quit)
|
||||
go emitter.Start(plugins, Settings.Middleware)
|
||||
go Start(quit)
|
||||
|
||||
for i := 0; i < 10; i++ {
|
||||
for i := 0; i < 1; i++ {
|
||||
// 2 http-output, 2 - test output request, 2 - test output http response
|
||||
wg.Add(6) // OPTIONS should be ignored
|
||||
input.EmitPOST()
|
||||
@@ -67,14 +62,9 @@ func TestHTTPOutput(t *testing.T) {
|
||||
}
|
||||
|
||||
wg.Wait()
|
||||
|
||||
close(quit)
|
||||
|
||||
activeWorkers := atomic.LoadInt64(&http_output.(*HTTPOutput).activeWorkers)
|
||||
|
||||
if activeWorkers < 50 {
|
||||
t.Error("Should create workers for each request", activeWorkers)
|
||||
}
|
||||
|
||||
Settings.modifierConfig = HTTPModifierConfig{}
|
||||
}
|
||||
|
||||
@@ -98,24 +88,22 @@ func TestHTTPOutputKeepOriginalHost(t *testing.T) {
|
||||
|
||||
output := NewHTTPOutput(server.URL, &HTTPOutputConfig{Debug: false, OriginalHost: true})
|
||||
|
||||
plugins := &InOutPlugins{
|
||||
Inputs: []io.Reader{input},
|
||||
Outputs: []io.Writer{output},
|
||||
}
|
||||
plugins.All = append(plugins.All, input, output)
|
||||
Plugins.Inputs = []io.Reader{input}
|
||||
Plugins.Outputs = []io.Writer{output}
|
||||
|
||||
emitter := NewEmitter(quit)
|
||||
go emitter.Start(plugins, Settings.Middleware)
|
||||
go Start(quit)
|
||||
|
||||
wg.Add(1)
|
||||
input.EmitGET()
|
||||
|
||||
wg.Wait()
|
||||
emitter.Close()
|
||||
|
||||
close(quit)
|
||||
|
||||
Settings.modifierConfig = HTTPModifierConfig{}
|
||||
}
|
||||
|
||||
func TestHTTPOutputSSL(t *testing.T) {
|
||||
func TestOutputHTTPSSL(t *testing.T) {
|
||||
wg := new(sync.WaitGroup)
|
||||
quit := make(chan int)
|
||||
|
||||
@@ -127,14 +115,10 @@ func TestHTTPOutputSSL(t *testing.T) {
|
||||
input := NewTestInput()
|
||||
output := NewHTTPOutput(server.URL, &HTTPOutputConfig{})
|
||||
|
||||
plugins := &InOutPlugins{
|
||||
Inputs: []io.Reader{input},
|
||||
Outputs: []io.Writer{output},
|
||||
}
|
||||
plugins.All = append(plugins.All, input, output)
|
||||
Plugins.Inputs = []io.Reader{input}
|
||||
Plugins.Outputs = []io.Writer{output}
|
||||
|
||||
emitter := NewEmitter(quit)
|
||||
go emitter.Start(plugins, Settings.Middleware)
|
||||
go Start(quit)
|
||||
|
||||
wg.Add(2)
|
||||
|
||||
@@ -142,55 +126,7 @@ func TestHTTPOutputSSL(t *testing.T) {
|
||||
input.EmitGET()
|
||||
|
||||
wg.Wait()
|
||||
emitter.Close()
|
||||
}
|
||||
|
||||
func TestHTTPOutputSessions(t *testing.T) {
|
||||
wg := new(sync.WaitGroup)
|
||||
quit := make(chan int)
|
||||
|
||||
input := NewTestInput()
|
||||
input.skipHeader = true
|
||||
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
|
||||
wg.Done()
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
Settings.RecognizeTCPSessions = true
|
||||
output := NewHTTPOutput(server.URL, &HTTPOutputConfig{Debug: true})
|
||||
|
||||
plugins := &InOutPlugins{
|
||||
Inputs: []io.Reader{input},
|
||||
Outputs: []io.Writer{output},
|
||||
}
|
||||
emitter := NewEmitter(quit)
|
||||
go emitter.Start(plugins, Settings.Middleware)
|
||||
|
||||
uuid1 := []byte("1234567890123456789a0000")
|
||||
uuid2 := []byte("1234567890123456789d0000")
|
||||
|
||||
for i := 0; i < 100; i++ {
|
||||
wg.Add(1) // OPTIONS should be ignored
|
||||
copy(uuid1[20:], randByte(4))
|
||||
input.EmitBytes([]byte("1 " + string(uuid1) + " 1\n" + "GET / HTTP/1.1\r\n\r\n"))
|
||||
}
|
||||
|
||||
for i := 0; i < 100; i++ {
|
||||
wg.Add(1) // OPTIONS should be ignored
|
||||
copy(uuid2[20:], randByte(4))
|
||||
input.EmitBytes([]byte("1 " + string(uuid2) + " 1\n" + "GET / HTTP/1.1\r\n\r\n"))
|
||||
}
|
||||
|
||||
wg.Wait()
|
||||
|
||||
if output.(*HTTPOutput).activeWorkers != 2 {
|
||||
t.Error("Should have only 2 workers", output.(*HTTPOutput).activeWorkers)
|
||||
}
|
||||
|
||||
emitter.Close()
|
||||
|
||||
Settings.RecognizeTCPSessions = false
|
||||
close(quit)
|
||||
}
|
||||
|
||||
func BenchmarkHTTPOutput(b *testing.B) {
|
||||
@@ -206,14 +142,10 @@ func BenchmarkHTTPOutput(b *testing.B) {
|
||||
input := NewTestInput()
|
||||
output := NewHTTPOutput(server.URL, &HTTPOutputConfig{})
|
||||
|
||||
plugins := &InOutPlugins{
|
||||
Inputs: []io.Reader{input},
|
||||
Outputs: []io.Writer{output},
|
||||
}
|
||||
plugins.All = append(plugins.All, input, output)
|
||||
Plugins.Inputs = []io.Reader{input}
|
||||
Plugins.Outputs = []io.Writer{output}
|
||||
|
||||
emitter := NewEmitter(quit)
|
||||
go emitter.Start(plugins, Settings.Middleware)
|
||||
go Start(quit)
|
||||
|
||||
for i := 0; i < b.N; i++ {
|
||||
wg.Add(1)
|
||||
@@ -221,5 +153,6 @@ func BenchmarkHTTPOutput(b *testing.B) {
|
||||
}
|
||||
|
||||
wg.Wait()
|
||||
emitter.Close()
|
||||
|
||||
close(quit)
|
||||
}
|
||||
|
||||
+5
-7
@@ -2,15 +2,13 @@ package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"github.com/Shopify/sarama"
|
||||
"github.com/Shopify/sarama/mocks"
|
||||
"github.com/buger/gor/proto"
|
||||
"io"
|
||||
"log"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/buger/goreplay/proto"
|
||||
|
||||
"github.com/Shopify/sarama"
|
||||
"github.com/Shopify/sarama/mocks"
|
||||
)
|
||||
|
||||
// KafkaOutput is used for sending payloads to kafka in JSON format.
|
||||
@@ -28,7 +26,7 @@ func NewKafkaOutput(address string, config *KafkaConfig) io.Writer {
|
||||
|
||||
var producer sarama.AsyncProducer
|
||||
|
||||
if mock, ok := config.producer.(*mocks.AsyncProducer); ok && mock != nil {
|
||||
if config.producer.(*mocks.AsyncProducer) != nil {
|
||||
producer = config.producer
|
||||
} else {
|
||||
c.Producer.RequiredAcks = sarama.WaitForLocal
|
||||
@@ -49,7 +47,7 @@ func NewKafkaOutput(address string, config *KafkaConfig) io.Writer {
|
||||
producer: producer,
|
||||
}
|
||||
|
||||
if Settings.Verbose {
|
||||
if Settings.verbose {
|
||||
// Start infinite loop for tracking errors for kafka producer.
|
||||
go o.ErrorHandler()
|
||||
}
|
||||
|
||||
@@ -1,10 +1,9 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/Shopify/sarama"
|
||||
"github.com/Shopify/sarama/mocks"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestOutputKafkaRAW(t *testing.T) {
|
||||
|
||||
+1
-1
@@ -4,7 +4,7 @@ package main
|
||||
type NullOutput struct {
|
||||
}
|
||||
|
||||
// NewNullOutput constructor for NullOutput
|
||||
// NullOutput constructor for NullOutput
|
||||
func NewNullOutput() (o *NullOutput) {
|
||||
return new(NullOutput)
|
||||
}
|
||||
|
||||
-129
@@ -1,129 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
_ "bufio"
|
||||
"fmt"
|
||||
_ "io"
|
||||
"log"
|
||||
"math/rand"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/aws/session"
|
||||
"github.com/aws/aws-sdk-go/service/s3"
|
||||
_ "github.com/aws/aws-sdk-go/service/s3/s3manager"
|
||||
)
|
||||
|
||||
// S3Output output plugin
|
||||
type S3Output struct {
|
||||
pathTemplate string
|
||||
|
||||
buffer *FileOutput
|
||||
session *session.Session
|
||||
config *FileOutputConfig
|
||||
closeCh chan struct{}
|
||||
}
|
||||
|
||||
// NewS3Output constructor for FileOutput, accepts path
|
||||
func NewS3Output(pathTemplate string, config *FileOutputConfig) *S3Output {
|
||||
if !PRO {
|
||||
log.Fatal("Using S3 output and input requires PRO license")
|
||||
return nil
|
||||
}
|
||||
|
||||
o := new(S3Output)
|
||||
o.pathTemplate = pathTemplate
|
||||
o.config = config
|
||||
o.config.onClose = o.onBufferUpdate
|
||||
|
||||
if config.bufferPath == "" {
|
||||
config.bufferPath = "/tmp"
|
||||
}
|
||||
|
||||
rnd := rand.Int63()
|
||||
bufferName := fmt.Sprintf("gor_output_s3_%d_buf_", rnd)
|
||||
|
||||
pathParts := strings.Split(pathTemplate, "/")
|
||||
bufferName += pathParts[len(pathParts)-1]
|
||||
|
||||
if strings.HasSuffix(o.pathTemplate, ".gz") {
|
||||
bufferName += ".gz"
|
||||
}
|
||||
|
||||
bufferPath := filepath.Join(config.bufferPath, bufferName)
|
||||
|
||||
o.buffer = NewFileOutput(bufferPath, config)
|
||||
o.connect()
|
||||
|
||||
return o
|
||||
}
|
||||
|
||||
func (o *S3Output) connect() {
|
||||
if o.session == nil {
|
||||
o.session = session.Must(session.NewSession(awsConfig()))
|
||||
log.Println("[S3 Output] S3 connection succesfully initialized")
|
||||
}
|
||||
}
|
||||
|
||||
func (o *S3Output) Write(data []byte) (n int, err error) {
|
||||
return o.buffer.Write(data)
|
||||
}
|
||||
|
||||
func (o *S3Output) String() string {
|
||||
return "S3 output: " + o.pathTemplate
|
||||
}
|
||||
|
||||
// Close close the buffer of the S3 connection
|
||||
func (o *S3Output) Close() error {
|
||||
return o.buffer.Close()
|
||||
}
|
||||
|
||||
func parseS3Url(path string) (bucket, key string) {
|
||||
path = path[5:] // stripping `s3://`
|
||||
sep := strings.IndexByte(path, '/')
|
||||
|
||||
bucket = path[:sep]
|
||||
key = path[sep+1:]
|
||||
|
||||
return bucket, key
|
||||
}
|
||||
|
||||
func (o *S3Output) keyPath(idx int) (bucket, key string) {
|
||||
bucket, key = parseS3Url(o.pathTemplate)
|
||||
|
||||
for name, fn := range dateFileNameFuncs {
|
||||
key = strings.Replace(key, name, fn(o.buffer), -1)
|
||||
}
|
||||
|
||||
key = setFileIndex(key, idx)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
func (o *S3Output) onBufferUpdate(path string) {
|
||||
svc := s3.New(o.session)
|
||||
idx := getFileIndex(path)
|
||||
bucket, key := o.keyPath(idx)
|
||||
|
||||
file, _ := os.Open(path)
|
||||
// reader := bufio.NewReader(file)
|
||||
|
||||
_, err := svc.PutObject(&s3.PutObjectInput{
|
||||
Body: file,
|
||||
Bucket: aws.String(bucket),
|
||||
Key: aws.String(key),
|
||||
})
|
||||
if err != nil {
|
||||
log.Printf("[S3 Output] Failed to upload data to %s/%s, %s\n", bucket, key, err)
|
||||
os.Remove(path)
|
||||
return
|
||||
}
|
||||
|
||||
os.Remove(path)
|
||||
|
||||
if o.closeCh != nil {
|
||||
o.closeCh <- struct{}{}
|
||||
}
|
||||
}
|
||||
+15
-52
@@ -1,9 +1,7 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"crypto/tls"
|
||||
"fmt"
|
||||
"hash/fnv"
|
||||
"io"
|
||||
"log"
|
||||
"net"
|
||||
@@ -16,48 +14,30 @@ import (
|
||||
type TCPOutput struct {
|
||||
address string
|
||||
limit int
|
||||
buf []chan []byte
|
||||
buf chan []byte
|
||||
bufStats *GorStat
|
||||
config *TCPOutputConfig
|
||||
}
|
||||
|
||||
type TCPOutputConfig struct {
|
||||
secure bool
|
||||
sticky bool
|
||||
}
|
||||
|
||||
// NewTCPOutput constructor for TCPOutput
|
||||
// Initialize 10 workers which hold keep-alive connection
|
||||
func NewTCPOutput(address string, config *TCPOutputConfig) io.Writer {
|
||||
func NewTCPOutput(address string) io.Writer {
|
||||
o := new(TCPOutput)
|
||||
|
||||
o.address = address
|
||||
o.config = config
|
||||
|
||||
if Settings.OutputTCPStats {
|
||||
o.bufStats = NewGorStat("output_tcp", 5000)
|
||||
o.buf = make(chan []byte, 100)
|
||||
if Settings.outputTCPStats {
|
||||
o.bufStats = NewGorStat("output_tcp")
|
||||
}
|
||||
|
||||
if o.config.sticky {
|
||||
// create 10 buffers and send the buffer index to the worker
|
||||
o.buf = make([]chan []byte, 10)
|
||||
for i := 0; i < 10; i++ {
|
||||
o.buf[i] = make(chan []byte, 100)
|
||||
go o.worker(i)
|
||||
}
|
||||
} else {
|
||||
// create 1 buffer and send its index (0) to all workers
|
||||
o.buf = make([]chan []byte, 1)
|
||||
o.buf[0] = make(chan []byte, 1000)
|
||||
for i := 0; i < 10; i++ {
|
||||
go o.worker(0)
|
||||
}
|
||||
for i := 0; i < 10; i++ {
|
||||
go o.worker()
|
||||
}
|
||||
|
||||
return o
|
||||
}
|
||||
|
||||
func (o *TCPOutput) worker(bufferIndex int) {
|
||||
func (o *TCPOutput) worker() {
|
||||
retries := 1
|
||||
conn, err := o.connect(o.address)
|
||||
for {
|
||||
@@ -79,29 +59,17 @@ func (o *TCPOutput) worker(bufferIndex int) {
|
||||
defer conn.Close()
|
||||
|
||||
for {
|
||||
data := <-o.buf[bufferIndex]
|
||||
conn.Write(data)
|
||||
conn.Write(<-o.buf)
|
||||
_, err := conn.Write([]byte(payloadSeparator))
|
||||
|
||||
if err != nil {
|
||||
log.Println("INFO: TCP output connection closed, reconnecting")
|
||||
o.buf[bufferIndex] <- data
|
||||
go o.worker(bufferIndex)
|
||||
log.Println("Lost connection with aggregator instance, reconnecting")
|
||||
go o.worker()
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (o *TCPOutput) getBufferIndex(data []byte) int {
|
||||
if !o.config.sticky {
|
||||
return 0
|
||||
}
|
||||
|
||||
hasher := fnv.New32a()
|
||||
hasher.Write(payloadMeta(data)[1])
|
||||
return int(hasher.Sum32()) % 10
|
||||
}
|
||||
|
||||
func (o *TCPOutput) Write(data []byte) (n int, err error) {
|
||||
if !isOriginPayload(data) {
|
||||
return len(data), nil
|
||||
@@ -111,22 +79,17 @@ func (o *TCPOutput) Write(data []byte) (n int, err error) {
|
||||
newBuf := make([]byte, len(data))
|
||||
copy(newBuf, data)
|
||||
|
||||
bufferIndex := o.getBufferIndex(data)
|
||||
o.buf[bufferIndex] <- newBuf
|
||||
o.buf <- newBuf
|
||||
|
||||
if Settings.OutputTCPStats {
|
||||
o.bufStats.Write(len(o.buf[bufferIndex]))
|
||||
if Settings.outputTCPStats {
|
||||
o.bufStats.Write(len(o.buf))
|
||||
}
|
||||
|
||||
return len(data), nil
|
||||
}
|
||||
|
||||
func (o *TCPOutput) connect(address string) (conn net.Conn, err error) {
|
||||
if o.config.secure {
|
||||
conn, err = tls.Dial("tcp", address, &tls.Config{})
|
||||
} else {
|
||||
conn, err = net.Dial("tcp", address)
|
||||
}
|
||||
conn, err = net.Dial("tcp", address)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
+11
-59
@@ -7,7 +7,6 @@ import (
|
||||
"net"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestTCPOutput(t *testing.T) {
|
||||
@@ -18,16 +17,12 @@ func TestTCPOutput(t *testing.T) {
|
||||
wg.Done()
|
||||
})
|
||||
input := NewTestInput()
|
||||
output := NewTCPOutput(listener.Addr().String(), &TCPOutputConfig{})
|
||||
output := NewTCPOutput(listener.Addr().String())
|
||||
|
||||
plugins := &InOutPlugins{
|
||||
Inputs: []io.Reader{input},
|
||||
Outputs: []io.Writer{output},
|
||||
}
|
||||
plugins.All = append(plugins.All, input, output)
|
||||
Plugins.Inputs = []io.Reader{input}
|
||||
Plugins.Outputs = []io.Writer{output}
|
||||
|
||||
emitter := NewEmitter(quit)
|
||||
go emitter.Start(plugins, Settings.Middleware)
|
||||
go Start(quit)
|
||||
|
||||
for i := 0; i < 100; i++ {
|
||||
wg.Add(1)
|
||||
@@ -35,7 +30,8 @@ func TestTCPOutput(t *testing.T) {
|
||||
}
|
||||
|
||||
wg.Wait()
|
||||
emitter.Close()
|
||||
|
||||
close(quit)
|
||||
}
|
||||
|
||||
func startTCP(cb func([]byte)) net.Listener {
|
||||
@@ -73,16 +69,12 @@ func BenchmarkTCPOutput(b *testing.B) {
|
||||
wg.Done()
|
||||
})
|
||||
input := NewTestInput()
|
||||
output := NewTCPOutput(listener.Addr().String(), &TCPOutputConfig{})
|
||||
output := NewTCPOutput(listener.Addr().String())
|
||||
|
||||
plugins := &InOutPlugins{
|
||||
Inputs: []io.Reader{input},
|
||||
Outputs: []io.Writer{output},
|
||||
}
|
||||
plugins.All = append(plugins.All, input, output)
|
||||
Plugins.Inputs = []io.Reader{input}
|
||||
Plugins.Outputs = []io.Writer{output}
|
||||
|
||||
emitter := NewEmitter(quit)
|
||||
go emitter.Start(plugins, Settings.Middleware)
|
||||
go Start(quit)
|
||||
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
@@ -91,46 +83,6 @@ func BenchmarkTCPOutput(b *testing.B) {
|
||||
}
|
||||
|
||||
wg.Wait()
|
||||
emitter.Close()
|
||||
}
|
||||
|
||||
func TestStickyDisable(t *testing.T) {
|
||||
tcpOutput := TCPOutput{config: &TCPOutputConfig{sticky: false}}
|
||||
|
||||
for i := 0; i < 1000; i++ {
|
||||
index := tcpOutput.getBufferIndex(getTestBytes())
|
||||
if index != 0 {
|
||||
t.Errorf("Sticky is disable. Got: %d want 0", index)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestBufferDistribution(t *testing.T) {
|
||||
numberOfWorkers := 10
|
||||
numberOfMessages := 1000000
|
||||
percentDistributionErrorRange := 20
|
||||
|
||||
buffer := make([]int, numberOfWorkers)
|
||||
tcpOutput := TCPOutput{config: &TCPOutputConfig{sticky: true}}
|
||||
for i := 0; i < numberOfMessages; i++ {
|
||||
buffer[tcpOutput.getBufferIndex(getTestBytes())]++
|
||||
}
|
||||
|
||||
expectedDistribution := numberOfMessages / numberOfWorkers
|
||||
lowerDistribution := expectedDistribution - (expectedDistribution * percentDistributionErrorRange / 100)
|
||||
upperDistribution := expectedDistribution + (expectedDistribution * percentDistributionErrorRange / 100)
|
||||
for i := 0; i < numberOfWorkers; i++ {
|
||||
if buffer[i] < lowerDistribution {
|
||||
t.Errorf("Under expected distribution. Got %d expected lower distribution %d", buffer[i], lowerDistribution)
|
||||
}
|
||||
if buffer[i] > upperDistribution {
|
||||
t.Errorf("Under expected distribution. Got %d expected upper distribution %d", buffer[i], upperDistribution)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func getTestBytes() []byte {
|
||||
reqh := payloadHeader(RequestPayload, uuid(), time.Now().UnixNano(), -1)
|
||||
reqb := append(reqh, []byte("GET / HTTP/1.1\r\nHost: www.w3.org\r\nUser-Agent: Go 1.1 package http\r\nAccept-Encoding: gzip\r\n\r\n")...)
|
||||
return reqb
|
||||
close(quit)
|
||||
}
|
||||
|
||||
+18
-27
@@ -5,6 +5,7 @@ import (
|
||||
"reflect"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// InOutPlugins struct for holding references to plugins
|
||||
@@ -17,7 +18,7 @@ type InOutPlugins struct {
|
||||
var pluginMu sync.Mutex
|
||||
|
||||
// Plugins holds all the plugin objects
|
||||
var plugins *InOutPlugins = new(InOutPlugins)
|
||||
var Plugins *InOutPlugins = new(InOutPlugins)
|
||||
|
||||
// extractLimitOptions detects if plugin get called with limiter support
|
||||
// Returns address and limit
|
||||
@@ -67,18 +68,18 @@ func registerPlugin(constructor interface{}, options ...interface{}) {
|
||||
|
||||
// Some of the output can be Readers as well because return responses
|
||||
if isR && !isW {
|
||||
plugins.Inputs = append(plugins.Inputs, pluginWrapper.(io.Reader))
|
||||
Plugins.Inputs = append(Plugins.Inputs, pluginWrapper.(io.Reader))
|
||||
}
|
||||
|
||||
if isW {
|
||||
plugins.Outputs = append(plugins.Outputs, pluginWrapper.(io.Writer))
|
||||
Plugins.Outputs = append(Plugins.Outputs, pluginWrapper.(io.Writer))
|
||||
}
|
||||
|
||||
plugins.All = append(plugins.All, plugin)
|
||||
Plugins.All = append(Plugins.All, plugin)
|
||||
}
|
||||
|
||||
// InitPlugins specify and initialize all available plugins
|
||||
func InitPlugins() *InOutPlugins {
|
||||
func InitPlugins() {
|
||||
pluginMu.Lock()
|
||||
defer pluginMu.Unlock()
|
||||
|
||||
@@ -90,43 +91,39 @@ func InitPlugins() *InOutPlugins {
|
||||
registerPlugin(NewDummyOutput)
|
||||
}
|
||||
|
||||
if Settings.OutputStdout {
|
||||
if Settings.outputStdout {
|
||||
registerPlugin(NewDummyOutput)
|
||||
}
|
||||
|
||||
if Settings.OutputNull {
|
||||
if Settings.outputNull {
|
||||
registerPlugin(NewNullOutput)
|
||||
}
|
||||
|
||||
engine := EnginePcap
|
||||
if Settings.InputRAWEngine == "raw_socket" {
|
||||
if Settings.inputRAWEngine == "raw_socket" {
|
||||
engine = EngineRawSocket
|
||||
} else if Settings.InputRAWEngine == "pcap_file" {
|
||||
} else if Settings.inputRAWEngine == "pcap_file" {
|
||||
engine = EnginePcapFile
|
||||
}
|
||||
|
||||
for _, options := range Settings.inputRAW {
|
||||
registerPlugin(NewRAWInput, options, engine, Settings.InputRAWTrackResponse, Settings.InputRAWExpire, Settings.InputRAWRealIPHeader, Settings.InputRAWProtocol, Settings.InputRAWBpfFilter, Settings.InputRAWTimestampType, Settings.InputRAWBufferSize)
|
||||
registerPlugin(NewRAWInput, options, engine, Settings.inputRAWTrackResponse, time.Duration(0), Settings.inputRAWRealIPHeader)
|
||||
}
|
||||
|
||||
for _, options := range Settings.inputTCP {
|
||||
registerPlugin(NewTCPInput, options, &Settings.InputTCPConfig)
|
||||
registerPlugin(NewTCPInput, options)
|
||||
}
|
||||
|
||||
for _, options := range Settings.outputTCP {
|
||||
registerPlugin(NewTCPOutput, options, &Settings.OutputTCPConfig)
|
||||
registerPlugin(NewTCPOutput, options)
|
||||
}
|
||||
|
||||
for _, options := range Settings.inputFile {
|
||||
registerPlugin(NewFileInput, options, Settings.InputFileLoop)
|
||||
registerPlugin(NewFileInput, options, Settings.inputFileLoop)
|
||||
}
|
||||
|
||||
for _, path := range Settings.outputFile {
|
||||
if strings.HasPrefix(path, "s3://") {
|
||||
registerPlugin(NewS3Output, path, &Settings.OutputFileConfig)
|
||||
} else {
|
||||
registerPlugin(NewFileOutput, path, &Settings.OutputFileConfig)
|
||||
}
|
||||
for _, options := range Settings.outputFile {
|
||||
registerPlugin(NewFileOutput, options, &Settings.outputFileConfig)
|
||||
}
|
||||
|
||||
for _, options := range Settings.inputHTTP {
|
||||
@@ -137,17 +134,13 @@ func InitPlugins() *InOutPlugins {
|
||||
// Fix: https://github.com/buger/gor/issues/174
|
||||
for _, header := range Settings.modifierConfig.headers {
|
||||
if header.Name == "Host" {
|
||||
Settings.OutputHTTPConfig.OriginalHost = true
|
||||
Settings.outputHTTPConfig.OriginalHost = true
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
for _, options := range Settings.outputHTTP {
|
||||
registerPlugin(NewHTTPOutput, options, &Settings.OutputHTTPConfig)
|
||||
}
|
||||
|
||||
for _, options := range Settings.outputBinary {
|
||||
registerPlugin(NewBinaryOutput, options, &Settings.OutputBinaryConfig)
|
||||
registerPlugin(NewHTTPOutput, options, &Settings.outputHTTPConfig)
|
||||
}
|
||||
|
||||
if Settings.outputKafkaConfig.host != "" && Settings.outputKafkaConfig.topic != "" {
|
||||
@@ -157,6 +150,4 @@ func InitPlugins() *InOutPlugins {
|
||||
if Settings.inputKafkaConfig.host != "" && Settings.inputKafkaConfig.topic != "" {
|
||||
registerPlugin(NewKafkaInput, "", &Settings.inputKafkaConfig)
|
||||
}
|
||||
|
||||
return plugins
|
||||
}
|
||||
|
||||
+13
-9
@@ -1,38 +1,42 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"io"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestPluginsRegistration(t *testing.T) {
|
||||
Plugins.Inputs = []io.Reader{}
|
||||
Plugins.Outputs = []io.Writer{}
|
||||
|
||||
Settings.inputDummy = MultiOption{"[]"}
|
||||
Settings.outputDummy = MultiOption{"[]"}
|
||||
Settings.outputHTTP = MultiOption{"www.example.com|10"}
|
||||
Settings.inputFile = MultiOption{"/dev/null"}
|
||||
|
||||
plugins := InitPlugins()
|
||||
InitPlugins()
|
||||
|
||||
if len(plugins.Inputs) != 2 {
|
||||
t.Errorf("Should be 2 inputs %d", len(plugins.Inputs))
|
||||
if len(Plugins.Inputs) != 2 {
|
||||
t.Errorf("Should be 2 inputs %d", len(Plugins.Inputs))
|
||||
}
|
||||
|
||||
if _, ok := plugins.Inputs[0].(*DummyInput); !ok {
|
||||
if _, ok := Plugins.Inputs[0].(*DummyInput); !ok {
|
||||
t.Errorf("First input should be DummyInput")
|
||||
}
|
||||
|
||||
if _, ok := plugins.Inputs[1].(*FileInput); !ok {
|
||||
if _, ok := Plugins.Inputs[1].(*FileInput); !ok {
|
||||
t.Errorf("Second input should be FileInput")
|
||||
}
|
||||
|
||||
if len(plugins.Outputs) != 2 {
|
||||
t.Errorf("Should be 2 output %d", len(plugins.Outputs))
|
||||
if len(Plugins.Outputs) != 2 {
|
||||
t.Errorf("Should be 2 output %d", len(Plugins.Outputs))
|
||||
}
|
||||
|
||||
if _, ok := plugins.Outputs[0].(*DummyOutput); !ok {
|
||||
if _, ok := Plugins.Outputs[0].(*DummyOutput); !ok {
|
||||
t.Errorf("First output should be DummyOutput")
|
||||
}
|
||||
|
||||
if l, ok := plugins.Outputs[1].(*Limiter); ok {
|
||||
if l, ok := Plugins.Outputs[1].(*Limiter); ok {
|
||||
if _, ok := l.plugin.(*HTTPOutput); !ok {
|
||||
t.Errorf("HTTPOutput should be wrapped in limiter")
|
||||
}
|
||||
|
||||
@@ -1,5 +0,0 @@
|
||||
package main
|
||||
|
||||
// PRO this value indicates if goreplay is running in PRO mode..
|
||||
// it must not be modified explicitly in production
|
||||
var PRO = false
|
||||
+66
-29
@@ -18,8 +18,7 @@ package proto
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
|
||||
"github.com/buger/goreplay/byteutils"
|
||||
"github.com/buger/gor/byteutils"
|
||||
)
|
||||
|
||||
// In HTTP newline defined by 2 bytes (for both windows and *nix support)
|
||||
@@ -42,23 +41,82 @@ func MIMEHeadersStartPos(payload []byte) int {
|
||||
return bytes.Index(payload, CLRF) + 2 // Find first line end
|
||||
}
|
||||
|
||||
func isLower(b byte) bool {
|
||||
if 'a' <= b && b <= 'z' {
|
||||
return true
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
func toUpper(b byte) byte {
|
||||
if 'a' <= b && b <= 'z' {
|
||||
b -= 'a' - 'A'
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
func toLower(b byte) byte {
|
||||
if 'A' <= b && b <= 'Z' {
|
||||
b += 'a' - 'A'
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
func headerIndex(payload []byte, name []byte) int {
|
||||
isLower := isLower(name[0])
|
||||
i := 0
|
||||
|
||||
for {
|
||||
// we need enough space for at least '\n' and the header name
|
||||
if i >= (len(payload) - len(name) - 1) {
|
||||
if i >= len(payload) {
|
||||
return -1
|
||||
}
|
||||
|
||||
if payload[i] == '\n' {
|
||||
i++
|
||||
if bytes.EqualFold(name, payload[i:i+len(name)]) {
|
||||
return i
|
||||
|
||||
// We are at the end
|
||||
if i == len(payload) {
|
||||
return -1
|
||||
}
|
||||
|
||||
if payload[i] == name[0] ||
|
||||
(!isLower && payload[i] == toLower(name[0])) ||
|
||||
(isLower && payload[i] == toUpper(name[0])) {
|
||||
|
||||
i++
|
||||
j := 1
|
||||
for {
|
||||
if j == len(name) {
|
||||
// Matched, and return start of the header
|
||||
return i - len(name)
|
||||
}
|
||||
|
||||
if payload[i] != name[j] {
|
||||
break
|
||||
}
|
||||
|
||||
// If compound header name do one more case check: Content-Length or Transfer-Encoding
|
||||
if name[j] == '-' {
|
||||
i++
|
||||
j++
|
||||
|
||||
if !(payload[i] == name[j] ||
|
||||
(!isLower && payload[i] == toLower(name[j])) ||
|
||||
(isLower && payload[i] == toUpper(name[j]))) {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
j++
|
||||
i++
|
||||
}
|
||||
}
|
||||
}
|
||||
i++
|
||||
|
||||
i++
|
||||
}
|
||||
|
||||
return -1
|
||||
}
|
||||
|
||||
@@ -279,35 +337,14 @@ func DeleteHeader(payload, name []byte) []byte {
|
||||
// Body returns request/response body
|
||||
func Body(payload []byte) []byte {
|
||||
// 4 -> len(EMPTY_LINE)
|
||||
if len(payload) < 4 {
|
||||
return []byte{}
|
||||
}
|
||||
|
||||
return payload[MIMEHeadersEndPos(payload):]
|
||||
}
|
||||
|
||||
// Path takes payload and retuns request path: Split(firstLine, ' ')[1]
|
||||
func Path(payload []byte) []byte {
|
||||
start := bytes.IndexByte(payload, ' ') + 1
|
||||
eol := bytes.IndexByte(payload[start:], '\r')
|
||||
end := bytes.IndexByte(payload[start:], ' ')
|
||||
|
||||
if eol > 0 {
|
||||
if end == -1 || eol < end {
|
||||
return payload[start : start+eol]
|
||||
}
|
||||
} else { // support for legacy clients
|
||||
eol = bytes.IndexByte(payload[start:], '\n')
|
||||
|
||||
if eol > 0 && (end == -1 || eol < end) {
|
||||
return payload[start : start+eol]
|
||||
}
|
||||
}
|
||||
|
||||
if end < 0 {
|
||||
return payload[start:]
|
||||
}
|
||||
|
||||
return payload[start : start+end]
|
||||
}
|
||||
|
||||
@@ -412,7 +449,7 @@ func Status(payload []byte) []byte {
|
||||
}
|
||||
|
||||
var httpMethods []string = []string{
|
||||
"GET ", "OPTI", "HEAD", "POST", "PUT ", "DELE", "TRAC", "CONN", "PATC" /* custom methods */, "BAN ", "PURG", "PROP", "MKCO", "COPY", "MOVE", "LOCK", "UNLO",
|
||||
"GET ", "OPTI", "HEAD", "POST", "PUT ", "DELE", "TRAC", "CONN", "PATC" /* custom methods */, "BAN", "PURG",
|
||||
}
|
||||
|
||||
func IsHTTPPayload(payload []byte) bool {
|
||||
|
||||
+4
-22
@@ -65,13 +65,13 @@ func TestHeader(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestMIMEHeadersEndPos(t *testing.T) {
|
||||
head := []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\n")
|
||||
head := []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org")
|
||||
payload := []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
|
||||
|
||||
end := MIMEHeadersEndPos(payload)
|
||||
|
||||
if !bytes.Equal(payload[:end], head) {
|
||||
t.Error("Wrong headers end position:", end, head, payload[:end])
|
||||
t.Error("Wrong headers end position:", end)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -80,10 +80,10 @@ func TestMIMEHeadersStartPos(t *testing.T) {
|
||||
payload := []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
|
||||
|
||||
start := MIMEHeadersStartPos(payload)
|
||||
end := MIMEHeadersEndPos(payload) - 4
|
||||
end := MIMEHeadersEndPos(payload)
|
||||
|
||||
if !bytes.Equal(payload[start:end], headers) {
|
||||
t.Error("Wrong headers end position:", start, end, payload[start:end])
|
||||
t.Error("Wrong headers end position:", start, end)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -237,24 +237,6 @@ func TestPath(t *testing.T) {
|
||||
if path = Path(payload); !bytes.Equal(path, []byte("/post")) {
|
||||
t.Error("Should find path", string(path))
|
||||
}
|
||||
|
||||
payload = []byte("GET /get\r\n\r\nHost: www.w3.org\r\n\r\n")
|
||||
|
||||
if path = Path(payload); !bytes.Equal(path, []byte("/get")) {
|
||||
t.Error("Should find path", string(path))
|
||||
}
|
||||
|
||||
payload = []byte("GET /get\n")
|
||||
|
||||
if path = Path(payload); !bytes.Equal(path, []byte("/get")) {
|
||||
t.Error("Should find path", string(path))
|
||||
}
|
||||
|
||||
payload = []byte("GET /get")
|
||||
|
||||
if path = Path(payload); !bytes.Equal(path, []byte("/get")) {
|
||||
t.Error("Should find path", string(path))
|
||||
}
|
||||
}
|
||||
|
||||
func TestSetPath(t *testing.T) {
|
||||
|
||||
+5
-20
@@ -7,25 +7,20 @@ import (
|
||||
"strconv"
|
||||
)
|
||||
|
||||
// These constants help to indicate the type of payload
|
||||
const (
|
||||
RequestPayload = '1'
|
||||
ResponsePayload = '2'
|
||||
ReplayedResponsePayload = '3'
|
||||
)
|
||||
|
||||
func randByte(len int) []byte {
|
||||
b := make([]byte, len/2)
|
||||
func uuid() []byte {
|
||||
b := make([]byte, 20)
|
||||
rand.Read(b)
|
||||
|
||||
h := make([]byte, len)
|
||||
hex.Encode(h, b)
|
||||
uuid := make([]byte, 40)
|
||||
hex.Encode(uuid, b)
|
||||
|
||||
return h
|
||||
}
|
||||
|
||||
func uuid() []byte {
|
||||
return randByte(24)
|
||||
return uuid
|
||||
}
|
||||
|
||||
var payloadSeparator = "\n🐵🙈🙉\n"
|
||||
@@ -94,16 +89,6 @@ func payloadMeta(payload []byte) [][]byte {
|
||||
return bytes.Split(payload[:headerSize], []byte{' '})
|
||||
}
|
||||
|
||||
func payloadID(payload []byte) []byte {
|
||||
idx := bytes.IndexByte(payload[2:], ' ')
|
||||
|
||||
if idx == -1 {
|
||||
return []byte{}
|
||||
}
|
||||
|
||||
return payload[2 : 2+idx]
|
||||
}
|
||||
|
||||
func isOriginPayload(payload []byte) bool {
|
||||
switch payload[0] {
|
||||
case RequestPayload, ResponsePayload:
|
||||
|
||||
@@ -1,16 +1,25 @@
|
||||
/*
|
||||
Package capture provides traffic sniffier using RAW sockets.
|
||||
Capture traffic from socket using RAW_SOCKET's http://en.wikipedia.org/wiki/Raw_socket
|
||||
RAW_SOCKET allows you to listen for traffic from any port (e.g. sniffing) because they operate on IP level.
|
||||
Package rawSocket provides traffic sniffier using RAW sockets.
|
||||
|
||||
Capture traffic from socket using RAW_SOCKET's
|
||||
http://en.wikipedia.org/wiki/Raw_socket
|
||||
|
||||
RAW_SOCKET allow you listen for traffic on any port (e.g. sniffing) because they operate on IP level.
|
||||
|
||||
Ports is TCP feature, same as flow control, reliable transmission and etc.
|
||||
|
||||
This package implements own TCP layer: TCP packets is parsed using tcp_packet.go, and flow control is managed by tcp_message.go
|
||||
*/
|
||||
package capture
|
||||
package rawSocket
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/binary"
|
||||
"fmt"
|
||||
"github.com/buger/gor/proto"
|
||||
"github.com/google/gopacket"
|
||||
"github.com/google/gopacket/layers"
|
||||
"github.com/google/gopacket/pcap"
|
||||
"io"
|
||||
"log"
|
||||
"net"
|
||||
@@ -20,14 +29,10 @@ import (
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/buger/goreplay/proto"
|
||||
|
||||
"github.com/google/gopacket"
|
||||
"github.com/google/gopacket/layers"
|
||||
"github.com/google/gopacket/pcap"
|
||||
)
|
||||
|
||||
var _ = fmt.Println
|
||||
|
||||
type packet struct {
|
||||
srcIP []byte
|
||||
data []byte
|
||||
@@ -36,7 +41,7 @@ type packet struct {
|
||||
|
||||
// Listener handle traffic capture
|
||||
type Listener struct {
|
||||
sync.Mutex
|
||||
mu sync.Mutex
|
||||
// buffer of TCPMessages waiting to be send
|
||||
// ID -> TCPMessage
|
||||
messages map[tcpID]*TCPMessage
|
||||
@@ -65,20 +70,11 @@ type Listener struct {
|
||||
trackResponse bool
|
||||
messageExpire time.Duration
|
||||
|
||||
bpfFilter string
|
||||
timestampType string
|
||||
overrideSnapLen bool
|
||||
immediateMode bool
|
||||
|
||||
bufferSize int64
|
||||
|
||||
conn net.PacketConn
|
||||
pcapHandles []*pcap.Handle
|
||||
|
||||
quit chan bool
|
||||
ready bool
|
||||
|
||||
protocol TCPProtocol
|
||||
quit chan bool
|
||||
readyCh chan bool
|
||||
}
|
||||
|
||||
type request struct {
|
||||
@@ -95,12 +91,13 @@ const (
|
||||
)
|
||||
|
||||
// NewListener creates and initializes new Listener object
|
||||
func NewListener(addr string, port string, engine int, trackResponse bool, expire time.Duration, protocol TCPProtocol, bpfFilter string, timestampType string, bufferSize int64, overrideSnapLen bool, immediateMode bool) (l *Listener) {
|
||||
func NewListener(addr string, port string, engine int, trackResponse bool, expire time.Duration) (l *Listener) {
|
||||
l = &Listener{}
|
||||
|
||||
l.packetsChan = make(chan *packet, 10000)
|
||||
l.messagesChan = make(chan *TCPMessage, 10000)
|
||||
l.quit = make(chan bool)
|
||||
l.readyCh = make(chan bool, 1)
|
||||
|
||||
l.messages = make(map[tcpID]*TCPMessage)
|
||||
l.ackAliases = make(map[uint32]uint32)
|
||||
@@ -108,12 +105,6 @@ func NewListener(addr string, port string, engine int, trackResponse bool, expir
|
||||
l.respAliases = make(map[uint32]*TCPMessage)
|
||||
l.respWithoutReq = make(map[uint32]tcpID)
|
||||
l.trackResponse = trackResponse
|
||||
l.protocol = protocol
|
||||
l.bpfFilter = bpfFilter
|
||||
l.timestampType = timestampType
|
||||
l.immediateMode = immediateMode
|
||||
l.bufferSize = bufferSize
|
||||
l.overrideSnapLen = overrideSnapLen
|
||||
|
||||
l.addr = addr
|
||||
_port, _ := strconv.Atoi(port)
|
||||
@@ -134,8 +125,6 @@ func NewListener(addr string, port string, engine int, trackResponse bool, expir
|
||||
go l.readPcap()
|
||||
case EnginePcapFile:
|
||||
go l.readPcapFile()
|
||||
case EngineRawSocket:
|
||||
go l.readRAWSocket()
|
||||
default:
|
||||
log.Fatal("Unknown traffic interception engine:", engine)
|
||||
}
|
||||
@@ -191,7 +180,7 @@ func (t *Listener) dispatchMessage(message *TCPMessage) {
|
||||
|
||||
t.deleteMessage(message)
|
||||
|
||||
if t.protocol == ProtocolHTTP && !message.complete {
|
||||
if !message.complete {
|
||||
if !message.IsIncoming {
|
||||
delete(t.respAliases, message.Ack)
|
||||
delete(t.respWithoutReq, message.Ack)
|
||||
@@ -252,7 +241,7 @@ func (e *DeviceNotFoundError) Error() string {
|
||||
devices, _ := pcap.FindAllDevs()
|
||||
|
||||
if len(devices) == 0 {
|
||||
return "Can't get list of network interfaces, ensure that you running Gor as root user or sudo.\nTo run as non-root users see this docs https://github.com/buger/goreplay/wiki/Running-as-non-root-user"
|
||||
return "Can't get list of network interfaces, ensure that you running Gor as root user or sudo.\nTo run as non-root users see this docs https://github.com/buger/gor/wiki/Running-as-non-root-user"
|
||||
}
|
||||
|
||||
var msg string
|
||||
@@ -314,9 +303,9 @@ func findPcapDevices(addr string) (interfaces []pcap.Interface, err error) {
|
||||
|
||||
if len(interfaces) == 0 {
|
||||
return nil, &DeviceNotFoundError{addr}
|
||||
} else {
|
||||
return interfaces, nil
|
||||
}
|
||||
|
||||
return interfaces, nil
|
||||
}
|
||||
|
||||
func (t *Listener) readPcap() {
|
||||
@@ -325,55 +314,25 @@ func (t *Listener) readPcap() {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
const BPFSupported = runtime.GOOS != "darwin"
|
||||
bpfSupported := true
|
||||
if runtime.GOOS == "darwin" {
|
||||
bpfSupported = false
|
||||
}
|
||||
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(len(devices))
|
||||
|
||||
for _, d := range devices {
|
||||
go func(device pcap.Interface) {
|
||||
inactive, err := pcap.NewInactiveHandle(device.Name)
|
||||
handle, err := pcap.OpenLive(device.Name, 65536, true, t.messageExpire)
|
||||
if err != nil {
|
||||
inactive.CleanUp()
|
||||
log.Println("Pcap Error while opening device", device.Name, err)
|
||||
wg.Done()
|
||||
return
|
||||
}
|
||||
|
||||
if t.timestampType != "" {
|
||||
if tt, terr := pcap.TimestampSourceFromString(t.timestampType); terr != nil {
|
||||
log.Println("Supported timestamp types: ", inactive.SupportedTimestamps(), device.Name)
|
||||
} else if terr := inactive.SetTimestampSource(tt); terr != nil {
|
||||
log.Println("Supported timestamp types: ", inactive.SupportedTimestamps(), device.Name)
|
||||
}
|
||||
}
|
||||
|
||||
if it, err := net.InterfaceByName(device.Name); err == nil && !t.overrideSnapLen {
|
||||
// Auto-guess max length of packet to capture
|
||||
inactive.SetSnapLen(it.MTU + 68*2)
|
||||
} else {
|
||||
inactive.SetSnapLen(65536)
|
||||
}
|
||||
inactive.SetTimeout(t.messageExpire)
|
||||
inactive.SetPromisc(true)
|
||||
inactive.SetImmediateMode(t.immediateMode)
|
||||
if t.immediateMode {
|
||||
log.Println("Setting immediate mode")
|
||||
}
|
||||
if t.bufferSize > 0 {
|
||||
inactive.SetBufferSize(int(t.bufferSize))
|
||||
}
|
||||
|
||||
handle, herr := inactive.Activate()
|
||||
if herr != nil {
|
||||
log.Printf("PCAP Activate device '%s' error: %s\n", device.Name, herr)
|
||||
wg.Done()
|
||||
return
|
||||
}
|
||||
|
||||
defer handle.Close()
|
||||
|
||||
t.Lock()
|
||||
t.mu.Lock()
|
||||
t.pcapHandles = append(t.pcapHandles, handle)
|
||||
|
||||
var bpfDstHost, bpfSrcHost string
|
||||
@@ -400,17 +359,13 @@ func (t *Listener) readPcap() {
|
||||
}
|
||||
}
|
||||
|
||||
if BPFSupported {
|
||||
if bpfSupported {
|
||||
var bpf string
|
||||
|
||||
if t.bpfFilter != "" {
|
||||
bpf = t.bpfFilter
|
||||
if t.trackResponse {
|
||||
bpf = "(tcp dst port " + strconv.Itoa(int(t.port)) + " and (" + bpfDstHost + ")) or (" + "tcp src port " + strconv.Itoa(int(t.port)) + " and (" + bpfSrcHost + "))"
|
||||
} else {
|
||||
if t.trackResponse {
|
||||
bpf = fmt.Sprintf("(tcp dst port %d and (%s)) or (tcp src port %d and (%s))", t.port, bpfDstHost, t.port, bpfSrcHost)
|
||||
} else {
|
||||
bpf = fmt.Sprintf("(tcp dst port %d and (%s))", t.port, bpfDstHost)
|
||||
}
|
||||
bpf = "tcp dst port " + strconv.Itoa(int(t.port)) + " and (" + bpfDstHost + ")"
|
||||
}
|
||||
|
||||
if err := handle.SetBPFFilter(bpf); err != nil {
|
||||
@@ -419,7 +374,7 @@ func (t *Listener) readPcap() {
|
||||
return
|
||||
}
|
||||
}
|
||||
t.Unlock()
|
||||
t.mu.Unlock()
|
||||
|
||||
var decoder gopacket.Decoder
|
||||
|
||||
@@ -460,14 +415,15 @@ func (t *Listener) readPcap() {
|
||||
of = 4
|
||||
case layers.LinkTypeLoop:
|
||||
of = 4
|
||||
case layers.LinkTypeRaw, layers.LayerTypeIPv4:
|
||||
case layers.LinkTypeRaw:
|
||||
of = 0
|
||||
case layers.LinkTypeLinuxSLL:
|
||||
of = 16
|
||||
default:
|
||||
log.Println("Unknown packet layer", decoder, packet)
|
||||
log.Println("Unknown packet layer", packet)
|
||||
break
|
||||
}
|
||||
|
||||
data = packet.Data()[of:]
|
||||
|
||||
version := uint8(data[0]) >> 4
|
||||
@@ -519,13 +475,13 @@ func (t *Listener) readPcap() {
|
||||
continue
|
||||
}
|
||||
|
||||
dataOffset := data[12] >> 4
|
||||
dataOffset := (data[12] & 0xF0) >> 4
|
||||
isFIN := data[13]&0x01 != 0
|
||||
|
||||
// We need only packets with data inside
|
||||
// Check that the buffer is larger than the size of the TCP header
|
||||
if len(data) > int(dataOffset*4) || isFIN {
|
||||
if !BPFSupported {
|
||||
if !bpfSupported {
|
||||
destPort := binary.BigEndian.Uint16(data[2:4])
|
||||
srcPort := binary.BigEndian.Uint16(data[0:2])
|
||||
|
||||
@@ -579,25 +535,14 @@ func (t *Listener) readPcap() {
|
||||
}
|
||||
|
||||
wg.Wait()
|
||||
t.Lock()
|
||||
t.ready = true
|
||||
t.Unlock()
|
||||
t.readyCh <- true
|
||||
}
|
||||
|
||||
func (t *Listener) readPcapFile() {
|
||||
if handle, err := pcap.OpenOffline(t.addr); err != nil {
|
||||
log.Fatal(err)
|
||||
} else {
|
||||
if t.bpfFilter != "" {
|
||||
if err := handle.SetBPFFilter(t.bpfFilter); err != nil {
|
||||
log.Println("BPF filter error:", err)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
t.Lock()
|
||||
t.ready = true
|
||||
t.Unlock()
|
||||
t.readyCh <- true
|
||||
packetSource := gopacket.NewPacketSource(handle, handle.LinkType())
|
||||
|
||||
for {
|
||||
@@ -615,12 +560,12 @@ func (t *Listener) readPcapFile() {
|
||||
tcp, _ := tcpLayer.(*layers.TCP)
|
||||
data = append(tcp.LayerContents(), tcp.LayerPayload()...)
|
||||
|
||||
if uint16(tcp.DstPort) == t.port {
|
||||
copy(data[0:2], []byte{byte(tcp.SrcPort >> 8), byte(tcp.SrcPort)})
|
||||
copy(data[2:4], []byte{byte(tcp.DstPort >> 8), byte(tcp.DstPort)})
|
||||
if tcp.SrcPort >= 32768 && tcp.SrcPort <= 61000 {
|
||||
copy(data[0:2], []byte{0, 0})
|
||||
copy(data[2:4], []byte{0, 1})
|
||||
} else {
|
||||
copy(data[0:2], []byte{byte(tcp.DstPort >> 8), byte(tcp.DstPort)})
|
||||
copy(data[2:4], []byte{byte(tcp.SrcPort >> 8), byte(tcp.SrcPort)})
|
||||
copy(data[0:2], []byte{0, 1})
|
||||
copy(data[2:4], []byte{0, 0})
|
||||
}
|
||||
} else {
|
||||
continue
|
||||
@@ -637,7 +582,7 @@ func (t *Listener) readPcapFile() {
|
||||
continue
|
||||
}
|
||||
|
||||
dataOffset := data[12] >> 4
|
||||
dataOffset := (data[12] & 0xF0) >> 4
|
||||
isFIN := data[13]&0x01 != 0
|
||||
|
||||
// We need only packets with data inside
|
||||
@@ -654,47 +599,44 @@ func (t *Listener) readPcapFile() {
|
||||
func (t *Listener) readRAWSocket() {
|
||||
conn, e := net.ListenPacket("ip:tcp", t.addr)
|
||||
t.conn = conn
|
||||
|
||||
if e != nil {
|
||||
log.Fatal(e)
|
||||
}
|
||||
|
||||
defer t.conn.Close()
|
||||
type RSPacket struct {
|
||||
buf []byte
|
||||
addr net.Addr
|
||||
err error
|
||||
n int
|
||||
}
|
||||
var bufChan = make(chan *RSPacket, 1000)
|
||||
t.Lock()
|
||||
t.ready = true
|
||||
t.Unlock()
|
||||
go func() {
|
||||
for {
|
||||
// Re-allocate data object to avoid data collision
|
||||
var buf [64 * 104 * 1024]byte
|
||||
// Note: ReadFrom receive messages without IP header
|
||||
n, addr, err := t.conn.ReadFrom(buf[:])
|
||||
bufChan <- &RSPacket{buf[:], addr, err, n}
|
||||
}
|
||||
}()
|
||||
|
||||
buf := make([]byte, 64*1024) // 64kb
|
||||
|
||||
t.readyCh <- true
|
||||
|
||||
for {
|
||||
packet := <-bufChan
|
||||
if packet.err != nil {
|
||||
if strings.HasSuffix(packet.err.Error(), "closed network connection") {
|
||||
// Note: ReadFrom receive messages without IP header
|
||||
n, addr, err := t.conn.ReadFrom(buf)
|
||||
|
||||
if err != nil {
|
||||
if strings.HasSuffix(err.Error(), "closed network connection") {
|
||||
return
|
||||
} else {
|
||||
continue
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
if packet.n > 0 {
|
||||
if t.isValidPacket(packet.buf[:packet.n]) {
|
||||
t.packetsChan <- t.buildPacket([]byte(packet.addr.(*net.IPAddr).IP), packet.buf[:packet.n], time.Now())
|
||||
if n > 0 {
|
||||
if t.isValidPacket(buf[:n]) {
|
||||
t.packetsChan <- t.buildPacket([]byte(addr.(*net.IPAddr).IP), buf[:n], time.Now())
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (t *Listener) buildPacket(packetSrcIP []byte, packetData []byte, timestamp time.Time) *packet {
|
||||
copyPacketSrcIP := make([]byte, 16)
|
||||
copyPacketData := make([]byte, len(packetData))
|
||||
|
||||
copy(copyPacketSrcIP, packetSrcIP)
|
||||
copy(copyPacketData, packetSrcIP)
|
||||
|
||||
return &packet{
|
||||
srcIP: packetSrcIP,
|
||||
data: packetData,
|
||||
@@ -707,13 +649,16 @@ func (t *Listener) isValidPacket(buf []byte) bool {
|
||||
// http://en.wikipedia.org/wiki/Transmission_Control_Protocol
|
||||
destPort := binary.BigEndian.Uint16(buf[2:4])
|
||||
srcPort := binary.BigEndian.Uint16(buf[0:2])
|
||||
|
||||
// Because RAW_SOCKET can't be bound to port, we have to control it by ourself
|
||||
if destPort == t.port || (t.trackResponse && srcPort == t.port) {
|
||||
// Get the 'data offset' (size of the TCP header in 32-bit words)
|
||||
dataOffset := buf[12] >> 4
|
||||
dataOffset := (buf[12] & 0xF0) >> 4
|
||||
|
||||
// We need only packets with data inside
|
||||
// Check that the buffer is larger than the size of the TCP header
|
||||
if len(buf) > int(dataOffset*4) {
|
||||
// We should create new buffer because go slices is pointers. So buffer data shoud be immutable.
|
||||
return true
|
||||
}
|
||||
}
|
||||
@@ -732,45 +677,32 @@ func (t *Listener) processTCPPacket(packet *TCPPacket) {
|
||||
}
|
||||
}()
|
||||
|
||||
var responseRequest *TCPMessage
|
||||
var message *TCPMessage
|
||||
|
||||
isIncoming := packet.DestPort == t.port
|
||||
|
||||
if t.protocol == ProtocolHTTP {
|
||||
if !isIncoming {
|
||||
responseRequest, _ = t.respAliases[packet.Ack]
|
||||
}
|
||||
// Seek for 100-expect chunks
|
||||
if parentAck, ok := t.seqWithData[packet.Seq]; ok {
|
||||
// In case if non-first data chunks comes first
|
||||
for _, m := range t.messages {
|
||||
if m.Ack == packet.Ack && bytes.Equal(m.packets[0].Addr, packet.Addr) {
|
||||
t.deleteMessage(m)
|
||||
|
||||
// Seek for 100-expect chunks
|
||||
// `packet.Ack != parentAck` is protection for clients who send data without ignoring server 100-continue response, e.g have data chunks have same Ack
|
||||
if parentAck, ok := t.seqWithData[packet.Seq]; ok && packet.Ack != parentAck {
|
||||
// Skip zero-length chunks https://github.com/buger/goreplay/issues/496
|
||||
if len(packet.Data) == 0 {
|
||||
return
|
||||
}
|
||||
if m.AssocMessage != nil {
|
||||
m.setAssocMessage(nil)
|
||||
}
|
||||
|
||||
// In case if non-first data chunks comes first
|
||||
for _, m := range t.messages {
|
||||
if m.Ack == packet.Ack && bytes.Equal(m.packets[0].Addr, packet.Addr) {
|
||||
t.deleteMessage(m)
|
||||
|
||||
if m.AssocMessage != nil {
|
||||
m.AssocMessage.setAssocMessage(nil)
|
||||
m.setAssocMessage(nil)
|
||||
}
|
||||
for _, pkt := range m.packets {
|
||||
// log.Println("Updating ack", parentAck, pkt.Ack)
|
||||
pkt.UpdateAck(parentAck)
|
||||
// Re-queue this packets
|
||||
t.processTCPPacket(pkt)
|
||||
}
|
||||
for _, pkt := range m.packets {
|
||||
// log.Println("Updating ack", parentAck, pkt.Ack)
|
||||
pkt.UpdateAck(parentAck)
|
||||
// Re-queue this packets
|
||||
t.processTCPPacket(pkt)
|
||||
}
|
||||
}
|
||||
|
||||
t.ackAliases[packet.Ack] = parentAck
|
||||
packet.UpdateAck(parentAck)
|
||||
}
|
||||
|
||||
t.ackAliases[packet.Ack] = parentAck
|
||||
packet.UpdateAck(parentAck)
|
||||
}
|
||||
|
||||
if isIncoming && packet.IsFIN {
|
||||
@@ -785,10 +717,16 @@ func (t *Listener) processTCPPacket(packet *TCPPacket) {
|
||||
packet.UpdateAck(alias)
|
||||
}
|
||||
|
||||
var responseRequest *TCPMessage
|
||||
|
||||
if !isIncoming {
|
||||
responseRequest, _ = t.respAliases[packet.Ack]
|
||||
}
|
||||
|
||||
message, ok := t.messages[packet.ID]
|
||||
|
||||
if !ok {
|
||||
message = NewTCPMessage(packet.Seq, packet.Ack, isIncoming, t.protocol, packet.timestamp)
|
||||
message = NewTCPMessage(packet.Seq, packet.Ack, isIncoming, packet.timestamp)
|
||||
t.messages[packet.ID] = message
|
||||
|
||||
if !isIncoming {
|
||||
@@ -805,10 +743,9 @@ func (t *Listener) processTCPPacket(packet *TCPPacket) {
|
||||
message.AddPacket(packet)
|
||||
|
||||
// Handling Expect: 100-continue requests
|
||||
if t.protocol == ProtocolHTTP && message.expectType == httpExpect100Continue && len(message.packets) == message.headerPacket+1 {
|
||||
seq := packet.Seq + uint32(len(packet.Data))
|
||||
if message.expectType == httpExpect100Continue && len(message.packets) == message.headerPacket+1 {
|
||||
seq := packet.Seq + uint32(message.Size())
|
||||
t.seqWithData[seq] = packet.Ack
|
||||
|
||||
message.DataSeq = seq
|
||||
message.complete = false
|
||||
|
||||
@@ -818,9 +755,8 @@ func (t *Listener) processTCPPacket(packet *TCPPacket) {
|
||||
t.deleteMessage(m)
|
||||
if m.AssocMessage != nil {
|
||||
message.setAssocMessage(m.AssocMessage)
|
||||
m.AssocMessage.setAssocMessage(nil)
|
||||
}
|
||||
|
||||
// log.Println("2: Adding ack alias:", m.Ack, packet.Ack)
|
||||
t.ackAliases[m.Ack] = packet.Ack
|
||||
|
||||
for _, pkt := range m.packets {
|
||||
@@ -834,13 +770,7 @@ func (t *Listener) processTCPPacket(packet *TCPPacket) {
|
||||
packet.Data = proto.DeleteHeader(packet.Data, bExpectHeader)
|
||||
}
|
||||
|
||||
// If client do sends Expect: 100-continue but do not respect server response
|
||||
if message.expectType == httpExpect100Continue && (message.headerPacket != -1 && len(message.packets) > message.headerPacket+1) {
|
||||
delete(t.seqWithData, message.DataSeq)
|
||||
seq := packet.Seq + uint32(len(packet.Data))
|
||||
t.seqWithData[seq] = packet.Ack
|
||||
message.DataSeq = seq
|
||||
}
|
||||
// log.Println("Received message:", string(message.Bytes()), message.ID(), t.messages)
|
||||
|
||||
if isIncoming {
|
||||
// If message have multiple packets, delete previous alias
|
||||
@@ -884,12 +814,20 @@ func (t *Listener) processTCPPacket(packet *TCPPacket) {
|
||||
}
|
||||
}
|
||||
|
||||
// Receiver TCP messages from the listener channel
|
||||
func (t *Listener) IsReady() bool {
|
||||
select {
|
||||
case <-t.readyCh:
|
||||
return true
|
||||
case <-time.After(5 * time.Second):
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
// Receive TCP messages from the listener channel
|
||||
func (t *Listener) Receiver() chan *TCPMessage {
|
||||
return t.messagesChan
|
||||
}
|
||||
|
||||
// Close tcp listener
|
||||
func (t *Listener) Close() {
|
||||
close(t.quit)
|
||||
if t.conn != nil {
|
||||
@@ -1,4 +1,4 @@
|
||||
package capture
|
||||
package rawSocket
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
@@ -11,7 +11,8 @@ import (
|
||||
|
||||
func TestRawListenerInput(t *testing.T) {
|
||||
var req, resp *TCPMessage
|
||||
listener := NewListener("", "0", EnginePcap, true, 10*time.Millisecond, ProtocolHTTP, "", "", 0, false, false)
|
||||
|
||||
listener := NewListener("", "0", EnginePcap, true, 10*time.Millisecond)
|
||||
defer listener.Close()
|
||||
|
||||
reqPacket := buildPacket(true, 1, 1, []byte("GET / HTTP/1.1\r\n\r\n"), time.Now())
|
||||
@@ -45,148 +46,10 @@ func TestRawListenerInput(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func firstPacket(payload []byte) *TCPPacket {
|
||||
return buildPacket(
|
||||
true,
|
||||
1,
|
||||
1,
|
||||
payload,
|
||||
time.Now(),
|
||||
)
|
||||
}
|
||||
|
||||
func nextPacket(prev *TCPPacket, payload []byte) *TCPPacket {
|
||||
return buildPacket(
|
||||
prev.SrcPort == 1,
|
||||
prev.Ack,
|
||||
prev.Seq+uint32(len(prev.Data)),
|
||||
payload,
|
||||
prev.timestamp.Add(time.Millisecond),
|
||||
)
|
||||
}
|
||||
|
||||
func responsePacket(prev *TCPPacket, payload []byte) *TCPPacket {
|
||||
return buildPacket(
|
||||
!(prev.SrcPort == 1),
|
||||
prev.Seq+uint32(len(prev.Data)),
|
||||
prev.Ack,
|
||||
payload,
|
||||
prev.timestamp.Add(time.Millisecond),
|
||||
)
|
||||
}
|
||||
|
||||
func TestHEADRequestNoBody(t *testing.T) {
|
||||
listener := NewListener("", "0", EnginePcap, true, 10*time.Millisecond, ProtocolHTTP, "", "", 0, false, false)
|
||||
defer listener.Close()
|
||||
|
||||
reqPacket := firstPacket([]byte("HEAD / HTTP/1.1\r\nContent-Length: 0\r\n\r\n"))
|
||||
respPacket := responsePacket(reqPacket, []byte("HTTP/1.1 200 OK\r\nContent-Length: 100\r\n\r\n"))
|
||||
|
||||
listener.packetsChan <- reqPacket.dump()
|
||||
listener.packetsChan <- respPacket.dump()
|
||||
|
||||
var req, resp *TCPMessage
|
||||
select {
|
||||
case req = <-listener.messagesChan:
|
||||
case <-time.After(time.Millisecond):
|
||||
t.Error("Should return request immediately")
|
||||
return
|
||||
}
|
||||
|
||||
if !req.IsIncoming {
|
||||
t.Error("Should be request")
|
||||
}
|
||||
|
||||
select {
|
||||
case resp = <-listener.messagesChan:
|
||||
case <-time.After(20 * time.Millisecond):
|
||||
t.Error("Should return response immediately")
|
||||
return
|
||||
}
|
||||
|
||||
if resp.IsIncoming {
|
||||
t.Error("Should be response")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSingleAck100Continue(t *testing.T) {
|
||||
listener := NewListener("", "0", EnginePcap, true, 10*time.Millisecond, ProtocolHTTP, "", "", 0, false, false)
|
||||
defer listener.Close()
|
||||
|
||||
reqPacket1 := firstPacket([]byte("POST / HTTP/1.1\r\nExpect: 100-continue\r\nContent-Length: 4\r\n\r\n"))
|
||||
respPacket1 := responsePacket(reqPacket1, []byte(""))
|
||||
respPacket2 := responsePacket(reqPacket1, []byte("HTTP/1.1 100 Continue\r\n"))
|
||||
reqPacket2 := responsePacket(respPacket2, []byte("DATA"))
|
||||
respPacket3 := responsePacket(reqPacket2, []byte("HTTP/1.1 200 OK\r\n\r\n"))
|
||||
|
||||
result := []byte("POST / HTTP/1.1\r\nContent-Length: 4\r\n\r\nDATA")
|
||||
|
||||
testRawListener100Continue(t, listener, result,
|
||||
reqPacket1,
|
||||
respPacket1, respPacket2,
|
||||
reqPacket2,
|
||||
respPacket3)
|
||||
}
|
||||
|
||||
func Test100ContinueWithoutWaiting(t *testing.T) {
|
||||
listener := NewListener("", "0", EnginePcap, true, 10*time.Millisecond, ProtocolHTTP, "", "", 0, false, false)
|
||||
defer listener.Close()
|
||||
|
||||
req1 := firstPacket([]byte("POST / HTTP/1.1\r\nExpect: 100-continue\r\nContent-Length: 4\r\n\r\n"))
|
||||
req2 := nextPacket(req1, []byte("DATA"))
|
||||
resp1 := responsePacket(req1, []byte("HTTP/1.1 100 Continue\r\n"))
|
||||
resp2 := responsePacket(req2, []byte("HTTP/1.1 200 OK\r\n\r\n"))
|
||||
|
||||
result := []byte("POST / HTTP/1.1\r\nContent-Length: 4\r\n\r\nDATA")
|
||||
|
||||
testRawListener100Continue(t, listener, result,
|
||||
req1, req2, resp1, resp2)
|
||||
}
|
||||
|
||||
// Client first sends data without waiting 100-continue, but once response received, generate packets based on Ack payload
|
||||
func Test100ContinueMixed(t *testing.T) {
|
||||
listener := NewListener("", "0", EnginePcap, true, 10*time.Millisecond, ProtocolHTTP, "", "", 0, false, false)
|
||||
defer listener.Close()
|
||||
|
||||
req1 := firstPacket([]byte("POST / HTTP/1.1\r\nExpect: 100-continue\r\nContent-Length: 12\r\n\r\n"))
|
||||
req2 := nextPacket(req1, []byte("DAT1"))
|
||||
resp1 := responsePacket(req1, []byte("HTTP/1.1 100 Continue\r\n\r\n"))
|
||||
req3 := responsePacket(resp1, []byte("DAT2"))
|
||||
req3.Seq = req2.Seq + uint32(len(req2.Data))
|
||||
req4 := nextPacket(req3, []byte("DAT3"))
|
||||
resp2 := responsePacket(req4, []byte("HTTP/1.1 200 OK\r\n\r\n"))
|
||||
|
||||
result := []byte("POST / HTTP/1.1\r\nContent-Length: 12\r\n\r\nDAT1DAT2DAT3")
|
||||
|
||||
testRawListener100Continue(t, listener, result,
|
||||
req1, req2, req3, req4, resp1, resp2)
|
||||
}
|
||||
|
||||
func TestDoubleAck100Continue(t *testing.T) {
|
||||
listener := NewListener("", "0", EnginePcap, true, 10*time.Millisecond, ProtocolHTTP, "", "", 0, false, false)
|
||||
defer listener.Close()
|
||||
|
||||
reqPacket1 := firstPacket([]byte("POST / HTTP/1.1\r\nExpect: 100-continue\r\nContent-Length: 4\r\n\r\n"))
|
||||
|
||||
respPacket1 := responsePacket(reqPacket1, []byte(""))
|
||||
respPacket2 := responsePacket(reqPacket1, []byte("HTTP/1.1 100 Continue\r\n"))
|
||||
reqPacket2 := responsePacket(respPacket2, []byte(""))
|
||||
reqPacket3 := responsePacket(respPacket2, []byte("DATA"))
|
||||
respPacket3 := responsePacket(reqPacket3, []byte("HTTP/1.1 200 OK\r\n\r\n"))
|
||||
|
||||
result := []byte("POST / HTTP/1.1\r\nContent-Length: 4\r\n\r\nDATA")
|
||||
|
||||
testRawListener100Continue(t, listener, result,
|
||||
reqPacket1,
|
||||
respPacket1, respPacket2,
|
||||
reqPacket2, reqPacket3,
|
||||
respPacket3)
|
||||
}
|
||||
|
||||
func TestRawListenerInputResponseByClose(t *testing.T) {
|
||||
var req, resp *TCPMessage
|
||||
|
||||
listener := NewListener("", "0", EnginePcap, true, 10*time.Millisecond, ProtocolHTTP, "", "", 0, false, false)
|
||||
listener := NewListener("", "0", EnginePcap, true, 10*time.Millisecond)
|
||||
defer listener.Close()
|
||||
|
||||
reqPacket := buildPacket(true, 1, 1, []byte("GET / HTTP/1.1\r\n\r\n"), time.Now())
|
||||
@@ -226,7 +89,7 @@ func TestRawListenerInputResponseByClose(t *testing.T) {
|
||||
func TestRawListenerInputWithoutResponse(t *testing.T) {
|
||||
var req *TCPMessage
|
||||
|
||||
listener := NewListener("", "0", EnginePcap, false, 10*time.Millisecond, ProtocolHTTP, "", "", 0, false, false)
|
||||
listener := NewListener("", "0", EnginePcap, false, 10*time.Millisecond)
|
||||
defer listener.Close()
|
||||
|
||||
reqPacket := buildPacket(true, 1, 1, []byte("GET / HTTP/1.1\r\n\r\n"), time.Now())
|
||||
@@ -248,11 +111,11 @@ func TestRawListenerInputWithoutResponse(t *testing.T) {
|
||||
func TestRawListenerResponse(t *testing.T) {
|
||||
var req, resp *TCPMessage
|
||||
|
||||
listener := NewListener("", "0", EnginePcap, true, 10*time.Millisecond, ProtocolHTTP, "", "", 0, false, false)
|
||||
listener := NewListener("", "0", EnginePcap, true, 10*time.Millisecond)
|
||||
defer listener.Close()
|
||||
|
||||
reqPacket := firstPacket([]byte("GET / HTTP/1.1\r\n\r\n"))
|
||||
respPacket := responsePacket(reqPacket, []byte("HTTP/1.1 200 OK\r\n\r\n"))
|
||||
reqPacket := buildPacket(true, 1, 1, []byte("GET / HTTP/1.1\r\n\r\n"), time.Now())
|
||||
respPacket := buildPacket(false, 1+uint32(len(reqPacket.Data)), 2, []byte("HTTP/1.1 200 OK\r\n\r\n"), time.Now())
|
||||
|
||||
// If response packet comes before request
|
||||
listener.packetsChan <- respPacket.dump()
|
||||
@@ -261,7 +124,7 @@ func TestRawListenerResponse(t *testing.T) {
|
||||
select {
|
||||
case req = <-listener.messagesChan:
|
||||
case <-time.After(time.Millisecond):
|
||||
t.Error("Should return request immediately")
|
||||
t.Error("Should return respose immediately")
|
||||
return
|
||||
}
|
||||
|
||||
@@ -285,37 +148,62 @@ func TestRawListenerResponse(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func get100ContinuePackets() (req []*TCPPacket, resp []*TCPPacket) {
|
||||
req1 := firstPacket([]byte("POST / HTTP/1.1\r\nExpect: 100-continue\r\nContent-Length: 2\r\n\r\n"))
|
||||
resp1 := responsePacket(req1, []byte("HTTP/1.1 100 Continue\r\n"))
|
||||
req2 := responsePacket(resp1, []byte("a"))
|
||||
req3 := nextPacket(req2, []byte("b"))
|
||||
resp2 := responsePacket(req3, []byte("HTTP/1.1 200 OK\r\n\r\n"))
|
||||
|
||||
return []*TCPPacket{req1, req2, req3}, []*TCPPacket{resp1, resp2}
|
||||
}
|
||||
|
||||
func TestShort100Continue(t *testing.T) {
|
||||
listener := NewListener("", "0", EnginePcap, true, 10*time.Millisecond, ProtocolHTTP, "", "", 0, false, false)
|
||||
listener := NewListener("", "0", EnginePcap, true, 10*time.Millisecond)
|
||||
defer listener.Close()
|
||||
|
||||
req, resp := get100ContinuePackets()
|
||||
reqPacket1 := buildPacket(true, 1, 1, []byte("POST / HTTP/1.1\r\nContent-Length: 2\r\nExpect: 100-continue\r\n\r\n"), time.Now())
|
||||
// Packet with data have different Seq
|
||||
reqPacket2 := buildPacket(true, 2, reqPacket1.Seq+uint32(len(reqPacket1.Data)), []byte("a"), time.Now())
|
||||
reqPacket3 := buildPacket(true, 2, reqPacket2.Seq+1, []byte("b"), time.Now())
|
||||
|
||||
respPacket1 := buildPacket(false, 10, 3, []byte("HTTP/1.1 100 Continue\r\n\r\n"), time.Now())
|
||||
|
||||
// panic(int(uint32(len(reqPacket1.Data)) + uint32(len(reqPacket2.Data)) + uint32(len(reqPacket3.Data))))
|
||||
respPacket2 := buildPacket(false, reqPacket3.Seq+1 /* len of data */, 2, []byte("HTTP/1.1 200 OK\r\n\r\n"), time.Now())
|
||||
|
||||
result := []byte("POST / HTTP/1.1\r\nContent-Length: 2\r\n\r\nab")
|
||||
|
||||
testRawListener100Continue(t, listener, result, req[0], req[1], req[2], resp[0], resp[1])
|
||||
testRawListener100Continue(t, listener, result, reqPacket1, reqPacket2, reqPacket3, respPacket1, respPacket2)
|
||||
}
|
||||
|
||||
// Response comes before Request
|
||||
func Test100ContinueWrongOrder(t *testing.T) {
|
||||
listener := NewListener("", "0", EnginePcap, true, 10*time.Millisecond, ProtocolHTTP, "", "", 0, false, false)
|
||||
listener := NewListener("", "0", EnginePcap, true, 10*time.Millisecond)
|
||||
defer listener.Close()
|
||||
|
||||
req, resp := get100ContinuePackets()
|
||||
reqPacket1 := buildPacket(true, 1, 1, []byte("POST / HTTP/1.1\r\nContent-Length: 2\r\nExpect: 100-continue\r\n\r\n"), time.Now())
|
||||
// Packet with data have different Seq
|
||||
reqPacket2 := buildPacket(true, 2, reqPacket1.Seq+uint32(len(reqPacket1.Data)), []byte("a"), time.Now())
|
||||
reqPacket3 := buildPacket(true, 2, reqPacket2.Seq+1, []byte("b"), time.Now())
|
||||
|
||||
respPacket1 := buildPacket(false, 10, 3, []byte("HTTP/1.1 100 Continue\r\n"), time.Now())
|
||||
|
||||
// panic(int(uint32(len(reqPacket1.Data)) + uint32(len(reqPacket2.Data)) + uint32(len(reqPacket3.Data))))
|
||||
respPacket2 := buildPacket(false, reqPacket3.Seq+1 /* len of data */, 2, []byte("HTTP/1.1 200 OK\r\n\r\n"), time.Now())
|
||||
|
||||
result := []byte("POST / HTTP/1.1\r\nContent-Length: 2\r\n\r\nab")
|
||||
|
||||
testRawListener100Continue(t, listener, result, resp[0], resp[1], req[0], req[1], req[2])
|
||||
testRawListener100Continue(t, listener, result, respPacket1, respPacket2, reqPacket1, reqPacket2, reqPacket3)
|
||||
}
|
||||
|
||||
func TestAlt100ContinueHeaderOrder(t *testing.T) {
|
||||
listener := NewListener("", "0", EnginePcap, true, 10*time.Millisecond)
|
||||
defer listener.Close()
|
||||
|
||||
reqPacket1 := buildPacket(true, 1, 1, []byte("POST / HTTP/1.1\r\nExpect: 100-continue\r\nContent-Length: 2\r\n\r\n"), time.Now())
|
||||
// Packet with data have different Seq
|
||||
reqPacket2 := buildPacket(true, 2, reqPacket1.Seq+uint32(len(reqPacket1.Data)), []byte("a"), time.Now())
|
||||
reqPacket3 := buildPacket(true, 2, reqPacket2.Seq+1, []byte("b"), time.Now())
|
||||
|
||||
respPacket1 := buildPacket(false, 10, 3, []byte("HTTP/1.1 100 Continue\r\n"), time.Now())
|
||||
|
||||
// panic(int(uint32(len(reqPacket1.Data)) + uint32(len(reqPacket2.Data)) + uint32(len(reqPacket3.Data))))
|
||||
respPacket2 := buildPacket(false, reqPacket3.Seq+1 /* len of data */, 2, []byte("HTTP/1.1 200 OK\r\n\r\n"), time.Now())
|
||||
|
||||
result := []byte("POST / HTTP/1.1\r\nContent-Length: 2\r\n\r\nab")
|
||||
|
||||
testRawListener100Continue(t, listener, result, reqPacket1, reqPacket2, reqPacket3, respPacket1, respPacket2)
|
||||
}
|
||||
|
||||
func testRawListener100Continue(t *testing.T, listener *Listener, result []byte, packets ...*TCPPacket) {
|
||||
@@ -419,6 +307,14 @@ func testChunkedSequence(t *testing.T, listener *Listener, packets ...*TCPPacket
|
||||
t.Fatal("packetsChan non empty:", listener.packetsChan)
|
||||
}
|
||||
|
||||
if len(listener.messagesChan) != 0 {
|
||||
t.Fatal("messagesChan non empty:", <-listener.messagesChan)
|
||||
}
|
||||
|
||||
if len(listener.messages) != 0 {
|
||||
t.Fatal("Messages non empty:", listener.messages)
|
||||
}
|
||||
|
||||
if len(listener.ackAliases) != 0 {
|
||||
t.Fatal("ackAliases non empty:", listener.ackAliases)
|
||||
}
|
||||
@@ -430,50 +326,55 @@ func testChunkedSequence(t *testing.T, listener *Listener, packets ...*TCPPacket
|
||||
if len(listener.respAliases) != 0 {
|
||||
t.Fatal("respAliases non empty:", listener.respAliases)
|
||||
}
|
||||
|
||||
if len(listener.respWithoutReq) != 0 {
|
||||
t.Fatal("respWithoutReq non empty:", listener.respWithoutReq)
|
||||
}
|
||||
}
|
||||
|
||||
// permutation using heap algorithm https://en.wikipedia.org/wiki/Heap%27s_algorithm
|
||||
func permutation(a []*TCPPacket, f func([]*TCPPacket)) {
|
||||
n := len(a)
|
||||
c := make([]int, n)
|
||||
f(a)
|
||||
i := 0
|
||||
for i < n {
|
||||
if c[i] < i {
|
||||
if i&1 != 1 {
|
||||
a[0], a[i] = a[i], a[0]
|
||||
} else {
|
||||
a[c[i]], a[i] = a[i], a[c[i]]
|
||||
}
|
||||
f(a)
|
||||
c[i]++
|
||||
i = 0
|
||||
} else {
|
||||
c[i] = 0
|
||||
i++
|
||||
}
|
||||
func permutation(n int, list []*TCPPacket) []*TCPPacket {
|
||||
if len(list) == 1 {
|
||||
return list
|
||||
}
|
||||
|
||||
k := n % len(list)
|
||||
|
||||
first := []*TCPPacket{list[k]}
|
||||
next := make([]*TCPPacket, len(list)-1)
|
||||
|
||||
copy(next, append(list[:k], list[k+1:]...))
|
||||
|
||||
return append(first, permutation(n/len(list), next)...)
|
||||
}
|
||||
|
||||
// Response comes before Request
|
||||
func TestRawListenerChunkedWrongOrder(t *testing.T) {
|
||||
listener := NewListener("", "0", EnginePcap, true, 10*time.Millisecond, ProtocolHTTP, "", "", 0, false, false)
|
||||
listener := NewListener("", "0", EnginePcap, true, 10*time.Millisecond)
|
||||
defer listener.Close()
|
||||
|
||||
reqPacket1 := firstPacket([]byte("POST / HTTP/1.1\r\nTransfer-Encoding: chunked\r\nExpect: 100-continue\r\n\r\n"))
|
||||
reqPacket1 := buildPacket(true, 1, 1, []byte("POST / HTTP/1.1\r\nTransfer-Encoding: chunked\r\nExpect: 100-continue\r\n\r\n"), time.Now())
|
||||
// Packet with data have different Seq
|
||||
reqPacket2 := buildPacket(true, 2, reqPacket1.Seq+uint32(len(reqPacket1.Data)), []byte("1\r\na\r\n"), time.Now())
|
||||
reqPacket3 := buildPacket(true, 2, reqPacket2.Seq+uint32(len(reqPacket2.Data)), []byte("1\r\nb\r\n"), time.Now())
|
||||
reqPacket4 := buildPacket(true, 2, reqPacket3.Seq+uint32(len(reqPacket3.Data)), []byte("0\r\n\r\n"), time.Now())
|
||||
|
||||
respPacket1 := responsePacket(reqPacket1, []byte("HTTP/1.1 100 Continue\r\n"))
|
||||
reqPacket2 := responsePacket(respPacket1, []byte("1\r\na\r\n"))
|
||||
reqPacket3 := nextPacket(reqPacket2, []byte("1\r\nb\r\n"))
|
||||
reqPacket4 := nextPacket(reqPacket3, []byte("0\r\n\r\n"))
|
||||
respPacket1 := buildPacket(false, 10, 3, []byte("HTTP/1.1 100 Continue\r\n\r\n"), time.Now())
|
||||
|
||||
respPacket2 := responsePacket(reqPacket4, []byte("HTTP/1.1 200 OK\r\n\r\n"))
|
||||
// panic(int(uint32(len(reqPacket1.Data)) + uint32(len(reqPacket2.Data)) + uint32(len(reqPacket3.Data))))
|
||||
respPacket2 := buildPacket(false, reqPacket4.Seq+5 /* len of data */, 2, []byte("HTTP/1.1 200 OK\r\n\r\n"), time.Now())
|
||||
|
||||
f := func(p []*TCPPacket) {
|
||||
testChunkedSequence(t, listener, p...)
|
||||
}
|
||||
// Should re-construct message from all possible combinations
|
||||
permutation([]*TCPPacket{reqPacket1, reqPacket2, reqPacket3, reqPacket4, respPacket1, respPacket2}, f)
|
||||
for i := 0; i < 6*5*4*3*2*1; i++ {
|
||||
|
||||
// if i < 54 || i > 57 {
|
||||
// continue
|
||||
// }
|
||||
|
||||
packets := permutation(i, []*TCPPacket{reqPacket1, reqPacket2, reqPacket3, reqPacket4, respPacket1, respPacket2})
|
||||
|
||||
t.Log("permutation:", i)
|
||||
testChunkedSequence(t, listener, packets...)
|
||||
}
|
||||
}
|
||||
|
||||
func chunkedPostMessage() []*TCPPacket {
|
||||
@@ -526,7 +427,7 @@ func getMessage() []*TCPPacket {
|
||||
|
||||
// Response comes before Request
|
||||
func TestRawListenerBench(t *testing.T) {
|
||||
l := NewListener("", "0", EnginePcap, true, 200*time.Millisecond, ProtocolHTTP, "", "", 0, false, false)
|
||||
l := NewListener("", "0", EnginePcap, true, 200*time.Millisecond)
|
||||
defer l.Close()
|
||||
|
||||
// Should re-construct message from all possible combinations
|
||||
@@ -574,44 +475,3 @@ func TestRawListenerBench(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestResponseZeroContentLength(t *testing.T) {
|
||||
var req, resp *TCPMessage
|
||||
listener := NewListener("", "0", EnginePcap, true, 10*time.Millisecond, ProtocolHTTP, "", "", 0, false, false)
|
||||
defer listener.Close()
|
||||
|
||||
reqPacket := firstPacket([]byte("POST /api/setup/install HTTP/1.1\r\nHost: localhost:22936\r\nUser-Agent: curl/7.57.0\r\nAccept: */*\r\nContent-Length: 0\r\nContent-Type: application/x-www-form-urlencoded\r\n\r\n"))
|
||||
respPacket := responsePacket(reqPacket, []byte("HTTP/1.1 200\r\nDate: Fri, 11 May 2018 15:09:10 GMT\r\nServer: Kestrel\r\nCache-Control: no-cache\r\nTransfer-Encoding: chunked\r\n\r\n"))
|
||||
respPacket2 := nextPacket(respPacket, []byte("0\r\n\r\n"))
|
||||
|
||||
// If response packet comes before request
|
||||
listener.packetsChan <- reqPacket.dump()
|
||||
listener.packetsChan <- respPacket.dump()
|
||||
listener.packetsChan <- respPacket2.dump()
|
||||
|
||||
select {
|
||||
case req = <-listener.messagesChan:
|
||||
case <-time.After(time.Millisecond):
|
||||
t.Error("Should return request immediately")
|
||||
return
|
||||
}
|
||||
|
||||
if !req.IsIncoming {
|
||||
t.Error("Should be request")
|
||||
}
|
||||
|
||||
select {
|
||||
case resp = <-listener.messagesChan:
|
||||
case <-time.After(time.Millisecond):
|
||||
t.Error("Should return response immediately")
|
||||
return
|
||||
}
|
||||
|
||||
if resp.IsIncoming {
|
||||
t.Error("Should be response")
|
||||
}
|
||||
|
||||
if !bytes.Equal(resp.UUID(), req.UUID()) {
|
||||
t.Error("Resp and Req UUID should be equal")
|
||||
}
|
||||
}
|
||||
@@ -1,27 +1,18 @@
|
||||
package capture
|
||||
package rawSocket
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/sha1"
|
||||
"encoding/binary"
|
||||
"encoding/hex"
|
||||
"github.com/buger/gor/proto"
|
||||
"log"
|
||||
"net"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/buger/goreplay/proto"
|
||||
)
|
||||
|
||||
// TCPProtocol is a number to indicate type of protocol
|
||||
type TCPProtocol uint8
|
||||
|
||||
const (
|
||||
// ProtocolHTTP ...
|
||||
ProtocolHTTP TCPProtocol = 0
|
||||
// ProtocolBinary ...
|
||||
ProtocolBinary TCPProtocol = 1
|
||||
)
|
||||
var _ = log.Println
|
||||
|
||||
// TCPMessage ensure that all TCP packets for given request is received, and processed in right sequence
|
||||
// Its needed because all TCP message can be fragmented or re-transmitted
|
||||
@@ -46,8 +37,6 @@ type TCPMessage struct {
|
||||
|
||||
delChan chan *TCPMessage
|
||||
|
||||
protocol TCPProtocol
|
||||
|
||||
/* HTTP specific variables */
|
||||
methodType httpMethodType
|
||||
bodyType httpBodyType
|
||||
@@ -59,9 +48,8 @@ type TCPMessage struct {
|
||||
}
|
||||
|
||||
// NewTCPMessage pointer created from a Acknowledgment number and a channel of messages readuy to be deleted
|
||||
func NewTCPMessage(Seq, Ack uint32, IsIncoming bool, protocol TCPProtocol, timestamp time.Time) (msg *TCPMessage) {
|
||||
msg = &TCPMessage{Seq: Seq, Ack: Ack, IsIncoming: IsIncoming, protocol: protocol, Start: timestamp}
|
||||
msg.Start = time.Now()
|
||||
func NewTCPMessage(Seq, Ack uint32, IsIncoming bool, timestamp time.Time) (msg *TCPMessage) {
|
||||
msg = &TCPMessage{Seq: Seq, Ack: Ack, IsIncoming: IsIncoming, Start: timestamp}
|
||||
|
||||
return
|
||||
}
|
||||
@@ -84,7 +72,7 @@ func (t *TCPMessage) Bytes() (output []byte) {
|
||||
return output
|
||||
}
|
||||
|
||||
// BodySize returns total body size
|
||||
// Size returns total body size
|
||||
func (t *TCPMessage) BodySize() (size int) {
|
||||
if len(t.packets) == 0 || t.headerPacket == -1 {
|
||||
return 0
|
||||
@@ -115,48 +103,52 @@ func (t *TCPMessage) Size() (size int) {
|
||||
// AddPacket to the message and ensure packet uniqueness
|
||||
// TCP allows that packet can be re-send multiple times
|
||||
func (t *TCPMessage) AddPacket(packet *TCPPacket) {
|
||||
packetFound := false
|
||||
|
||||
for _, pkt := range t.packets {
|
||||
if packet.Seq == pkt.Seq {
|
||||
return
|
||||
packetFound = true
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
// Packets not always captured in same Seq order, and sometimes we need to prepend
|
||||
if len(t.packets) == 0 || packet.Seq > t.packets[len(t.packets)-1].Seq {
|
||||
t.packets = append(t.packets, packet)
|
||||
} else if packet.Seq < t.packets[0].Seq {
|
||||
t.packets = append([]*TCPPacket{packet}, t.packets...)
|
||||
t.Seq = packet.Seq // Message Seq should indicated starting seq
|
||||
} else { // insert somewhere in the middle...
|
||||
for i, p := range t.packets {
|
||||
if packet.Seq < p.Seq {
|
||||
t.packets = append(t.packets[:i], append([]*TCPPacket{packet}, t.packets[i:]...)...)
|
||||
break
|
||||
if !packetFound {
|
||||
// Packets not always captured in same Seq order, and sometimes we need to prepend
|
||||
if len(t.packets) == 0 || packet.Seq > t.packets[len(t.packets)-1].Seq {
|
||||
t.packets = append(t.packets, packet)
|
||||
} else if packet.Seq < t.packets[0].Seq {
|
||||
t.packets = append([]*TCPPacket{packet}, t.packets...)
|
||||
t.Seq = packet.Seq // Message Seq should indicated starting seq
|
||||
} else { // insert somewhere in the middle...
|
||||
for i, p := range t.packets {
|
||||
if packet.Seq < p.Seq {
|
||||
t.packets = append(t.packets[:i], append([]*TCPPacket{packet}, t.packets[i:]...)...)
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if t.IsIncoming {
|
||||
t.End = time.Now()
|
||||
} else {
|
||||
t.End = time.Now().Add(time.Millisecond)
|
||||
}
|
||||
|
||||
if packet.OrigAck != 0 {
|
||||
t.DataAck = packet.OrigAck
|
||||
}
|
||||
|
||||
if packet.timestamp.Before(t.Start) || t.Start.IsZero() {
|
||||
if packet.timestamp.Before(t.Start) {
|
||||
t.Start = packet.timestamp
|
||||
}
|
||||
|
||||
if packet.timestamp.After(t.End) || t.End.IsZero() {
|
||||
t.End = packet.timestamp
|
||||
}
|
||||
}
|
||||
|
||||
t.checkSeqIntegrity()
|
||||
|
||||
if t.protocol == ProtocolHTTP {
|
||||
t.updateHeadersPacket()
|
||||
t.updateMethodType()
|
||||
t.updateBodyType()
|
||||
t.check100Continue()
|
||||
t.checkIfComplete()
|
||||
}
|
||||
t.updateHeadersPacket()
|
||||
t.updateMethodType()
|
||||
t.updateBodyType()
|
||||
t.checkIfComplete()
|
||||
t.check100Continue()
|
||||
}
|
||||
|
||||
// Check if there is missing packet
|
||||
@@ -190,7 +182,7 @@ func (t *TCPMessage) checkSeqIntegrity() {
|
||||
nextSeq := p.Seq + uint32(len(p.Data))
|
||||
|
||||
if np.Seq != nextSeq {
|
||||
if t.protocol == ProtocolHTTP && t.expectType == httpExpect100Continue {
|
||||
if t.expectType == httpExpect100Continue {
|
||||
if np.Seq != nextSeq+22 {
|
||||
t.seqMissing = true
|
||||
return
|
||||
@@ -206,9 +198,6 @@ func (t *TCPMessage) checkSeqIntegrity() {
|
||||
}
|
||||
|
||||
var bEmptyLine = []byte("\r\n\r\n")
|
||||
var bBR = []byte("\r\n")
|
||||
|
||||
// last-chunk always is 0\r\n\r\n\. More info https://tools.ietf.org/html/rfc2616#section-3.6.1
|
||||
var bChunkEnd = []byte("0\r\n\r\n")
|
||||
|
||||
func (t *TCPMessage) updateHeadersPacket() {
|
||||
@@ -225,63 +214,55 @@ func (t *TCPMessage) updateHeadersPacket() {
|
||||
}
|
||||
|
||||
for i, p := range t.packets {
|
||||
if len(p.Data) >= len(bEmptyLine) {
|
||||
if bytes.LastIndex(p.Data, bEmptyLine) != -1 {
|
||||
t.headerPacket = i
|
||||
return
|
||||
}
|
||||
} else if i > 0 && bytes.Equal(p.Data, bBR) {
|
||||
idx := bytes.LastIndex(t.packets[i-1].Data, bBR)
|
||||
if idx != -1 && idx == len(t.packets[i-1].Data)-len(bBR) {
|
||||
t.headerPacket = i
|
||||
return
|
||||
}
|
||||
if bytes.LastIndex(p.Data, bEmptyLine) != -1 {
|
||||
t.headerPacket = i
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
// checkIfComplete returns true if all of the packets that compse the message arrived.
|
||||
// isMultipart returns true if message contains from multiple tcp packets
|
||||
func (t *TCPMessage) checkIfComplete() {
|
||||
if t.seqMissing || t.headerPacket == -1 {
|
||||
// log.Println("Seq missing", t.seqMissing, t.packets)
|
||||
return
|
||||
}
|
||||
|
||||
if t.methodType == httpMethodNotFound {
|
||||
// log.Println("Method missing", t.methodType, t.packets)
|
||||
return
|
||||
}
|
||||
|
||||
// Responses can be emitted only if we found request
|
||||
if !t.IsIncoming && t.AssocMessage == nil {
|
||||
// log.Println("Assoc not found", t)
|
||||
return
|
||||
}
|
||||
|
||||
// log.Println("Found?", t)
|
||||
|
||||
switch t.bodyType {
|
||||
case httpBodyEmpty:
|
||||
// If one GET, OPTIONS, or HEAD request
|
||||
if t.methodType == httpMethodWithoutBody {
|
||||
t.complete = true
|
||||
case httpBodyContentLength:
|
||||
if t.contentLength == 0 || t.contentLength == t.BodySize() {
|
||||
} else {
|
||||
switch t.bodyType {
|
||||
case httpBodyEmpty:
|
||||
t.complete = true
|
||||
}
|
||||
case httpBodyChunked:
|
||||
lastPacket := t.packets[len(t.packets)-1]
|
||||
if bytes.LastIndex(lastPacket.Data, bChunkEnd) != -1 {
|
||||
t.complete = true
|
||||
}
|
||||
default:
|
||||
if len(t.packets) == 0 {
|
||||
return
|
||||
}
|
||||
case httpBodyContentLength:
|
||||
if t.contentLength == 0 || t.contentLength == t.BodySize() {
|
||||
t.complete = true
|
||||
}
|
||||
case httpBodyChunked:
|
||||
lastPacket := t.packets[len(t.packets)-1]
|
||||
if bytes.LastIndex(lastPacket.Data, bChunkEnd) != -1 {
|
||||
t.complete = true
|
||||
}
|
||||
default:
|
||||
if len(t.packets) == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
last := t.packets[len(t.packets)-1]
|
||||
if last.IsFIN {
|
||||
t.complete = true
|
||||
last := t.packets[len(t.packets)-1]
|
||||
if last.IsFIN {
|
||||
t.complete = true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -289,11 +270,19 @@ func (t *TCPMessage) checkIfComplete() {
|
||||
type httpMethodType uint8
|
||||
|
||||
const (
|
||||
httpMethodNotSet httpMethodType = 0
|
||||
httpMethodKnown httpMethodType = 1
|
||||
httpMethodNotFound httpMethodType = 2
|
||||
httpMethodNotSet httpMethodType = 0
|
||||
httpMethodWithBody httpMethodType = 1
|
||||
httpMethodWithoutBody httpMethodType = 2
|
||||
httpMethodNotFound httpMethodType = 3
|
||||
)
|
||||
|
||||
var methodsWithBody = [][]byte{
|
||||
[]byte("POST"),
|
||||
[]byte("PUT"),
|
||||
[]byte("PATCH"),
|
||||
[]byte("CONNECT"),
|
||||
}
|
||||
|
||||
func (t *TCPMessage) updateMethodType() {
|
||||
// if there is cache
|
||||
if t.methodType != httpMethodNotSet && t.methodType != httpMethodNotFound {
|
||||
@@ -310,7 +299,11 @@ func (t *TCPMessage) updateMethodType() {
|
||||
}
|
||||
|
||||
if t.IsIncoming {
|
||||
var method []byte
|
||||
|
||||
if mIdx := bytes.IndexByte(d[:8], ' '); mIdx != -1 {
|
||||
method = d[:mIdx]
|
||||
|
||||
// Check that after method we have absolute or relative path
|
||||
switch d[mIdx+1] {
|
||||
case '/', 'h', '*':
|
||||
@@ -323,14 +316,21 @@ func (t *TCPMessage) updateMethodType() {
|
||||
return
|
||||
}
|
||||
|
||||
t.methodType = httpMethodKnown
|
||||
for _, m := range methodsWithBody {
|
||||
if len(m) == len(method) && bytes.Equal(m, method) {
|
||||
t.methodType = httpMethodWithBody
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
t.methodType = httpMethodWithoutBody
|
||||
} else {
|
||||
if !bytes.Equal(d[:6], []byte("HTTP/1")) {
|
||||
t.methodType = httpMethodNotFound
|
||||
return
|
||||
}
|
||||
|
||||
t.methodType = httpMethodKnown
|
||||
t.methodType = httpMethodWithBody
|
||||
}
|
||||
}
|
||||
|
||||
@@ -378,20 +378,13 @@ func (t *TCPMessage) updateBodyType() {
|
||||
switch t.methodType {
|
||||
case httpMethodNotFound:
|
||||
return
|
||||
case httpMethodKnown:
|
||||
|
||||
if !t.IsIncoming &&
|
||||
t.AssocMessage != nil &&
|
||||
bytes.IndexByte(t.AssocMessage.Bytes(), ' ') > -1 &&
|
||||
bytes.Equal([]byte("HEAD"), proto.Method(t.AssocMessage.Bytes())) {
|
||||
// Need to check if this is a response to a head request,
|
||||
// in which case the body has to be empty regardless.
|
||||
t.bodyType = httpBodyEmpty
|
||||
return
|
||||
}
|
||||
|
||||
case httpMethodWithoutBody:
|
||||
t.bodyType = httpBodyEmpty
|
||||
return
|
||||
case httpMethodWithBody:
|
||||
if len(lengthB) > 0 {
|
||||
t.contentLength, _ = strconv.Atoi(string(lengthB))
|
||||
|
||||
if t.contentLength == 0 {
|
||||
t.bodyType = httpBodyEmpty
|
||||
} else {
|
||||
@@ -430,6 +423,10 @@ func (t *TCPMessage) check100Continue() {
|
||||
return
|
||||
}
|
||||
|
||||
if t.methodType != httpMethodWithBody {
|
||||
return
|
||||
}
|
||||
|
||||
if t.seqMissing || t.headerPacket == -1 {
|
||||
return
|
||||
}
|
||||
@@ -512,11 +509,3 @@ func (t *TCPMessage) ID() tcpID {
|
||||
func (t *TCPMessage) IP() net.IP {
|
||||
return net.IP(t.packets[0].Addr)
|
||||
}
|
||||
|
||||
func (t *TCPMessage) String() string {
|
||||
return strings.Join([]string{
|
||||
"Len packets: " + strconv.Itoa(len(t.packets)),
|
||||
"Data size:" + strconv.Itoa(len(t.Bytes())),
|
||||
"Data:" + string(t.Bytes()),
|
||||
}, "\n")
|
||||
}
|
||||
@@ -1,4 +1,4 @@
|
||||
package capture
|
||||
package rawSocket
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
@@ -36,7 +36,8 @@ func buildMessage(p *TCPPacket) *TCPMessage {
|
||||
if p.SrcPort == 1 {
|
||||
isIncoming = true
|
||||
}
|
||||
m := NewTCPMessage(p.Seq, p.Ack, isIncoming, ProtocolHTTP, p.timestamp)
|
||||
|
||||
m := NewTCPMessage(p.Seq, p.Ack, isIncoming, p.timestamp)
|
||||
m.AddPacket(p)
|
||||
|
||||
return m
|
||||
@@ -169,16 +170,16 @@ func TestTCPMessageMethodType(t *testing.T) {
|
||||
payload string
|
||||
expectedMethodType httpMethodType
|
||||
}{
|
||||
{true, "GET / HTTP/1.1\r\n\r\n", httpMethodKnown},
|
||||
{true, "GET * HTTP/1.1\r\n\r\n", httpMethodKnown},
|
||||
{true, "UNKNOWN / HTTP/1.1\r\n\r\n", httpMethodKnown},
|
||||
{true, "GET http://example.com HTTP/1.1\r\n\r\n", httpMethodKnown},
|
||||
{true, "POST / HTTP/1.1\r\n\r\n", httpMethodKnown},
|
||||
{true, "PUT / HTTP/1.1\r\n\r\n", httpMethodKnown},
|
||||
{true, "GET / HTTP/1.1\r\n\r\n", httpMethodWithoutBody},
|
||||
{true, "GET * HTTP/1.1\r\n\r\n", httpMethodWithoutBody},
|
||||
{true, "UNKNOWN / HTTP/1.1\r\n\r\n", httpMethodWithoutBody},
|
||||
{true, "GET http://example.com HTTP/1.1\r\n\r\n", httpMethodWithoutBody},
|
||||
{true, "POST / HTTP/1.1\r\n\r\n", httpMethodWithBody},
|
||||
{true, "PUT / HTTP/1.1\r\n\r\n", httpMethodWithBody},
|
||||
{true, "GET zxc HTTP/1.1\r\n\r\n", httpMethodNotFound},
|
||||
{true, "GET / HTTP\r\n\r\n", httpMethodNotFound},
|
||||
{true, "VERYLONGMETHOD / HTTP/1.1\r\n\r\n", httpMethodNotFound},
|
||||
{false, "HTTP/1.1 200 OK\r\n\r\n", httpMethodKnown},
|
||||
{false, "HTTP/1.1 200 OK\r\n\r\n", httpMethodWithBody},
|
||||
{false, "HTTP /1.1 200 OK\r\n\r\n", httpMethodNotFound},
|
||||
}
|
||||
|
||||
@@ -198,7 +199,6 @@ func TestTCPMessageBodyType(t *testing.T) {
|
||||
expectedBodyType httpBodyType
|
||||
}{
|
||||
{true, "GET / HTTP/1.1\r\n\r\n", httpBodyEmpty},
|
||||
{true, "GET / HTTP/1.1\r\nContent-Length: 2\r\n\r\nab", httpBodyContentLength},
|
||||
{true, "POST / HTTP/1.1\r\n\r\n", httpBodyEmpty},
|
||||
{true, "POST / HTTP/1.1\r\nUser-Agent: zxc\r\n\r\n", httpBodyEmpty},
|
||||
{false, "HTTP/1.1 200 OK\r\n\r\n", httpBodyEmpty},
|
||||
@@ -1,12 +1,15 @@
|
||||
package capture
|
||||
package rawSocket
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
"log"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
var _ = log.Println
|
||||
|
||||
// TCP Flags
|
||||
const (
|
||||
fFIN = 1 << iota
|
||||
@@ -74,9 +77,9 @@ func (t *TCPPacket) ParseBasic() {
|
||||
t.DataOffset = (t.Raw[12] & 0xF0) >> 4
|
||||
t.IsFIN = t.Raw[13]&0x01 != 0
|
||||
|
||||
if len(t.Raw) >= int(t.DataOffset*4) {
|
||||
t.Data = t.Raw[t.DataOffset*4:]
|
||||
}
|
||||
// log.Println("DataOffset:", t.DataOffset, t.DestPort, t.SrcPort, t.Seq, t.Ack)
|
||||
|
||||
t.Data = t.Raw[t.DataOffset*4:]
|
||||
}
|
||||
|
||||
func (t *TCPPacket) dump() *packet {
|
||||
@@ -99,6 +102,7 @@ func (t *TCPPacket) dump() *packet {
|
||||
}
|
||||
|
||||
copy(packetData[16:], t.Data)
|
||||
|
||||
return &packet{
|
||||
srcIP: packetSrcIP,
|
||||
data: packetData,
|
||||
File diff suppressed because one or more lines are too long
-105
@@ -1,105 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"log"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/aws/session"
|
||||
"github.com/aws/aws-sdk-go/service/s3"
|
||||
)
|
||||
|
||||
// S3ReadCloser ...
|
||||
type S3ReadCloser struct {
|
||||
bucket string
|
||||
key string
|
||||
offset int
|
||||
totalSize int
|
||||
readBytes int
|
||||
sess *session.Session
|
||||
buf *bytes.Buffer
|
||||
}
|
||||
|
||||
func awsConfig() *aws.Config {
|
||||
region := os.Getenv("AWS_DEFAULT_REGION")
|
||||
if region == "" {
|
||||
region = os.Getenv("AWS_REGION")
|
||||
if region == "" {
|
||||
region = "us-east-1"
|
||||
}
|
||||
}
|
||||
|
||||
config := &aws.Config{Region: aws.String(region)}
|
||||
|
||||
if endpoint := os.Getenv("AWS_ENDPOINT_URL"); endpoint != "" {
|
||||
config.Endpoint = aws.String(endpoint)
|
||||
log.Println("Custom endpoint:", endpoint)
|
||||
}
|
||||
|
||||
log.Println("Connecting to S3. Region: " + region)
|
||||
|
||||
config.CredentialsChainVerboseErrors = aws.Bool(true)
|
||||
|
||||
if os.Getenv("AWS_DEBUG") != "" {
|
||||
config.LogLevel = aws.LogLevel(aws.LogDebugWithHTTPBody)
|
||||
}
|
||||
|
||||
return config
|
||||
}
|
||||
|
||||
// NewS3ReadCloser returns new instance of S3 read closer
|
||||
func NewS3ReadCloser(path string) *S3ReadCloser {
|
||||
if !PRO {
|
||||
log.Fatal("Using S3 input and output require PRO license")
|
||||
return nil
|
||||
}
|
||||
|
||||
bucket, key := parseS3Url(path)
|
||||
sess := session.Must(session.NewSession(awsConfig()))
|
||||
|
||||
log.Println("[S3 Input] S3 connection succesfully initialized", path)
|
||||
|
||||
return &S3ReadCloser{
|
||||
bucket: bucket,
|
||||
key: key,
|
||||
sess: sess,
|
||||
buf: &bytes.Buffer{},
|
||||
}
|
||||
}
|
||||
|
||||
// Read reads buffer from s3 session
|
||||
func (s *S3ReadCloser) Read(b []byte) (n int, e error) {
|
||||
if s.readBytes == 0 || s.readBytes+len(b) > s.offset {
|
||||
svc := s3.New(s.sess)
|
||||
|
||||
objectRange := "bytes=" + strconv.Itoa(s.offset)
|
||||
s.offset += 1000000 // Reading in chunks of 1 mb
|
||||
objectRange += "-" + strconv.Itoa(s.offset-1)
|
||||
|
||||
params := &s3.GetObjectInput{
|
||||
Bucket: aws.String(s.bucket),
|
||||
Key: aws.String(s.key),
|
||||
Range: aws.String(objectRange),
|
||||
}
|
||||
resp, err := svc.GetObject(params)
|
||||
|
||||
if err != nil {
|
||||
log.Println("[S3 Input] Error during getting file", s.bucket, s.key, err)
|
||||
} else {
|
||||
s.totalSize, _ = strconv.Atoi(strings.Split(*resp.ContentRange, "/")[1])
|
||||
s.buf.ReadFrom(resp.Body)
|
||||
}
|
||||
}
|
||||
|
||||
s.readBytes += len(b)
|
||||
|
||||
return s.buf.Read(b)
|
||||
}
|
||||
|
||||
// Close is here to make S3ReadCloser satisfy ReadCloser interface
|
||||
func (s *S3ReadCloser) Close() error {
|
||||
return nil
|
||||
}
|
||||
-150
@@ -1,150 +0,0 @@
|
||||
// +build pro
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"math/rand"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/service/s3"
|
||||
)
|
||||
|
||||
func TestS3Output(t *testing.T) {
|
||||
bucket := aws.String("test-gor")
|
||||
rnd := rand.Int63()
|
||||
path := fmt.Sprintf("s3://test-gor/%d/requests.gz", rnd)
|
||||
|
||||
output := NewS3Output(path, &FileOutputConfig{queueLimit: 2})
|
||||
|
||||
svc := s3.New(output.session)
|
||||
|
||||
output.Write([]byte("1 1 1\ntest"))
|
||||
output.Write([]byte("1 1 1\ntest"))
|
||||
output.buffer.updateName()
|
||||
output.Write([]byte("1 1 1\ntest"))
|
||||
output.Write([]byte("1 1 1\ntest"))
|
||||
output.buffer.updateName()
|
||||
output.Write([]byte("1 1 1\ntest"))
|
||||
|
||||
time.Sleep(time.Second)
|
||||
|
||||
params := &s3.ListObjectsInput{
|
||||
Bucket: bucket,
|
||||
Prefix: aws.String(fmt.Sprintf("%d", rnd)),
|
||||
}
|
||||
|
||||
resp, _ := svc.ListObjects(params)
|
||||
if len(resp.Contents) != 2 {
|
||||
t.Error("Should create 2 objects", len(resp.Contents))
|
||||
} else {
|
||||
if *resp.Contents[0].Key != fmt.Sprintf("%d/requests_0.gz", rnd) ||
|
||||
*resp.Contents[1].Key != fmt.Sprintf("%d/requests_1.gz", rnd) {
|
||||
t.Error("Should assign proper names", resp.Contents)
|
||||
}
|
||||
}
|
||||
|
||||
for _, c := range resp.Contents {
|
||||
svc.DeleteObject(&s3.DeleteObjectInput{Bucket: bucket, Key: c.Key})
|
||||
}
|
||||
|
||||
matches, _ := filepath.Glob(fmt.Sprintf("/tmp/gor_output_s3_*"))
|
||||
for _, m := range matches {
|
||||
os.Remove(m)
|
||||
}
|
||||
}
|
||||
|
||||
func TestS3OutputQueueLimit(t *testing.T) {
|
||||
bucket := aws.String("test-gor")
|
||||
rnd := rand.Int63()
|
||||
path := fmt.Sprintf("s3://test-gor/%d/requests.gz", rnd)
|
||||
|
||||
output := NewS3Output(path, &FileOutputConfig{queueLimit: 100})
|
||||
output.closeCh = make(chan struct{}, 3)
|
||||
|
||||
svc := s3.New(output.session)
|
||||
|
||||
for i := 0; i < 3; i++ {
|
||||
for i := 0; i < 100; i++ {
|
||||
output.Write([]byte("1 1 1\ntest"))
|
||||
}
|
||||
output.buffer.updateName()
|
||||
}
|
||||
output.buffer.updateName()
|
||||
output.Write([]byte("1 1 1\ntest"))
|
||||
|
||||
for i := 0; i < 3; i++ {
|
||||
<-output.closeCh
|
||||
}
|
||||
|
||||
params := &s3.ListObjectsInput{
|
||||
Bucket: bucket,
|
||||
Prefix: aws.String(fmt.Sprintf("%d", rnd)),
|
||||
}
|
||||
|
||||
resp, _ := svc.ListObjects(params)
|
||||
if len(resp.Contents) != 3 {
|
||||
t.Error("Should create 3 object", len(resp.Contents))
|
||||
} else {
|
||||
if *resp.Contents[0].Key != fmt.Sprintf("%d/requests_0.gz", rnd) ||
|
||||
*resp.Contents[1].Key != fmt.Sprintf("%d/requests_1.gz", rnd) {
|
||||
t.Error("Should assign proper names", resp.Contents)
|
||||
}
|
||||
}
|
||||
|
||||
for _, c := range resp.Contents {
|
||||
svc.DeleteObject(&s3.DeleteObjectInput{Bucket: bucket, Key: c.Key})
|
||||
}
|
||||
|
||||
matches, _ := filepath.Glob(fmt.Sprintf("/tmp/gor_output_s3_*"))
|
||||
for _, m := range matches {
|
||||
os.Remove(m)
|
||||
}
|
||||
}
|
||||
|
||||
func TestInputFileFromS3(t *testing.T) {
|
||||
rnd := rand.Int63()
|
||||
path := fmt.Sprintf("s3://test-gor-eu/%d/requests.gz", rnd)
|
||||
|
||||
output := NewS3Output(path, &FileOutputConfig{queueLimit: 5000})
|
||||
output.closeCh = make(chan struct{}, 10)
|
||||
|
||||
for i := 0; i <= 20000; i++ {
|
||||
output.Write([]byte("1 1 1\ntest"))
|
||||
|
||||
if i%5000 == 0 {
|
||||
output.buffer.updateName()
|
||||
}
|
||||
}
|
||||
|
||||
output.Write([]byte("1 1 1\ntest"))
|
||||
|
||||
for i := 0; i < 2; i++ {
|
||||
<-output.closeCh
|
||||
}
|
||||
|
||||
input := NewFileInput(fmt.Sprintf("s3://test-gor-eu/%d", rnd), false)
|
||||
|
||||
buf := make([]byte, 1000)
|
||||
for i := 0; i <= 19999; i++ {
|
||||
input.Read(buf)
|
||||
}
|
||||
|
||||
// Cleanup artifacts
|
||||
bucket := aws.String("test-gor")
|
||||
svc := s3.New(output.session)
|
||||
params := &s3.ListObjectsInput{
|
||||
Bucket: bucket,
|
||||
Prefix: aws.String(fmt.Sprintf("%d", rnd)),
|
||||
}
|
||||
|
||||
resp, _ := svc.ListObjects(params)
|
||||
|
||||
for _, c := range resp.Contents {
|
||||
svc.DeleteObject(&s3.DeleteObjectInput{Bucket: bucket, Key: c.Key})
|
||||
}
|
||||
}
|
||||
+66
-355
@@ -1,26 +1,14 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"flag"
|
||||
"fmt"
|
||||
"github.com/spf13/pflag"
|
||||
"github.com/spf13/viper"
|
||||
_ "github.com/spf13/viper/remote"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"log"
|
||||
"net/http"
|
||||
"os"
|
||||
"regexp"
|
||||
"runtime"
|
||||
"strconv"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
var DEMO string
|
||||
var VERSION string
|
||||
|
||||
// MultiOption allows to specify multiple flags with same name and collects all values into array
|
||||
type MultiOption []string
|
||||
@@ -37,244 +25,126 @@ func (h *MultiOption) Set(value string) error {
|
||||
|
||||
// AppSettings is the struct of main configuration
|
||||
type AppSettings struct {
|
||||
Verbose bool
|
||||
Debug bool
|
||||
Stats bool
|
||||
ExitAfter time.Duration
|
||||
verbose bool
|
||||
debug bool
|
||||
stats bool
|
||||
exitAfter time.Duration
|
||||
|
||||
SplitOutput bool
|
||||
RecognizeTCPSessions bool
|
||||
Pprof string
|
||||
splitOutput bool
|
||||
|
||||
inputDummy MultiOption
|
||||
outputDummy MultiOption
|
||||
OutputStdout bool
|
||||
OutputNull bool
|
||||
outputStdout bool
|
||||
outputNull bool
|
||||
|
||||
inputTCP MultiOption
|
||||
InputTCPConfig TCPInputConfig
|
||||
outputTCP MultiOption
|
||||
OutputTCPConfig TCPOutputConfig
|
||||
OutputTCPStats bool
|
||||
inputTCP MultiOption
|
||||
outputTCP MultiOption
|
||||
outputTCPStats bool
|
||||
|
||||
inputFile MultiOption
|
||||
InputFileLoop bool
|
||||
inputFileLoop bool
|
||||
outputFile MultiOption
|
||||
OutputFileConfig FileOutputConfig
|
||||
outputFileConfig FileOutputConfig
|
||||
|
||||
inputRAW MultiOption
|
||||
InputRAWEngine string
|
||||
InputRAWTrackResponse bool
|
||||
InputRAWRealIPHeader string
|
||||
InputRAWExpire time.Duration
|
||||
InputRAWProtocol string
|
||||
InputRAWBpfFilter string
|
||||
InputRAWTimestampType string
|
||||
CopyBufferSize int64
|
||||
InputRAWImmediateMode bool
|
||||
InputRAWBufferSize int64
|
||||
InputRAWOverrideSnapLen bool
|
||||
inputRAW MultiOption
|
||||
inputRAWEngine string
|
||||
inputRAWTrackResponse bool
|
||||
inputRAWRealIPHeader string
|
||||
|
||||
InputRAWBufferSizeFlag string
|
||||
OutputFileSizeFlag string
|
||||
OutputFileMaxSizeFlag string
|
||||
CopyBufferSizeFlag string
|
||||
middleware string
|
||||
|
||||
Middleware string
|
||||
inputHTTP MultiOption
|
||||
outputHTTP MultiOption
|
||||
|
||||
inputHTTP MultiOption
|
||||
outputHTTP MultiOption
|
||||
PrettifyHTTP bool
|
||||
prettifyHTTP bool
|
||||
|
||||
OutputHTTPConfig HTTPOutputConfig
|
||||
|
||||
outputBinary MultiOption
|
||||
OutputBinaryConfig BinaryOutputConfig
|
||||
|
||||
modifierConfig HTTPModifierConfig
|
||||
outputHTTPConfig HTTPOutputConfig
|
||||
modifierConfig HTTPModifierConfig
|
||||
|
||||
inputKafkaConfig KafkaConfig
|
||||
outputKafkaConfig KafkaConfig
|
||||
|
||||
ConfigFile string
|
||||
ConfigServerAddress string
|
||||
RemoteConfigHost string
|
||||
}
|
||||
|
||||
// Settings holds Gor configuration
|
||||
var Settings AppSettings
|
||||
|
||||
func usage() {
|
||||
fmt.Printf("Gor is a simple http traffic replication tool written in Go. Its main goal is to replay traffic from production servers to staging and dev environments.\nProject page: https://github.com/buger/gor\nAuthor: <Leonid Bugaev> leonsbox@gmail.com\nCurrent Version: v%s\n\n", VERSION)
|
||||
fmt.Printf("Gor is a simple http traffic replication tool written in Go. Its main goal is to replay traffic from production servers to staging and dev environments.\nProject page: https://github.com/buger/gor\nAuthor: <Leonid Bugaev> leonsbox@gmail.com\nCurrent Version: %s\n\n", VERSION)
|
||||
flag.PrintDefaults()
|
||||
os.Exit(2)
|
||||
}
|
||||
|
||||
func readAndUpdateConfig(body io.ReadCloser) []byte {
|
||||
decoder := json.NewDecoder(body)
|
||||
|
||||
var t AppSettings
|
||||
err := decoder.Decode(&t)
|
||||
if err != nil {
|
||||
return []byte("Error while updating flags via POST request.")
|
||||
}
|
||||
Settings = t
|
||||
newConfig, err := json.Marshal(Settings)
|
||||
if err != nil {
|
||||
return []byte("Error while updating flags via POST request.")
|
||||
}
|
||||
err = viper.ReadConfig(bytes.NewBuffer(newConfig))
|
||||
if err != nil {
|
||||
return []byte("Error while updating flags via POST request.")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func updateConfig(respBody []byte) {
|
||||
var t AppSettings
|
||||
if err := json.Unmarshal(respBody, t); err != nil {
|
||||
return
|
||||
}
|
||||
Settings = t
|
||||
}
|
||||
|
||||
func flagz(res http.ResponseWriter, req *http.Request) {
|
||||
for k, v := range req.URL.Query() {
|
||||
if len(v) == 1 {
|
||||
viper.Set(k, v[0])
|
||||
viper.Unmarshal(&Settings)
|
||||
}
|
||||
}
|
||||
|
||||
if req.Method == "POST" {
|
||||
res.Write(readAndUpdateConfig(req.Body))
|
||||
}
|
||||
data, _ := json.MarshalIndent(Settings, "", " ")
|
||||
res.Write(data)
|
||||
}
|
||||
|
||||
func initConfigServer() {
|
||||
http.HandleFunc("/flagz", flagz)
|
||||
http.ListenAndServe(Settings.ConfigServerAddress, nil)
|
||||
}
|
||||
|
||||
func init() {
|
||||
flag.Usage = usage
|
||||
|
||||
flag.StringVar(&Settings.Pprof, "http-Pprof", "", "Enable profiling. Starts http server on specified port, exposing special /Debug/Pprof endpoint. Example: `:8181`")
|
||||
flag.BoolVar(&Settings.Verbose, "Verbose", false, "Turn on more Verbose output")
|
||||
flag.BoolVar(&Settings.Debug, "Debug", false, "Turn on Debug output, shows all intercepted traffic. Works only when with `Verbose` flag")
|
||||
flag.BoolVar(&Settings.Stats, "Stats", false, "Turn on queue Stats output")
|
||||
flag.BoolVar(&Settings.verbose, "verbose", false, "Turn on more verbose output")
|
||||
flag.BoolVar(&Settings.debug, "debug", false, "Turn on debug output, shows all intercepted traffic. Works only when with `verbose` flag")
|
||||
flag.BoolVar(&Settings.stats, "stats", false, "Turn on queue stats output")
|
||||
flag.DurationVar(&Settings.exitAfter, "exit-after", 0, "exit after specified duration")
|
||||
|
||||
if DEMO == "" {
|
||||
flag.DurationVar(&Settings.ExitAfter, "exit-after", 0, "exit after specified duration")
|
||||
} else {
|
||||
Settings.ExitAfter = 5 * time.Minute
|
||||
}
|
||||
|
||||
flag.BoolVar(&Settings.SplitOutput, "split-output", false, "By default each output gets same traffic. If set to `true` it splits traffic equally among all outputs.")
|
||||
|
||||
flag.BoolVar(&Settings.RecognizeTCPSessions, "recognize-tcp-sessions", false, "[PRO] If turned on http output will create separate worker for each TCP session. Splitting output will session based as well.")
|
||||
flag.BoolVar(&Settings.splitOutput, "split-output", false, "By default each output gets same traffic. If set to `true` it splits traffic equally among all outputs.")
|
||||
|
||||
flag.Var(&Settings.inputDummy, "input-dummy", "Used for testing outputs. Emits 'Get /' request every 1s")
|
||||
flag.Var(&Settings.outputDummy, "output-dummy", "DEPRECATED: use --output-stdout instead")
|
||||
|
||||
flag.BoolVar(&Settings.OutputStdout, "output-stdout", true, "Used for testing inputs. Just prints to console data coming from inputs.")
|
||||
flag.BoolVar(&Settings.outputStdout, "output-stdout", false, "Used for testing inputs. Just prints to console data coming from inputs.")
|
||||
|
||||
flag.BoolVar(&Settings.OutputNull, "output-null", false, "Used for testing inputs. Drops all requests.")
|
||||
flag.BoolVar(&Settings.outputNull, "output-null", false, "Used for testing inputs. Drops all requests.")
|
||||
|
||||
flag.Var(&Settings.inputTCP, "input-tcp", "Used for internal communication between Gor instances. Example: \n\t# Receive requests from other Gor instances on 28020 port, and redirect output to staging\n\tgor --input-tcp :28020 --output-http staging.com")
|
||||
flag.BoolVar(&Settings.InputTCPConfig.Secure, "input-tcp-Secure", false, "Turn on TLS security. Do not forget to specify certificate and key files.")
|
||||
flag.StringVar(&Settings.InputTCPConfig.CertificatePath, "input-tcp-certificate", "", "Path to PEM encoded certificate file. Used when TLS turned on.")
|
||||
flag.StringVar(&Settings.InputTCPConfig.KeyPath, "input-tcp-certificate-key", "", "Path to PEM encoded certificate key file. Used when TLS turned on.")
|
||||
|
||||
flag.Var(&Settings.outputTCP, "output-tcp", "Used for internal communication between Gor instances. Example: \n\t# Listen for requests on 80 port and forward them to other Gor instance on 28020 port\n\tgor --input-raw :80 --output-tcp replay.local:28020")
|
||||
flag.BoolVar(&Settings.OutputTCPConfig.secure, "output-tcp-Secure", false, "Use TLS Secure connection. --input-file on another end should have TLS turned on as well.")
|
||||
flag.BoolVar(&Settings.OutputTCPConfig.sticky, "output-tcp-sticky", false, "Use Sticky connection. Request/Response with same ID will be sent to the same connection.")
|
||||
flag.BoolVar(&Settings.OutputTCPStats, "output-tcp-Stats", false, "Report TCP output queue Stats to console every 5 seconds.")
|
||||
flag.BoolVar(&Settings.outputTCPStats, "output-tcp-stats", false, "Report TCP output queue stats to console every 5 seconds.")
|
||||
|
||||
flag.Var(&Settings.inputFile, "input-file", "Read requests from file: \n\tgor --input-file ./requests.gor --output-http staging.com")
|
||||
flag.BoolVar(&Settings.InputFileLoop, "input-file-loop", false, "Loop input files, useful for performance testing.")
|
||||
flag.BoolVar(&Settings.inputFileLoop, "input-file-loop", false, "Loop input files, useful for performance testing.")
|
||||
|
||||
flag.Var(&Settings.outputFile, "output-file", "Write incoming requests to file: \n\tgor --input-raw :80 --output-file ./requests.gor")
|
||||
flag.DurationVar(&Settings.OutputFileConfig.flushInterval, "output-file-flush-interval", time.Second, "Interval for forcing buffer flush to the file, default: 1s.")
|
||||
flag.BoolVar(&Settings.OutputFileConfig.append, "output-file-append", false, "The flushed chunk is appended to existence file or not. ")
|
||||
flag.StringVar(&Settings.OutputFileSizeFlag, "output-file-size-limit", "32mb", "Size of each chunk. Default: 32mb")
|
||||
flag.Int64Var(&Settings.OutputFileConfig.queueLimit, "output-file-queue-limit", 256, "The length of the chunk queue. Default: 256")
|
||||
flag.StringVar(&Settings.OutputFileMaxSizeFlag, "output-file-max-size-limit", "1TB", "Max size of output file, Default: 1TB")
|
||||
flag.DurationVar(&Settings.outputFileConfig.flushInterval, "output-file-flush-interval", time.Second, "Interval for forcing buffer flush to the file, default: 1s.")
|
||||
flag.BoolVar(&Settings.outputFileConfig.append, "output-file-append", false, "The flushed chunk is appended to existence file or not. ")
|
||||
|
||||
flag.StringVar(&Settings.OutputFileConfig.bufferPath, "output-file-buffer", "/tmp", "The path for temporary storing current buffer: \n\tgor --input-raw :80 --output-file s3://mybucket/logs/%Y-%m-%d.gz --output-file-buffer /mnt/logs")
|
||||
// Set default
|
||||
Settings.outputFileConfig.sizeLimit.Set("32mb")
|
||||
flag.Var(&Settings.outputFileConfig.sizeLimit, "output-file-size-limit", "Size of each chunk. Default: 32mb")
|
||||
flag.IntVar(&Settings.outputFileConfig.queueLimit, "output-file-queue-limit", 256, "The length of the chunk queue. Default: 256")
|
||||
|
||||
flag.BoolVar(&Settings.PrettifyHTTP, "prettify-http", false, "If enabled, will automatically decode requests and responses with: Content-Encodning: gzip and Transfer-Encoding: chunked. Useful for debugging, in conjuction with --output-stdout")
|
||||
flag.BoolVar(&Settings.prettifyHTTP, "prettify-http", false, "If enabled, will automatically decode requests and responses with: Content-Encodning: gzip and Transfer-Encoding: chunked. Useful for debugging, in conjuction with --output-stdout")
|
||||
|
||||
flag.Var(&Settings.inputRAW, "input-raw", "Capture traffic from given port (use RAW sockets and require *sudo* access):\n\t# Capture traffic from 8080 port\n\tgor --input-raw :8080 --output-http staging.com")
|
||||
|
||||
flag.BoolVar(&Settings.InputRAWTrackResponse, "input-raw-track-response", false, "If turned on Gor will track responses in addition to requests, and they will be available to Middleware and file output.")
|
||||
flag.BoolVar(&Settings.inputRAWTrackResponse, "input-raw-track-response", false, "If turned on Gor will track responses in addition to requests, and they will be available to middleware and file output.")
|
||||
|
||||
flag.StringVar(&Settings.InputRAWEngine, "input-raw-engine", "libpcap", "Intercept traffic using `libpcap` (default), and `raw_socket`")
|
||||
flag.StringVar(&Settings.inputRAWEngine, "input-raw-engine", "libpcap", "Intercept traffic using `libpcap` (default), and `raw_socket`")
|
||||
|
||||
flag.StringVar(&Settings.InputRAWProtocol, "input-raw-protocol", "http", "Specify application protocol of intercepted traffic. Possible values: http, binary")
|
||||
flag.StringVar(&Settings.inputRAWRealIPHeader, "input-raw-realip-header", "", "If not blank, injects header with given name and real IP value to the request payload. Usually this header should be named: X-Real-IP")
|
||||
|
||||
flag.StringVar(&Settings.InputRAWRealIPHeader, "input-raw-realip-header", "", "If not blank, injects header with given name and real IP value to the request payload. Usually this header should be named: X-Real-IP")
|
||||
|
||||
flag.DurationVar(&Settings.InputRAWExpire, "input-raw-expire", time.Second*2, "How much it should wait for the last TCP packet, till consider that TCP message complete.")
|
||||
|
||||
flag.StringVar(&Settings.InputRAWBpfFilter, "input-raw-bpf-filter", "", "BPF filter to write custom expressions. Can be useful in case of non standard network interfaces like tunneling or SPAN port. Example: --input-raw-bpf-filter 'dst port 80'")
|
||||
|
||||
flag.StringVar(&Settings.InputRAWTimestampType, "input-raw-timestamp-type", "", "Possible values: PCAP_TSTAMP_HOST, PCAP_TSTAMP_HOST_LOWPREC, PCAP_TSTAMP_HOST_HIPREC, PCAP_TSTAMP_ADAPTER, PCAP_TSTAMP_ADAPTER_UNSYNCED. This values not supported on all systems, GoReplay will tell you available values of you put wrong one.")
|
||||
flag.StringVar(&Settings.CopyBufferSizeFlag, "copy-buffer-size", "5mb", "Set the buffer size for an individual request (default 5MB)")
|
||||
flag.BoolVar(&Settings.InputRAWOverrideSnapLen, "input-raw-override-snaplen", false, "Override the capture snaplen to be 64k. Required for some Virtualized environments")
|
||||
flag.BoolVar(&Settings.InputRAWImmediateMode, "input-raw-immediate-mode", false, "Set pcap interface to immediate mode.")
|
||||
flag.StringVar(&Settings.InputRAWBufferSizeFlag, "input-raw-buffer-size", "0", "Controls size of the OS buffer which holds packets until they dispatched. Default value depends by system: in Linux around 2MB. If you see big package drop, increase this value.")
|
||||
|
||||
flag.StringVar(&Settings.Middleware, "Middleware", "", "Used for modifying traffic using external command")
|
||||
flag.StringVar(&Settings.middleware, "middleware", "", "Used for modifying traffic using external command")
|
||||
|
||||
// flag.Var(&Settings.inputHTTP, "input-http", "Read requests from HTTP, should be explicitly sent from your application:\n\t# Listen for http on 9000\n\tgor --input-http :9000 --output-http staging.com")
|
||||
|
||||
flag.Var(&Settings.outputHTTP, "output-http", "Forwards incoming requests to given http address.\n\t# Redirect all incoming requests to staging.com address \n\tgor --input-raw :80 --output-http http://staging.com")
|
||||
flag.IntVar(&Settings.outputHTTPConfig.BufferSize, "output-http-response-buffer", 0, "HTTP response buffer size, all data after this size will be discarded.")
|
||||
flag.IntVar(&Settings.outputHTTPConfig.workers, "output-http-workers", 0, "Gor uses dynamic worker scaling by default. Enter a number to run a set number of workers.")
|
||||
flag.IntVar(&Settings.outputHTTPConfig.redirectLimit, "output-http-redirects", 0, "Enable how often redirects should be followed.")
|
||||
flag.DurationVar(&Settings.outputHTTPConfig.Timeout, "output-http-timeout", 5*time.Second, "Specify HTTP request/response timeout. By default 5s. Example: --output-http-timeout 30s")
|
||||
flag.BoolVar(&Settings.outputHTTPConfig.TrackResponses, "output-http-track-response", false, "If turned on, HTTP output responses will be set to all outputs like stdout, file and etc.")
|
||||
|
||||
/* OutputHTTPConfig */
|
||||
flag.IntVar(&Settings.OutputHTTPConfig.BufferSize, "output-http-response-buffer", 0, "HTTP response buffer size, all data after this size will be discarded.")
|
||||
flag.BoolVar(&Settings.OutputHTTPConfig.CompatibilityMode, "output-http-compatibility-mode", false, "Use standard Go client, instead of built-in implementation. Can be slower, but more compatible.")
|
||||
flag.BoolVar(&Settings.outputHTTPConfig.stats, "output-http-stats", false, "Report http output queue stats to console every 5 seconds.")
|
||||
flag.BoolVar(&Settings.outputHTTPConfig.OriginalHost, "http-original-host", false, "Normally gor replaces the Host http header with the host supplied with --output-http. This option disables that behavior, preserving the original Host header.")
|
||||
flag.BoolVar(&Settings.outputHTTPConfig.Debug, "output-http-debug", false, "Enables http debug output.")
|
||||
|
||||
flag.IntVar(&Settings.OutputHTTPConfig.workersMin, "output-http-workers-min", 0, "Gor uses dynamic worker scaling. Enter a number to set a minimum number of workers. default = 1.")
|
||||
flag.IntVar(&Settings.OutputHTTPConfig.workersMax, "output-http-workers", 0, "Gor uses dynamic worker scaling. Enter a number to set a maximum number of workers. default = 0 = unlimited.")
|
||||
flag.IntVar(&Settings.OutputHTTPConfig.queueLen, "output-http-queue-len", 1000, "Number of requests that can be queued for output, if all workers are busy. default = 1000")
|
||||
flag.StringVar(&Settings.outputHTTPConfig.elasticSearch, "output-http-elasticsearch", "", "Send request and response stats to ElasticSearch:\n\tgor --input-raw :8080 --output-http staging.com --output-http-elasticsearch 'es_host:api_port/index_name'")
|
||||
|
||||
flag.IntVar(&Settings.OutputHTTPConfig.redirectLimit, "output-http-redirects", 0, "Enable how often redirects should be followed.")
|
||||
flag.DurationVar(&Settings.OutputHTTPConfig.Timeout, "output-http-timeout", 5*time.Second, "Specify HTTP request/response timeout. By default 5s. Example: --output-http-timeout 30s")
|
||||
flag.BoolVar(&Settings.OutputHTTPConfig.TrackResponses, "output-http-track-response", false, "If turned on, HTTP output responses will be set to all outputs like stdout, file and etc.")
|
||||
|
||||
flag.BoolVar(&Settings.OutputHTTPConfig.stats, "output-http-Stats", false, "Report http output queue Stats to console every N milliseconds. See output-http-Stats-ms")
|
||||
flag.IntVar(&Settings.OutputHTTPConfig.statsMs, "output-http-Stats-ms", 5000, "Report http output queue Stats to console every N milliseconds. default: 5000")
|
||||
flag.BoolVar(&Settings.OutputHTTPConfig.OriginalHost, "http-original-host", false, "Normally gor replaces the Host http header with the host supplied with --output-http. This option disables that behavior, preserving the original Host header.")
|
||||
flag.BoolVar(&Settings.OutputHTTPConfig.Debug, "output-http-Debug", false, "Enables http Debug output.")
|
||||
flag.StringVar(&Settings.OutputHTTPConfig.elasticSearch, "output-http-elasticsearch", "", "Send request and response Stats to ElasticSearch:\n\tgor --input-raw :8080 --output-http staging.com --output-http-elasticsearch 'es_host:api_port/index_name'")
|
||||
/* OutputHTTPConfig */
|
||||
|
||||
flag.Var(&Settings.outputBinary, "output-binary", "Forwards incoming binary payloads to given address.\n\t# Redirect all incoming requests to staging.com address \n\tgor --input-raw :80 --input-raw-protocol binary --output-binary staging.com:80")
|
||||
/* OutputBinaryConfig */
|
||||
flag.IntVar(&Settings.OutputBinaryConfig.BufferSize, "output-tcp-response-buffer", 0, "TCP response buffer size, all data after this size will be discarded.")
|
||||
flag.IntVar(&Settings.OutputBinaryConfig.workers, "output-binary-workers", 0, "Gor uses dynamic worker scaling by default. Enter a number to run a set number of workers.")
|
||||
flag.DurationVar(&Settings.OutputBinaryConfig.Timeout, "output-binary-timeout", 0, "Specify HTTP request/response timeout. By default 5s. Example: --output-binary-timeout 30s")
|
||||
flag.BoolVar(&Settings.OutputBinaryConfig.TrackResponses, "output-binary-track-response", false, "If turned on, Binary output responses will be set to all outputs like stdout, file and etc.")
|
||||
|
||||
flag.BoolVar(&Settings.OutputBinaryConfig.Debug, "output-binary-Debug", false, "Enables binary Debug output.")
|
||||
/* OutputBinaryConfig */
|
||||
|
||||
flag.StringVar(&Settings.outputKafkaConfig.host, "output-kafka-host", "", "Read request and response Stats from Kafka:\n\tgor --input-raw :8080 --output-kafka-host '192.168.0.1:9092,192.168.0.2:9092'")
|
||||
flag.StringVar(&Settings.outputKafkaConfig.topic, "output-kafka-topic", "", "Read request and response Stats from Kafka:\n\tgor --input-raw :8080 --output-kafka-topic 'kafka-log'")
|
||||
flag.StringVar(&Settings.outputKafkaConfig.host, "output-kafka-host", "", "Read request and response stats from Kafka:\n\tgor --input-raw :8080 --output-kafka-host '192.168.0.1:9092,192.168.0.2:9092'")
|
||||
flag.StringVar(&Settings.outputKafkaConfig.topic, "output-kafka-topic", "", "Read request and response stats from Kafka:\n\tgor --input-raw :8080 --output-kafka-topic 'kafka-log'")
|
||||
flag.BoolVar(&Settings.outputKafkaConfig.useJSON, "output-kafka-json-format", false, "If turned on, it will serialize messages from GoReplay text format to JSON.")
|
||||
|
||||
flag.StringVar(&Settings.inputKafkaConfig.host, "input-kafka-host", "", "Send request and response Stats to Kafka:\n\tgor --output-stdout --input-kafka-host '192.168.0.1:9092,192.168.0.2:9092'")
|
||||
flag.StringVar(&Settings.inputKafkaConfig.topic, "input-kafka-topic", "", "Send request and response Stats to Kafka:\n\tgor --output-stdout --input-kafka-topic 'kafka-log'")
|
||||
flag.StringVar(&Settings.inputKafkaConfig.host, "input-kafka-host", "", "Send request and response stats to Kafka:\n\tgor --output-stdout --input-kafka-host '192.168.0.1:9092,192.168.0.2:9092'")
|
||||
flag.StringVar(&Settings.inputKafkaConfig.topic, "input-kafka-topic", "", "Send request and response stats to Kafka:\n\tgor --output-stdout --input-kafka-topic 'kafka-log'")
|
||||
flag.BoolVar(&Settings.inputKafkaConfig.useJSON, "input-kafka-json-format", false, "If turned on, it will assume that messages coming in JSON format rather than GoReplay text format.")
|
||||
|
||||
flag.Var(&Settings.modifierConfig.headers, "http-set-header", "Inject additional headers to http reqest:\n\tgor --input-raw :8080 --output-http staging.com --http-set-header 'User-Agent: Gor'")
|
||||
flag.Var(&Settings.modifierConfig.headers, "output-http-header", "WARNING: `--output-http-header` DEPRECATED, use `--http-set-header` instead")
|
||||
|
||||
flag.Var(&Settings.modifierConfig.headerRewrite, "http-rewrite-header", "Rewrite the request header based on a mapping:\n\tgor --input-raw :8080 --output-http staging.com --http-rewrite-header Host: (.*).example.com,$1.beta.example.com")
|
||||
|
||||
flag.Var(&Settings.modifierConfig.params, "http-set-param", "Set request url param, if param already exists it will be overwritten:\n\tgor --input-raw :8080 --output-http staging.com --http-set-param api_key=1")
|
||||
|
||||
flag.Var(&Settings.modifierConfig.methods, "http-allow-method", "Whitelist of HTTP methods to replay. Anything else will be dropped:\n\tgor --input-raw :8080 --output-http staging.com --http-allow-method GET --http-allow-method OPTIONS")
|
||||
@@ -293,184 +163,25 @@ func init() {
|
||||
|
||||
flag.Var(&Settings.modifierConfig.headerNegativeFilters, "http-disallow-header", "A regexp to match a specific header against. Requests with matching headers will be dropped:\n\t gor --input-raw :8080 --output-http staging.com --http-disallow-header \"User-Agent: Replayed by Gor\"")
|
||||
|
||||
flag.Var(&Settings.modifierConfig.headerBasicAuthFilters, "http-basic-auth-filter", "A regexp to match the decoded basic auth string against. Requests with non-matching headers will be dropped:\n\t gor --input-raw :8080 --output-http staging.com --http-basic-auth-filter \"^customer[0-9].*\"")
|
||||
|
||||
flag.Var(&Settings.modifierConfig.headerHashFilters, "http-header-limiter", "Takes a fraction of requests, consistently taking or rejecting a request based on the FNV32-1A hash of a specific header:\n\t gor --input-raw :8080 --output-http staging.com --http-header-limiter user-id:25%")
|
||||
|
||||
flag.Var(&Settings.modifierConfig.headerHashFilters, "http-header-limiter", "Takes a fraction of requests, consistently taking or rejecting a request based on the FNV32-1A hash of a specific header:\n\t gor --input-raw :8080 --output-http staging.com --http-header-imiter user-id:25%")
|
||||
flag.Var(&Settings.modifierConfig.headerHashFilters, "output-http-header-hash-filter", "WARNING: `output-http-header-hash-filter` DEPRECATED, use `--http-header-hash-limiter` instead")
|
||||
|
||||
flag.Var(&Settings.modifierConfig.paramHashFilters, "http-param-limiter", "Takes a fraction of requests, consistently taking or rejecting a request based on the FNV32-1A hash of a specific GET param:\n\t gor --input-raw :8080 --output-http staging.com --http-param-limiter user_id:25%")
|
||||
|
||||
// default values, using for tests
|
||||
Settings.OutputFileConfig.sizeLimit = 33554432
|
||||
Settings.OutputFileConfig.outputFileMaxSize = 1099511627776
|
||||
Settings.CopyBufferSize = 5242880
|
||||
Settings.InputRAWBufferSize = 0
|
||||
|
||||
currrentDir, _ := os.Getwd()
|
||||
log.Printf("Locating config in folder: %s", currrentDir)
|
||||
|
||||
flag.StringVar(&Settings.ConfigFile, "config-file", "config.json", "The path to Goreplay config file.")
|
||||
viper.SetConfigFile(currrentDir + "/" +Settings.ConfigFile)
|
||||
|
||||
flag.StringVar(&Settings.ConfigServerAddress, "config-addr", ":9999", "The host address for config API.")
|
||||
viper.SetConfigFile(Settings.ConfigFile)
|
||||
|
||||
flag.StringVar(&Settings.RemoteConfigHost, "config-remote-host", "", "The host address for config API.")
|
||||
|
||||
pflag.CommandLine.AddGoFlagSet(flag.CommandLine)
|
||||
pflag.Parse()
|
||||
viper.BindPFlags(pflag.CommandLine)
|
||||
|
||||
// Searches for config file in given paths and read it
|
||||
if err := viper.ReadInConfig(); err != nil {
|
||||
log.Printf("Error reading config file, %s", err)
|
||||
return
|
||||
}
|
||||
|
||||
if viper.IsSet("RemoteConfigHost") {
|
||||
go pollRemoteConfig()
|
||||
}
|
||||
|
||||
fmt.Printf("Using config: %s\n", viper.ConfigFileUsed())
|
||||
|
||||
initConfigServer()
|
||||
}
|
||||
|
||||
func pollRemoteConfig() {
|
||||
for {
|
||||
req, err := http.NewRequest("GET", viper.GetString("RemoteConfigHost") + "/config.json", nil)
|
||||
if err != nil {
|
||||
log.Printf("Error while getting config from remote server., %s", err)
|
||||
continue
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
|
||||
client := &http.Client{}
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
log.Printf("Error while getting config from remote server., %s", err)
|
||||
continue
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
respBody, err := ioutil.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
log.Printf("Error while getting config from remote server., %s", err)
|
||||
continue
|
||||
}
|
||||
|
||||
updateConfig(respBody)
|
||||
resp.Body.Close()
|
||||
time.Sleep(time.Second)
|
||||
}
|
||||
}
|
||||
|
||||
func checkSettings() {
|
||||
outputFileSize, err := bufferParser(Settings.OutputFileSizeFlag, "32MB")
|
||||
if err != nil {
|
||||
log.Fatalf("output-file-size-limit error: %v\n", err)
|
||||
}
|
||||
Settings.OutputFileConfig.sizeLimit = outputFileSize
|
||||
|
||||
outputFileMaxSize, err := bufferParser(Settings.OutputFileMaxSizeFlag, "1TB")
|
||||
if err != nil {
|
||||
log.Fatalf("output-file-max-size-limit error: %v\n", err)
|
||||
}
|
||||
Settings.OutputFileConfig.outputFileMaxSize = outputFileMaxSize
|
||||
|
||||
copyBufferSize, err := bufferParser(Settings.CopyBufferSizeFlag, "5mb")
|
||||
if err != nil {
|
||||
log.Fatalf("copy-buffer-size error: %v\n", err)
|
||||
}
|
||||
Settings.CopyBufferSize = copyBufferSize
|
||||
|
||||
inputRAWBufferSize, err := bufferParser(Settings.InputRAWBufferSizeFlag, "0")
|
||||
if err != nil {
|
||||
log.Fatalf("input-raw-buffer-size error: %v\n", err)
|
||||
}
|
||||
Settings.InputRAWBufferSize = inputRAWBufferSize
|
||||
|
||||
// libpcap has bug in mac os x. More info: https://github.com/buger/goreplay/issues/730
|
||||
if Settings.InputRAWExpire == time.Second*2 && runtime.GOOS == "darwin" {
|
||||
Settings.InputRAWExpire = time.Second
|
||||
}
|
||||
}
|
||||
|
||||
var previousDebugTime = time.Now()
|
||||
var previousDebugTime int64
|
||||
var debugMutex sync.Mutex
|
||||
var pID = os.Getpid()
|
||||
|
||||
// Debug take an effect only if --Verbose flag specified
|
||||
// Debug gets called only if --verbose flag specified
|
||||
func Debug(args ...interface{}) {
|
||||
if Settings.Verbose {
|
||||
if Settings.verbose {
|
||||
debugMutex.Lock()
|
||||
defer debugMutex.Unlock()
|
||||
now := time.Now()
|
||||
diff := now.Sub(previousDebugTime).String()
|
||||
previousDebugTime = now
|
||||
fmt.Printf("[DEBUG][PID %d][%s][elapsed %s] ", pID, now.Format(time.StampNano), diff)
|
||||
diff := float64(now.UnixNano()-previousDebugTime) / 1000000
|
||||
previousDebugTime = now.UnixNano()
|
||||
debugMutex.Unlock()
|
||||
|
||||
fmt.Printf("[DEBUG][PID %d][%d][%fms] ", os.Getpid(), now.UnixNano(), diff)
|
||||
fmt.Println(args...)
|
||||
}
|
||||
}
|
||||
|
||||
// the following regexes follow Go semantics https://golang.org/ref/spec#Letters_and_digits
|
||||
var (
|
||||
rB = regexp.MustCompile(`(?i)^(?:0b|0x|0o)?[\da-f_]+$`)
|
||||
rKB = regexp.MustCompile(`(?i)^(?:0b|0x|0o)?[\da-f_]+kb$`)
|
||||
rMB = regexp.MustCompile(`(?i)^(?:0b|0x|0o)?[\da-f_]+mb$`)
|
||||
rGB = regexp.MustCompile(`(?i)^(?:0b|0x|0o)?[\da-f_]+gb$`)
|
||||
rTB = regexp.MustCompile(`(?i)^(?:0b|0x|0o)?[\da-f_]+tb$`)
|
||||
empt = regexp.MustCompile(`^[\n\t\r 0.\f\a]*$`)
|
||||
)
|
||||
|
||||
// bufferParser parses buffer to bytes from different bases and data units
|
||||
// size is the buffer in string, rpl act as a replacement for empty buffer.
|
||||
// e.g: (--output-file-size-limit "") may override default 32mb with empty buffer,
|
||||
// which can be solved by setting rpl by bufferParser(buffer, "32mb")
|
||||
func bufferParser(size, rpl string) (buffer int64, err error) {
|
||||
const (
|
||||
_ = 1 << (iota * 10)
|
||||
KB
|
||||
MB
|
||||
GB
|
||||
TB
|
||||
)
|
||||
|
||||
var (
|
||||
lmt = len(size) - 2
|
||||
s = []byte(size)
|
||||
)
|
||||
|
||||
if empt.Match(s) {
|
||||
size = rpl
|
||||
s = []byte(size)
|
||||
}
|
||||
|
||||
// recover, especially when buffer size overflows int64 i.e ~8019PBs
|
||||
defer func() {
|
||||
if e, ok := recover().(error); ok {
|
||||
err = e.(error)
|
||||
}
|
||||
}()
|
||||
|
||||
switch {
|
||||
case rB.Match(s):
|
||||
buffer, err = strconv.ParseInt(size, 0, 64)
|
||||
case rKB.Match(s):
|
||||
buffer, err = strconv.ParseInt(size[:lmt], 0, 64)
|
||||
buffer *= KB
|
||||
case rMB.Match(s):
|
||||
buffer, err = strconv.ParseInt(size[:lmt], 0, 64)
|
||||
buffer *= MB
|
||||
case rGB.Match(s):
|
||||
buffer, err = strconv.ParseInt(size[:lmt], 0, 64)
|
||||
buffer *= GB
|
||||
case rTB.Match(s):
|
||||
buffer, err = strconv.ParseInt(size[:lmt], 0, 64)
|
||||
buffer *= TB
|
||||
default:
|
||||
return 0, fmt.Errorf("invalid buffer %q", size)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
@@ -1,39 +0,0 @@
|
||||
name: goreplay
|
||||
version: '1.0'
|
||||
summary: GoReplay is an open-source tool for capturing and replaying live HTTP traffic
|
||||
description: |
|
||||
GoReplay is an open-source tool for capturing and replaying
|
||||
live HTTP traffic into a test environment in order to continuously
|
||||
test your system with real data. It can be used to increase confidence
|
||||
in code deployments, configuration changes and infrastructure changes.
|
||||
grade: stable
|
||||
confinement: strict
|
||||
base: core18
|
||||
parts:
|
||||
goreplay:
|
||||
plugin: go
|
||||
source: https://github.com/buger/goreplay.git
|
||||
go-importpath: github.com/buger/goreplay
|
||||
build-packages:
|
||||
- build-essential
|
||||
- libpcap-dev
|
||||
stage-packages:
|
||||
- libpcap0.8
|
||||
|
||||
apps:
|
||||
goreplay:
|
||||
command: bin/goreplay
|
||||
daemon: simple
|
||||
restart-condition: on-abnormal
|
||||
plugs:
|
||||
- home
|
||||
- network
|
||||
- network-bind
|
||||
- network-control
|
||||
- network-observe
|
||||
- netlink-connector
|
||||
- netlink-audit
|
||||
- bluetooth-control
|
||||
- firewall-control
|
||||
- x11
|
||||
|
||||
-196
@@ -1,196 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"crypto/tls"
|
||||
"io"
|
||||
"log"
|
||||
"net"
|
||||
"runtime/debug"
|
||||
"syscall"
|
||||
"time"
|
||||
)
|
||||
|
||||
// TCPClientConfig client configuration
|
||||
type TCPClientConfig struct {
|
||||
Debug bool
|
||||
ConnectionTimeout time.Duration
|
||||
Timeout time.Duration
|
||||
ResponseBufferSize int
|
||||
Secure bool
|
||||
}
|
||||
|
||||
// TCPClient client connection properties
|
||||
type TCPClient struct {
|
||||
baseURL string
|
||||
addr string
|
||||
conn net.Conn
|
||||
respBuf []byte
|
||||
config *TCPClientConfig
|
||||
redirectsCount int
|
||||
}
|
||||
|
||||
// NewTCPClient returns new TCPClient
|
||||
func NewTCPClient(addr string, config *TCPClientConfig) *TCPClient {
|
||||
if config.Timeout.Nanoseconds() == 0 {
|
||||
config.Timeout = 5 * time.Second
|
||||
}
|
||||
|
||||
config.ConnectionTimeout = config.Timeout
|
||||
|
||||
if config.ResponseBufferSize == 0 {
|
||||
config.ResponseBufferSize = 100 * 1024 // 100kb
|
||||
}
|
||||
|
||||
client := &TCPClient{config: config, addr: addr}
|
||||
client.respBuf = make([]byte, config.ResponseBufferSize)
|
||||
|
||||
return client
|
||||
}
|
||||
|
||||
// Connect creates a tcp connection of the client
|
||||
func (c *TCPClient) Connect() (err error) {
|
||||
c.Disconnect()
|
||||
|
||||
c.conn, err = net.DialTimeout("tcp", c.addr, c.config.ConnectionTimeout)
|
||||
|
||||
if c.config.Secure {
|
||||
tlsConn := tls.Client(c.conn, &tls.Config{InsecureSkipVerify: true})
|
||||
|
||||
if err = tlsConn.Handshake(); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
c.conn = tlsConn
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
// Disconnect closes the client connection
|
||||
func (c *TCPClient) Disconnect() {
|
||||
if c.conn != nil {
|
||||
c.conn.Close()
|
||||
c.conn = nil
|
||||
Debug("[TCPClient] Disconnected: ", c.baseURL)
|
||||
}
|
||||
}
|
||||
|
||||
func (c *TCPClient) isAlive() bool {
|
||||
one := make([]byte, 1)
|
||||
|
||||
// Ready 1 byte from socket without timeout to check if it not closed
|
||||
c.conn.SetReadDeadline(time.Now().Add(time.Millisecond))
|
||||
_, err := c.conn.Read(one)
|
||||
|
||||
if err == nil {
|
||||
return true
|
||||
} else if err == io.EOF {
|
||||
if c.config.Debug {
|
||||
Debug("[TCPClient] connection closed, reconnecting")
|
||||
}
|
||||
return false
|
||||
} else if err == syscall.EPIPE {
|
||||
Debug("Detected broken pipe.", err)
|
||||
return false
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
// Send sends data over created tcp connection
|
||||
func (c *TCPClient) Send(data []byte) (response []byte, err error) {
|
||||
// Don't exit on panic
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
Debug("[TCPClient]", r, string(data))
|
||||
|
||||
if _, ok := r.(error); !ok {
|
||||
log.Println("[TCPClient] Failed to send request: ", string(data))
|
||||
log.Println("PANIC: pkg:", r, debug.Stack())
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
if c.conn == nil || !c.isAlive() {
|
||||
Debug("[TCPClient] Connecting:", c.baseURL)
|
||||
if err = c.Connect(); err != nil {
|
||||
log.Println("[TCPClient] Connection error:", err)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
timeout := time.Now().Add(c.config.Timeout)
|
||||
|
||||
c.conn.SetWriteDeadline(timeout)
|
||||
|
||||
if c.config.Debug {
|
||||
Debug("[TCPClient] Sending:", string(data))
|
||||
}
|
||||
|
||||
if _, err = c.conn.Write(data); err != nil {
|
||||
Debug("[TCPClient] Write error:", err, c.baseURL)
|
||||
return
|
||||
}
|
||||
|
||||
var readBytes, n int
|
||||
var currentChunk []byte
|
||||
timeout = time.Now().Add(c.config.Timeout)
|
||||
|
||||
for {
|
||||
c.conn.SetReadDeadline(timeout)
|
||||
|
||||
if readBytes < len(c.respBuf) {
|
||||
n, err = c.conn.Read(c.respBuf[readBytes:])
|
||||
readBytes += n
|
||||
|
||||
if err != nil {
|
||||
if err == io.EOF {
|
||||
err = nil
|
||||
}
|
||||
break
|
||||
}
|
||||
} else {
|
||||
if currentChunk == nil {
|
||||
currentChunk = make([]byte, readChunkSize)
|
||||
}
|
||||
|
||||
n, err = c.conn.Read(currentChunk)
|
||||
|
||||
if err == io.EOF {
|
||||
break
|
||||
} else if err != nil {
|
||||
Debug("[TCPClient] Read the whole body error:", err, c.baseURL)
|
||||
break
|
||||
}
|
||||
|
||||
readBytes += int(n)
|
||||
}
|
||||
|
||||
if readBytes >= maxResponseSize {
|
||||
Debug("[TCPClient] Body is more than the max size", maxResponseSize,
|
||||
c.baseURL)
|
||||
break
|
||||
}
|
||||
|
||||
// For following chunks expect less timeout
|
||||
timeout = time.Now().Add(c.config.Timeout / 5)
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
Debug("[TCPClient] Response read error", err, c.conn, readBytes)
|
||||
return
|
||||
}
|
||||
|
||||
if readBytes > len(c.respBuf) {
|
||||
readBytes = len(c.respBuf)
|
||||
}
|
||||
|
||||
payload := make([]byte, readBytes)
|
||||
copy(payload, c.respBuf[:readBytes])
|
||||
|
||||
if c.config.Debug {
|
||||
Debug("[TCPClient] Received:", string(payload))
|
||||
}
|
||||
|
||||
return payload, err
|
||||
}
|
||||
+10
-24
@@ -3,51 +3,37 @@ package main
|
||||
import (
|
||||
"crypto/rand"
|
||||
"encoding/base64"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
)
|
||||
|
||||
// ErrorStopped is the error returned when the go routines reading the input is stopped.
|
||||
var ErrorStopped = errors.New("reading stopped")
|
||||
|
||||
// TestInput used for testing purpose, it allows emitting requests on demand
|
||||
type TestInput struct {
|
||||
data chan []byte
|
||||
skipHeader bool
|
||||
stop chan bool // Channel used only to indicate goroutine should shutdown
|
||||
}
|
||||
|
||||
// NewTestInput constructor for TestInput
|
||||
func NewTestInput() (i *TestInput) {
|
||||
i = new(TestInput)
|
||||
i.data = make(chan []byte, 100)
|
||||
i.stop = make(chan bool)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
func (i *TestInput) Read(data []byte) (int, error) {
|
||||
select {
|
||||
case buf := <-i.data:
|
||||
var header []byte
|
||||
buf := <-i.data
|
||||
|
||||
if !i.skipHeader {
|
||||
header = payloadHeader(RequestPayload, uuid(), time.Now().UnixNano(), -1)
|
||||
copy(data[0:len(header)], header)
|
||||
copy(data[len(header):], buf)
|
||||
} else {
|
||||
copy(data, buf)
|
||||
}
|
||||
var header []byte
|
||||
|
||||
return len(buf) + len(header), nil
|
||||
case <-time.After(10 * time.Second):
|
||||
return 0, fmt.Errorf("timed out waiting for read")
|
||||
if !i.skipHeader {
|
||||
header = payloadHeader(RequestPayload, uuid(), time.Now().UnixNano(), -1)
|
||||
copy(data[0:len(header)], header)
|
||||
copy(data[len(header):], buf)
|
||||
} else {
|
||||
copy(data, buf)
|
||||
}
|
||||
}
|
||||
|
||||
func (i *TestInput) Close() error {
|
||||
close(i.stop)
|
||||
return nil
|
||||
return len(buf) + len(header), nil
|
||||
}
|
||||
|
||||
func (i *TestInput) EmitBytes(data []byte) {
|
||||
|
||||
-27
@@ -1,27 +0,0 @@
|
||||
# Compiled Object files, Static and Dynamic libs (Shared Objects)
|
||||
*.o
|
||||
*.a
|
||||
*.so
|
||||
*.test
|
||||
|
||||
# Folders
|
||||
_obj
|
||||
_test
|
||||
.vagrant
|
||||
|
||||
# Architecture specific extensions/prefixes
|
||||
*.[568vq]
|
||||
[568vq].out
|
||||
|
||||
*.cgo1.go
|
||||
*.cgo2.c
|
||||
_cgo_defun.c
|
||||
_cgo_gotypes.go
|
||||
_cgo_export.*
|
||||
|
||||
_testmain.go
|
||||
|
||||
*.exe
|
||||
|
||||
coverage.txt
|
||||
profile.out
|
||||
-74
@@ -1,74 +0,0 @@
|
||||
run:
|
||||
timeout: 5m
|
||||
deadline: 10m
|
||||
|
||||
linters-settings:
|
||||
govet:
|
||||
check-shadowing: false
|
||||
golint:
|
||||
min-confidence: 0
|
||||
gocyclo:
|
||||
min-complexity: 99
|
||||
maligned:
|
||||
suggest-new: true
|
||||
dupl:
|
||||
threshold: 100
|
||||
goconst:
|
||||
min-len: 2
|
||||
min-occurrences: 3
|
||||
misspell:
|
||||
locale: US
|
||||
goimports:
|
||||
local-prefixes: github.com/Shopify/sarama
|
||||
gocritic:
|
||||
enabled-tags:
|
||||
- diagnostic
|
||||
- experimental
|
||||
- opinionated
|
||||
- performance
|
||||
- style
|
||||
disabled-checks:
|
||||
- wrapperFunc
|
||||
- ifElseChain
|
||||
funlen:
|
||||
lines: 300
|
||||
statements: 300
|
||||
|
||||
linters:
|
||||
disable-all: true
|
||||
enable:
|
||||
- bodyclose
|
||||
- deadcode
|
||||
- depguard
|
||||
- dogsled
|
||||
# - dupl
|
||||
- errcheck
|
||||
- funlen
|
||||
# - gocritic
|
||||
- gocyclo
|
||||
- gofmt
|
||||
- goimports
|
||||
# - golint
|
||||
- gosec
|
||||
# - gosimple
|
||||
- govet
|
||||
# - ineffassign
|
||||
- interfacer
|
||||
# - misspell
|
||||
# - nakedret
|
||||
# - scopelint
|
||||
# - staticcheck
|
||||
- structcheck
|
||||
# - stylecheck
|
||||
- typecheck
|
||||
- unconvert
|
||||
- unused
|
||||
- varcheck
|
||||
- whitespace
|
||||
# - goconst
|
||||
# - gochecknoinits
|
||||
|
||||
issues:
|
||||
exclude:
|
||||
- consider giving a name to these results
|
||||
- include an explanation for nolint directive
|
||||
-912
@@ -1,912 +0,0 @@
|
||||
# Changelog
|
||||
|
||||
#### Unreleased
|
||||
|
||||
#### Version 1.26.1 (2020-02-04)
|
||||
|
||||
Improvements:
|
||||
- Add requests-in-flight metric ([1539](https://github.com/Shopify/sarama/pull/1539))
|
||||
- Fix misleading example for cluster admin ([1595](https://github.com/Shopify/sarama/pull/1595))
|
||||
- Replace Travis with GitHub Actions, linters housekeeping ([1573](https://github.com/Shopify/sarama/pull/1573))
|
||||
- Allow BalanceStrategy to provide custom assignment data ([1592](https://github.com/Shopify/sarama/pull/1592))
|
||||
|
||||
Bug Fixes:
|
||||
- Adds back Consumer.Offsets.CommitInterval to fix API ([1590](https://github.com/Shopify/sarama/pull/1590))
|
||||
- Fix error message s/CommitInterval/AutoCommit.Interval ([1589](https://github.com/Shopify/sarama/pull/1589))
|
||||
|
||||
#### Version 1.26.0 (2020-01-24)
|
||||
|
||||
New Features:
|
||||
- Enable zstd compression
|
||||
([1574](https://github.com/Shopify/sarama/pull/1574),
|
||||
[1582](https://github.com/Shopify/sarama/pull/1582))
|
||||
- Support headers in tools kafka-console-producer
|
||||
([1549](https://github.com/Shopify/sarama/pull/1549))
|
||||
|
||||
Improvements:
|
||||
- Add SASL AuthIdentity to SASL frames (authzid)
|
||||
([1585](https://github.com/Shopify/sarama/pull/1585)).
|
||||
|
||||
Bug Fixes:
|
||||
- Sending messages with ZStd compression enabled fails in multiple ways
|
||||
([1252](https://github.com/Shopify/sarama/issues/1252)).
|
||||
- Use the broker for any admin on BrokerConfig
|
||||
([1571](https://github.com/Shopify/sarama/pull/1571)).
|
||||
- Set DescribeConfigRequest Version field
|
||||
([1576](https://github.com/Shopify/sarama/pull/1576)).
|
||||
- ConsumerGroup flooding logs with client/metadata update req
|
||||
([1578](https://github.com/Shopify/sarama/pull/1578)).
|
||||
- MetadataRequest version in DescribeCluster
|
||||
([1580](https://github.com/Shopify/sarama/pull/1580)).
|
||||
- Fix deadlock in consumer group handleError
|
||||
([1581](https://github.com/Shopify/sarama/pull/1581))
|
||||
- Fill in the Fetch{Request,Response} protocol
|
||||
([1582](https://github.com/Shopify/sarama/pull/1582)).
|
||||
- Retry topic request on ControllerNotAvailable
|
||||
([1586](https://github.com/Shopify/sarama/pull/1586)).
|
||||
|
||||
#### Version 1.25.0 (2020-01-13)
|
||||
|
||||
New Features:
|
||||
- Support TLS protocol in kafka-producer-performance
|
||||
([1538](https://github.com/Shopify/sarama/pull/1538)).
|
||||
- Add support for kafka 2.4.0
|
||||
([1552](https://github.com/Shopify/sarama/pull/1552)).
|
||||
|
||||
Improvements:
|
||||
- Allow the Consumer to disable auto-commit offsets
|
||||
([1164](https://github.com/Shopify/sarama/pull/1164)).
|
||||
- Produce records with consistent timestamps
|
||||
([1455](https://github.com/Shopify/sarama/pull/1455)).
|
||||
|
||||
Bug Fixes:
|
||||
- Fix incorrect SetTopicMetadata name mentions
|
||||
([1534](https://github.com/Shopify/sarama/pull/1534)).
|
||||
- Fix client.tryRefreshMetadata Println
|
||||
([1535](https://github.com/Shopify/sarama/pull/1535)).
|
||||
- Fix panic on calling updateMetadata on closed client
|
||||
([1531](https://github.com/Shopify/sarama/pull/1531)).
|
||||
- Fix possible faulty metrics in TestFuncProducing
|
||||
([1545](https://github.com/Shopify/sarama/pull/1545)).
|
||||
|
||||
#### Version 1.24.1 (2019-10-31)
|
||||
|
||||
New Features:
|
||||
- Add DescribeLogDirs Request/Response pair
|
||||
([1520](https://github.com/Shopify/sarama/pull/1520)).
|
||||
|
||||
Bug Fixes:
|
||||
- Fix ClusterAdmin returning invalid controller ID on DescribeCluster
|
||||
([1518](https://github.com/Shopify/sarama/pull/1518)).
|
||||
- Fix issue with consumergroup not rebalancing when new partition is added
|
||||
([1525](https://github.com/Shopify/sarama/pull/1525)).
|
||||
- Ensure consistent use of read/write deadlines
|
||||
([1529](https://github.com/Shopify/sarama/pull/1529)).
|
||||
|
||||
#### Version 1.24.0 (2019-10-09)
|
||||
|
||||
New Features:
|
||||
- Add sticky partition assignor
|
||||
([1416](https://github.com/Shopify/sarama/pull/1416)).
|
||||
- Switch from cgo zstd package to pure Go implementation
|
||||
([1477](https://github.com/Shopify/sarama/pull/1477)).
|
||||
|
||||
Improvements:
|
||||
- Allow creating ClusterAdmin from client
|
||||
([1415](https://github.com/Shopify/sarama/pull/1415)).
|
||||
- Set KafkaVersion in ListAcls method
|
||||
([1452](https://github.com/Shopify/sarama/pull/1452)).
|
||||
- Set request version in CreateACL ClusterAdmin method
|
||||
([1458](https://github.com/Shopify/sarama/pull/1458)).
|
||||
- Set request version in DeleteACL ClusterAdmin method
|
||||
([1461](https://github.com/Shopify/sarama/pull/1461)).
|
||||
- Handle missed error codes on TopicMetaDataRequest and GroupCoordinatorRequest
|
||||
([1464](https://github.com/Shopify/sarama/pull/1464)).
|
||||
- Remove direct usage of gofork
|
||||
([1465](https://github.com/Shopify/sarama/pull/1465)).
|
||||
- Add support for Go 1.13
|
||||
([1478](https://github.com/Shopify/sarama/pull/1478)).
|
||||
- Improve behavior of NewMockListAclsResponse
|
||||
([1481](https://github.com/Shopify/sarama/pull/1481)).
|
||||
|
||||
Bug Fixes:
|
||||
- Fix race condition in consumergroup example
|
||||
([1434](https://github.com/Shopify/sarama/pull/1434)).
|
||||
- Fix brokerProducer goroutine leak
|
||||
([1442](https://github.com/Shopify/sarama/pull/1442)).
|
||||
- Use released version of lz4 library
|
||||
([1469](https://github.com/Shopify/sarama/pull/1469)).
|
||||
- Set correct version in MockDeleteTopicsResponse
|
||||
([1484](https://github.com/Shopify/sarama/pull/1484)).
|
||||
- Fix CLI help message typo
|
||||
([1494](https://github.com/Shopify/sarama/pull/1494)).
|
||||
|
||||
Known Issues:
|
||||
- Please **don't** use Zstd, as it doesn't work right now.
|
||||
See https://github.com/Shopify/sarama/issues/1252
|
||||
|
||||
#### Version 1.23.1 (2019-07-22)
|
||||
|
||||
Bug Fixes:
|
||||
- Fix fetch delete bug record
|
||||
([1425](https://github.com/Shopify/sarama/pull/1425)).
|
||||
- Handle SASL/OAUTHBEARER token rejection
|
||||
([1428](https://github.com/Shopify/sarama/pull/1428)).
|
||||
|
||||
#### Version 1.23.0 (2019-07-02)
|
||||
|
||||
New Features:
|
||||
- Add support for Kafka 2.3.0
|
||||
([1418](https://github.com/Shopify/sarama/pull/1418)).
|
||||
- Add support for ListConsumerGroupOffsets v2
|
||||
([1374](https://github.com/Shopify/sarama/pull/1374)).
|
||||
- Add support for DeleteConsumerGroup
|
||||
([1417](https://github.com/Shopify/sarama/pull/1417)).
|
||||
- Add support for SASLVersion configuration
|
||||
([1410](https://github.com/Shopify/sarama/pull/1410)).
|
||||
- Add kerberos support
|
||||
([1366](https://github.com/Shopify/sarama/pull/1366)).
|
||||
|
||||
Improvements:
|
||||
- Improve sasl_scram_client example
|
||||
([1406](https://github.com/Shopify/sarama/pull/1406)).
|
||||
- Fix shutdown and race-condition in consumer-group example
|
||||
([1404](https://github.com/Shopify/sarama/pull/1404)).
|
||||
- Add support for error codes 77—81
|
||||
([1397](https://github.com/Shopify/sarama/pull/1397)).
|
||||
- Pool internal objects allocated per message
|
||||
([1385](https://github.com/Shopify/sarama/pull/1385)).
|
||||
- Reduce packet decoder allocations
|
||||
([1373](https://github.com/Shopify/sarama/pull/1373)).
|
||||
- Support timeout when fetching metadata
|
||||
([1359](https://github.com/Shopify/sarama/pull/1359)).
|
||||
|
||||
Bug Fixes:
|
||||
- Fix fetch size integer overflow
|
||||
([1376](https://github.com/Shopify/sarama/pull/1376)).
|
||||
- Handle and log throttled FetchResponses
|
||||
([1383](https://github.com/Shopify/sarama/pull/1383)).
|
||||
- Refactor misspelled word Resouce to Resource
|
||||
([1368](https://github.com/Shopify/sarama/pull/1368)).
|
||||
|
||||
#### Version 1.22.1 (2019-04-29)
|
||||
|
||||
Improvements:
|
||||
- Use zstd 1.3.8
|
||||
([1350](https://github.com/Shopify/sarama/pull/1350)).
|
||||
- Add support for SaslHandshakeRequest v1
|
||||
([1354](https://github.com/Shopify/sarama/pull/1354)).
|
||||
|
||||
Bug Fixes:
|
||||
- Fix V5 MetadataRequest nullable topics array
|
||||
([1353](https://github.com/Shopify/sarama/pull/1353)).
|
||||
- Use a different SCRAM client for each broker connection
|
||||
([1349](https://github.com/Shopify/sarama/pull/1349)).
|
||||
- Fix AllowAutoTopicCreation for MetadataRequest greater than v3
|
||||
([1344](https://github.com/Shopify/sarama/pull/1344)).
|
||||
|
||||
#### Version 1.22.0 (2019-04-09)
|
||||
|
||||
New Features:
|
||||
- Add Offline Replicas Operation to Client
|
||||
([1318](https://github.com/Shopify/sarama/pull/1318)).
|
||||
- Allow using proxy when connecting to broker
|
||||
([1326](https://github.com/Shopify/sarama/pull/1326)).
|
||||
- Implement ReadCommitted
|
||||
([1307](https://github.com/Shopify/sarama/pull/1307)).
|
||||
- Add support for Kafka 2.2.0
|
||||
([1331](https://github.com/Shopify/sarama/pull/1331)).
|
||||
- Add SASL SCRAM-SHA-512 and SCRAM-SHA-256 mechanismes
|
||||
([1331](https://github.com/Shopify/sarama/pull/1295)).
|
||||
|
||||
Improvements:
|
||||
- Unregister all broker metrics on broker stop
|
||||
([1232](https://github.com/Shopify/sarama/pull/1232)).
|
||||
- Add SCRAM authentication example
|
||||
([1303](https://github.com/Shopify/sarama/pull/1303)).
|
||||
- Add consumergroup examples
|
||||
([1304](https://github.com/Shopify/sarama/pull/1304)).
|
||||
- Expose consumer batch size metric
|
||||
([1296](https://github.com/Shopify/sarama/pull/1296)).
|
||||
- Add TLS options to console producer and consumer
|
||||
([1300](https://github.com/Shopify/sarama/pull/1300)).
|
||||
- Reduce client close bookkeeping
|
||||
([1297](https://github.com/Shopify/sarama/pull/1297)).
|
||||
- Satisfy error interface in create responses
|
||||
([1154](https://github.com/Shopify/sarama/pull/1154)).
|
||||
- Please lint gods
|
||||
([1346](https://github.com/Shopify/sarama/pull/1346)).
|
||||
|
||||
Bug Fixes:
|
||||
- Fix multi consumer group instance crash
|
||||
([1338](https://github.com/Shopify/sarama/pull/1338)).
|
||||
- Update lz4 to latest version
|
||||
([1347](https://github.com/Shopify/sarama/pull/1347)).
|
||||
- Retry ErrNotCoordinatorForConsumer in new consumergroup session
|
||||
([1231](https://github.com/Shopify/sarama/pull/1231)).
|
||||
- Fix cleanup error handler
|
||||
([1332](https://github.com/Shopify/sarama/pull/1332)).
|
||||
- Fix rate condition in PartitionConsumer
|
||||
([1156](https://github.com/Shopify/sarama/pull/1156)).
|
||||
|
||||
#### Version 1.21.0 (2019-02-24)
|
||||
|
||||
New Features:
|
||||
- Add CreateAclRequest, DescribeAclRequest, DeleteAclRequest
|
||||
([1236](https://github.com/Shopify/sarama/pull/1236)).
|
||||
- Add DescribeTopic, DescribeConsumerGroup, ListConsumerGroups, ListConsumerGroupOffsets admin requests
|
||||
([1178](https://github.com/Shopify/sarama/pull/1178)).
|
||||
- Implement SASL/OAUTHBEARER
|
||||
([1240](https://github.com/Shopify/sarama/pull/1240)).
|
||||
|
||||
Improvements:
|
||||
- Add Go mod support
|
||||
([1282](https://github.com/Shopify/sarama/pull/1282)).
|
||||
- Add error codes 73—76
|
||||
([1239](https://github.com/Shopify/sarama/pull/1239)).
|
||||
- Add retry backoff function
|
||||
([1160](https://github.com/Shopify/sarama/pull/1160)).
|
||||
- Maintain metadata in the producer even when retries are disabled
|
||||
([1189](https://github.com/Shopify/sarama/pull/1189)).
|
||||
- Include ReplicaAssignment in ListTopics
|
||||
([1274](https://github.com/Shopify/sarama/pull/1274)).
|
||||
- Add producer performance tool
|
||||
([1222](https://github.com/Shopify/sarama/pull/1222)).
|
||||
- Add support LogAppend timestamps
|
||||
([1258](https://github.com/Shopify/sarama/pull/1258)).
|
||||
|
||||
Bug Fixes:
|
||||
- Fix potential deadlock when a heartbeat request fails
|
||||
([1286](https://github.com/Shopify/sarama/pull/1286)).
|
||||
- Fix consuming compacted topic
|
||||
([1227](https://github.com/Shopify/sarama/pull/1227)).
|
||||
- Set correct Kafka version for DescribeConfigsRequest v1
|
||||
([1277](https://github.com/Shopify/sarama/pull/1277)).
|
||||
- Update kafka test version
|
||||
([1273](https://github.com/Shopify/sarama/pull/1273)).
|
||||
|
||||
#### Version 1.20.1 (2019-01-10)
|
||||
|
||||
New Features:
|
||||
- Add optional replica id in offset request
|
||||
([1100](https://github.com/Shopify/sarama/pull/1100)).
|
||||
|
||||
Improvements:
|
||||
- Implement DescribeConfigs Request + Response v1 & v2
|
||||
([1230](https://github.com/Shopify/sarama/pull/1230)).
|
||||
- Reuse compression objects
|
||||
([1185](https://github.com/Shopify/sarama/pull/1185)).
|
||||
- Switch from png to svg for GoDoc link in README
|
||||
([1243](https://github.com/Shopify/sarama/pull/1243)).
|
||||
- Fix typo in deprecation notice for FetchResponseBlock.Records
|
||||
([1242](https://github.com/Shopify/sarama/pull/1242)).
|
||||
- Fix typos in consumer metadata response file
|
||||
([1244](https://github.com/Shopify/sarama/pull/1244)).
|
||||
|
||||
Bug Fixes:
|
||||
- Revert to individual msg retries for non-idempotent
|
||||
([1203](https://github.com/Shopify/sarama/pull/1203)).
|
||||
- Respect MaxMessageBytes limit for uncompressed messages
|
||||
([1141](https://github.com/Shopify/sarama/pull/1141)).
|
||||
|
||||
#### Version 1.20.0 (2018-12-10)
|
||||
|
||||
New Features:
|
||||
- Add support for zstd compression
|
||||
([#1170](https://github.com/Shopify/sarama/pull/1170)).
|
||||
- Add support for Idempotent Producer
|
||||
([#1152](https://github.com/Shopify/sarama/pull/1152)).
|
||||
- Add support support for Kafka 2.1.0
|
||||
([#1229](https://github.com/Shopify/sarama/pull/1229)).
|
||||
- Add support support for OffsetCommit request/response pairs versions v1 to v5
|
||||
([#1201](https://github.com/Shopify/sarama/pull/1201)).
|
||||
- Add support support for OffsetFetch request/response pair up to version v5
|
||||
([#1198](https://github.com/Shopify/sarama/pull/1198)).
|
||||
|
||||
Improvements:
|
||||
- Export broker's Rack setting
|
||||
([#1173](https://github.com/Shopify/sarama/pull/1173)).
|
||||
- Always use latest patch version of Go on CI
|
||||
([#1202](https://github.com/Shopify/sarama/pull/1202)).
|
||||
- Add error codes 61 to 72
|
||||
([#1195](https://github.com/Shopify/sarama/pull/1195)).
|
||||
|
||||
Bug Fixes:
|
||||
- Fix build without cgo
|
||||
([#1182](https://github.com/Shopify/sarama/pull/1182)).
|
||||
- Fix go vet suggestion in consumer group file
|
||||
([#1209](https://github.com/Shopify/sarama/pull/1209)).
|
||||
- Fix typos in code and comments
|
||||
([#1228](https://github.com/Shopify/sarama/pull/1228)).
|
||||
|
||||
#### Version 1.19.0 (2018-09-27)
|
||||
|
||||
New Features:
|
||||
- Implement a higher-level consumer group
|
||||
([#1099](https://github.com/Shopify/sarama/pull/1099)).
|
||||
|
||||
Improvements:
|
||||
- Add support for Go 1.11
|
||||
([#1176](https://github.com/Shopify/sarama/pull/1176)).
|
||||
|
||||
Bug Fixes:
|
||||
- Fix encoding of `MetadataResponse` with version 2 and higher
|
||||
([#1174](https://github.com/Shopify/sarama/pull/1174)).
|
||||
- Fix race condition in mock async producer
|
||||
([#1174](https://github.com/Shopify/sarama/pull/1174)).
|
||||
|
||||
#### Version 1.18.0 (2018-09-07)
|
||||
|
||||
New Features:
|
||||
- Make `Partitioner.RequiresConsistency` vary per-message
|
||||
([#1112](https://github.com/Shopify/sarama/pull/1112)).
|
||||
- Add customizable partitioner
|
||||
([#1118](https://github.com/Shopify/sarama/pull/1118)).
|
||||
- Add `ClusterAdmin` support for `CreateTopic`, `DeleteTopic`, `CreatePartitions`,
|
||||
`DeleteRecords`, `DescribeConfig`, `AlterConfig`, `CreateACL`, `ListAcls`, `DeleteACL`
|
||||
([#1055](https://github.com/Shopify/sarama/pull/1055)).
|
||||
|
||||
Improvements:
|
||||
- Add support for Kafka 2.0.0
|
||||
([#1149](https://github.com/Shopify/sarama/pull/1149)).
|
||||
- Allow setting `LocalAddr` when dialing an address to support multi-homed hosts
|
||||
([#1123](https://github.com/Shopify/sarama/pull/1123)).
|
||||
- Simpler offset management
|
||||
([#1127](https://github.com/Shopify/sarama/pull/1127)).
|
||||
|
||||
Bug Fixes:
|
||||
- Fix mutation of `ProducerMessage.MetaData` when producing to Kafka
|
||||
([#1110](https://github.com/Shopify/sarama/pull/1110)).
|
||||
- Fix consumer block when response did not contain all the
|
||||
expected topic/partition blocks
|
||||
([#1086](https://github.com/Shopify/sarama/pull/1086)).
|
||||
- Fix consumer block when response contains only constrol messages
|
||||
([#1115](https://github.com/Shopify/sarama/pull/1115)).
|
||||
- Add timeout config for ClusterAdmin requests
|
||||
([#1142](https://github.com/Shopify/sarama/pull/1142)).
|
||||
- Add version check when producing message with headers
|
||||
([#1117](https://github.com/Shopify/sarama/pull/1117)).
|
||||
- Fix `MetadataRequest` for empty list of topics
|
||||
([#1132](https://github.com/Shopify/sarama/pull/1132)).
|
||||
- Fix producer topic metadata on-demand fetch when topic error happens in metadata response
|
||||
([#1125](https://github.com/Shopify/sarama/pull/1125)).
|
||||
|
||||
#### Version 1.17.0 (2018-05-30)
|
||||
|
||||
New Features:
|
||||
- Add support for gzip compression levels
|
||||
([#1044](https://github.com/Shopify/sarama/pull/1044)).
|
||||
- Add support for Metadata request/response pairs versions v1 to v5
|
||||
([#1047](https://github.com/Shopify/sarama/pull/1047),
|
||||
[#1069](https://github.com/Shopify/sarama/pull/1069)).
|
||||
- Add versioning to JoinGroup request/response pairs
|
||||
([#1098](https://github.com/Shopify/sarama/pull/1098))
|
||||
- Add support for CreatePartitions, DeleteGroups, DeleteRecords request/response pairs
|
||||
([#1065](https://github.com/Shopify/sarama/pull/1065),
|
||||
[#1096](https://github.com/Shopify/sarama/pull/1096),
|
||||
[#1027](https://github.com/Shopify/sarama/pull/1027)).
|
||||
- Add `Controller()` method to Client interface
|
||||
([#1063](https://github.com/Shopify/sarama/pull/1063)).
|
||||
|
||||
Improvements:
|
||||
- ConsumerMetadataReq/Resp has been migrated to FindCoordinatorReq/Resp
|
||||
([#1010](https://github.com/Shopify/sarama/pull/1010)).
|
||||
- Expose missing protocol parts: `msgSet` and `recordBatch`
|
||||
([#1049](https://github.com/Shopify/sarama/pull/1049)).
|
||||
- Add support for v1 DeleteTopics Request
|
||||
([#1052](https://github.com/Shopify/sarama/pull/1052)).
|
||||
- Add support for Go 1.10
|
||||
([#1064](https://github.com/Shopify/sarama/pull/1064)).
|
||||
- Claim support for Kafka 1.1.0
|
||||
([#1073](https://github.com/Shopify/sarama/pull/1073)).
|
||||
|
||||
Bug Fixes:
|
||||
- Fix FindCoordinatorResponse.encode to allow nil Coordinator
|
||||
([#1050](https://github.com/Shopify/sarama/pull/1050),
|
||||
[#1051](https://github.com/Shopify/sarama/pull/1051)).
|
||||
- Clear all metadata when we have the latest topic info
|
||||
([#1033](https://github.com/Shopify/sarama/pull/1033)).
|
||||
- Make `PartitionConsumer.Close` idempotent
|
||||
([#1092](https://github.com/Shopify/sarama/pull/1092)).
|
||||
|
||||
#### Version 1.16.0 (2018-02-12)
|
||||
|
||||
New Features:
|
||||
- Add support for the Create/Delete Topics request/response pairs
|
||||
([#1007](https://github.com/Shopify/sarama/pull/1007),
|
||||
[#1008](https://github.com/Shopify/sarama/pull/1008)).
|
||||
- Add support for the Describe/Create/Delete ACL request/response pairs
|
||||
([#1009](https://github.com/Shopify/sarama/pull/1009)).
|
||||
- Add support for the five transaction-related request/response pairs
|
||||
([#1016](https://github.com/Shopify/sarama/pull/1016)).
|
||||
|
||||
Improvements:
|
||||
- Permit setting version on mock producer responses
|
||||
([#999](https://github.com/Shopify/sarama/pull/999)).
|
||||
- Add `NewMockBrokerListener` helper for testing TLS connections
|
||||
([#1019](https://github.com/Shopify/sarama/pull/1019)).
|
||||
- Changed the default value for `Consumer.Fetch.Default` from 32KiB to 1MiB
|
||||
which results in much higher throughput in most cases
|
||||
([#1024](https://github.com/Shopify/sarama/pull/1024)).
|
||||
- Reuse the `time.Ticker` across fetch requests in the PartitionConsumer to
|
||||
reduce CPU and memory usage when processing many partitions
|
||||
([#1028](https://github.com/Shopify/sarama/pull/1028)).
|
||||
- Assign relative offsets to messages in the producer to save the brokers a
|
||||
recompression pass
|
||||
([#1002](https://github.com/Shopify/sarama/pull/1002),
|
||||
[#1015](https://github.com/Shopify/sarama/pull/1015)).
|
||||
|
||||
Bug Fixes:
|
||||
- Fix producing uncompressed batches with the new protocol format
|
||||
([#1032](https://github.com/Shopify/sarama/issues/1032)).
|
||||
- Fix consuming compacted topics with the new protocol format
|
||||
([#1005](https://github.com/Shopify/sarama/issues/1005)).
|
||||
- Fix consuming topics with a mix of protocol formats
|
||||
([#1021](https://github.com/Shopify/sarama/issues/1021)).
|
||||
- Fix consuming when the broker includes multiple batches in a single response
|
||||
([#1022](https://github.com/Shopify/sarama/issues/1022)).
|
||||
- Fix detection of `PartialTrailingMessage` when the partial message was
|
||||
truncated before the magic value indicating its version
|
||||
([#1030](https://github.com/Shopify/sarama/pull/1030)).
|
||||
- Fix expectation-checking in the mock of `SyncProducer.SendMessages`
|
||||
([#1035](https://github.com/Shopify/sarama/pull/1035)).
|
||||
|
||||
#### Version 1.15.0 (2017-12-08)
|
||||
|
||||
New Features:
|
||||
- Claim official support for Kafka 1.0, though it did already work
|
||||
([#984](https://github.com/Shopify/sarama/pull/984)).
|
||||
- Helper methods for Kafka version numbers to/from strings
|
||||
([#989](https://github.com/Shopify/sarama/pull/989)).
|
||||
- Implement CreatePartitions request/response
|
||||
([#985](https://github.com/Shopify/sarama/pull/985)).
|
||||
|
||||
Improvements:
|
||||
- Add error codes 45-60
|
||||
([#986](https://github.com/Shopify/sarama/issues/986)).
|
||||
|
||||
Bug Fixes:
|
||||
- Fix slow consuming for certain Kafka 0.11/1.0 configurations
|
||||
([#982](https://github.com/Shopify/sarama/pull/982)).
|
||||
- Correctly determine when a FetchResponse contains the new message format
|
||||
([#990](https://github.com/Shopify/sarama/pull/990)).
|
||||
- Fix producing with multiple headers
|
||||
([#996](https://github.com/Shopify/sarama/pull/996)).
|
||||
- Fix handling of truncated record batches
|
||||
([#998](https://github.com/Shopify/sarama/pull/998)).
|
||||
- Fix leaking metrics when closing brokers
|
||||
([#991](https://github.com/Shopify/sarama/pull/991)).
|
||||
|
||||
#### Version 1.14.0 (2017-11-13)
|
||||
|
||||
New Features:
|
||||
- Add support for the new Kafka 0.11 record-batch format, including the wire
|
||||
protocol and the necessary behavioural changes in the producer and consumer.
|
||||
Transactions and idempotency are not yet supported, but producing and
|
||||
consuming should work with all the existing bells and whistles (batching,
|
||||
compression, etc) as well as the new custom headers. Thanks to Vlad Hanciuta
|
||||
of Arista Networks for this work. Part of
|
||||
([#901](https://github.com/Shopify/sarama/issues/901)).
|
||||
|
||||
Bug Fixes:
|
||||
- Fix encoding of ProduceResponse versions in test
|
||||
([#970](https://github.com/Shopify/sarama/pull/970)).
|
||||
- Return partial replicas list when we have it
|
||||
([#975](https://github.com/Shopify/sarama/pull/975)).
|
||||
|
||||
#### Version 1.13.0 (2017-10-04)
|
||||
|
||||
New Features:
|
||||
- Support for FetchRequest version 3
|
||||
([#905](https://github.com/Shopify/sarama/pull/905)).
|
||||
- Permit setting version on mock FetchResponses
|
||||
([#939](https://github.com/Shopify/sarama/pull/939)).
|
||||
- Add a configuration option to support storing only minimal metadata for
|
||||
extremely large clusters
|
||||
([#937](https://github.com/Shopify/sarama/pull/937)).
|
||||
- Add `PartitionOffsetManager.ResetOffset` for backtracking tracked offsets
|
||||
([#932](https://github.com/Shopify/sarama/pull/932)).
|
||||
|
||||
Improvements:
|
||||
- Provide the block-level timestamp when consuming compressed messages
|
||||
([#885](https://github.com/Shopify/sarama/issues/885)).
|
||||
- `Client.Replicas` and `Client.InSyncReplicas` now respect the order returned
|
||||
by the broker, which can be meaningful
|
||||
([#930](https://github.com/Shopify/sarama/pull/930)).
|
||||
- Use a `Ticker` to reduce consumer timer overhead at the cost of higher
|
||||
variance in the actual timeout
|
||||
([#933](https://github.com/Shopify/sarama/pull/933)).
|
||||
|
||||
Bug Fixes:
|
||||
- Gracefully handle messages with negative timestamps
|
||||
([#907](https://github.com/Shopify/sarama/pull/907)).
|
||||
- Raise a proper error when encountering an unknown message version
|
||||
([#940](https://github.com/Shopify/sarama/pull/940)).
|
||||
|
||||
#### Version 1.12.0 (2017-05-08)
|
||||
|
||||
New Features:
|
||||
- Added support for the `ApiVersions` request and response pair, and Kafka
|
||||
version 0.10.2 ([#867](https://github.com/Shopify/sarama/pull/867)). Note
|
||||
that you still need to specify the Kafka version in the Sarama configuration
|
||||
for the time being.
|
||||
- Added a `Brokers` method to the Client which returns the complete set of
|
||||
active brokers ([#813](https://github.com/Shopify/sarama/pull/813)).
|
||||
- Added an `InSyncReplicas` method to the Client which returns the set of all
|
||||
in-sync broker IDs for the given partition, now that the Kafka versions for
|
||||
which this was misleading are no longer in our supported set
|
||||
([#872](https://github.com/Shopify/sarama/pull/872)).
|
||||
- Added a `NewCustomHashPartitioner` method which allows constructing a hash
|
||||
partitioner with a custom hash method in case the default (FNV-1a) is not
|
||||
suitable
|
||||
([#837](https://github.com/Shopify/sarama/pull/837),
|
||||
[#841](https://github.com/Shopify/sarama/pull/841)).
|
||||
|
||||
Improvements:
|
||||
- Recognize more Kafka error codes
|
||||
([#859](https://github.com/Shopify/sarama/pull/859)).
|
||||
|
||||
Bug Fixes:
|
||||
- Fix an issue where decoding a malformed FetchRequest would not return the
|
||||
correct error ([#818](https://github.com/Shopify/sarama/pull/818)).
|
||||
- Respect ordering of group protocols in JoinGroupRequests. This fix is
|
||||
transparent if you're using the `AddGroupProtocol` or
|
||||
`AddGroupProtocolMetadata` helpers; otherwise you will need to switch from
|
||||
the `GroupProtocols` field (now deprecated) to use `OrderedGroupProtocols`
|
||||
([#812](https://github.com/Shopify/sarama/issues/812)).
|
||||
- Fix an alignment-related issue with atomics on 32-bit architectures
|
||||
([#859](https://github.com/Shopify/sarama/pull/859)).
|
||||
|
||||
#### Version 1.11.0 (2016-12-20)
|
||||
|
||||
_Important:_ As of Sarama 1.11 it is necessary to set the config value of
|
||||
`Producer.Return.Successes` to true in order to use the SyncProducer. Previous
|
||||
versions would silently override this value when instantiating a SyncProducer
|
||||
which led to unexpected values and data races.
|
||||
|
||||
New Features:
|
||||
- Metrics! Thanks to Sébastien Launay for all his work on this feature
|
||||
([#701](https://github.com/Shopify/sarama/pull/701),
|
||||
[#746](https://github.com/Shopify/sarama/pull/746),
|
||||
[#766](https://github.com/Shopify/sarama/pull/766)).
|
||||
- Add support for LZ4 compression
|
||||
([#786](https://github.com/Shopify/sarama/pull/786)).
|
||||
- Add support for ListOffsetRequest v1 and Kafka 0.10.1
|
||||
([#775](https://github.com/Shopify/sarama/pull/775)).
|
||||
- Added a `HighWaterMarks` method to the Consumer which aggregates the
|
||||
`HighWaterMarkOffset` values of its child topic/partitions
|
||||
([#769](https://github.com/Shopify/sarama/pull/769)).
|
||||
|
||||
Bug Fixes:
|
||||
- Fixed producing when using timestamps, compression and Kafka 0.10
|
||||
([#759](https://github.com/Shopify/sarama/pull/759)).
|
||||
- Added missing decoder methods to DescribeGroups response
|
||||
([#756](https://github.com/Shopify/sarama/pull/756)).
|
||||
- Fix producer shutdown when `Return.Errors` is disabled
|
||||
([#787](https://github.com/Shopify/sarama/pull/787)).
|
||||
- Don't mutate configuration in SyncProducer
|
||||
([#790](https://github.com/Shopify/sarama/pull/790)).
|
||||
- Fix crash on SASL initialization failure
|
||||
([#795](https://github.com/Shopify/sarama/pull/795)).
|
||||
|
||||
#### Version 1.10.1 (2016-08-30)
|
||||
|
||||
Bug Fixes:
|
||||
- Fix the documentation for `HashPartitioner` which was incorrect
|
||||
([#717](https://github.com/Shopify/sarama/pull/717)).
|
||||
- Permit client creation even when it is limited by ACLs
|
||||
([#722](https://github.com/Shopify/sarama/pull/722)).
|
||||
- Several fixes to the consumer timer optimization code, regressions introduced
|
||||
in v1.10.0. Go's timers are finicky
|
||||
([#730](https://github.com/Shopify/sarama/pull/730),
|
||||
[#733](https://github.com/Shopify/sarama/pull/733),
|
||||
[#734](https://github.com/Shopify/sarama/pull/734)).
|
||||
- Handle consuming compressed relative offsets with Kafka 0.10
|
||||
([#735](https://github.com/Shopify/sarama/pull/735)).
|
||||
|
||||
#### Version 1.10.0 (2016-08-02)
|
||||
|
||||
_Important:_ As of Sarama 1.10 it is necessary to tell Sarama the version of
|
||||
Kafka you are running against (via the `config.Version` value) in order to use
|
||||
features that may not be compatible with old Kafka versions. If you don't
|
||||
specify this value it will default to 0.8.2 (the minimum supported), and trying
|
||||
to use more recent features (like the offset manager) will fail with an error.
|
||||
|
||||
_Also:_ The offset-manager's behaviour has been changed to match the upstream
|
||||
java consumer (see [#705](https://github.com/Shopify/sarama/pull/705) and
|
||||
[#713](https://github.com/Shopify/sarama/pull/713)). If you use the
|
||||
offset-manager, please ensure that you are committing one *greater* than the
|
||||
last consumed message offset or else you may end up consuming duplicate
|
||||
messages.
|
||||
|
||||
New Features:
|
||||
- Support for Kafka 0.10
|
||||
([#672](https://github.com/Shopify/sarama/pull/672),
|
||||
[#678](https://github.com/Shopify/sarama/pull/678),
|
||||
[#681](https://github.com/Shopify/sarama/pull/681), and others).
|
||||
- Support for configuring the target Kafka version
|
||||
([#676](https://github.com/Shopify/sarama/pull/676)).
|
||||
- Batch producing support in the SyncProducer
|
||||
([#677](https://github.com/Shopify/sarama/pull/677)).
|
||||
- Extend producer mock to allow setting expectations on message contents
|
||||
([#667](https://github.com/Shopify/sarama/pull/667)).
|
||||
|
||||
Improvements:
|
||||
- Support `nil` compressed messages for deleting in compacted topics
|
||||
([#634](https://github.com/Shopify/sarama/pull/634)).
|
||||
- Pre-allocate decoding errors, greatly reducing heap usage and GC time against
|
||||
misbehaving brokers ([#690](https://github.com/Shopify/sarama/pull/690)).
|
||||
- Re-use consumer expiry timers, removing one allocation per consumed message
|
||||
([#707](https://github.com/Shopify/sarama/pull/707)).
|
||||
|
||||
Bug Fixes:
|
||||
- Actually default the client ID to "sarama" like we say we do
|
||||
([#664](https://github.com/Shopify/sarama/pull/664)).
|
||||
- Fix a rare issue where `Client.Leader` could return the wrong error
|
||||
([#685](https://github.com/Shopify/sarama/pull/685)).
|
||||
- Fix a possible tight loop in the consumer
|
||||
([#693](https://github.com/Shopify/sarama/pull/693)).
|
||||
- Match upstream's offset-tracking behaviour
|
||||
([#705](https://github.com/Shopify/sarama/pull/705)).
|
||||
- Report UnknownTopicOrPartition errors from the offset manager
|
||||
([#706](https://github.com/Shopify/sarama/pull/706)).
|
||||
- Fix possible negative partition value from the HashPartitioner
|
||||
([#709](https://github.com/Shopify/sarama/pull/709)).
|
||||
|
||||
#### Version 1.9.0 (2016-05-16)
|
||||
|
||||
New Features:
|
||||
- Add support for custom offset manager retention durations
|
||||
([#602](https://github.com/Shopify/sarama/pull/602)).
|
||||
- Publish low-level mocks to enable testing of third-party producer/consumer
|
||||
implementations ([#570](https://github.com/Shopify/sarama/pull/570)).
|
||||
- Declare support for Golang 1.6
|
||||
([#611](https://github.com/Shopify/sarama/pull/611)).
|
||||
- Support for SASL plain-text auth
|
||||
([#648](https://github.com/Shopify/sarama/pull/648)).
|
||||
|
||||
Improvements:
|
||||
- Simplified broker locking scheme slightly
|
||||
([#604](https://github.com/Shopify/sarama/pull/604)).
|
||||
- Documentation cleanup
|
||||
([#605](https://github.com/Shopify/sarama/pull/605),
|
||||
[#621](https://github.com/Shopify/sarama/pull/621),
|
||||
[#654](https://github.com/Shopify/sarama/pull/654)).
|
||||
|
||||
Bug Fixes:
|
||||
- Fix race condition shutting down the OffsetManager
|
||||
([#658](https://github.com/Shopify/sarama/pull/658)).
|
||||
|
||||
#### Version 1.8.0 (2016-02-01)
|
||||
|
||||
New Features:
|
||||
- Full support for Kafka 0.9:
|
||||
- All protocol messages and fields
|
||||
([#586](https://github.com/Shopify/sarama/pull/586),
|
||||
[#588](https://github.com/Shopify/sarama/pull/588),
|
||||
[#590](https://github.com/Shopify/sarama/pull/590)).
|
||||
- Verified that TLS support works
|
||||
([#581](https://github.com/Shopify/sarama/pull/581)).
|
||||
- Fixed the OffsetManager compatibility
|
||||
([#585](https://github.com/Shopify/sarama/pull/585)).
|
||||
|
||||
Improvements:
|
||||
- Optimize for fewer system calls when reading from the network
|
||||
([#584](https://github.com/Shopify/sarama/pull/584)).
|
||||
- Automatically retry `InvalidMessage` errors to match upstream behaviour
|
||||
([#589](https://github.com/Shopify/sarama/pull/589)).
|
||||
|
||||
#### Version 1.7.0 (2015-12-11)
|
||||
|
||||
New Features:
|
||||
- Preliminary support for Kafka 0.9
|
||||
([#572](https://github.com/Shopify/sarama/pull/572)). This comes with several
|
||||
caveats:
|
||||
- Protocol-layer support is mostly in place
|
||||
([#577](https://github.com/Shopify/sarama/pull/577)), however Kafka 0.9
|
||||
renamed some messages and fields, which we did not in order to preserve API
|
||||
compatibility.
|
||||
- The producer and consumer work against 0.9, but the offset manager does
|
||||
not ([#573](https://github.com/Shopify/sarama/pull/573)).
|
||||
- TLS support may or may not work
|
||||
([#581](https://github.com/Shopify/sarama/pull/581)).
|
||||
|
||||
Improvements:
|
||||
- Don't wait for request timeouts on dead brokers, greatly speeding recovery
|
||||
when the TCP connection is left hanging
|
||||
([#548](https://github.com/Shopify/sarama/pull/548)).
|
||||
- Refactored part of the producer. The new version provides a much more elegant
|
||||
solution to [#449](https://github.com/Shopify/sarama/pull/449). It is also
|
||||
slightly more efficient, and much more precise in calculating batch sizes
|
||||
when compression is used
|
||||
([#549](https://github.com/Shopify/sarama/pull/549),
|
||||
[#550](https://github.com/Shopify/sarama/pull/550),
|
||||
[#551](https://github.com/Shopify/sarama/pull/551)).
|
||||
|
||||
Bug Fixes:
|
||||
- Fix race condition in consumer test mock
|
||||
([#553](https://github.com/Shopify/sarama/pull/553)).
|
||||
|
||||
#### Version 1.6.1 (2015-09-25)
|
||||
|
||||
Bug Fixes:
|
||||
- Fix panic that could occur if a user-supplied message value failed to encode
|
||||
([#449](https://github.com/Shopify/sarama/pull/449)).
|
||||
|
||||
#### Version 1.6.0 (2015-09-04)
|
||||
|
||||
New Features:
|
||||
- Implementation of a consumer offset manager using the APIs introduced in
|
||||
Kafka 0.8.2. The API is designed mainly for integration into a future
|
||||
high-level consumer, not for direct use, although it is *possible* to use it
|
||||
directly.
|
||||
([#461](https://github.com/Shopify/sarama/pull/461)).
|
||||
|
||||
Improvements:
|
||||
- CRC32 calculation is much faster on machines with SSE4.2 instructions,
|
||||
removing a major hotspot from most profiles
|
||||
([#255](https://github.com/Shopify/sarama/pull/255)).
|
||||
|
||||
Bug Fixes:
|
||||
- Make protocol decoding more robust against some malformed packets generated
|
||||
by go-fuzz ([#523](https://github.com/Shopify/sarama/pull/523),
|
||||
[#525](https://github.com/Shopify/sarama/pull/525)) or found in other ways
|
||||
([#528](https://github.com/Shopify/sarama/pull/528)).
|
||||
- Fix a potential race condition panic in the consumer on shutdown
|
||||
([#529](https://github.com/Shopify/sarama/pull/529)).
|
||||
|
||||
#### Version 1.5.0 (2015-08-17)
|
||||
|
||||
New Features:
|
||||
- TLS-encrypted network connections are now supported. This feature is subject
|
||||
to change when Kafka releases built-in TLS support, but for now this is
|
||||
enough to work with TLS-terminating proxies
|
||||
([#154](https://github.com/Shopify/sarama/pull/154)).
|
||||
|
||||
Improvements:
|
||||
- The consumer will not block if a single partition is not drained by the user;
|
||||
all other partitions will continue to consume normally
|
||||
([#485](https://github.com/Shopify/sarama/pull/485)).
|
||||
- Formatting of error strings has been much improved
|
||||
([#495](https://github.com/Shopify/sarama/pull/495)).
|
||||
- Internal refactoring of the producer for code cleanliness and to enable
|
||||
future work ([#300](https://github.com/Shopify/sarama/pull/300)).
|
||||
|
||||
Bug Fixes:
|
||||
- Fix a potential deadlock in the consumer on shutdown
|
||||
([#475](https://github.com/Shopify/sarama/pull/475)).
|
||||
|
||||
#### Version 1.4.3 (2015-07-21)
|
||||
|
||||
Bug Fixes:
|
||||
- Don't include the partitioner in the producer's "fetch partitions"
|
||||
circuit-breaker ([#466](https://github.com/Shopify/sarama/pull/466)).
|
||||
- Don't retry messages until the broker is closed when abandoning a broker in
|
||||
the producer ([#468](https://github.com/Shopify/sarama/pull/468)).
|
||||
- Update the import path for snappy-go, it has moved again and the API has
|
||||
changed slightly ([#486](https://github.com/Shopify/sarama/pull/486)).
|
||||
|
||||
#### Version 1.4.2 (2015-05-27)
|
||||
|
||||
Bug Fixes:
|
||||
- Update the import path for snappy-go, it has moved from google code to github
|
||||
([#456](https://github.com/Shopify/sarama/pull/456)).
|
||||
|
||||
#### Version 1.4.1 (2015-05-25)
|
||||
|
||||
Improvements:
|
||||
- Optimizations when decoding snappy messages, thanks to John Potocny
|
||||
([#446](https://github.com/Shopify/sarama/pull/446)).
|
||||
|
||||
Bug Fixes:
|
||||
- Fix hypothetical race conditions on producer shutdown
|
||||
([#450](https://github.com/Shopify/sarama/pull/450),
|
||||
[#451](https://github.com/Shopify/sarama/pull/451)).
|
||||
|
||||
#### Version 1.4.0 (2015-05-01)
|
||||
|
||||
New Features:
|
||||
- The consumer now implements `Topics()` and `Partitions()` methods to enable
|
||||
users to dynamically choose what topics/partitions to consume without
|
||||
instantiating a full client
|
||||
([#431](https://github.com/Shopify/sarama/pull/431)).
|
||||
- The partition-consumer now exposes the high water mark offset value returned
|
||||
by the broker via the `HighWaterMarkOffset()` method ([#339](https://github.com/Shopify/sarama/pull/339)).
|
||||
- Added a `kafka-console-consumer` tool capable of handling multiple
|
||||
partitions, and deprecated the now-obsolete `kafka-console-partitionConsumer`
|
||||
([#439](https://github.com/Shopify/sarama/pull/439),
|
||||
[#442](https://github.com/Shopify/sarama/pull/442)).
|
||||
|
||||
Improvements:
|
||||
- The producer's logging during retry scenarios is more consistent, more
|
||||
useful, and slightly less verbose
|
||||
([#429](https://github.com/Shopify/sarama/pull/429)).
|
||||
- The client now shuffles its initial list of seed brokers in order to prevent
|
||||
thundering herd on the first broker in the list
|
||||
([#441](https://github.com/Shopify/sarama/pull/441)).
|
||||
|
||||
Bug Fixes:
|
||||
- The producer now correctly manages its state if retries occur when it is
|
||||
shutting down, fixing several instances of confusing behaviour and at least
|
||||
one potential deadlock ([#419](https://github.com/Shopify/sarama/pull/419)).
|
||||
- The consumer now handles messages for different partitions asynchronously,
|
||||
making it much more resilient to specific user code ordering
|
||||
([#325](https://github.com/Shopify/sarama/pull/325)).
|
||||
|
||||
#### Version 1.3.0 (2015-04-16)
|
||||
|
||||
New Features:
|
||||
- The client now tracks consumer group coordinators using
|
||||
ConsumerMetadataRequests similar to how it tracks partition leadership using
|
||||
regular MetadataRequests ([#411](https://github.com/Shopify/sarama/pull/411)).
|
||||
This adds two methods to the client API:
|
||||
- `Coordinator(consumerGroup string) (*Broker, error)`
|
||||
- `RefreshCoordinator(consumerGroup string) error`
|
||||
|
||||
Improvements:
|
||||
- ConsumerMetadataResponses now automatically create a Broker object out of the
|
||||
ID/address/port combination for the Coordinator; accessing the fields
|
||||
individually has been deprecated
|
||||
([#413](https://github.com/Shopify/sarama/pull/413)).
|
||||
- Much improved handling of `OffsetOutOfRange` errors in the consumer.
|
||||
Consumers will fail to start if the provided offset is out of range
|
||||
([#418](https://github.com/Shopify/sarama/pull/418))
|
||||
and they will automatically shut down if the offset falls out of range
|
||||
([#424](https://github.com/Shopify/sarama/pull/424)).
|
||||
- Small performance improvement in encoding and decoding protocol messages
|
||||
([#427](https://github.com/Shopify/sarama/pull/427)).
|
||||
|
||||
Bug Fixes:
|
||||
- Fix a rare race condition in the client's background metadata refresher if
|
||||
it happens to be activated while the client is being closed
|
||||
([#422](https://github.com/Shopify/sarama/pull/422)).
|
||||
|
||||
#### Version 1.2.0 (2015-04-07)
|
||||
|
||||
Improvements:
|
||||
- The producer's behaviour when `Flush.Frequency` is set is now more intuitive
|
||||
([#389](https://github.com/Shopify/sarama/pull/389)).
|
||||
- The producer is now somewhat more memory-efficient during and after retrying
|
||||
messages due to an improved queue implementation
|
||||
([#396](https://github.com/Shopify/sarama/pull/396)).
|
||||
- The consumer produces much more useful logging output when leadership
|
||||
changes ([#385](https://github.com/Shopify/sarama/pull/385)).
|
||||
- The client's `GetOffset` method will now automatically refresh metadata and
|
||||
retry once in the event of stale information or similar
|
||||
([#394](https://github.com/Shopify/sarama/pull/394)).
|
||||
- Broker connections now have support for using TCP keepalives
|
||||
([#407](https://github.com/Shopify/sarama/issues/407)).
|
||||
|
||||
Bug Fixes:
|
||||
- The OffsetCommitRequest message now correctly implements all three possible
|
||||
API versions ([#390](https://github.com/Shopify/sarama/pull/390),
|
||||
[#400](https://github.com/Shopify/sarama/pull/400)).
|
||||
|
||||
#### Version 1.1.0 (2015-03-20)
|
||||
|
||||
Improvements:
|
||||
- Wrap the producer's partitioner call in a circuit-breaker so that repeatedly
|
||||
broken topics don't choke throughput
|
||||
([#373](https://github.com/Shopify/sarama/pull/373)).
|
||||
|
||||
Bug Fixes:
|
||||
- Fix the producer's internal reference counting in certain unusual scenarios
|
||||
([#367](https://github.com/Shopify/sarama/pull/367)).
|
||||
- Fix the consumer's internal reference counting in certain unusual scenarios
|
||||
([#369](https://github.com/Shopify/sarama/pull/369)).
|
||||
- Fix a condition where the producer's internal control messages could have
|
||||
gotten stuck ([#368](https://github.com/Shopify/sarama/pull/368)).
|
||||
- Fix an issue where invalid partition lists would be cached when asking for
|
||||
metadata for a non-existant topic ([#372](https://github.com/Shopify/sarama/pull/372)).
|
||||
|
||||
|
||||
#### Version 1.0.0 (2015-03-17)
|
||||
|
||||
Version 1.0.0 is the first tagged version, and is almost a complete rewrite. The primary differences with previous untagged versions are:
|
||||
|
||||
- The producer has been rewritten; there is now a `SyncProducer` with a blocking API, and an `AsyncProducer` that is non-blocking.
|
||||
- The consumer has been rewritten to only open one connection per broker instead of one connection per partition.
|
||||
- The main types of Sarama are now interfaces to make depedency injection easy; mock implementations for `Consumer`, `SyncProducer` and `AsyncProducer` are provided in the `github.com/Shopify/sarama/mocks` package.
|
||||
- For most uses cases, it is no longer necessary to open a `Client`; this will be done for you.
|
||||
- All the configuration values have been unified in the `Config` struct.
|
||||
- Much improved test suite.
|
||||
-20
@@ -1,20 +0,0 @@
|
||||
Copyright (c) 2013 Shopify
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining
|
||||
a copy of this software and associated documentation files (the
|
||||
"Software"), to deal in the Software without restriction, including
|
||||
without limitation the rights to use, copy, modify, merge, publish,
|
||||
distribute, sublicense, and/or sell copies of the Software, and to
|
||||
permit persons to whom the Software is furnished to do so, subject to
|
||||
the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be
|
||||
included in all copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
||||
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
||||
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
||||
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
||||
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
||||
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
||||
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
-27
@@ -1,27 +0,0 @@
|
||||
default: fmt get update test lint
|
||||
|
||||
GO := GO111MODULE=on GOPRIVATE=github.com/linkedin GOSUMDB=off go
|
||||
GOBUILD := CGO_ENABLED=0 $(GO) build $(BUILD_FLAG)
|
||||
GOTEST := $(GO) test -gcflags='-l' -p 3 -v -race -timeout 6m -coverprofile=profile.out -covermode=atomic
|
||||
|
||||
FILES := $(shell find . -name '*.go' -type f -not -name '*.pb.go' -not -name '*_generated.go' -not -name '*_test.go')
|
||||
TESTS := $(shell find . -name '*.go' -type f -not -name '*.pb.go' -not -name '*_generated.go' -name '*_test.go')
|
||||
|
||||
get:
|
||||
$(GO) get ./...
|
||||
$(GO) mod verify
|
||||
$(GO) mod tidy
|
||||
|
||||
update:
|
||||
$(GO) get -u -v all
|
||||
$(GO) mod verify
|
||||
$(GO) mod tidy
|
||||
|
||||
fmt:
|
||||
gofmt -s -l -w $(FILES) $(TESTS)
|
||||
|
||||
lint:
|
||||
golangci-lint run
|
||||
|
||||
test:
|
||||
$(GOTEST) ./...
|
||||
-36
@@ -1,36 +0,0 @@
|
||||
# sarama
|
||||
|
||||
[](https://godoc.org/github.com/Shopify/sarama)
|
||||
[](https://travis-ci.org/Shopify/sarama)
|
||||
[](https://codecov.io/gh/Shopify/sarama)
|
||||
|
||||
Sarama is an MIT-licensed Go client library for [Apache Kafka](https://kafka.apache.org/) version 0.8 (and later).
|
||||
|
||||
## Getting started
|
||||
|
||||
- API documentation and examples are available via [godoc](https://godoc.org/github.com/Shopify/sarama).
|
||||
- Mocks for testing are available in the [mocks](./mocks) subpackage.
|
||||
- The [examples](./examples) directory contains more elaborate example applications.
|
||||
- The [tools](./tools) directory contains command line tools that can be useful for testing, diagnostics, and instrumentation.
|
||||
|
||||
You might also want to look at the [Frequently Asked Questions](https://github.com/Shopify/sarama/wiki/Frequently-Asked-Questions).
|
||||
|
||||
## Compatibility and API stability
|
||||
|
||||
Sarama provides a "2 releases + 2 months" compatibility guarantee: we support
|
||||
the two latest stable releases of Kafka and Go, and we provide a two month
|
||||
grace period for older releases. This means we currently officially support
|
||||
Go 1.12 through 1.14, and Kafka 2.1 through 2.4, although older releases are
|
||||
still likely to work.
|
||||
|
||||
Sarama follows semantic versioning and provides API stability via the gopkg.in service.
|
||||
You can import a version with a guaranteed stable API via http://gopkg.in/Shopify/sarama.v1.
|
||||
A changelog is available [here](CHANGELOG.md).
|
||||
|
||||
## Contributing
|
||||
|
||||
- Get started by checking our [contribution guidelines](https://github.com/Shopify/sarama/blob/master/.github/CONTRIBUTING.md).
|
||||
- Read the [Sarama wiki](https://github.com/Shopify/sarama/wiki) for more technical and design details.
|
||||
- The [Kafka Protocol Specification](https://cwiki.apache.org/confluence/display/KAFKA/A+Guide+To+The+Kafka+Protocol) contains a wealth of useful information.
|
||||
- For more general issues, there is [a google group](https://groups.google.com/forum/#!forum/kafka-clients) for Kafka client developers.
|
||||
- If you have any questions, just ask!
|
||||
-14
@@ -1,14 +0,0 @@
|
||||
# We have 5 * 192MB ZK processes and 5 * 320MB Kafka processes => 2560MB
|
||||
MEMORY = 3072
|
||||
|
||||
Vagrant.configure("2") do |config|
|
||||
config.vm.box = "ubuntu/bionic64"
|
||||
|
||||
config.vm.provision :shell, path: "vagrant/provision.sh"
|
||||
|
||||
config.vm.network "private_network", ip: "192.168.100.67"
|
||||
|
||||
config.vm.provider "virtualbox" do |v|
|
||||
v.memory = MEMORY
|
||||
end
|
||||
end
|
||||
-138
@@ -1,138 +0,0 @@
|
||||
package sarama
|
||||
|
||||
//Resource holds information about acl resource type
|
||||
type Resource struct {
|
||||
ResourceType AclResourceType
|
||||
ResourceName string
|
||||
ResourcePatternType AclResourcePatternType
|
||||
}
|
||||
|
||||
func (r *Resource) encode(pe packetEncoder, version int16) error {
|
||||
pe.putInt8(int8(r.ResourceType))
|
||||
|
||||
if err := pe.putString(r.ResourceName); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if version == 1 {
|
||||
if r.ResourcePatternType == AclPatternUnknown {
|
||||
Logger.Print("Cannot encode an unknown resource pattern type, using Literal instead")
|
||||
r.ResourcePatternType = AclPatternLiteral
|
||||
}
|
||||
pe.putInt8(int8(r.ResourcePatternType))
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *Resource) decode(pd packetDecoder, version int16) (err error) {
|
||||
resourceType, err := pd.getInt8()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
r.ResourceType = AclResourceType(resourceType)
|
||||
|
||||
if r.ResourceName, err = pd.getString(); err != nil {
|
||||
return err
|
||||
}
|
||||
if version == 1 {
|
||||
pattern, err := pd.getInt8()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
r.ResourcePatternType = AclResourcePatternType(pattern)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
//Acl holds information about acl type
|
||||
type Acl struct {
|
||||
Principal string
|
||||
Host string
|
||||
Operation AclOperation
|
||||
PermissionType AclPermissionType
|
||||
}
|
||||
|
||||
func (a *Acl) encode(pe packetEncoder) error {
|
||||
if err := pe.putString(a.Principal); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := pe.putString(a.Host); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
pe.putInt8(int8(a.Operation))
|
||||
pe.putInt8(int8(a.PermissionType))
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (a *Acl) decode(pd packetDecoder, version int16) (err error) {
|
||||
if a.Principal, err = pd.getString(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if a.Host, err = pd.getString(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
operation, err := pd.getInt8()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
a.Operation = AclOperation(operation)
|
||||
|
||||
permissionType, err := pd.getInt8()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
a.PermissionType = AclPermissionType(permissionType)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
//ResourceAcls is an acl resource type
|
||||
type ResourceAcls struct {
|
||||
Resource
|
||||
Acls []*Acl
|
||||
}
|
||||
|
||||
func (r *ResourceAcls) encode(pe packetEncoder, version int16) error {
|
||||
if err := r.Resource.encode(pe, version); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := pe.putArrayLength(len(r.Acls)); err != nil {
|
||||
return err
|
||||
}
|
||||
for _, acl := range r.Acls {
|
||||
if err := acl.encode(pe); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *ResourceAcls) decode(pd packetDecoder, version int16) error {
|
||||
if err := r.Resource.decode(pd, version); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
n, err := pd.getArrayLength()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
r.Acls = make([]*Acl, n)
|
||||
for i := 0; i < n; i++ {
|
||||
r.Acls[i] = new(Acl)
|
||||
if err := r.Acls[i].decode(pd, version); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
-89
@@ -1,89 +0,0 @@
|
||||
package sarama
|
||||
|
||||
//CreateAclsRequest is an acl creation request
|
||||
type CreateAclsRequest struct {
|
||||
Version int16
|
||||
AclCreations []*AclCreation
|
||||
}
|
||||
|
||||
func (c *CreateAclsRequest) encode(pe packetEncoder) error {
|
||||
if err := pe.putArrayLength(len(c.AclCreations)); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, aclCreation := range c.AclCreations {
|
||||
if err := aclCreation.encode(pe, c.Version); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *CreateAclsRequest) decode(pd packetDecoder, version int16) (err error) {
|
||||
c.Version = version
|
||||
n, err := pd.getArrayLength()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
c.AclCreations = make([]*AclCreation, n)
|
||||
|
||||
for i := 0; i < n; i++ {
|
||||
c.AclCreations[i] = new(AclCreation)
|
||||
if err := c.AclCreations[i].decode(pd, version); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *CreateAclsRequest) key() int16 {
|
||||
return 30
|
||||
}
|
||||
|
||||
func (c *CreateAclsRequest) version() int16 {
|
||||
return c.Version
|
||||
}
|
||||
|
||||
func (c *CreateAclsRequest) headerVersion() int16 {
|
||||
return 1
|
||||
}
|
||||
|
||||
func (c *CreateAclsRequest) requiredVersion() KafkaVersion {
|
||||
switch c.Version {
|
||||
case 1:
|
||||
return V2_0_0_0
|
||||
default:
|
||||
return V0_11_0_0
|
||||
}
|
||||
}
|
||||
|
||||
//AclCreation is a wrapper around Resource and Acl type
|
||||
type AclCreation struct {
|
||||
Resource
|
||||
Acl
|
||||
}
|
||||
|
||||
func (a *AclCreation) encode(pe packetEncoder, version int16) error {
|
||||
if err := a.Resource.encode(pe, version); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := a.Acl.encode(pe); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (a *AclCreation) decode(pd packetDecoder, version int16) (err error) {
|
||||
if err := a.Resource.decode(pd, version); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := a.Acl.decode(pd, version); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
-94
@@ -1,94 +0,0 @@
|
||||
package sarama
|
||||
|
||||
import "time"
|
||||
|
||||
//CreateAclsResponse is a an acl response creation type
|
||||
type CreateAclsResponse struct {
|
||||
ThrottleTime time.Duration
|
||||
AclCreationResponses []*AclCreationResponse
|
||||
}
|
||||
|
||||
func (c *CreateAclsResponse) encode(pe packetEncoder) error {
|
||||
pe.putInt32(int32(c.ThrottleTime / time.Millisecond))
|
||||
|
||||
if err := pe.putArrayLength(len(c.AclCreationResponses)); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, aclCreationResponse := range c.AclCreationResponses {
|
||||
if err := aclCreationResponse.encode(pe); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *CreateAclsResponse) decode(pd packetDecoder, version int16) (err error) {
|
||||
throttleTime, err := pd.getInt32()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
c.ThrottleTime = time.Duration(throttleTime) * time.Millisecond
|
||||
|
||||
n, err := pd.getArrayLength()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
c.AclCreationResponses = make([]*AclCreationResponse, n)
|
||||
for i := 0; i < n; i++ {
|
||||
c.AclCreationResponses[i] = new(AclCreationResponse)
|
||||
if err := c.AclCreationResponses[i].decode(pd, version); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *CreateAclsResponse) key() int16 {
|
||||
return 30
|
||||
}
|
||||
|
||||
func (c *CreateAclsResponse) version() int16 {
|
||||
return 0
|
||||
}
|
||||
|
||||
func (c *CreateAclsResponse) headerVersion() int16 {
|
||||
return 0
|
||||
}
|
||||
|
||||
func (c *CreateAclsResponse) requiredVersion() KafkaVersion {
|
||||
return V0_11_0_0
|
||||
}
|
||||
|
||||
//AclCreationResponse is an acl creation response type
|
||||
type AclCreationResponse struct {
|
||||
Err KError
|
||||
ErrMsg *string
|
||||
}
|
||||
|
||||
func (a *AclCreationResponse) encode(pe packetEncoder) error {
|
||||
pe.putInt16(int16(a.Err))
|
||||
|
||||
if err := pe.putNullableString(a.ErrMsg); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (a *AclCreationResponse) decode(pd packetDecoder, version int16) (err error) {
|
||||
kerr, err := pd.getInt16()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
a.Err = KError(kerr)
|
||||
|
||||
if a.ErrMsg, err = pd.getNullableString(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
-62
@@ -1,62 +0,0 @@
|
||||
package sarama
|
||||
|
||||
//DeleteAclsRequest is a delete acl request
|
||||
type DeleteAclsRequest struct {
|
||||
Version int
|
||||
Filters []*AclFilter
|
||||
}
|
||||
|
||||
func (d *DeleteAclsRequest) encode(pe packetEncoder) error {
|
||||
if err := pe.putArrayLength(len(d.Filters)); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, filter := range d.Filters {
|
||||
filter.Version = d.Version
|
||||
if err := filter.encode(pe); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (d *DeleteAclsRequest) decode(pd packetDecoder, version int16) (err error) {
|
||||
d.Version = int(version)
|
||||
n, err := pd.getArrayLength()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
d.Filters = make([]*AclFilter, n)
|
||||
for i := 0; i < n; i++ {
|
||||
d.Filters[i] = new(AclFilter)
|
||||
d.Filters[i].Version = int(version)
|
||||
if err := d.Filters[i].decode(pd, version); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (d *DeleteAclsRequest) key() int16 {
|
||||
return 31
|
||||
}
|
||||
|
||||
func (d *DeleteAclsRequest) version() int16 {
|
||||
return int16(d.Version)
|
||||
}
|
||||
|
||||
func (c *DeleteAclsRequest) headerVersion() int16 {
|
||||
return 1
|
||||
}
|
||||
|
||||
func (d *DeleteAclsRequest) requiredVersion() KafkaVersion {
|
||||
switch d.Version {
|
||||
case 1:
|
||||
return V2_0_0_0
|
||||
default:
|
||||
return V0_11_0_0
|
||||
}
|
||||
}
|
||||
-163
@@ -1,163 +0,0 @@
|
||||
package sarama
|
||||
|
||||
import "time"
|
||||
|
||||
//DeleteAclsResponse is a delete acl response
|
||||
type DeleteAclsResponse struct {
|
||||
Version int16
|
||||
ThrottleTime time.Duration
|
||||
FilterResponses []*FilterResponse
|
||||
}
|
||||
|
||||
func (d *DeleteAclsResponse) encode(pe packetEncoder) error {
|
||||
pe.putInt32(int32(d.ThrottleTime / time.Millisecond))
|
||||
|
||||
if err := pe.putArrayLength(len(d.FilterResponses)); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, filterResponse := range d.FilterResponses {
|
||||
if err := filterResponse.encode(pe, d.Version); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (d *DeleteAclsResponse) decode(pd packetDecoder, version int16) (err error) {
|
||||
throttleTime, err := pd.getInt32()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
d.ThrottleTime = time.Duration(throttleTime) * time.Millisecond
|
||||
|
||||
n, err := pd.getArrayLength()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
d.FilterResponses = make([]*FilterResponse, n)
|
||||
|
||||
for i := 0; i < n; i++ {
|
||||
d.FilterResponses[i] = new(FilterResponse)
|
||||
if err := d.FilterResponses[i].decode(pd, version); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (d *DeleteAclsResponse) key() int16 {
|
||||
return 31
|
||||
}
|
||||
|
||||
func (d *DeleteAclsResponse) version() int16 {
|
||||
return d.Version
|
||||
}
|
||||
|
||||
func (d *DeleteAclsResponse) headerVersion() int16 {
|
||||
return 0
|
||||
}
|
||||
|
||||
func (d *DeleteAclsResponse) requiredVersion() KafkaVersion {
|
||||
return V0_11_0_0
|
||||
}
|
||||
|
||||
//FilterResponse is a filter response type
|
||||
type FilterResponse struct {
|
||||
Err KError
|
||||
ErrMsg *string
|
||||
MatchingAcls []*MatchingAcl
|
||||
}
|
||||
|
||||
func (f *FilterResponse) encode(pe packetEncoder, version int16) error {
|
||||
pe.putInt16(int16(f.Err))
|
||||
if err := pe.putNullableString(f.ErrMsg); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := pe.putArrayLength(len(f.MatchingAcls)); err != nil {
|
||||
return err
|
||||
}
|
||||
for _, matchingAcl := range f.MatchingAcls {
|
||||
if err := matchingAcl.encode(pe, version); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *FilterResponse) decode(pd packetDecoder, version int16) (err error) {
|
||||
kerr, err := pd.getInt16()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
f.Err = KError(kerr)
|
||||
|
||||
if f.ErrMsg, err = pd.getNullableString(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
n, err := pd.getArrayLength()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
f.MatchingAcls = make([]*MatchingAcl, n)
|
||||
for i := 0; i < n; i++ {
|
||||
f.MatchingAcls[i] = new(MatchingAcl)
|
||||
if err := f.MatchingAcls[i].decode(pd, version); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
//MatchingAcl is a matching acl type
|
||||
type MatchingAcl struct {
|
||||
Err KError
|
||||
ErrMsg *string
|
||||
Resource
|
||||
Acl
|
||||
}
|
||||
|
||||
func (m *MatchingAcl) encode(pe packetEncoder, version int16) error {
|
||||
pe.putInt16(int16(m.Err))
|
||||
if err := pe.putNullableString(m.ErrMsg); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := m.Resource.encode(pe, version); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := m.Acl.encode(pe); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *MatchingAcl) decode(pd packetDecoder, version int16) (err error) {
|
||||
kerr, err := pd.getInt16()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
m.Err = KError(kerr)
|
||||
|
||||
if m.ErrMsg, err = pd.getNullableString(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := m.Resource.decode(pd, version); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := m.Acl.decode(pd, version); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
-39
@@ -1,39 +0,0 @@
|
||||
package sarama
|
||||
|
||||
//DescribeAclsRequest is a secribe acl request type
|
||||
type DescribeAclsRequest struct {
|
||||
Version int
|
||||
AclFilter
|
||||
}
|
||||
|
||||
func (d *DescribeAclsRequest) encode(pe packetEncoder) error {
|
||||
d.AclFilter.Version = d.Version
|
||||
return d.AclFilter.encode(pe)
|
||||
}
|
||||
|
||||
func (d *DescribeAclsRequest) decode(pd packetDecoder, version int16) (err error) {
|
||||
d.Version = int(version)
|
||||
d.AclFilter.Version = int(version)
|
||||
return d.AclFilter.decode(pd, version)
|
||||
}
|
||||
|
||||
func (d *DescribeAclsRequest) key() int16 {
|
||||
return 29
|
||||
}
|
||||
|
||||
func (d *DescribeAclsRequest) version() int16 {
|
||||
return int16(d.Version)
|
||||
}
|
||||
|
||||
func (d *DescribeAclsRequest) headerVersion() int16 {
|
||||
return 1
|
||||
}
|
||||
|
||||
func (d *DescribeAclsRequest) requiredVersion() KafkaVersion {
|
||||
switch d.Version {
|
||||
case 1:
|
||||
return V2_0_0_0
|
||||
default:
|
||||
return V0_11_0_0
|
||||
}
|
||||
}
|
||||
-91
@@ -1,91 +0,0 @@
|
||||
package sarama
|
||||
|
||||
import "time"
|
||||
|
||||
//DescribeAclsResponse is a describe acl response type
|
||||
type DescribeAclsResponse struct {
|
||||
Version int16
|
||||
ThrottleTime time.Duration
|
||||
Err KError
|
||||
ErrMsg *string
|
||||
ResourceAcls []*ResourceAcls
|
||||
}
|
||||
|
||||
func (d *DescribeAclsResponse) encode(pe packetEncoder) error {
|
||||
pe.putInt32(int32(d.ThrottleTime / time.Millisecond))
|
||||
pe.putInt16(int16(d.Err))
|
||||
|
||||
if err := pe.putNullableString(d.ErrMsg); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := pe.putArrayLength(len(d.ResourceAcls)); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, resourceAcl := range d.ResourceAcls {
|
||||
if err := resourceAcl.encode(pe, d.Version); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (d *DescribeAclsResponse) decode(pd packetDecoder, version int16) (err error) {
|
||||
throttleTime, err := pd.getInt32()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
d.ThrottleTime = time.Duration(throttleTime) * time.Millisecond
|
||||
|
||||
kerr, err := pd.getInt16()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
d.Err = KError(kerr)
|
||||
|
||||
errmsg, err := pd.getString()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if errmsg != "" {
|
||||
d.ErrMsg = &errmsg
|
||||
}
|
||||
|
||||
n, err := pd.getArrayLength()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
d.ResourceAcls = make([]*ResourceAcls, n)
|
||||
|
||||
for i := 0; i < n; i++ {
|
||||
d.ResourceAcls[i] = new(ResourceAcls)
|
||||
if err := d.ResourceAcls[i].decode(pd, version); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (d *DescribeAclsResponse) key() int16 {
|
||||
return 29
|
||||
}
|
||||
|
||||
func (d *DescribeAclsResponse) version() int16 {
|
||||
return d.Version
|
||||
}
|
||||
|
||||
func (d *DescribeAclsResponse) headerVersion() int16 {
|
||||
return 0
|
||||
}
|
||||
|
||||
func (d *DescribeAclsResponse) requiredVersion() KafkaVersion {
|
||||
switch d.Version {
|
||||
case 1:
|
||||
return V2_0_0_0
|
||||
default:
|
||||
return V0_11_0_0
|
||||
}
|
||||
}
|
||||
-78
@@ -1,78 +0,0 @@
|
||||
package sarama
|
||||
|
||||
type AclFilter struct {
|
||||
Version int
|
||||
ResourceType AclResourceType
|
||||
ResourceName *string
|
||||
ResourcePatternTypeFilter AclResourcePatternType
|
||||
Principal *string
|
||||
Host *string
|
||||
Operation AclOperation
|
||||
PermissionType AclPermissionType
|
||||
}
|
||||
|
||||
func (a *AclFilter) encode(pe packetEncoder) error {
|
||||
pe.putInt8(int8(a.ResourceType))
|
||||
if err := pe.putNullableString(a.ResourceName); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if a.Version == 1 {
|
||||
pe.putInt8(int8(a.ResourcePatternTypeFilter))
|
||||
}
|
||||
|
||||
if err := pe.putNullableString(a.Principal); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := pe.putNullableString(a.Host); err != nil {
|
||||
return err
|
||||
}
|
||||
pe.putInt8(int8(a.Operation))
|
||||
pe.putInt8(int8(a.PermissionType))
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (a *AclFilter) decode(pd packetDecoder, version int16) (err error) {
|
||||
resourceType, err := pd.getInt8()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
a.ResourceType = AclResourceType(resourceType)
|
||||
|
||||
if a.ResourceName, err = pd.getNullableString(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if a.Version == 1 {
|
||||
pattern, err := pd.getInt8()
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
a.ResourcePatternTypeFilter = AclResourcePatternType(pattern)
|
||||
}
|
||||
|
||||
if a.Principal, err = pd.getNullableString(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if a.Host, err = pd.getNullableString(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
operation, err := pd.getInt8()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
a.Operation = AclOperation(operation)
|
||||
|
||||
permissionType, err := pd.getInt8()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
a.PermissionType = AclPermissionType(permissionType)
|
||||
|
||||
return nil
|
||||
}
|
||||
-55
@@ -1,55 +0,0 @@
|
||||
package sarama
|
||||
|
||||
type (
|
||||
AclOperation int
|
||||
|
||||
AclPermissionType int
|
||||
|
||||
AclResourceType int
|
||||
|
||||
AclResourcePatternType int
|
||||
)
|
||||
|
||||
// ref: https://github.com/apache/kafka/blob/trunk/clients/src/main/java/org/apache/kafka/common/acl/AclOperation.java
|
||||
const (
|
||||
AclOperationUnknown AclOperation = iota
|
||||
AclOperationAny
|
||||
AclOperationAll
|
||||
AclOperationRead
|
||||
AclOperationWrite
|
||||
AclOperationCreate
|
||||
AclOperationDelete
|
||||
AclOperationAlter
|
||||
AclOperationDescribe
|
||||
AclOperationClusterAction
|
||||
AclOperationDescribeConfigs
|
||||
AclOperationAlterConfigs
|
||||
AclOperationIdempotentWrite
|
||||
)
|
||||
|
||||
// ref: https://github.com/apache/kafka/blob/trunk/clients/src/main/java/org/apache/kafka/common/acl/AclPermissionType.java
|
||||
const (
|
||||
AclPermissionUnknown AclPermissionType = iota
|
||||
AclPermissionAny
|
||||
AclPermissionDeny
|
||||
AclPermissionAllow
|
||||
)
|
||||
|
||||
// ref: https://github.com/apache/kafka/blob/trunk/clients/src/main/java/org/apache/kafka/common/resource/ResourceType.java
|
||||
const (
|
||||
AclResourceUnknown AclResourceType = iota
|
||||
AclResourceAny
|
||||
AclResourceTopic
|
||||
AclResourceGroup
|
||||
AclResourceCluster
|
||||
AclResourceTransactionalID
|
||||
)
|
||||
|
||||
// ref: https://github.com/apache/kafka/blob/trunk/clients/src/main/java/org/apache/kafka/common/resource/PatternType.java
|
||||
const (
|
||||
AclPatternUnknown AclResourcePatternType = iota
|
||||
AclPatternAny
|
||||
AclPatternMatch
|
||||
AclPatternLiteral
|
||||
AclPatternPrefixed
|
||||
)
|
||||
-57
@@ -1,57 +0,0 @@
|
||||
package sarama
|
||||
|
||||
//AddOffsetsToTxnRequest adds offsets to a transaction request
|
||||
type AddOffsetsToTxnRequest struct {
|
||||
TransactionalID string
|
||||
ProducerID int64
|
||||
ProducerEpoch int16
|
||||
GroupID string
|
||||
}
|
||||
|
||||
func (a *AddOffsetsToTxnRequest) encode(pe packetEncoder) error {
|
||||
if err := pe.putString(a.TransactionalID); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
pe.putInt64(a.ProducerID)
|
||||
|
||||
pe.putInt16(a.ProducerEpoch)
|
||||
|
||||
if err := pe.putString(a.GroupID); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (a *AddOffsetsToTxnRequest) decode(pd packetDecoder, version int16) (err error) {
|
||||
if a.TransactionalID, err = pd.getString(); err != nil {
|
||||
return err
|
||||
}
|
||||
if a.ProducerID, err = pd.getInt64(); err != nil {
|
||||
return err
|
||||
}
|
||||
if a.ProducerEpoch, err = pd.getInt16(); err != nil {
|
||||
return err
|
||||
}
|
||||
if a.GroupID, err = pd.getString(); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (a *AddOffsetsToTxnRequest) key() int16 {
|
||||
return 25
|
||||
}
|
||||
|
||||
func (a *AddOffsetsToTxnRequest) version() int16 {
|
||||
return 0
|
||||
}
|
||||
|
||||
func (a *AddOffsetsToTxnRequest) headerVersion() int16 {
|
||||
return 1
|
||||
}
|
||||
|
||||
func (a *AddOffsetsToTxnRequest) requiredVersion() KafkaVersion {
|
||||
return V0_11_0_0
|
||||
}
|
||||
-49
@@ -1,49 +0,0 @@
|
||||
package sarama
|
||||
|
||||
import (
|
||||
"time"
|
||||
)
|
||||
|
||||
//AddOffsetsToTxnResponse is a response type for adding offsets to txns
|
||||
type AddOffsetsToTxnResponse struct {
|
||||
ThrottleTime time.Duration
|
||||
Err KError
|
||||
}
|
||||
|
||||
func (a *AddOffsetsToTxnResponse) encode(pe packetEncoder) error {
|
||||
pe.putInt32(int32(a.ThrottleTime / time.Millisecond))
|
||||
pe.putInt16(int16(a.Err))
|
||||
return nil
|
||||
}
|
||||
|
||||
func (a *AddOffsetsToTxnResponse) decode(pd packetDecoder, version int16) (err error) {
|
||||
throttleTime, err := pd.getInt32()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
a.ThrottleTime = time.Duration(throttleTime) * time.Millisecond
|
||||
|
||||
kerr, err := pd.getInt16()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
a.Err = KError(kerr)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (a *AddOffsetsToTxnResponse) key() int16 {
|
||||
return 25
|
||||
}
|
||||
|
||||
func (a *AddOffsetsToTxnResponse) version() int16 {
|
||||
return 0
|
||||
}
|
||||
|
||||
func (a *AddOffsetsToTxnResponse) headerVersion() int16 {
|
||||
return 0
|
||||
}
|
||||
|
||||
func (a *AddOffsetsToTxnResponse) requiredVersion() KafkaVersion {
|
||||
return V0_11_0_0
|
||||
}
|
||||
-81
@@ -1,81 +0,0 @@
|
||||
package sarama
|
||||
|
||||
//AddPartitionsToTxnRequest is a add paartition request
|
||||
type AddPartitionsToTxnRequest struct {
|
||||
TransactionalID string
|
||||
ProducerID int64
|
||||
ProducerEpoch int16
|
||||
TopicPartitions map[string][]int32
|
||||
}
|
||||
|
||||
func (a *AddPartitionsToTxnRequest) encode(pe packetEncoder) error {
|
||||
if err := pe.putString(a.TransactionalID); err != nil {
|
||||
return err
|
||||
}
|
||||
pe.putInt64(a.ProducerID)
|
||||
pe.putInt16(a.ProducerEpoch)
|
||||
|
||||
if err := pe.putArrayLength(len(a.TopicPartitions)); err != nil {
|
||||
return err
|
||||
}
|
||||
for topic, partitions := range a.TopicPartitions {
|
||||
if err := pe.putString(topic); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := pe.putInt32Array(partitions); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (a *AddPartitionsToTxnRequest) decode(pd packetDecoder, version int16) (err error) {
|
||||
if a.TransactionalID, err = pd.getString(); err != nil {
|
||||
return err
|
||||
}
|
||||
if a.ProducerID, err = pd.getInt64(); err != nil {
|
||||
return err
|
||||
}
|
||||
if a.ProducerEpoch, err = pd.getInt16(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
n, err := pd.getArrayLength()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
a.TopicPartitions = make(map[string][]int32)
|
||||
for i := 0; i < n; i++ {
|
||||
topic, err := pd.getString()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
partitions, err := pd.getInt32Array()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
a.TopicPartitions[topic] = partitions
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (a *AddPartitionsToTxnRequest) key() int16 {
|
||||
return 24
|
||||
}
|
||||
|
||||
func (a *AddPartitionsToTxnRequest) version() int16 {
|
||||
return 0
|
||||
}
|
||||
|
||||
func (a *AddPartitionsToTxnRequest) headerVersion() int16 {
|
||||
return 1
|
||||
}
|
||||
|
||||
func (a *AddPartitionsToTxnRequest) requiredVersion() KafkaVersion {
|
||||
return V0_11_0_0
|
||||
}
|
||||
-114
@@ -1,114 +0,0 @@
|
||||
package sarama
|
||||
|
||||
import (
|
||||
"time"
|
||||
)
|
||||
|
||||
//AddPartitionsToTxnResponse is a partition errors to transaction type
|
||||
type AddPartitionsToTxnResponse struct {
|
||||
ThrottleTime time.Duration
|
||||
Errors map[string][]*PartitionError
|
||||
}
|
||||
|
||||
func (a *AddPartitionsToTxnResponse) encode(pe packetEncoder) error {
|
||||
pe.putInt32(int32(a.ThrottleTime / time.Millisecond))
|
||||
if err := pe.putArrayLength(len(a.Errors)); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for topic, e := range a.Errors {
|
||||
if err := pe.putString(topic); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := pe.putArrayLength(len(e)); err != nil {
|
||||
return err
|
||||
}
|
||||
for _, partitionError := range e {
|
||||
if err := partitionError.encode(pe); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (a *AddPartitionsToTxnResponse) decode(pd packetDecoder, version int16) (err error) {
|
||||
throttleTime, err := pd.getInt32()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
a.ThrottleTime = time.Duration(throttleTime) * time.Millisecond
|
||||
|
||||
n, err := pd.getArrayLength()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
a.Errors = make(map[string][]*PartitionError)
|
||||
|
||||
for i := 0; i < n; i++ {
|
||||
topic, err := pd.getString()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
m, err := pd.getArrayLength()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
a.Errors[topic] = make([]*PartitionError, m)
|
||||
|
||||
for j := 0; j < m; j++ {
|
||||
a.Errors[topic][j] = new(PartitionError)
|
||||
if err := a.Errors[topic][j].decode(pd, version); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (a *AddPartitionsToTxnResponse) key() int16 {
|
||||
return 24
|
||||
}
|
||||
|
||||
func (a *AddPartitionsToTxnResponse) version() int16 {
|
||||
return 0
|
||||
}
|
||||
|
||||
func (a *AddPartitionsToTxnResponse) headerVersion() int16 {
|
||||
return 0
|
||||
}
|
||||
|
||||
func (a *AddPartitionsToTxnResponse) requiredVersion() KafkaVersion {
|
||||
return V0_11_0_0
|
||||
}
|
||||
|
||||
//PartitionError is a partition error type
|
||||
type PartitionError struct {
|
||||
Partition int32
|
||||
Err KError
|
||||
}
|
||||
|
||||
func (p *PartitionError) encode(pe packetEncoder) error {
|
||||
pe.putInt32(p.Partition)
|
||||
pe.putInt16(int16(p.Err))
|
||||
return nil
|
||||
}
|
||||
|
||||
func (p *PartitionError) decode(pd packetDecoder, version int16) (err error) {
|
||||
if p.Partition, err = pd.getInt32(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
kerr, err := pd.getInt16()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
p.Err = KError(kerr)
|
||||
|
||||
return nil
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user