More fixes

change package from `main -> goreplay`
this will allow importing `goreplay` as a package

.github/workflows/ci-docker.yaml

@@ -8,13 +8,13 @@ jobs:
     steps:
       -
         name: "Checkout repository"
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
       -
         name: "Set up Docker Buildx"
-        uses: docker/setup-buildx-action@v1
+        uses: docker/setup-buildx-action@v2
       -
         name: "Cache Docker layers"
-        uses: actions/cache@v2
+        uses: actions/cache@v3.0.5
         with:
           path: /tmp/.buildx-cache
           key: ${{ runner.os }}-buildx-${{ github.sha }}
@@ -28,7 +28,7 @@ jobs:
           password: ${{ secrets.DOCKERHUB_TOKEN }}
       -
         name: "Build and push docker image"
-        uses: docker/build-push-action@v2
+        uses: docker/build-push-action@v3
         with:
           context: .
           push: true

.github/workflows/ci-test.yaml

@@ -4,7 +4,7 @@ jobs:
   test:
     strategy:
       matrix:
-        go-version: [1.14.x, 1.15.x] # two latest minor versions
+        go-version: [1.18.x, 1.19.x] # two latest minor versions
     runs-on: ubuntu-latest
     steps:
     - name: update package index
@@ -16,8 +16,8 @@ jobs:
       with:
         go-version: ${{ matrix.go-version }}
     - name: checkout code
-      uses: actions/checkout@v2
-    - uses: actions/cache@v2
+      uses: actions/checkout@v3
+    - uses: actions/cache@v3.0.5
       with:
         path: |
           ~/go/pkg/mod              # Module download cache

.gitignore

@@ -1,10 +1,14 @@
+vendor
 *.swp
 *.gor
 *.rpm
 *.dep
+*.deb
 *.pkg
-
+*.exe
+*.pprof
 *.out
+hey
 
 *.bin
 
@@ -28,3 +32,4 @@ goreplay
 corpus
 crashers
 suppressions
+dist

COMM-LICENSE

@@ -2,7 +2,7 @@ END-USER LICENSE AGREEMENT
 
 ------------------------------------------------------------------------------
 
-IMPORTANT: THIS SOFTWARE END-USER LICENSE AGREEMENT ("EULA") IS A LEGAL AGREEMENT (“Agreement”) BETWEEN YOU (THE CUSTOMER, EITHER AS AN INDIVIDUAL OR, IF PURCHASED OR OTHERWISE ACQUIRED BY OR FOR AN ENTITY, AS AN ENTITY) AND GoReplay LLC. READ IT CAREFULLY BEFORE COMPLETING THE INSTALLATION PROCESS AND USING GOREPLAY PRO AND RELATED SOFTWARE COMPONENTS (“SOFTWARE”). IT PROVIDES A LICENSE TO USE THE SOFTWARE AND CONTAINS WARRANTY INFORMATION AND LIABILITY DISCLAIMERS. BY INSTALLING AND USING THE SOFTWARE, YOU ARE CONFIRMING YOUR ACCEPTANCE OF THE SOFTWARE AND AGREEING TO BECOME BOUND BY THE TERMS OF THIS AGREEMENT.
+IMPORTANT: THIS SOFTWARE END-USER LICENSE AGREEMENT ("EULA") IS A LEGAL AGREEMENT (“Agreement”) BETWEEN YOU (THE CUSTOMER, EITHER AS AN INDIVIDUAL OR, IF PURCHASED OR OTHERWISE ACQUIRED BY OR FOR AN ENTITY, AS AN ENTITY), hereafter "Customer", AND Replay Software LLC ("Licensor"). READ IT CAREFULLY BEFORE COMPLETING THE INSTALLATION PROCESS AND USING GOREPLAY PRO AND RELATED SOFTWARE COMPONENTS (“SOFTWARE”). IT PROVIDES A LICENSE TO USE THE SOFTWARE AND CONTAINS WARRANTY INFORMATION AND LIABILITY DISCLAIMERS. BY INSTALLING AND USING THE SOFTWARE, YOU ARE CONFIRMING YOUR ACCEPTANCE OF THE SOFTWARE AND AGREEING TO BECOME BOUND BY THE TERMS OF THIS AGREEMENT.
 
 ------------------------------------------------------------------------------
 
@@ -14,17 +14,17 @@ In order to use the Software under this Agreement, you must receive a “Source
 
 1.2 Unlimited Organization License. If you purchased an Organization License (included with the GoReplay Pro Software), you may install the Software on an unlimited number of Hosts. “Host” means any physical or virtual machine which is controlled by you. You may concurrently run the software on an unlimited number of Hosts.
 
-1.3 Appliance License. If you purchased an Appliance License, you may distribute the Software in any applications, frameworks, or elements (collectively referred to as an “Application” or “Applications”) that you develop using the Software in accordance with this EULA, provided that such distribution does not violate the restrictions set forth in section 3 of this EULA. You must not remove, obscure or interfere with any copyright, acknowledgment, attribution, trademark, warning or disclaimer statement affixed to, incorporated in or otherwise applied in connection with the Software. You are required to ensure that the Software is not reused by or with any applications other than those with which you distribute it as permitted herein. For example, if You install the Software on a customer’s server, that customer is not permitted to use the Software independently of your Application. You must inform GoReplay LLC of your knowledge of any infringing use of the Software by any of your customers. You are liable for compliance by those third parties with the terms and conditions of this EULA. You will not owe GoReplay LLC any royalties for your distribution of the Software in accordance with this EULA.
+1.3 Appliance License. If you purchased an Appliance License, you may distribute the Software in any applications, frameworks, or elements (collectively referred to as an “Application” or “Applications”) that you develop using the Software in accordance with this EULA, provided that such distribution does not violate the restrictions set forth in section 3 of this EULA. You must not remove, obscure or interfere with any copyright, acknowledgment, attribution, trademark, warning or disclaimer statement affixed to, incorporated in or otherwise applied in connection with the Software. You are required to ensure that the Software is not reused by or with any applications other than those with which you distribute it as permitted herein. For example, if You install the Software on a customer’s server, that customer is not permitted to use the Software independently of your Application. You must inform Licensor of your knowledge of any infringing use of the Software by any of your customers. You are liable for compliance by those third parties with the terms and conditions of this EULA. You will not owe Licensor any royalties for your distribution of the Software in accordance with this EULA.
 
 1.4 Archive Copies. You are entitled to make a reasonable amount of copies of the Software for archival purposes. Each copy must reproduce all copyright and other proprietary rights notices on or in the Software Product.
 
 1.5 Electronic Delivery. All Software and license documentation shall be delivered by electronic means unless otherwise specified on the applicable invoice or at the time of purchase. Software shall be deemed delivered when it is made available for download by you (“Delivery”).        
 
-2. Modifications. GoReplay LLC shall provide you with source code so that you can create Modifications of the original software. “Modification” means: (a) any addition to or deletion from the contents of a file included in the original Software or previous Modifications created by You, or (b) any new file that contains any part of the original Software or previous Modifications. While you retain all rights to any original work authored by you as part of the Modifications, We continue to own all copyright and other intellectual property rights in the Software.
+2. Modifications. Licensor shall provide you with source code so that you can create Modifications of the original software. “Modification” means: (a) any addition to or deletion from the contents of a file included in the original Software or previous Modifications created by You, or (b) any new file that contains any part of the original Software or previous Modifications. While you retain all rights to any original work authored by you as part of the Modifications, We continue to own all copyright and other intellectual property rights in the Software.
 
 3. Restricted Uses. 
 
-3.1 You shall not (and shall not allow any third party to): (a) decompile, disassemble, or otherwise reverse engineer the Software or attempt to reconstruct or discover any source code, underlying ideas, algorithms, file formats or programming interfaces of the Software by any means whatsoever (except and only to the extent that applicable law prohibits or restricts reverse engineering restrictions); (b) distribute, sell, sublicense, rent, lease or use the Software for time sharing, hosting, service provider or like purposes, except as expressly permitted under this Agreement; (c) redistribute the Software or Modifications other than by including the Software or a portion thereof within your own product, which must have substantially different functionality than the Software or Modifications and must not allow any third party to use the Software or Modifications, or any portions thereof, for software development or application development purposes; (d) redistribute the Software as part of a product, "appliance" or "virtual server"; (e) redistribute the Software on any server which is not directly under your control; (f) remove any product identification, proprietary, copyright or other notices contained in the Software; (g) modify any part of the Software, create a derivative work of any part of the Software (except as permitted in Section 4), or incorporate the Software, except to the extent expressly authorized in writing by GoReplay LLC; (h) publicly disseminate performance information or analysis (including, without limitation, benchmarks) from any source relating to the Software; (i) utilize any equipment, device, software, or other means designed to circumvent or remove any form of Source URL or copy protection used by GoReplay LLC in connection with the Software, or use the Software together with any authorization code, Source URL, serial number, or other copy protection device not supplied by GoReplay LLC; (j) use the Software to develop a product which is competitive with any GoReplay LLC product offerings; or (k) use unauthorized Source URLS or keycode(s) or distribute or publish Source URLs or keycode(s), except as may be expressly permitted by GoReplay LLC in writing. If your unique Source URL is ever published, GoReplay LLC reserves the right to terminate your access without notice.
+3.1 You shall not (and shall not allow any third party to): (a) decompile, disassemble, or otherwise reverse engineer the Software or attempt to reconstruct or discover any source code, underlying ideas, algorithms, file formats or programming interfaces of the Software by any means whatsoever (except and only to the extent that applicable law prohibits or restricts reverse engineering restrictions); (b) distribute, sell, sublicense, rent, lease or use the Software for time sharing, hosting, service provider or like purposes, except as expressly permitted under this Agreement; (c) redistribute the Software or Modifications other than by including the Software or a portion thereof within your own product, which must have substantially different functionality than the Software or Modifications and must not allow any third party to use the Software or Modifications, or any portions thereof, for software development or application development purposes; (d) redistribute the Software as part of a product, "appliance" or "virtual server"; (e) redistribute the Software on any server which is not directly under your control; (f) remove any product identification, proprietary, copyright or other notices contained in the Software; (g) modify any part of the Software, create a derivative work of any part of the Software (except as permitted in Section 4), or incorporate the Software, except to the extent expressly authorized in writing by Licensor; (h) publicly disseminate performance information or analysis (including, without limitation, benchmarks) from any source relating to the Software; (i) utilize any equipment, device, software, or other means designed to circumvent or remove any form of Source URL or copy protection used by Licensor in connection with the Software, or use the Software together with any authorization code, Source URL, serial number, or other copy protection device not supplied by Licensor; (j) use the Software to develop a product which is competitive with any Licensor product offerings; or (k) use unauthorized Source URLS or keycode(s) or distribute or publish Source URLs or keycode(s), except as may be expressly permitted by Licensor in writing. If your unique Source URL is ever published, Licensor reserves the right to terminate your access without notice.
 
 3.2 UNDER NO CIRCUMSTANCES MAY YOU USE THE SOFTWARE AS PART OF A PRODUCT OR SERVICE THAT PROVIDES SIMILAR FUNCTIONALITY TO THE SOFTWARE ITSELF.
 
@@ -32,35 +32,35 @@ The Open Source version of the Software (“LGPL Version”) is licensed
 under the terms of the GNU Lesser General Public License versions 3.0
 (“LGPL”) and not under this EULA. 
 
-4. Ownership. Notwithstanding anything to the contrary contained herein, except for the limited license rights expressly provided herein, GoReplay LLC and its suppliers have and will retain all rights, title and interest (including, without limitation, all patent, copyright, trademark, trade secret and other intellectual property rights) in and to the Software and all copies, modifications and derivative works thereof (including any changes which incorporate any of your ideas, feedback or suggestions). You acknowledge that you are obtaining only a limited license right to the Software, and that irrespective of any use of the words “purchase”, “sale” or like terms hereunder no ownership rights are being conveyed to you under this Agreement or otherwise. 
+4. Ownership. Notwithstanding anything to the contrary contained herein, except for the limited license rights expressly provided herein, Licensor and its suppliers have and will retain all rights, title and interest (including, without limitation, all patent, copyright, trademark, trade secret and other intellectual property rights) in and to the Software and all copies, modifications and derivative works thereof (including any changes which incorporate any of your ideas, feedback or suggestions). You acknowledge that you are obtaining only a limited license right to the Software, and that irrespective of any use of the words “purchase”, “sale” or like terms hereunder no ownership rights are being conveyed to you under this Agreement or otherwise. 
 
-5. Fees and Payment. The Software license fees will be due and payable in full as set forth in the applicable invoice or at the time of purchase. If the Software does not function properly within two weeks of purchase, please contact us within those two weeks for a refund. You shall be responsible for all taxes, withholdings, duties and levies arising from the order (excluding taxes based on the net income of GoReplay LLC). 
+5. Fees and Payment. The Software license fees will be due and payable in full as set forth in the applicable invoice or at the time of purchase. If the Software does not function properly within two weeks of purchase, please contact us within those two weeks for a refund. You shall be responsible for all taxes, withholdings, duties and levies arising from the order (excluding taxes based on the net income of Licensor). 
 
 6. Support, Maintenance and Services. Subject to the terms and conditions of this Agreement, as set forth in your invoice, and as set forth on the GoReplay Pro support page (https://github.com/buger/goreplay/wiki/Pro-Support), support and maintenance services may be included with the purchase of your license subscription.
 
 7. Term of Agreement. 
 
-7.1 Term. This Agreement is effective as of the Delivery of the Software and expires at such time as all license and service subscriptions hereunder have expired in accordance with their own terms (the “Term”). For clarification, the term of your license under this Agreement may be perpetual, limited for Evaluation Version, or designated as a fixed-term license in the Invoice, and shall be specified at your time of purchase. Either party may terminate this Agreement (including all related Invoices) if the other party: (a) fails to cure any material breach of this Agreement within thirty (30) days after written notice of such breach, provided that GoReplay LLC may terminate this Agreement immediately upon any breach of Section 3 or if you exceed any other restrictions contained in Section 1, unless otherwise specified in this agreement; (b) ceases operation without a successor; or (c) seeks protection under any bankruptcy, receivership, trust deed, creditors arrangement, composition or comparable proceeding, or if any such proceeding is instituted against such party (and not dismissed within sixty (60) days)). Termination is not an exclusive remedy and the exercise by either party of any remedy under this Agreement will be without prejudice to any other remedies it may have under this Agreement, by law, or otherwise.        
+7.1 Term. This Agreement is effective as of the Delivery of the Software and expires at such time as all license and service subscriptions hereunder have expired in accordance with their own terms (the “Term”). For clarification, the term of your license under this Agreement may be perpetual, limited for Evaluation Version, or designated as a fixed-term license in the Invoice, and shall be specified at your time of purchase. Either party may terminate this Agreement (including all related Invoices) if the other party: (a) fails to cure any material breach of this Agreement within thirty (30) days after written notice of such breach, provided that Licensor may terminate this Agreement immediately upon any breach of Section 3 or if you exceed any other restrictions contained in Section 1, unless otherwise specified in this agreement; (b) ceases operation without a successor; or (c) seeks protection under any bankruptcy, receivership, trust deed, creditors arrangement, composition or comparable proceeding, or if any such proceeding is instituted against such party (and not dismissed within sixty (60) days)). Termination is not an exclusive remedy and the exercise by either party of any remedy under this Agreement will be without prejudice to any other remedies it may have under this Agreement, by law, or otherwise.        
 
 7.2 Termination. Upon any termination of this Agreement, you shall cease any and all use of any Software and destroy all copies thereof. 
 
-7.3 Expiration of License. Upon the expiration of any term under this Agreement, (a) all Software updates and services pursuant to the license shall cease, (b) you may only continue to run existing installations of the Software, (c) you may not install the Software on any additional Hosts, and (d) any new installation of the Software shall require the purchase of a new license subscription from GoReplay LLC.
+7.3 Expiration of License. Upon the expiration of any term under this Agreement, (a) all Software updates and services pursuant to the license shall cease, (b) you may only continue to run existing installations of the Software, (c) you may not install the Software on any additional Hosts, and (d) any new installation of the Software shall require the purchase of a new license subscription from Licensor.
 
-8. Disclaimer of Warranties. The Software is provided "as is," with all faults, defects and errors, and without warranty of any kind. GoReplay LLC does not warrant that the Software will be free of bugs, errors, viruses or other defects, and GoReplay LLC shall have no liability of any kind for the use of or inability to use the Software, the Software content or any associated service, and you acknowledge that it is not technically practicable for GoReplay LLC to do so. 
-To the maximum extent permitted by applicable law, GoReplay LLC disclaims all warranties, express, implied, arising by law or otherwise, regarding the Software, the Software content and their respective performance or suitability for your intended use, including without limitation any implied warranty of merchantability, fitness for a particular purpose.
-Notwithstanding the foregoing, GoReplay LLC represents and warrants that it either owns the entire right to, title to, interest in, or has the right to license, the Software, and your proper use of the same will not violate any intellectual property or proprietary rights of another.
+8. Disclaimer of Warranties. The Software is provided "as is," with all faults, defects and errors, and without warranty of any kind. Licensor does not warrant that the Software will be free of bugs, errors, viruses or other defects, and Licensor shall have no liability of any kind for the use of or inability to use the Software, the Software content or any associated service, and you acknowledge that it is not technically practicable for Licensor to do so. 
+To the maximum extent permitted by applicable law, Licensor disclaims all warranties, express, implied, arising by law or otherwise, regarding the Software, the Software content and their respective performance or suitability for your intended use, including without limitation any implied warranty of merchantability, fitness for a particular purpose.
+Notwithstanding the foregoing, Licensor represents and warrants that it either owns the entire right to, title to, interest in, or has the right to license, the Software, and your proper use of the same will not violate any intellectual property or proprietary rights of another.
 
 9. Limitation of Liability. 
 
 In no event will either party be liable for any direct, indirect, consequential, incidental, special, exemplary, or punitive damages or liabilities whatsoever arising from or relating to the Software, the Software content or this Agreement, whether based on contract, tort (including negligence), strict liability or other theory, even if such party has been advised of the possibility of such damages.
 
-In no event will GoReplay LLC liability exceed the Software license price as indicated in the invoice. The existence of more than one claim will not enlarge or extend this limit.
+In no event will Licensor liability exceed the Software license price as indicated in the invoice. The existence of more than one claim will not enlarge or extend this limit.
 
-10. Remedies. Your exclusive remedy and GoReplay LLC’ entire liability for breach of this Agreement shall be limited, at GoReplay LLC’ sole and exclusive discretion, to (a) replacement of any defective software or documentation; or (b) refund of the license fee paid to GoReplay LLC, payable in accordance with GoReplay LLC' refund policy.
+10. Remedies. Your exclusive remedy and Licensor’ entire liability for breach of this Agreement shall be limited, at Licensor’ sole and exclusive discretion, to (a) replacement of any defective software or documentation; or (b) refund of the license fee paid to Licensor, payable in accordance with Licensor' refund policy.
 
 11. Acknowledgements.
 
-11.1 Consent to the Use of Data. You agree that GoReplay LLC and its affiliates may collect and use technical information gathered as part of the product support services to be used in an anonymized manner. GoReplay LLC may use this information solely to improve products and services and will not disclose this information in a form that personally identifies you.
+11.1 Consent to the Use of Data. You agree that Licensor and its affiliates may collect and use technical information gathered as part of the product support services to be used in an anonymized manner. Licensor may use this information solely to improve products and services and will not disclose this information in a form that personally identifies you.
 
 11.2 Verification. We or a certified auditor acting on our behalf, may, upon its reasonable request and at its expense, audit you with respect to the use of the Software, subject to your reasonable information security and confidentiality protections. Such audit may be conducted by mail, electronic means or through an in-person visit to your place of business. Any such in-person audit shall be conducted during regular business hours at your facilities and shall not unreasonably interfere with your business activities. We shall not remove, copy, or redistribute any electronic material during the course of an audit. If an audit reveals that you are using the Software in a way that is in material violation of the terms of the EULA, then you shall pay our reasonable costs of conducting the audit. In the case of a material violation, you agree to pay Us any amounts owing that are attributable to the unauthorized use. In the alternative, We reserve the right, at our sole option, to terminate the licenses for the Software.
 
@@ -73,13 +73,13 @@ In no event will GoReplay LLC liability exceed the Software license price as ind
 
 13.1 Entire Agreement. This Agreement sets forth our entire agreement with respect to the Software and the subject matter hereof and supersedes all prior and contemporaneous understandings and agreements whether written or oral.
 
-13.2 Amendment. GoReplay LLC reserves the right, in its sole discretion, to amend this Agreement from time. Amendments to this Agreement can be located at: https://github.com/buger/goreplay/blob/master/COMM-LICENSE.
+13.2 Amendment. Licensor reserves the right, in its sole discretion, to amend this Agreement from time. Amendments to this Agreement can be located at: https://github.com/buger/goreplay/blob/master/COMM-LICENSE.
 
-13.3 Assignment. You may not assign this Agreement or any of its rights under this Agreement without the prior written consent of GoReplay LLC and any attempted assignment without such consent shall be void.
+13.3 Assignment. You may not assign this Agreement or any of its rights under this Agreement without the prior written consent of Licensor and any attempted assignment without such consent shall be void.
 
 13.4 Export Compliance. Each party agrees to comply with all applicable laws and regulations, including laws, regulations, orders or other restrictions on export, re-export or redistribution of software.
 
-13.5 Indemnification. You agree to defend, indemnify, and hold harmless GoReplay LLC from and against any lawsuits, claims, losses, damages, fines and expenses (including attorneys' fees and costs) arising out of your use of the Software or breach of this Agreement.
+13.5 Indemnification. You agree to defend, indemnify, and hold harmless Licensor from and against any lawsuits, claims, losses, damages, fines and expenses (including attorneys' fees and costs) arising out of your use of the Software or breach of this Agreement.
 
 13.6 Governing Law. This Agreement is governed by the laws of the State of Oregon and the United States without regard to conflicts of laws provisions thereof, and without regard to the United Nations Convention on the International Sale of Goods or the Uniform Computer Information Transactions Act, as currently enacted by any jurisdiction or as may be codified or amended from time to time by any jurisdiction. The jurisdiction and venue for actions related to the subject matter hereof shall be the state of Oregon and United States federal courts located in Portland, Oregon, and both parties hereby submit to the personal jurisdiction of such courts. 
 
@@ -91,4 +91,4 @@ In no event will GoReplay LLC liability exceed the Software license price as ind
 
 13.10 Headings. The headings of sections and paragraphs of this Agreement are for convenience of reference only and are not intended to restrict, affect or be of any weight in the interpretation or construction of the provisions of such sections or paragraphs.
 
-14. Contact Information. If you have any questions about this EULA, or if you want to contact GoReplay LLC for any reason, please direct correspondence to info@gortool.com
+14. Contact Information. If you have any questions about this EULA, or if you want to contact Licensor for any reason, please direct correspondence to hello@goreplay.org

Dockerfile

@@ -1,4 +1,4 @@
-FROM alpine:3.12 as builder
+FROM alpine:3.16 as builder
 
 ARG RELEASE_VERSION
 
@@ -7,5 +7,6 @@ RUN wget https://github.com/buger/goreplay/releases/download/${RELEASE_VERSION}/
 RUN tar xzf gor.tar.gz
 
 FROM scratch
+COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
 COPY --from=builder /gor .
 ENTRYPOINT ["./gor"]

Dockerfile.dev

@@ -1,14 +1,12 @@
-FROM golang:1.14
+FROM golang:1.19
 
 RUN apt-get update && apt-get install ruby vim-common -y
 
 RUN apt-get install flex bison -y
-RUN wget http://www.tcpdump.org/release/libpcap-1.9.1.tar.gz && tar xzf libpcap-1.9.1.tar.gz && cd libpcap-1.9.1 && ./configure && make install
-
-RUN go get github.com/google/gopacket
-RUN go get -u golang.org/x/lint/golint
+RUN wget http://www.tcpdump.org/release/libpcap-1.10.0.tar.gz && tar xzf libpcap-1.10.0.tar.gz && cd libpcap-1.10.0 && ./configure && make install
 
 WORKDIR /go/src/github.com/buger/goreplay/
 ADD . /go/src/github.com/buger/goreplay/
 
+RUN go get golang.org/x/lint/golint
 RUN go get

Makefile

@@ -1,69 +1,98 @@
 SOURCE = $(shell ls -1 *.go | grep -v _test.go)
+PROJECT_NAME := goreplay
 SOURCE_PATH = /go/src/github.com/buger/goreplay/
 PORT = 8000
 FADDR = :8000
-CONTAINER=gor
-PREFIX=
-RUN = docker run -v `pwd`:$(SOURCE_PATH) -e AWS_ACCESS_KEY_ID=$(AWS_ACCESS_KEY_ID) -e AWS_SECRET_ACCESS_KEY=$(AWS_SECRET_ACCESS_KEY) -p 0.0.0.0:$(PORT):$(PORT) -t -i $(CONTAINER)
+DIST_PATH = dist
+CONTAINER_AMD=gor-amd64
+CONTAINER_ARM=gor-arm64
+RUN = docker run --rm -v `pwd`:$(SOURCE_PATH) -e AWS_ACCESS_KEY_ID=$(AWS_ACCESS_KEY_ID) -e AWS_SECRET_ACCESS_KEY=$(AWS_SECRET_ACCESS_KEY) -p 0.0.0.0:$(PORT):$(PORT) -t -i $(CONTAINER_AMD)
 BENCHMARK = BenchmarkRAWInput
 TEST = TestRawListenerBench
 BIN_NAME = gor
-VERSION = DEV-$(shell date +%s)
-LDFLAGS = -ldflags "-X main.VERSION=$(VERSION)$(PREFIX) -extldflags \"-static\" -X main.DEMO=$(DEMO)"
-MAC_LDFLAGS = -ldflags "-X main.VERSION=$(VERSION)$(PREFIX) -X main.DEMO=$(DEMO)"
-FADDR = ":8000"
+VERSION := DEV-$(shell date +%s)
+LDFLAGS = -ldflags "-X main.VERSION=$(VERSION) -extldflags \"-static\" -X main.DEMO=$(DEMO)"
+MAC_LDFLAGS = -ldflags "-X main.VERSION=$(VERSION) -X main.DEMO=$(DEMO)"
+DOCKER_FPM_CMD := docker run --rm -t -v `pwd`:/src -w /src fleetdm/fpm
 
-FPMCOMMON= \
-    --name goreplay \
+FPM_COMMON= \
+    --name $(PROJECT_NAME) \
     --description "GoReplay is an open-source network monitoring tool which can record your live traffic, and use it for shadowing, load testing, monitoring and detailed analysis." \
     -v $(VERSION) \
     --vendor "Leonid Bugaev" \
     -m "<support@goreplay.org>" \
     --url "https://goreplay.org" \
-    -s dir \
-    -C /tmp/gor-build \
+    -s dir
 
-release: release-x64 release-mac
+release: clean release-linux-amd64 release-linux-arm64 release-mac-amd64 release-mac-arm64 release-windows
 
-release-bin:
-	docker run -v `pwd`:$(SOURCE_PATH) -t --env GOOS=linux --env GOARCH=amd64  -i $(CONTAINER) go build -o $(BIN_NAME) -tags netgo $(LDFLAGS)
+.PHONY: vendor
+vendor:
+	go mod vendor
 
-release-bin-mac:
-	GOOS=darwin go build -o $(BIN_NAME) $(MAC_LDFLAGS)
+release-bin-linux-amd64: vendor
+	docker run --platform linux/amd64 --rm -v `pwd`:$(SOURCE_PATH) -t --env GOOS=linux --env GOARCH=amd64 -i $(CONTAINER_AMD) go build -mod=vendor -o $(BIN_NAME) -tags netgo $(LDFLAGS) ./cmd/gor/
 
-release-x64:
-	docker run -v `pwd`:$(SOURCE_PATH) -t --env GOOS=linux --env GOARCH=amd64  -i $(CONTAINER) go build -o $(BIN_NAME) -tags netgo $(LDFLAGS)
-	tar -czf gor_$(VERSION)$(PREFIX)_x64.tar.gz $(BIN_NAME)
-	mkdir -p /tmp/gor-build
-	mv ./$(BIN_NAME) /tmp/gor-build/$(BIN_NAME)
-	cd /tmp/gor-build
-	rm -f goreplay_$(VERSION)_amd64.deb
-	rm -f goreplay-$(VERSION)-1.x86_64.rpm
-	fpm $(FPMCOMMON) -a amd64 -t deb ./=/usr/local/bin
-	fpm $(FPMCOMMON) -a amd64 -t rpm ./=/usr/local/bin
-	rm -rf /tmp/gor-build
+release-bin-linux-arm64: vendor
+	docker run --platform linux/arm64 --rm -v `pwd`:$(SOURCE_PATH) -t --env GOOS=linux --env GOARCH=arm64 -i $(CONTAINER_ARM) go build -mod=vendor -o $(BIN_NAME) -tags netgo $(LDFLAGS) ./cmd/gor/
 
-release-x86:
-	docker run -v `pwd`:$(SOURCE_PATH) -t --env GOOS=linux --env GOARCH=386 -i $(CONTAINER) go build -o $(BIN_NAME) -tags netgo $(LDFLAGS)
-	tar -czf gor_$(VERSION)$(PREFIX)_x86.tar.gz $(BIN_NAME)
-	rm $(BIN_NAME)
+release-bin-mac-amd64: vendor
+	GOOS=darwin go build -mod=vendor -o $(BIN_NAME) $(MAC_LDFLAGS) ./cmd/gor/
 
-release-mac:
-	go build -o $(BIN_NAME) $(MAC_LDFLAGS)
-	tar -czf gor_$(VERSION)$(PREFIX)_mac.tar.gz $(BIN_NAME)
-	mkdir -p /tmp/gor-build
-	mv ./$(BIN_NAME) /tmp/gor-build/$(BIN_NAME)
-	cd /tmp/gor-build
-	rm -f goreplay-$(VERSION).pkg
-	fpm $(FPMCOMMON) -a amd64 -t osxpkg ./=/usr/local/bin
-	rm -rf /tmp/gor-build
+release-bin-mac-arm64: vendor
+	GOOS=darwin GOARCH=arm64 CGO_ENABLED=1 go build -mod=vendor -o $(BIN_NAME) $(MAC_LDFLAGS)
 
+release-bin-windows: vendor
+	docker run -it --rm -v `pwd`:$(SOURCE_PATH) -w $(SOURCE_PATH) -e CGO_ENABLED=1 docker.elastic.co/beats-dev/golang-crossbuild:1.19.2-main --build-cmd "make VERSION=$(VERSION) build" -p "windows/amd64" ./cmd/gor/
+	mv $(BIN_NAME) "$(BIN_NAME).exe"
+
+release-linux-amd64: dist release-bin-linux-amd64
+	tar -czf $(DIST_PATH)/gor_$(VERSION)_linux_amd64.tar.gz $(BIN_NAME)
+	$(DOCKER_FPM_CMD) $(FPM_COMMON) -f -t deb -a arm64 -p ./$(DIST_PATH) ./gor=/usr/local/bin
+	$(DOCKER_FPM_CMD) $(FPM_COMMON) -f -t rpm -a arm64 -p ./$(DIST_PATH) ./gor=/usr/local/bin
+	rm -rf $(BIN_NAME)
+
+release-linux-arm64: dist release-bin-linux-arm64
+	tar -czf $(DIST_PATH)/gor_$(VERSION)_linux_arm64.tar.gz $(BIN_NAME)
+	$(DOCKER_FPM_CMD) $(FPM_COMMON) -f -t deb -a arm64 -p ./$(DIST_PATH) ./gor=/usr/local/bin
+	$(DOCKER_FPM_CMD) $(FPM_COMMON) -f -t rpm -a arm64 -p ./$(DIST_PATH) ./gor=/usr/local/bin
+	rm -rf $(BIN_NAME)
+
+release-mac-amd64: dist release-bin-mac-amd64
+	tar -czf $(DIST_PATH)/gor_$(VERSION)_darwin_amd64.tar.gz $(BIN_NAME)
+	fpm $(FPM_COMMON) -f -t osxpkg -a amd64 -p ./$(DIST_PATH) ./gor=/usr/local/bin
+	mv ./$(DIST_PATH)/$(PROJECT_NAME)-$(VERSION).pkg ./$(DIST_PATH)/$(PROJECT_NAME)-$(VERSION)-amd64.pkg
+	rm -rf $(BIN_NAME)
+
+release-mac-arm64: dist release-bin-mac-arm64
+	tar -czf $(DIST_PATH)/gor_$(VERSION)_darwin_arm64.tar.gz $(BIN_NAME)
+	fpm $(FPM_COMMON) -f -t osxpkg -a arm64 -p ./$(DIST_PATH) ./gor=/usr/local/bin
+	mv ./$(DIST_PATH)/$(PROJECT_NAME)-$(VERSION).pkg ./$(DIST_PATH)/$(PROJECT_NAME)-$(VERSION)-arm64.pkg
+	rm -rf $(BIN_NAME)
+
+release-windows: dist release-bin-windows
+	zip $(DIST_PATH)/gor-$(VERSION)_windows.zip "$(BIN_NAME).exe"
+	rm -rf "$(BIN_NAME).exe"
+
+clean:
+	rm -rf $(DIST_PATH)
+
+build:
+	go build -mod=vendor -o $(BIN_NAME) $(LDFLAGS)
 
 install:
 	go install $(MAC_LDFLAGS)
 
-build:
-	docker build -t $(CONTAINER) -f Dockerfile.dev .
+build-env: build-amd64-env build-arm64-env
+
+build-amd64-env:
+	docker buildx build --platform linux/amd64 -t $(CONTAINER_AMD) -f Dockerfile.dev .
+
+build-arm64-env:
+	docker buildx build --platform linux/arm64 -t $(CONTAINER_ARM) -f Dockerfile.dev .
+
+build-docker:
+	docker build -t gor-dev -f Dockerfile .
 
 profile:
 	go build && ./$(BIN_NAME) --output-http="http://localhost:9000" --input-dummy 0 --input-raw :9000 --input-http :9000 --memprofile=./mem.out --cpuprofile=./cpu.out --stats --output-http-stats --output-http-timeout 100ms
@@ -128,19 +157,5 @@ replay:
 bash:
 	$(RUN) /bin/bash
 
-
-FPMCOMMON= \
-    --name gor \
-    --description "GoReplay is an open-source network monitoring tool which can record your live traffic, and use it for shadowing, load testing, monitoring and detailed analysis." \
-    -v $(VERSION) \
-    --vendor "Leonid Bugaev" \
-    -m "<support@goreplay.org>" \
-    --url "https://goreplay.org" \
-    -s dir \
-    -C /tmp/gor-build \
-
-build_packages:
-	mkdir -p /tmp/gor-build
-	go build -i -o /tmp/gor-build/$(BIN_NAME)
-	fpm $(FPMCOMMON) -a amd64 -t deb ./=/usr/local/bin
-	fpm $(FPMCOMMON) -a amd64 -t rpm ./=/usr/local/bin
+dist:
+	mkdir -p $(DIST_PATH)

README.md

@@ -4,7 +4,7 @@
 
 ## https://goreplay.org/
 
-GoReplay is an open-source network monitoring tool which can record your live traffic, and use it for shadowing, load testing, monitoring and detailed analysis.
+GoReplay is an open-source network monitoring tool which can record your live traffic and use it for shadowing, load testing, monitoring and detailed analysis.
 
 ## About
 
@@ -13,29 +13,28 @@ As your application grows, the effort required to test it also grows exponential
 GoReplay increases your confidence in code deployments, configuration and infrastructure changes.
 
 
-GoReplay offers unique approach for shadowing. Instead of being a proxy, GoReplay in background listen for traffic on your network interface, requiring no changes in your production infrastructure, rather then running GoReplay daemon on the same machine as your service.
+GoReplay offers a unique approach for shadowing. Instead of being a proxy, GoReplay listens in the background for traffic on your network interfaces, requiring no changes in your production infrastructure, other than running GoReplay daemon on the same machine as your service.
 
 ![Diagram](https://i.imgur.com/IN2xfDm.png)
 
 Check [latest documentation](http://github.com/buger/goreplay/wiki).
 
 ## Installation
-Download latest binary from https://github.com/buger/goreplay/releases or [compile by yourself](https://github.com/buger/goreplay/wiki/Compilation).
+Download the latest binary from https://github.com/buger/goreplay/releases or [compile by yourself](https://github.com/buger/goreplay/wiki/Compilation).
 
 ## Getting started
 
 The most basic setup will be `sudo ./gor --input-raw :8000 --output-stdout` which acts like tcpdump.
-If you already have test environment you can start replaying: `sudo ./gor --input-raw :8000 --output-http http://staging.env`.
+If you already have a test environment, you can start replaying by running: `sudo ./gor --input-raw :8000 --output-http http://staging.env`.
 
-See the our [documentation](https://github.com/buger/goreplay/wiki/) and [Getting started](https://github.com/buger/goreplay/wiki/Getting-Started) page for more info. 
+See our [documentation](https://github.com/buger/goreplay/wiki/) and the [Getting Started](https://github.com/buger/goreplay/wiki/Getting-Started) page for more info. 
 
 ## Newsletter
-Subscribe to our [newsletter](https://www.getdrip.com/forms/89690474/submissions/new) to stay informed about the latest features and changes to Gor project.
-
+Subscribe to our [newsletter](https://www.getdrip.com/forms/89690474/submissions/new) to stay informed about the latest features and changes to the Gor project.
 
 ## Want to Upgrade?
 
-We have created a [GoReplay PRO](https://goreplay.org/pro.html) extension which provides additional features such as support for binary protocols like Thrift or ProtocolBuffers, saving and replaying from cloud storage, TCP sessions replication, etc. The PRO version also includes a commercial-friendly license, dedicated support, and it also allows you to support high-quality open source development. 
+We have created a [GoReplay PRO](https://goreplay.org/pro.html) extension which provides additional features such as support for binary protocols like Thrift or ProtocolBuffers, saving and replaying from cloud storage, TCP session replication, etc. The PRO version also includes a commercial-friendly license, dedicated support, and it also allows you to support high-quality open source development. 
 
 
 ## Problems?

capture/capture.go

@@ -1,546 +0,0 @@
-package capture
-
-import (
-	"context"
-	"errors"
-	"fmt"
-	"io"
-	"log"
-	"net"
-	"os"
-	"runtime"
-	"sync"
-	"time"
-
-	"github.com/buger/goreplay/size"
-	"github.com/buger/goreplay/tcp"
-
-	"github.com/google/gopacket"
-	"github.com/google/gopacket/layers"
-	"github.com/google/gopacket/pcap"
-	"github.com/pbnjay/memory"
-	"golang.org/x/sys/unix"
-)
-
-// PacketHandler is a function that is used to handle packets
-type PacketHandler func(*tcp.Packet)
-
-// PcapOptions options that can be set on a pcap capture handle,
-// these options take effect on inactive pcap handles
-type PcapOptions struct {
-	BufferTimeout time.Duration `json:"input-raw-buffer-timeout"`
-	TimestampType string        `json:"input-raw-timestamp-type"`
-	BPFFilter     string        `json:"input-raw-bpf-filter"`
-	BufferSize    size.Size     `json:"input-raw-buffer-size"`
-	Promiscuous   bool          `json:"input-raw-promisc"`
-	Monitor       bool          `json:"input-raw-monitor"`
-	Snaplen       bool          `json:"input-raw-override-snaplen"`
-}
-
-// Listener handle traffic capture, this is its representation.
-type Listener struct {
-	sync.Mutex
-	Transport  string       // transport layer default to tcp
-	Activate   func() error // function is used to activate the engine. it must be called before reading packets
-	Handles    map[string]gopacket.ZeroCopyPacketDataSource
-	Interfaces []net.Interface
-	loopIndex  int
-	Reading    chan bool // this channel is closed when the listener has started reading packets
-	PcapOptions
-	Engine        EngineType
-	port          uint16 // src or/and dst port
-	trackResponse bool
-
-	host          string // pcap file name or interface (name, hardware addr, index or ip address)
-	maxAllowedMem int
-
-	closeDone chan struct{}
-	quit      chan struct{}
-}
-
-// EngineType ...
-type EngineType uint8
-
-// Available engines for intercepting traffic
-const (
-	EnginePcap EngineType = 1 << iota
-	EnginePcapFile
-	EngineRawSocket
-)
-
-// Set is here so that EngineType can implement flag.Var
-func (eng *EngineType) Set(v string) error {
-	switch v {
-	case "", "libpcap":
-		*eng = EnginePcap
-	case "pcap_file":
-		*eng = EnginePcapFile
-	case "raw_socket", "af_packet":
-		*eng = EngineRawSocket
-	default:
-		return fmt.Errorf("invalid engine %s", v)
-	}
-	return nil
-}
-
-func (eng *EngineType) String() (e string) {
-	switch *eng {
-	case EnginePcapFile:
-		e = "pcap_file"
-	case EnginePcap:
-		e = "libpcap"
-	case EngineRawSocket:
-		e = "raw_socket"
-	default:
-		e = ""
-	}
-	return e
-}
-
-// NewListener creates and initialize a new Listener. if transport or/and engine are invalid/unsupported
-// is "tcp" and "pcap", are assumed. l.Engine and l.Transport can help to get the values used.
-// if there is an error it will be associated with getting network interfaces
-func NewListener(host string, port uint16, transport string, engine EngineType, trackResponse bool, maxAllowedMem int) (l *Listener, err error) {
-	l = &Listener{}
-
-	l.host = host
-	l.port = port
-	l.Transport = "tcp"
-	if transport != "" {
-		l.Transport = transport
-	}
-	l.Handles = make(map[string]gopacket.ZeroCopyPacketDataSource)
-	l.trackResponse = trackResponse
-	l.closeDone = make(chan struct{})
-	l.quit = make(chan struct{})
-	l.Reading = make(chan bool)
-	l.maxAllowedMem = maxAllowedMem
-	switch engine {
-	default:
-		l.Engine = EnginePcap
-		l.Activate = l.activatePcap
-	case EngineRawSocket:
-		l.Engine = EngineRawSocket
-		l.Activate = l.activateRawSocket
-	case EnginePcapFile:
-		l.Engine = EnginePcapFile
-		l.Activate = l.activatePcapFile
-		return
-	}
-	err = l.setInterfaces()
-	if err != nil {
-		return nil, err
-	}
-	return
-}
-
-// SetPcapOptions set pcap options for all yet to be actived pcap handles
-// setting this on already activated handles will not have any effect
-func (l *Listener) SetPcapOptions(opts PcapOptions) {
-	l.PcapOptions = opts
-}
-
-// Listen listens for packets from the handles, and call handler on every packet received
-// until the context done signal is sent or there is unrecoverable error on all handles.
-// this function must be called after activating pcap handles
-func (l *Listener) Listen(ctx context.Context, handler PacketHandler) (err error) {
-	l.read(handler)
-	done := ctx.Done()
-	select {
-	case <-done:
-		close(l.quit) // signal close on all handles
-		<-l.closeDone // wait all handles to be closed
-		err = ctx.Err()
-	case <-l.closeDone: // all handles closed voluntarily
-	}
-	return
-}
-
-// ListenBackground is like listen but can run concurrently and signal error through channel
-func (l *Listener) ListenBackground(ctx context.Context, handler PacketHandler) chan error {
-	err := make(chan error, 1)
-	go func() {
-		defer close(err)
-		if e := l.Listen(ctx, handler); err != nil {
-			err <- e
-		}
-	}()
-	return err
-}
-
-// Filter returns automatic filter applied by goreplay
-// to a pcap handle of a specific interface
-func (l *Listener) Filter(ifi net.Interface) (filter string) {
-	// https://www.tcpdump.org/manpages/pcap-filter.7.html
-
-	port := fmt.Sprintf("portrange 0-%d", 1<<16-1)
-	if l.port != 0 {
-		port = fmt.Sprintf("port %d", l.port)
-	}
-	filter = fmt.Sprintf("%s dst %s", l.Transport, port)
-	if l.trackResponse {
-		filter = fmt.Sprintf("%s %s", l.Transport, port)
-	}
-
-	if listenAll(l.host) || isDevice(l.host, ifi) {
-		return "(" + filter + ")"
-	}
-	filter = fmt.Sprintf("(host %s and (%s))", l.host, filter)
-
-	log.Println("BPF filter: " + filter)
-	return
-}
-
-// PcapDumpHandler returns a handler to write packet data in PCAP
-// format, See http://wiki.wireshark.org/Development/LibpcapFileFormathandler.
-// if link layer is invalid Ethernet is assumed
-// func PcapDumpHandler(file *os.File, link layers.LinkType) (handler func(packet *tcp.Packet) error, err error) {
-// 	if link.String() == "" {
-// 		link = layers.LinkTypeEthernet
-// 	}
-// 	w := NewWriterNanos(file)
-// 	err = w.WriteFileHeader(64<<10, link)
-// 	if err != nil {
-// 		return nil, err
-// 	}
-// 	return func(packet *tcp.Packet) error {
-// 		return w.WritePacket(*packet.Info, packet.Data)
-// 	}, nil
-// }
-
-// PcapHandle returns new pcap Handle from dev on success.
-// this function should be called after setting all necessary options for this listener
-func (l *Listener) PcapHandle(ifi net.Interface) (handle *pcap.Handle, err error) {
-	var inactive *pcap.InactiveHandle
-	inactive, err = pcap.NewInactiveHandle(ifi.Name)
-	if err != nil {
-		return nil, fmt.Errorf("inactive handle error: %q, interface: %q", err, ifi.Name)
-	}
-	defer inactive.CleanUp()
-	if l.TimestampType != "" {
-		var ts pcap.TimestampSource
-		ts, err = pcap.TimestampSourceFromString(l.TimestampType)
-		err = inactive.SetTimestampSource(ts)
-		if err != nil {
-			return nil, fmt.Errorf("%q: supported timestamps: %q, interface: %q", err, inactive.SupportedTimestamps(), ifi.Name)
-		}
-	}
-	if l.Promiscuous {
-		if err = inactive.SetPromisc(l.Promiscuous); err != nil {
-			return nil, fmt.Errorf("promiscuous mode error: %q, interface: %q", err, ifi.Name)
-		}
-	}
-	if l.Monitor {
-		if err = inactive.SetRFMon(l.Monitor); err != nil && !errors.Is(err, pcap.CannotSetRFMon) {
-			return nil, fmt.Errorf("monitor mode error: %q, interface: %q", err, ifi.Name)
-		}
-	}
-	var snap int
-	if l.Snaplen {
-		snap = 64<<10 + 200
-	} else if ifi.MTU > 0 {
-		snap = ifi.MTU + 200
-	}
-	err = inactive.SetSnapLen(snap)
-	if err != nil {
-		return nil, fmt.Errorf("snapshot length error: %q, interface: %q", err, ifi.Name)
-	}
-	if l.BufferSize > 0 {
-		err = inactive.SetBufferSize(int(l.BufferSize))
-		if err != nil {
-			return nil, fmt.Errorf("handle buffer size error: %q, interface: %q", err, ifi.Name)
-		}
-	}
-	if l.BufferTimeout == 0 {
-		l.BufferTimeout = pcap.BlockForever
-	}
-	err = inactive.SetTimeout(l.BufferTimeout)
-	if err != nil {
-		return nil, fmt.Errorf("handle buffer timeout error: %q, interface: %q", err, ifi.Name)
-	}
-	handle, err = inactive.Activate()
-	if err != nil {
-		return nil, fmt.Errorf("PCAP Activate device error: %q, interface: %q", err, ifi.Name)
-	}
-	if l.BPFFilter != "" {
-		if l.BPFFilter[0] != '(' || l.BPFFilter[len(l.BPFFilter)-1] != ')' {
-			l.BPFFilter = "(" + l.BPFFilter + ")"
-		}
-	} else {
-		l.BPFFilter = l.Filter(ifi)
-	}
-	fmt.Println("Interface:", ifi.Name, ". BPF Filter:", l.BPFFilter)
-	err = handle.SetBPFFilter(l.BPFFilter)
-	if err != nil {
-		handle.Close()
-		return nil, fmt.Errorf("BPF filter error: %q%s, interface: %q", err, l.BPFFilter, ifi.Name)
-	}
-	return
-}
-
-// SocketHandle returns new unix ethernet handle associated with this listener settings
-func (l *Listener) SocketHandle(ifi net.Interface) (handle Socket, err error) {
-	handle, err = NewSocket(ifi)
-	if err != nil {
-		return nil, fmt.Errorf("sock raw error: %q, interface: %q", err, ifi.Name)
-	}
-	if err = handle.SetPromiscuous(l.Promiscuous || l.Monitor); err != nil {
-		return nil, fmt.Errorf("promiscuous mode error: %q, interface: %q", err, ifi.Name)
-	}
-	if l.BPFFilter != "" {
-		if l.BPFFilter[0] != '(' || l.BPFFilter[len(l.BPFFilter)-1] != ')' {
-			l.BPFFilter = "(" + l.BPFFilter + ")"
-		}
-	} else {
-		l.BPFFilter = l.Filter(ifi)
-	}
-	fmt.Println("BPF Filter: ", l.BPFFilter)
-	if err = handle.SetBPFFilter(l.BPFFilter); err != nil {
-		handle.Close()
-		return nil, fmt.Errorf("BPF filter error: %q%s, interface: %q", err, l.BPFFilter, ifi.Name)
-	}
-	handle.SetLoopbackIndex(int32(l.loopIndex))
-	return
-}
-
-func (l *Listener) read(handler PacketHandler) {
-	l.Lock()
-	defer l.Unlock()
-	for key, handle := range l.Handles {
-		go func(key string, hndl gopacket.ZeroCopyPacketDataSource) {
-			defer l.closeHandles(key)
-			linkSize := 14
-			linkType := int(layers.LinkTypeEthernet)
-			if _, ok := hndl.(*pcap.Handle); ok {
-				linkType = int(hndl.(*pcap.Handle).LinkType())
-				linkSize, ok = pcapLinkTypeLength(linkType)
-				if !ok {
-					if os.Getenv("GORDEBUG") != "0" {
-						log.Printf("can not identify link type of an interface '%s'\n", key)
-					}
-					return // can't find the linktype size
-				}
-			}
-
-			ticker := time.NewTicker(time.Second)
-
-			for {
-				select {
-				case <-l.quit:
-					return
-				case <-ticker.C:
-					if l.maxAllowedMem > 0 {
-						usedMemory, _ := memUsage()
-						if int(usedMemory) > l.maxAllowedMem {
-							log.Printf("[WARNING] Using more memory (%vmb) than allowed (%vmb). Enabling throttling for 1 second\n", bToMb(usedMemory), bToMb(uint64(l.maxAllowedMem)))
-							time.Sleep(time.Second)
-						}
-					}
-				default:
-					data, ci, err := hndl.ZeroCopyReadPacketData()
-					if err == nil {
-						pckt, err := tcp.ParsePacket(data, linkType, linkSize, &ci)
-						if err == nil {
-							handler(pckt)
-						}
-						continue
-					}
-					if enext, ok := err.(pcap.NextError); ok && enext == pcap.NextErrorTimeoutExpired {
-						continue
-					}
-					if eno, ok := err.(unix.Errno); ok && eno.Temporary() {
-						continue
-					}
-					if enet, ok := err.(*net.OpError); ok && (enet.Temporary() || enet.Timeout()) {
-						continue
-					}
-					if err == io.EOF || err == io.ErrClosedPipe {
-						log.Printf("stopped reading from %s interface with error %s\n", key, err)
-						return
-					}
-
-					log.Printf("stopped reading from %s interface with error %s\n", key, err)
-					return
-				}
-			}
-		}(key, handle)
-	}
-	close(l.Reading)
-}
-
-func (l *Listener) closeHandles(key string) {
-	l.Lock()
-	defer l.Unlock()
-	if handle, ok := l.Handles[key]; ok {
-		if _, ok = handle.(Socket); ok {
-			handle.(Socket).Close()
-		} else {
-			handle.(*pcap.Handle).Close()
-		}
-		delete(l.Handles, key)
-		if len(l.Handles) == 0 {
-			close(l.closeDone)
-		}
-	}
-}
-
-func (l *Listener) activatePcap() error {
-	var e error
-	var msg string
-	for _, ifi := range l.Interfaces {
-		var handle *pcap.Handle
-		handle, e = l.PcapHandle(ifi)
-		if e != nil {
-			msg += ("\n" + e.Error())
-			continue
-		}
-		l.Handles[ifi.Name] = handle
-	}
-	if len(l.Handles) == 0 {
-		return fmt.Errorf("pcap handles error:%s", msg)
-	}
-	return nil
-}
-
-func (l *Listener) activateRawSocket() error {
-	if runtime.GOOS != "linux" {
-		return fmt.Errorf("sock_raw is not stabilized on OS other than linux")
-	}
-	var msg string
-	var e error
-	for _, ifi := range l.Interfaces {
-		var handle Socket
-		handle, e = l.SocketHandle(ifi)
-		if e != nil {
-			msg += ("\n" + e.Error())
-			continue
-		}
-		l.Handles[ifi.Name] = handle
-	}
-	if len(l.Handles) == 0 {
-		return fmt.Errorf("raw socket handles error:%s", msg)
-	}
-	return nil
-}
-
-func (l *Listener) activatePcapFile() (err error) {
-	var handle *pcap.Handle
-	var e error
-	if handle, e = pcap.OpenOffline(l.host); e != nil {
-		return fmt.Errorf("open pcap file error: %q", e)
-	}
-	if l.BPFFilter != "" {
-		if l.BPFFilter[0] != '(' || l.BPFFilter[len(l.BPFFilter)-1] != ')' {
-			l.BPFFilter = "(" + l.BPFFilter + ")"
-		}
-	} else {
-		addr := l.host
-		l.host = ""
-		l.BPFFilter = l.Filter(net.Interface{})
-		l.host = addr
-	}
-	if e = handle.SetBPFFilter(l.BPFFilter); e != nil {
-		handle.Close()
-		return fmt.Errorf("BPF filter error: %q, filter: %s", e, l.BPFFilter)
-	}
-	l.Handles["pcap_file"] = handle
-	return
-}
-
-func (l *Listener) setInterfaces() (err error) {
-	var ifis []net.Interface
-	ifis, err = net.Interfaces()
-	if err != nil {
-		return
-	}
-	for i := range ifis {
-		if ifis[i].Flags&net.FlagLoopback != 0 {
-			l.loopIndex = ifis[i].Index
-		}
-		if ifis[i].Flags&net.FlagUp == 0 {
-			continue
-		}
-		if isDevice(l.host, ifis[i]) {
-			l.Interfaces = []net.Interface{ifis[i]}
-			return
-		}
-		addrs, e := ifis[i].Addrs()
-		if e != nil {
-			// don't give up on a failure from a single interface
-			continue
-		}
-		for _, addr := range addrs {
-			if cutMask(addr) == l.host {
-				l.Interfaces = []net.Interface{ifis[i]}
-				return
-			}
-		}
-	}
-	l.Interfaces = ifis
-	return
-}
-
-func cutMask(addr net.Addr) string {
-	mask := addr.String()
-	for i, v := range mask {
-		if v == '/' {
-			return mask[:i]
-		}
-	}
-	return mask
-}
-
-func isDevice(addr string, ifi net.Interface) bool {
-	return addr == ifi.Name || addr == fmt.Sprintf("%d", ifi.Index) || (addr != "" && addr == ifi.HardwareAddr.String())
-}
-
-func listenAll(addr string) bool {
-	switch addr {
-	case "", "0.0.0.0", "[::]", "::":
-		return true
-	}
-	return false
-}
-
-func pcapLinkTypeLength(lType int) (int, bool) {
-	switch layers.LinkType(lType) {
-	case layers.LinkTypeEthernet:
-		return 14, true
-	case layers.LinkTypeNull, layers.LinkTypeLoop:
-		return 4, true
-	case layers.LinkTypeRaw, 12, 14:
-		return 0, true
-	case layers.LinkTypeIPv4, layers.LinkTypeIPv6:
-		// (TODO:) look out for IP encapsulation?
-		return 0, true
-	case layers.LinkTypeLinuxSLL:
-		return 16, true
-	case layers.LinkTypeFDDI:
-		return 13, true
-	case 226 /*DLT_IPNET*/ :
-		// https://www.tcpdump.org/linktypes/LINKTYPE_IPNET.html
-		return 24, true
-	default:
-		return 0, false
-	}
-}
-
-func memUsage() (uint64, uint64) {
-	var m runtime.MemStats
-	runtime.ReadMemStats(&m)
-	// fmt.Printf("Current memory usage: ")
-	// // For info on each, see: https://golang.org/pkg/runtime/#MemStats
-	// fmt.Printf("Alloc = %v MiB", bToMb(m.Alloc))
-	// fmt.Printf("\tTotalAlloc = %v MiB", bToMb(m.TotalAlloc))
-	// fmt.Printf("\tSys = %v MiB", bToMb(m.Sys))
-	// fmt.Printf("\tNumGC = %v\n", m.NumGC)
-
-	// fmt.Printf("Total available memory %v MiB\n", bToMb(memory.TotalMemory()))
-	// fmt.Printf("Currently using %.2f %% of memory\n", 100*float64(m.Alloc+m.Sys)/float64(memory.TotalMemory()))
-
-	return m.Alloc + m.Sys, memory.TotalMemory()
-}
-
-func bToMb(b uint64) uint64 {
-	return b / 1024 / 1024
-}

capture/capture_test.go

@@ -1,313 +0,0 @@
-package capture
-
-import (
-	"context"
-	"encoding/binary"
-	"io/ioutil"
-	"net"
-	"os"
-	"sync/atomic"
-	"testing"
-	"time"
-
-	"github.com/google/gopacket/layers"
-	"github.com/google/gopacket/pcap"
-)
-
-var LoopBack = func() net.Interface {
-	ifis, _ := net.Interfaces()
-	for _, v := range ifis {
-		if v.Flags&net.FlagLoopback != 0 {
-			return v
-		}
-	}
-	return ifis[0]
-}()
-
-func TestSetInterfaces(t *testing.T) {
-	l := &Listener{}
-	l.host = "127.0.0.1"
-	l.setInterfaces()
-	if len(l.Interfaces) != 1 {
-		t.Error("expected a single interface")
-	}
-	l.host = LoopBack.HardwareAddr.String()
-	l.setInterfaces()
-	if l.Interfaces[0].Name != LoopBack.Name && len(l.Interfaces) != 1 {
-		t.Error("interface should be loop back interface")
-	}
-	l.host = ""
-	l.setInterfaces()
-	if len(l.Interfaces) < 1 {
-		t.Error("should get all interfaces")
-	}
-}
-
-func TestBPFFilter(t *testing.T) {
-	l := &Listener{}
-	l.host = "127.0.0.1"
-	l.Transport = "tcp"
-	l.setInterfaces()
-	filter := l.Filter(l.Interfaces[0])
-	if filter != "(tcp dst portrange 0-65535 and host 127.0.0.1)" {
-		t.Error("wrong filter", filter)
-	}
-	l.port = 8000
-	l.trackResponse = true
-	filter = l.Filter(l.Interfaces[0])
-	if filter != "(tcp port 8000 and host 127.0.0.1)" {
-		t.Error("wrong filter")
-	}
-}
-
-func TestPcapDump(t *testing.T) {
-	f, err := ioutil.TempFile("", "pcap_file")
-	if err != nil {
-		t.Error(err)
-	}
-	h, _ := PcapDumpHandler(f, layers.LinkTypeLoop)
-	packets := Packets(1, 5, 5, 4)
-	for i := 0; i < len(packets); i++ {
-		if i == 1 {
-			// change dst port
-			binary.BigEndian.PutUint16(packets[i].TransLayer[2:], 8001)
-		}
-		if i == 4 {
-			packets[i].Info.CaptureLength = 40
-		}
-		h(packets[i])
-	}
-	name := f.Name()
-	f.Close()
-	testPcapDumpEngine(name, t)
-}
-
-func testPcapDumpEngine(f string, t *testing.T) {
-	defer os.Remove(f)
-	l, err := NewListener(f, 8000, "", EnginePcapFile, true)
-	err = l.Activate()
-	if err != nil {
-		t.Errorf("expected error to be nil, got %q", err)
-		return
-	}
-	pckts := 0
-	ctx, cancel := context.WithCancel(context.Background())
-	defer cancel()
-	err = l.Listen(ctx, func(packet *Packet) {
-		if packet.Info.CaptureLength != 57 {
-			t.Errorf("expected packet length to be %d, got %d", 57, packet.Info.CaptureLength)
-		}
-		pckts++
-	})
-
-	if err != nil {
-		t.Errorf("expected error to be nil, got %q", err)
-	}
-	if pckts != 3 {
-		t.Errorf("expected %d packets, got %d packets", 3, pckts)
-	}
-}
-
-func TestPcapHandler(t *testing.T) {
-	l, err := NewListener(LoopBack.Name, 8000, "", EnginePcap, true)
-	if err != nil {
-		t.Errorf("expected error to be nil, got %v", err)
-		return
-	}
-	err = l.Activate()
-	if err != nil {
-		t.Errorf("expected error to be nil, got %v", err)
-		return
-	}
-	defer l.Handles[LoopBack.Name].(*pcap.Handle).Close()
-	if err != nil {
-		t.Errorf("expected error to be nil, got %v", err)
-		return
-	}
-	for i := 0; i < 5; i++ {
-		_, _ = net.Dial("tcp", "127.0.0.1:8000")
-	}
-	sts, _ := l.Handles[LoopBack.Name].(*pcap.Handle).Stats()
-	if sts.PacketsReceived < 5 {
-		t.Errorf("expected >=5 packets got %d", sts.PacketsReceived)
-	}
-}
-
-// func TestSocketHandler(t *testing.T) {
-// 	l, err := NewListener(LoopBack.Name, 8000, "", EngineRawSocket, true)
-// 	err = l.Activate()
-// 	if err != nil {
-// 		return
-// 	}
-// 	defer l.Handles[LoopBack.Name].(*SockRaw).Close()
-// 	if err != nil {
-// 		t.Errorf("expected error to be nil, got %v", err)
-// 		return
-// 	}
-// 	for i := 0; i < 5; i++ {
-// 		_, _ = net.Dial("tcp", "127.0.0.1:8000")
-// 	}
-// 	sts, _ := l.Handles[LoopBack.Name].(*SockRaw).Stats()
-// 	if sts.Packets < 5 {
-// 		t.Errorf("expected >=5 packets got %d", sts.Packets)
-// 	}
-// }
-
-func BenchmarkPcapDump(b *testing.B) {
-	f, err := ioutil.TempFile("", "pcap_file")
-	if err != nil {
-		b.Error(err)
-		return
-	}
-	now := time.Now()
-	defer os.Remove(f.Name())
-	h, _ := PcapDumpHandler(f, layers.LinkTypeLoop)
-	packets := Packets(1, b.N, 5, 4)
-	for i := 0; i < len(packets); i++ {
-		h(packets[i])
-	}
-	f.Close()
-	b.Logf("%d packets in %s", b.N, time.Since(now))
-}
-
-func BenchmarkPcapFile(b *testing.B) {
-	f, err := ioutil.TempFile("", "pcap_file")
-	if err != nil {
-		b.Error(err)
-		return
-	}
-	defer os.Remove(f.Name())
-	h, _ := PcapDumpHandler(f, layers.LinkTypeLoop)
-	packets := Packets(1, b.N, 5, 4)
-	for i := 0; i < len(packets); i++ {
-		h(packets[i])
-	}
-	name := f.Name()
-	f.Close()
-	b.ResetTimer()
-	var l *Listener
-	l, err = NewListener(name, 8000, "", EnginePcapFile, true)
-	if err != nil {
-		b.Error(err)
-		return
-	}
-	err = l.Activate()
-	if err != nil {
-		b.Error(err)
-		return
-	}
-	now := time.Now()
-	pckts := 0
-	ctx, cancel := context.WithCancel(context.Background())
-	defer cancel()
-	if err = l.Listen(ctx, func(packet *Packet) {
-		if packet.Info.CaptureLength != 49 {
-			b.Errorf("expected packet length to be %d, got %d", 49, packet.Info.CaptureLength)
-		}
-		pckts++
-	}); err != nil {
-		b.Error(err)
-	}
-	b.Logf("%d/%d packets in %s", pckts, b.N, time.Since(now))
-}
-
-// used to benchmark sock engine
-var buf [1024]byte
-
-func init() {
-	for i := 0; i < len(buf); i++ {
-		buf[i] = 0xff
-	}
-}
-
-func handler(n, counter *int32) PacketHandler {
-	return func(p *Packet) {
-		nn := int32(len(p.Data))
-		atomic.AddInt32(n, nn)
-		atomic.AddInt32(counter, 1)
-	}
-}
-
-func BenchmarkPcap(b *testing.B) {
-	var err error
-	n := new(int32)
-	counter := new(int32)
-	l, err := NewListener(LoopBack.Name, 8000, "", EnginePcap, false)
-	if err != nil {
-		b.Error(err)
-		return
-	}
-	l.PcapOptions.BPFFilter = "udp dst port 8000 and host 127.0.0.1"
-	err = l.Activate()
-	if err != nil {
-		b.Error(err)
-		return
-	}
-	errCh := l.ListenBackground(context.Background(), handler(n, counter))
-	select {
-	case <-l.Reading:
-	case err = <-errCh:
-		b.Error(err)
-		return
-	}
-	var conn net.Conn
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		b.StopTimer()
-		conn, err = net.Dial("udp", "127.0.0.1:8000")
-		if err != nil {
-			b.Error(err)
-			return
-		}
-		b.StartTimer()
-		_, err = conn.Write(buf[:])
-		if err != nil {
-			b.Error(err)
-			return
-		}
-	}
-	b.ReportMetric(float64(atomic.LoadInt32(n)), "buf")
-	b.ReportMetric(float64(atomic.LoadInt32(counter)), "packets")
-}
-
-func BenchmarkRawSocket(b *testing.B) {
-	var err error
-	n := new(int32)
-	counter := new(int32)
-	l, err := NewListener(LoopBack.Name, 8000, "", EngineRawSocket, false)
-	if err != nil {
-		b.Error(err)
-		return
-	}
-	l.PcapOptions.BPFFilter = "udp dst port 8000 and host 127.0.0.1"
-	err = l.Activate()
-	if err != nil {
-		b.Error(err)
-		return
-	}
-	errCh := l.ListenBackground(context.Background(), handler(n, counter))
-	select {
-	case <-l.Reading:
-	case err = <-errCh:
-		b.Error(err)
-		return
-	}
-	var conn net.Conn
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		b.StopTimer()
-		conn, err = net.Dial("udp", "127.0.0.1:8000")
-		if err != nil {
-			b.Error(err)
-			return
-		}
-		b.StartTimer()
-		_, err = conn.Write(buf[:])
-		if err != nil {
-			b.Error(err)
-			return
-		}
-	}
-	b.ReportMetric(float64(atomic.LoadInt32(n)), "buf")
-	b.ReportMetric(float64(atomic.LoadInt32(counter)), "packets")
-}

capture/packet_test.go

@@ -1,197 +0,0 @@
-package capture
-
-import (
-	"encoding/binary"
-	"errors"
-	"testing"
-	"time"
-
-	"github.com/google/gopacket"
-	"github.com/google/gopacket/layers"
-)
-
-func generateHeader4(seq uint32, length uint16) []byte {
-	hdr := make([]byte, 4+24+24, 4+24+24)
-	binary.BigEndian.PutUint32(hdr, uint32(layers.ProtocolFamilyIPv4))
-
-	ip := hdr[4:]
-	ip[0] = 4<<4 | 6
-	binary.BigEndian.PutUint16(ip[2:4], length+24+24)
-	ip[9] = uint8(layers.IPProtocolTCP)
-	copy(ip[12:16], []byte{127, 0, 0, 1})
-	copy(ip[16:], []byte{127, 0, 0, 1})
-
-	// set tcp header
-	tcp := ip[24:]
-	tcp[12] = 6 << 4
-	binary.BigEndian.PutUint16(tcp, 5535)
-	binary.BigEndian.PutUint16(tcp[2:], 8000)
-	binary.BigEndian.PutUint32(tcp[4:], seq)
-	return hdr
-}
-
-func generateHeader6(seq uint32, length uint16) []byte {
-	hdr := make([]byte, 4+40+32+24, 4+40+32+24)
-	binary.BigEndian.PutUint32(hdr, uint32(layers.ProtocolFamilyIPv6Linux))
-
-	ip := hdr[4:]
-	ip[0] = 6 << 4
-	binary.BigEndian.PutUint16(ip[4:], length+32+24)
-	var ipAddr [16]byte
-	ipAddr[15] = 0x01
-	copy(ip[8:], ip[:])
-	copy(ip[24:], ip[:])
-	copy(ip[40:], []byte{
-		// net-layer (IPv6-Opts)
-		0x2b, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-		0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-		uint8(layers.IPProtocolTCP), 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-		0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	})
-
-	// set tcp header
-	tcp := ip[40+32:]
-	tcp[12] = 6 << 4
-	binary.BigEndian.PutUint16(tcp, 5535)
-	binary.BigEndian.PutUint16(tcp[2:], 8000)
-	binary.BigEndian.PutUint32(tcp[4:], seq)
-	return hdr
-}
-
-func Packets(start uint32, _len int, length uint16, version byte) []*Packet {
-	var packets = make([]*Packet, _len)
-	for i := start; i < start+uint32(_len); i++ {
-		var h []byte
-		if version == 4 {
-			h = generateHeader4(i, length)
-		} else {
-			h = generateHeader6(i, length)
-		}
-		d := append(h, make([]byte, int(length))...)
-		ci := &gopacket.CaptureInfo{Length: len(d), CaptureLength: len(d), Timestamp: time.Now()}
-		packets[i-start] = NewPacket(d, int(layers.LinkTypeLoop), 4, ci)
-	}
-	return packets
-}
-
-func TestIPv4Packet(t *testing.T) {
-	pckt := packet(append(generateHeader4(1024, 10), make([]byte, 10)...))
-	if pckt.Err != nil {
-		t.Error(pckt)
-		return
-	}
-	if err := packet(pckt.Data[:2]).Err; !errors.Is(err, ErrHdrLength("Link")) {
-		t.Errorf("should fail with %q, got %q", ErrHdrLength("Link"), err)
-		return
-	}
-	if err := packet(pckt.Data[:20]).Err; !errors.Is(err, ErrHdrLength("IPv4")) {
-		t.Errorf("should fail with %q, got %q", ErrHdrLength("IPv4"), err)
-		return
-	}
-	if err := packet(pckt.Data[:27]).Err; !errors.Is(err, ErrHdrLength("IPv4 opts")) {
-		t.Errorf("should fail with %q, got %q", ErrHdrLength("IPv4 opts"), err)
-		return
-	}
-	if err := packet(pckt.Data[:40]).Err; !errors.Is(err, ErrHdrLength("TCP")) {
-		t.Errorf("should fail with %q, got %q", ErrHdrLength("TCP opts"), err)
-		return
-	}
-	if err := packet(pckt.Data[:50]).Err; !errors.Is(err, ErrHdrLength("TCP opts")) {
-		t.Errorf("should fail with %q, got %q", ErrHdrLength("TCP opts"), err)
-		return
-	}
-	pckt.TransLayer[12] = 0x10
-	if err := packet(pckt.Data[:50]).Err; !errors.Is(err, ErrHdrInvalid("TCP's data offset")) {
-		t.Errorf("should fail with %q, got %q", ErrHdrInvalid("TCP's data offset"), err)
-		return
-	}
-	pckt.TransLayer[12] = 0x60
-	if err := packet(pckt.Data[:28]).Err; !errors.Is(err, ErrHdrMissing("TCP")) {
-		t.Errorf("should fail with %q, got %q", ErrHdrMissing("TCP"), err)
-		return
-	}
-	pckt.NetLayer[9] = 0x02
-	if err := packet(pckt.Data).Err; !errors.Is(err, ErrHdrExpected("TCP")) {
-		t.Errorf("should fail with %q, got %q", ErrHdrExpected("TCP"), err)
-		return
-	}
-	pckt.NetLayer[9] = 0x06
-	pckt.NetLayer[0] = 0x44
-	if err := packet(pckt.Data).Err; !errors.Is(err, ErrHdrInvalid("IPv4's IHL")) {
-		t.Errorf("should fail with %q, got %q", ErrHdrInvalid("IPv4's IHL"), err)
-		return
-	}
-	pckt.NetLayer[0] = 0x56
-	if err := packet(pckt.Data).Err; !errors.Is(err, ErrHdrExpected("IPv4 or IPv6")) {
-		t.Errorf("should fail with %q, got %q", ErrHdrExpected("IPv4 or IPv6"), err)
-		return
-	}
-}
-
-func TestIPv6Packet(t *testing.T) {
-	pckt := packet(append(generateHeader6(1024, 10), make([]byte, 10)...))
-	if pckt.Err != nil {
-		t.Error(pckt)
-		return
-	}
-	if err := packet(pckt.Data[:4]).Err; !errors.Is(err, ErrHdrMissing("IPv4 or IPv6")) {
-		t.Errorf("should fail with %q, got %q", ErrHdrMissing("IPv4 or IPv6"), err)
-		return
-	}
-	if err := packet(pckt.Data[:40]).Err; !errors.Is(err, ErrHdrLength("IPv6")) {
-		t.Errorf("should fail with %q, got %q", ErrHdrLength("IPv6"), err)
-		return
-	}
-	if err := packet(pckt.Data[:52]).Err; !errors.Is(err, ErrHdrLength("IPv6 opts")) {
-		t.Errorf("should fail with %q, got %q", ErrHdrLength("IPv6 opts"), err)
-		return
-	}
-	if err := packet(pckt.Data[:80]).Err; !errors.Is(err, ErrHdrLength("TCP")) {
-		t.Errorf("should fail with %q, got %q", ErrHdrLength("TCP opts"), err)
-		return
-	}
-	if err := packet(pckt.Data[:98]).Err; !errors.Is(err, ErrHdrLength("TCP opts")) {
-		t.Errorf("should fail with %q, got %q", ErrHdrLength("TCP opts"), err)
-		return
-	}
-	pckt.TransLayer[12] = 0x10
-	if err := packet(pckt.Data).Err; !errors.Is(err, ErrHdrInvalid("TCP's data offset")) {
-		t.Errorf("should fail with %q, got %q", ErrHdrInvalid("TCP's data offset"), err)
-		return
-	}
-	pckt.TransLayer[12] = 0x60
-	if err := packet(pckt.Data[:76]).Err; !errors.Is(err, ErrHdrMissing("TCP")) {
-		t.Errorf("should fail with %q, got %q", ErrHdrMissing("TCP"), err)
-		return
-	}
-	pckt.NetLayer[56] = 0x02
-	if err := packet(pckt.Data).Err; !errors.Is(err, ErrHdrExpected("TCP")) {
-		t.Errorf("should fail with %q, got %q", ErrHdrExpected("TCP"), err)
-		return
-	}
-	pckt.NetLayer[56] = 0x06
-}
-
-func packet(data []byte) *Packet {
-	return NewPacket(data, int(layers.LinkTypeLoop), 4, &gopacket.CaptureInfo{})
-}
-
-func BenchmarkNewPacketIPv4(b *testing.B) {
-	data := append(generateHeader4(1204, 10), make([]byte, 10)...)
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		if err := packet(data).Err; err != nil {
-			b.Error(err)
-		}
-	}
-}
-
-func BenchmarkNewPacketIPv6(b *testing.B) {
-	data := append(generateHeader6(1024, 10), make([]byte, 10)...)
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		if err := packet(data).Err; err != nil {
-			b.Error(err)
-		}
-	}
-}

cmd/gor/gor.go

@@ -0,0 +1,265 @@
+// Gor is simple http traffic replication tool written in Go. Its main goal to replay traffic from production servers to staging and dev environments.
+// Now you can test your code on real user sessions in an automated and repeatable fashion.
+package main
+
+import (
+	"expvar"
+	"flag"
+	"fmt"
+	"net/http"
+	"net/http/httputil"
+	httppptof "net/http/pprof"
+	"os"
+	"os/signal"
+	"runtime"
+	"runtime/pprof"
+	"strings"
+	"syscall"
+	"time"
+
+	"github.com/buger/goreplay/pkg/binary"
+	"github.com/buger/goreplay/pkg/dummy"
+	"github.com/buger/goreplay/pkg/emitter"
+	"github.com/buger/goreplay/pkg/file"
+	gor_http "github.com/buger/goreplay/pkg/http"
+	"github.com/buger/goreplay/pkg/kafka"
+	"github.com/buger/goreplay/pkg/null"
+	"github.com/buger/goreplay/pkg/plugin"
+	"github.com/buger/goreplay/pkg/raw"
+	"github.com/buger/goreplay/pkg/settings"
+	"github.com/buger/goreplay/pkg/tcp"
+	"github.com/buger/goreplay/pkg/ws"
+
+	"github.com/rs/zerolog/log"
+)
+
+// Settings used for quick access to CLI flags
+var Settings = settings.Settings
+
+// NewPlugins specify and initialize all available plugins
+func NewPlugins() *plugin.InOutPlugins {
+	plugins := new(plugin.InOutPlugins)
+
+	for _, options := range Settings.InputDummy {
+		plugins.RegisterPlugin(dummy.NewDummyInput, options)
+	}
+
+	for range Settings.OutputDummy {
+		plugins.RegisterPlugin(dummy.NewDummyOutput)
+	}
+
+	if Settings.OutputStdout {
+		plugins.RegisterPlugin(dummy.NewDummyOutput)
+	}
+
+	if Settings.OutputNull {
+		plugins.RegisterPlugin(null.NewNullOutput)
+	}
+
+	for _, options := range Settings.InputRAW {
+		plugins.RegisterPlugin(raw.NewRAWInput, options, Settings.InputRAWConfig)
+	}
+
+	for _, options := range Settings.InputTCP {
+		plugins.RegisterPlugin(tcp.NewTCPInput, options, &Settings.InputTCPConfig)
+	}
+
+	for _, options := range Settings.OutputTCP {
+		plugins.RegisterPlugin(tcp.NewTCPOutput, options, &Settings.OutputTCPConfig)
+	}
+
+	for _, options := range Settings.OutputWebSocket {
+		plugins.RegisterPlugin(ws.NewWebSocketOutput, options, &Settings.OutputWebSocketConfig)
+	}
+
+	for _, options := range Settings.InputFile {
+		plugins.RegisterPlugin(file.NewFileInput, options, Settings.InputFileLoop, Settings.InputFileReadDepth, Settings.InputFileMaxWait, Settings.InputFileDryRun)
+	}
+
+	for _, path := range Settings.OutputFile {
+		if strings.HasPrefix(path, "s3://") {
+			plugins.RegisterPlugin(file.NewS3Output, path, &Settings.OutputFileConfig)
+		} else {
+			plugins.RegisterPlugin(file.NewFileOutput, path, &Settings.OutputFileConfig)
+		}
+	}
+
+	for _, options := range Settings.InputHTTP {
+		plugins.RegisterPlugin(gor_http.NewHTTPInput, options)
+	}
+
+	// If we explicitly set Host header http output should not rewrite it
+	// Fix: https://github.com/buger/gor/issues/174
+	for _, header := range Settings.ModifierConfig.Headers {
+		if header.Name == "Host" {
+			Settings.OutputHTTPConfig.OriginalHost = true
+			break
+		}
+	}
+
+	for _, options := range Settings.OutputHTTP {
+		plugins.RegisterPlugin(gor_http.NewHTTPOutput, options, &Settings.OutputHTTPConfig)
+	}
+
+	for _, options := range Settings.OutputBinary {
+		plugins.RegisterPlugin(binary.NewBinaryOutput, options, &Settings.OutputBinaryConfig)
+	}
+
+	if Settings.OutputKafkaConfig.Host != "" && Settings.OutputKafkaConfig.Topic != "" {
+		plugins.RegisterPlugin(kafka.NewKafkaOutput, "", &Settings.OutputKafkaConfig, &Settings.KafkaTLSConfig)
+	}
+
+	if Settings.InputKafkaConfig.Host != "" && Settings.InputKafkaConfig.Topic != "" {
+		plugins.RegisterPlugin(kafka.NewKafkaInput, "", &Settings.InputKafkaConfig, &Settings.KafkaTLSConfig)
+	}
+
+	return plugins
+}
+
+var (
+	cpuprofile = flag.String("cpuprofile", "", "write cpu profile to file")
+	memprofile = flag.String("memprofile", "", "write memory profile to this file")
+)
+
+func init() {
+	var defaultServeMux http.ServeMux
+	http.DefaultServeMux = &defaultServeMux
+
+	http.HandleFunc("/debug/vars", func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "application/json; charset=utf-8")
+		fmt.Fprintf(w, "{\n")
+		first := true
+		expvar.Do(func(kv expvar.KeyValue) {
+			if kv.Key == "memstats" || kv.Key == "cmdline" {
+				return
+			}
+
+			if !first {
+				fmt.Fprintf(w, ",\n")
+			}
+			first = false
+			fmt.Fprintf(w, "%q: %s", kv.Key, kv.Value)
+		})
+		fmt.Fprintf(w, "\n}\n")
+	})
+
+	http.HandleFunc("/debug/pprof/", httppptof.Index)
+	http.HandleFunc("/debug/pprof/cmdline", httppptof.Cmdline)
+	http.HandleFunc("/debug/pprof/profile", httppptof.Profile)
+	http.HandleFunc("/debug/pprof/symbol", httppptof.Symbol)
+	http.HandleFunc("/debug/pprof/trace", httppptof.Trace)
+}
+
+func loggingMiddleware(addr string, next http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path == "/loop" {
+			_, err := http.Get("http://" + addr)
+
+			if err != nil {
+				log.Error().Err(err).Msg("Error while calling loop endpoint")
+			}
+		}
+
+		rb, _ := httputil.DumpRequest(r, false)
+		log.Info().Msg(string(rb))
+		next.ServeHTTP(w, r)
+	})
+}
+
+func main() {
+	if os.Getenv("GOMAXPROCS") == "" {
+		runtime.GOMAXPROCS(runtime.NumCPU() * 2)
+	}
+
+	args := os.Args[1:]
+	var plugins *plugin.InOutPlugins
+	if len(args) > 0 && args[0] == "file-server" {
+		if len(args) != 2 {
+			log.Fatal().Msg("You should specify port and IP (optional) for the file server. Example: `gor file-server :80`")
+		}
+		dir, _ := os.Getwd()
+
+		log.Info().Msgf("Started example file server for current directory on address %s", args[1])
+
+		if err := http.ListenAndServe(args[1], loggingMiddleware(args[1], http.FileServer(http.Dir(dir)))); err != nil {
+			log.Fatal().Err(err).Msg("Failed to start file server")
+		}
+	} else {
+		flag.Parse()
+		settings.CheckSettings()
+		plugins = NewPlugins()
+	}
+
+	log.Printf("[PPID %d and PID %d] Version:%s\n", os.Getppid(), os.Getpid(), settings.VERSION)
+
+	if len(plugins.Inputs) == 0 || len(plugins.Outputs) == 0 {
+		log.Fatal().Msg("Required at least 1 input and 1 output")
+	}
+
+	if *memprofile != "" {
+		profileMEM(*memprofile)
+	}
+
+	if *cpuprofile != "" {
+		profileCPU(*cpuprofile)
+	}
+
+	if settings.Settings.Pprof != "" {
+		go func() {
+			if err := http.ListenAndServe(settings.Settings.Pprof, nil); err != nil {
+				log.Fatal().Err(err).Msg("Failed to start pprof server")
+			}
+		}()
+	}
+
+	closeCh := make(chan int)
+	emitter := emitter.New(&settings.Settings.EmitterConfig)
+	go emitter.Start(plugins)
+	if settings.Settings.ExitAfter > 0 {
+		log.Printf("Running gor for a duration of %s\n", settings.Settings.ExitAfter)
+
+		time.AfterFunc(settings.Settings.ExitAfter, func() {
+			log.Printf("gor run timeout %s\n", settings.Settings.ExitAfter)
+			close(closeCh)
+		})
+	}
+	c := make(chan os.Signal, 1)
+	signal.Notify(c, os.Interrupt, syscall.SIGTERM)
+	exit := 0
+	select {
+	case <-c:
+		exit = 1
+	case <-closeCh:
+		exit = 0
+	}
+	emitter.Close()
+	os.Exit(exit)
+}
+
+func profileCPU(cpuprofile string) {
+	if cpuprofile != "" {
+		f, err := os.Create(cpuprofile)
+		if err != nil {
+			log.Fatal().Err(err).Msg("Failed to create cpu profile file")
+		}
+		pprof.StartCPUProfile(f)
+
+		time.AfterFunc(30*time.Second, func() {
+			pprof.StopCPUProfile()
+			f.Close()
+		})
+	}
+}
+
+func profileMEM(memprofile string) {
+	if memprofile != "" {
+		f, err := os.Create(memprofile)
+		if err != nil {
+			log.Fatal().Err(err).Msg("Failed to create memory profile file")
+		}
+		time.AfterFunc(30*time.Second, func() {
+			pprof.WriteHeapProfile(f)
+			f.Close()
+		})
+	}
+}

docs/Capturing-and-replaying-traffic.md

@@ -37,6 +37,11 @@ sudo gor --input-raw :80 --input-raw-engine "raw_socket" --output-http "http://s
 
 You can read more about [[Replaying HTTP traffic]].
 
+You can use VXLAN or traffic mirroring from AWS to capture the traffic. The 4789 UDP port will be opened and that works as you are launched GoReplay on the source machine.
+
+```
+gor --input-raw :80 --input-raw-engine vxlan -output-stdout
+```
 
 ### Tracking original IP addresses
 You can use `--input-raw-realip-header` option to specify header name: If not blank, injects header with given name and real IP value to the request payload. Usually, this header should be named: `X-Real-IP`, but you can specify any name.

emitter.go

@@ -1,204 +0,0 @@
-package main
-
-import (
-	"fmt"
-	"hash/fnv"
-	"io"
-	"log"
-	"runtime"
-	"sync"
-	"time"
-
-	"github.com/buger/goreplay/byteutils"
-
-	"github.com/pbnjay/memory"
-)
-
-// Emitter represents an abject to manage plugins communication
-type Emitter struct {
-	sync.WaitGroup
-	plugins *InOutPlugins
-}
-
-// NewEmitter creates and initializes new Emitter object.
-func NewEmitter() *Emitter {
-	return &Emitter{}
-}
-
-// Start initialize loop for sending data from inputs to outputs
-func (e *Emitter) Start(plugins *InOutPlugins, middlewareCmd string) {
-	if Settings.CopyBufferSize < 1 {
-		Settings.CopyBufferSize = 5 << 20
-	}
-	e.plugins = plugins
-
-	if middlewareCmd != "" {
-		middleware := NewMiddleware(middlewareCmd)
-
-		for _, in := range plugins.Inputs {
-			middleware.ReadFrom(in)
-		}
-
-		e.plugins.Inputs = append(e.plugins.Inputs, middleware)
-		e.plugins.All = append(e.plugins.All, middleware)
-		e.Add(1)
-		go func() {
-			defer e.Done()
-			if err := CopyMulty(middleware, plugins.Outputs...); err != nil {
-				Debug(2, fmt.Sprintf("[EMITTER] error during copy: %q", err))
-			}
-		}()
-	} else {
-		for _, in := range plugins.Inputs {
-			e.Add(1)
-			go func(in PluginReader) {
-				defer e.Done()
-				if err := CopyMulty(in, plugins.Outputs...); err != nil {
-					Debug(2, fmt.Sprintf("[EMITTER] error during copy: %q", err))
-				}
-			}(in)
-		}
-	}
-}
-
-// Close closes all the goroutine and waits for it to finish.
-func (e *Emitter) Close() {
-	fmt.Println("Closing!")
-	for _, p := range e.plugins.All {
-		if cp, ok := p.(io.Closer); ok {
-			cp.Close()
-		}
-	}
-	if len(e.plugins.All) > 0 {
-		// wait for everything to stop
-		e.Wait()
-	}
-
-	fmt.Println("Closed!")
-	e.plugins.All = nil // avoid Close to make changes again
-}
-
-// CopyMulty copies from 1 reader to multiple writers
-func CopyMulty(src PluginReader, writers ...PluginWriter) error {
-	wIndex := 0
-	modifier := NewHTTPModifier(&Settings.ModifierConfig)
-	filteredRequests := make(map[string]int64)
-	filteredRequestsLastCleanTime := time.Now().UnixNano()
-	filteredCount := 0
-
-	for {
-		msg, err := src.PluginRead()
-		if err != nil {
-			if err == ErrorStopped || err == io.EOF {
-				return nil
-			}
-			return err
-		}
-
-		if msg != nil && len(msg.Data) > 0 {
-			if len(msg.Data) > int(Settings.CopyBufferSize) {
-				msg.Data = msg.Data[:Settings.CopyBufferSize]
-			}
-			meta := payloadMeta(msg.Meta)
-			if len(meta) < 3 {
-				Debug(2, fmt.Sprintf("[EMITTER] Found malformed record %q from %q", msg.Meta, src))
-				continue
-			}
-			requestID := byteutils.SliceToString(meta[1])
-			// start a subroutine only when necessary
-			if Settings.Verbose >= 3 {
-				Debug(3, "[EMITTER] input: ", byteutils.SliceToString(msg.Meta[:len(msg.Meta)-1]), " from: ", src)
-			}
-			if modifier != nil {
-				Debug(3, "[EMITTER] modifier:", requestID, "from:", src)
-				if isRequestPayload(msg.Meta) {
-					msg.Data = modifier.Rewrite(msg.Data)
-					// If modifier tells to skip request
-					if len(msg.Data) == 0 {
-						filteredRequests[requestID] = time.Now().UnixNano()
-						filteredCount++
-						continue
-					}
-					Debug(3, "[EMITTER] Rewritten input:", requestID, "from:", src)
-
-				} else {
-					if _, ok := filteredRequests[requestID]; ok {
-						delete(filteredRequests, requestID)
-						filteredCount--
-						continue
-					}
-				}
-			}
-
-			if Settings.PrettifyHTTP {
-				msg.Data = prettifyHTTP(msg.Data)
-				if len(msg.Data) == 0 {
-					continue
-				}
-			}
-
-			if Settings.SplitOutput {
-				if Settings.RecognizeTCPSessions {
-					if !PRO {
-						log.Fatal("Detailed TCP sessions work only with PRO license")
-					}
-					hasher := fnv.New32a()
-					hasher.Write(meta[1])
-
-					wIndex = int(hasher.Sum32()) % len(writers)
-					if _, err := writers[wIndex].PluginWrite(msg); err != nil {
-						return err
-					}
-				} else {
-					// Simple round robin
-					if _, err := writers[wIndex].PluginWrite(msg); err != nil {
-						return err
-					}
-
-					wIndex = (wIndex + 1) % len(writers)
-				}
-			} else {
-				for _, dst := range writers {
-					if _, err := dst.PluginWrite(msg); err != nil {
-						return err
-					}
-				}
-			}
-		}
-
-		// Run GC on each 1000 request
-		if filteredCount > 0 && filteredCount%1000 == 0 {
-			// Clean up filtered requests for which we didn't get a response to filter
-			now := time.Now().UnixNano()
-			if now-filteredRequestsLastCleanTime > int64(60*time.Second) {
-				for k, v := range filteredRequests {
-					if now-v > int64(60*time.Second) {
-						delete(filteredRequests, k)
-						filteredCount--
-					}
-				}
-				filteredRequestsLastCleanTime = time.Now().UnixNano()
-			}
-		}
-	}
-}
-
-func memUsage() (uint64, uint64) {
-	var m runtime.MemStats
-	runtime.ReadMemStats(&m)
-	// fmt.Printf("Current memory usage: ")
-	// // For info on each, see: https://golang.org/pkg/runtime/#MemStats
-	// fmt.Printf("Alloc = %v MiB", bToMb(m.Alloc))
-	// fmt.Printf("\tTotalAlloc = %v MiB", bToMb(m.TotalAlloc))
-	// fmt.Printf("\tSys = %v MiB", bToMb(m.Sys))
-	// fmt.Printf("\tNumGC = %v\n", m.NumGC)
-
-	// fmt.Printf("Total available memory %v MiB\n", bToMb(memory.TotalMemory()))
-	// fmt.Printf("Currently using %.2f %% of memory\n", 100*float64(m.Alloc+m.Sys)/float64(memory.TotalMemory()))
-
-	return m.Alloc + m.Sys, memory.TotalMemory()
-}
-
-func bToMb(b uint64) uint64 {
-	return b / 1024 / 1024
-}

emitter_test.go

@@ -1,263 +0,0 @@
-package main
-
-import (
-	"fmt"
-	"os"
-	"sync"
-	"sync/atomic"
-	"testing"
-	"time"
-)
-
-func TestMain(m *testing.M) {
-	PRO = true
-	code := m.Run()
-	os.Exit(code)
-}
-
-func TestEmitter(t *testing.T) {
-	wg := new(sync.WaitGroup)
-
-	input := NewTestInput()
-	output := NewTestOutput(func(*Message) {
-		wg.Done()
-	})
-
-	plugins := &InOutPlugins{
-		Inputs:  []PluginReader{input},
-		Outputs: []PluginWriter{output},
-	}
-	plugins.All = append(plugins.All, input, output)
-
-	emitter := NewEmitter()
-	go emitter.Start(plugins, Settings.Middleware)
-
-	for i := 0; i < 1000; i++ {
-		wg.Add(1)
-		input.EmitGET()
-	}
-
-	wg.Wait()
-	emitter.Close()
-}
-
-func TestEmitterFiltered(t *testing.T) {
-	wg := new(sync.WaitGroup)
-
-	input := NewTestInput()
-	input.skipHeader = true
-
-	output := NewTestOutput(func(*Message) {
-		wg.Done()
-	})
-
-	plugins := &InOutPlugins{
-		Inputs:  []PluginReader{input},
-		Outputs: []PluginWriter{output},
-	}
-	plugins.All = append(plugins.All, input, output)
-
-	methods := HTTPMethods{[]byte("GET")}
-	Settings.ModifierConfig = HTTPModifierConfig{Methods: methods}
-
-	emitter := &Emitter{}
-	go emitter.Start(plugins, "")
-
-	wg.Add(2)
-
-	id := uuid()
-	reqh := payloadHeader(RequestPayload, id, time.Now().UnixNano(), -1)
-	reqb := append(reqh, []byte("POST / HTTP/1.1\r\nHost: www.w3.org\r\nUser-Agent: Go 1.1 package http\r\nAccept-Encoding: gzip\r\n\r\n")...)
-
-	resh := payloadHeader(ResponsePayload, id, time.Now().UnixNano()+1, 1)
-	respb := append(resh, []byte("HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n")...)
-
-	input.EmitBytes(reqb)
-	input.EmitBytes(respb)
-
-	id = uuid()
-	reqh = payloadHeader(RequestPayload, id, time.Now().UnixNano(), -1)
-	reqb = append(reqh, []byte("GET / HTTP/1.1\r\nHost: www.w3.org\r\nUser-Agent: Go 1.1 package http\r\nAccept-Encoding: gzip\r\n\r\n")...)
-
-	resh = payloadHeader(ResponsePayload, id, time.Now().UnixNano()+1, 1)
-	respb = append(resh, []byte("HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n")...)
-
-	input.EmitBytes(reqb)
-	input.EmitBytes(respb)
-
-	wg.Wait()
-	emitter.Close()
-
-	Settings.ModifierConfig = HTTPModifierConfig{}
-}
-
-func TestEmitterSplitRoundRobin(t *testing.T) {
-	wg := new(sync.WaitGroup)
-
-	input := NewTestInput()
-
-	var counter1, counter2 int32
-
-	output1 := NewTestOutput(func(*Message) {
-		atomic.AddInt32(&counter1, 1)
-		wg.Done()
-	})
-
-	output2 := NewTestOutput(func(*Message) {
-		atomic.AddInt32(&counter2, 1)
-		wg.Done()
-	})
-
-	plugins := &InOutPlugins{
-		Inputs:  []PluginReader{input},
-		Outputs: []PluginWriter{output1, output2},
-	}
-
-	Settings.SplitOutput = true
-
-	emitter := NewEmitter()
-	go emitter.Start(plugins, Settings.Middleware)
-
-	for i := 0; i < 1000; i++ {
-		wg.Add(1)
-		input.EmitGET()
-	}
-
-	wg.Wait()
-
-	emitter.Close()
-
-	if counter1 == 0 || counter2 == 0 || counter1 != counter2 {
-		t.Errorf("Round robin should split traffic equally: %d vs %d", counter1, counter2)
-	}
-
-	Settings.SplitOutput = false
-}
-
-func TestEmitterRoundRobin(t *testing.T) {
-	wg := new(sync.WaitGroup)
-
-	input := NewTestInput()
-
-	var counter1, counter2 int32
-
-	output1 := NewTestOutput(func(*Message) {
-		counter1++
-		wg.Done()
-	})
-
-	output2 := NewTestOutput(func(*Message) {
-		counter2++
-		wg.Done()
-	})
-
-	plugins := &InOutPlugins{
-		Inputs:  []PluginReader{input},
-		Outputs: []PluginWriter{output1, output2},
-	}
-	plugins.All = append(plugins.All, input, output1, output2)
-
-	Settings.SplitOutput = true
-
-	emitter := NewEmitter()
-	go emitter.Start(plugins, Settings.Middleware)
-
-	for i := 0; i < 1000; i++ {
-		wg.Add(1)
-		input.EmitGET()
-	}
-
-	wg.Wait()
-	emitter.Close()
-
-	if counter1 == 0 || counter2 == 0 {
-		t.Errorf("Round robin should split traffic equally: %d vs %d", counter1, counter2)
-	}
-
-	Settings.SplitOutput = false
-}
-
-func TestEmitterSplitSession(t *testing.T) {
-	wg := new(sync.WaitGroup)
-	wg.Add(200)
-
-	input := NewTestInput()
-	input.skipHeader = true
-
-	var counter1, counter2 int32
-
-	output1 := NewTestOutput(func(msg *Message) {
-		if payloadID(msg.Meta)[0] == 'a' {
-			counter1++
-		}
-		wg.Done()
-	})
-
-	output2 := NewTestOutput(func(msg *Message) {
-		if payloadID(msg.Meta)[0] == 'b' {
-			counter2++
-		}
-		wg.Done()
-	})
-
-	plugins := &InOutPlugins{
-		Inputs:  []PluginReader{input},
-		Outputs: []PluginWriter{output1, output2},
-	}
-
-	Settings.SplitOutput = true
-	Settings.RecognizeTCPSessions = true
-
-	emitter := NewEmitter()
-	go emitter.Start(plugins, Settings.Middleware)
-
-	for i := 0; i < 200; i++ {
-		// Keep session but randomize
-		id := make([]byte, 20)
-		if i&1 == 0 { // for recognizeTCPSessions one should be odd and other will be even number
-			id[0] = 'a'
-		} else {
-			id[0] = 'b'
-		}
-		input.EmitBytes([]byte(fmt.Sprintf("1 %s 1 1\nGET / HTTP/1.1\r\n\r\n", id[:20])))
-	}
-
-	wg.Wait()
-
-	if counter1 != counter2 {
-		t.Errorf("Round robin should split traffic equally: %d vs %d", counter1, counter2)
-	}
-
-	Settings.SplitOutput = false
-	Settings.RecognizeTCPSessions = false
-	emitter.Close()
-}
-
-func BenchmarkEmitter(b *testing.B) {
-	wg := new(sync.WaitGroup)
-
-	input := NewTestInput()
-
-	output := NewTestOutput(func(*Message) {
-		wg.Done()
-	})
-
-	plugins := &InOutPlugins{
-		Inputs:  []PluginReader{input},
-		Outputs: []PluginWriter{output},
-	}
-	plugins.All = append(plugins.All, input, output)
-
-	emitter := NewEmitter()
-	go emitter.Start(plugins, Settings.Middleware)
-
-	b.ResetTimer()
-
-	for i := 0; i < b.N; i++ {
-		wg.Add(1)
-		input.EmitGET()
-	}
-
-	wg.Wait()
-	emitter.Close()
-}

examples/middleware/echo.sh

@@ -8,7 +8,7 @@
 # 
 
 function log {
-    if [[ ! -v GOR_TEST ]]; then # if we are not testing
+    if [[ -n "$GOR_TEST" ]]; then # if we are not testing
     # Logging to stderr, because stdout/stdin used for data transfer
     >&2 echo "[DEBUG][ECHO] $1"
     fi

examples/middleware/token_modifier.go

@@ -26,7 +26,7 @@ import (
 	"fmt"
 	"os"
 
-	"github.com/buger/goreplay/proto"
+	proto "github.com/buger/goreplay/pkg/http_proto"
 )
 
 // requestID -> originalToken

go.mod

@@ -1,21 +1,73 @@
 module github.com/buger/goreplay
 
-go 1.14
+go 1.19
 
 require (
 	github.com/Shopify/sarama v1.26.4
-	github.com/araddon/gou v0.0.0-20190110011759-c797efecbb61 // indirect
 	github.com/aws/aws-sdk-go v1.33.2
+	github.com/coocood/freecache v1.2.0
+	github.com/google/gopacket v1.1.20-0.20210429153827-3eaba0894325
+	github.com/gorilla/websocket v1.4.2
+	github.com/klauspost/compress v1.11.13 // indirect
+	github.com/mattbaird/elastigo v0.0.0-20170123220020-2fe47fd29e4b
+	github.com/rs/zerolog v1.28.0
+	github.com/stretchr/testify v1.7.0
+	github.com/xdg-go/scram v1.1.1
+	golang.org/x/net v0.0.0-20211209124913-491a49abca63
+	golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6
+	k8s.io/apimachinery v0.23.3
+	k8s.io/client-go v0.23.3
+)
+
+require (
+	github.com/araddon/gou v0.0.0-20211019181548-e7d08105776c // indirect
 	github.com/bitly/go-hostpool v0.1.0 // indirect
 	github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 // indirect
-	github.com/google/gopacket v1.1.18
-	github.com/klauspost/compress v1.10.10 // indirect
-	github.com/mattbaird/elastigo v0.0.0-20170123220020-2fe47fd29e4b
-	github.com/pbnjay/memory v0.0.0-20201129165224-b12e5d931931
-	github.com/pierrec/lz4 v2.5.2+incompatible // indirect
-	github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0 // indirect
-	github.com/smartystreets/goconvey v1.6.4 // indirect
-	github.com/stretchr/testify v1.5.1
-	golang.org/x/net v0.0.0-20200707034311-ab3426394381 // indirect
-	golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd
+	github.com/cespare/xxhash v1.1.0 // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/eapache/go-resiliency v1.2.0 // indirect
+	github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 // indirect
+	github.com/eapache/queue v1.1.0 // indirect
+	github.com/go-logr/logr v1.2.0 // indirect
+	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/golang/protobuf v1.5.2 // indirect
+	github.com/golang/snappy v0.0.1 // indirect
+	github.com/google/go-cmp v0.5.5 // indirect
+	github.com/google/gofuzz v1.1.0 // indirect
+	github.com/googleapis/gnostic v0.5.5 // indirect
+	github.com/hashicorp/go-uuid v1.0.2 // indirect
+	github.com/jcmturner/gofork v1.0.0 // indirect
+	github.com/jmespath/go-jmespath v0.3.0 // indirect
+	github.com/json-iterator/go v1.1.12 // indirect
+	github.com/mattn/go-colorable v0.1.12 // indirect
+	github.com/mattn/go-isatty v0.0.14 // indirect
+	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
+	github.com/modern-go/reflect2 v1.0.2 // indirect
+	github.com/pierrec/lz4 v2.4.1+incompatible // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563 // indirect
+	github.com/smartystreets/goconvey v1.7.2 // indirect
+	github.com/xdg-go/pbkdf2 v1.0.0 // indirect
+	github.com/xdg-go/stringprep v1.0.3 // indirect
+	golang.org/x/crypto v0.0.0-20210817164053-32db794688a5 // indirect
+	golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f // indirect
+	golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b // indirect
+	golang.org/x/text v0.3.7 // indirect
+	golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac // indirect
+	google.golang.org/appengine v1.6.7 // indirect
+	google.golang.org/protobuf v1.27.1 // indirect
+	gopkg.in/inf.v0 v0.9.1 // indirect
+	gopkg.in/jcmturner/aescts.v1 v1.0.1 // indirect
+	gopkg.in/jcmturner/dnsutils.v1 v1.0.1 // indirect
+	gopkg.in/jcmturner/gokrb5.v7 v7.5.0 // indirect
+	gopkg.in/jcmturner/rpc.v1 v1.1.0 // indirect
+	gopkg.in/yaml.v2 v2.4.0 // indirect
+	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
+	k8s.io/api v0.23.3 // indirect
+	k8s.io/klog/v2 v2.30.0 // indirect
+	k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65 // indirect
+	k8s.io/utils v0.0.0-20211116205334-6203023598ed // indirect
+	sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6 // indirect
+	sigs.k8s.io/structured-merge-diff/v4 v4.2.1 // indirect
+	sigs.k8s.io/yaml v1.2.0 // indirect
 )

go.sum

@@ -1,102 +1,662 @@
+cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=
+cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU=
+cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY=
+cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc=
+cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0=
+cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To=
+cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4=
+cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M=
+cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc=
+cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk=
+cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs=
+cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc=
+cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY=
+cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI=
+cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk=
+cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg=
+cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8=
+cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0=
+cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
+cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE=
+cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc=
+cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg=
+cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc=
+cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ=
+cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
+cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk=
+cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I=
+cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw=
+cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA=
+cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU=
+cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw=
+cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos=
+cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk=
+cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs=
+cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0=
+dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
+github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24=
+github.com/Azure/go-autorest/autorest v0.11.18/go.mod h1:dSiJPy22c3u0OtOKDNttNgqpNFY/GeWa7GH/Pz56QRA=
+github.com/Azure/go-autorest/autorest/adal v0.9.13/go.mod h1:W/MM4U6nLxnIskrw4UwWzlHfGjwUS50aOsc/I3yuU8M=
+github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74=
+github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k=
+github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8=
+github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU=
+github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
+github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ=
+github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE=
+github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
+github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
+github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
 github.com/Shopify/sarama v1.26.4 h1:+17TxUq/PJEAfZAll0T7XJjSgQWCpaQSoki/x5yN8o8=
 github.com/Shopify/sarama v1.26.4/go.mod h1:NbSGBSSndYaIhRcBtY9V0U7AyH+x71bG668AuWys/yU=
 github.com/Shopify/toxiproxy v2.1.4+incompatible h1:TKdv8HiTLgE5wdJuEML90aBgNWsokNbMijUGhmcoBJc=
 github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI=
-github.com/araddon/gou v0.0.0-20190110011759-c797efecbb61 h1:Xz25cuW4REGC5W5UtpMU3QItMIImag615HiQcRbxqKQ=
-github.com/araddon/gou v0.0.0-20190110011759-c797efecbb61/go.mod h1:ikc1XA58M+Rx7SEbf0bLJCfBkwayZ8T5jBo5FXK8Uz8=
+github.com/araddon/gou v0.0.0-20211019181548-e7d08105776c h1:XUqw//RExYoxW4Eie8MuKp8sEDAZI1gMHX/daUFgZww=
+github.com/araddon/gou v0.0.0-20211019181548-e7d08105776c/go.mod h1:ikc1XA58M+Rx7SEbf0bLJCfBkwayZ8T5jBo5FXK8Uz8=
+github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
 github.com/aws/aws-sdk-go v1.33.2 h1:8TVrnPnSD7I+AmDp66xBUvS3K0J+jH09YXdrkJ34ey0=
 github.com/aws/aws-sdk-go v1.33.2/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0=
 github.com/bitly/go-hostpool v0.1.0 h1:XKmsF6k5el6xHG3WPJ8U0Ku/ye7njX7W81Ng7O2ioR0=
 github.com/bitly/go-hostpool v0.1.0/go.mod h1:4gOCgp6+NZnVqlKyZ/iBZFTAJKembaVENUpMkpg42fw=
 github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 h1:DDGfHa7BWjL4YnC6+E63dPcxHo2sUxDIu8g3QgEJdRY=
 github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4=
+github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
+github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
+github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
+github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
+github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
+github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
+github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
+github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
+github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
+github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
+github.com/coocood/freecache v1.2.0 h1:p8RhjN6Y4DRBIMzdRlm1y+M7h7YJxye3lGW8/VvzCz0=
+github.com/coocood/freecache v1.2.0/go.mod h1:OKrEjkGVoxZhyWAJoeFi5BMLUJm2Tit0kpGkIr7NGYY=
+github.com/coreos/go-systemd/v22 v22.3.3-0.20220203105225-a9a7ef127534/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
+github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE=
 github.com/eapache/go-resiliency v1.2.0 h1:v7g92e/KSN71Rq7vSThKaWIq68fL4YHvWyiUKorFR1Q=
 github.com/eapache/go-resiliency v1.2.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs=
 github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 h1:YEetp8/yCZMuEPMUDHG0CW/brkkEp8mzqk2+ODEitlw=
 github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU=
 github.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc=
 github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I=
+github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc=
+github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
+github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
+github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po=
+github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
+github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
+github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
+github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k=
+github.com/form3tech-oss/jwt-go v3.2.3+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k=
 github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw=
 github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g=
 github.com/frankban/quicktest v1.7.2 h1:2QxQoC1TS09S7fhCPsrvqYdvP1H5M1P1ih5ABm3BTYk=
 github.com/frankban/quicktest v1.7.2/go.mod h1:jaStnuzAqU1AJdCO0l53JDCJrVDKcS03DbaAcR7Ks/o=
+github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
+github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
+github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
+github.com/getkin/kin-openapi v0.76.0/go.mod h1:660oXbgy5JFMKreazJaQTw7o+X00qeSyhcnluiMv+Xg=
+github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
+github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
+github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
+github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
+github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas=
+github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU=
+github.com/go-logr/logr v1.2.0 h1:QK40JKJyMdUDz+h+xvCsru/bJhvG0UxvePV0ufL/AcE=
+github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
+github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
+github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
+github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8=
+github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
 github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
+github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
+github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
+github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
+github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
+github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y=
+github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
+github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
+github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
+github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4=
+github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8=
+github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
+github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
+github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk=
+github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
+github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
+github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
+github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
+github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
+github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
+github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
+github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
+github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
+github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM=
+github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw=
+github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
 github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4=
 github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
+github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
+github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA=
+github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
+github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
-github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4=
 github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/gopacket v1.1.18 h1:lum7VRA9kdlvBi7/v2p7/zcbkduHaCH/SVVyurs7OpY=
-github.com/google/gopacket v1.1.18/go.mod h1:UdDNZ1OO62aGYVnPhxT1U6aI7ukYtA/kB8vaU0diBUM=
+github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU=
+github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g=
+github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/google/gopacket v1.1.20-0.20210429153827-3eaba0894325 h1:YmIcZ5Var3BAQ64AW98Iiys5Ih4fiU0xK41+8isC5Ec=
+github.com/google/gopacket v1.1.20-0.20210429153827-3eaba0894325/go.mod h1:riddUzxTSBpJXk3qBHtYr4qOhFhT6k/1c0E3qkQjQpA=
+github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
+github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
+github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
+github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
+github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
+github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
+github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
+github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
+github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
+github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
+github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
+github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
+github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
+github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
+github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
+github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
+github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
+github.com/googleapis/gnostic v0.5.1/go.mod h1:6U4PtQXGIEt/Z3h5MAT7FNofLnw9vXk2cUuW7uA/OeU=
+github.com/googleapis/gnostic v0.5.5 h1:9fHAtK0uDfpveeqqo1hkEZJcFvYXAiCN3UutL8F9xHw=
+github.com/googleapis/gnostic v0.5.5/go.mod h1:7+EbHbldMins07ALC74bsA81Ovc97DwqyJO1AENw9kA=
 github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8=
 github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
+github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
+github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc=
+github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
+github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
 github.com/hashicorp/go-uuid v1.0.2 h1:cfejS+Tpcp13yd5nYHWDI6qVCny6wyX2Mt5SGur2IGE=
 github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
+github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
+github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
+github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
+github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
+github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
+github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
 github.com/jcmturner/gofork v1.0.0 h1:J7uCkflzTEhUZ64xqKnkDxq3kzc96ajM1Gli5ktUem8=
 github.com/jcmturner/gofork v1.0.0/go.mod h1:MK8+TM0La+2rjBD4jE12Kj1pCCxK7d2LK/UM3ncEo0o=
 github.com/jmespath/go-jmespath v0.3.0 h1:OS12ieG61fsCg5+qLJ+SsW9NicxNkg3b25OyT2yCeUc=
 github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik=
+github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
+github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
+github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
+github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
+github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
 github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
 github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
+github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
+github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/klauspost/compress v1.9.8/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
-github.com/klauspost/compress v1.10.10 h1:a/y8CglcM7gLGYmlbP/stPE5sR3hbhFRUjCBfd/0B3I=
-github.com/klauspost/compress v1.10.10/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
+github.com/klauspost/compress v1.11.13 h1:eSvu8Tmq6j2psUJqJrLcWH6K3w5Dwc+qipbaA6eVEN4=
+github.com/klauspost/compress v1.11.13/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs=
 github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
-github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
+github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
+github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
+github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
 github.com/mattbaird/elastigo v0.0.0-20170123220020-2fe47fd29e4b h1:v29yPGHhOqw7VHEnTeQFAth3SsBrmwc8JfuhNY0G34k=
 github.com/mattbaird/elastigo v0.0.0-20170123220020-2fe47fd29e4b/go.mod h1:5MWrJXKRQyhQdUCF+vu6U5c4nQpg70vW3eHaU0/AYbU=
-github.com/pbnjay/memory v0.0.0-20201129165224-b12e5d931931 h1:EeWknjeRU+R3O4ghG7XZCpgSfJNStZyEP8aWyQwJM8s=
-github.com/pbnjay/memory v0.0.0-20201129165224-b12e5d931931/go.mod h1:RMU2gJXhratVxBDTFeOdNhd540tG57lt9FIUV0YLvIQ=
+github.com/mattn/go-colorable v0.1.12 h1:jF+Du6AlPIjs2BiUiQlKOX0rt3SujHxPnksPKZbaA40=
+github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4=
+github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y=
+github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=
+github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
+github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c=
+github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
+github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
+github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
+github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
+github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw=
+github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs=
+github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
+github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78=
+github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
+github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
+github.com/onsi/ginkgo v1.14.0 h1:2mOpI4JVVPBN+WQRa0WKH2eXR+Ey+uK4n7Zj0aYpIQA=
+github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY=
+github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
+github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
+github.com/onsi/gomega v1.10.1 h1:o0+MgICZLuZ7xjH7Vx6zS/zcu93/BEp1VwkIW1mEXCE=
+github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
+github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU=
+github.com/pierrec/lz4 v2.4.1+incompatible h1:mFe7ttWaflA46Mhqh+jUfjp2qTbPYxLB2/OyBppH9dg=
 github.com/pierrec/lz4 v2.4.1+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
-github.com/pierrec/lz4 v2.5.2+incompatible h1:WCjObylUIOlKy/+7Abdn34TLIkXiA4UWUMhxq9m9ZXI=
-github.com/pierrec/lz4 v2.5.2+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563 h1:dY6ETXrvDG7Sa4vE8ZQG4yqWg6UnOcbqTAahkV813vQ=
 github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
-github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0 h1:MkV+77GLUNo5oJ0jf870itWm3D0Sjh7+Za9gazKc5LQ=
-github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
-github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM=
-github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
-github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s=
-github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
+github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
+github.com/rs/xid v1.4.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
+github.com/rs/zerolog v1.28.0 h1:MirSo27VyNi7RJYP3078AA1+Cyzd2GB66qy3aUHvsWY=
+github.com/rs/zerolog v1.28.0/go.mod h1:NILgTygv/Uej1ra5XxGf82ZFSLk58MFGAUS2o6usyD0=
+github.com/smartystreets/assertions v1.2.0 h1:42S6lae5dvLc7BrLu/0ugRtcFVjoJNMC/N3yZFZkDFs=
+github.com/smartystreets/assertions v1.2.0/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo=
+github.com/smartystreets/goconvey v1.7.2 h1:9RBaZCeXEQ3UselpuwUQHltGVXvdwm6cv1hgR6gDIPg=
+github.com/smartystreets/goconvey v1.7.2/go.mod h1:Vw0tHAZW6lzCRk3xgdin6fKYcG+G3Pg9vgXWeJpQFMM=
+github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72 h1:qLC7fQah7D6K1B0ujays3HV9gkFtllcxhzImRR7ArPQ=
+github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
+github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk=
+github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
+github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
+github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
-github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4=
 github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
+github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
+github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE=
+github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU=
+github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0=
+github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c=
+github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
+github.com/xdg-go/scram v1.1.1 h1:VOMT+81stJgXW3CpHyqHN3AXDYIMsx56mEFrB37Mb/E=
+github.com/xdg-go/scram v1.1.1/go.mod h1:RaEWvsqvNKKvBPvcKeFjrG2cJqOkHTiyTpzz23ni57g=
+github.com/xdg-go/stringprep v1.0.3 h1:kdwGpVNwPFtjs98xCGkHjQtGKh86rDcRZN17QEMCOIs=
+github.com/xdg-go/stringprep v1.0.3/go.mod h1:W3f5j4i+9rC0kuIEJL0ky1VpHXQU3ocBgklLGvcBnW8=
 github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I=
 github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y=
+github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
+go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
+go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
+go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
+go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
+go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
+go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk=
+go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20200204104054-c9f3fb736b72/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
-golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20210817164053-32db794688a5 h1:HWj/xjIHfjYU5nVXpTM0s39J9CbLn7Cc5a7IC5rwsMQ=
+golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
+golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
+golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek=
+golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY=
+golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
+golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
+golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
+golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
+golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
+golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
+golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
+golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
+golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
+golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
+golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs=
+golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
+golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
+golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
+golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=
+golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=
+golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
+golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY=
+golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
+golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
+golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20200707034311-ab3426394381 h1:VXak5I6aEWmAXeQjA+QSZzlgNrpq9mjcfDemuexIKsU=
+golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
+golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
+golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc=
+golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
+golang.org/x/net v0.0.0-20211209124913-491a49abca63 h1:iocB37TsdFuN6IBRZ+ry36wrkoV51/tl5vOWqkcPGvY=
+golang.org/x/net v0.0.0-20211209124913-491a49abca63/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
+golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
+golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
+golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
+golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
+golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
+golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f h1:Qmd2pbz05z7z6lm0DrgQVVPuBm92jqujBKMHMOlOQEw=
+golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
+golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20190405154228-4b34438f7a67/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884=
+golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190606203320-7fc4e5ec1444/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200217220822-9197077df867/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210831042530-f4d43177bf5e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6 h1:foEbQz/B0Oz6YIqu/69kfXPYeFQAuuMYFkjaqXzl5Wo=
+golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b h1:9zKuko04nR4gjZ4+DNjHqRlAJqbJETHwiNKDqTfOjfE=
+golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
+golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk=
+golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
+golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac h1:7zkz7BUtwNFFqcowJ+RIgu2MaV/MapERkDIy+mwPyjs=
+golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
+golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
-golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
+golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw=
+golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw=
+golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8=
+golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200505023115-26f46d2f7ef8/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
+golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
+golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
+golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE=
+golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
+golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
+golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
+golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
+golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
+golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
+golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
+google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=
+google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
+google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
+google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
+google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
+google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
+google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
+google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
+google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
+google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
+google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
+google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE=
+google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE=
+google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM=
+google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc=
+google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg=
+google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE=
+google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8=
+google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU=
+google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94=
+google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
+google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
+google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
+google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
+google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c=
+google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
+google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
+google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
+google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
+google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8=
+google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA=
+google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U=
+google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
+google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA=
+google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20201019141844-1ed22bb0c154/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A=
+google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
+google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
+google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
+google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
+google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
+google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
+google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
+google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
+google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60=
+google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk=
+google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
+google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
+google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
+google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc=
+google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8=
+google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
+google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
+google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
+google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
+google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
+google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
+google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
+google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
+google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4=
+google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
+google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
+google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
+google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ=
+google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU=
+gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
+gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
+gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc=
+gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
 gopkg.in/jcmturner/aescts.v1 v1.0.1 h1:cVVZBK2b1zY26haWB4vbBiZrfFQnfbTVrE3xZq6hrEw=
 gopkg.in/jcmturner/aescts.v1 v1.0.1/go.mod h1:nsR8qBOg+OucoIW+WMhB3GspUQXq9XorLnQb9XtvcOo=
 gopkg.in/jcmturner/dnsutils.v1 v1.0.1 h1:cIuC1OLRGZrld+16ZJvvZxVJeKPsvd5eUIvxfoN5hSM=
@@ -107,6 +667,49 @@ gopkg.in/jcmturner/gokrb5.v7 v7.5.0 h1:a9tsXlIDD9SKxotJMK3niV7rPZAJeX2aD/0yg3qlI
 gopkg.in/jcmturner/gokrb5.v7 v7.5.0/go.mod h1:l8VISx+WGYp+Fp7KRbsiUuXTTOnxIc3Tuvyavf11/WM=
 gopkg.in/jcmturner/rpc.v1 v1.1.0 h1:QHIUxTX1ISuAv9dD2wJ9HWQVuWDX/Zc0PfeC2tjc4rU=
 gopkg.in/jcmturner/rpc.v1 v1.1.0/go.mod h1:YIdkC4XfD6GXbzje11McwsDuOlZQSb9W4vfLvuNnlv8=
+gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
+gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
+gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=
+gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
+gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=
+gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
+honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
+honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
+k8s.io/api v0.23.3 h1:KNrME8KHGr12Ozjf8ytOewKzZh6hl/hHUZeHddT3a38=
+k8s.io/api v0.23.3/go.mod h1:w258XdGyvCmnBj/vGzQMj6kzdufJZVUwEM1U2fRJwSQ=
+k8s.io/apimachinery v0.23.3 h1:7IW6jxNzrXTsP0c8yXz2E5Yx/WTzVPTsHIx/2Vm0cIk=
+k8s.io/apimachinery v0.23.3/go.mod h1:BEuFMMBaIbcOqVIJqNZJXGFTP4W6AycEpb5+m/97hrM=
+k8s.io/client-go v0.23.3 h1:23QYUmCQ/W6hW78xIwm3XqZrrKZM+LWDqW2zfo+szJs=
+k8s.io/client-go v0.23.3/go.mod h1:47oMd+YvAOqZM7pcQ6neJtBiFH7alOyfunYN48VsmwE=
+k8s.io/gengo v0.0.0-20210813121822-485abfe95c7c/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E=
+k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE=
+k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y=
+k8s.io/klog/v2 v2.30.0 h1:bUO6drIvCIsvZ/XFgfxoGFQU/a4Qkh0iAlvUR7vlHJw=
+k8s.io/klog/v2 v2.30.0/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0=
+k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65 h1:E3J9oCLlaobFUqsjG9DfKbP2BmgwBL2p7pn0A3dG9W4=
+k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65/go.mod h1:sX9MT8g7NVZM5lVL/j8QyCCJe8YSMW30QvGZWaCIDIk=
+k8s.io/utils v0.0.0-20210802155522-efc7438f0176/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
+k8s.io/utils v0.0.0-20211116205334-6203023598ed h1:ck1fRPWPJWsMd8ZRFsWc6mh/zHp5fZ/shhbrgPUxDAE=
+k8s.io/utils v0.0.0-20211116205334-6203023598ed/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
+rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
+rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
+rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
+sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6 h1:fD1pz4yfdADVNfFmcP2aBEtudwUQ1AlLnRBALr33v3s=
+sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6/go.mod h1:p4QtZmO4uMYipTQNzagwnNoseA6OxSUutVw05NhYDRs=
+sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw=
+sigs.k8s.io/structured-merge-diff/v4 v4.2.1 h1:bKCqE9GvQ5tiVHn5rfn1r+yao3aLQEaLzkkmAkf+A6Y=
+sigs.k8s.io/structured-merge-diff/v4 v4.2.1/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4=
+sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q=
+sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc=

gor.go

@@ -1,125 +0,0 @@
-// Gor is simple http traffic replication tool written in Go. Its main goal to replay traffic from production servers to staging and dev environments.
-// Now you can test your code on real user sessions in an automated and repeatable fashion.
-package main
-
-import (
-	"flag"
-	"log"
-	"net/http"
-	"net/http/httputil"
-	_ "net/http/pprof"
-	"os"
-	"os/signal"
-	"runtime"
-	_ "runtime/debug"
-	"runtime/pprof"
-	"syscall"
-	"time"
-)
-
-var (
-	cpuprofile = flag.String("cpuprofile", "", "write cpu profile to file")
-	memprofile = flag.String("memprofile", "", "write memory profile to this file")
-)
-
-func loggingMiddleware(next http.Handler) http.Handler {
-	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		rb, _ := httputil.DumpRequest(r, false)
-		log.Println(string(rb))
-		next.ServeHTTP(w, r)
-	})
-}
-
-func main() {
-	if os.Getenv("GOMAXPROCS") == "" {
-		runtime.GOMAXPROCS(runtime.NumCPU() * 2)
-	}
-
-	args := os.Args[1:]
-	var plugins *InOutPlugins
-	if len(args) > 0 && args[0] == "file-server" {
-		if len(args) != 2 {
-			log.Fatal("You should specify port and IP (optional) for the file server. Example: `gor file-server :80`")
-		}
-		dir, _ := os.Getwd()
-
-		Debug(0, "Started example file server for current directory on address ", args[1])
-
-		log.Fatal(http.ListenAndServe(args[1], loggingMiddleware(http.FileServer(http.Dir(dir)))))
-	} else {
-		flag.Parse()
-		checkSettings()
-		plugins = NewPlugins()
-	}
-
-	log.Printf("[PPID %d and PID %d] Version:%s\n", os.Getppid(), os.Getpid(), VERSION)
-
-	if len(plugins.Inputs) == 0 || len(plugins.Outputs) == 0 {
-		log.Fatal("Required at least 1 input and 1 output")
-	}
-
-	if *memprofile != "" {
-		profileMEM(*memprofile)
-	}
-
-	if *cpuprofile != "" {
-		profileCPU(*cpuprofile)
-	}
-
-	if Settings.Pprof != "" {
-		go func() {
-			log.Println(http.ListenAndServe(Settings.Pprof, nil))
-		}()
-	}
-
-	closeCh := make(chan int)
-	emitter := NewEmitter()
-	go emitter.Start(plugins, Settings.Middleware)
-	if Settings.ExitAfter > 0 {
-		log.Printf("Running gor for a duration of %s\n", Settings.ExitAfter)
-
-		time.AfterFunc(Settings.ExitAfter, func() {
-			log.Printf("gor run timeout %s\n", Settings.ExitAfter)
-			close(closeCh)
-		})
-	}
-	c := make(chan os.Signal, 1)
-	signal.Notify(c, os.Interrupt, syscall.SIGTERM)
-	exit := 0
-	select {
-	case <-c:
-		exit = 1
-	case <-closeCh:
-		exit = 0
-	}
-	emitter.Close()
-	os.Exit(exit)
-}
-
-func profileCPU(cpuprofile string) {
-	if cpuprofile != "" {
-		f, err := os.Create(cpuprofile)
-		if err != nil {
-			log.Fatal(err)
-		}
-		pprof.StartCPUProfile(f)
-
-		time.AfterFunc(30*time.Second, func() {
-			pprof.StopCPUProfile()
-			f.Close()
-		})
-	}
-}
-
-func profileMEM(memprofile string) {
-	if memprofile != "" {
-		f, err := os.Create(memprofile)
-		if err != nil {
-			log.Fatal(err)
-		}
-		time.AfterFunc(30*time.Second, func() {
-			pprof.WriteHeapProfile(f)
-			f.Close()
-		})
-	}
-}

gor_stat.go

@@ -1,64 +0,0 @@
-package main
-
-import (
-	"runtime"
-	"strconv"
-	"time"
-)
-
-type GorStat struct {
-	statName string
-	rateMs   int
-	latest   int
-	mean     int
-	max      int
-	count    int
-}
-
-func NewGorStat(statName string, rateMs int) (s *GorStat) {
-	s = new(GorStat)
-	s.statName = statName
-	s.rateMs = rateMs
-	s.latest = 0
-	s.mean = 0
-	s.max = 0
-	s.count = 0
-
-	if Settings.Stats {
-		go s.reportStats()
-	}
-	return
-}
-
-func (s *GorStat) Write(latest int) {
-	if Settings.Stats {
-		if latest > s.max {
-			s.max = latest
-		}
-		if latest != 0 {
-			s.mean = ((s.mean * s.count) + latest) / (s.count + 1)
-		}
-		s.latest = latest
-		s.count = s.count + 1
-	}
-}
-
-func (s *GorStat) Reset() {
-	s.latest = 0
-	s.max = 0
-	s.mean = 0
-	s.count = 0
-}
-
-func (s *GorStat) String() string {
-	return s.statName + ":" + strconv.Itoa(s.latest) + "," + strconv.Itoa(s.mean) + "," + strconv.Itoa(s.max) + "," + strconv.Itoa(s.count) + "," + strconv.Itoa(s.count/(s.rateMs/1000.0)) + "," + strconv.Itoa(runtime.NumGoroutine())
-}
-
-func (s *GorStat) reportStats() {
-	Debug(0, "\n", s.statName+":latest,mean,max,count,count/second,gcount")
-	for {
-		Debug(0, "\n", s)
-		s.Reset()
-		time.Sleep(time.Duration(s.rateMs) * time.Millisecond)
-	}
-}

http_prettifier.go

@@ -1,67 +0,0 @@
-package main
-
-import (
-	"bytes"
-	"compress/gzip"
-	"fmt"
-	"io/ioutil"
-	"net/http/httputil"
-	"strconv"
-
-	"github.com/buger/goreplay/proto"
-)
-
-func prettifyHTTP(p []byte) []byte {
-
-	tEnc := bytes.Equal(proto.Header(p, []byte("Transfer-Encoding")), []byte("chunked"))
-	cEnc := bytes.Equal(proto.Header(p, []byte("Content-Encoding")), []byte("gzip"))
-
-	if !(tEnc || cEnc) {
-		return p
-	}
-
-	headersPos := proto.MIMEHeadersEndPos(p)
-
-	if headersPos < 5 || headersPos > len(p) {
-		return p
-	}
-
-	headers := p[:headersPos]
-	content := p[headersPos:]
-
-	if tEnc {
-		buf := bytes.NewReader(content)
-		r := httputil.NewChunkedReader(buf)
-		content, _ = ioutil.ReadAll(r)
-
-		headers = proto.DeleteHeader(headers, []byte("Transfer-Encoding"))
-
-		newLen := strconv.Itoa(len(content))
-		headers = proto.SetHeader(headers, []byte("Content-Length"), []byte(newLen))
-	}
-
-	if cEnc {
-		buf := bytes.NewReader(content)
-		g, err := gzip.NewReader(buf)
-
-		if err != nil {
-			Debug(1, "[Prettifier] GZIP encoding error:", err)
-			return []byte{}
-		}
-
-		content, err = ioutil.ReadAll(g)
-		if err != nil {
-			Debug(1, fmt.Sprintf("[HTTP-PRETTIFIER] %q", err))
-			return p
-		}
-
-		headers = proto.DeleteHeader(headers, []byte("Content-Encoding"))
-
-		newLen := strconv.Itoa(len(content))
-		headers = proto.SetHeader(headers, []byte("Content-Length"), []byte(newLen))
-	}
-
-	newPayload := append(headers, content...)
-
-	return newPayload
-}

input_file.go

@@ -1,276 +0,0 @@
-package main
-
-import (
-	"bufio"
-	"bytes"
-	"compress/gzip"
-	"errors"
-	"fmt"
-	"io"
-	"os"
-	"path/filepath"
-	"strconv"
-	"strings"
-	"sync"
-	"sync/atomic"
-	"time"
-
-	"github.com/aws/aws-sdk-go/aws"
-	"github.com/aws/aws-sdk-go/aws/session"
-	"github.com/aws/aws-sdk-go/service/s3"
-)
-
-type fileInputReader struct {
-	reader    *bufio.Reader
-	data      []byte
-	file      io.ReadCloser
-	timestamp int64
-	closed    int32 // Value of 0 indicates that the file is still open.
-	s3        bool
-}
-
-func (f *fileInputReader) parseNext() error {
-	payloadSeparatorAsBytes := []byte(payloadSeparator)
-	var buffer bytes.Buffer
-	for {
-		line, err := f.reader.ReadBytes('\n')
-
-		if err != nil {
-			if err != io.EOF {
-				Debug(1, err)
-			} else {
-				f.Close()
-			}
-			return err
-		}
-
-		if bytes.Equal(payloadSeparatorAsBytes[1:], line) {
-			asBytes := buffer.Bytes()
-			meta := payloadMeta(asBytes)
-
-			f.timestamp, _ = strconv.ParseInt(string(meta[2]), 10, 64)
-			f.data = asBytes[:len(asBytes)-1]
-
-			return nil
-		}
-
-		buffer.Write(line)
-	}
-
-}
-
-func (f *fileInputReader) ReadPayload() []byte {
-	defer f.parseNext()
-
-	return f.data
-}
-
-// Close closes this plugin
-func (f *fileInputReader) Close() error {
-	if atomic.LoadInt32(&f.closed) == 0 {
-		atomic.StoreInt32(&f.closed, 1)
-		f.file.Close()
-	}
-
-	return nil
-}
-
-func newFileInputReader(path string) *fileInputReader {
-	var file io.ReadCloser
-	var err error
-
-	if strings.HasPrefix(path, "s3://") {
-		file = NewS3ReadCloser(path)
-	} else {
-		file, err = os.Open(path)
-	}
-
-	if err != nil {
-		Debug(0, fmt.Sprintf("[INPUT-FILE] err: %q", err))
-		return nil
-	}
-
-	r := &fileInputReader{file: file, closed: 0}
-	if strings.HasSuffix(path, ".gz") {
-		gzReader, err := gzip.NewReader(file)
-		if err != nil {
-			Debug(0, fmt.Sprintf("[INPUT-FILE] err: %q", err))
-			return nil
-		}
-		r.reader = bufio.NewReader(gzReader)
-	} else {
-		r.reader = bufio.NewReader(file)
-	}
-
-	r.parseNext()
-
-	return r
-}
-
-// FileInput can read requests generated by FileOutput
-type FileInput struct {
-	mu          sync.Mutex
-	data        chan []byte
-	exit        chan bool
-	path        string
-	readers     []*fileInputReader
-	speedFactor float64
-	loop        bool
-}
-
-// NewFileInput constructor for FileInput. Accepts file path as argument.
-func NewFileInput(path string, loop bool) (i *FileInput) {
-	i = new(FileInput)
-	i.data = make(chan []byte, 1000)
-	i.exit = make(chan bool)
-	i.path = path
-	i.speedFactor = 1
-	i.loop = loop
-
-	if err := i.init(); err != nil {
-		return
-	}
-
-	go i.emit()
-
-	return
-}
-
-func (i *FileInput) init() (err error) {
-	defer i.mu.Unlock()
-	i.mu.Lock()
-
-	var matches []string
-
-	if strings.HasPrefix(i.path, "s3://") {
-		sess := session.Must(session.NewSession(awsConfig()))
-		svc := s3.New(sess)
-
-		bucket, key := parseS3Url(i.path)
-
-		params := &s3.ListObjectsInput{
-			Bucket: aws.String(bucket),
-			Prefix: aws.String(key),
-		}
-
-		resp, err := svc.ListObjects(params)
-		if err != nil {
-			Debug(0, "[INPUT-FILE] Error while retreiving list of files from S3", i.path, err)
-			return err
-		}
-
-		for _, c := range resp.Contents {
-			matches = append(matches, "s3://"+bucket+"/"+(*c.Key))
-		}
-	} else if matches, err = filepath.Glob(i.path); err != nil{ 
-			Debug(0, "[INPUT-FILE] Wrong file pattern", i.path, err)
-			return
-		
-	}
-
-	if len(matches) == 0 {
-		Debug(0, "[INPUT-FILE] No files match pattern: ", i.path)
-		return errors.New("No matching files")
-	}
-
-	i.readers = make([]*fileInputReader, len(matches))
-
-	for idx, p := range matches {
-		i.readers[idx] = newFileInputReader(p)
-	}
-
-	return nil
-}
-
-// PluginRead reads message from this plugin
-func (i *FileInput) PluginRead() (*Message, error) {
-	var msg Message
-	select {
-	case <-i.exit:
-		return nil, ErrorStopped
-	case buf := <-i.data:
-		msg.Meta, msg.Data = payloadMetaWithBody(buf)
-		return &msg, nil
-	}
-}
-
-func (i *FileInput) String() string {
-	return "File input: " + i.path
-}
-
-// Find reader with smallest timestamp e.g next payload in row
-func (i *FileInput) nextReader() (next *fileInputReader) {
-	for _, r := range i.readers {
-		if r == nil || atomic.LoadInt32(&r.closed) != 0 {
-			continue
-		}
-
-		if next == nil || r.timestamp < next.timestamp {
-			next = r
-			continue
-		}
-	}
-
-	return
-}
-
-func (i *FileInput) emit() {
-	var lastTime int64 = -1
-
-	for {
-		select {
-		case <-i.exit:
-			return
-		default:
-		}
-
-		reader := i.nextReader()
-
-		if reader == nil {
-			if i.loop {
-				i.init()
-				lastTime = -1
-				continue
-			} else {
-				break
-			}
-		}
-
-		if lastTime != -1 {
-			diff := reader.timestamp - lastTime
-			lastTime = reader.timestamp
-
-			if i.speedFactor != 1 {
-				diff = int64(float64(diff) / i.speedFactor)
-			}
-
-			time.Sleep(time.Duration(diff))
-		} else {
-			lastTime = reader.timestamp
-		}
-
-		// Recheck if we have exited since last check.
-		select {
-		case <-i.exit:
-			return
-		default:
-			i.data <- reader.ReadPayload()
-		}
-	}
-
-	Debug(0, fmt.Sprintf("[INPUT-FILE] FileInput: end of file '%s'\n", i.path))
-
-}
-
-// Close closes this plugin
-func (i *FileInput) Close() error {
-	defer i.mu.Unlock()
-	i.mu.Lock()
-
-	close(i.exit)
-	for _, r := range i.readers {
-		r.Close()
-	}
-
-	return nil
-}

input_raw.go

@@ -1,229 +0,0 @@
-package main
-
-import (
-	"context"
-	"fmt"
-	"log"
-	"net"
-	"strconv"
-	"sync"
-	"time"
-
-	"github.com/buger/goreplay/capture"
-	"github.com/buger/goreplay/proto"
-	"github.com/buger/goreplay/size"
-	"github.com/buger/goreplay/tcp"
-)
-
-// TCPProtocol is a number to indicate type of protocol
-type TCPProtocol uint8
-
-const (
-	// ProtocolHTTP ...
-	ProtocolHTTP TCPProtocol = iota
-	// ProtocolBinary ...
-	ProtocolBinary
-)
-
-// Set is here so that TCPProtocol can implement flag.Var
-func (protocol *TCPProtocol) Set(v string) error {
-	switch v {
-	case "", "http":
-		*protocol = ProtocolHTTP
-	case "binary":
-		*protocol = ProtocolBinary
-	default:
-		return fmt.Errorf("unsupported protocol %s", v)
-	}
-	return nil
-}
-
-func (protocol *TCPProtocol) String() string {
-	switch *protocol {
-	case ProtocolBinary:
-		return "binary"
-	case ProtocolHTTP:
-		return "http"
-	default:
-		return ""
-	}
-}
-
-// RAWInputConfig represents configuration that can be applied on raw input
-type RAWInputConfig struct {
-	capture.PcapOptions
-	Expire         time.Duration      `json:"input-raw-expire"`
-	CopyBufferSize size.Size          `json:"copy-buffer-size"`
-	Engine         capture.EngineType `json:"input-raw-engine"`
-	TrackResponse  bool               `json:"input-raw-track-response"`
-	Protocol       TCPProtocol        `json:"input-raw-protocol"`
-	RealIPHeader   string             `json:"input-raw-realip-header"`
-	Stats          bool               `json:"input-raw-stats"`
-	quit           chan bool          // Channel used only to indicate goroutine should shutdown
-	host           string
-	port           uint16
-}
-
-// RAWInput used for intercepting traffic for given address
-type RAWInput struct {
-	sync.Mutex
-	RAWInputConfig
-	messageStats   []tcp.Stats
-	listener       *capture.Listener
-	message        chan *tcp.Message
-	cancelListener context.CancelFunc
-	closed         bool
-}
-
-// NewRAWInput constructor for RAWInput. Accepts raw input config as arguments.
-func NewRAWInput(address string, config RAWInputConfig) (i *RAWInput) {
-	i = new(RAWInput)
-	i.RAWInputConfig = config
-	i.message = make(chan *tcp.Message, 10000)
-	i.quit = make(chan bool)
-	var host, _port string
-	var err error
-	var port int
-	host, _port, err = net.SplitHostPort(address)
-	if err != nil {
-		log.Fatalf("input-raw: error while parsing address: %s", err)
-	}
-	if _port != "" {
-		port, err = strconv.Atoi(_port)
-	}
-
-	if err != nil {
-		log.Fatalf("parsing port error: %v", err)
-	}
-	i.host = host
-	i.port = uint16(port)
-
-	i.listen(address)
-
-	return
-}
-
-// PluginRead reads meassage from this plugin
-func (i *RAWInput) PluginRead() (*Message, error) {
-	var msgTCP *tcp.Message
-	var msg Message
-	select {
-	case <-i.quit:
-		return nil, ErrorStopped
-	case msgTCP = <-i.message:
-		msg.Data = msgTCP.Data()
-	}
-	var msgType byte = ResponsePayload
-	if msgTCP.IsRequest {
-		msgType = RequestPayload
-		if i.RealIPHeader != "" {
-			msg.Data = proto.SetHeader(msg.Data, []byte(i.RealIPHeader), []byte(msgTCP.SrcAddr))
-		}
-	}
-	msg.Meta = payloadHeader(msgType, msgTCP.UUID(), msgTCP.Start.UnixNano(), msgTCP.End.UnixNano()-msgTCP.Start.UnixNano())
-
-	// to be removed....
-	if msgTCP.Truncated {
-		Debug(2, "[INPUT-RAW] message truncated, increase copy-buffer-size")
-	}
-	// to be removed...
-	if msgTCP.TimedOut {
-		Debug(2, "[INPUT-RAW] message timeout reached, increase input-raw-expire")
-	}
-	if i.Stats {
-		stat := msgTCP.Stats
-		go i.addStats(stat)
-	}
-	msgTCP.Finalize()
-	msgTCP = nil
-	return &msg, nil
-}
-
-func (i *RAWInput) listen(address string) {
-	var err error
-	i.listener, err = capture.NewListener(i.host, i.port, "", i.Engine, i.TrackResponse, Settings.maxAllowedMem)
-	if err != nil {
-		log.Fatal(err)
-	}
-	i.listener.SetPcapOptions(i.PcapOptions)
-	err = i.listener.Activate()
-	if err != nil {
-		log.Fatal(err)
-	}
-	parser := tcp.NewMessageParser(i.CopyBufferSize, i.Expire, Debug, i.messageEmitter)
-
-	if i.Protocol == ProtocolHTTP {
-		parser.Start = http1StartHint
-		parser.End = http1EndHint
-	}
-	var ctx context.Context
-	ctx, i.cancelListener = context.WithCancel(context.Background())
-	errCh := i.listener.ListenBackground(ctx, parser.PacketHandler)
-	<-i.listener.Reading
-	Debug(1, i)
-	go func() {
-		<-errCh // the listener closed voluntarily
-		i.Close()
-	}()
-}
-
-func (i *RAWInput) messageEmitter(m *tcp.Message) {
-	i.message <- m
-}
-
-func (i *RAWInput) String() string {
-	return fmt.Sprintf("Intercepting traffic from: %s:%d", i.host, i.port)
-}
-
-// GetStats returns the stats so far and reset the stats
-func (i *RAWInput) GetStats() []tcp.Stats {
-	i.Lock()
-	defer func() {
-		i.messageStats = []tcp.Stats{}
-		i.Unlock()
-	}()
-	return i.messageStats
-}
-
-// Close closes the input raw listener
-func (i *RAWInput) Close() error {
-	i.Lock()
-	defer i.Unlock()
-	if i.closed {
-		return nil
-	}
-	i.cancelListener()
-	close(i.quit)
-	i.closed = true
-	return nil
-}
-
-func (i *RAWInput) addStats(mStats tcp.Stats) {
-	i.Lock()
-	if len(i.messageStats) >= 10000 {
-		i.messageStats = []tcp.Stats{}
-	}
-	i.messageStats = append(i.messageStats, mStats)
-	i.Unlock()
-}
-
-func http1StartHint(pckt *tcp.Packet) (isRequest, isResponse bool) {
-	if proto.HasRequestTitle(pckt.Payload) {
-		return true, false
-	}
-
-	if proto.HasResponseTitle(pckt.Payload) {
-		return false, true
-	}
-
-	// No request or response detected
-	return false, false
-}
-
-func http1EndHint(m *tcp.Message) bool {
-	if m.MissingChunk() {
-		return false
-	}
-
-	return proto.HasFullPayload(m, m.PacketData()...)
-}

internal/capture/af_packet.go

@@ -0,0 +1,32 @@
+//go:build !linux
+
+package capture
+
+import (
+	"fmt"
+	"time"
+
+	"github.com/google/gopacket"
+)
+
+func newAfpacketHandle(device string, snaplen int, block_size int, num_blocks int,
+	useVLAN bool, timeout time.Duration) (*afpacketHandle, error) {
+	return nil, fmt.Errorf("Not implemented")
+}
+
+func afpacketComputeSize(targetSizeMb int, snaplen int, pageSize int) (
+	frameSize int, blockSize int, numBlocks int, err error) {
+	return 0, 0, 0, fmt.Errorf("Not implemented")
+}
+
+type afpacketHandle struct{}
+
+// ReadPacketData satisfies PacketDataSource interface
+func (h *afpacketHandle) ReadPacketData() (data []byte, ci gopacket.CaptureInfo, err error) {
+	return nil, gopacket.CaptureInfo{}, fmt.Errorf("Not implemented")
+}
+
+// SetBPFFilter translates a BPF filter string into BPF RawInstruction and applies them.
+func (h *afpacketHandle) SetBPFFilter(filter string, snaplen int) (err error) {
+	return fmt.Errorf("Not implemented")
+}

internal/capture/af_packet_linux.go

@@ -0,0 +1,115 @@
+//go:build linux
+
+package capture
+
+import (
+	"fmt"
+	"time"
+
+	"github.com/google/gopacket"
+	"github.com/google/gopacket/afpacket"
+	"github.com/google/gopacket/layers"
+	"github.com/google/gopacket/pcap"
+	"golang.org/x/net/bpf"
+
+	_ "github.com/google/gopacket/layers"
+)
+
+type afpacketHandle struct {
+	TPacket *afpacket.TPacket
+}
+
+func newAfpacketHandle(device string, snaplen int, block_size int, num_blocks int,
+	useVLAN bool, timeout time.Duration) (*afpacketHandle, error) {
+
+	h := &afpacketHandle{}
+	var err error
+
+	if device == "any" {
+		h.TPacket, err = afpacket.NewTPacket(
+			afpacket.OptFrameSize(snaplen),
+			afpacket.OptBlockSize(block_size),
+			afpacket.OptNumBlocks(num_blocks),
+			afpacket.OptAddVLANHeader(false),
+			afpacket.OptPollTimeout(timeout),
+			afpacket.SocketRaw,
+			afpacket.TPacketVersion3)
+	} else {
+		h.TPacket, err = afpacket.NewTPacket(
+			afpacket.OptInterface(device),
+			afpacket.OptFrameSize(snaplen),
+			afpacket.OptBlockSize(block_size),
+			afpacket.OptNumBlocks(num_blocks),
+			afpacket.OptAddVLANHeader(false),
+			afpacket.OptPollTimeout(timeout),
+			afpacket.SocketRaw,
+			afpacket.TPacketVersion3)
+	}
+	return h, err
+}
+
+// ReadPacketData satisfies PacketDataSource interface
+func (h *afpacketHandle) ReadPacketData() (data []byte, ci gopacket.CaptureInfo, err error) {
+	return h.TPacket.ReadPacketData()
+}
+
+// SetBPFFilter translates a BPF filter string into BPF RawInstruction and applies them.
+func (h *afpacketHandle) SetBPFFilter(filter string, snaplen int) (err error) {
+	pcapBPF, err := pcap.CompileBPFFilter(layers.LinkTypeEthernet, snaplen, filter)
+	if err != nil {
+		return err
+	}
+	bpfIns := []bpf.RawInstruction{}
+	for _, ins := range pcapBPF {
+		bpfIns2 := bpf.RawInstruction{
+			Op: ins.Code,
+			Jt: ins.Jt,
+			Jf: ins.Jf,
+			K:  ins.K,
+		}
+		bpfIns = append(bpfIns, bpfIns2)
+	}
+	if h.TPacket.SetBPF(bpfIns); err != nil {
+		return err
+	}
+	return h.TPacket.SetBPF(bpfIns)
+}
+
+// LinkType returns ethernet link type.
+func (h *afpacketHandle) LinkType() layers.LinkType {
+	return layers.LinkTypeEthernet
+}
+
+// Close will close afpacket source.
+func (h *afpacketHandle) Close() {
+	h.TPacket.Close()
+}
+
+// SocketStats prints received, dropped, queue-freeze packet stats.
+func (h *afpacketHandle) SocketStats() (as afpacket.SocketStats, asv afpacket.SocketStatsV3, err error) {
+	return h.TPacket.SocketStats()
+}
+
+// afpacketComputeSize computes the block_size and the num_blocks in such a way that the
+// allocated mmap buffer is close to but smaller than target_size_mb.
+// The restriction is that the block_size must be divisible by both the
+// frame size and page size.
+func afpacketComputeSize(targetSizeMb int, snaplen int, pageSize int) (
+	frameSize int, blockSize int, numBlocks int, err error) {
+
+	if snaplen < pageSize {
+		frameSize = pageSize / (pageSize / snaplen)
+	} else {
+		frameSize = (snaplen/pageSize + 1) * pageSize
+	}
+
+	// 128 is the default from the gopacket library so just use that
+	blockSize = frameSize * 128
+	numBlocks = (targetSizeMb * 1024 * 1024) / blockSize
+
+	if numBlocks == 0 {
+		return 0, 0, 0, fmt.Errorf("Interface buffersize is too small")
+	}
+
+	return frameSize, blockSize, numBlocks, nil
+}

internal/capture/capture.go

@@ -0,0 +1,912 @@
+package capture
+
+import (
+	"context"
+	"errors"
+	"expvar"
+	"fmt"
+	"io"
+	"log"
+	"net"
+	"os"
+	"runtime"
+	"strings"
+	"sync"
+	"syscall"
+	"time"
+
+	"github.com/buger/goreplay/internal/size"
+	"github.com/buger/goreplay/internal/tcp"
+	"github.com/buger/goreplay/pkg/http_proto"
+
+	"github.com/google/gopacket"
+	"github.com/google/gopacket/layers"
+	"github.com/google/gopacket/pcap"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/rest"
+)
+
+var stats *expvar.Map
+
+func init() {
+	stats = expvar.NewMap("raw")
+	stats.Init()
+}
+
+// PacketHandler is a function that is used to handle packets
+type PacketHandler func(*tcp.Packet)
+
+type PcapStatProvider interface {
+	Stats() (*pcap.Stats, error)
+}
+
+type PcapSetFilter interface {
+	SetBPFFilter(string) error
+}
+
+// PcapOptions options that can be set on a pcap capture handle,
+// these options take effect on inactive pcap handles
+type PcapOptions struct {
+	BufferTimeout   time.Duration   `json:"input-raw-buffer-timeout"`
+	TimestampType   string          `json:"input-raw-timestamp-type"`
+	BPFFilter       string          `json:"input-raw-bpf-filter"`
+	BufferSize      size.Size       `json:"input-raw-buffer-size"`
+	Promiscuous     bool            `json:"input-raw-promisc"`
+	Monitor         bool            `json:"input-raw-monitor"`
+	Snaplen         bool            `json:"input-raw-override-snaplen"`
+	Engine          EngineType      `json:"input-raw-engine"`
+	VXLANPort       int             `json:"input-raw-vxlan-port"`
+	VXLANVNIs       []int           `json:"input-raw-vxlan-vni"`
+	VLAN            bool            `json:"input-raw-vlan"`
+	VLANVIDs        []int           `json:"input-raw-vlan-vid"`
+	Expire          time.Duration   `json:"input-raw-expire"`
+	TrackResponse   bool            `json:"input-raw-track-response"`
+	Protocol        tcp.TCPProtocol `json:"input-raw-protocol"`
+	RealIPHeader    string          `json:"input-raw-realip-header"`
+	Stats           bool            `json:"input-raw-stats"`
+	AllowIncomplete bool            `json:"input-raw-allow-incomplete"`
+	IgnoreInterface []string        `json:"input-raw-ignore-interface"`
+	Transport       string
+}
+
+// Listener handle traffic capture, this is its representation.
+type Listener struct {
+	sync.Mutex
+
+	config PcapOptions
+
+	Activate   func() error // function is used to activate the engine. it must be called before reading packets
+	Handles    map[string]packetHandle
+	Interfaces []pcap.Interface
+	loopIndex  int
+	Reading    chan bool // this channel is closed when the listener has started reading packets
+	messages   chan *tcp.Message
+
+	ports []uint16
+	host  string // pcap file name or interface (name, hardware addr, index or ip address)
+
+	closeDone chan struct{}
+	quit      chan struct{}
+	closed    bool
+}
+
+type packetHandle struct {
+	handler gopacket.PacketDataSource
+	ips     []net.IP
+}
+
+// EngineType ...
+type EngineType uint8
+
+// Available engines for intercepting traffic
+const (
+	EnginePcap EngineType = 1 << iota
+	EnginePcapFile
+	EngineRawSocket
+	EngineAFPacket
+	EngineVXLAN
+)
+
+// Set is here so that EngineType can implement flag.Var
+func (eng *EngineType) Set(v string) error {
+	switch v {
+	case "", "libpcap":
+		*eng = EnginePcap
+	case "pcap_file":
+		*eng = EnginePcapFile
+	case "raw_socket":
+		*eng = EngineRawSocket
+	case "af_packet":
+		*eng = EngineAFPacket
+	case "vxlan":
+		*eng = EngineVXLAN
+	default:
+		return fmt.Errorf("invalid engine %s", v)
+	}
+	return nil
+}
+
+func (eng *EngineType) String() (e string) {
+	switch *eng {
+	case EnginePcapFile:
+		e = "pcap_file"
+	case EnginePcap:
+		e = "libpcap"
+	case EngineRawSocket:
+		e = "raw_socket"
+	case EngineAFPacket:
+		e = "af_packet"
+	case EngineVXLAN:
+		e = "vxlan"
+	default:
+		e = ""
+	}
+	return e
+}
+
+// NewListener creates and initialize a new Listener. if transport or/and engine are invalid/unsupported
+// is "tcp" and "pcap", are assumed. l.Engine and l.Transport can help to get the values used.
+// if there is an error it will be associated with getting network interfaces
+func NewListener(host string, ports []uint16, config PcapOptions) (l *Listener, err error) {
+	l = &Listener{}
+
+	l.host = host
+	if l.host == "localhost" {
+		l.host = "127.0.0.1"
+	}
+	l.ports = ports
+
+	l.config = config
+	l.config.Transport = "tcp"
+	l.Handles = make(map[string]packetHandle)
+
+	l.closeDone = make(chan struct{})
+	l.quit = make(chan struct{})
+	l.Reading = make(chan bool)
+	l.messages = make(chan *tcp.Message, 10000)
+
+	if strings.HasPrefix(l.host, "k8s://") {
+		l.config.BPFFilter = l.Filter(pcap.Interface{}, k8sIPs(l.host[6:])...)
+	}
+
+	switch config.Engine {
+	default:
+		l.Activate = l.activatePcap
+	case EngineRawSocket:
+		l.Activate = l.activateRawSocket
+	case EngineAFPacket:
+		l.Activate = l.activateAFPacket
+	case EnginePcapFile:
+		l.Activate = l.activatePcapFile
+		return
+	case EngineVXLAN:
+		l.Activate = l.activateVxLanSocket
+		return
+	}
+
+	err = l.setInterfaces()
+	if err != nil {
+		return nil, err
+	}
+	return
+}
+
+// Listen listens for packets from the handles, and call handler on every packet received
+// until the context done signal is sent or there is unrecoverable error on all handles.
+// this function must be called after activating pcap handles
+func (l *Listener) Listen(ctx context.Context) (err error) {
+	l.Lock()
+	for key, handle := range l.Handles {
+		go l.readHandle(key, handle)
+	}
+	l.Unlock()
+
+	go func() {
+		for {
+			time.Sleep(time.Second)
+
+			if l.closed {
+				return
+			}
+
+			// Check for Pod IP changes
+			if strings.HasPrefix(l.host, "k8s://") {
+				newFilter := l.Filter(pcap.Interface{}, k8sIPs(l.host[6:])...)
+				if newFilter != l.config.BPFFilter {
+					fmt.Println("k8s pods configuration changed, new filter: ", newFilter)
+					for _, h := range l.Handles {
+						if _, ok := h.handler.(PcapSetFilter); ok {
+							h.handler.(PcapSetFilter).SetBPFFilter(newFilter)
+						}
+					}
+
+					l.config.BPFFilter = newFilter
+				}
+			}
+
+			var prevInterfaces []string
+			for _, in := range l.Interfaces {
+				prevInterfaces = append(prevInterfaces, in.Name)
+			}
+			l.setInterfaces()
+
+			for _, in := range l.Interfaces {
+				var found bool
+
+				for _, prev := range prevInterfaces {
+					if in.Name == prev {
+						found = true
+					}
+				}
+
+				if !found {
+					fmt.Println("Found new interface:", in.Name)
+					l.Lock()
+					l.Activate()
+
+					for key, handle := range l.Handles {
+						if key == in.Name {
+							fmt.Println("Activating capture on:", in.Name)
+							go l.readHandle(key, handle)
+							break
+						}
+					}
+					l.Unlock()
+				}
+			}
+		}
+	}()
+
+	close(l.Reading)
+	done := ctx.Done()
+	select {
+	case <-done:
+		close(l.quit) // signal close on all handles
+		<-l.closeDone // wait all handles to be closed
+		err = ctx.Err()
+	case <-l.closeDone: // all handles closed voluntarily
+	}
+
+	l.closed = true
+	return
+}
+
+// ListenBackground is like listen but can run concurrently and signal error through channel
+func (l *Listener) ListenBackground(ctx context.Context) chan error {
+	err := make(chan error, 1)
+	go func() {
+		defer close(err)
+		if e := l.Listen(ctx); err != nil {
+			err <- e
+		}
+	}()
+	return err
+}
+
+// Allowed format:
+//
+//	[namespace/]pod/[pod_name]
+//	[namespace/]deployment/[deployment_name]
+//	[namespace/]daemonset/[daemonset_name]
+//	[namespace/]labelSelector/[selector]
+//	[namespace/]fieldSelector/[selector]
+func k8sIPs(addr string) []string {
+	config, err := rest.InClusterConfig()
+	if err != nil {
+		panic(err.Error())
+	}
+
+	// creates the clientset
+	clientset, err := kubernetes.NewForConfig(config)
+	if err != nil {
+		panic(err.Error())
+	}
+
+	sections := strings.Split(addr, "/")
+
+	if len(sections) < 2 {
+		panic("Not supported k8s scheme. Allowed values: [namespace/]pod/[pod_name], [namespace/]deployment/[deployment_name], [namespace/]daemonset/[daemonset_name], [namespace/]label/[label-name]/[label-value]")
+	}
+
+	// If no namespace passed, assume it is ALL
+	switch sections[0] {
+	case "pod", "deployment", "daemonset", "labelSelector", "fieldSelector":
+		sections = append([]string{""}, sections...)
+	}
+
+	namespace, selectorType, selectorValue := sections[0], sections[1], sections[2]
+
+	labelSelector := ""
+	fieldSelector := ""
+
+	switch selectorType {
+	case "pod":
+		fieldSelector = "metadata.name=" + selectorValue
+	case "deployment":
+		labelSelector = "app=" + selectorValue
+	case "daemonset":
+		labelSelector = "pod-template-generation=1,name=" + selectorValue
+	case "labelSelector":
+		labelSelector = selectorValue
+	case "fieldSelector":
+		fieldSelector = selectorValue
+	}
+
+	pods, err := clientset.CoreV1().Pods(namespace).List(context.TODO(), metav1.ListOptions{LabelSelector: labelSelector, FieldSelector: fieldSelector})
+	if err != nil {
+		panic(err.Error())
+	}
+
+	var podIPs []string
+	for _, pod := range pods.Items {
+		for _, podIP := range pod.Status.PodIPs {
+			podIPs = append(podIPs, podIP.IP)
+		}
+	}
+	return podIPs
+}
+
+// Filter returns automatic filter applied by goreplay
+// to a pcap handle of a specific interface
+func (l *Listener) Filter(ifi pcap.Interface, hosts ...string) (filter string) {
+	// https://www.tcpdump.org/manpages/pcap-filter.7.html
+
+	if len(hosts) == 0 {
+		// If k8s have not found any IPs
+		if strings.HasPrefix(l.host, "k8s://") {
+			hosts = []string{}
+		} else {
+			hosts = []string{l.host}
+
+			if listenAll(l.host) || isDevice(l.host, ifi) {
+				hosts = interfaceAddresses(ifi)
+			}
+		}
+	}
+
+	filter = portsFilter(l.config.Transport, "dst", l.ports)
+
+	if len(hosts) != 0 && !l.config.Promiscuous {
+		filter = fmt.Sprintf("((%s) and (%s))", filter, hostsFilter("dst", hosts))
+	} else {
+		filter = fmt.Sprintf("(%s)", filter)
+	}
+
+	if l.config.TrackResponse {
+		responseFilter := portsFilter(l.config.Transport, "src", l.ports)
+
+		if len(hosts) != 0 && !l.config.Promiscuous {
+			responseFilter = fmt.Sprintf("((%s) and (%s))", responseFilter, hostsFilter("src", hosts))
+		} else {
+			responseFilter = fmt.Sprintf("(%s)", responseFilter)
+		}
+
+		filter = fmt.Sprintf("%s or %s", filter, responseFilter)
+	}
+
+	if l.config.VLAN {
+		if len(l.config.VLANVIDs) > 0 {
+			for _, vi := range l.config.VLANVIDs {
+				filter = fmt.Sprintf("vlan %d and ", vi) + filter
+			}
+		} else {
+			filter = "vlan and " + filter
+		}
+	}
+
+	return
+}
+
+// PcapHandle returns new pcap Handle from dev on success.
+// this function should be called after setting all necessary options for this listener
+func (l *Listener) PcapHandle(ifi pcap.Interface) (handle *pcap.Handle, err error) {
+	var inactive *pcap.InactiveHandle
+	inactive, err = pcap.NewInactiveHandle(ifi.Name)
+	if err != nil {
+		return nil, fmt.Errorf("inactive handle error: %q, interface: %q", err, ifi.Name)
+	}
+	defer inactive.CleanUp()
+
+	if l.config.TimestampType != "" && l.config.TimestampType != "go" {
+		var ts pcap.TimestampSource
+		ts, err = pcap.TimestampSourceFromString(l.config.TimestampType)
+		fmt.Println("Setting custom Timestamp Source. Supported values: `go`, ", inactive.SupportedTimestamps())
+		err = inactive.SetTimestampSource(ts)
+		if err != nil {
+			return nil, fmt.Errorf("%q: supported timestamps: %q, interface: %q", err, inactive.SupportedTimestamps(), ifi.Name)
+		}
+	}
+	if l.config.Promiscuous {
+		if err = inactive.SetPromisc(l.config.Promiscuous); err != nil {
+			return nil, fmt.Errorf("promiscuous mode error: %q, interface: %q", err, ifi.Name)
+		}
+	}
+	if l.config.Monitor {
+		if err = inactive.SetRFMon(l.config.Monitor); err != nil && !errors.Is(err, pcap.CannotSetRFMon) {
+			return nil, fmt.Errorf("monitor mode error: %q, interface: %q", err, ifi.Name)
+		}
+	}
+
+	var snap int
+
+	if !l.config.Snaplen {
+		infs, _ := net.Interfaces()
+		for _, i := range infs {
+			if i.Name == ifi.Name {
+				snap = i.MTU + 200
+			}
+		}
+	}
+
+	if snap == 0 {
+		snap = 64<<10 + 200
+	}
+
+	err = inactive.SetSnapLen(snap)
+	if err != nil {
+		return nil, fmt.Errorf("snapshot length error: %q, interface: %q", err, ifi.Name)
+	}
+	if l.config.BufferSize > 0 {
+		err = inactive.SetBufferSize(int(l.config.BufferSize))
+		if err != nil {
+			return nil, fmt.Errorf("handle buffer size error: %q, interface: %q", err, ifi.Name)
+		}
+	}
+	if l.config.BufferTimeout == 0 {
+		l.config.BufferTimeout = 2000 * time.Millisecond
+	}
+	err = inactive.SetTimeout(l.config.BufferTimeout)
+	if err != nil {
+		return nil, fmt.Errorf("handle buffer timeout error: %q, interface: %q", err, ifi.Name)
+	}
+	handle, err = inactive.Activate()
+	if err != nil {
+		return nil, fmt.Errorf("PCAP Activate device error: %q, interface: %q", err, ifi.Name)
+	}
+
+	bpfFilter := l.config.BPFFilter
+	if bpfFilter == "" {
+		bpfFilter = l.Filter(ifi)
+	}
+	fmt.Println("Interface:", ifi.Name, ". BPF Filter:", bpfFilter)
+	err = handle.SetBPFFilter(bpfFilter)
+	if err != nil {
+		handle.Close()
+		return nil, fmt.Errorf("BPF filter error: %q%s, interface: %q", err, bpfFilter, ifi.Name)
+	}
+	return
+}
+
+// SocketHandle returns new unix ethernet handle associated with this listener settings
+func (l *Listener) SocketHandle(ifi pcap.Interface) (handle Socket, err error) {
+	handle, err = NewSocket(ifi)
+	if err != nil {
+		return nil, fmt.Errorf("sock raw error: %q, interface: %q", err, ifi.Name)
+	}
+	if err = handle.SetPromiscuous(l.config.Promiscuous || l.config.Monitor); err != nil {
+		return nil, fmt.Errorf("promiscuous mode error: %q, interface: %q", err, ifi.Name)
+	}
+	if l.config.BPFFilter == "" {
+		l.config.BPFFilter = l.Filter(ifi)
+	}
+	fmt.Println("BPF Filter: ", l.config.BPFFilter)
+	if err = handle.SetBPFFilter(l.config.BPFFilter); err != nil {
+		handle.Close()
+		return nil, fmt.Errorf("BPF filter error: %q%s, interface: %q", err, l.config.BPFFilter, ifi.Name)
+	}
+	handle.SetLoopbackIndex(int32(l.loopIndex))
+	return
+}
+
+func http1StartHint(pckt *tcp.Packet) (isRequest, isResponse bool) {
+	if http_proto.HasRequestTitle(pckt.Payload) {
+		return true, false
+	}
+
+	if http_proto.HasResponseTitle(pckt.Payload) {
+		return false, true
+	}
+
+	// No request or response detected
+	return false, false
+}
+
+func http1EndHint(m *tcp.Message) bool {
+	if m.MissingChunk() {
+		return false
+	}
+
+	req, res := http1StartHint(m.Packets()[0])
+	return http_proto.HasFullPayload(m, m.PacketData()...) && (req || res)
+}
+
+func (l *Listener) readHandle(key string, hndl packetHandle) {
+	runtime.LockOSThread()
+
+	defer l.closeHandles(key)
+	linkSize := 14
+	linkType := int(layers.LinkTypeEthernet)
+	if _, ok := hndl.handler.(*pcap.Handle); ok {
+		linkType = int(hndl.handler.(*pcap.Handle).LinkType())
+		linkSize, ok = pcapLinkTypeLength(linkType, l.config.VLAN)
+		if !ok {
+			if os.Getenv("GORDEBUG") != "0" {
+				log.Printf("can not identify link type of an interface '%s'\n", key)
+			}
+			return // can't find the linktype size
+		}
+	}
+
+	messageParser := tcp.NewMessageParser(l.messages, l.ports, hndl.ips, l.config.Expire, l.config.AllowIncomplete)
+
+	if l.config.Protocol == tcp.ProtocolHTTP {
+		messageParser.Start = http1StartHint
+		messageParser.End = http1EndHint
+	}
+
+	timer := time.NewTicker(1 * time.Second)
+
+	for {
+		select {
+		case <-l.quit:
+			return
+		case <-timer.C:
+			if h, ok := hndl.handler.(PcapStatProvider); ok {
+				s, err := h.Stats()
+				if err == nil {
+					stats.Add("packets_received", int64(s.PacketsReceived))
+					stats.Add("packets_dropped", int64(s.PacketsDropped))
+					stats.Add("packets_if_dropped", int64(s.PacketsIfDropped))
+				}
+			}
+		default:
+			data, ci, err := hndl.handler.ReadPacketData()
+			if err == nil {
+				if l.config.TimestampType == "go" {
+					ci.Timestamp = time.Now()
+				}
+
+				messageParser.PacketHandler(&tcp.PcapPacket{
+					Data:     data,
+					LType:    linkType,
+					LTypeLen: linkSize,
+					Ci:       &ci,
+				})
+				continue
+			}
+			if enext, ok := err.(pcap.NextError); ok && enext == pcap.NextErrorTimeoutExpired {
+				continue
+			}
+			if eno, ok := err.(syscall.Errno); ok && eno.Temporary() {
+				continue
+			}
+			if enet, ok := err.(*net.OpError); ok && (enet.Temporary() || enet.Timeout()) {
+				continue
+			}
+			if err == io.EOF || err == io.ErrClosedPipe {
+				log.Printf("stopped reading from %s interface with error %s\n", key, err)
+				return
+			}
+
+			log.Printf("stopped reading from %s interface with error %s\n", key, err)
+			return
+		}
+	}
+}
+
+func (l *Listener) Messages() chan *tcp.Message {
+	return l.messages
+}
+
+func (l *Listener) closeHandles(key string) {
+	l.Lock()
+	defer l.Unlock()
+	if handle, ok := l.Handles[key]; ok {
+		if c, ok := handle.handler.(io.Closer); ok {
+			c.Close()
+		}
+
+		delete(l.Handles, key)
+		if len(l.Handles) == 0 {
+			close(l.closeDone)
+		}
+	}
+}
+
+func (l *Listener) activatePcap() error {
+	var e error
+	var msg string
+	for _, ifi := range l.Interfaces {
+		if _, found := l.Handles[ifi.Name]; found {
+			continue
+		}
+
+		var handle *pcap.Handle
+		handle, e = l.PcapHandle(ifi)
+		if e != nil {
+			msg += ("\n" + e.Error())
+			continue
+		}
+		l.Handles[ifi.Name] = packetHandle{
+			handler: handle,
+			ips:     interfaceIPs(ifi),
+		}
+	}
+	if len(l.Handles) == 0 {
+		return fmt.Errorf("pcap handles error:%s", msg)
+	}
+	return nil
+}
+
+func (l *Listener) activateVxLanSocket() error {
+	handler, err := newVXLANHandler(l.config.VXLANPort, l.config.VXLANVNIs)
+	if err != nil {
+		return err
+	}
+	l.Handles["vxlan"] = packetHandle{
+		handler: handler,
+	}
+
+	return nil
+}
+
+func (l *Listener) activateRawSocket() error {
+	if runtime.GOOS != "linux" {
+		return fmt.Errorf("sock_raw is not stabilized on OS other than linux")
+	}
+	var msg string
+	var e error
+	for _, ifi := range l.Interfaces {
+		if _, found := l.Handles[ifi.Name]; found {
+			continue
+		}
+
+		var handle Socket
+		handle, e = l.SocketHandle(ifi)
+		if e != nil {
+			msg += ("\n" + e.Error())
+			continue
+		}
+		l.Handles[ifi.Name] = packetHandle{
+			handler: handle,
+			ips:     interfaceIPs(ifi),
+		}
+	}
+	if len(l.Handles) == 0 {
+		return fmt.Errorf("raw socket handles error:%s", msg)
+	}
+	return nil
+}
+
+func (l *Listener) activatePcapFile() (err error) {
+	var handle *pcap.Handle
+	var e error
+	if handle, e = pcap.OpenOffline(l.host); e != nil {
+		return fmt.Errorf("open pcap file error: %q", e)
+	}
+
+	tmp := l.host
+	l.host = ""
+	l.config.BPFFilter = l.Filter(pcap.Interface{})
+	l.host = tmp
+
+	if e = handle.SetBPFFilter(l.config.BPFFilter); e != nil {
+		handle.Close()
+		return fmt.Errorf("BPF filter error: %q, filter: %s", e, l.config.BPFFilter)
+	}
+
+	fmt.Println("BPF Filter:", l.config.BPFFilter)
+
+	l.Handles["pcap_file"] = packetHandle{
+		handler: handle,
+	}
+	return
+}
+
+func (l *Listener) activateAFPacket() error {
+	szFrame, szBlock, numBlocks, err := afpacketComputeSize(32, 32<<10, os.Getpagesize())
+	if err != nil {
+		return err
+	}
+
+	var msg string
+	for _, ifi := range l.Interfaces {
+		if _, found := l.Handles[ifi.Name]; found {
+			continue
+		}
+
+		handle, err := newAfpacketHandle(ifi.Name, szFrame, szBlock, numBlocks, false, pcap.BlockForever)
+
+		if err != nil {
+			msg += ("\n" + err.Error())
+			continue
+		}
+
+		if l.config.BPFFilter == "" {
+			l.config.BPFFilter = l.Filter(ifi)
+		}
+		fmt.Println("Interface:", ifi.Name, ". BPF Filter:", l.config.BPFFilter)
+		handle.SetBPFFilter(l.config.BPFFilter, 64<<10)
+
+		l.Handles[ifi.Name] = packetHandle{
+			handler: handle,
+			ips:     interfaceIPs(ifi),
+		}
+	}
+
+	if len(l.Handles) == 0 {
+		return fmt.Errorf("pcap handles error:%s", msg)
+	}
+
+	return nil
+}
+
+func (l *Listener) setInterfaces() (err error) {
+	var pifis []pcap.Interface
+	pifis, err = pcap.FindAllDevs()
+	ifis, _ := net.Interfaces()
+	l.Interfaces = []pcap.Interface{}
+
+	if err != nil {
+		return
+	}
+
+	for _, pi := range pifis {
+		ignore := false
+		for _, ig := range l.config.IgnoreInterface {
+			if pi.Name == ig {
+				ignore = true
+				break
+			}
+		}
+
+		if ignore {
+			continue
+		}
+
+		if strings.HasPrefix(l.host, "k8s://") {
+			if !strings.HasPrefix(pi.Name, "veth") {
+				continue
+			}
+		}
+
+		if isDevice(l.host, pi) {
+			l.Interfaces = []pcap.Interface{pi}
+			return
+		}
+
+		var ni net.Interface
+		for _, i := range ifis {
+			if i.Name == pi.Name {
+				ni = i
+				break
+			}
+
+			addrs, _ := i.Addrs()
+			for _, a := range addrs {
+				for _, pa := range pi.Addresses {
+					if a.String() == pa.IP.String() {
+						ni = i
+						break
+					}
+				}
+			}
+		}
+
+		if ni.Flags&net.FlagLoopback != 0 {
+			l.loopIndex = ni.Index
+		}
+
+		if runtime.GOOS != "windows" {
+			if len(pi.Addresses) == 0 {
+				continue
+			}
+
+			if ni.Flags&net.FlagUp == 0 {
+				continue
+			}
+		}
+
+		l.Interfaces = append(l.Interfaces, pi)
+	}
+	return
+}
+
+func isDevice(addr string, ifi pcap.Interface) bool {
+	// Windows npcap loopback have no IPs
+	if addr == "127.0.0.1" && ifi.Name == `\Device\NPF_Loopback` {
+		return true
+	}
+
+	if addr == ifi.Name {
+		return true
+	}
+
+	if strings.HasSuffix(addr, "*") {
+		if strings.HasPrefix(ifi.Name, addr[:len(addr)-1]) {
+			return true
+		}
+	}
+
+	for _, _addr := range ifi.Addresses {
+		if _addr.IP.String() == addr {
+			return true
+		}
+	}
+
+	return false
+}
+
+func interfaceAddresses(ifi pcap.Interface) []string {
+	var hosts []string
+	for _, addr := range ifi.Addresses {
+		hosts = append(hosts, addr.IP.String())
+	}
+	return hosts
+}
+
+func interfaceIPs(ifi pcap.Interface) []net.IP {
+	var ips []net.IP
+	for _, addr := range ifi.Addresses {
+		ips = append(ips, addr.IP)
+	}
+	return ips
+}
+
+func listenAll(addr string) bool {
+	switch addr {
+	case "", "0.0.0.0", "[::]", "::":
+		return true
+	}
+	return false
+}
+
+func portsFilter(transport string, direction string, ports []uint16) string {
+	if len(ports) == 0 || ports[0] == 0 {
+		return fmt.Sprintf("%s %s portrange 0-%d", transport, direction, 1<<16-1)
+	}
+
+	var filters []string
+	for _, port := range ports {
+		filters = append(filters, fmt.Sprintf("%s %s port %d", transport, direction, port))
+	}
+	return strings.Join(filters, " or ")
+}
+
+func hostsFilter(direction string, hosts []string) string {
+	var hostsFilters []string
+	for _, host := range hosts {
+		hostsFilters = append(hostsFilters, fmt.Sprintf("%s host %s", direction, host))
+	}
+
+	return strings.Join(hostsFilters, " or ")
+}
+
+func pcapLinkTypeLength(lType int, vlan bool) (int, bool) {
+	switch layers.LinkType(lType) {
+	case layers.LinkTypeEthernet:
+		if vlan {
+			return 18, true
+		} else {
+			return 14, true
+		}
+	case layers.LinkTypeNull, layers.LinkTypeLoop:
+		return 4, true
+	case layers.LinkTypeRaw, 12, 14:
+		return 0, true
+	case layers.LinkTypeIPv4, layers.LinkTypeIPv6:
+		// (TODO:) look out for IP encapsulation?
+		return 0, true
+	case layers.LinkTypeLinuxSLL:
+		return 16, true
+	case layers.LinkTypeFDDI:
+		return 13, true
+	case 226 /*DLT_IPNET*/ :
+		// https://www.tcpdump.org/linktypes/LINKTYPE_IPNET.html
+		return 24, true
+	default:
+		return 0, false
+	}
+}

internal/capture/capture_test.go

@@ -0,0 +1,22 @@
+package capture
+
+import (
+	"testing"
+)
+
+func TestSetInterfaces(t *testing.T) {
+	listener := &Listener{
+		loopIndex: 99999,
+	}
+	listener.setInterfaces()
+
+	for _, nic := range listener.Interfaces {
+		if (len(nic.Addresses)) == 0 {
+			t.Errorf("nic %s was captured with 0 addresses", nic.Name)
+		}
+	}
+
+	if listener.loopIndex == 99999 {
+		t.Errorf("loopback nic index was not found")
+	}
+}

internal/capture/doc.go

@@ -9,27 +9,34 @@ example:
 
 // for the transport should be "tcp"
 listener, err := capture.NewListener(host, port, transport, engine, trackResponse)
-if err != nil {
-	// handle error
-}
+
+	if err != nil {
+		// handle error
+	}
+
 listener.SetPcapOptions(opts)
 err = listner.Activate()
-if err != nil {
-	// handle it
-}
 
-if err := listener.Listen(context.Background(), handler); err != nil {
-	 // handle error
-}
+	if err != nil {
+		// handle it
+	}
+
+	if err := listener.Listen(context.Background(), handler); err != nil {
+		 // handle error
+	}
+
 // or
-errCh := listener.ListenBackground(context.Background(), handler) // runs in the backgorund
+errCh := listener.ListenBackground(context.Background(), handler) // runs in the background
 select {
 case err := <- errCh:
+
 	// handle error
+
 case <-quit:
+
 	//
+
 case <- l.Reading: // if we have started reading
 }
-
 */
 package capture // import github.com/buger/goreplay/capture

internal/capture/dump.go

@@ -35,18 +35,18 @@ const versionMinor = 4
 // an append), you must call WriteFileHeader before WritePacket.  Packet
 // timestamps are written with nanosecond precision.
 //
-//  // Write a new file:
-//  f, _ := os.Create("/tmp/file.pcap")
-//  w := pcapgo.NewWriterNanos(f)
-//  w.WriteFileHeader(65536, layers.LinkTypeEthernet)  // new file, must do this.
-//  w.WritePacket(gopacket.CaptureInfo{...}, data1)
-//  f.Close()
-//  // Append to existing file (must have same snaplen and linktype)
-//  f2, _ := os.OpenFile("/tmp/fileNano.pcap", os.O_APPEND, 0700)
-//  w2 := pcapgo.NewWriter(f2)
-//  // no need for file header, it's already written.
-//  w2.WritePacket(gopacket.CaptureInfo{...}, data2)
-//  f2.Close()
+//	// Write a new file:
+//	f, _ := os.Create("/tmp/file.pcap")
+//	w := pcapgo.NewWriterNanos(f)
+//	w.WriteFileHeader(65536, layers.LinkTypeEthernet)  // new file, must do this.
+//	w.WritePacket(gopacket.CaptureInfo{...}, data1)
+//	f.Close()
+//	// Append to existing file (must have same snaplen and linktype)
+//	f2, _ := os.OpenFile("/tmp/fileNano.pcap", os.O_APPEND, 0700)
+//	w2 := pcapgo.NewWriter(f2)
+//	// no need for file header, it's already written.
+//	w2.WritePacket(gopacket.CaptureInfo{...}, data2)
+//	f2.Close()
 func NewWriterNanos(w io.Writer) *Writer {
 	return &Writer{w: w, tsScaler: nanosPerNano}
 }
@@ -54,20 +54,20 @@ func NewWriterNanos(w io.Writer) *Writer {
 // NewWriter returns a new writer object, for writing packet data out
 // to the given writer.  If this is a new empty writer (as opposed to
 // an append), you must call WriteFileHeader before WritePacket.
-// Packet timestamps are written witn microsecond precision.
+// Packet timestamps are written with microsecond precision.
 //
-//  // Write a new file:
-//  f, _ := os.Create("/tmp/file.pcap")
-//  w := pcapgo.NewWriter(f)
-//  w.WriteFileHeader(65536, layers.LinkTypeEthernet)  // new file, must do this.
-//  w.WritePacket(gopacket.CaptureInfo{...}, data1)
-//  f.Close()
-//  // Append to existing file (must have same snaplen and linktype)
-//  f2, _ := os.OpenFile("/tmp/file.pcap", os.O_APPEND, 0700)
-//  w2 := pcapgo.NewWriter(f2)
-//  // no need for file header, it's already written.
-//  w2.WritePacket(gopacket.CaptureInfo{...}, data2)
-//  f2.Close()
+//	// Write a new file:
+//	f, _ := os.Create("/tmp/file.pcap")
+//	w := pcapgo.NewWriter(f)
+//	w.WriteFileHeader(65536, layers.LinkTypeEthernet)  // new file, must do this.
+//	w.WritePacket(gopacket.CaptureInfo{...}, data1)
+//	f.Close()
+//	// Append to existing file (must have same snaplen and linktype)
+//	f2, _ := os.OpenFile("/tmp/file.pcap", os.O_APPEND, 0700)
+//	w2 := pcapgo.NewWriter(f2)
+//	// no need for file header, it's already written.
+//	w2.WritePacket(gopacket.CaptureInfo{...}, data2)
+//	f2.Close()
 func NewWriter(w io.Writer) *Writer {
 	return &Writer{w: w, tsScaler: nanosPerMicro}
 }

internal/capture/sock_linux.go

@@ -1,3 +1,5 @@
+//go:build linux && !arm64
+
 package capture
 
 import (
@@ -44,7 +46,23 @@ type SockRaw struct {
 }
 
 // NewSocket returns new M'maped sock_raw on packet version 2.
-func NewSocket(ifi net.Interface) (*SockRaw, error) {
+func NewSocket(pifi pcap.Interface) (*SockRaw, error) {
+	var ifi net.Interface
+
+	infs, _ := net.Interfaces()
+	found := false
+	for _, i := range infs {
+		if i.Name == pifi.Name {
+			ifi = i
+			found = true
+			break
+		}
+	}
+
+	if !found {
+		return nil, fmt.Errorf("can't find matching interface")
+	}
+
 	// sock create
 	fd, err := unix.Socket(unix.AF_PACKET, unix.SOCK_RAW, int(ETHALL))
 	if err != nil {
@@ -108,7 +126,7 @@ func NewSocket(ifi net.Interface) (*SockRaw, error) {
 }
 
 // ReadPacketData implements gopacket.PacketDataSource.
-func (sock *SockRaw) ZeroCopyReadPacketData() (buf []byte, ci gopacket.CaptureInfo, err error) {
+func (sock *SockRaw) ReadPacketData() (buf []byte, ci gopacket.CaptureInfo, err error) {
 	sock.mu.Lock()
 	defer sock.mu.Unlock()
 	var tpHdr *unix.Tpacket2Hdr
@@ -217,7 +235,7 @@ func (sock *SockRaw) SetBPFFilter(expr string) error {
 	return unix.SetsockoptSockFprog(sock.fd, unix.SOL_SOCKET, unix.SO_ATTACH_FILTER, fprog)
 }
 
-// SetPromiscuous sets promiscous mode to the required value. for better result capture on all interfaces instead.
+// SetPromiscuous sets promiscuous mode to the required value. for better result capture on all interfaces instead.
 // If it is enabled, traffic not destined for the interface will also be captured.
 func (sock *SockRaw) SetPromiscuous(b bool) error {
 	sock.mu.Lock()

internal/capture/sock_others.go

@@ -1,13 +1,14 @@
-// +build !linux
+//go:build !linux || arm64 || darwin
 
 package capture
 
 import (
 	"errors"
-	"net"
+
+	"github.com/google/gopacket/pcap"
 )
 
 // NewSocket returns new M'maped sock_raw on packet version 2.
-func NewSocket(_ net.Interface) (Socket, error) {
+func NewSocket(_ pcap.Interface) (Socket, error) {
 	return nil, errors.New("afpacket socket is only available on linux")
 }

internal/capture/socket.go

@@ -8,7 +8,7 @@ import (
 
 // Socket is any interface that defines the behaviors of Socket
 type Socket interface {
-	ZeroCopyReadPacketData() ([]byte, gopacket.CaptureInfo, error)
+	ReadPacketData() ([]byte, gopacket.CaptureInfo, error)
 	WritePacketData([]byte) error
 	SetBPFFilter(string) error
 	SetPromiscuous(bool) error

internal/capture/vxlan.go

@@ -0,0 +1,100 @@
+package capture
+
+import (
+	"errors"
+	"fmt"
+	"github.com/google/gopacket"
+	"github.com/google/gopacket/layers"
+	"net"
+	"time"
+)
+
+const VxLanPacketSize = 1526 //vxlan 8 B + ethernet II 1518 B
+
+type vxlanHandle struct {
+	connection    *net.UDPConn
+	packetChannel chan gopacket.Packet
+	vnis          []int
+}
+
+func newVXLANHandler(port int, vnis []int) (*vxlanHandle, error) {
+	if port == 0 {
+		port = 4789
+	}
+
+	addr := net.UDPAddr{
+		Port: port,
+		IP:   net.ParseIP("0.0.0.0"),
+	}
+
+	vxlanHandle := &vxlanHandle{}
+	con, err := net.ListenUDP("udp", &addr)
+	if err != nil {
+		return nil, fmt.Errorf(err.Error())
+	}
+	vxlanHandle.connection = con
+	vxlanHandle.packetChannel = make(chan gopacket.Packet, 1000)
+	vxlanHandle.vnis = vnis
+	go vxlanHandle.reader()
+
+	return vxlanHandle, nil
+}
+
+func (v *vxlanHandle) reader() {
+	for {
+		inputBytes := make([]byte, VxLanPacketSize)
+		length, _, err := v.connection.ReadFromUDP(inputBytes)
+		if err != nil {
+			if errors.Is(err, net.ErrClosed) {
+				return
+			}
+			continue
+		}
+		packet := gopacket.NewPacket(inputBytes[:length], layers.LayerTypeVXLAN, gopacket.NoCopy)
+		ci := packet.Metadata()
+		ci.Timestamp = time.Now()
+		ci.CaptureLength = length
+		ci.Length = length
+
+		if len(v.vnis) > 0 && !v.vniIsAllowed(packet) {
+			continue
+		}
+
+		v.packetChannel <- packet
+	}
+}
+
+func (v *vxlanHandle) vniIsAllowed(packet gopacket.Packet) bool {
+	defaultState := false
+	if layer := packet.Layer(layers.LayerTypeVXLAN); layer != nil {
+		vxlan, _ := layer.(*layers.VXLAN)
+		for _, vn := range v.vnis {
+			if vn > 0 && int(vxlan.VNI) == vn {
+				return true
+			}
+
+			if vn < 0 {
+				if int(vxlan.VNI) == -vn {
+					return false
+				}
+				defaultState = true
+			}
+		}
+	}
+	return defaultState
+}
+
+func (v *vxlanHandle) ReadPacketData() ([]byte, gopacket.CaptureInfo, error) {
+	packet := <-v.packetChannel
+	layer := packet.Layer(layers.LayerTypeVXLAN)
+	bytes := layer.LayerPayload()
+
+	return bytes, packet.Metadata().CaptureInfo, nil
+}
+
+func (v *vxlanHandle) Close() error {
+	if v.connection != nil {
+		return v.connection.Close()
+	}
+	return nil
+}

internal/ring/ring.go

@@ -0,0 +1,205 @@
+package ring
+
+import (
+	"errors"
+	"runtime"
+	"sync/atomic"
+	"time"
+)
+
+var (
+	// ErrDisposed is returned when an operation is performed on a disposed
+	// queue.
+	ErrDisposed = errors.New(`queue: disposed`)
+
+	// ErrTimeout is returned when an applicable queue operation times out.
+	ErrTimeout = errors.New(`queue: poll timed out`)
+
+	// ErrEmptyQueue is returned when an non-applicable queue operation was called
+	// due to the queue's empty item state
+	ErrEmptyQueue = errors.New(`queue: empty queue`)
+)
+
+// roundUp takes a uint64 greater than 0 and rounds it up to the next
+// power of 2.
+func roundUp(v uint64) uint64 {
+	v--
+	v |= v >> 1
+	v |= v >> 2
+	v |= v >> 4
+	v |= v >> 8
+	v |= v >> 16
+	v |= v >> 32
+	v++
+	return v
+}
+
+type node struct {
+	position uint64
+	data     interface{}
+}
+
+type nodes []node
+
+// RingBuffer is a MPMC buffer that achieves threadsafety with CAS operations
+// only.  A put on full or get on empty call will block until an item
+// is put or retrieved.  Calling Dispose on the RingBuffer will unblock
+// any blocked threads with an error.  This buffer is similar to the buffer
+// described here: http://www.1024cores.net/home/lock-free-algorithms/queues/bounded-mpmc-queue
+// with some minor additions.
+type RingBuffer struct {
+	_padding0      [8]uint64
+	queue          uint64
+	_padding1      [8]uint64
+	dequeue        uint64
+	_padding2      [8]uint64
+	mask, disposed uint64
+	_padding3      [8]uint64
+	nodes          nodes
+}
+
+func (rb *RingBuffer) init(size uint64) {
+	size = roundUp(size)
+	rb.nodes = make(nodes, size)
+	for i := uint64(0); i < size; i++ {
+		rb.nodes[i] = node{position: i}
+	}
+	rb.mask = size - 1 // so we don't have to do this with every put/get operation
+}
+
+// Put adds the provided item to the queue.  If the queue is full, this
+// call will block until an item is added to the queue or Dispose is called
+// on the queue.  An error will be returned if the queue is disposed.
+func (rb *RingBuffer) Put(item interface{}) error {
+	_, err := rb.put(item, false)
+	return err
+}
+
+// Offer adds the provided item to the queue if there is space.  If the queue
+// is full, this call will return false.  An error will be returned if the
+// queue is disposed.
+func (rb *RingBuffer) Offer(item interface{}) (bool, error) {
+	return rb.put(item, true)
+}
+
+func (rb *RingBuffer) put(item interface{}, offer bool) (bool, error) {
+	var n *node
+	pos := atomic.LoadUint64(&rb.queue)
+L:
+	for {
+		if atomic.LoadUint64(&rb.disposed) == 1 {
+			return false, ErrDisposed
+		}
+
+		n = &rb.nodes[pos&rb.mask]
+		seq := atomic.LoadUint64(&n.position)
+		switch dif := seq - pos; {
+		case dif == 0:
+			if atomic.CompareAndSwapUint64(&rb.queue, pos, pos+1) {
+				break L
+			}
+		case dif < 0:
+			panic(`Ring buffer in a compromised state during a put operation.`)
+		default:
+			pos = atomic.LoadUint64(&rb.queue)
+		}
+
+		if offer {
+			return false, nil
+		}
+
+		runtime.Gosched() // free up the cpu before the next iteration
+	}
+
+	n.data = item
+	atomic.StoreUint64(&n.position, pos+1)
+	return true, nil
+}
+
+// Get will return the next item in the queue.  This call will block
+// if the queue is empty.  This call will unblock when an item is added
+// to the queue or Dispose is called on the queue.  An error will be returned
+// if the queue is disposed.
+func (rb *RingBuffer) Get() (interface{}, error) {
+	return rb.Poll(0)
+}
+
+// Poll will return the next item in the queue.  This call will block
+// if the queue is empty.  This call will unblock when an item is added
+// to the queue, Dispose is called on the queue, or the timeout is reached. An
+// error will be returned if the queue is disposed or a timeout occurs. A
+// non-positive timeout will block indefinitely.
+func (rb *RingBuffer) Poll(timeout time.Duration) (interface{}, error) {
+	var (
+		n     *node
+		pos   = atomic.LoadUint64(&rb.dequeue)
+		start time.Time
+	)
+	if timeout > 0 {
+		start = time.Now()
+	}
+L:
+	for {
+		if atomic.LoadUint64(&rb.disposed) == 1 {
+			return nil, ErrDisposed
+		}
+
+		n = &rb.nodes[pos&rb.mask]
+		seq := atomic.LoadUint64(&n.position)
+		switch dif := seq - (pos + 1); {
+		case dif == 0:
+			if atomic.CompareAndSwapUint64(&rb.dequeue, pos, pos+1) {
+				break L
+			}
+		case dif < 0:
+			panic(`Ring buffer in compromised state during a get operation.`)
+		default:
+			pos = atomic.LoadUint64(&rb.dequeue)
+		}
+
+		if timeout > 0 && time.Since(start) >= timeout {
+			return nil, ErrTimeout
+		}
+
+		if timeout < 0 {
+			return nil, ErrTimeout
+		}
+
+		runtime.Gosched() // free up the cpu before the next iteration
+	}
+	data := n.data
+	n.data = nil
+	atomic.StoreUint64(&n.position, pos+rb.mask+1)
+	return data, nil
+}
+
+// Len returns the number of items in the queue.
+func (rb *RingBuffer) Len() uint64 {
+	return atomic.LoadUint64(&rb.queue) - atomic.LoadUint64(&rb.dequeue)
+}
+
+// Cap returns the capacity of this ring buffer.
+func (rb *RingBuffer) Cap() uint64 {
+	return uint64(len(rb.nodes))
+}
+
+// Dispose will dispose of this queue and free any blocked threads
+// in the Put and/or Get methods.  Calling those methods on a disposed
+// queue will return an error.
+func (rb *RingBuffer) Dispose() {
+	atomic.CompareAndSwapUint64(&rb.disposed, 0, 1)
+}
+
+// IsDisposed will return a bool indicating if this queue has been
+// disposed.
+func (rb *RingBuffer) IsDisposed() bool {
+	return atomic.LoadUint64(&rb.disposed) == 1
+}
+
+// NewRingBuffer will allocate, initialize, and return a ring buffer
+// with the specified size.
+func NewRingBuffer(size uint64) *RingBuffer {
+	rb := &RingBuffer{}
+	rb.init(size)
+	return rb
+}

internal/simpletime/time.go

@@ -0,0 +1,17 @@
+package simpletime
+
+import (
+	"time"
+)
+
+var Now time.Time
+
+func init() {
+	go func() {
+		for {
+			// Accurate enough
+			Now = time.Now()
+			time.Sleep(100 * time.Millisecond)
+		}
+	}()
+}

internal/tcp/doc.go

@@ -4,7 +4,6 @@ parsing, reassembling tcp packets, handling communication with engine listeners(
 and reporting errors and statistics of packets.
 the packets are parsed by following TCP way(https://en.wikipedia.org/wiki/Transmission_Control_Protocol#TCP_segment_structure).
 
-
 example:
 
 import "github.com/buger/goreplay/tcp"
@@ -22,6 +21,5 @@ you can use pool.End or/and pool.Start to set custom session behaviors
 
 debugLevel in debugger function indicates the priority of the logs, the bigger the number the lower
 the priority. errors are signified by debug level 4 for errors, 5 for discarded packets, and 6 for received packets.
-
 */
 package tcp // import github.com/buger/goreplay/tcp

internal/tcp/tcp_message.go

@@ -0,0 +1,444 @@
+package tcp
+
+import (
+	"encoding/binary"
+	"encoding/hex"
+	"fmt"
+	"net"
+	"reflect"
+	"sort"
+	"time"
+	"unsafe"
+
+	"github.com/buger/goreplay/pkg/http_proto"
+)
+
+// TCPProtocol is a number to indicate type of protocol
+type TCPProtocol uint8
+
+const (
+	// ProtocolHTTP ...
+	ProtocolHTTP TCPProtocol = iota
+	// ProtocolBinary ...
+	ProtocolBinary
+)
+
+// Set is here so that TCPProtocol can implement flag.Var
+func (protocol *TCPProtocol) Set(v string) error {
+	switch v {
+	case "", "http":
+		*protocol = ProtocolHTTP
+	case "binary":
+		*protocol = ProtocolBinary
+	default:
+		return fmt.Errorf("unsupported protocol %s", v)
+	}
+	return nil
+}
+
+func (protocol *TCPProtocol) String() string {
+	switch *protocol {
+	case ProtocolBinary:
+		return "binary"
+	case ProtocolHTTP:
+		return "http"
+	default:
+		return ""
+	}
+}
+
+// Stats every message carry its own stats object
+type Stats struct {
+	LostData  int
+	Length    int       // length of the data
+	Start     time.Time // first packet's timestamp
+	End       time.Time // last packet's timestamp
+	SrcAddr   string
+	DstAddr   string
+	Direction Dir
+	TimedOut  bool // timeout before getting the whole message
+	Truncated bool // last packet truncated due to max message size
+	IPversion byte
+}
+
+// Message is the representation of a tcp message
+type Message struct {
+	packets          []*Packet
+	parser           *MessageParser
+	feedback         interface{}
+	continueAdjusted bool
+	Stats
+}
+
+// UUID returns the UUID of a TCP request and its response.
+func (m *Message) UUID() []byte {
+	var streamID uint64
+	pckt := m.packets[0]
+
+	// check if response or request have generated the ID before.
+	if m.Direction == DirIncoming {
+		streamID = uint64(pckt.SrcPort)<<48 | uint64(pckt.DstPort)<<32 |
+			uint64(ip2int(pckt.SrcIP))
+	} else {
+		streamID = uint64(pckt.DstPort)<<48 | uint64(pckt.SrcPort)<<32 |
+			uint64(ip2int(pckt.DstIP))
+	}
+
+	id := make([]byte, 12)
+	binary.BigEndian.PutUint64(id, streamID)
+
+	if m.Direction == DirIncoming {
+		binary.BigEndian.PutUint32(id[8:], pckt.Ack)
+	} else {
+		binary.BigEndian.PutUint32(id[8:], pckt.Seq)
+	}
+
+	uuidHex := make([]byte, 24)
+	hex.Encode(uuidHex[:], id[:])
+
+	return uuidHex
+}
+
+func (m *Message) add(packet *Packet) bool {
+	// Skip duplicates
+	for _, p := range m.packets {
+		if p.Seq == packet.Seq {
+			return false
+		}
+	}
+
+	// Packets not always captured in same Seq order, and sometimes we need to prepend
+	if len(m.packets) == 0 || packet.Seq > m.packets[len(m.packets)-1].Seq {
+		m.packets = append(m.packets, packet)
+	} else if packet.Seq < m.packets[0].Seq {
+		m.packets = append([]*Packet{packet}, m.packets...)
+	} else { // insert somewhere in the middle...
+		for i, p := range m.packets {
+			if packet.Seq < p.Seq {
+				m.packets = append(m.packets[:i], append([]*Packet{packet}, m.packets[i:]...)...)
+				break
+			}
+		}
+	}
+
+	m.Length += len(packet.Payload)
+	m.LostData += int(packet.Lost)
+
+	if packet.Timestamp.After(m.End) || m.End.IsZero() {
+		m.End = packet.Timestamp
+	}
+
+	return true
+}
+
+// Packets returns packets of the message
+func (m *Message) Packets() []*Packet {
+	return m.packets
+}
+
+func (m *Message) MissingChunk() bool {
+	nextSeq := m.packets[0].Seq
+
+	for _, p := range m.packets {
+		if p.Seq != nextSeq {
+			return true
+		}
+
+		nextSeq += uint32(len(p.Payload))
+	}
+
+	return false
+}
+
+func (m *Message) PacketData() [][]byte {
+	tmp := make([][]byte, len(m.packets))
+
+	for i, p := range m.packets {
+		tmp[i] = p.Payload
+	}
+
+	return tmp
+}
+
+// Data returns data in this message
+func (m *Message) Data() []byte {
+	packetData := m.PacketData()
+	tmp := packetData[0]
+
+	if len(packetData) > 0 {
+		tmp, _ = copySlice(tmp, len(packetData[0]), packetData[1:]...)
+	}
+
+	// Remove Expect header, since its replay not fully supported
+	if state, ok := m.feedback.(*http_proto.HTTPState); ok {
+		if state.Continue100 {
+			tmp = http_proto.DeleteHeader(tmp, []byte("Expect"))
+		}
+	}
+
+	return tmp
+}
+
+// SetProtocolState set feedback/data that can be used later, e.g with End or Start hint
+func (m *Message) SetProtocolState(feedback interface{}) {
+	m.feedback = feedback
+}
+
+// ProtocolState returns feedback associated to this message
+func (m *Message) ProtocolState() interface{} {
+	return m.feedback
+}
+
+// Sort a helper to sort packets
+func (m *Message) Sort() {
+	sort.SliceStable(m.packets, func(i, j int) bool { return m.packets[i].Seq < m.packets[j].Seq })
+}
+
+// Emitter message handler
+type Emitter func(*Message)
+
+// HintEnd hints the parser to stop the session, see MessageParser.End
+// when set, it will be executed before checking FIN or RST flag
+type HintEnd func(*Message) bool
+
+// HintStart hints the parser to start the reassembling the message, see MessageParser.Start
+// when set, it will be called after checking SYN flag
+type HintStart func(*Packet) (IsRequest, IsOutgoing bool)
+
+// MessageParser holds data of all tcp messages in progress(still receiving/sending packets).
+// message is identified by its source port and dst port, and last 4bytes of src IP.
+type MessageParser struct {
+	m map[uint64]*Message
+
+	messageExpire  time.Duration // the maximum time to wait for the final packet, minimum is 100ms
+	allowIncompete bool
+	End            HintEnd
+	Start          HintStart
+	ticker         *time.Ticker
+	messages       chan *Message
+	packets        chan *PcapPacket
+	close          chan struct{} // to signal that we are able to close
+	ports          []uint16
+	ips            []net.IP
+}
+
+// NewMessageParser returns a new instance of message parser
+func NewMessageParser(messages chan *Message, ports []uint16, ips []net.IP, messageExpire time.Duration, allowIncompete bool) (parser *MessageParser) {
+	parser = new(MessageParser)
+
+	parser.messageExpire = messageExpire
+	if parser.messageExpire == 0 {
+		parser.messageExpire = time.Millisecond * 1000
+	}
+
+	parser.allowIncompete = allowIncompete
+
+	parser.packets = make(chan *PcapPacket, 10000)
+
+	if messages == nil {
+		messages = make(chan *Message, 1000)
+	}
+	parser.messages = messages
+
+	parser.m = make(map[uint64]*Message)
+	parser.ticker = time.NewTicker(time.Millisecond * 100)
+	parser.close = make(chan struct{}, 1)
+
+	parser.ports = ports
+	parser.ips = ips
+
+	go parser.wait()
+	return parser
+}
+
+var packetLen int
+
+// Packet returns packet handler
+func (parser *MessageParser) PacketHandler(packet *PcapPacket) {
+	packetLen++
+	parser.packets <- packet
+}
+
+func (parser *MessageParser) wait() {
+	var (
+		now time.Time
+	)
+	for {
+		select {
+		case pckt := <-parser.packets:
+			parser.processPacket(parser.parsePacket(pckt))
+		case now = <-parser.ticker.C:
+			parser.timer(now)
+		case <-parser.close:
+			parser.ticker.Stop()
+			// parser.Close should wait for this function to return
+			parser.close <- struct{}{}
+			return
+			// default:
+		}
+	}
+}
+
+func (parser *MessageParser) parsePacket(pcapPkt *PcapPacket) *Packet {
+	pckt, err := ParsePacket(pcapPkt.Data, pcapPkt.LType, pcapPkt.LTypeLen, pcapPkt.Ci, false)
+	if err != nil {
+		if _, empty := err.(EmptyPacket); !empty {
+			stats.Add("packet_error", 1)
+		}
+		return nil
+	}
+
+	for _, p := range parser.ports {
+		if pckt.DstPort == p && containsOrEmpty(pckt.DstIP, parser.ips) {
+			pckt.Direction = DirIncoming
+			break
+		} else if pckt.SrcPort == p && containsOrEmpty(pckt.SrcIP, parser.ips) {
+			pckt.Direction = DirOutcoming
+			break
+		}
+	}
+
+	return pckt
+}
+
+func containsOrEmpty(element net.IP, ipList []net.IP) bool {
+	if len(ipList) == 0 {
+		return true
+	}
+	for _, ip := range ipList {
+		if ip.Equal(element) {
+			return true
+		}
+	}
+	return false
+}
+
+func (parser *MessageParser) processPacket(pckt *Packet) {
+	if pckt == nil {
+		return
+	}
+
+	// Trying to build unique hash, but there is small chance of collision
+	// No matter if it is request or response, all packets in the same message have same
+	m, ok := parser.m[pckt.MessageID()]
+	switch {
+	case ok:
+		if m.Direction == DirUnknown {
+			if in, out := parser.Start(pckt); in || out {
+				if in {
+					m.Direction = DirIncoming
+				} else {
+					m.Direction = DirOutcoming
+				}
+			}
+		}
+		parser.addPacket(m, pckt)
+		return
+	case pckt.Direction == DirUnknown && parser.Start != nil:
+		if in, out := parser.Start(pckt); in || out {
+			if in {
+				pckt.Direction = DirIncoming
+			} else {
+				pckt.Direction = DirOutcoming
+			}
+		}
+	}
+
+	m = new(Message)
+	m.Direction = pckt.Direction
+	m.SrcAddr = pckt.SrcIP.String()
+	m.DstAddr = pckt.DstIP.String()
+
+	parser.m[pckt.MessageID()] = m
+
+	m.Start = pckt.Timestamp
+	m.parser = parser
+	parser.addPacket(m, pckt)
+}
+
+func (parser *MessageParser) addPacket(m *Message, pckt *Packet) bool {
+	if !m.add(pckt) {
+		return false
+	}
+
+	// If we are using protocol parsing, like HTTP, depend on its parsing func.
+	// For the binary procols wait for message to expire
+	if parser.End != nil {
+		if parser.End(m) {
+			parser.Emit(m)
+			return true
+		}
+
+		parser.Fix100Continue(m)
+	}
+
+	return true
+}
+
+func (parser *MessageParser) Fix100Continue(m *Message) {
+	// Only adjust a message once
+	if state, ok := m.feedback.(*http_proto.HTTPState); ok && state.Continue100 && !m.continueAdjusted {
+		// Shift Ack by given offset
+		// Size of "HTTP/1.1 100 Continue\r\n\r\n" message
+		for _, p := range m.packets {
+			p.messageID = 0
+			p.Ack += 25
+		}
+
+		// If next section was aready approved and received, merge messages
+		if next, found := parser.m[m.packets[0].MessageID()]; found {
+			for _, p := range next.packets {
+				parser.addPacket(m, p)
+			}
+		}
+
+		// Re-add (or override) again with new message and ID
+		parser.m[m.packets[0].MessageID()] = m
+		m.continueAdjusted = true
+	}
+}
+
+func (parser *MessageParser) Read() *Message {
+	m := <-parser.messages
+	return m
+}
+
+func (parser *MessageParser) Emit(m *Message) {
+	stats.Add("message_count", 1)
+
+	delete(parser.m, m.packets[0].MessageID())
+
+	parser.messages <- m
+}
+
+func GetUnexportedField(field reflect.Value) interface{} {
+	return reflect.NewAt(field.Type(), unsafe.Pointer(field.UnsafeAddr())).Elem().Interface()
+}
+
+var failMsg int
+
+func (parser *MessageParser) timer(now time.Time) {
+	packetLen = 0
+
+	packetQueueLen.Set(int64(len(parser.packets)))
+	messageQueueLen.Set(int64(len(parser.m)))
+
+	for _, m := range parser.m {
+		if now.Sub(m.End) > parser.messageExpire {
+			m.TimedOut = true
+			stats.Add("message_timeout_count", 1)
+			failMsg++
+			if parser.End == nil || parser.allowIncompete {
+				parser.Emit(m)
+			}
+
+			delete(parser.m, m.packets[0].MessageID())
+		}
+	}
+}
+
+func (parser *MessageParser) Close() error {
+	parser.close <- struct{}{}
+	<-parser.close // wait for timer to be closed!
+	return nil
+}

internal/tcp/tcp_packet.go

@@ -2,28 +2,54 @@ package tcp
 
 import (
 	"encoding/binary"
+	"expvar"
 	"fmt"
 	"net"
-	"sync"
 	"time"
 
 	"github.com/google/gopacket"
 )
 
-func copySlice(b, a []byte) []byte {
-	if cap(b) < len(a) {
-		b = make([]byte, len(a))
+func copySlice(to []byte, skip int, from ...[]byte) ([]byte, int) {
+	var totalLen int
+	for _, s := range from {
+		totalLen += len(s)
 	}
-	copy(b, a)
-	return b[:len(a)]
+	totalLen += skip
+
+	if len(to) < totalLen {
+		diff := totalLen - len(to)
+		to = append(to, make([]byte, diff)...)
+	}
+
+	for _, s := range from {
+		skip += copy(to[skip:], s)
+	}
+
+	return to, skip
 }
 
-var packetPool = sync.Pool{
-	New: func() interface{} {
-		return new(Packet)
-	},
+var stats *expvar.Map
+var packetQueueLen, messageQueueLen *expvar.Int
+
+func init() {
+	packetQueueLen = new(expvar.Int)
+	messageQueueLen = new(expvar.Int)
+
+	stats = expvar.NewMap("tcp")
+	stats.Init()
+	stats.Set("packet_queue", packetQueueLen)
+	stats.Set("message_queue", messageQueueLen)
 }
 
+type Dir int
+
+const (
+	DirUnknown = iota
+	DirIncoming
+	DirOutcoming
+)
+
 /*
 Packet represent data and layers of packet.
 parser extracts information from pcap Packet. functions of *Packet doesn't validate if packet is nil,
@@ -31,6 +57,7 @@ calllers must make sure that ParsePacket has'nt returned any error before callin
 function.
 */
 type Packet struct {
+	Direction          Dir
 	messageID          uint64
 	SrcIP, DstIP       net.IP
 	Version            uint8
@@ -39,24 +66,45 @@ type Packet struct {
 	ACK, SYN, FIN, RST bool
 	Lost               uint32
 	Retry              int
+	CaptureLength      int
 	Timestamp          time.Time
 	Payload            []byte
+	buf                []byte
+
+	created time.Time
+	gc      bool
+}
+
+type PcapPacket struct {
+	Data     []byte
+	LType    int
+	LTypeLen int
+	Ci       *gopacket.CaptureInfo
 }
 
 // ParsePacket parse raw packets
-func ParsePacket(data []byte, lType, lTypeLen int, cp *gopacket.CaptureInfo) (pckt *Packet, err error) {
-	pckt = packetPool.Get().(*Packet)
+func ParsePacket(data []byte, lType, lTypeLen int, ci *gopacket.CaptureInfo, allowEmpty bool) (pckt *Packet, err error) {
+	pckt = new(Packet)
+	if err := pckt.parse(data, lType, lTypeLen, ci, allowEmpty); err != nil {
+		return nil, err
+	}
+
+	return pckt, nil
+}
+
+func (pckt *Packet) parse(data []byte, lType, lTypeLen int, cp *gopacket.CaptureInfo, allowEmpty bool) error {
 	pckt.Retry = 0
 	pckt.messageID = 0
+	pckt.buf = pckt.buf[:]
 
 	// TODO: check resolution
 	pckt.Timestamp = cp.Timestamp
 
 	if len(data) < lTypeLen {
-		return nil, ErrHdrLength("Link")
+		return ErrHdrLength("Link")
 	}
 	if len(data) <= lTypeLen {
-		return nil, ErrHdrMissing("IPv4 or IPv6")
+		return ErrHdrMissing("IPv4 or IPv6")
 	}
 
 	ldata := data[lTypeLen:]
@@ -66,63 +114,72 @@ func ParsePacket(data []byte, lType, lTypeLen int, cp *gopacket.CaptureInfo) (pc
 	if ldata[0]>>4 == 4 {
 		// IPv4 header
 		if len(ldata) < 20 {
-			return nil, ErrHdrLength("IPv4")
+			return ErrHdrLength("IPv4")
 		}
 		proto = ldata[9]
 		ihl := int(ldata[0]&0x0F) * 4
 		if ihl < 20 {
-			return nil, ErrHdrInvalid("IPv4's IHL")
+			return ErrHdrInvalid("IPv4's IHL")
 		}
 		if len(ldata) < ihl {
-			return nil, ErrHdrLength("IPv4 opts")
+			return ErrHdrLength("IPv4 opts")
 		}
 		netLayer = ldata[:ihl]
 	} else if ldata[0]>>4 == 6 {
 		if len(ldata) < 40 {
-			return nil, ErrHdrLength("IPv6")
+			return ErrHdrLength("IPv6")
 		}
 		proto = ldata[6]
 		totalLen := 40
 		for ipv6ExtensionHdr(proto) {
 			hdr := len(ldata) - totalLen
 			if hdr < 8 {
-				return nil, ErrHdrExpected("IPv6 opts")
+				return ErrHdrExpected("IPv6 opts")
 			}
 			extLen := 8
 			if proto != 44 {
 				extLen = int(ldata[totalLen+1]+1) * 8
 			}
 			if hdr < extLen {
-				return nil, ErrHdrLength("IPv6 opts")
+				return ErrHdrLength("IPv6 opts")
 			}
 			proto = ldata[totalLen]
 			totalLen += extLen
 		}
 		netLayer = ldata[:totalLen]
 	} else {
-		return nil, ErrHdrExpected("IPv4 or IPv6")
+		return ErrHdrExpected("IPv4 or IPv6")
 	}
 	if proto != 6 {
-		return nil, ErrHdrExpected("TCP")
+		return ErrHdrExpected("TCP")
 	}
 	if len(data) <= len(netLayer) {
-		return nil, ErrHdrMissing("TCP")
+		return ErrHdrMissing("TCP")
 	}
 	ndata := ldata[len(netLayer):]
 	// TCP header
 	if len(ndata) < 20 {
-		return nil, ErrHdrLength("TCP")
+		return ErrHdrLength("TCP")
 	}
 	dOf := int(ndata[12]>>4) * 4
 	if dOf < 20 {
-		return nil, ErrHdrInvalid("TCP's ndata offset")
+		return ErrHdrInvalid("TCP's ndata offset")
 	}
 	if len(ndata) < dOf {
-		return nil, ErrHdrLength("TCP opts")
+		return ErrHdrLength("TCP opts")
 	}
 
-	if len(ndata[dOf:]) == 0 {
-		return nil, fmt.Errorf("Packet without Data")
+	// There are case when packet have padding but dOf shows its not
+	empty := true
+	for i := 0; i < len(ndata[dOf:]); i++ {
+		if ndata[dOf:][i] != 0 {
+			empty = false
+			break
+		}
+	}
+
+	if !allowEmpty && empty {
+		return EmptyPacket("")
 	}
 
 	if (netLayer[0] >> 4) == 4 {
@@ -139,6 +196,7 @@ func ParsePacket(data []byte, lType, lTypeLen int, cp *gopacket.CaptureInfo) (pc
 
 	transLayer = ndata[:dOf]
 
+	pckt.CaptureLength = cp.CaptureLength
 	pckt.SrcPort = binary.BigEndian.Uint16(transLayer[0:2])
 	pckt.DstPort = binary.BigEndian.Uint16(transLayer[2:4])
 	pckt.Seq = binary.BigEndian.Uint32(transLayer[4:8])
@@ -148,8 +206,10 @@ func ParsePacket(data []byte, lType, lTypeLen int, cp *gopacket.CaptureInfo) (pc
 	pckt.RST = transLayer[13]&0x04 != 0
 	pckt.ACK = transLayer[13]&0x10 != 0
 	pckt.Lost = uint32(cp.Length - cp.CaptureLength)
-	pckt.Payload = copySlice(pckt.Payload, ndata[dOf:])
-	return
+
+	pckt.Payload = ndata[dOf:]
+
+	return nil
 }
 
 func (pckt *Packet) MessageID() uint64 {
@@ -172,6 +232,12 @@ func (pckt *Packet) Dst() string {
 	return fmt.Sprintf("%s:%d", pckt.DstIP, pckt.DstPort)
 }
 
+type EmptyPacket string
+
+func (err EmptyPacket) Error() string {
+	return "Empty packet"
+}
+
 // ErrHdrLength returned on short header length
 type ErrHdrLength string
 

internal/tcp/tcp_test.go

@@ -0,0 +1,313 @@
+package tcp
+
+import (
+	"bytes"
+	"encoding/binary"
+	"testing"
+	"time"
+
+	proto "github.com/buger/goreplay/pkg/http_proto"
+
+	"github.com/google/gopacket"
+	"github.com/google/gopacket/layers"
+	"github.com/stretchr/testify/assert"
+)
+
+func generateHeader(request bool, seq uint32, length uint16) []byte {
+	hdr := make([]byte, 4+24+24)
+	binary.BigEndian.PutUint32(hdr, uint32(layers.ProtocolFamilyIPv4))
+
+	ip := hdr[4:]
+	ip[0] = 4<<4 | 6
+	binary.BigEndian.PutUint16(ip[2:4], length+24+24)
+	ip[9] = uint8(layers.IPProtocolTCP)
+	copy(ip[12:16], []byte{127, 0, 0, 1})
+	copy(ip[16:], []byte{127, 0, 0, 1})
+
+	// set tcp header
+	tcp := ip[24:]
+	tcp[12] = 6 << 4
+
+	if request {
+		binary.BigEndian.PutUint16(tcp, 5535)
+		binary.BigEndian.PutUint16(tcp[2:], 8000)
+	} else {
+		binary.BigEndian.PutUint16(tcp, 8000)
+		binary.BigEndian.PutUint16(tcp[2:], 5535)
+	}
+	binary.BigEndian.PutUint32(tcp[4:], seq)
+	return hdr
+}
+
+func GetPackets(request bool, start uint32, _len int, payload []byte) []*Packet {
+	var packets = make([]*Packet, _len)
+	var err error
+	for i := start; i < start+uint32(_len); i++ {
+		d := append(generateHeader(request, i, uint16(len(payload))), payload...)
+		ci := &gopacket.CaptureInfo{Length: len(d), CaptureLength: len(d), Timestamp: time.Now()}
+
+		packets[i-start], err = ParsePacket(d, int(layers.LinkTypeLoop), 4, ci, true)
+		if request {
+			packets[i-start].Direction = DirIncoming
+		} else {
+			packets[i-start].Direction = DirOutcoming
+		}
+		if err != nil {
+			panic(err)
+		}
+	}
+	return packets
+}
+
+func TestRequestResponseMapping(t *testing.T) {
+	packets := []*Packet{
+		{SrcPort: 60000, DstPort: 80, Ack: 1, Seq: 1, Direction: DirIncoming, Timestamp: time.Unix(1, 0), Payload: []byte("GET / HTTP/1.1\r\n")},
+		{SrcPort: 60000, DstPort: 80, Ack: 1, Seq: 17, Direction: DirIncoming, Timestamp: time.Unix(2, 0), Payload: []byte("Host: localhost\r\n\r\n")},
+
+		// Seq of first response packet match Ack of first request packet
+		{SrcPort: 80, DstPort: 60000, Ack: 36, Seq: 1, Direction: DirOutcoming, Timestamp: time.Unix(3, 0), Payload: []byte("HTTP/1.1 200 OK\r\n")},
+		{SrcPort: 80, DstPort: 60000, Ack: 36, Seq: 18, Direction: DirOutcoming, Timestamp: time.Unix(4, 0), Payload: []byte("Content-Length: 0\r\n\r\n")},
+
+		// Same TCP stream
+		{SrcPort: 60000, DstPort: 80, Ack: 39, Seq: 36, Direction: DirIncoming, Timestamp: time.Unix(5, 0), Payload: []byte("GET / HTTP/1.1\r\n")},
+		{SrcPort: 60000, DstPort: 80, Ack: 39, Seq: 52, Direction: DirIncoming, Timestamp: time.Unix(6, 0), Payload: []byte("Host: localhost\r\n\r\n")},
+
+		// Seq of first response packet match Ack of first request packet
+		{SrcPort: 80, DstPort: 60000, Ack: 71, Seq: 39, Direction: DirOutcoming, Timestamp: time.Unix(7, 0), Payload: []byte("HTTP/1.1 200 OK\r\n")},
+		{SrcPort: 80, DstPort: 60000, Ack: 71, Seq: 56, Direction: DirOutcoming, Timestamp: time.Unix(8, 0), Payload: []byte("Content-Length: 0\r\n\r\n")},
+	}
+
+	parser := NewMessageParser(nil, nil, nil, time.Second, false)
+	parser.Start = func(pckt *Packet) (bool, bool) {
+		return proto.HasRequestTitle(pckt.Payload), proto.HasResponseTitle(pckt.Payload)
+	}
+	parser.End = func(m *Message) bool {
+		return proto.HasFullPayload(m, m.PacketData()...)
+	}
+
+	for _, packet := range packets {
+		parser.processPacket(packet)
+	}
+
+	messages := []*Message{}
+	for i := 0; i < 4; i++ {
+		m := parser.Read()
+		messages = append(messages, m)
+	}
+
+	assert.Equal(t, int(messages[0].Direction), int(DirIncoming))
+	assert.Equal(t, int(messages[1].Direction), int(DirOutcoming))
+	assert.Equal(t, int(messages[2].Direction), int(DirIncoming))
+	assert.Equal(t, int(messages[3].Direction), int(DirOutcoming))
+
+	assert.Equal(t, messages[0].UUID(), messages[1].UUID())
+	assert.Equal(t, messages[2].UUID(), messages[3].UUID())
+
+	assert.NotEqual(t, messages[0].UUID(), messages[2].UUID())
+}
+
+func TestMessageParserWithHint(t *testing.T) {
+	parser := NewMessageParser(nil, nil, nil, time.Second, false)
+	parser.Start = func(pckt *Packet) (bool, bool) {
+		return proto.HasRequestTitle(pckt.Payload), proto.HasResponseTitle(pckt.Payload)
+	}
+	parser.End = func(m *Message) bool {
+		return proto.HasFullPayload(m, m.PacketData()...)
+	}
+
+	packets := []*Packet{
+		// Seq of first response packet match Ack of first request packet
+		{SrcPort: 80, DstPort: 60000, Ack: 1, Seq: 1, Direction: DirOutcoming, Timestamp: time.Unix(1, 0), Payload: []byte("HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nTransfer-Encoding: chunked\r\n\r\n7\r\n")},
+		{SrcPort: 80, DstPort: 60000, Ack: 1, Seq: 18, Direction: DirOutcoming, Timestamp: time.Unix(2, 0), Payload: []byte("\r\nMozilla\r\n9\r\nDeveloper\r")},
+		{SrcPort: 80, DstPort: 60000, Ack: 1, Seq: 42, Direction: DirOutcoming, Timestamp: time.Unix(3, 0), Payload: []byte("\n7\r\nNetwork\r\n0\r\n\r\n")},
+
+		{SrcPort: 60000, DstPort: 80, Ack: 60, Seq: 1, Direction: DirIncoming, Timestamp: time.Unix(4, 0), Payload: []byte("POST / HTTP/1.1\r\nContent-Type: text/plain\r\nContent-Length: 23\r\n\r\n")},
+		{SrcPort: 60000, DstPort: 80, Ack: 60, Seq: 66, Direction: DirIncoming, Timestamp: time.Unix(5, 0), Payload: []byte("MozillaDeveloper")},
+		{SrcPort: 60000, DstPort: 80, Ack: 60, Seq: 82, Direction: DirIncoming, Timestamp: time.Unix(6, 0), Payload: []byte("Network")},
+
+		{SrcPort: 80, DstPort: 60000, Ack: 89, Seq: 1, Direction: DirOutcoming, Timestamp: time.Unix(7, 0), Payload: []byte("HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nContent-Length: 0\r\n\r\n")},
+	}
+
+	for _, p := range packets {
+		parser.processPacket(p)
+	}
+
+	messages := []*Message{}
+	for i := 0; i < 3; i++ {
+		m := parser.Read()
+		messages = append(messages, m)
+	}
+
+	if !bytes.HasSuffix(messages[0].Data(), []byte("\n7\r\nNetwork\r\n0\r\n\r\n")) {
+		t.Errorf("expected to %q to have suffix %q", messages[0].Data(), []byte("\n7\r\nNetwork\r\n0\r\n\r\n"))
+	}
+
+	if !bytes.HasSuffix(messages[1].Data(), []byte("Network")) {
+		t.Errorf("expected to %q to have suffix %q", messages[1].Data(), []byte("Network"))
+	}
+
+	if !bytes.HasSuffix(messages[2].Data(), []byte("Content-Length: 0\r\n\r\n")) {
+		t.Errorf("expected to %q to have suffix %q", messages[2].Data(), []byte("Content-Length: 0\r\n\r\n"))
+	}
+}
+
+func TestMessageParserWrongOrder(t *testing.T) {
+	parser := NewMessageParser(nil, nil, nil, time.Second, false)
+	parser.Start = func(pckt *Packet) (bool, bool) {
+		return proto.HasRequestTitle(pckt.Payload), proto.HasResponseTitle(pckt.Payload)
+	}
+	parser.End = func(m *Message) bool {
+		return proto.HasFullPayload(m, m.PacketData()...)
+	}
+	packets := []*Packet{
+		// Seq of first response packet match Ack of first request packet
+		{SrcPort: 60000, DstPort: 80, Ack: 60, Seq: 66, Direction: DirIncoming, Timestamp: time.Unix(5, 0), Payload: []byte("MozillaDeveloper")},
+		{SrcPort: 80, DstPort: 60000, Ack: 1, Seq: 1, Direction: DirOutcoming, Timestamp: time.Unix(1, 0), Payload: []byte("HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nTransfer-Encoding: chunked\r\n\r\n7\r\n")},
+		{SrcPort: 80, DstPort: 60000, Ack: 1, Seq: 42, Direction: DirOutcoming, Timestamp: time.Unix(3, 0), Payload: []byte("\n7\r\nNetwork\r\n0\r\n\r\n")},
+
+		{SrcPort: 60000, DstPort: 80, Ack: 60, Seq: 1, Direction: DirIncoming, Timestamp: time.Unix(4, 0), Payload: []byte("POST / HTTP/1.1\r\nContent-Type: text/plain\r\nContent-Length: 23\r\n\r\n")},
+		{SrcPort: 80, DstPort: 60000, Ack: 1, Seq: 18, Direction: DirOutcoming, Timestamp: time.Unix(2, 0), Payload: []byte("\r\nMozilla\r\n9\r\nDeveloper\r")},
+
+		{SrcPort: 80, DstPort: 60000, Ack: 89, Seq: 1, Direction: DirOutcoming, Timestamp: time.Unix(7, 0), Payload: []byte("HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nContent-Length: 0\r\n\r\n")},
+		{SrcPort: 60000, DstPort: 80, Ack: 60, Seq: 82, Direction: DirIncoming, Timestamp: time.Unix(6, 0), Payload: []byte("Network")},
+	}
+
+	for _, p := range packets {
+		parser.processPacket(p)
+	}
+
+	m := parser.Read()
+
+	if !bytes.HasSuffix(m.Data(), []byte("\n7\r\nNetwork\r\n0\r\n\r\n")) {
+		t.Errorf("expected to %q to have suffix %q", m.Data(), []byte("\n7\r\nNetwork\r\n0\r\n\r\n"))
+	}
+
+	m = parser.Read()
+
+	if !bytes.HasSuffix(m.Data(), []byte("Content-Length: 0\r\n\r\n")) {
+		t.Errorf("expected to %q to have suffix %q", m.Data(), []byte("Content-Length: 0\r\n\r\n"))
+	}
+
+	m = parser.Read()
+
+	if !bytes.HasSuffix(m.Data(), []byte("Network")) {
+		t.Errorf("expected to %q to have suffix %q", m.Data(), []byte("Network"))
+	}
+}
+
+func TestMessageParserWithoutHint(t *testing.T) {
+	var data [63 << 10]byte
+	packets := GetPackets(true, 1, 10, data[:])
+
+	p := NewMessageParser(nil, nil, nil, time.Second, false)
+	for _, v := range packets {
+		p.processPacket(v)
+	}
+	m := p.Read()
+
+	if m.Length != 63<<10*10 {
+		t.Errorf("expected %d to equal %d", m.Length, 63<<10*10)
+	}
+}
+
+func TestMessageTimeoutReached(t *testing.T) {
+	const size = 63 << 11
+	var data [size >> 1]byte
+	packets := GetPackets(true, 1, 2, data[:])
+	p := NewMessageParser(nil, nil, nil, 100*time.Millisecond, true)
+	p.processPacket(packets[0])
+
+	time.Sleep(time.Millisecond * 20)
+
+	p.processPacket(packets[1])
+	m := p.Read()
+	if m.Length != size {
+		t.Errorf("expected %d to equal %d", m.Length, size)
+	}
+	if !m.TimedOut {
+		t.Error("expected message to be timeout")
+	}
+}
+
+func BenchmarkMessageUUID(b *testing.B) {
+	packets := GetPackets(true, 1, 5, nil)
+
+	var uuid []byte
+	parser := NewMessageParser(nil, nil, nil, 10*time.Millisecond, true)
+	for _, p := range packets {
+		parser.processPacket(p)
+	}
+
+	msg := parser.Read()
+
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		uuid = msg.UUID()
+	}
+	_ = uuid
+}
+
+func BenchmarkPacketParseAndSort(b *testing.B) {
+	m := new(Message)
+	m.packets = make([]*Packet, 100)
+	for i, v := range GetPackets(true, 1, 100, nil) {
+		m.packets[i] = v
+	}
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		m.Sort()
+	}
+}
+
+func BenchmarkMessageParserWithoutHint(b *testing.B) {
+	var chunk = []byte("111111111111111111111111111111")
+	packets := GetPackets(true, 1, 1000, chunk)
+	p := NewMessageParser(nil, nil, nil, 2*time.Second, false)
+	b.ResetTimer()
+	b.ReportMetric(float64(1000), "packets/op")
+	for i := 0; i < b.N; i++ {
+		for _, v := range packets {
+			p.processPacket(v)
+		}
+		p.Read()
+	}
+}
+
+func BenchmarkMessageParserWithHint(b *testing.B) {
+	var buf [1002][]byte
+	var chunk = []byte("1e\r\n111111111111111111111111111111\r\n")
+	buf[0] = []byte("HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nTransfer-Encoding: chunked\r\n\r\n")
+	for i := 1; i < 1000; i++ {
+		buf[i] = chunk
+	}
+	buf[1001] = []byte("0\r\n\r\n")
+	packets := make([]*Packet, len(buf))
+	for i := 0; i < len(buf); i++ {
+		packets[i] = GetPackets(false, 1, 1, buf[i])[0]
+	}
+
+	parser := NewMessageParser(nil, nil, nil, 2*time.Second, false)
+	parser.Start = func(pckt *Packet) (bool, bool) {
+		return false, proto.HasResponseTitle(pckt.Payload)
+	}
+	parser.End = func(m *Message) bool {
+		return proto.HasFullPayload(m, m.PacketData()...)
+	}
+	b.ResetTimer()
+	b.ReportMetric(float64(len(packets)), "packets/op")
+	b.ReportMetric(float64(1000), "chunks/op")
+	for i := 0; i < b.N; i++ {
+		for j := range packets {
+			parser.processPacket(packets[j])
+		}
+		parser.Read()
+	}
+}
+
+func BenchmarkNewAndParsePacket(b *testing.B) {
+	data := append(generateHeader(true, 1024, 10), make([]byte, 10)...)
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		ParsePacket(data, int(layers.LinkTypeLoop), 4, &gopacket.CaptureInfo{}, true)
+	}
+}

k8s/README.md

@@ -0,0 +1,144 @@
+# Native k8s integration
+
+After following steps below, you will be able to capture traffic inside k8s like this:
+
+```
+gor --input-raw k8s://namespace/deployment/app:80 --output-http http://replay.com
+```
+
+GoReplay will running as a daemonset (e.g. on each phisical k8s node. 
+It will also require giving required permission to have read access to K8s APIs, so it can dynamically filter traffic for a specific pods.
+
+Supported format for filtering required pods:
+
+```
+k8s://[namespace/]pod/[pod_name] - k8s://default/pod/nginx-7848d4b86f-5nxz8
+k8s://[namespace/]deployment/[deployment_name] - k8s://default/deployment/nginx
+k8s://[namespace/]daemonset/[daemonset_name] - k8s://default/daemonset/nginx
+k8s://[namespace/]labelSelector/[selector] - k8s://default/labelSelector/app=nginx
+k8s://[namespace/]fieldSelector/[selector] - k8s://default/fieldSelector/metadata.name=nginx-7848d4b86f-5nxz8
+```
+`namespace` is optional, omit to use all namespaces: `k8s://labelSelector/app=replay`
+
+## 1. Create a namespace
+`kubectl create namespace goreplay`
+
+## 2. Create the Kubernetes service account in the namespace:
+
+`kubectl create serviceaccount goreplay --namespace goreplay`
+
+## 3. Create Cluster Role which gives read-only access to the pods:
+`kubectl -n goreplay -f clusterrole.yaml apply`
+
+```yaml
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: pod-reader
+rules:
+- apiGroups: [""]
+  resources: ["pods"]
+  verbs: ["get", "watch", "list"]
+- apiGroups: [""]
+  resources: ["deployments"]
+  verbs: ["get", "watch", "list"]
+- apiGroups: [""]
+  resources: ["daemonset"]
+  verbs: ["get", "watch", "list"]
+```
+
+## 4. Attach role to goreplay service account
+`kubectl -n goreplay -f rolebinding.yaml apply`
+
+```yaml
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: goreplay-reader-binding
+subjects:
+- kind: ServiceAccount
+  name: goreplay
+  namespace: goreplay
+roleRef:
+  kind: ClusterRole
+  name: pod-reader
+  apiGroup: ""
+```
+
+## 5. Start goreplay daemonset
+
+`kubectl -n goreplay -f goreplay.yaml apply`
+
+In arguments, specify which service you want to capture. 
+Following format supported:
+
+
+```yaml
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: goreplay-daemon
+spec:
+  template:
+    spec:
+      hostNetwork: true
+      serviceAccountName: goreplay
+      containers:
+      - name: goreplay
+        image: buger/goreplay:2.0.0-rc2
+        command:
+          - "--input-raw k8s://deployments/nginx:80"
+          - "--output-stdout"
+```
+
+## 6. Create a simple http service (Optionally)
+
+`kubectl -n default -f nginx.yaml apply`
+
+```yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: nginx
+  labels:
+    app: nginx
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: nginx
+  template:
+    metadata:
+      labels:
+        app: nginx
+    spec:
+      containers:
+      - name: nginx
+        image: nginx
+        ports:
+        - containerPort: 80
+   
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: ngnix-service
+spec:
+  selector:
+    app: nginx
+  type: NodePort
+  ports:
+  - protocol: TCP
+    port: 80
+    targetPort: 80
+
+```
+
+
+## 7. Verify installation
+
+Find url for your service using `kubectl get svc` or `minikube service --url ngnix-service -n http`, and make a call to it.
+
+Get GoReplay logs, and check if it capture traffic of your service.
+`kubectl logs -n goreplay -l name=goreplay-daemon --all-containers`
+

k8s/clusterrole.yaml

@@ -0,0 +1,14 @@
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: pod-reader
+rules:
+- apiGroups: [""]
+  resources: ["pods"]
+  verbs: ["get", "watch", "list"]
+- apiGroups: ["apps"]
+  resources: ["deployments"]
+  verbs: ["get", "watch", "list"]
+- apiGroups: ["apps"]
+  resources: ["daemonsets"]
+  verbs: ["get", "watch", "list"]

k8s/goreplay.yaml

@@ -0,0 +1,15 @@
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: goreplay-daemon
+spec:
+  template:
+    spec:
+      hostNetwork: true
+      serviceAccountName: goreplay
+      containers:
+      - name: goreplay
+        image: buger/goreplay:2.0.0-rc2
+        command:
+          - "--input-raw k8s://deployments/nginx:80"
+          - "--output-stdout"

k8s/nginx.yaml

@@ -0,0 +1,36 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: nginx
+  labels:
+    app: nginx
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: nginx
+  template:
+    metadata:
+      labels:
+        app: nginx
+    spec:
+      containers:
+      - name: nginx
+        image: nginx
+        ports:
+        - containerPort: 80
+   
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: ngnix-service
+spec:
+  selector:
+    app: nginx
+  type: NodePort
+  ports:
+  - protocol: TCP
+    port: 80
+    targetPort: 80
+   

k8s/rolebinding.yaml

@@ -0,0 +1,12 @@
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: goreplay-reader-binding
+subjects:
+- kind: ServiceAccount
+  name: goreplay
+  namespace: goreplay
+roleRef:
+  kind: ClusterRole
+  name: pod-reader
+  apiGroup: ""

limiter_test.go

@@ -1,111 +0,0 @@
-// +build !race
-
-package main
-
-import (
-	"sync"
-	"testing"
-)
-
-func TestOutputLimiter(t *testing.T) {
-	wg := new(sync.WaitGroup)
-
-	input := NewTestInput()
-	output := NewLimiter(NewTestOutput(func(*Message) {
-		wg.Done()
-	}), "10")
-	wg.Add(10)
-
-	plugins := &InOutPlugins{
-		Inputs:  []PluginReader{input},
-		Outputs: []PluginWriter{output},
-	}
-	plugins.All = append(plugins.All, input, output)
-
-	emitter := NewEmitter()
-	go emitter.Start(plugins, Settings.Middleware)
-
-	for i := 0; i < 100; i++ {
-		input.EmitGET()
-	}
-
-	wg.Wait()
-	emitter.Close()
-}
-
-func TestInputLimiter(t *testing.T) {
-	wg := new(sync.WaitGroup)
-
-	input := NewLimiter(NewTestInput(), "10")
-	output := NewTestOutput(func(*Message) {
-		wg.Done()
-	})
-	wg.Add(10)
-
-	plugins := &InOutPlugins{
-		Inputs:  []PluginReader{input},
-		Outputs: []PluginWriter{output},
-	}
-	plugins.All = append(plugins.All, input, output)
-
-	emitter := NewEmitter()
-	go emitter.Start(plugins, Settings.Middleware)
-
-	for i := 0; i < 100; i++ {
-		input.(*Limiter).plugin.(*TestInput).EmitGET()
-	}
-
-	wg.Wait()
-	emitter.Close()
-}
-
-// Should limit all requests
-func TestPercentLimiter1(t *testing.T) {
-	wg := new(sync.WaitGroup)
-
-	input := NewTestInput()
-	output := NewLimiter(NewTestOutput(func(*Message) {
-		wg.Done()
-	}), "0%")
-
-	plugins := &InOutPlugins{
-		Inputs:  []PluginReader{input},
-		Outputs: []PluginWriter{output},
-	}
-	plugins.All = append(plugins.All, input, output)
-
-	emitter := NewEmitter()
-	go emitter.Start(plugins, Settings.Middleware)
-
-	for i := 0; i < 100; i++ {
-		input.EmitGET()
-	}
-
-	wg.Wait()
-}
-
-// Should not limit at all
-func TestPercentLimiter2(t *testing.T) {
-	wg := new(sync.WaitGroup)
-
-	input := NewTestInput()
-	output := NewLimiter(NewTestOutput(func(*Message) {
-		wg.Done()
-	}), "100%")
-	wg.Add(100)
-
-	plugins := &InOutPlugins{
-		Inputs:  []PluginReader{input},
-		Outputs: []PluginWriter{output},
-	}
-	plugins.All = append(plugins.All, input, output)
-
-	emitter := NewEmitter()
-	go emitter.Start(plugins, Settings.Middleware)
-
-	for i := 0; i < 100; i++ {
-		input.EmitGET()
-	}
-
-	wg.Wait()
-}

middleware_test.go

@@ -1,154 +0,0 @@
-package main
-
-import (
-	"bytes"
-	"context"
-	"os"
-	"os/exec"
-	"strings"
-	"sync/atomic"
-	"syscall"
-	"testing"
-
-	"github.com/buger/goreplay/proto"
-)
-
-const echoSh = "./examples/middleware/echo.sh"
-const tokenModifier = "go run ./examples/middleware/token_modifier.go"
-
-var noDebug = append(syscall.Environ(), "GOR_TEST=1")
-
-func initMiddleware(cmd *exec.Cmd, cancl context.CancelFunc, l PluginReader, c func(error)) *Middleware {
-	var m Middleware
-	m.data = make(chan *Message, 1000)
-	m.stop = make(chan bool)
-	m.commandCancel = cancl
-	m.Stdout, _ = cmd.StdoutPipe()
-	m.Stdin, _ = cmd.StdinPipe()
-	cmd.Stderr = os.Stderr
-	go m.read(m.Stdout)
-	go func() {
-		defer m.Close()
-		var err error
-		if err = cmd.Start(); err == nil {
-			err = cmd.Wait()
-		}
-		if err != nil {
-			c(err)
-		}
-	}()
-	m.ReadFrom(l)
-	return &m
-}
-
-func initCmd(command string, env []string) (*exec.Cmd, context.CancelFunc) {
-	commands := strings.Split(command, " ")
-	ctx, cancl := context.WithCancel(context.Background())
-	cmd := exec.CommandContext(ctx, commands[0], commands[1:]...)
-	cmd.Env = env
-	return cmd, cancl
-}
-
-func TestMiddlewareEarlyClose(t *testing.T) {
-	quit := make(chan struct{})
-	in := NewTestInput()
-	cmd, cancl := initCmd(echoSh, noDebug)
-	midd := initMiddleware(cmd, cancl, in, func(err error) {
-		if err != nil {
-			if e, ok := err.(*exec.ExitError); ok {
-				status := e.Sys().(syscall.WaitStatus)
-				if status.Signal() != syscall.SIGKILL {
-					t.Errorf("expected error to be signal killed. got %s", status.Signal().String())
-				}
-			}
-		}
-		quit <- struct{}{}
-	})
-	var body = []byte("OPTIONS / HTTP/1.1\r\nHost: example.org\r\n\r\n")
-	count := uint32(0)
-	out := NewTestOutput(func(msg *Message) {
-		if !bytes.Equal(body, msg.Data) {
-			t.Errorf("expected %q to equal %q", body, msg.Data)
-		}
-		atomic.AddUint32(&count, 1)
-		if atomic.LoadUint32(&count) == 5 {
-			quit <- struct{}{}
-		}
-	})
-	pl := &InOutPlugins{}
-	pl.Inputs = []PluginReader{midd, in}
-	pl.Outputs = []PluginWriter{out}
-	pl.All = []interface{}{midd, out, in}
-	e := NewEmitter()
-	go e.Start(pl, "")
-	for i := 0; i < 5; i++ {
-		in.EmitBytes(body)
-	}
-	<-quit
-	midd.Close()
-	<-quit
-}
-
-func TestTokenMiddleware(t *testing.T) {
-	quit := make(chan struct{})
-	in := NewTestInput()
-	in.skipHeader = true
-	cmd, cancl := initCmd(tokenModifier, noDebug)
-	midd := initMiddleware(cmd, cancl, in, func(err error) {})
-	req := []byte("1 932079936fa4306fc308d67588178d17d823647c 1439818823587396305 200\nGET /token HTTP/1.1\r\nHost: example.org\r\n\r\n")
-	res := []byte("2 932079936fa4306fc308d67588178d17d823647c 1439818823587396305 200\nHTTP/1.1 200 OK\r\nContent-Length: 10\r\nContent-Type: text/plain; charset=utf-8\r\n\r\n17d823647c")
-	rep := []byte("3 932079936fa4306fc308d67588178d17d823647c 1439818823587396305 200\nHTTP/1.1 200 OK\r\nContent-Length: 15\r\nContent-Type: text/plain; charset=utf-8\r\n\r\n932079936fa4306")
-	count := uint32(0)
-	out := NewTestOutput(func(msg *Message) {
-		if msg.Meta[0] == '1' && !bytes.Equal(payloadID(msg.Meta), payloadID(req)) {
-			token, _, _ := proto.PathParam(msg.Data, []byte("token"))
-			if !bytes.Equal(token, proto.Body(rep)) {
-				t.Error("expected the token to be equal to the replayed responses's token")
-			}
-		}
-		atomic.AddUint32(&count, 1)
-		if atomic.LoadUint32(&count) == 2 {
-			quit <- struct{}{}
-		}
-	})
-	pl := &InOutPlugins{}
-	pl.Inputs = []PluginReader{midd, in}
-	pl.Outputs = []PluginWriter{out}
-	pl.All = []interface{}{midd, out, in}
-	e := NewEmitter()
-	go e.Start(pl, "")
-	in.EmitBytes(req) // emit original request
-	in.EmitBytes(res) // emit its response
-	in.EmitBytes(rep) // emit replayed response
-	// emit the request which should have modified token
-	token := []byte("1 8e091765ae902fef8a2b7d9dd96 14398188235873 100\nGET /?token=17d823647c HTTP/1.1\r\nHost: example.org\r\n\r\n")
-	in.EmitBytes(token)
-	<-quit
-	midd.Close()
-}
-
-func TestMiddlewareWithPrettify(t *testing.T) {
-	Settings.PrettifyHTTP = true
-	quit := make(chan struct{})
-	in := NewTestInput()
-	cmd, cancl := initCmd(echoSh, noDebug)
-	midd := initMiddleware(cmd, cancl, in, func(err error) {})
-	var b1 = []byte("POST / HTTP/1.1\r\nHost: example.org\r\nTransfer-Encoding: chunked\r\n\r\n4\r\nWiki\r\n5\r\npedia\r\nE\r\n in\r\n\r\nchunks.\r\n0\r\n\r\n")
-	var b2 = []byte("POST / HTTP/1.1\r\nHost: example.org\r\nContent-Length: 25\r\n\r\nWikipedia in\r\n\r\nchunks.")
-	out := NewTestOutput(func(msg *Message) {
-		if !bytes.Equal(proto.Body(b2), proto.Body(msg.Data)) {
-			t.Errorf("expected %q body to equal %q body", b2, msg.Data)
-		}
-		quit <- struct{}{}
-	})
-	pl := &InOutPlugins{}
-	pl.Inputs = []PluginReader{midd, in}
-	pl.Outputs = []PluginWriter{out}
-	pl.All = []interface{}{midd, out, in}
-	e := NewEmitter()
-	go e.Start(pl, "")
-	in.EmitBytes(b1)
-	<-quit
-	midd.Close()
-	Settings.PrettifyHTTP = false
-}

nfpm.yaml

@@ -0,0 +1,21 @@
+# nfpm example config file
+#
+# check https://nfpm.goreleaser.com/configuration for detailed usage
+#
+name: "GoReplay"
+arch: ${PLATFORM}
+platform: "linux"
+version: ${VERSION}
+section: "default"
+priority: "extra"
+provides:
+- goreplay
+maintainer: "Leonid Bugaev <hello@goreplay.org>"
+description: |
+ GoReplay is the simplest and safest way to test your app using real traffic before you put it into production. 
+vendor: "GoReplay"
+homepage: "https://goreplay.org"
+license: "AGPL"
+contents:
+- src: ./${BIN_NAME}
+  dst: /usr/local/bin

pkg/binary/output_binary.go

@@ -1,12 +1,22 @@
-package main
+package binary
 
 import (
 	"sync/atomic"
 	"time"
 
-	"github.com/buger/goreplay/size"
+	"github.com/buger/goreplay/internal/size"
+	"github.com/buger/goreplay/pkg/plugin"
+	"github.com/buger/goreplay/pkg/proto"
+
+	"github.com/rs/zerolog/log"
 )
 
+const (
+	initialDynamicWorkers = 10
+)
+
+var outputLogger = log.With().Str("component", "output_binary").Logger()
+
 // BinaryOutputConfig struct for holding binary output configuration
 type BinaryOutputConfig struct {
 	Workers        int           `json:"output-binary-workers"`
@@ -25,24 +35,23 @@ type BinaryOutput struct {
 	// aligned at 64bit. See https://github.com/golang/go/issues/599
 	activeWorkers int64
 	address       string
-	queue         chan *Message
-	responses     chan response
+	queue         chan *plugin.Message
+	responses     chan plugin.Response
 	needWorker    chan int
 	quit          chan struct{}
 	config        *BinaryOutputConfig
-	queueStats    *GorStat
 }
 
 // NewBinaryOutput constructor for BinaryOutput
 // Initialize workers
-func NewBinaryOutput(address string, config *BinaryOutputConfig) PluginReadWriter {
+func NewBinaryOutput(address string, config *BinaryOutputConfig) plugin.ReadWriter {
 	o := new(BinaryOutput)
 
 	o.address = address
 	o.config = config
 
-	o.queue = make(chan *Message, 1000)
-	o.responses = make(chan response, 1000)
+	o.queue = make(chan *plugin.Message, 1000)
+	o.responses = make(chan plugin.Response, 1000)
 	o.needWorker = make(chan int, 1)
 	o.quit = make(chan struct{})
 
@@ -110,8 +119,8 @@ func (o *BinaryOutput) startWorker() {
 }
 
 // PluginWrite writes a message tothis plugin
-func (o *BinaryOutput) PluginWrite(msg *Message) (n int, err error) {
-	if !isRequestPayload(msg.Meta) {
+func (o *BinaryOutput) PluginWrite(msg *plugin.Message) (n int, err error) {
+	if !proto.IsRequestPayload(msg.Meta) {
 		return len(msg.Data), nil
 	}
 
@@ -129,37 +138,37 @@ func (o *BinaryOutput) PluginWrite(msg *Message) (n int, err error) {
 }
 
 // PluginRead reads a message from this plugin
-func (o *BinaryOutput) PluginRead() (*Message, error) {
-	var resp response
-	var msg Message
+func (o *BinaryOutput) PluginRead() (*plugin.Message, error) {
+	var resp plugin.Response
+	var msg plugin.Message
 	select {
 	case <-o.quit:
-		return nil, ErrorStopped
+		return nil, plugin.ErrorStopped
 	case resp = <-o.responses:
 	}
-	msg.Data = resp.payload
-	msg.Meta = payloadHeader(ReplayedResponsePayload, resp.uuid, resp.startedAt, resp.roundTripTime)
+	msg.Data = resp.Payload
+	msg.Meta = proto.PayloadHeader(proto.ReplayedResponsePayload, resp.UUID, resp.StartedAt, resp.RoundTripTime)
 
 	return &msg, nil
 }
 
-func (o *BinaryOutput) sendRequest(client *TCPClient, msg *Message) {
-	if !isRequestPayload(msg.Meta) {
+func (o *BinaryOutput) sendRequest(client *TCPClient, msg *plugin.Message) {
+	if !proto.IsRequestPayload(msg.Meta) {
 		return
 	}
 
-	uuid := payloadID(msg.Meta)
+	uuid := proto.PayloadID(msg.Meta)
 
 	start := time.Now()
 	resp, err := client.Send(msg.Data)
 	stop := time.Now()
 
 	if err != nil {
-		Debug(1, "Request error:", err)
+		outputLogger.Error().Err(err).Msg("Request error")
 	}
 
 	if o.config.TrackResponses {
-		o.responses <- response{resp, uuid, start.UnixNano(), stop.UnixNano() - start.UnixNano()}
+		o.responses <- plugin.Response{resp, uuid, start.UnixNano(), stop.UnixNano() - start.UnixNano()}
 	}
 }
 

pkg/binary/tcp_client.go

@@ -1,14 +1,18 @@
-package main
+package binary
 
 import (
 	"crypto/tls"
 	"io"
 	"net"
-	"runtime/debug"
 	"syscall"
 	"time"
 )
 
+const (
+	maxResponseSize = 1073741824
+	readChunkSize   = 64 * 1024
+)
+
 // TCPClientConfig client configuration
 type TCPClientConfig struct {
 	Debug              bool
@@ -70,7 +74,8 @@ func (c *TCPClient) Disconnect() {
 	if c.conn != nil {
 		c.conn.Close()
 		c.conn = nil
-		Debug(1, "[TCPClient] Disconnected: ", c.baseURL)
+
+		outputLogger.Warn().Msgf("Disconnected: %s", c.baseURL)
 	}
 }
 
@@ -84,10 +89,10 @@ func (c *TCPClient) isAlive() bool {
 	if err == nil {
 		return true
 	} else if err == io.EOF {
-		Debug(1, "[TCPClient] connection closed, reconnecting")
+		outputLogger.Warn().Msg("connection closed, reconnecting")
 		return false
 	} else if err == syscall.EPIPE {
-		Debug(1, "Detected broken pipe.", err)
+		outputLogger.Warn().Msg("broken pipe, reconnecting")
 		return false
 	}
 
@@ -99,19 +104,18 @@ func (c *TCPClient) Send(data []byte) (response []byte, err error) {
 	// Don't exit on panic
 	defer func() {
 		if r := recover(); r != nil {
-			Debug(1, "[TCPClient]", r, string(data))
+			outputLogger.Error().Msgf("PANIC: pkg: %v", r)
 
 			if _, ok := r.(error); !ok {
-				Debug(1, "[TCPClient] Failed to send request: ", string(data))
-				Debug(1, "PANIC: pkg:", r, debug.Stack())
+				outputLogger.Error().Stack().Msgf("faile to send request: %s", string(data))
 			}
 		}
 	}()
 
 	if c.conn == nil || !c.isAlive() {
-		Debug(1, "[TCPClient] Connecting:", c.baseURL)
+		outputLogger.Info().Msgf("Connecting: %s", c.baseURL)
 		if err = c.Connect(); err != nil {
-			Debug(1, "[TCPClient] Connection error:", err)
+			outputLogger.Error().Err(err).Msgf("Connection error: %s", c.baseURL)
 			return
 		}
 	}
@@ -121,11 +125,11 @@ func (c *TCPClient) Send(data []byte) (response []byte, err error) {
 	c.conn.SetWriteDeadline(timeout)
 
 	if c.config.Debug {
-		Debug(1, "[TCPClient] Sending:", string(data))
+		outputLogger.Debug().Msgf("Sending: %s", string(data))
 	}
 
 	if _, err = c.conn.Write(data); err != nil {
-		Debug(1, "[TCPClient] Write error:", err, c.baseURL)
+		outputLogger.Error().Err(err).Msgf("Write error: %s", c.baseURL)
 		return
 	}
 
@@ -156,7 +160,7 @@ func (c *TCPClient) Send(data []byte) (response []byte, err error) {
 			if err == io.EOF {
 				break
 			} else if err != nil {
-				Debug(1, "[TCPClient] Read the whole body error:", err, c.baseURL)
+				outputLogger.Error().Err(err).Msgf("Read error: %s", c.baseURL)
 				break
 			}
 
@@ -164,8 +168,7 @@ func (c *TCPClient) Send(data []byte) (response []byte, err error) {
 		}
 
 		if readBytes >= maxResponseSize {
-			Debug(1, "[TCPClient] Body is more than the max size", maxResponseSize,
-				c.baseURL)
+			outputLogger.Error().Msgf("Body is more than the max size: %d", maxResponseSize)
 			break
 		}
 
@@ -174,7 +177,7 @@ func (c *TCPClient) Send(data []byte) (response []byte, err error) {
 	}
 
 	if err != nil {
-		Debug(1, "[TCPClient] Response read error", err, c.conn, readBytes)
+		outputLogger.Error().Err(err).Msgf("Response read error")
 		return
 	}
 
@@ -186,7 +189,7 @@ func (c *TCPClient) Send(data []byte) (response []byte, err error) {
 	copy(payload, c.respBuf[:readBytes])
 
 	if c.config.Debug {
-		Debug(1, "[TCPClient] Received:", string(payload))
+		outputLogger.Debug().Msgf("Received: %s", string(payload))
 	}
 
 	return payload, err

pkg/dummy/input_dummy.go

@@ -1,7 +1,10 @@
-package main
+package dummy
 
 import (
 	"time"
+
+	"github.com/buger/goreplay/pkg/plugin"
+	"github.com/buger/goreplay/pkg/proto"
 )
 
 // DummyInput used for debugging. It generate 1 "GET /"" request per second.
@@ -22,13 +25,13 @@ func NewDummyInput(options string) (di *DummyInput) {
 }
 
 // PluginRead reads message from this plugin
-func (i *DummyInput) PluginRead() (*Message, error) {
-	var msg Message
+func (i *DummyInput) PluginRead() (*plugin.Message, error) {
+	var msg plugin.Message
 	select {
 	case <-i.quit:
-		return nil, ErrorStopped
+		return nil, plugin.ErrorStopped
 	case buf := <-i.data:
-		msg.Meta, msg.Data = payloadMetaWithBody(buf)
+		msg.Meta, msg.Data = proto.PayloadMetaWithBody(buf)
 		return &msg, nil
 	}
 }
@@ -36,16 +39,13 @@ func (i *DummyInput) PluginRead() (*Message, error) {
 func (i *DummyInput) emit() {
 	ticker := time.NewTicker(time.Second)
 
-	for {
-		select {
-		case <-ticker.C:
-			uuid := uuid()
-			reqh := payloadHeader(RequestPayload, uuid, time.Now().UnixNano(), -1)
-			i.data <- append(reqh, []byte("GET / HTTP/1.1\r\nHost: www.w3.org\r\nUser-Agent: Go 1.1 package http\r\nAccept-Encoding: gzip\r\n\r\n")...)
+	for range ticker.C {
+		uuid := proto.UUID()
+		reqh := proto.PayloadHeader(proto.RequestPayload, uuid, time.Now().UnixNano(), -1)
+		i.data <- append(reqh, []byte("GET / HTTP/1.1\r\nHost: www.w3.org\r\nUser-Agent: Go 1.1 package http\r\nAccept-Encoding: gzip\r\n\r\n")...)
 
-			resh := payloadHeader(ResponsePayload, uuid, time.Now().UnixNano()+1, 1)
-			i.data <- append(resh, []byte("HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n")...)
-		}
+		resh := proto.PayloadHeader(proto.ResponsePayload, uuid, time.Now().UnixNano()+1, 1)
+		i.data <- append(resh, []byte("HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n")...)
 	}
 }
 

pkg/dummy/output_dummy.go

@@ -1,7 +1,10 @@
-package main
+package dummy
 
 import (
 	"os"
+
+	"github.com/buger/goreplay/pkg/plugin"
+	"github.com/buger/goreplay/pkg/proto"
 )
 
 // DummyOutput used for debugging, prints all incoming requests
@@ -16,15 +19,15 @@ func NewDummyOutput() (di *DummyOutput) {
 }
 
 // PluginWrite writes message to this plugin
-func (i *DummyOutput) PluginWrite(msg *Message) (int, error) {
+func (i *DummyOutput) PluginWrite(msg *plugin.Message) (int, error) {
 	var n, nn int
-	var err error
-	n, err = os.Stdout.Write(msg.Meta)
-	nn, err = os.Stdout.Write(msg.Data)
+	n, _ = os.Stdout.Write(msg.Meta)
+	nn, _ = os.Stdout.Write(msg.Data)
 	n += nn
-	nn, err = os.Stdout.Write(payloadSeparatorAsBytes)
+	nn, _ = os.Stdout.Write(proto.PayloadSeparatorAsBytes)
 	n += nn
-	return n, err
+
+	return n, nil
 }
 
 func (i *DummyOutput) String() string {

pkg/elasticsearch/elasticsearch.go

@@ -1,17 +1,20 @@
-package main
+package elasticsearch
 
 import (
 	"encoding/json"
-	"log"
 	"net/url"
 	"strings"
 	"time"
 
-	"github.com/buger/goreplay/proto"
+	proto "github.com/buger/goreplay/pkg/http_proto"
 
 	elastigo "github.com/mattbaird/elastigo/lib"
+
+	"github.com/rs/zerolog/log"
 )
 
+var logger = log.With().Str("component", "elasticsearch").Logger()
+
 type ESUriErorr struct{}
 
 func (e *ESUriErorr) Error() string {
@@ -57,7 +60,7 @@ type ESRequestResponse struct {
 //
 // Proper format is: scheme://[userinfo@]host/index_name
 // userinfo is: user[:password]
-// net/url.Parse() does not fail if scheme is not provided but actualy does not
+// net/url.Parse() does not fail if scheme is not provided but actually does not
 // handle URI properly.
 // So we must 'validate' URI format to match requirements to use net/url.Parse()
 func parseURI(URI string) (err error, index string) {
@@ -88,7 +91,7 @@ func (p *ESPlugin) Init(URI string) {
 	err, p.Index = parseURI(URI)
 
 	if err != nil {
-		log.Fatal("Can't initialize ElasticSearch plugin.", err)
+		logger.Fatal().Err(err).Msg("Can't initialize ElasticSearch plugin.")
 	}
 
 	p.eConn = elastigo.NewConn()
@@ -101,7 +104,7 @@ func (p *ESPlugin) Init(URI string) {
 
 	go p.ErrorHandler()
 
-	Debug(1, "Initialized Elasticsearch Plugin")
+	logger.Info().Msg("Initialized Elasticsearch Plugin")
 	return
 }
 
@@ -113,7 +116,7 @@ func (p *ESPlugin) IndexerShutdown() {
 func (p *ESPlugin) ErrorHandler() {
 	for {
 		errBuf := <-p.indexor.ErrorChannel
-		Debug(1, "[ELASTICSEARCH]", errBuf.Err)
+		logger.Error().Err(errBuf.Err).Msg("Error indexing document")
 	}
 }
 
@@ -157,9 +160,10 @@ func (p *ESPlugin) ResponseAnalyze(req, resp []byte, start, stop time.Time) {
 		Rtt:                  rtt,
 		Timestamp:            t,
 	}
+
 	j, err := json.Marshal(&esResp)
 	if err != nil {
-		Debug(0, "[ELASTIC-RESPONSE]", err)
+		logger.Error().Err(err).Msg("Error marshaling ESRequestResponse")
 	} else {
 		p.indexor.Index(p.Index, "RequestResponse", "", "", "", &t, j)
 	}

pkg/elasticsearch/elasticsearch_test.go

@@ -1,4 +1,4 @@
-package main
+package elasticsearch
 
 import (
 	"testing"

pkg/emitter/emitter.go

@@ -0,0 +1,186 @@
+package emitter
+
+import (
+	"hash/fnv"
+	"io"
+	"sync"
+
+	"github.com/buger/goreplay/internal/byteutils"
+	"github.com/buger/goreplay/internal/size"
+	"github.com/buger/goreplay/pkg/http_modifier"
+	"github.com/buger/goreplay/pkg/middleware"
+	"github.com/buger/goreplay/pkg/plugin"
+	"github.com/buger/goreplay/pkg/prettify"
+	"github.com/buger/goreplay/pkg/pro"
+	"github.com/buger/goreplay/pkg/proto"
+
+	"github.com/coocood/freecache"
+	"github.com/rs/zerolog"
+	"github.com/rs/zerolog/log"
+)
+
+var logger = log.With().Str("component", "emitter").Logger()
+
+// Emitter represents an abject to manage plugins communication
+type Emitter struct {
+	sync.WaitGroup
+	plugins *plugin.InOutPlugins
+	config  *Config
+}
+
+// Config represents configuration for Emitter
+type Config struct {
+	CopyBufferSize       size.Size `json:"copy-buffer-size"`
+	Middleware           string    `json:"middleware"`
+	ModifierConfig       http_modifier.HTTPModifierConfig
+	PrettifyHTTP         bool `json:"prettify-http"`
+	SplitOutput          bool `json:"split-output"`
+	RecognizeTCPSessions bool `json:"recognize-tcp-sessions"`
+}
+
+// New creates and initializes new Emitter object.
+func New(configs ...*Config) *Emitter {
+	config := &Config{}
+	if len(configs) > 0 {
+		config = configs[0]
+	}
+
+	return &Emitter{
+		config: config,
+	}
+}
+
+// Start initialize loop for sending data from inputs to outputs
+func (e *Emitter) Start(plugins *plugin.InOutPlugins) {
+	if e.config.CopyBufferSize < 1 {
+		e.config.CopyBufferSize = 5 << 20
+	}
+	e.plugins = plugins
+
+	if e.config.Middleware != "" {
+		middleware := middleware.NewMiddleware(e.config.Middleware, nil)
+
+		for _, in := range plugins.Inputs {
+			middleware.ReadFrom(in)
+		}
+
+		e.plugins.Inputs = append(e.plugins.Inputs, middleware)
+		e.plugins.All = append(e.plugins.All, middleware)
+		e.Add(1)
+		go func() {
+			defer e.Done()
+			if err := e.CopyMulty(middleware, plugins.Outputs...); err != nil {
+				logger.Error().Err(err).Msg("error during copy")
+			}
+		}()
+	} else {
+		for _, in := range plugins.Inputs {
+			e.Add(1)
+			go func(in plugin.Reader) {
+				defer e.Done()
+				if err := e.CopyMulty(in, plugins.Outputs...); err != nil {
+					logger.Error().Err(err).Msg("error during copy")
+				}
+			}(in)
+		}
+	}
+}
+
+// Close closes all the goroutine and waits for it to finish.
+func (e *Emitter) Close() {
+	for _, p := range e.plugins.All {
+		if cp, ok := p.(io.Closer); ok {
+			cp.Close()
+		}
+	}
+	if len(e.plugins.All) > 0 {
+		// wait for everything to stop
+		e.Wait()
+	}
+	e.plugins.All = nil // avoid Close to make changes again
+}
+
+// CopyMulty copies from 1 reader to multiple writers
+func (e *Emitter) CopyMulty(src plugin.Reader, writers ...plugin.Writer) error {
+	wIndex := 0
+	modifier := http_modifier.NewHTTPModifier(&e.config.ModifierConfig)
+	filteredRequests := freecache.NewCache(200 * 1024 * 1024) // 200M
+
+	for {
+		msg, err := src.PluginRead()
+		if err != nil {
+			if err == plugin.ErrorStopped || err == io.EOF {
+				return nil
+			}
+			return err
+		}
+		if msg != nil && len(msg.Data) > 0 {
+			if len(msg.Data) > int(e.config.CopyBufferSize) {
+				msg.Data = msg.Data[:e.config.CopyBufferSize]
+			}
+			meta := proto.PayloadMeta(msg.Meta)
+			if len(meta) < 3 {
+				logger.Warn().Msgf("Found malformed record %q from %q", msg.Meta, src)
+				continue
+			}
+			requestID := meta[1]
+			// start a subroutine only when necessary
+			if log.Logger.GetLevel() == zerolog.DebugLevel {
+				logger.Debug().Msgf("input: %s from: %s", byteutils.SliceToString(msg.Meta[:len(msg.Meta)-1]), src)
+			}
+			if modifier != nil {
+				logger.Debug().Msgf("modifier: %s from: %s", requestID, src)
+				if proto.IsRequestPayload(msg.Meta) {
+					msg.Data = modifier.Rewrite(msg.Data)
+					// If modifier tells to skip request
+					if len(msg.Data) == 0 {
+						filteredRequests.Set(requestID, []byte{}, 60) //
+						continue
+					}
+					logger.Debug().Msgf("Rewritten input: %s from: %s", requestID, src)
+				} else {
+					_, err := filteredRequests.Get(requestID)
+					if err == nil {
+						filteredRequests.Del(requestID)
+						continue
+					}
+				}
+			}
+
+			if e.config.PrettifyHTTP {
+				msg.Data = prettify.PrettifyHTTP(msg.Data)
+				if len(msg.Data) == 0 {
+					continue
+				}
+			}
+
+			if e.config.SplitOutput {
+				if e.config.RecognizeTCPSessions {
+					if !pro.PRO {
+						logger.Fatal().Msg("Detailed TCP sessions work only with PRO license")
+					}
+					hasher := fnv.New32a()
+					hasher.Write(meta[1])
+
+					wIndex = int(hasher.Sum32()) % len(writers)
+					if _, err := writers[wIndex].PluginWrite(msg); err != nil {
+						return err
+					}
+				} else {
+					// Simple round robin
+					if _, err := writers[wIndex].PluginWrite(msg); err != nil {
+						return err
+					}
+
+					wIndex = (wIndex + 1) % len(writers)
+				}
+			} else {
+				for _, dst := range writers {
+					if _, err := dst.PluginWrite(msg); err != nil && err != io.ErrClosedPipe {
+						return err
+					}
+				}
+			}
+		}
+	}
+}

pkg/emitter/emitter_test.go

@@ -0,0 +1,344 @@
+package emitter
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"os"
+	"os/exec"
+	"strings"
+	"sync"
+	"sync/atomic"
+	"syscall"
+	"testing"
+	"time"
+
+	"github.com/buger/goreplay/pkg/http_modifier"
+	"github.com/buger/goreplay/pkg/middleware"
+	"github.com/buger/goreplay/pkg/plugin"
+	"github.com/buger/goreplay/pkg/pro"
+	"github.com/buger/goreplay/pkg/proto"
+	"github.com/buger/goreplay/pkg/test"
+)
+
+func TestMain(m *testing.M) {
+	pro.Enable()
+	code := m.Run()
+	os.Exit(code)
+}
+
+func TestEmitter(t *testing.T) {
+	wg := new(sync.WaitGroup)
+
+	input := test.NewTestInput()
+	output := test.NewTestOutput(func(*plugin.Message) {
+		wg.Done()
+	})
+
+	plugins := &plugin.InOutPlugins{
+		Inputs:  []plugin.Reader{input},
+		Outputs: []plugin.Writer{output},
+	}
+	plugins.All = append(plugins.All, input, output)
+
+	emitter := New()
+	go emitter.Start(plugins)
+
+	for i := 0; i < 1000; i++ {
+		wg.Add(1)
+		input.EmitGET()
+	}
+
+	wg.Wait()
+	emitter.Close()
+}
+
+func TestEmitterFiltered(t *testing.T) {
+	wg := new(sync.WaitGroup)
+
+	input := test.NewTestInput()
+	input.SkipHeader = true
+
+	output := test.NewTestOutput(func(*plugin.Message) {
+		wg.Done()
+	})
+
+	plugins := &plugin.InOutPlugins{
+		Inputs:  []plugin.Reader{input},
+		Outputs: []plugin.Writer{output},
+	}
+	plugins.All = append(plugins.All, input, output)
+
+	methods := http_modifier.HTTPMethods{[]byte("GET")}
+	emitter := New(&Config{
+		ModifierConfig: http_modifier.HTTPModifierConfig{Methods: methods},
+	})
+	go emitter.Start(plugins)
+
+	wg.Add(2)
+
+	id := proto.UUID()
+	reqh := proto.PayloadHeader(proto.RequestPayload, id, time.Now().UnixNano(), -1)
+	reqb := append(reqh, []byte("POST / HTTP/1.1\r\nHost: www.w3.org\r\nUser-Agent: Go 1.1 package http\r\nAccept-Encoding: gzip\r\n\r\n")...)
+
+	resh := proto.PayloadHeader(proto.ResponsePayload, id, time.Now().UnixNano()+1, 1)
+	respb := append(resh, []byte("HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n")...)
+
+	input.EmitBytes(reqb)
+	input.EmitBytes(respb)
+
+	id = proto.UUID()
+	reqh = proto.PayloadHeader(proto.RequestPayload, id, time.Now().UnixNano(), -1)
+	reqb = append(reqh, []byte("GET / HTTP/1.1\r\nHost: www.w3.org\r\nUser-Agent: Go 1.1 package http\r\nAccept-Encoding: gzip\r\n\r\n")...)
+
+	resh = proto.PayloadHeader(proto.ResponsePayload, id, time.Now().UnixNano()+1, 1)
+	respb = append(resh, []byte("HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n")...)
+
+	input.EmitBytes(reqb)
+	input.EmitBytes(respb)
+
+	wg.Wait()
+	emitter.Close()
+}
+
+func TestEmitterSplitRoundRobin(t *testing.T) {
+	wg := new(sync.WaitGroup)
+
+	input := test.NewTestInput()
+
+	var counter1, counter2 int32
+
+	output1 := test.NewTestOutput(func(*plugin.Message) {
+		atomic.AddInt32(&counter1, 1)
+		wg.Done()
+	})
+
+	output2 := test.NewTestOutput(func(*plugin.Message) {
+		atomic.AddInt32(&counter2, 1)
+		wg.Done()
+	})
+
+	plugins := &plugin.InOutPlugins{
+		Inputs:  []plugin.Reader{input},
+		Outputs: []plugin.Writer{output1, output2},
+	}
+
+	emitter := New(&Config{
+		SplitOutput: true,
+	})
+	go emitter.Start(plugins)
+
+	for i := 0; i < 1000; i++ {
+		wg.Add(1)
+		input.EmitGET()
+	}
+
+	wg.Wait()
+
+	emitter.Close()
+
+	if counter1 == 0 || counter2 == 0 || counter1 != counter2 {
+		t.Errorf("Round robin should split traffic equally: %d vs %d", counter1, counter2)
+	}
+}
+
+func TestEmitterRoundRobin(t *testing.T) {
+	wg := new(sync.WaitGroup)
+
+	input := test.NewTestInput()
+
+	var counter1, counter2 int32
+
+	output1 := test.NewTestOutput(func(*plugin.Message) {
+		counter1++
+		wg.Done()
+	})
+
+	output2 := test.NewTestOutput(func(*plugin.Message) {
+		counter2++
+		wg.Done()
+	})
+
+	plugins := &plugin.InOutPlugins{
+		Inputs:  []plugin.Reader{input},
+		Outputs: []plugin.Writer{output1, output2},
+	}
+	plugins.All = append(plugins.All, input, output1, output2)
+
+	emitter := New(&Config{
+		SplitOutput: true,
+	})
+	go emitter.Start(plugins)
+
+	for i := 0; i < 1000; i++ {
+		wg.Add(1)
+		input.EmitGET()
+	}
+
+	wg.Wait()
+	emitter.Close()
+
+	if counter1 == 0 || counter2 == 0 {
+		t.Errorf("Round robin should split traffic equally: %d vs %d", counter1, counter2)
+	}
+}
+
+func TestEmitterSplitSession(t *testing.T) {
+	wg := new(sync.WaitGroup)
+	wg.Add(200)
+
+	input := test.NewTestInput()
+	input.SkipHeader = true
+
+	var counter1, counter2 int32
+
+	output1 := test.NewTestOutput(func(msg *plugin.Message) {
+		if proto.PayloadID(msg.Meta)[0] == 'a' {
+			counter1++
+		}
+		wg.Done()
+	})
+
+	output2 := test.NewTestOutput(func(msg *plugin.Message) {
+		if proto.PayloadID(msg.Meta)[0] == 'b' {
+			counter2++
+		}
+		wg.Done()
+	})
+
+	plugins := &plugin.InOutPlugins{
+		Inputs:  []plugin.Reader{input},
+		Outputs: []plugin.Writer{output1, output2},
+	}
+
+	emitter := New(&Config{
+		SplitOutput:          true,
+		RecognizeTCPSessions: true,
+	})
+	go emitter.Start(plugins)
+
+	for i := 0; i < 200; i++ {
+		// Keep session but randomize
+		id := make([]byte, 20)
+		if i&1 == 0 { // for recognizeTCPSessions one should be odd and other will be even number
+			id[0] = 'a'
+		} else {
+			id[0] = 'b'
+		}
+		input.EmitBytes([]byte(fmt.Sprintf("1 %s 1 1\nGET / HTTP/1.1\r\n\r\n", id[:20])))
+	}
+
+	wg.Wait()
+
+	if counter1 != counter2 {
+		t.Errorf("Round robin should split traffic equally: %d vs %d", counter1, counter2)
+	}
+
+	emitter.Close()
+}
+
+func BenchmarkEmitter(b *testing.B) {
+	wg := new(sync.WaitGroup)
+
+	input := test.NewTestInput()
+
+	output := test.NewTestOutput(func(*plugin.Message) {
+		wg.Done()
+	})
+
+	plugins := &plugin.InOutPlugins{
+		Inputs:  []plugin.Reader{input},
+		Outputs: []plugin.Writer{output},
+	}
+	plugins.All = append(plugins.All, input, output)
+
+	emitter := New(&Config{})
+	go emitter.Start(plugins)
+
+	b.ResetTimer()
+
+	for i := 0; i < b.N; i++ {
+		wg.Add(1)
+		input.EmitGET()
+	}
+
+	wg.Wait()
+	emitter.Close()
+}
+
+const echoSh = "./examples/middleware/echo.sh"
+const tokenModifier = "go run ./examples/middleware/token_modifier.go"
+
+var withDebug = append(syscall.Environ(), "GOR_TEST=1")
+
+func initMiddleware(cmd *exec.Cmd, cancl context.CancelFunc, l plugin.Reader, c func(error)) *middleware.Middleware {
+	var m middleware.Middleware
+	m.Data = make(chan *plugin.Message, 1000)
+	m.Stop = make(chan bool)
+	m.CommandCancel = cancl
+	m.Stdout, _ = cmd.StdoutPipe()
+	m.Stdin, _ = cmd.StdinPipe()
+	cmd.Stderr = os.Stderr
+	go m.Read(m.Stdout)
+	go func() {
+		defer m.Close()
+		var err error
+		if err = cmd.Start(); err == nil {
+			err = cmd.Wait()
+		}
+		if err != nil {
+			c(err)
+		}
+	}()
+	m.ReadFrom(l)
+	return &m
+}
+
+func initCmd(command string, env []string) (*exec.Cmd, context.CancelFunc) {
+	commands := strings.Split(command, " ")
+	ctx, cancl := context.WithCancel(context.Background())
+	cmd := exec.CommandContext(ctx, commands[0], commands[1:]...)
+	cmd.Env = env
+	return cmd, cancl
+}
+
+func TestMiddlewareEarlyClose(t *testing.T) {
+	t.Skip()
+	quit := make(chan struct{})
+	in := test.NewTestInput()
+	cmd, cancl := initCmd(echoSh, withDebug)
+	midd := initMiddleware(cmd, cancl, in, func(err error) {
+		if err != nil {
+			if e, ok := err.(*exec.ExitError); ok {
+				status := e.Sys().(syscall.WaitStatus)
+				if status.Signal() != syscall.SIGKILL {
+					t.Errorf("expected error to be signal killed. got %s", status.Signal().String())
+				}
+			}
+		}
+		quit <- struct{}{}
+	})
+	var body = []byte("OPTIONS / HTTP/1.1\r\nHost: example.org\r\n\r\n")
+	count := uint32(0)
+	out := test.NewTestOutput(func(msg *plugin.Message) {
+		if !bytes.Equal(body, msg.Data) {
+			t.Errorf("expected %q to equal %q", body, msg.Data)
+		}
+		atomic.AddUint32(&count, 1)
+		if atomic.LoadUint32(&count) == 5 {
+			quit <- struct{}{}
+		}
+	})
+	pl := &plugin.InOutPlugins{}
+	pl.Inputs = []plugin.Reader{midd, in}
+	pl.Outputs = []plugin.Writer{out}
+	pl.All = []interface{}{midd, out, in}
+	e := New()
+	go e.Start(pl)
+	for i := 0; i < 5; i++ {
+		in.EmitBytes(body)
+	}
+	<-quit
+	midd.Close()
+	<-quit
+}

pkg/file/input_file.go

@@ -0,0 +1,464 @@
+package file
+
+import (
+	"bufio"
+	"bytes"
+	"compress/gzip"
+	"container/heap"
+	"errors"
+	"expvar"
+	"fmt"
+	"io"
+	"math"
+	"os"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/s3"
+	"github.com/buger/goreplay/pkg/plugin"
+	"github.com/buger/goreplay/pkg/proto"
+
+	"github.com/rs/zerolog/log"
+)
+
+var inputLogger = log.With().Str("component", "input_file").Logger()
+
+// InputFileConfig contains config of input file
+type InputFileConfig struct {
+	InputFileLoop      bool          `json:"input-file-loop"`
+	InputFileReadDepth int           `json:"input-file-read-depth"`
+	InputFileDryRun    bool          `json:"input-file-dry-run"`
+	InputFileMaxWait   time.Duration `json:"input-file-max-wait"`
+}
+
+type filePayload struct {
+	data      []byte
+	timestamp int64
+}
+
+// An IntHeap is a min-heap of ints.
+type payloadQueue struct {
+	sync.RWMutex
+	s []*filePayload
+}
+
+func (h payloadQueue) Len() int           { return len(h.s) }
+func (h payloadQueue) Less(i, j int) bool { return h.s[i].timestamp < h.s[j].timestamp }
+func (h payloadQueue) Swap(i, j int)      { h.s[i], h.s[j] = h.s[j], h.s[i] }
+
+func (h *payloadQueue) Push(x interface{}) {
+	// Push and Pop use pointer receivers because they modify the slice's length,
+	// not just its contents.
+	h.s = append(h.s, x.(*filePayload))
+}
+
+func (h *payloadQueue) Pop() interface{} {
+	old := h.s
+	n := len(old)
+	x := old[n-1]
+	h.s = old[0 : n-1]
+	return x
+}
+
+func (h payloadQueue) Idx(i int) *filePayload {
+	return h.s[i]
+}
+
+type fileInputReader struct {
+	reader    *bufio.Reader
+	file      io.ReadCloser
+	closed    int32 // Value of 0 indicates that the file is still open.
+	s3        bool
+	queue     payloadQueue
+	readDepth int
+	dryRun    bool
+	path      string
+}
+
+func (f *fileInputReader) parse(init chan struct{}) error {
+	payloadSeparatorAsBytes := []byte(proto.PayloadSeparator)
+	var buffer bytes.Buffer
+	var initialized bool
+
+	lineNum := 0
+
+	for {
+		line, err := f.reader.ReadBytes('\n')
+		lineNum++
+
+		if err != nil {
+			if err != io.EOF {
+				inputLogger.Error().Err(err).Msg("Error reading file")
+			}
+
+			f.Close()
+
+			if !initialized {
+				close(init)
+				initialized = true
+			}
+
+			return err
+		}
+
+		if bytes.Equal(payloadSeparatorAsBytes[1:], line) {
+			asBytes := buffer.Bytes()
+			meta := proto.PayloadMeta(asBytes)
+
+			if len(meta) < 3 {
+				inputLogger.Warn().Msgf("Found malformed record, file: %s, line %d", f.path, lineNum)
+				buffer = bytes.Buffer{}
+				continue
+			}
+
+			timestamp, _ := strconv.ParseInt(string(meta[2]), 10, 64)
+			data := asBytes[:len(asBytes)-1]
+
+			f.queue.Lock()
+			heap.Push(&f.queue, &filePayload{
+				timestamp: timestamp,
+				data:      data,
+			})
+			f.queue.Unlock()
+
+			for {
+				if f.queue.Len() < f.readDepth {
+					break
+				}
+
+				if !initialized {
+					close(init)
+					initialized = true
+				}
+
+				if !f.dryRun {
+					time.Sleep(100 * time.Millisecond)
+				}
+			}
+
+			buffer = bytes.Buffer{}
+			continue
+		}
+
+		buffer.Write(line)
+	}
+}
+
+func (f *fileInputReader) wait() {
+	for {
+		if atomic.LoadInt32(&f.closed) == 1 {
+			return
+		}
+
+		if f.queue.Len() > 0 {
+			return
+		}
+
+		if !f.dryRun {
+			time.Sleep(100 * time.Millisecond)
+		}
+	}
+
+	return
+}
+
+// Close closes this plugin
+func (f *fileInputReader) Close() error {
+	if atomic.LoadInt32(&f.closed) == 0 {
+		atomic.StoreInt32(&f.closed, 1)
+		f.file.Close()
+	}
+
+	return nil
+}
+
+func newFileInputReader(path string, readDepth int, dryRun bool) *fileInputReader {
+	var file io.ReadCloser
+	var err error
+
+	if strings.HasPrefix(path, "s3://") {
+		file = NewS3ReadCloser(path)
+	} else {
+		file, err = os.Open(path)
+	}
+
+	if err != nil {
+		inputLogger.Error().Err(err).Msg("Error opening file")
+		return nil
+	}
+
+	r := &fileInputReader{path: path, file: file, closed: 0, readDepth: readDepth, dryRun: dryRun}
+	if strings.HasSuffix(path, ".gz") {
+		gzReader, err := gzip.NewReader(file)
+		if err != nil {
+			inputLogger.Error().Err(err).Msg("Error opening compressed file")
+			return nil
+		}
+		r.reader = bufio.NewReader(gzReader)
+	} else {
+		r.reader = bufio.NewReader(file)
+	}
+
+	heap.Init(&r.queue)
+
+	init := make(chan struct{})
+	go r.parse(init)
+	<-init
+
+	return r
+}
+
+// FileInput can read requests generated by FileOutput
+type FileInput struct {
+	mu          sync.Mutex
+	data        chan []byte
+	exit        chan bool
+	path        string
+	readers     []*fileInputReader
+	speedFactor float64
+	loop        bool
+	readDepth   int
+	dryRun      bool
+	maxWait     time.Duration
+
+	stats *expvar.Map
+}
+
+// NewFileInput constructor for FileInput. Accepts file path as argument.
+func NewFileInput(path string, loop bool, readDepth int, maxWait time.Duration, dryRun bool) (i *FileInput) {
+	i = new(FileInput)
+	i.data = make(chan []byte, 1000)
+	i.exit = make(chan bool)
+	i.path = path
+	i.speedFactor = 1
+	i.loop = loop
+	i.readDepth = readDepth
+	i.stats = expvar.NewMap("file-" + path)
+	i.dryRun = dryRun
+	i.maxWait = maxWait
+
+	if err := i.init(); err != nil {
+		return
+	}
+
+	go i.emit()
+
+	return
+}
+
+func (i *FileInput) init() (err error) {
+	defer i.mu.Unlock()
+	i.mu.Lock()
+
+	var matches []string
+
+	if strings.HasPrefix(i.path, "s3://") {
+		sess := session.Must(session.NewSession(awsConfig()))
+		svc := s3.New(sess)
+
+		bucket, key := parseS3Url(i.path)
+
+		params := &s3.ListObjectsInput{
+			Bucket: aws.String(bucket),
+			Prefix: aws.String(key),
+		}
+
+		resp, err := svc.ListObjects(params)
+		if err != nil {
+			inputLogger.Error().Err(err).Msgf("Error while retrieving list of files from S3: %s", i.path)
+			return err
+		}
+
+		for _, c := range resp.Contents {
+			matches = append(matches, "s3://"+bucket+"/"+(*c.Key))
+		}
+	} else if matches, err = filepath.Glob(i.path); err != nil {
+		inputLogger.Error().Err(err).Msgf("Error while retrieving list of files: %s", i.path)
+		return
+	}
+
+	if len(matches) == 0 {
+		inputLogger.Error().Msgf("No files match pattern: %s", i.path)
+		return errors.New("no matching files")
+	}
+
+	i.readers = make([]*fileInputReader, len(matches))
+
+	for idx, p := range matches {
+		i.readers[idx] = newFileInputReader(p, i.readDepth, i.dryRun)
+	}
+
+	i.stats.Add("reader_count", int64(len(matches)))
+
+	return nil
+}
+
+// Limited enabled custom logic for file input
+func (i *FileInput) Limited() bool {
+	return false
+}
+
+// SetLimit sets limit for this plugin
+func (i *FileInput) SetLimit(limit float64) {
+	i.speedFactor = limit
+}
+
+// PluginRead reads message from this plugin
+func (i *FileInput) PluginRead() (*plugin.Message, error) {
+	var msg plugin.Message
+	select {
+	case <-i.exit:
+		return nil, plugin.ErrorStopped
+	case buf := <-i.data:
+		i.stats.Add("read_from", 1)
+		msg.Meta, msg.Data = proto.PayloadMetaWithBody(buf)
+		return &msg, nil
+	}
+}
+
+func (i *FileInput) String() string {
+	return "File input: " + i.path
+}
+
+// Find reader with smallest timestamp e.g next payload in row
+func (i *FileInput) nextReader() (next *fileInputReader) {
+	for _, r := range i.readers {
+		if r == nil {
+			continue
+		}
+
+		r.wait()
+
+		if r.queue.Len() == 0 {
+			continue
+		}
+
+		if next == nil || r.queue.Idx(0).timestamp < next.queue.Idx(0).timestamp {
+			next = r
+			continue
+		}
+	}
+
+	return
+}
+
+func (i *FileInput) emit() {
+	var lastTime int64 = -1
+
+	var maxWait, firstWait, minWait int64
+	minWait = math.MaxInt64
+
+	i.stats.Add("negative_wait", 0)
+
+	for {
+		select {
+		case <-i.exit:
+			return
+		default:
+		}
+
+		reader := i.nextReader()
+
+		if reader == nil {
+			if i.loop {
+				i.init()
+				lastTime = -1
+				continue
+			} else {
+				break
+			}
+		}
+
+		reader.queue.RLock()
+		payload := heap.Pop(&reader.queue).(*filePayload)
+		i.stats.Add("total_counter", 1)
+		i.stats.Add("total_bytes", int64(len(payload.data)))
+		reader.queue.RUnlock()
+
+		if lastTime != -1 {
+			diff := payload.timestamp - lastTime
+
+			if firstWait == 0 {
+				firstWait = diff
+			}
+
+			if i.speedFactor != 1 {
+				diff = int64(float64(diff) / i.speedFactor)
+			}
+
+			if i.maxWait > 0 && diff > int64(i.maxWait) {
+				diff = int64(i.maxWait)
+			}
+
+			if diff >= 0 {
+				lastTime = payload.timestamp
+
+				if !i.dryRun {
+					time.Sleep(time.Duration(diff))
+				}
+
+				i.stats.Add("total_wait", diff)
+
+				if diff > maxWait {
+					maxWait = diff
+				}
+
+				if diff < minWait {
+					minWait = diff
+				}
+			} else {
+				i.stats.Add("negative_wait", 1)
+			}
+		} else {
+			lastTime = payload.timestamp
+		}
+
+		// Recheck if we have exited since last check.
+		select {
+		case <-i.exit:
+			return
+		default:
+			if !i.dryRun {
+				i.data <- payload.data
+			}
+		}
+	}
+
+	i.stats.Set("first_wait", time.Duration(firstWait))
+	i.stats.Set("max_wait", time.Duration(maxWait))
+	i.stats.Set("min_wait", time.Duration(minWait))
+
+	inputLogger.Info().Msgf("FileInput: end of file '%s'", i.path)
+
+	if i.dryRun {
+		fmt.Printf("Records found: %v\nFiles processed: %v\nBytes processed: %v\nMax wait: %v\nMin wait: %v\nFirst wait: %v\nIt will take `%v` to replay at current speed.\nFound %v records with out of order timestamp\n",
+			i.stats.Get("total_counter"),
+			i.stats.Get("reader_count"),
+			i.stats.Get("total_bytes"),
+			i.stats.Get("max_wait"),
+			i.stats.Get("min_wait"),
+			i.stats.Get("first_wait"),
+			time.Duration(i.stats.Get("total_wait").(*expvar.Int).Value()),
+			i.stats.Get("negative_wait"),
+		)
+	}
+}
+
+// Close closes this plugin
+func (i *FileInput) Close() error {
+	defer i.mu.Unlock()
+	i.mu.Lock()
+
+	close(i.exit)
+	for _, r := range i.readers {
+		r.Close()
+	}
+
+	return nil
+}

pkg/file/input_file_test.go

@@ -1,4 +1,4 @@
-package main
+package file
 
 import (
 	"bytes"
@@ -10,79 +10,84 @@ import (
 	"sync"
 	"testing"
 	"time"
+
+	"github.com/buger/goreplay/pkg/emitter"
+	"github.com/buger/goreplay/pkg/plugin"
+	"github.com/buger/goreplay/pkg/proto"
+	"github.com/buger/goreplay/pkg/test"
 )
 
 func TestInputFileWithGET(t *testing.T) {
-	input := NewTestInput()
-	rg := NewRequestGenerator([]PluginReader{input}, func() { input.EmitGET() }, 1)
-	readPayloads := []*Message{}
+	input := test.NewTestInput()
+	rg := NewRequestGenerator([]plugin.Reader{input}, func() { input.EmitGET() }, 1)
+	readPayloads := []*plugin.Message{}
 
 	// Given a capture file with a GET request
 	expectedCaptureFile := CreateCaptureFile(rg)
 	defer expectedCaptureFile.TearDown()
 
 	// When the request is read from the capture file
-	err := ReadFromCaptureFile(expectedCaptureFile.file, 1, func(msg *Message) {
+	err := ReadFromCaptureFile(expectedCaptureFile.file, 1, func(msg *plugin.Message) {
 		readPayloads = append(readPayloads, msg)
 	})
 
 	// The read request should match the original request
 	if err != nil {
 		t.Error(err)
-	} else if !expectedCaptureFile.PayloadsEqual(readPayloads){
-			t.Error("Request read back from file should match")
-		
+	} else if !expectedCaptureFile.PayloadsEqual(readPayloads) {
+		t.Error("Request read back from file should match")
+
 	}
 }
 
 func TestInputFileWithPayloadLargerThan64Kb(t *testing.T) {
-	input := NewTestInput()
-	rg := NewRequestGenerator([]PluginReader{input}, func() { input.EmitSizedPOST(64 * 1024) }, 1)
-	readPayloads := []*Message{}
+	input := test.NewTestInput()
+	rg := NewRequestGenerator([]plugin.Reader{input}, func() { input.EmitSizedPOST(64 * 1024) }, 1)
+	readPayloads := []*plugin.Message{}
 
 	// Given a capture file with a request over 64Kb
 	expectedCaptureFile := CreateCaptureFile(rg)
 	defer expectedCaptureFile.TearDown()
 
 	// When the request is read from the capture file
-	err := ReadFromCaptureFile(expectedCaptureFile.file, 1, func(msg *Message) {
+	err := ReadFromCaptureFile(expectedCaptureFile.file, 1, func(msg *plugin.Message) {
 		readPayloads = append(readPayloads, msg)
 	})
 
 	// The read request should match the original request
 	if err != nil {
 		t.Error(err)
-	} else if !expectedCaptureFile.PayloadsEqual(readPayloads){ 
-			t.Error("Request read back from file should match")
-		
+	} else if !expectedCaptureFile.PayloadsEqual(readPayloads) {
+		t.Error("Request read back from file should match")
+
 	}
 
 }
 
 func TestInputFileWithGETAndPOST(t *testing.T) {
 
-	input := NewTestInput()
-	rg := NewRequestGenerator([]PluginReader{input}, func() {
+	input := test.NewTestInput()
+	rg := NewRequestGenerator([]plugin.Reader{input}, func() {
 		input.EmitGET()
 		input.EmitPOST()
 	}, 2)
-	readPayloads := []*Message{}
+	readPayloads := []*plugin.Message{}
 
 	// Given a capture file with a GET request
 	expectedCaptureFile := CreateCaptureFile(rg)
 	defer expectedCaptureFile.TearDown()
 
 	// When the requests are read from the capture file
-	err := ReadFromCaptureFile(expectedCaptureFile.file, 2, func(msg *Message) {
+	err := ReadFromCaptureFile(expectedCaptureFile.file, 2, func(msg *plugin.Message) {
 		readPayloads = append(readPayloads, msg)
 	})
 
 	// The read requests should match the original request
 	if err != nil {
 		t.Error(err)
-	} else if !expectedCaptureFile.PayloadsEqual(readPayloads){ 
-			t.Error("Request read back from file should match")
-		
+	} else if !expectedCaptureFile.PayloadsEqual(readPayloads) {
+		t.Error("Request read back from file should match")
+
 	}
 
 }
@@ -92,19 +97,19 @@ func TestInputFileMultipleFilesWithRequestsOnly(t *testing.T) {
 
 	file1, _ := os.OpenFile(fmt.Sprintf("/tmp/%d_0", rnd), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0660)
 	file1.Write([]byte("1 1 1\ntest1"))
-	file1.Write([]byte(payloadSeparator))
+	file1.Write([]byte(proto.PayloadSeparator))
 	file1.Write([]byte("1 1 3\ntest2"))
-	file1.Write([]byte(payloadSeparator))
+	file1.Write([]byte(proto.PayloadSeparator))
 	file1.Close()
 
 	file2, _ := os.OpenFile(fmt.Sprintf("/tmp/%d_1", rnd), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0660)
 	file2.Write([]byte("1 1 2\ntest3"))
-	file2.Write([]byte(payloadSeparator))
+	file2.Write([]byte(proto.PayloadSeparator))
 	file2.Write([]byte("1 1 4\ntest4"))
-	file2.Write([]byte(payloadSeparator))
+	file2.Write([]byte(proto.PayloadSeparator))
 	file2.Close()
 
-	input := NewFileInput(fmt.Sprintf("/tmp/%d*", rnd), false)
+	input := NewFileInput(fmt.Sprintf("/tmp/%d*", rnd), false, 100, 0, false)
 
 	for i := '1'; i <= '4'; i++ {
 		msg, _ := input.PluginRead()
@@ -124,13 +129,13 @@ func TestInputFileRequestsWithLatency(t *testing.T) {
 	defer file.Close()
 
 	file.Write([]byte("1 1 100000000\nrequest1"))
-	file.Write([]byte(payloadSeparator))
+	file.Write([]byte(proto.PayloadSeparator))
 	file.Write([]byte("1 2 150000000\nrequest2"))
-	file.Write([]byte(payloadSeparator))
+	file.Write([]byte(proto.PayloadSeparator))
 	file.Write([]byte("1 3 250000000\nrequest3"))
-	file.Write([]byte(payloadSeparator))
+	file.Write([]byte(proto.PayloadSeparator))
 
-	input := NewFileInput(fmt.Sprintf("/tmp/%d", rnd), false)
+	input := NewFileInput(fmt.Sprintf("/tmp/%d", rnd), false, 100, 0, false)
 
 	start := time.Now().UnixNano()
 	for i := 0; i < 3; i++ {
@@ -150,27 +155,27 @@ func TestInputFileMultipleFilesWithRequestsAndResponses(t *testing.T) {
 
 	file1, _ := os.OpenFile(fmt.Sprintf("/tmp/%d_0", rnd), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0660)
 	file1.Write([]byte("1 1 1\nrequest1"))
-	file1.Write([]byte(payloadSeparator))
+	file1.Write([]byte(proto.PayloadSeparator))
 	file1.Write([]byte("2 1 1\nresponse1"))
-	file1.Write([]byte(payloadSeparator))
+	file1.Write([]byte(proto.PayloadSeparator))
 	file1.Write([]byte("1 2 3\nrequest2"))
-	file1.Write([]byte(payloadSeparator))
+	file1.Write([]byte(proto.PayloadSeparator))
 	file1.Write([]byte("2 2 3\nresponse2"))
-	file1.Write([]byte(payloadSeparator))
+	file1.Write([]byte(proto.PayloadSeparator))
 	file1.Close()
 
 	file2, _ := os.OpenFile(fmt.Sprintf("/tmp/%d_1", rnd), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0660)
 	file2.Write([]byte("1 3 2\nrequest3"))
-	file2.Write([]byte(payloadSeparator))
+	file2.Write([]byte(proto.PayloadSeparator))
 	file2.Write([]byte("2 3 2\nresponse3"))
-	file2.Write([]byte(payloadSeparator))
+	file2.Write([]byte(proto.PayloadSeparator))
 	file2.Write([]byte("1 4 4\nrequest4"))
-	file2.Write([]byte(payloadSeparator))
+	file2.Write([]byte(proto.PayloadSeparator))
 	file2.Write([]byte("2 4 4\nresponse4"))
-	file2.Write([]byte(payloadSeparator))
+	file2.Write([]byte(proto.PayloadSeparator))
 	file2.Close()
 
-	input := NewFileInput(fmt.Sprintf("/tmp/%d*", rnd), false)
+	input := NewFileInput(fmt.Sprintf("/tmp/%d*", rnd), false, 100, 0, false)
 
 	for i := '1'; i <= '4'; i++ {
 		msg, _ := input.PluginRead()
@@ -193,12 +198,12 @@ func TestInputFileLoop(t *testing.T) {
 
 	file, _ := os.OpenFile(fmt.Sprintf("/tmp/%d", rnd), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0660)
 	file.Write([]byte("1 1 1\ntest1"))
-	file.Write([]byte(payloadSeparator))
+	file.Write([]byte(proto.PayloadSeparator))
 	file.Write([]byte("1 1 2\ntest2"))
-	file.Write([]byte(payloadSeparator))
+	file.Write([]byte(proto.PayloadSeparator))
 	file.Close()
 
-	input := NewFileInput(fmt.Sprintf("/tmp/%d", rnd), true)
+	input := NewFileInput(fmt.Sprintf("/tmp/%d", rnd), true, 100, 0, false)
 
 	// Even if we have just 2 requests in file, it should indifinitly loop
 	for i := 0; i < 1000; i++ {
@@ -214,19 +219,19 @@ func TestInputFileCompressed(t *testing.T) {
 
 	output := NewFileOutput(fmt.Sprintf("/tmp/%d_0.gz", rnd), &FileOutputConfig{FlushInterval: time.Minute, Append: true})
 	for i := 0; i < 1000; i++ {
-		output.PluginWrite(&Message{Meta: []byte("1 1 1\r\n"), Data: []byte("test")})
+		output.PluginWrite(&plugin.Message{Meta: []byte("1 1 1\r\n"), Data: []byte("test")})
 	}
 	name1 := output.file.Name()
 	output.Close()
 
 	output2 := NewFileOutput(fmt.Sprintf("/tmp/%d_1.gz", rnd), &FileOutputConfig{FlushInterval: time.Minute, Append: true})
 	for i := 0; i < 1000; i++ {
-		output2.PluginWrite(&Message{Meta: []byte("1 1 1\r\n"), Data: []byte("test")})
+		output2.PluginWrite(&plugin.Message{Meta: []byte("1 1 1\r\n"), Data: []byte("test")})
 	}
 	name2 := output2.file.Name()
 	output2.Close()
 
-	input := NewFileInput(fmt.Sprintf("/tmp/%d*", rnd), false)
+	input := NewFileInput(fmt.Sprintf("/tmp/%d*", rnd), false, 100, 0, false)
 	for i := 0; i < 2000; i++ {
 		input.PluginRead()
 	}
@@ -236,11 +241,11 @@ func TestInputFileCompressed(t *testing.T) {
 }
 
 type CaptureFile struct {
-	msgs []*Message
+	msgs []*plugin.Message
 	file *os.File
 }
 
-func NewExpectedCaptureFile(msgs []*Message, file *os.File) *CaptureFile {
+func NewExpectedCaptureFile(msgs []*plugin.Message, file *os.File) *CaptureFile {
 	ecf := new(CaptureFile)
 	ecf.file = file
 	ecf.msgs = msgs
@@ -254,12 +259,12 @@ func (expectedCaptureFile *CaptureFile) TearDown() {
 }
 
 type RequestGenerator struct {
-	inputs []PluginReader
+	inputs []plugin.Reader
 	emit   func()
 	wg     *sync.WaitGroup
 }
 
-func NewRequestGenerator(inputs []PluginReader, emit func(), count int) (rg *RequestGenerator) {
+func NewRequestGenerator(inputs []plugin.Reader, emit func(), count int) (rg *RequestGenerator) {
 	rg = new(RequestGenerator)
 	rg.inputs = inputs
 	rg.emit = emit
@@ -268,7 +273,7 @@ func NewRequestGenerator(inputs []PluginReader, emit func(), count int) (rg *Req
 	return
 }
 
-func (expectedCaptureFile *CaptureFile) PayloadsEqual(other []*Message) bool {
+func (expectedCaptureFile *CaptureFile) PayloadsEqual(other []*plugin.Message) bool {
 
 	if len(expectedCaptureFile.msgs) != len(other) {
 		return false
@@ -293,25 +298,25 @@ func CreateCaptureFile(requestGenerator *RequestGenerator) *CaptureFile {
 		panic(err)
 	}
 
-	readPayloads := []*Message{}
-	output := NewTestOutput(func(msg *Message) {
+	readPayloads := []*plugin.Message{}
+	output := test.NewTestOutput(func(msg *plugin.Message) {
 		readPayloads = append(readPayloads, msg)
 		requestGenerator.wg.Done()
 	})
 
 	outputFile := NewFileOutput(f.Name(), &FileOutputConfig{FlushInterval: time.Second, Append: true})
 
-	plugins := &InOutPlugins{
+	plugins := &plugin.InOutPlugins{
 		Inputs:  requestGenerator.inputs,
-		Outputs: []PluginWriter{output, outputFile},
+		Outputs: []plugin.Writer{output, outputFile},
 	}
 	for _, input := range requestGenerator.inputs {
 		plugins.All = append(plugins.All, input)
 	}
 	plugins.All = append(plugins.All, output, outputFile)
 
-	emitter := NewEmitter()
-	go emitter.Start(plugins, Settings.Middleware)
+	emitter := emitter.New()
+	go emitter.Start(plugins)
 
 	requestGenerator.emit()
 	requestGenerator.wg.Wait()
@@ -323,24 +328,24 @@ func CreateCaptureFile(requestGenerator *RequestGenerator) *CaptureFile {
 
 }
 
-func ReadFromCaptureFile(captureFile *os.File, count int, callback writeCallback) (err error) {
+func ReadFromCaptureFile(captureFile *os.File, count int, callback test.WriteCallback) (err error) {
 	wg := new(sync.WaitGroup)
 
-	input := NewFileInput(captureFile.Name(), false)
-	output := NewTestOutput(func(msg *Message) {
+	input := NewFileInput(captureFile.Name(), false, 100, 0, false)
+	output := test.NewTestOutput(func(msg *plugin.Message) {
 		callback(msg)
 		wg.Done()
 	})
 
-	plugins := &InOutPlugins{
-		Inputs:  []PluginReader{input},
-		Outputs: []PluginWriter{output},
+	plugins := &plugin.InOutPlugins{
+		Inputs:  []plugin.Reader{input},
+		Outputs: []plugin.Writer{output},
 	}
 	plugins.All = append(plugins.All, input, output)
 
 	wg.Add(count)
-	emitter := NewEmitter()
-	go emitter.Start(plugins, Settings.Middleware)
+	emitter := emitter.New()
+	go emitter.Start(plugins)
 
 	done := make(chan int, 1)
 	go func() {

pkg/file/output_file.go

@@ -1,4 +1,4 @@
-package main
+package file
 
 import (
 	"bufio"
@@ -6,19 +6,38 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"log"
+	"math/rand"
 	"os"
 	"path/filepath"
-	"runtime/debug"
 	"sort"
 	"strconv"
 	"strings"
 	"sync"
 	"time"
 
-	"github.com/buger/goreplay/size"
+	"github.com/buger/goreplay/internal/size"
+	"github.com/buger/goreplay/pkg/plugin"
+	"github.com/buger/goreplay/pkg/proto"
+
+	"github.com/rs/zerolog/log"
 )
 
+var outputLogger = log.With().Str("component", "output_file").Logger()
+var letters = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ")
+var instanceID string
+
+func init() {
+	instanceID = randSeq(8)
+}
+
+func randSeq(n int) string {
+	b := make([]rune, n)
+	for i := range b {
+		b[i] = letters[rand.Intn(len(letters))]
+	}
+	return string(b)
+}
+
 var dateFileNameFuncs = map[string]func(*FileOutput) string{
 	"%Y":  func(o *FileOutput) string { return time.Now().Format("2006") },
 	"%m":  func(o *FileOutput) string { return time.Now().Format("01") },
@@ -29,6 +48,7 @@ var dateFileNameFuncs = map[string]func(*FileOutput) string{
 	"%NS": func(o *FileOutput) string { return fmt.Sprint(time.Now().Nanosecond()) },
 	"%r":  func(o *FileOutput) string { return string(o.currentID) },
 	"%t":  func(o *FileOutput) string { return string(o.payloadType) },
+	"%i":  func(o *FileOutput) string { return instanceID },
 }
 
 // FileOutputConfig ...
@@ -45,17 +65,17 @@ type FileOutputConfig struct {
 // FileOutput output plugin
 type FileOutput struct {
 	sync.RWMutex
-	pathTemplate   string
-	currentName    string
-	file           *os.File
-	QueueLength    int
-	chunkSize      int
-	writer         io.Writer
-	requestPerFile bool
-	currentID      []byte
-	payloadType    []byte
-	closed         bool
-	totalFileSize  size.Size
+	pathTemplate    string
+	currentName     string
+	file            *os.File
+	QueueLength     int
+	writer          io.Writer
+	requestPerFile  bool
+	currentID       []byte
+	payloadType     []byte
+	closed          bool
+	currentFileSize int
+	totalFileSize   size.Size
 
 	config *FileOutputConfig
 }
@@ -65,7 +85,6 @@ func NewFileOutput(pathTemplate string, config *FileOutputConfig) *FileOutput {
 	o := new(FileOutput)
 	o.pathTemplate = pathTemplate
 	o.config = config
-	o.updateName()
 
 	if strings.Contains(pathTemplate, "%r") {
 		o.requestPerFile = true
@@ -81,7 +100,6 @@ func NewFileOutput(pathTemplate string, config *FileOutputConfig) *FileOutput {
 			if o.IsClosed() {
 				break
 			}
-			o.updateName()
 			o.flush()
 		}
 	}()
@@ -89,7 +107,8 @@ func NewFileOutput(pathTemplate string, config *FileOutputConfig) *FileOutput {
 	return o
 }
 
-func getFileIndex(name string) int {
+// GetFileIndex returns index of file
+func GetFileIndex(name string) int {
 	ext := filepath.Ext(name)
 	withoutExt := strings.TrimSuffix(name, ext)
 
@@ -136,7 +155,7 @@ func (s sortByFileIndex) Swap(i, j int) {
 
 func (s sortByFileIndex) Less(i, j int) bool {
 	if withoutIndex(s[i]) == withoutIndex(s[j]) {
-		return getFileIndex(s[i]) < getFileIndex(s[j])
+		return GetFileIndex(s[i]) < GetFileIndex(s[j])
 	}
 
 	return s[i] < s[j]
@@ -157,7 +176,7 @@ func (o *FileOutput) filename() string {
 
 		if o.currentName == "" ||
 			((o.config.QueueLimit > 0 && o.QueueLength >= o.config.QueueLimit) ||
-				(o.config.SizeLimit > 0 && o.chunkSize >= int(o.config.SizeLimit))) {
+				(o.config.SizeLimit > 0 && o.currentFileSize >= int(o.config.SizeLimit))) {
 			nextChunk = true
 		}
 
@@ -173,7 +192,7 @@ func (o *FileOutput) filename() string {
 			last := matches[len(matches)-1]
 
 			fileIndex := 0
-			if idx := getFileIndex(last); idx != -1 {
+			if idx := GetFileIndex(last); idx != -1 {
 				fileIndex = idx
 
 				if nextChunk {
@@ -196,10 +215,10 @@ func (o *FileOutput) updateName() {
 }
 
 // PluginWrite writes message to this plugin
-func (o *FileOutput) PluginWrite(msg *Message) (n int, err error) {
+func (o *FileOutput) PluginWrite(msg *plugin.Message) (n int, err error) {
 	if o.requestPerFile {
 		o.Lock()
-		meta := payloadMeta(msg.Meta)
+		meta := proto.PayloadMeta(msg.Meta)
 		o.currentID = meta[1]
 		o.payloadType = meta[0]
 		o.Unlock()
@@ -222,7 +241,7 @@ func (o *FileOutput) PluginWrite(msg *Message) (n int, err error) {
 		}
 
 		if err != nil {
-			log.Fatal(o, "Cannot open file %q. Error: %s", o.currentName, err)
+			outputLogger.Fatal().Err(err).Str("file", o.currentName).Msg("Cannot open file")
 		}
 
 		o.QueueLength = 0
@@ -232,13 +251,14 @@ func (o *FileOutput) PluginWrite(msg *Message) (n int, err error) {
 	n, err = o.writer.Write(msg.Meta)
 	nn, err = o.writer.Write(msg.Data)
 	n += nn
-	nn, err = o.writer.Write(payloadSeparatorAsBytes)
+	nn, err = o.writer.Write(proto.PayloadSeparatorAsBytes)
 	n += nn
 
 	o.totalFileSize += size.Size(n)
+	o.currentFileSize += n
 	o.QueueLength++
 
-	if Settings.OutputFileConfig.OutputFileMaxSize > 0 && o.totalFileSize >= Settings.OutputFileConfig.OutputFileMaxSize {
+	if o.config.OutputFileMaxSize > 0 && o.totalFileSize >= o.config.OutputFileMaxSize {
 		return n, errors.New("File output reached size limit")
 	}
 
@@ -249,7 +269,7 @@ func (o *FileOutput) flush() {
 	// Don't exit on panic
 	defer func() {
 		if r := recover(); r != nil {
-			Debug(0, "[OUTPUT-FILE] PANIC while file flush: ", r, o, string(debug.Stack()))
+			outputLogger.Error().Stack().Msgf("PANIC while file flush: %v", r)
 		}
 	}()
 
@@ -264,9 +284,9 @@ func (o *FileOutput) flush() {
 		}
 
 		if stat, err := o.file.Stat(); err == nil {
-			o.chunkSize = int(stat.Size())
+			o.currentFileSize = int(stat.Size())
 		} else {
-			Debug(0, "[OUTPUT-HTTP] error accessing file size", err)
+			outputLogger.Error().Err(err).Msgf("Error accessing file size")
 		}
 	}
 }
@@ -290,6 +310,8 @@ func (o *FileOutput) closeLocked() error {
 	}
 
 	o.closed = true
+	o.currentFileSize = 0
+
 	return nil
 }
 

pkg/file/output_file_test.go

@@ -1,4 +1,4 @@
-package main
+package file
 
 import (
 	"fmt"
@@ -11,23 +11,27 @@ import (
 	"testing"
 	"time"
 
-	"github.com/buger/goreplay/size"
+	"github.com/buger/goreplay/internal/size"
+	"github.com/buger/goreplay/pkg/emitter"
+	"github.com/buger/goreplay/pkg/plugin"
+	"github.com/buger/goreplay/pkg/proto"
+	"github.com/buger/goreplay/pkg/test"
 )
 
 func TestFileOutput(t *testing.T) {
 	wg := new(sync.WaitGroup)
 
-	input := NewTestInput()
+	input := test.NewTestInput()
 	output := NewFileOutput("/tmp/test_requests.gor", &FileOutputConfig{FlushInterval: time.Minute, Append: true})
 
-	plugins := &InOutPlugins{
-		Inputs:  []PluginReader{input},
-		Outputs: []PluginWriter{output},
+	plugins := &plugin.InOutPlugins{
+		Inputs:  []plugin.Reader{input},
+		Outputs: []plugin.Writer{output},
 	}
 	plugins.All = append(plugins.All, input, output)
 
-	emitter := NewEmitter()
-	go emitter.Start(plugins, Settings.Middleware)
+	em := emitter.New()
+	go em.Start(plugins)
 
 	for i := 0; i < 100; i++ {
 		wg.Add(2)
@@ -36,26 +40,26 @@ func TestFileOutput(t *testing.T) {
 	}
 	time.Sleep(100 * time.Millisecond)
 	output.flush()
-	emitter.Close()
+	em.Close()
 
 	var counter int64
-	input2 := NewFileInput("/tmp/test_requests.gor", false)
-	output2 := NewTestOutput(func(*Message) {
+	input2 := NewFileInput("/tmp/test_requests.gor", false, 100, 0, false)
+	output2 := test.NewTestOutput(func(*plugin.Message) {
 		atomic.AddInt64(&counter, 1)
 		wg.Done()
 	})
 
-	plugins2 := &InOutPlugins{
-		Inputs:  []PluginReader{input2},
-		Outputs: []PluginWriter{output2},
+	plugins2 := &plugin.InOutPlugins{
+		Inputs:  []plugin.Reader{input2},
+		Outputs: []plugin.Writer{output2},
 	}
 	plugins2.All = append(plugins2.All, input2, output2)
 
-	emitter2 := NewEmitter()
-	go emitter2.Start(plugins2, Settings.Middleware)
+	em2 := emitter.New()
+	go em2.Start(plugins2)
 
 	wg.Wait()
-	emitter2.Close()
+	em2.Close()
 }
 
 func TestFileOutputWithNameCleaning(t *testing.T) {
@@ -88,16 +92,16 @@ func TestFileOutputMultipleFiles(t *testing.T) {
 		t.Error("Should not initialize file if no writes")
 	}
 
-	output.PluginWrite(&Message{Meta: []byte("1 1 1\r\n"), Data: []byte("test")})
+	output.PluginWrite(&plugin.Message{Meta: []byte("1 1 1\r\n"), Data: []byte("test")})
 	name1 := output.file.Name()
 
-	output.PluginWrite(&Message{Meta: []byte("1 1 1\r\n"), Data: []byte("test")})
+	output.PluginWrite(&plugin.Message{Meta: []byte("1 1 1\r\n"), Data: []byte("test")})
 	name2 := output.file.Name()
 
 	time.Sleep(time.Second)
 	output.updateName()
 
-	output.PluginWrite(&Message{Meta: []byte("1 1 1\r\n"), Data: []byte("test")})
+	output.PluginWrite(&plugin.Message{Meta: []byte("1 1 1\r\n"), Data: []byte("test")})
 	name3 := output.file.Name()
 
 	if name2 != name1 {
@@ -119,16 +123,16 @@ func TestFileOutputFilePerRequest(t *testing.T) {
 		t.Error("Should not initialize file if no writes")
 	}
 
-	output.PluginWrite(&Message{Meta: []byte("1 1 1\r\n"), Data: []byte("test")})
+	output.PluginWrite(&plugin.Message{Meta: []byte("1 1 1\r\n"), Data: []byte("test")})
 	name1 := output.file.Name()
 
-	output.PluginWrite(&Message{Meta: []byte("1 2 1\r\n"), Data: []byte("test")})
+	output.PluginWrite(&plugin.Message{Meta: []byte("1 2 1\r\n"), Data: []byte("test")})
 	name2 := output.file.Name()
 
 	time.Sleep(time.Second)
 	output.updateName()
 
-	output.PluginWrite(&Message{Meta: []byte("1 3 1\r\n"), Data: []byte("test")})
+	output.PluginWrite(&plugin.Message{Meta: []byte("1 3 1\r\n"), Data: []byte("test")})
 	name3 := output.file.Name()
 
 	if name3 == name2 || name2 == name1 || name3 == name1 {
@@ -148,7 +152,7 @@ func TestFileOutputCompression(t *testing.T) {
 	}
 
 	for i := 0; i < 1000; i++ {
-		output.PluginWrite(&Message{Meta: []byte("1 1 1\r\n"), Data: []byte("test")})
+		output.PluginWrite(&plugin.Message{Meta: []byte("1 1 1\r\n"), Data: []byte("test")})
 	}
 
 	name := output.file.Name()
@@ -174,8 +178,8 @@ func TestGetFileIndex(t *testing.T) {
 	}
 
 	for _, c := range tests {
-		if getFileIndex(c.path) != c.index {
-			t.Error(c.path, "should be", c.index, "instead", getFileIndex(c.path))
+		if GetFileIndex(c.path) != c.index {
+			t.Error(c.path, "should be", c.index, "instead", GetFileIndex(c.path))
 		}
 	}
 }
@@ -207,15 +211,15 @@ func TestFileOutputAppendQueueLimitOverflow(t *testing.T) {
 
 	output := NewFileOutput(name, &FileOutputConfig{Append: false, FlushInterval: time.Minute, QueueLimit: 2})
 
-	output.PluginWrite(&Message{Meta: []byte("1 1 1\r\n"), Data: []byte("test")})
+	output.PluginWrite(&plugin.Message{Meta: []byte("1 1 1\r\n"), Data: []byte("test")})
 	name1 := output.file.Name()
 
-	output.PluginWrite(&Message{Meta: []byte("1 1 1\r\n"), Data: []byte("test")})
+	output.PluginWrite(&plugin.Message{Meta: []byte("1 1 1\r\n"), Data: []byte("test")})
 	name2 := output.file.Name()
 
 	output.updateName()
 
-	output.PluginWrite(&Message{Meta: []byte("1 1 1\r\n"), Data: []byte("test")})
+	output.PluginWrite(&plugin.Message{Meta: []byte("1 1 1\r\n"), Data: []byte("test")})
 	name3 := output.file.Name()
 
 	if name2 != name1 || name1 != fmt.Sprintf("/tmp/%d_0", rnd) {
@@ -236,15 +240,15 @@ func TestFileOutputAppendQueueLimitNoOverflow(t *testing.T) {
 
 	output := NewFileOutput(name, &FileOutputConfig{Append: false, FlushInterval: time.Minute, QueueLimit: 3})
 
-	output.PluginWrite(&Message{Meta: []byte("1 1 1\r\n"), Data: []byte("test")})
+	output.PluginWrite(&plugin.Message{Meta: []byte("1 1 1\r\n"), Data: []byte("test")})
 	name1 := output.file.Name()
 
-	output.PluginWrite(&Message{Meta: []byte("1 1 1\r\n"), Data: []byte("test")})
+	output.PluginWrite(&plugin.Message{Meta: []byte("1 1 1\r\n"), Data: []byte("test")})
 	name2 := output.file.Name()
 
 	output.updateName()
 
-	output.PluginWrite(&Message{Meta: []byte("1 1 1\r\n"), Data: []byte("test")})
+	output.PluginWrite(&plugin.Message{Meta: []byte("1 1 1\r\n"), Data: []byte("test")})
 	name3 := output.file.Name()
 
 	if name2 != name1 || name1 != fmt.Sprintf("/tmp/%d_0", rnd) {
@@ -265,15 +269,15 @@ func TestFileOutputAppendQueueLimitGzips(t *testing.T) {
 
 	output := NewFileOutput(name, &FileOutputConfig{Append: false, FlushInterval: time.Minute, QueueLimit: 2})
 
-	output.PluginWrite(&Message{Meta: []byte("1 1 1\r\n"), Data: []byte("test")})
+	output.PluginWrite(&plugin.Message{Meta: []byte("1 1 1\r\n"), Data: []byte("test")})
 	name1 := output.file.Name()
 
-	output.PluginWrite(&Message{Meta: []byte("1 1 1\r\n"), Data: []byte("test")})
+	output.PluginWrite(&plugin.Message{Meta: []byte("1 1 1\r\n"), Data: []byte("test")})
 	name2 := output.file.Name()
 
 	output.updateName()
 
-	output.PluginWrite(&Message{Meta: []byte("1 1 1\r\n"), Data: []byte("test")})
+	output.PluginWrite(&plugin.Message{Meta: []byte("1 1 1\r\n"), Data: []byte("test")})
 	name3 := output.file.Name()
 
 	if name2 != name1 || name1 != fmt.Sprintf("/tmp/%d_0.gz", rnd) {
@@ -304,20 +308,19 @@ func TestFileOutputAppendSizeLimitOverflow(t *testing.T) {
 
 	message := []byte("1 1 1\r\ntest")
 
-	messageSize := len(message) + len(payloadSeparator)
+	messageSize := len(message) + len(proto.PayloadSeparator)
 
 	output := NewFileOutput(name, &FileOutputConfig{Append: false, FlushInterval: time.Minute, SizeLimit: size.Size(2 * messageSize)})
 
-	output.PluginWrite(&Message{Meta: []byte("1 1 1\r\n"), Data: []byte("test")})
+	output.PluginWrite(&plugin.Message{Meta: []byte("1 1 1\r\n"), Data: []byte("test")})
 	name1 := output.file.Name()
 
-	output.PluginWrite(&Message{Meta: []byte("1 1 1\r\n"), Data: []byte("test")})
+	output.PluginWrite(&plugin.Message{Meta: []byte("1 1 1\r\n"), Data: []byte("test")})
 	name2 := output.file.Name()
 
 	output.flush()
-	output.updateName()
 
-	output.PluginWrite(&Message{Meta: []byte("1 1 1\r\n"), Data: []byte("test")})
+	output.PluginWrite(&plugin.Message{Meta: []byte("1 1 1\r\n"), Data: []byte("test")})
 	name3 := output.file.Name()
 
 	if name2 != name1 || name1 != fmt.Sprintf("/tmp/%d_0", rnd) {

pkg/file/output_s3.go

@@ -1,10 +1,9 @@
-package main
+package file
 
 import (
 	_ "bufio"
 	"fmt"
 	_ "io"
-	"log"
 	"math/rand"
 	"os"
 	"path/filepath"
@@ -14,8 +13,14 @@ import (
 	"github.com/aws/aws-sdk-go/aws/session"
 	"github.com/aws/aws-sdk-go/service/s3"
 	_ "github.com/aws/aws-sdk-go/service/s3/s3manager"
+	"github.com/buger/goreplay/pkg/plugin"
+	"github.com/buger/goreplay/pkg/pro"
+
+	"github.com/rs/zerolog/log"
 )
 
+var s3Logger = log.With().Str("component", "s3").Logger()
+
 // S3Output output plugin
 type S3Output struct {
 	pathTemplate string
@@ -28,8 +33,8 @@ type S3Output struct {
 
 // NewS3Output constructor for FileOutput, accepts path
 func NewS3Output(pathTemplate string, config *FileOutputConfig) *S3Output {
-	if !PRO {
-		log.Fatal("Using S3 output and input requires PRO license")
+	if !pro.PRO {
+		s3Logger.Fatal().Msg("Using S3 output and input requires PRO license")
 		return nil
 	}
 
@@ -63,12 +68,12 @@ func NewS3Output(pathTemplate string, config *FileOutputConfig) *S3Output {
 func (o *S3Output) connect() {
 	if o.session == nil {
 		o.session = session.Must(session.NewSession(awsConfig()))
-		log.Println("[S3 Output] S3 connection succesfully initialized")
+		s3Logger.Info().Msg("[S3 Output] S3 connection successfully initialized")
 	}
 }
 
 // PluginWrite writes message to this plugin
-func (o *S3Output) PluginWrite(msg *Message) (n int, err error) {
+func (o *S3Output) PluginWrite(msg *plugin.Message) (n int, err error) {
 	return o.buffer.PluginWrite(msg)
 }
 
@@ -105,12 +110,12 @@ func (o *S3Output) keyPath(idx int) (bucket, key string) {
 
 func (o *S3Output) onBufferUpdate(path string) {
 	svc := s3.New(o.session)
-	idx := getFileIndex(path)
+	idx := GetFileIndex(path)
 	bucket, key := o.keyPath(idx)
 
 	file, err := os.Open(path)
 	if err != nil {
-		Debug(0, fmt.Sprintf("[S3 Output] Failed to open file %q. err: %q", path, err))
+		s3Logger.Error().Err(err).Msgf("[S3 Output] Failed to open file %q", path)
 		return
 	}
 	defer os.Remove(path)
@@ -121,7 +126,7 @@ func (o *S3Output) onBufferUpdate(path string) {
 		Key:    aws.String(key),
 	})
 	if err != nil {
-		Debug(0, fmt.Sprintf("[S3 Output] Failed to upload data to %q/%q, %q", bucket, key, err))
+		s3Logger.Error().Err(err).Msgf("[S3 Output] Failed to upload data to %q/%q", bucket, key)
 		return
 	}
 

pkg/file/s3_reader.go

@@ -1,8 +1,8 @@
-package main
+package file
 
 import (
 	"bytes"
-	"log"
+
 	"os"
 	"strconv"
 	"strings"
@@ -10,6 +10,7 @@ import (
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/aws/session"
 	"github.com/aws/aws-sdk-go/service/s3"
+	"github.com/buger/goreplay/pkg/pro"
 )
 
 // S3ReadCloser ...
@@ -36,10 +37,10 @@ func awsConfig() *aws.Config {
 
 	if endpoint := os.Getenv("AWS_ENDPOINT_URL"); endpoint != "" {
 		config.Endpoint = aws.String(endpoint)
-		log.Println("Custom endpoint:", endpoint)
+		s3Logger.Debug().Msgf("Custom endpoint: %s", endpoint)
 	}
 
-	log.Println("Connecting to S3. Region: " + region)
+	s3Logger.Debug().Msgf("Connecting to S3. Region: %s", region)
 
 	config.CredentialsChainVerboseErrors = aws.Bool(true)
 
@@ -52,15 +53,15 @@ func awsConfig() *aws.Config {
 
 // NewS3ReadCloser returns new instance of S3 read closer
 func NewS3ReadCloser(path string) *S3ReadCloser {
-	if !PRO {
-		log.Fatal("Using S3 input and output require PRO license")
+	if !pro.PRO {
+		s3Logger.Fatal().Msg("Using S3 input and output require PRO license")
 		return nil
 	}
 
 	bucket, key := parseS3Url(path)
 	sess := session.Must(session.NewSession(awsConfig()))
 
-	log.Println("[S3 Input] S3 connection succesfully initialized", path)
+	s3Logger.Info().Msgf("S3 connection successfully initialized %v", path)
 
 	return &S3ReadCloser{
 		bucket: bucket,
@@ -87,7 +88,7 @@ func (s *S3ReadCloser) Read(b []byte) (n int, e error) {
 		resp, err := svc.GetObject(params)
 
 		if err != nil {
-			log.Println("[S3 Input] Error during getting file", s.bucket, s.key, err)
+			s3Logger.Error().Err(err).Msgf("Error during getting file %s %s", s.bucket, s.key)
 		} else {
 			s.totalSize, _ = strconv.Atoi(strings.Split(*resp.ContentRange, "/")[1])
 			s.buf.ReadFrom(resp.Body)

pkg/file/s3_test.go

@@ -1,6 +1,6 @@
-// +build pro
+//go:build pro
 
-package main
+package file
 
 import (
 	"fmt"
@@ -127,7 +127,7 @@ func TestInputFileFromS3(t *testing.T) {
 		<-output.closeCh
 	}
 
-	input := NewFileInput(fmt.Sprintf("s3://test-gor-eu/%d", rnd), false)
+	input := NewFileInput(fmt.Sprintf("s3://test-gor-eu/%d", rnd), false, 100, 0, false)
 
 	buf := make([]byte, 1000)
 	for i := 0; i <= 19999; i++ {

pkg/http/input_http.go

@@ -1,13 +1,19 @@
-package main
+package http
 
 import (
-	"log"
 	"net"
 	"net/http"
 	"net/http/httputil"
 	"time"
+
+	"github.com/buger/goreplay/pkg/plugin"
+	"github.com/buger/goreplay/pkg/proto"
+
+	"github.com/rs/zerolog/log"
 )
 
+var inputLogger = log.With().Str("component", "input_http").Logger()
+
 // HTTPInput used for sending requests to Gor via http
 type HTTPInput struct {
 	data     chan []byte
@@ -28,14 +34,14 @@ func NewHTTPInput(address string) (i *HTTPInput) {
 }
 
 // PluginRead reads message from this plugin
-func (i *HTTPInput) PluginRead() (*Message, error) {
-	var msg Message
+func (i *HTTPInput) PluginRead() (*plugin.Message, error) {
+	var msg plugin.Message
 	select {
 	case <-i.stop:
-		return nil, ErrorStopped
+		return nil, plugin.ErrorStopped
 	case buf := <-i.data:
 		msg.Data = buf
-		msg.Meta = payloadHeader(RequestPayload, uuid(), time.Now().UnixNano(), -1)
+		msg.Meta = proto.PayloadHeader(proto.RequestPayload, proto.UUID(), time.Now().UnixNano(), -1)
 		return &msg, nil
 	}
 }
@@ -64,14 +70,14 @@ func (i *HTTPInput) listen(address string) {
 
 	i.listener, err = net.Listen("tcp", address)
 	if err != nil {
-		log.Fatal("HTTP input listener failure:", err)
+		inputLogger.Fatal().Err(err).Msg("HTTP input listener failure")
 	}
 	i.address = i.listener.Addr().String()
 
 	go func() {
 		err = http.Serve(i.listener, mux)
 		if err != nil && err != http.ErrServerClosed {
-			log.Fatal("HTTP input serve failure ", err)
+			inputLogger.Fatal().Err(err).Msg("HTTP input serve failure")
 		}
 	}()
 }

pkg/http/input_http_test.go

@@ -1,4 +1,4 @@
-package main
+package http
 
 import (
 	"bytes"
@@ -7,6 +7,10 @@ import (
 	"sync"
 	"testing"
 	"time"
+
+	"github.com/buger/goreplay/pkg/emitter"
+	"github.com/buger/goreplay/pkg/plugin"
+	"github.com/buger/goreplay/pkg/test"
 )
 
 func TestHTTPInput(t *testing.T) {
@@ -14,18 +18,18 @@ func TestHTTPInput(t *testing.T) {
 
 	input := NewHTTPInput("127.0.0.1:0")
 	time.Sleep(time.Millisecond)
-	output := NewTestOutput(func(*Message) {
+	output := test.NewTestOutput(func(*plugin.Message) {
 		wg.Done()
 	})
 
-	plugins := &InOutPlugins{
-		Inputs:  []PluginReader{input},
-		Outputs: []PluginWriter{output},
+	plugins := &plugin.InOutPlugins{
+		Inputs:  []plugin.Reader{input},
+		Outputs: []plugin.Writer{output},
 	}
 	plugins.All = append(plugins.All, input, output)
 
-	emitter := NewEmitter()
-	go emitter.Start(plugins, Settings.Middleware)
+	emitter := emitter.New(&emitter.Config{})
+	go emitter.Start(plugins)
 
 	address := strings.Replace(input.address, "[::]", "127.0.0.1", -1)
 
@@ -45,22 +49,22 @@ func TestInputHTTPLargePayload(t *testing.T) {
 	large[n-1] = '0'
 
 	input := NewHTTPInput("127.0.0.1:0")
-	output := NewTestOutput(func(msg *Message) {
+	output := test.NewTestOutput(func(msg *plugin.Message) {
 		_len := len(msg.Data)
 		if _len >= n { // considering http body CRLF
 			t.Errorf("expected body to be >= %d", n)
 		}
 		wg.Done()
 	})
-	plugins := &InOutPlugins{
-		Inputs:  []PluginReader{input},
-		Outputs: []PluginWriter{output},
+	plugins := &plugin.InOutPlugins{
+		Inputs:  []plugin.Reader{input},
+		Outputs: []plugin.Writer{output},
 	}
 	plugins.All = append(plugins.All, input, output)
 
-	emitter := NewEmitter()
+	emitter := emitter.New()
 	defer emitter.Close()
-	go emitter.Start(plugins, Settings.Middleware)
+	go emitter.Start(plugins)
 
 	address := strings.Replace(input.address, "[::]", "127.0.0.1", -1)
 	var req *http.Request

pkg/http/output_http.go

@@ -1,11 +1,10 @@
-package main
+package http
 
 import (
 	"bufio"
 	"bytes"
 	"crypto/tls"
 	"fmt"
-	"log"
 	"math"
 	"net/http"
 	"net/http/httputil"
@@ -13,20 +12,26 @@ import (
 	"sync/atomic"
 	"time"
 
-	"github.com/buger/goreplay/size"
+	"github.com/buger/goreplay/internal/size"
+	"github.com/buger/goreplay/pkg/elasticsearch"
+	"github.com/buger/goreplay/pkg/plugin"
+	"github.com/buger/goreplay/pkg/proto"
+	"github.com/buger/goreplay/pkg/stats"
+
+	"github.com/rs/zerolog/log"
 )
 
+var outputLogger = log.With().Str("component", "output_http").Logger()
+
 const (
-	initialDynamicWorkers = 10
-	readChunkSize         = 64 * 1024
-	maxResponseSize       = 1073741824
+	readChunkSize = 64 * 1024
 )
 
 type response struct {
 	payload       []byte
 	uuid          []byte
-	roundTripTime int64
 	startedAt     int64
+	roundTripTime int64
 }
 
 // HTTPOutputConfig struct for holding http output configuration
@@ -48,76 +53,95 @@ type HTTPOutputConfig struct {
 	url            *url.URL
 }
 
+func (hoc *HTTPOutputConfig) Copy() *HTTPOutputConfig {
+	return &HTTPOutputConfig{
+		TrackResponses: hoc.TrackResponses,
+		Stats:          hoc.Stats,
+		OriginalHost:   hoc.OriginalHost,
+		RedirectLimit:  hoc.RedirectLimit,
+		WorkersMin:     hoc.WorkersMin,
+		WorkersMax:     hoc.WorkersMax,
+		StatsMs:        hoc.StatsMs,
+		QueueLen:       hoc.QueueLen,
+		ElasticSearch:  hoc.ElasticSearch,
+		Timeout:        hoc.Timeout,
+		WorkerTimeout:  hoc.WorkerTimeout,
+		BufferSize:     hoc.BufferSize,
+		SkipVerify:     hoc.SkipVerify,
+	}
+}
+
 // HTTPOutput plugin manage pool of workers which send request to replayed server
 // By default workers pool is dynamic and starts with 1 worker or workerMin workers
 // You can specify maximum number of workers using `--output-http-workers`
 type HTTPOutput struct {
 	activeWorkers int32
 	config        *HTTPOutputConfig
-	queueStats    *GorStat
-	elasticSearch *ESPlugin
+	queueStats    *stats.Stats
+	elasticSearch *elasticsearch.ESPlugin
 	client        *HTTPClient
 	stopWorker    chan struct{}
-	queue         chan *Message
-	responses     chan *response
+	queue         chan *plugin.Message
+	responses     chan *plugin.Response
 	stop          chan bool // Channel used only to indicate goroutine should shutdown
 }
 
 // NewHTTPOutput constructor for HTTPOutput
 // Initialize workers
-func NewHTTPOutput(address string, config *HTTPOutputConfig) PluginReadWriter {
+func NewHTTPOutput(address string, config *HTTPOutputConfig) plugin.ReadWriter {
 	o := new(HTTPOutput)
 	var err error
-	config.url, err = url.Parse(address)
+	newConfig := config.Copy()
+	newConfig.url, err = url.Parse(address)
 	if err != nil {
-		log.Fatal(fmt.Sprintf("[OUTPUT-HTTP] parse HTTP output URL error[%q]", err))
+		outputLogger.Fatal().Msg(fmt.Sprintf("[OUTPUT-HTTP] parse HTTP output URL error[%q]", err))
 	}
-	if config.url.Scheme == "" {
-		config.url.Scheme = "http"
+	if newConfig.url.Scheme == "" {
+		newConfig.url.Scheme = "http"
 	}
-	config.rawURL = config.url.String()
-	if config.Timeout < time.Millisecond*100 {
-		config.Timeout = time.Second
+	newConfig.rawURL = newConfig.url.String()
+	if newConfig.Timeout < time.Millisecond*100 {
+		newConfig.Timeout = time.Second
 	}
-	if config.BufferSize <= 0 {
-		config.BufferSize = 100 * 1024 // 100kb
+	if newConfig.BufferSize <= 0 {
+		newConfig.BufferSize = 100 * 1024 // 100kb
 	}
-	if config.WorkersMin <= 0 {
-		config.WorkersMin = 1
+	if newConfig.WorkersMin <= 0 {
+		newConfig.WorkersMin = 1
 	}
-	if config.WorkersMin > 1000 {
-		config.WorkersMin = 1000
+	if newConfig.WorkersMin > 1000 {
+		newConfig.WorkersMin = 1000
 	}
-	if config.WorkersMax <= 0 {
-		config.WorkersMax = math.MaxInt32 // idealy so large
+	if newConfig.WorkersMax <= 0 {
+		newConfig.WorkersMax = math.MaxInt32 // idealy so large
 	}
-	if config.WorkersMax < config.WorkersMin {
-		config.WorkersMax = config.WorkersMin
+	if newConfig.WorkersMax < newConfig.WorkersMin {
+		newConfig.WorkersMax = newConfig.WorkersMin
 	}
-	if config.QueueLen <= 0 {
-		config.QueueLen = 1000
+	if newConfig.QueueLen <= 0 {
+		newConfig.QueueLen = 1000
 	}
-	if config.RedirectLimit < 0 {
-		config.RedirectLimit = 0
+	if newConfig.RedirectLimit < 0 {
+		newConfig.RedirectLimit = 0
 	}
-	if config.WorkerTimeout <= 0 {
-		config.WorkerTimeout = time.Second * 2
+	if newConfig.WorkerTimeout <= 0 {
+		newConfig.WorkerTimeout = time.Second * 2
 	}
-	o.config = config
+	o.config = newConfig
 	o.stop = make(chan bool)
 	if o.config.Stats {
-		o.queueStats = NewGorStat("output_http", o.config.StatsMs)
+		o.queueStats = stats.New("output_http", o.config.StatsMs)
 	}
 
-	o.queue = make(chan *Message, o.config.QueueLen)
+	o.queue = make(chan *plugin.Message, o.config.QueueLen)
 	if o.config.TrackResponses {
-		o.responses = make(chan *response, o.config.QueueLen)
+		o.responses = make(chan *plugin.Response, o.config.QueueLen)
 	}
 	// it should not be buffered to avoid races
 	o.stopWorker = make(chan struct{})
 
 	if o.config.ElasticSearch != "" {
-		o.elasticSearch = new(ESPlugin)
+		o.elasticSearch = new(elasticsearch.ESPlugin)
 		o.elasticSearch.Init(o.config.ElasticSearch)
 	}
 	o.client = NewHTTPClient(o.config)
@@ -168,14 +192,14 @@ func (o *HTTPOutput) startWorker() {
 }
 
 // PluginWrite writes message to this plugin
-func (o *HTTPOutput) PluginWrite(msg *Message) (n int, err error) {
-	if !isRequestPayload(msg.Meta) {
+func (o *HTTPOutput) PluginWrite(msg *plugin.Message) (n int, err error) {
+	if !proto.IsRequestPayload(msg.Meta) {
 		return len(msg.Data), nil
 	}
 
 	select {
 	case <-o.stop:
-		return 0, ErrorStopped
+		return 0, plugin.ErrorStopped
 	case o.queue <- msg:
 	}
 
@@ -193,35 +217,36 @@ func (o *HTTPOutput) PluginWrite(msg *Message) (n int, err error) {
 }
 
 // PluginRead reads message from this plugin
-func (o *HTTPOutput) PluginRead() (*Message, error) {
+func (o *HTTPOutput) PluginRead() (*plugin.Message, error) {
 	if !o.config.TrackResponses {
-		return nil, ErrorStopped
+		return nil, plugin.ErrorStopped
 	}
-	var resp *response
-	var msg Message
+	var resp *plugin.Response
+	var msg plugin.Message
 	select {
 	case <-o.stop:
-		return nil, ErrorStopped
+		return nil, plugin.ErrorStopped
 	case resp = <-o.responses:
-		msg.Data = resp.payload
+		msg.Data = resp.Payload
 	}
 
-	msg.Meta = payloadHeader(ReplayedResponsePayload, resp.uuid, resp.roundTripTime, resp.startedAt)
+	msg.Meta = proto.PayloadHeader(proto.ReplayedResponsePayload, resp.UUID, resp.StartedAt, resp.RoundTripTime)
 
 	return &msg, nil
 }
 
-func (o *HTTPOutput) sendRequest(client *HTTPClient, msg *Message) {
-	if !isRequestPayload(msg.Meta) {
+func (o *HTTPOutput) sendRequest(client *HTTPClient, msg *plugin.Message) {
+	if !proto.IsRequestPayload(msg.Meta) {
 		return
 	}
-	uuid := payloadID(msg.Meta)
+
+	uuid := proto.PayloadID(msg.Meta)
 	start := time.Now()
 	resp, err := client.Send(msg.Data)
 	stop := time.Now()
 
 	if err != nil {
-		Debug(1, fmt.Sprintf("[HTTP-OUTPUT] error when sending: %q", err))
+		outputLogger.Error().Err(err).Msg("[HTTP-OUTPUT]")
 		return
 	}
 	if resp == nil {
@@ -229,7 +254,7 @@ func (o *HTTPOutput) sendRequest(client *HTTPClient, msg *Message) {
 	}
 
 	if o.config.TrackResponses {
-		o.responses <- &response{resp, uuid, start.UnixNano(), stop.UnixNano() - start.UnixNano()}
+		o.responses <- &plugin.Response{resp, uuid, start.UnixNano(), stop.UnixNano() - start.UnixNano()}
 	}
 
 	if o.elasticSearch != nil {
@@ -263,12 +288,12 @@ func NewHTTPClient(config *HTTPOutputConfig) *HTTPClient {
 		Timeout: client.config.Timeout,
 		CheckRedirect: func(req *http.Request, via []*http.Request) error {
 			if len(via) >= client.config.RedirectLimit {
-				Debug(1, fmt.Sprintf("[HTTPCLIENT] maximum output-http-redirects[%d] reached!", client.config.RedirectLimit))
+				outputLogger.Warn().Msgf("[HTTPCLIENT] maximum output-http-redirects[%d] reached!", client.config.RedirectLimit)
 				return http.ErrUseLastResponse
 			}
 			lastReq := via[len(via)-1]
 			resp := req.Response
-			Debug(2, fmt.Sprintf("[HTTPCLIENT] HTTP redirects from %q to %q with %q", lastReq.Host, req.Host, resp.Status))
+			outputLogger.Info().Msgf("[HTTPCLIENT] HTTP redirects from %q to %q with %q", lastReq.Host, req.Host, resp.Status)
 			return nil
 		},
 	}
@@ -321,5 +346,6 @@ func (c *HTTPClient) Send(data []byte) ([]byte, error) {
 	if c.config.TrackResponses {
 		return httputil.DumpResponse(resp, true)
 	}
+	_ = resp.Body.Close()
 	return nil, nil
 }

pkg/http/output_http_test.go

@@ -1,4 +1,4 @@
-package main
+package http
 
 import (
 	"io/ioutil"
@@ -7,12 +7,19 @@ import (
 	_ "net/http/httputil"
 	"sync"
 	"testing"
+
+	"github.com/buger/goreplay/pkg/emitter"
+	"github.com/buger/goreplay/pkg/http_modifier"
+	"github.com/buger/goreplay/pkg/plugin"
+	"github.com/buger/goreplay/pkg/pro"
+	"github.com/buger/goreplay/pkg/proto"
+	"github.com/buger/goreplay/pkg/test"
 )
 
 func TestHTTPOutput(t *testing.T) {
 	wg := new(sync.WaitGroup)
 
-	input := NewTestInput()
+	input := test.NewTestInput()
 
 	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
 		if req.Header.Get("User-Agent") != "Gor" {
@@ -36,23 +43,25 @@ func TestHTTPOutput(t *testing.T) {
 	}))
 	defer server.Close()
 
-	headers := HTTPHeaders{httpHeader{"User-Agent", "Gor"}}
-	methods := HTTPMethods{[]byte("GET"), []byte("PUT"), []byte("POST")}
-	Settings.ModifierConfig = HTTPModifierConfig{Headers: headers, Methods: methods}
+	headers := http_modifier.HTTPHeaders{http_modifier.HTTPHeader{"User-Agent", "Gor"}}
+	methods := http_modifier.HTTPMethods{[]byte("GET"), []byte("PUT"), []byte("POST")}
+	modifierConfig := http_modifier.HTTPModifierConfig{Headers: headers, Methods: methods}
 
 	httpOutput := NewHTTPOutput(server.URL, &HTTPOutputConfig{TrackResponses: false})
-	output := NewTestOutput(func(*Message) {
+	output := test.NewTestOutput(func(*plugin.Message) {
 		wg.Done()
 	})
 
-	plugins := &InOutPlugins{
-		Inputs:  []PluginReader{input},
-		Outputs: []PluginWriter{httpOutput, output},
+	plugins := &plugin.InOutPlugins{
+		Inputs:  []plugin.Reader{input},
+		Outputs: []plugin.Writer{httpOutput, output},
 	}
 	plugins.All = append(plugins.All, input, output, httpOutput)
 
-	emitter := NewEmitter()
-	go emitter.Start(plugins, Settings.Middleware)
+	emitter := emitter.New(&emitter.Config{
+		ModifierConfig: modifierConfig,
+	})
+	go emitter.Start(plugins)
 
 	for i := 0; i < 10; i++ {
 		// 2 http-output, 2 - test output request
@@ -64,14 +73,12 @@ func TestHTTPOutput(t *testing.T) {
 
 	wg.Wait()
 	emitter.Close()
-
-	Settings.ModifierConfig = HTTPModifierConfig{}
 }
 
 func TestHTTPOutputKeepOriginalHost(t *testing.T) {
 	wg := new(sync.WaitGroup)
 
-	input := NewTestInput()
+	input := test.NewTestInput()
 
 	server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
 		if req.Host != "custom-host.com" {
@@ -82,26 +89,27 @@ func TestHTTPOutputKeepOriginalHost(t *testing.T) {
 	}))
 	defer server.Close()
 
-	headers := HTTPHeaders{httpHeader{"Host", "custom-host.com"}}
-	Settings.ModifierConfig = HTTPModifierConfig{Headers: headers}
+	headers := http_modifier.HTTPHeaders{http_modifier.HTTPHeader{"Host", "custom-host.com"}}
+	modifierConfig := http_modifier.HTTPModifierConfig{Headers: headers}
 
 	output := NewHTTPOutput(server.URL, &HTTPOutputConfig{OriginalHost: true, SkipVerify: true})
 
-	plugins := &InOutPlugins{
-		Inputs:  []PluginReader{input},
-		Outputs: []PluginWriter{output},
+	plugins := &plugin.InOutPlugins{
+		Inputs:  []plugin.Reader{input},
+		Outputs: []plugin.Writer{output},
 	}
 	plugins.All = append(plugins.All, input, output)
 
-	emitter := NewEmitter()
-	go emitter.Start(plugins, Settings.Middleware)
+	emitter := emitter.New(&emitter.Config{
+		ModifierConfig: modifierConfig,
+	})
+	go emitter.Start(plugins)
 
 	wg.Add(1)
 	input.EmitGET()
 
 	wg.Wait()
 	emitter.Close()
-	Settings.ModifierConfig = HTTPModifierConfig{}
 }
 
 func TestHTTPOutputSSL(t *testing.T) {
@@ -112,17 +120,17 @@ func TestHTTPOutputSSL(t *testing.T) {
 		wg.Done()
 	}))
 
-	input := NewTestInput()
+	input := test.NewTestInput()
 	output := NewHTTPOutput(server.URL, &HTTPOutputConfig{SkipVerify: true})
 
-	plugins := &InOutPlugins{
-		Inputs:  []PluginReader{input},
-		Outputs: []PluginWriter{output},
+	plugins := &plugin.InOutPlugins{
+		Inputs:  []plugin.Reader{input},
+		Outputs: []plugin.Writer{output},
 	}
 	plugins.All = append(plugins.All, input, output)
 
-	emitter := NewEmitter()
-	go emitter.Start(plugins, Settings.Middleware)
+	emitter := emitter.New()
+	go emitter.Start(plugins)
 
 	wg.Add(2)
 
@@ -134,49 +142,49 @@ func TestHTTPOutputSSL(t *testing.T) {
 }
 
 func TestHTTPOutputSessions(t *testing.T) {
+	pro.Enable()
+
 	wg := new(sync.WaitGroup)
 
-	input := NewTestInput()
-	input.skipHeader = true
+	input := test.NewTestInput()
+	input.SkipHeader = true
 
 	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
 		wg.Done()
 	}))
 	defer server.Close()
 
-	Settings.RecognizeTCPSessions = true
-	Settings.SplitOutput = true
 	output := NewHTTPOutput(server.URL, &HTTPOutputConfig{})
 
-	plugins := &InOutPlugins{
-		Inputs:  []PluginReader{input},
-		Outputs: []PluginWriter{output},
+	plugins := &plugin.InOutPlugins{
+		Inputs:  []plugin.Reader{input},
+		Outputs: []plugin.Writer{output},
 	}
 	plugins.All = append(plugins.All, input, output)
-	emitter := NewEmitter()
-	go emitter.Start(plugins, Settings.Middleware)
+	emitter := emitter.New(&emitter.Config{
+		RecognizeTCPSessions: true,
+		SplitOutput:          true,
+	})
+	go emitter.Start(plugins)
 
 	uuid1 := []byte("1234567890123456789a0000")
 	uuid2 := []byte("1234567890123456789d0000")
 
 	for i := 0; i < 10; i++ {
 		wg.Add(1) // OPTIONS should be ignored
-		copy(uuid1[20:], randByte(4))
+		copy(uuid1[20:], proto.RandByte(4))
 		input.EmitBytes([]byte("1 " + string(uuid1) + " 1\n" + "GET / HTTP/1.1\r\n\r\n"))
 	}
 
 	for i := 0; i < 10; i++ {
 		wg.Add(1) // OPTIONS should be ignored
-		copy(uuid2[20:], randByte(4))
+		copy(uuid2[20:], proto.RandByte(4))
 		input.EmitBytes([]byte("1 " + string(uuid2) + " 1\n" + "GET / HTTP/1.1\r\n\r\n"))
 	}
 
 	wg.Wait()
 
 	emitter.Close()
-
-	Settings.RecognizeTCPSessions = false
-	Settings.SplitOutput = false
 }
 
 func BenchmarkHTTPOutput(b *testing.B) {
@@ -187,17 +195,17 @@ func BenchmarkHTTPOutput(b *testing.B) {
 	}))
 	defer server.Close()
 
-	input := NewTestInput()
+	input := test.NewTestInput()
 	output := NewHTTPOutput(server.URL, &HTTPOutputConfig{WorkersMax: 1})
 
-	plugins := &InOutPlugins{
-		Inputs:  []PluginReader{input},
-		Outputs: []PluginWriter{output},
+	plugins := &plugin.InOutPlugins{
+		Inputs:  []plugin.Reader{input},
+		Outputs: []plugin.Writer{output},
 	}
 	plugins.All = append(plugins.All, input, output)
 
-	emitter := NewEmitter()
-	go emitter.Start(plugins, Settings.Middleware)
+	emitter := emitter.New()
+	go emitter.Start(plugins)
 
 	for i := 0; i < b.N; i++ {
 		wg.Add(1)
@@ -216,17 +224,17 @@ func BenchmarkHTTPOutputTLS(b *testing.B) {
 	}))
 	defer server.Close()
 
-	input := NewTestInput()
+	input := test.NewTestInput()
 	output := NewHTTPOutput(server.URL, &HTTPOutputConfig{SkipVerify: true, WorkersMax: 1})
 
-	plugins := &InOutPlugins{
-		Inputs:  []PluginReader{input},
-		Outputs: []PluginWriter{output},
+	plugins := &plugin.InOutPlugins{
+		Inputs:  []plugin.Reader{input},
+		Outputs: []plugin.Writer{output},
 	}
 	plugins.All = append(plugins.All, input, output)
 
-	emitter := NewEmitter()
-	go emitter.Start(plugins, Settings.Middleware)
+	emitter := emitter.New()
+	go emitter.Start(plugins)
 
 	for i := 0; i < b.N; i++ {
 		wg.Add(1)

pkg/http_modifier/http_modifier.go

@@ -1,4 +1,4 @@
-package main
+package http_modifier
 
 import (
 	"bytes"
@@ -6,7 +6,7 @@ import (
 	"hash/fnv"
 	"strings"
 
-	"github.com/buger/goreplay/proto"
+	proto "github.com/buger/goreplay/pkg/http_proto"
 )
 
 type HTTPModifier struct {

pkg/http_modifier/http_modifier_settings.go

@@ -1,4 +1,4 @@
-package main
+package http_modifier
 
 import (
 	"errors"
@@ -24,9 +24,7 @@ type HTTPModifierConfig struct {
 	Methods                HTTPMethods                `json:"http-allow-method"`
 }
 
-//
 // Handling of --http-allow-header, --http-disallow-header options
-//
 type headerFilter struct {
 	name   []byte
 	regexp *regexp.Regexp
@@ -56,9 +54,7 @@ func (h *HTTPHeaderFilters) Set(value string) error {
 	return nil
 }
 
-//
 // Handling of --http-basic-auth-filter option
-//
 type basicAuthFilter struct {
 	regexp *regexp.Regexp
 }
@@ -82,9 +78,7 @@ func (h *HTTPHeaderBasicAuthFilters) Set(value string) error {
 	return nil
 }
 
-//
 // Handling of --http-allow-header-hash and --http-allow-param-hash options
-//
 type hashFilter struct {
 	name    []byte
 	percent uint32
@@ -129,16 +123,14 @@ func (h *HTTPHashFilters) Set(value string) error {
 	return nil
 }
 
-//
 // Handling of --http-set-header option
-//
-type httpHeader struct {
+type HTTPHeader struct {
 	Name  string
 	Value string
 }
 
 // HTTPHeaders is a slice of headers that must appended
-type HTTPHeaders []httpHeader
+type HTTPHeaders []HTTPHeader
 
 func (h *HTTPHeaders) String() string {
 	return fmt.Sprint(*h)
@@ -151,7 +143,7 @@ func (h *HTTPHeaders) Set(value string) error {
 		return errors.New("Expected `Key: Value`")
 	}
 
-	header := httpHeader{
+	header := HTTPHeader{
 		strings.TrimSpace(v[0]),
 		strings.TrimSpace(v[1]),
 	}
@@ -160,9 +152,7 @@ func (h *HTTPHeaders) Set(value string) error {
 	return nil
 }
 
-//
 // Handling of --http-set-param option
-//
 type httpParam struct {
 	Name  []byte
 	Value []byte
@@ -208,9 +198,7 @@ func (h *HTTPMethods) Set(value string) error {
 	return nil
 }
 
-//
 // Handling of --http-rewrite-url option
-//
 type urlRewrite struct {
 	src    *regexp.Regexp
 	target []byte
@@ -237,9 +225,7 @@ func (r *URLRewriteMap) Set(value string) error {
 	return nil
 }
 
-//
 // Handling of --http-rewrite-header option
-//
 type headerRewrite struct {
 	header []byte
 	src    *regexp.Regexp
@@ -275,9 +261,7 @@ func (r *HeaderRewriteMap) Set(value string) error {
 	return nil
 }
 
-//
 // Handling of --http-allow-url option
-//
 type urlRegexp struct {
 	regexp *regexp.Regexp
 }

pkg/http_modifier/http_modifier_settings_test.go

@@ -1,4 +1,4 @@
-package main
+package http_modifier
 
 import (
 	"testing"

pkg/http_modifier/http_modifier_test.go

@@ -1,10 +1,10 @@
-package main
+package http_modifier
 
 import (
 	"bytes"
 	"testing"
 
-	"github.com/buger/goreplay/proto"
+	proto "github.com/buger/goreplay/pkg/http_proto"
 )
 
 func TestHTTPModifierWithoutConfig(t *testing.T) {

pkg/http_proto/fuzz.go

@@ -1,6 +1,6 @@
-// +build gofuzz
+//go:build gofuzz
 
-package proto
+package http_proto
 
 func Fuzz(data []byte) int {
 

pkg/http_proto/proto.go

@@ -14,16 +14,18 @@ Example of HTTP payload for future references, new line symbols escaped:
 	\r\n
 	\r\n
 */
-package proto
+package http_proto
 
 import (
 	"bufio"
 	"bytes"
+
+	"github.com/buger/goreplay/internal/byteutils"
+
+	_ "fmt"
 	"net/http"
 	"net/textproto"
 	"strings"
-
-	"github.com/buger/goreplay/byteutils"
 )
 
 // CRLF In HTTP newline defined by 2 bytes (for both windows and *nix support)
@@ -76,9 +78,12 @@ func header(payload []byte, name []byte) (value []byte, headerStart, headerEnd,
 		headerEnd += headerStart
 		colonIndex = bytes.IndexByte(payload[headerStart:headerEnd], ':')
 		if colonIndex == -1 {
-			break
+			// Malformed header, skip, most likely packet with partial headers
+			headerStart = headerEnd + 1
+			continue
 		}
 		colonIndex += headerStart
+
 		if bytes.EqualFold(payload[headerStart:colonIndex], name) {
 			valueStart = colonIndex + 1
 			valueEnd = headerEnd - 2
@@ -198,7 +203,7 @@ func Body(payload []byte) []byte {
 	return payload[pos:]
 }
 
-// Path takes payload and retuns request path: Split(firstLine, ' ')[1]
+// Path takes payload and returns request path: Split(firstLine, ' ')[1]
 func Path(payload []byte) []byte {
 	if !HasRequestTitle(payload) {
 		return nil
@@ -447,34 +452,45 @@ type ProtocolStateSetter interface {
 	ProtocolState() interface{}
 }
 
-type httpProto struct {
-	body         int // body index
-	headerStart  int
-	headerParsed bool // we checked necessary headers
-	hasFullBody  bool // all chunks has been parsed
-	isChunked    bool // Transfer-Encoding: chunked
-	bodyLen      int  // Content-Length's value
-	hasTrailer   bool // Trailer header?
+type HTTPState struct {
+	Body           int // body index
+	HeaderStart    int
+	HeaderEnd      int
+	HeaderParsed   bool // we checked necessary headers
+	HasFullPayload bool // all chunks has been parsed
+	IsChunked      bool // Transfer-Encoding: chunked
+	BodyLen        int  // Content-Length's value
+	HasTrailer     bool // Trailer header?
+	Continue100    bool
 }
 
 // HasFullPayload checks if this message has full or valid payloads and returns true.
 // Message param is optional but recommended on cases where 'data' is storing
 // partial-to-full stream of bytes(packets).
 func HasFullPayload(m ProtocolStateSetter, payloads ...[]byte) bool {
-	var state *httpProto
+	var state *HTTPState
 	if m != nil {
-		state, _ = m.ProtocolState().(*httpProto)
+		state, _ = m.ProtocolState().(*HTTPState)
 	}
 	if state == nil {
-		state = new(httpProto)
+		state = new(HTTPState)
 		if m != nil {
 			m.SetProtocolState(state)
 		}
 	}
-	if state.headerStart < 1 {
+
+	// Http Packets can only start with a few things, check if this is one of them
+	if len(payloads) == 0 {
+		return false
+	}
+	if !HasRequestTitle(payloads[0]) && !HasResponseTitle(payloads[0]) {
+		return false
+	}
+
+	if state.HeaderStart < 1 {
 		for _, data := range payloads {
-			state.headerStart = MIMEHeadersStartPos(data)
-			if state.headerStart < 0 {
+			state.HeaderStart = MIMEHeadersStartPos(data)
+			if state.HeaderStart < 0 {
 				return false
 			} else {
 				break
@@ -482,7 +498,7 @@ func HasFullPayload(m ProtocolStateSetter, payloads ...[]byte) bool {
 		}
 	}
 
-	if state.body < 1 {
+	if state.Body < 1 || state.HeaderEnd < 1 {
 		var pos int
 		for _, data := range payloads {
 			endPos := MIMEHeadersEndPos(data)
@@ -490,32 +506,42 @@ func HasFullPayload(m ProtocolStateSetter, payloads ...[]byte) bool {
 				pos += len(data)
 			} else {
 				pos += endPos
+				state.HeaderEnd = pos
 			}
 
 			if endPos > 0 {
-				state.body = pos
+				state.Body = pos
 				break
 			}
 		}
 	}
-	if !state.headerParsed {
+
+	if state.HeaderEnd < 1 {
+		return false
+	}
+
+	if !state.HeaderParsed {
 		var pos int
 		for _, data := range payloads {
 			chunked := Header(data, []byte("Transfer-Encoding"))
 
 			if len(chunked) > 0 && bytes.Index(data, []byte("chunked")) > 0 {
-				state.isChunked = true
+				state.IsChunked = true
 				// trailers are generally not allowed in non-chunks body
-				state.hasTrailer = len(Header(data, []byte("Trailer"))) > 0
+				state.HasTrailer = len(Header(data, []byte("Trailer"))) > 0
 			} else {
 				contentLen := Header(data, []byte("Content-Length"))
-				state.bodyLen, _ = atoI(contentLen, 10)
+				state.BodyLen, _ = atoI(contentLen, 10)
 			}
 
 			pos += len(data)
 
-			if state.bodyLen > 0 || pos >= state.body {
-				state.headerParsed = true
+			if string(Header(data, []byte("Expect"))) == "100-continue" {
+				state.Continue100 = true
+			}
+
+			if state.BodyLen > 0 || pos >= state.Body {
+				state.HeaderParsed = true
 				break
 			}
 		}
@@ -525,22 +551,22 @@ func HasFullPayload(m ProtocolStateSetter, payloads ...[]byte) bool {
 	for _, data := range payloads {
 		bodyLen += len(data)
 	}
-	bodyLen -= state.body
+	bodyLen -= state.Body
 
-	if state.isChunked {
+	if state.IsChunked {
 		// check chunks
 		if bodyLen < 1 {
 			return false
 		}
 
 		// check trailer headers
-		if state.hasTrailer {
+		if state.HasTrailer {
 			if bytes.HasSuffix(payloads[len(payloads)-1], []byte("\r\n\r\n")) {
 				return true
 			}
 		} else {
 			if bytes.HasSuffix(payloads[len(payloads)-1], []byte("0\r\n\r\n")) {
-				state.hasFullBody = true
+				state.HasFullPayload = true
 				return true
 			}
 		}
@@ -549,7 +575,7 @@ func HasFullPayload(m ProtocolStateSetter, payloads ...[]byte) bool {
 	}
 
 	// check for content-length header
-	return state.bodyLen == bodyLen
+	return state.BodyLen == bodyLen
 }
 
 // this works with positive integers

pkg/http_proto/proto_test.go

@@ -1,4 +1,4 @@
-package proto
+package http_proto
 
 import (
 	"bytes"
@@ -56,6 +56,12 @@ func TestHeader(t *testing.T) {
 	if val = Header(payload, []byte("host")); !bytes.Equal(val, []byte("www.w3.org")) {
 		t.Error("Should find lower case 1 word header")
 	}
+
+	payload = []byte("GT\r\nContent-Length: 10\r\n\r\n")
+
+	if val = Header(payload, []byte("Content-Length")); !bytes.Equal(val, []byte("10")) {
+		t.Error("Should find in partial payload")
+	}
 }
 
 func TestMIMEHeadersEndPos(t *testing.T) {
@@ -479,6 +485,14 @@ func TestHasFullPayload(t *testing.T) {
 	if got != expected {
 		t.Errorf("expected %v to equal %v", got, expected)
 	}
+
+	// check with trailer and no header
+	m = "Content-Type: text/plain\r\nContent-Length: 23\r\n\r\nMozillaDeveloperNetwork"
+	got = HasFullPayload(nil, []byte(m))
+	expected = false
+	if got != expected {
+		t.Errorf("expected %v to equal %v", got, expected)
+	}
 }
 
 func BenchmarkHasFullPayload(b *testing.B) {

pkg/kafka/input_kafka.go

@@ -1,15 +1,18 @@
-package main
+package kafka
 
 import (
 	"encoding/json"
-	"log"
 	"strings"
 
 	"github.com/Shopify/sarama"
 	"github.com/Shopify/sarama/mocks"
+	"github.com/buger/goreplay/pkg/plugin"
+	"github.com/buger/goreplay/pkg/proto"
+
+	"github.com/rs/zerolog/log"
 )
 
-// KafkaInput is used for recieving Kafka messages and
+// KafkaInput is used for receiving Kafka messages and
 // transforming them into HTTP payloads.
 type KafkaInput struct {
 	config    *InputKafkaConfig
@@ -19,8 +22,8 @@ type KafkaInput struct {
 }
 
 // NewKafkaInput creates instance of kafka consumer client with TLS config
-func NewKafkaInput(address string, config *InputKafkaConfig, tlsConfig *KafkaTLSConfig) *KafkaInput {
-	c := NewKafkaConfig(tlsConfig)
+func NewKafkaInput(_ string, config *InputKafkaConfig, tlsConfig *KafkaTLSConfig) *KafkaInput {
+	c := NewKafkaConfig(&config.SASLConfig, tlsConfig)
 
 	var con sarama.Consumer
 
@@ -31,13 +34,13 @@ func NewKafkaInput(address string, config *InputKafkaConfig, tlsConfig *KafkaTLS
 		con, err = sarama.NewConsumer(strings.Split(config.Host, ","), c)
 
 		if err != nil {
-			log.Fatalln("Failed to start Sarama(Kafka) consumer:", err)
+			log.Fatal().Err(err).Msg("Failed to start Sarama(Kafka) consumer")
 		}
 	}
 
 	partitions, err := con.Partitions(config.Topic)
 	if err != nil {
-		log.Fatalln("Failed to collect Sarama(Kafka) partitions:", err)
+		log.Fatal().Err(err).Msg("Failed to collect Sarama(Kafka) partitions")
 	}
 
 	i := &KafkaInput{
@@ -50,7 +53,7 @@ func NewKafkaInput(address string, config *InputKafkaConfig, tlsConfig *KafkaTLS
 	for index, partition := range partitions {
 		consumer, err := con.ConsumePartition(config.Topic, partition, sarama.OffsetNewest)
 		if err != nil {
-			log.Fatalln("Failed to start Sarama(Kafka) partition consumer:", err)
+			log.Fatal().Err(err).Msg("Failed to start Sarama(Kafka) partition consumer")
 		}
 
 		go func(consumer sarama.PartitionConsumer) {
@@ -72,17 +75,17 @@ func NewKafkaInput(address string, config *InputKafkaConfig, tlsConfig *KafkaTLS
 // ErrorHandler should receive errors
 func (i *KafkaInput) ErrorHandler(consumer sarama.PartitionConsumer) {
 	for err := range consumer.Errors() {
-		Debug(1, "Failed to read access log entry:", err)
+		log.Error().Err(err).Msg("Failed to read access log entry")
 	}
 }
 
 // PluginRead a reads message from this plugin
-func (i *KafkaInput) PluginRead() (*Message, error) {
+func (i *KafkaInput) PluginRead() (*plugin.Message, error) {
 	var message *sarama.ConsumerMessage
-	var msg Message
+	var msg plugin.Message
 	select {
 	case <-i.quit:
-		return nil, ErrorStopped
+		return nil, plugin.ErrorStopped
 	case message = <-i.messages:
 	}
 
@@ -95,14 +98,14 @@ func (i *KafkaInput) PluginRead() (*Message, error) {
 		var err error
 		msg.Data, err = kafkaMessage.Dump()
 		if err != nil {
-			Debug(1, "[INPUT-KAFKA] failed to decode access log entry:", err)
+			log.Error().Err(err).Msg("Failed to decode access log entry")
 			return nil, err
 		}
 	}
 
 	// does it have meta
-	if isOriginPayload(msg.Data) {
-		msg.Meta, msg.Data = payloadMetaWithBody(msg.Data)
+	if proto.IsOriginPayload(msg.Data) {
+		msg.Meta, msg.Data = proto.PayloadMetaWithBody(msg.Data)
 	}
 
 	return &msg, nil

pkg/kafka/input_kafka_test.go

@@ -1,4 +1,4 @@
-package main
+package kafka
 
 import (
 	"testing"

pkg/kafka/kafka.go

@@ -1,7 +1,9 @@
-package main
+package kafka
 
 import (
 	"bytes"
+	"crypto/sha256"
+	"crypto/sha512"
 	"crypto/tls"
 	"crypto/x509"
 	"errors"
@@ -9,27 +11,37 @@ import (
 	"io/ioutil"
 	"log"
 
+	"github.com/buger/goreplay/pkg/http_proto"
+
 	"github.com/Shopify/sarama"
-	"github.com/buger/goreplay/proto"
+	"github.com/xdg-go/scram"
 )
 
+// SASLKafkaConfig SASL configuration
+type SASLKafkaConfig struct {
+	UseSASL   bool   `json:"input-kafka-use-sasl"`
+	Mechanism string `json:"input-kafka-mechanism"`
+	Username  string `json:"input-kafka-username"`
+	Password  string `json:"input-kafka-password"`
+}
+
 // InputKafkaConfig should contains required information to
 // build producers.
 type InputKafkaConfig struct {
-	producer sarama.AsyncProducer
-	consumer sarama.Consumer
-	Host     string `json:"input-kafka-host"`
-	Topic    string `json:"input-kafka-topic"`
-	UseJSON  bool   `json:"input-kafka-json-format"`
+	consumer   sarama.Consumer
+	Host       string `json:"input-kafka-host"`
+	Topic      string `json:"input-kafka-topic"`
+	UseJSON    bool   `json:"input-kafka-json-format"`
+	SASLConfig SASLKafkaConfig
 }
 
 // OutputKafkaConfig is the representation of kfka output configuration
 type OutputKafkaConfig struct {
-	producer sarama.AsyncProducer
-	consumer sarama.Consumer
-	Host     string `json:"output-kafka-host"`
-	Topic    string `json:"output-kafka-topic"`
-	UseJSON  bool   `json:"output-kafka-json-format"`
+	producer   sarama.AsyncProducer
+	Host       string `json:"output-kafka-host"`
+	Topic      string `json:"output-kafka-topic"`
+	UseJSON    bool   `json:"output-kafka-json-format"`
+	SASLConfig SASLKafkaConfig
 }
 
 // KafkaTLSConfig should contains TLS certificates for connecting to secured Kafka clusters
@@ -51,7 +63,6 @@ type KafkaMessage struct {
 	ReqHeaders map[string]string `json:"Req_Headers,omitempty"`
 }
 
-
 // NewTLSConfig loads TLS certificates
 func NewTLSConfig(clientCertFile, clientKeyFile, caCertFile string) (*tls.Config, error) {
 	tlsConfig := tls.Config{}
@@ -63,7 +74,7 @@ func NewTLSConfig(clientCertFile, clientKeyFile, caCertFile string) (*tls.Config
 		return &tlsConfig, errors.New("missing TLS client certificate in kafka")
 	}
 	// Load client cert
-	if  (clientCertFile != "") && (clientKeyFile != "") {
+	if (clientCertFile != "") && (clientKeyFile != "") {
 		cert, err := tls.LoadX509KeyPair(clientCertFile, clientKeyFile)
 		if err != nil {
 			return &tlsConfig, err
@@ -84,7 +95,7 @@ func NewTLSConfig(clientCertFile, clientKeyFile, caCertFile string) (*tls.Config
 }
 
 // NewKafkaConfig returns Kafka config with or without TLS
-func NewKafkaConfig(tlsConfig *KafkaTLSConfig) *sarama.Config {
+func NewKafkaConfig(saslConfig *SASLKafkaConfig, tlsConfig *KafkaTLSConfig) *sarama.Config {
 	config := sarama.NewConfig()
 	// Configuration options go here
 	if tlsConfig != nil && (tlsConfig.ClientCert != "" || tlsConfig.CACert != "") {
@@ -95,6 +106,18 @@ func NewKafkaConfig(tlsConfig *KafkaTLSConfig) *sarama.Config {
 		}
 		config.Net.TLS.Config = tlsConfig
 	}
+	if saslConfig.UseSASL {
+		mechanism := sarama.SASLMechanism(saslConfig.Mechanism)
+		config.Net.SASL.Enable = saslConfig.UseSASL
+		config.Net.SASL.Mechanism = mechanism
+		config.Net.SASL.User = saslConfig.Username
+		config.Net.SASL.Password = saslConfig.Password
+		if mechanism == sarama.SASLTypeSCRAMSHA256 {
+			config.Net.SASL.SCRAMClientGeneratorFunc = func() sarama.SCRAMClient { return &XDGSCRAMClient{HashGeneratorFcn: SHA256} }
+		} else if mechanism == sarama.SASLTypeSCRAMSHA512 {
+			config.Net.SASL.SCRAMClientGeneratorFunc = func() sarama.SCRAMClient { return &XDGSCRAMClient{HashGeneratorFcn: SHA512} }
+		}
+	}
 	return config
 }
 
@@ -105,14 +128,49 @@ func (m KafkaMessage) Dump() ([]byte, error) {
 
 	b.WriteString(fmt.Sprintf("%s %s %s\n", m.ReqType, m.ReqID, m.ReqTs))
 	b.WriteString(fmt.Sprintf("%s %s HTTP/1.1", m.ReqMethod, m.ReqURL))
-	b.Write(proto.CRLF)
+	b.Write(http_proto.CRLF)
 	for key, value := range m.ReqHeaders {
 		b.WriteString(fmt.Sprintf("%s: %s", key, value))
-		b.Write(proto.CRLF)
+		b.Write(http_proto.CRLF)
 	}
 
-	b.Write(proto.CRLF)
+	b.Write(http_proto.CRLF)
 	b.WriteString(m.ReqBody)
 
 	return b.Bytes(), nil
 }
+
+var (
+	// SHA256 SASLMechanism
+	SHA256 scram.HashGeneratorFcn = sha256.New
+	// SHA512 SASLMechanism
+	SHA512 scram.HashGeneratorFcn = sha512.New
+)
+
+// XDGSCRAMClient for SASL-Protocol
+type XDGSCRAMClient struct {
+	*scram.Client
+	*scram.ClientConversation
+	scram.HashGeneratorFcn
+}
+
+// Begin of XDGSCRAMClient
+func (x *XDGSCRAMClient) Begin(userName, password, authzID string) (err error) {
+	x.Client, err = x.HashGeneratorFcn.NewClient(userName, password, authzID)
+	if err != nil {
+		return err
+	}
+	x.ClientConversation = x.Client.NewConversation()
+	return nil
+}
+
+// Step of XDGSCRAMClient
+func (x *XDGSCRAMClient) Step(challenge string) (response string, err error) {
+	response, err = x.ClientConversation.Step(challenge)
+	return
+}
+
+// Done of XDGSCRAMClient
+func (x *XDGSCRAMClient) Done() bool {
+	return x.ClientConversation.Done()
+}

pkg/kafka/output_kafka.go

@@ -1,16 +1,19 @@
-package main
+package kafka
 
 import (
 	"encoding/json"
-	"log"
 	"strings"
 	"time"
 
-	"github.com/buger/goreplay/byteutils"
-	"github.com/buger/goreplay/proto"
+	"github.com/buger/goreplay/internal/byteutils"
+	"github.com/buger/goreplay/pkg/http_proto"
+	"github.com/buger/goreplay/pkg/plugin"
+	"github.com/buger/goreplay/pkg/proto"
 
 	"github.com/Shopify/sarama"
 	"github.com/Shopify/sarama/mocks"
+
+	"github.com/rs/zerolog/log"
 )
 
 // KafkaOutput is used for sending payloads to kafka in JSON format.
@@ -23,8 +26,8 @@ type KafkaOutput struct {
 const KafkaOutputFrequency = 500
 
 // NewKafkaOutput creates instance of kafka producer client  with TLS config
-func NewKafkaOutput(address string, config *OutputKafkaConfig, tlsConfig *KafkaTLSConfig) PluginWriter {
-	c := NewKafkaConfig(tlsConfig)
+func NewKafkaOutput(_ string, config *OutputKafkaConfig, tlsConfig *KafkaTLSConfig) plugin.Writer {
+	c := NewKafkaConfig(&config.SASLConfig, tlsConfig)
 
 	var producer sarama.AsyncProducer
 
@@ -40,7 +43,7 @@ func NewKafkaOutput(address string, config *OutputKafkaConfig, tlsConfig *KafkaT
 		var err error
 		producer, err = sarama.NewAsyncProducer(brokerList, c)
 		if err != nil {
-			log.Fatalln("Failed to start Sarama(Kafka) producer:", err)
+			log.Fatal().Err(err).Msg("Failed to start Sarama(Kafka) producer")
 		}
 	}
 
@@ -58,33 +61,33 @@ func NewKafkaOutput(address string, config *OutputKafkaConfig, tlsConfig *KafkaT
 // ErrorHandler should receive errors
 func (o *KafkaOutput) ErrorHandler() {
 	for err := range o.producer.Errors() {
-		Debug(1, "Failed to write access log entry:", err)
+		log.Error().Err(err).Msg("Failed to write access log entry")
 	}
 }
 
 // PluginWrite writes a message to this plugin
-func (o *KafkaOutput) PluginWrite(msg *Message) (n int, err error) {
+func (o *KafkaOutput) PluginWrite(msg *plugin.Message) (n int, err error) {
 	var message sarama.StringEncoder
 
 	if !o.config.UseJSON {
 		message = sarama.StringEncoder(byteutils.SliceToString(msg.Meta) + byteutils.SliceToString(msg.Data))
 	} else {
-		mimeHeader := proto.ParseHeaders(msg.Data)
+		mimeHeader := http_proto.ParseHeaders(msg.Data)
 		header := make(map[string]string)
 		for k, v := range mimeHeader {
 			header[k] = strings.Join(v, ", ")
 		}
 
-		meta := payloadMeta(msg.Meta)
+		meta := proto.PayloadMeta(msg.Meta)
 		req := msg.Data
 
 		kafkaMessage := KafkaMessage{
-			ReqURL:     byteutils.SliceToString(proto.Path(req)),
+			ReqURL:     byteutils.SliceToString(http_proto.Path(req)),
 			ReqType:    byteutils.SliceToString(meta[0]),
 			ReqID:      byteutils.SliceToString(meta[1]),
 			ReqTs:      byteutils.SliceToString(meta[2]),
-			ReqMethod:  byteutils.SliceToString(proto.Method(req)),
-			ReqBody:    byteutils.SliceToString(proto.Body(req)),
+			ReqMethod:  byteutils.SliceToString(http_proto.Method(req)),
+			ReqBody:    byteutils.SliceToString(http_proto.Body(req)),
 			ReqHeaders: header,
 		}
 		jsonMessage, _ := json.Marshal(&kafkaMessage)

pkg/kafka/output_kafka_test.go

@@ -1,10 +1,11 @@
-package main
+package kafka
 
 import (
 	"testing"
 
 	"github.com/Shopify/sarama"
 	"github.com/Shopify/sarama/mocks"
+	"github.com/buger/goreplay/pkg/plugin"
 )
 
 func TestOutputKafkaRAW(t *testing.T) {
@@ -19,7 +20,7 @@ func TestOutputKafkaRAW(t *testing.T) {
 		UseJSON:  false,
 	}, nil)
 
-	output.PluginWrite(&Message{Meta: []byte("1 2 3\n"), Data: []byte("GET / HTTP1.1\r\nHeader: 1\r\n\r\n")})
+	output.PluginWrite(&plugin.Message{Meta: []byte("1 2 3\n"), Data: []byte("GET / HTTP1.1\r\nHeader: 1\r\n\r\n")})
 
 	resp := <-producer.Successes()
 
@@ -42,7 +43,7 @@ func TestOutputKafkaJSON(t *testing.T) {
 		UseJSON:  true,
 	}, nil)
 
-	output.PluginWrite(&Message{Meta: []byte("1 2 3\n"), Data: []byte("GET / HTTP1.1\r\nHeader: 1\r\n\r\n")})
+	output.PluginWrite(&plugin.Message{Meta: []byte("1 2 3\n"), Data: []byte("GET / HTTP1.1\r\nHeader: 1\r\n\r\n")})
 
 	resp := <-producer.Successes()
 

pkg/middleware/middleware.go

@@ -1,4 +1,4 @@
-package main
+package middleware
 
 import (
 	"bufio"
@@ -11,30 +11,47 @@ import (
 	"strings"
 	"sync"
 	"syscall"
+
+	"github.com/buger/goreplay/pkg/plugin"
+	"github.com/buger/goreplay/pkg/prettify"
+	"github.com/buger/goreplay/pkg/proto"
+
+	"github.com/rs/zerolog"
+	"github.com/rs/zerolog/log"
 )
 
 // Middleware represents a middleware object
 type Middleware struct {
-	command       string
-	data          chan *Message
+	Command       string
+	Data          chan *plugin.Message
 	Stdin         io.Writer
 	Stdout        io.Reader
-	commandCancel context.CancelFunc
-	stop          chan bool // Channel used only to indicate goroutine should shutdown
+	CommandCancel context.CancelFunc
+	Stop          chan bool // Channel used only to indicate goroutine should shutdown
 	closed        bool
+	config        *MiddlewareConfig
 	mu            sync.RWMutex
 }
 
+// MiddlewareConfig represents a middleware configuration
+type MiddlewareConfig struct {
+	PrettifyHTTP bool
+}
+
 // NewMiddleware returns new middleware
-func NewMiddleware(command string) *Middleware {
+func NewMiddleware(command string, config *MiddlewareConfig) *Middleware {
 	m := new(Middleware)
-	m.command = command
-	m.data = make(chan *Message, 1000)
-	m.stop = make(chan bool)
+	m.Command = command
+	m.Data = make(chan *plugin.Message, 1000)
+	m.Stop = make(chan bool)
+	m.config = config
+	if m.config == nil {
+		m.config = &MiddlewareConfig{}
+	}
 
 	commands := strings.Split(command, " ")
 	ctx, cancl := context.WithCancel(context.Background())
-	m.commandCancel = cancl
+	m.CommandCancel = cancl
 	cmd := exec.CommandContext(ctx, commands[0], commands[1:]...)
 
 	m.Stdout, _ = cmd.StdoutPipe()
@@ -42,7 +59,7 @@ func NewMiddleware(command string) *Middleware {
 
 	cmd.Stderr = os.Stderr
 
-	go m.read(m.Stdout)
+	go m.Read(m.Stdout)
 
 	go func() {
 		defer m.Close()
@@ -57,7 +74,8 @@ func NewMiddleware(command string) *Middleware {
 					return
 				}
 			}
-			Debug(0, fmt.Sprintf("[MIDDLEWARE] command[%q] error: %q", command, err.Error()))
+
+			log.Error().Err(err).Msgf("[MIDDLEWARE] command[%q] error", command)
 		}
 	}()
 
@@ -65,12 +83,15 @@ func NewMiddleware(command string) *Middleware {
 }
 
 // ReadFrom start a worker to read from this plugin
-func (m *Middleware) ReadFrom(plugin PluginReader) {
-	Debug(2, fmt.Sprintf("[MIDDLEWARE] command[%q] Starting reading from %q", m.command, plugin))
+func (m *Middleware) ReadFrom(plugin plugin.Reader) {
+	if log.Logger.GetLevel() == zerolog.DebugLevel {
+		log.Debug().Msgf("command[%q] Starting reading from %q", m.Command, plugin)
+	}
+
 	go m.copy(m.Stdin, plugin)
 }
 
-func (m *Middleware) copy(to io.Writer, from PluginReader) {
+func (m *Middleware) copy(to io.Writer, from plugin.Reader) {
 	var buf, dst []byte
 
 	for {
@@ -82,8 +103,8 @@ func (m *Middleware) copy(to io.Writer, from PluginReader) {
 			continue
 		}
 		buf = msg.Data
-		if Settings.PrettifyHTTP {
-			buf = prettifyHTTP(msg.Data)
+		if m.config != nil && m.config.PrettifyHTTP {
+			buf = prettify.PrettifyHTTP(msg.Data)
 		}
 		dstLen := (len(buf)+len(msg.Meta))*2 + 1
 		// if enough space was previously allocated use it instead
@@ -104,7 +125,8 @@ func (m *Middleware) copy(to io.Writer, from PluginReader) {
 	}
 }
 
-func (m *Middleware) read(from io.Reader) {
+// Read reads from this plugin
+func (m *Middleware) Read(from io.Reader) {
 	reader := bufio.NewReader(from)
 	var line []byte
 	var e error
@@ -117,33 +139,33 @@ func (m *Middleware) read(from io.Reader) {
 		}
 		buf := make([]byte, (len(line)-1)/2)
 		if _, err := hex.Decode(buf, line[:len(line)-1]); err != nil {
-			Debug(0, fmt.Sprintf("[MIDDLEWARE] command[%q] failed to decode err: %q", m.command, err))
+			log.Error().Err(err).Msgf("[MIDDLEWARE] command[%q] failed to decode", m.Command)
 			continue
 		}
-		var msg Message
-		msg.Meta, msg.Data = payloadMetaWithBody(buf)
+		var msg plugin.Message
+		msg.Meta, msg.Data = proto.PayloadMetaWithBody(buf)
 		select {
-		case <-m.stop:
+		case <-m.Stop:
 			return
-		case m.data <- &msg:
+		case m.Data <- &msg:
 		}
 	}
 
 }
 
 // PluginRead reads message from this plugin
-func (m *Middleware) PluginRead() (msg *Message, err error) {
+func (m *Middleware) PluginRead() (msg *plugin.Message, err error) {
 	select {
-	case <-m.stop:
-		return nil, ErrorStopped
-	case msg = <-m.data:
+	case <-m.Stop:
+		return nil, plugin.ErrorStopped
+	case msg = <-m.Data:
 	}
 
 	return
 }
 
 func (m *Middleware) String() string {
-	return fmt.Sprintf("Modifying traffic using %q command", m.command)
+	return fmt.Sprintf("Modifying traffic using %q command", m.Command)
 }
 
 func (m *Middleware) isClosed() bool {
@@ -159,8 +181,8 @@ func (m *Middleware) Close() error {
 	}
 	m.mu.Lock()
 	defer m.mu.Unlock()
-	m.commandCancel()
-	close(m.stop)
+	m.CommandCancel()
+	close(m.Stop)
 	m.closed = true
 	return nil
 }

pkg/middleware/middleware_test.go

@@ -0,0 +1,65 @@
+package middleware
+
+//func TestTokenMiddleware(t *testing.T) {
+//	quit := make(chan struct{})
+//	in := test.NewTestInput()
+//	in.skipHeader = true
+//	cmd, cancl := initCmd(tokenModifier, withDebug)
+//	midd := initMiddleware(cmd, cancl, in, func(err error) {})
+//	req := []byte("1 932079936fa4306fc308d67588178d17d823647c 1439818823587396305 200\nGET /token HTTP/1.1\r\nHost: example.org\r\n\r\n")
+//	res := []byte("2 932079936fa4306fc308d67588178d17d823647c 1439818823587396305 200\nHTTP/1.1 200 OK\r\nContent-Length: 10\r\nContent-Type: text/plain; charset=utf-8\r\n\r\n17d823647c")
+//	rep := []byte("3 932079936fa4306fc308d67588178d17d823647c 1439818823587396305 200\nHTTP/1.1 200 OK\r\nContent-Length: 15\r\nContent-Type: text/plain; charset=utf-8\r\n\r\n932079936fa4306")
+//	count := uint32(0)
+//	out := test.NewTestOutput(func(msg *Message) {
+//		if msg.Meta[0] == '1' && !bytes.Equal(payloadID(msg.Meta), payloadID(req)) {
+//			token, _, _ := proto.PathParam(msg.Data, []byte("token"))
+//			if !bytes.Equal(token, proto.Body(rep)) {
+//				t.Errorf("expected the token %s to be equal to the replayed response's token %s", token, proto.Body(rep))
+//			}
+//		}
+//		atomic.AddUint32(&count, 1)
+//		if atomic.LoadUint32(&count) == 2 {
+//			quit <- struct{}{}
+//		}
+//	})
+//	pl := &InOutPlugins{}
+//	pl.Inputs = []PluginReader{midd, in}
+//	pl.Outputs = []PluginWriter{out}
+//	pl.All = []interface{}{midd, out, in}
+//	e := NewEmitter()
+//	go e.Start(pl, "")
+//	in.EmitBytes(req) // emit original request
+//	in.EmitBytes(res) // emit its response
+//	in.EmitBytes(rep) // emit replayed response
+//	// emit the request which should have modified token
+//	token := []byte("1 8e091765ae902fef8a2b7d9dd96 14398188235873 100\nGET /?token=17d823647c HTTP/1.1\r\nHost: example.org\r\n\r\n")
+//	in.EmitBytes(token)
+//	<-quit
+//	midd.Close()
+//}
+
+//func TestMiddlewareWithPrettify(t *testing.T) {
+//	Settings.PrettifyHTTP = true
+//	quit := make(chan struct{})
+//	in := test.NewTestInput()
+//	cmd, cancl := initCmd(echoSh, withDebug)
+//	midd := initMiddleware(cmd, cancl, in, func(err error) {})
+//	var b1 = []byte("POST / HTTP/1.1\r\nHost: example.org\r\nTransfer-Encoding: chunked\r\n\r\n4\r\nWiki\r\n5\r\npedia\r\nE\r\n in\r\n\r\nchunks.\r\n0\r\n\r\n")
+//	var b2 = []byte("POST / HTTP/1.1\r\nHost: example.org\r\nContent-Length: 25\r\n\r\nWikipedia in\r\n\r\nchunks.")
+//	out := test.NewTestOutput(func(msg *Message) {
+//		if !bytes.Equal(proto.Body(b2), proto.Body(msg.Data)) {
+//			t.Errorf("expected %q body to equal %q body", b2, msg.Data)
+//		}
+//		quit <- struct{}{}
+//	})
+//	pl := &InOutPlugins{}
+//	pl.Inputs = []PluginReader{midd, in}
+//	pl.Outputs = []PluginWriter{out}
+//	pl.All = []interface{}{midd, out, in}
+//	e := NewEmitter()
+//	go e.Start(pl, "")
+//	in.EmitBytes(b1)
+//	<-quit
+//	midd.Close()
+//	Settings.PrettifyHTTP = false
+//}

pkg/null/output_null.go

@@ -1,4 +1,6 @@
-package main
+package null
+
+import "github.com/buger/goreplay/pkg/plugin"
 
 // NullOutput used for debugging, prints nothing
 type NullOutput struct {
@@ -10,7 +12,7 @@ func NewNullOutput() (o *NullOutput) {
 }
 
 // PluginWrite writes message to this plugin
-func (o *NullOutput) PluginWrite(msg *Message) (int, error) {
+func (o *NullOutput) PluginWrite(msg *plugin.Message) (int, error) {
 	return len(msg.Data) + len(msg.Meta), nil
 }
 

pkg/plugin/limiter.go

@@ -1,4 +1,4 @@
-package main
+package plugin
 
 import (
 	"fmt"
@@ -11,9 +11,9 @@ import (
 
 // Limiter is a wrapper for input or output plugin which adds rate limiting
 type Limiter struct {
-	plugin    interface{}
-	limit     int
-	isPercent bool
+	Plugin    interface{}
+	Limit     int
+	IsPercent bool
 
 	currentRPS  int
 	currentTime int64
@@ -33,15 +33,15 @@ func parseLimitOptions(options string) (limit int, isPercent bool) {
 
 // NewLimiter constructor for Limiter, accepts plugin and options
 // `options` allow to sprcify relatve or absolute limiting
-func NewLimiter(plugin interface{}, options string) PluginReadWriter {
+func NewLimiter(plugin interface{}, options string) ReadWriter {
 	l := new(Limiter)
-	l.limit, l.isPercent = parseLimitOptions(options)
-	l.plugin = plugin
+	l.Limit, l.IsPercent = parseLimitOptions(options)
+	l.Plugin = plugin
 	l.currentTime = time.Now().UnixNano()
 
 	// FileInput have its own rate limiting. Unlike other inputs we not just dropping requests, we can slow down or speed up request emittion.
-	if fi, ok := l.plugin.(*FileInput); ok && l.isPercent {
-		fi.speedFactor = float64(l.limit) / float64(100)
+	if fi, ok := l.Plugin.(Limited); ok && l.IsPercent {
+		fi.SetLimit(float64(l.Limit) / float64(100))
 	}
 
 	return l
@@ -49,12 +49,12 @@ func NewLimiter(plugin interface{}, options string) PluginReadWriter {
 
 func (l *Limiter) isLimited() bool {
 	// File input have its own limiting algorithm
-	if _, ok := l.plugin.(*FileInput); ok && l.isPercent {
+	if _, ok := l.Plugin.(Limited); ok && l.IsPercent {
 		return false
 	}
 
-	if l.isPercent {
-		return l.limit <= rand.Intn(100)
+	if l.IsPercent {
+		return l.Limit <= rand.Intn(100)
 	}
 
 	if (time.Now().UnixNano() - l.currentTime) > time.Second.Nanoseconds() {
@@ -62,7 +62,7 @@ func (l *Limiter) isLimited() bool {
 		l.currentRPS = 0
 	}
 
-	if l.currentRPS >= l.limit {
+	if l.currentRPS >= l.Limit {
 		return true
 	}
 
@@ -76,7 +76,7 @@ func (l *Limiter) PluginWrite(msg *Message) (n int, err error) {
 	if l.isLimited() {
 		return 0, nil
 	}
-	if w, ok := l.plugin.(PluginWriter); ok {
+	if w, ok := l.Plugin.(Writer); ok {
 		return w.PluginWrite(msg)
 	}
 	// avoid further writing
@@ -85,7 +85,7 @@ func (l *Limiter) PluginWrite(msg *Message) (n int, err error) {
 
 // PluginRead reads message from this plugin
 func (l *Limiter) PluginRead() (msg *Message, err error) {
-	if r, ok := l.plugin.(PluginReader); ok {
+	if r, ok := l.Plugin.(Reader); ok {
 		msg, err = r.PluginRead()
 	} else {
 		// avoid further reading
@@ -100,12 +100,12 @@ func (l *Limiter) PluginRead() (msg *Message, err error) {
 }
 
 func (l *Limiter) String() string {
-	return fmt.Sprintf("Limiting %s to: %d (isPercent: %v)", l.plugin, l.limit, l.isPercent)
+	return fmt.Sprintf("Limiting %s to: %d (isPercent: %v)", l.Plugin, l.Limit, l.IsPercent)
 }
 
 // Close closes the resources.
 func (l *Limiter) Close() error {
-	if fi, ok := l.plugin.(io.Closer); ok {
+	if fi, ok := l.Plugin.(io.Closer); ok {
 		fi.Close()
 	}
 	return nil

pkg/plugin/limiter_test.go

@@ -0,0 +1,106 @@
+//go:build !race
+
+package plugin
+
+// func TestOutputLimiter(t *testing.T) {
+// 	wg := new(sync.WaitGroup)
+
+// 	input := test.NewTestInput()
+// 	output := NewLimiter(test.NewTestOutput(func(*plugin.Message) {
+// 		wg.Done()
+// 	}), "10")
+// 	wg.Add(10)
+
+// 	plugins := &InOutPlugins{
+// 		Inputs:  []Reader{input},
+// 		Outputs: []Writer{output},
+// 	}
+// 	plugins.All = append(plugins.All, input, output)
+
+// 	emitter := NewEmitter()
+// 	go emitter.Start(plugins, "")
+
+// 	for i := 0; i < 100; i++ {
+// 		input.EmitGET()
+// 	}
+
+// 	wg.Wait()
+// 	emitter.Close()
+// }
+
+// func TestInputLimiter(t *testing.T) {
+// 	wg := new(sync.WaitGroup)
+
+// 	input := NewLimiter(test.NewTestInput(), "10")
+// 	output := test.NewTestOutput(func(*Message) {
+// 		wg.Done()
+// 	})
+// 	wg.Add(10)
+
+// 	plugins := &InOutPlugins{
+// 		Inputs:  []Reader{input},
+// 		Outputs: []Writer{output},
+// 	}
+// 	plugins.All = append(plugins.All, input, output)
+
+// 	emitter := NewEmitter()
+// 	go emitter.Start(plugins, Settings.Middleware)
+
+// 	for i := 0; i < 100; i++ {
+// 		input.(*Limiter).plugin.(*TestInput).EmitGET()
+// 	}
+
+// 	wg.Wait()
+// 	emitter.Close()
+// }
+
+// // Should limit all requests
+// func TestPercentLimiter1(t *testing.T) {
+// 	wg := new(sync.WaitGroup)
+
+// 	input := test.NewTestInput()
+// 	output := NewLimiter(NewTestOutput(func(*Message) {
+// 		wg.Done()
+// 	}), "0%")
+
+// 	plugins := &InOutPlugins{
+// 		Inputs:  []Reader{input},
+// 		Outputs: []Writer{output},
+// 	}
+// 	plugins.All = append(plugins.All, input, output)
+
+// 	emitter := NewEmitter()
+// 	go emitter.Start(plugins, Settings.Middleware)
+
+// 	for i := 0; i < 100; i++ {
+// 		input.EmitGET()
+// 	}
+
+// 	wg.Wait()
+// }
+
+// // Should not limit at all
+// func TestPercentLimiter2(t *testing.T) {
+// 	wg := new(sync.WaitGroup)
+
+// 	input := test.NewTestInput()
+// 	output := NewLimiter(NewTestOutput(func(*Message) {
+// 		wg.Done()
+// 	}), "100%")
+// 	wg.Add(100)
+
+// 	plugins := &InOutPlugins{
+// 		Inputs:  []Reader{input},
+// 		Outputs: []Writer{output},
+// 	}
+// 	plugins.All = append(plugins.All, input, output)
+
+// 	emitter := NewEmitter()
+// 	go emitter.Start(plugins, Settings.Middleware)
+
+// 	for i := 0; i < 100; i++ {
+// 		input.EmitGET()
+// 	}
+
+// 	wg.Wait()
+// }

pkg/plugin/plugin.go

@@ -0,0 +1,104 @@
+package plugin
+
+import (
+	"errors"
+	"reflect"
+	"strings"
+)
+
+// ErrorStopped is the error returned when the go routines reading the input is stopped.
+var ErrorStopped = errors.New("reading stopped")
+
+// Message represents data across plugins
+type Message struct {
+	Meta []byte // metadata
+	Data []byte // actual data
+}
+
+// Reader is an interface for input plugins
+type Reader interface {
+	PluginRead() (msg *Message, err error)
+}
+
+// Writer is an interface for output plugins
+type Writer interface {
+	PluginWrite(msg *Message) (n int, err error)
+}
+
+// Limited is an interface for plugins that support limiting
+type Limited interface {
+	Limited() bool
+	SetLimit(float64)
+}
+
+// ReadWriter is an interface for plugins that support reading and writing
+type ReadWriter interface {
+	Reader
+	Writer
+}
+
+// Response is a response from a plugin
+type Response struct {
+	Payload       []byte
+	UUID          []byte
+	StartedAt     int64
+	RoundTripTime int64
+}
+
+// extractLimitOptions detects if plugin get called with limiter support
+// Returns address and limit
+func extractLimitOptions(options string) (string, string) {
+	split := strings.Split(options, "|")
+
+	if len(split) > 1 {
+		return split[0], split[1]
+	}
+
+	return split[0], ""
+}
+
+// InOutPlugins struct for holding references to plugins
+type InOutPlugins struct {
+	Inputs  []Reader
+	Outputs []Writer
+	All     []interface{}
+}
+
+// RegisterPlugin automatically detects type of plugin and initialize it
+//
+// See this article if curious about reflect stuff below: http://blog.burntsushi.net/type-parametric-functions-golang
+func (plugins *InOutPlugins) RegisterPlugin(constructor interface{}, options ...interface{}) {
+	var path, limit string
+	vc := reflect.ValueOf(constructor)
+
+	// Pre-processing options to make it work with reflect
+	vo := []reflect.Value{}
+	for _, oi := range options {
+		vo = append(vo, reflect.ValueOf(oi))
+	}
+
+	if len(vo) > 0 {
+		// Removing limit options from path
+		path, limit = extractLimitOptions(vo[0].String())
+
+		// Writing value back without limiter "|" options
+		vo[0] = reflect.ValueOf(path)
+	}
+
+	// Calling our constructor with list of given options
+	p := vc.Call(vo)[0].Interface()
+
+	if limit != "" {
+		p = NewLimiter(p, limit)
+	}
+
+	// Some of the output can be Readers as well because return responses
+	if r, ok := p.(Reader); ok {
+		plugins.Inputs = append(plugins.Inputs, r)
+	}
+
+	if w, ok := p.(Writer); ok {
+		plugins.Outputs = append(plugins.Outputs, w)
+	}
+	plugins.All = append(plugins.All, p)
+}

pkg/prettify/http_prettifier.go

@@ -0,0 +1,69 @@
+package prettify
+
+import (
+	"bytes"
+	"compress/gzip"
+	"io/ioutil"
+	"net/http/httputil"
+	"strconv"
+
+	"github.com/buger/goreplay/pkg/http_proto"
+
+	"github.com/rs/zerolog/log"
+)
+
+// PrettifyHTTP prettifies HTTP payload
+func PrettifyHTTP(p []byte) []byte {
+
+	tEnc := bytes.Equal(http_proto.Header(p, []byte("Transfer-Encoding")), []byte("chunked"))
+	cEnc := bytes.Equal(http_proto.Header(p, []byte("Content-Encoding")), []byte("gzip"))
+
+	if !(tEnc || cEnc) {
+		return p
+	}
+
+	headersPos := http_proto.MIMEHeadersEndPos(p)
+
+	if headersPos < 5 || headersPos > len(p) {
+		return p
+	}
+
+	headers := p[:headersPos]
+	content := p[headersPos:]
+
+	if tEnc {
+		buf := bytes.NewReader(content)
+		r := httputil.NewChunkedReader(buf)
+		content, _ = ioutil.ReadAll(r)
+
+		headers = http_proto.DeleteHeader(headers, []byte("Transfer-Encoding"))
+
+		newLen := strconv.Itoa(len(content))
+		headers = http_proto.SetHeader(headers, []byte("Content-Length"), []byte(newLen))
+	}
+
+	if cEnc {
+		buf := bytes.NewReader(content)
+		g, err := gzip.NewReader(buf)
+
+		if err != nil {
+			log.Error().Err(err).Msg("GZIP encoding error")
+			return []byte{}
+		}
+
+		content, err = ioutil.ReadAll(g)
+		if err != nil {
+			log.Error().Err(err).Msg("read error")
+			return p
+		}
+
+		headers = http_proto.DeleteHeader(headers, []byte("Content-Encoding"))
+
+		newLen := strconv.Itoa(len(content))
+		headers = http_proto.SetHeader(headers, []byte("Content-Length"), []byte(newLen))
+	}
+
+	newPayload := append(headers, content...)
+
+	return newPayload
+}

pkg/prettify/http_prettifier_test.go

@@ -1,4 +1,4 @@
-package main
+package prettify
 
 import (
 	"bytes"
@@ -6,7 +6,7 @@ import (
 	"strconv"
 	"testing"
 
-	"github.com/buger/goreplay/proto"
+	proto "github.com/buger/goreplay/pkg/http_proto"
 )
 
 func TestHTTPPrettifierGzip(t *testing.T) {
@@ -20,7 +20,7 @@ func TestHTTPPrettifierGzip(t *testing.T) {
 	payload := []byte("HTTP/1.1 200 OK\r\nContent-Length: " + size + "\r\nContent-Encoding: gzip\r\n\r\n")
 	payload = append(payload, b.Bytes()...)
 
-	newPayload := prettifyHTTP(payload)
+	newPayload := PrettifyHTTP(payload)
 
 	if string(newPayload) != "HTTP/1.1 200 OK\r\nContent-Length: 4\r\n\r\ntest" {
 		t.Errorf("Payload not match %q", string(newPayload))
@@ -30,7 +30,7 @@ func TestHTTPPrettifierGzip(t *testing.T) {
 func TestHTTPPrettifierChunked(t *testing.T) {
 	payload := []byte("POST / HTTP/1.1\r\nHost: www.w3.org\r\nTransfer-Encoding: chunked\r\n\r\n4\r\nWiki\r\n5\r\npedia\r\ne\r\n in\r\n\r\nchunks.\r\n0\r\n\r\n")
 
-	payload = prettifyHTTP(payload)
+	payload = PrettifyHTTP(payload)
 	if string(proto.Header(payload, []byte("Content-Length"))) != "23" {
 		t.Errorf("payload should have content length of 23")
 	}

pkg/pro/pro.go

@@ -1,5 +1,10 @@
-package main
+package pro
 
 // PRO this value indicates if goreplay is running in PRO mode..
 // it must not be modified explicitly in production
 var PRO = false
+
+// Enable enables PRO mode. Can be used ony in tests.
+func Enable() {
+	PRO = true
+}

pkg/proto/proto.go

@@ -1,4 +1,4 @@
-package main
+package proto
 
 import (
 	"bytes"
@@ -14,7 +14,8 @@ const (
 	ReplayedResponsePayload = '3'
 )
 
-func randByte(len int) []byte {
+// RandByte generates a random byte array of length len
+func RandByte(len int) []byte {
 	b := make([]byte, len/2)
 	rand.Read(b)
 
@@ -24,20 +25,26 @@ func randByte(len int) []byte {
 	return h
 }
 
-func uuid() []byte {
-	return randByte(24)
+// UUID generates a random 24 byte string
+func UUID() []byte {
+	return RandByte(24)
 }
 
-var payloadSeparator = "\n🐵🙈🙉\n"
+// PayloadSeparator is used to separate payloads
+var PayloadSeparator = "\n🐵🙈🙉\n"
 
-func payloadScanner(data []byte, atEOF bool) (advance int, token []byte, err error) {
+// PayloadSeparatorAsBytes is used to separate payloads
+var PayloadSeparatorAsBytes = []byte(PayloadSeparator)
+
+// PayloadScanner is used to split payloads
+func PayloadScanner(data []byte, atEOF bool) (advance int, token []byte, err error) {
 	if atEOF && len(data) == 0 {
 		return 0, nil, nil
 	}
 
-	if i := bytes.Index(data, []byte(payloadSeparator)); i >= 0 {
+	if i := bytes.Index(data, []byte(PayloadSeparator)); i >= 0 {
 		// We have a full newline-terminated line.
-		return i + len([]byte(payloadSeparator)), data[0:i], nil
+		return i + len([]byte(PayloadSeparator)), data[0:i], nil
 	}
 
 	if atEOF {
@@ -47,18 +54,20 @@ func payloadScanner(data []byte, atEOF bool) (advance int, token []byte, err err
 }
 
 // Timing is request start or round-trip time, depending on payloadType
-func payloadHeader(payloadType byte, uuid []byte, timing int64, latency int64) (header []byte) {
+func PayloadHeader(payloadType byte, uuid []byte, timing int64, latency int64) (header []byte) {
 	//Example:
 	//  3 f45590522cd1838b4a0d5c5aab80b77929dea3b3 13923489726487326 1231\n
 	return []byte(fmt.Sprintf("%c %s %d %d\n", payloadType, uuid, timing, latency))
 }
 
-func payloadBody(payload []byte) []byte {
+// PayloadBody returns the body of the payload
+func PayloadBody(payload []byte) []byte {
 	headerSize := bytes.IndexByte(payload, '\n')
 	return payload[headerSize+1:]
 }
 
-func payloadMeta(payload []byte) [][]byte {
+// PayloadMeta returns the meta data of the payload
+func PayloadMeta(payload []byte) [][]byte {
 	headerSize := bytes.IndexByte(payload, '\n')
 	if headerSize < 0 {
 		return nil
@@ -66,7 +75,8 @@ func payloadMeta(payload []byte) [][]byte {
 	return bytes.Split(payload[:headerSize], []byte{' '})
 }
 
-func payloadMetaWithBody(payload []byte) (meta, body []byte) {
+// PayloadMetaWithBody returns the meta data and body of the payload
+func PayloadMetaWithBody(payload []byte) (meta, body []byte) {
 	if i := bytes.IndexByte(payload, '\n'); i > 0 && len(payload) > i+1 {
 		meta = payload[:i+1]
 		body = payload[i+1:]
@@ -76,8 +86,9 @@ func payloadMetaWithBody(payload []byte) (meta, body []byte) {
 	return nil, payload
 }
 
-func payloadID(payload []byte) (id []byte) {
-	meta := payloadMeta(payload)
+// PayloadID returns the ID of the payload
+func PayloadID(payload []byte) (id []byte) {
+	meta := PayloadMeta(payload)
 
 	if len(meta) < 2 {
 		return
@@ -85,10 +96,12 @@ func payloadID(payload []byte) (id []byte) {
 	return meta[1]
 }
 
-func isOriginPayload(payload []byte) bool {
+// IsOriginPayload returns true if the payload is a request or response payload
+func IsOriginPayload(payload []byte) bool {
 	return payload[0] == RequestPayload || payload[0] == ResponsePayload
 }
 
-func isRequestPayload(payload []byte) bool {
+// IsRequestPayload returns true if the payload is a request payload
+func IsRequestPayload(payload []byte) bool {
 	return payload[0] == RequestPayload
 }

pkg/raw/input_raw.go

@@ -0,0 +1,181 @@
+package raw
+
+import (
+	"context"
+	"fmt"
+	"net"
+	"strconv"
+	"strings"
+	"sync"
+
+	"github.com/buger/goreplay/internal/capture"
+	"github.com/buger/goreplay/internal/tcp"
+	"github.com/buger/goreplay/pkg/http_proto"
+	"github.com/buger/goreplay/pkg/plugin"
+	"github.com/buger/goreplay/pkg/proto"
+
+	"github.com/rs/zerolog/log"
+)
+
+// RAWInputConfig represents configuration that can be applied on raw input
+type RAWInputConfig = capture.PcapOptions
+
+// RAWInput used for intercepting traffic for given address
+type RAWInput struct {
+	sync.Mutex
+	config         RAWInputConfig
+	messageStats   []tcp.Stats
+	listener       *capture.Listener
+	messageParser  *tcp.MessageParser
+	cancelListener context.CancelFunc
+	closed         bool
+
+	quit  chan bool // Channel used only to indicate goroutine should shutdown
+	host  string
+	ports []uint16
+}
+
+// NewRAWInput constructor for RAWInput. Accepts raw input config as arguments.
+func NewRAWInput(address string, config RAWInputConfig) (i *RAWInput) {
+	i = new(RAWInput)
+	i.config = config
+	i.quit = make(chan bool)
+
+	host, _ports, err := net.SplitHostPort(address)
+	if err != nil {
+		// If we are reading pcap file, no port needed
+		if strings.HasSuffix(address, "pcap") {
+			host = address
+			_ports = "0"
+			err = nil
+		} else if strings.HasPrefix(address, "k8s://") {
+			portIndex := strings.LastIndex(address, ":")
+			host = address[:portIndex]
+			_ports = address[portIndex+1:]
+		} else {
+			log.Fatal().Err(err).Msg("input-raw: error while parsing address")
+		}
+	}
+
+	if strings.HasSuffix(host, "pcap") {
+		i.config.Engine = capture.EnginePcapFile
+	}
+
+	var ports []uint16
+	if _ports != "" {
+		portsStr := strings.Split(_ports, ",")
+
+		for _, portStr := range portsStr {
+			port, err := strconv.Atoi(strings.TrimSpace(portStr))
+			if err != nil {
+				log.Fatal().Err(err).Msg("input-raw: error while parsing port")
+			}
+			ports = append(ports, uint16(port))
+
+		}
+	}
+
+	i.host = host
+	i.ports = ports
+
+	i.listen(address)
+
+	return
+}
+
+// PluginRead reads meassage from this plugin
+func (i *RAWInput) PluginRead() (*plugin.Message, error) {
+	var msgTCP *tcp.Message
+	var msg plugin.Message
+	select {
+	case <-i.quit:
+		return nil, plugin.ErrorStopped
+	case msgTCP = <-i.listener.Messages():
+		msg.Data = msgTCP.Data()
+	}
+
+	var msgType byte = proto.ResponsePayload
+	if msgTCP.Direction == tcp.DirIncoming {
+		msgType = proto.RequestPayload
+		if i.config.RealIPHeader != "" {
+			msg.Data = http_proto.SetHeader(msg.Data, []byte(i.config.RealIPHeader), []byte(msgTCP.SrcAddr))
+		}
+	}
+	msg.Meta = proto.PayloadHeader(msgType, msgTCP.UUID(), msgTCP.Start.UnixNano(), msgTCP.End.UnixNano()-msgTCP.Start.UnixNano())
+
+	// to be removed....
+	if msgTCP.Truncated {
+		log.Debug().Msg("message truncated, increase copy-buffer-size")
+	}
+	// to be removed...
+	if msgTCP.TimedOut {
+		log.Debug().Msg("message timeout reached, increase input-raw-expire")
+	}
+	if i.config.Stats {
+		stat := msgTCP.Stats
+		go i.addStats(stat)
+	}
+	msgTCP = nil
+	return &msg, nil
+}
+
+func (i *RAWInput) listen(address string) {
+	var err error
+	i.listener, err = capture.NewListener(i.host, i.ports, i.config)
+	if err != nil {
+		log.Fatal().Err(err).Msg("error while creating listener")
+	}
+
+	err = i.listener.Activate()
+	if err != nil {
+		log.Fatal().Err(err).Msg("error while activating listener")
+	}
+
+	var ctx context.Context
+	ctx, i.cancelListener = context.WithCancel(context.Background())
+	errCh := i.listener.ListenBackground(ctx)
+	<-i.listener.Reading
+
+	log.Info().Msg(i.String())
+
+	go func() {
+		<-errCh // the listener closed voluntarily
+		i.Close()
+	}()
+}
+
+func (i *RAWInput) String() string {
+	return fmt.Sprintf("Intercepting traffic from: %s:%s", i.host, strings.Join(strings.Fields(fmt.Sprint(i.ports)), ","))
+}
+
+// GetStats returns the stats so far and reset the stats
+func (i *RAWInput) GetStats() []tcp.Stats {
+	i.Lock()
+	defer func() {
+		i.messageStats = []tcp.Stats{}
+		i.Unlock()
+	}()
+	return i.messageStats
+}
+
+// Close closes the input raw listener
+func (i *RAWInput) Close() error {
+	i.Lock()
+	defer i.Unlock()
+	if i.closed {
+		return nil
+	}
+	i.cancelListener()
+	close(i.quit)
+	i.closed = true
+	return nil
+}
+
+func (i *RAWInput) addStats(mStats tcp.Stats) {
+	i.Lock()
+	if len(i.messageStats) >= 10000 {
+		i.messageStats = []tcp.Stats{}
+	}
+	i.messageStats = append(i.messageStats, mStats)
+	i.Unlock()
+}

pkg/raw/input_raw_test.go

@@ -1,4 +1,4 @@
-package main
+package raw
 
 import (
 	"bytes"
@@ -14,8 +14,13 @@ import (
 	"testing"
 	"time"
 
-	"github.com/buger/goreplay/capture"
-	"github.com/buger/goreplay/proto"
+	"github.com/buger/goreplay/internal/capture"
+	"github.com/buger/goreplay/internal/tcp"
+	"github.com/buger/goreplay/pkg/emitter"
+	gor_http "github.com/buger/goreplay/pkg/http"
+	gor_proto "github.com/buger/goreplay/pkg/http_proto"
+	"github.com/buger/goreplay/pkg/plugin"
+	"github.com/buger/goreplay/pkg/test"
 )
 
 const testRawExpire = time.Millisecond * 200
@@ -43,42 +48,47 @@ func TestRAWInputIPv4(t *testing.T) {
 	conf := RAWInputConfig{
 		Engine:        capture.EnginePcap,
 		Expire:        0,
-		Protocol:      ProtocolHTTP,
+		Protocol:      tcp.ProtocolHTTP,
 		TrackResponse: true,
 		RealIPHeader:  "X-Real-IP",
 	}
 	input := NewRAWInput(listener.Addr().String(), conf)
 
-	output := NewTestOutput(func(msg *Message) {
+	output := test.NewTestOutput(func(msg *plugin.Message) {
 		if msg.Meta[0] == '1' {
-			if len(proto.Header(msg.Data, []byte("X-Real-IP"))) == 0 {
+			if len(gor_proto.Header(msg.Data, []byte("X-Real-IP"))) == 0 {
 				t.Error("Should have X-Real-IP header")
 			}
 			reqCounter++
 		} else {
 			respCounter++
 		}
+
 		wg.Done()
 	})
 
-	plugins := &InOutPlugins{
-		Inputs:  []PluginReader{input},
-		Outputs: []PluginWriter{output},
+	plugins := &plugin.InOutPlugins{
+		Inputs:  []plugin.Reader{input},
+		Outputs: []plugin.Writer{output},
 	}
 	plugins.All = append(plugins.All, input, output)
 
 	addr := "http://127.0.0.1:" + port
-	emitter := NewEmitter()
+	emitter := emitter.New()
 	defer emitter.Close()
-	go emitter.Start(plugins, Settings.Middleware)
+	go emitter.Start(plugins)
+
+	// time.Sleep(time.Second)
 	for i := 0; i < 1; i++ {
 		wg.Add(2)
 		_, err = http.Get(addr)
+
 		if err != nil {
 			t.Error(err)
 			return
 		}
 	}
+
 	wg.Wait()
 	const want = 10
 	if reqCounter != respCounter && reqCounter != want {
@@ -108,30 +118,31 @@ func TestRAWInputNoKeepAlive(t *testing.T) {
 	conf := RAWInputConfig{
 		Engine:        capture.EnginePcap,
 		Expire:        testRawExpire,
-		Protocol:      ProtocolHTTP,
+		Protocol:      tcp.ProtocolHTTP,
 		TrackResponse: true,
 	}
 	input := NewRAWInput(":"+port, conf)
 	var respCounter, reqCounter int64
-	output := NewTestOutput(func(msg *Message) {
+	output := test.NewTestOutput(func(msg *plugin.Message) {
 		if msg.Meta[0] == '1' {
 			atomic.AddInt64(&reqCounter, 1)
+			wg.Done()
 		} else {
 			atomic.AddInt64(&respCounter, 1)
+			wg.Done()
 		}
-		wg.Done()
 	})
 
-	plugins := &InOutPlugins{
-		Inputs:  []PluginReader{input},
-		Outputs: []PluginWriter{output},
+	plugins := &plugin.InOutPlugins{
+		Inputs:  []plugin.Reader{input},
+		Outputs: []plugin.Writer{output},
 	}
 	plugins.All = append(plugins.All, input, output)
 
 	addr := "http://127.0.0.1:" + port
 
-	emitter := NewEmitter()
-	go emitter.Start(plugins, Settings.Middleware)
+	emitter := emitter.New()
+	go emitter.Start(plugins)
 
 	for i := 0; i < 10; i++ {
 		// request + response
@@ -173,12 +184,12 @@ func TestRAWInputIPv6(t *testing.T) {
 	var respCounter, reqCounter int64
 	conf := RAWInputConfig{
 		Engine:        capture.EnginePcap,
-		Protocol:      ProtocolHTTP,
+		Protocol:      tcp.ProtocolHTTP,
 		TrackResponse: true,
 	}
 	input := NewRAWInput(originAddr, conf)
 
-	output := NewTestOutput(func(msg *Message) {
+	output := test.NewTestOutput(func(msg *plugin.Message) {
 		if msg.Meta[0] == '1' {
 			atomic.AddInt64(&reqCounter, 1)
 		} else {
@@ -187,14 +198,14 @@ func TestRAWInputIPv6(t *testing.T) {
 		wg.Done()
 	})
 
-	plugins := &InOutPlugins{
-		Inputs:  []PluginReader{input},
-		Outputs: []PluginWriter{output},
+	plugins := &plugin.InOutPlugins{
+		Inputs:  []plugin.Reader{input},
+		Outputs: []plugin.Writer{output},
 	}
 
-	emitter := NewEmitter()
+	emitter := emitter.New()
 	addr := "http://" + originAddr
-	go emitter.Start(plugins, Settings.Middleware)
+	go emitter.Start(plugins)
 	for i := 0; i < 10; i++ {
 		// request + response
 		wg.Add(2)
@@ -228,10 +239,11 @@ func TestInputRAWChunkedEncoding(t *testing.T) {
 
 	originAddr := strings.Replace(origin.Listener.Addr().String(), "[::]", "127.0.0.1", -1)
 	conf := RAWInputConfig{
-		Engine:        capture.EnginePcap,
-		Expire:        time.Second,
-		Protocol:      ProtocolHTTP,
-		TrackResponse: true,
+		Engine:          capture.EnginePcap,
+		Expire:          time.Second,
+		Protocol:        tcp.ProtocolHTTP,
+		TrackResponse:   true,
+		AllowIncomplete: true,
 	}
 	input := NewRAWInput(originAddr, conf)
 
@@ -248,17 +260,17 @@ func TestInputRAWChunkedEncoding(t *testing.T) {
 	}))
 	defer replay.Close()
 
-	httpOutput := NewHTTPOutput(replay.URL, &HTTPOutputConfig{})
+	httpOutput := gor_http.NewHTTPOutput(replay.URL, &gor_http.HTTPOutputConfig{})
 
-	plugins := &InOutPlugins{
-		Inputs:  []PluginReader{input},
-		Outputs: []PluginWriter{httpOutput},
+	plugins := &plugin.InOutPlugins{
+		Inputs:  []plugin.Reader{input},
+		Outputs: []plugin.Writer{httpOutput},
 	}
 	plugins.All = append(plugins.All, input, httpOutput)
 
-	emitter := NewEmitter()
+	emitter := emitter.New()
 	defer emitter.Close()
-	go emitter.Start(plugins, Settings.Middleware)
+	go emitter.Start(plugins)
 	wg.Add(2)
 
 	curl := exec.Command("curl", "http://"+originAddr, "--header", "Transfer-Encoding: chunked", "--header", "Expect:", "--data-binary", "@README.md")
@@ -309,12 +321,12 @@ func BenchmarkRAWInputWithReplay(b *testing.B) {
 	conf := RAWInputConfig{
 		Engine:        capture.EnginePcap,
 		Expire:        testRawExpire,
-		Protocol:      ProtocolHTTP,
+		Protocol:      tcp.ProtocolHTTP,
 		TrackResponse: true,
 	}
 	input := NewRAWInput(originAddr, conf)
 
-	testOutput := NewTestOutput(func(msg *Message) {
+	testOutput := test.NewTestOutput(func(msg *plugin.Message) {
 		if msg.Meta[0] == '1' {
 			reqCounter++
 		} else {
@@ -322,15 +334,15 @@ func BenchmarkRAWInputWithReplay(b *testing.B) {
 		}
 		wg.Done()
 	})
-	httpOutput := NewHTTPOutput("http://"+replayAddr, &HTTPOutputConfig{})
+	httpOutput := gor_http.NewHTTPOutput("http://"+replayAddr, &gor_http.HTTPOutputConfig{})
 
-	plugins := &InOutPlugins{
-		Inputs:  []PluginReader{input},
-		Outputs: []PluginWriter{testOutput, httpOutput},
+	plugins := &plugin.InOutPlugins{
+		Inputs:  []plugin.Reader{input},
+		Outputs: []plugin.Writer{testOutput, httpOutput},
 	}
 
-	emitter := NewEmitter()
-	go emitter.Start(plugins, Settings.Middleware)
+	emitter := emitter.New()
+	go emitter.Start(plugins)
 	addr := "http://" + originAddr
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {

pkg/settings/settings.go

@@ -1,30 +1,70 @@
-package main
+package settings
 
 import (
 	"flag"
 	"fmt"
 	"os"
 	"strconv"
-	"strings"
-	"sync"
 	"time"
 
-	"github.com/buger/goreplay/size"
+	"github.com/buger/goreplay/internal/size"
+	"github.com/buger/goreplay/pkg/binary"
+	"github.com/buger/goreplay/pkg/emitter"
+	"github.com/buger/goreplay/pkg/file"
+	"github.com/buger/goreplay/pkg/http"
+	"github.com/buger/goreplay/pkg/http_modifier"
+	"github.com/buger/goreplay/pkg/kafka"
+	"github.com/buger/goreplay/pkg/raw"
+	"github.com/buger/goreplay/pkg/tcp"
+	"github.com/buger/goreplay/pkg/ws"
 )
 
 // DEMO indicates that goreplay is running in demo mode
 var DEMO string
 
 // MultiOption allows to specify multiple flags with same name and collects all values into array
-type MultiOption []string
+type MultiOption struct {
+	a *[]string
+}
 
 func (h *MultiOption) String() string {
-	return fmt.Sprint(*h)
+	if h.a == nil {
+		return ""
+	}
+	return fmt.Sprint(*h.a)
 }
 
 // Set gets called multiple times for each flag with same name
 func (h *MultiOption) Set(value string) error {
-	*h = append(*h, value)
+	if h.a == nil {
+		return nil
+	}
+
+	*h.a = append(*h.a, value)
+	return nil
+}
+
+// MultiOption allows to specify multiple flags with same name and collects all values into array
+type MultiIntOption struct {
+	a *[]int
+}
+
+func (h *MultiIntOption) String() string {
+	if h.a == nil {
+		return ""
+	}
+
+	return fmt.Sprint(*h.a)
+}
+
+// Set gets called multiple times for each flag with same name
+func (h *MultiIntOption) Set(value string) error {
+	if h.a == nil {
+		return nil
+	}
+
+	val, _ := strconv.Atoi(value)
+	*h.a = append(*h.a, val)
 	return nil
 }
 
@@ -34,53 +74,60 @@ type AppSettings struct {
 	Stats     bool          `json:"stats"`
 	ExitAfter time.Duration `json:"exit-after"`
 
-	SplitOutput          bool   `json:"split-output"`
-	RecognizeTCPSessions bool   `json:"recognize-tcp-sessions"`
-	Pprof                string `json:"http-pprof"`
+	Pprof string `json:"http-pprof"`
 
-	InputDummy   MultiOption `json:"input-dummy"`
-	OutputDummy  MultiOption
+	CopyBufferSize size.Size `json:"copy-buffer-size"`
+
+	InputDummy   []string `json:"input-dummy"`
+	OutputDummy  []string
 	OutputStdout bool `json:"output-stdout"`
 	OutputNull   bool `json:"output-null"`
 
-	InputTCP        MultiOption `json:"input-tcp"`
-	InputTCPConfig  TCPInputConfig
-	OutputTCP       MultiOption `json:"output-tcp"`
-	OutputTCPConfig TCPOutputConfig
+	InputTCP        []string `json:"input-tcp"`
+	InputTCPConfig  tcp.TCPInputConfig
+	OutputTCP       []string `json:"output-tcp"`
+	OutputTCPConfig tcp.TCPOutputConfig
 	OutputTCPStats  bool `json:"output-tcp-stats"`
 
-	InputFile        MultiOption `json:"input-file"`
-	InputFileLoop    bool        `json:"input-file-loop"`
-	OutputFile       MultiOption `json:"output-file"`
-	OutputFileConfig FileOutputConfig
+	OutputWebSocket       []string `json:"output-ws"`
+	OutputWebSocketConfig ws.WebSocketOutputConfig
+	OutputWebSocketStats  bool `json:"output-ws-stats"`
 
-	InputRAW MultiOption `json:"input_raw"`
-	RAWInputConfig
+	InputFile          []string      `json:"input-file"`
+	InputFileLoop      bool          `json:"input-file-loop"`
+	InputFileReadDepth int           `json:"input-file-read-depth"`
+	InputFileDryRun    bool          `json:"input-file-dry-run"`
+	InputFileMaxWait   time.Duration `json:"input-file-max-wait"`
+	OutputFile         []string      `json:"output-file"`
+	OutputFileConfig   file.FileOutputConfig
+
+	InputRAW       []string `json:"input_raw"`
+	InputRAWConfig raw.RAWInputConfig
 
 	Middleware string `json:"middleware"`
 
-	InputHTTP    MultiOption
-	OutputHTTP   MultiOption `json:"output-http"`
-	PrettifyHTTP bool        `json:"prettify-http"`
+	InputHTTP  []string
+	OutputHTTP []string `json:"output-http"`
 
-	MaxAllowedMem string `json:"max-allowed-mem"`
-	maxAllowedMem int    `json:"-"`
+	OutputHTTPConfig http.HTTPOutputConfig
+	EmitterConfig    emitter.Config
 
-	OutputHTTPConfig HTTPOutputConfig
+	OutputBinary       []string `json:"output-binary"`
+	OutputBinaryConfig binary.BinaryOutputConfig
 
-	OutputBinary       MultiOption `json:"output-binary"`
-	OutputBinaryConfig BinaryOutputConfig
+	ModifierConfig http_modifier.HTTPModifierConfig
 
-	ModifierConfig HTTPModifierConfig
-
-	InputKafkaConfig  InputKafkaConfig
-	OutputKafkaConfig OutputKafkaConfig
-	KafkaTLSConfig    KafkaTLSConfig
+	InputKafkaConfig  kafka.InputKafkaConfig
+	OutputKafkaConfig kafka.OutputKafkaConfig
+	KafkaTLSConfig    kafka.KafkaTLSConfig
 }
 
 // Settings holds Gor configuration
 var Settings AppSettings
 
+// VERSION the current version of goreplay
+var VERSION = "1.3.0"
+
 func usage() {
 	fmt.Printf("Gor is a simple http traffic replication tool written in Go. Its main goal is to replay traffic from production servers to staging and dev environments.\nProject page: https://github.com/buger/gor\nAuthor: <Leonid Bugaev> leonsbox@gmail.com\nCurrent Version: v%s\n\n", VERSION)
 	flag.PrintDefaults()
@@ -99,29 +146,38 @@ func init() {
 		Settings.ExitAfter = 5 * time.Minute
 	}
 
-	flag.BoolVar(&Settings.SplitOutput, "split-output", false, "By default each output gets same traffic. If set to `true` it splits traffic equally among all outputs.")
-	flag.BoolVar(&Settings.RecognizeTCPSessions, "recognize-tcp-sessions", false, "[PRO] If turned on http output will create separate worker for each TCP session. Splitting output will session based as well.")
+	flag.BoolVar(&Settings.EmitterConfig.SplitOutput, "split-output", false, "By default each output gets same traffic. If set to `true` it splits traffic equally among all outputs.")
+	flag.BoolVar(&Settings.EmitterConfig.RecognizeTCPSessions, "recognize-tcp-sessions", false, "[PRO] If turned on http output will create separate worker for each TCP session. Splitting output will session based as well.")
 
-	flag.Var(&Settings.InputDummy, "input-dummy", "Used for testing outputs. Emits 'Get /' request every 1s")
+	flag.Var(&MultiOption{&Settings.InputDummy}, "input-dummy", "Used for testing outputs. Emits 'Get /' request every 1s")
 	flag.BoolVar(&Settings.OutputStdout, "output-stdout", false, "Used for testing inputs. Just prints to console data coming from inputs.")
 	flag.BoolVar(&Settings.OutputNull, "output-null", false, "Used for testing inputs. Drops all requests.")
 
-	flag.Var(&Settings.InputTCP, "input-tcp", "Used for internal communication between Gor instances. Example: \n\t# Receive requests from other Gor instances on 28020 port, and redirect output to staging\n\tgor --input-tcp :28020 --output-http staging.com")
+	flag.Var(&MultiOption{&Settings.InputTCP}, "input-tcp", "Used for internal communication between Gor instances. Example: \n\t# Receive requests from other Gor instances on 28020 port, and redirect output to staging\n\tgor --input-tcp :28020 --output-http staging.com")
 	flag.BoolVar(&Settings.InputTCPConfig.Secure, "input-tcp-secure", false, "Turn on TLS security. Do not forget to specify certificate and key files.")
 	flag.StringVar(&Settings.InputTCPConfig.CertificatePath, "input-tcp-certificate", "", "Path to PEM encoded certificate file. Used when TLS turned on.")
 	flag.StringVar(&Settings.InputTCPConfig.KeyPath, "input-tcp-certificate-key", "", "Path to PEM encoded certificate key file. Used when TLS turned on.")
 
-	flag.Var(&Settings.OutputTCP, "output-tcp", "Used for internal communication between Gor instances. Example: \n\t# Listen for requests on 80 port and forward them to other Gor instance on 28020 port\n\tgor --input-raw :80 --output-tcp replay.local:28020")
+	flag.Var(&MultiOption{&Settings.OutputTCP}, "output-tcp", "Used for internal communication between Gor instances. Example: \n\t# Listen for requests on 80 port and forward them to other Gor instance on 28020 port\n\tgor --input-raw :80 --output-tcp replay.local:28020")
 	flag.BoolVar(&Settings.OutputTCPConfig.Secure, "output-tcp-secure", false, "Use TLS secure connection. --input-file on another end should have TLS turned on as well.")
 	flag.BoolVar(&Settings.OutputTCPConfig.SkipVerify, "output-tcp-skip-verify", false, "Don't verify hostname on TLS secure connection.")
 	flag.BoolVar(&Settings.OutputTCPConfig.Sticky, "output-tcp-sticky", false, "Use Sticky connection. Request/Response with same ID will be sent to the same connection.")
 	flag.IntVar(&Settings.OutputTCPConfig.Workers, "output-tcp-workers", 10, "Number of parallel tcp connections, default is 10")
 	flag.BoolVar(&Settings.OutputTCPStats, "output-tcp-stats", false, "Report TCP output queue stats to console every 5 seconds.")
 
-	flag.Var(&Settings.InputFile, "input-file", "Read requests from file: \n\tgor --input-file ./requests.gor --output-http staging.com")
-	flag.BoolVar(&Settings.InputFileLoop, "input-file-loop", false, "Loop input files, useful for performance testing.")
+	flag.Var(&MultiOption{&Settings.OutputWebSocket}, "output-ws", "Just like output tcp, just with WebSocket. Example: \n\t# Listen for requests on 80 port and forward them to other Gor instance on 28020 port\n\tgor --input-raw :80 --output-ws wss://replay.local:28020/endpoint")
+	flag.BoolVar(&Settings.OutputWebSocketConfig.SkipVerify, "output-ws-skip-verify", false, "Don't verify hostname on TLS secure connection.")
+	flag.BoolVar(&Settings.OutputWebSocketConfig.Sticky, "output-ws-sticky", false, "Use Sticky connection. Request/Response with same ID will be sent to the same connection.")
+	flag.IntVar(&Settings.OutputWebSocketConfig.Workers, "output-ws-workers", 10, "Number of parallel ws connections, default is 10")
+	flag.BoolVar(&Settings.OutputWebSocketStats, "output-ws-stats", false, "Report WebSocket output queue stats to console every 5 seconds.")
 
-	flag.Var(&Settings.OutputFile, "output-file", "Write incoming requests to file: \n\tgor --input-raw :80 --output-file ./requests.gor")
+	flag.Var(&MultiOption{&Settings.InputFile}, "input-file", "Read requests from file: \n\tgor --input-file ./requests.gor --output-http staging.com")
+	flag.BoolVar(&Settings.InputFileLoop, "input-file-loop", false, "Loop input files, useful for performance testing.")
+	flag.IntVar(&Settings.InputFileReadDepth, "input-file-read-depth", 100, "GoReplay tries to read and cache multiple records, in advance. In parallel it also perform sorting of requests, if they came out of order. Since it needs hold this buffer in memory, bigger values can cause worse performance")
+	flag.BoolVar(&Settings.InputFileDryRun, "input-file-dry-run", false, "Simulate reading from the data source without replaying it. You will get information about expected replay time, number of found records etc.")
+	flag.DurationVar(&Settings.InputFileMaxWait, "input-file-max-wait", 0, "Set the maximum time between requests. Can help in situations when you have too long periods between request, and you want to skip them. Example: --input-raw-max-wait 1s")
+
+	flag.Var(&MultiOption{&Settings.OutputFile}, "output-file", "Write incoming requests to file: \n\tgor --input-raw :80 --output-file ./requests.gor")
 	flag.DurationVar(&Settings.OutputFileConfig.FlushInterval, "output-file-flush-interval", time.Second, "Interval for forcing buffer flush to the file, default: 1s.")
 	flag.BoolVar(&Settings.OutputFileConfig.Append, "output-file-append", false, "The flushed chunk is appended to existence file or not. ")
 	flag.Var(&Settings.OutputFileConfig.SizeLimit, "output-file-size-limit", "Size of each chunk. Default: 32mb")
@@ -130,30 +186,35 @@ func init() {
 
 	flag.StringVar(&Settings.OutputFileConfig.BufferPath, "output-file-buffer", "/tmp", "The path for temporary storing current buffer: \n\tgor --input-raw :80 --output-file s3://mybucket/logs/%Y-%m-%d.gz --output-file-buffer /mnt/logs")
 
-	flag.BoolVar(&Settings.PrettifyHTTP, "prettify-http", false, "If enabled, will automatically decode requests and responses with: Content-Encoding: gzip and Transfer-Encoding: chunked. Useful for debugging, in conjuction with --output-stdout")
+	flag.BoolVar(&Settings.EmitterConfig.PrettifyHTTP, "prettify-http", false, "If enabled, will automatically decode requests and responses with: Content-Encoding: gzip and Transfer-Encoding: chunked. Useful for debugging, in conjunction with --output-stdout")
+
+	flag.Var(&Settings.CopyBufferSize, "copy-buffer-size", "Set the buffer size for an individual request (default 5MB)")
 
 	// input raw flags
-	flag.Var(&Settings.InputRAW, "input-raw", "Capture traffic from given port (use RAW sockets and require *sudo* access):\n\t# Capture traffic from 8080 port\n\tgor --input-raw :8080 --output-http staging.com")
-	flag.BoolVar(&Settings.TrackResponse, "input-raw-track-response", false, "If turned on Gor will track responses in addition to requests, and they will be available to middleware and file output.")
-	flag.Var(&Settings.Engine, "input-raw-engine", "Intercept traffic using `libpcap` (default), `raw_socket` or `pcap_file`")
-	flag.Var(&Settings.Protocol, "input-raw-protocol", "Specify application protocol of intercepted traffic. Possible values: http, binary")
-	flag.StringVar(&Settings.RealIPHeader, "input-raw-realip-header", "", "If not blank, injects header with given name and real IP value to the request payload. Usually this header should be named: X-Real-IP")
-	flag.DurationVar(&Settings.Expire, "input-raw-expire", time.Second*2, "How much it should wait for the last TCP packet, till consider that TCP message complete.")
-	flag.StringVar(&Settings.BPFFilter, "input-raw-bpf-filter", "", "BPF filter to write custom expressions. Can be useful in case of non standard network interfaces like tunneling or SPAN port. Example: --input-raw-bpf-filter 'dst port 80'")
-	flag.StringVar(&Settings.TimestampType, "input-raw-timestamp-type", "", "Possible values: PCAP_TSTAMP_HOST, PCAP_TSTAMP_HOST_LOWPREC, PCAP_TSTAMP_HOST_HIPREC, PCAP_TSTAMP_ADAPTER, PCAP_TSTAMP_ADAPTER_UNSYNCED. This values not supported on all systems, GoReplay will tell you available values of you put wrong one.")
-	flag.Var(&Settings.CopyBufferSize, "copy-buffer-size", "Set the buffer size for an individual request (default 5MB)")
-	flag.BoolVar(&Settings.Snaplen, "input-raw-override-snaplen", false, "Override the capture snaplen to be 64k. Required for some Virtualized environments")
-	flag.DurationVar(&Settings.BufferTimeout, "input-raw-buffer-timeout", 0, "set the pcap timeout. for immediate mode don't set this flag")
-	flag.Var(&Settings.BufferSize, "input-raw-buffer-size", "Controls size of the OS buffer which holds packets until they dispatched. Default value depends by system: in Linux around 2MB. If you see big package drop, increase this value.")
-	flag.BoolVar(&Settings.Promiscuous, "input-raw-promisc", false, "enable promiscuous mode")
-	flag.BoolVar(&Settings.Monitor, "input-raw-monitor", false, "enable RF monitor mode")
-	flag.BoolVar(&Settings.Stats, "input-raw-stats", false, "enable stats generator on raw TCP messages")
-
-	flag.StringVar(&Settings.MaxAllowedMem, "max-allowed-mem", "", "Allow specify memory which can be consumed by the process. Example: --max-allowed-mem 10%, --max-allowed-mem 200m")
+	flag.Var(&MultiOption{&Settings.InputRAW}, "input-raw", "Capture traffic from given port (use RAW sockets and require *sudo* access):\n\t# Capture traffic from 8080 port\n\tgor --input-raw :8080 --output-http staging.com")
+	flag.BoolVar(&Settings.InputRAWConfig.TrackResponse, "input-raw-track-response", false, "If turned on Gor will track responses in addition to requests, and they will be available to middleware and file output.")
+	flag.IntVar(&Settings.InputRAWConfig.VXLANPort, "input-raw-vxlan-port", 4789, "VXLAN port. Can be used only when engine set to `vxlan`. Default: 4789")
+	flag.Var(&MultiIntOption{&Settings.InputRAWConfig.VXLANVNIs}, "input-raw-vxlan-vni", "VXLAN VNI to capture. By default capture all VNIs. Ignore VNI by setting them with minus sign, example: `--input-raw-vxlan-vni -2`")
+	flag.BoolVar(&Settings.InputRAWConfig.VLAN, "input-raw-vlan", false, "Enable VLAN (802.1Q) support")
+	flag.Var(&MultiIntOption{&Settings.InputRAWConfig.VLANVIDs}, "input-raw-vlan-vid", "VLAN VID to capture. By default capture all VIDs")
+	flag.Var(&Settings.InputRAWConfig.Engine, "input-raw-engine", "Intercept traffic using `libpcap` (default), `raw_socket`, `pcap_file`, `vxlan`")
+	flag.Var(&Settings.InputRAWConfig.Protocol, "input-raw-protocol", "Specify application protocol of intercepted traffic. Possible values: http, binary")
+	flag.StringVar(&Settings.InputRAWConfig.RealIPHeader, "input-raw-realip-header", "", "If not blank, injects header with given name and real IP value to the request payload. Usually this header should be named: X-Real-IP")
+	flag.DurationVar(&Settings.InputRAWConfig.Expire, "input-raw-expire", time.Second*2, "How much it should wait for the last TCP packet, till consider that TCP message complete.")
+	flag.StringVar(&Settings.InputRAWConfig.BPFFilter, "input-raw-bpf-filter", "", "BPF filter to write custom expressions. Can be useful in case of non standard network interfaces like tunneling or SPAN port. Example: --input-raw-bpf-filter 'dst port 80'")
+	flag.StringVar(&Settings.InputRAWConfig.TimestampType, "input-raw-timestamp-type", "", "Possible values: PCAP_TSTAMP_HOST, PCAP_TSTAMP_HOST_LOWPREC, PCAP_TSTAMP_HOST_HIPREC, PCAP_TSTAMP_ADAPTER, PCAP_TSTAMP_ADAPTER_UNSYNCED. This values not supported on all systems, GoReplay will tell you available values of you put wrong one.")
+	flag.BoolVar(&Settings.InputRAWConfig.Snaplen, "input-raw-override-snaplen", false, "Override the capture snaplen to be 64k. Required for some Virtualized environments")
+	flag.DurationVar(&Settings.InputRAWConfig.BufferTimeout, "input-raw-buffer-timeout", 0, "set the pcap timeout. for immediate mode don't set this flag")
+	flag.Var(&Settings.InputRAWConfig.BufferSize, "input-raw-buffer-size", "Controls size of the OS buffer which holds packets until they dispatched. Default value depends by system: in Linux around 2MB. If you see big package drop, increase this value.")
+	flag.BoolVar(&Settings.InputRAWConfig.Promiscuous, "input-raw-promisc", false, "enable promiscuous mode")
+	flag.BoolVar(&Settings.InputRAWConfig.Monitor, "input-raw-monitor", false, "enable RF monitor mode")
+	flag.BoolVar(&Settings.InputRAWConfig.Stats, "input-raw-stats", false, "enable stats generator on raw TCP messages")
+	flag.BoolVar(&Settings.InputRAWConfig.AllowIncomplete, "input-raw-allow-incomplete", false, "If turned on Gor will record HTTP messages with missing packets")
+	flag.Var(&MultiOption{&Settings.InputRAWConfig.IgnoreInterface}, "input-raw-ignore-interface", "In case if you want listen for all interfaces except a few ones. Can be used in k8s environment. Example: --input-raw-ignore-interface cbr0 --input-raw-ignore-interface eth0 --input-raw-ignore-interface localhost")
 
 	flag.StringVar(&Settings.Middleware, "middleware", "", "Used for modifying traffic using external command")
 
-	flag.Var(&Settings.OutputHTTP, "output-http", "Forwards incoming requests to given http address.\n\t# Redirect all incoming requests to staging.com address \n\tgor --input-raw :80 --output-http http://staging.com")
+	flag.Var(&MultiOption{&Settings.OutputHTTP}, "output-http", "Forwards incoming requests to given http address.\n\t# Redirect all incoming requests to staging.com address \n\tgor --input-raw :80 --output-http http://staging.com")
 
 	/* outputHTTPConfig */
 	flag.Var(&Settings.OutputHTTPConfig.BufferSize, "output-http-response-buffer", "HTTP response buffer size, all data after this size will be discarded.")
@@ -173,7 +234,7 @@ func init() {
 	flag.StringVar(&Settings.OutputHTTPConfig.ElasticSearch, "output-http-elasticsearch", "", "Send request and response stats to ElasticSearch:\n\tgor --input-raw :8080 --output-http staging.com --output-http-elasticsearch 'es_host:api_port/index_name'")
 	/* outputHTTPConfig */
 
-	flag.Var(&Settings.OutputBinary, "output-binary", "Forwards incoming binary payloads to given address.\n\t# Redirect all incoming requests to staging.com address \n\tgor --input-raw :80 --input-raw-protocol binary --output-binary staging.com:80")
+	flag.Var(&MultiOption{&Settings.OutputBinary}, "output-binary", "Forwards incoming binary payloads to given address.\n\t# Redirect all incoming requests to staging.com address \n\tgor --input-raw :80 --input-raw-protocol binary --output-binary staging.com:80")
 
 	/* outputBinaryConfig */
 	flag.Var(&Settings.OutputBinaryConfig.BufferSize, "output-tcp-response-buffer", "TCP response buffer size, all data after this size will be discarded.")
@@ -187,16 +248,24 @@ func init() {
 	flag.StringVar(&Settings.OutputKafkaConfig.Host, "output-kafka-host", "", "Read request and response stats from Kafka:\n\tgor --input-raw :8080 --output-kafka-host '192.168.0.1:9092,192.168.0.2:9092'")
 	flag.StringVar(&Settings.OutputKafkaConfig.Topic, "output-kafka-topic", "", "Read request and response stats from Kafka:\n\tgor --input-raw :8080 --output-kafka-topic 'kafka-log'")
 	flag.BoolVar(&Settings.OutputKafkaConfig.UseJSON, "output-kafka-json-format", false, "If turned on, it will serialize messages from GoReplay text format to JSON.")
+	flag.BoolVar(&Settings.OutputKafkaConfig.SASLConfig.UseSASL, "output-kafka-use-sasl", false, "--output-kafka-use-sasl true")
+	flag.StringVar(&Settings.OutputKafkaConfig.SASLConfig.Mechanism, "output-kafka-mechanism", "", "mechanism\n\tgor --input-raw :8080 --output-kafka-mechanism 'SCRAM-SHA-512'")
+	flag.StringVar(&Settings.OutputKafkaConfig.SASLConfig.Username, "output-kafka-username", "", "username\n\tgor --input-raw :8080 --output-kafka-username 'username'")
+	flag.StringVar(&Settings.OutputKafkaConfig.SASLConfig.Password, "output-kafka-password", "", "password\n\tgor --input-raw :8080 --output-kafka-password 'password'")
 
 	flag.StringVar(&Settings.InputKafkaConfig.Host, "input-kafka-host", "", "Send request and response stats to Kafka:\n\tgor --output-stdout --input-kafka-host '192.168.0.1:9092,192.168.0.2:9092'")
 	flag.StringVar(&Settings.InputKafkaConfig.Topic, "input-kafka-topic", "", "Send request and response stats to Kafka:\n\tgor --output-stdout --input-kafka-topic 'kafka-log'")
 	flag.BoolVar(&Settings.InputKafkaConfig.UseJSON, "input-kafka-json-format", false, "If turned on, it will assume that messages coming in JSON format rather than  GoReplay text format.")
+	flag.BoolVar(&Settings.InputKafkaConfig.SASLConfig.UseSASL, "input-kafka-use-sasl", false, "use-sasl\n\t--use-sasl true")
+	flag.StringVar(&Settings.InputKafkaConfig.SASLConfig.Mechanism, "input-kafka-mechanism", "", "mechanism\n\tgor --input-raw :8080 --output-kafka-mechanism 'SCRAM-SHA-512'")
+	flag.StringVar(&Settings.InputKafkaConfig.SASLConfig.Username, "input-kafka-username", "", "username\n\tgor --input-raw :8080 --output-kafka-username 'username'")
+	flag.StringVar(&Settings.InputKafkaConfig.SASLConfig.Password, "input-kafka-password", "", "password\n\tgor --input-raw :8080 --output-kafka-password 'password'")
 
 	flag.StringVar(&Settings.KafkaTLSConfig.CACert, "kafka-tls-ca-cert", "", "CA certificate for Kafka TLS Config:\n\tgor  --input-raw :3000 --output-kafka-host '192.168.0.1:9092' --output-kafka-topic 'topic' --kafka-tls-ca-cert cacert.cer.pem --kafka-tls-client-cert client.cer.pem --kafka-tls-client-key client.key.pem")
 	flag.StringVar(&Settings.KafkaTLSConfig.ClientCert, "kafka-tls-client-cert", "", "Client certificate for Kafka TLS Config (mandatory with to kafka-tls-ca-cert and kafka-tls-client-key)")
 	flag.StringVar(&Settings.KafkaTLSConfig.ClientKey, "kafka-tls-client-key", "", "Client Key for Kafka TLS Config (mandatory with to kafka-tls-client-cert and kafka-tls-client-key)")
 
-	flag.Var(&Settings.ModifierConfig.Headers, "http-set-header", "Inject additional headers to http reqest:\n\tgor --input-raw :8080 --output-http staging.com --http-set-header 'User-Agent: Gor'")
+	flag.Var(&Settings.ModifierConfig.Headers, "http-set-header", "Inject additional headers to http request:\n\tgor --input-raw :8080 --output-http staging.com --http-set-header 'User-Agent: Gor'")
 	flag.Var(&Settings.ModifierConfig.HeaderRewrite, "http-rewrite-header", "Rewrite the request header based on a mapping:\n\tgor --input-raw :8080 --output-http staging.com --http-rewrite-header Host: (.*).example.com,$1.beta.example.com")
 	flag.Var(&Settings.ModifierConfig.Params, "http-set-param", "Set request url param, if param already exists it will be overwritten:\n\tgor --input-raw :8080 --output-http staging.com --http-set-param api_key=1")
 	flag.Var(&Settings.ModifierConfig.Methods, "http-allow-method", "Whitelist of HTTP methods to replay. Anything else will be dropped:\n\tgor --input-raw :8080 --output-http staging.com --http-allow-method GET --http-allow-method OPTIONS")
@@ -216,7 +285,7 @@ func init() {
 
 }
 
-func checkSettings() {
+func CheckSettings() {
 	if Settings.OutputFileConfig.SizeLimit < 1 {
 		Settings.OutputFileConfig.SizeLimit.Set("32mb")
 	}
@@ -226,35 +295,9 @@ func checkSettings() {
 	if Settings.CopyBufferSize < 1 {
 		Settings.CopyBufferSize.Set("5mb")
 	}
-
-	if Settings.MaxAllowedMem != "" {
-		_, totalMemory := memUsage()
-		fmt.Println("Total available memory:", bToMb(totalMemory), "Mb")
-		if strings.HasSuffix(Settings.MaxAllowedMem, "%") {
-			pct, _ := strconv.Atoi(strings.TrimSuffix(Settings.MaxAllowedMem, "%"))
-			Settings.maxAllowedMem = int((float64(pct) / 100) * float64(totalMemory))
-		} else {
-			var s size.Size
-			s.Set(Settings.MaxAllowedMem)
-			Settings.maxAllowedMem = int(s)
-		}
-
-		fmt.Println("Limiting memory usage to:", bToMb(uint64(Settings.maxAllowedMem)), "Mb")
-	}
 }
 
-var previousDebugTime = time.Now()
-var debugMutex sync.Mutex
-
-// Debug take an effect only if --verbose greater than 0 is specified
-func Debug(level int, args ...interface{}) {
-	if Settings.Verbose >= level {
-		debugMutex.Lock()
-		defer debugMutex.Unlock()
-		now := time.Now()
-		diff := now.Sub(previousDebugTime)
-		previousDebugTime = now
-		fmt.Fprintf(os.Stderr, "[DEBUG][elapsed %s]: ", diff)
-		fmt.Fprintln(os.Stderr, args...)
-	}
+// Set sets the global settings
+func Set(s AppSettings) {
+	Settings = s
 }

pkg/settings/settings_test.go

@@ -1,4 +1,4 @@
-package main
+package settings
 
 import (
 	"encoding/json"

pkg/stats/gor_stat.go

@@ -0,0 +1,66 @@
+package stats
+
+import (
+	"runtime"
+	"strconv"
+	"time"
+
+	"github.com/rs/zerolog/log"
+)
+
+// Stats is a struct that holds the stats for a given statName
+type Stats struct {
+	statName string
+	rateMs   int
+	latest   int
+	mean     int
+	max      int
+	count    int
+}
+
+// New creates a new Stats struct
+func New(statName string, rateMs int) (s *Stats) {
+	s = new(Stats)
+	s.statName = statName
+	s.rateMs = rateMs
+	s.latest = 0
+	s.mean = 0
+	s.max = 0
+	s.count = 0
+
+	go s.reportStats()
+
+	return
+}
+
+func (s *Stats) Write(latest int) {
+	if latest > s.max {
+		s.max = latest
+	}
+	if latest != 0 {
+		s.mean = ((s.mean * s.count) + latest) / (s.count + 1)
+	}
+	s.latest = latest
+	s.count = s.count + 1
+}
+
+// Reset resets the stats
+func (s *Stats) Reset() {
+	s.latest = 0
+	s.max = 0
+	s.mean = 0
+	s.count = 0
+}
+
+func (s *Stats) String() string {
+	return s.statName + ":" + strconv.Itoa(s.latest) + "," + strconv.Itoa(s.mean) + "," + strconv.Itoa(s.max) + "," + strconv.Itoa(s.count) + "," + strconv.Itoa(s.count/(s.rateMs/1000.0)) + "," + strconv.Itoa(runtime.NumGoroutine())
+}
+
+func (s *Stats) reportStats() {
+	log.Info().Msg(s.statName + ":latest,mean,max,count,count/second,gcount")
+	for {
+		log.Info().Msg(s.String())
+		s.Reset()
+		time.Sleep(time.Duration(s.rateMs) * time.Millisecond)
+	}
+}

pkg/tcp/input_tcp.go

@@ -1,18 +1,21 @@
-package main
+package tcp
 
 import (
 	"bufio"
 	"bytes"
 	"crypto/tls"
-	"fmt"
 	"io"
-	"log"
 	"net"
+
+	"github.com/buger/goreplay/pkg/plugin"
+	"github.com/buger/goreplay/pkg/proto"
+
+	"github.com/rs/zerolog/log"
 )
 
 // TCPInput used for internal communication
 type TCPInput struct {
-	data     chan *Message
+	data     chan *plugin.Message
 	listener net.Listener
 	address  string
 	config   *TCPInputConfig
@@ -29,7 +32,7 @@ type TCPInputConfig struct {
 // NewTCPInput constructor for TCPInput, accepts address with port
 func NewTCPInput(address string, config *TCPInputConfig) (i *TCPInput) {
 	i = new(TCPInput)
-	i.data = make(chan *Message, 1000)
+	i.data = make(chan *plugin.Message, 1000)
 	i.address = address
 	i.config = config
 	i.stop = make(chan bool)
@@ -40,10 +43,10 @@ func NewTCPInput(address string, config *TCPInputConfig) (i *TCPInput) {
 }
 
 // PluginRead returns data and details read from plugin
-func (i *TCPInput) PluginRead() (msg *Message, err error) {
+func (i *TCPInput) PluginRead() (msg *plugin.Message, err error) {
 	select {
 	case <-i.stop:
-		return nil, ErrorStopped
+		return nil, plugin.ErrorStopped
 	case msg = <-i.data:
 		return msg, nil
 	}
@@ -61,19 +64,19 @@ func (i *TCPInput) listen(address string) {
 	if i.config.Secure {
 		cer, err := tls.LoadX509KeyPair(i.config.CertificatePath, i.config.KeyPath)
 		if err != nil {
-			log.Fatalln("error while loading --input-tcp TLS certificate:", err)
+			log.Fatal().Err(err).Msg("error while loading --input-tcp TLS certificate")
 		}
 
 		config := &tls.Config{Certificates: []tls.Certificate{cer}}
 		listener, err := tls.Listen("tcp", address, config)
 		if err != nil {
-			log.Fatalln("[INPUT-TCP] failed to start INPUT-TCP listener:", err)
+			log.Fatal().Err(err).Msg("failed to start INPUT-TCP listener")
 		}
 		i.listener = listener
 	} else {
 		listener, err := net.Listen("tcp", address)
 		if err != nil {
-			log.Fatalln("failed to start INPUT-TCP listener:", err)
+			log.Fatal().Err(err).Msg("failed to start INPUT-TCP listener")
 		}
 		i.listener = listener
 	}
@@ -88,15 +91,13 @@ func (i *TCPInput) listen(address string) {
 				continue
 			}
 			if operr, ok := err.(*net.OpError); ok && operr.Err.Error() != "use of closed network connection" {
-				Debug(0, fmt.Sprintf("[INPUT-TCP] listener closed, err: %q", err))
+				log.Error().Err(err).Msg("failed to accept connection")
 			}
 			break
 		}
 	}()
 }
 
-var payloadSeparatorAsBytes = []byte(payloadSeparator)
-
 func (i *TCPInput) handleConnection(conn net.Conn) {
 	defer conn.Close()
 
@@ -110,16 +111,16 @@ func (i *TCPInput) handleConnection(conn net.Conn) {
 				continue
 			}
 			if err != io.EOF {
-				Debug(0, fmt.Sprintf("[INPUT-TCP] connection error: %q", err))
+				log.Err(err).Msg("failed to read from connection")
 			}
 			break
 		}
 
-		if bytes.Equal(payloadSeparatorAsBytes[1:], line) {
+		if bytes.Equal(proto.PayloadSeparatorAsBytes[1:], line) {
 			// unread the '\n' before monkeys
 			buffer.UnreadByte()
-			var msg Message
-			msg.Meta, msg.Data = payloadMetaWithBody(buffer.Bytes())
+			var msg plugin.Message
+			msg.Meta, msg.Data = proto.PayloadMetaWithBody(buffer.Bytes())
 			i.data <- &msg
 			buffer.Reset()
 		} else {

pkg/tcp/input_tcp_test.go

@@ -1,4 +1,4 @@
-package main
+package tcp
 
 import (
 	"bytes"
@@ -15,24 +15,29 @@ import (
 	"sync"
 	"testing"
 	"time"
+
+	"github.com/buger/goreplay/pkg/emitter"
+	"github.com/buger/goreplay/pkg/plugin"
+	"github.com/buger/goreplay/pkg/proto"
+	"github.com/buger/goreplay/pkg/test"
 )
 
 func TestTCPInput(t *testing.T) {
 	wg := new(sync.WaitGroup)
 
 	input := NewTCPInput("127.0.0.1:0", &TCPInputConfig{})
-	output := NewTestOutput(func(*Message) {
+	output := test.NewTestOutput(func(*plugin.Message) {
 		wg.Done()
 	})
 
-	plugins := &InOutPlugins{
-		Inputs:  []PluginReader{input},
-		Outputs: []PluginWriter{output},
+	plugins := &plugin.InOutPlugins{
+		Inputs:  []plugin.Reader{input},
+		Outputs: []plugin.Writer{output},
 	}
 	plugins.All = append(plugins.All, input, output)
 
-	emitter := NewEmitter()
-	go emitter.Start(plugins, Settings.Middleware)
+	emitter := emitter.New()
+	go emitter.Start(plugins)
 
 	tcpAddr, err := net.ResolveTCPAddr("tcp", input.listener.Addr().String())
 
@@ -50,7 +55,7 @@ func TestTCPInput(t *testing.T) {
 	for i := 0; i < 100; i++ {
 		wg.Add(1)
 		if _, err = conn.Write(msg); err == nil {
-			_, err = conn.Write(payloadSeparatorAsBytes)
+			_, err = conn.Write(proto.PayloadSeparatorAsBytes)
 		}
 		if err != nil {
 			t.Error(err)
@@ -106,18 +111,18 @@ func TestTCPInputSecure(t *testing.T) {
 		CertificatePath: serverCertPemFile.Name(),
 		KeyPath:         serverPrivPemFile.Name(),
 	})
-	output := NewTestOutput(func(*Message) {
+	output := test.NewTestOutput(func(*plugin.Message) {
 		wg.Done()
 	})
 
-	plugins := &InOutPlugins{
-		Inputs:  []PluginReader{input},
-		Outputs: []PluginWriter{output},
+	plugins := &plugin.InOutPlugins{
+		Inputs:  []plugin.Reader{input},
+		Outputs: []plugin.Writer{output},
 	}
 	plugins.All = append(plugins.All, input, output)
 
-	emitter := NewEmitter()
-	go emitter.Start(plugins, Settings.Middleware)
+	emitter := emitter.New()
+	go emitter.Start(plugins)
 
 	conf := &tls.Config{
 		InsecureSkipVerify: true,
@@ -134,7 +139,7 @@ func TestTCPInputSecure(t *testing.T) {
 	for i := 0; i < 100; i++ {
 		wg.Add(1)
 		conn.Write(msg)
-		conn.Write([]byte(payloadSeparator))
+		conn.Write([]byte(proto.PayloadSeparator))
 	}
 
 	wg.Wait()