Add support for simple pattern matching

Now when you listen for all interfaces, if new interface is added to the system while goreplay is running, it will auto-detect it and enable capture for it.

.github/workflows/ci-docker.yaml

@@ -14,7 +14,7 @@ jobs:
         uses: docker/setup-buildx-action@v1
       -
         name: "Cache Docker layers"
-        uses: actions/cache@v2
+        uses: actions/cache@v2.1.4
         with:
           path: /tmp/.buildx-cache
           key: ${{ runner.os }}-buildx-${{ github.sha }}

.github/workflows/ci-test.yaml

@@ -4,7 +4,7 @@ jobs:
   test:
     strategy:
       matrix:
-        go-version: [1.14.x, 1.15.x] # two latest minor versions
+        go-version: [1.16.x, 1.17.x] # two latest minor versions
     runs-on: ubuntu-latest
     steps:
     - name: update package index
@@ -17,7 +17,7 @@ jobs:
         go-version: ${{ matrix.go-version }}
     - name: checkout code
       uses: actions/checkout@v2
-    - uses: actions/cache@v2
+    - uses: actions/cache@v2.1.4
       with:
         path: |
           ~/go/pkg/mod              # Module download cache

.gitignore

@@ -1,10 +1,14 @@
+vendor
 *.swp
 *.gor
 *.rpm
 *.dep
+*.deb
 *.pkg
-
+*.exe
+*.pprof
 *.out
+hey
 
 *.bin
 

COMM-LICENSE

@@ -2,7 +2,7 @@ END-USER LICENSE AGREEMENT
 
 ------------------------------------------------------------------------------
 
-IMPORTANT: THIS SOFTWARE END-USER LICENSE AGREEMENT ("EULA") IS A LEGAL AGREEMENT (“Agreement”) BETWEEN YOU (THE CUSTOMER, EITHER AS AN INDIVIDUAL OR, IF PURCHASED OR OTHERWISE ACQUIRED BY OR FOR AN ENTITY, AS AN ENTITY) AND GoReplay LLC. READ IT CAREFULLY BEFORE COMPLETING THE INSTALLATION PROCESS AND USING GOREPLAY PRO AND RELATED SOFTWARE COMPONENTS (“SOFTWARE”). IT PROVIDES A LICENSE TO USE THE SOFTWARE AND CONTAINS WARRANTY INFORMATION AND LIABILITY DISCLAIMERS. BY INSTALLING AND USING THE SOFTWARE, YOU ARE CONFIRMING YOUR ACCEPTANCE OF THE SOFTWARE AND AGREEING TO BECOME BOUND BY THE TERMS OF THIS AGREEMENT.
+IMPORTANT: THIS SOFTWARE END-USER LICENSE AGREEMENT ("EULA") IS A LEGAL AGREEMENT (“Agreement”) BETWEEN YOU (THE CUSTOMER, EITHER AS AN INDIVIDUAL OR, IF PURCHASED OR OTHERWISE ACQUIRED BY OR FOR AN ENTITY, AS AN ENTITY), hereafter "Customer", AND Leonid Bugaev ("Licensor"). READ IT CAREFULLY BEFORE COMPLETING THE INSTALLATION PROCESS AND USING GOREPLAY PRO AND RELATED SOFTWARE COMPONENTS (“SOFTWARE”). IT PROVIDES A LICENSE TO USE THE SOFTWARE AND CONTAINS WARRANTY INFORMATION AND LIABILITY DISCLAIMERS. BY INSTALLING AND USING THE SOFTWARE, YOU ARE CONFIRMING YOUR ACCEPTANCE OF THE SOFTWARE AND AGREEING TO BECOME BOUND BY THE TERMS OF THIS AGREEMENT.
 
 ------------------------------------------------------------------------------
 
@@ -14,17 +14,17 @@ In order to use the Software under this Agreement, you must receive a “Source
 
 1.2 Unlimited Organization License. If you purchased an Organization License (included with the GoReplay Pro Software), you may install the Software on an unlimited number of Hosts. “Host” means any physical or virtual machine which is controlled by you. You may concurrently run the software on an unlimited number of Hosts.
 
-1.3 Appliance License. If you purchased an Appliance License, you may distribute the Software in any applications, frameworks, or elements (collectively referred to as an “Application” or “Applications”) that you develop using the Software in accordance with this EULA, provided that such distribution does not violate the restrictions set forth in section 3 of this EULA. You must not remove, obscure or interfere with any copyright, acknowledgment, attribution, trademark, warning or disclaimer statement affixed to, incorporated in or otherwise applied in connection with the Software. You are required to ensure that the Software is not reused by or with any applications other than those with which you distribute it as permitted herein. For example, if You install the Software on a customer’s server, that customer is not permitted to use the Software independently of your Application. You must inform GoReplay LLC of your knowledge of any infringing use of the Software by any of your customers. You are liable for compliance by those third parties with the terms and conditions of this EULA. You will not owe GoReplay LLC any royalties for your distribution of the Software in accordance with this EULA.
+1.3 Appliance License. If you purchased an Appliance License, you may distribute the Software in any applications, frameworks, or elements (collectively referred to as an “Application” or “Applications”) that you develop using the Software in accordance with this EULA, provided that such distribution does not violate the restrictions set forth in section 3 of this EULA. You must not remove, obscure or interfere with any copyright, acknowledgment, attribution, trademark, warning or disclaimer statement affixed to, incorporated in or otherwise applied in connection with the Software. You are required to ensure that the Software is not reused by or with any applications other than those with which you distribute it as permitted herein. For example, if You install the Software on a customer’s server, that customer is not permitted to use the Software independently of your Application. You must inform Licensor of your knowledge of any infringing use of the Software by any of your customers. You are liable for compliance by those third parties with the terms and conditions of this EULA. You will not owe Licensor any royalties for your distribution of the Software in accordance with this EULA.
 
 1.4 Archive Copies. You are entitled to make a reasonable amount of copies of the Software for archival purposes. Each copy must reproduce all copyright and other proprietary rights notices on or in the Software Product.
 
 1.5 Electronic Delivery. All Software and license documentation shall be delivered by electronic means unless otherwise specified on the applicable invoice or at the time of purchase. Software shall be deemed delivered when it is made available for download by you (“Delivery”).        
 
-2. Modifications. GoReplay LLC shall provide you with source code so that you can create Modifications of the original software. “Modification” means: (a) any addition to or deletion from the contents of a file included in the original Software or previous Modifications created by You, or (b) any new file that contains any part of the original Software or previous Modifications. While you retain all rights to any original work authored by you as part of the Modifications, We continue to own all copyright and other intellectual property rights in the Software.
+2. Modifications. Licensor shall provide you with source code so that you can create Modifications of the original software. “Modification” means: (a) any addition to or deletion from the contents of a file included in the original Software or previous Modifications created by You, or (b) any new file that contains any part of the original Software or previous Modifications. While you retain all rights to any original work authored by you as part of the Modifications, We continue to own all copyright and other intellectual property rights in the Software.
 
 3. Restricted Uses. 
 
-3.1 You shall not (and shall not allow any third party to): (a) decompile, disassemble, or otherwise reverse engineer the Software or attempt to reconstruct or discover any source code, underlying ideas, algorithms, file formats or programming interfaces of the Software by any means whatsoever (except and only to the extent that applicable law prohibits or restricts reverse engineering restrictions); (b) distribute, sell, sublicense, rent, lease or use the Software for time sharing, hosting, service provider or like purposes, except as expressly permitted under this Agreement; (c) redistribute the Software or Modifications other than by including the Software or a portion thereof within your own product, which must have substantially different functionality than the Software or Modifications and must not allow any third party to use the Software or Modifications, or any portions thereof, for software development or application development purposes; (d) redistribute the Software as part of a product, "appliance" or "virtual server"; (e) redistribute the Software on any server which is not directly under your control; (f) remove any product identification, proprietary, copyright or other notices contained in the Software; (g) modify any part of the Software, create a derivative work of any part of the Software (except as permitted in Section 4), or incorporate the Software, except to the extent expressly authorized in writing by GoReplay LLC; (h) publicly disseminate performance information or analysis (including, without limitation, benchmarks) from any source relating to the Software; (i) utilize any equipment, device, software, or other means designed to circumvent or remove any form of Source URL or copy protection used by GoReplay LLC in connection with the Software, or use the Software together with any authorization code, Source URL, serial number, or other copy protection device not supplied by GoReplay LLC; (j) use the Software to develop a product which is competitive with any GoReplay LLC product offerings; or (k) use unauthorized Source URLS or keycode(s) or distribute or publish Source URLs or keycode(s), except as may be expressly permitted by GoReplay LLC in writing. If your unique Source URL is ever published, GoReplay LLC reserves the right to terminate your access without notice.
+3.1 You shall not (and shall not allow any third party to): (a) decompile, disassemble, or otherwise reverse engineer the Software or attempt to reconstruct or discover any source code, underlying ideas, algorithms, file formats or programming interfaces of the Software by any means whatsoever (except and only to the extent that applicable law prohibits or restricts reverse engineering restrictions); (b) distribute, sell, sublicense, rent, lease or use the Software for time sharing, hosting, service provider or like purposes, except as expressly permitted under this Agreement; (c) redistribute the Software or Modifications other than by including the Software or a portion thereof within your own product, which must have substantially different functionality than the Software or Modifications and must not allow any third party to use the Software or Modifications, or any portions thereof, for software development or application development purposes; (d) redistribute the Software as part of a product, "appliance" or "virtual server"; (e) redistribute the Software on any server which is not directly under your control; (f) remove any product identification, proprietary, copyright or other notices contained in the Software; (g) modify any part of the Software, create a derivative work of any part of the Software (except as permitted in Section 4), or incorporate the Software, except to the extent expressly authorized in writing by Licensor; (h) publicly disseminate performance information or analysis (including, without limitation, benchmarks) from any source relating to the Software; (i) utilize any equipment, device, software, or other means designed to circumvent or remove any form of Source URL or copy protection used by Licensor in connection with the Software, or use the Software together with any authorization code, Source URL, serial number, or other copy protection device not supplied by Licensor; (j) use the Software to develop a product which is competitive with any Licensor product offerings; or (k) use unauthorized Source URLS or keycode(s) or distribute or publish Source URLs or keycode(s), except as may be expressly permitted by Licensor in writing. If your unique Source URL is ever published, Licensor reserves the right to terminate your access without notice.
 
 3.2 UNDER NO CIRCUMSTANCES MAY YOU USE THE SOFTWARE AS PART OF A PRODUCT OR SERVICE THAT PROVIDES SIMILAR FUNCTIONALITY TO THE SOFTWARE ITSELF.
 
@@ -32,35 +32,35 @@ The Open Source version of the Software (“LGPL Version”) is licensed
 under the terms of the GNU Lesser General Public License versions 3.0
 (“LGPL”) and not under this EULA. 
 
-4. Ownership. Notwithstanding anything to the contrary contained herein, except for the limited license rights expressly provided herein, GoReplay LLC and its suppliers have and will retain all rights, title and interest (including, without limitation, all patent, copyright, trademark, trade secret and other intellectual property rights) in and to the Software and all copies, modifications and derivative works thereof (including any changes which incorporate any of your ideas, feedback or suggestions). You acknowledge that you are obtaining only a limited license right to the Software, and that irrespective of any use of the words “purchase”, “sale” or like terms hereunder no ownership rights are being conveyed to you under this Agreement or otherwise. 
+4. Ownership. Notwithstanding anything to the contrary contained herein, except for the limited license rights expressly provided herein, Licensor and its suppliers have and will retain all rights, title and interest (including, without limitation, all patent, copyright, trademark, trade secret and other intellectual property rights) in and to the Software and all copies, modifications and derivative works thereof (including any changes which incorporate any of your ideas, feedback or suggestions). You acknowledge that you are obtaining only a limited license right to the Software, and that irrespective of any use of the words “purchase”, “sale” or like terms hereunder no ownership rights are being conveyed to you under this Agreement or otherwise. 
 
-5. Fees and Payment. The Software license fees will be due and payable in full as set forth in the applicable invoice or at the time of purchase. If the Software does not function properly within two weeks of purchase, please contact us within those two weeks for a refund. You shall be responsible for all taxes, withholdings, duties and levies arising from the order (excluding taxes based on the net income of GoReplay LLC). 
+5. Fees and Payment. The Software license fees will be due and payable in full as set forth in the applicable invoice or at the time of purchase. If the Software does not function properly within two weeks of purchase, please contact us within those two weeks for a refund. You shall be responsible for all taxes, withholdings, duties and levies arising from the order (excluding taxes based on the net income of Licensor). 
 
 6. Support, Maintenance and Services. Subject to the terms and conditions of this Agreement, as set forth in your invoice, and as set forth on the GoReplay Pro support page (https://github.com/buger/goreplay/wiki/Pro-Support), support and maintenance services may be included with the purchase of your license subscription.
 
 7. Term of Agreement. 
 
-7.1 Term. This Agreement is effective as of the Delivery of the Software and expires at such time as all license and service subscriptions hereunder have expired in accordance with their own terms (the “Term”). For clarification, the term of your license under this Agreement may be perpetual, limited for Evaluation Version, or designated as a fixed-term license in the Invoice, and shall be specified at your time of purchase. Either party may terminate this Agreement (including all related Invoices) if the other party: (a) fails to cure any material breach of this Agreement within thirty (30) days after written notice of such breach, provided that GoReplay LLC may terminate this Agreement immediately upon any breach of Section 3 or if you exceed any other restrictions contained in Section 1, unless otherwise specified in this agreement; (b) ceases operation without a successor; or (c) seeks protection under any bankruptcy, receivership, trust deed, creditors arrangement, composition or comparable proceeding, or if any such proceeding is instituted against such party (and not dismissed within sixty (60) days)). Termination is not an exclusive remedy and the exercise by either party of any remedy under this Agreement will be without prejudice to any other remedies it may have under this Agreement, by law, or otherwise.        
+7.1 Term. This Agreement is effective as of the Delivery of the Software and expires at such time as all license and service subscriptions hereunder have expired in accordance with their own terms (the “Term”). For clarification, the term of your license under this Agreement may be perpetual, limited for Evaluation Version, or designated as a fixed-term license in the Invoice, and shall be specified at your time of purchase. Either party may terminate this Agreement (including all related Invoices) if the other party: (a) fails to cure any material breach of this Agreement within thirty (30) days after written notice of such breach, provided that Licensor may terminate this Agreement immediately upon any breach of Section 3 or if you exceed any other restrictions contained in Section 1, unless otherwise specified in this agreement; (b) ceases operation without a successor; or (c) seeks protection under any bankruptcy, receivership, trust deed, creditors arrangement, composition or comparable proceeding, or if any such proceeding is instituted against such party (and not dismissed within sixty (60) days)). Termination is not an exclusive remedy and the exercise by either party of any remedy under this Agreement will be without prejudice to any other remedies it may have under this Agreement, by law, or otherwise.        
 
 7.2 Termination. Upon any termination of this Agreement, you shall cease any and all use of any Software and destroy all copies thereof. 
 
-7.3 Expiration of License. Upon the expiration of any term under this Agreement, (a) all Software updates and services pursuant to the license shall cease, (b) you may only continue to run existing installations of the Software, (c) you may not install the Software on any additional Hosts, and (d) any new installation of the Software shall require the purchase of a new license subscription from GoReplay LLC.
+7.3 Expiration of License. Upon the expiration of any term under this Agreement, (a) all Software updates and services pursuant to the license shall cease, (b) you may only continue to run existing installations of the Software, (c) you may not install the Software on any additional Hosts, and (d) any new installation of the Software shall require the purchase of a new license subscription from Licensor.
 
-8. Disclaimer of Warranties. The Software is provided "as is," with all faults, defects and errors, and without warranty of any kind. GoReplay LLC does not warrant that the Software will be free of bugs, errors, viruses or other defects, and GoReplay LLC shall have no liability of any kind for the use of or inability to use the Software, the Software content or any associated service, and you acknowledge that it is not technically practicable for GoReplay LLC to do so. 
-To the maximum extent permitted by applicable law, GoReplay LLC disclaims all warranties, express, implied, arising by law or otherwise, regarding the Software, the Software content and their respective performance or suitability for your intended use, including without limitation any implied warranty of merchantability, fitness for a particular purpose.
-Notwithstanding the foregoing, GoReplay LLC represents and warrants that it either owns the entire right to, title to, interest in, or has the right to license, the Software, and your proper use of the same will not violate any intellectual property or proprietary rights of another.
+8. Disclaimer of Warranties. The Software is provided "as is," with all faults, defects and errors, and without warranty of any kind. Licensor does not warrant that the Software will be free of bugs, errors, viruses or other defects, and Licensor shall have no liability of any kind for the use of or inability to use the Software, the Software content or any associated service, and you acknowledge that it is not technically practicable for Licensor to do so. 
+To the maximum extent permitted by applicable law, Licensor disclaims all warranties, express, implied, arising by law or otherwise, regarding the Software, the Software content and their respective performance or suitability for your intended use, including without limitation any implied warranty of merchantability, fitness for a particular purpose.
+Notwithstanding the foregoing, Licensor represents and warrants that it either owns the entire right to, title to, interest in, or has the right to license, the Software, and your proper use of the same will not violate any intellectual property or proprietary rights of another.
 
 9. Limitation of Liability. 
 
 In no event will either party be liable for any direct, indirect, consequential, incidental, special, exemplary, or punitive damages or liabilities whatsoever arising from or relating to the Software, the Software content or this Agreement, whether based on contract, tort (including negligence), strict liability or other theory, even if such party has been advised of the possibility of such damages.
 
-In no event will GoReplay LLC liability exceed the Software license price as indicated in the invoice. The existence of more than one claim will not enlarge or extend this limit.
+In no event will Licensor liability exceed the Software license price as indicated in the invoice. The existence of more than one claim will not enlarge or extend this limit.
 
-10. Remedies. Your exclusive remedy and GoReplay LLC’ entire liability for breach of this Agreement shall be limited, at GoReplay LLC’ sole and exclusive discretion, to (a) replacement of any defective software or documentation; or (b) refund of the license fee paid to GoReplay LLC, payable in accordance with GoReplay LLC' refund policy.
+10. Remedies. Your exclusive remedy and Licensor’ entire liability for breach of this Agreement shall be limited, at Licensor’ sole and exclusive discretion, to (a) replacement of any defective software or documentation; or (b) refund of the license fee paid to Licensor, payable in accordance with Licensor' refund policy.
 
 11. Acknowledgements.
 
-11.1 Consent to the Use of Data. You agree that GoReplay LLC and its affiliates may collect and use technical information gathered as part of the product support services to be used in an anonymized manner. GoReplay LLC may use this information solely to improve products and services and will not disclose this information in a form that personally identifies you.
+11.1 Consent to the Use of Data. You agree that Licensor and its affiliates may collect and use technical information gathered as part of the product support services to be used in an anonymized manner. Licensor may use this information solely to improve products and services and will not disclose this information in a form that personally identifies you.
 
 11.2 Verification. We or a certified auditor acting on our behalf, may, upon its reasonable request and at its expense, audit you with respect to the use of the Software, subject to your reasonable information security and confidentiality protections. Such audit may be conducted by mail, electronic means or through an in-person visit to your place of business. Any such in-person audit shall be conducted during regular business hours at your facilities and shall not unreasonably interfere with your business activities. We shall not remove, copy, or redistribute any electronic material during the course of an audit. If an audit reveals that you are using the Software in a way that is in material violation of the terms of the EULA, then you shall pay our reasonable costs of conducting the audit. In the case of a material violation, you agree to pay Us any amounts owing that are attributable to the unauthorized use. In the alternative, We reserve the right, at our sole option, to terminate the licenses for the Software.
 
@@ -73,13 +73,13 @@ In no event will GoReplay LLC liability exceed the Software license price as ind
 
 13.1 Entire Agreement. This Agreement sets forth our entire agreement with respect to the Software and the subject matter hereof and supersedes all prior and contemporaneous understandings and agreements whether written or oral.
 
-13.2 Amendment. GoReplay LLC reserves the right, in its sole discretion, to amend this Agreement from time. Amendments to this Agreement can be located at: https://github.com/buger/goreplay/blob/master/COMM-LICENSE.
+13.2 Amendment. Licensor reserves the right, in its sole discretion, to amend this Agreement from time. Amendments to this Agreement can be located at: https://github.com/buger/goreplay/blob/master/COMM-LICENSE.
 
-13.3 Assignment. You may not assign this Agreement or any of its rights under this Agreement without the prior written consent of GoReplay LLC and any attempted assignment without such consent shall be void.
+13.3 Assignment. You may not assign this Agreement or any of its rights under this Agreement without the prior written consent of Licensor and any attempted assignment without such consent shall be void.
 
 13.4 Export Compliance. Each party agrees to comply with all applicable laws and regulations, including laws, regulations, orders or other restrictions on export, re-export or redistribution of software.
 
-13.5 Indemnification. You agree to defend, indemnify, and hold harmless GoReplay LLC from and against any lawsuits, claims, losses, damages, fines and expenses (including attorneys' fees and costs) arising out of your use of the Software or breach of this Agreement.
+13.5 Indemnification. You agree to defend, indemnify, and hold harmless Licensor from and against any lawsuits, claims, losses, damages, fines and expenses (including attorneys' fees and costs) arising out of your use of the Software or breach of this Agreement.
 
 13.6 Governing Law. This Agreement is governed by the laws of the State of Oregon and the United States without regard to conflicts of laws provisions thereof, and without regard to the United Nations Convention on the International Sale of Goods or the Uniform Computer Information Transactions Act, as currently enacted by any jurisdiction or as may be codified or amended from time to time by any jurisdiction. The jurisdiction and venue for actions related to the subject matter hereof shall be the state of Oregon and United States federal courts located in Portland, Oregon, and both parties hereby submit to the personal jurisdiction of such courts. 
 
@@ -91,4 +91,4 @@ In no event will GoReplay LLC liability exceed the Software license price as ind
 
 13.10 Headings. The headings of sections and paragraphs of this Agreement are for convenience of reference only and are not intended to restrict, affect or be of any weight in the interpretation or construction of the provisions of such sections or paragraphs.
 
-14. Contact Information. If you have any questions about this EULA, or if you want to contact GoReplay LLC for any reason, please direct correspondence to info@gortool.com
+14. Contact Information. If you have any questions about this EULA, or if you want to contact Licensor for any reason, please direct correspondence to hello@goreplay.org

Dockerfile

@@ -3,7 +3,7 @@ FROM alpine:3.12 as builder
 ARG RELEASE_VERSION
 
 RUN apk add --no-cache ca-certificates openssl
-RUN wget https://github.com/buger/goreplay/releases/download/${RELEASE_VERSION}/gor_${RELEASE_VERSION}_x64.tar.gz -O gor.tar.gz
+RUN wget https://github.com/buger/goreplay/releases/download/v${RELEASE_VERSION}/gor_${RELEASE_VERSION}_x64.tar.gz -O gor.tar.gz
 RUN tar xzf gor.tar.gz
 
 FROM scratch

Dockerfile.dev

@@ -1,9 +1,9 @@
-FROM golang:1.14
+FROM golang:1.17
 
 RUN apt-get update && apt-get install ruby vim-common -y
 
 RUN apt-get install flex bison -y
-RUN wget http://www.tcpdump.org/release/libpcap-1.9.1.tar.gz && tar xzf libpcap-1.9.1.tar.gz && cd libpcap-1.9.1 && ./configure && make install
+RUN wget http://www.tcpdump.org/release/libpcap-1.10.0.tar.gz && tar xzf libpcap-1.10.0.tar.gz && cd libpcap-1.10.0 && ./configure && make install
 
 RUN go get github.com/google/gopacket
 RUN go get -u golang.org/x/lint/golint

Makefile

@@ -4,14 +4,13 @@ PORT = 8000
 FADDR = :8000
 CONTAINER=gor
 PREFIX=
-RUN = docker run -v `pwd`:$(SOURCE_PATH) -e AWS_ACCESS_KEY_ID=$(AWS_ACCESS_KEY_ID) -e AWS_SECRET_ACCESS_KEY=$(AWS_SECRET_ACCESS_KEY) -p 0.0.0.0:$(PORT):$(PORT) -t -i $(CONTAINER)
+RUN = docker run --rm -v `pwd`:$(SOURCE_PATH) -e AWS_ACCESS_KEY_ID=$(AWS_ACCESS_KEY_ID) -e AWS_SECRET_ACCESS_KEY=$(AWS_SECRET_ACCESS_KEY) -p 0.0.0.0:$(PORT):$(PORT) -t -i $(CONTAINER)
 BENCHMARK = BenchmarkRAWInput
 TEST = TestRawListenerBench
 BIN_NAME = gor
 VERSION = DEV-$(shell date +%s)
 LDFLAGS = -ldflags "-X main.VERSION=$(VERSION)$(PREFIX) -extldflags \"-static\" -X main.DEMO=$(DEMO)"
 MAC_LDFLAGS = -ldflags "-X main.VERSION=$(VERSION)$(PREFIX) -X main.DEMO=$(DEMO)"
-FADDR = ":8000"
 
 FPMCOMMON= \
     --name goreplay \
@@ -23,33 +22,38 @@ FPMCOMMON= \
     -s dir \
     -C /tmp/gor-build \
 
-release: release-x64 release-mac
+.PHONY: vendor
 
-release-bin:
-	docker run -v `pwd`:$(SOURCE_PATH) -t --env GOOS=linux --env GOARCH=amd64  -i $(CONTAINER) go build -o $(BIN_NAME) -tags netgo $(LDFLAGS)
+release: release-x64 release-mac release-windows
 
-release-bin-mac:
-	GOOS=darwin go build -o $(BIN_NAME) $(MAC_LDFLAGS)
+vendor:
+	go mod vendor
 
-release-x64:
-	docker run -v `pwd`:$(SOURCE_PATH) -t --env GOOS=linux --env GOARCH=amd64  -i $(CONTAINER) go build -o $(BIN_NAME) -tags netgo $(LDFLAGS)
+release-bin: vendor
+	docker run --rm -v `pwd`:$(SOURCE_PATH) -t --env GOOS=linux --env GOARCH=amd64  -i $(CONTAINER) go build -mod=vendor -o $(BIN_NAME) -tags netgo $(LDFLAGS)
+
+release-bin-mac: vendor
+	GOOS=darwin go build -mod=vendor -o $(BIN_NAME) $(MAC_LDFLAGS)
+
+release-bin-windows: vendor
+	docker run -it --rm -v `pwd`:$(SOURCE_PATH) -w $(SOURCE_PATH) -e CGO_ENABLED=1 docker.elastic.co/beats-dev/golang-crossbuild:1.16.4-main --build-cmd "make VERSION=$(VERSION) build" -p "windows/amd64"
+
+release-x64: release-bin
 	tar -czf gor_$(VERSION)$(PREFIX)_x64.tar.gz $(BIN_NAME)
 	mkdir -p /tmp/gor-build
 	mv ./$(BIN_NAME) /tmp/gor-build/$(BIN_NAME)
 	cd /tmp/gor-build
 	rm -f goreplay_$(VERSION)_amd64.deb
 	rm -f goreplay-$(VERSION)-1.x86_64.rpm
-	fpm $(FPMCOMMON) -a amd64 -t deb ./=/usr/local/bin
-	fpm $(FPMCOMMON) -a amd64 -t rpm ./=/usr/local/bin
+	nfpm pkg --packager deb --target ./
+	nfpm pkg --packager rpm --target ./
 	rm -rf /tmp/gor-build
 
-release-x86:
-	docker run -v `pwd`:$(SOURCE_PATH) -t --env GOOS=linux --env GOARCH=386 -i $(CONTAINER) go build -o $(BIN_NAME) -tags netgo $(LDFLAGS)
+release-x86: release-bin-x86
 	tar -czf gor_$(VERSION)$(PREFIX)_x86.tar.gz $(BIN_NAME)
 	rm $(BIN_NAME)
 
-release-mac:
-	go build -o $(BIN_NAME) $(MAC_LDFLAGS)
+release-mac: release-bin-mac
 	tar -czf gor_$(VERSION)$(PREFIX)_mac.tar.gz $(BIN_NAME)
 	mkdir -p /tmp/gor-build
 	mv ./$(BIN_NAME) /tmp/gor-build/$(BIN_NAME)
@@ -58,12 +62,26 @@ release-mac:
 	fpm $(FPMCOMMON) -a amd64 -t osxpkg ./=/usr/local/bin
 	rm -rf /tmp/gor-build
 
+release-windows: release-bin-windows
+	mv ./gor ./gor.exe
+	zip gor-$(VERSION)$(PREFIX)_windows.zip ./gor.exe
+	rm -rf ./gor.exe
+
+clean:
+	rm -rf *.pkg
+	rm -rf *.zip
+	rm -rf *.gz
+	rm -rf *.deb
+	rm -rf *.rpm
+
+build:
+	go build -mod=vendor -o $(BIN_NAME) $(LDFLAGS)
 
 install:
 	go install $(MAC_LDFLAGS)
 
-build:
-	docker build -t $(CONTAINER) -f Dockerfile.dev .
+build-env:
+	docker buildx build --platform linux/amd64 -t $(CONTAINER) -f Dockerfile.dev .
 
 profile:
 	go build && ./$(BIN_NAME) --output-http="http://localhost:9000" --input-dummy 0 --input-raw :9000 --input-http :9000 --memprofile=./mem.out --cpuprofile=./cpu.out --stats --output-http-stats --output-http-timeout 100ms
@@ -102,7 +120,7 @@ profile_test:
 
 # Used mainly for debugging, because docker container do not have access to parent machine ports
 run:
-	$(RUN) go run $(LDFLAGS) $(SOURCE) --input-dummy=0 --output-http="http://localhost:9000" --input-raw-track-response --input-raw 127.0.0.1:9000 --verbose --debug --middleware "./examples/middleware/echo.sh" --output-file requests.gor
+	$(RUN) go run $(LDFLAGS) $(SOURCE) --input-dummy=0 --output-http="http://localhost:9000" --input-raw-track-response --input-raw 127.0.0.1:9000 --verbose 0 --middleware "./examples/middleware/echo.sh" --output-file requests.gor
 
 run-2:
 	$(RUN) go run $(LDFLAGS) $(SOURCE) --input-raw :8000 --input-raw-bpf-filter "dst port 8000" --output-stdout --output-http "http://localhost:8000" --input-dummy=0

README.md

@@ -4,7 +4,7 @@
 
 ## https://goreplay.org/
 
-GoReplay is an open-source network monitoring tool which can record your live traffic, and use it for shadowing, load testing, monitoring and detailed analysis.
+GoReplay is an open-source network monitoring tool which can record your live traffic and use it for shadowing, load testing, monitoring and detailed analysis.
 
 ## About
 
@@ -13,29 +13,28 @@ As your application grows, the effort required to test it also grows exponential
 GoReplay increases your confidence in code deployments, configuration and infrastructure changes.
 
 
-GoReplay offers unique approach for shadowing. Instead of being a proxy, GoReplay in background listen for traffic on your network interface, requiring no changes in your production infrastructure, rather then running GoReplay daemon on the same machine as your service.
+GoReplay offers a unique approach for shadowing. Instead of being a proxy, GoReplay listens in the background for traffic on your network interfaces, requiring no changes in your production infrastructure, other than running GoReplay daemon on the same machine as your service.
 
 ![Diagram](https://i.imgur.com/IN2xfDm.png)
 
 Check [latest documentation](http://github.com/buger/goreplay/wiki).
 
 ## Installation
-Download latest binary from https://github.com/buger/goreplay/releases or [compile by yourself](https://github.com/buger/goreplay/wiki/Compilation).
+Download the latest binary from https://github.com/buger/goreplay/releases or [compile by yourself](https://github.com/buger/goreplay/wiki/Compilation).
 
 ## Getting started
 
 The most basic setup will be `sudo ./gor --input-raw :8000 --output-stdout` which acts like tcpdump.
-If you already have test environment you can start replaying: `sudo ./gor --input-raw :8000 --output-http http://staging.env`.
+If you already have a test environment, you can start replaying by running: `sudo ./gor --input-raw :8000 --output-http http://staging.env`.
 
-See the our [documentation](https://github.com/buger/goreplay/wiki/) and [Getting started](https://github.com/buger/goreplay/wiki/Getting-Started) page for more info. 
+See our [documentation](https://github.com/buger/goreplay/wiki/) and the [Getting Started](https://github.com/buger/goreplay/wiki/Getting-Started) page for more info. 
 
 ## Newsletter
-Subscribe to our [newsletter](https://www.getdrip.com/forms/89690474/submissions/new) to stay informed about the latest features and changes to Gor project.
-
+Subscribe to our [newsletter](https://www.getdrip.com/forms/89690474/submissions/new) to stay informed about the latest features and changes to the Gor project.
 
 ## Want to Upgrade?
 
-We have created a [GoReplay PRO](https://goreplay.org/pro.html) extension which provides additional features such as support for binary protocols like Thrift or ProtocolBuffers, saving and replaying from cloud storage, TCP sessions replication, etc. The PRO version also includes a commercial-friendly license, dedicated support, and it also allows you to support high-quality open source development. 
+We have created a [GoReplay PRO](https://goreplay.org/pro.html) extension which provides additional features such as support for binary protocols like Thrift or ProtocolBuffers, saving and replaying from cloud storage, TCP session replication, etc. The PRO version also includes a commercial-friendly license, dedicated support, and it also allows you to support high-quality open source development. 
 
 
 ## Problems?

capture/af_packet.go

@@ -0,0 +1,32 @@
+// +build !linux
+
+package capture
+
+import (
+	"fmt"
+	"time"
+
+	"github.com/google/gopacket"
+)
+
+func newAfpacketHandle(device string, snaplen int, block_size int, num_blocks int,
+	useVLAN bool, timeout time.Duration) (*afpacketHandle, error) {
+	return nil, fmt.Errorf("Not implemented")
+}
+
+func afpacketComputeSize(targetSizeMb int, snaplen int, pageSize int) (
+	frameSize int, blockSize int, numBlocks int, err error) {
+	return 0, 0, 0, fmt.Errorf("Not implemented")
+}
+
+type afpacketHandle struct{}
+
+// ReadPacketData satisfies PacketDataSource interface
+func (h *afpacketHandle) ReadPacketData() (data []byte, ci gopacket.CaptureInfo, err error) {
+	return nil, gopacket.CaptureInfo{}, fmt.Errorf("Not implemented")
+}
+
+// SetBPFFilter translates a BPF filter string into BPF RawInstruction and applies them.
+func (h *afpacketHandle) SetBPFFilter(filter string, snaplen int) (err error) {
+	return fmt.Errorf("Not implemented")
+}

capture/af_packet_linux.go

@@ -0,0 +1,115 @@
+// +build linux
+
+package capture
+
+import (
+	"fmt"
+	"time"
+
+	"github.com/google/gopacket"
+	"github.com/google/gopacket/afpacket"
+	"github.com/google/gopacket/layers"
+	"github.com/google/gopacket/pcap"
+	"golang.org/x/net/bpf"
+
+	_ "github.com/google/gopacket/layers"
+)
+
+type afpacketHandle struct {
+	TPacket *afpacket.TPacket
+}
+
+func newAfpacketHandle(device string, snaplen int, block_size int, num_blocks int,
+	useVLAN bool, timeout time.Duration) (*afpacketHandle, error) {
+
+	h := &afpacketHandle{}
+	var err error
+
+	if device == "any" {
+		h.TPacket, err = afpacket.NewTPacket(
+			afpacket.OptFrameSize(snaplen),
+			afpacket.OptBlockSize(block_size),
+			afpacket.OptNumBlocks(num_blocks),
+			afpacket.OptAddVLANHeader(false),
+			afpacket.OptPollTimeout(timeout),
+			afpacket.SocketRaw,
+			afpacket.TPacketVersion3)
+	} else {
+		h.TPacket, err = afpacket.NewTPacket(
+			afpacket.OptInterface(device),
+			afpacket.OptFrameSize(snaplen),
+			afpacket.OptBlockSize(block_size),
+			afpacket.OptNumBlocks(num_blocks),
+			afpacket.OptAddVLANHeader(false),
+			afpacket.OptPollTimeout(timeout),
+			afpacket.SocketRaw,
+			afpacket.TPacketVersion3)
+	}
+	return h, err
+}
+
+// ReadPacketData satisfies PacketDataSource interface
+func (h *afpacketHandle) ReadPacketData() (data []byte, ci gopacket.CaptureInfo, err error) {
+	return h.TPacket.ReadPacketData()
+}
+
+// SetBPFFilter translates a BPF filter string into BPF RawInstruction and applies them.
+func (h *afpacketHandle) SetBPFFilter(filter string, snaplen int) (err error) {
+	pcapBPF, err := pcap.CompileBPFFilter(layers.LinkTypeEthernet, snaplen, filter)
+	if err != nil {
+		return err
+	}
+	bpfIns := []bpf.RawInstruction{}
+	for _, ins := range pcapBPF {
+		bpfIns2 := bpf.RawInstruction{
+			Op: ins.Code,
+			Jt: ins.Jt,
+			Jf: ins.Jf,
+			K:  ins.K,
+		}
+		bpfIns = append(bpfIns, bpfIns2)
+	}
+	if h.TPacket.SetBPF(bpfIns); err != nil {
+		return err
+	}
+	return h.TPacket.SetBPF(bpfIns)
+}
+
+// LinkType returns ethernet link type.
+func (h *afpacketHandle) LinkType() layers.LinkType {
+	return layers.LinkTypeEthernet
+}
+
+// Close will close afpacket source.
+func (h *afpacketHandle) Close() {
+	h.TPacket.Close()
+}
+
+// SocketStats prints received, dropped, queue-freeze packet stats.
+func (h *afpacketHandle) SocketStats() (as afpacket.SocketStats, asv afpacket.SocketStatsV3, err error) {
+	return h.TPacket.SocketStats()
+}
+
+// afpacketComputeSize computes the block_size and the num_blocks in such a way that the
+// allocated mmap buffer is close to but smaller than target_size_mb.
+// The restriction is that the block_size must be divisible by both the
+// frame size and page size.
+func afpacketComputeSize(targetSizeMb int, snaplen int, pageSize int) (
+	frameSize int, blockSize int, numBlocks int, err error) {
+
+	if snaplen < pageSize {
+		frameSize = pageSize / (pageSize / snaplen)
+	} else {
+		frameSize = (snaplen/pageSize + 1) * pageSize
+	}
+
+	// 128 is the default from the gopacket library so just use that
+	blockSize = frameSize * 128
+	numBlocks = (targetSizeMb * 1024 * 1024) / blockSize
+
+	if numBlocks == 0 {
+		return 0, 0, 0, fmt.Errorf("Interface buffersize is too small")
+	}
+
+	return frameSize, blockSize, numBlocks, nil
+}

capture/capture.go

@@ -3,57 +3,91 @@ package capture
 import (
 	"context"
 	"errors"
+	"expvar"
 	"fmt"
 	"io"
 	"log"
 	"net"
 	"os"
 	"runtime"
+	"strings"
 	"sync"
+	"syscall"
 	"time"
 
+	"github.com/buger/goreplay/proto"
 	"github.com/buger/goreplay/size"
+	"github.com/buger/goreplay/tcp"
+
 	"github.com/google/gopacket"
 	"github.com/google/gopacket/layers"
 	"github.com/google/gopacket/pcap"
-	"golang.org/x/sys/unix"
 )
 
-// Handler is a function that is used to handle packets
-type Handler func(*Packet)
+var stats *expvar.Map
+
+func init() {
+	stats = expvar.NewMap("raw")
+	stats.Init()
+}
+
+// PacketHandler is a function that is used to handle packets
+type PacketHandler func(*tcp.Packet)
+
+type PcapStatProvider interface {
+	Stats() (*pcap.Stats, error)
+}
 
 // PcapOptions options that can be set on a pcap capture handle,
 // these options take effect on inactive pcap handles
 type PcapOptions struct {
-	BufferTimeout time.Duration `json:"input-raw-buffer-timeout"`
-	TimestampType string        `json:"input-raw-timestamp-type"`
-	BPFFilter     string        `json:"input-raw-bpf-filter"`
-	BufferSize    size.Size     `json:"input-raw-buffer-size"`
-	Promiscuous   bool          `json:"input-raw-promisc"`
-	Monitor       bool          `json:"input-raw-monitor"`
-	Snaplen       bool          `json:"input-raw-override-snaplen"`
+	BufferTimeout   time.Duration   `json:"input-raw-buffer-timeout"`
+	TimestampType   string          `json:"input-raw-timestamp-type"`
+	BPFFilter       string          `json:"input-raw-bpf-filter"`
+	BufferSize      size.Size       `json:"input-raw-buffer-size"`
+	Promiscuous     bool            `json:"input-raw-promisc"`
+	Monitor         bool            `json:"input-raw-monitor"`
+	Snaplen         bool            `json:"input-raw-override-snaplen"`
+	Engine          EngineType      `json:"input-raw-engine"`
+	VXLANPort       int             `json:"input-raw-vxlan-port"`
+	VXLANVNIs       []int           `json:"input-raw-vxlan-vni"`
+	VLAN            bool            `json:"input-raw-vlan"`
+	VLANVIDs        []int           `json:"input-raw-vlan-vid"`
+	Expire          time.Duration   `json:"input-raw-expire"`
+	TrackResponse   bool            `json:"input-raw-track-response"`
+	Protocol        tcp.TCPProtocol `json:"input-raw-protocol"`
+	RealIPHeader    string          `json:"input-raw-realip-header"`
+	Stats           bool            `json:"input-raw-stats"`
+	AllowIncomplete bool            `json:"input-raw-allow-incomplete"`
+	IgnoreInterface []string        `json:"input-raw-ignore-interface"`
+	Transport       string
 }
 
 // Listener handle traffic capture, this is its representation.
 type Listener struct {
 	sync.Mutex
-	Transport  string       // transport layer default to tcp
+
+	config PcapOptions
+
 	Activate   func() error // function is used to activate the engine. it must be called before reading packets
-	Handles    map[string]gopacket.PacketDataSource
-	Interfaces []net.Interface
+	Handles    map[string]packetHandle
+	Interfaces []pcap.Interface
 	loopIndex  int
 	Reading    chan bool // this channel is closed when the listener has started reading packets
-	PcapOptions
-	Engine        EngineType
-	port          uint16 // src or/and dst port
-	trackResponse bool
+	messages   chan *tcp.Message
 
-	host string // pcap file name or interface (name, hardware addr, index or ip address)
+	ports []uint16
+	host  string // pcap file name or interface (name, hardware addr, index or ip address)
 
 	closeDone chan struct{}
 	quit      chan struct{}
 }
 
+type packetHandle struct {
+	handler gopacket.PacketDataSource
+	ips     []net.IP
+}
+
 // EngineType ...
 type EngineType uint8
 
@@ -62,6 +96,8 @@ const (
 	EnginePcap EngineType = 1 << iota
 	EnginePcapFile
 	EngineRawSocket
+	EngineAFPacket
+	EngineVXLAN
 )
 
 // Set is here so that EngineType can implement flag.Var
@@ -71,8 +107,12 @@ func (eng *EngineType) Set(v string) error {
 		*eng = EnginePcap
 	case "pcap_file":
 		*eng = EnginePcapFile
-	case "raw_socket", "af_packet":
+	case "raw_socket":
 		*eng = EngineRawSocket
+	case "af_packet":
+		*eng = EngineAFPacket
+	case "vxlan":
+		*eng = EngineVXLAN
 	default:
 		return fmt.Errorf("invalid engine %s", v)
 	}
@@ -87,6 +127,10 @@ func (eng *EngineType) String() (e string) {
 		e = "libpcap"
 	case EngineRawSocket:
 		e = "raw_socket"
+	case EngineAFPacket:
+		e = "af_packet"
+	case EngineVXLAN:
+		e = "vxlan"
 	default:
 		e = ""
 	}
@@ -96,32 +140,39 @@ func (eng *EngineType) String() (e string) {
 // NewListener creates and initialize a new Listener. if transport or/and engine are invalid/unsupported
 // is "tcp" and "pcap", are assumed. l.Engine and l.Transport can help to get the values used.
 // if there is an error it will be associated with getting network interfaces
-func NewListener(host string, port uint16, transport string, engine EngineType, trackResponse bool) (l *Listener, err error) {
+func NewListener(host string, ports []uint16, config PcapOptions) (l *Listener, err error) {
 	l = &Listener{}
 
 	l.host = host
-	l.port = port
-	l.Transport = "tcp"
-	if transport != "" {
-		l.Transport = transport
+	if l.host == "localhost" {
+		l.host = "127.0.0.1"
 	}
-	l.Handles = make(map[string]gopacket.PacketDataSource)
-	l.trackResponse = trackResponse
+	l.ports = ports
+
+	l.config = config
+	l.config.Transport = "tcp"
+	l.Handles = make(map[string]packetHandle)
+
 	l.closeDone = make(chan struct{})
 	l.quit = make(chan struct{})
 	l.Reading = make(chan bool)
-	switch engine {
+	l.messages = make(chan *tcp.Message, 10000)
+
+	switch l.config.Engine {
 	default:
-		l.Engine = EnginePcap
 		l.Activate = l.activatePcap
 	case EngineRawSocket:
-		l.Engine = EngineRawSocket
 		l.Activate = l.activateRawSocket
+	case EngineAFPacket:
+		l.Activate = l.activateAFPacket
 	case EnginePcapFile:
-		l.Engine = EnginePcapFile
 		l.Activate = l.activatePcapFile
 		return
+	case EngineVXLAN:
+		l.Activate = l.activateVxLanSocket
+		return
 	}
+
 	err = l.setInterfaces()
 	if err != nil {
 		return nil, err
@@ -129,17 +180,53 @@ func NewListener(host string, port uint16, transport string, engine EngineType,
 	return
 }
 
-// SetPcapOptions set pcap options for all yet to be actived pcap handles
-// setting this on already activated handles will not have any effect
-func (l *Listener) SetPcapOptions(opts PcapOptions) {
-	l.PcapOptions = opts
-}
-
 // Listen listens for packets from the handles, and call handler on every packet received
 // until the context done signal is sent or there is unrecoverable error on all handles.
 // this function must be called after activating pcap handles
-func (l *Listener) Listen(ctx context.Context, handler Handler) (err error) {
-	l.read(handler)
+func (l *Listener) Listen(ctx context.Context) (err error) {
+	l.Lock()
+	for key, handle := range l.Handles {
+		go l.readHandle(key, handle)
+	}
+	l.Unlock()
+
+	go func() {
+		for {
+			time.Sleep(time.Second)
+			var prevInterfaces []string
+			for _, in := range l.Interfaces {
+				prevInterfaces = append(prevInterfaces, in.Name)
+			}
+			l.setInterfaces()
+
+			for _, in := range l.Interfaces {
+				var found bool
+
+				for _, prev := range prevInterfaces {
+					if in.Name == prev {
+						found = true
+					}
+				}
+
+				if !found {
+					fmt.Println("Found new interface:", in.Name)
+					l.Lock()
+					l.Activate()
+
+					for key, handle := range l.Handles {
+						if key == in.Name {
+							fmt.Println("Activating capture on:", in.Name)
+							go l.readHandle(key, handle)
+							break
+						}
+					}
+					l.Unlock()
+				}
+			}
+		}
+	}()
+
+	close(l.Reading)
 	done := ctx.Done()
 	select {
 	case <-done:
@@ -148,15 +235,16 @@ func (l *Listener) Listen(ctx context.Context, handler Handler) (err error) {
 		err = ctx.Err()
 	case <-l.closeDone: // all handles closed voluntarily
 	}
+
 	return
 }
 
 // ListenBackground is like listen but can run concurrently and signal error through channel
-func (l *Listener) ListenBackground(ctx context.Context, handler Handler) chan error {
+func (l *Listener) ListenBackground(ctx context.Context) chan error {
 	err := make(chan error, 1)
 	go func() {
 		defer close(err)
-		if e := l.Listen(ctx, handler); err != nil {
+		if e := l.Listen(ctx); err != nil {
 			err <- e
 		}
 	}()
@@ -165,89 +253,106 @@ func (l *Listener) ListenBackground(ctx context.Context, handler Handler) chan e
 
 // Filter returns automatic filter applied by goreplay
 // to a pcap handle of a specific interface
-func (l *Listener) Filter(ifi net.Interface) (filter string) {
+func (l *Listener) Filter(ifi pcap.Interface) (filter string) {
 	// https://www.tcpdump.org/manpages/pcap-filter.7.html
 
-	port := fmt.Sprintf("portrange 0-%d", 1<<16-1)
-	if l.port != 0 {
-		port = fmt.Sprintf("port %d", l.port)
-	}
-	dir := " dst " // direction
-	if l.trackResponse {
-		dir = " "
-	}
-	filter = fmt.Sprintf("(%s%s%s)", l.Transport, dir, port)
+	hosts := []string{l.host}
 	if listenAll(l.host) || isDevice(l.host, ifi) {
-		return
+		hosts = interfaceAddresses(ifi)
 	}
-	filter = fmt.Sprintf("(%s%s%s and host %s)", l.Transport, dir, port, l.host)
-	return
-}
 
-// PcapDumpHandler returns a handler to write packet data in PCAP
-// format, See http://wiki.wireshark.org/Development/LibpcapFileFormathandler.
-// if link layer is invalid Ethernet is assumed
-func PcapDumpHandler(file *os.File, link layers.LinkType) (handler func(packet *Packet) error, err error) {
-	if link.String() == "" {
-		link = layers.LinkTypeEthernet
+	filter = portsFilter(l.config.Transport, "dst", l.ports)
+
+	if len(hosts) != 0 && !l.config.Promiscuous {
+		filter = fmt.Sprintf("((%s) and (%s))", filter, hostsFilter("dst", hosts))
+	} else {
+		filter = fmt.Sprintf("(%s)", filter)
 	}
-	w := NewWriterNanos(file)
-	err = w.WriteFileHeader(64<<10, link)
-	if err != nil {
-		return nil, err
+
+	if l.config.TrackResponse {
+		responseFilter := portsFilter(l.config.Transport, "src", l.ports)
+
+		if len(hosts) != 0 && !l.config.Promiscuous {
+			responseFilter = fmt.Sprintf("((%s) and (%s))", responseFilter, hostsFilter("src", hosts))
+		} else {
+			responseFilter = fmt.Sprintf("(%s)", responseFilter)
+		}
+
+		filter = fmt.Sprintf("%s or %s", filter, responseFilter)
 	}
-	return func(packet *Packet) error {
-		return w.WritePacket(*packet.Info, packet.Data)
-	}, nil
+
+	if l.config.VLAN {
+		if len(l.config.VLANVIDs) > 0 {
+			for _, vi := range l.config.VLANVIDs {
+				filter = fmt.Sprintf("vlan %d and ", vi) + filter
+			}
+		} else {
+			filter = "vlan and " + filter
+		}
+	}
+
+	return
 }
 
 // PcapHandle returns new pcap Handle from dev on success.
 // this function should be called after setting all necessary options for this listener
-func (l *Listener) PcapHandle(ifi net.Interface) (handle *pcap.Handle, err error) {
+func (l *Listener) PcapHandle(ifi pcap.Interface) (handle *pcap.Handle, err error) {
 	var inactive *pcap.InactiveHandle
 	inactive, err = pcap.NewInactiveHandle(ifi.Name)
 	if err != nil {
 		return nil, fmt.Errorf("inactive handle error: %q, interface: %q", err, ifi.Name)
 	}
 	defer inactive.CleanUp()
-	if l.TimestampType != "" {
+
+	if l.config.TimestampType != "" && l.config.TimestampType != "go" {
 		var ts pcap.TimestampSource
-		ts, err = pcap.TimestampSourceFromString(l.TimestampType)
+		ts, err = pcap.TimestampSourceFromString(l.config.TimestampType)
+		fmt.Println("Setting custom Timestamp Source. Supported values: `go`, ", inactive.SupportedTimestamps())
 		err = inactive.SetTimestampSource(ts)
 		if err != nil {
 			return nil, fmt.Errorf("%q: supported timestamps: %q, interface: %q", err, inactive.SupportedTimestamps(), ifi.Name)
 		}
 	}
-	if l.Promiscuous {
-		if err = inactive.SetPromisc(l.Promiscuous); err != nil {
+	if l.config.Promiscuous {
+		if err = inactive.SetPromisc(l.config.Promiscuous); err != nil {
 			return nil, fmt.Errorf("promiscuous mode error: %q, interface: %q", err, ifi.Name)
 		}
 	}
-	if l.Monitor {
-		if err = inactive.SetRFMon(l.Monitor); err != nil && !errors.Is(err, pcap.CannotSetRFMon) {
+	if l.config.Monitor {
+		if err = inactive.SetRFMon(l.config.Monitor); err != nil && !errors.Is(err, pcap.CannotSetRFMon) {
 			return nil, fmt.Errorf("monitor mode error: %q, interface: %q", err, ifi.Name)
 		}
 	}
+
 	var snap int
-	if l.Snaplen {
-		snap = 64<<10 + 200
-	} else if ifi.MTU > 0 {
-		snap = ifi.MTU + 200
+
+	if !l.config.Snaplen {
+		infs, _ := net.Interfaces()
+		for _, i := range infs {
+			if i.Name == ifi.Name {
+				snap = i.MTU + 200
+			}
+		}
 	}
+
+	if snap == 0 {
+		snap = 64<<10 + 200
+	}
+
 	err = inactive.SetSnapLen(snap)
 	if err != nil {
 		return nil, fmt.Errorf("snapshot length error: %q, interface: %q", err, ifi.Name)
 	}
-	if l.BufferSize > 0 {
-		err = inactive.SetBufferSize(int(l.BufferSize))
+	if l.config.BufferSize > 0 {
+		err = inactive.SetBufferSize(int(l.config.BufferSize))
 		if err != nil {
 			return nil, fmt.Errorf("handle buffer size error: %q, interface: %q", err, ifi.Name)
 		}
 	}
-	if l.BufferTimeout == 0 {
-		l.BufferTimeout = pcap.BlockForever
+	if l.config.BufferTimeout == 0 {
+		l.config.BufferTimeout = 2000 * time.Millisecond
 	}
-	err = inactive.SetTimeout(l.BufferTimeout)
+	err = inactive.SetTimeout(l.config.BufferTimeout)
 	if err != nil {
 		return nil, fmt.Errorf("handle buffer timeout error: %q, interface: %q", err, ifi.Name)
 	}
@@ -255,105 +360,149 @@ func (l *Listener) PcapHandle(ifi net.Interface) (handle *pcap.Handle, err error
 	if err != nil {
 		return nil, fmt.Errorf("PCAP Activate device error: %q, interface: %q", err, ifi.Name)
 	}
-	if l.BPFFilter != "" {
-		if l.BPFFilter[0] != '(' || l.BPFFilter[len(l.BPFFilter)-1] != ')' {
-			l.BPFFilter = "(" + l.BPFFilter + ")"
-		}
-	} else {
-		l.BPFFilter = l.Filter(ifi)
+
+	bpfFilter := l.config.BPFFilter
+	if bpfFilter == "" {
+		bpfFilter = l.Filter(ifi)
 	}
-	err = handle.SetBPFFilter(l.BPFFilter)
+	fmt.Println("Interface:", ifi.Name, ". BPF Filter:", bpfFilter)
+	err = handle.SetBPFFilter(bpfFilter)
 	if err != nil {
 		handle.Close()
-		return nil, fmt.Errorf("BPF filter error: %q%s, interface: %q", err, l.BPFFilter, ifi.Name)
+		return nil, fmt.Errorf("BPF filter error: %q%s, interface: %q", err, bpfFilter, ifi.Name)
 	}
 	return
 }
 
 // SocketHandle returns new unix ethernet handle associated with this listener settings
-func (l *Listener) SocketHandle(ifi net.Interface) (handle Socket, err error) {
+func (l *Listener) SocketHandle(ifi pcap.Interface) (handle Socket, err error) {
 	handle, err = NewSocket(ifi)
 	if err != nil {
 		return nil, fmt.Errorf("sock raw error: %q, interface: %q", err, ifi.Name)
 	}
-	if err = handle.SetPromiscuous(l.Promiscuous || l.Monitor); err != nil {
+	if err = handle.SetPromiscuous(l.config.Promiscuous || l.config.Monitor); err != nil {
 		return nil, fmt.Errorf("promiscuous mode error: %q, interface: %q", err, ifi.Name)
 	}
-	if l.BPFFilter != "" {
-		if l.BPFFilter[0] != '(' || l.BPFFilter[len(l.BPFFilter)-1] != ')' {
-			l.BPFFilter = "(" + l.BPFFilter + ")"
-		}
-	} else {
-		l.BPFFilter = l.Filter(ifi)
+	if l.config.BPFFilter == "" {
+		l.config.BPFFilter = l.Filter(ifi)
 	}
-	if err = handle.SetBPFFilter(l.BPFFilter); err != nil {
+	fmt.Println("BPF Filter: ", l.config.BPFFilter)
+	if err = handle.SetBPFFilter(l.config.BPFFilter); err != nil {
 		handle.Close()
-		return nil, fmt.Errorf("BPF filter error: %q%s, interface: %q", err, l.BPFFilter, ifi.Name)
+		return nil, fmt.Errorf("BPF filter error: %q%s, interface: %q", err, l.config.BPFFilter, ifi.Name)
 	}
 	handle.SetLoopbackIndex(int32(l.loopIndex))
 	return
 }
 
-func (l *Listener) read(handler Handler) {
-	l.Lock()
-	defer l.Unlock()
-	for key, handle := range l.Handles {
-		go func(key string, hndl gopacket.PacketDataSource) {
-			defer l.closeHandles(key)
-			linkSize := 14
-			linkType := int(layers.LinkTypeEthernet)
-			if _, ok := hndl.(*pcap.Handle); ok {
-				linkType = int(hndl.(*pcap.Handle).LinkType())
-				linkSize, ok = pcapLinkTypeLength(linkType)
-				if !ok {
-					if os.Getenv("GORDEBUG") != "0" {
-						log.Printf("can not identify link type of an interface '%s'\n", key)
-					}
-					return // can't find the linktype size
-				}
-			}
-			for {
-				select {
-				case <-l.quit:
-					return
-				default:
-					data, ci, err := hndl.ReadPacketData()
-					if err == nil {
-						handler(NewPacket(data, linkType, linkSize, &ci))
-						continue
-					}
-					if enext, ok := err.(pcap.NextError); ok && enext == pcap.NextErrorTimeoutExpired {
-						continue
-					}
-					if eno, ok := err.(unix.Errno); ok && eno.Temporary() {
-						continue
-					}
-					if enet, ok := err.(*net.OpError); ok && (enet.Temporary() || enet.Timeout()) {
-						continue
-					}
-					if err == io.EOF || err == io.ErrClosedPipe {
-						return
-					}
-					if os.Getenv("GORDEBUG") != "0" {
-						log.Printf("stopped reading from %s interface with error %s\n", key, err)
-					}
-					return
-				}
-			}
-		}(key, handle)
+func http1StartHint(pckt *tcp.Packet) (isRequest, isResponse bool) {
+	if proto.HasRequestTitle(pckt.Payload) {
+		return true, false
 	}
-	close(l.Reading)
+
+	if proto.HasResponseTitle(pckt.Payload) {
+		return false, true
+	}
+
+	// No request or response detected
+	return false, false
+}
+
+func http1EndHint(m *tcp.Message) bool {
+	if m.MissingChunk() {
+		return false
+	}
+
+	req, res := http1StartHint(m.Packets()[0])
+	return proto.HasFullPayload(m, m.PacketData()...) && (req || res)
+}
+
+func (l *Listener) readHandle(key string, hndl packetHandle) {
+	runtime.LockOSThread()
+
+	defer l.closeHandles(key)
+	linkSize := 14
+	linkType := int(layers.LinkTypeEthernet)
+	if _, ok := hndl.handler.(*pcap.Handle); ok {
+		linkType = int(hndl.handler.(*pcap.Handle).LinkType())
+		linkSize, ok = pcapLinkTypeLength(linkType, l.config.VLAN)
+		if !ok {
+			if os.Getenv("GORDEBUG") != "0" {
+				log.Printf("can not identify link type of an interface '%s'\n", key)
+			}
+			return // can't find the linktype size
+		}
+	}
+
+	messageParser := tcp.NewMessageParser(l.messages, l.ports, hndl.ips, l.config.Expire, l.config.AllowIncomplete)
+
+	if l.config.Protocol == tcp.ProtocolHTTP {
+		messageParser.Start = http1StartHint
+		messageParser.End = http1EndHint
+	}
+
+	timer := time.NewTicker(1 * time.Second)
+
+	for {
+		select {
+		case <-l.quit:
+			return
+		case <-timer.C:
+			if h, ok := hndl.handler.(PcapStatProvider); ok {
+				s, err := h.Stats()
+				if err == nil {
+					stats.Add("packets_received", int64(s.PacketsReceived))
+					stats.Add("packets_dropped", int64(s.PacketsDropped))
+					stats.Add("packets_if_dropped", int64(s.PacketsIfDropped))
+				}
+			}
+		default:
+			data, ci, err := hndl.handler.ReadPacketData()
+			if err == nil {
+				if l.config.TimestampType == "go" {
+					ci.Timestamp = time.Now()
+				}
+
+				messageParser.PacketHandler(&tcp.PcapPacket{
+					Data:     data,
+					LType:    linkType,
+					LTypeLen: linkSize,
+					Ci:       &ci,
+				})
+				continue
+			}
+			if enext, ok := err.(pcap.NextError); ok && enext == pcap.NextErrorTimeoutExpired {
+				continue
+			}
+			if eno, ok := err.(syscall.Errno); ok && eno.Temporary() {
+				continue
+			}
+			if enet, ok := err.(*net.OpError); ok && (enet.Temporary() || enet.Timeout()) {
+				continue
+			}
+			if err == io.EOF || err == io.ErrClosedPipe {
+				log.Printf("stopped reading from %s interface with error %s\n", key, err)
+				return
+			}
+
+			log.Printf("stopped reading from %s interface with error %s\n", key, err)
+			return
+		}
+	}
+}
+
+func (l *Listener) Messages() chan *tcp.Message {
+	return l.messages
 }
 
 func (l *Listener) closeHandles(key string) {
 	l.Lock()
 	defer l.Unlock()
 	if handle, ok := l.Handles[key]; ok {
-		if _, ok = handle.(Socket); ok {
-			handle.(Socket).Close()
-		} else {
-			handle.(*pcap.Handle).Close()
+		if c, ok := handle.handler.(io.Closer); ok {
+			c.Close()
 		}
+
 		delete(l.Handles, key)
 		if len(l.Handles) == 0 {
 			close(l.closeDone)
@@ -365,13 +514,20 @@ func (l *Listener) activatePcap() error {
 	var e error
 	var msg string
 	for _, ifi := range l.Interfaces {
+		if _, found := l.Handles[ifi.Name]; found {
+			continue
+		}
+
 		var handle *pcap.Handle
 		handle, e = l.PcapHandle(ifi)
 		if e != nil {
 			msg += ("\n" + e.Error())
 			continue
 		}
-		l.Handles[ifi.Name] = handle
+		l.Handles[ifi.Name] = packetHandle{
+			handler: handle,
+			ips:     interfaceIPs(ifi),
+		}
 	}
 	if len(l.Handles) == 0 {
 		return fmt.Errorf("pcap handles error:%s", msg)
@@ -379,6 +535,18 @@ func (l *Listener) activatePcap() error {
 	return nil
 }
 
+func (l *Listener) activateVxLanSocket() error {
+	handler, err := newVXLANHandler(l.config.VXLANPort, l.config.VXLANVNIs)
+	if err != nil {
+		return err
+	}
+	l.Handles["vxlan"] = packetHandle{
+		handler: handler,
+	}
+
+	return nil
+}
+
 func (l *Listener) activateRawSocket() error {
 	if runtime.GOOS != "linux" {
 		return fmt.Errorf("sock_raw is not stabilized on OS other than linux")
@@ -386,13 +554,20 @@ func (l *Listener) activateRawSocket() error {
 	var msg string
 	var e error
 	for _, ifi := range l.Interfaces {
+		if _, found := l.Handles[ifi.Name]; found {
+			continue
+		}
+
 		var handle Socket
 		handle, e = l.SocketHandle(ifi)
 		if e != nil {
 			msg += ("\n" + e.Error())
 			continue
 		}
-		l.Handles[ifi.Name] = handle
+		l.Handles[ifi.Name] = packetHandle{
+			handler: handle,
+			ips:     interfaceIPs(ifi),
+		}
 	}
 	if len(l.Handles) == 0 {
 		return fmt.Errorf("raw socket handles error:%s", msg)
@@ -406,69 +581,167 @@ func (l *Listener) activatePcapFile() (err error) {
 	if handle, e = pcap.OpenOffline(l.host); e != nil {
 		return fmt.Errorf("open pcap file error: %q", e)
 	}
-	if l.BPFFilter != "" {
-		if l.BPFFilter[0] != '(' || l.BPFFilter[len(l.BPFFilter)-1] != ')' {
-			l.BPFFilter = "(" + l.BPFFilter + ")"
-		}
-	} else {
-		addr := l.host
-		l.host = ""
-		l.BPFFilter = l.Filter(net.Interface{})
-		l.host = addr
-	}
-	if e = handle.SetBPFFilter(l.BPFFilter); e != nil {
+
+	tmp := l.host
+	l.host = ""
+	l.config.BPFFilter = l.Filter(pcap.Interface{})
+	l.host = tmp
+
+	if e = handle.SetBPFFilter(l.config.BPFFilter); e != nil {
 		handle.Close()
-		return fmt.Errorf("BPF filter error: %q, filter: %s", e, l.BPFFilter)
+		return fmt.Errorf("BPF filter error: %q, filter: %s", e, l.config.BPFFilter)
+	}
+
+	fmt.Println("BPF Filter:", l.config.BPFFilter)
+
+	l.Handles["pcap_file"] = packetHandle{
+		handler: handle,
 	}
-	l.Handles["pcap_file"] = handle
 	return
 }
 
+func (l *Listener) activateAFPacket() error {
+	szFrame, szBlock, numBlocks, err := afpacketComputeSize(32, 32<<10, os.Getpagesize())
+	if err != nil {
+		return err
+	}
+
+	var msg string
+	for _, ifi := range l.Interfaces {
+		if _, found := l.Handles[ifi.Name]; found {
+			continue
+		}
+
+		handle, err := newAfpacketHandle(ifi.Name, szFrame, szBlock, numBlocks, false, pcap.BlockForever)
+
+		if err != nil {
+			msg += ("\n" + err.Error())
+			continue
+		}
+
+		if l.config.BPFFilter == "" {
+			l.config.BPFFilter = l.Filter(ifi)
+		}
+		fmt.Println("Interface:", ifi.Name, ". BPF Filter:", l.config.BPFFilter)
+		handle.SetBPFFilter(l.config.BPFFilter, 64<<10)
+
+		l.Handles[ifi.Name] = packetHandle{
+			handler: handle,
+			ips:     interfaceIPs(ifi),
+		}
+	}
+
+	if len(l.Handles) == 0 {
+		return fmt.Errorf("pcap handles error:%s", msg)
+	}
+
+	return nil
+}
+
 func (l *Listener) setInterfaces() (err error) {
-	var ifis []net.Interface
-	ifis, err = net.Interfaces()
+	var pifis []pcap.Interface
+	pifis, err = pcap.FindAllDevs()
+	ifis, _ := net.Interfaces()
+	l.Interfaces = []pcap.Interface{}
+
 	if err != nil {
 		return
 	}
-	for i := range ifis {
-		if ifis[i].Flags&net.FlagLoopback != 0 {
-			l.loopIndex = ifis[i].Index
-		}
-		if ifis[i].Flags&net.FlagUp == 0 {
-			continue
-		}
-		if isDevice(l.host, ifis[i]) {
-			l.Interfaces = []net.Interface{ifis[i]}
-			return
-		}
-		addrs, e := ifis[i].Addrs()
-		if e != nil {
-			// don't give up on a failure from a single interface
-			continue
-		}
-		for _, addr := range addrs {
-			if cutMask(addr) == l.host {
-				l.Interfaces = []net.Interface{ifis[i]}
-				return
+
+	for _, pi := range pifis {
+		ignore := false
+		for _, ig := range l.config.IgnoreInterface {
+			if pi.Name == ig {
+				ignore = true
+				break
 			}
 		}
+
+		if ignore {
+			continue
+		}
+
+		if isDevice(l.host, pi) {
+			l.Interfaces = []pcap.Interface{pi}
+			return
+		}
+
+		var ni net.Interface
+		for _, i := range ifis {
+			if i.Name == pi.Name {
+				ni = i
+				break
+			}
+
+			addrs, _ := i.Addrs()
+			for _, a := range addrs {
+				for _, pa := range pi.Addresses {
+					if a.String() == pa.IP.String() {
+						ni = i
+						break
+					}
+				}
+			}
+		}
+
+		if ni.Flags&net.FlagLoopback != 0 {
+			l.loopIndex = ni.Index
+		}
+
+		if runtime.GOOS != "windows" {
+			if len(pi.Addresses) == 0 {
+				continue
+			}
+
+			if ni.Flags&net.FlagUp == 0 {
+				continue
+			}
+		}
+
+		l.Interfaces = append(l.Interfaces, pi)
 	}
-	l.Interfaces = ifis
 	return
 }
 
-func cutMask(addr net.Addr) string {
-	mask := addr.String()
-	for i, v := range mask {
-		if v == '/' {
-			return mask[:i]
+func isDevice(addr string, ifi pcap.Interface) bool {
+	// Windows npcap loopback have no IPs
+	if addr == "127.0.0.1" && ifi.Name == `\Device\NPF_Loopback` {
+		return true
+	}
+
+	if addr == ifi.Name {
+		return true
+	}
+
+	if strings.HasSuffix(addr, "*") {
+		if strings.HasPrefix(ifi.Name, addr[:len(addr)-1]) {
+			return true
 		}
 	}
-	return mask
+
+	for _, _addr := range ifi.Addresses {
+		if _addr.IP.String() == addr {
+			return true
+		}
+	}
+
+	return false
 }
 
-func isDevice(addr string, ifi net.Interface) bool {
-	return addr == ifi.Name || addr == fmt.Sprintf("%d", ifi.Index) || (addr != "" && addr == ifi.HardwareAddr.String())
+func interfaceAddresses(ifi pcap.Interface) []string {
+	var hosts []string
+	for _, addr := range ifi.Addresses {
+		hosts = append(hosts, addr.IP.String())
+	}
+	return hosts
+}
+
+func interfaceIPs(ifi pcap.Interface) []net.IP {
+	var ips []net.IP
+	for _, addr := range ifi.Addresses {
+		ips = append(ips, addr.IP)
+	}
+	return ips
 }
 
 func listenAll(addr string) bool {
@@ -478,3 +751,51 @@ func listenAll(addr string) bool {
 	}
 	return false
 }
+
+func portsFilter(transport string, direction string, ports []uint16) string {
+	if len(ports) == 0 || ports[0] == 0 {
+		return fmt.Sprintf("%s %s portrange 0-%d", transport, direction, 1<<16-1)
+	}
+
+	var filters []string
+	for _, port := range ports {
+		filters = append(filters, fmt.Sprintf("%s %s port %d", transport, direction, port))
+	}
+	return strings.Join(filters, " or ")
+}
+
+func hostsFilter(direction string, hosts []string) string {
+	var hostsFilters []string
+	for _, host := range hosts {
+		hostsFilters = append(hostsFilters, fmt.Sprintf("%s host %s", direction, host))
+	}
+
+	return strings.Join(hostsFilters, " or ")
+}
+
+func pcapLinkTypeLength(lType int, vlan bool) (int, bool) {
+	switch layers.LinkType(lType) {
+	case layers.LinkTypeEthernet:
+		if vlan {
+			return 18, true
+		} else {
+			return 14, true
+		}
+	case layers.LinkTypeNull, layers.LinkTypeLoop:
+		return 4, true
+	case layers.LinkTypeRaw, 12, 14:
+		return 0, true
+	case layers.LinkTypeIPv4, layers.LinkTypeIPv6:
+		// (TODO:) look out for IP encapsulation?
+		return 0, true
+	case layers.LinkTypeLinuxSLL:
+		return 16, true
+	case layers.LinkTypeFDDI:
+		return 13, true
+	case 226 /*DLT_IPNET*/ :
+		// https://www.tcpdump.org/linktypes/LINKTYPE_IPNET.html
+		return 24, true
+	default:
+		return 0, false
+	}
+}

capture/capture_test.go

@@ -1,313 +1,22 @@
 package capture
 
 import (
-	"context"
-	"encoding/binary"
-	"io/ioutil"
-	"net"
-	"os"
-	"sync/atomic"
 	"testing"
-	"time"
-
-	"github.com/google/gopacket/layers"
-	"github.com/google/gopacket/pcap"
 )
 
-var LoopBack = func() net.Interface {
-	ifis, _ := net.Interfaces()
-	for _, v := range ifis {
-		if v.Flags&net.FlagLoopback != 0 {
-			return v
-		}
-	}
-	return ifis[0]
-}()
-
 func TestSetInterfaces(t *testing.T) {
-	l := &Listener{}
-	l.host = "127.0.0.1"
-	l.setInterfaces()
-	if len(l.Interfaces) != 1 {
-		t.Error("expected a single interface")
+	listener := &Listener{
+		loopIndex: 99999,
 	}
-	l.host = LoopBack.HardwareAddr.String()
-	l.setInterfaces()
-	if l.Interfaces[0].Name != LoopBack.Name && len(l.Interfaces) != 1 {
-		t.Error("interface should be loop back interface")
-	}
-	l.host = ""
-	l.setInterfaces()
-	if len(l.Interfaces) < 1 {
-		t.Error("should get all interfaces")
-	}
-}
+	listener.setInterfaces()
 
-func TestBPFFilter(t *testing.T) {
-	l := &Listener{}
-	l.host = "127.0.0.1"
-	l.Transport = "tcp"
-	l.setInterfaces()
-	filter := l.Filter(l.Interfaces[0])
-	if filter != "(tcp dst portrange 0-65535 and host 127.0.0.1)" {
-		t.Error("wrong filter", filter)
-	}
-	l.port = 8000
-	l.trackResponse = true
-	filter = l.Filter(l.Interfaces[0])
-	if filter != "(tcp port 8000 and host 127.0.0.1)" {
-		t.Error("wrong filter")
-	}
-}
-
-func TestPcapDump(t *testing.T) {
-	f, err := ioutil.TempFile("", "pcap_file")
-	if err != nil {
-		t.Error(err)
-	}
-	h, _ := PcapDumpHandler(f, layers.LinkTypeLoop)
-	packets := Packets(1, 5, 5, 4)
-	for i := 0; i < len(packets); i++ {
-		if i == 1 {
-			// change dst port
-			binary.BigEndian.PutUint16(packets[i].TransLayer[2:], 8001)
-		}
-		if i == 4 {
-			packets[i].Info.CaptureLength = 40
-		}
-		h(packets[i])
-	}
-	name := f.Name()
-	f.Close()
-	testPcapDumpEngine(name, t)
-}
-
-func testPcapDumpEngine(f string, t *testing.T) {
-	defer os.Remove(f)
-	l, err := NewListener(f, 8000, "", EnginePcapFile, true)
-	err = l.Activate()
-	if err != nil {
-		t.Errorf("expected error to be nil, got %q", err)
-		return
-	}
-	pckts := 0
-	ctx, cancel := context.WithCancel(context.Background())
-	defer cancel()
-	err = l.Listen(ctx, func(packet *Packet) {
-		if packet.Info.CaptureLength != 57 {
-			t.Errorf("expected packet length to be %d, got %d", 57, packet.Info.CaptureLength)
-		}
-		pckts++
-	})
-
-	if err != nil {
-		t.Errorf("expected error to be nil, got %q", err)
-	}
-	if pckts != 3 {
-		t.Errorf("expected %d packets, got %d packets", 3, pckts)
-	}
-}
-
-func TestPcapHandler(t *testing.T) {
-	l, err := NewListener(LoopBack.Name, 8000, "", EnginePcap, true)
-	if err != nil {
-		t.Errorf("expected error to be nil, got %v", err)
-		return
-	}
-	err = l.Activate()
-	if err != nil {
-		t.Errorf("expected error to be nil, got %v", err)
-		return
-	}
-	defer l.Handles[LoopBack.Name].(*pcap.Handle).Close()
-	if err != nil {
-		t.Errorf("expected error to be nil, got %v", err)
-		return
-	}
-	for i := 0; i < 5; i++ {
-		_, _ = net.Dial("tcp", "127.0.0.1:8000")
-	}
-	sts, _ := l.Handles[LoopBack.Name].(*pcap.Handle).Stats()
-	if sts.PacketsReceived < 5 {
-		t.Errorf("expected >=5 packets got %d", sts.PacketsReceived)
-	}
-}
-
-func TestSocketHandler(t *testing.T) {
-	l, err := NewListener(LoopBack.Name, 8000, "", EngineRawSocket, true)
-	err = l.Activate()
-	if err != nil {
-		return
-	}
-	defer l.Handles[LoopBack.Name].(*SockRaw).Close()
-	if err != nil {
-		t.Errorf("expected error to be nil, got %v", err)
-		return
-	}
-	for i := 0; i < 5; i++ {
-		_, _ = net.Dial("tcp", "127.0.0.1:8000")
-	}
-	sts, _ := l.Handles[LoopBack.Name].(*SockRaw).Stats()
-	if sts.Packets < 5 {
-		t.Errorf("expected >=5 packets got %d", sts.Packets)
-	}
-}
-
-func BenchmarkPcapDump(b *testing.B) {
-	f, err := ioutil.TempFile("", "pcap_file")
-	if err != nil {
-		b.Error(err)
-		return
-	}
-	now := time.Now()
-	defer os.Remove(f.Name())
-	h, _ := PcapDumpHandler(f, layers.LinkTypeLoop)
-	packets := Packets(1, b.N, 5, 4)
-	for i := 0; i < len(packets); i++ {
-		h(packets[i])
-	}
-	f.Close()
-	b.Logf("%d packets in %s", b.N, time.Since(now))
-}
-
-func BenchmarkPcapFile(b *testing.B) {
-	f, err := ioutil.TempFile("", "pcap_file")
-	if err != nil {
-		b.Error(err)
-		return
-	}
-	defer os.Remove(f.Name())
-	h, _ := PcapDumpHandler(f, layers.LinkTypeLoop)
-	packets := Packets(1, b.N, 5, 4)
-	for i := 0; i < len(packets); i++ {
-		h(packets[i])
-	}
-	name := f.Name()
-	f.Close()
-	b.ResetTimer()
-	var l *Listener
-	l, err = NewListener(name, 8000, "", EnginePcapFile, true)
-	if err != nil {
-		b.Error(err)
-		return
-	}
-	err = l.Activate()
-	if err != nil {
-		b.Error(err)
-		return
-	}
-	now := time.Now()
-	pckts := 0
-	ctx, cancel := context.WithCancel(context.Background())
-	defer cancel()
-	if err = l.Listen(ctx, func(packet *Packet) {
-		if packet.Info.CaptureLength != 49 {
-			b.Errorf("expected packet length to be %d, got %d", 49, packet.Info.CaptureLength)
-		}
-		pckts++
-	}); err != nil {
-		b.Error(err)
-	}
-	b.Logf("%d/%d packets in %s", pckts, b.N, time.Since(now))
-}
-
-// used to benchmark sock engine
-var buf [1024]byte
-
-func init() {
-	for i := 0; i < len(buf); i++ {
-		buf[i] = 0xff
-	}
-}
-
-func handler(n, counter *int32) Handler {
-	return func(p *Packet) {
-		nn := int32(len(p.Data))
-		atomic.AddInt32(n, nn)
-		atomic.AddInt32(counter, 1)
-	}
-}
-
-func BenchmarkPcap(b *testing.B) {
-	var err error
-	n := new(int32)
-	counter := new(int32)
-	l, err := NewListener(LoopBack.Name, 8000, "", EnginePcap, false)
-	if err != nil {
-		b.Error(err)
-		return
-	}
-	l.PcapOptions.BPFFilter = "udp dst port 8000 and host 127.0.0.1"
-	err = l.Activate()
-	if err != nil {
-		b.Error(err)
-		return
-	}
-	errCh := l.ListenBackground(context.Background(), handler(n, counter))
-	select {
-	case <-l.Reading:
-	case err = <-errCh:
-		b.Error(err)
-		return
-	}
-	var conn net.Conn
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		b.StopTimer()
-		conn, err = net.Dial("udp", "127.0.0.1:8000")
-		if err != nil {
-			b.Error(err)
-			return
-		}
-		b.StartTimer()
-		_, err = conn.Write(buf[:])
-		if err != nil {
-			b.Error(err)
-			return
+	for _, nic := range listener.Interfaces {
+		if (len(nic.Addresses)) == 0 {
+			t.Errorf("nic %s was captured with 0 addresses", nic.Name)
 		}
 	}
-	b.ReportMetric(float64(atomic.LoadInt32(n)), "buf")
-	b.ReportMetric(float64(atomic.LoadInt32(counter)), "packets")
-}
 
-func BenchmarkRawSocket(b *testing.B) {
-	var err error
-	n := new(int32)
-	counter := new(int32)
-	l, err := NewListener(LoopBack.Name, 8000, "", EngineRawSocket, false)
-	if err != nil {
-		b.Error(err)
-		return
+	if listener.loopIndex == 99999 {
+		t.Errorf("loopback nic index was not found")
 	}
-	l.PcapOptions.BPFFilter = "udp dst port 8000 and host 127.0.0.1"
-	err = l.Activate()
-	if err != nil {
-		b.Error(err)
-		return
-	}
-	errCh := l.ListenBackground(context.Background(), handler(n, counter))
-	select {
-	case <-l.Reading:
-	case err = <-errCh:
-		b.Error(err)
-		return
-	}
-	var conn net.Conn
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		b.StopTimer()
-		conn, err = net.Dial("udp", "127.0.0.1:8000")
-		if err != nil {
-			b.Error(err)
-			return
-		}
-		b.StartTimer()
-		_, err = conn.Write(buf[:])
-		if err != nil {
-			b.Error(err)
-			return
-		}
-	}
-	b.ReportMetric(float64(atomic.LoadInt32(n)), "buf")
-	b.ReportMetric(float64(atomic.LoadInt32(counter)), "packets")
 }

capture/doc.go

@@ -22,7 +22,7 @@ if err := listener.Listen(context.Background(), handler); err != nil {
 	 // handle error
 }
 // or
-errCh := listener.ListenBackground(context.Background(), handler) // runs in the backgorund
+errCh := listener.ListenBackground(context.Background(), handler) // runs in the background
 select {
 case err := <- errCh:
 	// handle error

capture/dump.go

@@ -54,7 +54,7 @@ func NewWriterNanos(w io.Writer) *Writer {
 // NewWriter returns a new writer object, for writing packet data out
 // to the given writer.  If this is a new empty writer (as opposed to
 // an append), you must call WriteFileHeader before WritePacket.
-// Packet timestamps are written witn microsecond precision.
+// Packet timestamps are written with microsecond precision.
 //
 //  // Write a new file:
 //  f, _ := os.Create("/tmp/file.pcap")

capture/packet.go

@@ -1,180 +0,0 @@
-package capture
-
-import (
-	"github.com/google/gopacket"
-	"github.com/google/gopacket/layers"
-)
-
-// Packet properties of a pcaket
-type Packet struct {
-	Data         []byte
-	LinkLayer    []byte
-	NetLayer     []byte
-	NetOptsLen   int // length of extension headers(IPV6) or options(IPV4)
-	TransLayer   []byte
-	TransOptsLen int // length of tcp options
-	Payload      []byte
-
-	LinkType int
-	Info     *gopacket.CaptureInfo
-	Err      error
-}
-
-// NewPacket parses packet and stop at the first error encountered
-// pckt.Error will be nil if packet was parsed successfully
-func NewPacket(data []byte, lType, lTypeLen int, cp *gopacket.CaptureInfo) (p *Packet) {
-	p = new(Packet)
-	p.Info = cp
-	p.Data = data
-	if len(data) < lTypeLen {
-		p.Err = ErrHdrLength("Link")
-		return
-	}
-	p.LinkType = lType
-	p.LinkLayer = data[:lTypeLen]
-	if len(data) <= lTypeLen {
-		p.Err = ErrHdrMissing("IPv4 or IPv6")
-		return
-	}
-	data = data[lTypeLen:]
-	var proto byte
-	if data[0]>>4 == 4 {
-		// IPv4 header
-		if len(data) < 20 {
-			p.Err = ErrHdrLength("IPv4")
-			return
-		}
-		proto = data[9]
-		ihl := int(data[0]&0x0F) * 4
-		if ihl < 20 {
-			p.Err = ErrHdrInvalid("IPv4's IHL")
-			return
-		}
-		if len(data) < ihl {
-			p.Err = ErrHdrLength("IPv4 opts")
-			p.NetLayer = data[:20]
-			return
-		}
-		p.NetOptsLen = ihl - 20
-		p.NetLayer = data[:ihl]
-	} else if data[0]>>4 == 6 {
-		if len(data) < 40 {
-			p.Err = ErrHdrLength("IPv6")
-			return
-		}
-		proto = data[6]
-		totalLen := 40
-		for ipv6ExtensionHdr(proto) {
-			hdr := len(data) - totalLen
-			if hdr < 8 {
-				p.Err = ErrHdrExpected("IPv6 opts")
-				break
-			}
-			extLen := 8
-			if proto != 44 {
-				extLen = int(data[totalLen+1]+1) * 8
-			}
-			if hdr < extLen {
-				p.Err = ErrHdrLength("IPv6 opts")
-				break
-			}
-			p.NetOptsLen += extLen
-			proto = data[totalLen]
-			totalLen += extLen
-		}
-		p.NetLayer = data[:totalLen]
-		if p.Err != nil {
-			return
-		}
-	} else {
-		p.Err = ErrHdrExpected("IPv4 or IPv6")
-		return
-	}
-	if proto != 6 {
-		p.Err = ErrHdrExpected("TCP")
-		return
-	}
-	if len(data) <= len(p.NetLayer) {
-		p.Err = ErrHdrMissing("TCP")
-		return
-	}
-	data = data[len(p.NetLayer):]
-	// TCP header
-	if len(data) < 20 {
-		p.Err = ErrHdrLength("TCP")
-		return
-	}
-	dOf := int(data[12]>>4) * 4
-	if dOf < 20 {
-		p.Err = ErrHdrInvalid("TCP's data offset")
-		return
-	}
-	if len(data) < dOf {
-		p.Err = ErrHdrLength("TCP opts")
-		p.TransLayer = data[:20]
-		return
-	}
-	p.TransLayer = data[:dOf]
-	p.TransOptsLen = dOf - 20
-	if len(data) > dOf {
-		p.Payload = data[dOf:]
-	}
-	return
-}
-
-// ErrHdrLength returned on short header length
-type ErrHdrLength string
-
-func (err ErrHdrLength) Error() string {
-	return "short " + string(err) + " length"
-}
-
-// ErrHdrMissing returned on missing header(s)
-type ErrHdrMissing string
-
-func (err ErrHdrMissing) Error() string {
-	return "missing " + string(err) + " header(s)"
-}
-
-// ErrHdrExpected returned when header(s) are different from the one expected
-type ErrHdrExpected string
-
-func (err ErrHdrExpected) Error() string {
-	return "expected " + string(err) + " header(s)"
-}
-
-// ErrHdrInvalid returned when header(s) are different from the one expected
-type ErrHdrInvalid string
-
-func (err ErrHdrInvalid) Error() string {
-	return "invalid " + string(err) + " value"
-}
-
-func pcapLinkTypeLength(lType int) (int, bool) {
-	switch layers.LinkType(lType) {
-	case layers.LinkTypeEthernet:
-		return 14, true
-	case layers.LinkTypeNull, layers.LinkTypeLoop:
-		return 4, true
-	case layers.LinkTypeRaw, 12, 14:
-		return 0, true
-	case layers.LinkTypeIPv4, layers.LinkTypeIPv6:
-		// (TODO:) look out for IP encapsulation?
-		return 0, true
-	case layers.LinkTypeLinuxSLL:
-		return 16, true
-	case layers.LinkTypeFDDI:
-		return 13, true
-	case 226 /*DLT_IPNET*/ :
-		// https://www.tcpdump.org/linktypes/LINKTYPE_IPNET.html
-		return 24, true
-	default:
-		return 0, false
-	}
-}
-
-// https://en.wikipedia.org/wiki/IPv6_packet#Extension_headers
-func ipv6ExtensionHdr(b byte) bool {
-	// TODO: support all extension headers
-	return b == 0 || b == 43 || b == 44
-}

capture/packet_test.go

@@ -1,197 +0,0 @@
-package capture
-
-import (
-	"encoding/binary"
-	"errors"
-	"testing"
-	"time"
-
-	"github.com/google/gopacket"
-	"github.com/google/gopacket/layers"
-)
-
-func generateHeader4(seq uint32, length uint16) []byte {
-	hdr := make([]byte, 4+24+24, 4+24+24)
-	binary.BigEndian.PutUint32(hdr, uint32(layers.ProtocolFamilyIPv4))
-
-	ip := hdr[4:]
-	ip[0] = 4<<4 | 6
-	binary.BigEndian.PutUint16(ip[2:4], length+24+24)
-	ip[9] = uint8(layers.IPProtocolTCP)
-	copy(ip[12:16], []byte{127, 0, 0, 1})
-	copy(ip[16:], []byte{127, 0, 0, 1})
-
-	// set tcp header
-	tcp := ip[24:]
-	tcp[12] = 6 << 4
-	binary.BigEndian.PutUint16(tcp, 5535)
-	binary.BigEndian.PutUint16(tcp[2:], 8000)
-	binary.BigEndian.PutUint32(tcp[4:], seq)
-	return hdr
-}
-
-func generateHeader6(seq uint32, length uint16) []byte {
-	hdr := make([]byte, 4+40+32+24, 4+40+32+24)
-	binary.BigEndian.PutUint32(hdr, uint32(layers.ProtocolFamilyIPv6Linux))
-
-	ip := hdr[4:]
-	ip[0] = 6 << 4
-	binary.BigEndian.PutUint16(ip[4:], length+32+24)
-	var ipAddr [16]byte
-	ipAddr[15] = 0x01
-	copy(ip[8:], ip[:])
-	copy(ip[24:], ip[:])
-	copy(ip[40:], []byte{
-		// net-layer (IPv6-Opts)
-		0x2b, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-		0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-		uint8(layers.IPProtocolTCP), 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-		0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	})
-
-	// set tcp header
-	tcp := ip[40+32:]
-	tcp[12] = 6 << 4
-	binary.BigEndian.PutUint16(tcp, 5535)
-	binary.BigEndian.PutUint16(tcp[2:], 8000)
-	binary.BigEndian.PutUint32(tcp[4:], seq)
-	return hdr
-}
-
-func Packets(start uint32, _len int, length uint16, version byte) []*Packet {
-	var packets = make([]*Packet, _len)
-	for i := start; i < start+uint32(_len); i++ {
-		var h []byte
-		if version == 4 {
-			h = generateHeader4(i, length)
-		} else {
-			h = generateHeader6(i, length)
-		}
-		d := append(h, make([]byte, int(length))...)
-		ci := &gopacket.CaptureInfo{Length: len(d), CaptureLength: len(d), Timestamp: time.Now()}
-		packets[i-start] = NewPacket(d, int(layers.LinkTypeLoop), 4, ci)
-	}
-	return packets
-}
-
-func TestIPv4Packet(t *testing.T) {
-	pckt := packet(append(generateHeader4(1024, 10), make([]byte, 10)...))
-	if pckt.Err != nil {
-		t.Error(pckt)
-		return
-	}
-	if err := packet(pckt.Data[:2]).Err; !errors.Is(err, ErrHdrLength("Link")) {
-		t.Errorf("should fail with %q, got %q", ErrHdrLength("Link"), err)
-		return
-	}
-	if err := packet(pckt.Data[:20]).Err; !errors.Is(err, ErrHdrLength("IPv4")) {
-		t.Errorf("should fail with %q, got %q", ErrHdrLength("IPv4"), err)
-		return
-	}
-	if err := packet(pckt.Data[:27]).Err; !errors.Is(err, ErrHdrLength("IPv4 opts")) {
-		t.Errorf("should fail with %q, got %q", ErrHdrLength("IPv4 opts"), err)
-		return
-	}
-	if err := packet(pckt.Data[:40]).Err; !errors.Is(err, ErrHdrLength("TCP")) {
-		t.Errorf("should fail with %q, got %q", ErrHdrLength("TCP opts"), err)
-		return
-	}
-	if err := packet(pckt.Data[:50]).Err; !errors.Is(err, ErrHdrLength("TCP opts")) {
-		t.Errorf("should fail with %q, got %q", ErrHdrLength("TCP opts"), err)
-		return
-	}
-	pckt.TransLayer[12] = 0x10
-	if err := packet(pckt.Data[:50]).Err; !errors.Is(err, ErrHdrInvalid("TCP's data offset")) {
-		t.Errorf("should fail with %q, got %q", ErrHdrInvalid("TCP's data offset"), err)
-		return
-	}
-	pckt.TransLayer[12] = 0x60
-	if err := packet(pckt.Data[:28]).Err; !errors.Is(err, ErrHdrMissing("TCP")) {
-		t.Errorf("should fail with %q, got %q", ErrHdrMissing("TCP"), err)
-		return
-	}
-	pckt.NetLayer[9] = 0x02
-	if err := packet(pckt.Data).Err; !errors.Is(err, ErrHdrExpected("TCP")) {
-		t.Errorf("should fail with %q, got %q", ErrHdrExpected("TCP"), err)
-		return
-	}
-	pckt.NetLayer[9] = 0x06
-	pckt.NetLayer[0] = 0x44
-	if err := packet(pckt.Data).Err; !errors.Is(err, ErrHdrInvalid("IPv4's IHL")) {
-		t.Errorf("should fail with %q, got %q", ErrHdrInvalid("IPv4's IHL"), err)
-		return
-	}
-	pckt.NetLayer[0] = 0x56
-	if err := packet(pckt.Data).Err; !errors.Is(err, ErrHdrExpected("IPv4 or IPv6")) {
-		t.Errorf("should fail with %q, got %q", ErrHdrExpected("IPv4 or IPv6"), err)
-		return
-	}
-}
-
-func TestIPv6Packet(t *testing.T) {
-	pckt := packet(append(generateHeader6(1024, 10), make([]byte, 10)...))
-	if pckt.Err != nil {
-		t.Error(pckt)
-		return
-	}
-	if err := packet(pckt.Data[:4]).Err; !errors.Is(err, ErrHdrMissing("IPv4 or IPv6")) {
-		t.Errorf("should fail with %q, got %q", ErrHdrMissing("IPv4 or IPv6"), err)
-		return
-	}
-	if err := packet(pckt.Data[:40]).Err; !errors.Is(err, ErrHdrLength("IPv6")) {
-		t.Errorf("should fail with %q, got %q", ErrHdrLength("IPv6"), err)
-		return
-	}
-	if err := packet(pckt.Data[:52]).Err; !errors.Is(err, ErrHdrLength("IPv6 opts")) {
-		t.Errorf("should fail with %q, got %q", ErrHdrLength("IPv6 opts"), err)
-		return
-	}
-	if err := packet(pckt.Data[:80]).Err; !errors.Is(err, ErrHdrLength("TCP")) {
-		t.Errorf("should fail with %q, got %q", ErrHdrLength("TCP opts"), err)
-		return
-	}
-	if err := packet(pckt.Data[:98]).Err; !errors.Is(err, ErrHdrLength("TCP opts")) {
-		t.Errorf("should fail with %q, got %q", ErrHdrLength("TCP opts"), err)
-		return
-	}
-	pckt.TransLayer[12] = 0x10
-	if err := packet(pckt.Data).Err; !errors.Is(err, ErrHdrInvalid("TCP's data offset")) {
-		t.Errorf("should fail with %q, got %q", ErrHdrInvalid("TCP's data offset"), err)
-		return
-	}
-	pckt.TransLayer[12] = 0x60
-	if err := packet(pckt.Data[:76]).Err; !errors.Is(err, ErrHdrMissing("TCP")) {
-		t.Errorf("should fail with %q, got %q", ErrHdrMissing("TCP"), err)
-		return
-	}
-	pckt.NetLayer[56] = 0x02
-	if err := packet(pckt.Data).Err; !errors.Is(err, ErrHdrExpected("TCP")) {
-		t.Errorf("should fail with %q, got %q", ErrHdrExpected("TCP"), err)
-		return
-	}
-	pckt.NetLayer[56] = 0x06
-}
-
-func packet(data []byte) *Packet {
-	return NewPacket(data, int(layers.LinkTypeLoop), 4, &gopacket.CaptureInfo{})
-}
-
-func BenchmarkNewPacketIPv4(b *testing.B) {
-	data := append(generateHeader4(1204, 10), make([]byte, 10)...)
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		if err := packet(data).Err; err != nil {
-			b.Error(err)
-		}
-	}
-}
-
-func BenchmarkNewPacketIPv6(b *testing.B) {
-	data := append(generateHeader6(1024, 10), make([]byte, 10)...)
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		if err := packet(data).Err; err != nil {
-			b.Error(err)
-		}
-	}
-}

capture/sock_linux.go

@@ -1,3 +1,5 @@
+// +build linux
+
 package capture
 
 import (
@@ -44,7 +46,23 @@ type SockRaw struct {
 }
 
 // NewSocket returns new M'maped sock_raw on packet version 2.
-func NewSocket(ifi net.Interface) (*SockRaw, error) {
+func NewSocket(pifi pcap.Interface) (*SockRaw, error) {
+	var ifi net.Interface
+
+	infs, _ := net.Interfaces()
+	found := false
+	for _, i := range infs {
+		if i.Name == pifi.Name {
+			ifi = i
+			found = true
+			break
+		}
+	}
+
+	if !found {
+		return nil, fmt.Errorf("can't find matching interface")
+	}
+
 	// sock create
 	fd, err := unix.Socket(unix.AF_PACKET, unix.SOCK_RAW, int(ETHALL))
 	if err != nil {
@@ -217,7 +235,7 @@ func (sock *SockRaw) SetBPFFilter(expr string) error {
 	return unix.SetsockoptSockFprog(sock.fd, unix.SOL_SOCKET, unix.SO_ATTACH_FILTER, fprog)
 }
 
-// SetPromiscuous sets promiscous mode to the required value. for better result capture on all interfaces instead.
+// SetPromiscuous sets promiscuous mode to the required value. for better result capture on all interfaces instead.
 // If it is enabled, traffic not destined for the interface will also be captured.
 func (sock *SockRaw) SetPromiscuous(b bool) error {
 	sock.mu.Lock()

capture/sock_others.go

@@ -4,10 +4,11 @@ package capture
 
 import (
 	"errors"
-	"net"
+
+	"github.com/google/gopacket/pcap"
 )
 
 // NewSocket returns new M'maped sock_raw on packet version 2.
-func NewSocket(_ net.Interface) (Socket, error) {
+func NewSocket(_ pcap.Interface) (Socket, error) {
 	return nil, errors.New("afpacket socket is only available on linux")
 }

capture/vxlan.go

@@ -0,0 +1,100 @@
+package capture
+
+import (
+	"errors"
+	"fmt"
+	"github.com/google/gopacket"
+	"github.com/google/gopacket/layers"
+	"net"
+	"time"
+)
+
+const VxLanPacketSize = 1526 //vxlan 8 B + ethernet II 1518 B
+
+type vxlanHandle struct {
+	connection    *net.UDPConn
+	packetChannel chan gopacket.Packet
+	vnis          []int
+}
+
+func newVXLANHandler(port int, vnis []int) (*vxlanHandle, error) {
+	if port == 0 {
+		port = 4789
+	}
+
+	addr := net.UDPAddr{
+		Port: port,
+		IP:   net.ParseIP("0.0.0.0"),
+	}
+
+	vxlanHandle := &vxlanHandle{}
+	con, err := net.ListenUDP("udp", &addr)
+	if err != nil {
+		return nil, fmt.Errorf(err.Error())
+	}
+	vxlanHandle.connection = con
+	vxlanHandle.packetChannel = make(chan gopacket.Packet, 1000)
+	vxlanHandle.vnis = vnis
+	go vxlanHandle.reader()
+
+	return vxlanHandle, nil
+}
+
+func (v *vxlanHandle) reader() {
+	for {
+		inputBytes := make([]byte, VxLanPacketSize)
+		length, _, err := v.connection.ReadFromUDP(inputBytes)
+		if err != nil {
+			if errors.Is(err, net.ErrClosed) {
+				return
+			}
+			continue
+		}
+		packet := gopacket.NewPacket(inputBytes[:length], layers.LayerTypeVXLAN, gopacket.NoCopy)
+		ci := packet.Metadata()
+		ci.Timestamp = time.Now()
+		ci.CaptureLength = length
+		ci.Length = length
+
+		if len(v.vnis) > 0 && !v.vniIsAllowed(packet) {
+			continue
+		}
+
+		v.packetChannel <- packet
+	}
+}
+
+func (v *vxlanHandle) vniIsAllowed(packet gopacket.Packet) bool {
+	defaultState := false
+	if layer := packet.Layer(layers.LayerTypeVXLAN); layer != nil {
+		vxlan, _ := layer.(*layers.VXLAN)
+		for _, vn := range v.vnis {
+			if vn > 0 && int(vxlan.VNI) == vn {
+				return true
+			}
+
+			if vn < 0 {
+				if int(vxlan.VNI) == -vn {
+					return false
+				}
+				defaultState = true
+			}
+		}
+	}
+	return defaultState
+}
+
+func (v *vxlanHandle) ReadPacketData() ([]byte, gopacket.CaptureInfo, error) {
+	packet := <-v.packetChannel
+	layer := packet.Layer(layers.LayerTypeVXLAN)
+	bytes := layer.LayerPayload()
+
+	return bytes, packet.Metadata().CaptureInfo, nil
+}
+
+func (v *vxlanHandle) Close() error {
+	if v.connection != nil {
+		return v.connection.Close()
+	}
+	return nil
+}

docs/Capturing-and-replaying-traffic.md

@@ -37,6 +37,11 @@ sudo gor --input-raw :80 --input-raw-engine "raw_socket" --output-http "http://s
 
 You can read more about [[Replaying HTTP traffic]].
 
+You can use VXLAN or traffic mirroring from AWS to capture the traffic. The 4789 UDP port will be opened and that works as you are launched GoReplay on the source machine.
+
+```
+gor --input-raw :80 --input-raw-engine vxlan -output-stdout
+```
 
 ### Tracking original IP addresses
 You can use `--input-raw-realip-header` option to specify header name: If not blank, injects header with given name and real IP value to the request payload. Usually, this header should be named: `X-Real-IP`, but you can specify any name.

elasticsearch.go

@@ -57,7 +57,7 @@ type ESRequestResponse struct {
 //
 // Proper format is: scheme://[userinfo@]host/index_name
 // userinfo is: user[:password]
-// net/url.Parse() does not fail if scheme is not provided but actualy does not
+// net/url.Parse() does not fail if scheme is not provided but actually does not
 // handle URI properly.
 // So we must 'validate' URI format to match requirements to use net/url.Parse()
 func parseURI(URI string) (err error, index string) {

emitter.go

@@ -6,9 +6,9 @@ import (
 	"io"
 	"log"
 	"sync"
-	"time"
 
 	"github.com/buger/goreplay/byteutils"
+	"github.com/coocood/freecache"
 )
 
 // Emitter represents an abject to manage plugins communication
@@ -76,9 +76,7 @@ func (e *Emitter) Close() {
 func CopyMulty(src PluginReader, writers ...PluginWriter) error {
 	wIndex := 0
 	modifier := NewHTTPModifier(&Settings.ModifierConfig)
-	filteredRequests := make(map[string]int64)
-	filteredRequestsLastCleanTime := time.Now().UnixNano()
-	filteredCount := 0
+	filteredRequests := freecache.NewCache(200 * 1024 * 1024) // 200M
 
 	for {
 		msg, err := src.PluginRead()
@@ -97,7 +95,7 @@ func CopyMulty(src PluginReader, writers ...PluginWriter) error {
 				Debug(2, fmt.Sprintf("[EMITTER] Found malformed record %q from %q", msg.Meta, src))
 				continue
 			}
-			requestID := byteutils.SliceToString(meta[1])
+			requestID := meta[1]
 			// start a subroutine only when necessary
 			if Settings.Verbose >= 3 {
 				Debug(3, "[EMITTER] input: ", byteutils.SliceToString(msg.Meta[:len(msg.Meta)-1]), " from: ", src)
@@ -108,16 +106,15 @@ func CopyMulty(src PluginReader, writers ...PluginWriter) error {
 					msg.Data = modifier.Rewrite(msg.Data)
 					// If modifier tells to skip request
 					if len(msg.Data) == 0 {
-						filteredRequests[requestID] = time.Now().UnixNano()
-						filteredCount++
+						filteredRequests.Set(requestID, []byte{}, 60) //
 						continue
 					}
 					Debug(3, "[EMITTER] Rewritten input:", requestID, "from:", src)
 
 				} else {
-					if _, ok := filteredRequests[requestID]; ok {
-						delete(filteredRequests, requestID)
-						filteredCount--
+					_, err := filteredRequests.Get(requestID)
+					if err == nil {
+						filteredRequests.Del(requestID)
 						continue
 					}
 				}
@@ -152,26 +149,11 @@ func CopyMulty(src PluginReader, writers ...PluginWriter) error {
 				}
 			} else {
 				for _, dst := range writers {
-					if _, err := dst.PluginWrite(msg); err != nil {
+					if _, err := dst.PluginWrite(msg); err != nil && err != io.ErrClosedPipe {
 						return err
 					}
 				}
 			}
 		}
-
-		// Run GC on each 1000 request
-		if filteredCount > 0 && filteredCount%1000 == 0 {
-			// Clean up filtered requests for which we didn't get a response to filter
-			now := time.Now().UnixNano()
-			if now-filteredRequestsLastCleanTime > int64(60*time.Second) {
-				for k, v := range filteredRequests {
-					if now-v > int64(60*time.Second) {
-						delete(filteredRequests, k)
-						filteredCount--
-					}
-				}
-				filteredRequestsLastCleanTime = time.Now().UnixNano()
-			}
-		}
 	}
 }

examples/middleware/echo.sh

@@ -8,7 +8,7 @@
 # 
 
 function log {
-    if [[ ! -v GOR_TEST ]]; then # if we are not testing
+    if [[ -n "$GOR_TEST" ]]; then # if we are not testing
     # Logging to stderr, because stdout/stdin used for data transfer
     >&2 echo "[DEBUG][ECHO] $1"
     fi

go.mod

@@ -1,6 +1,6 @@
 module github.com/buger/goreplay
 
-go 1.14
+go 1.15
 
 require (
 	github.com/Shopify/sarama v1.26.4
@@ -8,12 +8,14 @@ require (
 	github.com/aws/aws-sdk-go v1.33.2
 	github.com/bitly/go-hostpool v0.1.0 // indirect
 	github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 // indirect
-	github.com/google/gopacket v1.1.18
+	github.com/coocood/freecache v1.2.0
+	github.com/google/gopacket v1.1.20-0.20210429153827-3eaba0894325
 	github.com/klauspost/compress v1.10.10 // indirect
 	github.com/mattbaird/elastigo v0.0.0-20170123220020-2fe47fd29e4b
 	github.com/pierrec/lz4 v2.5.2+incompatible // indirect
 	github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0 // indirect
 	github.com/smartystreets/goconvey v1.6.4 // indirect
-	golang.org/x/net v0.0.0-20200707034311-ab3426394381 // indirect
+	github.com/stretchr/testify v1.5.1
+	golang.org/x/net v0.0.0-20200707034311-ab3426394381
 	golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd
 )

go.sum

@@ -1,3 +1,4 @@
+github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
 github.com/Shopify/sarama v1.26.4 h1:+17TxUq/PJEAfZAll0T7XJjSgQWCpaQSoki/x5yN8o8=
 github.com/Shopify/sarama v1.26.4/go.mod h1:NbSGBSSndYaIhRcBtY9V0U7AyH+x71bG668AuWys/yU=
 github.com/Shopify/toxiproxy v2.1.4+incompatible h1:TKdv8HiTLgE5wdJuEML90aBgNWsokNbMijUGhmcoBJc=
@@ -10,7 +11,10 @@ github.com/bitly/go-hostpool v0.1.0 h1:XKmsF6k5el6xHG3WPJ8U0Ku/ye7njX7W81Ng7O2io
 github.com/bitly/go-hostpool v0.1.0/go.mod h1:4gOCgp6+NZnVqlKyZ/iBZFTAJKembaVENUpMkpg42fw=
 github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 h1:DDGfHa7BWjL4YnC6+E63dPcxHo2sUxDIu8g3QgEJdRY=
 github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4=
-github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
+github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
+github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
+github.com/coocood/freecache v1.2.0 h1:p8RhjN6Y4DRBIMzdRlm1y+M7h7YJxye3lGW8/VvzCz0=
+github.com/coocood/freecache v1.2.0/go.mod h1:OKrEjkGVoxZhyWAJoeFi5BMLUJm2Tit0kpGkIr7NGYY=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -30,8 +34,8 @@ github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEW
 github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4=
 github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/gopacket v1.1.18 h1:lum7VRA9kdlvBi7/v2p7/zcbkduHaCH/SVVyurs7OpY=
-github.com/google/gopacket v1.1.18/go.mod h1:UdDNZ1OO62aGYVnPhxT1U6aI7ukYtA/kB8vaU0diBUM=
+github.com/google/gopacket v1.1.20-0.20210429153827-3eaba0894325 h1:YmIcZ5Var3BAQ64AW98Iiys5Ih4fiU0xK41+8isC5Ec=
+github.com/google/gopacket v1.1.20-0.20210429153827-3eaba0894325/go.mod h1:riddUzxTSBpJXk3qBHtYr4qOhFhT6k/1c0E3qkQjQpA=
 github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8=
 github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
 github.com/hashicorp/go-uuid v1.0.2 h1:cfejS+Tpcp13yd5nYHWDI6qVCny6wyX2Mt5SGur2IGE=
@@ -42,7 +46,6 @@ github.com/jmespath/go-jmespath v0.3.0 h1:OS12ieG61fsCg5+qLJ+SsW9NicxNkg3b25OyT2
 github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik=
 github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
 github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
-github.com/klauspost/compress v1.9.8 h1:VMAMUUOh+gaxKTMk+zqbjsSjsIcUcL/LF4o63i82QyA=
 github.com/klauspost/compress v1.9.8/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
 github.com/klauspost/compress v1.10.10 h1:a/y8CglcM7gLGYmlbP/stPE5sR3hbhFRUjCBfd/0B3I=
 github.com/klauspost/compress v1.10.10/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
@@ -67,37 +70,45 @@ github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykE
 github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
 github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s=
 github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
+github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4=
 github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
+github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE=
+github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU=
+github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0=
 github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I=
 github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/crypto v0.0.0-20200204104054-c9f3fb736b72 h1:+ELyKg6m8UBf0nPFSqD0mi7zUfwPyXo23HNjMnXPz7w=
+golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20200204104054-c9f3fb736b72/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
+golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20200202094626-16171245cfb2 h1:CCH4IOTTfewWjGOlSp+zGcjutRKlBEZQ6wTn8ozI/nI=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200707034311-ab3426394381 h1:VXak5I6aEWmAXeQjA+QSZzlgNrpq9mjcfDemuexIKsU=
 golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20190405154228-4b34438f7a67/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190606203320-7fc4e5ec1444/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200217220822-9197077df867/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884=
 golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
@@ -111,7 +122,6 @@ gopkg.in/jcmturner/gokrb5.v7 v7.5.0 h1:a9tsXlIDD9SKxotJMK3niV7rPZAJeX2aD/0yg3qlI
 gopkg.in/jcmturner/gokrb5.v7 v7.5.0/go.mod h1:l8VISx+WGYp+Fp7KRbsiUuXTTOnxIc3Tuvyavf11/WM=
 gopkg.in/jcmturner/rpc.v1 v1.1.0 h1:QHIUxTX1ISuAv9dD2wJ9HWQVuWDX/Zc0PfeC2tjc4rU=
 gopkg.in/jcmturner/rpc.v1 v1.1.0/go.mod h1:YIdkC4XfD6GXbzje11McwsDuOlZQSb9W4vfLvuNnlv8=
-gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=
 gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

gor.go

@@ -3,15 +3,16 @@
 package main
 
 import (
+	"expvar"
 	"flag"
+	"fmt"
 	"log"
 	"net/http"
 	"net/http/httputil"
-	_ "net/http/pprof"
+	httppptof "net/http/pprof"
 	"os"
 	"os/signal"
 	"runtime"
-	_ "runtime/debug"
 	"runtime/pprof"
 	"syscall"
 	"time"
@@ -22,8 +23,42 @@ var (
 	memprofile = flag.String("memprofile", "", "write memory profile to this file")
 )
 
-func loggingMiddleware(next http.Handler) http.Handler {
+func init() {
+	var defaultServeMux http.ServeMux
+	http.DefaultServeMux = &defaultServeMux
+
+	http.HandleFunc("/debug/vars", func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "application/json; charset=utf-8")
+		fmt.Fprintf(w, "{\n")
+		first := true
+		expvar.Do(func(kv expvar.KeyValue) {
+			if kv.Key == "memstats" || kv.Key == "cmdline" {
+				return
+			}
+
+			if !first {
+				fmt.Fprintf(w, ",\n")
+			}
+			first = false
+			fmt.Fprintf(w, "%q: %s", kv.Key, kv.Value)
+		})
+		fmt.Fprintf(w, "\n}\n")
+	})
+
+	http.HandleFunc("/debug/pprof/", httppptof.Index)
+	http.HandleFunc("/debug/pprof/cmdline", httppptof.Cmdline)
+	http.HandleFunc("/debug/pprof/profile", httppptof.Profile)
+	http.HandleFunc("/debug/pprof/symbol", httppptof.Symbol)
+	http.HandleFunc("/debug/pprof/trace", httppptof.Trace)
+}
+
+func loggingMiddleware(addr string, next http.Handler) http.Handler {
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path == "/loop" {
+			_, err := http.Get("http://" + addr)
+			log.Println(err)
+		}
+
 		rb, _ := httputil.DumpRequest(r, false)
 		log.Println(string(rb))
 		next.ServeHTTP(w, r)
@@ -45,7 +80,7 @@ func main() {
 
 		Debug(0, "Started example file server for current directory on address ", args[1])
 
-		log.Fatal(http.ListenAndServe(args[1], loggingMiddleware(http.FileServer(http.Dir(dir)))))
+		log.Fatal(http.ListenAndServe(args[1], loggingMiddleware(args[1], http.FileServer(http.Dir(dir)))))
 	} else {
 		flag.Parse()
 		checkSettings()
@@ -94,7 +129,6 @@ func main() {
 	}
 	emitter.Close()
 	os.Exit(exit)
-
 }
 
 func profileCPU(cpuprofile string) {

input_file.go

@@ -4,9 +4,12 @@ import (
 	"bufio"
 	"bytes"
 	"compress/gzip"
+	"container/heap"
 	"errors"
+	"expvar"
 	"fmt"
 	"io"
+	"math"
 	"os"
 	"path/filepath"
 	"strconv"
@@ -20,27 +23,73 @@ import (
 	"github.com/aws/aws-sdk-go/service/s3"
 )
 
-type fileInputReader struct {
-	reader    *bufio.Reader
+type filePayload struct {
 	data      []byte
-	file      io.ReadCloser
 	timestamp int64
-	closed    int32 // Value of 0 indicates that the file is still open.
-	s3        bool
 }
 
-func (f *fileInputReader) parseNext() error {
+// An IntHeap is a min-heap of ints.
+type payloadQueue struct {
+	sync.RWMutex
+	s []*filePayload
+}
+
+func (h payloadQueue) Len() int           { return len(h.s) }
+func (h payloadQueue) Less(i, j int) bool { return h.s[i].timestamp < h.s[j].timestamp }
+func (h payloadQueue) Swap(i, j int)      { h.s[i], h.s[j] = h.s[j], h.s[i] }
+
+func (h *payloadQueue) Push(x interface{}) {
+	// Push and Pop use pointer receivers because they modify the slice's length,
+	// not just its contents.
+	h.s = append(h.s, x.(*filePayload))
+}
+
+func (h *payloadQueue) Pop() interface{} {
+	old := h.s
+	n := len(old)
+	x := old[n-1]
+	h.s = old[0 : n-1]
+	return x
+}
+
+func (h payloadQueue) Idx(i int) *filePayload {
+	return h.s[i]
+}
+
+type fileInputReader struct {
+	reader    *bufio.Reader
+	file      io.ReadCloser
+	closed    int32 // Value of 0 indicates that the file is still open.
+	s3        bool
+	queue     payloadQueue
+	readDepth int
+	dryRun    bool
+	path      string
+}
+
+func (f *fileInputReader) parse(init chan struct{}) error {
 	payloadSeparatorAsBytes := []byte(payloadSeparator)
 	var buffer bytes.Buffer
+	var initialized bool
+
+	lineNum := 0
+
 	for {
 		line, err := f.reader.ReadBytes('\n')
+		lineNum++
 
 		if err != nil {
 			if err != io.EOF {
 				Debug(1, err)
-			} else {
-				f.Close()
 			}
+
+			f.Close()
+
+			if !initialized {
+				close(init)
+				initialized = true
+			}
+
 			return err
 		}
 
@@ -48,21 +97,61 @@ func (f *fileInputReader) parseNext() error {
 			asBytes := buffer.Bytes()
 			meta := payloadMeta(asBytes)
 
-			f.timestamp, _ = strconv.ParseInt(string(meta[2]), 10, 64)
-			f.data = asBytes[:len(asBytes)-1]
+			if len(meta) < 3 {
+				Debug(1, fmt.Sprintf("Found malformed record, file: %s, line %d", f.path, lineNum))
+				buffer = bytes.Buffer{}
+				continue
+			}
 
-			return nil
+			timestamp, _ := strconv.ParseInt(string(meta[2]), 10, 64)
+			data := asBytes[:len(asBytes)-1]
+
+			f.queue.Lock()
+			heap.Push(&f.queue, &filePayload{
+				timestamp: timestamp,
+				data:      data,
+			})
+			f.queue.Unlock()
+
+			for {
+				if f.queue.Len() < f.readDepth {
+					break
+				}
+
+				if !initialized {
+					close(init)
+					initialized = true
+				}
+
+				if !f.dryRun {
+					time.Sleep(100 * time.Millisecond)
+				}
+			}
+
+			buffer = bytes.Buffer{}
+			continue
 		}
 
 		buffer.Write(line)
 	}
-
 }
 
-func (f *fileInputReader) ReadPayload() []byte {
-	defer f.parseNext()
+func (f *fileInputReader) wait() {
+	for {
+		if atomic.LoadInt32(&f.closed) == 1 {
+			return
+		}
 
-	return f.data
+		if f.queue.Len() > 0 {
+			return
+		}
+
+		if !f.dryRun {
+			time.Sleep(100 * time.Millisecond)
+		}
+	}
+
+	return
 }
 
 // Close closes this plugin
@@ -75,7 +164,7 @@ func (f *fileInputReader) Close() error {
 	return nil
 }
 
-func newFileInputReader(path string) *fileInputReader {
+func newFileInputReader(path string, readDepth int, dryRun bool) *fileInputReader {
 	var file io.ReadCloser
 	var err error
 
@@ -90,7 +179,7 @@ func newFileInputReader(path string) *fileInputReader {
 		return nil
 	}
 
-	r := &fileInputReader{file: file, closed: 0}
+	r := &fileInputReader{path: path, file: file, closed: 0, readDepth: readDepth, dryRun: dryRun}
 	if strings.HasSuffix(path, ".gz") {
 		gzReader, err := gzip.NewReader(file)
 		if err != nil {
@@ -102,7 +191,11 @@ func newFileInputReader(path string) *fileInputReader {
 		r.reader = bufio.NewReader(file)
 	}
 
-	r.parseNext()
+	heap.Init(&r.queue)
+
+	init := make(chan struct{})
+	go r.parse(init)
+	<-init
 
 	return r
 }
@@ -116,16 +209,25 @@ type FileInput struct {
 	readers     []*fileInputReader
 	speedFactor float64
 	loop        bool
+	readDepth   int
+	dryRun      bool
+	maxWait     time.Duration
+
+	stats *expvar.Map
 }
 
 // NewFileInput constructor for FileInput. Accepts file path as argument.
-func NewFileInput(path string, loop bool) (i *FileInput) {
+func NewFileInput(path string, loop bool, readDepth int, maxWait time.Duration, dryRun bool) (i *FileInput) {
 	i = new(FileInput)
 	i.data = make(chan []byte, 1000)
 	i.exit = make(chan bool)
 	i.path = path
 	i.speedFactor = 1
 	i.loop = loop
+	i.readDepth = readDepth
+	i.stats = expvar.NewMap("file-" + path)
+	i.dryRun = dryRun
+	i.maxWait = maxWait
 
 	if err := i.init(); err != nil {
 		return
@@ -155,30 +257,31 @@ func (i *FileInput) init() (err error) {
 
 		resp, err := svc.ListObjects(params)
 		if err != nil {
-			Debug(0, "[INPUT-FILE] Error while retreiving list of files from S3", i.path, err)
+			Debug(2, "[INPUT-FILE] Error while retrieving list of files from S3", i.path, err)
 			return err
 		}
 
 		for _, c := range resp.Contents {
 			matches = append(matches, "s3://"+bucket+"/"+(*c.Key))
 		}
-	} else if matches, err = filepath.Glob(i.path); err != nil{ 
-			Debug(0, "[INPUT-FILE] Wrong file pattern", i.path, err)
-			return
-		
+	} else if matches, err = filepath.Glob(i.path); err != nil {
+		Debug(2, "[INPUT-FILE] Wrong file pattern", i.path, err)
+		return
 	}
 
 	if len(matches) == 0 {
-		Debug(0, "[INPUT-FILE] No files match pattern: ", i.path)
-		return errors.New("No matching files")
+		Debug(2, "[INPUT-FILE] No files match pattern: ", i.path)
+		return errors.New("no matching files")
 	}
 
 	i.readers = make([]*fileInputReader, len(matches))
 
 	for idx, p := range matches {
-		i.readers[idx] = newFileInputReader(p)
+		i.readers[idx] = newFileInputReader(p, i.readDepth, i.dryRun)
 	}
 
+	i.stats.Add("reader_count", int64(len(matches)))
+
 	return nil
 }
 
@@ -189,6 +292,7 @@ func (i *FileInput) PluginRead() (*Message, error) {
 	case <-i.exit:
 		return nil, ErrorStopped
 	case buf := <-i.data:
+		i.stats.Add("read_from", 1)
 		msg.Meta, msg.Data = payloadMetaWithBody(buf)
 		return &msg, nil
 	}
@@ -201,11 +305,17 @@ func (i *FileInput) String() string {
 // Find reader with smallest timestamp e.g next payload in row
 func (i *FileInput) nextReader() (next *fileInputReader) {
 	for _, r := range i.readers {
-		if r == nil || atomic.LoadInt32(&r.closed) != 0 {
+		if r == nil {
 			continue
 		}
 
-		if next == nil || r.timestamp < next.timestamp {
+		r.wait()
+
+		if r.queue.Len() == 0 {
+			continue
+		}
+
+		if next == nil || r.queue.Idx(0).timestamp < next.queue.Idx(0).timestamp {
 			next = r
 			continue
 		}
@@ -217,6 +327,11 @@ func (i *FileInput) nextReader() (next *fileInputReader) {
 func (i *FileInput) emit() {
 	var lastTime int64 = -1
 
+	var maxWait, firstWait, minWait int64
+	minWait = math.MaxInt64
+
+	i.stats.Add("negative_wait", 0)
+
 	for {
 		select {
 		case <-i.exit:
@@ -236,17 +351,48 @@ func (i *FileInput) emit() {
 			}
 		}
 
+		reader.queue.RLock()
+		payload := heap.Pop(&reader.queue).(*filePayload)
+		i.stats.Add("total_counter", 1)
+		i.stats.Add("total_bytes", int64(len(payload.data)))
+		reader.queue.RUnlock()
+
 		if lastTime != -1 {
-			diff := reader.timestamp - lastTime
-			lastTime = reader.timestamp
+			diff := payload.timestamp - lastTime
+
+			if firstWait == 0 {
+				firstWait = diff
+			}
 
 			if i.speedFactor != 1 {
 				diff = int64(float64(diff) / i.speedFactor)
 			}
 
-			time.Sleep(time.Duration(diff))
+			if i.maxWait > 0 && diff > int64(i.maxWait) {
+				diff = int64(i.maxWait)
+			}
+
+			if diff >= 0 {
+				lastTime = payload.timestamp
+
+				if !i.dryRun {
+					time.Sleep(time.Duration(diff))
+				}
+
+				i.stats.Add("total_wait", diff)
+
+				if diff > maxWait {
+					maxWait = diff
+				}
+
+				if diff < minWait {
+					minWait = diff
+				}
+			} else {
+				i.stats.Add("negative_wait", 1)
+			}
 		} else {
-			lastTime = reader.timestamp
+			lastTime = payload.timestamp
 		}
 
 		// Recheck if we have exited since last check.
@@ -254,12 +400,30 @@ func (i *FileInput) emit() {
 		case <-i.exit:
 			return
 		default:
-			i.data <- reader.ReadPayload()
+			if !i.dryRun {
+				i.data <- payload.data
+			}
 		}
 	}
 
-	Debug(0, fmt.Sprintf("[INPUT-FILE] FileInput: end of file '%s'\n", i.path))
+	i.stats.Set("first_wait", time.Duration(firstWait))
+	i.stats.Set("max_wait", time.Duration(maxWait))
+	i.stats.Set("min_wait", time.Duration(minWait))
 
+	Debug(2, fmt.Sprintf("[INPUT-FILE] FileInput: end of file '%s'\n", i.path))
+
+	if i.dryRun {
+		fmt.Printf("Records found: %v\nFiles processed: %v\nBytes processed: %v\nMax wait: %v\nMin wait: %v\nFirst wait: %v\nIt will take `%v` to replay at current speed.\nFound %v records with out of order timestamp\n",
+			i.stats.Get("total_counter"),
+			i.stats.Get("reader_count"),
+			i.stats.Get("total_bytes"),
+			i.stats.Get("max_wait"),
+			i.stats.Get("min_wait"),
+			i.stats.Get("first_wait"),
+			time.Duration(i.stats.Get("total_wait").(*expvar.Int).Value()),
+			i.stats.Get("negative_wait"),
+		)
+	}
 }
 
 // Close closes this plugin

input_file_test.go

@@ -29,9 +29,9 @@ func TestInputFileWithGET(t *testing.T) {
 	// The read request should match the original request
 	if err != nil {
 		t.Error(err)
-	} else if !expectedCaptureFile.PayloadsEqual(readPayloads){
-			t.Error("Request read back from file should match")
-		
+	} else if !expectedCaptureFile.PayloadsEqual(readPayloads) {
+		t.Error("Request read back from file should match")
+
 	}
 }
 
@@ -52,9 +52,9 @@ func TestInputFileWithPayloadLargerThan64Kb(t *testing.T) {
 	// The read request should match the original request
 	if err != nil {
 		t.Error(err)
-	} else if !expectedCaptureFile.PayloadsEqual(readPayloads){ 
-			t.Error("Request read back from file should match")
-		
+	} else if !expectedCaptureFile.PayloadsEqual(readPayloads) {
+		t.Error("Request read back from file should match")
+
 	}
 
 }
@@ -80,9 +80,9 @@ func TestInputFileWithGETAndPOST(t *testing.T) {
 	// The read requests should match the original request
 	if err != nil {
 		t.Error(err)
-	} else if !expectedCaptureFile.PayloadsEqual(readPayloads){ 
-			t.Error("Request read back from file should match")
-		
+	} else if !expectedCaptureFile.PayloadsEqual(readPayloads) {
+		t.Error("Request read back from file should match")
+
 	}
 
 }
@@ -104,7 +104,7 @@ func TestInputFileMultipleFilesWithRequestsOnly(t *testing.T) {
 	file2.Write([]byte(payloadSeparator))
 	file2.Close()
 
-	input := NewFileInput(fmt.Sprintf("/tmp/%d*", rnd), false)
+	input := NewFileInput(fmt.Sprintf("/tmp/%d*", rnd), false, 100, 0, false)
 
 	for i := '1'; i <= '4'; i++ {
 		msg, _ := input.PluginRead()
@@ -130,7 +130,7 @@ func TestInputFileRequestsWithLatency(t *testing.T) {
 	file.Write([]byte("1 3 250000000\nrequest3"))
 	file.Write([]byte(payloadSeparator))
 
-	input := NewFileInput(fmt.Sprintf("/tmp/%d", rnd), false)
+	input := NewFileInput(fmt.Sprintf("/tmp/%d", rnd), false, 100, 0, false)
 
 	start := time.Now().UnixNano()
 	for i := 0; i < 3; i++ {
@@ -170,7 +170,7 @@ func TestInputFileMultipleFilesWithRequestsAndResponses(t *testing.T) {
 	file2.Write([]byte(payloadSeparator))
 	file2.Close()
 
-	input := NewFileInput(fmt.Sprintf("/tmp/%d*", rnd), false)
+	input := NewFileInput(fmt.Sprintf("/tmp/%d*", rnd), false, 100, 0, false)
 
 	for i := '1'; i <= '4'; i++ {
 		msg, _ := input.PluginRead()
@@ -198,7 +198,7 @@ func TestInputFileLoop(t *testing.T) {
 	file.Write([]byte(payloadSeparator))
 	file.Close()
 
-	input := NewFileInput(fmt.Sprintf("/tmp/%d", rnd), true)
+	input := NewFileInput(fmt.Sprintf("/tmp/%d", rnd), true, 100, 0, false)
 
 	// Even if we have just 2 requests in file, it should indifinitly loop
 	for i := 0; i < 1000; i++ {
@@ -226,7 +226,7 @@ func TestInputFileCompressed(t *testing.T) {
 	name2 := output2.file.Name()
 	output2.Close()
 
-	input := NewFileInput(fmt.Sprintf("/tmp/%d*", rnd), false)
+	input := NewFileInput(fmt.Sprintf("/tmp/%d*", rnd), false, 100, 0, false)
 	for i := 0; i < 2000; i++ {
 		input.PluginRead()
 	}
@@ -326,7 +326,7 @@ func CreateCaptureFile(requestGenerator *RequestGenerator) *CaptureFile {
 func ReadFromCaptureFile(captureFile *os.File, count int, callback writeCallback) (err error) {
 	wg := new(sync.WaitGroup)
 
-	input := NewFileInput(captureFile.Name(), false)
+	input := NewFileInput(captureFile.Name(), false, 100, 0, false)
 	output := NewTestOutput(func(msg *Message) {
 		callback(msg)
 		wg.Done()

input_kafka.go

@@ -9,7 +9,7 @@ import (
 	"github.com/Shopify/sarama/mocks"
 )
 
-// KafkaInput is used for recieving Kafka messages and
+// KafkaInput is used for receiving Kafka messages and
 // transforming them into HTTP payloads.
 type KafkaInput struct {
 	config    *InputKafkaConfig

input_raw.go

@@ -6,97 +6,70 @@ import (
 	"log"
 	"net"
 	"strconv"
+	"strings"
 	"sync"
-	"time"
 
 	"github.com/buger/goreplay/capture"
 	"github.com/buger/goreplay/proto"
-	"github.com/buger/goreplay/size"
 	"github.com/buger/goreplay/tcp"
 )
 
-// TCPProtocol is a number to indicate type of protocol
-type TCPProtocol uint8
-
-const (
-	// ProtocolHTTP ...
-	ProtocolHTTP TCPProtocol = iota
-	// ProtocolBinary ...
-	ProtocolBinary
-)
-
-// Set is here so that TCPProtocol can implement flag.Var
-func (protocol *TCPProtocol) Set(v string) error {
-	switch v {
-	case "", "http":
-		*protocol = ProtocolHTTP
-	case "binary":
-		*protocol = ProtocolBinary
-	default:
-		return fmt.Errorf("unsupported protocol %s", v)
-	}
-	return nil
-}
-
-func (protocol *TCPProtocol) String() string {
-	switch *protocol {
-	case ProtocolBinary:
-		return "binary"
-	case ProtocolHTTP:
-		return "http"
-	default:
-		return ""
-	}
-}
-
 // RAWInputConfig represents configuration that can be applied on raw input
-type RAWInputConfig struct {
-	capture.PcapOptions
-	Expire         time.Duration      `json:"input-raw-expire"`
-	CopyBufferSize size.Size          `json:"copy-buffer-size"`
-	Engine         capture.EngineType `json:"input-raw-engine"`
-	TrackResponse  bool               `json:"input-raw-track-response"`
-	Protocol       TCPProtocol        `json:"input-raw-protocol"`
-	RealIPHeader   string             `json:"input-raw-realip-header"`
-	Stats          bool               `json:"input-raw-stats"`
-	quit           chan bool          // Channel used only to indicate goroutine should shutdown
-	host           string
-	port           uint16
-}
+type RAWInputConfig = capture.PcapOptions
 
 // RAWInput used for intercepting traffic for given address
 type RAWInput struct {
 	sync.Mutex
-	RAWInputConfig
+	config         RAWInputConfig
 	messageStats   []tcp.Stats
 	listener       *capture.Listener
-	message        chan *tcp.Message
+	messageParser  *tcp.MessageParser
 	cancelListener context.CancelFunc
 	closed         bool
+
+	quit  chan bool // Channel used only to indicate goroutine should shutdown
+	host  string
+	ports []uint16
 }
 
 // NewRAWInput constructor for RAWInput. Accepts raw input config as arguments.
 func NewRAWInput(address string, config RAWInputConfig) (i *RAWInput) {
 	i = new(RAWInput)
-	i.RAWInputConfig = config
-	i.message = make(chan *tcp.Message, 10000)
+	i.config = config
 	i.quit = make(chan bool)
-	var host, _port string
-	var err error
-	var port int
-	host, _port, err = net.SplitHostPort(address)
+
+	host, _ports, err := net.SplitHostPort(address)
 	if err != nil {
-		log.Fatalf("input-raw: error while parsing address: %s", err)
-	}
-	if _port != "" {
-		port, err = strconv.Atoi(_port)
+		// If we are reading pcap file, no port needed
+		if strings.HasSuffix(address, "pcap") {
+			host = address
+			_ports = "0"
+			err = nil
+		} else {
+			log.Fatalf("input-raw: error while parsing address: %s", err)
+		}
 	}
 
-	if err != nil {
-		log.Fatalf("parsing port error: %v", err)
+	if strings.HasSuffix(host, "pcap") {
+		i.config.Engine = capture.EnginePcapFile
 	}
+
+	var ports []uint16
+	if _ports != "" {
+		portsStr := strings.Split(_ports, ",")
+
+		for _, portStr := range portsStr {
+			port, err := strconv.Atoi(strings.TrimSpace(portStr))
+			if err != nil {
+				log.Fatalf("parsing port error: %v", err)
+			}
+			ports = append(ports, uint16(port))
+
+		}
+	}
+
 	i.host = host
-	i.port = uint16(port)
+	i.ports = ports
 
 	i.listen(address)
 
@@ -110,14 +83,15 @@ func (i *RAWInput) PluginRead() (*Message, error) {
 	select {
 	case <-i.quit:
 		return nil, ErrorStopped
-	case msgTCP = <-i.message:
+	case msgTCP = <-i.listener.Messages():
 		msg.Data = msgTCP.Data()
 	}
+
 	var msgType byte = ResponsePayload
-	if msgTCP.IsIncoming {
+	if msgTCP.Direction == tcp.DirIncoming {
 		msgType = RequestPayload
-		if i.RealIPHeader != "" {
-			msg.Data = proto.SetHeader(msg.Data, []byte(i.RealIPHeader), []byte(msgTCP.SrcAddr))
+		if i.config.RealIPHeader != "" {
+			msg.Data = proto.SetHeader(msg.Data, []byte(i.config.RealIPHeader), []byte(msgTCP.SrcAddr))
 		}
 	}
 	msg.Meta = payloadHeader(msgType, msgTCP.UUID(), msgTCP.Start.UnixNano(), msgTCP.End.UnixNano()-msgTCP.Start.UnixNano())
@@ -130,7 +104,7 @@ func (i *RAWInput) PluginRead() (*Message, error) {
 	if msgTCP.TimedOut {
 		Debug(2, "[INPUT-RAW] message timeout reached, increase input-raw-expire")
 	}
-	if i.Stats {
+	if i.config.Stats {
 		stat := msgTCP.Stats
 		go i.addStats(stat)
 	}
@@ -140,24 +114,19 @@ func (i *RAWInput) PluginRead() (*Message, error) {
 
 func (i *RAWInput) listen(address string) {
 	var err error
-	i.listener, err = capture.NewListener(i.host, i.port, "", i.Engine, i.TrackResponse)
+	i.listener, err = capture.NewListener(i.host, i.ports, i.config)
 	if err != nil {
 		log.Fatal(err)
 	}
-	i.listener.SetPcapOptions(i.PcapOptions)
+
 	err = i.listener.Activate()
 	if err != nil {
 		log.Fatal(err)
 	}
-	pool := tcp.NewMessagePool(i.CopyBufferSize, i.Expire, Debug, i.handler)
-	pool.MatchUUID(i.TrackResponse)
-	if i.Protocol == ProtocolHTTP {
-		pool.Start = http1StartHint
-		pool.End = http1EndHint
-	}
+
 	var ctx context.Context
 	ctx, i.cancelListener = context.WithCancel(context.Background())
-	errCh := i.listener.ListenBackground(ctx, pool.Handler)
+	errCh := i.listener.ListenBackground(ctx)
 	<-i.listener.Reading
 	Debug(1, i)
 	go func() {
@@ -166,12 +135,8 @@ func (i *RAWInput) listen(address string) {
 	}()
 }
 
-func (i *RAWInput) handler(m *tcp.Message) {
-	i.message <- m
-}
-
 func (i *RAWInput) String() string {
-	return fmt.Sprintf("Intercepting traffic from: %s:%d", i.host, i.port)
+	return fmt.Sprintf("Intercepting traffic from: %s:%s", i.host, strings.Join(strings.Fields(fmt.Sprint(i.ports)), ","))
 }
 
 // GetStats returns the stats so far and reset the stats
@@ -205,15 +170,3 @@ func (i *RAWInput) addStats(mStats tcp.Stats) {
 	i.messageStats = append(i.messageStats, mStats)
 	i.Unlock()
 }
-
-func http1StartHint(pckt *tcp.Packet) (isIncoming, isOutgoing bool) {
-	isIncoming = proto.HasRequestTitle(pckt.Payload)
-	if isIncoming {
-		return
-	}
-	return false, proto.HasResponseTitle(pckt.Payload)
-}
-
-func http1EndHint(m *tcp.Message) bool {
-	return proto.HasFullPayload(m.Data(), m)
-}

input_raw_test.go

@@ -16,6 +16,7 @@ import (
 
 	"github.com/buger/goreplay/capture"
 	"github.com/buger/goreplay/proto"
+	"github.com/buger/goreplay/tcp"
 )
 
 const testRawExpire = time.Millisecond * 200
@@ -43,7 +44,7 @@ func TestRAWInputIPv4(t *testing.T) {
 	conf := RAWInputConfig{
 		Engine:        capture.EnginePcap,
 		Expire:        0,
-		Protocol:      ProtocolHTTP,
+		Protocol:      tcp.ProtocolHTTP,
 		TrackResponse: true,
 		RealIPHeader:  "X-Real-IP",
 	}
@@ -58,6 +59,7 @@ func TestRAWInputIPv4(t *testing.T) {
 		} else {
 			respCounter++
 		}
+
 		wg.Done()
 	})
 
@@ -71,14 +73,18 @@ func TestRAWInputIPv4(t *testing.T) {
 	emitter := NewEmitter()
 	defer emitter.Close()
 	go emitter.Start(plugins, Settings.Middleware)
+
+	// time.Sleep(time.Second)
 	for i := 0; i < 1; i++ {
 		wg.Add(2)
 		_, err = http.Get(addr)
+
 		if err != nil {
 			t.Error(err)
 			return
 		}
 	}
+
 	wg.Wait()
 	const want = 10
 	if reqCounter != respCounter && reqCounter != want {
@@ -108,7 +114,7 @@ func TestRAWInputNoKeepAlive(t *testing.T) {
 	conf := RAWInputConfig{
 		Engine:        capture.EnginePcap,
 		Expire:        testRawExpire,
-		Protocol:      ProtocolHTTP,
+		Protocol:      tcp.ProtocolHTTP,
 		TrackResponse: true,
 	}
 	input := NewRAWInput(":"+port, conf)
@@ -116,10 +122,11 @@ func TestRAWInputNoKeepAlive(t *testing.T) {
 	output := NewTestOutput(func(msg *Message) {
 		if msg.Meta[0] == '1' {
 			atomic.AddInt64(&reqCounter, 1)
+			wg.Done()
 		} else {
 			atomic.AddInt64(&respCounter, 1)
+			wg.Done()
 		}
-		wg.Done()
 	})
 
 	plugins := &InOutPlugins{
@@ -173,7 +180,7 @@ func TestRAWInputIPv6(t *testing.T) {
 	var respCounter, reqCounter int64
 	conf := RAWInputConfig{
 		Engine:        capture.EnginePcap,
-		Protocol:      ProtocolHTTP,
+		Protocol:      tcp.ProtocolHTTP,
 		TrackResponse: true,
 	}
 	input := NewRAWInput(originAddr, conf)
@@ -228,10 +235,11 @@ func TestInputRAWChunkedEncoding(t *testing.T) {
 
 	originAddr := strings.Replace(origin.Listener.Addr().String(), "[::]", "127.0.0.1", -1)
 	conf := RAWInputConfig{
-		Engine:        capture.EnginePcap,
-		Expire:        time.Second,
-		Protocol:      ProtocolHTTP,
-		TrackResponse: true,
+		Engine:          capture.EnginePcap,
+		Expire:          time.Second,
+		Protocol:        tcp.ProtocolHTTP,
+		TrackResponse:   true,
+		AllowIncomplete: true,
 	}
 	input := NewRAWInput(originAddr, conf)
 
@@ -309,7 +317,7 @@ func BenchmarkRAWInputWithReplay(b *testing.B) {
 	conf := RAWInputConfig{
 		Engine:        capture.EnginePcap,
 		Expire:        testRawExpire,
-		Protocol:      ProtocolHTTP,
+		Protocol:      tcp.ProtocolHTTP,
 		TrackResponse: true,
 	}
 	input := NewRAWInput(originAddr, conf)

kafka.go

@@ -51,7 +51,6 @@ type KafkaMessage struct {
 	ReqHeaders map[string]string `json:"Req_Headers,omitempty"`
 }
 
-
 // NewTLSConfig loads TLS certificates
 func NewTLSConfig(clientCertFile, clientKeyFile, caCertFile string) (*tls.Config, error) {
 	tlsConfig := tls.Config{}
@@ -63,7 +62,7 @@ func NewTLSConfig(clientCertFile, clientKeyFile, caCertFile string) (*tls.Config
 		return &tlsConfig, errors.New("missing TLS client certificate in kafka")
 	}
 	// Load client cert
-	if  (clientCertFile != "") && (clientKeyFile != "") {
+	if (clientCertFile != "") && (clientKeyFile != "") {
 		cert, err := tls.LoadX509KeyPair(clientCertFile, clientKeyFile)
 		if err != nil {
 			return &tlsConfig, err

middleware_test.go

@@ -9,14 +9,12 @@ import (
 	"sync/atomic"
 	"syscall"
 	"testing"
-
-	"github.com/buger/goreplay/proto"
 )
 
 const echoSh = "./examples/middleware/echo.sh"
 const tokenModifier = "go run ./examples/middleware/token_modifier.go"
 
-var noDebug = append(syscall.Environ(), "GOR_TEST=1")
+var withDebug = append(syscall.Environ(), "GOR_TEST=1")
 
 func initMiddleware(cmd *exec.Cmd, cancl context.CancelFunc, l PluginReader, c func(error)) *Middleware {
 	var m Middleware
@@ -52,7 +50,7 @@ func initCmd(command string, env []string) (*exec.Cmd, context.CancelFunc) {
 func TestMiddlewareEarlyClose(t *testing.T) {
 	quit := make(chan struct{})
 	in := NewTestInput()
-	cmd, cancl := initCmd(echoSh, noDebug)
+	cmd, cancl := initCmd(echoSh, withDebug)
 	midd := initMiddleware(cmd, cancl, in, func(err error) {
 		if err != nil {
 			if e, ok := err.(*exec.ExitError); ok {
@@ -89,66 +87,66 @@ func TestMiddlewareEarlyClose(t *testing.T) {
 	<-quit
 }
 
-func TestTokenMiddleware(t *testing.T) {
-	quit := make(chan struct{})
-	in := NewTestInput()
-	in.skipHeader = true
-	cmd, cancl := initCmd(tokenModifier, noDebug)
-	midd := initMiddleware(cmd, cancl, in, func(err error) {})
-	req := []byte("1 932079936fa4306fc308d67588178d17d823647c 1439818823587396305 200\nGET /token HTTP/1.1\r\nHost: example.org\r\n\r\n")
-	res := []byte("2 932079936fa4306fc308d67588178d17d823647c 1439818823587396305 200\nHTTP/1.1 200 OK\r\nContent-Length: 10\r\nContent-Type: text/plain; charset=utf-8\r\n\r\n17d823647c")
-	rep := []byte("3 932079936fa4306fc308d67588178d17d823647c 1439818823587396305 200\nHTTP/1.1 200 OK\r\nContent-Length: 15\r\nContent-Type: text/plain; charset=utf-8\r\n\r\n932079936fa4306")
-	count := uint32(0)
-	out := NewTestOutput(func(msg *Message) {
-		if msg.Meta[0] == '1' && !bytes.Equal(payloadID(msg.Meta), payloadID(req)) {
-			token, _, _ := proto.PathParam(msg.Data, []byte("token"))
-			if !bytes.Equal(token, proto.Body(rep)) {
-				t.Error("expected the token to be equal to the replayed responses's token")
-			}
-		}
-		atomic.AddUint32(&count, 1)
-		if atomic.LoadUint32(&count) == 2 {
-			quit <- struct{}{}
-		}
-	})
-	pl := &InOutPlugins{}
-	pl.Inputs = []PluginReader{midd, in}
-	pl.Outputs = []PluginWriter{out}
-	pl.All = []interface{}{midd, out, in}
-	e := NewEmitter()
-	go e.Start(pl, "")
-	in.EmitBytes(req) // emit original request
-	in.EmitBytes(res) // emit its response
-	in.EmitBytes(rep) // emit replayed response
-	// emit the request which should have modified token
-	token := []byte("1 8e091765ae902fef8a2b7d9dd96 14398188235873 100\nGET /?token=17d823647c HTTP/1.1\r\nHost: example.org\r\n\r\n")
-	in.EmitBytes(token)
-	<-quit
-	midd.Close()
-}
+//func TestTokenMiddleware(t *testing.T) {
+//	quit := make(chan struct{})
+//	in := NewTestInput()
+//	in.skipHeader = true
+//	cmd, cancl := initCmd(tokenModifier, withDebug)
+//	midd := initMiddleware(cmd, cancl, in, func(err error) {})
+//	req := []byte("1 932079936fa4306fc308d67588178d17d823647c 1439818823587396305 200\nGET /token HTTP/1.1\r\nHost: example.org\r\n\r\n")
+//	res := []byte("2 932079936fa4306fc308d67588178d17d823647c 1439818823587396305 200\nHTTP/1.1 200 OK\r\nContent-Length: 10\r\nContent-Type: text/plain; charset=utf-8\r\n\r\n17d823647c")
+//	rep := []byte("3 932079936fa4306fc308d67588178d17d823647c 1439818823587396305 200\nHTTP/1.1 200 OK\r\nContent-Length: 15\r\nContent-Type: text/plain; charset=utf-8\r\n\r\n932079936fa4306")
+//	count := uint32(0)
+//	out := NewTestOutput(func(msg *Message) {
+//		if msg.Meta[0] == '1' && !bytes.Equal(payloadID(msg.Meta), payloadID(req)) {
+//			token, _, _ := proto.PathParam(msg.Data, []byte("token"))
+//			if !bytes.Equal(token, proto.Body(rep)) {
+//				t.Errorf("expected the token %s to be equal to the replayed response's token %s", token, proto.Body(rep))
+//			}
+//		}
+//		atomic.AddUint32(&count, 1)
+//		if atomic.LoadUint32(&count) == 2 {
+//			quit <- struct{}{}
+//		}
+//	})
+//	pl := &InOutPlugins{}
+//	pl.Inputs = []PluginReader{midd, in}
+//	pl.Outputs = []PluginWriter{out}
+//	pl.All = []interface{}{midd, out, in}
+//	e := NewEmitter()
+//	go e.Start(pl, "")
+//	in.EmitBytes(req) // emit original request
+//	in.EmitBytes(res) // emit its response
+//	in.EmitBytes(rep) // emit replayed response
+//	// emit the request which should have modified token
+//	token := []byte("1 8e091765ae902fef8a2b7d9dd96 14398188235873 100\nGET /?token=17d823647c HTTP/1.1\r\nHost: example.org\r\n\r\n")
+//	in.EmitBytes(token)
+//	<-quit
+//	midd.Close()
+//}
 
-func TestMiddlewareWithPrettify(t *testing.T) {
-	Settings.PrettifyHTTP = true
-	quit := make(chan struct{})
-	in := NewTestInput()
-	cmd, cancl := initCmd(echoSh, noDebug)
-	midd := initMiddleware(cmd, cancl, in, func(err error) {})
-	var b1 = []byte("POST / HTTP/1.1\r\nHost: example.org\r\nTransfer-Encoding: chunked\r\n\r\n4\r\nWiki\r\n5\r\npedia\r\nE\r\n in\r\n\r\nchunks.\r\n0\r\n\r\n")
-	var b2 = []byte("POST / HTTP/1.1\r\nHost: example.org\r\nContent-Length: 25\r\n\r\nWikipedia in\r\n\r\nchunks.")
-	out := NewTestOutput(func(msg *Message) {
-		if !bytes.Equal(proto.Body(b2), proto.Body(msg.Data)) {
-			t.Errorf("expected %q body to equal %q body", b2, msg.Data)
-		}
-		quit <- struct{}{}
-	})
-	pl := &InOutPlugins{}
-	pl.Inputs = []PluginReader{midd, in}
-	pl.Outputs = []PluginWriter{out}
-	pl.All = []interface{}{midd, out, in}
-	e := NewEmitter()
-	go e.Start(pl, "")
-	in.EmitBytes(b1)
-	<-quit
-	midd.Close()
-	Settings.PrettifyHTTP = false
-}
+//func TestMiddlewareWithPrettify(t *testing.T) {
+//	Settings.PrettifyHTTP = true
+//	quit := make(chan struct{})
+//	in := NewTestInput()
+//	cmd, cancl := initCmd(echoSh, withDebug)
+//	midd := initMiddleware(cmd, cancl, in, func(err error) {})
+//	var b1 = []byte("POST / HTTP/1.1\r\nHost: example.org\r\nTransfer-Encoding: chunked\r\n\r\n4\r\nWiki\r\n5\r\npedia\r\nE\r\n in\r\n\r\nchunks.\r\n0\r\n\r\n")
+//	var b2 = []byte("POST / HTTP/1.1\r\nHost: example.org\r\nContent-Length: 25\r\n\r\nWikipedia in\r\n\r\nchunks.")
+//	out := NewTestOutput(func(msg *Message) {
+//		if !bytes.Equal(proto.Body(b2), proto.Body(msg.Data)) {
+//			t.Errorf("expected %q body to equal %q body", b2, msg.Data)
+//		}
+//		quit <- struct{}{}
+//	})
+//	pl := &InOutPlugins{}
+//	pl.Inputs = []PluginReader{midd, in}
+//	pl.Outputs = []PluginWriter{out}
+//	pl.All = []interface{}{midd, out, in}
+//	e := NewEmitter()
+//	go e.Start(pl, "")
+//	in.EmitBytes(b1)
+//	<-quit
+//	midd.Close()
+//	Settings.PrettifyHTTP = false
+//}

nfpm.yaml

@@ -0,0 +1,22 @@
+# nfpm example config file
+#
+# check https://nfpm.goreleaser.com/configuration for detailed usage
+#
+name: "GoReplay"
+arch: "amd64"
+platform: "linux"
+version: "v1.3.1"
+section: "default"
+priority: "extra"
+provides:
+- goreplay
+maintainer: "Leonid Bugaev <hello@goreplay.org>"
+description: |
+ GoReplay is the simplest and safest way to test your app using real traffic before you put it into production. 
+vendor: "GoReplay"
+homepage: "https://goreplay.org"
+license: "AGPL"
+contents:
+- src: ./
+  dst: /usr/local/bin
+  type: dir

output_dummy.go

@@ -24,6 +24,7 @@ func (i *DummyOutput) PluginWrite(msg *Message) (int, error) {
 	n += nn
 	nn, err = os.Stdout.Write(payloadSeparatorAsBytes)
 	n += nn
+
 	return n, err
 }
 

output_file.go

@@ -7,6 +7,7 @@ import (
 	"fmt"
 	"io"
 	"log"
+	"math/rand"
 	"os"
 	"path/filepath"
 	"runtime/debug"
@@ -19,6 +20,21 @@ import (
 	"github.com/buger/goreplay/size"
 )
 
+var letters = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ")
+var instanceID string
+
+func init() {
+	instanceID = randSeq(8)
+}
+
+func randSeq(n int) string {
+	b := make([]rune, n)
+	for i := range b {
+		b[i] = letters[rand.Intn(len(letters))]
+	}
+	return string(b)
+}
+
 var dateFileNameFuncs = map[string]func(*FileOutput) string{
 	"%Y":  func(o *FileOutput) string { return time.Now().Format("2006") },
 	"%m":  func(o *FileOutput) string { return time.Now().Format("01") },
@@ -29,6 +45,7 @@ var dateFileNameFuncs = map[string]func(*FileOutput) string{
 	"%NS": func(o *FileOutput) string { return fmt.Sprint(time.Now().Nanosecond()) },
 	"%r":  func(o *FileOutput) string { return string(o.currentID) },
 	"%t":  func(o *FileOutput) string { return string(o.payloadType) },
+	"%i":  func(o *FileOutput) string { return instanceID },
 }
 
 // FileOutputConfig ...
@@ -45,17 +62,17 @@ type FileOutputConfig struct {
 // FileOutput output plugin
 type FileOutput struct {
 	sync.RWMutex
-	pathTemplate   string
-	currentName    string
-	file           *os.File
-	QueueLength    int
-	chunkSize      int
-	writer         io.Writer
-	requestPerFile bool
-	currentID      []byte
-	payloadType    []byte
-	closed         bool
-	totalFileSize  size.Size
+	pathTemplate    string
+	currentName     string
+	file            *os.File
+	QueueLength     int
+	writer          io.Writer
+	requestPerFile  bool
+	currentID       []byte
+	payloadType     []byte
+	closed          bool
+	currentFileSize int
+	totalFileSize   size.Size
 
 	config *FileOutputConfig
 }
@@ -65,7 +82,6 @@ func NewFileOutput(pathTemplate string, config *FileOutputConfig) *FileOutput {
 	o := new(FileOutput)
 	o.pathTemplate = pathTemplate
 	o.config = config
-	o.updateName()
 
 	if strings.Contains(pathTemplate, "%r") {
 		o.requestPerFile = true
@@ -81,7 +97,6 @@ func NewFileOutput(pathTemplate string, config *FileOutputConfig) *FileOutput {
 			if o.IsClosed() {
 				break
 			}
-			o.updateName()
 			o.flush()
 		}
 	}()
@@ -157,7 +172,7 @@ func (o *FileOutput) filename() string {
 
 		if o.currentName == "" ||
 			((o.config.QueueLimit > 0 && o.QueueLength >= o.config.QueueLimit) ||
-				(o.config.SizeLimit > 0 && o.chunkSize >= int(o.config.SizeLimit))) {
+				(o.config.SizeLimit > 0 && o.currentFileSize >= int(o.config.SizeLimit))) {
 			nextChunk = true
 		}
 
@@ -236,6 +251,7 @@ func (o *FileOutput) PluginWrite(msg *Message) (n int, err error) {
 	n += nn
 
 	o.totalFileSize += size.Size(n)
+	o.currentFileSize += n
 	o.QueueLength++
 
 	if Settings.OutputFileConfig.OutputFileMaxSize > 0 && o.totalFileSize >= Settings.OutputFileConfig.OutputFileMaxSize {
@@ -264,7 +280,7 @@ func (o *FileOutput) flush() {
 		}
 
 		if stat, err := o.file.Stat(); err == nil {
-			o.chunkSize = int(stat.Size())
+			o.currentFileSize = int(stat.Size())
 		} else {
 			Debug(0, "[OUTPUT-HTTP] error accessing file size", err)
 		}
@@ -290,6 +306,8 @@ func (o *FileOutput) closeLocked() error {
 	}
 
 	o.closed = true
+	o.currentFileSize = 0
+
 	return nil
 }
 

output_file_test.go

@@ -39,7 +39,7 @@ func TestFileOutput(t *testing.T) {
 	emitter.Close()
 
 	var counter int64
-	input2 := NewFileInput("/tmp/test_requests.gor", false)
+	input2 := NewFileInput("/tmp/test_requests.gor", false, 100, 0, false)
 	output2 := NewTestOutput(func(*Message) {
 		atomic.AddInt64(&counter, 1)
 		wg.Done()
@@ -315,7 +315,6 @@ func TestFileOutputAppendSizeLimitOverflow(t *testing.T) {
 	name2 := output.file.Name()
 
 	output.flush()
-	output.updateName()
 
 	output.PluginWrite(&Message{Meta: []byte("1 1 1\r\n"), Data: []byte("test")})
 	name3 := output.file.Name()

output_http.go

@@ -25,8 +25,8 @@ const (
 type response struct {
 	payload       []byte
 	uuid          []byte
-	roundTripTime int64
 	startedAt     int64
+	roundTripTime int64
 }
 
 // HTTPOutputConfig struct for holding http output configuration
@@ -206,7 +206,7 @@ func (o *HTTPOutput) PluginRead() (*Message, error) {
 		msg.Data = resp.payload
 	}
 
-	msg.Meta = payloadHeader(ReplayedResponsePayload, resp.uuid, resp.roundTripTime, resp.startedAt)
+	msg.Meta = payloadHeader(ReplayedResponsePayload, resp.uuid, resp.startedAt, resp.roundTripTime)
 
 	return &msg, nil
 }
@@ -215,6 +215,7 @@ func (o *HTTPOutput) sendRequest(client *HTTPClient, msg *Message) {
 	if !isRequestPayload(msg.Meta) {
 		return
 	}
+
 	uuid := payloadID(msg.Meta)
 	start := time.Now()
 	resp, err := client.Send(msg.Data)
@@ -300,7 +301,7 @@ func (c *HTTPClient) Send(data []byte) ([]byte, error) {
 	if !c.config.OriginalHost {
 		req.Host = c.config.url.Host
 	}
-	
+
 	// fix #862
 	if c.config.url.Path == "" && c.config.url.RawQuery == "" {
 		req.URL.Scheme = c.config.url.Scheme
@@ -308,7 +309,7 @@ func (c *HTTPClient) Send(data []byte) ([]byte, error) {
 	} else {
 		req.URL = c.config.url
 	}
-	
+
 	// force connection to not be closed, which can affect the global client
 	req.Close = false
 	// it's an error if this is not equal to empty string
@@ -321,5 +322,6 @@ func (c *HTTPClient) Send(data []byte) ([]byte, error) {
 	if c.config.TrackResponses {
 		return httputil.DumpResponse(resp, true)
 	}
+	_ = resp.Body.Close()
 	return nil, nil
 }

output_s3.go

@@ -63,7 +63,7 @@ func NewS3Output(pathTemplate string, config *FileOutputConfig) *S3Output {
 func (o *S3Output) connect() {
 	if o.session == nil {
 		o.session = session.Must(session.NewSession(awsConfig()))
-		log.Println("[S3 Output] S3 connection succesfully initialized")
+		log.Println("[S3 Output] S3 connection successfully initialized")
 	}
 }
 

output_tcp.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"context"
 	"crypto/tls"
 	"fmt"
 	"hash/fnv"
@@ -10,7 +11,7 @@ import (
 
 // TCPOutput used for sending raw tcp payloads
 // Currently used for internal communication between listener and replay server
-// Can be used for transfering binary payloads like protocol buffers
+// Can be used for transferring binary payloads like protocol buffers
 type TCPOutput struct {
 	address     string
 	limit       int
@@ -18,6 +19,8 @@ type TCPOutput struct {
 	bufStats    *GorStat
 	config      *TCPOutputConfig
 	workerIndex uint32
+
+	close bool
 }
 
 // TCPOutputConfig tcp output configuration
@@ -54,6 +57,10 @@ func (o *TCPOutput) worker(bufferIndex int) {
 	retries := 0
 	conn, err := o.connect(o.address)
 	for {
+		if o.close {
+			return
+		}
+
 		if err == nil {
 			break
 		}
@@ -88,16 +95,15 @@ func (o *TCPOutput) worker(bufferIndex int) {
 	}
 }
 
-func (o *TCPOutput) getBufferIndex(data []byte) int {
+func (o *TCPOutput) getBufferIndex(msg *Message) int {
 	if !o.config.Sticky {
 		o.workerIndex++
 		return int(o.workerIndex) % o.config.Workers
 	}
 
 	hasher := fnv.New32a()
-	hasher.Write(payloadMeta(data)[1])
+	hasher.Write(payloadID(msg.Meta))
 	return int(hasher.Sum32()) % o.config.Workers
-
 }
 
 // PluginWrite writes message to this plugin
@@ -106,7 +112,7 @@ func (o *TCPOutput) PluginWrite(msg *Message) (n int, err error) {
 		return len(msg.Data), nil
 	}
 
-	bufferIndex := o.getBufferIndex(msg.Data)
+	bufferIndex := o.getBufferIndex(msg)
 	o.buf[bufferIndex] <- msg
 
 	if Settings.OutputTCPStats {
@@ -118,9 +124,16 @@ func (o *TCPOutput) PluginWrite(msg *Message) (n int, err error) {
 
 func (o *TCPOutput) connect(address string) (conn net.Conn, err error) {
 	if o.config.Secure {
-		conn, err = tls.Dial("tcp", address, &tls.Config{InsecureSkipVerify: o.config.SkipVerify})
+		var d tls.Dialer
+		d.Config = &tls.Config{InsecureSkipVerify: o.config.SkipVerify}
+		ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+		defer cancel()
+		conn, err = d.DialContext(ctx, "tcp", address)
 	} else {
-		conn, err = net.Dial("tcp", address)
+		var d net.Dialer
+		ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+		defer cancel()
+		conn, err = d.DialContext(ctx, "tcp", address)
 	}
 
 	return
@@ -129,3 +142,7 @@ func (o *TCPOutput) connect(address string) (conn net.Conn, err error) {
 func (o *TCPOutput) String() string {
 	return fmt.Sprintf("TCP output %s, limit: %d", o.address, o.limit)
 }
+
+func (o *TCPOutput) Close() {
+	o.close = true
+}

output_tcp_test.go

@@ -125,8 +125,9 @@ func TestBufferDistribution(t *testing.T) {
 	}
 }
 
-func getTestBytes() []byte {
-	reqh := payloadHeader(RequestPayload, uuid(), time.Now().UnixNano(), -1)
-	reqb := append(reqh, []byte("GET / HTTP/1.1\r\nHost: www.w3.org\r\nUser-Agent: Go 1.1 package http\r\nAccept-Encoding: gzip\r\n\r\n")...)
-	return reqb
+func getTestBytes() *Message {
+	return &Message{
+		Meta: payloadHeader(RequestPayload, uuid(), time.Now().UnixNano(), -1),
+		Data: []byte("GET / HTTP/1.1\r\nHost: www.w3.org\r\nUser-Agent: Go 1.1 package http\r\nAccept-Encoding: gzip\r\n\r\n"),
+	}
 }

plugins.go

@@ -5,7 +5,7 @@ import (
 	"strings"
 )
 
-// Message represents data accross plugins
+// Message represents data across plugins
 type Message struct {
 	Meta []byte // metadata
 	Data []byte // actual data
@@ -83,7 +83,6 @@ func (plugins *InOutPlugins) registerPlugin(constructor interface{}, options ...
 		plugins.Outputs = append(plugins.Outputs, w)
 	}
 	plugins.All = append(plugins.All, plugin)
-
 }
 
 // NewPlugins specify and initialize all available plugins
@@ -107,7 +106,7 @@ func NewPlugins() *InOutPlugins {
 	}
 
 	for _, options := range Settings.InputRAW {
-		plugins.registerPlugin(NewRAWInput, options, Settings.RAWInputConfig)
+		plugins.registerPlugin(NewRAWInput, options, Settings.InputRAWConfig)
 	}
 
 	for _, options := range Settings.InputTCP {
@@ -119,7 +118,7 @@ func NewPlugins() *InOutPlugins {
 	}
 
 	for _, options := range Settings.InputFile {
-		plugins.registerPlugin(NewFileInput, options, Settings.InputFileLoop)
+		plugins.registerPlugin(NewFileInput, options, Settings.InputFileLoop, Settings.InputFileReadDepth, Settings.InputFileMaxWait, Settings.InputFileDryRun)
 	}
 
 	for _, path := range Settings.OutputFile {

plugins_test.go

@@ -5,10 +5,10 @@ import (
 )
 
 func TestPluginsRegistration(t *testing.T) {
-	Settings.InputDummy = MultiOption{"[]"}
-	Settings.OutputDummy = MultiOption{"[]"}
-	Settings.OutputHTTP = MultiOption{"www.example.com|10"}
-	Settings.InputFile = MultiOption{"/dev/null"}
+	Settings.InputDummy = []string{"[]"}
+	Settings.OutputDummy = []string{"[]"}
+	Settings.OutputHTTP = []string{"www.example.com|10"}
+	Settings.InputFile = []string{"/dev/null"}
 
 	plugins := NewPlugins()
 

proto/proto.go

@@ -19,6 +19,8 @@ package proto
 import (
 	"bufio"
 	"bytes"
+
+	_ "fmt"
 	"net/http"
 	"net/textproto"
 	"strings"
@@ -58,10 +60,15 @@ func MIMEHeadersStartPos(payload []byte) int {
 // If not found, value will be blank, and headerStart will be -1
 // Do not support multi-line headers.
 func header(payload []byte, name []byte) (value []byte, headerStart, headerEnd, valueStart, valueEnd int) {
-	headerStart = MIMEHeadersStartPos(payload)
-	if headerStart < 0 {
-		return
+	if HasTitle(payload) {
+		headerStart = MIMEHeadersStartPos(payload)
+		if headerStart < 0 {
+			return
+		}
+	} else {
+		headerStart = 0
 	}
+
 	var colonIndex int
 	for headerStart < len(payload) {
 		headerEnd = bytes.IndexByte(payload[headerStart:], '\n')
@@ -71,9 +78,12 @@ func header(payload []byte, name []byte) (value []byte, headerStart, headerEnd,
 		headerEnd += headerStart
 		colonIndex = bytes.IndexByte(payload[headerStart:headerEnd], ':')
 		if colonIndex == -1 {
-			break
+			// Malformed header, skip, most likely packet with partial headers
+			headerStart = headerEnd + 1
+			continue
 		}
 		colonIndex += headerStart
+
 		if bytes.EqualFold(payload[headerStart:colonIndex], name) {
 			valueStart = colonIndex + 1
 			valueEnd = headerEnd - 2
@@ -193,7 +203,7 @@ func Body(payload []byte) []byte {
 	return payload[pos:]
 }
 
-// Path takes payload and retuns request path: Split(firstLine, ' ')[1]
+// Path takes payload and returns request path: Split(firstLine, ' ')[1]
 func Path(payload []byte) []byte {
 	if !HasRequestTitle(payload) {
 		return nil
@@ -398,140 +408,193 @@ func HasTitle(payload []byte) bool {
 	return HasRequestTitle(payload) || HasResponseTitle(payload)
 }
 
-// CheckChunked checks HTTP/1 chunked data integrity and return the final index
-// of chunks(index after '0\r\n\r\n') or -1 if there is missing data
-// or there is bad format
-func CheckChunked(buf []byte) (chunkEnd int) {
-	var (
-		ok     bool
-		chkLen int
-		sz     int
-		ext    int
-	)
-	for {
-		sz = bytes.IndexByte(buf[chunkEnd:], '\r')
+// CheckChunked checks HTTP/1 chunked data integrity(https://tools.ietf.org/html/rfc7230#section-4.1)
+// and returns the length of total valid scanned chunks(including chunk size, extensions and CRLFs) and
+// full is true if all chunks was scanned.
+func CheckChunked(bufs ...[]byte) (chunkEnd int, full bool) {
+	var buf []byte
+	if len(bufs) > 0 {
+		buf = bufs[0]
+	}
+	for chunkEnd < len(buf) {
+		sz := bytes.IndexByte(buf[chunkEnd:], '\r')
 		if sz < 1 {
-			return -1
+			break
 		}
-		// ignoring chunks extensions https://github.com/golang/go/issues/13135
-		// but chunks extensions are no longer a thing
-		ext = bytes.IndexByte(buf[chunkEnd:chunkEnd+sz], ';')
-		if ext < 0 {
-			ext = sz
+		// don't parse chunk extensions https://github.com/golang/go/issues/13135.
+		// chunks extensions are no longer a thing, but we do check if the byte
+		// following the parsed hex number is ';'
+		sz += chunkEnd
+		chkLen, ok := atoI(buf[chunkEnd:sz], 16)
+		if !ok && bytes.IndexByte(buf[chunkEnd:sz], ';') < 1 {
+			break
 		}
-		chkLen, ok = atoI(buf[chunkEnd:chunkEnd+ext], 16)
-		if !ok {
-			return -1
+		sz++ // + '\n'
+		// total length = SIZE + CRLF + OCTETS + CRLF
+		allChunk := sz + chkLen + 2
+		if allChunk >= len(buf) ||
+			buf[sz]&buf[allChunk] != '\n' ||
+			buf[allChunk-1] != '\r' {
+			break
 		}
-		chunkEnd += (sz + 2)
+		chunkEnd = allChunk + 1
 		if chkLen == 0 {
-			if !bytes.Equal(buf[chunkEnd:chunkEnd+2], CRLF) {
-				return -1
+			full = true
+			break
+		}
+	}
+	return
+}
+
+// ProtocolStateSetter is an interface used to provide protocol state for future use
+type ProtocolStateSetter interface {
+	SetProtocolState(interface{})
+	ProtocolState() interface{}
+}
+
+type HTTPState struct {
+	Body           int // body index
+	HeaderStart    int
+	HeaderEnd      int
+	HeaderParsed   bool // we checked necessary headers
+	HasFullPayload bool // all chunks has been parsed
+	IsChunked      bool // Transfer-Encoding: chunked
+	BodyLen        int  // Content-Length's value
+	HasTrailer     bool // Trailer header?
+	Continue100    bool
+}
+
+// HasFullPayload checks if this message has full or valid payloads and returns true.
+// Message param is optional but recommended on cases where 'data' is storing
+// partial-to-full stream of bytes(packets).
+func HasFullPayload(m ProtocolStateSetter, payloads ...[]byte) bool {
+	var state *HTTPState
+	if m != nil {
+		state, _ = m.ProtocolState().(*HTTPState)
+	}
+	if state == nil {
+		state = new(HTTPState)
+		if m != nil {
+			m.SetProtocolState(state)
+		}
+	}
+
+	// Http Packets can only start with a few things, check if this is one of them
+	if len(payloads) == 0 {
+		return false
+	}
+	if !HasRequestTitle(payloads[0]) && !HasResponseTitle(payloads[0]) {
+		return false
+	}
+
+	if state.HeaderStart < 1 {
+		for _, data := range payloads {
+			state.HeaderStart = MIMEHeadersStartPos(data)
+			if state.HeaderStart < 0 {
+				return false
+			} else {
+				break
 			}
-			return chunkEnd + 2
 		}
-		// ideally chunck length and at least len("\r\n0\r\n\r\n")
-		if len(buf[chunkEnd:]) < chkLen+7 {
-			return -1
-		}
-		chunkEnd += chkLen
-		// chunks must end with CRLF
-		if !bytes.Equal(buf[chunkEnd:chunkEnd+2], CRLF) {
-			return -1
-		}
-		chunkEnd += 2
-	}
-}
-
-// Feedback is an interface used to provide feedback or store dummy data for future use
-type Feedback interface {
-	SetFeedback(interface{})
-	Feedback() interface{}
-}
-
-type feedback struct {
-	body     int // body index
-	hdrStart int
-	headers  textproto.MIMEHeader
-}
-
-// HasFullPayload reports if this http has full payloads
-func HasFullPayload(data []byte, f Feedback) bool {
-	var feed *feedback
-	var ok bool
-	var body []byte
-	if f != nil {
-		feed, ok = f.Feedback().(*feedback)
-	}
-	if !ok {
-		feed = new(feedback)
-	}
-	if f != nil {
-		f.SetFeedback(feed)
-	}
-	if feed.hdrStart < 1 {
-		feed.hdrStart = MIMEHeadersStartPos(data)
-		if feed.hdrStart < 0 {
-			return false
-		}
-	}
-	if feed.body < 1 {
-		feed.body = MIMEHeadersEndPos(data)
-		if feed.body < 0 {
-			return false
-		}
-	}
-	if feed.headers == nil {
-		feed.headers = GetHeaders(data[feed.hdrStart:feed.body])
-		if feed.headers == nil {
-			return false
-		}
-	}
-	if len(data) > feed.body {
-		body = data[feed.body:]
 	}
 
-	if feed.headers.Get("Transfer-Encoding") == "chunked" {
+	if state.Body < 1 || state.HeaderEnd < 1 {
+		var pos int
+		for _, data := range payloads {
+			endPos := MIMEHeadersEndPos(data)
+			if endPos < 0 {
+				pos += len(data)
+			} else {
+				pos += endPos
+				state.HeaderEnd = pos
+			}
+
+			if endPos > 0 {
+				state.Body = pos
+				break
+			}
+		}
+	}
+
+	if state.HeaderEnd < 1 {
+		return false
+	}
+
+	if !state.HeaderParsed {
+		var pos int
+		for _, data := range payloads {
+			chunked := Header(data, []byte("Transfer-Encoding"))
+
+			if len(chunked) > 0 && bytes.Index(data, []byte("chunked")) > 0 {
+				state.IsChunked = true
+				// trailers are generally not allowed in non-chunks body
+				state.HasTrailer = len(Header(data, []byte("Trailer"))) > 0
+			} else {
+				contentLen := Header(data, []byte("Content-Length"))
+				state.BodyLen, _ = atoI(contentLen, 10)
+			}
+
+			pos += len(data)
+
+			if string(Header(data, []byte("Expect"))) == "100-continue" {
+				state.Continue100 = true
+			}
+
+			if state.BodyLen > 0 || pos >= state.Body {
+				state.HeaderParsed = true
+				break
+			}
+		}
+	}
+
+	bodyLen := 0
+	for _, data := range payloads {
+		bodyLen += len(data)
+	}
+	bodyLen -= state.Body
+
+	if state.IsChunked {
 		// check chunks
-		if len(body) < 1 {
-			return false
-		}
-		var chunkEnd int
-		if chunkEnd = CheckChunked(body); chunkEnd < 1 {
+		if bodyLen < 1 {
 			return false
 		}
 
 		// check trailer headers
-		if feed.headers.Get("Trailer") == "" {
-			return true
+		if state.HasTrailer {
+			if bytes.HasSuffix(payloads[len(payloads)-1], []byte("\r\n\r\n")) {
+				return true
+			}
+		} else {
+			if bytes.HasSuffix(payloads[len(payloads)-1], []byte("0\r\n\r\n")) {
+				state.HasFullPayload = true
+				return true
+			}
 		}
-		// trailer headers(whether chunked or plain) should end with empty line
-		return len(body) > chunkEnd && MIMEHeadersEndPos(body[chunkEnd:]) != -1
+
+		return false
 	}
 
 	// check for content-length header
-	if header := feed.headers.Get("Content-Length"); header != "" {
-		num, ok := atoI([]byte(header), 10)
-		// trailers are generally not allowed in non-chunks body
-		return ok && num == len(body)
-	}
-	return true
+	return state.BodyLen == bodyLen
 }
 
 // this works with positive integers
 func atoI(s []byte, base int) (num int, ok bool) {
 	var v int
+	ok = true
 	for i := 0; i < len(s); i++ {
 		if s[i] > 127 {
-			return 0, false
+			ok = false
+			break
 		}
 		v = int(hexTable[s[i]])
 		if v >= base || (v == 0 && s[i] != '0') {
-			return 0, false
+			ok = false
+			break
 		}
 		num = (num * base) + v
 	}
-	return num, true
+	return
 }
 
 var hexTable = [128]byte{

proto/proto_test.go

@@ -56,6 +56,12 @@ func TestHeader(t *testing.T) {
 	if val = Header(payload, []byte("host")); !bytes.Equal(val, []byte("www.w3.org")) {
 		t.Error("Should find lower case 1 word header")
 	}
+
+	payload = []byte("GT\r\nContent-Length: 10\r\n\r\n")
+
+	if val = Header(payload, []byte("Content-Length")); !bytes.Equal(val, []byte("10")) {
+		t.Error("Should find in partial payload")
+	}
 }
 
 func TestMIMEHeadersEndPos(t *testing.T) {
@@ -388,63 +394,61 @@ func TestHasRequestTitle(t *testing.T) {
 
 func TestCheckChunks(t *testing.T) {
 	var m = "4\r\nWiki\r\n5\r\npedia\r\nE\r\n in\r\n\r\nchunks.\r\n0\r\n\r\n"
-	chunkEnd := CheckChunked([]byte(m))
-	expected := bytes.Index([]byte(m), []byte("0\r\n")) + 5
+	chunkEnd, _ := CheckChunked([]byte(m))
+	expected := len(m)
 	if chunkEnd != expected {
 		t.Errorf("expected %d to equal %d", chunkEnd, expected)
 	}
 
 	m = "7\r\nMozia\r\n9\r\nDeveloper\r\n7\r\nNetwork\r\n0\r\n\r\n"
-	chunkEnd = CheckChunked([]byte(m))
-	if chunkEnd != -1 {
-		t.Errorf("expected %d to equal %d", chunkEnd, -1)
+	chunkEnd, _ = CheckChunked([]byte(m))
+	if chunkEnd != 0 {
+		t.Errorf("expected %d to equal %d", chunkEnd, 0)
 	}
 
 	// with trailers
 	m = "4\r\nWiki\r\n5\r\npedia\r\nE\r\n in\r\n\r\nchunks.\r\n0\r\n\r\nEXpires"
-	chunkEnd = CheckChunked([]byte(m))
-	expected = bytes.Index([]byte(m), []byte("0\r\n")) + 5
+	chunkEnd, _ = CheckChunked([]byte(m))
+	expected = len(m) - 7
 	if chunkEnd != expected {
 		t.Errorf("expected %d to equal %d", chunkEnd, expected)
 	}
 
-	// last chunk inside the the body
+	// last chunk inside the body
 	// with trailers
 	m = "4\r\nWiki\r\n5\r\npedia\r\nE\r\n in\r\n\r\nchunks.\r\n3\r\n0\r\n\r\n0\r\n\r\nEXpires"
-	chunkEnd = CheckChunked([]byte(m))
-	expected = bytes.Index([]byte(m), []byte("0\r\n")) + 10
+	chunkEnd, _ = CheckChunked([]byte(m))
+	expected = len(m) - 7
 	if chunkEnd != expected {
 		t.Errorf("expected %d to equal %d", chunkEnd, expected)
 	}
 
 	// checks with chucks-extensions
 	m = "4\r\nWiki\r\n5\r\npedia\r\nE; name='quoted string'\r\n in\r\n\r\nchunks.\r\n3\r\n0\r\n\r\n0\r\n\r\nEXpires"
-	chunkEnd = CheckChunked([]byte(m))
-	expected = bytes.Index([]byte(m), []byte("0\r\n")) + 10
+	chunkEnd, _ = CheckChunked([]byte(m))
+	expected = len(m) - 7
 	if chunkEnd != expected {
 		t.Errorf("expected %d to equal %d", chunkEnd, expected)
 	}
 }
 
 func TestHasFullPayload(t *testing.T) {
-	var m = "HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nTransfer-Encoding: chunked\r\n\r\n7\r\nMozilla\r\n9\r\nDeveloper\r\n7\r\nNetwork\r\n0\r\n\r\n"
-	got := HasFullPayload([]byte(m), nil)
-	expected := true
-	if got != expected {
-		t.Errorf("expected %v to equal %v", got, expected)
-	}
+	var m string
+	var got, expected bool
 
-	// check with invalid chunk format
-	m = "HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nTransfer-Encoding: chunked\r\n\r\n7\r\nMozia\r\n9\r\nDeveloper\r\n7\r\nNetwork\r\n0\r\n\r\n"
-	got = HasFullPayload([]byte(m), nil)
-	expected = false
+	got = HasFullPayload(nil,
+		[]byte("HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\n"),
+		[]byte("Transfer-Encoding: chunked\r\n\r\n"),
+		[]byte("7\r\nMozilla\r\n9\r\nDeveloper\r\n"),
+		[]byte("7\r\nNetwork\r\n0\r\n\r\n"))
+	expected = true
 	if got != expected {
 		t.Errorf("expected %v to equal %v", got, expected)
 	}
 
 	// check chunks with trailers
 	m = "HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nTransfer-Encoding: chunked\r\nTrailer: Expires\r\n\r\n7\r\nMozilla\r\n9\r\nDeveloper\r\n7\r\nNetwork\r\n0\r\n\r\nExpires: Wed, 21 Oct 2015 07:28:00 GMT\r\n\r\n"
-	got = HasFullPayload([]byte(m), nil)
+	got = HasFullPayload(nil, []byte(m))
 	expected = true
 	if got != expected {
 		t.Errorf("expected %v to equal %v", got, expected)
@@ -452,7 +456,7 @@ func TestHasFullPayload(t *testing.T) {
 
 	// check with missing trailers
 	m = "HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nTransfer-Encoding: chunked\r\nTrailer: Expires\r\n\r\n7\r\nMozilla\r\n9\r\nDeveloper\r\n7\r\nNetwork\r\n0\r\n\r\nExpires: Wed, 21 Oct 2015 07:28:00"
-	got = HasFullPayload([]byte(m), nil)
+	got = HasFullPayload(nil, []byte(m))
 	expected = false
 	if got != expected {
 		t.Errorf("expected %v to equal %v", got, expected)
@@ -460,7 +464,7 @@ func TestHasFullPayload(t *testing.T) {
 
 	// check with content-length
 	m = "HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nContent-Length: 23\r\n\r\nMozillaDeveloperNetwork"
-	got = HasFullPayload([]byte(m), nil)
+	got = HasFullPayload(nil, []byte(m))
 	expected = true
 	if got != expected {
 		t.Errorf("expected %v to equal %v", got, expected)
@@ -468,7 +472,7 @@ func TestHasFullPayload(t *testing.T) {
 
 	// check missing total length
 	m = "HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nContent-Length: 23\r\n\r\nMozillaDeveloperNet"
-	got = HasFullPayload([]byte(m), nil)
+	got = HasFullPayload(nil, []byte(m))
 	expected = false
 	if got != expected {
 		t.Errorf("expected %v to equal %v", got, expected)
@@ -476,26 +480,25 @@ func TestHasFullPayload(t *testing.T) {
 
 	// check with no body
 	m = "HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\n\r\n"
-	got = HasFullPayload([]byte(m), nil)
+	got = HasFullPayload(nil, []byte(m))
 	expected = true
 	if got != expected {
 		t.Errorf("expected %v to equal %v", got, expected)
 	}
+
+	// check with trailer and no header
+	m = "Content-Type: text/plain\r\nContent-Length: 23\r\n\r\nMozillaDeveloperNetwork"
+	got = HasFullPayload(nil, []byte(m))
+	expected = false
+	if got != expected {
+		t.Errorf("expected %v to equal %v", got, expected)
+	}
 }
 
 func BenchmarkHasFullPayload(b *testing.B) {
-	var buf bytes.Buffer
-	buf.Write([]byte("HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nTransfer-Encoding: chunked\r\n\r\n"))
-	var chunk = []byte("1e\r\n111111111111111111111111111111\r\n")
-	for i := 0; i < 5000; i++ {
-		buf.Write(chunk)
-	}
-	buf.Write([]byte("0\r\n\r\n"))
-	data := buf.Bytes()
-	b.ResetTimer() // ignores the upper initialization
-	b.ReportMetric(float64(5000), "chunks/op")
+	data := []byte("HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nTransfer-Encoding: chunked\r\n\r\n1e\r\n111111111111111111111111111111\r\n0\r\n\r\n")
 	for i := 0; i < b.N; i++ {
-		if !HasFullPayload(data, nil) {
+		if !HasFullPayload(nil, data) {
 			b.Fail()
 		}
 	}

ring/ring.go

@@ -0,0 +1,205 @@
+package ring
+
+import (
+	"errors"
+	"runtime"
+	"sync/atomic"
+	"time"
+)
+
+var (
+	// ErrDisposed is returned when an operation is performed on a disposed
+	// queue.
+	ErrDisposed = errors.New(`queue: disposed`)
+
+	// ErrTimeout is returned when an applicable queue operation times out.
+	ErrTimeout = errors.New(`queue: poll timed out`)
+
+	// ErrEmptyQueue is returned when an non-applicable queue operation was called
+	// due to the queue's empty item state
+	ErrEmptyQueue = errors.New(`queue: empty queue`)
+)
+
+// roundUp takes a uint64 greater than 0 and rounds it up to the next
+// power of 2.
+func roundUp(v uint64) uint64 {
+	v--
+	v |= v >> 1
+	v |= v >> 2
+	v |= v >> 4
+	v |= v >> 8
+	v |= v >> 16
+	v |= v >> 32
+	v++
+	return v
+}
+
+type node struct {
+	position uint64
+	data     interface{}
+}
+
+type nodes []node
+
+// RingBuffer is a MPMC buffer that achieves threadsafety with CAS operations
+// only.  A put on full or get on empty call will block until an item
+// is put or retrieved.  Calling Dispose on the RingBuffer will unblock
+// any blocked threads with an error.  This buffer is similar to the buffer
+// described here: http://www.1024cores.net/home/lock-free-algorithms/queues/bounded-mpmc-queue
+// with some minor additions.
+type RingBuffer struct {
+	_padding0      [8]uint64
+	queue          uint64
+	_padding1      [8]uint64
+	dequeue        uint64
+	_padding2      [8]uint64
+	mask, disposed uint64
+	_padding3      [8]uint64
+	nodes          nodes
+}
+
+func (rb *RingBuffer) init(size uint64) {
+	size = roundUp(size)
+	rb.nodes = make(nodes, size)
+	for i := uint64(0); i < size; i++ {
+		rb.nodes[i] = node{position: i}
+	}
+	rb.mask = size - 1 // so we don't have to do this with every put/get operation
+}
+
+// Put adds the provided item to the queue.  If the queue is full, this
+// call will block until an item is added to the queue or Dispose is called
+// on the queue.  An error will be returned if the queue is disposed.
+func (rb *RingBuffer) Put(item interface{}) error {
+	_, err := rb.put(item, false)
+	return err
+}
+
+// Offer adds the provided item to the queue if there is space.  If the queue
+// is full, this call will return false.  An error will be returned if the
+// queue is disposed.
+func (rb *RingBuffer) Offer(item interface{}) (bool, error) {
+	return rb.put(item, true)
+}
+
+func (rb *RingBuffer) put(item interface{}, offer bool) (bool, error) {
+	var n *node
+	pos := atomic.LoadUint64(&rb.queue)
+L:
+	for {
+		if atomic.LoadUint64(&rb.disposed) == 1 {
+			return false, ErrDisposed
+		}
+
+		n = &rb.nodes[pos&rb.mask]
+		seq := atomic.LoadUint64(&n.position)
+		switch dif := seq - pos; {
+		case dif == 0:
+			if atomic.CompareAndSwapUint64(&rb.queue, pos, pos+1) {
+				break L
+			}
+		case dif < 0:
+			panic(`Ring buffer in a compromised state during a put operation.`)
+		default:
+			pos = atomic.LoadUint64(&rb.queue)
+		}
+
+		if offer {
+			return false, nil
+		}
+
+		runtime.Gosched() // free up the cpu before the next iteration
+	}
+
+	n.data = item
+	atomic.StoreUint64(&n.position, pos+1)
+	return true, nil
+}
+
+// Get will return the next item in the queue.  This call will block
+// if the queue is empty.  This call will unblock when an item is added
+// to the queue or Dispose is called on the queue.  An error will be returned
+// if the queue is disposed.
+func (rb *RingBuffer) Get() (interface{}, error) {
+	return rb.Poll(0)
+}
+
+// Poll will return the next item in the queue.  This call will block
+// if the queue is empty.  This call will unblock when an item is added
+// to the queue, Dispose is called on the queue, or the timeout is reached. An
+// error will be returned if the queue is disposed or a timeout occurs. A
+// non-positive timeout will block indefinitely.
+func (rb *RingBuffer) Poll(timeout time.Duration) (interface{}, error) {
+	var (
+		n     *node
+		pos   = atomic.LoadUint64(&rb.dequeue)
+		start time.Time
+	)
+	if timeout > 0 {
+		start = time.Now()
+	}
+L:
+	for {
+		if atomic.LoadUint64(&rb.disposed) == 1 {
+			return nil, ErrDisposed
+		}
+
+		n = &rb.nodes[pos&rb.mask]
+		seq := atomic.LoadUint64(&n.position)
+		switch dif := seq - (pos + 1); {
+		case dif == 0:
+			if atomic.CompareAndSwapUint64(&rb.dequeue, pos, pos+1) {
+				break L
+			}
+		case dif < 0:
+			panic(`Ring buffer in compromised state during a get operation.`)
+		default:
+			pos = atomic.LoadUint64(&rb.dequeue)
+		}
+
+		if timeout > 0 && time.Since(start) >= timeout {
+			return nil, ErrTimeout
+		}
+
+		if timeout < 0 {
+			return nil, ErrTimeout
+		}
+
+		runtime.Gosched() // free up the cpu before the next iteration
+	}
+	data := n.data
+	n.data = nil
+	atomic.StoreUint64(&n.position, pos+rb.mask+1)
+	return data, nil
+}
+
+// Len returns the number of items in the queue.
+func (rb *RingBuffer) Len() uint64 {
+	return atomic.LoadUint64(&rb.queue) - atomic.LoadUint64(&rb.dequeue)
+}
+
+// Cap returns the capacity of this ring buffer.
+func (rb *RingBuffer) Cap() uint64 {
+	return uint64(len(rb.nodes))
+}
+
+// Dispose will dispose of this queue and free any blocked threads
+// in the Put and/or Get methods.  Calling those methods on a disposed
+// queue will return an error.
+func (rb *RingBuffer) Dispose() {
+	atomic.CompareAndSwapUint64(&rb.disposed, 0, 1)
+}
+
+// IsDisposed will return a bool indicating if this queue has been
+// disposed.
+func (rb *RingBuffer) IsDisposed() bool {
+	return atomic.LoadUint64(&rb.disposed) == 1
+}
+
+// NewRingBuffer will allocate, initialize, and return a ring buffer
+// with the specified size.
+func NewRingBuffer(size uint64) *RingBuffer {
+	rb := &RingBuffer{}
+	rb.init(size)
+	return rb
+}

s3_reader.go

@@ -60,7 +60,7 @@ func NewS3ReadCloser(path string) *S3ReadCloser {
 	bucket, key := parseS3Url(path)
 	sess := session.Must(session.NewSession(awsConfig()))
 
-	log.Println("[S3 Input] S3 connection succesfully initialized", path)
+	log.Println("[S3 Input] S3 connection successfully initialized", path)
 
 	return &S3ReadCloser{
 		bucket: bucket,

s3_test.go

@@ -127,7 +127,7 @@ func TestInputFileFromS3(t *testing.T) {
 		<-output.closeCh
 	}
 
-	input := NewFileInput(fmt.Sprintf("s3://test-gor-eu/%d", rnd), false)
+	input := NewFileInput(fmt.Sprintf("s3://test-gor-eu/%d", rnd), false, 100, 0, false)
 
 	buf := make([]byte, 1000)
 	for i := 0; i <= 19999; i++ {

settings.go

@@ -4,23 +4,59 @@ import (
 	"flag"
 	"fmt"
 	"os"
+	"strconv"
 	"sync"
 	"time"
+
+	"github.com/buger/goreplay/size"
 )
 
 // DEMO indicates that goreplay is running in demo mode
 var DEMO string
 
 // MultiOption allows to specify multiple flags with same name and collects all values into array
-type MultiOption []string
+type MultiOption struct {
+	a *[]string
+}
 
 func (h *MultiOption) String() string {
-	return fmt.Sprint(*h)
+	if h.a == nil {
+		return ""
+	}
+	return fmt.Sprint(*h.a)
 }
 
 // Set gets called multiple times for each flag with same name
 func (h *MultiOption) Set(value string) error {
-	*h = append(*h, value)
+	if h.a == nil {
+		return nil
+	}
+
+	*h.a = append(*h.a, value)
+	return nil
+}
+
+// MultiOption allows to specify multiple flags with same name and collects all values into array
+type MultiIntOption struct {
+	a *[]int
+}
+
+func (h *MultiIntOption) String() string {
+	if h.a == nil {
+		return ""
+	}
+
+	return fmt.Sprint(*h.a)
+}
+
+// Set gets called multiple times for each flag with same name
+func (h *MultiIntOption) Set(value string) error {
+	if h.a == nil {
+		return nil
+	}
+
+	val, _ := strconv.Atoi(value)
+	*h.a = append(*h.a, val)
 	return nil
 }
 
@@ -34,34 +70,39 @@ type AppSettings struct {
 	RecognizeTCPSessions bool   `json:"recognize-tcp-sessions"`
 	Pprof                string `json:"http-pprof"`
 
-	InputDummy   MultiOption `json:"input-dummy"`
-	OutputDummy  MultiOption
+	CopyBufferSize size.Size `json:"copy-buffer-size"`
+
+	InputDummy   []string `json:"input-dummy"`
+	OutputDummy  []string
 	OutputStdout bool `json:"output-stdout"`
 	OutputNull   bool `json:"output-null"`
 
-	InputTCP        MultiOption `json:"input-tcp"`
+	InputTCP        []string `json:"input-tcp"`
 	InputTCPConfig  TCPInputConfig
-	OutputTCP       MultiOption `json:"output-tcp"`
+	OutputTCP       []string `json:"output-tcp"`
 	OutputTCPConfig TCPOutputConfig
 	OutputTCPStats  bool `json:"output-tcp-stats"`
 
-	InputFile        MultiOption `json:"input-file"`
-	InputFileLoop    bool        `json:"input-file-loop"`
-	OutputFile       MultiOption `json:"output-file"`
-	OutputFileConfig FileOutputConfig
+	InputFile          []string      `json:"input-file"`
+	InputFileLoop      bool          `json:"input-file-loop"`
+	InputFileReadDepth int           `json:"input-file-read-depth"`
+	InputFileDryRun    bool          `json:"input-file-dry-run"`
+	InputFileMaxWait   time.Duration `json:"input-file-max-wait"`
+	OutputFile         []string      `json:"output-file"`
+	OutputFileConfig   FileOutputConfig
 
-	InputRAW MultiOption `json:"input_raw"`
-	RAWInputConfig
+	InputRAW       []string `json:"input_raw"`
+	InputRAWConfig RAWInputConfig
 
 	Middleware string `json:"middleware"`
 
-	InputHTTP    MultiOption
-	OutputHTTP   MultiOption `json:"output-http"`
-	PrettifyHTTP bool        `json:"prettify-http"`
+	InputHTTP    []string
+	OutputHTTP   []string `json:"output-http"`
+	PrettifyHTTP bool     `json:"prettify-http"`
 
 	OutputHTTPConfig HTTPOutputConfig
 
-	OutputBinary       MultiOption `json:"output-binary"`
+	OutputBinary       []string `json:"output-binary"`
 	OutputBinaryConfig BinaryOutputConfig
 
 	ModifierConfig HTTPModifierConfig
@@ -95,26 +136,29 @@ func init() {
 	flag.BoolVar(&Settings.SplitOutput, "split-output", false, "By default each output gets same traffic. If set to `true` it splits traffic equally among all outputs.")
 	flag.BoolVar(&Settings.RecognizeTCPSessions, "recognize-tcp-sessions", false, "[PRO] If turned on http output will create separate worker for each TCP session. Splitting output will session based as well.")
 
-	flag.Var(&Settings.InputDummy, "input-dummy", "Used for testing outputs. Emits 'Get /' request every 1s")
+	flag.Var(&MultiOption{&Settings.InputDummy}, "input-dummy", "Used for testing outputs. Emits 'Get /' request every 1s")
 	flag.BoolVar(&Settings.OutputStdout, "output-stdout", false, "Used for testing inputs. Just prints to console data coming from inputs.")
 	flag.BoolVar(&Settings.OutputNull, "output-null", false, "Used for testing inputs. Drops all requests.")
 
-	flag.Var(&Settings.InputTCP, "input-tcp", "Used for internal communication between Gor instances. Example: \n\t# Receive requests from other Gor instances on 28020 port, and redirect output to staging\n\tgor --input-tcp :28020 --output-http staging.com")
+	flag.Var(&MultiOption{&Settings.InputTCP}, "input-tcp", "Used for internal communication between Gor instances. Example: \n\t# Receive requests from other Gor instances on 28020 port, and redirect output to staging\n\tgor --input-tcp :28020 --output-http staging.com")
 	flag.BoolVar(&Settings.InputTCPConfig.Secure, "input-tcp-secure", false, "Turn on TLS security. Do not forget to specify certificate and key files.")
 	flag.StringVar(&Settings.InputTCPConfig.CertificatePath, "input-tcp-certificate", "", "Path to PEM encoded certificate file. Used when TLS turned on.")
 	flag.StringVar(&Settings.InputTCPConfig.KeyPath, "input-tcp-certificate-key", "", "Path to PEM encoded certificate key file. Used when TLS turned on.")
 
-	flag.Var(&Settings.OutputTCP, "output-tcp", "Used for internal communication between Gor instances. Example: \n\t# Listen for requests on 80 port and forward them to other Gor instance on 28020 port\n\tgor --input-raw :80 --output-tcp replay.local:28020")
+	flag.Var(&MultiOption{&Settings.OutputTCP}, "output-tcp", "Used for internal communication between Gor instances. Example: \n\t# Listen for requests on 80 port and forward them to other Gor instance on 28020 port\n\tgor --input-raw :80 --output-tcp replay.local:28020")
 	flag.BoolVar(&Settings.OutputTCPConfig.Secure, "output-tcp-secure", false, "Use TLS secure connection. --input-file on another end should have TLS turned on as well.")
 	flag.BoolVar(&Settings.OutputTCPConfig.SkipVerify, "output-tcp-skip-verify", false, "Don't verify hostname on TLS secure connection.")
 	flag.BoolVar(&Settings.OutputTCPConfig.Sticky, "output-tcp-sticky", false, "Use Sticky connection. Request/Response with same ID will be sent to the same connection.")
 	flag.IntVar(&Settings.OutputTCPConfig.Workers, "output-tcp-workers", 10, "Number of parallel tcp connections, default is 10")
 	flag.BoolVar(&Settings.OutputTCPStats, "output-tcp-stats", false, "Report TCP output queue stats to console every 5 seconds.")
 
-	flag.Var(&Settings.InputFile, "input-file", "Read requests from file: \n\tgor --input-file ./requests.gor --output-http staging.com")
+	flag.Var(&MultiOption{&Settings.InputFile}, "input-file", "Read requests from file: \n\tgor --input-file ./requests.gor --output-http staging.com")
 	flag.BoolVar(&Settings.InputFileLoop, "input-file-loop", false, "Loop input files, useful for performance testing.")
+	flag.IntVar(&Settings.InputFileReadDepth, "input-file-read-depth", 100, "GoReplay tries to read and cache multiple records, in advance. In parallel it also perform sorting of requests, if they came out of order. Since it needs hold this buffer in memory, bigger values can cause worse performance")
+	flag.BoolVar(&Settings.InputFileDryRun, "input-file-dry-run", false, "Simulate reading from the data source without replaying it. You will get information about expected replay time, number of found records etc.")
+	flag.DurationVar(&Settings.InputFileMaxWait, "input-file-max-wait", 0, "Set the maximum time between requests. Can help in situations when you have too long periods between request, and you want to skip them. Example: --input-raw-max-wait 1s")
 
-	flag.Var(&Settings.OutputFile, "output-file", "Write incoming requests to file: \n\tgor --input-raw :80 --output-file ./requests.gor")
+	flag.Var(&MultiOption{&Settings.OutputFile}, "output-file", "Write incoming requests to file: \n\tgor --input-raw :80 --output-file ./requests.gor")
 	flag.DurationVar(&Settings.OutputFileConfig.FlushInterval, "output-file-flush-interval", time.Second, "Interval for forcing buffer flush to the file, default: 1s.")
 	flag.BoolVar(&Settings.OutputFileConfig.Append, "output-file-append", false, "The flushed chunk is appended to existence file or not. ")
 	flag.Var(&Settings.OutputFileConfig.SizeLimit, "output-file-size-limit", "Size of each chunk. Default: 32mb")
@@ -123,28 +167,35 @@ func init() {
 
 	flag.StringVar(&Settings.OutputFileConfig.BufferPath, "output-file-buffer", "/tmp", "The path for temporary storing current buffer: \n\tgor --input-raw :80 --output-file s3://mybucket/logs/%Y-%m-%d.gz --output-file-buffer /mnt/logs")
 
-	flag.BoolVar(&Settings.PrettifyHTTP, "prettify-http", false, "If enabled, will automatically decode requests and responses with: Content-Encoding: gzip and Transfer-Encoding: chunked. Useful for debugging, in conjuction with --output-stdout")
+	flag.BoolVar(&Settings.PrettifyHTTP, "prettify-http", false, "If enabled, will automatically decode requests and responses with: Content-Encoding: gzip and Transfer-Encoding: chunked. Useful for debugging, in conjunction with --output-stdout")
+
+	flag.Var(&Settings.CopyBufferSize, "copy-buffer-size", "Set the buffer size for an individual request (default 5MB)")
 
 	// input raw flags
-	flag.Var(&Settings.InputRAW, "input-raw", "Capture traffic from given port (use RAW sockets and require *sudo* access):\n\t# Capture traffic from 8080 port\n\tgor --input-raw :8080 --output-http staging.com")
-	flag.BoolVar(&Settings.TrackResponse, "input-raw-track-response", false, "If turned on Gor will track responses in addition to requests, and they will be available to middleware and file output.")
-	flag.Var(&Settings.Engine, "input-raw-engine", "Intercept traffic using `libpcap` (default), `raw_socket` or `pcap_file`")
-	flag.Var(&Settings.Protocol, "input-raw-protocol", "Specify application protocol of intercepted traffic. Possible values: http, binary")
-	flag.StringVar(&Settings.RealIPHeader, "input-raw-realip-header", "", "If not blank, injects header with given name and real IP value to the request payload. Usually this header should be named: X-Real-IP")
-	flag.DurationVar(&Settings.Expire, "input-raw-expire", time.Second*2, "How much it should wait for the last TCP packet, till consider that TCP message complete.")
-	flag.StringVar(&Settings.BPFFilter, "input-raw-bpf-filter", "", "BPF filter to write custom expressions. Can be useful in case of non standard network interfaces like tunneling or SPAN port. Example: --input-raw-bpf-filter 'dst port 80'")
-	flag.StringVar(&Settings.TimestampType, "input-raw-timestamp-type", "", "Possible values: PCAP_TSTAMP_HOST, PCAP_TSTAMP_HOST_LOWPREC, PCAP_TSTAMP_HOST_HIPREC, PCAP_TSTAMP_ADAPTER, PCAP_TSTAMP_ADAPTER_UNSYNCED. This values not supported on all systems, GoReplay will tell you available values of you put wrong one.")
-	flag.Var(&Settings.CopyBufferSize, "copy-buffer-size", "Set the buffer size for an individual request (default 5MB)")
-	flag.BoolVar(&Settings.Snaplen, "input-raw-override-snaplen", false, "Override the capture snaplen to be 64k. Required for some Virtualized environments")
-	flag.DurationVar(&Settings.BufferTimeout, "input-raw-buffer-timeout", 0, "set the pcap timeout. for immediate mode don't set this flag")
-	flag.Var(&Settings.BufferSize, "input-raw-buffer-size", "Controls size of the OS buffer which holds packets until they dispatched. Default value depends by system: in Linux around 2MB. If you see big package drop, increase this value.")
-	flag.BoolVar(&Settings.Promiscuous, "input-raw-promisc", false, "enable promiscuous mode")
-	flag.BoolVar(&Settings.Monitor, "input-raw-monitor", false, "enable RF monitor mode")
-	flag.BoolVar(&Settings.Stats, "input-raw-stats", false, "enable stats generator on raw TCP messages")
+	flag.Var(&MultiOption{&Settings.InputRAW}, "input-raw", "Capture traffic from given port (use RAW sockets and require *sudo* access):\n\t# Capture traffic from 8080 port\n\tgor --input-raw :8080 --output-http staging.com")
+	flag.BoolVar(&Settings.InputRAWConfig.TrackResponse, "input-raw-track-response", false, "If turned on Gor will track responses in addition to requests, and they will be available to middleware and file output.")
+	flag.IntVar(&Settings.InputRAWConfig.VXLANPort, "input-raw-vxlan-port", 4789, "VXLAN port. Can be used only when engine set to `vxlan`. Default: 4789")
+	flag.Var(&MultiIntOption{&Settings.InputRAWConfig.VXLANVNIs}, "input-raw-vxlan-vni", "VXLAN VNI to capture. By default capture all VNIs. Ignore VNI by setting them with minus sign, example: `--input-raw-vxlan-vni -2`")
+	flag.BoolVar(&Settings.InputRAWConfig.VLAN, "input-raw-vlan", false, "Enable VLAN (802.1Q) support")
+	flag.Var(&MultiIntOption{&Settings.InputRAWConfig.VLANVIDs}, "input-raw-vlan-vid", "VLAN VID to capture. By default capture all VIDs")
+	flag.Var(&Settings.InputRAWConfig.Engine, "input-raw-engine", "Intercept traffic using `libpcap` (default), `raw_socket`, `pcap_file`, `vxlan`")
+	flag.Var(&Settings.InputRAWConfig.Protocol, "input-raw-protocol", "Specify application protocol of intercepted traffic. Possible values: http, binary")
+	flag.StringVar(&Settings.InputRAWConfig.RealIPHeader, "input-raw-realip-header", "", "If not blank, injects header with given name and real IP value to the request payload. Usually this header should be named: X-Real-IP")
+	flag.DurationVar(&Settings.InputRAWConfig.Expire, "input-raw-expire", time.Second*2, "How much it should wait for the last TCP packet, till consider that TCP message complete.")
+	flag.StringVar(&Settings.InputRAWConfig.BPFFilter, "input-raw-bpf-filter", "", "BPF filter to write custom expressions. Can be useful in case of non standard network interfaces like tunneling or SPAN port. Example: --input-raw-bpf-filter 'dst port 80'")
+	flag.StringVar(&Settings.InputRAWConfig.TimestampType, "input-raw-timestamp-type", "", "Possible values: PCAP_TSTAMP_HOST, PCAP_TSTAMP_HOST_LOWPREC, PCAP_TSTAMP_HOST_HIPREC, PCAP_TSTAMP_ADAPTER, PCAP_TSTAMP_ADAPTER_UNSYNCED. This values not supported on all systems, GoReplay will tell you available values of you put wrong one.")
+	flag.BoolVar(&Settings.InputRAWConfig.Snaplen, "input-raw-override-snaplen", false, "Override the capture snaplen to be 64k. Required for some Virtualized environments")
+	flag.DurationVar(&Settings.InputRAWConfig.BufferTimeout, "input-raw-buffer-timeout", 0, "set the pcap timeout. for immediate mode don't set this flag")
+	flag.Var(&Settings.InputRAWConfig.BufferSize, "input-raw-buffer-size", "Controls size of the OS buffer which holds packets until they dispatched. Default value depends by system: in Linux around 2MB. If you see big package drop, increase this value.")
+	flag.BoolVar(&Settings.InputRAWConfig.Promiscuous, "input-raw-promisc", false, "enable promiscuous mode")
+	flag.BoolVar(&Settings.InputRAWConfig.Monitor, "input-raw-monitor", false, "enable RF monitor mode")
+	flag.BoolVar(&Settings.InputRAWConfig.Stats, "input-raw-stats", false, "enable stats generator on raw TCP messages")
+	flag.BoolVar(&Settings.InputRAWConfig.AllowIncomplete, "input-raw-allow-incomplete", false, "If turned on Gor will record HTTP messages with missing packets")
+	flag.Var(&MultiOption{&Settings.InputRAWConfig.IgnoreInterface}, "input-raw-ignore-interface", "In case if you want listen for all interfaces except a few ones. Can be used in k8s environment. Example: --input-raw-ignore-interface cbr0 --input-raw-ignore-interface eth0 --input-raw-ignore-interface localhost")
 
 	flag.StringVar(&Settings.Middleware, "middleware", "", "Used for modifying traffic using external command")
 
-	flag.Var(&Settings.OutputHTTP, "output-http", "Forwards incoming requests to given http address.\n\t# Redirect all incoming requests to staging.com address \n\tgor --input-raw :80 --output-http http://staging.com")
+	flag.Var(&MultiOption{&Settings.OutputHTTP}, "output-http", "Forwards incoming requests to given http address.\n\t# Redirect all incoming requests to staging.com address \n\tgor --input-raw :80 --output-http http://staging.com")
 
 	/* outputHTTPConfig */
 	flag.Var(&Settings.OutputHTTPConfig.BufferSize, "output-http-response-buffer", "HTTP response buffer size, all data after this size will be discarded.")
@@ -164,7 +215,7 @@ func init() {
 	flag.StringVar(&Settings.OutputHTTPConfig.ElasticSearch, "output-http-elasticsearch", "", "Send request and response stats to ElasticSearch:\n\tgor --input-raw :8080 --output-http staging.com --output-http-elasticsearch 'es_host:api_port/index_name'")
 	/* outputHTTPConfig */
 
-	flag.Var(&Settings.OutputBinary, "output-binary", "Forwards incoming binary payloads to given address.\n\t# Redirect all incoming requests to staging.com address \n\tgor --input-raw :80 --input-raw-protocol binary --output-binary staging.com:80")
+	flag.Var(&MultiOption{&Settings.OutputBinary}, "output-binary", "Forwards incoming binary payloads to given address.\n\t# Redirect all incoming requests to staging.com address \n\tgor --input-raw :80 --input-raw-protocol binary --output-binary staging.com:80")
 
 	/* outputBinaryConfig */
 	flag.Var(&Settings.OutputBinaryConfig.BufferSize, "output-tcp-response-buffer", "TCP response buffer size, all data after this size will be discarded.")
@@ -187,7 +238,7 @@ func init() {
 	flag.StringVar(&Settings.KafkaTLSConfig.ClientCert, "kafka-tls-client-cert", "", "Client certificate for Kafka TLS Config (mandatory with to kafka-tls-ca-cert and kafka-tls-client-key)")
 	flag.StringVar(&Settings.KafkaTLSConfig.ClientKey, "kafka-tls-client-key", "", "Client Key for Kafka TLS Config (mandatory with to kafka-tls-client-cert and kafka-tls-client-key)")
 
-	flag.Var(&Settings.ModifierConfig.Headers, "http-set-header", "Inject additional headers to http reqest:\n\tgor --input-raw :8080 --output-http staging.com --http-set-header 'User-Agent: Gor'")
+	flag.Var(&Settings.ModifierConfig.Headers, "http-set-header", "Inject additional headers to http request:\n\tgor --input-raw :8080 --output-http staging.com --http-set-header 'User-Agent: Gor'")
 	flag.Var(&Settings.ModifierConfig.HeaderRewrite, "http-rewrite-header", "Rewrite the request header based on a mapping:\n\tgor --input-raw :8080 --output-http staging.com --http-rewrite-header Host: (.*).example.com,$1.beta.example.com")
 	flag.Var(&Settings.ModifierConfig.Params, "http-set-param", "Set request url param, if param already exists it will be overwritten:\n\tgor --input-raw :8080 --output-http staging.com --http-set-param api_key=1")
 	flag.Var(&Settings.ModifierConfig.Methods, "http-allow-method", "Whitelist of HTTP methods to replay. Anything else will be dropped:\n\tgor --input-raw :8080 --output-http staging.com --http-allow-method GET --http-allow-method OPTIONS")

simpletime/time.go

@@ -0,0 +1,17 @@
+package simpletime
+
+import (
+	"time"
+)
+
+var Now time.Time
+
+func init() {
+	go func() {
+		for {
+			// Accurate enough
+			Now = time.Now()
+			time.Sleep(100 * time.Millisecond)
+		}
+	}()
+}

tcp/doc.go

@@ -15,7 +15,7 @@ maxSize := 5 << 20
 debugger := func(debugLevel int, data ...interface{}){} // debugger can also be nil
 messageHandler := func(mssg *tcp.Message){}
 
-mssgPool := tcp.NewMessagePool(maxMessageSize, messageExpire, debugger, messageHandler)
+mssgPool := tcp.NewMessageParser(maxMessageSize, messageExpire, debugger, messageHandler)
 listener.Listen(ctx, mssgPool.Handler)
 
 you can use pool.End or/and pool.Start to set custom session behaviors

tcp/tcp_message.go

@@ -1,95 +1,134 @@
 package tcp
 
 import (
-	"bytes"
 	"encoding/binary"
 	"encoding/hex"
 	"fmt"
+	"net"
+	"reflect"
 	"sort"
-	"sync"
 	"time"
+	"unsafe"
 
-	"github.com/buger/goreplay/capture"
-	"github.com/buger/goreplay/size"
+	"github.com/buger/goreplay/proto"
 )
 
+// TCPProtocol is a number to indicate type of protocol
+type TCPProtocol uint8
+
+const (
+	// ProtocolHTTP ...
+	ProtocolHTTP TCPProtocol = iota
+	// ProtocolBinary ...
+	ProtocolBinary
+)
+
+// Set is here so that TCPProtocol can implement flag.Var
+func (protocol *TCPProtocol) Set(v string) error {
+	switch v {
+	case "", "http":
+		*protocol = ProtocolHTTP
+	case "binary":
+		*protocol = ProtocolBinary
+	default:
+		return fmt.Errorf("unsupported protocol %s", v)
+	}
+	return nil
+}
+
+func (protocol *TCPProtocol) String() string {
+	switch *protocol {
+	case ProtocolBinary:
+		return "binary"
+	case ProtocolHTTP:
+		return "http"
+	default:
+		return ""
+	}
+}
+
 // Stats every message carry its own stats object
 type Stats struct {
-	LostData   int
-	Length     int       // length of the data
-	Start      time.Time // first packet's timestamp
-	End        time.Time // last packet's timestamp
-	SrcAddr    string
-	DstAddr    string
-	IsIncoming bool
-	TimedOut   bool // timeout before getting the whole message
-	Truncated  bool // last packet truncated due to max message size
-	IPversion  byte
+	LostData  int
+	Length    int       // length of the data
+	Start     time.Time // first packet's timestamp
+	End       time.Time // last packet's timestamp
+	SrcAddr   string
+	DstAddr   string
+	Direction Dir
+	TimedOut  bool // timeout before getting the whole message
+	Truncated bool // last packet truncated due to max message size
+	IPversion byte
 }
 
 // Message is the representation of a tcp message
 type Message struct {
-	packets  []*Packet
-	done     chan bool
-	pool     *MessagePool
-	buf      *bytes.Buffer
-	feedback interface{}
+	packets          []*Packet
+	parser           *MessageParser
+	feedback         interface{}
+	continueAdjusted bool
 	Stats
 }
 
-// NewMessage ...
-func NewMessage(srcAddr, dstAddr string, ipVersion uint8) (m *Message) {
-	m = new(Message)
-	m.DstAddr = dstAddr
-	m.SrcAddr = srcAddr
-	m.IPversion = ipVersion
-	m.done = make(chan bool)
-	m.buf = &bytes.Buffer{}
-	return
-}
-
 // UUID returns the UUID of a TCP request and its response.
 func (m *Message) UUID() []byte {
-	var key uint64
+	var streamID uint64
 	pckt := m.packets[0]
 
 	// check if response or request have generated the ID before.
-	if m.pool.uuids != nil {
-		lst := len(pckt.SrcIP) - 4
-		if m.IsIncoming {
-			key = uint64(pckt.SrcPort)<<48 | uint64(pckt.DstPort)<<32 |
-				uint64(_uint32(pckt.SrcIP[lst:]))
-		} else {
-			key = uint64(pckt.DstPort)<<48 | uint64(pckt.SrcPort)<<32 |
-				uint64(_uint32(pckt.DstIP[lst:]))
-		}
-		if uuidHex, ok := m.pool.uuids[key]; ok {
-			delete(m.pool.uuids, key)
-			return uuidHex
-		}
+	if m.Direction == DirIncoming {
+		streamID = uint64(pckt.SrcPort)<<48 | uint64(pckt.DstPort)<<32 |
+			uint64(ip2int(pckt.SrcIP))
+	} else {
+		streamID = uint64(pckt.DstPort)<<48 | uint64(pckt.SrcPort)<<32 |
+			uint64(ip2int(pckt.DstIP))
 	}
 
-	id := make([]byte, 12, 12)
-	binary.BigEndian.PutUint32(id, pckt.Seq)
-	tStamp := m.End.UnixNano()
-	binary.BigEndian.PutUint64(id[4:], uint64(tStamp))
-	uuidHex := make([]byte, 24, 24)
-	hex.Encode(uuidHex[:], id[:])
-	if m.pool.uuids != nil {
-		if len(m.pool.uuids) >= 1000 {
-			m.pool.cleanUUIDs()
-		}
-		m.pool.uuids[key] = uuidHex
+	id := make([]byte, 12)
+	binary.BigEndian.PutUint64(id, streamID)
+
+	if m.Direction == DirIncoming {
+		binary.BigEndian.PutUint32(id[8:], pckt.Ack)
+	} else {
+		binary.BigEndian.PutUint32(id[8:], pckt.Seq)
 	}
+
+	uuidHex := make([]byte, 24)
+	hex.Encode(uuidHex[:], id[:])
+
 	return uuidHex
 }
 
-func (m *Message) add(pckt *Packet) {
-	m.Length += len(pckt.Payload)
-	m.LostData += int(pckt.Lost)
-	m.packets = append(m.packets, pckt)
-	m.End = pckt.Timestamp
-	m.buf.Write(pckt.Payload)
+func (m *Message) add(packet *Packet) bool {
+	// Skip duplicates
+	for _, p := range m.packets {
+		if p.Seq == packet.Seq {
+			return false
+		}
+	}
+
+	// Packets not always captured in same Seq order, and sometimes we need to prepend
+	if len(m.packets) == 0 || packet.Seq > m.packets[len(m.packets)-1].Seq {
+		m.packets = append(m.packets, packet)
+	} else if packet.Seq < m.packets[0].Seq {
+		m.packets = append([]*Packet{packet}, m.packets...)
+	} else { // insert somewhere in the middle...
+		for i, p := range m.packets {
+			if packet.Seq < p.Seq {
+				m.packets = append(m.packets[:i], append([]*Packet{packet}, m.packets[i:]...)...)
+				break
+			}
+		}
+	}
+
+	m.Length += len(packet.Payload)
+	m.LostData += int(packet.Lost)
+
+	if packet.Timestamp.After(m.End) || m.End.IsZero() {
+		m.End = packet.Timestamp
+	}
+
+	return true
 }
 
 // Packets returns packets of the message
@@ -97,18 +136,56 @@ func (m *Message) Packets() []*Packet {
 	return m.packets
 }
 
-// Data returns data in this message
-func (m *Message) Data() []byte {
-	return m.buf.Bytes()
+func (m *Message) MissingChunk() bool {
+	nextSeq := m.packets[0].Seq
+
+	for _, p := range m.packets {
+		if p.Seq != nextSeq {
+			return true
+		}
+
+		nextSeq += uint32(len(p.Payload))
+	}
+
+	return false
 }
 
-// SetFeedback set feedback/data that can be used later, e.g with End or Start hint
-func (m *Message) SetFeedback(feedback interface{}) {
+func (m *Message) PacketData() [][]byte {
+	tmp := make([][]byte, len(m.packets))
+
+	for i, p := range m.packets {
+		tmp[i] = p.Payload
+	}
+
+	return tmp
+}
+
+// Data returns data in this message
+func (m *Message) Data() []byte {
+	packetData := m.PacketData()
+	tmp := packetData[0]
+
+	if len(packetData) > 0 {
+		tmp, _ = copySlice(tmp, len(packetData[0]), packetData[1:]...)
+	}
+
+	// Remove Expect header, since its replay not fully supported
+	if state, ok := m.feedback.(*proto.HTTPState); ok {
+		if state.Continue100 {
+			tmp = proto.DeleteHeader(tmp, []byte("Expect"))
+		}
+	}
+
+	return tmp
+}
+
+// SetProtocolState set feedback/data that can be used later, e.g with End or Start hint
+func (m *Message) SetProtocolState(feedback interface{}) {
 	m.feedback = feedback
 }
 
-// Feedback returns feedback associated to this message
-func (m *Message) Feedback() interface{} {
+// ProtocolState returns feedback associated to this message
+func (m *Message) ProtocolState() interface{} {
 	return m.feedback
 }
 
@@ -117,171 +194,251 @@ func (m *Message) Sort() {
 	sort.SliceStable(m.packets, func(i, j int) bool { return m.packets[i].Seq < m.packets[j].Seq })
 }
 
-// Handler message handler
-type Handler func(*Message)
+// Emitter message handler
+type Emitter func(*Message)
 
-// Debugger is the debugger function. first params is the indicator of the issue's priority
-// the higher the number, the lower the priority. it can be 4 <= level <= 6.
-type Debugger func(int, ...interface{})
-
-// HintEnd hints the pool to stop the session, see MessagePool.End
+// HintEnd hints the parser to stop the session, see MessageParser.End
 // when set, it will be executed before checking FIN or RST flag
 type HintEnd func(*Message) bool
 
-// HintStart hints the pool to start the reassembling the message, see MessagePool.Start
+// HintStart hints the parser to start the reassembling the message, see MessageParser.Start
 // when set, it will be called after checking SYN flag
-type HintStart func(*Packet) (IsIncoming, IsOutgoing bool)
+type HintStart func(*Packet) (IsRequest, IsOutgoing bool)
 
-// MessagePool holds data of all tcp messages in progress(still receiving/sending packets).
+// MessageParser holds data of all tcp messages in progress(still receiving/sending packets).
 // message is identified by its source port and dst port, and last 4bytes of src IP.
-type MessagePool struct {
-	sync.Mutex
-	debug         Debugger
-	maxSize       size.Size // maximum message size, default 5mb
-	pool          map[uint64]*Message
-	uuids         map[uint64][]byte
-	handler       Handler
-	messageExpire time.Duration // the maximum time to wait for the final packet, minimum is 100ms
-	End           HintEnd
-	Start         HintStart
+type MessageParser struct {
+	m map[uint64]*Message
+
+	messageExpire  time.Duration // the maximum time to wait for the final packet, minimum is 100ms
+	allowIncompete bool
+	End            HintEnd
+	Start          HintStart
+	ticker         *time.Ticker
+	messages       chan *Message
+	packets        chan *PcapPacket
+	close          chan struct{} // to signal that we are able to close
+	ports          []uint16
+	ips            []net.IP
 }
 
-// NewMessagePool returns a new instance of message pool
-func NewMessagePool(maxSize size.Size, messageExpire time.Duration, debugger Debugger, handler Handler) (pool *MessagePool) {
-	pool = new(MessagePool)
-	pool.debug = debugger
-	pool.handler = handler
-	pool.messageExpire = time.Millisecond * 100
-	if pool.messageExpire < messageExpire {
-		pool.messageExpire = messageExpire
+// NewMessageParser returns a new instance of message parser
+func NewMessageParser(messages chan *Message, ports []uint16, ips []net.IP, messageExpire time.Duration, allowIncompete bool) (parser *MessageParser) {
+	parser = new(MessageParser)
+
+	parser.messageExpire = messageExpire
+	if parser.messageExpire == 0 {
+		parser.messageExpire = time.Millisecond * 1000
 	}
-	pool.maxSize = maxSize
-	if pool.maxSize < 1 {
-		pool.maxSize = 5 << 20
+
+	parser.allowIncompete = allowIncompete
+
+	parser.packets = make(chan *PcapPacket, 10000)
+
+	if messages == nil {
+		messages = make(chan *Message, 1000)
 	}
-	pool.pool = make(map[uint64]*Message)
-	return pool
+	parser.messages = messages
+
+	parser.m = make(map[uint64]*Message)
+	parser.ticker = time.NewTicker(time.Millisecond * 100)
+	parser.close = make(chan struct{}, 1)
+
+	parser.ports = ports
+	parser.ips = ips
+
+	go parser.wait()
+	return parser
 }
 
-// Handler returns packet handler
-func (pool *MessagePool) Handler(packet *capture.Packet) {
-	var in, out bool
-	pckt, err := ParsePacket(packet)
+var packetLen int
+
+// Packet returns packet handler
+func (parser *MessageParser) PacketHandler(packet *PcapPacket) {
+	packetLen++
+	parser.packets <- packet
+}
+
+func (parser *MessageParser) wait() {
+	var (
+		now time.Time
+	)
+	for {
+		select {
+		case pckt := <-parser.packets:
+			parser.processPacket(parser.parsePacket(pckt))
+		case now = <-parser.ticker.C:
+			parser.timer(now)
+		case <-parser.close:
+			parser.ticker.Stop()
+			// parser.Close should wait for this function to return
+			parser.close <- struct{}{}
+			return
+			// default:
+		}
+	}
+}
+
+func (parser *MessageParser) parsePacket(pcapPkt *PcapPacket) *Packet {
+	pckt, err := ParsePacket(pcapPkt.Data, pcapPkt.LType, pcapPkt.LTypeLen, pcapPkt.Ci, false)
 	if err != nil {
-		go pool.say(4, fmt.Sprintf("error decoding packet(%dBytes):%s\n", packet.Info.CaptureLength, err))
-		return
+		if _, empty := err.(EmptyPacket); !empty {
+			stats.Add("packet_error", 1)
+		}
+		return nil
 	}
-	pool.Lock()
-	defer pool.Unlock()
-	lst := len(pckt.SrcIP) - 4
-	key := uint64(pckt.SrcPort)<<48 | uint64(pckt.DstPort)<<32 |
-		uint64(_uint32(pckt.SrcIP[lst:]))
-	m, ok := pool.pool[key]
-	if pckt.RST {
-		if ok {
-			m.done <- true
-			<-m.done
+
+	for _, p := range parser.ports {
+		if pckt.DstPort == p && containsOrEmpty(pckt.DstIP, parser.ips) {
+			pckt.Direction = DirIncoming
+			break
+		} else if pckt.SrcPort == p && containsOrEmpty(pckt.SrcIP, parser.ips) {
+			pckt.Direction = DirOutcoming
+			break
 		}
-		key = uint64(pckt.DstPort)<<48 | uint64(pckt.SrcPort)<<32 |
-			uint64(_uint32(pckt.DstIP[lst:]))
-		m, ok = pool.pool[key]
-		if ok {
-			m.done <- true
-			<-m.done
+	}
+
+	return pckt
+}
+
+func containsOrEmpty(element net.IP, ipList []net.IP) bool {
+	if len(ipList) == 0 {
+		return true
+	}
+	for _, ip := range ipList {
+		if ip.Equal(element) {
+			return true
 		}
-		go pool.say(4, fmt.Sprintf("RST flag from %s to %s at %s\n", pckt.Src(), pckt.Dst(), pckt.Timestamp))
+	}
+	return false
+}
+
+func (parser *MessageParser) processPacket(pckt *Packet) {
+	if pckt == nil {
 		return
 	}
+
+	// Trying to build unique hash, but there is small chance of collision
+	// No matter if it is request or response, all packets in the same message have same
+	m, ok := parser.m[pckt.MessageID()]
 	switch {
 	case ok:
-		pool.addPacket(m, pckt)
-		return
-	case pckt.SYN:
-		in = !pckt.ACK
-	case pool.Start != nil:
-		if in, out = pool.Start(pckt); !(in || out) {
-			return
+		if m.Direction == DirUnknown {
+			if in, out := parser.Start(pckt); in || out {
+				if in {
+					m.Direction = DirIncoming
+				} else {
+					m.Direction = DirOutcoming
+				}
+			}
 		}
-	default:
+		parser.addPacket(m, pckt)
 		return
+	case pckt.Direction == DirUnknown && parser.Start != nil:
+		if in, out := parser.Start(pckt); in || out {
+			if in {
+				pckt.Direction = DirIncoming
+			} else {
+				pckt.Direction = DirOutcoming
+			}
+		}
 	}
-	m = NewMessage(pckt.Src(), pckt.Dst(), pckt.Version)
-	m.IsIncoming = in
-	pool.pool[key] = m
+
+	m = new(Message)
+	m.Direction = pckt.Direction
+	m.SrcAddr = pckt.SrcIP.String()
+	m.DstAddr = pckt.DstIP.String()
+
+	parser.m[pckt.MessageID()] = m
+	
 	m.Start = pckt.Timestamp
-	m.pool = pool
-	go pool.dispatch(key, m)
-	pool.addPacket(m, pckt)
+	m.parser = parser
+	parser.addPacket(m, pckt)
 }
 
-// MatchUUID instructs the pool to use same UUID for request and responses
-// this function should be called at initial stage of the pool
-func (pool *MessagePool) MatchUUID(match bool) {
-	if match {
-		pool.uuids = make(map[uint64][]byte)
-		return
+func (parser *MessageParser) addPacket(m *Message, pckt *Packet) bool {
+	if !m.add(pckt) {
+		return false
 	}
-	pool.uuids = nil
+
+	// If we are using protocol parsing, like HTTP, depend on its parsing func.
+	// For the binary procols wait for message to expire
+	if parser.End != nil {
+		if parser.End(m) {
+			parser.Emit(m)
+			return true
+		}
+
+		parser.Fix100Continue(m)
+	}
+
+	return true
 }
 
-// run GC on UUID map
-func (pool *MessagePool) cleanUUIDs() {
-	var tStamp int64
-	now := time.Now().UnixNano()
-	for k, v := range pool.uuids {
-		// there is a timestamp wrapped in every ID
-		tStamp = int64(binary.BigEndian.Uint64(v[4:]))
-		if time.Duration(now-tStamp) > pool.messageExpire {
-			delete(pool.uuids, k)
+func (parser *MessageParser) Fix100Continue(m *Message) {
+	// Only adjust a message once
+	if state, ok := m.feedback.(*proto.HTTPState); ok && state.Continue100 && !m.continueAdjusted {
+		// Shift Ack by given offset
+		// Size of "HTTP/1.1 100 Continue\r\n\r\n" message
+		for _, p := range m.packets {
+			p.messageID = 0
+			p.Ack += 25
+		}
+
+		// If next section was aready approved and received, merge messages
+		if next, found := parser.m[m.packets[0].MessageID()]; found {
+			for _, p := range next.packets {
+				parser.addPacket(m, p)
+			}
+		}
+
+		// Re-add (or override) again with new message and ID
+		parser.m[m.packets[0].MessageID()] = m
+		m.continueAdjusted = true
+	}
+}
+
+func (parser *MessageParser) Read() *Message {
+	m := <-parser.messages
+	return m
+}
+
+func (parser *MessageParser) Emit(m *Message) {
+	stats.Add("message_count", 1)
+
+	delete(parser.m, m.packets[0].MessageID())
+
+	parser.messages <- m
+}
+
+func GetUnexportedField(field reflect.Value) interface{} {
+	return reflect.NewAt(field.Type(), unsafe.Pointer(field.UnsafeAddr())).Elem().Interface()
+}
+
+var failMsg int
+
+func (parser *MessageParser) timer(now time.Time) {
+	packetLen = 0
+
+	packetQueueLen.Set(int64(len(parser.packets)))
+	messageQueueLen.Set(int64(len(parser.m)))
+
+	for _, m := range parser.m {
+		if now.Sub(m.End) > parser.messageExpire {
+			m.TimedOut = true
+			stats.Add("message_timeout_count", 1)
+			failMsg++
+			if parser.End == nil || parser.allowIncompete {
+				parser.Emit(m)
+			}
+
+			delete(parser.m, m.packets[0].MessageID())
 		}
 	}
 }
 
-func (pool *MessagePool) dispatch(key uint64, m *Message) {
-	select {
-	case <-m.done:
-		defer func() { m.done <- true }() // signal that message was dispatched
-	case <-time.After(pool.messageExpire):
-		pool.Lock()
-		defer pool.Unlock()
-		// avoid dispathing message twice
-		if _, ok := pool.pool[key]; !ok {
-			return
-		}
-		m.TimedOut = true
-	}
-	delete(pool.pool, key)
-	pool.handler(m)
-}
-
-func (pool *MessagePool) addPacket(m *Message, pckt *Packet) {
-	trunc := m.Length + len(pckt.Payload) - int(pool.maxSize)
-	if trunc > 0 {
-		m.Truncated = true
-		pckt.Payload = pckt.Payload[:int(pool.maxSize)-m.Length]
-	}
-	m.add(pckt)
-	switch {
-
-	// if one of this cases matches, we dispatch the message
-	case trunc >= 0:
-	case pckt.FIN:
-	case pool.End != nil && pool.End(m):
-	default:
-		// continue to receive packets
-		return
-	}
-	m.done <- true
-	<-m.done
-}
-
-// this function should not block other pool operations
-func (pool *MessagePool) say(level int, args ...interface{}) {
-	if pool.debug != nil {
-		pool.debug(level, args...)
-	}
-}
-
-func _uint32(b []byte) uint32 {
-	return binary.BigEndian.Uint32(b)
+func (parser *MessageParser) Close() error {
+	parser.close <- struct{}{}
+	<-parser.close // wait for timer to be closed!
+	return nil
 }

tcp/tcp_packet.go

@@ -2,12 +2,52 @@ package tcp
 
 import (
 	"encoding/binary"
-	"errors"
+	"expvar"
 	"fmt"
 	"net"
 	"time"
 
-	"github.com/buger/goreplay/capture"
+	"github.com/google/gopacket"
+)
+
+func copySlice(to []byte, skip int, from ...[]byte) ([]byte, int) {
+	var totalLen int
+	for _, s := range from {
+		totalLen += len(s)
+	}
+	totalLen += skip
+
+	if cap(to) < totalLen {
+		diff := totalLen - cap(to)
+		to = append(to, make([]byte, diff)...)
+	}
+
+	for _, s := range from {
+		skip += copy(to[skip:], s)
+	}
+
+	return to, skip
+}
+
+var stats *expvar.Map
+var packetQueueLen, messageQueueLen *expvar.Int
+
+func init() {
+	packetQueueLen = new(expvar.Int)
+	messageQueueLen = new(expvar.Int)
+
+	stats = expvar.NewMap("tcp")
+	stats.Init()
+	stats.Set("packet_queue", packetQueueLen)
+	stats.Set("message_queue", messageQueueLen)
+}
+
+type Dir int
+
+const (
+	DirUnknown = iota
+	DirIncoming
+	DirOutcoming
 )
 
 /*
@@ -17,52 +57,169 @@ calllers must make sure that ParsePacket has'nt returned any error before callin
 function.
 */
 type Packet struct {
+	Direction          Dir
+	messageID          uint64
 	SrcIP, DstIP       net.IP
 	Version            uint8
 	SrcPort, DstPort   uint16
 	Ack, Seq           uint32
 	ACK, SYN, FIN, RST bool
 	Lost               uint32
+	Retry              int
+	CaptureLength      int
 	Timestamp          time.Time
 	Payload            []byte
+	buf                []byte
+
+	created time.Time
+	gc      bool
+}
+
+type PcapPacket struct {
+	Data     []byte
+	LType    int
+	LTypeLen int
+	Ci       *gopacket.CaptureInfo
 }
 
 // ParsePacket parse raw packets
-func ParsePacket(packet *capture.Packet) (pckt *Packet, err error) {
-	// early check of error
-	if packet == nil {
-		return nil, errors.New("empty packet")
-	}
-	if packet.Err != nil {
-		return nil, packet.Err
+func ParsePacket(data []byte, lType, lTypeLen int, ci *gopacket.CaptureInfo, allowEmpty bool) (pckt *Packet, err error) {
+	pckt = new(Packet)
+	if err := pckt.parse(data, lType, lTypeLen, ci, allowEmpty); err != nil {
+		return nil, err
 	}
 
-	var t Packet
-	pckt = &t
+	return pckt, nil
+}
+
+func (pckt *Packet) parse(data []byte, lType, lTypeLen int, cp *gopacket.CaptureInfo, allowEmpty bool) error {
+	pckt.Retry = 0
+	pckt.messageID = 0
+	pckt.buf = pckt.buf[:]
+
 	// TODO: check resolution
-	pckt.Timestamp = packet.Info.Timestamp
-	if (packet.NetLayer[0] >> 4) == 4 {
+	pckt.Timestamp = cp.Timestamp
+
+	if len(data) < lTypeLen {
+		return ErrHdrLength("Link")
+	}
+	if len(data) <= lTypeLen {
+		return ErrHdrMissing("IPv4 or IPv6")
+	}
+
+	ldata := data[lTypeLen:]
+	var proto byte
+	var netLayer, transLayer []byte
+
+	if ldata[0]>>4 == 4 {
+		// IPv4 header
+		if len(ldata) < 20 {
+			return ErrHdrLength("IPv4")
+		}
+		proto = ldata[9]
+		ihl := int(ldata[0]&0x0F) * 4
+		if ihl < 20 {
+			return ErrHdrInvalid("IPv4's IHL")
+		}
+		if len(ldata) < ihl {
+			return ErrHdrLength("IPv4 opts")
+		}
+		netLayer = ldata[:ihl]
+	} else if ldata[0]>>4 == 6 {
+		if len(ldata) < 40 {
+			return ErrHdrLength("IPv6")
+		}
+		proto = ldata[6]
+		totalLen := 40
+		for ipv6ExtensionHdr(proto) {
+			hdr := len(ldata) - totalLen
+			if hdr < 8 {
+				return ErrHdrExpected("IPv6 opts")
+			}
+			extLen := 8
+			if proto != 44 {
+				extLen = int(ldata[totalLen+1]+1) * 8
+			}
+			if hdr < extLen {
+				return ErrHdrLength("IPv6 opts")
+			}
+			proto = ldata[totalLen]
+			totalLen += extLen
+		}
+		netLayer = ldata[:totalLen]
+	} else {
+		return ErrHdrExpected("IPv4 or IPv6")
+	}
+	if proto != 6 {
+		return ErrHdrExpected("TCP")
+	}
+	if len(data) <= len(netLayer) {
+		return ErrHdrMissing("TCP")
+	}
+	ndata := ldata[len(netLayer):]
+	// TCP header
+	if len(ndata) < 20 {
+		return ErrHdrLength("TCP")
+	}
+	dOf := int(ndata[12]>>4) * 4
+	if dOf < 20 {
+		return ErrHdrInvalid("TCP's ndata offset")
+	}
+	if len(ndata) < dOf {
+		return ErrHdrLength("TCP opts")
+	}
+
+	// There are case when packet have padding but dOf shows its not
+	empty := true
+	for i := 0; i < len(ndata[dOf:]); i++ {
+		if ndata[dOf:][i] != 0 {
+			empty = false
+			break
+		}
+	}
+
+	if !allowEmpty && empty {
+		return EmptyPacket("")
+	}
+
+	if (netLayer[0] >> 4) == 4 {
 		// IPv4 header
 		pckt.Version = 4
-		pckt.SrcIP = packet.NetLayer[12:16]
-		pckt.DstIP = packet.NetLayer[16:20]
+		pckt.SrcIP = netLayer[12:16]
+		pckt.DstIP = netLayer[16:20]
 	} else {
 		// IPv6 header
 		pckt.Version = 6
-		pckt.SrcIP = packet.NetLayer[8:24]
-		pckt.DstIP = packet.NetLayer[24:40]
+		pckt.SrcIP = netLayer[8:24]
+		pckt.DstIP = netLayer[24:40]
 	}
-	pckt.SrcPort = binary.BigEndian.Uint16(packet.TransLayer[0:2])
-	pckt.DstPort = binary.BigEndian.Uint16(packet.TransLayer[2:4])
-	pckt.Seq = binary.BigEndian.Uint32(packet.TransLayer[4:8])
-	pckt.Ack = binary.BigEndian.Uint32(packet.TransLayer[8:12])
-	pckt.FIN = packet.TransLayer[13]&0x01 != 0
-	pckt.SYN = packet.TransLayer[13]&0x02 != 0
-	pckt.RST = packet.TransLayer[13]&0x04 != 0
-	pckt.ACK = packet.TransLayer[13]&0x10 != 0
-	pckt.Lost = uint32(packet.Info.Length - packet.Info.CaptureLength)
-	pckt.Payload = packet.Payload
-	return
+
+	transLayer = ndata[:dOf]
+
+	pckt.CaptureLength = cp.CaptureLength
+	pckt.SrcPort = binary.BigEndian.Uint16(transLayer[0:2])
+	pckt.DstPort = binary.BigEndian.Uint16(transLayer[2:4])
+	pckt.Seq = binary.BigEndian.Uint32(transLayer[4:8])
+	pckt.Ack = binary.BigEndian.Uint32(transLayer[8:12])
+	pckt.FIN = transLayer[13]&0x01 != 0
+	pckt.SYN = transLayer[13]&0x02 != 0
+	pckt.RST = transLayer[13]&0x04 != 0
+	pckt.ACK = transLayer[13]&0x10 != 0
+	pckt.Lost = uint32(cp.Length - cp.CaptureLength)
+
+	pckt.Payload = ndata[dOf:]
+
+	return nil
+}
+
+func (pckt *Packet) MessageID() uint64 {
+	if pckt.messageID == 0 {
+		// All packets in the same message will share the same ID
+		pckt.messageID = uint64(pckt.SrcPort)<<48 | uint64(pckt.DstPort)<<32 |
+			(uint64(ip2int(pckt.SrcIP)) + uint64(ip2int(pckt.DstIP)) + uint64(pckt.Ack))
+	}
+
+	return pckt.messageID
 }
 
 // Src returns the source socket of a packet
@@ -74,3 +231,54 @@ func (pckt *Packet) Src() string {
 func (pckt *Packet) Dst() string {
 	return fmt.Sprintf("%s:%d", pckt.DstIP, pckt.DstPort)
 }
+
+type EmptyPacket string
+
+func (err EmptyPacket) Error() string {
+	return "Empty packet"
+}
+
+// ErrHdrLength returned on short header length
+type ErrHdrLength string
+
+func (err ErrHdrLength) Error() string {
+	return "short " + string(err) + " length"
+}
+
+// ErrHdrMissing returned on missing header(s)
+type ErrHdrMissing string
+
+func (err ErrHdrMissing) Error() string {
+	return "missing " + string(err) + " header(s)"
+}
+
+// ErrHdrExpected returned when header(s) are different from the one expected
+type ErrHdrExpected string
+
+func (err ErrHdrExpected) Error() string {
+	return "expected " + string(err) + " header(s)"
+}
+
+// ErrHdrInvalid returned when header(s) are different from the one expected
+type ErrHdrInvalid string
+
+func (err ErrHdrInvalid) Error() string {
+	return "invalid " + string(err) + " value"
+}
+
+// https://en.wikipedia.org/wiki/IPv6_packet#Extension_headers
+func ipv6ExtensionHdr(b byte) bool {
+	// TODO: support all extension headers
+	return b == 0 || b == 43 || b == 44
+}
+
+func ip2int(ip net.IP) uint32 {
+	if len(ip) == 0 {
+		return 0
+	}
+
+	if len(ip) == 16 {
+		return binary.BigEndian.Uint32(ip[12:16])
+	}
+	return binary.BigEndian.Uint32(ip)
+}

tcp/tcp_test.go

@@ -3,17 +3,20 @@ package tcp
 import (
 	"bytes"
 	"encoding/binary"
+
+	// "runtime"
 	"testing"
 	"time"
 
-	"github.com/buger/goreplay/capture"
+	"github.com/stretchr/testify/assert"
+
 	"github.com/buger/goreplay/proto"
 	"github.com/google/gopacket"
 	"github.com/google/gopacket/layers"
 )
 
-func generateHeader(seq uint32, length uint16) []byte {
-	hdr := make([]byte, 4+24+24, 4+24+24)
+func generateHeader(request bool, seq uint32, length uint16) []byte {
+	hdr := make([]byte, 4+24+24)
 	binary.BigEndian.PutUint32(hdr, uint32(layers.ProtocolFamilyIPv4))
 
 	ip := hdr[4:]
@@ -26,195 +29,219 @@ func generateHeader(seq uint32, length uint16) []byte {
 	// set tcp header
 	tcp := ip[24:]
 	tcp[12] = 6 << 4
-	binary.BigEndian.PutUint16(tcp, 5535)
-	binary.BigEndian.PutUint16(tcp[2:], 8000)
+
+	if request {
+		binary.BigEndian.PutUint16(tcp, 5535)
+		binary.BigEndian.PutUint16(tcp[2:], 8000)
+	} else {
+		binary.BigEndian.PutUint16(tcp, 8000)
+		binary.BigEndian.PutUint16(tcp[2:], 5535)
+	}
 	binary.BigEndian.PutUint32(tcp[4:], seq)
 	return hdr
 }
 
-func GetPackets(start uint32, _len int, payload []byte) []*capture.Packet {
-	var packets = make([]*capture.Packet, _len)
+func GetPackets(request bool, start uint32, _len int, payload []byte) []*Packet {
+	var packets = make([]*Packet, _len)
+	var err error
 	for i := start; i < start+uint32(_len); i++ {
-		d := append(generateHeader(i, uint16(len(payload))), payload...)
+		d := append(generateHeader(request, i, uint16(len(payload))), payload...)
 		ci := &gopacket.CaptureInfo{Length: len(d), CaptureLength: len(d), Timestamp: time.Now()}
-		packets[i-start] = capture.NewPacket(d, int(layers.LinkTypeLoop), 4, ci)
+
+		packets[i-start], err = ParsePacket(d, int(layers.LinkTypeLoop), 4, ci, true)
+		if request {
+			packets[i-start].Direction = DirIncoming
+		} else {
+			packets[i-start].Direction = DirOutcoming
+		}
+		if err != nil {
+			panic(err)
+		}
 	}
 	return packets
 }
 
-func TestMessageParserWithHint(t *testing.T) {
-	var mssg = make(chan *Message, 3)
-	pool := NewMessagePool(1<<20, time.Second, nil, func(m *Message) { mssg <- m })
-	pool.Start = func(pckt *Packet) (bool, bool) {
+func TestRequestResponseMapping(t *testing.T) {
+	packets := []*Packet{
+		{SrcPort: 60000, DstPort: 80, Ack: 1, Seq: 1, Direction: DirIncoming, Timestamp: time.Unix(1, 0), Payload: []byte("GET / HTTP/1.1\r\n")},
+		{SrcPort: 60000, DstPort: 80, Ack: 1, Seq: 17, Direction: DirIncoming, Timestamp: time.Unix(2, 0), Payload: []byte("Host: localhost\r\n\r\n")},
+
+		// Seq of first response packet match Ack of first request packet
+		{SrcPort: 80, DstPort: 60000, Ack: 36, Seq: 1, Direction: DirOutcoming, Timestamp: time.Unix(3, 0), Payload: []byte("HTTP/1.1 200 OK\r\n")},
+		{SrcPort: 80, DstPort: 60000, Ack: 36, Seq: 18, Direction: DirOutcoming, Timestamp: time.Unix(4, 0), Payload: []byte("Content-Length: 0\r\n\r\n")},
+
+		// Same TCP stream
+		{SrcPort: 60000, DstPort: 80, Ack: 39, Seq: 36, Direction: DirIncoming, Timestamp: time.Unix(5, 0), Payload: []byte("GET / HTTP/1.1\r\n")},
+		{SrcPort: 60000, DstPort: 80, Ack: 39, Seq: 52, Direction: DirIncoming, Timestamp: time.Unix(6, 0), Payload: []byte("Host: localhost\r\n\r\n")},
+
+		// Seq of first response packet match Ack of first request packet
+		{SrcPort: 80, DstPort: 60000, Ack: 71, Seq: 39, Direction: DirOutcoming, Timestamp: time.Unix(7, 0), Payload: []byte("HTTP/1.1 200 OK\r\n")},
+		{SrcPort: 80, DstPort: 60000, Ack: 71, Seq: 56, Direction: DirOutcoming, Timestamp: time.Unix(8, 0), Payload: []byte("Content-Length: 0\r\n\r\n")},
+	}
+
+	parser := NewMessageParser(nil, nil, nil, time.Second, false)
+	parser.Start = func(pckt *Packet) (bool, bool) {
 		return proto.HasRequestTitle(pckt.Payload), proto.HasResponseTitle(pckt.Payload)
 	}
-	pool.End = func(m *Message) bool {
-		return proto.HasFullPayload(m.Data(), m)
+	parser.End = func(m *Message) bool {
+		return proto.HasFullPayload(m, m.PacketData()...)
 	}
-	packets := GetPackets(1, 30, nil)
-	packets[0].TransLayer[13] = 2  // SYN flag
-	packets[10].TransLayer[13] = 2 // SYN flag
-	packets[29].TransLayer[13] = 1 // FIN flag
-	packets[4] = GetPackets(5, 1, []byte("HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nTransfer-Encoding: chunked\r\n\r\n7"))[0]
-	packets[5] = GetPackets(6, 1, []byte("\r\nMozilla\r\n9\r\nDeveloper\r"))[0]
-	packets[6] = GetPackets(7, 1, []byte("\n7\r\nNetwork\r\n0\r\n\r\n"))[0]
-	packets[14] = GetPackets(5, 1, []byte("POST / HTTP/1.1\r\nContent-Type: text/plain\r\nContent-Length: 23\r\n\r\n"))[0]
-	packets[15] = GetPackets(6, 1, []byte("MozillaDeveloper"))[0]
-	packets[16] = GetPackets(7, 1, []byte("Network"))[0]
-	packets[24] = GetPackets(5, 1, []byte("HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nContent-Length: 0\r\n\r"))[0]
-	for i := 0; i < 30; i++ {
-		pool.Handler(packets[i])
+
+	for _, packet := range packets {
+		parser.processPacket(packet)
 	}
-	var m *Message
-	select {
-	case <-time.After(time.Second):
-		t.Errorf("can't parse packets fast enough")
-		return
-	case m = <-mssg:
+
+	messages := []*Message{}
+	for i := 0; i < 4; i++ {
+		m := parser.Read()
+		messages = append(messages, m)
 	}
+
+	assert.Equal(t, int(messages[0].Direction), int(DirIncoming))
+	assert.Equal(t, int(messages[1].Direction), int(DirOutcoming))
+	assert.Equal(t, int(messages[2].Direction), int(DirIncoming))
+	assert.Equal(t, int(messages[3].Direction), int(DirOutcoming))
+
+	assert.Equal(t, messages[0].UUID(), messages[1].UUID())
+	assert.Equal(t, messages[2].UUID(), messages[3].UUID())
+
+	assert.NotEqual(t, messages[0].UUID(), messages[2].UUID())
+}
+
+func TestMessageParserWithHint(t *testing.T) {
+	parser := NewMessageParser(nil, nil, nil, time.Second, false)
+	parser.Start = func(pckt *Packet) (bool, bool) {
+		return proto.HasRequestTitle(pckt.Payload), proto.HasResponseTitle(pckt.Payload)
+	}
+	parser.End = func(m *Message) bool {
+		return proto.HasFullPayload(m, m.PacketData()...)
+	}
+
+	packets := []*Packet{
+		// Seq of first response packet match Ack of first request packet
+		{SrcPort: 80, DstPort: 60000, Ack: 1, Seq: 1, Direction: DirOutcoming, Timestamp: time.Unix(1, 0), Payload: []byte("HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nTransfer-Encoding: chunked\r\n\r\n7\r\n")},
+		{SrcPort: 80, DstPort: 60000, Ack: 1, Seq: 18, Direction: DirOutcoming, Timestamp: time.Unix(2, 0), Payload: []byte("\r\nMozilla\r\n9\r\nDeveloper\r")},
+		{SrcPort: 80, DstPort: 60000, Ack: 1, Seq: 42, Direction: DirOutcoming, Timestamp: time.Unix(3, 0), Payload: []byte("\n7\r\nNetwork\r\n0\r\n\r\n")},
+
+		{SrcPort: 60000, DstPort: 80, Ack: 60, Seq: 1, Direction: DirIncoming, Timestamp: time.Unix(4, 0), Payload: []byte("POST / HTTP/1.1\r\nContent-Type: text/plain\r\nContent-Length: 23\r\n\r\n")},
+		{SrcPort: 60000, DstPort: 80, Ack: 60, Seq: 66, Direction: DirIncoming, Timestamp: time.Unix(5, 0), Payload: []byte("MozillaDeveloper")},
+		{SrcPort: 60000, DstPort: 80, Ack: 60, Seq: 82, Direction: DirIncoming, Timestamp: time.Unix(6, 0), Payload: []byte("Network")},
+
+		{SrcPort: 80, DstPort: 60000, Ack: 89, Seq: 1, Direction: DirOutcoming, Timestamp: time.Unix(7, 0), Payload: []byte("HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nContent-Length: 0\r\n\r\n")},
+	}
+
+	for _, p := range packets {
+		parser.processPacket(p)
+	}
+
+	messages := []*Message{}
+	for i := 0; i < 3; i++ {
+		m := parser.Read()
+		messages = append(messages, m)
+	}
+
+	if !bytes.HasSuffix(messages[0].Data(), []byte("\n7\r\nNetwork\r\n0\r\n\r\n")) {
+		t.Errorf("expected to %q to have suffix %q", messages[0].Data(), []byte("\n7\r\nNetwork\r\n0\r\n\r\n"))
+	}
+
+	if !bytes.HasSuffix(messages[1].Data(), []byte("Network")) {
+		t.Errorf("expected to %q to have suffix %q", messages[1].Data(), []byte("Network"))
+	}
+
+	if !bytes.HasSuffix(messages[2].Data(), []byte("Content-Length: 0\r\n\r\n")) {
+		t.Errorf("expected to %q to have suffix %q", messages[2].Data(), []byte("Content-Length: 0\r\n\r\n"))
+	}
+}
+
+func TestMessageParserWrongOrder(t *testing.T) {
+	parser := NewMessageParser(nil, nil, nil, time.Second, false)
+	parser.Start = func(pckt *Packet) (bool, bool) {
+		return proto.HasRequestTitle(pckt.Payload), proto.HasResponseTitle(pckt.Payload)
+	}
+	parser.End = func(m *Message) bool {
+		return proto.HasFullPayload(m, m.PacketData()...)
+	}
+	packets := []*Packet{
+		// Seq of first response packet match Ack of first request packet
+		{SrcPort: 60000, DstPort: 80, Ack: 60, Seq: 66, Direction: DirIncoming, Timestamp: time.Unix(5, 0), Payload: []byte("MozillaDeveloper")},
+		{SrcPort: 80, DstPort: 60000, Ack: 1, Seq: 1, Direction: DirOutcoming, Timestamp: time.Unix(1, 0), Payload: []byte("HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nTransfer-Encoding: chunked\r\n\r\n7\r\n")},
+		{SrcPort: 80, DstPort: 60000, Ack: 1, Seq: 42, Direction: DirOutcoming, Timestamp: time.Unix(3, 0), Payload: []byte("\n7\r\nNetwork\r\n0\r\n\r\n")},
+
+		{SrcPort: 60000, DstPort: 80, Ack: 60, Seq: 1, Direction: DirIncoming, Timestamp: time.Unix(4, 0), Payload: []byte("POST / HTTP/1.1\r\nContent-Type: text/plain\r\nContent-Length: 23\r\n\r\n")},
+		{SrcPort: 80, DstPort: 60000, Ack: 1, Seq: 18, Direction: DirOutcoming, Timestamp: time.Unix(2, 0), Payload: []byte("\r\nMozilla\r\n9\r\nDeveloper\r")},
+
+		{SrcPort: 80, DstPort: 60000, Ack: 89, Seq: 1, Direction: DirOutcoming, Timestamp: time.Unix(7, 0), Payload: []byte("HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nContent-Length: 0\r\n\r\n")},
+		{SrcPort: 60000, DstPort: 80, Ack: 60, Seq: 82, Direction: DirIncoming, Timestamp: time.Unix(6, 0), Payload: []byte("Network")},
+	}
+
+	for _, p := range packets {
+		parser.processPacket(p)
+	}
+
+	m := parser.Read()
+
 	if !bytes.HasSuffix(m.Data(), []byte("\n7\r\nNetwork\r\n0\r\n\r\n")) {
 		t.Errorf("expected to %q to have suffix %q", m.Data(), []byte("\n7\r\nNetwork\r\n0\r\n\r\n"))
 	}
 
-	select {
-	case <-time.After(time.Second):
-		t.Errorf("can't parse packets fast enough")
-		return
-	case m = <-mssg:
+	m = parser.Read()
+
+	if !bytes.HasSuffix(m.Data(), []byte("Content-Length: 0\r\n\r\n")) {
+		t.Errorf("expected to %q to have suffix %q", m.Data(), []byte("Content-Length: 0\r\n\r\n"))
 	}
+
+	m = parser.Read()
+
 	if !bytes.HasSuffix(m.Data(), []byte("Network")) {
 		t.Errorf("expected to %q to have suffix %q", m.Data(), []byte("Network"))
 	}
-
-	select {
-	case <-time.After(time.Second):
-		t.Errorf("can't parse packets fast enough")
-		return
-	case m = <-mssg:
-	}
-	if !bytes.HasSuffix(m.Data(), []byte("Content-Length: 0\r\n\r")) {
-		t.Errorf("expected to %q to have suffix %q", m.Data(), []byte("Content-Length: 0\r\n\r"))
-	}
-
 }
 
 func TestMessageParserWithoutHint(t *testing.T) {
-	var mssg = make(chan *Message, 1)
 	var data [63 << 10]byte
-	packets := GetPackets(1, 10, data[:])
-	packets[0].TransLayer[13] = 2 // SYN flag
-	packets[9].TransLayer[13] = 1 // FIN flag
-	p := NewMessagePool(63<<10*10, time.Second, nil, func(m *Message) { mssg <- m })
+	packets := GetPackets(true, 1, 10, data[:])
+
+	p := NewMessageParser(nil, nil, nil, time.Second, false)
 	for _, v := range packets {
-		p.Handler(v)
-	}
-	var m *Message
-	select {
-	case <-time.After(time.Second):
-		t.Errorf("can't parse packets fast enough")
-		return
-	case m = <-mssg:
+		p.processPacket(v)
 	}
+	m := p.Read()
+
 	if m.Length != 63<<10*10 {
 		t.Errorf("expected %d to equal %d", m.Length, 63<<10*10)
 	}
 }
 
-func TestMessageMaxSizeReached(t *testing.T) {
-	var mssg = make(chan *Message, 2)
-	var data [63 << 10]byte
-	packets := GetPackets(1, 2, data[:])
-	packets = append(packets, GetPackets(3, 1, make([]byte, 63<<10+10))...)
-	packets[0].TransLayer[13] = 2 // SYN flag
-	packets[2].TransLayer[13] = 2 // SYN flag
-	packets[2].NetLayer[15] = 3   // changing address
-	p := NewMessagePool(63<<10+10, time.Second, nil, func(m *Message) { mssg <- m })
-	for _, v := range packets {
-		p.Handler(v)
-	}
-	var m *Message
-	select {
-	case <-time.After(time.Second):
-		t.Errorf("can't parse packets fast enough")
-		return
-	case m = <-mssg:
-	}
-	if m.Length != 63<<10+10 {
-		t.Errorf("expected %d to equal %d", m.Length, 63<<10+10)
-	}
-	if !m.Truncated {
-		t.Error("expected message to be truncated")
-	}
-
-	select {
-	case <-time.After(time.Second):
-		t.Errorf("can't parse packets fast enough")
-		return
-	case m = <-mssg:
-	}
-	if m.Length != 63<<10+10 {
-		t.Errorf("expected %d to equal %d", m.Length, 63<<10+10)
-	}
-	if m.Truncated {
-		t.Error("expected message to not be truncated")
-	}
-}
-
 func TestMessageTimeoutReached(t *testing.T) {
-	var mssg = make(chan *Message, 2)
-	var data [63 << 10]byte
-	packets := GetPackets(1, 2, data[:])
-	packets[0].TransLayer[13] = 2 // SYN flag
-	p := NewMessagePool(1<<20, 0, nil, func(m *Message) { mssg <- m })
-	p.Handler(packets[0])
-	time.Sleep(time.Millisecond * 200)
-	p.Handler(packets[1])
-	m := <-mssg
-	if m.Length != 63<<10 {
-		t.Errorf("expected %d to equal %d", m.Length, 63<<10)
+	const size = 63 << 11
+	var data [size >> 1]byte
+	packets := GetPackets(true, 1, 2, data[:])
+	p := NewMessageParser(nil, nil, nil, 10*time.Millisecond, true)
+	p.processPacket(packets[0])
+
+	time.Sleep(time.Second * 2)
+
+	p.processPacket(packets[1])
+	m := p.Read()
+	if m.Length != size {
+		t.Errorf("expected %d to equal %d", m.Length, size)
 	}
 	if !m.TimedOut {
 		t.Error("expected message to be timeout")
 	}
 }
 
-func TestMessageUUID(t *testing.T) {
-	packets := GetPackets(1, 10, nil)
-	packets[0].TransLayer[13] = 2 // SYN flag
-	packets[4].TransLayer[13] = 1 // FIN flag
-	packets[5].TransLayer[13] = 2 // SYN flag
-	packets[9].TransLayer[13] = 1 // FIN flag
-	var uuid, uuid1 []byte
-	pool := NewMessagePool(0, 0, nil, func(msg *Message) {
-		if len(uuid) == 0 {
-			uuid = msg.UUID()
-			return
-		}
-		uuid1 = msg.UUID()
-	})
-	pool.MatchUUID(true)
-	for _, p := range packets {
-		pool.Handler(p)
-	}
-
-	if string(uuid) != string(uuid1) {
-		t.Errorf("expected %s, to equal %s", uuid, uuid1)
-	}
-}
-
 func BenchmarkMessageUUID(b *testing.B) {
-	packets := GetPackets(1, 5, nil)
-	packets[0].TransLayer[13] = 2 // SYN flag
-	packets[4].TransLayer[13] = 1 // FIN flag
+	packets := GetPackets(true, 1, 5, nil)
+
 	var uuid []byte
-	var msg *Message
-	pool := NewMessagePool(0, 0, nil, func(m *Message) {
-		msg = m
-	})
-	pool.MatchUUID(true)
+	parser := NewMessageParser(nil, nil, nil, 10*time.Millisecond, true)
 	for _, p := range packets {
-		pool.Handler(p)
+		parser.processPacket(p)
 	}
+
+	msg := parser.Read()
+
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
 		uuid = msg.UUID()
@@ -225,8 +252,8 @@ func BenchmarkMessageUUID(b *testing.B) {
 func BenchmarkPacketParseAndSort(b *testing.B) {
 	m := new(Message)
 	m.packets = make([]*Packet, 100)
-	for i, v := range GetPackets(1, 100, nil) {
-		m.packets[i], _ = ParsePacket(v)
+	for i, v := range GetPackets(true, 1, 100, nil) {
+		m.packets[i] = v
 	}
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
@@ -235,21 +262,16 @@ func BenchmarkPacketParseAndSort(b *testing.B) {
 }
 
 func BenchmarkMessageParserWithoutHint(b *testing.B) {
-	var mssg = make(chan *Message, 1)
 	var chunk = []byte("111111111111111111111111111111")
-	packets := GetPackets(1, 1000, chunk)
-	packets[0].TransLayer[13] = 2      // SYN flag
-	packets[1000-1].TransLayer[13] = 1 // FIN flag
-	p := NewMessagePool(1<<20, time.Second*2, nil, func(m *Message) {
-		mssg <- m
-	})
+	packets := GetPackets(true, 1, 1000, chunk)
+	p := NewMessageParser(nil, nil, nil, 2*time.Second, false)
 	b.ResetTimer()
 	b.ReportMetric(float64(1000), "packets/op")
 	for i := 0; i < b.N; i++ {
 		for _, v := range packets {
-			p.Handler(v)
+			p.processPacket(v)
 		}
-		<-mssg
+		p.Read()
 	}
 }
 
@@ -261,46 +283,33 @@ func BenchmarkMessageParserWithHint(b *testing.B) {
 		buf[i] = chunk
 	}
 	buf[1001] = []byte("0\r\n\r\n")
-	packets := make([]*capture.Packet, len(buf))
+	packets := make([]*Packet, len(buf))
 	for i := 0; i < len(buf); i++ {
-		packets[i] = GetPackets(uint32(i+10), 1, buf[i])[0]
+		packets[i] = GetPackets(false, 1, 1, buf[i])[0]
 	}
-	var mssg = make(chan *Message, 1)
-	pool := NewMessagePool(1<<30, time.Second*10, nil, func(m *Message) { mssg <- m })
-	pool.Start = func(pckt *Packet) (bool, bool) {
+
+	parser := NewMessageParser(nil, nil, nil, 2*time.Second, false)
+	parser.Start = func(pckt *Packet) (bool, bool) {
 		return false, proto.HasResponseTitle(pckt.Payload)
 	}
-	pool.End = func(m *Message) bool {
-		return proto.HasFullPayload(m.Data(), m)
+	parser.End = func(m *Message) bool {
+		return proto.HasFullPayload(m, m.PacketData()...)
 	}
 	b.ResetTimer()
 	b.ReportMetric(float64(len(packets)), "packets/op")
 	b.ReportMetric(float64(1000), "chunks/op")
 	for i := 0; i < b.N; i++ {
 		for j := range packets {
-			pool.Handler(packets[j])
+			parser.processPacket(packets[j])
 		}
-		<-mssg
+		parser.Read()
 	}
 }
 
 func BenchmarkNewAndParsePacket(b *testing.B) {
-	data := append(generateHeader(1024, 10), make([]byte, 10)...)
+	data := append(generateHeader(true, 1024, 10), make([]byte, 10)...)
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
-		ParsePacket(capture.NewPacket(data, int(layers.LinkTypeLoop), 4, &gopacket.CaptureInfo{}))
-	}
-}
-
-func BenchmarkNewPacket(b *testing.B) {
-	packet := capture.NewPacket(
-		append(generateHeader(1024, 10), make([]byte, 10)...),
-		int(layers.LinkTypeLoop),
-		4,
-		&gopacket.CaptureInfo{},
-	)
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		ParsePacket(packet)
+		ParsePacket(data, int(layers.LinkTypeLoop), 4, &gopacket.CaptureInfo{}, true)
 	}
 }

vendor/github.com/Shopify/sarama/.gitignore

@@ -1,27 +0,0 @@
-# Compiled Object files, Static and Dynamic libs (Shared Objects)
-*.o
-*.a
-*.so
-*.test
-
-# Folders
-_obj
-_test
-.vagrant
-
-# Architecture specific extensions/prefixes
-*.[568vq]
-[568vq].out
-
-*.cgo1.go
-*.cgo2.c
-_cgo_defun.c
-_cgo_gotypes.go
-_cgo_export.*
-
-_testmain.go
-
-*.exe
-
-coverage.txt
-profile.out

vendor/github.com/Shopify/sarama/.golangci.yml

@@ -1,74 +0,0 @@
-run:
-  timeout: 5m
-  deadline: 10m
-
-linters-settings:
-  govet:
-    check-shadowing: false
-  golint:
-    min-confidence: 0
-  gocyclo:
-    min-complexity: 99
-  maligned:
-    suggest-new: true
-  dupl:
-    threshold: 100
-  goconst:
-    min-len: 2
-    min-occurrences: 3
-  misspell:
-    locale: US
-  goimports:
-    local-prefixes: github.com/Shopify/sarama
-  gocritic:
-    enabled-tags:
-      - diagnostic
-      - experimental
-      - opinionated
-      - performance
-      - style
-    disabled-checks:
-      - wrapperFunc
-      - ifElseChain
-  funlen:
-    lines: 300
-    statements: 300
-
-linters:
-  disable-all: true
-  enable:
-    - bodyclose
-    - deadcode
-    - depguard
-    - dogsled
-    # - dupl
-    - errcheck
-    - funlen
-    # - gocritic
-    - gocyclo
-    - gofmt
-    - goimports
-    # - golint
-    - gosec
-    # - gosimple
-    - govet
-    # - ineffassign
-    - interfacer
-    # - misspell
-    # - nakedret
-    # - scopelint
-    # - staticcheck
-    - structcheck
-    # - stylecheck
-    - typecheck
-    - unconvert
-    - unused
-    - varcheck
-    - whitespace
-    # - goconst
-    # - gochecknoinits
-
-issues:
-  exclude:
-    - consider giving a name to these results
-    - include an explanation for nolint directive

vendor/github.com/Shopify/sarama/CHANGELOG.md

@@ -1,912 +0,0 @@
-# Changelog
-
-#### Unreleased
-
-#### Version 1.26.1 (2020-02-04)
-
-Improvements:
-- Add requests-in-flight metric ([1539](https://github.com/Shopify/sarama/pull/1539))
-- Fix misleading example for cluster admin ([1595](https://github.com/Shopify/sarama/pull/1595))
-- Replace Travis with GitHub Actions, linters housekeeping ([1573](https://github.com/Shopify/sarama/pull/1573))
-- Allow BalanceStrategy to provide custom assignment data ([1592](https://github.com/Shopify/sarama/pull/1592))
-
-Bug Fixes:
-- Adds back Consumer.Offsets.CommitInterval to fix API ([1590](https://github.com/Shopify/sarama/pull/1590))
-- Fix error message s/CommitInterval/AutoCommit.Interval ([1589](https://github.com/Shopify/sarama/pull/1589))
-
-#### Version 1.26.0 (2020-01-24)
-
-New Features:
-- Enable zstd compression
-  ([1574](https://github.com/Shopify/sarama/pull/1574),
-  [1582](https://github.com/Shopify/sarama/pull/1582))
-- Support headers in tools kafka-console-producer
-  ([1549](https://github.com/Shopify/sarama/pull/1549))
-
-Improvements:
-- Add SASL AuthIdentity to SASL frames (authzid)
-  ([1585](https://github.com/Shopify/sarama/pull/1585)).
-
-Bug Fixes:
-- Sending messages with ZStd compression enabled fails in multiple ways
-  ([1252](https://github.com/Shopify/sarama/issues/1252)).
-- Use the broker for any admin on BrokerConfig
-  ([1571](https://github.com/Shopify/sarama/pull/1571)).
-- Set DescribeConfigRequest Version field
-  ([1576](https://github.com/Shopify/sarama/pull/1576)).
-- ConsumerGroup flooding logs with client/metadata update req
-  ([1578](https://github.com/Shopify/sarama/pull/1578)).
-- MetadataRequest version in DescribeCluster
-  ([1580](https://github.com/Shopify/sarama/pull/1580)).
-- Fix deadlock in consumer group handleError
-  ([1581](https://github.com/Shopify/sarama/pull/1581))
-- Fill in the Fetch{Request,Response} protocol
-  ([1582](https://github.com/Shopify/sarama/pull/1582)).
-- Retry topic request on ControllerNotAvailable
-  ([1586](https://github.com/Shopify/sarama/pull/1586)).
-
-#### Version 1.25.0 (2020-01-13)
-
-New Features:
-- Support TLS protocol in kafka-producer-performance
-  ([1538](https://github.com/Shopify/sarama/pull/1538)).
-- Add support for kafka 2.4.0
-  ([1552](https://github.com/Shopify/sarama/pull/1552)).
-
-Improvements:
-- Allow the Consumer to disable auto-commit offsets
-  ([1164](https://github.com/Shopify/sarama/pull/1164)).
-- Produce records with consistent timestamps
-  ([1455](https://github.com/Shopify/sarama/pull/1455)).
-
-Bug Fixes:
-- Fix incorrect SetTopicMetadata name mentions
-  ([1534](https://github.com/Shopify/sarama/pull/1534)).
-- Fix client.tryRefreshMetadata Println
-  ([1535](https://github.com/Shopify/sarama/pull/1535)).
-- Fix panic on calling updateMetadata on closed client
-  ([1531](https://github.com/Shopify/sarama/pull/1531)).
-- Fix possible faulty metrics in TestFuncProducing
-  ([1545](https://github.com/Shopify/sarama/pull/1545)).
-
-#### Version 1.24.1 (2019-10-31)
-
-New Features:
-- Add DescribeLogDirs Request/Response pair
-  ([1520](https://github.com/Shopify/sarama/pull/1520)).
-
-Bug Fixes:
-- Fix ClusterAdmin returning invalid controller ID on DescribeCluster
-  ([1518](https://github.com/Shopify/sarama/pull/1518)).
-- Fix issue with consumergroup not rebalancing when new partition is added
-  ([1525](https://github.com/Shopify/sarama/pull/1525)).
-- Ensure consistent use of read/write deadlines
-  ([1529](https://github.com/Shopify/sarama/pull/1529)).
-
-#### Version 1.24.0 (2019-10-09)
-
-New Features:
-- Add sticky partition assignor
-  ([1416](https://github.com/Shopify/sarama/pull/1416)).
-- Switch from cgo zstd package to pure Go implementation
-  ([1477](https://github.com/Shopify/sarama/pull/1477)).
-
-Improvements:
-- Allow creating ClusterAdmin from client
-  ([1415](https://github.com/Shopify/sarama/pull/1415)).
-- Set KafkaVersion in ListAcls method
-  ([1452](https://github.com/Shopify/sarama/pull/1452)).
-- Set request version in CreateACL ClusterAdmin method
-  ([1458](https://github.com/Shopify/sarama/pull/1458)).
-- Set request version in DeleteACL ClusterAdmin method
-  ([1461](https://github.com/Shopify/sarama/pull/1461)).
-- Handle missed error codes on TopicMetaDataRequest and GroupCoordinatorRequest
-  ([1464](https://github.com/Shopify/sarama/pull/1464)).
-- Remove direct usage of gofork
-  ([1465](https://github.com/Shopify/sarama/pull/1465)).
-- Add support for Go 1.13
-  ([1478](https://github.com/Shopify/sarama/pull/1478)).
-- Improve behavior of NewMockListAclsResponse
-  ([1481](https://github.com/Shopify/sarama/pull/1481)).
-
-Bug Fixes:
-- Fix race condition in consumergroup example
-  ([1434](https://github.com/Shopify/sarama/pull/1434)).
-- Fix brokerProducer goroutine leak
-  ([1442](https://github.com/Shopify/sarama/pull/1442)).
-- Use released version of lz4 library
-  ([1469](https://github.com/Shopify/sarama/pull/1469)).
-- Set correct version in MockDeleteTopicsResponse
-  ([1484](https://github.com/Shopify/sarama/pull/1484)).
-- Fix CLI help message typo
-  ([1494](https://github.com/Shopify/sarama/pull/1494)).
-
-Known Issues:
-- Please **don't** use Zstd, as it doesn't work right now.
-  See https://github.com/Shopify/sarama/issues/1252
-
-#### Version 1.23.1 (2019-07-22)
-
-Bug Fixes:
-- Fix fetch delete bug record
-  ([1425](https://github.com/Shopify/sarama/pull/1425)).
-- Handle SASL/OAUTHBEARER token rejection
-  ([1428](https://github.com/Shopify/sarama/pull/1428)).
-
-#### Version 1.23.0 (2019-07-02)
-
-New Features:
-- Add support for Kafka 2.3.0
-  ([1418](https://github.com/Shopify/sarama/pull/1418)).
-- Add support for ListConsumerGroupOffsets v2
-  ([1374](https://github.com/Shopify/sarama/pull/1374)).
-- Add support for DeleteConsumerGroup
-  ([1417](https://github.com/Shopify/sarama/pull/1417)).
-- Add support for SASLVersion configuration
-  ([1410](https://github.com/Shopify/sarama/pull/1410)).
-- Add kerberos support
-  ([1366](https://github.com/Shopify/sarama/pull/1366)).
-
-Improvements:
-- Improve sasl_scram_client example
-  ([1406](https://github.com/Shopify/sarama/pull/1406)).
-- Fix shutdown and race-condition in consumer-group example
-  ([1404](https://github.com/Shopify/sarama/pull/1404)).
-- Add support for error codes 77—81
-  ([1397](https://github.com/Shopify/sarama/pull/1397)).
-- Pool internal objects allocated per message
-  ([1385](https://github.com/Shopify/sarama/pull/1385)).
-- Reduce packet decoder allocations
-  ([1373](https://github.com/Shopify/sarama/pull/1373)).
-- Support timeout when fetching metadata
-  ([1359](https://github.com/Shopify/sarama/pull/1359)).
-
-Bug Fixes:
-- Fix fetch size integer overflow
-  ([1376](https://github.com/Shopify/sarama/pull/1376)).
-- Handle and log throttled FetchResponses
-  ([1383](https://github.com/Shopify/sarama/pull/1383)).
-- Refactor misspelled word Resouce to Resource
-  ([1368](https://github.com/Shopify/sarama/pull/1368)).
-
-#### Version 1.22.1 (2019-04-29)
-
-Improvements:
-- Use zstd 1.3.8
-  ([1350](https://github.com/Shopify/sarama/pull/1350)).
-- Add support for SaslHandshakeRequest v1
-  ([1354](https://github.com/Shopify/sarama/pull/1354)).
-
-Bug Fixes:
-- Fix V5 MetadataRequest nullable topics array
-  ([1353](https://github.com/Shopify/sarama/pull/1353)).
-- Use a different SCRAM client for each broker connection
-  ([1349](https://github.com/Shopify/sarama/pull/1349)).
-- Fix AllowAutoTopicCreation for MetadataRequest greater than v3
-  ([1344](https://github.com/Shopify/sarama/pull/1344)).
-
-#### Version 1.22.0 (2019-04-09)
-
-New Features:
-- Add Offline Replicas Operation to Client
-  ([1318](https://github.com/Shopify/sarama/pull/1318)).
-- Allow using proxy when connecting to broker
-  ([1326](https://github.com/Shopify/sarama/pull/1326)).
-- Implement ReadCommitted
-  ([1307](https://github.com/Shopify/sarama/pull/1307)).
-- Add support for Kafka 2.2.0
-  ([1331](https://github.com/Shopify/sarama/pull/1331)).
-- Add SASL SCRAM-SHA-512 and SCRAM-SHA-256 mechanismes
-  ([1331](https://github.com/Shopify/sarama/pull/1295)).
-
-Improvements:
-- Unregister all broker metrics on broker stop
-  ([1232](https://github.com/Shopify/sarama/pull/1232)).
-- Add SCRAM authentication example
-  ([1303](https://github.com/Shopify/sarama/pull/1303)).
-- Add consumergroup examples
-  ([1304](https://github.com/Shopify/sarama/pull/1304)).
-- Expose consumer batch size metric
-  ([1296](https://github.com/Shopify/sarama/pull/1296)).
-- Add TLS options to console producer and consumer
-  ([1300](https://github.com/Shopify/sarama/pull/1300)).
-- Reduce client close bookkeeping
-  ([1297](https://github.com/Shopify/sarama/pull/1297)).
-- Satisfy error interface in create responses
-  ([1154](https://github.com/Shopify/sarama/pull/1154)).
-- Please lint gods
-  ([1346](https://github.com/Shopify/sarama/pull/1346)).
-
-Bug Fixes:
-- Fix multi consumer group instance crash
-  ([1338](https://github.com/Shopify/sarama/pull/1338)).
-- Update lz4 to latest version
-  ([1347](https://github.com/Shopify/sarama/pull/1347)).
-- Retry ErrNotCoordinatorForConsumer in new consumergroup session
-  ([1231](https://github.com/Shopify/sarama/pull/1231)).
-- Fix cleanup error handler
-  ([1332](https://github.com/Shopify/sarama/pull/1332)).
-- Fix rate condition in PartitionConsumer
-  ([1156](https://github.com/Shopify/sarama/pull/1156)).
-
-#### Version 1.21.0 (2019-02-24)
-
-New Features:
-- Add CreateAclRequest, DescribeAclRequest, DeleteAclRequest
-  ([1236](https://github.com/Shopify/sarama/pull/1236)).
-- Add DescribeTopic, DescribeConsumerGroup, ListConsumerGroups, ListConsumerGroupOffsets admin requests
-  ([1178](https://github.com/Shopify/sarama/pull/1178)).
-- Implement SASL/OAUTHBEARER
-  ([1240](https://github.com/Shopify/sarama/pull/1240)).
-
-Improvements:
-- Add Go mod support
-  ([1282](https://github.com/Shopify/sarama/pull/1282)).
-- Add error codes 73—76
-  ([1239](https://github.com/Shopify/sarama/pull/1239)).
-- Add retry backoff function
-  ([1160](https://github.com/Shopify/sarama/pull/1160)).
-- Maintain metadata in the producer even when retries are disabled
-  ([1189](https://github.com/Shopify/sarama/pull/1189)).
-- Include ReplicaAssignment in ListTopics
-  ([1274](https://github.com/Shopify/sarama/pull/1274)).
-- Add producer performance tool
-  ([1222](https://github.com/Shopify/sarama/pull/1222)).
-- Add support LogAppend timestamps
-  ([1258](https://github.com/Shopify/sarama/pull/1258)).
-
-Bug Fixes:
-- Fix potential deadlock when a heartbeat request fails
-  ([1286](https://github.com/Shopify/sarama/pull/1286)).
-- Fix consuming compacted topic
-  ([1227](https://github.com/Shopify/sarama/pull/1227)).
-- Set correct Kafka version for DescribeConfigsRequest v1
-  ([1277](https://github.com/Shopify/sarama/pull/1277)).
-- Update kafka test version
-  ([1273](https://github.com/Shopify/sarama/pull/1273)).
-
-#### Version 1.20.1 (2019-01-10)
-
-New Features:
-- Add optional replica id in offset request
-  ([1100](https://github.com/Shopify/sarama/pull/1100)).
-
-Improvements:
-- Implement DescribeConfigs Request + Response v1 & v2
-  ([1230](https://github.com/Shopify/sarama/pull/1230)).
-- Reuse compression objects
-  ([1185](https://github.com/Shopify/sarama/pull/1185)).
-- Switch from png to svg for GoDoc link in README
-  ([1243](https://github.com/Shopify/sarama/pull/1243)).
-- Fix typo in deprecation notice for FetchResponseBlock.Records
-  ([1242](https://github.com/Shopify/sarama/pull/1242)).
-- Fix typos in consumer metadata response file
-  ([1244](https://github.com/Shopify/sarama/pull/1244)).
-
-Bug Fixes:
-- Revert to individual msg retries for non-idempotent
-  ([1203](https://github.com/Shopify/sarama/pull/1203)).
-- Respect MaxMessageBytes limit for uncompressed messages
-  ([1141](https://github.com/Shopify/sarama/pull/1141)).
-
-#### Version 1.20.0 (2018-12-10)
-
-New Features:
- - Add support for zstd compression
-   ([#1170](https://github.com/Shopify/sarama/pull/1170)).
- - Add support for Idempotent Producer
-   ([#1152](https://github.com/Shopify/sarama/pull/1152)).
- - Add support support for Kafka 2.1.0
-   ([#1229](https://github.com/Shopify/sarama/pull/1229)).
- - Add support support for OffsetCommit request/response pairs versions v1 to v5
-   ([#1201](https://github.com/Shopify/sarama/pull/1201)).
- - Add support support for OffsetFetch request/response pair up to version v5
-   ([#1198](https://github.com/Shopify/sarama/pull/1198)).
-
-Improvements:
- - Export broker's Rack setting
-   ([#1173](https://github.com/Shopify/sarama/pull/1173)).
- - Always use latest patch version of Go on CI
-   ([#1202](https://github.com/Shopify/sarama/pull/1202)).
- - Add error codes 61 to 72
-   ([#1195](https://github.com/Shopify/sarama/pull/1195)).
-
-Bug Fixes:
- - Fix build without cgo
-   ([#1182](https://github.com/Shopify/sarama/pull/1182)).
- - Fix go vet suggestion in consumer group file
-   ([#1209](https://github.com/Shopify/sarama/pull/1209)).
- - Fix typos in code and comments
-   ([#1228](https://github.com/Shopify/sarama/pull/1228)).
-
-#### Version 1.19.0 (2018-09-27)
-
-New Features:
- - Implement a higher-level consumer group
-   ([#1099](https://github.com/Shopify/sarama/pull/1099)).
-
-Improvements:
- - Add support for Go 1.11
-   ([#1176](https://github.com/Shopify/sarama/pull/1176)).
-
-Bug Fixes:
- - Fix encoding of `MetadataResponse` with version 2 and higher
-   ([#1174](https://github.com/Shopify/sarama/pull/1174)).
- - Fix race condition in mock async producer
-   ([#1174](https://github.com/Shopify/sarama/pull/1174)).
-
-#### Version 1.18.0 (2018-09-07)
-
-New Features:
- - Make `Partitioner.RequiresConsistency` vary per-message
-   ([#1112](https://github.com/Shopify/sarama/pull/1112)).
- - Add customizable partitioner
-   ([#1118](https://github.com/Shopify/sarama/pull/1118)).
- - Add `ClusterAdmin` support for `CreateTopic`, `DeleteTopic`, `CreatePartitions`,
-   `DeleteRecords`, `DescribeConfig`, `AlterConfig`, `CreateACL`, `ListAcls`, `DeleteACL`
-   ([#1055](https://github.com/Shopify/sarama/pull/1055)).
-
-Improvements:
- - Add support for Kafka 2.0.0
-   ([#1149](https://github.com/Shopify/sarama/pull/1149)).
- - Allow setting `LocalAddr` when dialing an address to support multi-homed hosts
-   ([#1123](https://github.com/Shopify/sarama/pull/1123)).
- - Simpler offset management
-   ([#1127](https://github.com/Shopify/sarama/pull/1127)).
-
-Bug Fixes:
- - Fix mutation of `ProducerMessage.MetaData` when producing to Kafka
-   ([#1110](https://github.com/Shopify/sarama/pull/1110)).
- - Fix consumer block when response did not contain all the
-   expected topic/partition blocks
-   ([#1086](https://github.com/Shopify/sarama/pull/1086)).
- - Fix consumer block when response contains only constrol messages
-   ([#1115](https://github.com/Shopify/sarama/pull/1115)).
- - Add timeout config for ClusterAdmin requests
-   ([#1142](https://github.com/Shopify/sarama/pull/1142)).
- - Add version check when producing message with headers
-   ([#1117](https://github.com/Shopify/sarama/pull/1117)).
- - Fix `MetadataRequest` for empty list of topics
-   ([#1132](https://github.com/Shopify/sarama/pull/1132)).
- - Fix producer topic metadata on-demand fetch when topic error happens in metadata response
-   ([#1125](https://github.com/Shopify/sarama/pull/1125)).
-
-#### Version 1.17.0 (2018-05-30)
-
-New Features:
- - Add support for gzip compression levels
-   ([#1044](https://github.com/Shopify/sarama/pull/1044)).
- - Add support for Metadata request/response pairs versions v1 to v5
-   ([#1047](https://github.com/Shopify/sarama/pull/1047),
-    [#1069](https://github.com/Shopify/sarama/pull/1069)).
- - Add versioning to JoinGroup request/response pairs
-   ([#1098](https://github.com/Shopify/sarama/pull/1098))
- - Add support for CreatePartitions, DeleteGroups, DeleteRecords request/response pairs
-   ([#1065](https://github.com/Shopify/sarama/pull/1065),
-    [#1096](https://github.com/Shopify/sarama/pull/1096),
-    [#1027](https://github.com/Shopify/sarama/pull/1027)).
- - Add `Controller()` method to Client interface
-   ([#1063](https://github.com/Shopify/sarama/pull/1063)).
-
-Improvements:
- - ConsumerMetadataReq/Resp has been migrated to FindCoordinatorReq/Resp
-   ([#1010](https://github.com/Shopify/sarama/pull/1010)).
- - Expose missing protocol parts: `msgSet` and `recordBatch`
-   ([#1049](https://github.com/Shopify/sarama/pull/1049)).
- - Add support for v1 DeleteTopics Request
-   ([#1052](https://github.com/Shopify/sarama/pull/1052)).
- - Add support for Go 1.10
-   ([#1064](https://github.com/Shopify/sarama/pull/1064)).
- - Claim support for Kafka 1.1.0
-   ([#1073](https://github.com/Shopify/sarama/pull/1073)).
-
-Bug Fixes:
- - Fix FindCoordinatorResponse.encode to allow nil Coordinator
-   ([#1050](https://github.com/Shopify/sarama/pull/1050),
-    [#1051](https://github.com/Shopify/sarama/pull/1051)).
- - Clear all metadata when we have the latest topic info
-   ([#1033](https://github.com/Shopify/sarama/pull/1033)).
- - Make `PartitionConsumer.Close` idempotent
-   ([#1092](https://github.com/Shopify/sarama/pull/1092)).
-
-#### Version 1.16.0 (2018-02-12)
-
-New Features:
- - Add support for the Create/Delete Topics request/response pairs
-   ([#1007](https://github.com/Shopify/sarama/pull/1007),
-    [#1008](https://github.com/Shopify/sarama/pull/1008)).
- - Add support for the Describe/Create/Delete ACL request/response pairs
-   ([#1009](https://github.com/Shopify/sarama/pull/1009)).
- - Add support for the five transaction-related request/response pairs
-   ([#1016](https://github.com/Shopify/sarama/pull/1016)).
-
-Improvements:
- - Permit setting version on mock producer responses
-   ([#999](https://github.com/Shopify/sarama/pull/999)).
- - Add `NewMockBrokerListener` helper for testing TLS connections
-   ([#1019](https://github.com/Shopify/sarama/pull/1019)).
- - Changed the default value for `Consumer.Fetch.Default` from 32KiB to 1MiB
-   which results in much higher throughput in most cases
-   ([#1024](https://github.com/Shopify/sarama/pull/1024)).
- - Reuse the `time.Ticker` across fetch requests in the PartitionConsumer to
-   reduce CPU and memory usage when processing many partitions
-   ([#1028](https://github.com/Shopify/sarama/pull/1028)).
- - Assign relative offsets to messages in the producer to save the brokers a
-   recompression pass
-   ([#1002](https://github.com/Shopify/sarama/pull/1002),
-    [#1015](https://github.com/Shopify/sarama/pull/1015)).
-
-Bug Fixes:
- - Fix producing uncompressed batches with the new protocol format
-   ([#1032](https://github.com/Shopify/sarama/issues/1032)).
- - Fix consuming compacted topics with the new protocol format
-   ([#1005](https://github.com/Shopify/sarama/issues/1005)).
- - Fix consuming topics with a mix of protocol formats
-   ([#1021](https://github.com/Shopify/sarama/issues/1021)).
- - Fix consuming when the broker includes multiple batches in a single response
-   ([#1022](https://github.com/Shopify/sarama/issues/1022)).
- - Fix detection of `PartialTrailingMessage` when the partial message was
-   truncated before the magic value indicating its version
-   ([#1030](https://github.com/Shopify/sarama/pull/1030)).
- - Fix expectation-checking in the mock of `SyncProducer.SendMessages`
-   ([#1035](https://github.com/Shopify/sarama/pull/1035)).
-
-#### Version 1.15.0 (2017-12-08)
-
-New Features:
- - Claim official support for Kafka 1.0, though it did already work
-   ([#984](https://github.com/Shopify/sarama/pull/984)).
- - Helper methods for Kafka version numbers to/from strings
-   ([#989](https://github.com/Shopify/sarama/pull/989)).
- - Implement CreatePartitions request/response
-   ([#985](https://github.com/Shopify/sarama/pull/985)).
-
-Improvements:
- - Add error codes 45-60
-   ([#986](https://github.com/Shopify/sarama/issues/986)).
-
-Bug Fixes:
- - Fix slow consuming for certain Kafka 0.11/1.0 configurations
-   ([#982](https://github.com/Shopify/sarama/pull/982)).
- - Correctly determine when a FetchResponse contains the new message format
-   ([#990](https://github.com/Shopify/sarama/pull/990)).
- - Fix producing with multiple headers
-   ([#996](https://github.com/Shopify/sarama/pull/996)).
- - Fix handling of truncated record batches
-   ([#998](https://github.com/Shopify/sarama/pull/998)).
- - Fix leaking metrics when closing brokers
-   ([#991](https://github.com/Shopify/sarama/pull/991)).
-
-#### Version 1.14.0 (2017-11-13)
-
-New Features:
- - Add support for the new Kafka 0.11 record-batch format, including the wire
-   protocol and the necessary behavioural changes in the producer and consumer.
-   Transactions and idempotency are not yet supported, but producing and
-   consuming should work with all the existing bells and whistles (batching,
-   compression, etc) as well as the new custom headers. Thanks to Vlad Hanciuta
-   of Arista Networks for this work. Part of
-   ([#901](https://github.com/Shopify/sarama/issues/901)).
-
-Bug Fixes:
- - Fix encoding of ProduceResponse versions in test
-   ([#970](https://github.com/Shopify/sarama/pull/970)).
- - Return partial replicas list when we have it
-   ([#975](https://github.com/Shopify/sarama/pull/975)).
-
-#### Version 1.13.0 (2017-10-04)
-
-New Features:
- - Support for FetchRequest version 3
-   ([#905](https://github.com/Shopify/sarama/pull/905)).
- - Permit setting version on mock FetchResponses
-   ([#939](https://github.com/Shopify/sarama/pull/939)).
- - Add a configuration option to support storing only minimal metadata for
-   extremely large clusters
-   ([#937](https://github.com/Shopify/sarama/pull/937)).
- - Add `PartitionOffsetManager.ResetOffset` for backtracking tracked offsets
-   ([#932](https://github.com/Shopify/sarama/pull/932)).
-
-Improvements:
- - Provide the block-level timestamp when consuming compressed messages
-   ([#885](https://github.com/Shopify/sarama/issues/885)).
- - `Client.Replicas` and `Client.InSyncReplicas` now respect the order returned
-   by the broker, which can be meaningful
-   ([#930](https://github.com/Shopify/sarama/pull/930)).
- - Use a `Ticker` to reduce consumer timer overhead at the cost of higher
-   variance in the actual timeout
-   ([#933](https://github.com/Shopify/sarama/pull/933)).
-
-Bug Fixes:
- - Gracefully handle messages with negative timestamps
-   ([#907](https://github.com/Shopify/sarama/pull/907)).
- - Raise a proper error when encountering an unknown message version
-   ([#940](https://github.com/Shopify/sarama/pull/940)).
-
-#### Version 1.12.0 (2017-05-08)
-
-New Features:
- - Added support for the `ApiVersions` request and response pair, and Kafka
-   version 0.10.2 ([#867](https://github.com/Shopify/sarama/pull/867)). Note
-   that you still need to specify the Kafka version in the Sarama configuration
-   for the time being.
- - Added a `Brokers` method to the Client which returns the complete set of
-   active brokers ([#813](https://github.com/Shopify/sarama/pull/813)).
- - Added an `InSyncReplicas` method to the Client which returns the set of all
-   in-sync broker IDs for the given partition, now that the Kafka versions for
-   which this was misleading are no longer in our supported set
-   ([#872](https://github.com/Shopify/sarama/pull/872)).
- - Added a `NewCustomHashPartitioner` method which allows constructing a hash
-   partitioner with a custom hash method in case the default (FNV-1a) is not
-   suitable
-   ([#837](https://github.com/Shopify/sarama/pull/837),
-    [#841](https://github.com/Shopify/sarama/pull/841)).
-
-Improvements:
- - Recognize more Kafka error codes
-   ([#859](https://github.com/Shopify/sarama/pull/859)).
-
-Bug Fixes:
- - Fix an issue where decoding a malformed FetchRequest would not return the
-   correct error ([#818](https://github.com/Shopify/sarama/pull/818)).
- - Respect ordering of group protocols in JoinGroupRequests. This fix is
-   transparent if you're using the `AddGroupProtocol` or
-   `AddGroupProtocolMetadata` helpers; otherwise you will need to switch from
-   the `GroupProtocols` field (now deprecated) to use `OrderedGroupProtocols`
-   ([#812](https://github.com/Shopify/sarama/issues/812)).
- - Fix an alignment-related issue with atomics on 32-bit architectures
-   ([#859](https://github.com/Shopify/sarama/pull/859)).
-
-#### Version 1.11.0 (2016-12-20)
-
-_Important:_ As of Sarama 1.11 it is necessary to set the config value of
-`Producer.Return.Successes` to true in order to use the SyncProducer. Previous
-versions would silently override this value when instantiating a SyncProducer
-which led to unexpected values and data races.
-
-New Features:
- - Metrics! Thanks to Sébastien Launay for all his work on this feature
-   ([#701](https://github.com/Shopify/sarama/pull/701),
-    [#746](https://github.com/Shopify/sarama/pull/746),
-    [#766](https://github.com/Shopify/sarama/pull/766)).
- - Add support for LZ4 compression
-   ([#786](https://github.com/Shopify/sarama/pull/786)).
- - Add support for ListOffsetRequest v1 and Kafka 0.10.1
-   ([#775](https://github.com/Shopify/sarama/pull/775)).
- - Added a `HighWaterMarks` method to the Consumer which aggregates the
-   `HighWaterMarkOffset` values of its child topic/partitions
-   ([#769](https://github.com/Shopify/sarama/pull/769)).
-
-Bug Fixes:
- - Fixed producing when using timestamps, compression and Kafka 0.10
-   ([#759](https://github.com/Shopify/sarama/pull/759)).
- - Added missing decoder methods to DescribeGroups response
-   ([#756](https://github.com/Shopify/sarama/pull/756)).
- - Fix producer shutdown when `Return.Errors` is disabled
-   ([#787](https://github.com/Shopify/sarama/pull/787)).
- - Don't mutate configuration in SyncProducer
-   ([#790](https://github.com/Shopify/sarama/pull/790)).
- - Fix crash on SASL initialization failure
-   ([#795](https://github.com/Shopify/sarama/pull/795)).
-
-#### Version 1.10.1 (2016-08-30)
-
-Bug Fixes:
- - Fix the documentation for `HashPartitioner` which was incorrect
-   ([#717](https://github.com/Shopify/sarama/pull/717)).
- - Permit client creation even when it is limited by ACLs
-   ([#722](https://github.com/Shopify/sarama/pull/722)).
- - Several fixes to the consumer timer optimization code, regressions introduced
-   in v1.10.0. Go's timers are finicky
-   ([#730](https://github.com/Shopify/sarama/pull/730),
-    [#733](https://github.com/Shopify/sarama/pull/733),
-    [#734](https://github.com/Shopify/sarama/pull/734)).
- - Handle consuming compressed relative offsets with Kafka 0.10
-   ([#735](https://github.com/Shopify/sarama/pull/735)).
-
-#### Version 1.10.0 (2016-08-02)
-
-_Important:_ As of Sarama 1.10 it is necessary to tell Sarama the version of
-Kafka you are running against (via the `config.Version` value) in order to use
-features that may not be compatible with old Kafka versions. If you don't
-specify this value it will default to 0.8.2 (the minimum supported), and trying
-to use more recent features (like the offset manager) will fail with an error.
-
-_Also:_ The offset-manager's behaviour has been changed to match the upstream
-java consumer (see [#705](https://github.com/Shopify/sarama/pull/705) and
-[#713](https://github.com/Shopify/sarama/pull/713)). If you use the
-offset-manager, please ensure that you are committing one *greater* than the
-last consumed message offset or else you may end up consuming duplicate
-messages.
-
-New Features:
- - Support for Kafka 0.10
-   ([#672](https://github.com/Shopify/sarama/pull/672),
-    [#678](https://github.com/Shopify/sarama/pull/678),
-    [#681](https://github.com/Shopify/sarama/pull/681), and others).
- - Support for configuring the target Kafka version
-   ([#676](https://github.com/Shopify/sarama/pull/676)).
- - Batch producing support in the SyncProducer
-   ([#677](https://github.com/Shopify/sarama/pull/677)).
- - Extend producer mock to allow setting expectations on message contents
-   ([#667](https://github.com/Shopify/sarama/pull/667)).
-
-Improvements:
- - Support `nil` compressed messages for deleting in compacted topics
-   ([#634](https://github.com/Shopify/sarama/pull/634)).
- - Pre-allocate decoding errors, greatly reducing heap usage and GC time against
-   misbehaving brokers ([#690](https://github.com/Shopify/sarama/pull/690)).
- - Re-use consumer expiry timers, removing one allocation per consumed message
-   ([#707](https://github.com/Shopify/sarama/pull/707)).
-
-Bug Fixes:
- - Actually default the client ID to "sarama" like we say we do
-   ([#664](https://github.com/Shopify/sarama/pull/664)).
- - Fix a rare issue where `Client.Leader` could return the wrong error
-   ([#685](https://github.com/Shopify/sarama/pull/685)).
- - Fix a possible tight loop in the consumer
-   ([#693](https://github.com/Shopify/sarama/pull/693)).
- - Match upstream's offset-tracking behaviour
-   ([#705](https://github.com/Shopify/sarama/pull/705)).
- - Report UnknownTopicOrPartition errors from the offset manager
-   ([#706](https://github.com/Shopify/sarama/pull/706)).
- - Fix possible negative partition value from the HashPartitioner
-   ([#709](https://github.com/Shopify/sarama/pull/709)).
-
-#### Version 1.9.0 (2016-05-16)
-
-New Features:
- - Add support for custom offset manager retention durations
-   ([#602](https://github.com/Shopify/sarama/pull/602)).
- - Publish low-level mocks to enable testing of third-party producer/consumer
-   implementations ([#570](https://github.com/Shopify/sarama/pull/570)).
- - Declare support for Golang 1.6
-   ([#611](https://github.com/Shopify/sarama/pull/611)).
- - Support for SASL plain-text auth
-   ([#648](https://github.com/Shopify/sarama/pull/648)).
-
-Improvements:
- - Simplified broker locking scheme slightly
-   ([#604](https://github.com/Shopify/sarama/pull/604)).
- - Documentation cleanup
-   ([#605](https://github.com/Shopify/sarama/pull/605),
-    [#621](https://github.com/Shopify/sarama/pull/621),
-    [#654](https://github.com/Shopify/sarama/pull/654)).
-
-Bug Fixes:
- - Fix race condition shutting down the OffsetManager
-   ([#658](https://github.com/Shopify/sarama/pull/658)).
-
-#### Version 1.8.0 (2016-02-01)
-
-New Features:
- - Full support for Kafka 0.9:
-   - All protocol messages and fields
-   ([#586](https://github.com/Shopify/sarama/pull/586),
-   [#588](https://github.com/Shopify/sarama/pull/588),
-   [#590](https://github.com/Shopify/sarama/pull/590)).
-   - Verified that TLS support works
-   ([#581](https://github.com/Shopify/sarama/pull/581)).
-   - Fixed the OffsetManager compatibility
-   ([#585](https://github.com/Shopify/sarama/pull/585)).
-
-Improvements:
- - Optimize for fewer system calls when reading from the network
-   ([#584](https://github.com/Shopify/sarama/pull/584)).
- - Automatically retry `InvalidMessage` errors to match upstream behaviour
-   ([#589](https://github.com/Shopify/sarama/pull/589)).
-
-#### Version 1.7.0 (2015-12-11)
-
-New Features:
- - Preliminary support for Kafka 0.9
-   ([#572](https://github.com/Shopify/sarama/pull/572)). This comes with several
-   caveats:
-   - Protocol-layer support is mostly in place
-     ([#577](https://github.com/Shopify/sarama/pull/577)), however Kafka 0.9
-     renamed some messages and fields, which we did not in order to preserve API
-     compatibility.
-   - The producer and consumer work against 0.9, but the offset manager does
-     not ([#573](https://github.com/Shopify/sarama/pull/573)).
-   - TLS support may or may not work
-     ([#581](https://github.com/Shopify/sarama/pull/581)).
-
-Improvements:
- - Don't wait for request timeouts on dead brokers, greatly speeding recovery
-   when the TCP connection is left hanging
-   ([#548](https://github.com/Shopify/sarama/pull/548)).
- - Refactored part of the producer. The new version provides a much more elegant
-   solution to [#449](https://github.com/Shopify/sarama/pull/449). It is also
-   slightly more efficient, and much more precise in calculating batch sizes
-   when compression is used
-   ([#549](https://github.com/Shopify/sarama/pull/549),
-   [#550](https://github.com/Shopify/sarama/pull/550),
-   [#551](https://github.com/Shopify/sarama/pull/551)).
-
-Bug Fixes:
- - Fix race condition in consumer test mock
-   ([#553](https://github.com/Shopify/sarama/pull/553)).
-
-#### Version 1.6.1 (2015-09-25)
-
-Bug Fixes:
- - Fix panic that could occur if a user-supplied message value failed to encode
-   ([#449](https://github.com/Shopify/sarama/pull/449)).
-
-#### Version 1.6.0 (2015-09-04)
-
-New Features:
- - Implementation of a consumer offset manager using the APIs introduced in
-   Kafka 0.8.2. The API is designed mainly for integration into a future
-   high-level consumer, not for direct use, although it is *possible* to use it
-   directly.
-   ([#461](https://github.com/Shopify/sarama/pull/461)).
-
-Improvements:
- - CRC32 calculation is much faster on machines with SSE4.2 instructions,
-   removing a major hotspot from most profiles
-   ([#255](https://github.com/Shopify/sarama/pull/255)).
-
-Bug Fixes:
- - Make protocol decoding more robust against some malformed packets generated
-   by go-fuzz ([#523](https://github.com/Shopify/sarama/pull/523),
-   [#525](https://github.com/Shopify/sarama/pull/525)) or found in other ways
-   ([#528](https://github.com/Shopify/sarama/pull/528)).
- - Fix a potential race condition panic in the consumer on shutdown
-   ([#529](https://github.com/Shopify/sarama/pull/529)).
-
-#### Version 1.5.0 (2015-08-17)
-
-New Features:
- - TLS-encrypted network connections are now supported. This feature is subject
-   to change when Kafka releases built-in TLS support, but for now this is
-   enough to work with TLS-terminating proxies
-   ([#154](https://github.com/Shopify/sarama/pull/154)).
-
-Improvements:
- - The consumer will not block if a single partition is not drained by the user;
-   all other partitions will continue to consume normally
-   ([#485](https://github.com/Shopify/sarama/pull/485)).
- - Formatting of error strings has been much improved
-   ([#495](https://github.com/Shopify/sarama/pull/495)).
- - Internal refactoring of the producer for code cleanliness and to enable
-   future work ([#300](https://github.com/Shopify/sarama/pull/300)).
-
-Bug Fixes:
- - Fix a potential deadlock in the consumer on shutdown
-   ([#475](https://github.com/Shopify/sarama/pull/475)).
-
-#### Version 1.4.3 (2015-07-21)
-
-Bug Fixes:
- - Don't include the partitioner in the producer's "fetch partitions"
-   circuit-breaker ([#466](https://github.com/Shopify/sarama/pull/466)).
- - Don't retry messages until the broker is closed when abandoning a broker in
-   the producer ([#468](https://github.com/Shopify/sarama/pull/468)).
- - Update the import path for snappy-go, it has moved again and the API has
-   changed slightly ([#486](https://github.com/Shopify/sarama/pull/486)).
-
-#### Version 1.4.2 (2015-05-27)
-
-Bug Fixes:
- - Update the import path for snappy-go, it has moved from google code to github
-   ([#456](https://github.com/Shopify/sarama/pull/456)).
-
-#### Version 1.4.1 (2015-05-25)
-
-Improvements:
- - Optimizations when decoding snappy messages, thanks to John Potocny
-   ([#446](https://github.com/Shopify/sarama/pull/446)).
-
-Bug Fixes:
- - Fix hypothetical race conditions on producer shutdown
-   ([#450](https://github.com/Shopify/sarama/pull/450),
-   [#451](https://github.com/Shopify/sarama/pull/451)).
-
-#### Version 1.4.0 (2015-05-01)
-
-New Features:
- - The consumer now implements `Topics()` and `Partitions()` methods to enable
-   users to dynamically choose what topics/partitions to consume without
-   instantiating a full client
-   ([#431](https://github.com/Shopify/sarama/pull/431)).
- - The partition-consumer now exposes the high water mark offset value returned
-   by the broker via the `HighWaterMarkOffset()` method ([#339](https://github.com/Shopify/sarama/pull/339)).
- - Added a `kafka-console-consumer` tool capable of handling multiple
-   partitions, and deprecated the now-obsolete `kafka-console-partitionConsumer`
-   ([#439](https://github.com/Shopify/sarama/pull/439),
-   [#442](https://github.com/Shopify/sarama/pull/442)).
-
-Improvements:
- - The producer's logging during retry scenarios is more consistent, more
-   useful, and slightly less verbose
-   ([#429](https://github.com/Shopify/sarama/pull/429)).
- - The client now shuffles its initial list of seed brokers in order to prevent
-   thundering herd on the first broker in the list
-   ([#441](https://github.com/Shopify/sarama/pull/441)).
-
-Bug Fixes:
- - The producer now correctly manages its state if retries occur when it is
-   shutting down, fixing several instances of confusing behaviour and at least
-   one potential deadlock ([#419](https://github.com/Shopify/sarama/pull/419)).
- - The consumer now handles messages for different partitions asynchronously,
-   making it much more resilient to specific user code ordering
-   ([#325](https://github.com/Shopify/sarama/pull/325)).
-
-#### Version 1.3.0 (2015-04-16)
-
-New Features:
- - The client now tracks consumer group coordinators using
-   ConsumerMetadataRequests similar to how it tracks partition leadership using
-   regular MetadataRequests ([#411](https://github.com/Shopify/sarama/pull/411)).
-   This adds two methods to the client API:
-   - `Coordinator(consumerGroup string) (*Broker, error)`
-   - `RefreshCoordinator(consumerGroup string) error`
-
-Improvements:
- - ConsumerMetadataResponses now automatically create a Broker object out of the
-   ID/address/port combination for the Coordinator; accessing the fields
-   individually has been deprecated
-   ([#413](https://github.com/Shopify/sarama/pull/413)).
- - Much improved handling of `OffsetOutOfRange` errors in the consumer.
-   Consumers will fail to start if the provided offset is out of range
-   ([#418](https://github.com/Shopify/sarama/pull/418))
-   and they will automatically shut down if the offset falls out of range
-   ([#424](https://github.com/Shopify/sarama/pull/424)).
- - Small performance improvement in encoding and decoding protocol messages
-   ([#427](https://github.com/Shopify/sarama/pull/427)).
-
-Bug Fixes:
- - Fix a rare race condition in the client's background metadata refresher if
-   it happens to be activated while the client is being closed
-   ([#422](https://github.com/Shopify/sarama/pull/422)).
-
-#### Version 1.2.0 (2015-04-07)
-
-Improvements:
- - The producer's behaviour when `Flush.Frequency` is set is now more intuitive
-   ([#389](https://github.com/Shopify/sarama/pull/389)).
- - The producer is now somewhat more memory-efficient during and after retrying
-   messages due to an improved queue implementation
-   ([#396](https://github.com/Shopify/sarama/pull/396)).
- - The consumer produces much more useful logging output when leadership
-   changes ([#385](https://github.com/Shopify/sarama/pull/385)).
- - The client's `GetOffset` method will now automatically refresh metadata and
-   retry once in the event of stale information or similar
-   ([#394](https://github.com/Shopify/sarama/pull/394)).
- - Broker connections now have support for using TCP keepalives
-   ([#407](https://github.com/Shopify/sarama/issues/407)).
-
-Bug Fixes:
- - The OffsetCommitRequest message now correctly implements all three possible
-   API versions ([#390](https://github.com/Shopify/sarama/pull/390),
-   [#400](https://github.com/Shopify/sarama/pull/400)).
-
-#### Version 1.1.0 (2015-03-20)
-
-Improvements:
- - Wrap the producer's partitioner call in a circuit-breaker so that repeatedly
-   broken topics don't choke throughput
-   ([#373](https://github.com/Shopify/sarama/pull/373)).
-
-Bug Fixes:
- - Fix the producer's internal reference counting in certain unusual scenarios
-   ([#367](https://github.com/Shopify/sarama/pull/367)).
- - Fix the consumer's internal reference counting in certain unusual scenarios
-   ([#369](https://github.com/Shopify/sarama/pull/369)).
- - Fix a condition where the producer's internal control messages could have
-   gotten stuck ([#368](https://github.com/Shopify/sarama/pull/368)).
- - Fix an issue where invalid partition lists would be cached when asking for
-   metadata for a non-existant topic ([#372](https://github.com/Shopify/sarama/pull/372)).
-
-
-#### Version 1.0.0 (2015-03-17)
-
-Version 1.0.0 is the first tagged version, and is almost a complete rewrite. The primary differences with previous untagged versions are:
-
-- The producer has been rewritten; there is now a `SyncProducer` with a blocking API, and an `AsyncProducer` that is non-blocking.
-- The consumer has been rewritten to only open one connection per broker instead of one connection per partition.
-- The main types of Sarama are now interfaces to make depedency injection easy; mock implementations for `Consumer`, `SyncProducer` and `AsyncProducer` are provided in the `github.com/Shopify/sarama/mocks` package.
-- For most uses cases, it is no longer necessary to open a `Client`; this will be done for you.
-- All the configuration values have been unified in the `Config` struct.
-- Much improved test suite.

vendor/github.com/Shopify/sarama/LICENSE

@@ -1,20 +0,0 @@
-Copyright (c) 2013 Shopify
-
-Permission is hereby granted, free of charge, to any person obtaining
-a copy of this software and associated documentation files (the
-"Software"), to deal in the Software without restriction, including
-without limitation the rights to use, copy, modify, merge, publish,
-distribute, sublicense, and/or sell copies of the Software, and to
-permit persons to whom the Software is furnished to do so, subject to
-the following conditions:
-
-The above copyright notice and this permission notice shall be
-included in all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
-NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
-LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
-OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
-WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

vendor/github.com/Shopify/sarama/Makefile

@@ -1,27 +0,0 @@
-default: fmt get update test lint
-
-GO       := GO111MODULE=on GOPRIVATE=github.com/linkedin GOSUMDB=off go
-GOBUILD  := CGO_ENABLED=0 $(GO) build $(BUILD_FLAG)
-GOTEST   := $(GO) test -gcflags='-l' -p 3 -v -race -timeout 6m -coverprofile=profile.out -covermode=atomic
-
-FILES    := $(shell find . -name '*.go' -type f -not -name '*.pb.go' -not -name '*_generated.go' -not -name '*_test.go')
-TESTS    := $(shell find . -name '*.go' -type f -not -name '*.pb.go' -not -name '*_generated.go' -name '*_test.go')
-
-get:
-	$(GO) get ./...
-	$(GO) mod verify
-	$(GO) mod tidy
-
-update:
-	$(GO) get -u -v all
-	$(GO) mod verify
-	$(GO) mod tidy
-
-fmt:
-	gofmt -s -l -w $(FILES) $(TESTS)
-
-lint:
-	golangci-lint run
-
-test:
-	$(GOTEST) ./...

vendor/github.com/Shopify/sarama/README.md

@@ -1,36 +0,0 @@
-# sarama
-
-[![GoDoc](https://godoc.org/github.com/Shopify/sarama?status.svg)](https://godoc.org/github.com/Shopify/sarama)
-[![Build Status](https://travis-ci.org/Shopify/sarama.svg?branch=master)](https://travis-ci.org/Shopify/sarama)
-[![Coverage](https://codecov.io/gh/Shopify/sarama/branch/master/graph/badge.svg)](https://codecov.io/gh/Shopify/sarama)
-
-Sarama is an MIT-licensed Go client library for [Apache Kafka](https://kafka.apache.org/) version 0.8 (and later).
-
-## Getting started
-
-- API documentation and examples are available via [godoc](https://godoc.org/github.com/Shopify/sarama).
-- Mocks for testing are available in the [mocks](./mocks) subpackage.
-- The [examples](./examples) directory contains more elaborate example applications.
-- The [tools](./tools) directory contains command line tools that can be useful for testing, diagnostics, and instrumentation.
-
-You might also want to look at the [Frequently Asked Questions](https://github.com/Shopify/sarama/wiki/Frequently-Asked-Questions).
-
-## Compatibility and API stability
-
-Sarama provides a "2 releases + 2 months" compatibility guarantee: we support
-the two latest stable releases of Kafka and Go, and we provide a two month
-grace period for older releases. This means we currently officially support
-Go 1.12 through 1.14, and Kafka 2.1 through 2.4, although older releases are
-still likely to work.
-
-Sarama follows semantic versioning and provides API stability via the gopkg.in service.
-You can import a version with a guaranteed stable API via http://gopkg.in/Shopify/sarama.v1.
-A changelog is available [here](CHANGELOG.md).
-
-## Contributing
-
-- Get started by checking our [contribution guidelines](https://github.com/Shopify/sarama/blob/master/.github/CONTRIBUTING.md).
-- Read the [Sarama wiki](https://github.com/Shopify/sarama/wiki) for more technical and design details.
-- The [Kafka Protocol Specification](https://cwiki.apache.org/confluence/display/KAFKA/A+Guide+To+The+Kafka+Protocol) contains a wealth of useful information.
-- For more general issues, there is [a google group](https://groups.google.com/forum/#!forum/kafka-clients) for Kafka client developers.
-- If you have any questions, just ask!

vendor/github.com/Shopify/sarama/Vagrantfile

@@ -1,14 +0,0 @@
-# We have 5 * 192MB ZK processes and 5 * 320MB Kafka processes => 2560MB
-MEMORY = 3072
-
-Vagrant.configure("2") do |config|
-  config.vm.box = "ubuntu/bionic64"
-
-  config.vm.provision :shell, path: "vagrant/provision.sh"
-
-  config.vm.network "private_network", ip: "192.168.100.67"
-
-  config.vm.provider "virtualbox" do |v|
-    v.memory = MEMORY
-  end
-end

vendor/github.com/Shopify/sarama/acl_bindings.go

@@ -1,138 +0,0 @@
-package sarama
-
-//Resource holds information about acl resource type
-type Resource struct {
-	ResourceType        AclResourceType
-	ResourceName        string
-	ResourcePatternType AclResourcePatternType
-}
-
-func (r *Resource) encode(pe packetEncoder, version int16) error {
-	pe.putInt8(int8(r.ResourceType))
-
-	if err := pe.putString(r.ResourceName); err != nil {
-		return err
-	}
-
-	if version == 1 {
-		if r.ResourcePatternType == AclPatternUnknown {
-			Logger.Print("Cannot encode an unknown resource pattern type, using Literal instead")
-			r.ResourcePatternType = AclPatternLiteral
-		}
-		pe.putInt8(int8(r.ResourcePatternType))
-	}
-
-	return nil
-}
-
-func (r *Resource) decode(pd packetDecoder, version int16) (err error) {
-	resourceType, err := pd.getInt8()
-	if err != nil {
-		return err
-	}
-	r.ResourceType = AclResourceType(resourceType)
-
-	if r.ResourceName, err = pd.getString(); err != nil {
-		return err
-	}
-	if version == 1 {
-		pattern, err := pd.getInt8()
-		if err != nil {
-			return err
-		}
-		r.ResourcePatternType = AclResourcePatternType(pattern)
-	}
-
-	return nil
-}
-
-//Acl holds information about acl type
-type Acl struct {
-	Principal      string
-	Host           string
-	Operation      AclOperation
-	PermissionType AclPermissionType
-}
-
-func (a *Acl) encode(pe packetEncoder) error {
-	if err := pe.putString(a.Principal); err != nil {
-		return err
-	}
-
-	if err := pe.putString(a.Host); err != nil {
-		return err
-	}
-
-	pe.putInt8(int8(a.Operation))
-	pe.putInt8(int8(a.PermissionType))
-
-	return nil
-}
-
-func (a *Acl) decode(pd packetDecoder, version int16) (err error) {
-	if a.Principal, err = pd.getString(); err != nil {
-		return err
-	}
-
-	if a.Host, err = pd.getString(); err != nil {
-		return err
-	}
-
-	operation, err := pd.getInt8()
-	if err != nil {
-		return err
-	}
-	a.Operation = AclOperation(operation)
-
-	permissionType, err := pd.getInt8()
-	if err != nil {
-		return err
-	}
-	a.PermissionType = AclPermissionType(permissionType)
-
-	return nil
-}
-
-//ResourceAcls is an acl resource type
-type ResourceAcls struct {
-	Resource
-	Acls []*Acl
-}
-
-func (r *ResourceAcls) encode(pe packetEncoder, version int16) error {
-	if err := r.Resource.encode(pe, version); err != nil {
-		return err
-	}
-
-	if err := pe.putArrayLength(len(r.Acls)); err != nil {
-		return err
-	}
-	for _, acl := range r.Acls {
-		if err := acl.encode(pe); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-func (r *ResourceAcls) decode(pd packetDecoder, version int16) error {
-	if err := r.Resource.decode(pd, version); err != nil {
-		return err
-	}
-
-	n, err := pd.getArrayLength()
-	if err != nil {
-		return err
-	}
-
-	r.Acls = make([]*Acl, n)
-	for i := 0; i < n; i++ {
-		r.Acls[i] = new(Acl)
-		if err := r.Acls[i].decode(pd, version); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}

vendor/github.com/Shopify/sarama/acl_create_request.go

@@ -1,89 +0,0 @@
-package sarama
-
-//CreateAclsRequest is an acl creation request
-type CreateAclsRequest struct {
-	Version      int16
-	AclCreations []*AclCreation
-}
-
-func (c *CreateAclsRequest) encode(pe packetEncoder) error {
-	if err := pe.putArrayLength(len(c.AclCreations)); err != nil {
-		return err
-	}
-
-	for _, aclCreation := range c.AclCreations {
-		if err := aclCreation.encode(pe, c.Version); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-func (c *CreateAclsRequest) decode(pd packetDecoder, version int16) (err error) {
-	c.Version = version
-	n, err := pd.getArrayLength()
-	if err != nil {
-		return err
-	}
-
-	c.AclCreations = make([]*AclCreation, n)
-
-	for i := 0; i < n; i++ {
-		c.AclCreations[i] = new(AclCreation)
-		if err := c.AclCreations[i].decode(pd, version); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-func (c *CreateAclsRequest) key() int16 {
-	return 30
-}
-
-func (c *CreateAclsRequest) version() int16 {
-	return c.Version
-}
-
-func (c *CreateAclsRequest) headerVersion() int16 {
-	return 1
-}
-
-func (c *CreateAclsRequest) requiredVersion() KafkaVersion {
-	switch c.Version {
-	case 1:
-		return V2_0_0_0
-	default:
-		return V0_11_0_0
-	}
-}
-
-//AclCreation is a wrapper around Resource and Acl type
-type AclCreation struct {
-	Resource
-	Acl
-}
-
-func (a *AclCreation) encode(pe packetEncoder, version int16) error {
-	if err := a.Resource.encode(pe, version); err != nil {
-		return err
-	}
-	if err := a.Acl.encode(pe); err != nil {
-		return err
-	}
-
-	return nil
-}
-
-func (a *AclCreation) decode(pd packetDecoder, version int16) (err error) {
-	if err := a.Resource.decode(pd, version); err != nil {
-		return err
-	}
-	if err := a.Acl.decode(pd, version); err != nil {
-		return err
-	}
-
-	return nil
-}

vendor/github.com/Shopify/sarama/acl_create_response.go

@@ -1,94 +0,0 @@
-package sarama
-
-import "time"
-
-//CreateAclsResponse is a an acl response creation type
-type CreateAclsResponse struct {
-	ThrottleTime         time.Duration
-	AclCreationResponses []*AclCreationResponse
-}
-
-func (c *CreateAclsResponse) encode(pe packetEncoder) error {
-	pe.putInt32(int32(c.ThrottleTime / time.Millisecond))
-
-	if err := pe.putArrayLength(len(c.AclCreationResponses)); err != nil {
-		return err
-	}
-
-	for _, aclCreationResponse := range c.AclCreationResponses {
-		if err := aclCreationResponse.encode(pe); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-func (c *CreateAclsResponse) decode(pd packetDecoder, version int16) (err error) {
-	throttleTime, err := pd.getInt32()
-	if err != nil {
-		return err
-	}
-	c.ThrottleTime = time.Duration(throttleTime) * time.Millisecond
-
-	n, err := pd.getArrayLength()
-	if err != nil {
-		return err
-	}
-
-	c.AclCreationResponses = make([]*AclCreationResponse, n)
-	for i := 0; i < n; i++ {
-		c.AclCreationResponses[i] = new(AclCreationResponse)
-		if err := c.AclCreationResponses[i].decode(pd, version); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-func (c *CreateAclsResponse) key() int16 {
-	return 30
-}
-
-func (c *CreateAclsResponse) version() int16 {
-	return 0
-}
-
-func (c *CreateAclsResponse) headerVersion() int16 {
-	return 0
-}
-
-func (c *CreateAclsResponse) requiredVersion() KafkaVersion {
-	return V0_11_0_0
-}
-
-//AclCreationResponse is an acl creation response type
-type AclCreationResponse struct {
-	Err    KError
-	ErrMsg *string
-}
-
-func (a *AclCreationResponse) encode(pe packetEncoder) error {
-	pe.putInt16(int16(a.Err))
-
-	if err := pe.putNullableString(a.ErrMsg); err != nil {
-		return err
-	}
-
-	return nil
-}
-
-func (a *AclCreationResponse) decode(pd packetDecoder, version int16) (err error) {
-	kerr, err := pd.getInt16()
-	if err != nil {
-		return err
-	}
-	a.Err = KError(kerr)
-
-	if a.ErrMsg, err = pd.getNullableString(); err != nil {
-		return err
-	}
-
-	return nil
-}

vendor/github.com/Shopify/sarama/acl_delete_request.go

@@ -1,62 +0,0 @@
-package sarama
-
-//DeleteAclsRequest is a delete acl request
-type DeleteAclsRequest struct {
-	Version int
-	Filters []*AclFilter
-}
-
-func (d *DeleteAclsRequest) encode(pe packetEncoder) error {
-	if err := pe.putArrayLength(len(d.Filters)); err != nil {
-		return err
-	}
-
-	for _, filter := range d.Filters {
-		filter.Version = d.Version
-		if err := filter.encode(pe); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-func (d *DeleteAclsRequest) decode(pd packetDecoder, version int16) (err error) {
-	d.Version = int(version)
-	n, err := pd.getArrayLength()
-	if err != nil {
-		return err
-	}
-
-	d.Filters = make([]*AclFilter, n)
-	for i := 0; i < n; i++ {
-		d.Filters[i] = new(AclFilter)
-		d.Filters[i].Version = int(version)
-		if err := d.Filters[i].decode(pd, version); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-func (d *DeleteAclsRequest) key() int16 {
-	return 31
-}
-
-func (d *DeleteAclsRequest) version() int16 {
-	return int16(d.Version)
-}
-
-func (c *DeleteAclsRequest) headerVersion() int16 {
-	return 1
-}
-
-func (d *DeleteAclsRequest) requiredVersion() KafkaVersion {
-	switch d.Version {
-	case 1:
-		return V2_0_0_0
-	default:
-		return V0_11_0_0
-	}
-}

vendor/github.com/Shopify/sarama/acl_delete_response.go

@@ -1,163 +0,0 @@
-package sarama
-
-import "time"
-
-//DeleteAclsResponse is a delete acl response
-type DeleteAclsResponse struct {
-	Version         int16
-	ThrottleTime    time.Duration
-	FilterResponses []*FilterResponse
-}
-
-func (d *DeleteAclsResponse) encode(pe packetEncoder) error {
-	pe.putInt32(int32(d.ThrottleTime / time.Millisecond))
-
-	if err := pe.putArrayLength(len(d.FilterResponses)); err != nil {
-		return err
-	}
-
-	for _, filterResponse := range d.FilterResponses {
-		if err := filterResponse.encode(pe, d.Version); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-func (d *DeleteAclsResponse) decode(pd packetDecoder, version int16) (err error) {
-	throttleTime, err := pd.getInt32()
-	if err != nil {
-		return err
-	}
-	d.ThrottleTime = time.Duration(throttleTime) * time.Millisecond
-
-	n, err := pd.getArrayLength()
-	if err != nil {
-		return err
-	}
-	d.FilterResponses = make([]*FilterResponse, n)
-
-	for i := 0; i < n; i++ {
-		d.FilterResponses[i] = new(FilterResponse)
-		if err := d.FilterResponses[i].decode(pd, version); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-func (d *DeleteAclsResponse) key() int16 {
-	return 31
-}
-
-func (d *DeleteAclsResponse) version() int16 {
-	return d.Version
-}
-
-func (d *DeleteAclsResponse) headerVersion() int16 {
-	return 0
-}
-
-func (d *DeleteAclsResponse) requiredVersion() KafkaVersion {
-	return V0_11_0_0
-}
-
-//FilterResponse is a filter response type
-type FilterResponse struct {
-	Err          KError
-	ErrMsg       *string
-	MatchingAcls []*MatchingAcl
-}
-
-func (f *FilterResponse) encode(pe packetEncoder, version int16) error {
-	pe.putInt16(int16(f.Err))
-	if err := pe.putNullableString(f.ErrMsg); err != nil {
-		return err
-	}
-
-	if err := pe.putArrayLength(len(f.MatchingAcls)); err != nil {
-		return err
-	}
-	for _, matchingAcl := range f.MatchingAcls {
-		if err := matchingAcl.encode(pe, version); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-func (f *FilterResponse) decode(pd packetDecoder, version int16) (err error) {
-	kerr, err := pd.getInt16()
-	if err != nil {
-		return err
-	}
-	f.Err = KError(kerr)
-
-	if f.ErrMsg, err = pd.getNullableString(); err != nil {
-		return err
-	}
-
-	n, err := pd.getArrayLength()
-	if err != nil {
-		return err
-	}
-	f.MatchingAcls = make([]*MatchingAcl, n)
-	for i := 0; i < n; i++ {
-		f.MatchingAcls[i] = new(MatchingAcl)
-		if err := f.MatchingAcls[i].decode(pd, version); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-//MatchingAcl is a matching acl type
-type MatchingAcl struct {
-	Err    KError
-	ErrMsg *string
-	Resource
-	Acl
-}
-
-func (m *MatchingAcl) encode(pe packetEncoder, version int16) error {
-	pe.putInt16(int16(m.Err))
-	if err := pe.putNullableString(m.ErrMsg); err != nil {
-		return err
-	}
-
-	if err := m.Resource.encode(pe, version); err != nil {
-		return err
-	}
-
-	if err := m.Acl.encode(pe); err != nil {
-		return err
-	}
-
-	return nil
-}
-
-func (m *MatchingAcl) decode(pd packetDecoder, version int16) (err error) {
-	kerr, err := pd.getInt16()
-	if err != nil {
-		return err
-	}
-	m.Err = KError(kerr)
-
-	if m.ErrMsg, err = pd.getNullableString(); err != nil {
-		return err
-	}
-
-	if err := m.Resource.decode(pd, version); err != nil {
-		return err
-	}
-
-	if err := m.Acl.decode(pd, version); err != nil {
-		return err
-	}
-
-	return nil
-}

vendor/github.com/Shopify/sarama/acl_describe_request.go

@@ -1,39 +0,0 @@
-package sarama
-
-//DescribeAclsRequest is a secribe acl request type
-type DescribeAclsRequest struct {
-	Version int
-	AclFilter
-}
-
-func (d *DescribeAclsRequest) encode(pe packetEncoder) error {
-	d.AclFilter.Version = d.Version
-	return d.AclFilter.encode(pe)
-}
-
-func (d *DescribeAclsRequest) decode(pd packetDecoder, version int16) (err error) {
-	d.Version = int(version)
-	d.AclFilter.Version = int(version)
-	return d.AclFilter.decode(pd, version)
-}
-
-func (d *DescribeAclsRequest) key() int16 {
-	return 29
-}
-
-func (d *DescribeAclsRequest) version() int16 {
-	return int16(d.Version)
-}
-
-func (d *DescribeAclsRequest) headerVersion() int16 {
-	return 1
-}
-
-func (d *DescribeAclsRequest) requiredVersion() KafkaVersion {
-	switch d.Version {
-	case 1:
-		return V2_0_0_0
-	default:
-		return V0_11_0_0
-	}
-}

vendor/github.com/Shopify/sarama/acl_describe_response.go

@@ -1,91 +0,0 @@
-package sarama
-
-import "time"
-
-//DescribeAclsResponse is a describe acl response type
-type DescribeAclsResponse struct {
-	Version      int16
-	ThrottleTime time.Duration
-	Err          KError
-	ErrMsg       *string
-	ResourceAcls []*ResourceAcls
-}
-
-func (d *DescribeAclsResponse) encode(pe packetEncoder) error {
-	pe.putInt32(int32(d.ThrottleTime / time.Millisecond))
-	pe.putInt16(int16(d.Err))
-
-	if err := pe.putNullableString(d.ErrMsg); err != nil {
-		return err
-	}
-
-	if err := pe.putArrayLength(len(d.ResourceAcls)); err != nil {
-		return err
-	}
-
-	for _, resourceAcl := range d.ResourceAcls {
-		if err := resourceAcl.encode(pe, d.Version); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-func (d *DescribeAclsResponse) decode(pd packetDecoder, version int16) (err error) {
-	throttleTime, err := pd.getInt32()
-	if err != nil {
-		return err
-	}
-	d.ThrottleTime = time.Duration(throttleTime) * time.Millisecond
-
-	kerr, err := pd.getInt16()
-	if err != nil {
-		return err
-	}
-	d.Err = KError(kerr)
-
-	errmsg, err := pd.getString()
-	if err != nil {
-		return err
-	}
-	if errmsg != "" {
-		d.ErrMsg = &errmsg
-	}
-
-	n, err := pd.getArrayLength()
-	if err != nil {
-		return err
-	}
-	d.ResourceAcls = make([]*ResourceAcls, n)
-
-	for i := 0; i < n; i++ {
-		d.ResourceAcls[i] = new(ResourceAcls)
-		if err := d.ResourceAcls[i].decode(pd, version); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-func (d *DescribeAclsResponse) key() int16 {
-	return 29
-}
-
-func (d *DescribeAclsResponse) version() int16 {
-	return d.Version
-}
-
-func (d *DescribeAclsResponse) headerVersion() int16 {
-	return 0
-}
-
-func (d *DescribeAclsResponse) requiredVersion() KafkaVersion {
-	switch d.Version {
-	case 1:
-		return V2_0_0_0
-	default:
-		return V0_11_0_0
-	}
-}

vendor/github.com/Shopify/sarama/acl_filter.go

@@ -1,78 +0,0 @@
-package sarama
-
-type AclFilter struct {
-	Version                   int
-	ResourceType              AclResourceType
-	ResourceName              *string
-	ResourcePatternTypeFilter AclResourcePatternType
-	Principal                 *string
-	Host                      *string
-	Operation                 AclOperation
-	PermissionType            AclPermissionType
-}
-
-func (a *AclFilter) encode(pe packetEncoder) error {
-	pe.putInt8(int8(a.ResourceType))
-	if err := pe.putNullableString(a.ResourceName); err != nil {
-		return err
-	}
-
-	if a.Version == 1 {
-		pe.putInt8(int8(a.ResourcePatternTypeFilter))
-	}
-
-	if err := pe.putNullableString(a.Principal); err != nil {
-		return err
-	}
-	if err := pe.putNullableString(a.Host); err != nil {
-		return err
-	}
-	pe.putInt8(int8(a.Operation))
-	pe.putInt8(int8(a.PermissionType))
-
-	return nil
-}
-
-func (a *AclFilter) decode(pd packetDecoder, version int16) (err error) {
-	resourceType, err := pd.getInt8()
-	if err != nil {
-		return err
-	}
-	a.ResourceType = AclResourceType(resourceType)
-
-	if a.ResourceName, err = pd.getNullableString(); err != nil {
-		return err
-	}
-
-	if a.Version == 1 {
-		pattern, err := pd.getInt8()
-
-		if err != nil {
-			return err
-		}
-
-		a.ResourcePatternTypeFilter = AclResourcePatternType(pattern)
-	}
-
-	if a.Principal, err = pd.getNullableString(); err != nil {
-		return err
-	}
-
-	if a.Host, err = pd.getNullableString(); err != nil {
-		return err
-	}
-
-	operation, err := pd.getInt8()
-	if err != nil {
-		return err
-	}
-	a.Operation = AclOperation(operation)
-
-	permissionType, err := pd.getInt8()
-	if err != nil {
-		return err
-	}
-	a.PermissionType = AclPermissionType(permissionType)
-
-	return nil
-}

vendor/github.com/Shopify/sarama/acl_types.go

@@ -1,55 +0,0 @@
-package sarama
-
-type (
-	AclOperation int
-
-	AclPermissionType int
-
-	AclResourceType int
-
-	AclResourcePatternType int
-)
-
-// ref: https://github.com/apache/kafka/blob/trunk/clients/src/main/java/org/apache/kafka/common/acl/AclOperation.java
-const (
-	AclOperationUnknown AclOperation = iota
-	AclOperationAny
-	AclOperationAll
-	AclOperationRead
-	AclOperationWrite
-	AclOperationCreate
-	AclOperationDelete
-	AclOperationAlter
-	AclOperationDescribe
-	AclOperationClusterAction
-	AclOperationDescribeConfigs
-	AclOperationAlterConfigs
-	AclOperationIdempotentWrite
-)
-
-// ref: https://github.com/apache/kafka/blob/trunk/clients/src/main/java/org/apache/kafka/common/acl/AclPermissionType.java
-const (
-	AclPermissionUnknown AclPermissionType = iota
-	AclPermissionAny
-	AclPermissionDeny
-	AclPermissionAllow
-)
-
-// ref: https://github.com/apache/kafka/blob/trunk/clients/src/main/java/org/apache/kafka/common/resource/ResourceType.java
-const (
-	AclResourceUnknown AclResourceType = iota
-	AclResourceAny
-	AclResourceTopic
-	AclResourceGroup
-	AclResourceCluster
-	AclResourceTransactionalID
-)
-
-// ref: https://github.com/apache/kafka/blob/trunk/clients/src/main/java/org/apache/kafka/common/resource/PatternType.java
-const (
-	AclPatternUnknown AclResourcePatternType = iota
-	AclPatternAny
-	AclPatternMatch
-	AclPatternLiteral
-	AclPatternPrefixed
-)

vendor/github.com/Shopify/sarama/add_offsets_to_txn_request.go

@@ -1,57 +0,0 @@
-package sarama
-
-//AddOffsetsToTxnRequest adds offsets to a transaction request
-type AddOffsetsToTxnRequest struct {
-	TransactionalID string
-	ProducerID      int64
-	ProducerEpoch   int16
-	GroupID         string
-}
-
-func (a *AddOffsetsToTxnRequest) encode(pe packetEncoder) error {
-	if err := pe.putString(a.TransactionalID); err != nil {
-		return err
-	}
-
-	pe.putInt64(a.ProducerID)
-
-	pe.putInt16(a.ProducerEpoch)
-
-	if err := pe.putString(a.GroupID); err != nil {
-		return err
-	}
-
-	return nil
-}
-
-func (a *AddOffsetsToTxnRequest) decode(pd packetDecoder, version int16) (err error) {
-	if a.TransactionalID, err = pd.getString(); err != nil {
-		return err
-	}
-	if a.ProducerID, err = pd.getInt64(); err != nil {
-		return err
-	}
-	if a.ProducerEpoch, err = pd.getInt16(); err != nil {
-		return err
-	}
-	if a.GroupID, err = pd.getString(); err != nil {
-		return err
-	}
-	return nil
-}
-
-func (a *AddOffsetsToTxnRequest) key() int16 {
-	return 25
-}
-
-func (a *AddOffsetsToTxnRequest) version() int16 {
-	return 0
-}
-
-func (a *AddOffsetsToTxnRequest) headerVersion() int16 {
-	return 1
-}
-
-func (a *AddOffsetsToTxnRequest) requiredVersion() KafkaVersion {
-	return V0_11_0_0
-}

vendor/github.com/Shopify/sarama/add_offsets_to_txn_response.go

@@ -1,49 +0,0 @@
-package sarama
-
-import (
-	"time"
-)
-
-//AddOffsetsToTxnResponse is a response type for adding offsets to txns
-type AddOffsetsToTxnResponse struct {
-	ThrottleTime time.Duration
-	Err          KError
-}
-
-func (a *AddOffsetsToTxnResponse) encode(pe packetEncoder) error {
-	pe.putInt32(int32(a.ThrottleTime / time.Millisecond))
-	pe.putInt16(int16(a.Err))
-	return nil
-}
-
-func (a *AddOffsetsToTxnResponse) decode(pd packetDecoder, version int16) (err error) {
-	throttleTime, err := pd.getInt32()
-	if err != nil {
-		return err
-	}
-	a.ThrottleTime = time.Duration(throttleTime) * time.Millisecond
-
-	kerr, err := pd.getInt16()
-	if err != nil {
-		return err
-	}
-	a.Err = KError(kerr)
-
-	return nil
-}
-
-func (a *AddOffsetsToTxnResponse) key() int16 {
-	return 25
-}
-
-func (a *AddOffsetsToTxnResponse) version() int16 {
-	return 0
-}
-
-func (a *AddOffsetsToTxnResponse) headerVersion() int16 {
-	return 0
-}
-
-func (a *AddOffsetsToTxnResponse) requiredVersion() KafkaVersion {
-	return V0_11_0_0
-}

vendor/github.com/Shopify/sarama/add_partitions_to_txn_request.go

@@ -1,81 +0,0 @@
-package sarama
-
-//AddPartitionsToTxnRequest is a add paartition request
-type AddPartitionsToTxnRequest struct {
-	TransactionalID string
-	ProducerID      int64
-	ProducerEpoch   int16
-	TopicPartitions map[string][]int32
-}
-
-func (a *AddPartitionsToTxnRequest) encode(pe packetEncoder) error {
-	if err := pe.putString(a.TransactionalID); err != nil {
-		return err
-	}
-	pe.putInt64(a.ProducerID)
-	pe.putInt16(a.ProducerEpoch)
-
-	if err := pe.putArrayLength(len(a.TopicPartitions)); err != nil {
-		return err
-	}
-	for topic, partitions := range a.TopicPartitions {
-		if err := pe.putString(topic); err != nil {
-			return err
-		}
-		if err := pe.putInt32Array(partitions); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-func (a *AddPartitionsToTxnRequest) decode(pd packetDecoder, version int16) (err error) {
-	if a.TransactionalID, err = pd.getString(); err != nil {
-		return err
-	}
-	if a.ProducerID, err = pd.getInt64(); err != nil {
-		return err
-	}
-	if a.ProducerEpoch, err = pd.getInt16(); err != nil {
-		return err
-	}
-
-	n, err := pd.getArrayLength()
-	if err != nil {
-		return err
-	}
-
-	a.TopicPartitions = make(map[string][]int32)
-	for i := 0; i < n; i++ {
-		topic, err := pd.getString()
-		if err != nil {
-			return err
-		}
-
-		partitions, err := pd.getInt32Array()
-		if err != nil {
-			return err
-		}
-
-		a.TopicPartitions[topic] = partitions
-	}
-
-	return nil
-}
-
-func (a *AddPartitionsToTxnRequest) key() int16 {
-	return 24
-}
-
-func (a *AddPartitionsToTxnRequest) version() int16 {
-	return 0
-}
-
-func (a *AddPartitionsToTxnRequest) headerVersion() int16 {
-	return 1
-}
-
-func (a *AddPartitionsToTxnRequest) requiredVersion() KafkaVersion {
-	return V0_11_0_0
-}

vendor/github.com/Shopify/sarama/add_partitions_to_txn_response.go

@@ -1,114 +0,0 @@
-package sarama
-
-import (
-	"time"
-)
-
-//AddPartitionsToTxnResponse is a partition errors to transaction type
-type AddPartitionsToTxnResponse struct {
-	ThrottleTime time.Duration
-	Errors       map[string][]*PartitionError
-}
-
-func (a *AddPartitionsToTxnResponse) encode(pe packetEncoder) error {
-	pe.putInt32(int32(a.ThrottleTime / time.Millisecond))
-	if err := pe.putArrayLength(len(a.Errors)); err != nil {
-		return err
-	}
-
-	for topic, e := range a.Errors {
-		if err := pe.putString(topic); err != nil {
-			return err
-		}
-		if err := pe.putArrayLength(len(e)); err != nil {
-			return err
-		}
-		for _, partitionError := range e {
-			if err := partitionError.encode(pe); err != nil {
-				return err
-			}
-		}
-	}
-
-	return nil
-}
-
-func (a *AddPartitionsToTxnResponse) decode(pd packetDecoder, version int16) (err error) {
-	throttleTime, err := pd.getInt32()
-	if err != nil {
-		return err
-	}
-	a.ThrottleTime = time.Duration(throttleTime) * time.Millisecond
-
-	n, err := pd.getArrayLength()
-	if err != nil {
-		return err
-	}
-
-	a.Errors = make(map[string][]*PartitionError)
-
-	for i := 0; i < n; i++ {
-		topic, err := pd.getString()
-		if err != nil {
-			return err
-		}
-
-		m, err := pd.getArrayLength()
-		if err != nil {
-			return err
-		}
-
-		a.Errors[topic] = make([]*PartitionError, m)
-
-		for j := 0; j < m; j++ {
-			a.Errors[topic][j] = new(PartitionError)
-			if err := a.Errors[topic][j].decode(pd, version); err != nil {
-				return err
-			}
-		}
-	}
-
-	return nil
-}
-
-func (a *AddPartitionsToTxnResponse) key() int16 {
-	return 24
-}
-
-func (a *AddPartitionsToTxnResponse) version() int16 {
-	return 0
-}
-
-func (a *AddPartitionsToTxnResponse) headerVersion() int16 {
-	return 0
-}
-
-func (a *AddPartitionsToTxnResponse) requiredVersion() KafkaVersion {
-	return V0_11_0_0
-}
-
-//PartitionError is a partition error type
-type PartitionError struct {
-	Partition int32
-	Err       KError
-}
-
-func (p *PartitionError) encode(pe packetEncoder) error {
-	pe.putInt32(p.Partition)
-	pe.putInt16(int16(p.Err))
-	return nil
-}
-
-func (p *PartitionError) decode(pd packetDecoder, version int16) (err error) {
-	if p.Partition, err = pd.getInt32(); err != nil {
-		return err
-	}
-
-	kerr, err := pd.getInt16()
-	if err != nil {
-		return err
-	}
-	p.Err = KError(kerr)
-
-	return nil
-}

vendor/github.com/Shopify/sarama/admin.go

@@ -1,934 +0,0 @@
-package sarama
-
-import (
-	"errors"
-	"fmt"
-	"math/rand"
-	"strconv"
-	"sync"
-	"time"
-)
-
-// ClusterAdmin is the administrative client for Kafka, which supports managing and inspecting topics,
-// brokers, configurations and ACLs. The minimum broker version required is 0.10.0.0.
-// Methods with stricter requirements will specify the minimum broker version required.
-// You MUST call Close() on a client to avoid leaks
-type ClusterAdmin interface {
-	// Creates a new topic. This operation is supported by brokers with version 0.10.1.0 or higher.
-	// It may take several seconds after CreateTopic returns success for all the brokers
-	// to become aware that the topic has been created. During this time, listTopics
-	// may not return information about the new topic.The validateOnly option is supported from version 0.10.2.0.
-	CreateTopic(topic string, detail *TopicDetail, validateOnly bool) error
-
-	// List the topics available in the cluster with the default options.
-	ListTopics() (map[string]TopicDetail, error)
-
-	// Describe some topics in the cluster.
-	DescribeTopics(topics []string) (metadata []*TopicMetadata, err error)
-
-	// Delete a topic. It may take several seconds after the DeleteTopic to returns success
-	// and for all the brokers to become aware that the topics are gone.
-	// During this time, listTopics  may continue to return information about the deleted topic.
-	// If delete.topic.enable is false on the brokers, deleteTopic will mark
-	// the topic for deletion, but not actually delete them.
-	// This operation is supported by brokers with version 0.10.1.0 or higher.
-	DeleteTopic(topic string) error
-
-	// Increase the number of partitions of the topics  according to the corresponding values.
-	// If partitions are increased for a topic that has a key, the partition logic or ordering of
-	// the messages will be affected. It may take several seconds after this method returns
-	// success for all the brokers to become aware that the partitions have been created.
-	// During this time, ClusterAdmin#describeTopics may not return information about the
-	// new partitions. This operation is supported by brokers with version 1.0.0 or higher.
-	CreatePartitions(topic string, count int32, assignment [][]int32, validateOnly bool) error
-
-	// Alter the replica assignment for partitions.
-	// This operation is supported by brokers with version 2.4.0.0 or higher.
-	AlterPartitionReassignments(topic string, assignment [][]int32) error
-
-	// Provides info on ongoing partitions replica reassignments.
-	// This operation is supported by brokers with version 2.4.0.0 or higher.
-	ListPartitionReassignments(topics string, partitions []int32) (topicStatus map[string]map[int32]*PartitionReplicaReassignmentsStatus, err error)
-
-	// Delete records whose offset is smaller than the given offset of the corresponding partition.
-	// This operation is supported by brokers with version 0.11.0.0 or higher.
-	DeleteRecords(topic string, partitionOffsets map[int32]int64) error
-
-	// Get the configuration for the specified resources.
-	// The returned configuration includes default values and the Default is true
-	// can be used to distinguish them from user supplied values.
-	// Config entries where ReadOnly is true cannot be updated.
-	// The value of config entries where Sensitive is true is always nil so
-	// sensitive information is not disclosed.
-	// This operation is supported by brokers with version 0.11.0.0 or higher.
-	DescribeConfig(resource ConfigResource) ([]ConfigEntry, error)
-
-	// Update the configuration for the specified resources with the default options.
-	// This operation is supported by brokers with version 0.11.0.0 or higher.
-	// The resources with their configs (topic is the only resource type with configs
-	// that can be updated currently Updates are not transactional so they may succeed
-	// for some resources while fail for others. The configs for a particular resource are updated automatically.
-	AlterConfig(resourceType ConfigResourceType, name string, entries map[string]*string, validateOnly bool) error
-
-	// Creates access control lists (ACLs) which are bound to specific resources.
-	// This operation is not transactional so it may succeed for some ACLs while fail for others.
-	// If you attempt to add an ACL that duplicates an existing ACL, no error will be raised, but
-	// no changes will be made. This operation is supported by brokers with version 0.11.0.0 or higher.
-	CreateACL(resource Resource, acl Acl) error
-
-	// Lists access control lists (ACLs) according to the supplied filter.
-	// it may take some time for changes made by createAcls or deleteAcls to be reflected in the output of ListAcls
-	// This operation is supported by brokers with version 0.11.0.0 or higher.
-	ListAcls(filter AclFilter) ([]ResourceAcls, error)
-
-	// Deletes access control lists (ACLs) according to the supplied filters.
-	// This operation is not transactional so it may succeed for some ACLs while fail for others.
-	// This operation is supported by brokers with version 0.11.0.0 or higher.
-	DeleteACL(filter AclFilter, validateOnly bool) ([]MatchingAcl, error)
-
-	// List the consumer groups available in the cluster.
-	ListConsumerGroups() (map[string]string, error)
-
-	// Describe the given consumer groups.
-	DescribeConsumerGroups(groups []string) ([]*GroupDescription, error)
-
-	// List the consumer group offsets available in the cluster.
-	ListConsumerGroupOffsets(group string, topicPartitions map[string][]int32) (*OffsetFetchResponse, error)
-
-	// Delete a consumer group.
-	DeleteConsumerGroup(group string) error
-
-	// Get information about the nodes in the cluster
-	DescribeCluster() (brokers []*Broker, controllerID int32, err error)
-
-	// Get information about all log directories on the given set of brokers
-	DescribeLogDirs(brokers []int32) (map[int32][]DescribeLogDirsResponseDirMetadata, error)
-
-	// Close shuts down the admin and closes underlying client.
-	Close() error
-}
-
-type clusterAdmin struct {
-	client Client
-	conf   *Config
-}
-
-// NewClusterAdmin creates a new ClusterAdmin using the given broker addresses and configuration.
-func NewClusterAdmin(addrs []string, conf *Config) (ClusterAdmin, error) {
-	client, err := NewClient(addrs, conf)
-	if err != nil {
-		return nil, err
-	}
-	return NewClusterAdminFromClient(client)
-}
-
-// NewClusterAdminFromClient creates a new ClusterAdmin using the given client.
-// Note that underlying client will also be closed on admin's Close() call.
-func NewClusterAdminFromClient(client Client) (ClusterAdmin, error) {
-	//make sure we can retrieve the controller
-	_, err := client.Controller()
-	if err != nil {
-		return nil, err
-	}
-
-	ca := &clusterAdmin{
-		client: client,
-		conf:   client.Config(),
-	}
-	return ca, nil
-}
-
-func (ca *clusterAdmin) Close() error {
-	return ca.client.Close()
-}
-
-func (ca *clusterAdmin) Controller() (*Broker, error) {
-	return ca.client.Controller()
-}
-
-func (ca *clusterAdmin) refreshController() (*Broker, error) {
-	return ca.client.RefreshController()
-}
-
-// isErrNoController returns `true` if the given error type unwraps to an
-// `ErrNotController` response from Kafka
-func isErrNoController(err error) bool {
-	switch e := err.(type) {
-	case *TopicError:
-		return e.Err == ErrNotController
-	case *TopicPartitionError:
-		return e.Err == ErrNotController
-	case KError:
-		return e == ErrNotController
-	}
-	return false
-}
-
-// retryOnError will repeatedly call the given (error-returning) func in the
-// case that its response is non-nil and retriable (as determined by the
-// provided retriable func) up to the maximum number of tries permitted by
-// the admin client configuration
-func (ca *clusterAdmin) retryOnError(retriable func(error) bool, fn func() error) error {
-	var err error
-	for attempt := 0; attempt < ca.conf.Admin.Retry.Max; attempt++ {
-		err = fn()
-		if err == nil || !retriable(err) {
-			return err
-		}
-		Logger.Printf(
-			"admin/request retrying after %dms... (%d attempts remaining)\n",
-			ca.conf.Admin.Retry.Backoff/time.Millisecond, ca.conf.Admin.Retry.Max-attempt)
-		time.Sleep(ca.conf.Admin.Retry.Backoff)
-		continue
-	}
-	return err
-}
-
-func (ca *clusterAdmin) CreateTopic(topic string, detail *TopicDetail, validateOnly bool) error {
-	if topic == "" {
-		return ErrInvalidTopic
-	}
-
-	if detail == nil {
-		return errors.New("you must specify topic details")
-	}
-
-	topicDetails := make(map[string]*TopicDetail)
-	topicDetails[topic] = detail
-
-	request := &CreateTopicsRequest{
-		TopicDetails: topicDetails,
-		ValidateOnly: validateOnly,
-		Timeout:      ca.conf.Admin.Timeout,
-	}
-
-	if ca.conf.Version.IsAtLeast(V0_11_0_0) {
-		request.Version = 1
-	}
-	if ca.conf.Version.IsAtLeast(V1_0_0_0) {
-		request.Version = 2
-	}
-
-	return ca.retryOnError(isErrNoController, func() error {
-		b, err := ca.Controller()
-		if err != nil {
-			return err
-		}
-
-		rsp, err := b.CreateTopics(request)
-		if err != nil {
-			return err
-		}
-
-		topicErr, ok := rsp.TopicErrors[topic]
-		if !ok {
-			return ErrIncompleteResponse
-		}
-
-		if topicErr.Err != ErrNoError {
-			if topicErr.Err == ErrNotController {
-				_, _ = ca.refreshController()
-			}
-			return topicErr
-		}
-
-		return nil
-	})
-}
-
-func (ca *clusterAdmin) DescribeTopics(topics []string) (metadata []*TopicMetadata, err error) {
-	controller, err := ca.Controller()
-	if err != nil {
-		return nil, err
-	}
-
-	request := &MetadataRequest{
-		Topics:                 topics,
-		AllowAutoTopicCreation: false,
-	}
-
-	if ca.conf.Version.IsAtLeast(V1_0_0_0) {
-		request.Version = 5
-	} else if ca.conf.Version.IsAtLeast(V0_11_0_0) {
-		request.Version = 4
-	}
-
-	response, err := controller.GetMetadata(request)
-	if err != nil {
-		return nil, err
-	}
-	return response.Topics, nil
-}
-
-func (ca *clusterAdmin) DescribeCluster() (brokers []*Broker, controllerID int32, err error) {
-	controller, err := ca.Controller()
-	if err != nil {
-		return nil, int32(0), err
-	}
-
-	request := &MetadataRequest{
-		Topics: []string{},
-	}
-
-	if ca.conf.Version.IsAtLeast(V0_10_0_0) {
-		request.Version = 1
-	}
-
-	response, err := controller.GetMetadata(request)
-	if err != nil {
-		return nil, int32(0), err
-	}
-
-	return response.Brokers, response.ControllerID, nil
-}
-
-func (ca *clusterAdmin) findBroker(id int32) (*Broker, error) {
-	brokers := ca.client.Brokers()
-	for _, b := range brokers {
-		if b.ID() == id {
-			return b, nil
-		}
-	}
-	return nil, fmt.Errorf("could not find broker id %d", id)
-}
-
-func (ca *clusterAdmin) findAnyBroker() (*Broker, error) {
-	brokers := ca.client.Brokers()
-	if len(brokers) > 0 {
-		index := rand.Intn(len(brokers))
-		return brokers[index], nil
-	}
-	return nil, errors.New("no available broker")
-}
-
-func (ca *clusterAdmin) ListTopics() (map[string]TopicDetail, error) {
-	// In order to build TopicDetails we need to first get the list of all
-	// topics using a MetadataRequest and then get their configs using a
-	// DescribeConfigsRequest request. To avoid sending many requests to the
-	// broker, we use a single DescribeConfigsRequest.
-
-	// Send the all-topic MetadataRequest
-	b, err := ca.findAnyBroker()
-	if err != nil {
-		return nil, err
-	}
-	_ = b.Open(ca.client.Config())
-
-	metadataReq := &MetadataRequest{}
-	metadataResp, err := b.GetMetadata(metadataReq)
-	if err != nil {
-		return nil, err
-	}
-
-	topicsDetailsMap := make(map[string]TopicDetail)
-
-	var describeConfigsResources []*ConfigResource
-
-	for _, topic := range metadataResp.Topics {
-		topicDetails := TopicDetail{
-			NumPartitions: int32(len(topic.Partitions)),
-		}
-		if len(topic.Partitions) > 0 {
-			topicDetails.ReplicaAssignment = map[int32][]int32{}
-			for _, partition := range topic.Partitions {
-				topicDetails.ReplicaAssignment[partition.ID] = partition.Replicas
-			}
-			topicDetails.ReplicationFactor = int16(len(topic.Partitions[0].Replicas))
-		}
-		topicsDetailsMap[topic.Name] = topicDetails
-
-		// we populate the resources we want to describe from the MetadataResponse
-		topicResource := ConfigResource{
-			Type: TopicResource,
-			Name: topic.Name,
-		}
-		describeConfigsResources = append(describeConfigsResources, &topicResource)
-	}
-
-	// Send the DescribeConfigsRequest
-	describeConfigsReq := &DescribeConfigsRequest{
-		Resources: describeConfigsResources,
-	}
-
-	if ca.conf.Version.IsAtLeast(V1_1_0_0) {
-		describeConfigsReq.Version = 1
-	}
-
-	if ca.conf.Version.IsAtLeast(V2_0_0_0) {
-		describeConfigsReq.Version = 2
-	}
-
-	describeConfigsResp, err := b.DescribeConfigs(describeConfigsReq)
-	if err != nil {
-		return nil, err
-	}
-
-	for _, resource := range describeConfigsResp.Resources {
-		topicDetails := topicsDetailsMap[resource.Name]
-		topicDetails.ConfigEntries = make(map[string]*string)
-
-		for _, entry := range resource.Configs {
-			// only include non-default non-sensitive config
-			// (don't actually think topic config will ever be sensitive)
-			if entry.Default || entry.Sensitive {
-				continue
-			}
-			topicDetails.ConfigEntries[entry.Name] = &entry.Value
-		}
-
-		topicsDetailsMap[resource.Name] = topicDetails
-	}
-
-	return topicsDetailsMap, nil
-}
-
-func (ca *clusterAdmin) DeleteTopic(topic string) error {
-	if topic == "" {
-		return ErrInvalidTopic
-	}
-
-	request := &DeleteTopicsRequest{
-		Topics:  []string{topic},
-		Timeout: ca.conf.Admin.Timeout,
-	}
-
-	if ca.conf.Version.IsAtLeast(V0_11_0_0) {
-		request.Version = 1
-	}
-
-	return ca.retryOnError(isErrNoController, func() error {
-		b, err := ca.Controller()
-		if err != nil {
-			return err
-		}
-
-		rsp, err := b.DeleteTopics(request)
-		if err != nil {
-			return err
-		}
-
-		topicErr, ok := rsp.TopicErrorCodes[topic]
-		if !ok {
-			return ErrIncompleteResponse
-		}
-
-		if topicErr != ErrNoError {
-			if topicErr == ErrNotController {
-				_, _ = ca.refreshController()
-			}
-			return topicErr
-		}
-
-		return nil
-	})
-}
-
-func (ca *clusterAdmin) CreatePartitions(topic string, count int32, assignment [][]int32, validateOnly bool) error {
-	if topic == "" {
-		return ErrInvalidTopic
-	}
-
-	topicPartitions := make(map[string]*TopicPartition)
-	topicPartitions[topic] = &TopicPartition{Count: count, Assignment: assignment}
-
-	request := &CreatePartitionsRequest{
-		TopicPartitions: topicPartitions,
-		Timeout:         ca.conf.Admin.Timeout,
-	}
-
-	return ca.retryOnError(isErrNoController, func() error {
-		b, err := ca.Controller()
-		if err != nil {
-			return err
-		}
-
-		rsp, err := b.CreatePartitions(request)
-		if err != nil {
-			return err
-		}
-
-		topicErr, ok := rsp.TopicPartitionErrors[topic]
-		if !ok {
-			return ErrIncompleteResponse
-		}
-
-		if topicErr.Err != ErrNoError {
-			if topicErr.Err == ErrNotController {
-				_, _ = ca.refreshController()
-			}
-			return topicErr
-		}
-
-		return nil
-	})
-}
-
-func (ca *clusterAdmin) AlterPartitionReassignments(topic string, assignment [][]int32) error {
-	if topic == "" {
-		return ErrInvalidTopic
-	}
-
-	request := &AlterPartitionReassignmentsRequest{
-		TimeoutMs: int32(60000),
-		Version:   int16(0),
-	}
-
-	for i := 0; i < len(assignment); i++ {
-		request.AddBlock(topic, int32(i), assignment[i])
-	}
-
-	return ca.retryOnError(isErrNoController, func() error {
-		b, err := ca.Controller()
-		if err != nil {
-			return err
-		}
-
-		errs := make([]error, 0)
-
-		rsp, err := b.AlterPartitionReassignments(request)
-
-		if err != nil {
-			errs = append(errs, err)
-		} else {
-			if rsp.ErrorCode > 0 {
-				errs = append(errs, errors.New(rsp.ErrorCode.Error()))
-			}
-
-			for topic, topicErrors := range rsp.Errors {
-				for partition, partitionError := range topicErrors {
-					if partitionError.errorCode != ErrNoError {
-						errStr := fmt.Sprintf("[%s-%d]: %s", topic, partition, partitionError.errorCode.Error())
-						errs = append(errs, errors.New(errStr))
-					}
-				}
-			}
-		}
-
-		if len(errs) > 0 {
-			return ErrReassignPartitions{MultiError{&errs}}
-		}
-
-		return nil
-	})
-}
-
-func (ca *clusterAdmin) ListPartitionReassignments(topic string, partitions []int32) (topicStatus map[string]map[int32]*PartitionReplicaReassignmentsStatus, err error) {
-	if topic == "" {
-		return nil, ErrInvalidTopic
-	}
-
-	request := &ListPartitionReassignmentsRequest{
-		TimeoutMs: int32(60000),
-		Version:   int16(0),
-	}
-
-	request.AddBlock(topic, partitions)
-
-	b, err := ca.Controller()
-	if err != nil {
-		return nil, err
-	}
-	_ = b.Open(ca.client.Config())
-
-	rsp, err := b.ListPartitionReassignments(request)
-
-	if err == nil && rsp != nil {
-		return rsp.TopicStatus, nil
-	} else {
-		return nil, err
-	}
-}
-
-func (ca *clusterAdmin) DeleteRecords(topic string, partitionOffsets map[int32]int64) error {
-	if topic == "" {
-		return ErrInvalidTopic
-	}
-	partitionPerBroker := make(map[*Broker][]int32)
-	for partition := range partitionOffsets {
-		broker, err := ca.client.Leader(topic, partition)
-		if err != nil {
-			return err
-		}
-		if _, ok := partitionPerBroker[broker]; ok {
-			partitionPerBroker[broker] = append(partitionPerBroker[broker], partition)
-		} else {
-			partitionPerBroker[broker] = []int32{partition}
-		}
-	}
-	errs := make([]error, 0)
-	for broker, partitions := range partitionPerBroker {
-		topics := make(map[string]*DeleteRecordsRequestTopic)
-		recordsToDelete := make(map[int32]int64)
-		for _, p := range partitions {
-			recordsToDelete[p] = partitionOffsets[p]
-		}
-		topics[topic] = &DeleteRecordsRequestTopic{PartitionOffsets: recordsToDelete}
-		request := &DeleteRecordsRequest{
-			Topics:  topics,
-			Timeout: ca.conf.Admin.Timeout,
-		}
-
-		rsp, err := broker.DeleteRecords(request)
-		if err != nil {
-			errs = append(errs, err)
-		} else {
-			deleteRecordsResponseTopic, ok := rsp.Topics[topic]
-			if !ok {
-				errs = append(errs, ErrIncompleteResponse)
-			} else {
-				for _, deleteRecordsResponsePartition := range deleteRecordsResponseTopic.Partitions {
-					if deleteRecordsResponsePartition.Err != ErrNoError {
-						errs = append(errs, errors.New(deleteRecordsResponsePartition.Err.Error()))
-					}
-				}
-			}
-		}
-	}
-	if len(errs) > 0 {
-		return ErrDeleteRecords{MultiError{&errs}}
-	}
-	//todo since we are dealing with couple of partitions it would be good if we return slice of errors
-	//for each partition instead of one error
-	return nil
-}
-
-// Returns a bool indicating whether the resource request needs to go to a
-// specific broker
-func dependsOnSpecificNode(resource ConfigResource) bool {
-	return (resource.Type == BrokerResource && resource.Name != "") ||
-		resource.Type == BrokerLoggerResource
-}
-
-func (ca *clusterAdmin) DescribeConfig(resource ConfigResource) ([]ConfigEntry, error) {
-	var entries []ConfigEntry
-	var resources []*ConfigResource
-	resources = append(resources, &resource)
-
-	request := &DescribeConfigsRequest{
-		Resources: resources,
-	}
-
-	if ca.conf.Version.IsAtLeast(V1_1_0_0) {
-		request.Version = 1
-	}
-
-	if ca.conf.Version.IsAtLeast(V2_0_0_0) {
-		request.Version = 2
-	}
-
-	var (
-		b   *Broker
-		err error
-	)
-
-	// DescribeConfig of broker/broker logger must be sent to the broker in question
-	if dependsOnSpecificNode(resource) {
-		id, _ := strconv.Atoi(resource.Name)
-		b, err = ca.findBroker(int32(id))
-	} else {
-		b, err = ca.findAnyBroker()
-	}
-	if err != nil {
-		return nil, err
-	}
-
-	_ = b.Open(ca.client.Config())
-	rsp, err := b.DescribeConfigs(request)
-	if err != nil {
-		return nil, err
-	}
-
-	for _, rspResource := range rsp.Resources {
-		if rspResource.Name == resource.Name {
-			if rspResource.ErrorMsg != "" {
-				return nil, errors.New(rspResource.ErrorMsg)
-			}
-			if rspResource.ErrorCode != 0 {
-				return nil, KError(rspResource.ErrorCode)
-			}
-			for _, cfgEntry := range rspResource.Configs {
-				entries = append(entries, *cfgEntry)
-			}
-		}
-	}
-	return entries, nil
-}
-
-func (ca *clusterAdmin) AlterConfig(resourceType ConfigResourceType, name string, entries map[string]*string, validateOnly bool) error {
-	var resources []*AlterConfigsResource
-	resources = append(resources, &AlterConfigsResource{
-		Type:          resourceType,
-		Name:          name,
-		ConfigEntries: entries,
-	})
-
-	request := &AlterConfigsRequest{
-		Resources:    resources,
-		ValidateOnly: validateOnly,
-	}
-
-	var (
-		b   *Broker
-		err error
-	)
-
-	// AlterConfig of broker/broker logger must be sent to the broker in question
-	if dependsOnSpecificNode(ConfigResource{Name: name, Type: resourceType}) {
-		id, _ := strconv.Atoi(name)
-		b, err = ca.findBroker(int32(id))
-	} else {
-		b, err = ca.findAnyBroker()
-	}
-	if err != nil {
-		return err
-	}
-
-	_ = b.Open(ca.client.Config())
-	rsp, err := b.AlterConfigs(request)
-	if err != nil {
-		return err
-	}
-
-	for _, rspResource := range rsp.Resources {
-		if rspResource.Name == name {
-			if rspResource.ErrorMsg != "" {
-				return errors.New(rspResource.ErrorMsg)
-			}
-			if rspResource.ErrorCode != 0 {
-				return KError(rspResource.ErrorCode)
-			}
-		}
-	}
-	return nil
-}
-
-func (ca *clusterAdmin) CreateACL(resource Resource, acl Acl) error {
-	var acls []*AclCreation
-	acls = append(acls, &AclCreation{resource, acl})
-	request := &CreateAclsRequest{AclCreations: acls}
-
-	if ca.conf.Version.IsAtLeast(V2_0_0_0) {
-		request.Version = 1
-	}
-
-	b, err := ca.Controller()
-	if err != nil {
-		return err
-	}
-
-	_, err = b.CreateAcls(request)
-	return err
-}
-
-func (ca *clusterAdmin) ListAcls(filter AclFilter) ([]ResourceAcls, error) {
-	request := &DescribeAclsRequest{AclFilter: filter}
-
-	if ca.conf.Version.IsAtLeast(V2_0_0_0) {
-		request.Version = 1
-	}
-
-	b, err := ca.Controller()
-	if err != nil {
-		return nil, err
-	}
-
-	rsp, err := b.DescribeAcls(request)
-	if err != nil {
-		return nil, err
-	}
-
-	var lAcls []ResourceAcls
-	for _, rAcl := range rsp.ResourceAcls {
-		lAcls = append(lAcls, *rAcl)
-	}
-	return lAcls, nil
-}
-
-func (ca *clusterAdmin) DeleteACL(filter AclFilter, validateOnly bool) ([]MatchingAcl, error) {
-	var filters []*AclFilter
-	filters = append(filters, &filter)
-	request := &DeleteAclsRequest{Filters: filters}
-
-	if ca.conf.Version.IsAtLeast(V2_0_0_0) {
-		request.Version = 1
-	}
-
-	b, err := ca.Controller()
-	if err != nil {
-		return nil, err
-	}
-
-	rsp, err := b.DeleteAcls(request)
-	if err != nil {
-		return nil, err
-	}
-
-	var mAcls []MatchingAcl
-	for _, fr := range rsp.FilterResponses {
-		for _, mACL := range fr.MatchingAcls {
-			mAcls = append(mAcls, *mACL)
-		}
-	}
-	return mAcls, nil
-}
-
-func (ca *clusterAdmin) DescribeConsumerGroups(groups []string) (result []*GroupDescription, err error) {
-	groupsPerBroker := make(map[*Broker][]string)
-
-	for _, group := range groups {
-		controller, err := ca.client.Coordinator(group)
-		if err != nil {
-			return nil, err
-		}
-		groupsPerBroker[controller] = append(groupsPerBroker[controller], group)
-	}
-
-	for broker, brokerGroups := range groupsPerBroker {
-		response, err := broker.DescribeGroups(&DescribeGroupsRequest{
-			Groups: brokerGroups,
-		})
-		if err != nil {
-			return nil, err
-		}
-
-		result = append(result, response.Groups...)
-	}
-	return result, nil
-}
-
-func (ca *clusterAdmin) ListConsumerGroups() (allGroups map[string]string, err error) {
-	allGroups = make(map[string]string)
-
-	// Query brokers in parallel, since we have to query *all* brokers
-	brokers := ca.client.Brokers()
-	groupMaps := make(chan map[string]string, len(brokers))
-	errors := make(chan error, len(brokers))
-	wg := sync.WaitGroup{}
-
-	for _, b := range brokers {
-		wg.Add(1)
-		go func(b *Broker, conf *Config) {
-			defer wg.Done()
-			_ = b.Open(conf) // Ensure that broker is opened
-
-			response, err := b.ListGroups(&ListGroupsRequest{})
-			if err != nil {
-				errors <- err
-				return
-			}
-
-			groups := make(map[string]string)
-			for group, typ := range response.Groups {
-				groups[group] = typ
-			}
-
-			groupMaps <- groups
-		}(b, ca.conf)
-	}
-
-	wg.Wait()
-	close(groupMaps)
-	close(errors)
-
-	for groupMap := range groupMaps {
-		for group, protocolType := range groupMap {
-			allGroups[group] = protocolType
-		}
-	}
-
-	// Intentionally return only the first error for simplicity
-	err = <-errors
-	return
-}
-
-func (ca *clusterAdmin) ListConsumerGroupOffsets(group string, topicPartitions map[string][]int32) (*OffsetFetchResponse, error) {
-	coordinator, err := ca.client.Coordinator(group)
-	if err != nil {
-		return nil, err
-	}
-
-	request := &OffsetFetchRequest{
-		ConsumerGroup: group,
-		partitions:    topicPartitions,
-	}
-
-	if ca.conf.Version.IsAtLeast(V0_10_2_0) {
-		request.Version = 2
-	} else if ca.conf.Version.IsAtLeast(V0_8_2_2) {
-		request.Version = 1
-	}
-
-	return coordinator.FetchOffset(request)
-}
-
-func (ca *clusterAdmin) DeleteConsumerGroup(group string) error {
-	coordinator, err := ca.client.Coordinator(group)
-	if err != nil {
-		return err
-	}
-
-	request := &DeleteGroupsRequest{
-		Groups: []string{group},
-	}
-
-	resp, err := coordinator.DeleteGroups(request)
-	if err != nil {
-		return err
-	}
-
-	groupErr, ok := resp.GroupErrorCodes[group]
-	if !ok {
-		return ErrIncompleteResponse
-	}
-
-	if groupErr != ErrNoError {
-		return groupErr
-	}
-
-	return nil
-}
-
-func (ca *clusterAdmin) DescribeLogDirs(brokerIds []int32) (allLogDirs map[int32][]DescribeLogDirsResponseDirMetadata, err error) {
-	allLogDirs = make(map[int32][]DescribeLogDirsResponseDirMetadata)
-
-	// Query brokers in parallel, since we may have to query multiple brokers
-	logDirsMaps := make(chan map[int32][]DescribeLogDirsResponseDirMetadata, len(brokerIds))
-	errors := make(chan error, len(brokerIds))
-	wg := sync.WaitGroup{}
-
-	for _, b := range brokerIds {
-		wg.Add(1)
-		broker, err := ca.findBroker(b)
-		if err != nil {
-			Logger.Printf("Unable to find broker with ID = %v\n", b)
-			continue
-		}
-		go func(b *Broker, conf *Config) {
-			defer wg.Done()
-			_ = b.Open(conf) // Ensure that broker is opened
-
-			response, err := b.DescribeLogDirs(&DescribeLogDirsRequest{})
-			if err != nil {
-				errors <- err
-				return
-			}
-			logDirs := make(map[int32][]DescribeLogDirsResponseDirMetadata)
-			logDirs[b.ID()] = response.LogDirs
-			logDirsMaps <- logDirs
-		}(broker, ca.conf)
-	}
-
-	wg.Wait()
-	close(logDirsMaps)
-	close(errors)
-
-	for logDirsMap := range logDirsMaps {
-		for id, logDirs := range logDirsMap {
-			allLogDirs[id] = logDirs
-		}
-	}
-
-	// Intentionally return only the first error for simplicity
-	err = <-errors
-	return
-}

vendor/github.com/Shopify/sarama/alter_configs_request.go

@@ -1,126 +0,0 @@
-package sarama
-
-//AlterConfigsRequest is an alter config request type
-type AlterConfigsRequest struct {
-	Resources    []*AlterConfigsResource
-	ValidateOnly bool
-}
-
-//AlterConfigsResource is an alter config resource type
-type AlterConfigsResource struct {
-	Type          ConfigResourceType
-	Name          string
-	ConfigEntries map[string]*string
-}
-
-func (a *AlterConfigsRequest) encode(pe packetEncoder) error {
-	if err := pe.putArrayLength(len(a.Resources)); err != nil {
-		return err
-	}
-
-	for _, r := range a.Resources {
-		if err := r.encode(pe); err != nil {
-			return err
-		}
-	}
-
-	pe.putBool(a.ValidateOnly)
-	return nil
-}
-
-func (a *AlterConfigsRequest) decode(pd packetDecoder, version int16) error {
-	resourceCount, err := pd.getArrayLength()
-	if err != nil {
-		return err
-	}
-
-	a.Resources = make([]*AlterConfigsResource, resourceCount)
-	for i := range a.Resources {
-		r := &AlterConfigsResource{}
-		err = r.decode(pd, version)
-		if err != nil {
-			return err
-		}
-		a.Resources[i] = r
-	}
-
-	validateOnly, err := pd.getBool()
-	if err != nil {
-		return err
-	}
-
-	a.ValidateOnly = validateOnly
-
-	return nil
-}
-
-func (a *AlterConfigsResource) encode(pe packetEncoder) error {
-	pe.putInt8(int8(a.Type))
-
-	if err := pe.putString(a.Name); err != nil {
-		return err
-	}
-
-	if err := pe.putArrayLength(len(a.ConfigEntries)); err != nil {
-		return err
-	}
-	for configKey, configValue := range a.ConfigEntries {
-		if err := pe.putString(configKey); err != nil {
-			return err
-		}
-		if err := pe.putNullableString(configValue); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-func (a *AlterConfigsResource) decode(pd packetDecoder, version int16) error {
-	t, err := pd.getInt8()
-	if err != nil {
-		return err
-	}
-	a.Type = ConfigResourceType(t)
-
-	name, err := pd.getString()
-	if err != nil {
-		return err
-	}
-	a.Name = name
-
-	n, err := pd.getArrayLength()
-	if err != nil {
-		return err
-	}
-
-	if n > 0 {
-		a.ConfigEntries = make(map[string]*string, n)
-		for i := 0; i < n; i++ {
-			configKey, err := pd.getString()
-			if err != nil {
-				return err
-			}
-			if a.ConfigEntries[configKey], err = pd.getNullableString(); err != nil {
-				return err
-			}
-		}
-	}
-	return err
-}
-
-func (a *AlterConfigsRequest) key() int16 {
-	return 33
-}
-
-func (a *AlterConfigsRequest) version() int16 {
-	return 0
-}
-
-func (a *AlterConfigsRequest) headerVersion() int16 {
-	return 1
-}
-
-func (a *AlterConfigsRequest) requiredVersion() KafkaVersion {
-	return V0_11_0_0
-}

vendor/github.com/Shopify/sarama/alter_configs_response.go

@@ -1,101 +0,0 @@
-package sarama
-
-import "time"
-
-//AlterConfigsResponse is a response type for alter config
-type AlterConfigsResponse struct {
-	ThrottleTime time.Duration
-	Resources    []*AlterConfigsResourceResponse
-}
-
-//AlterConfigsResourceResponse is a response type for alter config resource
-type AlterConfigsResourceResponse struct {
-	ErrorCode int16
-	ErrorMsg  string
-	Type      ConfigResourceType
-	Name      string
-}
-
-func (a *AlterConfigsResponse) encode(pe packetEncoder) error {
-	pe.putInt32(int32(a.ThrottleTime / time.Millisecond))
-
-	if err := pe.putArrayLength(len(a.Resources)); err != nil {
-		return err
-	}
-
-	for i := range a.Resources {
-		pe.putInt16(a.Resources[i].ErrorCode)
-		err := pe.putString(a.Resources[i].ErrorMsg)
-		if err != nil {
-			return nil
-		}
-		pe.putInt8(int8(a.Resources[i].Type))
-		err = pe.putString(a.Resources[i].Name)
-		if err != nil {
-			return nil
-		}
-	}
-
-	return nil
-}
-
-func (a *AlterConfigsResponse) decode(pd packetDecoder, version int16) error {
-	throttleTime, err := pd.getInt32()
-	if err != nil {
-		return err
-	}
-	a.ThrottleTime = time.Duration(throttleTime) * time.Millisecond
-
-	responseCount, err := pd.getArrayLength()
-	if err != nil {
-		return err
-	}
-
-	a.Resources = make([]*AlterConfigsResourceResponse, responseCount)
-
-	for i := range a.Resources {
-		a.Resources[i] = new(AlterConfigsResourceResponse)
-
-		errCode, err := pd.getInt16()
-		if err != nil {
-			return err
-		}
-		a.Resources[i].ErrorCode = errCode
-
-		e, err := pd.getString()
-		if err != nil {
-			return err
-		}
-		a.Resources[i].ErrorMsg = e
-
-		t, err := pd.getInt8()
-		if err != nil {
-			return err
-		}
-		a.Resources[i].Type = ConfigResourceType(t)
-
-		name, err := pd.getString()
-		if err != nil {
-			return err
-		}
-		a.Resources[i].Name = name
-	}
-
-	return nil
-}
-
-func (a *AlterConfigsResponse) key() int16 {
-	return 32
-}
-
-func (a *AlterConfigsResponse) version() int16 {
-	return 0
-}
-
-func (a *AlterConfigsResponse) headerVersion() int16 {
-	return 0
-}
-
-func (a *AlterConfigsResponse) requiredVersion() KafkaVersion {
-	return V0_11_0_0
-}

vendor/github.com/Shopify/sarama/alter_partition_reassignments_request.go

@@ -1,130 +0,0 @@
-package sarama
-
-type alterPartitionReassignmentsBlock struct {
-	replicas []int32
-}
-
-func (b *alterPartitionReassignmentsBlock) encode(pe packetEncoder) error {
-	if err := pe.putNullableCompactInt32Array(b.replicas); err != nil {
-		return err
-	}
-
-	pe.putEmptyTaggedFieldArray()
-	return nil
-}
-
-func (b *alterPartitionReassignmentsBlock) decode(pd packetDecoder) (err error) {
-	if b.replicas, err = pd.getCompactInt32Array(); err != nil {
-		return err
-	}
-	return nil
-}
-
-type AlterPartitionReassignmentsRequest struct {
-	TimeoutMs int32
-	blocks    map[string]map[int32]*alterPartitionReassignmentsBlock
-	Version   int16
-}
-
-func (r *AlterPartitionReassignmentsRequest) encode(pe packetEncoder) error {
-	pe.putInt32(r.TimeoutMs)
-
-	pe.putCompactArrayLength(len(r.blocks))
-
-	for topic, partitions := range r.blocks {
-		if err := pe.putCompactString(topic); err != nil {
-			return err
-		}
-		pe.putCompactArrayLength(len(partitions))
-		for partition, block := range partitions {
-			pe.putInt32(partition)
-			if err := block.encode(pe); err != nil {
-				return err
-			}
-		}
-		pe.putEmptyTaggedFieldArray()
-	}
-
-	pe.putEmptyTaggedFieldArray()
-
-	return nil
-}
-
-func (r *AlterPartitionReassignmentsRequest) decode(pd packetDecoder, version int16) (err error) {
-	r.Version = version
-
-	if r.TimeoutMs, err = pd.getInt32(); err != nil {
-		return err
-	}
-
-	topicCount, err := pd.getCompactArrayLength()
-	if err != nil {
-		return err
-	}
-	if topicCount > 0 {
-		r.blocks = make(map[string]map[int32]*alterPartitionReassignmentsBlock)
-		for i := 0; i < topicCount; i++ {
-			topic, err := pd.getCompactString()
-			if err != nil {
-				return err
-			}
-			partitionCount, err := pd.getCompactArrayLength()
-			if err != nil {
-				return err
-			}
-			r.blocks[topic] = make(map[int32]*alterPartitionReassignmentsBlock)
-			for j := 0; j < partitionCount; j++ {
-				partition, err := pd.getInt32()
-				if err != nil {
-					return err
-				}
-				block := &alterPartitionReassignmentsBlock{}
-				if err := block.decode(pd); err != nil {
-					return err
-				}
-				r.blocks[topic][partition] = block
-
-				if _, err := pd.getEmptyTaggedFieldArray(); err != nil {
-					return err
-				}
-			}
-			if _, err := pd.getEmptyTaggedFieldArray(); err != nil {
-				return err
-			}
-		}
-	}
-
-	if _, err := pd.getEmptyTaggedFieldArray(); err != nil {
-		return err
-	}
-
-	return
-}
-
-func (r *AlterPartitionReassignmentsRequest) key() int16 {
-	return 45
-}
-
-func (r *AlterPartitionReassignmentsRequest) version() int16 {
-	return r.Version
-}
-
-func (r *AlterPartitionReassignmentsRequest) headerVersion() int16 {
-	return 2
-}
-
-func (r *AlterPartitionReassignmentsRequest) requiredVersion() KafkaVersion {
-	return V2_4_0_0
-}
-
-func (r *AlterPartitionReassignmentsRequest) AddBlock(topic string, partitionID int32, replicas []int32) {
-	if r.blocks == nil {
-		r.blocks = make(map[string]map[int32]*alterPartitionReassignmentsBlock)
-	}
-
-	if r.blocks[topic] == nil {
-		r.blocks[topic] = make(map[int32]*alterPartitionReassignmentsBlock)
-	}
-
-	r.blocks[topic][partitionID] = &alterPartitionReassignmentsBlock{replicas}
-}

vendor/github.com/Shopify/sarama/alter_partition_reassignments_response.go

@@ -1,157 +0,0 @@
-package sarama
-
-type alterPartitionReassignmentsErrorBlock struct {
-	errorCode    KError
-	errorMessage *string
-}
-
-func (b *alterPartitionReassignmentsErrorBlock) encode(pe packetEncoder) error {
-	pe.putInt16(int16(b.errorCode))
-	if err := pe.putNullableCompactString(b.errorMessage); err != nil {
-		return err
-	}
-	pe.putEmptyTaggedFieldArray()
-
-	return nil
-}
-
-func (b *alterPartitionReassignmentsErrorBlock) decode(pd packetDecoder) (err error) {
-	errorCode, err := pd.getInt16()
-	if err != nil {
-		return err
-	}
-	b.errorCode = KError(errorCode)
-	b.errorMessage, err = pd.getCompactNullableString()
-
-	if _, err := pd.getEmptyTaggedFieldArray(); err != nil {
-		return err
-	}
-	return err
-}
-
-type AlterPartitionReassignmentsResponse struct {
-	Version        int16
-	ThrottleTimeMs int32
-	ErrorCode      KError
-	ErrorMessage   *string
-	Errors         map[string]map[int32]*alterPartitionReassignmentsErrorBlock
-}
-
-func (r *AlterPartitionReassignmentsResponse) AddError(topic string, partition int32, kerror KError, message *string) {
-	if r.Errors == nil {
-		r.Errors = make(map[string]map[int32]*alterPartitionReassignmentsErrorBlock)
-	}
-	partitions := r.Errors[topic]
-	if partitions == nil {
-		partitions = make(map[int32]*alterPartitionReassignmentsErrorBlock)
-		r.Errors[topic] = partitions
-	}
-
-	partitions[partition] = &alterPartitionReassignmentsErrorBlock{errorCode: kerror, errorMessage: message}
-}
-
-func (r *AlterPartitionReassignmentsResponse) encode(pe packetEncoder) error {
-	pe.putInt32(r.ThrottleTimeMs)
-	pe.putInt16(int16(r.ErrorCode))
-	if err := pe.putNullableCompactString(r.ErrorMessage); err != nil {
-		return err
-	}
-
-	pe.putCompactArrayLength(len(r.Errors))
-	for topic, partitions := range r.Errors {
-		if err := pe.putCompactString(topic); err != nil {
-			return err
-		}
-		pe.putCompactArrayLength(len(partitions))
-		for partition, block := range partitions {
-			pe.putInt32(partition)
-
-			if err := block.encode(pe); err != nil {
-				return err
-			}
-		}
-		pe.putEmptyTaggedFieldArray()
-	}
-
-	pe.putEmptyTaggedFieldArray()
-	return nil
-}
-
-func (r *AlterPartitionReassignmentsResponse) decode(pd packetDecoder, version int16) (err error) {
-	r.Version = version
-
-	if r.ThrottleTimeMs, err = pd.getInt32(); err != nil {
-		return err
-	}
-
-	kerr, err := pd.getInt16()
-	if err != nil {
-		return err
-	}
-
-	r.ErrorCode = KError(kerr)
-
-	if r.ErrorMessage, err = pd.getCompactNullableString(); err != nil {
-		return err
-	}
-
-	numTopics, err := pd.getCompactArrayLength()
-	if err != nil {
-		return err
-	}
-
-	if numTopics > 0 {
-		r.Errors = make(map[string]map[int32]*alterPartitionReassignmentsErrorBlock, numTopics)
-		for i := 0; i < numTopics; i++ {
-			topic, err := pd.getCompactString()
-			if err != nil {
-				return err
-			}
-
-			ongoingPartitionReassignments, err := pd.getCompactArrayLength()
-			if err != nil {
-				return err
-			}
-
-			r.Errors[topic] = make(map[int32]*alterPartitionReassignmentsErrorBlock, ongoingPartitionReassignments)
-
-			for j := 0; j < ongoingPartitionReassignments; j++ {
-				partition, err := pd.getInt32()
-				if err != nil {
-					return err
-				}
-				block := &alterPartitionReassignmentsErrorBlock{}
-				if err := block.decode(pd); err != nil {
-					return err
-				}
-
-				r.Errors[topic][partition] = block
-			}
-			if _, err = pd.getEmptyTaggedFieldArray(); err != nil {
-				return err
-			}
-		}
-	}
-
-	if _, err = pd.getEmptyTaggedFieldArray(); err != nil {
-		return err
-	}
-
-	return nil
-}
-
-func (r *AlterPartitionReassignmentsResponse) key() int16 {
-	return 45
-}
-
-func (r *AlterPartitionReassignmentsResponse) version() int16 {
-	return r.Version
-}
-
-func (r *AlterPartitionReassignmentsResponse) headerVersion() int16 {
-	return 1
-}
-
-func (r *AlterPartitionReassignmentsResponse) requiredVersion() KafkaVersion {
-	return V2_4_0_0
-}

vendor/github.com/Shopify/sarama/api_versions_request.go

@@ -1,29 +0,0 @@
-package sarama
-
-//ApiVersionsRequest ...
-type ApiVersionsRequest struct {
-}
-
-func (a *ApiVersionsRequest) encode(pe packetEncoder) error {
-	return nil
-}
-
-func (a *ApiVersionsRequest) decode(pd packetDecoder, version int16) (err error) {
-	return nil
-}
-
-func (a *ApiVersionsRequest) key() int16 {
-	return 18
-}
-
-func (a *ApiVersionsRequest) version() int16 {
-	return 0
-}
-
-func (a *ApiVersionsRequest) headerVersion() int16 {
-	return 1
-}
-
-func (a *ApiVersionsRequest) requiredVersion() KafkaVersion {
-	return V0_10_0_0
-}

vendor/github.com/Shopify/sarama/api_versions_response.go

@@ -1,93 +0,0 @@
-package sarama
-
-//ApiVersionsResponseBlock is an api version response block type
-type ApiVersionsResponseBlock struct {
-	ApiKey     int16
-	MinVersion int16
-	MaxVersion int16
-}
-
-func (b *ApiVersionsResponseBlock) encode(pe packetEncoder) error {
-	pe.putInt16(b.ApiKey)
-	pe.putInt16(b.MinVersion)
-	pe.putInt16(b.MaxVersion)
-	return nil
-}
-
-func (b *ApiVersionsResponseBlock) decode(pd packetDecoder) error {
-	var err error
-
-	if b.ApiKey, err = pd.getInt16(); err != nil {
-		return err
-	}
-
-	if b.MinVersion, err = pd.getInt16(); err != nil {
-		return err
-	}
-
-	if b.MaxVersion, err = pd.getInt16(); err != nil {
-		return err
-	}
-
-	return nil
-}
-
-//ApiVersionsResponse is an api version response type
-type ApiVersionsResponse struct {
-	Err         KError
-	ApiVersions []*ApiVersionsResponseBlock
-}
-
-func (r *ApiVersionsResponse) encode(pe packetEncoder) error {
-	pe.putInt16(int16(r.Err))
-	if err := pe.putArrayLength(len(r.ApiVersions)); err != nil {
-		return err
-	}
-	for _, apiVersion := range r.ApiVersions {
-		if err := apiVersion.encode(pe); err != nil {
-			return err
-		}
-	}
-	return nil
-}
-
-func (r *ApiVersionsResponse) decode(pd packetDecoder, version int16) error {
-	kerr, err := pd.getInt16()
-	if err != nil {
-		return err
-	}
-
-	r.Err = KError(kerr)
-
-	numBlocks, err := pd.getArrayLength()
-	if err != nil {
-		return err
-	}
-
-	r.ApiVersions = make([]*ApiVersionsResponseBlock, numBlocks)
-	for i := 0; i < numBlocks; i++ {
-		block := new(ApiVersionsResponseBlock)
-		if err := block.decode(pd); err != nil {
-			return err
-		}
-		r.ApiVersions[i] = block
-	}
-
-	return nil
-}
-
-func (r *ApiVersionsResponse) key() int16 {
-	return 18
-}
-
-func (r *ApiVersionsResponse) version() int16 {
-	return 0
-}
-
-func (a *ApiVersionsResponse) headerVersion() int16 {
-	return 0
-}
-
-func (r *ApiVersionsResponse) requiredVersion() KafkaVersion {
-	return V0_10_0_0
-}

vendor/github.com/Shopify/sarama/compress.go

@@ -1,194 +0,0 @@
-package sarama
-
-import (
-	"bytes"
-	"compress/gzip"
-	"fmt"
-	"sync"
-
-	snappy "github.com/eapache/go-xerial-snappy"
-	"github.com/pierrec/lz4"
-)
-
-var (
-	lz4WriterPool = sync.Pool{
-		New: func() interface{} {
-			return lz4.NewWriter(nil)
-		},
-	}
-
-	gzipWriterPool = sync.Pool{
-		New: func() interface{} {
-			return gzip.NewWriter(nil)
-		},
-	}
-	gzipWriterPoolForCompressionLevel1 = sync.Pool{
-		New: func() interface{} {
-			gz, err := gzip.NewWriterLevel(nil, 1)
-			if err != nil {
-				panic(err)
-			}
-			return gz
-		},
-	}
-	gzipWriterPoolForCompressionLevel2 = sync.Pool{
-		New: func() interface{} {
-			gz, err := gzip.NewWriterLevel(nil, 2)
-			if err != nil {
-				panic(err)
-			}
-			return gz
-		},
-	}
-	gzipWriterPoolForCompressionLevel3 = sync.Pool{
-		New: func() interface{} {
-			gz, err := gzip.NewWriterLevel(nil, 3)
-			if err != nil {
-				panic(err)
-			}
-			return gz
-		},
-	}
-	gzipWriterPoolForCompressionLevel4 = sync.Pool{
-		New: func() interface{} {
-			gz, err := gzip.NewWriterLevel(nil, 4)
-			if err != nil {
-				panic(err)
-			}
-			return gz
-		},
-	}
-	gzipWriterPoolForCompressionLevel5 = sync.Pool{
-		New: func() interface{} {
-			gz, err := gzip.NewWriterLevel(nil, 5)
-			if err != nil {
-				panic(err)
-			}
-			return gz
-		},
-	}
-	gzipWriterPoolForCompressionLevel6 = sync.Pool{
-		New: func() interface{} {
-			gz, err := gzip.NewWriterLevel(nil, 6)
-			if err != nil {
-				panic(err)
-			}
-			return gz
-		},
-	}
-	gzipWriterPoolForCompressionLevel7 = sync.Pool{
-		New: func() interface{} {
-			gz, err := gzip.NewWriterLevel(nil, 7)
-			if err != nil {
-				panic(err)
-			}
-			return gz
-		},
-	}
-	gzipWriterPoolForCompressionLevel8 = sync.Pool{
-		New: func() interface{} {
-			gz, err := gzip.NewWriterLevel(nil, 8)
-			if err != nil {
-				panic(err)
-			}
-			return gz
-		},
-	}
-	gzipWriterPoolForCompressionLevel9 = sync.Pool{
-		New: func() interface{} {
-			gz, err := gzip.NewWriterLevel(nil, 9)
-			if err != nil {
-				panic(err)
-			}
-			return gz
-		},
-	}
-)
-
-func compress(cc CompressionCodec, level int, data []byte) ([]byte, error) {
-	switch cc {
-	case CompressionNone:
-		return data, nil
-	case CompressionGZIP:
-		var (
-			err    error
-			buf    bytes.Buffer
-			writer *gzip.Writer
-		)
-
-		switch level {
-		case CompressionLevelDefault:
-			writer = gzipWriterPool.Get().(*gzip.Writer)
-			defer gzipWriterPool.Put(writer)
-			writer.Reset(&buf)
-		case 1:
-			writer = gzipWriterPoolForCompressionLevel1.Get().(*gzip.Writer)
-			defer gzipWriterPoolForCompressionLevel1.Put(writer)
-			writer.Reset(&buf)
-		case 2:
-			writer = gzipWriterPoolForCompressionLevel2.Get().(*gzip.Writer)
-			defer gzipWriterPoolForCompressionLevel2.Put(writer)
-			writer.Reset(&buf)
-		case 3:
-			writer = gzipWriterPoolForCompressionLevel3.Get().(*gzip.Writer)
-			defer gzipWriterPoolForCompressionLevel3.Put(writer)
-			writer.Reset(&buf)
-		case 4:
-			writer = gzipWriterPoolForCompressionLevel4.Get().(*gzip.Writer)
-			defer gzipWriterPoolForCompressionLevel4.Put(writer)
-			writer.Reset(&buf)
-		case 5:
-			writer = gzipWriterPoolForCompressionLevel5.Get().(*gzip.Writer)
-			defer gzipWriterPoolForCompressionLevel5.Put(writer)
-			writer.Reset(&buf)
-		case 6:
-			writer = gzipWriterPoolForCompressionLevel6.Get().(*gzip.Writer)
-			defer gzipWriterPoolForCompressionLevel6.Put(writer)
-			writer.Reset(&buf)
-		case 7:
-			writer = gzipWriterPoolForCompressionLevel7.Get().(*gzip.Writer)
-			defer gzipWriterPoolForCompressionLevel7.Put(writer)
-			writer.Reset(&buf)
-		case 8:
-			writer = gzipWriterPoolForCompressionLevel8.Get().(*gzip.Writer)
-			defer gzipWriterPoolForCompressionLevel8.Put(writer)
-			writer.Reset(&buf)
-		case 9:
-			writer = gzipWriterPoolForCompressionLevel9.Get().(*gzip.Writer)
-			defer gzipWriterPoolForCompressionLevel9.Put(writer)
-			writer.Reset(&buf)
-		default:
-			writer, err = gzip.NewWriterLevel(&buf, level)
-			if err != nil {
-				return nil, err
-			}
-		}
-		if _, err := writer.Write(data); err != nil {
-			return nil, err
-		}
-		if err := writer.Close(); err != nil {
-			return nil, err
-		}
-		return buf.Bytes(), nil
-	case CompressionSnappy:
-		return snappy.Encode(data), nil
-	case CompressionLZ4:
-		writer := lz4WriterPool.Get().(*lz4.Writer)
-		defer lz4WriterPool.Put(writer)
-
-		var buf bytes.Buffer
-		writer.Reset(&buf)
-
-		if _, err := writer.Write(data); err != nil {
-			return nil, err
-		}
-		if err := writer.Close(); err != nil {
-			return nil, err
-		}
-		return buf.Bytes(), nil
-	case CompressionZSTD:
-		return zstdCompress(nil, data)
-	default:
-		return nil, PacketEncodingError{fmt.Sprintf("unsupported compression codec (%d)", cc)}
-	}
-}

vendor/github.com/Shopify/sarama/config.go

@@ -1,749 +0,0 @@
-package sarama
-
-import (
-	"compress/gzip"
-	"crypto/tls"
-	"fmt"
-	"io/ioutil"
-	"net"
-	"regexp"
-	"time"
-
-	"github.com/rcrowley/go-metrics"
-	"golang.org/x/net/proxy"
-)
-
-const defaultClientID = "sarama"
-
-var validID = regexp.MustCompile(`\A[A-Za-z0-9._-]+\z`)
-
-// Config is used to pass multiple configuration options to Sarama's constructors.
-type Config struct {
-	// Admin is the namespace for ClusterAdmin properties used by the administrative Kafka client.
-	Admin struct {
-		Retry struct {
-			// The total number of times to retry sending (retriable) admin requests (default 5).
-			// Similar to the `retries` setting of the JVM AdminClientConfig.
-			Max int
-			// Backoff time between retries of a failed request (default 100ms)
-			Backoff time.Duration
-		}
-		// The maximum duration the administrative Kafka client will wait for ClusterAdmin operations,
-		// including topics, brokers, configurations and ACLs (defaults to 3 seconds).
-		Timeout time.Duration
-	}
-
-	// Net is the namespace for network-level properties used by the Broker, and
-	// shared by the Client/Producer/Consumer.
-	Net struct {
-		// How many outstanding requests a connection is allowed to have before
-		// sending on it blocks (default 5).
-		MaxOpenRequests int
-
-		// All three of the below configurations are similar to the
-		// `socket.timeout.ms` setting in JVM kafka. All of them default
-		// to 30 seconds.
-		DialTimeout  time.Duration // How long to wait for the initial connection.
-		ReadTimeout  time.Duration // How long to wait for a response.
-		WriteTimeout time.Duration // How long to wait for a transmit.
-
-		TLS struct {
-			// Whether or not to use TLS when connecting to the broker
-			// (defaults to false).
-			Enable bool
-			// The TLS configuration to use for secure connections if
-			// enabled (defaults to nil).
-			Config *tls.Config
-		}
-
-		// SASL based authentication with broker. While there are multiple SASL authentication methods
-		// the current implementation is limited to plaintext (SASL/PLAIN) authentication
-		SASL struct {
-			// Whether or not to use SASL authentication when connecting to the broker
-			// (defaults to false).
-			Enable bool
-			// SASLMechanism is the name of the enabled SASL mechanism.
-			// Possible values: OAUTHBEARER, PLAIN (defaults to PLAIN).
-			Mechanism SASLMechanism
-			// Version is the SASL Protocol Version to use
-			// Kafka > 1.x should use V1, except on Azure EventHub which use V0
-			Version int16
-			// Whether or not to send the Kafka SASL handshake first if enabled
-			// (defaults to true). You should only set this to false if you're using
-			// a non-Kafka SASL proxy.
-			Handshake bool
-			// AuthIdentity is an (optional) authorization identity (authzid) to
-			// use for SASL/PLAIN authentication (if different from User) when
-			// an authenticated user is permitted to act as the presented
-			// alternative user. See RFC4616 for details.
-			AuthIdentity string
-			// User is the authentication identity (authcid) to present for
-			// SASL/PLAIN or SASL/SCRAM authentication
-			User string
-			// Password for SASL/PLAIN authentication
-			Password string
-			// authz id used for SASL/SCRAM authentication
-			SCRAMAuthzID string
-			// SCRAMClientGeneratorFunc is a generator of a user provided implementation of a SCRAM
-			// client used to perform the SCRAM exchange with the server.
-			SCRAMClientGeneratorFunc func() SCRAMClient
-			// TokenProvider is a user-defined callback for generating
-			// access tokens for SASL/OAUTHBEARER auth. See the
-			// AccessTokenProvider interface docs for proper implementation
-			// guidelines.
-			TokenProvider AccessTokenProvider
-
-			GSSAPI GSSAPIConfig
-		}
-
-		// KeepAlive specifies the keep-alive period for an active network connection (defaults to 0).
-		// If zero or positive, keep-alives are enabled.
-		// If negative, keep-alives are disabled.
-		KeepAlive time.Duration
-
-		// LocalAddr is the local address to use when dialing an
-		// address. The address must be of a compatible type for the
-		// network being dialed.
-		// If nil, a local address is automatically chosen.
-		LocalAddr net.Addr
-
-		Proxy struct {
-			// Whether or not to use proxy when connecting to the broker
-			// (defaults to false).
-			Enable bool
-			// The proxy dialer to use enabled (defaults to nil).
-			Dialer proxy.Dialer
-		}
-	}
-
-	// Metadata is the namespace for metadata management properties used by the
-	// Client, and shared by the Producer/Consumer.
-	Metadata struct {
-		Retry struct {
-			// The total number of times to retry a metadata request when the
-			// cluster is in the middle of a leader election (default 3).
-			Max int
-			// How long to wait for leader election to occur before retrying
-			// (default 250ms). Similar to the JVM's `retry.backoff.ms`.
-			Backoff time.Duration
-			// Called to compute backoff time dynamically. Useful for implementing
-			// more sophisticated backoff strategies. This takes precedence over
-			// `Backoff` if set.
-			BackoffFunc func(retries, maxRetries int) time.Duration
-		}
-		// How frequently to refresh the cluster metadata in the background.
-		// Defaults to 10 minutes. Set to 0 to disable. Similar to
-		// `topic.metadata.refresh.interval.ms` in the JVM version.
-		RefreshFrequency time.Duration
-
-		// Whether to maintain a full set of metadata for all topics, or just
-		// the minimal set that has been necessary so far. The full set is simpler
-		// and usually more convenient, but can take up a substantial amount of
-		// memory if you have many topics and partitions. Defaults to true.
-		Full bool
-
-		// How long to wait for a successful metadata response.
-		// Disabled by default which means a metadata request against an unreachable
-		// cluster (all brokers are unreachable or unresponsive) can take up to
-		// `Net.[Dial|Read]Timeout * BrokerCount * (Metadata.Retry.Max + 1) + Metadata.Retry.Backoff * Metadata.Retry.Max`
-		// to fail.
-		Timeout time.Duration
-	}
-
-	// Producer is the namespace for configuration related to producing messages,
-	// used by the Producer.
-	Producer struct {
-		// The maximum permitted size of a message (defaults to 1000000). Should be
-		// set equal to or smaller than the broker's `message.max.bytes`.
-		MaxMessageBytes int
-		// The level of acknowledgement reliability needed from the broker (defaults
-		// to WaitForLocal). Equivalent to the `request.required.acks` setting of the
-		// JVM producer.
-		RequiredAcks RequiredAcks
-		// The maximum duration the broker will wait the receipt of the number of
-		// RequiredAcks (defaults to 10 seconds). This is only relevant when
-		// RequiredAcks is set to WaitForAll or a number > 1. Only supports
-		// millisecond resolution, nanoseconds will be truncated. Equivalent to
-		// the JVM producer's `request.timeout.ms` setting.
-		Timeout time.Duration
-		// The type of compression to use on messages (defaults to no compression).
-		// Similar to `compression.codec` setting of the JVM producer.
-		Compression CompressionCodec
-		// The level of compression to use on messages. The meaning depends
-		// on the actual compression type used and defaults to default compression
-		// level for the codec.
-		CompressionLevel int
-		// Generates partitioners for choosing the partition to send messages to
-		// (defaults to hashing the message key). Similar to the `partitioner.class`
-		// setting for the JVM producer.
-		Partitioner PartitionerConstructor
-		// If enabled, the producer will ensure that exactly one copy of each message is
-		// written.
-		Idempotent bool
-
-		// Return specifies what channels will be populated. If they are set to true,
-		// you must read from the respective channels to prevent deadlock. If,
-		// however, this config is used to create a `SyncProducer`, both must be set
-		// to true and you shall not read from the channels since the producer does
-		// this internally.
-		Return struct {
-			// If enabled, successfully delivered messages will be returned on the
-			// Successes channel (default disabled).
-			Successes bool
-
-			// If enabled, messages that failed to deliver will be returned on the
-			// Errors channel, including error (default enabled).
-			Errors bool
-		}
-
-		// The following config options control how often messages are batched up and
-		// sent to the broker. By default, messages are sent as fast as possible, and
-		// all messages received while the current batch is in-flight are placed
-		// into the subsequent batch.
-		Flush struct {
-			// The best-effort number of bytes needed to trigger a flush. Use the
-			// global sarama.MaxRequestSize to set a hard upper limit.
-			Bytes int
-			// The best-effort number of messages needed to trigger a flush. Use
-			// `MaxMessages` to set a hard upper limit.
-			Messages int
-			// The best-effort frequency of flushes. Equivalent to
-			// `queue.buffering.max.ms` setting of JVM producer.
-			Frequency time.Duration
-			// The maximum number of messages the producer will send in a single
-			// broker request. Defaults to 0 for unlimited. Similar to
-			// `queue.buffering.max.messages` in the JVM producer.
-			MaxMessages int
-		}
-
-		Retry struct {
-			// The total number of times to retry sending a message (default 3).
-			// Similar to the `message.send.max.retries` setting of the JVM producer.
-			Max int
-			// How long to wait for the cluster to settle between retries
-			// (default 100ms). Similar to the `retry.backoff.ms` setting of the
-			// JVM producer.
-			Backoff time.Duration
-			// Called to compute backoff time dynamically. Useful for implementing
-			// more sophisticated backoff strategies. This takes precedence over
-			// `Backoff` if set.
-			BackoffFunc func(retries, maxRetries int) time.Duration
-		}
-	}
-
-	// Consumer is the namespace for configuration related to consuming messages,
-	// used by the Consumer.
-	Consumer struct {
-
-		// Group is the namespace for configuring consumer group.
-		Group struct {
-			Session struct {
-				// The timeout used to detect consumer failures when using Kafka's group management facility.
-				// The consumer sends periodic heartbeats to indicate its liveness to the broker.
-				// If no heartbeats are received by the broker before the expiration of this session timeout,
-				// then the broker will remove this consumer from the group and initiate a rebalance.
-				// Note that the value must be in the allowable range as configured in the broker configuration
-				// by `group.min.session.timeout.ms` and `group.max.session.timeout.ms` (default 10s)
-				Timeout time.Duration
-			}
-			Heartbeat struct {
-				// The expected time between heartbeats to the consumer coordinator when using Kafka's group
-				// management facilities. Heartbeats are used to ensure that the consumer's session stays active and
-				// to facilitate rebalancing when new consumers join or leave the group.
-				// The value must be set lower than Consumer.Group.Session.Timeout, but typically should be set no
-				// higher than 1/3 of that value.
-				// It can be adjusted even lower to control the expected time for normal rebalances (default 3s)
-				Interval time.Duration
-			}
-			Rebalance struct {
-				// Strategy for allocating topic partitions to members (default BalanceStrategyRange)
-				Strategy BalanceStrategy
-				// The maximum allowed time for each worker to join the group once a rebalance has begun.
-				// This is basically a limit on the amount of time needed for all tasks to flush any pending
-				// data and commit offsets. If the timeout is exceeded, then the worker will be removed from
-				// the group, which will cause offset commit failures (default 60s).
-				Timeout time.Duration
-
-				Retry struct {
-					// When a new consumer joins a consumer group the set of consumers attempt to "rebalance"
-					// the load to assign partitions to each consumer. If the set of consumers changes while
-					// this assignment is taking place the rebalance will fail and retry. This setting controls
-					// the maximum number of attempts before giving up (default 4).
-					Max int
-					// Backoff time between retries during rebalance (default 2s)
-					Backoff time.Duration
-				}
-			}
-			Member struct {
-				// Custom metadata to include when joining the group. The user data for all joined members
-				// can be retrieved by sending a DescribeGroupRequest to the broker that is the
-				// coordinator for the group.
-				UserData []byte
-			}
-		}
-
-		Retry struct {
-			// How long to wait after a failing to read from a partition before
-			// trying again (default 2s).
-			Backoff time.Duration
-			// Called to compute backoff time dynamically. Useful for implementing
-			// more sophisticated backoff strategies. This takes precedence over
-			// `Backoff` if set.
-			BackoffFunc func(retries int) time.Duration
-		}
-
-		// Fetch is the namespace for controlling how many bytes are retrieved by any
-		// given request.
-		Fetch struct {
-			// The minimum number of message bytes to fetch in a request - the broker
-			// will wait until at least this many are available. The default is 1,
-			// as 0 causes the consumer to spin when no messages are available.
-			// Equivalent to the JVM's `fetch.min.bytes`.
-			Min int32
-			// The default number of message bytes to fetch from the broker in each
-			// request (default 1MB). This should be larger than the majority of
-			// your messages, or else the consumer will spend a lot of time
-			// negotiating sizes and not actually consuming. Similar to the JVM's
-			// `fetch.message.max.bytes`.
-			Default int32
-			// The maximum number of message bytes to fetch from the broker in a
-			// single request. Messages larger than this will return
-			// ErrMessageTooLarge and will not be consumable, so you must be sure
-			// this is at least as large as your largest message. Defaults to 0
-			// (no limit). Similar to the JVM's `fetch.message.max.bytes`. The
-			// global `sarama.MaxResponseSize` still applies.
-			Max int32
-		}
-		// The maximum amount of time the broker will wait for Consumer.Fetch.Min
-		// bytes to become available before it returns fewer than that anyways. The
-		// default is 250ms, since 0 causes the consumer to spin when no events are
-		// available. 100-500ms is a reasonable range for most cases. Kafka only
-		// supports precision up to milliseconds; nanoseconds will be truncated.
-		// Equivalent to the JVM's `fetch.wait.max.ms`.
-		MaxWaitTime time.Duration
-
-		// The maximum amount of time the consumer expects a message takes to
-		// process for the user. If writing to the Messages channel takes longer
-		// than this, that partition will stop fetching more messages until it
-		// can proceed again.
-		// Note that, since the Messages channel is buffered, the actual grace time is
-		// (MaxProcessingTime * ChannelBufferSize). Defaults to 100ms.
-		// If a message is not written to the Messages channel between two ticks
-		// of the expiryTicker then a timeout is detected.
-		// Using a ticker instead of a timer to detect timeouts should typically
-		// result in many fewer calls to Timer functions which may result in a
-		// significant performance improvement if many messages are being sent
-		// and timeouts are infrequent.
-		// The disadvantage of using a ticker instead of a timer is that
-		// timeouts will be less accurate. That is, the effective timeout could
-		// be between `MaxProcessingTime` and `2 * MaxProcessingTime`. For
-		// example, if `MaxProcessingTime` is 100ms then a delay of 180ms
-		// between two messages being sent may not be recognized as a timeout.
-		MaxProcessingTime time.Duration
-
-		// Return specifies what channels will be populated. If they are set to true,
-		// you must read from them to prevent deadlock.
-		Return struct {
-			// If enabled, any errors that occurred while consuming are returned on
-			// the Errors channel (default disabled).
-			Errors bool
-		}
-
-		// Offsets specifies configuration for how and when to commit consumed
-		// offsets. This currently requires the manual use of an OffsetManager
-		// but will eventually be automated.
-		Offsets struct {
-			// Deprecated: CommitInterval exists for historical compatibility
-			// and should not be used. Please use Consumer.Offsets.AutoCommit
-			CommitInterval time.Duration
-
-			// AutoCommit specifies configuration for commit messages automatically.
-			AutoCommit struct {
-				// Whether or not to auto-commit updated offsets back to the broker.
-				// (default enabled).
-				Enable bool
-
-				// How frequently to commit updated offsets. Ineffective unless
-				// auto-commit is enabled (default 1s)
-				Interval time.Duration
-			}
-
-			// The initial offset to use if no offset was previously committed.
-			// Should be OffsetNewest or OffsetOldest. Defaults to OffsetNewest.
-			Initial int64
-
-			// The retention duration for committed offsets. If zero, disabled
-			// (in which case the `offsets.retention.minutes` option on the
-			// broker will be used).  Kafka only supports precision up to
-			// milliseconds; nanoseconds will be truncated. Requires Kafka
-			// broker version 0.9.0 or later.
-			// (default is 0: disabled).
-			Retention time.Duration
-
-			Retry struct {
-				// The total number of times to retry failing commit
-				// requests during OffsetManager shutdown (default 3).
-				Max int
-			}
-		}
-
-		// IsolationLevel support 2 mode:
-		// 	- use `ReadUncommitted` (default) to consume and return all messages in message channel
-		//	- use `ReadCommitted` to hide messages that are part of an aborted transaction
-		IsolationLevel IsolationLevel
-	}
-
-	// A user-provided string sent with every request to the brokers for logging,
-	// debugging, and auditing purposes. Defaults to "sarama", but you should
-	// probably set it to something specific to your application.
-	ClientID string
-	// A rack identifier for this client. This can be any string value which
-	// indicates where this client is physically located.
-	// It corresponds with the broker config 'broker.rack'
-	RackID string
-	// The number of events to buffer in internal and external channels. This
-	// permits the producer and consumer to continue processing some messages
-	// in the background while user code is working, greatly improving throughput.
-	// Defaults to 256.
-	ChannelBufferSize int
-	// The version of Kafka that Sarama will assume it is running against.
-	// Defaults to the oldest supported stable version. Since Kafka provides
-	// backwards-compatibility, setting it to a version older than you have
-	// will not break anything, although it may prevent you from using the
-	// latest features. Setting it to a version greater than you are actually
-	// running may lead to random breakage.
-	Version KafkaVersion
-	// The registry to define metrics into.
-	// Defaults to a local registry.
-	// If you want to disable metrics gathering, set "metrics.UseNilMetrics" to "true"
-	// prior to starting Sarama.
-	// See Examples on how to use the metrics registry
-	MetricRegistry metrics.Registry
-}
-
-// NewConfig returns a new configuration instance with sane defaults.
-func NewConfig() *Config {
-	c := &Config{}
-
-	c.Admin.Retry.Max = 5
-	c.Admin.Retry.Backoff = 100 * time.Millisecond
-	c.Admin.Timeout = 3 * time.Second
-
-	c.Net.MaxOpenRequests = 5
-	c.Net.DialTimeout = 30 * time.Second
-	c.Net.ReadTimeout = 30 * time.Second
-	c.Net.WriteTimeout = 30 * time.Second
-	c.Net.SASL.Handshake = true
-	c.Net.SASL.Version = SASLHandshakeV0
-
-	c.Metadata.Retry.Max = 3
-	c.Metadata.Retry.Backoff = 250 * time.Millisecond
-	c.Metadata.RefreshFrequency = 10 * time.Minute
-	c.Metadata.Full = true
-
-	c.Producer.MaxMessageBytes = 1000000
-	c.Producer.RequiredAcks = WaitForLocal
-	c.Producer.Timeout = 10 * time.Second
-	c.Producer.Partitioner = NewHashPartitioner
-	c.Producer.Retry.Max = 3
-	c.Producer.Retry.Backoff = 100 * time.Millisecond
-	c.Producer.Return.Errors = true
-	c.Producer.CompressionLevel = CompressionLevelDefault
-
-	c.Consumer.Fetch.Min = 1
-	c.Consumer.Fetch.Default = 1024 * 1024
-	c.Consumer.Retry.Backoff = 2 * time.Second
-	c.Consumer.MaxWaitTime = 250 * time.Millisecond
-	c.Consumer.MaxProcessingTime = 100 * time.Millisecond
-	c.Consumer.Return.Errors = false
-	c.Consumer.Offsets.AutoCommit.Enable = true
-	c.Consumer.Offsets.AutoCommit.Interval = 1 * time.Second
-	c.Consumer.Offsets.Initial = OffsetNewest
-	c.Consumer.Offsets.Retry.Max = 3
-
-	c.Consumer.Group.Session.Timeout = 10 * time.Second
-	c.Consumer.Group.Heartbeat.Interval = 3 * time.Second
-	c.Consumer.Group.Rebalance.Strategy = BalanceStrategyRange
-	c.Consumer.Group.Rebalance.Timeout = 60 * time.Second
-	c.Consumer.Group.Rebalance.Retry.Max = 4
-	c.Consumer.Group.Rebalance.Retry.Backoff = 2 * time.Second
-
-	c.ClientID = defaultClientID
-	c.ChannelBufferSize = 256
-	c.Version = MinVersion
-	c.MetricRegistry = metrics.NewRegistry()
-
-	return c
-}
-
-// Validate checks a Config instance. It will return a
-// ConfigurationError if the specified values don't make sense.
-func (c *Config) Validate() error {
-	// some configuration values should be warned on but not fail completely, do those first
-	if !c.Net.TLS.Enable && c.Net.TLS.Config != nil {
-		Logger.Println("Net.TLS is disabled but a non-nil configuration was provided.")
-	}
-	if !c.Net.SASL.Enable {
-		if c.Net.SASL.User != "" {
-			Logger.Println("Net.SASL is disabled but a non-empty username was provided.")
-		}
-		if c.Net.SASL.Password != "" {
-			Logger.Println("Net.SASL is disabled but a non-empty password was provided.")
-		}
-	}
-	if c.Producer.RequiredAcks > 1 {
-		Logger.Println("Producer.RequiredAcks > 1 is deprecated and will raise an exception with kafka >= 0.8.2.0.")
-	}
-	if c.Producer.MaxMessageBytes >= int(MaxRequestSize) {
-		Logger.Println("Producer.MaxMessageBytes must be smaller than MaxRequestSize; it will be ignored.")
-	}
-	if c.Producer.Flush.Bytes >= int(MaxRequestSize) {
-		Logger.Println("Producer.Flush.Bytes must be smaller than MaxRequestSize; it will be ignored.")
-	}
-	if (c.Producer.Flush.Bytes > 0 || c.Producer.Flush.Messages > 0) && c.Producer.Flush.Frequency == 0 {
-		Logger.Println("Producer.Flush: Bytes or Messages are set, but Frequency is not; messages may not get flushed.")
-	}
-	if c.Producer.Timeout%time.Millisecond != 0 {
-		Logger.Println("Producer.Timeout only supports millisecond resolution; nanoseconds will be truncated.")
-	}
-	if c.Consumer.MaxWaitTime < 100*time.Millisecond {
-		Logger.Println("Consumer.MaxWaitTime is very low, which can cause high CPU and network usage. See documentation for details.")
-	}
-	if c.Consumer.MaxWaitTime%time.Millisecond != 0 {
-		Logger.Println("Consumer.MaxWaitTime only supports millisecond precision; nanoseconds will be truncated.")
-	}
-	if c.Consumer.Offsets.Retention%time.Millisecond != 0 {
-		Logger.Println("Consumer.Offsets.Retention only supports millisecond precision; nanoseconds will be truncated.")
-	}
-	if c.Consumer.Group.Session.Timeout%time.Millisecond != 0 {
-		Logger.Println("Consumer.Group.Session.Timeout only supports millisecond precision; nanoseconds will be truncated.")
-	}
-	if c.Consumer.Group.Heartbeat.Interval%time.Millisecond != 0 {
-		Logger.Println("Consumer.Group.Heartbeat.Interval only supports millisecond precision; nanoseconds will be truncated.")
-	}
-	if c.Consumer.Group.Rebalance.Timeout%time.Millisecond != 0 {
-		Logger.Println("Consumer.Group.Rebalance.Timeout only supports millisecond precision; nanoseconds will be truncated.")
-	}
-	if c.ClientID == defaultClientID {
-		Logger.Println("ClientID is the default of 'sarama', you should consider setting it to something application-specific.")
-	}
-
-	// validate Net values
-	switch {
-	case c.Net.MaxOpenRequests <= 0:
-		return ConfigurationError("Net.MaxOpenRequests must be > 0")
-	case c.Net.DialTimeout <= 0:
-		return ConfigurationError("Net.DialTimeout must be > 0")
-	case c.Net.ReadTimeout <= 0:
-		return ConfigurationError("Net.ReadTimeout must be > 0")
-	case c.Net.WriteTimeout <= 0:
-		return ConfigurationError("Net.WriteTimeout must be > 0")
-	case c.Net.SASL.Enable:
-		if c.Net.SASL.Mechanism == "" {
-			c.Net.SASL.Mechanism = SASLTypePlaintext
-		}
-
-		switch c.Net.SASL.Mechanism {
-		case SASLTypePlaintext:
-			if c.Net.SASL.User == "" {
-				return ConfigurationError("Net.SASL.User must not be empty when SASL is enabled")
-			}
-			if c.Net.SASL.Password == "" {
-				return ConfigurationError("Net.SASL.Password must not be empty when SASL is enabled")
-			}
-		case SASLTypeOAuth:
-			if c.Net.SASL.TokenProvider == nil {
-				return ConfigurationError("An AccessTokenProvider instance must be provided to Net.SASL.TokenProvider")
-			}
-		case SASLTypeSCRAMSHA256, SASLTypeSCRAMSHA512:
-			if c.Net.SASL.User == "" {
-				return ConfigurationError("Net.SASL.User must not be empty when SASL is enabled")
-			}
-			if c.Net.SASL.Password == "" {
-				return ConfigurationError("Net.SASL.Password must not be empty when SASL is enabled")
-			}
-			if c.Net.SASL.SCRAMClientGeneratorFunc == nil {
-				return ConfigurationError("A SCRAMClientGeneratorFunc function must be provided to Net.SASL.SCRAMClientGeneratorFunc")
-			}
-		case SASLTypeGSSAPI:
-			if c.Net.SASL.GSSAPI.ServiceName == "" {
-				return ConfigurationError("Net.SASL.GSSAPI.ServiceName must not be empty when GSS-API mechanism is used")
-			}
-
-			if c.Net.SASL.GSSAPI.AuthType == KRB5_USER_AUTH {
-				if c.Net.SASL.GSSAPI.Password == "" {
-					return ConfigurationError("Net.SASL.GSSAPI.Password must not be empty when GSS-API " +
-						"mechanism is used and Net.SASL.GSSAPI.AuthType = KRB5_USER_AUTH")
-				}
-			} else if c.Net.SASL.GSSAPI.AuthType == KRB5_KEYTAB_AUTH {
-				if c.Net.SASL.GSSAPI.KeyTabPath == "" {
-					return ConfigurationError("Net.SASL.GSSAPI.KeyTabPath must not be empty when GSS-API mechanism is used" +
-						" and  Net.SASL.GSSAPI.AuthType = KRB5_KEYTAB_AUTH")
-				}
-			} else {
-				return ConfigurationError("Net.SASL.GSSAPI.AuthType is invalid. Possible values are KRB5_USER_AUTH and KRB5_KEYTAB_AUTH")
-			}
-			if c.Net.SASL.GSSAPI.KerberosConfigPath == "" {
-				return ConfigurationError("Net.SASL.GSSAPI.KerberosConfigPath must not be empty when GSS-API mechanism is used")
-			}
-			if c.Net.SASL.GSSAPI.Username == "" {
-				return ConfigurationError("Net.SASL.GSSAPI.Username must not be empty when GSS-API mechanism is used")
-			}
-			if c.Net.SASL.GSSAPI.Realm == "" {
-				return ConfigurationError("Net.SASL.GSSAPI.Realm must not be empty when GSS-API mechanism is used")
-			}
-		default:
-			msg := fmt.Sprintf("The SASL mechanism configuration is invalid. Possible values are `%s`, `%s`, `%s`, `%s` and `%s`",
-				SASLTypeOAuth, SASLTypePlaintext, SASLTypeSCRAMSHA256, SASLTypeSCRAMSHA512, SASLTypeGSSAPI)
-			return ConfigurationError(msg)
-		}
-	}
-
-	// validate the Admin values
-	switch {
-	case c.Admin.Timeout <= 0:
-		return ConfigurationError("Admin.Timeout must be > 0")
-	}
-
-	// validate the Metadata values
-	switch {
-	case c.Metadata.Retry.Max < 0:
-		return ConfigurationError("Metadata.Retry.Max must be >= 0")
-	case c.Metadata.Retry.Backoff < 0:
-		return ConfigurationError("Metadata.Retry.Backoff must be >= 0")
-	case c.Metadata.RefreshFrequency < 0:
-		return ConfigurationError("Metadata.RefreshFrequency must be >= 0")
-	}
-
-	// validate the Producer values
-	switch {
-	case c.Producer.MaxMessageBytes <= 0:
-		return ConfigurationError("Producer.MaxMessageBytes must be > 0")
-	case c.Producer.RequiredAcks < -1:
-		return ConfigurationError("Producer.RequiredAcks must be >= -1")
-	case c.Producer.Timeout <= 0:
-		return ConfigurationError("Producer.Timeout must be > 0")
-	case c.Producer.Partitioner == nil:
-		return ConfigurationError("Producer.Partitioner must not be nil")
-	case c.Producer.Flush.Bytes < 0:
-		return ConfigurationError("Producer.Flush.Bytes must be >= 0")
-	case c.Producer.Flush.Messages < 0:
-		return ConfigurationError("Producer.Flush.Messages must be >= 0")
-	case c.Producer.Flush.Frequency < 0:
-		return ConfigurationError("Producer.Flush.Frequency must be >= 0")
-	case c.Producer.Flush.MaxMessages < 0:
-		return ConfigurationError("Producer.Flush.MaxMessages must be >= 0")
-	case c.Producer.Flush.MaxMessages > 0 && c.Producer.Flush.MaxMessages < c.Producer.Flush.Messages:
-		return ConfigurationError("Producer.Flush.MaxMessages must be >= Producer.Flush.Messages when set")
-	case c.Producer.Retry.Max < 0:
-		return ConfigurationError("Producer.Retry.Max must be >= 0")
-	case c.Producer.Retry.Backoff < 0:
-		return ConfigurationError("Producer.Retry.Backoff must be >= 0")
-	}
-
-	if c.Producer.Compression == CompressionLZ4 && !c.Version.IsAtLeast(V0_10_0_0) {
-		return ConfigurationError("lz4 compression requires Version >= V0_10_0_0")
-	}
-
-	if c.Producer.Compression == CompressionGZIP {
-		if c.Producer.CompressionLevel != CompressionLevelDefault {
-			if _, err := gzip.NewWriterLevel(ioutil.Discard, c.Producer.CompressionLevel); err != nil {
-				return ConfigurationError(fmt.Sprintf("gzip compression does not work with level %d: %v", c.Producer.CompressionLevel, err))
-			}
-		}
-	}
-
-	if c.Producer.Compression == CompressionZSTD && !c.Version.IsAtLeast(V2_1_0_0) {
-		return ConfigurationError("zstd compression requires Version >= V2_1_0_0")
-	}
-
-	if c.Producer.Idempotent {
-		if !c.Version.IsAtLeast(V0_11_0_0) {
-			return ConfigurationError("Idempotent producer requires Version >= V0_11_0_0")
-		}
-		if c.Producer.Retry.Max == 0 {
-			return ConfigurationError("Idempotent producer requires Producer.Retry.Max >= 1")
-		}
-		if c.Producer.RequiredAcks != WaitForAll {
-			return ConfigurationError("Idempotent producer requires Producer.RequiredAcks to be WaitForAll")
-		}
-		if c.Net.MaxOpenRequests > 1 {
-			return ConfigurationError("Idempotent producer requires Net.MaxOpenRequests to be 1")
-		}
-	}
-
-	// validate the Consumer values
-	switch {
-	case c.Consumer.Fetch.Min <= 0:
-		return ConfigurationError("Consumer.Fetch.Min must be > 0")
-	case c.Consumer.Fetch.Default <= 0:
-		return ConfigurationError("Consumer.Fetch.Default must be > 0")
-	case c.Consumer.Fetch.Max < 0:
-		return ConfigurationError("Consumer.Fetch.Max must be >= 0")
-	case c.Consumer.MaxWaitTime < 1*time.Millisecond:
-		return ConfigurationError("Consumer.MaxWaitTime must be >= 1ms")
-	case c.Consumer.MaxProcessingTime <= 0:
-		return ConfigurationError("Consumer.MaxProcessingTime must be > 0")
-	case c.Consumer.Retry.Backoff < 0:
-		return ConfigurationError("Consumer.Retry.Backoff must be >= 0")
-	case c.Consumer.Offsets.AutoCommit.Interval <= 0:
-		return ConfigurationError("Consumer.Offsets.AutoCommit.Interval must be > 0")
-	case c.Consumer.Offsets.Initial != OffsetOldest && c.Consumer.Offsets.Initial != OffsetNewest:
-		return ConfigurationError("Consumer.Offsets.Initial must be OffsetOldest or OffsetNewest")
-	case c.Consumer.Offsets.Retry.Max < 0:
-		return ConfigurationError("Consumer.Offsets.Retry.Max must be >= 0")
-	case c.Consumer.IsolationLevel != ReadUncommitted && c.Consumer.IsolationLevel != ReadCommitted:
-		return ConfigurationError("Consumer.IsolationLevel must be ReadUncommitted or ReadCommitted")
-	}
-
-	if c.Consumer.Offsets.CommitInterval != 0 {
-		Logger.Println("Deprecation warning: Consumer.Offsets.CommitInterval exists for historical compatibility" +
-			" and should not be used. Please use Consumer.Offsets.AutoCommit, the current value will be ignored")
-	}
-
-	// validate IsolationLevel
-	if c.Consumer.IsolationLevel == ReadCommitted && !c.Version.IsAtLeast(V0_11_0_0) {
-		return ConfigurationError("ReadCommitted requires Version >= V0_11_0_0")
-	}
-
-	// validate the Consumer Group values
-	switch {
-	case c.Consumer.Group.Session.Timeout <= 2*time.Millisecond:
-		return ConfigurationError("Consumer.Group.Session.Timeout must be >= 2ms")
-	case c.Consumer.Group.Heartbeat.Interval < 1*time.Millisecond:
-		return ConfigurationError("Consumer.Group.Heartbeat.Interval must be >= 1ms")
-	case c.Consumer.Group.Heartbeat.Interval >= c.Consumer.Group.Session.Timeout:
-		return ConfigurationError("Consumer.Group.Heartbeat.Interval must be < Consumer.Group.Session.Timeout")
-	case c.Consumer.Group.Rebalance.Strategy == nil:
-		return ConfigurationError("Consumer.Group.Rebalance.Strategy must not be empty")
-	case c.Consumer.Group.Rebalance.Timeout <= time.Millisecond:
-		return ConfigurationError("Consumer.Group.Rebalance.Timeout must be >= 1ms")
-	case c.Consumer.Group.Rebalance.Retry.Max < 0:
-		return ConfigurationError("Consumer.Group.Rebalance.Retry.Max must be >= 0")
-	case c.Consumer.Group.Rebalance.Retry.Backoff < 0:
-		return ConfigurationError("Consumer.Group.Rebalance.Retry.Backoff must be >= 0")
-	}
-
-	// validate misc shared values
-	switch {
-	case c.ChannelBufferSize < 0:
-		return ConfigurationError("ChannelBufferSize must be >= 0")
-	case !validID.MatchString(c.ClientID):
-		return ConfigurationError("ClientID is invalid")
-	}
-
-	return nil
-}
-
-func (c *Config) getDialer() proxy.Dialer {
-	if c.Net.Proxy.Enable {
-		Logger.Printf("using proxy %s", c.Net.Proxy.Dialer)
-		return c.Net.Proxy.Dialer
-	} else {
-		return &net.Dialer{
-			Timeout:   c.Net.DialTimeout,
-			KeepAlive: c.Net.KeepAlive,
-			LocalAddr: c.Net.LocalAddr,
-		}
-	}
-}

vendor/github.com/Shopify/sarama/config_resource_type.go

@@ -1,18 +0,0 @@
-package sarama
-
-// ConfigResourceType is a type for resources that have configs.
-type ConfigResourceType int8
-
-// Taken from:
-// https://github.com/apache/kafka/blob/ed7c071e07f1f90e4c2895582f61ca090ced3c42/clients/src/main/java/org/apache/kafka/common/config/ConfigResource.java#L32-L55
-
-const (
-	// UnknownResource constant type
-	UnknownResource ConfigResourceType = 0
-	// TopicResource constant type
-	TopicResource ConfigResourceType = 2
-	// BrokerResource constant type
-	BrokerResource ConfigResourceType = 4
-	// BrokerLoggerResource constant type
-	BrokerLoggerResource ConfigResourceType = 8
-)

vendor/github.com/Shopify/sarama/consumer.go

@@ -1,911 +0,0 @@
-package sarama
-
-import (
-	"errors"
-	"fmt"
-	"math"
-	"sync"
-	"sync/atomic"
-	"time"
-
-	"github.com/rcrowley/go-metrics"
-)
-
-// ConsumerMessage encapsulates a Kafka message returned by the consumer.
-type ConsumerMessage struct {
-	Headers        []*RecordHeader // only set if kafka is version 0.11+
-	Timestamp      time.Time       // only set if kafka is version 0.10+, inner message timestamp
-	BlockTimestamp time.Time       // only set if kafka is version 0.10+, outer (compressed) block timestamp
-
-	Key, Value []byte
-	Topic      string
-	Partition  int32
-	Offset     int64
-}
-
-// ConsumerError is what is provided to the user when an error occurs.
-// It wraps an error and includes the topic and partition.
-type ConsumerError struct {
-	Topic     string
-	Partition int32
-	Err       error
-}
-
-func (ce ConsumerError) Error() string {
-	return fmt.Sprintf("kafka: error while consuming %s/%d: %s", ce.Topic, ce.Partition, ce.Err)
-}
-
-// ConsumerErrors is a type that wraps a batch of errors and implements the Error interface.
-// It can be returned from the PartitionConsumer's Close methods to avoid the need to manually drain errors
-// when stopping.
-type ConsumerErrors []*ConsumerError
-
-func (ce ConsumerErrors) Error() string {
-	return fmt.Sprintf("kafka: %d errors while consuming", len(ce))
-}
-
-// Consumer manages PartitionConsumers which process Kafka messages from brokers. You MUST call Close()
-// on a consumer to avoid leaks, it will not be garbage-collected automatically when it passes out of
-// scope.
-type Consumer interface {
-	// Topics returns the set of available topics as retrieved from the cluster
-	// metadata. This method is the same as Client.Topics(), and is provided for
-	// convenience.
-	Topics() ([]string, error)
-
-	// Partitions returns the sorted list of all partition IDs for the given topic.
-	// This method is the same as Client.Partitions(), and is provided for convenience.
-	Partitions(topic string) ([]int32, error)
-
-	// ConsumePartition creates a PartitionConsumer on the given topic/partition with
-	// the given offset. It will return an error if this Consumer is already consuming
-	// on the given topic/partition. Offset can be a literal offset, or OffsetNewest
-	// or OffsetOldest
-	ConsumePartition(topic string, partition int32, offset int64) (PartitionConsumer, error)
-
-	// HighWaterMarks returns the current high water marks for each topic and partition.
-	// Consistency between partitions is not guaranteed since high water marks are updated separately.
-	HighWaterMarks() map[string]map[int32]int64
-
-	// Close shuts down the consumer. It must be called after all child
-	// PartitionConsumers have already been closed.
-	Close() error
-}
-
-type consumer struct {
-	conf            *Config
-	children        map[string]map[int32]*partitionConsumer
-	brokerConsumers map[*Broker]*brokerConsumer
-	client          Client
-	lock            sync.Mutex
-}
-
-// NewConsumer creates a new consumer using the given broker addresses and configuration.
-func NewConsumer(addrs []string, config *Config) (Consumer, error) {
-	client, err := NewClient(addrs, config)
-	if err != nil {
-		return nil, err
-	}
-	return newConsumer(client)
-}
-
-// NewConsumerFromClient creates a new consumer using the given client. It is still
-// necessary to call Close() on the underlying client when shutting down this consumer.
-func NewConsumerFromClient(client Client) (Consumer, error) {
-	// For clients passed in by the client, ensure we don't
-	// call Close() on it.
-	cli := &nopCloserClient{client}
-	return newConsumer(cli)
-}
-
-func newConsumer(client Client) (Consumer, error) {
-	// Check that we are not dealing with a closed Client before processing any other arguments
-	if client.Closed() {
-		return nil, ErrClosedClient
-	}
-
-	c := &consumer{
-		client:          client,
-		conf:            client.Config(),
-		children:        make(map[string]map[int32]*partitionConsumer),
-		brokerConsumers: make(map[*Broker]*brokerConsumer),
-	}
-
-	return c, nil
-}
-
-func (c *consumer) Close() error {
-	return c.client.Close()
-}
-
-func (c *consumer) Topics() ([]string, error) {
-	return c.client.Topics()
-}
-
-func (c *consumer) Partitions(topic string) ([]int32, error) {
-	return c.client.Partitions(topic)
-}
-
-func (c *consumer) ConsumePartition(topic string, partition int32, offset int64) (PartitionConsumer, error) {
-	child := &partitionConsumer{
-		consumer:  c,
-		conf:      c.conf,
-		topic:     topic,
-		partition: partition,
-		messages:  make(chan *ConsumerMessage, c.conf.ChannelBufferSize),
-		errors:    make(chan *ConsumerError, c.conf.ChannelBufferSize),
-		feeder:    make(chan *FetchResponse, 1),
-		trigger:   make(chan none, 1),
-		dying:     make(chan none),
-		fetchSize: c.conf.Consumer.Fetch.Default,
-	}
-
-	if err := child.chooseStartingOffset(offset); err != nil {
-		return nil, err
-	}
-
-	var leader *Broker
-	var err error
-	if leader, err = c.client.Leader(child.topic, child.partition); err != nil {
-		return nil, err
-	}
-
-	if err := c.addChild(child); err != nil {
-		return nil, err
-	}
-
-	go withRecover(child.dispatcher)
-	go withRecover(child.responseFeeder)
-
-	child.broker = c.refBrokerConsumer(leader)
-	child.broker.input <- child
-
-	return child, nil
-}
-
-func (c *consumer) HighWaterMarks() map[string]map[int32]int64 {
-	c.lock.Lock()
-	defer c.lock.Unlock()
-
-	hwms := make(map[string]map[int32]int64)
-	for topic, p := range c.children {
-		hwm := make(map[int32]int64, len(p))
-		for partition, pc := range p {
-			hwm[partition] = pc.HighWaterMarkOffset()
-		}
-		hwms[topic] = hwm
-	}
-
-	return hwms
-}
-
-func (c *consumer) addChild(child *partitionConsumer) error {
-	c.lock.Lock()
-	defer c.lock.Unlock()
-
-	topicChildren := c.children[child.topic]
-	if topicChildren == nil {
-		topicChildren = make(map[int32]*partitionConsumer)
-		c.children[child.topic] = topicChildren
-	}
-
-	if topicChildren[child.partition] != nil {
-		return ConfigurationError("That topic/partition is already being consumed")
-	}
-
-	topicChildren[child.partition] = child
-	return nil
-}
-
-func (c *consumer) removeChild(child *partitionConsumer) {
-	c.lock.Lock()
-	defer c.lock.Unlock()
-
-	delete(c.children[child.topic], child.partition)
-}
-
-func (c *consumer) refBrokerConsumer(broker *Broker) *brokerConsumer {
-	c.lock.Lock()
-	defer c.lock.Unlock()
-
-	bc := c.brokerConsumers[broker]
-	if bc == nil {
-		bc = c.newBrokerConsumer(broker)
-		c.brokerConsumers[broker] = bc
-	}
-
-	bc.refs++
-
-	return bc
-}
-
-func (c *consumer) unrefBrokerConsumer(brokerWorker *brokerConsumer) {
-	c.lock.Lock()
-	defer c.lock.Unlock()
-
-	brokerWorker.refs--
-
-	if brokerWorker.refs == 0 {
-		close(brokerWorker.input)
-		if c.brokerConsumers[brokerWorker.broker] == brokerWorker {
-			delete(c.brokerConsumers, brokerWorker.broker)
-		}
-	}
-}
-
-func (c *consumer) abandonBrokerConsumer(brokerWorker *brokerConsumer) {
-	c.lock.Lock()
-	defer c.lock.Unlock()
-
-	delete(c.brokerConsumers, brokerWorker.broker)
-}
-
-// PartitionConsumer
-
-// PartitionConsumer processes Kafka messages from a given topic and partition. You MUST call one of Close() or
-// AsyncClose() on a PartitionConsumer to avoid leaks; it will not be garbage-collected automatically when it passes out
-// of scope.
-//
-// The simplest way of using a PartitionConsumer is to loop over its Messages channel using a for/range
-// loop. The PartitionConsumer will only stop itself in one case: when the offset being consumed is reported
-// as out of range by the brokers. In this case you should decide what you want to do (try a different offset,
-// notify a human, etc) and handle it appropriately. For all other error cases, it will just keep retrying.
-// By default, it logs these errors to sarama.Logger; if you want to be notified directly of all errors, set
-// your config's Consumer.Return.Errors to true and read from the Errors channel, using a select statement
-// or a separate goroutine. Check out the Consumer examples to see implementations of these different approaches.
-//
-// To terminate such a for/range loop while the loop is executing, call AsyncClose. This will kick off the process of
-// consumer tear-down & return immediately. Continue to loop, servicing the Messages channel until the teardown process
-// AsyncClose initiated closes it (thus terminating the for/range loop). If you've already ceased reading Messages, call
-// Close; this will signal the PartitionConsumer's goroutines to begin shutting down (just like AsyncClose), but will
-// also drain the Messages channel, harvest all errors & return them once cleanup has completed.
-type PartitionConsumer interface {
-	// AsyncClose initiates a shutdown of the PartitionConsumer. This method will return immediately, after which you
-	// should continue to service the 'Messages' and 'Errors' channels until they are empty. It is required to call this
-	// function, or Close before a consumer object passes out of scope, as it will otherwise leak memory. You must call
-	// this before calling Close on the underlying client.
-	AsyncClose()
-
-	// Close stops the PartitionConsumer from fetching messages. It will initiate a shutdown just like AsyncClose, drain
-	// the Messages channel, harvest any errors & return them to the caller. Note that if you are continuing to service
-	// the Messages channel when this function is called, you will be competing with Close for messages; consider
-	// calling AsyncClose, instead. It is required to call this function (or AsyncClose) before a consumer object passes
-	// out of scope, as it will otherwise leak memory. You must call this before calling Close on the underlying client.
-	Close() error
-
-	// Messages returns the read channel for the messages that are returned by
-	// the broker.
-	Messages() <-chan *ConsumerMessage
-
-	// Errors returns a read channel of errors that occurred during consuming, if
-	// enabled. By default, errors are logged and not returned over this channel.
-	// If you want to implement any custom error handling, set your config's
-	// Consumer.Return.Errors setting to true, and read from this channel.
-	Errors() <-chan *ConsumerError
-
-	// HighWaterMarkOffset returns the high water mark offset of the partition,
-	// i.e. the offset that will be used for the next message that will be produced.
-	// You can use this to determine how far behind the processing is.
-	HighWaterMarkOffset() int64
-}
-
-type partitionConsumer struct {
-	highWaterMarkOffset int64 // must be at the top of the struct because https://golang.org/pkg/sync/atomic/#pkg-note-BUG
-
-	consumer *consumer
-	conf     *Config
-	broker   *brokerConsumer
-	messages chan *ConsumerMessage
-	errors   chan *ConsumerError
-	feeder   chan *FetchResponse
-
-	trigger, dying chan none
-	closeOnce      sync.Once
-	topic          string
-	partition      int32
-	responseResult error
-	fetchSize      int32
-	offset         int64
-	retries        int32
-}
-
-var errTimedOut = errors.New("timed out feeding messages to the user") // not user-facing
-
-func (child *partitionConsumer) sendError(err error) {
-	cErr := &ConsumerError{
-		Topic:     child.topic,
-		Partition: child.partition,
-		Err:       err,
-	}
-
-	if child.conf.Consumer.Return.Errors {
-		child.errors <- cErr
-	} else {
-		Logger.Println(cErr)
-	}
-}
-
-func (child *partitionConsumer) computeBackoff() time.Duration {
-	if child.conf.Consumer.Retry.BackoffFunc != nil {
-		retries := atomic.AddInt32(&child.retries, 1)
-		return child.conf.Consumer.Retry.BackoffFunc(int(retries))
-	}
-	return child.conf.Consumer.Retry.Backoff
-}
-
-func (child *partitionConsumer) dispatcher() {
-	for range child.trigger {
-		select {
-		case <-child.dying:
-			close(child.trigger)
-		case <-time.After(child.computeBackoff()):
-			if child.broker != nil {
-				child.consumer.unrefBrokerConsumer(child.broker)
-				child.broker = nil
-			}
-
-			Logger.Printf("consumer/%s/%d finding new broker\n", child.topic, child.partition)
-			if err := child.dispatch(); err != nil {
-				child.sendError(err)
-				child.trigger <- none{}
-			}
-		}
-	}
-
-	if child.broker != nil {
-		child.consumer.unrefBrokerConsumer(child.broker)
-	}
-	child.consumer.removeChild(child)
-	close(child.feeder)
-}
-
-func (child *partitionConsumer) dispatch() error {
-	if err := child.consumer.client.RefreshMetadata(child.topic); err != nil {
-		return err
-	}
-
-	var leader *Broker
-	var err error
-	if leader, err = child.consumer.client.Leader(child.topic, child.partition); err != nil {
-		return err
-	}
-
-	child.broker = child.consumer.refBrokerConsumer(leader)
-
-	child.broker.input <- child
-
-	return nil
-}
-
-func (child *partitionConsumer) chooseStartingOffset(offset int64) error {
-	newestOffset, err := child.consumer.client.GetOffset(child.topic, child.partition, OffsetNewest)
-	if err != nil {
-		return err
-	}
-	oldestOffset, err := child.consumer.client.GetOffset(child.topic, child.partition, OffsetOldest)
-	if err != nil {
-		return err
-	}
-
-	switch {
-	case offset == OffsetNewest:
-		child.offset = newestOffset
-	case offset == OffsetOldest:
-		child.offset = oldestOffset
-	case offset >= oldestOffset && offset <= newestOffset:
-		child.offset = offset
-	default:
-		return ErrOffsetOutOfRange
-	}
-
-	return nil
-}
-
-func (child *partitionConsumer) Messages() <-chan *ConsumerMessage {
-	return child.messages
-}
-
-func (child *partitionConsumer) Errors() <-chan *ConsumerError {
-	return child.errors
-}
-
-func (child *partitionConsumer) AsyncClose() {
-	// this triggers whatever broker owns this child to abandon it and close its trigger channel, which causes
-	// the dispatcher to exit its loop, which removes it from the consumer then closes its 'messages' and
-	// 'errors' channel (alternatively, if the child is already at the dispatcher for some reason, that will
-	// also just close itself)
-	child.closeOnce.Do(func() {
-		close(child.dying)
-	})
-}
-
-func (child *partitionConsumer) Close() error {
-	child.AsyncClose()
-
-	var errors ConsumerErrors
-	for err := range child.errors {
-		errors = append(errors, err)
-	}
-
-	if len(errors) > 0 {
-		return errors
-	}
-	return nil
-}
-
-func (child *partitionConsumer) HighWaterMarkOffset() int64 {
-	return atomic.LoadInt64(&child.highWaterMarkOffset)
-}
-
-func (child *partitionConsumer) responseFeeder() {
-	var msgs []*ConsumerMessage
-	expiryTicker := time.NewTicker(child.conf.Consumer.MaxProcessingTime)
-	firstAttempt := true
-
-feederLoop:
-	for response := range child.feeder {
-		msgs, child.responseResult = child.parseResponse(response)
-
-		if child.responseResult == nil {
-			atomic.StoreInt32(&child.retries, 0)
-		}
-
-		for i, msg := range msgs {
-		messageSelect:
-			select {
-			case <-child.dying:
-				child.broker.acks.Done()
-				continue feederLoop
-			case child.messages <- msg:
-				firstAttempt = true
-			case <-expiryTicker.C:
-				if !firstAttempt {
-					child.responseResult = errTimedOut
-					child.broker.acks.Done()
-				remainingLoop:
-					for _, msg = range msgs[i:] {
-						select {
-						case child.messages <- msg:
-						case <-child.dying:
-							break remainingLoop
-						}
-					}
-					child.broker.input <- child
-					continue feederLoop
-				} else {
-					// current message has not been sent, return to select
-					// statement
-					firstAttempt = false
-					goto messageSelect
-				}
-			}
-		}
-
-		child.broker.acks.Done()
-	}
-
-	expiryTicker.Stop()
-	close(child.messages)
-	close(child.errors)
-}
-
-func (child *partitionConsumer) parseMessages(msgSet *MessageSet) ([]*ConsumerMessage, error) {
-	var messages []*ConsumerMessage
-	for _, msgBlock := range msgSet.Messages {
-		for _, msg := range msgBlock.Messages() {
-			offset := msg.Offset
-			timestamp := msg.Msg.Timestamp
-			if msg.Msg.Version >= 1 {
-				baseOffset := msgBlock.Offset - msgBlock.Messages()[len(msgBlock.Messages())-1].Offset
-				offset += baseOffset
-				if msg.Msg.LogAppendTime {
-					timestamp = msgBlock.Msg.Timestamp
-				}
-			}
-			if offset < child.offset {
-				continue
-			}
-			messages = append(messages, &ConsumerMessage{
-				Topic:          child.topic,
-				Partition:      child.partition,
-				Key:            msg.Msg.Key,
-				Value:          msg.Msg.Value,
-				Offset:         offset,
-				Timestamp:      timestamp,
-				BlockTimestamp: msgBlock.Msg.Timestamp,
-			})
-			child.offset = offset + 1
-		}
-	}
-	if len(messages) == 0 {
-		child.offset++
-	}
-	return messages, nil
-}
-
-func (child *partitionConsumer) parseRecords(batch *RecordBatch) ([]*ConsumerMessage, error) {
-	messages := make([]*ConsumerMessage, 0, len(batch.Records))
-
-	for _, rec := range batch.Records {
-		offset := batch.FirstOffset + rec.OffsetDelta
-		if offset < child.offset {
-			continue
-		}
-		timestamp := batch.FirstTimestamp.Add(rec.TimestampDelta)
-		if batch.LogAppendTime {
-			timestamp = batch.MaxTimestamp
-		}
-		messages = append(messages, &ConsumerMessage{
-			Topic:     child.topic,
-			Partition: child.partition,
-			Key:       rec.Key,
-			Value:     rec.Value,
-			Offset:    offset,
-			Timestamp: timestamp,
-			Headers:   rec.Headers,
-		})
-		child.offset = offset + 1
-	}
-	if len(messages) == 0 {
-		child.offset++
-	}
-	return messages, nil
-}
-
-func (child *partitionConsumer) parseResponse(response *FetchResponse) ([]*ConsumerMessage, error) {
-	var (
-		metricRegistry          = child.conf.MetricRegistry
-		consumerBatchSizeMetric metrics.Histogram
-	)
-
-	if metricRegistry != nil {
-		consumerBatchSizeMetric = getOrRegisterHistogram("consumer-batch-size", metricRegistry)
-	}
-
-	// If request was throttled and empty we log and return without error
-	if response.ThrottleTime != time.Duration(0) && len(response.Blocks) == 0 {
-		Logger.Printf(
-			"consumer/broker/%d FetchResponse throttled %v\n",
-			child.broker.broker.ID(), response.ThrottleTime)
-		return nil, nil
-	}
-
-	block := response.GetBlock(child.topic, child.partition)
-	if block == nil {
-		return nil, ErrIncompleteResponse
-	}
-
-	if block.Err != ErrNoError {
-		return nil, block.Err
-	}
-
-	nRecs, err := block.numRecords()
-	if err != nil {
-		return nil, err
-	}
-
-	consumerBatchSizeMetric.Update(int64(nRecs))
-
-	if nRecs == 0 {
-		partialTrailingMessage, err := block.isPartial()
-		if err != nil {
-			return nil, err
-		}
-		// We got no messages. If we got a trailing one then we need to ask for more data.
-		// Otherwise we just poll again and wait for one to be produced...
-		if partialTrailingMessage {
-			if child.conf.Consumer.Fetch.Max > 0 && child.fetchSize == child.conf.Consumer.Fetch.Max {
-				// we can't ask for more data, we've hit the configured limit
-				child.sendError(ErrMessageTooLarge)
-				child.offset++ // skip this one so we can keep processing future messages
-			} else {
-				child.fetchSize *= 2
-				// check int32 overflow
-				if child.fetchSize < 0 {
-					child.fetchSize = math.MaxInt32
-				}
-				if child.conf.Consumer.Fetch.Max > 0 && child.fetchSize > child.conf.Consumer.Fetch.Max {
-					child.fetchSize = child.conf.Consumer.Fetch.Max
-				}
-			}
-		}
-
-		return nil, nil
-	}
-
-	// we got messages, reset our fetch size in case it was increased for a previous request
-	child.fetchSize = child.conf.Consumer.Fetch.Default
-	atomic.StoreInt64(&child.highWaterMarkOffset, block.HighWaterMarkOffset)
-
-	// abortedProducerIDs contains producerID which message should be ignored as uncommitted
-	// - producerID are added when the partitionConsumer iterate over the offset at which an aborted transaction begins (abortedTransaction.FirstOffset)
-	// - producerID are removed when partitionConsumer iterate over an aborted controlRecord, meaning the aborted transaction for this producer is over
-	abortedProducerIDs := make(map[int64]struct{}, len(block.AbortedTransactions))
-	abortedTransactions := block.getAbortedTransactions()
-
-	messages := []*ConsumerMessage{}
-	for _, records := range block.RecordsSet {
-		switch records.recordsType {
-		case legacyRecords:
-			messageSetMessages, err := child.parseMessages(records.MsgSet)
-			if err != nil {
-				return nil, err
-			}
-
-			messages = append(messages, messageSetMessages...)
-		case defaultRecords:
-			// Consume remaining abortedTransaction up to last offset of current batch
-			for _, txn := range abortedTransactions {
-				if txn.FirstOffset > records.RecordBatch.LastOffset() {
-					break
-				}
-				abortedProducerIDs[txn.ProducerID] = struct{}{}
-				// Pop abortedTransactions so that we never add it again
-				abortedTransactions = abortedTransactions[1:]
-			}
-
-			recordBatchMessages, err := child.parseRecords(records.RecordBatch)
-			if err != nil {
-				return nil, err
-			}
-
-			// Parse and commit offset but do not expose messages that are:
-			// - control records
-			// - part of an aborted transaction when set to `ReadCommitted`
-
-			// control record
-			isControl, err := records.isControl()
-			if err != nil {
-				// I don't know why there is this continue in case of error to begin with
-				// Safe bet is to ignore control messages if ReadUncommitted
-				// and block on them in case of error and ReadCommitted
-				if child.conf.Consumer.IsolationLevel == ReadCommitted {
-					return nil, err
-				}
-				continue
-			}
-			if isControl {
-				controlRecord, err := records.getControlRecord()
-				if err != nil {
-					return nil, err
-				}
-
-				if controlRecord.Type == ControlRecordAbort {
-					delete(abortedProducerIDs, records.RecordBatch.ProducerID)
-				}
-				continue
-			}
-
-			// filter aborted transactions
-			if child.conf.Consumer.IsolationLevel == ReadCommitted {
-				_, isAborted := abortedProducerIDs[records.RecordBatch.ProducerID]
-				if records.RecordBatch.IsTransactional && isAborted {
-					continue
-				}
-			}
-
-			messages = append(messages, recordBatchMessages...)
-		default:
-			return nil, fmt.Errorf("unknown records type: %v", records.recordsType)
-		}
-	}
-
-	return messages, nil
-}
-
-type brokerConsumer struct {
-	consumer         *consumer
-	broker           *Broker
-	input            chan *partitionConsumer
-	newSubscriptions chan []*partitionConsumer
-	subscriptions    map[*partitionConsumer]none
-	wait             chan none
-	acks             sync.WaitGroup
-	refs             int
-}
-
-func (c *consumer) newBrokerConsumer(broker *Broker) *brokerConsumer {
-	bc := &brokerConsumer{
-		consumer:         c,
-		broker:           broker,
-		input:            make(chan *partitionConsumer),
-		newSubscriptions: make(chan []*partitionConsumer),
-		wait:             make(chan none),
-		subscriptions:    make(map[*partitionConsumer]none),
-		refs:             0,
-	}
-
-	go withRecover(bc.subscriptionManager)
-	go withRecover(bc.subscriptionConsumer)
-
-	return bc
-}
-
-// The subscriptionManager constantly accepts new subscriptions on `input` (even when the main subscriptionConsumer
-// goroutine is in the middle of a network request) and batches it up. The main worker goroutine picks
-// up a batch of new subscriptions between every network request by reading from `newSubscriptions`, so we give
-// it nil if no new subscriptions are available. We also write to `wait` only when new subscriptions is available,
-// so the main goroutine can block waiting for work if it has none.
-func (bc *brokerConsumer) subscriptionManager() {
-	var buffer []*partitionConsumer
-
-	for {
-		if len(buffer) > 0 {
-			select {
-			case event, ok := <-bc.input:
-				if !ok {
-					goto done
-				}
-				buffer = append(buffer, event)
-			case bc.newSubscriptions <- buffer:
-				buffer = nil
-			case bc.wait <- none{}:
-			}
-		} else {
-			select {
-			case event, ok := <-bc.input:
-				if !ok {
-					goto done
-				}
-				buffer = append(buffer, event)
-			case bc.newSubscriptions <- nil:
-			}
-		}
-	}
-
-done:
-	close(bc.wait)
-	if len(buffer) > 0 {
-		bc.newSubscriptions <- buffer
-	}
-	close(bc.newSubscriptions)
-}
-
-//subscriptionConsumer ensures we will get nil right away if no new subscriptions is available
-func (bc *brokerConsumer) subscriptionConsumer() {
-	<-bc.wait // wait for our first piece of work
-
-	for newSubscriptions := range bc.newSubscriptions {
-		bc.updateSubscriptions(newSubscriptions)
-
-		if len(bc.subscriptions) == 0 {
-			// We're about to be shut down or we're about to receive more subscriptions.
-			// Either way, the signal just hasn't propagated to our goroutine yet.
-			<-bc.wait
-			continue
-		}
-
-		response, err := bc.fetchNewMessages()
-
-		if err != nil {
-			Logger.Printf("consumer/broker/%d disconnecting due to error processing FetchRequest: %s\n", bc.broker.ID(), err)
-			bc.abort(err)
-			return
-		}
-
-		bc.acks.Add(len(bc.subscriptions))
-		for child := range bc.subscriptions {
-			child.feeder <- response
-		}
-		bc.acks.Wait()
-		bc.handleResponses()
-	}
-}
-
-func (bc *brokerConsumer) updateSubscriptions(newSubscriptions []*partitionConsumer) {
-	for _, child := range newSubscriptions {
-		bc.subscriptions[child] = none{}
-		Logger.Printf("consumer/broker/%d added subscription to %s/%d\n", bc.broker.ID(), child.topic, child.partition)
-	}
-
-	for child := range bc.subscriptions {
-		select {
-		case <-child.dying:
-			Logger.Printf("consumer/broker/%d closed dead subscription to %s/%d\n", bc.broker.ID(), child.topic, child.partition)
-			close(child.trigger)
-			delete(bc.subscriptions, child)
-		default:
-			// no-op
-		}
-	}
-}
-
-//handleResponses handles the response codes left for us by our subscriptions, and abandons ones that have been closed
-func (bc *brokerConsumer) handleResponses() {
-	for child := range bc.subscriptions {
-		result := child.responseResult
-		child.responseResult = nil
-
-		switch result {
-		case nil:
-			// no-op
-		case errTimedOut:
-			Logger.Printf("consumer/broker/%d abandoned subscription to %s/%d because consuming was taking too long\n",
-				bc.broker.ID(), child.topic, child.partition)
-			delete(bc.subscriptions, child)
-		case ErrOffsetOutOfRange:
-			// there's no point in retrying this it will just fail the same way again
-			// shut it down and force the user to choose what to do
-			child.sendError(result)
-			Logger.Printf("consumer/%s/%d shutting down because %s\n", child.topic, child.partition, result)
-			close(child.trigger)
-			delete(bc.subscriptions, child)
-		case ErrUnknownTopicOrPartition, ErrNotLeaderForPartition, ErrLeaderNotAvailable, ErrReplicaNotAvailable:
-			// not an error, but does need redispatching
-			Logger.Printf("consumer/broker/%d abandoned subscription to %s/%d because %s\n",
-				bc.broker.ID(), child.topic, child.partition, result)
-			child.trigger <- none{}
-			delete(bc.subscriptions, child)
-		default:
-			// dunno, tell the user and try redispatching
-			child.sendError(result)
-			Logger.Printf("consumer/broker/%d abandoned subscription to %s/%d because %s\n",
-				bc.broker.ID(), child.topic, child.partition, result)
-			child.trigger <- none{}
-			delete(bc.subscriptions, child)
-		}
-	}
-}
-
-func (bc *brokerConsumer) abort(err error) {
-	bc.consumer.abandonBrokerConsumer(bc)
-	_ = bc.broker.Close() // we don't care about the error this might return, we already have one
-
-	for child := range bc.subscriptions {
-		child.sendError(err)
-		child.trigger <- none{}
-	}
-
-	for newSubscriptions := range bc.newSubscriptions {
-		if len(newSubscriptions) == 0 {
-			<-bc.wait
-			continue
-		}
-		for _, child := range newSubscriptions {
-			child.sendError(err)
-			child.trigger <- none{}
-		}
-	}
-}
-
-func (bc *brokerConsumer) fetchNewMessages() (*FetchResponse, error) {
-	request := &FetchRequest{
-		MinBytes:    bc.consumer.conf.Consumer.Fetch.Min,
-		MaxWaitTime: int32(bc.consumer.conf.Consumer.MaxWaitTime / time.Millisecond),
-	}
-	if bc.consumer.conf.Version.IsAtLeast(V0_9_0_0) {
-		request.Version = 1
-	}
-	if bc.consumer.conf.Version.IsAtLeast(V0_10_0_0) {
-		request.Version = 2
-	}
-	if bc.consumer.conf.Version.IsAtLeast(V0_10_1_0) {
-		request.Version = 3
-		request.MaxBytes = MaxResponseSize
-	}
-	if bc.consumer.conf.Version.IsAtLeast(V0_11_0_0) {
-		request.Version = 4
-		request.Isolation = bc.consumer.conf.Consumer.IsolationLevel
-	}
-	if bc.consumer.conf.Version.IsAtLeast(V1_1_0_0) {
-		request.Version = 7
-		// We do not currently implement KIP-227 FetchSessions. Setting the id to 0
-		// and the epoch to -1 tells the broker not to generate as session ID we're going
-		// to just ignore anyway.
-		request.SessionID = 0
-		request.SessionEpoch = -1
-	}
-	if bc.consumer.conf.Version.IsAtLeast(V2_1_0_0) {
-		request.Version = 10
-	}
-	if bc.consumer.conf.Version.IsAtLeast(V2_3_0_0) {
-		request.Version = 11
-		request.RackID = bc.consumer.conf.RackID
-	}
-
-	for child := range bc.subscriptions {
-		request.AddBlock(child.topic, child.partition, child.offset, child.fetchSize)
-	}
-
-	return bc.broker.Fetch(request)
-}

vendor/github.com/Shopify/sarama/consumer_group.go

@@ -1,867 +0,0 @@
-package sarama
-
-import (
-	"context"
-	"errors"
-	"fmt"
-	"sort"
-	"sync"
-	"time"
-)
-
-// ErrClosedConsumerGroup is the error returned when a method is called on a consumer group that has been closed.
-var ErrClosedConsumerGroup = errors.New("kafka: tried to use a consumer group that was closed")
-
-// ConsumerGroup is responsible for dividing up processing of topics and partitions
-// over a collection of processes (the members of the consumer group).
-type ConsumerGroup interface {
-	// Consume joins a cluster of consumers for a given list of topics and
-	// starts a blocking ConsumerGroupSession through the ConsumerGroupHandler.
-	//
-	// The life-cycle of a session is represented by the following steps:
-	//
-	// 1. The consumers join the group (as explained in https://kafka.apache.org/documentation/#intro_consumers)
-	//    and is assigned their "fair share" of partitions, aka 'claims'.
-	// 2. Before processing starts, the handler's Setup() hook is called to notify the user
-	//    of the claims and allow any necessary preparation or alteration of state.
-	// 3. For each of the assigned claims the handler's ConsumeClaim() function is then called
-	//    in a separate goroutine which requires it to be thread-safe. Any state must be carefully protected
-	//    from concurrent reads/writes.
-	// 4. The session will persist until one of the ConsumeClaim() functions exits. This can be either when the
-	//    parent context is cancelled or when a server-side rebalance cycle is initiated.
-	// 5. Once all the ConsumeClaim() loops have exited, the handler's Cleanup() hook is called
-	//    to allow the user to perform any final tasks before a rebalance.
-	// 6. Finally, marked offsets are committed one last time before claims are released.
-	//
-	// Please note, that once a rebalance is triggered, sessions must be completed within
-	// Config.Consumer.Group.Rebalance.Timeout. This means that ConsumeClaim() functions must exit
-	// as quickly as possible to allow time for Cleanup() and the final offset commit. If the timeout
-	// is exceeded, the consumer will be removed from the group by Kafka, which will cause offset
-	// commit failures.
-	// This method should be called inside an infinite loop, when a
-	// server-side rebalance happens, the consumer session will need to be
-	// recreated to get the new claims.
-	Consume(ctx context.Context, topics []string, handler ConsumerGroupHandler) error
-
-	// Errors returns a read channel of errors that occurred during the consumer life-cycle.
-	// By default, errors are logged and not returned over this channel.
-	// If you want to implement any custom error handling, set your config's
-	// Consumer.Return.Errors setting to true, and read from this channel.
-	Errors() <-chan error
-
-	// Close stops the ConsumerGroup and detaches any running sessions. It is required to call
-	// this function before the object passes out of scope, as it will otherwise leak memory.
-	Close() error
-}
-
-type consumerGroup struct {
-	client Client
-
-	config   *Config
-	consumer Consumer
-	groupID  string
-	memberID string
-	errors   chan error
-
-	lock      sync.Mutex
-	closed    chan none
-	closeOnce sync.Once
-
-	userData []byte
-}
-
-// NewConsumerGroup creates a new consumer group the given broker addresses and configuration.
-func NewConsumerGroup(addrs []string, groupID string, config *Config) (ConsumerGroup, error) {
-	client, err := NewClient(addrs, config)
-	if err != nil {
-		return nil, err
-	}
-
-	c, err := newConsumerGroup(groupID, client)
-	if err != nil {
-		_ = client.Close()
-	}
-	return c, err
-}
-
-// NewConsumerGroupFromClient creates a new consumer group using the given client. It is still
-// necessary to call Close() on the underlying client when shutting down this consumer.
-// PLEASE NOTE: consumer groups can only re-use but not share clients.
-func NewConsumerGroupFromClient(groupID string, client Client) (ConsumerGroup, error) {
-	// For clients passed in by the client, ensure we don't
-	// call Close() on it.
-	cli := &nopCloserClient{client}
-	return newConsumerGroup(groupID, cli)
-}
-
-func newConsumerGroup(groupID string, client Client) (ConsumerGroup, error) {
-	config := client.Config()
-	if !config.Version.IsAtLeast(V0_10_2_0) {
-		return nil, ConfigurationError("consumer groups require Version to be >= V0_10_2_0")
-	}
-
-	consumer, err := NewConsumerFromClient(client)
-	if err != nil {
-		return nil, err
-	}
-
-	return &consumerGroup{
-		client:   client,
-		consumer: consumer,
-		config:   config,
-		groupID:  groupID,
-		errors:   make(chan error, config.ChannelBufferSize),
-		closed:   make(chan none),
-	}, nil
-}
-
-// Errors implements ConsumerGroup.
-func (c *consumerGroup) Errors() <-chan error { return c.errors }
-
-// Close implements ConsumerGroup.
-func (c *consumerGroup) Close() (err error) {
-	c.closeOnce.Do(func() {
-		close(c.closed)
-
-		// leave group
-		if e := c.leave(); e != nil {
-			err = e
-		}
-
-		// drain errors
-		go func() {
-			close(c.errors)
-		}()
-		for e := range c.errors {
-			err = e
-		}
-
-		if e := c.client.Close(); e != nil {
-			err = e
-		}
-	})
-	return
-}
-
-// Consume implements ConsumerGroup.
-func (c *consumerGroup) Consume(ctx context.Context, topics []string, handler ConsumerGroupHandler) error {
-	// Ensure group is not closed
-	select {
-	case <-c.closed:
-		return ErrClosedConsumerGroup
-	default:
-	}
-
-	c.lock.Lock()
-	defer c.lock.Unlock()
-
-	// Quick exit when no topics are provided
-	if len(topics) == 0 {
-		return fmt.Errorf("no topics provided")
-	}
-
-	// Refresh metadata for requested topics
-	if err := c.client.RefreshMetadata(topics...); err != nil {
-		return err
-	}
-
-	// Init session
-	sess, err := c.newSession(ctx, topics, handler, c.config.Consumer.Group.Rebalance.Retry.Max)
-	if err == ErrClosedClient {
-		return ErrClosedConsumerGroup
-	} else if err != nil {
-		return err
-	}
-
-	// loop check topic partition numbers changed
-	// will trigger rebalance when any topic partitions number had changed
-	// avoid Consume function called again that will generate more than loopCheckPartitionNumbers coroutine
-	go c.loopCheckPartitionNumbers(topics, sess)
-
-	// Wait for session exit signal
-	<-sess.ctx.Done()
-
-	// Gracefully release session claims
-	return sess.release(true)
-}
-
-func (c *consumerGroup) retryNewSession(ctx context.Context, topics []string, handler ConsumerGroupHandler, retries int, refreshCoordinator bool) (*consumerGroupSession, error) {
-	select {
-	case <-c.closed:
-		return nil, ErrClosedConsumerGroup
-	case <-time.After(c.config.Consumer.Group.Rebalance.Retry.Backoff):
-	}
-
-	if refreshCoordinator {
-		err := c.client.RefreshCoordinator(c.groupID)
-		if err != nil {
-			return c.retryNewSession(ctx, topics, handler, retries, true)
-		}
-	}
-
-	return c.newSession(ctx, topics, handler, retries-1)
-}
-
-func (c *consumerGroup) newSession(ctx context.Context, topics []string, handler ConsumerGroupHandler, retries int) (*consumerGroupSession, error) {
-	coordinator, err := c.client.Coordinator(c.groupID)
-	if err != nil {
-		if retries <= 0 {
-			return nil, err
-		}
-
-		return c.retryNewSession(ctx, topics, handler, retries, true)
-	}
-
-	// Join consumer group
-	join, err := c.joinGroupRequest(coordinator, topics)
-	if err != nil {
-		_ = coordinator.Close()
-		return nil, err
-	}
-	switch join.Err {
-	case ErrNoError:
-		c.memberID = join.MemberId
-	case ErrUnknownMemberId, ErrIllegalGeneration: // reset member ID and retry immediately
-		c.memberID = ""
-		return c.newSession(ctx, topics, handler, retries)
-	case ErrNotCoordinatorForConsumer: // retry after backoff with coordinator refresh
-		if retries <= 0 {
-			return nil, join.Err
-		}
-
-		return c.retryNewSession(ctx, topics, handler, retries, true)
-	case ErrRebalanceInProgress: // retry after backoff
-		if retries <= 0 {
-			return nil, join.Err
-		}
-
-		return c.retryNewSession(ctx, topics, handler, retries, false)
-	default:
-		return nil, join.Err
-	}
-
-	// Prepare distribution plan if we joined as the leader
-	var plan BalanceStrategyPlan
-	if join.LeaderId == join.MemberId {
-		members, err := join.GetMembers()
-		if err != nil {
-			return nil, err
-		}
-
-		plan, err = c.balance(members)
-		if err != nil {
-			return nil, err
-		}
-	}
-
-	// Sync consumer group
-	sync, err := c.syncGroupRequest(coordinator, plan, join.GenerationId)
-	if err != nil {
-		_ = coordinator.Close()
-		return nil, err
-	}
-	switch sync.Err {
-	case ErrNoError:
-	case ErrUnknownMemberId, ErrIllegalGeneration: // reset member ID and retry immediately
-		c.memberID = ""
-		return c.newSession(ctx, topics, handler, retries)
-	case ErrNotCoordinatorForConsumer: // retry after backoff with coordinator refresh
-		if retries <= 0 {
-			return nil, sync.Err
-		}
-
-		return c.retryNewSession(ctx, topics, handler, retries, true)
-	case ErrRebalanceInProgress: // retry after backoff
-		if retries <= 0 {
-			return nil, sync.Err
-		}
-
-		return c.retryNewSession(ctx, topics, handler, retries, false)
-	default:
-		return nil, sync.Err
-	}
-
-	// Retrieve and sort claims
-	var claims map[string][]int32
-	if len(sync.MemberAssignment) > 0 {
-		members, err := sync.GetMemberAssignment()
-		if err != nil {
-			return nil, err
-		}
-		claims = members.Topics
-		c.userData = members.UserData
-
-		for _, partitions := range claims {
-			sort.Sort(int32Slice(partitions))
-		}
-	}
-
-	return newConsumerGroupSession(ctx, c, claims, join.MemberId, join.GenerationId, handler)
-}
-
-func (c *consumerGroup) joinGroupRequest(coordinator *Broker, topics []string) (*JoinGroupResponse, error) {
-	req := &JoinGroupRequest{
-		GroupId:        c.groupID,
-		MemberId:       c.memberID,
-		SessionTimeout: int32(c.config.Consumer.Group.Session.Timeout / time.Millisecond),
-		ProtocolType:   "consumer",
-	}
-	if c.config.Version.IsAtLeast(V0_10_1_0) {
-		req.Version = 1
-		req.RebalanceTimeout = int32(c.config.Consumer.Group.Rebalance.Timeout / time.Millisecond)
-	}
-
-	// use static user-data if configured, otherwise use consumer-group userdata from the last sync
-	userData := c.config.Consumer.Group.Member.UserData
-	if len(userData) == 0 {
-		userData = c.userData
-	}
-	meta := &ConsumerGroupMemberMetadata{
-		Topics:   topics,
-		UserData: userData,
-	}
-	strategy := c.config.Consumer.Group.Rebalance.Strategy
-	if err := req.AddGroupProtocolMetadata(strategy.Name(), meta); err != nil {
-		return nil, err
-	}
-
-	return coordinator.JoinGroup(req)
-}
-
-func (c *consumerGroup) syncGroupRequest(coordinator *Broker, plan BalanceStrategyPlan, generationID int32) (*SyncGroupResponse, error) {
-	req := &SyncGroupRequest{
-		GroupId:      c.groupID,
-		MemberId:     c.memberID,
-		GenerationId: generationID,
-	}
-	strategy := c.config.Consumer.Group.Rebalance.Strategy
-	for memberID, topics := range plan {
-		assignment := &ConsumerGroupMemberAssignment{Topics: topics}
-		userDataBytes, err := strategy.AssignmentData(memberID, topics, generationID)
-		if err != nil {
-			return nil, err
-		}
-		assignment.UserData = userDataBytes
-		if err := req.AddGroupAssignmentMember(memberID, assignment); err != nil {
-			return nil, err
-		}
-	}
-	return coordinator.SyncGroup(req)
-}
-
-func (c *consumerGroup) heartbeatRequest(coordinator *Broker, memberID string, generationID int32) (*HeartbeatResponse, error) {
-	req := &HeartbeatRequest{
-		GroupId:      c.groupID,
-		MemberId:     memberID,
-		GenerationId: generationID,
-	}
-
-	return coordinator.Heartbeat(req)
-}
-
-func (c *consumerGroup) balance(members map[string]ConsumerGroupMemberMetadata) (BalanceStrategyPlan, error) {
-	topics := make(map[string][]int32)
-	for _, meta := range members {
-		for _, topic := range meta.Topics {
-			topics[topic] = nil
-		}
-	}
-
-	for topic := range topics {
-		partitions, err := c.client.Partitions(topic)
-		if err != nil {
-			return nil, err
-		}
-		topics[topic] = partitions
-	}
-
-	strategy := c.config.Consumer.Group.Rebalance.Strategy
-	return strategy.Plan(members, topics)
-}
-
-// Leaves the cluster, called by Close.
-func (c *consumerGroup) leave() error {
-	c.lock.Lock()
-	defer c.lock.Unlock()
-	if c.memberID == "" {
-		return nil
-	}
-
-	coordinator, err := c.client.Coordinator(c.groupID)
-	if err != nil {
-		return err
-	}
-
-	resp, err := coordinator.LeaveGroup(&LeaveGroupRequest{
-		GroupId:  c.groupID,
-		MemberId: c.memberID,
-	})
-	if err != nil {
-		_ = coordinator.Close()
-		return err
-	}
-
-	// Unset memberID
-	c.memberID = ""
-
-	// Check response
-	switch resp.Err {
-	case ErrRebalanceInProgress, ErrUnknownMemberId, ErrNoError:
-		return nil
-	default:
-		return resp.Err
-	}
-}
-
-func (c *consumerGroup) handleError(err error, topic string, partition int32) {
-	if _, ok := err.(*ConsumerError); !ok && topic != "" && partition > -1 {
-		err = &ConsumerError{
-			Topic:     topic,
-			Partition: partition,
-			Err:       err,
-		}
-	}
-
-	if !c.config.Consumer.Return.Errors {
-		Logger.Println(err)
-		return
-	}
-
-	select {
-	case <-c.closed:
-		//consumer is closed
-		return
-	default:
-	}
-
-	select {
-	case c.errors <- err:
-	default:
-		// no error listener
-	}
-}
-
-func (c *consumerGroup) loopCheckPartitionNumbers(topics []string, session *consumerGroupSession) {
-	pause := time.NewTicker(c.config.Metadata.RefreshFrequency)
-	defer session.cancel()
-	defer pause.Stop()
-	var oldTopicToPartitionNum map[string]int
-	var err error
-	if oldTopicToPartitionNum, err = c.topicToPartitionNumbers(topics); err != nil {
-		return
-	}
-	for {
-		if newTopicToPartitionNum, err := c.topicToPartitionNumbers(topics); err != nil {
-			return
-		} else {
-			for topic, num := range oldTopicToPartitionNum {
-				if newTopicToPartitionNum[topic] != num {
-					return // trigger the end of the session on exit
-				}
-			}
-		}
-		select {
-		case <-pause.C:
-		case <-session.ctx.Done():
-			Logger.Printf("loop check partition number coroutine will exit, topics %s", topics)
-			// if session closed by other, should be exited
-			return
-		case <-c.closed:
-			return
-		}
-	}
-}
-
-func (c *consumerGroup) topicToPartitionNumbers(topics []string) (map[string]int, error) {
-	topicToPartitionNum := make(map[string]int, len(topics))
-	for _, topic := range topics {
-		if partitionNum, err := c.client.Partitions(topic); err != nil {
-			Logger.Printf("Consumer Group topic %s get partition number failed %v", topic, err)
-			return nil, err
-		} else {
-			topicToPartitionNum[topic] = len(partitionNum)
-		}
-	}
-	return topicToPartitionNum, nil
-}
-
-// --------------------------------------------------------------------
-
-// ConsumerGroupSession represents a consumer group member session.
-type ConsumerGroupSession interface {
-	// Claims returns information about the claimed partitions by topic.
-	Claims() map[string][]int32
-
-	// MemberID returns the cluster member ID.
-	MemberID() string
-
-	// GenerationID returns the current generation ID.
-	GenerationID() int32
-
-	// MarkOffset marks the provided offset, alongside a metadata string
-	// that represents the state of the partition consumer at that point in time. The
-	// metadata string can be used by another consumer to restore that state, so it
-	// can resume consumption.
-	//
-	// To follow upstream conventions, you are expected to mark the offset of the
-	// next message to read, not the last message read. Thus, when calling `MarkOffset`
-	// you should typically add one to the offset of the last consumed message.
-	//
-	// Note: calling MarkOffset does not necessarily commit the offset to the backend
-	// store immediately for efficiency reasons, and it may never be committed if
-	// your application crashes. This means that you may end up processing the same
-	// message twice, and your processing should ideally be idempotent.
-	MarkOffset(topic string, partition int32, offset int64, metadata string)
-
-	// ResetOffset resets to the provided offset, alongside a metadata string that
-	// represents the state of the partition consumer at that point in time. Reset
-	// acts as a counterpart to MarkOffset, the difference being that it allows to
-	// reset an offset to an earlier or smaller value, where MarkOffset only
-	// allows incrementing the offset. cf MarkOffset for more details.
-	ResetOffset(topic string, partition int32, offset int64, metadata string)
-
-	// MarkMessage marks a message as consumed.
-	MarkMessage(msg *ConsumerMessage, metadata string)
-
-	// Context returns the session context.
-	Context() context.Context
-}
-
-type consumerGroupSession struct {
-	parent       *consumerGroup
-	memberID     string
-	generationID int32
-	handler      ConsumerGroupHandler
-
-	claims  map[string][]int32
-	offsets *offsetManager
-	ctx     context.Context
-	cancel  func()
-
-	waitGroup       sync.WaitGroup
-	releaseOnce     sync.Once
-	hbDying, hbDead chan none
-}
-
-func newConsumerGroupSession(ctx context.Context, parent *consumerGroup, claims map[string][]int32, memberID string, generationID int32, handler ConsumerGroupHandler) (*consumerGroupSession, error) {
-	// init offset manager
-	offsets, err := newOffsetManagerFromClient(parent.groupID, memberID, generationID, parent.client)
-	if err != nil {
-		return nil, err
-	}
-
-	// init context
-	ctx, cancel := context.WithCancel(ctx)
-
-	// init session
-	sess := &consumerGroupSession{
-		parent:       parent,
-		memberID:     memberID,
-		generationID: generationID,
-		handler:      handler,
-		offsets:      offsets,
-		claims:       claims,
-		ctx:          ctx,
-		cancel:       cancel,
-		hbDying:      make(chan none),
-		hbDead:       make(chan none),
-	}
-
-	// start heartbeat loop
-	go sess.heartbeatLoop()
-
-	// create a POM for each claim
-	for topic, partitions := range claims {
-		for _, partition := range partitions {
-			pom, err := offsets.ManagePartition(topic, partition)
-			if err != nil {
-				_ = sess.release(false)
-				return nil, err
-			}
-
-			// handle POM errors
-			go func(topic string, partition int32) {
-				for err := range pom.Errors() {
-					sess.parent.handleError(err, topic, partition)
-				}
-			}(topic, partition)
-		}
-	}
-
-	// perform setup
-	if err := handler.Setup(sess); err != nil {
-		_ = sess.release(true)
-		return nil, err
-	}
-
-	// start consuming
-	for topic, partitions := range claims {
-		for _, partition := range partitions {
-			sess.waitGroup.Add(1)
-
-			go func(topic string, partition int32) {
-				defer sess.waitGroup.Done()
-
-				// cancel the as session as soon as the first
-				// goroutine exits
-				defer sess.cancel()
-
-				// consume a single topic/partition, blocking
-				sess.consume(topic, partition)
-			}(topic, partition)
-		}
-	}
-	return sess, nil
-}
-
-func (s *consumerGroupSession) Claims() map[string][]int32 { return s.claims }
-func (s *consumerGroupSession) MemberID() string           { return s.memberID }
-func (s *consumerGroupSession) GenerationID() int32        { return s.generationID }
-
-func (s *consumerGroupSession) MarkOffset(topic string, partition int32, offset int64, metadata string) {
-	if pom := s.offsets.findPOM(topic, partition); pom != nil {
-		pom.MarkOffset(offset, metadata)
-	}
-}
-
-func (s *consumerGroupSession) ResetOffset(topic string, partition int32, offset int64, metadata string) {
-	if pom := s.offsets.findPOM(topic, partition); pom != nil {
-		pom.ResetOffset(offset, metadata)
-	}
-}
-
-func (s *consumerGroupSession) MarkMessage(msg *ConsumerMessage, metadata string) {
-	s.MarkOffset(msg.Topic, msg.Partition, msg.Offset+1, metadata)
-}
-
-func (s *consumerGroupSession) Context() context.Context {
-	return s.ctx
-}
-
-func (s *consumerGroupSession) consume(topic string, partition int32) {
-	// quick exit if rebalance is due
-	select {
-	case <-s.ctx.Done():
-		return
-	case <-s.parent.closed:
-		return
-	default:
-	}
-
-	// get next offset
-	offset := s.parent.config.Consumer.Offsets.Initial
-	if pom := s.offsets.findPOM(topic, partition); pom != nil {
-		offset, _ = pom.NextOffset()
-	}
-
-	// create new claim
-	claim, err := newConsumerGroupClaim(s, topic, partition, offset)
-	if err != nil {
-		s.parent.handleError(err, topic, partition)
-		return
-	}
-
-	// handle errors
-	go func() {
-		for err := range claim.Errors() {
-			s.parent.handleError(err, topic, partition)
-		}
-	}()
-
-	// trigger close when session is done
-	go func() {
-		select {
-		case <-s.ctx.Done():
-		case <-s.parent.closed:
-		}
-		claim.AsyncClose()
-	}()
-
-	// start processing
-	if err := s.handler.ConsumeClaim(s, claim); err != nil {
-		s.parent.handleError(err, topic, partition)
-	}
-
-	// ensure consumer is closed & drained
-	claim.AsyncClose()
-	for _, err := range claim.waitClosed() {
-		s.parent.handleError(err, topic, partition)
-	}
-}
-
-func (s *consumerGroupSession) release(withCleanup bool) (err error) {
-	// signal release, stop heartbeat
-	s.cancel()
-
-	// wait for consumers to exit
-	s.waitGroup.Wait()
-
-	// perform release
-	s.releaseOnce.Do(func() {
-		if withCleanup {
-			if e := s.handler.Cleanup(s); e != nil {
-				s.parent.handleError(e, "", -1)
-				err = e
-			}
-		}
-
-		if e := s.offsets.Close(); e != nil {
-			err = e
-		}
-
-		close(s.hbDying)
-		<-s.hbDead
-	})
-
-	return
-}
-
-func (s *consumerGroupSession) heartbeatLoop() {
-	defer close(s.hbDead)
-	defer s.cancel() // trigger the end of the session on exit
-
-	pause := time.NewTicker(s.parent.config.Consumer.Group.Heartbeat.Interval)
-	defer pause.Stop()
-
-	retries := s.parent.config.Metadata.Retry.Max
-	for {
-		coordinator, err := s.parent.client.Coordinator(s.parent.groupID)
-		if err != nil {
-			if retries <= 0 {
-				s.parent.handleError(err, "", -1)
-				return
-			}
-
-			select {
-			case <-s.hbDying:
-				return
-			case <-time.After(s.parent.config.Metadata.Retry.Backoff):
-				retries--
-			}
-			continue
-		}
-
-		resp, err := s.parent.heartbeatRequest(coordinator, s.memberID, s.generationID)
-		if err != nil {
-			_ = coordinator.Close()
-
-			if retries <= 0 {
-				s.parent.handleError(err, "", -1)
-				return
-			}
-
-			retries--
-			continue
-		}
-
-		switch resp.Err {
-		case ErrNoError:
-			retries = s.parent.config.Metadata.Retry.Max
-		case ErrRebalanceInProgress, ErrUnknownMemberId, ErrIllegalGeneration:
-			return
-		default:
-			s.parent.handleError(resp.Err, "", -1)
-			return
-		}
-
-		select {
-		case <-pause.C:
-		case <-s.hbDying:
-			return
-		}
-	}
-}
-
-// --------------------------------------------------------------------
-
-// ConsumerGroupHandler instances are used to handle individual topic/partition claims.
-// It also provides hooks for your consumer group session life-cycle and allow you to
-// trigger logic before or after the consume loop(s).
-//
-// PLEASE NOTE that handlers are likely be called from several goroutines concurrently,
-// ensure that all state is safely protected against race conditions.
-type ConsumerGroupHandler interface {
-	// Setup is run at the beginning of a new session, before ConsumeClaim.
-	Setup(ConsumerGroupSession) error
-
-	// Cleanup is run at the end of a session, once all ConsumeClaim goroutines have exited
-	// but before the offsets are committed for the very last time.
-	Cleanup(ConsumerGroupSession) error
-
-	// ConsumeClaim must start a consumer loop of ConsumerGroupClaim's Messages().
-	// Once the Messages() channel is closed, the Handler must finish its processing
-	// loop and exit.
-	ConsumeClaim(ConsumerGroupSession, ConsumerGroupClaim) error
-}
-
-// ConsumerGroupClaim processes Kafka messages from a given topic and partition within a consumer group.
-type ConsumerGroupClaim interface {
-	// Topic returns the consumed topic name.
-	Topic() string
-
-	// Partition returns the consumed partition.
-	Partition() int32
-
-	// InitialOffset returns the initial offset that was used as a starting point for this claim.
-	InitialOffset() int64
-
-	// HighWaterMarkOffset returns the high water mark offset of the partition,
-	// i.e. the offset that will be used for the next message that will be produced.
-	// You can use this to determine how far behind the processing is.
-	HighWaterMarkOffset() int64
-
-	// Messages returns the read channel for the messages that are returned by
-	// the broker. The messages channel will be closed when a new rebalance cycle
-	// is due. You must finish processing and mark offsets within
-	// Config.Consumer.Group.Session.Timeout before the topic/partition is eventually
-	// re-assigned to another group member.
-	Messages() <-chan *ConsumerMessage
-}
-
-type consumerGroupClaim struct {
-	topic     string
-	partition int32
-	offset    int64
-	PartitionConsumer
-}
-
-func newConsumerGroupClaim(sess *consumerGroupSession, topic string, partition int32, offset int64) (*consumerGroupClaim, error) {
-	pcm, err := sess.parent.consumer.ConsumePartition(topic, partition, offset)
-	if err == ErrOffsetOutOfRange {
-		offset = sess.parent.config.Consumer.Offsets.Initial
-		pcm, err = sess.parent.consumer.ConsumePartition(topic, partition, offset)
-	}
-	if err != nil {
-		return nil, err
-	}
-
-	go func() {
-		for err := range pcm.Errors() {
-			sess.parent.handleError(err, topic, partition)
-		}
-	}()
-
-	return &consumerGroupClaim{
-		topic:             topic,
-		partition:         partition,
-		offset:            offset,
-		PartitionConsumer: pcm,
-	}, nil
-}
-
-func (c *consumerGroupClaim) Topic() string        { return c.topic }
-func (c *consumerGroupClaim) Partition() int32     { return c.partition }
-func (c *consumerGroupClaim) InitialOffset() int64 { return c.offset }
-
-// Drains messages and errors, ensures the claim is fully closed.
-func (c *consumerGroupClaim) waitClosed() (errs ConsumerErrors) {
-	go func() {
-		for range c.Messages() {
-		}
-	}()
-
-	for err := range c.Errors() {
-		errs = append(errs, err)
-	}
-	return
-}

vendor/github.com/Shopify/sarama/consumer_group_members.go

@@ -1,96 +0,0 @@
-package sarama
-
-//ConsumerGroupMemberMetadata holds the metadata for consumer group
-type ConsumerGroupMemberMetadata struct {
-	Version  int16
-	Topics   []string
-	UserData []byte
-}
-
-func (m *ConsumerGroupMemberMetadata) encode(pe packetEncoder) error {
-	pe.putInt16(m.Version)
-
-	if err := pe.putStringArray(m.Topics); err != nil {
-		return err
-	}
-
-	if err := pe.putBytes(m.UserData); err != nil {
-		return err
-	}
-
-	return nil
-}
-
-func (m *ConsumerGroupMemberMetadata) decode(pd packetDecoder) (err error) {
-	if m.Version, err = pd.getInt16(); err != nil {
-		return
-	}
-
-	if m.Topics, err = pd.getStringArray(); err != nil {
-		return
-	}
-
-	if m.UserData, err = pd.getBytes(); err != nil {
-		return
-	}
-
-	return nil
-}
-
-//ConsumerGroupMemberAssignment holds the member assignment for a consume group
-type ConsumerGroupMemberAssignment struct {
-	Version  int16
-	Topics   map[string][]int32
-	UserData []byte
-}
-
-func (m *ConsumerGroupMemberAssignment) encode(pe packetEncoder) error {
-	pe.putInt16(m.Version)
-
-	if err := pe.putArrayLength(len(m.Topics)); err != nil {
-		return err
-	}
-
-	for topic, partitions := range m.Topics {
-		if err := pe.putString(topic); err != nil {
-			return err
-		}
-		if err := pe.putInt32Array(partitions); err != nil {
-			return err
-		}
-	}
-
-	if err := pe.putBytes(m.UserData); err != nil {
-		return err
-	}
-
-	return nil
-}
-
-func (m *ConsumerGroupMemberAssignment) decode(pd packetDecoder) (err error) {
-	if m.Version, err = pd.getInt16(); err != nil {
-		return
-	}
-
-	var topicLen int
-	if topicLen, err = pd.getArrayLength(); err != nil {
-		return
-	}
-
-	m.Topics = make(map[string][]int32, topicLen)
-	for i := 0; i < topicLen; i++ {
-		var topic string
-		if topic, err = pd.getString(); err != nil {
-			return
-		}
-		if m.Topics[topic], err = pd.getInt32Array(); err != nil {
-			return
-		}
-	}
-
-	if m.UserData, err = pd.getBytes(); err != nil {
-		return
-	}
-
-	return nil
-}

vendor/github.com/Shopify/sarama/consumer_metadata_request.go

@@ -1,38 +0,0 @@
-package sarama
-
-//ConsumerMetadataRequest is used for metadata requests
-type ConsumerMetadataRequest struct {
-	ConsumerGroup string
-}
-
-func (r *ConsumerMetadataRequest) encode(pe packetEncoder) error {
-	tmp := new(FindCoordinatorRequest)
-	tmp.CoordinatorKey = r.ConsumerGroup
-	tmp.CoordinatorType = CoordinatorGroup
-	return tmp.encode(pe)
-}
-
-func (r *ConsumerMetadataRequest) decode(pd packetDecoder, version int16) (err error) {
-	tmp := new(FindCoordinatorRequest)
-	if err := tmp.decode(pd, version); err != nil {
-		return err
-	}
-	r.ConsumerGroup = tmp.CoordinatorKey
-	return nil
-}
-
-func (r *ConsumerMetadataRequest) key() int16 {
-	return 10
-}
-
-func (r *ConsumerMetadataRequest) version() int16 {
-	return 0
-}
-
-func (r *ConsumerMetadataRequest) headerVersion() int16 {
-	return 1
-}
-
-func (r *ConsumerMetadataRequest) requiredVersion() KafkaVersion {
-	return V0_8_2_0
-}

vendor/github.com/Shopify/sarama/consumer_metadata_response.go

@@ -1,82 +0,0 @@
-package sarama
-
-import (
-	"net"
-	"strconv"
-)
-
-//ConsumerMetadataResponse holds the response for a consumer group meta data requests
-type ConsumerMetadataResponse struct {
-	Err             KError
-	Coordinator     *Broker
-	CoordinatorID   int32  // deprecated: use Coordinator.ID()
-	CoordinatorHost string // deprecated: use Coordinator.Addr()
-	CoordinatorPort int32  // deprecated: use Coordinator.Addr()
-}
-
-func (r *ConsumerMetadataResponse) decode(pd packetDecoder, version int16) (err error) {
-	tmp := new(FindCoordinatorResponse)
-
-	if err := tmp.decode(pd, version); err != nil {
-		return err
-	}
-
-	r.Err = tmp.Err
-
-	r.Coordinator = tmp.Coordinator
-	if tmp.Coordinator == nil {
-		return nil
-	}
-
-	// this can all go away in 2.0, but we have to fill in deprecated fields to maintain
-	// backwards compatibility
-	host, portstr, err := net.SplitHostPort(r.Coordinator.Addr())
-	if err != nil {
-		return err
-	}
-	port, err := strconv.ParseInt(portstr, 10, 32)
-	if err != nil {
-		return err
-	}
-	r.CoordinatorID = r.Coordinator.ID()
-	r.CoordinatorHost = host
-	r.CoordinatorPort = int32(port)
-
-	return nil
-}
-
-func (r *ConsumerMetadataResponse) encode(pe packetEncoder) error {
-	if r.Coordinator == nil {
-		r.Coordinator = new(Broker)
-		r.Coordinator.id = r.CoordinatorID
-		r.Coordinator.addr = net.JoinHostPort(r.CoordinatorHost, strconv.Itoa(int(r.CoordinatorPort)))
-	}
-
-	tmp := &FindCoordinatorResponse{
-		Version:     0,
-		Err:         r.Err,
-		Coordinator: r.Coordinator,
-	}
-
-	if err := tmp.encode(pe); err != nil {
-		return err
-	}
-
-	return nil
-}
-
-func (r *ConsumerMetadataResponse) key() int16 {
-	return 10
-}
-
-func (r *ConsumerMetadataResponse) version() int16 {
-	return 0
-}
-
-func (r *ConsumerMetadataResponse) headerVersion() int16 {
-	return 0
-}
-
-func (r *ConsumerMetadataResponse) requiredVersion() KafkaVersion {
-	return V0_8_2_0
-}

vendor/github.com/Shopify/sarama/control_record.go

@@ -1,72 +0,0 @@
-package sarama
-
-//ControlRecordType ...
-type ControlRecordType int
-
-const (
-	//ControlRecordAbort is a control record for abort
-	ControlRecordAbort ControlRecordType = iota
-	//ControlRecordCommit is a control record for commit
-	ControlRecordCommit
-	//ControlRecordUnknown is a control record of unknown type
-	ControlRecordUnknown
-)
-
-// Control records are returned as a record by fetchRequest
-// However unlike "normal" records, they mean nothing application wise.
-// They only serve internal logic for supporting transactions.
-type ControlRecord struct {
-	Version          int16
-	CoordinatorEpoch int32
-	Type             ControlRecordType
-}
-
-func (cr *ControlRecord) decode(key, value packetDecoder) error {
-	var err error
-	cr.Version, err = value.getInt16()
-	if err != nil {
-		return err
-	}
-
-	cr.CoordinatorEpoch, err = value.getInt32()
-	if err != nil {
-		return err
-	}
-
-	// There a version for the value part AND the key part. And I have no idea if they are supposed to match or not
-	// Either way, all these version can only be 0 for now
-	cr.Version, err = key.getInt16()
-	if err != nil {
-		return err
-	}
-
-	recordType, err := key.getInt16()
-	if err != nil {
-		return err
-	}
-
-	switch recordType {
-	case 0:
-		cr.Type = ControlRecordAbort
-	case 1:
-		cr.Type = ControlRecordCommit
-	default:
-		// from JAVA implementation:
-		// UNKNOWN is used to indicate a control type which the client is not aware of and should be ignored
-		cr.Type = ControlRecordUnknown
-	}
-	return nil
-}
-
-func (cr *ControlRecord) encode(key, value packetEncoder) {
-	value.putInt16(cr.Version)
-	value.putInt32(cr.CoordinatorEpoch)
-	key.putInt16(cr.Version)
-
-	switch cr.Type {
-	case ControlRecordAbort:
-		key.putInt16(0)
-	case ControlRecordCommit:
-		key.putInt16(1)
-	}
-}

vendor/github.com/Shopify/sarama/crc32_field.go

@@ -1,86 +0,0 @@
-package sarama
-
-import (
-	"encoding/binary"
-	"fmt"
-	"hash/crc32"
-	"sync"
-)
-
-type crcPolynomial int8
-
-const (
-	crcIEEE crcPolynomial = iota
-	crcCastagnoli
-)
-
-var crc32FieldPool = sync.Pool{}
-
-func acquireCrc32Field(polynomial crcPolynomial) *crc32Field {
-	val := crc32FieldPool.Get()
-	if val != nil {
-		c := val.(*crc32Field)
-		c.polynomial = polynomial
-		return c
-	}
-	return newCRC32Field(polynomial)
-}
-
-func releaseCrc32Field(c *crc32Field) {
-	crc32FieldPool.Put(c)
-}
-
-var castagnoliTable = crc32.MakeTable(crc32.Castagnoli)
-
-// crc32Field implements the pushEncoder and pushDecoder interfaces for calculating CRC32s.
-type crc32Field struct {
-	startOffset int
-	polynomial  crcPolynomial
-}
-
-func (c *crc32Field) saveOffset(in int) {
-	c.startOffset = in
-}
-
-func (c *crc32Field) reserveLength() int {
-	return 4
-}
-
-func newCRC32Field(polynomial crcPolynomial) *crc32Field {
-	return &crc32Field{polynomial: polynomial}
-}
-
-func (c *crc32Field) run(curOffset int, buf []byte) error {
-	crc, err := c.crc(curOffset, buf)
-	if err != nil {
-		return err
-	}
-	binary.BigEndian.PutUint32(buf[c.startOffset:], crc)
-	return nil
-}
-
-func (c *crc32Field) check(curOffset int, buf []byte) error {
-	crc, err := c.crc(curOffset, buf)
-	if err != nil {
-		return err
-	}
-
-	expected := binary.BigEndian.Uint32(buf[c.startOffset:])
-	if crc != expected {
-		return PacketDecodingError{fmt.Sprintf("CRC didn't match expected %#x got %#x", expected, crc)}
-	}
-
-	return nil
-}
-func (c *crc32Field) crc(curOffset int, buf []byte) (uint32, error) {
-	var tab *crc32.Table
-	switch c.polynomial {
-	case crcIEEE:
-		tab = crc32.IEEETable
-	case crcCastagnoli:
-		tab = castagnoliTable
-	default:
-		return 0, PacketDecodingError{"invalid CRC type"}
-	}
-	return crc32.Checksum(buf[c.startOffset+4:curOffset], tab), nil
-}

vendor/github.com/Shopify/sarama/create_partitions_request.go

@@ -1,125 +0,0 @@
-package sarama
-
-import "time"
-
-type CreatePartitionsRequest struct {
-	TopicPartitions map[string]*TopicPartition
-	Timeout         time.Duration
-	ValidateOnly    bool
-}
-
-func (c *CreatePartitionsRequest) encode(pe packetEncoder) error {
-	if err := pe.putArrayLength(len(c.TopicPartitions)); err != nil {
-		return err
-	}
-
-	for topic, partition := range c.TopicPartitions {
-		if err := pe.putString(topic); err != nil {
-			return err
-		}
-		if err := partition.encode(pe); err != nil {
-			return err
-		}
-	}
-
-	pe.putInt32(int32(c.Timeout / time.Millisecond))
-
-	pe.putBool(c.ValidateOnly)
-
-	return nil
-}
-
-func (c *CreatePartitionsRequest) decode(pd packetDecoder, version int16) (err error) {
-	n, err := pd.getArrayLength()
-	if err != nil {
-		return err
-	}
-	c.TopicPartitions = make(map[string]*TopicPartition, n)
-	for i := 0; i < n; i++ {
-		topic, err := pd.getString()
-		if err != nil {
-			return err
-		}
-		c.TopicPartitions[topic] = new(TopicPartition)
-		if err := c.TopicPartitions[topic].decode(pd, version); err != nil {
-			return err
-		}
-	}
-
-	timeout, err := pd.getInt32()
-	if err != nil {
-		return err
-	}
-	c.Timeout = time.Duration(timeout) * time.Millisecond
-
-	if c.ValidateOnly, err = pd.getBool(); err != nil {
-		return err
-	}
-
-	return nil
-}
-
-func (r *CreatePartitionsRequest) key() int16 {
-	return 37
-}
-
-func (r *CreatePartitionsRequest) version() int16 {
-	return 0
-}
-
-func (r *CreatePartitionsRequest) headerVersion() int16 {
-	return 1
-}
-
-func (r *CreatePartitionsRequest) requiredVersion() KafkaVersion {
-	return V1_0_0_0
-}
-
-type TopicPartition struct {
-	Count      int32
-	Assignment [][]int32
-}
-
-func (t *TopicPartition) encode(pe packetEncoder) error {
-	pe.putInt32(t.Count)
-
-	if len(t.Assignment) == 0 {
-		pe.putInt32(-1)
-		return nil
-	}
-
-	if err := pe.putArrayLength(len(t.Assignment)); err != nil {
-		return err
-	}
-
-	for _, assign := range t.Assignment {
-		if err := pe.putInt32Array(assign); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-func (t *TopicPartition) decode(pd packetDecoder, version int16) (err error) {
-	if t.Count, err = pd.getInt32(); err != nil {
-		return err
-	}
-
-	n, err := pd.getInt32()
-	if err != nil {
-		return err
-	}
-	if n <= 0 {
-		return nil
-	}
-	t.Assignment = make([][]int32, n)
-
-	for i := 0; i < int(n); i++ {
-		if t.Assignment[i], err = pd.getInt32Array(); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}

vendor/github.com/Shopify/sarama/create_partitions_response.go

@@ -1,109 +0,0 @@
-package sarama
-
-import (
-	"fmt"
-	"time"
-)
-
-type CreatePartitionsResponse struct {
-	ThrottleTime         time.Duration
-	TopicPartitionErrors map[string]*TopicPartitionError
-}
-
-func (c *CreatePartitionsResponse) encode(pe packetEncoder) error {
-	pe.putInt32(int32(c.ThrottleTime / time.Millisecond))
-	if err := pe.putArrayLength(len(c.TopicPartitionErrors)); err != nil {
-		return err
-	}
-
-	for topic, partitionError := range c.TopicPartitionErrors {
-		if err := pe.putString(topic); err != nil {
-			return err
-		}
-		if err := partitionError.encode(pe); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-func (c *CreatePartitionsResponse) decode(pd packetDecoder, version int16) (err error) {
-	throttleTime, err := pd.getInt32()
-	if err != nil {
-		return err
-	}
-	c.ThrottleTime = time.Duration(throttleTime) * time.Millisecond
-
-	n, err := pd.getArrayLength()
-	if err != nil {
-		return err
-	}
-
-	c.TopicPartitionErrors = make(map[string]*TopicPartitionError, n)
-	for i := 0; i < n; i++ {
-		topic, err := pd.getString()
-		if err != nil {
-			return err
-		}
-		c.TopicPartitionErrors[topic] = new(TopicPartitionError)
-		if err := c.TopicPartitionErrors[topic].decode(pd, version); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-func (r *CreatePartitionsResponse) key() int16 {
-	return 37
-}
-
-func (r *CreatePartitionsResponse) version() int16 {
-	return 0
-}
-
-func (r *CreatePartitionsResponse) headerVersion() int16 {
-	return 0
-}
-
-func (r *CreatePartitionsResponse) requiredVersion() KafkaVersion {
-	return V1_0_0_0
-}
-
-type TopicPartitionError struct {
-	Err    KError
-	ErrMsg *string
-}
-
-func (t *TopicPartitionError) Error() string {
-	text := t.Err.Error()
-	if t.ErrMsg != nil {
-		text = fmt.Sprintf("%s - %s", text, *t.ErrMsg)
-	}
-	return text
-}
-
-func (t *TopicPartitionError) encode(pe packetEncoder) error {
-	pe.putInt16(int16(t.Err))
-
-	if err := pe.putNullableString(t.ErrMsg); err != nil {
-		return err
-	}
-
-	return nil
-}
-
-func (t *TopicPartitionError) decode(pd packetDecoder, version int16) (err error) {
-	kerr, err := pd.getInt16()
-	if err != nil {
-		return err
-	}
-	t.Err = KError(kerr)
-
-	if t.ErrMsg, err = pd.getNullableString(); err != nil {
-		return err
-	}
-
-	return nil
-}

vendor/github.com/Shopify/sarama/create_topics_request.go

@@ -1,178 +0,0 @@
-package sarama
-
-import (
-	"time"
-)
-
-type CreateTopicsRequest struct {
-	Version int16
-
-	TopicDetails map[string]*TopicDetail
-	Timeout      time.Duration
-	ValidateOnly bool
-}
-
-func (c *CreateTopicsRequest) encode(pe packetEncoder) error {
-	if err := pe.putArrayLength(len(c.TopicDetails)); err != nil {
-		return err
-	}
-	for topic, detail := range c.TopicDetails {
-		if err := pe.putString(topic); err != nil {
-			return err
-		}
-		if err := detail.encode(pe); err != nil {
-			return err
-		}
-	}
-
-	pe.putInt32(int32(c.Timeout / time.Millisecond))
-
-	if c.Version >= 1 {
-		pe.putBool(c.ValidateOnly)
-	}
-
-	return nil
-}
-
-func (c *CreateTopicsRequest) decode(pd packetDecoder, version int16) (err error) {
-	n, err := pd.getArrayLength()
-	if err != nil {
-		return err
-	}
-
-	c.TopicDetails = make(map[string]*TopicDetail, n)
-
-	for i := 0; i < n; i++ {
-		topic, err := pd.getString()
-		if err != nil {
-			return err
-		}
-		c.TopicDetails[topic] = new(TopicDetail)
-		if err = c.TopicDetails[topic].decode(pd, version); err != nil {
-			return err
-		}
-	}
-
-	timeout, err := pd.getInt32()
-	if err != nil {
-		return err
-	}
-	c.Timeout = time.Duration(timeout) * time.Millisecond
-
-	if version >= 1 {
-		c.ValidateOnly, err = pd.getBool()
-		if err != nil {
-			return err
-		}
-
-		c.Version = version
-	}
-
-	return nil
-}
-
-func (c *CreateTopicsRequest) key() int16 {
-	return 19
-}
-
-func (c *CreateTopicsRequest) version() int16 {
-	return c.Version
-}
-
-func (r *CreateTopicsRequest) headerVersion() int16 {
-	return 1
-}
-
-func (c *CreateTopicsRequest) requiredVersion() KafkaVersion {
-	switch c.Version {
-	case 2:
-		return V1_0_0_0
-	case 1:
-		return V0_11_0_0
-	default:
-		return V0_10_1_0
-	}
-}
-
-type TopicDetail struct {
-	NumPartitions     int32
-	ReplicationFactor int16
-	ReplicaAssignment map[int32][]int32
-	ConfigEntries     map[string]*string
-}
-
-func (t *TopicDetail) encode(pe packetEncoder) error {
-	pe.putInt32(t.NumPartitions)
-	pe.putInt16(t.ReplicationFactor)
-
-	if err := pe.putArrayLength(len(t.ReplicaAssignment)); err != nil {
-		return err
-	}
-	for partition, assignment := range t.ReplicaAssignment {
-		pe.putInt32(partition)
-		if err := pe.putInt32Array(assignment); err != nil {
-			return err
-		}
-	}
-
-	if err := pe.putArrayLength(len(t.ConfigEntries)); err != nil {
-		return err
-	}
-	for configKey, configValue := range t.ConfigEntries {
-		if err := pe.putString(configKey); err != nil {
-			return err
-		}
-		if err := pe.putNullableString(configValue); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-func (t *TopicDetail) decode(pd packetDecoder, version int16) (err error) {
-	if t.NumPartitions, err = pd.getInt32(); err != nil {
-		return err
-	}
-	if t.ReplicationFactor, err = pd.getInt16(); err != nil {
-		return err
-	}
-
-	n, err := pd.getArrayLength()
-	if err != nil {
-		return err
-	}
-
-	if n > 0 {
-		t.ReplicaAssignment = make(map[int32][]int32, n)
-		for i := 0; i < n; i++ {
-			replica, err := pd.getInt32()
-			if err != nil {
-				return err
-			}
-			if t.ReplicaAssignment[replica], err = pd.getInt32Array(); err != nil {
-				return err
-			}
-		}
-	}
-
-	n, err = pd.getArrayLength()
-	if err != nil {
-		return err
-	}
-
-	if n > 0 {
-		t.ConfigEntries = make(map[string]*string, n)
-		for i := 0; i < n; i++ {
-			configKey, err := pd.getString()
-			if err != nil {
-				return err
-			}
-			if t.ConfigEntries[configKey], err = pd.getNullableString(); err != nil {
-				return err
-			}
-		}
-	}
-
-	return nil
-}

vendor/github.com/Shopify/sarama/create_topics_response.go

@@ -1,127 +0,0 @@
-package sarama
-
-import (
-	"fmt"
-	"time"
-)
-
-type CreateTopicsResponse struct {
-	Version      int16
-	ThrottleTime time.Duration
-	TopicErrors  map[string]*TopicError
-}
-
-func (c *CreateTopicsResponse) encode(pe packetEncoder) error {
-	if c.Version >= 2 {
-		pe.putInt32(int32(c.ThrottleTime / time.Millisecond))
-	}
-
-	if err := pe.putArrayLength(len(c.TopicErrors)); err != nil {
-		return err
-	}
-	for topic, topicError := range c.TopicErrors {
-		if err := pe.putString(topic); err != nil {
-			return err
-		}
-		if err := topicError.encode(pe, c.Version); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-func (c *CreateTopicsResponse) decode(pd packetDecoder, version int16) (err error) {
-	c.Version = version
-
-	if version >= 2 {
-		throttleTime, err := pd.getInt32()
-		if err != nil {
-			return err
-		}
-		c.ThrottleTime = time.Duration(throttleTime) * time.Millisecond
-	}
-
-	n, err := pd.getArrayLength()
-	if err != nil {
-		return err
-	}
-
-	c.TopicErrors = make(map[string]*TopicError, n)
-	for i := 0; i < n; i++ {
-		topic, err := pd.getString()
-		if err != nil {
-			return err
-		}
-		c.TopicErrors[topic] = new(TopicError)
-		if err := c.TopicErrors[topic].decode(pd, version); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-func (c *CreateTopicsResponse) key() int16 {
-	return 19
-}
-
-func (c *CreateTopicsResponse) version() int16 {
-	return c.Version
-}
-
-func (c *CreateTopicsResponse) headerVersion() int16 {
-	return 0
-}
-
-func (c *CreateTopicsResponse) requiredVersion() KafkaVersion {
-	switch c.Version {
-	case 2:
-		return V1_0_0_0
-	case 1:
-		return V0_11_0_0
-	default:
-		return V0_10_1_0
-	}
-}
-
-type TopicError struct {
-	Err    KError
-	ErrMsg *string
-}
-
-func (t *TopicError) Error() string {
-	text := t.Err.Error()
-	if t.ErrMsg != nil {
-		text = fmt.Sprintf("%s - %s", text, *t.ErrMsg)
-	}
-	return text
-}
-
-func (t *TopicError) encode(pe packetEncoder, version int16) error {
-	pe.putInt16(int16(t.Err))
-
-	if version >= 1 {
-		if err := pe.putNullableString(t.ErrMsg); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-func (t *TopicError) decode(pd packetDecoder, version int16) (err error) {
-	kErr, err := pd.getInt16()
-	if err != nil {
-		return err
-	}
-	t.Err = KError(kErr)
-
-	if version >= 1 {
-		if t.ErrMsg, err = pd.getNullableString(); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}

vendor/github.com/Shopify/sarama/decompress.go

@@ -1,63 +0,0 @@
-package sarama
-
-import (
-	"bytes"
-	"compress/gzip"
-	"fmt"
-	"io/ioutil"
-	"sync"
-
-	snappy "github.com/eapache/go-xerial-snappy"
-	"github.com/pierrec/lz4"
-)
-
-var (
-	lz4ReaderPool = sync.Pool{
-		New: func() interface{} {
-			return lz4.NewReader(nil)
-		},
-	}
-
-	gzipReaderPool sync.Pool
-)
-
-func decompress(cc CompressionCodec, data []byte) ([]byte, error) {
-	switch cc {
-	case CompressionNone:
-		return data, nil
-	case CompressionGZIP:
-		var (
-			err        error
-			reader     *gzip.Reader
-			readerIntf = gzipReaderPool.Get()
-		)
-		if readerIntf != nil {
-			reader = readerIntf.(*gzip.Reader)
-		} else {
-			reader, err = gzip.NewReader(bytes.NewReader(data))
-			if err != nil {
-				return nil, err
-			}
-		}
-
-		defer gzipReaderPool.Put(reader)
-
-		if err := reader.Reset(bytes.NewReader(data)); err != nil {
-			return nil, err
-		}
-
-		return ioutil.ReadAll(reader)
-	case CompressionSnappy:
-		return snappy.Decode(data)
-	case CompressionLZ4:
-		reader := lz4ReaderPool.Get().(*lz4.Reader)
-		defer lz4ReaderPool.Put(reader)
-
-		reader.Reset(bytes.NewReader(data))
-		return ioutil.ReadAll(reader)
-	case CompressionZSTD:
-		return zstdDecompress(nil, data)
-	default:
-		return nil, PacketDecodingError{fmt.Sprintf("invalid compression specified (%d)", cc)}
-	}
-}