Create probe.yaml

Adding a new flag `--input-kafka-offset` for supporting consumes from Kafka by specified offset.

.deepsource.toml

@@ -0,0 +1,28 @@
+version = 1
+
+exclude_patterns = [
+  "vendor/**"
+]
+
+[[analyzers]]
+name = "go"
+enabled = true
+
+  [analyzers.meta]
+  import_paths = ["github.com/ankitdobhal/goreplay"]
+
+[[analyzers]]
+name = "docker"
+enabled = true
+
+[[analyzers]]
+name = "ruby"
+enabled = true
+
+[[analyzers]]
+name = "javascript"
+enabled = true
+
+  [analyzers.meta]
+  environment = ["nodejs"]
+

.github/dependabot.yaml

@@ -0,0 +1,21 @@
+version: 2
+updates:
+  # Maintain dependencies for GitHub Actions
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+    pull-request-branch-name:
+      # Separate sections of the branch name with a hyphen
+      separator: "-"
+
+  - package-ecosystem: "gomod"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+      
+  - package-ecosystem: "docker"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+      

.github/workflows/ci-docker.yaml

@@ -0,0 +1,38 @@
+name: ci
+on:
+  release:
+    types: [published]
+jobs:
+  docker-build-and-push:
+    runs-on: ubuntu-latest
+    steps:
+      -
+        name: "Checkout repository"
+        uses: actions/checkout@v3
+      -
+        name: "Set up Docker Buildx"
+        uses: docker/setup-buildx-action@v2
+      -
+        name: "Cache Docker layers"
+        uses: actions/cache@v3.0.5
+        with:
+          path: /tmp/.buildx-cache
+          key: ${{ runner.os }}-buildx-${{ github.sha }}
+          restore-keys: |
+            ${{ runner.os }}-buildx-
+      -
+        name: "Login to Container Registry"
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      -
+        name: "Build and push docker image"
+        uses: docker/build-push-action@v3
+        with:
+          context: .
+          push: true
+          tags: ${{ github.repository }}:${{ github.event.release.tag_name }}
+          build-args: RELEASE_VERSION=${{ github.event.release.tag_name }}
+          cache-from: type=local,src=/tmp/.buildx-cache
+          cache-to: type=local,mode=max,dest=/tmp/.buildx-cache

.github/workflows/ci-test.yaml

@@ -0,0 +1,29 @@
+name: test
+on: [push, pull_request]
+jobs:
+  test:
+    strategy:
+      matrix:
+        go-version: [1.18.x, 1.19.x] # two latest minor versions
+    runs-on: ubuntu-latest
+    steps:
+    - name: update package index
+      run: sudo apt-get update
+    - name: install libpcap
+      run: sudo apt-get install libpcap-dev -y
+    - name: install Go
+      uses: actions/setup-go@v2
+      with:
+        go-version: ${{ matrix.go-version }}
+    - name: checkout code
+      uses: actions/checkout@v3
+    - uses: actions/cache@v3.0.5
+      with:
+        path: |
+          ~/go/pkg/mod              # Module download cache
+          ~/.cache/go-build         # Build cache (Linux)
+        key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
+        restore-keys: |
+          ${{ runner.os }}-go-
+    - name: test
+      run: sudo go test ./... -v -timeout 120s

.github/workflows/codesee-arch-diagram.yml

@@ -0,0 +1,23 @@
+# This workflow was added by CodeSee. Learn more at https://codesee.io/
+# This is v2.0 of this workflow file
+on:
+  push:
+    branches:
+      - master
+  pull_request_target:
+    types: [opened, synchronize, reopened]
+
+name: CodeSee
+
+permissions: read-all
+
+jobs:
+  codesee:
+    runs-on: ubuntu-latest
+    continue-on-error: true
+    name: Analyze the repo with CodeSee
+    steps:
+      - uses: Codesee-io/codesee-action@v2
+        with:
+          codesee-token: ${{ secrets.CODESEE_ARCH_DIAG_API_TOKEN }}
+          codesee-url: https://app.codesee.io

.github/workflows/probe.yaml

@@ -0,0 +1,38 @@
+name: AI Comment Handler
+
+
+on:
+  pull_request:
+    types: [opened] #[opened , labeled]
+  issue_comment:
+    types: [created]          
+  issues:
+    types: [opened] #[opened, labeled]  
+    
+# Define permissions needed for the workflow
+permissions:
+  issues: write
+  pull-requests: write
+  contents: read
+
+jobs:
+  trigger_probe_chat:
+    # Uncomment if you want to run on on specific lables, in this example `probe`
+    # if: |
+    #   (github.event_name == 'pull_request' && github.event.action == 'opened') || 
+    #   (github.event_name == 'issues' && github.event.action == 'opened') || 
+    #   (github.event_name == 'issue_comment' && github.event.action == 'created') || 
+    #   ((github.event_name == 'pull_request' || github.event_name == 'issues') && 
+    #    github.event.action == 'labeled' && github.event.label.name == 'probe')
+    # Use the reusable workflow from your repository (replace <your-org/repo> and <ref>)
+    uses: buger/probe/.github/workflows/probe.yml@main
+    # Pass required inputs
+    with:
+      command_prefix: "/probe" # Or '/ai', '/ask', etc.
+      # Optionally override the default npx command if the secret isn't set
+      # default_probe_chat_command: 'node path/to/custom/script.js'
+    # Pass ALL secrets from this repository to the reusable workflow
+    # This includes GITHUB_TOKEN, PROBE_CHAT_COMMAND (if set), ANTHROPIC_API_KEY, etc.
+    secrets:
+      ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
+      ANTHROPIC_API_URL: ${{ secrets.ANTHROPIC_API_URL }}

.github/workflows/semgrep.yml

@@ -0,0 +1,23 @@
+on:
+  pull_request: {}
+  push:
+    branches:
+    - main
+    - master
+    paths:
+    - .github/workflows/semgrep.yml
+  schedule:
+  # random HH:MM to avoid a load spike on GitHub Actions at 00:00
+  - cron: 53 23 * * *
+name: Semgrep
+jobs:
+  semgrep:
+    name: Scan
+    runs-on: ubuntu-20.04
+    env:
+      SEMGREP_APP_TOKEN: ${{ secrets.SEMGREP_APP_TOKEN }}
+    container:
+      image: returntocorp/semgrep
+    steps:
+    - uses: actions/checkout@v3
+    - run: semgrep ci

.gitignore

@@ -1,15 +1,21 @@
+vendor
 *.swp
 *.gor
 *.rpm
 *.dep
+*.deb
 *.pkg
-
+*.exe
+*.pprof
 *.out
+hey
 
 *.bin
-
+lib/
+output/
 *.gz
 *.zip
+.aider*
 
 *.class
 
@@ -28,3 +34,4 @@ goreplay
 corpus
 crashers
 suppressions
+dist

.travis.yml

@@ -1,6 +0,0 @@
-language: go
-go: 1.14
-script: sudo -E bash -c "source /etc/profile && eval '$(gimme 1.14)' && export GOPATH=$HOME/gopath:$GOPATH && go get && GORACE='halt_on_error=1' go test ./... -v -timeout 120s -race"
-
-before_install:
-  - sudo apt-get install libpcap-dev -y

COMM-LICENSE

@@ -2,7 +2,7 @@ END-USER LICENSE AGREEMENT
 
 ------------------------------------------------------------------------------
 
-IMPORTANT: THIS SOFTWARE END-USER LICENSE AGREEMENT ("EULA") IS A LEGAL AGREEMENT (“Agreement”) BETWEEN YOU (THE CUSTOMER, EITHER AS AN INDIVIDUAL OR, IF PURCHASED OR OTHERWISE ACQUIRED BY OR FOR AN ENTITY, AS AN ENTITY) AND GoReplay LLC. READ IT CAREFULLY BEFORE COMPLETING THE INSTALLATION PROCESS AND USING GOREPLAY PRO AND RELATED SOFTWARE COMPONENTS (“SOFTWARE”). IT PROVIDES A LICENSE TO USE THE SOFTWARE AND CONTAINS WARRANTY INFORMATION AND LIABILITY DISCLAIMERS. BY INSTALLING AND USING THE SOFTWARE, YOU ARE CONFIRMING YOUR ACCEPTANCE OF THE SOFTWARE AND AGREEING TO BECOME BOUND BY THE TERMS OF THIS AGREEMENT.
+IMPORTANT: THIS SOFTWARE END-USER LICENSE AGREEMENT ("EULA") IS A LEGAL AGREEMENT (“Agreement”) BETWEEN YOU (THE CUSTOMER, EITHER AS AN INDIVIDUAL OR, IF PURCHASED OR OTHERWISE ACQUIRED BY OR FOR AN ENTITY, AS AN ENTITY), hereafter "Customer", AND Replay Software LLC ("Licensor"). READ IT CAREFULLY BEFORE COMPLETING THE INSTALLATION PROCESS AND USING GOREPLAY PRO AND RELATED SOFTWARE COMPONENTS (“SOFTWARE”). IT PROVIDES A LICENSE TO USE THE SOFTWARE AND CONTAINS WARRANTY INFORMATION AND LIABILITY DISCLAIMERS. BY INSTALLING AND USING THE SOFTWARE, YOU ARE CONFIRMING YOUR ACCEPTANCE OF THE SOFTWARE AND AGREEING TO BECOME BOUND BY THE TERMS OF THIS AGREEMENT.
 
 ------------------------------------------------------------------------------
 
@@ -14,17 +14,17 @@ In order to use the Software under this Agreement, you must receive a “Source
 
 1.2 Unlimited Organization License. If you purchased an Organization License (included with the GoReplay Pro Software), you may install the Software on an unlimited number of Hosts. “Host” means any physical or virtual machine which is controlled by you. You may concurrently run the software on an unlimited number of Hosts.
 
-1.3 Appliance License. If you purchased an Appliance License, you may distribute the Software in any applications, frameworks, or elements (collectively referred to as an “Application” or “Applications”) that you develop using the Software in accordance with this EULA, provided that such distribution does not violate the restrictions set forth in section 3 of this EULA. You must not remove, obscure or interfere with any copyright, acknowledgment, attribution, trademark, warning or disclaimer statement affixed to, incorporated in or otherwise applied in connection with the Software. You are required to ensure that the Software is not reused by or with any applications other than those with which you distribute it as permitted herein. For example, if You install the Software on a customer’s server, that customer is not permitted to use the Software independently of your Application. You must inform GoReplay LLC of your knowledge of any infringing use of the Software by any of your customers. You are liable for compliance by those third parties with the terms and conditions of this EULA. You will not owe GoReplay LLC any royalties for your distribution of the Software in accordance with this EULA.
+1.3 Appliance License. If you purchased an Appliance License, you may distribute the Software in any applications, frameworks, or elements (collectively referred to as an “Application” or “Applications”) that you develop using the Software in accordance with this EULA, provided that such distribution does not violate the restrictions set forth in section 3 of this EULA. You must not remove, obscure or interfere with any copyright, acknowledgment, attribution, trademark, warning or disclaimer statement affixed to, incorporated in or otherwise applied in connection with the Software. You are required to ensure that the Software is not reused by or with any applications other than those with which you distribute it as permitted herein. For example, if You install the Software on a customer’s server, that customer is not permitted to use the Software independently of your Application. You must inform Licensor of your knowledge of any infringing use of the Software by any of your customers. You are liable for compliance by those third parties with the terms and conditions of this EULA. You will not owe Licensor any royalties for your distribution of the Software in accordance with this EULA.
 
 1.4 Archive Copies. You are entitled to make a reasonable amount of copies of the Software for archival purposes. Each copy must reproduce all copyright and other proprietary rights notices on or in the Software Product.
 
 1.5 Electronic Delivery. All Software and license documentation shall be delivered by electronic means unless otherwise specified on the applicable invoice or at the time of purchase. Software shall be deemed delivered when it is made available for download by you (“Delivery”).        
 
-2. Modifications. GoReplay LLC shall provide you with source code so that you can create Modifications of the original software. “Modification” means: (a) any addition to or deletion from the contents of a file included in the original Software or previous Modifications created by You, or (b) any new file that contains any part of the original Software or previous Modifications. While you retain all rights to any original work authored by you as part of the Modifications, We continue to own all copyright and other intellectual property rights in the Software.
+2. Modifications. Licensor shall provide you with source code so that you can create Modifications of the original software. “Modification” means: (a) any addition to or deletion from the contents of a file included in the original Software or previous Modifications created by You, or (b) any new file that contains any part of the original Software or previous Modifications. While you retain all rights to any original work authored by you as part of the Modifications, We continue to own all copyright and other intellectual property rights in the Software.
 
 3. Restricted Uses. 
 
-3.1 You shall not (and shall not allow any third party to): (a) decompile, disassemble, or otherwise reverse engineer the Software or attempt to reconstruct or discover any source code, underlying ideas, algorithms, file formats or programming interfaces of the Software by any means whatsoever (except and only to the extent that applicable law prohibits or restricts reverse engineering restrictions); (b) distribute, sell, sublicense, rent, lease or use the Software for time sharing, hosting, service provider or like purposes, except as expressly permitted under this Agreement; (c) redistribute the Software or Modifications other than by including the Software or a portion thereof within your own product, which must have substantially different functionality than the Software or Modifications and must not allow any third party to use the Software or Modifications, or any portions thereof, for software development or application development purposes; (d) redistribute the Software as part of a product, "appliance" or "virtual server"; (e) redistribute the Software on any server which is not directly under your control; (f) remove any product identification, proprietary, copyright or other notices contained in the Software; (g) modify any part of the Software, create a derivative work of any part of the Software (except as permitted in Section 4), or incorporate the Software, except to the extent expressly authorized in writing by GoReplay LLC; (h) publicly disseminate performance information or analysis (including, without limitation, benchmarks) from any source relating to the Software; (i) utilize any equipment, device, software, or other means designed to circumvent or remove any form of Source URL or copy protection used by GoReplay LLC in connection with the Software, or use the Software together with any authorization code, Source URL, serial number, or other copy protection device not supplied by GoReplay LLC; (j) use the Software to develop a product which is competitive with any GoReplay LLC product offerings; or (k) use unauthorized Source URLS or keycode(s) or distribute or publish Source URLs or keycode(s), except as may be expressly permitted by GoReplay LLC in writing. If your unique Source URL is ever published, GoReplay LLC reserves the right to terminate your access without notice.
+3.1 You shall not (and shall not allow any third party to): (a) decompile, disassemble, or otherwise reverse engineer the Software or attempt to reconstruct or discover any source code, underlying ideas, algorithms, file formats or programming interfaces of the Software by any means whatsoever (except and only to the extent that applicable law prohibits or restricts reverse engineering restrictions); (b) distribute, sell, sublicense, rent, lease or use the Software for time sharing, hosting, service provider or like purposes, except as expressly permitted under this Agreement; (c) redistribute the Software or Modifications other than by including the Software or a portion thereof within your own product, which must have substantially different functionality than the Software or Modifications and must not allow any third party to use the Software or Modifications, or any portions thereof, for software development or application development purposes; (d) redistribute the Software as part of a product, "appliance" or "virtual server"; (e) redistribute the Software on any server which is not directly under your control; (f) remove any product identification, proprietary, copyright or other notices contained in the Software; (g) modify any part of the Software, create a derivative work of any part of the Software (except as permitted in Section 4), or incorporate the Software, except to the extent expressly authorized in writing by Licensor; (h) publicly disseminate performance information or analysis (including, without limitation, benchmarks) from any source relating to the Software; (i) utilize any equipment, device, software, or other means designed to circumvent or remove any form of Source URL or copy protection used by Licensor in connection with the Software, or use the Software together with any authorization code, Source URL, serial number, or other copy protection device not supplied by Licensor; (j) use the Software to develop a product which is competitive with any Licensor product offerings; or (k) use unauthorized Source URLS or keycode(s) or distribute or publish Source URLs or keycode(s), except as may be expressly permitted by Licensor in writing. If your unique Source URL is ever published, Licensor reserves the right to terminate your access without notice.
 
 3.2 UNDER NO CIRCUMSTANCES MAY YOU USE THE SOFTWARE AS PART OF A PRODUCT OR SERVICE THAT PROVIDES SIMILAR FUNCTIONALITY TO THE SOFTWARE ITSELF.
 
@@ -32,35 +32,33 @@ The Open Source version of the Software (“LGPL Version”) is licensed
 under the terms of the GNU Lesser General Public License versions 3.0
 (“LGPL”) and not under this EULA. 
 
-4. Ownership. Notwithstanding anything to the contrary contained herein, except for the limited license rights expressly provided herein, GoReplay LLC and its suppliers have and will retain all rights, title and interest (including, without limitation, all patent, copyright, trademark, trade secret and other intellectual property rights) in and to the Software and all copies, modifications and derivative works thereof (including any changes which incorporate any of your ideas, feedback or suggestions). You acknowledge that you are obtaining only a limited license right to the Software, and that irrespective of any use of the words “purchase”, “sale” or like terms hereunder no ownership rights are being conveyed to you under this Agreement or otherwise. 
+4. Ownership. Notwithstanding anything to the contrary contained herein, except for the limited license rights expressly provided herein, Licensor and its suppliers have and will retain all rights, title and interest (including, without limitation, all patent, copyright, trademark, trade secret and other intellectual property rights) in and to the Software and all copies, modifications and derivative works thereof (including any changes which incorporate any of your ideas, feedback or suggestions). You acknowledge that you are obtaining only a limited license right to the Software, and that irrespective of any use of the words “purchase”, “sale” or like terms hereunder no ownership rights are being conveyed to you under this Agreement or otherwise. 
 
-5. Fees and Payment. The Software license fees will be due and payable in full as set forth in the applicable invoice or at the time of purchase. If the Software does not function properly within two weeks of purchase, please contact us within those two weeks for a refund. You shall be responsible for all taxes, withholdings, duties and levies arising from the order (excluding taxes based on the net income of GoReplay LLC). 
+5. Fees and Payment. The Software license fees will be due and payable in full as set forth in the applicable invoice or at the time of purchase. If the Software does not function properly within two weeks of purchase, please contact us within those two weeks for a refund. You shall be responsible for all taxes, withholdings, duties and levies arising from the order (excluding taxes based on the net income of Licensor). 
 
 6. Support, Maintenance and Services. Subject to the terms and conditions of this Agreement, as set forth in your invoice, and as set forth on the GoReplay Pro support page (https://github.com/buger/goreplay/wiki/Pro-Support), support and maintenance services may be included with the purchase of your license subscription.
 
 7. Term of Agreement. 
 
-7.1 Term. This Agreement is effective as of the Delivery of the Software and expires at such time as all license and service subscriptions hereunder have expired in accordance with their own terms (the “Term”). For clarification, the term of your license under this Agreement may be perpetual, limited for Evaluation Version, or designated as a fixed-term license in the Invoice, and shall be specified at your time of purchase. Either party may terminate this Agreement (including all related Invoices) if the other party: (a) fails to cure any material breach of this Agreement within thirty (30) days after written notice of such breach, provided that GoReplay LLC may terminate this Agreement immediately upon any breach of Section 3 or if you exceed any other restrictions contained in Section 1, unless otherwise specified in this agreement; (b) ceases operation without a successor; or (c) seeks protection under any bankruptcy, receivership, trust deed, creditors arrangement, composition or comparable proceeding, or if any such proceeding is instituted against such party (and not dismissed within sixty (60) days)). Termination is not an exclusive remedy and the exercise by either party of any remedy under this Agreement will be without prejudice to any other remedies it may have under this Agreement, by law, or otherwise.        
+7.1 Term. This Agreement is effective as of the Delivery of the Software and expires at such time as all license and service subscriptions hereunder have expired in accordance with their own terms (the “Term”). For clarification, the term of your license under this Agreement may be perpetual, limited for Evaluation Version, or designated as a fixed-term license in the Invoice, and shall be specified at your time of purchase. Either party may terminate this Agreement (including all related Invoices) if the other party: (a) fails to cure any material breach of this Agreement within thirty (30) days after written notice of such breach, provided that Licensor may terminate this Agreement immediately upon any breach of Section 3 or if you exceed any other restrictions contained in Section 1, unless otherwise specified in this agreement; (b) ceases operation without a successor; or (c) seeks protection under any bankruptcy, receivership, trust deed, creditors arrangement, composition or comparable proceeding, or if any such proceeding is instituted against such party (and not dismissed within sixty (60) days)). Termination is not an exclusive remedy and the exercise by either party of any remedy under this Agreement will be without prejudice to any other remedies it may have under this Agreement, by law, or otherwise.        
 
 7.2 Termination. Upon any termination of this Agreement, you shall cease any and all use of any Software and destroy all copies thereof. 
 
-7.3 Expiration of License. Upon the expiration of any term under this Agreement, (a) all Software updates and services pursuant to the license shall cease, (b) you may only continue to run existing installations of the Software, (c) you may not install the Software on any additional Hosts, and (d) any new installation of the Software shall require the purchase of a new license subscription from GoReplay LLC.
+7.3 Expiration of License. Upon the expiration of any term under this Agreement, (a) all Software updates and services pursuant to the license shall cease, (b) you may only continue to run existing installations of the Software, (c) you may not install the Software on any additional Hosts, and (d) any new installation of the Software shall require the purchase of a new license subscription from Licensor.
 
-8. Disclaimer of Warranties. The Software is provided "as is," with all faults, defects and errors, and without warranty of any kind. GoReplay LLC does not warrant that the Software will be free of bugs, errors, viruses or other defects, and GoReplay LLC shall have no liability of any kind for the use of or inability to use the Software, the Software content or any associated service, and you acknowledge that it is not technically practicable for GoReplay LLC to do so. 
-To the maximum extent permitted by applicable law, GoReplay LLC disclaims all warranties, express, implied, arising by law or otherwise, regarding the Software, the Software content and their respective performance or suitability for your intended use, including without limitation any implied warranty of merchantability, fitness for a particular purpose.
-Notwithstanding the foregoing, GoReplay LLC represents and warrants that it either owns the entire right to, title to, interest in, or has the right to license, the Software, and your proper use of the same will not violate any intellectual property or proprietary rights of another.
+8. Disclaimer of Warranties. The Software is provided "as is," with all faults, defects and errors, and without warranty of any kind. Licensor does not warrant that the Software will be free of bugs, errors, viruses or other defects, and Licensor shall have no liability of any kind for the use of or inability to use the Software, the Software content or any associated service, and you acknowledge that it is not technically practicable for Licensor to do so. 
+To the maximum extent permitted by applicable law, Licensor disclaims all warranties, express, implied, arising by law or otherwise, regarding the Software, the Software content and their respective performance or suitability for your intended use, including without limitation any implied warranty of merchantability, fitness for a particular purpose.
+Notwithstanding the foregoing, Licensor represents and warrants that it either owns the entire right to, title to, interest in, or has the right to license, the Software, and your proper use of the same will not violate any intellectual property or proprietary rights of another.
 
 9. Limitation of Liability. 
 
-In no event will either party be liable for any direct, indirect, consequential, incidental, special, exemplary, or punitive damages or liabilities whatsoever arising from or relating to the Software, the Software content or this Agreement, whether based on contract, tort (including negligence), strict liability or other theory, even if such party has been advised of the possibility of such damages.
+In no event will either party be liable for any direct, indirect, consequential, incidental, special, exemplary, or punitive damages or liabilities whatsoever arising from or relating to the Software, the Software content or this Agreement, whether based on contract, tort (including negligence), strict liability or other theory, even if such party has been advised of the possibility of such damages. Except for indemnification obligations in Section 13.5, in no event will Licensor liability exceed the Software license price as indicated in the invoice. The existence of more than one claim will not enlarge or extend this limit.
 
-In no event will GoReplay LLC liability exceed the Software license price as indicated in the invoice. The existence of more than one claim will not enlarge or extend this limit.
-
-10. Remedies. Your exclusive remedy and GoReplay LLC’ entire liability for breach of this Agreement shall be limited, at GoReplay LLC’ sole and exclusive discretion, to (a) replacement of any defective software or documentation; or (b) refund of the license fee paid to GoReplay LLC, payable in accordance with GoReplay LLC' refund policy.
+10. Remedies. Your exclusive remedy and Licensor’ entire liability for breach of this Agreement shall be limited, at Licensor’ sole and exclusive discretion, to (a) replacement of any defective software or documentation; or (b) refund of the license fee paid to Licensor, payable in accordance with Licensor' refund policy.
 
 11. Acknowledgements.
 
-11.1 Consent to the Use of Data. You agree that GoReplay LLC and its affiliates may collect and use technical information gathered as part of the product support services to be used in an anonymized manner. GoReplay LLC may use this information solely to improve products and services and will not disclose this information in a form that personally identifies you.
+11.1 Consent to the Use of Data. You agree that Licensor and its affiliates may collect and use technical information gathered as part of the product support services to be used in an anonymized manner. Licensor may use this information solely to improve products and services and will not disclose this information in a form that personally identifies you.
 
 11.2 Verification. We or a certified auditor acting on our behalf, may, upon its reasonable request and at its expense, audit you with respect to the use of the Software, subject to your reasonable information security and confidentiality protections. Such audit may be conducted by mail, electronic means or through an in-person visit to your place of business. Any such in-person audit shall be conducted during regular business hours at your facilities and shall not unreasonably interfere with your business activities. We shall not remove, copy, or redistribute any electronic material during the course of an audit. If an audit reveals that you are using the Software in a way that is in material violation of the terms of the EULA, then you shall pay our reasonable costs of conducting the audit. In the case of a material violation, you agree to pay Us any amounts owing that are attributable to the unauthorized use. In the alternative, We reserve the right, at our sole option, to terminate the licenses for the Software.
 
@@ -73,13 +71,13 @@ In no event will GoReplay LLC liability exceed the Software license price as ind
 
 13.1 Entire Agreement. This Agreement sets forth our entire agreement with respect to the Software and the subject matter hereof and supersedes all prior and contemporaneous understandings and agreements whether written or oral.
 
-13.2 Amendment. GoReplay LLC reserves the right, in its sole discretion, to amend this Agreement from time. Amendments to this Agreement can be located at: https://github.com/buger/goreplay/blob/master/COMM-LICENSE.
+13.2 Amendment. Licensor reserves the right, in its sole discretion, to amend this Agreement from time. Amendments to this Agreement can be located at: https://github.com/buger/goreplay/blob/master/COMM-LICENSE.
 
-13.3 Assignment. You may not assign this Agreement or any of its rights under this Agreement without the prior written consent of GoReplay LLC and any attempted assignment without such consent shall be void.
+13.3 Assignment. You may not assign this Agreement or any of its rights under this Agreement without the prior written consent of Licensor and any attempted assignment without such consent shall be void.
 
 13.4 Export Compliance. Each party agrees to comply with all applicable laws and regulations, including laws, regulations, orders or other restrictions on export, re-export or redistribution of software.
 
-13.5 Indemnification. You agree to defend, indemnify, and hold harmless GoReplay LLC from and against any lawsuits, claims, losses, damages, fines and expenses (including attorneys' fees and costs) arising out of your use of the Software or breach of this Agreement.
+13.5 Indemnification. Licensor agrees to defend, indemnify, and hold harmless You from and against any lawsuits, claims, losses, damages, fines and expenses (including attorneys' fees and costs) arising out of a claim brought by a third-party alleging that the Software directly infringes such third-party’s intellectual property rights; provided that Licensor shall have no indemnity obligation for claims arising out of your modification to the Software. You agree to defend, indemnify, and hold harmless Licensor from and against any lawsuits, claims, losses, damages, fines and expenses (including attorneys' fees and costs) brought by a third-party arising out of your modifications to the Software or breach of this Agreement.
 
 13.6 Governing Law. This Agreement is governed by the laws of the State of Oregon and the United States without regard to conflicts of laws provisions thereof, and without regard to the United Nations Convention on the International Sale of Goods or the Uniform Computer Information Transactions Act, as currently enacted by any jurisdiction or as may be codified or amended from time to time by any jurisdiction. The jurisdiction and venue for actions related to the subject matter hereof shall be the state of Oregon and United States federal courts located in Portland, Oregon, and both parties hereby submit to the personal jurisdiction of such courts. 
 
@@ -91,4 +89,4 @@ In no event will GoReplay LLC liability exceed the Software license price as ind
 
 13.10 Headings. The headings of sections and paragraphs of this Agreement are for convenience of reference only and are not intended to restrict, affect or be of any weight in the interpretation or construction of the provisions of such sections or paragraphs.
 
-14. Contact Information. If you have any questions about this EULA, or if you want to contact GoReplay LLC for any reason, please direct correspondence to info@gortool.com
+14. Contact Information. If you have any questions about this EULA, or if you want to contact Licensor for any reason, please direct correspondence to hello@goreplay.org

Dockerfile

@@ -1,8 +1,12 @@
-FROM alpine:latest as builder
+FROM alpine:3.16 as builder
+
+ARG RELEASE_VERSION
+
 RUN apk add --no-cache ca-certificates openssl
-RUN wget https://github.com/buger/goreplay/releases/download/v1.0.0/gor_1.0.0_x64.tar.gz -O gor.tar.gz
+RUN wget https://github.com/buger/goreplay/releases/download/${RELEASE_VERSION}/gor_${RELEASE_VERSION}_x64.tar.gz -O gor.tar.gz
 RUN tar xzf gor.tar.gz
 
 FROM scratch
+COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
 COPY --from=builder /gor .
 ENTRYPOINT ["./gor"]

Dockerfile.dev

@@ -1,14 +1,20 @@
-FROM golang:1.14
+ARG BASE_IMAGE
+FROM ${BASE_IMAGE}
 
-RUN apt-get update && apt-get install ruby vim-common -y
+RUN apk add --no-cache \
+    gcc \
+    g++ \
+    make \
+    linux-headers \
+    bison \
+    flex \
+    git \
+    wget
 
-RUN apt-get install flex bison -y
-RUN wget http://www.tcpdump.org/release/libpcap-1.9.1.tar.gz && tar xzf libpcap-1.9.1.tar.gz && cd libpcap-1.9.1 && ./configure && make install
-
-RUN go get github.com/google/gopacket
-RUN go get -u golang.org/x/lint/golint
+RUN wget http://www.tcpdump.org/release/libpcap-1.10.0.tar.gz && tar xzf libpcap-1.10.0.tar.gz && cd libpcap-1.10.0 && ./configure && make install
 
 WORKDIR /go/src/github.com/buger/goreplay/
 ADD . /go/src/github.com/buger/goreplay/
 
+RUN go get golang.org/x/lint/golint
 RUN go get

LICENSE.txt

@@ -3,7 +3,7 @@ Copyright (c) 2011-present Leonid Bugaev
 Portions of this software are licensed as follows:
 
 * All content residing under the "doc/" directory of this repository is licensed under "Creative Commons: CC BY-SA 4.0 license".
-* Files with the following names "pro.go", "s3_reader.go", "output_s3.go" are released under the commercial license specified in "COMM-LICENSE" file.
+* The file "pro.go" and all files ending with the "_pro.go" suffix are released under the commercial license specified in the "COMM-LICENSE" file.
 * Content outside of the above mentioned directories or restrictions above is available under the "LGPLv3" license as defined below.
 
 

Makefile

@@ -1,65 +1,102 @@
 SOURCE = $(shell ls -1 *.go | grep -v _test.go)
+PROJECT_NAME := goreplay
 SOURCE_PATH = /go/src/github.com/buger/goreplay/
 PORT = 8000
 FADDR = :8000
-CONTAINER=gor
-PREFIX=
-RUN = docker run -v `pwd`:$(SOURCE_PATH) -e AWS_ACCESS_KEY_ID=$(AWS_ACCESS_KEY_ID) -e AWS_SECRET_ACCESS_KEY=$(AWS_SECRET_ACCESS_KEY) -p 0.0.0.0:$(PORT):$(PORT) -t -i $(CONTAINER)
+DIST_PATH = dist
+CONTAINER_AMD=gor-amd64
+CONTAINER_ARM=gor-arm64
+RUN = docker run --rm -v `pwd`:$(SOURCE_PATH) -e AWS_ACCESS_KEY_ID=$(AWS_ACCESS_KEY_ID) -e AWS_SECRET_ACCESS_KEY=$(AWS_SECRET_ACCESS_KEY) -p 0.0.0.0:$(PORT):$(PORT) -t -i $(CONTAINER_AMD)
 BENCHMARK = BenchmarkRAWInput
 TEST = TestRawListenerBench
-VERSION = DEV-$(shell date +%s)
-LDFLAGS = -ldflags "-X main.VERSION=$(VERSION)$(PREFIX) -extldflags \"-static\" -X main.DEMO=$(DEMO)"
-MAC_LDFLAGS = -ldflags "-X main.VERSION=$(VERSION)$(PREFIX) -X main.DEMO=$(DEMO)"
-FADDR = ":8000"
+BIN_NAME = gor
+VERSION := DEV-$(shell date +%s)
+CUSTOM_TAGS := --tags "ngo$(if $(CUSTOM_BUILD_TAGS), $(CUSTOM_BUILD_TAGS),)"
+LDFLAGS = -ldflags "-X main.VERSION=$(VERSION) -extldflags \"-static\" -X main.DEMO=$(DEMO)"
+MAC_LDFLAGS = -ldflags "-X main.VERSION=$(VERSION) -X main.DEMO=$(DEMO)"
+DOCKER_FPM_CMD := docker run --rm -t -v `pwd`:/src -w /src fleetdm/fpm
 
-FPMCOMMON= \
-    --name goreplay \
-    --description "GoReplay is an open-source network monitoring tool which can record your live traffic, and use it for shadowing, load testing, monitoring and detailed analysis." \
-    -v $(VERSION) \
-    --vendor "Leonid Bugaev" \
-    -m "<support@goreplay.org>" \
-    --url "https://goreplay.org" \
-    -s dir \
-    -C /tmp/gor-build \
+FPM_COMMON= \
+				--name $(PROJECT_NAME) \
+				--description "GoReplay is an open-source network monitoring tool which can record your live traffic, and use it for shadowing, load testing, monitoring and detailed analysis." \
+				-v $(VERSION) \
+				--vendor "Leonid Bugaev" \
+				-m "<support@goreplay.org>" \
+				--url "https://goreplay.org" \
+				-s dir
 
-release: release-x64 release-mac
+release: clean release-linux-amd64 release-linux-arm64 release-mac-amd64 release-mac-arm64 release-windows
 
-release-bin:
-	docker run -v `pwd`:$(SOURCE_PATH) -t --env GOOS=linux --env GOARCH=amd64  -i gor go build -o gor -tags netgo $(LDFLAGS)
+.PHONY: vendor
+vendor:
+	go mod vendor
 
-release-x64:
-	docker run -v `pwd`:$(SOURCE_PATH) -t --env GOOS=linux --env GOARCH=amd64  -i $(CONTAINER) go build -o gor -tags netgo $(LDFLAGS)
-	tar -czf gor_$(VERSION)$(PREFIX)_x64.tar.gz gor
-	mkdir -p /tmp/gor-build
-	mv ./gor /tmp/gor-build/gor
-	cd /tmp/gor-build
-	fpm $(FPMCOMMON) -a amd64 -t deb ./=/usr/local/bin
-	fpm $(FPMCOMMON) -a amd64 -t rpm ./=/usr/local/bin
-	rm -rf /tmp/gor-build
+release-bin-linux-amd64: vendor
+	docker run --platform linux/amd64 --rm -v `pwd`:$(SOURCE_PATH) -t --env GOOS=linux --env GOARCH=amd64 -i $(CONTAINER_AMD) go build -mod=vendor -o $(BIN_NAME) $(CUSTOM_TAGS) $(LDFLAGS) ./cmd/gor/
 
-release-x86:
-	docker run -v `pwd`:$(SOURCE_PATH) -t --env GOOS=linux --env GOARCH=386 -i $(CONTAINER) go build -o gor -tags netgo $(LDFLAGS)
-	tar -czf gor_$(VERSION)$(PREFIX)_x86.tar.gz gor
-	rm gor
+release-bin-linux-arm64: vendor
+	docker run --platform linux/arm64 --rm -v `pwd`:$(SOURCE_PATH) -t --env GOOS=linux --env GOARCH=arm64 -i $(CONTAINER_ARM) go build -mod=vendor -o $(BIN_NAME) $(CUSTOM_TAGS) $(LDFLAGS) ./cmd/gor/
 
-release-mac:
-	go build -o gor $(MAC_LDFLAGS)
-	tar -czf gor_$(VERSION)$(PREFIX)_mac.tar.gz gor
-	mkdir -p /tmp/gor-build
-	mv ./gor /tmp/gor-build/gor
-	cd /tmp/gor-build
-	fpm $(FPMCOMMON) -a amd64 -t osxpkg ./=/usr/local/bin
-	rm -rf /tmp/gor-build
+release-bin-mac-amd64: vendor
+	GOOS=darwin go build -mod=vendor -o $(BIN_NAME) $(CUSTOM_TAGS) $(MAC_LDFLAGS) ./cmd/gor/
 
+release-bin-mac-arm64: vendor
+	GOOS=darwin GOARCH=arm64 CGO_ENABLED=1 go build -mod=vendor -o $(BIN_NAME) $(CUSTOM_TAGS) $(MAC_LDFLAGS)
 
-install:
-	go install $(MAC_LDFLAGS)
+release-bin-windows: vendor
+	docker run -it --rm -v `pwd`:$(SOURCE_PATH) -w $(SOURCE_PATH) -e CGO_ENABLED=1 docker.elastic.co/beats-dev/golang-crossbuild:1.19.2-main --build-cmd "make VERSION=$(VERSION) CUSTOM_BUILD_TAGS=$(CUSTOM_BUILD_TAGS) build" -p "windows/amd64" ./cmd/gor/
+	mv $(BIN_NAME) "$(BIN_NAME).exe"
+
+release-linux-amd64: dist release-bin-linux-amd64
+	tar -czf $(DIST_PATH)/gor_$(VERSION)_linux_amd64.tar.gz $(BIN_NAME)
+	$(DOCKER_FPM_CMD) $(FPM_COMMON) -f -t deb -a arm64 -p ./$(DIST_PATH) ./gor=/usr/local/bin
+	$(DOCKER_FPM_CMD) $(FPM_COMMON) -f -t rpm -a arm64 -p ./$(DIST_PATH) ./gor=/usr/local/bin
+	rm -rf $(BIN_NAME)
+
+release-linux-arm64: dist release-bin-linux-arm64
+	tar -czf $(DIST_PATH)/gor_$(VERSION)_linux_arm64.tar.gz $(BIN_NAME)
+	$(DOCKER_FPM_CMD) $(FPM_COMMON) -f -t deb -a arm64 -p ./$(DIST_PATH) ./gor=/usr/local/bin
+	$(DOCKER_FPM_CMD) $(FPM_COMMON) -f -t rpm -a arm64 -p ./$(DIST_PATH) ./gor=/usr/local/bin
+	rm -rf $(BIN_NAME)
+
+release-mac-amd64: dist release-bin-mac-amd64
+	tar -czf $(DIST_PATH)/gor_$(VERSION)_darwin_amd64.tar.gz $(BIN_NAME)
+	fpm $(FPM_COMMON) -f -t osxpkg -a amd64 -p ./$(DIST_PATH) ./gor=/usr/local/bin
+	mv ./$(DIST_PATH)/$(PROJECT_NAME)-$(VERSION).pkg ./$(DIST_PATH)/$(PROJECT_NAME)-$(VERSION)-amd64.pkg
+	rm -rf $(BIN_NAME)
+
+release-mac-arm64: dist release-bin-mac-arm64
+	tar -czf $(DIST_PATH)/gor_$(VERSION)_darwin_arm64.tar.gz $(BIN_NAME)
+	fpm $(FPM_COMMON) -f -t osxpkg -a arm64 -p ./$(DIST_PATH) ./gor=/usr/local/bin
+	mv ./$(DIST_PATH)/$(PROJECT_NAME)-$(VERSION).pkg ./$(DIST_PATH)/$(PROJECT_NAME)-$(VERSION)-arm64.pkg
+	rm -rf $(BIN_NAME)
+
+release-windows: dist release-bin-windows
+	zip $(DIST_PATH)/gor-$(VERSION)_windows.zip "$(BIN_NAME).exe"
+	rm -rf "$(BIN_NAME).exe"
+
+clean:
+	rm -rf $(DIST_PATH)
 
 build:
-	docker build -t $(CONTAINER) -f Dockerfile.dev .
+	go build -mod=vendor -o $(BIN_NAME) $(CUSTOM_TAGS) $(LDFLAGS)
+
+install:
+	go install $(CUSTOM_TAGS) $(MAC_LDFLAGS)
+
+build-env: build-amd64-env build-arm64-env
+
+build-amd64-env:
+	docker buildx build --build-arg BASE_IMAGE=golang:1.22-alpine --platform linux/amd64 -t $(CONTAINER_AMD) -f Dockerfile.dev --load .
+
+build-arm64-env:
+	docker buildx build --build-arg BASE_IMAGE=arm64v8/golang:1.22-alpine --platform linux/arm64 -t $(CONTAINER_ARM) -f Dockerfile.dev --load .
+
+build-docker:
+	docker build -t gor-dev -f Dockerfile .
 
 profile:
-	go build && ./gor --output-http="http://localhost:9000" --input-dummy 0 --input-raw :9000 --input-http :9000 --memprofile=./mem.out --cpuprofile=./cpu.out --stats --output-http-stats --output-http-timeout 100ms
+	go build && ./$(BIN_NAME) --output-http="http://localhost:9000" --input-dummy 0 --input-raw :9000 --input-http :9000 --memprofile=./mem.out --cpuprofile=./cpu.out --stats --output-http-stats --output-http-timeout 100ms
 
 lint:
 	$(RUN) golint $(PKG)
@@ -68,13 +105,13 @@ race:
 	$(RUN) go test ./... $(ARGS) -v -race -timeout 15s
 
 test:
-	$(RUN) go test ./. -timeout 120s $(LDFLAGS) $(ARGS)  -v
+	$(RUN) go test ./. -timeout 120s $(CUSTOM_TAGS) $(LDFLAGS) $(ARGS)  -v
 
 test_all:
-	$(RUN) go test ./... -timeout 120s $(LDFLAGS) $(ARGS) -v
+	$(RUN) go test ./... -timeout 120s $(CUSTOM_TAGS) $(LDFLAGS) $(ARGS) -v
 
 testone:
-	$(RUN) go test ./. -timeout 60s $(LDFLAGS) -run $(TEST) $(ARGS) -v
+	$(RUN) go test ./. -timeout 60s $(CUSTOM_TAGS) $(LDFLAGS) -run $(TEST) $(ARGS) -v
 
 cover:
 	$(RUN) go test $(ARGS) -race -v -timeout 15s -coverprofile=coverage.out
@@ -87,53 +124,39 @@ vet:
 	$(RUN) go vet
 
 bench:
-	$(RUN) go test $(LDFLAGS) -v -run NOT_EXISTING -bench $(BENCHMARK) -benchtime 5s
+	$(RUN) go test $(CUSTOM_TAGS) $(LDFLAGS) -v -run NOT_EXISTING -bench $(BENCHMARK) -benchtime 5s
 
 profile_test:
-	$(RUN) go test $(LDFLAGS) -run $(TEST) ./capture/. $(ARGS) -memprofile mem.mprof -cpuprofile cpu.out
-	$(RUN) go test $(LDFLAGS) -run $(TEST) ./capture/. $(ARGS) -c
+	$(RUN) go test $(CUSTOM_TAGS) $(LDFLAGS) -run $(TEST) ./capture/. $(ARGS) -memprofile mem.mprof -cpuprofile cpu.out
+	$(RUN) go test $(CUSTOM_TAGS) $(LDFLAGS) -run $(TEST) ./capture/. $(ARGS) -c
 
 # Used mainly for debugging, because docker container do not have access to parent machine ports
 run:
-	$(RUN) go run $(LDFLAGS) $(SOURCE) --input-dummy=0 --output-http="http://localhost:9000" --input-raw-track-response --input-raw 127.0.0.1:9000 --verbose --debug --middleware "./examples/middleware/echo.sh" --output-file requests.gor
+	$(RUN) go run $(CUSTOM_TAGS) $(LDFLAGS) $(SOURCE) --input-dummy=0 --output-http="http://localhost:9000" --input-raw-track-response --input-raw 127.0.0.1:9000 --verbose 0 --middleware "./examples/middleware/echo.sh" --output-file requests.gor
 
 run-2:
-	$(RUN) go run $(LDFLAGS) $(SOURCE) --input-raw :8000 --input-raw-bpf-filter "dst port 8000" --output-stdout --output-http "http://localhost:8000" --input-dummy=0
+	$(RUN) go run $(CUSTOM_TAGS) $(LDFLAGS) $(SOURCE) --input-raw :8000 --input-raw-bpf-filter "dst port 8000" --output-stdout --output-http "http://localhost:8000" --input-dummy=0
 
 run-3:
-	sudo -E go run $(SOURCE) --input-tcp :27001 --output-stdout
+	sudo -E go run $(CUSTOM_TAGS) $(SOURCE) --input-tcp :27001 --output-stdout
 
 run-arg:
-	sudo -E go run $(SOURCE) $(ARGS)
+	sudo -E go run $(CUSTOM_TAGS) $(SOURCE) $(ARGS)
 
 file-server:
-	go run $(SOURCE) file-server $(FADDR)
+	go run $(CUSTOM_TAGS) $(SOURCE) file-server $(FADDR)
 
 readpcap:
-	go run $(SOURCE) --input-raw $(FILE) --input-raw-track-response --input-raw-engine pcap_file --output-stdout
+	go run $(CUSTOM_TAGS) $(SOURCE) --input-raw $(FILE) --input-raw-track-response --input-raw-engine pcap_file --output-stdout
 
 record:
-	$(RUN) go run $(SOURCE) --input-dummy=0 --output-file=requests.gor --verbose --debug
+	$(RUN) go run $(CUSTOM_TAGS) $(SOURCE) --input-dummy=0 --output-file=requests.gor --verbose --debug
 
 replay:
-	$(RUN) go run $(SOURCE) --input-file=requests.bin --output-tcp=:9000 --verbose -h
+	$(RUN) go run $(CUSTOM_TAGS) $(SOURCE) --input-file=requests.bin --output-tcp=:9000 --verbose -h
 
 bash:
 	$(RUN) /bin/bash
 
-
-FPMCOMMON= \
-    --name goreplay \
-    --description "GoReplay is an open-source network monitoring tool which can record your live traffic, and use it for shadowing, load testing, monitoring and detailed analysis." \
-    -v $(VERSION) \
-    --vendor "Leonid Bugaev" \
-    -m "<support@goreplay.org>" \
-    --url "https://goreplay.org" \
-    -s dir \
-    -C /tmp/gor-build \
-
-build_packages:
-	mkdir -p /tmp/gor-build
-	go build -i -o /tmp/gor-build/gor
-	fpm $(FPMCOMMON) -a amd64 -t deb ./=/usr/local/bin
-	fpm $(FPMCOMMON) -a amd64 -t rpm ./=/usr/local/bin
+dist:
+	mkdir -p $(DIST_PATH)

README.md

@@ -1,10 +1,10 @@
-[![GitHub release](https://img.shields.io/github/release/buger/gor.svg?maxAge=3600)](https://github.com/buger/goreplay/releases) [![codebeat](https://codebeat.co/badges/6427d589-a78e-416c-a546-d299b4089893)](https://codebeat.co/projects/github-com-buger-gor) [![Go Report Card](https://goreportcard.com/badge/github.com/buger/gor)](https://goreportcard.com/report/github.com/buger/gor) [![Join the chat at https://gitter.im/buger/gor](https://badges.gitter.im/buger/gor.svg)](https://gitter.im/buger/gor?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) [![Reviewed by Hound](https://img.shields.io/badge/Reviewed_by-Hound-8E64B0.svg)](https://houndci.com)
+<a href="https://semgrep.dev/login?utm_source=github&utm_medium=badge&utm_campaign=growth-oss"><img src="https://img.shields.io/badge/semgrep-security-green.svg" /></a> [![GitHub release](https://img.shields.io/github/release/buger/gor.svg?maxAge=3600)](https://github.com/buger/goreplay/releases) [![codebeat](https://codebeat.co/badges/6427d589-a78e-416c-a546-d299b4089893)](https://codebeat.co/projects/github-com-buger-gor) [![Go Report Card](https://goreportcard.com/badge/github.com/buger/gor)](https://goreportcard.com/report/github.com/buger/gor) [![Join the chat at https://gitter.im/buger/gor](https://badges.gitter.im/buger/gor.svg)](https://gitter.im/buger/gor?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) [![Reviewed by Hound](https://img.shields.io/badge/Reviewed_by-Hound-8E64B0.svg)](https://houndci.com)
 
 ![Go Replay](http://i.imgur.com/ZG2ki5n.png)
 
 ## https://goreplay.org/
 
-GoReplay is an open-source network monitoring tool which can record your live traffic, and use it for shadowing, load testing, monitoring and detailed analysis.
+GoReplay is an open-source network monitoring tool which can record your live traffic and use it for shadowing, load testing, monitoring and detailed analysis.
 
 ## About
 
@@ -13,29 +13,28 @@ As your application grows, the effort required to test it also grows exponential
 GoReplay increases your confidence in code deployments, configuration and infrastructure changes.
 
 
-GoReplay offers unique approach for shadowing. Instead of being a proxy, GoReplay in background listen for traffic on your network interface, requiring no changes in your production infrastructure, rather then running GoReplay daemon on the same machine as your service.
+GoReplay offers a unique approach for shadowing. Instead of being a proxy, GoReplay listens in the background for traffic on your network interfaces, requiring no changes in your production infrastructure, other than running GoReplay daemon on the same machine as your service.
 
 ![Diagram](https://i.imgur.com/IN2xfDm.png)
 
 Check [latest documentation](http://github.com/buger/goreplay/wiki).
 
 ## Installation
-Download latest binary from https://github.com/buger/goreplay/releases or [compile by yourself](https://github.com/buger/goreplay/wiki/Compilation).
+Download the latest binary from https://github.com/buger/goreplay/releases or [compile by yourself](https://github.com/buger/goreplay/wiki/Compilation).
 
 ## Getting started
 
 The most basic setup will be `sudo ./gor --input-raw :8000 --output-stdout` which acts like tcpdump.
-If you already have test environment you can start replaying: `sudo ./gor --input-raw :8000 --output-http http://staging.env`.
+If you already have a test environment, you can start replaying by running: `sudo ./gor --input-raw :8000 --output-http http://staging.env`.
 
-See the our [documentation](https://github.com/buger/goreplay/wiki/) and [Getting started](https://github.com/buger/goreplay/wiki/Getting-Started) page for more info. 
+See our [documentation](https://github.com/buger/goreplay/wiki/) and the [Getting Started](https://github.com/buger/goreplay/wiki/Getting-Started) page for more info. 
 
 ## Newsletter
-Subscribe to our [newsletter](https://www.getdrip.com/forms/89690474/submissions/new) to stay informed about the latest features and changes to Gor project.
-
+Subscribe to our [newsletter](https://www.getdrip.com/forms/89690474/submissions/new) to stay informed about the latest features and changes to the Gor project.
 
 ## Want to Upgrade?
 
-We have created a [GoReplay PRO](https://goreplay.org/pro.html) extension which provides additional features such as support for binary protocols like Thrift or ProtocolBuffers, saving and replaying from cloud storage, TCP sessions replication, etc. The PRO version also includes a commercial-friendly license, dedicated support, and it also allows you to support high-quality open source development. 
+We have created a [GoReplay PRO](https://goreplay.org/pro.html) extension which provides additional features such as support for binary protocols like Thrift or ProtocolBuffers, saving and replaying from cloud storage, TCP session replication, etc. The PRO version also includes a commercial-friendly license, dedicated support, and it also allows you to support high-quality open source development. 
 
 
 ## Problems?

capture/listener.go

@@ -1,904 +0,0 @@
-/*
-Package capture provides traffic sniffier using RAW sockets.
-Capture traffic from socket using RAW_SOCKET's http://en.wikipedia.org/wiki/Raw_socket
-RAW_SOCKET allows you to listen for traffic from any port (e.g. sniffing) because they operate on IP level.
-Ports is TCP feature, same as flow control, reliable transmission and etc.
-This package implements own TCP layer: TCP packets is parsed using tcp_packet.go, and flow control is managed by tcp_message.go
-*/
-package capture
-
-import (
-	"bytes"
-	"encoding/binary"
-	"fmt"
-	"io"
-	"log"
-	"net"
-	"runtime"
-	"runtime/debug"
-	"strconv"
-	"strings"
-	"sync"
-	"time"
-
-	"github.com/buger/goreplay/proto"
-
-	"github.com/google/gopacket"
-	"github.com/google/gopacket/layers"
-	"github.com/google/gopacket/pcap"
-)
-
-type packet struct {
-	srcIP     []byte
-	data      []byte
-	timestamp time.Time
-}
-
-// Listener handle traffic capture
-type Listener struct {
-	sync.Mutex
-	// buffer of TCPMessages waiting to be send
-	// ID -> TCPMessage
-	messages map[tcpID]*TCPMessage
-
-	// Expect: 100-continue request is send in 2 tcp messages
-	// We store ACK aliases to merge this packets together
-	ackAliases map[uint32]uint32
-	// To get ACK of second message we need to compute its Seq and wait for them message
-	seqWithData map[uint32]uint32
-
-	// Ack -> Req
-	respAliases map[uint32]*TCPMessage
-
-	// Ack -> ID
-	respWithoutReq map[uint32]tcpID
-
-	// Messages ready to be send to client
-	packetsChan chan *packet
-
-	// Messages ready to be send to client
-	messagesChan chan *TCPMessage
-
-	addr string // IP to listen
-	port uint16 // Port to listen
-
-	trackResponse bool
-	messageExpire time.Duration
-
-	bpfFilter       string
-	timestampType   string
-	overrideSnapLen bool
-	immediateMode   bool
-
-	bufferSize int64
-
-	conn        net.PacketConn
-	pcapHandles []*pcap.Handle
-
-	quit  chan bool
-	ready bool
-
-	protocol TCPProtocol
-}
-
-type request struct {
-	id    tcpID
-	start time.Time
-	ack   uint32
-}
-
-// Available engines for intercepting traffic
-const (
-	EngineRawSocket = 1 << iota
-	EnginePcap
-	EnginePcapFile
-)
-
-// NewListener creates and initializes new Listener object
-func NewListener(addr string, port string, engine int, trackResponse bool, expire time.Duration, protocol TCPProtocol, bpfFilter string, timestampType string, bufferSize int64, overrideSnapLen bool, immediateMode bool) (l *Listener) {
-	l = &Listener{}
-
-	l.packetsChan = make(chan *packet, 10000)
-	l.messagesChan = make(chan *TCPMessage, 10000)
-	l.quit = make(chan bool)
-
-	l.messages = make(map[tcpID]*TCPMessage)
-	l.ackAliases = make(map[uint32]uint32)
-	l.seqWithData = make(map[uint32]uint32)
-	l.respAliases = make(map[uint32]*TCPMessage)
-	l.respWithoutReq = make(map[uint32]tcpID)
-	l.trackResponse = trackResponse
-	l.protocol = protocol
-	l.bpfFilter = bpfFilter
-	l.timestampType = timestampType
-	l.immediateMode = immediateMode
-	l.bufferSize = bufferSize
-	l.overrideSnapLen = overrideSnapLen
-
-	l.addr = addr
-	_port, _ := strconv.Atoi(port)
-	l.port = uint16(_port)
-
-	if expire.Nanoseconds() == 0 {
-		expire = 2000 * time.Millisecond
-	}
-
-	l.messageExpire = expire
-
-	go l.listen()
-
-	// Special case for testing
-	if l.port != 0 {
-		switch engine {
-		case EnginePcap:
-			go l.readPcap()
-		case EnginePcapFile:
-			go l.readPcapFile()
-		case EngineRawSocket:
-			go l.readRAWSocket()
-		default:
-			log.Fatal("Unknown traffic interception engine:", engine)
-		}
-	}
-
-	return
-}
-
-func (t *Listener) listen() {
-	gcTicker := time.Tick(t.messageExpire / 2)
-
-	for {
-		select {
-		case <-t.quit:
-			if t.conn != nil {
-				t.conn.Close()
-			}
-			return
-		case packet := <-t.packetsChan:
-			tcpPacket := ParseTCPPacket(packet.srcIP, packet.data, packet.timestamp)
-			t.processTCPPacket(tcpPacket)
-		case <-gcTicker:
-			now := time.Now()
-
-			// Dispatch requests before responses
-			for _, message := range t.messages {
-				if now.Sub(message.End) >= t.messageExpire {
-					t.dispatchMessage(message)
-				}
-			}
-		}
-	}
-}
-
-func (t *Listener) deleteMessage(message *TCPMessage) {
-	delete(t.messages, message.ID())
-	delete(t.ackAliases, message.Ack)
-	if message.DataAck != 0 {
-		delete(t.ackAliases, message.DataAck)
-	}
-	if message.DataSeq != 0 {
-		delete(t.seqWithData, message.DataSeq)
-	}
-
-	delete(t.respAliases, message.ResponseAck)
-}
-
-func (t *Listener) dispatchMessage(message *TCPMessage) {
-	// If already dispatched
-	if _, ok := t.messages[message.ID()]; !ok {
-		return
-	}
-
-	t.deleteMessage(message)
-
-	if t.protocol == ProtocolHTTP && !message.complete {
-		if !message.IsIncoming {
-			delete(t.respAliases, message.Ack)
-			delete(t.respWithoutReq, message.Ack)
-		}
-
-		return
-	}
-
-	if message.IsIncoming {
-		// If there were response before request
-		// log.Println("Looking for Response: ", t.respWithoutReq, message.ResponseAck)
-		if t.trackResponse {
-			if respID, ok := t.respWithoutReq[message.ResponseAck]; ok {
-				if resp, rok := t.messages[respID]; rok {
-					// if resp.AssocMessage == nil {
-					// log.Println("FOUND RESPONSE")
-					resp.setAssocMessage(message)
-					message.setAssocMessage(resp)
-
-					if resp.complete {
-						defer t.dispatchMessage(resp)
-					}
-					// }
-				}
-			}
-
-			if resp, ok := t.messages[message.ResponseID]; ok {
-				resp.setAssocMessage(message)
-			}
-		}
-	} else {
-		if message.AssocMessage == nil {
-			if responseRequest, ok := t.respAliases[message.Ack]; ok {
-				message.setAssocMessage(responseRequest)
-				responseRequest.setAssocMessage(message)
-			}
-		}
-
-		delete(t.respAliases, message.Ack)
-		delete(t.respWithoutReq, message.Ack)
-
-		// Do not track responses which have no associated requests
-		if message.AssocMessage == nil {
-			// log.Println("Can't dispatch resp", message.Seq, message.Ack, string(message.Bytes()))
-			return
-		}
-	}
-
-	t.messagesChan <- message
-}
-
-// DeviceNotFoundError raised if user specified wrong ip
-type DeviceNotFoundError struct {
-	addr string
-}
-
-func (e *DeviceNotFoundError) Error() string {
-	devices, _ := pcap.FindAllDevs()
-
-	if len(devices) == 0 {
-		return "Can't get list of network interfaces, ensure that you running Gor as root user or sudo.\nTo run as non-root users see this docs https://github.com/buger/goreplay/wiki/Running-as-non-root-user"
-	}
-
-	var msg string
-	msg += "Can't find interfaces with addr: " + e.addr + ". Provide available IP for intercepting traffic: \n"
-	for _, device := range devices {
-		msg += "Name: " + device.Name + "\n"
-		if device.Description != "" {
-			msg += "Description: " + device.Description + "\n"
-		}
-		for _, address := range device.Addresses {
-			msg += "- IP address: " + address.IP.String() + "\n"
-		}
-	}
-
-	return msg
-}
-
-func isLoopback(device pcap.Interface) bool {
-	if len(device.Addresses) == 0 {
-		return false
-	}
-
-	switch device.Addresses[0].IP.String() {
-	case "127.0.0.1", "::1":
-		return true
-	}
-
-	return false
-}
-
-func listenAllInterfaces(addr string) bool {
-	switch addr {
-	case "", "0.0.0.0", "[::]", "::":
-		return true
-	default:
-		return false
-	}
-}
-
-func findPcapDevices(addr string) (interfaces []pcap.Interface, err error) {
-	devices, err := pcap.FindAllDevs()
-	if err != nil {
-		log.Fatal(err)
-	}
-
-	for _, device := range devices {
-		if listenAllInterfaces(addr) && len(device.Addresses) > 0 || isLoopback(device) {
-			interfaces = append(interfaces, device)
-			continue
-		}
-
-		for _, address := range device.Addresses {
-			if device.Name == addr || address.IP.String() == addr {
-				interfaces = append(interfaces, device)
-				return interfaces, nil
-			}
-		}
-	}
-
-	if len(interfaces) == 0 {
-		return nil, &DeviceNotFoundError{addr}
-	}
-
-	return interfaces, nil
-}
-
-func (t *Listener) readPcap() {
-	devices, err := findPcapDevices(t.addr)
-	if err != nil {
-		log.Fatal(err)
-	}
-
-	const BPFSupported = runtime.GOOS != "darwin"
-
-	var wg sync.WaitGroup
-	wg.Add(len(devices))
-
-	for _, d := range devices {
-		go func(device pcap.Interface) {
-			inactive, err := pcap.NewInactiveHandle(device.Name)
-			if err != nil {
-				inactive.CleanUp()
-				log.Println("Pcap Error while opening device", device.Name, err)
-				wg.Done()
-				return
-			}
-
-			if t.timestampType != "" {
-				if tt, terr := pcap.TimestampSourceFromString(t.timestampType); terr != nil {
-					log.Println("Supported timestamp types: ", inactive.SupportedTimestamps(), device.Name)
-				} else if terr := inactive.SetTimestampSource(tt); terr != nil {
-					log.Println("Supported timestamp types: ", inactive.SupportedTimestamps(), device.Name)
-				}
-			}
-
-			if it, err := net.InterfaceByName(device.Name); err == nil && !t.overrideSnapLen {
-				// Auto-guess max length of packet to capture
-				inactive.SetSnapLen(it.MTU + 68*2)
-			} else {
-				inactive.SetSnapLen(65536)
-			}
-			inactive.SetTimeout(t.messageExpire)
-			inactive.SetPromisc(true)
-			inactive.SetImmediateMode(t.immediateMode)
-			if t.immediateMode {
-				log.Println("Setting immediate mode")
-			}
-			if t.bufferSize > 0 {
-				inactive.SetBufferSize(int(t.bufferSize))
-			}
-
-			handle, herr := inactive.Activate()
-			if herr != nil {
-				log.Printf("PCAP Activate device '%s' error: %s\n", device.Name, herr)
-				wg.Done()
-				return
-			}
-
-			defer handle.Close()
-
-			t.Lock()
-			t.pcapHandles = append(t.pcapHandles, handle)
-
-			var bpfDstHost, bpfSrcHost string
-			var loopback = isLoopback(device)
-
-			if loopback {
-				var allAddr []string
-				for _, dc := range devices {
-					for _, addr := range dc.Addresses {
-						allAddr = append(allAddr, "(dst host "+addr.IP.String()+" and src host "+addr.IP.String()+")")
-					}
-				}
-
-				bpfDstHost = strings.Join(allAddr, " or ")
-				bpfSrcHost = bpfDstHost
-			} else {
-				for i, addr := range device.Addresses {
-					bpfDstHost += "dst host " + addr.IP.String()
-					bpfSrcHost += "src host " + addr.IP.String()
-					if i != len(device.Addresses)-1 {
-						bpfDstHost += " or "
-						bpfSrcHost += " or "
-					}
-				}
-			}
-
-			if BPFSupported {
-				var bpf string
-
-				if t.bpfFilter != "" {
-					bpf = t.bpfFilter
-				} else {
-					if t.trackResponse {
-						bpf = fmt.Sprintf("(tcp dst port %d and (%s)) or (tcp src port %d and (%s))", t.port, bpfDstHost, t.port, bpfSrcHost)
-					} else {
-						bpf = fmt.Sprintf("(tcp dst port %d and (%s))", t.port, bpfDstHost)
-					}
-				}
-
-				if err := handle.SetBPFFilter(bpf); err != nil {
-					log.Println("BPF filter error:", err, "Device:", device.Name, bpf)
-					wg.Done()
-					return
-				}
-			}
-			t.Unlock()
-
-			var decoder gopacket.Decoder
-
-			// Special case for tunnel interface https://github.com/google/gopacket/issues/99
-			if handle.LinkType() == 12 {
-				decoder = layers.LayerTypeIPv4
-			} else {
-				decoder = handle.LinkType()
-			}
-
-			source := gopacket.NewPacketSource(handle, decoder)
-			source.Lazy = true
-			source.NoCopy = true
-
-			wg.Done()
-
-			var data, srcIP, dstIP []byte
-
-			for {
-				packet, err := source.NextPacket()
-
-				if err == io.EOF {
-					break
-				} else if err != nil {
-					continue
-				}
-
-				// We should remove network layer before parsing TCP/IP data
-				var of int
-				switch decoder {
-				case layers.LinkTypeEthernet:
-					of = 14
-				case layers.LinkTypePPP:
-					of = 1
-				case layers.LinkTypeFDDI:
-					of = 13
-				case layers.LinkTypeNull:
-					of = 4
-				case layers.LinkTypeLoop:
-					of = 4
-				case layers.LinkTypeRaw, layers.LayerTypeIPv4:
-					of = 0
-				case layers.LinkTypeLinuxSLL:
-					of = 16
-				default:
-					log.Println("Unknown packet layer", decoder, packet)
-					break
-				}
-				data = packet.Data()[of:]
-
-				version := uint8(data[0]) >> 4
-				ipLength := int(binary.BigEndian.Uint16(data[2:4]))
-
-				if version == 4 {
-					ihl := uint8(data[0]) & 0x0F
-
-					// Truncated IP info
-					if len(data) < int(ihl*4) {
-						continue
-					}
-
-					srcIP = data[12:16]
-					dstIP = data[16:20]
-
-					// Too small IP packet
-					if ipLength < 20 {
-						continue
-					}
-
-					// Invalid length
-					if int(ihl*4) > ipLength {
-						continue
-					}
-
-					if cmp := len(data) - ipLength; cmp > 0 {
-						data = data[:ipLength]
-					} else if cmp < 0 {
-						// Truncated packet
-						continue
-					}
-
-					data = data[ihl*4:]
-				} else {
-					// Truncated IP info
-					if len(data) < 40 {
-						continue
-					}
-
-					srcIP = data[8:24]
-					dstIP = data[24:40]
-
-					data = data[40:]
-				}
-
-				// Truncated TCP info
-				if len(data) <= 13 {
-					continue
-				}
-
-				dataOffset := data[12] >> 4
-				isFIN := data[13]&0x01 != 0
-
-				// We need only packets with data inside
-				// Check that the buffer is larger than the size of the TCP header
-				if len(data) > int(dataOffset*4) || isFIN {
-					if !BPFSupported {
-						destPort := binary.BigEndian.Uint16(data[2:4])
-						srcPort := binary.BigEndian.Uint16(data[0:2])
-
-						var addrCheck []byte
-
-						if destPort == t.port {
-							addrCheck = dstIP
-						}
-
-						if t.trackResponse && srcPort == t.port {
-							addrCheck = srcIP
-						}
-
-						if len(addrCheck) == 0 {
-							continue
-						}
-
-						addrMatched := false
-
-						if loopback {
-							for _, dc := range devices {
-								if addrMatched {
-									break
-								}
-								for _, a := range dc.Addresses {
-									if a.IP.Equal(net.IP(addrCheck)) {
-										addrMatched = true
-										break
-									}
-								}
-							}
-							addrMatched = true
-						} else {
-							for _, a := range device.Addresses {
-								if a.IP.Equal(net.IP(addrCheck)) {
-									addrMatched = true
-									break
-								}
-							}
-						}
-
-						if !addrMatched {
-							continue
-						}
-					}
-
-					t.packetsChan <- t.buildPacket(srcIP, data, packet.Metadata().Timestamp)
-				}
-			}
-		}(d)
-	}
-
-	wg.Wait()
-	t.Lock()
-	t.ready = true
-	t.Unlock()
-}
-
-func (t *Listener) readPcapFile() {
-	if handle, err := pcap.OpenOffline(t.addr); err != nil {
-		log.Fatal(err)
-	} else {
-		if t.bpfFilter != "" {
-			if err := handle.SetBPFFilter(t.bpfFilter); err != nil {
-				log.Println("BPF filter error:", err)
-				return
-			}
-		}
-
-		t.Lock()
-		t.ready = true
-		t.Unlock()
-		packetSource := gopacket.NewPacketSource(handle, handle.LinkType())
-
-		for {
-			packet, err := packetSource.NextPacket()
-			if err == io.EOF {
-				break
-			} else if err != nil {
-				log.Println("Error:", err)
-				continue
-			}
-
-			var addr, data []byte
-
-			if tcpLayer := packet.Layer(layers.LayerTypeTCP); tcpLayer != nil {
-				tcp, _ := tcpLayer.(*layers.TCP)
-				data = append(tcp.LayerContents(), tcp.LayerPayload()...)
-
-				if uint16(tcp.DstPort) == t.port {
-					copy(data[0:2], []byte{byte(tcp.SrcPort >> 8), byte(tcp.SrcPort)})
-					copy(data[2:4], []byte{byte(tcp.DstPort >> 8), byte(tcp.DstPort)})
-				} else {
-					copy(data[0:2], []byte{byte(tcp.DstPort >> 8), byte(tcp.DstPort)})
-					copy(data[2:4], []byte{byte(tcp.SrcPort >> 8), byte(tcp.SrcPort)})
-				}
-			} else {
-				continue
-			}
-
-			if ipLayer := packet.Layer(layers.LayerTypeIPv4); ipLayer != nil {
-				ip, _ := ipLayer.(*layers.IPv4)
-				addr = ip.SrcIP
-			} else if ipLayer = packet.Layer(layers.LayerTypeIPv6); ipLayer != nil {
-				ip, _ := ipLayer.(*layers.IPv6)
-				addr = ip.SrcIP
-			} else {
-				// log.Println("Can't find IP layer", packet)
-				continue
-			}
-
-			dataOffset := data[12] >> 4
-			isFIN := data[13]&0x01 != 0
-
-			// We need only packets with data inside
-			// Check that the buffer is larger than the size of the TCP header
-			if len(data) <= int(dataOffset*4) && !isFIN {
-				continue
-			}
-
-			t.packetsChan <- t.buildPacket(addr, data, packet.Metadata().Timestamp)
-		}
-	}
-}
-
-func (t *Listener) readRAWSocket() {
-	conn, e := net.ListenPacket("ip:tcp", t.addr)
-	t.conn = conn
-	if e != nil {
-		log.Fatal(e)
-	}
-	defer t.conn.Close()
-	type RSPacket struct {
-		buf  []byte
-		addr net.Addr
-		err  error
-		n    int
-	}
-	var bufChan = make(chan *RSPacket, 1000)
-	t.Lock()
-	t.ready = true
-	t.Unlock()
-	go func() {
-		for {
-			// Re-allocate data object to avoid data collision
-			var buf [64 * 104 * 1024]byte
-			// Note: ReadFrom receive messages without IP header
-			n, addr, err := t.conn.ReadFrom(buf[:])
-			bufChan <- &RSPacket{buf[:], addr, err, n}
-		}
-	}()
-	for {
-		packet := <-bufChan
-		if packet.err != nil {
-			if strings.HasSuffix(packet.err.Error(), "closed network connection") {
-				return
-			}
-			continue
-		}
-
-		if packet.n > 0 {
-			if t.isValidPacket(packet.buf[:packet.n]) {
-				t.packetsChan <- t.buildPacket([]byte(packet.addr.(*net.IPAddr).IP), packet.buf[:packet.n], time.Now())
-			}
-		}
-	}
-}
-
-func (t *Listener) buildPacket(packetSrcIP []byte, packetData []byte, timestamp time.Time) *packet {
-	return &packet{
-		srcIP:     packetSrcIP,
-		data:      packetData,
-		timestamp: timestamp,
-	}
-}
-
-func (t *Listener) isValidPacket(buf []byte) bool {
-	// To avoid full packet parsing every time, we manually parsing values needed for packet filtering
-	// http://en.wikipedia.org/wiki/Transmission_Control_Protocol
-	destPort := binary.BigEndian.Uint16(buf[2:4])
-	srcPort := binary.BigEndian.Uint16(buf[0:2])
-	// Because RAW_SOCKET can't be bound to port, we have to control it by ourself
-	if destPort == t.port || (t.trackResponse && srcPort == t.port) {
-		// Get the 'data offset' (size of the TCP header in 32-bit words)
-		dataOffset := buf[12] >> 4
-		// We need only packets with data inside
-		// Check that the buffer is larger than the size of the TCP header
-		if len(buf) > int(dataOffset*4) {
-			return true
-		}
-	}
-
-	return false
-}
-
-// Trying to add packet to existing message or creating new message
-//
-// For TCP message unique id is Acknowledgment number (see tcp_packet.go)
-func (t *Listener) processTCPPacket(packet *TCPPacket) {
-	// Don't exit on panic
-	defer func() {
-		if r := recover(); r != nil {
-			log.Println("PANIC: pkg:", r, packet, string(debug.Stack()))
-		}
-	}()
-
-	var responseRequest *TCPMessage
-	var message *TCPMessage
-
-	isIncoming := packet.DestPort == t.port
-
-	if t.protocol == ProtocolHTTP {
-		if !isIncoming {
-			responseRequest, _ = t.respAliases[packet.Ack]
-		}
-
-		// Seek for 100-expect chunks
-		// `packet.Ack != parentAck` is protection for clients who send data without ignoring server 100-continue response, e.g have data chunks have same Ack
-		if parentAck, ok := t.seqWithData[packet.Seq]; ok && packet.Ack != parentAck {
-			// Skip zero-length chunks https://github.com/buger/goreplay/issues/496
-			if len(packet.Data) == 0 {
-				return
-			}
-
-			// In case if non-first data chunks comes first
-			for _, m := range t.messages {
-				if m.Ack == packet.Ack && bytes.Equal(m.packets[0].Addr, packet.Addr) {
-					t.deleteMessage(m)
-
-					if m.AssocMessage != nil {
-						m.AssocMessage.setAssocMessage(nil)
-						m.setAssocMessage(nil)
-					}
-					for _, pkt := range m.packets {
-						// log.Println("Updating ack", parentAck, pkt.Ack)
-						pkt.UpdateAck(parentAck)
-						// Re-queue this packets
-						t.processTCPPacket(pkt)
-					}
-				}
-			}
-
-			t.ackAliases[packet.Ack] = parentAck
-			packet.UpdateAck(parentAck)
-		}
-	}
-
-	if isIncoming && packet.IsFIN {
-		if ma, ok := t.respAliases[packet.Seq]; ok {
-			if ma.packets[0].SrcPort == packet.SrcPort {
-				packet.UpdateAck(ma.Ack)
-			}
-		}
-	}
-
-	if alias, ok := t.ackAliases[packet.Ack]; ok {
-		packet.UpdateAck(alias)
-	}
-
-	message, ok := t.messages[packet.ID]
-
-	if !ok {
-		message = NewTCPMessage(packet.Seq, packet.Ack, isIncoming, t.protocol, packet.timestamp)
-		t.messages[packet.ID] = message
-
-		if !isIncoming {
-			if responseRequest != nil {
-				message.setAssocMessage(responseRequest)
-				responseRequest.setAssocMessage(message)
-			} else {
-				t.respWithoutReq[packet.Ack] = packet.ID
-			}
-		}
-	}
-
-	// Adding packet to message
-	message.AddPacket(packet)
-
-	// Handling Expect: 100-continue requests
-	if t.protocol == ProtocolHTTP && message.expectType == httpExpect100Continue && len(message.packets) == message.headerPacket+1 {
-		seq := packet.Seq + uint32(len(packet.Data))
-		t.seqWithData[seq] = packet.Ack
-
-		message.DataSeq = seq
-		message.complete = false
-
-		// In case if sequence packet came first
-		for _, m := range t.messages {
-			if m.Seq == seq {
-				t.deleteMessage(m)
-				if m.AssocMessage != nil {
-					message.setAssocMessage(m.AssocMessage)
-					m.AssocMessage.setAssocMessage(nil)
-				}
-
-				t.ackAliases[m.Ack] = packet.Ack
-
-				for _, pkt := range m.packets {
-					pkt.UpdateAck(packet.Ack)
-					message.AddPacket(pkt)
-				}
-			}
-		}
-
-		// Removing `Expect: 100-continue` header
-		packet.Data = proto.DeleteHeader(packet.Data, bExpectHeader)
-	}
-
-	// If client do sends Expect: 100-continue but do not respect server response
-	if message.expectType == httpExpect100Continue && (message.headerPacket != -1 && len(message.packets) > message.headerPacket+1) {
-		delete(t.seqWithData, message.DataSeq)
-		seq := packet.Seq + uint32(len(packet.Data))
-		t.seqWithData[seq] = packet.Ack
-		message.DataSeq = seq
-	}
-
-	if isIncoming {
-		// If message have multiple packets, delete previous alias
-		if len(message.packets) > 1 {
-			delete(t.respAliases, message.ResponseAck)
-		}
-
-		message.UpdateResponseAck()
-		t.respAliases[message.ResponseAck] = message
-	}
-
-	// If message contains only single packet immediately dispatch it
-	if message.complete {
-		// log.Println("COMPLETE!", isIncoming, message)
-		if isIncoming {
-			if t.trackResponse {
-				// log.Println("Found response!", message.ResponseID, t.messages)
-
-				if resp, ok := t.messages[message.ResponseID]; ok {
-					if resp.complete {
-						t.dispatchMessage(resp)
-					}
-
-					t.dispatchMessage(message)
-				}
-			} else {
-				t.dispatchMessage(message)
-			}
-		} else {
-			if message.AssocMessage == nil {
-				return
-			}
-
-			if req, ok := t.messages[message.AssocMessage.ID()]; ok {
-				if req.complete {
-					t.dispatchMessage(req)
-					t.dispatchMessage(message)
-				}
-			}
-		}
-	}
-}
-
-// Receiver TCP messages from the listener channel
-func (t *Listener) Receiver() chan *TCPMessage {
-	return t.messagesChan
-}
-
-// Close tcp listener
-func (t *Listener) Close() {
-	close(t.quit)
-	if t.conn != nil {
-		t.conn.Close()
-	}
-
-	for _, h := range t.pcapHandles {
-		h.Close()
-	}
-
-	return
-}

capture/listener_test.go

@@ -1,617 +0,0 @@
-package capture
-
-import (
-	"bytes"
-	"log"
-	"math/rand"
-	"sync/atomic"
-	"testing"
-	"time"
-)
-
-func TestRawListenerInput(t *testing.T) {
-	var req, resp *TCPMessage
-	listener := NewListener("", "0", EnginePcap, true, 10*time.Millisecond, ProtocolHTTP, "", "", 0, false, false)
-	defer listener.Close()
-
-	reqPacket := buildPacket(true, 1, 1, []byte("GET / HTTP/1.1\r\n\r\n"), time.Now())
-
-	respAck := reqPacket.Seq + uint32(len(reqPacket.Data))
-	respPacket := buildPacket(false, respAck, reqPacket.Seq+1, []byte("HTTP/1.1 200 OK\r\n\r\n"), time.Now())
-
-	listener.packetsChan <- reqPacket.dump()
-	listener.packetsChan <- respPacket.dump()
-
-	select {
-	case req = <-listener.messagesChan:
-	case <-time.After(time.Millisecond):
-		t.Error("Should return request immediately")
-		return
-	}
-
-	if !req.IsIncoming {
-		t.Error("Should be request")
-	}
-
-	select {
-	case resp = <-listener.messagesChan:
-	case <-time.After(20 * time.Millisecond):
-		t.Error("Should return response immediately")
-		return
-	}
-
-	if resp.IsIncoming {
-		t.Error("Should be response")
-	}
-}
-
-func firstPacket(payload []byte) *TCPPacket {
-	return buildPacket(
-		true,
-		1,
-		1,
-		payload,
-		time.Now(),
-	)
-}
-
-func nextPacket(prev *TCPPacket, payload []byte) *TCPPacket {
-	return buildPacket(
-		prev.SrcPort == 1,
-		prev.Ack,
-		prev.Seq+uint32(len(prev.Data)),
-		payload,
-		prev.timestamp.Add(time.Millisecond),
-	)
-}
-
-func responsePacket(prev *TCPPacket, payload []byte) *TCPPacket {
-	return buildPacket(
-		!(prev.SrcPort == 1),
-		prev.Seq+uint32(len(prev.Data)),
-		prev.Ack,
-		payload,
-		prev.timestamp.Add(time.Millisecond),
-	)
-}
-
-func TestHEADRequestNoBody(t *testing.T) {
-	listener := NewListener("", "0", EnginePcap, true, 10*time.Millisecond, ProtocolHTTP, "", "", 0, false, false)
-	defer listener.Close()
-
-	reqPacket := firstPacket([]byte("HEAD / HTTP/1.1\r\nContent-Length: 0\r\n\r\n"))
-	respPacket := responsePacket(reqPacket, []byte("HTTP/1.1 200 OK\r\nContent-Length: 100\r\n\r\n"))
-
-	listener.packetsChan <- reqPacket.dump()
-	listener.packetsChan <- respPacket.dump()
-
-	var req, resp *TCPMessage
-	select {
-	case req = <-listener.messagesChan:
-	case <-time.After(time.Millisecond):
-		t.Error("Should return request immediately")
-		return
-	}
-
-	if !req.IsIncoming {
-		t.Error("Should be request")
-	}
-
-	select {
-	case resp = <-listener.messagesChan:
-	case <-time.After(20 * time.Millisecond):
-		t.Error("Should return response immediately")
-		return
-	}
-
-	if resp.IsIncoming {
-		t.Error("Should be response")
-	}
-}
-
-func TestSingleAck100Continue(t *testing.T) {
-	listener := NewListener("", "0", EnginePcap, true, 10*time.Millisecond, ProtocolHTTP, "", "", 0, false, false)
-	defer listener.Close()
-
-	reqPacket1 := firstPacket([]byte("POST / HTTP/1.1\r\nExpect: 100-continue\r\nContent-Length: 4\r\n\r\n"))
-	respPacket1 := responsePacket(reqPacket1, []byte(""))
-	respPacket2 := responsePacket(reqPacket1, []byte("HTTP/1.1 100 Continue\r\n"))
-	reqPacket2 := responsePacket(respPacket2, []byte("DATA"))
-	respPacket3 := responsePacket(reqPacket2, []byte("HTTP/1.1 200 OK\r\n\r\n"))
-
-	result := []byte("POST / HTTP/1.1\r\nContent-Length: 4\r\n\r\nDATA")
-
-	testRawListener100Continue(t, listener, result,
-		reqPacket1,
-		respPacket1, respPacket2,
-		reqPacket2,
-		respPacket3)
-}
-
-func Test100ContinueWithoutWaiting(t *testing.T) {
-	listener := NewListener("", "0", EnginePcap, true, 10*time.Millisecond, ProtocolHTTP, "", "", 0, false, false)
-	defer listener.Close()
-
-	req1 := firstPacket([]byte("POST / HTTP/1.1\r\nExpect: 100-continue\r\nContent-Length: 4\r\n\r\n"))
-	req2 := nextPacket(req1, []byte("DATA"))
-	resp1 := responsePacket(req1, []byte("HTTP/1.1 100 Continue\r\n"))
-	resp2 := responsePacket(req2, []byte("HTTP/1.1 200 OK\r\n\r\n"))
-
-	result := []byte("POST / HTTP/1.1\r\nContent-Length: 4\r\n\r\nDATA")
-
-	testRawListener100Continue(t, listener, result,
-		req1, req2, resp1, resp2)
-}
-
-// Client first sends data without waiting 100-continue, but once response received, generate packets based on Ack payload
-func Test100ContinueMixed(t *testing.T) {
-	listener := NewListener("", "0", EnginePcap, true, 10*time.Millisecond, ProtocolHTTP, "", "", 0, false, false)
-	defer listener.Close()
-
-	req1 := firstPacket([]byte("POST / HTTP/1.1\r\nExpect: 100-continue\r\nContent-Length: 12\r\n\r\n"))
-	req2 := nextPacket(req1, []byte("DAT1"))
-	resp1 := responsePacket(req1, []byte("HTTP/1.1 100 Continue\r\n\r\n"))
-	req3 := responsePacket(resp1, []byte("DAT2"))
-	req3.Seq = req2.Seq + uint32(len(req2.Data))
-	req4 := nextPacket(req3, []byte("DAT3"))
-	resp2 := responsePacket(req4, []byte("HTTP/1.1 200 OK\r\n\r\n"))
-
-	result := []byte("POST / HTTP/1.1\r\nContent-Length: 12\r\n\r\nDAT1DAT2DAT3")
-
-	testRawListener100Continue(t, listener, result,
-		req1, req2, req3, req4, resp1, resp2)
-}
-
-func TestDoubleAck100Continue(t *testing.T) {
-	listener := NewListener("", "0", EnginePcap, true, 10*time.Millisecond, ProtocolHTTP, "", "", 0, false, false)
-	defer listener.Close()
-
-	reqPacket1 := firstPacket([]byte("POST / HTTP/1.1\r\nExpect: 100-continue\r\nContent-Length: 4\r\n\r\n"))
-
-	respPacket1 := responsePacket(reqPacket1, []byte(""))
-	respPacket2 := responsePacket(reqPacket1, []byte("HTTP/1.1 100 Continue\r\n"))
-	reqPacket2 := responsePacket(respPacket2, []byte(""))
-	reqPacket3 := responsePacket(respPacket2, []byte("DATA"))
-	respPacket3 := responsePacket(reqPacket3, []byte("HTTP/1.1 200 OK\r\n\r\n"))
-
-	result := []byte("POST / HTTP/1.1\r\nContent-Length: 4\r\n\r\nDATA")
-
-	testRawListener100Continue(t, listener, result,
-		reqPacket1,
-		respPacket1, respPacket2,
-		reqPacket2, reqPacket3,
-		respPacket3)
-}
-
-func TestRawListenerInputResponseByClose(t *testing.T) {
-	var req, resp *TCPMessage
-
-	listener := NewListener("", "0", EnginePcap, true, 10*time.Millisecond, ProtocolHTTP, "", "", 0, false, false)
-	defer listener.Close()
-
-	reqPacket := buildPacket(true, 1, 1, []byte("GET / HTTP/1.1\r\n\r\n"), time.Now())
-
-	respAck := reqPacket.Seq + uint32(len(reqPacket.Data))
-	respPacket := buildPacket(false, respAck, reqPacket.Seq+1, []byte("HTTP/1.1 200 OK\r\nConnection: close\r\n\r\nasd"), time.Now())
-	finPacket := buildPacket(false, respAck, reqPacket.Seq+2, []byte(""), time.Now())
-	finPacket.IsFIN = true
-
-	listener.packetsChan <- reqPacket.dump()
-	listener.packetsChan <- respPacket.dump()
-	listener.packetsChan <- finPacket.dump()
-
-	select {
-	case req = <-listener.messagesChan:
-	case <-time.After(time.Millisecond):
-		t.Error("Should return request immediately")
-		return
-	}
-
-	if !req.IsIncoming {
-		t.Error("Should be request")
-	}
-
-	select {
-	case resp = <-listener.messagesChan:
-	case <-time.After(20 * time.Millisecond):
-		t.Error("Should return response immediately")
-		return
-	}
-
-	if resp.IsIncoming {
-		t.Error("Should be response")
-	}
-}
-
-func TestRawListenerInputWithoutResponse(t *testing.T) {
-	var req *TCPMessage
-
-	listener := NewListener("", "0", EnginePcap, false, 10*time.Millisecond, ProtocolHTTP, "", "", 0, false, false)
-	defer listener.Close()
-
-	reqPacket := buildPacket(true, 1, 1, []byte("GET / HTTP/1.1\r\n\r\n"), time.Now())
-
-	listener.packetsChan <- reqPacket.dump()
-
-	select {
-	case req = <-listener.messagesChan:
-	case <-time.After(time.Millisecond):
-		t.Error("Should return request immediately")
-		return
-	}
-
-	if !req.IsIncoming {
-		t.Error("Should be request")
-	}
-}
-
-func TestRawListenerResponse(t *testing.T) {
-	var req, resp *TCPMessage
-
-	listener := NewListener("", "0", EnginePcap, true, 10*time.Millisecond, ProtocolHTTP, "", "", 0, false, false)
-	defer listener.Close()
-
-	reqPacket := firstPacket([]byte("GET / HTTP/1.1\r\n\r\n"))
-	respPacket := responsePacket(reqPacket, []byte("HTTP/1.1 200 OK\r\n\r\n"))
-
-	// If response packet comes before request
-	listener.packetsChan <- respPacket.dump()
-	listener.packetsChan <- reqPacket.dump()
-
-	select {
-	case req = <-listener.messagesChan:
-	case <-time.After(time.Millisecond):
-		t.Error("Should return request immediately")
-		return
-	}
-
-	if !req.IsIncoming {
-		t.Error("Should be request")
-	}
-
-	select {
-	case resp = <-listener.messagesChan:
-	case <-time.After(time.Millisecond):
-		t.Error("Should return response immediately")
-		return
-	}
-
-	if resp.IsIncoming {
-		t.Error("Should be response")
-	}
-
-	if !bytes.Equal(resp.UUID(), req.UUID()) {
-		t.Error("Resp and Req UUID should be equal")
-	}
-}
-
-func get100ContinuePackets() (req []*TCPPacket, resp []*TCPPacket) {
-	req1 := firstPacket([]byte("POST / HTTP/1.1\r\nExpect: 100-continue\r\nContent-Length: 2\r\n\r\n"))
-	resp1 := responsePacket(req1, []byte("HTTP/1.1 100 Continue\r\n"))
-	req2 := responsePacket(resp1, []byte("a"))
-	req3 := nextPacket(req2, []byte("b"))
-	resp2 := responsePacket(req3, []byte("HTTP/1.1 200 OK\r\n\r\n"))
-
-	return []*TCPPacket{req1, req2, req3}, []*TCPPacket{resp1, resp2}
-}
-
-func TestShort100Continue(t *testing.T) {
-	listener := NewListener("", "0", EnginePcap, true, 10*time.Millisecond, ProtocolHTTP, "", "", 0, false, false)
-	defer listener.Close()
-
-	req, resp := get100ContinuePackets()
-
-	result := []byte("POST / HTTP/1.1\r\nContent-Length: 2\r\n\r\nab")
-
-	testRawListener100Continue(t, listener, result, req[0], req[1], req[2], resp[0], resp[1])
-}
-
-// Response comes before Request
-func Test100ContinueWrongOrder(t *testing.T) {
-	listener := NewListener("", "0", EnginePcap, true, 10*time.Millisecond, ProtocolHTTP, "", "", 0, false, false)
-	defer listener.Close()
-
-	req, resp := get100ContinuePackets()
-
-	result := []byte("POST / HTTP/1.1\r\nContent-Length: 2\r\n\r\nab")
-
-	testRawListener100Continue(t, listener, result, resp[0], resp[1], req[0], req[1], req[2])
-}
-
-func testRawListener100Continue(t *testing.T, listener *Listener, result []byte, packets ...*TCPPacket) {
-	var req, resp *TCPMessage
-	for _, p := range packets {
-		listener.packetsChan <- p.dump()
-	}
-
-	select {
-	case req = <-listener.messagesChan:
-		break
-	case <-time.After(11 * time.Millisecond):
-		t.Error("Should return response after expire time")
-		return
-	}
-
-	if !bytes.Equal(req.Bytes(), result) {
-		t.Error("Should receive full message", string(req.Bytes()))
-	}
-
-	if !req.IsIncoming {
-		t.Error("Should be request")
-	}
-
-	select {
-	case resp = <-listener.messagesChan:
-		break
-	case <-time.After(21 * time.Millisecond):
-		t.Error("Should return response after expire time")
-		return
-	}
-
-	if resp.IsIncoming {
-		t.Error("Should be response")
-	}
-
-	if !bytes.Equal(resp.UUID(), req.UUID()) {
-		t.Error("Resp and Req UUID should be equal")
-	}
-}
-
-func testChunkedSequence(t *testing.T, listener *Listener, packets ...*TCPPacket) {
-	var r, req, resp *TCPMessage
-
-	for _, p := range packets {
-		listener.packetsChan <- p.dump()
-	}
-
-	select {
-	case r = <-listener.messagesChan:
-		if r.IsIncoming {
-			req = r
-		} else {
-			resp = r
-		}
-		break
-	case <-time.After(25 * time.Millisecond):
-		t.Error("Should return request after expire time")
-		return
-	}
-	select {
-	case r = <-listener.messagesChan:
-		if r.IsIncoming {
-			if req != nil {
-				t.Error("Request already received", r)
-				return
-			}
-			req = r
-		} else {
-			if resp != nil {
-				t.Error("Response already received", r)
-				return
-			}
-			resp = r
-		}
-		break
-	case <-time.After(25 * time.Millisecond):
-		t.Error("Should return request after expire time")
-		return
-	}
-
-	if !bytes.Equal(req.Bytes(), []byte("POST / HTTP/1.1\r\nTransfer-Encoding: chunked\r\n\r\n1\r\na\r\n1\r\nb\r\n0\r\n\r\n")) {
-		t.Error("Should receive full message", string(req.Bytes()))
-	}
-
-	if !req.IsIncoming {
-		t.Error("Should be request")
-	}
-
-	if resp.IsIncoming {
-		t.Error("Should be response")
-	}
-
-	if !bytes.Equal(resp.UUID(), req.UUID()) {
-		t.Error("Resp and Req UUID should be equal", string(resp.UUID()), string(req.UUID()))
-	}
-
-	time.Sleep(20 * time.Millisecond)
-
-	if len(listener.packetsChan) != 0 {
-		t.Fatal("packetsChan non empty:", listener.packetsChan)
-	}
-
-	if len(listener.ackAliases) != 0 {
-		t.Fatal("ackAliases non empty:", listener.ackAliases)
-	}
-
-	if len(listener.seqWithData) != 0 {
-		t.Fatal("seqWithData non empty:", listener.seqWithData)
-	}
-
-	if len(listener.respAliases) != 0 {
-		t.Fatal("respAliases non empty:", listener.respAliases)
-	}
-}
-
-// permutation using heap algorithm https://en.wikipedia.org/wiki/Heap%27s_algorithm
-func permutation(a []*TCPPacket, f func([]*TCPPacket)) {
-	n := len(a)
-	c := make([]int, n)
-	f(a)
-	i := 0
-	for i < n {
-		if c[i] < i {
-			if i&1 != 1 {
-				a[0], a[i] = a[i], a[0]
-			} else {
-				a[c[i]], a[i] = a[i], a[c[i]]
-			}
-			f(a)
-			c[i]++
-			i = 0
-		} else {
-			c[i] = 0
-			i++
-		}
-	}
-}
-
-// Response comes before Request
-func TestRawListenerChunkedWrongOrder(t *testing.T) {
-	listener := NewListener("", "0", EnginePcap, true, 10*time.Millisecond, ProtocolHTTP, "", "", 0, false, false)
-	defer listener.Close()
-
-	reqPacket1 := firstPacket([]byte("POST / HTTP/1.1\r\nTransfer-Encoding: chunked\r\nExpect: 100-continue\r\n\r\n"))
-
-	respPacket1 := responsePacket(reqPacket1, []byte("HTTP/1.1 100 Continue\r\n"))
-	reqPacket2 := responsePacket(respPacket1, []byte("1\r\na\r\n"))
-	reqPacket3 := nextPacket(reqPacket2, []byte("1\r\nb\r\n"))
-	reqPacket4 := nextPacket(reqPacket3, []byte("0\r\n\r\n"))
-
-	respPacket2 := responsePacket(reqPacket4, []byte("HTTP/1.1 200 OK\r\n\r\n"))
-
-	f := func(p []*TCPPacket) {
-		testChunkedSequence(t, listener, p...)
-	}
-	// Should re-construct message from all possible combinations
-	permutation([]*TCPPacket{reqPacket1, reqPacket2, reqPacket3, reqPacket4, respPacket1, respPacket2}, f)
-}
-
-func chunkedPostMessage() []*TCPPacket {
-	ack := uint32(rand.Int63())
-	seq := uint32(rand.Int63())
-
-	reqPacket1 := buildPacket(true, ack, seq, []byte("POST / HTTP/1.1\r\nTransfer-Encoding: chunked\r\n\r\n"), time.Now())
-	// Packet with data have different Seq
-	reqPacket2 := buildPacket(true, ack, seq+47, []byte("1\r\na\r\n"), time.Now())
-	reqPacket3 := buildPacket(true, ack, reqPacket2.Seq+5, []byte("1\r\nb\r\n"), time.Now())
-	reqPacket4 := buildPacket(true, ack, reqPacket3.Seq+5, []byte("0\r\n\r\n"), time.Now())
-
-	respPacket := buildPacket(false, reqPacket4.Seq+5 /* len of data */, ack, []byte("HTTP/1.1 200 OK\r\n\r\n"), time.Now())
-
-	return []*TCPPacket{
-		reqPacket1, reqPacket2, reqPacket3, reqPacket4, respPacket,
-	}
-}
-
-func postMessage() []*TCPPacket {
-	ack := uint32(rand.Int63())
-	seq2 := uint32(rand.Int63())
-	seq := uint32(rand.Int63())
-
-	c := 10000
-	data := make([]byte, c)
-	rand.Read(data)
-
-	head := []byte("POST / HTTP/1.1\r\nContent-Length: 9958\r\n\r\n")
-	for i := range head {
-		data[i] = head[i]
-	}
-
-	return []*TCPPacket{
-		buildPacket(true, ack, seq, data, time.Now()),
-		buildPacket(false, seq+uint32(len(data)), seq2, []byte("HTTP/1.1 200 OK\r\n\r\n"), time.Now()),
-	}
-}
-
-func getMessage() []*TCPPacket {
-	ack := uint32(rand.Int63())
-	seq2 := uint32(rand.Int63())
-	seq := uint32(rand.Int63())
-
-	return []*TCPPacket{
-		buildPacket(true, ack, seq, []byte("GET / HTTP/1.1\r\n\r\n"), time.Now()),
-		buildPacket(false, seq+18, seq2, []byte("HTTP/1.1 200 OK\r\n\r\n"), time.Now()),
-	}
-}
-
-// Response comes before Request
-func TestRawListenerBench(t *testing.T) {
-	l := NewListener("", "0", EnginePcap, true, 200*time.Millisecond, ProtocolHTTP, "", "", 0, false, false)
-	defer l.Close()
-
-	// Should re-construct message from all possible combinations
-	for i := 0; i < 1000; i++ {
-		go func(i int) {
-			for j := 0; j < 100; j++ {
-				var packets []*TCPPacket
-
-				if j%5 == 0 {
-					packets = chunkedPostMessage()
-				} else if j%3 == 0 {
-					packets = postMessage()
-				} else {
-					packets = getMessage()
-				}
-
-				for _, p := range packets {
-					// Randomly drop packets
-					if (i+j)%5 == 0 {
-						if rand.Int63()%3 == 0 {
-							continue
-						}
-					}
-
-					l.packetsChan <- p.dump()
-					time.Sleep(time.Millisecond)
-				}
-
-				time.Sleep(5 * time.Millisecond)
-			}
-		}(i)
-	}
-
-	ch := l.Receiver()
-
-	var count int32
-
-	for {
-		select {
-		case <-ch:
-			atomic.AddInt32(&count, 1)
-		case <-time.After(2000 * time.Millisecond):
-			log.Println("Emitted 200000 messages, captured: ", count, len(l.ackAliases), len(l.seqWithData), len(l.respAliases), len(l.respWithoutReq), len(l.packetsChan))
-			return
-		}
-	}
-}
-
-func TestResponseZeroContentLength(t *testing.T) {
-	var req, resp *TCPMessage
-	listener := NewListener("", "0", EnginePcap, true, 10*time.Millisecond, ProtocolHTTP, "", "", 0, false, false)
-	defer listener.Close()
-
-	reqPacket := firstPacket([]byte("POST /api/setup/install HTTP/1.1\r\nHost: localhost:22936\r\nUser-Agent: curl/7.57.0\r\nAccept: */*\r\nContent-Length: 0\r\nContent-Type: application/x-www-form-urlencoded\r\n\r\n"))
-	respPacket := responsePacket(reqPacket, []byte("HTTP/1.1 200\r\nDate: Fri, 11 May 2018 15:09:10 GMT\r\nServer: Kestrel\r\nCache-Control: no-cache\r\nTransfer-Encoding: chunked\r\n\r\n"))
-	respPacket2 := nextPacket(respPacket, []byte("0\r\n\r\n"))
-
-	// If response packet comes before request
-	listener.packetsChan <- reqPacket.dump()
-	listener.packetsChan <- respPacket.dump()
-	listener.packetsChan <- respPacket2.dump()
-
-	select {
-	case req = <-listener.messagesChan:
-	case <-time.After(time.Millisecond):
-		t.Error("Should return request immediately")
-		return
-	}
-
-	if !req.IsIncoming {
-		t.Error("Should be request")
-	}
-
-	select {
-	case resp = <-listener.messagesChan:
-	case <-time.After(time.Millisecond):
-		t.Error("Should return response immediately")
-		return
-	}
-
-	if resp.IsIncoming {
-		t.Error("Should be response")
-	}
-
-	if !bytes.Equal(resp.UUID(), req.UUID()) {
-		t.Error("Resp and Req UUID should be equal")
-	}
-}

capture/tcp_message.go

@@ -1,522 +0,0 @@
-package capture
-
-import (
-	"bytes"
-	"crypto/sha1"
-	"encoding/binary"
-	"encoding/hex"
-	"net"
-	"strconv"
-	"strings"
-	"time"
-
-	"github.com/buger/goreplay/proto"
-)
-
-// TCPProtocol is a number to indicate type of protocol
-type TCPProtocol uint8
-
-const (
-	// ProtocolHTTP ...
-	ProtocolHTTP TCPProtocol = 0
-	// ProtocolBinary ...
-	ProtocolBinary TCPProtocol = 1
-)
-
-// TCPMessage ensure that all TCP packets for given request is received, and processed in right sequence
-// Its needed because all TCP message can be fragmented or re-transmitted
-//
-// Each TCP Packet have 2 ids: acknowledgment - message_id, and sequence - packet_id
-// Message can be compiled from unique packets with same message_id which sorted by sequence
-// Message is received if we didn't receive any packets for 2000ms
-type TCPMessage struct {
-	Seq         uint32
-	Ack         uint32
-	ResponseAck uint32
-	ResponseID  tcpID
-	DataAck     uint32
-	DataSeq     uint32
-
-	AssocMessage *TCPMessage
-	Start        time.Time
-	End          time.Time
-	IsIncoming   bool
-
-	packets []*TCPPacket
-
-	delChan chan *TCPMessage
-
-	protocol TCPProtocol
-
-	/* HTTP specific variables */
-	methodType    httpMethodType
-	bodyType      httpBodyType
-	expectType    httpExpectType
-	seqMissing    bool
-	headerPacket  int
-	contentLength int
-	complete      bool
-}
-
-// NewTCPMessage pointer created from a Acknowledgment number and a channel of messages readuy to be deleted
-func NewTCPMessage(Seq, Ack uint32, IsIncoming bool, protocol TCPProtocol, timestamp time.Time) (msg *TCPMessage) {
-	msg = &TCPMessage{Seq: Seq, Ack: Ack, IsIncoming: IsIncoming, protocol: protocol, Start: timestamp}
-	msg.Start = time.Now()
-
-	return
-}
-
-func (t *TCPMessage) packetsData() (d [][]byte) {
-	d = make([][]byte, len(t.packets))
-	for i, p := range t.packets {
-		d[i] = p.Data
-	}
-
-	return
-}
-
-// Bytes return message content
-func (t *TCPMessage) Bytes() (output []byte) {
-	for _, p := range t.packets {
-		output = append(output, p.Data...)
-	}
-
-	return output
-}
-
-// BodySize returns total body size
-func (t *TCPMessage) BodySize() (size int) {
-	if len(t.packets) == 0 || t.headerPacket == -1 {
-		return 0
-	}
-
-	size += len(proto.Body(t.packets[t.headerPacket].Data))
-
-	for _, p := range t.packets[t.headerPacket+1:] {
-		size += len(p.Data)
-	}
-
-	return
-}
-
-// Size returns total size of message
-func (t *TCPMessage) Size() (size int) {
-	if len(t.packets) == 0 {
-		return 0
-	}
-
-	for _, p := range t.packets {
-		size += len(p.Data)
-	}
-
-	return
-}
-
-// AddPacket to the message and ensure packet uniqueness
-// TCP allows that packet can be re-send multiple times
-func (t *TCPMessage) AddPacket(packet *TCPPacket) {
-	for _, pkt := range t.packets {
-		if packet.Seq == pkt.Seq {
-			return
-		}
-	}
-
-	// Packets not always captured in same Seq order, and sometimes we need to prepend
-	if len(t.packets) == 0 || packet.Seq > t.packets[len(t.packets)-1].Seq {
-		t.packets = append(t.packets, packet)
-	} else if packet.Seq < t.packets[0].Seq {
-		t.packets = append([]*TCPPacket{packet}, t.packets...)
-		t.Seq = packet.Seq // Message Seq should indicated starting seq
-	} else { // insert somewhere in the middle...
-		for i, p := range t.packets {
-			if packet.Seq < p.Seq {
-				t.packets = append(t.packets[:i], append([]*TCPPacket{packet}, t.packets[i:]...)...)
-				break
-			}
-		}
-
-		if packet.OrigAck != 0 {
-			t.DataAck = packet.OrigAck
-		}
-
-		if packet.timestamp.Before(t.Start) || t.Start.IsZero() {
-			t.Start = packet.timestamp
-		}
-
-		if packet.timestamp.After(t.End) || t.End.IsZero() {
-			t.End = packet.timestamp
-		}
-	}
-
-	t.checkSeqIntegrity()
-
-	if t.protocol == ProtocolHTTP {
-		t.updateHeadersPacket()
-		t.updateMethodType()
-		t.updateBodyType()
-		t.check100Continue()
-		t.checkIfComplete()
-	}
-}
-
-// Check if there is missing packet
-func (t *TCPMessage) checkSeqIntegrity() {
-	if len(t.packets) == 1 {
-		t.seqMissing = false
-	}
-
-	offset := len(t.packets) - 1
-
-	if t.packets[offset].IsFIN {
-		offset--
-
-		if offset < 0 {
-			return
-		}
-	}
-
-	for i, p := range t.packets[:offset] {
-		if p.IsFIN {
-			continue
-		}
-
-		// If final packet
-		if len(t.packets) == i+1 {
-			t.seqMissing = false
-			return
-		}
-		np := t.packets[i+1]
-
-		nextSeq := p.Seq + uint32(len(p.Data))
-
-		if np.Seq != nextSeq {
-			if t.protocol == ProtocolHTTP && t.expectType == httpExpect100Continue {
-				if np.Seq != nextSeq+22 {
-					t.seqMissing = true
-					return
-				}
-			} else {
-				t.seqMissing = true
-				return
-			}
-		}
-	}
-
-	t.seqMissing = false
-}
-
-var bEmptyLine = []byte("\r\n\r\n")
-var bBR = []byte("\r\n")
-
-// last-chunk always is 0\r\n\r\n\. More info https://tools.ietf.org/html/rfc2616#section-3.6.1
-var bChunkEnd = []byte("0\r\n\r\n")
-
-func (t *TCPMessage) updateHeadersPacket() {
-	if len(t.packets) == 1 {
-		t.headerPacket = -1
-	}
-
-	if t.headerPacket != -1 {
-		return
-	}
-
-	if t.seqMissing {
-		return
-	}
-
-	for i, p := range t.packets {
-		if len(p.Data) >= len(bEmptyLine) {
-			if bytes.LastIndex(p.Data, bEmptyLine) != -1 {
-				t.headerPacket = i
-				return
-			}
-		} else if i > 0 && bytes.Equal(p.Data, bBR) {
-			idx := bytes.LastIndex(t.packets[i-1].Data, bBR)
-			if idx != -1 && idx == len(t.packets[i-1].Data)-len(bBR) {
-				t.headerPacket = i
-				return
-			}
-		}
-	}
-
-	return
-}
-
-// checkIfComplete returns true if all of the packets that compse the message arrived.
-func (t *TCPMessage) checkIfComplete() {
-	if t.seqMissing || t.headerPacket == -1 {
-		// log.Println("Seq missing", t.seqMissing, t.packets)
-		return
-	}
-
-	if t.methodType == httpMethodNotFound {
-		// log.Println("Method missing", t.methodType, t.packets)
-		return
-	}
-
-	// Responses can be emitted only if we found request
-	if !t.IsIncoming && t.AssocMessage == nil {
-		// log.Println("Assoc not found", t)
-		return
-	}
-
-	// log.Println("Found?", t)
-
-	switch t.bodyType {
-	case httpBodyEmpty:
-		t.complete = true
-	case httpBodyContentLength:
-		if t.contentLength == 0 || t.contentLength == t.BodySize() {
-			t.complete = true
-		}
-	case httpBodyChunked:
-		lastPacket := t.packets[len(t.packets)-1]
-		if bytes.LastIndex(lastPacket.Data, bChunkEnd) != -1 {
-			t.complete = true
-		}
-	default:
-		if len(t.packets) == 0 {
-			return
-		}
-
-		last := t.packets[len(t.packets)-1]
-		if last.IsFIN {
-			t.complete = true
-		}
-	}
-}
-
-type httpMethodType uint8
-
-const (
-	httpMethodNotSet   httpMethodType = 0
-	httpMethodKnown    httpMethodType = 1
-	httpMethodNotFound httpMethodType = 2
-)
-
-func (t *TCPMessage) updateMethodType() {
-	// if there is cache
-	if t.methodType != httpMethodNotSet && t.methodType != httpMethodNotFound {
-		return
-	}
-
-	d := t.packets[0].Data
-
-	// Minimum length fo request: GET / HTTP/1.1\r\n
-
-	if len(d) < 16 {
-		t.methodType = httpMethodNotFound
-		return
-	}
-
-	if t.IsIncoming {
-		if mIdx := bytes.IndexByte(d[:8], ' '); mIdx != -1 {
-			// Check that after method we have absolute or relative path
-			switch d[mIdx+1] {
-			case '/', 'h', '*':
-			default:
-				t.methodType = httpMethodNotFound
-				return
-			}
-		} else {
-			t.methodType = httpMethodNotFound
-			return
-		}
-
-		t.methodType = httpMethodKnown
-	} else {
-		if !bytes.Equal(d[:6], []byte("HTTP/1")) {
-			t.methodType = httpMethodNotFound
-			return
-		}
-
-		t.methodType = httpMethodKnown
-	}
-}
-
-type httpBodyType uint8
-
-const (
-	httpBodyNotSet          httpBodyType = 0
-	httpBodyEmpty           httpBodyType = 1
-	httpBodyContentLength   httpBodyType = 2
-	httpBodyChunked         httpBodyType = 3
-	httpBodyConnectionClose httpBodyType = 4
-)
-
-func (t *TCPMessage) updateBodyType() {
-	// if there is cache
-	if t.bodyType != httpBodyNotSet {
-		return
-	}
-
-	// Headers not received
-	if t.headerPacket == -1 {
-		return
-	}
-
-	var lengthB, encB, connB []byte
-
-	proto.ParseHeaders(t.packetsData(), func(header, value []byte) bool {
-		if proto.HeadersEqual(header, []byte("Content-Length")) {
-			lengthB = value
-			return false
-		}
-
-		if proto.HeadersEqual(header, []byte("Transfer-Encoding")) {
-			encB = value
-			return false
-		}
-
-		if proto.HeadersEqual(header, []byte("Connection")) {
-			connB = value
-		}
-
-		return true
-	})
-
-	switch t.methodType {
-	case httpMethodNotFound:
-		return
-	case httpMethodKnown:
-
-		if !t.IsIncoming &&
-			t.AssocMessage != nil &&
-			bytes.IndexByte(t.AssocMessage.Bytes(), ' ') > -1 &&
-			bytes.Equal([]byte("HEAD"), proto.Method(t.AssocMessage.Bytes())) {
-			// Need to check if this is a response to a head request,
-			// in which case the body has to be empty regardless.
-			t.bodyType = httpBodyEmpty
-			return
-		}
-
-		if len(lengthB) > 0 {
-			t.contentLength, _ = strconv.Atoi(string(lengthB))
-			if t.contentLength == 0 {
-				t.bodyType = httpBodyEmpty
-			} else {
-				t.bodyType = httpBodyContentLength
-			}
-			return
-		}
-
-		if len(encB) > 0 {
-			t.bodyType = httpBodyChunked
-			return
-		}
-
-		if len(connB) > 0 && bytes.Equal(connB, []byte("close")) {
-			t.bodyType = httpBodyConnectionClose
-			return
-		}
-	}
-
-	t.bodyType = httpBodyEmpty
-}
-
-type httpExpectType uint8
-
-const (
-	httpExpectNotSet      httpExpectType = 0
-	httpExpectEmpty       httpExpectType = 1
-	httpExpect100Continue httpExpectType = 2
-)
-
-var bExpectHeader = []byte("Expect")
-var bExpect100Value = []byte("100-continue")
-
-func (t *TCPMessage) check100Continue() {
-	if t.expectType != httpExpectNotSet || len(t.packets[0].Data) < 25 {
-		return
-	}
-
-	if t.seqMissing || t.headerPacket == -1 {
-		return
-	}
-
-	last := t.packets[len(t.packets)-1]
-	// reading last 4 bytes for double CRLF
-	if !bytes.HasSuffix(last.Data, bEmptyLine) {
-		return
-	}
-
-	var expectB []byte
-	proto.ParseHeaders(t.packetsData(), func(header, value []byte) bool {
-		if proto.HeadersEqual(header, bExpectHeader) {
-			expectB = value
-			return false
-		}
-
-		return true
-	})
-
-	if len(expectB) > 0 && bytes.Equal(bExpect100Value, expectB) {
-		t.expectType = httpExpect100Continue
-		return
-	}
-
-	t.expectType = httpExpectEmpty
-}
-
-func (t *TCPMessage) setAssocMessage(m *TCPMessage) {
-	t.AssocMessage = m
-	t.checkIfComplete()
-}
-
-// UpdateResponseAck should be called after packet is added
-func (t *TCPMessage) UpdateResponseAck() uint32 {
-	lastPacket := t.packets[len(t.packets)-1]
-	if lastPacket.IsFIN && len(t.packets) > 1 {
-		lastPacket = t.packets[len(t.packets)-2]
-	}
-
-	respAck := lastPacket.Seq + uint32(len(lastPacket.Data))
-
-	if t.ResponseAck != respAck {
-		t.ResponseAck = lastPacket.Seq + uint32(len(lastPacket.Data))
-
-		// We swappwed src and dst port
-		copy(t.ResponseID[:16], lastPacket.Addr)
-		copy(t.ResponseID[16:], lastPacket.Raw[2:4]) // Src port
-		copy(t.ResponseID[18:], lastPacket.Raw[0:2]) // Dest port
-		binary.BigEndian.PutUint32(t.ResponseID[20:24], t.ResponseAck)
-	}
-
-	return t.ResponseAck
-}
-
-func (t *TCPMessage) UUID() []byte {
-	var key []byte
-
-	if t.IsIncoming {
-		// log.Println("UUID:", t.Ack, t.Start.UnixNano())
-		key = strconv.AppendInt(key, t.Start.UnixNano(), 10)
-		key = strconv.AppendUint(key, uint64(t.Ack), 10)
-	} else {
-		// log.Println("RequestMessage:", t.AssocMessage.Ack, t.AssocMessage.Start.UnixNano())
-		key = strconv.AppendInt(key, t.AssocMessage.Start.UnixNano(), 10)
-		key = strconv.AppendUint(key, uint64(t.AssocMessage.Ack), 10)
-	}
-
-	uuid := make([]byte, 40)
-	sha := sha1.Sum(key)
-	hex.Encode(uuid, sha[:20])
-
-	return uuid
-}
-
-func (t *TCPMessage) ID() tcpID {
-	return t.packets[0].ID
-}
-
-func (t *TCPMessage) IP() net.IP {
-	return net.IP(t.packets[0].Addr)
-}
-
-func (t *TCPMessage) String() string {
-	return strings.Join([]string{
-		"Len packets: " + strconv.Itoa(len(t.packets)),
-		"Data size:" + strconv.Itoa(len(t.Bytes())),
-		"Data:" + string(t.Bytes()),
-	}, "\n")
-}

capture/tcp_message_test.go

@@ -1,257 +0,0 @@
-package capture
-
-import (
-	"bytes"
-	"encoding/binary"
-	_ "log"
-	"testing"
-	"time"
-)
-
-func buildPacket(isIncoming bool, Ack, Seq uint32, Data []byte, timestamp time.Time) (packet *TCPPacket) {
-	var srcPort, destPort uint16
-
-	// For tests `listening` port is 0
-	if isIncoming {
-		srcPort = 1
-	} else {
-		destPort = 1
-	}
-
-	buf := make([]byte, 16)
-	binary.BigEndian.PutUint16(buf[2:4], destPort)
-	binary.BigEndian.PutUint16(buf[0:2], srcPort)
-	binary.BigEndian.PutUint32(buf[4:8], Seq)
-	binary.BigEndian.PutUint32(buf[8:12], Ack)
-	buf[12] = 64
-	buf = append(buf, Data...)
-
-	packet = ParseTCPPacket([]byte("123"), buf, timestamp)
-
-	return packet
-}
-
-func buildMessage(p *TCPPacket) *TCPMessage {
-	isIncoming := false
-	if p.SrcPort == 1 {
-		isIncoming = true
-	}
-	m := NewTCPMessage(p.Seq, p.Ack, isIncoming, ProtocolHTTP, p.timestamp)
-	m.AddPacket(p)
-
-	return m
-}
-
-func TestTCPMessagePacketsOrder(t *testing.T) {
-	msg := buildMessage(buildPacket(true, 1, 1, []byte("a"), time.Now()))
-	msg.AddPacket(buildPacket(true, 1, 2, []byte("b"), time.Now()))
-
-	if !bytes.Equal(msg.Bytes(), []byte("ab")) {
-		t.Error("Should contatenate packets in right order")
-	}
-
-	// When first packet have wrong order (Seq)
-	msg = buildMessage(buildPacket(true, 1, 2, []byte("b"), time.Now()))
-	msg.AddPacket(buildPacket(true, 1, 1, []byte("a"), time.Now()))
-
-	if !bytes.Equal(msg.Bytes(), []byte("ab")) {
-		t.Error("Should contatenate packets in right order")
-	}
-
-	// Should ignore packets with same sequence
-	msg = buildMessage(buildPacket(true, 1, 1, []byte("a"), time.Now()))
-	msg.AddPacket(buildPacket(true, 1, 1, []byte("a"), time.Now()))
-
-	if !bytes.Equal(msg.Bytes(), []byte("a")) {
-		t.Error("Should ignore packet with same Seq")
-	}
-}
-
-func TestTCPMessageSize(t *testing.T) {
-	msg := buildMessage(buildPacket(true, 1, 1, []byte("POST / HTTP/1.1\r\nContent-Length: 2\r\n\r\na"), time.Now()))
-	msg.AddPacket(buildPacket(true, 1, 2, []byte("b"), time.Now()))
-
-	if msg.BodySize() != 2 {
-		t.Error("Should count only body", msg.BodySize())
-	}
-
-	if msg.Size() != 40 {
-		t.Error("Should count all sizes", msg.Size())
-	}
-}
-
-func TestTCPMessageIsComplete(t *testing.T) {
-	testCases := []struct {
-		direction         bool
-		payload           string
-		assocMessage      bool
-		expectedCompleted bool
-	}{
-		{true, "GET / HTTP/1.1\r\n\r\n", false, true},
-		{true, "HEAD / HTTP/1.1\r\n\r\n", false, true},
-		{false, "HTTP/1.1 200 OK\r\n\r\n", true, true},
-		{true, "POST / HTTP/1.1\r\nContent-Length: 1\r\n\r\na", false, true},
-		{true, "PUT / HTTP/1.1\r\nContent-Length: 1\r\n\r\na", false, true},
-		{false, "HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n", true, true},
-		{false, "HTTP/1.1 200 OK\r\nContent-Length: 1\r\n\r\na", true, true},
-		{false, "HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n0\r\n\r\n", true, true},
-
-		// chunked not finished
-		{false, "HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n", true, false},
-
-		// content-length != actual length
-		{true, "POST / HTTP/1.1\r\nContent-Length: 2\r\n\r\na", false, false},
-		{false, "HTTP/1.1 200 OK\r\nContent-Length: 10\r\n\r\na", true, false},
-		// non-valid http request
-		{true, "UNKNOWN asd HTTP/1.1\r\n\r\n", false, false},
-
-		// response without associated request
-		{false, "HTTP/1.1 200 OK\r\n\r\n", false, false},
-	}
-
-	for _, tc := range testCases {
-		msg := buildMessage(buildPacket(tc.direction, 1, 1, []byte(tc.payload), time.Now()))
-		if tc.assocMessage {
-			msg.AssocMessage = &TCPMessage{}
-		}
-		msg.checkIfComplete()
-
-		if msg.complete != tc.expectedCompleted {
-			t.Errorf("Payload %s: Expected %t, got %t.", tc.payload, tc.expectedCompleted, msg.complete)
-		}
-	}
-}
-
-func TestTCPMessageIsSeqMissing(t *testing.T) {
-	p1 := buildPacket(false, 1, 1, []byte("HTTP/1.1 200 OK\r\n"), time.Now())
-	p2 := buildPacket(false, 1, p1.Seq+uint32(len(p1.Data)), []byte("Content-Length: 10\r\n\r\n"), time.Now())
-	p3 := buildPacket(false, 1, p2.Seq+uint32(len(p2.Data)), []byte("a"), time.Now())
-
-	msg := buildMessage(p1)
-	if msg.seqMissing {
-		t.Error("Should be complete if have only 1 packet")
-	}
-
-	msg.AddPacket(p3)
-	if !msg.seqMissing {
-		t.Error("Should be incomplete because missing middle component")
-	}
-
-	msg.AddPacket(p2)
-	if msg.seqMissing {
-		t.Error("Should be complete once missing packet added")
-	}
-}
-
-func TestTCPMessageIsHeadersReceived(t *testing.T) {
-	p1 := buildPacket(false, 1, 1, []byte("HTTP/1.1 200 OK\r\n\r\n"), time.Now())
-	p2 := buildPacket(false, 1, p1.Seq+uint32(len(p1.Data)), []byte("Content-Length: 10\r\n\r\n"), time.Now())
-
-	msg := buildMessage(p1)
-	if msg.headerPacket == -1 {
-		t.Error("Should be complete if have only 1 packet", msg.headerPacket)
-	}
-
-	msg.AddPacket(p2)
-	if msg.headerPacket == -1 {
-		t.Error("Should found double new line: headers received")
-	}
-
-	msg = buildMessage(buildPacket(true, 1, 1, []byte("GET / HTTP/1.1\r\nContent-Length: 1\r\n"), time.Now()))
-	if msg.headerPacket != -1 {
-		t.Error("Should not find headers end")
-	}
-}
-
-func TestTCPMessageMethodType(t *testing.T) {
-	testCases := []struct {
-		direction          bool
-		payload            string
-		expectedMethodType httpMethodType
-	}{
-		{true, "GET / HTTP/1.1\r\n\r\n", httpMethodKnown},
-		{true, "GET * HTTP/1.1\r\n\r\n", httpMethodKnown},
-		{true, "UNKNOWN / HTTP/1.1\r\n\r\n", httpMethodKnown},
-		{true, "GET http://example.com HTTP/1.1\r\n\r\n", httpMethodKnown},
-		{true, "POST / HTTP/1.1\r\n\r\n", httpMethodKnown},
-		{true, "PUT / HTTP/1.1\r\n\r\n", httpMethodKnown},
-		{true, "GET zxc HTTP/1.1\r\n\r\n", httpMethodNotFound},
-		{true, "GET / HTTP\r\n\r\n", httpMethodNotFound},
-		{true, "VERYLONGMETHOD / HTTP/1.1\r\n\r\n", httpMethodNotFound},
-		{false, "HTTP/1.1 200 OK\r\n\r\n", httpMethodKnown},
-		{false, "HTTP /1.1 200 OK\r\n\r\n", httpMethodNotFound},
-	}
-
-	for _, tc := range testCases {
-		msg := buildMessage(buildPacket(tc.direction, 1, 1, []byte(tc.payload), time.Now()))
-
-		if msg.methodType != tc.expectedMethodType {
-			t.Errorf("Expected %d, got %d", tc.expectedMethodType, msg.methodType)
-		}
-	}
-}
-
-func TestTCPMessageBodyType(t *testing.T) {
-	testCases := []struct {
-		direction        bool
-		payload          string
-		expectedBodyType httpBodyType
-	}{
-		{true, "GET / HTTP/1.1\r\n\r\n", httpBodyEmpty},
-		{true, "GET / HTTP/1.1\r\nContent-Length: 2\r\n\r\nab", httpBodyContentLength},
-		{true, "POST / HTTP/1.1\r\n\r\n", httpBodyEmpty},
-		{true, "POST / HTTP/1.1\r\nUser-Agent: zxc\r\n\r\n", httpBodyEmpty},
-		{false, "HTTP/1.1 200 OK\r\n\r\n", httpBodyEmpty},
-		{true, "POST / HTTP/1.1\r\nContent-Length: 2\r\n\r\nab", httpBodyContentLength},
-		{false, "HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nab", httpBodyContentLength},
-		{true, "POST / HTTP/1.1\r\nTransfer-Encoding: chunked\r\n\r\n2\r\nab\r\n0\r\n\r\n", httpBodyChunked},
-		{false, "HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n2\r\nab\r\n0\r\n\r\n", httpBodyChunked},
-	}
-
-	for _, tc := range testCases {
-		msg := buildMessage(buildPacket(tc.direction, 1, 1, []byte(tc.payload), time.Now()))
-
-		if msg.bodyType != tc.expectedBodyType {
-			t.Errorf("Expected %d, got %d", tc.expectedBodyType, msg.bodyType)
-		}
-	}
-}
-
-func TestTCPMessageBodySize(t *testing.T) {
-	testCases := []struct {
-		direction    bool
-		payloads     []string
-		expectedSize int
-	}{
-		{true, []string{"GET / HTTP/1.1\r\n\r\n"}, 0},
-		{true, []string{"POST / HTTP/1.1\r\nContent-Length: 2\r\n\r\nab"}, 2},
-		{true, []string{"GET / HTTP/1.1\r\n", "Content-Length: 2\r\n\r\nab"}, 2},
-		{true, []string{"GET / HTTP/1.1\r\n", "Content-Length: 2\r\n\r\n", "ab"}, 2},
-	}
-
-	for _, tc := range testCases {
-		msg := buildMessage(buildPacket(tc.direction, 1, 1, []byte(tc.payloads[0]), time.Now()))
-
-		if len(tc.payloads) > 1 {
-			for _, p := range tc.payloads[1:] {
-				seq := uint32(1 + msg.Size())
-				msg.AddPacket(buildPacket(tc.direction, 1, seq, []byte(p), time.Now()))
-			}
-		}
-
-		if msg.BodySize() != tc.expectedSize {
-			t.Errorf("Expected %d, got %d", tc.expectedSize, msg.BodySize())
-		}
-	}
-}
-
-func TestTcpMessageStart(t *testing.T) {
-	start := time.Now().Add(-1 * time.Second)
-
-	msg := buildMessage(buildPacket(true, 1, 2, []byte("b"), time.Now()))
-	msg.AddPacket(buildPacket(true, 1, 1, []byte("POST / HTTP/1.1\r\nContent-Length: 2\r\n\r\na"), start))
-
-	if msg.Start != start {
-		t.Error("Message timestamp should be equal to the lowest related packet timestamp", start, msg.Start)
-	}
-}

capture/tcp_packet.go

@@ -1,129 +0,0 @@
-package capture
-
-import (
-	"encoding/binary"
-	"strconv"
-	"strings"
-	"time"
-)
-
-// TCP Flags
-const (
-	fFIN = 1 << iota
-	fSYN
-	fRST
-	fPSH
-	fACK
-	fURG
-	fECE
-	fCWR
-	fNS
-)
-
-type tcpID [24]byte
-
-// TCPPacket provides tcp packet parser
-// Packet structure: http://en.wikipedia.org/wiki/Transmission_Control_Protocol
-type TCPPacket struct {
-	SrcPort    uint16
-	DestPort   uint16
-	Seq        uint32
-	Ack        uint32
-	OrigAck    uint32
-	DataOffset uint8
-	IsFIN      bool
-
-	Raw       []byte
-	Data      []byte
-	Addr      []byte
-	timestamp time.Time
-	ID        tcpID
-}
-
-// ParseTCPPacket takes address and tcp payload and returns parsed TCPPacket
-func ParseTCPPacket(addr []byte, data []byte, timestamp time.Time) (p *TCPPacket) {
-	p = &TCPPacket{Raw: data}
-	p.ParseBasic()
-	p.Addr = addr
-	p.timestamp = timestamp
-	p.GenID()
-
-	return
-}
-
-func (p *TCPPacket) GenID() {
-	copy(p.ID[:16], p.Addr)
-	copy(p.ID[16:], p.Raw[0:2])  // Src port
-	copy(p.ID[18:], p.Raw[2:4])  // Dest port
-	copy(p.ID[20:], p.Raw[8:12]) // Ack
-}
-
-func (p *TCPPacket) UpdateAck(ack uint32) {
-	p.OrigAck = p.Ack
-	p.Ack = ack
-	binary.BigEndian.PutUint32(p.Raw[8:12], ack)
-	p.GenID()
-}
-
-// ParseBasic set of fields
-func (t *TCPPacket) ParseBasic() {
-	t.DestPort = binary.BigEndian.Uint16(t.Raw[2:4])
-	t.SrcPort = binary.BigEndian.Uint16(t.Raw[0:2])
-	t.Seq = binary.BigEndian.Uint32(t.Raw[4:8])
-	t.Ack = binary.BigEndian.Uint32(t.Raw[8:12])
-	t.DataOffset = (t.Raw[12] & 0xF0) >> 4
-	t.IsFIN = t.Raw[13]&0x01 != 0
-
-	if len(t.Raw) >= int(t.DataOffset*4) {
-		t.Data = t.Raw[t.DataOffset*4:]
-	}
-}
-
-func (t *TCPPacket) dump() *packet {
-
-	packetSrcIP := make([]byte, 16)
-	packetData := make([]byte, len(t.Data)+16)
-
-	copy(packetSrcIP, t.Addr)
-
-	binary.BigEndian.PutUint16(packetData[0:2], t.SrcPort)
-	binary.BigEndian.PutUint16(packetData[2:4], t.DestPort)
-
-	binary.BigEndian.PutUint32(packetData[4:8], t.Seq)
-	binary.BigEndian.PutUint32(packetData[8:12], t.Ack)
-
-	packetData[12] = 64
-
-	if t.IsFIN {
-		packetData[13] = packetData[13] | 0x01
-	}
-
-	copy(packetData[16:], t.Data)
-	return &packet{
-		srcIP:     packetSrcIP,
-		data:      packetData,
-		timestamp: t.timestamp,
-	}
-
-}
-
-// String output for a TCP Packet
-func (t *TCPPacket) String() string {
-	maxLen := len(t.Data)
-	if maxLen > 200 {
-		maxLen = 200
-	}
-
-	return strings.Join([]string{
-		"Addr: " + string(t.Addr),
-		"Source port: " + strconv.Itoa(int(t.SrcPort)),
-		"Dest port:" + strconv.Itoa(int(t.DestPort)),
-		"Sequence:" + strconv.Itoa(int(t.Seq)),
-		"Acknowledgment:" + strconv.Itoa(int(t.Ack)),
-		"Header len:" + strconv.Itoa(int(t.DataOffset)),
-		"FIN:" + strconv.FormatBool(t.IsFIN),
-
-		"Data size:" + strconv.Itoa(len(t.Data)),
-		"Data:" + string(t.Data[:maxLen]),
-	}, "\n")
-}

ce.go

@@ -0,0 +1,17 @@
+//go:build !pro
+
+package goreplay
+
+import (
+	"fmt"
+)
+
+// PRO this value indicates if goreplay is running in PRO mode.
+var PRO = false
+
+func SettingsHook(settings *AppSettings) {
+	if settings.RecognizeTCPSessions {
+		settings.RecognizeTCPSessions = false
+		fmt.Println("[ERROR] TCP session recognition is not supported in the open-source version of GoReplay")
+	}
+}

cmd/gor/gor.go

@@ -0,0 +1,161 @@
+// Gor is simple http traffic replication tool written in Go. Its main goal to replay traffic from production servers to staging and dev environments.
+// Now you can test your code on real user sessions in an automated and repeatable fashion.
+package main
+
+import (
+	"expvar"
+	"flag"
+	"fmt"
+	"github.com/buger/goreplay"
+	"log"
+	"net/http"
+	"net/http/httputil"
+	httppptof "net/http/pprof"
+	"os"
+	"os/signal"
+	"runtime"
+	"runtime/pprof"
+	"syscall"
+	"time"
+)
+
+var (
+	cpuprofile = flag.String("cpuprofile", "", "write cpu profile to file")
+	memprofile = flag.String("memprofile", "", "write memory profile to this file")
+)
+
+func init() {
+	var defaultServeMux http.ServeMux
+	http.DefaultServeMux = &defaultServeMux
+
+	http.HandleFunc("/debug/vars", func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "application/json; charset=utf-8")
+		fmt.Fprintf(w, "{\n")
+		first := true
+		expvar.Do(func(kv expvar.KeyValue) {
+			if kv.Key == "memstats" || kv.Key == "cmdline" {
+				return
+			}
+
+			if !first {
+				fmt.Fprintf(w, ",\n")
+			}
+			first = false
+			fmt.Fprintf(w, "%q: %s", kv.Key, kv.Value)
+		})
+		fmt.Fprintf(w, "\n}\n")
+	})
+
+	http.HandleFunc("/debug/pprof/", httppptof.Index)
+	http.HandleFunc("/debug/pprof/cmdline", httppptof.Cmdline)
+	http.HandleFunc("/debug/pprof/profile", httppptof.Profile)
+	http.HandleFunc("/debug/pprof/symbol", httppptof.Symbol)
+	http.HandleFunc("/debug/pprof/trace", httppptof.Trace)
+}
+
+func loggingMiddleware(addr string, next http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path == "/loop" {
+			_, err := http.Get("http://" + addr)
+			log.Println(err)
+		}
+
+		rb, _ := httputil.DumpRequest(r, false)
+		log.Println(string(rb))
+		next.ServeHTTP(w, r)
+	})
+}
+
+func main() {
+	if os.Getenv("GOMAXPROCS") == "" {
+		runtime.GOMAXPROCS(runtime.NumCPU() * 2)
+	}
+
+	args := os.Args[1:]
+	var plugins *goreplay.InOutPlugins
+	if len(args) > 0 && args[0] == "file-server" {
+		if len(args) != 2 {
+			log.Fatal("You should specify port and IP (optional) for the file server. Example: `gor file-server :80`")
+		}
+		dir, _ := os.Getwd()
+
+		goreplay.Debug(0, "Started example file server for current directory on address ", args[1])
+
+		log.Fatal(http.ListenAndServe(args[1], loggingMiddleware(args[1], http.FileServer(http.Dir(dir)))))
+	} else {
+		flag.Parse()
+		goreplay.CheckSettings()
+		plugins = goreplay.NewPlugins()
+	}
+
+	log.Printf("[PPID %d and PID %d] Version:%s\n", os.Getppid(), os.Getpid(), goreplay.VERSION)
+
+	if len(plugins.Inputs) == 0 || len(plugins.Outputs) == 0 {
+		log.Fatal("Required at least 1 input and 1 output")
+	}
+
+	if *memprofile != "" {
+		profileMEM(*memprofile)
+	}
+
+	if *cpuprofile != "" {
+		profileCPU(*cpuprofile)
+	}
+
+	if goreplay.Settings.Pprof != "" {
+		go func() {
+			log.Println(http.ListenAndServe(goreplay.Settings.Pprof, nil))
+		}()
+	}
+
+	closeCh := make(chan int)
+	emitter := goreplay.NewEmitter()
+	go emitter.Start(plugins, goreplay.Settings.Middleware)
+	if goreplay.Settings.ExitAfter > 0 {
+		log.Printf("Running gor for a duration of %s\n", goreplay.Settings.ExitAfter)
+
+		time.AfterFunc(goreplay.Settings.ExitAfter, func() {
+			log.Printf("gor run timeout %s\n", goreplay.Settings.ExitAfter)
+			close(closeCh)
+		})
+	}
+	c := make(chan os.Signal, 1)
+	signal.Notify(c, os.Interrupt, syscall.SIGTERM)
+	exit := 0
+	select {
+	case <-c:
+		exit = 1
+	case <-closeCh:
+		exit = 0
+	}
+	emitter.Close()
+	os.Exit(exit)
+}
+
+func profileCPU(cpuprofile string) {
+	if cpuprofile != "" {
+		f, err := os.Create(cpuprofile)
+		if err != nil {
+			log.Fatal(err)
+		}
+		pprof.StartCPUProfile(f)
+
+		time.AfterFunc(30*time.Second, func() {
+			pprof.StopCPUProfile()
+			f.Close()
+		})
+	}
+}
+
+func profileMEM(memprofile string) {
+	if memprofile != "" {
+		f, err := os.Create(memprofile)
+		if err != nil {
+			log.Fatal(err)
+		}
+		time.AfterFunc(30*time.Second, func() {
+			pprof.WriteHeapProfile(f)
+			f.Close()
+		})
+	}
+}

docs/Capturing-and-replaying-traffic.md

@@ -37,6 +37,11 @@ sudo gor --input-raw :80 --input-raw-engine "raw_socket" --output-http "http://s
 
 You can read more about [[Replaying HTTP traffic]].
 
+You can use VXLAN or traffic mirroring from AWS to capture the traffic. The 4789 UDP port will be opened and that works as you are launched GoReplay on the source machine.
+
+```
+gor --input-raw :80 --input-raw-engine vxlan -output-stdout
+```
 
 ### Tracking original IP addresses
 You can use `--input-raw-realip-header` option to specify header name: If not blank, injects header with given name and real IP value to the request payload. Usually, this header should be named: `X-Real-IP`, but you can specify any name.

elasticsearch.go

@@ -1,14 +1,13 @@
-package main
+package goreplay
 
 import (
 	"encoding/json"
+	"github.com/buger/goreplay/proto"
 	"log"
 	"net/url"
 	"strings"
 	"time"
 
-	"github.com/buger/goreplay/proto"
-
 	elastigo "github.com/mattbaird/elastigo/lib"
 )
 
@@ -57,7 +56,7 @@ type ESRequestResponse struct {
 //
 // Proper format is: scheme://[userinfo@]host/index_name
 // userinfo is: user[:password]
-// net/url.Parse() does not fail if scheme is not provided but actualy does not
+// net/url.Parse() does not fail if scheme is not provided but actually does not
 // handle URI properly.
 // So we must 'validate' URI format to match requirements to use net/url.Parse()
 func parseURI(URI string) (err error, index string) {
@@ -99,13 +98,9 @@ func (p *ESPlugin) Init(URI string) {
 	p.done = make(chan bool)
 	p.indexor.Start()
 
-	if Settings.Verbose {
-		// Only start the ErrorHandler goroutine when in verbose mode
-		// no need to burn ressources otherwise
-		go p.ErrorHandler()
-	}
+	go p.ErrorHandler()
 
-	log.Println("Initialized Elasticsearch Plugin")
+	Debug(1, "Initialized Elasticsearch Plugin")
 	return
 }
 
@@ -117,7 +112,7 @@ func (p *ESPlugin) IndexerShutdown() {
 func (p *ESPlugin) ErrorHandler() {
 	for {
 		errBuf := <-p.indexor.ErrorChannel
-		log.Println(errBuf.Err)
+		Debug(1, "[ELASTICSEARCH]", errBuf.Err)
 	}
 }
 
@@ -128,6 +123,7 @@ func (p *ESPlugin) RttDurationToMs(d time.Duration) int64 {
 	return int64(fl)
 }
 
+// ResponseAnalyze send req and resp to ES
 func (p *ESPlugin) ResponseAnalyze(req, resp []byte, start, stop time.Time) {
 	if len(resp) == 0 {
 		// nil http response - skipped elasticsearch export for this request
@@ -135,7 +131,6 @@ func (p *ESPlugin) ResponseAnalyze(req, resp []byte, start, stop time.Time) {
 	}
 	t := time.Now()
 	rtt := p.RttDurationToMs(stop.Sub(start))
-	req = payloadBody(req)
 
 	esResp := ESRequestResponse{
 		ReqURL:               string(proto.Path(req)),
@@ -163,7 +158,7 @@ func (p *ESPlugin) ResponseAnalyze(req, resp []byte, start, stop time.Time) {
 	}
 	j, err := json.Marshal(&esResp)
 	if err != nil {
-		log.Println(err)
+		Debug(0, "[ELASTIC-RESPONSE]", err)
 	} else {
 		p.indexor.Index(p.Index, "RequestResponse", "", "", "", &t, j)
 	}

elasticsearch_test.go

@@ -1,4 +1,4 @@
-package main
+package goreplay
 
 import (
 	"testing"

emitter.go

@@ -1,30 +1,33 @@
-package main
+package goreplay
 
 import (
-	"bytes"
+	"fmt"
+	"github.com/buger/goreplay/internal/byteutils"
 	"hash/fnv"
 	"io"
 	"log"
 	"sync"
-	"time"
+
+	"github.com/coocood/freecache"
 )
 
-type emitter struct {
+// Emitter represents an abject to manage plugins communication
+type Emitter struct {
 	sync.WaitGroup
-	quit chan int
+	plugins *InOutPlugins
 }
 
-// NewEmitter creates and initializes new `emitter` object.
-func NewEmitter(quit chan int) *emitter {
-	return &emitter{
-		quit: quit,
-	}
+// NewEmitter creates and initializes new Emitter object.
+func NewEmitter() *Emitter {
+	return &Emitter{}
 }
 
 // Start initialize loop for sending data from inputs to outputs
-func (e *emitter) Start(plugins *InOutPlugins, middlewareCmd string) {
-	e.Add(1)
-	defer e.Done()
+func (e *Emitter) Start(plugins *InOutPlugins, middlewareCmd string) {
+	if Settings.CopyBufferSize < 1 {
+		Settings.CopyBufferSize = 5 << 20
+	}
+	e.plugins = plugins
 
 	if middlewareCmd != "" {
 		middleware := NewMiddleware(middlewareCmd)
@@ -33,159 +36,93 @@ func (e *emitter) Start(plugins *InOutPlugins, middlewareCmd string) {
 			middleware.ReadFrom(in)
 		}
 
-		// We are going only to read responses, so using same ReadFrom method
-		for _, out := range plugins.Outputs {
-			if r, ok := out.(io.Reader); ok {
-				middleware.ReadFrom(r)
-			}
-		}
+		e.plugins.Inputs = append(e.plugins.Inputs, middleware)
+		e.plugins.All = append(e.plugins.All, middleware)
 		e.Add(1)
 		go func() {
 			defer e.Done()
-			if err := CopyMulty(e.quit, middleware, plugins.Outputs...); err != nil {
-				log.Println("Error during copy: ", err)
-				e.close()
-			}
-		}()
-		go func() {
-			for {
-				select {
-				case <-e.quit:
-					middleware.Close()
-					return
-				}
+			if err := CopyMulty(middleware, plugins.Outputs...); err != nil {
+				Debug(2, fmt.Sprintf("[EMITTER] error during copy: %q", err))
 			}
 		}()
 	} else {
 		for _, in := range plugins.Inputs {
 			e.Add(1)
-			go func(in io.Reader) {
+			go func(in PluginReader) {
 				defer e.Done()
-				if err := CopyMulty(e.quit, in, plugins.Outputs...); err != nil {
-					log.Println("Error during copy: ", err)
-					e.close()
+				if err := CopyMulty(in, plugins.Outputs...); err != nil {
+					Debug(2, fmt.Sprintf("[EMITTER] error during copy: %q", err))
 				}
 			}(in)
 		}
-
-		for _, out := range plugins.Outputs {
-			if r, ok := out.(io.Reader); ok {
-				e.Add(1)
-				go func(r io.Reader) {
-					defer e.Done()
-					if err := CopyMulty(e.quit, r, plugins.Outputs...); err != nil {
-						log.Println("Error during copy: ", err)
-						e.close()
-					}
-				}(r)
-			}
-		}
-	}
-
-	for {
-		select {
-		case <-e.quit:
-			finalize(plugins)
-			return
-		case <-time.After(100 * time.Millisecond):
-		}
-	}
-}
-
-func (e *emitter) close() {
-	select {
-	case <-e.quit:
-	default:
-		close(e.quit)
 	}
 }
 
 // Close closes all the goroutine and waits for it to finish.
-func (e *emitter) Close() {
-	e.close()
-	e.Wait()
+func (e *Emitter) Close() {
+	for _, p := range e.plugins.All {
+		if cp, ok := p.(io.Closer); ok {
+			cp.Close()
+		}
+	}
+	if len(e.plugins.All) > 0 {
+		// wait for everything to stop
+		e.Wait()
+	}
+	e.plugins.All = nil // avoid Close to make changes again
 }
 
 // CopyMulty copies from 1 reader to multiple writers
-func CopyMulty(stop chan int, src io.Reader, writers ...io.Writer) error {
-	buf := make([]byte, Settings.copyBufferSize)
+func CopyMulty(src PluginReader, writers ...PluginWriter) error {
 	wIndex := 0
 	modifier := NewHTTPModifier(&Settings.ModifierConfig)
-	filteredRequests := make(map[string]time.Time)
-	filteredRequestsLastCleanTime := time.Now()
+	filteredRequests := freecache.NewCache(200 * 1024 * 1024) // 200M
 
-	i := 0
 	for {
-		var nr int
-		nr, err := src.Read(buf)
-
-		select {
-		case <-stop:
-			return nil
-		default:
-		}
-
-		if err == io.EOF || err == ErrorStopped {
-			return nil
-		}
+		msg, err := src.PluginRead()
 		if err != nil {
+			if err == ErrorStopped || err == io.EOF {
+				return nil
+			}
 			return err
 		}
-
-		_maxN := nr
-		if nr > 500 {
-			_maxN = 500
-		}
-		if nr > 0 && len(buf) > nr {
-			payload := buf[:nr]
-			meta := payloadMeta(payload)
+		if msg != nil && len(msg.Data) > 0 {
+			if len(msg.Data) > int(Settings.CopyBufferSize) {
+				msg.Data = msg.Data[:Settings.CopyBufferSize]
+			}
+			meta := payloadMeta(msg.Meta)
 			if len(meta) < 3 {
-				if Settings.Debug {
-					Debug("[EMITTER] Found malformed record", string(payload[0:_maxN]), nr, "from:", src)
-				}
+				Debug(2, fmt.Sprintf("[EMITTER] Found malformed record %q from %q", msg.Meta, src))
 				continue
 			}
-			requestID := string(meta[1])
-
-			if nr >= 5*1024*1024 {
-				log.Println("INFO: Large packet... We received ", len(payload), " bytes from ", src)
+			requestID := meta[1]
+			// start a subroutine only when necessary
+			if Settings.Verbose >= 3 {
+				Debug(3, "[EMITTER] input: ", byteutils.SliceToString(msg.Meta[:len(msg.Meta)-1]), " from: ", src)
 			}
-
-			if Settings.Debug {
-				Debug("[EMITTER] input:", string(payload[0:_maxN]), nr, "from:", src)
-			}
-
 			if modifier != nil {
-				if isRequestPayload(payload) {
-					headSize := bytes.IndexByte(payload, '\n') + 1
-					body := payload[headSize:]
-					originalBodyLen := len(body)
-					body = modifier.Rewrite(body)
-
+				Debug(3, "[EMITTER] modifier:", requestID, "from:", src)
+				if isRequestPayload(msg.Meta) {
+					msg.Data = modifier.Rewrite(msg.Data)
 					// If modifier tells to skip request
-					if len(body) == 0 {
-						filteredRequests[requestID] = time.Now()
+					if len(msg.Data) == 0 {
+						filteredRequests.Set(requestID, []byte{}, 60) //
 						continue
 					}
+					Debug(3, "[EMITTER] Rewritten input:", requestID, "from:", src)
 
-					if originalBodyLen != len(body) {
-						payload = append(payload[:headSize], body...)
-					}
-
-					if Settings.Debug {
-						Debug("[EMITTER] Rewritten input:", len(payload), "First 500 bytes:", string(payload[0:_maxN]))
-					}
 				} else {
-					if _, ok := filteredRequests[requestID]; ok {
-						delete(filteredRequests, requestID)
+					_, err := filteredRequests.Get(requestID)
+					if err == nil {
+						filteredRequests.Del(requestID)
 						continue
 					}
 				}
 			}
 
 			if Settings.PrettifyHTTP {
-				payload = prettifyHTTP(payload)
-				if len(payload) == 0 {
+				msg.Data = prettifyHTTP(msg.Data)
+				if len(msg.Data) == 0 {
 					continue
 				}
 			}
@@ -196,49 +133,27 @@ func CopyMulty(stop chan int, src io.Reader, writers ...io.Writer) error {
 						log.Fatal("Detailed TCP sessions work only with PRO license")
 					}
 					hasher := fnv.New32a()
-					// First 20 bytes contain tcp session
-					id := payloadID(payload)
-					hasher.Write(id[:20])
+					hasher.Write(meta[1])
 
 					wIndex = int(hasher.Sum32()) % len(writers)
-					writers[wIndex].Write(payload)
+					if _, err := writers[wIndex].PluginWrite(msg); err != nil {
+						return err
+					}
 				} else {
 					// Simple round robin
-					if _, err := writers[wIndex].Write(payload); err != nil {
+					if _, err := writers[wIndex].PluginWrite(msg); err != nil {
 						return err
 					}
 
-					wIndex++
-
-					if wIndex >= len(writers) {
-						wIndex = 0
-					}
+					wIndex = (wIndex + 1) % len(writers)
 				}
 			} else {
 				for _, dst := range writers {
-					if _, err := dst.Write(payload); err != nil {
+					if _, err := dst.PluginWrite(msg); err != nil && err != io.ErrClosedPipe {
 						return err
 					}
 				}
 			}
-		} else if nr > 0 {
-			log.Println("WARN: Packet", nr, "bytes is too large to process. Consider increasing --copy-buffer-size")
 		}
-
-		// Run GC on each 1000 request
-		if i%1000 == 0 {
-			// Clean up filtered requests for which we didn't get a response to filter
-			now := time.Now()
-			if now.Sub(filteredRequestsLastCleanTime) > 60*time.Second {
-				for k, v := range filteredRequests {
-					if now.Sub(v) > 60*time.Second {
-						delete(filteredRequests, k)
-					}
-				}
-				filteredRequestsLastCleanTime = time.Now()
-			}
-		}
-
-		i++
 	}
 }

emitter_test.go

@@ -1,8 +1,7 @@
-package main
+package goreplay
 
 import (
-	"bytes"
-	"io"
+	"fmt"
 	"os"
 	"sync"
 	"sync/atomic"
@@ -18,20 +17,19 @@ func TestMain(m *testing.M) {
 
 func TestEmitter(t *testing.T) {
 	wg := new(sync.WaitGroup)
-	quit := make(chan int)
 
 	input := NewTestInput()
-	output := NewTestOutput(func(data []byte) {
+	output := NewTestOutput(func(*Message) {
 		wg.Done()
 	})
 
 	plugins := &InOutPlugins{
-		Inputs:  []io.Reader{input},
-		Outputs: []io.Writer{output},
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{output},
 	}
 	plugins.All = append(plugins.All, input, output)
 
-	emitter := NewEmitter(quit)
+	emitter := NewEmitter()
 	go emitter.Start(plugins, Settings.Middleware)
 
 	for i := 0; i < 1000; i++ {
@@ -45,32 +43,31 @@ func TestEmitter(t *testing.T) {
 
 func TestEmitterFiltered(t *testing.T) {
 	wg := new(sync.WaitGroup)
-	quit := make(chan int)
 
 	input := NewTestInput()
 	input.skipHeader = true
 
-	output := NewTestOutput(func(data []byte) {
+	output := NewTestOutput(func(*Message) {
 		wg.Done()
 	})
 
 	plugins := &InOutPlugins{
-		Inputs:  []io.Reader{input},
-		Outputs: []io.Writer{output},
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{output},
 	}
 	plugins.All = append(plugins.All, input, output)
 
 	methods := HTTPMethods{[]byte("GET")}
 	Settings.ModifierConfig = HTTPModifierConfig{Methods: methods}
 
-	emitter := &emitter{quit: quit}
+	emitter := &Emitter{}
 	go emitter.Start(plugins, "")
 
 	wg.Add(2)
 
 	id := uuid()
 	reqh := payloadHeader(RequestPayload, id, time.Now().UnixNano(), -1)
-	reqb := append(reqh, []byte("GET / HTTP/1.1\r\nHost: www.w3.org\r\nUser-Agent: Go 1.1 package http\r\nAccept-Encoding: gzip\r\n\r\n")...)
+	reqb := append(reqh, []byte("POST / HTTP/1.1\r\nHost: www.w3.org\r\nUser-Agent: Go 1.1 package http\r\nAccept-Encoding: gzip\r\n\r\n")...)
 
 	resh := payloadHeader(ResponsePayload, id, time.Now().UnixNano()+1, 1)
 	respb := append(resh, []byte("HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n")...)
@@ -80,7 +77,7 @@ func TestEmitterFiltered(t *testing.T) {
 
 	id = uuid()
 	reqh = payloadHeader(RequestPayload, id, time.Now().UnixNano(), -1)
-	reqb = append(reqh, []byte("POST / HTTP/1.1\r\nHost: www.w3.org\r\nUser-Agent: Go 1.1 package http\r\nAccept-Encoding: gzip\r\n\r\n")...)
+	reqb = append(reqh, []byte("GET / HTTP/1.1\r\nHost: www.w3.org\r\nUser-Agent: Go 1.1 package http\r\nAccept-Encoding: gzip\r\n\r\n")...)
 
 	resh = payloadHeader(ResponsePayload, id, time.Now().UnixNano()+1, 1)
 	respb = append(resh, []byte("HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n")...)
@@ -96,30 +93,29 @@ func TestEmitterFiltered(t *testing.T) {
 
 func TestEmitterSplitRoundRobin(t *testing.T) {
 	wg := new(sync.WaitGroup)
-	quit := make(chan int)
 
 	input := NewTestInput()
 
 	var counter1, counter2 int32
 
-	output1 := NewTestOutput(func(data []byte) {
+	output1 := NewTestOutput(func(*Message) {
 		atomic.AddInt32(&counter1, 1)
 		wg.Done()
 	})
 
-	output2 := NewTestOutput(func(data []byte) {
+	output2 := NewTestOutput(func(*Message) {
 		atomic.AddInt32(&counter2, 1)
 		wg.Done()
 	})
 
 	plugins := &InOutPlugins{
-		Inputs:  []io.Reader{input},
-		Outputs: []io.Writer{output1, output2},
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{output1, output2},
 	}
 
 	Settings.SplitOutput = true
 
-	emitter := NewEmitter(quit)
+	emitter := NewEmitter()
 	go emitter.Start(plugins, Settings.Middleware)
 
 	for i := 0; i < 1000; i++ {
@@ -140,31 +136,30 @@ func TestEmitterSplitRoundRobin(t *testing.T) {
 
 func TestEmitterRoundRobin(t *testing.T) {
 	wg := new(sync.WaitGroup)
-	quit := make(chan int)
 
 	input := NewTestInput()
 
 	var counter1, counter2 int32
 
-	output1 := NewTestOutput(func(data []byte) {
-		atomic.AddInt32(&counter1, 1)
+	output1 := NewTestOutput(func(*Message) {
+		counter1++
 		wg.Done()
 	})
 
-	output2 := NewTestOutput(func(data []byte) {
-		atomic.AddInt32(&counter2, 1)
+	output2 := NewTestOutput(func(*Message) {
+		counter2++
 		wg.Done()
 	})
 
 	plugins := &InOutPlugins{
-		Inputs:  []io.Reader{input},
-		Outputs: []io.Writer{output1, output2},
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{output1, output2},
 	}
 	plugins.All = append(plugins.All, input, output1, output2)
 
 	Settings.SplitOutput = true
 
-	emitter := NewEmitter(quit)
+	emitter := NewEmitter()
 	go emitter.Start(plugins, Settings.Middleware)
 
 	for i := 0; i < 1000; i++ {
@@ -183,67 +178,53 @@ func TestEmitterRoundRobin(t *testing.T) {
 }
 
 func TestEmitterSplitSession(t *testing.T) {
-	wg1 := new(sync.WaitGroup)
-	wg2 := new(sync.WaitGroup)
-	wg1.Add(1000)
-	wg2.Add(1000)
-
-	// Base uuids, only 1 letter changed
-	uuid1 := []byte("1234567890123456789a0000")
-	uuid2 := []byte("1234567890123456789d0000")
-
-	quit := make(chan int)
+	wg := new(sync.WaitGroup)
+	wg.Add(200)
 
 	input := NewTestInput()
 	input.skipHeader = true
 
 	var counter1, counter2 int32
 
-	output1 := NewTestOutput(func(data []byte) {
-		atomic.AddInt32(&counter1, 1)
-		if !bytes.Equal(uuid1[:20], payloadID(data)[:20]) {
-			t.Errorf("All tcp sessions should have same id")
+	output1 := NewTestOutput(func(msg *Message) {
+		if payloadID(msg.Meta)[0] == 'a' {
+			counter1++
 		}
-
-		wg1.Done()
+		wg.Done()
 	})
 
-	output2 := NewTestOutput(func(data []byte) {
-		atomic.AddInt32(&counter2, 1)
-		if !bytes.Equal(uuid2[:20], payloadID(data)[:20]) {
-			t.Errorf("All tcp sessions should have same id")
+	output2 := NewTestOutput(func(msg *Message) {
+		if payloadID(msg.Meta)[0] == 'b' {
+			counter2++
 		}
-
-		wg2.Done()
+		wg.Done()
 	})
 
 	plugins := &InOutPlugins{
-		Inputs:  []io.Reader{input},
-		Outputs: []io.Writer{output1, output2},
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{output1, output2},
 	}
 
 	Settings.SplitOutput = true
 	Settings.RecognizeTCPSessions = true
 
-	emitter := NewEmitter(quit)
+	emitter := NewEmitter()
 	go emitter.Start(plugins, Settings.Middleware)
 
-	for i := 0; i < 1000; i++ {
-		// Keep session but randomize ACK
-		copy(uuid1[20:], randByte(4))
-		input.EmitBytes([]byte("1 " + string(uuid1) + " 1\n" + "GET / HTTP/1.1\r\n\r\n"))
+	for i := 0; i < 200; i++ {
+		// Keep session but randomize
+		id := make([]byte, 20)
+		if i&1 == 0 { // for recognizeTCPSessions one should be odd and other will be even number
+			id[0] = 'a'
+		} else {
+			id[0] = 'b'
+		}
+		input.EmitBytes([]byte(fmt.Sprintf("1 %s 1 1\nGET / HTTP/1.1\r\n\r\n", id[:20])))
 	}
 
-	for i := 0; i < 1000; i++ {
-		// Keep session but randomize ACK
-		copy(uuid2[20:], randByte(4))
-		input.EmitBytes([]byte("1 " + string(uuid2) + " 1\n" + "GET / HTTP/1.1\r\n\r\n"))
-	}
+	wg.Wait()
 
-	wg1.Wait()
-	wg2.Wait()
-
-	if counter1 != 1000 || counter2 != 1000 {
+	if counter1 != counter2 {
 		t.Errorf("Round robin should split traffic equally: %d vs %d", counter1, counter2)
 	}
 
@@ -254,21 +235,20 @@ func TestEmitterSplitSession(t *testing.T) {
 
 func BenchmarkEmitter(b *testing.B) {
 	wg := new(sync.WaitGroup)
-	quit := make(chan int)
 
 	input := NewTestInput()
 
-	output := NewTestOutput(func(data []byte) {
+	output := NewTestOutput(func(*Message) {
 		wg.Done()
 	})
 
 	plugins := &InOutPlugins{
-		Inputs:  []io.Reader{input},
-		Outputs: []io.Writer{output},
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{output},
 	}
 	plugins.All = append(plugins.All, input, output)
 
-	emitter := NewEmitter(quit)
+	emitter := NewEmitter()
 	go emitter.Start(plugins, Settings.Middleware)
 
 	b.ResetTimer()

examples/middleware/echo.sh

@@ -8,8 +8,10 @@
 # 
 
 function log {
+    if [[ -n "$GOR_TEST" ]]; then # if we are not testing
     # Logging to stderr, because stdout/stdin used for data transfer
     >&2 echo "[DEBUG][ECHO] $1"
+    fi
 }
 
 while read line; do

examples/middleware/token_modifier.go

@@ -24,9 +24,8 @@ import (
 	"bytes"
 	"encoding/hex"
 	"fmt"
-	"os"
-
 	"github.com/buger/goreplay/proto"
+	"os"
 )
 
 // requestID -> originalToken
@@ -115,6 +114,8 @@ func encode(buf []byte) []byte {
 }
 
 func Debug(args ...interface{}) {
-	fmt.Fprint(os.Stderr, "[DEBUG][TOKEN-MOD] ")
-	fmt.Fprintln(os.Stderr, args...)
+	if os.Getenv("GOR_TEST") == "" { // if we are not testing
+		fmt.Fprint(os.Stderr, "[DEBUG][TOKEN-MOD] ")
+		fmt.Fprintln(os.Stderr, args...)
+	}
 }

go.mod

@@ -1,18 +1,80 @@
 module github.com/buger/goreplay
 
-go 1.14
+go 1.21
 
 require (
-	github.com/Shopify/sarama v1.26.4
-	github.com/araddon/gou v0.0.0-20190110011759-c797efecbb61 // indirect
-	github.com/aws/aws-sdk-go v1.33.2
+	github.com/Shopify/sarama v1.38.1
+	github.com/aws/aws-sdk-go v1.44.262
+	github.com/coocood/freecache v1.2.3
+	github.com/google/gopacket v1.1.20-0.20210429153827-3eaba0894325
+	github.com/gorilla/websocket v1.5.0
+	github.com/klauspost/compress v1.16.5 // indirect
+	github.com/mattbaird/elastigo v0.0.0-20170123220020-2fe47fd29e4b
+	github.com/stretchr/testify v1.8.2
+	github.com/xdg-go/scram v1.1.2
+	golang.org/x/net v0.34.0
+	golang.org/x/sys v0.29.0
+	k8s.io/apimachinery v0.27.1
+	k8s.io/client-go v0.27.1
+)
+
+require (
+	github.com/araddon/gou v0.0.0-20211019181548-e7d08105776c // indirect
 	github.com/bitly/go-hostpool v0.1.0 // indirect
 	github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 // indirect
-	github.com/google/gopacket v1.1.17
-	github.com/klauspost/compress v1.10.10 // indirect
-	github.com/mattbaird/elastigo v0.0.0-20170123220020-2fe47fd29e4b
-	github.com/pierrec/lz4 v2.5.2+incompatible // indirect
-	github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0 // indirect
-	github.com/smartystreets/goconvey v1.6.4 // indirect
-	golang.org/x/net v0.0.0-20200707034311-ab3426394381 // indirect
+	github.com/cespare/xxhash/v2 v2.2.0 // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/eapache/go-resiliency v1.3.0 // indirect
+	github.com/eapache/go-xerial-snappy v0.0.0-20230111030713-bf00bc1b83b6 // indirect
+	github.com/eapache/queue v1.1.0 // indirect
+	github.com/emicklei/go-restful/v3 v3.10.2 // indirect
+	github.com/go-logr/logr v1.2.4 // indirect
+	github.com/go-openapi/jsonpointer v0.19.6 // indirect
+	github.com/go-openapi/jsonreference v0.20.2 // indirect
+	github.com/go-openapi/swag v0.22.3 // indirect
+	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/golang/protobuf v1.5.3 // indirect
+	github.com/golang/snappy v0.0.4 // indirect
+	github.com/google/gnostic v0.6.9 // indirect
+	github.com/google/go-cmp v0.5.9 // indirect
+	github.com/google/gofuzz v1.2.0 // indirect
+	github.com/google/uuid v1.3.0 // indirect
+	github.com/hashicorp/errwrap v1.1.0 // indirect
+	github.com/hashicorp/go-multierror v1.1.1 // indirect
+	github.com/hashicorp/go-uuid v1.0.3 // indirect
+	github.com/jcmturner/aescts/v2 v2.0.0 // indirect
+	github.com/jcmturner/dnsutils/v2 v2.0.0 // indirect
+	github.com/jcmturner/gofork v1.7.6 // indirect
+	github.com/jcmturner/gokrb5/v8 v8.4.4 // indirect
+	github.com/jcmturner/rpc/v2 v2.0.3 // indirect
+	github.com/jmespath/go-jmespath v0.4.0 // indirect
+	github.com/josharian/intern v1.0.0 // indirect
+	github.com/json-iterator/go v1.1.12 // indirect
+	github.com/mailru/easyjson v0.7.7 // indirect
+	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
+	github.com/modern-go/reflect2 v1.0.2 // indirect
+	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
+	github.com/pierrec/lz4/v4 v4.1.17 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect
+	github.com/smartystreets/goconvey v1.7.2 // indirect
+	github.com/xdg-go/pbkdf2 v1.0.0 // indirect
+	github.com/xdg-go/stringprep v1.0.4 // indirect
+	golang.org/x/crypto v0.32.0 // indirect
+	golang.org/x/oauth2 v0.8.0 // indirect
+	golang.org/x/term v0.28.0 // indirect
+	golang.org/x/text v0.21.0 // indirect
+	golang.org/x/time v0.3.0 // indirect
+	google.golang.org/appengine v1.6.7 // indirect
+	google.golang.org/protobuf v1.30.0 // indirect
+	gopkg.in/inf.v0 v0.9.1 // indirect
+	gopkg.in/yaml.v2 v2.4.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
+	k8s.io/api v0.27.1 // indirect
+	k8s.io/klog/v2 v2.100.1 // indirect
+	k8s.io/kube-openapi v0.0.0-20230501164219-8b0f38b5fd1f // indirect
+	k8s.io/utils v0.0.0-20230505201702-9f6742963106 // indirect
+	sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
+	sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect
+	sigs.k8s.io/yaml v1.3.0 // indirect
 )

go.sum

@@ -1,117 +1,384 @@
-github.com/Shopify/sarama v1.26.4 h1:+17TxUq/PJEAfZAll0T7XJjSgQWCpaQSoki/x5yN8o8=
-github.com/Shopify/sarama v1.26.4/go.mod h1:NbSGBSSndYaIhRcBtY9V0U7AyH+x71bG668AuWys/yU=
-github.com/Shopify/toxiproxy v2.1.4+incompatible h1:TKdv8HiTLgE5wdJuEML90aBgNWsokNbMijUGhmcoBJc=
-github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI=
-github.com/araddon/gou v0.0.0-20190110011759-c797efecbb61 h1:Xz25cuW4REGC5W5UtpMU3QItMIImag615HiQcRbxqKQ=
-github.com/araddon/gou v0.0.0-20190110011759-c797efecbb61/go.mod h1:ikc1XA58M+Rx7SEbf0bLJCfBkwayZ8T5jBo5FXK8Uz8=
-github.com/aws/aws-sdk-go v1.33.2 h1:8TVrnPnSD7I+AmDp66xBUvS3K0J+jH09YXdrkJ34ey0=
-github.com/aws/aws-sdk-go v1.33.2/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0=
+cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
+github.com/Shopify/sarama v1.38.1 h1:lqqPUPQZ7zPqYlWpTh+LQ9bhYNu2xJL6k1SJN4WVe2A=
+github.com/Shopify/sarama v1.38.1/go.mod h1:iwv9a67Ha8VNa+TifujYoWGxWnu2kNVAQdSdZ4X2o5g=
+github.com/Shopify/toxiproxy/v2 v2.5.0 h1:i4LPT+qrSlKNtQf5QliVjdP08GyAH8+BUIc9gT0eahc=
+github.com/Shopify/toxiproxy/v2 v2.5.0/go.mod h1:yhM2epWtAmel9CB8r2+L+PCmhH6yH2pITaPAo7jxJl0=
+github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
+github.com/araddon/gou v0.0.0-20211019181548-e7d08105776c h1:XUqw//RExYoxW4Eie8MuKp8sEDAZI1gMHX/daUFgZww=
+github.com/araddon/gou v0.0.0-20211019181548-e7d08105776c/go.mod h1:ikc1XA58M+Rx7SEbf0bLJCfBkwayZ8T5jBo5FXK8Uz8=
+github.com/aws/aws-sdk-go v1.44.262 h1:gyXpcJptWoNkK+DiAiaBltlreoWKQXjAIh6FRh60F+I=
+github.com/aws/aws-sdk-go v1.44.262/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI=
 github.com/bitly/go-hostpool v0.1.0 h1:XKmsF6k5el6xHG3WPJ8U0Ku/ye7njX7W81Ng7O2ioR0=
 github.com/bitly/go-hostpool v0.1.0/go.mod h1:4gOCgp6+NZnVqlKyZ/iBZFTAJKembaVENUpMkpg42fw=
 github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 h1:DDGfHa7BWjL4YnC6+E63dPcxHo2sUxDIu8g3QgEJdRY=
 github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4=
-github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
+github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0=
+github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
+github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
+github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
+github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
+github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
+github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
+github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
+github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
+github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
+github.com/coocood/freecache v1.2.3 h1:lcBwpZrwBZRZyLk/8EMyQVXRiFl663cCuMOrjCALeto=
+github.com/coocood/freecache v1.2.3/go.mod h1:RBUWa/Cy+OHdfTGFEhEuE1pMCMX51Ncizj7rthiQ3vk=
+github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/eapache/go-resiliency v1.2.0 h1:v7g92e/KSN71Rq7vSThKaWIq68fL4YHvWyiUKorFR1Q=
-github.com/eapache/go-resiliency v1.2.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs=
-github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 h1:YEetp8/yCZMuEPMUDHG0CW/brkkEp8mzqk2+ODEitlw=
-github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU=
+github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE=
+github.com/eapache/go-resiliency v1.3.0 h1:RRL0nge+cWGlxXbUzJ7yMcq6w2XBEr19dCN6HECGaT0=
+github.com/eapache/go-resiliency v1.3.0/go.mod h1:5yPzW0MIvSe0JDsv0v+DvcjEv2FyD6iZYSs1ZI+iQho=
+github.com/eapache/go-xerial-snappy v0.0.0-20230111030713-bf00bc1b83b6 h1:8yY/I9ndfrgrXUbOGObLHKBR4Fl3nZXwM2c7OYTT8hM=
+github.com/eapache/go-xerial-snappy v0.0.0-20230111030713-bf00bc1b83b6/go.mod h1:YvSRo5mw33fLEx1+DlK6L2VV43tJt5Eyel9n9XBcR+0=
 github.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc=
 github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I=
+github.com/emicklei/go-restful/v3 v3.10.2 h1:hIovbnmBTLjHXkqEBUz3HGpXZdM7ZrE9fJIZIqlJLqE=
+github.com/emicklei/go-restful/v3 v3.10.2/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc=
+github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
+github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
+github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ=
+github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
+github.com/flowstack/go-jsonschema v0.1.1/go.mod h1:yL7fNggx1o8rm9RlgXv7hTBWxdBM0rVwpMwimd3F3N0=
 github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw=
 github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g=
-github.com/frankban/quicktest v1.7.2 h1:2QxQoC1TS09S7fhCPsrvqYdvP1H5M1P1ih5ABm3BTYk=
-github.com/frankban/quicktest v1.7.2/go.mod h1:jaStnuzAqU1AJdCO0l53JDCJrVDKcS03DbaAcR7Ks/o=
-github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
-github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4=
-github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
+github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
+github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ=
+github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
+github.com/go-openapi/jsonpointer v0.19.6 h1:eCs3fxoIi3Wh6vtgmLTOjdhSpiqphQ+DaPn38N2ZdrE=
+github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs=
+github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE=
+github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k=
+github.com/go-openapi/swag v0.22.3 h1:yMBqmnQ0gyZvEb/+KzuWZOXgllrXT4SADYbvDaXHv/g=
+github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14=
+github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 h1:p104kn46Q8WdvHunIJ9dAyjPVtrBPhSr3KT2yUst43I=
+github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE=
+github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
+github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
+github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
+github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
+github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
+github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
+github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
+github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
+github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
+github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
+github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
+github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
+github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
+github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
+github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg=
+github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
+github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM=
+github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/google/gnostic v0.6.9 h1:ZK/5VhkoX835RikCHpSUJV9a+S3e1zLh59YnyWeBW+0=
+github.com/google/gnostic v0.6.9/go.mod h1:Nm8234We1lq6iB9OmlgNv3nH91XLLVZHCDayfA3xq+E=
+github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
+github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
-github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4=
 github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/gopacket v1.1.17 h1:rMrlX2ZY2UbvT+sdz3+6J+pp2z+msCq9MxTU6ymxbBY=
-github.com/google/gopacket v1.1.17/go.mod h1:UdDNZ1OO62aGYVnPhxT1U6aI7ukYtA/kB8vaU0diBUM=
+github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
+github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0=
+github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/google/gopacket v1.1.20-0.20210429153827-3eaba0894325 h1:YmIcZ5Var3BAQ64AW98Iiys5Ih4fiU0xK41+8isC5Ec=
+github.com/google/gopacket v1.1.20-0.20210429153827-3eaba0894325/go.mod h1:riddUzxTSBpJXk3qBHtYr4qOhFhT6k/1c0E3qkQjQpA=
+github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 h1:K6RDEckDVWvDI9JAJYCmNdQXq6neHJOYx3V6jnqNEec=
+github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
+github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
+github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8=
 github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
-github.com/hashicorp/go-uuid v1.0.2 h1:cfejS+Tpcp13yd5nYHWDI6qVCny6wyX2Mt5SGur2IGE=
+github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4=
+github.com/gorilla/sessions v1.2.1/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/zI+bUmuGM=
+github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc=
+github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
+github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw=
+github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
+github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I=
+github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
+github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo=
+github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM=
 github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
-github.com/jcmturner/gofork v1.0.0 h1:J7uCkflzTEhUZ64xqKnkDxq3kzc96ajM1Gli5ktUem8=
-github.com/jcmturner/gofork v1.0.0/go.mod h1:MK8+TM0La+2rjBD4jE12Kj1pCCxK7d2LK/UM3ncEo0o=
-github.com/jmespath/go-jmespath v0.3.0 h1:OS12ieG61fsCg5+qLJ+SsW9NicxNkg3b25OyT2yCeUc=
-github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik=
+github.com/hashicorp/go-uuid v1.0.3 h1:2gKiV6YVmrJ1i2CKKa9obLvRieoRGviZFL26PcT/Co8=
+github.com/hashicorp/go-uuid v1.0.3/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
+github.com/jcmturner/aescts/v2 v2.0.0 h1:9YKLH6ey7H4eDBXW8khjYslgyqG2xZikXP0EQFKrle8=
+github.com/jcmturner/aescts/v2 v2.0.0/go.mod h1:AiaICIRyfYg35RUkr8yESTqvSy7csK90qZ5xfvvsoNs=
+github.com/jcmturner/dnsutils/v2 v2.0.0 h1:lltnkeZGL0wILNvrNiVCR6Ro5PGU/SeBvVO/8c/iPbo=
+github.com/jcmturner/dnsutils/v2 v2.0.0/go.mod h1:b0TnjGOvI/n42bZa+hmXL+kFJZsFT7G4t3HTlQ184QM=
+github.com/jcmturner/gofork v1.7.6 h1:QH0l3hzAU1tfT3rZCnW5zXl+orbkNMMRGJfdJjHVETg=
+github.com/jcmturner/gofork v1.7.6/go.mod h1:1622LH6i/EZqLloHfE7IeZ0uEJwMSUyQ/nDd82IeqRo=
+github.com/jcmturner/goidentity/v6 v6.0.1 h1:VKnZd2oEIMorCTsFBnJWbExfNN7yZr3EhJAxwOkZg6o=
+github.com/jcmturner/goidentity/v6 v6.0.1/go.mod h1:X1YW3bgtvwAXju7V3LCIMpY0Gbxyjn/mY9zx4tFonSg=
+github.com/jcmturner/gokrb5/v8 v8.4.4 h1:x1Sv4HaTpepFkXbt2IkL29DXRf8sOfZXo8eRKh687T8=
+github.com/jcmturner/gokrb5/v8 v8.4.4/go.mod h1:1btQEpgT6k+unzCwX1KdWMEwPPkkgBtP+F6aCACiMrs=
+github.com/jcmturner/rpc/v2 v2.0.3 h1:7FXXj8Ti1IaVFpSAziCZWNzbNuZmnvw/i6CqLNdWfZY=
+github.com/jcmturner/rpc/v2 v2.0.3/go.mod h1:VUJYCIDm3PVOEHw8sgt091/20OJjskO/YJki3ELg/Hc=
+github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
+github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
+github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
+github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
+github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
+github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
+github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
+github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
 github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
 github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
-github.com/klauspost/compress v1.9.8 h1:VMAMUUOh+gaxKTMk+zqbjsSjsIcUcL/LF4o63i82QyA=
-github.com/klauspost/compress v1.9.8/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
-github.com/klauspost/compress v1.10.10 h1:a/y8CglcM7gLGYmlbP/stPE5sR3hbhFRUjCBfd/0B3I=
-github.com/klauspost/compress v1.10.10/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
-github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
-github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs=
+github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
+github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
+github.com/klauspost/compress v1.16.5 h1:IFV2oUNUzZaz+XyusxpLzpzS8Pt5rh0Z16For/djlyI=
+github.com/klauspost/compress v1.16.5/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
 github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
+github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
+github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0=
+github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
-github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
+github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
+github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
+github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
 github.com/mattbaird/elastigo v0.0.0-20170123220020-2fe47fd29e4b h1:v29yPGHhOqw7VHEnTeQFAth3SsBrmwc8JfuhNY0G34k=
 github.com/mattbaird/elastigo v0.0.0-20170123220020-2fe47fd29e4b/go.mod h1:5MWrJXKRQyhQdUCF+vu6U5c4nQpg70vW3eHaU0/AYbU=
-github.com/pierrec/lz4 v2.4.1+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
-github.com/pierrec/lz4 v2.5.2+incompatible h1:WCjObylUIOlKy/+7Abdn34TLIkXiA4UWUMhxq9m9ZXI=
-github.com/pierrec/lz4 v2.5.2+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
+github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
+github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
+github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
+github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
+github.com/onsi/ginkgo/v2 v2.9.1 h1:zie5Ly042PD3bsCvsSOPvRnFwyo3rKe64TJlD6nu0mk=
+github.com/onsi/ginkgo/v2 v2.9.1/go.mod h1:FEcmzVcCHl+4o9bQZVab+4dC9+j+91t2FHSzmGAPfuo=
+github.com/onsi/gomega v1.27.4 h1:Z2AnStgsdSayCMDiCU42qIz+HLqEPcgiOCXjAU/w+8E=
+github.com/onsi/gomega v1.27.4/go.mod h1:riYq/GJKh8hhoM01HN6Vmuy93AarCXCBGpvFDK3q3fQ=
+github.com/pierrec/lz4/v4 v4.1.17 h1:kV4Ip+/hUBC+8T6+2EgburRtkE9ef4nbY3f4dFhGjMc=
+github.com/pierrec/lz4/v4 v4.1.17/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
-github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0 h1:MkV+77GLUNo5oJ0jf870itWm3D0Sjh7+Za9gazKc5LQ=
-github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
-github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM=
-github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
-github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s=
-github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
+github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 h1:N/ElC8H3+5XpJzTSTfLsJV/mx9Q9g7kxmchpfZyxgzM=
+github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
+github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
+github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ=
+github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog=
+github.com/smartystreets/assertions v1.2.0 h1:42S6lae5dvLc7BrLu/0ugRtcFVjoJNMC/N3yZFZkDFs=
+github.com/smartystreets/assertions v1.2.0/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo=
+github.com/smartystreets/goconvey v1.7.2 h1:9RBaZCeXEQ3UselpuwUQHltGVXvdwm6cv1hgR6gDIPg=
+github.com/smartystreets/goconvey v1.7.2/go.mod h1:Vw0tHAZW6lzCRk3xgdin6fKYcG+G3Pg9vgXWeJpQFMM=
+github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
+github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
+github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
+github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
+github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
-github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4=
 github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
-github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I=
-github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y=
+github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
+github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
+github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8=
+github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
+github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE=
+github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU=
+github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0=
+github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c=
+github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
+github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY=
+github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4=
+github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8=
+github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM=
+github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
+github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ=
+github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y=
+github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
+go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/crypto v0.0.0-20200204104054-c9f3fb736b72 h1:+ELyKg6m8UBf0nPFSqD0mi7zUfwPyXo23HNjMnXPz7w=
-golang.org/x/crypto v0.0.0-20200204104054-c9f3fb736b72/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
-golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI=
+golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
+golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58=
+golang.org/x/crypto v0.32.0 h1:euUpcYgM8WcP71gNpTqQCn6rC2t6ULUPiOzfWaXVVfc=
+golang.org/x/crypto v0.32.0/go.mod h1:ZnnJkOaASj8g0AjIduWNlq2NRxL0PlBrbKVyZ6V/Ugc=
+golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
+golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
+golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
+golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
+golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
+golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20200202094626-16171245cfb2 h1:CCH4IOTTfewWjGOlSp+zGcjutRKlBEZQ6wTn8ozI/nI=
-golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20200707034311-ab3426394381 h1:VXak5I6aEWmAXeQjA+QSZzlgNrpq9mjcfDemuexIKsU=
-golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
+golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
+golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
+golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
+golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco=
+golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
+golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
+golang.org/x/net v0.34.0 h1:Mb7Mrk043xzHgnRM88suvJFwzVrRfHEHJEl5/71CKw0=
+golang.org/x/net v0.34.0/go.mod h1:di0qlW3YNM5oh6GqDGQr92MyTozJPmybPK4Ev/Gm31k=
+golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/oauth2 v0.8.0 h1:6dkIjl3j3LtZ/O3sTgZTMsLKSftL/B8Zgq4huOIIUu8=
+golang.org/x/oauth2 v0.8.0/go.mod h1:yr7u4HXZRm1R1kBWqr/xKNqewf0plRYoB7sla+BCIXE=
+golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ=
+golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
+golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20190405154228-4b34438f7a67/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884=
+golang.org/x/sys v0.0.0-20190606203320-7fc4e5ec1444/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200217220822-9197077df867/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
+golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.29.0 h1:TPYlXGxvx1MGTn2GiZDhnjPA9wZzZeGKHHmKhHYvgaU=
+golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
+golang.org/x/term v0.28.0 h1:/Ts8HFuMR2E6IP/jlo7QVLZHggjKQbhu/7H0LJFr3Gg=
+golang.org/x/term v0.28.0/go.mod h1:Sw/lC2IAUZ92udQNf3WodGtn4k/XoLyZoh8v/8uiwek=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
+golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
+golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
+golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
+golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo=
+golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
+golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4=
+golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
+golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
-golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
+golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
+golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
+golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg=
+golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
+google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c=
+google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
+google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
+google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
+google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
+google.golang.org/genproto v0.0.0-20220107163113-42d7afdf6368/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
+google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
+google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
+google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
+google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
+google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0=
+google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
+google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34=
+google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
+google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
+google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
+google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
+google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
+google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
+google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
+google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
+google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
+google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng=
+google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/jcmturner/aescts.v1 v1.0.1 h1:cVVZBK2b1zY26haWB4vbBiZrfFQnfbTVrE3xZq6hrEw=
-gopkg.in/jcmturner/aescts.v1 v1.0.1/go.mod h1:nsR8qBOg+OucoIW+WMhB3GspUQXq9XorLnQb9XtvcOo=
-gopkg.in/jcmturner/dnsutils.v1 v1.0.1 h1:cIuC1OLRGZrld+16ZJvvZxVJeKPsvd5eUIvxfoN5hSM=
-gopkg.in/jcmturner/dnsutils.v1 v1.0.1/go.mod h1:m3v+5svpVOhtFAP/wSz+yzh4Mc0Fg7eRhxkJMWSIz9Q=
-gopkg.in/jcmturner/goidentity.v3 v3.0.0 h1:1duIyWiTaYvVx3YX2CYtpJbUFd7/UuPYCfgXtQ3VTbI=
-gopkg.in/jcmturner/goidentity.v3 v3.0.0/go.mod h1:oG2kH0IvSYNIu80dVAyu/yoefjq1mNfM5bm88whjWx4=
-gopkg.in/jcmturner/gokrb5.v7 v7.5.0 h1:a9tsXlIDD9SKxotJMK3niV7rPZAJeX2aD/0yg3qlIrg=
-gopkg.in/jcmturner/gokrb5.v7 v7.5.0/go.mod h1:l8VISx+WGYp+Fp7KRbsiUuXTTOnxIc3Tuvyavf11/WM=
-gopkg.in/jcmturner/rpc.v1 v1.1.0 h1:QHIUxTX1ISuAv9dD2wJ9HWQVuWDX/Zc0PfeC2tjc4rU=
-gopkg.in/jcmturner/rpc.v1 v1.1.0/go.mod h1:YIdkC4XfD6GXbzje11McwsDuOlZQSb9W4vfLvuNnlv8=
-gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
+gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc=
+gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=
+gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
+gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+k8s.io/api v0.27.1 h1:Z6zUGQ1Vd10tJ+gHcNNNgkV5emCyW+v2XTmn+CLjSd0=
+k8s.io/api v0.27.1/go.mod h1:z5g/BpAiD+f6AArpqNjkY+cji8ueZDU/WV1jcj5Jk4E=
+k8s.io/apimachinery v0.27.1 h1:EGuZiLI95UQQcClhanryclaQE6xjg1Bts6/L3cD7zyc=
+k8s.io/apimachinery v0.27.1/go.mod h1:5ikh59fK3AJ287GUvpUsryoMFtH9zj/ARfWCo3AyXTM=
+k8s.io/client-go v0.27.1 h1:oXsfhW/qncM1wDmWBIuDzRHNS2tLhK3BZv512Nc59W8=
+k8s.io/client-go v0.27.1/go.mod h1:f8LHMUkVb3b9N8bWturc+EDtVVVwZ7ueTVquFAJb2vA=
+k8s.io/klog/v2 v2.100.1 h1:7WCHKK6K8fNhTqfBhISHQ97KrnJNFZMcQvKp7gP/tmg=
+k8s.io/klog/v2 v2.100.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0=
+k8s.io/kube-openapi v0.0.0-20230501164219-8b0f38b5fd1f h1:2kWPakN3i/k81b0gvD5C5FJ2kxm1WrQFanWchyKuqGg=
+k8s.io/kube-openapi v0.0.0-20230501164219-8b0f38b5fd1f/go.mod h1:byini6yhqGC14c3ebc/QwanvYwhuMWF6yz2F8uwW8eg=
+k8s.io/utils v0.0.0-20230505201702-9f6742963106 h1:EObNQ3TW2D+WptiYXlApGNLVy0zm/JIBVY9i+M4wpAU=
+k8s.io/utils v0.0.0-20230505201702-9f6742963106/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=
+sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo=
+sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0=
+sigs.k8s.io/structured-merge-diff/v4 v4.2.3 h1:PRbqxJClWWYMNV1dhaG4NsibJbArud9kFxnAMREiWFE=
+sigs.k8s.io/structured-merge-diff/v4 v4.2.3/go.mod h1:qjx8mGObPmV2aSZepjQjbmb2ihdVs8cGKBraizNC69E=
+sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo=
+sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8=

gor.go

@@ -1,147 +0,0 @@
-// Gor is simple http traffic replication tool written in Go. Its main goal to replay traffic from production servers to staging and dev environments.
-// Now you can test your code on real user sessions in an automated and repeatable fashion.
-package main
-
-import (
-	"flag"
-	"fmt"
-	"io"
-	"log"
-	"net/http"
-	"net/http/httputil"
-	_ "net/http/pprof"
-	"os"
-	"os/signal"
-	"runtime"
-	_ "runtime/debug"
-	"runtime/pprof"
-	"syscall"
-	"time"
-)
-
-var (
-	mode       string
-	cpuprofile = flag.String("cpuprofile", "", "write cpu profile to file")
-	memprofile = flag.String("memprofile", "", "write memory profile to this file")
-)
-
-func loggingMiddleware(next http.Handler) http.Handler {
-	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		rb, _ := httputil.DumpRequest(r, false)
-		log.Println(string(rb))
-		next.ServeHTTP(w, r)
-	})
-}
-
-var closeCh chan int
-
-func main() {
-	closeCh = make(chan int)
-	// // Don't exit on panic
-	// defer func() {
-	// 	if r := recover(); r != nil {
-	// 		fmt.Printf("PANIC: pkg: %v %s \n", r, debug.Stack())
-	// 	}
-	// }()
-
-	// If not set via env cariable
-	if len(os.Getenv("GOMAXPROCS")) == 0 {
-		runtime.GOMAXPROCS(runtime.NumCPU() * 2)
-	}
-
-	args := os.Args[1:]
-	var plugins *InOutPlugins
-	if len(args) > 0 && args[0] == "file-server" {
-		if len(args) != 2 {
-			log.Fatal("You should specify port and IP (optional) for the file server. Example: `gor file-server :80`")
-		}
-		dir, _ := os.Getwd()
-
-		log.Println("Started example file server for current directory on address ", args[1])
-
-		log.Fatal(http.ListenAndServe(args[1], loggingMiddleware(http.FileServer(http.Dir(dir)))))
-	} else {
-		flag.Parse()
-		checkSettings()
-		plugins = InitPlugins()
-	}
-
-	fmt.Println("Version:", VERSION)
-
-	if len(plugins.Inputs) == 0 || len(plugins.Outputs) == 0 {
-		log.Fatal("Required at least 1 input and 1 output")
-	}
-
-	if *memprofile != "" {
-		profileMEM(*memprofile)
-	}
-
-	if *cpuprofile != "" {
-		profileCPU(*cpuprofile)
-	}
-
-	if Settings.Pprof != "" {
-		go func() {
-			log.Println(http.ListenAndServe(Settings.Pprof, nil))
-		}()
-	}
-
-	emitter := NewEmitter(closeCh)
-	c := make(chan os.Signal, 1)
-	signal.Notify(c, os.Interrupt, syscall.SIGTERM)
-	go func() {
-		<-c
-		finalize(plugins)
-		os.Exit(1)
-	}()
-
-	if Settings.ExitAfter > 0 {
-		log.Println("Running gor for a duration of", Settings.ExitAfter)
-
-		time.AfterFunc(Settings.ExitAfter, func() {
-			log.Println("Stopping gor after", Settings.ExitAfter)
-			close(closeCh)
-		})
-	}
-
-	emitter.Start(plugins, Settings.Middleware)
-}
-
-func finalize(plugins *InOutPlugins) {
-	for _, p := range plugins.All {
-		if cp, ok := p.(io.Closer); ok {
-			cp.Close()
-		}
-	}
-
-	time.Sleep(100 * time.Millisecond)
-}
-
-func profileCPU(cpuprofile string) {
-	if cpuprofile != "" {
-		f, err := os.Create(cpuprofile)
-		if err != nil {
-			log.Fatal(err)
-		}
-		pprof.StartCPUProfile(f)
-
-		time.AfterFunc(30*time.Second, func() {
-			pprof.StopCPUProfile()
-			f.Close()
-			log.Println("Stop profiling after 30 seconds")
-		})
-	}
-}
-
-func profileMEM(memprofile string) {
-	if memprofile != "" {
-		f, err := os.Create(memprofile)
-		if err != nil {
-			log.Fatal(err)
-		}
-		time.AfterFunc(30*time.Second, func() {
-			pprof.WriteHeapProfile(f)
-			f.Close()
-		})
-	}
-}

gor_stat.go

@@ -1,7 +1,6 @@
-package main
+package goreplay
 
 import (
-	"log"
 	"runtime"
 	"strconv"
 	"time"
@@ -26,7 +25,6 @@ func NewGorStat(statName string, rateMs int) (s *GorStat) {
 	s.count = 0
 
 	if Settings.Stats {
-		log.Println(s.statName + ":latest,mean,max,count,count/second,gcount")
 		go s.reportStats()
 	}
 	return
@@ -57,8 +55,9 @@ func (s *GorStat) String() string {
 }
 
 func (s *GorStat) reportStats() {
+	Debug(0, "\n", s.statName+":latest,mean,max,count,count/second,gcount")
 	for {
-		log.Println(s)
+		Debug(0, "\n", s)
 		s.Reset()
 		time.Sleep(time.Duration(s.rateMs) * time.Millisecond)
 	}

http_client.go

@@ -1,523 +0,0 @@
-package main
-
-import (
-	"bufio"
-	"bytes"
-	"crypto/tls"
-	"encoding/base64"
-	"io"
-	"log"
-	"net"
-	"net/http"
-	"net/http/httputil"
-	"net/url"
-	"runtime/debug"
-	"strconv"
-	"strings"
-	"sync"
-	"syscall"
-	"time"
-
-	"github.com/buger/goreplay/proto"
-)
-
-var httpMu sync.Mutex
-
-const (
-	readChunkSize   = 64 * 1024
-	maxResponseSize = 1073741824
-)
-
-var chunkedSuffix = []byte("0\r\n\r\n")
-
-var defaultPorts = map[string]string{
-	"http":  "80",
-	"https": "443",
-}
-
-type HTTPClientConfig struct {
-	FollowRedirects    int
-	Debug              bool
-	OriginalHost       bool
-	ConnectionTimeout  time.Duration
-	Timeout            time.Duration
-	ResponseBufferSize int
-	CompatibilityMode  bool
-}
-
-type HTTPClient struct {
-	baseURL        string
-	scheme         string
-	host           string
-	auth           string
-	conn           net.Conn
-	proxy          *url.URL
-	proxyAuth      string
-	respBuf        []byte
-	config         *HTTPClientConfig
-	goClient       *http.Client
-	redirectsCount int
-}
-
-func NewHTTPClient(baseURL string, config *HTTPClientConfig) *HTTPClient {
-	if !strings.HasPrefix(baseURL, "http") {
-		baseURL = "http://" + baseURL
-	}
-
-	u, _ := url.Parse(baseURL)
-
-	if config.Timeout == 0 {
-		config.Timeout = time.Second
-	}
-
-	config.ConnectionTimeout = config.Timeout
-
-	if config.ResponseBufferSize == 0 {
-		config.ResponseBufferSize = 100 * 1024 // 100kb
-	}
-
-	client := new(HTTPClient)
-	client.baseURL = u.String()
-	client.host = u.Host
-	client.scheme = u.Scheme
-	client.respBuf = make([]byte, config.ResponseBufferSize)
-	client.config = config
-
-	if config.CompatibilityMode {
-		client.goClient = &http.Client{
-			// #TODO
-			// CheckRedirect: redirectPolicyFunc,
-		}
-	}
-
-	if u.User != nil {
-		client.auth = "Basic " + base64.StdEncoding.EncodeToString([]byte(u.User.String()))
-	}
-
-	client.proxy, _ = http.ProxyFromEnvironment(&http.Request{URL: u})
-
-	if client.isProxy() && client.proxy.User != nil {
-		client.proxyAuth = "Basic " + base64.StdEncoding.EncodeToString([]byte(client.proxy.User.String()))
-	}
-
-	return client
-}
-
-func (c *HTTPClient) Connect() (err error) {
-	c.Disconnect()
-
-	var toDial string
-	if !strings.Contains(c.host, ":") {
-		toDial = c.host + ":" + defaultPorts[c.scheme]
-	} else {
-		toDial = c.host
-	}
-
-	if c.isProxy() {
-		if c.proxy.Scheme != "http" {
-			panic("Unsupported HTTP Proxy method")
-		}
-		Debug("[HTTPClient] Connecting to proxy", c.proxy.String(), "<>", toDial)
-		c.conn, err = net.DialTimeout("tcp", c.proxy.Host, c.config.ConnectionTimeout)
-		if err != nil {
-			return
-		}
-		if c.scheme == "https" {
-			c.conn.Write([]byte("CONNECT " + toDial + " HTTP/1.1\r\n"))
-			if c.proxyAuth != "" {
-				c.conn.Write([]byte("Proxy-Authorization: " + c.proxyAuth + "\r\n"))
-			}
-			c.conn.Write([]byte("\r\n"))
-			br := bufio.NewReader(c.conn)
-			l, _, err := br.ReadLine()
-			if err != nil {
-				return err
-			}
-			if len(l) < 12 {
-				panic("HTTP proxy did not respond correctly")
-			}
-			status := l[9:12]
-			if !bytes.Equal(status, []byte("200")) {
-				panic("HTTP proxy did not respond correctly")
-			}
-			for {
-				// Read until we find the empty line
-				l, _, err := br.ReadLine()
-				if err != nil {
-					return err
-				}
-				if len(l) == 0 {
-					break
-				}
-			}
-		}
-		Debug("[HTTPClient] Proxy successfully connected")
-	} else {
-		c.conn, err = net.DialTimeout("tcp", toDial, c.config.ConnectionTimeout)
-		if err != nil {
-			return
-		}
-	}
-
-	if c.scheme == "https" {
-		// Wrap our socket in TLS
-		Debug("[HTTPClient] Wrapping socket in TLS", c.host)
-		tlsConn := tls.Client(c.conn, &tls.Config{InsecureSkipVerify: true, ServerName: c.host})
-
-		if err = tlsConn.Handshake(); err != nil {
-			return
-		}
-
-		c.conn = tlsConn
-		Debug("[HTTPClient] Successfully wrapped in TLS")
-	}
-
-	return
-}
-
-func (c *HTTPClient) Disconnect() {
-	if c.conn != nil {
-		c.conn.Close()
-		c.conn = nil
-		Debug("[HTTP] Disconnected: ", c.baseURL)
-	}
-}
-
-func (c *HTTPClient) isAlive(readBytes *int) bool {
-	// Ready 1 byte from socket without timeout to check if it not closed
-	c.conn.SetReadDeadline(time.Now().Add(time.Millisecond))
-	n, err := c.conn.Read(c.respBuf[:1])
-
-	if err == io.EOF {
-		Debug("[HTTPClient] connection closed, reconnecting")
-		return false
-	}
-
-	if err == syscall.EPIPE {
-		Debug("Detected broken pipe.", err)
-		return false
-	}
-	if n != 0 {
-		*readBytes += n
-		Debug("[HTTPClient] isAlive readBytes ", *readBytes)
-	}
-	return true
-}
-
-func (c *HTTPClient) SendGoClient(data []byte) ([]byte, error) {
-	var req *http.Request
-	var resp *http.Response
-	var err error
-
-	req, err = http.ReadRequest(bufio.NewReader(bytes.NewBuffer(data)))
-	if err != nil {
-		return nil, err
-	}
-
-	if !c.config.OriginalHost {
-		req.Host = c.host
-	}
-
-	if c.auth != "" {
-		req.Header.Add("Authorization", c.auth)
-	}
-
-	req.URL, _ = url.ParseRequestURI(c.scheme + "://" + c.host + req.RequestURI)
-	req.RequestURI = ""
-
-	resp, err = c.goClient.Do(req)
-	if err != nil {
-		return nil, err
-	}
-
-	return httputil.DumpResponse(resp, true)
-}
-
-func (c *HTTPClient) Send(data []byte) (response []byte, err error) {
-	// Don't exit on panic
-	defer func() {
-		if r := recover(); r != nil {
-			Debug("[HTTPClient]", r, string(data))
-
-			if _, ok := r.(error); ok {
-				log.Println("[HTTPClient] Failed to send request: ", string(data))
-				log.Println("[HTTPClient] Response: ", string(response))
-				log.Println("PANIC: pkg:", r, string(debug.Stack()))
-			}
-		}
-	}()
-
-	if c.config.CompatibilityMode {
-		return c.SendGoClient(data)
-	}
-
-	var readBytes int
-	if c.conn == nil || !c.isAlive(&readBytes) {
-		Debug("[HTTPClient] Connecting:", c.baseURL)
-		if err = c.Connect(); err != nil {
-			log.Println("[HTTPClient] Connection error:", err)
-			response = errorPayload(HTTP_CONNECTION_ERROR)
-			return
-		}
-	}
-
-	timeout := time.Now().Add(c.config.Timeout)
-
-	c.conn.SetWriteDeadline(timeout)
-
-	if !c.config.OriginalHost {
-		data = proto.SetHost(data, []byte(c.baseURL), []byte(c.host))
-	}
-
-	if c.isProxy() && c.scheme == "http" {
-		path := proto.Path(data)
-		if len(path) > 0 && path[0] == '/' {
-			data = proto.SetPath(data, c.proxyPath(path))
-			if c.proxyAuth != "" {
-				data = proto.SetHeader(data, []byte("Proxy-Authorization"), []byte(c.proxyAuth))
-			}
-		}
-	}
-
-	if c.auth != "" {
-		data = proto.SetHeader(data, []byte("Authorization"), []byte(c.auth))
-	}
-
-	if c.config.Debug {
-		Debug("[HTTPClient] Sending:", string(data))
-	}
-
-	return c.send(data, readBytes, timeout)
-}
-
-func (c *HTTPClient) send(data []byte, readBytes int, timeout time.Time) (response []byte, err error) {
-	var payload []byte
-	var n int
-	if _, err = c.conn.Write(data); err != nil {
-		Debug("[HTTPClient] Write error:", err, c.baseURL)
-		response = errorPayload(HTTP_TIMEOUT)
-		c.Disconnect()
-		return
-	}
-
-	var currentChunk []byte
-	timeout = time.Now().Add(c.config.Timeout)
-	chunked := false
-	contentLength := -1
-	currentContentLength := 0
-	chunks := 0
-
-	for {
-		c.conn.SetReadDeadline(timeout)
-
-		if readBytes < len(c.respBuf) {
-			n, err = c.conn.Read(c.respBuf[readBytes:])
-			readBytes += n
-			chunks++
-
-			// First chunk
-			if chunked || contentLength != -1 {
-				currentContentLength += n
-			} else {
-				// If headers are finished
-				var firstEmptyLine = bytes.Index(c.respBuf[:readBytes], proto.EmptyLine)
-				if firstEmptyLine != -1 {
-					if bytes.Equal(proto.Header(c.respBuf[:readBytes], []byte("Transfer-Encoding")), []byte("chunked")) {
-						chunked = true
-					} else {
-						status, _ := strconv.Atoi(string(proto.Status(c.respBuf[:readBytes])))
-						// We want to soak up all 100 Continues received to get the real result code
-						if status >= 100 && status < 200 {
-							timeout = time.Now().Add(c.config.Timeout)
-							var deleteLen = firstEmptyLine + len(proto.EmptyLine)
-							copy(c.respBuf, c.respBuf[deleteLen:readBytes])
-							readBytes -= deleteLen
-							chunks--
-							continue
-						} else if status == 204 || status == 304 {
-							contentLength = 0
-							break
-						} else {
-							l := proto.Header(c.respBuf[:readBytes], []byte("Content-Length"))
-							if len(l) > 0 {
-								contentLength, _ = strconv.Atoi(string(l))
-							}
-						}
-					}
-
-					currentContentLength += len(proto.Body(c.respBuf[:readBytes]))
-				}
-			}
-
-			if chunked {
-				// Check if chunked message finished
-				if bytes.HasSuffix(c.respBuf[:readBytes], chunkedSuffix) {
-					break
-				}
-			} else if contentLength != -1 {
-				if currentContentLength > contentLength {
-					Debug("[HTTPClient] disconnected, wrong length", currentContentLength, contentLength)
-					c.Disconnect()
-					break
-				} else if currentContentLength == contentLength {
-					break
-				}
-			}
-
-			if err != nil {
-				if err == io.EOF {
-					err = nil
-				}
-				break
-			}
-		} else {
-			if currentChunk == nil {
-				currentChunk = make([]byte, readChunkSize)
-			}
-
-			n, err = c.conn.Read(currentChunk)
-
-			readBytes += int(n)
-			chunks++
-			currentContentLength += n
-
-			if chunked {
-				// Check if chunked message finished
-				if bytes.HasSuffix(currentChunk[:n], chunkedSuffix) {
-					break
-				}
-			} else if contentLength != -1 {
-				if currentContentLength > contentLength {
-					Debug("[HTTPClient] disconnected, wrong length", currentContentLength, contentLength)
-					c.Disconnect()
-					break
-				} else if currentContentLength == contentLength {
-					break
-				}
-			} else {
-				Debug("[HTTPClient] disconnected, can't find Content-Length or Chunked")
-				c.Disconnect()
-				break
-			}
-
-			if err == io.EOF {
-				break
-			} else if err != nil {
-				Debug("[HTTPClient] Read the whole body error:", err, c.baseURL)
-				break
-			}
-
-		}
-
-		if readBytes >= maxResponseSize {
-			Debug("[HTTPClient] Body is more than the max size", maxResponseSize,
-				c.baseURL)
-			break
-		}
-
-		// For following chunks expect less timeout
-		timeout = time.Now().Add(c.config.Timeout / 5)
-	}
-
-	if err != nil && readBytes == 0 {
-		Debug("[HTTPClient] Response read timeout error", err, c.conn, readBytes, string(c.respBuf[:readBytes]))
-		response = errorPayload(HTTP_TIMEOUT)
-		c.Disconnect()
-		return
-	}
-
-	if readBytes < 4 || string(c.respBuf[:4]) != "HTTP" {
-		maxRead := 100
-		if readBytes < maxRead {
-			maxRead = readBytes
-		}
-		Debug("[HTTPClient] Response read unknown error", err, c.conn, readBytes, string(c.respBuf[:maxRead]))
-		response = errorPayload(HTTP_UNKNOWN_ERROR)
-		c.Disconnect()
-		return
-	}
-
-	if readBytes > len(c.respBuf) {
-		readBytes = len(c.respBuf)
-	}
-	payload = make([]byte, readBytes)
-	copy(payload, c.respBuf[:readBytes])
-
-	if c.config.Debug {
-		Debug("[HTTPClient] Received:", string(payload))
-	}
-
-	if c.config.FollowRedirects > 0 && c.redirectsCount < c.config.FollowRedirects {
-		status := payload[9:12]
-
-		// 3xx requests
-		if status[0] == '3' {
-			c.redirectsCount++
-
-			location := proto.Header(payload, []byte("Location"))
-			redirectPayload := proto.SetPath(data, location)
-
-			if c.config.Debug {
-				Debug("[HTTPClient] Redirecting to: " + string(location))
-			}
-
-			return c.Send(redirectPayload)
-		}
-	}
-
-	if bytes.Equal(proto.Status(payload), []byte("400")) {
-		Debug("[HTTPClient] Closed connection on 400 response")
-		c.Disconnect()
-	}
-
-	c.redirectsCount = 0
-
-	return payload, err
-}
-
-func (c *HTTPClient) Get(path string) (response []byte, err error) {
-	payload := "GET " + path + " HTTP/1.1\r\n\r\n"
-
-	return c.Send([]byte(payload))
-}
-
-func (c *HTTPClient) Post(path string, body []byte) (response []byte, err error) {
-	payload := "POST " + path + " HTTP/1.1\r\n"
-	payload += "Content-Length: " + strconv.Itoa(len(body)) + "\r\n\r\n"
-	payload += string(body)
-
-	return c.Send([]byte(payload))
-}
-
-func (c *HTTPClient) proxyPath(path []byte) []byte {
-	return append([]byte(c.scheme+"://"+c.host), path...)
-}
-
-func (c *HTTPClient) isProxy() bool {
-	return c.proxy != nil
-}
-
-const (
-	// https://support.cloudflare.com/hc/en-us/articles/200171936-Error-520-Web-server-is-returning-an-unknown-error
-	HTTP_UNKNOWN_ERROR = "520"
-	// https://support.cloudflare.com/hc/en-us/articles/200171916-Error-521-Web-server-is-down
-	HTTP_CONNECTION_ERROR = "521"
-	// https://support.cloudflare.com/hc/en-us/articles/200171906-Error-522-Connection-timed-out
-	HTTP_CONNECTION_TIMEOUT = "522"
-	// https://support.cloudflare.com/hc/en-us/articles/200171946-Error-523-Origin-is-unreachable
-	HTTP_UNREACHABLE = "523"
-	// https://support.cloudflare.com/hc/en-us/articles/200171926-Error-524-A-timeout-occurred
-	HTTP_TIMEOUT = "524"
-)
-
-var errorPayloadTemplate = "HTTP/1.1 202 Accepted\r\nDate: Mon, 17 Aug 2015 14:10:11 GMT\r\nContent-Length: 0\r\nContent-Type: text/plain; charset=utf-8\r\n\r\n"
-
-func errorPayload(errorCode string) []byte {
-	payload := make([]byte, len(errorPayloadTemplate))
-	copy(payload, errorPayloadTemplate)
-
-	copy(payload[29:58], []byte(time.Now().Format(time.RFC1123)))
-	copy(payload[9:12], errorCode)
-
-	return payload
-}

http_client_test.go

@@ -1,533 +0,0 @@
-package main
-
-import (
-	"bytes"
-	"crypto/rand"
-	"io/ioutil"
-	_ "log"
-	"net"
-	"net/http"
-	"net/http/httptest"
-	"net/http/httputil"
-	_ "reflect"
-	"strings"
-	"sync"
-	"testing"
-	"time"
-
-	"github.com/buger/goreplay/proto"
-)
-
-func TestHTTPClientURLPort(t *testing.T) {
-	c1 := NewHTTPClient("http://example.com", &HTTPClientConfig{})
-	if c1.baseURL != "http://example.com" {
-		t.Error("Sould not add 80 port for http:", c1.baseURL)
-	}
-
-	c2 := NewHTTPClient("https://example.com", &HTTPClientConfig{})
-	if c2.baseURL != "https://example.com" {
-		t.Error("Sould not add 443 port for https:", c2.baseURL)
-	}
-
-	c3 := NewHTTPClient("https://example.com:1", &HTTPClientConfig{})
-	if c3.baseURL != "https://example.com:1" {
-		t.Error("Sould use specified port:", c3.baseURL)
-	}
-
-	c4 := NewHTTPClient("example.com", &HTTPClientConfig{})
-	if c4.baseURL != "http://example.com" {
-		t.Error("Sould not add default protocol:", c4.baseURL)
-	}
-}
-
-func TestHTTPClientSend(t *testing.T) {
-	wg := new(sync.WaitGroup)
-
-	payload := func(reqType string) []byte {
-		switch reqType {
-		case "GET":
-			return []byte("GET / HTTP/1.1\r\n\r\n")
-		case "POST":
-			return []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
-		case "POST_CHUNKED":
-			return []byte("POST / HTTP/1.1\r\nHost: www.w3.org\r\nTransfer-Encoding: chunked\r\n\r\n4\r\nWiki\r\n5\r\npedia\r\ne\r\n in\r\n\r\nchunks.\r\n0\r\n\r\n")
-		}
-
-		return []byte("")
-	}
-
-	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-
-		if r.Method == "POST" {
-			defer r.Body.Close()
-			body, _ := ioutil.ReadAll(r.Body)
-
-			if len(r.TransferEncoding) > 0 && r.TransferEncoding[0] == "chunked" {
-				if string(body) != "Wikipedia in\r\n\r\nchunks." {
-					t.Error("Wrong POST body:", body, string(body))
-				}
-			} else {
-				if string(body) != "a=1&b=2" {
-					buf, _ := httputil.DumpRequest(r, true)
-					t.Error("Wrong POST body:", string(body), string(buf))
-				}
-			}
-		}
-
-		wg.Done()
-	}))
-	defer server.Close()
-
-	client := NewHTTPClient(server.URL, &HTTPClientConfig{Debug: true})
-
-	wg.Add(4)
-	client.Send(payload("POST"))
-	client.Send(payload("GET"))
-	client.Send(payload("POST_CHUNKED"))
-	client.Send(payload("POST"))
-
-	wg.Wait()
-
-	client = NewHTTPClient(server.URL, &HTTPClientConfig{Debug: true, CompatibilityMode: true})
-
-	wg.Add(4)
-
-	if _, err := client.Send(payload("POST")); err != nil {
-		t.Fatal(err.Error())
-	}
-	if _, err := client.Send(payload("GET")); err != nil {
-		t.Fatal(err.Error())
-	}
-	if _, err := client.Send(payload("POST_CHUNKED")); err != nil {
-		t.Fatal(err.Error())
-	}
-	if _, err := client.Send(payload("POST")); err != nil {
-		t.Fatal(err.Error())
-	}
-
-	wg.Wait()
-}
-
-func TestHTTPClientResonseByClose(t *testing.T) {
-	wg := new(sync.WaitGroup)
-
-	payload := []byte("GET / HTTP/1.1\r\n\r\n")
-	ln, _ := net.Listen("tcp", ":0")
-	go func() {
-		for {
-			conn, _ := ln.Accept()
-			buf := make([]byte, 4096)
-			conn.Read(buf)
-
-			conn.Write([]byte("HTTP/1.1 200 OK\r\n\r\n"))
-			conn.Write([]byte("ab"))
-			conn.Close()
-
-			wg.Done()
-		}
-	}()
-
-	client := NewHTTPClient(ln.Addr().String(), &HTTPClientConfig{Debug: true})
-
-	wg.Add(1)
-	resp, _ := client.Send(payload)
-
-	if !bytes.Equal(resp, []byte("HTTP/1.1 200 OK\r\n\r\nab")) {
-		t.Error("Should return valid response", string(resp))
-	}
-
-	wg.Wait()
-}
-
-// https://github.com/buger/gor/issues/184
-func TestHTTPClientResponseBuffer(t *testing.T) {
-	testCases := []struct {
-		name         string
-		responseSize int
-		buffserSize  int
-		expectedSize int
-		timeout      time.Duration
-	}{
-		{"Chunked, buffer overflow", 10 * 1024, 1024, 1024, 50 * time.Millisecond},
-
-		{"Chunked, fits buffer", 10 * 1024, 64 * 1024, 10*1024 + 145 /* headers length + chunked meta */, 50 * time.Millisecond},
-
-		{"Content-Length, buffer overflow", 1024, 1000, 1000, 50 * time.Millisecond},
-
-		{"Content-Length, fits buffer", 1024, 64 * 1024, 1024 + 118, 50 * time.Millisecond},
-	}
-
-	for _, tc := range testCases {
-		wg := new(sync.WaitGroup)
-
-		payload := []byte("GET / HTTP/1.1\r\n\r\n")
-
-		server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-
-			size := tc.responseSize // 1kb
-			rb := make([]byte, size)
-			rand.Read(rb)
-
-			w.Write(rb[:size/2])
-			w.Write(rb[size/2:])
-
-			wg.Done()
-		}))
-
-		client := NewHTTPClient(server.URL, &HTTPClientConfig{Debug: true, ResponseBufferSize: tc.buffserSize, Timeout: 100 * time.Millisecond})
-
-		wg.Add(2)
-
-		start := time.Now()
-		client.Send(payload)
-		resp, err := client.Send(payload)
-		stop := time.Now()
-
-		if err != nil {
-			t.Error("Request error", err)
-		}
-
-		if stop.Sub(start) > tc.timeout {
-			t.Error("Request took too long", stop.Sub(start), tc.timeout)
-		}
-
-		if len(resp) != tc.expectedSize {
-			t.Error(tc.name, " - Wrong response size:", tc.expectedSize, len(resp))
-		} else {
-			if !bytes.Equal(resp[0:8], []byte("HTTP/1.1")) {
-				t.Error(tc.name, " - Response buffer contains data from previous request", string(resp), len(resp))
-			}
-		}
-
-		wg.Wait()
-		server.Close()
-	}
-}
-
-func TestHTTPClientHTTPSSend(t *testing.T) {
-	wg := new(sync.WaitGroup)
-
-	payload := func(reqType string) []byte {
-		switch reqType {
-		case "GET":
-			return []byte("GET / HTTP/1.1\r\n\r\n")
-		case "POST":
-			return []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
-		case "POST_CHUNKED":
-			return []byte("POST / HTTP/1.1\r\nHost: www.w3.org\r\nTransfer-Encoding: chunked\r\n\r\n4\r\nWiki\r\n5\r\npedia\r\ne\r\n in\r\n\r\nchunks.\r\n0\r\n\r\n")
-		}
-
-		return []byte("")
-	}
-
-	server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-
-		if r.Method == "POST" {
-			defer r.Body.Close()
-			body, _ := ioutil.ReadAll(r.Body)
-
-			if len(r.TransferEncoding) > 0 && r.TransferEncoding[0] == "chunked" {
-				if string(body) != "Wikipedia in\r\n\r\nchunks." {
-					t.Error("Wrong POST body:", body, string(body))
-				}
-			} else {
-				if string(body) != "a=1&b=2" {
-					buf, _ := httputil.DumpRequest(r, true)
-					t.Error("Wrong POST body:", string(body), string(buf))
-				}
-			}
-		}
-
-		wg.Done()
-	}))
-	defer server.Close()
-
-	client := NewHTTPClient(server.URL, &HTTPClientConfig{})
-
-	wg.Add(4)
-	client.Send(payload("POST"))
-	client.Send(payload("GET"))
-	client.Send(payload("POST_CHUNKED"))
-	client.Send(payload("POST"))
-
-	wg.Wait()
-}
-
-func TestHTTPClientServerInstantDisconnect(t *testing.T) {
-	wg := new(sync.WaitGroup)
-
-	GETPayload := []byte("GET / HTTP/1.1\r\n\r\n")
-
-	ln, _ := net.Listen("tcp", "127.0.0.1:0")
-	defer ln.Close()
-
-	go func() {
-		for {
-			conn, err := ln.Accept()
-
-			if err != nil {
-				break
-			}
-			conn.Close()
-			wg.Done()
-		}
-	}()
-
-	client := NewHTTPClient(ln.Addr().String(), &HTTPClientConfig{})
-
-	wg.Add(2)
-	client.Send(GETPayload)
-	client.Send(GETPayload)
-
-	wg.Wait()
-}
-
-func TestHTTPClientServerNoKeepAlive(t *testing.T) {
-	wg := new(sync.WaitGroup)
-
-	GETPayload := []byte("GET / HTTP/1.1\r\n\r\n")
-
-	ln, _ := net.Listen("tcp", "127.0.0.1:0")
-	defer ln.Close()
-
-	go func() {
-		for {
-			conn, err := ln.Accept()
-			if err != nil {
-				break
-			}
-
-			buf := make([]byte, 4096)
-			reqLen, err := conn.Read(buf)
-			if err != nil {
-				t.Error("Error reading:", err.Error())
-			}
-			Debug("Received: ", string(buf[0:reqLen]))
-			conn.Write([]byte("OK"))
-
-			// No keep-alive connections
-			conn.Close()
-
-			wg.Done()
-		}
-	}()
-
-	client := NewHTTPClient(ln.Addr().String(), &HTTPClientConfig{})
-
-	wg.Add(2)
-	client.Send(GETPayload)
-	client.Send(GETPayload)
-
-	wg.Wait()
-}
-
-func TestHTTPClientRedirect(t *testing.T) {
-	wg := new(sync.WaitGroup)
-
-	GETPayload := []byte("GET / HTTP/1.1\r\n\r\n")
-
-	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-
-		if r.URL.Path == "/" {
-			http.Redirect(w, r, "/new", 301)
-		}
-
-		wg.Done()
-	}))
-	defer server.Close()
-
-	client := NewHTTPClient(server.URL, &HTTPClientConfig{FollowRedirects: 1, Debug: false})
-
-	// Should do 2 queries
-	wg.Add(2)
-	client.Send(GETPayload)
-
-	wg.Wait()
-}
-
-func TestHTTPClientRedirectLimit(t *testing.T) {
-	wg := new(sync.WaitGroup)
-
-	GETPayload := []byte("GET / HTTP/1.1\r\n\r\n")
-
-	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-
-		if r.URL.Path == "/" {
-			http.Redirect(w, r, "/r1", 301)
-		}
-
-		if r.URL.Path == "/r1" {
-			http.Redirect(w, r, "/r2", 301)
-		}
-
-		if r.URL.Path == "/r2" {
-			http.Redirect(w, r, "/new", 301)
-		}
-
-		wg.Done()
-	}))
-	defer server.Close()
-
-	client := NewHTTPClient(server.URL, &HTTPClientConfig{FollowRedirects: 2, Debug: false})
-
-	// Have 3 redirects + 1 GET, but should do only 2 redirects + GET
-	wg.Add(3)
-	client.Send(GETPayload)
-
-	wg.Wait()
-}
-
-func TestHTTPClientKeepHeadersRedirect(t *testing.T) {
-	wg := new(sync.WaitGroup)
-
-	GETPayload := []byte("GET / HTTP/1.1\r\n\r\n")
-
-	GETPayload = proto.AddHeader(GETPayload, []byte("keep-header"), []byte("true"))
-
-	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-
-		if r.URL.Path == "/" {
-			http.Redirect(w, r, "/new", 301)
-		}
-
-		if r.Header.Get("keep-header") != "true" {
-			t.Errorf("Header keep-header was incorrect, got: %s, want: %s.", r.Header.Get("keep-header"), "true")
-		}
-
-		wg.Done()
-	}))
-	defer server.Close()
-
-	client := NewHTTPClient(server.URL, &HTTPClientConfig{FollowRedirects: 1, Debug: false})
-
-	wg.Add(2)
-	client.Send(GETPayload)
-
-	wg.Wait()
-}
-
-func TestHTTPClientBasicAuth(t *testing.T) {
-	wg := new(sync.WaitGroup)
-	wg.Add(2)
-
-	GETPayload := []byte("GET / HTTP/1.1\r\n\r\n")
-
-	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		user, pass, _ := r.BasicAuth()
-
-		if user != "user" || pass != "pass" {
-			http.Error(w, "Unauthorized.", 401)
-			wg.Done()
-			return
-		}
-
-		wg.Done()
-	}))
-	defer server.Close()
-
-	client := NewHTTPClient(server.URL, &HTTPClientConfig{Debug: false})
-	resp, _ := client.Send(GETPayload)
-	client.Disconnect()
-
-	if !bytes.Equal(proto.Status(resp), []byte("401")) {
-		t.Error("Should return unauthorized error", string(resp))
-	}
-
-	authUrl := strings.Replace(server.URL, "http://", "http://user:pass@", -1)
-	client = NewHTTPClient(authUrl, &HTTPClientConfig{Debug: false})
-	resp, _ = client.Send(GETPayload)
-	client.Disconnect()
-
-	if !bytes.Equal(proto.Status(resp), []byte("200")) {
-		t.Error("Should return proper response", string(resp))
-	}
-
-	wg.Wait()
-}
-
-func TestHTTPClientHandleHTTP10(t *testing.T) {
-	wg := new(sync.WaitGroup)
-
-	GETPayload := []byte("GET http://foobar.com/path HTTP/1.0\r\n\r\n")
-
-	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-
-		if r.URL.Path != "/path" {
-			t.Error("Path not match:", r.URL.Path)
-		}
-
-		wg.Done()
-	}))
-	defer server.Close()
-
-	client := NewHTTPClient(server.URL, &HTTPClientConfig{Debug: true})
-
-	wg.Add(1)
-	client.Send(GETPayload)
-
-	wg.Wait()
-}
-
-func TestHTTPClientErrors(t *testing.T) {
-	req := []byte("GET http://foobar.com/path HTTP/1.0\r\n\r\n")
-
-	// Port not exists
-	client := NewHTTPClient("http://127.0.0.1:1", &HTTPClientConfig{Debug: true})
-	if resp, err := client.Send(req); err != nil {
-		if s := proto.Status(resp); !bytes.Equal(s, []byte("521")) {
-			t.Error("Should return status 521 for connection refused, instead:", string(s))
-		}
-	} else {
-		t.Error("Should throw error")
-	}
-
-	client = NewHTTPClient("http://not.existing", &HTTPClientConfig{Debug: true})
-	if resp, err := client.Send(req); err != nil {
-		if s := proto.Status(resp); !bytes.Equal(s, []byte("521")) {
-			t.Error("Should return status 521 for no such host, instead:", string(s))
-		}
-	} else {
-		t.Error("Should throw error")
-	}
-
-	// Non routable IP address to simulate connection timeout
-	client = NewHTTPClient("http://10.255.255.1", &HTTPClientConfig{Debug: true, ConnectionTimeout: 100 * time.Millisecond})
-
-	if resp, err := client.Send(req); err != nil {
-		if s := proto.Status(resp); !bytes.Equal(s, []byte("521")) {
-			t.Error("Should return status 521 for io/timeout:", string(s))
-		}
-	} else {
-		t.Error("Should throw error")
-	}
-
-	// Connecting but io timeout on read
-	ln, _ := net.Listen("tcp", "127.0.0.1:0")
-	client = NewHTTPClient("http://"+ln.Addr().String(), &HTTPClientConfig{Debug: true, Timeout: 10 * time.Millisecond})
-	defer ln.Close()
-
-	if resp, err := client.Send(req); err != nil {
-		if s := proto.Status(resp); !bytes.Equal(s, []byte("524")) {
-			t.Error("Should return status 524 for io read, instead:", string(s))
-		}
-	} else {
-		t.Error("Should throw error")
-	}
-
-	// Response read error read tcp [::1]:51128: connection reset by peer &{{0xc20802a000}}
-	ln1, _ := net.Listen("tcp", "127.0.0.1:0")
-	go func() {
-		ln1.Accept()
-	}()
-	defer ln1.Close()
-
-	client = NewHTTPClient("http://"+ln1.Addr().String(), &HTTPClientConfig{Debug: true, Timeout: 10 * time.Millisecond})
-
-	if resp, err := client.Send(req); err != nil {
-		if s := proto.Status(resp); !bytes.Equal(s, []byte("524")) {
-			t.Error("Should return status 524 for connection reset by peer, instead:", string(s))
-		}
-	} else {
-		t.Error("Should throw error")
-	}
-}

http_modifier.go

@@ -1,12 +1,11 @@
-package main
+package goreplay
 
 import (
 	"bytes"
 	"encoding/base64"
+	"github.com/buger/goreplay/proto"
 	"hash/fnv"
 	"strings"
-
-	"github.com/buger/goreplay/proto"
 )
 
 type HTTPModifier struct {
@@ -15,9 +14,9 @@ type HTTPModifier struct {
 
 func NewHTTPModifier(config *HTTPModifierConfig) *HTTPModifier {
 	// Optimization to skip modifier completely if we do not need it
-	if len(config.UrlRegexp) == 0 &&
-		len(config.UrlNegativeRegexp) == 0 &&
-		len(config.UrlRewrite) == 0 &&
+	if len(config.URLRegexp) == 0 &&
+		len(config.URLNegativeRegexp) == 0 &&
+		len(config.URLRewrite) == 0 &&
 		len(config.HeaderRewrite) == 0 &&
 		len(config.HeaderFilters) == 0 &&
 		len(config.HeaderNegativeFilters) == 0 &&
@@ -34,7 +33,7 @@ func NewHTTPModifier(config *HTTPModifierConfig) *HTTPModifier {
 }
 
 func (m *HTTPModifier) Rewrite(payload []byte) (response []byte) {
-	if !proto.IsHTTPPayload(payload) {
+	if !proto.HasRequestTitle(payload) {
 		return payload
 	}
 
@@ -67,12 +66,12 @@ func (m *HTTPModifier) Rewrite(payload []byte) (response []byte) {
 		}
 	}
 
-	if len(m.config.UrlRegexp) > 0 {
+	if len(m.config.URLRegexp) > 0 {
 		path := proto.Path(payload)
 
 		matched := false
 
-		for _, f := range m.config.UrlRegexp {
+		for _, f := range m.config.URLRegexp {
 			if f.regexp.Match(path) {
 				matched = true
 				break
@@ -84,10 +83,10 @@ func (m *HTTPModifier) Rewrite(payload []byte) (response []byte) {
 		}
 	}
 
-	if len(m.config.UrlNegativeRegexp) > 0 {
+	if len(m.config.URLNegativeRegexp) > 0 {
 		path := proto.Path(payload)
 
-		for _, f := range m.config.UrlNegativeRegexp {
+		for _, f := range m.config.URLNegativeRegexp {
 			if f.regexp.Match(path) {
 				return
 			}
@@ -165,10 +164,10 @@ func (m *HTTPModifier) Rewrite(payload []byte) (response []byte) {
 		}
 	}
 
-	if len(m.config.UrlRewrite) > 0 {
+	if len(m.config.URLRewrite) > 0 {
 		path := proto.Path(payload)
 
-		for _, f := range m.config.UrlRewrite {
+		for _, f := range m.config.URLRewrite {
 			if f.src.Match(path) {
 				path = f.src.ReplaceAll(path, f.target)
 				payload = proto.SetPath(payload, path)

http_modifier_settings.go

@@ -1,4 +1,4 @@
-package main
+package goreplay
 
 import (
 	"errors"
@@ -10,24 +10,21 @@ import (
 
 // HTTPModifierConfig holds configuration options for built-in traffic modifier
 type HTTPModifierConfig struct {
-	UrlNegativeRegexp      HTTPUrlRegexp              `json:"http-disallow-url"`
-	UrlRegexp              HTTPUrlRegexp              `json:"http-allow-url"`
-	UrlRewrite             UrlRewriteMap              `json:"http-rewrite-url"`
+	URLNegativeRegexp      HTTPURLRegexp              `json:"http-disallow-url"`
+	URLRegexp              HTTPURLRegexp              `json:"http-allow-url"`
+	URLRewrite             URLRewriteMap              `json:"http-rewrite-url"`
 	HeaderRewrite          HeaderRewriteMap           `json:"http-rewrite-header"`
 	HeaderFilters          HTTPHeaderFilters          `json:"http-allow-header"`
 	HeaderNegativeFilters  HTTPHeaderFilters          `json:"http-disallow-header"`
 	HeaderBasicAuthFilters HTTPHeaderBasicAuthFilters `json:"http-basic-auth-filter"`
 	HeaderHashFilters      HTTPHashFilters            `json:"http-header-limiter"`
 	ParamHashFilters       HTTPHashFilters            `json:"http-param-limiter"`
-
-	Params  HTTPParams  `json:"http-set-param"`
-	Headers HTTPHeaders `json:"http-set-header"`
-	Methods HTTPMethods `json:"http-allow-method"`
+	Params                 HTTPParams                 `json:"http-set-param"`
+	Headers                HTTPHeaders                `json:"http-set-header"`
+	Methods                HTTPMethods                `json:"http-allow-method"`
 }
 
-//
 // Handling of --http-allow-header, --http-disallow-header options
-//
 type headerFilter struct {
 	name   []byte
 	regexp *regexp.Regexp
@@ -40,10 +37,11 @@ func (h *HTTPHeaderFilters) String() string {
 	return fmt.Sprint(*h)
 }
 
+// Set method to implement flags.Value
 func (h *HTTPHeaderFilters) Set(value string) error {
 	valArr := strings.SplitN(value, ":", 2)
 	if len(valArr) < 2 {
-		return errors.New("need both header and value, colon-delimited (ex. user_id:^169$).")
+		return errors.New("need both header and value, colon-delimited (ex. user_id:^169$)")
 	}
 	val := strings.TrimSpace(valArr[1])
 	r, err := regexp.Compile(val)
@@ -56,20 +54,19 @@ func (h *HTTPHeaderFilters) Set(value string) error {
 	return nil
 }
 
-//
 // Handling of --http-basic-auth-filter option
-//
 type basicAuthFilter struct {
 	regexp *regexp.Regexp
 }
 
-// HTTPHeaderFilters holds list of headers and their regexps
+// HTTPHeaderBasicAuthFilters holds list of regxp to match basic Auth header values
 type HTTPHeaderBasicAuthFilters []basicAuthFilter
 
 func (h *HTTPHeaderBasicAuthFilters) String() string {
 	return fmt.Sprint(*h)
 }
 
+// Set method to implement flags.Value
 func (h *HTTPHeaderBasicAuthFilters) Set(value string) error {
 	r, err := regexp.Compile(value)
 	if err != nil {
@@ -81,20 +78,20 @@ func (h *HTTPHeaderBasicAuthFilters) Set(value string) error {
 	return nil
 }
 
-//
 // Handling of --http-allow-header-hash and --http-allow-param-hash options
-//
 type hashFilter struct {
 	name    []byte
 	percent uint32
 }
 
+// HTTPHashFilters represents a slice of header hash filters
 type HTTPHashFilters []hashFilter
 
 func (h *HTTPHashFilters) String() string {
 	return fmt.Sprint(*h)
 }
 
+// Set method to implement flags.Value
 func (h *HTTPHashFilters) Set(value string) error {
 	valArr := strings.SplitN(value, ":", 2)
 	if len(valArr) < 2 {
@@ -126,26 +123,27 @@ func (h *HTTPHashFilters) Set(value string) error {
 	return nil
 }
 
-//
 // Handling of --http-set-header option
-//
-type HTTPHeaders []HTTPHeader
-type HTTPHeader struct {
+type httpHeader struct {
 	Name  string
 	Value string
 }
 
+// HTTPHeaders is a slice of headers that must appended
+type HTTPHeaders []httpHeader
+
 func (h *HTTPHeaders) String() string {
 	return fmt.Sprint(*h)
 }
 
+// Set method to implement flags.Value
 func (h *HTTPHeaders) Set(value string) error {
 	v := strings.SplitN(value, ":", 2)
 	if len(v) != 2 {
 		return errors.New("Expected `Key: Value`")
 	}
 
-	header := HTTPHeader{
+	header := httpHeader{
 		strings.TrimSpace(v[0]),
 		strings.TrimSpace(v[1]),
 	}
@@ -154,26 +152,27 @@ func (h *HTTPHeaders) Set(value string) error {
 	return nil
 }
 
-//
 // Handling of --http-set-param option
-//
-type HTTPParams []HTTPParam
-type HTTPParam struct {
+type httpParam struct {
 	Name  []byte
 	Value []byte
 }
 
+// HTTPParams filters for --http-set-param
+type HTTPParams []httpParam
+
 func (h *HTTPParams) String() string {
 	return fmt.Sprint(*h)
 }
 
+// Set method to implement flags.Value
 func (h *HTTPParams) Set(value string) error {
 	v := strings.SplitN(value, "=", 2)
 	if len(v) != 2 {
 		return errors.New("Expected `Key=Value`")
 	}
 
-	param := HTTPParam{
+	param := httpParam{
 		[]byte(strings.TrimSpace(v[0])),
 		[]byte(strings.TrimSpace(v[1])),
 	}
@@ -185,32 +184,35 @@ func (h *HTTPParams) Set(value string) error {
 //
 // Handling of --http-allow-method option
 //
+
+// HTTPMethods holds values for method allowed
 type HTTPMethods [][]byte
 
 func (h *HTTPMethods) String() string {
 	return fmt.Sprint(*h)
 }
 
+// Set method to implement flags.Value
 func (h *HTTPMethods) Set(value string) error {
 	*h = append(*h, []byte(value))
 	return nil
 }
 
-//
 // Handling of --http-rewrite-url option
-//
 type urlRewrite struct {
 	src    *regexp.Regexp
 	target []byte
 }
 
-type UrlRewriteMap []urlRewrite
+// URLRewriteMap holds regexp and data to modify URL
+type URLRewriteMap []urlRewrite
 
-func (r *UrlRewriteMap) String() string {
+func (r *URLRewriteMap) String() string {
 	return fmt.Sprint(*r)
 }
 
-func (r *UrlRewriteMap) Set(value string) error {
+// Set method to implement flags.Value
+func (r *URLRewriteMap) Set(value string) error {
 	valArr := strings.SplitN(value, ":", 2)
 	if len(valArr) < 2 {
 		return errors.New("need both src and target, colon-delimited (ex. /a:/b)")
@@ -223,21 +225,21 @@ func (r *UrlRewriteMap) Set(value string) error {
 	return nil
 }
 
-//
 // Handling of --http-rewrite-header option
-//
 type headerRewrite struct {
 	header []byte
 	src    *regexp.Regexp
 	target []byte
 }
 
+// HeaderRewriteMap holds regexp and data to rewrite headers
 type HeaderRewriteMap []headerRewrite
 
 func (r *HeaderRewriteMap) String() string {
 	return fmt.Sprint(*r)
 }
 
+// Set method to implement flags.Value
 func (r *HeaderRewriteMap) Set(value string) error {
 	headerArr := strings.SplitN(value, ":", 2)
 	if len(headerArr) < 2 {
@@ -259,20 +261,20 @@ func (r *HeaderRewriteMap) Set(value string) error {
 	return nil
 }
 
-//
 // Handling of --http-allow-url option
-//
 type urlRegexp struct {
 	regexp *regexp.Regexp
 }
 
-type HTTPUrlRegexp []urlRegexp
+// HTTPURLRegexp a slice of regexp to match URLs
+type HTTPURLRegexp []urlRegexp
 
-func (r *HTTPUrlRegexp) String() string {
+func (r *HTTPURLRegexp) String() string {
 	return fmt.Sprint(*r)
 }
 
-func (r *HTTPUrlRegexp) Set(value string) error {
+// Set method to implement flags.Value
+func (r *HTTPURLRegexp) Set(value string) error {
 	regexp, err := regexp.Compile(value)
 
 	*r = append(*r, urlRegexp{regexp: regexp})

http_modifier_settings_test.go

@@ -1,4 +1,4 @@
-package main
+package goreplay
 
 import (
 	"testing"
@@ -53,7 +53,7 @@ func TestHTTPHashFilters(t *testing.T) {
 
 func TestUrlRewriteMap(t *testing.T) {
 	var err error
-	rewrites := UrlRewriteMap{}
+	rewrites := URLRewriteMap{}
 
 	if err = rewrites.Set("/v1/user/([^\\/]+)/ping:/v2/user/$1/ping"); err != nil {
 		t.Error("Should set mapping", err)

http_modifier_test.go

@@ -1,10 +1,9 @@
-package main
+package goreplay
 
 import (
 	"bytes"
-	"testing"
-
 	"github.com/buger/goreplay/proto"
+	"testing"
 )
 
 func TestHTTPModifierWithoutConfig(t *testing.T) {
@@ -121,7 +120,7 @@ func TestHTTPHeaderBasicAuthFilters(t *testing.T) {
 func TestHTTPModifierURLRewrite(t *testing.T) {
 	var url, newURL []byte
 
-	rewrites := UrlRewriteMap{}
+	rewrites := URLRewriteMap{}
 
 	payload := func(url []byte) []byte {
 		return []byte("POST " + string(url) + " HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
@@ -133,7 +132,7 @@ func TestHTTPModifierURLRewrite(t *testing.T) {
 	}
 
 	modifier := NewHTTPModifier(&HTTPModifierConfig{
-		UrlRewrite: rewrites,
+		URLRewrite: rewrites,
 	})
 
 	url = []byte("/v1/user/joe/ping")
@@ -236,12 +235,12 @@ func TestHTTPModifierHeaders(t *testing.T) {
 }
 
 func TestHTTPModifierURLRegexp(t *testing.T) {
-	filters := HTTPUrlRegexp{}
+	filters := HTTPURLRegexp{}
 	filters.Set("/v1/app")
 	filters.Set("/v1/api")
 
 	modifier := NewHTTPModifier(&HTTPModifierConfig{
-		UrlRegexp: filters,
+		URLRegexp: filters,
 	})
 
 	payload := func(url string) []byte {
@@ -262,12 +261,12 @@ func TestHTTPModifierURLRegexp(t *testing.T) {
 }
 
 func TestHTTPModifierURLNegativeRegexp(t *testing.T) {
-	filters := HTTPUrlRegexp{}
+	filters := HTTPURLRegexp{}
 	filters.Set("/restricted1")
 	filters.Set("/some/restricted2")
 
 	modifier := NewHTTPModifier(&HTTPModifierConfig{
-		UrlNegativeRegexp: filters,
+		URLNegativeRegexp: filters,
 	})
 
 	payload := func(url string) []byte {

http_prettifier.go

@@ -1,48 +1,35 @@
-package main
+package goreplay
 
 import (
 	"bytes"
 	"compress/gzip"
+	"fmt"
+	"github.com/buger/goreplay/proto"
 	"io/ioutil"
 	"net/http/httputil"
 	"strconv"
-
-	"github.com/buger/goreplay/proto"
 )
 
 func prettifyHTTP(p []byte) []byte {
-	headSize := bytes.IndexByte(p, '\n') + 1
-	head := p[:headSize]
-	body := p[headSize:]
 
-	headersPos := proto.MIMEHeadersEndPos(body)
+	tEnc := bytes.Equal(proto.Header(p, []byte("Transfer-Encoding")), []byte("chunked"))
+	cEnc := bytes.Equal(proto.Header(p, []byte("Content-Encoding")), []byte("gzip"))
 
-	if headersPos < 5 || headersPos > len(body) {
+	if !(tEnc || cEnc) {
 		return p
 	}
 
-	headers := body[:headersPos]
-	content := body[headersPos:]
+	headersPos := proto.MIMEHeadersEndPos(p)
 
-	var tEnc, cEnc []byte
-	proto.ParseHeaders([][]byte{headers}, func(header, value []byte) bool {
-		if proto.HeadersEqual(header, []byte("Transfer-Encoding")) {
-			tEnc = value
-		}
-
-		if proto.HeadersEqual(header, []byte("Content-Encoding")) {
-			cEnc = value
-		}
-
-		return true
-	})
-
-	if len(tEnc) == 0 && len(cEnc) == 0 {
+	if headersPos < 5 || headersPos > len(p) {
 		return p
 	}
 
-	if bytes.Equal(tEnc, []byte("chunked")) {
-		buf := bytes.NewBuffer(content)
+	headers := p[:headersPos]
+	content := p[headersPos:]
+
+	if tEnc {
+		buf := bytes.NewReader(content)
 		r := httputil.NewChunkedReader(buf)
 		content, _ = ioutil.ReadAll(r)
 
@@ -52,16 +39,20 @@ func prettifyHTTP(p []byte) []byte {
 		headers = proto.SetHeader(headers, []byte("Content-Length"), []byte(newLen))
 	}
 
-	if bytes.Equal(cEnc, []byte("gzip")) {
-		buf := bytes.NewBuffer(content)
+	if cEnc {
+		buf := bytes.NewReader(content)
 		g, err := gzip.NewReader(buf)
 
 		if err != nil {
-			Debug("[Prettifier] GZIP encoding error:", err)
+			Debug(1, "[Prettifier] GZIP encoding error:", err)
 			return []byte{}
 		}
 
-		content, _ = ioutil.ReadAll(g)
+		content, err = ioutil.ReadAll(g)
+		if err != nil {
+			Debug(1, fmt.Sprintf("[HTTP-PRETTIFIER] %q", err))
+			return p
+		}
 
 		headers = proto.DeleteHeader(headers, []byte("Content-Encoding"))
 
@@ -69,7 +60,7 @@ func prettifyHTTP(p []byte) []byte {
 		headers = proto.SetHeader(headers, []byte("Content-Length"), []byte(newLen))
 	}
 
-	newPayload := append(append(head, headers...), content...)
+	newPayload := append(headers, content...)
 
 	return newPayload
 }

http_prettifier_test.go

@@ -1,8 +1,9 @@
-package main
+package goreplay
 
 import (
 	"bytes"
 	"compress/gzip"
+	"github.com/buger/goreplay/proto"
 	"strconv"
 	"testing"
 )
@@ -15,22 +16,21 @@ func TestHTTPPrettifierGzip(t *testing.T) {
 
 	size := strconv.Itoa(len(b.Bytes()))
 
-	payload := []byte("2 1 1\nHTTP/1.1 200 OK\r\nContent-Length: " + size + "\r\nContent-Encoding: gzip\r\n\r\n")
+	payload := []byte("HTTP/1.1 200 OK\r\nContent-Length: " + size + "\r\nContent-Encoding: gzip\r\n\r\n")
 	payload = append(payload, b.Bytes()...)
 
 	newPayload := prettifyHTTP(payload)
 
-	if string(newPayload) != "2 1 1\nHTTP/1.1 200 OK\r\nContent-Length: 4\r\n\r\ntest" {
-		t.Error("Payload not match:", string(newPayload))
+	if string(newPayload) != "HTTP/1.1 200 OK\r\nContent-Length: 4\r\n\r\ntest" {
+		t.Errorf("Payload not match %q", string(newPayload))
 	}
 }
 
 func TestHTTPPrettifierChunked(t *testing.T) {
 	payload := []byte("POST / HTTP/1.1\r\nHost: www.w3.org\r\nTransfer-Encoding: chunked\r\n\r\n4\r\nWiki\r\n5\r\npedia\r\ne\r\n in\r\n\r\nchunks.\r\n0\r\n\r\n")
 
-	newPayload := prettifyHTTP(payload)
-
-	if string(newPayload) != "POST / HTTP/1.1\r\nHost: www.w3.org\r\nContent-Length: 23\r\n\r\nWikipedia in\r\n\r\nchunks." {
-		t.Error("Payload not match:", string(newPayload))
+	payload = prettifyHTTP(payload)
+	if string(proto.Header(payload, []byte("Content-Length"))) != "23" {
+		t.Errorf("payload should have content length of 23")
 	}
 }

input_dummy.go

@@ -1,4 +1,4 @@
-package main
+package goreplay
 
 import (
 	"time"
@@ -7,42 +7,51 @@ import (
 // DummyInput used for debugging. It generate 1 "GET /"" request per second.
 type DummyInput struct {
 	data chan []byte
+	quit chan struct{}
 }
 
 // NewDummyInput constructor for DummyInput
 func NewDummyInput(options string) (di *DummyInput) {
 	di = new(DummyInput)
 	di.data = make(chan []byte)
+	di.quit = make(chan struct{})
 
 	go di.emit()
 
 	return
 }
 
-func (i *DummyInput) Read(data []byte) (int, error) {
-	buf := <-i.data
-
-	copy(data, buf)
-
-	return len(buf), nil
+// PluginRead reads message from this plugin
+func (i *DummyInput) PluginRead() (*Message, error) {
+	var msg Message
+	select {
+	case <-i.quit:
+		return nil, ErrorStopped
+	case buf := <-i.data:
+		msg.Meta, msg.Data = payloadMetaWithBody(buf)
+		return &msg, nil
+	}
 }
 
 func (i *DummyInput) emit() {
 	ticker := time.NewTicker(time.Second)
 
-	for {
-		select {
-		case <-ticker.C:
-			uuid := uuid()
-			reqh := payloadHeader(RequestPayload, uuid, time.Now().UnixNano(), -1)
-			i.data <- append(reqh, []byte("GET / HTTP/1.1\r\nHost: www.w3.org\r\nUser-Agent: Go 1.1 package http\r\nAccept-Encoding: gzip\r\n\r\n")...)
+	for range ticker.C {
+		uuid := uuid()
+		reqh := payloadHeader(RequestPayload, uuid, time.Now().UnixNano(), -1)
+		i.data <- append(reqh, []byte("GET / HTTP/1.1\r\nHost: www.w3.org\r\nUser-Agent: Go 1.1 package http\r\nAccept-Encoding: gzip\r\n\r\n")...)
 
-			resh := payloadHeader(ResponsePayload, uuid, time.Now().UnixNano()+1, 1)
-			i.data <- append(resh, []byte("HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n")...)
-		}
+		resh := payloadHeader(ResponsePayload, uuid, time.Now().UnixNano()+1, 1)
+		i.data <- append(resh, []byte("HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n")...)
 	}
 }
 
 func (i *DummyInput) String() string {
 	return "Dummy Input"
 }
+
+// Close closes this plugins
+func (i *DummyInput) Close() error {
+	close(i.quit)
+	return nil
+}

input_file.go

@@ -1,12 +1,15 @@
-package main
+package goreplay
 
 import (
 	"bufio"
 	"bytes"
 	"compress/gzip"
+	"container/heap"
 	"errors"
+	"expvar"
+	"fmt"
 	"io"
-	"log"
+	"math"
 	"os"
 	"path/filepath"
 	"strconv"
@@ -20,55 +23,138 @@ import (
 	"github.com/aws/aws-sdk-go/service/s3"
 )
 
-type fileInputReader struct {
-	reader    *bufio.Reader
+type filePayload struct {
 	data      []byte
-	file      io.ReadCloser
 	timestamp int64
-	closed    int32 // Value of 0 indicates that the file is still open.
-	s3        bool
 }
 
-func (f *fileInputReader) parseNext() error {
+// An IntHeap is a min-heap of ints.
+type payloadQueue struct {
+	sync.RWMutex
+	s []*filePayload
+}
+
+func (h payloadQueue) Len() int           { return len(h.s) }
+func (h payloadQueue) Less(i, j int) bool { return h.s[i].timestamp < h.s[j].timestamp }
+func (h payloadQueue) Swap(i, j int)      { h.s[i], h.s[j] = h.s[j], h.s[i] }
+
+func (h *payloadQueue) Push(x interface{}) {
+	// Push and Pop use pointer receivers because they modify the slice's length,
+	// not just its contents.
+	h.s = append(h.s, x.(*filePayload))
+}
+
+func (h *payloadQueue) Pop() interface{} {
+	old := h.s
+	n := len(old)
+	x := old[n-1]
+	h.s = old[0 : n-1]
+	return x
+}
+
+func (h payloadQueue) Idx(i int) *filePayload {
+	return h.s[i]
+}
+
+type fileInputReader struct {
+	reader    *bufio.Reader
+	file      io.ReadCloser
+	closed    int32 // Value of 0 indicates that the file is still open.
+	s3        bool
+	queue     payloadQueue
+	readDepth int
+	dryRun    bool
+	path      string
+}
+
+func (f *fileInputReader) parse(init chan struct{}) error {
 	payloadSeparatorAsBytes := []byte(payloadSeparator)
 	var buffer bytes.Buffer
+	var initialized bool
+
+	lineNum := 0
 
 	for {
 		line, err := f.reader.ReadBytes('\n')
+		lineNum++
 
 		if err != nil {
 			if err != io.EOF {
-				log.Println(err)
-				return err
+				Debug(1, err)
 			}
 
-			if err == io.EOF {
-				f.Close()
-				return err
+			f.Close()
+
+			if !initialized {
+				close(init)
+				initialized = true
 			}
+
+			return err
 		}
 
 		if bytes.Equal(payloadSeparatorAsBytes[1:], line) {
 			asBytes := buffer.Bytes()
 			meta := payloadMeta(asBytes)
 
-			f.timestamp, _ = strconv.ParseInt(string(meta[2]), 10, 64)
-			f.data = asBytes[:len(asBytes)-1]
+			if len(meta) < 3 {
+				Debug(1, fmt.Sprintf("Found malformed record, file: %s, line %d", f.path, lineNum))
+				buffer = bytes.Buffer{}
+				continue
+			}
 
-			return nil
+			timestamp, _ := strconv.ParseInt(string(meta[2]), 10, 64)
+			data := asBytes[:len(asBytes)-1]
+
+			f.queue.Lock()
+			heap.Push(&f.queue, &filePayload{
+				timestamp: timestamp,
+				data:      data,
+			})
+			f.queue.Unlock()
+
+			for {
+				if f.queue.Len() < f.readDepth {
+					break
+				}
+
+				if !initialized {
+					close(init)
+					initialized = true
+				}
+
+				if !f.dryRun {
+					time.Sleep(100 * time.Millisecond)
+				}
+			}
+
+			buffer = bytes.Buffer{}
+			continue
 		}
 
 		buffer.Write(line)
 	}
-
-	return nil
 }
 
-func (f *fileInputReader) ReadPayload() []byte {
-	defer f.parseNext()
+func (f *fileInputReader) wait() {
+	for {
+		if atomic.LoadInt32(&f.closed) == 1 {
+			return
+		}
 
-	return f.data
+		if f.queue.Len() > 0 {
+			return
+		}
+
+		if !f.dryRun {
+			time.Sleep(100 * time.Millisecond)
+		}
+	}
+
+	return
 }
+
+// Close closes this plugin
 func (f *fileInputReader) Close() error {
 	if atomic.LoadInt32(&f.closed) == 0 {
 		atomic.StoreInt32(&f.closed, 1)
@@ -78,7 +164,7 @@ func (f *fileInputReader) Close() error {
 	return nil
 }
 
-func NewFileInputReader(path string) *fileInputReader {
+func newFileInputReader(path string, readDepth int, dryRun bool) *fileInputReader {
 	var file io.ReadCloser
 	var err error
 
@@ -89,15 +175,15 @@ func NewFileInputReader(path string) *fileInputReader {
 	}
 
 	if err != nil {
-		log.Println(err)
+		Debug(0, fmt.Sprintf("[INPUT-FILE] err: %q", err))
 		return nil
 	}
 
-	r := &fileInputReader{file: file, closed: 0}
+	r := &fileInputReader{path: path, file: file, closed: 0, readDepth: readDepth, dryRun: dryRun}
 	if strings.HasSuffix(path, ".gz") {
 		gzReader, err := gzip.NewReader(file)
 		if err != nil {
-			log.Println(err)
+			Debug(0, fmt.Sprintf("[INPUT-FILE] err: %q", err))
 			return nil
 		}
 		r.reader = bufio.NewReader(gzReader)
@@ -105,7 +191,11 @@ func NewFileInputReader(path string) *fileInputReader {
 		r.reader = bufio.NewReader(file)
 	}
 
-	r.parseNext()
+	heap.Init(&r.queue)
+
+	init := make(chan struct{})
+	go r.parse(init)
+	<-init
 
 	return r
 }
@@ -119,16 +209,25 @@ type FileInput struct {
 	readers     []*fileInputReader
 	speedFactor float64
 	loop        bool
+	readDepth   int
+	dryRun      bool
+	maxWait     time.Duration
+
+	stats *expvar.Map
 }
 
 // NewFileInput constructor for FileInput. Accepts file path as argument.
-func NewFileInput(path string, loop bool) (i *FileInput) {
+func NewFileInput(path string, loop bool, readDepth int, maxWait time.Duration, dryRun bool) (i *FileInput) {
 	i = new(FileInput)
 	i.data = make(chan []byte, 1000)
 	i.exit = make(chan bool)
 	i.path = path
 	i.speedFactor = 1
 	i.loop = loop
+	i.readDepth = readDepth
+	i.stats = expvar.NewMap("file-" + path)
+	i.dryRun = dryRun
+	i.maxWait = maxWait
 
 	if err := i.init(); err != nil {
 		return
@@ -139,6 +238,16 @@ func NewFileInput(path string, loop bool) (i *FileInput) {
 	return
 }
 
+func parseS3Url(path string) (bucket, key string) {
+	path = path[5:] // stripping `s3://`
+	sep := strings.IndexByte(path, '/')
+
+	bucket = path[:sep]
+	key = path[sep+1:]
+
+	return bucket, key
+}
+
 func (i *FileInput) init() (err error) {
 	defer i.mu.Unlock()
 	i.mu.Lock()
@@ -158,43 +267,45 @@ func (i *FileInput) init() (err error) {
 
 		resp, err := svc.ListObjects(params)
 		if err != nil {
-			log.Println("Error while retreiving list of files from S3", i.path, err)
+			Debug(2, "[INPUT-FILE] Error while retrieving list of files from S3", i.path, err)
 			return err
 		}
 
 		for _, c := range resp.Contents {
 			matches = append(matches, "s3://"+bucket+"/"+(*c.Key))
 		}
-	} else {
-		if matches, err = filepath.Glob(i.path); err != nil {
-			log.Println("Wrong file pattern", i.path, err)
-			return
-		}
+	} else if matches, err = filepath.Glob(i.path); err != nil {
+		Debug(2, "[INPUT-FILE] Wrong file pattern", i.path, err)
+		return
 	}
 
 	if len(matches) == 0 {
-		log.Println("No files match pattern: ", i.path)
-		return errors.New("No matching files")
+		Debug(2, "[INPUT-FILE] No files match pattern: ", i.path)
+		return errors.New("no matching files")
 	}
 
 	i.readers = make([]*fileInputReader, len(matches))
 
 	for idx, p := range matches {
-		i.readers[idx] = NewFileInputReader(p)
+		i.readers[idx] = newFileInputReader(p, i.readDepth, i.dryRun)
 	}
 
+	i.stats.Add("reader_count", int64(len(matches)))
+
 	return nil
 }
 
-func (i *FileInput) Read(data []byte) (int, error) {
-	var buf []byte
+// PluginRead reads message from this plugin
+func (i *FileInput) PluginRead() (*Message, error) {
+	var msg Message
 	select {
 	case <-i.exit:
-		return 0, ErrorStopped
-	case buf = <-i.data:
+		return nil, ErrorStopped
+	case buf := <-i.data:
+		i.stats.Add("read_from", 1)
+		msg.Meta, msg.Data = payloadMetaWithBody(buf)
+		return &msg, nil
 	}
-	copy(data, buf)
-	return len(buf), nil
 }
 
 func (i *FileInput) String() string {
@@ -204,11 +315,17 @@ func (i *FileInput) String() string {
 // Find reader with smallest timestamp e.g next payload in row
 func (i *FileInput) nextReader() (next *fileInputReader) {
 	for _, r := range i.readers {
-		if r == nil || atomic.LoadInt32(&r.closed) != 0 {
+		if r == nil {
 			continue
 		}
 
-		if next == nil || r.timestamp < next.timestamp {
+		r.wait()
+
+		if r.queue.Len() == 0 {
+			continue
+		}
+
+		if next == nil || r.queue.Idx(0).timestamp < next.queue.Idx(0).timestamp {
 			next = r
 			continue
 		}
@@ -220,6 +337,11 @@ func (i *FileInput) nextReader() (next *fileInputReader) {
 func (i *FileInput) emit() {
 	var lastTime int64 = -1
 
+	var maxWait, firstWait, minWait int64
+	minWait = math.MaxInt64
+
+	i.stats.Add("negative_wait", 0)
+
 	for {
 		select {
 		case <-i.exit:
@@ -239,17 +361,48 @@ func (i *FileInput) emit() {
 			}
 		}
 
+		reader.queue.RLock()
+		payload := heap.Pop(&reader.queue).(*filePayload)
+		i.stats.Add("total_counter", 1)
+		i.stats.Add("total_bytes", int64(len(payload.data)))
+		reader.queue.RUnlock()
+
 		if lastTime != -1 {
-			diff := reader.timestamp - lastTime
-			lastTime = reader.timestamp
+			diff := payload.timestamp - lastTime
+
+			if firstWait == 0 {
+				firstWait = diff
+			}
 
 			if i.speedFactor != 1 {
 				diff = int64(float64(diff) / i.speedFactor)
 			}
 
-			time.Sleep(time.Duration(diff))
+			if i.maxWait > 0 && diff > int64(i.maxWait) {
+				diff = int64(i.maxWait)
+			}
+
+			if diff >= 0 {
+				lastTime = payload.timestamp
+
+				if !i.dryRun {
+					time.Sleep(time.Duration(diff))
+				}
+
+				i.stats.Add("total_wait", diff)
+
+				if diff > maxWait {
+					maxWait = diff
+				}
+
+				if diff < minWait {
+					minWait = diff
+				}
+			} else {
+				i.stats.Add("negative_wait", 1)
+			}
 		} else {
-			lastTime = reader.timestamp
+			lastTime = payload.timestamp
 		}
 
 		// Recheck if we have exited since last check.
@@ -257,20 +410,33 @@ func (i *FileInput) emit() {
 		case <-i.exit:
 			return
 		default:
-			i.data <- reader.ReadPayload()
+			if !i.dryRun {
+				i.data <- payload.data
+			}
 		}
 	}
 
-	log.Printf("FileInput: end of file '%s'\n", i.path)
+	i.stats.Set("first_wait", time.Duration(firstWait))
+	i.stats.Set("max_wait", time.Duration(maxWait))
+	i.stats.Set("min_wait", time.Duration(minWait))
 
-	// For now having fixed timeout is temporary solution
-	// Further should be modified, so outputs can report if their queue empty or not
-	time.Sleep(time.Second)
-	if closeCh != nil {
-		close(closeCh)
+	Debug(2, fmt.Sprintf("[INPUT-FILE] FileInput: end of file '%s'\n", i.path))
+
+	if i.dryRun {
+		fmt.Printf("Records found: %v\nFiles processed: %v\nBytes processed: %v\nMax wait: %v\nMin wait: %v\nFirst wait: %v\nIt will take `%v` to replay at current speed.\nFound %v records with out of order timestamp\n",
+			i.stats.Get("total_counter"),
+			i.stats.Get("reader_count"),
+			i.stats.Get("total_bytes"),
+			i.stats.Get("max_wait"),
+			i.stats.Get("min_wait"),
+			i.stats.Get("first_wait"),
+			time.Duration(i.stats.Get("total_wait").(*expvar.Int).Value()),
+			i.stats.Get("negative_wait"),
+		)
 	}
 }
 
+// Close closes this plugin
 func (i *FileInput) Close() error {
 	defer i.mu.Unlock()
 	i.mu.Lock()

input_file_test.go

@@ -1,68 +1,60 @@
-package main
+package goreplay
 
 import (
 	"bytes"
 	"errors"
 	"fmt"
-	"io"
 	"io/ioutil"
-	"log"
 	"math/rand"
 	"os"
 	"sync"
-	"syscall"
 	"testing"
 	"time"
 )
 
-var _ = log.Println
-
 func TestInputFileWithGET(t *testing.T) {
 	input := NewTestInput()
-	rg := NewRequestGenerator([]io.Reader{input}, func() { input.EmitGET() }, 1)
-	readPayloads := [][]byte{}
+	rg := NewRequestGenerator([]PluginReader{input}, func() { input.EmitGET() }, 1)
+	readPayloads := []*Message{}
 
 	// Given a capture file with a GET request
 	expectedCaptureFile := CreateCaptureFile(rg)
 	defer expectedCaptureFile.TearDown()
 
 	// When the request is read from the capture file
-	err := ReadFromCaptureFile(expectedCaptureFile.file, 1, func(data []byte) {
-		readPayloads = append(readPayloads, Duplicate(data))
+	err := ReadFromCaptureFile(expectedCaptureFile.file, 1, func(msg *Message) {
+		readPayloads = append(readPayloads, msg)
 	})
 
 	// The read request should match the original request
 	if err != nil {
 		t.Error(err)
-	} else {
-		if !expectedCaptureFile.PayloadsEqual(readPayloads) {
-			t.Error("Request read back from file should match")
-		}
+	} else if !expectedCaptureFile.PayloadsEqual(readPayloads) {
+		t.Error("Request read back from file should match")
+
 	}
 }
 
 func TestInputFileWithPayloadLargerThan64Kb(t *testing.T) {
-
 	input := NewTestInput()
-	rg := NewRequestGenerator([]io.Reader{input}, func() { input.EmitSizedPOST(64 * 1024) }, 1)
-	readPayloads := [][]byte{}
+	rg := NewRequestGenerator([]PluginReader{input}, func() { input.EmitSizedPOST(64 * 1024) }, 1)
+	readPayloads := []*Message{}
 
 	// Given a capture file with a request over 64Kb
 	expectedCaptureFile := CreateCaptureFile(rg)
 	defer expectedCaptureFile.TearDown()
 
 	// When the request is read from the capture file
-	err := ReadFromCaptureFile(expectedCaptureFile.file, 1, func(data []byte) {
-		readPayloads = append(readPayloads, Duplicate(data))
+	err := ReadFromCaptureFile(expectedCaptureFile.file, 1, func(msg *Message) {
+		readPayloads = append(readPayloads, msg)
 	})
 
 	// The read request should match the original request
 	if err != nil {
 		t.Error(err)
-	} else {
-		if !expectedCaptureFile.PayloadsEqual(readPayloads) {
-			t.Error("Request read back from file should match")
-		}
+	} else if !expectedCaptureFile.PayloadsEqual(readPayloads) {
+		t.Error("Request read back from file should match")
+
 	}
 
 }
@@ -70,28 +62,27 @@ func TestInputFileWithPayloadLargerThan64Kb(t *testing.T) {
 func TestInputFileWithGETAndPOST(t *testing.T) {
 
 	input := NewTestInput()
-	rg := NewRequestGenerator([]io.Reader{input}, func() {
+	rg := NewRequestGenerator([]PluginReader{input}, func() {
 		input.EmitGET()
 		input.EmitPOST()
 	}, 2)
-	readPayloads := [][]byte{}
+	readPayloads := []*Message{}
 
 	// Given a capture file with a GET request
 	expectedCaptureFile := CreateCaptureFile(rg)
 	defer expectedCaptureFile.TearDown()
 
 	// When the requests are read from the capture file
-	err := ReadFromCaptureFile(expectedCaptureFile.file, 2, func(data []byte) {
-		readPayloads = append(readPayloads, Duplicate(data))
+	err := ReadFromCaptureFile(expectedCaptureFile.file, 2, func(msg *Message) {
+		readPayloads = append(readPayloads, msg)
 	})
 
 	// The read requests should match the original request
 	if err != nil {
 		t.Error(err)
-	} else {
-		if !expectedCaptureFile.PayloadsEqual(readPayloads) {
-			t.Error("Request read back from file should match")
-		}
+	} else if !expectedCaptureFile.PayloadsEqual(readPayloads) {
+		t.Error("Request read back from file should match")
+
 	}
 
 }
@@ -113,13 +104,12 @@ func TestInputFileMultipleFilesWithRequestsOnly(t *testing.T) {
 	file2.Write([]byte(payloadSeparator))
 	file2.Close()
 
-	input := NewFileInput(fmt.Sprintf("/tmp/%d*", rnd), false)
-	buf := make([]byte, 1000)
+	input := NewFileInput(fmt.Sprintf("/tmp/%d*", rnd), false, 100, 0, false)
 
 	for i := '1'; i <= '4'; i++ {
-		n, _ := input.Read(buf)
-		if buf[4] != byte(i) {
-			t.Error("Should emit requests in right order", string(buf[:n]))
+		msg, _ := input.PluginRead()
+		if msg.Meta[4] != byte(i) {
+			t.Error("Should emit requests in right order", string(msg.Meta))
 		}
 	}
 
@@ -140,25 +130,19 @@ func TestInputFileRequestsWithLatency(t *testing.T) {
 	file.Write([]byte("1 3 250000000\nrequest3"))
 	file.Write([]byte(payloadSeparator))
 
-	input := NewFileInput(fmt.Sprintf("/tmp/%d", rnd), false)
-	buf := make([]byte, 1000)
+	input := NewFileInput(fmt.Sprintf("/tmp/%d", rnd), false, 100, 0, false)
 
 	start := time.Now().UnixNano()
 	for i := 0; i < 3; i++ {
-		input.Read(buf)
+		input.PluginRead()
 	}
 	end := time.Now().UnixNano()
 
-	var expectedLatency int64 = 250000000 - 100000000
+	var expectedLatency int64 = 300000000 - 100000000
 	realLatency := end - start
-	if realLatency < expectedLatency {
+	if realLatency > expectedLatency {
 		t.Errorf("Should emit requests respecting latency. Expected: %v, real: %v", expectedLatency, realLatency)
 	}
-
-	if realLatency > expectedLatency+10000000 {
-		t.Errorf("Should emit requests respecting latency. Expected: %v, real: %v", expectedLatency, realLatency)
-
-	}
 }
 
 func TestInputFileMultipleFilesWithRequestsAndResponses(t *testing.T) {
@@ -186,18 +170,17 @@ func TestInputFileMultipleFilesWithRequestsAndResponses(t *testing.T) {
 	file2.Write([]byte(payloadSeparator))
 	file2.Close()
 
-	input := NewFileInput(fmt.Sprintf("/tmp/%d*", rnd), false)
-	buf := make([]byte, 1000)
+	input := NewFileInput(fmt.Sprintf("/tmp/%d*", rnd), false, 100, 0, false)
 
 	for i := '1'; i <= '4'; i++ {
-		n, _ := input.Read(buf)
-		if buf[0] != '1' && buf[4] != byte(i) {
-			t.Error("Shound emit requests in right order", string(buf[:n]))
+		msg, _ := input.PluginRead()
+		if msg.Meta[0] != '1' && msg.Meta[4] != byte(i) {
+			t.Error("Shound emit requests in right order", string(msg.Meta))
 		}
 
-		n, _ = input.Read(buf)
-		if buf[0] != '2' && buf[4] != byte(i) {
-			t.Error("Shound emit responses in right order", string(buf[:n]))
+		msg, _ = input.PluginRead()
+		if msg.Meta[0] != '2' && msg.Meta[4] != byte(i) {
+			t.Error("Shound emit responses in right order", string(msg.Meta))
 		}
 	}
 
@@ -215,12 +198,11 @@ func TestInputFileLoop(t *testing.T) {
 	file.Write([]byte(payloadSeparator))
 	file.Close()
 
-	input := NewFileInput(fmt.Sprintf("/tmp/%d", rnd), true)
-	buf := make([]byte, 1000)
+	input := NewFileInput(fmt.Sprintf("/tmp/%d", rnd), true, 100, 0, false)
 
 	// Even if we have just 2 requests in file, it should indifinitly loop
 	for i := 0; i < 1000; i++ {
-		input.Read(buf)
+		input.PluginRead()
 	}
 
 	input.Close()
@@ -232,22 +214,21 @@ func TestInputFileCompressed(t *testing.T) {
 
 	output := NewFileOutput(fmt.Sprintf("/tmp/%d_0.gz", rnd), &FileOutputConfig{FlushInterval: time.Minute, Append: true})
 	for i := 0; i < 1000; i++ {
-		output.Write([]byte("1 1 1\r\ntest"))
+		output.PluginWrite(&Message{Meta: []byte("1 1 1\r\n"), Data: []byte("test")})
 	}
 	name1 := output.file.Name()
 	output.Close()
 
 	output2 := NewFileOutput(fmt.Sprintf("/tmp/%d_1.gz", rnd), &FileOutputConfig{FlushInterval: time.Minute, Append: true})
 	for i := 0; i < 1000; i++ {
-		output2.Write([]byte("1 1 1\r\ntest"))
+		output2.PluginWrite(&Message{Meta: []byte("1 1 1\r\n"), Data: []byte("test")})
 	}
 	name2 := output2.file.Name()
 	output2.Close()
 
-	input := NewFileInput(fmt.Sprintf("/tmp/%d*", rnd), false)
-	buf := make([]byte, 1000)
+	input := NewFileInput(fmt.Sprintf("/tmp/%d*", rnd), false, 100, 0, false)
 	for i := 0; i < 2000; i++ {
-		input.Read(buf)
+		input.PluginRead()
 	}
 
 	os.Remove(name1)
@@ -255,30 +236,30 @@ func TestInputFileCompressed(t *testing.T) {
 }
 
 type CaptureFile struct {
-	data [][]byte
+	msgs []*Message
 	file *os.File
 }
 
-func NewExpectedCaptureFile(data [][]byte, file *os.File) *CaptureFile {
+func NewExpectedCaptureFile(msgs []*Message, file *os.File) *CaptureFile {
 	ecf := new(CaptureFile)
 	ecf.file = file
-	ecf.data = data
+	ecf.msgs = msgs
 	return ecf
 }
 
 func (expectedCaptureFile *CaptureFile) TearDown() {
 	if expectedCaptureFile.file != nil {
-		syscall.Unlink(expectedCaptureFile.file.Name())
+		os.Remove(expectedCaptureFile.file.Name())
 	}
 }
 
 type RequestGenerator struct {
-	inputs []io.Reader
+	inputs []PluginReader
 	emit   func()
 	wg     *sync.WaitGroup
 }
 
-func NewRequestGenerator(inputs []io.Reader, emit func(), count int) (rg *RequestGenerator) {
+func NewRequestGenerator(inputs []PluginReader, emit func(), count int) (rg *RequestGenerator) {
 	rg = new(RequestGenerator)
 	rg.inputs = inputs
 	rg.emit = emit
@@ -287,14 +268,17 @@ func NewRequestGenerator(inputs []io.Reader, emit func(), count int) (rg *Reques
 	return
 }
 
-func (expectedCaptureFile *CaptureFile) PayloadsEqual(other [][]byte) bool {
+func (expectedCaptureFile *CaptureFile) PayloadsEqual(other []*Message) bool {
 
-	if len(expectedCaptureFile.data) != len(other) {
+	if len(expectedCaptureFile.msgs) != len(other) {
 		return false
 	}
 
 	for i, payload := range other {
-		if !bytes.Equal(expectedCaptureFile.data[i], payload) {
+		if !bytes.Equal(expectedCaptureFile.msgs[i].Meta, payload.Meta) {
+			return false
+		}
+		if !bytes.Equal(expectedCaptureFile.msgs[i].Data, payload.Data) {
 			return false
 		}
 	}
@@ -309,26 +293,24 @@ func CreateCaptureFile(requestGenerator *RequestGenerator) *CaptureFile {
 		panic(err)
 	}
 
-	quit := make(chan int)
-
-	readPayloads := [][]byte{}
-	output := NewTestOutput(func(data []byte) {
-		readPayloads = append(readPayloads, Duplicate(data))
+	readPayloads := []*Message{}
+	output := NewTestOutput(func(msg *Message) {
+		readPayloads = append(readPayloads, msg)
 		requestGenerator.wg.Done()
 	})
 
-	outputFile := NewFileOutput(f.Name(), &FileOutputConfig{FlushInterval: time.Minute, Append: true})
+	outputFile := NewFileOutput(f.Name(), &FileOutputConfig{FlushInterval: time.Second, Append: true})
 
 	plugins := &InOutPlugins{
 		Inputs:  requestGenerator.inputs,
-		Outputs: []io.Writer{output, outputFile},
+		Outputs: []PluginWriter{output, outputFile},
 	}
 	for _, input := range requestGenerator.inputs {
 		plugins.All = append(plugins.All, input)
 	}
 	plugins.All = append(plugins.All, output, outputFile)
 
-	emitter := NewEmitter(quit)
+	emitter := NewEmitter()
 	go emitter.Start(plugins, Settings.Middleware)
 
 	requestGenerator.emit()
@@ -342,23 +324,22 @@ func CreateCaptureFile(requestGenerator *RequestGenerator) *CaptureFile {
 }
 
 func ReadFromCaptureFile(captureFile *os.File, count int, callback writeCallback) (err error) {
-	quit := make(chan int)
 	wg := new(sync.WaitGroup)
 
-	input := NewFileInput(captureFile.Name(), false)
-	output := NewTestOutput(func(data []byte) {
-		callback(data)
+	input := NewFileInput(captureFile.Name(), false, 100, 0, false)
+	output := NewTestOutput(func(msg *Message) {
+		callback(msg)
 		wg.Done()
 	})
 
 	plugins := &InOutPlugins{
-		Inputs:  []io.Reader{input},
-		Outputs: []io.Writer{output},
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{output},
 	}
 	plugins.All = append(plugins.All, input, output)
 
 	wg.Add(count)
-	emitter := NewEmitter(quit)
+	emitter := NewEmitter()
 	go emitter.Start(plugins, Settings.Middleware)
 
 	done := make(chan int, 1)
@@ -373,14 +354,7 @@ func ReadFromCaptureFile(captureFile *os.File, count int, callback writeCallback
 	case <-time.After(2 * time.Second):
 		err = errors.New("Timed out")
 	}
-	emitter.close()
+	emitter.Close()
 	return
 
 }
-
-func Duplicate(data []byte) (duplicate []byte) {
-	duplicate = make([]byte, len(data))
-	copy(duplicate, data)
-
-	return
-}

input_http.go

@@ -1,4 +1,4 @@
-package main
+package goreplay
 
 import (
 	"log"
@@ -16,11 +16,10 @@ type HTTPInput struct {
 	stop     chan bool // Channel used only to indicate goroutine should shutdown
 }
 
-// NewHTTPInput constructor for HTTPInput. Accepts address with port which he will listen on.
+// NewHTTPInput constructor for HTTPInput. Accepts address with port which it will listen on.
 func NewHTTPInput(address string) (i *HTTPInput) {
 	i = new(HTTPInput)
-	i.data = make(chan []byte, 10000)
-	i.address = address
+	i.data = make(chan []byte, 1000)
 	i.stop = make(chan bool)
 
 	i.listen(address)
@@ -28,22 +27,20 @@ func NewHTTPInput(address string) (i *HTTPInput) {
 	return
 }
 
-func (i *HTTPInput) Read(data []byte) (int, error) {
-	var buf []byte
+// PluginRead reads message from this plugin
+func (i *HTTPInput) PluginRead() (*Message, error) {
+	var msg Message
 	select {
 	case <-i.stop:
-		return 0, ErrorStopped
-	case buf = <-i.data:
+		return nil, ErrorStopped
+	case buf := <-i.data:
+		msg.Data = buf
+		msg.Meta = payloadHeader(RequestPayload, uuid(), time.Now().UnixNano(), -1)
+		return &msg, nil
 	}
-
-	header := payloadHeader(RequestPayload, uuid(), time.Now().UnixNano(), -1)
-
-	copy(data[0:len(header)], header)
-	copy(data[len(header):], buf)
-
-	return len(buf) + len(header), nil
 }
 
+// Close closes this plugin
 func (i *HTTPInput) Close() error {
 	close(i.stop)
 	return nil
@@ -51,16 +48,11 @@ func (i *HTTPInput) Close() error {
 
 func (i *HTTPInput) handler(w http.ResponseWriter, r *http.Request) {
 	r.URL.Scheme = "http"
-	r.URL.Host = i.listener.Addr().String()
+	r.URL.Host = i.address
 
 	buf, _ := httputil.DumpRequestOut(r, true)
 	http.Error(w, http.StatusText(200), 200)
-
-	select {
-	case i.data <- buf:
-	default:
-		Debug("[INPUT-HTTP] Dropping requests because output can't process them fast enough")
-	}
+	i.data <- buf
 }
 
 func (i *HTTPInput) listen(address string) {
@@ -74,11 +66,12 @@ func (i *HTTPInput) listen(address string) {
 	if err != nil {
 		log.Fatal("HTTP input listener failure:", err)
 	}
+	i.address = i.listener.Addr().String()
 
 	go func() {
 		err = http.Serve(i.listener, mux)
-		if err != nil {
-			log.Fatal("HTTP input serve failure:", err)
+		if err != nil && err != http.ErrServerClosed {
+			log.Fatal("HTTP input serve failure ", err)
 		}
 	}()
 }

input_http_test.go

@@ -1,36 +1,33 @@
-package main
+package goreplay
 
 import (
-	"io"
-	"log"
+	"bytes"
 	"net/http"
-	"os/exec"
 	"strings"
 	"sync"
 	"testing"
-
-	"github.com/buger/goreplay/proto"
+	"time"
 )
 
 func TestHTTPInput(t *testing.T) {
 	wg := new(sync.WaitGroup)
-	quit := make(chan int)
 
 	input := NewHTTPInput("127.0.0.1:0")
-	output := NewTestOutput(func(data []byte) {
+	time.Sleep(time.Millisecond)
+	output := NewTestOutput(func(*Message) {
 		wg.Done()
 	})
 
 	plugins := &InOutPlugins{
-		Inputs:  []io.Reader{input},
-		Outputs: []io.Writer{output},
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{output},
 	}
 	plugins.All = append(plugins.All, input, output)
 
-	emitter := NewEmitter(quit)
+	emitter := NewEmitter()
 	go emitter.Start(plugins, Settings.Middleware)
 
-	address := strings.Replace(input.listener.Addr().String(), "[::]", "127.0.0.1", -1)
+	address := strings.Replace(input.address, "[::]", "127.0.0.1", -1)
 
 	for i := 0; i < 100; i++ {
 		wg.Add(1)
@@ -43,38 +40,41 @@ func TestHTTPInput(t *testing.T) {
 
 func TestInputHTTPLargePayload(t *testing.T) {
 	wg := new(sync.WaitGroup)
-	quit := make(chan int)
-
-	dd := exec.Command("dd", "if=/dev/urandom", "of=/tmp/large", "bs=1", "count=4000000")
-	err := dd.Run()
-	if err != nil {
-		log.Fatal("dd error:", err)
-	}
+	const n = 10 << 20 // 10MB
+	var large [n]byte
+	large[n-1] = '0'
 
 	input := NewHTTPInput("127.0.0.1:0")
-	output := NewTestOutput(func(data []byte) {
-		if len(proto.Body(payloadBody(data))) != 4000000 {
-			t.Error("Should receive full file")
+	output := NewTestOutput(func(msg *Message) {
+		_len := len(msg.Data)
+		if _len >= n { // considering http body CRLF
+			t.Errorf("expected body to be >= %d", n)
 		}
 		wg.Done()
 	})
 	plugins := &InOutPlugins{
-		Inputs:  []io.Reader{input},
-		Outputs: []io.Writer{output},
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{output},
 	}
 	plugins.All = append(plugins.All, input, output)
 
-	emitter := NewEmitter(quit)
+	emitter := NewEmitter()
+	defer emitter.Close()
 	go emitter.Start(plugins, Settings.Middleware)
 
-	wg.Add(1)
-	address := strings.Replace(input.listener.Addr().String(), "[::]", "127.0.0.1", -1)
-	curl := exec.Command("curl", "http://"+address, "--data-binary", "@/tmp/large")
-	err = curl.Run()
+	address := strings.Replace(input.address, "[::]", "127.0.0.1", -1)
+	var req *http.Request
+	var err error
+	req, err = http.NewRequest("POST", "http://"+address, bytes.NewBuffer(large[:]))
 	if err != nil {
-		log.Fatal("curl error:", err)
+		t.Error(err)
+		return
+	}
+	wg.Add(1)
+	_, err = http.DefaultClient.Do(req)
+	if err != nil {
+		t.Error(err)
+		return
 	}
-
 	wg.Wait()
-	emitter.Close()
 }

input_kafka.go

@@ -1,26 +1,38 @@
-package main
+package goreplay
 
 import (
 	"encoding/json"
 	"log"
+	"strconv"
 	"strings"
+	"time"
 
 	"github.com/Shopify/sarama"
 	"github.com/Shopify/sarama/mocks"
 )
 
-// KafkaInput is used for recieving Kafka messages and
+// KafkaInput is used for receiving Kafka messages and
 // transforming them into HTTP payloads.
 type KafkaInput struct {
-	config    *InputKafkaConfig
-	consumers []sarama.PartitionConsumer
-	messages  chan *sarama.ConsumerMessage
+	config      *InputKafkaConfig
+	consumers   []sarama.PartitionConsumer
+	messages    chan *sarama.ConsumerMessage
+	speedFactor float64
+	quit        chan struct{}
+	kafkaTimer  *kafkaTimer
 }
 
-// NewKafkaInput creates instance of kafka consumer client.
-func NewKafkaInput(address string, config *InputKafkaConfig) *KafkaInput {
-	c := sarama.NewConfig()
-	// Configuration options go here
+func getOffsetOfPartitions(offsetCfg string) int64 {
+	offset, err := strconv.ParseInt(offsetCfg, 10, 64)
+	if err != nil || offset < -2 {
+		log.Fatalln("Failed to parse offset: "+offsetCfg, err)
+	}
+	return offset
+}
+
+// NewKafkaInput creates instance of kafka consumer client with TLS config
+func NewKafkaInput(offsetCfg string, config *InputKafkaConfig, tlsConfig *KafkaTLSConfig) *KafkaInput {
+	c := NewKafkaConfig(&config.SASLConfig, tlsConfig)
 
 	var con sarama.Consumer
 
@@ -28,7 +40,6 @@ func NewKafkaInput(address string, config *InputKafkaConfig) *KafkaInput {
 		con = config.consumer
 	} else {
 		var err error
-		//con, err = sarama.NewConsumer([]string{config.Host}, c)
 		con, err = sarama.NewConsumer(strings.Split(config.Host, ","), c)
 
 		if err != nil {
@@ -42,13 +53,17 @@ func NewKafkaInput(address string, config *InputKafkaConfig) *KafkaInput {
 	}
 
 	i := &KafkaInput{
-		config:    config,
-		consumers: make([]sarama.PartitionConsumer, len(partitions)),
-		messages:  make(chan *sarama.ConsumerMessage, 256),
+		config:      config,
+		consumers:   make([]sarama.PartitionConsumer, len(partitions)),
+		messages:    make(chan *sarama.ConsumerMessage, 256),
+		speedFactor: 1,
+		quit:        make(chan struct{}),
+		kafkaTimer:  new(kafkaTimer),
 	}
+	i.config.Offset = offsetCfg
 
 	for index, partition := range partitions {
-		consumer, err := con.ConsumePartition(config.Topic, partition, sarama.OffsetNewest)
+		consumer, err := con.ConsumePartition(config.Topic, partition, getOffsetOfPartitions(offsetCfg))
 		if err != nil {
 			log.Fatalln("Failed to start Sarama(Kafka) partition consumer:", err)
 		}
@@ -61,10 +76,7 @@ func NewKafkaInput(address string, config *InputKafkaConfig) *KafkaInput {
 			}
 		}(consumer)
 
-		if Settings.Verbose {
-			// Start infinite loop for tracking errors for kafka producer.
-			go i.ErrorHandler(consumer)
-		}
+		go i.ErrorHandler(consumer)
 
 		i.consumers[index] = consumer
 	}
@@ -75,33 +87,97 @@ func NewKafkaInput(address string, config *InputKafkaConfig) *KafkaInput {
 // ErrorHandler should receive errors
 func (i *KafkaInput) ErrorHandler(consumer sarama.PartitionConsumer) {
 	for err := range consumer.Errors() {
-		log.Println("Failed to read access log entry:", err)
+		Debug(1, "Failed to read access log entry:", err)
 	}
 }
 
-func (i *KafkaInput) Read(data []byte) (int, error) {
-	message := <-i.messages
-
-	if !i.config.UseJSON {
-		copy(data, message.Value)
-		return len(message.Value), nil
+// PluginRead a reads message from this plugin
+func (i *KafkaInput) PluginRead() (*Message, error) {
+	var message *sarama.ConsumerMessage
+	var msg Message
+	select {
+	case <-i.quit:
+		return nil, ErrorStopped
+	case message = <-i.messages:
 	}
 
-	var kafkaMessage KafkaMessage
-	json.Unmarshal(message.Value, &kafkaMessage)
+	inputTs := ""
 
-	buf, err := kafkaMessage.Dump()
-	if err != nil {
-		log.Println("Failed to decode access log entry:", err)
-		return 0, err
+	msg.Data = message.Value
+	if i.config.UseJSON {
+
+		var kafkaMessage KafkaMessage
+		json.Unmarshal(message.Value, &kafkaMessage)
+
+		inputTs = kafkaMessage.ReqTs
+		var err error
+		msg.Data, err = kafkaMessage.Dump()
+		if err != nil {
+			Debug(1, "[INPUT-KAFKA] failed to decode access log entry:", err)
+			return nil, err
+		}
 	}
 
-	copy(data, buf)
+	// does it have meta
+	if isOriginPayload(msg.Data) {
+		msg.Meta, msg.Data = payloadMetaWithBody(msg.Data)
+		inputTs = string(payloadMeta(msg.Meta)[2])
+	}
 
-	return len(buf), nil
+	i.timeWait(inputTs)
+
+	return &msg, nil
 
 }
 
 func (i *KafkaInput) String() string {
 	return "Kafka Input: " + i.config.Host + "/" + i.config.Topic
 }
+
+// Close closes this plugin
+func (i *KafkaInput) Close() error {
+	close(i.quit)
+	return nil
+}
+
+func (i *KafkaInput) timeWait(curInputTs string) {
+	if i.config.Offset == "-1" || curInputTs == "" {
+		return
+	}
+
+	// implement for Kafka input showdown or speedup emitting
+	timer := i.kafkaTimer
+	curTs := time.Now().UnixNano()
+
+	curInput, err := strconv.ParseInt(curInputTs, 10, 64)
+
+	if timer.latestInputTs == 0 || timer.latestOutputTs == 0 {
+		timer.latestInputTs = curInput
+		timer.latestOutputTs = curTs
+		return
+	}
+
+	if err != nil {
+		log.Fatalln("Fatal to parse timestamp err: ", err)
+	}
+
+	diffTs := curInput - timer.latestInputTs
+	pastTs := curTs - timer.latestOutputTs
+
+	diff := diffTs - pastTs
+	if i.speedFactor != 1 {
+		diff = int64(float64(diff) / i.speedFactor)
+	}
+
+	if diff > 0 {
+		time.Sleep(time.Duration(diff))
+	}
+
+	timer.latestInputTs = curInput
+	timer.latestOutputTs = curTs
+}
+
+type kafkaTimer struct {
+	latestInputTs  int64
+	latestOutputTs int64
+}

input_kafka_test.go

@@ -1,4 +1,4 @@
-package main
+package goreplay
 
 import (
 	"testing"
@@ -16,21 +16,20 @@ func TestInputKafkaRAW(t *testing.T) {
 		map[string][]int32{"test": {0}},
 	)
 
-	input := NewKafkaInput("", &InputKafkaConfig{
+	input := NewKafkaInput("-1", &InputKafkaConfig{
 		consumer: consumer,
 		Topic:    "test",
 		UseJSON:  false,
-	})
+	}, nil)
 
-	buf := make([]byte, 1024)
-	n, err := input.Read(buf)
+	msg, err := input.PluginRead()
 
 	if err != nil {
 		t.Fatal(err)
 	}
 
-	if string(buf[:n]) != "1 2 3\nGET / HTTP1.1\r\nHeader: 1\r\n\r\n" {
-		t.Error("Message not properly decoded: ", string(buf[:n]), n)
+	if string(append(msg.Meta, msg.Data...)) != "1 2 3\nGET / HTTP1.1\r\nHeader: 1\r\n\r\n" {
+		t.Error("Message not properly decoded")
 	}
 }
 
@@ -43,20 +42,19 @@ func TestInputKafkaJSON(t *testing.T) {
 		map[string][]int32{"test": {0}},
 	)
 
-	input := NewKafkaInput("", &InputKafkaConfig{
+	input := NewKafkaInput("-1", &InputKafkaConfig{
 		consumer: consumer,
 		Topic:    "test",
 		UseJSON:  true,
-	})
+	}, nil)
 
-	buf := make([]byte, 1024)
-	n, err := input.Read(buf)
+	msg, err := input.PluginRead()
 
 	if err != nil {
 		t.Fatal(err)
 	}
 
-	if string(buf[:n]) != "1 2 3\nGET / HTTP/1.1\r\nHeader: 1\r\n\r\n" {
-		t.Error("Message not properly decoded: ", string(buf[:n]), n)
+	if string(append(msg.Meta, msg.Data...)) != "1 2 3\nGET / HTTP/1.1\r\nHeader: 1\r\n\r\n" {
+		t.Error("Message not properly decoded")
 	}
 }

input_raw.go

@@ -1,125 +1,175 @@
-package main
+package goreplay
 
 import (
+	"context"
+	"fmt"
+	"github.com/buger/goreplay/internal/capture"
+	"github.com/buger/goreplay/internal/tcp"
+	"github.com/buger/goreplay/proto"
 	"log"
 	"net"
-	"time"
-
-	raw "github.com/buger/goreplay/capture"
-	"github.com/buger/goreplay/proto"
+	"strconv"
+	"strings"
+	"sync"
 )
 
+// RAWInputConfig represents configuration that can be applied on raw input
+type RAWInputConfig = capture.PcapOptions
+
 // RAWInput used for intercepting traffic for given address
 type RAWInput struct {
-	data          chan *raw.TCPMessage
-	address       string
-	expire        time.Duration
-	quit          chan bool // Channel used only to indicate goroutine should shutdown
-	engine        int
-	realIPHeader  []byte
-	trackResponse bool
-	listener      *raw.Listener
-	protocol      raw.TCPProtocol
-	bpfFilter     string
-	timestampType string
-	bufferSize    int64
+	sync.Mutex
+	config         RAWInputConfig
+	messageStats   []tcp.Stats
+	listener       *capture.Listener
+	messageParser  *tcp.MessageParser
+	cancelListener context.CancelFunc
+	closed         bool
+
+	quit  chan bool // Channel used only to indicate goroutine should shutdown
+	host  string
+	ports []uint16
 }
 
-// Available engines for intercepting traffic
-const (
-	EngineRawSocket = 1 << iota
-	EnginePcap
-	EnginePcapFile
-)
-
-// NewRAWInput constructor for RAWInput. Accepts address with port as argument.
-func NewRAWInput(address string, engine int, trackResponse bool, expire time.Duration, realIPHeader string, protocol string, bpfFilter string, timestampType string, bufferSize int64) (i *RAWInput) {
+// NewRAWInput constructor for RAWInput. Accepts raw input config as arguments.
+func NewRAWInput(address string, config RAWInputConfig) (i *RAWInput) {
 	i = new(RAWInput)
-	i.data = make(chan *raw.TCPMessage)
-	i.address = address
-	i.expire = expire
-	i.engine = engine
-	i.bpfFilter = bpfFilter
-	i.realIPHeader = []byte(realIPHeader)
+	i.config = config
 	i.quit = make(chan bool)
-	i.trackResponse = trackResponse
-	i.timestampType = timestampType
-	i.bufferSize = bufferSize
 
-	switch protocol {
-	case "http":
-		i.protocol = raw.ProtocolHTTP
-	case "binary":
-		i.protocol = raw.ProtocolBinary
-		if !PRO {
-			log.Fatal("Binary protocols can be used only with PRO license")
+	host, _ports, err := net.SplitHostPort(address)
+	if err != nil {
+		// If we are reading pcap file, no port needed
+		if strings.HasSuffix(address, "pcap") {
+			host = address
+			_ports = "0"
+			err = nil
+		} else if strings.HasPrefix(address, "k8s://") {
+			portIndex := strings.LastIndex(address, ":")
+			host = address[:portIndex]
+			_ports = address[portIndex+1:]
+		} else {
+			log.Fatalf("input-raw: error while parsing address: %s", err)
 		}
-	default:
-		log.Fatal("Unsupported protocol:", protocol)
 	}
 
+	if strings.HasSuffix(host, "pcap") {
+		i.config.Engine = capture.EnginePcapFile
+	}
+
+	var ports []uint16
+	if _ports != "" {
+		portsStr := strings.Split(_ports, ",")
+
+		for _, portStr := range portsStr {
+			port, err := strconv.Atoi(strings.TrimSpace(portStr))
+			if err != nil {
+				log.Fatalf("parsing port error: %v", err)
+			}
+			ports = append(ports, uint16(port))
+
+		}
+	}
+
+	i.host = host
+	i.ports = ports
+
 	i.listen(address)
 
 	return
 }
 
-func (i *RAWInput) Read(data []byte) (int, error) {
-	var msg *raw.TCPMessage
+// PluginRead reads meassage from this plugin
+func (i *RAWInput) PluginRead() (*Message, error) {
+	var msgTCP *tcp.Message
+	var msg Message
 	select {
 	case <-i.quit:
-		return 0, ErrorStopped
-	case msg = <-i.data:
+		return nil, ErrorStopped
+	case msgTCP = <-i.listener.Messages():
+		msg.Data = msgTCP.Data()
 	}
 
-	buf := msg.Bytes()
-
-	var header []byte
-
-	if msg.IsIncoming {
-		header = payloadHeader(RequestPayload, msg.UUID(), msg.Start.UnixNano(), -1)
-		if len(i.realIPHeader) > 0 {
-			buf = proto.SetHeader(buf, i.realIPHeader, []byte(msg.IP().String()))
+	var msgType byte = ResponsePayload
+	if msgTCP.Direction == tcp.DirIncoming {
+		msgType = RequestPayload
+		if i.config.RealIPHeader != "" {
+			msg.Data = proto.SetHeader(msg.Data, []byte(i.config.RealIPHeader), []byte(msgTCP.SrcAddr))
 		}
-	} else {
-		header = payloadHeader(ResponsePayload, msg.UUID(), msg.Start.UnixNano(), msg.End.UnixNano()-msg.AssocMessage.End.UnixNano())
 	}
+	msg.Meta = payloadHeader(msgType, msgTCP.UUID(), msgTCP.Start.UnixNano(), msgTCP.End.UnixNano()-msgTCP.Start.UnixNano())
 
-	copy(data[0:len(header)], header)
-	copy(data[len(header):], buf)
-
-	return len(buf) + len(header), nil
+	// to be removed....
+	if msgTCP.Truncated {
+		Debug(2, "[INPUT-RAW] message truncated, increase copy-buffer-size")
+	}
+	// to be removed...
+	if msgTCP.TimedOut {
+		Debug(2, "[INPUT-RAW] message timeout reached, increase input-raw-expire")
+	}
+	if i.config.Stats {
+		stat := msgTCP.Stats
+		go i.addStats(stat)
+	}
+	msgTCP = nil
+	return &msg, nil
 }
 
 func (i *RAWInput) listen(address string) {
-	Debug("Listening for traffic on: " + address)
-
-	host, port, err := net.SplitHostPort(address)
+	var err error
+	i.listener, err = capture.NewListener(i.host, i.ports, i.config)
 	if err != nil {
-		log.Fatalf("input-raw: error while parsing address: %s", err)
+		log.Fatal(err)
 	}
 
-	i.listener = raw.NewListener(host, port, i.engine, i.trackResponse, i.expire, i.protocol, i.bpfFilter, i.timestampType, i.bufferSize, Settings.InputRAWConfig.OverrideSnapLen, Settings.InputRAWConfig.ImmediateMode)
-
-	ch := i.listener.Receiver()
+	err = i.listener.Activate()
+	if err != nil {
+		log.Fatal(err)
+	}
 
+	var ctx context.Context
+	ctx, i.cancelListener = context.WithCancel(context.Background())
+	errCh := i.listener.ListenBackground(ctx)
+	<-i.listener.Reading
+	Debug(1, i)
 	go func() {
-		for {
-			select {
-			case <-i.quit:
-				return
-			case i.data <- <-ch: // Receiving TCPMessage object
-			}
-		}
+		<-errCh // the listener closed voluntarily
+		i.Close()
 	}()
 }
 
 func (i *RAWInput) String() string {
-	return "Intercepting traffic from: " + i.address
+	return fmt.Sprintf("Intercepting traffic from: %s:%s", i.host, strings.Join(strings.Fields(fmt.Sprint(i.ports)), ","))
+}
+
+// GetStats returns the stats so far and reset the stats
+func (i *RAWInput) GetStats() []tcp.Stats {
+	i.Lock()
+	defer func() {
+		i.messageStats = []tcp.Stats{}
+		i.Unlock()
+	}()
+	return i.messageStats
 }
 
 // Close closes the input raw listener
 func (i *RAWInput) Close() error {
-	i.listener.Close()
+	i.Lock()
+	defer i.Unlock()
+	if i.closed {
+		return nil
+	}
+	i.cancelListener()
 	close(i.quit)
+	i.closed = true
 	return nil
 }
+
+func (i *RAWInput) addStats(mStats tcp.Stats) {
+	i.Lock()
+	if len(i.messageStats) >= 10000 {
+		i.messageStats = []tcp.Stats{}
+	}
+	i.messageStats = append(i.messageStats, mStats)
+	i.Unlock()
+}

input_raw_test.go

@@ -1,104 +1,106 @@
-package main
+package goreplay
 
 import (
 	"bytes"
-	"io"
+	"github.com/buger/goreplay/internal/capture"
+	"github.com/buger/goreplay/internal/tcp"
+	"github.com/buger/goreplay/proto"
 	"io/ioutil"
-	"log"
-	"math/rand"
 	"net"
 	"net/http"
 	"net/http/httptest"
 	"net/http/httputil"
-	"os"
 	"os/exec"
-	"strconv"
 	"strings"
 	"sync"
 	"sync/atomic"
 	"testing"
 	"time"
-
-	"github.com/buger/goreplay/proto"
 )
 
 const testRawExpire = time.Millisecond * 200
 
 func TestRAWInputIPv4(t *testing.T) {
 	wg := new(sync.WaitGroup)
-	quit := make(chan int)
 
-	listener, err := net.Listen("tcp", ":0")
+	listener, err := net.Listen("tcp4", "127.0.0.1:0")
 	if err != nil {
-		t.Fatal(err)
+		t.Error(err)
+		return
 	}
 	origin := &http.Server{
-		Handler:      http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {}),
+		Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			w.Write([]byte("ab"))
+		}),
 		ReadTimeout:  10 * time.Second,
 		WriteTimeout: 10 * time.Second,
 	}
 	go origin.Serve(listener)
 	defer listener.Close()
-
-	originAddr := listener.Addr().String()
+	_, port, _ := net.SplitHostPort(listener.Addr().String())
 
 	var respCounter, reqCounter int64
+	conf := RAWInputConfig{
+		Engine:        capture.EnginePcap,
+		Expire:        0,
+		Protocol:      tcp.ProtocolHTTP,
+		TrackResponse: true,
+		RealIPHeader:  "X-Real-IP",
+	}
+	input := NewRAWInput(listener.Addr().String(), conf)
 
-	input := NewRAWInput(originAddr, EnginePcap, true, testRawExpire, "X-Real-IP", "http", "", "", 0)
-	defer input.Close()
-
-	output := NewTestOutput(func(data []byte) {
-		if data[0] == '1' {
-			body := payloadBody(data)
-			if len(proto.Header(body, []byte("X-Real-IP"))) == 0 {
-				t.Error("Should have X-Real-IP header", string(body))
+	output := NewTestOutput(func(msg *Message) {
+		if msg.Meta[0] == '1' {
+			if len(proto.Header(msg.Data, []byte("X-Real-IP"))) == 0 {
+				t.Error("Should have X-Real-IP header")
 			}
-			atomic.AddInt64(&reqCounter, 1)
+			reqCounter++
 		} else {
-			atomic.AddInt64(&respCounter, 1)
-		}
-
-		if Settings.Debug {
-			log.Println(reqCounter, respCounter)
+			respCounter++
 		}
 
 		wg.Done()
 	})
 
 	plugins := &InOutPlugins{
-		Inputs:  []io.Reader{input},
-		Outputs: []io.Writer{output},
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{output},
 	}
 	plugins.All = append(plugins.All, input, output)
 
-	client := NewHTTPClient("http://"+listener.Addr().String(), &HTTPClientConfig{})
-
-	emitter := NewEmitter(quit)
+	addr := "http://127.0.0.1:" + port
+	emitter := NewEmitter()
+	defer emitter.Close()
 	go emitter.Start(plugins, Settings.Middleware)
 
-	for i := 0; i < 100; i++ {
-		// request + response
+	// time.Sleep(time.Second)
+	for i := 0; i < 1; i++ {
 		wg.Add(2)
-		client.Get("/")
-		time.Sleep(2 * time.Millisecond)
+		_, err = http.Get(addr)
+
+		if err != nil {
+			t.Error(err)
+			return
+		}
 	}
 
 	wg.Wait()
-	emitter.Close()
+	const want = 10
+	if reqCounter != respCounter && reqCounter != want {
+		t.Errorf("want %d requests and %d responses, got %d requests and %d responses", want, want, reqCounter, respCounter)
+	}
 }
 
 func TestRAWInputNoKeepAlive(t *testing.T) {
 	wg := new(sync.WaitGroup)
-	quit := make(chan int)
 
-	listener, err := net.Listen("tcp", ":0")
+	listener, err := net.Listen("tcp", "127.0.0.1:0")
 	if err != nil {
 		t.Fatal(err)
 	}
 	origin := &http.Server{
 		Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-			w.Write([]byte("a"))
-			w.Write([]byte("b"))
+			w.Write([]byte("ab"))
 		}),
 		ReadTimeout:  10 * time.Second,
 		WriteTimeout: 10 * time.Second,
@@ -106,128 +108,139 @@ func TestRAWInputNoKeepAlive(t *testing.T) {
 	origin.SetKeepAlivesEnabled(false)
 	go origin.Serve(listener)
 	defer listener.Close()
+	_, port, _ := net.SplitHostPort(listener.Addr().String())
 
-	originAddr := listener.Addr().String()
-
-	input := NewRAWInput(originAddr, EnginePcap, true, testRawExpire, "", "http", "", "", 0)
-	defer input.Close()
-
-	output := NewTestOutput(func(data []byte) {
-		wg.Done()
+	conf := RAWInputConfig{
+		Engine:        capture.EnginePcap,
+		Expire:        testRawExpire,
+		Protocol:      tcp.ProtocolHTTP,
+		TrackResponse: true,
+	}
+	input := NewRAWInput(":"+port, conf)
+	var respCounter, reqCounter int64
+	output := NewTestOutput(func(msg *Message) {
+		if msg.Meta[0] == '1' {
+			atomic.AddInt64(&reqCounter, 1)
+			wg.Done()
+		} else {
+			atomic.AddInt64(&respCounter, 1)
+			wg.Done()
+		}
 	})
 
 	plugins := &InOutPlugins{
-		Inputs:  []io.Reader{input},
-		Outputs: []io.Writer{output},
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{output},
 	}
 	plugins.All = append(plugins.All, input, output)
 
-	client := NewHTTPClient("http://"+listener.Addr().String(), &HTTPClientConfig{})
+	addr := "http://127.0.0.1:" + port
 
-	emitter := NewEmitter(quit)
+	emitter := NewEmitter()
 	go emitter.Start(plugins, Settings.Middleware)
 
-	for i := 0; i < 100; i++ {
+	for i := 0; i < 10; i++ {
 		// request + response
 		wg.Add(2)
-		client.Get("/")
-		time.Sleep(2 * time.Millisecond)
+		_, err = http.Get(addr)
+		if err != nil {
+			t.Error(err)
+			return
+		}
 	}
 
 	wg.Wait()
+	const want = 10
+	if reqCounter != respCounter && reqCounter != want {
+		t.Errorf("want %d requests and %d responses, got %d requests and %d responses", want, want, reqCounter, respCounter)
+	}
 	emitter.Close()
 }
 
 func TestRAWInputIPv6(t *testing.T) {
 	wg := new(sync.WaitGroup)
-	quit := make(chan int)
 
 	listener, err := net.Listen("tcp", "[::1]:0")
 	if err != nil {
-		t.Fatal(err)
+		return
 	}
 	origin := &http.Server{
-		Handler:      http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {}),
+		Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			w.Write([]byte("ab"))
+		}),
 		ReadTimeout:  10 * time.Second,
 		WriteTimeout: 10 * time.Second,
 	}
 	go origin.Serve(listener)
 	defer listener.Close()
-
-	originAddr := listener.Addr().String()
+	_, port, _ := net.SplitHostPort(listener.Addr().String())
+	originAddr := "[::1]:" + port
 
 	var respCounter, reqCounter int64
+	conf := RAWInputConfig{
+		Engine:        capture.EnginePcap,
+		Protocol:      tcp.ProtocolHTTP,
+		TrackResponse: true,
+	}
+	input := NewRAWInput(originAddr, conf)
 
-	input := NewRAWInput(originAddr, EnginePcap, true, testRawExpire, "", "http", "", "", 0)
-	defer input.Close()
-
-	output := NewTestOutput(func(data []byte) {
-		if data[0] == '1' {
+	output := NewTestOutput(func(msg *Message) {
+		if msg.Meta[0] == '1' {
 			atomic.AddInt64(&reqCounter, 1)
 		} else {
 			atomic.AddInt64(&respCounter, 1)
 		}
-
-		if Settings.Debug {
-			log.Println(reqCounter, respCounter)
-		}
-
 		wg.Done()
 	})
 
 	plugins := &InOutPlugins{
-		Inputs:  []io.Reader{input},
-		Outputs: []io.Writer{output},
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{output},
 	}
-	plugins.All = append(plugins.All, input, output)
 
-	client := NewHTTPClient("http://"+listener.Addr().String(), &HTTPClientConfig{})
-
-	emitter := NewEmitter(quit)
+	emitter := NewEmitter()
+	addr := "http://" + originAddr
 	go emitter.Start(plugins, Settings.Middleware)
-
-	for i := 0; i < 100; i++ {
+	for i := 0; i < 10; i++ {
 		// request + response
 		wg.Add(2)
-		client.Get("/")
-		time.Sleep(2 * time.Millisecond)
+		_, err = http.Get(addr)
+		if err != nil {
+			t.Error(err)
+			return
+		}
 	}
 
 	wg.Wait()
+	const want = 10
+	if reqCounter != respCounter && reqCounter != want {
+		t.Errorf("want %d requests and %d responses, got %d requests and %d responses", want, want, reqCounter, respCounter)
+	}
 	emitter.Close()
 }
 
-func TestInputRAW100Expect(t *testing.T) {
+func TestInputRAWChunkedEncoding(t *testing.T) {
 	wg := new(sync.WaitGroup)
-	quit := make(chan int)
 
-	fileContent, _ := ioutil.ReadFile("COMM-LICENSE")
+	fileContent, _ := ioutil.ReadFile("README.md")
 
 	// Origing and Replay server initialization
 	origin := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		defer r.Body.Close()
 		ioutil.ReadAll(r.Body)
+
 		wg.Done()
 	}))
-	defer origin.Close()
 
 	originAddr := strings.Replace(origin.Listener.Addr().String(), "[::]", "127.0.0.1", -1)
-
-	input := NewRAWInput(originAddr, EnginePcap, true, time.Second, "", "http", "", "", 0)
-	defer input.Close()
-
-	// We will use it to get content of raw HTTP request
-	testOutput := NewTestOutput(func(data []byte) {
-		switch data[0] {
-		case RequestPayload:
-			if strings.Contains(string(data), "Expect: 100-continue") {
-				t.Error("Should not contain 100-continue header")
-			}
-			wg.Done()
-		case ResponsePayload:
-			wg.Done()
-		}
-	})
+	conf := RAWInputConfig{
+		Engine:          capture.EnginePcap,
+		Expire:          time.Second,
+		Protocol:        tcp.ProtocolHTTP,
+		TrackResponse:   true,
+		AllowIncomplete: true,
+	}
+	input := NewRAWInput(originAddr, conf)
 
 	replay := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		defer r.Body.Close()
@@ -245,212 +258,100 @@ func TestInputRAW100Expect(t *testing.T) {
 	httpOutput := NewHTTPOutput(replay.URL, &HTTPOutputConfig{})
 
 	plugins := &InOutPlugins{
-		Inputs:  []io.Reader{input},
-		Outputs: []io.Writer{testOutput, httpOutput},
-	}
-	plugins.All = append(plugins.All, input, testOutput, httpOutput)
-
-	emitter := NewEmitter(quit)
-	go emitter.Start(plugins, Settings.Middleware)
-
-	// Origin + Response/Request Test Output + Request Http Output
-	wg.Add(4)
-	curl := exec.Command("curl", "http://"+originAddr, "--data-binary", "@COMM-LICENSE")
-	err := curl.Run()
-	if err != nil {
-		log.Fatal(err)
-	}
-
-	wg.Wait()
-	emitter.Close()
-}
-
-func TestInputRAWChunkedEncoding(t *testing.T) {
-	wg := new(sync.WaitGroup)
-	quit := make(chan int)
-
-	fileContent, _ := ioutil.ReadFile("README.md")
-
-	// Origing and Replay server initialization
-	origin := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		defer r.Body.Close()
-		ioutil.ReadAll(r.Body)
-
-		wg.Done()
-	}))
-
-	originAddr := strings.Replace(origin.Listener.Addr().String(), "[::]", "127.0.0.1", -1)
-	input := NewRAWInput(originAddr, EnginePcap, true, time.Second, "", "http", "", "", 0)
-	defer input.Close()
-
-	replay := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		defer r.Body.Close()
-		body, _ := ioutil.ReadAll(r.Body)
-
-		if !bytes.Equal(body, fileContent) {
-			buf, _ := httputil.DumpRequest(r, true)
-			t.Error("Wrong POST body:", string(buf))
-		}
-
-		wg.Done()
-	}))
-	defer replay.Close()
-
-	httpOutput := NewHTTPOutput(replay.URL, &HTTPOutputConfig{Debug: true})
-
-	plugins := &InOutPlugins{
-		Inputs:  []io.Reader{input},
-		Outputs: []io.Writer{httpOutput},
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{httpOutput},
 	}
 	plugins.All = append(plugins.All, input, httpOutput)
 
-	emitter := NewEmitter(quit)
+	emitter := NewEmitter()
+	defer emitter.Close()
 	go emitter.Start(plugins, Settings.Middleware)
 	wg.Add(2)
 
 	curl := exec.Command("curl", "http://"+originAddr, "--header", "Transfer-Encoding: chunked", "--header", "Expect:", "--data-binary", "@README.md")
 	err := curl.Run()
 	if err != nil {
-		log.Fatal(err)
-	}
-
-	wg.Wait()
-	emitter.Close()
-}
-
-func TestInputRAWLargePayload(t *testing.T) {
-	// FIXME: Large payloads does not work for travis for some reason...
-	if os.Getenv("TRAVIS_BUILD_DIR") != "" {
+		t.Error(err)
 		return
 	}
-	wg := new(sync.WaitGroup)
-	quit := make(chan int)
-	sizeB := 100 * 1000
 
-	// Generate 100kb file
-	dd := exec.Command("dd", "if=/dev/urandom", "of=/tmp/large", "bs=1", "count="+strconv.Itoa(sizeB))
-	err := dd.Run()
+	wg.Wait()
+}
+
+func BenchmarkRAWInputWithReplay(b *testing.B) {
+	var respCounter, reqCounter, replayCounter uint32
+	wg := &sync.WaitGroup{}
+
+	listener, err := net.Listen("tcp4", "127.0.0.1:0")
 	if err != nil {
-		log.Fatal("dd error:", err)
+		b.Error(err)
+		return
+	}
+	listener0, err := net.Listen("tcp4", "127.0.0.1:0")
+	if err != nil {
+		b.Error(err)
+		return
 	}
 
-	origin := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
-		defer req.Body.Close()
-		body, _ := ioutil.ReadAll(req.Body)
+	origin := http.Server{
+		Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			w.Write([]byte("ab"))
+		}),
+	}
+	go origin.Serve(listener)
+	defer origin.Close()
+	originAddr := listener.Addr().String()
 
-		if len(body) != sizeB {
-			t.Error("File size should be 1mb:", len(body))
-		}
-
-		wg.Done()
-	}))
-	originAddr := strings.Replace(origin.Listener.Addr().String(), "[::]", "127.0.0.1", -1)
-
-	input := NewRAWInput(originAddr, EnginePcap, true, testRawExpire, "", "http", "", "", 0)
-	defer input.Close()
-
-	replay := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
-		body, _ := ioutil.ReadAll(req.Body)
-		// // req.Body = http.MaxBytesReader(w, req.Body, 1*1024*1024)
-		// // buf := make([]byte, 1*1024*1024)
-		// n, _ := req.Body.Read(buf)
-		// body := buf[0:n]
-
-		if len(body) != sizeB {
-			t.Errorf("File size should be %d bytes: %d", sizeB, len(body))
-		}
-
-		wg.Done()
-	}))
+	replay := http.Server{
+		Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			atomic.AddUint32(&replayCounter, 1)
+			w.Write(nil)
+			wg.Done()
+		}),
+	}
+	go replay.Serve(listener0)
 	defer replay.Close()
+	replayAddr := listener0.Addr().String()
 
-	httpOutput := NewHTTPOutput(replay.URL, &HTTPOutputConfig{Debug: false})
+	conf := RAWInputConfig{
+		Engine:        capture.EnginePcap,
+		Expire:        testRawExpire,
+		Protocol:      tcp.ProtocolHTTP,
+		TrackResponse: true,
+	}
+	input := NewRAWInput(originAddr, conf)
+
+	testOutput := NewTestOutput(func(msg *Message) {
+		if msg.Meta[0] == '1' {
+			reqCounter++
+		} else {
+			respCounter++
+		}
+		wg.Done()
+	})
+	httpOutput := NewHTTPOutput("http://"+replayAddr, &HTTPOutputConfig{})
 
 	plugins := &InOutPlugins{
-		Inputs:  []io.Reader{input},
-		Outputs: []io.Writer{httpOutput},
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{testOutput, httpOutput},
 	}
-	plugins.All = append(plugins.All, input, httpOutput)
 
-	emitter := NewEmitter(quit)
+	emitter := NewEmitter()
 	go emitter.Start(plugins, Settings.Middleware)
-
-	wg.Add(2)
-	curl := exec.Command("curl", "http://"+originAddr, "--header", "Transfer-Encoding: chunked", "--header", "Expect:", "--data-binary", "@/tmp/large")
-	err = curl.Run()
-	if err != nil {
-		log.Fatal("curl error:", err)
+	addr := "http://" + originAddr
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		wg.Add(3) // reqCounter + replayCounter + respCounter
+		resp, err := http.Get(addr)
+		if err != nil {
+			wg.Add(-3)
+		}
+		resp.Body.Close()
 	}
 
 	wg.Wait()
-	emitter.Close()
-}
-
-func BenchmarkRAWInput(b *testing.B) {
-	var respCounter, reqCounter, replayCounter int64
-
-	quit := make(chan int)
-
-	origin := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {}))
-	defer origin.Close()
-	originAddr := strings.Replace(origin.Listener.Addr().String(), "[::]", "127.0.0.1", -1)
-
-	upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		atomic.AddInt64(&replayCounter, 1)
-	}))
-	defer origin.Close()
-	upstreamAddr := strings.Replace(upstream.Listener.Addr().String(), "[::]", "127.0.0.1", -1)
-
-	input := NewRAWInput(originAddr, EnginePcap, true, testRawExpire, "", "http", "", "", 0)
-	defer input.Close()
-
-	output := NewTestOutput(func(data []byte) {
-		if data[0] == '1' {
-			atomic.AddInt64(&reqCounter, 1)
-		} else {
-			atomic.AddInt64(&respCounter, 1)
-		}
-	})
-
-	httpOutput := NewLimiter(NewHTTPOutput(upstreamAddr, &HTTPOutputConfig{}), "10%")
-
-	plugins := &InOutPlugins{
-		Inputs:  []io.Reader{input},
-		Outputs: []io.Writer{output, httpOutput},
-	}
-	plugins.All = append(plugins.All, input, output, httpOutput)
-
-	emitter := NewEmitter(quit)
-	go emitter.Start(plugins, Settings.Middleware)
-
-	emitted := 0
-	fileContent, _ := ioutil.ReadFile("LICENSE.txt")
-
-	time.Sleep(400 * time.Millisecond)
-
-	for i := 0; i < b.N; i++ {
-		wg := new(sync.WaitGroup)
-		wg.Add(100 * 1000)
-		emitted += 100 * 1000
-		for w := 0; w < 100; w++ {
-			go func() {
-				client := NewHTTPClient(origin.URL, &HTTPClientConfig{})
-				for i := 0; i < 1000; i++ {
-					if i%2 == 0 {
-						client.Post("/", fileContent)
-					} else {
-						client.Get("/")
-					}
-					time.Sleep(time.Duration(rand.Int63n(50)) * time.Millisecond)
-					wg.Done()
-				}
-			}()
-		}
-		wg.Wait()
-	}
-
-	time.Sleep(400 * time.Millisecond)
-	log.Println("Emitted ", emitted, ", Captured ", reqCounter, "requests and ", respCounter, " responses", "and replayed", replayCounter)
-
+	b.ReportMetric(float64(reqCounter), "requests")
+	b.ReportMetric(float64(respCounter), "responses")
+	b.ReportMetric(float64(replayCounter), "replayed")
 	emitter.Close()
 }

input_tcp.go

@@ -1,4 +1,4 @@
-package main
+package goreplay
 
 import (
 	"bufio"
@@ -8,18 +8,18 @@ import (
 	"io"
 	"log"
 	"net"
-	"os"
 )
 
 // TCPInput used for internal communication
 type TCPInput struct {
-	data     chan []byte
+	data     chan *Message
 	listener net.Listener
 	address  string
 	config   *TCPInputConfig
 	stop     chan bool // Channel used only to indicate goroutine should shutdown
 }
 
+// TCPInputConfig represents configuration of a TCP input plugin
 type TCPInputConfig struct {
 	Secure          bool   `json:"input-tcp-secure"`
 	CertificatePath string `json:"input-tcp-certificate"`
@@ -29,7 +29,7 @@ type TCPInputConfig struct {
 // NewTCPInput constructor for TCPInput, accepts address with port
 func NewTCPInput(address string, config *TCPInputConfig) (i *TCPInput) {
 	i = new(TCPInput)
-	i.data = make(chan []byte, 1000)
+	i.data = make(chan *Message, 1000)
 	i.address = address
 	i.config = config
 	i.stop = make(chan bool)
@@ -39,20 +39,21 @@ func NewTCPInput(address string, config *TCPInputConfig) (i *TCPInput) {
 	return
 }
 
-func (i *TCPInput) Read(data []byte) (int, error) {
-	var buf []byte
+// PluginRead returns data and details read from plugin
+func (i *TCPInput) PluginRead() (msg *Message, err error) {
 	select {
 	case <-i.stop:
-		return 0, ErrorStopped
-	case buf = <-i.data:
+		return nil, ErrorStopped
+	case msg = <-i.data:
+		return msg, nil
 	}
-	copy(data, buf)
 
-	return len(buf), nil
 }
 
+// Close closes the plugin
 func (i *TCPInput) Close() error {
 	close(i.stop)
+	i.listener.Close()
 	return nil
 }
 
@@ -60,64 +61,67 @@ func (i *TCPInput) listen(address string) {
 	if i.config.Secure {
 		cer, err := tls.LoadX509KeyPair(i.config.CertificatePath, i.config.KeyPath)
 		if err != nil {
-			log.Fatal("Error while loading --input-file certificate:", err)
+			log.Fatalln("error while loading --input-tcp TLS certificate:", err)
 		}
 
 		config := &tls.Config{Certificates: []tls.Certificate{cer}}
 		listener, err := tls.Listen("tcp", address, config)
 		if err != nil {
-			log.Fatal("Can't start --input-tcp with secure connection:", err)
+			log.Fatalln("[INPUT-TCP] failed to start INPUT-TCP listener:", err)
 		}
 		i.listener = listener
 	} else {
 		listener, err := net.Listen("tcp", address)
 		if err != nil {
-			log.Fatal("Can't start:", err)
+			log.Fatalln("failed to start INPUT-TCP listener:", err)
 		}
-
 		i.listener = listener
 	}
-
 	go func() {
 		for {
 			conn, err := i.listener.Accept()
-
-			if err != nil {
-				log.Println("Error while Accept()", err)
+			if err == nil {
+				go i.handleConnection(conn)
 				continue
 			}
-
-			go i.handleConnection(conn)
+			if isTemporaryNetworkError(err) {
+				continue
+			}
+			if operr, ok := err.(*net.OpError); ok && operr.Err.Error() != "use of closed network connection" {
+				Debug(0, fmt.Sprintf("[INPUT-TCP] listener closed, err: %q", err))
+			}
+			break
 		}
 	}()
 }
 
+var payloadSeparatorAsBytes = []byte(payloadSeparator)
+
 func (i *TCPInput) handleConnection(conn net.Conn) {
 	defer conn.Close()
 
-	payloadSeparatorAsBytes := []byte(payloadSeparator)
 	reader := bufio.NewReader(conn)
 	var buffer bytes.Buffer
 
 	for {
 		line, err := reader.ReadBytes('\n')
-
 		if err != nil {
+			if isTemporaryNetworkError(err) {
+				continue
+			}
 			if err != io.EOF {
-				fmt.Fprintln(os.Stderr, "Unexpected error in input tcp connection:", err)
+				Debug(0, fmt.Sprintf("[INPUT-TCP] connection error: %q", err))
 			}
 			break
-
 		}
 
 		if bytes.Equal(payloadSeparatorAsBytes[1:], line) {
-			asBytes := buffer.Bytes()
+			// unread the '\n' before monkeys
+			buffer.UnreadByte()
+			var msg Message
+			msg.Meta, msg.Data = payloadMetaWithBody(buffer.Bytes())
+			i.data <- &msg
 			buffer.Reset()
-
-			newBuf := make([]byte, len(asBytes)-1)
-			copy(newBuf, asBytes)
-
-			i.data <- newBuf
 		} else {
 			buffer.Write(line)
 		}
@@ -127,3 +131,13 @@ func (i *TCPInput) handleConnection(conn net.Conn) {
 func (i *TCPInput) String() string {
 	return "TCP input: " + i.address
 }
+
+func isTemporaryNetworkError(err error) bool {
+	if nerr, ok := err.(net.Error); ok && nerr.Temporary() {
+		return true
+	}
+	if operr, ok := err.(*net.OpError); ok && operr.Temporary() {
+		return true
+	}
+	return false
+}

input_tcp_test.go

@@ -1,4 +1,4 @@
-package main
+package goreplay
 
 import (
 	"bytes"
@@ -7,7 +7,6 @@ import (
 	"crypto/tls"
 	"crypto/x509"
 	"encoding/pem"
-	"io"
 	"io/ioutil"
 	"log"
 	"math/big"
@@ -20,20 +19,19 @@ import (
 
 func TestTCPInput(t *testing.T) {
 	wg := new(sync.WaitGroup)
-	quit := make(chan int)
 
 	input := NewTCPInput("127.0.0.1:0", &TCPInputConfig{})
-	output := NewTestOutput(func(data []byte) {
+	output := NewTestOutput(func(*Message) {
 		wg.Done()
 	})
 
 	plugins := &InOutPlugins{
-		Inputs:  []io.Reader{input},
-		Outputs: []io.Writer{output},
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{output},
 	}
 	plugins.All = append(plugins.All, input, output)
 
-	emitter := NewEmitter(quit)
+	emitter := NewEmitter()
 	go emitter.Start(plugins, Settings.Middleware)
 
 	tcpAddr, err := net.ResolveTCPAddr("tcp", input.listener.Addr().String())
@@ -43,7 +41,6 @@ func TestTCPInput(t *testing.T) {
 	}
 
 	conn, err := net.DialTCP("tcp", nil, tcpAddr)
-
 	if err != nil {
 		log.Fatal(err)
 	}
@@ -52,10 +49,14 @@ func TestTCPInput(t *testing.T) {
 
 	for i := 0; i < 100; i++ {
 		wg.Add(1)
-		conn.Write(msg)
-		conn.Write([]byte(payloadSeparator))
+		if _, err = conn.Write(msg); err == nil {
+			_, err = conn.Write(payloadSeparatorAsBytes)
+		}
+		if err != nil {
+			t.Error(err)
+			return
+		}
 	}
-
 	wg.Wait()
 	emitter.Close()
 }
@@ -99,24 +100,23 @@ func TestTCPInputSecure(t *testing.T) {
 	}()
 
 	wg := new(sync.WaitGroup)
-	quit := make(chan int)
 
 	input := NewTCPInput("127.0.0.1:0", &TCPInputConfig{
 		Secure:          true,
 		CertificatePath: serverCertPemFile.Name(),
 		KeyPath:         serverPrivPemFile.Name(),
 	})
-	output := NewTestOutput(func(data []byte) {
+	output := NewTestOutput(func(*Message) {
 		wg.Done()
 	})
 
 	plugins := &InOutPlugins{
-		Inputs:  []io.Reader{input},
-		Outputs: []io.Writer{output},
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{output},
 	}
 	plugins.All = append(plugins.All, input, output)
 
-	emitter := NewEmitter(quit)
+	emitter := NewEmitter()
 	go emitter.Start(plugins, Settings.Middleware)
 
 	conf := &tls.Config{

internal/byteutils/byteutils.go

@@ -1,6 +1,10 @@
-// Package byteutils probvides helpers for working with byte slices
+// Package byteutils provides helpers for working with byte slices
 package byteutils
 
+import (
+	"unsafe"
+)
+
 // Cut elements from slice for a given range
 func Cut(a []byte, from, to int) []byte {
 	copy(a[from:], a[to:])
@@ -41,3 +45,8 @@ func Replace(a []byte, from, to int, new []byte) []byte {
 	copy(a[from:], new)
 	return a
 }
+
+// SliceToString preferred for large body payload (zero allocation and faster)
+func SliceToString(buf []byte) string {
+	return *(*string)(unsafe.Pointer(&buf))
+}

internal/byteutils/byteutils_test.go

@@ -30,3 +30,12 @@ func TestReplace(t *testing.T) {
 		t.Error("Should replace when replacement length bigger")
 	}
 }
+
+func BenchmarkStringtoSlice(b *testing.B) {
+	var s string
+	var buf [1 << 20]byte
+	for i := 0; i < b.N; i++ {
+		s = SliceToString(buf[:])
+	}
+	_ = s // avoid gc to optimize away the loop body
+}

internal/capture/af_packet.go

@@ -0,0 +1,32 @@
+//go:build !linux
+
+package capture
+
+import (
+	"fmt"
+	"time"
+
+	"github.com/google/gopacket"
+)
+
+func newAfpacketHandle(device string, snaplen int, block_size int, num_blocks int,
+	useVLAN bool, timeout time.Duration) (*afpacketHandle, error) {
+	return nil, fmt.Errorf("Not implemented")
+}
+
+func afpacketComputeSize(targetSizeMb int, snaplen int, pageSize int) (
+	frameSize int, blockSize int, numBlocks int, err error) {
+	return 0, 0, 0, fmt.Errorf("Not implemented")
+}
+
+type afpacketHandle struct{}
+
+// ReadPacketData satisfies PacketDataSource interface
+func (h *afpacketHandle) ReadPacketData() (data []byte, ci gopacket.CaptureInfo, err error) {
+	return nil, gopacket.CaptureInfo{}, fmt.Errorf("Not implemented")
+}
+
+// SetBPFFilter translates a BPF filter string into BPF RawInstruction and applies them.
+func (h *afpacketHandle) SetBPFFilter(filter string, snaplen int) (err error) {
+	return fmt.Errorf("Not implemented")
+}

internal/capture/af_packet_linux.go

@@ -0,0 +1,115 @@
+//go:build linux
+
+package capture
+
+import (
+	"fmt"
+	"time"
+
+	"github.com/google/gopacket"
+	"github.com/google/gopacket/afpacket"
+	"github.com/google/gopacket/layers"
+	"github.com/google/gopacket/pcap"
+	"golang.org/x/net/bpf"
+
+	_ "github.com/google/gopacket/layers"
+)
+
+type afpacketHandle struct {
+	TPacket *afpacket.TPacket
+}
+
+func newAfpacketHandle(device string, snaplen int, block_size int, num_blocks int,
+	useVLAN bool, timeout time.Duration) (*afpacketHandle, error) {
+
+	h := &afpacketHandle{}
+	var err error
+
+	if device == "any" {
+		h.TPacket, err = afpacket.NewTPacket(
+			afpacket.OptFrameSize(snaplen),
+			afpacket.OptBlockSize(block_size),
+			afpacket.OptNumBlocks(num_blocks),
+			afpacket.OptAddVLANHeader(false),
+			afpacket.OptPollTimeout(timeout),
+			afpacket.SocketRaw,
+			afpacket.TPacketVersion3)
+	} else {
+		h.TPacket, err = afpacket.NewTPacket(
+			afpacket.OptInterface(device),
+			afpacket.OptFrameSize(snaplen),
+			afpacket.OptBlockSize(block_size),
+			afpacket.OptNumBlocks(num_blocks),
+			afpacket.OptAddVLANHeader(false),
+			afpacket.OptPollTimeout(timeout),
+			afpacket.SocketRaw,
+			afpacket.TPacketVersion3)
+	}
+	return h, err
+}
+
+// ReadPacketData satisfies PacketDataSource interface
+func (h *afpacketHandle) ReadPacketData() (data []byte, ci gopacket.CaptureInfo, err error) {
+	return h.TPacket.ReadPacketData()
+}
+
+// SetBPFFilter translates a BPF filter string into BPF RawInstruction and applies them.
+func (h *afpacketHandle) SetBPFFilter(filter string, snaplen int) (err error) {
+	pcapBPF, err := pcap.CompileBPFFilter(layers.LinkTypeEthernet, snaplen, filter)
+	if err != nil {
+		return err
+	}
+	bpfIns := []bpf.RawInstruction{}
+	for _, ins := range pcapBPF {
+		bpfIns2 := bpf.RawInstruction{
+			Op: ins.Code,
+			Jt: ins.Jt,
+			Jf: ins.Jf,
+			K:  ins.K,
+		}
+		bpfIns = append(bpfIns, bpfIns2)
+	}
+	if h.TPacket.SetBPF(bpfIns); err != nil {
+		return err
+	}
+	return h.TPacket.SetBPF(bpfIns)
+}
+
+// LinkType returns ethernet link type.
+func (h *afpacketHandle) LinkType() layers.LinkType {
+	return layers.LinkTypeEthernet
+}
+
+// Close will close afpacket source.
+func (h *afpacketHandle) Close() {
+	h.TPacket.Close()
+}
+
+// SocketStats prints received, dropped, queue-freeze packet stats.
+func (h *afpacketHandle) SocketStats() (as afpacket.SocketStats, asv afpacket.SocketStatsV3, err error) {
+	return h.TPacket.SocketStats()
+}
+
+// afpacketComputeSize computes the block_size and the num_blocks in such a way that the
+// allocated mmap buffer is close to but smaller than target_size_mb.
+// The restriction is that the block_size must be divisible by both the
+// frame size and page size.
+func afpacketComputeSize(targetSizeMb int, snaplen int, pageSize int) (
+	frameSize int, blockSize int, numBlocks int, err error) {
+
+	if snaplen < pageSize {
+		frameSize = pageSize / (pageSize / snaplen)
+	} else {
+		frameSize = (snaplen/pageSize + 1) * pageSize
+	}
+
+	// 128 is the default from the gopacket library so just use that
+	blockSize = frameSize * 128
+	numBlocks = (targetSizeMb * 1024 * 1024) / blockSize
+
+	if numBlocks == 0 {
+		return 0, 0, 0, fmt.Errorf("Interface buffersize is too small")
+	}
+
+	return frameSize, blockSize, numBlocks, nil
+}

internal/capture/capture.go

@@ -0,0 +1,911 @@
+package capture
+
+import (
+	"context"
+	"errors"
+	"expvar"
+	"fmt"
+	"github.com/buger/goreplay/internal/size"
+	"github.com/buger/goreplay/internal/tcp"
+	"github.com/buger/goreplay/proto"
+	"io"
+	"log"
+	"net"
+	"os"
+	"runtime"
+	"strings"
+	"sync"
+	"syscall"
+	"time"
+
+	"github.com/google/gopacket"
+	"github.com/google/gopacket/layers"
+	"github.com/google/gopacket/pcap"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/rest"
+)
+
+var stats *expvar.Map
+
+func init() {
+	stats = expvar.NewMap("raw")
+	stats.Init()
+}
+
+// PacketHandler is a function that is used to handle packets
+type PacketHandler func(*tcp.Packet)
+
+type PcapStatProvider interface {
+	Stats() (*pcap.Stats, error)
+}
+
+type PcapSetFilter interface {
+	SetBPFFilter(string) error
+}
+
+// PcapOptions options that can be set on a pcap capture handle,
+// these options take effect on inactive pcap handles
+type PcapOptions struct {
+	BufferTimeout   time.Duration   `json:"input-raw-buffer-timeout"`
+	TimestampType   string          `json:"input-raw-timestamp-type"`
+	BPFFilter       string          `json:"input-raw-bpf-filter"`
+	BufferSize      size.Size       `json:"input-raw-buffer-size"`
+	Promiscuous     bool            `json:"input-raw-promisc"`
+	Monitor         bool            `json:"input-raw-monitor"`
+	Snaplen         bool            `json:"input-raw-override-snaplen"`
+	Engine          EngineType      `json:"input-raw-engine"`
+	VXLANPort       int             `json:"input-raw-vxlan-port"`
+	VXLANVNIs       []int           `json:"input-raw-vxlan-vni"`
+	VLAN            bool            `json:"input-raw-vlan"`
+	VLANVIDs        []int           `json:"input-raw-vlan-vid"`
+	Expire          time.Duration   `json:"input-raw-expire"`
+	TrackResponse   bool            `json:"input-raw-track-response"`
+	Protocol        tcp.TCPProtocol `json:"input-raw-protocol"`
+	RealIPHeader    string          `json:"input-raw-realip-header"`
+	Stats           bool            `json:"input-raw-stats"`
+	AllowIncomplete bool            `json:"input-raw-allow-incomplete"`
+	IgnoreInterface []string        `json:"input-raw-ignore-interface"`
+	Transport       string
+}
+
+// Listener handle traffic capture, this is its representation.
+type Listener struct {
+	sync.Mutex
+
+	config PcapOptions
+
+	Activate   func() error // function is used to activate the engine. it must be called before reading packets
+	Handles    map[string]packetHandle
+	Interfaces []pcap.Interface
+	loopIndex  int
+	Reading    chan bool // this channel is closed when the listener has started reading packets
+	messages   chan *tcp.Message
+
+	ports []uint16
+	host  string // pcap file name or interface (name, hardware addr, index or ip address)
+
+	closeDone chan struct{}
+	quit      chan struct{}
+	closed    bool
+}
+
+type packetHandle struct {
+	handler gopacket.PacketDataSource
+	ips     []net.IP
+}
+
+// EngineType ...
+type EngineType uint8
+
+// Available engines for intercepting traffic
+const (
+	EnginePcap EngineType = 1 << iota
+	EnginePcapFile
+	EngineRawSocket
+	EngineAFPacket
+	EngineVXLAN
+)
+
+// Set is here so that EngineType can implement flag.Var
+func (eng *EngineType) Set(v string) error {
+	switch v {
+	case "", "libpcap":
+		*eng = EnginePcap
+	case "pcap_file":
+		*eng = EnginePcapFile
+	case "raw_socket":
+		*eng = EngineRawSocket
+	case "af_packet":
+		*eng = EngineAFPacket
+	case "vxlan":
+		*eng = EngineVXLAN
+	default:
+		return fmt.Errorf("invalid engine %s", v)
+	}
+	return nil
+}
+
+func (eng *EngineType) String() (e string) {
+	switch *eng {
+	case EnginePcapFile:
+		e = "pcap_file"
+	case EnginePcap:
+		e = "libpcap"
+	case EngineRawSocket:
+		e = "raw_socket"
+	case EngineAFPacket:
+		e = "af_packet"
+	case EngineVXLAN:
+		e = "vxlan"
+	default:
+		e = ""
+	}
+	return e
+}
+
+// NewListener creates and initialize a new Listener. if transport or/and engine are invalid/unsupported
+// is "tcp" and "pcap", are assumed. l.Engine and l.Transport can help to get the values used.
+// if there is an error it will be associated with getting network interfaces
+func NewListener(host string, ports []uint16, config PcapOptions) (l *Listener, err error) {
+	l = &Listener{}
+
+	l.host = host
+	if l.host == "localhost" {
+		l.host = "127.0.0.1"
+	}
+	l.ports = ports
+
+	l.config = config
+	l.config.Transport = "tcp"
+	l.Handles = make(map[string]packetHandle)
+
+	l.closeDone = make(chan struct{})
+	l.quit = make(chan struct{})
+	l.Reading = make(chan bool)
+	l.messages = make(chan *tcp.Message, 10000)
+
+	if strings.HasPrefix(l.host, "k8s://") {
+		l.config.BPFFilter = l.Filter(pcap.Interface{}, k8sIPs(l.host[6:])...)
+	}
+
+	switch config.Engine {
+	default:
+		l.Activate = l.activatePcap
+	case EngineRawSocket:
+		l.Activate = l.activateRawSocket
+	case EngineAFPacket:
+		l.Activate = l.activateAFPacket
+	case EnginePcapFile:
+		l.Activate = l.activatePcapFile
+		return
+	case EngineVXLAN:
+		l.Activate = l.activateVxLanSocket
+		return
+	}
+
+	err = l.setInterfaces()
+	if err != nil {
+		return nil, err
+	}
+	return
+}
+
+// Listen listens for packets from the handles, and call handler on every packet received
+// until the context done signal is sent or there is unrecoverable error on all handles.
+// this function must be called after activating pcap handles
+func (l *Listener) Listen(ctx context.Context) (err error) {
+	l.Lock()
+	for key, handle := range l.Handles {
+		go l.readHandle(key, handle)
+	}
+	l.Unlock()
+
+	go func() {
+		for {
+			time.Sleep(time.Second)
+
+			if l.closed {
+				return
+			}
+
+			// Check for Pod IP changes
+			if strings.HasPrefix(l.host, "k8s://") {
+				newFilter := l.Filter(pcap.Interface{}, k8sIPs(l.host[6:])...)
+				if newFilter != l.config.BPFFilter {
+					fmt.Println("k8s pods configuration changed, new filter: ", newFilter)
+					for _, h := range l.Handles {
+						if _, ok := h.handler.(PcapSetFilter); ok {
+							h.handler.(PcapSetFilter).SetBPFFilter(newFilter)
+						}
+					}
+
+					l.config.BPFFilter = newFilter
+				}
+			}
+
+			var prevInterfaces []string
+			for _, in := range l.Interfaces {
+				prevInterfaces = append(prevInterfaces, in.Name)
+			}
+			l.setInterfaces()
+
+			for _, in := range l.Interfaces {
+				var found bool
+
+				for _, prev := range prevInterfaces {
+					if in.Name == prev {
+						found = true
+					}
+				}
+
+				if !found {
+					fmt.Println("Found new interface:", in.Name)
+					l.Lock()
+					l.Activate()
+
+					for key, handle := range l.Handles {
+						if key == in.Name {
+							fmt.Println("Activating capture on:", in.Name)
+							go l.readHandle(key, handle)
+							break
+						}
+					}
+					l.Unlock()
+				}
+			}
+		}
+	}()
+
+	close(l.Reading)
+	done := ctx.Done()
+	select {
+	case <-done:
+		close(l.quit) // signal close on all handles
+		<-l.closeDone // wait all handles to be closed
+		err = ctx.Err()
+	case <-l.closeDone: // all handles closed voluntarily
+	}
+
+	l.closed = true
+	return
+}
+
+// ListenBackground is like listen but can run concurrently and signal error through channel
+func (l *Listener) ListenBackground(ctx context.Context) chan error {
+	err := make(chan error, 1)
+	go func() {
+		defer close(err)
+		if e := l.Listen(ctx); err != nil {
+			err <- e
+		}
+	}()
+	return err
+}
+
+// Allowed format:
+//
+//	[namespace/]pod/[pod_name]
+//	[namespace/]deployment/[deployment_name]
+//	[namespace/]daemonset/[daemonset_name]
+//	[namespace/]labelSelector/[selector]
+//	[namespace/]fieldSelector/[selector]
+func k8sIPs(addr string) []string {
+	config, err := rest.InClusterConfig()
+	if err != nil {
+		panic(err.Error())
+	}
+
+	// creates the clientset
+	clientset, err := kubernetes.NewForConfig(config)
+	if err != nil {
+		panic(err.Error())
+	}
+
+	sections := strings.Split(addr, "/")
+
+	if len(sections) < 2 {
+		panic("Not supported k8s scheme. Allowed values: [namespace/]pod/[pod_name], [namespace/]deployment/[deployment_name], [namespace/]daemonset/[daemonset_name], [namespace/]label/[label-name]/[label-value]")
+	}
+
+	// If no namespace passed, assume it is ALL
+	switch sections[0] {
+	case "pod", "deployment", "daemonset", "labelSelector", "fieldSelector":
+		sections = append([]string{""}, sections...)
+	}
+
+	namespace, selectorType, selectorValue := sections[0], sections[1], sections[2]
+
+	labelSelector := ""
+	fieldSelector := ""
+
+	switch selectorType {
+	case "pod":
+		fieldSelector = "metadata.name=" + selectorValue
+	case "deployment":
+		labelSelector = "app=" + selectorValue
+	case "daemonset":
+		labelSelector = "pod-template-generation=1,name=" + selectorValue
+	case "labelSelector":
+		labelSelector = selectorValue
+	case "fieldSelector":
+		fieldSelector = selectorValue
+	}
+
+	pods, err := clientset.CoreV1().Pods(namespace).List(context.TODO(), metav1.ListOptions{LabelSelector: labelSelector, FieldSelector: fieldSelector})
+	if err != nil {
+		panic(err.Error())
+	}
+
+	var podIPs []string
+	for _, pod := range pods.Items {
+		for _, podIP := range pod.Status.PodIPs {
+			podIPs = append(podIPs, podIP.IP)
+		}
+	}
+	return podIPs
+}
+
+// Filter returns automatic filter applied by goreplay
+// to a pcap handle of a specific interface
+func (l *Listener) Filter(ifi pcap.Interface, hosts ...string) (filter string) {
+	// https://www.tcpdump.org/manpages/pcap-filter.7.html
+
+	if len(hosts) == 0 {
+		// If k8s have not found any IPs
+		if strings.HasPrefix(l.host, "k8s://") {
+			hosts = []string{}
+		} else {
+			hosts = []string{l.host}
+
+			if listenAll(l.host) || isDevice(l.host, ifi) {
+				hosts = interfaceAddresses(ifi)
+			}
+		}
+	}
+
+	filter = portsFilter(l.config.Transport, "dst", l.ports)
+
+	if len(hosts) != 0 && !l.config.Promiscuous {
+		filter = fmt.Sprintf("((%s) and (%s))", filter, hostsFilter("dst", hosts))
+	} else {
+		filter = fmt.Sprintf("(%s)", filter)
+	}
+
+	if l.config.TrackResponse {
+		responseFilter := portsFilter(l.config.Transport, "src", l.ports)
+
+		if len(hosts) != 0 && !l.config.Promiscuous {
+			responseFilter = fmt.Sprintf("((%s) and (%s))", responseFilter, hostsFilter("src", hosts))
+		} else {
+			responseFilter = fmt.Sprintf("(%s)", responseFilter)
+		}
+
+		filter = fmt.Sprintf("%s or %s", filter, responseFilter)
+	}
+
+	if l.config.VLAN {
+		if len(l.config.VLANVIDs) > 0 {
+			for _, vi := range l.config.VLANVIDs {
+				filter = fmt.Sprintf("vlan %d and ", vi) + filter
+			}
+		} else {
+			filter = "vlan and " + filter
+		}
+	}
+
+	return
+}
+
+// PcapHandle returns new pcap Handle from dev on success.
+// this function should be called after setting all necessary options for this listener
+func (l *Listener) PcapHandle(ifi pcap.Interface) (handle *pcap.Handle, err error) {
+	var inactive *pcap.InactiveHandle
+	inactive, err = pcap.NewInactiveHandle(ifi.Name)
+	if err != nil {
+		return nil, fmt.Errorf("inactive handle error: %q, interface: %q", err, ifi.Name)
+	}
+	defer inactive.CleanUp()
+
+	if l.config.TimestampType != "" && l.config.TimestampType != "go" {
+		var ts pcap.TimestampSource
+		ts, err = pcap.TimestampSourceFromString(l.config.TimestampType)
+		fmt.Println("Setting custom Timestamp Source. Supported values: `go`, ", inactive.SupportedTimestamps())
+		err = inactive.SetTimestampSource(ts)
+		if err != nil {
+			return nil, fmt.Errorf("%q: supported timestamps: %q, interface: %q", err, inactive.SupportedTimestamps(), ifi.Name)
+		}
+	}
+	if l.config.Promiscuous {
+		if err = inactive.SetPromisc(l.config.Promiscuous); err != nil {
+			return nil, fmt.Errorf("promiscuous mode error: %q, interface: %q", err, ifi.Name)
+		}
+	}
+	if l.config.Monitor {
+		if err = inactive.SetRFMon(l.config.Monitor); err != nil && !errors.Is(err, pcap.CannotSetRFMon) {
+			return nil, fmt.Errorf("monitor mode error: %q, interface: %q", err, ifi.Name)
+		}
+	}
+
+	var snap int
+
+	if !l.config.Snaplen {
+		infs, _ := net.Interfaces()
+		for _, i := range infs {
+			if i.Name == ifi.Name {
+				snap = i.MTU + 200
+			}
+		}
+	}
+
+	if snap == 0 {
+		snap = 64<<10 + 200
+	}
+
+	err = inactive.SetSnapLen(snap)
+	if err != nil {
+		return nil, fmt.Errorf("snapshot length error: %q, interface: %q", err, ifi.Name)
+	}
+	if l.config.BufferSize > 0 {
+		err = inactive.SetBufferSize(int(l.config.BufferSize))
+		if err != nil {
+			return nil, fmt.Errorf("handle buffer size error: %q, interface: %q", err, ifi.Name)
+		}
+	}
+	if l.config.BufferTimeout == 0 {
+		l.config.BufferTimeout = 2000 * time.Millisecond
+	}
+	err = inactive.SetTimeout(l.config.BufferTimeout)
+	if err != nil {
+		return nil, fmt.Errorf("handle buffer timeout error: %q, interface: %q", err, ifi.Name)
+	}
+	handle, err = inactive.Activate()
+	if err != nil {
+		return nil, fmt.Errorf("PCAP Activate device error: %q, interface: %q", err, ifi.Name)
+	}
+
+	bpfFilter := l.config.BPFFilter
+	if bpfFilter == "" {
+		bpfFilter = l.Filter(ifi)
+	}
+	fmt.Println("Interface:", ifi.Name, ". BPF Filter:", bpfFilter)
+	err = handle.SetBPFFilter(bpfFilter)
+	if err != nil {
+		handle.Close()
+		return nil, fmt.Errorf("BPF filter error: %q%s, interface: %q", err, bpfFilter, ifi.Name)
+	}
+	return
+}
+
+// SocketHandle returns new unix ethernet handle associated with this listener settings
+func (l *Listener) SocketHandle(ifi pcap.Interface) (handle Socket, err error) {
+	handle, err = NewSocket(ifi)
+	if err != nil {
+		return nil, fmt.Errorf("sock raw error: %q, interface: %q", err, ifi.Name)
+	}
+	if err = handle.SetPromiscuous(l.config.Promiscuous || l.config.Monitor); err != nil {
+		return nil, fmt.Errorf("promiscuous mode error: %q, interface: %q", err, ifi.Name)
+	}
+	if l.config.BPFFilter == "" {
+		l.config.BPFFilter = l.Filter(ifi)
+	}
+	fmt.Println("BPF Filter: ", l.config.BPFFilter)
+	if err = handle.SetBPFFilter(l.config.BPFFilter); err != nil {
+		handle.Close()
+		return nil, fmt.Errorf("BPF filter error: %q%s, interface: %q", err, l.config.BPFFilter, ifi.Name)
+	}
+	handle.SetLoopbackIndex(int32(l.loopIndex))
+	return
+}
+
+func http1StartHint(pckt *tcp.Packet) (isRequest, isResponse bool) {
+	if proto.HasRequestTitle(pckt.Payload) {
+		return true, false
+	}
+
+	if proto.HasResponseTitle(pckt.Payload) {
+		return false, true
+	}
+
+	// No request or response detected
+	return false, false
+}
+
+func http1EndHint(m *tcp.Message) bool {
+	if m.MissingChunk() {
+		return false
+	}
+
+	req, res := http1StartHint(m.Packets()[0])
+	return proto.HasFullPayload(m, m.PacketData()...) && (req || res)
+}
+
+func (l *Listener) readHandle(key string, hndl packetHandle) {
+	runtime.LockOSThread()
+
+	defer l.closeHandles(key)
+	linkSize := 14
+	linkType := int(layers.LinkTypeEthernet)
+	if _, ok := hndl.handler.(*pcap.Handle); ok {
+		linkType = int(hndl.handler.(*pcap.Handle).LinkType())
+		linkSize, ok = pcapLinkTypeLength(linkType, l.config.VLAN)
+		if !ok {
+			if os.Getenv("GORDEBUG") != "0" {
+				log.Printf("can not identify link type of an interface '%s'\n", key)
+			}
+			return // can't find the linktype size
+		}
+	}
+
+	messageParser := tcp.NewMessageParser(l.messages, l.ports, hndl.ips, l.config.Expire, l.config.AllowIncomplete)
+
+	if l.config.Protocol == tcp.ProtocolHTTP {
+		messageParser.Start = http1StartHint
+		messageParser.End = http1EndHint
+	}
+
+	timer := time.NewTicker(1 * time.Second)
+
+	for {
+		select {
+		case <-l.quit:
+			return
+		case <-timer.C:
+			if h, ok := hndl.handler.(PcapStatProvider); ok {
+				s, err := h.Stats()
+				if err == nil {
+					stats.Add("packets_received", int64(s.PacketsReceived))
+					stats.Add("packets_dropped", int64(s.PacketsDropped))
+					stats.Add("packets_if_dropped", int64(s.PacketsIfDropped))
+				}
+			}
+		default:
+			data, ci, err := hndl.handler.ReadPacketData()
+			if err == nil {
+				if l.config.TimestampType == "go" {
+					ci.Timestamp = time.Now()
+				}
+
+				messageParser.PacketHandler(&tcp.PcapPacket{
+					Data:     data,
+					LType:    linkType,
+					LTypeLen: linkSize,
+					Ci:       &ci,
+				})
+				continue
+			}
+			if enext, ok := err.(pcap.NextError); ok && enext == pcap.NextErrorTimeoutExpired {
+				continue
+			}
+			if eno, ok := err.(syscall.Errno); ok && eno.Temporary() {
+				continue
+			}
+			if enet, ok := err.(*net.OpError); ok && (enet.Temporary() || enet.Timeout()) {
+				continue
+			}
+			if err == io.EOF || err == io.ErrClosedPipe {
+				log.Printf("stopped reading from %s interface with error %s\n", key, err)
+				return
+			}
+
+			log.Printf("stopped reading from %s interface with error %s\n", key, err)
+			return
+		}
+	}
+}
+
+func (l *Listener) Messages() chan *tcp.Message {
+	return l.messages
+}
+
+func (l *Listener) closeHandles(key string) {
+	l.Lock()
+	defer l.Unlock()
+	if handle, ok := l.Handles[key]; ok {
+		if c, ok := handle.handler.(io.Closer); ok {
+			c.Close()
+		}
+
+		delete(l.Handles, key)
+		if len(l.Handles) == 0 {
+			close(l.closeDone)
+		}
+	}
+}
+
+func (l *Listener) activatePcap() error {
+	var e error
+	var msg string
+	for _, ifi := range l.Interfaces {
+		if _, found := l.Handles[ifi.Name]; found {
+			continue
+		}
+
+		var handle *pcap.Handle
+		handle, e = l.PcapHandle(ifi)
+		if e != nil {
+			msg += ("\n" + e.Error())
+			continue
+		}
+		l.Handles[ifi.Name] = packetHandle{
+			handler: handle,
+			ips:     interfaceIPs(ifi),
+		}
+	}
+	if len(l.Handles) == 0 {
+		return fmt.Errorf("pcap handles error:%s", msg)
+	}
+	return nil
+}
+
+func (l *Listener) activateVxLanSocket() error {
+	handler, err := newVXLANHandler(l.config.VXLANPort, l.config.VXLANVNIs)
+	if err != nil {
+		return err
+	}
+	l.Handles["vxlan"] = packetHandle{
+		handler: handler,
+	}
+
+	return nil
+}
+
+func (l *Listener) activateRawSocket() error {
+	if runtime.GOOS != "linux" {
+		return fmt.Errorf("sock_raw is not stabilized on OS other than linux")
+	}
+	var msg string
+	var e error
+	for _, ifi := range l.Interfaces {
+		if _, found := l.Handles[ifi.Name]; found {
+			continue
+		}
+
+		var handle Socket
+		handle, e = l.SocketHandle(ifi)
+		if e != nil {
+			msg += ("\n" + e.Error())
+			continue
+		}
+		l.Handles[ifi.Name] = packetHandle{
+			handler: handle,
+			ips:     interfaceIPs(ifi),
+		}
+	}
+	if len(l.Handles) == 0 {
+		return fmt.Errorf("raw socket handles error:%s", msg)
+	}
+	return nil
+}
+
+func (l *Listener) activatePcapFile() (err error) {
+	var handle *pcap.Handle
+	var e error
+	if handle, e = pcap.OpenOffline(l.host); e != nil {
+		return fmt.Errorf("open pcap file error: %q", e)
+	}
+
+	tmp := l.host
+	l.host = ""
+	l.config.BPFFilter = l.Filter(pcap.Interface{})
+	l.host = tmp
+
+	if e = handle.SetBPFFilter(l.config.BPFFilter); e != nil {
+		handle.Close()
+		return fmt.Errorf("BPF filter error: %q, filter: %s", e, l.config.BPFFilter)
+	}
+
+	fmt.Println("BPF Filter:", l.config.BPFFilter)
+
+	l.Handles["pcap_file"] = packetHandle{
+		handler: handle,
+	}
+	return
+}
+
+func (l *Listener) activateAFPacket() error {
+	szFrame, szBlock, numBlocks, err := afpacketComputeSize(32, 32<<10, os.Getpagesize())
+	if err != nil {
+		return err
+	}
+
+	var msg string
+	for _, ifi := range l.Interfaces {
+		if _, found := l.Handles[ifi.Name]; found {
+			continue
+		}
+
+		handle, err := newAfpacketHandle(ifi.Name, szFrame, szBlock, numBlocks, false, pcap.BlockForever)
+
+		if err != nil {
+			msg += ("\n" + err.Error())
+			continue
+		}
+
+		if l.config.BPFFilter == "" {
+			l.config.BPFFilter = l.Filter(ifi)
+		}
+		fmt.Println("Interface:", ifi.Name, ". BPF Filter:", l.config.BPFFilter)
+		handle.SetBPFFilter(l.config.BPFFilter, 64<<10)
+
+		l.Handles[ifi.Name] = packetHandle{
+			handler: handle,
+			ips:     interfaceIPs(ifi),
+		}
+	}
+
+	if len(l.Handles) == 0 {
+		return fmt.Errorf("pcap handles error:%s", msg)
+	}
+
+	return nil
+}
+
+func (l *Listener) setInterfaces() (err error) {
+	var pifis []pcap.Interface
+	pifis, err = pcap.FindAllDevs()
+	ifis, _ := net.Interfaces()
+	l.Interfaces = []pcap.Interface{}
+
+	if err != nil {
+		return
+	}
+
+	for _, pi := range pifis {
+		ignore := false
+		for _, ig := range l.config.IgnoreInterface {
+			if pi.Name == ig {
+				ignore = true
+				break
+			}
+		}
+
+		if ignore {
+			continue
+		}
+
+		if strings.HasPrefix(l.host, "k8s://") {
+			if !strings.HasPrefix(pi.Name, "veth") {
+				continue
+			}
+		}
+
+		if isDevice(l.host, pi) {
+			l.Interfaces = []pcap.Interface{pi}
+			return
+		}
+
+		var ni net.Interface
+		for _, i := range ifis {
+			if i.Name == pi.Name {
+				ni = i
+				break
+			}
+
+			addrs, _ := i.Addrs()
+			for _, a := range addrs {
+				for _, pa := range pi.Addresses {
+					if a.String() == pa.IP.String() {
+						ni = i
+						break
+					}
+				}
+			}
+		}
+
+		if ni.Flags&net.FlagLoopback != 0 {
+			l.loopIndex = ni.Index
+		}
+
+		if runtime.GOOS != "windows" {
+			if len(pi.Addresses) == 0 {
+				continue
+			}
+
+			if ni.Flags&net.FlagUp == 0 {
+				continue
+			}
+		}
+
+		l.Interfaces = append(l.Interfaces, pi)
+	}
+	return
+}
+
+func isDevice(addr string, ifi pcap.Interface) bool {
+	// Windows npcap loopback have no IPs
+	if addr == "127.0.0.1" && ifi.Name == `\Device\NPF_Loopback` {
+		return true
+	}
+
+	if addr == ifi.Name {
+		return true
+	}
+
+	if strings.HasSuffix(addr, "*") {
+		if strings.HasPrefix(ifi.Name, addr[:len(addr)-1]) {
+			return true
+		}
+	}
+
+	for _, _addr := range ifi.Addresses {
+		if _addr.IP.String() == addr {
+			return true
+		}
+	}
+
+	return false
+}
+
+func interfaceAddresses(ifi pcap.Interface) []string {
+	var hosts []string
+	for _, addr := range ifi.Addresses {
+		hosts = append(hosts, addr.IP.String())
+	}
+	return hosts
+}
+
+func interfaceIPs(ifi pcap.Interface) []net.IP {
+	var ips []net.IP
+	for _, addr := range ifi.Addresses {
+		ips = append(ips, addr.IP)
+	}
+	return ips
+}
+
+func listenAll(addr string) bool {
+	switch addr {
+	case "", "0.0.0.0", "[::]", "::":
+		return true
+	}
+	return false
+}
+
+func portsFilter(transport string, direction string, ports []uint16) string {
+	if len(ports) == 0 || ports[0] == 0 {
+		return fmt.Sprintf("%s %s portrange 0-%d", transport, direction, 1<<16-1)
+	}
+
+	var filters []string
+	for _, port := range ports {
+		filters = append(filters, fmt.Sprintf("%s %s port %d", transport, direction, port))
+	}
+	return strings.Join(filters, " or ")
+}
+
+func hostsFilter(direction string, hosts []string) string {
+	var hostsFilters []string
+	for _, host := range hosts {
+		hostsFilters = append(hostsFilters, fmt.Sprintf("%s host %s", direction, host))
+	}
+
+	return strings.Join(hostsFilters, " or ")
+}
+
+func pcapLinkTypeLength(lType int, vlan bool) (int, bool) {
+	switch layers.LinkType(lType) {
+	case layers.LinkTypeEthernet:
+		if vlan {
+			return 18, true
+		} else {
+			return 14, true
+		}
+	case layers.LinkTypeNull, layers.LinkTypeLoop:
+		return 4, true
+	case layers.LinkTypeRaw, 12, 14:
+		return 0, true
+	case layers.LinkTypeIPv4, layers.LinkTypeIPv6:
+		// (TODO:) look out for IP encapsulation?
+		return 0, true
+	case layers.LinkTypeLinuxSLL:
+		return 16, true
+	case layers.LinkTypeFDDI:
+		return 13, true
+	case 226 /*DLT_IPNET*/ :
+		// https://www.tcpdump.org/linktypes/LINKTYPE_IPNET.html
+		return 24, true
+	default:
+		return 0, false
+	}
+}

internal/capture/capture_test.go

@@ -0,0 +1,22 @@
+package capture
+
+import (
+	"testing"
+)
+
+func TestSetInterfaces(t *testing.T) {
+	listener := &Listener{
+		loopIndex: 99999,
+	}
+	listener.setInterfaces()
+
+	for _, nic := range listener.Interfaces {
+		if (len(nic.Addresses)) == 0 {
+			t.Errorf("nic %s was captured with 0 addresses", nic.Name)
+		}
+	}
+
+	if listener.loopIndex == 99999 {
+		t.Errorf("loopback nic index was not found")
+	}
+}

internal/capture/doc.go

@@ -0,0 +1,42 @@
+/*
+Package capture provides traffic sniffier using AF_PACKET, pcap or pcap file.
+it allows you to listen for traffic from any port (e.g. sniffing) because they operate on IP level.
+Ports is TCP/IP feature, same as flow control, reliable transmission and etc.
+Currently this package implements TCP layer: flow control is managed under tcp package.
+BPF filters can also be applied.
+
+example:
+
+// for the transport should be "tcp"
+listener, err := capture.NewListener(host, port, transport, engine, trackResponse)
+
+	if err != nil {
+		// handle error
+	}
+
+listener.SetPcapOptions(opts)
+err = listner.Activate()
+
+	if err != nil {
+		// handle it
+	}
+
+	if err := listener.Listen(context.Background(), handler); err != nil {
+		 // handle error
+	}
+
+// or
+errCh := listener.ListenBackground(context.Background(), handler) // runs in the background
+select {
+case err := <- errCh:
+
+	// handle error
+
+case <-quit:
+
+	//
+
+case <- l.Reading: // if we have started reading
+}
+*/
+package capture // import github.com/buger/goreplay/capture

internal/capture/dump.go

@@ -0,0 +1,126 @@
+// https://github.com/google/gopacket/blob/403ca653c4/pcapgo/read.go
+
+package capture
+
+import (
+	"encoding/binary"
+	"fmt"
+	"io"
+	"time"
+
+	"github.com/google/gopacket"
+	"github.com/google/gopacket/layers"
+)
+
+// Writer wraps an underlying io.Writer to write packet data in PCAP
+// format.  See http://wiki.wireshark.org/Development/LibpcapFileFormat
+// for information on the file format.
+//
+// For those that care, we currently write v2.4 files with nanosecond
+// or microsecond timestamp resolution and little-endian encoding.
+type Writer struct {
+	w        io.Writer
+	tsScaler int
+	// Moving this into the struct seems to save an allocation for each call to writePacketHeader
+	buf [16]byte
+}
+
+const magicNanoseconds = 0xA1B23C4D
+const magicMicroseconds = 0xA1B2C3D4
+const versionMajor = 2
+const versionMinor = 4
+
+// NewWriterNanos returns a new writer object, for writing packet data out
+// to the given writer.  If this is a new empty writer (as opposed to
+// an append), you must call WriteFileHeader before WritePacket.  Packet
+// timestamps are written with nanosecond precision.
+//
+//	// Write a new file:
+//	f, _ := os.Create("/tmp/file.pcap")
+//	w := pcapgo.NewWriterNanos(f)
+//	w.WriteFileHeader(65536, layers.LinkTypeEthernet)  // new file, must do this.
+//	w.WritePacket(gopacket.CaptureInfo{...}, data1)
+//	f.Close()
+//	// Append to existing file (must have same snaplen and linktype)
+//	f2, _ := os.OpenFile("/tmp/fileNano.pcap", os.O_APPEND, 0700)
+//	w2 := pcapgo.NewWriter(f2)
+//	// no need for file header, it's already written.
+//	w2.WritePacket(gopacket.CaptureInfo{...}, data2)
+//	f2.Close()
+func NewWriterNanos(w io.Writer) *Writer {
+	return &Writer{w: w, tsScaler: nanosPerNano}
+}
+
+// NewWriter returns a new writer object, for writing packet data out
+// to the given writer.  If this is a new empty writer (as opposed to
+// an append), you must call WriteFileHeader before WritePacket.
+// Packet timestamps are written with microsecond precision.
+//
+//	// Write a new file:
+//	f, _ := os.Create("/tmp/file.pcap")
+//	w := pcapgo.NewWriter(f)
+//	w.WriteFileHeader(65536, layers.LinkTypeEthernet)  // new file, must do this.
+//	w.WritePacket(gopacket.CaptureInfo{...}, data1)
+//	f.Close()
+//	// Append to existing file (must have same snaplen and linktype)
+//	f2, _ := os.OpenFile("/tmp/file.pcap", os.O_APPEND, 0700)
+//	w2 := pcapgo.NewWriter(f2)
+//	// no need for file header, it's already written.
+//	w2.WritePacket(gopacket.CaptureInfo{...}, data2)
+//	f2.Close()
+func NewWriter(w io.Writer) *Writer {
+	return &Writer{w: w, tsScaler: nanosPerMicro}
+}
+
+// WriteFileHeader writes a file header out to the writer.
+// This must be called exactly once per output.
+func (w *Writer) WriteFileHeader(snaplen uint32, linktype layers.LinkType) error {
+	var buf [24]byte
+	if w.tsScaler == nanosPerMicro {
+		binary.LittleEndian.PutUint32(buf[0:4], magicMicroseconds)
+	} else {
+		binary.LittleEndian.PutUint32(buf[0:4], magicNanoseconds)
+	}
+	binary.LittleEndian.PutUint16(buf[4:6], versionMajor)
+	binary.LittleEndian.PutUint16(buf[6:8], versionMinor)
+	// bytes 8:12 stay 0 (timezone = UTC)
+	// bytes 12:16 stay 0 (sigfigs is always set to zero, according to
+	//   http://wiki.wireshark.org/Development/LibpcapFileFormat
+	binary.LittleEndian.PutUint32(buf[16:20], snaplen)
+	binary.LittleEndian.PutUint32(buf[20:24], uint32(linktype))
+	_, err := w.w.Write(buf[:])
+	return err
+}
+
+const nanosPerMicro = 1000
+const nanosPerNano = 1
+
+func (w *Writer) writePacketHeader(ci gopacket.CaptureInfo) error {
+	t := ci.Timestamp
+	if t.IsZero() {
+		t = time.Now()
+	}
+	secs := t.Unix()
+	usecs := t.Nanosecond() / w.tsScaler
+	binary.LittleEndian.PutUint32(w.buf[0:4], uint32(secs))
+	binary.LittleEndian.PutUint32(w.buf[4:8], uint32(usecs))
+	binary.LittleEndian.PutUint32(w.buf[8:12], uint32(ci.CaptureLength))
+	binary.LittleEndian.PutUint32(w.buf[12:16], uint32(ci.Length))
+	_, err := w.w.Write(w.buf[:])
+	return err
+}
+
+// WritePacket writes the given packet data out to the file.
+func (w *Writer) WritePacket(ci gopacket.CaptureInfo, data []byte) error {
+	if ci.CaptureLength != len(data) {
+		return fmt.Errorf("capture length %d does not match data length %d", ci.CaptureLength, len(data))
+	}
+	if ci.CaptureLength > ci.Length {
+		return fmt.Errorf("invalid capture info %+v:  capture length > length", ci)
+	}
+	if err := w.writePacketHeader(ci); err != nil {
+		return fmt.Errorf("error writing packet header: %v", err)
+	}
+	_, err := w.w.Write(data)
+	return err
+}

internal/capture/sock_linux.go

@@ -0,0 +1,278 @@
+//go:build linux && !arm64
+
+package capture
+
+import (
+	"fmt"
+	"net"
+	"sync"
+	"time"
+	"unsafe"
+
+	"golang.org/x/sys/unix"
+
+	"github.com/google/gopacket"
+	"github.com/google/gopacket/layers"
+	"github.com/google/gopacket/pcap"
+)
+
+const (
+	// ETHALL htons(ETH_P_ALL)
+	ETHALL uint16 = unix.ETH_P_ALL<<8 | unix.ETH_P_ALL>>8
+	// BLOCKSIZE ring buffer block_size
+	BLOCKSIZE = 64 << 10
+	// BLOCKNR ring buffer block_nr
+	BLOCKNR = (2 << 20) / BLOCKSIZE // 2mb / 64kb
+	// FRAMESIZE ring buffer frame_size
+	FRAMESIZE = BLOCKSIZE
+	// FRAMENR ring buffer frame_nr
+	FRAMENR = BLOCKNR * BLOCKSIZE / FRAMESIZE
+	// MAPHUGE2MB 2mb huge map
+	MAPHUGE2MB = 21 << unix.MAP_HUGE_SHIFT
+)
+
+var tpacket2hdrlen = tpAlign(int(unsafe.Sizeof(unix.Tpacket2Hdr{})))
+
+// SockRaw is a linux M'maped af_packet socket
+type SockRaw struct {
+	mu          sync.Mutex
+	fd          int
+	ifindex     int
+	snaplen     int
+	pollTimeout uintptr
+	frame       uint32 // current frame
+	buf         []byte // points to the memory space of the ring buffer shared with the kernel.
+	loopIndex   int32  // this field must filled to avoid reading packet twice on a loopback device
+}
+
+// NewSocket returns new M'maped sock_raw on packet version 2.
+func NewSocket(pifi pcap.Interface) (*SockRaw, error) {
+	var ifi net.Interface
+
+	infs, _ := net.Interfaces()
+	found := false
+	for _, i := range infs {
+		if i.Name == pifi.Name {
+			ifi = i
+			found = true
+			break
+		}
+	}
+
+	if !found {
+		return nil, fmt.Errorf("can't find matching interface")
+	}
+
+	// sock create
+	fd, err := unix.Socket(unix.AF_PACKET, unix.SOCK_RAW, int(ETHALL))
+	if err != nil {
+		return nil, err
+	}
+	sock := &SockRaw{
+		fd:          fd,
+		ifindex:     ifi.Index,
+		snaplen:     FRAMESIZE,
+		pollTimeout: ^uintptr(0),
+	}
+
+	// set packet version
+	err = unix.SetsockoptInt(fd, unix.SOL_PACKET, unix.PACKET_VERSION, unix.TPACKET_V2)
+	if err != nil {
+		unix.Close(fd)
+		return nil, fmt.Errorf("setsockopt packet_version: %v", err)
+	}
+
+	// bind to interface
+	addr := unix.RawSockaddrLinklayer{
+		Family:   unix.AF_PACKET,
+		Protocol: ETHALL,
+		Ifindex:  int32(ifi.Index),
+	}
+	_, _, e := unix.Syscall(
+		unix.SYS_BIND,
+		uintptr(fd),
+		uintptr(unsafe.Pointer(&addr)),
+		uintptr(unix.SizeofSockaddrLinklayer),
+	)
+	if e != 0 {
+		unix.Close(fd)
+		return nil, e
+	}
+
+	// create shared-memory ring buffer
+	tp := &unix.TpacketReq{
+		Block_size: BLOCKSIZE,
+		Block_nr:   BLOCKNR,
+		Frame_size: FRAMESIZE,
+		Frame_nr:   FRAMENR,
+	}
+	err = unix.SetsockoptTpacketReq(sock.fd, unix.SOL_PACKET, unix.PACKET_RX_RING, tp)
+	if err != nil {
+		unix.Close(fd)
+		return nil, fmt.Errorf("setsockopt packet_rx_ring: %v", err)
+	}
+	sock.buf, err = unix.Mmap(
+		sock.fd,
+		0,
+		BLOCKSIZE*BLOCKNR,
+		unix.PROT_READ|unix.PROT_WRITE,
+		unix.MAP_SHARED|MAPHUGE2MB,
+	)
+	if err != nil {
+		unix.Close(fd)
+		return nil, fmt.Errorf("socket mmap error: %v", err)
+	}
+	return sock, nil
+}
+
+// ReadPacketData implements gopacket.PacketDataSource.
+func (sock *SockRaw) ReadPacketData() (buf []byte, ci gopacket.CaptureInfo, err error) {
+	sock.mu.Lock()
+	defer sock.mu.Unlock()
+	var tpHdr *unix.Tpacket2Hdr
+	poll := &unix.PollFd{
+		Fd:     int32(sock.fd),
+		Events: unix.POLLIN,
+	}
+	var i int
+read:
+	i = int(sock.frame * FRAMESIZE)
+	tpHdr = (*unix.Tpacket2Hdr)(unsafe.Pointer(&sock.buf[i]))
+	sock.frame = (sock.frame + 1) % FRAMENR
+
+	if tpHdr.Status&unix.TP_STATUS_USER == 0 {
+		_, _, e := unix.Syscall(unix.SYS_POLL, uintptr(unsafe.Pointer(poll)), 1, sock.pollTimeout)
+		if e != 0 && e != unix.EINTR {
+			return buf, ci, e
+		}
+		// it might be some other frame with data!
+		if tpHdr.Status&unix.TP_STATUS_USER == 0 {
+			goto read
+		}
+	}
+	tpHdr.Status = unix.TP_STATUS_KERNEL
+	sockAddr := (*unix.RawSockaddrLinklayer)(unsafe.Pointer(&sock.buf[i+tpacket2hdrlen]))
+
+	// parse out repeating packets on loopback
+	if sockAddr.Ifindex == sock.loopIndex && sock.frame%2 != 0 {
+		goto read
+	}
+
+	ci.Length = int(tpHdr.Len)
+	ci.Timestamp = time.Unix(int64(tpHdr.Sec), int64(tpHdr.Nsec))
+	ci.InterfaceIndex = int(sockAddr.Ifindex)
+	buf = make([]byte, tpHdr.Snaplen)
+	ci.CaptureLength = copy(buf, sock.buf[i+int(tpHdr.Mac):])
+
+	return
+}
+
+// Close closes the underlying socket
+func (sock *SockRaw) Close() (err error) {
+	sock.mu.Lock()
+	defer sock.mu.Unlock()
+	if sock.fd != -1 {
+		unix.Munmap(sock.buf)
+		sock.buf = nil
+		err = unix.Close(sock.fd)
+		sock.fd = -1
+	}
+	return
+}
+
+// SetSnapLen sets the maximum capture length to the given value.
+// for this to take effects on the kernel level SetBPFilter should be called too.
+func (sock *SockRaw) SetSnapLen(snap int) error {
+	sock.mu.Lock()
+	defer sock.mu.Unlock()
+	if snap < 0 {
+		return fmt.Errorf("expected %d snap length to be at least 0", snap)
+	}
+	if snap > FRAMESIZE {
+		snap = FRAMESIZE
+	}
+	sock.snaplen = snap
+	return nil
+}
+
+// SetTimeout sets poll wait timeout for the socket.
+// negative value will block forever
+func (sock *SockRaw) SetTimeout(t time.Duration) error {
+	sock.mu.Lock()
+	defer sock.mu.Unlock()
+	sock.pollTimeout = uintptr(t)
+	return nil
+}
+
+// GetSnapLen returns the maximum capture length
+func (sock *SockRaw) GetSnapLen() int {
+	sock.mu.Lock()
+	defer sock.mu.Unlock()
+	return sock.snaplen
+}
+
+// SetBPFFilter compiles and sets a BPF filter for the socket handle.
+func (sock *SockRaw) SetBPFFilter(expr string) error {
+	sock.mu.Lock()
+	defer sock.mu.Unlock()
+	if expr == "" {
+		return unix.SetsockoptInt(sock.fd, unix.SOL_SOCKET, unix.SO_DETACH_FILTER, 0)
+	}
+	filter, err := pcap.CompileBPFFilter(layers.LinkTypeEthernet, sock.snaplen, expr)
+	if err != nil {
+		return err
+	}
+	if len(filter) > int(^uint16(0)) {
+		return fmt.Errorf("filters out of range 0-%d", ^uint16(0))
+	}
+	if len(filter) == 0 {
+		return unix.SetsockoptInt(sock.fd, unix.SOL_SOCKET, unix.SO_DETACH_FILTER, 0)
+	}
+	fprog := &unix.SockFprog{
+		Len:    uint16(len(filter)),
+		Filter: &(*(*[]unix.SockFilter)(unsafe.Pointer(&filter)))[0],
+	}
+	return unix.SetsockoptSockFprog(sock.fd, unix.SOL_SOCKET, unix.SO_ATTACH_FILTER, fprog)
+}
+
+// SetPromiscuous sets promiscuous mode to the required value. for better result capture on all interfaces instead.
+// If it is enabled, traffic not destined for the interface will also be captured.
+func (sock *SockRaw) SetPromiscuous(b bool) error {
+	sock.mu.Lock()
+	defer sock.mu.Unlock()
+	mreq := unix.PacketMreq{
+		Ifindex: int32(sock.ifindex),
+		Type:    unix.PACKET_MR_PROMISC,
+	}
+
+	opt := unix.PACKET_ADD_MEMBERSHIP
+	if !b {
+		opt = unix.PACKET_DROP_MEMBERSHIP
+	}
+
+	return unix.SetsockoptPacketMreq(sock.fd, unix.SOL_PACKET, opt, &mreq)
+}
+
+// Stats returns number of packets and dropped packets. This will be the number of packets/dropped packets since the last call to stats (not the cummulative sum!).
+func (sock *SockRaw) Stats() (*unix.TpacketStats, error) {
+	sock.mu.Lock()
+	defer sock.mu.Unlock()
+	return unix.GetsockoptTpacketStats(sock.fd, unix.SOL_PACKET, unix.PACKET_STATISTICS)
+}
+
+// SetLoopbackIndex necessary to avoid reading packet twice on a loopback device
+func (sock *SockRaw) SetLoopbackIndex(i int32) {
+	sock.mu.Lock()
+	defer sock.mu.Unlock()
+	sock.loopIndex = i
+}
+
+// WritePacketData transmits a raw packet.
+func (sock *SockRaw) WritePacketData(pkt []byte) error {
+	_, err := unix.Write(sock.fd, pkt)
+	return err
+}
+
+func tpAlign(x int) int {
+	return int((uint(x) + unix.TPACKET_ALIGNMENT - 1) &^ (unix.TPACKET_ALIGNMENT - 1))
+}

internal/capture/sock_others.go

@@ -0,0 +1,14 @@
+//go:build !linux || arm64 || darwin
+
+package capture
+
+import (
+	"errors"
+
+	"github.com/google/gopacket/pcap"
+)
+
+// NewSocket returns new M'maped sock_raw on packet version 2.
+func NewSocket(_ pcap.Interface) (Socket, error) {
+	return nil, errors.New("afpacket socket is only available on linux")
+}

internal/capture/socket.go

@@ -0,0 +1,20 @@
+package capture
+
+import (
+	"time"
+
+	"github.com/google/gopacket"
+)
+
+// Socket is any interface that defines the behaviors of Socket
+type Socket interface {
+	ReadPacketData() ([]byte, gopacket.CaptureInfo, error)
+	WritePacketData([]byte) error
+	SetBPFFilter(string) error
+	SetPromiscuous(bool) error
+	SetSnapLen(int) error
+	GetSnapLen() int
+	SetTimeout(time.Duration) error
+	SetLoopbackIndex(i int32)
+	Close() error
+}

internal/capture/vxlan.go

@@ -0,0 +1,100 @@
+package capture
+
+import (
+	"errors"
+	"fmt"
+	"github.com/google/gopacket"
+	"github.com/google/gopacket/layers"
+	"net"
+	"time"
+)
+
+const VxLanPacketSize = 1526 //vxlan 8 B + ethernet II 1518 B
+
+type vxlanHandle struct {
+	connection    *net.UDPConn
+	packetChannel chan gopacket.Packet
+	vnis          []int
+}
+
+func newVXLANHandler(port int, vnis []int) (*vxlanHandle, error) {
+	if port == 0 {
+		port = 4789
+	}
+
+	addr := net.UDPAddr{
+		Port: port,
+		IP:   net.ParseIP("0.0.0.0"),
+	}
+
+	vxlanHandle := &vxlanHandle{}
+	con, err := net.ListenUDP("udp", &addr)
+	if err != nil {
+		return nil, fmt.Errorf(err.Error())
+	}
+	vxlanHandle.connection = con
+	vxlanHandle.packetChannel = make(chan gopacket.Packet, 1000)
+	vxlanHandle.vnis = vnis
+	go vxlanHandle.reader()
+
+	return vxlanHandle, nil
+}
+
+func (v *vxlanHandle) reader() {
+	for {
+		inputBytes := make([]byte, VxLanPacketSize)
+		length, _, err := v.connection.ReadFromUDP(inputBytes)
+		if err != nil {
+			if errors.Is(err, net.ErrClosed) {
+				return
+			}
+			continue
+		}
+		packet := gopacket.NewPacket(inputBytes[:length], layers.LayerTypeVXLAN, gopacket.NoCopy)
+		ci := packet.Metadata()
+		ci.Timestamp = time.Now()
+		ci.CaptureLength = length
+		ci.Length = length
+
+		if len(v.vnis) > 0 && !v.vniIsAllowed(packet) {
+			continue
+		}
+
+		v.packetChannel <- packet
+	}
+}
+
+func (v *vxlanHandle) vniIsAllowed(packet gopacket.Packet) bool {
+	defaultState := false
+	if layer := packet.Layer(layers.LayerTypeVXLAN); layer != nil {
+		vxlan, _ := layer.(*layers.VXLAN)
+		for _, vn := range v.vnis {
+			if vn > 0 && int(vxlan.VNI) == vn {
+				return true
+			}
+
+			if vn < 0 {
+				if int(vxlan.VNI) == -vn {
+					return false
+				}
+				defaultState = true
+			}
+		}
+	}
+	return defaultState
+}
+
+func (v *vxlanHandle) ReadPacketData() ([]byte, gopacket.CaptureInfo, error) {
+	packet := <-v.packetChannel
+	layer := packet.Layer(layers.LayerTypeVXLAN)
+	bytes := layer.LayerPayload()
+
+	return bytes, packet.Metadata().CaptureInfo, nil
+}
+
+func (v *vxlanHandle) Close() error {
+	if v.connection != nil {
+		return v.connection.Close()
+	}
+	return nil
+}

internal/ring/ring.go

@@ -0,0 +1,205 @@
+package ring
+
+import (
+	"errors"
+	"runtime"
+	"sync/atomic"
+	"time"
+)
+
+var (
+	// ErrDisposed is returned when an operation is performed on a disposed
+	// queue.
+	ErrDisposed = errors.New(`queue: disposed`)
+
+	// ErrTimeout is returned when an applicable queue operation times out.
+	ErrTimeout = errors.New(`queue: poll timed out`)
+
+	// ErrEmptyQueue is returned when an non-applicable queue operation was called
+	// due to the queue's empty item state
+	ErrEmptyQueue = errors.New(`queue: empty queue`)
+)
+
+// roundUp takes a uint64 greater than 0 and rounds it up to the next
+// power of 2.
+func roundUp(v uint64) uint64 {
+	v--
+	v |= v >> 1
+	v |= v >> 2
+	v |= v >> 4
+	v |= v >> 8
+	v |= v >> 16
+	v |= v >> 32
+	v++
+	return v
+}
+
+type node struct {
+	position uint64
+	data     interface{}
+}
+
+type nodes []node
+
+// RingBuffer is a MPMC buffer that achieves threadsafety with CAS operations
+// only.  A put on full or get on empty call will block until an item
+// is put or retrieved.  Calling Dispose on the RingBuffer will unblock
+// any blocked threads with an error.  This buffer is similar to the buffer
+// described here: http://www.1024cores.net/home/lock-free-algorithms/queues/bounded-mpmc-queue
+// with some minor additions.
+type RingBuffer struct {
+	_padding0      [8]uint64
+	queue          uint64
+	_padding1      [8]uint64
+	dequeue        uint64
+	_padding2      [8]uint64
+	mask, disposed uint64
+	_padding3      [8]uint64
+	nodes          nodes
+}
+
+func (rb *RingBuffer) init(size uint64) {
+	size = roundUp(size)
+	rb.nodes = make(nodes, size)
+	for i := uint64(0); i < size; i++ {
+		rb.nodes[i] = node{position: i}
+	}
+	rb.mask = size - 1 // so we don't have to do this with every put/get operation
+}
+
+// Put adds the provided item to the queue.  If the queue is full, this
+// call will block until an item is added to the queue or Dispose is called
+// on the queue.  An error will be returned if the queue is disposed.
+func (rb *RingBuffer) Put(item interface{}) error {
+	_, err := rb.put(item, false)
+	return err
+}
+
+// Offer adds the provided item to the queue if there is space.  If the queue
+// is full, this call will return false.  An error will be returned if the
+// queue is disposed.
+func (rb *RingBuffer) Offer(item interface{}) (bool, error) {
+	return rb.put(item, true)
+}
+
+func (rb *RingBuffer) put(item interface{}, offer bool) (bool, error) {
+	var n *node
+	pos := atomic.LoadUint64(&rb.queue)
+L:
+	for {
+		if atomic.LoadUint64(&rb.disposed) == 1 {
+			return false, ErrDisposed
+		}
+
+		n = &rb.nodes[pos&rb.mask]
+		seq := atomic.LoadUint64(&n.position)
+		switch dif := seq - pos; {
+		case dif == 0:
+			if atomic.CompareAndSwapUint64(&rb.queue, pos, pos+1) {
+				break L
+			}
+		case dif < 0:
+			panic(`Ring buffer in a compromised state during a put operation.`)
+		default:
+			pos = atomic.LoadUint64(&rb.queue)
+		}
+
+		if offer {
+			return false, nil
+		}
+
+		runtime.Gosched() // free up the cpu before the next iteration
+	}
+
+	n.data = item
+	atomic.StoreUint64(&n.position, pos+1)
+	return true, nil
+}
+
+// Get will return the next item in the queue.  This call will block
+// if the queue is empty.  This call will unblock when an item is added
+// to the queue or Dispose is called on the queue.  An error will be returned
+// if the queue is disposed.
+func (rb *RingBuffer) Get() (interface{}, error) {
+	return rb.Poll(0)
+}
+
+// Poll will return the next item in the queue.  This call will block
+// if the queue is empty.  This call will unblock when an item is added
+// to the queue, Dispose is called on the queue, or the timeout is reached. An
+// error will be returned if the queue is disposed or a timeout occurs. A
+// non-positive timeout will block indefinitely.
+func (rb *RingBuffer) Poll(timeout time.Duration) (interface{}, error) {
+	var (
+		n     *node
+		pos   = atomic.LoadUint64(&rb.dequeue)
+		start time.Time
+	)
+	if timeout > 0 {
+		start = time.Now()
+	}
+L:
+	for {
+		if atomic.LoadUint64(&rb.disposed) == 1 {
+			return nil, ErrDisposed
+		}
+
+		n = &rb.nodes[pos&rb.mask]
+		seq := atomic.LoadUint64(&n.position)
+		switch dif := seq - (pos + 1); {
+		case dif == 0:
+			if atomic.CompareAndSwapUint64(&rb.dequeue, pos, pos+1) {
+				break L
+			}
+		case dif < 0:
+			panic(`Ring buffer in compromised state during a get operation.`)
+		default:
+			pos = atomic.LoadUint64(&rb.dequeue)
+		}
+
+		if timeout > 0 && time.Since(start) >= timeout {
+			return nil, ErrTimeout
+		}
+
+		if timeout < 0 {
+			return nil, ErrTimeout
+		}
+
+		runtime.Gosched() // free up the cpu before the next iteration
+	}
+	data := n.data
+	n.data = nil
+	atomic.StoreUint64(&n.position, pos+rb.mask+1)
+	return data, nil
+}
+
+// Len returns the number of items in the queue.
+func (rb *RingBuffer) Len() uint64 {
+	return atomic.LoadUint64(&rb.queue) - atomic.LoadUint64(&rb.dequeue)
+}
+
+// Cap returns the capacity of this ring buffer.
+func (rb *RingBuffer) Cap() uint64 {
+	return uint64(len(rb.nodes))
+}
+
+// Dispose will dispose of this queue and free any blocked threads
+// in the Put and/or Get methods.  Calling those methods on a disposed
+// queue will return an error.
+func (rb *RingBuffer) Dispose() {
+	atomic.CompareAndSwapUint64(&rb.disposed, 0, 1)
+}
+
+// IsDisposed will return a bool indicating if this queue has been
+// disposed.
+func (rb *RingBuffer) IsDisposed() bool {
+	return atomic.LoadUint64(&rb.disposed) == 1
+}
+
+// NewRingBuffer will allocate, initialize, and return a ring buffer
+// with the specified size.
+func NewRingBuffer(size uint64) *RingBuffer {
+	rb := &RingBuffer{}
+	rb.init(size)
+	return rb
+}

internal/simpletime/time.go

@@ -0,0 +1,17 @@
+package simpletime
+
+import (
+	"time"
+)
+
+var Now time.Time
+
+func init() {
+	go func() {
+		for {
+			// Accurate enough
+			Now = time.Now()
+			time.Sleep(100 * time.Millisecond)
+		}
+	}()
+}

internal/size/size.go

@@ -0,0 +1,64 @@
+package size
+
+import (
+	"fmt"
+	"regexp"
+	"strconv"
+)
+
+// Size represents size that implements flag.Var
+type Size int64
+
+// the following regexes follow Go semantics https://golang.org/ref/spec#Letters_and_digits
+var (
+	rB  = regexp.MustCompile(`(?i)^(?:0b|0x|0o)?[\da-f_]+$`)
+	rKB = regexp.MustCompile(`(?i)^(?:0b|0x|0o)?[\da-f_]+kb$`)
+	rMB = regexp.MustCompile(`(?i)^(?:0b|0x|0o)?[\da-f_]+mb$`)
+	rGB = regexp.MustCompile(`(?i)^(?:0b|0x|0o)?[\da-f_]+gb$`)
+	rTB = regexp.MustCompile(`(?i)^(?:0b|0x|0o)?[\da-f_]+tb$`)
+)
+
+// Set parses size to integer from different bases and data units
+func (siz *Size) Set(size string) (err error) {
+	if size == "" {
+		return
+	}
+	const (
+		_ = 1 << (iota * 10)
+		KB
+		MB
+		GB
+		TB
+	)
+
+	var (
+		lmt = len(size) - 2
+		s   = []byte(size)
+	)
+
+	var _len int64
+	switch {
+	case rB.Match(s):
+		_len, err = strconv.ParseInt(size, 0, 64)
+	case rKB.Match(s):
+		_len, err = strconv.ParseInt(size[:lmt], 0, 64)
+		_len *= KB
+	case rMB.Match(s):
+		_len, err = strconv.ParseInt(size[:lmt], 0, 64)
+		_len *= MB
+	case rGB.Match(s):
+		_len, err = strconv.ParseInt(size[:lmt], 0, 64)
+		_len *= GB
+	case rTB.Match(s):
+		_len, err = strconv.ParseInt(size[:lmt], 0, 64)
+		_len *= TB
+	default:
+		return fmt.Errorf("invalid _len %q", size)
+	}
+	*siz = Size(_len)
+	return
+}
+
+func (siz *Size) String() string {
+	return fmt.Sprintf("%d", *siz)
+}

internal/size/size_test.go

@@ -0,0 +1,29 @@
+package size
+
+import "testing"
+
+func TestParseDataUnit(t *testing.T) {
+	var d = map[string]int{
+		"42mb":                 42 << 20,
+		"4_2":                  42,
+		"00":                   0,
+		"0":                    0,
+		"0_600tb":              384 << 40,
+		"0600Tb":               384 << 40,
+		"0o12Mb":               10 << 20,
+		"0b_10010001111_1kb":   2335 << 10,
+		"1024":                 1 << 10,
+		"0b111":                7,
+		"0x12gB":               18 << 30,
+		"0x_67_7a_2f_cc_40_c6": 113774485586118,
+		"121562380192901":      121562380192901,
+	}
+	var buf Size
+	var err error
+	for k, v := range d {
+		err = buf.Set(k)
+		if err != nil || buf != Size(v) {
+			t.Errorf("Error parsing %s: %v", k, err)
+		}
+	}
+}

internal/tcp/doc.go

@@ -0,0 +1,25 @@
+/*
+Package tcp implements TCP transport layer protocol, it is responsible for
+parsing, reassembling tcp packets, handling communication with engine listeners(github.com/buger/goreplay/capture),
+and reporting errors and statistics of packets.
+the packets are parsed by following TCP way(https://en.wikipedia.org/wiki/Transmission_Control_Protocol#TCP_segment_structure).
+
+example:
+
+import "github.com/buger/goreplay/tcp"
+
+messageExpire := time.Second*5
+maxSize := 5 << 20
+
+debugger := func(debugLevel int, data ...interface{}){} // debugger can also be nil
+messageHandler := func(mssg *tcp.Message){}
+
+mssgPool := tcp.NewMessageParser(maxMessageSize, messageExpire, debugger, messageHandler)
+listener.Listen(ctx, mssgPool.Handler)
+
+you can use pool.End or/and pool.Start to set custom session behaviors
+
+debugLevel in debugger function indicates the priority of the logs, the bigger the number the lower
+the priority. errors are signified by debug level 4 for errors, 5 for discarded packets, and 6 for received packets.
+*/
+package tcp // import github.com/buger/goreplay/tcp

internal/tcp/tcp_message.go

@@ -0,0 +1,443 @@
+package tcp
+
+import (
+	"encoding/binary"
+	"encoding/hex"
+	"fmt"
+	"github.com/buger/goreplay/proto"
+	"net"
+	"reflect"
+	"sort"
+	"time"
+	"unsafe"
+)
+
+// TCPProtocol is a number to indicate type of protocol
+type TCPProtocol uint8
+
+const (
+	// ProtocolHTTP ...
+	ProtocolHTTP TCPProtocol = iota
+	// ProtocolBinary ...
+	ProtocolBinary
+)
+
+// Set is here so that TCPProtocol can implement flag.Var
+func (protocol *TCPProtocol) Set(v string) error {
+	switch v {
+	case "", "http":
+		*protocol = ProtocolHTTP
+	case "binary":
+		*protocol = ProtocolBinary
+	default:
+		return fmt.Errorf("unsupported protocol %s", v)
+	}
+	return nil
+}
+
+func (protocol *TCPProtocol) String() string {
+	switch *protocol {
+	case ProtocolBinary:
+		return "binary"
+	case ProtocolHTTP:
+		return "http"
+	default:
+		return ""
+	}
+}
+
+// Stats every message carry its own stats object
+type Stats struct {
+	LostData  int
+	Length    int       // length of the data
+	Start     time.Time // first packet's timestamp
+	End       time.Time // last packet's timestamp
+	SrcAddr   string
+	DstAddr   string
+	Direction Dir
+	TimedOut  bool // timeout before getting the whole message
+	Truncated bool // last packet truncated due to max message size
+	IPversion byte
+}
+
+// Message is the representation of a tcp message
+type Message struct {
+	packets          []*Packet
+	parser           *MessageParser
+	feedback         interface{}
+	continueAdjusted bool
+	Stats
+}
+
+// UUID returns the UUID of a TCP request and its response.
+func (m *Message) UUID() []byte {
+	var streamID uint64
+	pckt := m.packets[0]
+
+	// check if response or request have generated the ID before.
+	if m.Direction == DirIncoming {
+		streamID = uint64(pckt.SrcPort)<<48 | uint64(pckt.DstPort)<<32 |
+			uint64(ip2int(pckt.SrcIP))
+	} else {
+		streamID = uint64(pckt.DstPort)<<48 | uint64(pckt.SrcPort)<<32 |
+			uint64(ip2int(pckt.DstIP))
+	}
+
+	id := make([]byte, 12)
+	binary.BigEndian.PutUint64(id, streamID)
+
+	if m.Direction == DirIncoming {
+		binary.BigEndian.PutUint32(id[8:], pckt.Ack)
+	} else {
+		binary.BigEndian.PutUint32(id[8:], pckt.Seq)
+	}
+
+	uuidHex := make([]byte, 24)
+	hex.Encode(uuidHex[:], id[:])
+
+	return uuidHex
+}
+
+func (m *Message) add(packet *Packet) bool {
+	// Skip duplicates
+	for _, p := range m.packets {
+		if p.Seq == packet.Seq {
+			return false
+		}
+	}
+
+	// Packets not always captured in same Seq order, and sometimes we need to prepend
+	if len(m.packets) == 0 || packet.Seq > m.packets[len(m.packets)-1].Seq {
+		m.packets = append(m.packets, packet)
+	} else if packet.Seq < m.packets[0].Seq {
+		m.packets = append([]*Packet{packet}, m.packets...)
+	} else { // insert somewhere in the middle...
+		for i, p := range m.packets {
+			if packet.Seq < p.Seq {
+				m.packets = append(m.packets[:i], append([]*Packet{packet}, m.packets[i:]...)...)
+				break
+			}
+		}
+	}
+
+	m.Length += len(packet.Payload)
+	m.LostData += int(packet.Lost)
+
+	if packet.Timestamp.After(m.End) || m.End.IsZero() {
+		m.End = packet.Timestamp
+	}
+
+	return true
+}
+
+// Packets returns packets of the message
+func (m *Message) Packets() []*Packet {
+	return m.packets
+}
+
+func (m *Message) MissingChunk() bool {
+	nextSeq := m.packets[0].Seq
+
+	for _, p := range m.packets {
+		if p.Seq != nextSeq {
+			return true
+		}
+
+		nextSeq += uint32(len(p.Payload))
+	}
+
+	return false
+}
+
+func (m *Message) PacketData() [][]byte {
+	tmp := make([][]byte, len(m.packets))
+
+	for i, p := range m.packets {
+		tmp[i] = p.Payload
+	}
+
+	return tmp
+}
+
+// Data returns data in this message
+func (m *Message) Data() []byte {
+	packetData := m.PacketData()
+	tmp := packetData[0]
+
+	if len(packetData) > 0 {
+		tmp, _ = copySlice(tmp, len(packetData[0]), packetData[1:]...)
+	}
+
+	// Remove Expect header, since its replay not fully supported
+	if state, ok := m.feedback.(*proto.HTTPState); ok {
+		if state.Continue100 {
+			tmp = proto.DeleteHeader(tmp, []byte("Expect"))
+		}
+	}
+
+	return tmp
+}
+
+// SetProtocolState set feedback/data that can be used later, e.g with End or Start hint
+func (m *Message) SetProtocolState(feedback interface{}) {
+	m.feedback = feedback
+}
+
+// ProtocolState returns feedback associated to this message
+func (m *Message) ProtocolState() interface{} {
+	return m.feedback
+}
+
+// Sort a helper to sort packets
+func (m *Message) Sort() {
+	sort.SliceStable(m.packets, func(i, j int) bool { return m.packets[i].Seq < m.packets[j].Seq })
+}
+
+// Emitter message handler
+type Emitter func(*Message)
+
+// HintEnd hints the parser to stop the session, see MessageParser.End
+// when set, it will be executed before checking FIN or RST flag
+type HintEnd func(*Message) bool
+
+// HintStart hints the parser to start the reassembling the message, see MessageParser.Start
+// when set, it will be called after checking SYN flag
+type HintStart func(*Packet) (IsRequest, IsOutgoing bool)
+
+// MessageParser holds data of all tcp messages in progress(still receiving/sending packets).
+// message is identified by its source port and dst port, and last 4bytes of src IP.
+type MessageParser struct {
+	m map[uint64]*Message
+
+	messageExpire  time.Duration // the maximum time to wait for the final packet, minimum is 100ms
+	allowIncompete bool
+	End            HintEnd
+	Start          HintStart
+	ticker         *time.Ticker
+	messages       chan *Message
+	packets        chan *PcapPacket
+	close          chan struct{} // to signal that we are able to close
+	ports          []uint16
+	ips            []net.IP
+}
+
+// NewMessageParser returns a new instance of message parser
+func NewMessageParser(messages chan *Message, ports []uint16, ips []net.IP, messageExpire time.Duration, allowIncompete bool) (parser *MessageParser) {
+	parser = new(MessageParser)
+
+	parser.messageExpire = messageExpire
+	if parser.messageExpire == 0 {
+		parser.messageExpire = time.Millisecond * 1000
+	}
+
+	parser.allowIncompete = allowIncompete
+
+	parser.packets = make(chan *PcapPacket, 10000)
+
+	if messages == nil {
+		messages = make(chan *Message, 1000)
+	}
+	parser.messages = messages
+
+	parser.m = make(map[uint64]*Message)
+	parser.ticker = time.NewTicker(time.Millisecond * 100)
+	parser.close = make(chan struct{}, 1)
+
+	parser.ports = ports
+	parser.ips = ips
+
+	go parser.wait()
+	return parser
+}
+
+var packetLen int
+
+// Packet returns packet handler
+func (parser *MessageParser) PacketHandler(packet *PcapPacket) {
+	packetLen++
+	parser.packets <- packet
+}
+
+func (parser *MessageParser) wait() {
+	var (
+		now time.Time
+	)
+	for {
+		select {
+		case pckt := <-parser.packets:
+			parser.processPacket(parser.parsePacket(pckt))
+		case now = <-parser.ticker.C:
+			parser.timer(now)
+		case <-parser.close:
+			parser.ticker.Stop()
+			// parser.Close should wait for this function to return
+			parser.close <- struct{}{}
+			return
+			// default:
+		}
+	}
+}
+
+func (parser *MessageParser) parsePacket(pcapPkt *PcapPacket) *Packet {
+	pckt, err := ParsePacket(pcapPkt.Data, pcapPkt.LType, pcapPkt.LTypeLen, pcapPkt.Ci, false)
+	if err != nil {
+		if _, empty := err.(EmptyPacket); !empty {
+			stats.Add("packet_error", 1)
+		}
+		return nil
+	}
+
+	for _, p := range parser.ports {
+		if pckt.DstPort == p && containsOrEmpty(pckt.DstIP, parser.ips) {
+			pckt.Direction = DirIncoming
+			break
+		} else if pckt.SrcPort == p && containsOrEmpty(pckt.SrcIP, parser.ips) {
+			pckt.Direction = DirOutcoming
+			break
+		}
+	}
+
+	return pckt
+}
+
+func containsOrEmpty(element net.IP, ipList []net.IP) bool {
+	if len(ipList) == 0 {
+		return true
+	}
+	for _, ip := range ipList {
+		if ip.Equal(element) {
+			return true
+		}
+	}
+	return false
+}
+
+func (parser *MessageParser) processPacket(pckt *Packet) {
+	if pckt == nil {
+		return
+	}
+
+	// Trying to build unique hash, but there is small chance of collision
+	// No matter if it is request or response, all packets in the same message have same
+	m, ok := parser.m[pckt.MessageID()]
+	switch {
+	case ok:
+		if m.Direction == DirUnknown {
+			if in, out := parser.Start(pckt); in || out {
+				if in {
+					m.Direction = DirIncoming
+				} else {
+					m.Direction = DirOutcoming
+				}
+			}
+		}
+		parser.addPacket(m, pckt)
+		return
+	case pckt.Direction == DirUnknown && parser.Start != nil:
+		if in, out := parser.Start(pckt); in || out {
+			if in {
+				pckt.Direction = DirIncoming
+			} else {
+				pckt.Direction = DirOutcoming
+			}
+		}
+	}
+
+	m = new(Message)
+	m.Direction = pckt.Direction
+	m.SrcAddr = pckt.SrcIP.String()
+	m.DstAddr = pckt.DstIP.String()
+
+	parser.m[pckt.MessageID()] = m
+
+	m.Start = pckt.Timestamp
+	m.parser = parser
+	parser.addPacket(m, pckt)
+}
+
+func (parser *MessageParser) addPacket(m *Message, pckt *Packet) bool {
+	if !m.add(pckt) {
+		return false
+	}
+
+	// If we are using protocol parsing, like HTTP, depend on its parsing func.
+	// For the binary procols wait for message to expire
+	if parser.End != nil {
+		if parser.End(m) {
+			parser.Emit(m)
+			return true
+		}
+
+		parser.Fix100Continue(m)
+	}
+
+	return true
+}
+
+func (parser *MessageParser) Fix100Continue(m *Message) {
+	// Only adjust a message once
+	if state, ok := m.feedback.(*proto.HTTPState); ok && state.Continue100 && !m.continueAdjusted {
+		// Shift Ack by given offset
+		// Size of "HTTP/1.1 100 Continue\r\n\r\n" message
+		for _, p := range m.packets {
+			p.messageID = 0
+			p.Ack += 25
+		}
+
+		// If next section was aready approved and received, merge messages
+		if next, found := parser.m[m.packets[0].MessageID()]; found {
+			for _, p := range next.packets {
+				parser.addPacket(m, p)
+			}
+		}
+
+		// Re-add (or override) again with new message and ID
+		parser.m[m.packets[0].MessageID()] = m
+		m.continueAdjusted = true
+	}
+}
+
+func (parser *MessageParser) Read() *Message {
+	m := <-parser.messages
+	return m
+}
+
+func (parser *MessageParser) Emit(m *Message) {
+	stats.Add("message_count", 1)
+
+	delete(parser.m, m.packets[0].MessageID())
+
+	parser.messages <- m
+}
+
+func GetUnexportedField(field reflect.Value) interface{} {
+	return reflect.NewAt(field.Type(), unsafe.Pointer(field.UnsafeAddr())).Elem().Interface()
+}
+
+var failMsg int
+
+func (parser *MessageParser) timer(now time.Time) {
+	packetLen = 0
+
+	packetQueueLen.Set(int64(len(parser.packets)))
+	messageQueueLen.Set(int64(len(parser.m)))
+
+	for _, m := range parser.m {
+		if now.Sub(m.End) > parser.messageExpire {
+			m.TimedOut = true
+			stats.Add("message_timeout_count", 1)
+			failMsg++
+			if parser.End == nil || parser.allowIncompete {
+				parser.Emit(m)
+			}
+
+			delete(parser.m, m.packets[0].MessageID())
+		}
+	}
+}
+
+func (parser *MessageParser) Close() error {
+	parser.close <- struct{}{}
+	<-parser.close // wait for timer to be closed!
+	return nil
+}

internal/tcp/tcp_packet.go

@@ -0,0 +1,284 @@
+package tcp
+
+import (
+	"encoding/binary"
+	"expvar"
+	"fmt"
+	"net"
+	"time"
+
+	"github.com/google/gopacket"
+)
+
+func copySlice(to []byte, skip int, from ...[]byte) ([]byte, int) {
+	var totalLen int
+	for _, s := range from {
+		totalLen += len(s)
+	}
+	totalLen += skip
+
+	if len(to) < totalLen {
+		diff := totalLen - len(to)
+		to = append(to, make([]byte, diff)...)
+	}
+
+	for _, s := range from {
+		skip += copy(to[skip:], s)
+	}
+
+	return to, skip
+}
+
+var stats *expvar.Map
+var packetQueueLen, messageQueueLen *expvar.Int
+
+func init() {
+	packetQueueLen = new(expvar.Int)
+	messageQueueLen = new(expvar.Int)
+
+	stats = expvar.NewMap("tcp")
+	stats.Init()
+	stats.Set("packet_queue", packetQueueLen)
+	stats.Set("message_queue", messageQueueLen)
+}
+
+type Dir int
+
+const (
+	DirUnknown = iota
+	DirIncoming
+	DirOutcoming
+)
+
+/*
+Packet represent data and layers of packet.
+parser extracts information from pcap Packet. functions of *Packet doesn't validate if packet is nil,
+calllers must make sure that ParsePacket has'nt returned any error before calling any other
+function.
+*/
+type Packet struct {
+	Direction          Dir
+	messageID          uint64
+	SrcIP, DstIP       net.IP
+	Version            uint8
+	SrcPort, DstPort   uint16
+	Ack, Seq           uint32
+	ACK, SYN, FIN, RST bool
+	Lost               uint32
+	Retry              int
+	CaptureLength      int
+	Timestamp          time.Time
+	Payload            []byte
+	buf                []byte
+
+	created time.Time
+	gc      bool
+}
+
+type PcapPacket struct {
+	Data     []byte
+	LType    int
+	LTypeLen int
+	Ci       *gopacket.CaptureInfo
+}
+
+// ParsePacket parse raw packets
+func ParsePacket(data []byte, lType, lTypeLen int, ci *gopacket.CaptureInfo, allowEmpty bool) (pckt *Packet, err error) {
+	pckt = new(Packet)
+	if err := pckt.parse(data, lType, lTypeLen, ci, allowEmpty); err != nil {
+		return nil, err
+	}
+
+	return pckt, nil
+}
+
+func (pckt *Packet) parse(data []byte, lType, lTypeLen int, cp *gopacket.CaptureInfo, allowEmpty bool) error {
+	pckt.Retry = 0
+	pckt.messageID = 0
+	pckt.buf = pckt.buf[:]
+
+	// TODO: check resolution
+	pckt.Timestamp = cp.Timestamp
+
+	if len(data) < lTypeLen {
+		return ErrHdrLength("Link")
+	}
+	if len(data) <= lTypeLen {
+		return ErrHdrMissing("IPv4 or IPv6")
+	}
+
+	ldata := data[lTypeLen:]
+	var proto byte
+	var netLayer, transLayer []byte
+
+	if ldata[0]>>4 == 4 {
+		// IPv4 header
+		if len(ldata) < 20 {
+			return ErrHdrLength("IPv4")
+		}
+		proto = ldata[9]
+		ihl := int(ldata[0]&0x0F) * 4
+		if ihl < 20 {
+			return ErrHdrInvalid("IPv4's IHL")
+		}
+		if len(ldata) < ihl {
+			return ErrHdrLength("IPv4 opts")
+		}
+		netLayer = ldata[:ihl]
+	} else if ldata[0]>>4 == 6 {
+		if len(ldata) < 40 {
+			return ErrHdrLength("IPv6")
+		}
+		proto = ldata[6]
+		totalLen := 40
+		for ipv6ExtensionHdr(proto) {
+			hdr := len(ldata) - totalLen
+			if hdr < 8 {
+				return ErrHdrExpected("IPv6 opts")
+			}
+			extLen := 8
+			if proto != 44 {
+				extLen = int(ldata[totalLen+1]+1) * 8
+			}
+			if hdr < extLen {
+				return ErrHdrLength("IPv6 opts")
+			}
+			proto = ldata[totalLen]
+			totalLen += extLen
+		}
+		netLayer = ldata[:totalLen]
+	} else {
+		return ErrHdrExpected("IPv4 or IPv6")
+	}
+	if proto != 6 {
+		return ErrHdrExpected("TCP")
+	}
+	if len(data) <= len(netLayer) {
+		return ErrHdrMissing("TCP")
+	}
+	ndata := ldata[len(netLayer):]
+	// TCP header
+	if len(ndata) < 20 {
+		return ErrHdrLength("TCP")
+	}
+	dOf := int(ndata[12]>>4) * 4
+	if dOf < 20 {
+		return ErrHdrInvalid("TCP's ndata offset")
+	}
+	if len(ndata) < dOf {
+		return ErrHdrLength("TCP opts")
+	}
+
+	// There are case when packet have padding but dOf shows its not
+	empty := true
+	for i := 0; i < len(ndata[dOf:]); i++ {
+		if ndata[dOf:][i] != 0 {
+			empty = false
+			break
+		}
+	}
+
+	if !allowEmpty && empty {
+		return EmptyPacket("")
+	}
+
+	if (netLayer[0] >> 4) == 4 {
+		// IPv4 header
+		pckt.Version = 4
+		pckt.SrcIP = netLayer[12:16]
+		pckt.DstIP = netLayer[16:20]
+	} else {
+		// IPv6 header
+		pckt.Version = 6
+		pckt.SrcIP = netLayer[8:24]
+		pckt.DstIP = netLayer[24:40]
+	}
+
+	transLayer = ndata[:dOf]
+
+	pckt.CaptureLength = cp.CaptureLength
+	pckt.SrcPort = binary.BigEndian.Uint16(transLayer[0:2])
+	pckt.DstPort = binary.BigEndian.Uint16(transLayer[2:4])
+	pckt.Seq = binary.BigEndian.Uint32(transLayer[4:8])
+	pckt.Ack = binary.BigEndian.Uint32(transLayer[8:12])
+	pckt.FIN = transLayer[13]&0x01 != 0
+	pckt.SYN = transLayer[13]&0x02 != 0
+	pckt.RST = transLayer[13]&0x04 != 0
+	pckt.ACK = transLayer[13]&0x10 != 0
+	pckt.Lost = uint32(cp.Length - cp.CaptureLength)
+
+	pckt.Payload = ndata[dOf:]
+
+	return nil
+}
+
+func (pckt *Packet) MessageID() uint64 {
+	if pckt.messageID == 0 {
+		// All packets in the same message will share the same ID
+		pckt.messageID = uint64(pckt.SrcPort)<<48 | uint64(pckt.DstPort)<<32 |
+			(uint64(ip2int(pckt.SrcIP)) + uint64(ip2int(pckt.DstIP)) + uint64(pckt.Ack))
+	}
+
+	return pckt.messageID
+}
+
+// Src returns the source socket of a packet
+func (pckt *Packet) Src() string {
+	return fmt.Sprintf("%s:%d", pckt.SrcIP, pckt.SrcPort)
+}
+
+// Dst returns destination socket
+func (pckt *Packet) Dst() string {
+	return fmt.Sprintf("%s:%d", pckt.DstIP, pckt.DstPort)
+}
+
+type EmptyPacket string
+
+func (err EmptyPacket) Error() string {
+	return "Empty packet"
+}
+
+// ErrHdrLength returned on short header length
+type ErrHdrLength string
+
+func (err ErrHdrLength) Error() string {
+	return "short " + string(err) + " length"
+}
+
+// ErrHdrMissing returned on missing header(s)
+type ErrHdrMissing string
+
+func (err ErrHdrMissing) Error() string {
+	return "missing " + string(err) + " header(s)"
+}
+
+// ErrHdrExpected returned when header(s) are different from the one expected
+type ErrHdrExpected string
+
+func (err ErrHdrExpected) Error() string {
+	return "expected " + string(err) + " header(s)"
+}
+
+// ErrHdrInvalid returned when header(s) are different from the one expected
+type ErrHdrInvalid string
+
+func (err ErrHdrInvalid) Error() string {
+	return "invalid " + string(err) + " value"
+}
+
+// https://en.wikipedia.org/wiki/IPv6_packet#Extension_headers
+func ipv6ExtensionHdr(b byte) bool {
+	// TODO: support all extension headers
+	return b == 0 || b == 43 || b == 44
+}
+
+func ip2int(ip net.IP) uint32 {
+	if len(ip) == 0 {
+		return 0
+	}
+
+	if len(ip) == 16 {
+		return binary.BigEndian.Uint32(ip[12:16])
+	}
+	return binary.BigEndian.Uint32(ip)
+}

internal/tcp/tcp_test.go

@@ -0,0 +1,315 @@
+package tcp
+
+import (
+	"bytes"
+	"encoding/binary"
+	"github.com/buger/goreplay/proto"
+
+	// "runtime"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+
+	"github.com/google/gopacket"
+	"github.com/google/gopacket/layers"
+)
+
+func generateHeader(request bool, seq uint32, length uint16) []byte {
+	hdr := make([]byte, 4+24+24)
+	binary.BigEndian.PutUint32(hdr, uint32(layers.ProtocolFamilyIPv4))
+
+	ip := hdr[4:]
+	ip[0] = 4<<4 | 6
+	binary.BigEndian.PutUint16(ip[2:4], length+24+24)
+	ip[9] = uint8(layers.IPProtocolTCP)
+	copy(ip[12:16], []byte{127, 0, 0, 1})
+	copy(ip[16:], []byte{127, 0, 0, 1})
+
+	// set tcp header
+	tcp := ip[24:]
+	tcp[12] = 6 << 4
+
+	if request {
+		binary.BigEndian.PutUint16(tcp, 5535)
+		binary.BigEndian.PutUint16(tcp[2:], 8000)
+	} else {
+		binary.BigEndian.PutUint16(tcp, 8000)
+		binary.BigEndian.PutUint16(tcp[2:], 5535)
+	}
+	binary.BigEndian.PutUint32(tcp[4:], seq)
+	return hdr
+}
+
+func GetPackets(request bool, start uint32, _len int, payload []byte) []*Packet {
+	var packets = make([]*Packet, _len)
+	var err error
+	for i := start; i < start+uint32(_len); i++ {
+		d := append(generateHeader(request, i, uint16(len(payload))), payload...)
+		ci := &gopacket.CaptureInfo{Length: len(d), CaptureLength: len(d), Timestamp: time.Now()}
+
+		packets[i-start], err = ParsePacket(d, int(layers.LinkTypeLoop), 4, ci, true)
+		if request {
+			packets[i-start].Direction = DirIncoming
+		} else {
+			packets[i-start].Direction = DirOutcoming
+		}
+		if err != nil {
+			panic(err)
+		}
+	}
+	return packets
+}
+
+func TestRequestResponseMapping(t *testing.T) {
+	packets := []*Packet{
+		{SrcPort: 60000, DstPort: 80, Ack: 1, Seq: 1, Direction: DirIncoming, Timestamp: time.Unix(1, 0), Payload: []byte("GET / HTTP/1.1\r\n")},
+		{SrcPort: 60000, DstPort: 80, Ack: 1, Seq: 17, Direction: DirIncoming, Timestamp: time.Unix(2, 0), Payload: []byte("Host: localhost\r\n\r\n")},
+
+		// Seq of first response packet match Ack of first request packet
+		{SrcPort: 80, DstPort: 60000, Ack: 36, Seq: 1, Direction: DirOutcoming, Timestamp: time.Unix(3, 0), Payload: []byte("HTTP/1.1 200 OK\r\n")},
+		{SrcPort: 80, DstPort: 60000, Ack: 36, Seq: 18, Direction: DirOutcoming, Timestamp: time.Unix(4, 0), Payload: []byte("Content-Length: 0\r\n\r\n")},
+
+		// Same TCP stream
+		{SrcPort: 60000, DstPort: 80, Ack: 39, Seq: 36, Direction: DirIncoming, Timestamp: time.Unix(5, 0), Payload: []byte("GET / HTTP/1.1\r\n")},
+		{SrcPort: 60000, DstPort: 80, Ack: 39, Seq: 52, Direction: DirIncoming, Timestamp: time.Unix(6, 0), Payload: []byte("Host: localhost\r\n\r\n")},
+
+		// Seq of first response packet match Ack of first request packet
+		{SrcPort: 80, DstPort: 60000, Ack: 71, Seq: 39, Direction: DirOutcoming, Timestamp: time.Unix(7, 0), Payload: []byte("HTTP/1.1 200 OK\r\n")},
+		{SrcPort: 80, DstPort: 60000, Ack: 71, Seq: 56, Direction: DirOutcoming, Timestamp: time.Unix(8, 0), Payload: []byte("Content-Length: 0\r\n\r\n")},
+	}
+
+	parser := NewMessageParser(nil, nil, nil, time.Second, false)
+	parser.Start = func(pckt *Packet) (bool, bool) {
+		return proto.HasRequestTitle(pckt.Payload), proto.HasResponseTitle(pckt.Payload)
+	}
+	parser.End = func(m *Message) bool {
+		return proto.HasFullPayload(m, m.PacketData()...)
+	}
+
+	for _, packet := range packets {
+		parser.processPacket(packet)
+	}
+
+	messages := []*Message{}
+	for i := 0; i < 4; i++ {
+		m := parser.Read()
+		messages = append(messages, m)
+	}
+
+	assert.Equal(t, int(messages[0].Direction), int(DirIncoming))
+	assert.Equal(t, int(messages[1].Direction), int(DirOutcoming))
+	assert.Equal(t, int(messages[2].Direction), int(DirIncoming))
+	assert.Equal(t, int(messages[3].Direction), int(DirOutcoming))
+
+	assert.Equal(t, messages[0].UUID(), messages[1].UUID())
+	assert.Equal(t, messages[2].UUID(), messages[3].UUID())
+
+	assert.NotEqual(t, messages[0].UUID(), messages[2].UUID())
+}
+
+func TestMessageParserWithHint(t *testing.T) {
+	parser := NewMessageParser(nil, nil, nil, time.Second, false)
+	parser.Start = func(pckt *Packet) (bool, bool) {
+		return proto.HasRequestTitle(pckt.Payload), proto.HasResponseTitle(pckt.Payload)
+	}
+	parser.End = func(m *Message) bool {
+		return proto.HasFullPayload(m, m.PacketData()...)
+	}
+
+	packets := []*Packet{
+		// Seq of first response packet match Ack of first request packet
+		{SrcPort: 80, DstPort: 60000, Ack: 1, Seq: 1, Direction: DirOutcoming, Timestamp: time.Unix(1, 0), Payload: []byte("HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nTransfer-Encoding: chunked\r\n\r\n7\r\n")},
+		{SrcPort: 80, DstPort: 60000, Ack: 1, Seq: 18, Direction: DirOutcoming, Timestamp: time.Unix(2, 0), Payload: []byte("\r\nMozilla\r\n9\r\nDeveloper\r")},
+		{SrcPort: 80, DstPort: 60000, Ack: 1, Seq: 42, Direction: DirOutcoming, Timestamp: time.Unix(3, 0), Payload: []byte("\n7\r\nNetwork\r\n0\r\n\r\n")},
+
+		{SrcPort: 60000, DstPort: 80, Ack: 60, Seq: 1, Direction: DirIncoming, Timestamp: time.Unix(4, 0), Payload: []byte("POST / HTTP/1.1\r\nContent-Type: text/plain\r\nContent-Length: 23\r\n\r\n")},
+		{SrcPort: 60000, DstPort: 80, Ack: 60, Seq: 66, Direction: DirIncoming, Timestamp: time.Unix(5, 0), Payload: []byte("MozillaDeveloper")},
+		{SrcPort: 60000, DstPort: 80, Ack: 60, Seq: 82, Direction: DirIncoming, Timestamp: time.Unix(6, 0), Payload: []byte("Network")},
+
+		{SrcPort: 80, DstPort: 60000, Ack: 89, Seq: 1, Direction: DirOutcoming, Timestamp: time.Unix(7, 0), Payload: []byte("HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nContent-Length: 0\r\n\r\n")},
+	}
+
+	for _, p := range packets {
+		parser.processPacket(p)
+	}
+
+	messages := []*Message{}
+	for i := 0; i < 3; i++ {
+		m := parser.Read()
+		messages = append(messages, m)
+	}
+
+	if !bytes.HasSuffix(messages[0].Data(), []byte("\n7\r\nNetwork\r\n0\r\n\r\n")) {
+		t.Errorf("expected to %q to have suffix %q", messages[0].Data(), []byte("\n7\r\nNetwork\r\n0\r\n\r\n"))
+	}
+
+	if !bytes.HasSuffix(messages[1].Data(), []byte("Network")) {
+		t.Errorf("expected to %q to have suffix %q", messages[1].Data(), []byte("Network"))
+	}
+
+	if !bytes.HasSuffix(messages[2].Data(), []byte("Content-Length: 0\r\n\r\n")) {
+		t.Errorf("expected to %q to have suffix %q", messages[2].Data(), []byte("Content-Length: 0\r\n\r\n"))
+	}
+}
+
+func TestMessageParserWrongOrder(t *testing.T) {
+	parser := NewMessageParser(nil, nil, nil, time.Second, false)
+	parser.Start = func(pckt *Packet) (bool, bool) {
+		return proto.HasRequestTitle(pckt.Payload), proto.HasResponseTitle(pckt.Payload)
+	}
+	parser.End = func(m *Message) bool {
+		return proto.HasFullPayload(m, m.PacketData()...)
+	}
+	packets := []*Packet{
+		// Seq of first response packet match Ack of first request packet
+		{SrcPort: 60000, DstPort: 80, Ack: 60, Seq: 66, Direction: DirIncoming, Timestamp: time.Unix(5, 0), Payload: []byte("MozillaDeveloper")},
+		{SrcPort: 80, DstPort: 60000, Ack: 1, Seq: 1, Direction: DirOutcoming, Timestamp: time.Unix(1, 0), Payload: []byte("HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nTransfer-Encoding: chunked\r\n\r\n7\r\n")},
+		{SrcPort: 80, DstPort: 60000, Ack: 1, Seq: 42, Direction: DirOutcoming, Timestamp: time.Unix(3, 0), Payload: []byte("\n7\r\nNetwork\r\n0\r\n\r\n")},
+
+		{SrcPort: 60000, DstPort: 80, Ack: 60, Seq: 1, Direction: DirIncoming, Timestamp: time.Unix(4, 0), Payload: []byte("POST / HTTP/1.1\r\nContent-Type: text/plain\r\nContent-Length: 23\r\n\r\n")},
+		{SrcPort: 80, DstPort: 60000, Ack: 1, Seq: 18, Direction: DirOutcoming, Timestamp: time.Unix(2, 0), Payload: []byte("\r\nMozilla\r\n9\r\nDeveloper\r")},
+
+		{SrcPort: 80, DstPort: 60000, Ack: 89, Seq: 1, Direction: DirOutcoming, Timestamp: time.Unix(7, 0), Payload: []byte("HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nContent-Length: 0\r\n\r\n")},
+		{SrcPort: 60000, DstPort: 80, Ack: 60, Seq: 82, Direction: DirIncoming, Timestamp: time.Unix(6, 0), Payload: []byte("Network")},
+	}
+
+	for _, p := range packets {
+		parser.processPacket(p)
+	}
+
+	m := parser.Read()
+
+	if !bytes.HasSuffix(m.Data(), []byte("\n7\r\nNetwork\r\n0\r\n\r\n")) {
+		t.Errorf("expected to %q to have suffix %q", m.Data(), []byte("\n7\r\nNetwork\r\n0\r\n\r\n"))
+	}
+
+	m = parser.Read()
+
+	if !bytes.HasSuffix(m.Data(), []byte("Content-Length: 0\r\n\r\n")) {
+		t.Errorf("expected to %q to have suffix %q", m.Data(), []byte("Content-Length: 0\r\n\r\n"))
+	}
+
+	m = parser.Read()
+
+	if !bytes.HasSuffix(m.Data(), []byte("Network")) {
+		t.Errorf("expected to %q to have suffix %q", m.Data(), []byte("Network"))
+	}
+}
+
+func TestMessageParserWithoutHint(t *testing.T) {
+	var data [63 << 10]byte
+	packets := GetPackets(true, 1, 10, data[:])
+
+	p := NewMessageParser(nil, nil, nil, time.Second, false)
+	for _, v := range packets {
+		p.processPacket(v)
+	}
+	m := p.Read()
+
+	if m.Length != 63<<10*10 {
+		t.Errorf("expected %d to equal %d", m.Length, 63<<10*10)
+	}
+}
+
+func TestMessageTimeoutReached(t *testing.T) {
+	const size = 63 << 11
+	var data [size >> 1]byte
+	packets := GetPackets(true, 1, 2, data[:])
+	p := NewMessageParser(nil, nil, nil, 100*time.Millisecond, true)
+	p.processPacket(packets[0])
+
+	time.Sleep(time.Millisecond * 20)
+
+	p.processPacket(packets[1])
+	m := p.Read()
+	if m.Length != size {
+		t.Errorf("expected %d to equal %d", m.Length, size)
+	}
+	if !m.TimedOut {
+		t.Error("expected message to be timeout")
+	}
+}
+
+func BenchmarkMessageUUID(b *testing.B) {
+	packets := GetPackets(true, 1, 5, nil)
+
+	var uuid []byte
+	parser := NewMessageParser(nil, nil, nil, 10*time.Millisecond, true)
+	for _, p := range packets {
+		parser.processPacket(p)
+	}
+
+	msg := parser.Read()
+
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		uuid = msg.UUID()
+	}
+	_ = uuid
+}
+
+func BenchmarkPacketParseAndSort(b *testing.B) {
+	m := new(Message)
+	m.packets = make([]*Packet, 100)
+	for i, v := range GetPackets(true, 1, 100, nil) {
+		m.packets[i] = v
+	}
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		m.Sort()
+	}
+}
+
+func BenchmarkMessageParserWithoutHint(b *testing.B) {
+	var chunk = []byte("111111111111111111111111111111")
+	packets := GetPackets(true, 1, 1000, chunk)
+	p := NewMessageParser(nil, nil, nil, 2*time.Second, false)
+	b.ResetTimer()
+	b.ReportMetric(float64(1000), "packets/op")
+	for i := 0; i < b.N; i++ {
+		for _, v := range packets {
+			p.processPacket(v)
+		}
+		p.Read()
+	}
+}
+
+func BenchmarkMessageParserWithHint(b *testing.B) {
+	var buf [1002][]byte
+	var chunk = []byte("1e\r\n111111111111111111111111111111\r\n")
+	buf[0] = []byte("HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nTransfer-Encoding: chunked\r\n\r\n")
+	for i := 1; i < 1000; i++ {
+		buf[i] = chunk
+	}
+	buf[1001] = []byte("0\r\n\r\n")
+	packets := make([]*Packet, len(buf))
+	for i := 0; i < len(buf); i++ {
+		packets[i] = GetPackets(false, 1, 1, buf[i])[0]
+	}
+
+	parser := NewMessageParser(nil, nil, nil, 2*time.Second, false)
+	parser.Start = func(pckt *Packet) (bool, bool) {
+		return false, proto.HasResponseTitle(pckt.Payload)
+	}
+	parser.End = func(m *Message) bool {
+		return proto.HasFullPayload(m, m.PacketData()...)
+	}
+	b.ResetTimer()
+	b.ReportMetric(float64(len(packets)), "packets/op")
+	b.ReportMetric(float64(1000), "chunks/op")
+	for i := 0; i < b.N; i++ {
+		for j := range packets {
+			parser.processPacket(packets[j])
+		}
+		parser.Read()
+	}
+}
+
+func BenchmarkNewAndParsePacket(b *testing.B) {
+	data := append(generateHeader(true, 1024, 10), make([]byte, 10)...)
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		ParsePacket(data, int(layers.LinkTypeLoop), 4, &gopacket.CaptureInfo{}, true)
+	}
+}

k8s/README.md

@@ -0,0 +1,172 @@
+# Native k8s integration
+
+After following steps below, you will be able to capture traffic inside k8s like this:
+
+```
+gor --input-raw k8s://namespace/deployment/app:80 --output-http http://replay.com
+```
+
+GoReplay will running as a daemonset (e.g. on each phisical k8s node. 
+It will also require giving required permission to have read access to K8s APIs, so it can dynamically filter traffic for a specific pods.
+
+Supported format for filtering required pods:
+
+```
+k8s://[namespace/]pod/[pod_name] - k8s://default/pod/nginx-7848d4b86f-5nxz8
+k8s://[namespace/]deployment/[deployment_name] - k8s://default/deployment/nginx
+k8s://[namespace/]daemonset/[daemonset_name] - k8s://default/daemonset/nginx
+k8s://[namespace/]labelSelector/[selector] - k8s://default/labelSelector/app=nginx
+k8s://[namespace/]fieldSelector/[selector] - k8s://default/fieldSelector/metadata.name=nginx-7848d4b86f-5nxz8
+```
+`namespace` is optional, omit to use all namespaces: `k8s://labelSelector/app=replay`
+
+## 1. Create a namespace
+`kubectl create namespace goreplay`
+
+## 2. Create the Kubernetes service account in the namespace:
+
+`kubectl create serviceaccount goreplay --namespace goreplay`
+
+## 3. Create Cluster Role which gives read-only access to the pods:
+`kubectl -n goreplay -f clusterrole.yaml apply`
+
+```yaml
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: pod-reader
+rules:
+- apiGroups: [""]
+  resources: ["pods"]
+  verbs: ["get", "watch", "list"]
+- apiGroups: [""]
+  resources: ["deployments"]
+  verbs: ["get", "watch", "list"]
+- apiGroups: [""]
+  resources: ["daemonset"]
+  verbs: ["get", "watch", "list"]
+```
+
+## 4. Attach role to goreplay service account
+`kubectl -n goreplay -f rolebinding.yaml apply`
+
+```yaml
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: goreplay-reader-binding
+subjects:
+- kind: ServiceAccount
+  name: goreplay
+  namespace: goreplay
+roleRef:
+  kind: ClusterRole
+  name: pod-reader
+  apiGroup: ""
+```
+
+## 5. Start goreplay daemonset
+
+`kubectl -n goreplay -f goreplay.yaml apply`
+
+In arguments, specify which service you want to capture. 
+Following format supported:
+
+
+```yaml
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: goreplay-daemon
+spec:
+  selector:
+    matchLabels:
+      app: goreplay
+  template:
+    metadata:
+      labels:
+        app: goreplay
+    spec:
+      hostNetwork: true
+      serviceAccountName: goreplay
+      containers:
+      - name: goreplay
+        image: buger/goreplay:2.0.0-rc4
+        args:
+          - "--input-raw"
+          - "k8s://deployments/nginx:80"
+          - "--output-stdout"
+```
+
+## 6. Create a simple http service (Optionally)
+
+`kubectl -n default -f nginx.yaml apply`
+
+```yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: nginx
+  labels:
+    app: nginx
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: nginx
+  template:
+    metadata:
+      labels:
+        app: nginx
+    spec:
+      containers:
+      - name: nginx
+        image: nginx
+        ports:
+        - containerPort: 80
+   
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: ngnix-service
+spec:
+  selector:
+    app: nginx
+  type: NodePort
+  ports:
+  - protocol: TCP
+    port: 80
+    targetPort: 80
+
+```
+
+
+## 7. Verify installation, and debugging tips
+
+Find url for your service using `kubectl get svc` or `minikube service --url ngnix-service -n http`, and make a call to it.
+
+Get GoReplay logs, and check if it capture traffic of your service.
+`kubectl logs -n goreplay -l app=goreplay --all-containers`
+
+Describe daemonset:
+`kubectl describe daemonset goreplay-daemon -n goreplay`
+
+Get GoReplay pod list:
+`kubectl get pods -n goreplay -l app=goreplay`
+
+Get logs for specific pod (take data from previous step):
+`kubectl logs goreplay-daemon-<replace> -n goreplay`
+
+Get related k8s events:
+`kubectl get events -n goreplay --field-selector involvedObject.name=goreplay-daemon-<replace>`
+
+## 8. Debuggin telemetry
+
+We provde a special script which will get all require logs, and help you find installation errors. 
+
+`curl -s https://raw.githubusercontent.com/buger/goreplay/refs/heads/master/k8s/collect_goreplay_telemetry.sh | bash`
+
+If you are using microk8s or similar, you can also specific prefix for your kubectl command like this:
+
+`curl -s https://raw.githubusercontent.com/buger/goreplay/refs/heads/master/k8s/collect_goreplay_telemetry.sh | bash -s -- "microk8s kubectl"`

k8s/clusterrole.yaml

@@ -0,0 +1,14 @@
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: pod-reader
+rules:
+- apiGroups: [""]
+  resources: ["pods"]
+  verbs: ["get", "watch", "list"]
+- apiGroups: ["apps"]
+  resources: ["deployments"]
+  verbs: ["get", "watch", "list"]
+- apiGroups: ["apps"]
+  resources: ["daemonsets"]
+  verbs: ["get", "watch", "list"]

k8s/collect_goreplay_telemetry.sh

@@ -0,0 +1,123 @@
+#!/usr/bin/env bash
+#
+# collect_goreplay_telemetry.sh
+#
+# Gathers telemetry from a GoReplay DaemonSet in the 'goreplay' namespace.
+# Works on macOS and Linux, assuming 'kubectl' (or compatible) is installed.
+#
+# Usage examples:
+#   ./collect_goreplay_telemetry.sh
+#   ./collect_goreplay_telemetry.sh "microk8s kubectl"
+
+set -euo pipefail
+
+########################################
+# Determine kubectl command
+########################################
+if [[ $# -gt 0 ]]; then
+  # If an argument was provided, use that as the kubectl command
+  KUBECTL="$*"
+else
+  # Default to 'kubectl'
+  KUBECTL="kubectl"
+fi
+
+########################################
+# Check that the base command exists
+########################################
+# For "microk8s kubectl", we only check "microk8s" in PATH. For "oc" we check "oc".
+BASE_CMD="${KUBECTL%% *}"  # everything before the first space
+if ! command -v "${BASE_CMD}" >/dev/null 2>&1; then
+  echo "ERROR: '${BASE_CMD}' not found in PATH. Please install or configure it first."
+  exit 1
+fi
+
+echo "Using kubectl command: $KUBECTL"
+echo
+
+########################################
+# Helper function to print and run commands
+########################################
+run_cmd() {
+  echo "Command: $*"
+  eval "$*"
+}
+
+########################################
+# 1. Print logs from ALL GoReplay pods
+########################################
+echo "=================================================="
+echo "1. Gathering logs from all goreplay pods (all containers)..."
+echo "=================================================="
+run_cmd "$KUBECTL logs -n goreplay -l app=goreplay --all-containers" || {
+  echo "WARNING: Failed to get logs from pods with label app=goreplay"
+}
+
+########################################
+# 2. Describe the GoReplay DaemonSet
+########################################
+echo
+echo "=================================================="
+echo "2. Describing DaemonSet goreplay-daemon..."
+echo "=================================================="
+run_cmd "$KUBECTL describe daemonset goreplay-daemon -n goreplay" || {
+  echo "WARNING: Failed to describe daemonset goreplay-daemon"
+}
+
+########################################
+# 3. Get list of GoReplay pods (full output)
+########################################
+echo
+echo "=================================================="
+echo "3. Listing goreplay pods (full output)..."
+echo "=================================================="
+
+# Print full output (no -o name here):
+run_cmd "$KUBECTL get pods -n goreplay -l app=goreplay"
+
+# Then retrieve just the names for further processing:
+echo
+echo "Getting goreplay pod names for telemetry collection..."
+pods=$($KUBECTL get pods -n goreplay -l app=goreplay -o name 2>/dev/null) || {
+  echo "ERROR: Failed to list pods with label app=goreplay"
+  exit 1
+}
+echo "Found pods:"
+echo "$pods"
+echo
+
+########################################
+# 4. For each pod, gather logs, describe, and get events
+########################################
+for pod in $pods; do
+  # pod looks like "pod/goreplay-daemon-xyz"
+  pod_name="${pod##*/}"  # remove "pod/" prefix
+
+  echo "=================================================="
+  echo "LOGS for pod: ${pod_name}"
+  echo "=================================================="
+  run_cmd "$KUBECTL logs ${pod_name} -n goreplay" || {
+    echo "WARNING: Failed to get logs for pod ${pod_name}"
+  }
+
+  echo
+  echo "--------------------------------------------------"
+  echo "DESCRIBE for pod: ${pod_name}"
+  echo "--------------------------------------------------"
+  run_cmd "$KUBECTL describe pod -n goreplay ${pod_name}" || {
+    echo "WARNING: Failed to describe pod ${pod_name}"
+  }
+
+  echo
+  echo "--------------------------------------------------"
+  echo "EVENTS for pod: ${pod_name}"
+  echo "--------------------------------------------------"
+  run_cmd "$KUBECTL get events -n goreplay --field-selector involvedObject.name=${pod_name}" || {
+    echo "WARNING: Failed to get events for pod ${pod_name}"
+  }
+  echo
+done
+
+echo "=================================================="
+echo "Telemetry collection complete."
+echo "=================================================="

k8s/goreplay.yaml

@@ -0,0 +1,22 @@
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: goreplay-daemon
+spec:
+  selector:
+    matchLabels:
+      app: goreplay
+  template:
+    metadata:
+      labels:
+        app: goreplay
+    spec:
+      hostNetwork: true
+      serviceAccountName: goreplay
+      containers:
+      - name: goreplay
+        image: buger/gor:v2.0.0-rc4
+        args:
+        - "--input-raw k8s://deployments/nginx:80"
+        - "--output-stdout"
+        - "--verbose"

k8s/nginx.yaml

@@ -0,0 +1,36 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: nginx
+  labels:
+    app: nginx
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: nginx
+  template:
+    metadata:
+      labels:
+        app: nginx
+    spec:
+      containers:
+      - name: nginx
+        image: nginx
+        ports:
+        - containerPort: 80
+   
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: ngnix-service
+spec:
+  selector:
+    app: nginx
+  type: NodePort
+  ports:
+  - protocol: TCP
+    port: 80
+    targetPort: 80
+   

k8s/rolebinding.yaml

@@ -0,0 +1,12 @@
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: goreplay-reader-binding
+subjects:
+- kind: ServiceAccount
+  name: goreplay
+  namespace: goreplay
+roleRef:
+  kind: ClusterRole
+  name: pod-reader
+  apiGroup: ""

kafka.go

@@ -1,30 +1,54 @@
-package main
+package goreplay
 
 import (
 	"bytes"
+	"crypto/sha256"
+	"crypto/sha512"
+	"crypto/tls"
+	"crypto/x509"
+	"errors"
 	"fmt"
+	"github.com/buger/goreplay/proto"
+	"io/ioutil"
+	"log"
 
 	"github.com/Shopify/sarama"
-	"github.com/buger/goreplay/proto"
+	"github.com/xdg-go/scram"
 )
 
-// KafkaConfig should contains required information to
-// build producers.
-
-type InputKafkaConfig struct {
-	producer sarama.AsyncProducer
-	consumer sarama.Consumer
-	Host     string `json:"input-kafka-host"`
-	Topic    string `json:"input-kafka-topic"`
-	UseJSON  bool   `json:"input-kafka-json-format"`
+// SASLKafkaConfig SASL configuration
+type SASLKafkaConfig struct {
+	UseSASL   bool   `json:"input-kafka-use-sasl"`
+	Mechanism string `json:"input-kafka-mechanism"`
+	Username  string `json:"input-kafka-username"`
+	Password  string `json:"input-kafka-password"`
 }
 
+// InputKafkaConfig should contains required information to
+// build producers.
+type InputKafkaConfig struct {
+	consumer   sarama.Consumer
+	Host       string `json:"input-kafka-host"`
+	Topic      string `json:"input-kafka-topic"`
+	UseJSON    bool   `json:"input-kafka-json-format"`
+	Offset     string  `json:"input-kafka-offset"`
+	SASLConfig SASLKafkaConfig
+}
+
+// OutputKafkaConfig is the representation of kfka output configuration
 type OutputKafkaConfig struct {
-	producer sarama.AsyncProducer
-	consumer sarama.Consumer
-	Host     string `json:"output-kafka-host"`
-	Topic    string `json:"output-kafka-topic"`
-	UseJSON  bool   `json:"output-kafka-json-format"`
+	producer   sarama.AsyncProducer
+	Host       string `json:"output-kafka-host"`
+	Topic      string `json:"output-kafka-topic"`
+	UseJSON    bool   `json:"output-kafka-json-format"`
+	SASLConfig SASLKafkaConfig
+}
+
+// KafkaTLSConfig should contains TLS certificates for connecting to secured Kafka clusters
+type KafkaTLSConfig struct {
+	CACert     string `json:"kafka-tls-ca-cert"`
+	ClientCert string `json:"kafka-tls-client-cert"`
+	ClientKey  string `json:"kafka-tls-client-key"`
 }
 
 // KafkaMessage should contains catched request information that should be
@@ -39,6 +63,64 @@ type KafkaMessage struct {
 	ReqHeaders map[string]string `json:"Req_Headers,omitempty"`
 }
 
+// NewTLSConfig loads TLS certificates
+func NewTLSConfig(clientCertFile, clientKeyFile, caCertFile string) (*tls.Config, error) {
+	tlsConfig := tls.Config{}
+
+	if clientCertFile != "" && clientKeyFile == "" {
+		return &tlsConfig, errors.New("Missing key of client certificate in kafka")
+	}
+	if clientCertFile == "" && clientKeyFile != "" {
+		return &tlsConfig, errors.New("missing TLS client certificate in kafka")
+	}
+	// Load client cert
+	if (clientCertFile != "") && (clientKeyFile != "") {
+		cert, err := tls.LoadX509KeyPair(clientCertFile, clientKeyFile)
+		if err != nil {
+			return &tlsConfig, err
+		}
+		tlsConfig.Certificates = []tls.Certificate{cert}
+	}
+	// Load CA cert
+	if caCertFile != "" {
+		caCert, err := ioutil.ReadFile(caCertFile)
+		if err != nil {
+			return &tlsConfig, err
+		}
+		caCertPool := x509.NewCertPool()
+		caCertPool.AppendCertsFromPEM(caCert)
+		tlsConfig.RootCAs = caCertPool
+	}
+	return &tlsConfig, nil
+}
+
+// NewKafkaConfig returns Kafka config with or without TLS
+func NewKafkaConfig(saslConfig *SASLKafkaConfig, tlsConfig *KafkaTLSConfig) *sarama.Config {
+	config := sarama.NewConfig()
+	// Configuration options go here
+	if tlsConfig != nil && (tlsConfig.ClientCert != "" || tlsConfig.CACert != "") {
+		config.Net.TLS.Enable = true
+		tlsConfig, err := NewTLSConfig(tlsConfig.ClientCert, tlsConfig.ClientKey, tlsConfig.CACert)
+		if err != nil {
+			log.Fatal(err)
+		}
+		config.Net.TLS.Config = tlsConfig
+	}
+	if saslConfig.UseSASL {
+		mechanism := sarama.SASLMechanism(saslConfig.Mechanism)
+		config.Net.SASL.Enable = saslConfig.UseSASL
+		config.Net.SASL.Mechanism = mechanism
+		config.Net.SASL.User = saslConfig.Username
+		config.Net.SASL.Password = saslConfig.Password
+		if mechanism == sarama.SASLTypeSCRAMSHA256 {
+			config.Net.SASL.SCRAMClientGeneratorFunc = func() sarama.SCRAMClient { return &XDGSCRAMClient{HashGeneratorFcn: SHA256} }
+		} else if mechanism == sarama.SASLTypeSCRAMSHA512 {
+			config.Net.SASL.SCRAMClientGeneratorFunc = func() sarama.SCRAMClient { return &XDGSCRAMClient{HashGeneratorFcn: SHA512} }
+		}
+	}
+	return config
+}
+
 // Dump returns the given request in its HTTP/1.x wire
 // representation.
 func (m KafkaMessage) Dump() ([]byte, error) {
@@ -46,14 +128,49 @@ func (m KafkaMessage) Dump() ([]byte, error) {
 
 	b.WriteString(fmt.Sprintf("%s %s %s\n", m.ReqType, m.ReqID, m.ReqTs))
 	b.WriteString(fmt.Sprintf("%s %s HTTP/1.1", m.ReqMethod, m.ReqURL))
-	b.Write(proto.CLRF)
+	b.Write(proto.CRLF)
 	for key, value := range m.ReqHeaders {
 		b.WriteString(fmt.Sprintf("%s: %s", key, value))
-		b.Write(proto.CLRF)
+		b.Write(proto.CRLF)
 	}
 
-	b.Write(proto.CLRF)
+	b.Write(proto.CRLF)
 	b.WriteString(m.ReqBody)
 
 	return b.Bytes(), nil
 }
+
+var (
+	// SHA256 SASLMechanism
+	SHA256 scram.HashGeneratorFcn = sha256.New
+	// SHA512 SASLMechanism
+	SHA512 scram.HashGeneratorFcn = sha512.New
+)
+
+// XDGSCRAMClient for SASL-Protocol
+type XDGSCRAMClient struct {
+	*scram.Client
+	*scram.ClientConversation
+	scram.HashGeneratorFcn
+}
+
+// Begin of XDGSCRAMClient
+func (x *XDGSCRAMClient) Begin(userName, password, authzID string) (err error) {
+	x.Client, err = x.HashGeneratorFcn.NewClient(userName, password, authzID)
+	if err != nil {
+		return err
+	}
+	x.ClientConversation = x.Client.NewConversation()
+	return nil
+}
+
+// Step of XDGSCRAMClient
+func (x *XDGSCRAMClient) Step(challenge string) (response string, err error) {
+	response, err = x.ClientConversation.Step(challenge)
+	return
+}
+
+// Done of XDGSCRAMClient
+func (x *XDGSCRAMClient) Done() bool {
+	return x.ClientConversation.Done()
+}

limiter.go

@@ -1,4 +1,4 @@
-package main
+package goreplay
 
 import (
 	"fmt"
@@ -20,8 +20,8 @@ type Limiter struct {
 }
 
 func parseLimitOptions(options string) (limit int, isPercent bool) {
-	if strings.Contains(options, "%") {
-		limit, _ = strconv.Atoi(strings.Split(options, "%")[0])
+	if n := strings.Index(options, "%"); n > 0 {
+		limit, _ = strconv.Atoi(options[:n])
 		isPercent = true
 	} else {
 		limit, _ = strconv.Atoi(options)
@@ -31,25 +31,52 @@ func parseLimitOptions(options string) (limit int, isPercent bool) {
 	return
 }
 
+func newLimiterExceptions(l *Limiter) {
+
+	if !l.isPercent {
+		return
+	}
+	speedFactor := float64(l.limit) / float64(100)
+
+	// FileInput、KafkaInput have its own rate limiting. Unlike other inputs we not just dropping requests, we can slow down or speed up request emittion.
+	switch input := l.plugin.(type) {
+	case *FileInput:
+		input.speedFactor = speedFactor
+	case *KafkaInput:
+		input.speedFactor = speedFactor
+	}
+}
+
 // NewLimiter constructor for Limiter, accepts plugin and options
 // `options` allow to sprcify relatve or absolute limiting
-func NewLimiter(plugin interface{}, options string) io.ReadWriter {
+func NewLimiter(plugin interface{}, options string) PluginReadWriter {
 	l := new(Limiter)
 	l.limit, l.isPercent = parseLimitOptions(options)
 	l.plugin = plugin
 	l.currentTime = time.Now().UnixNano()
 
-	// FileInput have its own rate limiting. Unlike other inputs we not just dropping requests, we can slow down or speed up request emittion.
-	if fi, ok := l.plugin.(*FileInput); ok && l.isPercent {
-		fi.speedFactor = float64(l.limit) / float64(100)
-	}
+	newLimiterExceptions(l)
 
 	return l
 }
 
+func (l *Limiter) isLimitedExceptions() bool {
+	if !l.isPercent {
+		return false
+	}
+	// Fileinput、Kafkainput have its own limiting algorithm
+	switch l.plugin.(type) {
+	case *FileInput:
+		return true
+	case *KafkaInput:
+		return true
+	default:
+		return false
+	}
+}
+
 func (l *Limiter) isLimited() bool {
-	// File input have its own limiting algorithm
-	if _, ok := l.plugin.(*FileInput); ok && l.isPercent {
+	if l.isLimitedExceptions() {
 		return false
 	}
 
@@ -71,24 +98,29 @@ func (l *Limiter) isLimited() bool {
 	return false
 }
 
-func (l *Limiter) Write(data []byte) (n int, err error) {
+// PluginWrite writes message to this plugin
+func (l *Limiter) PluginWrite(msg *Message) (n int, err error) {
 	if l.isLimited() {
 		return 0, nil
 	}
-
-	n, err = l.plugin.(io.Writer).Write(data)
-	return
+	if w, ok := l.plugin.(PluginWriter); ok {
+		return w.PluginWrite(msg)
+	}
+	// avoid further writing
+	return 0, io.ErrClosedPipe
 }
 
-func (l *Limiter) Read(data []byte) (n int, err error) {
-	if r, ok := l.plugin.(io.Reader); ok {
-		n, err = r.Read(data)
+// PluginRead reads message from this plugin
+func (l *Limiter) PluginRead() (msg *Message, err error) {
+	if r, ok := l.plugin.(PluginReader); ok {
+		msg, err = r.PluginRead()
 	} else {
-		return 0, nil
+		// avoid further reading
+		return nil, io.ErrClosedPipe
 	}
 
 	if l.isLimited() {
-		return 0, nil
+		return nil, nil
 	}
 
 	return
@@ -100,7 +132,7 @@ func (l *Limiter) String() string {
 
 // Close closes the resources.
 func (l *Limiter) Close() error {
-	if fi, ok := l.plugin.(io.ReadCloser); ok {
+	if fi, ok := l.plugin.(io.Closer); ok {
 		fi.Close()
 	}
 	return nil

limiter_test.go

@@ -1,30 +1,28 @@
-// +build !race
+//go:build !race
 
-package main
+package goreplay
 
 import (
-	"io"
 	"sync"
 	"testing"
 )
 
 func TestOutputLimiter(t *testing.T) {
 	wg := new(sync.WaitGroup)
-	quit := make(chan int)
 
 	input := NewTestInput()
-	output := NewLimiter(NewTestOutput(func(data []byte) {
+	output := NewLimiter(NewTestOutput(func(*Message) {
 		wg.Done()
 	}), "10")
 	wg.Add(10)
 
 	plugins := &InOutPlugins{
-		Inputs:  []io.Reader{input},
-		Outputs: []io.Writer{output},
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{output},
 	}
 	plugins.All = append(plugins.All, input, output)
 
-	emitter := NewEmitter(quit)
+	emitter := NewEmitter()
 	go emitter.Start(plugins, Settings.Middleware)
 
 	for i := 0; i < 100; i++ {
@@ -37,21 +35,20 @@ func TestOutputLimiter(t *testing.T) {
 
 func TestInputLimiter(t *testing.T) {
 	wg := new(sync.WaitGroup)
-	quit := make(chan int)
 
 	input := NewLimiter(NewTestInput(), "10")
-	output := NewTestOutput(func(data []byte) {
+	output := NewTestOutput(func(*Message) {
 		wg.Done()
 	})
 	wg.Add(10)
 
 	plugins := &InOutPlugins{
-		Inputs:  []io.Reader{input},
-		Outputs: []io.Writer{output},
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{output},
 	}
 	plugins.All = append(plugins.All, input, output)
 
-	emitter := NewEmitter(quit)
+	emitter := NewEmitter()
 	go emitter.Start(plugins, Settings.Middleware)
 
 	for i := 0; i < 100; i++ {
@@ -65,20 +62,19 @@ func TestInputLimiter(t *testing.T) {
 // Should limit all requests
 func TestPercentLimiter1(t *testing.T) {
 	wg := new(sync.WaitGroup)
-	quit := make(chan int)
 
 	input := NewTestInput()
-	output := NewLimiter(NewTestOutput(func(data []byte) {
+	output := NewLimiter(NewTestOutput(func(*Message) {
 		wg.Done()
 	}), "0%")
 
 	plugins := &InOutPlugins{
-		Inputs:  []io.Reader{input},
-		Outputs: []io.Writer{output},
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{output},
 	}
 	plugins.All = append(plugins.All, input, output)
 
-	emitter := NewEmitter(quit)
+	emitter := NewEmitter()
 	go emitter.Start(plugins, Settings.Middleware)
 
 	for i := 0; i < 100; i++ {
@@ -86,27 +82,25 @@ func TestPercentLimiter1(t *testing.T) {
 	}
 
 	wg.Wait()
-	emitter.Close()
 }
 
 // Should not limit at all
 func TestPercentLimiter2(t *testing.T) {
 	wg := new(sync.WaitGroup)
-	quit := make(chan int)
 
 	input := NewTestInput()
-	output := NewLimiter(NewTestOutput(func(data []byte) {
+	output := NewLimiter(NewTestOutput(func(*Message) {
 		wg.Done()
 	}), "100%")
 	wg.Add(100)
 
 	plugins := &InOutPlugins{
-		Inputs:  []io.Reader{input},
-		Outputs: []io.Writer{output},
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{output},
 	}
 	plugins.All = append(plugins.All, input, output)
 
-	emitter := NewEmitter(quit)
+	emitter := NewEmitter()
 	go emitter.Start(plugins, Settings.Middleware)
 
 	for i := 0; i < 100; i++ {
@@ -114,5 +108,4 @@ func TestPercentLimiter2(t *testing.T) {
 	}
 
 	wg.Wait()
-	emitter.Close()
 }

middleware.go

@@ -1,38 +1,41 @@
-package main
+package goreplay
 
 import (
 	"bufio"
+	"context"
 	"encoding/hex"
 	"fmt"
 	"io"
-	"log"
 	"os"
 	"os/exec"
 	"strings"
 	"sync"
+	"syscall"
 )
 
+// Middleware represents a middleware object
 type Middleware struct {
-	command string
-
-	data chan []byte
-
-	mu sync.Mutex
-
-	Stdin  io.Writer
-	Stdout io.Reader
-
-	stop chan bool // Channel used only to indicate goroutine should shutdown
+	command       string
+	data          chan *Message
+	Stdin         io.Writer
+	Stdout        io.Reader
+	commandCancel context.CancelFunc
+	stop          chan bool // Channel used only to indicate goroutine should shutdown
+	closed        bool
+	mu            sync.RWMutex
 }
 
+// NewMiddleware returns new middleware
 func NewMiddleware(command string) *Middleware {
 	m := new(Middleware)
 	m.command = command
-	m.data = make(chan []byte, 1000)
+	m.data = make(chan *Message, 1000)
 	m.stop = make(chan bool)
 
 	commands := strings.Split(command, " ")
-	cmd := exec.Command(commands[0], commands[1:]...)
+	ctx, cancl := context.WithCancel(context.Background())
+	m.commandCancel = cancl
+	cmd := exec.CommandContext(ctx, commands[0], commands[1:]...)
 
 	m.Stdout, _ = cmd.StdoutPipe()
 	m.Stdin, _ = cmd.StdinPipe()
@@ -42,62 +45,61 @@ func NewMiddleware(command string) *Middleware {
 	go m.read(m.Stdout)
 
 	go func() {
-		err := cmd.Start()
-
-		if err != nil {
-			log.Fatal(err)
+		defer m.Close()
+		var err error
+		if err = cmd.Start(); err == nil {
+			err = cmd.Wait()
 		}
-
-		err = cmd.Wait()
-
 		if err != nil {
-			log.Fatal(err)
+			if e, ok := err.(*exec.ExitError); ok {
+				status := e.Sys().(syscall.WaitStatus)
+				if status.Signal() == syscall.SIGKILL /*killed or context canceld */ {
+					return
+				}
+			}
+			Debug(0, fmt.Sprintf("[MIDDLEWARE] command[%q] error: %q", command, err.Error()))
 		}
 	}()
 
 	return m
 }
 
-func (m *Middleware) ReadFrom(plugin io.Reader) {
-	Debug("[MIDDLEWARE-MASTER] Starting reading from", plugin)
+// ReadFrom start a worker to read from this plugin
+func (m *Middleware) ReadFrom(plugin PluginReader) {
+	Debug(2, fmt.Sprintf("[MIDDLEWARE] command[%q] Starting reading from %q", m.command, plugin))
 	go m.copy(m.Stdin, plugin)
 }
 
-func (m *Middleware) copy(to io.Writer, from io.Reader) {
-	buf := make([]byte, 5*1024*1024)
-	dst := make([]byte, len(buf)*4)
+func (m *Middleware) copy(to io.Writer, from PluginReader) {
+	var buf, dst []byte
 
 	for {
-		nr, _ := from.Read(buf)
-		if nr == 0 || nr > len(buf) {
+		msg, err := from.PluginRead()
+		if err != nil {
+			return
+		}
+		if msg == nil || len(msg.Data) == 0 {
 			continue
 		}
-
-		payload := buf[0:nr]
-
+		buf = msg.Data
 		if Settings.PrettifyHTTP {
-			payload = prettifyHTTP(payload)
-			nr = len(payload)
-
-			if nr*2 > len(dst) {
-				continue
-			}
+			buf = prettifyHTTP(msg.Data)
 		}
-
-		if Settings.PrettifyHTTP {
-			payload = prettifyHTTP(payload)
-			nr = len(payload)
+		dstLen := (len(buf)+len(msg.Meta))*2 + 1
+		// if enough space was previously allocated use it instead
+		if dstLen > len(dst) {
+			dst = make([]byte, dstLen)
 		}
+		n := hex.Encode(dst, msg.Meta)
+		n += hex.Encode(dst[n:], buf)
+		dst[n] = '\n'
 
-		hex.Encode(dst, payload)
-		dst[nr*2] = '\n'
-
-		m.mu.Lock()
-		to.Write(dst[0 : nr*2+1])
-		m.mu.Unlock()
-
-		if Settings.Debug {
-			Debug("[MIDDLEWARE-MASTER] Sending:", string(buf[0:nr]), "From:", from)
+		n, err = to.Write(dst[:n+1])
+		if err == nil {
+			continue
+		}
+		if m.isClosed() {
+			return
 		}
 	}
 }
@@ -106,52 +108,59 @@ func (m *Middleware) read(from io.Reader) {
 	reader := bufio.NewReader(from)
 	var line []byte
 	var e error
-
 	for {
 		if line, e = reader.ReadBytes('\n'); e != nil {
-			if e == io.EOF {
-				continue
-			} else {
-				break
+			if m.isClosed() {
+				return
 			}
+			continue
 		}
-
-		buf := make([]byte, len(line)/2)
+		buf := make([]byte, (len(line)-1)/2)
 		if _, err := hex.Decode(buf, line[:len(line)-1]); err != nil {
-			fmt.Fprintln(os.Stderr, "Failed to decode input payload", err, len(line), string(line[:len(line)-1]))
+			Debug(0, fmt.Sprintf("[MIDDLEWARE] command[%q] failed to decode err: %q", m.command, err))
+			continue
 		}
-
-		if Settings.Debug {
-			Debug("[MIDDLEWARE-MASTER] Received:", string(buf))
-		}
-
+		var msg Message
+		msg.Meta, msg.Data = payloadMetaWithBody(buf)
 		select {
 		case <-m.stop:
 			return
-		case m.data <- buf:
+		case m.data <- &msg:
 		}
 	}
 
+}
+
+// PluginRead reads message from this plugin
+func (m *Middleware) PluginRead() (msg *Message, err error) {
+	select {
+	case <-m.stop:
+		return nil, ErrorStopped
+	case msg = <-m.data:
+	}
+
 	return
 }
 
-func (m *Middleware) Read(data []byte) (int, error) {
-	var buf []byte
-	select {
-	case <-m.stop:
-		return 0, ErrorStopped
-	case buf = <-m.data:
-	}
-
-	copy(data, buf)
-	return len(buf), nil
-}
-
 func (m *Middleware) String() string {
-	return fmt.Sprintf("Modifying traffic using '%s' command", m.command)
+	return fmt.Sprintf("Modifying traffic using %q command", m.command)
 }
 
+func (m *Middleware) isClosed() bool {
+	m.mu.RLock()
+	defer m.mu.RUnlock()
+	return m.closed
+}
+
+// Close closes this plugin
 func (m *Middleware) Close() error {
+	if m.isClosed() {
+		return nil
+	}
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	m.commandCancel()
 	close(m.stop)
+	m.closed = true
 	return nil
 }

middleware/README.md

@@ -117,6 +117,115 @@ Package expose following functions to process raw HTTP payloads:
 Also it is totally legit to use standard `Buffer` functions like `indexOf` for processing the HTTP payload. Just do not forget that if you modify the body, update the `Content-Length` header with a new value. And if you modify any of the headers, line endings should be `\r\n`. Rest is up to your imagination.
 
 
+## Masking PII Data
+
+This middleware provides functionality to mask Personally Identifiable Information (PII) data in HTTP requests based on specified headers and JSON paths. It allows you to define a configuration object that specifies which headers and JSON fields should be masked and the type of data they represent.
+
+```javascript
+const gor = require("goreplay_middleware");
+const faker = require("faker");
+
+// Initialize the middleware
+gor.init();
+
+// Configuration for masking PII data
+const maskConfig = {
+  headers: [
+    { name: "Authorization", type: "token" },
+    { name: "X-API-Key", type: "token" },
+    { name: "X-User-Email", type: "email" },
+    { name: "X-User-Name", type: "name" },
+  ],
+  jsonPaths: [
+    { path: "$.user.email", type: "email" },
+    { path: "$.user.name", type: "name" },
+    { path: "$.user.phone", type: "phone" },
+    { path: "$.user.address", type: "address" },
+  ],
+};
+
+// Function to mask a value based on its type
+function maskValue(type) {
+  switch (type) {
+    case "email":
+      return faker.internet.email();
+    case "name":
+      return faker.name.findName();
+    case "phone":
+      return faker.phone.phoneNumber();
+    case "address":
+      return faker.address.streetAddress();
+    case "token":
+      return faker.random.alphaNumeric(32);
+    default:
+      return "***";
+  }
+}
+
+// Middleware function to mask PII data
+gor.on("message", (data) => {
+  // Mask headers
+  maskConfig.headers.forEach((header) => {
+    const value = gor.httpHeader(data.http, header.name);
+    if (value) {
+      data.http = gor.setHttpHeader(data.http, header.name, maskValue(header.type));
+    }
+  });
+
+  // Mask JSON fields
+  const body = gor.httpBody(data.http);
+  if (body) {
+    try {
+      const jsonBody = JSON.parse(body.toString());
+      maskConfig.jsonPaths.forEach((field) => {
+        const value = eval(`jsonBody${field.path.slice(1)}`);
+        if (value) {
+          eval(`jsonBody${field.path.slice(1)} = maskValue(field.type)`);
+        }
+      });
+      data.http = gor.setHttpBody(data.http, Buffer.from(JSON.stringify(jsonBody)));
+    } catch (error) {
+      console.error("Error parsing JSON body:", error);
+    }
+  }
+
+  return data;
+});
+```
+
+### Configuration
+
+The `maskConfig` object is used to configure the masking behavior. It consists of two properties:
+
+- `headers`: An array of objects representing the headers to be masked. Each object should have the following properties:
+  - `name`: The name of the header.
+  - `type`: The type of data the header represents (e.g., "email", "name", "token").
+
+- `jsonPaths`: An array of objects representing the JSON paths to be masked. Each object should have the following properties:
+  - `path`: The JSON path to the field to be masked (e.g., "$.user.email").
+  - `type`: The type of data the field represents (e.g., "email", "name", "phone", "address").
+
+### Masking Function
+
+The `maskValue` function is responsible for generating masked values based on the data type. It uses the Faker library to generate realistic-looking masked data for different types such as email, name, phone, address, and token. You can extend this function to support additional data types or customize the masking behavior.
+
+### Middleware Function
+
+The middleware function is triggered for each HTTP message (request or response) processed by GoReplay. It performs the following steps:
+
+1. Iterate over the specified headers in the `maskConfig` and mask their values using the `maskValue` function based on the associated data type.
+
+2. Parse the JSON body of the request (if present) and iterate over the specified JSON paths in the `maskConfig`. If a value exists at a given path, replace it with a masked value generated by the `maskValue` function based on the associated data type.
+
+3. Update the request body with the masked JSON data.
+
+4. Return the modified HTTP message.
+
+Note: The middleware uses the `eval` function to dynamically access and modify the JSON object based on the provided paths. Exercise caution when using `eval` and ensure that the paths are properly validated to prevent potential security risks.
+
+To use this middleware, make sure to install the required dependencies (`goreplay_middleware` and `faker`), configure the `maskConfig` object according to your needs, and run the middleware with GoReplay.
+
+
 ## Support
 
 Feel free to ask questions here and by sending email to [support@goreplay.org](mailto:support@goreplay.org). Commercial support is available and welcomed 🙈.

middleware_test.go

@@ -1,229 +1,152 @@
-package main
+package goreplay
 
 import (
 	"bytes"
-	"crypto/rand"
-	"encoding/hex"
-	"io"
-	"net/http"
-	"net/http/httptest"
+	"context"
+	"os"
+	"os/exec"
 	"strings"
-	"sync"
+	"sync/atomic"
+	"syscall"
 	"testing"
-	"time"
-
-	"github.com/buger/goreplay/proto"
 )
 
-type fakeServiceCb func(string, int, []byte)
+const echoSh = "./examples/middleware/echo.sh"
+const tokenModifier = "go run ./examples/middleware/token_modifier.go"
 
-// Simple service that generate token on request, and require this token for accesing to secure area
-func NewFakeSecureService(wg *sync.WaitGroup, cb fakeServiceCb) *httptest.Server {
-	active_tokens := make([]string, 0)
-	var mu sync.Mutex
+var withDebug = append(syscall.Environ(), "GOR_TEST=1")
 
-	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
-		mu.Lock()
-		defer mu.Unlock()
-		Debug("Received request: " + req.URL.String())
+func initMiddleware(cmd *exec.Cmd, cancl context.CancelFunc, l PluginReader, c func(error)) *Middleware {
+	var m Middleware
+	m.data = make(chan *Message, 1000)
+	m.stop = make(chan bool)
+	m.commandCancel = cancl
+	m.Stdout, _ = cmd.StdoutPipe()
+	m.Stdin, _ = cmd.StdinPipe()
+	cmd.Stderr = os.Stderr
+	go m.read(m.Stdout)
+	go func() {
+		defer m.Close()
+		var err error
+		if err = cmd.Start(); err == nil {
+			err = cmd.Wait()
+		}
+		if err != nil {
+			c(err)
+		}
+	}()
+	m.ReadFrom(l)
+	return &m
+}
 
-		switch req.URL.Path {
-		case "/token":
-			// Generate random token
-			token_length := 10
-			buf := make([]byte, token_length)
-			rand.Read(buf)
-			token := hex.EncodeToString(buf)
-			active_tokens = append(active_tokens, token)
+func initCmd(command string, env []string) (*exec.Cmd, context.CancelFunc) {
+	commands := strings.Split(command, " ")
+	ctx, cancl := context.WithCancel(context.Background())
+	cmd := exec.CommandContext(ctx, commands[0], commands[1:]...)
+	cmd.Env = env
+	return cmd, cancl
+}
 
-			w.Write([]byte(token))
-
-			cb(req.URL.Path, 200, []byte(token))
-		case "/secure":
-			token := req.URL.Query().Get("token")
-			token_found := false
-
-			for _, t := range active_tokens {
-				if t == token {
-					token_found = true
-					break
+func TestMiddlewareEarlyClose(t *testing.T) {
+	quit := make(chan struct{})
+	in := NewTestInput()
+	cmd, cancl := initCmd(echoSh, withDebug)
+	midd := initMiddleware(cmd, cancl, in, func(err error) {
+		if err != nil {
+			if e, ok := err.(*exec.ExitError); ok {
+				status := e.Sys().(syscall.WaitStatus)
+				if status.Signal() != syscall.SIGKILL {
+					t.Errorf("expected error to be signal killed. got %s", status.Signal().String())
 				}
 			}
-
-			if token_found {
-				w.WriteHeader(http.StatusAccepted)
-				cb(req.URL.Path, 202, []byte(nil))
-			} else {
-				w.WriteHeader(http.StatusForbidden)
-				cb(req.URL.Path, 403, []byte(nil))
-			}
 		}
-
-		wg.Done()
-	}))
-
-	return server
-}
-
-func TestFakeSecureService(t *testing.T) {
-	var resp, token []byte
-
-	wg := new(sync.WaitGroup)
-
-	server := NewFakeSecureService(wg, func(path string, status int, resp []byte) {
+		quit <- struct{}{}
 	})
-	defer server.Close()
-
-	wg.Add(3)
-
-	client := NewHTTPClient(server.URL, &HTTPClientConfig{Debug: true})
-	resp, _ = client.Get("/token")
-	token = proto.Body(resp)
-
-	// Right token
-	resp, _ = client.Get("/secure?token=" + string(token))
-	if !bytes.Equal(proto.Status(resp), []byte("202")) {
-		t.Error("Valid token should return status 202:", string(proto.Status(resp)))
-	}
-
-	// Wrong tokens forbidden
-	resp, _ = client.Get("/secure?token=wrong")
-	if !bytes.Equal(proto.Status(resp), []byte("403")) {
-		t.Error("Wrong token should returns status 403:", string(proto.Status(resp)))
-	}
-
-	wg.Wait()
-}
-
-func TestEchoMiddleware(t *testing.T) {
-	wg := new(sync.WaitGroup)
-
-	from := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		w.Header().Set("Env", "prod")
-		w.Header().Set("RequestPath", r.URL.Path)
-		wg.Done()
-	}))
-	defer from.Close()
-
-	to := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		w.Header().Set("Env", "test")
-		w.Header().Set("RequestPath", r.URL.Path)
-		wg.Done()
-	}))
-	defer to.Close()
-
-	quit := make(chan int)
-
-	Settings.Middleware = "./examples/middleware/echo.sh"
-
-	// Catch traffic from one service
-	fromAddr := strings.Replace(from.Listener.Addr().String(), "[::]", "127.0.0.1", -1)
-	input := NewRAWInput(fromAddr, EnginePcap, true, testRawExpire, "", "http", "", "", 0)
-	defer input.Close()
-
-	// And redirect to another
-	output := NewHTTPOutput(to.URL, &HTTPOutputConfig{Debug: false})
-
-	plugins := &InOutPlugins{
-		Inputs:  []io.Reader{input},
-		Outputs: []io.Writer{output},
-	}
-	plugins.All = append(plugins.All, input, output)
-
-	// Start Gor
-	emitter := NewEmitter(quit)
-	go emitter.Start(plugins, Settings.Middleware)
-
-	// Wait till middleware initialization
-	time.Sleep(100 * time.Millisecond)
-
-	// Should receive 2 requests from original + 2 from replayed
-	client := NewHTTPClient(from.URL, &HTTPClientConfig{Debug: false})
-
-	for i := 0; i < 10; i++ {
-		wg.Add(4)
-		// Request should be echoed
-		client.Get("/a")
-		time.Sleep(5 * time.Millisecond)
-		client.Get("/b")
-		time.Sleep(5 * time.Millisecond)
-	}
-
-	wg.Wait()
-	emitter.Close()
-	time.Sleep(200 * time.Millisecond)
-
-	Settings.Middleware = ""
-}
-
-func TestTokenMiddleware(t *testing.T) {
-	var resp, token []byte
-
-	wg := new(sync.WaitGroup)
-
-	from := NewFakeSecureService(wg, func(path string, status int, tok []byte) {
-		time.Sleep(10 * time.Millisecond)
-	})
-	defer from.Close()
-
-	to := NewFakeSecureService(wg, func(path string, status int, tok []byte) {
-		switch path {
-		case "/secure":
-			if status != 202 {
-				t.Error("Server should receive valid rewritten token")
-			}
+	var body = []byte("OPTIONS / HTTP/1.1\r\nHost: example.org\r\n\r\n")
+	count := uint32(0)
+	out := NewTestOutput(func(msg *Message) {
+		if !bytes.Equal(body, msg.Data) {
+			t.Errorf("expected %q to equal %q", body, msg.Data)
+		}
+		atomic.AddUint32(&count, 1)
+		if atomic.LoadUint32(&count) == 5 {
+			quit <- struct{}{}
 		}
-
-		time.Sleep(10 * time.Millisecond)
 	})
-	defer to.Close()
-
-	quit := make(chan int)
-
-	Settings.Middleware = "go run ./examples/middleware/token_modifier.go"
-
-	fromAddr := strings.Replace(from.Listener.Addr().String(), "[::]", "127.0.0.1", -1)
-	// Catch traffic from one service
-	input := NewRAWInput(fromAddr, EnginePcap, true, testRawExpire, "", "http", "", "", 0)
-	defer input.Close()
-
-	// And redirect to another
-	output := NewHTTPOutput(to.URL, &HTTPOutputConfig{Debug: true})
-
-	plugins := &InOutPlugins{
-		Inputs:  []io.Reader{input},
-		Outputs: []io.Writer{output},
+	pl := &InOutPlugins{}
+	pl.Inputs = []PluginReader{midd, in}
+	pl.Outputs = []PluginWriter{out}
+	pl.All = []interface{}{midd, out, in}
+	e := NewEmitter()
+	go e.Start(pl, "")
+	for i := 0; i < 5; i++ {
+		in.EmitBytes(body)
 	}
-	plugins.All = append(plugins.All, input, output)
-
-	// Start Gor
-	emitter := NewEmitter(quit)
-	go emitter.Start(plugins, Settings.Middleware)
-
-	// Wait for middleware to initialize
-	// Give go compiller time to build programm
-	time.Sleep(500 * time.Millisecond)
-
-	// Should receive 2 requests from original + 2 from replayed
-	wg.Add(4)
-
-	client := NewHTTPClient(from.URL, &HTTPClientConfig{Debug: true})
-
-	// Sending traffic to original service
-	resp, _ = client.Get("/token")
-	token = proto.Body(resp)
-
-	// When delay is too smal, middleware does not always rewrite requests in time
-	// Hopefuly client will have delay more then 100ms :)
-	time.Sleep(100 * time.Millisecond)
-
-	resp, _ = client.Get("/secure?token=" + string(token))
-	if !bytes.Equal(proto.Status(resp), []byte("202")) {
-		t.Error("Valid token should return 202:", proto.Status(resp))
-	}
-
-	wg.Wait()
-	emitter.Close()
-	time.Sleep(100 * time.Millisecond)
-	Settings.Middleware = ""
+	<-quit
+	midd.Close()
+	<-quit
 }
+
+//func TestTokenMiddleware(t *testing.T) {
+//	quit := make(chan struct{})
+//	in := NewTestInput()
+//	in.skipHeader = true
+//	cmd, cancl := initCmd(tokenModifier, withDebug)
+//	midd := initMiddleware(cmd, cancl, in, func(err error) {})
+//	req := []byte("1 932079936fa4306fc308d67588178d17d823647c 1439818823587396305 200\nGET /token HTTP/1.1\r\nHost: example.org\r\n\r\n")
+//	res := []byte("2 932079936fa4306fc308d67588178d17d823647c 1439818823587396305 200\nHTTP/1.1 200 OK\r\nContent-Length: 10\r\nContent-Type: text/plain; charset=utf-8\r\n\r\n17d823647c")
+//	rep := []byte("3 932079936fa4306fc308d67588178d17d823647c 1439818823587396305 200\nHTTP/1.1 200 OK\r\nContent-Length: 15\r\nContent-Type: text/plain; charset=utf-8\r\n\r\n932079936fa4306")
+//	count := uint32(0)
+//	out := NewTestOutput(func(msg *Message) {
+//		if msg.Meta[0] == '1' && !bytes.Equal(payloadID(msg.Meta), payloadID(req)) {
+//			token, _, _ := proto.PathParam(msg.Data, []byte("token"))
+//			if !bytes.Equal(token, proto.Body(rep)) {
+//				t.Errorf("expected the token %s to be equal to the replayed response's token %s", token, proto.Body(rep))
+//			}
+//		}
+//		atomic.AddUint32(&count, 1)
+//		if atomic.LoadUint32(&count) == 2 {
+//			quit <- struct{}{}
+//		}
+//	})
+//	pl := &InOutPlugins{}
+//	pl.Inputs = []PluginReader{midd, in}
+//	pl.Outputs = []PluginWriter{out}
+//	pl.All = []interface{}{midd, out, in}
+//	e := NewEmitter()
+//	go e.Start(pl, "")
+//	in.EmitBytes(req) // emit original request
+//	in.EmitBytes(res) // emit its response
+//	in.EmitBytes(rep) // emit replayed response
+//	// emit the request which should have modified token
+//	token := []byte("1 8e091765ae902fef8a2b7d9dd96 14398188235873 100\nGET /?token=17d823647c HTTP/1.1\r\nHost: example.org\r\n\r\n")
+//	in.EmitBytes(token)
+//	<-quit
+//	midd.Close()
+//}
+
+//func TestMiddlewareWithPrettify(t *testing.T) {
+//	Settings.PrettifyHTTP = true
+//	quit := make(chan struct{})
+//	in := NewTestInput()
+//	cmd, cancl := initCmd(echoSh, withDebug)
+//	midd := initMiddleware(cmd, cancl, in, func(err error) {})
+//	var b1 = []byte("POST / HTTP/1.1\r\nHost: example.org\r\nTransfer-Encoding: chunked\r\n\r\n4\r\nWiki\r\n5\r\npedia\r\nE\r\n in\r\n\r\nchunks.\r\n0\r\n\r\n")
+//	var b2 = []byte("POST / HTTP/1.1\r\nHost: example.org\r\nContent-Length: 25\r\n\r\nWikipedia in\r\n\r\nchunks.")
+//	out := NewTestOutput(func(msg *Message) {
+//		if !bytes.Equal(proto.Body(b2), proto.Body(msg.Data)) {
+//			t.Errorf("expected %q body to equal %q body", b2, msg.Data)
+//		}
+//		quit <- struct{}{}
+//	})
+//	pl := &InOutPlugins{}
+//	pl.Inputs = []PluginReader{midd, in}
+//	pl.Outputs = []PluginWriter{out}
+//	pl.All = []interface{}{midd, out, in}
+//	e := NewEmitter()
+//	go e.Start(pl, "")
+//	in.EmitBytes(b1)
+//	<-quit
+//	midd.Close()
+//	Settings.PrettifyHTTP = false
+//}

nfpm.yaml

@@ -0,0 +1,21 @@
+# nfpm example config file
+#
+# check https://nfpm.goreleaser.com/configuration for detailed usage
+#
+name: "GoReplay"
+arch: ${PLATFORM}
+platform: "linux"
+version: ${VERSION}
+section: "default"
+priority: "extra"
+provides:
+- goreplay
+maintainer: "Leonid Bugaev <hello@goreplay.org>"
+description: |
+ GoReplay is the simplest and safest way to test your app using real traffic before you put it into production. 
+vendor: "GoReplay"
+homepage: "https://goreplay.org"
+license: "AGPL"
+contents:
+- src: ./${BIN_NAME}
+  dst: /usr/local/bin

output_binary.go

@@ -1,16 +1,21 @@
-package main
+//go:build !pro
+
+package goreplay
 
 import (
-	"io"
-	"sync/atomic"
+	"errors"
 	"time"
+
+	"github.com/buger/goreplay/internal/size"
 )
 
+var _ PluginWriter = (*BinaryOutput)(nil)
+
 // BinaryOutputConfig struct for holding binary output configuration
 type BinaryOutputConfig struct {
 	Workers        int           `json:"output-binary-workers"`
 	Timeout        time.Duration `json:"output-binary-timeout"`
-	BufferSize     int           `json:"output-tcp-response-buffer"`
+	BufferSize     size.Size     `json:"output-tcp-response-buffer"`
 	Debug          bool          `json:"output-binary-debug"`
 	TrackResponses bool          `json:"output-binary-track-response"`
 }
@@ -19,158 +24,30 @@ type BinaryOutputConfig struct {
 // By default workers pool is dynamic and starts with 10 workers
 // You can specify fixed number of workers using `--output-tcp-workers`
 type BinaryOutput struct {
-	// Keep this as first element of struct because it guarantees 64bit
-	// alignment. atomic.* functions crash on 32bit machines if operand is not
-	// aligned at 64bit. See https://github.com/golang/go/issues/599
-	activeWorkers int64
-
 	address string
-	queue   chan []byte
-
-	responses chan response
-
-	needWorker chan int
-
-	config *BinaryOutputConfig
-
-	queueStats *GorStat
 }
 
 // NewBinaryOutput constructor for BinaryOutput
 // Initialize workers
-func NewBinaryOutput(address string, config *BinaryOutputConfig) io.Writer {
-	o := new(BinaryOutput)
-
-	o.address = address
-	o.config = config
-
-	o.queue = make(chan []byte, 1000)
-	o.responses = make(chan response, 1000)
-	o.needWorker = make(chan int, 1)
-
-	// Initial workers count
-	if o.config.Workers == 0 {
-		o.needWorker <- initialDynamicWorkers
-	} else {
-		o.needWorker <- o.config.Workers
-	}
-
-	go o.workerMaster()
-
-	return o
+func NewBinaryOutput(address string, config *BinaryOutputConfig) PluginReadWriter {
+	return &BinaryOutput{address: address}
 }
 
-func (o *BinaryOutput) workerMaster() {
-	for {
-		newWorkers := <-o.needWorker
-		for i := 0; i < newWorkers; i++ {
-			go o.startWorker()
-		}
-
-		// Disable dynamic scaling if workers poll fixed size
-		if o.config.Workers != 0 {
-			return
-		}
-	}
+// PluginWrite writes a message to this plugin
+func (o *BinaryOutput) PluginWrite(msg *Message) (n int, err error) {
+	return 0, errors.New("binary output is only available in PRO version")
 }
 
-func (o *BinaryOutput) startWorker() {
-	client := NewTCPClient(o.address, &TCPClientConfig{
-		Debug:              o.config.Debug,
-		Timeout:            o.config.Timeout,
-		ResponseBufferSize: o.config.BufferSize,
-	})
-
-	deathCount := 0
-
-	atomic.AddInt64(&o.activeWorkers, 1)
-
-	for {
-		select {
-		case data := <-o.queue:
-			o.sendRequest(client, data)
-			deathCount = 0
-		case <-time.After(time.Millisecond * 100):
-			// When dynamic scaling enabled workers die after 2s of inactivity
-			if o.config.Workers == 0 {
-				deathCount++
-			} else {
-				continue
-			}
-
-			if deathCount > 20 {
-				workersCount := atomic.LoadInt64(&o.activeWorkers)
-
-				// At least 1 startWorker should be alive
-				if workersCount != 1 {
-					atomic.AddInt64(&o.activeWorkers, -1)
-					return
-				}
-			}
-		}
-	}
-}
-
-func (o *BinaryOutput) Write(data []byte) (n int, err error) {
-	if !isRequestPayload(data) {
-		return len(data), nil
-	}
-
-	buf := make([]byte, len(data))
-	copy(buf, data)
-
-	o.queue <- buf
-
-	if o.config.Workers == 0 {
-		workersCount := atomic.LoadInt64(&o.activeWorkers)
-
-		if len(o.queue) > int(workersCount) {
-			o.needWorker <- len(o.queue)
-		}
-	}
-
-	return len(data), nil
-}
-
-func (o *BinaryOutput) Read(data []byte) (int, error) {
-	resp := <-o.responses
-
-	Debug("[OUTPUT-TCP] Received response:", string(resp.payload))
-
-	header := payloadHeader(ReplayedResponsePayload, resp.uuid, resp.startedAt, resp.roundTripTime)
-	copy(data[0:len(header)], header)
-	copy(data[len(header):], resp.payload)
-
-	return len(resp.payload) + len(header), nil
-}
-
-func (o *BinaryOutput) sendRequest(client *TCPClient, request []byte) {
-	meta := payloadMeta(request)
-	if len(meta) < 2 {
-		return
-	}
-
-	if !isRequestPayload(request) {
-		return
-	}
-
-	uuid := meta[1]
-
-	body := payloadBody(request)
-
-	start := time.Now()
-	resp, err := client.Send(body)
-	stop := time.Now()
-
-	if err != nil {
-		Debug("Request error:", err)
-	}
-
-	if o.config.TrackResponses {
-		o.responses <- response{resp, uuid, start.UnixNano(), stop.UnixNano() - start.UnixNano()}
-	}
+// PluginRead reads a message from this plugin
+func (o *BinaryOutput) PluginRead() (*Message, error) {
+	return nil, errors.New("binary output is only available in PRO version")
 }
 
 func (o *BinaryOutput) String() string {
-	return "Binary output: " + o.address
+	return "Binary output: " + o.address + " (PRO version required)"
+}
+
+// Close closes this plugin for reading
+func (o *BinaryOutput) Close() error {
+	return nil
 }

output_binary_pro.go

@@ -0,0 +1,178 @@
+//go:build pro
+
+package goreplay
+
+import (
+	"sync/atomic"
+	"time"
+
+	"github.com/buger/goreplay/internal/size"
+)
+
+var _ PluginWriter = (*BinaryOutput)(nil)
+
+// BinaryOutputConfig struct for holding binary output configuration
+type BinaryOutputConfig struct {
+	Workers        int           `json:"output-binary-workers"`
+	Timeout        time.Duration `json:"output-binary-timeout"`
+	BufferSize     size.Size     `json:"output-tcp-response-buffer"`
+	Debug          bool          `json:"output-binary-debug"`
+	TrackResponses bool          `json:"output-binary-track-response"`
+}
+
+// BinaryOutput plugin manage pool of workers which send request to replayed server
+// By default workers pool is dynamic and starts with 10 workers
+// You can specify fixed number of workers using `--output-tcp-workers`
+type BinaryOutput struct {
+	// Keep this as first element of struct because it guarantees 64bit
+	// alignment. atomic.* functions crash on 32bit machines if operand is not
+	// aligned at 64bit. See https://github.com/golang/go/issues/599
+	activeWorkers int64
+	address       string
+	queue         chan *Message
+	responses     chan response
+	needWorker    chan int
+	quit          chan struct{}
+	config        *BinaryOutputConfig
+	queueStats    *GorStat
+}
+
+// NewBinaryOutput constructor for BinaryOutput
+// Initialize workers
+func NewBinaryOutput(address string, config *BinaryOutputConfig) PluginReadWriter {
+	o := new(BinaryOutput)
+
+	o.address = address
+	o.config = config
+
+	o.queue = make(chan *Message, 1000)
+	o.responses = make(chan response, 1000)
+	o.needWorker = make(chan int, 1)
+	o.quit = make(chan struct{})
+
+	// Initial workers count
+	if o.config.Workers == 0 {
+		o.needWorker <- initialDynamicWorkers
+	} else {
+		o.needWorker <- o.config.Workers
+	}
+
+	go o.workerMaster()
+
+	return o
+}
+
+func (o *BinaryOutput) workerMaster() {
+	for {
+		newWorkers := <-o.needWorker
+		for i := 0; i < newWorkers; i++ {
+			go o.startWorker()
+		}
+
+		// Disable dynamic scaling if workers poll fixed size
+		if o.config.Workers != 0 {
+			return
+		}
+	}
+}
+
+func (o *BinaryOutput) startWorker() {
+	client := NewTCPClient(o.address, &TCPClientConfig{
+		Debug:              o.config.Debug,
+		Timeout:            o.config.Timeout,
+		ResponseBufferSize: int(o.config.BufferSize),
+	})
+
+	deathCount := 0
+
+	atomic.AddInt64(&o.activeWorkers, 1)
+
+	for {
+		select {
+		case msg := <-o.queue:
+			o.sendRequest(client, msg)
+			deathCount = 0
+		case <-time.After(time.Millisecond * 100):
+			// When dynamic scaling enabled workers die after 2s of inactivity
+			if o.config.Workers == 0 {
+				deathCount++
+			} else {
+				continue
+			}
+
+			if deathCount > 20 {
+				workersCount := atomic.LoadInt64(&o.activeWorkers)
+
+				// At least 1 startWorker should be alive
+				if workersCount != 1 {
+					atomic.AddInt64(&o.activeWorkers, -1)
+					return
+				}
+			}
+		}
+	}
+}
+
+// PluginWrite writes a message tothis plugin
+func (o *BinaryOutput) PluginWrite(msg *Message) (n int, err error) {
+	if !isRequestPayload(msg.Meta) {
+		return len(msg.Data), nil
+	}
+
+	o.queue <- msg
+
+	if o.config.Workers == 0 {
+		workersCount := atomic.LoadInt64(&o.activeWorkers)
+
+		if len(o.queue) > int(workersCount) {
+			o.needWorker <- len(o.queue)
+		}
+	}
+
+	return len(msg.Data) + len(msg.Meta), nil
+}
+
+// PluginRead reads a message from this plugin
+func (o *BinaryOutput) PluginRead() (*Message, error) {
+	var resp response
+	var msg Message
+	select {
+	case <-o.quit:
+		return nil, ErrorStopped
+	case resp = <-o.responses:
+	}
+	msg.Data = resp.payload
+	msg.Meta = payloadHeader(ReplayedResponsePayload, resp.uuid, resp.startedAt, resp.roundTripTime)
+
+	return &msg, nil
+}
+
+func (o *BinaryOutput) sendRequest(client *TCPClient, msg *Message) {
+	if !isRequestPayload(msg.Meta) {
+		return
+	}
+
+	uuid := payloadID(msg.Meta)
+
+	start := time.Now()
+	resp, err := client.Send(msg.Data)
+	stop := time.Now()
+
+	if err != nil {
+		Debug(1, "Request error:", err)
+	}
+
+	if o.config.TrackResponses {
+		o.responses <- response{resp, uuid, start.UnixNano(), stop.UnixNano() - start.UnixNano()}
+	}
+}
+
+func (o *BinaryOutput) String() string {
+	return "Binary output: " + o.address
+}
+
+// Close closes this plugin for reading
+func (o *BinaryOutput) Close() error {
+	close(o.quit)
+	return nil
+}

output_dummy.go

@@ -1,7 +1,7 @@
-package main
+package goreplay
 
 import (
-	"fmt"
+	"os"
 )
 
 // DummyOutput used for debugging, prints all incoming requests
@@ -15,10 +15,17 @@ func NewDummyOutput() (di *DummyOutput) {
 	return
 }
 
-func (i *DummyOutput) Write(data []byte) (int, error) {
-	fmt.Println(string(data))
+// PluginWrite writes message to this plugin
+func (i *DummyOutput) PluginWrite(msg *Message) (int, error) {
+	var n, nn int
+	var err error
+	n, err = os.Stdout.Write(msg.Meta)
+	nn, err = os.Stdout.Write(msg.Data)
+	n += nn
+	nn, err = os.Stdout.Write(payloadSeparatorAsBytes)
+	n += nn
 
-	return len(data), nil
+	return n, err
 }
 
 func (i *DummyOutput) String() string {

output_file.go

@@ -1,12 +1,14 @@
-package main
+package goreplay
 
 import (
 	"bufio"
 	"compress/gzip"
 	"errors"
 	"fmt"
+	"github.com/buger/goreplay/internal/size"
 	"io"
 	"log"
+	"math/rand"
 	"os"
 	"path/filepath"
 	"runtime/debug"
@@ -17,6 +19,21 @@ import (
 	"time"
 )
 
+var letters = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ")
+var instanceID string
+
+func init() {
+	instanceID = randSeq(8)
+}
+
+func randSeq(n int) string {
+	b := make([]rune, n)
+	for i := range b {
+		b[i] = letters[rand.Intn(len(letters))]
+	}
+	return string(b)
+}
+
 var dateFileNameFuncs = map[string]func(*FileOutput) string{
 	"%Y":  func(o *FileOutput) string { return time.Now().Format("2006") },
 	"%m":  func(o *FileOutput) string { return time.Now().Format("01") },
@@ -27,33 +44,34 @@ var dateFileNameFuncs = map[string]func(*FileOutput) string{
 	"%NS": func(o *FileOutput) string { return fmt.Sprint(time.Now().Nanosecond()) },
 	"%r":  func(o *FileOutput) string { return string(o.currentID) },
 	"%t":  func(o *FileOutput) string { return string(o.payloadType) },
+	"%i":  func(o *FileOutput) string { return instanceID },
 }
 
 // FileOutputConfig ...
 type FileOutputConfig struct {
 	FlushInterval     time.Duration `json:"output-file-flush-interval"`
-	sizeLimit         int64
-	outputFileMaxSize int64
-	QueueLimit        int64  `json:"output-file-queue-limit"`
-	Append            bool   `json:"output-file-append"`
-	BufferPath        string `json:"output-file-buffer"`
+	SizeLimit         size.Size     `json:"output-file-size-limit"`
+	OutputFileMaxSize size.Size     `json:"output-file-max-size-limit"`
+	QueueLimit        int           `json:"output-file-queue-limit"`
+	Append            bool          `json:"output-file-append"`
+	BufferPath        string        `json:"output-file-buffer"`
 	onClose           func(string)
 }
 
 // FileOutput output plugin
 type FileOutput struct {
 	sync.RWMutex
-	pathTemplate   string
-	currentName    string
-	file           *os.File
-	queueLength    int64
-	chunkSize      int
-	writer         io.Writer
-	requestPerFile bool
-	currentID      []byte
-	payloadType    []byte
-	closed         bool
-	totalFileSize  int64
+	pathTemplate    string
+	currentName     string
+	file            *os.File
+	QueueLength     int
+	writer          io.Writer
+	requestPerFile  bool
+	currentID       []byte
+	payloadType     []byte
+	closed          bool
+	currentFileSize int
+	totalFileSize   size.Size
 
 	config *FileOutputConfig
 }
@@ -63,7 +81,6 @@ func NewFileOutput(pathTemplate string, config *FileOutputConfig) *FileOutput {
 	o := new(FileOutput)
 	o.pathTemplate = pathTemplate
 	o.config = config
-	o.updateName()
 
 	if strings.Contains(pathTemplate, "%r") {
 		o.requestPerFile = true
@@ -79,7 +96,6 @@ func NewFileOutput(pathTemplate string, config *FileOutputConfig) *FileOutput {
 			if o.IsClosed() {
 				break
 			}
-			o.updateName()
 			o.flush()
 		}
 	}()
@@ -154,8 +170,8 @@ func (o *FileOutput) filename() string {
 		nextChunk := false
 
 		if o.currentName == "" ||
-			((o.config.QueueLimit > 0 && o.queueLength >= o.config.QueueLimit) ||
-				(o.config.sizeLimit > 0 && o.chunkSize >= int(o.config.sizeLimit))) {
+			((o.config.QueueLimit > 0 && o.QueueLength >= o.config.QueueLimit) ||
+				(o.config.SizeLimit > 0 && o.currentFileSize >= int(o.config.SizeLimit))) {
 			nextChunk = true
 		}
 
@@ -193,10 +209,11 @@ func (o *FileOutput) updateName() {
 	o.Unlock()
 }
 
-func (o *FileOutput) Write(data []byte) (n int, err error) {
+// PluginWrite writes message to this plugin
+func (o *FileOutput) PluginWrite(msg *Message) (n int, err error) {
 	if o.requestPerFile {
 		o.Lock()
-		meta := payloadMeta(data)
+		meta := payloadMeta(msg.Meta)
 		o.currentID = meta[1]
 		o.payloadType = meta[0]
 		o.Unlock()
@@ -222,29 +239,32 @@ func (o *FileOutput) Write(data []byte) (n int, err error) {
 			log.Fatal(o, "Cannot open file %q. Error: %s", o.currentName, err)
 		}
 
-		o.queueLength = 0
+		o.QueueLength = 0
 	}
 
-	n, _ = o.writer.Write(data)
-	nSeparator, _ := o.writer.Write([]byte(payloadSeparator))
+	var nn int
+	n, err = o.writer.Write(msg.Meta)
+	nn, err = o.writer.Write(msg.Data)
+	n += nn
+	nn, err = o.writer.Write(payloadSeparatorAsBytes)
+	n += nn
 
-	n += nSeparator
+	o.totalFileSize += size.Size(n)
+	o.currentFileSize += n
+	o.QueueLength++
 
-	o.totalFileSize += int64(n)
-	o.queueLength++
-
-	if Settings.OutputFileConfig.outputFileMaxSize > 0 && o.totalFileSize >= Settings.OutputFileConfig.outputFileMaxSize {
+	if Settings.OutputFileConfig.OutputFileMaxSize > 0 && o.totalFileSize >= Settings.OutputFileConfig.OutputFileMaxSize {
 		return n, errors.New("File output reached size limit")
 	}
 
-	return n, nil
+	return n, err
 }
 
 func (o *FileOutput) flush() {
 	// Don't exit on panic
 	defer func() {
 		if r := recover(); r != nil {
-			log.Println("PANIC while file flush: ", r, o, string(debug.Stack()))
+			Debug(0, "[OUTPUT-FILE] PANIC while file flush: ", r, o, string(debug.Stack()))
 		}
 	}()
 
@@ -259,9 +279,9 @@ func (o *FileOutput) flush() {
 		}
 
 		if stat, err := o.file.Stat(); err == nil {
-			o.chunkSize = int(stat.Size())
+			o.currentFileSize = int(stat.Size())
 		} else {
-			log.Println("Error accessing file sats", err)
+			Debug(0, "[OUTPUT-HTTP] error accessing file size", err)
 		}
 	}
 }
@@ -285,6 +305,8 @@ func (o *FileOutput) closeLocked() error {
 	}
 
 	o.closed = true
+	o.currentFileSize = 0
+
 	return nil
 }
 

output_file_test.go

@@ -1,8 +1,8 @@
-package main
+package goreplay
 
 import (
 	"fmt"
-	"io"
+	"github.com/buger/goreplay/internal/size"
 	"math/rand"
 	"os"
 	"reflect"
@@ -15,18 +15,17 @@ import (
 
 func TestFileOutput(t *testing.T) {
 	wg := new(sync.WaitGroup)
-	quit := make(chan int)
 
 	input := NewTestInput()
 	output := NewFileOutput("/tmp/test_requests.gor", &FileOutputConfig{FlushInterval: time.Minute, Append: true})
 
 	plugins := &InOutPlugins{
-		Inputs:  []io.Reader{input},
-		Outputs: []io.Writer{output},
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{output},
 	}
 	plugins.All = append(plugins.All, input, output)
 
-	emitter := NewEmitter(quit)
+	emitter := NewEmitter()
 	go emitter.Start(plugins, Settings.Middleware)
 
 	for i := 0; i < 100; i++ {
@@ -39,20 +38,19 @@ func TestFileOutput(t *testing.T) {
 	emitter.Close()
 
 	var counter int64
-	input2 := NewFileInput("/tmp/test_requests.gor", false)
-	output2 := NewTestOutput(func(data []byte) {
+	input2 := NewFileInput("/tmp/test_requests.gor", false, 100, 0, false)
+	output2 := NewTestOutput(func(*Message) {
 		atomic.AddInt64(&counter, 1)
 		wg.Done()
 	})
 
 	plugins2 := &InOutPlugins{
-		Inputs:  []io.Reader{input2},
-		Outputs: []io.Writer{output2},
+		Inputs:  []PluginReader{input2},
+		Outputs: []PluginWriter{output2},
 	}
 	plugins2.All = append(plugins2.All, input2, output2)
 
-	quit2 := make(chan int)
-	emitter2 := NewEmitter(quit2)
+	emitter2 := NewEmitter()
 	go emitter2.Start(plugins2, Settings.Middleware)
 
 	wg.Wait()
@@ -89,16 +87,16 @@ func TestFileOutputMultipleFiles(t *testing.T) {
 		t.Error("Should not initialize file if no writes")
 	}
 
-	output.Write([]byte("1 1 1\r\ntest"))
+	output.PluginWrite(&Message{Meta: []byte("1 1 1\r\n"), Data: []byte("test")})
 	name1 := output.file.Name()
 
-	output.Write([]byte("1 1 1\r\ntest"))
+	output.PluginWrite(&Message{Meta: []byte("1 1 1\r\n"), Data: []byte("test")})
 	name2 := output.file.Name()
 
 	time.Sleep(time.Second)
 	output.updateName()
 
-	output.Write([]byte("1 1 1\r\ntest"))
+	output.PluginWrite(&Message{Meta: []byte("1 1 1\r\n"), Data: []byte("test")})
 	name3 := output.file.Name()
 
 	if name2 != name1 {
@@ -120,16 +118,16 @@ func TestFileOutputFilePerRequest(t *testing.T) {
 		t.Error("Should not initialize file if no writes")
 	}
 
-	output.Write([]byte("1 1 1\ntest"))
+	output.PluginWrite(&Message{Meta: []byte("1 1 1\r\n"), Data: []byte("test")})
 	name1 := output.file.Name()
 
-	output.Write([]byte("1 2 1\ntest"))
+	output.PluginWrite(&Message{Meta: []byte("1 2 1\r\n"), Data: []byte("test")})
 	name2 := output.file.Name()
 
 	time.Sleep(time.Second)
 	output.updateName()
 
-	output.Write([]byte("1 3 1\ntest"))
+	output.PluginWrite(&Message{Meta: []byte("1 3 1\r\n"), Data: []byte("test")})
 	name3 := output.file.Name()
 
 	if name3 == name2 || name2 == name1 || name3 == name1 {
@@ -149,7 +147,7 @@ func TestFileOutputCompression(t *testing.T) {
 	}
 
 	for i := 0; i < 1000; i++ {
-		output.Write([]byte("1 1 1\r\ntest"))
+		output.PluginWrite(&Message{Meta: []byte("1 1 1\r\n"), Data: []byte("test")})
 	}
 
 	name := output.file.Name()
@@ -163,30 +161,6 @@ func TestFileOutputCompression(t *testing.T) {
 	os.Remove(name)
 }
 
-func TestParseDataUnit(t *testing.T) {
-	var d = map[string]int64{
-		"42mb":                 42 << 20,
-		"4_2":                  42,
-		"00":                   0,
-		"\n\n 0.0\r\t\f":       0,
-		"0_600tb":              384 << 40,
-		"0600Tb":               384 << 40,
-		"0o12Mb":               10 << 20,
-		"0b_10010001111_1kb":   2335 << 10,
-		"1024":                 1 << 10,
-		"0b111":                7,
-		"0x12gB":               18 << 30,
-		"0x_67_7a_2f_cc_40_c6": 113774485586118,
-		"121562380192901":      121562380192901,
-	}
-	for k, v := range d {
-		n, err := bufferParser(k, "0")
-		if err != nil || n != v {
-			t.Errorf("Error parsing %s: %v", k, err)
-		}
-	}
-}
-
 func TestGetFileIndex(t *testing.T) {
 	var tests = []struct {
 		path  string
@@ -232,15 +206,15 @@ func TestFileOutputAppendQueueLimitOverflow(t *testing.T) {
 
 	output := NewFileOutput(name, &FileOutputConfig{Append: false, FlushInterval: time.Minute, QueueLimit: 2})
 
-	output.Write([]byte("1 1 1\r\ntest"))
+	output.PluginWrite(&Message{Meta: []byte("1 1 1\r\n"), Data: []byte("test")})
 	name1 := output.file.Name()
 
-	output.Write([]byte("1 1 1\r\ntest"))
+	output.PluginWrite(&Message{Meta: []byte("1 1 1\r\n"), Data: []byte("test")})
 	name2 := output.file.Name()
 
 	output.updateName()
 
-	output.Write([]byte("1 1 1\r\ntest"))
+	output.PluginWrite(&Message{Meta: []byte("1 1 1\r\n"), Data: []byte("test")})
 	name3 := output.file.Name()
 
 	if name2 != name1 || name1 != fmt.Sprintf("/tmp/%d_0", rnd) {
@@ -261,15 +235,15 @@ func TestFileOutputAppendQueueLimitNoOverflow(t *testing.T) {
 
 	output := NewFileOutput(name, &FileOutputConfig{Append: false, FlushInterval: time.Minute, QueueLimit: 3})
 
-	output.Write([]byte("1 1 1\r\ntest"))
+	output.PluginWrite(&Message{Meta: []byte("1 1 1\r\n"), Data: []byte("test")})
 	name1 := output.file.Name()
 
-	output.Write([]byte("1 1 1\r\ntest"))
+	output.PluginWrite(&Message{Meta: []byte("1 1 1\r\n"), Data: []byte("test")})
 	name2 := output.file.Name()
 
 	output.updateName()
 
-	output.Write([]byte("1 1 1\r\ntest"))
+	output.PluginWrite(&Message{Meta: []byte("1 1 1\r\n"), Data: []byte("test")})
 	name3 := output.file.Name()
 
 	if name2 != name1 || name1 != fmt.Sprintf("/tmp/%d_0", rnd) {
@@ -290,15 +264,15 @@ func TestFileOutputAppendQueueLimitGzips(t *testing.T) {
 
 	output := NewFileOutput(name, &FileOutputConfig{Append: false, FlushInterval: time.Minute, QueueLimit: 2})
 
-	output.Write([]byte("1 1 1\r\ntest"))
+	output.PluginWrite(&Message{Meta: []byte("1 1 1\r\n"), Data: []byte("test")})
 	name1 := output.file.Name()
 
-	output.Write([]byte("1 1 1\r\ntest"))
+	output.PluginWrite(&Message{Meta: []byte("1 1 1\r\n"), Data: []byte("test")})
 	name2 := output.file.Name()
 
 	output.updateName()
 
-	output.Write([]byte("1 1 1\r\ntest"))
+	output.PluginWrite(&Message{Meta: []byte("1 1 1\r\n"), Data: []byte("test")})
 	name3 := output.file.Name()
 
 	if name2 != name1 || name1 != fmt.Sprintf("/tmp/%d_0.gz", rnd) {
@@ -331,18 +305,17 @@ func TestFileOutputAppendSizeLimitOverflow(t *testing.T) {
 
 	messageSize := len(message) + len(payloadSeparator)
 
-	output := NewFileOutput(name, &FileOutputConfig{Append: false, FlushInterval: time.Minute, sizeLimit: 2 * int64(messageSize)})
+	output := NewFileOutput(name, &FileOutputConfig{Append: false, FlushInterval: time.Minute, SizeLimit: size.Size(2 * messageSize)})
 
-	output.Write([]byte("1 1 1\r\ntest"))
+	output.PluginWrite(&Message{Meta: []byte("1 1 1\r\n"), Data: []byte("test")})
 	name1 := output.file.Name()
 
-	output.Write([]byte("1 1 1\r\ntest"))
+	output.PluginWrite(&Message{Meta: []byte("1 1 1\r\n"), Data: []byte("test")})
 	name2 := output.file.Name()
 
 	output.flush()
-	output.updateName()
 
-	output.Write([]byte("1 1 1\r\ntest"))
+	output.PluginWrite(&Message{Meta: []byte("1 1 1\r\n"), Data: []byte("test")})
 	name3 := output.file.Name()
 
 	if name2 != name1 || name1 != fmt.Sprintf("/tmp/%d_0", rnd) {

output_http.go

@@ -1,39 +1,107 @@
-package main
+package goreplay
 
 import (
+	"bufio"
+	"bytes"
+	"crypto/tls"
 	"fmt"
-	"io"
 	"log"
+	"math"
+	"net/http"
+	"net/http/httputil"
+	"net/url"
 	"sync/atomic"
 	"time"
 
-	"github.com/buger/goreplay/proto"
+	"github.com/buger/goreplay/internal/size"
 )
 
-var _ = fmt.Println
+const (
+	initialDynamicWorkers = 10
+	readChunkSize         = 64 * 1024
+	maxResponseSize       = 1073741824
+)
 
-const initialDynamicWorkers = 10
+type response struct {
+	payload       []byte
+	uuid          []byte
+	startedAt     int64
+	roundTripTime int64
+}
+
+// HTTPOutputConfig struct for holding http output configuration
+type HTTPOutputConfig struct {
+	TrackResponses    bool          `json:"output-http-track-response"`
+	Stats             bool          `json:"output-http-stats"`
+	OriginalHost      bool          `json:"output-http-original-host"`
+	RedirectLimit     int           `json:"output-http-redirect-limit"`
+	WorkersMin        int           `json:"output-http-workers-min"`
+	WorkersMax        int           `json:"output-http-workers"`
+	StatsMs           int           `json:"output-http-stats-ms"`
+	QueueLen          int           `json:"output-http-queue-len"`
+	ElasticSearch     string        `json:"output-http-elasticsearch"`
+	Timeout           time.Duration `json:"output-http-timeout"`
+	WorkerTimeout     time.Duration `json:"output-http-worker-timeout"`
+	BufferSize        size.Size     `json:"output-http-response-buffer"`
+	SkipVerify        bool          `json:"output-http-skip-verify"`
+	CompatibilityMode bool          `json:"output-http-compatibility-mode"`
+	RequestGroup      string        `json:"output-http-request-group"`
+	Debug             bool          `json:"output-http-debug"`
+	rawURL            string
+	url               *url.URL
+}
+
+func (hoc *HTTPOutputConfig) Copy() *HTTPOutputConfig {
+	return &HTTPOutputConfig{
+		TrackResponses:    hoc.TrackResponses,
+		Stats:             hoc.Stats,
+		OriginalHost:      hoc.OriginalHost,
+		RedirectLimit:     hoc.RedirectLimit,
+		WorkersMin:        hoc.WorkersMin,
+		WorkersMax:        hoc.WorkersMax,
+		StatsMs:           hoc.StatsMs,
+		QueueLen:          hoc.QueueLen,
+		ElasticSearch:     hoc.ElasticSearch,
+		Timeout:           hoc.Timeout,
+		WorkerTimeout:     hoc.WorkerTimeout,
+		BufferSize:        hoc.BufferSize,
+		SkipVerify:        hoc.SkipVerify,
+		CompatibilityMode: hoc.CompatibilityMode,
+		RequestGroup:      hoc.RequestGroup,
+		Debug:             hoc.Debug,
+	}
+}
+
+// HTTPOutput plugin manage pool of workers which send request to replayed server
+// By default workers pool is dynamic and starts with 1 worker or workerMin workers
+// You can specify maximum number of workers using `--output-http-workers`
+type HTTPOutput struct {
+	activeWorkers  int64
+	config         *HTTPOutputConfig
+	queueStats     *GorStat
+	elasticSearch  *ESPlugin
+	client         *HTTPClient
+	stopWorker     chan struct{}
+	queue          chan *Message
+	responses      chan *response
+	stop           chan bool // Channel used only to indicate goroutine should shutdown
+	workerSessions map[string]*httpWorker
+}
 
 type httpWorker struct {
 	output       *HTTPOutput
 	client       *HTTPClient
 	lastActivity time.Time
-	queue        chan []byte
+	queue        chan *Message
 	stop         chan bool
 }
 
-func newHTTPWorker(output *HTTPOutput, queue chan []byte) *httpWorker {
-	client := NewHTTPClient(output.address, &HTTPClientConfig{
-		FollowRedirects:    output.config.RedirectLimit,
-		Debug:              output.config.Debug,
-		OriginalHost:       output.config.OriginalHost,
-		Timeout:            output.config.Timeout,
-		ResponseBufferSize: output.config.BufferSize,
-	})
+func newHTTPWorker(output *HTTPOutput, queue chan *Message) *httpWorker {
+	client := NewHTTPClient(output.config)
 
-	w := &httpWorker{client: client}
+	w := &httpWorker{client: client, output: output}
 	if queue == nil {
-		w.queue = make(chan []byte, 100)
+		w.queue = make(chan *Message, 100)
 	} else {
 		w.queue = queue
 	}
@@ -42,8 +110,8 @@ func newHTTPWorker(output *HTTPOutput, queue chan []byte) *httpWorker {
 	go func() {
 		for {
 			select {
-			case payload := <-w.queue:
-				output.sendRequest(client, payload)
+			case msg := <-w.queue:
+				output.sendRequest(client, msg)
 			case <-w.stop:
 				return
 			}
@@ -53,103 +121,74 @@ func newHTTPWorker(output *HTTPOutput, queue chan []byte) *httpWorker {
 	return w
 }
 
-type response struct {
-	payload       []byte
-	uuid          []byte
-	roundTripTime int64
-	startedAt     int64
-}
-
-// HTTPOutputConfig struct for holding http output configuration
-type HTTPOutputConfig struct {
-	RedirectLimit int `json:"output-http-redirect-limit"`
-
-	Stats      bool `json:"output-http-stats"`
-	WorkersMin int  `json:"output-http-workers-min"`
-	WorkersMax int  `json:"output-http-workers"`
-	StatsMs    int  `json:"output-http-stats-ms"`
-	Workers    int
-	QueueLen   int `json:"output-http-queue-len"`
-
-	ElasticSearch string `json:"output-http-elasticsearch"`
-
-	Timeout      time.Duration `json:"output-http-timeout"`
-	OriginalHost bool          `json:"output-http-original-host"`
-	BufferSize   int           `json:"output-http-response-buffer"`
-
-	CompatibilityMode bool `json:"output-http-compatibility-mode"`
-
-	RequestGroup string
-
-	Debug bool `json:"output-http-debug"`
-
-	TrackResponses bool `json:"output-http-track-response"`
-}
-
-// HTTPOutput plugin manage pool of workers which send request to replayed server
-// By default workers pool is dynamic and starts with 10 workers
-// You can specify fixed number of workers using `--output-http-workers`
-type HTTPOutput struct {
-	// Keep this as first element of struct because it guarantees 64bit
-	// alignment. atomic.* functions crash on 32bit machines if operand is not
-	// aligned at 64bit. See https://github.com/golang/go/issues/599
-	activeWorkers int64
-
-	workerSessions map[string]*httpWorker
-
-	address string
-	limit   int
-	queue   chan []byte
-
-	responses chan response
-
-	needWorker chan int
-
-	config *HTTPOutputConfig
-
-	queueStats *GorStat
-
-	elasticSearch *ESPlugin
-
-	stop chan bool // Channel used only to indicate goroutine should shutdown
-}
-
 // NewHTTPOutput constructor for HTTPOutput
 // Initialize workers
-func NewHTTPOutput(address string, config *HTTPOutputConfig) io.Writer {
+func NewHTTPOutput(address string, config *HTTPOutputConfig) PluginReadWriter {
 	o := new(HTTPOutput)
-
-	o.address = address
-	o.config = config
+	var err error
+	newConfig := config.Copy()
+	newConfig.url, err = url.Parse(address)
+	if err != nil {
+		log.Fatal(fmt.Sprintf("[OUTPUT-HTTP] parse HTTP output URL error[%q]", err))
+	}
+	if newConfig.url.Scheme == "" {
+		newConfig.url.Scheme = "http"
+	}
+	newConfig.rawURL = newConfig.url.String()
+	if newConfig.Timeout < time.Millisecond*100 {
+		newConfig.Timeout = time.Second
+	}
+	if newConfig.BufferSize <= 0 {
+		newConfig.BufferSize = 100 * 1024 // 100kb
+	}
+	if newConfig.WorkersMin <= 0 {
+		newConfig.WorkersMin = 1
+	}
+	if newConfig.WorkersMin > 1000 {
+		newConfig.WorkersMin = 1000
+	}
+	if newConfig.WorkersMax <= 0 {
+		newConfig.WorkersMax = math.MaxInt32 // ideally so large
+	}
+	if newConfig.WorkersMax < newConfig.WorkersMin {
+		newConfig.WorkersMax = newConfig.WorkersMin
+	}
+	if newConfig.QueueLen <= 0 {
+		newConfig.QueueLen = 1000
+	}
+	if newConfig.RedirectLimit < 0 {
+		newConfig.RedirectLimit = 0
+	}
+	if newConfig.WorkerTimeout <= 0 {
+		newConfig.WorkerTimeout = time.Second * 2
+	}
+	o.config = newConfig
 	o.stop = make(chan bool)
-
 	if o.config.Stats {
 		o.queueStats = NewGorStat("output_http", o.config.StatsMs)
 	}
 
-	o.queue = make(chan []byte, o.config.QueueLen)
-	o.responses = make(chan response, o.config.QueueLen)
-	o.needWorker = make(chan int, 1)
-
-	// Initial workers count
-	if o.config.WorkersMax == 0 {
-		o.needWorker <- initialDynamicWorkers
-	} else {
-		o.needWorker <- o.config.WorkersMax
+	o.queue = make(chan *Message, o.config.QueueLen)
+	if o.config.TrackResponses {
+		o.responses = make(chan *response, o.config.QueueLen)
 	}
+	// it should not be buffered to avoid races
+	o.stopWorker = make(chan struct{})
 
 	if o.config.ElasticSearch != "" {
 		o.elasticSearch = new(ESPlugin)
 		o.elasticSearch.Init(o.config.ElasticSearch)
 	}
+	o.client = NewHTTPClient(o.config)
 
 	if Settings.RecognizeTCPSessions {
-		if !PRO {
-			log.Fatal("Detailed TCP sessions work only with PRO license")
-		}
 		o.workerSessions = make(map[string]*httpWorker, 100)
 		go o.sessionWorkerMaster()
 	} else {
+		o.activeWorkers += int64(o.config.WorkersMin)
+		for i := 0; i < o.config.WorkersMin; i++ {
+			go o.startWorker()
+		}
 		go o.workerMaster()
 	}
 
@@ -157,12 +196,29 @@ func NewHTTPOutput(address string, config *HTTPOutputConfig) io.Writer {
 }
 
 func (o *HTTPOutput) workerMaster() {
+	var timer = time.NewTimer(o.config.WorkerTimeout)
+	defer func() {
+		// recover from panics caused by trying to send in
+		// a closed chan(o.stopWorker)
+		recover()
+	}()
+	defer timer.Stop()
 	for {
-		newWorkers := <-o.needWorker
-		atomic.AddInt64(&o.activeWorkers, int64(newWorkers))
-		for i := 0; i < newWorkers; i++ {
-			go o.startWorker()
+		select {
+		case <-o.stop:
+			return
+		default:
+			<-timer.C
 		}
+		// rollback workers
+	rollback:
+		if atomic.LoadInt64(&o.activeWorkers) > int64(o.config.WorkersMin) && len(o.queue) < 1 {
+			// close one worker
+			o.stopWorker <- struct{}{}
+			atomic.AddInt64(&o.activeWorkers, -1)
+			goto rollback
+		}
+		timer.Reset(o.config.WorkerTimeout)
 	}
 }
 
@@ -171,8 +227,8 @@ func (o *HTTPOutput) sessionWorkerMaster() {
 
 	for {
 		select {
-		case p := <-o.queue:
-			id := payloadID(p)
+		case msg := <-o.queue:
+			id := payloadID(msg.Meta)
 			sessionID := string(id[0:20])
 			worker, ok := o.workerSessions[sessionID]
 
@@ -182,7 +238,7 @@ func (o *HTTPOutput) sessionWorkerMaster() {
 				o.workerSessions[sessionID] = worker
 			}
 
-			worker.queue <- p
+			worker.queue <- msg
 			worker.lastActivity = time.Now()
 		case <-gc:
 			now := time.Now()
@@ -199,50 +255,26 @@ func (o *HTTPOutput) sessionWorkerMaster() {
 }
 
 func (o *HTTPOutput) startWorker() {
-	client := NewHTTPClient(o.address, &HTTPClientConfig{
-		FollowRedirects:    o.config.RedirectLimit,
-		Debug:              o.config.Debug,
-		OriginalHost:       o.config.OriginalHost,
-		Timeout:            o.config.Timeout,
-		ResponseBufferSize: o.config.BufferSize,
-		CompatibilityMode:  o.config.CompatibilityMode,
-	})
-
 	for {
 		select {
-		case <-o.stop:
+		case <-o.stopWorker:
 			return
-		case data := <-o.queue:
-			o.sendRequest(client, data)
-		case <-time.After(2 * time.Second):
-			// When dynamic scaling enabled workers die after 2s of inactivity
-			if o.config.WorkersMin == o.config.WorkersMax {
-				continue
-			}
-
-			workersCount := int(atomic.LoadInt64(&o.activeWorkers))
-
-			// At least 1 startWorker should be alive
-			if workersCount != 1 && workersCount > o.config.WorkersMin {
-				atomic.AddInt64(&o.activeWorkers, -1)
-				return
-			}
+		case msg := <-o.queue:
+			o.sendRequest(o.client, msg)
 		}
 	}
 }
 
-func (o *HTTPOutput) Write(data []byte) (n int, err error) {
-	if !isRequestPayload(data) {
-		return len(data), nil
+// PluginWrite writes message to this plugin
+func (o *HTTPOutput) PluginWrite(msg *Message) (n int, err error) {
+	if !isRequestPayload(msg.Meta) {
+		return len(msg.Data), nil
 	}
 
-	buf := make([]byte, len(data))
-	copy(buf, data)
-
 	select {
 	case <-o.stop:
 		return 0, ErrorStopped
-	case o.queue <- buf:
+	case o.queue <- msg:
 	}
 
 	if o.config.Stats {
@@ -259,74 +291,147 @@ func (o *HTTPOutput) Write(data []byte) (n int, err error) {
 				extraWorkersReq = maxWorkersAvailable
 			}
 			if extraWorkersReq > 0 {
-				o.needWorker <- extraWorkersReq
+				for i := 0; i < extraWorkersReq; i++ {
+					go o.startWorker()
+					atomic.AddInt64(&o.activeWorkers, 1)
+				}
 			}
 		}
 	}
 
-	return len(data), nil
+	return len(msg.Data) + len(msg.Meta), nil
 }
 
-func (o *HTTPOutput) Read(data []byte) (int, error) {
-	var resp response
+// PluginRead reads message from this plugin
+func (o *HTTPOutput) PluginRead() (*Message, error) {
+	if !o.config.TrackResponses {
+		return nil, ErrorStopped
+	}
+	var resp *response
+	var msg Message
 	select {
 	case <-o.stop:
-		return 0, ErrorStopped
+		return nil, ErrorStopped
 	case resp = <-o.responses:
+		msg.Data = resp.payload
 	}
 
-	if Settings.Debug {
-		Debug("[OUTPUT-HTTP] Received response:", string(resp.payload))
-	}
+	msg.Meta = payloadHeader(ReplayedResponsePayload, resp.uuid, resp.startedAt, resp.roundTripTime)
 
-	header := payloadHeader(ReplayedResponsePayload, resp.uuid, resp.roundTripTime, resp.startedAt)
-	copy(data[0:len(header)], header)
-	copy(data[len(header):], resp.payload)
-
-	return len(resp.payload) + len(header), nil
+	return &msg, nil
 }
 
-func (o *HTTPOutput) sendRequest(client *HTTPClient, request []byte) {
-	meta := payloadMeta(request)
-
-	if Settings.Debug {
-		Debug(meta)
-	}
-
-	if len(meta) < 2 {
-		return
-	}
-	uuid := meta[1]
-
-	body := payloadBody(request)
-	if !proto.IsHTTPPayload(body) {
+func (o *HTTPOutput) sendRequest(client *HTTPClient, msg *Message) {
+	if !isRequestPayload(msg.Meta) {
 		return
 	}
 
+	uuid := payloadID(msg.Meta)
 	start := time.Now()
-	resp, err := client.Send(body)
+	resp, err := client.Send(msg.Data)
 	stop := time.Now()
 
 	if err != nil {
-		log.Println("Error when sending ", err, time.Now())
-		Debug("Request error:", err)
+		Debug(1, fmt.Sprintf("[HTTP-OUTPUT] error when sending: %q", err))
+		return
+	}
+	if resp == nil {
+		return
 	}
 
 	if o.config.TrackResponses {
-		o.responses <- response{resp, uuid, start.UnixNano(), stop.UnixNano() - start.UnixNano()}
+		o.responses <- &response{resp, uuid, start.UnixNano(), stop.UnixNano() - start.UnixNano()}
 	}
 
 	if o.elasticSearch != nil {
-		o.elasticSearch.ResponseAnalyze(request, resp, start, stop)
+		o.elasticSearch.ResponseAnalyze(msg.Data, resp, start, stop)
 	}
 }
 
 func (o *HTTPOutput) String() string {
-	return "HTTP output: " + o.address
+	return "HTTP output: " + o.config.rawURL
 }
 
 // Close closes the data channel so that data
 func (o *HTTPOutput) Close() error {
 	close(o.stop)
+	close(o.stopWorker)
 	return nil
 }
+
+// HTTPClient holds configurations for a single HTTP client
+type HTTPClient struct {
+	config *HTTPOutputConfig
+	Client *http.Client
+}
+
+// NewHTTPClient returns new http client with check redirects policy
+func NewHTTPClient(config *HTTPOutputConfig) *HTTPClient {
+	client := new(HTTPClient)
+	client.config = config
+	var transport *http.Transport
+	client.Client = &http.Client{
+		Timeout: client.config.Timeout,
+		CheckRedirect: func(req *http.Request, via []*http.Request) error {
+			if len(via) >= client.config.RedirectLimit {
+				Debug(1, fmt.Sprintf("[HTTPCLIENT] maximum output-http-redirects[%d] reached!", client.config.RedirectLimit))
+				return http.ErrUseLastResponse
+			}
+			lastReq := via[len(via)-1]
+			resp := req.Response
+			Debug(2, fmt.Sprintf("[HTTPCLIENT] HTTP redirects from %q to %q with %q", lastReq.Host, req.Host, resp.Status))
+			return nil
+		},
+	}
+	if config.SkipVerify {
+		// clone to avoid modifying global default RoundTripper
+		transport = http.DefaultTransport.(*http.Transport).Clone()
+		transport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true}
+		client.Client.Transport = transport
+	}
+
+	return client
+}
+
+// Send sends an http request using client created by NewHTTPClient
+func (c *HTTPClient) Send(data []byte) ([]byte, error) {
+	var req *http.Request
+	var resp *http.Response
+	var err error
+
+	req, err = http.ReadRequest(bufio.NewReader(bytes.NewReader(data)))
+	if err != nil {
+		return nil, err
+	}
+	// we don't send CONNECT or OPTIONS request
+	if req.Method == http.MethodConnect {
+		return nil, nil
+	}
+
+	if !c.config.OriginalHost {
+		req.Host = c.config.url.Host
+	}
+
+	// fix #862
+	if c.config.url.Path == "" && c.config.url.RawQuery == "" {
+		req.URL.Scheme = c.config.url.Scheme
+		req.URL.Host = c.config.url.Host
+	} else {
+		req.URL = c.config.url
+	}
+
+	// force connection to not be closed, which can affect the global client
+	req.Close = false
+	// it's an error if this is not equal to empty string
+	req.RequestURI = ""
+
+	resp, err = c.Client.Do(req)
+	if err != nil {
+		return nil, err
+	}
+	if c.config.TrackResponses {
+		return httputil.DumpResponse(resp, true)
+	}
+	_ = resp.Body.Close()
+	return nil, nil
+}

output_http_test.go

@@ -1,20 +1,16 @@
-package main
+package goreplay
 
 import (
-	"io"
 	"io/ioutil"
 	"net/http"
 	"net/http/httptest"
 	_ "net/http/httputil"
 	"sync"
-	"sync/atomic"
 	"testing"
-	"time"
 )
 
 func TestHTTPOutput(t *testing.T) {
 	wg := new(sync.WaitGroup)
-	quit := make(chan int)
 
 	input := NewTestInput()
 
@@ -40,47 +36,40 @@ func TestHTTPOutput(t *testing.T) {
 	}))
 	defer server.Close()
 
-	headers := HTTPHeaders{HTTPHeader{"User-Agent", "Gor"}}
+	headers := HTTPHeaders{httpHeader{"User-Agent", "Gor"}}
 	methods := HTTPMethods{[]byte("GET"), []byte("PUT"), []byte("POST")}
 	Settings.ModifierConfig = HTTPModifierConfig{Headers: headers, Methods: methods}
 
-	http_output := NewHTTPOutput(server.URL, &HTTPOutputConfig{Debug: true, TrackResponses: true})
-	output := NewTestOutput(func(data []byte) {
+	httpOutput := NewHTTPOutput(server.URL, &HTTPOutputConfig{TrackResponses: false})
+	output := NewTestOutput(func(*Message) {
 		wg.Done()
 	})
 
 	plugins := &InOutPlugins{
-		Inputs:  []io.Reader{input},
-		Outputs: []io.Writer{http_output, output},
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{httpOutput, output},
 	}
-	plugins.All = append(plugins.All, input, output, http_output)
+	plugins.All = append(plugins.All, input, output, httpOutput)
 
-	emitter := NewEmitter(quit)
+	emitter := NewEmitter()
 	go emitter.Start(plugins, Settings.Middleware)
 
 	for i := 0; i < 10; i++ {
-		// 2 http-output, 2 - test output request, 2 - test output http response
-		wg.Add(6) // OPTIONS should be ignored
+		// 2 http-output, 2 - test output request
+		wg.Add(4) // OPTIONS should be ignored
 		input.EmitPOST()
 		input.EmitOPTIONS()
 		input.EmitGET()
 	}
 
 	wg.Wait()
-	close(quit)
-
-	activeWorkers := atomic.LoadInt64(&http_output.(*HTTPOutput).activeWorkers)
-
-	if activeWorkers < 50 {
-		t.Error("Should create workers for each request", activeWorkers)
-	}
+	emitter.Close()
 
 	Settings.ModifierConfig = HTTPModifierConfig{}
 }
 
 func TestHTTPOutputKeepOriginalHost(t *testing.T) {
 	wg := new(sync.WaitGroup)
-	quit := make(chan int)
 
 	input := NewTestInput()
 
@@ -93,18 +82,18 @@ func TestHTTPOutputKeepOriginalHost(t *testing.T) {
 	}))
 	defer server.Close()
 
-	headers := HTTPHeaders{HTTPHeader{"Host", "custom-host.com"}}
+	headers := HTTPHeaders{httpHeader{"Host", "custom-host.com"}}
 	Settings.ModifierConfig = HTTPModifierConfig{Headers: headers}
 
-	output := NewHTTPOutput(server.URL, &HTTPOutputConfig{Debug: false, OriginalHost: true})
+	output := NewHTTPOutput(server.URL, &HTTPOutputConfig{OriginalHost: true, SkipVerify: true})
 
 	plugins := &InOutPlugins{
-		Inputs:  []io.Reader{input},
-		Outputs: []io.Writer{output},
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{output},
 	}
 	plugins.All = append(plugins.All, input, output)
 
-	emitter := NewEmitter(quit)
+	emitter := NewEmitter()
 	go emitter.Start(plugins, Settings.Middleware)
 
 	wg.Add(1)
@@ -117,7 +106,6 @@ func TestHTTPOutputKeepOriginalHost(t *testing.T) {
 
 func TestHTTPOutputSSL(t *testing.T) {
 	wg := new(sync.WaitGroup)
-	quit := make(chan int)
 
 	// Origing and Replay server initialization
 	server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
@@ -125,15 +113,15 @@ func TestHTTPOutputSSL(t *testing.T) {
 	}))
 
 	input := NewTestInput()
-	output := NewHTTPOutput(server.URL, &HTTPOutputConfig{})
+	output := NewHTTPOutput(server.URL, &HTTPOutputConfig{SkipVerify: true})
 
 	plugins := &InOutPlugins{
-		Inputs:  []io.Reader{input},
-		Outputs: []io.Writer{output},
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{output},
 	}
 	plugins.All = append(plugins.All, input, output)
 
-	emitter := NewEmitter(quit)
+	emitter := NewEmitter()
 	go emitter.Start(plugins, Settings.Middleware)
 
 	wg.Add(2)
@@ -147,7 +135,6 @@ func TestHTTPOutputSSL(t *testing.T) {
 
 func TestHTTPOutputSessions(t *testing.T) {
 	wg := new(sync.WaitGroup)
-	quit := make(chan int)
 
 	input := NewTestInput()
 	input.skipHeader = true
@@ -158,25 +145,27 @@ func TestHTTPOutputSessions(t *testing.T) {
 	defer server.Close()
 
 	Settings.RecognizeTCPSessions = true
-	output := NewHTTPOutput(server.URL, &HTTPOutputConfig{Debug: true})
+	Settings.SplitOutput = true
+	output := NewHTTPOutput(server.URL, &HTTPOutputConfig{})
 
 	plugins := &InOutPlugins{
-		Inputs:  []io.Reader{input},
-		Outputs: []io.Writer{output},
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{output},
 	}
-	emitter := NewEmitter(quit)
+	plugins.All = append(plugins.All, input, output)
+	emitter := NewEmitter()
 	go emitter.Start(plugins, Settings.Middleware)
 
 	uuid1 := []byte("1234567890123456789a0000")
 	uuid2 := []byte("1234567890123456789d0000")
 
-	for i := 0; i < 100; i++ {
+	for i := 0; i < 10; i++ {
 		wg.Add(1) // OPTIONS should be ignored
 		copy(uuid1[20:], randByte(4))
 		input.EmitBytes([]byte("1 " + string(uuid1) + " 1\n" + "GET / HTTP/1.1\r\n\r\n"))
 	}
 
-	for i := 0; i < 100; i++ {
+	for i := 0; i < 10; i++ {
 		wg.Add(1) // OPTIONS should be ignored
 		copy(uuid2[20:], randByte(4))
 		input.EmitBytes([]byte("1 " + string(uuid2) + " 1\n" + "GET / HTTP/1.1\r\n\r\n"))
@@ -184,35 +173,59 @@ func TestHTTPOutputSessions(t *testing.T) {
 
 	wg.Wait()
 
-	if output.(*HTTPOutput).activeWorkers != 2 {
-		t.Error("Should have only 2 workers", output.(*HTTPOutput).activeWorkers)
-	}
-
 	emitter.Close()
 
 	Settings.RecognizeTCPSessions = false
+	Settings.SplitOutput = false
 }
 
 func BenchmarkHTTPOutput(b *testing.B) {
 	wg := new(sync.WaitGroup)
-	quit := make(chan int)
 
-	server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		time.Sleep(50 * time.Millisecond)
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		wg.Done()
 	}))
 	defer server.Close()
 
 	input := NewTestInput()
-	output := NewHTTPOutput(server.URL, &HTTPOutputConfig{})
+	output := NewHTTPOutput(server.URL, &HTTPOutputConfig{WorkersMax: 1})
 
 	plugins := &InOutPlugins{
-		Inputs:  []io.Reader{input},
-		Outputs: []io.Writer{output},
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{output},
 	}
 	plugins.All = append(plugins.All, input, output)
 
-	emitter := NewEmitter(quit)
+	emitter := NewEmitter()
+	go emitter.Start(plugins, Settings.Middleware)
+
+	for i := 0; i < b.N; i++ {
+		wg.Add(1)
+		input.EmitPOST()
+	}
+
+	wg.Wait()
+	emitter.Close()
+}
+
+func BenchmarkHTTPOutputTLS(b *testing.B) {
+	wg := new(sync.WaitGroup)
+
+	server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		wg.Done()
+	}))
+	defer server.Close()
+
+	input := NewTestInput()
+	output := NewHTTPOutput(server.URL, &HTTPOutputConfig{SkipVerify: true, WorkersMax: 1})
+
+	plugins := &InOutPlugins{
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{output},
+	}
+	plugins.All = append(plugins.All, input, output)
+
+	emitter := NewEmitter()
 	go emitter.Start(plugins, Settings.Middleware)
 
 	for i := 0; i < b.N; i++ {

output_kafka.go

@@ -1,14 +1,13 @@
-package main
+package goreplay
 
 import (
 	"encoding/json"
-	"io"
+	"github.com/buger/goreplay/internal/byteutils"
+	"github.com/buger/goreplay/proto"
 	"log"
 	"strings"
 	"time"
 
-	"github.com/buger/goreplay/proto"
-
 	"github.com/Shopify/sarama"
 	"github.com/Shopify/sarama/mocks"
 )
@@ -22,9 +21,9 @@ type KafkaOutput struct {
 // KafkaOutputFrequency in milliseconds
 const KafkaOutputFrequency = 500
 
-// NewKafkaOutput creates instance of kafka producer client.
-func NewKafkaOutput(address string, config *OutputKafkaConfig) io.Writer {
-	c := sarama.NewConfig()
+// NewKafkaOutput creates instance of kafka producer client  with TLS config
+func NewKafkaOutput(_ string, config *OutputKafkaConfig, tlsConfig *KafkaTLSConfig) PluginWriter {
+	c := NewKafkaConfig(&config.SASLConfig, tlsConfig)
 
 	var producer sarama.AsyncProducer
 
@@ -49,10 +48,8 @@ func NewKafkaOutput(address string, config *OutputKafkaConfig) io.Writer {
 		producer: producer,
 	}
 
-	if Settings.Verbose {
-		// Start infinite loop for tracking errors for kafka producer.
-		go o.ErrorHandler()
-	}
+	// Start infinite loop for tracking errors for kafka producer.
+	go o.ErrorHandler()
 
 	return o
 }
@@ -60,36 +57,37 @@ func NewKafkaOutput(address string, config *OutputKafkaConfig) io.Writer {
 // ErrorHandler should receive errors
 func (o *KafkaOutput) ErrorHandler() {
 	for err := range o.producer.Errors() {
-		log.Println("Failed to write access log entry:", err)
+		Debug(1, "Failed to write access log entry:", err)
 	}
 }
 
-func (o *KafkaOutput) Write(data []byte) (n int, err error) {
+// PluginWrite writes a message to this plugin
+func (o *KafkaOutput) PluginWrite(msg *Message) (n int, err error) {
 	var message sarama.StringEncoder
 
 	if !o.config.UseJSON {
-		message = sarama.StringEncoder(data)
+		message = sarama.StringEncoder(byteutils.SliceToString(msg.Meta) + byteutils.SliceToString(msg.Data))
 	} else {
-		headers := make(map[string]string)
-		proto.ParseHeaders([][]byte{data}, func(header []byte, value []byte) bool {
-			headers[string(header)] = string(value)
-			return true
-		})
+		mimeHeader := proto.ParseHeaders(msg.Data)
+		header := make(map[string]string)
+		for k, v := range mimeHeader {
+			header[k] = strings.Join(v, ", ")
+		}
 
-		meta := payloadMeta(data)
-		req := payloadBody(data)
+		meta := payloadMeta(msg.Meta)
+		req := msg.Data
 
 		kafkaMessage := KafkaMessage{
-			ReqURL:     string(proto.Path(req)),
-			ReqType:    string(meta[0]),
-			ReqID:      string(meta[1]),
-			ReqTs:      string(meta[2]),
-			ReqMethod:  string(proto.Method(req)),
-			ReqBody:    string(proto.Body(req)),
-			ReqHeaders: headers,
+			ReqURL:     byteutils.SliceToString(proto.Path(req)),
+			ReqType:    byteutils.SliceToString(meta[0]),
+			ReqID:      byteutils.SliceToString(meta[1]),
+			ReqTs:      byteutils.SliceToString(meta[2]),
+			ReqMethod:  byteutils.SliceToString(proto.Method(req)),
+			ReqBody:    byteutils.SliceToString(proto.Body(req)),
+			ReqHeaders: header,
 		}
 		jsonMessage, _ := json.Marshal(&kafkaMessage)
-		message = sarama.StringEncoder(jsonMessage)
+		message = sarama.StringEncoder(byteutils.SliceToString(jsonMessage))
 	}
 
 	o.producer.Input() <- &sarama.ProducerMessage{

output_kafka_test.go

@@ -1,4 +1,4 @@
-package main
+package goreplay
 
 import (
 	"testing"
@@ -17,16 +17,16 @@ func TestOutputKafkaRAW(t *testing.T) {
 		producer: producer,
 		Topic:    "test",
 		UseJSON:  false,
-	})
+	}, nil)
 
-	output.Write([]byte("1 2 3\nGET / HTTP1.1\r\nHeader: 1\r\n\r\n"))
+	output.PluginWrite(&Message{Meta: []byte("1 2 3\n"), Data: []byte("GET / HTTP1.1\r\nHeader: 1\r\n\r\n")})
 
 	resp := <-producer.Successes()
 
 	data, _ := resp.Value.Encode()
 
 	if string(data) != "1 2 3\nGET / HTTP1.1\r\nHeader: 1\r\n\r\n" {
-		t.Error("Message not properly encoded: ", string(data))
+		t.Errorf("Message not properly encoded: %q", data)
 	}
 }
 
@@ -40,15 +40,15 @@ func TestOutputKafkaJSON(t *testing.T) {
 		producer: producer,
 		Topic:    "test",
 		UseJSON:  true,
-	})
+	}, nil)
 
-	output.Write([]byte("1 2 3\nGET / HTTP1.1\r\nHeader: 1\r\n\r\n"))
+	output.PluginWrite(&Message{Meta: []byte("1 2 3\n"), Data: []byte("GET / HTTP1.1\r\nHeader: 1\r\n\r\n")})
 
 	resp := <-producer.Successes()
 
 	data, _ := resp.Value.Encode()
 
-	if string(data) != `{"Req_URL":"/","Req_Type":"1","Req_ID":"2","Req_Ts":"3","Req_Method":"GET","Req_Headers":{"Header":"1"}}` {
+	if string(data) != `{"Req_URL":"","Req_Type":"1","Req_ID":"2","Req_Ts":"3","Req_Method":"GET"}` {
 		t.Error("Message not properly encoded: ", string(data))
 	}
 }

output_null.go

@@ -1,4 +1,4 @@
-package main
+package goreplay
 
 // NullOutput used for debugging, prints nothing
 type NullOutput struct {
@@ -9,8 +9,9 @@ func NewNullOutput() (o *NullOutput) {
 	return new(NullOutput)
 }
 
-func (o *NullOutput) Write(data []byte) (int, error) {
-	return len(data), nil
+// PluginWrite writes message to this plugin
+func (o *NullOutput) PluginWrite(msg *Message) (int, error) {
+	return len(msg.Data) + len(msg.Meta), nil
 }
 
 func (o *NullOutput) String() string {

output_s3.go

@@ -1,129 +1,29 @@
-package main
+//go:build !pro
+
+package goreplay
 
 import (
-	_ "bufio"
+	"errors"
 	"fmt"
-	_ "io"
-	"log"
-	"math/rand"
-	"os"
-	"path/filepath"
-	"strings"
-
-	"github.com/aws/aws-sdk-go/aws"
-	"github.com/aws/aws-sdk-go/aws/session"
-	"github.com/aws/aws-sdk-go/service/s3"
-	_ "github.com/aws/aws-sdk-go/service/s3/s3manager"
 )
 
 // S3Output output plugin
-type S3Output struct {
-	pathTemplate string
-
-	buffer  *FileOutput
-	session *session.Session
-	config  *FileOutputConfig
-	closeCh chan struct{}
-}
+type S3Output struct{}
 
 // NewS3Output constructor for FileOutput, accepts path
 func NewS3Output(pathTemplate string, config *FileOutputConfig) *S3Output {
-	if !PRO {
-		log.Fatal("Using S3 output and input requires PRO license")
-		return nil
-	}
-
-	o := new(S3Output)
-	o.pathTemplate = pathTemplate
-	o.config = config
-	o.config.onClose = o.onBufferUpdate
-
-	if config.BufferPath == "" {
-		config.BufferPath = "/tmp"
-	}
-
-	rnd := rand.Int63()
-	bufferName := fmt.Sprintf("gor_output_s3_%d_buf_", rnd)
-
-	pathParts := strings.Split(pathTemplate, "/")
-	bufferName += pathParts[len(pathParts)-1]
-
-	if strings.HasSuffix(o.pathTemplate, ".gz") {
-		bufferName += ".gz"
-	}
-
-	bufferPath := filepath.Join(config.BufferPath, bufferName)
-
-	o.buffer = NewFileOutput(bufferPath, config)
-	o.connect()
-
-	return o
+	fmt.Println("S3 output is only available in the pro version")
+	return &S3Output{}
 }
 
-func (o *S3Output) connect() {
-	if o.session == nil {
-		o.session = session.Must(session.NewSession(awsConfig()))
-		log.Println("[S3 Output] S3 connection succesfully initialized")
-	}
-}
-
-func (o *S3Output) Write(data []byte) (n int, err error) {
-	return o.buffer.Write(data)
+func (o *S3Output) PluginWrite(msg *Message) (n int, err error) {
+	return 0, errors.New("S3 output is only available in the pro version")
 }
 
 func (o *S3Output) String() string {
-	return "S3 output: " + o.pathTemplate
+	return "S3 output (pro version only)"
 }
 
-// Close close the buffer of the S3 connection
 func (o *S3Output) Close() error {
-	return o.buffer.Close()
-}
-
-func parseS3Url(path string) (bucket, key string) {
-	path = path[5:] // stripping `s3://`
-	sep := strings.IndexByte(path, '/')
-
-	bucket = path[:sep]
-	key = path[sep+1:]
-
-	return bucket, key
-}
-
-func (o *S3Output) keyPath(idx int) (bucket, key string) {
-	bucket, key = parseS3Url(o.pathTemplate)
-
-	for name, fn := range dateFileNameFuncs {
-		key = strings.Replace(key, name, fn(o.buffer), -1)
-	}
-
-	key = setFileIndex(key, idx)
-
-	return
-}
-
-func (o *S3Output) onBufferUpdate(path string) {
-	svc := s3.New(o.session)
-	idx := getFileIndex(path)
-	bucket, key := o.keyPath(idx)
-
-	file, _ := os.Open(path)
-	// reader := bufio.NewReader(file)
-
-	_, err := svc.PutObject(&s3.PutObjectInput{
-		Body:   file,
-		Bucket: aws.String(bucket),
-		Key:    aws.String(key),
-	})
-	if err != nil {
-		log.Printf("[S3 Output] Failed to upload data to %s/%s, %s\n", bucket, key, err)
-		os.Remove(path)
-		return
-	}
-
-	os.Remove(path)
-
-	if o.closeCh != nil {
-		o.closeCh <- struct{}{}
-	}
+	return errors.New("S3 output is only available in the pro version")
 }

output_s3_pro.go

@@ -0,0 +1,117 @@
+//go:build pro
+
+package goreplay
+
+import (
+	_ "bufio"
+	"fmt"
+	_ "io"
+	"log"
+	"math/rand"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/s3"
+	_ "github.com/aws/aws-sdk-go/service/s3/s3manager"
+)
+
+var _ PluginWriter = (*S3Output)(nil)
+
+// S3Output output plugin
+type S3Output struct {
+	pathTemplate string
+
+	buffer  *FileOutput
+	session *session.Session
+	config  *FileOutputConfig
+	closeC  chan struct{}
+}
+
+// NewS3Output constructor for FileOutput, accepts path
+func NewS3Output(pathTemplate string, config *FileOutputConfig) *S3Output {
+	o := new(S3Output)
+	o.pathTemplate = pathTemplate
+	o.config = config
+	o.config.onClose = o.onBufferUpdate
+
+	if config.BufferPath == "" {
+		config.BufferPath = "/tmp"
+	}
+
+	rnd := rand.Int63()
+	buffer_name := fmt.Sprintf("gor_output_s3_%d_buf_", rnd)
+
+	pathParts := strings.Split(pathTemplate, "/")
+	buffer_name += pathParts[len(pathParts)-1]
+
+	if strings.HasSuffix(o.pathTemplate, ".gz") {
+		buffer_name += ".gz"
+	}
+
+	buffer_path := filepath.Join(config.BufferPath, buffer_name)
+
+	o.buffer = NewFileOutput(buffer_path, config)
+	o.connect()
+
+	return o
+}
+
+func (o *S3Output) connect() {
+	if o.session == nil {
+		o.session = session.Must(session.NewSession(awsConfig()))
+		log.Println("[S3 Output] S3 connection succesfully initialized")
+	}
+}
+
+func (o *S3Output) PluginWrite(msg *Message) (n int, err error) {
+	return o.buffer.PluginWrite(msg)
+}
+
+func (o *S3Output) String() string {
+	return "S3 output: " + o.pathTemplate
+}
+
+func (o *S3Output) Close() error {
+	return o.buffer.Close()
+}
+
+func (o *S3Output) keyPath(idx int) (bucket, key string) {
+	bucket, key = parseS3Url(o.pathTemplate)
+
+	for name, fn := range dateFileNameFuncs {
+		key = strings.Replace(key, name, fn(o.buffer), -1)
+	}
+
+	key = setFileIndex(key, idx)
+
+	return
+}
+
+func (o *S3Output) onBufferUpdate(path string) {
+	svc := s3.New(o.session)
+	idx := getFileIndex(path)
+	bucket, key := o.keyPath(idx)
+
+	file, _ := os.Open(path)
+	// reader := bufio.NewReader(file)
+
+	_, err := svc.PutObject(&s3.PutObjectInput{
+		Body:   file,
+		Bucket: aws.String(bucket),
+		Key:    aws.String(key),
+	})
+	if err != nil {
+		log.Printf("[S3 Output] Failed to upload data to %s/%s, %s\n", bucket, key, err)
+		os.Remove(path)
+		return
+	}
+
+	os.Remove(path)
+
+	if o.closeC != nil {
+		o.closeC <- struct{}{}
+	}
+}

output_tcp.go

@@ -1,34 +1,42 @@
-package main
+package goreplay
 
 import (
+	"context"
 	"crypto/tls"
 	"fmt"
 	"hash/fnv"
-	"io"
-	"log"
 	"net"
 	"time"
 )
 
 // TCPOutput used for sending raw tcp payloads
 // Currently used for internal communication between listener and replay server
-// Can be used for transfering binary payloads like protocol buffers
+// Can be used for transferring binary payloads like protocol buffers
 type TCPOutput struct {
-	address  string
-	limit    int
-	buf      []chan []byte
-	bufStats *GorStat
-	config   *TCPOutputConfig
+	address     string
+	limit       int
+	buf         []chan *Message
+	bufStats    *GorStat
+	config      *TCPOutputConfig
+	workerIndex uint32
+
+	close bool
 }
 
+// TCPOutputConfig tcp output configuration
 type TCPOutputConfig struct {
-	Secure bool `json:"output-tcp-secure"`
-	Sticky bool `json:"output-tcp-sticky"`
+	Secure     bool `json:"output-tcp-secure"`
+	Sticky     bool `json:"output-tcp-sticky"`
+	SkipVerify bool `json:"output-tcp-skip-verify"`
+	Workers    int  `json:"output-tcp-workers"`
+
+	GetInitMessage     func() *Message                         `json:"-"`
+	WriteBeforeMessage func(conn net.Conn, msg *Message) error `json:"-"`
 }
 
 // NewTCPOutput constructor for TCPOutput
-// Initialize 10 workers which hold keep-alive connection
-func NewTCPOutput(address string, config *TCPOutputConfig) io.Writer {
+// Initialize X workers which hold keep-alive connection
+func NewTCPOutput(address string, config *TCPOutputConfig) PluginWriter {
 	o := new(TCPOutput)
 
 	o.address = address
@@ -38,34 +46,29 @@ func NewTCPOutput(address string, config *TCPOutputConfig) io.Writer {
 		o.bufStats = NewGorStat("output_tcp", 5000)
 	}
 
-	if o.config.Sticky {
-		// create 10 buffers and send the buffer index to the worker
-		o.buf = make([]chan []byte, 10)
-		for i := 0; i < 10; i++ {
-			o.buf[i] = make(chan []byte, 100)
-			go o.worker(i)
-		}
-	} else {
-		// create 1 buffer and send its index (0) to all workers
-		o.buf = make([]chan []byte, 1)
-		o.buf[0] = make(chan []byte, 1000)
-		for i := 0; i < 10; i++ {
-			go o.worker(0)
-		}
+	// create X buffers and send the buffer index to the worker
+	o.buf = make([]chan *Message, o.config.Workers)
+	for i := 0; i < o.config.Workers; i++ {
+		o.buf[i] = make(chan *Message, 100)
+		go o.worker(i)
 	}
 
 	return o
 }
 
 func (o *TCPOutput) worker(bufferIndex int) {
-	retries := 1
+	retries := 0
 	conn, err := o.connect(o.address)
 	for {
+		if o.close {
+			return
+		}
+
 		if err == nil {
 			break
 		}
 
-		log.Println("Can't connect to aggregator instance, reconnecting in 1 second. Retries:", retries)
+		Debug(1, fmt.Sprintf("Can't connect to aggregator instance, reconnecting in 1 second. Retries:%d", retries))
 		time.Sleep(1 * time.Second)
 
 		conn, err = o.connect(o.address)
@@ -73,59 +76,83 @@ func (o *TCPOutput) worker(bufferIndex int) {
 	}
 
 	if retries > 0 {
-		log.Println("Connected to aggregator instance after ", retries, " retries")
+		Debug(2, fmt.Sprintf("Connected to aggregator instance after %d retries", retries))
 	}
 
 	defer conn.Close()
 
-	for {
-		data := <-o.buf[bufferIndex]
-		conn.Write(data)
-		_, err := conn.Write([]byte(payloadSeparator))
+	if o.config.GetInitMessage != nil {
+		msg := o.config.GetInitMessage()
+		_ = o.writeToConnection(conn, msg)
+	}
 
+	for {
+		msg := <-o.buf[bufferIndex]
+		err = o.writeToConnection(conn, msg)
 		if err != nil {
-			log.Println("INFO: TCP output connection closed, reconnecting")
-			o.buf[bufferIndex] <- data
+			Debug(2, "INFO: TCP output connection closed, reconnecting")
 			go o.worker(bufferIndex)
+			o.buf[bufferIndex] <- msg
 			break
 		}
 	}
 }
 
-func (o *TCPOutput) getBufferIndex(data []byte) int {
+func (o *TCPOutput) writeToConnection(conn net.Conn, msg *Message) (err error) {
+	if o.config.WriteBeforeMessage != nil {
+		err = o.config.WriteBeforeMessage(conn, msg)
+	}
+
+	if err == nil {
+		if _, err = conn.Write(msg.Meta); err == nil {
+			if _, err = conn.Write(msg.Data); err == nil {
+				_, err = conn.Write(payloadSeparatorAsBytes)
+			}
+		}
+	}
+
+	return err
+}
+
+func (o *TCPOutput) getBufferIndex(msg *Message) int {
 	if !o.config.Sticky {
-		return 0
+		o.workerIndex++
+		return int(o.workerIndex) % o.config.Workers
 	}
 
 	hasher := fnv.New32a()
-	hasher.Write(payloadMeta(data)[1])
-	return int(hasher.Sum32()) % 10
+	hasher.Write(payloadID(msg.Meta))
+	return int(hasher.Sum32()) % o.config.Workers
 }
 
-func (o *TCPOutput) Write(data []byte) (n int, err error) {
-	if !isOriginPayload(data) {
-		return len(data), nil
+// PluginWrite writes message to this plugin
+func (o *TCPOutput) PluginWrite(msg *Message) (n int, err error) {
+	if !isOriginPayload(msg.Meta) {
+		return len(msg.Data), nil
 	}
 
-	// We have to copy, because sending data in multiple threads
-	newBuf := make([]byte, len(data))
-	copy(newBuf, data)
-
-	bufferIndex := o.getBufferIndex(data)
-	o.buf[bufferIndex] <- newBuf
+	bufferIndex := o.getBufferIndex(msg)
+	o.buf[bufferIndex] <- msg
 
 	if Settings.OutputTCPStats {
 		o.bufStats.Write(len(o.buf[bufferIndex]))
 	}
 
-	return len(data), nil
+	return len(msg.Data) + len(msg.Meta), nil
 }
 
 func (o *TCPOutput) connect(address string) (conn net.Conn, err error) {
 	if o.config.Secure {
-		conn, err = tls.Dial("tcp", address, &tls.Config{})
+		var d tls.Dialer
+		d.Config = &tls.Config{InsecureSkipVerify: o.config.SkipVerify}
+		ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+		defer cancel()
+		conn, err = d.DialContext(ctx, "tcp", address)
 	} else {
-		conn, err = net.Dial("tcp", address)
+		var d net.Dialer
+		ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+		defer cancel()
+		conn, err = d.DialContext(ctx, "tcp", address)
 	}
 
 	return
@@ -134,3 +161,7 @@ func (o *TCPOutput) connect(address string) (conn net.Conn, err error) {
 func (o *TCPOutput) String() string {
 	return fmt.Sprintf("TCP output %s, limit: %d", o.address, o.limit)
 }
+
+func (o *TCPOutput) Close() {
+	o.close = true
+}

output_tcp_test.go

@@ -1,41 +1,25 @@
-package main
+package goreplay
 
 import (
 	"bufio"
-	"io"
 	"log"
 	"net"
+	"strings"
 	"sync"
 	"testing"
 	"time"
+
+	"github.com/stretchr/testify/assert"
 )
 
 func TestTCPOutput(t *testing.T) {
 	wg := new(sync.WaitGroup)
-	quit := make(chan int)
 
 	listener := startTCP(func(data []byte) {
 		wg.Done()
 	})
-	input := NewTestInput()
-	output := NewTCPOutput(listener.Addr().String(), &TCPOutputConfig{})
-
-	plugins := &InOutPlugins{
-		Inputs:  []io.Reader{input},
-		Outputs: []io.Writer{output},
-	}
-	plugins.All = append(plugins.All, input, output)
-
-	emitter := NewEmitter(quit)
-	go emitter.Start(plugins, Settings.Middleware)
-
-	for i := 0; i < 100; i++ {
-		wg.Add(1)
-		input.EmitGET()
-	}
-
-	wg.Wait()
-	emitter.Close()
+	output := NewTCPOutput(listener.Addr().String(), &TCPOutputConfig{Workers: 10})
+	runTCPOutput(wg, output, 10, false)
 }
 
 func startTCP(cb func([]byte)) net.Listener {
@@ -48,9 +32,9 @@ func startTCP(cb func([]byte)) net.Listener {
 	go func() {
 		for {
 			conn, _ := listener.Accept()
-			defer conn.Close()
 
-			go func() {
+			go func(conn net.Conn) {
+				defer conn.Close()
 				reader := bufio.NewReader(conn)
 				scanner := bufio.NewScanner(reader)
 				scanner.Split(payloadScanner)
@@ -58,7 +42,7 @@ func startTCP(cb func([]byte)) net.Listener {
 				for scanner.Scan() {
 					cb(scanner.Bytes())
 				}
-			}()
+			}(conn)
 		}
 	}()
 
@@ -67,51 +51,50 @@ func startTCP(cb func([]byte)) net.Listener {
 
 func BenchmarkTCPOutput(b *testing.B) {
 	wg := new(sync.WaitGroup)
-	quit := make(chan int)
 
 	listener := startTCP(func(data []byte) {
 		wg.Done()
 	})
 	input := NewTestInput()
-	output := NewTCPOutput(listener.Addr().String(), &TCPOutputConfig{})
-
-	plugins := &InOutPlugins{
-		Inputs:  []io.Reader{input},
-		Outputs: []io.Writer{output},
-	}
-	plugins.All = append(plugins.All, input, output)
-
-	emitter := NewEmitter(quit)
-	go emitter.Start(plugins, Settings.Middleware)
-
-	b.ResetTimer()
+	input.data = make(chan []byte, b.N)
 	for i := 0; i < b.N; i++ {
-		wg.Add(1)
 		input.EmitGET()
 	}
+	wg.Add(b.N)
+	output := NewTCPOutput(listener.Addr().String(), &TCPOutputConfig{Workers: 10})
+
+	plugins := &InOutPlugins{
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{output},
+	}
+
+	emitter := NewEmitter()
+	// avoid counting above initialization
+	b.ResetTimer()
+	go emitter.Start(plugins, Settings.Middleware)
 
 	wg.Wait()
 	emitter.Close()
 }
 
 func TestStickyDisable(t *testing.T) {
-	tcpOutput := TCPOutput{config: &TCPOutputConfig{Sticky: false}}
+	tcpOutput := TCPOutput{config: &TCPOutputConfig{Sticky: false, Workers: 10}}
 
-	for i := 0; i < 1000; i++ {
+	for i := 0; i < 10; i++ {
 		index := tcpOutput.getBufferIndex(getTestBytes())
-		if index != 0 {
-			t.Errorf("Sticky is disable. Got: %d want 0", index)
+		if index != (i+1)%10 {
+			t.Errorf("Sticky is disable. Got: %d want %d", index, (i+1)%10)
 		}
 	}
 }
 
 func TestBufferDistribution(t *testing.T) {
 	numberOfWorkers := 10
-	numberOfMessages := 1000000
+	numberOfMessages := 10000
 	percentDistributionErrorRange := 20
 
 	buffer := make([]int, numberOfWorkers)
-	tcpOutput := TCPOutput{config: &TCPOutputConfig{Sticky: true}}
+	tcpOutput := TCPOutput{config: &TCPOutputConfig{Sticky: true, Workers: 10}}
 	for i := 0; i < numberOfMessages; i++ {
 		buffer[tcpOutput.getBufferIndex(getTestBytes())]++
 	}
@@ -129,8 +112,82 @@ func TestBufferDistribution(t *testing.T) {
 	}
 }
 
-func getTestBytes() []byte {
-	reqh := payloadHeader(RequestPayload, uuid(), time.Now().UnixNano(), -1)
-	reqb := append(reqh, []byte("GET / HTTP/1.1\r\nHost: www.w3.org\r\nUser-Agent: Go 1.1 package http\r\nAccept-Encoding: gzip\r\n\r\n")...)
-	return reqb
+func getTestBytes() *Message {
+	return &Message{
+		Meta: payloadHeader(RequestPayload, uuid(), time.Now().UnixNano(), -1),
+		Data: []byte("GET / HTTP/1.1\r\nHost: www.w3.org\r\nUser-Agent: Go 1.1 package http\r\nAccept-Encoding: gzip\r\n\r\n"),
+	}
+}
+
+func TestTCPOutputGetInitMessage(t *testing.T) {
+	wg := new(sync.WaitGroup)
+
+	var dataList [][]byte
+	listener := startTCP(func(data []byte) {
+		dataList = append(dataList, data)
+		wg.Done()
+	})
+	getInitMessage := func() *Message {
+		return &Message{
+			Meta: []byte{},
+			Data: []byte("test1"),
+		}
+	}
+	output := NewTCPOutput(listener.Addr().String(), &TCPOutputConfig{Workers: 1, GetInitMessage: getInitMessage})
+
+	runTCPOutput(wg, output, 1, true)
+
+	if assert.Equal(t, 2, len(dataList)) {
+		assert.Equal(t, "test1", string(dataList[0]))
+	}
+}
+
+func TestTCPOutputGetInitMessageAndWriteBeforeMessage(t *testing.T) {
+	wg := new(sync.WaitGroup)
+
+	var dataList [][]byte
+	listener := startTCP(func(data []byte) {
+		dataList = append(dataList, data)
+		wg.Done()
+	})
+	getInitMessage := func() *Message {
+		return &Message{
+			Meta: []byte{},
+			Data: []byte("test2"),
+		}
+	}
+	writeBeforeMessage := func(conn net.Conn, _ *Message) error {
+		_, err := conn.Write([]byte("before"))
+		return err
+	}
+	output := NewTCPOutput(listener.Addr().String(), &TCPOutputConfig{Workers: 1, GetInitMessage: getInitMessage, WriteBeforeMessage: writeBeforeMessage})
+
+	runTCPOutput(wg, output, 1, true)
+
+	if assert.Equal(t, 2, len(dataList)) {
+		assert.Equal(t, "beforetest2", string(dataList[0]))
+		assert.True(t, strings.HasPrefix(string(dataList[1]), "before"))
+	}
+}
+
+func runTCPOutput(wg *sync.WaitGroup, output PluginWriter, repeat int, initMessage bool) {
+	input := NewTestInput()
+	plugins := &InOutPlugins{
+		Inputs:  []PluginReader{input},
+		Outputs: []PluginWriter{output},
+	}
+
+	emitter := NewEmitter()
+	go emitter.Start(plugins, Settings.Middleware)
+
+	if initMessage {
+		wg.Add(1)
+	}
+	for i := 0; i < repeat; i++ {
+		wg.Add(1)
+		input.EmitGET()
+	}
+
+	wg.Wait()
+	emitter.Close()
 }

output_ws.go

@@ -0,0 +1,162 @@
+package goreplay
+
+import (
+	"context"
+	"crypto/tls"
+	"encoding/base64"
+	"fmt"
+	"hash/fnv"
+	"log"
+	"net/http"
+	"net/url"
+	"strings"
+	"time"
+
+	"github.com/gorilla/websocket"
+)
+
+// WebSocketOutput used for sending raw tcp payloads
+// Can be used for transferring binary payloads like protocol buffers
+type WebSocketOutput struct {
+	address     string
+	limit       int
+	buf         []chan *Message
+	bufStats    *GorStat
+	config      *WebSocketOutputConfig
+	workerIndex uint32
+	headers     http.Header
+
+	close bool
+}
+
+// WebSocketOutputConfig WebSocket output configuration
+type WebSocketOutputConfig struct {
+	Sticky     bool `json:"output-ws-sticky"`
+	SkipVerify bool `json:"output-ws-skip-verify"`
+	Workers    int  `json:"output-ws-workers"`
+
+	Headers map[string][]string `json:"output-ws-headers"`
+}
+
+// NewWebSocketOutput constructor for WebSocketOutput
+// Initialize X workers which hold keep-alive connection
+func NewWebSocketOutput(address string, config *WebSocketOutputConfig) PluginWriter {
+	o := new(WebSocketOutput)
+
+	u, err := url.Parse(address)
+	if err != nil {
+		log.Fatal(fmt.Sprintf("[OUTPUT-WS] parse WS output URL error[%q]", err))
+	}
+
+	o.config = config
+	o.headers = http.Header{
+		"Authorization": []string{"Basic " + base64.StdEncoding.EncodeToString([]byte(u.User.String()))},
+	}
+	for k, values := range config.Headers {
+		for _, v := range values {
+			o.headers.Add(k, v)
+		}
+	}
+
+	u.User = nil // must be after creating the headers
+	o.address = u.String()
+
+	if Settings.OutputWebSocketStats {
+		o.bufStats = NewGorStat("output_ws", 5000)
+	}
+
+	// create X buffers and send the buffer index to the worker
+	o.buf = make([]chan *Message, o.config.Workers)
+	for i := 0; i < o.config.Workers; i++ {
+		o.buf[i] = make(chan *Message, 100)
+		go o.worker(i)
+	}
+
+	return o
+}
+
+func (o *WebSocketOutput) worker(bufferIndex int) {
+	retries := 0
+	conn, err := o.connect(o.address)
+	for {
+		if o.close {
+			return
+		}
+
+		if err == nil {
+			break
+		}
+
+		Debug(1, fmt.Sprintf("Can't connect to aggregator instance, reconnecting in 1 second. Retries:%d", retries))
+		time.Sleep(1 * time.Second)
+
+		conn, err = o.connect(o.address)
+		retries++
+	}
+
+	if retries > 0 {
+		Debug(2, fmt.Sprintf("Connected to aggregator instance after %d retries", retries))
+	}
+
+	defer conn.Close()
+
+	for {
+		msg := <-o.buf[bufferIndex]
+		err = conn.WriteMessage(websocket.BinaryMessage, append(msg.Meta, msg.Data...))
+		if err != nil {
+			Debug(2, "INFO: WebSocket output connection closed, reconnecting "+err.Error())
+			go o.worker(bufferIndex)
+			o.buf[bufferIndex] <- msg
+			break
+		}
+	}
+}
+
+func (o *WebSocketOutput) getBufferIndex(msg *Message) int {
+	if !o.config.Sticky {
+		o.workerIndex++
+		return int(o.workerIndex) % o.config.Workers
+	}
+
+	hasher := fnv.New32a()
+	hasher.Write(payloadID(msg.Meta))
+	return int(hasher.Sum32()) % o.config.Workers
+}
+
+// PluginWrite writes message to this plugin
+func (o *WebSocketOutput) PluginWrite(msg *Message) (n int, err error) {
+	if !isOriginPayload(msg.Meta) {
+		return len(msg.Data), nil
+	}
+
+	bufferIndex := o.getBufferIndex(msg)
+	o.buf[bufferIndex] <- msg
+
+	if Settings.OutputTCPStats {
+		o.bufStats.Write(len(o.buf[bufferIndex]))
+	}
+
+	return len(msg.Data) + len(msg.Meta), nil
+}
+
+func (o *WebSocketOutput) connect(address string) (conn *websocket.Conn, err error) {
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+
+	d := websocket.DefaultDialer
+	if strings.HasPrefix(address, "wss://") {
+		d.TLSClientConfig = &tls.Config{InsecureSkipVerify: o.config.SkipVerify}
+	}
+
+	conn, _, err = d.DialContext(ctx, address, o.headers)
+	return
+}
+
+func (o *WebSocketOutput) String() string {
+	return fmt.Sprintf("WebSocket output %s, limit: %d", o.address, o.limit)
+}
+
+// Close closes the output
+func (o *WebSocketOutput) Close() {
+	o.close = true
+}