Update README.md

* Track FIN packets to check for closed connection

* Add tests for FIN / fix bugs

* Fmt changes

.dockerignore

@@ -0,0 +1,3 @@
+*.tar.gz
+gor
+gor.test

.gitignore

@@ -1,3 +1,19 @@
 *.swp
-
 *.gor
+
+*.out
+
+*.bin
+
+*.gz
+
+*.class
+
+*.test
+.idea
+*.iml
+gor
+
+*.mprof
+
+*.pcap

.gitmodules

@@ -0,0 +1,12 @@
+[submodule "vendor/github.com/mattbaird/elastigo"]
+	path = vendor/github.com/mattbaird/elastigo
+	url = https://github.com/mattbaird/elastigo
+[submodule "vendor/github.com/araddon/gou"]
+	path = vendor/github.com/araddon/gou
+	url = https://github.com/araddon/gou
+[submodule "vendor/github.com/bitly/go-hostpool"]
+	path = vendor/github.com/bitly/go-hostpool
+	url = https://github.com/bitly/go-hostpool
+[submodule "vendor/github.com/bmizerany/assert"]
+	path = vendor/github.com/bmizerany/assert
+	url = https://github.com/bmizerany/assert

.travis.yml

@@ -1,3 +1,6 @@
-language: go 
-go: 1.2.1
-script: sudo -E bash -c "source /etc/profile && gvm use go1.2.1 && export GOPATH=$HOME/gopath:$GOPATH && go get && go test -v"
+language: go
+go: 1.6
+script: sudo -E bash -c "source /etc/profile && eval '$(gimme 1.6)' && export GOPATH=$HOME/gopath:$GOPATH && go get && GORACE='halt_on_error=1' go test ./... -v -timeout 120s -race"
+
+before_install:
+  - sudo apt-get install libpcap-dev -y

CHANGELOG.md

@@ -1,25 +0,0 @@
-v0.7.0 - 31 Oct 2013
-* New modular architecture. Listener and Replay functionality merged.
-* Added option to equally split traffic between multiple outputs: --split-output true
-* Saving requests to file and replaying from it
-* Injecting custom headers to http requests
-* Advanced stats using ElasticSearch
-
-v0.3.5 - 15 Sep 2013
-* Significantly improved test coverage
-* Fixed bug with redirect replay https://github.com/buger/gor/pull/15
-* Added limit on listener side
-* Improved stability (catch and log panic, instead of exiting)
-* Added License file
-
-v0.3.3 - 22 Jun 2013
-* Using TCP instead of UDP for communication between Listener and Replay
-* Significantly improved performance
-* Fixed bugs causing locking and message dropping (concurrency issues)
-* Rewrote concurrency model to use more channels
-
-v0.3 - 10 Jun 2013
-* Use RAW_SOCKETS instead of tcpdump
-* Own TCP stack
-* All HTTP request types support
-* Simplified request parsing

COMM-LICENSE

@@ -0,0 +1,93 @@
+END-USER LICENSE AGREEMENT
+
+------------------------------------------------------------------------------
+
+IMPORTANT: THIS SOFTWARE END-USER LICENSE AGREEMENT ("EULA") IS A LEGAL AGREEMENT (“Agreement”) BETWEEN YOU (THE CUSTOMER, EITHER AS AN INDIVIDUAL OR, IF PURCHASED OR OTHERWISE ACQUIRED BY OR FOR AN ENTITY, AS AN ENTITY) AND GoReplay LLC. READ IT CAREFULLY BEFORE COMPLETING THE INSTALLATION PROCESS AND USING GOREPLAY PRO AND RELATED SOFTWARE COMPONENTS (“SOFTWARE”). IT PROVIDES A LICENSE TO USE THE SOFTWARE AND CONTAINS WARRANTY INFORMATION AND LIABILITY DISCLAIMERS. BY INSTALLING AND USING THE SOFTWARE, YOU ARE CONFIRMING YOUR ACCEPTANCE OF THE SOFTWARE AND AGREEING TO BECOME BOUND BY THE TERMS OF THIS AGREEMENT.
+
+------------------------------------------------------------------------------
+
+In order to use the Software under this Agreement, you must receive a “Source URL” at the time of purchase, in accordance with the scope of use and other terms specified for each type of Software and as set forth in this Section 1 of this Agreement. 
+
+1. License Grant
+
+1.1 General Use. This Agreement grants you a non-exclusive, non-transferable, limited license to the use rights for the Software, without the right to grant sublicenses, subject to the terms and conditions in this Agreement. The Software is licensed, not sold.
+
+1.2 Unlimited Organization License. If you purchased an Organization License (included with the GoReplay Pro Software), you may install the Software on an unlimited number of Hosts. “Host” means any physical or virtual machine which is controlled by you. You may concurrently run the software on an unlimited number of Hosts.
+
+1.3 Appliance License. If you purchased an Appliance License, you may distribute the Software in any applications, frameworks, or elements (collectively referred to as an “Application” or “Applications”) that you develop using the Software in accordance with this EULA, provided that such distribution does not violate the restrictions set forth in section 3 of this EULA. You must not remove, obscure or interfere with any copyright, acknowledgment, attribution, trademark, warning or disclaimer statement affixed to, incorporated in or otherwise applied in connection with the Software. You are required to ensure that the Software is not reused by or with any applications other than those with which you distribute it as permitted herein. For example, if You install the Software on a customer’s server, that customer is not permitted to use the Software independently of your Application. You must inform GoReplay LLC of your knowledge of any infringing use of the Software by any of your customers. You are liable for compliance by those third parties with the terms and conditions of this EULA. You will not owe GoReplay LLC any royalties for your distribution of the Software in accordance with this EULA.
+
+1.4 Archive Copies. You are entitled to make a reasonable amount of copies of the Software for archival purposes. Each copy must reproduce all copyright and other proprietary rights notices on or in the Software Product.
+
+1.5 Electronic Delivery. All Software and license documentation shall be delivered by electronic means unless otherwise specified on the applicable invoice or at the time of purchase. Software shall be deemed delivered when it is made available for download by you (“Delivery”).        
+
+2. Modifications. GoReplay LLC shall provide you with source code so that you can create Modifications of the original software. “Modification” means: (a) any addition to or deletion from the contents of a file included in the original Software or previous Modifications created by You, or (b) any new file that contains any part of the original Software or previous Modifications. While you retain all rights to any original work authored by you as part of the Modifications, We continue to own all copyright and other intellectual property rights in the Software.
+
+3. Restricted Uses. 
+
+3.1 You shall not (and shall not allow any third party to): (a) decompile, disassemble, or otherwise reverse engineer the Software or attempt to reconstruct or discover any source code, underlying ideas, algorithms, file formats or programming interfaces of the Software by any means whatsoever (except and only to the extent that applicable law prohibits or restricts reverse engineering restrictions); (b) distribute, sell, sublicense, rent, lease or use the Software for time sharing, hosting, service provider or like purposes, except as expressly permitted under this Agreement; (c) redistribute the Software or Modifications other than by including the Software or a portion thereof within your own product, which must have substantially different functionality than the Software or Modifications and must not allow any third party to use the Software or Modifications, or any portions thereof, for software development or application development purposes; (d) redistribute the Software as part of a product, "appliance" or "virtual server"; (e) redistribute the Software on any server which is not directly under your control; (f) remove any product identification, proprietary, copyright or other notices contained in the Software; (g) modify any part of the Software, create a derivative work of any part of the Software (except as permitted in Section 4), or incorporate the Software, except to the extent expressly authorized in writing by GoReplay LLC; (h) publicly disseminate performance information or analysis (including, without limitation, benchmarks) from any source relating to the Software; (i) utilize any equipment, device, software, or other means designed to circumvent or remove any form of Source URL or copy protection used by GoReplay LLC in connection with the Software, or use the Software together with any authorization code, Source URL, serial number, or other copy protection device not supplied by GoReplay LLC; (j) use the Software to develop a product which is competitive with any GoReplay LLC product offerings; or (k) use unauthorized Source URLS or keycode(s) or distribute or publish Source URLs or keycode(s), except as may be expressly permitted by GoReplay LLC in writing. If your unique Source URL is ever published, GoReplay LLC reserves the right to terminate your access without notice.
+
+3.2 UNDER NO CIRCUMSTANCES MAY YOU USE THE SOFTWARE AS PART OF A PRODUCT OR SERVICE THAT PROVIDES SIMILAR FUNCTIONALITY TO THE SOFTWARE ITSELF.
+
+The Open Source version of the Software (“LGPL Version”) is licensed
+under the terms of the GNU Lesser General Public License versions 3.0
+(“LGPL”) and not under this EULA. 
+
+4. Ownership. Notwithstanding anything to the contrary contained herein, except for the limited license rights expressly provided herein, GoReplay LLC and its suppliers have and will retain all rights, title and interest (including, without limitation, all patent, copyright, trademark, trade secret and other intellectual property rights) in and to the Software and all copies, modifications and derivative works thereof (including any changes which incorporate any of your ideas, feedback or suggestions). You acknowledge that you are obtaining only a limited license right to the Software, and that irrespective of any use of the words “purchase”, “sale” or like terms hereunder no ownership rights are being conveyed to you under this Agreement or otherwise. 
+
+5. Fees and Payment. The Software license fees will be due and payable in full as set forth in the applicable invoice or at the time of purchase. If the Software does not function properly within two weeks of purchase, please contact us within those two weeks for a refund. You shall be responsible for all taxes, withholdings, duties and levies arising from the order (excluding taxes based on the net income of GoReplay LLC). 
+
+6. Support, Maintenance and Services. Subject to the terms and conditions of this Agreement, as set forth in your invoice, and as set forth on the GoReplay Pro support page (https://github.com/buger/gor/wiki/Pro-Support), support and maintenance services may be included with the purchase of your license subscription.
+
+7. Term of Agreement. 
+
+7.1 Term. This Agreement is effective as of the Delivery of the Software and expires at such time as all license and service subscriptions hereunder have expired in accordance with their own terms (the “Term”). For clarification, the term of your license under this Agreement may be perpetual, limited for Evaluation Version, or designated as a fixed-term license in the Invoice, and shall be specified at your time of purchase. Either party may terminate this Agreement (including all related Invoices) if the other party: (a) fails to cure any material breach of this Agreement within thirty (30) days after written notice of such breach, provided that GoReplay LLC may terminate this Agreement immediately upon any breach of Section 3 or if you exceed any other restrictions contained in Section 1, unless otherwise specified in this agreement; (b) ceases operation without a successor; or (c) seeks protection under any bankruptcy, receivership, trust deed, creditors arrangement, composition or comparable proceeding, or if any such proceeding is instituted against such party (and not dismissed within sixty (60) days)). Termination is not an exclusive remedy and the exercise by either party of any remedy under this Agreement will be without prejudice to any other remedies it may have under this Agreement, by law, or otherwise.        
+
+7.2 Termination. Upon any termination of this Agreement, you shall cease any and all use of any Software and destroy all copies thereof. 
+
+7.3 Expiration of License. Upon the expiration of any term under this Agreement, (a) all Software updates and services pursuant to the license shall cease, (b) you may only continue to run existing installations of the Software, (c) you may not install the Software on any additional Hosts, and (d) any new installation of the Software shall require the purchase of a new license subscription from GoReplay LLC.
+
+8. Disclaimer of Warranties. The Software is provided "as is," with all faults, defects and errors, and without warranty of any kind. GoReplay LLC does not warrant that the Software will be free of bugs, errors, viruses or other defects, and GoReplay LLC shall have no liability of any kind for the use of or inability to use the Software, the Software content or any associated service, and you acknowledge that it is not technically practicable for GoReplay LLC to do so. 
+To the maximum extent permitted by applicable law, GoReplay LLC disclaims all warranties, express, implied, arising by law or otherwise, regarding the Software, the Software content and their respective performance or suitability for your intended use, including without limitation any implied warranty of merchantability, fitness for a particular purpose.
+
+9. Limitation of Liability. 
+
+In no event will GoReplay LLC be liable for any direct, indirect, consequential, incidental, special, exemplary, or punitive damages or liabilities whatsoever arising from or relating to the Software, the Software content or this Agreement, whether based on contract, tort (including negligence), strict liability or other theory, even if GoReplay LLC has been advised of the possibility of such damages.
+
+In no event will GoReplay LLC liability exceed the Software license price as indicated in the invoice. The existence of more than one claim will not enlarge or extend this limit.
+
+10. Remedies. Your exclusive remedy and GoReplay LLC’ entire liability for breach of this Agreement shall be limited, at GoReplay LLC’ sole and exclusive discretion, to (a) replacement of any defective software or documentation; or (b) refund of the license fee paid to GoReplay LLC, payable in accordance with GoReplay LLC' refund policy.
+
+11. Acknowledgements.
+
+11.1 Consent to the Use of Data. You agree that GoReplay LLC and its affiliates may collect and use technical information gathered as part of the product support services. GoReplay LLC may use this information solely to improve products and services and will not disclose this information in a form that personally identifies you.
+
+11.2 Verification. We or a certified auditor acting on our behalf, may, upon its reasonable request and at its expense, audit you with respect to the use of the Software. Such audit may be conducted by mail, electronic means or through an in-person visit to your place of business. Any such in-person audit shall be conducted during regular business hours at your facilities and shall not unreasonably interfere with your business activities. We shall not remove, copy, or redistribute any electronic material during the course of an audit. If an audit reveals that you are using the Software in a way that is in material violation of the terms of the EULA, then you shall pay our reasonable costs of conducting the audit. In the case of a material violation, you agree to pay Us any amounts owing that are attributable to the unauthorized use. In the alternative, We reserve the right, at our sole option, to terminate the licenses for the Software.
+
+11.3 Government End Users. If the Software and related documentation are supplied to or purchased by or on behalf of the United States Government, then the Software is deemed to be "commercial software" as that term is used in the Federal Acquisition Regulation system. Rights of the United States shall not exceed the minimum rights set forth in FAR 52.227-19 for "restricted computer software". All other terms and conditions of this Agreement apply.
+
+12. Third Party Software.  Examples included in Software may provide links to third party libraries or code (collectively “Third Party Software”) to implement various functions. Third Party Software does not comprise part of the Software. In some cases, access to Third Party Software may be included along with the Software delivery as a convenience for demonstration purposes. Such source code and libraries may be included in the “…/examples” source tree delivered with the Software and do not comprise the Software. Licensee acknowledges (1) that some part of Third Party Software may require additional licensing of copyright and patents from the owners of such, and (2) that distribution of any of the Software referencing or including any portion of a Third Party Software may require appropriate licensing from such third parties.
+
+
+13. Miscellaneous
+
+13.1 Entire Agreement. This Agreement sets forth our entire agreement with respect to the Software and the subject matter hereof and supersedes all prior and contemporaneous understandings and agreements whether written or oral.
+
+13.2 Amendment. GoReplay LLC reserves the right, in its sole discretion, to amend this Agreement from time. Amendments to this Agreement can be located at: https://github.com/buger/gor/blob/master/COMM-LICENSE.
+
+13.3 Assignment. You may not assign this Agreement or any of its rights under this Agreement without the prior written consent of GoReplay LLC and any attempted assignment without such consent shall be void.
+
+13.4 Export Compliance. You agree to comply with all applicable laws and regulations, including laws, regulations, orders or other restrictions on export, re-export or redistribution of software.
+
+13.5 Indemnification. You agree to defend, indemnify, and hold harmless GoReplay LLC from and against any lawsuits, claims, losses, damages, fines and expenses (including attorneys' fees and costs) arising out of your use of the Software or breach of this Agreement.
+
+13.6 Governing Law. This Agreement is governed by the laws of the State of Oregon and the United States without regard to conflicts of laws provisions thereof, and without regard to the United Nations Convention on the International Sale of Goods or the Uniform Computer Information Transactions Act, as currently enacted by any jurisdiction or as may be codified or amended from time to time by any jurisdiction. The jurisdiction and venue for actions related to the subject matter hereof shall be the state of Oregon and United States federal courts located in Portland, Oregon, and both parties hereby submit to the personal jurisdiction of such courts. 
+
+13.7 Attorneys’ Fees and Costs. The prevailing party in any action to enforce this Agreement will be entitled to recover its attorneys’ fees and costs in connection with such action. 
+
+13.8 Severability. If any provision of this Agreement is held by a court of competent jurisdiction to be invalid, illegal, or unenforceable, the remainder of this Agreement will remain in full force and effect.
+
+13.9 Waiver. Failure or neglect by either party to enforce at any time any of the provisions of this licence Agreement shall not be construed or deemed to be a waiver of that party's rights under this Agreement.
+
+13.10 Headings. The headings of sections and paragraphs of this Agreement are for convenience of reference only and are not intended to restrict, affect or be of any weight in the interpretation or construction of the provisions of such sections or paragraphs.
+
+14. Contact Information. If you have any questions about this EULA, or if you want to contact GoReplay LLC for any reason, please direct correspondence to info@gortool.com

Dockerfile

@@ -0,0 +1,22 @@
+FROM golang:latest
+
+RUN apt-get update && apt-get install ruby vim-common -y
+
+# Install Java for middleware testing
+RUN echo "deb http://ppa.launchpad.net/webupd8team/java/ubuntu trusty main" | tee /etc/apt/sources.list.d/webupd8team-java.list
+RUN echo "deb-src http://ppa.launchpad.net/webupd8team/java/ubuntu trusty main" | tee -a /etc/apt/sources.list.d/webupd8team-java.list
+RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys EEA14886
+RUN apt-get update -y
+RUN echo oracle-java7-installer shared/accepted-oracle-license-v1-1 select true | /usr/bin/debconf-set-selections
+RUN apt-get install oracle-java8-installer -y
+
+RUN apt-get install flex bison -y
+RUN wget http://www.tcpdump.org/release/libpcap-1.7.4.tar.gz && tar xzf libpcap-1.7.4.tar.gz && cd libpcap-1.7.4 && ./configure && make install
+RUN go get github.com/google/gopacket
+RUN go get -u github.com/golang/lint/golint
+
+WORKDIR /go/src/github.com/buger/gor/
+ADD . /go/src/github.com/buger/gor/
+
+RUN javac -cp /tmp/commons-io-2.4/commons-io-2.4.jar ./examples/middleware/echo.java
+RUN go get

LICENSE.txt

@@ -1,191 +1,19 @@
-Apache License
-Version 2.0, January 2004
-http://www.apache.org/licenses/
+GoReplay is an Open Source project licensed under the terms of
+the LGPLv3 license.  Please see <http://www.gnu.org/licenses/lgpl-3.0.html>
+for license text.
 
-TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+As a special exception to the GNU Lesser General Public License version 3
+("LGPL3"), the copyright holders of this Library give you permission to
+convey to a third party a Combined Work that links statically or dynamically
+to this Library without providing any Minimal Corresponding Source or
+Minimal Application Code as set out in 4d or providing the installation
+information set out in section 4e, provided that you comply with the other
+provisions of LGPL3 and provided that you meet, for the Application the
+terms and conditions of the license(s) which apply to the Application.
 
-1. Definitions.
+TLDR: You are free to use Gor subpackages like `byteutils` or `proto` in your commercial projects.
 
-"License" shall mean the terms and conditions for use, reproduction, and
-distribution as defined by Sections 1 through 9 of this document.
 
-"Licensor" shall mean the copyright owner or entity authorized by the copyright
-owner that is granting the License.
-
-"Legal Entity" shall mean the union of the acting entity and all other entities
-that control, are controlled by, or are under common control with that entity.
-For the purposes of this definition, "control" means (i) the power, direct or
-indirect, to cause the direction or management of such entity, whether by
-contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the
-outstanding shares, or (iii) beneficial ownership of such entity.
-
-"You" (or "Your") shall mean an individual or Legal Entity exercising
-permissions granted by this License.
-
-"Source" form shall mean the preferred form for making modifications, including
-but not limited to software source code, documentation source, and configuration
-files.
-
-"Object" form shall mean any form resulting from mechanical transformation or
-translation of a Source form, including but not limited to compiled object code,
-generated documentation, and conversions to other media types.
-
-"Work" shall mean the work of authorship, whether in Source or Object form, made
-available under the License, as indicated by a copyright notice that is included
-in or attached to the work (an example is provided in the Appendix below).
-
-"Derivative Works" shall mean any work, whether in Source or Object form, that
-is based on (or derived from) the Work and for which the editorial revisions,
-annotations, elaborations, or other modifications represent, as a whole, an
-original work of authorship. For the purposes of this License, Derivative Works
-shall not include works that remain separable from, or merely link (or bind by
-name) to the interfaces of, the Work and Derivative Works thereof.
-
-"Contribution" shall mean any work of authorship, including the original version
-of the Work and any modifications or additions to that Work or Derivative Works
-thereof, that is intentionally submitted to Licensor for inclusion in the Work
-by the copyright owner or by an individual or Legal Entity authorized to submit
-on behalf of the copyright owner. For the purposes of this definition,
-"submitted" means any form of electronic, verbal, or written communication sent
-to the Licensor or its representatives, including but not limited to
-communication on electronic mailing lists, source code control systems, and
-issue tracking systems that are managed by, or on behalf of, the Licensor for
-the purpose of discussing and improving the Work, but excluding communication
-that is conspicuously marked or otherwise designated in writing by the copyright
-owner as "Not a Contribution."
-
-"Contributor" shall mean Licensor and any individual or Legal Entity on behalf
-of whom a Contribution has been received by Licensor and subsequently
-incorporated within the Work.
-
-2. Grant of Copyright License.
-
-Subject to the terms and conditions of this License, each Contributor hereby
-grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free,
-irrevocable copyright license to reproduce, prepare Derivative Works of,
-publicly display, publicly perform, sublicense, and distribute the Work and such
-Derivative Works in Source or Object form.
-
-3. Grant of Patent License.
-
-Subject to the terms and conditions of this License, each Contributor hereby
-grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free,
-irrevocable (except as stated in this section) patent license to make, have
-made, use, offer to sell, sell, import, and otherwise transfer the Work, where
-such license applies only to those patent claims licensable by such Contributor
-that are necessarily infringed by their Contribution(s) alone or by combination
-of their Contribution(s) with the Work to which such Contribution(s) was
-submitted. If You institute patent litigation against any entity (including a
-cross-claim or counterclaim in a lawsuit) alleging that the Work or a
-Contribution incorporated within the Work constitutes direct or contributory
-patent infringement, then any patent licenses granted to You under this License
-for that Work shall terminate as of the date such litigation is filed.
-
-4. Redistribution.
-
-You may reproduce and distribute copies of the Work or Derivative Works thereof
-in any medium, with or without modifications, and in Source or Object form,
-provided that You meet the following conditions:
-
-You must give any other recipients of the Work or Derivative Works a copy of
-this License; and
-You must cause any modified files to carry prominent notices stating that You
-changed the files; and
-You must retain, in the Source form of any Derivative Works that You distribute,
-all copyright, patent, trademark, and attribution notices from the Source form
-of the Work, excluding those notices that do not pertain to any part of the
-Derivative Works; and
-If the Work includes a "NOTICE" text file as part of its distribution, then any
-Derivative Works that You distribute must include a readable copy of the
-attribution notices contained within such NOTICE file, excluding those notices
-that do not pertain to any part of the Derivative Works, in at least one of the
-following places: within a NOTICE text file distributed as part of the
-Derivative Works; within the Source form or documentation, if provided along
-with the Derivative Works; or, within a display generated by the Derivative
-Works, if and wherever such third-party notices normally appear. The contents of
-the NOTICE file are for informational purposes only and do not modify the
-License. You may add Your own attribution notices within Derivative Works that
-You distribute, alongside or as an addendum to the NOTICE text from the Work,
-provided that such additional attribution notices cannot be construed as
-modifying the License.
-You may add Your own copyright statement to Your modifications and may provide
-additional or different license terms and conditions for use, reproduction, or
-distribution of Your modifications, or for any such Derivative Works as a whole,
-provided Your use, reproduction, and distribution of the Work otherwise complies
-with the conditions stated in this License.
-
-5. Submission of Contributions.
-
-Unless You explicitly state otherwise, any Contribution intentionally submitted
-for inclusion in the Work by You to the Licensor shall be under the terms and
-conditions of this License, without any additional terms or conditions.
-Notwithstanding the above, nothing herein shall supersede or modify the terms of
-any separate license agreement you may have executed with Licensor regarding
-such Contributions.
-
-6. Trademarks.
-
-This License does not grant permission to use the trade names, trademarks,
-service marks, or product names of the Licensor, except as required for
-reasonable and customary use in describing the origin of the Work and
-reproducing the content of the NOTICE file.
-
-7. Disclaimer of Warranty.
-
-Unless required by applicable law or agreed to in writing, Licensor provides the
-Work (and each Contributor provides its Contributions) on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied,
-including, without limitation, any warranties or conditions of TITLE,
-NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are
-solely responsible for determining the appropriateness of using or
-redistributing the Work and assume any risks associated with Your exercise of
-permissions under this License.
-
-8. Limitation of Liability.
-
-In no event and under no legal theory, whether in tort (including negligence),
-contract, or otherwise, unless required by applicable law (such as deliberate
-and grossly negligent acts) or agreed to in writing, shall any Contributor be
-liable to You for damages, including any direct, indirect, special, incidental,
-or consequential damages of any character arising as a result of this License or
-out of the use or inability to use the Work (including but not limited to
-damages for loss of goodwill, work stoppage, computer failure or malfunction, or
-any and all other commercial damages or losses), even if such Contributor has
-been advised of the possibility of such damages.
-
-9. Accepting Warranty or Additional Liability.
-
-While redistributing the Work or Derivative Works thereof, You may choose to
-offer, and charge a fee for, acceptance of support, warranty, indemnity, or
-other liability obligations and/or rights consistent with this License. However,
-in accepting such obligations, You may act only on Your own behalf and on Your
-sole responsibility, not on behalf of any other Contributor, and only if You
-agree to indemnify, defend, and hold each Contributor harmless for any liability
-incurred by, or claims asserted against, such Contributor by reason of your
-accepting any such warranty or additional liability.
-
-END OF TERMS AND CONDITIONS
-
-APPENDIX: How to apply the Apache License to your work
-
-To apply the Apache License to your work, attach the following boilerplate
-notice, with the fields enclosed by brackets "[]" replaced with your own
-identifying information. (Don't include the brackets!) The text should be
-enclosed in the appropriate comment syntax for the file format. We also
-recommend that a file or class name and description of purpose be included on
-the same "printed page" as the copyright notice for easier identification within
-third-party archives.
-
-   Copyright [yyyy] [name of copyright owner]
-
-   Licensed under the Apache License, Version 2.0 (the "License");
-   you may not use this file except in compliance with the License.
-   You may obtain a copy of the License at
-
-     http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
+GoReplay Pro has a commercial-friendly license allowing private forks
+and modifications of GoReplay.  Please see https://goreplay.org/pro.html for
+more detail.  You can find the commercial license terms in COMM-LICENSE.

Makefile

@@ -1,11 +1,89 @@
-all: build-x86 build-x64
+SOURCE = emitter.go gor.go gor_stat.go input_dummy.go input_file.go input_raw.go input_tcp.go limiter.go output_dummy.go output_null.go output_file.go input_http.go output_http.go output_tcp.go plugins.go settings.go test_input.go elasticsearch.go http_modifier.go http_modifier_settings.go http_client.go middleware.go protocol.go output_file_settings.go
+SOURCE_PATH = /go/src/github.com/buger/gor/
+PORT = 8000
+FADDR = :8000
+RUN = docker run -v `pwd`:$(SOURCE_PATH) -p 0.0.0.0:$(PORT):$(PORT) -t -i gor
+BENCHMARK = BenchmarkRAWInput
+TEST = TestRawListenerBench
+VERSION = DEV-$(shell date +%s)
+LDFLAGS = -ldflags "-X main.VERSION=$(VERSION) -extldflags \"-static\""
+MAC_LDFLAGS = -ldflags "-X main.VERSION=$(VERSION)"
+FADDR = ":8000"
 
-build-x64:
-	GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build
-	tar -czf gor_x64.tar.gz gor
-	rm gor
+release: release-x64 release-mac
 
-build-x86:
-	GOOS=linux GOARCH=386 CGO_ENABLED=0 go build
-	tar -czf gor_x86.tar.gz gor
-	rm gor
+release-x64:
+	docker run -v `pwd`:$(SOURCE_PATH) -t --env GOOS=linux --env GOARCH=amd64  -i gor go build -tags netgo $(LDFLAGS) && tar -czf gor_$(VERSION)_x64.tar.gz gor && rm gor
+
+release-x86:
+	docker run -v `pwd`:$(SOURCE_PATH) -t --env GOOS=linux --env GOARCH=386 -i gor go build -tags netgo $(LDFLAGS) && tar -czf gor_$(VERSION)_x86.tar.gz gor && rm gor
+
+release-mac:
+	go build $(MAC_LDFLAGS) && tar -czf gor_$(VERSION)_mac.tar.gz gor && rm gor
+
+build:
+	docker build -t gor .
+
+
+profile:
+	go build && ./gor --output-http="http://localhost:9000" --input-dummy 0 --input-raw :9000 --input-http :9000 --memprofile=./mem.out --cpuprofile=./cpu.out --stats --output-http-stats --output-http-timeout 100ms
+
+lint:
+	$(RUN) golint $(PKG)
+
+race:
+	$(RUN) go test ./... $(ARGS) -v -race -timeout 15s
+
+test:
+	$(RUN) go test ./. -timeout 60s $(LDFLAGS) $(ARGS)  -v
+
+test_all:
+	$(RUN) go test ./... -timeout 60s $(LDFLAGS) $(ARGS) -v
+
+testone:
+	$(RUN) go test ./... -timeout 4s $(LDFLAGS) -run $(TEST) $(ARGS) -v
+
+cover:
+	$(RUN) go test $(ARGS) -race -v -timeout 15s -coverprofile=coverage.out
+	go tool cover -html=coverage.out
+
+fmt:
+	$(RUN) gofmt -w -s ./..
+
+vet:
+	$(RUN) go vet
+
+bench:
+	$(RUN) go test $(LDFLAGS) -v -run NOT_EXISTING -bench $(BENCHMARK) -benchtime 5s
+
+profile_test:
+	$(RUN) go test $(LDFLAGS) -run $(TEST) ./raw_socket_listener/. $(ARGS) -memprofile mem.mprof -cpuprofile cpu.out
+	$(RUN) go test $(LDFLAGS) -run $(TEST) ./raw_socket_listener/. $(ARGS) -c
+
+# Used mainly for debugging, because docker container do not have access to parent machine ports
+run:
+	$(RUN) go run $(LDFLAGS) $(SOURCE) --input-dummy=0 --output-http="http://localhost:9000" --input-raw-track-response --input-raw 127.0.0.1:9000 --verbose --debug --middleware "./examples/middleware/echo.sh" --output-file requests.gor
+
+run-2:
+	sudo -E go run $(SOURCE) --input-dummy="" --output-tcp localhost:27001 --verbose --debug
+
+run-3:
+	sudo -E go run $(SOURCE) --input-tcp :27001 --output-stdout
+
+run-arg:
+	sudo -E go run $(SOURCE) $(ARGS)
+
+file-server:
+	go run $(SOURCE) file-server $(FADDR)
+
+readpcap:
+	go run $(SOURCE) --input-raw $(FILE) --input-raw-engine pcap_file --output-null
+
+record:
+	$(RUN) go run $(SOURCE) --input-dummy=0 --output-file=requests.gor --verbose --debug
+
+replay:
+	$(RUN) go run $(SOURCE) --input-file=requests.bin --output-tcp=:9000 --verbose -h
+
+bash:
+	$(RUN) /bin/bash

README.md

@@ -1,228 +1,73 @@
-[![Stories in Ready](https://badge.waffle.io/buger/gor.png?label=ready)](https://waffle.io/buger/gor)  
-[![Build Status](https://travis-ci.org/buger/gor.png?branch=master)](https://travis-ci.org/buger/gor)
-
-## About
-
-Gor is a simple http traffic replication tool written in Go.
-Its main goal is to replay traffic from production servers to staging and dev environments.
-
-
-Now you can test your code on real user sessions in an automated and repeatable fashion.
-**No more falling down in production!**
-
-Here is basic workflow: The listener server catches http traffic and sends it to the replay server or saves to file.The replay server forwards traffic to a given address.
-
-
-![Diagram](http://i.imgur.com/9mqj2SK.png)
-
-
-## Examples
-
-### Capture traffic from port
-```bash
-# Run on servers where you want to catch traffic. You can run it on each `web` machine.
-sudo gor --input-raw :80 --output-tcp replay.local:28020
-
-# Replay server (replay.local).
-gor --input-tcp replay.local:28020 --output-http http://staging.com
-```
-
-### Using 1 Gor instance for both listening and replaying
-It's recommended to use separate server for replaying traffic, but if you have enough CPU resources you can use single Gor instance.
-
-```
-sudo gor --input-raw :80 --output-http "http://staging.com"
-```
-
-## Advanced use
-
-### Rate limiting
-Both replay and listener support rate limiting. It can be useful if you want
-forward only part of production traffic and not overload your staging
-environment. You can specify your desired requests per second using the
-"|" operator after the server address:
-
-#### Limiting replay
-```
-# staging.server will not get more than 10 requests per second
-gor --input-tcp :28020 --output-http "http://staging.com|10"
-```
-
-#### Limiting listener
-```
-# replay server will not get more than 10 requests per second
-# useful for high-load environments
-gor --input-raw :80 --output-tcp "replay.local:28020|10"
-```
-
-### Forward to multiple addresses
-
-You can forward traffic to multiple endpoints. Just add multiple --output-* arguments.
-```
-gor --input-tcp :28020 --output-http "http://staging.com"  --output-http "http://dev.com"
-```
-
-#### Splitting traffic
-By default it will send same traffic to all outputs, but you have options to equally split it:
-
-```
-gor --input-tcp :28020 --output-http "http://staging.com"  --output-http "http://dev.com" --split-output true
-```
-
-### Saving requests to file
-You can save requests to file, and replay them later:
-```
-# write to file
-gor --input-raw :80 --output-file requests.gor
-
-# read from file
-gor --input-file requests.gor --output-http "http://staging.com"
-```
-
-**Note:** Replay will preserve the original time differences between requests.
-
-### Injecting headers
-
-Additional headers can be injected/overwritten into requests during replay. This may be useful if the hostname that staging responds to differs from production, you need to identify requests generated by Gor, or enable feature flagged functionality in an application:
-
-```
-gor --input-raw :80 --output-http "http://staging.server" \
-    --output-http-header "Host: staging.server" \
-    --output-http-header "User-Agent: Replayed by Gor" -header " \
-    --output-http-header "Enable-Feature-X: true"
-```
-
-## Filtering HTTP methods
+[![GitHub release](https://img.shields.io/github/release/buger/gor.svg?maxAge=3600)](https://github.com/buger/gor/releases) [![codebeat](https://codebeat.co/badges/6427d589-a78e-416c-a546-d299b4089893)](https://codebeat.co/projects/github-com-buger-gor) [![Go Report Card](https://goreportcard.com/badge/github.com/buger/gor)](https://goreportcard.com/report/github.com/buger/gor) [![Join the chat at https://gitter.im/buger/gor](https://badges.gitter.im/buger/gor.svg)](https://gitter.im/buger/gor?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
 
-Requests not matching a specified whitelist can be filtered out. For example to strip non-nullipotent requests:
+![Go Replay](http://i.imgur.com/ZG2ki5n.png)
 
-```
-gor --input-raw :80 --output-http "http://staging.server" \
-    --output-http-method GET \
-    --output-http-method OPTIONS
-```
+## https://goreplay.org/
 
-### Basic Auth
-
-If your development or staging environment is protected by Basic Authentication then those credentials can be injected in during the replay:
-
-```
-gor --input-raw :80 --output-http "http://user:pass@staging .com"
-```
-
-Note: This will overwrite any Authorization headers in the original request.
-
-## Stats 
-### ElasticSearch 
-For deep response analyze based on url, cookie, user-agent and etc. you can export response metadata to ElasticSearch. See [ELASTICSEARCH.md](ELASTICSEARCH.md) for more details.
-
-```
-gor --input-tcp :80 --output-http "http://staging.com" --output-http-elasticsearch "es_host:api_port/index_name"
-```
-
-
-## Additional help
-
-Feel free to ask question directly by email or by creating github issue.
-
-## Latest releases (including binaries)
-
-https://github.com/buger/gor/releases
-
-## Command line reference
-`gor -h` output:
-```
-  -cpuprofile="": write cpu profile to file
-  -memprofile="": write memory profile to this file
-  
-  -input-dummy=[]: Used for testing outputs. Emits 'Get /' request every 1s
-
-  -input-file=[]: Read requests from file: 
-    gor --input-file ./requests.gor --output-http staging.com
-
-  -input-raw=[]: Capture traffic from given port (use RAW sockets and require *sudo* access):
-    # Capture traffic from 8080 port
-    gor --input-raw :8080 --output-http staging.com
-
-  -input-tcp=[]: Used for internal communication between Gor instances. Example: 
-    # Receive requests from other Gor instances on 28020 port, and redirect output to staging
-    gor --input-tcp :28020 --output-http staging.com
-
-  -output-dummy=[]: Used for testing inputs. Just prints data coming from inputs.
-
-  -output-file=[]: Write incoming requests to file: 
-    gor --input-raw :80 --output-file ./requests.gor
-
-  -output-http=[]: Forwards incoming requests to given http address.
-    # Redirect all incoming requests to staging.com address 
-    gor --input-raw :80 --output-http http://staging.com
-
-  -output-http-elasticsearch="": Send request and response stats to ElasticSearch:
-    gor --input-raw :8080 --output-http staging.com --output-http-elasticsearch 'es_host:api_port/index_name'
-
-  -output-http-header=[]: Inject additional headers to http reqest:
-    gor --input-raw :8080 --output-http staging.com --output-http-header 'User-Agent: Gor'
-    
-  -output-tcp=[]: Used for internal communication between Gor instances. Example: 
-    # Listen for requests on 80 port and forward them to other Gor instance on 28020 port
-    gor --input-raw :80 --output-tcp replay.local:28020
-    
-  -split-output=false: By default each output gets same traffic. If set to `true` it splits traffic equally among all outputs.
-```
-
-## Building from source
-1. Setup standard Go environment http://golang.org/doc/code.html and ensure that $GOPATH environment variable properly set.
-2. `go get github.com/buger/gor`.
-3. `cd $GOPATH/src/github.com/buger/gor`
-4. `go build ./bin/gor.go` to get binary, or `go run ./bin/gor.go` to build and run (useful for development)
-
-## FAQ
-
-### What OS are supported?
-For now only Linux based. *BSD (including MacOS is not supported yet, check https://github.com/buger/gor/issues/22 for details)
-
-### Why does the `--input-raw` requires sudo or root access?
-Listener works by sniffing traffic from a given port. It's accessible
-only by using sudo or root access.
-
-### I'm getting 'too many open files' error
-Typical linux shell has a small open files soft limit at 1024. You can easily raise that when you do this before starting your gor replay process:
-  
-  ulimit -n 64000
-
-More about ulimit: http://blog.thecodingmachine.com/content/solving-too-many-open-files-exception-red5-or-any-other-application
-
-## Tuning
-
-To achieve the top most performance you should tune the source server system limits:
-
-    net.ipv4.tcp_max_tw_buckets = 65536
-    net.ipv4.tcp_tw_recycle = 1
-    net.ipv4.tcp_tw_reuse = 0
-    net.ipv4.tcp_max_syn_backlog = 131072
-    net.ipv4.tcp_syn_retries = 3
-    net.ipv4.tcp_synack_retries = 3
-    net.ipv4.tcp_retries1 = 3
-    net.ipv4.tcp_retries2 = 8
-    net.ipv4.tcp_rmem = 16384 174760 349520
-    net.ipv4.tcp_wmem = 16384 131072 262144
-    net.ipv4.tcp_mem = 262144 524288 1048576
-    net.ipv4.tcp_max_orphans = 65536
-    net.ipv4.tcp_fin_timeout = 10
-    net.ipv4.tcp_low_latency = 1
-    net.ipv4.tcp_syncookies = 0
-
-
-## Contributing
-
-1. Fork it
-2. Create your feature branch (git checkout -b my-new-feature)
-3. Commit your changes (git commit -am 'Added some feature')
-4. Push to the branch (git push origin my-new-feature)
-5. Create new Pull Request
-
-## Companies using Gor
-
-* http://granify.com
-* [GOV.UK](https://www.gov.uk) ([Government Digital Service](http://digital.cabinetoffice.gov.uk/))
-* To add your company drop me a line to github.com/buger or leonsbox@gmail.com
+## About
+
+GoReplay is the simplest and safest way to test your app using real traffic before you put it into production.
+
+As your application grows, the effort required to test it also grows exponentially. GoReplay offers you the simple idea of reusing your existing traffic for testing, which makes it incredibly powerful. Our state of art technique allows to analyze and record your application traffic without affecting it. This eliminates the risks that come with putting a third party component in the critical path. 
+
+GoReplay increases your confidence in code deployments, configuration changes and infrastructure changes. Did we mention that no coding is required?
+
+
+
+Here is basic workflow: The listener server catches http traffic and sends it to the replay server or saves to file. The replay server forwards traffic to a given address.
+
+![Diagram](http://i.imgur.com/9mqj2SK.png)
+
+Check [latest documentation](http://github.com/buger/gor/wiki).
+
+## Installation
+Download latest binary from https://github.com/buger/gor/releases or [compile by yourself](https://github.com/buger/gor/wiki/Compilation).
+
+## Getting started
+
+The most basic setup will be `sudo ./gor --input-raw :8000 --output-stdout` which acts like tcpdump.
+If you already have test environment you can start replaying: `sudo ./gor --input-raw :8000 --output-http http://staging.env`.
+
+See the our [documentation](https://github.com/buger/gor/wiki/) and [Getting started](https://github.com/buger/gor/wiki/Getting-Started) page for more info. 
+## Newsletter
+Subscribe to our [newsletter](https://www.getdrip.com/forms/89690474/submissions/new) to stay informed about the latest features and changes to Gor project.
+
+
+## Want to Upgrade?
+
+We have created a [GoReplay PRO](https://goreplay.org/pro.html) extension which provides additional features such as support for binary protocols like Thrift or ProtocolBuffers, saving and replaying from cloud storage, TCP sessions replication, etc. The PRO version also includes a commercial-friendly license, dedicated support, and it also allows you to support high-quality open source development. 
+
+
+## Problems?
+If you have a problem, please review the [FAQ](https://github.com/buger/gor/wiki/FAQ) and [Troubleshooting](https://github.com/buger/gor/wiki/Troubleshooting) wiki pages. Searching the [issues](https://github.com/buger/gor/issues) for your problem is also a good idea.
+
+All bug-reports and suggestions should go though Github Issues or our [Google Group](https://groups.google.com/forum/#!forum/gor-users) (you can just send email to gor-users@googlegroups.com).
+If you have a private question feel free to send email to support@gortool.com.
+
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (git checkout -b my-new-feature)
+3. Commit your changes (git commit -am 'Added some feature')
+4. Push to the branch (git push origin my-new-feature)
+5. Create new Pull Request
+
+## Companies using Gor
+
+* [GOV.UK](https://www.gov.uk) - UK Government Digital Service
+* [theguardian.com](http://theguardian.com) - Most popular online newspaper in the UK
+* [TomTom](http://www.tomtom.com/) - Global leader in navigation, traffic and map products, GPS Sport Watches and fleet management solutions.
+* [3SCALE](http://www.3scale.net/) - API infrastructure to manage your APIs for internal or external users
+* [Optionlab](http://www.opinionlab.com) - Optimize customer experience and drive engagement across multiple channels
+* [TubeMogul](http://tubemogul.com) - Software for Brand Advertising
+* [Videology](http://www.videologygroup.com/) - Video advertising platform
+* [ForeksMobile](http://foreksmobile.com/) -  One of the leading financial application development company in Turkey
+* [Granify](http://granify.com) - AI backed SaaS solution that enables online retailers to maximise their sales
+* And many more!
+
+If you are using Gor we are happy add you to the list and share your story, just write to: hello@goreplay.org
+
+## Author
+
+Leonid Bugaev, [@buger](https://twitter.com/buger), https://leonsbox.com

byteutils/byteutils.go

@@ -0,0 +1,40 @@
+// Package byteutils probvides helpers for working with byte slices
+package byteutils
+
+// Cut elements from slice for a given range
+func Cut(a []byte, from, to int) []byte {
+	copy(a[from:], a[to:])
+	a = a[:len(a)-to+from]
+
+	return a
+}
+
+// Insert new slice at specified position
+func Insert(a []byte, i int, b []byte) []byte {
+	a = append(a, make([]byte, len(b))...)
+	copy(a[i+len(b):], a[i:])
+	copy(a[i:i+len(b)], b)
+
+	return a
+}
+
+// Replace function unlike bytes.Replace allows you to specify range
+func Replace(a []byte, from, to int, new []byte) []byte {
+	lenDiff := len(new) - (to - from)
+
+	if lenDiff > 0 {
+		// Extend if new segment bigger
+		a = append(a, make([]byte, lenDiff)...)
+		copy(a[to+lenDiff:], a[to:])
+		copy(a[from:from+len(new)], new)
+
+		return a
+	} else if lenDiff < 0 {
+		copy(a[from:], new)
+		copy(a[from+len(new):], a[to:])
+		return a[:len(a)+lenDiff]
+	} else { // same size
+		copy(a[from:], new)
+		return a
+	}
+}

byteutils/byteutils_test.go

@@ -0,0 +1,32 @@
+package byteutils
+
+import (
+	"bytes"
+	"testing"
+)
+
+func TestCut(t *testing.T) {
+	if !bytes.Equal(Cut([]byte("123456"), 2, 4), []byte("1256")) {
+		t.Error("Should properly cut")
+	}
+}
+
+func TestInsert(t *testing.T) {
+	if !bytes.Equal(Insert([]byte("123456"), 2, []byte("abcd")), []byte("12abcd3456")) {
+		t.Error("Should insert into middle of slice")
+	}
+}
+
+func TestReplace(t *testing.T) {
+	if !bytes.Equal(Replace([]byte("123456"), 2, 4, []byte("ab")), []byte("12ab56")) {
+		t.Error("Should replace when same length")
+	}
+
+	if !bytes.Equal(Replace([]byte("123456"), 2, 4, []byte("abcd")), []byte("12abcd56")) {
+		t.Error("Should replace when replacement length bigger")
+	}
+
+	if !bytes.Equal(Replace([]byte("123456"), 2, 5, []byte("ab")), []byte("12ab6")) {
+		t.Error("Should replace when replacement length bigger")
+	}
+}

circle.yml

@@ -0,0 +1,7 @@
+dependencies:
+  pre:
+    - sudo apt-get install libpcap-dev -y
+
+test:
+  override:
+    - sudo bash -l -c "export GOPATH='/home/ubuntu/.go_workspace:/usr/local/go_workspace:/home/ubuntu/.go_project' && GORACE='halt_on_error=1' /usr/local/go/bin/go test ./... -v -timeout 120s -race"

elasticsearch.go

@@ -0,0 +1,154 @@
+package main
+
+import (
+	"encoding/json"
+	"github.com/mattbaird/elastigo/lib"
+	"github.com/buger/gor/proto"
+	"log"
+	"regexp"
+	"time"
+)
+
+type ESUriErorr struct{}
+
+func (e *ESUriErorr) Error() string {
+	return "Wrong ElasticSearch URL format. Expected to be: host:port/index_name"
+}
+
+type ESPlugin struct {
+	Active  bool
+	ApiPort string
+	eConn   *elastigo.Conn
+	Host    string
+	Index   string
+	indexor *elastigo.BulkIndexer
+	done    chan bool
+}
+
+type ESRequestResponse struct {
+	ReqURL               string `json:"Req_URL"`
+	ReqMethod            string `json:"Req_Method"`
+	ReqUserAgent         string `json:"Req_User-Agent"`
+	ReqAcceptLanguage    string `json:"Req_Accept-Language,omitempty"`
+	ReqAccept            string `json:"Req_Accept,omitempty"`
+	ReqAcceptEncoding    string `json:"Req_Accept-Encoding,omitempty"`
+	ReqIfModifiedSince   string `json:"Req_If-Modified-Since,omitempty"`
+	ReqConnection        string `json:"Req_Connection,omitempty"`
+	ReqCookies           string `json:"Req_Cookies,omitempty"`
+	RespStatus           string `json:"Resp_Status"`
+	RespStatusCode       string `json:"Resp_Status-Code"`
+	RespProto            string `json:"Resp_Proto,omitempty"`
+	RespContentLength    string `json:"Resp_Content-Length,omitempty"`
+	RespContentType      string `json:"Resp_Content-Type,omitempty"`
+	RespTransferEncoding string `json:"Resp_Transfer-Encoding,omitempty"`
+	RespContentEncoding  string `json:"Resp_Content-Encoding,omitempty"`
+	RespExpires          string `json:"Resp_Expires,omitempty"`
+	RespCacheControl     string `json:"Resp_Cache-Control,omitempty"`
+	RespVary             string `json:"Resp_Vary,omitempty"`
+	RespSetCookie        string `json:"Resp_Set-Cookie,omitempty"`
+	Rtt                  int64  `json:"RTT"`
+	Timestamp            time.Time
+}
+
+// Parse ElasticSearch URI
+//
+// Proper format is: host:port/index_name
+func parseURI(URI string) (err error, host string, port string, index string) {
+	rURI := regexp.MustCompile("(.+):([0-9]+)/(.+)")
+	match := rURI.FindAllStringSubmatch(URI, -1)
+
+	if len(match) == 0 {
+		err = new(ESUriErorr)
+	} else {
+		host = match[0][1]
+		port = match[0][2]
+		index = match[0][3]
+	}
+
+	return
+}
+
+func (p *ESPlugin) Init(URI string) {
+	var err error
+
+	err, p.Host, p.ApiPort, p.Index = parseURI(URI)
+
+	if err != nil {
+		log.Fatal("Can't initialize ElasticSearch plugin.", err)
+	}
+	p.eConn = elastigo.NewConn()
+	p.eConn.SetPort(p.ApiPort)
+	p.eConn.SetHosts([]string{p.Host})
+
+	p.indexor = p.eConn.NewBulkIndexerErrors(50, 60)
+	p.done = make(chan bool)
+	p.indexor.Start()
+
+	// Only start the ErrorHandler goroutine when in verbose mode
+	// no need to burn ressources otherwise
+	go p.ErrorHandler()
+
+	log.Println("Initialized Elasticsearch Plugin")
+	return
+}
+
+func (p *ESPlugin) IndexerShutdown() {
+	p.indexor.Stop()
+	return
+}
+
+func (p *ESPlugin) ErrorHandler() {
+	for {
+		errBuf := <-p.indexor.ErrorChannel
+		log.Println(errBuf.Err)
+	}
+}
+
+func (p *ESPlugin) RttDurationToMs(d time.Duration) int64 {
+	sec := d / time.Second
+	nsec := d % time.Second
+	fl := float64(sec) + float64(nsec)*1e-6
+	return int64(fl)
+}
+
+func (p *ESPlugin) ResponseAnalyze(req, resp []byte, start, stop time.Time) {
+	if len(resp) == 0 {
+		// nil http response - skipped elasticsearch export for this request
+		return
+	}
+	t := time.Now()
+	rtt := p.RttDurationToMs(stop.Sub(start))
+	req = payloadBody(req)
+
+	esResp := ESRequestResponse{
+		ReqURL:               string(proto.Path(req)),
+		ReqMethod:            string(proto.Method(req)),
+		ReqUserAgent:         string(proto.Header(req, []byte("User-Agent"))),
+		ReqAcceptLanguage:    string(proto.Header(req, []byte("Accept-Language"))),
+		ReqAccept:            string(proto.Header(req, []byte("Accept"))),
+		ReqAcceptEncoding:    string(proto.Header(req, []byte("Accept-Encoding"))),
+		ReqIfModifiedSince:   string(proto.Header(req, []byte("If-Modified-Since"))),
+		ReqConnection:        string(proto.Header(req, []byte("Connection"))),
+		ReqCookies:           string(proto.Header(req, []byte("Cookie"))),
+		RespStatus:           string(proto.Status(resp)),
+		RespStatusCode:       string(proto.Status(resp)),
+		RespProto:            string(proto.Method(resp)),
+		RespContentLength:    string(proto.Header(resp, []byte("Content-Length"))),
+		RespContentType:      string(proto.Header(resp, []byte("Content-Type"))),
+		RespTransferEncoding: string(proto.Header(resp, []byte("Transfer-Encoding"))),
+		RespContentEncoding:  string(proto.Header(resp, []byte("Content-Encoding"))),
+		RespExpires:          string(proto.Header(resp, []byte("Expires"))),
+		RespCacheControl:     string(proto.Header(resp, []byte("Cache-Control"))),
+		RespVary:             string(proto.Header(resp, []byte("Vary"))),
+		RespSetCookie:        string(proto.Header(resp, []byte("Set-Cookie"))),
+		Rtt:                  rtt,
+		Timestamp:            t,
+	}
+	j, err := json.Marshal(&esResp)
+	if err != nil {
+		log.Println(err)
+	} else {
+		p.indexor.Index(p.Index, "RequestResponse", "", "", "", &t, j)
+	}
+	return
+}

elasticsearch/elasticsearch.go

@@ -1,153 +0,0 @@
-package elasticsearch
-
-import (
-	"encoding/json"
-	"github.com/mattbaird/elastigo/api"
-	"github.com/mattbaird/elastigo/core"
-	"log"
-	"net/http"
-	"regexp"
-	"time"
-)
-
-type ESUriErorr struct{}
-
-func (e *ESUriErorr) Error() string {
-	return "Wrong ElasticSearch URL format. Expected to be: host:port/index_name"
-}
-
-type ESPlugin struct {
-	Active  bool
-	ApiPort string
-	Host    string
-	Index   string
-	indexor *core.BulkIndexer
-	done    chan bool
-}
-
-type ESRequestResponse struct {
-	ReqUrl               string         `json:"Req_URL"`
-	ReqMethod            string         `json:"Req_Method"`
-	ReqUserAgent         string         `json:"Req_User-Agent"`
-	ReqAcceptLanguage    string         `json:"Req_Accept-Language,omitempty"`
-	ReqAccept            string         `json:"Req_Accept,omitempty"`
-	ReqAcceptEncoding    string         `json:"Req_Accept-Encoding,omitempty"`
-	ReqIfModifiedSince   string         `json:"Req_If-Modified-Since,omitempty"`
-	ReqConnection        string         `json:"Req_Connection,omitempty"`
-	ReqCookies           []*http.Cookie `json:"Req_Cookies,omitempty"`
-	RespStatus           string         `json:"Resp_Status"`
-	RespStatusCode       int            `json:"Resp_Status-Code"`
-	RespProto            string         `json:"Resp_Proto,omitempty"`
-	RespContentLength    int64          `json:"Resp_Content-Length,omitempty"`
-	RespContentType      string         `json:"Resp_Content-Type,omitempty"`
-	RespTransferEncoding []string       `json:"Resp_Transfer-Encoding,omitempty"`
-	RespContentEncoding  string         `json:"Resp_Content-Encoding,omitempty"`
-	RespExpires          string         `json:"Resp_Expires,omitempty"`
-	RespCacheControl     string         `json:"Resp_Cache-Control,omitempty"`
-	RespVary             string         `json:"Resp_Vary,omitempty"`
-	RespSetCookie        string         `json:"Resp_Set-Cookie,omitempty"`
-	Rtt                  int64          `json:"RTT"`
-	Timestamp            time.Time
-}
-
-// Parse ElasticSearch URI
-//
-// Proper format is: host:port/index_name
-func parseURI(URI string) (err error, host string, port string, index string) {
-	rURI := regexp.MustCompile("(.+):([0-9]+)/(.+)")
-	match := rURI.FindAllStringSubmatch(URI, -1)
-
-	if len(match) == 0 {
-		err = new(ESUriErorr)
-	} else {
-		host = match[0][1]
-		port = match[0][2]
-		index = match[0][3]
-	}
-
-	return
-}
-
-func (p *ESPlugin) Init(URI string) {
-	var err error
-
-	err, p.Host, p.ApiPort, p.Index = parseURI(URI)
-
-	if err != nil {
-		log.Fatal("Can't initialize ElasticSearch plugin.", err)
-	}
-
-	api.Domain = p.Host
-	api.Port = p.ApiPort
-
-	p.indexor = core.NewBulkIndexerErrors(50, 60)
-	p.done = make(chan bool)
-	p.indexor.Run(p.done)
-
-	// Only start the ErrorHandler goroutine when in verbose mode
-	// no need to burn ressources otherwise
-	// go p.ErrorHandler()
-
-	log.Println("Initialized Elasticsearch Plugin")
-	return
-}
-
-func (p *ESPlugin) IndexerShutdown() {
-	p.done <- true
-	return
-}
-
-func (p *ESPlugin) ErrorHandler() {
-	for {
-		errBuf := <-p.indexor.ErrorChannel
-		log.Println(errBuf.Err)
-	}
-}
-
-func (p *ESPlugin) RttDurationToMs(d time.Duration) int64 {
-	sec := d / time.Second
-	nsec := d % time.Second
-	fl := float64(sec) + float64(nsec)*1e-6
-	return int64(fl)
-}
-
-func (p *ESPlugin) ResponseAnalyze(req *http.Request, resp *http.Response, start, stop time.Time) {
-	if resp == nil {
-		// nil http response - skipped elasticsearch export for this request
-		return
-	}
-	t := time.Now()
-	rtt := p.RttDurationToMs(stop.Sub(start))
-
-	esResp := ESRequestResponse{
-		ReqUrl:               req.URL.String(),
-		ReqMethod:            req.Method,
-		ReqUserAgent:         req.UserAgent(),
-		ReqAcceptLanguage:    req.Header.Get("Accept-Language"),
-		ReqAccept:            req.Header.Get("Accept"),
-		ReqAcceptEncoding:    req.Header.Get("Accept-Encoding"),
-		ReqIfModifiedSince:   req.Header.Get("If-Modified-Since"),
-		ReqConnection:        req.Header.Get("Connection"),
-		ReqCookies:           req.Cookies(),
-		RespStatus:           resp.Status,
-		RespStatusCode:       resp.StatusCode,
-		RespProto:            resp.Proto,
-		RespContentLength:    resp.ContentLength,
-		RespContentType:      resp.Header.Get("Content-Type"),
-		RespTransferEncoding: resp.TransferEncoding,
-		RespContentEncoding:  resp.Header.Get("Content-Encoding"),
-		RespExpires:          resp.Header.Get("Expires"),
-		RespCacheControl:     resp.Header.Get("Cache-Control"),
-		RespVary:             resp.Header.Get("Vary"),
-		RespSetCookie:        resp.Header.Get("Set-Cookie"),
-		Rtt:                  rtt,
-		Timestamp:            t,
-	}
-	j, err := json.Marshal(&esResp)
-	if err != nil {
-		log.Println(err)
-	} else {
-		p.indexor.Index(p.Index, "RequestResponse", "", "", &t, j)
-	}
-	return
-}

emitter.go

@@ -1,33 +1,88 @@
 package main
 
 import (
+	"bytes"
 	"io"
+	"time"
 )
 
+// Start initialize loop for sending data from inputs to outputs
 func Start(stop chan int) {
-	for _, in := range Plugins.Inputs {
-		go CopyMulty(in, Plugins.Outputs...)
+	if Settings.middleware != "" {
+		middleware := NewMiddleware(Settings.middleware)
+
+		for _, in := range Plugins.Inputs {
+			middleware.ReadFrom(in)
+		}
+
+		// We going only to read responses, so using same ReadFrom method
+		for _, out := range Plugins.Outputs {
+			if r, ok := out.(io.Reader); ok {
+				middleware.ReadFrom(r)
+			}
+		}
+
+		go CopyMulty(middleware, Plugins.Outputs...)
+	} else {
+		for _, in := range Plugins.Inputs {
+			go CopyMulty(in, Plugins.Outputs...)
+		}
 	}
 
-	select {
-	case <-stop:
-		return
+	for {
+		select {
+		case <-stop:
+			finalize()
+			return
+		case <-time.After(100 * time.Millisecond):
+		}
 	}
 }
 
-// Copy from 1 reader to multiple writers
+// CopyMulty copies from 1 reader to multiple writers
 func CopyMulty(src io.Reader, writers ...io.Writer) (err error) {
-	buf := make([]byte, 32*1024)
+	buf := make([]byte, 5*1024*1024)
 	wIndex := 0
+	modifier := NewHTTPModifier(&Settings.modifierConfig)
 
 	for {
 		nr, er := src.Read(buf)
-		if nr > 0 && len(buf) > nr{
-			Debug("Sending", src, ": ", string(buf[0:nr]))
+
+		if nr > 0 && len(buf) > nr {
+			payload := buf[:nr]
+
+			_maxN := nr
+			if nr > 500 {
+				_maxN = 500
+			}
+
+			if Settings.debug {
+				Debug("[EMITTER] input:", string(payload[0:_maxN]), nr, "from:", src)
+			}
+
+			if modifier != nil && isRequestPayload(payload) {
+				headSize := bytes.IndexByte(payload, '\n') + 1
+				body := payload[headSize:]
+				originalBodyLen := len(body)
+				body = modifier.Rewrite(body)
+
+				// If modifier tells to skip request
+				if len(body) == 0 {
+					continue
+				}
+
+				if originalBodyLen != len(body) {
+					payload = append(payload[:headSize], body...)
+				}
+
+				if Settings.debug {
+					Debug("[EMITTER] Rewrittern input:", len(payload), "First 500 bytes:", string(payload[0:_maxN]))
+				}
+			}
 
 			if Settings.splitOutput {
 				// Simple round robin
-				writers[wIndex].Write(buf[0:nr])
+				writers[wIndex].Write(payload)
 
 				wIndex++
 
@@ -36,7 +91,7 @@ func CopyMulty(src io.Reader, writers ...io.Writer) (err error) {
 				}
 			} else {
 				for _, dst := range writers {
-					dst.Write(buf[0:nr])
+					dst.Write(payload)
 				}
 			}
 

examples/middleware/echo.java

@@ -0,0 +1,27 @@
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+
+public class echo {
+    public static void main(String[] args) {
+        if(args != null){
+            for(String arg : args){
+                System.out.println(arg);
+            }
+
+        }
+
+        BufferedReader stdin = new BufferedReader(new InputStreamReader(
+                System.in));
+        String line = null;
+
+        try {
+            while ((line = stdin.readLine()) != null) {
+
+                System.out.println(line);
+
+            }
+        } catch (IOException e) {
+        }
+    }
+}

examples/middleware/echo.rb

@@ -0,0 +1,22 @@
+#!/usr/bin/env ruby
+# encoding: utf-8
+while data = STDIN.gets # continiously read line from STDIN
+  next unless data
+  data = data.chomp # remove end of line symbol
+  
+  decoded = [data].pack("H*") # decode base64 encoded request
+  
+  # dedoded value is raw HTTP payload, example:
+  #   
+  #   POST /post HTTP/1.1
+  #   Content-Length: 7
+  #   Host: www.w3.org
+  #
+  #   a=1&b=2"
+  
+  encoded = decoded.unpack("H*").first # encoding back to base64
+  
+  # Emit request back
+  # You can skip this if want to filter out request
+  STDOUT.puts encoded
+end

examples/middleware/echo.sh

@@ -0,0 +1,46 @@
+#!/usr/bin/env bash
+#
+# `xxd` utility included into vim-common package
+# It allow hex decoding/encoding
+# 
+# This example may broke if you request contains `null` string, you may consider using pipes instead.
+# See: https://github.com/buger/gor/issues/309
+# 
+
+function log {
+    # Logging to stderr, because stdout/stdin used for data transfer
+    >&2 echo "[DEBUG][ECHO] $1"
+}
+
+while read line; do
+    decoded=$(echo -e "$line" | xxd -r -p)
+
+    header=$(echo -e "$decoded" | head -n +1)
+    payload=$(echo -e "$decoded" | tail -n +2)
+
+    encoded=$(echo -e "$header\n$payload" | xxd -p | tr -d "\\n")
+
+    log ""
+    log "==================================="
+
+    case ${header:0:1} in
+    "1")
+        log "Request type: Request"
+        ;;
+    "2")
+        log "Request type: Original Response"
+        ;;
+    "3")
+        log "Request type: Replayed Response"
+        ;;
+    *)
+        log "Unknown request type $header"
+    esac
+    echo "$encoded"
+
+    log "==================================="
+
+    log "Original data: $line"
+    log "Decoded request: $decoded"
+    log "Encoded data: $encoded"
+done;

examples/middleware/token_modifier.go

@@ -0,0 +1,119 @@
+/*
+This middleware made for auth system that randomly generate access tokens, which used later for accessing secure content. Since there is no pre-defined token value, naive approach without middleware (or if middleware use only request payloads) will fail, because replayed server have own tokens, not synced with origin. To fix this, our middleware should take in account responses of replayed and origin server, store `originalToken -> replayedToken` aliases and rewrite all requests using this token to use replayed alias. See `middleware_test.go#TestTokenMiddleware` test for examples of using this middleware.
+
+How middleware works:
+
+                   Original request      +--------------+
++-------------+----------STDIN---------->+              |
+|  Gor input  |                          |  Middleware  |
++-------------+----------STDIN---------->+              |
+                   Original response     +------+---+---+
+                                                |   ^
++-------------+    Modified request             v   |
+| Gor output  +<---------STDOUT-----------------+   |
++-----+-------+                                     |
+      |                                             |
+      |            Replayed response                |
+      +------------------STDIN----------------->----+
+*/
+
+package main
+
+import (
+	"bufio"
+	"bytes"
+	"encoding/hex"
+	"fmt"
+	"github.com/buger/gor/proto"
+	"os"
+)
+
+// requestID -> originalToken
+var originalTokens map[string][]byte
+
+// originalToken -> replayedToken
+var tokenAliases map[string][]byte
+
+func main() {
+	originalTokens = make(map[string][]byte)
+	tokenAliases = make(map[string][]byte)
+
+	scanner := bufio.NewScanner(os.Stdin)
+
+	for scanner.Scan() {
+		encoded := scanner.Bytes()
+		buf := make([]byte, len(encoded)/2)
+		hex.Decode(buf, encoded)
+
+		process(buf)
+	}
+}
+
+func process(buf []byte) {
+	// First byte indicate payload type, possible values:
+	//  1 - Request
+	//  2 - Response
+	//  3 - ReplayedResponse
+	payloadType := buf[0]
+	headerSize := bytes.IndexByte(buf, '\n') + 1
+	header := buf[:headerSize-1]
+
+	// Header contains space separated values of: request type, request id, and request start time (or round-trip time for responses)
+	meta := bytes.Split(header, []byte(" "))
+	// For each request you should receive 3 payloads (request, response, replayed response) with same request id
+	reqID := string(meta[1])
+	payload := buf[headerSize:]
+
+	Debug("Received payload:", string(buf))
+
+	switch payloadType {
+	case '1': // Request
+		if bytes.Equal(proto.Path(payload), []byte("/token")) {
+			originalTokens[reqID] = []byte{}
+			Debug("Found token request:", reqID)
+		} else {
+			token, vs, _ := proto.PathParam(payload, []byte("token"))
+
+			if vs != -1 { // If there is GET token param
+				if alias, ok := tokenAliases[string(token)]; ok {
+					// Rewrite original token to alias
+					payload = proto.SetPathParam(payload, []byte("token"), alias)
+
+					// Copy modified payload to our buffer
+					buf = append(buf[:headerSize], payload...)
+				}
+			}
+		}
+
+		// Emitting data back
+		os.Stdout.Write(encode(buf))
+	case '2': // Original response
+		if _, ok := originalTokens[reqID]; ok {
+			// Token is inside response body
+			secureToken := proto.Body(payload)
+			originalTokens[reqID] = secureToken
+			Debug("Remember origial token:", string(secureToken))
+		}
+	case '3': // Replayed response
+		if originalToken, ok := originalTokens[reqID]; ok {
+			delete(originalTokens, reqID)
+			secureToken := proto.Body(payload)
+			tokenAliases[string(originalToken)] = secureToken
+
+			Debug("Create alias for new token token, was:", string(originalToken), "now:", string(secureToken))
+		}
+	}
+}
+
+func encode(buf []byte) []byte {
+	dst := make([]byte, len(buf)*2+1)
+	hex.Encode(dst, buf)
+	dst[len(dst)-1] = '\n'
+
+	return dst
+}
+
+func Debug(args ...interface{}) {
+	fmt.Fprint(os.Stderr, "[DEBUG][TOKEN-MOD] ")
+	fmt.Fprintln(os.Stderr, args...)
+}

gor.go

@@ -5,10 +5,16 @@ package main
 import (
 	"flag"
 	"fmt"
+	"io"
 	"log"
+	"net/http"
+	"net/http/httputil"
 	"os"
-	"runtime/debug"
+	"os/signal"
+	"runtime"
+	_ "runtime/debug"
 	"runtime/pprof"
+	"syscall"
 	"time"
 )
 
@@ -18,21 +24,44 @@ var (
 	memprofile = flag.String("memprofile", "", "write memory profile to this file")
 )
 
+func loggingMiddleware(next http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		rb, _ := httputil.DumpRequest(r, false)
+		log.Println(string(rb))
+		next.ServeHTTP(w, r)
+	})
+}
+
 func main() {
-	// Don't exit on panic
-	defer func() {
-		if r := recover(); r != nil {
-			if _, ok := r.(error); !ok {
-				fmt.Printf("PANIC: pkg: %v %s \n", r, debug.Stack())
-			}
+	// // Don't exit on panic
+	// defer func() {
+	// 	if r := recover(); r != nil {
+	// 		fmt.Printf("PANIC: pkg: %v %s \n", r, debug.Stack())
+	// 	}
+	// }()
+
+	// If not set via env cariable
+	if len(os.Getenv("GOMAXPROCS")) == 0 {
+		runtime.GOMAXPROCS(runtime.NumCPU() * 2)
+	}
+
+	args := os.Args[1:]
+	if len(args) > 0 && args[0] == "file-server" {
+		if len(args) != 2 {
+			log.Fatal("You should specify port and IP (optional) for the file server. Example: `gor file-server :80`")
 		}
-	}()
+		dir, _ := os.Getwd()
+
+		log.Println("Started example file server for current dirrectory on address ", args[1])
+
+		log.Fatal(http.ListenAndServe(args[1], loggingMiddleware(http.FileServer(http.Dir(dir)))))
+	} else {
+		flag.Parse()
+		InitPlugins()
+	}
 
 	fmt.Println("Version:", VERSION)
 
-	flag.Parse()
-	InitPlugins()
-
 	if len(Plugins.Inputs) == 0 || len(Plugins.Outputs) == 0 {
 		log.Fatal("Required at least 1 input and 1 output")
 	}
@@ -45,7 +74,35 @@ func main() {
 		profileCPU(*cpuprofile)
 	}
 
-	Start(nil)
+	c := make(chan os.Signal, 1)
+	signal.Notify(c, os.Interrupt, syscall.SIGTERM)
+	go func() {
+		<-c
+		finalize()
+		os.Exit(1)
+	}()
+
+	if Settings.exitAfter > 0 {
+		log.Println("Running gor for a duration of", Settings.exitAfter)
+		closeCh := make(chan int)
+
+		time.AfterFunc(Settings.exitAfter, func() {
+			log.Println("Stopping gor after", Settings.exitAfter)
+			close(closeCh)
+		})
+
+		Start(closeCh)
+	} else {
+		Start(nil)
+	}
+}
+
+func finalize() {
+	for _, p := range Plugins.All {
+		if cp, ok := p.(io.Closer); ok {
+			cp.Close()
+		}
+	}
 }
 
 func profileCPU(cpuprofile string) {
@@ -56,10 +113,10 @@ func profileCPU(cpuprofile string) {
 		}
 		pprof.StartCPUProfile(f)
 
-		time.AfterFunc(60*time.Second, func() {
+		time.AfterFunc(30*time.Second, func() {
 			pprof.StopCPUProfile()
 			f.Close()
-			log.Println("Stop profiling after 60 seconds")
+			log.Println("Stop profiling after 30 seconds")
 		})
 	}
 }
@@ -70,9 +127,9 @@ func profileMEM(memprofile string) {
 		if err != nil {
 			log.Fatal(err)
 		}
-		time.AfterFunc(60*time.Second, func() {
+		time.AfterFunc(30*time.Second, func() {
 			pprof.WriteHeapProfile(f)
 			f.Close()
 		})
 	}
-}
+}

gor_stat.go

@@ -1,9 +1,10 @@
 package main
 
 import (
-	"time"
 	"log"
+	"runtime"
 	"strconv"
+	"time"
 )
 
 const (
@@ -11,11 +12,11 @@ const (
 )
 
 type GorStat struct {
-	statName	string
-	latest		int
-	mean	 	int
-	max			int
-	count		int
+	statName string
+	latest   int
+	mean     int
+	max      int
+	count    int
 }
 
 func NewGorStat(statName string) (s *GorStat) {
@@ -27,7 +28,7 @@ func NewGorStat(statName string) (s *GorStat) {
 	s.count = 0
 
 	if Settings.stats {
-		log.Println(s.statName + ":latest,mean,max,count,count/second")
+		log.Println(s.statName + ":latest,mean,max,count,count/second,gcount")
 		go s.reportStats()
 	}
 	return
@@ -54,14 +55,13 @@ func (s *GorStat) Reset() {
 }
 
 func (s *GorStat) String() string {
-	return s.statName + ":" + strconv.Itoa(s.latest) + "," + strconv.Itoa(s.mean) + "," + strconv.Itoa(s.max) + "," + strconv.Itoa(s.count) + "," + strconv.Itoa(s.count / rate)
+	return s.statName + ":" + strconv.Itoa(s.latest) + "," + strconv.Itoa(s.mean) + "," + strconv.Itoa(s.max) + "," + strconv.Itoa(s.count) + "," + strconv.Itoa(s.count/rate) + "," + strconv.Itoa(runtime.NumGoroutine())
 }
 
 func (s *GorStat) reportStats() {
 	for {
-			log.Println(s)
-			s.Reset()
-			time.Sleep(rate * time.Second)
+		log.Println(s)
+		s.Reset()
+		time.Sleep(rate * time.Second)
 	}
 }
-

homebrew/gor.rb

@@ -0,0 +1,46 @@
+require "language/go"
+
+class Gor < Formula
+  desc "Real-time HTTP traffic replay tool written in Go"
+  homepage "https://gortool.com"
+  url "https://github.com/buger/gor/archive/v0.14.0.tar.gz"
+  sha256 "62260a6f5cabde571b91d5762fba9c47691643df0a58565cbe808854cd064dc8"
+  head "https://github.com/buger/gor.git"
+
+  bottle do
+    cellar :any_skip_relocation
+    sha256 "c382403de70a41b7445920a02051f5e82030704aaaae70cfcd4e8f401cc87f6a" => :el_capitan
+    sha256 "4b76b3785584897800e87967f1af9510208faefe46f57d7bd6f8b40a7133c19b" => :yosemite
+    sha256 "d186cb1566d33ab8f78215e69934f49dd96becb1c236905b4502d94399ae1974" => :mavericks
+  end
+
+  depends_on "go" => :build
+
+  go_resource "github.com/bitly/go-hostpool" do
+    url "https://github.com/bitly/go-hostpool.git",
+      :revision => "d0e59c22a56e8dadfed24f74f452cea5a52722d2"
+  end
+
+  go_resource "github.com/buger/elastigo" do
+    url "https://github.com/buger/elastigo.git",
+      :revision => "23fcfd9db0d8be2189a98fdab77a4c90fcc3a1e9"
+  end
+
+  go_resource "github.com/google/gopacket" do
+    url "https://github.com/google/gopacket.git",
+      :revision => "aa09ced736460d76535444c825932a0742975f7d"
+  end
+
+  def install
+    ENV["GOPATH"] = buildpath
+    mkdir_p buildpath/"src/github.com/buger/"
+    ln_sf buildpath, buildpath/"src/github.com/buger/gor"
+    Language::Go.stage_deps resources, buildpath/"src"
+
+    system "go", "build", "-o", "#{bin}/gor", "-ldflags", "-X main.VERSION \"#{version}\""
+  end
+
+  test do
+    assert_match version.to_s, shell_output("#{bin}/gor", 1)
+  end
+end

http_client.go

@@ -0,0 +1,357 @@
+package main
+
+import (
+	"bytes"
+	"crypto/tls"
+	"io"
+	"log"
+	"net"
+	"net/url"
+	"runtime/debug"
+	"strconv"
+	"strings"
+	"sync"
+	"syscall"
+	"time"
+
+	"github.com/buger/gor/proto"
+)
+
+var httpMu sync.Mutex
+
+const (
+	readChunkSize   = 64 * 1024
+	maxResponseSize = 1073741824
+)
+
+var chunkedSuffix = []byte("0\r\n\r\n")
+
+var defaultPorts = map[string]string{
+	"http":  "80",
+	"https": "443",
+}
+
+type HTTPClientConfig struct {
+	FollowRedirects    int
+	Debug              bool
+	OriginalHost       bool
+	ConnectionTimeout  time.Duration
+	Timeout            time.Duration
+	ResponseBufferSize int
+}
+
+type HTTPClient struct {
+	baseURL        string
+	scheme         string
+	host           string
+	conn           net.Conn
+	respBuf        []byte
+	config         *HTTPClientConfig
+	redirectsCount int
+}
+
+func NewHTTPClient(baseURL string, config *HTTPClientConfig) *HTTPClient {
+	if !strings.HasPrefix(baseURL, "http") {
+		baseURL = "http://" + baseURL
+	}
+
+	u, _ := url.Parse(baseURL)
+	if !strings.Contains(u.Host, ":") {
+		if u.Scheme != "http" {
+			u.Host += ":" + defaultPorts[u.Scheme]
+		}
+	}
+
+	if config.Timeout.Nanoseconds() == 0 {
+		config.Timeout = 5 * time.Second
+	}
+
+	config.ConnectionTimeout = config.Timeout
+
+	if config.ResponseBufferSize == 0 {
+		config.ResponseBufferSize = 100 * 1024 // 100kb
+	}
+
+	client := new(HTTPClient)
+	client.baseURL = u.String()
+	client.host = u.Host
+	client.scheme = u.Scheme
+	client.respBuf = make([]byte, config.ResponseBufferSize)
+	client.config = config
+
+	return client
+}
+
+func (c *HTTPClient) Connect() (err error) {
+	c.Disconnect()
+
+	if !strings.Contains(c.host, ":") {
+		c.conn, err = net.DialTimeout("tcp", c.host+":80", c.config.ConnectionTimeout)
+	} else {
+		c.conn, err = net.DialTimeout("tcp", c.host, c.config.ConnectionTimeout)
+	}
+
+	if c.scheme == "https" {
+		tlsConn := tls.Client(c.conn, &tls.Config{InsecureSkipVerify: true})
+
+		if err = tlsConn.Handshake(); err != nil {
+			return
+		}
+
+		c.conn = tlsConn
+	}
+
+	return
+}
+
+func (c *HTTPClient) Disconnect() {
+	if c.conn != nil {
+		c.conn.Close()
+		c.conn = nil
+		Debug("[HTTP] Disconnected: ", c.baseURL)
+	}
+}
+
+func (c *HTTPClient) isAlive() bool {
+	one := make([]byte, 1)
+
+	// Ready 1 byte from socket without timeout to check if it not closed
+	c.conn.SetReadDeadline(time.Now().Add(time.Millisecond))
+	_, err := c.conn.Read(one)
+
+	if err == nil {
+		return true
+	} else if err == io.EOF {
+		if c.config.Debug {
+			Debug("[HTTPClient] connection closed, reconnecting")
+		}
+		return false
+	} else if err == syscall.EPIPE {
+		Debug("Detected broken pipe.", err)
+		return false
+	}
+
+	return true
+}
+
+func (c *HTTPClient) Send(data []byte) (response []byte, err error) {
+	// Don't exit on panic
+	defer func() {
+		if r := recover(); r != nil {
+			Debug("[HTTPClient]", r, string(data))
+
+			if _, ok := r.(error); !ok {
+				log.Println("[HTTPClient] Failed to send request: ", string(data))
+				log.Println("PANIC: pkg:", r, debug.Stack())
+			}
+		}
+	}()
+
+	if c.conn == nil || !c.isAlive() {
+		Debug("[HTTPClient] Connecting:", c.baseURL)
+		if err = c.Connect(); err != nil {
+			log.Println("[HTTPClient] Connection error:", err)
+			response = errorPayload(HTTP_CONNECTION_ERROR)
+			return
+		}
+	}
+
+	timeout := time.Now().Add(c.config.Timeout)
+
+	c.conn.SetWriteDeadline(timeout)
+
+	if !c.config.OriginalHost {
+		data = proto.SetHost(data, []byte(c.baseURL), []byte(c.host))
+	}
+
+	if c.config.Debug {
+		Debug("[HTTPClient] Sending:", string(data))
+	}
+
+	if _, err = c.conn.Write(data); err != nil {
+		Debug("[HTTPClient] Write error:", err, c.baseURL)
+		response = errorPayload(HTTP_TIMEOUT)
+		return
+	}
+
+	var readBytes, n int
+	var currentChunk []byte
+	timeout = time.Now().Add(c.config.Timeout)
+	chunked := false
+	contentLength := -1
+	currentContentLength := 0
+	chunks := 0
+
+	for {
+		c.conn.SetReadDeadline(timeout)
+
+		if readBytes < len(c.respBuf) {
+			n, err = c.conn.Read(c.respBuf[readBytes:])
+			readBytes += n
+			chunks++
+
+			if err != nil {
+				if err == io.EOF {
+					err = nil
+				}
+				break
+			}
+
+			// First chunk
+			if chunked || contentLength != -1 {
+				currentContentLength += n
+			} else {
+				// If headers are finished
+				if bytes.Contains(c.respBuf[:readBytes], proto.EmptyLine) {
+					if bytes.Equal(proto.Header(c.respBuf, []byte("Transfer-Encoding")), []byte("chunked")) {
+						chunked = true
+					} else {
+						l := proto.Header(c.respBuf, []byte("Content-Length"))
+						if len(l) > 0 {
+							contentLength, _ = strconv.Atoi(string(l))
+						}
+					}
+
+					currentContentLength += len(proto.Body(c.respBuf[:readBytes]))
+				}
+			}
+
+			if chunked {
+				// Check if chunked message finished
+				if bytes.HasSuffix(c.respBuf[:readBytes], chunkedSuffix) {
+					break
+				}
+			} else if contentLength != -1 {
+				if currentContentLength > contentLength {
+					Debug("[HTTPClient] disconnected, wrong length", currentContentLength, contentLength)
+					c.Disconnect()
+					break
+				} else if currentContentLength == contentLength {
+					break
+				}
+			}
+		} else {
+			if currentChunk == nil {
+				currentChunk = make([]byte, readChunkSize)
+			}
+
+			n, err = c.conn.Read(currentChunk)
+
+			if err == io.EOF {
+				break
+			} else if err != nil {
+				Debug("[HTTPClient] Read the whole body error:", err, c.baseURL)
+				break
+			}
+
+			readBytes += int(n)
+			chunks++
+			currentContentLength += n
+
+			if chunked {
+				// Check if chunked message finished
+				if bytes.HasSuffix(currentChunk[:n], chunkedSuffix) {
+					break
+				}
+			} else if contentLength != -1 {
+				if currentContentLength > contentLength {
+					Debug("[HTTPClient] disconnected, wrong length", currentContentLength, contentLength)
+					c.Disconnect()
+					break
+				} else if currentContentLength == contentLength {
+					break
+				}
+			} else {
+				Debug("[HTTPClient] disconnected, can't find Content-Length or Chunked")
+				c.Disconnect()
+				break
+			}
+		}
+
+		if readBytes >= maxResponseSize {
+			Debug("[HTTPClient] Body is more than the max size", maxResponseSize,
+				c.baseURL)
+			break
+		}
+
+		// For following chunks expect less timeout
+		timeout = time.Now().Add(c.config.Timeout / 5)
+	}
+
+	if err != nil {
+		Debug("[HTTPClient] Response read error", err, c.conn, readBytes)
+		response = errorPayload(HTTP_TIMEOUT)
+		return
+	}
+
+	if readBytes > len(c.respBuf) {
+		readBytes = len(c.respBuf)
+	}
+	payload := make([]byte, readBytes)
+	copy(payload, c.respBuf[:readBytes])
+
+	if c.config.Debug {
+		Debug("[HTTPClient] Received:", string(payload))
+	}
+
+	if c.config.FollowRedirects > 0 && c.redirectsCount < c.config.FollowRedirects {
+		status := payload[9:12]
+
+		// 3xx requests
+		if status[0] == '3' {
+			c.redirectsCount++
+
+			location := proto.Header(payload, []byte("Location"))
+			redirectPayload := []byte("GET " + string(location) + " HTTP/1.1\r\n\r\n")
+
+			if c.config.Debug {
+				Debug("[HTTPClient] Redirecting to: " + string(location))
+			}
+
+			return c.Send(redirectPayload)
+		}
+	}
+
+	c.redirectsCount = 0
+
+	return payload, err
+}
+
+func (c *HTTPClient) Get(path string) (response []byte, err error) {
+	payload := "GET " + path + " HTTP/1.1\r\n\r\n"
+
+	return c.Send([]byte(payload))
+}
+
+func (c *HTTPClient) Post(path string, body []byte) (response []byte, err error) {
+	payload := "POST " + path + " HTTP/1.1\r\n"
+	payload += "Content-Length: " + strconv.Itoa(len(body)) + "\r\n\r\n"
+	payload += string(body)
+
+	return c.Send([]byte(payload))
+}
+
+const (
+	// https://support.cloudflare.com/hc/en-us/articles/200171936-Error-520-Web-server-is-returning-an-unknown-error
+	HTTP_UNKNOWN_ERROR = "520"
+	// https://support.cloudflare.com/hc/en-us/articles/200171916-Error-521-Web-server-is-down
+	HTTP_CONNECTION_ERROR = "521"
+	// https://support.cloudflare.com/hc/en-us/articles/200171906-Error-522-Connection-timed-out
+	HTTP_CONNECTION_TIMEOUT = "522"
+	// https://support.cloudflare.com/hc/en-us/articles/200171946-Error-523-Origin-is-unreachable
+	HTTP_UNREACHABLE = "523"
+	// https://support.cloudflare.com/hc/en-us/articles/200171926-Error-524-A-timeout-occurred
+	HTTP_TIMEOUT = "524"
+)
+
+var errorPayloadTemplate = "HTTP/1.1 202 Accepted\r\nDate: Mon, 17 Aug 2015 14:10:11 GMT\r\nContent-Length: 0\r\nContent-Type: text/plain; charset=utf-8\r\n\r\n"
+
+func errorPayload(errorCode string) []byte {
+	payload := make([]byte, len(errorPayloadTemplate))
+	copy(payload, errorPayloadTemplate)
+
+	copy(payload[29:58], []byte(time.Now().Format(time.RFC1123)))
+	copy(payload[9:12], errorCode)
+
+	return payload
+}

http_client_test.go

@@ -0,0 +1,444 @@
+package main
+
+import (
+	"bytes"
+	"crypto/rand"
+	"github.com/buger/gor/proto"
+	"io/ioutil"
+	_ "log"
+	"net"
+	"net/http"
+	"net/http/httptest"
+	"net/http/httputil"
+	_ "reflect"
+	"sync"
+	"testing"
+	"time"
+)
+
+func TestHTTPClientURLPort(t *testing.T) {
+	c1 := NewHTTPClient("http://example.com", &HTTPClientConfig{})
+	if c1.baseURL != "http://example.com" {
+		t.Error("Sould not add 80 port for http:", c1.baseURL)
+	}
+
+	c2 := NewHTTPClient("https://example.com", &HTTPClientConfig{})
+	if c2.baseURL != "https://example.com:443" {
+		t.Error("Sould add 443 port for https:", c2.baseURL)
+	}
+
+	c3 := NewHTTPClient("https://example.com:1", &HTTPClientConfig{})
+	if c3.baseURL != "https://example.com:1" {
+		t.Error("Sould use specified port:", c3.baseURL)
+	}
+
+	c4 := NewHTTPClient("example.com", &HTTPClientConfig{})
+	if c4.baseURL != "http://example.com" {
+		t.Error("Sould not add default protocol:", c4.baseURL)
+	}
+}
+
+func TestHTTPClientSend(t *testing.T) {
+	wg := new(sync.WaitGroup)
+
+	payload := func(reqType string) []byte {
+		switch reqType {
+		case "GET":
+			return []byte("GET / HTTP/1.1\r\n\r\n")
+		case "POST":
+			return []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
+		case "POST_CHUNKED":
+			return []byte("POST / HTTP/1.1\r\nHost: www.w3.org\r\nTransfer-Encoding: chunked\r\n\r\n4\r\nWiki\r\n5\r\npedia\r\ne\r\n in\r\n\r\nchunks.\r\n0\r\n\r\n")
+		}
+
+		return []byte("")
+	}
+
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+
+		if r.Method == "POST" {
+			defer r.Body.Close()
+			body, _ := ioutil.ReadAll(r.Body)
+
+			if len(r.TransferEncoding) > 0 && r.TransferEncoding[0] == "chunked" {
+				if string(body) != "Wikipedia in\r\n\r\nchunks." {
+					t.Error("Wrong POST body:", body, string(body))
+				}
+			} else {
+				if string(body) != "a=1&b=2" {
+					buf, _ := httputil.DumpRequest(r, true)
+					t.Error("Wrong POST body:", string(body), string(buf))
+				}
+			}
+		}
+
+		wg.Done()
+	}))
+	defer server.Close()
+
+	client := NewHTTPClient(server.URL, &HTTPClientConfig{Debug: true})
+
+	wg.Add(4)
+	client.Send(payload("POST"))
+	client.Send(payload("GET"))
+	client.Send(payload("POST_CHUNKED"))
+	client.Send(payload("POST"))
+
+	wg.Wait()
+}
+
+func TestHTTPClientResonseByClose(t *testing.T) {
+	wg := new(sync.WaitGroup)
+
+	payload := []byte("GET / HTTP/1.1\r\n\r\n")
+	ln, _ := net.Listen("tcp", ":0")
+	go func(){
+		for {
+			conn, _ := ln.Accept()
+			buf := make([]byte, 4096)
+			conn.Read(buf)
+
+			conn.Write([]byte("HTTP/1.1 200 OK\r\n\r\n"))
+			conn.Write([]byte("ab"))
+			conn.Close()
+
+			wg.Done()
+		}
+	}()
+
+	client := NewHTTPClient(ln.Addr().String(), &HTTPClientConfig{Debug: true})
+
+	wg.Add(1)
+	resp, _ := client.Send(payload)
+
+	if !bytes.Equal(resp, []byte("HTTP/1.1 200 OK\r\n\r\nab")) {
+		t.Error("Should return valid response", string(resp))
+	}
+
+	wg.Wait()
+}
+
+// https://github.com/buger/gor/issues/184
+func TestHTTPClientResponseBuffer(t *testing.T) {
+	testCases := []struct {
+		name         string
+		responseSize int
+		buffserSize  int
+		expectedSize int
+		timeout      time.Duration
+	}{
+		{"Chunked, buffer overflow", 10 * 1024, 1024, 1024, 50 * time.Millisecond},
+
+		{"Chunked, fits buffer", 10 * 1024, 64 * 1024, 10*1024 + 145 /* headers length + chunked meta */, 50 * time.Millisecond},
+
+		{"Content-Length, buffer overflow", 1024, 1000, 1000, 50 * time.Millisecond},
+
+		{"Content-Length, fits buffer", 1024, 64 * 1024, 1024 + 118, 50 * time.Millisecond},
+	}
+
+	for _, tc := range testCases {
+		wg := new(sync.WaitGroup)
+
+		payload := []byte("GET / HTTP/1.1\r\n\r\n")
+
+		server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+
+			size := tc.responseSize // 1kb
+			rb := make([]byte, size)
+			rand.Read(rb)
+
+			w.Write(rb[:size/2])
+			w.Write(rb[size/2:])
+
+			wg.Done()
+		}))
+
+		client := NewHTTPClient(server.URL, &HTTPClientConfig{Debug: true, ResponseBufferSize: tc.buffserSize, Timeout: 100 * time.Millisecond})
+
+		wg.Add(2)
+
+		start := time.Now()
+		client.Send(payload)
+		resp, err := client.Send(payload)
+		stop := time.Now()
+
+		if err != nil {
+			t.Error("Request error", err)
+		}
+
+		if stop.Sub(start) > tc.timeout {
+			t.Error("Request took too long", stop.Sub(start), tc.timeout)
+		}
+
+		if len(resp) != tc.expectedSize {
+			t.Error(tc.name, " - Wrong response size:", tc.expectedSize, len(resp))
+		} else {
+			if !bytes.Equal(resp[0:8], []byte("HTTP/1.1")) {
+				t.Error(tc.name, " - Response buffer contains data from previous request", string(resp), len(resp))
+			}
+		}
+
+		wg.Wait()
+		server.Close()
+	}
+}
+
+func TestHTTPClientHTTPSSend(t *testing.T) {
+	wg := new(sync.WaitGroup)
+
+	payload := func(reqType string) []byte {
+		switch reqType {
+		case "GET":
+			return []byte("GET / HTTP/1.1\r\n\r\n")
+		case "POST":
+			return []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
+		case "POST_CHUNKED":
+			return []byte("POST / HTTP/1.1\r\nHost: www.w3.org\r\nTransfer-Encoding: chunked\r\n\r\n4\r\nWiki\r\n5\r\npedia\r\ne\r\n in\r\n\r\nchunks.\r\n0\r\n\r\n")
+		}
+
+		return []byte("")
+	}
+
+	server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+
+		if r.Method == "POST" {
+			defer r.Body.Close()
+			body, _ := ioutil.ReadAll(r.Body)
+
+			if len(r.TransferEncoding) > 0 && r.TransferEncoding[0] == "chunked" {
+				if string(body) != "Wikipedia in\r\n\r\nchunks." {
+					t.Error("Wrong POST body:", body, string(body))
+				}
+			} else {
+				if string(body) != "a=1&b=2" {
+					buf, _ := httputil.DumpRequest(r, true)
+					t.Error("Wrong POST body:", string(body), string(buf))
+				}
+			}
+		}
+
+		wg.Done()
+	}))
+	defer server.Close()
+
+	client := NewHTTPClient(server.URL, &HTTPClientConfig{})
+
+	wg.Add(4)
+	client.Send(payload("POST"))
+	client.Send(payload("GET"))
+	client.Send(payload("POST_CHUNKED"))
+	client.Send(payload("POST"))
+
+	wg.Wait()
+}
+
+func TestHTTPClientServerInstantDisconnect(t *testing.T) {
+	wg := new(sync.WaitGroup)
+
+	GETPayload := []byte("GET / HTTP/1.1\r\n\r\n")
+
+	ln, _ := net.Listen("tcp", "127.0.0.1:0")
+	defer ln.Close()
+
+	go func() {
+		for {
+			conn, err := ln.Accept()
+
+			if err != nil {
+				break
+			}
+			conn.Close()
+			wg.Done()
+		}
+	}()
+
+	client := NewHTTPClient(ln.Addr().String(), &HTTPClientConfig{})
+
+	wg.Add(2)
+	client.Send(GETPayload)
+	client.Send(GETPayload)
+
+	wg.Wait()
+}
+
+func TestHTTPClientServerNoKeepAlive(t *testing.T) {
+	wg := new(sync.WaitGroup)
+
+	GETPayload := []byte("GET / HTTP/1.1\r\n\r\n")
+
+	ln, _ := net.Listen("tcp", "127.0.0.1:0")
+	defer ln.Close()
+
+	go func() {
+		for {
+			conn, err := ln.Accept()
+			if err != nil {
+				break
+			}
+
+			buf := make([]byte, 4096)
+			reqLen, err := conn.Read(buf)
+			if err != nil {
+				t.Error("Error reading:", err.Error())
+			}
+			Debug("Received: ", string(buf[0:reqLen]))
+			conn.Write([]byte("OK"))
+
+			// No keep-alive connections
+			conn.Close()
+
+			wg.Done()
+		}
+	}()
+
+	client := NewHTTPClient(ln.Addr().String(), &HTTPClientConfig{})
+
+	wg.Add(2)
+	client.Send(GETPayload)
+	client.Send(GETPayload)
+
+	wg.Wait()
+}
+
+func TestHTTPClientRedirect(t *testing.T) {
+	wg := new(sync.WaitGroup)
+
+	GETPayload := []byte("GET / HTTP/1.1\r\n\r\n")
+
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+
+		if r.URL.Path == "/" {
+			http.Redirect(w, r, "/new", 301)
+		}
+
+		wg.Done()
+	}))
+	defer server.Close()
+
+	client := NewHTTPClient(server.URL, &HTTPClientConfig{FollowRedirects: 1, Debug: false})
+
+	// Should do 2 queries
+	wg.Add(2)
+	client.Send(GETPayload)
+
+	wg.Wait()
+}
+
+func TestHTTPClientRedirectLimit(t *testing.T) {
+	wg := new(sync.WaitGroup)
+
+	GETPayload := []byte("GET / HTTP/1.1\r\n\r\n")
+
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+
+		if r.URL.Path == "/" {
+			http.Redirect(w, r, "/r1", 301)
+		}
+
+		if r.URL.Path == "/r1" {
+			http.Redirect(w, r, "/r2", 301)
+		}
+
+		if r.URL.Path == "/r2" {
+			http.Redirect(w, r, "/new", 301)
+		}
+
+		wg.Done()
+	}))
+	defer server.Close()
+
+	client := NewHTTPClient(server.URL, &HTTPClientConfig{FollowRedirects: 2, Debug: false})
+
+	// Have 3 redirects + 1 GET, but should do only 2 redirects + GET
+	wg.Add(3)
+	client.Send(GETPayload)
+
+	wg.Wait()
+}
+
+func TestHTTPClientHandleHTTP10(t *testing.T) {
+	wg := new(sync.WaitGroup)
+
+	GETPayload := []byte("GET http://foobar.com/path HTTP/1.0\r\n\r\n")
+
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+
+		if r.URL.Path != "/path" {
+			t.Error("Path not match:", r.URL.Path)
+		}
+
+		wg.Done()
+	}))
+	defer server.Close()
+
+	client := NewHTTPClient(server.URL, &HTTPClientConfig{Debug: true})
+
+	wg.Add(1)
+	client.Send(GETPayload)
+
+	wg.Wait()
+}
+
+func TestHTTPClientErrors(t *testing.T) {
+	req := []byte("GET http://foobar.com/path HTTP/1.0\r\n\r\n")
+
+	// Port not exists
+	client := NewHTTPClient("http://127.0.0.1:1", &HTTPClientConfig{Debug: true})
+	if resp, err := client.Send(req); err != nil {
+		if s := proto.Status(resp); !bytes.Equal(s, []byte("521")) {
+			t.Error("Should return status 521 for connection refused, instead:", string(s))
+		}
+	} else {
+		t.Error("Should throw error")
+	}
+
+	client = NewHTTPClient("http://not.existing", &HTTPClientConfig{Debug: true})
+	if resp, err := client.Send(req); err != nil {
+		if s := proto.Status(resp); !bytes.Equal(s, []byte("521")) {
+			t.Error("Should return status 521 for no such host, instead:", string(s))
+		}
+	} else {
+		t.Error("Should throw error")
+	}
+
+	// Non routable IP address to simulate connection timeout
+	client = NewHTTPClient("http://10.255.255.1", &HTTPClientConfig{Debug: true, ConnectionTimeout: 100 * time.Millisecond})
+
+	if resp, err := client.Send(req); err != nil {
+		if s := proto.Status(resp); !bytes.Equal(s, []byte("521")) {
+			t.Error("Should return status 521 for io/timeout:", string(s))
+		}
+	} else {
+		t.Error("Should throw error")
+	}
+
+	// Connecting but io timeout on read
+	ln, _ := net.Listen("tcp", "127.0.0.1:0")
+	client = NewHTTPClient("http://"+ln.Addr().String(), &HTTPClientConfig{Debug: true, Timeout: 10 * time.Millisecond})
+	defer ln.Close()
+
+	if resp, err := client.Send(req); err != nil {
+		if s := proto.Status(resp); !bytes.Equal(s, []byte("524")) {
+			t.Error("Should return status 524 for io read, instead:", string(s))
+		}
+	} else {
+		t.Error("Should throw error")
+	}
+
+	// Response read error read tcp [::1]:51128: connection reset by peer &{{0xc20802a000}}
+	ln1, _ := net.Listen("tcp", "127.0.0.1:0")
+	go func() {
+		ln1.Accept()
+	}()
+	defer ln1.Close()
+
+	client = NewHTTPClient("http://"+ln1.Addr().String(), &HTTPClientConfig{Debug: true, Timeout: 10 * time.Millisecond})
+
+	if resp, err := client.Send(req); err != nil {
+		if s := proto.Status(resp); !bytes.Equal(s, []byte("524")) {
+			t.Error("Should return status 524 for connection reset by peer, instead:", string(s))
+		}
+	} else {
+		t.Error("Should throw error")
+	}
+}

http_modifier.go

@@ -0,0 +1,157 @@
+package main
+
+import (
+	"bytes"
+	"hash/fnv"
+
+	"github.com/buger/gor/proto"
+)
+
+type HTTPModifier struct {
+	config *HTTPModifierConfig
+}
+
+func NewHTTPModifier(config *HTTPModifierConfig) *HTTPModifier {
+	// Optimization to skip modifier completely if we do not need it
+	if len(config.urlRegexp) == 0 &&
+		len(config.urlNegativeRegexp) == 0 &&
+		len(config.urlRewrite) == 0 &&
+		len(config.headerFilters) == 0 &&
+		len(config.headerNegativeFilters) == 0 &&
+		len(config.headerHashFilters) == 0 &&
+		len(config.paramHashFilters) == 0 &&
+		len(config.params) == 0 &&
+		len(config.headers) == 0 &&
+		len(config.methods) == 0 {
+		return nil
+	}
+
+	return &HTTPModifier{config: config}
+}
+
+func (m *HTTPModifier) Rewrite(payload []byte) (response []byte) {
+	if !proto.IsHTTPPayload(payload) {
+		return payload
+	}
+
+	if len(m.config.methods) > 0 {
+		method := proto.Method(payload)
+
+		matched := false
+
+		for _, m := range m.config.methods {
+			if bytes.Equal(method, m) {
+				matched = true
+				break
+			}
+		}
+
+		if !matched {
+			return
+		}
+	}
+
+	if len(m.config.headers) > 0 {
+		for _, header := range m.config.headers {
+			payload = proto.SetHeader(payload, []byte(header.Name), []byte(header.Value))
+		}
+	}
+
+	if len(m.config.params) > 0 {
+		for _, param := range m.config.params {
+			payload = proto.SetPathParam(payload, param.Name, param.Value)
+		}
+	}
+
+	if len(m.config.urlRegexp) > 0 {
+		path := proto.Path(payload)
+
+		matched := false
+
+		for _, f := range m.config.urlRegexp {
+			if f.regexp.Match(path) {
+				matched = true
+				break
+			}
+		}
+
+		if !matched {
+			return
+		}
+	}
+
+	if len(m.config.urlNegativeRegexp) > 0 {
+		path := proto.Path(payload)
+
+		for _, f := range m.config.urlNegativeRegexp {
+			if f.regexp.Match(path) {
+				return
+			}
+		}
+	}
+
+	if len(m.config.headerFilters) > 0 {
+		for _, f := range m.config.headerFilters {
+			value := proto.Header(payload, f.name)
+
+			if len(value) > 0 && !f.regexp.Match(value) {
+				return
+			}
+		}
+	}
+
+	if len(m.config.headerNegativeFilters) > 0 {
+		for _, f := range m.config.headerNegativeFilters {
+			value := proto.Header(payload, f.name)
+
+			if len(value) > 0 && f.regexp.Match(value) {
+				return
+			}
+		}
+	}
+
+	if len(m.config.headerHashFilters) > 0 {
+		for _, f := range m.config.headerHashFilters {
+			value := proto.Header(payload, f.name)
+
+			if len(value) > 0 {
+				hasher := fnv.New32a()
+				hasher.Write(value)
+
+				if (hasher.Sum32() % 100) >= f.percent {
+					return
+				}
+			}
+		}
+	}
+
+	if len(m.config.paramHashFilters) > 0 {
+		for _, f := range m.config.paramHashFilters {
+			value, s, _ := proto.PathParam(payload, f.name)
+
+			if s != -1 {
+				hasher := fnv.New32a()
+				hasher.Write(value)
+
+				if (hasher.Sum32() % 100) >= f.percent {
+					return
+				}
+			}
+		}
+	}
+
+	if len(m.config.urlRewrite) > 0 {
+		path := proto.Path(payload)
+
+		for _, f := range m.config.urlRewrite {
+			if f.src.Match(path) {
+				path = f.src.ReplaceAll(path, f.target)
+				payload = proto.SetPath(payload, path)
+
+				break
+			}
+		}
+	}
+
+	return payload
+}

http_modifier_settings.go

@@ -0,0 +1,215 @@
+package main
+
+import (
+	"errors"
+	"fmt"
+	"regexp"
+	"strconv"
+	"strings"
+)
+
+// HTTPModifierConfig holds configuration options for built-in traffic modifier
+type HTTPModifierConfig struct {
+	urlNegativeRegexp     HTTPUrlRegexp
+	urlRegexp             HTTPUrlRegexp
+	urlRewrite            UrlRewriteMap
+	headerFilters         HTTPHeaderFilters
+	headerNegativeFilters HTTPHeaderFilters
+	headerHashFilters     HTTPHashFilters
+	paramHashFilters      HTTPHashFilters
+
+	params  HTTPParams
+	headers HTTPHeaders
+	methods HTTPMethods
+}
+
+//
+// Handling of --http-allow-header, --http-disallow-header options
+//
+type headerFilter struct {
+	name   []byte
+	regexp *regexp.Regexp
+}
+
+// HTTPHeaderFilters holds list of headers and their regexps
+type HTTPHeaderFilters []headerFilter
+
+func (h *HTTPHeaderFilters) String() string {
+	return fmt.Sprint(*h)
+}
+
+func (h *HTTPHeaderFilters) Set(value string) error {
+	valArr := strings.SplitN(value, ":", 2)
+	if len(valArr) < 2 {
+		return errors.New("need both header and value, colon-delimited (ex. user_id:^169$).")
+	}
+	r, err := regexp.Compile(valArr[1])
+	if err != nil {
+		return err
+	}
+
+	*h = append(*h, headerFilter{name: []byte(valArr[0]), regexp: r})
+
+	return nil
+}
+
+//
+// Handling of --http-allow-header-hash and --http-allow-param-hash options
+//
+type hashFilter struct {
+	name    []byte
+	percent uint32
+}
+
+type HTTPHashFilters []hashFilter
+
+func (h *HTTPHashFilters) String() string {
+	return fmt.Sprint(*h)
+}
+
+func (h *HTTPHashFilters) Set(value string) error {
+	valArr := strings.SplitN(value, ":", 2)
+	if len(valArr) < 2 {
+		return errors.New("need both header and value, colon-delimited (ex. user_id:50%)")
+	}
+
+	f := hashFilter{name: []byte(valArr[0])}
+
+	if strings.Contains(valArr[1], "%") {
+		p, _ := strconv.ParseInt(valArr[1][:len(valArr[1])-1], 0, 0)
+		f.percent = uint32(p)
+	} else if strings.Contains(valArr[1], "/") {
+		// DEPRECATED format
+		var num, den uint64
+
+		fracArr := strings.Split(valArr[1], "/")
+		num, _ = strconv.ParseUint(fracArr[0], 10, 64)
+		den, _ = strconv.ParseUint(fracArr[1], 10, 64)
+
+		f.percent = uint32((float64(num) / float64(den)) * 100)
+	} else {
+		return errors.New("Value should be percent and contain '%'")
+	}
+
+	*h = append(*h, f)
+
+	return nil
+}
+
+//
+// Handling of --http-set-header option
+//
+type HTTPHeaders []HTTPHeader
+type HTTPHeader struct {
+	Name  string
+	Value string
+}
+
+func (h *HTTPHeaders) String() string {
+	return fmt.Sprint(*h)
+}
+
+func (h *HTTPHeaders) Set(value string) error {
+	v := strings.SplitN(value, ":", 2)
+	if len(v) != 2 {
+		return errors.New("Expected `Key: Value`")
+	}
+
+	header := HTTPHeader{
+		strings.TrimSpace(v[0]),
+		strings.TrimSpace(v[1]),
+	}
+
+	*h = append(*h, header)
+	return nil
+}
+
+//
+// Handling of --http-set-param option
+//
+type HTTPParams []HTTPParam
+type HTTPParam struct {
+	Name  []byte
+	Value []byte
+}
+
+func (h *HTTPParams) String() string {
+	return fmt.Sprint(*h)
+}
+
+func (h *HTTPParams) Set(value string) error {
+	v := strings.SplitN(value, "=", 2)
+	if len(v) != 2 {
+		return errors.New("Expected `Key=Value`")
+	}
+
+	param := HTTPParam{
+		[]byte(strings.TrimSpace(v[0])),
+		[]byte(strings.TrimSpace(v[1])),
+	}
+
+	*h = append(*h, param)
+	return nil
+}
+
+//
+// Handling of --http-allow-method option
+//
+type HTTPMethods [][]byte
+
+func (h *HTTPMethods) String() string {
+	return fmt.Sprint(*h)
+}
+
+func (h *HTTPMethods) Set(value string) error {
+	*h = append(*h, []byte(value))
+	return nil
+}
+
+//
+// Handling of --http-rewrite-url option
+//
+type urlRewrite struct {
+	src    *regexp.Regexp
+	target []byte
+}
+
+type UrlRewriteMap []urlRewrite
+
+func (r *UrlRewriteMap) String() string {
+	return fmt.Sprint(*r)
+}
+
+func (r *UrlRewriteMap) Set(value string) error {
+	valArr := strings.SplitN(value, ":", 2)
+	if len(valArr) < 2 {
+		return errors.New("need both src and target, colon-delimited (ex. /a:/b)")
+	}
+	regexp, err := regexp.Compile(valArr[0])
+	if err != nil {
+		return err
+	}
+	*r = append(*r, urlRewrite{src: regexp, target: []byte(valArr[1])})
+	return nil
+}
+
+//
+// Handling of --http-allow-url option
+//
+type urlRegexp struct {
+	regexp *regexp.Regexp
+}
+
+type HTTPUrlRegexp []urlRegexp
+
+func (r *HTTPUrlRegexp) String() string {
+	return fmt.Sprint(*r)
+}
+
+func (r *HTTPUrlRegexp) Set(value string) error {
+	regexp, err := regexp.Compile(value)
+
+	*r = append(*r, urlRegexp{regexp: regexp})
+
+	return err
+}

http_modifier_settings_test.go

@@ -0,0 +1,65 @@
+package main
+
+import (
+	"testing"
+)
+
+func TestHTTPHeaderFilters(t *testing.T) {
+	filters := HTTPHeaderFilters{}
+
+	err := filters.Set("Header1:^$")
+	if err != nil {
+		t.Error("Should not error on Header1:^$")
+	}
+
+	err = filters.Set("Header2:^:$")
+	if err != nil {
+		t.Error("Should not error on Header2:^:$")
+	}
+
+	// Missing colon
+	err = filters.Set("Header3-^$")
+	if err == nil {
+		t.Error("Should error on Header2:^:$")
+	}
+}
+
+func TestHTTPHashFilters(t *testing.T) {
+	filters := HTTPHashFilters{}
+
+	err := filters.Set("Header1:1/2")
+	if err != nil {
+		t.Error("Should support old syntax")
+	}
+
+	if filters[0].percent != 50 {
+		t.Error("Wrong percentage", filters[0].percent)
+	}
+
+	err = filters.Set("Header2:1")
+	if err == nil {
+		t.Error("Should error on Header2 because no % symbol")
+	}
+
+	err = filters.Set("Header2:10%")
+	if err != nil {
+		t.Error("Should pass")
+	}
+
+	if filters[1].percent != 10 {
+		t.Error("Wrong percentage", filters[1].percent)
+	}
+}
+
+func TestUrlRewriteMap(t *testing.T) {
+	var err error
+	rewrites := UrlRewriteMap{}
+
+	if err = rewrites.Set("/v1/user/([^\\/]+)/ping:/v2/user/$1/ping"); err != nil {
+		t.Error("Should set mapping", err)
+	}
+
+	if err = rewrites.Set("/v1/user/([^\\/]+)/ping"); err == nil {
+		t.Error("Should not set mapping without :")
+	}
+}

http_modifier_test.go

@@ -0,0 +1,247 @@
+package main
+
+import (
+	"bytes"
+	"github.com/buger/gor/proto"
+	"testing"
+)
+
+func TestHTTPModifierWithoutConfig(t *testing.T) {
+	if NewHTTPModifier(&HTTPModifierConfig{}) != nil {
+		t.Error("If no config specified should not be initialized")
+	}
+}
+
+func TestHTTPModifierHeaderFilters(t *testing.T) {
+	filters := HTTPHeaderFilters{}
+	filters.Set("Host:^www.w3.org$")
+
+	modifier := NewHTTPModifier(&HTTPModifierConfig{
+		headerFilters: filters,
+	})
+
+	payload := []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
+
+	if len(modifier.Rewrite(payload)) == 0 {
+		t.Error("Request should pass filters")
+	}
+
+	filters = HTTPHeaderFilters{}
+	// Setting filter that not match our header
+	filters.Set("Host:^www.w4.org$")
+
+	modifier = NewHTTPModifier(&HTTPModifierConfig{
+		headerFilters: filters,
+	})
+
+	if len(modifier.Rewrite(payload)) != 0 {
+		t.Error("Request should not pass filters")
+	}
+}
+
+func TestHTTPModifierHeaderNegativeFilters(t *testing.T) {
+	filters := HTTPHeaderFilters{}
+	filters.Set("Host:^www.w3.org$")
+
+	modifier := NewHTTPModifier(&HTTPModifierConfig{
+		headerNegativeFilters: filters,
+	})
+
+	payload := []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w4.org\r\n\r\na=1&b=2")
+
+	if len(modifier.Rewrite(payload)) == 0 {
+		t.Error("Request should pass filters")
+	}
+
+	filters = HTTPHeaderFilters{}
+	// Setting filter that not match our header
+	filters.Set("Host:^www.w4.org$")
+
+	modifier = NewHTTPModifier(&HTTPModifierConfig{
+		headerNegativeFilters: filters,
+	})
+
+	if len(modifier.Rewrite(payload)) != 0 {
+		t.Error("Request should not pass filters")
+	}
+}
+
+func TestHTTPModifierURLRewrite(t *testing.T) {
+	var url, newURL []byte
+
+	rewrites := UrlRewriteMap{}
+
+	payload := func(url []byte) []byte {
+		return []byte("POST " + string(url) + " HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
+	}
+
+	err := rewrites.Set("/v1/user/([^\\/]+)/ping:/v2/user/$1/ping")
+	if err != nil {
+		t.Error("Should not error on /v1/user/([^\\/]+)/ping:/v2/user/$1/ping")
+	}
+
+	modifier := NewHTTPModifier(&HTTPModifierConfig{
+		urlRewrite: rewrites,
+	})
+
+	url = []byte("/v1/user/joe/ping")
+	if newURL = proto.Path(modifier.Rewrite(payload(url))); bytes.Equal(newURL, url) {
+		t.Error("Request url should have been rewritten, wasn't", string(newURL))
+	}
+
+	url = []byte("/v1/user/ping")
+	if newURL = proto.Path(modifier.Rewrite(payload(url))); !bytes.Equal(newURL, url) {
+		t.Error("Request url should have been rewritten, wasn't", string(newURL))
+	}
+}
+
+func TestHTTPModifierHeaderHashFilters(t *testing.T) {
+	filters := HTTPHashFilters{}
+	filters.Set("Header2:1/2")
+
+	modifier := NewHTTPModifier(&HTTPModifierConfig{
+		headerHashFilters: filters,
+	})
+
+	payload := func(header []byte) []byte {
+		return []byte("POST / HTTP/1.1\r\n" + string(header) + "Content-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
+	}
+
+	if p := modifier.Rewrite(payload([]byte(""))); len(p) == 0 {
+		t.Error("Request should pass filters if Header does not exist")
+	}
+
+	if p := modifier.Rewrite(payload([]byte("Header2: 3\r\n"))); len(p) > 0 {
+		t.Error("Request should not pass filters, Header2 hash too high")
+	}
+
+	if p := modifier.Rewrite(payload([]byte("Header2: 1\r\n"))); len(p) == 0 {
+		t.Error("Request should pass filters")
+	}
+}
+
+func TestHTTPModifierParamHashFilters(t *testing.T) {
+	filters := HTTPHashFilters{}
+	filters.Set("user_id:1/2")
+
+	modifier := NewHTTPModifier(&HTTPModifierConfig{
+		paramHashFilters: filters,
+	})
+
+	payload := func(value []byte) []byte {
+		return []byte("POST /" + string(value) + " HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
+	}
+
+	if p := modifier.Rewrite(payload([]byte(""))); len(p) == 0 {
+		t.Error("Request should pass filters if param does not exist")
+	}
+
+	if p := modifier.Rewrite(payload([]byte("?user_id=3"))); len(p) > 0 {
+		t.Error("Request should not pass filters", string(p))
+	}
+
+	if p := modifier.Rewrite(payload([]byte("?user_id=1"))); len(p) == 0 {
+		t.Error("Request should pass filters")
+	}
+}
+
+func TestHTTPModifierHeaders(t *testing.T) {
+	headers := HTTPHeaders{}
+	headers.Set("Header1:1")
+	headers.Set("Host:localhost")
+
+	modifier := NewHTTPModifier(&HTTPModifierConfig{
+		headers: headers,
+	})
+
+	payload := []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
+	newPayload := []byte("POST /post HTTP/1.1\r\nHeader1: 1\r\nContent-Length: 7\r\nHost: localhost\r\n\r\na=1&b=2")
+
+	if payload = modifier.Rewrite(payload); !bytes.Equal(payload, newPayload) {
+		t.Error("Should update request headers", string(payload))
+	}
+}
+
+func TestHTTPModifierURLRegexp(t *testing.T) {
+	filters := HTTPUrlRegexp{}
+	filters.Set("/v1/app")
+	filters.Set("/v1/api")
+
+	modifier := NewHTTPModifier(&HTTPModifierConfig{
+		urlRegexp: filters,
+	})
+
+	payload := func(url string) []byte {
+		return []byte("POST " + url + " HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
+	}
+
+	if len(modifier.Rewrite(payload("/v1/app/test"))) == 0 {
+		t.Error("Should pass url")
+	}
+
+	if len(modifier.Rewrite(payload("/v1/api/test"))) == 0 {
+		t.Error("Should pass url")
+	}
+
+	if len(modifier.Rewrite(payload("/other"))) > 0 {
+		t.Error("Should not pass url")
+	}
+}
+
+func TestHTTPModifierURLNegativeRegexp(t *testing.T) {
+	filters := HTTPUrlRegexp{}
+	filters.Set("/restricted1")
+	filters.Set("/some/restricted2")
+
+	modifier := NewHTTPModifier(&HTTPModifierConfig{
+		urlNegativeRegexp: filters,
+	})
+
+	payload := func(url string) []byte {
+		return []byte("POST " + url + " HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
+	}
+
+	if len(modifier.Rewrite(payload("/v1/app/test"))) == 0 {
+		t.Error("Should pass url")
+	}
+
+	if len(modifier.Rewrite(payload("/restricted1"))) > 0 {
+		t.Error("Should not pass url")
+	}
+
+	if len(modifier.Rewrite(payload("/some/restricted2"))) > 0 {
+		t.Error("Should not pass url")
+	}
+}
+
+func TestHTTPModifierSetHeader(t *testing.T) {
+	filters := HTTPHeaders{}
+	filters.Set("User-Agent:Gor")
+
+	modifier := NewHTTPModifier(&HTTPModifierConfig{
+		headers: filters,
+	})
+
+	payload := []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
+	payloadAfter := []byte("POST /post HTTP/1.1\r\nUser-Agent: Gor\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
+
+	if payload = modifier.Rewrite(payload); !bytes.Equal(payloadAfter, payload) {
+		t.Error("Should add new header", string(payload))
+	}
+}
+
+func TestHTTPModifierSetParam(t *testing.T) {
+	filters := HTTPParams{}
+	filters.Set("api_key=1")
+
+	modifier := NewHTTPModifier(&HTTPModifierConfig{
+		params: filters,
+	})
+
+	payload := []byte("POST /post?api_key=1234 HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
+	payloadAfter := []byte("POST /post?api_key=1 HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
+
+	if payload = modifier.Rewrite(payload); !bytes.Equal(payloadAfter, payload) {
+		t.Error("Should override param", string(payload))
+	}
+}

input_dummy.go

@@ -4,10 +4,12 @@ import (
 	"time"
 )
 
+// DummyInput used for debugging. It generate 1 "GET /"" request per second.
 type DummyInput struct {
 	data chan []byte
 }
 
+// NewDummyInput constructor for DummyInput
 func NewDummyInput(options string) (di *DummyInput) {
 	di = new(DummyInput)
 	di.data = make(chan []byte)
@@ -19,6 +21,7 @@ func NewDummyInput(options string) (di *DummyInput) {
 
 func (i *DummyInput) Read(data []byte) (int, error) {
 	buf := <-i.data
+
 	copy(data, buf)
 
 	return len(buf), nil
@@ -30,7 +33,12 @@ func (i *DummyInput) emit() {
 	for {
 		select {
 		case <-ticker.C:
-			i.data <- []byte("GET / HTTP/1.1\r\n\r\n")
+			uuid := uuid()
+			reqh := payloadHeader(RequestPayload, uuid, time.Now().UnixNano(), -1)
+			i.data <- append(reqh, []byte("GET / HTTP/1.1\r\nHost: www.w3.org\r\nUser-Agent: Go 1.1 package http\r\nAccept-Encoding: gzip\r\n\r\n")...)
+
+			resh := payloadHeader(ResponsePayload, uuid, time.Now().UnixNano()+1, 1)
+			i.data <- append(resh, []byte("HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n")...)
 		}
 	}
 }

input_file.go

@@ -1,37 +1,159 @@
 package main
 
 import (
-	"encoding/gob"
+	"bufio"
+	"bytes"
+	"compress/gzip"
+	"errors"
+	"io"
 	"log"
 	"os"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"sync"
 	"time"
 )
 
-type FileInput struct {
-	data    chan []byte
-	path    string
-	decoder *gob.Decoder
+type fileInputReader struct {
+	reader    *bufio.Reader
+	data      []byte
+	file      *os.File
+	timestamp int64
 }
 
-func NewFileInput(path string) (i *FileInput) {
+func (f *fileInputReader) parseNext() error {
+	payloadSeparatorAsBytes := []byte(payloadSeparator)
+	var buffer bytes.Buffer
+
+	for {
+		line, err := f.reader.ReadBytes('\n')
+
+		if err != nil {
+			if err != io.EOF {
+				log.Println(err)
+				return err
+			}
+
+			if err == io.EOF {
+				f.file.Close()
+				f.file = nil
+				return err
+			}
+		}
+
+		if bytes.Equal(payloadSeparatorAsBytes[1:], line) {
+			asBytes := buffer.Bytes()
+			meta := payloadMeta(asBytes)
+
+			f.timestamp, _ = strconv.ParseInt(string(meta[2]), 10, 64)
+			f.data = asBytes[:len(asBytes)-1]
+
+			return nil
+		}
+
+		buffer.Write(line)
+	}
+
+	return nil
+}
+
+func (f *fileInputReader) ReadPayload() []byte {
+	defer f.parseNext()
+
+	return f.data
+}
+func (f *fileInputReader) Close() error {
+	if f.file != nil {
+		f.file.Close()
+	}
+
+	return nil
+}
+
+func NewFileInputReader(path string) *fileInputReader {
+	file, err := os.Open(path)
+
+	if err != nil {
+		log.Println(err)
+		return nil
+	}
+
+	r := &fileInputReader{file: file}
+	if strings.HasSuffix(path, ".gz") {
+		gzReader, err := gzip.NewReader(file)
+		if err != nil {
+			log.Println(err)
+			return nil
+		}
+		r.reader = bufio.NewReader(gzReader)
+	} else {
+		r.reader = bufio.NewReader(file)
+	}
+
+	r.parseNext()
+
+	return r
+}
+
+// FileInput can read requests generated by FileOutput
+type FileInput struct {
+	mu          sync.Mutex
+	data        chan []byte
+	exit        chan bool
+	path        string
+	readers     []*fileInputReader
+	speedFactor float64
+	loop        bool
+}
+
+// NewFileInput constructor for FileInput. Accepts file path as argument.
+func NewFileInput(path string, loop bool) (i *FileInput) {
 	i = new(FileInput)
-	i.data = make(chan []byte)
+	i.data = make(chan []byte, 1000)
+	i.exit = make(chan bool, 1)
 	i.path = path
-	i.Init(path)
+	i.speedFactor = 1
+	i.loop = loop
+
+	if err := i.init(); err != nil {
+		return
+	}
 
 	go i.emit()
 
 	return
 }
 
-func (i *FileInput) Init(path string) {
-	file, err := os.Open(path)
+type NextFileNotFound struct{}
 
-	if err != nil {
-		log.Fatal(i, "Cannot open file %q. Error: %s", path, err)
+func (_ *NextFileNotFound) Error() string {
+	return "There is no new files"
+}
+
+func (i *FileInput) init() (err error) {
+	defer i.mu.Unlock()
+	i.mu.Lock()
+
+	var matches []string
+
+	if matches, err = filepath.Glob(i.path); err != nil {
+		log.Println("Wrong file pattern", i.path, err)
+		return
 	}
 
-	i.decoder = gob.NewDecoder(file)
+	if len(matches) == 0 {
+		log.Println("No files match pattern: ", i.path)
+		return errors.New("No matching files")
+	}
+
+	i.readers = make([]*fileInputReader, len(matches))
+
+	for idx, p := range matches {
+		i.readers[idx] = NewFileInputReader(p)
+	}
+
+	return nil
 }
 
 func (i *FileInput) Read(data []byte) (int, error) {
@@ -45,22 +167,72 @@ func (i *FileInput) String() string {
 	return "File input: " + i.path
 }
 
+// Find reader with smallest timestamp e.g next payload in row
+func (i *FileInput) nextReader() (next *fileInputReader) {
+	for _, r := range i.readers {
+		if r == nil || r.file == nil {
+			continue
+		}
+
+		if next == nil || r.timestamp < next.timestamp {
+			next = r
+			continue
+		}
+	}
+
+	return
+}
+
 func (i *FileInput) emit() {
-	var lastTime int64
+	var lastTime int64 = -1
 
 	for {
-		raw := new(RawRequest)
-		err := i.decoder.Decode(raw)
-
-		if err != nil {
+		select {
+		case <-i.exit:
 			return
+		default:
 		}
 
-		if lastTime != 0 {
-			time.Sleep(time.Duration(raw.Timestamp - lastTime))
-			lastTime = raw.Timestamp
+		reader := i.nextReader()
+
+		if reader == nil {
+			if i.loop {
+				i.init()
+				lastTime = -1
+				continue
+			} else {
+				break
+			}
 		}
 
-		i.data <- raw.Request
+		if lastTime != -1 {
+			diff := reader.timestamp - lastTime
+			lastTime = reader.timestamp
+
+			if i.speedFactor != 1 {
+				diff = int64(float64(diff) / i.speedFactor)
+			}
+
+			time.Sleep(time.Duration(diff))
+		} else {
+			lastTime = reader.timestamp
+		}
+
+		i.data <- reader.ReadPayload()
 	}
+
+	log.Printf("FileInput: end of file '%s'\n", i.path)
+}
+
+func (i *FileInput) Close() error {
+	defer i.mu.Unlock()
+	i.mu.Lock()
+
+	i.exit <- true
+
+	for _, r := range i.readers {
+		r.Close()
+	}
+
+	return nil
 }

input_file_test.go

@@ -0,0 +1,382 @@
+package main
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"log"
+	"math/rand"
+	"os"
+	"sync"
+	"syscall"
+	"testing"
+	"time"
+)
+
+var _ = log.Println
+
+func TestInputFileWithGET(t *testing.T) {
+
+	input := NewTestInput()
+	rg := NewRequestGenerator([]io.Reader{input}, func() { input.EmitGET() }, 1)
+	readPayloads := [][]byte{}
+
+	// Given a capture file with a GET request
+	expectedCaptureFile := CreateCaptureFile(rg)
+	defer expectedCaptureFile.TearDown()
+
+	// When the request is read from the capture file
+	err := ReadFromCaptureFile(expectedCaptureFile.file, 1, func(data []byte) {
+		readPayloads = append(readPayloads, Duplicate(data))
+	})
+
+	// The read request should match the original request
+	if err != nil {
+		t.Error(err)
+	} else {
+		if !expectedCaptureFile.PayloadsEqual(readPayloads) {
+			t.Error("Request read back from file should match")
+		}
+	}
+}
+
+func TestInputFileWithPayloadLargerThan64Kb(t *testing.T) {
+
+	input := NewTestInput()
+	rg := NewRequestGenerator([]io.Reader{input}, func() { input.EmitSizedPOST(64 * 1024) }, 1)
+	readPayloads := [][]byte{}
+
+	// Given a capture file with a request over 64Kb
+	expectedCaptureFile := CreateCaptureFile(rg)
+	defer expectedCaptureFile.TearDown()
+
+	// When the request is read from the capture file
+	err := ReadFromCaptureFile(expectedCaptureFile.file, 1, func(data []byte) {
+		readPayloads = append(readPayloads, Duplicate(data))
+	})
+
+	// The read request should match the original request
+	if err != nil {
+		t.Error(err)
+	} else {
+		if !expectedCaptureFile.PayloadsEqual(readPayloads) {
+			t.Error("Request read back from file should match")
+		}
+	}
+
+}
+
+func TestInputFileWithGETAndPOST(t *testing.T) {
+
+	input := NewTestInput()
+	rg := NewRequestGenerator([]io.Reader{input}, func() {
+		input.EmitGET()
+		input.EmitPOST()
+	}, 2)
+	readPayloads := [][]byte{}
+
+	// Given a capture file with a GET request
+	expectedCaptureFile := CreateCaptureFile(rg)
+	defer expectedCaptureFile.TearDown()
+
+	// When the requests are read from the capture file
+	err := ReadFromCaptureFile(expectedCaptureFile.file, 2, func(data []byte) {
+		readPayloads = append(readPayloads, Duplicate(data))
+	})
+
+	// The read requests should match the original request
+	if err != nil {
+		t.Error(err)
+	} else {
+		if !expectedCaptureFile.PayloadsEqual(readPayloads) {
+			t.Error("Request read back from file should match")
+		}
+	}
+
+}
+
+func TestInputFileMultipleFilesWithRequestsOnly(t *testing.T) {
+	rnd := rand.Int63()
+
+	file1, _ := os.OpenFile(fmt.Sprintf("/tmp/%d_0", rnd), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0660)
+	file1.Write([]byte("1 1 1\ntest1"))
+	file1.Write([]byte(payloadSeparator))
+	file1.Write([]byte("1 1 3\ntest2"))
+	file1.Write([]byte(payloadSeparator))
+	file1.Close()
+
+	file2, _ := os.OpenFile(fmt.Sprintf("/tmp/%d_1", rnd), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0660)
+	file2.Write([]byte("1 1 2\ntest3"))
+	file2.Write([]byte(payloadSeparator))
+	file2.Write([]byte("1 1 4\ntest4"))
+	file2.Write([]byte(payloadSeparator))
+	file2.Close()
+
+	input := NewFileInput(fmt.Sprintf("/tmp/%d*", rnd), false)
+	buf := make([]byte, 1000)
+
+	for i := '1'; i <= '4'; i++ {
+		n, _ := input.Read(buf)
+		if buf[4] != byte(i) {
+			t.Error("Should emit requests in right order", string(buf[:n]))
+		}
+	}
+
+	os.Remove(file1.Name())
+	os.Remove(file2.Name())
+}
+
+func TestInputFileRequestsWithLatency(t *testing.T) {
+	rnd := rand.Int63()
+
+	file, _ := os.OpenFile(fmt.Sprintf("/tmp/%d", rnd), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0660)
+	defer file.Close()
+
+	file.Write([]byte("1 1 100000000\nrequest1"))
+	file.Write([]byte(payloadSeparator))
+	file.Write([]byte("1 2 150000000\nrequest2"))
+	file.Write([]byte(payloadSeparator))
+	file.Write([]byte("1 3 250000000\nrequest3"))
+	file.Write([]byte(payloadSeparator))
+
+	input := NewFileInput(fmt.Sprintf("/tmp/%d", rnd), false)
+	buf := make([]byte, 1000)
+
+	start := time.Now().UnixNano()
+	for i := 0; i < 3; i++ {
+		input.Read(buf)
+	}
+	end := time.Now().UnixNano()
+
+	var expectedLatency int64 = 250000000 - 100000000
+	realLatency := end - start
+	if realLatency < expectedLatency {
+		t.Errorf("Should emit requests respecting latency. Expected: %v, real: %v", expectedLatency, realLatency)
+	}
+
+	if realLatency > expectedLatency+10000000 {
+		t.Errorf("Should emit requests respecting latency. Expected: %v, real: %v", expectedLatency, realLatency)
+
+	}
+}
+
+func TestInputFileMultipleFilesWithRequestsAndResponses(t *testing.T) {
+	rnd := rand.Int63()
+
+	file1, _ := os.OpenFile(fmt.Sprintf("/tmp/%d_0", rnd), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0660)
+	file1.Write([]byte("1 1 1\nrequest1"))
+	file1.Write([]byte(payloadSeparator))
+	file1.Write([]byte("2 1 1\nresponse1"))
+	file1.Write([]byte(payloadSeparator))
+	file1.Write([]byte("1 2 3\nrequest2"))
+	file1.Write([]byte(payloadSeparator))
+	file1.Write([]byte("2 2 3\nresponse2"))
+	file1.Write([]byte(payloadSeparator))
+	file1.Close()
+
+	file2, _ := os.OpenFile(fmt.Sprintf("/tmp/%d_1", rnd), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0660)
+	file2.Write([]byte("1 3 2\nrequest3"))
+	file2.Write([]byte(payloadSeparator))
+	file2.Write([]byte("2 3 2\nresponse3"))
+	file2.Write([]byte(payloadSeparator))
+	file2.Write([]byte("1 4 4\nrequest4"))
+	file2.Write([]byte(payloadSeparator))
+	file2.Write([]byte("2 4 4\nresponse4"))
+	file2.Write([]byte(payloadSeparator))
+	file2.Close()
+
+	input := NewFileInput(fmt.Sprintf("/tmp/%d*", rnd), false)
+	buf := make([]byte, 1000)
+
+	for i := '1'; i <= '4'; i++ {
+		n, _ := input.Read(buf)
+		if buf[0] != '1' && buf[4] != byte(i) {
+			t.Error("Shound emit requests in right order", string(buf[:n]))
+		}
+
+		n, _ = input.Read(buf)
+		if buf[0] != '2' && buf[4] != byte(i) {
+			t.Error("Shound emit responses in right order", string(buf[:n]))
+		}
+	}
+
+	os.Remove(file1.Name())
+	os.Remove(file2.Name())
+}
+
+func TestInputFileLoop(t *testing.T) {
+	rnd := rand.Int63()
+
+	file, _ := os.OpenFile(fmt.Sprintf("/tmp/%d", rnd), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0660)
+	file.Write([]byte("1 1 1\ntest1"))
+	file.Write([]byte(payloadSeparator))
+	file.Write([]byte("1 1 2\ntest2"))
+	file.Write([]byte(payloadSeparator))
+	file.Close()
+
+	input := NewFileInput(fmt.Sprintf("/tmp/%d", rnd), true)
+	buf := make([]byte, 1000)
+
+	// Even if we have just 2 requests in file, it should indifinitly loop
+	for i := 0; i < 1000; i++ {
+		input.Read(buf)
+	}
+
+	input.Close()
+	os.Remove(file.Name())
+}
+
+func TestInputFileCompressed(t *testing.T) {
+	rnd := rand.Int63()
+
+	output := NewFileOutput(fmt.Sprintf("/tmp/%d_0.gz", rnd), &FileOutputConfig{flushInterval: time.Minute, append: true})
+	for i := 0; i < 1000; i++ {
+		output.Write([]byte("1 1 1\r\ntest"))
+	}
+	name1 := output.file.Name()
+	output.Close()
+
+	output2 := NewFileOutput(fmt.Sprintf("/tmp/%d_1.gz", rnd), &FileOutputConfig{flushInterval: time.Minute, append: true})
+	for i := 0; i < 1000; i++ {
+		output2.Write([]byte("1 1 1\r\ntest"))
+	}
+	name2 := output2.file.Name()
+	output2.Close()
+
+	input := NewFileInput(fmt.Sprintf("/tmp/%d*", rnd), false)
+	buf := make([]byte, 1000)
+	for i := 0; i < 2000; i++ {
+		input.Read(buf)
+	}
+
+	os.Remove(name1)
+	os.Remove(name2)
+}
+
+type CaptureFile struct {
+	data [][]byte
+	file *os.File
+}
+
+func NewExpectedCaptureFile(data [][]byte, file *os.File) *CaptureFile {
+	ecf := new(CaptureFile)
+	ecf.file = file
+	ecf.data = data
+	return ecf
+}
+
+func (expectedCaptureFile *CaptureFile) TearDown() {
+	if expectedCaptureFile.file != nil {
+		syscall.Unlink(expectedCaptureFile.file.Name())
+	}
+}
+
+type RequestGenerator struct {
+	inputs []io.Reader
+	emit   func()
+	wg     *sync.WaitGroup
+}
+
+func NewRequestGenerator(inputs []io.Reader, emit func(), count int) (rg *RequestGenerator) {
+	rg = new(RequestGenerator)
+	rg.inputs = inputs
+	rg.emit = emit
+	rg.wg = new(sync.WaitGroup)
+	rg.wg.Add(count)
+	return
+}
+
+func (expectedCaptureFile *CaptureFile) PayloadsEqual(other [][]byte) bool {
+
+	if len(expectedCaptureFile.data) != len(other) {
+		return false
+	}
+
+	for i, payload := range other {
+		if !bytes.Equal(expectedCaptureFile.data[i], payload) {
+			return false
+		}
+	}
+
+	return true
+
+}
+
+func CreateCaptureFile(requestGenerator *RequestGenerator) *CaptureFile {
+
+	f, err := ioutil.TempFile("", "testmainconf")
+	if err != nil {
+		panic(err)
+	}
+
+	quit := make(chan int)
+
+	readPayloads := [][]byte{}
+	output := NewTestOutput(func(data []byte) {
+		readPayloads = append(readPayloads, Duplicate(data))
+
+		requestGenerator.wg.Done()
+	})
+
+	outputFile := NewFileOutput(f.Name(), &FileOutputConfig{flushInterval: time.Minute, append: true})
+
+	Plugins.Inputs = requestGenerator.inputs
+	Plugins.Outputs = []io.Writer{output, outputFile}
+
+	go Start(quit)
+
+	requestGenerator.emit()
+	requestGenerator.wg.Wait()
+
+	time.Sleep(100 * time.Millisecond)
+	outputFile.Close()
+
+	close(quit)
+
+	return NewExpectedCaptureFile(readPayloads, f)
+
+}
+
+func ReadFromCaptureFile(captureFile *os.File, count int, callback writeCallback) (err error) {
+
+	quit := make(chan int)
+	wg := new(sync.WaitGroup)
+
+	input := NewFileInput(captureFile.Name(), false)
+	output := NewTestOutput(func(data []byte) {
+		callback(data)
+		wg.Done()
+	})
+
+	Plugins.Inputs = []io.Reader{input}
+	Plugins.Outputs = []io.Writer{output}
+
+	wg.Add(count)
+	go Start(quit)
+
+	done := make(chan int, 1)
+	go func() {
+		wg.Wait()
+		done <- 1
+	}()
+
+	select {
+	case <-done:
+		break
+	case <-time.After(2 * time.Second):
+		err = errors.New("Timed out")
+	}
+	close(quit)
+
+	return
+
+}
+
+func Duplicate(data []byte) (duplicate []byte) {
+	duplicate = make([]byte, len(data))
+	copy(duplicate, data)
+
+	return
+}

input_http.go

@@ -0,0 +1,76 @@
+package main
+
+import (
+	"log"
+	"net"
+	"net/http"
+	"net/http/httputil"
+	"time"
+)
+
+// HTTPInput used for sending requests to Gor via http
+type HTTPInput struct {
+	data     chan []byte
+	address  string
+	listener net.Listener
+}
+
+// NewHTTPInput constructor for HTTPInput. Accepts address with port which he will listen on.
+func NewHTTPInput(address string) (i *HTTPInput) {
+	i = new(HTTPInput)
+	i.data = make(chan []byte, 10000)
+	i.address = address
+
+	i.listen(address)
+
+	return
+}
+
+func (i *HTTPInput) Read(data []byte) (int, error) {
+	buf := <-i.data
+
+	header := payloadHeader(RequestPayload, uuid(), time.Now().UnixNano(), -1)
+
+	copy(data[0:len(header)], header)
+	copy(data[len(header):], buf)
+
+	return len(buf) + len(header), nil
+}
+
+func (i *HTTPInput) handler(w http.ResponseWriter, r *http.Request) {
+	r.URL.Scheme = "http"
+	r.URL.Host = i.listener.Addr().String()
+
+	buf, _ := httputil.DumpRequestOut(r, true)
+	http.Error(w, http.StatusText(200), 200)
+
+	select {
+	case i.data <- buf:
+	default:
+		Debug("[INPUT-HTTP] Dropping requests because output can't process them fast enough")
+	}
+}
+
+func (i *HTTPInput) listen(address string) {
+	var err error
+
+	mux := http.NewServeMux()
+
+	mux.HandleFunc("/", i.handler)
+
+	i.listener, err = net.Listen("tcp", address)
+	if err != nil {
+		log.Fatal("HTTP input listener failure:", err)
+	}
+
+	go func() {
+		err = http.Serve(i.listener, mux)
+		if err != nil {
+			log.Fatal("HTTP input serve failure:", err)
+		}
+	}()
+}
+
+func (i *HTTPInput) String() string {
+	return "HTTP input: " + i.address
+}

input_http_test.go

@@ -0,0 +1,72 @@
+package main
+
+import (
+	"github.com/buger/gor/proto"
+	"io"
+	"log"
+	"net/http"
+	"os/exec"
+	"strings"
+	"sync"
+	"testing"
+)
+
+func TestHTTPInput(t *testing.T) {
+	wg := new(sync.WaitGroup)
+	quit := make(chan int)
+
+	input := NewHTTPInput("127.0.0.1:0")
+	output := NewTestOutput(func(data []byte) {
+		wg.Done()
+	})
+
+	Plugins.Inputs = []io.Reader{input}
+	Plugins.Outputs = []io.Writer{output}
+
+	go Start(quit)
+
+	address := strings.Replace(input.listener.Addr().String(), "[::]", "127.0.0.1", -1)
+
+	for i := 0; i < 100; i++ {
+		wg.Add(1)
+		http.Get("http://" + address + "/")
+	}
+
+	wg.Wait()
+
+	close(quit)
+}
+
+func TestInputHTTPLargePayload(t *testing.T) {
+	wg := new(sync.WaitGroup)
+	quit := make(chan int)
+
+	dd := exec.Command("dd", "if=/dev/urandom", "of=/tmp/large", "bs=1", "count=4000000")
+	err := dd.Run()
+	if err != nil {
+		log.Fatal("dd error:", err)
+	}
+
+	input := NewHTTPInput("127.0.0.1:0")
+	output := NewTestOutput(func(data []byte) {
+		if len(proto.Body(payloadBody(data))) != 4000000 {
+			t.Error("Should receive full file")
+		}
+		wg.Done()
+	})
+	Plugins.Inputs = []io.Reader{input}
+	Plugins.Outputs = []io.Writer{output}
+
+	go Start(quit)
+
+	wg.Add(1)
+	address := strings.Replace(input.listener.Addr().String(), "[::]", "127.0.0.1", -1)
+	curl := exec.Command("curl", "http://"+address, "--data-binary", "@/tmp/large")
+	err = curl.Run()
+	if err != nil {
+		log.Fatal("curl error:", err)
+	}
+
+	wg.Wait()
+	close(quit)
+}

input_raw.go

@@ -1,53 +1,111 @@
 package main
 
 import (
+	"github.com/buger/gor/proto"
 	raw "github.com/buger/gor/raw_socket_listener"
 	"log"
 	"net"
-	"strings"
+	"time"
 )
 
+// RAWInput used for intercepting traffic for given address
 type RAWInput struct {
-	data    chan []byte
-	address string
+	data          chan *raw.TCPMessage
+	address       string
+	expire        time.Duration
+	quit          chan bool
+	engine        int
+	realIPHeader  []byte
+	trackResponse bool
+	listener      *raw.Listener
 }
 
-func NewRAWInput(address string) (i *RAWInput) {
-	i = new(RAWInput)
-	i.data = make(chan []byte)
-	i.address = address
+// Available engines for intercepting traffic
+const (
+	EngineRawSocket = 1 << iota
+	EnginePcap
+	EnginePcapFile
+)
 
-	go i.listen(address)
+// NewRAWInput constructor for RAWInput. Accepts address with port as argument.
+func NewRAWInput(address string, engine int, trackResponse bool, expire time.Duration, realIPHeader string) (i *RAWInput) {
+	i = new(RAWInput)
+	i.data = make(chan *raw.TCPMessage)
+	i.address = address
+	i.expire = expire
+	i.engine = engine
+	i.realIPHeader = []byte(realIPHeader)
+	i.quit = make(chan bool)
+	i.trackResponse = trackResponse
+
+	i.listen(address)
+	i.listener.IsReady()
 
 	return
 }
 
 func (i *RAWInput) Read(data []byte) (int, error) {
-	buf := <-i.data
-	copy(data, buf)
+	msg := <-i.data
+	buf := msg.Bytes()
 
-	return len(buf), nil
+	var header []byte
+
+	if msg.IsIncoming {
+		header = payloadHeader(RequestPayload, msg.UUID(), msg.Start.UnixNano(), -1)
+		if len(i.realIPHeader) > 0 {
+			buf = proto.SetHeader(buf, i.realIPHeader, []byte(msg.IP().String()))
+		}
+	} else {
+		header = payloadHeader(ResponsePayload, msg.UUID(), msg.AssocMessage.Start.UnixNano(), msg.End.UnixNano()-msg.AssocMessage.Start.UnixNano())
+	}
+
+	copy(data[0:len(header)], header)
+	copy(data[len(header):], buf)
+
+	return len(buf) + len(header), nil
 }
 
 func (i *RAWInput) listen(address string) {
-	address = strings.Replace(address, "[::]", "127.0.0.1", -1)
+	Debug("Listening for traffic on: " + address)
 
 	host, port, err := net.SplitHostPort(address)
 
+	if i.engine == EnginePcapFile {
+		host = address
+		port = "1"
+		err = nil
+	}
+
 	if err != nil {
 		log.Fatal("input-raw: error while parsing address", err)
 	}
 
-	listener := raw.NewListener(host, port)
+	i.listener = raw.NewListener(host, port, i.engine, i.trackResponse, i.expire)
 
-	for {
-		// Receiving TCPMessage object
-		m := listener.Receive()
+	ch := i.listener.Receiver()
 
-		i.data <- m.Bytes()
-	}
+	go func() {
+		for {
+			select {
+			case <-i.quit:
+				return
+			default:
+			}
+
+			// Receiving TCPMessage object
+			m := <-ch
+
+			i.data <- m
+		}
+	}()
 }
 
 func (i *RAWInput) String() string {
-	return "RAW Socket input: " + i.address
+	return "Intercepting traffic from: " + i.address
+}
+
+func (i *RAWInput) Close() error {
+	i.listener.Close()
+	close(i.quit)
+	return nil
 }

input_raw_test.go

@@ -1,21 +1,113 @@
 package main
 
 import (
+	"bytes"
+	"github.com/buger/gor/proto"
 	"io"
+	"io/ioutil"
+	"log"
+	"math/rand"
+	"net"
 	"net/http"
+	"net/http/httptest"
+	"net/http/httputil"
+	"os"
+	"os/exec"
+	"strconv"
 	"strings"
 	"sync"
+	"sync/atomic"
 	"testing"
+	"time"
 )
 
-func TestRAWInput(t *testing.T) {
+const testRawExpire = time.Millisecond * 200
 
+func TestRAWInputIPv4(t *testing.T) {
 	wg := new(sync.WaitGroup)
 	quit := make(chan int)
 
-	listener := startHTTP(func(req *http.Request) {})
+	listener, err := net.Listen("tcp", ":0")
+	if err != nil {
+		t.Fatal(err)
+	}
+	origin := &http.Server{
+		Handler:      http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {}),
+		ReadTimeout:  10 * time.Second,
+		WriteTimeout: 10 * time.Second,
+	}
+	go origin.Serve(listener)
+	defer listener.Close()
+
+	originAddr := listener.Addr().String()
+
+	var respCounter, reqCounter int64
+
+	input := NewRAWInput(originAddr, EnginePcap, true, testRawExpire, "X-Real-IP")
+	defer input.Close()
+
+	output := NewTestOutput(func(data []byte) {
+		if data[0] == '1' {
+			body := payloadBody(data)
+			if len(proto.Header(body, []byte("X-Real-IP"))) == 0 {
+				t.Error("Should have X-Real-IP header", string(body))
+			}
+			atomic.AddInt64(&reqCounter, 1)
+		} else {
+			atomic.AddInt64(&respCounter, 1)
+		}
+
+		if Settings.debug {
+			log.Println(reqCounter, respCounter)
+		}
+
+		wg.Done()
+	})
+
+	Plugins.Inputs = []io.Reader{input}
+	Plugins.Outputs = []io.Writer{output}
+
+	client := NewHTTPClient("http://"+listener.Addr().String(), &HTTPClientConfig{})
+
+	go Start(quit)
+
+	for i := 0; i < 100; i++ {
+		// request + response
+		wg.Add(2)
+		client.Get("/")
+		time.Sleep(2 * time.Millisecond)
+	}
+
+	wg.Wait()
+
+	close(quit)
+}
+
+func TestRAWInputNoKeepAlive(t *testing.T) {
+	wg := new(sync.WaitGroup)
+	quit := make(chan int)
+
+	listener, err := net.Listen("tcp", ":0")
+	if err != nil {
+		t.Fatal(err)
+	}
+	origin := &http.Server{
+		Handler:      http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+				w.Write([]byte("a"))
+				w.Write([]byte("b"))
+			}),
+		ReadTimeout:  10 * time.Second,
+		WriteTimeout: 10 * time.Second,
+	}
+	origin.SetKeepAlivesEnabled(false)
+	go origin.Serve(listener)
+	defer listener.Close()
+
+	originAddr := listener.Addr().String()
+
+	input := NewRAWInput(originAddr, EnginePcap, true, testRawExpire, "")
+	defer input.Close()
 
-	input := NewRAWInput(listener.Addr().String())
 	output := NewTestOutput(func(data []byte) {
 		wg.Done()
 	})
@@ -23,17 +115,309 @@ func TestRAWInput(t *testing.T) {
 	Plugins.Inputs = []io.Reader{input}
 	Plugins.Outputs = []io.Writer{output}
 
-	address := strings.Replace(listener.Addr().String(), "[::]", "127.0.0.1", -1)
+	client := NewHTTPClient("http://"+listener.Addr().String(), &HTTPClientConfig{})
 
 	go Start(quit)
 
 	for i := 0; i < 100; i++ {
-		wg.Add(1)
-		res, _ := http.Get("http://" + address)
-		res.Body.Close()
+		// request + response
+		wg.Add(2)
+		client.Get("/")
+		time.Sleep(2 * time.Millisecond)
 	}
 
 	wg.Wait()
 
 	close(quit)
 }
+
+func TestRAWInputIPv6(t *testing.T) {
+	wg := new(sync.WaitGroup)
+	quit := make(chan int)
+
+	listener, err := net.Listen("tcp", "[::1]:0")
+	if err != nil {
+		t.Fatal(err)
+	}
+	origin := &http.Server{
+		Handler:      http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {}),
+		ReadTimeout:  10 * time.Second,
+		WriteTimeout: 10 * time.Second,
+	}
+	go origin.Serve(listener)
+	defer listener.Close()
+
+	originAddr := listener.Addr().String()
+
+	var respCounter, reqCounter int64
+
+	input := NewRAWInput(originAddr, EnginePcap, true, testRawExpire, "")
+	defer input.Close()
+
+	output := NewTestOutput(func(data []byte) {
+		if data[0] == '1' {
+			atomic.AddInt64(&reqCounter, 1)
+		} else {
+			atomic.AddInt64(&respCounter, 1)
+		}
+
+		if Settings.debug {
+			log.Println(reqCounter, respCounter)
+		}
+
+		wg.Done()
+	})
+
+	Plugins.Inputs = []io.Reader{input}
+	Plugins.Outputs = []io.Writer{output}
+
+	client := NewHTTPClient("http://"+listener.Addr().String(), &HTTPClientConfig{})
+
+	go Start(quit)
+
+	for i := 0; i < 100; i++ {
+		// request + response
+		wg.Add(2)
+		client.Get("/")
+		time.Sleep(2 * time.Millisecond)
+	}
+
+	wg.Wait()
+	close(quit)
+}
+
+func TestInputRAW100Expect(t *testing.T) {
+	wg := new(sync.WaitGroup)
+	quit := make(chan int)
+
+	fileContent, _ := ioutil.ReadFile("COMM-LICENSE")
+
+	// Origing and Replay server initialization
+	origin := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		defer r.Body.Close()
+		ioutil.ReadAll(r.Body)
+		wg.Done()
+	}))
+	defer origin.Close()
+
+	originAddr := strings.Replace(origin.Listener.Addr().String(), "[::]", "127.0.0.1", -1)
+
+	input := NewRAWInput(originAddr, EnginePcap, true, time.Second, "")
+	defer input.Close()
+
+	// We will use it to get content of raw HTTP request
+	testOutput := NewTestOutput(func(data []byte) {
+		switch data[0] {
+		case RequestPayload:
+			if strings.Contains(string(data), "Expect: 100-continue") {
+				t.Error("Should not contain 100-continue header")
+			}
+			wg.Done()
+		case ResponsePayload:
+			wg.Done()
+		}
+	})
+
+	replay := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		defer r.Body.Close()
+		body, _ := ioutil.ReadAll(r.Body)
+
+		if !bytes.Equal(body, fileContent) {
+			buf, _ := httputil.DumpRequest(r, true)
+			t.Error("Wrong POST body:", string(buf))
+		}
+
+		wg.Done()
+	}))
+	defer replay.Close()
+
+	httpOutput := NewHTTPOutput(replay.URL, &HTTPOutputConfig{})
+
+	Plugins.Inputs = []io.Reader{input}
+	Plugins.Outputs = []io.Writer{testOutput, httpOutput}
+
+	go Start(quit)
+
+	// Origin + Response/Request Test Output + Request Http Output
+	wg.Add(4)
+	curl := exec.Command("curl", "http://"+originAddr, "--data-binary", "@COMM-LICENSE")
+	err := curl.Run()
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	wg.Wait()
+	close(quit)
+}
+
+func TestInputRAWChunkedEncoding(t *testing.T) {
+	wg := new(sync.WaitGroup)
+	quit := make(chan int)
+
+	fileContent, _ := ioutil.ReadFile("README.md")
+
+	// Origing and Replay server initialization
+	origin := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		defer r.Body.Close()
+		ioutil.ReadAll(r.Body)
+
+		wg.Done()
+	}))
+
+	originAddr := strings.Replace(origin.Listener.Addr().String(), "[::]", "127.0.0.1", -1)
+	input := NewRAWInput(originAddr, EnginePcap, true, time.Second, "")
+	defer input.Close()
+
+	replay := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		defer r.Body.Close()
+		body, _ := ioutil.ReadAll(r.Body)
+
+		if !bytes.Equal(body, fileContent) {
+			buf, _ := httputil.DumpRequest(r, true)
+			t.Error("Wrong POST body:", string(buf))
+		}
+
+		wg.Done()
+	}))
+	defer replay.Close()
+
+	httpOutput := NewHTTPOutput(replay.URL, &HTTPOutputConfig{Debug: true})
+
+	Plugins.Inputs = []io.Reader{input}
+	Plugins.Outputs = []io.Writer{httpOutput}
+
+	go Start(quit)
+
+	wg.Add(2)
+
+	curl := exec.Command("curl", "http://"+originAddr, "--header", "Transfer-Encoding: chunked", "--header", "Expect:", "--data-binary", "@README.md")
+	err := curl.Run()
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	wg.Wait()
+
+	close(quit)
+}
+
+func TestInputRAWLargePayload(t *testing.T) {
+	// FIXME: Large payloads does not work for travis for some reason...
+	if os.Getenv("TRAVIS_BUILD_DIR") != "" {
+		return
+	}
+	wg := new(sync.WaitGroup)
+	quit := make(chan int)
+	sizeB := 100 * 1000
+
+	// Generate 100kb file
+	dd := exec.Command("dd", "if=/dev/urandom", "of=/tmp/large", "bs=1", "count="+strconv.Itoa(sizeB))
+	err := dd.Run()
+	if err != nil {
+		log.Fatal("dd error:", err)
+	}
+
+	origin := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
+		defer req.Body.Close()
+		body, _ := ioutil.ReadAll(req.Body)
+
+		if len(body) != sizeB {
+			t.Error("File size should be 1mb:", len(body))
+		}
+
+		wg.Done()
+	}))
+	originAddr := strings.Replace(origin.Listener.Addr().String(), "[::]", "127.0.0.1", -1)
+
+	input := NewRAWInput(originAddr, EnginePcap, true, testRawExpire, "")
+	defer input.Close()
+
+	replay := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
+		body, _ := ioutil.ReadAll(req.Body)
+		// // req.Body = http.MaxBytesReader(w, req.Body, 1*1024*1024)
+		// // buf := make([]byte, 1*1024*1024)
+		// n, _ := req.Body.Read(buf)
+		// body := buf[0:n]
+
+		if len(body) != sizeB {
+			t.Errorf("File size should be %d bytes: %d", sizeB, len(body))
+		}
+
+		wg.Done()
+	}))
+	defer replay.Close()
+
+	httpOutput := NewHTTPOutput(replay.URL, &HTTPOutputConfig{Debug: false})
+
+	Plugins.Inputs = []io.Reader{input}
+	Plugins.Outputs = []io.Writer{httpOutput}
+
+	go Start(quit)
+
+	wg.Add(2)
+	curl := exec.Command("curl", "http://"+originAddr, "--header", "Transfer-Encoding: chunked", "--header", "Expect:", "--data-binary", "@/tmp/large")
+	err = curl.Run()
+	if err != nil {
+		log.Fatal("curl error:", err)
+	}
+
+	wg.Wait()
+	close(quit)
+}
+
+func BenchmarkRAWInput(b *testing.B) {
+	quit := make(chan int)
+
+	origin := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {}))
+	defer origin.Close()
+	originAddr := strings.Replace(origin.Listener.Addr().String(), "[::]", "127.0.0.1", -1)
+
+	var respCounter, reqCounter int64
+
+	input := NewRAWInput(originAddr, EnginePcap, true, testRawExpire, "")
+	defer input.Close()
+
+	output := NewTestOutput(func(data []byte) {
+		if data[0] == '1' {
+			atomic.AddInt64(&reqCounter, 1)
+		} else {
+			atomic.AddInt64(&respCounter, 1)
+		}
+
+		// log.Println("Captured ", reqCounter, "requests and ", respCounter, " responses")
+	})
+
+	Plugins.Inputs = []io.Reader{input}
+	Plugins.Outputs = []io.Writer{output}
+
+	go Start(quit)
+
+	emitted := 0
+	fileContent, _ := ioutil.ReadFile("LICENSE.txt")
+
+	for i := 0; i < b.N; i++ {
+		wg := new(sync.WaitGroup)
+		wg.Add(10 * 100)
+		emitted += 10 * 100
+		for w := 0; w < 100; w++ {
+			go func() {
+				client := NewHTTPClient(origin.URL, &HTTPClientConfig{})
+				for i := 0; i < 10; i++ {
+					if rand.Int63n(2) == 0 {
+						client.Post("/", fileContent)
+					} else {
+						client.Get("/")
+					}
+					time.Sleep(time.Duration(rand.Int63n(50)) * time.Millisecond)
+					wg.Done()
+				}
+			}()
+		}
+		wg.Wait()
+	}
+
+	time.Sleep(400 * time.Millisecond)
+	log.Println("Emitted ", emitted, ", Captured ", reqCounter, "requests and ", respCounter, " responses")
+
+	close(quit)
+}

input_tcp.go

@@ -2,23 +2,25 @@ package main
 
 import (
 	"bufio"
+	"bytes"
+	"fmt"
 	"io"
 	"log"
 	"net"
+	"os"
 )
 
-// Can be tested using nc tool:
-//    echo "asdad" | nc 127.0.0.1 27017
-//
+// TCPInput used for internal communication
 type TCPInput struct {
 	data     chan []byte
 	address  string
 	listener net.Listener
 }
 
+// NewTCPInput constructor for TCPInput, accepts address with port
 func NewTCPInput(address string) (i *TCPInput) {
 	i = new(TCPInput)
-	i.data = make(chan []byte)
+	i.data = make(chan []byte, 1000)
 	i.address = address
 
 	i.listen(address)
@@ -58,23 +60,31 @@ func (i *TCPInput) listen(address string) {
 func (i *TCPInput) handleConnection(conn net.Conn) {
 	defer conn.Close()
 
+	payloadSeparatorAsBytes := []byte(payloadSeparator)
 	reader := bufio.NewReader(conn)
+	var buffer bytes.Buffer
 
 	for {
-		buf,err := reader.ReadBytes('¶')
-		buf_len := len(buf)
-		if buf_len > 0 {
-			new_buf_len := len(buf) - 2
-			if new_buf_len > 0 {
-				new_buf := make([]byte, new_buf_len)
-				copy(new_buf, buf[:new_buf_len])
-				i.data <- new_buf
-				if err != nil {
-					if err != io.EOF {
-						log.Printf("error: %s\n", err)
-					}
-				}
+		line, err := reader.ReadBytes('\n')
+
+		if err != nil {
+			if err != io.EOF {
+				fmt.Fprintln(os.Stderr, "Unexpected error in input tcp connection:", err)
 			}
+			break
+
+		}
+
+		if bytes.Equal(payloadSeparatorAsBytes[1:], line) {
+			asBytes := buffer.Bytes()
+			buffer.Reset()
+
+			newBuf := make([]byte, len(asBytes)-1)
+			copy(newBuf, asBytes)
+
+			i.data <- newBuf
+		} else {
+			buffer.Write(line)
 		}
 	}
 }

input_tcp_test.go

@@ -12,7 +12,7 @@ func TestTCPInput(t *testing.T) {
 	wg := new(sync.WaitGroup)
 	quit := make(chan int)
 
-	input := NewTCPInput(":0")
+	input := NewTCPInput("127.0.0.1:0")
 	output := NewTestOutput(func(data []byte) {
 		wg.Done()
 	})
@@ -34,79 +34,15 @@ func TestTCPInput(t *testing.T) {
 		log.Fatal(err)
 	}
 
-	msg := []byte("GET / HTTP/1.1\r\n\r\n")
+	msg := []byte("1 1 1\nGET / HTTP/1.1\r\n\r\n")
 
 	for i := 0; i < 100; i++ {
 		wg.Add(1)
-		new_buf := make([]byte, len(msg) + 2)
-		msg = append(msg,[]byte("¶")...)
-		copy(new_buf, msg)
-		conn.Write(new_buf)
+		conn.Write(msg)
+		conn.Write([]byte(payloadSeparator))
 	}
 
 	wg.Wait()
 
 	close(quit)
 }
-
-func BenchmarkTCPInput(b *testing.B) {
-	wg := new(sync.WaitGroup)
-	quit := make(chan int)
-
-	input := NewTCPInput(":0")
-	output := NewTestOutput(func(data []byte) {
-		wg.Done()
-	})
-
-	Plugins.Inputs = []io.Reader{input}
-	Plugins.Outputs = []io.Writer{output}
-
-	go Start(quit)
-
-	tcpAddr, err := net.ResolveTCPAddr("tcp", input.listener.Addr().String())
-
-	if err != nil {
-		log.Fatal(err)
-	}
-
-	var connections []net.Conn
-
-	// Creating simple pool of workers, same as output_tcp have
-	dataChan := make(chan []byte, 1000)
-
-	for i := 0; i < 10; i++ {
-		conn, _ := net.DialTCP("tcp", nil, tcpAddr)
-		connections = append(connections, conn)
-
-		go func(conn net.Conn) {
-			for {
-				data := <-dataChan
-
-				new_buf := make([]byte, len(data) + 2)
-				data = append(data,[]byte("¶")...)
-				copy(new_buf, data)
-				conn.Write(new_buf)
-			}
-		}(conn)
-	}
-
-	if err != nil {
-		log.Fatal(err)
-	}
-
-	msg := []byte("GET / HTTP/1.1\r\n\r\n")
-
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		wg.Add(1)
-		dataChan <- msg
-	}
-
-	wg.Wait()
-
-	for _, conn := range connections {
-		conn.Close()
-	}
-
-	close(quit)
-}

limiter.go

@@ -3,43 +3,94 @@ package main
 import (
 	"fmt"
 	"io"
+	"math/rand"
+	"strconv"
+	"strings"
 	"time"
 )
 
+// Limiter is a wrapper for input or output plugin which adds rate limiting
 type Limiter struct {
-	writer io.Writer
-	limit  int
+	plugin    interface{}
+	limit     int
+	isPercent bool
 
 	currentRPS  int
 	currentTime int64
 }
 
-func NewLimiter(writer io.Writer, limit int) (l *Limiter) {
-	l = new(Limiter)
-	l.limit = limit
-	l.writer = writer
-	l.currentTime = time.Now().UnixNano()
+func parseLimitOptions(options string) (limit int, isPercent bool) {
+	if strings.Contains(options, "%") {
+		limit, _ = strconv.Atoi(strings.Split(options, "%")[0])
+		isPercent = true
+	} else {
+		limit, _ = strconv.Atoi(options)
+		isPercent = false
+	}
 
 	return
 }
 
-func (l *Limiter) Write(data []byte) (n int, err error) {
+// NewLimiter constructor for Limiter, accepts plugin and options
+// `options` allow to sprcify relatve or absolute limiting
+func NewLimiter(plugin interface{}, options string) io.ReadWriter {
+	l := new(Limiter)
+	l.limit, l.isPercent = parseLimitOptions(options)
+	l.plugin = plugin
+	l.currentTime = time.Now().UnixNano()
+
+	// FileInput have its own rate limiting. Unlike other inputs we not just dropping requests, we can slow down or speed up request emittion.
+	if fi, ok := l.plugin.(*FileInput); ok && l.isPercent {
+		fi.speedFactor = float64(l.limit) / float64(100)
+	}
+
+	return l
+}
+
+func (l *Limiter) isLimited() bool {
+	// File input have its own limiting algorithm
+	if _, ok := l.plugin.(*FileInput); ok && l.isPercent {
+		return false
+	}
+
+	if l.isPercent {
+		return l.limit <= rand.Intn(100)
+	}
+
 	if (time.Now().UnixNano() - l.currentTime) > time.Second.Nanoseconds() {
 		l.currentTime = time.Now().UnixNano()
 		l.currentRPS = 0
 	}
 
 	if l.currentRPS >= l.limit {
+		return true
+	}
+
+	l.currentRPS++
+
+	return false
+}
+
+func (l *Limiter) Write(data []byte) (n int, err error) {
+	if l.isLimited() {
 		return 0, nil
 	}
 
-	n, err = l.writer.Write(data)
+	n, err = l.plugin.(io.Writer).Write(data)
 
-	l.currentRPS++
+	return
+}
+
+func (l *Limiter) Read(data []byte) (n int, err error) {
+	n, err = l.plugin.(io.Reader).Read(data)
+
+	if l.isLimited() {
+		return 0, nil
+	}
 
 	return
 }
 
 func (l *Limiter) String() string {
-	return fmt.Sprintf("Limiting %s to: %d", l.writer, l.limit)
+	return fmt.Sprintf("Limiting %s to: %d (isPercent: %b)", l.plugin, l.limit, l.isPercent)
 }

limiter_test.go

@@ -1,3 +1,5 @@
+// +build !race
+
 package main
 
 import (
@@ -6,14 +8,14 @@ import (
 	"testing"
 )
 
-func TestLimiter(t *testing.T) {
+func TestOutputLimiter(t *testing.T) {
 	wg := new(sync.WaitGroup)
 	quit := make(chan int)
 
 	input := NewTestInput()
 	output := NewLimiter(NewTestOutput(func(data []byte) {
 		wg.Done()
-	}), 10)
+	}), "10")
 	wg.Add(10)
 
 	Plugins.Inputs = []io.Reader{input}
@@ -29,3 +31,76 @@ func TestLimiter(t *testing.T) {
 
 	close(quit)
 }
+
+func TestInputLimiter(t *testing.T) {
+	wg := new(sync.WaitGroup)
+	quit := make(chan int)
+
+	input := NewLimiter(NewTestInput(), "10")
+	output := NewTestOutput(func(data []byte) {
+		wg.Done()
+	})
+	wg.Add(10)
+
+	Plugins.Inputs = []io.Reader{input}
+	Plugins.Outputs = []io.Writer{output}
+
+	go Start(quit)
+
+	for i := 0; i < 100; i++ {
+		input.(*Limiter).plugin.(*TestInput).EmitGET()
+	}
+
+	wg.Wait()
+
+	close(quit)
+}
+
+// Should limit all requests
+func TestPercentLimiter1(t *testing.T) {
+	wg := new(sync.WaitGroup)
+	quit := make(chan int)
+
+	input := NewTestInput()
+	output := NewLimiter(NewTestOutput(func(data []byte) {
+		wg.Done()
+	}), "0%")
+
+	Plugins.Inputs = []io.Reader{input}
+	Plugins.Outputs = []io.Writer{output}
+
+	go Start(quit)
+
+	for i := 0; i < 100; i++ {
+		input.EmitGET()
+	}
+
+	wg.Wait()
+
+	close(quit)
+}
+
+// Should not limit at all
+func TestPercentLimiter2(t *testing.T) {
+	wg := new(sync.WaitGroup)
+	quit := make(chan int)
+
+	input := NewTestInput()
+	output := NewLimiter(NewTestOutput(func(data []byte) {
+		wg.Done()
+	}), "100%")
+	wg.Add(100)
+
+	Plugins.Inputs = []io.Reader{input}
+	Plugins.Outputs = []io.Writer{output}
+
+	go Start(quit)
+
+	for i := 0; i < 100; i++ {
+		input.EmitGET()
+	}
+
+	wg.Wait()
+
+	close(quit)
+}

middleware.go

@@ -0,0 +1,125 @@
+package main
+
+import (
+	"bufio"
+	"encoding/hex"
+	"fmt"
+	"io"
+	"log"
+	"os"
+	"os/exec"
+	"strings"
+	"sync"
+)
+
+type Middleware struct {
+	command string
+
+	data chan []byte
+
+	mu sync.Mutex
+
+	Stdin  io.Writer
+	Stdout io.Reader
+}
+
+func NewMiddleware(command string) *Middleware {
+	m := new(Middleware)
+	m.command = command
+	m.data = make(chan []byte, 1000)
+
+	commands := strings.Split(command, " ")
+	cmd := exec.Command(commands[0], commands[1:]...)
+
+	m.Stdout, _ = cmd.StdoutPipe()
+	m.Stdin, _ = cmd.StdinPipe()
+
+	if Settings.verbose {
+		cmd.Stderr = os.Stderr
+	}
+
+	go m.read(m.Stdout)
+
+	go func() {
+		err := cmd.Start()
+
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		err = cmd.Wait()
+
+		if err != nil {
+			log.Fatal(err)
+		}
+	}()
+
+	return m
+}
+
+func (m *Middleware) ReadFrom(plugin io.Reader) {
+	Debug("[MIDDLEWARE-MASTER] Starting reading from", plugin)
+	go m.copy(m.Stdin, plugin)
+}
+
+func (m *Middleware) copy(to io.Writer, from io.Reader) {
+	buf := make([]byte, 5*1024*1024)
+	dst := make([]byte, len(buf)*2)
+
+	for {
+		nr, _ := from.Read(buf)
+		if nr > 0 && len(buf) > nr {
+
+			hex.Encode(dst, buf[0:nr])
+			dst[nr*2] = '\n'
+
+			m.mu.Lock()
+			to.Write(dst[0 : nr*2+1])
+			m.mu.Unlock()
+
+			if Settings.debug {
+				Debug("[MIDDLEWARE-MASTER] Sending:", string(buf[0:nr]), "From:", from)
+			}
+		}
+	}
+}
+
+func (m *Middleware) read(from io.Reader) {
+	reader := bufio.NewReader(from)
+	var line []byte
+	var e error
+
+	for {
+		if line, e = reader.ReadBytes('\n'); e != nil {
+			if e == io.EOF {
+				continue
+			} else {
+				break
+			}
+		}
+
+		buf := make([]byte, len(line)/2)
+		if _, err := hex.Decode(buf, line[:len(line)-1]); err != nil {
+			fmt.Fprintln(os.Stderr, "Failed to decode input payload", err, len(line))
+		}
+
+		if Settings.debug {
+			Debug("[MIDDLEWARE-MASTER] Received:", string(buf))
+		}
+
+		m.data <- buf
+	}
+
+	return
+}
+
+func (m *Middleware) Read(data []byte) (int, error) {
+	buf := <-m.data
+	copy(data, buf)
+
+	return len(buf), nil
+}
+
+func (m *Middleware) String() string {
+	return fmt.Sprintf("Modifying traffic using '%s' command", m.command)
+}

middleware_test.go

@@ -0,0 +1,220 @@
+package main
+
+import (
+	"bytes"
+	"crypto/rand"
+	"encoding/hex"
+	"github.com/buger/gor/proto"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+)
+
+type fakeServiceCb func(string, int, []byte)
+
+// Simple service that generate token on request, and require this token for accesing to secure area
+func NewFakeSecureService(wg *sync.WaitGroup, cb fakeServiceCb) *httptest.Server {
+	active_tokens := make([]string, 0)
+	var mu sync.Mutex
+
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
+		mu.Lock()
+		defer mu.Unlock()
+		Debug("Received request: " + req.URL.String())
+
+		switch req.URL.Path {
+		case "/token":
+			// Generate random token
+			token_length := 10
+			buf := make([]byte, token_length)
+			rand.Read(buf)
+			token := hex.EncodeToString(buf)
+			active_tokens = append(active_tokens, token)
+
+			w.Write([]byte(token))
+
+			cb(req.URL.Path, 200, []byte(token))
+		case "/secure":
+			token := req.URL.Query().Get("token")
+			token_found := false
+
+			for _, t := range active_tokens {
+				if t == token {
+					token_found = true
+					break
+				}
+			}
+
+			if token_found {
+				w.WriteHeader(http.StatusAccepted)
+				cb(req.URL.Path, 202, []byte(nil))
+			} else {
+				w.WriteHeader(http.StatusForbidden)
+				cb(req.URL.Path, 403, []byte(nil))
+			}
+		}
+
+		wg.Done()
+	}))
+
+	return server
+}
+
+func TestFakeSecureService(t *testing.T) {
+	var resp, token []byte
+
+	wg := new(sync.WaitGroup)
+
+	server := NewFakeSecureService(wg, func(path string, status int, resp []byte) {
+	})
+	defer server.Close()
+
+	wg.Add(3)
+
+	client := NewHTTPClient(server.URL, &HTTPClientConfig{Debug: true})
+	resp, _ = client.Get("/token")
+	token = proto.Body(resp)
+
+	// Right token
+	resp, _ = client.Get("/secure?token=" + string(token))
+	if !bytes.Equal(proto.Status(resp), []byte("202")) {
+		t.Error("Valid token should return status 202:", string(proto.Status(resp)))
+	}
+
+	// Wrong tokens forbidden
+	resp, _ = client.Get("/secure?token=wrong")
+	if !bytes.Equal(proto.Status(resp), []byte("403")) {
+		t.Error("Wrong token should returns status 403:", string(proto.Status(resp)))
+	}
+
+	wg.Wait()
+}
+
+func TestEchoMiddleware(t *testing.T) {
+	wg := new(sync.WaitGroup)
+
+	from := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Env", "prod")
+		w.Header().Set("RequestPath", r.URL.Path)
+		wg.Done()
+	}))
+	defer from.Close()
+
+	to := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Env", "test")
+		w.Header().Set("RequestPath", r.URL.Path)
+		wg.Done()
+	}))
+	defer to.Close()
+
+	quit := make(chan int)
+
+	Settings.middleware = "./examples/middleware/echo.sh"
+
+	// Catch traffic from one service
+	fromAddr := strings.Replace(from.Listener.Addr().String(), "[::]", "127.0.0.1", -1)
+	input := NewRAWInput(fromAddr, EnginePcap, true, testRawExpire, "")
+	defer input.Close()
+
+	// And redirect to another
+	output := NewHTTPOutput(to.URL, &HTTPOutputConfig{Debug: false})
+
+	Plugins.Inputs = []io.Reader{input}
+	Plugins.Outputs = []io.Writer{output}
+
+	// Start Gor
+	go Start(quit)
+
+	// Wait till middleware initialization
+	time.Sleep(100 * time.Millisecond)
+
+	// Should receive 2 requests from original + 2 from replayed
+	client := NewHTTPClient(from.URL, &HTTPClientConfig{Debug: false})
+
+	for i := 0; i < 10; i++ {
+		wg.Add(4)
+		// Request should be echoed
+		client.Get("/a")
+		time.Sleep(5 * time.Millisecond)
+		client.Get("/b")
+		time.Sleep(5 * time.Millisecond)
+	}
+
+	wg.Wait()
+	close(quit)
+	time.Sleep(200 * time.Millisecond)
+
+	Settings.middleware = ""
+}
+
+func TestTokenMiddleware(t *testing.T) {
+	var resp, token []byte
+
+	wg := new(sync.WaitGroup)
+
+	from := NewFakeSecureService(wg, func(path string, status int, tok []byte) {
+		time.Sleep(10 * time.Millisecond)
+	})
+	defer from.Close()
+
+	to := NewFakeSecureService(wg, func(path string, status int, tok []byte) {
+		switch path {
+		case "/secure":
+			if status != 202 {
+				t.Error("Server should receive valid rewritten token")
+			}
+		}
+
+		time.Sleep(10 * time.Millisecond)
+	})
+	defer to.Close()
+
+	quit := make(chan int)
+
+	Settings.middleware = "go run ./examples/middleware/token_modifier.go"
+
+	fromAddr := strings.Replace(from.Listener.Addr().String(), "[::]", "127.0.0.1", -1)
+	// Catch traffic from one service
+	input := NewRAWInput(fromAddr, EnginePcap, true, testRawExpire, "")
+	defer input.Close()
+
+	// And redirect to another
+	output := NewHTTPOutput(to.URL, &HTTPOutputConfig{Debug: true})
+
+	Plugins.Inputs = []io.Reader{input}
+	Plugins.Outputs = []io.Writer{output}
+
+	// Start Gor
+	go Start(quit)
+
+	// Wait for middleware to initialize
+	// Give go compiller time to build programm
+	time.Sleep(500 * time.Millisecond)
+
+	// Should receive 2 requests from original + 2 from replayed
+	wg.Add(4)
+
+	client := NewHTTPClient(from.URL, &HTTPClientConfig{Debug: true})
+
+	// Sending traffic to original service
+	resp, _ = client.Get("/token")
+	token = proto.Body(resp)
+
+	// When delay is too smal, middleware does not always rewrite requests in time
+	// Hopefuly client will have delay more then 100ms :)
+	time.Sleep(100 * time.Millisecond)
+
+	resp, _ = client.Get("/secure?token=" + string(token))
+	if !bytes.Equal(proto.Status(resp), []byte("202")) {
+		t.Error("Valid token should return 202:", proto.Status(resp))
+	}
+
+	wg.Wait()
+	close(quit)
+	time.Sleep(100 * time.Millisecond)
+	Settings.middleware = ""
+}

output_dummy.go

@@ -4,17 +4,19 @@ import (
 	"fmt"
 )
 
+// DummyOutput used for debugging, prints all incoming requests
 type DummyOutput struct {
 }
 
-func NewDummyOutput(options string) (di *DummyOutput) {
+// NewDummyOutput constructor for DummyOutput
+func NewDummyOutput() (di *DummyOutput) {
 	di = new(DummyOutput)
 
 	return
 }
 
 func (i *DummyOutput) Write(data []byte) (int, error) {
-	fmt.Println("Writing message: ", data)
+	fmt.Println(string(data))
 
 	return len(data), nil
 }

output_file.go

@@ -1,51 +1,241 @@
 package main
 
 import (
-	"encoding/gob"
+	"bufio"
+	"compress/gzip"
+	"fmt"
+	"io"
 	"log"
 	"os"
+	"path/filepath"
+	"runtime/debug"
+	"sort"
+	"strconv"
+	"strings"
+	"sync"
 	"time"
 )
 
-type RawRequest struct {
-	Timestamp int64
-	Request   []byte
+var dateFileNameFuncs = map[string]func() string{
+	"%Y":  func() string { return time.Now().Format("2006") },
+	"%m":  func() string { return time.Now().Format("01") },
+	"%d":  func() string { return time.Now().Format("02") },
+	"%H":  func() string { return time.Now().Format("15") },
+	"%M":  func() string { return time.Now().Format("04") },
+	"%S":  func() string { return time.Now().Format("05") },
+	"%NS": func() string { return fmt.Sprint(time.Now().Nanosecond()) },
 }
 
+type FileOutputConfig struct {
+	flushInterval time.Duration
+	sizeLimit     unitSizeVar
+	queueLimit    int
+	append        bool
+}
+
+// FileOutput output plugin
 type FileOutput struct {
-	path    string
-	encoder *gob.Encoder
-	file    *os.File
+	mu           sync.Mutex
+	pathTemplate string
+	currentName  string
+	file         *os.File
+	queueLength  int
+	chunkSize    int
+	writer       io.Writer
+
+	config *FileOutputConfig
 }
 
-func NewFileOutput(path string) (o *FileOutput) {
-	o = new(FileOutput)
-	o.path = path
-	o.Init(path)
+// NewFileOutput constructor for FileOutput, accepts path
+func NewFileOutput(pathTemplate string, config *FileOutputConfig) *FileOutput {
+	o := new(FileOutput)
+	o.pathTemplate = pathTemplate
+	o.config = config
+	o.updateName()
 
-	return
+	go func() {
+		for {
+			time.Sleep(time.Second)
+			o.updateName()
+		}
+	}()
+
+	return o
 }
 
-func (o *FileOutput) Init(path string) {
-	var err error
+func getFileIndex(name string) int {
+	ext := filepath.Ext(name)
+	withoutExt := strings.TrimSuffix(name, ext)
 
-	o.file, err = os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0660)
-
-	if err != nil {
-		log.Fatal(o, "Cannot open file %q. Error: %s", path, err)
+	if idx := strings.LastIndex(withoutExt, "_"); idx != -1 {
+		if i, err := strconv.Atoi(withoutExt[idx+1:]); err == nil {
+			return i
+		}
 	}
 
-	o.encoder = gob.NewEncoder(o.file)
+	return -1
+}
+
+func setFileIndex(name string, idx int) string {
+	idxS := strconv.Itoa(idx)
+	ext := filepath.Ext(name)
+	withoutExt := strings.TrimSuffix(name, ext)
+
+	if i := strings.LastIndex(withoutExt, "_"); i != -1 {
+		if _, err := strconv.Atoi(withoutExt[i+1:]); err == nil {
+			withoutExt = withoutExt[:i]
+		}
+	}
+
+	return withoutExt + "_" + idxS + ext
+}
+
+func withoutIndex(s string) string {
+	if i := strings.LastIndex(s, "_"); i != -1 {
+		return s[:i]
+	}
+
+	return s
+}
+
+type sortByFileIndex []string
+
+func (s sortByFileIndex) Len() int {
+	return len(s)
+}
+
+func (s sortByFileIndex) Swap(i, j int) {
+	s[i], s[j] = s[j], s[i]
+}
+
+func (s sortByFileIndex) Less(i, j int) bool {
+	if withoutIndex(s[i]) == withoutIndex(s[j]) {
+		return getFileIndex(s[i]) < getFileIndex(s[j])
+	}
+
+	return s[i] < s[j]
+}
+
+func (o *FileOutput) filename() string {
+	defer o.mu.Unlock()
+	o.mu.Lock()
+
+	path := o.pathTemplate
+
+	for name, fn := range dateFileNameFuncs {
+		path = strings.Replace(path, name, fn(), -1)
+	}
+
+	if !o.config.append {
+		nextChunk := false
+
+		if o.currentName == "" ||
+			((o.config.queueLimit > 0 && o.queueLength >= o.config.queueLimit) ||
+				(o.config.sizeLimit > 0 && o.chunkSize >= int(o.config.sizeLimit))) {
+			nextChunk = true
+		}
+
+		ext := filepath.Ext(path)
+		withoutExt := strings.TrimSuffix(path, ext)
+
+		if matches, err := filepath.Glob(withoutExt + "*" + ext); err == nil {
+			if len(matches) == 0 {
+				return setFileIndex(path, 0)
+			}
+			sort.Sort(sortByFileIndex(matches))
+
+			last := matches[len(matches)-1]
+
+			fileIndex := 0
+			if idx := getFileIndex(last); idx != -1 {
+				fileIndex = idx
+
+				if nextChunk {
+					fileIndex++
+				}
+			}
+
+			return setFileIndex(last, fileIndex)
+		}
+	}
+
+	return path
+}
+
+func (o *FileOutput) updateName() {
+	o.currentName = filepath.Clean(o.filename())
 }
 
 func (o *FileOutput) Write(data []byte) (n int, err error) {
-	raw := RawRequest{time.Now().UnixNano(), data}
+	if !isOriginPayload(data) {
+		return len(data), nil
+	}
 
-	o.encoder.Encode(raw)
+	if o.file == nil || o.currentName != o.file.Name() {
+		o.mu.Lock()
+		o.Close()
+
+		o.file, err = os.OpenFile(o.currentName, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0660)
+		o.file.Sync()
+
+		if strings.HasSuffix(o.currentName, ".gz") {
+			o.writer = gzip.NewWriter(o.file)
+		} else {
+			o.writer = bufio.NewWriter(o.file)
+		}
+
+		if err != nil {
+			log.Fatal(o, "Cannot open file %q. Error: %s", o.currentName, err)
+		}
+
+		o.queueLength = 0
+		o.mu.Unlock()
+	}
+
+	o.writer.Write(data)
+	o.writer.Write([]byte(payloadSeparator))
+
+	o.queueLength++
 
 	return len(data), nil
 }
 
+func (o *FileOutput) flush() {
+	// Don't exit on panic
+	defer func() {
+		if r := recover(); r != nil {
+			log.Println("PANIC while file flush: ", r, o, string(debug.Stack()))
+		}
+	}()
+
+	defer o.mu.Unlock()
+	o.mu.Lock()
+
+	if o.file != nil {
+		if strings.HasSuffix(o.currentName, ".gz") {
+			o.writer.(*gzip.Writer).Flush()
+		} else {
+			o.writer.(*bufio.Writer).Flush()
+		}
+
+		if stat, err := o.file.Stat(); err != nil {
+			o.chunkSize = int(stat.Size())
+		}
+	}
+}
+
 func (o *FileOutput) String() string {
-	return "File output: " + o.path
+	return "File output: " + o.file.Name()
+}
+
+func (o *FileOutput) Close() error {
+	if o.file != nil {
+		if strings.HasSuffix(o.currentName, ".gz") {
+			o.writer.(*gzip.Writer).Close()
+		} else {
+			o.writer.(*bufio.Writer).Flush()
+		}
+		o.file.Close()
+	}
+	return nil
 }

output_file_settings.go

@@ -0,0 +1,39 @@
+package main
+
+import (
+	"strconv"
+	"strings"
+)
+
+var dataUnitMap = map[byte]int64{
+	'k': 1024,
+	'm': 1024 * 1024,
+	'g': 1024 * 1024 * 1024,
+}
+
+func parseDataUnit(s string) int64 {
+	// Allow kb, mb, gb
+	if strings.HasSuffix(s, "b") {
+		s = s[:len(s)-1]
+	}
+
+	if unit, ok := dataUnitMap[s[len(s)-1]]; ok {
+		size, _ := strconv.ParseInt(s[:len(s)-1], 10, 64)
+		return unit * size
+	} else {
+		// If no unit specified use bytes
+		size, _ := strconv.ParseInt(s, 10, 64)
+		return size
+	}
+}
+
+type unitSizeVar int64
+
+func (u unitSizeVar) String() string {
+	return strconv.Itoa(int(u))
+}
+
+func (u unitSizeVar) Set(s string) error {
+	u = unitSizeVar(parseDataUnit(s))
+	return nil
+}

output_file_test.go

@@ -1,9 +1,16 @@
 package main
 
 import (
+	"fmt"
 	"io"
+	"math/rand"
+	"os"
+	"reflect"
+	"sort"
 	"sync"
+	"sync/atomic"
 	"testing"
+	"time"
 )
 
 func TestFileOutput(t *testing.T) {
@@ -11,7 +18,7 @@ func TestFileOutput(t *testing.T) {
 	quit := make(chan int)
 
 	input := NewTestInput()
-	output := NewFileOutput("/tmp/test_requests.gor")
+	output := NewFileOutput("/tmp/test_requests.gor", &FileOutputConfig{flushInterval: time.Minute, append: true})
 
 	Plugins.Inputs = []io.Reader{input}
 	Plugins.Outputs = []io.Writer{output}
@@ -23,12 +30,17 @@ func TestFileOutput(t *testing.T) {
 		input.EmitGET()
 		input.EmitPOST()
 	}
+	time.Sleep(100 * time.Millisecond)
+	output.flush()
+
 	close(quit)
 
 	quit = make(chan int)
 
-	input2 := NewFileInput("/tmp/test_requests.gor")
+	var counter int64
+	input2 := NewFileInput("/tmp/test_requests.gor", false)
 	output2 := NewTestOutput(func(data []byte) {
+		atomic.AddInt64(&counter, 1)
 		wg.Done()
 	})
 
@@ -40,3 +52,234 @@ func TestFileOutput(t *testing.T) {
 	wg.Wait()
 	close(quit)
 }
+
+func TestFileOutputWithNameCleaning(t *testing.T) {
+	output := &FileOutput{pathTemplate: "./test_requests.gor", config: &FileOutputConfig{flushInterval: time.Minute, append: false}}
+	expectedFileName := "test_requests_0.gor"
+	output.updateName()
+
+	if expectedFileName != output.currentName {
+		t.Errorf("Expected path %s but got %s", expectedFileName, output.currentName)
+	}
+
+}
+
+func TestFileOutputPathTemplate(t *testing.T) {
+	output := &FileOutput{pathTemplate: "/tmp/log-%Y-%m-%d-%S", config: &FileOutputConfig{flushInterval: time.Minute, append: true}}
+	now := time.Now()
+	expectedPath := fmt.Sprintf("/tmp/log-%s-%s-%s-%s", now.Format("2006"), now.Format("01"), now.Format("02"), now.Format("05"))
+	path := output.filename()
+
+	if expectedPath != path {
+		t.Errorf("Expected path %s but got %s", expectedPath, path)
+	}
+}
+
+func TestFileOutputMultipleFiles(t *testing.T) {
+	output := NewFileOutput("/tmp/log-%Y-%m-%d-%S", &FileOutputConfig{append: true, flushInterval: time.Minute})
+
+	if output.file != nil {
+		t.Error("Should not initialize file if no writes")
+	}
+
+	output.Write([]byte("1 1 1\r\ntest"))
+	name1 := output.file.Name()
+
+	output.Write([]byte("1 1 1\r\ntest"))
+	name2 := output.file.Name()
+
+	time.Sleep(time.Second)
+	output.updateName()
+
+	output.Write([]byte("1 1 1\r\ntest"))
+	name3 := output.file.Name()
+
+	if name2 != name1 {
+		t.Error("Fast changes should happen in same file:", name1, name2, name3)
+	}
+
+	if name3 == name1 {
+		t.Error("File name should change:", name1, name2, name3)
+	}
+
+	os.Remove(name1)
+	os.Remove(name3)
+}
+
+func TestFileOutputCompression(t *testing.T) {
+	output := NewFileOutput("/tmp/log-%Y-%m-%d-%S.gz", &FileOutputConfig{append: true, flushInterval: time.Minute})
+
+	if output.file != nil {
+		t.Error("Should not initialize file if no writes")
+	}
+
+	for i := 0; i < 1000; i++ {
+		output.Write([]byte("1 1 1\r\ntest"))
+	}
+
+	name := output.file.Name()
+	output.Close()
+
+	s, _ := os.Stat(name)
+	if s.Size() == 12*1000 {
+		t.Error("Should be compressed file:", s.Size())
+	}
+
+	os.Remove(name)
+}
+
+func TestParseDataUnit(t *testing.T) {
+	var tests = []struct {
+		value string
+		size  int64
+	}{
+		{"100kb", dataUnitMap['k'] * 100},
+		{"100k", dataUnitMap['k'] * 100},
+		{"1kb", dataUnitMap['k']},
+		{"1g", dataUnitMap['g']},
+		{"10m", dataUnitMap['m'] * 10},
+		{"zsaa312", 0},
+	}
+
+	for _, c := range tests {
+		if parseDataUnit(c.value) != c.size {
+			t.Error(c.value, "should be", c.size, "instead", parseDataUnit(c.value))
+		}
+	}
+}
+
+func TestGetFileIndex(t *testing.T) {
+	var tests = []struct {
+		path  string
+		index int
+	}{
+		{"/tmp/logs", -1},
+		{"/tmp/logs_1", 1},
+		{"/tmp/logs_2.gz", 2},
+		{"/tmp/logs_0.gz", 0},
+	}
+
+	for _, c := range tests {
+		if getFileIndex(c.path) != c.index {
+			t.Error(c.path, "should be", c.index, "instead", getFileIndex(c.path))
+		}
+	}
+}
+
+func TestSetFileIndex(t *testing.T) {
+	var tests = []struct {
+		path    string
+		index   int
+		newPath string
+	}{
+		{"/tmp/logs", 0, "/tmp/logs_0"},
+		{"/tmp/logs.gz", 1, "/tmp/logs_1.gz"},
+		{"/tmp/logs_1", 0, "/tmp/logs_0"},
+		{"/tmp/logs_0", 10, "/tmp/logs_10"},
+		{"/tmp/logs_0.gz", 10, "/tmp/logs_10.gz"},
+		{"/tmp/logs_underscores.gz", 10, "/tmp/logs_underscores_10.gz"},
+	}
+
+	for _, c := range tests {
+		if setFileIndex(c.path, c.index) != c.newPath {
+			t.Error(c.path, "should be", c.newPath, "instead", setFileIndex(c.path, c.index))
+		}
+	}
+}
+
+func TestFileOutputAppendQueueLimitOverflow(t *testing.T) {
+	rnd := rand.Int63()
+	name := fmt.Sprintf("/tmp/%d", rnd)
+
+	output := NewFileOutput(name, &FileOutputConfig{append: false, flushInterval: time.Minute, queueLimit: 2})
+
+	output.Write([]byte("1 1 1\r\ntest"))
+	name1 := output.file.Name()
+
+	output.Write([]byte("1 1 1\r\ntest"))
+	name2 := output.file.Name()
+
+	output.updateName()
+
+	output.Write([]byte("1 1 1\r\ntest"))
+	name3 := output.file.Name()
+
+	if name2 != name1 || name1 != fmt.Sprintf("/tmp/%d_0", rnd) {
+		t.Error("Fast changes should happen in same file:", name1, name2, name3)
+	}
+
+	if name3 == name1 || name3 != fmt.Sprintf("/tmp/%d_1", rnd) {
+		t.Error("File name should change:", name1, name2, name3)
+	}
+
+	os.Remove(name1)
+	os.Remove(name3)
+}
+
+func TestFileOutputAppendQueueLimitNoOverflow(t *testing.T) {
+	rnd := rand.Int63()
+	name := fmt.Sprintf("/tmp/%d", rnd)
+
+	output := NewFileOutput(name, &FileOutputConfig{append: false, flushInterval: time.Minute, queueLimit: 3})
+
+	output.Write([]byte("1 1 1\r\ntest"))
+	name1 := output.file.Name()
+
+	output.Write([]byte("1 1 1\r\ntest"))
+	name2 := output.file.Name()
+
+	output.updateName()
+
+	output.Write([]byte("1 1 1\r\ntest"))
+	name3 := output.file.Name()
+
+	if name2 != name1 || name1 != fmt.Sprintf("/tmp/%d_0", rnd) {
+		t.Error("Fast changes should happen in same file:", name1, name2, name3)
+	}
+
+	if name3 != name1 || name3 != fmt.Sprintf("/tmp/%d_0", rnd) {
+		t.Error("File name should not change:", name1, name2, name3)
+	}
+
+	os.Remove(name1)
+	os.Remove(name3)
+}
+
+func TestFileOutputAppendQueueLimitGzips(t *testing.T) {
+	rnd := rand.Int63()
+	name := fmt.Sprintf("/tmp/%d.gz", rnd)
+
+	output := NewFileOutput(name, &FileOutputConfig{append: false, flushInterval: time.Minute, queueLimit: 2})
+
+	output.Write([]byte("1 1 1\r\ntest"))
+	name1 := output.file.Name()
+
+	output.Write([]byte("1 1 1\r\ntest"))
+	name2 := output.file.Name()
+
+	output.updateName()
+
+	output.Write([]byte("1 1 1\r\ntest"))
+	name3 := output.file.Name()
+
+	if name2 != name1 || name1 != fmt.Sprintf("/tmp/%d_0.gz", rnd) {
+		t.Error("Fast changes should happen in same file:", name1, name2, name3)
+	}
+
+	if name3 == name1 || name3 != fmt.Sprintf("/tmp/%d_1.gz", rnd) {
+		t.Error("File name should change:", name1, name2, name3)
+	}
+
+	os.Remove(name1)
+	os.Remove(name3)
+}
+
+func TestFileOutputSort(t *testing.T) {
+	var files = []string{"2016_0", "2014_10", "2015_0", "2015_10", "2015_2"}
+	var expected = []string{"2014_10", "2015_0", "2015_2", "2015_10", "2016_0"}
+	sort.Sort(sortByFileIndex(files))
+
+	if !reflect.DeepEqual(files, expected) {
+		t.Error("Should properly sort file names using indexes", files, expected)
+	}
+}

output_http.go

@@ -1,150 +1,213 @@
 package main
 
 import (
-	"bufio"
-	"bytes"
-	es "github.com/buger/gor/elasticsearch"
 	"io"
-	"log"
-	"net/http"
-	"net/url"
-	"strconv"
-	"strings"
+	"sync/atomic"
 	"time"
+
+	"github.com/buger/gor/proto"
 )
 
-type RedirectNotAllowed struct{}
+const initialDynamicWorkers = 10
 
-func (e *RedirectNotAllowed) Error() string {
-	return "Redirects not allowed"
+type response struct {
+	payload       []byte
+	uuid          []byte
+	roundTripTime int64
+	startedAt     int64
 }
 
-// customCheckRedirect disables redirects https://github.com/buger/gor/pull/15
-func customCheckRedirect(req *http.Request, via []*http.Request) error {
-	if len(via) >= 0 {
-		return new(RedirectNotAllowed)
-	}
-	return nil
-}
+// HTTPOutputConfig struct for holding http output configuration
+type HTTPOutputConfig struct {
+	redirectLimit int
 
-// ParseRequest in []byte returns a http request or an error
-func ParseRequest(data []byte) (request *http.Request, err error) {
-	buf := bytes.NewBuffer(data)
-	reader := bufio.NewReader(buf)
+	stats   bool
+	workers int
 
-	request, err = http.ReadRequest(reader)
+	elasticSearch string
 
-	return
+	Timeout      time.Duration
+	OriginalHost bool
+	BufferSize   int
+
+	Debug bool
+
+	TrackResponses bool
 }
 
+// HTTPOutput plugin manage pool of workers which send request to replayed server
+// By default workers pool is dynamic and starts with 10 workers
+// You can specify fixed number of workers using `--output-http-workers`
 type HTTPOutput struct {
+	// Keep this as first element of struct because it guarantees 64bit
+	// alignment. atomic.* functions crash on 32bit machines if operand is not
+	// aligned at 64bit. See https://github.com/golang/go/issues/599
+	activeWorkers int64
+
 	address string
 	limit   int
-	buf     chan []byte
+	queue   chan []byte
 
-	headers HTTPHeaders
-	methods HTTPMethods
+	responses chan response
 
-	elasticSearch *es.ESPlugin
+	needWorker chan int
 
-	bufStats *GorStat
+	config *HTTPOutputConfig
+
+	queueStats *GorStat
+
+	elasticSearch *ESPlugin
 }
 
-func NewHTTPOutput(options string, headers HTTPHeaders, methods HTTPMethods, elasticSearchAddr string) io.Writer {
+// NewHTTPOutput constructor for HTTPOutput
+// Initialize workers
+func NewHTTPOutput(address string, config *HTTPOutputConfig) io.Writer {
 	o := new(HTTPOutput)
 
-	optionsArr := strings.Split(options, "|")
-	address := optionsArr[0]
-
-	if !strings.HasPrefix(address, "http") {
-		address = "http://" + address
-	}
-
 	o.address = address
-	o.headers = headers
-	o.methods = methods
+	o.config = config
 
-	o.buf = make(chan []byte, 100)
-	o.bufStats = NewGorStat("output_http")
-
-	if elasticSearchAddr != "" {
-		o.elasticSearch = new(es.ESPlugin)
-		o.elasticSearch.Init(elasticSearchAddr)
+	if o.config.stats {
+		o.queueStats = NewGorStat("output_http")
 	}
 
-	if len(optionsArr) > 1 {
-		o.limit, _ = strconv.Atoi(optionsArr[1])
-	}
+	o.queue = make(chan []byte, 1000)
+	o.responses = make(chan response, 1000)
+	o.needWorker = make(chan int, 1)
 
-	for i := 0; i < 10; i++ {
-		go o.worker(i)
-	}
-
-	if o.limit > 0 {
-		return NewLimiter(o, o.limit)
+	// Initial workers count
+	if o.config.workers == 0 {
+		o.needWorker <- initialDynamicWorkers
 	} else {
-		return o
+		o.needWorker <- o.config.workers
+	}
+
+	if o.config.elasticSearch != "" {
+		o.elasticSearch = new(ESPlugin)
+		o.elasticSearch.Init(o.config.elasticSearch)
+	}
+
+	if len(Settings.middleware) > 0 {
+		o.config.TrackResponses = true
+	}
+
+	go o.workerMaster()
+
+	return o
+}
+
+func (o *HTTPOutput) workerMaster() {
+	for {
+		newWorkers := <-o.needWorker
+		for i := 0; i < newWorkers; i++ {
+			go o.startWorker()
+		}
+
+		// Disable dynamic scaling if workers poll fixed size
+		if o.config.workers != 0 {
+			return
+		}
 	}
 }
 
-func (o *HTTPOutput) worker(n int) {
-	client := &http.Client{
-		CheckRedirect: customCheckRedirect,
-	}
+func (o *HTTPOutput) startWorker() {
+	client := NewHTTPClient(o.address, &HTTPClientConfig{
+		FollowRedirects:    o.config.redirectLimit,
+		Debug:              o.config.Debug,
+		OriginalHost:       o.config.OriginalHost,
+		Timeout:            o.config.Timeout,
+		ResponseBufferSize: o.config.BufferSize,
+	})
+
+	deathCount := 0
+
+	atomic.AddInt64(&o.activeWorkers, 1)
 
 	for {
-		data := <-o.buf
-		o.sendRequest(client, data)
+		select {
+		case data := <-o.queue:
+			o.sendRequest(client, data)
+			deathCount = 0
+		case <-time.After(time.Millisecond * 100):
+			// When dynamic scaling enabled workers die after 2s of inactivity
+			if o.config.workers == 0 {
+				deathCount++
+			} else {
+				continue
+			}
+
+			if deathCount > 20 {
+				workersCount := atomic.LoadInt64(&o.activeWorkers)
+
+				// At least 1 startWorker should be alive
+				if workersCount != 1 {
+					atomic.AddInt64(&o.activeWorkers, -1)
+					return
+				}
+			}
+		}
 	}
 }
 
 func (o *HTTPOutput) Write(data []byte) (n int, err error) {
+	if !isRequestPayload(data) {
+		return len(data), nil
+	}
+
 	buf := make([]byte, len(data))
 	copy(buf, data)
 
-	o.buf <- buf
-	o.bufStats.Write(len(o.buf))
+	o.queue <- buf
+
+	if o.config.stats {
+		o.queueStats.Write(len(o.queue))
+	}
+
+	if o.config.workers == 0 {
+		workersCount := atomic.LoadInt64(&o.activeWorkers)
+
+		if len(o.queue) > int(workersCount) {
+			o.needWorker <- len(o.queue)
+		}
+	}
 
 	return len(data), nil
 }
 
-func (o *HTTPOutput) sendRequest(client *http.Client, data []byte) {
-	request, err := ParseRequest(data)
+func (o *HTTPOutput) Read(data []byte) (int, error) {
+	resp := <-o.responses
 
-	if err != nil {
-		log.Println("Cannot parse request", string(data), err)
+	Debug("[OUTPUT-HTTP] Received response:", string(resp.payload))
+
+	header := payloadHeader(ReplayedResponsePayload, resp.uuid, resp.startedAt, resp.roundTripTime)
+	copy(data[0:len(header)], header)
+	copy(data[len(header):], resp.payload)
+
+	return len(resp.payload) + len(header), nil
+}
+
+func (o *HTTPOutput) sendRequest(client *HTTPClient, request []byte) {
+	meta := payloadMeta(request)
+	if len(meta) < 2 {
 		return
 	}
+	uuid := meta[1]
 
-	if len(o.methods) > 0 && !o.methods.Contains(request.Method) {
+	body := payloadBody(request)
+	if !proto.IsHTTPPayload(body) {
 		return
 	}
 
-	// Change HOST of original request
-	URL := o.address + request.URL.Path + "?" + request.URL.RawQuery
-
-	request.RequestURI = ""
-	request.URL, _ = url.ParseRequestURI(URL)
-
-	for _, header := range o.headers {
-		request.Header.Set(header.Name, header.Value)
-	}
-
 	start := time.Now()
-	resp, err := client.Do(request)
+	resp, err := client.Send(body)
 	stop := time.Now()
 
-	// We should not count Redirect as errors
-	if urlErr, ok := err.(*url.Error); ok {
-		if _, ok := urlErr.Err.(*RedirectNotAllowed); ok {
-			err = nil
-		}
+	if err != nil {
+		Debug("Request error:", err)
 	}
 
-	if err == nil {
-		defer resp.Body.Close()
-	} else {
-		log.Println("Request error:", err)
+	if o.config.TrackResponses {
+		o.responses <- response{resp, uuid, start.UnixNano(), stop.UnixNano() - start.UnixNano()}
 	}
 
 	if o.elasticSearch != nil {

output_http_test.go

@@ -2,35 +2,22 @@ package main
 
 import (
 	"io"
-	"net"
+	"io/ioutil"
 	"net/http"
+	"net/http/httptest"
+	_ "net/http/httputil"
 	"sync"
 	"testing"
 	"time"
 )
 
-func startHTTP(cb func(*http.Request)) net.Listener {
-	handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		go cb(r)
-	})
-
-	listener, _ := net.Listen("tcp", ":0")
-
-	go http.Serve(listener, handler)
-
-	return listener
-}
-
 func TestHTTPOutput(t *testing.T) {
 	wg := new(sync.WaitGroup)
 	quit := make(chan int)
 
 	input := NewTestInput()
 
-	headers := HTTPHeaders{HTTPHeader{"User-Agent", "Gor"}}
-	methods := HTTPMethods{"GET", "PUT", "POST"}
-
-	listener := startHTTP(func(req *http.Request) {
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
 		if req.Header.Get("User-Agent") != "Gor" {
 			t.Error("Wrong header")
 		}
@@ -39,10 +26,24 @@ func TestHTTPOutput(t *testing.T) {
 			t.Error("Wrong method")
 		}
 
-		wg.Done()
-	})
+		if req.Method == "POST" {
+			defer req.Body.Close()
+			body, _ := ioutil.ReadAll(req.Body)
 
-	output := NewHTTPOutput(listener.Addr().String(), headers, methods, "")
+			if string(body) != "a=1&b=2" {
+				t.Error("Wrong POST body:", string(body))
+			}
+		}
+
+		wg.Done()
+	}))
+	defer server.Close()
+
+	headers := HTTPHeaders{HTTPHeader{"User-Agent", "Gor"}}
+	methods := HTTPMethods{[]byte("GET"), []byte("PUT"), []byte("POST")}
+	Settings.modifierConfig = HTTPModifierConfig{headers: headers, methods: methods}
+
+	output := NewHTTPOutput(server.URL, &HTTPOutputConfig{Debug: true})
 
 	Plugins.Inputs = []io.Reader{input}
 	Plugins.Outputs = []io.Writer{output}
@@ -50,7 +51,7 @@ func TestHTTPOutput(t *testing.T) {
 	go Start(quit)
 
 	for i := 0; i < 100; i++ {
-		wg.Add(2)
+		wg.Add(2) // OPTIONS should be ignored
 		input.EmitPOST()
 		input.EmitOPTIONS()
 		input.EmitGET()
@@ -59,23 +60,83 @@ func TestHTTPOutput(t *testing.T) {
 	wg.Wait()
 
 	close(quit)
+
+	Settings.modifierConfig = HTTPModifierConfig{}
+}
+
+func TestHTTPOutputKeepOriginalHost(t *testing.T) {
+	wg := new(sync.WaitGroup)
+	quit := make(chan int)
+
+	input := NewTestInput()
+
+	server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
+		if req.Host != "custom-host.com" {
+			t.Error("Wrong header", req.Host)
+		}
+
+		wg.Done()
+	}))
+	defer server.Close()
+
+	headers := HTTPHeaders{HTTPHeader{"Host", "custom-host.com"}}
+	Settings.modifierConfig = HTTPModifierConfig{headers: headers}
+
+	output := NewHTTPOutput(server.URL, &HTTPOutputConfig{Debug: false, OriginalHost: true})
+
+	Plugins.Inputs = []io.Reader{input}
+	Plugins.Outputs = []io.Writer{output}
+
+	go Start(quit)
+
+	wg.Add(1)
+	input.EmitGET()
+
+	wg.Wait()
+
+	close(quit)
+
+	Settings.modifierConfig = HTTPModifierConfig{}
+}
+
+func TestOutputHTTPSSL(t *testing.T) {
+	wg := new(sync.WaitGroup)
+	quit := make(chan int)
+
+	// Origing and Replay server initialization
+	server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		wg.Done()
+	}))
+
+	input := NewTestInput()
+	output := NewHTTPOutput(server.URL, &HTTPOutputConfig{})
+
+	Plugins.Inputs = []io.Reader{input}
+	Plugins.Outputs = []io.Writer{output}
+
+	go Start(quit)
+
+	wg.Add(2)
+
+	input.EmitPOST()
+	input.EmitGET()
+
+	wg.Wait()
+	close(quit)
 }
 
 func BenchmarkHTTPOutput(b *testing.B) {
 	wg := new(sync.WaitGroup)
 	quit := make(chan int)
 
-	input := NewTestInput()
-
-	headers := HTTPHeaders{HTTPHeader{"User-Agent", "Gor"}}
-	methods := HTTPMethods{"GET", "PUT", "POST"}
-
-	listener := startHTTP(func(req *http.Request) {
+	server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		time.Sleep(50 * time.Millisecond)
 		wg.Done()
-	})
+	}))
+	defer server.Close()
 
-	output := NewHTTPOutput(listener.Addr().String(), headers, methods, "")
+	input := NewTestInput()
+	output := NewHTTPOutput(server.URL, &HTTPOutputConfig{})
 
 	Plugins.Inputs = []io.Reader{input}
 	Plugins.Outputs = []io.Writer{output}

output_null.go

@@ -0,0 +1,18 @@
+package main
+
+// NullOutput used for debugging, prints nothing
+type NullOutput struct {
+}
+
+// NullOutput constructor for NullOutput
+func NewNullOutput() (o *NullOutput) {
+    return new(NullOutput)
+}
+
+func (o *NullOutput) Write(data []byte) (int, error) {
+    return len(data), nil
+}
+
+func (o *NullOutput) String() string {
+    return "Null Output"
+}

output_tcp.go

@@ -5,56 +5,85 @@ import (
 	"io"
 	"log"
 	"net"
-	"strconv"
-	"strings"
+	"time"
 )
 
+// TCPOutput used for sending raw tcp payloads
+// Currently used for internal communication between listener and replay server
+// Can be used for transfering binary payloads like protocol buffers
 type TCPOutput struct {
-	address string
-	limit   int
-	buf     chan []byte
+	address  string
+	limit    int
+	buf      chan []byte
 	bufStats *GorStat
 }
 
-func NewTCPOutput(options string) io.Writer {
+// NewTCPOutput constructor for TCPOutput
+// Initialize 10 workers which hold keep-alive connection
+func NewTCPOutput(address string) io.Writer {
 	o := new(TCPOutput)
 
-	optionsArr := strings.Split(options, "|")
-	o.address = optionsArr[0]
+	o.address = address
 
 	o.buf = make(chan []byte, 100)
-	o.bufStats = NewGorStat("output_tcp")
-
-	if len(optionsArr) > 1 {
-		o.limit, _ = strconv.Atoi(optionsArr[1])
+	if Settings.outputTCPStats {
+		o.bufStats = NewGorStat("output_tcp")
 	}
 
 	for i := 0; i < 10; i++ {
 		go o.worker()
 	}
 
-	if o.limit > 0 {
-		return NewLimiter(o, o.limit)
-	} else {
-		return o
-	}
+	return o
 }
 
 func (o *TCPOutput) worker() {
-	conn, _ := o.connect(o.address)
+	retries := 1
+	conn, err := o.connect(o.address)
+	for {
+		if err == nil {
+			break
+		}
+
+		log.Println("Can't connect to aggregator instance, reconnecting in 1 second. Retries:", retries)
+		time.Sleep(1 * time.Second)
+
+		conn, err = o.connect(o.address)
+		retries++
+	}
+
+	if retries > 0 {
+		log.Println("Connected to aggregator instance after ", retries, " retries")
+	}
+
 	defer conn.Close()
 
 	for {
 		conn.Write(<-o.buf)
+		_, err := conn.Write([]byte(payloadSeparator))
+
+		if err != nil {
+			log.Println("Lost connection with aggregator instance, reconnecting")
+			go o.worker()
+			break
+		}
 	}
 }
 
 func (o *TCPOutput) Write(data []byte) (n int, err error) {
-	new_buf := make([]byte, len(data) + 2)
-	data = append(data,[]byte("¶")...)
-	copy(new_buf, data)
-	o.buf <- new_buf
-	o.bufStats.Write(len(o.buf))
+	if !isOriginPayload(data) {
+		return len(data), nil
+	}
+
+	// We have to copy, because sending data in multiple threads
+	newBuf := make([]byte, len(data))
+	copy(newBuf, data)
+
+	o.buf <- newBuf
+
+	if Settings.outputTCPStats {
+		o.bufStats.Write(len(o.buf))
+	}
 
 	return len(data), nil
 }
@@ -62,10 +91,6 @@ func (o *TCPOutput) Write(data []byte) (n int, err error) {
 func (o *TCPOutput) connect(address string) (conn net.Conn, err error) {
 	conn, err = net.Dial("tcp", address)
 
-	if err != nil {
-		log.Println("Connection error ", err, o.address)
-	}
-
 	return
 }
 

output_tcp_test.go

@@ -35,7 +35,7 @@ func TestTCPOutput(t *testing.T) {
 }
 
 func startTCP(cb func([]byte)) net.Listener {
-	listener, err := net.Listen("tcp", ":0")
+	listener, err := net.Listen("tcp", "127.0.0.1:0")
 
 	if err != nil {
 		log.Fatal("Can't start:", err)
@@ -44,22 +44,16 @@ func startTCP(cb func([]byte)) net.Listener {
 	go func() {
 		for {
 			conn, _ := listener.Accept()
-			
+			defer conn.Close()
+
 			go func() {
 				reader := bufio.NewReader(conn)
-				for {
-					buf,err := reader.ReadBytes('¶')
-					new_buf_len := len(buf) - 2
-					new_buf := make([]byte, new_buf_len)
-					copy(new_buf, buf[:new_buf_len])
-					if err != nil {
-						if err != io.EOF {
-							log.Printf("error: %s\n", err)
-						}
-					}
-					cb(new_buf)
+				scanner := bufio.NewScanner(reader)
+				scanner.Split(payloadScanner)
+
+				for scanner.Scan() {
+					cb(scanner.Bytes())
 				}
-				conn.Close()
 			}()
 		}
 	}()

plugins.go

@@ -2,45 +2,144 @@ package main
 
 import (
 	"io"
+	"reflect"
+	"strings"
+	"sync"
+	"time"
 )
 
+// InOutPlugins struct for holding references to plugins
 type InOutPlugins struct {
 	Inputs  []io.Reader
 	Outputs []io.Writer
+	All     []interface{}
 }
 
+var pluginMu sync.Mutex
+
+// Plugins holds all the plugin objects
 var Plugins *InOutPlugins = new(InOutPlugins)
 
-func InitPlugins() {
-	for _, options := range Settings.inputDummy {
-		Plugins.Inputs = append(Plugins.Inputs, NewDummyInput(options))
+// extractLimitOptions detects if plugin get called with limiter support
+// Returns address and limit
+func extractLimitOptions(options string) (string, string) {
+	split := strings.Split(options, "|")
+
+	if len(split) > 1 {
+		return split[0], split[1]
 	}
 
-	for _, options := range Settings.outputDummy {
-		Plugins.Outputs = append(Plugins.Outputs, NewDummyOutput(options))
+	return split[0], ""
+}
+
+// Automatically detects type of plugin and initialize it
+//
+// See this article if curious about relfect stuff below: http://blog.burntsushi.net/type-parametric-functions-golang
+func registerPlugin(constructor interface{}, options ...interface{}) {
+	var path, limit string
+	vc := reflect.ValueOf(constructor)
+
+	// Pre-processing options to make it work with reflect
+	vo := []reflect.Value{}
+	for _, oi := range options {
+		vo = append(vo, reflect.ValueOf(oi))
+	}
+
+	if len(vo) > 0 {
+		// Removing limit options from path
+		path, limit = extractLimitOptions(vo[0].String())
+
+		// Writing value back without limiter "|" options
+		vo[0] = reflect.ValueOf(path)
+	}
+
+	// Calling our constructor with list of given options
+	plugin := vc.Call(vo)[0].Interface()
+	pluginWrapper := plugin
+
+	if limit != "" {
+		pluginWrapper = NewLimiter(plugin, limit)
+	} else {
+		pluginWrapper = plugin
+	}
+
+	_, isR := plugin.(io.Reader)
+	_, isW := plugin.(io.Writer)
+
+	// Some of the output can be Readers as well because return responses
+	if isR && !isW {
+		Plugins.Inputs = append(Plugins.Inputs, pluginWrapper.(io.Reader))
+	}
+
+	if isW {
+		Plugins.Outputs = append(Plugins.Outputs, pluginWrapper.(io.Writer))
+	}
+
+	Plugins.All = append(Plugins.All, plugin)
+}
+
+// InitPlugins specify and initialize all available plugins
+func InitPlugins() {
+	pluginMu.Lock()
+	defer pluginMu.Unlock()
+
+	for _, options := range Settings.inputDummy {
+		registerPlugin(NewDummyInput, options)
+	}
+
+	for range Settings.outputDummy {
+		registerPlugin(NewDummyOutput)
+	}
+
+	if Settings.outputStdout {
+		registerPlugin(NewDummyOutput)
+	}
+
+	if Settings.outputNull {
+		registerPlugin(NewNullOutput)
+	}
+
+	engine := EnginePcap
+	if Settings.inputRAWEngine == "raw_socket" {
+		engine = EngineRawSocket
+	} else if Settings.inputRAWEngine == "pcap_file" {
+		engine = EnginePcapFile
 	}
 
 	for _, options := range Settings.inputRAW {
-		Plugins.Inputs = append(Plugins.Inputs, NewRAWInput(options))
+		registerPlugin(NewRAWInput, options, engine, Settings.inputRAWTrackResponse, time.Duration(0), Settings.inputRAWRealIPHeader)
 	}
 
 	for _, options := range Settings.inputTCP {
-		Plugins.Inputs = append(Plugins.Inputs, NewTCPInput(options))
+		registerPlugin(NewTCPInput, options)
 	}
 
 	for _, options := range Settings.outputTCP {
-		Plugins.Outputs = append(Plugins.Outputs, NewTCPOutput(options))
+		registerPlugin(NewTCPOutput, options)
 	}
 
 	for _, options := range Settings.inputFile {
-		Plugins.Inputs = append(Plugins.Inputs, NewFileInput(options))
+		registerPlugin(NewFileInput, options, Settings.inputFileLoop)
 	}
 
 	for _, options := range Settings.outputFile {
-		Plugins.Outputs = append(Plugins.Outputs, NewFileOutput(options))
+		registerPlugin(NewFileOutput, options, &Settings.outputFileConfig)
+	}
+
+	for _, options := range Settings.inputHTTP {
+		registerPlugin(NewHTTPInput, options)
+	}
+
+	// If we explicitly set Host header http output should not rewrite it
+	// Fix: https://github.com/buger/gor/issues/174
+	for _, header := range Settings.modifierConfig.headers {
+		if header.Name == "Host" {
+			Settings.outputHTTPConfig.OriginalHost = true
+			break
+		}
 	}
 
 	for _, options := range Settings.outputHTTP {
-		Plugins.Outputs = append(Plugins.Outputs, NewHTTPOutput(options, Settings.outputHTTPHeaders, Settings.outputHTTPMethods, Settings.outputHTTPElasticSearch))
+		registerPlugin(NewHTTPOutput, options, &Settings.outputHTTPConfig)
 	}
 }

plugins_test.go

@@ -0,0 +1,47 @@
+package main
+
+import (
+	"io"
+	"testing"
+)
+
+func TestPluginsRegistration(t *testing.T) {
+	Plugins.Inputs = []io.Reader{}
+	Plugins.Outputs = []io.Writer{}
+
+	Settings.inputDummy = MultiOption{"[]"}
+	Settings.outputDummy = MultiOption{"[]"}
+	Settings.outputHTTP = MultiOption{"www.example.com|10"}
+	Settings.inputFile = MultiOption{"/dev/null"}
+
+	InitPlugins()
+
+	if len(Plugins.Inputs) != 2 {
+		t.Errorf("Should be 2 inputs %d", len(Plugins.Inputs))
+	}
+
+	if _, ok := Plugins.Inputs[0].(*DummyInput); !ok {
+		t.Errorf("First input should be DummyInput")
+	}
+
+	if _, ok := Plugins.Inputs[1].(*FileInput); !ok {
+		t.Errorf("Second input should be FileInput")
+	}
+
+	if len(Plugins.Outputs) != 2 {
+		t.Errorf("Should be 2 output %d", len(Plugins.Outputs))
+	}
+
+	if _, ok := Plugins.Outputs[0].(*DummyOutput); !ok {
+		t.Errorf("First output should be DummyOutput")
+	}
+
+	if l, ok := Plugins.Outputs[1].(*Limiter); ok {
+		if _, ok := l.plugin.(*HTTPOutput); !ok {
+			t.Errorf("HTTPOutput should be wrapped in limiter")
+		}
+	} else {
+		t.Errorf("Second output should be Limiter")
+	}
+
+}

proto/proto.go

@@ -0,0 +1,462 @@
+/*
+Package proto provides byte-level interaction with HTTP request payload.
+
+Example of HTTP payload for future references, new line symbols escaped:
+
+	POST /upload HTTP/1.1\r\n
+	User-Agent: Gor\r\n
+	Content-Length: 11\r\n
+	\r\n
+	Hello world
+
+	GET /index.html HTTP/1.1\r\n
+	User-Agent: Gor\r\n
+	\r\n
+	\r\n
+*/
+package proto
+
+import (
+	"bytes"
+	"github.com/buger/gor/byteutils"
+)
+
+// In HTTP newline defined by 2 bytes (for both windows and *nix support)
+var CLRF = []byte("\r\n")
+
+// New line acts as separator: end of Headers or Body (in some cases)
+var EmptyLine = []byte("\r\n\r\n")
+
+// Separator for Header line. Header looks like: `HeaderName: value`
+var HeaderDelim = []byte(": ")
+
+// MIMEHeadersEndPos finds end of the Headers section, which should end with empty line.
+func MIMEHeadersEndPos(payload []byte) int {
+	return bytes.Index(payload, EmptyLine)
+}
+
+// MIMEHeadersStartPos finds start of Headers section
+// It just finds position of second line (first contains location and method).
+func MIMEHeadersStartPos(payload []byte) int {
+	return bytes.Index(payload, CLRF) + 2 // Find first line end
+}
+
+func isLower(b byte) bool {
+	if 'a' <= b && b <= 'z' {
+		return true
+	}
+
+	return false
+}
+
+func toUpper(b byte) byte {
+	if 'a' <= b && b <= 'z' {
+		b -= 'a' - 'A'
+	}
+	return b
+}
+
+func toLower(b byte) byte {
+	if 'A' <= b && b <= 'Z' {
+		b += 'a' - 'A'
+	}
+	return b
+}
+
+func headerIndex(payload []byte, name []byte) int {
+	isLower := isLower(name[0])
+	i := 0
+
+	for {
+		if i >= len(payload) {
+			return -1
+		}
+
+		if payload[i] == '\n' {
+			i++
+
+			// We are at the end
+			if i == len(payload) {
+				return -1
+			}
+
+			if payload[i] == name[0] ||
+				(!isLower && payload[i] == toLower(name[0])) ||
+				(isLower && payload[i] == toUpper(name[0])) {
+
+				i++
+				j := 1
+				for {
+					if j == len(name) {
+						// Matched, and return start of the header
+						return i - len(name)
+					}
+
+					if payload[i] != name[j] {
+						break
+					}
+
+					// If compound header name do one more case check: Content-Length or Transfer-Encoding
+					if name[j] == '-' {
+						i++
+						j++
+
+						if !(payload[i] == name[j] ||
+							(!isLower && payload[i] == toLower(name[j])) ||
+							(isLower && payload[i] == toUpper(name[j]))) {
+							break
+						}
+					}
+
+					j++
+					i++
+				}
+			}
+		}
+
+		i++
+	}
+
+	return -1
+}
+
+// header return value and positions of header/value start/end.
+// If not found, value will be blank, and headerStart will be -1
+// Do not support multi-line headers.
+func header(payload []byte, name []byte) (value []byte, headerStart, headerEnd, valueStart, valueEnd int) {
+	headerStart = headerIndex(payload, name)
+
+	if headerStart == -1 {
+		return
+	}
+
+	valueStart = headerStart + len(name) + 1 // Skip ":" after header name
+	headerEnd = valueStart + bytes.IndexByte(payload[valueStart:], '\n')
+
+	for valueStart < headerEnd { // Ignore empty space after ':'
+		if payload[valueStart] == ' ' {
+			valueStart++
+		} else {
+			break
+		}
+	}
+
+	valueEnd = valueStart + bytes.IndexByte(payload[valueStart:], '\n')
+
+	if payload[headerEnd-1] == '\r' {
+		valueEnd--
+	}
+
+	// ignore empty space at end of header value
+	for valueStart < valueEnd {
+		if payload[valueEnd-1] == ' ' {
+			valueEnd--
+		} else {
+			break
+		}
+	}
+	value = payload[valueStart:valueEnd]
+
+	return
+}
+
+// Works only with ASCII
+func HeadersEqual(h1 []byte, h2 []byte) bool {
+	if len(h1) != len(h2) {
+		return false
+	}
+
+	for i, c1 := range h1 {
+		c2 := h2[i]
+
+		switch int(c1) - int(c2) {
+		case 0, 32, -32:
+		default:
+			return false
+		}
+	}
+
+	return true
+}
+
+// Parsing headers from multiple payloads
+func ParseHeaders(payloads [][]byte, cb func(header []byte, value []byte) bool) {
+
+	hS := [2]int{0, 0}
+	hE := [2]int{-1, -1}
+	vS := [2]int{-1, -1}
+	vE := [2]int{-1, -1}
+
+	i := 0
+	pIdx := 0
+	lineBreaks := 0
+
+	for {
+		if len(payloads)-1 < pIdx {
+			break
+		}
+
+		p := payloads[pIdx]
+
+		if len(p)-1 < i {
+			pIdx++
+			i = 0
+			continue
+		}
+
+		switch p[i] {
+		case '\r', '\n':
+			lineBreaks++
+
+			// End of headers
+			if lineBreaks == 4 {
+				return
+			}
+
+			if lineBreaks > 1 {
+				break
+			}
+
+			vE = [2]int{pIdx, i}
+
+			if vS[1] != -1 && vE[1] != -1 &&
+				hS[1] != -1 && hE[1] != -1 {
+
+				var header, value []byte
+
+				phS, phE, pvS, pvE := payloads[hS[0]], payloads[hE[0]], payloads[vS[0]], payloads[vE[0]]
+
+				// If in same payload
+				if hS[0] == hE[0] {
+					header = phS[hS[1]:hE[1]]
+				} else {
+					header = make([]byte, len(phS)-hS[1]+hE[1])
+					copy(header, phS[hS[1]:])
+					copy(header[len(phS)-hS[1]:], phE[:hE[1]])
+				}
+
+				if vS[0] == vE[0] {
+					value = pvS[vS[1]:vE[1]]
+				} else {
+					value = make([]byte, len(pvS)-vS[1]+vE[1])
+					copy(value, pvS[vS[1]:])
+					copy(value[len(pvS)-vS[1]:], pvE[:vE[1]])
+				}
+
+				if !cb(header, value) {
+					return
+				}
+			}
+
+			// Header found, reset values
+			vS = [2]int{-1, -1}
+			vE = [2]int{-1, -1}
+			hS = [2]int{-1, -1}
+			hE = [2]int{-1, -1}
+		case ':':
+			hE = [2]int{pIdx, i}
+		default:
+			lineBreaks = 0
+
+			if hS[1] == -1 {
+				hS = [2]int{pIdx, i}
+			} else {
+				if hE[1] == -1 {
+					break
+				}
+
+				if vS[1] == -1 {
+					if p[i] == ' ' {
+						break
+					}
+
+					vS = [2]int{pIdx, i}
+				}
+			}
+		}
+
+		i++
+	}
+
+	return
+}
+
+// Header returns header value, if header not found, value will be blank
+func Header(payload, name []byte) []byte {
+	val, _, _, _, _ := header(payload, name)
+
+	return val
+}
+
+// SetHeader sets header value. If header not found it creates new one.
+// Returns modified request payload
+func SetHeader(payload, name, value []byte) []byte {
+	_, hs, _, vs, ve := header(payload, name)
+
+	if hs != -1 {
+		// If header found we just replace its value
+		return byteutils.Replace(payload, vs, ve, value)
+	}
+
+	return AddHeader(payload, name, value)
+}
+
+// AddHeader takes http payload and appends new header to the start of headers section
+// Returns modified request payload
+func AddHeader(payload, name, value []byte) []byte {
+	header := make([]byte, len(name)+2+len(value)+2)
+	copy(header[0:], name)
+	copy(header[len(name):], HeaderDelim)
+	copy(header[len(name)+2:], value)
+	copy(header[len(header)-2:], CLRF)
+
+	mimeStart := MIMEHeadersStartPos(payload)
+
+	return byteutils.Insert(payload, mimeStart, header)
+}
+
+// DelHeader takes http payload and removes header name from headers section
+// Returns modified request payload
+func DeleteHeader(payload, name []byte) []byte {
+	_, hs, he, _, _ := header(payload, name)
+	if hs != -1 {
+		newHeader := make([]byte, len(payload)-(he-hs)-1)
+		copy(newHeader[:hs], payload[:hs])
+		copy(newHeader[hs:], payload[he+1:])
+		return newHeader
+	}
+	return payload
+}
+
+// Body returns request/response body
+func Body(payload []byte) []byte {
+	// 4 -> len(EMPTY_LINE)
+	return payload[MIMEHeadersEndPos(payload)+4:]
+}
+
+// Path takes payload and retuns request path: Split(firstLine, ' ')[1]
+func Path(payload []byte) []byte {
+	start := bytes.IndexByte(payload, ' ') + 1
+	end := bytes.IndexByte(payload[start:], ' ')
+
+	return payload[start : start+end]
+}
+
+// SetPath takes payload, sets new path and returns modified payload
+func SetPath(payload, path []byte) []byte {
+	start := bytes.IndexByte(payload, ' ') + 1
+	end := bytes.IndexByte(payload[start:], ' ')
+
+	return byteutils.Replace(payload, start, start+end, path)
+}
+
+// PathParam returns URL query attribute by given name, if no found: valueStart will be -1
+func PathParam(payload, name []byte) (value []byte, valueStart, valueEnd int) {
+	path := Path(payload)
+
+	if paramStart := bytes.Index(path, append(name, '=')); paramStart != -1 {
+		valueStart := paramStart + len(name) + 1
+		paramEnd := bytes.IndexByte(path[valueStart:], '&')
+
+		// Param can end with '&' (another param), or end of line
+		if paramEnd == -1 { // It is final param
+			paramEnd = len(path)
+		} else {
+			paramEnd += valueStart
+		}
+
+		return path[valueStart:paramEnd], valueStart, paramEnd
+	}
+
+	return []byte(""), -1, -1
+}
+
+// SetPathParam takes payload and updates path Query attribute
+// If query param not found, it will append new
+// Returns modified payload
+func SetPathParam(payload, name, value []byte) []byte {
+	path := Path(payload)
+	_, vs, ve := PathParam(payload, name)
+
+	if vs != -1 { // If param found, replace its value and set new Path
+		newPath := make([]byte, len(path))
+		copy(newPath, path)
+		newPath = byteutils.Replace(newPath, vs, ve, value)
+
+		return SetPath(payload, newPath)
+	}
+
+	// if param not found append to end of url
+	// Adding 2 because of '?' or '&' at start, and '=' in middle
+	newParam := make([]byte, len(name)+len(value)+2)
+
+	if bytes.IndexByte(path, '?') == -1 {
+		newParam[0] = '?'
+	} else {
+		newParam[0] = '&'
+	}
+
+	// Copy "param=value" into buffer, after it looks like "?param=value"
+	copy(newParam[1:], name)
+	newParam[1+len(name)] = '='
+	copy(newParam[2+len(name):], value)
+
+	// Append param to the end of path
+	newPath := make([]byte, len(path)+len(newParam))
+	copy(newPath, path)
+	copy(newPath[len(path):], newParam)
+
+	return SetPath(payload, newPath)
+}
+
+// SetHost updates Host header for HTTP/1.1 or updates host in path for HTTP/1.0 or Proxy requests
+// Returns modified payload
+func SetHost(payload, url, host []byte) []byte {
+	// If this is HTTP 1.0 traffic or proxy traffic it may include host right into path variable, so instead of setting Host header we rewrite Path
+	// Fix for https://github.com/buger/gor/issues/156
+	if path := Path(payload); bytes.HasPrefix(path, []byte("http")) {
+		hostStart := bytes.IndexByte(path, ':') // : position "https?:"
+		hostStart += 3                          // Skip 1 ':' and 2 '\'
+		hostEnd := hostStart + bytes.IndexByte(path[hostStart:], '/')
+
+		newPath := make([]byte, len(path))
+		copy(newPath, path)
+		newPath = byteutils.Replace(newPath, 0, hostEnd, url)
+
+		return SetPath(payload, newPath)
+	}
+
+	return SetHeader(payload, []byte("Host"), host)
+}
+
+// Method returns HTTP method
+func Method(payload []byte) []byte {
+	end := bytes.IndexByte(payload, ' ')
+
+	return payload[:end]
+}
+
+// Status returns response status.
+// It happend to be in same position as request payload path
+func Status(payload []byte) []byte {
+	return Path(payload)
+}
+
+var httpMethods []string = []string{
+	"GET ", "OPTI", "HEAD", "POST", "PUT ", "DELE", "TRAC", "CONN" /* custom methods */, "BAN", "PURG",
+}
+
+func IsHTTPPayload(payload []byte) bool {
+	if len(payload) < 4 {
+		return false
+	}
+
+	method := string(payload[0:4])
+
+	for _, m := range httpMethods {
+		if method == m {
+			return true
+		}
+	}
+	return false
+}

proto/proto_test.go

@@ -0,0 +1,242 @@
+package proto
+
+import (
+	"bytes"
+	"reflect"
+	"testing"
+)
+
+func TestHeader(t *testing.T) {
+	var payload, val []byte
+	var headerStart int
+
+	// Value with space at start
+	payload = []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
+
+	if val = Header(payload, []byte("Content-Length")); !bytes.Equal(val, []byte("7")) {
+		t.Error("Should find header value")
+	}
+
+	// Value with space at end
+	payload = []byte("POST /post HTTP/1.1\r\nContent-Length: 7 \r\nHost: www.w3.org\r\n\r\na=1&b=2")
+
+	if val = Header(payload, []byte("Content-Length")); !bytes.Equal(val, []byte("7")) {
+		t.Error("Should find header value without space after 7")
+	}
+
+	// Value without space at start
+	payload = []byte("POST /post HTTP/1.1\r\nContent-Length:7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
+
+	if val = Header(payload, []byte("Content-Length")); !bytes.Equal(val, []byte("7")) {
+		t.Error("Should find header value without space after :")
+	}
+
+	// Value is empty
+	payload = []byte("GET /p HTTP/1.1\r\nCookie:\r\nHost: www.w3.org\r\n\r\n")
+
+	if val = Header(payload, []byte("Cookie")); len(val) > 0 {
+		t.Error("Should return empty value")
+	}
+
+	// Wrong delimeter
+	payload = []byte("GET /p HTTP/1.1\r\nCookie: 123\nHost: www.w3.org\r\n\r\n")
+
+	if val = Header(payload, []byte("Cookie")); !bytes.Equal(val, []byte("123")) {
+		t.Error("Should handle wrong header delimeter")
+	}
+
+	// Header not found
+	if _, headerStart, _, _, _ = header(payload, []byte("Not-Found")); headerStart != -1 {
+		t.Error("Should not found header")
+	}
+
+	// Lower case headers
+	payload = []byte("POST /post HTTP/1.1\r\ncontent-length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
+
+	if val = Header(payload, []byte("Content-Length")); !bytes.Equal(val, []byte("7")) {
+		t.Error("Should find lower case 2 word header")
+	}
+
+	payload = []byte("POST /post HTTP/1.1\r\ncontent-length: 7\r\nhost: www.w3.org\r\n\r\na=1&b=2")
+
+	if val = Header(payload, []byte("host")); !bytes.Equal(val, []byte("www.w3.org")) {
+		t.Error("Should find lower case 1 word header")
+	}
+}
+
+func TestMIMEHeadersEndPos(t *testing.T) {
+	head := []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org")
+	payload := []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
+
+	end := MIMEHeadersEndPos(payload)
+
+	if !bytes.Equal(payload[:end], head) {
+		t.Error("Wrong headers end position:", end)
+	}
+}
+
+func TestMIMEHeadersStartPos(t *testing.T) {
+	headers := []byte("Content-Length: 7\r\nHost: www.w3.org")
+	payload := []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
+
+	start := MIMEHeadersStartPos(payload)
+	end := MIMEHeadersEndPos(payload)
+
+	if !bytes.Equal(payload[start:end], headers) {
+		t.Error("Wrong headers end position:", start, end)
+	}
+}
+
+func TestSetHeader(t *testing.T) {
+	var payload, payloadAfter []byte
+
+	payload = []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
+	payloadAfter = []byte("POST /post HTTP/1.1\r\nContent-Length: 14\r\nHost: www.w3.org\r\n\r\na=1&b=2")
+
+	if payload = SetHeader(payload, []byte("Content-Length"), []byte("14")); !bytes.Equal(payload, payloadAfter) {
+		t.Error("Should update header if it exists", string(payload))
+	}
+
+	payload = []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
+	payloadAfter = []byte("POST /post HTTP/1.1\r\nUser-Agent: Gor\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
+
+	if payload = SetHeader(payload, []byte("User-Agent"), []byte("Gor")); !bytes.Equal(payload, payloadAfter) {
+		t.Error("Should add header if not found", string(payload))
+	}
+}
+
+func TestDeleteHeader(t *testing.T) {
+	var payload, payloadAfter []byte
+
+	payload = []byte("POST /post HTTP/1.1\r\nUser-Agent: Gor\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
+	payloadAfter = []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
+
+	if payload = DeleteHeader(payload, []byte("User-Agent")); !bytes.Equal(payload, payloadAfter) {
+		t.Error("Should delete header if found", string(payload), string(payloadAfter))
+	}
+
+	//Whitespace at end of User-Agent
+	payload = []byte("POST /post HTTP/1.1\r\nUser-Agent: Gor \r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
+	payloadAfter = []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
+
+	if payload = DeleteHeader(payload, []byte("User-Agent")); !bytes.Equal(payload, payloadAfter) {
+		t.Error("Should delete header if found", string(payload), string(payloadAfter))
+	}
+}
+
+func TestParseHeaders(t *testing.T) {
+	payload := [][]byte{[]byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.or"), []byte("g\r\nUser-Ag"), []byte("ent:Chrome\r\n\r\n"), []byte("Fake-Header: asda")}
+
+	headers := make(map[string]string)
+
+	ParseHeaders(payload, func(header []byte, value []byte) bool {
+		headers[string(header)] = string(value)
+		return true
+	})
+
+	expected := map[string]string{
+		"Content-Length": "7",
+		"Host":           "www.w3.org",
+		"User-Agent":     "Chrome",
+	}
+	if !reflect.DeepEqual(headers, expected) {
+		t.Error("Headers do not properly parsed", headers)
+	}
+}
+
+func TestHeaderEquals(t *testing.T) {
+	tests := []struct {
+		h1     string
+		h2     string
+		equals bool
+	}{
+		{"Content-Length", "content-length", true},
+		{"content-length", "Content-Length", true},
+		{"content-Pength", "Content-Length", false},
+		{"Host", "Content-Length", false},
+	}
+
+	for _, tc := range tests {
+		if HeadersEqual([]byte(tc.h1), []byte(tc.h2)) != tc.equals {
+			t.Error(tc)
+		}
+	}
+}
+
+func TestPath(t *testing.T) {
+	var path, payload []byte
+
+	payload = []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
+
+	if path = Path(payload); !bytes.Equal(path, []byte("/post")) {
+		t.Error("Should find path", string(path))
+	}
+}
+
+func TestSetPath(t *testing.T) {
+	var payload, payloadAfter []byte
+
+	payload = []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
+	payloadAfter = []byte("POST /new_path HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
+
+	if payload = SetPath(payload, []byte("/new_path")); !bytes.Equal(payload, payloadAfter) {
+		t.Error("Should replace path", string(payload))
+	}
+}
+
+func TestPathParam(t *testing.T) {
+	var payload []byte
+
+	payload = []byte("POST /post?param=test&user_id=1 HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
+
+	if val, _, _ := PathParam(payload, []byte("param")); !bytes.Equal(val, []byte("test")) {
+		t.Error("Should detect attribute", string(val))
+	}
+
+	if val, _, _ := PathParam(payload, []byte("user_id")); !bytes.Equal(val, []byte("1")) {
+		t.Error("Should detect attribute", string(val))
+	}
+}
+
+func TestSetPathParam(t *testing.T) {
+	var payload, payloadAfter []byte
+
+	payload = []byte("POST /post?param=test&user_id=1 HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
+	payloadAfter = []byte("POST /post?param=new&user_id=1 HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
+
+	if payload = SetPathParam(payload, []byte("param"), []byte("new")); !bytes.Equal(payload, payloadAfter) {
+		t.Error("Should replace existing value", string(payload))
+	}
+
+	payload = []byte("POST /post?param=test&user_id=1 HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
+	payloadAfter = []byte("POST /post?param=test&user_id=2 HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
+
+	if payload = SetPathParam(payload, []byte("user_id"), []byte("2")); !bytes.Equal(payload, payloadAfter) {
+		t.Error("Should replace existing value", string(payload))
+	}
+
+	payload = []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
+	payloadAfter = []byte("POST /post?param=test HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
+
+	if payload = SetPathParam(payload, []byte("param"), []byte("test")); !bytes.Equal(payload, payloadAfter) {
+		t.Error("Should set param if url have no params", string(payload))
+	}
+
+	payload = []byte("POST /post?param=test HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
+	payloadAfter = []byte("POST /post?param=test&user_id=1 HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
+
+	if payload = SetPathParam(payload, []byte("user_id"), []byte("1")); !bytes.Equal(payload, payloadAfter) {
+		t.Error("Should set param at the end if url params", string(payload))
+	}
+}
+
+func TestSetHostHTTP10(t *testing.T) {
+	var payload, payloadAfter []byte
+
+	payload = []byte("POST http://example.com/post HTTP/1.0\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
+	payloadAfter = []byte("POST http://new.com/post HTTP/1.0\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
+
+	if payload = SetHost(payload, []byte("http://new.com"), []byte("new.com")); !bytes.Equal(payload, payloadAfter) {
+		t.Error("Should replace host", string(payload))
+	}
+}

protocol.go

@@ -0,0 +1,103 @@
+package main
+
+import (
+	"bytes"
+	"crypto/rand"
+	"encoding/hex"
+	"strconv"
+)
+
+const (
+	RequestPayload          = '1'
+	ResponsePayload         = '2'
+	ReplayedResponsePayload = '3'
+)
+
+func uuid() []byte {
+	b := make([]byte, 20)
+	rand.Read(b)
+
+	uuid := make([]byte, 40)
+	hex.Encode(uuid, b)
+
+	return uuid
+}
+
+var payloadSeparator = "\n🐵🙈🙉\n"
+
+func payloadScanner(data []byte, atEOF bool) (advance int, token []byte, err error) {
+	if atEOF && len(data) == 0 {
+		return 0, nil, nil
+	}
+
+	if i := bytes.Index(data, []byte(payloadSeparator)); i >= 0 {
+		// We have a full newline-terminated line.
+		return i + len([]byte(payloadSeparator)), data[0:i], nil
+	}
+
+	if atEOF {
+		return len(data), data, nil
+	}
+	return 0, nil, nil
+}
+
+// Timing is request start or round-trip time, depending on payloadType
+func payloadHeader(payloadType byte, uuid []byte, timing int64, latency int64) (header []byte) {
+	var sTime, sLatency string
+
+	sTime = strconv.FormatInt(timing, 10)
+	if latency != -1 {
+		sLatency = strconv.FormatInt(latency, 10)
+	}
+
+	//Example:
+	//  3 f45590522cd1838b4a0d5c5aab80b77929dea3b3 1231\n
+	// `+ 1` indicates space characters or end of line
+	headerLen := 1 + 1 + len(uuid) + 1 + len(sTime) + 1
+
+	if latency != -1 {
+		headerLen += len(sLatency) + 1
+	}
+
+	header = make([]byte, headerLen)
+	header[0] = payloadType
+	header[1] = ' '
+	header[2+len(uuid)] = ' '
+	header[len(header)-1] = '\n'
+
+	copy(header[2:], uuid)
+	copy(header[3+len(uuid):], sTime)
+
+	if latency != -1 {
+		header[3+len(uuid)+len(sTime)] = ' '
+		copy(header[4+len(uuid)+len(sTime):], sLatency)
+	}
+
+	return header
+}
+
+func payloadBody(payload []byte) []byte {
+	headerSize := bytes.IndexByte(payload, '\n')
+	return payload[headerSize+1:]
+}
+
+func payloadMeta(payload []byte) [][]byte {
+	headerSize := bytes.IndexByte(payload, '\n')
+	if headerSize < 0 {
+		headerSize = 0
+	}
+	return bytes.Split(payload[:headerSize], []byte{' '})
+}
+
+func isOriginPayload(payload []byte) bool {
+	switch payload[0] {
+	case RequestPayload, ResponsePayload:
+		return true
+	default:
+		return false
+	}
+}
+
+func isRequestPayload(payload []byte) bool {
+	return payload[0] == RequestPayload
+}

raw_socket_listener/listener.go

@@ -1,105 +1,643 @@
-package raw_socket
+/*
+Package rawSocket provides traffic sniffier using RAW sockets.
+
+Capture traffic from socket using RAW_SOCKET's
+http://en.wikipedia.org/wiki/Raw_socket
+
+RAW_SOCKET allow you listen for traffic on any port (e.g. sniffing) because they operate on IP level.
+
+Ports is TCP feature, same as flow control, reliable transmission and etc.
+
+This package implements own TCP layer: TCP packets is parsed using tcp_packet.go, and flow control is managed by tcp_message.go
+*/
+package rawSocket
 
 import (
+	"bytes"
 	"encoding/binary"
+	"fmt"
+	"github.com/buger/gor/proto"
+	"github.com/google/gopacket"
+	"github.com/google/gopacket/layers"
+	"github.com/google/gopacket/pcap"
+	"io"
 	"log"
 	"net"
+	"runtime"
+	"runtime/debug"
 	"strconv"
+	"strings"
+	"sync"
+	"time"
 )
 
-// Capture traffic from socket using RAW_SOCKET's
-// http://en.wikipedia.org/wiki/Raw_socket
-//
-// RAW_SOCKET allow you listen for traffic on any port (e.g. sniffing) because they operate on IP level.
-// Ports is TCP feature, same as flow control, reliable transmission and etc.
-// Since we can't use default TCP libraries RAWTCPLitener implements own TCP layer
-// TCP packets is parsed using tcp_packet.go, and flow control is managed by tcp_message.go
+var _ = fmt.Println
+
+// Listener handle traffic capture
 type Listener struct {
-	messages map[string]*TCPMessage // buffer of TCPMessages waiting to be send
+	mu sync.Mutex
+	// buffer of TCPMessages waiting to be send
+	// ID -> TCPMessage
+	messages map[tcpID]*TCPMessage
 
-	c_packets  chan *TCPPacket
-	c_messages chan *TCPMessage // Messages ready to be send to client
+	// Expect: 100-continue request is send in 2 tcp messages
+	// We store ACK aliases to merge this packets together
+	ackAliases map[uint32]uint32
+	// To get ACK of second message we need to compute its Seq and wait for them message
+	seqWithData map[uint32]uint32
 
-	c_del_message chan *TCPMessage // Used for notifications about completed or expired messages
+	// Ack -> Req
+	respAliases map[uint32]*TCPMessage
+
+	// Ack -> ID
+	respWithoutReq map[uint32]tcpID
+
+	// Messages ready to be send to client
+	packetsChan chan []byte
+
+	// Messages ready to be send to client
+	messagesChan chan *TCPMessage
 
 	addr string // IP to listen
-	port int    // Port to listen
+	port uint16 // Port to listen
+
+	trackResponse bool
+	messageExpire time.Duration
+
+	conn        net.PacketConn
+	pcapHandles []*pcap.Handle
+
+	quit    chan bool
+	readyCh chan bool
 }
 
-// RAWTCPListen creates a listener to capture traffic from RAW_SOCKET
-func NewListener(addr string, port string) (rawListener *Listener) {
-	rawListener = &Listener{}
+type request struct {
+	id    tcpID
+	start time.Time
+	ack   uint32
+}
 
-	rawListener.c_packets = make(chan *TCPPacket, 100)
-	rawListener.c_messages = make(chan *TCPMessage, 100)
-	rawListener.c_del_message = make(chan *TCPMessage, 100)
-	rawListener.messages = make(map[string]*TCPMessage)
+// Available engines for intercepting traffic
+const (
+	EngineRawSocket = 1 << iota
+	EnginePcap
+	EnginePcapFile
+)
 
-	rawListener.addr = addr
-	rawListener.port, _ = strconv.Atoi(port)
+// NewListener creates and initializes new Listener object
+func NewListener(addr string, port string, engine int, trackResponse bool, expire time.Duration) (l *Listener) {
+	l = &Listener{}
 
-	go rawListener.listen()
-	go rawListener.readRAWSocket()
+	l.packetsChan = make(chan []byte, 10000)
+	l.messagesChan = make(chan *TCPMessage, 10000)
+	l.quit = make(chan bool)
+	l.readyCh = make(chan bool, 1)
+
+	l.messages = make(map[tcpID]*TCPMessage)
+	l.ackAliases = make(map[uint32]uint32)
+	l.seqWithData = make(map[uint32]uint32)
+	l.respAliases = make(map[uint32]*TCPMessage)
+	l.respWithoutReq = make(map[uint32]tcpID)
+	l.trackResponse = trackResponse
+
+	l.addr = addr
+	_port, _ := strconv.Atoi(port)
+	l.port = uint16(_port)
+
+	if expire.Nanoseconds() == 0 {
+		expire = 2000 * time.Millisecond
+	}
+
+	l.messageExpire = expire
+
+	go l.listen()
+
+	// Special case for testing
+	if l.port != 0 {
+		switch engine {
+		case EngineRawSocket:
+			go l.readRAWSocket()
+		case EnginePcap:
+			go l.readPcap()
+		case EnginePcapFile:
+			go l.readPcapFile()
+		default:
+			log.Fatal("Unknown traffic interception engine:", engine)
+		}
+	}
 
 	return
 }
 
 func (t *Listener) listen() {
+	gcTicker := time.Tick(t.messageExpire / 2)
+
 	for {
 		select {
-		// If message ready for deletion it means that its also complete or expired by timeout
-		case message := <-t.c_del_message:
-			t.c_messages <- message
-			delete(t.messages, message.ID)
-
-		// We need to use channels to process each packet to avoid data races
-		case packet := <-t.c_packets:
+		case <-t.quit:
+			if t.conn != nil {
+				t.conn.Close()
+			}
+			return
+		case data := <-t.packetsChan:
+			packet := ParseTCPPacket(data[:16], data[16:])
 			t.processTCPPacket(packet)
+		case <-gcTicker:
+			now := time.Now()
+
+			// Dispatch requests before responses
+			for _, message := range t.messages {
+				if now.Sub(message.End) >= t.messageExpire {
+					t.dispatchMessage(message)
+				}
+			}
+		}
+	}
+}
+
+func (t *Listener) deleteMessage(message *TCPMessage) {
+	delete(t.messages, message.ID())
+	delete(t.ackAliases, message.Ack)
+	if message.DataAck != 0 {
+		delete(t.ackAliases, message.DataAck)
+	}
+	if message.DataSeq != 0 {
+		delete(t.seqWithData, message.DataSeq)
+	}
+
+	delete(t.respAliases, message.ResponseAck)
+}
+
+func (t *Listener) dispatchMessage(message *TCPMessage) {
+	// If already dispatched
+	if _, ok := t.messages[message.ID()]; !ok {
+		return
+	}
+
+	t.deleteMessage(message)
+
+	if !message.complete {
+		if !message.IsIncoming {
+			delete(t.respAliases, message.Ack)
+			delete(t.respWithoutReq, message.Ack)
+		}
+
+		return
+	}
+
+	if message.IsIncoming {
+		// If there were response before request
+		// log.Println("Looking for Response: ", t.respWithoutReq, message.ResponseAck)
+		if t.trackResponse {
+			if respID, ok := t.respWithoutReq[message.ResponseAck]; ok {
+				if resp, rok := t.messages[respID]; rok {
+					// if resp.AssocMessage == nil {
+					// log.Println("FOUND RESPONSE")
+					resp.setAssocMessage(message)
+					message.setAssocMessage(resp)
+
+					if resp.complete {
+						defer t.dispatchMessage(resp)
+					}
+					// }
+				}
+			}
+
+			if resp, ok := t.messages[message.ResponseID]; ok {
+				resp.setAssocMessage(message)
+			}
+		}
+	} else {
+		if message.AssocMessage == nil {
+			if responseRequest, ok := t.respAliases[message.Ack]; ok {
+				message.setAssocMessage(responseRequest)
+				responseRequest.setAssocMessage(message)
+			}
+		}
+
+		delete(t.respAliases, message.Ack)
+		delete(t.respWithoutReq, message.Ack)
+
+		// Do not track responses which have no associated requests
+		if message.AssocMessage == nil {
+			// log.Println("Can't dispatch resp", message.Seq, message.Ack, string(message.Bytes()))
+			return
+		}
+	}
+
+	t.messagesChan <- message
+}
+
+// DeviceNotFoundError raised if user specified wrong ip
+type DeviceNotFoundError struct {
+	addr string
+}
+
+func (e *DeviceNotFoundError) Error() string {
+	devices, _ := pcap.FindAllDevs()
+
+	if len(devices) == 0 {
+		return "Can't get list of network interfaces, ensure that you running Gor as root user or sudo.\nTo run as non-root users see this docs https://github.com/buger/gor/wiki/Running-as-non-root-user"
+	}
+
+	var msg string
+	msg += "Can't find interfaces with addr: " + e.addr + ". Provide available IP for intercepting traffic: \n"
+	for _, device := range devices {
+		msg += "Name: " + device.Name + "\n"
+		if device.Description != "" {
+			msg += "Description: " + device.Description + "\n"
+		}
+		for _, address := range device.Addresses {
+			msg += "- IP address: " + address.IP.String() + "\n"
+		}
+	}
+
+	return msg
+}
+
+func isLoopback(device pcap.Interface) bool {
+	if len(device.Addresses) == 0 {
+		return false
+	}
+
+	switch device.Addresses[0].IP.String() {
+	case "127.0.0.1", "::1":
+		return true
+	}
+
+	return false
+}
+
+func listenAllInterfaces(addr string) bool {
+	switch addr {
+	case "", "0.0.0.0", "[::]", "::":
+		return true
+	default:
+		return false
+	}
+}
+
+func findPcapDevices(addr string) (interfaces []pcap.Interface, err error) {
+	devices, err := pcap.FindAllDevs()
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	for _, device := range devices {
+		if listenAllInterfaces(addr) && len(device.Addresses) > 0 || isLoopback(device) {
+			interfaces = append(interfaces, device)
+			continue
+		}
+
+		for _, address := range device.Addresses {
+			if device.Name == addr || address.IP.String() == addr {
+				interfaces = append(interfaces, device)
+				return interfaces, nil
+			}
+		}
+	}
+
+	if len(interfaces) == 0 {
+		return nil, &DeviceNotFoundError{addr}
+	} else {
+		return interfaces, nil
+	}
+}
+
+func (t *Listener) readPcap() {
+	devices, err := findPcapDevices(t.addr)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	bpfSupported := true
+	if runtime.GOOS == "darwin" {
+		bpfSupported = false
+	}
+
+	var wg sync.WaitGroup
+	wg.Add(len(devices))
+
+	for _, d := range devices {
+		go func(device pcap.Interface) {
+			handle, err := pcap.OpenLive(device.Name, 65536, true, t.messageExpire)
+			if err != nil {
+				log.Println("Pcap Error while opening device", device.Name, err)
+				wg.Done()
+				return
+			}
+			defer handle.Close()
+
+			t.mu.Lock()
+			t.pcapHandles = append(t.pcapHandles, handle)
+
+			var bpfDstHost, bpfSrcHost string
+			var loopback = isLoopback(device)
+
+			if loopback {
+				var allAddr []string
+				for _, dc := range devices {
+					for _, addr := range dc.Addresses {
+						allAddr = append(allAddr, "(dst host "+addr.IP.String()+" and src host "+addr.IP.String()+")")
+					}
+				}
+
+				bpfDstHost = strings.Join(allAddr, " or ")
+				bpfSrcHost = bpfDstHost
+			} else {
+				for i, addr := range device.Addresses {
+					bpfDstHost += "dst host " + addr.IP.String()
+					bpfSrcHost += "src host " + addr.IP.String()
+					if i != len(device.Addresses)-1 {
+						bpfDstHost += " or "
+						bpfSrcHost += " or "
+					}
+				}
+			}
+
+			if bpfSupported {
+				var bpf string
+
+				if t.trackResponse {
+					bpf = "(tcp dst port " + strconv.Itoa(int(t.port)) + " and (" + bpfDstHost + ")) or (" + "tcp src port " + strconv.Itoa(int(t.port)) + " and (" + bpfSrcHost + "))"
+				} else {
+					bpf = "tcp dst port " + strconv.Itoa(int(t.port)) + " and (" + bpfDstHost + ")"
+				}
+
+				if err := handle.SetBPFFilter(bpf); err != nil {
+					log.Println("BPF filter error:", err, "Device:", device.Name, bpf)
+					wg.Done()
+					return
+				}
+			}
+			t.mu.Unlock()
+
+			var decoder gopacket.Decoder
+
+			// Special case for tunnel interface https://github.com/google/gopacket/issues/99
+			if handle.LinkType() == 12 {
+				decoder = layers.LayerTypeIPv4
+			} else {
+				decoder = handle.LinkType()
+			}
+
+			source := gopacket.NewPacketSource(handle, decoder)
+			source.Lazy = true
+			source.NoCopy = true
+
+			wg.Done()
+
+			var data, srcIP, dstIP []byte
+
+			for {
+				packet, err := source.NextPacket()
+
+				if err == io.EOF {
+					break
+				} else if err != nil {
+					continue
+				}
+
+				// We should remove network layer before parsing TCP/IP data
+				var of int
+				switch decoder {
+				case layers.LinkTypeEthernet:
+					of = 14
+				case layers.LinkTypePPP:
+					of = 1
+				case layers.LinkTypeFDDI:
+					of = 13
+				case layers.LinkTypeNull:
+					of = 4
+				case layers.LinkTypeLoop:
+					of = 4
+				case layers.LinkTypeRaw:
+					of = 0
+				case layers.LinkTypeLinuxSLL:
+					of = 16
+				default:
+					log.Println("Unknown packet layer", packet)
+					break
+				}
+
+				data = packet.Data()[of:]
+
+				version := uint8(data[0]) >> 4
+				ipLength := int(binary.BigEndian.Uint16(data[2:4]))
+
+				if version == 4 {
+					ihl := uint8(data[0]) & 0x0F
+
+					// Truncated IP info
+					if len(data) < int(ihl*4) {
+						continue
+					}
+
+					srcIP = data[12:16]
+					dstIP = data[16:20]
+
+					// Too small IP packet
+					if ipLength < 20 {
+						continue
+					}
+
+					// Invalid length
+					if int(ihl*4) > ipLength {
+						continue
+					}
+
+					if cmp := len(data) - ipLength; cmp > 0 {
+						data = data[:ipLength]
+					} else if cmp < 0 {
+						// Truncated packet
+						continue
+					}
+
+					data = data[ihl*4:]
+				} else {
+					// Truncated IP info
+					if len(data) < 40 {
+						continue
+					}
+
+					srcIP = data[8:24]
+					dstIP = data[24:40]
+
+					data = data[40:]
+				}
+
+				// Truncated TCP info
+				if len(data) < 13 {
+					continue
+				}
+
+				dataOffset := (data[12] & 0xF0) >> 4
+				isFIN := data[13]&0x01 != 0
+
+				// We need only packets with data inside
+				// Check that the buffer is larger than the size of the TCP header
+				if len(data) > int(dataOffset*4) || isFIN {
+					if !bpfSupported {
+						destPort := binary.BigEndian.Uint16(data[2:4])
+						srcPort := binary.BigEndian.Uint16(data[0:2])
+
+						var addrCheck []byte
+
+						if destPort == t.port {
+							addrCheck = dstIP
+						}
+
+						if t.trackResponse && srcPort == t.port {
+							addrCheck = srcIP
+						}
+
+						if len(addrCheck) == 0 {
+							continue
+						}
+
+						addrMatched := false
+
+						if loopback {
+							for _, dc := range devices {
+								if addrMatched {
+									break
+								}
+								for _, a := range dc.Addresses {
+									if a.IP.Equal(net.IP(addrCheck)) {
+										addrMatched = true
+										break
+									}
+								}
+							}
+							addrMatched = true
+						} else {
+							for _, a := range device.Addresses {
+								if a.IP.Equal(net.IP(addrCheck)) {
+									addrMatched = true
+									break
+								}
+							}
+						}
+
+						if !addrMatched {
+							continue
+						}
+					}
+
+					newBuf := make([]byte, len(data)+16)
+					copy(newBuf[:16], srcIP)
+					copy(newBuf[16:], data)
+
+					t.packetsChan <- newBuf
+				}
+			}
+		}(d)
+	}
+
+	wg.Wait()
+	t.readyCh <- true
+}
+
+func (t *Listener) readPcapFile() {
+	if handle, err := pcap.OpenOffline(t.addr); err != nil {
+		log.Fatal(err)
+	} else {
+		t.readyCh <- true
+		packetSource := gopacket.NewPacketSource(handle, handle.LinkType())
+
+		for {
+			packet, err := packetSource.NextPacket()
+			if err == io.EOF {
+				break
+			} else if err != nil {
+				log.Println("Error:", err)
+				continue
+			}
+
+			var addr, data []byte
+
+			if tcpLayer := packet.Layer(layers.LayerTypeTCP); tcpLayer != nil {
+				tcp, _ := tcpLayer.(*layers.TCP)
+				data = append(tcp.LayerContents(), tcp.LayerPayload()...)
+				copy(data[2:4], []byte{0, 1})
+			} else {
+				continue
+			}
+
+			if ipLayer := packet.Layer(layers.LayerTypeIPv4); ipLayer != nil {
+				ip, _ := ipLayer.(*layers.IPv4)
+				addr = ip.SrcIP
+			} else if ipLayer = packet.Layer(layers.LayerTypeIPv6); ipLayer != nil {
+				ip, _ := ipLayer.(*layers.IPv6)
+				addr = ip.SrcIP
+			} else {
+				// log.Println("Can't find IP layer", packet)
+				continue
+			}
+
+			dataOffset := (data[12] & 0xF0) >> 4
+
+			// We need only packets with data inside
+			// Check that the buffer is larger than the size of the TCP header
+			if len(data) <= int(dataOffset*4) {
+				continue
+			}
+
+			newBuf := make([]byte, len(data)+16)
+			copy(newBuf[:16], addr)
+			copy(newBuf[16:], data)
+
+			t.packetsChan <- newBuf
 		}
 	}
 }
 
 func (t *Listener) readRAWSocket() {
-	conn, e := net.ListenPacket("ip4:tcp", t.addr)
-	defer conn.Close()
+	conn, e := net.ListenPacket("ip:tcp", t.addr)
+	t.conn = conn
 
 	if e != nil {
 		log.Fatal(e)
 	}
 
-	buf := make([]byte, 4096*2)
+	defer t.conn.Close()
+
+	buf := make([]byte, 64*1024) // 64kb
+
+	t.readyCh <- true
 
 	for {
 		// Note: ReadFrom receive messages without IP header
-		n, addr, err := conn.ReadFrom(buf)
+		n, addr, err := t.conn.ReadFrom(buf)
 
 		if err != nil {
-			log.Println("Error:", err)
-			continue
+			if strings.HasSuffix(err.Error(), "closed network connection") {
+				return
+			} else {
+				continue
+			}
 		}
 
 		if n > 0 {
-			t.parsePacket(addr, buf[:n])
+			if t.isValidPacket(buf[:n]) {
+				newBuf := make([]byte, n+16)
+				copy(newBuf[16:], buf[:n])
+				copy(newBuf[:16], []byte(addr.(*net.IPAddr).IP))
+
+				t.packetsChan <- newBuf
+			}
 		}
 	}
 }
 
-func (t *Listener) parsePacket(addr net.Addr, buf []byte) {
-	if t.isIncomingDataPacket(buf) {
-		new_buf := make([]byte, len(buf))
-		copy(new_buf, buf)
-
-		t.c_packets <- ParseTCPPacket(addr, new_buf)
-	}
-}
-
-func (t *Listener) isIncomingDataPacket(buf []byte) bool {
+func (t *Listener) isValidPacket(buf []byte) bool {
 	// To avoid full packet parsing every time, we manually parsing values needed for packet filtering
 	// http://en.wikipedia.org/wiki/Transmission_Control_Protocol
-	dest_port := binary.BigEndian.Uint16(buf[2:4])
+	destPort := binary.BigEndian.Uint16(buf[2:4])
+	srcPort := binary.BigEndian.Uint16(buf[0:2])
 
 	// Because RAW_SOCKET can't be bound to port, we have to control it by ourself
-	if int(dest_port) == t.port {
+	if destPort == t.port || (t.trackResponse && srcPort == t.port) {
 		// Get the 'data offset' (size of the TCP header in 32-bit words)
 		dataOffset := (buf[12] & 0xF0) >> 4
 
@@ -118,24 +656,164 @@ func (t *Listener) isIncomingDataPacket(buf []byte) bool {
 //
 // For TCP message unique id is Acknowledgment number (see tcp_packet.go)
 func (t *Listener) processTCPPacket(packet *TCPPacket) {
-	defer func() { recover() }()
+	// Don't exit on panic
+	defer func() {
+		if r := recover(); r != nil {
+			log.Println("PANIC: pkg:", r, packet, string(debug.Stack()))
+		}
+	}()
+
+	// log.Println("Processing packet:", packet.Ack, packet.Seq, packet.ID)
 
 	var message *TCPMessage
-	m_id := packet.Addr.String() + strconv.Itoa(int(packet.Ack))
 
-	message, ok := t.messages[m_id]
+	isIncoming := packet.DestPort == t.port
+
+	// Seek for 100-expect chunks
+	if parentAck, ok := t.seqWithData[packet.Seq]; ok {
+		// In case if non-first data chunks comes first
+		for _, m := range t.messages {
+			if m.Ack == packet.Ack && bytes.Equal(m.packets[0].Addr, packet.Addr) {
+				t.deleteMessage(m)
+
+				if m.AssocMessage != nil {
+					m.setAssocMessage(nil)
+				}
+
+				for _, pkt := range m.packets {
+					// log.Println("Updating ack", parentAck, pkt.Ack)
+					pkt.UpdateAck(parentAck)
+					// Re-queue this packets
+					t.processTCPPacket(pkt)
+				}
+			}
+		}
+
+		t.ackAliases[packet.Ack] = parentAck
+		packet.UpdateAck(parentAck)
+	}
+
+	if alias, ok := t.ackAliases[packet.Ack]; ok {
+		packet.UpdateAck(alias)
+	}
+
+	var responseRequest *TCPMessage
+
+	if !isIncoming {
+		responseRequest, _ = t.respAliases[packet.Ack]
+	}
+
+	message, ok := t.messages[packet.ID]
 
 	if !ok {
-		// We sending c_del_message channel, so message object can communicate with Listener and notify it if message completed
-		message = NewTCPMessage(m_id, t.c_del_message)
-		t.messages[m_id] = message
+		message = NewTCPMessage(packet.Seq, packet.Ack, isIncoming)
+		t.messages[packet.ID] = message
+
+		if !isIncoming {
+			if responseRequest != nil {
+				message.setAssocMessage(responseRequest)
+				responseRequest.setAssocMessage(message)
+			} else {
+				t.respWithoutReq[packet.Ack] = packet.ID
+			}
+		}
 	}
 
 	// Adding packet to message
-	message.c_packets <- packet
+	message.AddPacket(packet)
+
+	// Handling Expect: 100-continue requests
+	if message.expectType == httpExpect100Continue && len(message.packets) == message.headerPacket+1 {
+		seq := packet.Seq + uint32(message.Size())
+		t.seqWithData[seq] = packet.Ack
+		message.DataSeq = seq
+		message.complete = false
+
+		// In case if sequence packet came first
+		for _, m := range t.messages {
+			if m.Seq == seq {
+				t.deleteMessage(m)
+				if m.AssocMessage != nil {
+					message.setAssocMessage(m.AssocMessage)
+				}
+				// log.Println("2: Adding ack alias:", m.Ack, packet.Ack)
+				t.ackAliases[m.Ack] = packet.Ack
+
+				for _, pkt := range m.packets {
+					pkt.UpdateAck(packet.Ack)
+					message.AddPacket(pkt)
+				}
+			}
+		}
+
+		// Removing `Expect: 100-continue` header
+		packet.Data = proto.DeleteHeader(packet.Data, bExpectHeader)
+	}
+
+	// log.Println("Received message:", string(message.Bytes()), message.ID(), t.messages)
+
+	if isIncoming {
+		// If message have multiple packets, delete previous alias
+		if len(message.packets) > 1 {
+			delete(t.respAliases, message.ResponseAck)
+		}
+
+		message.UpdateResponseAck()
+		t.respAliases[message.ResponseAck] = message
+	}
+
+	// If message contains only single packet immediately dispatch it
+	if message.complete {
+		if isIncoming {
+			if t.trackResponse {
+				if resp, ok := t.messages[message.ResponseID]; ok {
+					if resp.complete {
+						t.dispatchMessage(resp)
+					}
+
+					t.dispatchMessage(message)
+				}
+			} else {
+				t.dispatchMessage(message)
+			}
+		} else {
+			if message.AssocMessage == nil {
+				return
+			}
+
+			if req, ok := t.messages[message.AssocMessage.ID()]; ok {
+				if req.complete {
+					t.dispatchMessage(req)
+					t.dispatchMessage(message)
+				}
+			}
+		}
+	}
+}
+
+func (t *Listener) IsReady() bool {
+	select {
+	case <-t.readyCh:
+		return true
+	case <-time.After(5 * time.Second):
+		return false
+	}
 }
 
 // Receive TCP messages from the listener channel
-func (t *Listener) Receive() *TCPMessage {
-	return <-t.c_messages
+func (t *Listener) Receiver() chan *TCPMessage {
+	return t.messagesChan
+}
+
+func (t *Listener) Close() {
+	close(t.quit)
+	if t.conn != nil {
+		t.conn.Close()
+	}
+
+	for _, h := range t.pcapHandles {
+		h.Close()
+	}
+
+	return
 }

raw_socket_listener/listener_test.go

@@ -0,0 +1,477 @@
+package rawSocket
+
+import (
+	"bytes"
+	"log"
+	"math/rand"
+	"sync/atomic"
+	"testing"
+	"time"
+)
+
+func TestRawListenerInput(t *testing.T) {
+	var req, resp *TCPMessage
+
+	listener := NewListener("", "0", EnginePcap, true, 10*time.Millisecond)
+	defer listener.Close()
+
+	reqPacket := buildPacket(true, 1, 1, []byte("GET / HTTP/1.1\r\n\r\n"))
+
+	respAck := reqPacket.Seq + uint32(len(reqPacket.Data))
+	respPacket := buildPacket(false, respAck, reqPacket.Seq+1, []byte("HTTP/1.1 200 OK\r\n\r\n"))
+
+	listener.packetsChan <- reqPacket.Dump()
+	listener.packetsChan <- respPacket.Dump()
+
+	select {
+	case req = <-listener.messagesChan:
+	case <-time.After(time.Millisecond):
+		t.Error("Should return request immediately")
+		return
+	}
+
+	if !req.IsIncoming {
+		t.Error("Should be request")
+	}
+
+	select {
+	case resp = <-listener.messagesChan:
+	case <-time.After(20 * time.Millisecond):
+		t.Error("Should return response immediately")
+		return
+	}
+
+	if resp.IsIncoming {
+		t.Error("Should be response")
+	}
+}
+
+func TestRawListenerInputResponseByClose(t *testing.T) {
+	var req, resp *TCPMessage
+
+	listener := NewListener("", "0", EnginePcap, true, 10*time.Millisecond)
+	defer listener.Close()
+
+	reqPacket := buildPacket(true, 1, 1, []byte("GET / HTTP/1.1\r\n\r\n"))
+
+	respAck := reqPacket.Seq + uint32(len(reqPacket.Data))
+	respPacket := buildPacket(false, respAck, reqPacket.Seq+1, []byte("HTTP/1.1 200 OK\r\nConnection: close\r\n\r\nasd"))
+	finPacket := buildPacket(false, respAck, reqPacket.Seq+2, []byte(""))
+	finPacket.IsFIN = true
+
+	listener.packetsChan <- reqPacket.Dump()
+	listener.packetsChan <- respPacket.Dump()
+	listener.packetsChan <- finPacket.Dump()
+
+	select {
+	case req = <-listener.messagesChan:
+	case <-time.After(time.Millisecond):
+		t.Error("Should return request immediately")
+		return
+	}
+
+	if !req.IsIncoming {
+		t.Error("Should be request")
+	}
+
+	select {
+	case resp = <-listener.messagesChan:
+	case <-time.After(20 * time.Millisecond):
+		t.Error("Should return response immediately")
+		return
+	}
+
+	if resp.IsIncoming {
+		t.Error("Should be response")
+	}
+}
+
+func TestRawListenerInputWithoutResponse(t *testing.T) {
+	var req *TCPMessage
+
+	listener := NewListener("", "0", EnginePcap, false, 10*time.Millisecond)
+	defer listener.Close()
+
+	reqPacket := buildPacket(true, 1, 1, []byte("GET / HTTP/1.1\r\n\r\n"))
+
+	listener.packetsChan <- reqPacket.Dump()
+
+	select {
+	case req = <-listener.messagesChan:
+	case <-time.After(time.Millisecond):
+		t.Error("Should return request immediately")
+		return
+	}
+
+	if !req.IsIncoming {
+		t.Error("Should be request")
+	}
+}
+
+func TestRawListenerResponse(t *testing.T) {
+	var req, resp *TCPMessage
+
+	listener := NewListener("", "0", EnginePcap, true, 10*time.Millisecond)
+	defer listener.Close()
+
+	reqPacket := buildPacket(true, 1, 1, []byte("GET / HTTP/1.1\r\n\r\n"))
+	respPacket := buildPacket(false, 1+uint32(len(reqPacket.Data)), 2, []byte("HTTP/1.1 200 OK\r\n\r\n"))
+
+	// If response packet comes before request
+	listener.packetsChan <- respPacket.Dump()
+	listener.packetsChan <- reqPacket.Dump()
+
+	select {
+	case req = <-listener.messagesChan:
+	case <-time.After(time.Millisecond):
+		t.Error("Should return respose immediately")
+		return
+	}
+
+	if !req.IsIncoming {
+		t.Error("Should be request")
+	}
+
+	select {
+	case resp = <-listener.messagesChan:
+	case <-time.After(time.Millisecond):
+		t.Error("Should return response immediately")
+		return
+	}
+
+	if resp.IsIncoming {
+		t.Error("Should be response")
+	}
+
+	if !bytes.Equal(resp.UUID(), req.UUID()) {
+		t.Error("Resp and Req UUID should be equal")
+	}
+}
+
+func TestShort100Continue(t *testing.T) {
+	listener := NewListener("", "0", EnginePcap, true, 10*time.Millisecond)
+	defer listener.Close()
+
+	reqPacket1 := buildPacket(true, 1, 1, []byte("POST / HTTP/1.1\r\nContent-Length: 2\r\nExpect: 100-continue\r\n\r\n"))
+	// Packet with data have different Seq
+	reqPacket2 := buildPacket(true, 2, reqPacket1.Seq+uint32(len(reqPacket1.Data)), []byte("a"))
+	reqPacket3 := buildPacket(true, 2, reqPacket2.Seq+1, []byte("b"))
+
+	respPacket1 := buildPacket(false, 10, 3, []byte("HTTP/1.1 100 Continue\r\n\r\n"))
+
+	// panic(int(uint32(len(reqPacket1.Data)) + uint32(len(reqPacket2.Data)) + uint32(len(reqPacket3.Data))))
+	respPacket2 := buildPacket(false, reqPacket3.Seq+1 /* len of data */, 2, []byte("HTTP/1.1 200 OK\r\n\r\n"))
+
+	result := []byte("POST / HTTP/1.1\r\nContent-Length: 2\r\n\r\nab")
+
+	testRawListener100Continue(t, listener, result, reqPacket1, reqPacket2, reqPacket3, respPacket1, respPacket2)
+}
+
+// Response comes before Request
+func Test100ContinueWrongOrder(t *testing.T) {
+	listener := NewListener("", "0", EnginePcap, true, 10*time.Millisecond)
+	defer listener.Close()
+
+	reqPacket1 := buildPacket(true, 1, 1, []byte("POST / HTTP/1.1\r\nContent-Length: 2\r\nExpect: 100-continue\r\n\r\n"))
+	// Packet with data have different Seq
+	reqPacket2 := buildPacket(true, 2, reqPacket1.Seq+uint32(len(reqPacket1.Data)), []byte("a"))
+	reqPacket3 := buildPacket(true, 2, reqPacket2.Seq+1, []byte("b"))
+
+	respPacket1 := buildPacket(false, 10, 3, []byte("HTTP/1.1 100 Continue\r\n"))
+
+	// panic(int(uint32(len(reqPacket1.Data)) + uint32(len(reqPacket2.Data)) + uint32(len(reqPacket3.Data))))
+	respPacket2 := buildPacket(false, reqPacket3.Seq+1 /* len of data */, 2, []byte("HTTP/1.1 200 OK\r\n\r\n"))
+
+	result := []byte("POST / HTTP/1.1\r\nContent-Length: 2\r\n\r\nab")
+
+	testRawListener100Continue(t, listener, result, respPacket1, respPacket2, reqPacket1, reqPacket2, reqPacket3)
+}
+
+func TestAlt100ContinueHeaderOrder(t *testing.T) {
+	listener := NewListener("", "0", EnginePcap, true, 10*time.Millisecond)
+	defer listener.Close()
+
+	reqPacket1 := buildPacket(true, 1, 1, []byte("POST / HTTP/1.1\r\nExpect: 100-continue\r\nContent-Length: 2\r\n\r\n"))
+	// Packet with data have different Seq
+	reqPacket2 := buildPacket(true, 2, reqPacket1.Seq+uint32(len(reqPacket1.Data)), []byte("a"))
+	reqPacket3 := buildPacket(true, 2, reqPacket2.Seq+1, []byte("b"))
+
+	respPacket1 := buildPacket(false, 10, 3, []byte("HTTP/1.1 100 Continue\r\n"))
+
+	// panic(int(uint32(len(reqPacket1.Data)) + uint32(len(reqPacket2.Data)) + uint32(len(reqPacket3.Data))))
+	respPacket2 := buildPacket(false, reqPacket3.Seq+1 /* len of data */, 2, []byte("HTTP/1.1 200 OK\r\n\r\n"))
+
+	result := []byte("POST / HTTP/1.1\r\nContent-Length: 2\r\n\r\nab")
+
+	testRawListener100Continue(t, listener, result, reqPacket1, reqPacket2, reqPacket3, respPacket1, respPacket2)
+}
+
+func testRawListener100Continue(t *testing.T, listener *Listener, result []byte, packets ...*TCPPacket) {
+	var req, resp *TCPMessage
+	for _, p := range packets {
+		listener.packetsChan <- p.Dump()
+	}
+
+	select {
+	case req = <-listener.messagesChan:
+		break
+	case <-time.After(11 * time.Millisecond):
+		t.Error("Should return response after expire time")
+		return
+	}
+
+	if !bytes.Equal(req.Bytes(), result) {
+		t.Error("Should receive full message", string(req.Bytes()))
+	}
+
+	if !req.IsIncoming {
+		t.Error("Should be request")
+	}
+
+	select {
+	case resp = <-listener.messagesChan:
+		break
+	case <-time.After(21 * time.Millisecond):
+		t.Error("Should return response after expire time")
+		return
+	}
+
+	if resp.IsIncoming {
+		t.Error("Should be response")
+	}
+
+	if !bytes.Equal(resp.UUID(), req.UUID()) {
+		t.Error("Resp and Req UUID should be equal")
+	}
+}
+
+func testChunkedSequence(t *testing.T, listener *Listener, packets ...*TCPPacket) {
+	var r, req, resp *TCPMessage
+
+	for _, p := range packets {
+		listener.packetsChan <- p.Dump()
+	}
+
+	select {
+	case r = <-listener.messagesChan:
+		if r.IsIncoming {
+			req = r
+		} else {
+			resp = r
+		}
+		break
+	case <-time.After(25 * time.Millisecond):
+		t.Error("Should return request after expire time")
+		return
+	}
+	select {
+	case r = <-listener.messagesChan:
+		if r.IsIncoming {
+			if req != nil {
+				t.Error("Request already received", r)
+				return
+			}
+			req = r
+		} else {
+			if resp != nil {
+				t.Error("Response already received", r)
+				return
+			}
+			resp = r
+		}
+		break
+	case <-time.After(25 * time.Millisecond):
+		t.Error("Should return request after expire time")
+		return
+	}
+
+	if !bytes.Equal(req.Bytes(), []byte("POST / HTTP/1.1\r\nTransfer-Encoding: chunked\r\n\r\n1\r\na\r\n1\r\nb\r\n0\r\n\r\n")) {
+		t.Error("Should receive full message", string(req.Bytes()))
+	}
+
+	if !req.IsIncoming {
+		t.Error("Should be request")
+	}
+
+	if resp.IsIncoming {
+		t.Error("Should be response")
+	}
+
+	if !bytes.Equal(resp.UUID(), req.UUID()) {
+		t.Error("Resp and Req UUID should be equal", string(resp.UUID()), string(req.UUID()))
+	}
+
+	time.Sleep(20 * time.Millisecond)
+
+	if len(listener.packetsChan) != 0 {
+		t.Fatal("packetsChan non empty:", listener.packetsChan)
+	}
+
+	if len(listener.messagesChan) != 0 {
+		t.Fatal("messagesChan non empty:", <-listener.messagesChan)
+	}
+
+	if len(listener.messages) != 0 {
+		t.Fatal("Messages non empty:", listener.messages)
+	}
+
+	if len(listener.ackAliases) != 0 {
+		t.Fatal("ackAliases non empty:", listener.ackAliases)
+	}
+
+	if len(listener.seqWithData) != 0 {
+		t.Fatal("seqWithData non empty:", listener.seqWithData)
+	}
+
+	if len(listener.respAliases) != 0 {
+		t.Fatal("respAliases non empty:", listener.respAliases)
+	}
+
+	if len(listener.respWithoutReq) != 0 {
+		t.Fatal("respWithoutReq non empty:", listener.respWithoutReq)
+	}
+}
+
+func permutation(n int, list []*TCPPacket) []*TCPPacket {
+	if len(list) == 1 {
+		return list
+	}
+
+	k := n % len(list)
+
+	first := []*TCPPacket{list[k]}
+	next := make([]*TCPPacket, len(list)-1)
+
+	copy(next, append(list[:k], list[k+1:]...))
+
+	return append(first, permutation(n/len(list), next)...)
+}
+
+// Response comes before Request
+func TestRawListenerChunkedWrongOrder(t *testing.T) {
+	listener := NewListener("", "0", EnginePcap, true, 10*time.Millisecond)
+	defer listener.Close()
+
+	reqPacket1 := buildPacket(true, 1, 1, []byte("POST / HTTP/1.1\r\nTransfer-Encoding: chunked\r\nExpect: 100-continue\r\n\r\n"))
+	// Packet with data have different Seq
+	reqPacket2 := buildPacket(true, 2, reqPacket1.Seq+uint32(len(reqPacket1.Data)), []byte("1\r\na\r\n"))
+	reqPacket3 := buildPacket(true, 2, reqPacket2.Seq+uint32(len(reqPacket2.Data)), []byte("1\r\nb\r\n"))
+	reqPacket4 := buildPacket(true, 2, reqPacket3.Seq+uint32(len(reqPacket3.Data)), []byte("0\r\n\r\n"))
+
+	respPacket1 := buildPacket(false, 10, 3, []byte("HTTP/1.1 100 Continue\r\n\r\n"))
+
+	// panic(int(uint32(len(reqPacket1.Data)) + uint32(len(reqPacket2.Data)) + uint32(len(reqPacket3.Data))))
+	respPacket2 := buildPacket(false, reqPacket4.Seq+5 /* len of data */, 2, []byte("HTTP/1.1 200 OK\r\n\r\n"))
+
+	// Should re-construct message from all possible combinations
+	for i := 0; i < 6*5*4*3*2*1; i++ {
+
+		// if i < 54 || i > 57 {
+		// 	continue
+		// }
+
+		packets := permutation(i, []*TCPPacket{reqPacket1, reqPacket2, reqPacket3, reqPacket4, respPacket1, respPacket2})
+
+		t.Log("permutation:", i)
+		testChunkedSequence(t, listener, packets...)
+	}
+}
+
+func chunkedPostMessage() []*TCPPacket {
+	ack := uint32(rand.Int63())
+	seq := uint32(rand.Int63())
+
+	reqPacket1 := buildPacket(true, ack, seq, []byte("POST / HTTP/1.1\r\nTransfer-Encoding: chunked\r\n\r\n"))
+	// Packet with data have different Seq
+	reqPacket2 := buildPacket(true, ack, seq+47, []byte("1\r\na\r\n"))
+	reqPacket3 := buildPacket(true, ack, reqPacket2.Seq+5, []byte("1\r\nb\r\n"))
+	reqPacket4 := buildPacket(true, ack, reqPacket3.Seq+5, []byte("0\r\n\r\n"))
+
+	respPacket := buildPacket(false, reqPacket4.Seq+5 /* len of data */, ack, []byte("HTTP/1.1 200 OK\r\n\r\n"))
+
+	return []*TCPPacket{
+		reqPacket1, reqPacket2, reqPacket3, reqPacket4, respPacket,
+	}
+}
+
+func postMessage() []*TCPPacket {
+	ack := uint32(rand.Int63())
+	seq2 := uint32(rand.Int63())
+	seq := uint32(rand.Int63())
+
+	c := 10000
+	data := make([]byte, c)
+	rand.Read(data)
+
+	head := []byte("POST / HTTP/1.1\r\nContent-Length: 9958\r\n\r\n")
+	for i := range head {
+		data[i] = head[i]
+	}
+
+	return []*TCPPacket{
+		buildPacket(true, ack, seq, data),
+		buildPacket(false, seq+uint32(len(data)), seq2, []byte("HTTP/1.1 200 OK\r\n\r\n")),
+	}
+}
+
+func getMessage() []*TCPPacket {
+	ack := uint32(rand.Int63())
+	seq2 := uint32(rand.Int63())
+	seq := uint32(rand.Int63())
+
+	return []*TCPPacket{
+		buildPacket(true, ack, seq, []byte("GET / HTTP/1.1\r\n\r\n")),
+		buildPacket(false, seq+18, seq2, []byte("HTTP/1.1 200 OK\r\n\r\n")),
+	}
+}
+
+// Response comes before Request
+func TestRawListenerBench(t *testing.T) {
+	l := NewListener("", "0", EnginePcap, true, 200*time.Millisecond)
+	defer l.Close()
+
+	// Should re-construct message from all possible combinations
+	for i := 0; i < 1000; i++ {
+		go func(i int) {
+			for j := 0; j < 100; j++ {
+				var packets []*TCPPacket
+
+				if j%5 == 0 {
+					packets = chunkedPostMessage()
+				} else if j%3 == 0 {
+					packets = postMessage()
+				} else {
+					packets = getMessage()
+				}
+
+				for _, p := range packets {
+					// Randomly drop packets
+					if (i+j)%5 == 0 {
+						if rand.Int63()%3 == 0 {
+							continue
+						}
+					}
+
+					l.packetsChan <- p.Dump()
+					time.Sleep(time.Millisecond)
+				}
+
+				time.Sleep(5 * time.Millisecond)
+			}
+		}(i)
+	}
+
+	ch := l.Receiver()
+
+	var count int32
+
+	for {
+		select {
+		case <-ch:
+			atomic.AddInt32(&count, 1)
+		case <-time.After(2000 * time.Millisecond):
+			log.Println("Emitted 200000 messages, captured: ", count, len(l.ackAliases), len(l.seqWithData), len(l.respAliases), len(l.respWithoutReq), len(l.packetsChan))
+			return
+		}
+	}
+}

raw_socket_listener/tcp_message.go

@@ -1,12 +1,18 @@
-package raw_socket
+package rawSocket
 
 import (
+	"bytes"
+	"crypto/sha1"
+	"encoding/binary"
+	"encoding/hex"
+	"github.com/buger/gor/proto"
 	"log"
-	"sort"
+	"net"
+	"strconv"
 	"time"
 )
 
-const MSG_EXPIRE = 2000 * time.Millisecond
+var _ = log.Println
 
 // TCPMessage ensure that all TCP packets for given request is received, and processed in right sequence
 // Its needed because all TCP message can be fragmented or re-transmitted
@@ -15,57 +21,81 @@ const MSG_EXPIRE = 2000 * time.Millisecond
 // Message can be compiled from unique packets with same message_id which sorted by sequence
 // Message is received if we didn't receive any packets for 2000ms
 type TCPMessage struct {
-	ID      string // Message ID
+	Seq         uint32
+	Ack         uint32
+	ResponseAck uint32
+	ResponseID  tcpID
+	DataAck     uint32
+	DataSeq     uint32
+
+	AssocMessage *TCPMessage
+	Start        time.Time
+	End          time.Time
+	IsIncoming   bool
+
 	packets []*TCPPacket
 
-	timer *time.Timer // Used for expire check
+	delChan chan *TCPMessage
 
-	c_packets chan *TCPPacket
-
-	c_del_message chan *TCPMessage
+	/* HTTP specific variables */
+	methodType    httpMethodType
+	bodyType      httpBodyType
+	expectType    httpExpectType
+	seqMissing    bool
+	headerPacket  int
+	contentLength int
+	complete      bool
 }
 
 // NewTCPMessage pointer created from a Acknowledgment number and a channel of messages readuy to be deleted
-func NewTCPMessage(ID string, c_del chan *TCPMessage) (msg *TCPMessage) {
-	msg = &TCPMessage{ID: ID}
-
-	msg.c_packets = make(chan *TCPPacket)
-	msg.c_del_message = c_del // used for notifying that message completed or expired
-
-	// Every time we receive packet we reset this timer
-	msg.timer = time.AfterFunc(MSG_EXPIRE, msg.Timeout)
-
-	go msg.listen()
+func NewTCPMessage(Seq, Ack uint32, IsIncoming bool) (msg *TCPMessage) {
+	msg = &TCPMessage{Seq: Seq, Ack: Ack, IsIncoming: IsIncoming}
+	msg.Start = time.Now()
 
 	return
 }
 
-func (t *TCPMessage) listen() {
-	for {
-		select {
-		case packet, more := <-t.c_packets:
-			if more {
-				t.AddPacket(packet)
-			} else {
-				// Stop loop if channel closed
-				return
-			}
-		}
+func (t *TCPMessage) packetsData() (d [][]byte) {
+	d = make([][]byte, len(t.packets))
+	for i, p := range t.packets {
+		d[i] = p.Data
 	}
+
+	return
 }
 
-// Timeout notifies message to stop listening, close channel and message ready to be sent
-func (t *TCPMessage) Timeout() {
-	close(t.c_packets)   // Notify to stop listen loop and close channel
-	t.c_del_message <- t // Notify RAWListener that message is ready to be send to replay server
-}
-
-// Bytes sorts packets in right orders and return message content
+// Bytes return message content
 func (t *TCPMessage) Bytes() (output []byte) {
-	sort.Sort(BySeq(t.packets))
+	for _, p := range t.packets {
+		output = append(output, p.Data...)
+	}
 
-	for _, v := range t.packets {
-		output = append(output, v.Data...)
+	return output
+}
+
+// Size returns total body size
+func (t *TCPMessage) BodySize() (size int) {
+	if len(t.packets) == 0 || t.headerPacket == -1 {
+		return 0
+	}
+
+	size += len(proto.Body(t.packets[t.headerPacket].Data))
+
+	for _, p := range t.packets[t.headerPacket+1:] {
+		size += len(p.Data)
+	}
+
+	return
+}
+
+// Size returns total size of message
+func (t *TCPMessage) Size() (size int) {
+	if len(t.packets) == 0 {
+		return 0
+	}
+
+	for _, p := range t.packets {
+		size += len(p.Data)
 	}
 
 	return
@@ -83,12 +113,393 @@ func (t *TCPMessage) AddPacket(packet *TCPPacket) {
 		}
 	}
 
-	if packetFound {
-		log.Println("Received packet with same sequence")
-	} else {
-		t.packets = append(t.packets, packet)
+	if !packetFound {
+		// Packets not always captured in same Seq order, and sometimes we need to prepend
+		if len(t.packets) == 0 || packet.Seq > t.packets[len(t.packets)-1].Seq {
+			t.packets = append(t.packets, packet)
+		} else if packet.Seq < t.packets[0].Seq {
+			t.packets = append([]*TCPPacket{packet}, t.packets...)
+			t.Seq = packet.Seq // Message Seq should indicated starting seq
+		} else { // insert somewhere in the middle...
+			for i, p := range t.packets {
+				if packet.Seq < p.Seq {
+					t.packets = append(t.packets[:i], append([]*TCPPacket{packet}, t.packets[i:]...)...)
+					break
+				}
+			}
+		}
+
+		if t.IsIncoming {
+			t.End = time.Now()
+		} else {
+			t.End = time.Now().Add(time.Millisecond)
+		}
+
+		if packet.OrigAck != 0 {
+			t.DataAck = packet.OrigAck
+		}
 	}
 
-	// Reset message timeout timer
-	t.timer.Reset(MSG_EXPIRE)
+	t.checkSeqIntegrity()
+	t.updateHeadersPacket()
+	t.updateMethodType()
+	t.updateBodyType()
+	t.checkIfComplete()
+	t.check100Continue()
+}
+
+// Check if there is missing packet
+func (t *TCPMessage) checkSeqIntegrity() {
+	if len(t.packets) == 1 {
+		t.seqMissing = false
+	}
+
+	offset := len(t.packets) - 1
+
+	if t.packets[offset].IsFIN {
+		offset--
+
+		if offset < 0 {
+			return
+		}
+	}
+
+	for i, p := range t.packets[:offset] {
+		if p.IsFIN {
+			continue
+		}
+
+		// If final packet
+		if len(t.packets) == i+1 {
+			t.seqMissing = false
+			return
+		}
+		np := t.packets[i+1]
+
+		nextSeq := p.Seq + uint32(len(p.Data))
+
+		if np.Seq != nextSeq {
+			if t.expectType == httpExpect100Continue {
+				if np.Seq != nextSeq+22 {
+					t.seqMissing = true
+					return
+				}
+			} else {
+				t.seqMissing = true
+				return
+			}
+		}
+	}
+
+	t.seqMissing = false
+}
+
+var bEmptyLine = []byte("\r\n\r\n")
+var bChunkEnd = []byte("0\r\n\r\n")
+
+func (t *TCPMessage) updateHeadersPacket() {
+	if len(t.packets) == 1 {
+		t.headerPacket = -1
+	}
+
+	if t.headerPacket != -1 {
+		return
+	}
+
+	if t.seqMissing {
+		return
+	}
+
+	for i, p := range t.packets {
+		if bytes.LastIndex(p.Data, bEmptyLine) != -1 {
+			t.headerPacket = i
+			return
+		}
+	}
+
+	return
+}
+
+// isMultipart returns true if message contains from multiple tcp packets
+func (t *TCPMessage) checkIfComplete() {
+	if t.seqMissing || t.headerPacket == -1 {
+		return
+	}
+
+	if t.methodType == httpMethodNotFound {
+		return
+	}
+
+	// Responses can be emitted only if we found request
+	if !t.IsIncoming && t.AssocMessage == nil {
+		return
+	}
+
+	// If one GET, OPTIONS, or HEAD request
+	if t.methodType == httpMethodWithoutBody {
+		t.complete = true
+	} else {
+		switch t.bodyType {
+		case httpBodyEmpty:
+			t.complete = true
+		case httpBodyContentLength:
+			if t.contentLength == 0 || t.contentLength == t.BodySize() {
+				t.complete = true
+			}
+		case httpBodyChunked:
+			lastPacket := t.packets[len(t.packets)-1]
+			if bytes.LastIndex(lastPacket.Data, bChunkEnd) != -1 {
+				t.complete = true
+			}
+		default:
+			if len(t.packets) == 0 {
+				return
+			}
+
+			last := t.packets[len(t.packets)-1]
+			if last.IsFIN {
+				t.complete = true
+			}
+		}
+	}
+}
+
+type httpMethodType uint8
+
+const (
+	httpMethodNotSet      httpMethodType = 0
+	httpMethodWithBody    httpMethodType = 1
+	httpMethodWithoutBody httpMethodType = 2
+	httpMethodNotFound    httpMethodType = 3
+)
+
+var methodsWithBody = [][]byte{
+	[]byte("POST"),
+	[]byte("PUT"),
+	[]byte("PATCH"),
+	[]byte("CONNECT"),
+}
+
+func (t *TCPMessage) updateMethodType() {
+	// if there is cache
+	if t.methodType != httpMethodNotSet && t.methodType != httpMethodNotFound {
+		return
+	}
+
+	d := t.packets[0].Data
+
+	// Minimum length fo request: GET / HTTP/1.1\r\n
+
+	if len(d) < 16 {
+		t.methodType = httpMethodNotFound
+		return
+	}
+
+	if t.IsIncoming {
+		var method []byte
+
+		if mIdx := bytes.IndexByte(d[:8], ' '); mIdx != -1 {
+			method = d[:mIdx]
+
+			// Check that after method we have absolute or relative path
+			switch d[mIdx+1] {
+			case '/', 'h', '*':
+			default:
+				t.methodType = httpMethodNotFound
+				return
+			}
+		} else {
+			t.methodType = httpMethodNotFound
+			return
+		}
+
+		for _, m := range methodsWithBody {
+			if len(m) == len(method) && bytes.Equal(m, method) {
+				t.methodType = httpMethodWithBody
+				return
+			}
+		}
+
+		t.methodType = httpMethodWithoutBody
+	} else {
+		if !bytes.Equal(d[:6], []byte("HTTP/1")) {
+			t.methodType = httpMethodNotFound
+			return
+		}
+
+		t.methodType = httpMethodWithBody
+	}
+}
+
+type httpBodyType uint8
+
+const (
+	httpBodyNotSet          httpBodyType = 0
+	httpBodyEmpty           httpBodyType = 1
+	httpBodyContentLength   httpBodyType = 2
+	httpBodyChunked         httpBodyType = 3
+	httpBodyConnectionClose httpBodyType = 4
+)
+
+func (t *TCPMessage) updateBodyType() {
+	// if there is cache
+	if t.bodyType != httpBodyNotSet {
+		return
+	}
+
+	// Headers not received
+	if t.headerPacket == -1 {
+		return
+	}
+
+	switch t.methodType {
+	case httpMethodNotFound:
+		return
+	case httpMethodWithoutBody:
+		t.bodyType = httpBodyEmpty
+		return
+	case httpMethodWithBody:
+		var lengthB, encB, connB []byte
+
+		proto.ParseHeaders(t.packetsData(), func(header, value []byte)bool{
+			if proto.HeadersEqual(header, []byte("Content-Length")) {
+				lengthB = value
+				return false
+			}
+
+			if proto.HeadersEqual(header, []byte("Transfer-Encoding")) {
+				encB = value
+				return false
+			}
+
+			if proto.HeadersEqual(header, []byte("Connection")) {
+				connB = value
+				return false
+			}
+
+			return true
+		})
+
+		if len(lengthB) > 0 {
+			t.contentLength, _ = strconv.Atoi(string(lengthB))
+
+			if t.contentLength == 0 {
+				t.bodyType = httpBodyEmpty
+			} else {
+				t.bodyType = httpBodyContentLength
+			}
+			return
+		}
+
+		if len(encB) > 0 {
+			t.bodyType = httpBodyChunked
+			return
+		}
+
+		if len(connB) > 0 && bytes.Equal(connB, []byte("close")) {
+			t.bodyType = httpBodyConnectionClose
+			return
+		}
+	}
+
+	t.bodyType = httpBodyEmpty
+}
+
+type httpExpectType uint8
+
+const (
+	httpExpectNotSet      httpExpectType = 0
+	httpExpectEmpty       httpExpectType = 1
+	httpExpect100Continue httpExpectType = 2
+)
+
+var bExpectHeader = []byte("Expect")
+var bExpect100Value = []byte("100-continue")
+
+func (t *TCPMessage) check100Continue() {
+	if t.expectType != httpExpectNotSet || len(t.packets[0].Data) < 25 {
+		return
+	}
+
+	if t.methodType != httpMethodWithBody {
+		return
+	}
+
+	if t.seqMissing || t.headerPacket == -1 {
+		return
+	}
+
+	last := t.packets[len(t.packets)-1]
+	// reading last 4 bytes for double CRLF
+	if !bytes.HasSuffix(last.Data, bEmptyLine) {
+		return
+	}
+
+	var expectB []byte
+	proto.ParseHeaders(t.packetsData(), func(header, value []byte)bool{
+		if proto.HeadersEqual(header, bExpectHeader) {
+			expectB = value
+			return false
+		}
+
+		return true
+	})
+
+	if len(expectB) > 0 && bytes.Equal(bExpect100Value, expectB) {
+		t.expectType = httpExpect100Continue
+		return
+	}
+
+	t.expectType = httpExpectEmpty
+}
+
+func (t *TCPMessage) setAssocMessage(m *TCPMessage) {
+	t.AssocMessage = m
+	t.checkIfComplete()
+}
+
+// UpdateResponseAck should be called after packet is added
+func (t *TCPMessage) UpdateResponseAck() uint32 {
+	lastPacket := t.packets[len(t.packets)-1]
+	respAck := lastPacket.Seq + uint32(len(lastPacket.Data))
+
+	if t.ResponseAck != respAck {
+		t.ResponseAck = lastPacket.Seq + uint32(len(lastPacket.Data))
+
+		// We swappwed src and dst port
+		copy(t.ResponseID[:16], lastPacket.Addr)
+		copy(t.ResponseID[16:], lastPacket.Raw[2:4]) // Src port
+		copy(t.ResponseID[18:], lastPacket.Raw[0:2]) // Dest port
+		binary.BigEndian.PutUint32(t.ResponseID[20:24], t.ResponseAck)
+	}
+
+	return t.ResponseAck
+}
+
+func (t *TCPMessage) UUID() []byte {
+	var key []byte
+
+	if t.IsIncoming {
+		// log.Println("UUID:", t.Ack, t.Start.UnixNano())
+		key = strconv.AppendInt(key, t.Start.UnixNano(), 10)
+		key = strconv.AppendUint(key, uint64(t.Ack), 10)
+	} else {
+		// log.Println("RequestMessage:", t.AssocMessage.Ack, t.AssocMessage.Start.UnixNano())
+		key = strconv.AppendInt(key, t.AssocMessage.Start.UnixNano(), 10)
+		key = strconv.AppendUint(key, uint64(t.AssocMessage.Ack), 10)
+	}
+
+	uuid := make([]byte, 40)
+	sha := sha1.Sum(key)
+	hex.Encode(uuid, sha[:20])
+
+	return uuid
+}
+
+func (t *TCPMessage) ID() tcpID {
+	return t.packets[0].ID
+}
+
+func (t *TCPMessage) IP() net.IP {
+	return net.IP(t.packets[0].Addr)
 }

raw_socket_listener/tcp_message_test.go

@@ -0,0 +1,245 @@
+package rawSocket
+
+import (
+	"bytes"
+	"encoding/binary"
+	_ "log"
+	"testing"
+)
+
+func buildPacket(isIncoming bool, Ack, Seq uint32, Data []byte) (packet *TCPPacket) {
+	var srcPort, destPort uint16
+
+	// For tests `listening` port is 0
+	if isIncoming {
+		srcPort = 1
+	} else {
+		destPort = 1
+	}
+
+	buf := make([]byte, 16)
+	binary.BigEndian.PutUint16(buf[2:4], destPort)
+	binary.BigEndian.PutUint16(buf[0:2], srcPort)
+	binary.BigEndian.PutUint32(buf[4:8], Seq)
+	binary.BigEndian.PutUint32(buf[8:12], Ack)
+	buf[12] = 64
+	buf = append(buf, Data...)
+
+	packet = ParseTCPPacket([]byte("123"), buf)
+
+	return packet
+}
+
+func buildMessage(p *TCPPacket) *TCPMessage {
+	isIncoming := false
+	if p.SrcPort == 1 {
+		isIncoming = true
+	}
+
+	m := NewTCPMessage(p.Seq, p.Ack, isIncoming)
+	m.AddPacket(p)
+
+	return m
+}
+
+func TestTCPMessagePacketsOrder(t *testing.T) {
+	msg := buildMessage(buildPacket(true, 1, 1, []byte("a")))
+	msg.AddPacket(buildPacket(true, 1, 2, []byte("b")))
+
+	if !bytes.Equal(msg.Bytes(), []byte("ab")) {
+		t.Error("Should contatenate packets in right order")
+	}
+
+	// When first packet have wrong order (Seq)
+	msg = buildMessage(buildPacket(true, 1, 2, []byte("b")))
+	msg.AddPacket(buildPacket(true, 1, 1, []byte("a")))
+
+	if !bytes.Equal(msg.Bytes(), []byte("ab")) {
+		t.Error("Should contatenate packets in right order")
+	}
+
+	// Should ignore packets with same sequence
+	msg = buildMessage(buildPacket(true, 1, 1, []byte("a")))
+	msg.AddPacket(buildPacket(true, 1, 1, []byte("a")))
+
+	if !bytes.Equal(msg.Bytes(), []byte("a")) {
+		t.Error("Should ignore packet with same Seq")
+	}
+}
+
+func TestTCPMessageSize(t *testing.T) {
+	msg := buildMessage(buildPacket(true, 1, 1, []byte("POST / HTTP/1.1\r\nContent-Length: 2\r\n\r\na")))
+	msg.AddPacket(buildPacket(true, 1, 2, []byte("b")))
+
+	if msg.BodySize() != 2 {
+		t.Error("Should count only body", msg.BodySize())
+	}
+
+	if msg.Size() != 40 {
+		t.Error("Should count all sizes", msg.Size())
+	}
+}
+
+func TestTCPMessageIsComplete(t *testing.T) {
+	testCases := []struct {
+		direction         bool
+		payload           string
+		assocMessage      bool
+		expectedCompleted bool
+	}{
+		{true, "GET / HTTP/1.1\r\n\r\n", false, true},
+		{true, "HEAD / HTTP/1.1\r\n\r\n", false, true},
+		{false, "HTTP/1.1 200 OK\r\n\r\n", true, true},
+		{true, "POST / HTTP/1.1\r\nContent-Length: 1\r\n\r\na", false, true},
+		{true, "PUT / HTTP/1.1\r\nContent-Length: 1\r\n\r\na", false, true},
+		{false, "HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n", true, true},
+		{false, "HTTP/1.1 200 OK\r\nContent-Length: 1\r\n\r\na", true, true},
+		{false, "HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n0\r\n\r\n", true, true},
+
+		// chunked not finished
+		{false, "HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n", true, false},
+
+		// content-length != actual length
+		{true, "POST / HTTP/1.1\r\nContent-Length: 2\r\n\r\na", false, false},
+		{false, "HTTP/1.1 200 OK\r\nContent-Length: 10\r\n\r\na", true, false},
+		// non-valid http request
+		{true, "UNKNOWN asd HTTP/1.1\r\n\r\n", false, false},
+
+		// response without associated request
+		{false, "HTTP/1.1 200 OK\r\n\r\n", false, false},
+	}
+
+	for _, tc := range testCases {
+		msg := buildMessage(buildPacket(tc.direction, 1, 1, []byte(tc.payload)))
+		if tc.assocMessage {
+			msg.AssocMessage = &TCPMessage{}
+		}
+		msg.checkIfComplete()
+
+		if msg.complete != tc.expectedCompleted {
+			t.Errorf("Payload %s: Expected %t, got %t.", tc.payload, tc.expectedCompleted, msg.complete)
+		}
+	}
+}
+
+func TestTCPMessageIsSeqMissing(t *testing.T) {
+	p1 := buildPacket(false, 1, 1, []byte("HTTP/1.1 200 OK\r\n"))
+	p2 := buildPacket(false, 1, p1.Seq+uint32(len(p1.Data)), []byte("Content-Length: 10\r\n\r\n"))
+	p3 := buildPacket(false, 1, p2.Seq+uint32(len(p2.Data)), []byte("a"))
+
+	msg := buildMessage(p1)
+	if msg.seqMissing {
+		t.Error("Should be complete if have only 1 packet")
+	}
+
+	msg.AddPacket(p3)
+	if !msg.seqMissing {
+		t.Error("Should be incomplete because missing middle component")
+	}
+
+	msg.AddPacket(p2)
+	if msg.seqMissing {
+		t.Error("Should be complete once missing packet added")
+	}
+}
+
+func TestTCPMessageIsHeadersReceived(t *testing.T) {
+	p1 := buildPacket(false, 1, 1, []byte("HTTP/1.1 200 OK\r\n\r\n"))
+	p2 := buildPacket(false, 1, p1.Seq+uint32(len(p1.Data)), []byte("Content-Length: 10\r\n\r\n"))
+
+	msg := buildMessage(p1)
+	if msg.headerPacket == -1 {
+		t.Error("Should be complete if have only 1 packet", msg.headerPacket)
+	}
+
+	msg.AddPacket(p2)
+	if msg.headerPacket == -1 {
+		t.Error("Should found double new line: headers received")
+	}
+
+	msg = buildMessage(buildPacket(true, 1, 1, []byte("GET / HTTP/1.1\r\nContent-Length: 1\r\n")))
+	if msg.headerPacket != -1 {
+		t.Error("Should not find headers end")
+	}
+}
+
+func TestTCPMessageMethodType(t *testing.T) {
+	testCases := []struct {
+		direction          bool
+		payload            string
+		expectedMethodType httpMethodType
+	}{
+		{true, "GET / HTTP/1.1\r\n\r\n", httpMethodWithoutBody},
+		{true, "GET * HTTP/1.1\r\n\r\n", httpMethodWithoutBody},
+		{true, "UNKNOWN / HTTP/1.1\r\n\r\n", httpMethodWithoutBody},
+		{true, "GET http://example.com HTTP/1.1\r\n\r\n", httpMethodWithoutBody},
+		{true, "POST / HTTP/1.1\r\n\r\n", httpMethodWithBody},
+		{true, "PUT / HTTP/1.1\r\n\r\n", httpMethodWithBody},
+		{true, "GET zxc HTTP/1.1\r\n\r\n", httpMethodNotFound},
+		{true, "GET / HTTP\r\n\r\n", httpMethodNotFound},
+		{true, "VERYLONGMETHOD / HTTP/1.1\r\n\r\n", httpMethodNotFound},
+		{false, "HTTP/1.1 200 OK\r\n\r\n", httpMethodWithBody},
+		{false, "HTTP /1.1 200 OK\r\n\r\n", httpMethodNotFound},
+	}
+
+	for _, tc := range testCases {
+		msg := buildMessage(buildPacket(tc.direction, 1, 1, []byte(tc.payload)))
+
+		if msg.methodType != tc.expectedMethodType {
+			t.Errorf("Expected %d, got %d", tc.expectedMethodType, msg.methodType)
+		}
+	}
+}
+
+func TestTCPMessageBodyType(t *testing.T) {
+	testCases := []struct {
+		direction        bool
+		payload          string
+		expectedBodyType httpBodyType
+	}{
+		{true, "GET / HTTP/1.1\r\n\r\n", httpBodyEmpty},
+		{true, "POST / HTTP/1.1\r\n\r\n", httpBodyEmpty},
+		{true, "POST / HTTP/1.1\r\nUser-Agent: zxc\r\n\r\n", httpBodyEmpty},
+		{false, "HTTP/1.1 200 OK\r\n\r\n", httpBodyEmpty},
+		{true, "POST / HTTP/1.1\r\nContent-Length: 2\r\n\r\nab", httpBodyContentLength},
+		{false, "HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nab", httpBodyContentLength},
+		{true, "POST / HTTP/1.1\r\nTransfer-Encoding: chunked\r\n\r\n2\r\nab\r\n0\r\n\r\n", httpBodyChunked},
+		{false, "HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n2\r\nab\r\n0\r\n\r\n", httpBodyChunked},
+	}
+
+	for _, tc := range testCases {
+		msg := buildMessage(buildPacket(tc.direction, 1, 1, []byte(tc.payload)))
+
+		if msg.bodyType != tc.expectedBodyType {
+			t.Errorf("Expected %d, got %d", tc.expectedBodyType, msg.bodyType)
+		}
+	}
+}
+
+func TestTCPMessageBodySize(t *testing.T) {
+	testCases := []struct {
+		direction    bool
+		payloads     []string
+		expectedSize int
+	}{
+		{true, []string{"GET / HTTP/1.1\r\n\r\n"}, 0},
+		{true, []string{"POST / HTTP/1.1\r\nContent-Length: 2\r\n\r\nab"}, 2},
+		{true, []string{"GET / HTTP/1.1\r\n", "Content-Length: 2\r\n\r\nab"}, 2},
+		{true, []string{"GET / HTTP/1.1\r\n", "Content-Length: 2\r\n\r\n", "ab"}, 2},
+	}
+
+	for _, tc := range testCases {
+		msg := buildMessage(buildPacket(tc.direction, 1, 1, []byte(tc.payloads[0])))
+
+		if len(tc.payloads) > 1 {
+			for _, p := range tc.payloads[1:] {
+				seq := uint32(1 + msg.Size())
+				msg.AddPacket(buildPacket(tc.direction, 1, seq, []byte(p)))
+			}
+		}
+
+		if msg.BodySize() != tc.expectedSize {
+			t.Errorf("Expected %d, got %d", tc.expectedSize, msg.BodySize())
+		}
+	}
+}

raw_socket_listener/tcp_packet.go

@@ -1,100 +1,124 @@
-package raw_socket
+package rawSocket
 
 import (
 	"encoding/binary"
-	"net"
+	"log"
 	"strconv"
 	"strings"
 )
 
+var _ = log.Println
+
 // TCP Flags
 const (
-	TCP_FIN = 1 << iota
-	TCP_SYN
-	TCP_RST
-	TCP_PSH
-	TCP_ACK
-	TCP_URG
-	TCP_ECE
-	TCP_CWR
-	TCP_NS
+	fFIN = 1 << iota
+	fSYN
+	fRST
+	fPSH
+	fACK
+	fURG
+	fECE
+	fCWR
+	fNS
 )
 
-// Simple TCP packet parser
-//
+type tcpID [24]byte
+
+// TCPPacket provides tcp packet parser
 // Packet structure: http://en.wikipedia.org/wiki/Transmission_Control_Protocol
 type TCPPacket struct {
 	SrcPort    uint16
 	DestPort   uint16
 	Seq        uint32
 	Ack        uint32
+	OrigAck    uint32
 	DataOffset uint8
-	Flags      uint16
-	Window     uint16
-	Checksum   uint16
-	Urgent     uint16
+	IsFIN      bool
 
+	Raw  []byte
 	Data []byte
-
-	Addr net.Addr
+	Addr []byte
+	ID   tcpID
 }
 
-func ParseTCPPacket(addr net.Addr, b []byte) (p *TCPPacket) {
-	p = &TCPPacket{Data: b}
+// ParseTCPPacket takes address and tcp payload and returns parsed TCPPacket
+func ParseTCPPacket(addr []byte, data []byte) (p *TCPPacket) {
+	p = &TCPPacket{Raw: data}
 	p.ParseBasic()
 	p.Addr = addr
+	p.GenID()
 
-	return p
+	return
 }
 
-// Parse TCP Packet, inspired by: https://github.com/miekg/pcap/blob/master/packet.go
-func (t *TCPPacket) Parse() {
-	t.ParseBasic()
-	t.SrcPort = binary.BigEndian.Uint16(t.Data[0:2])
-	t.DestPort = binary.BigEndian.Uint16(t.Data[2:4])
-	t.Flags = binary.BigEndian.Uint16(t.Data[12:14]) & 0x1FF
-	t.Window = binary.BigEndian.Uint16(t.Data[14:16])
-	t.Checksum = binary.BigEndian.Uint16(t.Data[16:18])
-	t.Urgent = binary.BigEndian.Uint16(t.Data[18:20])
+func (p *TCPPacket) GenID() {
+	copy(p.ID[:16], p.Addr)
+	copy(p.ID[16:], p.Raw[0:2])  // Src port
+	copy(p.ID[18:], p.Raw[2:4])  // Dest port
+	copy(p.ID[20:], p.Raw[8:12]) // Ack
+}
+
+func (p *TCPPacket) UpdateAck(ack uint32) {
+	p.OrigAck = p.Ack
+	p.Ack = ack
+	binary.BigEndian.PutUint32(p.Raw[8:12], ack)
+	p.GenID()
 }
 
 // ParseBasic set of fields
 func (t *TCPPacket) ParseBasic() {
-	t.Seq = binary.BigEndian.Uint32(t.Data[4:8])
-	t.Ack = binary.BigEndian.Uint32(t.Data[8:12])
-	t.DataOffset = (t.Data[12] & 0xF0) >> 4
+	t.DestPort = binary.BigEndian.Uint16(t.Raw[2:4])
+	t.SrcPort = binary.BigEndian.Uint16(t.Raw[0:2])
+	t.Seq = binary.BigEndian.Uint32(t.Raw[4:8])
+	t.Ack = binary.BigEndian.Uint32(t.Raw[8:12])
+	t.DataOffset = (t.Raw[12] & 0xF0) >> 4
+	t.IsFIN = t.Raw[13]&0x01 != 0
 
-	t.Data = t.Data[t.DataOffset*4:]
+	// log.Println("DataOffset:", t.DataOffset, t.DestPort, t.SrcPort, t.Seq, t.Ack)
+
+	t.Data = t.Raw[t.DataOffset*4:]
+}
+
+func (t *TCPPacket) Dump() []byte {
+	buf := make([]byte, len(t.Data)+16+16)
+	copy(buf[:16], t.Addr)
+
+	tcpBuf := buf[16:]
+
+	binary.BigEndian.PutUint16(tcpBuf[2:4], t.DestPort)
+	binary.BigEndian.PutUint16(tcpBuf[0:2], t.SrcPort)
+
+	binary.BigEndian.PutUint32(tcpBuf[4:8], t.Seq)
+	binary.BigEndian.PutUint32(tcpBuf[8:12], t.Ack)
+
+	tcpBuf[12] = 64
+
+	if t.IsFIN {
+		tcpBuf[13] = tcpBuf[13] | 0x01
+	}
+
+	copy(tcpBuf[16:], t.Data)
+
+	return buf
 }
 
 // String output for a TCP Packet
 func (t *TCPPacket) String() string {
+	maxLen := len(t.Data)
+	if maxLen > 200 {
+		maxLen = 200
+	}
+
 	return strings.Join([]string{
+		"Addr: " + string(t.Addr),
 		"Source port: " + strconv.Itoa(int(t.SrcPort)),
 		"Dest port:" + strconv.Itoa(int(t.DestPort)),
 		"Sequence:" + strconv.Itoa(int(t.Seq)),
 		"Acknowledgment:" + strconv.Itoa(int(t.Ack)),
 		"Header len:" + strconv.Itoa(int(t.DataOffset)),
+		"FIN:" + strconv.FormatBool(t.IsFIN),
 
-		"Flag ns:" + strconv.FormatBool(t.Flags&TCP_NS != 0),
-		"Flag crw:" + strconv.FormatBool(t.Flags&TCP_CWR != 0),
-		"Flag ece:" + strconv.FormatBool(t.Flags&TCP_ECE != 0),
-		"Flag urg:" + strconv.FormatBool(t.Flags&TCP_URG != 0),
-		"Flag ack:" + strconv.FormatBool(t.Flags&TCP_ACK != 0),
-		"Flag psh:" + strconv.FormatBool(t.Flags&TCP_PSH != 0),
-		"Flag rst:" + strconv.FormatBool(t.Flags&TCP_RST != 0),
-		"Flag syn:" + strconv.FormatBool(t.Flags&TCP_SYN != 0),
-		"Flag fin:" + strconv.FormatBool(t.Flags&TCP_FIN != 0),
-
-		"Window size:" + strconv.Itoa(int(t.Window)),
-		"Checksum:" + strconv.Itoa(int(t.Checksum)),
-
-		"Data:" + string(t.Data),
+		"Data size:" + strconv.Itoa(len(t.Data)),
+		"Data:" + string(t.Data[:maxLen]),
 	}, "\n")
 }
-
-type BySeq []*TCPPacket
-
-func (a BySeq) Len() int			{ return len(a) }
-func (a BySeq) Swap(i, j int)		{ a[i], a[j] = a[j], a[i] }
-func (a BySeq) Less(i, j int) bool	{ return a[i].Seq < a[j].Seq }

settings.go

@@ -3,38 +3,65 @@ package main
 import (
 	"flag"
 	"fmt"
-	"log"
 	"os"
+	"sync"
+	"time"
 )
 
-const (
-	VERSION = "0.8"
-)
+var VERSION string
 
+// MultiOption allows to specify multiple flags with same name and collects all values into array
+type MultiOption []string
+
+func (h *MultiOption) String() string {
+	return fmt.Sprint(*h)
+}
+
+// Set gets called multiple times for each flag with same name
+func (h *MultiOption) Set(value string) error {
+	*h = append(*h, value)
+	return nil
+}
+
+// AppSettings is the struct of main configuration
 type AppSettings struct {
-	verbose bool
-	stats bool
+	verbose   bool
+	debug     bool
+	stats     bool
+	exitAfter time.Duration
 
 	splitOutput bool
 
-	inputDummy  MultiOption
-	outputDummy MultiOption
+	inputDummy   MultiOption
+	outputDummy  MultiOption
+	outputStdout bool
+	outputNull   bool
 
-	inputTCP  MultiOption
-	outputTCP MultiOption
+	inputTCP       MultiOption
+	outputTCP      MultiOption
+	outputTCPStats bool
 
-	inputFile  MultiOption
-	outputFile MultiOption
+	inputFile        MultiOption
+	inputFileLoop    bool
+	outputFile       MultiOption
+	outputFileConfig FileOutputConfig
 
-	inputRAW MultiOption
+	inputRAW              MultiOption
+	inputRAWEngine        string
+	inputRAWTrackResponse bool
+	inputRAWRealIPHeader  string
 
-	outputHTTP              MultiOption
-	outputHTTPHeaders       HTTPHeaders
-	outputHTTPMethods       HTTPMethods
-	outputHTTPElasticSearch string
+	middleware string
+
+	inputHTTP  MultiOption
+	outputHTTP MultiOption
+
+	outputHTTPConfig HTTPOutputConfig
+	modifierConfig   HTTPModifierConfig
 }
 
-var Settings AppSettings = AppSettings{}
+// Settings holds Gor configuration
+var Settings AppSettings
 
 func usage() {
 	fmt.Printf("Gor is a simple http traffic replication tool written in Go. Its main goal is to replay traffic from production servers to staging and dev environments.\nProject page: https://github.com/buger/gor\nAuthor: <Leonid Bugaev> leonsbox@gmail.com\nCurrent Version: %s\n\n", VERSION)
@@ -45,30 +72,100 @@ func usage() {
 func init() {
 	flag.Usage = usage
 
-	flag.BoolVar(&Settings.verbose, "verbose", false, "Turn on verbose/debug output")
+	flag.BoolVar(&Settings.verbose, "verbose", false, "Turn on more verbose output")
+	flag.BoolVar(&Settings.debug, "debug", false, "Turn on debug output, shows all intercepted traffic. Works only when with `verbose` flag")
 	flag.BoolVar(&Settings.stats, "stats", false, "Turn on queue stats output")
+	flag.DurationVar(&Settings.exitAfter, "exit-after", 0, "exit after specified duration")
 
 	flag.BoolVar(&Settings.splitOutput, "split-output", false, "By default each output gets same traffic. If set to `true` it splits traffic equally among all outputs.")
 
 	flag.Var(&Settings.inputDummy, "input-dummy", "Used for testing outputs. Emits 'Get /' request every 1s")
-	flag.Var(&Settings.outputDummy, "output-dummy", "Used for testing inputs. Just prints data coming from inputs.")
+	flag.Var(&Settings.outputDummy, "output-dummy", "DEPRECATED: use --output-stdout instead")
+
+	flag.BoolVar(&Settings.outputStdout, "output-stdout", false, "Used for testing inputs. Just prints to console data coming from inputs.")
+
+	flag.BoolVar(&Settings.outputNull, "output-null", false, "Used for testing inputs. Drops all requests.")
 
 	flag.Var(&Settings.inputTCP, "input-tcp", "Used for internal communication between Gor instances. Example: \n\t# Receive requests from other Gor instances on 28020 port, and redirect output to staging\n\tgor --input-tcp :28020 --output-http staging.com")
 	flag.Var(&Settings.outputTCP, "output-tcp", "Used for internal communication between Gor instances. Example: \n\t# Listen for requests on 80 port and forward them to other Gor instance on 28020 port\n\tgor --input-raw :80 --output-tcp replay.local:28020")
+	flag.BoolVar(&Settings.outputTCPStats, "output-tcp-stats", false, "Report TCP output queue stats to console every 5 seconds.")
 
 	flag.Var(&Settings.inputFile, "input-file", "Read requests from file: \n\tgor --input-file ./requests.gor --output-http staging.com")
+	flag.BoolVar(&Settings.inputFileLoop, "input-file-loop", false, "Loop input files, useful for performance testing.")
+
 	flag.Var(&Settings.outputFile, "output-file", "Write incoming requests to file: \n\tgor --input-raw :80 --output-file ./requests.gor")
+	flag.DurationVar(&Settings.outputFileConfig.flushInterval, "output-file-flush-interval", time.Minute, "Interval for forcing buffer flush to the file, default: 60s.")
+	flag.BoolVar(&Settings.outputFileConfig.append, "output-file-append", false, "The flushed chunk is appended to existence file or not. ")
+
+	// Set default
+	Settings.outputFileConfig.sizeLimit.Set("32mb")
+	flag.Var(&Settings.outputFileConfig.sizeLimit, "output-file-size-limit", "Size of each chunk. Default: 32mb")
+	flag.IntVar(&Settings.outputFileConfig.queueLimit, "output-file-queue-limit", 256, "The length of the chunk queue. Default: 256")
 
 	flag.Var(&Settings.inputRAW, "input-raw", "Capture traffic from given port (use RAW sockets and require *sudo* access):\n\t# Capture traffic from 8080 port\n\tgor --input-raw :8080 --output-http staging.com")
 
+	flag.BoolVar(&Settings.inputRAWTrackResponse, "input-raw-track-response", false, "If turned on Gor will track responses in addition to requests, and they will be available to middleware and file output.")
+
+	flag.StringVar(&Settings.inputRAWEngine, "input-raw-engine", "libpcap", "Intercept traffic using `libpcap` (default), and `raw_socket`")
+
+	flag.StringVar(&Settings.inputRAWRealIPHeader, "input-raw-realip-header", "", "If not blank, injects header with given name and real IP value to the request payload. Usually this header should be named: X-Real-IP")
+
+	flag.StringVar(&Settings.middleware, "middleware", "", "Used for modifying traffic using external command")
+
+	// flag.Var(&Settings.inputHTTP, "input-http", "Read requests from HTTP, should be explicitly sent from your application:\n\t# Listen for http on 9000\n\tgor --input-http :9000 --output-http staging.com")
+
 	flag.Var(&Settings.outputHTTP, "output-http", "Forwards incoming requests to given http address.\n\t# Redirect all incoming requests to staging.com address \n\tgor --input-raw :80 --output-http http://staging.com")
-	flag.Var(&Settings.outputHTTPHeaders, "output-http-header", "Inject additional headers to http reqest:\n\tgor --input-raw :8080 --output-http staging.com --output-http-header 'User-Agent: Gor'")
-	flag.Var(&Settings.outputHTTPMethods, "output-http-method", "Whitelist of HTTP methods to replay. Anything else will be dropped:\n\tgor --input-raw :8080 --output-http staging.com --output-http-method GET --output-http-method OPTIONS")
-	flag.StringVar(&Settings.outputHTTPElasticSearch, "output-http-elasticsearch", "", "Send request and response stats to ElasticSearch:\n\tgor --input-raw :8080 --output-http staging.com --output-http-elasticsearch 'es_host:api_port/index_name'")
+	flag.IntVar(&Settings.outputHTTPConfig.BufferSize, "output-http-response-buffer", 0, "HTTP response buffer size, all data after this size will be discarded.")
+	flag.IntVar(&Settings.outputHTTPConfig.workers, "output-http-workers", 0, "Gor uses dynamic worker scaling by default.  Enter a number to run a set number of workers.")
+	flag.IntVar(&Settings.outputHTTPConfig.redirectLimit, "output-http-redirects", 0, "Enable how often redirects should be followed.")
+	flag.DurationVar(&Settings.outputHTTPConfig.Timeout, "output-http-timeout", 0, "Specify HTTP request/response timeout. By default 5s. Example: --output-http-timeout 30s")
+
+	flag.BoolVar(&Settings.outputHTTPConfig.stats, "output-http-stats", false, "Report http output queue stats to console every 5 seconds.")
+	flag.BoolVar(&Settings.outputHTTPConfig.OriginalHost, "http-original-host", false, "Normally gor replaces the Host http header with the host supplied with --output-http.  This option disables that behavior, preserving the original Host header.")
+	flag.BoolVar(&Settings.outputHTTPConfig.Debug, "output-http-debug", false, "Enables http debug output.")
+
+	flag.StringVar(&Settings.outputHTTPConfig.elasticSearch, "output-http-elasticsearch", "", "Send request and response stats to ElasticSearch:\n\tgor --input-raw :8080 --output-http staging.com --output-http-elasticsearch 'es_host:api_port/index_name'")
+
+	flag.Var(&Settings.modifierConfig.headers, "http-set-header", "Inject additional headers to http reqest:\n\tgor --input-raw :8080 --output-http staging.com --http-set-header 'User-Agent: Gor'")
+	flag.Var(&Settings.modifierConfig.headers, "output-http-header", "WARNING: `--output-http-header` DEPRECATED, use `--http-set-header` instead")
+
+	flag.Var(&Settings.modifierConfig.params, "http-set-param", "Set request url param, if param already exists it will be overwritten:\n\tgor --input-raw :8080 --output-http staging.com --http-set-param api_key=1")
+
+	flag.Var(&Settings.modifierConfig.methods, "http-allow-method", "Whitelist of HTTP methods to replay. Anything else will be dropped:\n\tgor --input-raw :8080 --output-http staging.com --http-allow-method GET --http-allow-method OPTIONS")
+	flag.Var(&Settings.modifierConfig.methods, "output-http-method", "WARNING: `--output-http-method` DEPRECATED, use `--http-allow-method` instead")
+
+	flag.Var(&Settings.modifierConfig.urlRegexp, "http-allow-url", "A regexp to match requests against. Filter get matched against full url with domain. Anything else will be dropped:\n\t gor --input-raw :8080 --output-http staging.com --http-allow-url ^www.")
+	flag.Var(&Settings.modifierConfig.urlRegexp, "output-http-url-regexp", "WARNING: `--output-http-url-regexp` DEPRECATED, use `--http-allow-url` instead")
+
+	flag.Var(&Settings.modifierConfig.urlNegativeRegexp, "http-disallow-url", "A regexp to match requests against. Filter get matched against full url with domain. Anything else will be forwarded:\n\t gor --input-raw :8080 --output-http staging.com --http-disallow-url ^www.")
+
+	flag.Var(&Settings.modifierConfig.urlRewrite, "http-rewrite-url", "Rewrite the request url based on a mapping:\n\tgor --input-raw :8080 --output-http staging.com --http-rewrite-url /v1/user/([^\\/]+)/ping:/v2/user/$1/ping")
+	flag.Var(&Settings.modifierConfig.urlRewrite, "output-http-rewrite-url", "WARNING: `--output-http-rewrite-url` DEPRECATED, use `--http-rewrite-url` instead")
+
+	flag.Var(&Settings.modifierConfig.headerFilters, "http-allow-header", "A regexp to match a specific header against. Requests with non-matching headers will be dropped:\n\t gor --input-raw :8080 --output-http staging.com --http-allow-header api-version:^v1")
+	flag.Var(&Settings.modifierConfig.headerFilters, "output-http-header-filter", "WARNING: `--output-http-header-filter` DEPRECATED, use `--http-allow-header` instead")
+
+	flag.Var(&Settings.modifierConfig.headerNegativeFilters, "http-disallow-header", "A regexp to match a specific header against. Requests with matching headers will be dropped:\n\t gor --input-raw :8080 --output-http staging.com --http-disallow-header \"User-Agent: Replayed by Gor\"")
+
+	flag.Var(&Settings.modifierConfig.headerHashFilters, "http-header-limiter", "Takes a fraction of requests, consistently taking or rejecting a request based on the FNV32-1A hash of a specific header:\n\t gor --input-raw :8080 --output-http staging.com --http-header-imiter user-id:25%")
+	flag.Var(&Settings.modifierConfig.headerHashFilters, "output-http-header-hash-filter", "WARNING: `output-http-header-hash-filter` DEPRECATED, use `--http-header-hash-limiter` instead")
+
+	flag.Var(&Settings.modifierConfig.paramHashFilters, "http-param-limiter", "Takes a fraction of requests, consistently taking or rejecting a request based on the FNV32-1A hash of a specific GET param:\n\t gor --input-raw :8080 --output-http staging.com --http-param-limiter user_id:25%")
 }
 
+var previousDebugTime int64
+var debugMutex sync.Mutex
+
+// Debug gets called only if --verbose flag specified
 func Debug(args ...interface{}) {
 	if Settings.verbose {
-		log.Println(args...)
+		debugMutex.Lock()
+		now := time.Now()
+		diff := float64(now.UnixNano()-previousDebugTime) / 1000000
+		previousDebugTime = now.UnixNano()
+		debugMutex.Unlock()
+
+		fmt.Printf("[DEBUG][PID %d][%d][%fms] ", os.Getpid(), now.UnixNano(), diff)
+		fmt.Println(args...)
 	}
 }

settings_headers.go

@@ -1,32 +0,0 @@
-package main
-
-import (
-	"errors"
-	"fmt"
-	"strings"
-)
-
-type HTTPHeaders []HTTPHeader
-type HTTPHeader struct {
-	Name  string
-	Value string
-}
-
-func (h *HTTPHeaders) String() string {
-	return fmt.Sprint(*h)
-}
-
-func (h *HTTPHeaders) Set(value string) error {
-	v := strings.SplitN(value, ":", 2)
-	if len(v) != 2 {
-		return errors.New("Expected `Key: Value`")
-	}
-
-	header := HTTPHeader{
-		strings.TrimSpace(v[0]),
-		strings.TrimSpace(v[1]),
-	}
-
-	*h = append(*h, header)
-	return nil
-}

settings_methods.go

@@ -1,26 +0,0 @@
-package main
-
-import (
-	"fmt"
-	"strings"
-)
-
-type HTTPMethods []string
-
-func (h *HTTPMethods) String() string {
-	return fmt.Sprint(*h)
-}
-
-func (h *HTTPMethods) Set(value string) error {
-	*h = append(*h, strings.ToUpper(value))
-	return nil
-}
-
-func (h *HTTPMethods) Contains(value string) bool {
-	for _, method := range *h {
-		if value == method {
-			return true
-		}
-	}
-	return false
-}

settings_methods_test.go

@@ -1,24 +0,0 @@
-package main
-
-import (
-	"testing"
-)
-
-func TestHTTPMethods(t *testing.T) {
-	methods := HTTPMethods{}
-
-	methods.Set("lower")
-	methods.Set("UPPER")
-
-	if !methods.Contains("LOWER") {
-		t.Error("Does not contain LOWER")
-	}
-
-	if !methods.Contains("UPPER") {
-		t.Error("Does not contain UPPER")
-	}
-
-	if methods.Contains("ABSENT") {
-		t.Error("Does contain ABSENT")
-	}
-}

settings_option.go

@@ -1,16 +0,0 @@
-package main
-
-import (
-	"fmt"
-)
-
-type MultiOption []string
-
-func (h *MultiOption) String() string {
-	return fmt.Sprint(*h)
-}
-
-func (h *MultiOption) Set(value string) error {
-	*h = append(*h, value)
-	return nil
-}

test_input.go

@@ -1,9 +1,17 @@
 package main
 
+import (
+	"crypto/rand"
+	"encoding/base64"
+	"time"
+)
+
+// TestInput used for testing purpose, it allows emitting requests on demand
 type TestInput struct {
 	data chan []byte
 }
 
+// NewTestInput constructor for TestInput
 func NewTestInput() (i *TestInput) {
 	i = new(TestInput)
 	i.data = make(chan []byte, 100)
@@ -13,19 +21,53 @@ func NewTestInput() (i *TestInput) {
 
 func (i *TestInput) Read(data []byte) (int, error) {
 	buf := <-i.data
-	copy(data, buf)
 
-	return len(buf), nil
+	header := payloadHeader(RequestPayload, uuid(), time.Now().UnixNano(), -1)
+	copy(data[0:len(header)], header)
+	copy(data[len(header):], buf)
+
+	return len(buf) + len(header), nil
 }
 
+// EmitGET emits GET request without headers
 func (i *TestInput) EmitGET() {
 	i.data <- []byte("GET / HTTP/1.1\r\n\r\n")
 }
 
+// EmitPOST emits POST request with Content-Length
 func (i *TestInput) EmitPOST() {
-	i.data <- []byte("POST /pub/WWW/ HTTP/1.1\nHost: www.w3.org\r\n\r\na=1&b=2\r\n\r\n")
+	i.data <- []byte("POST /pub/WWW/ HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
 }
 
+// EmitChunkedPOST emits POST request with `Transfer-Encoding: chunked` and chunked body
+func (i *TestInput) EmitChunkedPOST() {
+	i.data <- []byte("POST /pub/WWW/ HTTP/1.1\r\nHost: www.w3.org\r\nTransfer-Encoding: chunked\r\n\r\n4\r\nWiki\r\n5\r\npedia\r\ne\r\n in\r\n\r\nchunks.\r\n0\r\n\r\n")
+}
+
+// EmitLargePOST emits POST request with large payload (5mb)
+func (i *TestInput) EmitLargePOST() {
+	size := 5 * 1024 * 1024 // 5 MB
+	rb := make([]byte, size)
+	rand.Read(rb)
+
+	rs := base64.URLEncoding.EncodeToString(rb)
+
+	i.data <- []byte("POST / HTTP/1.1\nHost: www.w3.org\nContent-Length:5242880\r\n\r\n" + rs)
+	Debug("Sent large POST")
+}
+
+// EmitSizedPOST emit a POST with a payload set to a supplied size
+func (i *TestInput) EmitSizedPOST(payloadSize int) {
+	rb := make([]byte, payloadSize)
+	rand.Read(rb)
+
+	rs := base64.URLEncoding.EncodeToString(rb)
+
+	i.data <- []byte("POST / HTTP/1.1\nHost: www.w3.org\nContent-Length:5242880\r\n\r\n" + rs)
+	Debug("Sent large POST")
+}
+
+// EmitOPTIONS emits OPTIONS request, similar to GET
 func (i *TestInput) EmitOPTIONS() {
 	i.data <- []byte("OPTIONS / HTTP/1.1\nHost: www.w3.org\r\n\r\n")
 }

test_output.go

@@ -2,10 +2,12 @@ package main
 
 type writeCallback func(data []byte)
 
+// TestOutput used in testing to intercept any output into callback
 type TestOutput struct {
 	cb writeCallback
 }
 
+// NewTestOutput constructor for TestOutput, accepts callback which get called on each incoming Write
 func NewTestOutput(cb writeCallback) (i *TestOutput) {
 	i = new(TestOutput)
 	i.cb = cb