Add tests for listener and tcp_message

This commit is contained in:
Leonid Bugaev
2016-04-19 20:43:54 +05:00
parent bb2adbce65
commit 9fc7cdc0d3
8 changed files with 438 additions and 44 deletions
+7 -4
View File
@@ -1,14 +1,14 @@
SOURCE = emitter.go gor.go gor_stat.go input_dummy.go input_file.go input_raw.go input_tcp.go limiter.go output_dummy.go output_file.go input_http.go output_http.go output_tcp.go plugins.go settings.go test_input.go elasticsearch.go http_modifier.go http_modifier_settings.go http_client.go middleware.go protocol.go
SOURCE_PATH = /gopath/src/github.com/buger/gor/
SOURCE_PATH = /go/src/github.com/buger/gor/
release: release-x86 release-x64
release-x64:
docker run -v `pwd`:$(SOURCE_PATH) -t --env GOOS=linux --env GOARCH=amd64 --env CGO_ENABLED=0 -i gor go build -ldflags "-X main.VERSION $(VERSION)"&& tar -czf gor_$(VERSION)_x64.tar.gz gor && rm gor
docker run -v `pwd`:$(SOURCE_PATH) -t --env GOOS=linux --env GOARCH=amd64 --env CGO_ENABLED=0 -i gor go build -ldflags "-X main.VERSION=$(VERSION)"&& tar -czf gor_$(VERSION)_x64.tar.gz gor && rm gor
release-x86:
docker run -v `pwd`:$(SOURCE_PATH) -t --env GOOS=linux --env GOARCH=386 --env CGO_ENABLED=0 -i gor go build -ldflags "-X main.VERSION $(VERSION)" && tar -czf gor_$(VERSION)_x86.tar.gz gor && rm gor
docker run -v `pwd`:$(SOURCE_PATH) -t --env GOOS=linux --env GOARCH=386 --env CGO_ENABLED=0 -i gor go build -ldflags "-X main.VERSION=$(VERSION)" && tar -czf gor_$(VERSION)_x86.tar.gz gor && rm gor
dbuild:
docker build -t gor .
@@ -24,7 +24,10 @@ drace:
docker run -v `pwd`:$(SOURCE_PATH) -t -i --env GORACE="halt_on_error=1" gor go test ./... $(ARGS) -v -race -timeout 15s
dtest:
docker run -v `pwd`:$(SOURCE_PATH) -t -i gor go test ./... -timeout 60s $(ARGS) -v
docker run -v `pwd`:$(SOURCE_PATH) -t -i gor go test ./... -timeout 10s $(ARGS) -v
dtestone:
docker run -v `pwd`:$(SOURCE_PATH) -t -i gor go test ./. -timeout 4s -run $(TEST) $(ARGS) -v
dcover:
docker run -v `pwd`:$(SOURCE_PATH) -t -i --env GORACE="halt_on_error=1" gor go test $(ARGS) -race -v -timeout 15s -coverprofile=coverage.out
+1 -1
View File
@@ -59,7 +59,7 @@ func (i *RAWInput) listen(address string) {
log.Fatal("input-raw: error while parsing address", err)
}
i.listener = raw.NewListener(host, port, i.expire, true)
i.listener = raw.NewListener(host, port, i.expire)
for {
select {
+2
View File
@@ -86,6 +86,8 @@ func TestInputRAW100Expect(t *testing.T) {
// We will use it to get content of raw HTTP request
testOutput := NewTestOutput(func(data []byte) {
log.Println("Received", string(data[:100]))
switch data[0] {
case RequestPayload:
if strings.Contains(string(data), "Expect: 100-continue") {
+89 -18
View File
@@ -26,6 +26,7 @@ import (
// Listener handle traffic capture
type Listener struct {
// buffer of TCPMessages waiting to be send
// ID -> TCPMessage
messages map[string]*TCPMessage
// Expect: 100-continue request is send in 2 tcp messages
@@ -34,8 +35,12 @@ type Listener struct {
// To get ACK of second message we need to compute its Seq and wait for them message
seqWithData map[uint32]uint32
// Ack -> Req
respAliases map[uint32]*request
// Ack -> ID
respWithoutReq map[uint32]string
// Messages ready to be send to client
packetsChan chan *TCPPacket
@@ -47,8 +52,6 @@ type Listener struct {
messageExpire time.Duration
captureResponse bool
conn net.PacketConn
quit chan bool
}
@@ -59,8 +62,8 @@ type request struct {
}
// NewListener creates and initializes new Listener object
func NewListener(addr string, port string, expire time.Duration, captureResponse bool) (l *Listener) {
l = &Listener{captureResponse: captureResponse}
func NewListener(addr string, port string, expire time.Duration) (l *Listener) {
l = &Listener{}
l.packetsChan = make(chan *TCPPacket, 10000)
l.messagesChan = make(chan *TCPMessage, 10000)
@@ -70,6 +73,7 @@ func NewListener(addr string, port string, expire time.Duration, captureResponse
l.ackAliases = make(map[uint32]uint32)
l.seqWithData = make(map[uint32]uint32)
l.respAliases = make(map[uint32]*request)
l.respWithoutReq = make(map[uint32]string)
l.addr = addr
_port, _ := strconv.Atoi(port)
@@ -82,7 +86,11 @@ func NewListener(addr string, port string, expire time.Duration, captureResponse
l.messageExpire = expire
go l.listen()
go l.readRAWSocket()
// Special case for testing
if l.port != 0 {
go l.readRAWSocket()
}
return
}
@@ -93,7 +101,9 @@ func (t *Listener) listen() {
for {
select {
case <-t.quit:
t.conn.Close()
if t.conn != nil {
t.conn.Close()
}
return
// We need to use channels to process each packet to avoid data races
case packet := <-t.packetsChan:
@@ -112,17 +122,38 @@ func (t *Listener) listen() {
}
func (t *Listener) dispatchMessage(message *TCPMessage) {
log.Println("Dispatching:", message, string(message.Bytes()))
delete(t.ackAliases, message.Ack)
delete(t.messages, message.ID)
if !message.IsIncoming {
if message.IsIncoming {
// If there were response before request
// log.Println("Checking", message.ResponseAck, t.respWithoutReq)
if respID, ok := t.respWithoutReq[message.ResponseAck]; ok {
log.Println("FOUND RESP WITHOUT REQ")
if resp, rok := t.messages[respID]; rok {
if resp.RequestAck == 0 {
resp.RequestAck = message.Ack
resp.RequestStart = message.Start
if resp.IsFinished() {
defer t.dispatchMessage(resp)
}
}
}
}
} else {
delete(t.respAliases, message.Ack)
// log.Println("Removing alias:", message.Ack, t.respWithoutReq)
delete(t.respWithoutReq, message.Ack)
// Do not track responses which have no associated requests
if message.RequestAck == 0 {
log.Println("DAMN")
return
}
}
t.messagesChan <- message
}
@@ -171,7 +202,7 @@ func (t *Listener) isValidPacket(buf []byte) bool {
srcPort := binary.BigEndian.Uint16(buf[0:2])
// Because RAW_SOCKET can't be bound to port, we have to control it by ourself
if destPort == t.port || (t.captureResponse && srcPort == t.port) {
if destPort == t.port || srcPort == t.port {
// Get the 'data offset' (size of the TCP header in 32-bit words)
dataOffset := (buf[12] & 0xF0) >> 4
@@ -206,16 +237,19 @@ func (t *Listener) processTCPPacket(packet *TCPPacket) {
if parentAck, ok := t.seqWithData[packet.Seq]; ok {
t.ackAliases[packet.Ack] = parentAck
log.Println("Adding ack alias", packet.Ack, parentAck)
packet.Ack = parentAck
delete(t.seqWithData, packet.Seq)
}
if alias, ok := t.ackAliases[packet.Ack]; ok {
log.Println("Using ack alisas", packet.Ack, alias)
packet.Ack = alias
}
var responseRequest *request
if t.captureResponse && !isIncoming {
if !isIncoming {
responseRequest, _ = t.respAliases[packet.Ack]
}
@@ -223,13 +257,30 @@ func (t *Listener) processTCPPacket(packet *TCPPacket) {
message, ok := t.messages[mID]
if len(packet.Data) >= 20 {
log.Println("New packet:", packet.Seq, packet.Ack, string(packet.Data[:20]))
} else {
log.Println("New packet:", packet.Seq, packet.Ack, string(packet.Data))
}
if !ok {
message = NewTCPMessage(mID, packet.Ack, isIncoming)
message = NewTCPMessage(mID, packet.Seq, packet.Ack, isIncoming)
t.messages[mID] = message
if !isIncoming && responseRequest != nil {
message.RequestStart = responseRequest.start
message.RequestAck = responseRequest.ack
if !isIncoming {
if responseRequest != nil {
message.RequestStart = responseRequest.start
message.RequestAck = responseRequest.ack
} else {
log.Println("Can't find request", packet.Ack, t.respAliases)
t.respWithoutReq[packet.Ack] = mID
}
}
if len(packet.Data) >= 20 {
log.Println("New message:", message, string(packet.Data[:20]))
} else {
log.Println("New message:", message, string(packet.Data))
}
}
@@ -237,21 +288,39 @@ func (t *Listener) processTCPPacket(packet *TCPPacket) {
if len(packet.Data) > 4 && bytes.Equal(packet.Data[0:4], bPOST) {
// reading last 20 bytes (not counting CRLF): last header value (if no body presented)
if bytes.Equal(packet.Data[len(packet.Data)-24:len(packet.Data)-4], bExpect100ContinueCheck) {
t.seqWithData[packet.Seq+uint32(len(packet.Data))] = packet.Ack
seq := packet.Seq+uint32(len(packet.Data))
t.seqWithData[seq] = packet.Ack
// In case if sequence packet came first
for _id, m := range t.messages {
if m.Seq == seq {
panic("asds")
t.ackAliases[m.Ack] = packet.Ack
for _, pkt := range m.packets {
message.AddPacket(pkt)
}
delete(t.messages, _id)
}
}
// Removing `Expect: 100-continue` header
packet.Data = append(packet.Data[:len(packet.Data)-24], packet.Data[len(packet.Data)-2:]...)
}
}
if t.captureResponse && isIncoming {
if isIncoming {
// If message have multiple packets, delete previous alias
if len(message.packets) > 0 {
delete(t.respAliases, message.ResponseAck)
}
responseAck := packet.Seq + uint32(len(packet.Data))
responseAck := packet.Seq + uint32(message.Size()) + uint32(len(packet.Data))
t.respAliases[responseAck] = &request{message.Start, message.Ack}
log.Println("ADDING RESP ACK", responseAck, message.Ack)
message.ResponseAck = responseAck
}
@@ -259,7 +328,7 @@ func (t *Listener) processTCPPacket(packet *TCPPacket) {
message.AddPacket(packet)
// If message contains only single packet immediately dispatch it
if !message.IsMultipart() {
if message.IsFinished() {
t.dispatchMessage(message)
}
}
@@ -271,6 +340,8 @@ func (t *Listener) Receive() *TCPMessage {
func (t *Listener) Close() {
close(t.quit)
t.conn.Close()
if t.conn != nil {
t.conn.Close()
}
return
}
+140
View File
@@ -0,0 +1,140 @@
package rawSocket
import (
"testing"
"time"
"bytes"
_ "log"
)
func TestRawListenerInput(t *testing.T) {
var req, resp *TCPMessage
listener := NewListener("", "0", 10 * time.Millisecond)
defer listener.Close()
reqPacket := buildPacket(true, 1, 1, []byte("GET / HTTP/1.1"))
listener.packetsChan <- reqPacket
respAck := reqPacket.Seq + uint32(len(reqPacket.Data))
respPacket := buildPacket(false, respAck, reqPacket.Seq + 1, []byte("HTTP/1.1 200 OK"))
listener.packetsChan <- respPacket
select {
case req = <- listener.messagesChan:
case <- time.After(time.Millisecond):
t.Error("Should return respose immediately")
return
}
if !req.IsIncoming {
t.Error("Should be request")
}
select {
case resp = <- listener.messagesChan:
case <- time.After(time.Millisecond):
t.Error("Should return response immediately")
return
}
if resp.IsIncoming {
t.Error("Should be response")
}
}
func TestRawListenerResponse(t *testing.T) {
var req, resp *TCPMessage
listener := NewListener("", "0", 10 * time.Millisecond)
defer listener.Close()
reqPacket := buildPacket(true, 1, 1, []byte("GET / HTTP/1.1"))
respPacket := buildPacket(false, 1 + uint32(len(reqPacket.Data)), 2, []byte("HTTP/1.1 200 OK"))
// If response packet comes before request
listener.packetsChan <- respPacket
listener.packetsChan <- reqPacket
select {
case req = <- listener.messagesChan:
case <- time.After(time.Millisecond):
t.Error("Should return respose immediately")
return
}
if !req.IsIncoming {
t.Error("Should be request")
}
select {
case resp = <- listener.messagesChan:
case <- time.After(time.Millisecond):
t.Error("Should return response immediately")
return
}
if resp.IsIncoming {
t.Error("Should be response")
}
if !bytes.Equal(resp.UUID(), req.UUID()) {
t.Error("Resp and Req UUID should be equal")
}
}
func TestRawListener100Continue(t *testing.T) {
var req, resp *TCPMessage
listener := NewListener("", "0", 10 * time.Millisecond)
defer listener.Close()
reqPacket1 := buildPacket(true, 1, 1, []byte("POST / HTTP/1.1\r\nContent-Length: 2\r\nExpect: 100-continue\r\n\r\n"))
// Packet with data have different Seq
reqPacket2 := buildPacket(true, 2, reqPacket1.Seq + uint32(len(reqPacket1.Data)), []byte("a"))
reqPacket3 := buildPacket(true, 2, reqPacket2.Seq + 1, []byte("b"))
respPacket1 := buildPacket(false, 10, 3, []byte("HTTP/1.1 100 Continue\r\n"))
respPacket2 := buildPacket(false, reqPacket3.Seq + uint32(len(reqPacket1.Data)) + uint32(len(reqPacket2.Data)) + uint32(len(reqPacket3.Data)), 2, []byte("HTTP/1.1 200 OK\r\n"))
listener.processTCPPacket(reqPacket1)
listener.processTCPPacket(reqPacket2)
listener.processTCPPacket(reqPacket3)
listener.processTCPPacket(respPacket1)
listener.processTCPPacket(respPacket2)
select {
case req = <- listener.messagesChan:
break
case <- time.After(11 * time.Millisecond):
t.Error("Should return respose after expire time")
return
}
if !bytes.Equal(req.Bytes(), []byte("POST / HTTP/1.1\r\nContent-Length: 2\r\n\r\nab")) {
t.Error("Should recived full message")
}
if !req.IsIncoming {
t.Error("Should be request")
}
select {
case resp = <- listener.messagesChan:
break
case <- time.After(21 * time.Millisecond):
t.Error("Should return response after expire time")
return
}
if resp.IsIncoming {
t.Error("Should be response")
}
if !bytes.Equal(resp.UUID(), req.UUID()) {
t.Error("Resp and Req UUID should be equal")
}
}
+42 -13
View File
@@ -18,6 +18,7 @@ import (
// Message is received if we didn't receive any packets for 2000ms
type TCPMessage struct {
ID string // Message ID
Seq uint32
Ack uint32
ResponseAck uint32
RequestStart time.Time
@@ -32,8 +33,8 @@ type TCPMessage struct {
}
// NewTCPMessage pointer created from a Acknowledgment number and a channel of messages readuy to be deleted
func NewTCPMessage(ID string, Ack uint32, IsIncoming bool) (msg *TCPMessage) {
msg = &TCPMessage{ID: ID, Ack: Ack, IsIncoming: IsIncoming}
func NewTCPMessage(ID string, Seq, Ack uint32, IsIncoming bool) (msg *TCPMessage) {
msg = &TCPMessage{ID: ID, Seq: Seq, Ack: Ack, IsIncoming: IsIncoming}
msg.Start = time.Now()
return
@@ -48,8 +49,12 @@ func (t *TCPMessage) Bytes() (output []byte) {
return output
}
// Size returns total size of message
func (t *TCPMessage) Size() (size int) {
// Size returns total body size
func (t *TCPMessage) BodySize() (size int) {
if len(t.packets) == 0 {
return 0
}
size += len(proto.Body(t.packets[0].Data))
for _, p := range t.packets[1:] {
@@ -59,6 +64,19 @@ func (t *TCPMessage) Size() (size int) {
return
}
// Size returns total size of message
func (t *TCPMessage) Size() (size int) {
if len(t.packets) == 0 {
return 0
}
for _, p := range t.packets {
size += len(p.Data)
}
return
}
// AddPacket to the message and ensure packet uniqueness
// TCP allows that packet can be re-send multiple times
func (t *TCPMessage) AddPacket(packet *TCPPacket) {
@@ -86,9 +104,9 @@ func (t *TCPMessage) AddPacket(packet *TCPPacket) {
}
// isMultipart returns true if message contains from multiple tcp packets
func (t *TCPMessage) IsMultipart() bool {
func (t *TCPMessage) IsFinished() bool {
if len(t.packets) > 1 {
return true
return false
}
payload := t.packets[0].Data
@@ -97,7 +115,7 @@ func (t *TCPMessage) IsMultipart() bool {
if t.IsIncoming {
// If one GET, OPTIONS, or HEAD request
if bytes.Equal(m, []byte("GET ")) || bytes.Equal(m, []byte("OPTI")) || bytes.Equal(m, []byte("HEAD")) {
return false
return true
} else {
// Sometimes header comes after the body :(
if bytes.Equal(m, []byte("POST")) || bytes.Equal(m, []byte("PUT ")) || bytes.Equal(m, []byte("PATC")) {
@@ -105,28 +123,38 @@ func (t *TCPMessage) IsMultipart() bool {
l, _ := strconv.Atoi(string(length))
// If content-length equal current body length
if l > 0 && l == t.Size() {
return false
if l > 0 && l == t.BodySize() {
return true
}
}
}
}
} else {
// Request not found
// Can be because response came first or request request was just missing
if t.RequestAck == 0 {
return false
}
if length := proto.Header(payload, []byte("Content-Length")); len(length) > 0 {
if length[0] == '0' {
return false
return true
}
l, _ := strconv.Atoi(string(length))
// If content-length equal current body length
if l > 0 && l == t.Size() {
return false
if l > 0 && l == t.BodySize() {
return true
}
} else {
if enc := proto.Header(payload, []byte("Transfer-Encoding")); len(enc) == 0 {
return true
}
}
}
return true
return false
}
func (t *TCPMessage) UUID() []byte {
@@ -146,3 +174,4 @@ func (t *TCPMessage) UUID() []byte {
return uuid
}
+155
View File
@@ -0,0 +1,155 @@
package rawSocket
import (
"testing"
"net"
"strconv"
"bytes"
_ "log"
)
func buildPacket(isIncoming bool, Ack, Seq uint32, Data []byte) (packet *TCPPacket) {
packet = &TCPPacket{
Addr: &net.IPAddr{net.IP{}, ""},
Ack: Ack,
Seq: Seq,
Data: Data,
}
// For tests `listening` port is 0
if isIncoming {
packet.SrcPort = 1
} else {
packet.DestPort = 1
}
return packet
}
func buildMessage(p *TCPPacket) *TCPMessage {
id := p.Addr.String() + strconv.Itoa(int(p.DestPort)) + strconv.Itoa(int(p.Ack))
isIncoming := false
if p.SrcPort == 1 {
isIncoming = true
}
m := NewTCPMessage(id, p.Seq, p.Ack, isIncoming)
m.AddPacket(p)
return m
}
func TestTCPMessagePacketsOrder(t *testing.T) {
msg := buildMessage(buildPacket(true, 1, 1, []byte("a")))
msg.AddPacket(buildPacket(true, 1, 2, []byte("b")))
if !bytes.Equal(msg.Bytes(), []byte("ab")) {
t.Error("Should contatenate packets in right order")
}
// When first packet have wrong order (Seq)
msg = buildMessage(buildPacket(true, 1, 2, []byte("b")))
msg.AddPacket(buildPacket(true, 1, 1, []byte("a")))
if !bytes.Equal(msg.Bytes(), []byte("ab")) {
t.Error("Should contatenate packets in right order")
}
// Should ignore packets with same sequence
msg = buildMessage(buildPacket(true, 1, 1, []byte("a")))
msg.AddPacket(buildPacket(true, 1, 1, []byte("a")))
if !bytes.Equal(msg.Bytes(), []byte("a")) {
t.Error("Should ignore packet with same Seq")
}
}
func TestTCPMessageSize(t *testing.T) {
msg := buildMessage(buildPacket(true, 1, 1, []byte("POST / HTTP/1.1\r\nContent-Length: 2\r\n\r\na")))
msg.AddPacket(buildPacket(true, 1, 2, []byte("b")))
if msg.BodySize() != 2 {
t.Error("Should count only body", msg.BodySize())
}
if msg.Size() != 40 {
t.Error("Should count all sizes", msg.Size())
}
}
func TestTCPMessageIsFinished(t *testing.T) {
methodsWithoutBodies := []string{"GET","OPTIONS","HEAD"}
for _, m := range methodsWithoutBodies {
msg := buildMessage(buildPacket(true, 1, 1, []byte(m + " / HTTP/1.1")))
if !msg.IsFinished() {
t.Error(m, " request should be finished")
}
}
methodsWithBodies := []string{"POST","PUT","PATCH"}
for _, m := range methodsWithBodies {
msg := buildMessage(buildPacket(true, 1, 1, []byte(m + " / HTTP/1.1\r\nContent-Length: 1\r\n\r\na")))
if !msg.IsFinished() {
t.Error(m, " should be finished as body length == content length")
}
msg = buildMessage(buildPacket(true, 1, 1, []byte(m + " / HTTP/1.1\r\nContent-Length: 2\r\n\r\na")))
if msg.IsFinished() {
t.Error(m, " should not be finished as body length != content length")
}
}
msg := buildMessage(buildPacket(true, 1, 1, []byte("UNKNOWN / HTTP/1.1\r\n\r\n")))
if msg.IsFinished() {
t.Error("non http or wrong methods considered as not finished")
}
// Responses
msg = buildMessage(buildPacket(false, 1, 1, []byte("HTTP/1.1 200 OK\r\n\r\n")))
msg.RequestAck = 1
if !msg.IsFinished() {
t.Error("Should mark simple response as finished")
}
msg = buildMessage(buildPacket(false, 1, 1, []byte("HTTP/1.1 200 OK\r\n\r\n")))
msg.RequestAck = 0
if msg.IsFinished() {
t.Error("Should not mark responses without associated requests")
}
msg = buildMessage(buildPacket(false, 1, 1, []byte("HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n")))
msg.RequestAck = 1
if msg.IsFinished() {
t.Error("Should mark chunked response as non finished")
}
msg = buildMessage(buildPacket(false, 1, 1, []byte("HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n")))
msg.RequestAck = 1
if !msg.IsFinished() {
t.Error("Should mark Content-Length: 0 respones as finished")
}
msg = buildMessage(buildPacket(false, 1, 1, []byte("HTTP/1.1 200 OK\r\nContent-Length: 1\r\n\r\na")))
msg.RequestAck = 1
if !msg.IsFinished() {
t.Error("Should mark valid Content-Length respones as finished")
}
msg = buildMessage(buildPacket(false, 1, 1, []byte("HTTP/1.1 200 OK\r\nContent-Length: 10\r\n\r\na")))
msg.RequestAck = 1
if msg.IsFinished() {
t.Error("Should not mark not valid Content-Length respones as finished")
}
}
+2 -8
View File
@@ -70,8 +70,8 @@ func (t *TCPPacket) ParseBasic() {
// String output for a TCP Packet
func (t *TCPPacket) String() string {
maxLen := len(t.Data)
if maxLen > 500 {
maxLen = 500
if maxLen > 200 {
maxLen = 200
}
return strings.Join([]string{
@@ -99,9 +99,3 @@ func (t *TCPPacket) String() string {
"Data:" + string(t.Data[:maxLen]),
}, "\n")
}
type sortBySeq []*TCPPacket
func (a sortBySeq) Len() int { return len(a) }
func (a sortBySeq) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
func (a sortBySeq) Less(i, j int) bool { return a[i].Seq < a[j].Seq }