Server reload (#63)

* add reload feature for cache type: redis, memcache, memcache_binary
* add more metrics
* add fuzz test for redis/memcache protocol

.gitignore

@@ -24,3 +24,4 @@ cmd/apicli/apicli
 cmd/anzi/anzi
 cmd/enri/enri
 coverage.txt
+go.sum

.travis.yml

@@ -1,7 +1,7 @@
 language: go
 
 go:
-  - 1.11
+  - 1.12
 
 # Only clone the most recent commit.
 git:
@@ -27,10 +27,7 @@ before_script:
   - curl -sfL https://install.goreleaser.com/github.com/golangci/golangci-lint.sh | sh -s -- -b $GOPATH/bin
 
 install:
-  - bash scripts/install-travis-etcd.sh
-  - docker pull grokzen/redis-cluster:4.0.9
-  - docker run -e "IP=0.0.0.0" -d -p 7000-7007:7000-7007 grokzen/redis-cluster:4.0.9
-  - docker run -e "IP=0.0.0.0" -d -p 8000-8007:8000-8007 grokzen/redis-cluster:4.0.9
+  - bash -x scripts/init-ci-all.sh
 
 script:
   - ./scripts/codecov.sh

README.md

@@ -33,4 +33,4 @@ Overlord已被哔哩哔哩用于生产环境。
 
 -------------
 
-*You are welcome to use Overlord in your product, and feel free to let us know~ :)*
+*Please report bugs, concerns, suggestions by issues, or join QQ-group 716486124 to discuss problems around source code.*

ci/README.md

@@ -0,0 +1,23 @@
+# overlord 集成测试工具集
+
+## 确定性测试
+
+### redis-cluster
+
+1. 测试发送随机长度,随机的命令的
+2. 测试发送不支持的命令的时候的结果
+
+### proxy 模式测试
+
+1. 测试发送单命令、多命令、超长命令的情况
+2. 测试发送不支持的命令的测试结果。
+3. 测试后端断开连接的时候的结果
+4. 分别测试 request 模式下的三种协议支持
+
+## 基准测试
+
+测试各种基本(get/set)命令的 benchmark 。
+
+## FUZZ测试
+
+利用 go-fuzz 库开启测试

ci/fuzz/.gitignore

@@ -0,0 +1,4 @@
+*/corpus/*
+*/crashers/*
+*/suppressions/*
+*/*.zip

ci/fuzz/mcparser/fuzz.go

@@ -0,0 +1,114 @@
+package mcparser
+
+import (
+	"bytes"
+	"io"
+	"net"
+	"sync/atomic"
+	"time"
+
+	"overlord/pkg/bufio"
+	libnet "overlord/pkg/net"
+	"overlord/proxy/proto"
+	"overlord/proxy/proto/memcache"
+)
+
+var (
+	pc   proto.ProxyConn
+	msgs []*proto.Message
+	nc   *libnet.Conn
+)
+
+func Fuzz(data []byte) int {
+	conn := _createConn(data)
+	nc := libnet.NewConn(conn, time.Second, time.Second)
+	pc = memcache.NewProxyConn(nc)
+	msgs = proto.GetMsgs(4)
+	nmsgs, err := pc.Decode(msgs)
+	if err == bufio.ErrBufferFull {
+		return -1
+	}
+
+	if err != nil {
+		return 1
+	}
+
+	if len(nmsgs) >= 1 {
+		return 1
+	}
+
+	return 0
+}
+
+const (
+	stateClosed  = 1
+	stateOpening = 0
+)
+
+type mockAddr string
+
+func (m mockAddr) Network() string {
+	return "tcp"
+}
+func (m mockAddr) String() string {
+	return string(m)
+}
+
+type mockConn struct {
+	addr   mockAddr
+	buf    *bytes.Buffer
+	err    error
+	closed int32
+}
+
+func (m *mockConn) Read(b []byte) (n int, err error) {
+	if atomic.LoadInt32(&m.closed) == stateClosed {
+		return 0, io.EOF
+	}
+	if m.err != nil {
+		err = m.err
+		return
+	}
+	return m.buf.Read(b)
+}
+
+func (m *mockConn) Write(b []byte) (n int, err error) {
+	if atomic.LoadInt32(&m.closed) == stateClosed {
+		return 0, io.EOF
+	}
+
+	if m.err != nil {
+		err = m.err
+		return
+	}
+	return m.buf.Write(b)
+}
+
+// writeBuffers impl the net.buffersWriter to support writev
+func (m *mockConn) writeBuffers(buf *net.Buffers) (int64, error) {
+	if m.err != nil {
+		return 0, m.err
+	}
+	return buf.WriteTo(m.buf)
+}
+
+func (m *mockConn) Close() error {
+	atomic.StoreInt32(&m.closed, stateClosed)
+	return nil
+}
+
+func (m *mockConn) LocalAddr() net.Addr  { return m.addr }
+func (m *mockConn) RemoteAddr() net.Addr { return m.addr }
+
+func (m *mockConn) SetDeadline(t time.Time) error      { return nil }
+func (m *mockConn) SetReadDeadline(t time.Time) error  { return nil }
+func (m *mockConn) SetWriteDeadline(t time.Time) error { return nil }
+
+// _createConn is useful tools for handler test
+func _createConn(data []byte) net.Conn {
+	mconn := &mockConn{
+		addr: "127.0.0.1:12345",
+		buf:  bytes.NewBuffer(data),
+	}
+	return mconn
+}

ci/fuzz/mcparser/gen/main.go

@@ -0,0 +1,16 @@
+package main
+
+import (
+	"github.com/dvyukov/go-fuzz/gen"
+)
+
+var zdata = []string{
+	`SET A 1 1 1\r\n1\r\n`,
+	"GET A",
+}
+
+func main() {
+	for _, data := range zdata {
+		gen.Emit([]byte(data), nil, true)
+	}
+}

ci/fuzz/redisparser/corpus/0

@@ -0,0 +1,37 @@
+*3
+$4
+MGET
+$4
+baka
+$4
+kaba
+*5
+$4
+MSET
+$1
+a
+$1
+b
+$3
+eee
+$5
+12345
+*3
+$4
+MGET
+$4
+enen
+$4
+nime
+*2
+$3
+GET
+$5
+abcde
+*3
+$3
+DEL
+$1
+a
+$1
+b

ci/fuzz/redisparser/corpus/1

@@ -0,0 +1,7 @@
+*3
+$5
+SETNX
+$1
+a
+$10
+abcdeabcde

ci/fuzz/redisparser/corpus/2

@@ -0,0 +1 @@
+set a b

ci/fuzz/redisparser/corpus/3

@@ -0,0 +1 @@
+get a

ci/fuzz/redisparser/gen/main.go

@@ -0,0 +1,18 @@
+package main
+
+import (
+	"github.com/dvyukov/go-fuzz/gen"
+)
+
+var zdata = []string{
+	"*3\r\n$4\r\nMGET\r\n$4\r\nbaka\r\n$4\r\nkaba\r\n*5\r\n$4\r\nMSET\r\n$1\r\na\r\n$1\r\nb\r\n$3\r\neee\r\n$5\r\n12345\r\n*3\r\n$4\r\nMGET\r\n$4\r\nenen\r\n$4\r\nnime\r\n*2\r\n$3\r\nGET\r\n$5\r\nabcde\r\n*3\r\n$3\r\nDEL\r\n$1\r\na\r\n$1\r\nb\r\n",
+	"*3\r\n$5\r\nSETNX\r\n$1\r\na\r\n$10\r\nabcdeabcde\r\n",
+	"set a b\r\n",
+	"get a\r\n",
+}
+
+func main() {
+	for _, data := range zdata {
+		gen.Emit([]byte(data), nil, true)
+	}
+}

ci/fuzz/redisparser/redisparser.go

@@ -0,0 +1,114 @@
+package redisparser
+
+import (
+	"bytes"
+	"io"
+	"net"
+	"sync/atomic"
+	"time"
+
+	"overlord/pkg/bufio"
+	libnet "overlord/pkg/net"
+	"overlord/proxy/proto"
+	"overlord/proxy/proto/redis"
+)
+
+var (
+	pc proto.ProxyConn
+	msgs []*proto.Message
+	nc *libnet.Conn
+)
+
+func Fuzz(data []byte) int {
+	conn := _createConn(data)
+	nc := libnet.NewConn(conn, time.Second, time.Second)
+	pc = redis.NewProxyConn(nc)
+	msgs = proto.GetMsgs(4)
+	nmsgs, err := pc.Decode(msgs)
+	if err == bufio.ErrBufferFull {
+		return -1
+	}
+
+	if err != nil {
+		return 1
+	}
+
+	if len(nmsgs) >= 1 {
+		return 1
+	}
+
+	return 0
+}
+
+const (
+	stateClosed  = 1
+	stateOpening = 0
+)
+
+type mockAddr string
+
+func (m mockAddr) Network() string {
+	return "tcp"
+}
+func (m mockAddr) String() string {
+	return string(m)
+}
+
+type mockConn struct {
+	addr   mockAddr
+	buf   *bytes.Buffer
+	err    error
+	closed int32
+}
+
+func (m *mockConn) Read(b []byte) (n int, err error) {
+	if atomic.LoadInt32(&m.closed) == stateClosed {
+		return 0, io.EOF
+	}
+	if m.err != nil {
+		err = m.err
+		return
+	}
+	return m.buf.Read(b)
+}
+
+func (m *mockConn) Write(b []byte) (n int, err error) {
+	if atomic.LoadInt32(&m.closed) == stateClosed {
+		return 0, io.EOF
+	}
+
+	if m.err != nil {
+		err = m.err
+		return
+	}
+	return m.buf.Write(b)
+}
+
+// writeBuffers impl the net.buffersWriter to support writev
+func (m *mockConn) writeBuffers(buf *net.Buffers) (int64, error) {
+	if m.err != nil {
+		return 0, m.err
+	}
+	return buf.WriteTo(m.buf)
+}
+
+func (m *mockConn) Close() error {
+	atomic.StoreInt32(&m.closed, stateClosed)
+	return nil
+}
+
+func (m *mockConn) LocalAddr() net.Addr  { return m.addr }
+func (m *mockConn) RemoteAddr() net.Addr { return m.addr }
+
+func (m *mockConn) SetDeadline(t time.Time) error      { return nil }
+func (m *mockConn) SetReadDeadline(t time.Time) error  { return nil }
+func (m *mockConn) SetWriteDeadline(t time.Time) error { return nil }
+
+// _createConn is useful tools for handler test
+func _createConn(data []byte) net.Conn {
+	mconn := &mockConn{
+		addr:   "127.0.0.1:12345",
+		buf:   bytes.NewBuffer(data),
+	}
+	return mconn
+}