chore(test): setup a "flappy tests" system (#201)

.github/workflows/unit-tests.yml

@@ -26,6 +26,7 @@ jobs:
           - test.files2
           - test.packages
           - test.examples
+          - test.flappy
     runs-on: ubuntu-latest
     timeout-minutes: 21
     steps:

Makefile

@@ -55,19 +55,24 @@ examples.build: install_gnodev examples.precompile
 
 ########################################
 # Test suite
-.PHONY: test test.go test.gno test.files1 test.files2 test.realm test.packages
-test: test.gno test.go
+.PHONY: test test.go test.gno test.files1 test.files2 test.realm test.packages test.flappy
+test: test.gno test.go test.flappy
 	@echo "Full test suite finished."
 
 test.gno: test.files1 test.files2 test.realm test.packages test.examples
 	go test tests/*.go -v -run "TestFileStr"
 	go test tests/*.go -v -run "TestSelectors"
 
+test.flappy:
+	# flappy tests should work "sometimes" (at least once)
+	TEST_STABILITY=flappy go run -modfile ./misc/devdeps/go.mod moul.io/testman test -test.v -timeout=20m -retry=10 -run ^TestFlappy \
+		./pkgs/bft/consensus ./pkgs/bft/blockchain ./pkgs/bft/mempool ./pkgs/p2p
+
 test.go:
 	go test . -v
 	# -p 1 shows test failures as they come
 	# maybe another way to do this?
-	go test ./pkgs/... -v -p 1
+	go test ./pkgs/... -v -p 1 -timeout=20m
 
 test.files1:
 	go test tests/*.go -v -test.short -run "TestFiles1/" --timeout 20m
@@ -104,5 +109,5 @@ genproto2:
 	find pkgs | grep -v "^pkgs\/amino" | grep "pbbindings" | xargs rm
 	find pkgs | grep -v "^pkgs\/amino" | grep "pb.go" | xargs rm
 	#@rm gno.proto || true
-	@rm pbbindings.go || true 
+	@rm pbbindings.go || true
 	@rm gno.pb.go || true

go.mod

@@ -6,6 +6,7 @@ require (
 	github.com/btcsuite/btcd v0.20.1-beta
 	github.com/btcsuite/btcutil v1.0.2
 	github.com/davecgh/go-spew v1.1.1
+	github.com/dgraph-io/badger/v3 v3.2103.2
 	github.com/fortytw2/leaktest v1.3.0
 	github.com/gdamore/tcell/v2 v2.1.0
 	github.com/gnolang/cors v1.8.1
@@ -18,6 +19,7 @@ require (
 	github.com/jaekwon/testify v1.6.1
 	github.com/jmhodges/levigo v1.0.0
 	github.com/libp2p/go-buffer-pool v0.0.2
+	github.com/linxGnu/grocksdb v1.7.2
 	github.com/mattn/go-runewidth v0.0.10
 	github.com/pelletier/go-toml v1.9.3
 	github.com/stretchr/testify v1.7.1
@@ -34,23 +36,34 @@ require (
 )
 
 require (
+	github.com/cespare/xxhash v1.1.0 // indirect
+	github.com/cespare/xxhash/v2 v2.1.1 // indirect
+	github.com/dgraph-io/ristretto v0.1.0 // indirect
+	github.com/dustin/go-humanize v1.0.0 // indirect
 	github.com/facebookgo/ensure v0.0.0-20200202191622-63f1cf65ac4c // indirect
 	github.com/facebookgo/stack v0.0.0-20160209184415-751773369052 // indirect
 	github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4 // indirect
 	github.com/gdamore/encoding v1.0.0 // indirect
-	github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db // indirect
+	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b // indirect
+	github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6 // indirect
+	github.com/golang/snappy v0.0.3 // indirect
+	github.com/google/flatbuffers v1.12.1 // indirect
 	github.com/gorilla/securecookie v1.1.1 // indirect
 	github.com/gorilla/sessions v1.2.1 // indirect
+	github.com/klauspost/compress v1.12.3 // indirect
 	github.com/kr/text v0.2.0 // indirect
 	github.com/lucasb-eyer/go-colorful v1.0.3 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/rivo/uniseg v0.1.0 // indirect
+	go.opencensus.io v0.22.5 // indirect
 	go.uber.org/atomic v1.7.0 // indirect
 	golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1 // indirect
 	golang.org/x/text v0.3.6 // indirect
 	gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
-	gopkg.in/yaml.v2 v2.2.2 // indirect
-	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
+	gopkg.in/yaml.v2 v2.2.4 // indirect
+	gopkg.in/yaml.v3 v3.0.0-20220512140231-539c8e751b99 // indirect
 )
 
 replace github.com/gdamore/tcell/v2 => github.com/gnolang/tcell/v2 v2.1.0

misc/devdeps/README.md

@@ -0,0 +1 @@
+This folder contains configuration of various development dependencies.

misc/devdeps/deps.go

@@ -0,0 +1,11 @@
+//go:build tools
+// +build tools
+
+package gno
+
+// this file is used to allow "go mod" to be aware of some development dependencies.
+
+import (
+	// required by Makefile for flappy tests
+	_ "moul.io/testman"
+)

misc/devdeps/go.mod

@@ -0,0 +1,12 @@
+module github.com/gnolang/gno/misc/devops
+
+go 1.18
+
+require moul.io/testman v1.5.0
+
+require (
+	github.com/peterbourgon/ff/v3 v3.0.0 // indirect
+	moul.io/banner v1.0.1 // indirect
+	moul.io/motd v1.0.0 // indirect
+	moul.io/u v1.6.0 // indirect
+)

misc/devdeps/go.sum

@@ -0,0 +1,52 @@
+github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls=
+github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
+github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=
+github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
+github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/pelletier/go-toml v1.6.0/go.mod h1:5N711Q9dKgbdkxHL+MEfF31hpT7l0S0s/t2kKREewys=
+github.com/peterbourgon/ff/v3 v3.0.0 h1:eQzEmNahuOjQXfuegsKQTSTDbf4dNvr/eNLrmJhiH7M=
+github.com/peterbourgon/ff/v3 v3.0.0/go.mod h1:UILIFjRH5a/ar8TjXYLTkIvSvekZqPm5Eb/qbGk6CT0=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0=
+github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+go.uber.org/goleak v1.1.10 h1:z+mqJhf6ss6BSfSM671tgKyZBFPTTJM+HLxnhPC3wu0=
+go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/lint v0.0.0-20190930215403-16217165b5de h1:5hukYrvBGR8/eNkX5mdUezrA6JiaEZDtJb9Ei+1LlBs=
+golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20191108193012-7d206e10da11 h1:Yq9t9jnGoR+dBuitxdo9l6Q7xh/zOyNnYUtDKaQ3x0E=
+golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v3 v3.0.0-20190709130402-674ba3eaed22/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+moul.io/banner v1.0.1 h1:+WsemGLhj2pOajw2eR5VYjLhOIqs0XhIRYchzTyMLk0=
+moul.io/banner v1.0.1/go.mod h1:XwvIGKkhKRKyN1vIdmR5oaKQLIkMhkMqrsHpS94QzAU=
+moul.io/godev v1.7.0 h1:PgnL7BsCQPPjKwu9V0oxIVm2MyZAHAN2sl0S3+E37U0=
+moul.io/godev v1.7.0/go.mod h1:5lgSpI1oH7xWpLl2Ew/Nsgk8DiNM6FzN9WV9+lgW8RQ=
+moul.io/motd v1.0.0 h1:Trk4fPibDfPJf2iCBSQC8ws7Q02sMwivQdVEFAjCPto=
+moul.io/motd v1.0.0/go.mod h1:39rvZ0lC2oRhHDY2VoPyZ8r70VKqeJye3QAxjeLDJso=
+moul.io/testman v1.5.0 h1:tN1XEzLxYh8ZYy1wET6leWufnTl7BcZELkSNiro/yEo=
+moul.io/testman v1.5.0/go.mod h1:b4/5+lMsMDJtwuh25Cr0eVJ5Y4B2lSPfkzDtfct070g=
+moul.io/u v1.6.0 h1:hZwwppk8LZJMGb12ggEAMPfE27MWzhEVgFxgHxwtfMg=
+moul.io/u v1.6.0/go.mod h1:yd3/IoYRIJaZWAJV2rYHvM2EPp/Pp0zSNraB5IPX+hw=

pkgs/bft/blockchain/reactor_test.go

@@ -20,6 +20,7 @@ import (
 	"github.com/gnolang/gno/pkgs/errors"
 	"github.com/gnolang/gno/pkgs/log"
 	"github.com/gnolang/gno/pkgs/p2p"
+	"github.com/gnolang/gno/pkgs/testutils"
 )
 
 var config *cfg.Config
@@ -173,7 +174,9 @@ func TestNoBlockResponse(t *testing.T) {
 // or without significant refactoring of the module.
 // Alternatively we could actually dial a TCP conn but
 // that seems extreme.
-func TestBadBlockStopsPeer(t *testing.T) {
+func TestFlappyBadBlockStopsPeer(t *testing.T) {
+	testutils.FilterStability(t, testutils.Flappy)
+
 	config = cfg.ResetTestRoot("blockchain_reactor_test")
 	defer os.RemoveAll(config.RootDir)
 	genDoc, privVals := randGenesisDoc(1, false, 30)

pkgs/bft/consensus/replay_test.go

@@ -32,6 +32,7 @@ import (
 	"github.com/gnolang/gno/pkgs/events"
 	"github.com/gnolang/gno/pkgs/log"
 	"github.com/gnolang/gno/pkgs/random"
+	"github.com/gnolang/gno/pkgs/testutils"
 )
 
 func TestMain(m *testing.M) {
@@ -534,7 +535,9 @@ func TestHandshakeReplayOne(t *testing.T) {
 }
 
 // Sync from caught up
-func TestHandshakeReplayNone(t *testing.T) {
+func TestFlappyHandshakeReplayNone(t *testing.T) {
+	testutils.FilterStability(t, testutils.Flappy)
+
 	for _, m := range modes {
 		testHandshakeReplay(t, config, numBlocks, m, nil)
 	}

pkgs/bft/mempool/reactor_test.go

@@ -19,6 +19,7 @@ import (
 	"github.com/gnolang/gno/pkgs/p2p"
 	p2pcfg "github.com/gnolang/gno/pkgs/p2p/config"
 	"github.com/gnolang/gno/pkgs/p2p/mock"
+	"github.com/gnolang/gno/pkgs/testutils"
 )
 
 type peerState struct {
@@ -195,7 +196,9 @@ func TestBroadcastTxForPeerStopsWhenPeerStops(t *testing.T) {
 	leaktest.CheckTimeout(t, 10*time.Second)()
 }
 
-func TestBroadcastTxForPeerStopsWhenReactorStops(t *testing.T) {
+func TestFlappyBroadcastTxForPeerStopsWhenReactorStops(t *testing.T) {
+	testutils.FilterStability(t, testutils.Flappy)
+
 	if testing.Short() {
 		t.Skip("skipping test in short mode.")
 	}

pkgs/p2p/transport_test.go

@@ -11,6 +11,7 @@ import (
 	"github.com/gnolang/gno/pkgs/amino"
 	"github.com/gnolang/gno/pkgs/crypto/ed25519"
 	"github.com/gnolang/gno/pkgs/p2p/conn"
+	"github.com/gnolang/gno/pkgs/testutils"
 )
 
 var defaultNodeName = "host_peer"
@@ -210,7 +211,9 @@ func testDialer(dialAddr NetAddress, errc chan error) {
 	errc <- nil
 }
 
-func TestTransportMultiplexAcceptNonBlocking(t *testing.T) {
+func TestFlappyTransportMultiplexAcceptNonBlocking(t *testing.T) {
+	testutils.FilterStability(t, testutils.Flappy)
+
 	mt := testSetupMultiplexTransport(t)
 
 	var (

pkgs/testutils/flappy.go

@@ -0,0 +1,24 @@
+package testutils
+
+import (
+	"fmt"
+	"os"
+	"testing"
+)
+
+type StabilityMode string
+
+const (
+	Stable StabilityMode = "stable"
+	Flappy StabilityMode = "flappy"
+	Broken StabilityMode = "broken"
+)
+
+func FilterStability(t *testing.T, mode StabilityMode) {
+	t.Helper()
+
+	filter := os.Getenv("STABILITY_FILTER")
+	if filter != string(mode) {
+		t.Skip(fmt.Sprintf("skip test with %q stability", mode))
+	}
+}