Fix goroutine leak caused by updating rate quotas (#11371)

Make sure that when we modify a rate quota, we stop the existing goroutine before starting the new one.

changelog/11371.txt

@@ -0,0 +1,3 @@
+```release-note:bug
+core: Fix goroutine leak when updating rate limit quota
+```

go.mod

@@ -156,6 +156,7 @@ require (
 	go.etcd.io/etcd v0.5.0-alpha.5.0.20200425165423-262c93980547
 	go.mongodb.org/mongo-driver v1.4.6
 	go.uber.org/atomic v1.6.0
+	go.uber.org/goleak v1.1.10
 	golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83
 	golang.org/x/net v0.0.0-20201110031124-69a78807bb2b
 	golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d

helper/metricsutil/wrapped_metrics.go

@@ -88,8 +88,9 @@ func (m *ClusterMetricSink) MeasureSinceWithLabels(key []string, start time.Time
 
 // BlackholeSink is a default suitable for use in unit tests.
 func BlackholeSink() *ClusterMetricSink {
-	sink, _ := metrics.New(metrics.DefaultConfig(""),
+	conf := metrics.DefaultConfig("")
-		&metrics.BlackholeSink{})
+	conf.EnableRuntimeMetrics = false
+	sink, _ := metrics.New(conf, &metrics.BlackholeSink{})
 	cms := &ClusterMetricSink{
 		ClusterName: atomic.Value{},
 		Sink:        sink,

vault/quotas/quotas.go

@@ -261,14 +261,16 @@ func NewManager(logger log.Logger, walkFunc leaseWalkFunc, ms *metricsutil.Clust
 	return manager, nil
 }
 
-// SetQuota adds a new quota rule to the db.
+// SetQuota adds or updates a quota rule.
 func (m *Manager) SetQuota(ctx context.Context, qType string, quota Quota, loading bool) error {
 	m.lock.Lock()
 	defer m.lock.Unlock()
 	return m.setQuotaLocked(ctx, qType, quota, loading)
 }
 
-// setQuotaLocked should be called with the manager's lock held
+// setQuotaLocked adds or updates a quota rule, modifying the db as well as
+// any runtime elements such as goroutines.
+// It should be called with the write lock held.
 func (m *Manager) setQuotaLocked(ctx context.Context, qType string, quota Quota, loading bool) error {
 	if qType == TypeLeaseCount.String() {
 		m.setIsPerfStandby(quota)
@@ -277,13 +279,17 @@ func (m *Manager) setQuotaLocked(ctx context.Context, qType string, quota Quota,
 	txn := m.db.Txn(true)
 	defer txn.Abort()
 
-	raw, err := txn.First(qType, "id", quota.quotaID())
+	raw, err := txn.First(qType, indexID, quota.quotaID())
 	if err != nil {
 		return err
 	}
 
 	// If there already exists an entry in the db, remove that first.
 	if raw != nil {
+		quota := raw.(Quota)
+		if err := quota.close(); err != nil {
+			return err
+		}
 		err = txn.Delete(qType, raw)
 		if err != nil {
 			return err

vault/quotas/quotas_rate_limit.go

@@ -319,6 +319,7 @@ func (rlq *RateLimitQuota) allow(req *Request) (Response, error) {
 }
 
 // close stops the current running client purge loop.
+// It should be called with the write lock held.
 func (rlq *RateLimitQuota) close() error {
 	if rlq.purgeBlocked {
 		close(rlq.closePurgeBlockedCh)

vault/quotas/quotas_rate_limit_test.go

@@ -1,6 +1,7 @@
 package quotas
 
 import (
+	"context"
 	"fmt"
 	"math"
 	"sync"
@@ -12,6 +13,7 @@ import (
 	"github.com/hashicorp/vault/sdk/helper/logging"
 	"github.com/stretchr/testify/require"
 	"go.uber.org/atomic"
+	"go.uber.org/goleak"
 )
 
 type clientResult struct {
@@ -34,6 +36,9 @@ func TestNewRateLimitQuota(t *testing.T) {
 		t.Run(tc.name, func(t *testing.T) {
 			err := tc.rlq.initialize(logging.NewVaultLogger(log.Trace), metricsutil.BlackholeSink())
 			require.Equal(t, tc.expectErr, err != nil, err)
+			if err == nil {
+				require.Nil(t, tc.rlq.close())
+			}
 		})
 	}
 }
@@ -61,6 +66,7 @@ func TestRateLimitQuota_Allow(t *testing.T) {
 	}
 
 	require.NoError(t, rlq.initialize(logging.NewVaultLogger(log.Trace), metricsutil.BlackholeSink()))
+	defer rlq.close()
 
 	var wg sync.WaitGroup
 
@@ -135,6 +141,7 @@ func TestRateLimitQuota_Allow_WithBlock(t *testing.T) {
 	}
 
 	require.NoError(t, rlq.initialize(logging.NewVaultLogger(log.Trace), metricsutil.BlackholeSink()))
+	defer rlq.close()
 	require.True(t, rlq.getPurgeBlocked())
 
 	var wg sync.WaitGroup
@@ -204,3 +211,15 @@ func TestRateLimitQuota_Allow_WithBlock(t *testing.T) {
 		}
 	}()
 }
+
+func TestRateLimitQuota_Update(t *testing.T) {
+	defer goleak.VerifyNone(t)
+	qm, err := NewManager(logging.NewVaultLogger(log.Trace), nil, metricsutil.BlackholeSink())
+	require.NoError(t, err)
+
+	quota := NewRateLimitQuota("quota1", "", "", 10, time.Second, 0)
+	require.NoError(t, qm.SetQuota(context.Background(), TypeRateLimit.String(), quota, true))
+	require.NoError(t, qm.SetQuota(context.Background(), TypeRateLimit.String(), quota, true))
+
+	require.Nil(t, quota.close())
+}

vendor/go.uber.org/goleak/.gitignore

@@ -0,0 +1,5 @@
+vendor/
+/bin
+/lint.log
+/cover.out
+/cover.html

vendor/go.uber.org/goleak/.travis.yml

@@ -0,0 +1,25 @@
+sudo: false
+language: go
+go_import_path: go.uber.org/goleak
+
+env:
+  global:
+    - GO111MODULE=on
+
+matrix:
+  include:
+  - go: 1.12.x
+  - go: 1.13.x
+  - go: 1.14.x
+    env: LINT=1
+
+install:
+  - make install
+
+script:
+  - test -z "$LINT" || make lint
+  - make test
+
+after_success:
+  - make cover
+  - bash <(curl -s https://codecov.io/bash)

vendor/go.uber.org/goleak/CHANGELOG.md

@@ -0,0 +1,24 @@
+# Changelog
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
+and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
+
+## [1.1.0]
+### Added
+- [#49]: Add option to ignore current goroutines, which checks for any additional leaks and allows for incremental adoption of goleak in larger projects.
+
+Thanks to @denis-tingajkin for their contributions to this release.
+
+## [1.0.0]
+### Changed
+- Migrate to Go modules.
+
+### Fixed
+- Ignore trace related goroutines that cause false positives with -trace.
+
+## 0.10.0
+- Initial release.
+
+[1.0.0]: https://github.com/uber-go/goleak/compare/v0.10.0...v1.0.0
+[#49]: https://github.com/uber-go/goleak/pull/49

vendor/go.uber.org/goleak/LICENSE

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2018 Uber Technologies, Inc.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

vendor/go.uber.org/goleak/Makefile

@@ -0,0 +1,41 @@
+export GOBIN ?= $(shell pwd)/bin
+
+GOLINT = $(GOBIN)/golint
+
+GO_FILES := $(shell \
+	find . '(' -path '*/.*' -o -path './vendor' ')' -prune \
+	-o -name '*.go' -print | cut -b3-)
+
+.PHONY: build
+build:
+	go build ./...
+
+.PHONY: install
+install:
+	go mod download
+
+.PHONY: test
+test:
+	go test -v -race ./...
+	go test -v -trace=/dev/null .
+
+.PHONY: cover
+cover:
+	go test -race -coverprofile=cover.out -coverpkg=./... ./...
+	go tool cover -html=cover.out -o cover.html
+
+$(GOLINT):
+	go install golang.org/x/lint/golint
+
+.PHONY: lint
+lint: $(GOLINT)
+	@rm -rf lint.log
+	@echo "Checking formatting..."
+	@gofmt -d -s $(GO_FILES) 2>&1 | tee lint.log
+	@echo "Checking vet..."
+	@go vet ./... 2>&1 | tee -a lint.log
+	@echo "Checking lint..."
+	@$(GOLINT) ./... 2>&1 | tee -a lint.log
+	@echo "Checking for unresolved FIXMEs..."
+	@git grep -i fixme | grep -v -e '^vendor/' -e '^Makefile' | tee -a lint.log
+	@[ ! -s lint.log ]

vendor/go.uber.org/goleak/README.md

@@ -0,0 +1,71 @@
+# goleak [![GoDoc][doc-img]][doc] [![Build Status][ci-img]][ci] [![Coverage Status][cov-img]][cov]
+
+Goroutine leak detector to help avoid Goroutine leaks.
+
+## Installation
+
+You can use `go get` to get the latest version:
+
+`go get -u go.uber.org/goleak`
+
+`goleak` also supports semver releases. It is compatible with Go 1.5+.
+
+## Quick Start
+
+To verify that there are no unexpected goroutines running at the end of a test:
+
+```go
+func TestA(t *testing.T) {
+	defer goleak.VerifyNone(t)
+
+	// test logic here.
+}
+```
+
+Instead of checking for leaks at the end of every test, `goleak` can also be run
+at the end of every test package by creating a `TestMain` function for your
+package:
+
+```go
+func TestMain(m *testing.M) {
+	goleak.VerifyTestMain(m)
+}
+```
+
+## Determine Source of Package Leaks
+
+When verifying leaks using `TestMain`, the leak test is only run once after all tests
+have been run. This is typically enough to ensure there's no goroutines leaked from
+tests, but when there are leaks, it's hard to determine which test is causing them.
+
+You can use the following bash script to determine the source of the failing test:
+
+```sh
+# Create a test binary which will be used to run each test individually
+$ go test -c -o tests
+
+# Run each test individually, printing "." for successful tests, or the test name
+# for failing tests.
+$ for test in $(go test -list . | grep -E "^(Test|Example)"); do ./tests -test.run "^$test\$" &>/dev/null && echo -n "." || echo "\n$test failed"; done
+```
+
+This will only print names of failing tests which can be investigated individually. E.g.,
+
+```
+.....
+TestLeakyTest failed
+.......
+```
+
+## Stability
+
+goleak is v1 and follows [SemVer](http://semver.org/) strictly.
+
+No breaking changes will be made to exported APIs before 2.0.
+
+[doc-img]: https://godoc.org/go.uber.org/goleak?status.svg
+[doc]: https://godoc.org/go.uber.org/goleak
+[ci-img]: https://travis-ci.com/uber-go/goleak.svg?branch=master
+[ci]: https://travis-ci.com/uber-go/goleak
+[cov-img]: https://codecov.io/gh/uber-go/goleak/branch/master/graph/badge.svg
+[cov]: https://codecov.io/gh/uber-go/goleak

vendor/go.uber.org/goleak/doc.go

@@ -0,0 +1,22 @@
+// Copyright (c) 2018 Uber Technologies, Inc.
+
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+// THE SOFTWARE.
+
+// Package goleak is a Goroutine leak detector.
+package goleak // import "go.uber.org/goleak"

vendor/go.uber.org/goleak/glide.yaml

@@ -0,0 +1,8 @@
+package: go.uber.org/goleak
+import: []
+testImport:
+- package: github.com/stretchr/testify
+  version: ^1.1.4
+  subpackages:
+  - assert
+  - require

vendor/go.uber.org/goleak/go.mod

@@ -0,0 +1,11 @@
+module go.uber.org/goleak
+
+go 1.13
+
+require (
+	github.com/kr/pretty v0.1.0 // indirect
+	github.com/stretchr/testify v1.4.0
+	golang.org/x/lint v0.0.0-20190930215403-16217165b5de
+	golang.org/x/tools v0.0.0-20191108193012-7d206e10da11 // indirect
+	gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect
+)

vendor/go.uber.org/goleak/go.sum

@@ -0,0 +1,30 @@
+github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
+github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
+github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/lint v0.0.0-20190930215403-16217165b5de h1:5hukYrvBGR8/eNkX5mdUezrA6JiaEZDtJb9Ei+1LlBs=
+golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20191108193012-7d206e10da11 h1:Yq9t9jnGoR+dBuitxdo9l6Q7xh/zOyNnYUtDKaQ3x0E=
+golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

vendor/go.uber.org/goleak/internal/stack/stacks.go

@@ -0,0 +1,155 @@
+// Copyright (c) 2017 Uber Technologies, Inc.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+// THE SOFTWARE.
+
+package stack
+
+import (
+	"bufio"
+	"bytes"
+	"fmt"
+	"io"
+	"runtime"
+	"strconv"
+	"strings"
+)
+
+const _defaultBufferSize = 64 * 1024 // 64 KiB
+
+// Stack represents a single Goroutine's stack.
+type Stack struct {
+	id            int
+	state         string
+	firstFunction string
+	fullStack     *bytes.Buffer
+}
+
+// ID returns the goroutine ID.
+func (s Stack) ID() int {
+	return s.id
+}
+
+// State returns the Goroutine's state.
+func (s Stack) State() string {
+	return s.state
+}
+
+// Full returns the full stack trace for this goroutine.
+func (s Stack) Full() string {
+	return s.fullStack.String()
+}
+
+// FirstFunction returns the name of the first function on the stack.
+func (s Stack) FirstFunction() string {
+	return s.firstFunction
+}
+
+func (s Stack) String() string {
+	return fmt.Sprintf(
+		"Goroutine %v in state %v, with %v on top of the stack:\n%s",
+		s.id, s.state, s.firstFunction, s.Full())
+}
+
+func getStacks(all bool) []Stack {
+	var stacks []Stack
+
+	var curStack *Stack
+	stackReader := bufio.NewReader(bytes.NewReader(getStackBuffer(all)))
+	for {
+		line, err := stackReader.ReadString('\n')
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			// We're reading using bytes.NewReader which should never fail.
+			panic("bufio.NewReader failed on a fixed string")
+		}
+
+		// If we see the goroutine header, start a new stack.
+		isFirstLine := false
+		if strings.HasPrefix(line, "goroutine ") {
+			// flush any previous stack
+			if curStack != nil {
+				stacks = append(stacks, *curStack)
+			}
+			id, goState := parseGoStackHeader(line)
+			curStack = &Stack{
+				id:        id,
+				state:     goState,
+				fullStack: &bytes.Buffer{},
+			}
+			isFirstLine = true
+		}
+		curStack.fullStack.WriteString(line)
+		if !isFirstLine && curStack.firstFunction == "" {
+			curStack.firstFunction = parseFirstFunc(line)
+		}
+	}
+
+	if curStack != nil {
+		stacks = append(stacks, *curStack)
+	}
+	return stacks
+}
+
+// All returns the stacks for all running goroutines.
+func All() []Stack {
+	return getStacks(true)
+}
+
+// Current returns the stack for the current goroutine.
+func Current() Stack {
+	return getStacks(false)[0]
+}
+
+func getStackBuffer(all bool) []byte {
+	for i := _defaultBufferSize; ; i *= 2 {
+		buf := make([]byte, i)
+		if n := runtime.Stack(buf, all); n < i {
+			return buf[:n]
+		}
+	}
+}
+
+func parseFirstFunc(line string) string {
+	line = strings.TrimSpace(line)
+	if idx := strings.LastIndex(line, "("); idx > 0 {
+		return line[:idx]
+	}
+	panic(fmt.Sprintf("function calls missing parents: %q", line))
+}
+
+// parseGoStackHeader parses a stack header that looks like:
+// goroutine 643 [runnable]:\n
+// And returns the goroutine ID, and the state.
+func parseGoStackHeader(line string) (goroutineID int, state string) {
+	line = strings.TrimSuffix(line, ":\n")
+	parts := strings.SplitN(line, " ", 3)
+	if len(parts) != 3 {
+		panic(fmt.Sprintf("unexpected stack header format: %q", line))
+	}
+
+	id, err := strconv.Atoi(parts[1])
+	if err != nil {
+		panic(fmt.Sprintf("failed to parse goroutine ID: %v in line %q", parts[1], line))
+	}
+
+	state = strings.TrimSuffix(strings.TrimPrefix(parts[2], "["), "]")
+	return id, state
+}

vendor/go.uber.org/goleak/leaks.go

@@ -0,0 +1,80 @@
+// Copyright (c) 2017 Uber Technologies, Inc.
+
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+// THE SOFTWARE.
+
+package goleak
+
+import (
+	"fmt"
+
+	"go.uber.org/goleak/internal/stack"
+)
+
+// TestingT is the minimal subset of testing.TB that we use.
+type TestingT interface {
+	Error(...interface{})
+}
+
+// filterStacks will filter any stacks excluded by the given opts.
+// filterStacks modifies the passed in stacks slice.
+func filterStacks(stacks []stack.Stack, skipID int, opts *opts) []stack.Stack {
+	filtered := stacks[:0]
+	for _, stack := range stacks {
+		// Always skip the running goroutine.
+		if stack.ID() == skipID {
+			continue
+		}
+		// Run any default or user-specified filters.
+		if opts.filter(stack) {
+			continue
+		}
+		filtered = append(filtered, stack)
+	}
+	return filtered
+}
+
+// Find looks for extra goroutines, and returns a descriptive error if
+// any are found.
+func Find(options ...Option) error {
+	cur := stack.Current().ID()
+
+	opts := buildOpts(options...)
+	var stacks []stack.Stack
+	retry := true
+	for i := 0; retry; i++ {
+		stacks = filterStacks(stack.All(), cur, opts)
+
+		if len(stacks) == 0 {
+			return nil
+		}
+		retry = opts.retry(i)
+	}
+
+	return fmt.Errorf("found unexpected goroutines:\n%s", stacks)
+}
+
+// VerifyNone marks the given TestingT as failed if any extra goroutines are
+// found by Find. This is a helper method to make it easier to integrate in
+// tests by doing:
+// 	defer VerifyNone(t)
+func VerifyNone(t TestingT, options ...Option) {
+	if err := Find(options...); err != nil {
+		t.Error(err)
+	}
+}

vendor/go.uber.org/goleak/options.go

@@ -0,0 +1,164 @@
+// Copyright (c) 2017 Uber Technologies, Inc.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+// THE SOFTWARE.
+
+package goleak
+
+import (
+	"strings"
+	"time"
+
+	"go.uber.org/goleak/internal/stack"
+)
+
+// Option lets users specify custom verifications.
+type Option interface {
+	apply(*opts)
+}
+
+// We retry up to 20 times if we can't find the goroutine that
+// we are looking for. In between each attempt, we will sleep for
+// a short while to let any running goroutines complete.
+const _defaultRetries = 20
+
+type opts struct {
+	filters    []func(stack.Stack) bool
+	maxRetries int
+	maxSleep   time.Duration
+}
+
+// optionFunc lets us easily write options without a custom type.
+type optionFunc func(*opts)
+
+func (f optionFunc) apply(opts *opts) { f(opts) }
+
+// IgnoreTopFunction ignores any goroutines where the specified function
+// is at the top of the stack. The function name should be fully qualified,
+// e.g., go.uber.org/goleak.IgnoreTopFunction
+func IgnoreTopFunction(f string) Option {
+	return addFilter(func(s stack.Stack) bool {
+		return s.FirstFunction() == f
+	})
+}
+
+// IgnoreCurrent records all current goroutines when the option is created, and ignores
+// them in any future Find/Verify calls.
+func IgnoreCurrent() Option {
+	excludeIDSet := map[int]bool{}
+	for _, s := range stack.All() {
+		excludeIDSet[s.ID()] = true
+	}
+	return addFilter(func(s stack.Stack) bool {
+		return excludeIDSet[s.ID()]
+	})
+}
+
+func maxSleep(d time.Duration) Option {
+	return optionFunc(func(opts *opts) {
+		opts.maxSleep = d
+	})
+}
+
+func addFilter(f func(stack.Stack) bool) Option {
+	return optionFunc(func(opts *opts) {
+		opts.filters = append(opts.filters, f)
+	})
+}
+
+func buildOpts(options ...Option) *opts {
+	opts := &opts{
+		maxRetries: _defaultRetries,
+		maxSleep:   100 * time.Millisecond,
+	}
+	opts.filters = append(opts.filters,
+		isTestStack,
+		isSyscallStack,
+		isStdLibStack,
+		isTraceStack,
+	)
+	for _, option := range options {
+		option.apply(opts)
+	}
+	return opts
+}
+
+func (vo *opts) filter(s stack.Stack) bool {
+	for _, filter := range vo.filters {
+		if filter(s) {
+			return true
+		}
+	}
+	return false
+}
+
+func (vo *opts) retry(i int) bool {
+	if i >= vo.maxRetries {
+		return false
+	}
+
+	d := time.Duration(int(time.Microsecond) << uint(i))
+	if d > vo.maxSleep {
+		d = vo.maxSleep
+	}
+	time.Sleep(d)
+	return true
+}
+
+// isTestStack is a default filter installed to automatically skip goroutines
+// that the testing package runs while the user's tests are running.
+func isTestStack(s stack.Stack) bool {
+	// Until go1.7, the main goroutine ran RunTests, which started
+	// the test in a separate goroutine and waited for that test goroutine
+	// to end by waiting on a channel.
+	// Since go1.7, a separate goroutine is started to wait for signals.
+	// T.Parallel is for parallel tests, which are blocked until all serial
+	// tests have run with T.Parallel at the top of the stack.
+	switch s.FirstFunction() {
+	case "testing.RunTests", "testing.(*T).Run", "testing.(*T).Parallel":
+		// In pre1.7 and post-1.7, background goroutines started by the testing
+		// package are blocked waiting on a channel.
+		return strings.HasPrefix(s.State(), "chan receive")
+	}
+	return false
+}
+
+func isSyscallStack(s stack.Stack) bool {
+	// Typically runs in the background when code uses CGo:
+	// https://github.com/golang/go/issues/16714
+	return s.FirstFunction() == "runtime.goexit" && strings.HasPrefix(s.State(), "syscall")
+}
+
+func isStdLibStack(s stack.Stack) bool {
+	// Importing os/signal starts a background goroutine.
+	// The name of the function at the top has changed between versions.
+	if f := s.FirstFunction(); f == "os/signal.signal_recv" || f == "os/signal.loop" {
+		return true
+	}
+
+	// Using signal.Notify will start a runtime goroutine.
+	return strings.Contains(s.Full(), "runtime.ensureSigM")
+}
+
+func isTraceStack(s stack.Stack) bool {
+	if f := s.FirstFunction(); f != "runtime.goparkunlock" {
+		return false
+	}
+
+	return strings.Contains(s.Full(), "runtime.ReadTrace")
+}

vendor/go.uber.org/goleak/testmain.go

@@ -0,0 +1,63 @@
+// Copyright (c) 2017 Uber Technologies, Inc.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+// THE SOFTWARE.
+
+package goleak
+
+import (
+	"fmt"
+	"io"
+	"os"
+)
+
+// Variables for stubbing in unit tests.
+var (
+	_osExit             = os.Exit
+	_osStderr io.Writer = os.Stderr
+)
+
+// TestingM is the minimal subset of testing.M that we use.
+type TestingM interface {
+	Run() int
+}
+
+// VerifyTestMain can be used in a TestMain function for package tests to
+// verify that there were no goroutine leaks.
+// To use it, your TestMain function should look like:
+//
+//  func TestMain(m *testing.M) {
+//    goleak.VerifyTestMain(m)
+//  }
+//
+// See https://golang.org/pkg/testing/#hdr-Main for more details.
+//
+// This will run all tests as per normal, and if they were successful, look
+// for any goroutine leaks and fail the tests if any leaks were found.
+func VerifyTestMain(m TestingM, options ...Option) {
+	exitCode := m.Run()
+
+	if exitCode == 0 {
+		if err := Find(options...); err != nil {
+			fmt.Fprintf(_osStderr, "goleak: Errors on successful test run: %v\n", err)
+			exitCode = 1
+		}
+	}
+
+	_osExit(exitCode)
+}

vendor/go.uber.org/goleak/tools.go

@@ -0,0 +1,28 @@
+// Copyright (c) 2019 Uber Technologies, Inc.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+// THE SOFTWARE.
+
+// +build tools
+
+package goleak
+
+import (
+	// Tools we use during development.
+	_ "golang.org/x/lint/golint"
+)

vendor/modules.txt

@@ -1270,6 +1270,10 @@ go.opencensus.io/trace/tracestate
 # go.uber.org/atomic v1.6.0
 ## explicit
 go.uber.org/atomic
+# go.uber.org/goleak v1.1.10
+## explicit
+go.uber.org/goleak
+go.uber.org/goleak/internal/stack
 # go.uber.org/multierr v1.5.0
 go.uber.org/multierr
 # go.uber.org/zap v1.14.1