diff --git a/.changelog/14441.txt b/.changelog/14441.txt new file mode 100644 index 000000000..1a3419981 --- /dev/null +++ b/.changelog/14441.txt @@ -0,0 +1,3 @@ +```release-note:improvement +autopilot: upgrade to raft-autopilot library +``` diff --git a/command/agent/operator_endpoint.go b/command/agent/operator_endpoint.go index 224ddabe2..4c8d278cd 100644 --- a/command/agent/operator_endpoint.go +++ b/command/agent/operator_endpoint.go @@ -2,21 +2,20 @@ package agent import ( "context" + "fmt" "io" "net" "net/http" - "strings" - - "fmt" "strconv" + "strings" "time" - "github.com/hashicorp/consul/agent/consul/autopilot" "github.com/hashicorp/go-msgpack/codec" + "github.com/hashicorp/raft" + "github.com/hashicorp/nomad/api" cstructs "github.com/hashicorp/nomad/client/structs" "github.com/hashicorp/nomad/nomad/structs" - "github.com/hashicorp/raft" ) // OperatorRequest is used route operator/raft API requests to the implementing @@ -184,7 +183,7 @@ func (s *HTTPServer) OperatorServerHealth(resp http.ResponseWriter, req *http.Re return nil, nil } - var reply autopilot.OperatorHealthReply + var reply structs.OperatorHealthReply if err := s.agent.RPC("Operator.ServerHealth", &args, &reply); err != nil { return nil, err } diff --git a/go.mod b/go.mod index 01f5c2d0f..a352fd879 100644 --- a/go.mod +++ b/go.mod @@ -40,7 +40,6 @@ require ( github.com/gorilla/websocket v1.5.0 github.com/gosuri/uilive v0.0.4 github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 - github.com/hashicorp/consul v1.7.8 github.com/hashicorp/consul-template v0.29.3-0.20220829190305-21d2c9bb9752 github.com/hashicorp/consul/api v1.14.0 github.com/hashicorp/consul/sdk v0.11.0 @@ -78,6 +77,7 @@ require ( github.com/hashicorp/net-rpc-msgpackrpc v0.0.0-20151116020338-a14192a58a69 github.com/hashicorp/nomad/api v0.0.0-20220829153708-e1e5bb1dcefb github.com/hashicorp/raft v1.3.9 + github.com/hashicorp/raft-autopilot v0.1.6 github.com/hashicorp/raft-boltdb/v2 v2.2.0 github.com/hashicorp/serf v0.9.7 github.com/hashicorp/vault/api v1.7.2 diff --git a/go.sum b/go.sum index 51cc87fed..79bdc0c74 100644 --- a/go.sum +++ b/go.sum @@ -55,7 +55,6 @@ github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8= github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/Azure/go-autorest v10.8.1+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= -github.com/Azure/go-autorest v10.15.3+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs= github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= github.com/Azure/go-autorest/autorest v0.9.0/go.mod h1:xyHB1BMZT0cuDHU7I0+g046+BFDTQ8rEZB0s4Yfa6bI= @@ -128,14 +127,12 @@ github.com/Microsoft/hcsshim v0.9.3/go.mod h1:7pLA8lDk46WKDWlVsENo92gC0XFa8rbKfy github.com/Microsoft/hcsshim/test v0.0.0-20201218223536-d3e5debf77da/go.mod h1:5hlzMzRKMLyo42nCZ9oml8AdTlq/0cvIaBv6tK1RehU= github.com/Microsoft/hcsshim/test v0.0.0-20210227013316-43a75bb4edd3/go.mod h1:mw7qgWloBUl75W/gVH3cQszUg1+gUITj7D6NY7ywVnY= github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= -github.com/NYTimes/gziphandler v1.0.1/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/PuerkitoBio/purell v1.0.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/Shopify/logrus-bugsnag v0.0.0-20171204204709-577dee27f20d/go.mod h1:HI8ITrYtUY+O+ZhtlqUnD8+KwNPOyugEhfP9fdUIaEQ= -github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= github.com/VividCortex/ewma v1.1.1 h1:MnEK4VOv6n0RSY4vtRe3h11qjxL3+t0B8yOL8iMXdcM= github.com/VividCortex/ewma v1.1.1/go.mod h1:2Tkkvm3sRDVXaiyucHiACn4cqf7DpdyLvmxzcbUokwA= github.com/abdullin/seq v0.0.0-20160510034733-d5467c17e7af h1:DBNMBMuMiWYu0b+8KMJuWmfCkcxl09JwdlqwDZZ6U14= @@ -162,7 +159,6 @@ github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hC github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878/go.mod h1:3AMJUQhVx52RsWOnlkpikZr01T/yAVN2gn0861vByNg= -github.com/armon/go-metrics v0.3.4/go.mod h1:4O98XIr/9W0sxpJ8UaYkvjk10Iff7SnFrb4QAOwNTFc= github.com/armon/go-metrics v0.3.9/go.mod h1:4O98XIr/9W0sxpJ8UaYkvjk10Iff7SnFrb4QAOwNTFc= github.com/armon/go-metrics v0.3.10/go.mod h1:4O98XIr/9W0sxpJ8UaYkvjk10Iff7SnFrb4QAOwNTFc= github.com/armon/go-metrics v0.4.0 h1:yCQqn7dwca4ITXb+CbubHmedzaQYHhNhrEXLYUeEe8Q= @@ -236,7 +232,6 @@ github.com/circonus-labs/circonusllhist v0.1.3 h1:TJH+oke8D16535+jHExHj4nQvzlZrj github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= -github.com/cncf/udpa/go v0.0.0-20200313221541-5f7e5dd04533/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4 h1:hzAQntlaYRkVSFEfj9OTWlVV1H155FMD8BTKktLv0QI= @@ -347,7 +342,6 @@ github.com/containernetworking/plugins v1.1.1/go.mod h1:Sr5TH/eBsGLXK/h71HeLfX19 github.com/containers/ocicrypt v1.0.1/go.mod h1:MeJDzk1RJHv89LjsH0Sp5KTY3ZYkjXO/C+bKAeWFIrc= github.com/containers/ocicrypt v1.1.0/go.mod h1:b8AOe0YR67uU8OqfVNcznfFpAzu3rdgUV4GP9qXPfu4= github.com/containers/ocicrypt v1.1.1/go.mod h1:Dm55fwWm1YZAjYRaJ94z2mfZikIyIN4B0oB3dj3jFxY= -github.com/coredns/coredns v1.1.2/go.mod h1:zASH/MVDgR6XZTbxvOnsZfffS+31vg6Ackf/wo1+AM0= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-iptables v0.4.5/go.mod h1:/mVI274lEDI2ns62jHCDnCyBF9Iwsmekav8Dbxlm1MU= @@ -411,7 +405,6 @@ github.com/docker/docker v20.10.17+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05 github.com/docker/docker-credential-helpers v0.6.3/go.mod h1:WRaJzqw3CTB9bk10avuGsjVBZsD05qeibJ1/TYlvc0Y= github.com/docker/docker-credential-helpers v0.6.4 h1:axCks+yV+2MR3/kZhAmy07yC56WZ2Pwu/fKWtKuZB0o= github.com/docker/docker-credential-helpers v0.6.4/go.mod h1:ofX3UI0Gz1TteYBjtgs07O36Pyasyp66D2uKT7H8W1c= -github.com/docker/go-connections v0.3.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ= github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= github.com/docker/go-events v0.0.0-20170721190031-9461782956ad/go.mod h1:Uw6UezgYA44ePAFQYUehOuCzmy5zmg/+nl2ZfMWGkpA= @@ -431,7 +424,6 @@ github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3 github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= -github.com/elazarl/go-bindata-assetfs v0.0.0-20160803192304-e1a2a7ec64b0/go.mod h1:v+YaWX3bdea5J/mo8dSETolEo7R71Vk1u8bnjau5yw4= github.com/elazarl/go-bindata-assetfs v1.0.1-0.20200509193318-234c15e7648f h1:AwZUiMWfYSmIiHdFJIubTSs8BFIFoMmUFbeuwBzHIPs= github.com/elazarl/go-bindata-assetfs v1.0.1-0.20200509193318-234c15e7648f/go.mod h1:v+YaWX3bdea5J/mo8dSETolEo7R71Vk1u8bnjau5yw4= github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= @@ -442,7 +434,6 @@ github.com/endocrimes/go-winio v0.4.13-0.20190628114223-fb47a8b41948/go.mod h1:q github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= -github.com/envoyproxy/go-control-plane v0.9.5/go.mod h1:OXl5to++W0ctG+EHWTFUjiypVxC/Y4VLc/KFU+al13s= github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= @@ -486,14 +477,12 @@ github.com/go-ini/ini v1.25.4/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3I github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= -github.com/go-ldap/ldap v3.0.2+incompatible/go.mod h1:qfd9rJvER9Q0/D/Sqn1DfHRoBp40uXYvFoEVrNEPqRc= github.com/go-ldap/ldap/v3 v3.1.10/go.mod h1:5Zun81jBTabRaI8lzN7E1JjyEl1g6zI6u9pd8luAK4Q= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= -github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dTyBNF8= github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-openapi/jsonpointer v0.0.0-20160704185906-46af16f9f7b1/go.mod h1:+35s3my2LFTysnkMfxsJBAMHj/DoqoB9knIWoYG/Vk0= @@ -509,7 +498,6 @@ github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= -github.com/go-test/deep v1.0.2-0.20181118220953-042da051cf31/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA= github.com/go-test/deep v1.0.2/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA= github.com/go-test/deep v1.0.3 h1:ZrJSEWsXzPOxaZnFteGEfooLba+ju3FYIbOrS+rQd68= github.com/go-test/deep v1.0.3/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA= @@ -573,7 +561,6 @@ github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaS github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= @@ -665,14 +652,10 @@ github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgf github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= -github.com/hashicorp/consul v1.7.8 h1:hp308KxAf3zWoGuwp2e+0UUhrm6qHjeBQk3jCZ+bjcY= -github.com/hashicorp/consul v1.7.8/go.mod h1:urbfGaVZDmnXC6geg0LYPh/SRUk1E8nfmDHpz+Q0nLw= github.com/hashicorp/consul-template v0.29.3-0.20220829190305-21d2c9bb9752 h1:VjEbNw/ZtuaQRz3HOHIeinO7qZKX3XIPO33A9tIcsZI= github.com/hashicorp/consul-template v0.29.3-0.20220829190305-21d2c9bb9752/go.mod h1:aiT2d9ReQd7VtFZJELlt1SfEOiiRRpca9Ot/jcyWQps= -github.com/hashicorp/consul/api v1.4.0/go.mod h1:xc8u05kyMa3Wjr9eEAsIAo3dg8+LywT5E/Cl7cNS5nU= github.com/hashicorp/consul/api v1.14.0 h1:Y64GIJ8hYTu+tuGekwO4G4ardXoiCivX9wv1iP/kihk= github.com/hashicorp/consul/api v1.14.0/go.mod h1:bcaw5CSZ7NE9qfOfKCI1xb7ZKjzu/MyvQkCLTfqLqxQ= -github.com/hashicorp/consul/sdk v0.4.0/go.mod h1:fY08Y9z5SvJqevyZNy6WWPXiG3KwBPAvlcdx16zZ0fM= github.com/hashicorp/consul/sdk v0.10.0/go.mod h1:yPkX5Q6CsxTFMjQQDJwzeNmUUF5NUGGbrDsv9wTb8cw= github.com/hashicorp/consul/sdk v0.11.0 h1:HRzj8YSCln2yGgCumN5CL8lYlD3gBurnervJRJAZyC4= github.com/hashicorp/consul/sdk v0.11.0/go.mod h1:yPkX5Q6CsxTFMjQQDJwzeNmUUF5NUGGbrDsv9wTb8cw= @@ -682,7 +665,6 @@ github.com/hashicorp/errwrap v0.0.0-20141028054710-7554cd9344ce/go.mod h1:YH+1FK github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= -github.com/hashicorp/go-bexpr v0.1.2/go.mod h1:ANbpTX1oAql27TZkKVeW8p1w8NTdnyzPe/0qqPCKohU= github.com/hashicorp/go-bexpr v0.1.11 h1:6DqdA/KBjurGby9yTY0bmkathya0lfwF2SeuubCI7dY= github.com/hashicorp/go-bexpr v0.1.11/go.mod h1:f03lAo0duBlDIUMGCuad8oLcgejw4m7U+N8T+6Kz1AE= github.com/hashicorp/go-checkpoint v0.0.0-20171009173528-1545e56e46de h1:XDCSythtg8aWSRSO29uwhgh7b127fWr+m5SemqjSUL8= @@ -691,7 +673,6 @@ github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtng github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= -github.com/hashicorp/go-connlimit v0.2.0/go.mod h1:OUj9FGL1tPIhl/2RCfzYHrIiWj+VVPGNyVPnUX8AqS0= github.com/hashicorp/go-connlimit v0.3.0 h1:oAojHGjFxUTTTA8c5XXnDqWJ2HLuWbDiBPTpWvNzvqM= github.com/hashicorp/go-connlimit v0.3.0/go.mod h1:OUj9FGL1tPIhl/2RCfzYHrIiWj+VVPGNyVPnUX8AqS0= github.com/hashicorp/go-cty-funcs v0.0.0-20200930094925-2721b1e36840 h1:kgvybwEeu0SXktbB2y3uLHX9lklLo+nzUwh59A3jzQc= @@ -703,8 +684,6 @@ github.com/hashicorp/go-envparse v0.0.0-20180119215841-310ca1881b22/go.mod h1:/N github.com/hashicorp/go-gatedio v0.5.0 h1:Jm1X5yP4yCqqWj5L1TgW7iZwCVPGtVc+mro5r/XX7Tg= github.com/hashicorp/go-getter v1.6.1 h1:NASsgP4q6tL94WH6nJxKWj8As2H/2kop/bB1d8JMyRY= github.com/hashicorp/go-getter v1.6.1/go.mod h1:IZCrswsZPeWv9IkVnLElzRU/gz/QPi6pZHn4tv6vbwA= -github.com/hashicorp/go-hclog v0.0.0-20180709165350-ff2cf002a8dd/go.mod h1:9bjs9uLqI8l75knNv3lV1kA55veR+WUPSiKIWcQHudI= -github.com/hashicorp/go-hclog v0.8.0/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= github.com/hashicorp/go-hclog v0.9.1/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= github.com/hashicorp/go-hclog v0.12.0/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= @@ -713,15 +692,12 @@ github.com/hashicorp/go-hclog v0.16.2/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39 github.com/hashicorp/go-hclog v1.2.2 h1:ihRI7YFwcZdiSD7SIenIhHfQH3OuDvWerAUBZbeQS3M= github.com/hashicorp/go-hclog v1.2.2/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M= github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= -github.com/hashicorp/go-immutable-radix v1.1.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= -github.com/hashicorp/go-immutable-radix v1.2.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= github.com/hashicorp/go-immutable-radix v1.3.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= github.com/hashicorp/go-immutable-radix v1.3.1 h1:DKHmCUm2hRBK510BaiZlwvpD40f8bJFeZnpfm2KLowc= github.com/hashicorp/go-immutable-radix v1.3.1/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= github.com/hashicorp/go-kms-wrapping/entropy v0.1.0/go.mod h1:d1g9WGtAunDNpek8jUIEJnBlbgKS1N2Q61QkHiZyR1g= github.com/hashicorp/go-kms-wrapping/v2 v2.0.5 h1:rOFDv+3k05mnW0oaDLffhVUwg03Csn0mvfO98Wdd2bE= github.com/hashicorp/go-kms-wrapping/v2 v2.0.5/go.mod h1:sDQAfwJGv25uGPZA04x87ERglCG6avnRcBT9wYoMII8= -github.com/hashicorp/go-memdb v1.0.3/go.mod h1:LWQ8R70vPrS4OEY9k28D2z8/Zzyu34NVzeRibGAzHO0= github.com/hashicorp/go-memdb v1.3.3 h1:oGfEWrFuxtIUF3W2q/Jzt6G85TrMk9ey6XfYLvVe1Wo= github.com/hashicorp/go-memdb v1.3.3/go.mod h1:uBTr1oQbtuMgd1SSGoR8YV27eT3sBHbYiNm53bMpgSg= github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= @@ -733,16 +709,12 @@ github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHh github.com/hashicorp/go-multierror v1.1.0/go.mod h1:spPvp8C1qA32ftKqdAHm4hHTbPw+vmowP0z+KUhOZdA= github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= -github.com/hashicorp/go-plugin v1.0.1/go.mod h1:++UyYGoz3o5w9ZzAdZxtQKrWWP+iqPBn3cQptSMzBuY= github.com/hashicorp/go-plugin v1.4.3 h1:DXmvivbWD5qdiBts9TpBC7BYL1Aia5sxbRgQB+v6UZM= github.com/hashicorp/go-plugin v1.4.3/go.mod h1:5fGEH17QVwTTcR0zV7yhDPLLmFX9YSZ38b18Udy6vYQ= -github.com/hashicorp/go-raftchunking v0.6.1/go.mod h1:cGlg3JtDy7qy6c/3Bu660Mic1JF+7lWqIwCFSb08fX0= github.com/hashicorp/go-retryablehttp v0.5.3/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs= -github.com/hashicorp/go-retryablehttp v0.5.4/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs= github.com/hashicorp/go-retryablehttp v0.6.6/go.mod h1:vAew36LZh98gCBJNLH42IQ1ER/9wtLZZ8meHqQvEYWY= github.com/hashicorp/go-retryablehttp v0.7.0 h1:eu1EI/mbirUgP5C8hVsTNaGZreBDlYiwC1FZWkvQPQ4= github.com/hashicorp/go-retryablehttp v0.7.0/go.mod h1:vAew36LZh98gCBJNLH42IQ1ER/9wtLZZ8meHqQvEYWY= -github.com/hashicorp/go-rootcerts v1.0.1/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8= github.com/hashicorp/go-rootcerts v1.0.2 h1:jzhAVGtqPKbwpyCPELlgNWhE1znq+qwJtW5Oi2viEzc= github.com/hashicorp/go-rootcerts v1.0.2/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8= github.com/hashicorp/go-safetemp v1.0.0 h1:2HR189eFNrjHQyENnQMMpCiBAsRxzbTMIgBhEyExpmo= @@ -779,7 +751,6 @@ github.com/hashicorp/go-version v1.1.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09 github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mOkIeek= github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= -github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc= @@ -788,46 +759,38 @@ github.com/hashicorp/hcl v1.0.1-0.20201016140508-a07e7d50bbee h1:8B4HqvMUtYSjsGk github.com/hashicorp/hcl v1.0.1-0.20201016140508-a07e7d50bbee/go.mod h1:gwlu9+/P9MmKtYrMsHeFRZPXj2CTPm11TDnMeaRHS7g= github.com/hashicorp/hcl/v2 v2.9.2-0.20220525143345-ab3cae0737bc h1:32lGaCPq5JPYNgFFTjl/cTIar9UWWxCbimCs5G2hMHg= github.com/hashicorp/hcl/v2 v2.9.2-0.20220525143345-ab3cae0737bc/go.mod h1:odKNpEeZv3COD+++SQcPyACuKOlM5eBoQlzRyN5utIQ= -github.com/hashicorp/hil v0.0.0-20160711231837-1e86c6b523c5/go.mod h1:KHvg/R2/dPtaePb16oW4qIyzkMxXOL38xjRN64adsts= github.com/hashicorp/logutils v1.0.0 h1:dLEQVugN8vlakKOUE3ihGLTZJRB4j+M2cdTm/ORI65Y= github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= -github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= github.com/hashicorp/mdns v1.0.1/go.mod h1:4gW7WsVCke5TE7EPeYliwHlRUyBtfCwuFwuMg2DmyNY= github.com/hashicorp/mdns v1.0.4 h1:sY0CMhFmjIPDMlTB+HfymFHCaYLhgifZ0QhjaYKD/UQ= github.com/hashicorp/mdns v1.0.4/go.mod h1:mtBihi+LeNXGtG8L9dX59gAEa12BDtBQSp4v/YAJqrc= -github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= -github.com/hashicorp/memberlist v0.2.2/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOnAH9VT3Sh9MUE= github.com/hashicorp/memberlist v0.3.0/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOnAH9VT3Sh9MUE= github.com/hashicorp/memberlist v0.3.1 h1:MXgUXLqva1QvpVEDQW1IQLG0wivQAtmFlHRQ+1vWZfM= github.com/hashicorp/memberlist v0.3.1/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOnAH9VT3Sh9MUE= github.com/hashicorp/net-rpc-msgpackrpc v0.0.0-20151116020338-a14192a58a69 h1:lc3c72qGlIMDqQpQH82Y4vaglRMMFdJbziYWriR4UcE= github.com/hashicorp/net-rpc-msgpackrpc v0.0.0-20151116020338-a14192a58a69/go.mod h1:/z+jUGRBlwVpUZfjute9jWaF6/HuhjuFQuL1YXzVD1Q= github.com/hashicorp/raft v1.1.0/go.mod h1:4Ak7FSPnuvmb0GV6vgIAJ4vYT4bek9bb6Q+7HVbyzqM= -github.com/hashicorp/raft v1.1.1/go.mod h1:vPAJM8Asw6u8LxC3eJCUZmRP/E4QmUGE1R7g7k8sG/8= -github.com/hashicorp/raft v1.1.2/go.mod h1:vPAJM8Asw6u8LxC3eJCUZmRP/E4QmUGE1R7g7k8sG/8= +github.com/hashicorp/raft v1.2.0/go.mod h1:vPAJM8Asw6u8LxC3eJCUZmRP/E4QmUGE1R7g7k8sG/8= github.com/hashicorp/raft v1.3.9 h1:9yuo1aR0bFTr1cw7pj3S2Bk6MhJCsnr2NAxvIBrP2x4= github.com/hashicorp/raft v1.3.9/go.mod h1:4Ak7FSPnuvmb0GV6vgIAJ4vYT4bek9bb6Q+7HVbyzqM= +github.com/hashicorp/raft-autopilot v0.1.6 h1:C1q3RNF2FfXNZfHWbvVAu0QixaQK8K5pX4O5lh+9z4I= +github.com/hashicorp/raft-autopilot v0.1.6/go.mod h1:Af4jZBwaNOI+tXfIqIdbcAnh/UyyqIMj/pOISIfhArw= github.com/hashicorp/raft-boltdb v0.0.0-20171010151810-6e5ba93211ea/go.mod h1:pNv7Wc3ycL6F5oOWn+tPGo2gWD4a5X+yp/ntwdKLjRk= github.com/hashicorp/raft-boltdb v0.0.0-20210409134258-03c10cc3d4ea h1:RxcPJuutPRM8PUOyiweMmkuNO+RJyfy2jds2gfvgNmU= github.com/hashicorp/raft-boltdb v0.0.0-20210409134258-03c10cc3d4ea/go.mod h1:qRd6nFJYYS6Iqnc/8HcUmko2/2Gw8qTFEmxDLii6W5I= github.com/hashicorp/raft-boltdb/v2 v2.2.0 h1:/CVN9LSAcH50L3yp2TsPFIpeyHn1m3VF6kiutlDE3Nw= github.com/hashicorp/raft-boltdb/v2 v2.2.0/go.mod h1:SgPUD5TP20z/bswEr210SnkUFvQP/YjKV95aaiTbeMQ= -github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= -github.com/hashicorp/serf v0.9.3/go.mod h1:UWDWwZeL5cuWDJdl0C6wrvrUwEqtQ4ZKBKKENpqIUyk= github.com/hashicorp/serf v0.9.7 h1:hkdgbqizGQHuU5IPqYM1JdSMV8nKfpuOnZYXssk9muY= github.com/hashicorp/serf v0.9.7/go.mod h1:TXZNMjZQijwlDvp+r0b63xZ45H7JmCmgg4gpTwn9UV4= -github.com/hashicorp/vault/api v1.0.4/go.mod h1:gDcqh3WGcR1cpF5AJz/B1UFheUEneMoIospckxBxk6Q= github.com/hashicorp/vault/api v1.7.2 h1:kawHE7s/4xwrdKbkmwQi0wYaIeUhk5ueek7ljuezCVQ= github.com/hashicorp/vault/api v1.7.2/go.mod h1:xbfA+1AvxFseDzxxdWaL0uO99n1+tndus4GCrtouy0M= github.com/hashicorp/vault/api/auth/kubernetes v0.2.0 h1:ScdzRAF8JZIdJYP4oprZKsIS4GSTCaTP4iG2JJlJDvA= github.com/hashicorp/vault/api/auth/kubernetes v0.2.0/go.mod h1:2BKADs9mwqAycDK/6tiHRh2sX0SPnC0DN4wHjJoAirw= -github.com/hashicorp/vault/sdk v0.1.13/go.mod h1:B+hVj7TpuQY1Y/GPbCpffmgd+tSEwvhkWnjtSYCaS2M= github.com/hashicorp/vault/sdk v0.5.1 h1:zly/TmNgOXCGgWIRA8GojyXzG817POtVh3uzIwzZx+8= github.com/hashicorp/vault/sdk v0.5.1/go.mod h1:DoGraE9kKGNcVgPmTuX357Fm6WAx1Okvde8Vp3dPDoU= github.com/hashicorp/vic v1.5.1-0.20190403131502-bbfe86ec9443 h1:O/pT5C1Q3mVXMyuqg7yuAWUg/jMZR1/0QTzTRdNR6Uw= github.com/hashicorp/vic v1.5.1-0.20190403131502-bbfe86ec9443/go.mod h1:bEpDU35nTu0ey1EXjwNwPjI9xErAsoOCmcMb9GKvyxo= github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb/go.mod h1:+NfK9FKeTrX5uv1uIXGdwYDTeHna2qgaIlx54MXqjAM= -github.com/hashicorp/yamux v0.0.0-20181012175058-2f1d1f20f75d/go.mod h1:+NfK9FKeTrX5uv1uIXGdwYDTeHna2qgaIlx54MXqjAM= github.com/hashicorp/yamux v0.0.0-20211028200310-0bc27b27de87 h1:xixZ2bWeofWV68J+x6AzmKuVM/JWCQwkWm6GW/MUR6I= github.com/hashicorp/yamux v0.0.0-20211028200310-0bc27b27de87/go.mod h1:CtWFDAQgb7dxtzFs4tWbplKIe2jSi3+5vKbgIO0SLnQ= github.com/hexdigest/gowrap v1.1.7/go.mod h1:Z+nBFUDLa01iaNM+/jzoOA1JJ7sm51rnYFauKFUB5fs= @@ -973,21 +936,16 @@ github.com/mitchellh/go-ps v0.0.0-20190716172923-621e5597135b h1:9+ke9YJ9KGWw5AN github.com/mitchellh/go-ps v0.0.0-20190716172923-621e5597135b/go.mod h1:r1VsdOzOPt1ZSrGZWFoNhsAedKnEd6r9Np1+5blZCWk= github.com/mitchellh/go-testing-interface v0.0.0-20171004221916-a61a99592b77/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= -github.com/mitchellh/go-testing-interface v1.0.3/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= github.com/mitchellh/go-testing-interface v1.14.1 h1:jrgshOhYAUVNMAJiKbEu7EqAwgJJ2JqpQmpLJOu07cU= github.com/mitchellh/go-testing-interface v1.14.1/go.mod h1:gfgS7OtZj6MA4U1UrDRp04twqAjfvlZyCfX3sDjEym8= github.com/mitchellh/go-wordwrap v0.0.0-20150314170334-ad45545899c7/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUbP2oI0UX1GXzOo= github.com/mitchellh/go-wordwrap v1.0.0/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUbP2oI0UX1GXzOo= github.com/mitchellh/go-wordwrap v1.0.1 h1:TLuKupo69TCn6TQSyGxwI1EblZZEsQ0vMlAFQflz0v0= github.com/mitchellh/go-wordwrap v1.0.1/go.mod h1:R62XHJLzvMFRBbcrT7m7WgmE1eOyTSsCt+hzestvNj0= -github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= -github.com/mitchellh/hashstructure v0.0.0-20170609045927-2bca23e0e452/go.mod h1:QjSHrPWS+BGUVBYkbTZWEnOh3G1DutKwClXU/ABz6AQ= github.com/mitchellh/hashstructure v1.1.0 h1:P6P1hdjqAAknpY/M1CGipelZgp+4y9ja9kmUZPXP+H0= github.com/mitchellh/hashstructure v1.1.0/go.mod h1:xUDAozZz0Wmdiufv0uyhnHkUTN6/6d8ulp4AwfLKrmA= -github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/mitchellh/mapstructure v1.2.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= @@ -995,7 +953,6 @@ github.com/mitchellh/osext v0.0.0-20151018003038-5e2d6d41470f/go.mod h1:OkQIRizQ github.com/mitchellh/pointerstructure v1.2.1 h1:ZhBBeX8tSlRpu/FFhXH4RC4OJzFlqsQhoHZAz4x7TIw= github.com/mitchellh/pointerstructure v1.2.1/go.mod h1:BRAsLI5zgXmw97Lf6s25bs8ohIXc3tViBH44KcwB2g4= github.com/mitchellh/reflectwalk v1.0.0/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= -github.com/mitchellh/reflectwalk v1.0.1/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ= github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= github.com/moby/locker v1.0.1/go.mod h1:S7SDdo5zpBK84bzzVlKr2V0hz+7x9hWbYC/kq7oQppc= @@ -1104,7 +1061,6 @@ github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144T github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pelletier/go-toml v1.8.1/go.mod h1:T2/BmBdy8dvIRq1a/8aqjN41wvWlN4lrapLU/GW4pbc= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= -github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pierrec/lz4 v2.5.2+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pierrec/lz4 v2.6.1+incompatible h1:9UY3+iC23yxF0UfGaYrGplQ+79Rg+h/q9FV9ix19jjM= github.com/pierrec/lz4 v2.6.1+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= @@ -1165,7 +1121,6 @@ github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1 github.com/prometheus/procfs v0.7.3 h1:4jVXhlkAyzOScmCkXBTOLRLTz8EeU+eyjrwB/EPq0VU= github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= -github.com/rboyer/safeio v0.2.1/go.mod h1:Cq/cEPK+YXFn622lsQ0K4KsPZSPtaptHHEldsy7Fmig= github.com/renier/xmlrpc v0.0.0-20170708154548-ce4a1a486c03 h1:Wdi9nwnhFNAlseAOekn6B5G/+GMtks9UKbvRU/CMM/o= github.com/renier/xmlrpc v0.0.0-20170708154548-ce4a1a486c03/go.mod h1:gRAiPF5C5Nd0eyyRdqIu9qTiFSoZzpTq727b5B8fkkU= github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= @@ -1196,10 +1151,8 @@ github.com/seccomp/libseccomp-golang v0.10.0 h1:aA4bp+/Zzi0BnWZ2F1wgNBs5gTpm+na2 github.com/seccomp/libseccomp-golang v0.10.0/go.mod h1:JA8cRccbGaA1s33RQf7Y1+q9gHmZX1yB/z9WDN1C6fg= github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= -github.com/shirou/gopsutil v0.0.0-20181107111621-48177ef5f880/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= github.com/shirou/gopsutil/v3 v3.22.7 h1:flKnuCMfUUrO+oAvwAd6GKZgnPzr098VA/UJ14nhJd4= github.com/shirou/gopsutil/v3 v3.22.7/go.mod h1:s648gW4IywYzUfE/KjXxUsqrqx/T2xO5VqOXxONeRfI= -github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q1u/4XEfrquwF8Lw7D7y5cD8CuHnfIc= github.com/shoenig/test v0.3.1 h1:dhGZztS6nQuvJ0o0RtUiQHaEO4hhArh/WmWwik3Ols0= github.com/shoenig/test v0.3.1/go.mod h1:xYtyGBC5Q3kzCNyJg/SjgNpfAa2kvmgA0i5+lQso8x0= github.com/shopspring/decimal v1.2.0 h1:abSATXmQEYyShuxI4/vyW3tV1MrKAJzCZ/0zLUXYbsQ= @@ -1346,6 +1299,7 @@ go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= go.uber.org/goleak v1.1.12 h1:gZAh5/EyT/HQwlpkCy6wTpqfH9H8Lz8zbm3dZh+OyzA= go.uber.org/goleak v1.1.12/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= @@ -1363,7 +1317,6 @@ golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20191106202628-ed6320f186d4/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200220183623-bac4c82f6975/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200414173820-0848c9571904/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200422194213-44a606286825/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= @@ -1517,12 +1470,10 @@ golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190129075346-302c3dd5f1cc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190209173611-3b5209105503/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1637,7 +1588,6 @@ golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fq golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.1-0.20181227161524-e6919f6577db/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= @@ -1677,6 +1627,7 @@ golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20190907020128-2ca718005c18/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -1770,7 +1721,6 @@ google.golang.org/cloud v0.0.0-20151119220103-975617b05ea8/go.mod h1:0H1ncTHf11K google.golang.org/genproto v0.0.0-20170818010345-ee236bd376b0/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190404172233-64821d5d2107/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= @@ -1837,12 +1787,10 @@ google.golang.org/genproto v0.0.0-20220314164441-57ef72a4c106 h1:ErU+UA6wxadoU8n google.golang.org/genproto v0.0.0-20220314164441-57ef72a4c106/go.mod h1:hAL49I2IFola2sVEjAn7MEwsja0xp51I0tlGAf9hz4E= google.golang.org/grpc v0.0.0-20160317175043-d3ddb4469d5a/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.8.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= -google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= -google.golang.org/grpc v1.22.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.24.0/go.mod h1:XDChyiUovWa60DnaeDeZmSW86xtLtjtZbwvSiRnRtcA= @@ -1889,7 +1837,6 @@ google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175 google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= -gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d/go.mod h1:cuepJuh7vyXfUyUwEgHQXw849cJrilpS5NeIjOWESAw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20141024133853-64131543e789/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -1941,12 +1888,10 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -k8s.io/api v0.0.0-20190325185214-7544f9db76f6/go.mod h1:iuAfoD4hCxJ8Onx9kaTIt30j7jUFS00AXQi6QMi99vA= k8s.io/api v0.18.2/go.mod h1:SJCWI7OLzhZSvbY7U8zwNl9UA4o1fizoug34OV/2r78= k8s.io/api v0.20.1/go.mod h1:KqwcCVogGxQY3nBlRpwt+wpAMF/KjaCc7RpywacvqUo= k8s.io/api v0.20.4/go.mod h1:++lNL1AJMkDymriNniQsWRkMDzRaX2Y/POTUi8yvqYQ= k8s.io/api v0.20.6/go.mod h1:X9e8Qag6JV/bL5G6bU8sdVRltWKmdHsFUGS3eVndqE8= -k8s.io/apimachinery v0.0.0-20190223001710-c182ff3b9841/go.mod h1:ccL7Eh7zubPUSh9A3USN90/OzHNSVN6zxzde07TDCL0= k8s.io/apimachinery v0.18.2/go.mod h1:9SnR/e11v5IbyPCGbvJViimtJ0SwHG4nfZFjU77ftcA= k8s.io/apimachinery v0.20.1/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRpU= k8s.io/apimachinery v0.20.4/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRpU= @@ -1958,7 +1903,6 @@ k8s.io/client-go v0.18.2/go.mod h1:Xcm5wVGXX9HAA2JJ2sSBUn3tCJ+4SVlCbl2MNNv+CIU= k8s.io/client-go v0.20.1/go.mod h1:/zcHdt1TeWSd5HoUe6elJmHSQ6uLLgp4bIJHVEuy+/Y= k8s.io/client-go v0.20.4/go.mod h1:LiMv25ND1gLUdBeYxBIwKpkSC5IsozMMmOOeSJboP+k= k8s.io/client-go v0.20.6/go.mod h1:nNQMnOvEUEsOzRRFIIkdmYOjAZrC8bgq0ExboWSU1I0= -k8s.io/client-go v8.0.0+incompatible/go.mod h1:7vJpHMYJwNQCWgzmNV+VYUl1zCObLyodBc8nIyt8L5s= k8s.io/code-generator v0.19.7/go.mod h1:lwEq3YnLYb/7uVXLorOJfxg+cUu2oihFhHZ0n9NIla0= k8s.io/component-base v0.20.1/go.mod h1:guxkoJnNoh8LNrbtiQOlyp2Y2XFCZQmrcg2n/DeYNLk= k8s.io/component-base v0.20.4/go.mod h1:t4p9EdiagbVCJKrQ1RsA5/V4rFQNDfRlevJajlGwgjI= diff --git a/nomad/autopilot.go b/nomad/autopilot.go index 5e42fff54..3d56ef4df 100644 --- a/nomad/autopilot.go +++ b/nomad/autopilot.go @@ -3,10 +3,12 @@ package nomad import ( "context" "fmt" + "strconv" metrics "github.com/armon/go-metrics" - "github.com/hashicorp/consul/agent/consul/autopilot" + "github.com/hashicorp/nomad/nomad/structs" "github.com/hashicorp/raft" + autopilot "github.com/hashicorp/raft-autopilot" "github.com/hashicorp/serf/serf" ) @@ -20,63 +22,54 @@ const ( AutopilotVersionTag = "ap_version" ) -// AutopilotDelegate is a Nomad delegate for autopilot operations. +// AutopilotDelegate is a Nomad delegate for autopilot operations. It implements +// the autopilot.ApplicationIntegration interface, and the methods required for +// that interface have been documented as such below. type AutopilotDelegate struct { server *Server } +// AutopilotConfig is used to retrieve the latest configuration from the Nomad +// delegate. This method is required to implement the ApplicationIntegration +// interface. func (d *AutopilotDelegate) AutopilotConfig() *autopilot.Config { c := d.server.getOrCreateAutopilotConfig() if c == nil { return nil } - conf := &autopilot.Config{ CleanupDeadServers: c.CleanupDeadServers, LastContactThreshold: c.LastContactThreshold, MaxTrailingLogs: c.MaxTrailingLogs, MinQuorum: c.MinQuorum, ServerStabilizationTime: c.ServerStabilizationTime, - DisableUpgradeMigration: c.DisableUpgradeMigration, - ModifyIndex: c.ModifyIndex, - CreateIndex: c.CreateIndex, - } - - if c.EnableRedundancyZones { - conf.RedundancyZoneTag = AutopilotRZTag - } - if c.EnableCustomUpgrades { - conf.UpgradeVersionTag = AutopilotVersionTag + Ext: autopilotConfigExt(c), } return conf } -func (d *AutopilotDelegate) FetchStats(ctx context.Context, servers []serf.Member) map[string]*autopilot.ServerStats { +// FetchServerStats will be called by autopilot to request Nomad fetch the +// server stats out of band. This method is required to implement the +// ApplicationIntegration interface +func (d *AutopilotDelegate) FetchServerStats(ctx context.Context, servers map[raft.ServerID]*autopilot.Server) map[raft.ServerID]*autopilot.ServerStats { return d.server.statsFetcher.Fetch(ctx, servers) } -func (d *AutopilotDelegate) IsServer(m serf.Member) (*autopilot.ServerInfo, error) { - ok, parts := isNomadServer(m) - if !ok || parts.Region != d.server.Region() { - return nil, nil - } - - server := &autopilot.ServerInfo{ - Name: m.Name, - ID: parts.ID, - Addr: parts.Addr, - Build: parts.Build, - Status: m.Status, - } - return server, nil +// KnownServers will be called by autopilot to request the list of servers known +// to Nomad. This method is required to implement the ApplicationIntegration +// interface +func (d *AutopilotDelegate) KnownServers() map[raft.ServerID]*autopilot.Server { + return d.server.autopilotServers() } -// NotifyHealth heartbeats a metric for monitoring if we're the leader. -func (d *AutopilotDelegate) NotifyHealth(health autopilot.OperatorHealthReply) { +// NotifyState will be called when the autopilot state is updated. The Nomad +// leader heartbeats a metric for monitoring based on this information. This +// method is required to implement the ApplicationIntegration interface +func (d *AutopilotDelegate) NotifyState(state *autopilot.State) { if d.server.raft.State() == raft.Leader { - metrics.SetGauge([]string{"nomad", "autopilot", "failure_tolerance"}, float32(health.FailureTolerance)) - if health.Healthy { + metrics.SetGauge([]string{"nomad", "autopilot", "failure_tolerance"}, float32(state.FailureTolerance)) + if state.Healthy { metrics.SetGauge([]string{"nomad", "autopilot", "healthy"}, 1) } else { metrics.SetGauge([]string{"nomad", "autopilot", "healthy"}, 0) @@ -84,24 +77,170 @@ func (d *AutopilotDelegate) NotifyHealth(health autopilot.OperatorHealthReply) { } } -func (d *AutopilotDelegate) PromoteNonVoters(conf *autopilot.Config, health autopilot.OperatorHealthReply) ([]raft.Server, error) { - future := d.server.raft.GetConfiguration() - if err := future.Error(); err != nil { - return nil, fmt.Errorf("failed to get raft configuration: %v", err) +// RemoveFailedServer will be called by autopilot to notify Nomad to remove the +// server in a failed state. This method is required to implement the +// ApplicationIntegration interface. (Note this is expected to return +// immediately so we'll spawn a goroutine for it.) +func (d *AutopilotDelegate) RemoveFailedServer(failedSrv *autopilot.Server) { + go func() { + err := d.server.RemoveFailedNode(failedSrv.Name) + if err != nil { + d.server.logger.Error("could not remove failed server", "server", string(failedSrv.ID)) + } + }() +} + +// MinRaftProtocol returns the lowest supported Raft protocol among alive +// servers +func (s *Server) MinRaftProtocol() (int, error) { + return minRaftProtocol(s.serf.Members(), isNomadServer) +} + +// GetClusterHealth is used to get the current health of the servers, as known +// by the leader. +func (s *Server) GetClusterHealth() *structs.OperatorHealthReply { + + state := s.autopilot.GetState() + if state == nil { + // this behavior seems odd but its functionally equivalent to 1.8.5 where if + // autopilot didn't have a health reply yet it would just return no error + return nil } - return autopilot.PromoteStableServers(conf, health, future.Configuration().Servers), nil + health := &structs.OperatorHealthReply{ + Healthy: state.Healthy, + FailureTolerance: state.FailureTolerance, + } + + for _, srv := range state.Servers { + srvHealth := structs.ServerHealth{ + ID: string(srv.Server.ID), + Name: srv.Server.Name, + Address: string(srv.Server.Address), + Version: srv.Server.Version, + Leader: srv.State == autopilot.RaftLeader, + Voter: srv.State == autopilot.RaftLeader || srv.State == autopilot.RaftVoter, + LastContact: srv.Stats.LastContact, + LastTerm: srv.Stats.LastTerm, + LastIndex: srv.Stats.LastIndex, + Healthy: srv.Health.Healthy, + StableSince: srv.Health.StableSince, + } + + switch srv.Server.NodeStatus { + case autopilot.NodeAlive: + srvHealth.SerfStatus = serf.StatusAlive + case autopilot.NodeLeft: + srvHealth.SerfStatus = serf.StatusLeft + case autopilot.NodeFailed: + srvHealth.SerfStatus = serf.StatusFailed + default: + srvHealth.SerfStatus = serf.StatusNone + } + + health.Servers = append(health.Servers, srvHealth) + } + + return health } -func (d *AutopilotDelegate) Raft() *raft.Raft { - return d.server.raft +// ------------------- +// helper functions + +func minRaftProtocol(members []serf.Member, serverFunc func(serf.Member) (bool, *serverParts)) (int, error) { + minVersion := -1 + for _, m := range members { + if m.Status != serf.StatusAlive { + continue + } + + ok, server := serverFunc(m) + if !ok { + return -1, fmt.Errorf("not a Nomad server") + } + if server == nil { + continue + } + + vsn, ok := m.Tags["raft_vsn"] + if !ok { + vsn = "1" + } + raftVsn, err := strconv.Atoi(vsn) + if err != nil { + return -1, err + } + + if minVersion == -1 || raftVsn < minVersion { + minVersion = raftVsn + } + } + + if minVersion == -1 { + return minVersion, fmt.Errorf("No servers found") + } + + return minVersion, nil } -func (d *AutopilotDelegate) SerfLAN() *serf.Serf { - return d.server.serf +func (s *Server) autopilotServers() map[raft.ServerID]*autopilot.Server { + servers := make(map[raft.ServerID]*autopilot.Server) + + for _, member := range s.serf.Members() { + srv, err := s.autopilotServer(member) + if err != nil { + s.logger.Warn("Error parsing server info", "name", member.Name, "error", err) + continue + } else if srv == nil { + // this member was a client + continue + } + + servers[srv.ID] = srv + } + + return servers } -func (d *AutopilotDelegate) SerfWAN() *serf.Serf { - // serf WAN isn't supported in nomad yet - return nil +func (s *Server) autopilotServer(m serf.Member) (*autopilot.Server, error) { + ok, srv := isNomadServer(m) + if !ok { + return nil, nil + } + + return s.autopilotServerFromMetadata(srv) +} + +func (s *Server) autopilotServerFromMetadata(srv *serverParts) (*autopilot.Server, error) { + server := &autopilot.Server{ + Name: srv.Name, + ID: raft.ServerID(srv.ID), + Address: raft.ServerAddress(srv.Addr.String()), + Version: srv.Build.String(), + RaftVersion: srv.RaftVersion, + Ext: s.autopilotServerExt(srv), + } + + switch srv.Status { + case serf.StatusLeft: + server.NodeStatus = autopilot.NodeLeft + case serf.StatusAlive, serf.StatusLeaving: + // we want to treat leaving as alive to prevent autopilot from + // prematurely removing the node. + server.NodeStatus = autopilot.NodeAlive + case serf.StatusFailed: + server.NodeStatus = autopilot.NodeFailed + default: + server.NodeStatus = autopilot.NodeUnknown + } + + members := s.serf.Members() + for _, member := range members { + if member.Name == srv.Name { + server.Meta = member.Tags + break + } + } + + return server, nil } diff --git a/nomad/autopilot_oss.go b/nomad/autopilot_oss.go new file mode 100644 index 000000000..f18781aed --- /dev/null +++ b/nomad/autopilot_oss.go @@ -0,0 +1,26 @@ +//go:build !ent +// +build !ent + +package nomad + +import ( + autopilot "github.com/hashicorp/raft-autopilot" + + "github.com/hashicorp/nomad/nomad/structs" +) + +func (s *Server) autopilotPromoter() autopilot.Promoter { + return autopilot.DefaultPromoter() +} + +// autopilotServerExt returns the autopilot-enterprise.Server extensions needed +// for ENT feature support, but this is the empty OSS implementation. +func (s *Server) autopilotServerExt(_ *serverParts) interface{} { + return nil +} + +// autopilotConfigExt returns the autopilot-enterprise.Config extensions needed +// for ENT feature support, but this is the empty OSS implementation. +func autopilotConfigExt(_ *structs.AutopilotConfig) interface{} { + return nil +} diff --git a/nomad/autopilot_test.go b/nomad/autopilot_test.go index 8be118a58..417474126 100644 --- a/nomad/autopilot_test.go +++ b/nomad/autopilot_test.go @@ -5,14 +5,18 @@ import ( "testing" "time" - "github.com/hashicorp/consul/agent/consul/autopilot" + // TODO: replace this with our own helper "github.com/hashicorp/consul/sdk/testutil/retry" + "github.com/hashicorp/raft" + autopilot "github.com/hashicorp/raft-autopilot" + "github.com/hashicorp/serf/serf" + "github.com/hashicorp/nomad/ci" "github.com/hashicorp/nomad/testutil" - "github.com/hashicorp/raft" - "github.com/hashicorp/serf/serf" ) +var _ autopilot.ApplicationIntegration = (*AutopilotDelegate)(nil) + // wantPeers determines whether the server has the given // number of voting raft peers. func wantPeers(s *Server, peers int) error { @@ -21,7 +25,13 @@ func wantPeers(s *Server, peers int) error { return err } - n := autopilot.NumPeers(future.Configuration()) + var n int + for _, server := range future.Configuration().Servers { + if server.Suffrage == raft.Voter { + n++ + } + } + if got, want := n, peers; got != want { return fmt.Errorf("server %v: got %d peers want %d\n\tservers: %#+v", s.config.NodeName, got, want, future.Configuration().Servers) } @@ -68,7 +78,6 @@ func wantRaft(servers []*Server) error { func TestAutopilot_CleanupDeadServer(t *testing.T) { ci.Parallel(t) - t.Run("raft_v2", func(t *testing.T) { testCleanupDeadServer(t, 2) }) t.Run("raft_v3", func(t *testing.T) { testCleanupDeadServer(t, 3) }) } @@ -351,9 +360,9 @@ func TestAutopilot_PromoteNonVoter(t *testing.T) { if servers[1].Suffrage != raft.Nonvoter { r.Fatalf("bad: %v", servers) } - health := s1.autopilot.GetServerHealth(string(servers[1].ID)) + health := s1.autopilot.GetServerHealth(raft.ServerID(servers[1].ID)) if health == nil { - r.Fatalf("nil health, %v", s1.autopilot.GetClusterHealth()) + r.Fatalf("nil health, %v", s1.GetClusterHealth()) } if !health.Healthy { r.Fatalf("bad: %v", health) diff --git a/nomad/leader.go b/nomad/leader.go index 50913e96a..950931ce9 100644 --- a/nomad/leader.go +++ b/nomad/leader.go @@ -288,7 +288,7 @@ func (s *Server) establishLeadership(stopCh chan struct{}) error { // Initialize and start the autopilot routine s.getOrCreateAutopilotConfig() - s.autopilot.Start() + s.autopilot.Start(s.shutdownCtx) // Initialize scheduler configuration. schedulerConfig := s.getOrCreateSchedulerConfig() @@ -1302,7 +1302,7 @@ func (s *Server) addRaftPeer(m serf.Member, parts *serverParts) error { // but we want to avoid doing that if possible to prevent useless Raft // log entries. If the address is the same but the ID changed, remove the // old server before adding the new one. - minRaftProtocol, err := s.autopilot.MinRaftProtocol() + minRaftProtocol, err := s.MinRaftProtocol() if err != nil { return err } @@ -1372,7 +1372,7 @@ func (s *Server) removeRaftPeer(m serf.Member, parts *serverParts) error { return err } - minRaftProtocol, err := s.autopilot.MinRaftProtocol() + minRaftProtocol, err := s.MinRaftProtocol() if err != nil { return err } diff --git a/nomad/leader_test.go b/nomad/leader_test.go index 5289d413c..2eab55473 100644 --- a/nomad/leader_test.go +++ b/nomad/leader_test.go @@ -1175,7 +1175,7 @@ func TestLeader_UpgradeRaftVersion(t *testing.T) { } for _, s := range []*Server{s1, s3} { - minVer, err := s.autopilot.MinRaftProtocol() + minVer, err := s.MinRaftProtocol() if err != nil { t.Fatal(err) } @@ -1227,16 +1227,8 @@ func TestLeader_UpgradeRaftVersion(t *testing.T) { func TestLeader_Reelection(t *testing.T) { ci.Parallel(t) - raftProtocols := []int{1, 2, 3} - for _, p := range raftProtocols { - t.Run(fmt.Sprintf("Leader Election - Protocol version %d", p), func(t *testing.T) { - leaderElectionTest(t, raft.ProtocolVersion(p)) - }) - } + const raftProtocol = 3 -} - -func leaderElectionTest(t *testing.T, raftProtocol raft.ProtocolVersion) { s1, cleanupS1 := TestServer(t, func(c *Config) { c.BootstrapExpect = 3 c.RaftConfig.ProtocolVersion = raftProtocol @@ -1298,19 +1290,20 @@ func TestLeader_RollRaftServer(t *testing.T) { ci.SkipSlow(t, "flaky on GHA; #12358") s1, cleanupS1 := TestServer(t, func(c *Config) { - c.RaftConfig.ProtocolVersion = 2 + c.BootstrapExpect = 3 + c.RaftConfig.ProtocolVersion = 3 }) defer cleanupS1() s2, cleanupS2 := TestServer(t, func(c *Config) { c.BootstrapExpect = 3 - c.RaftConfig.ProtocolVersion = 2 + c.RaftConfig.ProtocolVersion = 3 }) defer cleanupS2() s3, cleanupS3 := TestServer(t, func(c *Config) { c.BootstrapExpect = 3 - c.RaftConfig.ProtocolVersion = 2 + c.RaftConfig.ProtocolVersion = 3 }) defer cleanupS3() @@ -1330,25 +1323,14 @@ func TestLeader_RollRaftServer(t *testing.T) { s.RemoveFailedNode(s1.config.NodeID) retry.Run(t, func(r *retry.R) { - minVer, err := s.autopilot.MinRaftProtocol() - if err != nil { - r.Fatal(err) - } - if got, want := minVer, 2; got != want { - r.Fatalf("got min raft version %d want %d", got, want) - } - configFuture := s.raft.GetConfiguration() - if err != nil { - r.Fatal(err) - } if len(configFuture.Configuration().Servers) != 2 { r.Fatalf("expected 2 servers, got %d", len(configFuture.Configuration().Servers)) } }) } - // Replace the dead server with one running raft protocol v3 + // Replace the dead server s4, cleanupS4 := TestServer(t, func(c *Config) { c.BootstrapExpect = 3 c.RaftConfig.ProtocolVersion = 3 @@ -1357,7 +1339,19 @@ func TestLeader_RollRaftServer(t *testing.T) { TestJoin(t, s2, s3, s4) servers[0] = s4 - // Kill the second v2 server + for _, s := range []*Server{s3, s4} { + retry.RunWith(&retry.Counter{ + Count: int(10 * testutil.TestMultiplier()), + Wait: time.Duration(testutil.TestMultiplier()) * time.Second * 2, + }, t, func(r *retry.R) { + configFuture := s.raft.GetConfiguration() + if len(configFuture.Configuration().Servers) != 3 { + r.Fatalf("expected 3 servers, got %d", len(configFuture.Configuration().Servers)) + } + }) + } + + // Kill the second server s2.Shutdown() for _, s := range []*Server{s3, s4} { @@ -1365,26 +1359,16 @@ func TestLeader_RollRaftServer(t *testing.T) { retry.RunWith(&retry.Counter{ Count: int(10 * testutil.TestMultiplier()), - Wait: time.Duration(testutil.TestMultiplier()) * time.Second, + Wait: time.Duration(testutil.TestMultiplier()) * time.Second * 2, }, t, func(r *retry.R) { - minVer, err := s.autopilot.MinRaftProtocol() - if err != nil { - r.Fatal(err) - } - if got, want := minVer, 2; got != want { - r.Fatalf("got min raft version %d want %d", got, want) - } - configFuture := s.raft.GetConfiguration() - if err != nil { - r.Fatal(err) - } - if len(configFuture.Configuration().Servers) != 2 { + if len(configFuture.Configuration().Servers) != 3 { r.Fatalf("expected 2 servers, got %d", len(configFuture.Configuration().Servers)) } }) } - // Replace another dead server with one running raft protocol v3 + + // Replace the 2nd dead server s5, cleanupS5 := TestServer(t, func(c *Config) { c.BootstrapExpect = 3 c.RaftConfig.ProtocolVersion = 3 @@ -1393,17 +1377,29 @@ func TestLeader_RollRaftServer(t *testing.T) { TestJoin(t, s3, s4, s5) servers[1] = s5 - // Kill the last v2 server, now minRaftProtocol should be 3 + for _, s := range []*Server{s3, s4, s5} { + retry.RunWith(&retry.Counter{ + Count: int(10 * testutil.TestMultiplier()), + Wait: time.Duration(testutil.TestMultiplier()) * time.Second * 2, + }, t, func(r *retry.R) { + configFuture := s.raft.GetConfiguration() + if len(configFuture.Configuration().Servers) != 3 { + r.Fatalf("expected 3 servers, got %d", len(configFuture.Configuration().Servers)) + } + }) + } + + // Kill the last old server s3.Shutdown() for _, s := range []*Server{s4, s5} { - s.RemoveFailedNode(s2.config.NodeID) + s.RemoveFailedNode(s3.config.NodeID) retry.RunWith(&retry.Counter{ Count: int(10 * testutil.TestMultiplier()), - Wait: time.Duration(testutil.TestMultiplier()) * time.Second, + Wait: time.Duration(testutil.TestMultiplier()) * time.Second * 2, }, t, func(r *retry.R) { - minVer, err := s.autopilot.MinRaftProtocol() + minVer, err := s.MinRaftProtocol() if err != nil { r.Fatal(err) } @@ -1415,7 +1411,7 @@ func TestLeader_RollRaftServer(t *testing.T) { if err != nil { r.Fatal(err) } - if len(configFuture.Configuration().Servers) != 2 { + if len(configFuture.Configuration().Servers) != 3 { r.Fatalf("expected 2 servers, got %d", len(configFuture.Configuration().Servers)) } }) diff --git a/nomad/operator_endpoint.go b/nomad/operator_endpoint.go index 40c27669e..b21f3256b 100644 --- a/nomad/operator_endpoint.go +++ b/nomad/operator_endpoint.go @@ -9,13 +9,12 @@ import ( log "github.com/hashicorp/go-hclog" "github.com/hashicorp/go-msgpack/codec" + "github.com/hashicorp/raft" + "github.com/hashicorp/serf/serf" - "github.com/hashicorp/consul/agent/consul/autopilot" cstructs "github.com/hashicorp/nomad/client/structs" "github.com/hashicorp/nomad/helper/snapshot" "github.com/hashicorp/nomad/nomad/structs" - "github.com/hashicorp/raft" - "github.com/hashicorp/serf/serf" ) // Operator endpoint is used to perform low-level operator tasks for Nomad. @@ -183,7 +182,7 @@ REMOVE: // doing if you are calling this. If you remove a peer that's known to // Serf, for example, it will come back when the leader does a reconcile // pass. - minRaftProtocol, err := op.srv.autopilot.MinRaftProtocol() + minRaftProtocol, err := op.srv.MinRaftProtocol() if err != nil { return err } @@ -270,7 +269,7 @@ func (op *Operator) AutopilotSetConfiguration(args *structs.AutopilotSetConfigRe } // ServerHealth is used to get the current health of the servers. -func (op *Operator) ServerHealth(args *structs.GenericRequest, reply *autopilot.OperatorHealthReply) error { +func (op *Operator) ServerHealth(args *structs.GenericRequest, reply *structs.OperatorHealthReply) error { // This must be sent to the leader, so we fix the args since we are // re-using a structure where we don't support all the options. args.AllowStale = false @@ -288,7 +287,7 @@ func (op *Operator) ServerHealth(args *structs.GenericRequest, reply *autopilot. } // Exit early if the min Raft version is too low - minRaftProtocol, err := op.srv.autopilot.MinRaftProtocol() + minRaftProtocol, err := op.srv.MinRaftProtocol() if err != nil { return fmt.Errorf("error getting server raft protocol versions: %s", err) } @@ -296,7 +295,7 @@ func (op *Operator) ServerHealth(args *structs.GenericRequest, reply *autopilot. return fmt.Errorf("all servers must have raft_protocol set to 3 or higher to use this endpoint") } - *reply = op.srv.autopilot.GetClusterHealth() + *reply = *op.srv.GetClusterHealth() return nil } diff --git a/nomad/serf.go b/nomad/serf.go index 9a6d31a87..adcf80a28 100644 --- a/nomad/serf.go +++ b/nomad/serf.go @@ -196,7 +196,8 @@ func (s *Server) maybeBootstrap() { // Attempt a live bootstrap! var configuration raft.Configuration var addrs []string - minRaftVersion, err := s.autopilot.MinRaftProtocol() + + minRaftVersion, err := s.MinRaftProtocol() if err != nil { s.logger.Error("failed to read server raft versions", "error", err) } diff --git a/nomad/server.go b/nomad/server.go index f32f7d880..32df0f448 100644 --- a/nomad/server.go +++ b/nomad/server.go @@ -20,11 +20,16 @@ import ( "time" "github.com/armon/go-metrics" - "github.com/hashicorp/consul/agent/consul/autopilot" consulapi "github.com/hashicorp/consul/api" log "github.com/hashicorp/go-hclog" multierror "github.com/hashicorp/go-multierror" lru "github.com/hashicorp/golang-lru" + "github.com/hashicorp/raft" + autopilot "github.com/hashicorp/raft-autopilot" + raftboltdb "github.com/hashicorp/raft-boltdb/v2" + "github.com/hashicorp/serf/serf" + "go.etcd.io/bbolt" + "github.com/hashicorp/nomad/command/agent/consul" "github.com/hashicorp/nomad/helper" "github.com/hashicorp/nomad/helper/codec" @@ -38,10 +43,6 @@ import ( "github.com/hashicorp/nomad/nomad/structs/config" "github.com/hashicorp/nomad/nomad/volumewatcher" "github.com/hashicorp/nomad/scheduler" - "github.com/hashicorp/raft" - raftboltdb "github.com/hashicorp/raft-boltdb/v2" - "github.com/hashicorp/serf/serf" - "go.etcd.io/bbolt" ) const ( @@ -752,7 +753,7 @@ func (s *Server) Leave() error { // for some sane period of time. isLeader := s.IsLeader() if isLeader && numPeers > 1 { - minRaftProtocol, err := s.autopilot.MinRaftProtocol() + minRaftProtocol, err := s.MinRaftProtocol() if err != nil { return err } diff --git a/nomad/server_setup_oss.go b/nomad/server_setup_oss.go index 7abf7a41e..5e0684729 100644 --- a/nomad/server_setup_oss.go +++ b/nomad/server_setup_oss.go @@ -4,7 +4,7 @@ package nomad import ( - "github.com/hashicorp/consul/agent/consul/autopilot" + autopilot "github.com/hashicorp/raft-autopilot" ) type EnterpriseState struct{} @@ -20,7 +20,15 @@ func (es *EnterpriseState) ReloadLicense(_ *Config) error { func (s *Server) setupEnterprise(config *Config) error { // Set up the OSS version of autopilot apDelegate := &AutopilotDelegate{s} - s.autopilot = autopilot.NewAutopilot(s.logger, apDelegate, config.AutopilotInterval, config.ServerHealthInterval) + + s.autopilot = autopilot.New( + s.raft, + apDelegate, + autopilot.WithLogger(s.logger), + autopilot.WithReconcileInterval(config.AutopilotInterval), + autopilot.WithUpdateInterval(config.ServerHealthInterval), + autopilot.WithPromoter(s.autopilotPromoter()), + ) return nil } diff --git a/nomad/stats_fetcher.go b/nomad/stats_fetcher.go index 9967698b6..d06bffba5 100644 --- a/nomad/stats_fetcher.go +++ b/nomad/stats_fetcher.go @@ -2,13 +2,15 @@ package nomad import ( "context" + "net" "sync" log "github.com/hashicorp/go-hclog" + "github.com/hashicorp/raft" + autopilot "github.com/hashicorp/raft-autopilot" - "github.com/hashicorp/consul/agent/consul/autopilot" "github.com/hashicorp/nomad/helper/pool" - "github.com/hashicorp/serf/serf" + "github.com/hashicorp/nomad/nomad/structs" ) // StatsFetcher has two functions for autopilot. First, lets us fetch all the @@ -22,7 +24,7 @@ type StatsFetcher struct { logger log.Logger pool *pool.ConnPool region string - inflight map[string]struct{} + inflight map[raft.ServerID]struct{} inflightLock sync.Mutex } @@ -32,7 +34,7 @@ func NewStatsFetcher(logger log.Logger, pool *pool.ConnPool, region string) *Sta logger: logger.Named("stats_fetcher"), pool: pool, region: region, - inflight: make(map[string]struct{}), + inflight: make(map[raft.ServerID]struct{}), } } @@ -40,39 +42,47 @@ func NewStatsFetcher(logger log.Logger, pool *pool.ConnPool, region string) *Sta // cancel this when the context is canceled because we only want one in-flight // RPC to each server, so we let it finish and then clean up the in-flight // tracking. -func (f *StatsFetcher) fetch(server *serverParts, replyCh chan *autopilot.ServerStats) { +func (f *StatsFetcher) fetch(server *autopilot.Server, replyCh chan *autopilot.ServerStats) { var args struct{} - var reply autopilot.ServerStats - err := f.pool.RPC(f.region, server.Addr, "Status.RaftStats", &args, &reply) + var reply structs.RaftStats + + // defer some cleanup to notify everything else that the fetching is no longer occurring + // this is easier than trying to make the conditionals line up just right. + defer func() { + f.inflightLock.Lock() + delete(f.inflight, server.ID) + f.inflightLock.Unlock() + }() + + addr, err := net.ResolveTCPAddr("tcp", string(server.Address)) if err != nil { - f.logger.Warn("failed retrieving server health", "server", server.Name, "error", err) - } else { - replyCh <- &reply + f.logger.Warn("error resolving TCP address for server", + "address", server.Address, + "error", err) + return } - f.inflightLock.Lock() - delete(f.inflight, server.ID) - f.inflightLock.Unlock() + err = f.pool.RPC(f.region, addr, "Status.RaftStats", &args, &reply) + if err != nil { + f.logger.Warn("error getting server health", "server", server.Name, "error", err) + return + } + + replyCh <- reply.ToAutopilotServerStats() } // Fetch will attempt to query all the servers in parallel. -func (f *StatsFetcher) Fetch(ctx context.Context, members []serf.Member) map[string]*autopilot.ServerStats { +func (f *StatsFetcher) Fetch(ctx context.Context, servers map[raft.ServerID]*autopilot.Server) map[raft.ServerID]*autopilot.ServerStats { type workItem struct { - server *serverParts + server *autopilot.Server replyCh chan *autopilot.ServerStats } - var servers []*serverParts - for _, s := range members { - if ok, parts := isNomadServer(s); ok { - servers = append(servers, parts) - } - } // Skip any servers that have inflight requests. var work []*workItem f.inflightLock.Lock() - for _, server := range servers { - if _, ok := f.inflight[server.ID]; ok { + for id, server := range servers { + if _, ok := f.inflight[id]; ok { f.logger.Warn("failed retrieving server health; last request still outstanding", "server", server.Name) } else { workItem := &workItem{ @@ -80,7 +90,7 @@ func (f *StatsFetcher) Fetch(ctx context.Context, members []serf.Member) map[str replyCh: make(chan *autopilot.ServerStats, 1), } work = append(work, workItem) - f.inflight[server.ID] = struct{}{} + f.inflight[id] = struct{}{} go f.fetch(workItem.server, workItem.replyCh) } } @@ -88,14 +98,28 @@ func (f *StatsFetcher) Fetch(ctx context.Context, members []serf.Member) map[str // Now wait for the results to come in, or for the context to be // canceled. - replies := make(map[string]*autopilot.ServerStats) + replies := make(map[raft.ServerID]*autopilot.ServerStats) for _, workItem := range work { + // Drain the reply first if there is one. + select { + case reply := <-workItem.replyCh: + replies[workItem.server.ID] = reply + continue + default: + } + select { case reply := <-workItem.replyCh: replies[workItem.server.ID] = reply case <-ctx.Done(): - f.logger.Warn("failed retrieving server health", "server", workItem.server.Name, "error", ctx.Err()) + f.logger.Warn("failed retrieving server health", + "server", workItem.server.Name, "error", ctx.Err()) + + f.inflightLock.Lock() + delete(f.inflight, workItem.server.ID) + f.inflightLock.Unlock() + } } return replies diff --git a/nomad/stats_fetcher_test.go b/nomad/stats_fetcher_test.go index 36362d604..442ff4477 100644 --- a/nomad/stats_fetcher_test.go +++ b/nomad/stats_fetcher_test.go @@ -7,6 +7,7 @@ import ( "github.com/hashicorp/nomad/ci" "github.com/hashicorp/nomad/testutil" + "github.com/hashicorp/raft" ) func TestStatsFetcher(t *testing.T) { @@ -47,13 +48,14 @@ func TestStatsFetcher(t *testing.T) { func() { ctx, cancel := context.WithTimeout(context.Background(), time.Second) defer cancel() - stats := s1.statsFetcher.Fetch(ctx, s1.Members()) + + stats := s1.statsFetcher.Fetch(ctx, s1.autopilotServers()) if len(stats) != 3 { t.Fatalf("bad: %#v", stats) } for id, stat := range stats { switch id { - case s1.config.NodeID, s2.config.NodeID, s3.config.NodeID: + case raft.ServerID(s1.config.NodeID), raft.ServerID(s2.config.NodeID), raft.ServerID(s3.config.NodeID): // OK default: t.Fatalf("bad: %s", id) @@ -68,20 +70,20 @@ func TestStatsFetcher(t *testing.T) { // Fake an in-flight request to server 3 and make sure we don't fetch // from it. func() { - s1.statsFetcher.inflight[string(s3.config.NodeID)] = struct{}{} - defer delete(s1.statsFetcher.inflight, string(s3.config.NodeID)) + s1.statsFetcher.inflight[raft.ServerID(s3.config.NodeID)] = struct{}{} + defer delete(s1.statsFetcher.inflight, raft.ServerID(s3.config.NodeID)) ctx, cancel := context.WithTimeout(context.Background(), time.Second) defer cancel() - stats := s1.statsFetcher.Fetch(ctx, s1.Members()) + stats := s1.statsFetcher.Fetch(ctx, s1.autopilotServers()) if len(stats) != 2 { t.Fatalf("bad: %#v", stats) } for id, stat := range stats { switch id { - case s1.config.NodeID, s2.config.NodeID: + case raft.ServerID(s1.config.NodeID), raft.ServerID(s2.config.NodeID): // OK - case s3.config.NodeID: + case raft.ServerID(s3.config.NodeID): t.Fatalf("bad") default: t.Fatalf("bad: %s", id) diff --git a/nomad/status_endpoint.go b/nomad/status_endpoint.go index 7b87afee7..88fa754a2 100644 --- a/nomad/status_endpoint.go +++ b/nomad/status_endpoint.go @@ -7,7 +7,6 @@ import ( log "github.com/hashicorp/go-hclog" - "github.com/hashicorp/consul/agent/consul/autopilot" "github.com/hashicorp/nomad/nomad/structs" ) @@ -97,7 +96,7 @@ func (s *Status) Members(args *structs.GenericRequest, reply *structs.ServerMemb } // RaftStats is used by Autopilot to query the raft stats of the local server. -func (s *Status) RaftStats(args struct{}, reply *autopilot.ServerStats) error { +func (s *Status) RaftStats(args struct{}, reply *structs.RaftStats) error { stats := s.srv.raft.Stats() var err error diff --git a/nomad/structs/autopilot.go b/nomad/structs/autopilot.go new file mode 100644 index 000000000..7520e084a --- /dev/null +++ b/nomad/structs/autopilot.go @@ -0,0 +1,82 @@ +package structs + +import ( + "time" + + autopilot "github.com/hashicorp/raft-autopilot" + "github.com/hashicorp/serf/serf" +) + +// OperatorHealthReply is a representation of the overall health of the cluster +type OperatorHealthReply struct { + // Healthy is true if all the servers in the cluster are healthy. + Healthy bool + + // FailureTolerance is the number of healthy servers that could be lost without + // an outage occurring. + FailureTolerance int + + // Servers holds the health of each server. + Servers []ServerHealth +} + +// ServerHealth is the health (from the leader's point of view) of a server. +type ServerHealth struct { + // ID is the raft ID of the server. + ID string + + // Name is the node name of the server. + Name string + + // Address is the address of the server. + Address string + + // The status of the SerfHealth check for the server. + SerfStatus serf.MemberStatus + + // Version is the Nomad version of the server. + Version string + + // Leader is whether this server is currently the leader. + Leader bool + + // LastContact is the time since this node's last contact with the leader. + LastContact time.Duration + + // LastTerm is the highest leader term this server has a record of in its Raft log. + LastTerm uint64 + + // LastIndex is the last log index this server has a record of in its Raft log. + LastIndex uint64 + + // Healthy is whether or not the server is healthy according to the current + // Autopilot config. + Healthy bool + + // Voter is whether this is a voting server. + Voter bool + + // StableSince is the last time this server's Healthy value changed. + StableSince time.Time +} + +// RaftStats holds miscellaneous Raft metrics for a server, used by autopilot. +type RaftStats struct { + // LastContact is the time since this node's last contact with the leader. + LastContact string + + // LastTerm is the highest leader term this server has a record of in its Raft log. + LastTerm uint64 + + // LastIndex is the last log index this server has a record of in its Raft log. + LastIndex uint64 +} + +func (s *RaftStats) ToAutopilotServerStats() *autopilot.ServerStats { + duration, _ := time.ParseDuration(s.LastContact) + return &autopilot.ServerStats{ + LastContact: duration, + LastTerm: s.LastTerm, + LastIndex: s.LastIndex, + } +}