Backport of connect: deployments should wait for Connect sidecar checks into release/1.6.x #19365

Co-authored-by: Tim Gross <tgross@hashicorp.com>
This commit is contained in:
hc-github-team-nomad-core 2023-12-07 13:27:56 -06:00 committed by GitHub
parent e1a6a4c42f
commit a8c0b2ebb5
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 71 additions and 0 deletions

3
.changelog/19334.txt Normal file
View File

@ -0,0 +1,3 @@
```release-note:bug
connect: Fixed a bug where deployments would not wait for Connect sidecar task health checks to pass
```

View File

@ -575,6 +575,8 @@ func evaluateConsulChecks(services []*structs.Service, registrations *servicereg
} }
} }
// The sidecar service and checks are created when the service is
// registered, not on job registration, so they won't appear in the jobspec.
if !maps.Equal(expChecks, regChecks) { if !maps.Equal(expChecks, regChecks) {
return false return false
} }
@ -595,6 +597,21 @@ func evaluateConsulChecks(services []*structs.Service, registrations *servicereg
} }
} }
} }
// Check the health of Connect sidecars, so we don't accidentally
// mark an allocation healthy before min_healthy_time. These don't
// currently support on_update.
if service.SidecarService != nil {
for _, check := range service.SidecarChecks {
switch check.Status {
case api.HealthWarning:
return false
case api.HealthCritical:
return false
}
}
}
} }
} }

View File

@ -1466,6 +1466,42 @@ func TestTracker_evaluateConsulChecks(t *testing.T) {
}, },
}, },
}, },
{
name: "failing sidecar checks only",
exp: false,
tg: &structs.TaskGroup{
Services: []*structs.Service{{
Name: "group-s1",
Checks: []*structs.ServiceCheck{
{Name: "c1"},
},
}},
},
registrations: &serviceregistration.AllocRegistration{
Tasks: map[string]*serviceregistration.ServiceRegistrations{
"group": {
Services: map[string]*serviceregistration.ServiceRegistration{
"abc123": {
ServiceID: "abc123",
Checks: []*consulapi.AgentCheck{
{
Name: "c1",
Status: consulapi.HealthPassing,
},
},
SidecarService: &consulapi.AgentService{},
SidecarChecks: []*consulapi.AgentCheck{
{
Name: "sidecar-check",
Status: consulapi.HealthCritical,
},
},
},
},
},
},
},
},
} }
for _, tc := range cases { for _, tc := range cases {

View File

@ -144,6 +144,12 @@ type ServiceRegistration struct {
// Checks is the status of the registered checks. // Checks is the status of the registered checks.
Checks []*api.AgentCheck Checks []*api.AgentCheck
// SidecarService is the AgentService registered in Consul for any Connect sidecar
SidecarService *api.AgentService
// SidecarChecks is the status of the registered checks for any Connect sidecar
SidecarChecks []*api.AgentCheck
} }
func (s *ServiceRegistration) copy() *ServiceRegistration { func (s *ServiceRegistration) copy() *ServiceRegistration {

View File

@ -1489,6 +1489,15 @@ func (c *ServiceClient) AllocRegistrations(allocID string) (*serviceregistration
sreg.Checks = append(sreg.Checks, check) sreg.Checks = append(sreg.Checks, check)
} }
} }
if sidecarService := getNomadSidecar(serviceID, services); sidecarService != nil {
sreg.SidecarService = sidecarService
for _, check := range checks {
if check.ServiceID == sidecarService.ID {
sreg.SidecarChecks = append(sreg.SidecarChecks, check)
}
}
}
} }
} }