Fix panicky xDS test flakes (#16305)

* Add defensive guard to make some tests less flaky and panic less

* Do the actual fix

agent/xds/testing.go

@@ -85,6 +85,8 @@ type TestEnvoy struct {
 	EnvoyVersion string
 
 	deltaStream *TestADSDeltaStream // Incremental v3
+
+	closed bool
 }
 
 // NewTestEnvoy creates a TestEnvoy instance.
@@ -225,9 +227,9 @@ func (e *TestEnvoy) Close() error {
 	defer e.mu.Unlock()
 
 	// unblock the recv chans to simulate recv errors when client disconnects
-	if e.deltaStream != nil && e.deltaStream.recvCh != nil {
+	if !e.closed && e.deltaStream.recvCh != nil {
 		close(e.deltaStream.recvCh)
-		e.deltaStream = nil
+		e.closed = true
 	}
 	if e.cancel != nil {
 		e.cancel()