backport of commit d425c90e0f5acc6947c3d3e32a3e54942d1cd2bf (#18674)

Co-authored-by: Luiz Aoqui <luiz@hashicorp.com>

.changelog/18664.txt

@@ -0,0 +1,3 @@
+```release-note:bug
+client: ensure null dynamic node metadata values are removed from memory
+```

client/meta_endpoint.go

@@ -45,6 +45,16 @@ func (n *NodeMeta) Apply(args *structs.NodeMetaApplyRequest, reply *structs.Node
 		dyn = maps.Clone(n.c.metaDynamic)
 		maps.Copy(dyn, args.Meta)
 
+		// Delete null values from the dynamic metadata if they are also not
+		// static. Static null values must be kept so their removal is
+		// persisted in client state.
+		for k, v := range args.Meta {
+			_, static := n.c.metaStatic[k]
+			if v == nil && !static {
+				delete(dyn, k)
+			}
+		}
+
 		if stateErr = n.c.stateDB.PutNodeMeta(dyn); stateErr != nil {
 			return
 		}

client/meta_endpoint_test.go

@@ -96,3 +96,88 @@ func TestNodeMeta_Validation(t *testing.T) {
 	err = c1.ClientRPC("NodeMeta.Apply", applyReq, &resp)
 	must.ErrorContains(t, err, "*")
 }
+
+func TestNodeMeta_unset(t *testing.T) {
+	ci.Parallel(t)
+
+	s, cleanupS := nomad.TestServer(t, nil)
+	defer cleanupS()
+	testutil.WaitForLeader(t, s.RPC)
+
+	c1, cleanup := TestClient(t, func(c *config.Config) {
+		c.Servers = []string{s.GetConfig().RPCAddr.String()}
+		c.Node.Meta["static_meta"] = "true"
+	})
+	defer cleanup()
+
+	// Set dynamic node metadata.
+	applyReq := &structs.NodeMetaApplyRequest{
+		NodeID: c1.NodeID(),
+		Meta: map[string]*string{
+			"dynamic_meta": pointer.Of("true"),
+		},
+	}
+	var resp structs.NodeMetaResponse
+	err := c1.ClientRPC("NodeMeta.Apply", applyReq, &resp)
+	must.NoError(t, err)
+
+	// Check static_meta:
+	//   1. must be present in Static.
+	//   2. must be present in Meta
+	must.Eq(t, resp.Static["static_meta"], "true")
+	must.Eq(t, resp.Meta["static_meta"], "true")
+
+	// Check dynamic_meta:
+	//   1. must be present in Dynamic.
+	//   2. must be present in Meta
+	must.Eq(t, *resp.Dynamic["dynamic_meta"], "true")
+	must.Eq(t, resp.Meta["dynamic_meta"], "true")
+
+	// Unset static node metada.
+	applyReq = &structs.NodeMetaApplyRequest{
+		NodeID: c1.NodeID(),
+		Meta: map[string]*string{
+			"static_meta": nil,
+		},
+	}
+	err = c1.ClientRPC("NodeMeta.Apply", applyReq, &resp)
+	must.NoError(t, err)
+
+	// Check static_meta:
+	//   1. must be present in Static.
+	//   2. must not be present in Meta
+	//   3. must be nil in Dynamic to persist its removal even on restarts.
+	must.Eq(t, resp.Static["static_meta"], "true")
+	must.MapNotContainsKey(t, resp.Meta, "static_meta")
+	must.Nil(t, resp.Dynamic["static_meta"])
+
+	// Check dynamic_meta:
+	//   1. must be present in Dynamic.
+	//   2. must be present in Meta
+	must.Eq(t, *resp.Dynamic["dynamic_meta"], "true")
+	must.Eq(t, resp.Meta["dynamic_meta"], "true")
+
+	// Unset dynamic node metada.
+	applyReq = &structs.NodeMetaApplyRequest{
+		NodeID: c1.NodeID(),
+		Meta: map[string]*string{
+			"dynamic_meta": nil,
+		},
+	}
+	err = c1.ClientRPC("NodeMeta.Apply", applyReq, &resp)
+	must.NoError(t, err)
+
+	// Check static_meta:
+	//   1. must be present in Static.
+	//   2. must not be present in Meta
+	//   3. must be nil in Dynamic to persist its removal even on restarts.
+	must.Eq(t, resp.Static["static_meta"], "true")
+	must.MapNotContainsKey(t, resp.Meta, "static_meta")
+	must.Nil(t, resp.Dynamic["static_meta"])
+
+	// Check dynamic_meta:
+	//   1. must not be present in Dynamic.
+	//   2. must not be present in Meta
+	must.MapNotContainsKey(t, resp.Dynamic, "dynamic_meta")
+	must.MapNotContainsKey(t, resp.Meta, "dynamic_meta")
+}