From 7ddac6e43784f0ecd05247ad6a90bd51845329f4 Mon Sep 17 00:00:00 2001 From: Kit Haines Date: Thu, 26 Jan 2023 10:21:13 -0500 Subject: [PATCH] Vault 11795 vault cli verify s ign (#18437) * The verify-sign command in it's cleanest existing form. * Working state * Updates to proper verification syntax Co-authored-by: 'Alex Scheel' * make fmt * Git CI caught some stuff. * Some tests * PR-review updates. * make fmt. Co-authored-by: 'Alex Scheel' --- changelog/18437.txt | 3 + command/commands.go | 5 + command/pki_verify_sign_command.go | 231 +++++++++++++++ command/pki_verify_sign_test.go | 437 +++++++++++++++++++++++++++++ 4 files changed, 676 insertions(+) create mode 100644 changelog/18437.txt create mode 100644 command/pki_verify_sign_command.go create mode 100644 command/pki_verify_sign_test.go diff --git a/changelog/18437.txt b/changelog/18437.txt new file mode 100644 index 000000000..9ca8a8dc3 --- /dev/null +++ b/changelog/18437.txt @@ -0,0 +1,3 @@ +```release-note:improvement +client/pki: Add a new command verify-sign which checks the relationship between two certificates. +``` diff --git a/command/commands.go b/command/commands.go index 5b309c6cd..4432a6fbc 100644 --- a/command/commands.go +++ b/command/commands.go @@ -543,6 +543,11 @@ func initCommands(ui, serverCmdUi cli.Ui, runOpts *RunOptions) { BaseCommand: getBaseCommand(), }, nil }, + "pki verify-sign": func() (cli.Command, error) { + return &PKIVerifySignCommand{ + BaseCommand: getBaseCommand(), + }, nil + }, "plugin": func() (cli.Command, error) { return &PluginCommand{ BaseCommand: getBaseCommand(), diff --git a/command/pki_verify_sign_command.go b/command/pki_verify_sign_command.go new file mode 100644 index 000000000..ff2976cd1 --- /dev/null +++ b/command/pki_verify_sign_command.go @@ -0,0 +1,231 @@ +package command + +import ( + "bytes" + "crypto/x509" + "encoding/json" + "fmt" + "strconv" + "strings" + + "github.com/ghodss/yaml" + "github.com/hashicorp/vault/api" + "github.com/hashicorp/vault/sdk/helper/certutil" + "github.com/ryanuber/columnize" +) + +type PKIVerifySignCommand struct { + *BaseCommand + + flagConfig string + flagReturnIndicator string + flagDefaultDisabled bool + flagList bool +} + +func (c *PKIVerifySignCommand) Synopsis() string { + return "Check Whether One Certificate Validates Another Specified Certificate" +} + +func (c *PKIVerifySignCommand) Help() string { + helpText := ` +Usage: vault pki verify-sign POSSIBLE-ISSUER POSSIBLE-ISSUED +Here POSSIBLE-ISSUER and POSSIBLE-ISSUED are the fully name-spaced path to the certificate, +for instance: 'ns1/mount1/issuer/issuerName/json' +Returns five fields of information: +- signature_match: was the key of the issuer used to sign the issued +- path_match: the possible issuer appears in the valid certificate chain of the issued +- key_id_match: does the key-id of the issuer match the key_id of the subject +- subject_match: does the subject name of the issuer match the issuer subject of the issued +- trust_match: if someone trusted the parent issuer, is the chain provided sufficient to trust the child issued +` + return strings.TrimSpace(helpText) +} + +func (c *PKIVerifySignCommand) Flags() *FlagSets { + set := c.flagSet(FlagSetHTTP | FlagSetOutputFormat) + return set +} + +func (c *PKIVerifySignCommand) Run(args []string) int { + f := c.Flags() + if err := f.Parse(args); err != nil { + c.UI.Error(err.Error()) + return 1 + } + + args = f.Args() + + if len(args) < 2 { + if len(args) == 0 { + c.UI.Error("Not enough arguments (expected potential issuer and issued, got nothing)") + } else { + c.UI.Error("Not enough arguments (expected both potential issuer and issued, got only one)") + } + return 1 + } else if len(args) > 2 { + c.UI.Error(fmt.Sprintf("Too many arguments (expected only potential issuer and issued, got %d arguments)", len(args))) + for _, arg := range args { + if strings.HasPrefix(arg, "-") { + c.UI.Warn(fmt.Sprintf("Options (%v) must be specified before positional arguments (%v)", arg, args[0])) + break + } + } + return 1 + } + + issuer := sanitizePath(args[0]) + issued := sanitizePath(args[1]) + + client, err := c.Client() + if err != nil { + c.UI.Error(fmt.Sprintf("Failed to obtain client: %s", err)) + return 1 + } + + err, results := verifySignBetween(client, issuer, issued) + if err != nil { + c.UI.Error(fmt.Sprintf("Failed to run verification: %v", err)) + return pkiRetUsage + } + + c.outputResults(results, issuer, issued) + + return 0 +} + +func verifySignBetween(client *api.Client, issuerPath string, issuedPath string) (error, map[string]bool) { + // Note that this eats warnings + + // Fetch and Parse the Potential Issuer: + issuerResp, err := client.Logical().Read(issuerPath) + if err != nil { + return fmt.Errorf("error: unable to fetch issuer %v: %w", issuerPath, err), nil + } + issuerCertPem := issuerResp.Data["certificate"].(string) + issuerCertBundle, err := certutil.ParsePEMBundle(issuerCertPem) + if err != nil { + return err, nil + } + issuerKeyId := issuerCertBundle.Certificate.SubjectKeyId + + // Fetch and Parse the Potential Issued Cert + issuedCertResp, err := client.Logical().Read(issuedPath) + if err != nil { + return fmt.Errorf("error: unable to fetch issuer %v: %w", issuerPath, err), nil + } + if len(issuedPath) <= 2 { + return fmt.Errorf(fmt.Sprintf("%v", issuedPath)), nil + } + caChainRaw := issuedCertResp.Data["ca_chain"] + if caChainRaw == nil { + return fmt.Errorf("no ca_chain information on %v", issuedPath), nil + } + caChainCast := caChainRaw.([]interface{}) + caChain := make([]string, len(caChainCast)) + for i, cert := range caChainCast { + caChain[i] = cert.(string) + } + issuedCertPem := issuedCertResp.Data["certificate"].(string) + issuedCertBundle, err := certutil.ParsePEMBundle(issuedCertPem) + if err != nil { + return err, nil + } + parentKeyId := issuedCertBundle.Certificate.AuthorityKeyId + + // Check the Chain-Match + rootCertPool := x509.NewCertPool() + rootCertPool.AddCert(issuerCertBundle.Certificate) + checkTrustPathOptions := x509.VerifyOptions{ + Roots: rootCertPool, + } + trust := false + trusts, err := issuedCertBundle.Certificate.Verify(checkTrustPathOptions) + if err != nil && !strings.Contains(err.Error(), "certificate signed by unknown authority") { + return err, nil + } else if err == nil { + for _, chain := range trusts { + // Output of this Should Only Have One Trust with Chain of Length Two (Child followed by Parent) + for _, cert := range chain { + if issuedCertBundle.Certificate.Equal(cert) { + trust = true + break + } + } + } + } + + pathMatch := false + for _, cert := range caChain { + if strings.TrimSpace(cert) == strings.TrimSpace(issuerCertPem) { // TODO: Decode into ASN1 to Check + pathMatch = true + break + } + } + + signatureMatch := false + err = issuedCertBundle.Certificate.CheckSignatureFrom(issuerCertBundle.Certificate) + if err == nil { + signatureMatch = true + } + + result := map[string]bool{ + // This comparison isn't strictly correct, despite a standard ordering these are sets + "subject_match": bytes.Equal(issuerCertBundle.Certificate.RawSubject, issuedCertBundle.Certificate.RawIssuer), + "path_match": pathMatch, + "trust_match": trust, // TODO: Refactor into a reasonable function + "key_id_match": bytes.Equal(parentKeyId, issuerKeyId), + "signature_match": signatureMatch, + } + + return nil, result +} + +func (c *PKIVerifySignCommand) outputResults(results map[string]bool, potentialParent, potentialChild string) error { + switch Format(c.UI) { + case "", "table": + return c.outputResultsTable(results, potentialParent, potentialChild) + case "json": + return c.outputResultsJSON(results) + case "yaml": + return c.outputResultsYAML(results) + default: + return fmt.Errorf("unknown output format: %v", Format(c.UI)) + } +} + +func (c *PKIVerifySignCommand) outputResultsTable(results map[string]bool, potentialParent, potentialChild string) error { + c.UI.Output("issuer:" + potentialParent) + c.UI.Output("issued:" + potentialChild + "\n") + data := []string{"field" + hopeDelim + "value"} + for field, finding := range results { + row := field + hopeDelim + strconv.FormatBool(finding) + data = append(data, row) + } + c.UI.Output(tableOutput(data, &columnize.Config{ + Delim: hopeDelim, + })) + c.UI.Output("\n") + + return nil +} + +func (c *PKIVerifySignCommand) outputResultsJSON(results map[string]bool) error { + bytes, err := json.MarshalIndent(results, "", " ") + if err != nil { + return err + } + + c.UI.Output(string(bytes)) + return nil +} + +func (c *PKIVerifySignCommand) outputResultsYAML(results map[string]bool) error { + bytes, err := yaml.Marshal(results) + if err != nil { + return err + } + + c.UI.Output(string(bytes)) + return nil +} diff --git a/command/pki_verify_sign_test.go b/command/pki_verify_sign_test.go new file mode 100644 index 000000000..0a3b239bb --- /dev/null +++ b/command/pki_verify_sign_test.go @@ -0,0 +1,437 @@ +package command + +import ( + "bytes" + "encoding/json" + "fmt" + "strings" + "testing" + + "github.com/hashicorp/vault/api" +) + +func TestPKIVerifySign(t *testing.T) { + t.Parallel() + + client, closer := testVaultServer(t) + defer closer() + + // Relationship Map to Create + // pki-root | pki-newroot + // RootX1 RootX2 RootX4 RootX3 + // | | + // ---------------------------------------------- + // v v + // IntX1 IntX2 pki-int + // | | + // v v + // IntX3 (-----------------------) IntX3 + // + // Here X1,X2 have the same name (same mount) + // RootX4 uses the same key as RootX1 (but a different common_name/subject) + // RootX3 has the same name, and is on a different mount + // RootX1 has issued IntX1; RootX3 has issued IntX2 + createComplicatedIssuerSetUp(t, client) + + cases := []struct { + name string + args []string + expectedMatches map[string]bool + jsonOut bool + shouldError bool + expectErrorCont string + expectErrorNotCont string + nonJsonOutputCont string + }{ + { + "rootX1-matches-rootX1", + []string{"pki", "verify-sign", "-format=json", "pki-root/issuer/rootX1", "pki-root/issuer/rootX1"}, + map[string]bool{ + "key_id_match": true, + "path_match": true, + "signature_match": true, + "subject_match": true, + "trust_match": true, + }, + true, + false, + "", + "", + "", + }, + { + "rootX1-on-rootX2-onlySameName", + []string{"pki", "verify-sign", "-format=json", "pki-root/issuer/rootX1", "pki-root/issuer/rootX2"}, + map[string]bool{ + "key_id_match": false, + "path_match": false, + "signature_match": false, + "subject_match": true, + "trust_match": false, + }, + true, + false, + "", + "", + "", + }, + } + for _, testCase := range cases { + var errString string + var results map[string]interface{} + var stdOut string + + if testCase.jsonOut { + results, errString = execPKIVerifyJson(t, client, false, testCase.shouldError, testCase.args) + } else { + stdOut, errString = execPKIVerifyNonJson(t, client, testCase.shouldError, testCase.args) + } + + // Verify Error Behavior + if testCase.shouldError { + if errString == "" { + t.Fatalf("Expected error in Testcase %s : no error produced, got results %s", testCase.name, results) + } + if testCase.expectErrorCont != "" && !strings.Contains(errString, testCase.expectErrorCont) { + t.Fatalf("Expected error in Testcase %s to contain %s, but got error %s", testCase.name, testCase.expectErrorCont, errString) + } + if testCase.expectErrorNotCont != "" && strings.Contains(errString, testCase.expectErrorNotCont) { + t.Fatalf("Expected error in Testcase %s to not contain %s, but got error %s", testCase.name, testCase.expectErrorNotCont, errString) + } + } else { + if errString != "" { + t.Fatalf("Error in Testcase %s : no error expected, but got error: %s", testCase.name, errString) + } + } + + // Verify Output + if testCase.jsonOut { + isMatch, errString := verifyExpectedJson(testCase.expectedMatches, results) + if !isMatch { + t.Fatalf("Expected Results for Testcase %s, do not match returned results %s", testCase.name, errString) + } + } else { + if !strings.Contains(stdOut, testCase.nonJsonOutputCont) { + t.Fatalf("Expected standard output for Testcase %s to contain %s, but got %s", testCase.name, testCase.nonJsonOutputCont, stdOut) + } + } + + } +} + +func execPKIVerifyJson(t *testing.T, client *api.Client, expectErrorUnmarshalling bool, expectErrorOut bool, callArgs []string) (map[string]interface{}, string) { + stdout, stderr := execPKIVerifyNonJson(t, client, expectErrorOut, callArgs) + + var results map[string]interface{} + if err := json.Unmarshal([]byte(stdout), &results); err != nil && !expectErrorUnmarshalling { + t.Fatalf("failed to decode json response : %v \n json: \n%v", err, stdout) + } + + return results, stderr +} + +func execPKIVerifyNonJson(t *testing.T, client *api.Client, expectErrorOut bool, callArgs []string) (string, string) { + stdout := bytes.NewBuffer(nil) + stderr := bytes.NewBuffer(nil) + runOpts := &RunOptions{ + Stdout: stdout, + Stderr: stderr, + Client: client, + } + + code := RunCustom(callArgs, runOpts) + if !expectErrorOut && code != 0 { + t.Fatalf("running command `%v` unsuccessful (ret %v)\nerr: %v", strings.Join(callArgs, " "), code, stderr.String()) + } + + t.Log(stdout.String() + stderr.String()) + + return stdout.String(), stderr.String() +} + +func convertListOfInterfaceToString(list []interface{}, sep string) string { + newList := make([]string, len(list)) + for i, interfa := range list { + newList[i] = interfa.(string) + } + return strings.Join(newList, sep) +} + +func createComplicatedIssuerSetUp(t *testing.T, client *api.Client) { + // Relationship Map to Create + // pki-root | pki-newroot + // RootX1 RootX2 RootX4 RootX3 + // | | + // ---------------------------------------------- + // v v + // IntX1 IntX2 pki-int + // | | + // v v + // IntX3 (-----------------------) IntX3 + // + // Here X1,X2 have the same name (same mount) + // RootX4 uses the same key as RootX1 (but a different common_name/subject) + // RootX3 has the same name, and is on a different mount + // RootX1 has issued IntX1; RootX3 has issued IntX2 + + if err := client.Sys().Mount("pki-root", &api.MountInput{ + Type: "pki", + Config: api.MountConfigInput{ + MaxLeaseTTL: "36500d", + }, + }); err != nil { + t.Fatalf("pki mount error: %#v", err) + } + + if err := client.Sys().Mount("pki-newroot", &api.MountInput{ + Type: "pki", + Config: api.MountConfigInput{ + MaxLeaseTTL: "36500d", + }, + }); err != nil { + t.Fatalf("pki mount error: %#v", err) + } + + if err := client.Sys().Mount("pki-int", &api.MountInput{ + Type: "pki", + Config: api.MountConfigInput{ + MaxLeaseTTL: "36500d", + }, + }); err != nil { + t.Fatalf("pki mount error: %#v", err) + } + + resp, err := client.Logical().Write("pki-root/root/generate/internal", map[string]interface{}{ + "key_type": "ec", + "common_name": "Root X", + "ttl": "3650d", + "issuer_name": "rootX1", + "key_name": "rootX1", + }) + t.Logf("%s", resp.Data) + if err != nil || resp == nil { + t.Fatalf("failed to prime CA: %v", err) + } + + resp, err = client.Logical().Write("pki-root/root/generate/internal", map[string]interface{}{ + "key_type": "ec", + "common_name": "Root X", + "ttl": "3650d", + "issuer_name": "rootX2", + }) + t.Logf("%s", resp.Data) + if err != nil || resp == nil { + t.Fatalf("failed to prime CA: %v", err) + } + + if resp, err := client.Logical().Write("pki-newroot/root/generate/internal", map[string]interface{}{ + "key_type": "ec", + "common_name": "Root X", + "ttl": "3650d", + "issuer_name": "rootX3", + }); err != nil || resp == nil { + t.Fatalf("failed to prime CA: %v", err) + } + + if resp, err := client.Logical().Write("pki-root/root/generate/existing", map[string]interface{}{ + "common_name": "Root X4", + "ttl": "3650d", + "issuer_name": "rootX4", + "key_ref": "rootX1", + }); err != nil || resp == nil { + t.Fatalf("failed to prime CA: %v", err) + } + + // Intermediate X1 + int1CsrResp, err := client.Logical().Write("pki-int/intermediate/generate/internal", map[string]interface{}{ + "key_type": "rsa", + "common_name": "Int X1", + "ttl": "3650d", + }) + if err != nil || int1CsrResp == nil { + t.Fatalf("failed to generate CSR: %v", err) + } + int1CsrRaw, ok := int1CsrResp.Data["csr"] + if !ok { + t.Fatalf("no csr produced when generating intermediate, resp: %v", int1CsrResp) + } + int1Csr := int1CsrRaw.(string) + int1CertResp, err := client.Logical().Write("pki-root/issuer/rootX1/sign-intermediate", map[string]interface{}{ + "csr": int1Csr, + }) + if err != nil || int1CertResp == nil { + t.Fatalf("failed to sign CSR: %v", err) + } + int1CertChainRaw, ok := int1CertResp.Data["ca_chain"] + if !ok { + t.Fatalf("no ca_chain produced when signing intermediate, resp: %v", int1CertResp) + } + int1CertChain := convertListOfInterfaceToString(int1CertChainRaw.([]interface{}), "\n") + importInt1Resp, err := client.Logical().Write("pki-int/issuers/import/cert", map[string]interface{}{ + "pem_bundle": int1CertChain, + }) + if err != nil || importInt1Resp == nil { + t.Fatalf("failed to import certificate: %v", err) + } + importIssuerIdMap, ok := importInt1Resp.Data["mapping"] + if !ok { + t.Fatalf("no mapping data returned on issuer import: %v", importInt1Resp) + } + importIssuerId := "" + for key, value := range importIssuerIdMap.(map[string]interface{}) { + if value != nil && len(value.(string)) > 0 { + importIssuerId = key + break + } + } + if resp, err := client.Logical().Write("pki-int/issuer/"+importIssuerId, map[string]interface{}{ + "issuer_name": "intX1", + }); err != nil || resp == nil { + t.Fatalf("error naming issuer %v", err) + } + + // Intermediate X2 + int2CsrResp, err := client.Logical().Write("pki-int/intermediate/generate/internal", map[string]interface{}{ + "key_type": "ed25519", + "common_name": "Int X2", + "ttl": "3650d", + }) + if err != nil || int2CsrResp == nil { + t.Fatalf("failed to generate CSR: %v", err) + } + int2CsrRaw, ok := int2CsrResp.Data["csr"] + if !ok { + t.Fatalf("no csr produced when generating intermediate, resp: %v", int2CsrResp) + } + int2Csr := int2CsrRaw.(string) + int2CertResp, err := client.Logical().Write("pki-newroot/issuer/rootX3/sign-intermediate", map[string]interface{}{ + "csr": int2Csr, + }) + if err != nil || int2CertResp == nil { + t.Fatalf("failed to sign CSR: %v", err) + } + int2CertChainRaw, ok := int2CertResp.Data["ca_chain"] + if !ok { + t.Fatalf("no ca_chain produced when signing intermediate, resp: %v", int2CertResp) + } + int2CertChain := convertListOfInterfaceToString(int2CertChainRaw.([]interface{}), "\n") + importInt2Resp, err := client.Logical().Write("pki-int/issuers/import/cert", map[string]interface{}{ + "pem_bundle": int2CertChain, + "issuer_name": "intX2", + }) + if err != nil || importInt2Resp == nil { + t.Fatalf("failed to import certificate: %v", err) + } + importIssuer2IdMap, ok := importInt2Resp.Data["mapping"] + if !ok { + t.Fatalf("no mapping data returned on issuer import: %v", importInt2Resp) + } + importIssuer2Id := "" + for key, value := range importIssuer2IdMap.(map[string]interface{}) { + if value != nil && len(value.(string)) > 0 { + importIssuer2Id = key + break + } + } + if resp, err := client.Logical().Write("pki-int/issuer/"+importIssuer2Id, map[string]interface{}{ + "issuer_name": "intX2", + }); err != nil || resp == nil { + t.Fatalf("error naming issuer %v", err) + } + + // Intermediate X3 + int3CsrResp, err := client.Logical().Write("pki-int/intermediate/generate/internal", map[string]interface{}{ + "key_type": "rsa", + "common_name": "Int X3", + "ttl": "3650d", + }) + if err != nil || int3CsrResp == nil { + t.Fatalf("failed to generate CSR: %v", err) + } + int3CsrRaw, ok := int3CsrResp.Data["csr"] + if !ok { + t.Fatalf("no csr produced when generating intermediate, resp: %v", int3CsrResp) + } + int3Csr := int3CsrRaw.(string) + // sign by intX1 and import + int3CertResp1, err := client.Logical().Write("pki-int/issuer/intX1/sign-intermediate", map[string]interface{}{ + "csr": int3Csr, + }) + if err != nil || int3CertResp1 == nil { + t.Fatalf("failed to sign CSR: %v", err) + } + int3CertChainRaw1, ok := int3CertResp1.Data["ca_chain"] + if !ok { + t.Fatalf("no ca_chain produced when signing intermediate, resp: %v", int3CertResp1) + } + int3CertChain1 := convertListOfInterfaceToString(int3CertChainRaw1.([]interface{}), "\n") + importInt3Resp1, err := client.Logical().Write("pki-int/issuers/import/cert", map[string]interface{}{ + "pem_bundle": int3CertChain1, + }) + if err != nil || importInt3Resp1 == nil { + t.Fatalf("failed to import certificate: %v", err) + } + importIssuer3IdMap1, ok := importInt3Resp1.Data["mapping"] + if !ok { + t.Fatalf("no mapping data returned on issuer import: %v", importInt2Resp) + } + importIssuer3Id1 := "" + for key, value := range importIssuer3IdMap1.(map[string]interface{}) { + if value != nil && len(value.(string)) > 0 { + importIssuer3Id1 = key + break + } + } + if resp, err := client.Logical().Write("pki-int/issuer/"+importIssuer3Id1, map[string]interface{}{ + "issuer_name": "intX3", + }); err != nil || resp == nil { + t.Fatalf("error naming issuer %v", err) + } + // sign by intX2 and import + int3CertResp2, err := client.Logical().Write("pki-int/issuer/intX2/sign-intermediate", map[string]interface{}{ + "csr": int3Csr, + }) + if err != nil || int3CertResp2 == nil { + t.Fatalf("failed to sign CSR: %v", err) + } + int3CertChainRaw2, ok := int3CertResp2.Data["ca_chain"] + if !ok { + t.Fatalf("no ca_chain produced when signing intermediate, resp: %v", int3CertResp2) + } + int3CertChain2 := convertListOfInterfaceToString(int3CertChainRaw2.([]interface{}), "\n") + importInt3Resp2, err := client.Logical().Write("pki-int/issuers/import/cert", map[string]interface{}{ + "pem_bundle": int3CertChain2, + }) + if err != nil || importInt3Resp2 == nil { + t.Fatalf("failed to import certificate: %v", err) + } + importIssuer3IdMap2, ok := importInt3Resp2.Data["mapping"] + if !ok { + t.Fatalf("no mapping data returned on issuer import: %v", importInt2Resp) + } + importIssuer3Id2 := "" + for key, value := range importIssuer3IdMap2.(map[string]interface{}) { + if value != nil && len(value.(string)) > 0 { + importIssuer3Id2 = key + break + } + } + if resp, err := client.Logical().Write("pki-int/issuer/"+importIssuer3Id2, map[string]interface{}{ + "issuer_name": "intX3also", + }); err != nil || resp == nil { + t.Fatalf("error naming issuer %v", err) + } +} + +func verifyExpectedJson(expectedResults map[string]bool, results map[string]interface{}) (isMatch bool, error string) { + if len(expectedResults) != len(results) { + return false, fmt.Sprintf("Different Number of Keys in Expected Results (%d), than results (%d)", + len(expectedResults), len(results)) + } + for key, value := range expectedResults { + if results[key].(bool) != value { + return false, fmt.Sprintf("Different value for key %s : expected %t got %s", key, value, results[key]) + } + } + return true, "" +}