audit: support a configurable prefix string to write before each message (#2359)

A static token at the beginning of a log line can help systems parse
logs better. For example, rsyslog and syslog-ng will recognize the
'@cee: ' prefix and will parse the rest of the line as a valid json message.
This is useful in environments where there is a mix of structured and
unstructured logs.

audit/format_json.go

@@ -8,13 +8,22 @@ import (
 
 // JSONFormatWriter is an AuditFormatWriter implementation that structures data into
 // a JSON format.
-type JSONFormatWriter struct{}
+type JSONFormatWriter struct {
+	Prefix string
+}
 
 func (f *JSONFormatWriter) WriteRequest(w io.Writer, req *AuditRequestEntry) error {
 	if req == nil {
 		return fmt.Errorf("request entry was nil, cannot encode")
 	}
 
+	if len(f.Prefix) > 0 {
+		_, err := w.Write([]byte(f.Prefix))
+		if err != nil {
+			return err
+		}
+	}
+
 	enc := json.NewEncoder(w)
 	return enc.Encode(req)
 }
@@ -24,6 +33,13 @@ func (f *JSONFormatWriter) WriteResponse(w io.Writer, resp *AuditResponseEntry)
 		return fmt.Errorf("response entry was nil, cannot encode")
 	}
 
+	if len(f.Prefix) > 0 {
+		_, err := w.Write([]byte(f.Prefix))
+		if err != nil {
+			return err
+		}
+	}
+
 	enc := json.NewEncoder(w)
 	return enc.Encode(resp)
 }

audit/format_json_test.go

@@ -19,6 +19,7 @@ func TestFormatJSON_formatRequest(t *testing.T) {
 		Auth   *logical.Auth
 		Req    *logical.Request
 		Err    error
+		Prefix string
 		Result string
 	}{
 		"auth, request": {
@@ -37,6 +38,26 @@ func TestFormatJSON_formatRequest(t *testing.T) {
 				},
 			},
 			errors.New("this is an error"),
+			"",
+			testFormatJSONReqBasicStr,
+		},
+		"auth, request with prefix": {
+			&logical.Auth{ClientToken: "foo", Policies: []string{"root"}},
+			&logical.Request{
+				Operation: logical.UpdateOperation,
+				Path:      "/foo",
+				Connection: &logical.Connection{
+					RemoteAddr: "127.0.0.1",
+				},
+				WrapInfo: &logical.RequestWrapInfo{
+					TTL: 60 * time.Second,
+				},
+				Headers: map[string][]string{
+					"foo": []string{"bar"},
+				},
+			},
+			errors.New("this is an error"),
+			"@cee: ",
 			testFormatJSONReqBasicStr,
 		},
 	}
@@ -44,7 +65,9 @@ func TestFormatJSON_formatRequest(t *testing.T) {
 	for name, tc := range cases {
 		var buf bytes.Buffer
 		formatter := AuditFormatter{
-			AuditFormatWriter: &JSONFormatWriter{},
+			AuditFormatWriter: &JSONFormatWriter{
+				Prefix: tc.Prefix,
+			},
 		}
 		salter, _ := salt.NewSalt(nil, nil)
 		config := FormatterConfig{
@@ -54,13 +77,17 @@ func TestFormatJSON_formatRequest(t *testing.T) {
 			t.Fatalf("bad: %s\nerr: %s", name, err)
 		}
 
+		if !strings.HasPrefix(buf.String(), tc.Prefix) {
+			t.Fatalf("no prefix: %s \n log: %s\nprefix: %s", name, tc.Result, tc.Prefix)
+		}
+
 		var expectedjson = new(AuditRequestEntry)
 		if err := jsonutil.DecodeJSON([]byte(tc.Result), &expectedjson); err != nil {
 			t.Fatalf("bad json: %s", err)
 		}
 
 		var actualjson = new(AuditRequestEntry)
-		if err := jsonutil.DecodeJSON([]byte(buf.String()), &actualjson); err != nil {
+		if err := jsonutil.DecodeJSON([]byte(buf.String())[len(tc.Prefix):], &actualjson); err != nil {
 			t.Fatalf("bad json: %s", err)
 		}
 
@@ -71,7 +98,7 @@ func TestFormatJSON_formatRequest(t *testing.T) {
 			t.Fatalf("unable to marshal json: %s", err)
 		}
 
-		if strings.TrimSpace(buf.String()) != string(expectedBytes) {
+		if !strings.HasSuffix(strings.TrimSpace(buf.String()), string(expectedBytes)) {
 			t.Fatalf(
 				"bad: %s\nResult:\n\n'%s'\n\nExpected:\n\n'%s'",
 				name, buf.String(), string(expectedBytes))

audit/format_jsonx.go

@@ -10,13 +10,22 @@ import (
 
 // JSONxFormatWriter is an AuditFormatWriter implementation that structures data into
 // a XML format.
-type JSONxFormatWriter struct{}
+type JSONxFormatWriter struct {
+	Prefix string
+}
 
 func (f *JSONxFormatWriter) WriteRequest(w io.Writer, req *AuditRequestEntry) error {
 	if req == nil {
 		return fmt.Errorf("request entry was nil, cannot encode")
 	}
 
+	if len(f.Prefix) > 0 {
+		_, err := w.Write([]byte(f.Prefix))
+		if err != nil {
+			return err
+		}
+	}
+
 	jsonBytes, err := json.Marshal(req)
 	if err != nil {
 		return err
@@ -36,6 +45,13 @@ func (f *JSONxFormatWriter) WriteResponse(w io.Writer, resp *AuditResponseEntry)
 		return fmt.Errorf("response entry was nil, cannot encode")
 	}
 
+	if len(f.Prefix) > 0 {
+		_, err := w.Write([]byte(f.Prefix))
+		if err != nil {
+			return err
+		}
+	}
+
 	jsonBytes, err := json.Marshal(resp)
 	if err != nil {
 		return err

audit/format_jsonx_test.go

@@ -17,6 +17,7 @@ func TestFormatJSONx_formatRequest(t *testing.T) {
 		Auth     *logical.Auth
 		Req      *logical.Request
 		Err      error
+		Prefix   string
 		Result   string
 		Expected string
 	}{
@@ -37,6 +38,27 @@ func TestFormatJSONx_formatRequest(t *testing.T) {
 			},
 			errors.New("this is an error"),
 			"",
+			"",
+			`<json:object name="auth"><json:string name="accessor"></json:string><json:string name="client_token"></json:string><json:string name="display_name"></json:string><json:null name="metadata" /><json:array name="policies"><json:string>root</json:string></json:array></json:object><json:string name="error">this is an error</json:string><json:object name="request"><json:string name="client_token"></json:string><json:string name="client_token_accessor"></json:string><json:null name="data" /><json:object name="headers"><json:array name="foo"><json:string>bar</json:string></json:array></json:object><json:string name="id"></json:string><json:string name="operation">update</json:string><json:string name="path">/foo</json:string><json:string name="remote_address">127.0.0.1</json:string><json:number name="wrap_ttl">60</json:number></json:object><json:string name="type">request</json:string>`,
+		},
+		"auth, request with prefix": {
+			&logical.Auth{ClientToken: "foo", Policies: []string{"root"}},
+			&logical.Request{
+				Operation: logical.UpdateOperation,
+				Path:      "/foo",
+				Connection: &logical.Connection{
+					RemoteAddr: "127.0.0.1",
+				},
+				WrapInfo: &logical.RequestWrapInfo{
+					TTL: 60 * time.Second,
+				},
+				Headers: map[string][]string{
+					"foo": []string{"bar"},
+				},
+			},
+			errors.New("this is an error"),
+			"",
+			"@cee: ",
 			`<json:object name="auth"><json:string name="accessor"></json:string><json:string name="client_token"></json:string><json:string name="display_name"></json:string><json:null name="metadata" /><json:array name="policies"><json:string>root</json:string></json:array></json:object><json:string name="error">this is an error</json:string><json:object name="request"><json:string name="client_token"></json:string><json:string name="client_token_accessor"></json:string><json:null name="data" /><json:object name="headers"><json:array name="foo"><json:string>bar</json:string></json:array></json:object><json:string name="id"></json:string><json:string name="operation">update</json:string><json:string name="path">/foo</json:string><json:string name="remote_address">127.0.0.1</json:string><json:number name="wrap_ttl">60</json:number></json:object><json:string name="type">request</json:string>`,
 		},
 	}
@@ -44,7 +66,9 @@ func TestFormatJSONx_formatRequest(t *testing.T) {
 	for name, tc := range cases {
 		var buf bytes.Buffer
 		formatter := AuditFormatter{
-			AuditFormatWriter: &JSONxFormatWriter{},
+			AuditFormatWriter: &JSONxFormatWriter{
+				Prefix: tc.Prefix,
+			},
 		}
 		salter, _ := salt.NewSalt(nil, nil)
 		config := FormatterConfig{
@@ -55,7 +79,11 @@ func TestFormatJSONx_formatRequest(t *testing.T) {
 			t.Fatalf("bad: %s\nerr: %s", name, err)
 		}
 
-		if strings.TrimSpace(buf.String()) != string(tc.Expected) {
+		if !strings.HasPrefix(buf.String(), tc.Prefix) {
+			t.Fatalf("no prefix: %s \n log: %s\nprefix: %s", name, tc.Result, tc.Prefix)
+		}
+
+		if !strings.HasSuffix(strings.TrimSpace(buf.String()), string(tc.Expected)) {
 			t.Fatalf(
 				"bad: %s\nResult:\n\n'%s'\n\nExpected:\n\n'%s'",
 				name, strings.TrimSpace(buf.String()), string(tc.Expected))

builtin/audit/file/backend.go

@@ -76,9 +76,13 @@ func Factory(conf *audit.BackendConfig) (audit.Backend, error) {
 
 	switch format {
 	case "json":
-		b.formatter.AuditFormatWriter = &audit.JSONFormatWriter{}
+		b.formatter.AuditFormatWriter = &audit.JSONFormatWriter{
+			Prefix: conf.Config["prefix"],
+		}
 	case "jsonx":
-		b.formatter.AuditFormatWriter = &audit.JSONxFormatWriter{}
+		b.formatter.AuditFormatWriter = &audit.JSONxFormatWriter{
+			Prefix: conf.Config["prefix"],
+		}
 	}
 
 	// Ensure that the file can be successfully opened for writing;

builtin/audit/socket/backend.go

@@ -87,9 +87,13 @@ func Factory(conf *audit.BackendConfig) (audit.Backend, error) {
 
 	switch format {
 	case "json":
-		b.formatter.AuditFormatWriter = &audit.JSONFormatWriter{}
+		b.formatter.AuditFormatWriter = &audit.JSONFormatWriter{
+			Prefix: conf.Config["prefix"],
+		}
 	case "jsonx":
-		b.formatter.AuditFormatWriter = &audit.JSONxFormatWriter{}
+		b.formatter.AuditFormatWriter = &audit.JSONxFormatWriter{
+			Prefix: conf.Config["prefix"],
+		}
 	}
 
 	return b, nil

builtin/audit/syslog/backend.go

@@ -74,9 +74,13 @@ func Factory(conf *audit.BackendConfig) (audit.Backend, error) {
 
 	switch format {
 	case "json":
-		b.formatter.AuditFormatWriter = &audit.JSONFormatWriter{}
+		b.formatter.AuditFormatWriter = &audit.JSONFormatWriter{
+			Prefix: conf.Config["prefix"],
+		}
 	case "jsonx":
-		b.formatter.AuditFormatWriter = &audit.JSONxFormatWriter{}
+		b.formatter.AuditFormatWriter = &audit.JSONxFormatWriter{
+			Prefix: conf.Config["prefix"],
+		}
 	}
 
 	return b, nil

website/source/docs/audit/file.html.md

@@ -85,6 +85,12 @@ Following are the configuration options available for the backend.
             Allows selecting the output format. Valid values are `json` (the
             default) and `jsonx`, which formats the normal log entries as XML.
       </li>
+      <li>
+        <span class="param">prefix</span>
+        <span class="param-flags">optional</span>
+            Allows a customizable string prefix to write before the actual log
+            line. Defaults to an empty string.
+      </li>
     </ul>
   </dd>
 </dl>

website/source/docs/audit/socket.html.md

@@ -74,6 +74,12 @@ Following are the configuration options available for the backend.
         <span class="param-flags">optional</span>
             Sets the timeout for writes to the socket. Defaults to "2s" (2 seconds).
         </li>
+      <li>
+        <span class="param">prefix</span>
+        <span class="param-flags">optional</span>
+            Allows a customizable string prefix to write before the actual log
+            line. Defaults to an empty string.
+      </li>
     </ul>
   </dd>
 </dl>

website/source/docs/audit/syslog.html.md

@@ -71,6 +71,12 @@ Following are the configuration options available for the backend.
             Allows selecting the output format. Valid values are `json` (the
             default) and `jsonx`, which formats the normal log entries as XML.
       </li>
+      <li>
+        <span class="param">prefix</span>
+        <span class="param-flags">optional</span>
+            Allows a customizable string prefix to write before the actual log
+            line. Defaults to an empty string.
+      </li>
     </ul>
   </dd>
 </dl>