config: refactor to consolidate all File->Source loading

Previously the logic for reading ConfigFiles and produces Sources was split
between NewBuilder and Build. This commit moves all of the logic into NewBuilder
so that Build() can operate entirely on Sources.

This change is in preparation for logging warnings when files have an
unsupported extension.

It also reduces the scope of BuilderOpts, and gets us very close to removing
Builder.options.
This commit is contained in:
Daniel Nephin 2020-05-01 18:45:15 -04:00
parent c820a8de88
commit 2c2da41b3d
4 changed files with 140 additions and 64 deletions

View File

@ -81,8 +81,7 @@ type Builder struct {
err error err error
} }
// NewBuilder returns a new configuration builder based on the given command // NewBuilder returns a new configuration Builder from the BuilderOpts.
// line flags.
func NewBuilder(opts BuilderOpts) (*Builder, error) { func NewBuilder(opts BuilderOpts) (*Builder, error) {
newSource := func(name string, v interface{}) Source { newSource := func(name string, v interface{}) Source {
b, err := json.MarshalIndent(v, "", " ") b, err := json.MarshalIndent(v, "", " ")
@ -97,7 +96,7 @@ func NewBuilder(opts BuilderOpts) (*Builder, error) {
Head: []Source{DefaultSource(), DefaultEnterpriseSource()}, Head: []Source{DefaultSource(), DefaultEnterpriseSource()},
} }
if b.boolVal(b.options.DevMode) { if b.boolVal(opts.DevMode) {
b.Head = append(b.Head, DevSource()) b.Head = append(b.Head, DevSource())
} }
@ -106,17 +105,17 @@ func NewBuilder(opts BuilderOpts) (*Builder, error) {
// we need to merge all slice values defined in flags before we // we need to merge all slice values defined in flags before we
// merge the config files since the flag values for slices are // merge the config files since the flag values for slices are
// otherwise appended instead of prepended. // otherwise appended instead of prepended.
slices, values := b.splitSlicesAndValues(b.options.Config) slices, values := b.splitSlicesAndValues(opts.Config)
b.Head = append(b.Head, newSource("flags.slices", slices)) b.Head = append(b.Head, newSource("flags.slices", slices))
for _, path := range b.options.ConfigFiles { for _, path := range opts.ConfigFiles {
sources, err := b.ReadPath(path) sources, err := sourcesFromPath(path, opts)
if err != nil { if err != nil {
return nil, err return nil, err
} }
b.Sources = append(b.Sources, sources...) b.Sources = append(b.Sources, sources...)
} }
b.Tail = append(b.Tail, newSource("flags.values", values)) b.Tail = append(b.Tail, newSource("flags.values", values))
for i, s := range b.options.HCL { for i, s := range opts.HCL {
b.Tail = append(b.Tail, Source{ b.Tail = append(b.Tail, Source{
Name: fmt.Sprintf("flags-%d.hcl", i), Name: fmt.Sprintf("flags-%d.hcl", i),
Format: "hcl", Format: "hcl",
@ -124,16 +123,16 @@ func NewBuilder(opts BuilderOpts) (*Builder, error) {
}) })
} }
b.Tail = append(b.Tail, NonUserSource(), DefaultConsulSource(), OverrideEnterpriseSource(), DefaultVersionSource()) b.Tail = append(b.Tail, NonUserSource(), DefaultConsulSource(), OverrideEnterpriseSource(), DefaultVersionSource())
if b.boolVal(b.options.DevMode) { if b.boolVal(opts.DevMode) {
b.Tail = append(b.Tail, DevConsulSource()) b.Tail = append(b.Tail, DevConsulSource())
} }
return b, nil return b, nil
} }
// ReadPath reads a single config file or all files in a directory (but // sourcesFromPath reads a single config file or all files in a directory (but
// not its sub-directories) and appends them to the list of config // not its sub-directories) and returns Sources created from the
// sources. // files.
func (b *Builder) ReadPath(path string) ([]Source, error) { func sourcesFromPath(path string, options BuilderOpts) ([]Source, error) {
f, err := os.Open(path) f, err := os.Open(path)
if err != nil { if err != nil {
return nil, fmt.Errorf("config: Open failed on %s. %s", path, err) return nil, fmt.Errorf("config: Open failed on %s. %s", path, err)
@ -146,7 +145,12 @@ func (b *Builder) ReadPath(path string) ([]Source, error) {
} }
if !fi.IsDir() { if !fi.IsDir() {
src, err := b.ReadFile(path) if !shouldParseFile(path, options.ConfigFormat) {
// TODO: log warning
return nil, nil
}
src, err := newSourceFromFile(path, options.ConfigFormat)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -181,36 +185,46 @@ func (b *Builder) ReadPath(path string) ([]Source, error) {
continue continue
} }
if b.shouldParseFile(fp) { if !shouldParseFile(fp, options.ConfigFormat) {
src, err := b.ReadFile(fp) // TODO: log warning
if err != nil { continue
return nil, err
}
sources = append(sources, src)
} }
src, err := newSourceFromFile(fp, options.ConfigFormat)
if err != nil {
return nil, err
}
sources = append(sources, src)
} }
return sources, nil return sources, nil
} }
// ReadFile parses a JSON or HCL config file and appends it to the list of // newSourceFromFile creates a Source from the contents of the file at path.
// config sources. func newSourceFromFile(path string, format string) (Source, error) {
func (b *Builder) ReadFile(path string) (Source, error) {
data, err := ioutil.ReadFile(path) data, err := ioutil.ReadFile(path)
if err != nil { if err != nil {
return Source{}, fmt.Errorf("config: ReadFile failed on %s: %s", path, err) return Source{}, fmt.Errorf("config: failed to read %s: %s", path, err)
} }
return Source{Name: path, Data: string(data)}, nil if format == "" {
format = formatFromFileExtension(path)
}
return Source{Name: path, Data: string(data), Format: format}, nil
} }
// shouldParse file determines whether the file to be read is of a supported extension // shouldParse file determines whether the file to be read is of a supported extension
func (b *Builder) shouldParseFile(path string) bool { func shouldParseFile(path string, configFormat string) bool {
srcFormat := FormatFrom(path) srcFormat := formatFromFileExtension(path)
return configFormat != "" || srcFormat == "hcl" || srcFormat == "json"
}
// If config-format is not set, only read files with supported extensions func formatFromFileExtension(name string) string {
if b.options.ConfigFormat == "" && srcFormat != "hcl" && srcFormat != "json" { switch {
return false case strings.HasSuffix(name, ".json"):
return "json"
case strings.HasSuffix(name, ".hcl"):
return "hcl"
default:
return ""
} }
return true
} }
type byName []os.FileInfo type byName []os.FileInfo
@ -240,9 +254,7 @@ func (b *Builder) Build() (rt RuntimeConfig, err error) {
b.err = nil b.err = nil
b.Warnings = nil b.Warnings = nil
// ---------------------------------------------------------------- // TODO: move to NewBuilder to remove Builder.options field
// merge config sources as follows
//
configFormat := b.options.ConfigFormat configFormat := b.options.ConfigFormat
if configFormat != "" && configFormat != "json" && configFormat != "hcl" { if configFormat != "" && configFormat != "json" && configFormat != "hcl" {
return RuntimeConfig{}, fmt.Errorf("config: -config-format must be either 'hcl' or 'json'") return RuntimeConfig{}, fmt.Errorf("config: -config-format must be either 'hcl' or 'json'")
@ -251,19 +263,7 @@ func (b *Builder) Build() (rt RuntimeConfig, err error) {
// build the list of config sources // build the list of config sources
var srcs []Source var srcs []Source
srcs = append(srcs, b.Head...) srcs = append(srcs, b.Head...)
for _, src := range b.Sources { srcs = append(srcs, b.Sources...)
// skip file if it should not be parsed
if !b.shouldParseFile(src.Name) {
continue
}
// if config-format is set, files of any extension will be interpreted in that format
src.Format = FormatFrom(src.Name)
if configFormat != "" {
src.Format = configFormat
}
srcs = append(srcs, src)
}
srcs = append(srcs, b.Tail...) srcs = append(srcs, b.Tail...)
// parse the config sources into a configuration // parse the config sources into a configuration

View File

@ -0,0 +1,93 @@
package config
import (
"fmt"
"io/ioutil"
"os"
"path/filepath"
"testing"
"github.com/stretchr/testify/require"
)
func TestShouldParseFile(t *testing.T) {
var testcases = []struct {
filename string
configFormat string
expected bool
}{
{filename: "config.json", expected: true},
{filename: "config.hcl", expected: true},
{filename: "config", configFormat: "hcl", expected: true},
{filename: "config.js", configFormat: "json", expected: true},
{filename: "config.yaml", expected: false},
}
for _, tc := range testcases {
name := fmt.Sprintf("filename=%s, format=%s", tc.filename, tc.configFormat)
t.Run(name, func(t *testing.T) {
require.Equal(t, tc.expected, shouldParseFile(tc.filename, tc.configFormat))
})
}
}
func TestNewBuilder_PopulatesSourcesFromConfigFiles(t *testing.T) {
paths := setupConfigFiles(t)
b, err := NewBuilder(BuilderOpts{ConfigFiles: paths})
require.NoError(t, err)
expected := []Source{
{Name: paths[0], Format: "hcl", Data: "content a"},
{Name: paths[1], Format: "json", Data: "content b"},
{Name: filepath.Join(paths[3], "a.hcl"), Format: "hcl", Data: "content a"},
{Name: filepath.Join(paths[3], "b.json"), Format: "json", Data: "content b"},
}
require.Equal(t, expected, b.Sources)
}
func TestNewBuilder_PopulatesSourcesFromConfigFiles_WithConfigFormat(t *testing.T) {
paths := setupConfigFiles(t)
b, err := NewBuilder(BuilderOpts{ConfigFiles: paths, ConfigFormat: "hcl"})
require.NoError(t, err)
expected := []Source{
{Name: paths[0], Format: "hcl", Data: "content a"},
{Name: paths[1], Format: "hcl", Data: "content b"},
{Name: paths[2], Format: "hcl", Data: "content c"},
{Name: filepath.Join(paths[3], "a.hcl"), Format: "hcl", Data: "content a"},
{Name: filepath.Join(paths[3], "b.json"), Format: "hcl", Data: "content b"},
{Name: filepath.Join(paths[3], "c.yaml"), Format: "hcl", Data: "content c"},
}
require.Equal(t, expected, b.Sources)
}
// TODO: this would be much nicer with gotest.tools/fs
func setupConfigFiles(t *testing.T) []string {
t.Helper()
path, err := ioutil.TempDir("", t.Name())
require.NoError(t, err)
t.Cleanup(func() { os.RemoveAll(path) })
subpath := filepath.Join(path, "sub")
err = os.Mkdir(subpath, 0755)
require.NoError(t, err)
for _, dir := range []string{path, subpath} {
err = ioutil.WriteFile(filepath.Join(dir, "a.hcl"), []byte("content a"), 0644)
require.NoError(t, err)
err = ioutil.WriteFile(filepath.Join(dir, "b.json"), []byte("content b"), 0644)
require.NoError(t, err)
err = ioutil.WriteFile(filepath.Join(dir, "c.yaml"), []byte("content c"), 0644)
require.NoError(t, err)
}
return []string{
filepath.Join(path, "a.hcl"),
filepath.Join(path, "b.json"),
filepath.Join(path, "c.yaml"),
subpath,
}
}

View File

@ -3,7 +3,6 @@ package config
import ( import (
"encoding/json" "encoding/json"
"fmt" "fmt"
"strings"
"github.com/hashicorp/consul/lib/decode" "github.com/hashicorp/consul/lib/decode"
"github.com/hashicorp/hcl" "github.com/hashicorp/hcl"
@ -21,17 +20,6 @@ type Source struct {
Data string Data string
} }
func FormatFrom(name string) string {
switch {
case strings.HasSuffix(name, ".json"):
return "json"
case strings.HasSuffix(name, ".hcl"):
return "hcl"
default:
return ""
}
}
// Parse parses a config fragment in either JSON or HCL format. // Parse parses a config fragment in either JSON or HCL format.
func Parse(data string, format string) (c Config, md mapstructure.Metadata, err error) { func Parse(data string, format string) (c Config, md mapstructure.Metadata, err error) {
var raw map[string]interface{} var raw map[string]interface{}

View File

@ -6003,16 +6003,11 @@ func TestFullConfig(t *testing.T) {
t.Fatalf("ParseFlags: %s", err) t.Fatalf("ParseFlags: %s", err)
} }
// ensure that all fields are set to unique non-zero values
// if err := nonZero("Config", nil, c); err != nil {
// t.Fatal(err)
// }
b, err := NewBuilder(flags) b, err := NewBuilder(flags)
if err != nil { if err != nil {
t.Fatalf("NewBuilder: %s", err) t.Fatalf("NewBuilder: %s", err)
} }
b.Sources = append(b.Sources, Source{Name: "full." + format, Data: data}) b.Sources = append(b.Sources, Source{Name: "full." + format, Data: data, Format: format})
b.Tail = append(b.Tail, tail[format]...) b.Tail = append(b.Tail, tail[format]...)
b.Tail = append(b.Tail, VersionSource("JNtPSav3", "R909Hblt", "ZT1JOQLn")) b.Tail = append(b.Tail, VersionSource("JNtPSav3", "R909Hblt", "ZT1JOQLn"))