open-vault/vendor/github.com/circonus-labs/circonus-gometrics/api/graph.go
Brian Kassouf ed14061578
Raft Storage Backend (#6888)
* Work on raft backend

* Add logstore locally

* Add encryptor and unsealable interfaces

* Add clustering support to raft

* Remove client and handler

* Bootstrap raft on init

* Cleanup raft logic a bit

* More raft work

* Work on TLS config

* More work on bootstrapping

* Fix build

* More work on bootstrapping

* More bootstrapping work

* fix build

* Remove consul dep

* Fix build

* merged oss/master into raft-storage

* Work on bootstrapping

* Get bootstrapping to work

* Clean up FMS and node-id

* Update local node ID logic

* Cleanup node-id change

* Work on snapshotting

* Raft: Add remove peer API (#906)

* Add remove peer API

* Add some comments

* Fix existing snapshotting (#909)

* Raft get peers API (#912)

* Read raft configuration

* address review feedback

* Use the Leadership Transfer API to step-down the active node (#918)

* Raft join and unseal using Shamir keys (#917)

* Raft join using shamir

* Store AEAD instead of master key

* Split the raft join process to answer the challenge after a successful unseal

* get the follower to standby state

* Make unseal work

* minor changes

* Some input checks

* reuse the shamir seal access instead of new default seal access

* refactor joinRaftSendAnswer function

* Synchronously send answer in auto-unseal case

* Address review feedback

* Raft snapshots (#910)

* Fix existing snapshotting

* implement the noop snapshotting

* Add comments and switch log libraries

* add some snapshot tests

* add snapshot test file

* add TODO

* More work on raft snapshotting

* progress on the ConfigStore strategy

* Don't use two buckets

* Update the snapshot store logic to hide the file logic

* Add more backend tests

* Cleanup code a bit

* [WIP] Raft recovery (#938)

* Add recovery functionality

* remove fmt.Printfs

* Fix a few fsm bugs

* Add max size value for raft backend (#942)

* Add max size value for raft backend

* Include physical.ErrValueTooLarge in the message

* Raft snapshot Take/Restore API  (#926)

* Inital work on raft snapshot APIs

* Always redirect snapshot install/download requests

* More work on the snapshot APIs

* Cleanup code a bit

* On restore handle special cases

* Use the seal to encrypt the sha sum file

* Add sealer mechanism and fix some bugs

* Call restore while state lock is held

* Send restore cb trigger through raft log

* Make error messages nicer

* Add test helpers

* Add snapshot test

* Add shamir unseal test

* Add more raft snapshot API tests

* Fix locking

* Change working to initalize

* Add underlying raw object to test cluster core

* Move leaderUUID to core

* Add raft TLS rotation logic (#950)

* Add TLS rotation logic

* Cleanup logic a bit

* Add/Remove from follower state on add/remove peer

* add comments

* Update more comments

* Update request_forwarding_service.proto

* Make sure we populate all nodes in the followerstate obj

* Update times

* Apply review feedback

* Add more raft config setting (#947)

* Add performance config setting

* Add more config options and fix tests

* Test Raft Recovery (#944)

* Test raft recovery

* Leave out a node during recovery

* remove unused struct

* Update physical/raft/snapshot_test.go

* Update physical/raft/snapshot_test.go

* fix vendoring

* Switch to new raft interface

* Remove unused files

* Switch a gogo -> proto instance

* Remove unneeded vault dep in go.sum

* Update helper/testhelpers/testhelpers.go

Co-Authored-By: Calvin Leung Huang <cleung2010@gmail.com>

* Update vault/cluster/cluster.go

* track active key within the keyring itself (#6915)

* track active key within the keyring itself

* lookup and store using the active key ID

* update docstring

* minor refactor

* Small text fixes (#6912)

* Update physical/raft/raft.go

Co-Authored-By: Calvin Leung Huang <cleung2010@gmail.com>

* review feedback

* Move raft logical system into separate file

* Update help text a bit

* Enforce cluster addr is set and use it for raft bootstrapping

* Fix tests

* fix http test panic

* Pull in latest raft-snapshot library

* Add comment
2019-06-20 12:14:58 -07:00

357 lines
14 KiB
Go

// Copyright 2016 Circonus, Inc. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// Graph API support - Fetch, Create, Update, Delete, and Search
// See: https://login.circonus.com/resources/api/calls/graph
package api
import (
"encoding/json"
"fmt"
"net/url"
"regexp"
"github.com/circonus-labs/circonus-gometrics/api/config"
)
// GraphAccessKey defines an access key for a graph
type GraphAccessKey struct {
Active bool `json:"active,omitempty"` // boolean
Height uint `json:"height,omitempty"` // uint
Key string `json:"key,omitempty"` // string
Legend bool `json:"legend,omitempty"` // boolean
LockDate bool `json:"lock_date,omitempty"` // boolean
LockMode string `json:"lock_mode,omitempty"` // string
LockRangeEnd uint `json:"lock_range_end,omitempty"` // uint
LockRangeStart uint `json:"lock_range_start,omitempty"` // uint
LockShowTimes bool `json:"lock_show_times,omitempty"` // boolean
LockZoom string `json:"lock_zoom,omitempty"` // string
Nickname string `json:"nickname,omitempty"` // string
Title bool `json:"title,omitempty"` // boolean
Width uint `json:"width,omitempty"` // uint
XLabels bool `json:"x_labels,omitempty"` // boolean
YLabels bool `json:"y_labels,omitempty"` // boolean
}
// GraphComposite defines a composite
type GraphComposite struct {
Axis string `json:"axis"` // string
Color string `json:"color"` // string
DataFormula *string `json:"data_formula"` // string or null
Hidden bool `json:"hidden"` // boolean
LegendFormula *string `json:"legend_formula"` // string or null
Name string `json:"name"` // string
Stack *uint `json:"stack"` // uint or null
}
// GraphDatapoint defines a datapoint
type GraphDatapoint struct {
Alpha *float64 `json:"alpha,string,omitempty"` // float64
Axis string `json:"axis,omitempty"` // string
CAQL *string `json:"caql,omitempty"` // string or null
CheckID uint `json:"check_id,omitempty"` // uint
Color *string `json:"color,omitempty"` // string
DataFormula *string `json:"data_formula"` // string or null
Derive interface{} `json:"derive,omitempty"` // BUG doc: string, api: string or boolean(for caql statements)
Hidden bool `json:"hidden"` // boolean
LegendFormula *string `json:"legend_formula"` // string or null
MetricName string `json:"metric_name,omitempty"` // string
MetricType string `json:"metric_type,omitempty"` // string
Name string `json:"name"` // string
Search *string `json:"search"` // string or null
Stack *uint `json:"stack"` // uint or null
}
// GraphGuide defines a guide
type GraphGuide struct {
Color string `json:"color"` // string
DataFormula *string `json:"data_formula"` // string or null
Hidden bool `json:"hidden"` // boolean
LegendFormula *string `json:"legend_formula"` // string or null
Name string `json:"name"` // string
}
// GraphMetricCluster defines a metric cluster
type GraphMetricCluster struct {
AggregateFunc string `json:"aggregate_function,omitempty"` // string
Axis string `json:"axis,omitempty"` // string
Color *string `json:"color,omitempty"` // string
DataFormula *string `json:"data_formula"` // string or null
Hidden bool `json:"hidden"` // boolean
LegendFormula *string `json:"legend_formula"` // string or null
MetricCluster string `json:"metric_cluster,omitempty"` // string
Name string `json:"name,omitempty"` // string
Stack *uint `json:"stack"` // uint or null
}
// GraphOverlaySet defines an overlay set for a graph
type GraphOverlaySet struct {
Overlays map[string]GraphOverlay `json:"overlays"`
Title string `json:"title"`
}
// GraphOverlay defines a single overlay in an overlay set
type GraphOverlay struct {
DataOpts OverlayDataOptions `json:"data_opts,omitempty"` // OverlayDataOptions
ID string `json:"id,omitempty"` // string
Title string `json:"title,omitempty"` // string
UISpecs OverlayUISpecs `json:"ui_specs,omitempty"` // OverlayUISpecs
}
// OverlayUISpecs defines UI specs for overlay
type OverlayUISpecs struct {
Decouple bool `json:"decouple,omitempty"` // boolean
ID string `json:"id,omitempty"` // string
Label string `json:"label,omitempty"` // string
Type string `json:"type,omitempty"` // string
Z string `json:"z,omitempty"` // int encoded as string BUG doc: numeric, api: string
}
// OverlayDataOptions defines overlay options for data. Note, each overlay type requires
// a _subset_ of the options. See Graph API documentation (URL above) for details.
type OverlayDataOptions struct {
Alerts string `json:"alerts,omitempty"` // int encoded as string BUG doc: numeric, api: string
ArrayOutput string `json:"array_output,omitempty"` // int encoded as string BUG doc: numeric, api: string
BasePeriod string `json:"base_period,omitempty"` // int encoded as string BUG doc: numeric, api: string
Delay string `json:"delay,omitempty"` // int encoded as string BUG doc: numeric, api: string
Extension string `json:"extension,omitempty"` // string
GraphTitle string `json:"graph_title,omitempty"` // string
GraphUUID string `json:"graph_id,omitempty"` // string
InPercent string `json:"in_percent,omitempty"` // boolean encoded as string BUG doc: boolean, api: string
Inverse string `json:"inverse,omitempty"` // int encoded as string BUG doc: numeric, api: string
Method string `json:"method,omitempty"` // string
Model string `json:"model,omitempty"` // string
ModelEnd string `json:"model_end,omitempty"` // string
ModelPeriod string `json:"model_period,omitempty"` // string
ModelRelative string `json:"model_relative,omitempty"` // int encoded as string BUG doc: numeric, api: string
Out string `json:"out,omitempty"` // string
Prequel string `json:"prequel,omitempty"` // int
Presets string `json:"presets,omitempty"` // string
Quantiles string `json:"quantiles,omitempty"` // string
SeasonLength string `json:"season_length,omitempty"` // int encoded as string BUG doc: numeric, api: string
Sensitivity string `json:"sensitivity,omitempty"` // int encoded as string BUG doc: numeric, api: string
SingleValue string `json:"single_value,omitempty"` // int encoded as string BUG doc: numeric, api: string
TargetPeriod string `json:"target_period,omitempty"` // string
TimeOffset string `json:"time_offset,omitempty"` // string
TimeShift string `json:"time_shift,omitempty"` // int encoded as string BUG doc: numeric, api: string
Transform string `json:"transform,omitempty"` // string
Version string `json:"version,omitempty"` // int encoded as string BUG doc: numeric, api: string
Window string `json:"window,omitempty"` // int encoded as string BUG doc: numeric, api: string
XShift string `json:"x_shift,omitempty"` // string
}
// Graph defines a graph. See https://login.circonus.com/resources/api/calls/graph for more information.
type Graph struct {
AccessKeys []GraphAccessKey `json:"access_keys,omitempty"` // [] len >= 0
CID string `json:"_cid,omitempty"` // string
Composites []GraphComposite `json:"composites,omitempty"` // [] len >= 0
Datapoints []GraphDatapoint `json:"datapoints,omitempt"` // [] len >= 0
Description string `json:"description,omitempty"` // string
Guides []GraphGuide `json:"guides,omitempty"` // [] len >= 0
LineStyle *string `json:"line_style"` // string or null
LogLeftY *int `json:"logarithmic_left_y,string,omitempty"` // int encoded as string or null BUG doc: number (not string)
LogRightY *int `json:"logarithmic_right_y,string,omitempty"` // int encoded as string or null BUG doc: number (not string)
MaxLeftY *float64 `json:"max_left_y,string,omitempty"` // float64 encoded as string or null BUG doc: number (not string)
MaxRightY *float64 `json:"max_right_y,string,omitempty"` // float64 encoded as string or null BUG doc: number (not string)
MetricClusters []GraphMetricCluster `json:"metric_clusters,omitempty"` // [] len >= 0
MinLeftY *float64 `json:"min_left_y,string,omitempty"` // float64 encoded as string or null BUG doc: number (not string)
MinRightY *float64 `json:"min_right_y,string,omitempty"` // float64 encoded as string or null BUG doc: number (not string)
Notes *string `json:"notes,omitempty"` // string or null
OverlaySets *map[string]GraphOverlaySet `json:"overlay_sets,omitempty"` // GroupOverLaySets or null
Style *string `json:"style"` // string or null
Tags []string `json:"tags,omitempty"` // [] len >= 0
Title string `json:"title,omitempty"` // string
}
// NewGraph returns a Graph (with defaults, if applicable)
func NewGraph() *Graph {
return &Graph{}
}
// FetchGraph retrieves graph with passed cid.
func (a *API) FetchGraph(cid CIDType) (*Graph, error) {
if cid == nil || *cid == "" {
return nil, fmt.Errorf("Invalid graph CID [none]")
}
graphCID := string(*cid)
matched, err := regexp.MatchString(config.GraphCIDRegex, graphCID)
if err != nil {
return nil, err
}
if !matched {
return nil, fmt.Errorf("Invalid graph CID [%s]", graphCID)
}
result, err := a.Get(graphCID)
if err != nil {
return nil, err
}
if a.Debug {
a.Log.Printf("[DEBUG] fetch graph, received JSON: %s", string(result))
}
graph := new(Graph)
if err := json.Unmarshal(result, graph); err != nil {
return nil, err
}
return graph, nil
}
// FetchGraphs retrieves all graphs available to the API Token.
func (a *API) FetchGraphs() (*[]Graph, error) {
result, err := a.Get(config.GraphPrefix)
if err != nil {
return nil, err
}
var graphs []Graph
if err := json.Unmarshal(result, &graphs); err != nil {
return nil, err
}
return &graphs, nil
}
// UpdateGraph updates passed graph.
func (a *API) UpdateGraph(cfg *Graph) (*Graph, error) {
if cfg == nil {
return nil, fmt.Errorf("Invalid graph config [nil]")
}
graphCID := string(cfg.CID)
matched, err := regexp.MatchString(config.GraphCIDRegex, graphCID)
if err != nil {
return nil, err
}
if !matched {
return nil, fmt.Errorf("Invalid graph CID [%s]", graphCID)
}
jsonCfg, err := json.Marshal(cfg)
if err != nil {
return nil, err
}
if a.Debug {
a.Log.Printf("[DEBUG] update graph, sending JSON: %s", string(jsonCfg))
}
result, err := a.Put(graphCID, jsonCfg)
if err != nil {
return nil, err
}
graph := &Graph{}
if err := json.Unmarshal(result, graph); err != nil {
return nil, err
}
return graph, nil
}
// CreateGraph creates a new graph.
func (a *API) CreateGraph(cfg *Graph) (*Graph, error) {
if cfg == nil {
return nil, fmt.Errorf("Invalid graph config [nil]")
}
jsonCfg, err := json.Marshal(cfg)
if err != nil {
return nil, err
}
if a.Debug {
a.Log.Printf("[DEBUG] update graph, sending JSON: %s", string(jsonCfg))
}
result, err := a.Post(config.GraphPrefix, jsonCfg)
if err != nil {
return nil, err
}
graph := &Graph{}
if err := json.Unmarshal(result, graph); err != nil {
return nil, err
}
return graph, nil
}
// DeleteGraph deletes passed graph.
func (a *API) DeleteGraph(cfg *Graph) (bool, error) {
if cfg == nil {
return false, fmt.Errorf("Invalid graph config [nil]")
}
return a.DeleteGraphByCID(CIDType(&cfg.CID))
}
// DeleteGraphByCID deletes graph with passed cid.
func (a *API) DeleteGraphByCID(cid CIDType) (bool, error) {
if cid == nil || *cid == "" {
return false, fmt.Errorf("Invalid graph CID [none]")
}
graphCID := string(*cid)
matched, err := regexp.MatchString(config.GraphCIDRegex, graphCID)
if err != nil {
return false, err
}
if !matched {
return false, fmt.Errorf("Invalid graph CID [%s]", graphCID)
}
_, err = a.Delete(graphCID)
if err != nil {
return false, err
}
return true, nil
}
// SearchGraphs returns graphs matching the specified search query
// and/or filter. If nil is passed for both parameters all graphs
// will be returned.
func (a *API) SearchGraphs(searchCriteria *SearchQueryType, filterCriteria *SearchFilterType) (*[]Graph, error) {
q := url.Values{}
if searchCriteria != nil && *searchCriteria != "" {
q.Set("search", string(*searchCriteria))
}
if filterCriteria != nil && len(*filterCriteria) > 0 {
for filter, criteria := range *filterCriteria {
for _, val := range criteria {
q.Add(filter, val)
}
}
}
if q.Encode() == "" {
return a.FetchGraphs()
}
reqURL := url.URL{
Path: config.GraphPrefix,
RawQuery: q.Encode(),
}
result, err := a.Get(reqURL.String())
if err != nil {
return nil, fmt.Errorf("[ERROR] API call error %+v", err)
}
var graphs []Graph
if err := json.Unmarshal(result, &graphs); err != nil {
return nil, err
}
return &graphs, nil
}