2013-12-23 19:38:51 +00:00
|
|
|
package agent
|
|
|
|
|
|
|
|
import (
|
|
|
|
"encoding/json"
|
2014-11-18 16:03:36 +00:00
|
|
|
"fmt"
|
2017-09-25 18:40:42 +00:00
|
|
|
"net"
|
2013-12-23 19:38:51 +00:00
|
|
|
"net/http"
|
2014-03-20 00:50:57 +00:00
|
|
|
"net/http/pprof"
|
2015-04-14 00:31:53 +00:00
|
|
|
"net/url"
|
2017-07-15 07:07:08 +00:00
|
|
|
"regexp"
|
2014-02-05 22:36:13 +00:00
|
|
|
"strconv"
|
2014-08-22 19:59:47 +00:00
|
|
|
"strings"
|
2013-12-23 22:26:34 +00:00
|
|
|
"time"
|
2014-08-22 19:59:47 +00:00
|
|
|
|
2016-08-09 22:41:15 +00:00
|
|
|
"github.com/armon/go-metrics"
|
2017-08-23 14:52:48 +00:00
|
|
|
"github.com/hashicorp/consul/acl"
|
2017-07-06 10:34:00 +00:00
|
|
|
"github.com/hashicorp/consul/agent/structs"
|
2014-08-22 19:59:47 +00:00
|
|
|
"github.com/mitchellh/mapstructure"
|
2013-12-23 19:38:51 +00:00
|
|
|
)
|
|
|
|
|
2017-09-26 06:11:19 +00:00
|
|
|
// MethodNotAllowedError should be returned by a handler when the HTTP method is not allowed.
|
|
|
|
type MethodNotAllowedError struct {
|
|
|
|
Method string
|
|
|
|
Allow []string
|
|
|
|
}
|
|
|
|
|
|
|
|
func (e MethodNotAllowedError) Error() string {
|
|
|
|
return fmt.Sprintf("method %s not allowed", e.Method)
|
|
|
|
}
|
|
|
|
|
2017-05-19 09:53:41 +00:00
|
|
|
// HTTPServer provides an HTTP api for an agent.
|
2013-12-23 19:38:51 +00:00
|
|
|
type HTTPServer struct {
|
2017-05-19 09:53:41 +00:00
|
|
|
*http.Server
|
2017-07-10 20:51:25 +00:00
|
|
|
agent *Agent
|
|
|
|
blacklist *Blacklist
|
|
|
|
|
|
|
|
// proto is filled by the agent to "http" or "https".
|
2017-05-24 13:22:56 +00:00
|
|
|
proto string
|
2017-09-25 18:40:42 +00:00
|
|
|
addr net.Addr
|
2013-12-23 19:38:51 +00:00
|
|
|
}
|
|
|
|
|
2017-09-25 18:40:42 +00:00
|
|
|
func NewHTTPServer(addr net.Addr, a *Agent) *HTTPServer {
|
2017-07-10 20:51:25 +00:00
|
|
|
s := &HTTPServer{
|
2017-09-25 18:40:42 +00:00
|
|
|
Server: &http.Server{Addr: addr.String()},
|
2017-07-10 20:51:25 +00:00
|
|
|
agent: a,
|
2017-09-25 18:40:42 +00:00
|
|
|
blacklist: NewBlacklist(a.config.HTTPBlockEndpoints),
|
|
|
|
addr: addr,
|
2017-07-10 20:51:25 +00:00
|
|
|
}
|
2017-09-25 18:40:42 +00:00
|
|
|
|
2017-07-10 20:51:25 +00:00
|
|
|
s.Server.Handler = s.handler(a.config.EnableDebug)
|
2017-05-19 09:53:41 +00:00
|
|
|
return s
|
2014-11-14 19:39:19 +00:00
|
|
|
}
|
|
|
|
|
2017-05-19 09:53:41 +00:00
|
|
|
// handler is used to attach our handlers to the mux
|
|
|
|
func (s *HTTPServer) handler(enableDebug bool) http.Handler {
|
|
|
|
mux := http.NewServeMux()
|
|
|
|
|
|
|
|
// handleFuncMetrics takes the given pattern and handler and wraps to produce
|
|
|
|
// metrics based on the pattern and request.
|
|
|
|
handleFuncMetrics := func(pattern string, handler http.HandlerFunc) {
|
|
|
|
// Get the parts of the pattern. We omit any initial empty for the
|
|
|
|
// leading slash, and put an underscore as a "thing" placeholder if we
|
|
|
|
// see a trailing slash, which means the part after is parsed. This lets
|
|
|
|
// us distinguish from things like /v1/query and /v1/query/<query id>.
|
|
|
|
var parts []string
|
|
|
|
for i, part := range strings.Split(pattern, "/") {
|
|
|
|
if part == "" {
|
|
|
|
if i == 0 {
|
|
|
|
continue
|
|
|
|
}
|
2016-08-09 22:41:15 +00:00
|
|
|
part = "_"
|
|
|
|
}
|
2017-05-19 09:53:41 +00:00
|
|
|
parts = append(parts, part)
|
2016-08-09 22:41:15 +00:00
|
|
|
}
|
|
|
|
|
2017-05-19 09:53:41 +00:00
|
|
|
// Register the wrapper, which will close over the expensive-to-compute
|
|
|
|
// parts from above.
|
2017-08-08 08:31:38 +00:00
|
|
|
// TODO (kyhavlov): Convert this to utilize metric labels in a major release
|
2017-05-19 09:53:41 +00:00
|
|
|
wrapper := func(resp http.ResponseWriter, req *http.Request) {
|
|
|
|
start := time.Now()
|
|
|
|
handler(resp, req)
|
|
|
|
key := append([]string{"consul", "http", req.Method}, parts...)
|
|
|
|
metrics.MeasureSince(key, start)
|
|
|
|
}
|
|
|
|
mux.HandleFunc(pattern, wrapper)
|
2016-08-09 22:41:15 +00:00
|
|
|
}
|
|
|
|
|
2017-05-19 09:53:41 +00:00
|
|
|
mux.HandleFunc("/", s.Index)
|
2013-12-25 01:09:51 +00:00
|
|
|
|
2016-10-26 02:20:24 +00:00
|
|
|
// API V1.
|
|
|
|
if s.agent.config.ACLDatacenter != "" {
|
2017-08-03 00:05:18 +00:00
|
|
|
handleFuncMetrics("/v1/acl/bootstrap", s.wrap(s.ACLBootstrap))
|
2017-05-19 09:53:41 +00:00
|
|
|
handleFuncMetrics("/v1/acl/create", s.wrap(s.ACLCreate))
|
|
|
|
handleFuncMetrics("/v1/acl/update", s.wrap(s.ACLUpdate))
|
|
|
|
handleFuncMetrics("/v1/acl/destroy/", s.wrap(s.ACLDestroy))
|
|
|
|
handleFuncMetrics("/v1/acl/info/", s.wrap(s.ACLGet))
|
|
|
|
handleFuncMetrics("/v1/acl/clone/", s.wrap(s.ACLClone))
|
|
|
|
handleFuncMetrics("/v1/acl/list", s.wrap(s.ACLList))
|
|
|
|
handleFuncMetrics("/v1/acl/replication", s.wrap(s.ACLReplicationStatus))
|
2017-07-26 18:03:43 +00:00
|
|
|
handleFuncMetrics("/v1/agent/token/", s.wrap(s.AgentToken))
|
2015-07-30 18:23:09 +00:00
|
|
|
} else {
|
2017-08-03 00:05:18 +00:00
|
|
|
handleFuncMetrics("/v1/acl/bootstrap", s.wrap(ACLDisabled))
|
2017-05-19 09:53:41 +00:00
|
|
|
handleFuncMetrics("/v1/acl/create", s.wrap(ACLDisabled))
|
|
|
|
handleFuncMetrics("/v1/acl/update", s.wrap(ACLDisabled))
|
|
|
|
handleFuncMetrics("/v1/acl/destroy/", s.wrap(ACLDisabled))
|
|
|
|
handleFuncMetrics("/v1/acl/info/", s.wrap(ACLDisabled))
|
|
|
|
handleFuncMetrics("/v1/acl/clone/", s.wrap(ACLDisabled))
|
|
|
|
handleFuncMetrics("/v1/acl/list", s.wrap(ACLDisabled))
|
|
|
|
handleFuncMetrics("/v1/acl/replication", s.wrap(ACLDisabled))
|
2017-07-26 18:03:43 +00:00
|
|
|
handleFuncMetrics("/v1/agent/token/", s.wrap(ACLDisabled))
|
2015-07-29 23:33:25 +00:00
|
|
|
}
|
2017-05-19 09:53:41 +00:00
|
|
|
handleFuncMetrics("/v1/agent/self", s.wrap(s.AgentSelf))
|
|
|
|
handleFuncMetrics("/v1/agent/maintenance", s.wrap(s.AgentNodeMaintenance))
|
|
|
|
handleFuncMetrics("/v1/agent/reload", s.wrap(s.AgentReload))
|
|
|
|
handleFuncMetrics("/v1/agent/monitor", s.wrap(s.AgentMonitor))
|
2017-08-08 20:05:38 +00:00
|
|
|
handleFuncMetrics("/v1/agent/metrics", s.wrap(s.AgentMetrics))
|
2017-05-19 09:53:41 +00:00
|
|
|
handleFuncMetrics("/v1/agent/services", s.wrap(s.AgentServices))
|
|
|
|
handleFuncMetrics("/v1/agent/checks", s.wrap(s.AgentChecks))
|
|
|
|
handleFuncMetrics("/v1/agent/members", s.wrap(s.AgentMembers))
|
|
|
|
handleFuncMetrics("/v1/agent/join/", s.wrap(s.AgentJoin))
|
|
|
|
handleFuncMetrics("/v1/agent/leave", s.wrap(s.AgentLeave))
|
|
|
|
handleFuncMetrics("/v1/agent/force-leave/", s.wrap(s.AgentForceLeave))
|
|
|
|
handleFuncMetrics("/v1/agent/check/register", s.wrap(s.AgentRegisterCheck))
|
|
|
|
handleFuncMetrics("/v1/agent/check/deregister/", s.wrap(s.AgentDeregisterCheck))
|
|
|
|
handleFuncMetrics("/v1/agent/check/pass/", s.wrap(s.AgentCheckPass))
|
|
|
|
handleFuncMetrics("/v1/agent/check/warn/", s.wrap(s.AgentCheckWarn))
|
|
|
|
handleFuncMetrics("/v1/agent/check/fail/", s.wrap(s.AgentCheckFail))
|
|
|
|
handleFuncMetrics("/v1/agent/check/update/", s.wrap(s.AgentCheckUpdate))
|
|
|
|
handleFuncMetrics("/v1/agent/service/register", s.wrap(s.AgentRegisterService))
|
|
|
|
handleFuncMetrics("/v1/agent/service/deregister/", s.wrap(s.AgentDeregisterService))
|
|
|
|
handleFuncMetrics("/v1/agent/service/maintenance/", s.wrap(s.AgentServiceMaintenance))
|
|
|
|
handleFuncMetrics("/v1/catalog/register", s.wrap(s.CatalogRegister))
|
|
|
|
handleFuncMetrics("/v1/catalog/deregister", s.wrap(s.CatalogDeregister))
|
|
|
|
handleFuncMetrics("/v1/catalog/datacenters", s.wrap(s.CatalogDatacenters))
|
|
|
|
handleFuncMetrics("/v1/catalog/nodes", s.wrap(s.CatalogNodes))
|
|
|
|
handleFuncMetrics("/v1/catalog/services", s.wrap(s.CatalogServices))
|
|
|
|
handleFuncMetrics("/v1/catalog/service/", s.wrap(s.CatalogServiceNodes))
|
|
|
|
handleFuncMetrics("/v1/catalog/node/", s.wrap(s.CatalogNodeServices))
|
2016-10-26 02:20:24 +00:00
|
|
|
if !s.agent.config.DisableCoordinates {
|
2017-05-19 09:53:41 +00:00
|
|
|
handleFuncMetrics("/v1/coordinate/datacenters", s.wrap(s.CoordinateDatacenters))
|
|
|
|
handleFuncMetrics("/v1/coordinate/nodes", s.wrap(s.CoordinateNodes))
|
2016-10-26 02:20:24 +00:00
|
|
|
} else {
|
2017-05-19 09:53:41 +00:00
|
|
|
handleFuncMetrics("/v1/coordinate/datacenters", s.wrap(coordinateDisabled))
|
|
|
|
handleFuncMetrics("/v1/coordinate/nodes", s.wrap(coordinateDisabled))
|
2016-10-26 02:20:24 +00:00
|
|
|
}
|
2017-05-19 09:53:41 +00:00
|
|
|
handleFuncMetrics("/v1/event/fire/", s.wrap(s.EventFire))
|
|
|
|
handleFuncMetrics("/v1/event/list", s.wrap(s.EventList))
|
|
|
|
handleFuncMetrics("/v1/health/node/", s.wrap(s.HealthNodeChecks))
|
|
|
|
handleFuncMetrics("/v1/health/checks/", s.wrap(s.HealthServiceChecks))
|
|
|
|
handleFuncMetrics("/v1/health/state/", s.wrap(s.HealthChecksInState))
|
|
|
|
handleFuncMetrics("/v1/health/service/", s.wrap(s.HealthServiceNodes))
|
|
|
|
handleFuncMetrics("/v1/internal/ui/nodes", s.wrap(s.UINodes))
|
|
|
|
handleFuncMetrics("/v1/internal/ui/node/", s.wrap(s.UINodeInfo))
|
|
|
|
handleFuncMetrics("/v1/internal/ui/services", s.wrap(s.UIServices))
|
|
|
|
handleFuncMetrics("/v1/kv/", s.wrap(s.KVSEndpoint))
|
|
|
|
handleFuncMetrics("/v1/operator/raft/configuration", s.wrap(s.OperatorRaftConfiguration))
|
|
|
|
handleFuncMetrics("/v1/operator/raft/peer", s.wrap(s.OperatorRaftPeer))
|
|
|
|
handleFuncMetrics("/v1/operator/keyring", s.wrap(s.OperatorKeyringEndpoint))
|
|
|
|
handleFuncMetrics("/v1/operator/autopilot/configuration", s.wrap(s.OperatorAutopilotConfiguration))
|
|
|
|
handleFuncMetrics("/v1/operator/autopilot/health", s.wrap(s.OperatorServerHealth))
|
|
|
|
handleFuncMetrics("/v1/query", s.wrap(s.PreparedQueryGeneral))
|
|
|
|
handleFuncMetrics("/v1/query/", s.wrap(s.PreparedQuerySpecific))
|
|
|
|
handleFuncMetrics("/v1/session/create", s.wrap(s.SessionCreate))
|
|
|
|
handleFuncMetrics("/v1/session/destroy/", s.wrap(s.SessionDestroy))
|
|
|
|
handleFuncMetrics("/v1/session/renew/", s.wrap(s.SessionRenew))
|
|
|
|
handleFuncMetrics("/v1/session/info/", s.wrap(s.SessionGet))
|
|
|
|
handleFuncMetrics("/v1/session/node/", s.wrap(s.SessionsForNode))
|
|
|
|
handleFuncMetrics("/v1/session/list", s.wrap(s.SessionList))
|
|
|
|
handleFuncMetrics("/v1/status/leader", s.wrap(s.StatusLeader))
|
|
|
|
handleFuncMetrics("/v1/status/peers", s.wrap(s.StatusPeers))
|
|
|
|
handleFuncMetrics("/v1/snapshot", s.wrap(s.Snapshot))
|
|
|
|
handleFuncMetrics("/v1/txn", s.wrap(s.Txn))
|
2016-05-11 04:41:47 +00:00
|
|
|
|
2016-10-26 02:20:24 +00:00
|
|
|
// Debug endpoints.
|
2014-03-20 00:50:57 +00:00
|
|
|
if enableDebug {
|
2017-05-19 09:53:41 +00:00
|
|
|
handleFuncMetrics("/debug/pprof/", pprof.Index)
|
|
|
|
handleFuncMetrics("/debug/pprof/cmdline", pprof.Cmdline)
|
|
|
|
handleFuncMetrics("/debug/pprof/profile", pprof.Profile)
|
|
|
|
handleFuncMetrics("/debug/pprof/symbol", pprof.Symbol)
|
2014-03-20 00:50:57 +00:00
|
|
|
}
|
2014-04-23 19:57:06 +00:00
|
|
|
|
2015-11-30 19:24:08 +00:00
|
|
|
// Use the custom UI dir if provided.
|
2017-05-11 17:31:58 +00:00
|
|
|
if s.agent.config.UIDir != "" {
|
2017-05-19 09:53:41 +00:00
|
|
|
mux.Handle("/ui/", http.StripPrefix("/ui/", http.FileServer(http.Dir(s.agent.config.UIDir))))
|
2017-04-21 00:02:42 +00:00
|
|
|
} else if s.agent.config.EnableUI {
|
2017-05-19 09:53:41 +00:00
|
|
|
mux.Handle("/ui/", http.StripPrefix("/ui/", http.FileServer(assetFS())))
|
2015-02-11 21:25:04 +00:00
|
|
|
}
|
2017-05-19 09:53:41 +00:00
|
|
|
return mux
|
2013-12-23 19:38:51 +00:00
|
|
|
}
|
|
|
|
|
2017-07-15 07:07:08 +00:00
|
|
|
// aclEndpointRE is used to find old ACL endpoints that take tokens in the URL
|
|
|
|
// so that we can redact them. The ACL endpoints that take the token in the URL
|
|
|
|
// are all of the form /v1/acl/<verb>/<token>, and can optionally include query
|
|
|
|
// parameters which are indicated by a question mark. We capture the part before
|
|
|
|
// the token, the token, and any query parameters after, and then reassemble as
|
|
|
|
// $1<hidden>$3 (the token in $2 isn't used), which will give:
|
|
|
|
//
|
|
|
|
// /v1/acl/clone/foo -> /v1/acl/clone/<hidden>
|
|
|
|
// /v1/acl/clone/foo?token=bar -> /v1/acl/clone/<hidden>?token=<hidden>
|
|
|
|
//
|
|
|
|
// The query parameter in the example above is obfuscated like any other, after
|
|
|
|
// this regular expression is applied, so the regular expression substitution
|
|
|
|
// results in:
|
|
|
|
//
|
|
|
|
// /v1/acl/clone/foo?token=bar -> /v1/acl/clone/<hidden>?token=bar
|
|
|
|
// ^---- $1 ----^^- $2 -^^-- $3 --^
|
|
|
|
//
|
|
|
|
// And then the loop that looks for parameters called "token" does the last
|
|
|
|
// step to get to the final redacted form.
|
|
|
|
var (
|
|
|
|
aclEndpointRE = regexp.MustCompile("^(/v1/acl/[^/]+/)([^?]+)([?]?.*)$")
|
|
|
|
)
|
|
|
|
|
2013-12-23 19:38:51 +00:00
|
|
|
// wrap is used to wrap functions to make them more convenient
|
2017-05-11 17:37:47 +00:00
|
|
|
func (s *HTTPServer) wrap(handler func(resp http.ResponseWriter, req *http.Request) (interface{}, error)) http.HandlerFunc {
|
|
|
|
return func(resp http.ResponseWriter, req *http.Request) {
|
2017-09-25 18:40:42 +00:00
|
|
|
setHeaders(resp, s.agent.config.HTTPResponseHeaders)
|
|
|
|
setTranslateAddr(resp, s.agent.config.TranslateWANAddrs)
|
2014-12-28 04:53:19 +00:00
|
|
|
|
2015-04-12 18:17:31 +00:00
|
|
|
// Obfuscate any tokens from appearing in the logs
|
2015-04-14 00:31:53 +00:00
|
|
|
formVals, err := url.ParseQuery(req.URL.RawQuery)
|
|
|
|
if err != nil {
|
2017-05-19 09:53:41 +00:00
|
|
|
s.agent.logger.Printf("[ERR] http: Failed to decode query: %s from=%s", err, req.RemoteAddr)
|
2017-08-23 19:19:11 +00:00
|
|
|
resp.WriteHeader(http.StatusInternalServerError)
|
2015-04-14 00:31:53 +00:00
|
|
|
return
|
|
|
|
}
|
2015-04-12 18:17:31 +00:00
|
|
|
logURL := req.URL.String()
|
2015-04-14 00:31:53 +00:00
|
|
|
if tokens, ok := formVals["token"]; ok {
|
2015-04-12 18:17:31 +00:00
|
|
|
for _, token := range tokens {
|
2015-06-12 07:09:51 +00:00
|
|
|
if token == "" {
|
|
|
|
logURL += "<hidden>"
|
|
|
|
continue
|
|
|
|
}
|
2015-04-12 18:17:31 +00:00
|
|
|
logURL = strings.Replace(logURL, token, "<hidden>", -1)
|
|
|
|
}
|
|
|
|
}
|
2017-07-15 07:07:08 +00:00
|
|
|
logURL = aclEndpointRE.ReplaceAllString(logURL, "$1<hidden>$3")
|
2015-04-12 18:17:31 +00:00
|
|
|
|
2017-07-10 20:51:25 +00:00
|
|
|
if s.blacklist.Block(req.URL.Path) {
|
|
|
|
errMsg := "Endpoint is blocked by agent configuration"
|
|
|
|
s.agent.logger.Printf("[ERR] http: Request %s %v, error: %v from=%s", req.Method, logURL, err, req.RemoteAddr)
|
|
|
|
resp.WriteHeader(http.StatusForbidden)
|
|
|
|
fmt.Fprint(resp, errMsg)
|
|
|
|
return
|
|
|
|
}
|
|
|
|
|
2017-09-26 06:11:19 +00:00
|
|
|
isMethodNotAllowed := func(err error) bool {
|
|
|
|
_, ok := err.(MethodNotAllowedError)
|
|
|
|
return ok
|
|
|
|
}
|
|
|
|
|
2017-05-11 17:37:47 +00:00
|
|
|
handleErr := func(err error) {
|
2017-05-19 09:53:41 +00:00
|
|
|
s.agent.logger.Printf("[ERR] http: Request %s %v, error: %v from=%s", req.Method, logURL, err, req.RemoteAddr)
|
2017-08-23 14:52:48 +00:00
|
|
|
switch {
|
|
|
|
case acl.IsErrPermissionDenied(err) || acl.IsErrNotFound(err):
|
2017-08-23 19:19:11 +00:00
|
|
|
resp.WriteHeader(http.StatusForbidden)
|
2017-08-23 14:52:48 +00:00
|
|
|
fmt.Fprint(resp, err.Error())
|
2017-09-01 22:02:50 +00:00
|
|
|
case structs.IsErrRPCRateExceeded(err):
|
|
|
|
resp.WriteHeader(http.StatusTooManyRequests)
|
2017-09-26 06:11:19 +00:00
|
|
|
case isMethodNotAllowed(err):
|
|
|
|
// RFC2616 states that for 405 Method Not Allowed the response
|
|
|
|
// MUST include an Allow header containing the list of valid
|
|
|
|
// methods for the requested resource.
|
|
|
|
// https://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html
|
|
|
|
resp.Header()["Allow"] = err.(MethodNotAllowedError).Allow
|
|
|
|
resp.WriteHeader(http.StatusMethodNotAllowed) // 405
|
2017-09-01 22:02:50 +00:00
|
|
|
fmt.Fprint(resp, err.Error())
|
2017-08-23 14:52:48 +00:00
|
|
|
default:
|
2017-08-23 19:19:11 +00:00
|
|
|
resp.WriteHeader(http.StatusInternalServerError)
|
2017-08-23 14:52:48 +00:00
|
|
|
fmt.Fprint(resp, err.Error())
|
2017-05-11 17:37:47 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2013-12-23 19:38:51 +00:00
|
|
|
// Invoke the handler
|
2013-12-23 22:26:34 +00:00
|
|
|
start := time.Now()
|
|
|
|
defer func() {
|
2017-10-17 18:38:24 +00:00
|
|
|
s.agent.logger.Printf("[DEBUG] http: Request %s %v (%v) from=%s", req.Method, logURL, time.Since(start), req.RemoteAddr)
|
2013-12-23 22:26:34 +00:00
|
|
|
}()
|
2013-12-24 00:20:51 +00:00
|
|
|
obj, err := handler(resp, req)
|
2013-12-23 19:38:51 +00:00
|
|
|
if err != nil {
|
2017-05-11 17:37:47 +00:00
|
|
|
handleErr(err)
|
|
|
|
return
|
|
|
|
}
|
|
|
|
if obj == nil {
|
2013-12-23 19:38:51 +00:00
|
|
|
return
|
|
|
|
}
|
|
|
|
|
2017-05-11 17:37:47 +00:00
|
|
|
buf, err := s.marshalJSON(req, obj)
|
|
|
|
if err != nil {
|
|
|
|
handleErr(err)
|
|
|
|
return
|
2013-12-23 19:38:51 +00:00
|
|
|
}
|
2017-05-11 17:37:47 +00:00
|
|