package nomad import ( "bytes" "context" "errors" "fmt" "io" "net" "time" log "github.com/hashicorp/go-hclog" sframer "github.com/hashicorp/nomad/client/lib/streamframer" cstructs "github.com/hashicorp/nomad/client/structs" "github.com/hashicorp/nomad/command/agent/monitor" "github.com/hashicorp/nomad/command/agent/pprof" "github.com/hashicorp/nomad/helper" "github.com/hashicorp/nomad/nomad/structs" "github.com/ugorji/go/codec" ) type Agent struct { srv *Server } func (a *Agent) register() { a.srv.streamingRpcs.Register("Agent.Monitor", a.monitor) } func (a *Agent) Profile(args *structs.AgentPprofRequest, reply *structs.AgentPprofResponse) error { // Check ACL for agent write aclObj, err := a.srv.ResolveToken(args.AuthToken) if err != nil { return err } else if aclObj != nil && !aclObj.AllowAgentWrite() { return structs.ErrPermissionDenied } // Forward to different region if necessary // this would typically be done in a.srv.forward() but since // we are targeting a specific server, not just the leader // we must manually handle region forwarding here. region := args.RequestRegion() if region == "" { return fmt.Errorf("missing target RPC") } if region != a.srv.config.Region { // Mark that we are forwarding args.SetForwarded() return a.srv.forwardRegion(region, "Agent.Profile", args, reply) } // Targeting a node, forward request to node if args.NodeID != "" { return a.forwardProfileClient(args, reply) } // Handle serverID not equal to ours if args.ServerID != "" { serverToFwd, err := a.forwardFor(args.ServerID, region) if err != nil { return err } if serverToFwd != nil { return a.srv.forwardServer(serverToFwd, "Agent.Profile", args, reply) } } // If ACLs are disabled, EnableDebug must be enabled if aclObj == nil && !a.srv.config.EnableDebug { return structs.ErrPermissionDenied } // Process the request on this server var resp []byte var headers map[string]string // Determine which profile to run and generate profile. // Blocks for args.Seconds // Our RPC endpoints currently don't support context // or request cancellation so using server shutdownCtx as a // best effort. switch args.ReqType { case pprof.CPUReq: resp, headers, err = pprof.CPUProfile(a.srv.shutdownCtx, args.Seconds) case pprof.CmdReq: resp, headers, err = pprof.Cmdline() case pprof.LookupReq: resp, headers, err = pprof.Profile(args.Profile, args.Debug, args.GC) case pprof.TraceReq: resp, headers, err = pprof.Trace(a.srv.shutdownCtx, args.Seconds) default: err = structs.NewErrRPCCoded(404, "Unknown profile request type") } if err != nil { if pprof.IsErrProfileNotFound(err) { return structs.NewErrRPCCoded(404, err.Error()) } return structs.NewErrRPCCoded(500, err.Error()) } // Copy profile response to reply reply.Payload = resp reply.HTTPHeaders = headers reply.AgentID = a.srv.serf.LocalMember().Name return nil } func (a *Agent) monitor(conn io.ReadWriteCloser) { defer conn.Close() // Decode args var args cstructs.MonitorRequest decoder := codec.NewDecoder(conn, structs.MsgpackHandle) encoder := codec.NewEncoder(conn, structs.MsgpackHandle) if err := decoder.Decode(&args); err != nil { handleStreamResultError(err, helper.Int64ToPtr(500), encoder) return } // Check agent read permissions if aclObj, err := a.srv.ResolveToken(args.AuthToken); err != nil { handleStreamResultError(err, nil, encoder) return } else if aclObj != nil && !aclObj.AllowAgentRead() { handleStreamResultError(structs.ErrPermissionDenied, helper.Int64ToPtr(403), encoder) return } logLevel := log.LevelFromString(args.LogLevel) if args.LogLevel == "" { logLevel = log.LevelFromString("INFO") } if logLevel == log.NoLevel { handleStreamResultError(errors.New("Unknown log level"), helper.Int64ToPtr(400), encoder) return } // Targeting a node, forward request to node if args.NodeID != "" { a.forwardMonitorClient(conn, args, encoder, decoder) // forwarded request has ended, return return } region := args.RequestRegion() if region == "" { handleStreamResultError(fmt.Errorf("missing target RPC"), helper.Int64ToPtr(400), encoder) return } if region != a.srv.config.Region { // Mark that we are forwarding args.SetForwarded() } // Try to forward request to remote region/server if args.ServerID != "" { serverToFwd, err := a.forwardFor(args.ServerID, region) if err != nil { handleStreamResultError(err, helper.Int64ToPtr(400), encoder) return } if serverToFwd != nil { a.forwardMonitorServer(conn, serverToFwd, args, encoder, decoder) return } } // NodeID was empty, ServerID was equal to this server, monitor this server ctx, cancel := context.WithCancel(context.Background()) defer cancel() monitor := monitor.New(512, a.srv.logger, &log.LoggerOptions{ Level: logLevel, JSONFormat: args.LogJSON, }) frames := make(chan *sframer.StreamFrame, 32) errCh := make(chan error) var buf bytes.Buffer frameCodec := codec.NewEncoder(&buf, structs.JsonHandle) framer := sframer.NewStreamFramer(frames, 1*time.Second, 200*time.Millisecond, 1024) framer.Run() defer framer.Destroy() // goroutine to detect remote side closing go func() { if _, err := conn.Read(nil); err != nil { // One end of the pipe explicitly closed, exit cancel() return } select { case <-ctx.Done(): return } }() logCh := monitor.Start() defer monitor.Stop() initialOffset := int64(0) // receive logs and build frames go func() { defer framer.Destroy() LOOP: for { select { case log := <-logCh: if err := framer.Send("", "log", log, initialOffset); err != nil { select { case errCh <- err: case <-ctx.Done(): } break LOOP } case <-ctx.Done(): break LOOP } } }() var streamErr error OUTER: for { select { case frame, ok := <-frames: if !ok { // frame may have been closed when an error // occurred. Check once more for an error. select { case streamErr = <-errCh: // There was a pending error! default: // No error, continue on } break OUTER } var resp cstructs.StreamErrWrapper if args.PlainText { resp.Payload = frame.Data } else { if err := frameCodec.Encode(frame); err != nil { streamErr = err break OUTER } resp.Payload = buf.Bytes() buf.Reset() } if err := encoder.Encode(resp); err != nil { streamErr = err break OUTER } encoder.Reset(conn) case <-ctx.Done(): break OUTER } } if streamErr != nil { handleStreamResultError(streamErr, helper.Int64ToPtr(500), encoder) return } } // forwardFor returns a serverParts for a request to be forwarded to. // A response of nil, nil indicates that the current server is equal to the // serverID and region so the request should not be forwarded. func (a *Agent) forwardFor(serverID, region string) (*serverParts, error) { var target *serverParts var err error if serverID == "leader" { isLeader, remoteLeader := a.srv.getLeader() if !isLeader && remoteLeader != nil { target = remoteLeader } else if !isLeader && remoteLeader == nil { return nil, structs.ErrNoLeader } else if isLeader { // This server is current leader do not forward return nil, nil } } else { target, err = a.srv.getServer(region, serverID) if err != nil { return nil, err } } // Unable to find a server if target == nil { return nil, fmt.Errorf("unknown nomad server %s", serverID) } // ServerID is this current server, // No need to forward request if target.Name == a.srv.LocalMember().Name { return nil, nil } return target, nil } func (a *Agent) forwardMonitorClient(conn io.ReadWriteCloser, args cstructs.MonitorRequest, encoder *codec.Encoder, decoder *codec.Decoder) { nodeID := args.NodeID snap, err := a.srv.State().Snapshot() if err != nil { handleStreamResultError(err, nil, encoder) return } node, err := snap.NodeByID(nil, nodeID) if err != nil { handleStreamResultError(err, helper.Int64ToPtr(500), encoder) return } if node == nil { err := fmt.Errorf("Unknown node %q", nodeID) handleStreamResultError(err, helper.Int64ToPtr(400), encoder) return } if err := nodeSupportsRpc(node); err != nil { handleStreamResultError(err, helper.Int64ToPtr(400), encoder) return } // Get the Connection to the client either by fowarding to another server // or creating direct stream var clientConn net.Conn state, ok := a.srv.getNodeConn(nodeID) if !ok { // Determine the server that has a connection to the node srv, err := a.srv.serverWithNodeConn(nodeID, a.srv.Region()) if err != nil { var code *int64 if structs.IsErrNoNodeConn(err) { code = helper.Int64ToPtr(404) } handleStreamResultError(err, code, encoder) return } conn, err := a.srv.streamingRpc(srv, "Agent.Monitor") if err != nil { handleStreamResultError(err, nil, encoder) return } clientConn = conn } else { stream, err := NodeStreamingRpc(state.Session, "Agent.Monitor") if err != nil { handleStreamResultError(err, nil, encoder) return } clientConn = stream } defer clientConn.Close() // Send the Request outEncoder := codec.NewEncoder(clientConn, structs.MsgpackHandle) if err := outEncoder.Encode(args); err != nil { handleStreamResultError(err, nil, encoder) return } structs.Bridge(conn, clientConn) return } func (a *Agent) forwardMonitorServer(conn io.ReadWriteCloser, server *serverParts, args cstructs.MonitorRequest, encoder *codec.Encoder, decoder *codec.Decoder) { // empty ServerID to prevent forwarding loop args.ServerID = "" serverConn, err := a.srv.streamingRpc(server, "Agent.Monitor") if err != nil { handleStreamResultError(err, helper.Int64ToPtr(500), encoder) return } defer serverConn.Close() // Send the Request outEncoder := codec.NewEncoder(serverConn, structs.MsgpackHandle) if err := outEncoder.Encode(args); err != nil { handleStreamResultError(err, helper.Int64ToPtr(500), encoder) return } structs.Bridge(conn, serverConn) return } func (a *Agent) forwardProfileClient(args *structs.AgentPprofRequest, reply *structs.AgentPprofResponse) error { nodeID := args.NodeID snap, err := a.srv.State().Snapshot() if err != nil { return structs.NewErrRPCCoded(500, err.Error()) } node, err := snap.NodeByID(nil, nodeID) if err != nil { return structs.NewErrRPCCoded(500, err.Error()) } if node == nil { err := fmt.Errorf("Unknown node %q", nodeID) return structs.NewErrRPCCoded(404, err.Error()) } if err := nodeSupportsRpc(node); err != nil { return structs.NewErrRPCCoded(400, err.Error()) } // Get the Connection to the client either by fowarding to another server // or creating direct stream state, ok := a.srv.getNodeConn(nodeID) if !ok { // Determine the server that has a connection to the node srv, err := a.srv.serverWithNodeConn(nodeID, a.srv.Region()) if err != nil { code := 500 if structs.IsErrNoNodeConn(err) { code = 404 } return structs.NewErrRPCCoded(code, err.Error()) } rpcErr := a.srv.forwardServer(srv, "Agent.Profile", args, reply) if rpcErr != nil { return rpcErr } } else { // NodeRpc rpcErr := NodeRpc(state.Session, "Agent.Profile", args, reply) if rpcErr != nil { return rpcErr } } return nil }