open-nomad/plugins/drivers/proto/driver.proto
Tim Gross aa8927abb4
volumes: return better error messages for unsupported task drivers (#8030)
When an allocation runs for a task driver that can't support volume mounts,
the mounting will fail in a way that can be hard to understand. With host
volumes this usually means failing silently, whereas with CSI the operator
gets inscrutable internals exposed in the `nomad alloc status`.

This changeset adds a MountConfig field to the task driver Capabilities
response. We validate this when the `csi_hook` or `volume_hook` fires and
return a user-friendly error.

Note that we don't currently have a way to get driver capabilities up to the
server, except through attributes. Validating this when the user initially
submits the jobspec would be even better than what we're doing here (and could
be useful for all our other capabilities), but that's out of scope for this
changeset.

Also note that the MountConfig enum starts with "supports all" in order to
support community plugins in a backwards compatible way, rather than cutting
them off from volume mounting unexpectedly.
2020-05-21 09:18:02 -04:00

706 lines
20 KiB
Protocol Buffer

syntax = "proto3";
package hashicorp.nomad.plugins.drivers.proto;
option go_package = "proto";
import "google/protobuf/duration.proto";
import "google/protobuf/timestamp.proto";
import "github.com/hashicorp/nomad/plugins/shared/hclspec/hcl_spec.proto";
import "github.com/hashicorp/nomad/plugins/shared/structs/proto/attribute.proto";
// Driver service defines RPCs used to communicate with a nomad runtime driver.
// Some rpcs may not be implemented by the driver based on it's capabilities.
service Driver {
// TaskConfigSchema returns the schema for parsing the driver
// configuration of a task.
rpc TaskConfigSchema(TaskConfigSchemaRequest) returns (TaskConfigSchemaResponse) {}
// Capabilities returns a set of features which the driver implements. Some
// RPCs are not possible to implement on some runtimes, this allows the
// driver to indicate if it doesn't support these RPCs and features.
rpc Capabilities(CapabilitiesRequest) returns (CapabilitiesResponse) {}
// Fingerprint starts a stream which emits information about the driver
// including whether the driver healthy and able to function in the
// existing environment.
//
// The driver should immediately stream a FingerprintResponse when the RPC
// is initially called, then send any additional responses if there is a
// change in the driver's state.
rpc Fingerprint(FingerprintRequest) returns (stream FingerprintResponse) {}
// RecoverTask is used when a task has been started but the driver may not
// know about it. Such is the case if the driver restarts or is upgraded.
rpc RecoverTask(RecoverTaskRequest) returns (RecoverTaskResponse) {}
// StartTask starts and tracks the task on the implemented runtime
rpc StartTask(StartTaskRequest) returns (StartTaskResponse) {}
// WaitTask blocks until the given task exits, returning the result of the
// task. It may be called after the task has exited, but before the task is
// destroyed.
rpc WaitTask(WaitTaskRequest) returns (WaitTaskResponse) {}
// StopTask stops a given task by sending the desired signal to the process.
// If the task does not exit on its own within the given timeout, it will be
// forcefully killed.
rpc StopTask(StopTaskRequest) returns (StopTaskResponse) {}
// DestroyTask removes the task from the driver's internal state and cleans
// up any additional resources created by the driver. It cannot be called
// on a running task, unless force is set to true.
rpc DestroyTask(DestroyTaskRequest) returns (DestroyTaskResponse) {}
// InspectTask returns detailed information for the given task
rpc InspectTask(InspectTaskRequest) returns (InspectTaskResponse) {}
// TaskStats collects and returns runtime metrics for the given task
rpc TaskStats(TaskStatsRequest) returns (stream TaskStatsResponse) {}
// TaskEvents starts a streaming RPC where all task events emitted by the
// driver are streamed to the caller.
rpc TaskEvents(TaskEventsRequest) returns (stream DriverTaskEvent) {}
// The following RPCs are only implemented if the driver sets the
// corresponding capability.
// SignalTask sends a signal to the task
rpc SignalTask(SignalTaskRequest) returns (SignalTaskResponse) {}
// ExecTask executes a command inside the tasks execution context
rpc ExecTask(ExecTaskRequest) returns (ExecTaskResponse) {}
// ExecTaskStreaming executes a command inside the tasks execution context
// and streams back results
rpc ExecTaskStreaming(stream ExecTaskStreamingRequest) returns (stream ExecTaskStreamingResponse) {}
// CreateNetwork is implemented when the driver needs to create the network
// namespace instead of allowing the Nomad client to do.
rpc CreateNetwork(CreateNetworkRequest) returns (CreateNetworkResponse) {}
// DestroyNetwork destroys a previously created network. This rpc is only
// implemented if the driver needs to manage network namespace creation.
rpc DestroyNetwork(DestroyNetworkRequest) returns (DestroyNetworkResponse) {}
}
message TaskConfigSchemaRequest {}
message TaskConfigSchemaResponse {
// Spec is the configuration schema for the job driver config stanza
hashicorp.nomad.plugins.shared.hclspec.Spec spec = 1;
}
message CapabilitiesRequest {}
message CapabilitiesResponse {
// Capabilities provides a way for the driver to denote if it implements
// non-core RPCs. Some Driver service RPCs expose additional information
// or functionality outside of the core task management functions. These
// RPCs are only implemented if the driver sets the corresponding capability.
DriverCapabilities capabilities = 1;
}
message FingerprintRequest {}
message FingerprintResponse {
// Attributes are key/value pairs that annotate the nomad client and can be
// used in scheduling constraints and affinities.
map<string, hashicorp.nomad.plugins.shared.structs.Attribute> attributes = 1;
enum HealthState {
UNDETECTED = 0;
UNHEALTHY = 1;
HEALTHY = 2;
}
// Health is used to determine the state of the health the driver is in.
// Health can be one of the following states:
// * UNDETECTED: driver dependencies are not met and the driver can not start
// * UNHEALTHY: driver dependencies are met but the driver is unable to
// perform operations due to some other problem
// * HEALTHY: driver is able to perform all operations
HealthState health = 2;
// HealthDescription is a human readable message describing the current
// state of driver health
string health_description = 3;
}
message RecoverTaskRequest {
// TaskId is the ID of the target task
string task_id = 1;
// Handle is the TaskHandle returned from StartTask
TaskHandle handle = 2;
}
message RecoverTaskResponse {}
message StartTaskRequest {
// Task configuration to launch
TaskConfig task = 1;
}
message StartTaskResponse {
enum Result {
SUCCESS = 0;
RETRY = 1;
FATAL = 2;
}
// Result is set depending on the type of error that occurred while starting
// a task:
//
// * SUCCESS: No error occurred, handle is set
// * RETRY: An error occurred, but is recoverable and the RPC should be retried
// * FATAL: A fatal error occurred and is not likely to succeed if retried
//
// If Result is not successful, the DriverErrorMsg will be set.
Result result = 1;
// DriverErrorMsg is set if an error occurred
string driver_error_msg = 2;
// Handle is opaque to the client, but must be stored in order to recover
// the task.
TaskHandle handle = 3;
// NetworkOverride is set if the driver sets network settings and the service ip/port
// needs to be set differently.
NetworkOverride network_override = 4;
}
message WaitTaskRequest {
// TaskId is the ID of the target task
string task_id = 1;
}
message WaitTaskResponse {
// Result is the exit status of the task
ExitResult result = 1;
// Err is set if any driver error occurred while waiting for the task
string err = 2;
}
message StopTaskRequest {
// TaskId is the ID of the target task
string task_id = 1;
// Timeout defines the amount of time to wait before forcefully killing
// the task. For example, on Unix clients, this means sending a SIGKILL to
// the process.
google.protobuf.Duration timeout = 2;
// Signal can be set to override the Task's configured shutdown signal
string signal = 3;
}
message StopTaskResponse {}
message DestroyTaskRequest {
// TaskId is the ID of the target task
string task_id = 1;
// Force destroys the task even if it is still in a running state
bool force = 2;
}
message DestroyTaskResponse {}
message InspectTaskRequest {
// TaskId is the ID of the target task
string task_id = 1;
}
message InspectTaskResponse {
// Task details
TaskStatus task = 1;
// Driver details for task
TaskDriverStatus driver = 2;
// NetworkOverride info if set
NetworkOverride network_override = 3;
}
message TaskStatsRequest {
// TaskId is the ID of the target task
string task_id = 1;
// CollectionInterval is the interval at which to stream stats to the caller
google.protobuf.Duration collection_interval = 2;
}
message TaskStatsResponse {
// Stats for the task
TaskStats stats = 1;
}
message TaskEventsRequest {}
message SignalTaskRequest {
// TaskId is the ID of the target task
string task_id = 1;
// Signal is the operating system signal to send to the task. Ex: SIGHUP
string signal = 2;
}
message SignalTaskResponse {}
message ExecTaskRequest {
// TaskId is the ID of the target task
string task_id = 1;
// Command is the command to execute in the task environment
repeated string command = 2;
// Timeout is the amount of time to wait for the command to stop.
// Defaults to 0 (run forever)
google.protobuf.Duration timeout = 3;
}
message ExecTaskResponse {
// Stdout from the exec
bytes stdout = 1;
// Stderr from the exec
bytes stderr = 2;
// Result from the exec
ExitResult result = 3;
}
message ExecTaskStreamingIOOperation {
bytes data = 1;
bool close = 2;
}
message ExecTaskStreamingRequest {
message Setup {
string task_id = 1;
repeated string command = 2;
bool tty = 3;
}
message TerminalSize {
int32 height = 1;
int32 width = 2;
}
Setup setup = 1;
TerminalSize tty_size = 2;
ExecTaskStreamingIOOperation stdin = 3;
}
message ExecTaskStreamingResponse {
ExecTaskStreamingIOOperation stdout = 1;
ExecTaskStreamingIOOperation stderr = 2;
bool exited = 3;
ExitResult result = 4;
}
message CreateNetworkRequest {
// AllocID of the allocation the network is associated with
string alloc_id = 1;
}
message CreateNetworkResponse {
NetworkIsolationSpec isolation_spec = 1;
// created indicates that the network namespace is newly created
// as a result of this request. if false, the NetworkIsolationSpec
// value returned is an existing spec.
bool created = 2;
}
message DestroyNetworkRequest {
// AllocID of the allocation the network is associated with
string alloc_id = 1;
NetworkIsolationSpec isolation_spec = 2;
}
message DestroyNetworkResponse {}
message DriverCapabilities {
// SendSignals indicates that the driver can send process signals (ex. SIGUSR1)
// to the task.
bool send_signals = 1;
// Exec indicates that the driver supports executing arbitrary commands
// in the task's execution environment.
bool exec = 2;
enum FSIsolation {
NONE = 0;
CHROOT = 1;
IMAGE = 2;
}
// FsIsolation indicates what kind of filesystem isolation a driver supports.
FSIsolation fs_isolation = 3;
repeated NetworkIsolationSpec.NetworkIsolationMode network_isolation_modes = 4;
bool must_create_network = 5;
enum MountConfigs {
option allow_alias = true;
UNKNOWN_MOUNTS = 0; // treated as ANY_MOUNTS for backwards compatibility
ANY_MOUNTS = 0;
NO_MOUNTS = 1;
}
// MountConfigs indicates whether the driver supports mount configurations.
MountConfigs mount_configs = 6;
}
message NetworkIsolationSpec {
enum NetworkIsolationMode {
HOST = 0;
GROUP = 1;
TASK = 2;
NONE = 3;
}
NetworkIsolationMode mode = 1;
string path = 2;
map<string,string> labels = 3;
}
message TaskConfig {
// Id of the task, recommended to the globally unique, must be unique to the driver.
string id = 1;
// Name of the task
string name = 2;
// MsgpackDriverConfig is the encoded driver configuation of the task
bytes msgpack_driver_config = 3;
// Env is the a set of key/value pairs to be set as environment variables
map<string, string> env = 4;
// DeviceEnv is the set of environment variables that are defined by device
// plugins. This allows the driver to differentiate environment variables
// set by the device plugins and those by the user. When populating the
// task's environment env should be used.
map<string, string> device_env = 5;
// Resources defines the resources to isolate
Resources resources = 6;
// Mounts is a list of targets to bind mount into the task directory
repeated Mount mounts = 7;
// Devices is a list of system devices to mount into the task's execution
// environment.
repeated Device devices = 8;
// User defines the operating system user the tasks should run as
string user = 9;
// AllocDir is the directory on the host where the allocation directory
// exists.
string alloc_dir = 10;
// StdoutPath is the path to the file to open and write task stdout to
string stdout_path = 11;
// StderrPath is the path to the file to open and write task stderr to
string stderr_path = 12;
// TaskGroupName is the name of the task group which this task is a member of
string task_group_name = 13;
// JobName is the name of the job of which this task is part of
string job_name = 14;
// AllocId is the ID of the associated allocation
string alloc_id = 15;
// NetworkIsolationSpec specifies the configuration for the network namespace
// to use for the task. *Only supported on Linux
NetworkIsolationSpec network_isolation_spec = 16;
}
message Resources {
// AllocatedResources are the resources set for the task
AllocatedTaskResources allocated_resources = 1;
// LinuxResources are the computed values to set for specific Linux features
LinuxResources linux_resources = 2;
}
message AllocatedTaskResources {
AllocatedCpuResources cpu = 1;
AllocatedMemoryResources memory = 2;
repeated NetworkResource networks = 5;
}
message AllocatedCpuResources {
int64 cpu_shares = 1;
}
message AllocatedMemoryResources {
int64 memory_mb = 2;
}
message NetworkResource {
string device = 1;
string cidr = 2;
string ip = 3;
int32 mbits = 4;
repeated NetworkPort reserved_ports = 5;
repeated NetworkPort dynamic_ports = 6;
}
message NetworkPort {
string label = 1;
int32 value = 2;
}
message LinuxResources {
// CPU CFS (Completely Fair Scheduler) period. Default: 0 (not specified)
int64 cpu_period = 1;
// CPU CFS (Completely Fair Scheduler) quota. Default: 0 (not specified)
int64 cpu_quota = 2;
// CPU shares (relative weight vs. other containers). Default: 0 (not specified)
int64 cpu_shares = 3;
// Memory limit in bytes. Default: 0 (not specified)
int64 memory_limit_bytes = 4;
// OOMScoreAdj adjusts the oom-killer score. Default: 0 (not specified)
int64 oom_score_adj = 5;
// CpusetCpus constrains the allowed set of logical CPUs. Default: "" (not specified)
string cpuset_cpus = 6;
// CpusetMems constrains the allowed set of memory nodes. Default: "" (not specified)
string cpuset_mems = 7;
// PercentTicks is a compatibility option for docker and should not be used
double PercentTicks = 8;
}
message Mount {
// TaskPath is the file path within the task directory to mount to
string task_path = 1;
// HostPath is the file path on the host to mount from
string host_path = 2;
// Readonly if set true, mounts the path in readonly mode
bool readonly = 3;
}
message Device {
// TaskPath is the file path within the task to mount the device to
string task_path = 1;
// HostPath is the path on the host to the source device
string host_path = 2;
// CgroupPermissions defines the Cgroup permissions of the device.
// One or more of the following options can be set:
// * r - allows the task to read from the specified device.
// * w - allows the task to write to the specified device.
// * m - allows the task to create device files that do not yet exist.
//
// Example: "rw"
string cgroup_permissions = 3;
}
enum TaskState {
UNKNOWN = 0;
RUNNING = 1;
EXITED = 2;
}
// TaskHandle is created when starting a task and is used to recover task
message TaskHandle {
// Version is used by the driver to version the DriverState schema.
// Version 0 is reserved by Nomad and should not be used.
int32 version = 1;
// Config is the TaskConfig for the task
TaskConfig config = 2;
// State is the state of the task's execution
TaskState state = 3;
// DriverState is the encoded state for the specific driver
bytes driver_state = 4;
}
// NetworkOverride contains network settings which the driver may override
// for the task, such as when the driver is setting up the task's network.
message NetworkOverride {
// PortMap can be set to replace ports with driver-specific mappings
map<string,int32> port_map = 1;
// Addr is the IP address for the task created by the driver
string addr = 2;
// AutoAdvertise indicates whether the driver thinks services that choose
// to auto_advertise_addresses should use this IP instead of the host's.
bool auto_advertise = 3;
}
// ExitResult contains information about the exit status of a task
message ExitResult {
// ExitCode returned from the task on exit
int32 exit_code = 1;
// Signal is set if a signal was sent to the task
int32 signal = 2;
// OomKilled is true if the task exited as a result of the OOM Killer
bool oom_killed = 3;
}
// TaskStatus includes information of a specific task
message TaskStatus {
string id = 1;
string name = 2;
// State is the state of the task's execution
TaskState state = 3;
// StartedAt is the timestamp when the task was started
google.protobuf.Timestamp started_at = 4;
// CompletedAt is the timestamp when the task exited.
// If the task is still running, CompletedAt will not be set
google.protobuf.Timestamp completed_at = 5;
// Result is set when CompletedAt is set.
ExitResult result = 6;
}
message TaskDriverStatus {
// Attributes is a set of string/string key value pairs specific to the
// implementing driver
map<string, string> attributes = 1;
}
message TaskStats {
// Id of the task
string id = 1;
// Timestamp for which the stats were collected
google.protobuf.Timestamp timestamp = 2;
// AggResourceUsage is the aggreate usage of all processes
TaskResourceUsage agg_resource_usage = 3;
// ResourceUsageByPid breaks the usage stats by process
map<string, TaskResourceUsage> resource_usage_by_pid = 4;
}
message TaskResourceUsage {
// CPU usage stats
CPUUsage cpu = 1;
// Memory usage stats
MemoryUsage memory = 2;
}
message CPUUsage {
double system_mode = 1;
double user_mode = 2;
double total_ticks = 3;
uint64 throttled_periods = 4;
uint64 throttled_time = 5;
double percent = 6;
enum Fields {
SYSTEM_MODE = 0;
USER_MODE = 1;
TOTAL_TICKS = 2;
THROTTLED_PERIODS = 3;
THROTTLED_TIME = 4;
PERCENT = 5;
}
// MeasuredFields indicates which fields were actually sampled
repeated Fields measured_fields = 7;
}
message MemoryUsage {
uint64 rss = 1;
uint64 cache = 2;
uint64 max_usage = 3;
uint64 kernel_usage = 4;
uint64 kernel_max_usage = 5;
uint64 usage = 7;
uint64 swap = 8;
enum Fields {
RSS = 0;
CACHE = 1;
MAX_USAGE = 2;
KERNEL_USAGE = 3;
KERNEL_MAX_USAGE = 4;
USAGE = 5;
SWAP = 6;
}
// MeasuredFields indicates which fields were actually sampled
repeated Fields measured_fields = 6;
}
message DriverTaskEvent {
// TaskId is the id of the task for the event
string task_id = 1;
// AllocId of the task for the event
string alloc_id = 2;
// TaskName is the name of the task for the event
string task_name = 3;
// Timestamp when the event occurred
google.protobuf.Timestamp timestamp = 4;
// Message is the body of the event
string message = 5;
// Annotations allows for additional key/value data to be sent along with the event
map<string,string> annotations = 6;
}