task_execution.proto

syntax = "proto3";

package tes;

// Import HTTP RESTful annotations.
import "google/api/annotations.proto";

// Task describes an instance of a task.
message Task {

  // OUTPUT ONLY
  //
  // Task identifier assigned by the server.
  string id = 1;

  // OUTPUT ONLY
  State state = 2;

  // OPTIONAL
  string name = 3;

  // OPTIONAL
  //
  // Describes the project this task is associated with.
  // Commonly used for billing on cloud providers (AWS, Google Cloud, etc).
  string project = 4;

  // OPTIONAL
  string description = 5;

  // OPTIONAL
  //
  // Input files.
  // Inputs will be downloaded and mounted into the executor container.
  repeated TaskParameter inputs = 6;

  // OPTIONAL
  //
  // Output files.
  // Outputs will be uploaded from the executor container to long-term storage.
  repeated TaskParameter outputs = 7;

  // OPTIONAL
  //
  // Request that the task be run with these resources.
  Resources resources = 8;

  // REQUIRED
  //
  // A list of executors to be run, sequentially.
  repeated Executor executors = 9;

  // OPTIONAL
  //
  // Declared volumes.
  // Volumes are shared between executors. Volumes for inputs and outputs are 
  // inferred and should not be delcared here.
  repeated string volumes = 10;

  // OPTIONAL
  //
  // A key-value map of arbitrary tags.
  map<string, string> tags = 11;

  // OUTPUT ONLY
  //
  // Task logging information.
  // Normally, this will contain only one entry, but in the case where
  // a task fails and is retried, an entry will be appended to this list.
  repeated TaskLog logs = 12;
}

enum FileType {
  FILE = 0;
  DIRECTORY = 1;
}

// TaskParameter describes input and output files for a Task.
message TaskParameter {

  // OPTIONAL
  string name = 1;

  // OPTIONAL
  string description = 2;

  // REQUIRED
  //
  // URL in long term storage, for example:
  // s3://my-object-store/file1
  // gs://my-bucket/file2
  // file:///path/to/my/file
  // /path/to/my/file
  // etc...
  string url = 3;

  // REQUIRED
  //
  // Path of the file inside the container.
  string path = 4;

  // REQUIRED
  //
  // Type of the file, FILE or DIRECTORY
  FileType type = 5;

  // OPTIONAL
  //
  // File contents literal. 
  // Implementations should support a minimum of 128 KiB in this field and may define its own maximum.
  // UTF-8 encoded
  string contents = 6;
}

// Ports describes the port mapping between the container and host.
message Ports {

  // REQUIRED 
  //
  // Port number opened inside the container.
  uint32 container = 1;

  // OPTIONAL 
  //
  // Port number opened on the host. Must be greater than 1024.
  // Defaults to 0, which assigns a random port on the host.
  uint32 host = 2;
}

// Executor describes a command to run, and its environment.
message Executor {

  // REQUIRED
  //
  // Name of the container image, for example:
  // ubuntu
  // quay.io/aptible/ubuntu
  // gcr.io/my-org/my-image
  // etc...
  string image_name = 1;

  // REQUIRED
  //
  // The command to be executed.
  repeated string cmd = 2;

  // OPTIONAL
  //
  // The working directory that the command will be executed in.
  // Defaults to the directory set by the container image.
  string workdir = 3;

  // OPTIONAL
  //
  // Path inside the container to a file which will be piped
  // to the command's stdin.
  string stdin = 6;

  // OPTIONAL
  //
  // Path inside the container to a file where the command's
  // stdout will be written to.
  string stdout = 4;

  // OPTIONAL
  //
  // Path inside the container to a file where the command's
  // stderr will be written to.
  string stderr = 5;

  // OPTIONAL
  //
  // Port to expose from within the container, blank if none.
  repeated Ports ports = 7;

  // OPTIONAL
  //
  // Enviromental variables to set within the container.
  map<string,string> environ = 8;
}

// Resources describes the resources requested by a task.
message Resources {

  // OPTIONAL
  //
  // Requested number of CPUs
  uint32 cpu_cores = 1;

  // OPTIONAL
  //
  // Is the task allowed to run on preemptible compute instances (e.g. AWS Spot)?
  bool preemptible = 2;

  // OPTIONAL
  //
  // Requested RAM required in gigabytes (GB)
  double ram_gb = 3;

  // OPTIONAL
  //
  // Requested disk size in gigabytes (GB)
  double size_gb = 4;

  // OPTIONAL
  //
  // Request that the task be run in these compute zones.
  repeated string zones = 5;
}

// OUTPUT ONLY
//
// TaskLog describes logging information related to a Task.
message TaskLog {

  // REQUIRED
  //
  // Logs for each executor
  repeated ExecutorLog logs = 1;

  // OPTIONAL
  //
  // Arbitrary logging metadata included by the implementation.
  map<string, string> metadata = 2;

  // OPTIONAL
  //
  // When the task started, in RFC 3339 format.
  string start_time = 3;

  // OPTIONAL
  //
  // When the task ended, in RFC 3339 format.
  string end_time = 4;

  // REQUIRED
  //
  // Information about all output files. Directory outputs are
  // flattened into separate items.
  repeated OutputFileLog outputs = 5;
}

// OUTPUT ONLY
//
// ExecutorLog describes logging information related to an Executor.
message ExecutorLog {

  // OPTIONAL
  //
  // Time the executor started, in RFC 3339 format.
  string start_time = 2;

  // OPTIONAL
  //
  // Time the executor ended, in RFC 3339 format.
  string end_time = 3;

  // OPTIONAL
  //
  // Stdout tail.
  // This is not guaranteed to be the entire log.
  // Implementations determine the maximum size.
  string stdout = 4;

  // OPTIONAL
  //
  // Stderr tail.
  // This is not guaranteed to be the entire log.
  // Implementations determine the maximum size.
  string stderr = 5;

  // REQUIRED
  //
  // Exit code.
  int32 exit_code = 6;

  // OPTIONAL
  //
  // IP address of host.
  string host_ip = 7;

  // OPTIONAL
  //
  // Ports mapped between the container and host.
  repeated Ports ports = 8;
}

// OUTPUT ONLY
//
// OutputFileLog describes a single output file. This describes
// file details after the task has completed successfully,
// for logging purposes.
message OutputFileLog {

  // REQUIRED
  //
  // URL of the file in storage, e.g. s3://bucket/file.txt
  string url = 1;

  // REQUIRED
  //
  // Path of the file inside the container.
  string path = 2;

  // REQUIRED
  //
  // Size of the file in bytes.
  int64 size_bytes = 3;
}

// OUTPUT ONLY
//
// Task states.
enum State {
  UNKNOWN = 0;
  QUEUED = 1;
  INITIALIZING = 2;
  RUNNING = 3;
  // An implementation *may* have the ability to pause a task,
  // but this is not required.
  PAUSED = 4;
  COMPLETE = 5;
  ERROR = 6;
  SYSTEM_ERROR = 7;
  CANCELED = 8;
}

// OUTPUT ONLY
//
// CreateTaskResponse describes a response from the CreateTask endpoint.
message CreateTaskResponse {
  // REQUIRED
  //
  // Task identifier assigned by the server.
  string id = 1;
}

// GetTaskRequest describes a request to the GetTask endpoint.
message GetTaskRequest {
  // REQUIRED
  //
  // Task identifier.
  string id = 1;

  // OPTIONAL
  //
  // Affects the fields included in the returned Task messages.
  // See TaskView below.
  TaskView view = 2;
}

// ListTasksRequest describes a request to the ListTasks service endpoint.
message ListTasksRequest {

  // OPTIONAL
  //
  // Filter the task list to include tasks in this project.
  string project = 1;

  // OPTIONAL
  //
  // Filter the list to include tasks where the name matches this prefix.
  // If unspecified, no task name filtering is done.
  string name_prefix	= 2;

  // OPTIONAL
  //
  // Number of tasks to return in one page.
  // Must be less than 2048. Defaults to 256.
  uint32 page_size = 3;

  // OPTIONAL
  //
  // Page token is used to retrieve the next page of results.
  // If unspecified, returns the first page of results.
  // See ListTasksResponse.next_page_token
  string page_token = 4;

  // OPTIONAL
  //
  // Affects the fields included in the returned Task messages.
  // See TaskView below.
  TaskView view = 5;
}

// TaskView affects the fields returned by the ListTasks endpoint.
//
// Some of the fields in task can be large strings (e.g. logs),
// which can be a burden on the network. In the default BASIC view,
// these heavyweight fields are not included, however, a client may
// request the FULL version to include these fields.
enum TaskView {

  // Task message will include ONLY the fields:
  //   Task.Id
  //   Task.State
  MINIMAL = 0;

  // Task message will include all fields EXCEPT:
  //   Task.ExecutorLog.stdout
  //   Task.ExecutorLog.stderr
  //   TaskParameter.Contents in Task.Inputs
  BASIC = 1;

  // Task message includes all fields.
  FULL = 2;
}

// OUTPUT ONLY
//
// ListTasksResponse describes a response from the ListTasks endpoint.
message ListTasksResponse {

  // REQUIRED
  //
  // List of lightweight task descriptions.
  repeated Task tasks = 1;

  // OPTIONAL
  //
  // Token used to return the next page of results.
  // See TaskListRequest.next_page_token
  string next_page_token = 2;
}

// CancelTaskRequest describes a request to the CancelTask endpoint.
message CancelTaskRequest {
  // REQUIRED
  //
  // Task identifier.
  string id = 1;
}

// OUTPUT ONLY
//
// CancelTaskResponse describes a response from the CancelTask endpoint.
message CancelTaskResponse {}

// ServiceInfoRequest describes a request to the ServiceInfo endpoint.
message ServiceInfoRequest {}

// OUTPUT ONLY
//
// ServiceInfo describes information about the service,
// such as storage details, resource availability,
// and other documentation.
message ServiceInfo {

  // Returns the name of the service, e.g. "ohsu-compbio-funnel".
  string name = 1;

  // Returns a documentation string, e.g. "Hey, we're OHSU Comp. Bio!".
  string doc = 2;

  // Lists some, but not necessarily all, storage locations supported by the service.
  //
  // Must be in a valid URL format.
  // e.g. 
  // file:///path/to/local/funnel-storage
  // s3://ohsu-compbio-funnel/storage
  // etc.
  repeated string storage = 3;
}

// TaskService describes the HTTP/gRPC service API provided by TES
// services to create, list, get, update tasks.
service TaskService {

  // GetServiceInfo provides information about the service,
  // such as storage details, resource availability, and 
  // other documentation.
  rpc GetServiceInfo(ServiceInfoRequest) returns (ServiceInfo) {
    option (google.api.http) = {
      get: "/v1/tasks/service-info"
    };
  }

  // Create a new task.
  rpc CreateTask(Task) returns (CreateTaskResponse) {
    option (google.api.http) = {
      post: "/v1/tasks"
      body: "*"
    };
  }

  // List tasks.
  // TaskView is requested as such: "v1/tasks?view=BASIC"
  rpc ListTasks(ListTasksRequest) returns (ListTasksResponse) {
    option (google.api.http) = {
      get: "/v1/tasks"
    };
  }

  // Get a task.
  // TaskView is requested as such: "v1/tasks/{id}?view=FULL"
  rpc GetTask(GetTaskRequest) returns (Task) {
      option (google.api.http) = {
        get: "/v1/tasks/{id}"
      };
  }

  // Cancel a task.
  rpc CancelTask(CancelTaskRequest) returns (CancelTaskResponse) {
    option (google.api.http) = {
      post: "/v1/tasks/{id}:cancel"
    };
  }

}