2018-02-12 23:22:41 +00:00
|
|
|
//
|
|
|
|
// Copyright (c) 2018, Joyent, Inc. All rights reserved.
|
|
|
|
//
|
|
|
|
// This Source Code Form is subject to the terms of the Mozilla Public
|
|
|
|
// License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
|
|
// file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
|
|
|
//
|
|
|
|
|
|
|
|
package storage
|
|
|
|
|
|
|
|
import (
|
|
|
|
"context"
|
2019-04-12 15:51:37 +00:00
|
|
|
"encoding/json"
|
2018-02-12 23:22:41 +00:00
|
|
|
"io"
|
|
|
|
"net/http"
|
|
|
|
"net/url"
|
|
|
|
"path"
|
|
|
|
"strconv"
|
|
|
|
"strings"
|
|
|
|
"time"
|
|
|
|
|
|
|
|
"github.com/joyent/triton-go/client"
|
|
|
|
tt "github.com/joyent/triton-go/errors"
|
|
|
|
"github.com/pkg/errors"
|
|
|
|
)
|
|
|
|
|
|
|
|
type ObjectsClient struct {
|
|
|
|
client *client.Client
|
|
|
|
}
|
|
|
|
|
2019-04-12 15:51:37 +00:00
|
|
|
// AbortMpuInput represents parameters to an AbortMpu operation
|
|
|
|
type AbortMpuInput struct {
|
|
|
|
PartsDirectoryPath string
|
|
|
|
}
|
|
|
|
|
|
|
|
func (s *ObjectsClient) AbortMultipartUpload(ctx context.Context, input *AbortMpuInput) error {
|
|
|
|
return abortMpu(*s, ctx, input)
|
|
|
|
}
|
|
|
|
|
|
|
|
// CommitMpuInput represents parameters to a CommitMpu operation
|
|
|
|
type CommitMpuInput struct {
|
|
|
|
Id string
|
|
|
|
Headers map[string]string
|
|
|
|
Body CommitMpuBody
|
|
|
|
}
|
|
|
|
|
|
|
|
// CommitMpuBody represents the body of a CommitMpu request
|
|
|
|
type CommitMpuBody struct {
|
|
|
|
Parts []string `json:"parts"`
|
|
|
|
}
|
|
|
|
|
|
|
|
func (s *ObjectsClient) CommitMultipartUpload(ctx context.Context, input *CommitMpuInput) error {
|
|
|
|
return commitMpu(*s, ctx, input)
|
|
|
|
}
|
|
|
|
|
|
|
|
// CreateMpuInput represents parameters to a CreateMpu operation.
|
|
|
|
type CreateMpuInput struct {
|
|
|
|
Body CreateMpuBody
|
|
|
|
ContentLength uint64
|
|
|
|
ContentMD5 string
|
|
|
|
DurabilityLevel uint64
|
|
|
|
ForceInsert bool //Force the creation of the directory tree
|
|
|
|
}
|
|
|
|
|
|
|
|
// CreateMpuOutput represents the response from a CreateMpu operation
|
|
|
|
type CreateMpuOutput struct {
|
|
|
|
Id string `json:"id"`
|
|
|
|
PartsDirectory string `json:"partsDirectory"`
|
|
|
|
}
|
|
|
|
|
|
|
|
// CreateMpuBody represents the body of a CreateMpu request.
|
|
|
|
type CreateMpuBody struct {
|
|
|
|
ObjectPath string `json:"objectPath"`
|
|
|
|
Headers map[string]string `json:"headers,omitempty"`
|
|
|
|
}
|
|
|
|
|
|
|
|
func (s *ObjectsClient) CreateMultipartUpload(ctx context.Context, input *CreateMpuInput) (*CreateMpuOutput, error) {
|
|
|
|
return createMpu(*s, ctx, input)
|
|
|
|
}
|
|
|
|
|
2018-02-12 23:22:41 +00:00
|
|
|
// GetObjectInput represents parameters to a GetObject operation.
|
|
|
|
type GetInfoInput struct {
|
|
|
|
ObjectPath string
|
|
|
|
Headers map[string]string
|
|
|
|
}
|
|
|
|
|
|
|
|
// GetObjectOutput contains the outputs for a GetObject operation. It is your
|
|
|
|
// responsibility to ensure that the io.ReadCloser ObjectReader is closed.
|
|
|
|
type GetInfoOutput struct {
|
|
|
|
ContentLength uint64
|
|
|
|
ContentType string
|
|
|
|
LastModified time.Time
|
|
|
|
ContentMD5 string
|
|
|
|
ETag string
|
|
|
|
Metadata map[string]string
|
|
|
|
}
|
|
|
|
|
|
|
|
// GetInfo sends a HEAD request to an object in the Manta service. This function
|
|
|
|
// does not return a response body.
|
|
|
|
func (s *ObjectsClient) GetInfo(ctx context.Context, input *GetInfoInput) (*GetInfoOutput, error) {
|
|
|
|
absPath := absFileInput(s.client.AccountName, input.ObjectPath)
|
|
|
|
|
|
|
|
headers := &http.Header{}
|
|
|
|
for key, value := range input.Headers {
|
|
|
|
headers.Set(key, value)
|
|
|
|
}
|
|
|
|
|
|
|
|
reqInput := client.RequestInput{
|
|
|
|
Method: http.MethodHead,
|
|
|
|
Path: string(absPath),
|
|
|
|
Headers: headers,
|
|
|
|
}
|
|
|
|
_, respHeaders, err := s.client.ExecuteRequestStorage(ctx, reqInput)
|
|
|
|
if err != nil {
|
|
|
|
return nil, errors.Wrap(err, "unable to get info")
|
|
|
|
}
|
|
|
|
|
|
|
|
response := &GetInfoOutput{
|
|
|
|
ContentType: respHeaders.Get("Content-Type"),
|
|
|
|
ContentMD5: respHeaders.Get("Content-MD5"),
|
|
|
|
ETag: respHeaders.Get("Etag"),
|
|
|
|
}
|
|
|
|
|
|
|
|
lastModified, err := time.Parse(time.RFC1123, respHeaders.Get("Last-Modified"))
|
|
|
|
if err == nil {
|
|
|
|
response.LastModified = lastModified
|
|
|
|
}
|
|
|
|
|
|
|
|
contentLength, err := strconv.ParseUint(respHeaders.Get("Content-Length"), 10, 64)
|
|
|
|
if err == nil {
|
|
|
|
response.ContentLength = contentLength
|
|
|
|
}
|
|
|
|
|
|
|
|
metadata := map[string]string{}
|
|
|
|
for key, values := range respHeaders {
|
|
|
|
if strings.HasPrefix(key, "m-") {
|
|
|
|
metadata[key] = strings.Join(values, ", ")
|
|
|
|
}
|
|
|
|
}
|
|
|
|
response.Metadata = metadata
|
|
|
|
|
|
|
|
return response, nil
|
|
|
|
}
|
|
|
|
|
|
|
|
// IsDir is a convenience wrapper around the GetInfo function which takes an
|
|
|
|
// ObjectPath and returns a boolean whether or not the object is a directory
|
|
|
|
// type in Manta. Returns an error if GetInfo failed upstream for some reason.
|
|
|
|
func (s *ObjectsClient) IsDir(ctx context.Context, objectPath string) (bool, error) {
|
|
|
|
info, err := s.GetInfo(ctx, &GetInfoInput{
|
|
|
|
ObjectPath: objectPath,
|
|
|
|
})
|
|
|
|
if err != nil {
|
|
|
|
return false, err
|
|
|
|
}
|
|
|
|
if info != nil {
|
|
|
|
return strings.HasSuffix(info.ContentType, "type=directory"), nil
|
|
|
|
}
|
|
|
|
return false, nil
|
|
|
|
}
|
|
|
|
|
|
|
|
// GetObjectInput represents parameters to a GetObject operation.
|
|
|
|
type GetObjectInput struct {
|
|
|
|
ObjectPath string
|
|
|
|
Headers map[string]string
|
|
|
|
}
|
|
|
|
|
|
|
|
// GetObjectOutput contains the outputs for a GetObject operation. It is your
|
|
|
|
// responsibility to ensure that the io.ReadCloser ObjectReader is closed.
|
|
|
|
type GetObjectOutput struct {
|
|
|
|
ContentLength uint64
|
|
|
|
ContentType string
|
|
|
|
LastModified time.Time
|
|
|
|
ContentMD5 string
|
|
|
|
ETag string
|
|
|
|
Metadata map[string]string
|
|
|
|
ObjectReader io.ReadCloser
|
|
|
|
}
|
|
|
|
|
|
|
|
// Get retrieves an object from the Manta service. If error is nil (i.e. the
|
|
|
|
// call returns successfully), it is your responsibility to close the
|
|
|
|
// io.ReadCloser named ObjectReader in the operation output.
|
|
|
|
func (s *ObjectsClient) Get(ctx context.Context, input *GetObjectInput) (*GetObjectOutput, error) {
|
|
|
|
absPath := absFileInput(s.client.AccountName, input.ObjectPath)
|
|
|
|
|
|
|
|
headers := &http.Header{}
|
|
|
|
for key, value := range input.Headers {
|
|
|
|
headers.Set(key, value)
|
|
|
|
}
|
|
|
|
|
|
|
|
reqInput := client.RequestInput{
|
|
|
|
Method: http.MethodGet,
|
|
|
|
Path: string(absPath),
|
|
|
|
Headers: headers,
|
|
|
|
}
|
|
|
|
respBody, respHeaders, err := s.client.ExecuteRequestStorage(ctx, reqInput)
|
|
|
|
if err != nil {
|
|
|
|
return nil, errors.Wrap(err, "unable to get object")
|
|
|
|
}
|
|
|
|
|
|
|
|
response := &GetObjectOutput{
|
|
|
|
ContentType: respHeaders.Get("Content-Type"),
|
|
|
|
ContentMD5: respHeaders.Get("Content-MD5"),
|
|
|
|
ETag: respHeaders.Get("Etag"),
|
|
|
|
ObjectReader: respBody,
|
|
|
|
}
|
|
|
|
|
|
|
|
lastModified, err := time.Parse(time.RFC1123, respHeaders.Get("Last-Modified"))
|
|
|
|
if err == nil {
|
|
|
|
response.LastModified = lastModified
|
|
|
|
}
|
|
|
|
|
|
|
|
contentLength, err := strconv.ParseUint(respHeaders.Get("Content-Length"), 10, 64)
|
|
|
|
if err == nil {
|
|
|
|
response.ContentLength = contentLength
|
|
|
|
}
|
|
|
|
|
|
|
|
metadata := map[string]string{}
|
|
|
|
for key, values := range respHeaders {
|
|
|
|
if strings.HasPrefix(key, "m-") {
|
|
|
|
metadata[key] = strings.Join(values, ", ")
|
|
|
|
}
|
|
|
|
}
|
|
|
|
response.Metadata = metadata
|
|
|
|
|
|
|
|
return response, nil
|
|
|
|
}
|
|
|
|
|
|
|
|
// DeleteObjectInput represents parameters to a DeleteObject operation.
|
|
|
|
type DeleteObjectInput struct {
|
|
|
|
ObjectPath string
|
|
|
|
Headers map[string]string
|
|
|
|
}
|
|
|
|
|
|
|
|
// DeleteObject deletes an object.
|
|
|
|
func (s *ObjectsClient) Delete(ctx context.Context, input *DeleteObjectInput) error {
|
|
|
|
absPath := absFileInput(s.client.AccountName, input.ObjectPath)
|
|
|
|
|
|
|
|
headers := &http.Header{}
|
|
|
|
for key, value := range input.Headers {
|
|
|
|
headers.Set(key, value)
|
|
|
|
}
|
|
|
|
|
|
|
|
reqInput := client.RequestInput{
|
|
|
|
Method: http.MethodDelete,
|
|
|
|
Path: string(absPath),
|
|
|
|
Headers: headers,
|
|
|
|
}
|
|
|
|
respBody, _, err := s.client.ExecuteRequestStorage(ctx, reqInput)
|
|
|
|
if respBody != nil {
|
|
|
|
defer respBody.Close()
|
|
|
|
}
|
|
|
|
if err != nil {
|
|
|
|
return errors.Wrap(err, "unable to delete object")
|
|
|
|
}
|
|
|
|
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2019-04-12 15:51:37 +00:00
|
|
|
// GetMpuInput represents parameters to a GetMpu operation
|
|
|
|
type GetMpuInput struct {
|
|
|
|
PartsDirectoryPath string
|
|
|
|
}
|
|
|
|
|
|
|
|
type GetMpuHeaders struct {
|
|
|
|
ContentLength int64 `json:"content-length"`
|
|
|
|
ContentMd5 string `json:"content-md5"`
|
|
|
|
}
|
|
|
|
|
|
|
|
type GetMpuOutput struct {
|
|
|
|
Id string `json:"id"`
|
|
|
|
State string `json:"state"`
|
|
|
|
PartsDirectory string `json:"partsDirectory"`
|
|
|
|
TargetObject string `json:"targetObject"`
|
|
|
|
Headers GetMpuHeaders `json:"headers"`
|
|
|
|
NumCopies int64 `json:"numCopies"`
|
|
|
|
CreationTimeMs int64 `json:"creationTimeMs"`
|
|
|
|
}
|
|
|
|
|
|
|
|
func (s *ObjectsClient) GetMultipartUpload(ctx context.Context, input *GetMpuInput) (*GetMpuOutput, error) {
|
|
|
|
return getMpu(*s, ctx, input)
|
|
|
|
}
|
|
|
|
|
|
|
|
type ListMpuPartsInput struct {
|
|
|
|
Id string
|
|
|
|
}
|
|
|
|
|
|
|
|
type ListMpuPart struct {
|
|
|
|
ETag string
|
|
|
|
PartNumber int
|
|
|
|
Size int64
|
|
|
|
}
|
|
|
|
|
|
|
|
type ListMpuPartsOutput struct {
|
|
|
|
Parts []ListMpuPart
|
|
|
|
}
|
|
|
|
|
|
|
|
func (s *ObjectsClient) ListMultipartUploadParts(ctx context.Context, input *ListMpuPartsInput) (*ListMpuPartsOutput, error) {
|
|
|
|
return listMpuParts(*s, ctx, input)
|
|
|
|
}
|
|
|
|
|
2018-02-12 23:22:41 +00:00
|
|
|
// PutObjectMetadataInput represents parameters to a PutObjectMetadata operation.
|
|
|
|
type PutObjectMetadataInput struct {
|
|
|
|
ObjectPath string
|
|
|
|
ContentType string
|
|
|
|
Metadata map[string]string
|
|
|
|
}
|
|
|
|
|
|
|
|
// PutObjectMetadata allows you to overwrite the HTTP headers for an already
|
|
|
|
// existing object, without changing the data. Note this is an idempotent "replace"
|
|
|
|
// operation, so you must specify the complete set of HTTP headers you want
|
|
|
|
// stored on each request.
|
|
|
|
//
|
|
|
|
// You cannot change "critical" headers:
|
|
|
|
// - Content-Length
|
|
|
|
// - Content-MD5
|
|
|
|
// - Durability-Level
|
|
|
|
func (s *ObjectsClient) PutMetadata(ctx context.Context, input *PutObjectMetadataInput) error {
|
|
|
|
absPath := absFileInput(s.client.AccountName, input.ObjectPath)
|
|
|
|
query := &url.Values{}
|
|
|
|
query.Set("metadata", "true")
|
|
|
|
|
|
|
|
headers := &http.Header{}
|
|
|
|
headers.Set("Content-Type", input.ContentType)
|
|
|
|
for key, value := range input.Metadata {
|
|
|
|
headers.Set(key, value)
|
|
|
|
}
|
|
|
|
|
|
|
|
reqInput := client.RequestInput{
|
|
|
|
Method: http.MethodPut,
|
|
|
|
Path: string(absPath),
|
|
|
|
Query: query,
|
|
|
|
Headers: headers,
|
|
|
|
}
|
|
|
|
respBody, _, err := s.client.ExecuteRequestStorage(ctx, reqInput)
|
|
|
|
if respBody != nil {
|
|
|
|
defer respBody.Close()
|
|
|
|
}
|
|
|
|
if err != nil {
|
|
|
|
return errors.Wrap(err, "unable to put metadata")
|
|
|
|
}
|
|
|
|
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
// PutObjectInput represents parameters to a PutObject operation.
|
|
|
|
type PutObjectInput struct {
|
|
|
|
ObjectPath string
|
|
|
|
DurabilityLevel uint64
|
|
|
|
ContentType string
|
|
|
|
ContentMD5 string
|
|
|
|
IfMatch string
|
|
|
|
IfModifiedSince *time.Time
|
|
|
|
ContentLength uint64
|
|
|
|
MaxContentLength uint64
|
|
|
|
ObjectReader io.Reader
|
|
|
|
Headers map[string]string
|
|
|
|
ForceInsert bool //Force the creation of the directory tree
|
|
|
|
}
|
|
|
|
|
|
|
|
func (s *ObjectsClient) Put(ctx context.Context, input *PutObjectInput) error {
|
|
|
|
absPath := absFileInput(s.client.AccountName, input.ObjectPath)
|
|
|
|
if input.ForceInsert {
|
|
|
|
absDirName := _AbsCleanPath(path.Dir(string(absPath)))
|
|
|
|
exists, err := checkDirectoryTreeExists(*s, ctx, absDirName)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
if !exists {
|
|
|
|
err := createDirectory(*s, ctx, absDirName)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
return putObject(*s, ctx, input, absPath)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
return putObject(*s, ctx, input, absPath)
|
|
|
|
}
|
|
|
|
|
2019-04-12 15:51:37 +00:00
|
|
|
// UploadPartInput represents parameters to a UploadPart operation.
|
|
|
|
type UploadPartInput struct {
|
|
|
|
Id string
|
|
|
|
PartNum uint64
|
|
|
|
ContentMD5 string
|
|
|
|
Headers map[string]string
|
|
|
|
ObjectReader io.Reader
|
|
|
|
}
|
|
|
|
|
|
|
|
// UploadPartOutput represents the response from a
|
|
|
|
type UploadPartOutput struct {
|
|
|
|
Part string `json:"part"`
|
|
|
|
}
|
|
|
|
|
|
|
|
func (s *ObjectsClient) UploadPart(ctx context.Context, input *UploadPartInput) (*UploadPartOutput, error) {
|
|
|
|
return uploadPart(*s, ctx, input)
|
|
|
|
}
|
|
|
|
|
2018-02-12 23:22:41 +00:00
|
|
|
// _AbsCleanPath is an internal type that means the input has been
|
|
|
|
// path.Clean()'ed and is an absolute path.
|
|
|
|
type _AbsCleanPath string
|
|
|
|
|
|
|
|
func absFileInput(accountName, objPath string) _AbsCleanPath {
|
|
|
|
cleanInput := path.Clean(objPath)
|
|
|
|
if strings.HasPrefix(cleanInput, path.Join("/", accountName, "/")) {
|
|
|
|
return _AbsCleanPath(cleanInput)
|
|
|
|
}
|
|
|
|
|
|
|
|
cleanAbs := path.Clean(path.Join("/", accountName, objPath))
|
|
|
|
return _AbsCleanPath(cleanAbs)
|
|
|
|
}
|
|
|
|
|
|
|
|
func putObject(c ObjectsClient, ctx context.Context, input *PutObjectInput, absPath _AbsCleanPath) error {
|
|
|
|
if input.MaxContentLength != 0 && input.ContentLength != 0 {
|
|
|
|
return errors.New("ContentLength and MaxContentLength may not both be set to non-zero values.")
|
|
|
|
}
|
|
|
|
|
|
|
|
headers := &http.Header{}
|
|
|
|
for key, value := range input.Headers {
|
|
|
|
headers.Set(key, value)
|
|
|
|
}
|
|
|
|
if input.DurabilityLevel != 0 {
|
|
|
|
headers.Set("Durability-Level", strconv.FormatUint(input.DurabilityLevel, 10))
|
|
|
|
}
|
|
|
|
if input.ContentType != "" {
|
|
|
|
headers.Set("Content-Type", input.ContentType)
|
|
|
|
}
|
|
|
|
if input.ContentMD5 != "" {
|
|
|
|
headers.Set("Content-MD$", input.ContentMD5)
|
|
|
|
}
|
|
|
|
if input.IfMatch != "" {
|
|
|
|
headers.Set("If-Match", input.IfMatch)
|
|
|
|
}
|
|
|
|
if input.IfModifiedSince != nil {
|
|
|
|
headers.Set("If-Modified-Since", input.IfModifiedSince.Format(time.RFC1123))
|
|
|
|
}
|
|
|
|
if input.ContentLength != 0 {
|
|
|
|
headers.Set("Content-Length", strconv.FormatUint(input.ContentLength, 10))
|
|
|
|
}
|
|
|
|
if input.MaxContentLength != 0 {
|
|
|
|
headers.Set("Max-Content-Length", strconv.FormatUint(input.MaxContentLength, 10))
|
|
|
|
}
|
|
|
|
|
|
|
|
reqInput := client.RequestNoEncodeInput{
|
|
|
|
Method: http.MethodPut,
|
|
|
|
Path: string(absPath),
|
|
|
|
Headers: headers,
|
|
|
|
Body: input.ObjectReader,
|
|
|
|
}
|
|
|
|
respBody, _, err := c.client.ExecuteRequestNoEncode(ctx, reqInput)
|
|
|
|
if respBody != nil {
|
|
|
|
defer respBody.Close()
|
|
|
|
}
|
|
|
|
if err != nil {
|
|
|
|
return errors.Wrap(err, "unable to put object")
|
|
|
|
}
|
|
|
|
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
func createDirectory(c ObjectsClient, ctx context.Context, absPath _AbsCleanPath) error {
|
|
|
|
dirClient := &DirectoryClient{
|
|
|
|
client: c.client,
|
|
|
|
}
|
|
|
|
|
|
|
|
// An abspath starts w/ a leading "/" which gets added to the slice as an
|
|
|
|
// empty string. Start all array math at 1.
|
|
|
|
parts := strings.Split(string(absPath), "/")
|
|
|
|
if len(parts) < 2 {
|
|
|
|
return errors.New("no path components to create directory")
|
|
|
|
}
|
|
|
|
|
|
|
|
folderPath := parts[1]
|
|
|
|
// Don't attempt to create a manta account as a directory
|
|
|
|
for i := 2; i < len(parts); i++ {
|
|
|
|
part := parts[i]
|
|
|
|
folderPath = path.Clean(path.Join("/", folderPath, part))
|
|
|
|
err := dirClient.Put(ctx, &PutDirectoryInput{
|
|
|
|
DirectoryName: folderPath,
|
|
|
|
})
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2019-04-12 15:51:37 +00:00
|
|
|
func abortMpu(c ObjectsClient, ctx context.Context, input *AbortMpuInput) error {
|
|
|
|
reqInput := client.RequestInput{
|
|
|
|
Method: http.MethodPost,
|
|
|
|
Path: input.PartsDirectoryPath + "/abort",
|
|
|
|
Headers: &http.Header{},
|
|
|
|
Body: nil,
|
|
|
|
}
|
|
|
|
respBody, _, err := c.client.ExecuteRequestStorage(ctx, reqInput)
|
|
|
|
if err != nil {
|
|
|
|
return errors.Wrap(err, "unable to abort mpu")
|
|
|
|
}
|
|
|
|
|
|
|
|
if respBody != nil {
|
|
|
|
defer respBody.Close()
|
|
|
|
}
|
|
|
|
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
func commitMpu(c ObjectsClient, ctx context.Context, input *CommitMpuInput) error {
|
|
|
|
headers := &http.Header{}
|
|
|
|
for key, value := range input.Headers {
|
|
|
|
headers.Set(key, value)
|
|
|
|
}
|
|
|
|
|
|
|
|
// The mpu directory prefix length is derived from the final character
|
|
|
|
// in the mpu identifier which we'll call P. The mpu prefix itself is
|
|
|
|
// the first P characters of the mpu identifier. In order to derive the
|
|
|
|
// correct directory structure we need to parse this information from
|
|
|
|
// the mpu identifier
|
|
|
|
id := input.Id
|
|
|
|
idLength := len(id)
|
|
|
|
prefixLen, err := strconv.Atoi(id[idLength-1 : idLength])
|
|
|
|
if err != nil {
|
|
|
|
return errors.Wrap(err, "unable to commit mpu due to invalid mpu prefix length")
|
|
|
|
}
|
|
|
|
prefix := id[:prefixLen]
|
|
|
|
partPath := "/" + c.client.AccountName + "/uploads/" + prefix + "/" + input.Id + "/commit"
|
|
|
|
|
|
|
|
reqInput := client.RequestInput{
|
|
|
|
Method: http.MethodPost,
|
|
|
|
Path: partPath,
|
|
|
|
Headers: headers,
|
|
|
|
Body: input.Body,
|
|
|
|
}
|
|
|
|
respBody, _, err := c.client.ExecuteRequestStorage(ctx, reqInput)
|
|
|
|
if err != nil {
|
|
|
|
return errors.Wrap(err, "unable to commit mpu")
|
|
|
|
}
|
|
|
|
|
|
|
|
if respBody != nil {
|
|
|
|
defer respBody.Close()
|
|
|
|
}
|
|
|
|
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
func createMpu(c ObjectsClient, ctx context.Context, input *CreateMpuInput) (*CreateMpuOutput, error) {
|
|
|
|
absPath := absFileInput(c.client.AccountName, input.Body.ObjectPath)
|
|
|
|
|
|
|
|
// Because some clients will be treating Manta like S3, they will
|
|
|
|
// include slashes in object names which we'll need to convert to
|
|
|
|
// directories
|
|
|
|
if input.ForceInsert {
|
|
|
|
absDirName := _AbsCleanPath(path.Dir(string(absPath)))
|
|
|
|
exists, _ := checkDirectoryTreeExists(c, ctx, absDirName)
|
|
|
|
if !exists {
|
|
|
|
err := createDirectory(c, ctx, absDirName)
|
|
|
|
if err != nil {
|
|
|
|
return nil, errors.Wrap(err, "unable to create directory for create mpu operation")
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
headers := &http.Header{}
|
|
|
|
for key, value := range input.Body.Headers {
|
|
|
|
headers.Set(key, value)
|
|
|
|
}
|
|
|
|
if input.DurabilityLevel != 0 {
|
|
|
|
headers.Set("Durability-Level", strconv.FormatUint(input.DurabilityLevel, 10))
|
|
|
|
}
|
|
|
|
if input.ContentLength != 0 {
|
|
|
|
headers.Set("Content-Length", strconv.FormatUint(input.ContentLength, 10))
|
|
|
|
}
|
|
|
|
if input.ContentMD5 != "" {
|
|
|
|
headers.Set("Content-MD5", input.ContentMD5)
|
|
|
|
}
|
|
|
|
|
|
|
|
input.Body.ObjectPath = string(absPath)
|
|
|
|
reqInput := client.RequestInput{
|
|
|
|
Method: http.MethodPost,
|
|
|
|
Path: "/" + c.client.AccountName + "/uploads",
|
|
|
|
Headers: headers,
|
|
|
|
Body: input.Body,
|
|
|
|
}
|
|
|
|
respBody, _, err := c.client.ExecuteRequestStorage(ctx, reqInput)
|
|
|
|
if err != nil {
|
|
|
|
return nil, errors.Wrap(err, "unable to create mpu")
|
|
|
|
}
|
|
|
|
if respBody != nil {
|
|
|
|
defer respBody.Close()
|
|
|
|
}
|
|
|
|
|
|
|
|
response := &CreateMpuOutput{}
|
|
|
|
decoder := json.NewDecoder(respBody)
|
|
|
|
if err = decoder.Decode(&response); err != nil {
|
|
|
|
return nil, errors.Wrap(err, "unable to decode create mpu response")
|
|
|
|
}
|
|
|
|
|
|
|
|
return response, nil
|
|
|
|
}
|
|
|
|
|
|
|
|
func getMpu(c ObjectsClient, ctx context.Context, input *GetMpuInput) (*GetMpuOutput, error) {
|
|
|
|
headers := &http.Header{}
|
|
|
|
|
|
|
|
reqInput := client.RequestInput{
|
|
|
|
Method: http.MethodGet,
|
|
|
|
Path: input.PartsDirectoryPath + "/state",
|
|
|
|
Headers: headers,
|
|
|
|
}
|
|
|
|
respBody, _, err := c.client.ExecuteRequestStorage(ctx, reqInput)
|
|
|
|
if err != nil {
|
|
|
|
return nil, errors.Wrap(err, "unable to get mpu")
|
|
|
|
}
|
|
|
|
|
|
|
|
response := &GetMpuOutput{}
|
|
|
|
decoder := json.NewDecoder(respBody)
|
|
|
|
if err = decoder.Decode(&response); err != nil {
|
|
|
|
return nil, errors.Wrap(err, "unable to decode get mpu response")
|
|
|
|
}
|
|
|
|
|
|
|
|
return response, nil
|
|
|
|
}
|
|
|
|
|
|
|
|
func listMpuParts(c ObjectsClient, ctx context.Context, input *ListMpuPartsInput) (*ListMpuPartsOutput, error) {
|
|
|
|
id := input.Id
|
|
|
|
idLength := len(id)
|
|
|
|
prefixLen, err := strconv.Atoi(id[idLength-1 : idLength])
|
|
|
|
if err != nil {
|
|
|
|
return nil, errors.Wrap(err, "unable to upload part")
|
|
|
|
}
|
|
|
|
prefix := id[:prefixLen]
|
|
|
|
partPath := "/" + c.client.AccountName + "/uploads/" + prefix + "/" + input.Id + "/"
|
|
|
|
listDirInput := ListDirectoryInput{
|
|
|
|
DirectoryName: partPath,
|
|
|
|
}
|
|
|
|
|
|
|
|
dirClient := &DirectoryClient{
|
|
|
|
client: c.client,
|
|
|
|
}
|
|
|
|
|
|
|
|
listDirOutput, err := dirClient.List(ctx, &listDirInput)
|
|
|
|
if err != nil {
|
|
|
|
return nil, errors.Wrap(err, "unable to list mpu parts")
|
|
|
|
}
|
|
|
|
|
|
|
|
var parts []ListMpuPart
|
|
|
|
for num, part := range listDirOutput.Entries {
|
|
|
|
parts = append(parts, ListMpuPart{
|
|
|
|
ETag: part.ETag,
|
|
|
|
PartNumber: num,
|
|
|
|
Size: int64(part.Size),
|
|
|
|
})
|
|
|
|
}
|
|
|
|
|
|
|
|
listMpuPartsOutput := &ListMpuPartsOutput{
|
|
|
|
Parts: parts,
|
|
|
|
}
|
|
|
|
|
|
|
|
return listMpuPartsOutput, nil
|
|
|
|
}
|
|
|
|
|
|
|
|
func uploadPart(c ObjectsClient, ctx context.Context, input *UploadPartInput) (*UploadPartOutput, error) {
|
|
|
|
headers := &http.Header{}
|
|
|
|
for key, value := range input.Headers {
|
|
|
|
headers.Set(key, value)
|
|
|
|
}
|
|
|
|
|
|
|
|
if input.ContentMD5 != "" {
|
|
|
|
headers.Set("Content-MD5", input.ContentMD5)
|
|
|
|
}
|
|
|
|
|
|
|
|
// The mpu directory prefix length is derived from the final character
|
|
|
|
// in the mpu identifier which we'll call P. The mpu prefix itself is
|
|
|
|
// the first P characters of the mpu identifier. In order to derive the
|
|
|
|
// correct directory structure we need to parse this information from
|
|
|
|
// the mpu identifier
|
|
|
|
id := input.Id
|
|
|
|
idLength := len(id)
|
|
|
|
partNum := strconv.FormatUint(input.PartNum, 10)
|
|
|
|
prefixLen, err := strconv.Atoi(id[idLength-1 : idLength])
|
|
|
|
if err != nil {
|
|
|
|
return nil, errors.Wrap(err, "unable to upload part due to invalid mpu prefix length")
|
|
|
|
}
|
|
|
|
prefix := id[:prefixLen]
|
|
|
|
partPath := "/" + c.client.AccountName + "/uploads/" + prefix + "/" + input.Id + "/" + partNum
|
|
|
|
|
|
|
|
reqInput := client.RequestNoEncodeInput{
|
|
|
|
Method: http.MethodPut,
|
|
|
|
Path: partPath,
|
|
|
|
Headers: headers,
|
|
|
|
Body: input.ObjectReader,
|
|
|
|
}
|
|
|
|
respBody, respHeader, err := c.client.ExecuteRequestNoEncode(ctx, reqInput)
|
|
|
|
if respBody != nil {
|
|
|
|
defer respBody.Close()
|
|
|
|
}
|
|
|
|
if err != nil {
|
|
|
|
return nil, errors.Wrap(err, "unable to upload part")
|
|
|
|
}
|
|
|
|
|
|
|
|
uploadPartOutput := &UploadPartOutput{
|
|
|
|
Part: respHeader.Get("Etag"),
|
|
|
|
}
|
|
|
|
return uploadPartOutput, nil
|
|
|
|
}
|
|
|
|
|
2018-02-12 23:22:41 +00:00
|
|
|
func checkDirectoryTreeExists(c ObjectsClient, ctx context.Context, absPath _AbsCleanPath) (bool, error) {
|
|
|
|
exists, err := c.IsDir(ctx, string(absPath))
|
|
|
|
if err != nil {
|
2019-04-12 15:51:37 +00:00
|
|
|
if tt.IsResourceNotFoundError(err) || tt.IsStatusNotFoundCode(err) {
|
2018-02-12 23:22:41 +00:00
|
|
|
return false, nil
|
|
|
|
}
|
|
|
|
return false, err
|
|
|
|
}
|
|
|
|
if exists {
|
|
|
|
return true, nil
|
|
|
|
}
|
|
|
|
|
|
|
|
return false, nil
|
|
|
|
}
|