435c0d9fc8
This PR switches the Nomad repository from using govendor to Go modules for managing dependencies. Aspects of the Nomad workflow remain pretty much the same. The usual Makefile targets should continue to work as they always did. The API submodule simply defers to the parent Nomad version on the repository, keeping the semantics of API versioning that currently exists.
430 lines
11 KiB
Go
430 lines
11 KiB
Go
// Copyright 2011 Google Inc. All rights reserved.
|
|
// Use of this source code is governed by the Apache 2.0
|
|
// license that can be found in the LICENSE file.
|
|
|
|
package datastore
|
|
|
|
import (
|
|
"fmt"
|
|
"reflect"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/golang/protobuf/proto"
|
|
"google.golang.org/appengine"
|
|
pb "google.golang.org/appengine/internal/datastore"
|
|
)
|
|
|
|
var (
|
|
typeOfBlobKey = reflect.TypeOf(appengine.BlobKey(""))
|
|
typeOfByteSlice = reflect.TypeOf([]byte(nil))
|
|
typeOfByteString = reflect.TypeOf(ByteString(nil))
|
|
typeOfGeoPoint = reflect.TypeOf(appengine.GeoPoint{})
|
|
typeOfTime = reflect.TypeOf(time.Time{})
|
|
typeOfKeyPtr = reflect.TypeOf(&Key{})
|
|
typeOfEntityPtr = reflect.TypeOf(&Entity{})
|
|
)
|
|
|
|
// typeMismatchReason returns a string explaining why the property p could not
|
|
// be stored in an entity field of type v.Type().
|
|
func typeMismatchReason(pValue interface{}, v reflect.Value) string {
|
|
entityType := "empty"
|
|
switch pValue.(type) {
|
|
case int64:
|
|
entityType = "int"
|
|
case bool:
|
|
entityType = "bool"
|
|
case string:
|
|
entityType = "string"
|
|
case float64:
|
|
entityType = "float"
|
|
case *Key:
|
|
entityType = "*datastore.Key"
|
|
case time.Time:
|
|
entityType = "time.Time"
|
|
case appengine.BlobKey:
|
|
entityType = "appengine.BlobKey"
|
|
case appengine.GeoPoint:
|
|
entityType = "appengine.GeoPoint"
|
|
case ByteString:
|
|
entityType = "datastore.ByteString"
|
|
case []byte:
|
|
entityType = "[]byte"
|
|
}
|
|
return fmt.Sprintf("type mismatch: %s versus %v", entityType, v.Type())
|
|
}
|
|
|
|
type propertyLoader struct {
|
|
// m holds the number of times a substruct field like "Foo.Bar.Baz" has
|
|
// been seen so far. The map is constructed lazily.
|
|
m map[string]int
|
|
}
|
|
|
|
func (l *propertyLoader) load(codec *structCodec, structValue reflect.Value, p Property, requireSlice bool) string {
|
|
var v reflect.Value
|
|
var sliceIndex int
|
|
|
|
name := p.Name
|
|
|
|
// If name ends with a '.', the last field is anonymous.
|
|
// In this case, strings.Split will give us "" as the
|
|
// last element of our fields slice, which will match the ""
|
|
// field name in the substruct codec.
|
|
fields := strings.Split(name, ".")
|
|
|
|
for len(fields) > 0 {
|
|
var decoder fieldCodec
|
|
var ok bool
|
|
|
|
// Cut off the last field (delimited by ".") and find its parent
|
|
// in the codec.
|
|
// eg. for name "A.B.C.D", split off "A.B.C" and try to
|
|
// find a field in the codec with this name.
|
|
// Loop again with "A.B", etc.
|
|
for i := len(fields); i > 0; i-- {
|
|
parent := strings.Join(fields[:i], ".")
|
|
decoder, ok = codec.fields[parent]
|
|
if ok {
|
|
fields = fields[i:]
|
|
break
|
|
}
|
|
}
|
|
|
|
// If we never found a matching field in the codec, return
|
|
// error message.
|
|
if !ok {
|
|
return "no such struct field"
|
|
}
|
|
|
|
v = initField(structValue, decoder.path)
|
|
if !v.IsValid() {
|
|
return "no such struct field"
|
|
}
|
|
if !v.CanSet() {
|
|
return "cannot set struct field"
|
|
}
|
|
|
|
if decoder.structCodec != nil {
|
|
codec = decoder.structCodec
|
|
structValue = v
|
|
}
|
|
|
|
if v.Kind() == reflect.Slice && v.Type() != typeOfByteSlice {
|
|
if l.m == nil {
|
|
l.m = make(map[string]int)
|
|
}
|
|
sliceIndex = l.m[p.Name]
|
|
l.m[p.Name] = sliceIndex + 1
|
|
for v.Len() <= sliceIndex {
|
|
v.Set(reflect.Append(v, reflect.New(v.Type().Elem()).Elem()))
|
|
}
|
|
structValue = v.Index(sliceIndex)
|
|
requireSlice = false
|
|
}
|
|
}
|
|
|
|
var slice reflect.Value
|
|
if v.Kind() == reflect.Slice && v.Type().Elem().Kind() != reflect.Uint8 {
|
|
slice = v
|
|
v = reflect.New(v.Type().Elem()).Elem()
|
|
} else if requireSlice {
|
|
return "multiple-valued property requires a slice field type"
|
|
}
|
|
|
|
// Convert indexValues to a Go value with a meaning derived from the
|
|
// destination type.
|
|
pValue := p.Value
|
|
if iv, ok := pValue.(indexValue); ok {
|
|
meaning := pb.Property_NO_MEANING
|
|
switch v.Type() {
|
|
case typeOfBlobKey:
|
|
meaning = pb.Property_BLOBKEY
|
|
case typeOfByteSlice:
|
|
meaning = pb.Property_BLOB
|
|
case typeOfByteString:
|
|
meaning = pb.Property_BYTESTRING
|
|
case typeOfGeoPoint:
|
|
meaning = pb.Property_GEORSS_POINT
|
|
case typeOfTime:
|
|
meaning = pb.Property_GD_WHEN
|
|
case typeOfEntityPtr:
|
|
meaning = pb.Property_ENTITY_PROTO
|
|
}
|
|
var err error
|
|
pValue, err = propValue(iv.value, meaning)
|
|
if err != nil {
|
|
return err.Error()
|
|
}
|
|
}
|
|
|
|
if errReason := setVal(v, pValue); errReason != "" {
|
|
// Set the slice back to its zero value.
|
|
if slice.IsValid() {
|
|
slice.Set(reflect.Zero(slice.Type()))
|
|
}
|
|
return errReason
|
|
}
|
|
|
|
if slice.IsValid() {
|
|
slice.Index(sliceIndex).Set(v)
|
|
}
|
|
|
|
return ""
|
|
}
|
|
|
|
// setVal sets v to the value pValue.
|
|
func setVal(v reflect.Value, pValue interface{}) string {
|
|
switch v.Kind() {
|
|
case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
|
|
x, ok := pValue.(int64)
|
|
if !ok && pValue != nil {
|
|
return typeMismatchReason(pValue, v)
|
|
}
|
|
if v.OverflowInt(x) {
|
|
return fmt.Sprintf("value %v overflows struct field of type %v", x, v.Type())
|
|
}
|
|
v.SetInt(x)
|
|
case reflect.Bool:
|
|
x, ok := pValue.(bool)
|
|
if !ok && pValue != nil {
|
|
return typeMismatchReason(pValue, v)
|
|
}
|
|
v.SetBool(x)
|
|
case reflect.String:
|
|
switch x := pValue.(type) {
|
|
case appengine.BlobKey:
|
|
v.SetString(string(x))
|
|
case ByteString:
|
|
v.SetString(string(x))
|
|
case string:
|
|
v.SetString(x)
|
|
default:
|
|
if pValue != nil {
|
|
return typeMismatchReason(pValue, v)
|
|
}
|
|
}
|
|
case reflect.Float32, reflect.Float64:
|
|
x, ok := pValue.(float64)
|
|
if !ok && pValue != nil {
|
|
return typeMismatchReason(pValue, v)
|
|
}
|
|
if v.OverflowFloat(x) {
|
|
return fmt.Sprintf("value %v overflows struct field of type %v", x, v.Type())
|
|
}
|
|
v.SetFloat(x)
|
|
case reflect.Ptr:
|
|
x, ok := pValue.(*Key)
|
|
if !ok && pValue != nil {
|
|
return typeMismatchReason(pValue, v)
|
|
}
|
|
if _, ok := v.Interface().(*Key); !ok {
|
|
return typeMismatchReason(pValue, v)
|
|
}
|
|
v.Set(reflect.ValueOf(x))
|
|
case reflect.Struct:
|
|
switch v.Type() {
|
|
case typeOfTime:
|
|
x, ok := pValue.(time.Time)
|
|
if !ok && pValue != nil {
|
|
return typeMismatchReason(pValue, v)
|
|
}
|
|
v.Set(reflect.ValueOf(x))
|
|
case typeOfGeoPoint:
|
|
x, ok := pValue.(appengine.GeoPoint)
|
|
if !ok && pValue != nil {
|
|
return typeMismatchReason(pValue, v)
|
|
}
|
|
v.Set(reflect.ValueOf(x))
|
|
default:
|
|
ent, ok := pValue.(*Entity)
|
|
if !ok {
|
|
return typeMismatchReason(pValue, v)
|
|
}
|
|
|
|
// Recursively load nested struct
|
|
pls, err := newStructPLS(v.Addr().Interface())
|
|
if err != nil {
|
|
return err.Error()
|
|
}
|
|
|
|
// if ent has a Key value and our struct has a Key field,
|
|
// load the Entity's Key value into the Key field on the struct.
|
|
if ent.Key != nil && pls.codec.keyField != -1 {
|
|
|
|
pls.v.Field(pls.codec.keyField).Set(reflect.ValueOf(ent.Key))
|
|
}
|
|
|
|
err = pls.Load(ent.Properties)
|
|
if err != nil {
|
|
return err.Error()
|
|
}
|
|
}
|
|
case reflect.Slice:
|
|
x, ok := pValue.([]byte)
|
|
if !ok {
|
|
if y, yok := pValue.(ByteString); yok {
|
|
x, ok = []byte(y), true
|
|
}
|
|
}
|
|
if !ok && pValue != nil {
|
|
return typeMismatchReason(pValue, v)
|
|
}
|
|
if v.Type().Elem().Kind() != reflect.Uint8 {
|
|
return typeMismatchReason(pValue, v)
|
|
}
|
|
v.SetBytes(x)
|
|
default:
|
|
return typeMismatchReason(pValue, v)
|
|
}
|
|
return ""
|
|
}
|
|
|
|
// initField is similar to reflect's Value.FieldByIndex, in that it
|
|
// returns the nested struct field corresponding to index, but it
|
|
// initialises any nil pointers encountered when traversing the structure.
|
|
func initField(val reflect.Value, index []int) reflect.Value {
|
|
for _, i := range index[:len(index)-1] {
|
|
val = val.Field(i)
|
|
if val.Kind() == reflect.Ptr {
|
|
if val.IsNil() {
|
|
val.Set(reflect.New(val.Type().Elem()))
|
|
}
|
|
val = val.Elem()
|
|
}
|
|
}
|
|
return val.Field(index[len(index)-1])
|
|
}
|
|
|
|
// loadEntity loads an EntityProto into PropertyLoadSaver or struct pointer.
|
|
func loadEntity(dst interface{}, src *pb.EntityProto) (err error) {
|
|
ent, err := protoToEntity(src)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if e, ok := dst.(PropertyLoadSaver); ok {
|
|
return e.Load(ent.Properties)
|
|
}
|
|
return LoadStruct(dst, ent.Properties)
|
|
}
|
|
|
|
func (s structPLS) Load(props []Property) error {
|
|
var fieldName, reason string
|
|
var l propertyLoader
|
|
for _, p := range props {
|
|
if errStr := l.load(s.codec, s.v, p, p.Multiple); errStr != "" {
|
|
// We don't return early, as we try to load as many properties as possible.
|
|
// It is valid to load an entity into a struct that cannot fully represent it.
|
|
// That case returns an error, but the caller is free to ignore it.
|
|
fieldName, reason = p.Name, errStr
|
|
}
|
|
}
|
|
if reason != "" {
|
|
return &ErrFieldMismatch{
|
|
StructType: s.v.Type(),
|
|
FieldName: fieldName,
|
|
Reason: reason,
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func protoToEntity(src *pb.EntityProto) (*Entity, error) {
|
|
props, rawProps := src.Property, src.RawProperty
|
|
outProps := make([]Property, 0, len(props)+len(rawProps))
|
|
for {
|
|
var (
|
|
x *pb.Property
|
|
noIndex bool
|
|
)
|
|
if len(props) > 0 {
|
|
x, props = props[0], props[1:]
|
|
} else if len(rawProps) > 0 {
|
|
x, rawProps = rawProps[0], rawProps[1:]
|
|
noIndex = true
|
|
} else {
|
|
break
|
|
}
|
|
|
|
var value interface{}
|
|
if x.Meaning != nil && *x.Meaning == pb.Property_INDEX_VALUE {
|
|
value = indexValue{x.Value}
|
|
} else {
|
|
var err error
|
|
value, err = propValue(x.Value, x.GetMeaning())
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
outProps = append(outProps, Property{
|
|
Name: x.GetName(),
|
|
Value: value,
|
|
NoIndex: noIndex,
|
|
Multiple: x.GetMultiple(),
|
|
})
|
|
}
|
|
|
|
var key *Key
|
|
if src.Key != nil {
|
|
// Ignore any error, since nested entity values
|
|
// are allowed to have an invalid key.
|
|
key, _ = protoToKey(src.Key)
|
|
}
|
|
return &Entity{key, outProps}, nil
|
|
}
|
|
|
|
// propValue returns a Go value that combines the raw PropertyValue with a
|
|
// meaning. For example, an Int64Value with GD_WHEN becomes a time.Time.
|
|
func propValue(v *pb.PropertyValue, m pb.Property_Meaning) (interface{}, error) {
|
|
switch {
|
|
case v.Int64Value != nil:
|
|
if m == pb.Property_GD_WHEN {
|
|
return fromUnixMicro(*v.Int64Value), nil
|
|
} else {
|
|
return *v.Int64Value, nil
|
|
}
|
|
case v.BooleanValue != nil:
|
|
return *v.BooleanValue, nil
|
|
case v.StringValue != nil:
|
|
if m == pb.Property_BLOB {
|
|
return []byte(*v.StringValue), nil
|
|
} else if m == pb.Property_BLOBKEY {
|
|
return appengine.BlobKey(*v.StringValue), nil
|
|
} else if m == pb.Property_BYTESTRING {
|
|
return ByteString(*v.StringValue), nil
|
|
} else if m == pb.Property_ENTITY_PROTO {
|
|
var ent pb.EntityProto
|
|
err := proto.Unmarshal([]byte(*v.StringValue), &ent)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return protoToEntity(&ent)
|
|
} else {
|
|
return *v.StringValue, nil
|
|
}
|
|
case v.DoubleValue != nil:
|
|
return *v.DoubleValue, nil
|
|
case v.Referencevalue != nil:
|
|
key, err := referenceValueToKey(v.Referencevalue)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return key, nil
|
|
case v.Pointvalue != nil:
|
|
// NOTE: Strangely, latitude maps to X, longitude to Y.
|
|
return appengine.GeoPoint{Lat: v.Pointvalue.GetX(), Lng: v.Pointvalue.GetY()}, nil
|
|
}
|
|
return nil, nil
|
|
}
|
|
|
|
// indexValue is a Property value that is created when entities are loaded from
|
|
// an index, such as from a projection query.
|
|
//
|
|
// Such Property values do not contain all of the metadata required to be
|
|
// faithfully represented as a Go value, and are instead represented as an
|
|
// opaque indexValue. Load the properties into a concrete struct type (e.g. by
|
|
// passing a struct pointer to Iterator.Next) to reconstruct actual Go values
|
|
// of type int, string, time.Time, etc.
|
|
type indexValue struct {
|
|
value *pb.PropertyValue
|
|
}
|