properties.go 17 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608
  1. // Protocol Buffers for Go with Gadgets
  2. //
  3. // Copyright (c) 2013, The GoGo Authors. All rights reserved.
  4. // http://github.com/gogo/protobuf
  5. //
  6. // Go support for Protocol Buffers - Google's data interchange format
  7. //
  8. // Copyright 2010 The Go Authors. All rights reserved.
  9. // https://github.com/golang/protobuf
  10. //
  11. // Redistribution and use in source and binary forms, with or without
  12. // modification, are permitted provided that the following conditions are
  13. // met:
  14. //
  15. // * Redistributions of source code must retain the above copyright
  16. // notice, this list of conditions and the following disclaimer.
  17. // * Redistributions in binary form must reproduce the above
  18. // copyright notice, this list of conditions and the following disclaimer
  19. // in the documentation and/or other materials provided with the
  20. // distribution.
  21. // * Neither the name of Google Inc. nor the names of its
  22. // contributors may be used to endorse or promote products derived from
  23. // this software without specific prior written permission.
  24. //
  25. // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  26. // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  27. // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  28. // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  29. // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  30. // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  31. // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  32. // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  33. // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  34. // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  35. // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  36. package proto
  37. /*
  38. * Routines for encoding data into the wire format for protocol buffers.
  39. */
  40. import (
  41. "fmt"
  42. "log"
  43. "os"
  44. "reflect"
  45. "sort"
  46. "strconv"
  47. "strings"
  48. "sync"
  49. )
  50. const debug bool = false
  51. // Constants that identify the encoding of a value on the wire.
  52. const (
  53. WireVarint = 0
  54. WireFixed64 = 1
  55. WireBytes = 2
  56. WireStartGroup = 3
  57. WireEndGroup = 4
  58. WireFixed32 = 5
  59. )
  60. // tagMap is an optimization over map[int]int for typical protocol buffer
  61. // use-cases. Encoded protocol buffers are often in tag order with small tag
  62. // numbers.
  63. type tagMap struct {
  64. fastTags []int
  65. slowTags map[int]int
  66. }
  67. // tagMapFastLimit is the upper bound on the tag number that will be stored in
  68. // the tagMap slice rather than its map.
  69. const tagMapFastLimit = 1024
  70. func (p *tagMap) get(t int) (int, bool) {
  71. if t > 0 && t < tagMapFastLimit {
  72. if t >= len(p.fastTags) {
  73. return 0, false
  74. }
  75. fi := p.fastTags[t]
  76. return fi, fi >= 0
  77. }
  78. fi, ok := p.slowTags[t]
  79. return fi, ok
  80. }
  81. func (p *tagMap) put(t int, fi int) {
  82. if t > 0 && t < tagMapFastLimit {
  83. for len(p.fastTags) < t+1 {
  84. p.fastTags = append(p.fastTags, -1)
  85. }
  86. p.fastTags[t] = fi
  87. return
  88. }
  89. if p.slowTags == nil {
  90. p.slowTags = make(map[int]int)
  91. }
  92. p.slowTags[t] = fi
  93. }
  94. // StructProperties represents properties for all the fields of a struct.
  95. // decoderTags and decoderOrigNames should only be used by the decoder.
  96. type StructProperties struct {
  97. Prop []*Properties // properties for each field
  98. reqCount int // required count
  99. decoderTags tagMap // map from proto tag to struct field number
  100. decoderOrigNames map[string]int // map from original name to struct field number
  101. order []int // list of struct field numbers in tag order
  102. // OneofTypes contains information about the oneof fields in this message.
  103. // It is keyed by the original name of a field.
  104. OneofTypes map[string]*OneofProperties
  105. }
  106. // OneofProperties represents information about a specific field in a oneof.
  107. type OneofProperties struct {
  108. Type reflect.Type // pointer to generated struct type for this oneof field
  109. Field int // struct field number of the containing oneof in the message
  110. Prop *Properties
  111. }
  112. // Implement the sorting interface so we can sort the fields in tag order, as recommended by the spec.
  113. // See encode.go, (*Buffer).enc_struct.
  114. func (sp *StructProperties) Len() int { return len(sp.order) }
  115. func (sp *StructProperties) Less(i, j int) bool {
  116. return sp.Prop[sp.order[i]].Tag < sp.Prop[sp.order[j]].Tag
  117. }
  118. func (sp *StructProperties) Swap(i, j int) { sp.order[i], sp.order[j] = sp.order[j], sp.order[i] }
  119. // Properties represents the protocol-specific behavior of a single struct field.
  120. type Properties struct {
  121. Name string // name of the field, for error messages
  122. OrigName string // original name before protocol compiler (always set)
  123. JSONName string // name to use for JSON; determined by protoc
  124. Wire string
  125. WireType int
  126. Tag int
  127. Required bool
  128. Optional bool
  129. Repeated bool
  130. Packed bool // relevant for repeated primitives only
  131. Enum string // set for enum types only
  132. proto3 bool // whether this is known to be a proto3 field
  133. oneof bool // whether this is a oneof field
  134. Default string // default value
  135. HasDefault bool // whether an explicit default was provided
  136. CustomType string
  137. CastType string
  138. StdTime bool
  139. StdDuration bool
  140. WktPointer bool
  141. stype reflect.Type // set for struct types only
  142. ctype reflect.Type // set for custom types only
  143. sprop *StructProperties // set for struct types only
  144. mtype reflect.Type // set for map types only
  145. MapKeyProp *Properties // set for map types only
  146. MapValProp *Properties // set for map types only
  147. }
  148. // String formats the properties in the protobuf struct field tag style.
  149. func (p *Properties) String() string {
  150. s := p.Wire
  151. s += ","
  152. s += strconv.Itoa(p.Tag)
  153. if p.Required {
  154. s += ",req"
  155. }
  156. if p.Optional {
  157. s += ",opt"
  158. }
  159. if p.Repeated {
  160. s += ",rep"
  161. }
  162. if p.Packed {
  163. s += ",packed"
  164. }
  165. s += ",name=" + p.OrigName
  166. if p.JSONName != p.OrigName {
  167. s += ",json=" + p.JSONName
  168. }
  169. if p.proto3 {
  170. s += ",proto3"
  171. }
  172. if p.oneof {
  173. s += ",oneof"
  174. }
  175. if len(p.Enum) > 0 {
  176. s += ",enum=" + p.Enum
  177. }
  178. if p.HasDefault {
  179. s += ",def=" + p.Default
  180. }
  181. return s
  182. }
  183. // Parse populates p by parsing a string in the protobuf struct field tag style.
  184. func (p *Properties) Parse(s string) {
  185. // "bytes,49,opt,name=foo,def=hello!"
  186. fields := strings.Split(s, ",") // breaks def=, but handled below.
  187. if len(fields) < 2 {
  188. fmt.Fprintf(os.Stderr, "proto: tag has too few fields: %q\n", s)
  189. return
  190. }
  191. p.Wire = fields[0]
  192. switch p.Wire {
  193. case "varint":
  194. p.WireType = WireVarint
  195. case "fixed32":
  196. p.WireType = WireFixed32
  197. case "fixed64":
  198. p.WireType = WireFixed64
  199. case "zigzag32":
  200. p.WireType = WireVarint
  201. case "zigzag64":
  202. p.WireType = WireVarint
  203. case "bytes", "group":
  204. p.WireType = WireBytes
  205. // no numeric converter for non-numeric types
  206. default:
  207. fmt.Fprintf(os.Stderr, "proto: tag has unknown wire type: %q\n", s)
  208. return
  209. }
  210. var err error
  211. p.Tag, err = strconv.Atoi(fields[1])
  212. if err != nil {
  213. return
  214. }
  215. outer:
  216. for i := 2; i < len(fields); i++ {
  217. f := fields[i]
  218. switch {
  219. case f == "req":
  220. p.Required = true
  221. case f == "opt":
  222. p.Optional = true
  223. case f == "rep":
  224. p.Repeated = true
  225. case f == "packed":
  226. p.Packed = true
  227. case strings.HasPrefix(f, "name="):
  228. p.OrigName = f[5:]
  229. case strings.HasPrefix(f, "json="):
  230. p.JSONName = f[5:]
  231. case strings.HasPrefix(f, "enum="):
  232. p.Enum = f[5:]
  233. case f == "proto3":
  234. p.proto3 = true
  235. case f == "oneof":
  236. p.oneof = true
  237. case strings.HasPrefix(f, "def="):
  238. p.HasDefault = true
  239. p.Default = f[4:] // rest of string
  240. if i+1 < len(fields) {
  241. // Commas aren't escaped, and def is always last.
  242. p.Default += "," + strings.Join(fields[i+1:], ",")
  243. break outer
  244. }
  245. case strings.HasPrefix(f, "embedded="):
  246. p.OrigName = strings.Split(f, "=")[1]
  247. case strings.HasPrefix(f, "customtype="):
  248. p.CustomType = strings.Split(f, "=")[1]
  249. case strings.HasPrefix(f, "casttype="):
  250. p.CastType = strings.Split(f, "=")[1]
  251. case f == "stdtime":
  252. p.StdTime = true
  253. case f == "stdduration":
  254. p.StdDuration = true
  255. case f == "wktptr":
  256. p.WktPointer = true
  257. }
  258. }
  259. }
  260. var protoMessageType = reflect.TypeOf((*Message)(nil)).Elem()
  261. // setFieldProps initializes the field properties for submessages and maps.
  262. func (p *Properties) setFieldProps(typ reflect.Type, f *reflect.StructField, lockGetProp bool) {
  263. isMap := typ.Kind() == reflect.Map
  264. if len(p.CustomType) > 0 && !isMap {
  265. p.ctype = typ
  266. p.setTag(lockGetProp)
  267. return
  268. }
  269. if p.StdTime && !isMap {
  270. p.setTag(lockGetProp)
  271. return
  272. }
  273. if p.StdDuration && !isMap {
  274. p.setTag(lockGetProp)
  275. return
  276. }
  277. if p.WktPointer && !isMap {
  278. p.setTag(lockGetProp)
  279. return
  280. }
  281. switch t1 := typ; t1.Kind() {
  282. case reflect.Struct:
  283. p.stype = typ
  284. case reflect.Ptr:
  285. if t1.Elem().Kind() == reflect.Struct {
  286. p.stype = t1.Elem()
  287. }
  288. case reflect.Slice:
  289. switch t2 := t1.Elem(); t2.Kind() {
  290. case reflect.Ptr:
  291. switch t3 := t2.Elem(); t3.Kind() {
  292. case reflect.Struct:
  293. p.stype = t3
  294. }
  295. case reflect.Struct:
  296. p.stype = t2
  297. }
  298. case reflect.Map:
  299. p.mtype = t1
  300. p.MapKeyProp = &Properties{}
  301. p.MapKeyProp.init(reflect.PtrTo(p.mtype.Key()), "Key", f.Tag.Get("protobuf_key"), nil, lockGetProp)
  302. p.MapValProp = &Properties{}
  303. vtype := p.mtype.Elem()
  304. if vtype.Kind() != reflect.Ptr && vtype.Kind() != reflect.Slice {
  305. // The value type is not a message (*T) or bytes ([]byte),
  306. // so we need encoders for the pointer to this type.
  307. vtype = reflect.PtrTo(vtype)
  308. }
  309. p.MapValProp.CustomType = p.CustomType
  310. p.MapValProp.StdDuration = p.StdDuration
  311. p.MapValProp.StdTime = p.StdTime
  312. p.MapValProp.WktPointer = p.WktPointer
  313. p.MapValProp.init(vtype, "Value", f.Tag.Get("protobuf_val"), nil, lockGetProp)
  314. }
  315. p.setTag(lockGetProp)
  316. }
  317. func (p *Properties) setTag(lockGetProp bool) {
  318. if p.stype != nil {
  319. if lockGetProp {
  320. p.sprop = GetProperties(p.stype)
  321. } else {
  322. p.sprop = getPropertiesLocked(p.stype)
  323. }
  324. }
  325. }
  326. var (
  327. marshalerType = reflect.TypeOf((*Marshaler)(nil)).Elem()
  328. )
  329. // Init populates the properties from a protocol buffer struct tag.
  330. func (p *Properties) Init(typ reflect.Type, name, tag string, f *reflect.StructField) {
  331. p.init(typ, name, tag, f, true)
  332. }
  333. func (p *Properties) init(typ reflect.Type, name, tag string, f *reflect.StructField, lockGetProp bool) {
  334. // "bytes,49,opt,def=hello!"
  335. p.Name = name
  336. p.OrigName = name
  337. if tag == "" {
  338. return
  339. }
  340. p.Parse(tag)
  341. p.setFieldProps(typ, f, lockGetProp)
  342. }
  343. var (
  344. propertiesMu sync.RWMutex
  345. propertiesMap = make(map[reflect.Type]*StructProperties)
  346. )
  347. // GetProperties returns the list of properties for the type represented by t.
  348. // t must represent a generated struct type of a protocol message.
  349. func GetProperties(t reflect.Type) *StructProperties {
  350. if t.Kind() != reflect.Struct {
  351. panic("proto: type must have kind struct")
  352. }
  353. // Most calls to GetProperties in a long-running program will be
  354. // retrieving details for types we have seen before.
  355. propertiesMu.RLock()
  356. sprop, ok := propertiesMap[t]
  357. propertiesMu.RUnlock()
  358. if ok {
  359. if collectStats {
  360. stats.Chit++
  361. }
  362. return sprop
  363. }
  364. propertiesMu.Lock()
  365. sprop = getPropertiesLocked(t)
  366. propertiesMu.Unlock()
  367. return sprop
  368. }
  369. // getPropertiesLocked requires that propertiesMu is held.
  370. func getPropertiesLocked(t reflect.Type) *StructProperties {
  371. if prop, ok := propertiesMap[t]; ok {
  372. if collectStats {
  373. stats.Chit++
  374. }
  375. return prop
  376. }
  377. if collectStats {
  378. stats.Cmiss++
  379. }
  380. prop := new(StructProperties)
  381. // in case of recursive protos, fill this in now.
  382. propertiesMap[t] = prop
  383. // build properties
  384. prop.Prop = make([]*Properties, t.NumField())
  385. prop.order = make([]int, t.NumField())
  386. isOneofMessage := false
  387. for i := 0; i < t.NumField(); i++ {
  388. f := t.Field(i)
  389. p := new(Properties)
  390. name := f.Name
  391. p.init(f.Type, name, f.Tag.Get("protobuf"), &f, false)
  392. oneof := f.Tag.Get("protobuf_oneof") // special case
  393. if oneof != "" {
  394. isOneofMessage = true
  395. // Oneof fields don't use the traditional protobuf tag.
  396. p.OrigName = oneof
  397. }
  398. prop.Prop[i] = p
  399. prop.order[i] = i
  400. if debug {
  401. print(i, " ", f.Name, " ", t.String(), " ")
  402. if p.Tag > 0 {
  403. print(p.String())
  404. }
  405. print("\n")
  406. }
  407. }
  408. // Re-order prop.order.
  409. sort.Sort(prop)
  410. type oneofMessage interface {
  411. XXX_OneofFuncs() (func(Message, *Buffer) error, func(Message, int, int, *Buffer) (bool, error), func(Message) int, []interface{})
  412. }
  413. if om, ok := reflect.Zero(reflect.PtrTo(t)).Interface().(oneofMessage); isOneofMessage && ok {
  414. var oots []interface{}
  415. _, _, _, oots = om.XXX_OneofFuncs()
  416. // Interpret oneof metadata.
  417. prop.OneofTypes = make(map[string]*OneofProperties)
  418. for _, oot := range oots {
  419. oop := &OneofProperties{
  420. Type: reflect.ValueOf(oot).Type(), // *T
  421. Prop: new(Properties),
  422. }
  423. sft := oop.Type.Elem().Field(0)
  424. oop.Prop.Name = sft.Name
  425. oop.Prop.Parse(sft.Tag.Get("protobuf"))
  426. // There will be exactly one interface field that
  427. // this new value is assignable to.
  428. for i := 0; i < t.NumField(); i++ {
  429. f := t.Field(i)
  430. if f.Type.Kind() != reflect.Interface {
  431. continue
  432. }
  433. if !oop.Type.AssignableTo(f.Type) {
  434. continue
  435. }
  436. oop.Field = i
  437. break
  438. }
  439. prop.OneofTypes[oop.Prop.OrigName] = oop
  440. }
  441. }
  442. // build required counts
  443. // build tags
  444. reqCount := 0
  445. prop.decoderOrigNames = make(map[string]int)
  446. for i, p := range prop.Prop {
  447. if strings.HasPrefix(p.Name, "XXX_") {
  448. // Internal fields should not appear in tags/origNames maps.
  449. // They are handled specially when encoding and decoding.
  450. continue
  451. }
  452. if p.Required {
  453. reqCount++
  454. }
  455. prop.decoderTags.put(p.Tag, i)
  456. prop.decoderOrigNames[p.OrigName] = i
  457. }
  458. prop.reqCount = reqCount
  459. return prop
  460. }
  461. // A global registry of enum types.
  462. // The generated code will register the generated maps by calling RegisterEnum.
  463. var enumValueMaps = make(map[string]map[string]int32)
  464. var enumStringMaps = make(map[string]map[int32]string)
  465. // RegisterEnum is called from the generated code to install the enum descriptor
  466. // maps into the global table to aid parsing text format protocol buffers.
  467. func RegisterEnum(typeName string, unusedNameMap map[int32]string, valueMap map[string]int32) {
  468. if _, ok := enumValueMaps[typeName]; ok {
  469. panic("proto: duplicate enum registered: " + typeName)
  470. }
  471. enumValueMaps[typeName] = valueMap
  472. if _, ok := enumStringMaps[typeName]; ok {
  473. panic("proto: duplicate enum registered: " + typeName)
  474. }
  475. enumStringMaps[typeName] = unusedNameMap
  476. }
  477. // EnumValueMap returns the mapping from names to integers of the
  478. // enum type enumType, or a nil if not found.
  479. func EnumValueMap(enumType string) map[string]int32 {
  480. return enumValueMaps[enumType]
  481. }
  482. // A registry of all linked message types.
  483. // The string is a fully-qualified proto name ("pkg.Message").
  484. var (
  485. protoTypedNils = make(map[string]Message) // a map from proto names to typed nil pointers
  486. protoMapTypes = make(map[string]reflect.Type) // a map from proto names to map types
  487. revProtoTypes = make(map[reflect.Type]string)
  488. )
  489. // RegisterType is called from generated code and maps from the fully qualified
  490. // proto name to the type (pointer to struct) of the protocol buffer.
  491. func RegisterType(x Message, name string) {
  492. if _, ok := protoTypedNils[name]; ok {
  493. // TODO: Some day, make this a panic.
  494. log.Printf("proto: duplicate proto type registered: %s", name)
  495. return
  496. }
  497. t := reflect.TypeOf(x)
  498. if v := reflect.ValueOf(x); v.Kind() == reflect.Ptr && v.Pointer() == 0 {
  499. // Generated code always calls RegisterType with nil x.
  500. // This check is just for extra safety.
  501. protoTypedNils[name] = x
  502. } else {
  503. protoTypedNils[name] = reflect.Zero(t).Interface().(Message)
  504. }
  505. revProtoTypes[t] = name
  506. }
  507. // RegisterMapType is called from generated code and maps from the fully qualified
  508. // proto name to the native map type of the proto map definition.
  509. func RegisterMapType(x interface{}, name string) {
  510. if reflect.TypeOf(x).Kind() != reflect.Map {
  511. panic(fmt.Sprintf("RegisterMapType(%T, %q); want map", x, name))
  512. }
  513. if _, ok := protoMapTypes[name]; ok {
  514. log.Printf("proto: duplicate proto type registered: %s", name)
  515. return
  516. }
  517. t := reflect.TypeOf(x)
  518. protoMapTypes[name] = t
  519. revProtoTypes[t] = name
  520. }
  521. // MessageName returns the fully-qualified proto name for the given message type.
  522. func MessageName(x Message) string {
  523. type xname interface {
  524. XXX_MessageName() string
  525. }
  526. if m, ok := x.(xname); ok {
  527. return m.XXX_MessageName()
  528. }
  529. return revProtoTypes[reflect.TypeOf(x)]
  530. }
  531. // MessageType returns the message type (pointer to struct) for a named message.
  532. // The type is not guaranteed to implement proto.Message if the name refers to a
  533. // map entry.
  534. func MessageType(name string) reflect.Type {
  535. if t, ok := protoTypedNils[name]; ok {
  536. return reflect.TypeOf(t)
  537. }
  538. return protoMapTypes[name]
  539. }
  540. // A registry of all linked proto files.
  541. var (
  542. protoFiles = make(map[string][]byte) // file name => fileDescriptor
  543. )
  544. // RegisterFile is called from generated code and maps from the
  545. // full file name of a .proto file to its compressed FileDescriptorProto.
  546. func RegisterFile(filename string, fileDescriptor []byte) {
  547. protoFiles[filename] = fileDescriptor
  548. }
  549. // FileDescriptor returns the compressed FileDescriptorProto for a .proto file.
  550. func FileDescriptor(filename string) []byte { return protoFiles[filename] }