encode.go 9.9 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319
  1. // Copyright 2019 The Go Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style
  3. // license that can be found in the LICENSE file.
  4. package proto
  5. import (
  6. "google.golang.org/protobuf/encoding/protowire"
  7. "google.golang.org/protobuf/internal/encoding/messageset"
  8. "google.golang.org/protobuf/internal/order"
  9. "google.golang.org/protobuf/internal/pragma"
  10. "google.golang.org/protobuf/reflect/protoreflect"
  11. "google.golang.org/protobuf/runtime/protoiface"
  12. )
  13. // MarshalOptions configures the marshaler.
  14. //
  15. // Example usage:
  16. // b, err := MarshalOptions{Deterministic: true}.Marshal(m)
  17. type MarshalOptions struct {
  18. pragma.NoUnkeyedLiterals
  19. // AllowPartial allows messages that have missing required fields to marshal
  20. // without returning an error. If AllowPartial is false (the default),
  21. // Marshal will return an error if there are any missing required fields.
  22. AllowPartial bool
  23. // Deterministic controls whether the same message will always be
  24. // serialized to the same bytes within the same binary.
  25. //
  26. // Setting this option guarantees that repeated serialization of
  27. // the same message will return the same bytes, and that different
  28. // processes of the same binary (which may be executing on different
  29. // machines) will serialize equal messages to the same bytes.
  30. // It has no effect on the resulting size of the encoded message compared
  31. // to a non-deterministic marshal.
  32. //
  33. // Note that the deterministic serialization is NOT canonical across
  34. // languages. It is not guaranteed to remain stable over time. It is
  35. // unstable across different builds with schema changes due to unknown
  36. // fields. Users who need canonical serialization (e.g., persistent
  37. // storage in a canonical form, fingerprinting, etc.) must define
  38. // their own canonicalization specification and implement their own
  39. // serializer rather than relying on this API.
  40. //
  41. // If deterministic serialization is requested, map entries will be
  42. // sorted by keys in lexographical order. This is an implementation
  43. // detail and subject to change.
  44. Deterministic bool
  45. // UseCachedSize indicates that the result of a previous Size call
  46. // may be reused.
  47. //
  48. // Setting this option asserts that:
  49. //
  50. // 1. Size has previously been called on this message with identical
  51. // options (except for UseCachedSize itself).
  52. //
  53. // 2. The message and all its submessages have not changed in any
  54. // way since the Size call.
  55. //
  56. // If either of these invariants is violated,
  57. // the results are undefined and may include panics or corrupted output.
  58. //
  59. // Implementations MAY take this option into account to provide
  60. // better performance, but there is no guarantee that they will do so.
  61. // There is absolutely no guarantee that Size followed by Marshal with
  62. // UseCachedSize set will perform equivalently to Marshal alone.
  63. UseCachedSize bool
  64. }
  65. // Marshal returns the wire-format encoding of m.
  66. func Marshal(m Message) ([]byte, error) {
  67. // Treat nil message interface as an empty message; nothing to output.
  68. if m == nil {
  69. return nil, nil
  70. }
  71. out, err := MarshalOptions{}.marshal(nil, m.ProtoReflect())
  72. if len(out.Buf) == 0 && err == nil {
  73. out.Buf = emptyBytesForMessage(m)
  74. }
  75. return out.Buf, err
  76. }
  77. // Marshal returns the wire-format encoding of m.
  78. func (o MarshalOptions) Marshal(m Message) ([]byte, error) {
  79. // Treat nil message interface as an empty message; nothing to output.
  80. if m == nil {
  81. return nil, nil
  82. }
  83. out, err := o.marshal(nil, m.ProtoReflect())
  84. if len(out.Buf) == 0 && err == nil {
  85. out.Buf = emptyBytesForMessage(m)
  86. }
  87. return out.Buf, err
  88. }
  89. // emptyBytesForMessage returns a nil buffer if and only if m is invalid,
  90. // otherwise it returns a non-nil empty buffer.
  91. //
  92. // This is to assist the edge-case where user-code does the following:
  93. // m1.OptionalBytes, _ = proto.Marshal(m2)
  94. // where they expect the proto2 "optional_bytes" field to be populated
  95. // if any only if m2 is a valid message.
  96. func emptyBytesForMessage(m Message) []byte {
  97. if m == nil || !m.ProtoReflect().IsValid() {
  98. return nil
  99. }
  100. return emptyBuf[:]
  101. }
  102. // MarshalAppend appends the wire-format encoding of m to b,
  103. // returning the result.
  104. func (o MarshalOptions) MarshalAppend(b []byte, m Message) ([]byte, error) {
  105. // Treat nil message interface as an empty message; nothing to append.
  106. if m == nil {
  107. return b, nil
  108. }
  109. out, err := o.marshal(b, m.ProtoReflect())
  110. return out.Buf, err
  111. }
  112. // MarshalState returns the wire-format encoding of a message.
  113. //
  114. // This method permits fine-grained control over the marshaler.
  115. // Most users should use Marshal instead.
  116. func (o MarshalOptions) MarshalState(in protoiface.MarshalInput) (protoiface.MarshalOutput, error) {
  117. return o.marshal(in.Buf, in.Message)
  118. }
  119. // marshal is a centralized function that all marshal operations go through.
  120. // For profiling purposes, avoid changing the name of this function or
  121. // introducing other code paths for marshal that do not go through this.
  122. func (o MarshalOptions) marshal(b []byte, m protoreflect.Message) (out protoiface.MarshalOutput, err error) {
  123. allowPartial := o.AllowPartial
  124. o.AllowPartial = true
  125. if methods := protoMethods(m); methods != nil && methods.Marshal != nil &&
  126. !(o.Deterministic && methods.Flags&protoiface.SupportMarshalDeterministic == 0) {
  127. in := protoiface.MarshalInput{
  128. Message: m,
  129. Buf: b,
  130. }
  131. if o.Deterministic {
  132. in.Flags |= protoiface.MarshalDeterministic
  133. }
  134. if o.UseCachedSize {
  135. in.Flags |= protoiface.MarshalUseCachedSize
  136. }
  137. if methods.Size != nil {
  138. sout := methods.Size(protoiface.SizeInput{
  139. Message: m,
  140. Flags: in.Flags,
  141. })
  142. if cap(b) < len(b)+sout.Size {
  143. in.Buf = make([]byte, len(b), growcap(cap(b), len(b)+sout.Size))
  144. copy(in.Buf, b)
  145. }
  146. in.Flags |= protoiface.MarshalUseCachedSize
  147. }
  148. out, err = methods.Marshal(in)
  149. } else {
  150. out.Buf, err = o.marshalMessageSlow(b, m)
  151. }
  152. if err != nil {
  153. return out, err
  154. }
  155. if allowPartial {
  156. return out, nil
  157. }
  158. return out, checkInitialized(m)
  159. }
  160. func (o MarshalOptions) marshalMessage(b []byte, m protoreflect.Message) ([]byte, error) {
  161. out, err := o.marshal(b, m)
  162. return out.Buf, err
  163. }
  164. // growcap scales up the capacity of a slice.
  165. //
  166. // Given a slice with a current capacity of oldcap and a desired
  167. // capacity of wantcap, growcap returns a new capacity >= wantcap.
  168. //
  169. // The algorithm is mostly identical to the one used by append as of Go 1.14.
  170. func growcap(oldcap, wantcap int) (newcap int) {
  171. if wantcap > oldcap*2 {
  172. newcap = wantcap
  173. } else if oldcap < 1024 {
  174. // The Go 1.14 runtime takes this case when len(s) < 1024,
  175. // not when cap(s) < 1024. The difference doesn't seem
  176. // significant here.
  177. newcap = oldcap * 2
  178. } else {
  179. newcap = oldcap
  180. for 0 < newcap && newcap < wantcap {
  181. newcap += newcap / 4
  182. }
  183. if newcap <= 0 {
  184. newcap = wantcap
  185. }
  186. }
  187. return newcap
  188. }
  189. func (o MarshalOptions) marshalMessageSlow(b []byte, m protoreflect.Message) ([]byte, error) {
  190. if messageset.IsMessageSet(m.Descriptor()) {
  191. return o.marshalMessageSet(b, m)
  192. }
  193. fieldOrder := order.AnyFieldOrder
  194. if o.Deterministic {
  195. // TODO: This should use a more natural ordering like NumberFieldOrder,
  196. // but doing so breaks golden tests that make invalid assumption about
  197. // output stability of this implementation.
  198. fieldOrder = order.LegacyFieldOrder
  199. }
  200. var err error
  201. order.RangeFields(m, fieldOrder, func(fd protoreflect.FieldDescriptor, v protoreflect.Value) bool {
  202. b, err = o.marshalField(b, fd, v)
  203. return err == nil
  204. })
  205. if err != nil {
  206. return b, err
  207. }
  208. b = append(b, m.GetUnknown()...)
  209. return b, nil
  210. }
  211. func (o MarshalOptions) marshalField(b []byte, fd protoreflect.FieldDescriptor, value protoreflect.Value) ([]byte, error) {
  212. switch {
  213. case fd.IsList():
  214. return o.marshalList(b, fd, value.List())
  215. case fd.IsMap():
  216. return o.marshalMap(b, fd, value.Map())
  217. default:
  218. b = protowire.AppendTag(b, fd.Number(), wireTypes[fd.Kind()])
  219. return o.marshalSingular(b, fd, value)
  220. }
  221. }
  222. func (o MarshalOptions) marshalList(b []byte, fd protoreflect.FieldDescriptor, list protoreflect.List) ([]byte, error) {
  223. if fd.IsPacked() && list.Len() > 0 {
  224. b = protowire.AppendTag(b, fd.Number(), protowire.BytesType)
  225. b, pos := appendSpeculativeLength(b)
  226. for i, llen := 0, list.Len(); i < llen; i++ {
  227. var err error
  228. b, err = o.marshalSingular(b, fd, list.Get(i))
  229. if err != nil {
  230. return b, err
  231. }
  232. }
  233. b = finishSpeculativeLength(b, pos)
  234. return b, nil
  235. }
  236. kind := fd.Kind()
  237. for i, llen := 0, list.Len(); i < llen; i++ {
  238. var err error
  239. b = protowire.AppendTag(b, fd.Number(), wireTypes[kind])
  240. b, err = o.marshalSingular(b, fd, list.Get(i))
  241. if err != nil {
  242. return b, err
  243. }
  244. }
  245. return b, nil
  246. }
  247. func (o MarshalOptions) marshalMap(b []byte, fd protoreflect.FieldDescriptor, mapv protoreflect.Map) ([]byte, error) {
  248. keyf := fd.MapKey()
  249. valf := fd.MapValue()
  250. keyOrder := order.AnyKeyOrder
  251. if o.Deterministic {
  252. keyOrder = order.GenericKeyOrder
  253. }
  254. var err error
  255. order.RangeEntries(mapv, keyOrder, func(key protoreflect.MapKey, value protoreflect.Value) bool {
  256. b = protowire.AppendTag(b, fd.Number(), protowire.BytesType)
  257. var pos int
  258. b, pos = appendSpeculativeLength(b)
  259. b, err = o.marshalField(b, keyf, key.Value())
  260. if err != nil {
  261. return false
  262. }
  263. b, err = o.marshalField(b, valf, value)
  264. if err != nil {
  265. return false
  266. }
  267. b = finishSpeculativeLength(b, pos)
  268. return true
  269. })
  270. return b, err
  271. }
  272. // When encoding length-prefixed fields, we speculatively set aside some number of bytes
  273. // for the length, encode the data, and then encode the length (shifting the data if necessary
  274. // to make room).
  275. const speculativeLength = 1
  276. func appendSpeculativeLength(b []byte) ([]byte, int) {
  277. pos := len(b)
  278. b = append(b, "\x00\x00\x00\x00"[:speculativeLength]...)
  279. return b, pos
  280. }
  281. func finishSpeculativeLength(b []byte, pos int) []byte {
  282. mlen := len(b) - pos - speculativeLength
  283. msiz := protowire.SizeVarint(uint64(mlen))
  284. if msiz != speculativeLength {
  285. for i := 0; i < msiz-speculativeLength; i++ {
  286. b = append(b, 0)
  287. }
  288. copy(b[pos+msiz:], b[pos+speculativeLength:])
  289. b = b[:pos+msiz+mlen]
  290. }
  291. protowire.AppendVarint(b[:pos], uint64(mlen))
  292. return b
  293. }