encode.go

Documentation: google.golang.org/protobuf/proto

     1  // Copyright 2019 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package proto
     6  
     7  import (
     8  	"google.golang.org/protobuf/encoding/protowire"
     9  	"google.golang.org/protobuf/internal/encoding/messageset"
    10  	"google.golang.org/protobuf/internal/order"
    11  	"google.golang.org/protobuf/internal/pragma"
    12  	"google.golang.org/protobuf/reflect/protoreflect"
    13  	"google.golang.org/protobuf/runtime/protoiface"
    14  )
    15  
    16  // MarshalOptions configures the marshaler.
    17  //
    18  // Example usage:
    19  //
    20  //	b, err := MarshalOptions{Deterministic: true}.Marshal(m)
    21  type MarshalOptions struct {
    22  	pragma.NoUnkeyedLiterals
    23  
    24  	// AllowPartial allows messages that have missing required fields to marshal
    25  	// without returning an error. If AllowPartial is false (the default),
    26  	// Marshal will return an error if there are any missing required fields.
    27  	AllowPartial bool
    28  
    29  	// Deterministic controls whether the same message will always be
    30  	// serialized to the same bytes within the same binary.
    31  	//
    32  	// Setting this option guarantees that repeated serialization of
    33  	// the same message will return the same bytes, and that different
    34  	// processes of the same binary (which may be executing on different
    35  	// machines) will serialize equal messages to the same bytes.
    36  	// It has no effect on the resulting size of the encoded message compared
    37  	// to a non-deterministic marshal.
    38  	//
    39  	// Note that the deterministic serialization is NOT canonical across
    40  	// languages. It is not guaranteed to remain stable over time. It is
    41  	// unstable across different builds with schema changes due to unknown
    42  	// fields. Users who need canonical serialization (e.g., persistent
    43  	// storage in a canonical form, fingerprinting, etc.) must define
    44  	// their own canonicalization specification and implement their own
    45  	// serializer rather than relying on this API.
    46  	//
    47  	// If deterministic serialization is requested, map entries will be
    48  	// sorted by keys in lexographical order. This is an implementation
    49  	// detail and subject to change.
    50  	Deterministic bool
    51  
    52  	// UseCachedSize indicates that the result of a previous Size call
    53  	// may be reused.
    54  	//
    55  	// Setting this option asserts that:
    56  	//
    57  	// 1. Size has previously been called on this message with identical
    58  	// options (except for UseCachedSize itself).
    59  	//
    60  	// 2. The message and all its submessages have not changed in any
    61  	// way since the Size call.
    62  	//
    63  	// If either of these invariants is violated,
    64  	// the results are undefined and may include panics or corrupted output.
    65  	//
    66  	// Implementations MAY take this option into account to provide
    67  	// better performance, but there is no guarantee that they will do so.
    68  	// There is absolutely no guarantee that Size followed by Marshal with
    69  	// UseCachedSize set will perform equivalently to Marshal alone.
    70  	UseCachedSize bool
    71  }
    72  
    73  // Marshal returns the wire-format encoding of m.
    74  func Marshal(m Message) ([]byte, error) {
    75  	// Treat nil message interface as an empty message; nothing to output.
    76  	if m == nil {
    77  		return nil, nil
    78  	}
    79  
    80  	out, err := MarshalOptions{}.marshal(nil, m.ProtoReflect())
    81  	if len(out.Buf) == 0 && err == nil {
    82  		out.Buf = emptyBytesForMessage(m)
    83  	}
    84  	return out.Buf, err
    85  }
    86  
    87  // Marshal returns the wire-format encoding of m.
    88  func (o MarshalOptions) Marshal(m Message) ([]byte, error) {
    89  	// Treat nil message interface as an empty message; nothing to output.
    90  	if m == nil {
    91  		return nil, nil
    92  	}
    93  
    94  	out, err := o.marshal(nil, m.ProtoReflect())
    95  	if len(out.Buf) == 0 && err == nil {
    96  		out.Buf = emptyBytesForMessage(m)
    97  	}
    98  	return out.Buf, err
    99  }
   100  
   101  // emptyBytesForMessage returns a nil buffer if and only if m is invalid,
   102  // otherwise it returns a non-nil empty buffer.
   103  //
   104  // This is to assist the edge-case where user-code does the following:
   105  //
   106  //	m1.OptionalBytes, _ = proto.Marshal(m2)
   107  //
   108  // where they expect the proto2 "optional_bytes" field to be populated
   109  // if any only if m2 is a valid message.
   110  func emptyBytesForMessage(m Message) []byte {
   111  	if m == nil || !m.ProtoReflect().IsValid() {
   112  		return nil
   113  	}
   114  	return emptyBuf[:]
   115  }
   116  
   117  // MarshalAppend appends the wire-format encoding of m to b,
   118  // returning the result.
   119  func (o MarshalOptions) MarshalAppend(b []byte, m Message) ([]byte, error) {
   120  	// Treat nil message interface as an empty message; nothing to append.
   121  	if m == nil {
   122  		return b, nil
   123  	}
   124  
   125  	out, err := o.marshal(b, m.ProtoReflect())
   126  	return out.Buf, err
   127  }
   128  
   129  // MarshalState returns the wire-format encoding of a message.
   130  //
   131  // This method permits fine-grained control over the marshaler.
   132  // Most users should use [Marshal] instead.
   133  func (o MarshalOptions) MarshalState(in protoiface.MarshalInput) (protoiface.MarshalOutput, error) {
   134  	return o.marshal(in.Buf, in.Message)
   135  }
   136  
   137  // marshal is a centralized function that all marshal operations go through.
   138  // For profiling purposes, avoid changing the name of this function or
   139  // introducing other code paths for marshal that do not go through this.
   140  func (o MarshalOptions) marshal(b []byte, m protoreflect.Message) (out protoiface.MarshalOutput, err error) {
   141  	allowPartial := o.AllowPartial
   142  	o.AllowPartial = true
   143  	if methods := protoMethods(m); methods != nil && methods.Marshal != nil &&
   144  		!(o.Deterministic && methods.Flags&protoiface.SupportMarshalDeterministic == 0) {
   145  		in := protoiface.MarshalInput{
   146  			Message: m,
   147  			Buf:     b,
   148  		}
   149  		if o.Deterministic {
   150  			in.Flags |= protoiface.MarshalDeterministic
   151  		}
   152  		if o.UseCachedSize {
   153  			in.Flags |= protoiface.MarshalUseCachedSize
   154  		}
   155  		if methods.Size != nil {
   156  			sout := methods.Size(protoiface.SizeInput{
   157  				Message: m,
   158  				Flags:   in.Flags,
   159  			})
   160  			if cap(b) < len(b)+sout.Size {
   161  				in.Buf = make([]byte, len(b), growcap(cap(b), len(b)+sout.Size))
   162  				copy(in.Buf, b)
   163  			}
   164  			in.Flags |= protoiface.MarshalUseCachedSize
   165  		}
   166  		out, err = methods.Marshal(in)
   167  	} else {
   168  		out.Buf, err = o.marshalMessageSlow(b, m)
   169  	}
   170  	if err != nil {
   171  		return out, err
   172  	}
   173  	if allowPartial {
   174  		return out, nil
   175  	}
   176  	return out, checkInitialized(m)
   177  }
   178  
   179  func (o MarshalOptions) marshalMessage(b []byte, m protoreflect.Message) ([]byte, error) {
   180  	out, err := o.marshal(b, m)
   181  	return out.Buf, err
   182  }
   183  
   184  // growcap scales up the capacity of a slice.
   185  //
   186  // Given a slice with a current capacity of oldcap and a desired
   187  // capacity of wantcap, growcap returns a new capacity >= wantcap.
   188  //
   189  // The algorithm is mostly identical to the one used by append as of Go 1.14.
   190  func growcap(oldcap, wantcap int) (newcap int) {
   191  	if wantcap > oldcap*2 {
   192  		newcap = wantcap
   193  	} else if oldcap < 1024 {
   194  		// The Go 1.14 runtime takes this case when len(s) < 1024,
   195  		// not when cap(s) < 1024. The difference doesn't seem
   196  		// significant here.
   197  		newcap = oldcap * 2
   198  	} else {
   199  		newcap = oldcap
   200  		for 0 < newcap && newcap < wantcap {
   201  			newcap += newcap / 4
   202  		}
   203  		if newcap <= 0 {
   204  			newcap = wantcap
   205  		}
   206  	}
   207  	return newcap
   208  }
   209  
   210  func (o MarshalOptions) marshalMessageSlow(b []byte, m protoreflect.Message) ([]byte, error) {
   211  	if messageset.IsMessageSet(m.Descriptor()) {
   212  		return o.marshalMessageSet(b, m)
   213  	}
   214  	fieldOrder := order.AnyFieldOrder
   215  	if o.Deterministic {
   216  		// TODO: This should use a more natural ordering like NumberFieldOrder,
   217  		// but doing so breaks golden tests that make invalid assumption about
   218  		// output stability of this implementation.
   219  		fieldOrder = order.LegacyFieldOrder
   220  	}
   221  	var err error
   222  	order.RangeFields(m, fieldOrder, func(fd protoreflect.FieldDescriptor, v protoreflect.Value) bool {
   223  		b, err = o.marshalField(b, fd, v)
   224  		return err == nil
   225  	})
   226  	if err != nil {
   227  		return b, err
   228  	}
   229  	b = append(b, m.GetUnknown()...)
   230  	return b, nil
   231  }
   232  
   233  func (o MarshalOptions) marshalField(b []byte, fd protoreflect.FieldDescriptor, value protoreflect.Value) ([]byte, error) {
   234  	switch {
   235  	case fd.IsList():
   236  		return o.marshalList(b, fd, value.List())
   237  	case fd.IsMap():
   238  		return o.marshalMap(b, fd, value.Map())
   239  	default:
   240  		b = protowire.AppendTag(b, fd.Number(), wireTypes[fd.Kind()])
   241  		return o.marshalSingular(b, fd, value)
   242  	}
   243  }
   244  
   245  func (o MarshalOptions) marshalList(b []byte, fd protoreflect.FieldDescriptor, list protoreflect.List) ([]byte, error) {
   246  	if fd.IsPacked() && list.Len() > 0 {
   247  		b = protowire.AppendTag(b, fd.Number(), protowire.BytesType)
   248  		b, pos := appendSpeculativeLength(b)
   249  		for i, llen := 0, list.Len(); i < llen; i++ {
   250  			var err error
   251  			b, err = o.marshalSingular(b, fd, list.Get(i))
   252  			if err != nil {
   253  				return b, err
   254  			}
   255  		}
   256  		b = finishSpeculativeLength(b, pos)
   257  		return b, nil
   258  	}
   259  
   260  	kind := fd.Kind()
   261  	for i, llen := 0, list.Len(); i < llen; i++ {
   262  		var err error
   263  		b = protowire.AppendTag(b, fd.Number(), wireTypes[kind])
   264  		b, err = o.marshalSingular(b, fd, list.Get(i))
   265  		if err != nil {
   266  			return b, err
   267  		}
   268  	}
   269  	return b, nil
   270  }
   271  
   272  func (o MarshalOptions) marshalMap(b []byte, fd protoreflect.FieldDescriptor, mapv protoreflect.Map) ([]byte, error) {
   273  	keyf := fd.MapKey()
   274  	valf := fd.MapValue()
   275  	keyOrder := order.AnyKeyOrder
   276  	if o.Deterministic {
   277  		keyOrder = order.GenericKeyOrder
   278  	}
   279  	var err error
   280  	order.RangeEntries(mapv, keyOrder, func(key protoreflect.MapKey, value protoreflect.Value) bool {
   281  		b = protowire.AppendTag(b, fd.Number(), protowire.BytesType)
   282  		var pos int
   283  		b, pos = appendSpeculativeLength(b)
   284  
   285  		b, err = o.marshalField(b, keyf, key.Value())
   286  		if err != nil {
   287  			return false
   288  		}
   289  		b, err = o.marshalField(b, valf, value)
   290  		if err != nil {
   291  			return false
   292  		}
   293  		b = finishSpeculativeLength(b, pos)
   294  		return true
   295  	})
   296  	return b, err
   297  }
   298  
   299  // When encoding length-prefixed fields, we speculatively set aside some number of bytes
   300  // for the length, encode the data, and then encode the length (shifting the data if necessary
   301  // to make room).
   302  const speculativeLength = 1
   303  
   304  func appendSpeculativeLength(b []byte) ([]byte, int) {
   305  	pos := len(b)
   306  	b = append(b, "\x00\x00\x00\x00"[:speculativeLength]...)
   307  	return b, pos
   308  }
   309  
   310  func finishSpeculativeLength(b []byte, pos int) []byte {
   311  	mlen := len(b) - pos - speculativeLength
   312  	msiz := protowire.SizeVarint(uint64(mlen))
   313  	if msiz != speculativeLength {
   314  		for i := 0; i < msiz-speculativeLength; i++ {
   315  			b = append(b, 0)
   316  		}
   317  		copy(b[pos+msiz:], b[pos+speculativeLength:])
   318  		b = b[:pos+msiz+mlen]
   319  	}
   320  	protowire.AppendVarint(b[:pos], uint64(mlen))
   321  	return b
   322  }
   323
View as plain text