protodelim.go 4.6 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160
  1. // Copyright 2022 The Go Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style
  3. // license that can be found in the LICENSE file.
  4. // Package protodelim marshals and unmarshals varint size-delimited messages.
  5. package protodelim
  6. import (
  7. "bufio"
  8. "encoding/binary"
  9. "fmt"
  10. "io"
  11. "google.golang.org/protobuf/encoding/protowire"
  12. "google.golang.org/protobuf/internal/errors"
  13. "google.golang.org/protobuf/proto"
  14. )
  15. // MarshalOptions is a configurable varint size-delimited marshaler.
  16. type MarshalOptions struct{ proto.MarshalOptions }
  17. // MarshalTo writes a varint size-delimited wire-format message to w.
  18. // If w returns an error, MarshalTo returns it unchanged.
  19. func (o MarshalOptions) MarshalTo(w io.Writer, m proto.Message) (int, error) {
  20. msgBytes, err := o.MarshalOptions.Marshal(m)
  21. if err != nil {
  22. return 0, err
  23. }
  24. sizeBytes := protowire.AppendVarint(nil, uint64(len(msgBytes)))
  25. sizeWritten, err := w.Write(sizeBytes)
  26. if err != nil {
  27. return sizeWritten, err
  28. }
  29. msgWritten, err := w.Write(msgBytes)
  30. if err != nil {
  31. return sizeWritten + msgWritten, err
  32. }
  33. return sizeWritten + msgWritten, nil
  34. }
  35. // MarshalTo writes a varint size-delimited wire-format message to w
  36. // with the default options.
  37. //
  38. // See the documentation for [MarshalOptions.MarshalTo].
  39. func MarshalTo(w io.Writer, m proto.Message) (int, error) {
  40. return MarshalOptions{}.MarshalTo(w, m)
  41. }
  42. // UnmarshalOptions is a configurable varint size-delimited unmarshaler.
  43. type UnmarshalOptions struct {
  44. proto.UnmarshalOptions
  45. // MaxSize is the maximum size in wire-format bytes of a single message.
  46. // Unmarshaling a message larger than MaxSize will return an error.
  47. // A zero MaxSize will default to 4 MiB.
  48. // Setting MaxSize to -1 disables the limit.
  49. MaxSize int64
  50. }
  51. const defaultMaxSize = 4 << 20 // 4 MiB, corresponds to the default gRPC max request/response size
  52. // SizeTooLargeError is an error that is returned when the unmarshaler encounters a message size
  53. // that is larger than its configured [UnmarshalOptions.MaxSize].
  54. type SizeTooLargeError struct {
  55. // Size is the varint size of the message encountered
  56. // that was larger than the provided MaxSize.
  57. Size uint64
  58. // MaxSize is the MaxSize limit configured in UnmarshalOptions, which Size exceeded.
  59. MaxSize uint64
  60. }
  61. func (e *SizeTooLargeError) Error() string {
  62. return fmt.Sprintf("message size %d exceeded unmarshaler's maximum configured size %d", e.Size, e.MaxSize)
  63. }
  64. // Reader is the interface expected by [UnmarshalFrom].
  65. // It is implemented by *[bufio.Reader].
  66. type Reader interface {
  67. io.Reader
  68. io.ByteReader
  69. }
  70. // UnmarshalFrom parses and consumes a varint size-delimited wire-format message
  71. // from r.
  72. // The provided message must be mutable (e.g., a non-nil pointer to a message).
  73. //
  74. // The error is [io.EOF] error only if no bytes are read.
  75. // If an EOF happens after reading some but not all the bytes,
  76. // UnmarshalFrom returns a non-io.EOF error.
  77. // In particular if r returns a non-io.EOF error, UnmarshalFrom returns it unchanged,
  78. // and if only a size is read with no subsequent message, [io.ErrUnexpectedEOF] is returned.
  79. func (o UnmarshalOptions) UnmarshalFrom(r Reader, m proto.Message) error {
  80. var sizeArr [binary.MaxVarintLen64]byte
  81. sizeBuf := sizeArr[:0]
  82. for i := range sizeArr {
  83. b, err := r.ReadByte()
  84. if err != nil {
  85. // Immediate EOF is unexpected.
  86. if err == io.EOF && i != 0 {
  87. break
  88. }
  89. return err
  90. }
  91. sizeBuf = append(sizeBuf, b)
  92. if b < 0x80 {
  93. break
  94. }
  95. }
  96. size, n := protowire.ConsumeVarint(sizeBuf)
  97. if n < 0 {
  98. return protowire.ParseError(n)
  99. }
  100. maxSize := o.MaxSize
  101. if maxSize == 0 {
  102. maxSize = defaultMaxSize
  103. }
  104. if maxSize != -1 && size > uint64(maxSize) {
  105. return errors.Wrap(&SizeTooLargeError{Size: size, MaxSize: uint64(maxSize)}, "")
  106. }
  107. var b []byte
  108. var err error
  109. if br, ok := r.(*bufio.Reader); ok {
  110. // Use the []byte from the bufio.Reader instead of having to allocate one.
  111. // This reduces CPU usage and allocated bytes.
  112. b, err = br.Peek(int(size))
  113. if err == nil {
  114. defer br.Discard(int(size))
  115. } else {
  116. b = nil
  117. }
  118. }
  119. if b == nil {
  120. b = make([]byte, size)
  121. _, err = io.ReadFull(r, b)
  122. }
  123. if err == io.EOF {
  124. return io.ErrUnexpectedEOF
  125. }
  126. if err != nil {
  127. return err
  128. }
  129. if err := o.Unmarshal(b, m); err != nil {
  130. return err
  131. }
  132. return nil
  133. }
  134. // UnmarshalFrom parses and consumes a varint size-delimited wire-format message
  135. // from r with the default options.
  136. // The provided message must be mutable (e.g., a non-nil pointer to a message).
  137. //
  138. // See the documentation for [UnmarshalOptions.UnmarshalFrom].
  139. func UnmarshalFrom(r Reader, m proto.Message) error {
  140. return UnmarshalOptions{}.UnmarshalFrom(r, m)
  141. }