text_parse.go 29 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901
  1. // Copyright 2014 The Prometheus Authors
  2. // Licensed under the Apache License, Version 2.0 (the "License");
  3. // you may not use this file except in compliance with the License.
  4. // You may obtain a copy of the License at
  5. //
  6. // http://www.apache.org/licenses/LICENSE-2.0
  7. //
  8. // Unless required by applicable law or agreed to in writing, software
  9. // distributed under the License is distributed on an "AS IS" BASIS,
  10. // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11. // See the License for the specific language governing permissions and
  12. // limitations under the License.
  13. package expfmt
  14. import (
  15. "bufio"
  16. "bytes"
  17. "errors"
  18. "fmt"
  19. "io"
  20. "math"
  21. "strconv"
  22. "strings"
  23. "unicode/utf8"
  24. dto "github.com/prometheus/client_model/go"
  25. "google.golang.org/protobuf/proto"
  26. "github.com/prometheus/common/model"
  27. )
  28. // A stateFn is a function that represents a state in a state machine. By
  29. // executing it, the state is progressed to the next state. The stateFn returns
  30. // another stateFn, which represents the new state. The end state is represented
  31. // by nil.
  32. type stateFn func() stateFn
  33. // ParseError signals errors while parsing the simple and flat text-based
  34. // exchange format.
  35. type ParseError struct {
  36. Line int
  37. Msg string
  38. }
  39. // Error implements the error interface.
  40. func (e ParseError) Error() string {
  41. return fmt.Sprintf("text format parsing error in line %d: %s", e.Line, e.Msg)
  42. }
  43. // TextParser is used to parse the simple and flat text-based exchange format. Its
  44. // zero value is ready to use.
  45. type TextParser struct {
  46. metricFamiliesByName map[string]*dto.MetricFamily
  47. buf *bufio.Reader // Where the parsed input is read through.
  48. err error // Most recent error.
  49. lineCount int // Tracks the line count for error messages.
  50. currentByte byte // The most recent byte read.
  51. currentToken bytes.Buffer // Re-used each time a token has to be gathered from multiple bytes.
  52. currentMF *dto.MetricFamily
  53. currentMetric *dto.Metric
  54. currentLabelPair *dto.LabelPair
  55. currentLabelPairs []*dto.LabelPair // Temporarily stores label pairs while parsing a metric line.
  56. // The remaining member variables are only used for summaries/histograms.
  57. currentLabels map[string]string // All labels including '__name__' but excluding 'quantile'/'le'
  58. // Summary specific.
  59. summaries map[uint64]*dto.Metric // Key is created with LabelsToSignature.
  60. currentQuantile float64
  61. // Histogram specific.
  62. histograms map[uint64]*dto.Metric // Key is created with LabelsToSignature.
  63. currentBucket float64
  64. // These tell us if the currently processed line ends on '_count' or
  65. // '_sum' respectively and belong to a summary/histogram, representing the sample
  66. // count and sum of that summary/histogram.
  67. currentIsSummaryCount, currentIsSummarySum bool
  68. currentIsHistogramCount, currentIsHistogramSum bool
  69. // These indicate if the metric name from the current line being parsed is inside
  70. // braces and if that metric name was found respectively.
  71. currentMetricIsInsideBraces, currentMetricInsideBracesIsPresent bool
  72. }
  73. // TextToMetricFamilies reads 'in' as the simple and flat text-based exchange
  74. // format and creates MetricFamily proto messages. It returns the MetricFamily
  75. // proto messages in a map where the metric names are the keys, along with any
  76. // error encountered.
  77. //
  78. // If the input contains duplicate metrics (i.e. lines with the same metric name
  79. // and exactly the same label set), the resulting MetricFamily will contain
  80. // duplicate Metric proto messages. Similar is true for duplicate label
  81. // names. Checks for duplicates have to be performed separately, if required.
  82. // Also note that neither the metrics within each MetricFamily are sorted nor
  83. // the label pairs within each Metric. Sorting is not required for the most
  84. // frequent use of this method, which is sample ingestion in the Prometheus
  85. // server. However, for presentation purposes, you might want to sort the
  86. // metrics, and in some cases, you must sort the labels, e.g. for consumption by
  87. // the metric family injection hook of the Prometheus registry.
  88. //
  89. // Summaries and histograms are rather special beasts. You would probably not
  90. // use them in the simple text format anyway. This method can deal with
  91. // summaries and histograms if they are presented in exactly the way the
  92. // text.Create function creates them.
  93. //
  94. // This method must not be called concurrently. If you want to parse different
  95. // input concurrently, instantiate a separate Parser for each goroutine.
  96. func (p *TextParser) TextToMetricFamilies(in io.Reader) (map[string]*dto.MetricFamily, error) {
  97. p.reset(in)
  98. for nextState := p.startOfLine; nextState != nil; nextState = nextState() {
  99. // Magic happens here...
  100. }
  101. // Get rid of empty metric families.
  102. for k, mf := range p.metricFamiliesByName {
  103. if len(mf.GetMetric()) == 0 {
  104. delete(p.metricFamiliesByName, k)
  105. }
  106. }
  107. // If p.err is io.EOF now, we have run into a premature end of the input
  108. // stream. Turn this error into something nicer and more
  109. // meaningful. (io.EOF is often used as a signal for the legitimate end
  110. // of an input stream.)
  111. if p.err != nil && errors.Is(p.err, io.EOF) {
  112. p.parseError("unexpected end of input stream")
  113. }
  114. return p.metricFamiliesByName, p.err
  115. }
  116. func (p *TextParser) reset(in io.Reader) {
  117. p.metricFamiliesByName = map[string]*dto.MetricFamily{}
  118. if p.buf == nil {
  119. p.buf = bufio.NewReader(in)
  120. } else {
  121. p.buf.Reset(in)
  122. }
  123. p.err = nil
  124. p.lineCount = 0
  125. if p.summaries == nil || len(p.summaries) > 0 {
  126. p.summaries = map[uint64]*dto.Metric{}
  127. }
  128. if p.histograms == nil || len(p.histograms) > 0 {
  129. p.histograms = map[uint64]*dto.Metric{}
  130. }
  131. p.currentQuantile = math.NaN()
  132. p.currentBucket = math.NaN()
  133. p.currentMF = nil
  134. }
  135. // startOfLine represents the state where the next byte read from p.buf is the
  136. // start of a line (or whitespace leading up to it).
  137. func (p *TextParser) startOfLine() stateFn {
  138. p.lineCount++
  139. p.currentMetricIsInsideBraces = false
  140. p.currentMetricInsideBracesIsPresent = false
  141. if p.skipBlankTab(); p.err != nil {
  142. // This is the only place that we expect to see io.EOF,
  143. // which is not an error but the signal that we are done.
  144. // Any other error that happens to align with the start of
  145. // a line is still an error.
  146. if errors.Is(p.err, io.EOF) {
  147. p.err = nil
  148. }
  149. return nil
  150. }
  151. switch p.currentByte {
  152. case '#':
  153. return p.startComment
  154. case '\n':
  155. return p.startOfLine // Empty line, start the next one.
  156. case '{':
  157. p.currentMetricIsInsideBraces = true
  158. return p.readingLabels
  159. }
  160. return p.readingMetricName
  161. }
  162. // startComment represents the state where the next byte read from p.buf is the
  163. // start of a comment (or whitespace leading up to it).
  164. func (p *TextParser) startComment() stateFn {
  165. if p.skipBlankTab(); p.err != nil {
  166. return nil // Unexpected end of input.
  167. }
  168. if p.currentByte == '\n' {
  169. return p.startOfLine
  170. }
  171. if p.readTokenUntilWhitespace(); p.err != nil {
  172. return nil // Unexpected end of input.
  173. }
  174. // If we have hit the end of line already, there is nothing left
  175. // to do. This is not considered a syntax error.
  176. if p.currentByte == '\n' {
  177. return p.startOfLine
  178. }
  179. keyword := p.currentToken.String()
  180. if keyword != "HELP" && keyword != "TYPE" {
  181. // Generic comment, ignore by fast forwarding to end of line.
  182. for p.currentByte != '\n' {
  183. if p.currentByte, p.err = p.buf.ReadByte(); p.err != nil {
  184. return nil // Unexpected end of input.
  185. }
  186. }
  187. return p.startOfLine
  188. }
  189. // There is something. Next has to be a metric name.
  190. if p.skipBlankTab(); p.err != nil {
  191. return nil // Unexpected end of input.
  192. }
  193. if p.readTokenAsMetricName(); p.err != nil {
  194. return nil // Unexpected end of input.
  195. }
  196. if p.currentByte == '\n' {
  197. // At the end of the line already.
  198. // Again, this is not considered a syntax error.
  199. return p.startOfLine
  200. }
  201. if !isBlankOrTab(p.currentByte) {
  202. p.parseError("invalid metric name in comment")
  203. return nil
  204. }
  205. p.setOrCreateCurrentMF()
  206. if p.skipBlankTab(); p.err != nil {
  207. return nil // Unexpected end of input.
  208. }
  209. if p.currentByte == '\n' {
  210. // At the end of the line already.
  211. // Again, this is not considered a syntax error.
  212. return p.startOfLine
  213. }
  214. switch keyword {
  215. case "HELP":
  216. return p.readingHelp
  217. case "TYPE":
  218. return p.readingType
  219. }
  220. panic(fmt.Sprintf("code error: unexpected keyword %q", keyword))
  221. }
  222. // readingMetricName represents the state where the last byte read (now in
  223. // p.currentByte) is the first byte of a metric name.
  224. func (p *TextParser) readingMetricName() stateFn {
  225. if p.readTokenAsMetricName(); p.err != nil {
  226. return nil
  227. }
  228. if p.currentToken.Len() == 0 {
  229. p.parseError("invalid metric name")
  230. return nil
  231. }
  232. p.setOrCreateCurrentMF()
  233. // Now is the time to fix the type if it hasn't happened yet.
  234. if p.currentMF.Type == nil {
  235. p.currentMF.Type = dto.MetricType_UNTYPED.Enum()
  236. }
  237. p.currentMetric = &dto.Metric{}
  238. // Do not append the newly created currentMetric to
  239. // currentMF.Metric right now. First wait if this is a summary,
  240. // and the metric exists already, which we can only know after
  241. // having read all the labels.
  242. if p.skipBlankTabIfCurrentBlankTab(); p.err != nil {
  243. return nil // Unexpected end of input.
  244. }
  245. return p.readingLabels
  246. }
  247. // readingLabels represents the state where the last byte read (now in
  248. // p.currentByte) is either the first byte of the label set (i.e. a '{'), or the
  249. // first byte of the value (otherwise).
  250. func (p *TextParser) readingLabels() stateFn {
  251. // Summaries/histograms are special. We have to reset the
  252. // currentLabels map, currentQuantile and currentBucket before starting to
  253. // read labels.
  254. if p.currentMF.GetType() == dto.MetricType_SUMMARY || p.currentMF.GetType() == dto.MetricType_HISTOGRAM {
  255. p.currentLabels = map[string]string{}
  256. p.currentLabels[string(model.MetricNameLabel)] = p.currentMF.GetName()
  257. p.currentQuantile = math.NaN()
  258. p.currentBucket = math.NaN()
  259. }
  260. if p.currentByte != '{' {
  261. return p.readingValue
  262. }
  263. return p.startLabelName
  264. }
  265. // startLabelName represents the state where the next byte read from p.buf is
  266. // the start of a label name (or whitespace leading up to it).
  267. func (p *TextParser) startLabelName() stateFn {
  268. if p.skipBlankTab(); p.err != nil {
  269. return nil // Unexpected end of input.
  270. }
  271. if p.currentByte == '}' {
  272. p.currentMetric.Label = append(p.currentMetric.Label, p.currentLabelPairs...)
  273. p.currentLabelPairs = nil
  274. if p.skipBlankTab(); p.err != nil {
  275. return nil // Unexpected end of input.
  276. }
  277. return p.readingValue
  278. }
  279. if p.readTokenAsLabelName(); p.err != nil {
  280. return nil // Unexpected end of input.
  281. }
  282. if p.currentToken.Len() == 0 {
  283. p.parseError(fmt.Sprintf("invalid label name for metric %q", p.currentMF.GetName()))
  284. return nil
  285. }
  286. if p.skipBlankTabIfCurrentBlankTab(); p.err != nil {
  287. return nil // Unexpected end of input.
  288. }
  289. if p.currentByte != '=' {
  290. if p.currentMetricIsInsideBraces {
  291. if p.currentMetricInsideBracesIsPresent {
  292. p.parseError(fmt.Sprintf("multiple metric names for metric %q", p.currentMF.GetName()))
  293. return nil
  294. }
  295. switch p.currentByte {
  296. case ',':
  297. p.setOrCreateCurrentMF()
  298. if p.currentMF.Type == nil {
  299. p.currentMF.Type = dto.MetricType_UNTYPED.Enum()
  300. }
  301. p.currentMetric = &dto.Metric{}
  302. p.currentMetricInsideBracesIsPresent = true
  303. return p.startLabelName
  304. case '}':
  305. p.setOrCreateCurrentMF()
  306. if p.currentMF.Type == nil {
  307. p.currentMF.Type = dto.MetricType_UNTYPED.Enum()
  308. }
  309. p.currentMetric = &dto.Metric{}
  310. p.currentMetric.Label = append(p.currentMetric.Label, p.currentLabelPairs...)
  311. p.currentLabelPairs = nil
  312. if p.skipBlankTab(); p.err != nil {
  313. return nil // Unexpected end of input.
  314. }
  315. return p.readingValue
  316. default:
  317. p.parseError(fmt.Sprintf("unexpected end of metric name %q", p.currentByte))
  318. return nil
  319. }
  320. }
  321. p.parseError(fmt.Sprintf("expected '=' after label name, found %q", p.currentByte))
  322. p.currentLabelPairs = nil
  323. return nil
  324. }
  325. p.currentLabelPair = &dto.LabelPair{Name: proto.String(p.currentToken.String())}
  326. if p.currentLabelPair.GetName() == string(model.MetricNameLabel) {
  327. p.parseError(fmt.Sprintf("label name %q is reserved", model.MetricNameLabel))
  328. return nil
  329. }
  330. // Special summary/histogram treatment. Don't add 'quantile' and 'le'
  331. // labels to 'real' labels.
  332. if !(p.currentMF.GetType() == dto.MetricType_SUMMARY && p.currentLabelPair.GetName() == model.QuantileLabel) &&
  333. !(p.currentMF.GetType() == dto.MetricType_HISTOGRAM && p.currentLabelPair.GetName() == model.BucketLabel) {
  334. p.currentLabelPairs = append(p.currentLabelPairs, p.currentLabelPair)
  335. }
  336. // Check for duplicate label names.
  337. labels := make(map[string]struct{})
  338. for _, l := range p.currentLabelPairs {
  339. lName := l.GetName()
  340. if _, exists := labels[lName]; !exists {
  341. labels[lName] = struct{}{}
  342. } else {
  343. p.parseError(fmt.Sprintf("duplicate label names for metric %q", p.currentMF.GetName()))
  344. p.currentLabelPairs = nil
  345. return nil
  346. }
  347. }
  348. return p.startLabelValue
  349. }
  350. // startLabelValue represents the state where the next byte read from p.buf is
  351. // the start of a (quoted) label value (or whitespace leading up to it).
  352. func (p *TextParser) startLabelValue() stateFn {
  353. if p.skipBlankTab(); p.err != nil {
  354. return nil // Unexpected end of input.
  355. }
  356. if p.currentByte != '"' {
  357. p.parseError(fmt.Sprintf("expected '\"' at start of label value, found %q", p.currentByte))
  358. return nil
  359. }
  360. if p.readTokenAsLabelValue(); p.err != nil {
  361. return nil
  362. }
  363. if !model.LabelValue(p.currentToken.String()).IsValid() {
  364. p.parseError(fmt.Sprintf("invalid label value %q", p.currentToken.String()))
  365. return nil
  366. }
  367. p.currentLabelPair.Value = proto.String(p.currentToken.String())
  368. // Special treatment of summaries:
  369. // - Quantile labels are special, will result in dto.Quantile later.
  370. // - Other labels have to be added to currentLabels for signature calculation.
  371. if p.currentMF.GetType() == dto.MetricType_SUMMARY {
  372. if p.currentLabelPair.GetName() == model.QuantileLabel {
  373. if p.currentQuantile, p.err = parseFloat(p.currentLabelPair.GetValue()); p.err != nil {
  374. // Create a more helpful error message.
  375. p.parseError(fmt.Sprintf("expected float as value for 'quantile' label, got %q", p.currentLabelPair.GetValue()))
  376. p.currentLabelPairs = nil
  377. return nil
  378. }
  379. } else {
  380. p.currentLabels[p.currentLabelPair.GetName()] = p.currentLabelPair.GetValue()
  381. }
  382. }
  383. // Similar special treatment of histograms.
  384. if p.currentMF.GetType() == dto.MetricType_HISTOGRAM {
  385. if p.currentLabelPair.GetName() == model.BucketLabel {
  386. if p.currentBucket, p.err = parseFloat(p.currentLabelPair.GetValue()); p.err != nil {
  387. // Create a more helpful error message.
  388. p.parseError(fmt.Sprintf("expected float as value for 'le' label, got %q", p.currentLabelPair.GetValue()))
  389. return nil
  390. }
  391. } else {
  392. p.currentLabels[p.currentLabelPair.GetName()] = p.currentLabelPair.GetValue()
  393. }
  394. }
  395. if p.skipBlankTab(); p.err != nil {
  396. return nil // Unexpected end of input.
  397. }
  398. switch p.currentByte {
  399. case ',':
  400. return p.startLabelName
  401. case '}':
  402. if p.currentMF == nil {
  403. p.parseError("invalid metric name")
  404. return nil
  405. }
  406. p.currentMetric.Label = append(p.currentMetric.Label, p.currentLabelPairs...)
  407. p.currentLabelPairs = nil
  408. if p.skipBlankTab(); p.err != nil {
  409. return nil // Unexpected end of input.
  410. }
  411. return p.readingValue
  412. default:
  413. p.parseError(fmt.Sprintf("unexpected end of label value %q", p.currentLabelPair.GetValue()))
  414. p.currentLabelPairs = nil
  415. return nil
  416. }
  417. }
  418. // readingValue represents the state where the last byte read (now in
  419. // p.currentByte) is the first byte of the sample value (i.e. a float).
  420. func (p *TextParser) readingValue() stateFn {
  421. // When we are here, we have read all the labels, so for the
  422. // special case of a summary/histogram, we can finally find out
  423. // if the metric already exists.
  424. if p.currentMF.GetType() == dto.MetricType_SUMMARY {
  425. signature := model.LabelsToSignature(p.currentLabels)
  426. if summary := p.summaries[signature]; summary != nil {
  427. p.currentMetric = summary
  428. } else {
  429. p.summaries[signature] = p.currentMetric
  430. p.currentMF.Metric = append(p.currentMF.Metric, p.currentMetric)
  431. }
  432. } else if p.currentMF.GetType() == dto.MetricType_HISTOGRAM {
  433. signature := model.LabelsToSignature(p.currentLabels)
  434. if histogram := p.histograms[signature]; histogram != nil {
  435. p.currentMetric = histogram
  436. } else {
  437. p.histograms[signature] = p.currentMetric
  438. p.currentMF.Metric = append(p.currentMF.Metric, p.currentMetric)
  439. }
  440. } else {
  441. p.currentMF.Metric = append(p.currentMF.Metric, p.currentMetric)
  442. }
  443. if p.readTokenUntilWhitespace(); p.err != nil {
  444. return nil // Unexpected end of input.
  445. }
  446. value, err := parseFloat(p.currentToken.String())
  447. if err != nil {
  448. // Create a more helpful error message.
  449. p.parseError(fmt.Sprintf("expected float as value, got %q", p.currentToken.String()))
  450. return nil
  451. }
  452. switch p.currentMF.GetType() {
  453. case dto.MetricType_COUNTER:
  454. p.currentMetric.Counter = &dto.Counter{Value: proto.Float64(value)}
  455. case dto.MetricType_GAUGE:
  456. p.currentMetric.Gauge = &dto.Gauge{Value: proto.Float64(value)}
  457. case dto.MetricType_UNTYPED:
  458. p.currentMetric.Untyped = &dto.Untyped{Value: proto.Float64(value)}
  459. case dto.MetricType_SUMMARY:
  460. // *sigh*
  461. if p.currentMetric.Summary == nil {
  462. p.currentMetric.Summary = &dto.Summary{}
  463. }
  464. switch {
  465. case p.currentIsSummaryCount:
  466. p.currentMetric.Summary.SampleCount = proto.Uint64(uint64(value))
  467. case p.currentIsSummarySum:
  468. p.currentMetric.Summary.SampleSum = proto.Float64(value)
  469. case !math.IsNaN(p.currentQuantile):
  470. p.currentMetric.Summary.Quantile = append(
  471. p.currentMetric.Summary.Quantile,
  472. &dto.Quantile{
  473. Quantile: proto.Float64(p.currentQuantile),
  474. Value: proto.Float64(value),
  475. },
  476. )
  477. }
  478. case dto.MetricType_HISTOGRAM:
  479. // *sigh*
  480. if p.currentMetric.Histogram == nil {
  481. p.currentMetric.Histogram = &dto.Histogram{}
  482. }
  483. switch {
  484. case p.currentIsHistogramCount:
  485. p.currentMetric.Histogram.SampleCount = proto.Uint64(uint64(value))
  486. case p.currentIsHistogramSum:
  487. p.currentMetric.Histogram.SampleSum = proto.Float64(value)
  488. case !math.IsNaN(p.currentBucket):
  489. p.currentMetric.Histogram.Bucket = append(
  490. p.currentMetric.Histogram.Bucket,
  491. &dto.Bucket{
  492. UpperBound: proto.Float64(p.currentBucket),
  493. CumulativeCount: proto.Uint64(uint64(value)),
  494. },
  495. )
  496. }
  497. default:
  498. p.err = fmt.Errorf("unexpected type for metric name %q", p.currentMF.GetName())
  499. }
  500. if p.currentByte == '\n' {
  501. return p.startOfLine
  502. }
  503. return p.startTimestamp
  504. }
  505. // startTimestamp represents the state where the next byte read from p.buf is
  506. // the start of the timestamp (or whitespace leading up to it).
  507. func (p *TextParser) startTimestamp() stateFn {
  508. if p.skipBlankTab(); p.err != nil {
  509. return nil // Unexpected end of input.
  510. }
  511. if p.readTokenUntilWhitespace(); p.err != nil {
  512. return nil // Unexpected end of input.
  513. }
  514. timestamp, err := strconv.ParseInt(p.currentToken.String(), 10, 64)
  515. if err != nil {
  516. // Create a more helpful error message.
  517. p.parseError(fmt.Sprintf("expected integer as timestamp, got %q", p.currentToken.String()))
  518. return nil
  519. }
  520. p.currentMetric.TimestampMs = proto.Int64(timestamp)
  521. if p.readTokenUntilNewline(false); p.err != nil {
  522. return nil // Unexpected end of input.
  523. }
  524. if p.currentToken.Len() > 0 {
  525. p.parseError(fmt.Sprintf("spurious string after timestamp: %q", p.currentToken.String()))
  526. return nil
  527. }
  528. return p.startOfLine
  529. }
  530. // readingHelp represents the state where the last byte read (now in
  531. // p.currentByte) is the first byte of the docstring after 'HELP'.
  532. func (p *TextParser) readingHelp() stateFn {
  533. if p.currentMF.Help != nil {
  534. p.parseError(fmt.Sprintf("second HELP line for metric name %q", p.currentMF.GetName()))
  535. return nil
  536. }
  537. // Rest of line is the docstring.
  538. if p.readTokenUntilNewline(true); p.err != nil {
  539. return nil // Unexpected end of input.
  540. }
  541. p.currentMF.Help = proto.String(p.currentToken.String())
  542. return p.startOfLine
  543. }
  544. // readingType represents the state where the last byte read (now in
  545. // p.currentByte) is the first byte of the type hint after 'HELP'.
  546. func (p *TextParser) readingType() stateFn {
  547. if p.currentMF.Type != nil {
  548. p.parseError(fmt.Sprintf("second TYPE line for metric name %q, or TYPE reported after samples", p.currentMF.GetName()))
  549. return nil
  550. }
  551. // Rest of line is the type.
  552. if p.readTokenUntilNewline(false); p.err != nil {
  553. return nil // Unexpected end of input.
  554. }
  555. metricType, ok := dto.MetricType_value[strings.ToUpper(p.currentToken.String())]
  556. if !ok {
  557. p.parseError(fmt.Sprintf("unknown metric type %q", p.currentToken.String()))
  558. return nil
  559. }
  560. p.currentMF.Type = dto.MetricType(metricType).Enum()
  561. return p.startOfLine
  562. }
  563. // parseError sets p.err to a ParseError at the current line with the given
  564. // message.
  565. func (p *TextParser) parseError(msg string) {
  566. p.err = ParseError{
  567. Line: p.lineCount,
  568. Msg: msg,
  569. }
  570. }
  571. // skipBlankTab reads (and discards) bytes from p.buf until it encounters a byte
  572. // that is neither ' ' nor '\t'. That byte is left in p.currentByte.
  573. func (p *TextParser) skipBlankTab() {
  574. for {
  575. if p.currentByte, p.err = p.buf.ReadByte(); p.err != nil || !isBlankOrTab(p.currentByte) {
  576. return
  577. }
  578. }
  579. }
  580. // skipBlankTabIfCurrentBlankTab works exactly as skipBlankTab but doesn't do
  581. // anything if p.currentByte is neither ' ' nor '\t'.
  582. func (p *TextParser) skipBlankTabIfCurrentBlankTab() {
  583. if isBlankOrTab(p.currentByte) {
  584. p.skipBlankTab()
  585. }
  586. }
  587. // readTokenUntilWhitespace copies bytes from p.buf into p.currentToken. The
  588. // first byte considered is the byte already read (now in p.currentByte). The
  589. // first whitespace byte encountered is still copied into p.currentByte, but not
  590. // into p.currentToken.
  591. func (p *TextParser) readTokenUntilWhitespace() {
  592. p.currentToken.Reset()
  593. for p.err == nil && !isBlankOrTab(p.currentByte) && p.currentByte != '\n' {
  594. p.currentToken.WriteByte(p.currentByte)
  595. p.currentByte, p.err = p.buf.ReadByte()
  596. }
  597. }
  598. // readTokenUntilNewline copies bytes from p.buf into p.currentToken. The first
  599. // byte considered is the byte already read (now in p.currentByte). The first
  600. // newline byte encountered is still copied into p.currentByte, but not into
  601. // p.currentToken. If recognizeEscapeSequence is true, two escape sequences are
  602. // recognized: '\\' translates into '\', and '\n' into a line-feed character.
  603. // All other escape sequences are invalid and cause an error.
  604. func (p *TextParser) readTokenUntilNewline(recognizeEscapeSequence bool) {
  605. p.currentToken.Reset()
  606. escaped := false
  607. for p.err == nil {
  608. if recognizeEscapeSequence && escaped {
  609. switch p.currentByte {
  610. case '\\':
  611. p.currentToken.WriteByte(p.currentByte)
  612. case 'n':
  613. p.currentToken.WriteByte('\n')
  614. case '"':
  615. p.currentToken.WriteByte('"')
  616. default:
  617. p.parseError(fmt.Sprintf("invalid escape sequence '\\%c'", p.currentByte))
  618. return
  619. }
  620. escaped = false
  621. } else {
  622. switch p.currentByte {
  623. case '\n':
  624. return
  625. case '\\':
  626. escaped = true
  627. default:
  628. p.currentToken.WriteByte(p.currentByte)
  629. }
  630. }
  631. p.currentByte, p.err = p.buf.ReadByte()
  632. }
  633. }
  634. // readTokenAsMetricName copies a metric name from p.buf into p.currentToken.
  635. // The first byte considered is the byte already read (now in p.currentByte).
  636. // The first byte not part of a metric name is still copied into p.currentByte,
  637. // but not into p.currentToken.
  638. func (p *TextParser) readTokenAsMetricName() {
  639. p.currentToken.Reset()
  640. // A UTF-8 metric name must be quoted and may have escaped characters.
  641. quoted := false
  642. escaped := false
  643. if !isValidMetricNameStart(p.currentByte) {
  644. return
  645. }
  646. for p.err == nil {
  647. if escaped {
  648. switch p.currentByte {
  649. case '\\':
  650. p.currentToken.WriteByte(p.currentByte)
  651. case 'n':
  652. p.currentToken.WriteByte('\n')
  653. case '"':
  654. p.currentToken.WriteByte('"')
  655. default:
  656. p.parseError(fmt.Sprintf("invalid escape sequence '\\%c'", p.currentByte))
  657. return
  658. }
  659. escaped = false
  660. } else {
  661. switch p.currentByte {
  662. case '"':
  663. quoted = !quoted
  664. if !quoted {
  665. p.currentByte, p.err = p.buf.ReadByte()
  666. return
  667. }
  668. case '\n':
  669. p.parseError(fmt.Sprintf("metric name %q contains unescaped new-line", p.currentToken.String()))
  670. return
  671. case '\\':
  672. escaped = true
  673. default:
  674. p.currentToken.WriteByte(p.currentByte)
  675. }
  676. }
  677. p.currentByte, p.err = p.buf.ReadByte()
  678. if !isValidMetricNameContinuation(p.currentByte, quoted) || (!quoted && p.currentByte == ' ') {
  679. return
  680. }
  681. }
  682. }
  683. // readTokenAsLabelName copies a label name from p.buf into p.currentToken.
  684. // The first byte considered is the byte already read (now in p.currentByte).
  685. // The first byte not part of a label name is still copied into p.currentByte,
  686. // but not into p.currentToken.
  687. func (p *TextParser) readTokenAsLabelName() {
  688. p.currentToken.Reset()
  689. // A UTF-8 label name must be quoted and may have escaped characters.
  690. quoted := false
  691. escaped := false
  692. if !isValidLabelNameStart(p.currentByte) {
  693. return
  694. }
  695. for p.err == nil {
  696. if escaped {
  697. switch p.currentByte {
  698. case '\\':
  699. p.currentToken.WriteByte(p.currentByte)
  700. case 'n':
  701. p.currentToken.WriteByte('\n')
  702. case '"':
  703. p.currentToken.WriteByte('"')
  704. default:
  705. p.parseError(fmt.Sprintf("invalid escape sequence '\\%c'", p.currentByte))
  706. return
  707. }
  708. escaped = false
  709. } else {
  710. switch p.currentByte {
  711. case '"':
  712. quoted = !quoted
  713. if !quoted {
  714. p.currentByte, p.err = p.buf.ReadByte()
  715. return
  716. }
  717. case '\n':
  718. p.parseError(fmt.Sprintf("label name %q contains unescaped new-line", p.currentToken.String()))
  719. return
  720. case '\\':
  721. escaped = true
  722. default:
  723. p.currentToken.WriteByte(p.currentByte)
  724. }
  725. }
  726. p.currentByte, p.err = p.buf.ReadByte()
  727. if !isValidLabelNameContinuation(p.currentByte, quoted) || (!quoted && p.currentByte == '=') {
  728. return
  729. }
  730. }
  731. }
  732. // readTokenAsLabelValue copies a label value from p.buf into p.currentToken.
  733. // In contrast to the other 'readTokenAs...' functions, which start with the
  734. // last read byte in p.currentByte, this method ignores p.currentByte and starts
  735. // with reading a new byte from p.buf. The first byte not part of a label value
  736. // is still copied into p.currentByte, but not into p.currentToken.
  737. func (p *TextParser) readTokenAsLabelValue() {
  738. p.currentToken.Reset()
  739. escaped := false
  740. for {
  741. if p.currentByte, p.err = p.buf.ReadByte(); p.err != nil {
  742. return
  743. }
  744. if escaped {
  745. switch p.currentByte {
  746. case '"', '\\':
  747. p.currentToken.WriteByte(p.currentByte)
  748. case 'n':
  749. p.currentToken.WriteByte('\n')
  750. default:
  751. p.parseError(fmt.Sprintf("invalid escape sequence '\\%c'", p.currentByte))
  752. p.currentLabelPairs = nil
  753. return
  754. }
  755. escaped = false
  756. continue
  757. }
  758. switch p.currentByte {
  759. case '"':
  760. return
  761. case '\n':
  762. p.parseError(fmt.Sprintf("label value %q contains unescaped new-line", p.currentToken.String()))
  763. return
  764. case '\\':
  765. escaped = true
  766. default:
  767. p.currentToken.WriteByte(p.currentByte)
  768. }
  769. }
  770. }
  771. func (p *TextParser) setOrCreateCurrentMF() {
  772. p.currentIsSummaryCount = false
  773. p.currentIsSummarySum = false
  774. p.currentIsHistogramCount = false
  775. p.currentIsHistogramSum = false
  776. name := p.currentToken.String()
  777. if p.currentMF = p.metricFamiliesByName[name]; p.currentMF != nil {
  778. return
  779. }
  780. // Try out if this is a _sum or _count for a summary/histogram.
  781. summaryName := summaryMetricName(name)
  782. if p.currentMF = p.metricFamiliesByName[summaryName]; p.currentMF != nil {
  783. if p.currentMF.GetType() == dto.MetricType_SUMMARY {
  784. if isCount(name) {
  785. p.currentIsSummaryCount = true
  786. }
  787. if isSum(name) {
  788. p.currentIsSummarySum = true
  789. }
  790. return
  791. }
  792. }
  793. histogramName := histogramMetricName(name)
  794. if p.currentMF = p.metricFamiliesByName[histogramName]; p.currentMF != nil {
  795. if p.currentMF.GetType() == dto.MetricType_HISTOGRAM {
  796. if isCount(name) {
  797. p.currentIsHistogramCount = true
  798. }
  799. if isSum(name) {
  800. p.currentIsHistogramSum = true
  801. }
  802. return
  803. }
  804. }
  805. p.currentMF = &dto.MetricFamily{Name: proto.String(name)}
  806. p.metricFamiliesByName[name] = p.currentMF
  807. }
  808. func isValidLabelNameStart(b byte) bool {
  809. return (b >= 'a' && b <= 'z') || (b >= 'A' && b <= 'Z') || b == '_' || b == '"'
  810. }
  811. func isValidLabelNameContinuation(b byte, quoted bool) bool {
  812. return isValidLabelNameStart(b) || (b >= '0' && b <= '9') || (quoted && utf8.ValidString(string(b)))
  813. }
  814. func isValidMetricNameStart(b byte) bool {
  815. return isValidLabelNameStart(b) || b == ':'
  816. }
  817. func isValidMetricNameContinuation(b byte, quoted bool) bool {
  818. return isValidLabelNameContinuation(b, quoted) || b == ':'
  819. }
  820. func isBlankOrTab(b byte) bool {
  821. return b == ' ' || b == '\t'
  822. }
  823. func isCount(name string) bool {
  824. return len(name) > 6 && name[len(name)-6:] == "_count"
  825. }
  826. func isSum(name string) bool {
  827. return len(name) > 4 && name[len(name)-4:] == "_sum"
  828. }
  829. func isBucket(name string) bool {
  830. return len(name) > 7 && name[len(name)-7:] == "_bucket"
  831. }
  832. func summaryMetricName(name string) string {
  833. switch {
  834. case isCount(name):
  835. return name[:len(name)-6]
  836. case isSum(name):
  837. return name[:len(name)-4]
  838. default:
  839. return name
  840. }
  841. }
  842. func histogramMetricName(name string) string {
  843. switch {
  844. case isCount(name):
  845. return name[:len(name)-6]
  846. case isSum(name):
  847. return name[:len(name)-4]
  848. case isBucket(name):
  849. return name[:len(name)-7]
  850. default:
  851. return name
  852. }
  853. }
  854. func parseFloat(s string) (float64, error) {
  855. if strings.ContainsAny(s, "pP_") {
  856. return 0, errors.New("unsupported character in float")
  857. }
  858. return strconv.ParseFloat(s, 64)
  859. }