V8 API Reference, 7.2.502.16 (for Deno 0.2.4)
scanner.h
1 // Copyright 2011 the V8 project authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4 
5 // Features shared by parsing and pre-parsing scanners.
6 
7 #ifndef V8_PARSING_SCANNER_H_
8 #define V8_PARSING_SCANNER_H_
9 
10 #include <algorithm>
11 
12 #include "src/allocation.h"
13 #include "src/base/logging.h"
14 #include "src/char-predicates.h"
15 #include "src/globals.h"
16 #include "src/message-template.h"
17 #include "src/parsing/token.h"
18 #include "src/pointer-with-payload.h"
19 #include "src/unicode-decoder.h"
20 #include "src/unicode.h"
21 
22 namespace v8 {
23 namespace internal {
24 
25 class AstRawString;
26 class AstValueFactory;
27 class ExternalOneByteString;
28 class ExternalTwoByteString;
29 class ParserRecorder;
30 class RuntimeCallStats;
31 class Zone;
32 
33 // ---------------------------------------------------------------------
34 // Buffered stream of UTF-16 code units, using an internal UTF-16 buffer.
35 // A code unit is a 16 bit value representing either a 16 bit code point
36 // or one part of a surrogate pair that make a single 21 bit code point.
38  public:
39  static const uc32 kEndOfInput = -1;
40 
41  virtual ~Utf16CharacterStream() = default;
42 
43  V8_INLINE void set_parser_error() {
44  buffer_cursor_ = buffer_end_;
45  has_parser_error_ = true;
46  }
47  V8_INLINE void reset_parser_error_flag() { has_parser_error_ = false; }
48  V8_INLINE bool has_parser_error() const { return has_parser_error_; }
49 
50  inline uc32 Peek() {
51  if (V8_LIKELY(buffer_cursor_ < buffer_end_)) {
52  return static_cast<uc32>(*buffer_cursor_);
53  } else if (ReadBlockChecked()) {
54  return static_cast<uc32>(*buffer_cursor_);
55  } else {
56  return kEndOfInput;
57  }
58  }
59 
60  // Returns and advances past the next UTF-16 code unit in the input
61  // stream. If there are no more code units it returns kEndOfInput.
62  inline uc32 Advance() {
63  uc32 result = Peek();
64  buffer_cursor_++;
65  return result;
66  }
67 
68  // Returns and advances past the next UTF-16 code unit in the input stream
69  // that meets the checks requirement. If there are no more code units it
70  // returns kEndOfInput.
71  template <typename FunctionType>
72  V8_INLINE uc32 AdvanceUntil(FunctionType check) {
73  while (true) {
74  auto next_cursor_pos =
75  std::find_if(buffer_cursor_, buffer_end_, [&check](uint16_t raw_c0_) {
76  uc32 c0_ = static_cast<uc32>(raw_c0_);
77  return check(c0_);
78  });
79 
80  if (next_cursor_pos == buffer_end_) {
81  buffer_cursor_ = buffer_end_;
82  if (!ReadBlockChecked()) {
83  buffer_cursor_++;
84  return kEndOfInput;
85  }
86  } else {
87  buffer_cursor_ = next_cursor_pos + 1;
88  return static_cast<uc32>(*next_cursor_pos);
89  }
90  }
91  }
92 
93  // Go back one by one character in the input stream.
94  // This undoes the most recent Advance().
95  inline void Back() {
96  // The common case - if the previous character is within
97  // buffer_start_ .. buffer_end_ will be handles locally.
98  // Otherwise, a new block is requested.
99  if (V8_LIKELY(buffer_cursor_ > buffer_start_)) {
100  buffer_cursor_--;
101  } else {
102  ReadBlockAt(pos() - 1);
103  }
104  }
105 
106  inline size_t pos() const {
107  return buffer_pos_ + (buffer_cursor_ - buffer_start_);
108  }
109 
110  inline void Seek(size_t pos) {
111  if (V8_LIKELY(pos >= buffer_pos_ &&
112  pos < (buffer_pos_ + (buffer_end_ - buffer_start_)))) {
113  buffer_cursor_ = buffer_start_ + (pos - buffer_pos_);
114  } else {
115  ReadBlockAt(pos);
116  }
117  }
118 
119  // Returns true if the stream could access the V8 heap after construction.
120  bool can_be_cloned_for_parallel_access() const {
121  return can_be_cloned() && !can_access_heap();
122  }
123 
124  // Returns true if the stream can be cloned with Clone.
125  // TODO(rmcilroy): Remove this once ChunkedStreams can be cloned.
126  virtual bool can_be_cloned() const = 0;
127 
128  // Clones the character stream to enable another independent scanner to access
129  // the same underlying stream.
130  virtual std::unique_ptr<Utf16CharacterStream> Clone() const = 0;
131 
132  // Returns true if the stream could access the V8 heap after construction.
133  virtual bool can_access_heap() const = 0;
134 
135  RuntimeCallStats* runtime_call_stats() const { return runtime_call_stats_; }
136  void set_runtime_call_stats(RuntimeCallStats* runtime_call_stats) {
137  runtime_call_stats_ = runtime_call_stats;
138  }
139 
140  protected:
141  Utf16CharacterStream(const uint16_t* buffer_start,
142  const uint16_t* buffer_cursor,
143  const uint16_t* buffer_end, size_t buffer_pos)
144  : buffer_start_(buffer_start),
145  buffer_cursor_(buffer_cursor),
146  buffer_end_(buffer_end),
147  buffer_pos_(buffer_pos) {}
148  Utf16CharacterStream() : Utf16CharacterStream(nullptr, nullptr, nullptr, 0) {}
149 
150  bool ReadBlockChecked() {
151  size_t position = pos();
152  USE(position);
153  bool success = !has_parser_error() && ReadBlock();
154 
155  // Post-conditions: 1, We should always be at the right position.
156  // 2, Cursor should be inside the buffer.
157  // 3, We should have more characters available iff success.
158  DCHECK_EQ(pos(), position);
159  DCHECK_LE(buffer_cursor_, buffer_end_);
160  DCHECK_LE(buffer_start_, buffer_cursor_);
161  DCHECK_EQ(success, buffer_cursor_ < buffer_end_);
162  return success;
163  }
164 
165  void ReadBlockAt(size_t new_pos) {
166  // The callers of this method (Back/Back2/Seek) should handle the easy
167  // case (seeking within the current buffer), and we should only get here
168  // if we actually require new data.
169  // (This is really an efficiency check, not a correctness invariant.)
170  DCHECK(new_pos < buffer_pos_ ||
171  new_pos >= buffer_pos_ + (buffer_end_ - buffer_start_));
172 
173  // Change pos() to point to new_pos.
174  buffer_pos_ = new_pos;
175  buffer_cursor_ = buffer_start_;
176  DCHECK_EQ(pos(), new_pos);
177  ReadBlockChecked();
178  }
179 
180  // Read more data, and update buffer_*_ to point to it.
181  // Returns true if more data was available.
182  //
183  // ReadBlock() may modify any of the buffer_*_ members, but must sure that
184  // the result of pos() remains unaffected.
185  //
186  // Examples:
187  // - a stream could either fill a separate buffer. Then buffer_start_ and
188  // buffer_cursor_ would point to the beginning of the buffer, and
189  // buffer_pos would be the old pos().
190  // - a stream with existing buffer chunks would set buffer_start_ and
191  // buffer_end_ to cover the full chunk, and then buffer_cursor_ would
192  // point into the middle of the buffer, while buffer_pos_ would describe
193  // the start of the buffer.
194  virtual bool ReadBlock() = 0;
195 
196  const uint16_t* buffer_start_;
197  const uint16_t* buffer_cursor_;
198  const uint16_t* buffer_end_;
199  size_t buffer_pos_;
200  RuntimeCallStats* runtime_call_stats_;
201  bool has_parser_error_ = false;
202 };
203 
204 // ----------------------------------------------------------------------------
205 // JavaScript Scanner.
206 
207 class Scanner {
208  public:
209  // Scoped helper for a re-settable bookmark.
211  public:
212  explicit BookmarkScope(Scanner* scanner)
213  : scanner_(scanner),
214  bookmark_(kNoBookmark),
215  had_parser_error_(scanner->has_parser_error()) {
216  DCHECK_NOT_NULL(scanner_);
217  }
218  ~BookmarkScope() = default;
219 
220  void Set();
221  void Apply();
222  bool HasBeenSet() const;
223  bool HasBeenApplied() const;
224 
225  private:
226  static const size_t kNoBookmark;
227  static const size_t kBookmarkWasApplied;
228  static const size_t kBookmarkAtFirstPos;
229 
230  Scanner* scanner_;
231  size_t bookmark_;
232  bool had_parser_error_;
233 
234  DISALLOW_COPY_AND_ASSIGN(BookmarkScope);
235  };
236 
237  // Sets the Scanner into an error state to stop further scanning and terminate
238  // the parsing by only returning ILLEGAL tokens after that.
239  V8_INLINE void set_parser_error() {
240  if (!has_parser_error()) {
241  c0_ = kEndOfInput;
242  source_->set_parser_error();
243  for (TokenDesc& desc : token_storage_) desc.token = Token::ILLEGAL;
244  }
245  }
246  V8_INLINE void reset_parser_error_flag() {
247  source_->reset_parser_error_flag();
248  }
249  V8_INLINE bool has_parser_error() const {
250  return source_->has_parser_error();
251  }
252 
253  // Representation of an interval of source positions.
254  struct Location {
255  Location(int b, int e) : beg_pos(b), end_pos(e) { }
256  Location() : beg_pos(0), end_pos(0) { }
257 
258  int length() const { return end_pos - beg_pos; }
259  bool IsValid() const { return beg_pos >= 0 && end_pos >= beg_pos; }
260 
261  static Location invalid() { return Location(-1, -1); }
262 
263  int beg_pos;
264  int end_pos;
265  };
266 
267  // -1 is outside of the range of any real source code.
268  static const int kNoOctalLocation = -1;
269  static const uc32 kEndOfInput = Utf16CharacterStream::kEndOfInput;
270 
271  explicit Scanner(Utf16CharacterStream* source, bool is_module);
272 
273  void Initialize();
274 
275  // Returns the next token and advances input.
276  Token::Value Next();
277  // Returns the token following peek()
278  Token::Value PeekAhead();
279  // Returns the current token again.
280  Token::Value current_token() const { return current().token; }
281 
282  // Returns the location information for the current token
283  // (the token last returned by Next()).
284  const Location& location() const { return current().location; }
285 
286  // This error is specifically an invalid hex or unicode escape sequence.
287  bool has_error() const { return scanner_error_ != MessageTemplate::kNone; }
288  MessageTemplate error() const { return scanner_error_; }
289  const Location& error_location() const { return scanner_error_location_; }
290 
291  bool has_invalid_template_escape() const {
292  return current().invalid_template_escape_message != MessageTemplate::kNone;
293  }
294  MessageTemplate invalid_template_escape_message() const {
295  DCHECK(has_invalid_template_escape());
296  return current().invalid_template_escape_message;
297  }
298 
299  void clear_invalid_template_escape_message() {
300  DCHECK(has_invalid_template_escape());
301  current_->invalid_template_escape_message = MessageTemplate::kNone;
302  }
303 
304  Location invalid_template_escape_location() const {
305  DCHECK(has_invalid_template_escape());
306  return current().invalid_template_escape_location;
307  }
308 
309  // Similar functions for the upcoming token.
310 
311  // One token look-ahead (past the token returned by Next()).
312  Token::Value peek() const { return next().token; }
313 
314  const Location& peek_location() const { return next().location; }
315 
316  bool literal_contains_escapes() const {
317  return LiteralContainsEscapes(current());
318  }
319 
320  const AstRawString* CurrentSymbol(AstValueFactory* ast_value_factory) const;
321 
322  const AstRawString* NextSymbol(AstValueFactory* ast_value_factory) const;
323  const AstRawString* CurrentRawSymbol(
324  AstValueFactory* ast_value_factory) const;
325 
326  double DoubleValue();
327 
328  const char* CurrentLiteralAsCString(Zone* zone) const;
329 
330  inline bool CurrentMatches(Token::Value token) const {
331  DCHECK(Token::IsKeyword(token));
332  return current().token == token;
333  }
334 
335  template <size_t N>
336  bool NextLiteralEquals(const char (&s)[N]) {
337  DCHECK_EQ(Token::STRING, peek());
338  // The length of the token is used to make sure the literal equals without
339  // taking escape sequences (e.g., "use \x73trict") or line continuations
340  // (e.g., "use \(newline) strict") into account.
341  if (!is_next_literal_one_byte()) return false;
342  if (peek_location().length() != N + 1) return false;
343 
344  Vector<const uint8_t> next = next_literal_one_byte_string();
345  const char* chars = reinterpret_cast<const char*>(next.start());
346  return next.length() == N - 1 && strncmp(s, chars, N - 1) == 0;
347  }
348 
349  // Returns the location of the last seen octal literal.
350  Location octal_position() const { return octal_pos_; }
351  void clear_octal_position() {
352  octal_pos_ = Location::invalid();
353  octal_message_ = MessageTemplate::kNone;
354  }
355  MessageTemplate octal_message() const { return octal_message_; }
356 
357  // Returns the value of the last smi that was scanned.
358  uint32_t smi_value() const { return current().smi_value_; }
359 
360  // Seek forward to the given position. This operation does not
361  // work in general, for instance when there are pushed back
362  // characters, but works for seeking forward until simple delimiter
363  // tokens, which is what it is used for.
364  void SeekForward(int pos);
365 
366  // Returns true if there was a line terminator before the peek'ed token,
367  // possibly inside a multi-line comment.
368  bool HasLineTerminatorBeforeNext() const {
369  return next().after_line_terminator;
370  }
371 
372  bool HasLineTerminatorAfterNext() {
373  Token::Value ensure_next_next = PeekAhead();
374  USE(ensure_next_next);
375  return next_next().after_line_terminator;
376  }
377 
378  // Scans the input as a regular expression pattern, next token must be /(=).
379  // Returns true if a pattern is scanned.
380  bool ScanRegExpPattern();
381  // Scans the input as regular expression flags. Returns the flags on success.
382  Maybe<RegExp::Flags> ScanRegExpFlags();
383 
384  // Scans the input as a template literal
385  Token::Value ScanTemplateContinuation() {
386  DCHECK_EQ(next().token, Token::RBRACE);
387  DCHECK_EQ(source_pos() - 1, next().location.beg_pos);
388  return ScanTemplateSpan();
389  }
390 
391  Handle<String> SourceUrl(Isolate* isolate) const;
392  Handle<String> SourceMappingUrl(Isolate* isolate) const;
393 
394  bool FoundHtmlComment() const { return found_html_comment_; }
395 
396  bool allow_harmony_private_fields() const {
397  return allow_harmony_private_fields_;
398  }
399  void set_allow_harmony_private_fields(bool allow) {
400  allow_harmony_private_fields_ = allow;
401  }
402  bool allow_harmony_numeric_separator() const {
403  return allow_harmony_numeric_separator_;
404  }
405  void set_allow_harmony_numeric_separator(bool allow) {
406  allow_harmony_numeric_separator_ = allow;
407  }
408 
409  const Utf16CharacterStream* stream() const { return source_; }
410 
411  private:
412  // Scoped helper for saving & restoring scanner error state.
413  // This is used for tagged template literals, in which normally forbidden
414  // escape sequences are allowed.
415  class ErrorState;
416 
417  // LiteralBuffer - Collector of chars of literals.
418  class LiteralBuffer {
419  public:
420  LiteralBuffer() : backing_store_(), position_(0), is_one_byte_(true) {}
421 
422  ~LiteralBuffer() { backing_store_.Dispose(); }
423 
424  V8_INLINE void AddChar(char code_unit) {
425  DCHECK(IsValidAscii(code_unit));
426  AddOneByteChar(static_cast<byte>(code_unit));
427  }
428 
429  V8_INLINE void AddChar(uc32 code_unit) {
430  if (is_one_byte()) {
431  if (code_unit <= static_cast<uc32>(unibrow::Latin1::kMaxChar)) {
432  AddOneByteChar(static_cast<byte>(code_unit));
433  return;
434  }
435  ConvertToTwoByte();
436  }
437  AddTwoByteChar(code_unit);
438  }
439 
440  bool is_one_byte() const { return is_one_byte_; }
441 
442  bool Equals(Vector<const char> keyword) const {
443  return is_one_byte() && keyword.length() == position_ &&
444  (memcmp(keyword.start(), backing_store_.start(), position_) == 0);
445  }
446 
447  Vector<const uint16_t> two_byte_literal() const {
448  DCHECK(!is_one_byte());
449  DCHECK_EQ(position_ & 0x1, 0);
450  return Vector<const uint16_t>(
451  reinterpret_cast<const uint16_t*>(backing_store_.start()),
452  position_ >> 1);
453  }
454 
455  Vector<const uint8_t> one_byte_literal() const {
456  DCHECK(is_one_byte());
457  return Vector<const uint8_t>(
458  reinterpret_cast<const uint8_t*>(backing_store_.start()), position_);
459  }
460 
461  int length() const { return is_one_byte() ? position_ : (position_ >> 1); }
462 
463  void Start() {
464  position_ = 0;
465  is_one_byte_ = true;
466  }
467 
468  Handle<String> Internalize(Isolate* isolate) const;
469 
470  private:
471  static const int kInitialCapacity = 16;
472  static const int kGrowthFactory = 4;
473  static const int kMinConversionSlack = 256;
474  static const int kMaxGrowth = 1 * MB;
475 
476  inline bool IsValidAscii(char code_unit) {
477  // Control characters and printable characters span the range of
478  // valid ASCII characters (0-127). Chars are unsigned on some
479  // platforms which causes compiler warnings if the validity check
480  // tests the lower bound >= 0 as it's always true.
481  return iscntrl(code_unit) || isprint(code_unit);
482  }
483 
484  V8_INLINE void AddOneByteChar(byte one_byte_char) {
485  DCHECK(is_one_byte());
486  if (position_ >= backing_store_.length()) ExpandBuffer();
487  backing_store_[position_] = one_byte_char;
488  position_ += kOneByteSize;
489  }
490 
491  void AddTwoByteChar(uc32 code_unit);
492  int NewCapacity(int min_capacity);
493  void ExpandBuffer();
494  void ConvertToTwoByte();
495 
496  Vector<byte> backing_store_;
497  int position_;
498 
499  bool is_one_byte_;
500 
501  DISALLOW_COPY_AND_ASSIGN(LiteralBuffer);
502  };
503 
504  // The current and look-ahead token.
505  struct TokenDesc {
506  Location location = {0, 0};
507  LiteralBuffer literal_chars;
508  LiteralBuffer raw_literal_chars;
509  Token::Value token = Token::UNINITIALIZED;
510  MessageTemplate invalid_template_escape_message = MessageTemplate::kNone;
511  Location invalid_template_escape_location;
512  uint32_t smi_value_ = 0;
513  bool after_line_terminator = false;
514 
515 #ifdef DEBUG
516  bool CanAccessLiteral() const {
517  return token == Token::PRIVATE_NAME || token == Token::ILLEGAL ||
518  token == Token::UNINITIALIZED || token == Token::REGEXP_LITERAL ||
519  token == Token::ESCAPED_KEYWORD ||
520  IsInRange(token, Token::NUMBER, Token::STRING) ||
521  (Token::IsAnyIdentifier(token) && !Token::IsKeyword(token)) ||
522  IsInRange(token, Token::TEMPLATE_SPAN, Token::TEMPLATE_TAIL);
523  }
524  bool CanAccessRawLiteral() const {
525  return token == Token::ILLEGAL || token == Token::UNINITIALIZED ||
526  IsInRange(token, Token::TEMPLATE_SPAN, Token::TEMPLATE_TAIL);
527  }
528 #endif // DEBUG
529  };
530 
531  enum NumberKind {
532  BINARY,
533  OCTAL,
534  IMPLICIT_OCTAL,
535  HEX,
536  DECIMAL,
537  DECIMAL_WITH_LEADING_ZERO
538  };
539 
540  static const int kCharacterLookaheadBufferSize = 1;
541  static const int kMaxAscii = 127;
542 
543  // Scans octal escape sequence. Also accepts "\0" decimal escape sequence.
544  template <bool capture_raw>
545  uc32 ScanOctalEscape(uc32 c, int length);
546 
547  // Call this after setting source_ to the input.
548  void Init() {
549  // Set c0_ (one character ahead)
550  STATIC_ASSERT(kCharacterLookaheadBufferSize == 1);
551  Advance();
552 
553  current_ = &token_storage_[0];
554  next_ = &token_storage_[1];
555  next_next_ = &token_storage_[2];
556 
557  found_html_comment_ = false;
558  scanner_error_ = MessageTemplate::kNone;
559  }
560 
561  void ReportScannerError(const Location& location, MessageTemplate error) {
562  if (has_error()) return;
563  scanner_error_ = error;
564  scanner_error_location_ = location;
565  }
566 
567  void ReportScannerError(int pos, MessageTemplate error) {
568  if (has_error()) return;
569  scanner_error_ = error;
570  scanner_error_location_ = Location(pos, pos + 1);
571  }
572 
573  // Seek to the next_ token at the given position.
574  void SeekNext(size_t position);
575 
576  V8_INLINE void AddLiteralChar(uc32 c) { next().literal_chars.AddChar(c); }
577 
578  V8_INLINE void AddLiteralChar(char c) { next().literal_chars.AddChar(c); }
579 
580  V8_INLINE void AddRawLiteralChar(uc32 c) {
581  next().raw_literal_chars.AddChar(c);
582  }
583 
584  V8_INLINE void AddLiteralCharAdvance() {
585  AddLiteralChar(c0_);
586  Advance();
587  }
588 
589  // Low-level scanning support.
590  template <bool capture_raw = false>
591  void Advance() {
592  if (capture_raw) {
593  AddRawLiteralChar(c0_);
594  }
595  c0_ = source_->Advance();
596  }
597 
598  template <typename FunctionType>
599  V8_INLINE void AdvanceUntil(FunctionType check) {
600  c0_ = source_->AdvanceUntil(check);
601  }
602 
603  bool CombineSurrogatePair() {
604  DCHECK(!unibrow::Utf16::IsLeadSurrogate(kEndOfInput));
605  if (unibrow::Utf16::IsLeadSurrogate(c0_)) {
606  uc32 c1 = source_->Advance();
607  DCHECK(!unibrow::Utf16::IsTrailSurrogate(kEndOfInput));
608  if (unibrow::Utf16::IsTrailSurrogate(c1)) {
609  c0_ = unibrow::Utf16::CombineSurrogatePair(c0_, c1);
610  return true;
611  }
612  source_->Back();
613  }
614  return false;
615  }
616 
617  void PushBack(uc32 ch) {
618  DCHECK_LE(c0_, static_cast<uc32>(unibrow::Utf16::kMaxNonSurrogateCharCode));
619  source_->Back();
620  c0_ = ch;
621  }
622 
623  uc32 Peek() const { return source_->Peek(); }
624 
625  inline Token::Value Select(Token::Value tok) {
626  Advance();
627  return tok;
628  }
629 
630  inline Token::Value Select(uc32 next, Token::Value then, Token::Value else_) {
631  Advance();
632  if (c0_ == next) {
633  Advance();
634  return then;
635  } else {
636  return else_;
637  }
638  }
639  // Returns the literal string, if any, for the current token (the
640  // token last returned by Next()). The string is 0-terminated.
641  // Literal strings are collected for identifiers, strings, numbers as well
642  // as for template literals. For template literals we also collect the raw
643  // form.
644  // These functions only give the correct result if the literal was scanned
645  // when a LiteralScope object is alive.
646  //
647  // Current usage of these functions is unfortunately a little undisciplined,
648  // and is_literal_one_byte() + is_literal_one_byte_string() is also
649  // requested for tokens that do not have a literal. Hence, we treat any
650  // token as a one-byte literal. E.g. Token::FUNCTION pretends to have a
651  // literal "function".
652  Vector<const uint8_t> literal_one_byte_string() const {
653  DCHECK(current().CanAccessLiteral() || Token::IsKeyword(current().token));
654  return current().literal_chars.one_byte_literal();
655  }
656  Vector<const uint16_t> literal_two_byte_string() const {
657  DCHECK(current().CanAccessLiteral() || Token::IsKeyword(current().token));
658  return current().literal_chars.two_byte_literal();
659  }
660  bool is_literal_one_byte() const {
661  DCHECK(current().CanAccessLiteral() || Token::IsKeyword(current().token));
662  return current().literal_chars.is_one_byte();
663  }
664  // Returns the literal string for the next token (the token that
665  // would be returned if Next() were called).
666  Vector<const uint8_t> next_literal_one_byte_string() const {
667  DCHECK(next().CanAccessLiteral());
668  return next().literal_chars.one_byte_literal();
669  }
670  Vector<const uint16_t> next_literal_two_byte_string() const {
671  DCHECK(next().CanAccessLiteral());
672  return next().literal_chars.two_byte_literal();
673  }
674  bool is_next_literal_one_byte() const {
675  DCHECK(next().CanAccessLiteral());
676  return next().literal_chars.is_one_byte();
677  }
678  Vector<const uint8_t> raw_literal_one_byte_string() const {
679  DCHECK(current().CanAccessRawLiteral());
680  return current().raw_literal_chars.one_byte_literal();
681  }
682  Vector<const uint16_t> raw_literal_two_byte_string() const {
683  DCHECK(current().CanAccessRawLiteral());
684  return current().raw_literal_chars.two_byte_literal();
685  }
686  bool is_raw_literal_one_byte() const {
687  DCHECK(current().CanAccessRawLiteral());
688  return current().raw_literal_chars.is_one_byte();
689  }
690 
691  template <bool capture_raw, bool unicode = false>
692  uc32 ScanHexNumber(int expected_length);
693  // Scan a number of any length but not bigger than max_value. For example, the
694  // number can be 000000001, so it's very long in characters but its value is
695  // small.
696  template <bool capture_raw>
697  uc32 ScanUnlimitedLengthHexNumber(int max_value, int beg_pos);
698 
699  // Scans a single JavaScript token.
700  V8_INLINE Token::Value ScanSingleToken();
701  V8_INLINE void Scan();
702  // Performance hack: pass through a pre-calculated "next()" value to avoid
703  // having to re-calculate it in Scan. You'd think the compiler would be able
704  // to hoist the next() calculation out of the inlined Scan method, but seems
705  // that pointer aliasing analysis fails show that this is safe.
706  V8_INLINE void Scan(TokenDesc* next_desc);
707 
708  V8_INLINE Token::Value SkipWhiteSpace();
709  Token::Value SkipSingleHTMLComment();
710  Token::Value SkipSingleLineComment();
711  Token::Value SkipSourceURLComment();
712  void TryToParseSourceURLComment();
713  Token::Value SkipMultiLineComment();
714  // Scans a possible HTML comment -- begins with '<!'.
715  Token::Value ScanHtmlComment();
716 
717  bool ScanDigitsWithNumericSeparators(bool (*predicate)(uc32 ch),
718  bool is_check_first_digit);
719  bool ScanDecimalDigits();
720  // Optimized function to scan decimal number as Smi.
721  bool ScanDecimalAsSmi(uint64_t* value);
722  bool ScanDecimalAsSmiWithNumericSeparators(uint64_t* value);
723  bool ScanHexDigits();
724  bool ScanBinaryDigits();
725  bool ScanSignedInteger();
726  bool ScanOctalDigits();
727  bool ScanImplicitOctalDigits(int start_pos, NumberKind* kind);
728 
729  Token::Value ScanNumber(bool seen_period);
730  V8_INLINE Token::Value ScanIdentifierOrKeyword();
731  V8_INLINE Token::Value ScanIdentifierOrKeywordInner();
732  Token::Value ScanIdentifierOrKeywordInnerSlow(bool escaped,
733  bool can_be_keyword);
734 
735  Token::Value ScanString();
736  Token::Value ScanPrivateName();
737 
738  // Scans an escape-sequence which is part of a string and adds the
739  // decoded character to the current literal. Returns true if a pattern
740  // is scanned.
741  template <bool capture_raw>
742  bool ScanEscape();
743 
744  // Decodes a Unicode escape-sequence which is part of an identifier.
745  // If the escape sequence cannot be decoded the result is kBadChar.
746  uc32 ScanIdentifierUnicodeEscape();
747  // Helper for the above functions.
748  template <bool capture_raw>
749  uc32 ScanUnicodeEscape();
750 
751  Token::Value ScanTemplateSpan();
752 
753  // Return the current source position.
754  int source_pos() {
755  return static_cast<int>(source_->pos()) - kCharacterLookaheadBufferSize;
756  }
757 
758  static bool LiteralContainsEscapes(const TokenDesc& token) {
759  Location location = token.location;
760  int source_length = (location.end_pos - location.beg_pos);
761  if (token.token == Token::STRING) {
762  // Subtract delimiters.
763  source_length -= 2;
764  }
765  return token.literal_chars.length() != source_length;
766  }
767 
768 #ifdef DEBUG
769  void SanityCheckTokenDesc(const TokenDesc&) const;
770 #endif
771 
772  TokenDesc& next() { return *next_; }
773 
774  const TokenDesc& current() const { return *current_; }
775  const TokenDesc& next() const { return *next_; }
776  const TokenDesc& next_next() const { return *next_next_; }
777 
778  TokenDesc* current_; // desc for current token (as returned by Next())
779  TokenDesc* next_; // desc for next token (one token look-ahead)
780  TokenDesc* next_next_; // desc for the token after next (after PeakAhead())
781 
782  // Input stream. Must be initialized to an Utf16CharacterStream.
783  Utf16CharacterStream* const source_;
784 
785  // One Unicode character look-ahead; c0_ < 0 at the end of the input.
786  uc32 c0_;
787 
788  TokenDesc token_storage_[3];
789 
790  // Whether this scanner encountered an HTML comment.
791  bool found_html_comment_;
792 
793  // Harmony flags to allow ESNext features.
794  bool allow_harmony_private_fields_;
795  bool allow_harmony_numeric_separator_;
796 
797  const bool is_module_;
798 
799  // Values parsed from magic comments.
800  LiteralBuffer source_url_;
801  LiteralBuffer source_mapping_url_;
802 
803  // Last-seen positions of potentially problematic tokens.
804  Location octal_pos_;
805  MessageTemplate octal_message_;
806 
807  MessageTemplate scanner_error_;
808  Location scanner_error_location_;
809 };
810 
811 } // namespace internal
812 } // namespace v8
813 
814 #endif // V8_PARSING_SCANNER_H_
Definition: libplatform.h:13
Definition: v8.h:3740