7 #include "src/parsing/scanner.h" 13 #include "src/ast/ast-value-factory.h" 14 #include "src/conversions-inl.h" 15 #include "src/objects/bigint.h" 16 #include "src/parsing/scanner-inl.h" 17 #include "src/zone/zone.h" 25 : message_stack_(message_stack),
26 old_message_(*message_stack),
27 location_stack_(location_stack),
28 old_location_(*location_stack) {
29 *message_stack_ = MessageTemplate::kNone;
30 *location_stack_ = Location::invalid();
34 *message_stack_ = old_message_;
35 *location_stack_ = old_location_;
38 void MoveErrorTo(TokenDesc* dest) {
39 if (*message_stack_ == MessageTemplate::kNone) {
42 if (dest->invalid_template_escape_message == MessageTemplate::kNone) {
43 dest->invalid_template_escape_message = *message_stack_;
44 dest->invalid_template_escape_location = *location_stack_;
46 *message_stack_ = MessageTemplate::kNone;
47 *location_stack_ = Location::invalid();
51 MessageTemplate*
const message_stack_;
52 MessageTemplate
const old_message_;
62 return isolate->factory()->InternalizeOneByteString(one_byte_literal());
64 return isolate->factory()->InternalizeTwoByteString(two_byte_literal());
67 int Scanner::LiteralBuffer::NewCapacity(
int min_capacity) {
68 int capacity = Max(min_capacity, backing_store_.length());
69 int new_capacity = Min(capacity * kGrowthFactory, capacity + kMaxGrowth);
73 void Scanner::LiteralBuffer::ExpandBuffer() {
74 Vector<byte> new_store = Vector<byte>::New(NewCapacity(kInitialCapacity));
75 MemCopy(new_store.start(), backing_store_.start(), position_);
76 backing_store_.Dispose();
77 backing_store_ = new_store;
80 void Scanner::LiteralBuffer::ConvertToTwoByte() {
81 DCHECK(is_one_byte());
82 Vector<byte> new_store;
83 int new_content_size = position_ * kUC16Size;
84 if (new_content_size >= backing_store_.length()) {
87 new_store = Vector<byte>::New(NewCapacity(new_content_size));
89 new_store = backing_store_;
91 uint8_t* src = backing_store_.start();
92 uint16_t* dst =
reinterpret_cast<uint16_t*
>(new_store.start());
93 for (
int i = position_ - 1;
i >= 0;
i--) {
96 if (new_store.start() != backing_store_.start()) {
97 backing_store_.Dispose();
98 backing_store_ = new_store;
100 position_ = new_content_size;
101 is_one_byte_ =
false;
104 void Scanner::LiteralBuffer::AddTwoByteChar(uc32 code_unit) {
105 DCHECK(!is_one_byte());
106 if (position_ >= backing_store_.length()) ExpandBuffer();
108 static_cast<uc32>(unibrow::Utf16::kMaxNonSurrogateCharCode)) {
109 *
reinterpret_cast<uint16_t*
>(&backing_store_[position_]) = code_unit;
110 position_ += kUC16Size;
112 *
reinterpret_cast<uint16_t*
>(&backing_store_[position_]) =
113 unibrow::Utf16::LeadSurrogate(code_unit);
114 position_ += kUC16Size;
115 if (position_ >= backing_store_.length()) ExpandBuffer();
116 *
reinterpret_cast<uint16_t*
>(&backing_store_[position_]) =
117 unibrow::Utf16::TrailSurrogate(code_unit);
118 position_ += kUC16Size;
125 const size_t Scanner::BookmarkScope::kBookmarkAtFirstPos =
126 std::numeric_limits<size_t>::max() - 2;
127 const size_t Scanner::BookmarkScope::kNoBookmark =
128 std::numeric_limits<size_t>::max() - 1;
129 const size_t Scanner::BookmarkScope::kBookmarkWasApplied =
130 std::numeric_limits<size_t>::max();
132 void Scanner::BookmarkScope::Set() {
133 DCHECK_EQ(bookmark_, kNoBookmark);
139 DCHECK_IMPLIES(scanner_->current().token == Token::UNINITIALIZED,
140 scanner_->current().location.beg_pos ==
141 scanner_->next().location.beg_pos);
142 bookmark_ = (scanner_->current().token == Token::UNINITIALIZED)
143 ? kBookmarkAtFirstPos
144 : scanner_->location().beg_pos;
147 void Scanner::BookmarkScope::Apply() {
148 DCHECK(HasBeenSet());
149 if (had_parser_error_) {
150 scanner_->set_parser_error();
152 scanner_->reset_parser_error_flag();
153 if (bookmark_ == kBookmarkAtFirstPos) {
154 scanner_->SeekNext(0);
156 scanner_->SeekNext(bookmark_);
158 DCHECK_EQ(scanner_->location().beg_pos,
static_cast<int>(bookmark_));
161 bookmark_ = kBookmarkWasApplied;
164 bool Scanner::BookmarkScope::HasBeenSet()
const {
165 return bookmark_ != kNoBookmark && bookmark_ != kBookmarkWasApplied;
168 bool Scanner::BookmarkScope::HasBeenApplied()
const {
169 return bookmark_ == kBookmarkWasApplied;
175 Scanner::Scanner(Utf16CharacterStream* source,
bool is_module)
177 found_html_comment_(false),
178 allow_harmony_numeric_separator_(false),
179 is_module_(is_module),
180 octal_pos_(Location::invalid()),
181 octal_message_(MessageTemplate::
kNone) {
182 DCHECK_NOT_NULL(source);
185 void Scanner::Initialize() {
189 next().after_line_terminator =
true;
193 template <
bool capture_raw,
bool unicode>
194 uc32 Scanner::ScanHexNumber(
int expected_length) {
195 DCHECK_LE(expected_length, 4);
197 int begin = source_pos() - 2;
199 for (
int i = 0;
i < expected_length;
i++) {
200 int d = HexValue(c0_);
202 ReportScannerError(Location(begin, begin + expected_length + 2),
204 ? MessageTemplate::kInvalidUnicodeEscapeSequence
205 : MessageTemplate::kInvalidHexEscapeSequence);
209 Advance<capture_raw>();
215 template <
bool capture_raw>
216 uc32 Scanner::ScanUnlimitedLengthHexNumber(
int max_value,
int beg_pos) {
218 int d = HexValue(c0_);
219 if (d < 0)
return -1;
224 ReportScannerError(Location(beg_pos, source_pos() + 1),
225 MessageTemplate::kUndefinedUnicodeCodePoint);
228 Advance<capture_raw>();
235 Token::Value Scanner::Next() {
237 TokenDesc* previous = current_;
243 if (V8_LIKELY(next_next().token == Token::UNINITIALIZED)) {
247 previous->after_line_terminator =
false;
251 next_next_ = previous;
252 previous->token = Token::UNINITIALIZED;
253 DCHECK_NE(Token::UNINITIALIZED, current().token);
255 return current().token;
258 Token::Value Scanner::PeekAhead() {
259 DCHECK(next().token != Token::DIV);
260 DCHECK(next().token != Token::ASSIGN_DIV);
262 if (next_next().token != Token::UNINITIALIZED) {
263 return next_next().token;
265 TokenDesc* temp = next_;
267 next().after_line_terminator =
false;
271 return next_next().token;
274 Token::Value Scanner::SkipSingleHTMLComment() {
276 ReportScannerError(source_pos(), MessageTemplate::kHtmlCommentInModule);
277 return Token::ILLEGAL;
279 return SkipSingleLineComment();
282 Token::Value Scanner::SkipSingleLineComment() {
288 AdvanceUntil([](uc32 c0_) {
return unibrow::IsLineTerminator(c0_); });
290 return Token::WHITESPACE;
293 Token::Value Scanner::SkipSourceURLComment() {
294 TryToParseSourceURLComment();
295 while (c0_ != kEndOfInput && !unibrow::IsLineTerminator(c0_)) {
299 return Token::WHITESPACE;
302 void Scanner::TryToParseSourceURLComment() {
305 DCHECK(!IsWhiteSpaceOrLineTerminator(kEndOfInput));
306 if (!IsWhiteSpace(c0_))
return;
311 while (c0_ != kEndOfInput && !IsWhiteSpaceOrLineTerminator(c0_) &&
316 if (!name.is_one_byte())
return;
317 Vector<const uint8_t> name_literal = name.one_byte_literal();
318 LiteralBuffer* value;
319 if (name_literal == STATIC_CHAR_VECTOR(
"sourceURL")) {
320 value = &source_url_;
321 }
else if (name_literal == STATIC_CHAR_VECTOR(
"sourceMappingURL")) {
322 value = &source_mapping_url_;
330 while (IsWhiteSpace(c0_)) {
333 while (c0_ != kEndOfInput && !unibrow::IsLineTerminator(c0_)) {
335 if (c0_ ==
'"' || c0_ ==
'\'') {
339 if (IsWhiteSpace(c0_)) {
346 while (c0_ != kEndOfInput && !unibrow::IsLineTerminator(c0_)) {
347 if (!IsWhiteSpace(c0_)) {
355 Token::Value Scanner::SkipMultiLineComment() {
359 while (c0_ != kEndOfInput) {
360 DCHECK(!unibrow::IsLineTerminator(kEndOfInput));
361 if (!HasLineTerminatorBeforeNext() && unibrow::IsLineTerminator(c0_)) {
364 next().after_line_terminator =
true;
367 while (V8_UNLIKELY(c0_ ==
'*')) {
371 return Token::WHITESPACE;
378 return Token::ILLEGAL;
381 Token::Value Scanner::ScanHtmlComment() {
385 if (c0_ !=
'-' || Peek() !=
'-') {
391 found_html_comment_ =
true;
392 return SkipSingleHTMLComment();
396 void Scanner::SanityCheckTokenDesc(
const TokenDesc& token)
const {
400 switch (token.token) {
401 case Token::UNINITIALIZED:
404 case Token::TEMPLATE_SPAN:
405 case Token::TEMPLATE_TAIL:
408 DCHECK_EQ(token.invalid_template_escape_message, MessageTemplate::kNone);
414 void Scanner::SeekForward(
int pos) {
417 if (pos == next().location.beg_pos)
return;
418 int current_pos = source_pos();
419 DCHECK_EQ(next().location.end_pos, current_pos);
421 DCHECK(pos >= current_pos);
422 if (pos != current_pos) {
428 next().after_line_terminator =
false;
433 template <
bool capture_raw>
434 bool Scanner::ScanEscape() {
436 Advance<capture_raw>();
439 DCHECK(!unibrow::IsLineTerminator(kEndOfInput));
440 if (!capture_raw && unibrow::IsLineTerminator(c)) {
442 if (IsCarriageReturn(c) && IsLineFeed(c0_)) Advance();
450 case 'b' : c =
'\b';
break;
451 case 'f' : c =
'\f';
break;
452 case 'n' : c =
'\n';
break;
453 case 'r' : c =
'\r';
break;
454 case 't' : c =
'\t';
break;
456 c = ScanUnicodeEscape<capture_raw>();
457 if (c < 0)
return false;
464 c = ScanHexNumber<capture_raw>(2);
465 if (c < 0)
return false;
476 c = ScanOctalEscape<capture_raw>(c, 2);
485 template <
bool capture_raw>
486 uc32 Scanner::ScanOctalEscape(uc32 c,
int length) {
489 for (;
i < length;
i++) {
491 if (d < 0 || d > 7)
break;
493 if (nx >= 256)
break;
495 Advance<capture_raw>();
502 if (c !=
'0' ||
i > 0 || IsNonOctalDecimalDigit(c0_)) {
503 octal_pos_ = Location(source_pos() -
i - 1, source_pos() - 1);
504 octal_message_ = capture_raw ? MessageTemplate::kTemplateOctalLiteral
505 : MessageTemplate::kStrictOctalEscape;
510 Token::Value Scanner::ScanString() {
514 next().literal_chars.Start();
516 if (V8_UNLIKELY(c0_ == kEndOfInput))
return Token::ILLEGAL;
517 if ((V8_UNLIKELY(static_cast<uint32_t>(c0_) >= kMaxAscii) &&
518 !unibrow::IsStringLiteralLineTerminator(c0_)) ||
519 !MayTerminateString(character_scan_flags[c0_])) {
521 AdvanceUntil([
this](uc32 c0) {
522 if (V8_UNLIKELY(static_cast<uint32_t>(c0) > kMaxAscii)) {
523 if (V8_UNLIKELY(unibrow::IsStringLiteralLineTerminator(c0))) {
529 uint8_t char_flags = character_scan_flags[c0];
530 if (MayTerminateString(char_flags))
return true;
537 return Token::STRING;
542 if (V8_UNLIKELY(c0_ == kEndOfInput || !ScanEscape<false>())) {
543 return Token::ILLEGAL;
547 if (V8_UNLIKELY(c0_ == kEndOfInput ||
548 unibrow::IsStringLiteralLineTerminator(c0_))) {
549 return Token::ILLEGAL;
551 DCHECK_NE(quote, c0_);
552 DCHECK((c0_ ==
'\'' || c0_ ==
'"'));
553 AddLiteralCharAdvance();
557 Token::Value Scanner::ScanPrivateName() {
558 if (!allow_harmony_private_fields()) {
559 ReportScannerError(source_pos(),
560 MessageTemplate::kInvalidOrUnexpectedToken);
561 return Token::ILLEGAL;
564 next().literal_chars.Start();
566 DCHECK(!IsIdentifierStart(kEndOfInput));
567 if (!IsIdentifierStart(Peek())) {
568 ReportScannerError(source_pos(),
569 MessageTemplate::kInvalidOrUnexpectedToken);
570 return Token::ILLEGAL;
573 AddLiteralCharAdvance();
574 Token::Value token = ScanIdentifierOrKeywordInner();
575 return token == Token::ILLEGAL ? Token::ILLEGAL : Token::PRIVATE_NAME;
578 Token::Value Scanner::ScanTemplateSpan() {
595 ErrorState scanner_error_state(&scanner_error_, &scanner_error_location_);
596 ErrorState octal_error_state(&octal_message_, &octal_pos_);
598 Token::Value result = Token::TEMPLATE_SPAN;
599 next().literal_chars.Start();
600 next().raw_literal_chars.Start();
601 const bool capture_raw =
true;
606 result = Token::TEMPLATE_TAIL;
608 }
else if (c ==
'$' && Peek() ==
'{') {
612 }
else if (c ==
'\\') {
614 DCHECK(!unibrow::IsLineTerminator(kEndOfInput));
615 if (capture_raw) AddRawLiteralChar(
'\\');
616 if (unibrow::IsLineTerminator(c0_)) {
621 if (lastChar ==
'\r') {
623 if (c0_ ==
'\n') Advance();
626 if (capture_raw) AddRawLiteralChar(lastChar);
628 bool success = ScanEscape<capture_raw>();
630 DCHECK_EQ(!success, has_error());
633 scanner_error_state.MoveErrorTo(next_);
634 octal_error_state.MoveErrorTo(next_);
645 if (c0_ ==
'\n') Advance();
648 if (capture_raw) AddRawLiteralChar(c);
652 next().location.end_pos = source_pos();
653 next().token = result;
658 Handle<String> Scanner::SourceUrl(Isolate* isolate)
const {
660 if (source_url_.length() > 0) {
661 tmp = source_url_.Internalize(isolate);
666 Handle<String> Scanner::SourceMappingUrl(Isolate* isolate)
const {
668 if (source_mapping_url_.length() > 0) {
669 tmp = source_mapping_url_.Internalize(isolate);
674 bool Scanner::ScanDigitsWithNumericSeparators(
bool (*predicate)(uc32 ch),
675 bool is_check_first_digit) {
677 if (is_check_first_digit && !predicate(c0_))
return false;
679 bool separator_seen =
false;
680 while (predicate(c0_) || c0_ ==
'_') {
684 ReportScannerError(Location(source_pos(), source_pos() + 1),
685 MessageTemplate::kContinuousNumericSeparator);
688 separator_seen =
true;
691 separator_seen =
false;
692 AddLiteralCharAdvance();
695 if (separator_seen) {
696 ReportScannerError(Location(source_pos(), source_pos() + 1),
697 MessageTemplate::kTrailingNumericSeparator);
704 bool Scanner::ScanDecimalDigits() {
705 if (allow_harmony_numeric_separator()) {
706 return ScanDigitsWithNumericSeparators(&IsDecimalDigit,
false);
708 while (IsDecimalDigit(c0_)) {
709 AddLiteralCharAdvance();
714 bool Scanner::ScanDecimalAsSmiWithNumericSeparators(uint64_t* value) {
715 bool separator_seen =
false;
716 while (IsDecimalDigit(c0_) || c0_ ==
'_') {
720 ReportScannerError(Location(source_pos(), source_pos() + 1),
721 MessageTemplate::kContinuousNumericSeparator);
724 separator_seen =
true;
727 separator_seen =
false;
728 *value = 10 * *value + (c0_ -
'0');
729 uc32 first_char = c0_;
731 AddLiteralChar(first_char);
734 if (separator_seen) {
735 ReportScannerError(Location(source_pos(), source_pos() + 1),
736 MessageTemplate::kTrailingNumericSeparator);
743 bool Scanner::ScanDecimalAsSmi(uint64_t* value) {
744 if (allow_harmony_numeric_separator()) {
745 return ScanDecimalAsSmiWithNumericSeparators(value);
748 while (IsDecimalDigit(c0_)) {
749 *value = 10 * *value + (c0_ -
'0');
750 uc32 first_char = c0_;
752 AddLiteralChar(first_char);
757 bool Scanner::ScanBinaryDigits() {
758 if (allow_harmony_numeric_separator()) {
759 return ScanDigitsWithNumericSeparators(&IsBinaryDigit,
true);
763 if (!IsBinaryDigit(c0_)) {
767 while (IsBinaryDigit(c0_)) {
768 AddLiteralCharAdvance();
773 bool Scanner::ScanOctalDigits() {
774 if (allow_harmony_numeric_separator()) {
775 return ScanDigitsWithNumericSeparators(&IsOctalDigit,
true);
779 if (!IsOctalDigit(c0_)) {
783 while (IsOctalDigit(c0_)) {
784 AddLiteralCharAdvance();
789 bool Scanner::ScanImplicitOctalDigits(
int start_pos,
790 Scanner::NumberKind* kind) {
791 *kind = IMPLICIT_OCTAL;
795 if (IsNonOctalDecimalDigit(c0_)) {
796 *kind = DECIMAL_WITH_LEADING_ZERO;
799 if (!IsOctalDigit(c0_)) {
801 octal_pos_ = Location(start_pos, source_pos());
802 octal_message_ = MessageTemplate::kStrictOctalLiteral;
805 AddLiteralCharAdvance();
809 bool Scanner::ScanHexDigits() {
810 if (allow_harmony_numeric_separator()) {
811 return ScanDigitsWithNumericSeparators(&IsHexDigit,
true);
815 if (!IsHexDigit(c0_)) {
819 while (IsHexDigit(c0_)) {
820 AddLiteralCharAdvance();
825 bool Scanner::ScanSignedInteger() {
826 if (c0_ ==
'+' || c0_ ==
'-') AddLiteralCharAdvance();
828 if (!IsDecimalDigit(c0_))
return false;
829 return ScanDecimalDigits();
832 Token::Value Scanner::ScanNumber(
bool seen_period) {
833 DCHECK(IsDecimalDigit(c0_));
835 NumberKind kind = DECIMAL;
837 next().literal_chars.Start();
838 bool at_start = !seen_period;
839 int start_pos = source_pos();
843 if (allow_harmony_numeric_separator() && c0_ ==
'_') {
844 return Token::ILLEGAL;
847 if (!ScanDecimalDigits())
return Token::ILLEGAL;
851 AddLiteralCharAdvance();
855 if (c0_ ==
'x' || c0_ ==
'X') {
856 AddLiteralCharAdvance();
858 if (!ScanHexDigits())
return Token::ILLEGAL;
859 }
else if (c0_ ==
'o' || c0_ ==
'O') {
860 AddLiteralCharAdvance();
862 if (!ScanOctalDigits())
return Token::ILLEGAL;
863 }
else if (c0_ ==
'b' || c0_ ==
'B') {
864 AddLiteralCharAdvance();
866 if (!ScanBinaryDigits())
return Token::ILLEGAL;
867 }
else if (IsOctalDigit(c0_)) {
868 kind = IMPLICIT_OCTAL;
869 if (!ScanImplicitOctalDigits(start_pos, &kind)) {
870 return Token::ILLEGAL;
872 if (kind == DECIMAL_WITH_LEADING_ZERO) {
875 }
else if (IsNonOctalDecimalDigit(c0_)) {
876 kind = DECIMAL_WITH_LEADING_ZERO;
877 }
else if (allow_harmony_numeric_separator() && c0_ ==
'_') {
878 ReportScannerError(Location(source_pos(), source_pos() + 1),
879 MessageTemplate::kZeroDigitNumericSeparator);
880 return Token::ILLEGAL;
885 if (kind == DECIMAL || kind == DECIMAL_WITH_LEADING_ZERO) {
890 if (!ScanDecimalAsSmi(&value)) {
891 return Token::ILLEGAL;
894 if (next().literal_chars.one_byte_literal().length() <= 10 &&
895 value <= Smi::kMaxValue && c0_ !=
'.' && !IsIdentifierStart(c0_)) {
896 next().smi_value_ =
static_cast<uint32_t>(value);
898 if (kind == DECIMAL_WITH_LEADING_ZERO) {
899 octal_pos_ = Location(start_pos, source_pos());
900 octal_message_ = MessageTemplate::kStrictDecimalWithLeadingZero;
906 if (!ScanDecimalDigits())
return Token::ILLEGAL;
909 AddLiteralCharAdvance();
910 if (allow_harmony_numeric_separator() && c0_ ==
'_') {
911 return Token::ILLEGAL;
913 if (!ScanDecimalDigits())
return Token::ILLEGAL;
918 bool is_bigint =
false;
919 if (c0_ ==
'n' && !seen_period &&
920 (kind == DECIMAL || kind == HEX || kind == OCTAL || kind == BINARY)) {
924 static const int kMaxBigIntCharacters = BigInt::kMaxLengthBits / 4;
925 int length = source_pos() - start_pos - (kind != DECIMAL ? 2 : 0);
926 if (length > kMaxBigIntCharacters) {
927 ReportScannerError(Location(start_pos, source_pos()),
928 MessageTemplate::kBigIntTooBig);
929 return Token::ILLEGAL;
934 }
else if (c0_ ==
'e' || c0_ ==
'E') {
938 if (!(kind == DECIMAL || kind == DECIMAL_WITH_LEADING_ZERO))
939 return Token::ILLEGAL;
942 AddLiteralCharAdvance();
944 if (!ScanSignedInteger())
return Token::ILLEGAL;
951 if (IsDecimalDigit(c0_) || IsIdentifierStart(c0_)) {
952 return Token::ILLEGAL;
955 if (kind == DECIMAL_WITH_LEADING_ZERO) {
956 octal_pos_ = Location(start_pos, source_pos());
957 octal_message_ = MessageTemplate::kStrictDecimalWithLeadingZero;
960 return is_bigint ? Token::BIGINT : Token::NUMBER;
963 uc32 Scanner::ScanIdentifierUnicodeEscape() {
965 if (c0_ !=
'u')
return -1;
967 return ScanUnicodeEscape<false>();
970 template <
bool capture_raw>
971 uc32 Scanner::ScanUnicodeEscape() {
975 int begin = source_pos() - 2;
976 Advance<capture_raw>();
977 uc32 cp = ScanUnlimitedLengthHexNumber<capture_raw>(0x10FFFF, begin);
978 if (cp < 0 || c0_ !=
'}') {
979 ReportScannerError(source_pos(),
980 MessageTemplate::kInvalidUnicodeEscapeSequence);
983 Advance<capture_raw>();
986 const bool unicode =
true;
987 return ScanHexNumber<capture_raw, unicode>(4);
990 Token::Value Scanner::ScanIdentifierOrKeywordInnerSlow(
bool escaped,
991 bool can_be_keyword) {
995 uc32 c = ScanIdentifierUnicodeEscape();
999 DCHECK(!IsIdentifierPart(-1));
1000 if (c ==
'\\' || !IsIdentifierPart(c)) {
1001 return Token::ILLEGAL;
1003 can_be_keyword = can_be_keyword && CharCanBeKeyword(c);
1005 }
else if (IsIdentifierPart(c0_) ||
1006 (CombineSurrogatePair() && IsIdentifierPart(c0_))) {
1007 can_be_keyword = can_be_keyword && CharCanBeKeyword(c0_);
1008 AddLiteralCharAdvance();
1014 if (can_be_keyword && next().literal_chars.is_one_byte()) {
1015 Vector<const uint8_t> chars = next().literal_chars.one_byte_literal();
1016 Token::Value token =
1017 KeywordOrIdentifierToken(chars.start(), chars.length());
1019 if (token == Token::FUTURE_STRICT_RESERVED_WORD) {
1020 if (escaped)
return Token::ESCAPED_STRICT_RESERVED_WORD;
1023 if (token == Token::IDENTIFIER)
return token;
1025 if (!escaped)
return token;
1027 if (token == Token::LET || token == Token::STATIC) {
1028 return Token::ESCAPED_STRICT_RESERVED_WORD;
1030 return Token::ESCAPED_KEYWORD;
1033 return Token::IDENTIFIER;
1036 bool Scanner::ScanRegExpPattern() {
1037 DCHECK_EQ(Token::UNINITIALIZED, next_next().token);
1038 DCHECK(next().token == Token::DIV || next().token == Token::ASSIGN_DIV);
1041 bool in_character_class =
false;
1046 next().literal_chars.Start();
1047 if (next().token == Token::ASSIGN_DIV) {
1048 AddLiteralChar(
'=');
1051 while (c0_ !=
'/' || in_character_class) {
1052 if (c0_ == kEndOfInput || unibrow::IsLineTerminator(c0_)) {
1056 AddLiteralCharAdvance();
1057 if (c0_ == kEndOfInput || unibrow::IsLineTerminator(c0_)) {
1060 AddLiteralCharAdvance();
1072 if (c0_ ==
'[') in_character_class =
true;
1073 if (c0_ ==
']') in_character_class =
false;
1074 AddLiteralCharAdvance();
1079 next().token = Token::REGEXP_LITERAL;
1084 Maybe<RegExp::Flags> Scanner::ScanRegExpFlags() {
1085 DCHECK_EQ(Token::REGEXP_LITERAL, next().token);
1089 while (IsIdentifierPart(c0_)) {
1093 flag = RegExp::kGlobal;
1096 flag = RegExp::kIgnoreCase;
1099 flag = RegExp::kMultiline;
1102 flag = RegExp::kDotAll;
1105 flag = RegExp::kUnicode;
1108 flag = RegExp::kSticky;
1111 return Nothing<RegExp::Flags>();
1114 return Nothing<RegExp::Flags>();
1120 next().location.end_pos = source_pos();
1124 const AstRawString* Scanner::CurrentSymbol(
1125 AstValueFactory* ast_value_factory)
const {
1126 if (is_literal_one_byte()) {
1127 return ast_value_factory->GetOneByteString(literal_one_byte_string());
1129 return ast_value_factory->GetTwoByteString(literal_two_byte_string());
1132 const AstRawString* Scanner::NextSymbol(
1133 AstValueFactory* ast_value_factory)
const {
1134 if (is_next_literal_one_byte()) {
1135 return ast_value_factory->GetOneByteString(next_literal_one_byte_string());
1137 return ast_value_factory->GetTwoByteString(next_literal_two_byte_string());
1140 const AstRawString* Scanner::CurrentRawSymbol(
1141 AstValueFactory* ast_value_factory)
const {
1142 if (is_raw_literal_one_byte()) {
1143 return ast_value_factory->GetOneByteString(raw_literal_one_byte_string());
1145 return ast_value_factory->GetTwoByteString(raw_literal_two_byte_string());
1149 double Scanner::DoubleValue() {
1150 DCHECK(is_literal_one_byte());
1151 return StringToDouble(
1152 literal_one_byte_string(),
1153 ALLOW_HEX | ALLOW_OCTAL | ALLOW_IMPLICIT_OCTAL | ALLOW_BINARY);
1156 const char* Scanner::CurrentLiteralAsCString(Zone* zone)
const {
1157 DCHECK(is_literal_one_byte());
1158 Vector<const uint8_t> vector = literal_one_byte_string();
1159 int length = vector.length();
1160 char* buffer = zone->NewArray<
char>(length + 1);
1161 memcpy(buffer, vector.start(), length);
1162 buffer[length] =
'\0';
1166 void Scanner::SeekNext(
size_t position) {
1174 for (TokenDesc& token : token_storage_) {
1175 token.token = Token::UNINITIALIZED;
1176 token.invalid_template_escape_message = MessageTemplate::kNone;
1179 source_->Seek(position);
1181 c0_ = source_->Advance();
1182 next().after_line_terminator =
false;
1184 DCHECK_EQ(next().location.beg_pos, static_cast<int>(position));