projects / gcc.git / blob
? search:


0baf4e4e24b0b0213c085b2bc9b5a4292ca36895
[gcc.git] / gcc / go / gofrontend / lex.cc
1 // lex.cc -- Go frontend lexer.
2
3 // Copyright 2009 The Go Authors. All rights reserved.
4 // Use of this source code is governed by a BSD-style
5 // license that can be found in the LICENSE file.
6
7 #include "go-system.h"
8 #include "go-diagnostics.h"
9
10 #include "lex.h"
11
12 // Manage mapping from keywords to the Keyword codes.
13
14 class Keywords
15 {
16 public:
17 // The structure which maps keywords to codes.
18 struct Mapping
19 {
20 // Keyword string.
21 const char* keystring;
22 // Keyword code.
23 Keyword keycode;
24 };
25
26 // Return the parsecode corresponding to KEYSTRING, or
27 // KEYWORD_INVALID if it is not a keyword.
28 Keyword
29 keyword_to_code(const char* keyword, size_t len) const;
30
31 // Return the string for a keyword.
32 const char*
33 keyword_to_string(Keyword) const;
34
35 private:
36 static const Mapping mapping_[];
37 static const int count_;
38 };
39
40 // Mapping from keyword string to keyword code. This array must be
41 // kept in sorted order, and the order must match the Keyword enum.
42 // Strings are looked up using bsearch.
43
44 const Keywords::Mapping
45 Keywords::mapping_[] =
46 {
47 { NULL, KEYWORD_INVALID },
48 { "__asm__", KEYWORD_ASM },
49 { "break", KEYWORD_BREAK },
50 { "case", KEYWORD_CASE },
51 { "chan", KEYWORD_CHAN },
52 { "const", KEYWORD_CONST },
53 { "continue", KEYWORD_CONTINUE },
54 { "default", KEYWORD_DEFAULT },
55 { "defer", KEYWORD_DEFER },
56 { "else", KEYWORD_ELSE },
57 { "fallthrough", KEYWORD_FALLTHROUGH },
58 { "for", KEYWORD_FOR },
59 { "func", KEYWORD_FUNC },
60 { "go", KEYWORD_GO },
61 { "goto", KEYWORD_GOTO },
62 { "if", KEYWORD_IF },
63 { "import", KEYWORD_IMPORT },
64 { "interface", KEYWORD_INTERFACE },
65 { "map", KEYWORD_MAP },
66 { "package", KEYWORD_PACKAGE },
67 { "range", KEYWORD_RANGE },
68 { "return", KEYWORD_RETURN },
69 { "select", KEYWORD_SELECT },
70 { "struct", KEYWORD_STRUCT },
71 { "switch", KEYWORD_SWITCH },
72 { "type", KEYWORD_TYPE },
73 { "var", KEYWORD_VAR }
74 };
75
76 // Number of entries in the map.
77
78 const int Keywords::count_ =
79 sizeof(Keywords::mapping_) / sizeof(Keywords::mapping_[0]);
80
81 // Comparison function passed to bsearch.
82
83 extern "C"
84 {
85
86 struct Keywords_search_key
87 {
88 const char* str;
89 size_t len;
90 };
91
92 static int
93 keyword_compare(const void* keyv, const void* mapv)
94 {
95 const Keywords_search_key* key =
96 static_cast<const Keywords_search_key*>(keyv);
97 const Keywords::Mapping* map =
98 static_cast<const Keywords::Mapping*>(mapv);
99 if (map->keystring == NULL)
100 return 1;
101 int i = strncmp(key->str, map->keystring, key->len);
102 if (i != 0)
103 return i;
104 if (map->keystring[key->len] != '\0')
105 return -1;
106 return 0;
107 }
108
109 } // End extern "C".
110
111 // Convert a string to a keyword code. Return KEYWORD_INVALID if the
112 // string is not a keyword.
113
114 Keyword
115 Keywords::keyword_to_code(const char* keyword, size_t len) const
116 {
117 Keywords_search_key key;
118 key.str = keyword;
119 key.len = len;
120 void* mapv = bsearch(&key,
121 this->mapping_,
122 this->count_,
123 sizeof(this->mapping_[0]),
124 keyword_compare);
125 if (mapv == NULL)
126 return KEYWORD_INVALID;
127 Mapping* map = static_cast<Mapping*>(mapv);
128 return map->keycode;
129 }
130
131 // Convert a keyword code to a string.
132
133 const char*
134 Keywords::keyword_to_string(Keyword code) const
135 {
136 go_assert(code > KEYWORD_INVALID && code < this->count_);
137 const Mapping* map = &this->mapping_[code];
138 go_assert(map->keycode == code);
139 return map->keystring;
140 }
141
142 // There is one instance of the Keywords class.
143
144 static Keywords keywords;
145
146 // Class Token.
147
148 // Make a general token.
149
150 Token::Token(Classification classification, Location location)
151 : classification_(classification), location_(location)
152 {
153 }
154
155 // Destroy a token.
156
157 Token::~Token()
158 {
159 this->clear();
160 }
161
162 // Clear a token--release memory.
163
164 void
165 Token::clear()
166 {
167 if (this->classification_ == TOKEN_INTEGER
168 || this->classification_ == TOKEN_CHARACTER)
169 mpz_clear(this->u_.integer_value);
170 else if (this->classification_ == TOKEN_FLOAT
171 || this->classification_ == TOKEN_IMAGINARY)
172 mpfr_clear(this->u_.float_value);
173 }
174
175 // Construct a token.
176
177 Token::Token(const Token& tok)
178 : classification_(tok.classification_), location_(tok.location_)
179 {
180 switch (this->classification_)
181 {
182 case TOKEN_INVALID:
183 case TOKEN_EOF:
184 break;
185 case TOKEN_KEYWORD:
186 this->u_.keyword = tok.u_.keyword;
187 break;
188 case TOKEN_IDENTIFIER:
189 case TOKEN_STRING:
190 this->u_.string_value = tok.u_.string_value;
191 break;
192 case TOKEN_OPERATOR:
193 this->u_.op = tok.u_.op;
194 break;
195 case TOKEN_CHARACTER:
196 case TOKEN_INTEGER:
197 mpz_init_set(this->u_.integer_value, tok.u_.integer_value);
198 break;
199 case TOKEN_FLOAT:
200 case TOKEN_IMAGINARY:
201 mpfr_init_set(this->u_.float_value, tok.u_.float_value, MPFR_RNDN);
202 break;
203 default:
204 go_unreachable();
205 }
206 }
207
208 // Assign to a token.
209
210 Token&
211 Token::operator=(const Token& tok)
212 {
213 this->clear();
214 this->classification_ = tok.classification_;
215 this->location_ = tok.location_;
216 switch (tok.classification_)
217 {
218 case TOKEN_INVALID:
219 case TOKEN_EOF:
220 break;
221 case TOKEN_KEYWORD:
222 this->u_.keyword = tok.u_.keyword;
223 break;
224 case TOKEN_IDENTIFIER:
225 this->u_.identifier_value.name = tok.u_.identifier_value.name;
226 this->u_.identifier_value.is_exported =
227 tok.u_.identifier_value.is_exported;
228 break;
229 case TOKEN_STRING:
230 this->u_.string_value = tok.u_.string_value;
231 break;
232 case TOKEN_OPERATOR:
233 this->u_.op = tok.u_.op;
234 break;
235 case TOKEN_CHARACTER:
236 case TOKEN_INTEGER:
237 mpz_init_set(this->u_.integer_value, tok.u_.integer_value);
238 break;
239 case TOKEN_FLOAT:
240 case TOKEN_IMAGINARY:
241 mpfr_init_set(this->u_.float_value, tok.u_.float_value, MPFR_RNDN);
242 break;
243 default:
244 go_unreachable();
245 }
246 return *this;
247 }
248
249 // Print the token for debugging.
250
251 void
252 Token::print(FILE* file) const
253 {
254 switch (this->classification_)
255 {
256 case TOKEN_INVALID:
257 fprintf(file, "invalid");
258 break;
259 case TOKEN_EOF:
260 fprintf(file, "EOF");
261 break;
262 case TOKEN_KEYWORD:
263 fprintf(file, "keyword %s", keywords.keyword_to_string(this->u_.keyword));
264 break;
265 case TOKEN_IDENTIFIER:
266 fprintf(file, "identifier \"%s\"", this->u_.string_value->c_str());
267 break;
268 case TOKEN_STRING:
269 fprintf(file, "quoted string \"%s\"", this->u_.string_value->c_str());
270 break;
271 case TOKEN_CHARACTER:
272 fprintf(file, "character ");
273 mpz_out_str(file, 10, this->u_.integer_value);
274 break;
275 case TOKEN_INTEGER:
276 fprintf(file, "integer ");
277 mpz_out_str(file, 10, this->u_.integer_value);
278 break;
279 case TOKEN_FLOAT:
280 fprintf(file, "float ");
281 mpfr_out_str(file, 10, 0, this->u_.float_value, MPFR_RNDN);
282 break;
283 case TOKEN_IMAGINARY:
284 fprintf(file, "imaginary ");
285 mpfr_out_str(file, 10, 0, this->u_.float_value, MPFR_RNDN);
286 break;
287 case TOKEN_OPERATOR:
288 fprintf(file, "operator ");
289 switch (this->u_.op)
290 {
291 case OPERATOR_INVALID:
292 fprintf(file, "invalid");
293 break;
294 case OPERATOR_OROR:
295 fprintf(file, "||");
296 break;
297 case OPERATOR_ANDAND:
298 fprintf(file, "&&");
299 break;
300 case OPERATOR_EQEQ:
301 fprintf(file, "==");
302 break;
303 case OPERATOR_NOTEQ:
304 fprintf(file, "!=");
305 break;
306 case OPERATOR_LT:
307 fprintf(file, "<");
308 break;
309 case OPERATOR_LE:
310 fprintf(file, "<=");
311 break;
312 case OPERATOR_GT:
313 fprintf(file, ">");
314 break;
315 case OPERATOR_GE:
316 fprintf(file, ">=");
317 break;
318 case OPERATOR_PLUS:
319 fprintf(file, "+");
320 break;
321 case OPERATOR_MINUS:
322 fprintf(file, "-");
323 break;
324 case OPERATOR_OR:
325 fprintf(file, "|");
326 break;
327 case OPERATOR_XOR:
328 fprintf(file, "^");
329 break;
330 case OPERATOR_MULT:
331 fprintf(file, "*");
332 break;
333 case OPERATOR_DIV:
334 fprintf(file, "/");
335 break;
336 case OPERATOR_MOD:
337 fprintf(file, "%%");
338 break;
339 case OPERATOR_LSHIFT:
340 fprintf(file, "<<");
341 break;
342 case OPERATOR_RSHIFT:
343 fprintf(file, ">>");
344 break;
345 case OPERATOR_AND:
346 fprintf(file, "&");
347 break;
348 case OPERATOR_BITCLEAR:
349 fprintf(file, "&^");
350 break;
351 case OPERATOR_NOT:
352 fprintf(file, "!");
353 break;
354 case OPERATOR_CHANOP:
355 fprintf(file, "<-");
356 break;
357 case OPERATOR_EQ:
358 fprintf(file, "=");
359 break;
360 case OPERATOR_PLUSEQ:
361 fprintf(file, "+=");
362 break;
363 case OPERATOR_MINUSEQ:
364 fprintf(file, "-=");
365 break;
366 case OPERATOR_OREQ:
367 fprintf(file, "|=");
368 break;
369 case OPERATOR_XOREQ:
370 fprintf(file, "^=");
371 break;
372 case OPERATOR_MULTEQ:
373 fprintf(file, "*=");
374 break;
375 case OPERATOR_DIVEQ:
376 fprintf(file, "/=");
377 break;
378 case OPERATOR_MODEQ:
379 fprintf(file, "%%=");
380 break;
381 case OPERATOR_LSHIFTEQ:
382 fprintf(file, "<<=");
383 break;
384 case OPERATOR_RSHIFTEQ:
385 fprintf(file, ">>=");
386 break;
387 case OPERATOR_ANDEQ:
388 fprintf(file, "&=");
389 break;
390 case OPERATOR_BITCLEAREQ:
391 fprintf(file, "&^=");
392 break;
393 case OPERATOR_PLUSPLUS:
394 fprintf(file, "++");
395 break;
396 case OPERATOR_MINUSMINUS:
397 fprintf(file, "--");
398 break;
399 case OPERATOR_COLON:
400 fprintf(file, ":");
401 break;
402 case OPERATOR_COLONEQ:
403 fprintf(file, ":=");
404 break;
405 case OPERATOR_SEMICOLON:
406 fprintf(file, ";");
407 break;
408 case OPERATOR_DOT:
409 fprintf(file, ".");
410 break;
411 case OPERATOR_COMMA:
412 fprintf(file, ",");
413 break;
414 case OPERATOR_LPAREN:
415 fprintf(file, "(");
416 break;
417 case OPERATOR_RPAREN:
418 fprintf(file, ")");
419 break;
420 case OPERATOR_LCURLY:
421 fprintf(file, "{");
422 break;
423 case OPERATOR_RCURLY:
424 fprintf(file, "}");
425 break;
426 case OPERATOR_LSQUARE:
427 fprintf(file, "[");
428 break;
429 case OPERATOR_RSQUARE:
430 fprintf(file, "]");
431 break;
432 default:
433 go_unreachable();
434 }
435 break;
436 default:
437 go_unreachable();
438 }
439 }
440
441 // Class Lex.
442
443 Lex::Lex(const char* input_file_name, FILE* input_file, Linemap* linemap)
444 : input_file_name_(input_file_name), input_file_(input_file),
445 linemap_(linemap), linebuf_(NULL), linebufsize_(120), linesize_(0),
446 lineoff_(0), lineno_(0), add_semi_at_eol_(false), pragmas_(0),
447 extern_(), linknames_(NULL)
448 {
449 this->linebuf_ = new char[this->linebufsize_];
450 this->linemap_->start_file(input_file_name, 0);
451 }
452
453 Lex::~Lex()
454 {
455 delete[] this->linebuf_;
456 }
457
458 // Read a new line from the file.
459
460 ssize_t
461 Lex::get_line()
462 {
463 char* buf = this->linebuf_;
464 size_t size = this->linebufsize_;
465
466 FILE* file = this->input_file_;
467 size_t cur = 0;
468 while (true)
469 {
470 int c = getc(file);
471 if (c == EOF)
472 {
473 if (cur == 0)
474 return -1;
475 break;
476 }
477 if (cur + 1 >= size)
478 {
479 size_t ns = 2 * size + 1;
480 if (ns < size || static_cast<ssize_t>(ns) < 0)
481 go_error_at(this->location(), "out of memory");
482 char* nb = new char[ns];
483 memcpy(nb, buf, cur);
484 delete[] buf;
485 buf = nb;
486 size = ns;
487 }
488 buf[cur] = c;
489 ++cur;
490
491 if (c == '\n')
492 break;
493 }
494
495 buf[cur] = '\0';
496
497 this->linebuf_ = buf;
498 this->linebufsize_ = size;
499
500 return cur;
501 }
502
503 // See if we need to read a new line. Return true if there is a new
504 // line, false if we are at EOF.
505
506 bool
507 Lex::require_line()
508 {
509 if (this->lineoff_ < this->linesize_)
510 return true;
511
512 ssize_t got = this->get_line();
513 if (got < 0)
514 return false;
515 ++this->lineno_;
516 this->linesize_= got;
517 this->lineoff_ = 0;
518
519 this->linemap_->start_line(this->lineno_, this->linesize_);
520
521 return true;
522 }
523
524 // Get the current location.
525
526 Location
527 Lex::location() const
528 {
529 return this->linemap_->get_location(this->lineoff_ + 1);
530 }
531
532 // Get a location slightly before the current one. This is used for
533 // slightly more efficient handling of operator tokens.
534
535 Location
536 Lex::earlier_location(int chars) const
537 {
538 return this->linemap_->get_location(this->lineoff_ + 1 - chars);
539 }
540
541 // Get the next token.
542
543 Token
544 Lex::next_token()
545 {
546 bool saw_cpp_comment = false;
547 while (true)
548 {
549 if (!this->require_line())
550 {
551 bool add_semi_at_eol = this->add_semi_at_eol_;
552 this->add_semi_at_eol_ = false;
553 if (add_semi_at_eol)
554 return this->make_operator(OPERATOR_SEMICOLON, 1);
555 return this->make_eof_token();
556 }
557
558 if (!saw_cpp_comment)
559 this->extern_.clear();
560 saw_cpp_comment = false;
561
562 const char* p = this->linebuf_ + this->lineoff_;
563 const char* pend = this->linebuf_ + this->linesize_;
564
565 while (p < pend)
566 {
567 unsigned char cc = *p;
568 switch (cc)
569 {
570 case ' ': case '\t': case '\r':
571 ++p;
572 // Skip whitespace quickly.
573 while (*p == ' ' || *p == '\t' || *p == '\r')
574 ++p;
575 break;
576
577 case '\n':
578 {
579 ++p;
580 bool add_semi_at_eol = this->add_semi_at_eol_;
581 this->add_semi_at_eol_ = false;
582 if (add_semi_at_eol)
583 {
584 this->lineoff_ = p - this->linebuf_;
585 return this->make_operator(OPERATOR_SEMICOLON, 1);
586 }
587 }
588 break;
589
590 case '/':
591 if (p[1] == '/')
592 {
593 this->lineoff_ = p + 2 - this->linebuf_;
594 this->skip_cpp_comment();
595 p = pend;
596 if (p[-1] == '\n' && this->add_semi_at_eol_)
597 --p;
598 saw_cpp_comment = true;
599 }
600 else if (p[1] == '*')
601 {
602 this->lineoff_ = p + 2 - this->linebuf_;
603 Location location = this->location();
604 bool found_newline = false;
605 if (!this->skip_c_comment(&found_newline))
606 return Token::make_invalid_token(location);
607 if (found_newline && this->add_semi_at_eol_)
608 {
609 this->add_semi_at_eol_ = false;
610 return this->make_operator(OPERATOR_SEMICOLON, 1);
611 }
612 p = this->linebuf_ + this->lineoff_;
613 pend = this->linebuf_ + this->linesize_;
614 }
615 else if (p[1] == '=')
616 {
617 this->add_semi_at_eol_ = false;
618 this->lineoff_ = p + 2 - this->linebuf_;
619 return this->make_operator(OPERATOR_DIVEQ, 2);
620 }
621 else
622 {
623 this->add_semi_at_eol_ = false;
624 this->lineoff_ = p + 1 - this->linebuf_;
625 return this->make_operator(OPERATOR_DIV, 1);
626 }
627 break;
628
629 case 'A': case 'B': case 'C': case 'D': case 'E': case 'F':
630 case 'G': case 'H': case 'I': case 'J': case 'K': case 'L':
631 case 'M': case 'N': case 'O': case 'P': case 'Q': case 'R':
632 case 'S': case 'T': case 'U': case 'V': case 'W': case 'X':
633 case 'Y': case 'Z':
634 case 'a': case 'b': case 'c': case 'd': case 'e': case 'f':
635 case 'g': case 'h': case 'i': case 'j': case 'k': case 'l':
636 case 'm': case 'n': case 'o': case 'p': case 'q': case 'r':
637 case 's': case 't': case 'u': case 'v': case 'w': case 'x':
638 case 'y': case 'z':
639 case '_':
640 this->lineoff_ = p - this->linebuf_;
641 return this->gather_identifier();
642
643 case '0': case '1': case '2': case '3': case '4':
644 case '5': case '6': case '7': case '8': case '9':
645 this->add_semi_at_eol_ = true;
646 this->lineoff_ = p - this->linebuf_;
647 return this->gather_number();
648
649 case '\'':
650 this->add_semi_at_eol_ = true;
651 this->lineoff_ = p - this->linebuf_;
652 return this->gather_character();
653
654 case '"':
655 this->add_semi_at_eol_ = true;
656 this->lineoff_ = p - this->linebuf_;
657 return this->gather_string();
658
659 case '`':
660 this->add_semi_at_eol_ = true;
661 this->lineoff_ = p - this->linebuf_;
662 return this->gather_raw_string();
663
664 case '<':
665 case '>':
666 case '&':
667 if (p + 2 < pend)
668 {
669 this->add_semi_at_eol_ = false;
670 Operator op = this->three_character_operator(cc, p[1], p[2]);
671 if (op != OPERATOR_INVALID)
672 {
673 this->lineoff_ = p + 3 - this->linebuf_;
674 return this->make_operator(op, 3);
675 }
676 }
677 // Fall through.
678 case '|':
679 case '=':
680 case '!':
681 case '+':
682 case '-':
683 case '^':
684 case '*':
685 // '/' handled above.
686 case '%':
687 case ':':
688 case ';':
689 case ',':
690 case '(': case ')':
691 case '{': case '}':
692 case '[': case ']':
693 {
694 this->add_semi_at_eol_ = false;
695 Operator op = this->two_character_operator(cc, p[1]);
696 int chars;
697 if (op != OPERATOR_INVALID)
698 {
699 ++p;
700 chars = 2;
701 }
702 else
703 {
704 op = this->one_character_operator(cc);
705 chars = 1;
706 }
707 this->lineoff_ = p + 1 - this->linebuf_;
708 return this->make_operator(op, chars);
709 }
710
711 case '.':
712 if (p[1] >= '0' && p[1] <= '9')
713 {
714 this->add_semi_at_eol_ = true;
715 this->lineoff_ = p - this->linebuf_;
716 return this->gather_number();
717 }
718 if (p[1] == '.' && p[2] == '.')
719 {
720 this->add_semi_at_eol_ = false;
721 this->lineoff_ = p + 3 - this->linebuf_;
722 return this->make_operator(OPERATOR_ELLIPSIS, 3);
723 }
724 this->add_semi_at_eol_ = false;
725 this->lineoff_ = p + 1 - this->linebuf_;
726 return this->make_operator(OPERATOR_DOT, 1);
727
728 default:
729 {
730 unsigned int ci;
731 bool issued_error;
732 this->lineoff_ = p - this->linebuf_;
733 const char *pnext = this->advance_one_utf8_char(p, &ci,
734 &issued_error);
735
736 // Ignore byte order mark at start of file.
737 if (ci == 0xfeff)
738 {
739 p = pnext;
740 break;
741 }
742
743 if (Lex::is_unicode_letter(ci))
744 return this->gather_identifier();
745
746 if (!issued_error && Lex::is_unicode_digit(ci))
747 {
748 go_error_at(this->location(),
749 "identifier cannot begin with digit");
750 issued_error = true;
751 }
752
753 if (!issued_error)
754 go_error_at(this->location(),
755 "invalid character 0x%x in input file",
756 ci);
757
758 p = pend;
759
760 break;
761 }
762 }
763 }
764
765 this->lineoff_ = p - this->linebuf_;
766 }
767 }
768
769 // Fetch one UTF-8 character from a string. Set *VALUE to the value.
770 // Return the number of bytes read from the string. Returns 0 if the
771 // string does not point to a valid UTF-8 character.
772
773 int
774 Lex::fetch_char(const char* p, unsigned int* value)
775 {
776 unsigned char c = *p;
777 if (c <= 0x7f)
778 {
779 *value = c;
780 return 1;
781 }
782 else if ((c & 0xe0) == 0xc0
783 && (p[1] & 0xc0) == 0x80)
784 {
785 *value = (((c & 0x1f) << 6)
786 + (p[1] & 0x3f));
787 if (*value <= 0x7f)
788 {
789 *value = 0xfffd;
790 return 0;
791 }
792 return 2;
793 }
794 else if ((c & 0xf0) == 0xe0
795 && (p[1] & 0xc0) == 0x80
796 && (p[2] & 0xc0) == 0x80)
797 {
798 *value = (((c & 0xf) << 12)
799 + ((p[1] & 0x3f) << 6)
800 + (p[2] & 0x3f));
801 if (*value <= 0x7ff)
802 {
803 *value = 0xfffd;
804 return 0;
805 }
806 return 3;
807 }
808 else if ((c & 0xf8) == 0xf0
809 && (p[1] & 0xc0) == 0x80
810 && (p[2] & 0xc0) == 0x80
811 && (p[3] & 0xc0) == 0x80)
812 {
813 *value = (((c & 0x7) << 18)
814 + ((p[1] & 0x3f) << 12)
815 + ((p[2] & 0x3f) << 6)
816 + (p[3] & 0x3f));
817 if (*value <= 0xffff)
818 {
819 *value = 0xfffd;
820 return 0;
821 }
822 return 4;
823 }
824 else
825 {
826 /* Invalid encoding. Return the Unicode replacement
827 character. */
828 *value = 0xfffd;
829 return 0;
830 }
831 }
832
833 // Advance one UTF-8 character. Return the pointer beyond the
834 // character. Set *VALUE to the value. Set *ISSUED_ERROR if an error
835 // was issued.
836
837 const char*
838 Lex::advance_one_utf8_char(const char* p, unsigned int* value,
839 bool* issued_error)
840 {
841 *issued_error = false;
842
843 if (*p == '\0')
844 {
845 go_error_at(this->location(), "invalid NUL byte");
846 *issued_error = true;
847 *value = 0;
848 return p + 1;
849 }
850
851 int adv = Lex::fetch_char(p, value);
852 if (adv == 0)
853 {
854 go_error_at(this->location(), "invalid UTF-8 encoding");
855 *issued_error = true;
856 return p + 1;
857 }
858
859 // Warn about byte order mark, except at start of file.
860 if (*value == 0xfeff && (this->lineno_ != 1 || this->lineoff_ != 0))
861 {
862 go_error_at(this->location(), "Unicode (UTF-8) BOM in middle of file");
863 *issued_error = true;
864 }
865
866 return p + adv;
867 }
868
869 // Pick up an identifier.
870
871 Token
872 Lex::gather_identifier()
873 {
874 const char* pstart = this->linebuf_ + this->lineoff_;
875 const char* p = pstart;
876 const char* pend = this->linebuf_ + this->linesize_;
877 bool is_first = true;
878 bool is_exported = false;
879 bool has_non_ascii_char = false;
880 std::string buf;
881 while (p < pend)
882 {
883 unsigned char cc = *p;
884 if (cc <= 0x7f)
885 {
886 if ((cc < 'A' || cc > 'Z')
887 && (cc < 'a' || cc > 'z')
888 && cc != '_'
889 && (cc < '0' || cc > '9'))
890 {
891 // Check for an invalid character here, as we get better
892 // error behaviour if we swallow them as part of the
893 // identifier we are building.
894 if ((cc >= ' ' && cc < 0x7f)
895 || cc == '\t'
896 || cc == '\r'
897 || cc == '\n')
898 break;
899
900 this->lineoff_ = p - this->linebuf_;
901 go_error_at(this->location(),
902 "invalid character 0x%x in identifier",
903 cc);
904 if (!has_non_ascii_char)
905 {
906 buf.assign(pstart, p - pstart);
907 has_non_ascii_char = true;
908 }
909 if (!Lex::is_invalid_identifier(buf))
910 buf.append("$INVALID$");
911 }
912 ++p;
913 if (is_first)
914 {
915 is_exported = cc >= 'A' && cc <= 'Z';
916 is_first = false;
917 }
918 if (has_non_ascii_char)
919 buf.push_back(cc);
920 }
921 else
922 {
923 unsigned int ci;
924 bool issued_error;
925 this->lineoff_ = p - this->linebuf_;
926 const char* pnext = this->advance_one_utf8_char(p, &ci,
927 &issued_error);
928 bool is_invalid = false;
929 if (!Lex::is_unicode_letter(ci) && !Lex::is_unicode_digit(ci))
930 {
931 // There is no valid place for a non-ASCII character
932 // other than an identifier, so we get better error
933 // handling behaviour if we swallow this character after
934 // giving an error.
935 if (!issued_error)
936 go_error_at(this->location(),
937 "invalid character 0x%x in identifier",
938 ci);
939 is_invalid = true;
940 }
941 if (is_first)
942 {
943 is_exported = Lex::is_unicode_uppercase(ci);
944 is_first = false;
945 }
946 if (!has_non_ascii_char)
947 {
948 buf.assign(pstart, p - pstart);
949 has_non_ascii_char = true;
950 }
951 if (is_invalid && !Lex::is_invalid_identifier(buf))
952 buf.append("$INVALID$");
953 buf.append(p, pnext - p);
954 p = pnext;
955 }
956 }
957 Location location = this->location();
958 this->add_semi_at_eol_ = true;
959 this->lineoff_ = p - this->linebuf_;
960 if (has_non_ascii_char)
961 return Token::make_identifier_token(buf, is_exported, location);
962 else
963 {
964 Keyword code = keywords.keyword_to_code(pstart, p - pstart);
965 if (code == KEYWORD_INVALID)
966 return Token::make_identifier_token(std::string(pstart, p - pstart),
967 is_exported, location);
968 else
969 {
970 switch (code)
971 {
972 case KEYWORD_BREAK:
973 case KEYWORD_CONTINUE:
974 case KEYWORD_FALLTHROUGH:
975 case KEYWORD_RETURN:
976 break;
977 default:
978 this->add_semi_at_eol_ = false;
979 break;
980 }
981 return Token::make_keyword_token(code, location);
982 }
983 }
984 }
985
986 // Return whether C is a hex digit.
987
988 bool
989 Lex::is_hex_digit(char c)
990 {
991 return ((c >= '0' && c <= '9')
992 || (c >= 'A' && c <= 'F')
993 || (c >= 'a' && c <= 'f'));
994 }
995
996 // Return whether C is a valid digit in BASE.
997
998 bool
999 Lex::is_base_digit(int base, char c)
1000 {
1001 switch (base)
1002 {
1003 case 2:
1004 return c == '0' || c == '1';
1005 case 8:
1006 return c >= '0' && c <= '7';
1007 case 10:
1008 return c >= '0' && c <= '9';
1009 case 16:
1010 return Lex::is_hex_digit(c);
1011 default:
1012 go_unreachable();
1013 }
1014 }
1015
1016 // not a hex value
1017 #define NHV 100
1018
1019 // for use by Lex::hex_val
1020 static const unsigned char hex_value_lookup_table[256] =
1021 {
1022 NHV, NHV, NHV, NHV, NHV, NHV, NHV, NHV, // NUL SOH STX ETX EOT ENQ ACK BEL
1023 NHV, NHV, NHV, NHV, NHV, NHV, NHV, NHV, // BS HT LF VT FF CR SO SI
1024 NHV, NHV, NHV, NHV, NHV, NHV, NHV, NHV, // DLE DC1 DC2 DC3 DC4 NAK SYN ETB
1025 NHV, NHV, NHV, NHV, NHV, NHV, NHV, NHV, // CAN EM SUB ESC FS GS RS US
1026 NHV, NHV, NHV, NHV, NHV, NHV, NHV, NHV, // SP ! " # $ % & '
1027 NHV, NHV, NHV, NHV, NHV, NHV, NHV, NHV, // ( ) * + , - . /
1028 0, 1, 2, 3, 4, 5, 6, 7, // 0 1 2 3 4 5 6 7
1029 8, 9, NHV, NHV, NHV, NHV, NHV, NHV, // 8 9 : ; < = > ?
1030 NHV, 10, 11, 12, 13, 14, 15, NHV, // @ A B C D E F G
1031 NHV, NHV, NHV, NHV, NHV, NHV, NHV, NHV, // H I J K L M N O
1032 NHV, NHV, NHV, NHV, NHV, NHV, NHV, NHV, // P Q R S T U V W
1033 NHV, NHV, NHV, NHV, NHV, NHV, NHV, NHV, // X Y Z [ \ ] ^ _
1034 NHV, 10, 11, 12, 13, 14, 15, NHV, // ` a b c d e f g
1035 NHV, NHV, NHV, NHV, NHV, NHV, NHV, NHV, // h i j k l m n o
1036 NHV, NHV, NHV, NHV, NHV, NHV, NHV, NHV, // p q r s t u v w
1037 NHV, NHV, NHV, NHV, NHV, NHV, NHV, NHV, // x y z { | } ~
1038 NHV, NHV, NHV, NHV, NHV, NHV, NHV, NHV, //
1039 NHV, NHV, NHV, NHV, NHV, NHV, NHV, NHV, //
1040 NHV, NHV, NHV, NHV, NHV, NHV, NHV, NHV, //
1041 NHV, NHV, NHV, NHV, NHV, NHV, NHV, NHV, //
1042 NHV, NHV, NHV, NHV, NHV, NHV, NHV, NHV, //
1043 NHV, NHV, NHV, NHV, NHV, NHV, NHV, NHV, //
1044 NHV, NHV, NHV, NHV, NHV, NHV, NHV, NHV, //
1045 NHV, NHV, NHV, NHV, NHV, NHV, NHV, NHV, //
1046 NHV, NHV, NHV, NHV, NHV, NHV, NHV, NHV, //
1047 NHV, NHV, NHV, NHV, NHV, NHV, NHV, NHV, //
1048 NHV, NHV, NHV, NHV, NHV, NHV, NHV, NHV, //
1049 NHV, NHV, NHV, NHV, NHV, NHV, NHV, NHV, //
1050 NHV, NHV, NHV, NHV, NHV, NHV, NHV, NHV, //
1051 NHV, NHV, NHV, NHV, NHV, NHV, NHV, NHV, //
1052 NHV, NHV, NHV, NHV, NHV, NHV, NHV, NHV, //
1053 NHV, NHV, NHV, NHV, NHV, NHV, NHV, NHV //
1054 };
1055
1056 unsigned
1057 Lex::hex_val(char c)
1058 {
1059 return hex_value_lookup_table[static_cast<unsigned char>(c)];
1060 }
1061
1062 // Return whether an exponent could start at P, in base BASE.
1063
1064 bool
1065 Lex::could_be_exponent(int base, const char* p, const char* pend)
1066 {
1067 switch (base)
1068 {
1069 case 10:
1070 if (*p != 'e' && *p != 'E')
1071 return false;
1072 break;
1073 case 16:
1074 if (*p != 'p' && *p != 'P')
1075 return false;
1076 break;
1077 default:
1078 go_unreachable();
1079 }
1080 ++p;
1081 if (p >= pend)
1082 return false;
1083 if (*p == '+' || *p == '-')
1084 {
1085 ++p;
1086 if (p >= pend)
1087 return false;
1088 }
1089 return *p >= '0' && *p <= '9';
1090 }
1091
1092 // Pick up a number.
1093
1094 Token
1095 Lex::gather_number()
1096 {
1097 const char* pstart = this->linebuf_ + this->lineoff_;
1098 const char* p = pstart;
1099 const char* pend = this->linebuf_ + this->linesize_;
1100
1101 Location location = this->location();
1102
1103 int base = 10;
1104 std::string num;
1105 if (*p == '0')
1106 {
1107 int basecheck;
1108 int off;
1109 if (p[1] == 'x' || p[1] == 'X')
1110 {
1111 base = 16;
1112 basecheck = 16;
1113 off = 2;
1114 }
1115 else if (p[1] == 'o' || p[1] == 'O')
1116 {
1117 base = 8;
1118 basecheck = 8;
1119 off = 2;
1120 }
1121 else if (p[1] == 'b' || p[1] == 'B')
1122 {
1123 base = 2;
1124 basecheck = 2;
1125 off = 2;
1126 }
1127 else
1128 {
1129 // Old style octal literal. May also be the start of a
1130 // floating-point number (e.g., 09.2, 09e2) or an imaginary
1131 // literal (e.g., 09i), so we have to accept decimal digits.
1132 base = 8;
1133 basecheck = 10;
1134 off = 0;
1135 }
1136
1137 p += off;
1138 if (*p == '_' && Lex::is_base_digit(basecheck, p[1]))
1139 ++p;
1140
1141 while (Lex::is_base_digit(basecheck, *p))
1142 {
1143 num.push_back(*p);
1144 ++p;
1145 if (*p == '_' && Lex::is_base_digit(basecheck, p[1]))
1146 ++p;
1147 }
1148
1149 // We must see at least one valid digit, except for a case like
1150 // 0x.0p1.
1151 if (num.length() == 0 && (base != 16 || *p != '.'))
1152 {
1153 go_error_at(this->location(), "invalid numeric literal");
1154 this->lineoff_ = p - this->linebuf_;
1155 mpz_t val;
1156 mpz_init_set_ui(val, 0);
1157 Token ret = Token::make_integer_token(val, location);
1158 mpz_clear(val);
1159 return ret;
1160 }
1161
1162 bool is_float = false;
1163 // A number that looks like an old-style octal literal might
1164 // actually be the beginning of a floating-point or imaginary
1165 // literal, in which case the value is decimal digits. Handle
1166 // that case below by treating the leading '0' as decimal.
1167 if (off == 0
1168 && (*p == '.' || *p == 'i' || Lex::could_be_exponent(10, p, pend)))
1169 {
1170 is_float = true;
1171 base = 10;
1172 }
1173 else if (base == 16
1174 && (*p == '.' || Lex::could_be_exponent(16, p, pend)))
1175 is_float = true;
1176
1177 if (!is_float)
1178 {
1179 mpz_t val;
1180 int r = mpz_init_set_str(val, num.c_str(), base);
1181 if (r != 0)
1182 {
1183 const char *errword;
1184 switch (base)
1185 {
1186 case 2:
1187 errword = "binary";
1188 break;
1189 case 8:
1190 errword = "octal";
1191 break;
1192 case 16:
1193 errword = "hex";
1194 break;
1195 default:
1196 go_unreachable();
1197 }
1198 go_error_at(this->location(), "invalid %s literal", errword);
1199 }
1200
1201 bool is_imaginary = *p == 'i';
1202 if (is_imaginary)
1203 ++p;
1204
1205 this->lineoff_ = p - this->linebuf_;
1206
1207 if (*p == 'e' || *p == 'E' || *p == 'p' || *p == 'P')
1208 {
1209 go_error_at(location,
1210 "invalid prefix for floating constant");
1211 this->skip_exponent();
1212 }
1213
1214 if (!is_imaginary)
1215 {
1216 Token ret = Token::make_integer_token(val, location);
1217 mpz_clear(val);
1218 return ret;
1219 }
1220 else
1221 {
1222 mpfr_t ival;
1223 mpfr_init_set_z(ival, val, MPFR_RNDN);
1224 mpz_clear(val);
1225 Token ret = Token::make_imaginary_token(ival, location);
1226 mpfr_clear(ival);
1227 return ret;
1228 }
1229 }
1230 }
1231
1232 while (p < pend)
1233 {
1234 if (*p == '_' && p[1] >= '0' && p[1] <= '9')
1235 ++p;
1236 else if (*p < '0' || *p > '9')
1237 break;
1238 num.push_back(*p);
1239 ++p;
1240 }
1241
1242 if (*p != '.' && *p != 'i' && !Lex::could_be_exponent(base, p, pend))
1243 {
1244 mpz_t val;
1245 int r = mpz_init_set_str(val, num.c_str(), 10);
1246 go_assert(r == 0);
1247
1248 this->lineoff_ = p - this->linebuf_;
1249
1250 if (*p == 'e' || *p == 'E' || *p == 'p' || *p == 'P')
1251 {
1252 go_error_at(location,
1253 "invalid prefix for floating constant");
1254 this->skip_exponent();
1255 }
1256
1257 Token ret = Token::make_integer_token(val, location);
1258 mpz_clear(val);
1259 return ret;
1260 }
1261
1262 if (*p != 'i')
1263 {
1264 bool dot = *p == '.';
1265
1266 num.push_back(*p);
1267 ++p;
1268
1269 if (!dot)
1270 {
1271 if (*p == '+' || *p == '-')
1272 {
1273 num.push_back(*p);
1274 ++p;
1275 }
1276 }
1277
1278 bool first = true;
1279 while (p < pend)
1280 {
1281 if (!first && *p == '_' && Lex::is_base_digit(base, p[1]))
1282 ++p;
1283 else if (!Lex::is_base_digit(base, *p))
1284 break;
1285 num.push_back(*p);
1286 ++p;
1287 first = false;
1288 }
1289
1290 if (dot && Lex::could_be_exponent(base, p, pend))
1291 {
1292 num.push_back(*p);
1293 ++p;
1294 if (*p == '+' || *p == '-')
1295 {
1296 num.push_back(*p);
1297 ++p;
1298 }
1299 first = true;
1300 while (p < pend)
1301 {
1302 if (!first && *p == '_' && p[1] >= '0' && p[1] <= '9')
1303 ++p;
1304 else if (*p < '0' || *p > '9')
1305 break;
1306 num.push_back(*p);
1307 ++p;
1308 first = false;
1309 }
1310 }
1311 else if (dot && base == 16)
1312 {
1313 go_error_at(this->location(),
1314 "invalid hex floating-point literal with no exponent");
1315 num.append("p0");
1316 }
1317 }
1318
1319 mpfr_clear_overflow();
1320 mpfr_t val;
1321 int r = mpfr_init_set_str(val, num.c_str(), base, MPFR_RNDN);
1322 go_assert(r == 0);
1323 if (mpfr_overflow_p())
1324 go_error_at(this->location(),
1325 "floating-point exponent too large to represent");
1326
1327 bool is_imaginary = *p == 'i';
1328 if (is_imaginary)
1329 ++p;
1330
1331 this->lineoff_ = p - this->linebuf_;
1332
1333 if (*p == 'e' || *p == 'E' || *p == 'p' || *p == 'P')
1334 {
1335 go_error_at(location,
1336 "invalid prefix for floating constant");
1337 this->skip_exponent();
1338 }
1339
1340 if (is_imaginary)
1341 {
1342 Token ret = Token::make_imaginary_token(val, location);
1343 mpfr_clear(val);
1344 return ret;
1345 }
1346 else
1347 {
1348 Token ret = Token::make_float_token(val, location);
1349 mpfr_clear(val);
1350 return ret;
1351 }
1352 }
1353
1354 // Skip an exponent after reporting an error.
1355
1356 void
1357 Lex::skip_exponent()
1358 {
1359 const char* p = this->linebuf_ + this->lineoff_;
1360 const char* pend = this->linebuf_ + this->linesize_;
1361 if (*p != 'e' && *p != 'E' && *p != 'p' && *p != 'P')
1362 return;
1363 ++p;
1364 if (*p == '+' || *p == '-')
1365 ++p;
1366 while (p < pend)
1367 {
1368 if ((*p < '0' || *p > '9') && *p != '_')
1369 break;
1370 ++p;
1371 }
1372 this->lineoff_ = p - this->linebuf_;
1373 }
1374
1375 // Advance one character, possibly escaped. Return the pointer beyond
1376 // the character. Set *VALUE to the character. Set *IS_CHARACTER if
1377 // this is a character (e.g., 'a' or '\u1234') rather than a byte
1378 // value (e.g., '\001').
1379
1380 const char*
1381 Lex::advance_one_char(const char* p, bool is_single_quote, unsigned int* value,
1382 bool* is_character)
1383 {
1384 *value = 0;
1385 *is_character = true;
1386 if (*p != '\\')
1387 {
1388 bool issued_error;
1389 const char* ret = this->advance_one_utf8_char(p, value, &issued_error);
1390 if (is_single_quote
1391 && (*value == '\'' || *value == '\n')
1392 && !issued_error)
1393 go_error_at(this->location(), "invalid character literal");
1394 return ret;
1395 }
1396 else
1397 {
1398 ++p;
1399 switch (*p)
1400 {
1401 case '0': case '1': case '2': case '3':
1402 case '4': case '5': case '6': case '7':
1403 *is_character = false;
1404 if (p[1] >= '0' && p[1] <= '7'
1405 && p[2] >= '0' && p[2] <= '7')
1406 {
1407 *value = ((Lex::octal_value(p[0]) << 6)
1408 + (Lex::octal_value(p[1]) << 3)
1409 + Lex::octal_value(p[2]));
1410 if (*value > 255)
1411 {
1412 go_error_at(this->location(), "invalid octal constant");
1413 *value = 255;
1414 }
1415 return p + 3;
1416 }
1417 go_error_at(this->location(), "invalid octal character");
1418 return (p[1] >= '0' && p[1] <= '7'
1419 ? p + 2
1420 : p + 1);
1421
1422 case 'x':
1423 *is_character = false;
1424 if (Lex::is_hex_digit(p[1]) && Lex::is_hex_digit(p[2]))
1425 {
1426 *value = (Lex::hex_val(p[1]) << 4) + Lex::hex_val(p[2]);
1427 return p + 3;
1428 }
1429 go_error_at(this->location(), "invalid hex character");
1430 return (Lex::is_hex_digit(p[1])
1431 ? p + 2
1432 : p + 1);
1433
1434 case 'a':
1435 *value = '\a';
1436 return p + 1;
1437 case 'b':
1438 *value = '\b';
1439 return p + 1;
1440 case 'f':
1441 *value = '\f';
1442 return p + 1;
1443 case 'n':
1444 *value = '\n';
1445 return p + 1;
1446 case 'r':
1447 *value = '\r';
1448 return p + 1;
1449 case 't':
1450 *value = '\t';
1451 return p + 1;
1452 case 'v':
1453 *value = '\v';
1454 return p + 1;
1455 case '\\':
1456 *value = '\\';
1457 return p + 1;
1458 case '\'':
1459 if (!is_single_quote)
1460 go_error_at(this->location(), "invalid quoted character");
1461 *value = '\'';
1462 return p + 1;
1463 case '"':
1464 if (is_single_quote)
1465 go_error_at(this->location(), "invalid quoted character");
1466 *value = '"';
1467 return p + 1;
1468
1469 case 'u':
1470 if (Lex::is_hex_digit(p[1]) && Lex::is_hex_digit(p[2])
1471 && Lex::is_hex_digit(p[3]) && Lex::is_hex_digit(p[4]))
1472 {
1473 *value = ((Lex::hex_val(p[1]) << 12)
1474 + (Lex::hex_val(p[2]) << 8)
1475 + (Lex::hex_val(p[3]) << 4)
1476 + Lex::hex_val(p[4]));
1477 if (*value >= 0xd800 && *value < 0xe000)
1478 {
1479 go_error_at(this->location(),
1480 "invalid unicode code point 0x%x",
1481 *value);
1482 // Use the replacement character.
1483 *value = 0xfffd;
1484 }
1485 return p + 5;
1486 }
1487 go_error_at(this->location(), "invalid little unicode code point");
1488 return p + 1;
1489
1490 case 'U':
1491 if (Lex::is_hex_digit(p[1]) && Lex::is_hex_digit(p[2])
1492 && Lex::is_hex_digit(p[3]) && Lex::is_hex_digit(p[4])
1493 && Lex::is_hex_digit(p[5]) && Lex::is_hex_digit(p[6])
1494 && Lex::is_hex_digit(p[7]) && Lex::is_hex_digit(p[8]))
1495 {
1496 *value = ((Lex::hex_val(p[1]) << 28)
1497 + (Lex::hex_val(p[2]) << 24)
1498 + (Lex::hex_val(p[3]) << 20)
1499 + (Lex::hex_val(p[4]) << 16)
1500 + (Lex::hex_val(p[5]) << 12)
1501 + (Lex::hex_val(p[6]) << 8)
1502 + (Lex::hex_val(p[7]) << 4)
1503 + Lex::hex_val(p[8]));
1504 if (*value > 0x10ffff
1505 || (*value >= 0xd800 && *value < 0xe000))
1506 {
1507 go_error_at(this->location(),
1508 "invalid unicode code point 0x%x",
1509 *value);
1510 // Use the replacement character.
1511 *value = 0xfffd;
1512 }
1513 return p + 9;
1514 }
1515 go_error_at(this->location(), "invalid big unicode code point");
1516 return p + 1;
1517
1518 default:
1519 go_error_at(this->location(), "invalid character after %<\\%>");
1520 *value = *p;
1521 return p + 1;
1522 }
1523 }
1524 }
1525
1526 // Append V to STR. IS_CHARACTER is true for a character which should
1527 // be stored in UTF-8, false for a general byte value which should be
1528 // stored directly.
1529
1530 void
1531 Lex::append_char(unsigned int v, bool is_character, std::string* str,
1532 Location location)
1533 {
1534 char buf[4];
1535 size_t len;
1536 if (v <= 0x7f || !is_character)
1537 {
1538 buf[0] = v;
1539 len = 1;
1540 }
1541 else if (v <= 0x7ff)
1542 {
1543 buf[0] = 0xc0 + (v >> 6);
1544 buf[1] = 0x80 + (v & 0x3f);
1545 len = 2;
1546 }
1547 else
1548 {
1549 if (v > 0x10ffff)
1550 {
1551 go_warning_at(location, 0,
1552 "unicode code point 0x%x out of range in string", v);
1553 // Turn it into the "replacement character".
1554 v = 0xfffd;
1555 }
1556 if (v >= 0xd800 && v < 0xe000)
1557 {
1558 go_warning_at(location, 0,
1559 "unicode code point 0x%x is invalid surrogate pair", v);
1560 v = 0xfffd;
1561 }
1562 if (v <= 0xffff)
1563 {
1564 buf[0] = 0xe0 + (v >> 12);
1565 buf[1] = 0x80 + ((v >> 6) & 0x3f);
1566 buf[2] = 0x80 + (v & 0x3f);
1567 len = 3;
1568 }
1569 else
1570 {
1571 buf[0] = 0xf0 + (v >> 18);
1572 buf[1] = 0x80 + ((v >> 12) & 0x3f);
1573 buf[2] = 0x80 + ((v >> 6) & 0x3f);
1574 buf[3] = 0x80 + (v & 0x3f);
1575 len = 4;
1576 }
1577 }
1578 str->append(buf, len);
1579 }
1580
1581 // Pick up a character literal.
1582
1583 Token
1584 Lex::gather_character()
1585 {
1586 ++this->lineoff_;
1587 const char* pstart = this->linebuf_ + this->lineoff_;
1588 const char* p = pstart;
1589
1590 unsigned int value;
1591 bool is_character;
1592 p = this->advance_one_char(p, true, &value, &is_character);
1593
1594 if (*p != '\'')
1595 {
1596 go_error_at(this->location(), "unterminated character constant");
1597 this->lineoff_ = p - this->linebuf_;
1598 return this->make_invalid_token();
1599 }
1600
1601 mpz_t val;
1602 mpz_init_set_ui(val, value);
1603
1604 Location location = this->location();
1605 this->lineoff_ = p + 1 - this->linebuf_;
1606 Token ret = Token::make_character_token(val, location);
1607 mpz_clear(val);
1608 return ret;
1609 }
1610
1611 // Pick up a quoted string.
1612
1613 Token
1614 Lex::gather_string()
1615 {
1616 const char* pstart = this->linebuf_ + this->lineoff_ + 1;
1617 const char* p = pstart;
1618 const char* pend = this->linebuf_ + this->linesize_;
1619
1620 std::string value;
1621 while (*p != '"')
1622 {
1623 Location loc = this->location();
1624 unsigned int c;
1625 bool is_character;
1626 this->lineoff_ = p - this->linebuf_;
1627 p = this->advance_one_char(p, false, &c, &is_character);
1628 if (p >= pend)
1629 {
1630 go_error_at(this->location(), "unterminated string");
1631 --p;
1632 break;
1633 }
1634 Lex::append_char(c, is_character, &value, loc);
1635 }
1636
1637 Location location = this->location();
1638 this->lineoff_ = p + 1 - this->linebuf_;
1639 return Token::make_string_token(value, location);
1640 }
1641
1642 // Pick up a raw string.
1643
1644 Token
1645 Lex::gather_raw_string()
1646 {
1647 const char* p = this->linebuf_ + this->lineoff_ + 1;
1648 const char* pend = this->linebuf_ + this->linesize_;
1649 Location location = this->location();
1650
1651 std::string value;
1652 while (true)
1653 {
1654 while (p < pend)
1655 {
1656 if (*p == '`')
1657 {
1658 this->lineoff_ = p + 1 - this->linebuf_;
1659 return Token::make_string_token(value, location);
1660 }
1661 Location loc = this->location();
1662 unsigned int c;
1663 bool issued_error;
1664 this->lineoff_ = p - this->linebuf_;
1665 p = this->advance_one_utf8_char(p, &c, &issued_error);
1666 // "Carriage return characters ('\r') inside raw string literals
1667 // are discarded from the raw string value."
1668 if (c != '\r')
1669 Lex::append_char(c, true, &value, loc);
1670 }
1671 this->lineoff_ = p - this->linebuf_;
1672 if (!this->require_line())
1673 {
1674 go_error_at(location, "unterminated raw string");
1675 return Token::make_string_token(value, location);
1676 }
1677 p = this->linebuf_ + this->lineoff_;
1678 pend = this->linebuf_ + this->linesize_;
1679 }
1680 }
1681
1682 // If C1 C2 C3 are a three character operator, return the code.
1683
1684 Operator
1685 Lex::three_character_operator(char c1, char c2, char c3)
1686 {
1687 if (c3 == '=')
1688 {
1689 if (c1 == '<' && c2 == '<')
1690 return OPERATOR_LSHIFTEQ;
1691 else if (c1 == '>' && c2 == '>')
1692 return OPERATOR_RSHIFTEQ;
1693 else if (c1 == '&' && c2 == '^')
1694 return OPERATOR_BITCLEAREQ;
1695 }
1696 return OPERATOR_INVALID;
1697 }
1698
1699 // If C1 C2 are a two character operator, return the code.
1700
1701 Operator
1702 Lex::two_character_operator(char c1, char c2)
1703 {
1704 switch (c1)
1705 {
1706 case '|':
1707 if (c2 == '|')
1708 return OPERATOR_OROR;
1709 else if (c2 == '=')
1710 return OPERATOR_OREQ;
1711 break;
1712 case '&':
1713 if (c2 == '&')
1714 return OPERATOR_ANDAND;
1715 else if (c2 == '^')
1716 return OPERATOR_BITCLEAR;
1717 else if (c2 == '=')
1718 return OPERATOR_ANDEQ;
1719 break;
1720 case '^':
1721 if (c2 == '=')
1722 return OPERATOR_XOREQ;
1723 break;
1724 case '=':
1725 if (c2 == '=')
1726 return OPERATOR_EQEQ;
1727 break;
1728 case '!':
1729 if (c2 == '=')
1730 return OPERATOR_NOTEQ;
1731 break;
1732 case '<':
1733 if (c2 == '=')
1734 return OPERATOR_LE;
1735 else if (c2 == '<')
1736 return OPERATOR_LSHIFT;
1737 else if (c2 == '-')
1738 return OPERATOR_CHANOP;
1739 break;
1740 case '>':
1741 if (c2 == '=')
1742 return OPERATOR_GE;
1743 else if (c2 == '>')
1744 return OPERATOR_RSHIFT;
1745 break;
1746 case '*':
1747 if (c2 == '=')
1748 return OPERATOR_MULTEQ;
1749 break;
1750 case '/':
1751 if (c2 == '=')
1752 return OPERATOR_DIVEQ;
1753 break;
1754 case '%':
1755 if (c2 == '=')
1756 return OPERATOR_MODEQ;
1757 break;
1758 case '+':
1759 if (c2 == '+')
1760 {
1761 this->add_semi_at_eol_ = true;
1762 return OPERATOR_PLUSPLUS;
1763 }
1764 else if (c2 == '=')
1765 return OPERATOR_PLUSEQ;
1766 break;
1767 case '-':
1768 if (c2 == '-')
1769 {
1770 this->add_semi_at_eol_ = true;
1771 return OPERATOR_MINUSMINUS;
1772 }
1773 else if (c2 == '=')
1774 return OPERATOR_MINUSEQ;
1775 break;
1776 case ':':
1777 if (c2 == '=')
1778 return OPERATOR_COLONEQ;
1779 break;
1780 default:
1781 break;
1782 }
1783 return OPERATOR_INVALID;
1784 }
1785
1786 // If character C is an operator, return the code.
1787
1788 Operator
1789 Lex::one_character_operator(char c)
1790 {
1791 switch (c)
1792 {
1793 case '<':
1794 return OPERATOR_LT;
1795 case '>':
1796 return OPERATOR_GT;
1797 case '+':
1798 return OPERATOR_PLUS;
1799 case '-':
1800 return OPERATOR_MINUS;
1801 case '|':
1802 return OPERATOR_OR;
1803 case '^':
1804 return OPERATOR_XOR;
1805 case '*':
1806 return OPERATOR_MULT;
1807 case '/':
1808 return OPERATOR_DIV;
1809 case '%':
1810 return OPERATOR_MOD;
1811 case '&':
1812 return OPERATOR_AND;
1813 case '!':
1814 return OPERATOR_NOT;
1815 case '=':
1816 return OPERATOR_EQ;
1817 case ':':
1818 return OPERATOR_COLON;
1819 case ';':
1820 return OPERATOR_SEMICOLON;
1821 case '.':
1822 return OPERATOR_DOT;
1823 case ',':
1824 return OPERATOR_COMMA;
1825 case '(':
1826 return OPERATOR_LPAREN;
1827 case ')':
1828 this->add_semi_at_eol_ = true;
1829 return OPERATOR_RPAREN;
1830 case '{':
1831 return OPERATOR_LCURLY;
1832 case '}':
1833 this->add_semi_at_eol_ = true;
1834 return OPERATOR_RCURLY;
1835 case '[':
1836 return OPERATOR_LSQUARE;
1837 case ']':
1838 this->add_semi_at_eol_ = true;
1839 return OPERATOR_RSQUARE;
1840 default:
1841 return OPERATOR_INVALID;
1842 }
1843 }
1844
1845 // Skip a C-style comment.
1846
1847 bool
1848 Lex::skip_c_comment(bool* found_newline)
1849 {
1850 while (true)
1851 {
1852 if (!this->require_line())
1853 {
1854 go_error_at(this->location(), "unterminated comment");
1855 return false;
1856 }
1857
1858 const char* p = this->linebuf_ + this->lineoff_;
1859 const char* pend = this->linebuf_ + this->linesize_;
1860
1861 while (p < pend)
1862 {
1863 if (p[0] == '*' && p + 1 < pend && p[1] == '/')
1864 {
1865 this->lineoff_ = p + 2 - this->linebuf_;
1866 return true;
1867 }
1868
1869 if (p[0] == '\n')
1870 *found_newline = true;
1871
1872 this->lineoff_ = p - this->linebuf_;
1873 unsigned int c;
1874 bool issued_error;
1875 p = this->advance_one_utf8_char(p, &c, &issued_error);
1876 }
1877
1878 this->lineoff_ = p - this->linebuf_;
1879 }
1880 }
1881
1882 // Skip a C++-style comment.
1883
1884 void
1885 Lex::skip_cpp_comment()
1886 {
1887 // Ensure that if EXTERN_ is set, it means that we just saw a
1888 // //extern comment.
1889 this->extern_.clear();
1890
1891 Location loc = this->location();
1892 size_t lineoff = this->lineoff_;
1893
1894 const char* p = this->linebuf_ + lineoff;
1895 const char* pend = this->linebuf_ + this->linesize_;
1896
1897 const char* pcheck = p;
1898 bool saw_error = false;
1899 while (pcheck < pend)
1900 {
1901 this->lineoff_ = pcheck - this->linebuf_;
1902 unsigned int c;
1903 bool issued_error;
1904 pcheck = this->advance_one_utf8_char(pcheck, &c, &issued_error);
1905 if (issued_error)
1906 saw_error = true;
1907 }
1908
1909 if (saw_error)
1910 return;
1911
1912 // Recognize various magic comments at the start of a line.
1913
1914 if (lineoff != 2)
1915 {
1916 // Not at the start of the line. (lineoff == 2 because of the
1917 // two characters in "//").
1918 return;
1919 }
1920
1921 while (pend > p
1922 && (pend[-1] == ' ' || pend[-1] == '\t'
1923 || pend[-1] == '\r' || pend[-1] == '\n'))
1924 --pend;
1925
1926 // A C++ comment at the start of the line of the form
1927 // //line FILE:LINENO
1928 // is interpreted as setting the file name and line number of the
1929 // next source line.
1930 if (pend - p > 5 && memcmp(p, "line ", 5) == 0)
1931 {
1932 p += 5;
1933 while (p < pend && *p == ' ')
1934 ++p;
1935 const char* pcolon = static_cast<const char*>(memchr(p, ':', pend - p));
1936 if (pcolon != NULL
1937 && pcolon[1] >= '0'
1938 && pcolon[1] <= '9')
1939 {
1940 char* plend;
1941 long lineno = strtol(pcolon + 1, &plend, 10);
1942 if (plend > pcolon + 1
1943 && (plend == pend
1944 || *plend < '0'
1945 || *plend > '9')
1946 && lineno > 0
1947 && lineno < 0x7fffffff)
1948 {
1949 unsigned int filelen = pcolon - p;
1950 char* file = new char[filelen + 1];
1951 memcpy(file, p, filelen);
1952 file[filelen] = '\0';
1953
1954 this->linemap_->start_file(file, lineno);
1955 this->lineno_ = lineno - 1;
1956
1957 p = plend;
1958 }
1959 }
1960 return;
1961 }
1962
1963 // As a special gccgo extension, a C++ comment at the start of the
1964 // line of the form
1965 // //extern NAME
1966 // which immediately precedes a function declaration means that the
1967 // external name of the function declaration is NAME. This is
1968 // normally used to permit Go code to call a C function.
1969 if (pend - p > 7 && memcmp(p, "extern ", 7) == 0)
1970 {
1971 p += 7;
1972 while (p < pend && (*p == ' ' || *p == '\t'))
1973 ++p;
1974 if (pend > p)
1975 this->extern_ = std::string(p, pend - p);
1976 return;
1977 }
1978
1979 // All other special comments start with "go:".
1980
1981 if (pend - p < 4 || memcmp(p, "go:", 3) != 0)
1982 return;
1983
1984 const char *ps = p + 3;
1985 while (ps < pend && *ps != ' ' && *ps != '\t')
1986 ++ps;
1987 std::string verb = std::string(p, ps - p);
1988
1989 if (verb == "go:linkname")
1990 {
1991 // As in the gc compiler, set the external link name for a Go symbol.
1992 std::string go_name;
1993 std::string ext_name;
1994 bool is_exported = false;
1995 if (ps < pend)
1996 {
1997 while (ps < pend && (*ps == ' ' || *ps == '\t'))
1998 ++ps;
1999 if (ps < pend)
2000 {
2001 const char* pg = ps;
2002
2003 unsigned int c;
2004 bool issued_error;
2005 ps = this->advance_one_utf8_char(ps, &c, &issued_error);
2006 is_exported = Lex::is_unicode_uppercase(c);
2007
2008 while (ps < pend && *ps != ' ' && *ps != '\t')
2009 ++ps;
2010 if (ps <= pend)
2011 go_name = std::string(pg, ps - pg);
2012 while (ps < pend && (*ps == ' ' || *ps == '\t'))
2013 ++ps;
2014 }
2015 if (ps < pend)
2016 {
2017 const char* pc = ps;
2018 while (ps < pend && *ps != ' ' && *ps != '\t')
2019 ++ps;
2020 if (ps <= pend)
2021 ext_name = std::string(pc, ps - pc);
2022 }
2023 if (ps != pend)
2024 {
2025 go_name.clear();
2026 ext_name.clear();
2027 }
2028 }
2029 if (go_name.empty())
2030 go_error_at(loc, "usage: %<//go:linkname%> localname [linkname]");
2031 else
2032 {
2033 if (this->linknames_ == NULL)
2034 this->linknames_ = new Linknames();
2035 (*this->linknames_)[go_name] = Linkname(ext_name, is_exported, loc);
2036 }
2037 }
2038 else if (verb == "go:nointerface")
2039 {
2040 // For field tracking analysis: a //go:nointerface comment means
2041 // that the next interface method should not be stored in the
2042 // type descriptor. This permits it to be discarded if it is
2043 // not needed.
2044 this->pragmas_ |= GOPRAGMA_NOINTERFACE;
2045 }
2046 else if (verb == "go:noescape")
2047 {
2048 // Applies to the next function declaration. Any arguments do
2049 // not escape.
2050 // FIXME: Not implemented.
2051 this->pragmas_ |= GOPRAGMA_NOESCAPE;
2052 }
2053 else if (verb == "go:nosplit")
2054 {
2055 // Applies to the next function. Do not split the stack when
2056 // entering the function.
2057 this->pragmas_ |= GOPRAGMA_NOSPLIT;
2058 }
2059 else if (verb == "go:noinline")
2060 {
2061 // Applies to the next function. Do not inline the function.
2062 this->pragmas_ |= GOPRAGMA_NOINLINE;
2063 }
2064 else if (verb == "go:notinheap")
2065 {
2066 // Applies to the next type. The type does not live in the heap.
2067 this->pragmas_ |= GOPRAGMA_NOTINHEAP;
2068 }
2069 else if (verb == "go:systemstack")
2070 {
2071 // Applies to the next function. It must run on the system stack.
2072 // FIXME: Should only work when compiling the runtime package.
2073 // FIXME: Not implemented.
2074 this->pragmas_ |= GOPRAGMA_SYSTEMSTACK;
2075 }
2076 else if (verb == "go:nowritebarrier")
2077 {
2078 // Applies to the next function. If the function needs to use
2079 // any write barriers, it should emit an error instead.
2080 // FIXME: Should only work when compiling the runtime package.
2081 this->pragmas_ |= GOPRAGMA_NOWRITEBARRIER;
2082 }
2083 else if (verb == "go:nowritebarrierrec")
2084 {
2085 // Applies to the next function. If the function, or any
2086 // function that it calls, needs to use any write barriers, it
2087 // should emit an error instead.
2088 // FIXME: Should only work when compiling the runtime package.
2089 this->pragmas_ |= GOPRAGMA_NOWRITEBARRIERREC;
2090 }
2091 else if (verb == "go:yeswritebarrierrec")
2092 {
2093 // Applies to the next function. Disables go:nowritebarrierrec
2094 // when looking at callees; write barriers are permitted here.
2095 // FIXME: Should only work when compiling the runtime package.
2096 this->pragmas_ |= GOPRAGMA_YESWRITEBARRIERREC;
2097 }
2098 else if (verb == "go:cgo_unsafe_args")
2099 {
2100 // Applies to the next function. Taking the address of any
2101 // argument implies taking the address of all arguments.
2102 // FIXME: Not implemented.
2103 this->pragmas_ |= GOPRAGMA_CGOUNSAFEARGS;
2104 }
2105 else if (verb == "go:uintptrescapes")
2106 {
2107 // Applies to the next function. If an argument is a pointer
2108 // converted to uintptr, then the pointer escapes.
2109 // FIXME: Not implemented.
2110 this->pragmas_ |= GOPRAGMA_UINTPTRESCAPES;
2111 }
2112 }
2113
2114 // The Unicode tables use this struct.
2115
2116 struct Unicode_range
2117 {
2118 // The low end of the range.
2119 unsigned int low;
2120 // The high end of the range.
2121 unsigned int high;
2122 // The stride. This entries represents low, low + stride, low + 2 *
2123 // stride, etc., up to high.
2124 unsigned int stride;
2125 };
2126
2127 // A table of whitespace characters--Unicode code points classified as
2128 // "Space", "C" locale whitespace characters, the "next line" control
2129 // character (0085), the line separator (2028), the paragraph
2130 // separator (2029), and the "zero-width non-break space" (feff).
2131
2132 static const Unicode_range unicode_space[] =
2133 {
2134 { 0x0009, 0x000d, 1 },
2135 { 0x0020, 0x0020, 1 },
2136 { 0x0085, 0x0085, 1 },
2137 { 0x00a0, 0x00a0, 1 },
2138 { 0x1680, 0x1680, 1 },
2139 { 0x180e, 0x180e, 1 },
2140 { 0x2000, 0x200a, 1 },
2141 { 0x2028, 0x2029, 1 },
2142 { 0x202f, 0x202f, 1 },
2143 { 0x205f, 0x205f, 1 },
2144 { 0x3000, 0x3000, 1 },
2145 { 0xfeff, 0xfeff, 1 },
2146 };
2147
2148 // A table of Unicode digits--Unicode code points classified as
2149 // "Digit".
2150
2151 static const Unicode_range unicode_digits[] =
2152 {
2153 { 0x0030, 0x0039, 1},
2154 { 0x0660, 0x0669, 1},
2155 { 0x06f0, 0x06f9, 1},
2156 { 0x07c0, 0x07c9, 1},
2157 { 0x0966, 0x096f, 1},
2158 { 0x09e6, 0x09ef, 1},
2159 { 0x0a66, 0x0a6f, 1},
2160 { 0x0ae6, 0x0aef, 1},
2161 { 0x0b66, 0x0b6f, 1},
2162 { 0x0be6, 0x0bef, 1},
2163 { 0x0c66, 0x0c6f, 1},
2164 { 0x0ce6, 0x0cef, 1},
2165 { 0x0d66, 0x0d6f, 1},
2166 { 0x0e50, 0x0e59, 1},
2167 { 0x0ed0, 0x0ed9, 1},
2168 { 0x0f20, 0x0f29, 1},
2169 { 0x1040, 0x1049, 1},
2170 { 0x17e0, 0x17e9, 1},
2171 { 0x1810, 0x1819, 1},
2172 { 0x1946, 0x194f, 1},
2173 { 0x19d0, 0x19d9, 1},
2174 { 0x1b50, 0x1b59, 1},
2175 { 0xff10, 0xff19, 1},
2176 { 0x104a0, 0x104a9, 1},
2177 { 0x1d7ce, 0x1d7ff, 1},
2178 };
2179
2180 // A table of Unicode letters--Unicode code points classified as
2181 // "Letter".
2182
2183 static const Unicode_range unicode_letters[] =
2184 {
2185 { 0x0041, 0x005a, 1},
2186 { 0x0061, 0x007a, 1},
2187 { 0x00aa, 0x00b5, 11},
2188 { 0x00ba, 0x00c0, 6},
2189 { 0x00c1, 0x00d6, 1},
2190 { 0x00d8, 0x00f6, 1},
2191 { 0x00f8, 0x02c1, 1},
2192 { 0x02c6, 0x02d1, 1},
2193 { 0x02e0, 0x02e4, 1},
2194 { 0x02ec, 0x02ee, 2},
2195 { 0x0370, 0x0374, 1},
2196 { 0x0376, 0x0377, 1},
2197 { 0x037a, 0x037d, 1},
2198 { 0x037f, 0x0386, 7},
2199 { 0x0388, 0x038a, 1},
2200 { 0x038c, 0x038e, 2},
2201 { 0x038f, 0x03a1, 1},
2202 { 0x03a3, 0x03f5, 1},
2203 { 0x03f7, 0x0481, 1},
2204 { 0x048a, 0x052f, 1},
2205 { 0x0531, 0x0556, 1},
2206 { 0x0559, 0x0561, 8},
2207 { 0x0562, 0x0587, 1},
2208 { 0x05d0, 0x05ea, 1},
2209 { 0x05f0, 0x05f2, 1},
2210 { 0x0620, 0x064a, 1},
2211 { 0x066e, 0x066f, 1},
2212 { 0x0671, 0x06d3, 1},
2213 { 0x06d5, 0x06e5, 16},
2214 { 0x06e6, 0x06ee, 8},
2215 { 0x06ef, 0x06fa, 11},
2216 { 0x06fb, 0x06fc, 1},
2217 { 0x06ff, 0x0710, 17},
2218 { 0x0712, 0x072f, 1},
2219 { 0x074d, 0x07a5, 1},
2220 { 0x07b1, 0x07ca, 25},
2221 { 0x07cb, 0x07ea, 1},
2222 { 0x07f4, 0x07f5, 1},
2223 { 0x07fa, 0x0800, 6},
2224 { 0x0801, 0x0815, 1},
2225 { 0x081a, 0x0824, 10},
2226 { 0x0828, 0x0840, 24},
2227 { 0x0841, 0x0858, 1},
2228 { 0x08a0, 0x08b4, 1},
2229 { 0x0904, 0x0939, 1},
2230 { 0x093d, 0x0950, 19},
2231 { 0x0958, 0x0961, 1},
2232 { 0x0971, 0x0980, 1},
2233 { 0x0985, 0x098c, 1},
2234 { 0x098f, 0x0990, 1},
2235 { 0x0993, 0x09a8, 1},
2236 { 0x09aa, 0x09b0, 1},
2237 { 0x09b2, 0x09b6, 4},
2238 { 0x09b7, 0x09b9, 1},
2239 { 0x09bd, 0x09ce, 17},
2240 { 0x09dc, 0x09dd, 1},
2241 { 0x09df, 0x09e1, 1},
2242 { 0x09f0, 0x09f1, 1},
2243 { 0x0a05, 0x0a0a, 1},
2244 { 0x0a0f, 0x0a10, 1},
2245 { 0x0a13, 0x0a28, 1},
2246 { 0x0a2a, 0x0a30, 1},
2247 { 0x0a32, 0x0a33, 1},
2248 { 0x0a35, 0x0a36, 1},
2249 { 0x0a38, 0x0a39, 1},
2250 { 0x0a59, 0x0a5c, 1},
2251 { 0x0a5e, 0x0a72, 20},
2252 { 0x0a73, 0x0a74, 1},
2253 { 0x0a85, 0x0a8d, 1},
2254 { 0x0a8f, 0x0a91, 1},
2255 { 0x0a93, 0x0aa8, 1},
2256 { 0x0aaa, 0x0ab0, 1},
2257 { 0x0ab2, 0x0ab3, 1},
2258 { 0x0ab5, 0x0ab9, 1},
2259 { 0x0abd, 0x0ad0, 19},
2260 { 0x0ae0, 0x0ae1, 1},
2261 { 0x0af9, 0x0b05, 12},
2262 { 0x0b06, 0x0b0c, 1},
2263 { 0x0b0f, 0x0b10, 1},
2264 { 0x0b13, 0x0b28, 1},
2265 { 0x0b2a, 0x0b30, 1},
2266 { 0x0b32, 0x0b33, 1},
2267 { 0x0b35, 0x0b39, 1},
2268 { 0x0b3d, 0x0b5c, 31},
2269 { 0x0b5d, 0x0b5f, 2},
2270 { 0x0b60, 0x0b61, 1},
2271 { 0x0b71, 0x0b83, 18},
2272 { 0x0b85, 0x0b8a, 1},
2273 { 0x0b8e, 0x0b90, 1},
2274 { 0x0b92, 0x0b95, 1},
2275 { 0x0b99, 0x0b9a, 1},
2276 { 0x0b9c, 0x0b9e, 2},
2277 { 0x0b9f, 0x0ba3, 4},
2278 { 0x0ba4, 0x0ba8, 4},
2279 { 0x0ba9, 0x0baa, 1},
2280 { 0x0bae, 0x0bb9, 1},
2281 { 0x0bd0, 0x0c05, 53},
2282 { 0x0c06, 0x0c0c, 1},
2283 { 0x0c0e, 0x0c10, 1},
2284 { 0x0c12, 0x0c28, 1},
2285 { 0x0c2a, 0x0c39, 1},
2286 { 0x0c3d, 0x0c58, 27},
2287 { 0x0c59, 0x0c5a, 1},
2288 { 0x0c60, 0x0c61, 1},
2289 { 0x0c85, 0x0c8c, 1},
2290 { 0x0c8e, 0x0c90, 1},
2291 { 0x0c92, 0x0ca8, 1},
2292 { 0x0caa, 0x0cb3, 1},
2293 { 0x0cb5, 0x0cb9, 1},
2294 { 0x0cbd, 0x0cde, 33},
2295 { 0x0ce0, 0x0ce1, 1},
2296 { 0x0cf1, 0x0cf2, 1},
2297 { 0x0d05, 0x0d0c, 1},
2298 { 0x0d0e, 0x0d10, 1},
2299 { 0x0d12, 0x0d3a, 1},
2300 { 0x0d3d, 0x0d5f, 17},
2301 { 0x0d60, 0x0d61, 1},
2302 { 0x0d7a, 0x0d7f, 1},
2303 { 0x0d85, 0x0d96, 1},
2304 { 0x0d9a, 0x0db1, 1},
2305 { 0x0db3, 0x0dbb, 1},
2306 { 0x0dbd, 0x0dc0, 3},
2307 { 0x0dc1, 0x0dc6, 1},
2308 { 0x0e01, 0x0e30, 1},
2309 { 0x0e32, 0x0e33, 1},
2310 { 0x0e40, 0x0e46, 1},
2311 { 0x0e81, 0x0e82, 1},
2312 { 0x0e84, 0x0e87, 3},
2313 { 0x0e88, 0x0e8a, 2},
2314 { 0x0e8d, 0x0e94, 7},
2315 { 0x0e95, 0x0e97, 1},
2316 { 0x0e99, 0x0e9f, 1},
2317 { 0x0ea1, 0x0ea3, 1},
2318 { 0x0ea5, 0x0ea7, 2},
2319 { 0x0eaa, 0x0eab, 1},
2320 { 0x0ead, 0x0eb0, 1},
2321 { 0x0eb2, 0x0eb3, 1},
2322 { 0x0ebd, 0x0ec0, 3},
2323 { 0x0ec1, 0x0ec4, 1},
2324 { 0x0ec6, 0x0edc, 22},
2325 { 0x0edd, 0x0edf, 1},
2326 { 0x0f00, 0x0f40, 64},
2327 { 0x0f41, 0x0f47, 1},
2328 { 0x0f49, 0x0f6c, 1},
2329 { 0x0f88, 0x0f8c, 1},
2330 { 0x1000, 0x102a, 1},
2331 { 0x103f, 0x1050, 17},
2332 { 0x1051, 0x1055, 1},
2333 { 0x105a, 0x105d, 1},
2334 { 0x1061, 0x1065, 4},
2335 { 0x1066, 0x106e, 8},
2336 { 0x106f, 0x1070, 1},
2337 { 0x1075, 0x1081, 1},
2338 { 0x108e, 0x10a0, 18},
2339 { 0x10a1, 0x10c5, 1},
2340 { 0x10c7, 0x10cd, 6},
2341 { 0x10d0, 0x10fa, 1},
2342 { 0x10fc, 0x1248, 1},
2343 { 0x124a, 0x124d, 1},
2344 { 0x1250, 0x1256, 1},
2345 { 0x1258, 0x125a, 2},
2346 { 0x125b, 0x125d, 1},
2347 { 0x1260, 0x1288, 1},
2348 { 0x128a, 0x128d, 1},
2349 { 0x1290, 0x12b0, 1},
2350 { 0x12b2, 0x12b5, 1},
2351 { 0x12b8, 0x12be, 1},
2352 { 0x12c0, 0x12c2, 2},
2353 { 0x12c3, 0x12c5, 1},
2354 { 0x12c8, 0x12d6, 1},
2355 { 0x12d8, 0x1310, 1},
2356 { 0x1312, 0x1315, 1},
2357 { 0x1318, 0x135a, 1},
2358 { 0x1380, 0x138f, 1},
2359 { 0x13a0, 0x13f5, 1},
2360 { 0x13f8, 0x13fd, 1},
2361 { 0x1401, 0x166c, 1},
2362 { 0x166f, 0x167f, 1},
2363 { 0x1681, 0x169a, 1},
2364 { 0x16a0, 0x16ea, 1},
2365 { 0x16f1, 0x16f8, 1},
2366 { 0x1700, 0x170c, 1},
2367 { 0x170e, 0x1711, 1},
2368 { 0x1720, 0x1731, 1},
2369 { 0x1740, 0x1751, 1},
2370 { 0x1760, 0x176c, 1},
2371 { 0x176e, 0x1770, 1},
2372 { 0x1780, 0x17b3, 1},
2373 { 0x17d7, 0x17dc, 5},
2374 { 0x1820, 0x1877, 1},
2375 { 0x1880, 0x18a8, 1},
2376 { 0x18aa, 0x18b0, 6},
2377 { 0x18b1, 0x18f5, 1},
2378 { 0x1900, 0x191e, 1},
2379 { 0x1950, 0x196d, 1},
2380 { 0x1970, 0x1974, 1},
2381 { 0x1980, 0x19ab, 1},
2382 { 0x19b0, 0x19c9, 1},
2383 { 0x1a00, 0x1a16, 1},
2384 { 0x1a20, 0x1a54, 1},
2385 { 0x1aa7, 0x1b05, 94},
2386 { 0x1b06, 0x1b33, 1},
2387 { 0x1b45, 0x1b4b, 1},
2388 { 0x1b83, 0x1ba0, 1},
2389 { 0x1bae, 0x1baf, 1},
2390 { 0x1bba, 0x1be5, 1},
2391 { 0x1c00, 0x1c23, 1},
2392 { 0x1c4d, 0x1c4f, 1},
2393 { 0x1c5a, 0x1c7d, 1},
2394 { 0x1ce9, 0x1cec, 1},
2395 { 0x1cee, 0x1cf1, 1},
2396 { 0x1cf5, 0x1cf6, 1},
2397 { 0x1d00, 0x1dbf, 1},
2398 { 0x1e00, 0x1f15, 1},
2399 { 0x1f18, 0x1f1d, 1},
2400 { 0x1f20, 0x1f45, 1},
2401 { 0x1f48, 0x1f4d, 1},
2402 { 0x1f50, 0x1f57, 1},
2403 { 0x1f59, 0x1f5f, 2},
2404 { 0x1f60, 0x1f7d, 1},
2405 { 0x1f80, 0x1fb4, 1},
2406 { 0x1fb6, 0x1fbc, 1},
2407 { 0x1fbe, 0x1fc2, 4},
2408 { 0x1fc3, 0x1fc4, 1},
2409 { 0x1fc6, 0x1fcc, 1},
2410 { 0x1fd0, 0x1fd3, 1},
2411 { 0x1fd6, 0x1fdb, 1},
2412 { 0x1fe0, 0x1fec, 1},
2413 { 0x1ff2, 0x1ff4, 1},
2414 { 0x1ff6, 0x1ffc, 1},
2415 { 0x2071, 0x207f, 14},
2416 { 0x2090, 0x209c, 1},
2417 { 0x2102, 0x2107, 5},
2418 { 0x210a, 0x2113, 1},
2419 { 0x2115, 0x2119, 4},
2420 { 0x211a, 0x211d, 1},
2421 { 0x2124, 0x212a, 2},
2422 { 0x212b, 0x212d, 1},
2423 { 0x212f, 0x2139, 1},
2424 { 0x213c, 0x213f, 1},
2425 { 0x2145, 0x2149, 1},
2426 { 0x214e, 0x2183, 53},
2427 { 0x2184, 0x2c00, 2684},
2428 { 0x2c01, 0x2c2e, 1},
2429 { 0x2c30, 0x2c5e, 1},
2430 { 0x2c60, 0x2ce4, 1},
2431 { 0x2ceb, 0x2cee, 1},
2432 { 0x2cf2, 0x2cf3, 1},
2433 { 0x2d00, 0x2d25, 1},
2434 { 0x2d27, 0x2d2d, 6},
2435 { 0x2d30, 0x2d67, 1},
2436 { 0x2d6f, 0x2d80, 17},
2437 { 0x2d81, 0x2d96, 1},
2438 { 0x2da0, 0x2da6, 1},
2439 { 0x2da8, 0x2dae, 1},
2440 { 0x2db0, 0x2db6, 1},
2441 { 0x2db8, 0x2dbe, 1},
2442 { 0x2dc0, 0x2dc6, 1},
2443 { 0x2dc8, 0x2dce, 1},
2444 { 0x2dd0, 0x2dd6, 1},
2445 { 0x2dd8, 0x2dde, 1},
2446 { 0x2e2f, 0x3005, 470},
2447 { 0x3006, 0x3031, 43},
2448 { 0x3032, 0x3035, 1},
2449 { 0x303b, 0x303c, 1},
2450 { 0x3041, 0x3096, 1},
2451 { 0x309d, 0x309f, 1},
2452 { 0x30a1, 0x30fa, 1},
2453 { 0x30fc, 0x30ff, 1},
2454 { 0x3105, 0x312d, 1},
2455 { 0x3131, 0x318e, 1},
2456 { 0x31a0, 0x31ba, 1},
2457 { 0x31f0, 0x31ff, 1},
2458 { 0x3400, 0x4db5, 1},
2459 { 0x4e00, 0x9fd5, 1},
2460 { 0xa000, 0xa48c, 1},
2461 { 0xa4d0, 0xa4fd, 1},
2462 { 0xa500, 0xa60c, 1},
2463 { 0xa610, 0xa61f, 1},
2464 { 0xa62a, 0xa62b, 1},
2465 { 0xa640, 0xa66e, 1},
2466 { 0xa67f, 0xa69d, 1},
2467 { 0xa6a0, 0xa6e5, 1},
2468 { 0xa717, 0xa71f, 1},
2469 { 0xa722, 0xa788, 1},
2470 { 0xa78b, 0xa7ad, 1},
2471 { 0xa7b0, 0xa7b7, 1},
2472 { 0xa7f7, 0xa801, 1},
2473 { 0xa803, 0xa805, 1},
2474 { 0xa807, 0xa80a, 1},
2475 { 0xa80c, 0xa822, 1},
2476 { 0xa840, 0xa873, 1},
2477 { 0xa882, 0xa8b3, 1},
2478 { 0xa8f2, 0xa8f7, 1},
2479 { 0xa8fb, 0xa8fd, 2},
2480 { 0xa90a, 0xa925, 1},
2481 { 0xa930, 0xa946, 1},
2482 { 0xa960, 0xa97c, 1},
2483 { 0xa984, 0xa9b2, 1},
2484 { 0xa9cf, 0xa9e0, 17},
2485 { 0xa9e1, 0xa9e4, 1},
2486 { 0xa9e6, 0xa9ef, 1},
2487 { 0xa9fa, 0xa9fe, 1},
2488 { 0xaa00, 0xaa28, 1},
2489 { 0xaa40, 0xaa42, 1},
2490 { 0xaa44, 0xaa4b, 1},
2491 { 0xaa60, 0xaa76, 1},
2492 { 0xaa7a, 0xaa7e, 4},
2493 { 0xaa7f, 0xaaaf, 1},
2494 { 0xaab1, 0xaab5, 4},
2495 { 0xaab6, 0xaab9, 3},
2496 { 0xaaba, 0xaabd, 1},
2497 { 0xaac0, 0xaac2, 2},
2498 { 0xaadb, 0xaadd, 1},
2499 { 0xaae0, 0xaaea, 1},
2500 { 0xaaf2, 0xaaf4, 1},
2501 { 0xab01, 0xab06, 1},
2502 { 0xab09, 0xab0e, 1},
2503 { 0xab11, 0xab16, 1},
2504 { 0xab20, 0xab26, 1},
2505 { 0xab28, 0xab2e, 1},
2506 { 0xab30, 0xab5a, 1},
2507 { 0xab5c, 0xab65, 1},
2508 { 0xab70, 0xabe2, 1},
2509 { 0xac00, 0xd7a3, 1},
2510 { 0xd7b0, 0xd7c6, 1},
2511 { 0xd7cb, 0xd7fb, 1},
2512 { 0xf900, 0xfa6d, 1},
2513 { 0xfa70, 0xfad9, 1},
2514 { 0xfb00, 0xfb06, 1},
2515 { 0xfb13, 0xfb17, 1},
2516 { 0xfb1d, 0xfb1f, 2},
2517 { 0xfb20, 0xfb28, 1},
2518 { 0xfb2a, 0xfb36, 1},
2519 { 0xfb38, 0xfb3c, 1},
2520 { 0xfb3e, 0xfb40, 2},
2521 { 0xfb41, 0xfb43, 2},
2522 { 0xfb44, 0xfb46, 2},
2523 { 0xfb47, 0xfbb1, 1},
2524 { 0xfbd3, 0xfd3d, 1},
2525 { 0xfd50, 0xfd8f, 1},
2526 { 0xfd92, 0xfdc7, 1},
2527 { 0xfdf0, 0xfdfb, 1},
2528 { 0xfe70, 0xfe74, 1},
2529 { 0xfe76, 0xfefc, 1},
2530 { 0xff21, 0xff3a, 1},
2531 { 0xff41, 0xff5a, 1},
2532 { 0xff66, 0xffbe, 1},
2533 { 0xffc2, 0xffc7, 1},
2534 { 0xffca, 0xffcf, 1},
2535 { 0xffd2, 0xffd7, 1},
2536 { 0xffda, 0xffdc, 1},
2537 { 0x10000, 0x1000b, 1},
2538 { 0x1000d, 0x10026, 1},
2539 { 0x10028, 0x1003a, 1},
2540 { 0x1003c, 0x1003d, 1},
2541 { 0x1003f, 0x1004d, 1},
2542 { 0x10050, 0x1005d, 1},
2543 { 0x10080, 0x100fa, 1},
2544 { 0x10280, 0x1029c, 1},
2545 { 0x102a0, 0x102d0, 1},
2546 { 0x10300, 0x1031f, 1},
2547 { 0x10330, 0x10340, 1},
2548 { 0x10342, 0x10349, 1},
2549 { 0x10350, 0x10375, 1},
2550 { 0x10380, 0x1039d, 1},
2551 { 0x103a0, 0x103c3, 1},
2552 { 0x103c8, 0x103cf, 1},
2553 { 0x10400, 0x1049d, 1},
2554 { 0x10500, 0x10527, 1},
2555 { 0x10530, 0x10563, 1},
2556 { 0x10600, 0x10736, 1},
2557 { 0x10740, 0x10755, 1},
2558 { 0x10760, 0x10767, 1},
2559 { 0x10800, 0x10805, 1},
2560 { 0x10808, 0x1080a, 2},
2561 { 0x1080b, 0x10835, 1},
2562 { 0x10837, 0x10838, 1},
2563 { 0x1083c, 0x1083f, 3},
2564 { 0x10840, 0x10855, 1},
2565 { 0x10860, 0x10876, 1},
2566 { 0x10880, 0x1089e, 1},
2567 { 0x108e0, 0x108f2, 1},
2568 { 0x108f4, 0x108f5, 1},
2569 { 0x10900, 0x10915, 1},
2570 { 0x10920, 0x10939, 1},
2571 { 0x10980, 0x109b7, 1},
2572 { 0x109be, 0x109bf, 1},
2573 { 0x10a00, 0x10a10, 16},
2574 { 0x10a11, 0x10a13, 1},
2575 { 0x10a15, 0x10a17, 1},
2576 { 0x10a19, 0x10a33, 1},
2577 { 0x10a60, 0x10a7c, 1},
2578 { 0x10a80, 0x10a9c, 1},
2579 { 0x10ac0, 0x10ac7, 1},
2580 { 0x10ac9, 0x10ae4, 1},
2581 { 0x10b00, 0x10b35, 1},
2582 { 0x10b40, 0x10b55, 1},
2583 { 0x10b60, 0x10b72, 1},
2584 { 0x10b80, 0x10b91, 1},
2585 { 0x10c00, 0x10c48, 1},
2586 { 0x10c80, 0x10cb2, 1},
2587 { 0x10cc0, 0x10cf2, 1},
2588 { 0x11003, 0x11037, 1},
2589 { 0x11083, 0x110af, 1},
2590 { 0x110d0, 0x110e8, 1},
2591 { 0x11103, 0x11126, 1},
2592 { 0x11150, 0x11172, 1},
2593 { 0x11176, 0x11183, 13},
2594 { 0x11184, 0x111b2, 1},
2595 { 0x111c1, 0x111c4, 1},
2596 { 0x111da, 0x111dc, 2},
2597 { 0x11200, 0x11211, 1},
2598 { 0x11213, 0x1122b, 1},
2599 { 0x11280, 0x11286, 1},
2600 { 0x11288, 0x1128a, 2},
2601 { 0x1128b, 0x1128d, 1},
2602 { 0x1128f, 0x1129d, 1},
2603 { 0x1129f, 0x112a8, 1},
2604 { 0x112b0, 0x112de, 1},
2605 { 0x11305, 0x1130c, 1},
2606 { 0x1130f, 0x11310, 1},
2607 { 0x11313, 0x11328, 1},
2608 { 0x1132a, 0x11330, 1},
2609 { 0x11332, 0x11333, 1},
2610 { 0x11335, 0x11339, 1},
2611 { 0x1133d, 0x11350, 19},
2612 { 0x1135d, 0x11361, 1},
2613 { 0x11480, 0x114af, 1},
2614 { 0x114c4, 0x114c5, 1},
2615 { 0x114c7, 0x11580, 185},
2616 { 0x11581, 0x115ae, 1},
2617 { 0x115d8, 0x115db, 1},
2618 { 0x11600, 0x1162f, 1},
2619 { 0x11644, 0x11680, 60},
2620 { 0x11681, 0x116aa, 1},
2621 { 0x11700, 0x11719, 1},
2622 { 0x118a0, 0x118df, 1},
2623 { 0x118ff, 0x11ac0, 449},
2624 { 0x11ac1, 0x11af8, 1},
2625 { 0x12000, 0x12399, 1},
2626 { 0x12480, 0x12543, 1},
2627 { 0x13000, 0x1342e, 1},
2628 { 0x14400, 0x14646, 1},
2629 { 0x16800, 0x16a38, 1},
2630 { 0x16a40, 0x16a5e, 1},
2631 { 0x16ad0, 0x16aed, 1},
2632 { 0x16b00, 0x16b2f, 1},
2633 { 0x16b40, 0x16b43, 1},
2634 { 0x16b63, 0x16b77, 1},
2635 { 0x16b7d, 0x16b8f, 1},
2636 { 0x16f00, 0x16f44, 1},
2637 { 0x16f50, 0x16f93, 67},
2638 { 0x16f94, 0x16f9f, 1},
2639 { 0x1b000, 0x1b001, 1},
2640 { 0x1bc00, 0x1bc6a, 1},
2641 { 0x1bc70, 0x1bc7c, 1},
2642 { 0x1bc80, 0x1bc88, 1},
2643 { 0x1bc90, 0x1bc99, 1},
2644 { 0x1d400, 0x1d454, 1},
2645 { 0x1d456, 0x1d49c, 1},
2646 { 0x1d49e, 0x1d49f, 1},
2647 { 0x1d4a2, 0x1d4a5, 3},
2648 { 0x1d4a6, 0x1d4a9, 3},
2649 { 0x1d4aa, 0x1d4ac, 1},
2650 { 0x1d4ae, 0x1d4b9, 1},
2651 { 0x1d4bb, 0x1d4bd, 2},
2652 { 0x1d4be, 0x1d4c3, 1},
2653 { 0x1d4c5, 0x1d505, 1},
2654 { 0x1d507, 0x1d50a, 1},
2655 { 0x1d50d, 0x1d514, 1},
2656 { 0x1d516, 0x1d51c, 1},
2657 { 0x1d51e, 0x1d539, 1},
2658 { 0x1d53b, 0x1d53e, 1},
2659 { 0x1d540, 0x1d544, 1},
2660 { 0x1d546, 0x1d54a, 4},
2661 { 0x1d54b, 0x1d550, 1},
2662 { 0x1d552, 0x1d6a5, 1},
2663 { 0x1d6a8, 0x1d6c0, 1},
2664 { 0x1d6c2, 0x1d6da, 1},
2665 { 0x1d6dc, 0x1d6fa, 1},
2666 { 0x1d6fc, 0x1d714, 1},
2667 { 0x1d716, 0x1d734, 1},
2668 { 0x1d736, 0x1d74e, 1},
2669 { 0x1d750, 0x1d76e, 1},
2670 { 0x1d770, 0x1d788, 1},
2671 { 0x1d78a, 0x1d7a8, 1},
2672 { 0x1d7aa, 0x1d7c2, 1},
2673 { 0x1d7c4, 0x1d7cb, 1},
2674 { 0x1e800, 0x1e8c4, 1},
2675 { 0x1ee00, 0x1ee03, 1},
2676 { 0x1ee05, 0x1ee1f, 1},
2677 { 0x1ee21, 0x1ee22, 1},
2678 { 0x1ee24, 0x1ee27, 3},
2679 { 0x1ee29, 0x1ee32, 1},
2680 { 0x1ee34, 0x1ee37, 1},
2681 { 0x1ee39, 0x1ee3b, 2},
2682 { 0x1ee42, 0x1ee47, 5},
2683 { 0x1ee49, 0x1ee4d, 2},
2684 { 0x1ee4e, 0x1ee4f, 1},
2685 { 0x1ee51, 0x1ee52, 1},
2686 { 0x1ee54, 0x1ee57, 3},
2687 { 0x1ee59, 0x1ee61, 2},
2688 { 0x1ee62, 0x1ee64, 2},
2689 { 0x1ee67, 0x1ee6a, 1},
2690 { 0x1ee6c, 0x1ee72, 1},
2691 { 0x1ee74, 0x1ee77, 1},
2692 { 0x1ee79, 0x1ee7c, 1},
2693 { 0x1ee7e, 0x1ee80, 2},
2694 { 0x1ee81, 0x1ee89, 1},
2695 { 0x1ee8b, 0x1ee9b, 1},
2696 { 0x1eea1, 0x1eea3, 1},
2697 { 0x1eea5, 0x1eea9, 1},
2698 { 0x1eeab, 0x1eebb, 1},
2699 { 0x20000, 0x2a6d6, 1},
2700 { 0x2a700, 0x2b734, 1},
2701 { 0x2b740, 0x2b81d, 1},
2702 { 0x2b820, 0x2cea1, 1},
2703 { 0x2f800, 0x2fa1d, 1},
2704 };
2705
2706 // A table of Unicode uppercase letters--Unicode code points
2707 // classified as "Letter, uppercase".
2708
2709 static const Unicode_range unicode_uppercase_letters[] =
2710 {
2711 { 0x0041, 0x005a, 1},
2712 { 0x00c0, 0x00d6, 1},
2713 { 0x00d8, 0x00de, 1},
2714 { 0x0100, 0x0136, 2},
2715 { 0x0139, 0x0147, 2},
2716 { 0x014a, 0x0178, 2},
2717 { 0x0179, 0x017d, 2},
2718 { 0x0181, 0x0182, 1},
2719 { 0x0184, 0x0186, 2},
2720 { 0x0187, 0x0189, 2},
2721 { 0x018a, 0x018b, 1},
2722 { 0x018e, 0x0191, 1},
2723 { 0x0193, 0x0194, 1},
2724 { 0x0196, 0x0198, 1},
2725 { 0x019c, 0x019d, 1},
2726 { 0x019f, 0x01a0, 1},
2727 { 0x01a2, 0x01a6, 2},
2728 { 0x01a7, 0x01a9, 2},
2729 { 0x01ac, 0x01ae, 2},
2730 { 0x01af, 0x01b1, 2},
2731 { 0x01b2, 0x01b3, 1},
2732 { 0x01b5, 0x01b7, 2},
2733 { 0x01b8, 0x01bc, 4},
2734 { 0x01c4, 0x01cd, 3},
2735 { 0x01cf, 0x01db, 2},
2736 { 0x01de, 0x01ee, 2},
2737 { 0x01f1, 0x01f4, 3},
2738 { 0x01f6, 0x01f8, 1},
2739 { 0x01fa, 0x0232, 2},
2740 { 0x023a, 0x023b, 1},
2741 { 0x023d, 0x023e, 1},
2742 { 0x0241, 0x0243, 2},
2743 { 0x0244, 0x0246, 1},
2744 { 0x0248, 0x024e, 2},
2745 { 0x0370, 0x0372, 2},
2746 { 0x0376, 0x037f, 9},
2747 { 0x0386, 0x0388, 2},
2748 { 0x0389, 0x038a, 1},
2749 { 0x038c, 0x038e, 2},
2750 { 0x038f, 0x0391, 2},
2751 { 0x0392, 0x03a1, 1},
2752 { 0x03a3, 0x03ab, 1},
2753 { 0x03cf, 0x03d2, 3},
2754 { 0x03d3, 0x03d4, 1},
2755 { 0x03d8, 0x03ee, 2},
2756 { 0x03f4, 0x03f7, 3},
2757 { 0x03f9, 0x03fa, 1},
2758 { 0x03fd, 0x042f, 1},
2759 { 0x0460, 0x0480, 2},
2760 { 0x048a, 0x04c0, 2},
2761 { 0x04c1, 0x04cd, 2},
2762 { 0x04d0, 0x052e, 2},
2763 { 0x0531, 0x0556, 1},
2764 { 0x10a0, 0x10c5, 1},
2765 { 0x10c7, 0x10cd, 6},
2766 { 0x1e00, 0x1e94, 2},
2767 { 0x1e9e, 0x1efe, 2},
2768 { 0x1f08, 0x1f0f, 1},
2769 { 0x1f18, 0x1f1d, 1},
2770 { 0x1f28, 0x1f2f, 1},
2771 { 0x1f38, 0x1f3f, 1},
2772 { 0x1f48, 0x1f4d, 1},
2773 { 0x1f59, 0x1f5f, 2},
2774 { 0x1f68, 0x1f6f, 1},
2775 { 0x1fb8, 0x1fbb, 1},
2776 { 0x1fc8, 0x1fcb, 1},
2777 { 0x1fd8, 0x1fdb, 1},
2778 { 0x1fe8, 0x1fec, 1},
2779 { 0x1ff8, 0x1ffb, 1},
2780 { 0x2102, 0x2107, 5},
2781 { 0x210b, 0x210d, 1},
2782 { 0x2110, 0x2112, 1},
2783 { 0x2115, 0x2119, 4},
2784 { 0x211a, 0x211d, 1},
2785 { 0x2124, 0x212a, 2},
2786 { 0x212b, 0x212d, 1},
2787 { 0x2130, 0x2133, 1},
2788 { 0x213e, 0x213f, 1},
2789 { 0x2145, 0x2183, 62},
2790 { 0x2c00, 0x2c2e, 1},
2791 { 0x2c60, 0x2c62, 2},
2792 { 0x2c63, 0x2c64, 1},
2793 { 0x2c67, 0x2c6d, 2},
2794 { 0x2c6e, 0x2c70, 1},
2795 { 0x2c72, 0x2c75, 3},
2796 { 0x2c7e, 0x2c80, 1},
2797 { 0x2c82, 0x2ce2, 2},
2798 { 0x2ceb, 0x2ced, 2},
2799 { 0x2cf2, 0xa640, 31054},
2800 { 0xa642, 0xa66c, 2},
2801 { 0xa680, 0xa69a, 2},
2802 { 0xa722, 0xa72e, 2},
2803 { 0xa732, 0xa76e, 2},
2804 { 0xa779, 0xa77d, 2},
2805 { 0xa77e, 0xa786, 2},
2806 { 0xa78b, 0xa78d, 2},
2807 { 0xa790, 0xa792, 2},
2808 { 0xa796, 0xa7aa, 2},
2809 { 0xa7ab, 0xa7ad, 1},
2810 { 0xa7b0, 0xa7b1, 1},
2811 { 0xff21, 0xff3a, 1},
2812 { 0x10400, 0x10427, 1},
2813 { 0x118a0, 0x118bf, 1},
2814 { 0x1d400, 0x1d419, 1},
2815 { 0x1d434, 0x1d44d, 1},
2816 { 0x1d468, 0x1d481, 1},
2817 { 0x1d49c, 0x1d49e, 2},
2818 { 0x1d49f, 0x1d4a5, 3},
2819 { 0x1d4a6, 0x1d4a9, 3},
2820 { 0x1d4aa, 0x1d4ac, 1},
2821 { 0x1d4ae, 0x1d4b5, 1},
2822 { 0x1d4d0, 0x1d4e9, 1},
2823 { 0x1d504, 0x1d505, 1},
2824 { 0x1d507, 0x1d50a, 1},
2825 { 0x1d50d, 0x1d514, 1},
2826 { 0x1d516, 0x1d51c, 1},
2827 { 0x1d538, 0x1d539, 1},
2828 { 0x1d53b, 0x1d53e, 1},
2829 { 0x1d540, 0x1d544, 1},
2830 { 0x1d546, 0x1d54a, 4},
2831 { 0x1d54b, 0x1d550, 1},
2832 { 0x1d56c, 0x1d585, 1},
2833 { 0x1d5a0, 0x1d5b9, 1},
2834 { 0x1d5d4, 0x1d5ed, 1},
2835 { 0x1d608, 0x1d621, 1},
2836 { 0x1d63c, 0x1d655, 1},
2837 { 0x1d670, 0x1d689, 1},
2838 { 0x1d6a8, 0x1d6c0, 1},
2839 { 0x1d6e2, 0x1d6fa, 1},
2840 { 0x1d71c, 0x1d734, 1},
2841 { 0x1d756, 0x1d76e, 1},
2842 { 0x1d790, 0x1d7a8, 1},
2843 { 0x1d7ca, 0x1d7ca, 1},
2844 };
2845
2846 // Return true if C is in RANGES.
2847
2848 bool
2849 Lex::is_in_unicode_range(unsigned int c, const Unicode_range* ranges,
2850 size_t range_size)
2851 {
2852 if (c < 0x100)
2853 {
2854 // The common case is a small value, and we know that it will be
2855 // in the first few entries of the table. Do a linear scan
2856 // rather than a binary search.
2857 for (size_t i = 0; i < range_size; ++i)
2858 {
2859 const Unicode_range* p = &ranges[i];
2860 if (c <= p->high)
2861 {
2862 if (c < p->low)
2863 return false;
2864 return (c - p->low) % p->stride == 0;
2865 }
2866 }
2867 return false;
2868 }
2869 else
2870 {
2871 size_t lo = 0;
2872 size_t hi = range_size;
2873 while (lo < hi)
2874 {
2875 size_t mid = lo + (hi - lo) / 2;
2876 const Unicode_range* p = &ranges[mid];
2877 if (c < p->low)
2878 hi = mid;
2879 else if (c > p->high)
2880 lo = mid + 1;
2881 else
2882 return (c - p->low) % p->stride == 0;
2883 }
2884 return false;
2885 }
2886 }
2887
2888 // Return whether C is a space character.
2889
2890 bool
2891 Lex::is_unicode_space(unsigned int c)
2892 {
2893 return Lex::is_in_unicode_range(c, unicode_space,
2894 ARRAY_SIZE(unicode_space));
2895 }
2896
2897 // Return whether C is a Unicode digit--a Unicode code point
2898 // classified as "Digit".
2899
2900 bool
2901 Lex::is_unicode_digit(unsigned int c)
2902 {
2903 return Lex::is_in_unicode_range(c, unicode_digits,
2904 ARRAY_SIZE(unicode_digits));
2905 }
2906
2907 // Return whether C is a Unicode letter--a Unicode code point
2908 // classified as "Letter".
2909
2910 bool
2911 Lex::is_unicode_letter(unsigned int c)
2912 {
2913 return Lex::is_in_unicode_range(c, unicode_letters,
2914 ARRAY_SIZE(unicode_letters));
2915 }
2916
2917 // Return whether C is a Unicode uppercase letter. a Unicode code
2918 // point classified as "Letter, uppercase".
2919
2920 bool
2921 Lex::is_unicode_uppercase(unsigned int c)
2922 {
2923 return Lex::is_in_unicode_range(c, unicode_uppercase_letters,
2924 ARRAY_SIZE(unicode_uppercase_letters));
2925 }
2926
2927 // Return whether the identifier NAME should be exported. NAME is a
2928 // mangled name which includes only ASCII characters.
2929
2930 bool
2931 Lex::is_exported_mangled_name(const std::string& name)
2932 {
2933 unsigned char c = name[0];
2934 if (c != '.')
2935 return c >= 'A' && c <= 'Z';
2936 else
2937 {
2938 const char* p = name.data();
2939 size_t len = name.length();
2940 if (len < 4 || p[1] != '.' || (p[2] != 'u' && p[2] != 'U'))
2941 return false;
2942 unsigned int ci = 0;
2943 size_t want = (p[2] == 'u' ? 4 : 8);
2944 if (len < want + 3)
2945 return false;
2946 for (size_t i = 3; i < want; ++i)
2947 {
2948 c = p[i];
2949 if (!Lex::is_hex_digit(c))
2950 return false;
2951 ci <<= 4;
2952 ci |= Lex::hex_val(c);
2953 }
2954 return Lex::is_unicode_uppercase(ci);
2955 }
2956 }
2957
2958 // Return whether the identifier NAME should be exported. NAME is a
2959 // an unmangled utf-8 string and may contain non-ASCII characters.
2960
2961 bool
2962 Lex::is_exported_name(const std::string& name)
2963 {
2964 unsigned int uchar;
2965 if (Lex::fetch_char(name.c_str(), &uchar) != 0)
2966 return Lex::is_unicode_letter(uchar) && Lex::is_unicode_uppercase(uchar);
2967 return false;
2968 }
2969
2970 // Return whether the identifier NAME contains an invalid character.
2971 // This is based on how we handle invalid characters in
2972 // gather_identifier.
2973
2974 bool
2975 Lex::is_invalid_identifier(const std::string& name)
2976 {
2977 return name.find("$INVALID$") != std::string::npos;
2978 }