mirror of
https://github.com/apple/swift.git
synced 2025-12-21 12:14:44 +01:00
2509f50e45
For invalid code, lexer is forgiving enough to allow single quote '\'' as the starting point of string literals. Later, parser assumes the string literals are always using double quote "\"", and passes such knowledge to SwiftSyntax side, leading to the round-trip failure we observed in the radar. This commit fixes the issue by introducing another token kind for single quote. rdar://51071021
352 lines
14 KiB
Python
352 lines
14 KiB
Python
from Classification import classification_by_name
|
|
from Node import error # noqa: I201
|
|
from kinds import lowercase_first_word # noqa: I201
|
|
|
|
|
|
class Token(object):
|
|
"""
|
|
Represents the specification for a Token in the TokenSyntax file.
|
|
"""
|
|
|
|
def __init__(self, name, kind, serialization_code, unprefixed_kind=None,
|
|
text=None, classification='None', is_keyword=False):
|
|
self.name = name
|
|
self.kind = kind
|
|
if unprefixed_kind is None:
|
|
self.unprefixed_kind = kind
|
|
else:
|
|
self.unprefixed_kind = unprefixed_kind
|
|
self.serialization_code = serialization_code
|
|
self.text = text or ""
|
|
self.classification = classification_by_name(classification)
|
|
self.is_keyword = is_keyword
|
|
|
|
def swift_kind(self):
|
|
name = lowercase_first_word(self.name)
|
|
if self.is_keyword:
|
|
return name + 'Keyword'
|
|
return name
|
|
|
|
|
|
class Keyword(Token):
|
|
"""
|
|
Represents a keyword token.
|
|
"""
|
|
|
|
def __init__(self, name, text, serialization_code,
|
|
classification='Keyword'):
|
|
Token.__init__(self, name, 'kw_' + text, serialization_code,
|
|
unprefixed_kind=text, text=text,
|
|
classification=classification, is_keyword=True)
|
|
|
|
def macro_name(self):
|
|
return "KEYWORD"
|
|
|
|
|
|
class SwiftKeyword(Keyword):
|
|
def macro_name(self):
|
|
return "SWIFT_KEYWORD"
|
|
|
|
|
|
class DeclKeyword(SwiftKeyword):
|
|
def macro_name(self):
|
|
return "DECL_KEYWORD"
|
|
|
|
|
|
class StmtKeyword(SwiftKeyword):
|
|
def macro_name(self):
|
|
return "STMT_KEYWORD"
|
|
|
|
|
|
class ExprKeyword(SwiftKeyword):
|
|
def macro_name(self):
|
|
return "EXPR_KEYWORD"
|
|
|
|
|
|
class PatternKeyword(SwiftKeyword):
|
|
def macro_name(self):
|
|
return "PAT_KEYWORD"
|
|
|
|
|
|
class SilKeyword(Keyword):
|
|
def macro_name(self):
|
|
return "SIL_KEYWORD"
|
|
|
|
|
|
class PoundKeyword(Token):
|
|
def __init__(self, name, kind, text, serialization_code,
|
|
classification='Keyword'):
|
|
Token.__init__(self, name, 'pound_' + kind, serialization_code,
|
|
unprefixed_kind=kind, text=text,
|
|
classification=classification, is_keyword=True)
|
|
|
|
def macro_name(self):
|
|
return "POUND_KEYWORD"
|
|
|
|
|
|
class PoundObjectLiteral(PoundKeyword):
|
|
def __init__(self, name, kind, text, serialization_code, description,
|
|
protocol, classification='ObjectLiteral'):
|
|
PoundKeyword.__init__(self, name, kind, text, serialization_code,
|
|
classification)
|
|
self.description = description
|
|
self.protocol = protocol
|
|
|
|
def macro_name(self):
|
|
return "POUND_OBJECT_LITERAL"
|
|
|
|
|
|
class PoundConfig(PoundKeyword):
|
|
def macro_name(self):
|
|
return "POUND_CONFIG"
|
|
|
|
|
|
class PoundDirectiveKeyword(PoundKeyword):
|
|
def __init__(self, name, kind, text, serialization_code,
|
|
classification='PoundDirectiveKeyword'):
|
|
PoundKeyword.__init__(self, name, kind, text, serialization_code,
|
|
classification)
|
|
|
|
def macro_name(self):
|
|
return "POUND_DIRECTIVE_KEYWORD"
|
|
|
|
|
|
class PoundConditionalDirectiveKeyword(PoundDirectiveKeyword):
|
|
def __init__(self, name, kind, text, serialization_code,
|
|
classification='PoundDirectiveKeyword'):
|
|
PoundKeyword.__init__(self, name, kind, text, serialization_code,
|
|
classification)
|
|
|
|
def macro_name(self):
|
|
return "POUND_COND_DIRECTIVE_KEYWORD"
|
|
|
|
|
|
class Punctuator(Token):
|
|
def macro_name(self):
|
|
return "PUNCTUATOR"
|
|
|
|
|
|
class Literal(Token):
|
|
def macro_name(self):
|
|
return "LITERAL"
|
|
|
|
|
|
class Misc(Token):
|
|
def macro_name(self):
|
|
return "MISC"
|
|
|
|
|
|
SYNTAX_TOKENS = [
|
|
# Keywords that start decls
|
|
DeclKeyword('Associatedtype', 'associatedtype', serialization_code=1),
|
|
DeclKeyword('Class', 'class', serialization_code=2),
|
|
DeclKeyword('Deinit', 'deinit', serialization_code=3),
|
|
DeclKeyword('Enum', 'enum', serialization_code=4),
|
|
DeclKeyword('Extension', 'extension', serialization_code=5),
|
|
DeclKeyword('Func', 'func', serialization_code=6),
|
|
DeclKeyword('Import', 'import', serialization_code=7),
|
|
DeclKeyword('Init', 'init', serialization_code=8),
|
|
DeclKeyword('Inout', 'inout', serialization_code=9),
|
|
DeclKeyword('Let', 'let', serialization_code=10),
|
|
DeclKeyword('Operator', 'operator', serialization_code=11),
|
|
DeclKeyword('Precedencegroup', 'precedencegroup', serialization_code=12),
|
|
DeclKeyword('Protocol', 'protocol', serialization_code=13),
|
|
DeclKeyword('Struct', 'struct', serialization_code=14),
|
|
DeclKeyword('Subscript', 'subscript', serialization_code=15),
|
|
DeclKeyword('Typealias', 'typealias', serialization_code=16),
|
|
DeclKeyword('Var', 'var', serialization_code=17),
|
|
|
|
DeclKeyword('Fileprivate', 'fileprivate', serialization_code=18),
|
|
DeclKeyword('Internal', 'internal', serialization_code=19),
|
|
DeclKeyword('Private', 'private', serialization_code=20),
|
|
DeclKeyword('Public', 'public', serialization_code=21),
|
|
DeclKeyword('Static', 'static', serialization_code=22),
|
|
|
|
# Statement keywords
|
|
StmtKeyword('Defer', 'defer', serialization_code=23),
|
|
StmtKeyword('If', 'if', serialization_code=24),
|
|
StmtKeyword('Guard', 'guard', serialization_code=25),
|
|
StmtKeyword('Do', 'do', serialization_code=26),
|
|
StmtKeyword('Repeat', 'repeat', serialization_code=27),
|
|
StmtKeyword('Else', 'else', serialization_code=28),
|
|
StmtKeyword('For', 'for', serialization_code=29),
|
|
StmtKeyword('In', 'in', serialization_code=30),
|
|
StmtKeyword('While', 'while', serialization_code=31),
|
|
StmtKeyword('Return', 'return', serialization_code=32),
|
|
StmtKeyword('Break', 'break', serialization_code=33),
|
|
StmtKeyword('Continue', 'continue', serialization_code=34),
|
|
StmtKeyword('Fallthrough', 'fallthrough', serialization_code=35),
|
|
StmtKeyword('Switch', 'switch', serialization_code=36),
|
|
StmtKeyword('Case', 'case', serialization_code=37),
|
|
StmtKeyword('Default', 'default', serialization_code=38),
|
|
StmtKeyword('Where', 'where', serialization_code=39),
|
|
StmtKeyword('Catch', 'catch', serialization_code=40),
|
|
StmtKeyword('Throw', 'throw', serialization_code=50),
|
|
|
|
# Expression keywords
|
|
ExprKeyword('As', 'as', serialization_code=41),
|
|
ExprKeyword('Any', 'Any', serialization_code=42),
|
|
ExprKeyword('False', 'false', serialization_code=43),
|
|
ExprKeyword('Is', 'is', serialization_code=44),
|
|
ExprKeyword('Nil', 'nil', serialization_code=45),
|
|
ExprKeyword('Rethrows', 'rethrows', serialization_code=46),
|
|
ExprKeyword('Super', 'super', serialization_code=47),
|
|
ExprKeyword('Self', 'self', serialization_code=48),
|
|
ExprKeyword('CapitalSelf', 'Self', serialization_code=49),
|
|
ExprKeyword('True', 'true', serialization_code=51),
|
|
ExprKeyword('Try', 'try', serialization_code=52),
|
|
ExprKeyword('Throws', 'throws', serialization_code=53),
|
|
|
|
Keyword('__FILE__', '__FILE__', serialization_code=54),
|
|
Keyword('__LINE__', '__LINE__', serialization_code=55),
|
|
Keyword('__COLUMN__', '__COLUMN__', serialization_code=56),
|
|
Keyword('__FUNCTION__', '__FUNCTION__', serialization_code=57),
|
|
Keyword('__DSO_HANDLE__', '__DSO_HANDLE__', serialization_code=58),
|
|
|
|
# Pattern keywords
|
|
PatternKeyword('Wildcard', '_', serialization_code=59),
|
|
|
|
# Punctuators
|
|
Punctuator('LeftParen', 'l_paren', text='(', serialization_code=88),
|
|
Punctuator('RightParen', 'r_paren', text=')', serialization_code=89),
|
|
Punctuator('LeftBrace', 'l_brace', text='{', serialization_code=90),
|
|
Punctuator('RightBrace', 'r_brace', text='}', serialization_code=91),
|
|
Punctuator('LeftSquareBracket', 'l_square', text='[',
|
|
serialization_code=92),
|
|
Punctuator('RightSquareBracket', 'r_square', text=']',
|
|
serialization_code=93),
|
|
Punctuator('LeftAngle', 'l_angle', text='<', serialization_code=94),
|
|
Punctuator('RightAngle', 'r_angle', text='>', serialization_code=95),
|
|
|
|
Punctuator('Period', 'period', text='.', serialization_code=85),
|
|
Punctuator('PrefixPeriod', 'period_prefix', text='.',
|
|
serialization_code=87),
|
|
Punctuator('Comma', 'comma', text=',', serialization_code=84),
|
|
Punctuator('Ellipsis', 'ellipsis', text='...', serialization_code=118),
|
|
Punctuator('Colon', 'colon', text=':', serialization_code=82),
|
|
Punctuator('Semicolon', 'semi', text=';', serialization_code=83),
|
|
Punctuator('Equal', 'equal', text='=', serialization_code=86),
|
|
Punctuator('AtSign', 'at_sign', text='@', classification='Attribute',
|
|
serialization_code=80),
|
|
Punctuator('Pound', 'pound', text='#', serialization_code=81),
|
|
|
|
Punctuator('PrefixAmpersand', 'amp_prefix', text='&',
|
|
serialization_code=96),
|
|
Punctuator('Arrow', 'arrow', text='->', serialization_code=78),
|
|
|
|
|
|
Punctuator('Backtick', 'backtick', text='`', serialization_code=79),
|
|
|
|
Punctuator('Backslash', 'backslash', text='\\\\', serialization_code=100),
|
|
|
|
Punctuator('ExclamationMark', 'exclaim_postfix', text='!',
|
|
serialization_code=99),
|
|
|
|
Punctuator('PostfixQuestionMark', 'question_postfix', text='?',
|
|
serialization_code=97),
|
|
Punctuator('InfixQuestionMark', 'question_infix', text='?',
|
|
serialization_code=98),
|
|
|
|
Punctuator('StringQuote', 'string_quote', text='\\\"',
|
|
classification='StringLiteral', serialization_code=102),
|
|
Punctuator('SingleQuote', 'single_quote', text='\\\'',
|
|
classification='StringLiteral', serialization_code=120),
|
|
Punctuator('MultilineStringQuote', 'multiline_string_quote',
|
|
text='\\\"\\\"\\\"', classification='StringLiteral',
|
|
serialization_code=103),
|
|
|
|
# Keywords prefixed with a '#'.
|
|
|
|
PoundKeyword('PoundKeyPath', 'keyPath', text='#keyPath',
|
|
serialization_code=74),
|
|
PoundKeyword('PoundLine', 'line', text='#line',
|
|
serialization_code=69),
|
|
PoundKeyword('PoundSelector', 'selector', text='#selector',
|
|
serialization_code=73),
|
|
PoundKeyword('PoundFile', 'file', text='#file',
|
|
serialization_code=68),
|
|
PoundKeyword('PoundColumn', 'column', text='#column',
|
|
serialization_code=70),
|
|
PoundKeyword('PoundFunction', 'function', text='#function',
|
|
serialization_code=72),
|
|
PoundKeyword('PoundDsohandle', 'dsohandle', text='#dsohandle',
|
|
serialization_code=71),
|
|
PoundKeyword('PoundAssert', 'assert', text='#assert',
|
|
serialization_code=117),
|
|
|
|
PoundDirectiveKeyword('PoundSourceLocation', 'sourceLocation',
|
|
text='#sourceLocation', serialization_code=65),
|
|
PoundDirectiveKeyword('PoundWarning', 'warning', text='#warning',
|
|
serialization_code=66),
|
|
PoundDirectiveKeyword('PoundError', 'error', text='#error',
|
|
serialization_code=67),
|
|
|
|
PoundConditionalDirectiveKeyword('PoundIf', 'if', text='#if',
|
|
serialization_code=64),
|
|
PoundConditionalDirectiveKeyword('PoundElse', 'else', text='#else',
|
|
serialization_code=62),
|
|
PoundConditionalDirectiveKeyword('PoundElseif', 'elseif',
|
|
text='#elseif', serialization_code=63),
|
|
PoundConditionalDirectiveKeyword('PoundEndif', 'endif',
|
|
text='#endif', serialization_code=61),
|
|
|
|
PoundConfig('PoundAvailable', 'available', text='#available',
|
|
serialization_code=60),
|
|
|
|
PoundObjectLiteral('PoundFileLiteral', 'fileLiteral',
|
|
text='#fileLiteral', serialization_code=76,
|
|
description='file reference',
|
|
protocol='ExpressibleByFileReferenceLiteral'),
|
|
PoundObjectLiteral('PoundImageLiteral', 'imageLiteral',
|
|
text='#imageLiteral', serialization_code=77,
|
|
description='image',
|
|
protocol='ExpressibleByImageLiteral'),
|
|
PoundObjectLiteral('PoundColorLiteral', 'colorLiteral',
|
|
text='#colorLiteral', serialization_code=75,
|
|
description='color',
|
|
protocol='ExpressibleByColorLiteral'),
|
|
|
|
Literal('IntegerLiteral', 'integer_literal',
|
|
classification='IntegerLiteral', serialization_code=111),
|
|
Literal('FloatingLiteral', 'floating_literal',
|
|
classification='FloatingLiteral', serialization_code=112),
|
|
Literal('StringLiteral', 'string_literal',
|
|
classification='StringLiteral', serialization_code=113),
|
|
|
|
Misc('Unknown', 'unknown', serialization_code=115),
|
|
Misc('Identifier', 'identifier', classification=None,
|
|
serialization_code=105),
|
|
Misc('UnspacedBinaryOperator', 'oper_binary_unspaced',
|
|
serialization_code=107),
|
|
Misc('SpacedBinaryOperator', 'oper_binary_spaced', serialization_code=108),
|
|
Misc('PostfixOperator', 'oper_postfix', serialization_code=110),
|
|
Misc('PrefixOperator', 'oper_prefix', serialization_code=109),
|
|
Misc('DollarIdentifier', 'dollarident', classification='DollarIdentifier',
|
|
serialization_code=106),
|
|
|
|
Misc('ContextualKeyword', 'contextual_keyword', classification='Keyword',
|
|
serialization_code=114),
|
|
Misc('RawStringDelimiter', 'raw_string_delimiter', serialization_code=119),
|
|
Misc('StringSegment', 'string_segment', classification='StringLiteral',
|
|
serialization_code=104),
|
|
Misc('StringInterpolationAnchor', 'string_interpolation_anchor',
|
|
text=')', classification='StringInterpolationAnchor',
|
|
serialization_code=101),
|
|
Misc('Yield', 'kw_yield', serialization_code=116, text='yield'),
|
|
|
|
]
|
|
|
|
SYNTAX_TOKEN_MAP = {token.name + 'Token': token for token in SYNTAX_TOKENS}
|
|
|
|
|
|
def verify_no_duplicate_serialization_codes(tokens):
|
|
used_codes = set()
|
|
for token in tokens:
|
|
if token.serialization_code in used_codes:
|
|
error("Serialization code %d used twice for tokens" %
|
|
token.serialization_code)
|
|
used_codes.add(token.serialization_code)
|
|
|
|
|
|
verify_no_duplicate_serialization_codes(SYNTAX_TOKENS)
|