averello/swift-mirror
1
0
Fork 0
mirror of https://github.com/apple/swift.git synced 2025-12-21 12:14:44 +01:00
Code Issues Packages Projects Releases Wiki Activity

Lexer: refactor constructors to bring the parameter count back to a sane number

Browse Source 
Now we have a clear separation between a primary lexer, which scans the whole
buffer, and a sublexer, which can be created from a primary lexer to scan a
part of the buffer.


Swift SVN r7077
This commit is contained in:
Dmitri Hrybenko
2013-08-09 02:06:50 +00:00
parent 591410ea23
commit c92fd7cbdd
3 changed files with 118 additions and 101 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
lib/Parse/Parser.cpp
View File

@@ -199,12 +199,15 @@ std::vector<Token> swift::tokenize(SourceManager &SM, unsigned BufferID,
unsigned Offset, unsigned EndOffset,
bool KeepComments,
bool TokenizeInterpolatedString) {
Lexer L(SM, BufferID, /*Diags=*/nullptr, /*InSILMode=*/false, KeepComments,
Offset, EndOffset);
if (Offset == 0 && EndOffset == 0)
EndOffset = SM->getMemoryBuffer(BufferID)->getBufferSize();
Lexer L(SM, BufferID, /*Diags=*/nullptr, /*InSILMode=*/false, KeepComments);
Lexer SubLexer(L, Offset, EndOffset);
std::vector<Token> Tokens;
do {
Tokens.emplace_back();
L.lex(Tokens.back());
SubLexer.lex(Tokens.back());
if (Tokens.back().is(tok::string_literal) && TokenizeInterpolatedString) {
Token StrTok = Tokens.back();
Tokens.pop_back();
@@ -286,9 +289,8 @@ SourceLoc Parser::consumeStartingLess() {
return consumeToken();
// Skip the starting '<' in the existing token.
SourceLoc Loc = Tok.getLoc();
StringRef Remaining = Tok.getText().substr(1);
Tok.setToken(L->getTokenKind(Remaining), Remaining);
SourceLoc Loc = Tok.getLoc();
Tok = L->getTokenAt(Loc.getAdvancedLoc(1));
return Loc;
}
@@ -300,8 +302,7 @@ SourceLoc Parser::consumeStartingGreater() {
// Skip the starting '>' in the existing token.
SourceLoc Loc = Tok.getLoc();
StringRef Remaining = Tok.getText().substr(1);
Tok.setToken(L->getTokenKind(Remaining), Remaining);
Tok = L->getTokenAt(Loc.getAdvancedLoc(1));
return Loc;
}