godotengine
/
godot
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #32808 from bojidar-bg/30937-less-strict-mixed-spacing 

Allow mixed tabs and spaces when indentation does not depend on tab size
This commit is contained in:
Rémi Verschelde 2019-10-25 23:16:04 +02:00 committed by GitHub
parent 6a74113150 afbde3314a
commit c43c3747cf
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 103 additions and 78 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

76
modules/gdscript/gdscript_parser.cpp
View File

@ -89,8 +89,8 @@ bool GDScriptParser::_enter_indent_block(BlockNode *p_block) {
if (tokenizer->get_token() != GDScriptTokenizer::TK_NEWLINE) { if (tokenizer->get_token() != GDScriptTokenizer::TK_NEWLINE) {
// be more python-like // be more python-like
int current = tab_level.back()->get(); IndentLevel current_level = indent_level.back()->get();
tab_level.push_back(current); indent_level.push_back(current_level);
return true; return true;
//_set_error("newline expected after ':'."); //_set_error("newline expected after ':'.");
//return false; //return false;
@ -105,12 +105,19 @@ bool GDScriptParser::_enter_indent_block(BlockNode *p_block) {
} else if (tokenizer->get_token(1) != GDScriptTokenizer::TK_NEWLINE) { } else if (tokenizer->get_token(1) != GDScriptTokenizer::TK_NEWLINE) {
int indent = tokenizer->get_token_line_indent(); int indent = tokenizer->get_token_line_indent();
int current = tab_level.back()->get(); int tabs = tokenizer->get_token_line_tab_indent();
if (indent <= current) { IndentLevel current_level = indent_level.back()->get();
IndentLevel new_indent(indent, tabs);
if (new_indent.is_mixed(current_level)) {
_set_error("Mixed tabs and spaces in indentation.");
return false; return false;
} }
tab_level.push_back(indent); if (indent <= current_level.indent) {
return false;
}
indent_level.push_back(new_indent);
tokenizer->advance(); tokenizer->advance();
return true; return true;
@ -2225,7 +2232,7 @@ GDScriptParser::PatternNode *GDScriptParser::_parse_pattern(bool p_static) {
} }
void GDScriptParser::_parse_pattern_block(BlockNode *p_block, Vector<PatternBranchNode *> &p_branches, bool p_static) { void GDScriptParser::_parse_pattern_block(BlockNode *p_block, Vector<PatternBranchNode *> &p_branches, bool p_static) {
int indent_level = tab_level.back()->get(); IndentLevel current_level = indent_level.back()->get();
p_block->has_return = true; p_block->has_return = true;
@ -2240,7 +2247,7 @@ void GDScriptParser::_parse_pattern_block(BlockNode *p_block, Vector<PatternBran
if (error_set) if (error_set)
return; return;
if (indent_level > tab_level.back()->get()) { if (current_level.indent > indent_level.back()->get().indent) {
break; // go back a level break; // go back a level
} }
@ -2695,7 +2702,7 @@ void GDScriptParser::_transform_match_statment(MatchNode *p_match_statement) {
void GDScriptParser::_parse_block(BlockNode *p_block, bool p_static) { void GDScriptParser::_parse_block(BlockNode *p_block, bool p_static) {
int indent_level = tab_level.back()->get(); IndentLevel current_level = indent_level.back()->get();
#ifdef DEBUG_ENABLED #ifdef DEBUG_ENABLED
@ -2708,9 +2715,13 @@ void GDScriptParser::_parse_block(BlockNode *p_block, bool p_static) {
bool is_first_line = true; bool is_first_line = true;
while (true) { while (true) {
if (!is_first_line && tab_level.back()->prev() && tab_level.back()->prev()->get() == indent_level) { if (!is_first_line && indent_level.back()->prev() && indent_level.back()->prev()->get().indent == current_level.indent) {
if (indent_level.back()->prev()->get().is_mixed(current_level)) {
_set_error("Mixed tabs and spaces in indentation.");
return;
}
// pythonic single-line expression, don't parse future lines // pythonic single-line expression, don't parse future lines
tab_level.pop_back(); indent_level.pop_back();
p_block->end_line = tokenizer->get_token_line(); p_block->end_line = tokenizer->get_token_line();
return; return;
} }
@ -2720,7 +2731,7 @@ void GDScriptParser::_parse_block(BlockNode *p_block, bool p_static) {
if (error_set) if (error_set)
return; return;
if (indent_level > tab_level.back()->get()) { if (current_level.indent > indent_level.back()->get().indent) {
p_block->end_line = tokenizer->get_token_line(); p_block->end_line = tokenizer->get_token_line();
return; //go back a level return; //go back a level
} }
@ -2924,14 +2935,14 @@ void GDScriptParser::_parse_block(BlockNode *p_block, bool p_static) {
while (tokenizer->get_token() == GDScriptTokenizer::TK_NEWLINE && _parse_newline()) while (tokenizer->get_token() == GDScriptTokenizer::TK_NEWLINE && _parse_newline())
; ;
if (tab_level.back()->get() < indent_level) { //not at current indent level if (indent_level.back()->get().indent < current_level.indent) { //not at current indent level
p_block->end_line = tokenizer->get_token_line(); p_block->end_line = tokenizer->get_token_line();
return; return;
} }
if (tokenizer->get_token() == GDScriptTokenizer::TK_CF_ELIF) { if (tokenizer->get_token() == GDScriptTokenizer::TK_CF_ELIF) {
if (tab_level.back()->get() > indent_level) { if (indent_level.back()->get().indent > current_level.indent) {
_set_error("Invalid indentation."); _set_error("Invalid indentation.");
return; return;
@ -2979,7 +2990,7 @@ void GDScriptParser::_parse_block(BlockNode *p_block, bool p_static) {
} else if (tokenizer->get_token() == GDScriptTokenizer::TK_CF_ELSE) { } else if (tokenizer->get_token() == GDScriptTokenizer::TK_CF_ELSE) {
if (tab_level.back()->get() > indent_level) { if (indent_level.back()->get().indent > current_level.indent) {
_set_error("Invalid indentation."); _set_error("Invalid indentation.");
return; return;
} }
@ -3351,32 +3362,45 @@ bool GDScriptParser::_parse_newline() {
if (tokenizer->get_token(1) != GDScriptTokenizer::TK_EOF && tokenizer->get_token(1) != GDScriptTokenizer::TK_NEWLINE) { if (tokenizer->get_token(1) != GDScriptTokenizer::TK_EOF && tokenizer->get_token(1) != GDScriptTokenizer::TK_NEWLINE) {
IndentLevel current_level = indent_level.back()->get();
int indent = tokenizer->get_token_line_indent(); int indent = tokenizer->get_token_line_indent();
int current_indent = tab_level.back()->get(); int tabs = tokenizer->get_token_line_tab_indent();
IndentLevel new_level(indent, tabs);
if (indent > current_indent) { if (new_level.is_mixed(current_level)) {
_set_error("Mixed tabs and spaces in indentation.");
return false;
}
if (indent > current_level.indent) {
_set_error("Unexpected indentation."); _set_error("Unexpected indentation.");
return false; return false;
} }
if (indent < current_indent) { if (indent < current_level.indent) {
while (indent < current_indent) { while (indent < current_level.indent) {
//exit block //exit block
if (tab_level.size() == 1) { if (indent_level.size() == 1) {
_set_error("Invalid indentation. Bug?"); _set_error("Invalid indentation. Bug?");
return false; return false;
} }
tab_level.pop_back(); indent_level.pop_back();
if (tab_level.back()->get() < indent) { if (indent_level.back()->get().indent < indent) {
_set_error("Unindent does not match any outer indentation level."); _set_error("Unindent does not match any outer indentation level.");
return false; return false;
} }
current_indent = tab_level.back()->get();
if (indent_level.back()->get().is_mixed(current_level)) {
_set_error("Mixed tabs and spaces in indentation.");
return false;
}
current_level = indent_level.back()->get();
} }
tokenizer->advance(); tokenizer->advance();
@ -3474,7 +3498,7 @@ void GDScriptParser::_parse_extends(ClassNode *p_class) {
void GDScriptParser::_parse_class(ClassNode *p_class) { void GDScriptParser::_parse_class(ClassNode *p_class) {
int indent_level = tab_level.back()->get(); IndentLevel current_level = indent_level.back()->get();
while (true) { while (true) {
@ -3482,7 +3506,7 @@ void GDScriptParser::_parse_class(ClassNode *p_class) {
if (error_set) if (error_set)
return; return;
if (indent_level > tab_level.back()->get()) { if (current_level.indent > indent_level.back()->get().indent) {
p_class->end_line = tokenizer->get_token_line(); p_class->end_line = tokenizer->get_token_line();
return; //go back a level return; //go back a level
} }
@ -8562,8 +8586,8 @@ void GDScriptParser::clear() {
validating = false; validating = false;
for_completion = false; for_completion = false;
error_set = false; error_set = false;
tab_level.clear(); indent_level.clear();
tab_level.push_back(0); indent_level.push_back(IndentLevel(0, 0));
error_line = 0; error_line = 0;
error_column = 0; error_column = 0;
pending_newline = -1; pending_newline = -1;

22
modules/gdscript/gdscript_parser.h
View File

@ -552,7 +552,27 @@ private:
int pending_newline; int pending_newline;
List<int> tab_level; struct IndentLevel {
int indent;
int tabs;
bool is_mixed(IndentLevel other) {
return (
(indent == other.indent && tabs != other.tabs) ||
(indent > other.indent && tabs < other.tabs) ||
(indent < other.indent && tabs > other.tabs));
}
IndentLevel() :
indent(0),
tabs(0) {}
IndentLevel(int p_indent, int p_tabs) :
indent(p_indent),
tabs(p_tabs) {}
};
List<IndentLevel> indent_level;
String base_path; String base_path;
String self_path; String self_path;

73
modules/gdscript/gdscript_tokenizer.cpp
View File

@ -450,11 +450,11 @@ void GDScriptTokenizerText::_make_error(const String &p_error) {
tk_rb_pos = (tk_rb_pos + 1) % TK_RB_SIZE; tk_rb_pos = (tk_rb_pos + 1) % TK_RB_SIZE;
} }
void GDScriptTokenizerText::_make_newline(int p_spaces) { void GDScriptTokenizerText::_make_newline(int p_indentation, int p_tabs) {
TokenData &tk = tk_rb[tk_rb_pos]; TokenData &tk = tk_rb[tk_rb_pos];
tk.type = TK_NEWLINE; tk.type = TK_NEWLINE;
tk.constant = p_spaces; tk.constant = Vector2(p_indentation, p_tabs);
tk.line = line; tk.line = line;
tk.col = column; tk.col = column;
tk_rb_pos = (tk_rb_pos + 1) % TK_RB_SIZE; tk_rb_pos = (tk_rb_pos + 1) % TK_RB_SIZE;
@ -511,33 +511,6 @@ void GDScriptTokenizerText::_advance() {
case ' ': case ' ':
INCPOS(1); INCPOS(1);
continue; continue;
case '\n': {
line++;
INCPOS(1);
column = 1;
int i = 0;
while (true) {
if (GETCHAR(i) == ' ') {
if (file_indent_type == INDENT_NONE) file_indent_type = INDENT_SPACES;
if (file_indent_type != INDENT_SPACES) {
_make_error("Spaces used for indentation in tab-indented file!");
return;
}
} else if (GETCHAR(i) == '\t') {
if (file_indent_type == INDENT_NONE) file_indent_type = INDENT_TABS;
if (file_indent_type != INDENT_TABS) {
_make_error("Tabs used for indentation in space-indented file!");
return;
}
} else {
break; // not indentation anymore
}
i++;
}
_make_newline(i);
return;
}
case '#': { // line comment skip case '#': { // line comment skip
#ifdef DEBUG_ENABLED #ifdef DEBUG_ENABLED
String comment; String comment;
@ -565,33 +538,34 @@ void GDScriptTokenizerText::_advance() {
ignore_warnings = true; ignore_warnings = true;
} }
#endif // DEBUG_ENABLED #endif // DEBUG_ENABLED
INCPOS(1); FALLTHROUGH;
column = 1; }
case '\n': {
line++; line++;
INCPOS(1);
bool used_spaces = false;
int tabs = 0;
column = 1;
int i = 0; int i = 0;
while (true) { while (true) {
if (GETCHAR(i) == ' ') { if (GETCHAR(i) == ' ') {
if (file_indent_type == INDENT_NONE) file_indent_type = INDENT_SPACES; i++;
if (file_indent_type != INDENT_SPACES) { used_spaces = true;
_make_error("Spaces used for indentation in tab-indented file!");
return;
}
} else if (GETCHAR(i) == '\t') { } else if (GETCHAR(i) == '\t') {
if (file_indent_type == INDENT_NONE) file_indent_type = INDENT_TABS; if (used_spaces) {
if (file_indent_type != INDENT_TABS) { _make_error("Spaces used before tabs on a line");
_make_error("Tabs used for indentation in space-indented file!");
return; return;
} }
i++;
tabs++;
} else { } else {
break; // not indentation anymore break; // not indentation anymore
} }
i++;
} }
_make_newline(i); _make_newline(i, tabs);
return; return;
}
} break;
case '/': { case '/': {
switch (GETCHAR(1)) { switch (GETCHAR(1)) {
@ -1108,7 +1082,6 @@ void GDScriptTokenizerText::set_code(const String &p_code) {
ignore_warnings = false; ignore_warnings = false;
#endif // DEBUG_ENABLED #endif // DEBUG_ENABLED
last_error = ""; last_error = "";
file_indent_type = INDENT_NONE;
for (int i = 0; i < MAX_LOOKAHEAD + 1; i++) for (int i = 0; i < MAX_LOOKAHEAD + 1; i++)
_advance(); _advance();
} }
@ -1183,7 +1156,17 @@ int GDScriptTokenizerText::get_token_line_indent(int p_offset) const {
int ofs = (TK_RB_SIZE + tk_rb_pos + p_offset - MAX_LOOKAHEAD - 1) % TK_RB_SIZE; int ofs = (TK_RB_SIZE + tk_rb_pos + p_offset - MAX_LOOKAHEAD - 1) % TK_RB_SIZE;
ERR_FAIL_COND_V(tk_rb[ofs].type != TK_NEWLINE, 0); ERR_FAIL_COND_V(tk_rb[ofs].type != TK_NEWLINE, 0);
return tk_rb[ofs].constant; return tk_rb[ofs].constant.operator Vector2().x;
}
int GDScriptTokenizerText::get_token_line_tab_indent(int p_offset) const {
ERR_FAIL_COND_V(p_offset <= -MAX_LOOKAHEAD, 0);
ERR_FAIL_COND_V(p_offset >= MAX_LOOKAHEAD, 0);
int ofs = (TK_RB_SIZE + tk_rb_pos + p_offset - MAX_LOOKAHEAD - 1) % TK_RB_SIZE;
ERR_FAIL_COND_V(tk_rb[ofs].type != TK_NEWLINE, 0);
return tk_rb[ofs].constant.operator Vector2().y;
} }
String GDScriptTokenizerText::get_token_error(int p_offset) const { String GDScriptTokenizerText::get_token_error(int p_offset) const {

10
modules/gdscript/gdscript_tokenizer.h
View File

@ -168,6 +168,7 @@ public:
virtual int get_token_line(int p_offset = 0) const = 0; virtual int get_token_line(int p_offset = 0) const = 0;
virtual int get_token_column(int p_offset = 0) const = 0; virtual int get_token_column(int p_offset = 0) const = 0;
virtual int get_token_line_indent(int p_offset = 0) const = 0; virtual int get_token_line_indent(int p_offset = 0) const = 0;
virtual int get_token_line_tab_indent(int p_offset = 0) const = 0;
virtual String get_token_error(int p_offset = 0) const = 0; virtual String get_token_error(int p_offset = 0) const = 0;
virtual void advance(int p_amount = 1) = 0; virtual void advance(int p_amount = 1) = 0;
#ifdef DEBUG_ENABLED #ifdef DEBUG_ENABLED
@ -205,7 +206,7 @@ class GDScriptTokenizerText : public GDScriptTokenizer {
}; };
void _make_token(Token p_type); void _make_token(Token p_type);
void _make_newline(int p_spaces = 0); void _make_newline(int p_indentation = 0, int p_tabs = 0);
void _make_identifier(const StringName &p_identifier); void _make_identifier(const StringName &p_identifier);
void _make_built_in_func(GDScriptFunctions::Function p_func); void _make_built_in_func(GDScriptFunctions::Function p_func);
void _make_constant(const Variant &p_constant); void _make_constant(const Variant &p_constant);
@ -222,11 +223,6 @@ class GDScriptTokenizerText : public GDScriptTokenizer {
int tk_rb_pos; int tk_rb_pos;
String last_error; String last_error;
bool error_flag; bool error_flag;
enum {
INDENT_NONE,
INDENT_SPACES,
INDENT_TABS,
} file_indent_type;
#ifdef DEBUG_ENABLED #ifdef DEBUG_ENABLED
Vector<Pair<int, String> > warning_skips; Vector<Pair<int, String> > warning_skips;
@ -245,6 +241,7 @@ public:
virtual int get_token_line(int p_offset = 0) const; virtual int get_token_line(int p_offset = 0) const;
virtual int get_token_column(int p_offset = 0) const; virtual int get_token_column(int p_offset = 0) const;
virtual int get_token_line_indent(int p_offset = 0) const; virtual int get_token_line_indent(int p_offset = 0) const;
virtual int get_token_line_tab_indent(int p_offset = 0) const;
virtual const Variant &get_token_constant(int p_offset = 0) const; virtual const Variant &get_token_constant(int p_offset = 0) const;
virtual String get_token_error(int p_offset = 0) const; virtual String get_token_error(int p_offset = 0) const;
virtual void advance(int p_amount = 1); virtual void advance(int p_amount = 1);
@ -283,6 +280,7 @@ public:
virtual int get_token_line(int p_offset = 0) const; virtual int get_token_line(int p_offset = 0) const;
virtual int get_token_column(int p_offset = 0) const; virtual int get_token_column(int p_offset = 0) const;
virtual int get_token_line_indent(int p_offset = 0) const; virtual int get_token_line_indent(int p_offset = 0) const;
virtual int get_token_line_tab_indent(int p_offset = 0) const { return 0; }
virtual const Variant &get_token_constant(int p_offset = 0) const; virtual const Variant &get_token_constant(int p_offset = 0) const;
virtual String get_token_error(int p_offset = 0) const; virtual String get_token_error(int p_offset = 0) const;
virtual void advance(int p_amount = 1); virtual void advance(int p_amount = 1);