godotengine
/
godot
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #72058 from bitsawer/shader_preprocessor_fixes 

Several shader preprocessor parser fixes and improvements
This commit is contained in:
Rémi Verschelde 2023-01-26 22:51:06 +01:00
parent 0a19973245 6f5598979f
commit c250f6eeb6
No known key found for this signature in database
GPG Key ID: C3336907360768E1
3 changed files with 114 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
servers/rendering/shader_language.cpp
View File

@ -622,7 +622,7 @@ ShaderLanguage::Token ShaderLanguage::_get_token() {
char_idx += 2;
include_positions.resize(include_positions.size() - 1); // Pop back.
tk_line = include_positions[include_positions.size() - 1].line; // Restore line.
tk_line = include_positions[include_positions.size() - 1].line - 1; // Restore line.
} else {
return _make_token(TK_ERROR, "Invalid include enter/exit hint token (@@> and @@<)");

145
servers/rendering/shader_preprocessor.cpp
View File

@ -78,19 +78,46 @@ char32_t ShaderPreprocessor::Tokenizer::peek() {
return 0;
}
int ShaderPreprocessor::Tokenizer::consume_line_continuations(int p_offset) {
int skips = 0;
for (int i = index + p_offset; i < size; i++) {
char32_t c = code[i];
if (c == '\\') {
if (i + 1 < size && code[i + 1] == '\n') {
// This line ends with "\" and "\n" continuation.
add_generated(Token('\n', line));
line++;
skips++;
i = i + 2;
index = i;
} else {
break;
}
} else if (!is_whitespace(c)) {
break;
}
}
return skips;
}
LocalVector<ShaderPreprocessor::Token> ShaderPreprocessor::Tokenizer::advance(char32_t p_what) {
LocalVector<ShaderPreprocessor::Token> tokens;
while (index < size) {
char32_t c = code[index++];
tokens.push_back(ShaderPreprocessor::Token(c, line));
if (c == '\\' && consume_line_continuations(-1) > 0) {
continue;
}
if (c == '\n') {
add_generated(ShaderPreprocessor::Token('\n', line));
line++;
}
tokens.push_back(ShaderPreprocessor::Token(c, line));
if (c == p_what || c == 0) {
return tokens;
}
@ -104,6 +131,11 @@ void ShaderPreprocessor::Tokenizer::skip_whitespace() {
}
}
bool ShaderPreprocessor::Tokenizer::consume_empty_line() {
// Read until newline and return true if the content was all whitespace/empty.
return tokens_to_string(advance('\n')).strip_edges().size() == 0;
}
String ShaderPreprocessor::Tokenizer::get_identifier(bool *r_is_cursor, bool p_started) {
if (r_is_cursor != nullptr) {
*r_is_cursor = false;
@ -113,6 +145,10 @@ String ShaderPreprocessor::Tokenizer::get_identifier(bool *r_is_cursor, bool p_s
while (true) {
char32_t c = peek();
if (c == '\\' && consume_line_continuations(0) > 0) {
continue;
}
if (is_char_end(c) || c == '(' || c == ')' || c == ',' || c == ';') {
break;
}
@ -146,8 +182,10 @@ String ShaderPreprocessor::Tokenizer::get_identifier(bool *r_is_cursor, bool p_s
String ShaderPreprocessor::Tokenizer::peek_identifier() {
const int original = index;
const int original_line = line;
String id = get_identifier();
index = original;
line = original_line;
return id;
}
@ -485,7 +523,9 @@ void ShaderPreprocessor::process_else(Tokenizer *p_tokenizer) {
state->previous_region->to_line = line - 1;
}
p_tokenizer->advance('\n');
if (!p_tokenizer->consume_empty_line()) {
set_error(RTR("Invalid else."), p_tokenizer->get_line());
}
bool skip = false;
for (int i = 0; i < state->current_branch->conditions.size(); i++) {
@ -508,17 +548,21 @@ void ShaderPreprocessor::process_else(Tokenizer *p_tokenizer) {
}
void ShaderPreprocessor::process_endif(Tokenizer *p_tokenizer) {
const int line = p_tokenizer->get_line();
state->condition_depth--;
if (state->condition_depth < 0) {
set_error(RTR("Unmatched endif."), p_tokenizer->get_line());
set_error(RTR("Unmatched endif."), line);
return;
}
if (state->previous_region != nullptr) {
state->previous_region->to_line = p_tokenizer->get_line() - 1;
state->previous_region->to_line = line - 1;
state->previous_region = state->previous_region->parent;
}
p_tokenizer->advance('\n');
if (!p_tokenizer->consume_empty_line()) {
set_error(RTR("Invalid endif."), line);
}
state->current_branch = state->current_branch->parent;
state->branches.pop_back();
@ -574,12 +618,10 @@ void ShaderPreprocessor::process_ifdef(Tokenizer *p_tokenizer) {
return;
}
p_tokenizer->skip_whitespace();
if (!is_char_end(p_tokenizer->peek())) {
if (!p_tokenizer->consume_empty_line()) {
set_error(RTR("Invalid ifdef."), line);
return;
}
p_tokenizer->advance('\n');
bool success = state->defines.has(label);
start_branch_condition(p_tokenizer, success);
@ -598,12 +640,10 @@ void ShaderPreprocessor::process_ifndef(Tokenizer *p_tokenizer) {
return;
}
p_tokenizer->skip_whitespace();
if (!is_char_end(p_tokenizer->peek())) {
if (!p_tokenizer->consume_empty_line()) {
set_error(RTR("Invalid ifndef."), line);
return;
}
p_tokenizer->advance('\n');
bool success = !state->defines.has(label);
start_branch_condition(p_tokenizer, success);
@ -628,9 +668,8 @@ void ShaderPreprocessor::process_include(Tokenizer *p_tokenizer) {
}
}
path = path.substr(0, path.length() - 1);
p_tokenizer->skip_whitespace();
if (path.is_empty() || !is_char_end(p_tokenizer->peek())) {
if (path.is_empty() || !p_tokenizer->consume_empty_line()) {
set_error(RTR("Invalid path."), line);
return;
}
@ -728,25 +767,24 @@ void ShaderPreprocessor::process_pragma(Tokenizer *p_tokenizer) {
return;
}
p_tokenizer->advance('\n');
if (!p_tokenizer->consume_empty_line()) {
set_error(RTR("Invalid pragma directive."), line);
return;
}
}
void ShaderPreprocessor::process_undef(Tokenizer *p_tokenizer) {
const int line = p_tokenizer->get_line();
const String label = p_tokenizer->get_identifier();
if (label.is_empty() || !state->defines.has(label)) {
set_error(RTR("Invalid name."), line);
return;
}
p_tokenizer->skip_whitespace();
if (!is_char_end(p_tokenizer->peek())) {
if (label.is_empty() || !p_tokenizer->consume_empty_line()) {
set_error(RTR("Invalid undef."), line);
return;
}
memdelete(state->defines[label]);
state->defines.erase(label);
if (state->defines.has(label)) {
memdelete(state->defines[label]);
state->defines.erase(label);
}
}
void ShaderPreprocessor::add_region(int p_line, bool p_enabled, Region *p_parent_region) {
@ -957,15 +995,57 @@ bool ShaderPreprocessor::expand_macros_once(const String &p_line, int p_line_num
String body = define->body;
if (define->arguments.size() > 0) {
// Complex macro with arguments.
int args_start = index + key.length();
int args_end = p_line.find(")", args_start);
if (args_start == -1 || args_end == -1) {
set_error(RTR("Missing macro argument parenthesis."), p_line_number);
return false;
int args_start = -1;
int args_end = -1;
int brackets_open = 0;
Vector<String> args;
for (int i = index_start - 1; i < p_line.length(); i++) {
bool add_argument = false;
bool reached_end = false;
char32_t c = p_line[i];
if (c == '(') {
brackets_open++;
if (brackets_open == 1) {
args_start = i + 1;
args_end = -1;
}
} else if (c == ')') {
brackets_open--;
if (brackets_open == 0) {
args_end = i;
add_argument = true;
reached_end = true;
}
} else if (c == ',') {
if (brackets_open == 1) {
args_end = i;
add_argument = true;
}
}
if (add_argument) {
if (args_start == -1 || args_end == -1) {
set_error(RTR("Invalid macro argument list."), p_line_number);
return false;
}
String arg = p_line.substr(args_start, args_end - args_start).strip_edges();
if (arg.is_empty()) {
set_error(RTR("Invalid macro argument."), p_line_number);
return false;
}
args.append(arg);
args_start = args_end + 1;
}
if (reached_end) {
break;
}
}
String values = result.substr(args_start + 1, args_end - (args_start + 1));
Vector<String> args = values.split(",");
if (args.size() != define->arguments.size()) {
set_error(RTR("Invalid macro argument count."), p_line_number);
return false;
@ -987,9 +1067,6 @@ bool ShaderPreprocessor::expand_macros_once(const String &p_line, int p_line_num
result = result.substr(0, index) + " " + body + " " + result.substr(args_end + 1, result.length());
} else {
result = result.substr(0, index) + body + result.substr(index + key.length(), result.length() - (index + key.length()));
// Manually reset index_start to where the body value of the define finishes.
// This ensures we don't skip another instance of this macro in the string.
index_start = index + body.length() + 1;
}
r_expanded = result;

2
servers/rendering/shader_preprocessor.h
View File

@ -93,11 +93,13 @@ private:
int get_line() const;
int get_index() const;
char32_t peek();
int consume_line_continuations(int p_offset);
void get_and_clear_generated(Vector<Token> *r_out);
void backtrack(char32_t p_what);
LocalVector<Token> advance(char32_t p_what);
void skip_whitespace();
bool consume_empty_line();
String get_identifier(bool *r_is_cursor = nullptr, bool p_started = false);
String peek_identifier();
Token get_token();