godotengine
/
godot
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Convert 3.x shaders

This commit is contained in:
nikitalita 2024-09-30 03:12:56 -07:00
parent 324ad7571a
commit 018f1ed97e
11 changed files with 3885 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

74
editor/plugins/text_shader_editor.cpp
View File

@ -42,6 +42,10 @@
#include "servers/rendering/shader_preprocessor.h"
#include "servers/rendering/shader_types.h"
#ifndef DISABLE_DEPRECATED
#include "servers/rendering/shader_converter.h"
#endif
/*** SHADER SYNTAX HIGHLIGHTER ****/
Dictionary GDShaderSyntaxHighlighter::_get_line_syntax_highlighting_impl(int p_line) {
@ -699,6 +703,28 @@ void TextShaderEditor::_menu_option(int p_option) {
case EDIT_COMPLETE: {
code_editor->get_text_editor()->request_code_completion();
} break;
#ifndef DISABLE_DEPRECATED
case EDIT_CONVERT: {
if (shader.is_null()) {
return;
}
String code = code_editor->get_text_editor()->get_text();
if (code.is_empty()) {
return;
}
ShaderDeprecatedConverter converter;
if (!converter.is_code_deprecated(code)) {
if (converter.get_error_text() != String()) {
shader_convert_error_dialog->set_text(vformat(RTR("Line %d: %s"), converter.get_error_line(), converter.get_error_text()));
shader_convert_error_dialog->popup_centered();
ERR_PRINT("Shader conversion failed: " + converter.get_error_text());
}
confirm_convert_shader->popup_centered();
return;
}
_convert_shader();
} break;
#endif
case SEARCH_FIND: {
code_editor->get_find_replace_bar()->popup_search();
} break;
@ -755,6 +781,36 @@ void TextShaderEditor::_notification(int p_what) {
} break;
}
}
#ifndef DISABLE_DEPRECATED
void TextShaderEditor::_convert_shader() {
if (shader.is_null()) {
return;
}
String code = code_editor->get_text_editor()->get_text();
if (code.is_empty()) {
return;
}
ShaderDeprecatedConverter converter;
if (!converter.convert_code(code)) {
String err_text = converter.get_error_text();
if (err_text.is_empty()) {
err_text = TTR("Unknown error occurred while converting the shader.");
} else if (converter.get_error_line() > 0) {
err_text = vformat("%s (line %d)", err_text, converter.get_error_line());
}
shader_convert_error_dialog->set_text(err_text);
shader_convert_error_dialog->popup_centered();
ERR_PRINT("Shader conversion failed: " + err_text);
return;
}
String new_code = converter.emit_code();
// Ensure undoable.
code_editor->get_text_editor()->set_text(new_code);
code_editor->get_text_editor()->tag_saved_version();
code_editor->_validate_script();
}
#endif
void TextShaderEditor::_editor_settings_changed() {
if (!EditorThemeManager::is_generated_theme_outdated() &&
@ -1172,6 +1228,11 @@ TextShaderEditor::TextShaderEditor() {
edit_menu->get_popup()->add_shortcut(ED_GET_SHORTCUT("ui_text_completion_query"), EDIT_COMPLETE);
edit_menu->get_popup()->connect(SceneStringName(id_pressed), callable_mp(this, &TextShaderEditor::_menu_option));
edit_menu->get_popup()->add_separator();
#ifndef DISABLE_DEPRECATED
edit_menu->get_popup()->add_item(TTR("Convert 3.x Shader"), EDIT_CONVERT);
edit_menu->get_popup()->connect(SceneStringName(id_pressed), callable_mp(this, &TextShaderEditor::_menu_option));
#endif
search_menu = memnew(MenuButton);
search_menu->set_shortcut_context(this);
search_menu->set_text(TTR("Search"));
@ -1254,7 +1315,20 @@ TextShaderEditor::TextShaderEditor() {
disk_changed->connect("custom_action", callable_mp(this, &TextShaderEditor::save_external_data));
add_child(disk_changed);
#ifndef DISABLE_DEPRECATED
shader_convert_error_dialog = memnew(AcceptDialog);
shader_convert_error_dialog->set_title(TTR("Error converting shader"));
shader_convert_error_dialog->set_hide_on_ok(true);
add_child(shader_convert_error_dialog);
confirm_convert_shader = memnew(ConfirmationDialog);
confirm_convert_shader->set_title(TTR("Confirm Convert 3.x Shader"));
confirm_convert_shader->set_text(TTR("This shader does not appear to be a 3.x shader.\nAre you sure you want to convert it?"));
confirm_convert_shader->get_ok_button()->set_text(TTR("Convert"));
confirm_convert_shader->get_cancel_button()->set_text(TTR("Cancel"));
confirm_convert_shader->connect("confirmed", callable_mp(this, &TextShaderEditor::_convert_shader));
add_child(confirm_convert_shader);
#endif
_editor_settings_changed();
code_editor->show_toggle_scripts_button(); // TODO: Disabled for now, because it doesn't work properly.
}

10
editor/plugins/text_shader_editor.h
View File

@ -125,6 +125,7 @@ class TextShaderEditor : public ShaderEditor {
EDIT_TOGGLE_WORD_WRAP,
EDIT_TOGGLE_COMMENT,
EDIT_COMPLETE,
EDIT_CONVERT,
SEARCH_FIND,
SEARCH_FIND_NEXT,
SEARCH_FIND_PREV,
@ -150,13 +151,20 @@ class TextShaderEditor : public ShaderEditor {
ConfirmationDialog *disk_changed = nullptr;
ShaderTextEditor *code_editor = nullptr;
#ifndef DISABLE_DEPRECATED
AcceptDialog *shader_convert_error_dialog = nullptr;
ConfirmationDialog *confirm_convert_shader = nullptr;
#endif
bool compilation_success = true;
void _menu_option(int p_option);
void _prepare_edit_menu();
mutable Ref<Shader> shader;
mutable Ref<ShaderInclude> shader_inc;
#ifndef DISABLE_DEPRECATED
void _convert_shader();
#endif
void _editor_settings_changed();
void _apply_editor_settings();
void _project_settings_changed();

69
scene/resources/shader.cpp
View File

@ -32,8 +32,11 @@
#include "shader.compat.inc"
#include "core/io/file_access.h"
#include "scene/scene_string_names.h"
#include "servers/rendering/rendering_server_globals.h"
#include "servers/rendering/shader_language.h"
#include "servers/rendering/shader_preprocessor.h"
#include "servers/rendering/shader_types.h"
#include "servers/rendering_server.h"
#include "texture.h"
@ -46,6 +49,12 @@
#endif
#endif
#ifndef DISABLE_DEPRECATED
#include "servers/rendering/shader_converter.h"
#endif
#define _LOAD_COMPAT_META_PROPERTY "_load_compat"
Shader::Mode Shader::get_mode() const {
return mode;
}
@ -70,6 +79,13 @@ void Shader::set_include_path(const String &p_path) {
include_path = p_path;
}
#ifndef DISABLE_DEPRECATED
ShaderLanguage::DataType _get_global_shader_uniform_type(const StringName &p_name) {
RS::GlobalShaderParameterType gvt = RenderingServerGlobals::material_storage->global_shader_parameter_get_type(p_name);
return (ShaderLanguage::DataType)RS::global_shader_uniform_type_get_shader_datatype(gvt);
}
#endif
void Shader::set_code(const String &p_code) {
for (const Ref<ShaderInclude> &E : include_dependencies) {
E->disconnect_changed(callable_mp(this, &Shader::_dependency_changed));
@ -77,6 +93,43 @@ void Shader::set_code(const String &p_code) {
code = p_code;
String pp_code = p_code;
#ifndef DISABLE_DEPRECATED
if (get_meta(_LOAD_COMPAT_META_PROPERTY, false)) {
// check if the Shader code compiles; if not, it's probably an old shader.
ShaderLanguage sl;
ShaderLanguage::ShaderCompileInfo info;
String mode_string = ShaderLanguage::get_shader_type(p_code);
RS::ShaderMode new_mode;
if (mode_string == "canvas_item") {
new_mode = RS::SHADER_CANVAS_ITEM;
} else if (mode_string == "particles") {
new_mode = RS::SHADER_PARTICLES;
} else if (mode_string == "spatial") {
new_mode = RS::SHADER_SPATIAL;
} else {
new_mode = RS::SHADER_MAX;
}
if (new_mode != RS::SHADER_MAX) {
info.functions = ShaderTypes::get_singleton()->get_functions(new_mode);
info.render_modes = ShaderTypes::get_singleton()->get_modes(new_mode);
info.shader_types = ShaderTypes::get_singleton()->get_types();
info.global_shader_uniform_type_func = _get_global_shader_uniform_type;
Error err = sl.compile(p_code, info);
if (err) {
ShaderDeprecatedConverter sdc;
if (sdc.is_code_deprecated(p_code)) {
ERR_FAIL_COND_MSG(!sdc.convert_code(p_code), vformat("Shader conversion failed (line %d): %s", sdc.get_error_line(), sdc.get_error_text()));
code = sdc.emit_code();
pp_code = code;
} else if (sdc.get_error_text() != "") { // Preprocessing failed.
WARN_PRINT(vformat("Shader conversion failed (line %d): %s", sdc.get_error_line(), sdc.get_error_text()));
} // If the code is reported as not deprecated, let it fall through to the compile step after this if block so that we get the full compile error.
}
}
}
#endif
{
String path = get_path();
@ -88,7 +141,7 @@ void Shader::set_code(const String &p_code) {
// 2) Server does not do interaction with Resource filetypes, this is a scene level feature.
HashSet<Ref<ShaderInclude>> new_include_dependencies;
ShaderPreprocessor preprocessor;
Error result = preprocessor.preprocess(p_code, path, pp_code, nullptr, nullptr, nullptr, &new_include_dependencies);
Error result = preprocessor.preprocess(code, path, pp_code, nullptr, nullptr, nullptr, &new_include_dependencies);
if (result == OK) {
// This ensures previous include resources are not freed and then re-loaded during parse (which would make compiling slower)
include_dependencies = new_include_dependencies;
@ -237,6 +290,20 @@ Array Shader::_get_shader_uniform_list(bool p_get_groups) {
return ret;
}
void Shader::_start_load(const StringName &p_res_format_type, int p_res_format_version) {
#ifndef DISABLE_DEPRECATED
if ((p_res_format_type == "binary" && p_res_format_version == 3) || (p_res_format_type == "text" && p_res_format_version == 2)) {
set_meta(_LOAD_COMPAT_META_PROPERTY, true);
}
#endif
}
void Shader::_finish_load(const StringName &p_res_format_type, int p_res_format_version) {
#ifndef DISABLE_DEPRECATED
set_meta(_LOAD_COMPAT_META_PROPERTY, Variant());
#endif
}
void Shader::_bind_methods() {
ClassDB::bind_method(D_METHOD("get_mode"), &Shader::get_mode);

3
scene/resources/shader.h
View File

@ -94,6 +94,9 @@ public:
virtual RID get_rid() const override;
virtual void _start_load(const StringName &p_res_format_type, int p_res_format_version) override;
virtual void _finish_load(const StringName &p_res_format_type, int p_res_format_version) override;
Shader();
~Shader();
};

2393
servers/rendering/shader_converter.cpp Normal file
View File

File diff suppressed because it is too large Load Diff

284
servers/rendering/shader_converter.h Normal file
View File

@ -0,0 +1,284 @@
/**************************************************************************/
/* shader_converter.h */
/**************************************************************************/
/* This file is part of: */
/* GODOT ENGINE */
/* https://godotengine.org */
/**************************************************************************/
/* Copyright (c) 2014-present Godot Engine contributors (see AUTHORS.md). */
/* Copyright (c) 2007-2014 Juan Linietsky, Ariel Manzur. */
/* */
/* Permission is hereby granted, free of charge, to any person obtaining */
/* a copy of this software and associated documentation files (the */
/* "Software"), to deal in the Software without restriction, including */
/* without limitation the rights to use, copy, modify, merge, publish, */
/* distribute, sublicense, and/or sell copies of the Software, and to */
/* permit persons to whom the Software is furnished to do so, subject to */
/* the following conditions: */
/* */
/* The above copyright notice and this permission notice shall be */
/* included in all copies or substantial portions of the Software. */
/* */
/* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, */
/* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF */
/* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. */
/* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY */
/* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, */
/* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE */
/* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */
/**************************************************************************/
#ifndef SHADER_CONVERTER_H
#define SHADER_CONVERTER_H
#ifndef DISABLE_DEPRECATED
#include "core/templates/pair.h"
#include "servers/rendering/shader_language.h"
#include "servers/rendering_server.h"
class ShaderDeprecatedConverter {
public:
using TokenType = ShaderLanguage::TokenType;
using Token = ShaderLanguage::Token;
using TT = TokenType;
using TokenE = List<Token>::Element;
ShaderDeprecatedConverter() {};
bool is_code_deprecated(const String &p_code);
bool convert_code(const String &p_code);
String get_error_text() const;
int get_error_line() const;
String emit_code() const;
void set_warning_comments(bool p_add_comments);
void set_fail_on_unported(bool p_fail_on_unported);
void set_assume_correct(bool p_assume_correct);
void set_force_reserved_word_replacement(bool p_force_reserved_word_replacement);
void set_verbose_comments(bool p_verbose_comments);
static bool tokentype_is_identifier(const TokenType &p_tk_type);
static bool tokentype_is_new_reserved_keyword(const TokenType &p_tk_type);
static bool tokentype_is_new_type(const TokenType &p_tk_type);
static bool tokentype_is_new_hint(const TokenType &p_tk);
static bool id_is_new_builtin_func(const String &p_name);
static String get_tokentype_text(TokenType p_tk_type);
static bool has_builtin_rename(RS::ShaderMode p_mode, const String &p_name, const String &p_function = "");
static String get_builtin_rename(const String &p_name);
static bool has_hint_replacement(const String &p_name);
static TokenType get_hint_replacement(const String &p_name);
static bool is_renamed_render_mode(RS::ShaderMode p_mode, const String &p_name);
static String get_render_mode_rename(const String &p_name);
static bool has_removed_render_mode(RS::ShaderMode p_mode, const String &p_name);
static bool can_remove_render_mode(const String &p_name);
static bool has_removed_type(const String &p_name);
static bool is_renamed_main_function(RS::ShaderMode p_mode, const String &p_name);
static bool is_renamee_main_function(RS::ShaderMode p_mode, const String &p_name);
static String get_main_function_rename(const String &p_name);
static TokenType get_renamed_function_type(const String &p_name);
static int get_renamed_function_arg_count(const String &p_name);
static bool is_removed_builtin(RS::ShaderMode p_mode, const String &p_name, const String &p_function = "");
static TokenType get_removed_builtin_uniform_type(const String &p_name);
static Vector<TokenType> get_removed_builtin_hints(const String &p_name);
static bool _rename_has_special_handling(const String &p_name);
static void _get_builtin_renames_list(List<String> *r_list);
static void _get_render_mode_renames_list(List<String> *r_list);
static void _get_hint_renames_list(List<String> *r_list);
static void _get_function_renames_list(List<String> *r_list);
static void _get_render_mode_removals_list(List<String> *r_list);
static void _get_builtin_removals_list(List<String> *r_list);
static void _get_type_removals_list(List<String> *r_list);
static void _get_new_builtin_funcs_list(List<String> *r_list);
static Vector<String> _get_funcs_builtin_rename(RS::ShaderMode p_mode, const String &p_name);
static Vector<String> _get_funcs_builtin_removal(RS::ShaderMode p_mode, const String &p_name);
struct RenamedBuiltins {
const char *name;
const char *replacement;
const Vector<Pair<RS::ShaderMode, Vector<String>>> mode_functions;
const bool special_handling;
};
struct RenamedRenderModes {
const RS::ShaderMode mode;
const char *name;
const char *replacement;
};
struct RenamedHints {
const char *name;
const ShaderLanguage::TokenType replacement;
};
struct RenamedFunctions {
const RS::ShaderMode mode;
const ShaderLanguage::TokenType type;
const int arg_count;
const char *name;
const char *replacement;
};
struct RemovedRenderModes {
const RS::ShaderMode mode;
const char *name;
const bool can_remove;
};
struct RemovedBuiltins {
const char *name;
const ShaderLanguage::TokenType uniform_type;
const Vector<ShaderLanguage::TokenType> hints;
const Vector<Pair<RS::ShaderMode, Vector<String>>> mode_functions;
};
private:
struct UniformDecl {
List<Token>::Element *start_pos = nullptr;
List<Token>::Element *uniform_stmt_pos = nullptr;
List<Token>::Element *end_pos = nullptr;
List<Token>::Element *interp_qual_pos = nullptr;
List<Token>::Element *type_pos = nullptr;
List<Token>::Element *name_pos = nullptr;
Vector<List<Token>::Element *> hint_poses;
bool is_array = false;
bool has_uniform_qual() const {
return start_pos != nullptr && ShaderLanguage::is_token_uniform_qual(start_pos->get().type);
}
bool has_interp_qual() const {
return interp_qual_pos != nullptr;
}
};
struct VarDecl {
List<Token>::Element *start_pos = nullptr;
List<Token>::Element *end_pos = nullptr; // semicolon or comma or right paren
List<Token>::Element *type_pos = nullptr;
List<Token>::Element *name_pos = nullptr;
bool is_array = false;
bool new_arr_style_decl = false;
bool is_func_arg = false;
void clear() {
start_pos = nullptr;
end_pos = nullptr;
type_pos = nullptr;
name_pos = nullptr;
}
};
struct FunctionDecl {
List<Token>::Element *start_pos = nullptr;
List<Token>::Element *type_pos = nullptr;
List<Token>::Element *name_pos = nullptr;
List<Token>::Element *args_start_pos = nullptr; // left paren
List<Token>::Element *args_end_pos = nullptr; // right paren
List<Token>::Element *body_start_pos = nullptr; // left curly
List<Token>::Element *body_end_pos = nullptr; // right curly - end of function
bool is_renamed_main_function(RS::ShaderMode p_mode) const;
bool is_new_main_function(RS::ShaderMode p_mode) const;
int arg_count = 0;
bool has_array_return_type = false;
void clear() {
type_pos = nullptr;
name_pos = nullptr;
args_start_pos = nullptr;
args_end_pos = nullptr;
body_start_pos = nullptr;
body_end_pos = nullptr;
}
};
static const RenamedBuiltins renamed_builtins[];
static const RenamedRenderModes renamed_render_modes[];
static const RenamedHints renamed_hints[];
static const RenamedFunctions renamed_functions[];
static const RemovedRenderModes removed_render_modes[];
static const RemovedBuiltins removed_builtins[];
static const char *removed_types[];
static const char *old_builtin_funcs[];
static HashSet<String> _new_builtin_funcs;
String old_code;
List<Token> code_tokens;
List<Token>::Element *curr_ptr = nullptr;
List<Token>::Element *after_shader_decl = nullptr;
HashMap<String, UniformDecl> uniform_decls;
HashMap<String, Vector<VarDecl>> var_decls;
HashMap<String, FunctionDecl> function_decls;
HashMap<String, HashSet<String>> scope_declarations;
RenderingServer::ShaderMode shader_mode = RenderingServer::ShaderMode::SHADER_MAX;
bool warning_comments = true;
bool verbose_comments = false;
bool fail_on_unported = true;
bool function_pass_failed = false;
bool var_pass_failed = false;
String err_str;
int err_line = 0;
Token eof_token{ ShaderLanguage::TK_EOF, {}, 0, 0, 0, 0 };
static RS::ShaderMode get_shader_mode_from_string(const String &p_mode);
String get_token_literal_text(const Token &p_tk) const;
static Token mkTok(TokenType p_type, const StringName &p_text = StringName(), double constant = 0, uint16_t p_line = 0);
static bool token_is_skippable(const Token &p_tk);
static bool token_is_type(const Token &p_tk);
static bool token_is_hint(const Token &p_tk);
void reset();
bool _preprocess_code();
List<Token>::Element *get_next_token();
List<Token>::Element *get_prev_token();
List<Token>::Element *remove_cur_and_get_next();
TokenType peek_next_tk_type(uint32_t p_count = 1) const;
TokenType peek_prev_tk_type(uint32_t p_count = 1) const;
List<Token>::Element *get_pos() const;
bool reset_to(List<Token>::Element *p_pos);
bool insert_after(const Vector<Token> &p_token_list, List<Token>::Element *p_pos);
bool insert_before(const Vector<Token> &p_token_list, List<Token>::Element *p_pos);
bool insert_after(const Token &p_token, List<Token>::Element *p_pos);
bool insert_before(const Token &p_token, List<Token>::Element *p_pos);
List<Token>::Element *replace_curr(const Token &p_token);
List<Token>::Element *_get_next_token_ptr(List<Token>::Element *p_curr_ptr) const;
List<Token>::Element *_get_prev_token_ptr(List<Token>::Element *p_curr_ptr) const;
TokenType _peek_tk_type(int64_t p_count, List<Token>::Element **r_pos = nullptr) const;
bool scope_has_decl(const String &p_scope, const String &p_name) const;
bool _handle_new_keyword_rename(TokenType p_tk_type, const String &p_name, bool p_detected_3x, HashMap<TokenType, String> &p_func_renames);
bool _has_any_preprocessor_directives();
bool _is_code_deprecated();
bool _parse_uniform();
static bool _tok_is_start_of_decl(const Token &p_tk);
bool _skip_uniform();
bool _parse_uniforms();
bool _skip_array_size();
bool _skip_struct();
bool _check_deprecated_type(TokenE *p_type_tok);
bool _add_warning_comment_before(const String &p_comment, List<Token>::Element *p_pos);
bool _add_comment_at_eol(const String &p_comment, List<Token>::Element *p_pos);
bool _process_func_decl_statement(TokenE *p_start_tok, TokenE *p_type_tok, bool p_second_pass = false);
bool _process_decl_statement(TokenE *p_start_tok, TokenE *p_type_tok, const String &p_scope = "<global>", bool p_func_args = false);
bool _parse_decls(bool p_first_pass);
bool _insert_uniform_declaration(const String &p_name);
List<Token>::Element *_remove_from_curr_to(List<Token>::Element *p_end);
List<Token>::Element *_get_end_of_closure();
static HashSet<String> _construct_new_builtin_funcs();
enum {
NEW_IDENT = -1
};
};
#endif // DISABLE_DEPRECATED
#endif // SHADER_CONVERTER_H

93
servers/rendering/shader_language.cpp
View File

@ -230,6 +230,13 @@ const char *ShaderLanguage::token_names[TK_MAX] = {
"CURSOR",
"ERROR",
"EOF",
"TAB",
"CR",
"SPACE",
"NEWLINE",
"BLOCK_COMMENT",
"LINE_COMMENT",
"PREPROC_DIRECTIVE",
};
String ShaderLanguage::get_token_text(Token p_token) {
@ -250,6 +257,8 @@ ShaderLanguage::Token ShaderLanguage::_make_token(TokenType p_type, const String
tk.type = p_type;
tk.text = p_text;
tk.line = tk_line;
tk.pos = tk_start_pos;
tk.length = char_idx - tk_start_pos;
if (tk.type == TK_ERROR) {
_set_error(p_text);
}
@ -402,8 +411,11 @@ const ShaderLanguage::KeyWord ShaderLanguage::keyword_list[] = {
ShaderLanguage::Token ShaderLanguage::_get_token() {
#define GETCHAR(m_idx) (((char_idx + m_idx) < code.length()) ? code[char_idx + m_idx] : char32_t(0))
#define IF_DBG_MK_TK(m_type) \
if (unlikely(debug_parse)) \
return _make_token(m_type);
while (true) {
tk_start_pos = char_idx;
char_idx++;
switch (GETCHAR(-1)) {
case 0:
@ -411,11 +423,17 @@ ShaderLanguage::Token ShaderLanguage::_get_token() {
case 0xFFFF:
return _make_token(TK_CURSOR); //for completion
case '\t':
IF_DBG_MK_TK(TK_TAB);
continue;
case '\r':
IF_DBG_MK_TK(TK_CR);
continue;
case ' ':
IF_DBG_MK_TK(TK_SPACE);
continue;
case '\n':
tk_line++;
IF_DBG_MK_TK(TK_NEWLINE);
continue;
case '/': {
switch (GETCHAR(0)) {
@ -424,10 +442,12 @@ ShaderLanguage::Token ShaderLanguage::_get_token() {
char_idx++;
while (true) {
if (GETCHAR(0) == 0) {
IF_DBG_MK_TK(TK_BLOCK_COMMENT);
return _make_token(TK_EOF);
}
if (GETCHAR(0) == '*' && GETCHAR(1) == '/') {
char_idx += 2;
IF_DBG_MK_TK(TK_BLOCK_COMMENT);
break;
} else if (GETCHAR(0) == '\n') {
tk_line++;
@ -441,11 +461,13 @@ ShaderLanguage::Token ShaderLanguage::_get_token() {
while (true) {
if (GETCHAR(0) == '\n') {
IF_DBG_MK_TK(TK_LINE_COMMENT);
tk_line++;
char_idx++;
break;
}
if (GETCHAR(0) == 0) {
IF_DBG_MK_TK(TK_LINE_COMMENT);
return _make_token(TK_EOF);
}
char_idx++;
@ -702,6 +724,23 @@ ShaderLanguage::Token ShaderLanguage::_get_token() {
return _make_token(TK_ERROR, "Invalid include enter/exit hint token (@@> and @@<)");
}
} break;
case '#': {
if (!debug_parse) { // We shouldn't get here if the preprocessor is enabled and doing a non-debug parse.
return _make_token(TK_ERROR, "Unexpected pre-processor directive (Is the pre-processor enabled?)");
}
while (true) {
char32_t c = GETCHAR(0);
if (c == '\\' && GETCHAR(1) == '\n') {
char_idx += 2;
tk_line++;
continue;
}
if (c == '\n' || c == 0) {
return _make_token(TK_PREPROC_DIRECTIVE);
}
char_idx++;
}
} break;
default: {
char_idx--; //go back one, since we have no idea what this is
@ -767,7 +806,7 @@ ShaderLanguage::Token ShaderLanguage::_get_token() {
lut_case = CASE_EXPONENT;
} else if (symbol == 'f' && !hexa_found) {
if (!period_found && !exponent_found) {
error = true;
error = !debug_parse;
}
float_suffix_found = true;
end_suffix_found = true;
@ -886,6 +925,8 @@ ShaderLanguage::Token ShaderLanguage::_get_token() {
tk.constant = str.to_float();
}
tk.line = tk_line;
tk.pos = tk_start_pos;
tk.length = char_idx - tk_start_pos;
return tk;
}
@ -934,6 +975,7 @@ ShaderLanguage::Token ShaderLanguage::_get_token() {
return Token();
#undef GETCHAR
#undef IF_DBG_MK_TK
}
bool ShaderLanguage::_lookup_next(Token &r_tk) {
@ -956,22 +998,46 @@ ShaderLanguage::Token ShaderLanguage::_peek() {
return tk;
}
String ShaderLanguage::token_debug(const String &p_code) {
String ShaderLanguage::token_debug(const String &p_code, bool p_debug_parse) {
clear();
code = p_code;
debug_parse = p_debug_parse;
String output;
Token tk = _get_token();
while (tk.type != TK_EOF && tk.type != TK_ERROR) {
output += itos(tk_line) + ": " + get_token_text(tk) + "\n";
String literal_text = p_code.substr(tk.pos, tk.length);
output += itos(tk_line) + " (" + itos(tk.pos) + ":" + itos(tk.pos + tk.length) + "): " + get_token_text(tk) + " [" + literal_text;
String suffix = "]\n";
// add error string if invalid float constant
if (debug_parse && tk.type == TK_FLOAT_CONSTANT && literal_text.ends_with("f") && !literal_text.contains(".") && !literal_text.contains("e")) {
output += " (Invalid float constant)]\n";
} else {
output += "]\n";
}
tk = _get_token();
}
debug_parse = false;
return output;
}
void ShaderLanguage::token_debug_stream(const String &p_code, List<Token> &r_output, bool p_debug_parse) {
clear();
code = p_code;
debug_parse = p_debug_parse;
Token tk = _get_token();
while (tk.type != TK_EOF && tk.type != TK_ERROR) {
r_output.push_back(tk);
tk = _get_token();
}
debug_parse = false;
}
bool ShaderLanguage::is_token_variable_datatype(TokenType p_type) {
return (
p_type == TK_TYPE_VOID ||
@ -1064,6 +1130,12 @@ bool ShaderLanguage::is_token_arg_qual(TokenType p_type) {
p_type == TK_ARG_INOUT);
}
bool ShaderLanguage::is_token_uniform_qual(TokenType p_type) {
return (
p_type == TK_INSTANCE ||
p_type == TK_GLOBAL);
}
ShaderLanguage::DataPrecision ShaderLanguage::get_token_precision(TokenType p_type) {
if (p_type == TK_PRECISION_LOW) {
return PRECISION_LOWP;
@ -4118,6 +4190,17 @@ bool ShaderLanguage::is_token_hint(TokenType p_type) {
return int(p_type) > int(TK_RENDER_MODE) && int(p_type) < int(TK_SHADER_TYPE);
}
bool ShaderLanguage::is_token_keyword(TokenType p_type) {
int idx = 0;
while (keyword_list[idx].text) {
if (keyword_list[idx].token == p_type) {
return true;
}
idx++;
}
return false;
}
bool ShaderLanguage::convert_constant(ConstantNode *p_constant, DataType p_to_type, Scalar *p_value) {
if (p_constant->datatype == p_to_type) {
if (p_value) {

22
servers/rendering/shader_language.h
View File

@ -194,7 +194,15 @@ public:
TK_CURSOR,
TK_ERROR,
TK_EOF,
TK_MAX
TK_TAB, // for debug purposes
TK_CR,
TK_SPACE,
TK_NEWLINE,
TK_BLOCK_COMMENT,
TK_LINE_COMMENT,
TK_PREPROC_DIRECTIVE,
TK_MAX,
TK_REG_MAX = TK_TAB,
};
/* COMPILER */
@ -788,6 +796,8 @@ public:
StringName text;
double constant;
uint16_t line;
uint16_t length;
int32_t pos;
bool is_integer_constant() const {
return type == TK_INT_CONSTANT || type == TK_UINT_CONSTANT;
}
@ -803,6 +813,7 @@ public:
static DataInterpolation get_token_interpolation(TokenType p_type);
static bool is_token_precision(TokenType p_type);
static bool is_token_arg_qual(TokenType p_type);
static bool is_token_uniform_qual(TokenType p_type);
static DataPrecision get_token_precision(TokenType p_type);
static String get_precision_name(DataPrecision p_type);
static String get_interpolation_name(DataInterpolation p_interpolation);
@ -814,6 +825,7 @@ public:
static bool is_token_operator(TokenType p_type);
static bool is_token_operator_assign(TokenType p_type);
static bool is_token_hint(TokenType p_type);
static bool is_token_keyword(TokenType p_type);
static bool convert_constant(ConstantNode *p_constant, DataType p_to_type, Scalar *p_value = nullptr);
static DataType get_scalar_type(DataType p_type);
@ -1006,6 +1018,7 @@ private:
String code;
int char_idx = 0;
int tk_line = 0;
int tk_start_pos = 0;
StringName shader_type_identifier;
StringName current_function;
@ -1140,6 +1153,8 @@ private:
StringName completion_struct;
int completion_argument = 0;
bool debug_parse = false;
#ifdef DEBUG_ENABLED
uint32_t keyword_completion_context;
#endif // DEBUG_ENABLED
@ -1222,7 +1237,10 @@ public:
ShaderNode *get_shader();
String token_debug(const String &p_code);
String token_debug(const String &p_code, bool p_debug_parse = false);
void token_debug_stream(const String &p_code, List<Token> &r_output, bool p_debug_parse);
ShaderLanguage::Operator get_op(const TokenType &p_token) const;
ShaderLanguage();
~ShaderLanguage();

799
tests/servers/rendering/test_shader_converter.h Normal file
View File

@ -0,0 +1,799 @@
/**************************************************************************/
/* test_shader_converter.h */
/**************************************************************************/
/* This file is part of: */
/* GODOT ENGINE */
/* https://godotengine.org */
/**************************************************************************/
/* Copyright (c) 2014-present Godot Engine contributors (see AUTHORS.md). */
/* Copyright (c) 2007-2014 Juan Linietsky, Ariel Manzur. */
/* */
/* Permission is hereby granted, free of charge, to any person obtaining */
/* a copy of this software and associated documentation files (the */
/* "Software"), to deal in the Software without restriction, including */
/* without limitation the rights to use, copy, modify, merge, publish, */
/* distribute, sublicense, and/or sell copies of the Software, and to */
/* permit persons to whom the Software is furnished to do so, subject to */
/* the following conditions: */
/* */
/* The above copyright notice and this permission notice shall be */
/* included in all copies or substantial portions of the Software. */
/* */
/* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, */
/* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF */
/* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. */
/* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY */
/* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, */
/* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE */
/* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */
/**************************************************************************/
#ifndef TEST_SHADER_CONVERTER_H
#define TEST_SHADER_CONVERTER_H
#ifndef DISABLE_DEPRECATED
#include "servers/rendering/shader_converter.h"
#include "servers/rendering/shader_language.h"
#include "servers/rendering/shader_types.h"
#include "tests/test_macros.h"
#include <cctype>
namespace TestShaderConverter {
void erase_all_empty(Vector<String> &p_vec) {
int idx = p_vec.find(" ");
while (idx >= 0) {
p_vec.remove_at(idx);
idx = p_vec.find(" ");
}
}
bool is_variable_char(unsigned char c) {
return std::isalnum(c) || c == '_';
}
bool is_operator_char(unsigned char c) {
return (c == '*') || (c == '+') || (c == '-') || (c == '/') || ((c >= '<') && (c <= '>'));
}
// Remove unnecessary spaces from a line.
String remove_spaces(String &p_str) {
String res;
// Result is guaranteed to not be longer than the input.
res.resize(p_str.size());
int wp = 0;
char32_t last = 0;
bool has_removed = false;
for (int n = 0; n < p_str.size(); n++) {
// These test cases only use ASCII.
unsigned char c = static_cast<unsigned char>(p_str[n]);
if (std::isblank(c)) {
has_removed = true;
} else {
if (has_removed) {
// Insert a space to avoid joining things that could potentially form a new token.
// E.g. "float x" or "- -".
if ((is_variable_char(c) && is_variable_char(last)) ||
(is_operator_char(c) && is_operator_char(last))) {
res[wp++] = ' ';
}
has_removed = false;
}
res[wp++] = c;
last = c;
}
}
res.resize(wp);
return res;
}
// The pre-processor changes indentation and inserts spaces when inserting macros.
// Re-format the code, without changing its meaning, to make it easier to compare.
String compact_spaces(String &p_str) {
Vector<String> lines = p_str.split("\n", false);
erase_all_empty(lines);
for (String &line : lines) {
line = remove_spaces(line);
}
return String("\n").join(lines);
}
void get_keyword_set(HashSet<String> &p_keywords) {
List<String> keywords;
ShaderLanguage::get_keyword_list(&keywords);
for (const String &keyword : keywords) {
p_keywords.insert(keyword);
}
}
#define CHECK_SHADER_EQ(a, b) CHECK_EQ(compact_spaces(a), compact_spaces(b))
#define CHECK_SHADER_NE(a, b) CHECK_NE(compact_spaces(a), compact_spaces(b))
void get_compile_info(ShaderLanguage::ShaderCompileInfo &info, RenderingServer::ShaderMode p_mode) {
info.functions = ShaderTypes::get_singleton()->get_functions(p_mode);
info.render_modes = ShaderTypes::get_singleton()->get_modes(p_mode);
info.shader_types = ShaderTypes::get_singleton()->get_types();
// Only used by editor for completion, so it's not important for these tests.
info.global_shader_uniform_type_func = [](const StringName &p_name) -> ShaderLanguage::DataType {
return ShaderLanguage::TYPE_SAMPLER2D;
};
}
RenderingServer::ShaderMode get_shader_mode(const String &p_mode_string) {
if (p_mode_string == "canvas_item") {
return RS::SHADER_CANVAS_ITEM;
} else if (p_mode_string == "particles") {
return RS::SHADER_PARTICLES;
} else if (p_mode_string == "spatial") {
return RS::SHADER_SPATIAL;
} else if (p_mode_string == "sky") {
return RS::SHADER_SKY;
} else if (p_mode_string == "fog") {
return RS::SHADER_FOG;
} else {
return RS::SHADER_MAX;
}
}
String get_shader_mode_name(const RenderingServer::ShaderMode &p_mode_string) {
switch (p_mode_string) {
case RS::SHADER_CANVAS_ITEM:
return "canvas_item";
case RS::SHADER_PARTICLES:
return "particles";
case RS::SHADER_SPATIAL:
return "spatial";
case RS::SHADER_SKY:
return "sky";
case RS::SHADER_FOG:
return "fog";
default:
return "unknown";
}
}
using SL = ShaderLanguage;
using SDC = ShaderDeprecatedConverter;
#define TEST_CONVERSION(m_old_code, m_expected, m_is_deprecated) \
{ \
ShaderDeprecatedConverter _i_converter; \
CHECK_EQ(_i_converter.is_code_deprecated(m_old_code), m_is_deprecated); \
CHECK(_i_converter.convert_code(m_old_code)); \
String _i_new_code = _i_converter.emit_code(); \
CHECK_EQ(_i_new_code, m_expected); \
}
TEST_CASE("[ShaderDeprecatedConverter] Simple conversion with arrays") {
String code = "shader_type particles; void vertex() { float xy[2] = {1.0,1.1}; }";
String expected = "shader_type particles; void process() { float xy[2] = {1.0,1.1}; }";
TEST_CONVERSION(code, expected, true);
}
TEST_CASE("[ShaderDeprecatedConverter] Test warning comments") {
// Test that warning comments are added when fail_on_unported is false and warning_comments is true
String code = "shader_type spatial;\nrender_mode specular_phong;";
String expected = "shader_type spatial;\n/* !convert WARNING: Deprecated render mode 'specular_phong' is not supported by this version of Godot. */\nrender_mode specular_phong;";
ShaderDeprecatedConverter converter;
CHECK(converter.is_code_deprecated(code));
converter.set_warning_comments(true);
converter.set_fail_on_unported(false);
CHECK(converter.convert_code(code));
String new_code = converter.emit_code();
CHECK_EQ(new_code, expected);
}
TEST_CASE("[ShaderDeprecatedConverter] Simple conversion with arrays") {
String code = "shader_type particles; struct foo{float bar;} void vertex() { float xy[2] = {1.0,1.1}; }";
String expected = "shader_type particles; struct foo{float bar;} void process() { float xy[2] = {1.0,1.1}; }";
TEST_CONVERSION(code, expected, true);
}
TEST_CASE("[ShaderDeprecatedConverter] new-style array declaration") {
String code = "shader_type spatial; void vertex() { float[2] xy = {1.0,1.1}; }";
// code should be the same
TEST_CONVERSION(code, code, false);
}
TEST_CASE("[ShaderDeprecatedConverter] Simple conversion") {
String code = "shader_type particles; void vertex() { float x = 1.0; }";
String expected = "shader_type particles; void process() { float x = 1.0; }";
TEST_CONVERSION(code, expected, true);
}
TEST_CASE("[ShaderDeprecatedConverter] Replace non-conformant float literals") {
String code = "shader_type spatial; const float x = 1f;";
String expected = "shader_type spatial; const float x = 1.0f;";
TEST_CONVERSION(code, expected, true);
}
TEST_CASE("[ShaderDeprecatedConverter] particles::vertex() -> particles::process()") {
SUBCASE("basic") {
String code = "shader_type particles; void vertex() { float x = 1.0; }";
String expected = "shader_type particles; void process() { float x = 1.0; }";
TEST_CONVERSION(code, expected, true);
}
SUBCASE("with another function named `process` without correct signature") {
String code = "shader_type particles; void vertex() {} float process() { return 1.0; }";
String expected = "shader_type particles; void process() {} float process_() { return 1.0; }";
TEST_CONVERSION(code, expected, true);
}
SUBCASE("with another function named `process` with correct signature") {
String code = "shader_type particles; void vertex() {} void process() {}";
// Should be unchanged.
TEST_CONVERSION(code, code, false);
}
SUBCASE("with another function named `process` that is called") {
String code = "shader_type particles; float process() { return 1.0; } void vertex() { float foo = process(); }";
String expected = "shader_type particles; float process_() { return 1.0; } void process() { float foo = process_(); }";
TEST_CONVERSION(code, expected, true);
}
SUBCASE("with another function named `process` which calls `vertex`") {
String code = "shader_type particles; float process() {foo(); return 1.0;} void vertex() {} void foo() { vertex(); }";
String expected = "shader_type particles; float process_() {foo(); return 1.0;} void process() {} void foo() { process(); }";
TEST_CONVERSION(code, expected, true);
}
SUBCASE("No function named `vertex`") {
String code = "shader_type particles; void process() {}";
// Should be unchanged.
TEST_CONVERSION(code, code, false);
}
}
TEST_CASE("[ShaderDeprecatedConverter] CLEARCOAT_GLOSS -> CLEARCOAT_ROUGHNESS") {
SUBCASE("Left-hand simple assignment") {
String code("shader_type spatial; void fragment() {\n"
"CLEARCOAT_GLOSS = 1.0;\n"
"}\n");
String expected("shader_type spatial; void fragment() {\n"
"CLEARCOAT_ROUGHNESS = (1.0 - (1.0));\n"
"}\n");
TEST_CONVERSION(code, expected, true);
}
SUBCASE("Left-hand *= assignment") {
String code("shader_type spatial; void fragment() {\n"
"CLEARCOAT_GLOSS *= 0.5;\n"
"}\n");
String expected("shader_type spatial; void fragment() {\n"
"CLEARCOAT_ROUGHNESS = (1.0 - ((1.0 - CLEARCOAT_ROUGHNESS) * 0.5));\n"
"}\n");
TEST_CONVERSION(code, expected, true);
}
SUBCASE("Right-hand usage") {
String code("shader_type spatial; void fragment() {\n"
"float foo = CLEARCOAT_GLOSS;\n"
"}\n");
String expected("shader_type spatial; void fragment() {\n"
"float foo = (1.0 - CLEARCOAT_ROUGHNESS);\n"
"}\n");
TEST_CONVERSION(code, expected, true);
}
SUBCASE("both usages") {
String code("shader_type spatial; void fragment() {\n"
"float foo = (CLEARCOAT_GLOSS *= 0.5);\n"
"}\n");
String expected("shader_type spatial; void fragment() {\n"
"float foo = ((1.0 - (CLEARCOAT_ROUGHNESS = (1.0 - ((1.0 - CLEARCOAT_ROUGHNESS) * 0.5)))));\n"
"}\n");
TEST_CONVERSION(code, expected, true);
}
}
TEST_CASE("[ShaderDeprecatedConverter] Wrap INDEX in int()") {
SUBCASE("basic") {
String code("shader_type particles; void vertex() {\n"
"float foo = INDEX/2;\n"
"}\n");
String expected("shader_type particles; void process() {\n"
"float foo = int(INDEX)/2;\n"
"}\n");
TEST_CONVERSION(code, expected, true);
}
SUBCASE("Without clobbering existing casts") {
String code("shader_type particles; void vertex() {\n"
"float foo = int(INDEX/2) * int(INDEX) * 2 * float(INDEX);\n"
"}\n");
String expected("shader_type particles; void process() {\n"
"float foo = int(int(INDEX)/2) * int(INDEX) * 2 * float(INDEX);\n"
"}\n");
TEST_CONVERSION(code, expected, true);
}
}
TEST_CASE("[ShaderDeprecatedConverter] All hint renames") {
String code_template = "shader_type spatial; uniform sampler2D foo : %s;";
// get all the hint renames
List<String> hints;
ShaderDeprecatedConverter::_get_hint_renames_list(&hints);
SUBCASE("No renamed hints present in current keyword list") {
HashSet<String> keywords_set;
get_keyword_set(keywords_set);
for (const String &hint : hints) {
CHECK_FALSE(keywords_set.has(hint));
}
}
SUBCASE("All renamed hints are replaced") {
for (const String &hint : hints) {
ShaderDeprecatedConverter::TokenType type = ShaderDeprecatedConverter::get_hint_replacement(hint);
String rename = ShaderDeprecatedConverter::get_tokentype_text(type);
String code = vformat(code_template, hint);
String expected = vformat(code_template, rename);
TEST_CONVERSION(code, expected, true);
}
}
}
TEST_CASE("[ShaderDeprecatedConverter] Built-in renames") {
// Get all the built-in renames.
List<String> builtins;
ShaderDeprecatedConverter::_get_builtin_renames_list(&builtins);
// remove built-ins that have special handling, we test those above
for (List<String>::Element *E = builtins.front(); E; E = E->next()) {
if (ShaderDeprecatedConverter::_rename_has_special_handling(E->get())) {
List<String>::Element *prev = E->prev();
builtins.erase(E);
E = prev;
}
}
Vector<RS::ShaderMode> modes = { RS::SHADER_SPATIAL, RS::SHADER_CANVAS_ITEM, RS::SHADER_PARTICLES };
HashMap<RS::ShaderMode, HashMap<String, Vector<String>>> rename_func_map;
for (RS::ShaderMode mode : modes) {
rename_func_map[mode] = HashMap<String, Vector<String>>();
for (const String &builtin : builtins) {
rename_func_map[mode][builtin] = ShaderDeprecatedConverter::_get_funcs_builtin_rename(mode, builtin);
}
}
SUBCASE("All renamed built-ins are not currently built-in") {
for (RS::ShaderMode mode : modes) {
ShaderLanguage::ShaderCompileInfo info;
get_compile_info(info, mode);
for (const String &builtin : builtins) {
// Now get the funcs applicable for this mode and built-in.
for (const String &func : rename_func_map[mode][builtin]) {
// The built-in should not be present in the built-ins list.
auto &finfo = info.functions[func];
if (finfo.built_ins.has(builtin)) {
WARN_PRINT(vformat("Renamed 3.x Built-in %s is present in function %s", builtin, func));
}
CHECK_FALSE(finfo.built_ins.has(builtin));
}
}
}
}
SUBCASE("All renamed built-ins are replaced") {
String code_template = "shader_type %s; void %s() { %s; }";
for (RS::ShaderMode mode : modes) {
for (const String &builtin : builtins) {
// Now get the funcs applicable for this mode and built-in.
String rename = ShaderDeprecatedConverter::get_builtin_rename(builtin);
for (const String &func : rename_func_map[mode][builtin]) {
String code = vformat(code_template, get_shader_mode_name(mode), func, builtin);
String expected = vformat(code_template, get_shader_mode_name(mode), func, rename);
TEST_CONVERSION(code, expected, true);
}
}
}
}
SUBCASE("No renaming built-ins in non-candidate functions") {
String code_template = "shader_type %s; void %s() { float %s = 1.0; %s += 1.0; }";
for (RS::ShaderMode mode : modes) {
ShaderLanguage::ShaderCompileInfo info;
get_compile_info(info, mode);
for (const String &builtin : builtins) {
Vector<String> non_funcs;
for (KeyValue<StringName, ShaderLanguage::FunctionInfo> &func : info.functions) {
if (func.key == "global") {
continue;
}
if (!rename_func_map[mode][builtin].has(func.key)) {
non_funcs.push_back(func.key);
}
}
String rename = ShaderDeprecatedConverter::get_builtin_rename(builtin);
for (const String &func : non_funcs) {
String code = vformat(code_template, get_shader_mode_name(mode), func, builtin, builtin);
// The code should not change.
TEST_CONVERSION(code, code, false);
}
}
}
}
SUBCASE("No renaming built-ins in candidate functions with built-in declared") {
String code_template = "shader_type %s; void %s() { float %s = 1.0; %s += 1.0; }";
for (RS::ShaderMode mode : modes) {
for (const String &builtin : builtins) {
for (const String &func : rename_func_map[mode][builtin]) {
String code = vformat(code_template, get_shader_mode_name(mode), func, builtin, builtin);
// The code should not change.
TEST_CONVERSION(code, code, false);
}
}
}
}
}
// TODO: Remove this when the MODULATE built-in PR lands.
// If this fails, remove the MODULATE entry from ShaderDeprecatedConverter::removed_builtins, then remove this test and the following test.
TEST_CASE("[ShaderDeprecatedConverter] MODULATE is not a built-in") {
ShaderLanguage::ShaderCompileInfo info;
get_compile_info(info, RS::ShaderMode::SHADER_CANVAS_ITEM);
SUBCASE("MODULATE is not a built-in") {
for (const String &func : Vector<String>{ "vertex", "fragment", "light" }) {
auto &finfo = info.functions[func];
CHECK_FALSE(finfo.built_ins.has("MODULATE"));
}
}
}
// Don't remove this one if the above doesn't fail too.
TEST_CASE("[ShaderDeprecatedConverter] MODULATE handling") {
ShaderLanguage::ShaderCompileInfo info;
get_compile_info(info, RS::ShaderMode::SHADER_CANVAS_ITEM);
SUBCASE("Fails to compile") {
for (const String &func : Vector<String>{ "vertex", "fragment", "light" }) {
String code = vformat("shader_type canvas_item; void %s() { MODULATE; }", func);
ShaderLanguage sl;
CHECK_NE(sl.compile(code, info), Error::OK);
}
}
SUBCASE("Fails to convert on fail_on_unported=true") {
for (const String &func : Vector<String>{ "vertex", "fragment", "light" }) {
String code = vformat("shader_type canvas_item; void %s() { MODULATE; }", func);
ShaderDeprecatedConverter converter;
CHECK(converter.is_code_deprecated(code));
converter.set_fail_on_unported(true);
CHECK_FALSE(converter.convert_code(code));
}
}
SUBCASE("Conversion succeeds on fail_on_unported=false") {
for (const String &func : Vector<String>{ "vertex", "fragment", "light" }) {
String code = vformat("shader_type canvas_item; void %s() { MODULATE; }", func);
ShaderDeprecatedConverter converter;
CHECK(converter.is_code_deprecated(code));
converter.set_fail_on_unported(false);
CHECK(converter.convert_code(code));
String new_code = converter.emit_code();
CHECK(new_code.find("/*") != -1);
}
}
}
TEST_CASE("[ShaderDeprecatedConverter] Uniform declarations for removed builtins") {
// Test uniform declaration inserts for removed builtins for all shader types.
String code_template = "shader_type %s;%s void %s() { %s; }";
String uniform_template = "\nuniform %s %s : %s;\n";
// Get all the removed built-ins.
List<String> builtins;
ShaderDeprecatedConverter::_get_builtin_removals_list(&builtins);
Vector<RS::ShaderMode> modes = { RS::SHADER_SPATIAL, RS::SHADER_CANVAS_ITEM, RS::SHADER_PARTICLES };
HashMap<RS::ShaderMode, ShaderLanguage::ShaderCompileInfo> compiler_infos;
SUBCASE("Removed built-ins are not currently built-in") {
for (RS::ShaderMode mode : modes) {
ShaderLanguage::ShaderCompileInfo info;
get_compile_info(info, mode);
for (const String &builtin : builtins) {
Vector<String> funcs = ShaderDeprecatedConverter::_get_funcs_builtin_removal(mode, builtin);
for (const String &func : funcs) {
const ShaderLanguage::FunctionInfo &finfo = info.functions[func];
if (finfo.built_ins.has(builtin)) {
WARN_PRINT(vformat("Removed 3.x Built-in %s is present in function %s", builtin, func));
}
CHECK_FALSE(finfo.built_ins.has(builtin));
}
}
}
}
SUBCASE("All removed built-ins have uniform declarations") {
for (RS::ShaderMode mode : modes) {
for (const String &builtin : builtins) {
// now get the funcs applicable for this mode and builtins
ShaderLanguage::TokenType type = ShaderDeprecatedConverter::get_removed_builtin_uniform_type(builtin);
if (type == ShaderDeprecatedConverter::TokenType::TK_ERROR) {
continue;
}
Vector<ShaderLanguage::TokenType> hints = ShaderDeprecatedConverter::get_removed_builtin_hints(builtin);
Vector<String> funcs = ShaderDeprecatedConverter::_get_funcs_builtin_removal(mode, builtin);
String hint_string = "";
for (int i = 0; i < hints.size(); i++) {
hint_string += ShaderDeprecatedConverter::get_tokentype_text(hints[i]);
if (i < hints.size() - 1) {
hint_string += ", ";
}
}
String uniform_decl = vformat(uniform_template, ShaderDeprecatedConverter::get_tokentype_text(type), builtin, hint_string);
for (const String &func : funcs) {
String code = vformat(code_template, get_shader_mode_name(mode), "", func, builtin);
if (type == ShaderDeprecatedConverter::TokenType::TK_ERROR) { // Unported builtins with no uniform declaration
ShaderDeprecatedConverter converter;
CHECK(converter.is_code_deprecated(code));
CHECK_FALSE(converter.convert_code(code));
converter.set_fail_on_unported(false);
CHECK(converter.convert_code(code));
continue;
}
String expected = vformat(code_template, get_shader_mode_name(mode), uniform_decl, func, builtin);
TEST_CONVERSION(code, expected, true);
}
}
}
}
}
// Reserved keywords (i.e. non-built-in function keywords that have a discrete token type)
TEST_CASE("[ShaderDeprecatedConverter] Replacement of reserved keywords used as identifiers") {
Vector<String> keywords;
for (int i = 0; i < ShaderLanguage::TK_MAX; i++) {
if (ShaderDeprecatedConverter::tokentype_is_new_reserved_keyword(static_cast<ShaderLanguage::TokenType>(i))) {
keywords.push_back(ShaderDeprecatedConverter::get_tokentype_text(static_cast<ShaderLanguage::TokenType>(i)));
}
}
Vector<String> hint_keywords;
for (int i = 0; i < SL::TK_MAX; i++) {
if (SDC::tokentype_is_new_hint(static_cast<SL::TokenType>(i))) {
hint_keywords.push_back(SDC::get_tokentype_text(static_cast<SL::TokenType>(i)));
}
}
Vector<String> uniform_quals;
for (int i = 0; i < SL::TK_MAX; i++) {
if (SL::is_token_uniform_qual(static_cast<SL::TokenType>(i))) {
uniform_quals.push_back(SDC::get_tokentype_text(static_cast<SL::TokenType>(i)));
}
}
Vector<String> shader_types_to_test = { "spatial", "canvas_item", "particles" };
static const char *decl_test_template[]{
"shader_type %s;\nvoid %s() {}\n",
"shader_type %s;\nvoid test_func() {float %s;}\n",
"shader_type %s;\nuniform sampler2D %s;\n",
"shader_type %s;\nconst float %s = 1.0;\n",
"shader_type %s;\nvarying float %s;\n",
nullptr
};
// NOTE: if this fails, the current behavior of the converter to replace these has to be changed.
SUBCASE("Code with reserved keywords used as identifiers fail to compile") {
ShaderLanguage::ShaderCompileInfo info;
get_compile_info(info, RS::SHADER_SPATIAL);
for (const String &shader_type : shader_types_to_test) {
for (const String &keyword : keywords) {
for (int i = 0; decl_test_template[i] != nullptr; i++) {
String code = vformat(decl_test_template[i], shader_type, keyword);
ShaderLanguage sl;
CHECK_NE(sl.compile(code, info), Error::OK);
}
}
}
}
SUBCASE("Code with reserved keywords used as identifiers is converted successfully") {
for (const String &shader_type : shader_types_to_test) {
ShaderLanguage::ShaderCompileInfo info;
get_compile_info(info, get_shader_mode(shader_type));
for (const String &keyword : keywords) {
for (int i = 0; decl_test_template[i] != nullptr; i++) {
if (shader_type == "particles" && String(decl_test_template[i]).contains("varying")) {
continue;
}
String code = vformat(decl_test_template[i], shader_type, keyword);
String expected = vformat(decl_test_template[i], shader_type, keyword + "_");
TEST_CONVERSION(code, expected, true);
}
}
}
}
static const char *new_hint_test = "shader_type spatial;\nuniform sampler2D foo : %s; const float %s = 1.0;\n";
SUBCASE("New hints used as hints are not replaced") {
for (const String &hint : hint_keywords) {
String code = vformat(new_hint_test, hint, "bar");
// Code should not change.
TEST_CONVERSION(code, code, false);
}
}
SUBCASE("Mixed new hints used as hints and new hints used as identifiers") {
for (const String &hint : hint_keywords) {
String code = vformat(new_hint_test, hint, hint);
// Should not change.
ShaderDeprecatedConverter converter;
CHECK_FALSE(converter.is_code_deprecated(code)); // Should be detected as not deprecated.
converter.set_warning_comments(false);
CHECK(converter.convert_code(code));
String new_code = converter.emit_code();
// Code should not change
CHECK_EQ(new_code, code);
// Check for warning comment
converter.set_warning_comments(true);
new_code = converter.emit_code();
CHECK(new_code.contains("/* !convert WARNING:"));
}
}
static const char *non_id_keyword_test = "shader_type spatial;\n%s uniform sampler2D foo; const float %s = 1.0;\n";
SUBCASE("New keywords not used as identifiers are not replaced") {
for (const String &qual : uniform_quals) {
// e.g. "shader_type spatial;\nglobal uniform sampler2D foo; const float bar = 1.0;\n"
String code = vformat(non_id_keyword_test, qual, "bar");
// Code should not change.
TEST_CONVERSION(code, code, false);
}
}
SUBCASE("Mixed idiomatic new reserved words and new reserved words used as identifiers") {
for (const String &qual : uniform_quals) {
// e.g. "shader_type spatial;\nglobal uniform sampler2D foo; const float global = 1.0;\n"
String code = vformat(non_id_keyword_test, qual, qual);
// Should not change.
ShaderDeprecatedConverter converter;
CHECK_FALSE(converter.is_code_deprecated(code)); // Should be detected as not deprecated.
converter.set_warning_comments(false);
CHECK(converter.convert_code(code));
String new_code = converter.emit_code();
// Code should not change
CHECK_EQ(new_code, code);
// Check for warning comment
converter.set_warning_comments(true);
new_code = converter.emit_code();
CHECK(new_code.contains("/* !convert WARNING:"));
}
}
}
TEST_CASE("[ShaderDeprecatedConverter] Convert default 3.x nodetree shader") {
static const char *default_3x_nodtree_shader =
R"(shader_type spatial;
render_mode blend_mix, depth_draw_always, cull_back, diffuse_burley, specular_schlick_ggx;
uniform sampler2D texture_0: hint_albedo;
void node_bsdf_principled(vec4 color, float subsurface, vec4 subsurface_color,
float metallic, float specular, float roughness, float clearcoat,
float clearcoat_roughness, float anisotropy, float transmission,
float IOR, out vec3 albedo, out float sss_strength_out,
out float metallic_out, out float specular_out,
out float roughness_out, out float clearcoat_out,
out float clearcoat_gloss_out, out float anisotropy_out,
out float transmission_out, out float ior) {
metallic = clamp(metallic, 0.0, 1.0);
transmission = clamp(transmission, 0.0, 1.0);
subsurface = subsurface * (1.0 - metallic);
albedo = mix(color.rgb, subsurface_color.rgb, subsurface);
sss_strength_out = subsurface;
metallic_out = metallic;
specular_out = pow((IOR - 1.0)/(IOR + 1.0), 2)/0.08;
roughness_out = roughness;
clearcoat_out = clearcoat * (1.0 - transmission);
clearcoat_gloss_out = 1.0 - clearcoat_roughness;
anisotropy_out = clamp(anisotropy, 0.0, 1.0);
transmission_out = (1.0 - transmission) * (1.0 - metallic);
ior = IOR;
}
void node_tex_image(vec3 co, sampler2D ima, out vec4 color, out float alpha) {
color = texture(ima, co.xy);
alpha = color.a;
}
void vertex () {
}
void fragment () {
// node: 'Image Texture'
// type: 'ShaderNodeTexImage'
// input sockets handling
vec3 node0_in0_vector = vec3(0.0, 0.0, 0.0);
// output sockets definitions
vec4 node0_out0_color;
float node0_out1_alpha;
node0_in0_vector = vec3(UV, 0.0);
node_tex_image(node0_in0_vector, texture_0, node0_out0_color, node0_out1_alpha);
// node: 'Principled BSDF'
// type: 'ShaderNodeBsdfPrincipled'
// input sockets handling
vec4 node1_in0_basecolor = node0_out0_color;
float node1_in1_subsurface = float(0.0);
vec3 node1_in2_subsurfaceradius = vec3(1.0, 0.20000000298023224,
0.10000000149011612);
vec4 node1_in3_subsurfacecolor = vec4(0.800000011920929, 0.800000011920929,
0.800000011920929, 1.0);
float node1_in4_metallic = float(0.0);
float node1_in5_specular = float(0.5);
float node1_in6_speculartint = float(0.0);
float node1_in7_roughness = float(1.0);
float node1_in8_anisotropic = float(0.0);
float node1_in9_anisotropicrotation = float(0.0);
float node1_in10_sheen = float(0.0);
float node1_in11_sheentint = float(0.5);
float node1_in12_clearcoat = float(0.0);
float node1_in13_clearcoatroughness = float(0.029999999329447746);
float node1_in14_ior = float(1.4500000476837158);
float node1_in15_transmission = float(0.0);
float node1_in16_transmissionroughness = float(0.0);
vec4 node1_in17_emission = vec4(0.0, 0.0, 0.0, 1.0);
float node1_in18_emissionstrength = float(1.0);
float node1_in19_alpha = float(1.0);
vec3 node1_in20_normal = NORMAL;
vec3 node1_in21_clearcoatnormal = vec3(0.0, 0.0, 0.0);
vec3 node1_in22_tangent = TANGENT;
// output sockets definitions
vec3 node1_bsdf_out0_albedo;
float node1_bsdf_out1_sss_strength;
float node1_bsdf_out3_specular;
float node1_bsdf_out2_metallic;
float node1_bsdf_out4_roughness;
float node1_bsdf_out5_clearcoat;
float node1_bsdf_out6_clearcoat_gloss;
float node1_bsdf_out7_anisotropy;
float node1_bsdf_out8_transmission;
float node1_bsdf_out9_ior;
node_bsdf_principled(node1_in0_basecolor, node1_in1_subsurface,
node1_in3_subsurfacecolor, node1_in4_metallic, node1_in5_specular,
node1_in7_roughness, node1_in12_clearcoat, node1_in13_clearcoatroughness,
node1_in8_anisotropic, node1_in15_transmission, node1_in14_ior,
node1_bsdf_out0_albedo, node1_bsdf_out1_sss_strength, node1_bsdf_out2_metallic,
node1_bsdf_out3_specular, node1_bsdf_out4_roughness, node1_bsdf_out5_clearcoat,
node1_bsdf_out6_clearcoat_gloss, node1_bsdf_out7_anisotropy,
node1_bsdf_out8_transmission, node1_bsdf_out9_ior);
ALBEDO = node1_bsdf_out0_albedo;
SSS_STRENGTH = node1_bsdf_out1_sss_strength;
SPECULAR = node1_bsdf_out3_specular;
METALLIC = node1_bsdf_out2_metallic;
ROUGHNESS = node1_bsdf_out4_roughness;
CLEARCOAT = node1_bsdf_out5_clearcoat;
CLEARCOAT_GLOSS = node1_bsdf_out6_clearcoat_gloss;
NORMAL = node1_in20_normal;
// uncomment it when you need it
// TRANSMISSION = vec3(1.0, 1.0, 1.0) * node1_bsdf_out8_transmission;
// uncomment it when you are modifing TANGENT
// TANGENT = normalize(cross(cross(node1_in22_tangent, NORMAL), NORMAL));
// BINORMAL = cross(TANGENT, NORMAL);
// uncomment it when you have tangent(UV) set
// ANISOTROPY = node1_bsdf_out7_anisotropy;
}
)";
ShaderLanguage sl;
ShaderLanguage::ShaderCompileInfo info;
get_compile_info(info, RS::SHADER_SPATIAL);
SUBCASE("Default 3.x nodetree shader does not compile") {
CHECK_NE(sl.compile(default_3x_nodtree_shader, info), Error::OK);
}
sl.clear();
SUBCASE("Convert default 3.x nodetree shader") {
ShaderDeprecatedConverter converter;
CHECK(converter.convert_code(default_3x_nodtree_shader));
String new_code = converter.emit_code();
CHECK(new_code.find("/*") == -1);
CHECK_FALSE(converter.get_error_line());
}
SUBCASE("Converted default 3.x nodetree shader compiles") {
ShaderDeprecatedConverter converter;
CHECK(converter.convert_code(default_3x_nodtree_shader));
String new_code = converter.emit_code();
CHECK_EQ(sl.compile(new_code, info), Error::OK);
}
sl.clear();
}
} // namespace TestShaderConverter
#undef TEST_CONVERSION
#endif // DISABLE_DEPRECATED
#endif // TEST_SHADER_CONVERTER_H

144
tests/servers/rendering/test_shader_language.h Normal file
View File

@ -0,0 +1,144 @@
/**************************************************************************/
/* test_shader_language.h */
/**************************************************************************/
/* This file is part of: */
/* GODOT ENGINE */
/* https://godotengine.org */
/**************************************************************************/
/* Copyright (c) 2014-present Godot Engine contributors (see AUTHORS.md). */
/* Copyright (c) 2007-2014 Juan Linietsky, Ariel Manzur. */
/* */
/* Permission is hereby granted, free of charge, to any person obtaining */
/* a copy of this software and associated documentation files (the */
/* "Software"), to deal in the Software without restriction, including */
/* without limitation the rights to use, copy, modify, merge, publish, */
/* distribute, sublicense, and/or sell copies of the Software, and to */
/* permit persons to whom the Software is furnished to do so, subject to */
/* the following conditions: */
/* */
/* The above copyright notice and this permission notice shall be */
/* included in all copies or substantial portions of the Software. */
/* */
/* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, */
/* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF */
/* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. */
/* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY */
/* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, */
/* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE */
/* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */
/**************************************************************************/
#ifndef TEST_SHADER_LANGUAGE_H
#define TEST_SHADER_LANGUAGE_H
#include "servers/rendering/shader_language.h"
#include "servers/rendering/shader_types.h"
#include "tests/test_macros.h"
#include <cctype>
namespace TestShaderLanguage {
void get_compile_info(ShaderLanguage::ShaderCompileInfo &info, RenderingServer::ShaderMode p_mode) {
info.functions = ShaderTypes::get_singleton()->get_functions(p_mode);
info.render_modes = ShaderTypes::get_singleton()->get_modes(p_mode);
info.shader_types = ShaderTypes::get_singleton()->get_types();
// Only used by editor for completion, so it's not important for these tests.
info.global_shader_uniform_type_func = [](const StringName &p_name) -> ShaderLanguage::DataType {
return ShaderLanguage::TYPE_SAMPLER2D;
};
}
RenderingServer::ShaderMode get_shader_mode(const String &p_mode_string) {
if (p_mode_string == "canvas_item") {
return RS::SHADER_CANVAS_ITEM;
} else if (p_mode_string == "particles") {
return RS::SHADER_PARTICLES;
} else if (p_mode_string == "spatial") {
return RS::SHADER_SPATIAL;
} else if (p_mode_string == "sky") {
return RS::SHADER_SKY;
} else if (p_mode_string == "fog") {
return RS::SHADER_FOG;
} else {
return RS::SHADER_MAX;
}
}
TEST_CASE("[ShaderLanguage] Minimal Script") {
ShaderLanguage sl;
ShaderLanguage::ShaderCompileInfo info;
get_compile_info(info, RS::SHADER_SPATIAL);
String code = "shader_type spatial;";
CHECK_EQ(sl.compile(code, info), Error::OK);
}
// No keywords (except for built-in functions) should be valid identifiers.
TEST_CASE("[ShaderLanguage] Ensure no reserved keywords are valid identifiers") {
List<String> keywords;
List<String> builtin_functions;
ShaderLanguage::get_keyword_list(&keywords);
ShaderLanguage::get_builtin_funcs(&builtin_functions);
HashSet<String> builtin_set;
for (const String &keyword : builtin_functions) {
builtin_set.insert(keyword);
}
HashSet<String> non_func_keywords_set;
for (const String &keyword : keywords) {
if (!builtin_set.has(keyword)) {
non_func_keywords_set.insert(keyword);
}
}
static const char *decl_test_template[]{
"shader_type %s;\nvoid %s() {}\n",
"shader_type %s;\nvoid vertex() {float %s;}\n",
"shader_type %s;\nuniform sampler2D %s;\n",
"shader_type %s;\nconst float %s = 1.0;\n",
nullptr
};
static const char *varying_template = "shader_type %s;\nvarying float %s;\n";
Vector<String> non_varying_types = { "particles", "sky", "fog" };
auto shader_types_to_test = ShaderTypes::get_singleton()->get_types();
for (auto shader_type : shader_types_to_test) {
ShaderLanguage::ShaderCompileInfo info;
get_compile_info(info, get_shader_mode(shader_type));
// test templates with non-keyword identifiers
for (int i = 0; decl_test_template[i] != nullptr; i++) {
String code = vformat(decl_test_template[i], shader_type, "foo");
String result;
ShaderLanguage sl;
CHECK_EQ(sl.compile(code, info), Error::OK);
}
if (!non_varying_types.has(shader_type)) {
String code = vformat(varying_template, shader_type, "foo");
String result;
ShaderLanguage sl;
CHECK_EQ(sl.compile(code, info), Error::OK);
}
for (const String &keyword : non_func_keywords_set) {
for (int i = 0; decl_test_template[i] != nullptr; i++) {
String code = vformat(decl_test_template[i], shader_type, keyword);
String result;
ShaderLanguage sl;
CHECK_NE(sl.compile(code, info), Error::OK);
}
if (!non_varying_types.has(shader_type)) {
String code = vformat(varying_template, shader_type, keyword);
String result;
ShaderLanguage sl;
CHECK_NE(sl.compile(code, info), Error::OK);
}
}
}
}
} // namespace TestShaderLanguage
#endif // TEST_SHADER_LANGUAGE_H

4
tests/test_main.cpp
View File

@ -129,6 +129,8 @@
#include "tests/scene/test_viewport.h"
#include "tests/scene/test_visual_shader.h"
#include "tests/scene/test_window.h"
#include "tests/servers/rendering/test_shader_converter.h"
#include "tests/servers/rendering/test_shader_language.h"
#include "tests/servers/rendering/test_shader_preprocessor.h"
#include "tests/servers/test_text_server.h"
#include "tests/test_validate_testing.h"
@ -266,7 +268,7 @@ struct GodotTestCaseListener : public doctest::IReporter {
String name = String(p_in.m_name);
String suite_name = String(p_in.m_test_suite);
if (name.contains("[SceneTree]") || name.contains("[Editor]")) {
if (name.contains("[SceneTree]") || name.contains("[Editor]") || name.contains("[ShaderDeprecatedConverter]") || name.contains("[ShaderLanguage]")) {
memnew(MessageQueue);
memnew(Input);