From 018f1ed97e632e5ffd27be75bbc487d9f9fd780e Mon Sep 17 00:00:00 2001 From: nikitalita <69168929+nikitalita@users.noreply.github.com> Date: Mon, 30 Sep 2024 03:12:56 -0700 Subject: [PATCH] Convert 3.x shaders --- editor/plugins/text_shader_editor.cpp | 74 + editor/plugins/text_shader_editor.h | 10 +- scene/resources/shader.cpp | 69 +- scene/resources/shader.h | 3 + servers/rendering/shader_converter.cpp | 2393 +++++++++++++++++ servers/rendering/shader_converter.h | 284 ++ servers/rendering/shader_language.cpp | 93 +- servers/rendering/shader_language.h | 22 +- .../servers/rendering/test_shader_converter.h | 799 ++++++ .../servers/rendering/test_shader_language.h | 144 + tests/test_main.cpp | 4 +- 11 files changed, 3885 insertions(+), 10 deletions(-) create mode 100644 servers/rendering/shader_converter.cpp create mode 100644 servers/rendering/shader_converter.h create mode 100644 tests/servers/rendering/test_shader_converter.h create mode 100644 tests/servers/rendering/test_shader_language.h diff --git a/editor/plugins/text_shader_editor.cpp b/editor/plugins/text_shader_editor.cpp index 0ff7aaa3fe4..bc333f96cdf 100644 --- a/editor/plugins/text_shader_editor.cpp +++ b/editor/plugins/text_shader_editor.cpp @@ -42,6 +42,10 @@ #include "servers/rendering/shader_preprocessor.h" #include "servers/rendering/shader_types.h" +#ifndef DISABLE_DEPRECATED +#include "servers/rendering/shader_converter.h" +#endif + /*** SHADER SYNTAX HIGHLIGHTER ****/ Dictionary GDShaderSyntaxHighlighter::_get_line_syntax_highlighting_impl(int p_line) { @@ -699,6 +703,28 @@ void TextShaderEditor::_menu_option(int p_option) { case EDIT_COMPLETE: { code_editor->get_text_editor()->request_code_completion(); } break; +#ifndef DISABLE_DEPRECATED + case EDIT_CONVERT: { + if (shader.is_null()) { + return; + } + String code = code_editor->get_text_editor()->get_text(); + if (code.is_empty()) { + return; + } + ShaderDeprecatedConverter converter; + if (!converter.is_code_deprecated(code)) { + if (converter.get_error_text() != String()) { + shader_convert_error_dialog->set_text(vformat(RTR("Line %d: %s"), converter.get_error_line(), converter.get_error_text())); + shader_convert_error_dialog->popup_centered(); + ERR_PRINT("Shader conversion failed: " + converter.get_error_text()); + } + confirm_convert_shader->popup_centered(); + return; + } + _convert_shader(); + } break; +#endif case SEARCH_FIND: { code_editor->get_find_replace_bar()->popup_search(); } break; @@ -755,6 +781,36 @@ void TextShaderEditor::_notification(int p_what) { } break; } } +#ifndef DISABLE_DEPRECATED +void TextShaderEditor::_convert_shader() { + if (shader.is_null()) { + return; + } + String code = code_editor->get_text_editor()->get_text(); + if (code.is_empty()) { + return; + } + ShaderDeprecatedConverter converter; + if (!converter.convert_code(code)) { + String err_text = converter.get_error_text(); + if (err_text.is_empty()) { + err_text = TTR("Unknown error occurred while converting the shader."); + } else if (converter.get_error_line() > 0) { + err_text = vformat("%s (line %d)", err_text, converter.get_error_line()); + } + + shader_convert_error_dialog->set_text(err_text); + shader_convert_error_dialog->popup_centered(); + ERR_PRINT("Shader conversion failed: " + err_text); + return; + } + String new_code = converter.emit_code(); + // Ensure undoable. + code_editor->get_text_editor()->set_text(new_code); + code_editor->get_text_editor()->tag_saved_version(); + code_editor->_validate_script(); +} +#endif void TextShaderEditor::_editor_settings_changed() { if (!EditorThemeManager::is_generated_theme_outdated() && @@ -1172,6 +1228,11 @@ TextShaderEditor::TextShaderEditor() { edit_menu->get_popup()->add_shortcut(ED_GET_SHORTCUT("ui_text_completion_query"), EDIT_COMPLETE); edit_menu->get_popup()->connect(SceneStringName(id_pressed), callable_mp(this, &TextShaderEditor::_menu_option)); + edit_menu->get_popup()->add_separator(); +#ifndef DISABLE_DEPRECATED + edit_menu->get_popup()->add_item(TTR("Convert 3.x Shader"), EDIT_CONVERT); + edit_menu->get_popup()->connect(SceneStringName(id_pressed), callable_mp(this, &TextShaderEditor::_menu_option)); +#endif search_menu = memnew(MenuButton); search_menu->set_shortcut_context(this); search_menu->set_text(TTR("Search")); @@ -1254,7 +1315,20 @@ TextShaderEditor::TextShaderEditor() { disk_changed->connect("custom_action", callable_mp(this, &TextShaderEditor::save_external_data)); add_child(disk_changed); +#ifndef DISABLE_DEPRECATED + shader_convert_error_dialog = memnew(AcceptDialog); + shader_convert_error_dialog->set_title(TTR("Error converting shader")); + shader_convert_error_dialog->set_hide_on_ok(true); + add_child(shader_convert_error_dialog); + confirm_convert_shader = memnew(ConfirmationDialog); + confirm_convert_shader->set_title(TTR("Confirm Convert 3.x Shader")); + confirm_convert_shader->set_text(TTR("This shader does not appear to be a 3.x shader.\nAre you sure you want to convert it?")); + confirm_convert_shader->get_ok_button()->set_text(TTR("Convert")); + confirm_convert_shader->get_cancel_button()->set_text(TTR("Cancel")); + confirm_convert_shader->connect("confirmed", callable_mp(this, &TextShaderEditor::_convert_shader)); + add_child(confirm_convert_shader); +#endif _editor_settings_changed(); code_editor->show_toggle_scripts_button(); // TODO: Disabled for now, because it doesn't work properly. } diff --git a/editor/plugins/text_shader_editor.h b/editor/plugins/text_shader_editor.h index 55efb4e30ed..e2538bc0e29 100644 --- a/editor/plugins/text_shader_editor.h +++ b/editor/plugins/text_shader_editor.h @@ -125,6 +125,7 @@ class TextShaderEditor : public ShaderEditor { EDIT_TOGGLE_WORD_WRAP, EDIT_TOGGLE_COMMENT, EDIT_COMPLETE, + EDIT_CONVERT, SEARCH_FIND, SEARCH_FIND_NEXT, SEARCH_FIND_PREV, @@ -150,13 +151,20 @@ class TextShaderEditor : public ShaderEditor { ConfirmationDialog *disk_changed = nullptr; ShaderTextEditor *code_editor = nullptr; +#ifndef DISABLE_DEPRECATED + AcceptDialog *shader_convert_error_dialog = nullptr; + ConfirmationDialog *confirm_convert_shader = nullptr; +#endif + bool compilation_success = true; void _menu_option(int p_option); void _prepare_edit_menu(); mutable Ref shader; mutable Ref shader_inc; - +#ifndef DISABLE_DEPRECATED + void _convert_shader(); +#endif void _editor_settings_changed(); void _apply_editor_settings(); void _project_settings_changed(); diff --git a/scene/resources/shader.cpp b/scene/resources/shader.cpp index 46d38146a67..bc8f0293783 100644 --- a/scene/resources/shader.cpp +++ b/scene/resources/shader.cpp @@ -32,8 +32,11 @@ #include "shader.compat.inc" #include "core/io/file_access.h" +#include "scene/scene_string_names.h" +#include "servers/rendering/rendering_server_globals.h" #include "servers/rendering/shader_language.h" #include "servers/rendering/shader_preprocessor.h" +#include "servers/rendering/shader_types.h" #include "servers/rendering_server.h" #include "texture.h" @@ -46,6 +49,12 @@ #endif #endif +#ifndef DISABLE_DEPRECATED +#include "servers/rendering/shader_converter.h" +#endif + +#define _LOAD_COMPAT_META_PROPERTY "_load_compat" + Shader::Mode Shader::get_mode() const { return mode; } @@ -70,6 +79,13 @@ void Shader::set_include_path(const String &p_path) { include_path = p_path; } +#ifndef DISABLE_DEPRECATED +ShaderLanguage::DataType _get_global_shader_uniform_type(const StringName &p_name) { + RS::GlobalShaderParameterType gvt = RenderingServerGlobals::material_storage->global_shader_parameter_get_type(p_name); + return (ShaderLanguage::DataType)RS::global_shader_uniform_type_get_shader_datatype(gvt); +} +#endif + void Shader::set_code(const String &p_code) { for (const Ref &E : include_dependencies) { E->disconnect_changed(callable_mp(this, &Shader::_dependency_changed)); @@ -77,6 +93,43 @@ void Shader::set_code(const String &p_code) { code = p_code; String pp_code = p_code; +#ifndef DISABLE_DEPRECATED + if (get_meta(_LOAD_COMPAT_META_PROPERTY, false)) { + // check if the Shader code compiles; if not, it's probably an old shader. + + ShaderLanguage sl; + ShaderLanguage::ShaderCompileInfo info; + String mode_string = ShaderLanguage::get_shader_type(p_code); + + RS::ShaderMode new_mode; + if (mode_string == "canvas_item") { + new_mode = RS::SHADER_CANVAS_ITEM; + } else if (mode_string == "particles") { + new_mode = RS::SHADER_PARTICLES; + } else if (mode_string == "spatial") { + new_mode = RS::SHADER_SPATIAL; + } else { + new_mode = RS::SHADER_MAX; + } + if (new_mode != RS::SHADER_MAX) { + info.functions = ShaderTypes::get_singleton()->get_functions(new_mode); + info.render_modes = ShaderTypes::get_singleton()->get_modes(new_mode); + info.shader_types = ShaderTypes::get_singleton()->get_types(); + info.global_shader_uniform_type_func = _get_global_shader_uniform_type; + Error err = sl.compile(p_code, info); + if (err) { + ShaderDeprecatedConverter sdc; + if (sdc.is_code_deprecated(p_code)) { + ERR_FAIL_COND_MSG(!sdc.convert_code(p_code), vformat("Shader conversion failed (line %d): %s", sdc.get_error_line(), sdc.get_error_text())); + code = sdc.emit_code(); + pp_code = code; + } else if (sdc.get_error_text() != "") { // Preprocessing failed. + WARN_PRINT(vformat("Shader conversion failed (line %d): %s", sdc.get_error_line(), sdc.get_error_text())); + } // If the code is reported as not deprecated, let it fall through to the compile step after this if block so that we get the full compile error. + } + } + } +#endif { String path = get_path(); @@ -88,7 +141,7 @@ void Shader::set_code(const String &p_code) { // 2) Server does not do interaction with Resource filetypes, this is a scene level feature. HashSet> new_include_dependencies; ShaderPreprocessor preprocessor; - Error result = preprocessor.preprocess(p_code, path, pp_code, nullptr, nullptr, nullptr, &new_include_dependencies); + Error result = preprocessor.preprocess(code, path, pp_code, nullptr, nullptr, nullptr, &new_include_dependencies); if (result == OK) { // This ensures previous include resources are not freed and then re-loaded during parse (which would make compiling slower) include_dependencies = new_include_dependencies; @@ -237,6 +290,20 @@ Array Shader::_get_shader_uniform_list(bool p_get_groups) { return ret; } +void Shader::_start_load(const StringName &p_res_format_type, int p_res_format_version) { +#ifndef DISABLE_DEPRECATED + if ((p_res_format_type == "binary" && p_res_format_version == 3) || (p_res_format_type == "text" && p_res_format_version == 2)) { + set_meta(_LOAD_COMPAT_META_PROPERTY, true); + } +#endif +} + +void Shader::_finish_load(const StringName &p_res_format_type, int p_res_format_version) { +#ifndef DISABLE_DEPRECATED + set_meta(_LOAD_COMPAT_META_PROPERTY, Variant()); +#endif +} + void Shader::_bind_methods() { ClassDB::bind_method(D_METHOD("get_mode"), &Shader::get_mode); diff --git a/scene/resources/shader.h b/scene/resources/shader.h index 682fbd7ea6a..ae568981fe4 100644 --- a/scene/resources/shader.h +++ b/scene/resources/shader.h @@ -94,6 +94,9 @@ public: virtual RID get_rid() const override; + virtual void _start_load(const StringName &p_res_format_type, int p_res_format_version) override; + virtual void _finish_load(const StringName &p_res_format_type, int p_res_format_version) override; + Shader(); ~Shader(); }; diff --git a/servers/rendering/shader_converter.cpp b/servers/rendering/shader_converter.cpp new file mode 100644 index 00000000000..5ad9f7dee47 --- /dev/null +++ b/servers/rendering/shader_converter.cpp @@ -0,0 +1,2393 @@ +/**************************************************************************/ +/* shader_converter.cpp */ +/**************************************************************************/ +/* This file is part of: */ +/* GODOT ENGINE */ +/* https://godotengine.org */ +/**************************************************************************/ +/* Copyright (c) 2014-present Godot Engine contributors (see AUTHORS.md). */ +/* Copyright (c) 2007-2014 Juan Linietsky, Ariel Manzur. */ +/* */ +/* Permission is hereby granted, free of charge, to any person obtaining */ +/* a copy of this software and associated documentation files (the */ +/* "Software"), to deal in the Software without restriction, including */ +/* without limitation the rights to use, copy, modify, merge, publish, */ +/* distribute, sublicense, and/or sell copies of the Software, and to */ +/* permit persons to whom the Software is furnished to do so, subject to */ +/* the following conditions: */ +/* */ +/* The above copyright notice and this permission notice shall be */ +/* included in all copies or substantial portions of the Software. */ +/* */ +/* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, */ +/* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF */ +/* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. */ +/* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY */ +/* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, */ +/* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE */ +/* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ +/**************************************************************************/ + +#include "core/error/error_macros.h" +#include "core/templates/pair.h" +#ifndef DISABLE_DEPRECATED + +#include "shader_converter.h" + +#define SL ShaderLanguage + +const char *ShaderDeprecatedConverter::old_builtin_funcs[]{ + "abs", + "acos", + "acosh", + "all", + "any", + "asin", + "asinh", + "atan", + "atanh", + "bool", + "bvec2", + "bvec3", + "bvec4", + "ceil", + "clamp", + "cos", + "cosh", + "cross", + "dFdx", + "dFdy", + "degrees", + "determinant", + "distance", + "dot", + "equal", + "exp", + "exp2", + "faceforward", + "float", + "floatBitsToInt", + "floatBitsToUint", + "floor", + "fract", + "fwidth", + "greaterThan", + "greaterThanEqual", + "int", + "intBitsToFloat", + "inverse", + "inversesqrt", + "isinf", + "isnan", + "ivec2", + "ivec3", + "ivec4", + "length", + "lessThan", + "lessThanEqual", + "log", + "log2", + "mat2", + "mat3", + "mat4", + "matrixCompMult", + "max", + "min", + "mix", + "mod", + "modf", + "normalize", + "not", + "notEqual", + "outerProduct", + "pow", + "radians", + "reflect", + "refract", + "round", + "roundEven", + "sign", + "sin", + "sinh", + "smoothstep", + "sqrt", + "step", + "tan", + "tanh", + "texelFetch", + "texture", + "textureGrad", + "textureLod", + "textureProj", + "textureProjLod", + "textureSize", + "transpose", + "trunc", + "uint", + "uintBitsToFloat", + "uvec2", + "uvec3", + "uvec4", + "vec2", + "vec3", + "vec4", + nullptr +}; + +const ShaderDeprecatedConverter::RenamedBuiltins ShaderDeprecatedConverter::renamed_builtins[] = { + { "ALPHA_SCISSOR", "ALPHA_SCISSOR_THRESHOLD", { { RS::SHADER_SPATIAL, { "fragment" } } }, false }, + { "CAMERA_MATRIX", "INV_VIEW_MATRIX", { { RS::SHADER_SPATIAL, { "vertex", "fragment", "light" } } }, false }, + { "INV_CAMERA_MATRIX", "VIEW_MATRIX", { { RS::SHADER_SPATIAL, { "vertex", "fragment", "light" } } }, false }, + { "NORMALMAP", "NORMAL_MAP", { { RS::SHADER_CANVAS_ITEM, { "fragment" } }, { RS::SHADER_SPATIAL, { "fragment" } } }, false }, + { "NORMALMAP_DEPTH", "NORMAL_MAP_DEPTH", { { RS::SHADER_CANVAS_ITEM, { "fragment" } }, { RS::SHADER_SPATIAL, { "fragment" } } }, false }, + { "TRANSMISSION", "BACKLIGHT", { { RS::SHADER_SPATIAL, { "fragment", "light" } } }, false }, + { "WORLD_MATRIX", "MODEL_MATRIX", { { RS::SHADER_CANVAS_ITEM, { "vertex" } }, { RS::SHADER_SPATIAL, { "vertex", "fragment", "light" } } }, false }, + { "CLEARCOAT_GLOSS", "CLEARCOAT_ROUGHNESS", { { RS::SHADER_SPATIAL, { "fragment" } } }, true }, // Usages require inversion, manually handled + { "INDEX", "INDEX", { { RS::SHADER_PARTICLES, { "vertex" } } }, true }, // No rename, was previously an int (vs. uint), usages require wrapping in `int()`. + { nullptr, nullptr, {}, false }, +}; + +const ShaderDeprecatedConverter::RenamedRenderModes ShaderDeprecatedConverter::renamed_render_modes[] = { + { RS::SHADER_SPATIAL, "depth_draw_alpha_prepass", "depth_prepass_alpha" }, + { RS::SHADER_MAX, nullptr, nullptr }, +}; + +const ShaderDeprecatedConverter::RenamedHints ShaderDeprecatedConverter::renamed_hints[]{ + { "hint_albedo", SL::TokenType::TK_HINT_SOURCE_COLOR }, + { "hint_aniso", SL::TokenType::TK_HINT_ANISOTROPY_TEXTURE }, + { "hint_black", SL::TokenType::TK_HINT_DEFAULT_BLACK_TEXTURE }, + { "hint_black_albedo", SL::TokenType::TK_HINT_DEFAULT_BLACK_TEXTURE }, + { "hint_color", SL::TokenType::TK_HINT_SOURCE_COLOR }, + { "hint_transparent", SL::TokenType::TK_HINT_DEFAULT_TRANSPARENT_TEXTURE }, + { "hint_white", SL::TokenType::TK_HINT_DEFAULT_WHITE_TEXTURE }, + { nullptr, {} }, +}; + +const ShaderDeprecatedConverter::RenamedFunctions ShaderDeprecatedConverter::renamed_functions[]{ + { RS::SHADER_PARTICLES, SL::TK_TYPE_VOID, 0, "vertex", "process" }, + { RS::SHADER_MAX, SL::TK_EMPTY, 0, nullptr, nullptr }, +}; + +const ShaderDeprecatedConverter::RemovedRenderModes ShaderDeprecatedConverter::removed_render_modes[]{ + { RS::SHADER_SPATIAL, "specular_blinn", false }, + { RS::SHADER_SPATIAL, "specular_phong", false }, + { RS::SHADER_SPATIAL, "async_visible", true }, + { RS::SHADER_SPATIAL, "async_hidden", true }, + { RS::SHADER_MAX, nullptr, false }, +}; + +// These necessitate adding a uniform to the shader. +const ShaderDeprecatedConverter::RemovedBuiltins ShaderDeprecatedConverter::removed_builtins[]{ + { "SCREEN_TEXTURE", SL::TK_TYPE_SAMPLER2D, { SL::TK_HINT_SCREEN_TEXTURE, SL::TK_FILTER_LINEAR_MIPMAP }, { { RS::SHADER_SPATIAL, { "fragment" } }, { RS::SHADER_CANVAS_ITEM, { "fragment" } } } }, + { "DEPTH_TEXTURE", SL::TK_TYPE_SAMPLER2D, { SL::TK_HINT_DEPTH_TEXTURE, SL::TK_FILTER_LINEAR_MIPMAP }, { { RS::SHADER_SPATIAL, { "fragment" } } } }, + { "NORMAL_ROUGHNESS_TEXTURE", SL::TK_TYPE_SAMPLER2D, { SL::TK_HINT_NORMAL_ROUGHNESS_TEXTURE, SL::TK_FILTER_LINEAR_MIPMAP }, { { RS::SHADER_SPATIAL, { "fragment" } } } }, + { "MODULATE", SL::TK_ERROR, {}, { { RS::SHADER_CANVAS_ITEM, { "vertex", "fragment", "light" } } } }, // TODO: remove this when the MODULATE PR lands. + { nullptr, SL::TK_EMPTY, {}, {} }, +}; + +const char *ShaderDeprecatedConverter::removed_types[]{ + nullptr, +}; + +HashSet ShaderDeprecatedConverter::_new_builtin_funcs = HashSet(); + +HashSet ShaderDeprecatedConverter::_construct_new_builtin_funcs() { + List current_builtin_funcs; + ShaderLanguage::get_builtin_funcs(¤t_builtin_funcs); + HashSet old_funcs; + for (int i = 0; old_builtin_funcs[i] != nullptr; i++) { + old_funcs.insert(old_builtin_funcs[i]); + } + HashSet new_funcs; + for (List::Element *E = current_builtin_funcs.front(); E; E = E->next()) { + if (!old_funcs.has(E->get())) { + new_funcs.insert(E->get()); + } + } + return new_funcs; +} + +String ShaderDeprecatedConverter::get_builtin_rename(const String &p_name) { + for (int i = 0; renamed_builtins[i].name != nullptr; i++) { + if (renamed_builtins[i].name == p_name) { + return renamed_builtins[i].replacement; + } + } + return String(); +} + +bool ShaderDeprecatedConverter::has_builtin_rename(RS::ShaderMode p_mode, const String &p_name, const String &p_function) { + for (int i = 0; renamed_builtins[i].name != nullptr; i++) { + if (renamed_builtins[i].name == p_name) { + for (int j = 0; j < renamed_builtins[i].mode_functions.size(); j++) { + if (renamed_builtins[i].mode_functions[j].first == p_mode) { + if (p_function == "") { // Empty function means don't check function. + return true; + } + for (int k = 0; k < renamed_builtins[i].mode_functions[j].second.size(); k++) { + if (renamed_builtins[i].mode_functions[j].second[k] == p_function) { + return true; + } + } + } + } + } + } + return false; +} + +SL::TokenType ShaderDeprecatedConverter::get_removed_builtin_uniform_type(const String &p_name) { + for (int i = 0; removed_builtins[i].name != nullptr; i++) { + if (removed_builtins[i].name == p_name) { + return removed_builtins[i].uniform_type; + } + } + return SL::TK_EMPTY; +} + +Vector ShaderDeprecatedConverter::get_removed_builtin_hints(const String &p_name) { + for (int i = 0; removed_builtins[i].name != nullptr; i++) { + if (removed_builtins[i].name == p_name) { + return removed_builtins[i].hints; + } + } + return Vector(); +} + +bool ShaderDeprecatedConverter::_rename_has_special_handling(const String &p_name) { + for (int i = 0; renamed_builtins[i].name != nullptr; i++) { + if (renamed_builtins[i].name == p_name) { + return renamed_builtins[i].special_handling; + } + } + return false; +} + +void ShaderDeprecatedConverter::_get_builtin_renames_list(List *r_list) { + for (int i = 0; renamed_builtins[i].name != nullptr; i++) { + r_list->push_back(renamed_builtins[i].name); + } +} + +void ShaderDeprecatedConverter::_get_render_mode_renames_list(List *r_list) { + for (int i = 0; renamed_render_modes[i].name != nullptr; i++) { + r_list->push_back(renamed_render_modes[i].name); + } +} + +void ShaderDeprecatedConverter::_get_hint_renames_list(List *r_list) { + for (int i = 0; renamed_hints[i].name != nullptr; i++) { + r_list->push_back(renamed_hints[i].name); + } +} + +void ShaderDeprecatedConverter::_get_function_renames_list(List *r_list) { + for (int i = 0; renamed_functions[i].name != nullptr; i++) { + r_list->push_back(renamed_functions[i].name); + } +} + +void ShaderDeprecatedConverter::_get_render_mode_removals_list(List *r_list) { + for (int i = 0; removed_render_modes[i].name != nullptr; i++) { + r_list->push_back(removed_render_modes[i].name); + } +} + +void ShaderDeprecatedConverter::_get_builtin_removals_list(List *r_list) { + for (int i = 0; removed_builtins[i].name != nullptr; i++) { + r_list->push_back(removed_builtins[i].name); + } +} + +void ShaderDeprecatedConverter::_get_type_removals_list(List *r_list) { + for (int i = 0; removed_types[i] != nullptr; i++) { + r_list->push_back(removed_types[i]); + } +} + +Vector ShaderDeprecatedConverter::_get_funcs_builtin_rename(RS::ShaderMode p_mode, const String &p_name) { + Vector funcs; + for (int i = 0; renamed_builtins[i].name != nullptr; i++) { + if (renamed_builtins[i].name == p_name) { + for (int j = 0; j < renamed_builtins[i].mode_functions.size(); j++) { + if (renamed_builtins[i].mode_functions[j].first == p_mode) { + for (int k = 0; k < renamed_builtins[i].mode_functions[j].second.size(); k++) { + funcs.push_back(renamed_builtins[i].mode_functions[j].second[k]); + } + } + } + } + } + return funcs; +} + +Vector ShaderDeprecatedConverter::_get_funcs_builtin_removal(RS::ShaderMode p_mode, const String &p_name) { + Vector funcs; + for (int i = 0; removed_builtins[i].name != nullptr; i++) { + if (removed_builtins[i].name == p_name) { + for (int j = 0; j < removed_builtins[i].mode_functions.size(); j++) { + if (removed_builtins[i].mode_functions[j].first == p_mode) { + for (int k = 0; k < removed_builtins[i].mode_functions[j].second.size(); k++) { + funcs.push_back(removed_builtins[i].mode_functions[j].second[k]); + } + } + } + } + } + return funcs; +} + +bool ShaderDeprecatedConverter::is_removed_builtin(RS::ShaderMode p_mode, const String &p_name, const String &p_function) { + for (int i = 0; removed_builtins[i].name != nullptr; i++) { + if (removed_builtins[i].name == p_name) { + for (int j = 0; j < removed_builtins[i].mode_functions.size(); j++) { + if (removed_builtins[i].mode_functions[j].first == p_mode) { + if (p_function == "") { // Empty function means don't check function. + return true; + } + for (int k = 0; k < removed_builtins[i].mode_functions[j].second.size(); k++) { + if (removed_builtins[i].mode_functions[j].second[k] == p_function) { + return true; + } + } + } + } + } + } + return false; +} + +bool ShaderDeprecatedConverter::has_hint_replacement(const String &p_name) { + for (int i = 0; renamed_hints[i].name != nullptr; i++) { + if (renamed_hints[i].name == p_name) { + return true; + } + } + return false; +} + +SL::TokenType ShaderDeprecatedConverter::get_hint_replacement(const String &p_name) { + for (int i = 0; renamed_hints[i].name != nullptr; i++) { + if (renamed_hints[i].name == p_name) { + return renamed_hints[i].replacement; + } + } + return {}; +} + +bool ShaderDeprecatedConverter::is_renamed_render_mode(RS::ShaderMode p_mode, const String &p_name) { + for (int i = 0; renamed_render_modes[i].name != nullptr; i++) { + if (renamed_render_modes[i].mode == p_mode && renamed_render_modes[i].name == p_name) { + return true; + } + } + return false; +} + +String ShaderDeprecatedConverter::get_render_mode_rename(const String &p_name) { + for (int i = 0; renamed_render_modes[i].name != nullptr; i++) { + if (renamed_render_modes[i].name == p_name) { + return renamed_render_modes[i].replacement; + } + } + return {}; +} + +bool ShaderDeprecatedConverter::is_renamed_main_function(RS::ShaderMode p_mode, const String &p_name) { + for (int i = 0; renamed_functions[i].name != nullptr; i++) { + if (renamed_functions[i].mode == p_mode && renamed_functions[i].name == p_name) { + return true; + } + } + return false; +} + +bool ShaderDeprecatedConverter::is_renamee_main_function(RS::ShaderMode p_mode, const String &p_name) { + for (int i = 0; renamed_functions[i].name != nullptr; i++) { + if (renamed_functions[i].mode == p_mode && renamed_functions[i].replacement == p_name) { + return true; + } + } + return false; +} + +SL::TokenType ShaderDeprecatedConverter::get_renamed_function_type(const String &p_name) { + for (int i = 0; renamed_functions[i].name != nullptr; i++) { + if (renamed_functions[i].name == p_name) { + return renamed_functions[i].type; + } + } + return SL::TK_MAX; +} + +int ShaderDeprecatedConverter::get_renamed_function_arg_count(const String &p_name) { + for (int i = 0; renamed_functions[i].name != nullptr; i++) { + if (renamed_functions[i].name == p_name) { + return renamed_functions[i].arg_count; + } + } + return -1; +} + +bool ShaderDeprecatedConverter::FunctionDecl::is_renamed_main_function(RS::ShaderMode p_mode) const { + if (!name_pos || !type_pos) { + return false; + } + if (ShaderDeprecatedConverter::is_renamed_main_function(p_mode, name_pos->get().text) && type_pos->get().type == get_renamed_function_type(name_pos->get().text) && arg_count == get_renamed_function_arg_count(name_pos->get().text)) { + return true; + } + return false; +} + +bool ShaderDeprecatedConverter::FunctionDecl::is_new_main_function(RS::ShaderMode p_mode) const { + if (!name_pos || !type_pos) { + return false; + } + for (int i = 0; renamed_functions[i].name != nullptr; i++) { + if (renamed_functions[i].mode == p_mode && renamed_functions[i].replacement == name_pos->get().text && renamed_functions[i].type == type_pos->get().type && arg_count == renamed_functions[i].arg_count) { + return true; + } + } + return false; +} + +String ShaderDeprecatedConverter::get_main_function_rename(const String &p_name) { + for (int i = 0; renamed_functions[i].name != nullptr; i++) { + if (renamed_functions[i].name == p_name) { + return renamed_functions[i].replacement; + } + } + return String(); +} + +bool ShaderDeprecatedConverter::has_removed_render_mode(RS::ShaderMode p_mode, const String &p_name) { + for (int i = 0; removed_render_modes[i].name != nullptr; i++) { + if (removed_render_modes[i].mode == p_mode && removed_render_modes[i].name == p_name) { + return true; + } + } + return false; +} + +bool ShaderDeprecatedConverter::can_remove_render_mode(const String &p_name) { + for (int i = 0; removed_render_modes[i].name != nullptr; i++) { + if (removed_render_modes[i].name == p_name) { + return removed_render_modes[i].can_remove; + } + } + return false; +} + +bool ShaderDeprecatedConverter::has_removed_type(const String &p_name) { + for (int i = 0; removed_types[i] != nullptr; i++) { + if (removed_types[i] == p_name) { + return true; + } + } + return false; +} + +static constexpr const char *token_to_str[] = { + "", // TK_EMPTY + "", // TK_IDENTIFIER + "true", + "false", + "", // TK_FLOAT_CONSTANT + "", // TK_INT_CONSTANT + "", // TK_UINT_CONSTANT + "", // TK_STRING_CONSTANT + "void", + "bool", + "bvec2", + "bvec3", + "bvec4", + "int", + "ivec2", + "ivec3", + "ivec4", + "uint", + "uvec2", + "uvec3", + "uvec4", + "float", + "vec2", + "vec3", + "vec4", + "mat2", + "mat3", + "mat4", + "sampler2D", + "isampler2D", + "usampler2D", + "sampler2DArray", + "isampler2DArray", + "usampler2DArray", + "sampler3D", + "isampler3D", + "usampler3D", + "samplerCube", + "samplerCubeArray", + "samplerExternalOES", + "flat", + "smooth", + "const", + "struct", + "lowp", + "mediump", + "highp", + "==", + "!=", + "<", + "<=", + ">", + ">=", + "&&", + "||", + "!", + "+", + "-", + "*", + "/", + "%", + "<<", + ">>", + "=", + "+=", + "-=", + "*=", + "/=", + "%=", + "<<=", + ">>=", + "&=", + "|=", + "^=", + "&", + "|", + "^", + "~", + "++", + "--", + "if", + "else", + "for", + "while", + "do", + "switch", + "case", + "default", + "break", + "continue", + "return", + "discard", + "[", + "]", + "{", + "}", + "(", + ")", + "?", + ",", + ":", + ";", + ".", + "uniform", + "group_uniforms", + "instance", + "global", + "varying", + "in", + "out", + "inout", + "render_mode", + "hint_default_white", + "hint_default_black", + "hint_default_transparent", + "hint_normal", + "hint_roughness_normal", + "hint_roughness_r", + "hint_roughness_g", + "hint_roughness_b", + "hint_roughness_a", + "hint_roughness_gray", + "hint_anisotropy", + "source_color", + "hint_range", + "hint_enum", + "instance_index", + "hint_screen_texture", + "hint_normal_roughness_texture", + "hint_depth_texture", + "filter_nearest", + "filter_linear", + "filter_nearest_mipmap", + "filter_linear_mipmap", + "filter_nearest_mipmap_anisotropic", + "filter_linear_mipmap_anisotropic", + "repeat_enable", + "repeat_disable", + "shader_type", + "", // TK_CURSOR + "", // TK_ERROR + "", // TK_EOF + "\t", + "\r", + " ", + "\n", + "", // TK_BLOCK_COMMENT + "", // TK_LINE_COMMENT + "", // TK_PREPROC_DIRECTIVE +}; +static_assert(ShaderLanguage::TK_MAX == sizeof(token_to_str) / sizeof(token_to_str[0]), "token_to_str length does not match token count (Did TK_MAX change?)"); + +bool ShaderDeprecatedConverter::token_is_skippable(const Token &p_tk) { + switch (p_tk.type) { + case ShaderLanguage::TK_TAB: + case ShaderLanguage::TK_CR: + case ShaderLanguage::TK_SPACE: + case ShaderLanguage::TK_NEWLINE: + case ShaderLanguage::TK_BLOCK_COMMENT: + case ShaderLanguage::TK_LINE_COMMENT: + case ShaderLanguage::TK_PREPROC_DIRECTIVE: + return true; + default: + break; + } + return false; +} + +List::Element *ShaderDeprecatedConverter::_get_next_token_ptr(List::Element *p_curr_ptr) const { + ERR_FAIL_COND_V(p_curr_ptr == nullptr, p_curr_ptr); + if (p_curr_ptr->next() == nullptr) { + return p_curr_ptr; + } + p_curr_ptr = p_curr_ptr->next(); + while (token_is_skippable(p_curr_ptr->get())) { + if (p_curr_ptr->next() == nullptr) { + return p_curr_ptr; + } + p_curr_ptr = p_curr_ptr->next(); + } + return p_curr_ptr; +} + +List::Element *ShaderDeprecatedConverter::_get_prev_token_ptr(List::Element *_curr_ptr) const { + ERR_FAIL_COND_V(_curr_ptr == nullptr, _curr_ptr); + if (_curr_ptr->prev() == nullptr) { + return _curr_ptr; + } + _curr_ptr = _curr_ptr->prev(); + while (token_is_skippable(_curr_ptr->get())) { + if (_curr_ptr->prev() == nullptr) { + return _curr_ptr; + } + _curr_ptr = _curr_ptr->prev(); + } + return _curr_ptr; +} + +List::Element *ShaderDeprecatedConverter::get_next_token() { + curr_ptr = _get_next_token_ptr(curr_ptr); + return curr_ptr; +} + +List::Element *ShaderDeprecatedConverter::get_prev_token() { + curr_ptr = _get_prev_token_ptr(curr_ptr); + return curr_ptr; +} + +List::Element *ShaderDeprecatedConverter::remove_cur_and_get_next() { + ERR_FAIL_COND_V(!curr_ptr, nullptr); + List::Element *prev = curr_ptr->prev(); + if (!prev) { + prev = curr_ptr->next(); + code_tokens.erase(curr_ptr); + while (token_is_skippable(prev->get())) { + if (prev->next() == nullptr) { + return prev; + } + prev = prev->next(); + } + return prev; + } + code_tokens.erase(curr_ptr); + curr_ptr = prev; + return get_next_token(); +} + +SL::TokenType ShaderDeprecatedConverter::_peek_tk_type(int64_t p_count, List::Element **r_pos) const { + ERR_FAIL_COND_V(!curr_ptr, ShaderLanguage::TK_EOF); + if (p_count == 0) { + return curr_ptr->get().type; + } + + bool backwards = p_count < 0; + uint64_t max_count = abs(p_count); + TokenE *start_ptr = curr_ptr; + for (uint64_t i = 0; i < max_count; i++) { + TokenE *_ptr = backwards ? _get_prev_token_ptr(start_ptr) : _get_next_token_ptr(start_ptr); + if (!_ptr) { + if (r_pos) { + *r_pos = start_ptr; + } + return ShaderLanguage::TK_EOF; + } + start_ptr = _ptr; + } + if (r_pos) { + *r_pos = start_ptr; + } + return start_ptr->get().type; +} + +bool ShaderDeprecatedConverter::scope_has_decl(const String &p_scope, const String &p_name) const { + if (uniform_decls.has(p_name) || + (scope_declarations.has("") && scope_declarations[""].has(p_name)) || + (scope_declarations.has(p_scope) && scope_declarations[p_scope].has(p_name))) { + return true; + } + return false; +} + +SL::TokenType ShaderDeprecatedConverter::peek_next_tk_type(uint32_t p_count) const { + return _peek_tk_type(p_count); +} + +SL::TokenType ShaderDeprecatedConverter::peek_prev_tk_type(uint32_t p_count) const { + return _peek_tk_type(-((int64_t)p_count)); +} + +List::Element *ShaderDeprecatedConverter::get_pos() const { + ERR_FAIL_COND_V(!curr_ptr, nullptr); + return curr_ptr; +} + +bool ShaderDeprecatedConverter::reset_to(List::Element *p_pos) { + ERR_FAIL_COND_V(p_pos == nullptr, false); + curr_ptr = p_pos; + return true; +} + +bool ShaderDeprecatedConverter::insert_after(const Vector &p_token_list, List::Element *p_pos) { + ERR_FAIL_COND_V(p_pos == nullptr, false); + for (int i = p_token_list.size() - 1; i >= 0; i--) { + const Token &tk = p_token_list[i]; + code_tokens.insert_after(p_pos, { tk.type, tk.text, tk.constant, tk.line, tk.length, NEW_IDENT }); + } + return true; +} + +bool ShaderDeprecatedConverter::insert_before(const Vector &p_token_list, List::Element *p_pos) { + ERR_FAIL_COND_V(p_pos == nullptr, false); + for (const Token &tk : p_token_list) { + code_tokens.insert_before(p_pos, { tk.type, tk.text, tk.constant, tk.line, tk.length, NEW_IDENT }); + } + return true; +} + +bool ShaderDeprecatedConverter::insert_after(const Token &p_token, List::Element *p_pos) { + ERR_FAIL_COND_V(p_pos == nullptr, false); + Token new_token = p_token; + new_token.pos = NEW_IDENT; + code_tokens.insert_after(p_pos, new_token); + return true; +} + +bool ShaderDeprecatedConverter::insert_before(const Token &p_token, List::Element *p_pos) { + ERR_FAIL_COND_V(p_pos == nullptr, false); + Token new_token = p_token; + new_token.pos = NEW_IDENT; + code_tokens.insert_before(p_pos, new_token); + return true; +} + +List::Element *ShaderDeprecatedConverter::replace_curr(const Token &p_token) { + ERR_FAIL_COND_V(curr_ptr == nullptr, nullptr); + Token new_token = p_token; + new_token.pos = NEW_IDENT; + List::Element *prev = curr_ptr; + curr_ptr = code_tokens.insert_before(curr_ptr, new_token); + ERR_FAIL_COND_V(!code_tokens.erase(prev), nullptr); + return curr_ptr; +} + +SL::Token ShaderDeprecatedConverter::mkTok(TokenType p_type, const StringName &p_text, double p_constant, uint16_t p_line) { + return { p_type, p_text, p_constant, p_line, 0, NEW_IDENT }; +} + +bool ShaderDeprecatedConverter::_insert_uniform_declaration(const String &p_name) { + if (after_shader_decl == nullptr) { + return false; + } + TokenType type = get_removed_builtin_uniform_type(p_name); + Vector hints = get_removed_builtin_hints(p_name); + Vector uni_decl = { mkTok(TT::TK_NEWLINE), mkTok(TT::TK_UNIFORM), mkTok(TT::TK_SPACE), mkTok(type), + mkTok(TT::TK_SPACE), mkTok(TT::TK_IDENTIFIER, p_name), mkTok(TT::TK_SPACE), mkTok(TT::TK_COLON), + mkTok(TT::TK_SPACE) }; + for (int i = 0; i < hints.size(); i++) { + uni_decl.append(mkTok(hints[i])); + if (i < hints.size() - 1) { + uni_decl.append(mkTok(TT::TK_COMMA)); + uni_decl.append(mkTok(TT::TK_SPACE)); + } + } + uni_decl.append_array({ mkTok(TT::TK_SEMICOLON), mkTok(TT::TK_NEWLINE) }); + if (!insert_after(uni_decl, after_shader_decl)) { + return false; + } + TokenE *cur_pos = get_pos(); + reset_to(after_shader_decl); + UniformDecl uni; + uni.start_pos = get_next_token(); // uniform + uni.type_pos = get_next_token(); // type + uni.name_pos = get_next_token(); // id + get_next_token(); // colon + for (int i = 0; i < hints.size(); i++) { + uni.hint_poses.push_back(get_next_token()); // hint + if (i < hints.size() - 1) { + get_next_token(); // comma + } + } + uni.end_pos = get_next_token(); + uniform_decls[p_name] = uni; + reset_to(cur_pos); + return true; +} + +RS::ShaderMode ShaderDeprecatedConverter::get_shader_mode_from_string(const String &p_mode) { + if (p_mode == "spatial") { + return RS::SHADER_SPATIAL; + } else if (p_mode == "canvas_item") { + return RS::SHADER_CANVAS_ITEM; + } else if (p_mode == "particles") { + return RS::SHADER_PARTICLES; + } else { // 3.x didn't support anything else. + return RS::SHADER_MAX; + } +} +// Remove from the current token to end (exclsusive) and return the new current token. +List::Element *ShaderDeprecatedConverter::_remove_from_curr_to(TokenE *p_end) { + ERR_FAIL_COND_V(p_end == nullptr, nullptr); + while (curr_ptr != p_end) { + TokenE *next = curr_ptr->next(); + code_tokens.erase(curr_ptr); + curr_ptr = next; + } + return curr_ptr; +} + +String ShaderDeprecatedConverter::get_tokentype_text(TokenType p_tk_type) { + return token_to_str[p_tk_type]; +} + +List::Element *ShaderDeprecatedConverter::_get_end_of_closure() { + int additional_closures = 0; + TokenE *ptr = curr_ptr; + bool start_is_scope_start = false; + switch (ptr->get().type) { + case TT::TK_CURLY_BRACKET_OPEN: + case TT::TK_PARENTHESIS_OPEN: + case TT::TK_BRACKET_OPEN: + start_is_scope_start = true; + break; + default: + break; + } + for (; ptr; ptr = ptr->next()) { + switch (ptr->get().type) { + case TT::TK_CURLY_BRACKET_OPEN: + case TT::TK_PARENTHESIS_OPEN: + case TT::TK_BRACKET_OPEN: { + additional_closures++; + } break; + case TT::TK_CURLY_BRACKET_CLOSE: + case TT::TK_PARENTHESIS_CLOSE: + case TT::TK_BRACKET_CLOSE: { + if (additional_closures > 0) { + additional_closures--; + if (start_is_scope_start && additional_closures == 0) { + return ptr; + } + } else { + return ptr; + } + } break; + case TT::TK_SEMICOLON: + case TT::TK_COMMA: { + if (additional_closures <= 0) { + return _get_prev_token_ptr(ptr); + } + } break; + case TT::TK_EOF: + case TT::TK_ERROR: { + err_line = curr_ptr->get().line + 1; + err_str = ptr->get().type == TT::TK_ERROR ? vformat(RTR("Parser Error (%s) ", ptr->get().text)) : vformat(RTR("Could not find end of closure for token '%s'"), get_tokentype_text(curr_ptr->get().type)); + return ptr; + } break; + default: + break; + } + } + return ptr; +} + +bool ShaderDeprecatedConverter::token_is_type(const Token &p_tk) { + return (ShaderLanguage::is_token_datatype(p_tk.type)) || (p_tk.type == TT::TK_IDENTIFIER && has_removed_type(p_tk.text)); +} + +bool ShaderDeprecatedConverter::token_is_hint(const Token &p_tk) { + if (p_tk.type == TT::TK_IDENTIFIER) { + return has_hint_replacement(p_tk.text); + } + return SL::is_token_hint(p_tk.type); +} + +String ShaderDeprecatedConverter::get_token_literal_text(const Token &p_tk) const { + switch (p_tk.type) { + case TT::TK_PREPROC_DIRECTIVE: + case TT::TK_LINE_COMMENT: + case TT::TK_BLOCK_COMMENT: + case TT::TK_IDENTIFIER: { // Identifiers prefixed with `__` are modified to `_dup_` by the SL parser + if (p_tk.pos == NEW_IDENT) { + return p_tk.text; + } else { + return old_code.substr(p_tk.pos, p_tk.length); + } + } break; + case TT::TK_INT_CONSTANT: + case TT::TK_FLOAT_CONSTANT: + case TT::TK_UINT_CONSTANT: { + if (p_tk.pos == NEW_IDENT) { + // Fix for 3.x float constants not having a decimal point. + if (!p_tk.is_integer_constant() && p_tk.text != "") { + return p_tk.text; + } + String const_str = rtos(p_tk.constant); + if (!p_tk.is_integer_constant() && !const_str.contains(".")) { + const_str += ".0"; + } + return const_str; + } else { + return old_code.substr(p_tk.pos, p_tk.length); + } + } break; + case TT::TK_ERROR: + case TT::TK_EOF: { + return ""; + } break; + default: + break; + } + return token_to_str[p_tk.type]; +} + +bool ShaderDeprecatedConverter::tokentype_is_identifier(const TokenType &p_tk_type) { + return p_tk_type == TT::TK_IDENTIFIER || tokentype_is_new_reserved_keyword(p_tk_type); +} + +bool ShaderDeprecatedConverter::tokentype_is_new_type(const TokenType &p_type) { + // the following types are in both 3.x and 4.x + return (!( + p_type == TT::TK_TYPE_VOID || + p_type == TT::TK_TYPE_BOOL || + p_type == TT::TK_TYPE_BVEC2 || + p_type == TT::TK_TYPE_BVEC3 || + p_type == TT::TK_TYPE_BVEC4 || + p_type == TT::TK_TYPE_INT || + p_type == TT::TK_TYPE_IVEC2 || + p_type == TT::TK_TYPE_IVEC3 || + p_type == TT::TK_TYPE_IVEC4 || + p_type == TT::TK_TYPE_UINT || + p_type == TT::TK_TYPE_UVEC2 || + p_type == TT::TK_TYPE_UVEC3 || + p_type == TT::TK_TYPE_UVEC4 || + p_type == TT::TK_TYPE_FLOAT || + p_type == TT::TK_TYPE_VEC2 || + p_type == TT::TK_TYPE_VEC3 || + p_type == TT::TK_TYPE_VEC4 || + p_type == TT::TK_TYPE_MAT2 || + p_type == TT::TK_TYPE_MAT3 || + p_type == TT::TK_TYPE_MAT4 || + p_type == TT::TK_TYPE_SAMPLER2D || + p_type == TT::TK_TYPE_ISAMPLER2D || + p_type == TT::TK_TYPE_USAMPLER2D || + p_type == TT::TK_TYPE_SAMPLER2DARRAY || + p_type == TT::TK_TYPE_ISAMPLER2DARRAY || + p_type == TT::TK_TYPE_USAMPLER2DARRAY || + p_type == TT::TK_TYPE_SAMPLER3D || + p_type == TT::TK_TYPE_ISAMPLER3D || + p_type == TT::TK_TYPE_USAMPLER3D || + p_type == TT::TK_TYPE_SAMPLERCUBE)) && + SL::is_token_datatype(p_type); +} + +// checks for reserved keywords only found in 4.x +bool ShaderDeprecatedConverter::tokentype_is_new_reserved_keyword(const TokenType &tk_type) { + switch (tk_type) { + // The following keyword tokens are in both 3.x and 4.x. + case TT::TK_ARG_IN: + case TT::TK_ARG_INOUT: + case TT::TK_ARG_OUT: + case TT::TK_CF_BREAK: + case TT::TK_CF_CASE: + case TT::TK_CF_CONTINUE: + case TT::TK_CF_DEFAULT: + case TT::TK_CF_DISCARD: + case TT::TK_CF_DO: + case TT::TK_CF_ELSE: + case TT::TK_CF_FOR: + case TT::TK_CF_IF: + case TT::TK_CF_RETURN: + case TT::TK_CF_SWITCH: + case TT::TK_CF_WHILE: + case TT::TK_CONST: + case TT::TK_ERROR: + case TT::TK_FALSE: + case TT::TK_HINT_NORMAL_TEXTURE: + case TT::TK_HINT_RANGE: + case TT::TK_INTERPOLATION_FLAT: + case TT::TK_INTERPOLATION_SMOOTH: + case TT::TK_PRECISION_HIGH: + case TT::TK_PRECISION_LOW: + case TT::TK_PRECISION_MID: + case TT::TK_RENDER_MODE: + case TT::TK_SHADER_TYPE: + case TT::TK_STRUCT: + case TT::TK_TRUE: + case TT::TK_TYPE_BOOL: + case TT::TK_TYPE_BVEC2: + case TT::TK_TYPE_BVEC3: + case TT::TK_TYPE_BVEC4: + case TT::TK_TYPE_FLOAT: + case TT::TK_TYPE_INT: + case TT::TK_TYPE_ISAMPLER2D: + case TT::TK_TYPE_ISAMPLER2DARRAY: + case TT::TK_TYPE_ISAMPLER3D: + case TT::TK_TYPE_IVEC2: + case TT::TK_TYPE_IVEC3: + case TT::TK_TYPE_IVEC4: + case TT::TK_TYPE_MAT2: + case TT::TK_TYPE_MAT3: + case TT::TK_TYPE_MAT4: + case TT::TK_TYPE_SAMPLER2D: + case TT::TK_TYPE_SAMPLER2DARRAY: + case TT::TK_TYPE_SAMPLER3D: + case TT::TK_TYPE_SAMPLERCUBE: + case TT::TK_TYPE_UINT: + case TT::TK_TYPE_USAMPLER2D: + case TT::TK_TYPE_USAMPLER2DARRAY: + case TT::TK_TYPE_USAMPLER3D: + case TT::TK_TYPE_UVEC2: + case TT::TK_TYPE_UVEC3: + case TT::TK_TYPE_UVEC4: + case TT::TK_TYPE_VEC2: + case TT::TK_TYPE_VEC3: + case TT::TK_TYPE_VEC4: + case TT::TK_TYPE_VOID: + case TT::TK_UNIFORM: + case TT::TK_VARYING: + return false; + default: + break; + } + return SL::is_token_keyword(tk_type); +} + +bool ShaderDeprecatedConverter::tokentype_is_new_hint(const TokenType &tk_type) { + switch (tk_type) { + case TT::TK_HINT_NORMAL_TEXTURE: // These two are in both 3.x and 4.x. + case TT::TK_HINT_RANGE: + return false; + default: + break; + } + return SL::is_token_hint(tk_type); +} + +bool ShaderDeprecatedConverter::id_is_new_builtin_func(const String &p_name) { + if (_new_builtin_funcs.is_empty()) { + _new_builtin_funcs = _construct_new_builtin_funcs(); + } + return _new_builtin_funcs.has(p_name); +} + +void ShaderDeprecatedConverter::_get_new_builtin_funcs_list(List *r_list) { + if (_new_builtin_funcs.is_empty()) { + _new_builtin_funcs = _construct_new_builtin_funcs(); + } + for (const String &k : _new_builtin_funcs) { + r_list->push_back(k); + } +} + +bool ShaderDeprecatedConverter::_add_warning_comment_before(const String &p_comment, List::Element *p_pos) { + // Peek back until we hit a newline or the start of the file (EOF). + TokenE *start_pos = p_pos; + if (!start_pos) { + return false; + } + while (start_pos->prev() && start_pos->get().type != TT::TK_NEWLINE && start_pos->get().type != TT::TK_EOF) { + start_pos = start_pos->prev(); + } + String comment = "/* !convert WARNING: " + p_comment + " */\n"; + // Check if the token after this is a block comment and has the same comment. + if (start_pos->next() && start_pos->next()->get().type == TT::TK_BLOCK_COMMENT && get_token_literal_text(start_pos->next()->get()) == comment) { + return true; + } + + return insert_after(mkTok(TT::TK_BLOCK_COMMENT, comment), start_pos); +} + +bool ShaderDeprecatedConverter::_add_comment_at_eol(const String &p_comment, List::Element *p_pos) { + // Peek forward until we hit a newline or the end of the file (EOF). + TokenE *start_pos = p_pos ? p_pos : get_pos(); + if (!start_pos) { + return false; + } + while (start_pos->get().type != TT::TK_NEWLINE && start_pos->get().type != TT::TK_EOF) { + start_pos = start_pos->next(); + } + String comment = "/* !convert: " + p_comment + " */"; + if (start_pos->prev() && start_pos->prev()->get().type == TT::TK_BLOCK_COMMENT && get_token_literal_text(start_pos->prev()->get()) == comment) { + return true; + } + return insert_before(mkTok(TT::TK_BLOCK_COMMENT, comment), start_pos); +} + +void ShaderDeprecatedConverter::reset() { + ShaderLanguage sl; + code_tokens.clear(); + sl.token_debug_stream(old_code, code_tokens, true); + code_tokens.push_back(eof_token); + code_tokens.push_front(eof_token); + uniform_decls.clear(); + var_decls.clear(); + function_decls.clear(); + scope_declarations.clear(); + after_shader_decl = code_tokens.front(); + curr_ptr = code_tokens.front(); +} + +#define COND_MSG_FAIL(m_cond, m_msg) \ + if (unlikely(m_cond)) { \ + err_str = m_msg; \ + return false; \ + } +#define COND_LINE_MSG_FAIL(m_cond, m_line, m_msg) \ + if (unlikely(m_cond)) { \ + err_line = m_line + 1; \ + err_str = m_msg; \ + return false; \ + } +#define LINE_MSG_FAIL(m_line, m_msg) \ + err_line = m_line + 1; \ + err_str = m_msg; \ + return false; +#define MSG_FAIL(m_msg) \ + err_str = m_msg; \ + return false; + +#define EOF_FAIL(m_tok_E) \ + COND_MSG_FAIL(m_tok_E == nullptr, RTR("Unexpected end of file")); \ + COND_LINE_MSG_FAIL(m_tok_E->get().type == TT::TK_EOF || m_tok_E->get().type == TT::TK_ERROR, m_tok_E->get().line, m_tok_E->get().type == TT::TK_ERROR ? vformat(RTR("Parser Error (%s) ", m_tok_E->get().text)) : RTR("Unexpected end of file")); +#define CLOSURE_FAIL(m_tok_E) \ + COND_MSG_FAIL(m_tok_E == nullptr, RTR("Unexpected end of file")); \ + if (unlikely(m_tok_E->get().type == TT::TK_EOF || m_tok_E->get().type == TT::TK_ERROR)) { \ + return false; \ + } + +// At uniform statement. +bool ShaderDeprecatedConverter::_parse_uniform() { + UniformDecl uni; + uni.start_pos = get_pos(); + DEV_ASSERT(uni.start_pos && uni.start_pos->get().type == TT::TK_UNIFORM); + uni.uniform_stmt_pos = uni.start_pos; + if (SL::is_token_uniform_qual(peek_prev_tk_type())) { // 3.x doesn't support these. + uni.start_pos = get_prev_token(); + get_next_token(); // Back to the uniform. + } + TokenE *next_tk = get_next_token(); + EOF_FAIL(next_tk); + while (SL::is_token_precision(next_tk->get().type) || SL::is_token_interpolation(next_tk->get().type)) { + if (SL::is_token_interpolation(next_tk->get().type)) { // Interpolations are not supported for uniforms in newer versions of Godot. + uni.interp_qual_pos = next_tk; + } + next_tk = get_next_token(); + EOF_FAIL(next_tk); + } + COND_LINE_MSG_FAIL(!token_is_type(next_tk->get()), next_tk->get().line, RTR("Expected type after 'uniform'")); + uni.type_pos = next_tk; + next_tk = get_next_token(); + EOF_FAIL(next_tk); + if (next_tk->get().type == TT::TK_BRACKET_OPEN) { + uni.is_array = true; + if (!_skip_array_size()) { + return false; + } + next_tk = get_next_token(); + EOF_FAIL(next_tk); + } + COND_LINE_MSG_FAIL(!tokentype_is_identifier(next_tk->get().type), next_tk->get().line, RTR("Expected identifier after uniform type")); + String name = get_token_literal_text(next_tk->get()); + uni.name_pos = next_tk; + next_tk = get_next_token(); + EOF_FAIL(next_tk); + if (next_tk->get().type == TT::TK_BRACKET_OPEN) { + uni.is_array = true; + if (!_skip_array_size()) { + return false; + } + next_tk = get_next_token(); + EOF_FAIL(next_tk); + } + if (next_tk->get().type == TT::TK_COLON) { + while (true) { + next_tk = get_next_token(); + EOF_FAIL(next_tk); + COND_LINE_MSG_FAIL(!token_is_hint(next_tk->get()), next_tk->get().line, RTR("Expected hint after ':' in uniform declaration")); + uni.hint_poses.push_back(next_tk); + next_tk = get_next_token(); + EOF_FAIL(next_tk); + if (next_tk->get().type == TT::TK_PARENTHESIS_OPEN) { + next_tk = _get_end_of_closure(); + CLOSURE_FAIL(next_tk); + COND_LINE_MSG_FAIL(next_tk->get().type != TT::TK_PARENTHESIS_CLOSE, next_tk->get().line, RTR("Expected ')' after hint range")); + reset_to(next_tk); // Skip the hint range. + next_tk = get_next_token(); + EOF_FAIL(next_tk); + } + if (next_tk->get().type != TT::TK_COMMA) { + break; + } + } + } + if (next_tk->get().type == TT::TK_OP_ASSIGN) { + next_tk = _get_end_of_closure(); + CLOSURE_FAIL(next_tk); + reset_to(next_tk); // Skip the assignment. + next_tk = get_next_token(); + } + uni.end_pos = next_tk; + EOF_FAIL(uni.end_pos); + COND_LINE_MSG_FAIL(uni.end_pos->get().type != TT::TK_SEMICOLON, uni.end_pos->get().line, RTR("Expected ';' after uniform declaration")); + uniform_decls[name] = uni; + return true; +} + +bool ShaderDeprecatedConverter::_skip_uniform() { + TokenE *cur_tok = get_pos(); + DEV_ASSERT(cur_tok && cur_tok->get().type == TT::TK_UNIFORM); + for (KeyValue &kv : uniform_decls) { + if (kv.value.uniform_stmt_pos == cur_tok) { + reset_to(kv.value.end_pos); + return true; + } + } + LINE_MSG_FAIL(cur_tok->get().line, RTR("Uniform declaration not found")); +} + +bool ShaderDeprecatedConverter::_skip_array_size() { + TokenE *next_tk = get_pos(); + DEV_ASSERT(next_tk && next_tk->get().type == TT::TK_BRACKET_OPEN); + next_tk = _get_end_of_closure(); + CLOSURE_FAIL(next_tk); + COND_LINE_MSG_FAIL(next_tk->get().type != TT::TK_BRACKET_CLOSE, next_tk->get().line, RTR("Expected ']' after array type")); + reset_to(next_tk); // Skip to end. + return true; +} + +bool ShaderDeprecatedConverter::_skip_struct() { + DEV_ASSERT(get_pos() && get_pos()->get().type == TT::TK_STRUCT); + TokenE *struct_name = get_next_token(); + EOF_FAIL(struct_name); + TokenE *struct_body_start; + if (struct_name->get().type == TT::TK_CURLY_BRACKET_OPEN) { + struct_body_start = struct_name; + } else { + struct_body_start = get_next_token(); + } + EOF_FAIL(struct_body_start); + COND_LINE_MSG_FAIL(struct_body_start->get().type != TT::TK_CURLY_BRACKET_OPEN, struct_body_start->get().line, RTR("Expected '{' after struct declaration")); + TokenE *struct_body_end = _get_end_of_closure(); + CLOSURE_FAIL(struct_body_end); + COND_LINE_MSG_FAIL(struct_body_end->get().type != TT::TK_CURLY_BRACKET_CLOSE, struct_body_start->get().line, RTR("Expected '}' bracket at end of struct declaration")); + reset_to(struct_body_end); + if (tokentype_is_identifier(peek_next_tk_type())) { + get_next_token(); + } + return true; +} + +bool ShaderDeprecatedConverter::_tok_is_start_of_decl(const Token &p_tk) { + return token_is_type(p_tk) || p_tk.type == TT::TK_CONST || p_tk.type == TT::TK_VARYING || SL::is_token_precision(p_tk.type) || SL::is_token_interpolation(p_tk.type); +} + +// Past the start and type tokens, at the id or bracket open token. +bool ShaderDeprecatedConverter::_process_decl_statement(TokenE *p_start_tok, TokenE *p_type_tok, const String &p_scope, bool p_func_args) { + while (true) { + EOF_FAIL(p_start_tok); + EOF_FAIL(p_type_tok); + COND_LINE_MSG_FAIL(!token_is_type(p_type_tok->get()), p_type_tok->get().line, RTR("Expected type in declaration")); + TokenE *next_tk = get_pos(); + VarDecl var; + var.start_pos = p_start_tok; + var.type_pos = p_type_tok; + var.is_func_arg = p_func_args; + EOF_FAIL(next_tk); + if (next_tk->get().type == TT::TK_BRACKET_OPEN) { + var.is_array = true; + var.new_arr_style_decl = true; + if (!_skip_array_size()) { + return false; + } + next_tk = get_next_token(); + EOF_FAIL(next_tk); + } + COND_LINE_MSG_FAIL(!tokentype_is_identifier(next_tk->get().type), next_tk->get().line, RTR("Expected identifier after type in declaration")); + var.name_pos = next_tk; + String name = get_token_literal_text(var.name_pos->get()); + next_tk = get_next_token(); + EOF_FAIL(next_tk); + TokenE *end_pos = next_tk; + if (next_tk->get().type == TT::TK_BRACKET_OPEN) { + var.is_array = true; + if (!_skip_array_size()) { + return false; + } + end_pos = get_next_token(); + next_tk = end_pos; + EOF_FAIL(next_tk); + } + if (next_tk->get().type == TT::TK_OP_ASSIGN) { + end_pos = _get_end_of_closure(); + CLOSURE_FAIL(end_pos); + reset_to(end_pos); // Skip the assignment. + if (end_pos->get().type == TT::TK_PARENTHESIS_CLOSE && p_func_args) { + next_tk = end_pos; + end_pos = end_pos->prev(); // Including whitespace before parenthesis. + } else { + next_tk = get_next_token(); // comma or semi-colon + EOF_FAIL(next_tk); + end_pos = next_tk; + } + } + var.end_pos = end_pos; + COND_LINE_MSG_FAIL(p_func_args && !(next_tk->get().type == TT::TK_COMMA || next_tk->get().type == TT::TK_PARENTHESIS_CLOSE), next_tk->get().line, RTR("Expected ',' , or ')' after function argument declaration")); + COND_LINE_MSG_FAIL(!p_func_args && !(next_tk->get().type == TT::TK_SEMICOLON || next_tk->get().type == TT::TK_COMMA), next_tk->get().line, RTR("Expected ',' or ';' after declaration")); + if (var_decls.has(name)) { + var_decls[name].push_back(var); + } else { + var_decls[name] = { var }; + } + if (!scope_declarations.has(p_scope)) { + scope_declarations[p_scope] = HashSet(); + } + scope_declarations[p_scope].insert(name); + if (next_tk->get().type == TT::TK_COMMA) { + next_tk = get_next_token(); + EOF_FAIL(next_tk); + p_start_tok = next_tk; + if (p_func_args) { + while (next_tk->get().type == TT::TK_CONST || + SL::is_token_precision(next_tk->get().type) || + SL::is_token_arg_qual(next_tk->get().type) || + SL::is_token_interpolation(next_tk->get().type)) { + next_tk = get_next_token(); + EOF_FAIL(next_tk); + } + p_type_tok = next_tk; // next_tk is type + COND_LINE_MSG_FAIL(!token_is_type(p_type_tok->get()), p_type_tok->get().line, RTR("Expected type after comma in function argument declaration")); + next_tk = get_next_token(); // id + EOF_FAIL(next_tk); + } // otherwise, this is a compound declaration, leave type_tok as is + } else if (next_tk->get().type == TT::TK_PARENTHESIS_CLOSE) { + break; + } else if (next_tk->get().type == TT::TK_SEMICOLON) { + break; + } + } + return true; +}; + +// Past the start and type tokens, at the id or bracket open token. +bool ShaderDeprecatedConverter::_process_func_decl_statement(TokenE *p_start_tok, TokenE *p_type_tok, bool p_first_pass) { + FunctionDecl func; + func.start_pos = p_start_tok; // type or const + func.type_pos = p_type_tok; // type + TokenE *next_tk = get_pos(); // id or array size + if (next_tk->get().type == TT::TK_BRACKET_OPEN) { + func.has_array_return_type = true; + if (!_skip_array_size()) { + return false; + } + next_tk = get_next_token(); + EOF_FAIL(next_tk); + } + func.name_pos = next_tk; // id + String name = get_token_literal_text(func.name_pos->get()); + func.args_start_pos = get_next_token(); // paren + EOF_FAIL(func.args_start_pos); + if (peek_next_tk_type() == TT::TK_PARENTHESIS_CLOSE) { + func.args_end_pos = get_next_token(); + func.arg_count = 0; + } else { // Args are present. + func.args_end_pos = _get_end_of_closure(); + CLOSURE_FAIL(func.args_end_pos); + COND_LINE_MSG_FAIL(func.args_end_pos->get().type != TT::TK_PARENTHESIS_CLOSE, func.args_end_pos->get().line, RTR("Expected ')' after function arguments")); + if (!p_first_pass) { // first_pass == false means we've already parsed the args. + // Skip the args. + reset_to(func.args_end_pos); + } else { + TokenE *start_pos = get_next_token(); + TokenE *type_pos = start_pos; + while (type_pos->get().type == TT::TK_CONST || SL::is_token_precision(type_pos->get().type) || SL::is_token_arg_qual(type_pos->get().type) || SL::is_token_interpolation(type_pos->get().type)) { + type_pos = get_next_token(); + EOF_FAIL(type_pos); + } + COND_LINE_MSG_FAIL(!token_is_type(type_pos->get()), type_pos->get().line, RTR("Expected type in function argument declaration")); + get_next_token(); // id or open bracket + int var_count = var_decls.size(); + if (!_process_decl_statement(start_pos, type_pos, name, true)) { + return false; + } + COND_LINE_MSG_FAIL(get_pos() != func.args_end_pos, get_pos()->get().line, RTR("Expected ')' after function arguments")); + func.arg_count = var_decls.size() - var_count; + } + } + // Currently at paren close. + func.body_start_pos = get_next_token(); // Curly open. + EOF_FAIL(func.body_start_pos); + COND_LINE_MSG_FAIL(func.body_start_pos->get().type != TT::TK_CURLY_BRACKET_OPEN, func.body_start_pos->get().line, RTR("Expected '{' after function declaration")); + func.body_end_pos = _get_end_of_closure(); + CLOSURE_FAIL(func.body_end_pos); + COND_LINE_MSG_FAIL(func.body_end_pos->get().type != TT::TK_CURLY_BRACKET_CLOSE, func.body_start_pos->get().line, RTR("Expected '}' bracket")); + if (p_first_pass) { // p_first_pass == false means the functions have already been processed. + function_decls[name] = func; +#ifdef DEBUG_ENABLED + } else { + if (!function_decls.has(name)) { + LINE_MSG_FAIL(func.start_pos->get().line, vformat(RTR("Function declaration not found in third pass (%s)"), name)); + } else { + // Compare our values to ensure they match. + FunctionDecl &first_pass = function_decls[name]; + // Don't check arg count, as it's not set in the second pass. + bool matches = first_pass.start_pos == func.start_pos && first_pass.type_pos == func.type_pos && first_pass.name_pos == func.name_pos && first_pass.args_start_pos == func.args_start_pos && first_pass.args_end_pos == func.args_end_pos && first_pass.body_start_pos == func.body_start_pos && first_pass.body_end_pos == func.body_end_pos; + COND_LINE_MSG_FAIL(!matches, func.start_pos->get().line, vformat(RTR("Function declaration mismatch in third pass (%s)"), name)); + } +#endif + } + return true; +} + +bool ShaderDeprecatedConverter::_parse_decls(bool p_first_pass) { + reset_to(after_shader_decl); + String curr_func = ""; + while (true) { + TokenE *cur_tok = get_next_token(); + if (cur_tok->get().type == TT::TK_EOF) { + break; + } + + if (!p_first_pass) { + for (KeyValue &E : function_decls) { + FunctionDecl &func = E.value; + if (cur_tok == func.args_start_pos) { + curr_func = E.key; + } else if (cur_tok == func.body_end_pos) { + curr_func = ""; + } + } + } + if (cur_tok->get().type == TT::TK_STRUCT) { + if (!_skip_struct()) { + return false; + } + continue; + } + if (cur_tok->get().type == TT::TK_UNIFORM) { + if (!_skip_uniform()) { + return false; + } + continue; + } + TokenE *start_pos = cur_tok; + if (!_tok_is_start_of_decl(cur_tok->get())) { + continue; + } + while (_tok_is_start_of_decl(cur_tok->get())) { + if (token_is_type(cur_tok->get())) { + break; + } + cur_tok = get_next_token(); + EOF_FAIL(cur_tok); + } + ERR_CONTINUE(!token_is_type(cur_tok->get())); + TokenE *type_pos = cur_tok; + + bool is_decl = tokentype_is_identifier(peek_next_tk_type()); + bool is_function = peek_next_tk_type(2) == TT::TK_PARENTHESIS_OPEN; + if (!is_decl) { + // Check if this is an array declaration. + TokenE *next_tk = get_next_token(); + if (next_tk->get().type == TT::TK_BRACKET_OPEN) { + if (!_skip_array_size()) { + return false; + } + next_tk = get_pos(); + EOF_FAIL(next_tk); + COND_LINE_MSG_FAIL(next_tk->get().type != TT::TK_BRACKET_CLOSE, next_tk->get().line, RTR("Expected ']' after array type")); + TokenE *next_next_tk = get_next_token(); + if (next_next_tk && next_next_tk->get().type == TT::TK_IDENTIFIER) { + is_decl = true; + if (peek_next_tk_type() == TT::TK_PARENTHESIS_OPEN) { + is_function = true; + } else { + is_function = false; + } + } + } + reset_to(cur_tok); // Backup to the Bracket open. + } + COND_LINE_MSG_FAIL(is_function && curr_func != "", cur_tok->get().line, RTR("Unexpected function declaration")); + if (!is_decl) { + continue; + } + TokenE *id_tok = get_next_token(); // Id or bracket open. + EOF_FAIL(id_tok); + if (is_function) { // Function declaration. + if (!_process_func_decl_statement(start_pos, type_pos, p_first_pass)) { + return false; + } + // Backup to before the curly bracket open. + get_prev_token(); + } else if (!p_first_pass) { // Other non-uniform declaration (global const, varying, locals, etc.). + if (!_process_decl_statement(start_pos, type_pos, curr_func)) { + return false; + } + } + } + return true; +} + +bool ShaderDeprecatedConverter::_parse_uniforms() { + reset_to(after_shader_decl); + while (true) { + TokenE *cur_tok = get_next_token(); + if (cur_tok->get().type == TT::TK_EOF) { + break; + } + switch (cur_tok->get().type) { + case TT::TK_UNIFORM: { + if (!_parse_uniform()) { + return false; + } + } break; + default: + break; + } + } + return true; +} + +bool ShaderDeprecatedConverter::_preprocess_code() { + COND_MSG_FAIL(code_tokens.size() == 0, RTR("Empty shader file")); + StringName mode_string; + { + COND_MSG_FAIL(code_tokens.size() < 3, RTR("Invalid shader file")); + TokenE *first_token = get_next_token(); + EOF_FAIL(first_token); + COND_LINE_MSG_FAIL(first_token->get().type != TT::TK_SHADER_TYPE, first_token->get().line, RTR("Shader type must be first token")); + TokenE *id_token = get_next_token(); + EOF_FAIL(id_token); + COND_LINE_MSG_FAIL(id_token->get().type != TT::TK_IDENTIFIER, id_token->get().line, RTR("Invalid shader type")); + mode_string = id_token->get().text; + TokenE *token = get_next_token(); + EOF_FAIL(token); + COND_LINE_MSG_FAIL(token->get().type != TT::TK_SEMICOLON, token->get().line, RTR("Expected semi-colon after shader type")); + shader_mode = get_shader_mode_from_string(mode_string); + } + after_shader_decl = get_pos(); + + /*** + * The first pass gets the uniform declarations; we require this is to ensure idempotency for inserting new uniforms and replacing type hints. + * The second pass gets the function declarations; these are used for determining if a renamed built-in is valid in the current scope. + * The third pass gets the variable declarations; These are used for determining if renamed built-ins have been previously declared, and for detecting new keywords used as identifiers. + */ + // first pass, get uniform declarations. + if (!_parse_uniforms()) { + err_str = vformat(RTR("First pre-process pass failed: %s"), err_str); + curr_ptr = code_tokens.front(); + return false; + } + // Second pass, get function declarations. + if (!_parse_decls(true)) { + function_pass_failed = true; + err_str = vformat(RTR("Second pre-process pass failed: %s"), err_str); + curr_ptr = code_tokens.front(); + return false; + } + // Third pass, get variable declarations. + if (!_parse_decls(false)) { + var_pass_failed = true; + err_str = vformat(RTR("Third pre-process pass failed: %s"), err_str); + curr_ptr = code_tokens.front(); + return false; + } + curr_ptr = code_tokens.front(); + return true; +} + +int ShaderDeprecatedConverter::get_error_line() const { + return err_line; +} + +bool ShaderDeprecatedConverter::is_code_deprecated(const String &p_code) { + // Quick check to see if it's a shader file with a deprecated type. + String mode_str = SL::get_shader_type(p_code); + if (mode_str.is_empty()) { + // If it failed, it's because it was prefixed with a preproc directive (4.x only) or it's not a shader file. + return false; + } + RS::ShaderMode mode = get_shader_mode_from_string(mode_str); + if (mode == RS::SHADER_MAX) { + return false; + } + old_code = p_code; + reset(); + if (_has_any_preprocessor_directives()) { + return false; + } + if (!_preprocess_code()) { + return false; + } + return _is_code_deprecated(); +} + +bool ShaderDeprecatedConverter::_has_any_preprocessor_directives() { + TokenE *cur_tok = code_tokens.front(); + while (cur_tok) { + if (cur_tok->get().type == TT::TK_PREPROC_DIRECTIVE) { + return true; + } + cur_tok = cur_tok->next(); + } + return false; +} + +bool ShaderDeprecatedConverter::_is_code_deprecated() { + reset_to(after_shader_decl); + + // Negative cases first, then positive cases. + + // Check declarations for negative cases. + for (const KeyValue &E : uniform_decls) { + const UniformDecl &uni = E.value; + if (uni.is_array) { // 3.x did not have array uniforms. + return false; + } else if (tokentype_is_new_type(uni.type_pos->get().type)) { // Usage of new type. + return false; + } else if (uni.has_uniform_qual()) { // 3.x did not have uniform qualifiers. + return false; + } + for (const TokenE *hint : uni.hint_poses) { + if (tokentype_is_new_hint(hint->get().type)) { // Usage of new hint. + return false; + } + } + } + + for (const KeyValue &E : function_decls) { + const FunctionDecl &func = E.value; + if (func.has_array_return_type) { // 3.x did not have array return types. + return false; + } else if (tokentype_is_new_type(func.type_pos->get().type)) { // Usage of new type. + return false; + } else if (func.is_new_main_function(shader_mode)) { // Has the process function with the same signature. + return false; + } + } + + for (const KeyValue> &E : var_decls) { + for (const VarDecl &var_decl : E.value) { + if (var_decl.is_array && var_decl.is_func_arg) { // 3.x did not allow array function arguments. + return false; + } else if (var_decl.new_arr_style_decl) { // 3.x did not have the `float[] x` style of array declarations. + return false; + } else if (tokentype_is_new_type(var_decl.type_pos->get().type)) { // Usage of new type. + return false; + } + } + } + + // Check token stream for negative cases. + { + reset_to(after_shader_decl); + String curr_func = ""; + while (true) { + TokenE *cur_tok = get_next_token(); + DEV_ASSERT(cur_tok); + if (cur_tok->get().type == TT::TK_EOF || cur_tok->get().type == TT::TK_ERROR) { + break; + } + for (KeyValue &E : function_decls) { + FunctionDecl &func = E.value; + if (cur_tok == func.args_start_pos) { + curr_func = E.key; + break; + } else if (cur_tok == func.body_end_pos) { + curr_func = ""; + break; + } + } + if (cur_tok->get().type == TT::TK_STRUCT) { + if (!_skip_struct()) { + return false; + } + continue; + } + String id = get_token_literal_text(cur_tok->get()); + if (tokentype_is_new_type(cur_tok->get().type) && peek_next_tk_type() == TT::TK_IDENTIFIER) { // Declaration of an identifier with a new type. + return false; + } else if (cur_tok->get().type == TT::TK_IDENTIFIER) { + if (has_builtin_rename(shader_mode, id, curr_func) || is_removed_builtin(shader_mode, id, curr_func)) { + if (scope_has_decl(curr_func, id) || function_decls.has(id)) { + // The renamed built-ins are global identifiers in 3.x and can't be redefined in either the global scope or the function scope they're valid for. + // If they were declared previously within the global or current scope, this would be a 4.x shader. + return false; + } + } else if (id_is_new_builtin_func(id) && peek_next_tk_type() == TT::TK_PARENTHESIS_OPEN && !function_decls.has(id)) { // Use of a new built-in function without a corresponding declaration. + return false; + } + } else if (tokentype_is_new_reserved_keyword(cur_tok->get().type) && (!scope_has_decl(curr_func, id) && !function_decls.has(id))) { // Use of new keyword without corresponding declaration. + return false; + } + } + } + + // Positive cases. + + // Check declarations for positive cases. + for (const KeyValue &E : uniform_decls) { + const UniformDecl &uni = E.value; + if (uni.type_pos->get().type == TT::TK_IDENTIFIER && has_removed_type(get_token_literal_text(uni.type_pos->get()))) { // Unported 3.x type. + return true; + } else if (tokentype_is_new_reserved_keyword(uni.name_pos->get().type)) { // Uniform name is a new reserved keyword. + return true; + } else if (uni.has_interp_qual()) { // Newer versions of Godot disallow interpolation qualifiers for uniforms. + return true; + } + for (const TokenE *hint : uni.hint_poses) { + if (hint->get().type == TT::TK_IDENTIFIER && has_hint_replacement(get_token_literal_text(hint->get()))) { + return true; + } + } + } + + for (const KeyValue &E : function_decls) { + const FunctionDecl &func = E.value; + String name = get_token_literal_text(func.name_pos->get()); + String type_name = get_token_literal_text(func.type_pos->get()); + if (func.type_pos->get().type == TT::TK_IDENTIFIER && has_removed_type(type_name)) { // Unported 3.x type. + return true; + } else if (func.is_renamed_main_function(shader_mode)) { // Matching renamed function. + return true; + } else if (tokentype_is_new_reserved_keyword(func.name_pos->get().type)) { // Function identifier is new reserved keyword. + return true; + } else if (id_is_new_builtin_func(name)) { // Declaration of function with the same name as a new built-in function. + return true; + } + } + + for (const KeyValue> &E : var_decls) { + for (const VarDecl &var_decl : E.value) { + if (var_decl.type_pos->get().type == TT::TK_IDENTIFIER && has_removed_type(get_token_literal_text(var_decl.type_pos->get()))) { // Unported 3.x type. + return true; + } else if (tokentype_is_new_reserved_keyword(var_decl.name_pos->get().type)) { // Id is new reserved keyword. + return true; + } + } + } + + bool is_3x = false; + String curr_func = ""; + reset_to(after_shader_decl); + // Check token stream for positive cases. + while (true) { + TokenE *cur_tok = get_next_token(); + if (cur_tok->get().type == TT::TK_EOF || cur_tok->get().type == TT::TK_ERROR) { + break; + } + + for (KeyValue &E : function_decls) { + FunctionDecl &func = E.value; + if (cur_tok == func.body_start_pos) { + curr_func = E.key; + break; + } else if (cur_tok == func.body_end_pos) { + curr_func = ""; + break; + } + } + if (cur_tok->get().type == TT::TK_STRUCT) { + if (!_skip_struct()) { + return false; + } + continue; + } + + switch (cur_tok->get().type) { + case TT::TK_FLOAT_CONSTANT: { + String const_str = get_token_literal_text(cur_tok->get()); + // 3.x float constants allowed for a value without a decimal point if it ended in `f` (e.g. `1f`). + if (const_str.ends_with("f") && const_str.find(".") == -1 && const_str.find("e") == -1) { + return true; + } + } break; + case TT::TK_RENDER_MODE: { + while (true) { + TokenE *next_tk = get_next_token(); + if (next_tk->get().type == TT::TK_IDENTIFIER) { + String id_text = get_token_literal_text(next_tk->get()); + if (is_renamed_render_mode(shader_mode, id_text) || has_removed_render_mode(shader_mode, id_text)) { + return true; + } + } else { + COND_LINE_MSG_FAIL(next_tk->get().type != TT::TK_COMMA && next_tk->get().type != TT::TK_SEMICOLON, next_tk->get().line, "Invalid render mode declaration"); + } + if (next_tk->get().type == TT::TK_SEMICOLON) { + break; + } + } + } break; + case TT::TK_IDENTIFIER: { + String id = get_token_literal_text(cur_tok->get()); + if (has_builtin_rename(shader_mode, id, curr_func) || is_removed_builtin(shader_mode, id, curr_func)) { + if ((!scope_has_decl(curr_func, id) && !function_decls.has(id))) { + is_3x = true; + return true; + // Do not stop checking in this case; the third pass may have failed and the decls may be incomplete. + } + } else if (has_removed_type(id) && peek_next_tk_type() == TT::TK_IDENTIFIER) { + // Declaration with unported 3.x type. + return true; + } + } break; + default: + break; + } + } + return is_3x; +} + +String ShaderDeprecatedConverter::get_error_text() const { + return err_str; +} + +bool ShaderDeprecatedConverter::_check_deprecated_type(TokenE *p_type_pos) { + if (p_type_pos->get().type == TT::TK_IDENTIFIER && has_removed_type(get_token_literal_text(p_type_pos->get()))) { + const String i_err_msg = vformat(RTR("Deprecated type '%s' is not supported by this version of Godot."), get_token_literal_text(p_type_pos->get())); + COND_LINE_MSG_FAIL(fail_on_unported, p_type_pos->get().line, i_err_msg); + _add_warning_comment_before(i_err_msg, p_type_pos); + } + return true; +} + +bool ShaderDeprecatedConverter::_handle_new_keyword_rename(TokenType p_tk_type, const String &p_name, bool p_detected_3x, HashMap &p_new_reserved_word_renames) { + if (tokentype_is_new_reserved_keyword(p_tk_type)) { + if (!p_detected_3x) { + // If we're not sure it's a 3.x shader, just add a comment. + _add_warning_comment_before(vformat(RTR("Identifier '%s' is a reserved word in this version of Godot."), p_name), get_pos()); + return false; + } + if (!p_new_reserved_word_renames.has(p_tk_type)) { + String rename = p_name + String("_"); + while (function_decls.has(rename) || uniform_decls.has(rename) || var_decls.has(rename)) { + rename += "_"; + } + p_new_reserved_word_renames[p_tk_type] = rename; + } + return true; + } + return false; +} + +bool ShaderDeprecatedConverter::convert_code(const String &p_code) { + /** + * We need to do the following: + * * Replace everything in RenamesMap3To4::shaders_renames + * * the usage of SCREEN_TEXTURE, DEPTH_TEXTURE, and NORMAL_ROUGHNESS_TEXTURE necessitates adding a uniform declaration at the top of the file + * * async_visible and async_hidden render modes need to be removed + * * If shader_type is "particles", need to rename the function "void vertex()" to "void process()" + * * Invert all usages of CLEARCOAT_GLOSS: + * * Invert all lefthand assignments: + * - `CLEARCOAT_GLOSS = 5.0 / foo;` + * becomes: `CLEARCOAT_ROUGHNESS = (1.0 - (5.0 / foo));`, + * - `CLEARCOAT_GLOSS *= 1.1;` + * becomes `CLEARCOAT_ROUGHNESS = (1.0 - ((1.0 - CLEARCOAT_ROUGHNESS) * 1.1));` + * * Invert all righthand usages + * - `foo = CLEARCOAT_GLOSS;` + * becomes: `foo = (1.0 - CLEARCOAT_ROUGHNESS);` + * * Wrap `INDEX` in `int()` casts if necessary. + * * Check for use of `specular_blinn` and `specular_phong` render modes; not supported in 4.x, throw an error. + * * Check for use of `MODULATE`; not supported in 4.x, throw an error. + * * Check for use of unported `samplerExternalOES` 3.x type; not supported in 4.x, throw an error. + * * Check for use of new reserved keywords as identifiers; rename them if necessary. + * * Check for use of new built-in functions with a corresponding declaration; rename them if necessary. + */ + old_code = p_code; + reset(); + if (_has_any_preprocessor_directives()) { // We refuse to process any shader with preprocessor directives, as they're not supported in 3.x and they make our parsing assumptions invalid. + err_str = RTR("Cannot convert new shader with pre-processor directives."); + return false; + } + if (!_preprocess_code()) { + return false; + } + bool detected_3x = _is_code_deprecated(); // Calls preprocess_code(). + if (!detected_3x && err_str != "") { + return false; + } // We don't fail if the code is detected as not deprecated, as the user may have forced it; instead, we just avoid doing the more dicey replacements, like renaming new keywords. + COND_MSG_FAIL(shader_mode == RS::SHADER_MAX, RTR("Detected Shader type is not a 3.x type.")); // However, we do fail if it's a new shader type, because we don't do any replacements for those. + err_str = ""; + curr_ptr = after_shader_decl; + + // Renaming changed hints. + Vector all_hints; + for (KeyValue &E : uniform_decls) { + UniformDecl &uni = E.value; + if (uni.has_interp_qual()) { // Removing interpolation qualifiers before the type name, which was allowed in 3.x. + reset_to(uni.interp_qual_pos); + remove_cur_and_get_next(); + uni.interp_qual_pos = nullptr; + reset_to(after_shader_decl); + } + String name = get_token_literal_text(uni.name_pos->get()); + for (int i = 0; i < uni.hint_poses.size(); i++) { + TokenE *hint = uni.hint_poses[i]; + String hint_name = get_token_literal_text(hint->get()); + if (hint->get().type == TT::TK_IDENTIFIER && has_hint_replacement(hint_name)) { + // replace the hint + reset_to(hint); + hint = replace_curr(mkTok(get_hint_replacement(hint_name))); + uni.hint_poses.write[i] = hint; + reset_to(after_shader_decl); + } + all_hints.push_back(hint); + } + } + + // Renaming new reserved keywords used as identifiers (e.g "global", "instance"). + // To ensure idempotency, we only do this if we know for certain that the new keyword was used in a declaration. + HashMap new_reserved_word_renames; + HashMap func_renames; + HashMap nonfunc_globals_renames; // Only used if a function is renamed and an existing global conflicts with the rename. + + for (KeyValue &E : uniform_decls) { + UniformDecl &uni = E.value; + if (!_check_deprecated_type(uni.type_pos)) { + return false; + } + + TokenType type = uni.name_pos->get().type; + if (_handle_new_keyword_rename(type, get_token_literal_text(uni.name_pos->get()), detected_3x, new_reserved_word_renames)) { + reset_to(uni.name_pos); + uni.name_pos = replace_curr(mkTok(TT::TK_IDENTIFIER, new_reserved_word_renames[type])); + reset_to(after_shader_decl); + } + } + for (KeyValue> &E : var_decls) { + if (E.value.is_empty()) { + continue; + } + // Check for deprecated type. + for (int i = 0; i < E.value.size(); i++) { + if (!_check_deprecated_type(E.value[i].type_pos)) { + return false; + } + } + + TT name_tok_type = E.value[0].name_pos->get().type; + String name = get_token_literal_text(E.value[0].name_pos->get()); + if (_handle_new_keyword_rename(name_tok_type, name, detected_3x, new_reserved_word_renames)) { + for (int i = 0; i < E.value.size(); i++) { + // replace the identifier + + reset_to(E.value[i].name_pos); + if (E.value[i].name_pos == E.value[i].start_pos) { + E.value.write[i].name_pos = replace_curr(mkTok(TT::TK_IDENTIFIER, new_reserved_word_renames[name_tok_type])); + E.value.write[i].start_pos = E.value.write[i].name_pos; + } else { + E.value.write[i].name_pos = replace_curr(mkTok(TT::TK_IDENTIFIER, new_reserved_word_renames[name_tok_type])); + } + reset_to(after_shader_decl); + } + } + } + bool has_new_main_function = false; + Vector new_main_function_names; + for (KeyValue &E : function_decls) { + if (E.value.is_new_main_function(shader_mode)) { + has_new_main_function = true; + break; + } + } + for (KeyValue &E : function_decls) { + FunctionDecl &var = E.value; + TT tok_type = var.name_pos->get().type; + if (!_check_deprecated_type(var.type_pos)) { + return false; + } + String name = get_token_literal_text(var.name_pos->get()); + if (!has_new_main_function && var.is_renamed_main_function(shader_mode)) { + // replace the function name + String rename = get_main_function_rename(name); + reset_to(var.name_pos); + var.name_pos = replace_curr(mkTok(TT::TK_IDENTIFIER, rename)); + reset_to(after_shader_decl); + func_renames[name] = rename; + // Only doing this because "process" is a common word and we don't want to clobber an existing function/global named that. + // Won't clobber a pre-exising "process" function that has the correct main signature because of the check before. + if (function_decls.has(rename) || scope_has_decl("", rename)) { + String rerename = rename + String("_"); + while (function_decls.has(rerename) || uniform_decls.has(rerename) || var_decls.has(rerename)) { + rerename += "_"; + } + if (function_decls.has(rename)) { + func_renames[rename] = rerename; + FunctionDecl &rere_func = function_decls[rename]; + reset_to(rere_func.name_pos); + rere_func.name_pos = replace_curr(mkTok(TT::TK_IDENTIFIER, rerename)); + reset_to(after_shader_decl); + } else if (uniform_decls.has(rename)) { + nonfunc_globals_renames[rename] = rerename; + UniformDecl &rere_uni = uniform_decls[rename]; + reset_to(rere_uni.name_pos); + rere_uni.name_pos = replace_curr(mkTok(TT::TK_IDENTIFIER, rerename)); + reset_to(after_shader_decl); + } else if (var_decls.has(rename) && scope_declarations.has("") && scope_declarations[""].has(rename)) { + nonfunc_globals_renames[rename] = rerename; + for (int i = 0; i < var_decls[rename].size(); i++) { + VarDecl &rere_var = var_decls[rename].write[i]; + reset_to(rere_var.name_pos); + rere_var.name_pos = replace_curr(mkTok(TT::TK_IDENTIFIER, rerename)); + reset_to(after_shader_decl); + } + } + } + } else if (id_is_new_builtin_func(name)) { + if (!detected_3x) { // If we're not sure it's a 3.x shader, just add a comment. + _add_warning_comment_before(vformat(RTR("Function '%s' is a built-in function in this version of Godot."), name), var.start_pos); + } else { + String rename = name + String("_"); + while (function_decls.has(rename) || uniform_decls.has(rename) || var_decls.has(rename)) { + rename += "_"; + } + func_renames[name] = rename; + // replace the function name + reset_to(var.name_pos); + var.name_pos = replace_curr(mkTok(TT::TK_IDENTIFIER, rename)); + reset_to(after_shader_decl); + } + } else if (_handle_new_keyword_rename(tok_type, name, detected_3x, new_reserved_word_renames)) { + reset_to(var.name_pos); + var.name_pos = replace_curr(mkTok(TT::TK_IDENTIFIER, new_reserved_word_renames[tok_type])); + reset_to(after_shader_decl); + } + } + bool in_function = false; + String curr_func = ""; + reset_to(after_shader_decl); + static Vector uniform_qualifiers = { "global", "instance" }; + while (true) { + TokenE *cur_tok = get_next_token(); + if (cur_tok->get().type == TT::TK_EOF) { + break; + } + for (KeyValue &E : function_decls) { + FunctionDecl &func = E.value; + if (cur_tok == func.args_start_pos) { + in_function = true; + curr_func = E.key; // The key is the ORIGINAL function name, not the potentially renamed one. + } else if (in_function && cur_tok == func.body_end_pos) { + in_function = false; + curr_func = ""; + } + } + if (cur_tok->get().type == TT::TK_STRUCT) { + if (!_skip_struct()) { + return false; + } + continue; + } + String cur_tok_text = get_token_literal_text(cur_tok->get()); + if (cur_tok->get().pos != NEW_IDENT && new_reserved_word_renames.has(cur_tok->get().type)) { + if (!(scope_has_decl(curr_func, cur_tok_text) || (function_decls.has(cur_tok_text) && peek_next_tk_type() == TT::TK_PARENTHESIS_OPEN))) { + continue; // Don't replace if this rename is not in the current scope or is not a renamed function call. + } + // Just extra insurance against replacing legit new keywords. + if (uniform_qualifiers.has(cur_tok_text)) { + if (peek_next_tk_type() == TT::TK_UNIFORM) { + continue; // Don't replace uniform qualifiers. + } + } else if (all_hints.has(cur_tok)) { + continue; // Hint, don't replace it. + } else if (peek_prev_tk_type() == TT::TK_PERIOD) { + continue; // Struct member access, don't replace it. + } + cur_tok = replace_curr(mkTok(TT::TK_IDENTIFIER, new_reserved_word_renames[cur_tok->get().type])); + continue; + } + switch (cur_tok->get().type) { + case TT::TK_FLOAT_CONSTANT: { + // Earlier versions of Godot 3.x (< 3.5) allowed the use of the `f` sigil with float constants without a decimal place. + if (cur_tok_text.ends_with("f") && !cur_tok_text.contains(".") && !cur_tok_text.contains("e")) { + cur_tok_text = cur_tok_text.substr(0, cur_tok_text.length() - 1) + ".0f"; + cur_tok = replace_curr(mkTok(TT::TK_FLOAT_CONSTANT, cur_tok_text, 0xdeadbeef)); + } + } break; + case TT::TK_RENDER_MODE: { + // we only care about the ones for spatial + if (shader_mode == RenderingServer::ShaderMode::SHADER_SPATIAL) { + while (true) { + TokenE *next_tk = get_next_token(); + if (next_tk->get().type == TT::TK_IDENTIFIER) { + String id_text = get_token_literal_text(next_tk->get()); + if (has_removed_render_mode(shader_mode, id_text)) { + if (!can_remove_render_mode(id_text)) { + const String i_err_msg = vformat(RTR("Deprecated render mode '%s' is not supported by this version of Godot."), id_text); + COND_LINE_MSG_FAIL(fail_on_unported, next_tk->get().line, i_err_msg); + _add_warning_comment_before(i_err_msg, next_tk); + } else { + if (peek_next_tk_type() == TT::TK_COMMA) { + TokenE *comma = get_next_token(); + reset_to(next_tk); // Reset to the identifier. + EOF_FAIL(comma->next()); + next_tk = _remove_from_curr_to(comma->next()); // Inclusive of comma. + } else if (peek_prev_tk_type() == TT::TK_COMMA && peek_next_tk_type() == TT::TK_SEMICOLON) { + TokenE *end = get_next_token(); + reset_to(next_tk); // Back to identifier. + next_tk = get_prev_token(); // comma + next_tk = _remove_from_curr_to(end); // Exclusive of semi-colon. + break; // We're at the end of the render_mode declaration. + } else if (peek_prev_tk_type() == TT::TK_RENDER_MODE && peek_next_tk_type() == TT::TK_SEMICOLON) { + // Remove the whole line. + TokenE *semi = get_next_token(); + COND_LINE_MSG_FAIL(!semi->next(), semi->get().line, "Unexpected EOF???"); // We should always have an EOF token at the end of the stream. + reset_to(next_tk); // Back to identifier. + next_tk = get_prev_token(); // render_mode + next_tk = _remove_from_curr_to(semi->next()); // Inclusive of semi-colon. + break; + } else { + // we shouldn't be here + LINE_MSG_FAIL(next_tk->get().line, RTR("Unexpected token after render mode declaration.")); + } + } + } else if (is_renamed_render_mode(shader_mode, id_text)) { + next_tk = replace_curr(mkTok(TT::TK_IDENTIFIER, get_render_mode_rename(id_text))); + } + } else { + COND_LINE_MSG_FAIL(next_tk->get().type != TT::TK_COMMA && next_tk->get().type != TT::TK_SEMICOLON, next_tk->get().line, RTR("Expected ',' or ';' after render mode declaration.")); + } + if (next_tk->get().type == TT::TK_SEMICOLON) { + break; + } + } + } + } break; + case TT::TK_IDENTIFIER: { + if (cur_tok->get().pos == NEW_IDENT) { // Skip already-replaced identifiers. + break; + } + if (peek_prev_tk_type() == TT::TK_PERIOD) { + break; // Struct member access, don't replace it. + } + if (func_renames.has(cur_tok_text) && peek_next_tk_type() == TT::TK_PARENTHESIS_OPEN) { // Function call. + cur_tok = replace_curr(mkTok(TT::TK_IDENTIFIER, func_renames[cur_tok_text])); + } else if (nonfunc_globals_renames.has(cur_tok_text) && peek_next_tk_type() != TT::TK_PARENTHESIS_OPEN) { + cur_tok = replace_curr(mkTok(TT::TK_IDENTIFIER, nonfunc_globals_renames[cur_tok_text])); + } else if (is_removed_builtin(shader_mode, cur_tok_text, curr_func) && !scope_has_decl(curr_func, cur_tok_text)) { + if (get_removed_builtin_uniform_type(cur_tok_text) == TT::TK_ERROR) { + const String i_err_str = vformat(RTR("Deprecated built-in '%s' is not supported by this version of Godot"), cur_tok_text); + COND_LINE_MSG_FAIL(fail_on_unported, cur_tok->get().line, i_err_str); + _add_warning_comment_before(i_err_str, cur_tok); + } + COND_LINE_MSG_FAIL(!_insert_uniform_declaration(cur_tok_text), cur_tok->get().line, RTR("Failed to insert uniform declaration")); + UniformDecl &uniform_decl = uniform_decls[cur_tok_text]; + all_hints.append_array(uniform_decl.hint_poses); + } else if (cur_tok_text == "INDEX" && has_builtin_rename(shader_mode, cur_tok_text, curr_func) && !scope_has_decl(curr_func, cur_tok_text)) { + // INDEX was an int in 3.x, but is a uint in later versions. + // Need to wrap it in a `int()` cast. + // This is safe because this will only trigger if the `particles` function is "vertex" (which is renamed to "process"). + + // Don't do this if it's already wrapped in a int(), uint() or float(). + if (peek_prev_tk_type() == TT::TK_PARENTHESIS_OPEN && peek_next_tk_type() == TT::TK_PARENTHESIS_CLOSE) { + TT peeked_type = peek_prev_tk_type(2); + if (peeked_type == TT::TK_TYPE_INT || peeked_type == TT::TK_TYPE_UINT || peeked_type == TT::TK_TYPE_FLOAT) { + break; + } + } + insert_before({ mkTok(TT::TK_TYPE_INT), mkTok(TT::TK_PARENTHESIS_OPEN) }, cur_tok); + insert_after(mkTok(TT::TK_PARENTHESIS_CLOSE), cur_tok); + } else if (cur_tok_text == "CLEARCOAT_GLOSS" && has_builtin_rename(shader_mode, cur_tok_text, curr_func) && !scope_has_decl(curr_func, cur_tok_text)) { + cur_tok = replace_curr(mkTok(TT::TK_IDENTIFIER, "CLEARCOAT_ROUGHNESS")); + List::Element *assign_closure_end = nullptr; + switch (peek_next_tk_type()) { + case TT::TK_OP_ASSIGN: + case TT::TK_OP_ASSIGN_ADD: + case TT::TK_OP_ASSIGN_SUB: + case TT::TK_OP_ASSIGN_MUL: + case TT::TK_OP_ASSIGN_DIV: { + assign_closure_end = _get_end_of_closure(); + CLOSURE_FAIL(assign_closure_end); + + TokenE *assign_tk = get_next_token(); + TokenE *insert_pos = assign_tk; + if (assign_tk->next() && assign_tk->next()->get().type == TT::TK_SPACE) { + insert_pos = assign_tk->next(); + } + // " = (1.0 - (" + Vector assign_prefix = { + mkTok(TT::TK_OP_ASSIGN), + mkTok(TT::TK_SPACE), + mkTok(TT::TK_PARENTHESIS_OPEN), + mkTok(TT::TK_FLOAT_CONSTANT, {}, 1.0), + mkTok(TT::TK_SPACE), + mkTok(TT::TK_OP_SUB), + mkTok(TT::TK_SPACE), + mkTok(TT::TK_PARENTHESIS_OPEN), + }; + if (assign_tk->get().type != TT::TK_OP_ASSIGN) { + // " = (1.0 - ((1.0 - CLEARCOAT_ROUGHNESS) {op} + assign_prefix.append_array( + { mkTok(TT::TK_PARENTHESIS_OPEN), + mkTok(TT::TK_FLOAT_CONSTANT, {}, 1.0), + mkTok(TT::TK_SPACE), + mkTok(TT::TK_OP_SUB), + mkTok(TT::TK_SPACE), + mkTok(TT::TK_IDENTIFIER, "CLEARCOAT_ROUGHNESS"), + mkTok(TT::TK_PARENTHESIS_CLOSE), + mkTok(TT::TK_SPACE) }); + } + switch (assign_tk->get().type) { + case TT::TK_OP_ASSIGN_ADD: { + assign_prefix.append_array({ mkTok(TT::TK_OP_ADD), mkTok(TT::TK_SPACE) }); + } break; + case TT::TK_OP_ASSIGN_SUB: { + assign_prefix.append_array({ mkTok(TT::TK_OP_SUB), mkTok(TT::TK_SPACE) }); + } break; + case TT::TK_OP_ASSIGN_MUL: { + assign_prefix.append_array({ mkTok(TT::TK_OP_MUL), mkTok(TT::TK_SPACE) }); + } break; + case TT::TK_OP_ASSIGN_DIV: { + assign_prefix.append_array({ mkTok(TT::TK_OP_DIV), mkTok(TT::TK_SPACE) }); + } break; + default: + break; + } + insert_after(assign_prefix, insert_pos); + + // remove the assignment token + if (assign_tk != insert_pos && insert_pos->next()) { + // remove the extraneous space too if necessary + _remove_from_curr_to(insert_pos->next()); // Exclusive of the token after the space + } else { + remove_cur_and_get_next(); + } + // "))" + insert_after({ mkTok(TT::TK_PARENTHESIS_CLOSE), mkTok(TT::TK_PARENTHESIS_CLOSE) }, assign_closure_end); + reset_to(cur_tok); + + } break; + + default: + break; + } + + // Check for right-hand usage: if previous token is anything but a `{`, `}` or `;`. + if (peek_prev_tk_type() == TT::TK_SEMICOLON || + peek_prev_tk_type() == TT::TK_CURLY_BRACKET_OPEN || + peek_prev_tk_type() == TT::TK_CURLY_BRACKET_CLOSE) { + break; + } + + // Invert right-hand usage. + Vector right_hand_prefix = { // "(1.0 - ("; + mkTok(TT::TK_PARENTHESIS_OPEN), + mkTok(TT::TK_FLOAT_CONSTANT, {}, 1.0), + mkTok(TT::TK_SPACE), + mkTok(TT::TK_OP_SUB), + mkTok(TT::TK_SPACE) + }; + if (assign_closure_end) { + right_hand_prefix.append_array({ mkTok(TT::TK_PARENTHESIS_OPEN) }); + insert_after({ mkTok(TT::TK_PARENTHESIS_CLOSE), mkTok(TT::TK_PARENTHESIS_CLOSE) }, assign_closure_end); + } else { + insert_after(mkTok(TT::TK_PARENTHESIS_CLOSE), cur_tok); + } + insert_before(right_hand_prefix, cur_tok); + } else if (has_builtin_rename(shader_mode, cur_tok_text, curr_func) && !scope_has_decl(curr_func, cur_tok_text)) { + cur_tok = replace_curr(mkTok(TT::TK_IDENTIFIER, get_builtin_rename(cur_tok_text))); + } + } break; // End of identifier case. + case TT::TK_ERROR: { + LINE_MSG_FAIL(cur_tok->get().line, "Parser error ( " + cur_tok->get().text + ")"); + } break; + default: + break; + } + } + return true; +} + +String ShaderDeprecatedConverter::emit_code() const { + if (code_tokens.size() == 0) { + return ""; + } + String new_code = ""; + const TokenE *start = code_tokens.front()->next(); // skip TK_EOF token at start + for (const TokenE *E = start; E; E = E->next()) { + const Token &tk = E->get(); + ERR_FAIL_COND_V(tk.type < 0 || tk.type > TT::TK_MAX, ""); + bool end = false; + String tok_text = get_token_literal_text(tk); + switch (tk.type) { + case TT::TK_ERROR: + case TT::TK_EOF: { + end = true; + } break; + case TT::TK_BLOCK_COMMENT: { + if (tk.pos == NEW_IDENT) { + if (warning_comments && tok_text.contains("!convert WARNING:")) { + new_code += tok_text; + } else if (verbose_comments && tok_text.contains("!convert")) { + new_code += tok_text; + } else { + break; + } + } else { + new_code += tok_text; + } + } break; + default: { + new_code += tok_text; + } break; + } + if (end) { + break; + } + } + + return new_code; +} + +void ShaderDeprecatedConverter::set_warning_comments(bool p_add_comments) { + warning_comments = p_add_comments; +} +void ShaderDeprecatedConverter::set_fail_on_unported(bool p_fail_on_unported) { + fail_on_unported = p_fail_on_unported; +} +void ShaderDeprecatedConverter::set_verbose_comments(bool p_verbose_comments) { + verbose_comments = p_verbose_comments; +} + +#endif // DISABLE_DEPRECATED diff --git a/servers/rendering/shader_converter.h b/servers/rendering/shader_converter.h new file mode 100644 index 00000000000..7ed390e5c05 --- /dev/null +++ b/servers/rendering/shader_converter.h @@ -0,0 +1,284 @@ +/**************************************************************************/ +/* shader_converter.h */ +/**************************************************************************/ +/* This file is part of: */ +/* GODOT ENGINE */ +/* https://godotengine.org */ +/**************************************************************************/ +/* Copyright (c) 2014-present Godot Engine contributors (see AUTHORS.md). */ +/* Copyright (c) 2007-2014 Juan Linietsky, Ariel Manzur. */ +/* */ +/* Permission is hereby granted, free of charge, to any person obtaining */ +/* a copy of this software and associated documentation files (the */ +/* "Software"), to deal in the Software without restriction, including */ +/* without limitation the rights to use, copy, modify, merge, publish, */ +/* distribute, sublicense, and/or sell copies of the Software, and to */ +/* permit persons to whom the Software is furnished to do so, subject to */ +/* the following conditions: */ +/* */ +/* The above copyright notice and this permission notice shall be */ +/* included in all copies or substantial portions of the Software. */ +/* */ +/* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, */ +/* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF */ +/* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. */ +/* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY */ +/* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, */ +/* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE */ +/* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ +/**************************************************************************/ + +#ifndef SHADER_CONVERTER_H +#define SHADER_CONVERTER_H + +#ifndef DISABLE_DEPRECATED + +#include "core/templates/pair.h" +#include "servers/rendering/shader_language.h" +#include "servers/rendering_server.h" + +class ShaderDeprecatedConverter { +public: + using TokenType = ShaderLanguage::TokenType; + using Token = ShaderLanguage::Token; + using TT = TokenType; + using TokenE = List::Element; + + ShaderDeprecatedConverter() {}; + bool is_code_deprecated(const String &p_code); + bool convert_code(const String &p_code); + String get_error_text() const; + int get_error_line() const; + String emit_code() const; + void set_warning_comments(bool p_add_comments); + void set_fail_on_unported(bool p_fail_on_unported); + void set_assume_correct(bool p_assume_correct); + void set_force_reserved_word_replacement(bool p_force_reserved_word_replacement); + void set_verbose_comments(bool p_verbose_comments); + + static bool tokentype_is_identifier(const TokenType &p_tk_type); + static bool tokentype_is_new_reserved_keyword(const TokenType &p_tk_type); + static bool tokentype_is_new_type(const TokenType &p_tk_type); + static bool tokentype_is_new_hint(const TokenType &p_tk); + + static bool id_is_new_builtin_func(const String &p_name); + + static String get_tokentype_text(TokenType p_tk_type); + + static bool has_builtin_rename(RS::ShaderMode p_mode, const String &p_name, const String &p_function = ""); + static String get_builtin_rename(const String &p_name); + + static bool has_hint_replacement(const String &p_name); + static TokenType get_hint_replacement(const String &p_name); + + static bool is_renamed_render_mode(RS::ShaderMode p_mode, const String &p_name); + static String get_render_mode_rename(const String &p_name); + + static bool has_removed_render_mode(RS::ShaderMode p_mode, const String &p_name); + static bool can_remove_render_mode(const String &p_name); + + static bool has_removed_type(const String &p_name); + + static bool is_renamed_main_function(RS::ShaderMode p_mode, const String &p_name); + static bool is_renamee_main_function(RS::ShaderMode p_mode, const String &p_name); + static String get_main_function_rename(const String &p_name); + static TokenType get_renamed_function_type(const String &p_name); + static int get_renamed_function_arg_count(const String &p_name); + + static bool is_removed_builtin(RS::ShaderMode p_mode, const String &p_name, const String &p_function = ""); + static TokenType get_removed_builtin_uniform_type(const String &p_name); + static Vector get_removed_builtin_hints(const String &p_name); + + static bool _rename_has_special_handling(const String &p_name); + + static void _get_builtin_renames_list(List *r_list); + static void _get_render_mode_renames_list(List *r_list); + static void _get_hint_renames_list(List *r_list); + static void _get_function_renames_list(List *r_list); + static void _get_render_mode_removals_list(List *r_list); + static void _get_builtin_removals_list(List *r_list); + static void _get_type_removals_list(List *r_list); + static void _get_new_builtin_funcs_list(List *r_list); + static Vector _get_funcs_builtin_rename(RS::ShaderMode p_mode, const String &p_name); + static Vector _get_funcs_builtin_removal(RS::ShaderMode p_mode, const String &p_name); + + struct RenamedBuiltins { + const char *name; + const char *replacement; + const Vector>> mode_functions; + const bool special_handling; + }; + + struct RenamedRenderModes { + const RS::ShaderMode mode; + const char *name; + const char *replacement; + }; + + struct RenamedHints { + const char *name; + const ShaderLanguage::TokenType replacement; + }; + + struct RenamedFunctions { + const RS::ShaderMode mode; + const ShaderLanguage::TokenType type; + const int arg_count; + const char *name; + const char *replacement; + }; + + struct RemovedRenderModes { + const RS::ShaderMode mode; + const char *name; + const bool can_remove; + }; + + struct RemovedBuiltins { + const char *name; + const ShaderLanguage::TokenType uniform_type; + const Vector hints; + const Vector>> mode_functions; + }; + +private: + struct UniformDecl { + List::Element *start_pos = nullptr; + List::Element *uniform_stmt_pos = nullptr; + List::Element *end_pos = nullptr; + List::Element *interp_qual_pos = nullptr; + List::Element *type_pos = nullptr; + List::Element *name_pos = nullptr; + Vector::Element *> hint_poses; + + bool is_array = false; + bool has_uniform_qual() const { + return start_pos != nullptr && ShaderLanguage::is_token_uniform_qual(start_pos->get().type); + } + bool has_interp_qual() const { + return interp_qual_pos != nullptr; + } + }; + struct VarDecl { + List::Element *start_pos = nullptr; + List::Element *end_pos = nullptr; // semicolon or comma or right paren + List::Element *type_pos = nullptr; + List::Element *name_pos = nullptr; + bool is_array = false; + bool new_arr_style_decl = false; + bool is_func_arg = false; + void clear() { + start_pos = nullptr; + end_pos = nullptr; + type_pos = nullptr; + name_pos = nullptr; + } + }; + + struct FunctionDecl { + List::Element *start_pos = nullptr; + List::Element *type_pos = nullptr; + List::Element *name_pos = nullptr; + List::Element *args_start_pos = nullptr; // left paren + List::Element *args_end_pos = nullptr; // right paren + List::Element *body_start_pos = nullptr; // left curly + List::Element *body_end_pos = nullptr; // right curly - end of function + bool is_renamed_main_function(RS::ShaderMode p_mode) const; + bool is_new_main_function(RS::ShaderMode p_mode) const; + + int arg_count = 0; + bool has_array_return_type = false; + void clear() { + type_pos = nullptr; + name_pos = nullptr; + args_start_pos = nullptr; + args_end_pos = nullptr; + body_start_pos = nullptr; + body_end_pos = nullptr; + } + }; + static const RenamedBuiltins renamed_builtins[]; + static const RenamedRenderModes renamed_render_modes[]; + static const RenamedHints renamed_hints[]; + static const RenamedFunctions renamed_functions[]; + static const RemovedRenderModes removed_render_modes[]; + static const RemovedBuiltins removed_builtins[]; + static const char *removed_types[]; + static const char *old_builtin_funcs[]; + static HashSet _new_builtin_funcs; + String old_code; + List code_tokens; + List::Element *curr_ptr = nullptr; + List::Element *after_shader_decl = nullptr; + HashMap uniform_decls; + HashMap> var_decls; + HashMap function_decls; + HashMap> scope_declarations; + RenderingServer::ShaderMode shader_mode = RenderingServer::ShaderMode::SHADER_MAX; + + bool warning_comments = true; + bool verbose_comments = false; + bool fail_on_unported = true; + + bool function_pass_failed = false; + bool var_pass_failed = false; + String err_str; + int err_line = 0; + + Token eof_token{ ShaderLanguage::TK_EOF, {}, 0, 0, 0, 0 }; + + static RS::ShaderMode get_shader_mode_from_string(const String &p_mode); + + String get_token_literal_text(const Token &p_tk) const; + static Token mkTok(TokenType p_type, const StringName &p_text = StringName(), double constant = 0, uint16_t p_line = 0); + static bool token_is_skippable(const Token &p_tk); + static bool token_is_type(const Token &p_tk); + static bool token_is_hint(const Token &p_tk); + + void reset(); + bool _preprocess_code(); + List::Element *get_next_token(); + List::Element *get_prev_token(); + List::Element *remove_cur_and_get_next(); + TokenType peek_next_tk_type(uint32_t p_count = 1) const; + TokenType peek_prev_tk_type(uint32_t p_count = 1) const; + List::Element *get_pos() const; + bool reset_to(List::Element *p_pos); + bool insert_after(const Vector &p_token_list, List::Element *p_pos); + bool insert_before(const Vector &p_token_list, List::Element *p_pos); + bool insert_after(const Token &p_token, List::Element *p_pos); + bool insert_before(const Token &p_token, List::Element *p_pos); + List::Element *replace_curr(const Token &p_token); + List::Element *_get_next_token_ptr(List::Element *p_curr_ptr) const; + List::Element *_get_prev_token_ptr(List::Element *p_curr_ptr) const; + TokenType _peek_tk_type(int64_t p_count, List::Element **r_pos = nullptr) const; + + bool scope_has_decl(const String &p_scope, const String &p_name) const; + bool _handle_new_keyword_rename(TokenType p_tk_type, const String &p_name, bool p_detected_3x, HashMap &p_func_renames); + + bool _has_any_preprocessor_directives(); + bool _is_code_deprecated(); + bool _parse_uniform(); + static bool _tok_is_start_of_decl(const Token &p_tk); + bool _skip_uniform(); + bool _parse_uniforms(); + bool _skip_array_size(); + bool _skip_struct(); + bool _check_deprecated_type(TokenE *p_type_tok); + bool _add_warning_comment_before(const String &p_comment, List::Element *p_pos); + bool _add_comment_at_eol(const String &p_comment, List::Element *p_pos); + bool _process_func_decl_statement(TokenE *p_start_tok, TokenE *p_type_tok, bool p_second_pass = false); + bool _process_decl_statement(TokenE *p_start_tok, TokenE *p_type_tok, const String &p_scope = "", bool p_func_args = false); + bool _parse_decls(bool p_first_pass); + bool _insert_uniform_declaration(const String &p_name); + List::Element *_remove_from_curr_to(List::Element *p_end); + List::Element *_get_end_of_closure(); + static HashSet _construct_new_builtin_funcs(); + + enum { + NEW_IDENT = -1 + }; +}; +#endif // DISABLE_DEPRECATED + +#endif // SHADER_CONVERTER_H diff --git a/servers/rendering/shader_language.cpp b/servers/rendering/shader_language.cpp index 879a83f5198..ec05f5dc5bf 100644 --- a/servers/rendering/shader_language.cpp +++ b/servers/rendering/shader_language.cpp @@ -230,6 +230,13 @@ const char *ShaderLanguage::token_names[TK_MAX] = { "CURSOR", "ERROR", "EOF", + "TAB", + "CR", + "SPACE", + "NEWLINE", + "BLOCK_COMMENT", + "LINE_COMMENT", + "PREPROC_DIRECTIVE", }; String ShaderLanguage::get_token_text(Token p_token) { @@ -250,6 +257,8 @@ ShaderLanguage::Token ShaderLanguage::_make_token(TokenType p_type, const String tk.type = p_type; tk.text = p_text; tk.line = tk_line; + tk.pos = tk_start_pos; + tk.length = char_idx - tk_start_pos; if (tk.type == TK_ERROR) { _set_error(p_text); } @@ -402,8 +411,11 @@ const ShaderLanguage::KeyWord ShaderLanguage::keyword_list[] = { ShaderLanguage::Token ShaderLanguage::_get_token() { #define GETCHAR(m_idx) (((char_idx + m_idx) < code.length()) ? code[char_idx + m_idx] : char32_t(0)) - +#define IF_DBG_MK_TK(m_type) \ + if (unlikely(debug_parse)) \ + return _make_token(m_type); while (true) { + tk_start_pos = char_idx; char_idx++; switch (GETCHAR(-1)) { case 0: @@ -411,11 +423,17 @@ ShaderLanguage::Token ShaderLanguage::_get_token() { case 0xFFFF: return _make_token(TK_CURSOR); //for completion case '\t': + IF_DBG_MK_TK(TK_TAB); + continue; case '\r': + IF_DBG_MK_TK(TK_CR); + continue; case ' ': + IF_DBG_MK_TK(TK_SPACE); continue; case '\n': tk_line++; + IF_DBG_MK_TK(TK_NEWLINE); continue; case '/': { switch (GETCHAR(0)) { @@ -424,10 +442,12 @@ ShaderLanguage::Token ShaderLanguage::_get_token() { char_idx++; while (true) { if (GETCHAR(0) == 0) { + IF_DBG_MK_TK(TK_BLOCK_COMMENT); return _make_token(TK_EOF); } if (GETCHAR(0) == '*' && GETCHAR(1) == '/') { char_idx += 2; + IF_DBG_MK_TK(TK_BLOCK_COMMENT); break; } else if (GETCHAR(0) == '\n') { tk_line++; @@ -441,11 +461,13 @@ ShaderLanguage::Token ShaderLanguage::_get_token() { while (true) { if (GETCHAR(0) == '\n') { + IF_DBG_MK_TK(TK_LINE_COMMENT); tk_line++; char_idx++; break; } if (GETCHAR(0) == 0) { + IF_DBG_MK_TK(TK_LINE_COMMENT); return _make_token(TK_EOF); } char_idx++; @@ -702,6 +724,23 @@ ShaderLanguage::Token ShaderLanguage::_get_token() { return _make_token(TK_ERROR, "Invalid include enter/exit hint token (@@> and @@<)"); } } break; + case '#': { + if (!debug_parse) { // We shouldn't get here if the preprocessor is enabled and doing a non-debug parse. + return _make_token(TK_ERROR, "Unexpected pre-processor directive (Is the pre-processor enabled?)"); + } + while (true) { + char32_t c = GETCHAR(0); + if (c == '\\' && GETCHAR(1) == '\n') { + char_idx += 2; + tk_line++; + continue; + } + if (c == '\n' || c == 0) { + return _make_token(TK_PREPROC_DIRECTIVE); + } + char_idx++; + } + } break; default: { char_idx--; //go back one, since we have no idea what this is @@ -767,7 +806,7 @@ ShaderLanguage::Token ShaderLanguage::_get_token() { lut_case = CASE_EXPONENT; } else if (symbol == 'f' && !hexa_found) { if (!period_found && !exponent_found) { - error = true; + error = !debug_parse; } float_suffix_found = true; end_suffix_found = true; @@ -886,6 +925,8 @@ ShaderLanguage::Token ShaderLanguage::_get_token() { tk.constant = str.to_float(); } tk.line = tk_line; + tk.pos = tk_start_pos; + tk.length = char_idx - tk_start_pos; return tk; } @@ -934,6 +975,7 @@ ShaderLanguage::Token ShaderLanguage::_get_token() { return Token(); #undef GETCHAR +#undef IF_DBG_MK_TK } bool ShaderLanguage::_lookup_next(Token &r_tk) { @@ -956,22 +998,46 @@ ShaderLanguage::Token ShaderLanguage::_peek() { return tk; } -String ShaderLanguage::token_debug(const String &p_code) { +String ShaderLanguage::token_debug(const String &p_code, bool p_debug_parse) { clear(); code = p_code; + debug_parse = p_debug_parse; String output; - Token tk = _get_token(); + while (tk.type != TK_EOF && tk.type != TK_ERROR) { - output += itos(tk_line) + ": " + get_token_text(tk) + "\n"; + String literal_text = p_code.substr(tk.pos, tk.length); + output += itos(tk_line) + " (" + itos(tk.pos) + ":" + itos(tk.pos + tk.length) + "): " + get_token_text(tk) + " [" + literal_text; + String suffix = "]\n"; + // add error string if invalid float constant + if (debug_parse && tk.type == TK_FLOAT_CONSTANT && literal_text.ends_with("f") && !literal_text.contains(".") && !literal_text.contains("e")) { + output += " (Invalid float constant)]\n"; + } else { + output += "]\n"; + } tk = _get_token(); } + debug_parse = false; return output; } +void ShaderLanguage::token_debug_stream(const String &p_code, List &r_output, bool p_debug_parse) { + clear(); + + code = p_code; + debug_parse = p_debug_parse; + Token tk = _get_token(); + + while (tk.type != TK_EOF && tk.type != TK_ERROR) { + r_output.push_back(tk); + tk = _get_token(); + } + debug_parse = false; +} + bool ShaderLanguage::is_token_variable_datatype(TokenType p_type) { return ( p_type == TK_TYPE_VOID || @@ -1064,6 +1130,12 @@ bool ShaderLanguage::is_token_arg_qual(TokenType p_type) { p_type == TK_ARG_INOUT); } +bool ShaderLanguage::is_token_uniform_qual(TokenType p_type) { + return ( + p_type == TK_INSTANCE || + p_type == TK_GLOBAL); +} + ShaderLanguage::DataPrecision ShaderLanguage::get_token_precision(TokenType p_type) { if (p_type == TK_PRECISION_LOW) { return PRECISION_LOWP; @@ -4118,6 +4190,17 @@ bool ShaderLanguage::is_token_hint(TokenType p_type) { return int(p_type) > int(TK_RENDER_MODE) && int(p_type) < int(TK_SHADER_TYPE); } +bool ShaderLanguage::is_token_keyword(TokenType p_type) { + int idx = 0; + while (keyword_list[idx].text) { + if (keyword_list[idx].token == p_type) { + return true; + } + idx++; + } + return false; +} + bool ShaderLanguage::convert_constant(ConstantNode *p_constant, DataType p_to_type, Scalar *p_value) { if (p_constant->datatype == p_to_type) { if (p_value) { diff --git a/servers/rendering/shader_language.h b/servers/rendering/shader_language.h index 48df77f6bbf..10ac3dd4bcd 100644 --- a/servers/rendering/shader_language.h +++ b/servers/rendering/shader_language.h @@ -194,7 +194,15 @@ public: TK_CURSOR, TK_ERROR, TK_EOF, - TK_MAX + TK_TAB, // for debug purposes + TK_CR, + TK_SPACE, + TK_NEWLINE, + TK_BLOCK_COMMENT, + TK_LINE_COMMENT, + TK_PREPROC_DIRECTIVE, + TK_MAX, + TK_REG_MAX = TK_TAB, }; /* COMPILER */ @@ -788,6 +796,8 @@ public: StringName text; double constant; uint16_t line; + uint16_t length; + int32_t pos; bool is_integer_constant() const { return type == TK_INT_CONSTANT || type == TK_UINT_CONSTANT; } @@ -803,6 +813,7 @@ public: static DataInterpolation get_token_interpolation(TokenType p_type); static bool is_token_precision(TokenType p_type); static bool is_token_arg_qual(TokenType p_type); + static bool is_token_uniform_qual(TokenType p_type); static DataPrecision get_token_precision(TokenType p_type); static String get_precision_name(DataPrecision p_type); static String get_interpolation_name(DataInterpolation p_interpolation); @@ -814,6 +825,7 @@ public: static bool is_token_operator(TokenType p_type); static bool is_token_operator_assign(TokenType p_type); static bool is_token_hint(TokenType p_type); + static bool is_token_keyword(TokenType p_type); static bool convert_constant(ConstantNode *p_constant, DataType p_to_type, Scalar *p_value = nullptr); static DataType get_scalar_type(DataType p_type); @@ -1006,6 +1018,7 @@ private: String code; int char_idx = 0; int tk_line = 0; + int tk_start_pos = 0; StringName shader_type_identifier; StringName current_function; @@ -1140,6 +1153,8 @@ private: StringName completion_struct; int completion_argument = 0; + bool debug_parse = false; + #ifdef DEBUG_ENABLED uint32_t keyword_completion_context; #endif // DEBUG_ENABLED @@ -1222,7 +1237,10 @@ public: ShaderNode *get_shader(); - String token_debug(const String &p_code); + String token_debug(const String &p_code, bool p_debug_parse = false); + void token_debug_stream(const String &p_code, List &r_output, bool p_debug_parse); + + ShaderLanguage::Operator get_op(const TokenType &p_token) const; ShaderLanguage(); ~ShaderLanguage(); diff --git a/tests/servers/rendering/test_shader_converter.h b/tests/servers/rendering/test_shader_converter.h new file mode 100644 index 00000000000..0a1664919bf --- /dev/null +++ b/tests/servers/rendering/test_shader_converter.h @@ -0,0 +1,799 @@ +/**************************************************************************/ +/* test_shader_converter.h */ +/**************************************************************************/ +/* This file is part of: */ +/* GODOT ENGINE */ +/* https://godotengine.org */ +/**************************************************************************/ +/* Copyright (c) 2014-present Godot Engine contributors (see AUTHORS.md). */ +/* Copyright (c) 2007-2014 Juan Linietsky, Ariel Manzur. */ +/* */ +/* Permission is hereby granted, free of charge, to any person obtaining */ +/* a copy of this software and associated documentation files (the */ +/* "Software"), to deal in the Software without restriction, including */ +/* without limitation the rights to use, copy, modify, merge, publish, */ +/* distribute, sublicense, and/or sell copies of the Software, and to */ +/* permit persons to whom the Software is furnished to do so, subject to */ +/* the following conditions: */ +/* */ +/* The above copyright notice and this permission notice shall be */ +/* included in all copies or substantial portions of the Software. */ +/* */ +/* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, */ +/* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF */ +/* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. */ +/* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY */ +/* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, */ +/* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE */ +/* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ +/**************************************************************************/ + +#ifndef TEST_SHADER_CONVERTER_H +#define TEST_SHADER_CONVERTER_H + +#ifndef DISABLE_DEPRECATED +#include "servers/rendering/shader_converter.h" +#include "servers/rendering/shader_language.h" +#include "servers/rendering/shader_types.h" +#include "tests/test_macros.h" + +#include + +namespace TestShaderConverter { + +void erase_all_empty(Vector &p_vec) { + int idx = p_vec.find(" "); + while (idx >= 0) { + p_vec.remove_at(idx); + idx = p_vec.find(" "); + } +} + +bool is_variable_char(unsigned char c) { + return std::isalnum(c) || c == '_'; +} + +bool is_operator_char(unsigned char c) { + return (c == '*') || (c == '+') || (c == '-') || (c == '/') || ((c >= '<') && (c <= '>')); +} + +// Remove unnecessary spaces from a line. +String remove_spaces(String &p_str) { + String res; + // Result is guaranteed to not be longer than the input. + res.resize(p_str.size()); + int wp = 0; + char32_t last = 0; + bool has_removed = false; + + for (int n = 0; n < p_str.size(); n++) { + // These test cases only use ASCII. + unsigned char c = static_cast(p_str[n]); + if (std::isblank(c)) { + has_removed = true; + } else { + if (has_removed) { + // Insert a space to avoid joining things that could potentially form a new token. + // E.g. "float x" or "- -". + if ((is_variable_char(c) && is_variable_char(last)) || + (is_operator_char(c) && is_operator_char(last))) { + res[wp++] = ' '; + } + has_removed = false; + } + res[wp++] = c; + last = c; + } + } + res.resize(wp); + return res; +} + +// The pre-processor changes indentation and inserts spaces when inserting macros. +// Re-format the code, without changing its meaning, to make it easier to compare. +String compact_spaces(String &p_str) { + Vector lines = p_str.split("\n", false); + erase_all_empty(lines); + for (String &line : lines) { + line = remove_spaces(line); + } + return String("\n").join(lines); +} + +void get_keyword_set(HashSet &p_keywords) { + List keywords; + ShaderLanguage::get_keyword_list(&keywords); + for (const String &keyword : keywords) { + p_keywords.insert(keyword); + } +} + +#define CHECK_SHADER_EQ(a, b) CHECK_EQ(compact_spaces(a), compact_spaces(b)) +#define CHECK_SHADER_NE(a, b) CHECK_NE(compact_spaces(a), compact_spaces(b)) + +void get_compile_info(ShaderLanguage::ShaderCompileInfo &info, RenderingServer::ShaderMode p_mode) { + info.functions = ShaderTypes::get_singleton()->get_functions(p_mode); + info.render_modes = ShaderTypes::get_singleton()->get_modes(p_mode); + info.shader_types = ShaderTypes::get_singleton()->get_types(); + // Only used by editor for completion, so it's not important for these tests. + info.global_shader_uniform_type_func = [](const StringName &p_name) -> ShaderLanguage::DataType { + return ShaderLanguage::TYPE_SAMPLER2D; + }; +} + +RenderingServer::ShaderMode get_shader_mode(const String &p_mode_string) { + if (p_mode_string == "canvas_item") { + return RS::SHADER_CANVAS_ITEM; + } else if (p_mode_string == "particles") { + return RS::SHADER_PARTICLES; + } else if (p_mode_string == "spatial") { + return RS::SHADER_SPATIAL; + } else if (p_mode_string == "sky") { + return RS::SHADER_SKY; + } else if (p_mode_string == "fog") { + return RS::SHADER_FOG; + } else { + return RS::SHADER_MAX; + } +} + +String get_shader_mode_name(const RenderingServer::ShaderMode &p_mode_string) { + switch (p_mode_string) { + case RS::SHADER_CANVAS_ITEM: + return "canvas_item"; + case RS::SHADER_PARTICLES: + return "particles"; + case RS::SHADER_SPATIAL: + return "spatial"; + case RS::SHADER_SKY: + return "sky"; + case RS::SHADER_FOG: + return "fog"; + default: + return "unknown"; + } +} +using SL = ShaderLanguage; +using SDC = ShaderDeprecatedConverter; + +#define TEST_CONVERSION(m_old_code, m_expected, m_is_deprecated) \ + { \ + ShaderDeprecatedConverter _i_converter; \ + CHECK_EQ(_i_converter.is_code_deprecated(m_old_code), m_is_deprecated); \ + CHECK(_i_converter.convert_code(m_old_code)); \ + String _i_new_code = _i_converter.emit_code(); \ + CHECK_EQ(_i_new_code, m_expected); \ + } + +TEST_CASE("[ShaderDeprecatedConverter] Simple conversion with arrays") { + String code = "shader_type particles; void vertex() { float xy[2] = {1.0,1.1}; }"; + String expected = "shader_type particles; void process() { float xy[2] = {1.0,1.1}; }"; + TEST_CONVERSION(code, expected, true); +} + +TEST_CASE("[ShaderDeprecatedConverter] Test warning comments") { + // Test that warning comments are added when fail_on_unported is false and warning_comments is true + String code = "shader_type spatial;\nrender_mode specular_phong;"; + String expected = "shader_type spatial;\n/* !convert WARNING: Deprecated render mode 'specular_phong' is not supported by this version of Godot. */\nrender_mode specular_phong;"; + ShaderDeprecatedConverter converter; + CHECK(converter.is_code_deprecated(code)); + converter.set_warning_comments(true); + converter.set_fail_on_unported(false); + CHECK(converter.convert_code(code)); + String new_code = converter.emit_code(); + CHECK_EQ(new_code, expected); +} + +TEST_CASE("[ShaderDeprecatedConverter] Simple conversion with arrays") { + String code = "shader_type particles; struct foo{float bar;} void vertex() { float xy[2] = {1.0,1.1}; }"; + String expected = "shader_type particles; struct foo{float bar;} void process() { float xy[2] = {1.0,1.1}; }"; + TEST_CONVERSION(code, expected, true); +} + +TEST_CASE("[ShaderDeprecatedConverter] new-style array declaration") { + String code = "shader_type spatial; void vertex() { float[2] xy = {1.0,1.1}; }"; + // code should be the same + TEST_CONVERSION(code, code, false); +} + +TEST_CASE("[ShaderDeprecatedConverter] Simple conversion") { + String code = "shader_type particles; void vertex() { float x = 1.0; }"; + String expected = "shader_type particles; void process() { float x = 1.0; }"; + TEST_CONVERSION(code, expected, true); +} + +TEST_CASE("[ShaderDeprecatedConverter] Replace non-conformant float literals") { + String code = "shader_type spatial; const float x = 1f;"; + String expected = "shader_type spatial; const float x = 1.0f;"; + TEST_CONVERSION(code, expected, true); +} + +TEST_CASE("[ShaderDeprecatedConverter] particles::vertex() -> particles::process()") { + SUBCASE("basic") { + String code = "shader_type particles; void vertex() { float x = 1.0; }"; + String expected = "shader_type particles; void process() { float x = 1.0; }"; + TEST_CONVERSION(code, expected, true); + } + SUBCASE("with another function named `process` without correct signature") { + String code = "shader_type particles; void vertex() {} float process() { return 1.0; }"; + String expected = "shader_type particles; void process() {} float process_() { return 1.0; }"; + TEST_CONVERSION(code, expected, true); + } + SUBCASE("with another function named `process` with correct signature") { + String code = "shader_type particles; void vertex() {} void process() {}"; + // Should be unchanged. + TEST_CONVERSION(code, code, false); + } + + SUBCASE("with another function named `process` that is called") { + String code = "shader_type particles; float process() { return 1.0; } void vertex() { float foo = process(); }"; + String expected = "shader_type particles; float process_() { return 1.0; } void process() { float foo = process_(); }"; + TEST_CONVERSION(code, expected, true); + } + SUBCASE("with another function named `process` which calls `vertex`") { + String code = "shader_type particles; float process() {foo(); return 1.0;} void vertex() {} void foo() { vertex(); }"; + String expected = "shader_type particles; float process_() {foo(); return 1.0;} void process() {} void foo() { process(); }"; + TEST_CONVERSION(code, expected, true); + } + SUBCASE("No function named `vertex`") { + String code = "shader_type particles; void process() {}"; + // Should be unchanged. + TEST_CONVERSION(code, code, false); + } +} + +TEST_CASE("[ShaderDeprecatedConverter] CLEARCOAT_GLOSS -> CLEARCOAT_ROUGHNESS") { + SUBCASE("Left-hand simple assignment") { + String code("shader_type spatial; void fragment() {\n" + "CLEARCOAT_GLOSS = 1.0;\n" + "}\n"); + String expected("shader_type spatial; void fragment() {\n" + "CLEARCOAT_ROUGHNESS = (1.0 - (1.0));\n" + "}\n"); + TEST_CONVERSION(code, expected, true); + } + SUBCASE("Left-hand *= assignment") { + String code("shader_type spatial; void fragment() {\n" + "CLEARCOAT_GLOSS *= 0.5;\n" + "}\n"); + String expected("shader_type spatial; void fragment() {\n" + "CLEARCOAT_ROUGHNESS = (1.0 - ((1.0 - CLEARCOAT_ROUGHNESS) * 0.5));\n" + "}\n"); + TEST_CONVERSION(code, expected, true); + } + SUBCASE("Right-hand usage") { + String code("shader_type spatial; void fragment() {\n" + "float foo = CLEARCOAT_GLOSS;\n" + "}\n"); + String expected("shader_type spatial; void fragment() {\n" + "float foo = (1.0 - CLEARCOAT_ROUGHNESS);\n" + "}\n"); + TEST_CONVERSION(code, expected, true); + } + SUBCASE("both usages") { + String code("shader_type spatial; void fragment() {\n" + "float foo = (CLEARCOAT_GLOSS *= 0.5);\n" + "}\n"); + String expected("shader_type spatial; void fragment() {\n" + "float foo = ((1.0 - (CLEARCOAT_ROUGHNESS = (1.0 - ((1.0 - CLEARCOAT_ROUGHNESS) * 0.5)))));\n" + "}\n"); + TEST_CONVERSION(code, expected, true); + } +} +TEST_CASE("[ShaderDeprecatedConverter] Wrap INDEX in int()") { + SUBCASE("basic") { + String code("shader_type particles; void vertex() {\n" + "float foo = INDEX/2;\n" + "}\n"); + String expected("shader_type particles; void process() {\n" + "float foo = int(INDEX)/2;\n" + "}\n"); + + TEST_CONVERSION(code, expected, true); + } + SUBCASE("Without clobbering existing casts") { + String code("shader_type particles; void vertex() {\n" + "float foo = int(INDEX/2) * int(INDEX) * 2 * float(INDEX);\n" + "}\n"); + String expected("shader_type particles; void process() {\n" + "float foo = int(int(INDEX)/2) * int(INDEX) * 2 * float(INDEX);\n" + "}\n"); + TEST_CONVERSION(code, expected, true); + } +} + +TEST_CASE("[ShaderDeprecatedConverter] All hint renames") { + String code_template = "shader_type spatial; uniform sampler2D foo : %s;"; + // get all the hint renames + List hints; + ShaderDeprecatedConverter::_get_hint_renames_list(&hints); + + SUBCASE("No renamed hints present in current keyword list") { + HashSet keywords_set; + get_keyword_set(keywords_set); + for (const String &hint : hints) { + CHECK_FALSE(keywords_set.has(hint)); + } + } + + SUBCASE("All renamed hints are replaced") { + for (const String &hint : hints) { + ShaderDeprecatedConverter::TokenType type = ShaderDeprecatedConverter::get_hint_replacement(hint); + String rename = ShaderDeprecatedConverter::get_tokentype_text(type); + String code = vformat(code_template, hint); + String expected = vformat(code_template, rename); + TEST_CONVERSION(code, expected, true); + } + } +} + +TEST_CASE("[ShaderDeprecatedConverter] Built-in renames") { + // Get all the built-in renames. + List builtins; + ShaderDeprecatedConverter::_get_builtin_renames_list(&builtins); + // remove built-ins that have special handling, we test those above + for (List::Element *E = builtins.front(); E; E = E->next()) { + if (ShaderDeprecatedConverter::_rename_has_special_handling(E->get())) { + List::Element *prev = E->prev(); + builtins.erase(E); + E = prev; + } + } + Vector modes = { RS::SHADER_SPATIAL, RS::SHADER_CANVAS_ITEM, RS::SHADER_PARTICLES }; + HashMap>> rename_func_map; + for (RS::ShaderMode mode : modes) { + rename_func_map[mode] = HashMap>(); + for (const String &builtin : builtins) { + rename_func_map[mode][builtin] = ShaderDeprecatedConverter::_get_funcs_builtin_rename(mode, builtin); + } + } + + SUBCASE("All renamed built-ins are not currently built-in") { + for (RS::ShaderMode mode : modes) { + ShaderLanguage::ShaderCompileInfo info; + get_compile_info(info, mode); + for (const String &builtin : builtins) { + // Now get the funcs applicable for this mode and built-in. + for (const String &func : rename_func_map[mode][builtin]) { + // The built-in should not be present in the built-ins list. + auto &finfo = info.functions[func]; + if (finfo.built_ins.has(builtin)) { + WARN_PRINT(vformat("Renamed 3.x Built-in %s is present in function %s", builtin, func)); + } + CHECK_FALSE(finfo.built_ins.has(builtin)); + } + } + } + } + + SUBCASE("All renamed built-ins are replaced") { + String code_template = "shader_type %s; void %s() { %s; }"; + for (RS::ShaderMode mode : modes) { + for (const String &builtin : builtins) { + // Now get the funcs applicable for this mode and built-in. + String rename = ShaderDeprecatedConverter::get_builtin_rename(builtin); + for (const String &func : rename_func_map[mode][builtin]) { + String code = vformat(code_template, get_shader_mode_name(mode), func, builtin); + String expected = vformat(code_template, get_shader_mode_name(mode), func, rename); + TEST_CONVERSION(code, expected, true); + } + } + } + } + SUBCASE("No renaming built-ins in non-candidate functions") { + String code_template = "shader_type %s; void %s() { float %s = 1.0; %s += 1.0; }"; + for (RS::ShaderMode mode : modes) { + ShaderLanguage::ShaderCompileInfo info; + get_compile_info(info, mode); + for (const String &builtin : builtins) { + Vector non_funcs; + for (KeyValue &func : info.functions) { + if (func.key == "global") { + continue; + } + if (!rename_func_map[mode][builtin].has(func.key)) { + non_funcs.push_back(func.key); + } + } + String rename = ShaderDeprecatedConverter::get_builtin_rename(builtin); + for (const String &func : non_funcs) { + String code = vformat(code_template, get_shader_mode_name(mode), func, builtin, builtin); + // The code should not change. + TEST_CONVERSION(code, code, false); + } + } + } + } + SUBCASE("No renaming built-ins in candidate functions with built-in declared") { + String code_template = "shader_type %s; void %s() { float %s = 1.0; %s += 1.0; }"; + for (RS::ShaderMode mode : modes) { + for (const String &builtin : builtins) { + for (const String &func : rename_func_map[mode][builtin]) { + String code = vformat(code_template, get_shader_mode_name(mode), func, builtin, builtin); + // The code should not change. + TEST_CONVERSION(code, code, false); + } + } + } + } +} + +// TODO: Remove this when the MODULATE built-in PR lands. +// If this fails, remove the MODULATE entry from ShaderDeprecatedConverter::removed_builtins, then remove this test and the following test. +TEST_CASE("[ShaderDeprecatedConverter] MODULATE is not a built-in") { + ShaderLanguage::ShaderCompileInfo info; + get_compile_info(info, RS::ShaderMode::SHADER_CANVAS_ITEM); + SUBCASE("MODULATE is not a built-in") { + for (const String &func : Vector{ "vertex", "fragment", "light" }) { + auto &finfo = info.functions[func]; + CHECK_FALSE(finfo.built_ins.has("MODULATE")); + } + } +} + +// Don't remove this one if the above doesn't fail too. +TEST_CASE("[ShaderDeprecatedConverter] MODULATE handling") { + ShaderLanguage::ShaderCompileInfo info; + get_compile_info(info, RS::ShaderMode::SHADER_CANVAS_ITEM); + SUBCASE("Fails to compile") { + for (const String &func : Vector{ "vertex", "fragment", "light" }) { + String code = vformat("shader_type canvas_item; void %s() { MODULATE; }", func); + ShaderLanguage sl; + CHECK_NE(sl.compile(code, info), Error::OK); + } + } + SUBCASE("Fails to convert on fail_on_unported=true") { + for (const String &func : Vector{ "vertex", "fragment", "light" }) { + String code = vformat("shader_type canvas_item; void %s() { MODULATE; }", func); + ShaderDeprecatedConverter converter; + CHECK(converter.is_code_deprecated(code)); + converter.set_fail_on_unported(true); + CHECK_FALSE(converter.convert_code(code)); + } + } + + SUBCASE("Conversion succeeds on fail_on_unported=false") { + for (const String &func : Vector{ "vertex", "fragment", "light" }) { + String code = vformat("shader_type canvas_item; void %s() { MODULATE; }", func); + ShaderDeprecatedConverter converter; + CHECK(converter.is_code_deprecated(code)); + converter.set_fail_on_unported(false); + CHECK(converter.convert_code(code)); + String new_code = converter.emit_code(); + CHECK(new_code.find("/*") != -1); + } + } +} + +TEST_CASE("[ShaderDeprecatedConverter] Uniform declarations for removed builtins") { + // Test uniform declaration inserts for removed builtins for all shader types. + String code_template = "shader_type %s;%s void %s() { %s; }"; + String uniform_template = "\nuniform %s %s : %s;\n"; + // Get all the removed built-ins. + List builtins; + ShaderDeprecatedConverter::_get_builtin_removals_list(&builtins); + Vector modes = { RS::SHADER_SPATIAL, RS::SHADER_CANVAS_ITEM, RS::SHADER_PARTICLES }; + HashMap compiler_infos; + + SUBCASE("Removed built-ins are not currently built-in") { + for (RS::ShaderMode mode : modes) { + ShaderLanguage::ShaderCompileInfo info; + get_compile_info(info, mode); + for (const String &builtin : builtins) { + Vector funcs = ShaderDeprecatedConverter::_get_funcs_builtin_removal(mode, builtin); + for (const String &func : funcs) { + const ShaderLanguage::FunctionInfo &finfo = info.functions[func]; + if (finfo.built_ins.has(builtin)) { + WARN_PRINT(vformat("Removed 3.x Built-in %s is present in function %s", builtin, func)); + } + CHECK_FALSE(finfo.built_ins.has(builtin)); + } + } + } + } + + SUBCASE("All removed built-ins have uniform declarations") { + for (RS::ShaderMode mode : modes) { + for (const String &builtin : builtins) { + // now get the funcs applicable for this mode and builtins + ShaderLanguage::TokenType type = ShaderDeprecatedConverter::get_removed_builtin_uniform_type(builtin); + if (type == ShaderDeprecatedConverter::TokenType::TK_ERROR) { + continue; + } + Vector hints = ShaderDeprecatedConverter::get_removed_builtin_hints(builtin); + Vector funcs = ShaderDeprecatedConverter::_get_funcs_builtin_removal(mode, builtin); + String hint_string = ""; + for (int i = 0; i < hints.size(); i++) { + hint_string += ShaderDeprecatedConverter::get_tokentype_text(hints[i]); + if (i < hints.size() - 1) { + hint_string += ", "; + } + } + String uniform_decl = vformat(uniform_template, ShaderDeprecatedConverter::get_tokentype_text(type), builtin, hint_string); + for (const String &func : funcs) { + String code = vformat(code_template, get_shader_mode_name(mode), "", func, builtin); + if (type == ShaderDeprecatedConverter::TokenType::TK_ERROR) { // Unported builtins with no uniform declaration + ShaderDeprecatedConverter converter; + CHECK(converter.is_code_deprecated(code)); + CHECK_FALSE(converter.convert_code(code)); + converter.set_fail_on_unported(false); + CHECK(converter.convert_code(code)); + continue; + } + String expected = vformat(code_template, get_shader_mode_name(mode), uniform_decl, func, builtin); + TEST_CONVERSION(code, expected, true); + } + } + } + } +} + +// Reserved keywords (i.e. non-built-in function keywords that have a discrete token type) +TEST_CASE("[ShaderDeprecatedConverter] Replacement of reserved keywords used as identifiers") { + Vector keywords; + for (int i = 0; i < ShaderLanguage::TK_MAX; i++) { + if (ShaderDeprecatedConverter::tokentype_is_new_reserved_keyword(static_cast(i))) { + keywords.push_back(ShaderDeprecatedConverter::get_tokentype_text(static_cast(i))); + } + } + Vector hint_keywords; + for (int i = 0; i < SL::TK_MAX; i++) { + if (SDC::tokentype_is_new_hint(static_cast(i))) { + hint_keywords.push_back(SDC::get_tokentype_text(static_cast(i))); + } + } + Vector uniform_quals; + for (int i = 0; i < SL::TK_MAX; i++) { + if (SL::is_token_uniform_qual(static_cast(i))) { + uniform_quals.push_back(SDC::get_tokentype_text(static_cast(i))); + } + } + Vector shader_types_to_test = { "spatial", "canvas_item", "particles" }; + + static const char *decl_test_template[]{ + "shader_type %s;\nvoid %s() {}\n", + "shader_type %s;\nvoid test_func() {float %s;}\n", + "shader_type %s;\nuniform sampler2D %s;\n", + "shader_type %s;\nconst float %s = 1.0;\n", + "shader_type %s;\nvarying float %s;\n", + nullptr + }; + // NOTE: if this fails, the current behavior of the converter to replace these has to be changed. + SUBCASE("Code with reserved keywords used as identifiers fail to compile") { + ShaderLanguage::ShaderCompileInfo info; + get_compile_info(info, RS::SHADER_SPATIAL); + for (const String &shader_type : shader_types_to_test) { + for (const String &keyword : keywords) { + for (int i = 0; decl_test_template[i] != nullptr; i++) { + String code = vformat(decl_test_template[i], shader_type, keyword); + ShaderLanguage sl; + CHECK_NE(sl.compile(code, info), Error::OK); + } + } + } + } + SUBCASE("Code with reserved keywords used as identifiers is converted successfully") { + for (const String &shader_type : shader_types_to_test) { + ShaderLanguage::ShaderCompileInfo info; + get_compile_info(info, get_shader_mode(shader_type)); + for (const String &keyword : keywords) { + for (int i = 0; decl_test_template[i] != nullptr; i++) { + if (shader_type == "particles" && String(decl_test_template[i]).contains("varying")) { + continue; + } + String code = vformat(decl_test_template[i], shader_type, keyword); + String expected = vformat(decl_test_template[i], shader_type, keyword + "_"); + TEST_CONVERSION(code, expected, true); + } + } + } + } + static const char *new_hint_test = "shader_type spatial;\nuniform sampler2D foo : %s; const float %s = 1.0;\n"; + SUBCASE("New hints used as hints are not replaced") { + for (const String &hint : hint_keywords) { + String code = vformat(new_hint_test, hint, "bar"); + // Code should not change. + TEST_CONVERSION(code, code, false); + } + } + + SUBCASE("Mixed new hints used as hints and new hints used as identifiers") { + for (const String &hint : hint_keywords) { + String code = vformat(new_hint_test, hint, hint); + // Should not change. + ShaderDeprecatedConverter converter; + CHECK_FALSE(converter.is_code_deprecated(code)); // Should be detected as not deprecated. + converter.set_warning_comments(false); + CHECK(converter.convert_code(code)); + String new_code = converter.emit_code(); + // Code should not change + CHECK_EQ(new_code, code); + // Check for warning comment + converter.set_warning_comments(true); + new_code = converter.emit_code(); + CHECK(new_code.contains("/* !convert WARNING:")); + } + } + static const char *non_id_keyword_test = "shader_type spatial;\n%s uniform sampler2D foo; const float %s = 1.0;\n"; + SUBCASE("New keywords not used as identifiers are not replaced") { + for (const String &qual : uniform_quals) { + // e.g. "shader_type spatial;\nglobal uniform sampler2D foo; const float bar = 1.0;\n" + String code = vformat(non_id_keyword_test, qual, "bar"); + // Code should not change. + TEST_CONVERSION(code, code, false); + } + } + + SUBCASE("Mixed idiomatic new reserved words and new reserved words used as identifiers") { + for (const String &qual : uniform_quals) { + // e.g. "shader_type spatial;\nglobal uniform sampler2D foo; const float global = 1.0;\n" + String code = vformat(non_id_keyword_test, qual, qual); + // Should not change. + ShaderDeprecatedConverter converter; + CHECK_FALSE(converter.is_code_deprecated(code)); // Should be detected as not deprecated. + converter.set_warning_comments(false); + CHECK(converter.convert_code(code)); + String new_code = converter.emit_code(); + // Code should not change + CHECK_EQ(new_code, code); + // Check for warning comment + converter.set_warning_comments(true); + new_code = converter.emit_code(); + CHECK(new_code.contains("/* !convert WARNING:")); + } + } +} + +TEST_CASE("[ShaderDeprecatedConverter] Convert default 3.x nodetree shader") { + static const char *default_3x_nodtree_shader = + R"(shader_type spatial; +render_mode blend_mix, depth_draw_always, cull_back, diffuse_burley, specular_schlick_ggx; + +uniform sampler2D texture_0: hint_albedo; + + +void node_bsdf_principled(vec4 color, float subsurface, vec4 subsurface_color, + float metallic, float specular, float roughness, float clearcoat, + float clearcoat_roughness, float anisotropy, float transmission, + float IOR, out vec3 albedo, out float sss_strength_out, + out float metallic_out, out float specular_out, + out float roughness_out, out float clearcoat_out, + out float clearcoat_gloss_out, out float anisotropy_out, + out float transmission_out, out float ior) { + metallic = clamp(metallic, 0.0, 1.0); + transmission = clamp(transmission, 0.0, 1.0); + + subsurface = subsurface * (1.0 - metallic); + + albedo = mix(color.rgb, subsurface_color.rgb, subsurface); + sss_strength_out = subsurface; + metallic_out = metallic; + specular_out = pow((IOR - 1.0)/(IOR + 1.0), 2)/0.08; + roughness_out = roughness; + clearcoat_out = clearcoat * (1.0 - transmission); + clearcoat_gloss_out = 1.0 - clearcoat_roughness; + anisotropy_out = clamp(anisotropy, 0.0, 1.0); + transmission_out = (1.0 - transmission) * (1.0 - metallic); + ior = IOR; +} + + +void node_tex_image(vec3 co, sampler2D ima, out vec4 color, out float alpha) { + color = texture(ima, co.xy); + alpha = color.a; +} + +void vertex () { +} + +void fragment () { + + // node: 'Image Texture' + // type: 'ShaderNodeTexImage' + // input sockets handling + vec3 node0_in0_vector = vec3(0.0, 0.0, 0.0); + // output sockets definitions + vec4 node0_out0_color; + float node0_out1_alpha; + + node0_in0_vector = vec3(UV, 0.0); + node_tex_image(node0_in0_vector, texture_0, node0_out0_color, node0_out1_alpha); + + + // node: 'Principled BSDF' + // type: 'ShaderNodeBsdfPrincipled' + // input sockets handling + vec4 node1_in0_basecolor = node0_out0_color; + float node1_in1_subsurface = float(0.0); + vec3 node1_in2_subsurfaceradius = vec3(1.0, 0.20000000298023224, + 0.10000000149011612); + vec4 node1_in3_subsurfacecolor = vec4(0.800000011920929, 0.800000011920929, + 0.800000011920929, 1.0); + float node1_in4_metallic = float(0.0); + float node1_in5_specular = float(0.5); + float node1_in6_speculartint = float(0.0); + float node1_in7_roughness = float(1.0); + float node1_in8_anisotropic = float(0.0); + float node1_in9_anisotropicrotation = float(0.0); + float node1_in10_sheen = float(0.0); + float node1_in11_sheentint = float(0.5); + float node1_in12_clearcoat = float(0.0); + float node1_in13_clearcoatroughness = float(0.029999999329447746); + float node1_in14_ior = float(1.4500000476837158); + float node1_in15_transmission = float(0.0); + float node1_in16_transmissionroughness = float(0.0); + vec4 node1_in17_emission = vec4(0.0, 0.0, 0.0, 1.0); + float node1_in18_emissionstrength = float(1.0); + float node1_in19_alpha = float(1.0); + vec3 node1_in20_normal = NORMAL; + vec3 node1_in21_clearcoatnormal = vec3(0.0, 0.0, 0.0); + vec3 node1_in22_tangent = TANGENT; + // output sockets definitions + vec3 node1_bsdf_out0_albedo; + float node1_bsdf_out1_sss_strength; + float node1_bsdf_out3_specular; + float node1_bsdf_out2_metallic; + float node1_bsdf_out4_roughness; + float node1_bsdf_out5_clearcoat; + float node1_bsdf_out6_clearcoat_gloss; + float node1_bsdf_out7_anisotropy; + float node1_bsdf_out8_transmission; + float node1_bsdf_out9_ior; + + node_bsdf_principled(node1_in0_basecolor, node1_in1_subsurface, + node1_in3_subsurfacecolor, node1_in4_metallic, node1_in5_specular, + node1_in7_roughness, node1_in12_clearcoat, node1_in13_clearcoatroughness, + node1_in8_anisotropic, node1_in15_transmission, node1_in14_ior, + node1_bsdf_out0_albedo, node1_bsdf_out1_sss_strength, node1_bsdf_out2_metallic, + node1_bsdf_out3_specular, node1_bsdf_out4_roughness, node1_bsdf_out5_clearcoat, + node1_bsdf_out6_clearcoat_gloss, node1_bsdf_out7_anisotropy, + node1_bsdf_out8_transmission, node1_bsdf_out9_ior); + + + ALBEDO = node1_bsdf_out0_albedo; + SSS_STRENGTH = node1_bsdf_out1_sss_strength; + SPECULAR = node1_bsdf_out3_specular; + METALLIC = node1_bsdf_out2_metallic; + ROUGHNESS = node1_bsdf_out4_roughness; + CLEARCOAT = node1_bsdf_out5_clearcoat; + CLEARCOAT_GLOSS = node1_bsdf_out6_clearcoat_gloss; + NORMAL = node1_in20_normal; + // uncomment it when you need it + // TRANSMISSION = vec3(1.0, 1.0, 1.0) * node1_bsdf_out8_transmission; + // uncomment it when you are modifing TANGENT + // TANGENT = normalize(cross(cross(node1_in22_tangent, NORMAL), NORMAL)); + // BINORMAL = cross(TANGENT, NORMAL); + // uncomment it when you have tangent(UV) set + // ANISOTROPY = node1_bsdf_out7_anisotropy; +} + )"; + + ShaderLanguage sl; + ShaderLanguage::ShaderCompileInfo info; + get_compile_info(info, RS::SHADER_SPATIAL); + SUBCASE("Default 3.x nodetree shader does not compile") { + CHECK_NE(sl.compile(default_3x_nodtree_shader, info), Error::OK); + } + sl.clear(); + SUBCASE("Convert default 3.x nodetree shader") { + ShaderDeprecatedConverter converter; + CHECK(converter.convert_code(default_3x_nodtree_shader)); + String new_code = converter.emit_code(); + CHECK(new_code.find("/*") == -1); + CHECK_FALSE(converter.get_error_line()); + } + + SUBCASE("Converted default 3.x nodetree shader compiles") { + ShaderDeprecatedConverter converter; + CHECK(converter.convert_code(default_3x_nodtree_shader)); + String new_code = converter.emit_code(); + CHECK_EQ(sl.compile(new_code, info), Error::OK); + } + sl.clear(); +} + +} // namespace TestShaderConverter +#undef TEST_CONVERSION +#endif // DISABLE_DEPRECATED + +#endif // TEST_SHADER_CONVERTER_H diff --git a/tests/servers/rendering/test_shader_language.h b/tests/servers/rendering/test_shader_language.h new file mode 100644 index 00000000000..b51d567983a --- /dev/null +++ b/tests/servers/rendering/test_shader_language.h @@ -0,0 +1,144 @@ +/**************************************************************************/ +/* test_shader_language.h */ +/**************************************************************************/ +/* This file is part of: */ +/* GODOT ENGINE */ +/* https://godotengine.org */ +/**************************************************************************/ +/* Copyright (c) 2014-present Godot Engine contributors (see AUTHORS.md). */ +/* Copyright (c) 2007-2014 Juan Linietsky, Ariel Manzur. */ +/* */ +/* Permission is hereby granted, free of charge, to any person obtaining */ +/* a copy of this software and associated documentation files (the */ +/* "Software"), to deal in the Software without restriction, including */ +/* without limitation the rights to use, copy, modify, merge, publish, */ +/* distribute, sublicense, and/or sell copies of the Software, and to */ +/* permit persons to whom the Software is furnished to do so, subject to */ +/* the following conditions: */ +/* */ +/* The above copyright notice and this permission notice shall be */ +/* included in all copies or substantial portions of the Software. */ +/* */ +/* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, */ +/* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF */ +/* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. */ +/* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY */ +/* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, */ +/* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE */ +/* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ +/**************************************************************************/ + +#ifndef TEST_SHADER_LANGUAGE_H +#define TEST_SHADER_LANGUAGE_H + +#include "servers/rendering/shader_language.h" +#include "servers/rendering/shader_types.h" + +#include "tests/test_macros.h" + +#include + +namespace TestShaderLanguage { + +void get_compile_info(ShaderLanguage::ShaderCompileInfo &info, RenderingServer::ShaderMode p_mode) { + info.functions = ShaderTypes::get_singleton()->get_functions(p_mode); + info.render_modes = ShaderTypes::get_singleton()->get_modes(p_mode); + info.shader_types = ShaderTypes::get_singleton()->get_types(); + // Only used by editor for completion, so it's not important for these tests. + info.global_shader_uniform_type_func = [](const StringName &p_name) -> ShaderLanguage::DataType { + return ShaderLanguage::TYPE_SAMPLER2D; + }; +} + +RenderingServer::ShaderMode get_shader_mode(const String &p_mode_string) { + if (p_mode_string == "canvas_item") { + return RS::SHADER_CANVAS_ITEM; + } else if (p_mode_string == "particles") { + return RS::SHADER_PARTICLES; + } else if (p_mode_string == "spatial") { + return RS::SHADER_SPATIAL; + } else if (p_mode_string == "sky") { + return RS::SHADER_SKY; + } else if (p_mode_string == "fog") { + return RS::SHADER_FOG; + } else { + return RS::SHADER_MAX; + } +} + +TEST_CASE("[ShaderLanguage] Minimal Script") { + ShaderLanguage sl; + ShaderLanguage::ShaderCompileInfo info; + get_compile_info(info, RS::SHADER_SPATIAL); + String code = "shader_type spatial;"; + CHECK_EQ(sl.compile(code, info), Error::OK); +} + +// No keywords (except for built-in functions) should be valid identifiers. +TEST_CASE("[ShaderLanguage] Ensure no reserved keywords are valid identifiers") { + List keywords; + List builtin_functions; + ShaderLanguage::get_keyword_list(&keywords); + ShaderLanguage::get_builtin_funcs(&builtin_functions); + + HashSet builtin_set; + for (const String &keyword : builtin_functions) { + builtin_set.insert(keyword); + } + + HashSet non_func_keywords_set; + for (const String &keyword : keywords) { + if (!builtin_set.has(keyword)) { + non_func_keywords_set.insert(keyword); + } + } + + static const char *decl_test_template[]{ + "shader_type %s;\nvoid %s() {}\n", + "shader_type %s;\nvoid vertex() {float %s;}\n", + "shader_type %s;\nuniform sampler2D %s;\n", + "shader_type %s;\nconst float %s = 1.0;\n", + nullptr + }; + static const char *varying_template = "shader_type %s;\nvarying float %s;\n"; + Vector non_varying_types = { "particles", "sky", "fog" }; + + auto shader_types_to_test = ShaderTypes::get_singleton()->get_types(); + for (auto shader_type : shader_types_to_test) { + ShaderLanguage::ShaderCompileInfo info; + get_compile_info(info, get_shader_mode(shader_type)); + // test templates with non-keyword identifiers + + for (int i = 0; decl_test_template[i] != nullptr; i++) { + String code = vformat(decl_test_template[i], shader_type, "foo"); + String result; + ShaderLanguage sl; + CHECK_EQ(sl.compile(code, info), Error::OK); + } + if (!non_varying_types.has(shader_type)) { + String code = vformat(varying_template, shader_type, "foo"); + String result; + ShaderLanguage sl; + CHECK_EQ(sl.compile(code, info), Error::OK); + } + + for (const String &keyword : non_func_keywords_set) { + for (int i = 0; decl_test_template[i] != nullptr; i++) { + String code = vformat(decl_test_template[i], shader_type, keyword); + String result; + ShaderLanguage sl; + CHECK_NE(sl.compile(code, info), Error::OK); + } + if (!non_varying_types.has(shader_type)) { + String code = vformat(varying_template, shader_type, keyword); + String result; + ShaderLanguage sl; + CHECK_NE(sl.compile(code, info), Error::OK); + } + } + } +} + +} // namespace TestShaderLanguage + +#endif // TEST_SHADER_LANGUAGE_H diff --git a/tests/test_main.cpp b/tests/test_main.cpp index 12ff3ad4bc9..3838586af48 100644 --- a/tests/test_main.cpp +++ b/tests/test_main.cpp @@ -129,6 +129,8 @@ #include "tests/scene/test_viewport.h" #include "tests/scene/test_visual_shader.h" #include "tests/scene/test_window.h" +#include "tests/servers/rendering/test_shader_converter.h" +#include "tests/servers/rendering/test_shader_language.h" #include "tests/servers/rendering/test_shader_preprocessor.h" #include "tests/servers/test_text_server.h" #include "tests/test_validate_testing.h" @@ -266,7 +268,7 @@ struct GodotTestCaseListener : public doctest::IReporter { String name = String(p_in.m_name); String suite_name = String(p_in.m_test_suite); - if (name.contains("[SceneTree]") || name.contains("[Editor]")) { + if (name.contains("[SceneTree]") || name.contains("[Editor]") || name.contains("[ShaderDeprecatedConverter]") || name.contains("[ShaderLanguage]")) { memnew(MessageQueue); memnew(Input);