495 lines
15 KiB
C++
495 lines
15 KiB
C++
/**************************************************************************/
|
|
/* gdscript_tokenizer_buffer.cpp */
|
|
/**************************************************************************/
|
|
/* This file is part of: */
|
|
/* GODOT ENGINE */
|
|
/* https://godotengine.org */
|
|
/**************************************************************************/
|
|
/* Copyright (c) 2014-present Godot Engine contributors (see AUTHORS.md). */
|
|
/* Copyright (c) 2007-2014 Juan Linietsky, Ariel Manzur. */
|
|
/* */
|
|
/* Permission is hereby granted, free of charge, to any person obtaining */
|
|
/* a copy of this software and associated documentation files (the */
|
|
/* "Software"), to deal in the Software without restriction, including */
|
|
/* without limitation the rights to use, copy, modify, merge, publish, */
|
|
/* distribute, sublicense, and/or sell copies of the Software, and to */
|
|
/* permit persons to whom the Software is furnished to do so, subject to */
|
|
/* the following conditions: */
|
|
/* */
|
|
/* The above copyright notice and this permission notice shall be */
|
|
/* included in all copies or substantial portions of the Software. */
|
|
/* */
|
|
/* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, */
|
|
/* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF */
|
|
/* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. */
|
|
/* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY */
|
|
/* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, */
|
|
/* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE */
|
|
/* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */
|
|
/**************************************************************************/
|
|
|
|
#include "gdscript_tokenizer_buffer.h"
|
|
|
|
#include "core/io/compression.h"
|
|
#include "core/io/marshalls.h"
|
|
|
|
#define TOKENIZER_VERSION 100
|
|
|
|
int GDScriptTokenizerBuffer::_token_to_binary(const Token &p_token, Vector<uint8_t> &r_buffer, int p_start, HashMap<StringName, uint32_t> &r_identifiers_map, HashMap<Variant, uint32_t, VariantHasher, VariantComparator> &r_constants_map) {
|
|
int pos = p_start;
|
|
|
|
int token_type = p_token.type & TOKEN_MASK;
|
|
|
|
switch (p_token.type) {
|
|
case GDScriptTokenizer::Token::ANNOTATION:
|
|
case GDScriptTokenizer::Token::IDENTIFIER: {
|
|
// Add identifier to map.
|
|
int identifier_pos;
|
|
StringName id = p_token.get_identifier();
|
|
if (r_identifiers_map.has(id)) {
|
|
identifier_pos = r_identifiers_map[id];
|
|
} else {
|
|
identifier_pos = r_identifiers_map.size();
|
|
r_identifiers_map[id] = identifier_pos;
|
|
}
|
|
token_type |= identifier_pos << TOKEN_BITS;
|
|
} break;
|
|
case GDScriptTokenizer::Token::ERROR:
|
|
case GDScriptTokenizer::Token::LITERAL: {
|
|
// Add literal to map.
|
|
int constant_pos;
|
|
if (r_constants_map.has(p_token.literal)) {
|
|
constant_pos = r_constants_map[p_token.literal];
|
|
} else {
|
|
constant_pos = r_constants_map.size();
|
|
r_constants_map[p_token.literal] = constant_pos;
|
|
}
|
|
token_type |= constant_pos << TOKEN_BITS;
|
|
} break;
|
|
default:
|
|
break;
|
|
}
|
|
|
|
// Encode token.
|
|
int token_len;
|
|
if (token_type & TOKEN_MASK) {
|
|
token_len = 8;
|
|
r_buffer.resize(pos + token_len);
|
|
encode_uint32(token_type | TOKEN_BYTE_MASK, &r_buffer.write[pos]);
|
|
pos += 4;
|
|
} else {
|
|
token_len = 5;
|
|
r_buffer.resize(pos + token_len);
|
|
r_buffer.write[pos] = token_type;
|
|
pos++;
|
|
}
|
|
encode_uint32(p_token.start_line, &r_buffer.write[pos]);
|
|
return token_len;
|
|
}
|
|
|
|
GDScriptTokenizer::Token GDScriptTokenizerBuffer::_binary_to_token(const uint8_t *p_buffer) {
|
|
Token token;
|
|
const uint8_t *b = p_buffer;
|
|
|
|
uint32_t token_type = decode_uint32(b);
|
|
token.type = (Token::Type)(token_type & TOKEN_MASK);
|
|
if (token_type & TOKEN_BYTE_MASK) {
|
|
b += 4;
|
|
} else {
|
|
b++;
|
|
}
|
|
token.start_line = decode_uint32(b);
|
|
token.end_line = token.start_line;
|
|
|
|
token.literal = token.get_name();
|
|
if (token.type == Token::CONST_NAN) {
|
|
token.literal = String("NAN"); // Special case since name and notation are different.
|
|
}
|
|
|
|
switch (token.type) {
|
|
case GDScriptTokenizer::Token::ANNOTATION:
|
|
case GDScriptTokenizer::Token::IDENTIFIER: {
|
|
// Get name from map.
|
|
int identifier_pos = token_type >> TOKEN_BITS;
|
|
if (unlikely(identifier_pos >= identifiers.size())) {
|
|
Token error;
|
|
error.type = Token::ERROR;
|
|
error.literal = "Identifier index out of bounds.";
|
|
return error;
|
|
}
|
|
token.literal = identifiers[identifier_pos];
|
|
} break;
|
|
case GDScriptTokenizer::Token::ERROR:
|
|
case GDScriptTokenizer::Token::LITERAL: {
|
|
// Get literal from map.
|
|
int constant_pos = token_type >> TOKEN_BITS;
|
|
if (unlikely(constant_pos >= constants.size())) {
|
|
Token error;
|
|
error.type = Token::ERROR;
|
|
error.literal = "Constant index out of bounds.";
|
|
return error;
|
|
}
|
|
token.literal = constants[constant_pos];
|
|
} break;
|
|
default:
|
|
break;
|
|
}
|
|
|
|
return token;
|
|
}
|
|
|
|
Error GDScriptTokenizerBuffer::set_code_buffer(const Vector<uint8_t> &p_buffer) {
|
|
const uint8_t *buf = p_buffer.ptr();
|
|
ERR_FAIL_COND_V(p_buffer.size() < 12 || p_buffer[0] != 'G' || p_buffer[1] != 'D' || p_buffer[2] != 'S' || p_buffer[3] != 'C', ERR_INVALID_DATA);
|
|
|
|
int version = decode_uint32(&buf[4]);
|
|
ERR_FAIL_COND_V_MSG(version > TOKENIZER_VERSION, ERR_INVALID_DATA, "Binary GDScript is too recent! Please use a newer engine version.");
|
|
|
|
int decompressed_size = decode_uint32(&buf[8]);
|
|
|
|
Vector<uint8_t> contents;
|
|
if (decompressed_size == 0) {
|
|
contents = p_buffer.slice(12);
|
|
} else {
|
|
contents.resize(decompressed_size);
|
|
int result = Compression::decompress(contents.ptrw(), contents.size(), &buf[12], p_buffer.size() - 12, Compression::MODE_ZSTD);
|
|
ERR_FAIL_COND_V_MSG(result != decompressed_size, ERR_INVALID_DATA, "Error decompressing GDScript tokenizer buffer.");
|
|
}
|
|
|
|
int total_len = contents.size();
|
|
buf = contents.ptr();
|
|
uint32_t identifier_count = decode_uint32(&buf[0]);
|
|
uint32_t constant_count = decode_uint32(&buf[4]);
|
|
uint32_t token_line_count = decode_uint32(&buf[8]);
|
|
uint32_t token_count = decode_uint32(&buf[16]);
|
|
|
|
const uint8_t *b = &buf[20];
|
|
total_len -= 20;
|
|
|
|
identifiers.resize(identifier_count);
|
|
for (uint32_t i = 0; i < identifier_count; i++) {
|
|
uint32_t len = decode_uint32(b);
|
|
total_len -= 4;
|
|
ERR_FAIL_COND_V((len * 4u) > (uint32_t)total_len, ERR_INVALID_DATA);
|
|
b += 4;
|
|
Vector<uint32_t> cs;
|
|
cs.resize(len);
|
|
for (uint32_t j = 0; j < len; j++) {
|
|
uint8_t tmp[4];
|
|
for (uint32_t k = 0; k < 4; k++) {
|
|
tmp[k] = b[j * 4 + k] ^ 0xb6;
|
|
}
|
|
cs.write[j] = decode_uint32(tmp);
|
|
}
|
|
|
|
String s(reinterpret_cast<const char32_t *>(cs.ptr()), len);
|
|
b += len * 4;
|
|
total_len -= len * 4;
|
|
identifiers.write[i] = s;
|
|
}
|
|
|
|
constants.resize(constant_count);
|
|
for (uint32_t i = 0; i < constant_count; i++) {
|
|
Variant v;
|
|
int len;
|
|
Error err = decode_variant(v, b, total_len, &len, false);
|
|
if (err) {
|
|
return err;
|
|
}
|
|
b += len;
|
|
total_len -= len;
|
|
constants.write[i] = v;
|
|
}
|
|
|
|
for (uint32_t i = 0; i < token_line_count; i++) {
|
|
ERR_FAIL_COND_V(total_len < 8, ERR_INVALID_DATA);
|
|
uint32_t token_index = decode_uint32(b);
|
|
b += 4;
|
|
uint32_t line = decode_uint32(b);
|
|
b += 4;
|
|
total_len -= 8;
|
|
token_lines[token_index] = line;
|
|
}
|
|
for (uint32_t i = 0; i < token_line_count; i++) {
|
|
ERR_FAIL_COND_V(total_len < 8, ERR_INVALID_DATA);
|
|
uint32_t token_index = decode_uint32(b);
|
|
b += 4;
|
|
uint32_t column = decode_uint32(b);
|
|
b += 4;
|
|
total_len -= 8;
|
|
token_columns[token_index] = column;
|
|
}
|
|
|
|
tokens.resize(token_count);
|
|
for (uint32_t i = 0; i < token_count; i++) {
|
|
int token_len = 5;
|
|
if ((*b) & TOKEN_BYTE_MASK) {
|
|
token_len = 8;
|
|
}
|
|
ERR_FAIL_COND_V(total_len < token_len, ERR_INVALID_DATA);
|
|
Token token = _binary_to_token(b);
|
|
b += token_len;
|
|
ERR_FAIL_INDEX_V(token.type, Token::TK_MAX, ERR_INVALID_DATA);
|
|
tokens.write[i] = token;
|
|
total_len -= token_len;
|
|
}
|
|
|
|
ERR_FAIL_COND_V(total_len > 0, ERR_INVALID_DATA);
|
|
|
|
return OK;
|
|
}
|
|
|
|
Vector<uint8_t> GDScriptTokenizerBuffer::parse_code_string(const String &p_code, CompressMode p_compress_mode) {
|
|
HashMap<StringName, uint32_t> identifier_map;
|
|
HashMap<Variant, uint32_t, VariantHasher, VariantComparator> constant_map;
|
|
Vector<uint8_t> token_buffer;
|
|
HashMap<uint32_t, uint32_t> token_lines;
|
|
HashMap<uint32_t, uint32_t> token_columns;
|
|
|
|
GDScriptTokenizerText tokenizer;
|
|
tokenizer.set_source_code(p_code);
|
|
tokenizer.set_multiline_mode(true); // Ignore whitespace tokens.
|
|
Token current = tokenizer.scan();
|
|
int token_pos = 0;
|
|
int last_token_line = 0;
|
|
int token_counter = 0;
|
|
|
|
while (current.type != Token::TK_EOF) {
|
|
int token_len = _token_to_binary(current, token_buffer, token_pos, identifier_map, constant_map);
|
|
token_pos += token_len;
|
|
if (token_counter > 0 && current.start_line > last_token_line) {
|
|
token_lines[token_counter] = current.start_line;
|
|
token_columns[token_counter] = current.start_column;
|
|
}
|
|
last_token_line = current.end_line;
|
|
|
|
current = tokenizer.scan();
|
|
token_counter++;
|
|
}
|
|
|
|
// Reverse maps.
|
|
Vector<StringName> rev_identifier_map;
|
|
rev_identifier_map.resize(identifier_map.size());
|
|
for (const KeyValue<StringName, uint32_t> &E : identifier_map) {
|
|
rev_identifier_map.write[E.value] = E.key;
|
|
}
|
|
Vector<Variant> rev_constant_map;
|
|
rev_constant_map.resize(constant_map.size());
|
|
for (const KeyValue<Variant, uint32_t> &E : constant_map) {
|
|
rev_constant_map.write[E.value] = E.key;
|
|
}
|
|
HashMap<uint32_t, uint32_t> rev_token_lines;
|
|
for (const KeyValue<uint32_t, uint32_t> &E : token_lines) {
|
|
rev_token_lines[E.value] = E.key;
|
|
}
|
|
|
|
// Remove continuation lines from map.
|
|
for (int line : tokenizer.get_continuation_lines()) {
|
|
if (rev_token_lines.has(line)) {
|
|
token_lines.erase(rev_token_lines[line]);
|
|
token_columns.erase(rev_token_lines[line]);
|
|
}
|
|
}
|
|
|
|
Vector<uint8_t> contents;
|
|
contents.resize(20);
|
|
encode_uint32(identifier_map.size(), &contents.write[0]);
|
|
encode_uint32(constant_map.size(), &contents.write[4]);
|
|
encode_uint32(token_lines.size(), &contents.write[8]);
|
|
encode_uint32(0, &contents.write[12]); // Unused, kept for compatibility. Please remove at next `TOKENIZER_VERSION` increment.
|
|
encode_uint32(token_counter, &contents.write[16]);
|
|
|
|
int buf_pos = 20;
|
|
|
|
// Save identifiers.
|
|
for (const StringName &id : rev_identifier_map) {
|
|
String s = id.operator String();
|
|
int len = s.length();
|
|
|
|
contents.resize(buf_pos + (len + 1) * 4);
|
|
|
|
encode_uint32(len, &contents.write[buf_pos]);
|
|
buf_pos += 4;
|
|
|
|
for (int i = 0; i < len; i++) {
|
|
uint8_t tmp[4];
|
|
encode_uint32(s[i], tmp);
|
|
|
|
for (int b = 0; b < 4; b++) {
|
|
contents.write[buf_pos + b] = tmp[b] ^ 0xb6;
|
|
}
|
|
|
|
buf_pos += 4;
|
|
}
|
|
}
|
|
|
|
// Save constants.
|
|
for (const Variant &v : rev_constant_map) {
|
|
int len;
|
|
// Objects cannot be constant, never encode objects.
|
|
Error err = encode_variant(v, nullptr, len, false);
|
|
ERR_FAIL_COND_V_MSG(err != OK, Vector<uint8_t>(), "Error when trying to encode Variant.");
|
|
contents.resize(buf_pos + len);
|
|
encode_variant(v, &contents.write[buf_pos], len, false);
|
|
buf_pos += len;
|
|
}
|
|
|
|
// Save lines and columns.
|
|
contents.resize(buf_pos + token_lines.size() * 16);
|
|
for (const KeyValue<uint32_t, uint32_t> &e : token_lines) {
|
|
encode_uint32(e.key, &contents.write[buf_pos]);
|
|
buf_pos += 4;
|
|
encode_uint32(e.value, &contents.write[buf_pos]);
|
|
buf_pos += 4;
|
|
}
|
|
for (const KeyValue<uint32_t, uint32_t> &e : token_columns) {
|
|
encode_uint32(e.key, &contents.write[buf_pos]);
|
|
buf_pos += 4;
|
|
encode_uint32(e.value, &contents.write[buf_pos]);
|
|
buf_pos += 4;
|
|
}
|
|
|
|
// Store tokens.
|
|
contents.append_array(token_buffer);
|
|
|
|
Vector<uint8_t> buf;
|
|
|
|
// Save header.
|
|
buf.resize(12);
|
|
buf.write[0] = 'G';
|
|
buf.write[1] = 'D';
|
|
buf.write[2] = 'S';
|
|
buf.write[3] = 'C';
|
|
encode_uint32(TOKENIZER_VERSION, &buf.write[4]);
|
|
|
|
switch (p_compress_mode) {
|
|
case COMPRESS_NONE:
|
|
encode_uint32(0u, &buf.write[8]);
|
|
buf.append_array(contents);
|
|
break;
|
|
|
|
case COMPRESS_ZSTD: {
|
|
encode_uint32(contents.size(), &buf.write[8]);
|
|
Vector<uint8_t> compressed;
|
|
int max_size = Compression::get_max_compressed_buffer_size(contents.size(), Compression::MODE_ZSTD);
|
|
compressed.resize(max_size);
|
|
|
|
int compressed_size = Compression::compress(compressed.ptrw(), contents.ptr(), contents.size(), Compression::MODE_ZSTD);
|
|
ERR_FAIL_COND_V_MSG(compressed_size < 0, Vector<uint8_t>(), "Error compressing GDScript tokenizer buffer.");
|
|
compressed.resize(compressed_size);
|
|
|
|
buf.append_array(compressed);
|
|
} break;
|
|
}
|
|
|
|
return buf;
|
|
}
|
|
|
|
int GDScriptTokenizerBuffer::get_cursor_line() const {
|
|
return 0;
|
|
}
|
|
|
|
int GDScriptTokenizerBuffer::get_cursor_column() const {
|
|
return 0;
|
|
}
|
|
|
|
void GDScriptTokenizerBuffer::set_cursor_position(int p_line, int p_column) {
|
|
}
|
|
|
|
void GDScriptTokenizerBuffer::set_multiline_mode(bool p_state) {
|
|
multiline_mode = p_state;
|
|
}
|
|
|
|
bool GDScriptTokenizerBuffer::is_past_cursor() const {
|
|
return false;
|
|
}
|
|
|
|
void GDScriptTokenizerBuffer::push_expression_indented_block() {
|
|
indent_stack_stack.push_back(indent_stack);
|
|
}
|
|
|
|
void GDScriptTokenizerBuffer::pop_expression_indented_block() {
|
|
ERR_FAIL_COND(indent_stack_stack.is_empty());
|
|
indent_stack = indent_stack_stack.back()->get();
|
|
indent_stack_stack.pop_back();
|
|
}
|
|
|
|
GDScriptTokenizer::Token GDScriptTokenizerBuffer::scan() {
|
|
// Add final newline.
|
|
if (current >= tokens.size() && !last_token_was_newline) {
|
|
Token newline;
|
|
newline.type = Token::NEWLINE;
|
|
newline.start_line = current_line;
|
|
newline.end_line = current_line;
|
|
last_token_was_newline = true;
|
|
return newline;
|
|
}
|
|
|
|
// Resolve pending indentation change.
|
|
if (pending_indents > 0) {
|
|
pending_indents--;
|
|
Token indent;
|
|
indent.type = Token::INDENT;
|
|
indent.start_line = current_line;
|
|
indent.end_line = current_line;
|
|
return indent;
|
|
} else if (pending_indents < 0) {
|
|
pending_indents++;
|
|
Token dedent;
|
|
dedent.type = Token::DEDENT;
|
|
dedent.start_line = current_line;
|
|
dedent.end_line = current_line;
|
|
return dedent;
|
|
}
|
|
|
|
if (current >= tokens.size()) {
|
|
if (!indent_stack.is_empty()) {
|
|
pending_indents -= indent_stack.size();
|
|
indent_stack.clear();
|
|
return scan();
|
|
}
|
|
Token eof;
|
|
eof.type = Token::TK_EOF;
|
|
return eof;
|
|
};
|
|
|
|
if (!last_token_was_newline && token_lines.has(current)) {
|
|
current_line = token_lines[current];
|
|
uint32_t current_column = token_columns[current];
|
|
|
|
// Check if there's a need to indent/dedent.
|
|
if (!multiline_mode) {
|
|
uint32_t previous_indent = 0;
|
|
if (!indent_stack.is_empty()) {
|
|
previous_indent = indent_stack.back()->get();
|
|
}
|
|
if (current_column - 1 > previous_indent) {
|
|
pending_indents++;
|
|
indent_stack.push_back(current_column - 1);
|
|
} else {
|
|
while (current_column - 1 < previous_indent) {
|
|
pending_indents--;
|
|
indent_stack.pop_back();
|
|
if (indent_stack.is_empty()) {
|
|
break;
|
|
}
|
|
previous_indent = indent_stack.back()->get();
|
|
}
|
|
}
|
|
|
|
Token newline;
|
|
newline.type = Token::NEWLINE;
|
|
newline.start_line = current_line;
|
|
newline.end_line = current_line;
|
|
last_token_was_newline = true;
|
|
|
|
return newline;
|
|
}
|
|
}
|
|
|
|
last_token_was_newline = false;
|
|
|
|
Token token = tokens[current++];
|
|
return token;
|
|
}
|