From 4373a0bb865265f07507d36e6c151a556f3d94e8 Mon Sep 17 00:00:00 2001 From: bruvzg <7645683+bruvzg@users.noreply.github.com> Date: Fri, 12 Nov 2021 10:12:37 +0200 Subject: [PATCH] [TextServer] Add ICU Unicode security and spoofing detection. --- doc/classes/TextServer.xml | 21 + doc/classes/TextServerExtension.xml | 15 + modules/text_server_adv/SCsub | 11 +- modules/text_server_adv/text_server_adv.cpp | 63 + modules/text_server_adv/text_server_adv.h | 4 + servers/text/text_server_extension.cpp | 19 + servers/text/text_server_extension.h | 5 + servers/text_server.cpp | 4 + servers/text_server.h | 4 + thirdparty/README.md | 2 + thirdparty/icu4c/godot_data.json | 3 +- thirdparty/icu4c/i18n/scriptset.cpp | 313 ++++ thirdparty/icu4c/i18n/scriptset.h | 86 + thirdparty/icu4c/i18n/ucln_in.cpp | 65 + thirdparty/icu4c/i18n/ucln_in.h | 74 + thirdparty/icu4c/i18n/unicode/uspoof.h | 1577 +++++++++++++++++++ thirdparty/icu4c/i18n/uspoof.cpp | 839 ++++++++++ thirdparty/icu4c/i18n/uspoof_impl.cpp | 959 +++++++++++ thirdparty/icu4c/i18n/uspoof_impl.h | 343 ++++ thirdparty/icu4c/icudt71l.dat | Bin 4226000 -> 4271680 bytes 20 files changed, 4403 insertions(+), 4 deletions(-) create mode 100644 thirdparty/icu4c/i18n/scriptset.cpp create mode 100644 thirdparty/icu4c/i18n/scriptset.h create mode 100644 thirdparty/icu4c/i18n/ucln_in.cpp create mode 100644 thirdparty/icu4c/i18n/ucln_in.h create mode 100644 thirdparty/icu4c/i18n/unicode/uspoof.h create mode 100644 thirdparty/icu4c/i18n/uspoof.cpp create mode 100644 thirdparty/icu4c/i18n/uspoof_impl.cpp create mode 100644 thirdparty/icu4c/i18n/uspoof_impl.h diff --git a/doc/classes/TextServer.xml b/doc/classes/TextServer.xml index f4c9ade7d14..9f35ddc3532 100644 --- a/doc/classes/TextServer.xml +++ b/doc/classes/TextServer.xml @@ -937,6 +937,16 @@ Returns [code]true[/code] if the server supports a feature. + + + + + + Returns index of the first string in [code]dict[/dict] which is visually confusable with the [code]string[/string], or [code]-1[/code] if none is found. + [b]Note:[/b] This method doesn't detect invisible characters, for spoof detection use it in combination with [method spoof_check]. + [b]Note:[/b] Always returns [code]-1[/code] if the server does not support the [constant FEATURE_UNICODE_SECURITY] feature. + + @@ -1476,6 +1486,14 @@ Aligns shaped text to the given tab-stops. + + + + + Returns [code]true[/code] if [code]string[/code] is likely to be an attempt at confusing the reader. + [b]Note:[/b] Always returns [code]false[/code] if the server does not support the [constant FEATURE_UNICODE_SECURITY] feature. + + @@ -1733,6 +1751,9 @@ TextServer supports UAX #31 identifier validation, see [method is_valid_identifier]. + + TextServer supports [url=https://unicode.org/reports/tr36/]Unicode Technical Report #36[/url] and [url=https://unicode.org/reports/tr39/]Unicode Technical Standard #39[/url] based spoof detection features. + Contour point is on the curve. diff --git a/doc/classes/TextServerExtension.xml b/doc/classes/TextServerExtension.xml index 482460cb3ba..c686a06e5e3 100644 --- a/doc/classes/TextServerExtension.xml +++ b/doc/classes/TextServerExtension.xml @@ -934,6 +934,14 @@ Returns [code]true[/code] if the server supports a feature. + + + + + + Returns index of the first string in [code]dict[/dict] which is visually confusable with the [code]string[/string], or [code]-1[/code] if none is found. + + @@ -1488,6 +1496,13 @@ [b]Note:[/b] This method is used by default line/word breaking methods, and its implementation might be omitted if custom line breaking in implemented. + + + + + Returns [code]true[/code] if [code]string[/code] is likely to be an attempt at confusing the reader. + + diff --git a/modules/text_server_adv/SCsub b/modules/text_server_adv/SCsub index d212fe62b4a..73e5c2bf743 100644 --- a/modules/text_server_adv/SCsub +++ b/modules/text_server_adv/SCsub @@ -121,7 +121,7 @@ if env["builtin_harfbuzz"]: env_harfbuzz.Append(CCFLAGS=["-DHAVE_ICU"]) if env["builtin_icu"]: - env_harfbuzz.Prepend(CPPPATH=["#thirdparty/icu4c/common/"]) + env_harfbuzz.Prepend(CPPPATH=["#thirdparty/icu4c/common/", "#thirdparty/icu4c/i18n/"]) env_harfbuzz.Append(CCFLAGS=["-DU_HAVE_LIB_SUFFIX=1", "-DU_LIB_SUFFIX_C_NAME=_godot", "-DHAVE_ICU_BUILTIN"]) if freetype_enabled: @@ -439,6 +439,10 @@ if env["builtin_icu"]: "common/uvectr32.cpp", "common/uvectr64.cpp", "common/wintz.cpp", + "i18n/scriptset.cpp", + "i18n/ucln_in.cpp", + "i18n/uspoof.cpp", + "i18n/uspoof_impl.cpp", ] thirdparty_sources = [thirdparty_dir + file for file in thirdparty_sources] @@ -451,7 +455,7 @@ if env["builtin_icu"]: else: thirdparty_sources += ["icu_data/icudata_stub.cpp"] - env_icu.Prepend(CPPPATH=["#thirdparty/icu4c/common/"]) + env_icu.Prepend(CPPPATH=["#thirdparty/icu4c/common/", "#thirdparty/icu4c/i18n/"]) env_icu.Append( CXXFLAGS=[ "-DU_STATIC_IMPLEMENTATION", @@ -463,6 +467,7 @@ if env["builtin_icu"]: "-DUCONFIG_NO_IDNA", "-DUCONFIG_NO_FILE_IO", "-DUCONFIG_NO_TRANSLITERATION", + "-DUCONFIG_NO_REGULAR_EXPRESSIONS", "-DPKGDATA_MODE=static", "-DU_ENABLE_DYLOAD=0", "-DU_HAVE_LIB_SUFFIX=1", @@ -480,7 +485,7 @@ if env["builtin_icu"]: if env_text_server_adv["tools"]: env_text_server_adv.Append(CXXFLAGS=["-DICU_STATIC_DATA"]) - env_text_server_adv.Prepend(CPPPATH=["#thirdparty/icu4c/common/"]) + env_text_server_adv.Prepend(CPPPATH=["#thirdparty/icu4c/common/", "#thirdparty/icu4c/i18n/"]) lib = env_icu.add_library("icu_builtin", thirdparty_sources) thirdparty_obj += lib diff --git a/modules/text_server_adv/text_server_adv.cpp b/modules/text_server_adv/text_server_adv.cpp index f8dbbc2e61c..bb49fb52482 100644 --- a/modules/text_server_adv/text_server_adv.cpp +++ b/modules/text_server_adv/text_server_adv.cpp @@ -347,6 +347,7 @@ bool TextServerAdvanced::has_feature(Feature p_feature) const { case FEATURE_CONTEXT_SENSITIVE_CASE_CONVERSION: case FEATURE_USE_SUPPORT_DATA: case FEATURE_UNICODE_IDENTIFIERS: + case FEATURE_UNICODE_SECURITY: return true; default: { } @@ -5640,6 +5641,68 @@ String TextServerAdvanced::percent_sign(const String &p_language) const { return "%"; } +int TextServerAdvanced::is_confusable(const String &p_string, const PackedStringArray &p_dict) const { + UErrorCode status = U_ZERO_ERROR; + int match_index = -1; + + Char16String utf16 = p_string.utf16(); + Vector skeletons; + skeletons.resize(p_dict.size()); + + USpoofChecker *sc = uspoof_open(&status); + uspoof_setChecks(sc, USPOOF_CONFUSABLE, &status); + for (int i = 0; i < p_dict.size(); i++) { + Char16String word = p_dict[i].utf16(); + int32_t len = uspoof_getSkeleton(sc, 0, word.get_data(), -1, NULL, 0, &status); + skeletons.write[i] = (UChar *)memalloc(++len * sizeof(UChar)); + status = U_ZERO_ERROR; + uspoof_getSkeleton(sc, 0, word.get_data(), -1, skeletons.write[i], len, &status); + } + + int32_t len = uspoof_getSkeleton(sc, 0, utf16.get_data(), -1, NULL, 0, &status); + UChar *skel = (UChar *)memalloc(++len * sizeof(UChar)); + status = U_ZERO_ERROR; + uspoof_getSkeleton(sc, 0, utf16.get_data(), -1, skel, len, &status); + for (int i = 0; i < skeletons.size(); i++) { + if (u_strcmp(skel, skeletons[i]) == 0) { + match_index = i; + break; + } + } + memfree(skel); + + for (int i = 0; i < skeletons.size(); i++) { + memfree(skeletons.write[i]); + } + uspoof_close(sc); + + ERR_FAIL_COND_V_MSG(U_FAILURE(status), -1, u_errorName(status)); + + return match_index; +} + +bool TextServerAdvanced::spoof_check(const String &p_string) const { + UErrorCode status = U_ZERO_ERROR; + Char16String utf16 = p_string.utf16(); + + USet *allowed = uset_openEmpty(); + uset_addAll(allowed, uspoof_getRecommendedSet(&status)); + uset_addAll(allowed, uspoof_getInclusionSet(&status)); + + USpoofChecker *sc = uspoof_open(&status); + uspoof_setAllowedChars(sc, allowed, &status); + uspoof_setRestrictionLevel(sc, USPOOF_MODERATELY_RESTRICTIVE); + + int32_t bitmask = uspoof_check(sc, utf16.get_data(), -1, NULL, &status); + + uspoof_close(sc); + uset_close(allowed); + + ERR_FAIL_COND_V_MSG(U_FAILURE(status), false, u_errorName(status)); + + return (bitmask != 0); +} + String TextServerAdvanced::strip_diacritics(const String &p_string) const { UErrorCode err = U_ZERO_ERROR; diff --git a/modules/text_server_adv/text_server_adv.h b/modules/text_server_adv/text_server_adv.h index 170d92f8fcc..b337abea7a2 100644 --- a/modules/text_server_adv/text_server_adv.h +++ b/modules/text_server_adv/text_server_adv.h @@ -101,6 +101,7 @@ using namespace godot; #include #include #include +#include #include #include @@ -701,6 +702,9 @@ public: virtual PackedInt32Array string_get_word_breaks(const String &p_string, const String &p_language = "") const override; + virtual int is_confusable(const String &p_string, const PackedStringArray &p_dict) const override; + virtual bool spoof_check(const String &p_string) const override; + virtual String strip_diacritics(const String &p_string) const override; virtual bool is_valid_identifier(const String &p_string) const override; diff --git a/servers/text/text_server_extension.cpp b/servers/text/text_server_extension.cpp index c2387be80d7..59310ab69bf 100644 --- a/servers/text/text_server_extension.cpp +++ b/servers/text/text_server_extension.cpp @@ -301,6 +301,9 @@ void TextServerExtension::_bind_methods() { GDVIRTUAL_BIND(string_get_word_breaks, "string", "language"); + GDVIRTUAL_BIND(is_confusable, "string", "dict"); + GDVIRTUAL_BIND(spoof_check, "string"); + GDVIRTUAL_BIND(string_to_upper, "string", "language"); GDVIRTUAL_BIND(string_to_lower, "string", "language"); @@ -1547,6 +1550,22 @@ PackedInt32Array TextServerExtension::string_get_word_breaks(const String &p_str return PackedInt32Array(); } +int TextServerExtension::is_confusable(const String &p_string, const PackedStringArray &p_dict) const { + int ret; + if (GDVIRTUAL_CALL(is_confusable, p_string, p_dict, ret)) { + return ret; + } + return TextServer::is_confusable(p_string, p_dict); +} + +bool TextServerExtension::spoof_check(const String &p_string) const { + bool ret; + if (GDVIRTUAL_CALL(spoof_check, p_string, ret)) { + return ret; + } + return TextServer::spoof_check(p_string); +} + TextServerExtension::TextServerExtension() { //NOP } diff --git a/servers/text/text_server_extension.h b/servers/text/text_server_extension.h index 3814e2ad884..81af1b60e56 100644 --- a/servers/text/text_server_extension.h +++ b/servers/text/text_server_extension.h @@ -507,6 +507,11 @@ public: Array parse_structured_text(StructuredTextParser p_parser_type, const Array &p_args, const String &p_text) const; GDVIRTUAL3RC(Array, parse_structured_text, StructuredTextParser, const Array &, const String &); + virtual int is_confusable(const String &p_string, const PackedStringArray &p_dict) const override; + virtual bool spoof_check(const String &p_string) const override; + GDVIRTUAL2RC(int, is_confusable, const String &, const PackedStringArray &); + GDVIRTUAL1RC(bool, spoof_check, const String &); + TextServerExtension(); ~TextServerExtension(); }; diff --git a/servers/text_server.cpp b/servers/text_server.cpp index 1ea05795f11..fd63a7b99d0 100644 --- a/servers/text_server.cpp +++ b/servers/text_server.cpp @@ -446,6 +446,9 @@ void TextServer::_bind_methods() { ClassDB::bind_method(D_METHOD("string_get_word_breaks", "string", "language"), &TextServer::string_get_word_breaks, DEFVAL("")); + ClassDB::bind_method(D_METHOD("is_confusable", "string", "dict"), &TextServer::is_confusable); + ClassDB::bind_method(D_METHOD("spoof_check", "string"), &TextServer::spoof_check); + ClassDB::bind_method(D_METHOD("strip_diacritics", "string"), &TextServer::strip_diacritics); ClassDB::bind_method(D_METHOD("is_valid_identifier", "string"), &TextServer::is_valid_identifier); @@ -547,6 +550,7 @@ void TextServer::_bind_methods() { BIND_ENUM_CONSTANT(FEATURE_CONTEXT_SENSITIVE_CASE_CONVERSION); BIND_ENUM_CONSTANT(FEATURE_USE_SUPPORT_DATA); BIND_ENUM_CONSTANT(FEATURE_UNICODE_IDENTIFIERS); + BIND_ENUM_CONSTANT(FEATURE_UNICODE_SECURITY); /* FT Contour Point Types */ BIND_ENUM_CONSTANT(CONTOUR_CURVE_TAG_ON); diff --git a/servers/text_server.h b/servers/text_server.h index 0e57bcbb6cf..5874b8f6e84 100644 --- a/servers/text_server.h +++ b/servers/text_server.h @@ -149,6 +149,7 @@ public: FEATURE_CONTEXT_SENSITIVE_CASE_CONVERSION = 1 << 11, FEATURE_USE_SUPPORT_DATA = 1 << 12, FEATURE_UNICODE_IDENTIFIERS = 1 << 13, + FEATURE_UNICODE_SECURITY = 1 << 14, }; enum ContourPointTag { @@ -464,6 +465,9 @@ public: // String functions. virtual PackedInt32Array string_get_word_breaks(const String &p_string, const String &p_language = "") const = 0; + virtual int is_confusable(const String &p_string, const PackedStringArray &p_dict) const { return -1; }; + virtual bool spoof_check(const String &p_string) const { return false; }; + virtual String strip_diacritics(const String &p_string) const; virtual bool is_valid_identifier(const String &p_string) const; diff --git a/thirdparty/README.md b/thirdparty/README.md index b06d9cec81b..6db011d3c65 100644 --- a/thirdparty/README.md +++ b/thirdparty/README.md @@ -231,6 +231,8 @@ Files extracted from upstream source: Files extracted from upstream source: - the `common` folder +- `scriptset.*`, `ucln_in.*`, `uspoof.cpp"` and `uspoof_impl.cpp` from the `i18n` folder +- `uspoof.h` from the `i18n/unicode` folder - `LICENSE` Files generated from upstream source: diff --git a/thirdparty/icu4c/godot_data.json b/thirdparty/icu4c/godot_data.json index 3a9c28af4cc..e36e2b078b5 100644 --- a/thirdparty/icu4c/godot_data.json +++ b/thirdparty/icu4c/godot_data.json @@ -6,5 +6,6 @@ brkitr_tree: include misc: include normalization: include + confusables: include } -} \ No newline at end of file +} diff --git a/thirdparty/icu4c/i18n/scriptset.cpp b/thirdparty/icu4c/i18n/scriptset.cpp new file mode 100644 index 00000000000..6a1db8c01c3 --- /dev/null +++ b/thirdparty/icu4c/i18n/scriptset.cpp @@ -0,0 +1,313 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* +********************************************************************** +* Copyright (C) 2014, International Business Machines +* Corporation and others. All Rights Reserved. +********************************************************************** +* +* scriptset.cpp +* +* created on: 2013 Jan 7 +* created by: Andy Heninger +*/ + +#include "unicode/utypes.h" + +#include "unicode/uchar.h" +#include "unicode/unistr.h" + +#include "scriptset.h" +#include "uassert.h" +#include "cmemory.h" + +U_NAMESPACE_BEGIN + +//---------------------------------------------------------------------------- +// +// ScriptSet implementation +// +//---------------------------------------------------------------------------- +ScriptSet::ScriptSet() { + uprv_memset(bits, 0, sizeof(bits)); +} + +ScriptSet::~ScriptSet() { +} + +ScriptSet::ScriptSet(const ScriptSet &other) { + *this = other; +} + +ScriptSet & ScriptSet::operator =(const ScriptSet &other) { + uprv_memcpy(bits, other.bits, sizeof(bits)); + return *this; +} + +bool ScriptSet::operator == (const ScriptSet &other) const { + for (uint32_t i=0; i= SCRIPT_LIMIT) { + status = U_ILLEGAL_ARGUMENT_ERROR; + return FALSE; + } + uint32_t index = script / 32; + uint32_t bit = 1 << (script & 31); + return ((bits[index] & bit) != 0); +} + + +ScriptSet &ScriptSet::set(UScriptCode script, UErrorCode &status) { + if (U_FAILURE(status)) { + return *this; + } + if (script < 0 || (int32_t)script >= SCRIPT_LIMIT) { + status = U_ILLEGAL_ARGUMENT_ERROR; + return *this; + } + uint32_t index = script / 32; + uint32_t bit = 1 << (script & 31); + bits[index] |= bit; + return *this; +} + +ScriptSet &ScriptSet::reset(UScriptCode script, UErrorCode &status) { + if (U_FAILURE(status)) { + return *this; + } + if (script < 0 || (int32_t)script >= SCRIPT_LIMIT) { + status = U_ILLEGAL_ARGUMENT_ERROR; + return *this; + } + uint32_t index = script / 32; + uint32_t bit = 1 << (script & 31); + bits[index] &= ~bit; + return *this; +} + + + +ScriptSet &ScriptSet::Union(const ScriptSet &other) { + for (uint32_t i=0; iintersect(t); + } + return *this; +} + +UBool ScriptSet::intersects(const ScriptSet &other) const { + for (uint32_t i=0; i 0) { + count++; + x &= (x - 1); // and off the least significant one bit. + } + } + return count; +} + +int32_t ScriptSet::hashCode() const { + int32_t hash = 0; + for (int32_t i=0; i= 0; i = nextSetBit(i + 1)) { + if (!firstTime) { + dest.append((UChar)0x20); + } + firstTime = FALSE; + const char *scriptName = uscript_getShortName((UScriptCode(i))); + dest.append(UnicodeString(scriptName, -1, US_INV)); + } + return dest; +} + +ScriptSet &ScriptSet::parseScripts(const UnicodeString &scriptString, UErrorCode &status) { + resetAll(); + if (U_FAILURE(status)) { + return *this; + } + UnicodeString oneScriptName; + for (int32_t i=0; i 0) { + char buf[40]; + oneScriptName.extract(0, oneScriptName.length(), buf, sizeof(buf)-1, US_INV); + buf[sizeof(buf)-1] = 0; + int32_t sc = u_getPropertyValueEnum(UCHAR_SCRIPT, buf); + if (sc == UCHAR_INVALID_CODE) { + status = U_ILLEGAL_ARGUMENT_ERROR; + } else { + this->set((UScriptCode)sc, status); + } + if (U_FAILURE(status)) { + return *this; + } + oneScriptName.remove(); + } + } + return *this; +} + +void ScriptSet::setScriptExtensions(UChar32 codePoint, UErrorCode& status) { + if (U_FAILURE(status)) { return; } + static const int32_t FIRST_GUESS_SCRIPT_CAPACITY = 20; + MaybeStackArray scripts; + UErrorCode internalStatus = U_ZERO_ERROR; + int32_t script_count = -1; + + while (TRUE) { + script_count = uscript_getScriptExtensions( + codePoint, scripts.getAlias(), scripts.getCapacity(), &internalStatus); + if (internalStatus == U_BUFFER_OVERFLOW_ERROR) { + // Need to allocate more space + if (scripts.resize(script_count) == NULL) { + status = U_MEMORY_ALLOCATION_ERROR; + return; + } + internalStatus = U_ZERO_ERROR; + } else { + break; + } + } + + // Check if we failed for some reason other than buffer overflow + if (U_FAILURE(internalStatus)) { + status = internalStatus; + return; + } + + // Load the scripts into the ScriptSet and return + for (int32_t i = 0; i < script_count; i++) { + this->set(scripts[i], status); + if (U_FAILURE(status)) { return; } + } +} + +U_NAMESPACE_END + +U_CAPI UBool U_EXPORT2 +uhash_equalsScriptSet(const UElement key1, const UElement key2) { + icu::ScriptSet *s1 = static_cast(key1.pointer); + icu::ScriptSet *s2 = static_cast(key2.pointer); + return (*s1 == *s2); +} + +U_CAPI int8_t U_EXPORT2 +uhash_compareScriptSet(UElement key0, UElement key1) { + icu::ScriptSet *s0 = static_cast(key0.pointer); + icu::ScriptSet *s1 = static_cast(key1.pointer); + int32_t diff = s0->countMembers() - s1->countMembers(); + if (diff != 0) return static_cast(diff); + int32_t i0 = s0->nextSetBit(0); + int32_t i1 = s1->nextSetBit(0); + while ((diff = i0-i1) == 0 && i0 > 0) { + i0 = s0->nextSetBit(i0+1); + i1 = s1->nextSetBit(i1+1); + } + return (int8_t)diff; +} + +U_CAPI int32_t U_EXPORT2 +uhash_hashScriptSet(const UElement key) { + icu::ScriptSet *s = static_cast(key.pointer); + return s->hashCode(); +} + +U_CAPI void U_EXPORT2 +uhash_deleteScriptSet(void *obj) { + icu::ScriptSet *s = static_cast(obj); + delete s; +} diff --git a/thirdparty/icu4c/i18n/scriptset.h b/thirdparty/icu4c/i18n/scriptset.h new file mode 100644 index 00000000000..51980ab7b3e --- /dev/null +++ b/thirdparty/icu4c/i18n/scriptset.h @@ -0,0 +1,86 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* +********************************************************************** +* Copyright (C) 2013, International Business Machines +* Corporation and others. All Rights Reserved. +********************************************************************** +* +* scriptset.h +* +* created on: 2013 Jan 7 +* created by: Andy Heninger +*/ + +#ifndef __SCRIPTSET_H__ +#define __SCRIPTSET_H__ + +#include "unicode/utypes.h" +#include "unicode/uobject.h" +#include "unicode/uscript.h" + +#include "uelement.h" + +U_NAMESPACE_BEGIN + +//------------------------------------------------------------------------------- +// +// ScriptSet - A bit set representing a set of scripts. +// +// This class was originally used exclusively with script sets appearing +// as part of the spoof check whole script confusable binary data. Its +// use has since become more general, but the continued use to wrap +// prebuilt binary data does constrain the design. +// +//------------------------------------------------------------------------------- +class U_I18N_API ScriptSet: public UMemory { + public: + static constexpr int32_t SCRIPT_LIMIT = 224; // multiple of 32! + + ScriptSet(); + ScriptSet(const ScriptSet &other); + ~ScriptSet(); + + bool operator == (const ScriptSet &other) const; + bool operator != (const ScriptSet &other) const {return !(*this == other);} + ScriptSet & operator = (const ScriptSet &other); + + UBool test(UScriptCode script, UErrorCode &status) const; + ScriptSet &Union(const ScriptSet &other); + ScriptSet &set(UScriptCode script, UErrorCode &status); + ScriptSet &reset(UScriptCode script, UErrorCode &status); + ScriptSet &intersect(const ScriptSet &other); + ScriptSet &intersect(UScriptCode script, UErrorCode &status); + UBool intersects(const ScriptSet &other) const; // Sets contain at least one script in common. + UBool contains(const ScriptSet &other) const; // All set bits in other are also set in this. + + ScriptSet &setAll(); + ScriptSet &resetAll(); + int32_t countMembers() const; + int32_t hashCode() const; + int32_t nextSetBit(int32_t script) const; + + UBool isEmpty() const; + + UnicodeString &displayScripts(UnicodeString &dest) const; // append script names to dest string. + ScriptSet & parseScripts(const UnicodeString &scriptsString, UErrorCode &status); // Replaces ScriptSet contents. + + // Wraps around UScript::getScriptExtensions() and adds the corresponding scripts to this instance. + void setScriptExtensions(UChar32 codePoint, UErrorCode& status); + + private: + uint32_t bits[SCRIPT_LIMIT / 32]; +}; + +U_NAMESPACE_END + +U_CAPI UBool U_EXPORT2 +uhash_compareScriptSet(const UElement key1, const UElement key2); + +U_CAPI int32_t U_EXPORT2 +uhash_hashScriptSet(const UElement key); + +U_CAPI void U_EXPORT2 +uhash_deleteScriptSet(void *obj); + +#endif // __SCRIPTSET_H__ diff --git a/thirdparty/icu4c/i18n/ucln_in.cpp b/thirdparty/icu4c/i18n/ucln_in.cpp new file mode 100644 index 00000000000..f29cbe41dde --- /dev/null +++ b/thirdparty/icu4c/i18n/ucln_in.cpp @@ -0,0 +1,65 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* +****************************************************************************** +* * +* Copyright (C) 2001-2014, International Business Machines * +* Corporation and others. All Rights Reserved. * +* * +****************************************************************************** +* file name: ucln_in.cpp +* encoding: UTF-8 +* tab size: 8 (not used) +* indentation:4 +* +* created on: 2001July05 +* created by: George Rhoten +*/ + +#include "ucln.h" +#include "ucln_in.h" +#include "mutex.h" +#include "uassert.h" + +/** Auto-client for UCLN_I18N **/ +#define UCLN_TYPE UCLN_I18N +#include "ucln_imp.h" + +/* Leave this copyright notice here! It needs to go somewhere in this library. */ +static const char copyright[] = U_COPYRIGHT_STRING; + +static cleanupFunc *gCleanupFunctions[UCLN_I18N_COUNT]; + +static UBool U_CALLCONV i18n_cleanup(void) +{ + int32_t libType = UCLN_I18N_START; + (void)copyright; /* Suppress unused variable warning with clang. */ + + while (++libType + * This class, based on Unicode Technical Report #36 and + * Unicode Technical Standard #39, has two main functions: + * + *
    + *
  1. Checking whether two strings are visually confusable with each other, such as "Harvest" and + * "Ηarvest", where the second string starts with the Greek capital letter Eta.
  2. + *
  3. Checking whether an individual string is likely to be an attempt at confusing the reader (spoof + * detection), such as "paypal" with some Latin characters substituted with Cyrillic look-alikes.
  4. + *
+ * + *

+ * Although originally designed as a method for flagging suspicious identifier strings such as URLs, + * USpoofChecker has a number of other practical use cases, such as preventing attempts to evade bad-word + * content filters. + * + *

+ * The functions of this class are exposed as C API, with a handful of syntactical conveniences for C++. + * + *

Confusables

+ * + *

+ * The following example shows how to use USpoofChecker to check for confusability between two strings: + * + * \code{.c} + * UErrorCode status = U_ZERO_ERROR; + * UChar* str1 = (UChar*) u"Harvest"; + * UChar* str2 = (UChar*) u"\u0397arvest"; // with U+0397 GREEK CAPITAL LETTER ETA + * + * USpoofChecker* sc = uspoof_open(&status); + * uspoof_setChecks(sc, USPOOF_CONFUSABLE, &status); + * + * int32_t bitmask = uspoof_areConfusable(sc, str1, -1, str2, -1, &status); + * UBool result = bitmask != 0; + * // areConfusable: 1 (status: U_ZERO_ERROR) + * printf("areConfusable: %d (status: %s)\n", result, u_errorName(status)); + * uspoof_close(sc); + * \endcode + * + *

+ * The call to {@link uspoof_open} creates a USpoofChecker object; the call to {@link uspoof_setChecks} + * enables confusable checking and disables all other checks; the call to {@link uspoof_areConfusable} performs the + * confusability test; and the following line extracts the result out of the return value. For best performance, + * the instance should be created once (e.g., upon application startup), and the efficient + * {@link uspoof_areConfusable} method can be used at runtime. + * + *

+ * The type {@link LocalUSpoofCheckerPointer} is exposed for C++ programmers. It will automatically call + * {@link uspoof_close} when the object goes out of scope: + * + * \code{.cpp} + * UErrorCode status = U_ZERO_ERROR; + * LocalUSpoofCheckerPointer sc(uspoof_open(&status)); + * uspoof_setChecks(sc.getAlias(), USPOOF_CONFUSABLE, &status); + * // ... + * \endcode + * + * UTS 39 defines two strings to be confusable if they map to the same skeleton string. A skeleton can + * be thought of as a "hash code". {@link uspoof_getSkeleton} computes the skeleton for a particular string, so + * the following snippet is equivalent to the example above: + * + * \code{.c} + * UErrorCode status = U_ZERO_ERROR; + * UChar* str1 = (UChar*) u"Harvest"; + * UChar* str2 = (UChar*) u"\u0397arvest"; // with U+0397 GREEK CAPITAL LETTER ETA + * + * USpoofChecker* sc = uspoof_open(&status); + * uspoof_setChecks(sc, USPOOF_CONFUSABLE, &status); + * + * // Get skeleton 1 + * int32_t skel1Len = uspoof_getSkeleton(sc, 0, str1, -1, NULL, 0, &status); + * UChar* skel1 = (UChar*) malloc(++skel1Len * sizeof(UChar)); + * status = U_ZERO_ERROR; + * uspoof_getSkeleton(sc, 0, str1, -1, skel1, skel1Len, &status); + * + * // Get skeleton 2 + * int32_t skel2Len = uspoof_getSkeleton(sc, 0, str2, -1, NULL, 0, &status); + * UChar* skel2 = (UChar*) malloc(++skel2Len * sizeof(UChar)); + * status = U_ZERO_ERROR; + * uspoof_getSkeleton(sc, 0, str2, -1, skel2, skel2Len, &status); + * + * // Are the skeletons the same? + * UBool result = u_strcmp(skel1, skel2) == 0; + * // areConfusable: 1 (status: U_ZERO_ERROR) + * printf("areConfusable: %d (status: %s)\n", result, u_errorName(status)); + * uspoof_close(sc); + * free(skel1); + * free(skel2); + * \endcode + * + * If you need to check if a string is confusable with any string in a dictionary of many strings, rather than calling + * {@link uspoof_areConfusable} many times in a loop, {@link uspoof_getSkeleton} can be used instead, as shown below: + * + * \code{.c} + * UErrorCode status = U_ZERO_ERROR; + * #define DICTIONARY_LENGTH 2 + * UChar* dictionary[DICTIONARY_LENGTH] = { (UChar*) u"lorem", (UChar*) u"ipsum" }; + * UChar* skeletons[DICTIONARY_LENGTH]; + * UChar* str = (UChar*) u"1orern"; + * + * // Setup: + * USpoofChecker* sc = uspoof_open(&status); + * uspoof_setChecks(sc, USPOOF_CONFUSABLE, &status); + * for (size_t i=0; iNote: Since the Unicode confusables mapping table is frequently updated, confusable skeletons are not + * guaranteed to be the same between ICU releases. We therefore recommend that you always compute confusable skeletons + * at runtime and do not rely on creating a permanent, or difficult to update, database of skeletons. + * + *

Spoof Detection

+ * + * The following snippet shows a minimal example of using USpoofChecker to perform spoof detection on a + * string: + * + * \code{.c} + * UErrorCode status = U_ZERO_ERROR; + * UChar* str = (UChar*) u"p\u0430ypal"; // with U+0430 CYRILLIC SMALL LETTER A + * + * // Get the default set of allowable characters: + * USet* allowed = uset_openEmpty(); + * uset_addAll(allowed, uspoof_getRecommendedSet(&status)); + * uset_addAll(allowed, uspoof_getInclusionSet(&status)); + * + * USpoofChecker* sc = uspoof_open(&status); + * uspoof_setAllowedChars(sc, allowed, &status); + * uspoof_setRestrictionLevel(sc, USPOOF_MODERATELY_RESTRICTIVE); + * + * int32_t bitmask = uspoof_check(sc, str, -1, NULL, &status); + * UBool result = bitmask != 0; + * // fails checks: 1 (status: U_ZERO_ERROR) + * printf("fails checks: %d (status: %s)\n", result, u_errorName(status)); + * uspoof_close(sc); + * uset_close(allowed); + * \endcode + * + * As in the case for confusability checking, it is good practice to create one USpoofChecker instance at + * startup, and call the cheaper {@link uspoof_check} online. We specify the set of + * allowed characters to be those with type RECOMMENDED or INCLUSION, according to the recommendation in UTS 39. + * + * In addition to {@link uspoof_check}, the function {@link uspoof_checkUTF8} is exposed for UTF8-encoded char* strings, + * and {@link uspoof_checkUnicodeString} is exposed for C++ programmers. + * + * If the {@link USPOOF_AUX_INFO} check is enabled, a limited amount of information on why a string failed the checks + * is available in the returned bitmask. For complete information, use the {@link uspoof_check2} class of functions + * with a {@link USpoofCheckResult} parameter: + * + * \code{.c} + * UErrorCode status = U_ZERO_ERROR; + * UChar* str = (UChar*) u"p\u0430ypal"; // with U+0430 CYRILLIC SMALL LETTER A + * + * // Get the default set of allowable characters: + * USet* allowed = uset_openEmpty(); + * uset_addAll(allowed, uspoof_getRecommendedSet(&status)); + * uset_addAll(allowed, uspoof_getInclusionSet(&status)); + * + * USpoofChecker* sc = uspoof_open(&status); + * uspoof_setAllowedChars(sc, allowed, &status); + * uspoof_setRestrictionLevel(sc, USPOOF_MODERATELY_RESTRICTIVE); + * + * USpoofCheckResult* checkResult = uspoof_openCheckResult(&status); + * int32_t bitmask = uspoof_check2(sc, str, -1, checkResult, &status); + * + * int32_t failures1 = bitmask; + * int32_t failures2 = uspoof_getCheckResultChecks(checkResult, &status); + * assert(failures1 == failures2); + * // checks that failed: 0x00000010 (status: U_ZERO_ERROR) + * printf("checks that failed: %#010x (status: %s)\n", failures1, u_errorName(status)); + * + * // Cleanup: + * uspoof_close(sc); + * uset_close(allowed); + * uspoof_closeCheckResult(checkResult); + * \endcode + * + * C++ users can take advantage of a few syntactical conveniences. The following snippet is functionally + * equivalent to the one above: + * + * \code{.cpp} + * UErrorCode status = U_ZERO_ERROR; + * UnicodeString str((UChar*) u"p\u0430ypal"); // with U+0430 CYRILLIC SMALL LETTER A + * + * // Get the default set of allowable characters: + * UnicodeSet allowed; + * allowed.addAll(*uspoof_getRecommendedUnicodeSet(&status)); + * allowed.addAll(*uspoof_getInclusionUnicodeSet(&status)); + * + * LocalUSpoofCheckerPointer sc(uspoof_open(&status)); + * uspoof_setAllowedChars(sc.getAlias(), allowed.toUSet(), &status); + * uspoof_setRestrictionLevel(sc.getAlias(), USPOOF_MODERATELY_RESTRICTIVE); + * + * LocalUSpoofCheckResultPointer checkResult(uspoof_openCheckResult(&status)); + * int32_t bitmask = uspoof_check2UnicodeString(sc.getAlias(), str, checkResult.getAlias(), &status); + * + * int32_t failures1 = bitmask; + * int32_t failures2 = uspoof_getCheckResultChecks(checkResult.getAlias(), &status); + * assert(failures1 == failures2); + * // checks that failed: 0x00000010 (status: U_ZERO_ERROR) + * printf("checks that failed: %#010x (status: %s)\n", failures1, u_errorName(status)); + * + * // Explicit cleanup not necessary. + * \endcode + * + * The return value is a bitmask of the checks that failed. In this case, there was one check that failed: + * {@link USPOOF_RESTRICTION_LEVEL}, corresponding to the fifth bit (16). The possible checks are: + * + *
    + *
  • RESTRICTION_LEVEL: flags strings that violate the + * Restriction Level test as specified in UTS + * 39; in most cases, this means flagging strings that contain characters from multiple different scripts.
  • + *
  • INVISIBLE: flags strings that contain invisible characters, such as zero-width spaces, or character + * sequences that are likely not to display, such as multiple occurrences of the same non-spacing mark.
  • + *
  • CHAR_LIMIT: flags strings that contain characters outside of a specified set of acceptable + * characters. See {@link uspoof_setAllowedChars} and {@link uspoof_setAllowedLocales}.
  • + *
  • MIXED_NUMBERS: flags strings that contain digits from multiple different numbering systems.
  • + *
+ * + *

+ * These checks can be enabled independently of each other. For example, if you were interested in checking for only the + * INVISIBLE and MIXED_NUMBERS conditions, you could do: + * + * \code{.c} + * UErrorCode status = U_ZERO_ERROR; + * UChar* str = (UChar*) u"8\u09EA"; // 8 mixed with U+09EA BENGALI DIGIT FOUR + * + * USpoofChecker* sc = uspoof_open(&status); + * uspoof_setChecks(sc, USPOOF_INVISIBLE | USPOOF_MIXED_NUMBERS, &status); + * + * int32_t bitmask = uspoof_check2(sc, str, -1, NULL, &status); + * UBool result = bitmask != 0; + * // fails checks: 1 (status: U_ZERO_ERROR) + * printf("fails checks: %d (status: %s)\n", result, u_errorName(status)); + * uspoof_close(sc); + * \endcode + * + * Here is an example in C++ showing how to compute the restriction level of a string: + * + * \code{.cpp} + * UErrorCode status = U_ZERO_ERROR; + * UnicodeString str((UChar*) u"p\u0430ypal"); // with U+0430 CYRILLIC SMALL LETTER A + * + * // Get the default set of allowable characters: + * UnicodeSet allowed; + * allowed.addAll(*uspoof_getRecommendedUnicodeSet(&status)); + * allowed.addAll(*uspoof_getInclusionUnicodeSet(&status)); + * + * LocalUSpoofCheckerPointer sc(uspoof_open(&status)); + * uspoof_setAllowedChars(sc.getAlias(), allowed.toUSet(), &status); + * uspoof_setRestrictionLevel(sc.getAlias(), USPOOF_MODERATELY_RESTRICTIVE); + * uspoof_setChecks(sc.getAlias(), USPOOF_RESTRICTION_LEVEL | USPOOF_AUX_INFO, &status); + * + * LocalUSpoofCheckResultPointer checkResult(uspoof_openCheckResult(&status)); + * int32_t bitmask = uspoof_check2UnicodeString(sc.getAlias(), str, checkResult.getAlias(), &status); + * + * URestrictionLevel restrictionLevel = uspoof_getCheckResultRestrictionLevel(checkResult.getAlias(), &status); + * // Since USPOOF_AUX_INFO was enabled, the restriction level is also available in the upper bits of the bitmask: + * assert((restrictionLevel & bitmask) == restrictionLevel); + * // Restriction level: 0x50000000 (status: U_ZERO_ERROR) + * printf("Restriction level: %#010x (status: %s)\n", restrictionLevel, u_errorName(status)); + * \endcode + * + * The code '0x50000000' corresponds to the restriction level USPOOF_MINIMALLY_RESTRICTIVE. Since + * USPOOF_MINIMALLY_RESTRICTIVE is weaker than USPOOF_MODERATELY_RESTRICTIVE, the string fails the check. + * + * Note: The Restriction Level is the most powerful of the checks. The full logic is documented in + * UTS 39, but the basic idea is that strings + * are restricted to contain characters from only a single script, except that most scripts are allowed to have + * Latin characters interspersed. Although the default restriction level is HIGHLY_RESTRICTIVE, it is + * recommended that users set their restriction level to MODERATELY_RESTRICTIVE, which allows Latin mixed + * with all other scripts except Cyrillic, Greek, and Cherokee, with which it is often confusable. For more details on + * the levels, see UTS 39 or {@link URestrictionLevel}. The Restriction Level test is aware of the set of + * allowed characters set in {@link uspoof_setAllowedChars}. Note that characters which have script code + * COMMON or INHERITED, such as numbers and punctuation, are ignored when computing whether a string has multiple + * scripts. + * + *

Additional Information

+ * + * A USpoofChecker instance may be used repeatedly to perform checks on any number of identifiers. + * + * Thread Safety: The test functions for checking a single identifier, or for testing whether + * two identifiers are possible confusable, are thread safe. They may called concurrently, from multiple threads, + * using the same USpoofChecker instance. + * + * More generally, the standard ICU thread safety rules apply: functions that take a const USpoofChecker parameter are + * thread safe. Those that take a non-const USpoofChecker are not thread safe.. + * + * @stable ICU 4.6 + */ + +U_CDECL_BEGIN + +struct USpoofChecker; +/** + * @stable ICU 4.2 + */ +typedef struct USpoofChecker USpoofChecker; /**< typedef for C of USpoofChecker */ + +struct USpoofCheckResult; +/** + * @see uspoof_openCheckResult + * @stable ICU 58 + */ +typedef struct USpoofCheckResult USpoofCheckResult; + +/** + * Enum for the kinds of checks that USpoofChecker can perform. + * These enum values are used both to select the set of checks that + * will be performed, and to report results from the check function. + * + * @stable ICU 4.2 + */ +typedef enum USpoofChecks { + /** + * When performing the two-string {@link uspoof_areConfusable} test, this flag in the return value indicates + * that the two strings are visually confusable and that they are from the same script, according to UTS 39 section + * 4. + * + * @see uspoof_areConfusable + * @stable ICU 4.2 + */ + USPOOF_SINGLE_SCRIPT_CONFUSABLE = 1, + + /** + * When performing the two-string {@link uspoof_areConfusable} test, this flag in the return value indicates + * that the two strings are visually confusable and that they are not from the same script, according to UTS + * 39 section 4. + * + * @see uspoof_areConfusable + * @stable ICU 4.2 + */ + USPOOF_MIXED_SCRIPT_CONFUSABLE = 2, + + /** + * When performing the two-string {@link uspoof_areConfusable} test, this flag in the return value indicates + * that the two strings are visually confusable and that they are not from the same script but both of them are + * single-script strings, according to UTS 39 section 4. + * + * @see uspoof_areConfusable + * @stable ICU 4.2 + */ + USPOOF_WHOLE_SCRIPT_CONFUSABLE = 4, + + /** + * Enable this flag in {@link uspoof_setChecks} to turn on all types of confusables. You may set + * the checks to some subset of SINGLE_SCRIPT_CONFUSABLE, MIXED_SCRIPT_CONFUSABLE, or WHOLE_SCRIPT_CONFUSABLE to + * make {@link uspoof_areConfusable} return only those types of confusables. + * + * @see uspoof_areConfusable + * @see uspoof_getSkeleton + * @stable ICU 58 + */ + USPOOF_CONFUSABLE = USPOOF_SINGLE_SCRIPT_CONFUSABLE | USPOOF_MIXED_SCRIPT_CONFUSABLE | USPOOF_WHOLE_SCRIPT_CONFUSABLE, + +#ifndef U_HIDE_DEPRECATED_API + /** + * This flag is deprecated and no longer affects the behavior of SpoofChecker. + * + * @deprecated ICU 58 Any case confusable mappings were removed from UTS 39; the corresponding ICU API was deprecated. + */ + USPOOF_ANY_CASE = 8, +#endif /* U_HIDE_DEPRECATED_API */ + + /** + * Check that an identifier is no looser than the specified RestrictionLevel. + * The default if {@link uspoof_setRestrictionLevel} is not called is HIGHLY_RESTRICTIVE. + * + * If USPOOF_AUX_INFO is enabled the actual restriction level of the + * identifier being tested will also be returned by uspoof_check(). + * + * @see URestrictionLevel + * @see uspoof_setRestrictionLevel + * @see USPOOF_AUX_INFO + * + * @stable ICU 51 + */ + USPOOF_RESTRICTION_LEVEL = 16, + +#ifndef U_HIDE_DEPRECATED_API + /** Check that an identifier contains only characters from a + * single script (plus chars from the common and inherited scripts.) + * Applies to checks of a single identifier check only. + * @deprecated ICU 51 Use RESTRICTION_LEVEL instead. + */ + USPOOF_SINGLE_SCRIPT = USPOOF_RESTRICTION_LEVEL, +#endif /* U_HIDE_DEPRECATED_API */ + + /** Check an identifier for the presence of invisible characters, + * such as zero-width spaces, or character sequences that are + * likely not to display, such as multiple occurrences of the same + * non-spacing mark. This check does not test the input string as a whole + * for conformance to any particular syntax for identifiers. + */ + USPOOF_INVISIBLE = 32, + + /** Check that an identifier contains only characters from a specified set + * of acceptable characters. See {@link uspoof_setAllowedChars} and + * {@link uspoof_setAllowedLocales}. Note that a string that fails this check + * will also fail the {@link USPOOF_RESTRICTION_LEVEL} check. + */ + USPOOF_CHAR_LIMIT = 64, + + /** + * Check that an identifier does not mix numbers from different numbering systems. + * For more information, see UTS 39 section 5.3. + * + * @stable ICU 51 + */ + USPOOF_MIXED_NUMBERS = 128, + + /** + * Check that an identifier does not have a combining character following a character in which that + * combining character would be hidden; for example 'i' followed by a U+0307 combining dot. + * + * More specifically, the following characters are forbidden from preceding a U+0307: + *
    + *
  • Those with the Soft_Dotted Unicode property (which includes 'i' and 'j')
  • + *
  • Latin lowercase letter 'l'
  • + *
  • Dotless 'i' and 'j' ('ı' and 'ȷ', U+0131 and U+0237)
  • + *
  • Any character whose confusable prototype ends with such a character + * (Soft_Dotted, 'l', 'ı', or 'ȷ')
  • + *
+ * In addition, combining characters are allowed between the above characters and U+0307 except those + * with combining class 0 or combining class "Above" (230, same class as U+0307). + * + * This list and the number of combing characters considered by this check may grow over time. + * + * @stable ICU 62 + */ + USPOOF_HIDDEN_OVERLAY = 256, + + /** + * Enable all spoof checks. + * + * @stable ICU 4.6 + */ + USPOOF_ALL_CHECKS = 0xFFFF, + + /** + * Enable the return of auxiliary (non-error) information in the + * upper bits of the check results value. + * + * If this "check" is not enabled, the results of {@link uspoof_check} will be + * zero when an identifier passes all of the enabled checks. + * + * If this "check" is enabled, (uspoof_check() & {@link USPOOF_ALL_CHECKS}) will + * be zero when an identifier passes all checks. + * + * @stable ICU 51 + */ + USPOOF_AUX_INFO = 0x40000000 + + } USpoofChecks; + + + /** + * Constants from UAX #39 for use in {@link uspoof_setRestrictionLevel}, and + * for returned identifier restriction levels in check results. + * + * @stable ICU 51 + * + * @see uspoof_setRestrictionLevel + * @see uspoof_check + */ + typedef enum URestrictionLevel { + /** + * All characters in the string are in the identifier profile and all characters in the string are in the + * ASCII range. + * + * @stable ICU 51 + */ + USPOOF_ASCII = 0x10000000, + /** + * The string classifies as ASCII-Only, or all characters in the string are in the identifier profile and + * the string is single-script, according to the definition in UTS 39 section 5.1. + * + * @stable ICU 53 + */ + USPOOF_SINGLE_SCRIPT_RESTRICTIVE = 0x20000000, + /** + * The string classifies as Single Script, or all characters in the string are in the identifier profile and + * the string is covered by any of the following sets of scripts, according to the definition in UTS 39 + * section 5.1: + *
    + *
  • Latin + Han + Bopomofo (or equivalently: Latn + Hanb)
  • + *
  • Latin + Han + Hiragana + Katakana (or equivalently: Latn + Jpan)
  • + *
  • Latin + Han + Hangul (or equivalently: Latn +Kore)
  • + *
+ * This is the default restriction in ICU. + * + * @stable ICU 51 + */ + USPOOF_HIGHLY_RESTRICTIVE = 0x30000000, + /** + * The string classifies as Highly Restrictive, or all characters in the string are in the identifier profile + * and the string is covered by Latin and any one other Recommended or Aspirational script, except Cyrillic, + * Greek, and Cherokee. + * + * @stable ICU 51 + */ + USPOOF_MODERATELY_RESTRICTIVE = 0x40000000, + /** + * All characters in the string are in the identifier profile. Allow arbitrary mixtures of scripts. + * + * @stable ICU 51 + */ + USPOOF_MINIMALLY_RESTRICTIVE = 0x50000000, + /** + * Any valid identifiers, including characters outside of the Identifier Profile. + * + * @stable ICU 51 + */ + USPOOF_UNRESTRICTIVE = 0x60000000, + /** + * Mask for selecting the Restriction Level bits from the return value of {@link uspoof_check}. + * + * @stable ICU 53 + */ + USPOOF_RESTRICTION_LEVEL_MASK = 0x7F000000, +#ifndef U_HIDE_INTERNAL_API + /** + * An undefined restriction level. + * @internal + */ + USPOOF_UNDEFINED_RESTRICTIVE = -1 +#endif /* U_HIDE_INTERNAL_API */ + } URestrictionLevel; + +/** + * Create a Unicode Spoof Checker, configured to perform all + * checks except for USPOOF_LOCALE_LIMIT and USPOOF_CHAR_LIMIT. + * Note that additional checks may be added in the future, + * resulting in the changes to the default checking behavior. + * + * @param status The error code, set if this function encounters a problem. + * @return the newly created Spoof Checker + * @stable ICU 4.2 + */ +U_CAPI USpoofChecker * U_EXPORT2 +uspoof_open(UErrorCode *status); + + +/** + * Open a Spoof checker from its serialized form, stored in 32-bit-aligned memory. + * Inverse of uspoof_serialize(). + * The memory containing the serialized data must remain valid and unchanged + * as long as the spoof checker, or any cloned copies of the spoof checker, + * are in use. Ownership of the memory remains with the caller. + * The spoof checker (and any clones) must be closed prior to deleting the + * serialized data. + * + * @param data a pointer to 32-bit-aligned memory containing the serialized form of spoof data + * @param length the number of bytes available at data; + * can be more than necessary + * @param pActualLength receives the actual number of bytes at data taken up by the data; + * can be NULL + * @param pErrorCode ICU error code + * @return the spoof checker. + * + * @see uspoof_open + * @see uspoof_serialize + * @stable ICU 4.2 + */ +U_CAPI USpoofChecker * U_EXPORT2 +uspoof_openFromSerialized(const void *data, int32_t length, int32_t *pActualLength, + UErrorCode *pErrorCode); + +/** + * Open a Spoof Checker from the source form of the spoof data. + * The input corresponds to the Unicode data file confusables.txt + * as described in Unicode UAX #39. The syntax of the source data + * is as described in UAX #39 for this file, and the content of + * this file is acceptable input. + * + * The character encoding of the (char *) input text is UTF-8. + * + * @param confusables a pointer to the confusable characters definitions, + * as found in file confusables.txt from unicode.org. + * @param confusablesLen The length of the confusables text, or -1 if the + * input string is zero terminated. + * @param confusablesWholeScript + * Deprecated in ICU 58. No longer used. + * @param confusablesWholeScriptLen + * Deprecated in ICU 58. No longer used. + * @param errType In the event of an error in the input, indicates + * which of the input files contains the error. + * The value is one of USPOOF_SINGLE_SCRIPT_CONFUSABLE or + * USPOOF_WHOLE_SCRIPT_CONFUSABLE, or + * zero if no errors are found. + * @param pe In the event of an error in the input, receives the position + * in the input text (line, offset) of the error. + * @param status an in/out ICU UErrorCode. Among the possible errors is + * U_PARSE_ERROR, which is used to report syntax errors + * in the input. + * @return A spoof checker that uses the rules from the input files. + * @stable ICU 4.2 + */ +U_CAPI USpoofChecker * U_EXPORT2 +uspoof_openFromSource(const char *confusables, int32_t confusablesLen, + const char *confusablesWholeScript, int32_t confusablesWholeScriptLen, + int32_t *errType, UParseError *pe, UErrorCode *status); + + +/** + * Close a Spoof Checker, freeing any memory that was being held by + * its implementation. + * @stable ICU 4.2 + */ +U_CAPI void U_EXPORT2 +uspoof_close(USpoofChecker *sc); + +/** + * Clone a Spoof Checker. The clone will be set to perform the same checks + * as the original source. + * + * @param sc The source USpoofChecker + * @param status The error code, set if this function encounters a problem. + * @return + * @stable ICU 4.2 + */ +U_CAPI USpoofChecker * U_EXPORT2 +uspoof_clone(const USpoofChecker *sc, UErrorCode *status); + + +/** + * Specify the bitmask of checks that will be performed by {@link uspoof_check}. Calling this method + * overwrites any checks that may have already been enabled. By default, all checks are enabled. + * + * To enable specific checks and disable all others, + * OR together only the bit constants for the desired checks. + * For example, to fail strings containing characters outside of + * the set specified by {@link uspoof_setAllowedChars} and + * also strings that contain digits from mixed numbering systems: + * + *
+ * {@code
+ * uspoof_setChecks(USPOOF_CHAR_LIMIT | USPOOF_MIXED_NUMBERS);
+ * }
+ * 
+ * + * To disable specific checks and enable all others, + * start with ALL_CHECKS and "AND away" the not-desired checks. + * For example, if you are not planning to use the {@link uspoof_areConfusable} functionality, + * it is good practice to disable the CONFUSABLE check: + * + *
+ * {@code
+ * uspoof_setChecks(USPOOF_ALL_CHECKS & ~USPOOF_CONFUSABLE);
+ * }
+ * 
+ * + * Note that methods such as {@link uspoof_setAllowedChars}, {@link uspoof_setAllowedLocales}, and + * {@link uspoof_setRestrictionLevel} will enable certain checks when called. Those methods will OR the check they + * enable onto the existing bitmask specified by this method. For more details, see the documentation of those + * methods. + * + * @param sc The USpoofChecker + * @param checks The set of checks that this spoof checker will perform. + * The value is a bit set, obtained by OR-ing together + * values from enum USpoofChecks. + * @param status The error code, set if this function encounters a problem. + * @stable ICU 4.2 + * + */ +U_CAPI void U_EXPORT2 +uspoof_setChecks(USpoofChecker *sc, int32_t checks, UErrorCode *status); + +/** + * Get the set of checks that this Spoof Checker has been configured to perform. + * + * @param sc The USpoofChecker + * @param status The error code, set if this function encounters a problem. + * @return The set of checks that this spoof checker will perform. + * The value is a bit set, obtained by OR-ing together + * values from enum USpoofChecks. + * @stable ICU 4.2 + * + */ +U_CAPI int32_t U_EXPORT2 +uspoof_getChecks(const USpoofChecker *sc, UErrorCode *status); + +/** + * Set the loosest restriction level allowed for strings. The default if this is not called is + * {@link USPOOF_HIGHLY_RESTRICTIVE}. Calling this method enables the {@link USPOOF_RESTRICTION_LEVEL} and + * {@link USPOOF_MIXED_NUMBERS} checks, corresponding to Sections 5.1 and 5.2 of UTS 39. To customize which checks are + * to be performed by {@link uspoof_check}, see {@link uspoof_setChecks}. + * + * @param sc The USpoofChecker + * @param restrictionLevel The loosest restriction level allowed. + * @see URestrictionLevel + * @stable ICU 51 + */ +U_CAPI void U_EXPORT2 +uspoof_setRestrictionLevel(USpoofChecker *sc, URestrictionLevel restrictionLevel); + + +/** + * Get the Restriction Level that will be tested if the checks include {@link USPOOF_RESTRICTION_LEVEL}. + * + * @return The restriction level + * @see URestrictionLevel + * @stable ICU 51 + */ +U_CAPI URestrictionLevel U_EXPORT2 +uspoof_getRestrictionLevel(const USpoofChecker *sc); + +/** + * Limit characters that are acceptable in identifiers being checked to those + * normally used with the languages associated with the specified locales. + * Any previously specified list of locales is replaced by the new settings. + * + * A set of languages is determined from the locale(s), and + * from those a set of acceptable Unicode scripts is determined. + * Characters from this set of scripts, along with characters from + * the "common" and "inherited" Unicode Script categories + * will be permitted. + * + * Supplying an empty string removes all restrictions; + * characters from any script will be allowed. + * + * The {@link USPOOF_CHAR_LIMIT} test is automatically enabled for this + * USpoofChecker when calling this function with a non-empty list + * of locales. + * + * The Unicode Set of characters that will be allowed is accessible + * via the uspoof_getAllowedChars() function. uspoof_setAllowedLocales() + * will replace any previously applied set of allowed characters. + * + * Adjustments, such as additions or deletions of certain classes of characters, + * can be made to the result of uspoof_setAllowedLocales() by + * fetching the resulting set with uspoof_getAllowedChars(), + * manipulating it with the Unicode Set API, then resetting the + * spoof detectors limits with uspoof_setAllowedChars(). + * + * @param sc The USpoofChecker + * @param localesList A list list of locales, from which the language + * and associated script are extracted. The locales + * are comma-separated if there is more than one. + * White space may not appear within an individual locale, + * but is ignored otherwise. + * The locales are syntactically like those from the + * HTTP Accept-Language header. + * If the localesList is empty, no restrictions will be placed on + * the allowed characters. + * + * @param status The error code, set if this function encounters a problem. + * @stable ICU 4.2 + */ +U_CAPI void U_EXPORT2 +uspoof_setAllowedLocales(USpoofChecker *sc, const char *localesList, UErrorCode *status); + +/** + * Get a list of locales for the scripts that are acceptable in strings + * to be checked. If no limitations on scripts have been specified, + * an empty string will be returned. + * + * uspoof_setAllowedChars() will reset the list of allowed to be empty. + * + * The format of the returned list is the same as that supplied to + * uspoof_setAllowedLocales(), but returned list may not be identical + * to the originally specified string; the string may be reformatted, + * and information other than languages from + * the originally specified locales may be omitted. + * + * @param sc The USpoofChecker + * @param status The error code, set if this function encounters a problem. + * @return A string containing a list of locales corresponding + * to the acceptable scripts, formatted like an + * HTTP Accept Language value. + * + * @stable ICU 4.2 + */ +U_CAPI const char * U_EXPORT2 +uspoof_getAllowedLocales(USpoofChecker *sc, UErrorCode *status); + + +/** + * Limit the acceptable characters to those specified by a Unicode Set. + * Any previously specified character limit is + * is replaced by the new settings. This includes limits on + * characters that were set with the uspoof_setAllowedLocales() function. + * + * The USPOOF_CHAR_LIMIT test is automatically enabled for this + * USpoofChecker by this function. + * + * @param sc The USpoofChecker + * @param chars A Unicode Set containing the list of + * characters that are permitted. Ownership of the set + * remains with the caller. The incoming set is cloned by + * this function, so there are no restrictions on modifying + * or deleting the USet after calling this function. + * @param status The error code, set if this function encounters a problem. + * @stable ICU 4.2 + */ +U_CAPI void U_EXPORT2 +uspoof_setAllowedChars(USpoofChecker *sc, const USet *chars, UErrorCode *status); + + +/** + * Get a USet for the characters permitted in an identifier. + * This corresponds to the limits imposed by the Set Allowed Characters + * functions. Limitations imposed by other checks will not be + * reflected in the set returned by this function. + * + * The returned set will be frozen, meaning that it cannot be modified + * by the caller. + * + * Ownership of the returned set remains with the Spoof Detector. The + * returned set will become invalid if the spoof detector is closed, + * or if a new set of allowed characters is specified. + * + * + * @param sc The USpoofChecker + * @param status The error code, set if this function encounters a problem. + * @return A USet containing the characters that are permitted by + * the USPOOF_CHAR_LIMIT test. + * @stable ICU 4.2 + */ +U_CAPI const USet * U_EXPORT2 +uspoof_getAllowedChars(const USpoofChecker *sc, UErrorCode *status); + + +/** + * Check the specified string for possible security issues. + * The text to be checked will typically be an identifier of some sort. + * The set of checks to be performed is specified with uspoof_setChecks(). + * + * \note + * Consider using the newer API, {@link uspoof_check2}, instead. + * The newer API exposes additional information from the check procedure + * and is otherwise identical to this method. + * + * @param sc The USpoofChecker + * @param id The identifier to be checked for possible security issues, + * in UTF-16 format. + * @param length the length of the string to be checked, expressed in + * 16 bit UTF-16 code units, or -1 if the string is + * zero terminated. + * @param position Deprecated in ICU 51. Always returns zero. + * Originally, an out parameter for the index of the first + * string position that failed a check. + * This parameter may be NULL. + * @param status The error code, set if an error occurred while attempting to + * perform the check. + * Spoofing or security issues detected with the input string are + * not reported here, but through the function's return value. + * @return An integer value with bits set for any potential security + * or spoofing issues detected. The bits are defined by + * enum USpoofChecks. (returned_value & USPOOF_ALL_CHECKS) + * will be zero if the input string passes all of the + * enabled checks. + * @see uspoof_check2 + * @stable ICU 4.2 + */ +U_CAPI int32_t U_EXPORT2 +uspoof_check(const USpoofChecker *sc, + const UChar *id, int32_t length, + int32_t *position, + UErrorCode *status); + + +/** + * Check the specified string for possible security issues. + * The text to be checked will typically be an identifier of some sort. + * The set of checks to be performed is specified with uspoof_setChecks(). + * + * \note + * Consider using the newer API, {@link uspoof_check2UTF8}, instead. + * The newer API exposes additional information from the check procedure + * and is otherwise identical to this method. + * + * @param sc The USpoofChecker + * @param id A identifier to be checked for possible security issues, in UTF8 format. + * @param length the length of the string to be checked, or -1 if the string is + * zero terminated. + * @param position Deprecated in ICU 51. Always returns zero. + * Originally, an out parameter for the index of the first + * string position that failed a check. + * This parameter may be NULL. + * @param status The error code, set if an error occurred while attempting to + * perform the check. + * Spoofing or security issues detected with the input string are + * not reported here, but through the function's return value. + * If the input contains invalid UTF-8 sequences, + * a status of U_INVALID_CHAR_FOUND will be returned. + * @return An integer value with bits set for any potential security + * or spoofing issues detected. The bits are defined by + * enum USpoofChecks. (returned_value & USPOOF_ALL_CHECKS) + * will be zero if the input string passes all of the + * enabled checks. + * @see uspoof_check2UTF8 + * @stable ICU 4.2 + */ +U_CAPI int32_t U_EXPORT2 +uspoof_checkUTF8(const USpoofChecker *sc, + const char *id, int32_t length, + int32_t *position, + UErrorCode *status); + + +/** + * Check the specified string for possible security issues. + * The text to be checked will typically be an identifier of some sort. + * The set of checks to be performed is specified with uspoof_setChecks(). + * + * @param sc The USpoofChecker + * @param id The identifier to be checked for possible security issues, + * in UTF-16 format. + * @param length the length of the string to be checked, or -1 if the string is + * zero terminated. + * @param checkResult An instance of USpoofCheckResult to be filled with + * details about the identifier. Can be NULL. + * @param status The error code, set if an error occurred while attempting to + * perform the check. + * Spoofing or security issues detected with the input string are + * not reported here, but through the function's return value. + * @return An integer value with bits set for any potential security + * or spoofing issues detected. The bits are defined by + * enum USpoofChecks. (returned_value & USPOOF_ALL_CHECKS) + * will be zero if the input string passes all of the + * enabled checks. Any information in this bitmask will be + * consistent with the information saved in the optional + * checkResult parameter. + * @see uspoof_openCheckResult + * @see uspoof_check2UTF8 + * @see uspoof_check2UnicodeString + * @stable ICU 58 + */ +U_CAPI int32_t U_EXPORT2 +uspoof_check2(const USpoofChecker *sc, + const UChar* id, int32_t length, + USpoofCheckResult* checkResult, + UErrorCode *status); + +/** + * Check the specified string for possible security issues. + * The text to be checked will typically be an identifier of some sort. + * The set of checks to be performed is specified with uspoof_setChecks(). + * + * This version of {@link uspoof_check} accepts a USpoofCheckResult, which + * returns additional information about the identifier. For more + * information, see {@link uspoof_openCheckResult}. + * + * @param sc The USpoofChecker + * @param id A identifier to be checked for possible security issues, in UTF8 format. + * @param length the length of the string to be checked, or -1 if the string is + * zero terminated. + * @param checkResult An instance of USpoofCheckResult to be filled with + * details about the identifier. Can be NULL. + * @param status The error code, set if an error occurred while attempting to + * perform the check. + * Spoofing or security issues detected with the input string are + * not reported here, but through the function's return value. + * @return An integer value with bits set for any potential security + * or spoofing issues detected. The bits are defined by + * enum USpoofChecks. (returned_value & USPOOF_ALL_CHECKS) + * will be zero if the input string passes all of the + * enabled checks. Any information in this bitmask will be + * consistent with the information saved in the optional + * checkResult parameter. + * @see uspoof_openCheckResult + * @see uspoof_check2 + * @see uspoof_check2UnicodeString + * @stable ICU 58 + */ +U_CAPI int32_t U_EXPORT2 +uspoof_check2UTF8(const USpoofChecker *sc, + const char *id, int32_t length, + USpoofCheckResult* checkResult, + UErrorCode *status); + +/** + * Create a USpoofCheckResult, used by the {@link uspoof_check2} class of functions to return + * information about the identifier. Information includes: + *
    + *
  • A bitmask of the checks that failed
  • + *
  • The identifier's restriction level (UTS 39 section 5.2)
  • + *
  • The set of numerics in the string (UTS 39 section 5.3)
  • + *
+ * The data held in a USpoofCheckResult is cleared whenever it is passed into a new call + * of {@link uspoof_check2}. + * + * @param status The error code, set if this function encounters a problem. + * @return the newly created USpoofCheckResult + * @see uspoof_check2 + * @see uspoof_check2UTF8 + * @see uspoof_check2UnicodeString + * @stable ICU 58 + */ +U_CAPI USpoofCheckResult* U_EXPORT2 +uspoof_openCheckResult(UErrorCode *status); + +/** + * Close a USpoofCheckResult, freeing any memory that was being held by + * its implementation. + * + * @param checkResult The instance of USpoofCheckResult to close + * @stable ICU 58 + */ +U_CAPI void U_EXPORT2 +uspoof_closeCheckResult(USpoofCheckResult *checkResult); + +/** + * Indicates which of the spoof check(s) have failed. The value is a bitwise OR of the constants for the tests + * in question: USPOOF_RESTRICTION_LEVEL, USPOOF_CHAR_LIMIT, and so on. + * + * @param checkResult The instance of USpoofCheckResult created by {@link uspoof_openCheckResult} + * @param status The error code, set if an error occurred. + * @return An integer value with bits set for any potential security + * or spoofing issues detected. The bits are defined by + * enum USpoofChecks. (returned_value & USPOOF_ALL_CHECKS) + * will be zero if the input string passes all of the + * enabled checks. + * @see uspoof_setChecks + * @stable ICU 58 + */ +U_CAPI int32_t U_EXPORT2 +uspoof_getCheckResultChecks(const USpoofCheckResult *checkResult, UErrorCode *status); + +/** + * Gets the restriction level that the text meets, if the USPOOF_RESTRICTION_LEVEL check + * was enabled; otherwise, undefined. + * + * @param checkResult The instance of USpoofCheckResult created by {@link uspoof_openCheckResult} + * @param status The error code, set if an error occurred. + * @return The restriction level contained in the USpoofCheckResult + * @see uspoof_setRestrictionLevel + * @stable ICU 58 + */ +U_CAPI URestrictionLevel U_EXPORT2 +uspoof_getCheckResultRestrictionLevel(const USpoofCheckResult *checkResult, UErrorCode *status); + +/** + * Gets the set of numerics found in the string, if the USPOOF_MIXED_NUMBERS check was enabled; + * otherwise, undefined. The set will contain the zero digit from each decimal number system found + * in the input string. Ownership of the returned USet remains with the USpoofCheckResult. + * The USet will be free'd when {@link uspoof_closeCheckResult} is called. + * + * @param checkResult The instance of USpoofCheckResult created by {@link uspoof_openCheckResult} + * @return The set of numerics contained in the USpoofCheckResult + * @param status The error code, set if an error occurred. + * @stable ICU 58 + */ +U_CAPI const USet* U_EXPORT2 +uspoof_getCheckResultNumerics(const USpoofCheckResult *checkResult, UErrorCode *status); + + +/** + * Check the whether two specified strings are visually confusable. + * + * If the strings are confusable, the return value will be nonzero, as long as + * {@link USPOOF_CONFUSABLE} was enabled in uspoof_setChecks(). + * + * The bits in the return value correspond to flags for each of the classes of + * confusables applicable to the two input strings. According to UTS 39 + * section 4, the possible flags are: + * + *
    + *
  • {@link USPOOF_SINGLE_SCRIPT_CONFUSABLE}
  • + *
  • {@link USPOOF_MIXED_SCRIPT_CONFUSABLE}
  • + *
  • {@link USPOOF_WHOLE_SCRIPT_CONFUSABLE}
  • + *
+ * + * If one or more of the above flags were not listed in uspoof_setChecks(), this + * function will never report that class of confusable. The check + * {@link USPOOF_CONFUSABLE} enables all three flags. + * + * + * @param sc The USpoofChecker + * @param id1 The first of the two identifiers to be compared for + * confusability. The strings are in UTF-16 format. + * @param length1 the length of the first identifier, expressed in + * 16 bit UTF-16 code units, or -1 if the string is + * nul terminated. + * @param id2 The second of the two identifiers to be compared for + * confusability. The identifiers are in UTF-16 format. + * @param length2 The length of the second identifiers, expressed in + * 16 bit UTF-16 code units, or -1 if the string is + * nul terminated. + * @param status The error code, set if an error occurred while attempting to + * perform the check. + * Confusability of the identifiers is not reported here, + * but through this function's return value. + * @return An integer value with bit(s) set corresponding to + * the type of confusability found, as defined by + * enum USpoofChecks. Zero is returned if the identifiers + * are not confusable. + * + * @stable ICU 4.2 + */ +U_CAPI int32_t U_EXPORT2 +uspoof_areConfusable(const USpoofChecker *sc, + const UChar *id1, int32_t length1, + const UChar *id2, int32_t length2, + UErrorCode *status); + + + +/** + * A version of {@link uspoof_areConfusable} accepting strings in UTF-8 format. + * + * @param sc The USpoofChecker + * @param id1 The first of the two identifiers to be compared for + * confusability. The strings are in UTF-8 format. + * @param length1 the length of the first identifiers, in bytes, or -1 + * if the string is nul terminated. + * @param id2 The second of the two identifiers to be compared for + * confusability. The strings are in UTF-8 format. + * @param length2 The length of the second string in bytes, or -1 + * if the string is nul terminated. + * @param status The error code, set if an error occurred while attempting to + * perform the check. + * Confusability of the strings is not reported here, + * but through this function's return value. + * @return An integer value with bit(s) set corresponding to + * the type of confusability found, as defined by + * enum USpoofChecks. Zero is returned if the strings + * are not confusable. + * + * @stable ICU 4.2 + * + * @see uspoof_areConfusable + */ +U_CAPI int32_t U_EXPORT2 +uspoof_areConfusableUTF8(const USpoofChecker *sc, + const char *id1, int32_t length1, + const char *id2, int32_t length2, + UErrorCode *status); + + + + +/** + * Get the "skeleton" for an identifier. + * Skeletons are a transformation of the input identifier; + * Two identifiers are confusable if their skeletons are identical. + * See Unicode UAX #39 for additional information. + * + * Using skeletons directly makes it possible to quickly check + * whether an identifier is confusable with any of some large + * set of existing identifiers, by creating an efficiently + * searchable collection of the skeletons. + * + * @param sc The USpoofChecker + * @param type Deprecated in ICU 58. You may pass any number. + * Originally, controlled which of the Unicode confusable data + * tables to use. + * @param id The input identifier whose skeleton will be computed. + * @param length The length of the input identifier, expressed in 16 bit + * UTF-16 code units, or -1 if the string is zero terminated. + * @param dest The output buffer, to receive the skeleton string. + * @param destCapacity The length of the output buffer, in 16 bit units. + * The destCapacity may be zero, in which case the function will + * return the actual length of the skeleton. + * @param status The error code, set if an error occurred while attempting to + * perform the check. + * @return The length of the skeleton string. The returned length + * is always that of the complete skeleton, even when the + * supplied buffer is too small (or of zero length) + * + * @stable ICU 4.2 + * @see uspoof_areConfusable + */ +U_CAPI int32_t U_EXPORT2 +uspoof_getSkeleton(const USpoofChecker *sc, + uint32_t type, + const UChar *id, int32_t length, + UChar *dest, int32_t destCapacity, + UErrorCode *status); + +/** + * Get the "skeleton" for an identifier. + * Skeletons are a transformation of the input identifier; + * Two identifiers are confusable if their skeletons are identical. + * See Unicode UAX #39 for additional information. + * + * Using skeletons directly makes it possible to quickly check + * whether an identifier is confusable with any of some large + * set of existing identifiers, by creating an efficiently + * searchable collection of the skeletons. + * + * @param sc The USpoofChecker + * @param type Deprecated in ICU 58. You may pass any number. + * Originally, controlled which of the Unicode confusable data + * tables to use. + * @param id The UTF-8 format identifier whose skeleton will be computed. + * @param length The length of the input string, in bytes, + * or -1 if the string is zero terminated. + * @param dest The output buffer, to receive the skeleton string. + * @param destCapacity The length of the output buffer, in bytes. + * The destCapacity may be zero, in which case the function will + * return the actual length of the skeleton. + * @param status The error code, set if an error occurred while attempting to + * perform the check. Possible Errors include U_INVALID_CHAR_FOUND + * for invalid UTF-8 sequences, and + * U_BUFFER_OVERFLOW_ERROR if the destination buffer is too small + * to hold the complete skeleton. + * @return The length of the skeleton string, in bytes. The returned length + * is always that of the complete skeleton, even when the + * supplied buffer is too small (or of zero length) + * + * @stable ICU 4.2 + */ +U_CAPI int32_t U_EXPORT2 +uspoof_getSkeletonUTF8(const USpoofChecker *sc, + uint32_t type, + const char *id, int32_t length, + char *dest, int32_t destCapacity, + UErrorCode *status); + +/** + * Get the set of Candidate Characters for Inclusion in Identifiers, as defined + * in http://unicode.org/Public/security/latest/xidmodifications.txt + * and documented in http://www.unicode.org/reports/tr39/, Unicode Security Mechanisms. + * + * The returned set is frozen. Ownership of the set remains with the ICU library; it must not + * be deleted by the caller. + * + * @param status The error code, set if a problem occurs while creating the set. + * + * @stable ICU 51 + */ +U_CAPI const USet * U_EXPORT2 +uspoof_getInclusionSet(UErrorCode *status); + +/** + * Get the set of characters from Recommended Scripts for Inclusion in Identifiers, as defined + * in http://unicode.org/Public/security/latest/xidmodifications.txt + * and documented in http://www.unicode.org/reports/tr39/, Unicode Security Mechanisms. + * + * The returned set is frozen. Ownership of the set remains with the ICU library; it must not + * be deleted by the caller. + * + * @param status The error code, set if a problem occurs while creating the set. + * + * @stable ICU 51 + */ +U_CAPI const USet * U_EXPORT2 +uspoof_getRecommendedSet(UErrorCode *status); + +/** + * Serialize the data for a spoof detector into a chunk of memory. + * The flattened spoof detection tables can later be used to efficiently + * instantiate a new Spoof Detector. + * + * The serialized spoof checker includes only the data compiled from the + * Unicode data tables by uspoof_openFromSource(); it does not include + * include any other state or configuration that may have been set. + * + * @param sc the Spoof Detector whose data is to be serialized. + * @param data a pointer to 32-bit-aligned memory to be filled with the data, + * can be NULL if capacity==0 + * @param capacity the number of bytes available at data, + * or 0 for preflighting + * @param status an in/out ICU UErrorCode; possible errors include: + * - U_BUFFER_OVERFLOW_ERROR if the data storage block is too small for serialization + * - U_ILLEGAL_ARGUMENT_ERROR the data or capacity parameters are bad + * @return the number of bytes written or needed for the spoof data + * + * @see utrie2_openFromSerialized() + * @stable ICU 4.2 + */ +U_CAPI int32_t U_EXPORT2 +uspoof_serialize(USpoofChecker *sc, + void *data, int32_t capacity, + UErrorCode *status); + +U_CDECL_END + +#if U_SHOW_CPLUSPLUS_API + +U_NAMESPACE_BEGIN + +/** + * \class LocalUSpoofCheckerPointer + * "Smart pointer" class, closes a USpoofChecker via uspoof_close(). + * For most methods see the LocalPointerBase base class. + * + * @see LocalPointerBase + * @see LocalPointer + * @stable ICU 4.4 + */ +/** + * \cond + * Note: Doxygen is giving a bogus warning on this U_DEFINE_LOCAL_OPEN_POINTER. + * For now, suppress with a Doxygen cond + */ +U_DEFINE_LOCAL_OPEN_POINTER(LocalUSpoofCheckerPointer, USpoofChecker, uspoof_close); +/** \endcond */ + +/** + * \class LocalUSpoofCheckResultPointer + * "Smart pointer" class, closes a USpoofCheckResult via `uspoof_closeCheckResult()`. + * For most methods see the LocalPointerBase base class. + * + * @see LocalPointerBase + * @see LocalPointer + * @stable ICU 58 + */ + +/** + * \cond + * Note: Doxygen is giving a bogus warning on this U_DEFINE_LOCAL_OPEN_POINTER. + * For now, suppress with a Doxygen cond + */ +U_DEFINE_LOCAL_OPEN_POINTER(LocalUSpoofCheckResultPointer, USpoofCheckResult, uspoof_closeCheckResult); +/** \endcond */ + +U_NAMESPACE_END + +/** + * Limit the acceptable characters to those specified by a Unicode Set. + * Any previously specified character limit is + * is replaced by the new settings. This includes limits on + * characters that were set with the uspoof_setAllowedLocales() function. + * + * The USPOOF_CHAR_LIMIT test is automatically enabled for this + * USoofChecker by this function. + * + * @param sc The USpoofChecker + * @param chars A Unicode Set containing the list of + * characters that are permitted. Ownership of the set + * remains with the caller. The incoming set is cloned by + * this function, so there are no restrictions on modifying + * or deleting the UnicodeSet after calling this function. + * @param status The error code, set if this function encounters a problem. + * @stable ICU 4.2 + */ +U_CAPI void U_EXPORT2 +uspoof_setAllowedUnicodeSet(USpoofChecker *sc, const icu::UnicodeSet *chars, UErrorCode *status); + + +/** + * Get a UnicodeSet for the characters permitted in an identifier. + * This corresponds to the limits imposed by the Set Allowed Characters / + * UnicodeSet functions. Limitations imposed by other checks will not be + * reflected in the set returned by this function. + * + * The returned set will be frozen, meaning that it cannot be modified + * by the caller. + * + * Ownership of the returned set remains with the Spoof Detector. The + * returned set will become invalid if the spoof detector is closed, + * or if a new set of allowed characters is specified. + * + * + * @param sc The USpoofChecker + * @param status The error code, set if this function encounters a problem. + * @return A UnicodeSet containing the characters that are permitted by + * the USPOOF_CHAR_LIMIT test. + * @stable ICU 4.2 + */ +U_CAPI const icu::UnicodeSet * U_EXPORT2 +uspoof_getAllowedUnicodeSet(const USpoofChecker *sc, UErrorCode *status); + +/** + * Check the specified string for possible security issues. + * The text to be checked will typically be an identifier of some sort. + * The set of checks to be performed is specified with uspoof_setChecks(). + * + * \note + * Consider using the newer API, {@link uspoof_check2UnicodeString}, instead. + * The newer API exposes additional information from the check procedure + * and is otherwise identical to this method. + * + * @param sc The USpoofChecker + * @param id A identifier to be checked for possible security issues. + * @param position Deprecated in ICU 51. Always returns zero. + * Originally, an out parameter for the index of the first + * string position that failed a check. + * This parameter may be NULL. + * @param status The error code, set if an error occurred while attempting to + * perform the check. + * Spoofing or security issues detected with the input string are + * not reported here, but through the function's return value. + * @return An integer value with bits set for any potential security + * or spoofing issues detected. The bits are defined by + * enum USpoofChecks. (returned_value & USPOOF_ALL_CHECKS) + * will be zero if the input string passes all of the + * enabled checks. + * @see uspoof_check2UnicodeString + * @stable ICU 4.2 + */ +U_CAPI int32_t U_EXPORT2 +uspoof_checkUnicodeString(const USpoofChecker *sc, + const icu::UnicodeString &id, + int32_t *position, + UErrorCode *status); + +/** + * Check the specified string for possible security issues. + * The text to be checked will typically be an identifier of some sort. + * The set of checks to be performed is specified with uspoof_setChecks(). + * + * @param sc The USpoofChecker + * @param id A identifier to be checked for possible security issues. + * @param checkResult An instance of USpoofCheckResult to be filled with + * details about the identifier. Can be NULL. + * @param status The error code, set if an error occurred while attempting to + * perform the check. + * Spoofing or security issues detected with the input string are + * not reported here, but through the function's return value. + * @return An integer value with bits set for any potential security + * or spoofing issues detected. The bits are defined by + * enum USpoofChecks. (returned_value & USPOOF_ALL_CHECKS) + * will be zero if the input string passes all of the + * enabled checks. Any information in this bitmask will be + * consistent with the information saved in the optional + * checkResult parameter. + * @see uspoof_openCheckResult + * @see uspoof_check2 + * @see uspoof_check2UTF8 + * @stable ICU 58 + */ +U_CAPI int32_t U_EXPORT2 +uspoof_check2UnicodeString(const USpoofChecker *sc, + const icu::UnicodeString &id, + USpoofCheckResult* checkResult, + UErrorCode *status); + +/** + * A version of {@link uspoof_areConfusable} accepting UnicodeStrings. + * + * @param sc The USpoofChecker + * @param s1 The first of the two identifiers to be compared for + * confusability. The strings are in UTF-8 format. + * @param s2 The second of the two identifiers to be compared for + * confusability. The strings are in UTF-8 format. + * @param status The error code, set if an error occurred while attempting to + * perform the check. + * Confusability of the identifiers is not reported here, + * but through this function's return value. + * @return An integer value with bit(s) set corresponding to + * the type of confusability found, as defined by + * enum USpoofChecks. Zero is returned if the identifiers + * are not confusable. + * + * @stable ICU 4.2 + * + * @see uspoof_areConfusable + */ +U_CAPI int32_t U_EXPORT2 +uspoof_areConfusableUnicodeString(const USpoofChecker *sc, + const icu::UnicodeString &s1, + const icu::UnicodeString &s2, + UErrorCode *status); + +/** + * Get the "skeleton" for an identifier. + * Skeletons are a transformation of the input identifier; + * Two identifiers are confusable if their skeletons are identical. + * See Unicode UAX #39 for additional information. + * + * Using skeletons directly makes it possible to quickly check + * whether an identifier is confusable with any of some large + * set of existing identifiers, by creating an efficiently + * searchable collection of the skeletons. + * + * @param sc The USpoofChecker. + * @param type Deprecated in ICU 58. You may pass any number. + * Originally, controlled which of the Unicode confusable data + * tables to use. + * @param id The input identifier whose skeleton will be computed. + * @param dest The output identifier, to receive the skeleton string. + * @param status The error code, set if an error occurred while attempting to + * perform the check. + * @return A reference to the destination (skeleton) string. + * + * @stable ICU 4.2 + */ +U_I18N_API icu::UnicodeString & U_EXPORT2 +uspoof_getSkeletonUnicodeString(const USpoofChecker *sc, + uint32_t type, + const icu::UnicodeString &id, + icu::UnicodeString &dest, + UErrorCode *status); + +/** + * Get the set of Candidate Characters for Inclusion in Identifiers, as defined + * in http://unicode.org/Public/security/latest/xidmodifications.txt + * and documented in http://www.unicode.org/reports/tr39/, Unicode Security Mechanisms. + * + * The returned set is frozen. Ownership of the set remains with the ICU library; it must not + * be deleted by the caller. + * + * @param status The error code, set if a problem occurs while creating the set. + * + * @stable ICU 51 + */ +U_CAPI const icu::UnicodeSet * U_EXPORT2 +uspoof_getInclusionUnicodeSet(UErrorCode *status); + +/** + * Get the set of characters from Recommended Scripts for Inclusion in Identifiers, as defined + * in http://unicode.org/Public/security/latest/xidmodifications.txt + * and documented in http://www.unicode.org/reports/tr39/, Unicode Security Mechanisms. + * + * The returned set is frozen. Ownership of the set remains with the ICU library; it must not + * be deleted by the caller. + * + * @param status The error code, set if a problem occurs while creating the set. + * + * @stable ICU 51 + */ +U_CAPI const icu::UnicodeSet * U_EXPORT2 +uspoof_getRecommendedUnicodeSet(UErrorCode *status); + +#endif /* U_SHOW_CPLUSPLUS_API */ + +#endif /* UCONFIG_NO_NORMALIZATION */ + +#endif /* USPOOF_H */ diff --git a/thirdparty/icu4c/i18n/uspoof.cpp b/thirdparty/icu4c/i18n/uspoof.cpp new file mode 100644 index 00000000000..dd4618baa70 --- /dev/null +++ b/thirdparty/icu4c/i18n/uspoof.cpp @@ -0,0 +1,839 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* +*************************************************************************** +* Copyright (C) 2008-2015, International Business Machines Corporation +* and others. All Rights Reserved. +*************************************************************************** +* file name: uspoof.cpp +* encoding: UTF-8 +* tab size: 8 (not used) +* indentation:4 +* +* created on: 2008Feb13 +* created by: Andy Heninger +* +* Unicode Spoof Detection +*/ +#include "unicode/utypes.h" +#include "unicode/normalizer2.h" +#include "unicode/uspoof.h" +#include "unicode/ustring.h" +#include "unicode/utf16.h" +#include "cmemory.h" +#include "cstring.h" +#include "mutex.h" +#include "scriptset.h" +#include "uassert.h" +#include "ucln_in.h" +#include "uspoof_impl.h" +#include "umutex.h" + + +#if !UCONFIG_NO_NORMALIZATION + +U_NAMESPACE_USE + + +// +// Static Objects used by the spoof impl, their thread safe initialization and their cleanup. +// +static UnicodeSet *gInclusionSet = NULL; +static UnicodeSet *gRecommendedSet = NULL; +static const Normalizer2 *gNfdNormalizer = NULL; +static UInitOnce gSpoofInitStaticsOnce = U_INITONCE_INITIALIZER; + +namespace { + +UBool U_CALLCONV +uspoof_cleanup(void) { + delete gInclusionSet; + gInclusionSet = NULL; + delete gRecommendedSet; + gRecommendedSet = NULL; + gNfdNormalizer = NULL; + gSpoofInitStaticsOnce.reset(); + return TRUE; +} + +void U_CALLCONV initializeStatics(UErrorCode &status) { + static const char16_t *inclusionPat = + u"['\\-.\\:\\u00B7\\u0375\\u058A\\u05F3\\u05F4\\u06FD\\u06FE\\u0F0B\\u200C" + u"\\u200D\\u2010\\u2019\\u2027\\u30A0\\u30FB]"; + gInclusionSet = new UnicodeSet(UnicodeString(inclusionPat), status); + if (gInclusionSet == NULL) { + status = U_MEMORY_ALLOCATION_ERROR; + return; + } + gInclusionSet->freeze(); + + // Note: data from IdentifierStatus.txt & IdentifierType.txt + // There is tooling to generate this constant in the unicodetools project: + // org.unicode.text.tools.RecommendedSetGenerator + // It will print the Java and C++ code to the console for easy copy-paste into this file. + static const char16_t *recommendedPat = + u"[0-9A-Z_a-z\\u00C0-\\u00D6\\u00D8-\\u00F6\\u00F8-\\u0131\\u0134-\\u013E" + u"\\u0141-\\u0148\\u014A-\\u017E\\u018F\\u01A0\\u01A1\\u01AF\\u01B0\\u01CD-" + u"\\u01DC\\u01DE-\\u01E3\\u01E6-\\u01F0\\u01F4\\u01F5\\u01F8-\\u021B\\u021E" + u"\\u021F\\u0226-\\u0233\\u0259\\u02BB\\u02BC\\u02EC\\u0300-\\u0304\\u0306-" + u"\\u030C\\u030F-\\u0311\\u0313\\u0314\\u031B\\u0323-\\u0328\\u032D\\u032E" + u"\\u0330\\u0331\\u0335\\u0338\\u0339\\u0342\\u0345\\u037B-\\u037D\\u0386" + u"\\u0388-\\u038A\\u038C\\u038E-\\u03A1\\u03A3-\\u03CE\\u03FC-\\u045F\\u048A-" + u"\\u04FF\\u0510-\\u0529\\u052E\\u052F\\u0531-\\u0556\\u0559\\u0561-\\u0586" + u"\\u05B4\\u05D0-\\u05EA\\u05EF-\\u05F2\\u0620-\\u063F\\u0641-\\u0655\\u0660-" + u"\\u0669\\u0670-\\u0672\\u0674\\u0679-\\u068D\\u068F-\\u06A0\\u06A2-\\u06D3" + u"\\u06D5\\u06E5\\u06E6\\u06EE-\\u06FC\\u06FF\\u0750-\\u07B1\\u0870-\\u0887" + u"\\u0889-\\u088E\\u08A0-\\u08AC\\u08B2\\u08B5-\\u08C9\\u0901-\\u094D\\u094F" + u"\\u0950\\u0956\\u0957\\u0960-\\u0963\\u0966-\\u096F\\u0971-\\u0977\\u0979-" + u"\\u097F\\u0981-\\u0983\\u0985-\\u098C\\u098F\\u0990\\u0993-\\u09A8\\u09AA-" + u"\\u09B0\\u09B2\\u09B6-\\u09B9\\u09BC-\\u09C4\\u09C7\\u09C8\\u09CB-\\u09CE" + u"\\u09D7\\u09E0-\\u09E3\\u09E6-\\u09F1\\u09FE\\u0A01-\\u0A03\\u0A05-\\u0A0A" + u"\\u0A0F\\u0A10\\u0A13-\\u0A28\\u0A2A-\\u0A30\\u0A32\\u0A35\\u0A38\\u0A39" + u"\\u0A3C\\u0A3E-\\u0A42\\u0A47\\u0A48\\u0A4B-\\u0A4D\\u0A5C\\u0A66-\\u0A74" + u"\\u0A81-\\u0A83\\u0A85-\\u0A8D\\u0A8F-\\u0A91\\u0A93-\\u0AA8\\u0AAA-\\u0AB0" + u"\\u0AB2\\u0AB3\\u0AB5-\\u0AB9\\u0ABC-\\u0AC5\\u0AC7-\\u0AC9\\u0ACB-\\u0ACD" + u"\\u0AD0\\u0AE0-\\u0AE3\\u0AE6-\\u0AEF\\u0AFA-\\u0AFF\\u0B01-\\u0B03\\u0B05-" + u"\\u0B0C\\u0B0F\\u0B10\\u0B13-\\u0B28\\u0B2A-\\u0B30\\u0B32\\u0B33\\u0B35-" + u"\\u0B39\\u0B3C-\\u0B43\\u0B47\\u0B48\\u0B4B-\\u0B4D\\u0B55-\\u0B57\\u0B5F-" + u"\\u0B61\\u0B66-\\u0B6F\\u0B71\\u0B82\\u0B83\\u0B85-\\u0B8A\\u0B8E-\\u0B90" + u"\\u0B92-\\u0B95\\u0B99\\u0B9A\\u0B9C\\u0B9E\\u0B9F\\u0BA3\\u0BA4\\u0BA8-" + u"\\u0BAA\\u0BAE-\\u0BB9\\u0BBE-\\u0BC2\\u0BC6-\\u0BC8\\u0BCA-\\u0BCD\\u0BD0" + u"\\u0BD7\\u0BE6-\\u0BEF\\u0C01-\\u0C0C\\u0C0E-\\u0C10\\u0C12-\\u0C28\\u0C2A-" + u"\\u0C33\\u0C35-\\u0C39\\u0C3C-\\u0C44\\u0C46-\\u0C48\\u0C4A-\\u0C4D\\u0C55" + u"\\u0C56\\u0C5D\\u0C60\\u0C61\\u0C66-\\u0C6F\\u0C80\\u0C82\\u0C83\\u0C85-" + u"\\u0C8C\\u0C8E-\\u0C90\\u0C92-\\u0CA8\\u0CAA-\\u0CB3\\u0CB5-\\u0CB9\\u0CBC-" + u"\\u0CC4\\u0CC6-\\u0CC8\\u0CCA-\\u0CCD\\u0CD5\\u0CD6\\u0CDD\\u0CE0-\\u0CE3" + u"\\u0CE6-\\u0CEF\\u0CF1\\u0CF2\\u0D00\\u0D02\\u0D03\\u0D05-\\u0D0C\\u0D0E-" + u"\\u0D10\\u0D12-\\u0D3A\\u0D3D-\\u0D43\\u0D46-\\u0D48\\u0D4A-\\u0D4E\\u0D54-" + u"\\u0D57\\u0D60\\u0D61\\u0D66-\\u0D6F\\u0D7A-\\u0D7F\\u0D82\\u0D83\\u0D85-" + u"\\u0D8E\\u0D91-\\u0D96\\u0D9A-\\u0DA5\\u0DA7-\\u0DB1\\u0DB3-\\u0DBB\\u0DBD" + u"\\u0DC0-\\u0DC6\\u0DCA\\u0DCF-\\u0DD4\\u0DD6\\u0DD8-\\u0DDE\\u0DF2\\u0E01-" + u"\\u0E32\\u0E34-\\u0E3A\\u0E40-\\u0E4E\\u0E50-\\u0E59\\u0E81\\u0E82\\u0E84" + u"\\u0E86-\\u0E8A\\u0E8C-\\u0EA3\\u0EA5\\u0EA7-\\u0EB2\\u0EB4-\\u0EBD\\u0EC0-" + u"\\u0EC4\\u0EC6\\u0EC8-\\u0ECD\\u0ED0-\\u0ED9\\u0EDE\\u0EDF\\u0F00\\u0F20-" + u"\\u0F29\\u0F35\\u0F37\\u0F3E-\\u0F42\\u0F44-\\u0F47\\u0F49-\\u0F4C\\u0F4E-" + u"\\u0F51\\u0F53-\\u0F56\\u0F58-\\u0F5B\\u0F5D-\\u0F68\\u0F6A-\\u0F6C\\u0F71" + u"\\u0F72\\u0F74\\u0F7A-\\u0F80\\u0F82-\\u0F84\\u0F86-\\u0F92\\u0F94-\\u0F97" + u"\\u0F99-\\u0F9C\\u0F9E-\\u0FA1\\u0FA3-\\u0FA6\\u0FA8-\\u0FAB\\u0FAD-\\u0FB8" + u"\\u0FBA-\\u0FBC\\u0FC6\\u1000-\\u1049\\u1050-\\u109D\\u10C7\\u10CD\\u10D0-" + u"\\u10F0\\u10F7-\\u10FA\\u10FD-\\u10FF\\u1200-\\u1248\\u124A-\\u124D\\u1250-" + u"\\u1256\\u1258\\u125A-\\u125D\\u1260-\\u1288\\u128A-\\u128D\\u1290-\\u12B0" + u"\\u12B2-\\u12B5\\u12B8-\\u12BE\\u12C0\\u12C2-\\u12C5\\u12C8-\\u12D6\\u12D8-" + u"\\u1310\\u1312-\\u1315\\u1318-\\u135A\\u135D-\\u135F\\u1380-\\u138F\\u1780-" + u"\\u17A2\\u17A5-\\u17A7\\u17A9-\\u17B3\\u17B6-\\u17CD\\u17D0\\u17D2\\u17D7" + u"\\u17DC\\u17E0-\\u17E9\\u1C90-\\u1CBA\\u1CBD-\\u1CBF\\u1E00-\\u1E99\\u1E9E" + u"\\u1EA0-\\u1EF9\\u1F00-\\u1F15\\u1F18-\\u1F1D\\u1F20-\\u1F45\\u1F48-\\u1F4D" + u"\\u1F50-\\u1F57\\u1F59\\u1F5B\\u1F5D\\u1F5F-\\u1F70\\u1F72\\u1F74\\u1F76" + u"\\u1F78\\u1F7A\\u1F7C\\u1F80-\\u1FB4\\u1FB6-\\u1FBA\\u1FBC\\u1FC2-\\u1FC4" + u"\\u1FC6-\\u1FC8\\u1FCA\\u1FCC\\u1FD0-\\u1FD2\\u1FD6-\\u1FDA\\u1FE0-\\u1FE2" + u"\\u1FE4-\\u1FEA\\u1FEC\\u1FF2-\\u1FF4\\u1FF6-\\u1FF8\\u1FFA\\u1FFC\\u2D27" + u"\\u2D2D\\u2D80-\\u2D96\\u2DA0-\\u2DA6\\u2DA8-\\u2DAE\\u2DB0-\\u2DB6\\u2DB8-" + u"\\u2DBE\\u2DC0-\\u2DC6\\u2DC8-\\u2DCE\\u2DD0-\\u2DD6\\u2DD8-\\u2DDE\\u3005-" + u"\\u3007\\u3041-\\u3096\\u3099\\u309A\\u309D\\u309E\\u30A1-\\u30FA\\u30FC-" + u"\\u30FE\\u3105-\\u312D\\u312F\\u31A0-\\u31BF\\u3400-\\u4DBF\\u4E00-\\u9FFF" + u"\\uA67F\\uA717-\\uA71F\\uA788\\uA78D\\uA792\\uA793\\uA7AA\\uA7AE\\uA7B8" + u"\\uA7B9\\uA7C0-\\uA7CA\\uA7D0\\uA7D1\\uA7D3\\uA7D5-\\uA7D9\\uA9E7-\\uA9FE" + u"\\uAA60-\\uAA76\\uAA7A-\\uAA7F\\uAB01-\\uAB06\\uAB09-\\uAB0E\\uAB11-\\uAB16" + u"\\uAB20-\\uAB26\\uAB28-\\uAB2E\\uAB66\\uAB67\\uAC00-\\uD7A3\\uFA0E\\uFA0F" + u"\\uFA11\\uFA13\\uFA14\\uFA1F\\uFA21\\uFA23\\uFA24\\uFA27-\\uFA29\\U00011301" + u"\\U00011303\\U0001133B\\U0001133C\\U00016FF0\\U00016FF1\\U0001B11F-" + u"\\U0001B122\\U0001B150-\\U0001B152\\U0001B164-\\U0001B167\\U0001DF00-" + u"\\U0001DF1E\\U0001E7E0-\\U0001E7E6\\U0001E7E8-\\U0001E7EB\\U0001E7ED" + u"\\U0001E7EE\\U0001E7F0-\\U0001E7FE\\U00020000-\\U0002A6DF\\U0002A700-" + u"\\U0002B738\\U0002B740-\\U0002B81D\\U0002B820-\\U0002CEA1\\U0002CEB0-" + u"\\U0002EBE0\\U00030000-\\U0003134A]"; + + gRecommendedSet = new UnicodeSet(UnicodeString(recommendedPat), status); + if (gRecommendedSet == NULL) { + status = U_MEMORY_ALLOCATION_ERROR; + delete gInclusionSet; + return; + } + gRecommendedSet->freeze(); + gNfdNormalizer = Normalizer2::getNFDInstance(status); + ucln_i18n_registerCleanup(UCLN_I18N_SPOOF, uspoof_cleanup); +} + +} // namespace + +U_CFUNC void uspoof_internalInitStatics(UErrorCode *status) { + umtx_initOnce(gSpoofInitStaticsOnce, &initializeStatics, *status); +} + +U_CAPI USpoofChecker * U_EXPORT2 +uspoof_open(UErrorCode *status) { + umtx_initOnce(gSpoofInitStaticsOnce, &initializeStatics, *status); + if (U_FAILURE(*status)) { + return NULL; + } + SpoofImpl *si = new SpoofImpl(*status); + if (si == NULL) { + *status = U_MEMORY_ALLOCATION_ERROR; + return NULL; + } + if (U_FAILURE(*status)) { + delete si; + return NULL; + } + return si->asUSpoofChecker(); +} + + +U_CAPI USpoofChecker * U_EXPORT2 +uspoof_openFromSerialized(const void *data, int32_t length, int32_t *pActualLength, + UErrorCode *status) { + if (U_FAILURE(*status)) { + return NULL; + } + + if (data == NULL) { + *status = U_ILLEGAL_ARGUMENT_ERROR; + return NULL; + } + + umtx_initOnce(gSpoofInitStaticsOnce, &initializeStatics, *status); + if (U_FAILURE(*status)) + { + return NULL; + } + + SpoofData *sd = new SpoofData(data, length, *status); + if (sd == NULL) { + *status = U_MEMORY_ALLOCATION_ERROR; + return NULL; + } + + if (U_FAILURE(*status)) { + delete sd; + return NULL; + } + + SpoofImpl *si = new SpoofImpl(sd, *status); + if (si == NULL) { + *status = U_MEMORY_ALLOCATION_ERROR; + delete sd; // explicit delete as the destructor for si won't be called. + return NULL; + } + + if (U_FAILURE(*status)) { + delete si; // no delete for sd, as the si destructor will delete it. + return NULL; + } + + if (pActualLength != NULL) { + *pActualLength = sd->size(); + } + return si->asUSpoofChecker(); +} + + +U_CAPI USpoofChecker * U_EXPORT2 +uspoof_clone(const USpoofChecker *sc, UErrorCode *status) { + const SpoofImpl *src = SpoofImpl::validateThis(sc, *status); + if (src == NULL) { + return NULL; + } + SpoofImpl *result = new SpoofImpl(*src, *status); // copy constructor + if (result == NULL) { + *status = U_MEMORY_ALLOCATION_ERROR; + return NULL; + } + if (U_FAILURE(*status)) { + delete result; + result = NULL; + } + return result->asUSpoofChecker(); +} + + +U_CAPI void U_EXPORT2 +uspoof_close(USpoofChecker *sc) { + UErrorCode status = U_ZERO_ERROR; + SpoofImpl *This = SpoofImpl::validateThis(sc, status); + delete This; +} + + +U_CAPI void U_EXPORT2 +uspoof_setChecks(USpoofChecker *sc, int32_t checks, UErrorCode *status) { + SpoofImpl *This = SpoofImpl::validateThis(sc, *status); + if (This == NULL) { + return; + } + + // Verify that the requested checks are all ones (bits) that + // are acceptable, known values. + if (checks & ~(USPOOF_ALL_CHECKS | USPOOF_AUX_INFO)) { + *status = U_ILLEGAL_ARGUMENT_ERROR; + return; + } + + This->fChecks = checks; +} + + +U_CAPI int32_t U_EXPORT2 +uspoof_getChecks(const USpoofChecker *sc, UErrorCode *status) { + const SpoofImpl *This = SpoofImpl::validateThis(sc, *status); + if (This == NULL) { + return 0; + } + return This->fChecks; +} + +U_CAPI void U_EXPORT2 +uspoof_setRestrictionLevel(USpoofChecker *sc, URestrictionLevel restrictionLevel) { + UErrorCode status = U_ZERO_ERROR; + SpoofImpl *This = SpoofImpl::validateThis(sc, status); + if (This != NULL) { + This->fRestrictionLevel = restrictionLevel; + This->fChecks |= USPOOF_RESTRICTION_LEVEL; + } +} + +U_CAPI URestrictionLevel U_EXPORT2 +uspoof_getRestrictionLevel(const USpoofChecker *sc) { + UErrorCode status = U_ZERO_ERROR; + const SpoofImpl *This = SpoofImpl::validateThis(sc, status); + if (This == NULL) { + return USPOOF_UNRESTRICTIVE; + } + return This->fRestrictionLevel; +} + +U_CAPI void U_EXPORT2 +uspoof_setAllowedLocales(USpoofChecker *sc, const char *localesList, UErrorCode *status) { + SpoofImpl *This = SpoofImpl::validateThis(sc, *status); + if (This == NULL) { + return; + } + This->setAllowedLocales(localesList, *status); +} + +U_CAPI const char * U_EXPORT2 +uspoof_getAllowedLocales(USpoofChecker *sc, UErrorCode *status) { + SpoofImpl *This = SpoofImpl::validateThis(sc, *status); + if (This == NULL) { + return NULL; + } + return This->getAllowedLocales(*status); +} + + +U_CAPI const USet * U_EXPORT2 +uspoof_getAllowedChars(const USpoofChecker *sc, UErrorCode *status) { + const UnicodeSet *result = uspoof_getAllowedUnicodeSet(sc, status); + return result->toUSet(); +} + +U_CAPI const UnicodeSet * U_EXPORT2 +uspoof_getAllowedUnicodeSet(const USpoofChecker *sc, UErrorCode *status) { + const SpoofImpl *This = SpoofImpl::validateThis(sc, *status); + if (This == NULL) { + return NULL; + } + return This->fAllowedCharsSet; +} + + +U_CAPI void U_EXPORT2 +uspoof_setAllowedChars(USpoofChecker *sc, const USet *chars, UErrorCode *status) { + const UnicodeSet *set = UnicodeSet::fromUSet(chars); + uspoof_setAllowedUnicodeSet(sc, set, status); +} + + +U_CAPI void U_EXPORT2 +uspoof_setAllowedUnicodeSet(USpoofChecker *sc, const UnicodeSet *chars, UErrorCode *status) { + SpoofImpl *This = SpoofImpl::validateThis(sc, *status); + if (This == NULL) { + return; + } + if (chars->isBogus()) { + *status = U_ILLEGAL_ARGUMENT_ERROR; + return; + } + UnicodeSet *clonedSet = chars->clone(); + if (clonedSet == NULL || clonedSet->isBogus()) { + *status = U_MEMORY_ALLOCATION_ERROR; + return; + } + clonedSet->freeze(); + delete This->fAllowedCharsSet; + This->fAllowedCharsSet = clonedSet; + This->fChecks |= USPOOF_CHAR_LIMIT; +} + + +U_CAPI int32_t U_EXPORT2 +uspoof_check(const USpoofChecker *sc, + const UChar *id, int32_t length, + int32_t *position, + UErrorCode *status) { + + // Backwards compatibility: + if (position != NULL) { + *position = 0; + } + + // Delegate to uspoof_check2 + return uspoof_check2(sc, id, length, NULL, status); +} + + +U_CAPI int32_t U_EXPORT2 +uspoof_check2(const USpoofChecker *sc, + const UChar* id, int32_t length, + USpoofCheckResult* checkResult, + UErrorCode *status) { + + const SpoofImpl *This = SpoofImpl::validateThis(sc, *status); + if (This == NULL) { + return 0; + } + if (length < -1) { + *status = U_ILLEGAL_ARGUMENT_ERROR; + return 0; + } + UnicodeString idStr((length == -1), id, length); // Aliasing constructor. + int32_t result = uspoof_check2UnicodeString(sc, idStr, checkResult, status); + return result; +} + + +U_CAPI int32_t U_EXPORT2 +uspoof_checkUTF8(const USpoofChecker *sc, + const char *id, int32_t length, + int32_t *position, + UErrorCode *status) { + + // Backwards compatibility: + if (position != NULL) { + *position = 0; + } + + // Delegate to uspoof_check2 + return uspoof_check2UTF8(sc, id, length, NULL, status); +} + + +U_CAPI int32_t U_EXPORT2 +uspoof_check2UTF8(const USpoofChecker *sc, + const char *id, int32_t length, + USpoofCheckResult* checkResult, + UErrorCode *status) { + + if (U_FAILURE(*status)) { + return 0; + } + UnicodeString idStr = UnicodeString::fromUTF8(StringPiece(id, length>=0 ? length : static_cast(uprv_strlen(id)))); + int32_t result = uspoof_check2UnicodeString(sc, idStr, checkResult, status); + return result; +} + + +U_CAPI int32_t U_EXPORT2 +uspoof_areConfusable(const USpoofChecker *sc, + const UChar *id1, int32_t length1, + const UChar *id2, int32_t length2, + UErrorCode *status) { + SpoofImpl::validateThis(sc, *status); + if (U_FAILURE(*status)) { + return 0; + } + if (length1 < -1 || length2 < -1) { + *status = U_ILLEGAL_ARGUMENT_ERROR; + return 0; + } + + UnicodeString id1Str((length1==-1), id1, length1); // Aliasing constructor + UnicodeString id2Str((length2==-1), id2, length2); // Aliasing constructor + return uspoof_areConfusableUnicodeString(sc, id1Str, id2Str, status); +} + + +U_CAPI int32_t U_EXPORT2 +uspoof_areConfusableUTF8(const USpoofChecker *sc, + const char *id1, int32_t length1, + const char *id2, int32_t length2, + UErrorCode *status) { + SpoofImpl::validateThis(sc, *status); + if (U_FAILURE(*status)) { + return 0; + } + if (length1 < -1 || length2 < -1) { + *status = U_ILLEGAL_ARGUMENT_ERROR; + return 0; + } + UnicodeString id1Str = UnicodeString::fromUTF8(StringPiece(id1, length1>=0? length1 : static_cast(uprv_strlen(id1)))); + UnicodeString id2Str = UnicodeString::fromUTF8(StringPiece(id2, length2>=0? length2 : static_cast(uprv_strlen(id2)))); + int32_t results = uspoof_areConfusableUnicodeString(sc, id1Str, id2Str, status); + return results; +} + + +U_CAPI int32_t U_EXPORT2 +uspoof_areConfusableUnicodeString(const USpoofChecker *sc, + const icu::UnicodeString &id1, + const icu::UnicodeString &id2, + UErrorCode *status) { + const SpoofImpl *This = SpoofImpl::validateThis(sc, *status); + if (U_FAILURE(*status)) { + return 0; + } + // + // See section 4 of UAX 39 for the algorithm for checking whether two strings are confusable, + // and for definitions of the types (single, whole, mixed-script) of confusables. + + // We only care about a few of the check flags. Ignore the others. + // If no tests relevant to this function have been specified, return an error. + // TODO: is this really the right thing to do? It's probably an error on the caller's part, + // but logically we would just return 0 (no error). + if ((This->fChecks & USPOOF_CONFUSABLE) == 0) { + *status = U_INVALID_STATE_ERROR; + return 0; + } + + // Compute the skeletons and check for confusability. + UnicodeString id1Skeleton; + uspoof_getSkeletonUnicodeString(sc, 0 /* deprecated */, id1, id1Skeleton, status); + UnicodeString id2Skeleton; + uspoof_getSkeletonUnicodeString(sc, 0 /* deprecated */, id2, id2Skeleton, status); + if (U_FAILURE(*status)) { return 0; } + if (id1Skeleton != id2Skeleton) { + return 0; + } + + // If we get here, the strings are confusable. Now we just need to set the flags for the appropriate classes + // of confusables according to UTS 39 section 4. + // Start by computing the resolved script sets of id1 and id2. + ScriptSet id1RSS; + This->getResolvedScriptSet(id1, id1RSS, *status); + ScriptSet id2RSS; + This->getResolvedScriptSet(id2, id2RSS, *status); + + // Turn on all applicable flags + int32_t result = 0; + if (id1RSS.intersects(id2RSS)) { + result |= USPOOF_SINGLE_SCRIPT_CONFUSABLE; + } else { + result |= USPOOF_MIXED_SCRIPT_CONFUSABLE; + if (!id1RSS.isEmpty() && !id2RSS.isEmpty()) { + result |= USPOOF_WHOLE_SCRIPT_CONFUSABLE; + } + } + + // Turn off flags that the user doesn't want + if ((This->fChecks & USPOOF_SINGLE_SCRIPT_CONFUSABLE) == 0) { + result &= ~USPOOF_SINGLE_SCRIPT_CONFUSABLE; + } + if ((This->fChecks & USPOOF_MIXED_SCRIPT_CONFUSABLE) == 0) { + result &= ~USPOOF_MIXED_SCRIPT_CONFUSABLE; + } + if ((This->fChecks & USPOOF_WHOLE_SCRIPT_CONFUSABLE) == 0) { + result &= ~USPOOF_WHOLE_SCRIPT_CONFUSABLE; + } + + return result; +} + + +U_CAPI int32_t U_EXPORT2 +uspoof_checkUnicodeString(const USpoofChecker *sc, + const icu::UnicodeString &id, + int32_t *position, + UErrorCode *status) { + + // Backwards compatibility: + if (position != NULL) { + *position = 0; + } + + // Delegate to uspoof_check2 + return uspoof_check2UnicodeString(sc, id, NULL, status); +} + +namespace { + +int32_t checkImpl(const SpoofImpl* This, const UnicodeString& id, CheckResult* checkResult, UErrorCode* status) { + U_ASSERT(This != NULL); + U_ASSERT(checkResult != NULL); + checkResult->clear(); + int32_t result = 0; + + if (0 != (This->fChecks & USPOOF_RESTRICTION_LEVEL)) { + URestrictionLevel idRestrictionLevel = This->getRestrictionLevel(id, *status); + if (idRestrictionLevel > This->fRestrictionLevel) { + result |= USPOOF_RESTRICTION_LEVEL; + } + checkResult->fRestrictionLevel = idRestrictionLevel; + } + + if (0 != (This->fChecks & USPOOF_MIXED_NUMBERS)) { + UnicodeSet numerics; + This->getNumerics(id, numerics, *status); + if (numerics.size() > 1) { + result |= USPOOF_MIXED_NUMBERS; + } + checkResult->fNumerics = numerics; // UnicodeSet::operator= + } + + if (0 != (This->fChecks & USPOOF_HIDDEN_OVERLAY)) { + int32_t index = This->findHiddenOverlay(id, *status); + if (index != -1) { + result |= USPOOF_HIDDEN_OVERLAY; + } + } + + + if (0 != (This->fChecks & USPOOF_CHAR_LIMIT)) { + int32_t i; + UChar32 c; + int32_t length = id.length(); + for (i=0; ifAllowedCharsSet->contains(c)) { + result |= USPOOF_CHAR_LIMIT; + break; + } + } + } + + if (0 != (This->fChecks & USPOOF_INVISIBLE)) { + // This check needs to be done on NFD input + UnicodeString nfdText; + gNfdNormalizer->normalize(id, nfdText, *status); + int32_t nfdLength = nfdText.length(); + + // scan for more than one occurrence of the same non-spacing mark + // in a sequence of non-spacing marks. + int32_t i; + UChar32 c; + UChar32 firstNonspacingMark = 0; + UBool haveMultipleMarks = FALSE; + UnicodeSet marksSeenSoFar; // Set of combining marks in a single combining sequence. + + for (i=0; ifChecks = result; + return checkResult->toCombinedBitmask(This->fChecks); +} + +} // namespace + +U_CAPI int32_t U_EXPORT2 +uspoof_check2UnicodeString(const USpoofChecker *sc, + const icu::UnicodeString &id, + USpoofCheckResult* checkResult, + UErrorCode *status) { + const SpoofImpl *This = SpoofImpl::validateThis(sc, *status); + if (This == NULL) { + return FALSE; + } + + if (checkResult != NULL) { + CheckResult* ThisCheckResult = CheckResult::validateThis(checkResult, *status); + if (ThisCheckResult == NULL) { + return FALSE; + } + return checkImpl(This, id, ThisCheckResult, status); + } else { + // Stack-allocate the checkResult since this method doesn't return it + CheckResult stackCheckResult; + return checkImpl(This, id, &stackCheckResult, status); + } +} + + +U_CAPI int32_t U_EXPORT2 +uspoof_getSkeleton(const USpoofChecker *sc, + uint32_t type, + const UChar *id, int32_t length, + UChar *dest, int32_t destCapacity, + UErrorCode *status) { + + SpoofImpl::validateThis(sc, *status); + if (U_FAILURE(*status)) { + return 0; + } + if (length<-1 || destCapacity<0 || (destCapacity==0 && dest!=NULL)) { + *status = U_ILLEGAL_ARGUMENT_ERROR; + return 0; + } + + UnicodeString idStr((length==-1), id, length); // Aliasing constructor + UnicodeString destStr; + uspoof_getSkeletonUnicodeString(sc, type, idStr, destStr, status); + destStr.extract(dest, destCapacity, *status); + return destStr.length(); +} + + + +U_I18N_API UnicodeString & U_EXPORT2 +uspoof_getSkeletonUnicodeString(const USpoofChecker *sc, + uint32_t /*type*/, + const UnicodeString &id, + UnicodeString &dest, + UErrorCode *status) { + const SpoofImpl *This = SpoofImpl::validateThis(sc, *status); + if (U_FAILURE(*status)) { + return dest; + } + + UnicodeString nfdId; + gNfdNormalizer->normalize(id, nfdId, *status); + + // Apply the skeleton mapping to the NFD normalized input string + // Accumulate the skeleton, possibly unnormalized, in a UnicodeString. + int32_t inputIndex = 0; + UnicodeString skelStr; + int32_t normalizedLen = nfdId.length(); + for (inputIndex=0; inputIndex < normalizedLen; ) { + UChar32 c = nfdId.char32At(inputIndex); + inputIndex += U16_LENGTH(c); + This->fSpoofData->confusableLookup(c, skelStr); + } + + gNfdNormalizer->normalize(skelStr, dest, *status); + return dest; +} + + +U_CAPI int32_t U_EXPORT2 +uspoof_getSkeletonUTF8(const USpoofChecker *sc, + uint32_t type, + const char *id, int32_t length, + char *dest, int32_t destCapacity, + UErrorCode *status) { + SpoofImpl::validateThis(sc, *status); + if (U_FAILURE(*status)) { + return 0; + } + if (length<-1 || destCapacity<0 || (destCapacity==0 && dest!=NULL)) { + *status = U_ILLEGAL_ARGUMENT_ERROR; + return 0; + } + + UnicodeString srcStr = UnicodeString::fromUTF8(StringPiece(id, length>=0 ? length : static_cast(uprv_strlen(id)))); + UnicodeString destStr; + uspoof_getSkeletonUnicodeString(sc, type, srcStr, destStr, status); + if (U_FAILURE(*status)) { + return 0; + } + + int32_t lengthInUTF8 = 0; + u_strToUTF8(dest, destCapacity, &lengthInUTF8, + destStr.getBuffer(), destStr.length(), status); + return lengthInUTF8; +} + + +U_CAPI int32_t U_EXPORT2 +uspoof_serialize(USpoofChecker *sc,void *buf, int32_t capacity, UErrorCode *status) { + SpoofImpl *This = SpoofImpl::validateThis(sc, *status); + if (This == NULL) { + U_ASSERT(U_FAILURE(*status)); + return 0; + } + + return This->fSpoofData->serialize(buf, capacity, *status); +} + +U_CAPI const USet * U_EXPORT2 +uspoof_getInclusionSet(UErrorCode *status) { + umtx_initOnce(gSpoofInitStaticsOnce, &initializeStatics, *status); + return gInclusionSet->toUSet(); +} + +U_CAPI const USet * U_EXPORT2 +uspoof_getRecommendedSet(UErrorCode *status) { + umtx_initOnce(gSpoofInitStaticsOnce, &initializeStatics, *status); + return gRecommendedSet->toUSet(); +} + +U_I18N_API const UnicodeSet * U_EXPORT2 +uspoof_getInclusionUnicodeSet(UErrorCode *status) { + umtx_initOnce(gSpoofInitStaticsOnce, &initializeStatics, *status); + return gInclusionSet; +} + +U_I18N_API const UnicodeSet * U_EXPORT2 +uspoof_getRecommendedUnicodeSet(UErrorCode *status) { + umtx_initOnce(gSpoofInitStaticsOnce, &initializeStatics, *status); + return gRecommendedSet; +} + +//------------------ +// CheckResult APIs +//------------------ + +U_CAPI USpoofCheckResult* U_EXPORT2 +uspoof_openCheckResult(UErrorCode *status) { + CheckResult* checkResult = new CheckResult(); + if (checkResult == NULL) { + *status = U_MEMORY_ALLOCATION_ERROR; + return NULL; + } + return checkResult->asUSpoofCheckResult(); +} + +U_CAPI void U_EXPORT2 +uspoof_closeCheckResult(USpoofCheckResult* checkResult) { + UErrorCode status = U_ZERO_ERROR; + CheckResult* This = CheckResult::validateThis(checkResult, status); + delete This; +} + +U_CAPI int32_t U_EXPORT2 +uspoof_getCheckResultChecks(const USpoofCheckResult *checkResult, UErrorCode *status) { + const CheckResult* This = CheckResult::validateThis(checkResult, *status); + if (U_FAILURE(*status)) { return 0; } + return This->fChecks; +} + +U_CAPI URestrictionLevel U_EXPORT2 +uspoof_getCheckResultRestrictionLevel(const USpoofCheckResult *checkResult, UErrorCode *status) { + const CheckResult* This = CheckResult::validateThis(checkResult, *status); + if (U_FAILURE(*status)) { return USPOOF_UNRESTRICTIVE; } + return This->fRestrictionLevel; +} + +U_CAPI const USet* U_EXPORT2 +uspoof_getCheckResultNumerics(const USpoofCheckResult *checkResult, UErrorCode *status) { + const CheckResult* This = CheckResult::validateThis(checkResult, *status); + if (U_FAILURE(*status)) { return NULL; } + return This->fNumerics.toUSet(); +} + + + +#endif // !UCONFIG_NO_NORMALIZATION diff --git a/thirdparty/icu4c/i18n/uspoof_impl.cpp b/thirdparty/icu4c/i18n/uspoof_impl.cpp new file mode 100644 index 00000000000..b283d813210 --- /dev/null +++ b/thirdparty/icu4c/i18n/uspoof_impl.cpp @@ -0,0 +1,959 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* +********************************************************************** +* Copyright (C) 2008-2016, International Business Machines +* Corporation and others. All Rights Reserved. +********************************************************************** +*/ + +#include "unicode/utypes.h" +#include "unicode/uspoof.h" +#include "unicode/uchar.h" +#include "unicode/uniset.h" +#include "unicode/utf16.h" +#include "utrie2.h" +#include "cmemory.h" +#include "cstring.h" +#include "scriptset.h" +#include "umutex.h" +#include "udataswp.h" +#include "uassert.h" +#include "ucln_in.h" +#include "uspoof_impl.h" + +#if !UCONFIG_NO_NORMALIZATION + + +U_NAMESPACE_BEGIN + +UOBJECT_DEFINE_RTTI_IMPLEMENTATION(SpoofImpl) + +SpoofImpl::SpoofImpl(SpoofData *data, UErrorCode& status) { + construct(status); + fSpoofData = data; +} + +SpoofImpl::SpoofImpl(UErrorCode& status) { + construct(status); + + // TODO: Call this method where it is actually needed, instead of in the + // constructor, to allow for lazy data loading. See #12696. + fSpoofData = SpoofData::getDefault(status); +} + +SpoofImpl::SpoofImpl() { + UErrorCode status = U_ZERO_ERROR; + construct(status); + + // TODO: Call this method where it is actually needed, instead of in the + // constructor, to allow for lazy data loading. See #12696. + fSpoofData = SpoofData::getDefault(status); +} + +void SpoofImpl::construct(UErrorCode& status) { + fChecks = USPOOF_ALL_CHECKS; + fSpoofData = NULL; + fAllowedCharsSet = NULL; + fAllowedLocales = NULL; + fRestrictionLevel = USPOOF_HIGHLY_RESTRICTIVE; + + if (U_FAILURE(status)) { return; } + + UnicodeSet *allowedCharsSet = new UnicodeSet(0, 0x10ffff); + fAllowedCharsSet = allowedCharsSet; + fAllowedLocales = uprv_strdup(""); + if (fAllowedCharsSet == NULL || fAllowedLocales == NULL) { + status = U_MEMORY_ALLOCATION_ERROR; + return; + } + allowedCharsSet->freeze(); +} + + +// Copy Constructor, used by the user level clone() function. +SpoofImpl::SpoofImpl(const SpoofImpl &src, UErrorCode &status) : + fChecks(USPOOF_ALL_CHECKS), fSpoofData(NULL), fAllowedCharsSet(NULL) , + fAllowedLocales(NULL) { + if (U_FAILURE(status)) { + return; + } + fChecks = src.fChecks; + if (src.fSpoofData != NULL) { + fSpoofData = src.fSpoofData->addReference(); + } + fAllowedCharsSet = src.fAllowedCharsSet->clone(); + fAllowedLocales = uprv_strdup(src.fAllowedLocales); + if (fAllowedCharsSet == NULL || fAllowedLocales == NULL) { + status = U_MEMORY_ALLOCATION_ERROR; + } + fRestrictionLevel = src.fRestrictionLevel; +} + +SpoofImpl::~SpoofImpl() { + if (fSpoofData != NULL) { + fSpoofData->removeReference(); // Will delete if refCount goes to zero. + } + delete fAllowedCharsSet; + uprv_free((void *)fAllowedLocales); +} + +// Cast this instance as a USpoofChecker for the C API. +USpoofChecker *SpoofImpl::asUSpoofChecker() { + return exportForC(); +} + +// +// Incoming parameter check on Status and the SpoofChecker object +// received from the C API. +// +const SpoofImpl *SpoofImpl::validateThis(const USpoofChecker *sc, UErrorCode &status) { + auto* This = validate(sc, status); + if (U_FAILURE(status)) { + return NULL; + } + if (This->fSpoofData != NULL && !This->fSpoofData->validateDataVersion(status)) { + return NULL; + } + return This; +} + +SpoofImpl *SpoofImpl::validateThis(USpoofChecker *sc, UErrorCode &status) { + return const_cast + (SpoofImpl::validateThis(const_cast(sc), status)); +} + + +void SpoofImpl::setAllowedLocales(const char *localesList, UErrorCode &status) { + UnicodeSet allowedChars; + UnicodeSet *tmpSet = NULL; + const char *locStart = localesList; + const char *locEnd = NULL; + const char *localesListEnd = localesList + uprv_strlen(localesList); + int32_t localeListCount = 0; // Number of locales provided by caller. + + // Loop runs once per locale from the localesList, a comma separated list of locales. + do { + locEnd = uprv_strchr(locStart, ','); + if (locEnd == NULL) { + locEnd = localesListEnd; + } + while (*locStart == ' ') { + locStart++; + } + const char *trimmedEnd = locEnd-1; + while (trimmedEnd > locStart && *trimmedEnd == ' ') { + trimmedEnd--; + } + if (trimmedEnd <= locStart) { + break; + } + const char *locale = uprv_strndup(locStart, (int32_t)(trimmedEnd + 1 - locStart)); + localeListCount++; + + // We have one locale from the locales list. + // Add the script chars for this locale to the accumulating set of allowed chars. + // If the locale is no good, we will be notified back via status. + addScriptChars(locale, &allowedChars, status); + uprv_free((void *)locale); + if (U_FAILURE(status)) { + break; + } + locStart = locEnd + 1; + } while (locStart < localesListEnd); + + // If our caller provided an empty list of locales, we disable the allowed characters checking + if (localeListCount == 0) { + uprv_free((void *)fAllowedLocales); + fAllowedLocales = uprv_strdup(""); + tmpSet = new UnicodeSet(0, 0x10ffff); + if (fAllowedLocales == NULL || tmpSet == NULL) { + status = U_MEMORY_ALLOCATION_ERROR; + return; + } + tmpSet->freeze(); + delete fAllowedCharsSet; + fAllowedCharsSet = tmpSet; + fChecks &= ~USPOOF_CHAR_LIMIT; + return; + } + + + // Add all common and inherited characters to the set of allowed chars. + UnicodeSet tempSet; + tempSet.applyIntPropertyValue(UCHAR_SCRIPT, USCRIPT_COMMON, status); + allowedChars.addAll(tempSet); + tempSet.applyIntPropertyValue(UCHAR_SCRIPT, USCRIPT_INHERITED, status); + allowedChars.addAll(tempSet); + + // If anything went wrong, we bail out without changing + // the state of the spoof checker. + if (U_FAILURE(status)) { + return; + } + + // Store the updated spoof checker state. + tmpSet = allowedChars.clone(); + const char *tmpLocalesList = uprv_strdup(localesList); + if (tmpSet == NULL || tmpLocalesList == NULL) { + status = U_MEMORY_ALLOCATION_ERROR; + return; + } + uprv_free((void *)fAllowedLocales); + fAllowedLocales = tmpLocalesList; + tmpSet->freeze(); + delete fAllowedCharsSet; + fAllowedCharsSet = tmpSet; + fChecks |= USPOOF_CHAR_LIMIT; +} + + +const char * SpoofImpl::getAllowedLocales(UErrorCode &/*status*/) { + return fAllowedLocales; +} + + +// Given a locale (a language), add all the characters from all of the scripts used with that language +// to the allowedChars UnicodeSet + +void SpoofImpl::addScriptChars(const char *locale, UnicodeSet *allowedChars, UErrorCode &status) { + UScriptCode scripts[30]; + + int32_t numScripts = uscript_getCode(locale, scripts, UPRV_LENGTHOF(scripts), &status); + if (U_FAILURE(status)) { + return; + } + if (status == U_USING_DEFAULT_WARNING) { + status = U_ILLEGAL_ARGUMENT_ERROR; + return; + } + UnicodeSet tmpSet; + int32_t i; + for (i=0; iaddAll(tmpSet); + } +} + +// Computes the augmented script set for a code point, according to UTS 39 section 5.1. +void SpoofImpl::getAugmentedScriptSet(UChar32 codePoint, ScriptSet& result, UErrorCode& status) { + result.resetAll(); + result.setScriptExtensions(codePoint, status); + if (U_FAILURE(status)) { return; } + + // Section 5.1 step 1 + if (result.test(USCRIPT_HAN, status)) { + result.set(USCRIPT_HAN_WITH_BOPOMOFO, status); + result.set(USCRIPT_JAPANESE, status); + result.set(USCRIPT_KOREAN, status); + } + if (result.test(USCRIPT_HIRAGANA, status)) { + result.set(USCRIPT_JAPANESE, status); + } + if (result.test(USCRIPT_KATAKANA, status)) { + result.set(USCRIPT_JAPANESE, status); + } + if (result.test(USCRIPT_HANGUL, status)) { + result.set(USCRIPT_KOREAN, status); + } + if (result.test(USCRIPT_BOPOMOFO, status)) { + result.set(USCRIPT_HAN_WITH_BOPOMOFO, status); + } + + // Section 5.1 step 2 + if (result.test(USCRIPT_COMMON, status) || result.test(USCRIPT_INHERITED, status)) { + result.setAll(); + } +} + +// Computes the resolved script set for a string, according to UTS 39 section 5.1. +void SpoofImpl::getResolvedScriptSet(const UnicodeString& input, ScriptSet& result, UErrorCode& status) const { + getResolvedScriptSetWithout(input, USCRIPT_CODE_LIMIT, result, status); +} + +// Computes the resolved script set for a string, omitting characters having the specified script. +// If USCRIPT_CODE_LIMIT is passed as the second argument, all characters are included. +void SpoofImpl::getResolvedScriptSetWithout(const UnicodeString& input, UScriptCode script, ScriptSet& result, UErrorCode& status) const { + result.setAll(); + + ScriptSet temp; + UChar32 codePoint; + for (int32_t i = 0; i < input.length(); i += U16_LENGTH(codePoint)) { + codePoint = input.char32At(i); + + // Compute the augmented script set for the character + getAugmentedScriptSet(codePoint, temp, status); + if (U_FAILURE(status)) { return; } + + // Intersect the augmented script set with the resolved script set, but only if the character doesn't + // have the script specified in the function call + if (script == USCRIPT_CODE_LIMIT || !temp.test(script, status)) { + result.intersect(temp); + } + } +} + +// Computes the set of numerics for a string, according to UTS 39 section 5.3. +void SpoofImpl::getNumerics(const UnicodeString& input, UnicodeSet& result, UErrorCode& /*status*/) const { + result.clear(); + + UChar32 codePoint; + for (int32_t i = 0; i < input.length(); i += U16_LENGTH(codePoint)) { + codePoint = input.char32At(i); + + // Store a representative character for each kind of decimal digit + if (u_charType(codePoint) == U_DECIMAL_DIGIT_NUMBER) { + // Store the zero character as a representative for comparison. + // Unicode guarantees it is codePoint - value + result.add(codePoint - (UChar32)u_getNumericValue(codePoint)); + } + } +} + +// Computes the restriction level of a string, according to UTS 39 section 5.2. +URestrictionLevel SpoofImpl::getRestrictionLevel(const UnicodeString& input, UErrorCode& status) const { + // Section 5.2 step 1: + if (!fAllowedCharsSet->containsAll(input)) { + return USPOOF_UNRESTRICTIVE; + } + + // Section 5.2 step 2 + // Java use a static UnicodeSet for this test. In C++, avoid the static variable + // and just do a simple for loop. + UBool allASCII = TRUE; + for (int32_t i=0, length=input.length(); i 0x7f) { + allASCII = FALSE; + break; + } + } + if (allASCII) { + return USPOOF_ASCII; + } + + // Section 5.2 steps 3: + ScriptSet resolvedScriptSet; + getResolvedScriptSet(input, resolvedScriptSet, status); + if (U_FAILURE(status)) { return USPOOF_UNRESTRICTIVE; } + + // Section 5.2 step 4: + if (!resolvedScriptSet.isEmpty()) { + return USPOOF_SINGLE_SCRIPT_RESTRICTIVE; + } + + // Section 5.2 step 5: + ScriptSet resolvedNoLatn; + getResolvedScriptSetWithout(input, USCRIPT_LATIN, resolvedNoLatn, status); + if (U_FAILURE(status)) { return USPOOF_UNRESTRICTIVE; } + + // Section 5.2 step 6: + if (resolvedNoLatn.test(USCRIPT_HAN_WITH_BOPOMOFO, status) + || resolvedNoLatn.test(USCRIPT_JAPANESE, status) + || resolvedNoLatn.test(USCRIPT_KOREAN, status)) { + return USPOOF_HIGHLY_RESTRICTIVE; + } + + // Section 5.2 step 7: + if (!resolvedNoLatn.isEmpty() + && !resolvedNoLatn.test(USCRIPT_CYRILLIC, status) + && !resolvedNoLatn.test(USCRIPT_GREEK, status) + && !resolvedNoLatn.test(USCRIPT_CHEROKEE, status)) { + return USPOOF_MODERATELY_RESTRICTIVE; + } + + // Section 5.2 step 8: + return USPOOF_MINIMALLY_RESTRICTIVE; +} + +int32_t SpoofImpl::findHiddenOverlay(const UnicodeString& input, UErrorCode&) const { + bool sawLeadCharacter = false; + for (int32_t i=0; iconfusableLookup(cp, skelStr); + UChar32 finalCp = skelStr.char32At(skelStr.moveIndex32(skelStr.length(), -1)); + if (finalCp != cp && isIllegalCombiningDotLeadCharacterNoLookup(finalCp)) { + return true; + } + return false; +} + + + +// Convert a text format hex number. Utility function used by builder code. Static. +// Input: UChar *string text. Output: a UChar32 +// Input has been pre-checked, and will have no non-hex chars. +// The number must fall in the code point range of 0..0x10ffff +// Static Function. +UChar32 SpoofImpl::ScanHex(const UChar *s, int32_t start, int32_t limit, UErrorCode &status) { + if (U_FAILURE(status)) { + return 0; + } + U_ASSERT(limit-start > 0); + uint32_t val = 0; + int i; + for (i=start; i9) { + digitVal = 0xa + (s[i] - 0x41); // Upper Case 'A' + } + if (digitVal>15) { + digitVal = 0xa + (s[i] - 0x61); // Lower Case 'a' + } + U_ASSERT(digitVal <= 0xf); + val <<= 4; + val += digitVal; + } + if (val > 0x10ffff) { + status = U_PARSE_ERROR; + val = 0; + } + return (UChar32)val; +} + + +//----------------------------------------- +// +// class CheckResult Implementation +// +//----------------------------------------- + +CheckResult::CheckResult() { + clear(); +} + +USpoofCheckResult* CheckResult::asUSpoofCheckResult() { + return exportForC(); +} + +// +// Incoming parameter check on Status and the CheckResult object +// received from the C API. +// +const CheckResult* CheckResult::validateThis(const USpoofCheckResult *ptr, UErrorCode &status) { + return validate(ptr, status); +} + +CheckResult* CheckResult::validateThis(USpoofCheckResult *ptr, UErrorCode &status) { + return validate(ptr, status); +} + +void CheckResult::clear() { + fChecks = 0; + fNumerics.clear(); + fRestrictionLevel = USPOOF_UNDEFINED_RESTRICTIVE; +} + +int32_t CheckResult::toCombinedBitmask(int32_t enabledChecks) { + if ((enabledChecks & USPOOF_AUX_INFO) != 0 && fRestrictionLevel != USPOOF_UNDEFINED_RESTRICTIVE) { + return fChecks | fRestrictionLevel; + } else { + return fChecks; + } +} + +CheckResult::~CheckResult() { +} + +//---------------------------------------------------------------------------------------------- +// +// class SpoofData Implementation +// +//---------------------------------------------------------------------------------------------- + + +UBool SpoofData::validateDataVersion(UErrorCode &status) const { + if (U_FAILURE(status) || + fRawData == NULL || + fRawData->fMagic != USPOOF_MAGIC || + fRawData->fFormatVersion[0] != USPOOF_CONFUSABLE_DATA_FORMAT_VERSION || + fRawData->fFormatVersion[1] != 0 || + fRawData->fFormatVersion[2] != 0 || + fRawData->fFormatVersion[3] != 0) { + status = U_INVALID_FORMAT_ERROR; + return FALSE; + } + return TRUE; +} + +static UBool U_CALLCONV +spoofDataIsAcceptable(void *context, + const char * /* type */, const char * /*name*/, + const UDataInfo *pInfo) { + if( + pInfo->size >= 20 && + pInfo->isBigEndian == U_IS_BIG_ENDIAN && + pInfo->charsetFamily == U_CHARSET_FAMILY && + pInfo->dataFormat[0] == 0x43 && // dataFormat="Cfu " + pInfo->dataFormat[1] == 0x66 && + pInfo->dataFormat[2] == 0x75 && + pInfo->dataFormat[3] == 0x20 && + pInfo->formatVersion[0] == USPOOF_CONFUSABLE_DATA_FORMAT_VERSION + ) { + UVersionInfo *version = static_cast(context); + if(version != NULL) { + uprv_memcpy(version, pInfo->dataVersion, 4); + } + return TRUE; + } else { + return FALSE; + } +} + +// Methods for the loading of the default confusables data file. The confusable +// data is loaded only when it is needed. +// +// SpoofData::getDefault() - Return the default confusables data, and call the +// initOnce() if it is not available. Adds a reference +// to the SpoofData that the caller is responsible for +// decrementing when they are done with the data. +// +// uspoof_loadDefaultData - Called once, from initOnce(). The resulting SpoofData +// is shared by all spoof checkers using the default data. +// +// uspoof_cleanupDefaultData - Called during cleanup. +// + +static UInitOnce gSpoofInitDefaultOnce = U_INITONCE_INITIALIZER; +static SpoofData* gDefaultSpoofData; + +static UBool U_CALLCONV +uspoof_cleanupDefaultData(void) { + if (gDefaultSpoofData) { + // Will delete, assuming all user-level spoof checkers were closed. + gDefaultSpoofData->removeReference(); + gDefaultSpoofData = nullptr; + gSpoofInitDefaultOnce.reset(); + } + return TRUE; +} + +static void U_CALLCONV uspoof_loadDefaultData(UErrorCode& status) { + UDataMemory *udm = udata_openChoice(nullptr, "cfu", "confusables", + spoofDataIsAcceptable, + nullptr, // context, would receive dataVersion if supplied. + &status); + if (U_FAILURE(status)) { return; } + gDefaultSpoofData = new SpoofData(udm, status); + if (U_FAILURE(status)) { + delete gDefaultSpoofData; + gDefaultSpoofData = nullptr; + return; + } + if (gDefaultSpoofData == nullptr) { + status = U_MEMORY_ALLOCATION_ERROR; + return; + } + ucln_i18n_registerCleanup(UCLN_I18N_SPOOFDATA, uspoof_cleanupDefaultData); +} + +SpoofData* SpoofData::getDefault(UErrorCode& status) { + umtx_initOnce(gSpoofInitDefaultOnce, &uspoof_loadDefaultData, status); + if (U_FAILURE(status)) { return NULL; } + gDefaultSpoofData->addReference(); + return gDefaultSpoofData; +} + + + +SpoofData::SpoofData(UDataMemory *udm, UErrorCode &status) +{ + reset(); + if (U_FAILURE(status)) { + return; + } + fUDM = udm; + // fRawData is non-const because it may be constructed by the data builder. + fRawData = reinterpret_cast( + const_cast(udata_getMemory(udm))); + validateDataVersion(status); + initPtrs(status); +} + + +SpoofData::SpoofData(const void *data, int32_t length, UErrorCode &status) +{ + reset(); + if (U_FAILURE(status)) { + return; + } + if ((size_t)length < sizeof(SpoofDataHeader)) { + status = U_INVALID_FORMAT_ERROR; + return; + } + if (data == NULL) { + status = U_ILLEGAL_ARGUMENT_ERROR; + return; + } + void *ncData = const_cast(data); + fRawData = static_cast(ncData); + if (length < fRawData->fLength) { + status = U_INVALID_FORMAT_ERROR; + return; + } + validateDataVersion(status); + initPtrs(status); +} + + +// Spoof Data constructor for use from data builder. +// Initializes a new, empty data area that will be populated later. +SpoofData::SpoofData(UErrorCode &status) { + reset(); + if (U_FAILURE(status)) { + return; + } + fDataOwned = true; + + // The spoof header should already be sized to be a multiple of 16 bytes. + // Just in case it's not, round it up. + uint32_t initialSize = (sizeof(SpoofDataHeader) + 15) & ~15; + U_ASSERT(initialSize == sizeof(SpoofDataHeader)); + + fRawData = static_cast(uprv_malloc(initialSize)); + fMemLimit = initialSize; + if (fRawData == NULL) { + status = U_MEMORY_ALLOCATION_ERROR; + return; + } + uprv_memset(fRawData, 0, initialSize); + + fRawData->fMagic = USPOOF_MAGIC; + fRawData->fFormatVersion[0] = USPOOF_CONFUSABLE_DATA_FORMAT_VERSION; + fRawData->fFormatVersion[1] = 0; + fRawData->fFormatVersion[2] = 0; + fRawData->fFormatVersion[3] = 0; + initPtrs(status); +} + +// reset() - initialize all fields. +// Should be updated if any new fields are added. +// Called by constructors to put things in a known initial state. +void SpoofData::reset() { + fRawData = NULL; + fDataOwned = FALSE; + fUDM = NULL; + fMemLimit = 0; + fRefCount = 1; + fCFUKeys = NULL; + fCFUValues = NULL; + fCFUStrings = NULL; +} + + +// SpoofData::initPtrs() +// Initialize the pointers to the various sections of the raw data. +// +// This function is used both during the Trie building process (multiple +// times, as the individual data sections are added), and +// during the opening of a Spoof Checker from prebuilt data. +// +// The pointers for non-existent data sections (identified by an offset of 0) +// are set to NULL. +// +// Note: During building the data, adding each new data section +// reallocs the raw data area, which likely relocates it, which +// in turn requires reinitializing all of the pointers into it, hence +// multiple calls to this function during building. +// +void SpoofData::initPtrs(UErrorCode &status) { + fCFUKeys = NULL; + fCFUValues = NULL; + fCFUStrings = NULL; + if (U_FAILURE(status)) { + return; + } + if (fRawData->fCFUKeys != 0) { + fCFUKeys = (int32_t *)((char *)fRawData + fRawData->fCFUKeys); + } + if (fRawData->fCFUStringIndex != 0) { + fCFUValues = (uint16_t *)((char *)fRawData + fRawData->fCFUStringIndex); + } + if (fRawData->fCFUStringTable != 0) { + fCFUStrings = (UChar *)((char *)fRawData + fRawData->fCFUStringTable); + } +} + + +SpoofData::~SpoofData() { + if (fDataOwned) { + uprv_free(fRawData); + } + fRawData = NULL; + if (fUDM != NULL) { + udata_close(fUDM); + } + fUDM = NULL; +} + + +void SpoofData::removeReference() { + if (umtx_atomic_dec(&fRefCount) == 0) { + delete this; + } +} + + +SpoofData *SpoofData::addReference() { + umtx_atomic_inc(&fRefCount); + return this; +} + + +void *SpoofData::reserveSpace(int32_t numBytes, UErrorCode &status) { + if (U_FAILURE(status)) { + return NULL; + } + if (!fDataOwned) { + UPRV_UNREACHABLE_EXIT; + } + + numBytes = (numBytes + 15) & ~15; // Round up to a multiple of 16 + uint32_t returnOffset = fMemLimit; + fMemLimit += numBytes; + fRawData = static_cast(uprv_realloc(fRawData, fMemLimit)); + fRawData->fLength = fMemLimit; + uprv_memset((char *)fRawData + returnOffset, 0, numBytes); + initPtrs(status); + return (char *)fRawData + returnOffset; +} + +int32_t SpoofData::serialize(void *buf, int32_t capacity, UErrorCode &status) const { + int32_t dataSize = fRawData->fLength; + if (capacity < dataSize) { + status = U_BUFFER_OVERFLOW_ERROR; + return dataSize; + } + uprv_memcpy(buf, fRawData, dataSize); + return dataSize; +} + +int32_t SpoofData::size() const { + return fRawData->fLength; +} + +//------------------------------- +// +// Front-end APIs for SpoofData +// +//------------------------------- + +int32_t SpoofData::confusableLookup(UChar32 inChar, UnicodeString &dest) const { + // Perform a binary search. + // [lo, hi), i.e lo is inclusive, hi is exclusive. + // The result after the loop will be in lo. + int32_t lo = 0; + int32_t hi = length(); + do { + int32_t mid = (lo + hi) / 2; + if (codePointAt(mid) > inChar) { + hi = mid; + } else if (codePointAt(mid) < inChar) { + lo = mid; + } else { + // Found result. Break early. + lo = mid; + break; + } + } while (hi - lo > 1); + + // Did we find an entry? If not, the char maps to itself. + if (codePointAt(lo) != inChar) { + dest.append(inChar); + return 1; + } + + // Add the element to the string builder and return. + return appendValueTo(lo, dest); +} + +int32_t SpoofData::length() const { + return fRawData->fCFUKeysSize; +} + +UChar32 SpoofData::codePointAt(int32_t index) const { + return ConfusableDataUtils::keyToCodePoint(fCFUKeys[index]); +} + +int32_t SpoofData::appendValueTo(int32_t index, UnicodeString& dest) const { + int32_t stringLength = ConfusableDataUtils::keyToLength(fCFUKeys[index]); + + // Value is either a char (for strings of length 1) or + // an index into the string table (for longer strings) + uint16_t value = fCFUValues[index]; + if (stringLength == 1) { + dest.append((UChar)value); + } else { + dest.append(fCFUStrings + value, stringLength); + } + + return stringLength; +} + + +U_NAMESPACE_END + +U_NAMESPACE_USE + +//----------------------------------------------------------------------------- +// +// uspoof_swap - byte swap and char encoding swap of spoof data +// +//----------------------------------------------------------------------------- +U_CAPI int32_t U_EXPORT2 +uspoof_swap(const UDataSwapper *ds, const void *inData, int32_t length, void *outData, + UErrorCode *status) { + + if (status == NULL || U_FAILURE(*status)) { + return 0; + } + if(ds==NULL || inData==NULL || length<-1 || (length>0 && outData==NULL)) { + *status=U_ILLEGAL_ARGUMENT_ERROR; + return 0; + } + + // + // Check that the data header is for spoof data. + // (Header contents are defined in gencfu.cpp) + // + const UDataInfo *pInfo = (const UDataInfo *)((const char *)inData+4); + if(!( pInfo->dataFormat[0]==0x43 && /* dataFormat="Cfu " */ + pInfo->dataFormat[1]==0x66 && + pInfo->dataFormat[2]==0x75 && + pInfo->dataFormat[3]==0x20 && + pInfo->formatVersion[0]==USPOOF_CONFUSABLE_DATA_FORMAT_VERSION && + pInfo->formatVersion[1]==0 && + pInfo->formatVersion[2]==0 && + pInfo->formatVersion[3]==0 )) { + udata_printError(ds, "uspoof_swap(): data format %02x.%02x.%02x.%02x " + "(format version %02x %02x %02x %02x) is not recognized\n", + pInfo->dataFormat[0], pInfo->dataFormat[1], + pInfo->dataFormat[2], pInfo->dataFormat[3], + pInfo->formatVersion[0], pInfo->formatVersion[1], + pInfo->formatVersion[2], pInfo->formatVersion[3]); + *status=U_UNSUPPORTED_ERROR; + return 0; + } + + // + // Swap the data header. (This is the generic ICU Data Header, not the uspoof Specific + // header). This swap also conveniently gets us + // the size of the ICU d.h., which lets us locate the start + // of the uspoof specific data. + // + int32_t headerSize=udata_swapDataHeader(ds, inData, length, outData, status); + + + // + // Get the Spoof Data Header, and check that it appears to be OK. + // + // + const uint8_t *inBytes =(const uint8_t *)inData+headerSize; + SpoofDataHeader *spoofDH = (SpoofDataHeader *)inBytes; + if (ds->readUInt32(spoofDH->fMagic) != USPOOF_MAGIC || + ds->readUInt32(spoofDH->fLength) < sizeof(SpoofDataHeader)) + { + udata_printError(ds, "uspoof_swap(): Spoof Data header is invalid.\n"); + *status=U_UNSUPPORTED_ERROR; + return 0; + } + + // + // Prefight operation? Just return the size + // + int32_t spoofDataLength = ds->readUInt32(spoofDH->fLength); + int32_t totalSize = headerSize + spoofDataLength; + if (length < 0) { + return totalSize; + } + + // + // Check that length passed in is consistent with length from Spoof data header. + // + if (length < totalSize) { + udata_printError(ds, "uspoof_swap(): too few bytes (%d after ICU Data header) for spoof data.\n", + spoofDataLength); + *status=U_INDEX_OUTOFBOUNDS_ERROR; + return 0; + } + + + // + // Swap the Data. Do the data itself first, then the Spoof Data Header, because + // we need to reference the header to locate the data, and an + // inplace swap of the header leaves it unusable. + // + uint8_t *outBytes = (uint8_t *)outData + headerSize; + SpoofDataHeader *outputDH = (SpoofDataHeader *)outBytes; + + int32_t sectionStart; + int32_t sectionLength; + + // + // If not swapping in place, zero out the output buffer before starting. + // Gaps may exist between the individual sections, and these must be zeroed in + // the output buffer. The simplest way to do that is to just zero the whole thing. + // + if (inBytes != outBytes) { + uprv_memset(outBytes, 0, spoofDataLength); + } + + // Confusables Keys Section (fCFUKeys) + sectionStart = ds->readUInt32(spoofDH->fCFUKeys); + sectionLength = ds->readUInt32(spoofDH->fCFUKeysSize) * 4; + ds->swapArray32(ds, inBytes+sectionStart, sectionLength, outBytes+sectionStart, status); + + // String Index Section + sectionStart = ds->readUInt32(spoofDH->fCFUStringIndex); + sectionLength = ds->readUInt32(spoofDH->fCFUStringIndexSize) * 2; + ds->swapArray16(ds, inBytes+sectionStart, sectionLength, outBytes+sectionStart, status); + + // String Table Section + sectionStart = ds->readUInt32(spoofDH->fCFUStringTable); + sectionLength = ds->readUInt32(spoofDH->fCFUStringTableLen) * 2; + ds->swapArray16(ds, inBytes+sectionStart, sectionLength, outBytes+sectionStart, status); + + // And, last, swap the header itself. + // int32_t fMagic // swap this + // uint8_t fFormatVersion[4] // Do not swap this, just copy + // int32_t fLength and all the rest // Swap the rest, all is 32 bit stuff. + // + uint32_t magic = ds->readUInt32(spoofDH->fMagic); + ds->writeUInt32((uint32_t *)&outputDH->fMagic, magic); + + if (outputDH->fFormatVersion != spoofDH->fFormatVersion) { + uprv_memcpy(outputDH->fFormatVersion, spoofDH->fFormatVersion, sizeof(spoofDH->fFormatVersion)); + } + // swap starting at fLength + ds->swapArray32(ds, &spoofDH->fLength, sizeof(SpoofDataHeader)-8 /* minus magic and fFormatVersion[4] */, &outputDH->fLength, status); + + return totalSize; +} + +#endif + + diff --git a/thirdparty/icu4c/i18n/uspoof_impl.h b/thirdparty/icu4c/i18n/uspoof_impl.h new file mode 100644 index 00000000000..e75ae262bdd --- /dev/null +++ b/thirdparty/icu4c/i18n/uspoof_impl.h @@ -0,0 +1,343 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* +*************************************************************************** +* Copyright (C) 2008-2013, International Business Machines Corporation +* and others. All Rights Reserved. +*************************************************************************** +* +* uspoof_impl.h +* +* Implementation header for spoof detection +* +*/ + +#ifndef USPOOFIM_H +#define USPOOFIM_H + +#include "uassert.h" +#include "unicode/utypes.h" +#include "unicode/uspoof.h" +#include "unicode/uscript.h" +#include "unicode/udata.h" +#include "udataswp.h" +#include "utrie2.h" + +#if !UCONFIG_NO_NORMALIZATION + +#ifdef __cplusplus + +#include "capi_helper.h" + +U_NAMESPACE_BEGIN + +// The maximum length (in UTF-16 UChars) of the skeleton replacement string resulting from +// a single input code point. This is function of the unicode.org data. +#define USPOOF_MAX_SKELETON_EXPANSION 20 + +// The default stack buffer size for copies or conversions or normalizations +// of input strings being checked. (Used in multiple places.) +#define USPOOF_STACK_BUFFER_SIZE 100 + +// Magic number for sanity checking spoof data. +#define USPOOF_MAGIC 0x3845fdef + +// Magic number for sanity checking spoof checkers. +#define USPOOF_CHECK_MAGIC 0x2734ecde + +class ScriptSet; +class SpoofData; +struct SpoofDataHeader; +class ConfusableDataUtils; + +/** + * Class SpoofImpl corresponds directly to the plain C API opaque type + * USpoofChecker. One can be cast to the other. + */ +class SpoofImpl : public UObject, + public IcuCApiHelper { +public: + SpoofImpl(SpoofData *data, UErrorCode& status); + SpoofImpl(UErrorCode& status); + SpoofImpl(); + void construct(UErrorCode& status); + virtual ~SpoofImpl(); + + /** Copy constructor, used by the user level uspoof_clone() function. + */ + SpoofImpl(const SpoofImpl &src, UErrorCode &status); + + USpoofChecker *asUSpoofChecker(); + static SpoofImpl *validateThis(USpoofChecker *sc, UErrorCode &status); + static const SpoofImpl *validateThis(const USpoofChecker *sc, UErrorCode &status); + + /** Set and Get AllowedLocales, implementations of the corresponding API */ + void setAllowedLocales(const char *localesList, UErrorCode &status); + const char * getAllowedLocales(UErrorCode &status); + + // Add (union) to the UnicodeSet all of the characters for the scripts used for + // the specified locale. Part of the implementation of setAllowedLocales. + void addScriptChars(const char *locale, UnicodeSet *allowedChars, UErrorCode &status); + + // Functions implementing the features of UTS 39 section 5. + static void getAugmentedScriptSet(UChar32 codePoint, ScriptSet& result, UErrorCode& status); + void getResolvedScriptSet(const UnicodeString& input, ScriptSet& result, UErrorCode& status) const; + void getResolvedScriptSetWithout(const UnicodeString& input, UScriptCode script, ScriptSet& result, UErrorCode& status) const; + void getNumerics(const UnicodeString& input, UnicodeSet& result, UErrorCode& status) const; + URestrictionLevel getRestrictionLevel(const UnicodeString& input, UErrorCode& status) const; + + int32_t findHiddenOverlay(const UnicodeString& input, UErrorCode& status) const; + bool isIllegalCombiningDotLeadCharacter(UChar32 cp) const; + + /** parse a hex number. Untility used by the builders. */ + static UChar32 ScanHex(const UChar *s, int32_t start, int32_t limit, UErrorCode &status); + + static UClassID U_EXPORT2 getStaticClassID(void); + virtual UClassID getDynamicClassID(void) const override; + + // + // Data Members + // + + int32_t fChecks; // Bit vector of checks to perform. + + SpoofData *fSpoofData; + + const UnicodeSet *fAllowedCharsSet; // The UnicodeSet of allowed characters. + // for this Spoof Checker. Defaults to all chars. + + const char *fAllowedLocales; // The list of allowed locales. + URestrictionLevel fRestrictionLevel; // The maximum restriction level for an acceptable identifier. +}; + +/** + * Class CheckResult corresponds directly to the plain C API opaque type + * USpoofCheckResult. One can be cast to the other. + */ +class CheckResult : public UObject, + public IcuCApiHelper { +public: + CheckResult(); + virtual ~CheckResult(); + + USpoofCheckResult *asUSpoofCheckResult(); + static CheckResult *validateThis(USpoofCheckResult *ptr, UErrorCode &status); + static const CheckResult *validateThis(const USpoofCheckResult *ptr, UErrorCode &status); + + void clear(); + + // Used to convert this CheckResult to the older int32_t return value API + int32_t toCombinedBitmask(int32_t expectedChecks); + + // Data Members + int32_t fChecks; // Bit vector of checks that were failed. + UnicodeSet fNumerics; // Set of numerics found in the string. + URestrictionLevel fRestrictionLevel; // The restriction level of the string. +}; + + +// +// Confusable Mappings Data Structures, version 2.0 +// +// For the confusable data, we are essentially implementing a map, +// key: a code point +// value: a string. Most commonly one char in length, but can be more. +// +// The keys are stored as a sorted array of 32 bit ints. +// bits 0-23 a code point value +// bits 24-31 length of value string, in UChars (between 1 and 256 UChars). +// The key table is sorted in ascending code point order. (not on the +// 32 bit int value, the flag bits do not participate in the sorting.) +// +// Lookup is done by means of a binary search in the key table. +// +// The corresponding values are kept in a parallel array of 16 bit ints. +// If the value string is of length 1, it is literally in the value array. +// For longer strings, the value array contains an index into the strings table. +// +// String Table: +// The strings table contains all of the value strings (those of length two or greater) +// concatenated together into one long UChar (UTF-16) array. +// +// There is no nul character or other mark between adjacent strings. +// +//---------------------------------------------------------------------------- +// +// Changes from format version 1 to format version 2: +// 1) Removal of the whole-script confusable data tables. +// 2) Removal of the SL/SA/ML/MA and multi-table flags in the key bitmask. +// 3) Expansion of string length value in the key bitmask from 2 bits to 8 bits. +// 4) Removal of the string lengths table since 8 bits is sufficient for the +// lengths of all entries in confusables.txt. + + + +// Internal functions for manipulating confusable data table keys +#define USPOOF_CONFUSABLE_DATA_FORMAT_VERSION 2 // version for ICU 58 +class ConfusableDataUtils { +public: + inline static UChar32 keyToCodePoint(int32_t key) { + return key & 0x00ffffff; + } + inline static int32_t keyToLength(int32_t key) { + return ((key & 0xff000000) >> 24) + 1; + } + inline static int32_t codePointAndLengthToKey(UChar32 codePoint, int32_t length) { + U_ASSERT((codePoint & 0x00ffffff) == codePoint); + U_ASSERT(length <= 256); + return codePoint | ((length - 1) << 24); + } +}; + + +//------------------------------------------------------------------------------------- +// +// SpoofData +// +// A small class that wraps the raw (usually memory mapped) spoof data. +// Serves two primary functions: +// 1. Convenience. Contains real pointers to the data, to avoid dealing with +// the offsets in the raw data. +// 2. Reference counting. When a spoof checker is cloned, the raw data is shared +// and must be retained until all checkers using the data are closed. +// Nothing in this struct includes state that is specific to any particular +// USpoofDetector object. +// +//--------------------------------------------------------------------------------------- +class SpoofData: public UMemory { + public: + static SpoofData* getDefault(UErrorCode &status); // Get standard ICU spoof data. + static void releaseDefault(); // Cleanup reference to default spoof data. + + SpoofData(UErrorCode &status); // Create new spoof data wrapper. + // Only used when building new data from rules. + + // Constructor for use when creating from prebuilt default data. + // A UDataMemory is what the ICU internal data loading functions provide. + // The udm is adopted by the SpoofData. + SpoofData(UDataMemory *udm, UErrorCode &status); + + // Constructor for use when creating from serialized data. + // + SpoofData(const void *serializedData, int32_t length, UErrorCode &status); + + // Check raw Spoof Data Version compatibility. + // Return true it looks good. + UBool validateDataVersion(UErrorCode &status) const; + + ~SpoofData(); // Destructor not normally used. + // Use removeReference() instead. + // Reference Counting functions. + // Clone of a user-level spoof detector increments the ref count on the data. + // Close of a user-level spoof detector decrements the ref count. + // If the data is owned by us, it will be deleted when count goes to zero. + SpoofData *addReference(); + void removeReference(); + + // Reset all fields to an initial state. + // Called from the top of all constructors. + void reset(); + + // Copy this instance's raw data buffer to the specified address. + int32_t serialize(void *buf, int32_t capacity, UErrorCode &status) const; + + // Get the total number of bytes of data backed by this SpoofData. + // Not to be confused with length, which returns the number of confusable entries. + int32_t size() const; + + // Get the confusable skeleton transform for a single code point. + // The result is a string with a length between 1 and 18 as of Unicode 9. + // This is the main public endpoint for this class. + // @return The length in UTF-16 code units of the substitution string. + int32_t confusableLookup(UChar32 inChar, UnicodeString &dest) const; + + // Get the number of confusable entries in this SpoofData. + int32_t length() const; + + // Get the code point (key) at the specified index. + UChar32 codePointAt(int32_t index) const; + + // Get the confusable skeleton (value) at the specified index. + // Append it to the specified UnicodeString&. + // @return The length in UTF-16 code units of the skeleton string. + int32_t appendValueTo(int32_t index, UnicodeString& dest) const; + + private: + // Reserve space in the raw data. For use by builder when putting together a + // new set of data. Init the new storage to zero, to prevent inconsistent + // results if it is not all otherwise set by the requester. + // Return: + // pointer to the new space that was added by this function. + void *reserveSpace(int32_t numBytes, UErrorCode &status); + + // initialize the pointers from this object to the raw data. + void initPtrs(UErrorCode &status); + + SpoofDataHeader *fRawData; // Ptr to the raw memory-mapped data + UBool fDataOwned; // True if the raw data is owned, and needs + // to be deleted when refcount goes to zero. + UDataMemory *fUDM; // If not NULL, our data came from a + // UDataMemory, which we must close when + // we are done. + + uint32_t fMemLimit; // Limit of available raw data space + u_atomic_int32_t fRefCount; + + // Confusable data + int32_t *fCFUKeys; + uint16_t *fCFUValues; + UChar *fCFUStrings; + + friend class ConfusabledataBuilder; +}; + +//--------------------------------------------------------------------------------------- +// +// Raw Binary Data Formats, as loaded from the ICU data file, +// or as built by the builder. +// +//--------------------------------------------------------------------------------------- +struct SpoofDataHeader { + int32_t fMagic; // (0x3845fdef) + uint8_t fFormatVersion[4]; // Data Format. Same as the value in struct UDataInfo + // if there is one associated with this data. + int32_t fLength; // Total length in bytes of this spoof data, + // including all sections, not just the header. + + // The following four sections refer to data representing the confusable data + // from the Unicode.org data from "confusables.txt" + + int32_t fCFUKeys; // byte offset to Keys table (from SpoofDataHeader *) + int32_t fCFUKeysSize; // number of entries in keys table (32 bits each) + + // TODO: change name to fCFUValues, for consistency. + int32_t fCFUStringIndex; // byte offset to String Indexes table + int32_t fCFUStringIndexSize; // number of entries in String Indexes table (16 bits each) + // (number of entries must be same as in Keys table + + int32_t fCFUStringTable; // byte offset of String table + int32_t fCFUStringTableLen; // length of string table (in 16 bit UChars) + + // The following sections are for data from xidmodifications.txt + + int32_t unused[15]; // Padding, Room for Expansion +}; + + + +U_NAMESPACE_END +#endif /* __cplusplus */ + +/** + * Endianness swap function for binary spoof data. + * @internal + */ +U_CAPI int32_t U_EXPORT2 +uspoof_swap(const UDataSwapper *ds, const void *inData, int32_t length, void *outData, + UErrorCode *status); + + +#endif + +#endif /* USPOOFIM_H */ + diff --git a/thirdparty/icu4c/icudt71l.dat b/thirdparty/icu4c/icudt71l.dat index 4c2c1c4d16c62eca0da1e2e6c1769e021d2f9ac8..3fa3af9c2315d6b5df2c70487ca38ade6e188030 100644 GIT binary patch delta 46587 zcmXWk2fUBv|3C2iGY`i($KE4*?@cz9WF-|+Sy>6$Bp;iWgzS)HXBP=cl8_LRR9ccs zlB}rz^L&5*$9X*O*ZaEe>%KqZ`rPAiI7hCH3-3NNF+6>GqHYib$3sDIHYo`HmPdYx z1Zm+QXmm$3C@VK?EE+VHU3L}?I>=yG(O`hQw6|#RoNV+_(O{K4D|gE-hl>VZ$Sj{1 z4KB*vCyEBKNDwSJTQn#xv%FO%Xe`^7E+2G?1c~6t)bhbF9Rq(UAG|8Zwx|$nl}pxF z2+qpF1u6zX)B`?KF(@Vno~anLl9iiP3I@szYbpg( z!9JPgXtm(Hto(JgAc%XX%2W^X$V=s_2X*AwhSh^s@~rGC2R5o6j7rbW)r09W%RkkF zHL}r>8o?fUc0|qK)PsuM$+dzDa_re!!GH47n%Y4@({X7}?Vy&de57{JQts|pC+I5+ zN9zZZ($}}EAIz1(jQYVEId)`&;3IkD=;lu0pCV}k`u7|HSFY*7=~6ko2SE@Hn)V!8*hemig z1Mih>Wjpx*-PJv1FB0mpavbBCz(jphA{i}aJ%zo8(U&z$F{07WTA@vteTq`DOK)D1{K zu_c-*?&cnvb1$uEO&jhv=mGf<57SY*lYB&WkzHkXda8TVN8Ohv)lbo%^fTtu;UGL% zJ(OXL=GkzN2#r^~V8F{vSI>~MSfYM|WxUC9RCq!x8(MB{`-6Pj`t_wXey`7!B&j)uNi zF?oWMe8m~g<3bx;O1}dBskqEvT(QC5@*3B zKJrt5f)v74-SxwiP?Ta(^Iu$1f^v3LURIQqWfiJYjT+RVHg%~-eHze^Ml_}gO=-qm zw6L%HXsK>PdmiK=I`Bxs%10Gl>89>Z`gQ$j2B`-#L_L(z(IEUR&+B`E7kP>4yuu7- zGMhQfWghccz#@^C*wgh4fprCrI?N=uO{v5Pnkj<0<+xfI%K`Fhd!} zaN|bEk&Iy+lbFg2Oyd=1FqcIvW*IB&Z>3x%SMyff{BP0mHe1=Q^F6tfUFr`=zghf{ zefmCTKc8@rL;4TPBl4&`CXaK1Z#l2;H+fP1!6o&dT;?yX#LfTTimP1XIybm&@W1>= z$W3LK2vHi*m?qrA{YkElbW?Yiy<~6MhkiWGAjYuUx&!>m?_5k+{KXai<|eOa@OH6+ zm8{C(gP0)*uV;gLBb)TU!&ZIovYq$z?_eiCah?nO%I{p{5`S`;zerrSxWP^SBgp86 zEn^T0%Lq|*oD3O*aH>or3t7pgFFQG?XKXtjpgm7m-&6MDS;lEkW{3K${El-Oo&PWB z_?>XF2PB>xgp$ZWMv}=)7IIL}nEPl&PkPapan_BO6PT=iiI?+J~IT;j5hEBvj#Ca=qzGDtBUDc&8jt8-A8qLiQl6{$rXn$V0cbfu^5#>w$= z0+X1`6cRHmW?DImh3ZAT!Fo2biSMjCCx7NVe-KLbq*6U8GN?1kWSK%1vXg_n6tJ!^ z#i&AcYEn1V@!wFI?E${!mBKf*=!_$w5wXrQ=O6Dlajg1-!;07PHydZSr0A^9cty z$fq3Pj4@vmWDdgN%t0hVO6DL)%N&IAQz&6^2eoweWw?POIjrx5_DO;)-ckx?aR(#I zP&P{tt*Kp`M%<_G3E5K)$P$DH$}x;%5lgK5nZNjtP}U$6C6U1*nN+fqgPi0dH+jfQ zKJrt5f)wHo3R8rlS%Yvfic^A;l+s_CGL)qp<@HyP6{(cf{8v^~p(@p^tWFJeO=?k_ zI@IM(>eG5U0YWRi}_atc#B=3J&l){uKkLfA!o{2yvl6m=$k9&F`or2AZhZ}SdYt>4DG>h1D9xkK(`7w@y157=Y=gAWbZ>p?$~`}mms ze8K?^a>)8mb3wyZ1fq#pHYKtmeQm?ku(8Fz6v_t3(&Eon`A?T6_=#~kLr zlj0FN^C*w;I9=#UH@ee6{u$67R z%lkI|L>}NEhxnAk9JTI@{F-n0ma}}v_x!+*{KPqa<~+Y}m1|t*1~=2^{~yIIZu2ky z5#)5+O_&HNq>@Hva+8O=I4iON)=D%EnD|LTew)T9=*sY6}v zq#liOy6300x($!encnnaxOJm>mND81#xjoaOkiS8(=4YLGflqC^z?zRFoT)QV!Z)d zc$=;4Zs-4fii3R4aZd0hr-}GO z>x?|eY&PfdVdY)+u#b<~&nFzrGD&jo(x z5|=rW*Eg7q&F2p(S(VR~kuB=2w9D^1A3D&P$MgH#6!7+Pq<~W`pQ}&EQ+&y3&KMAU z%{Tl`w4mE*GLcGVvXGS=1%1k8VTw?c;*_Kmr71%>8qkg>ZPT9t1>FbJa%aqI{KvC*xB#(>_u0)KT;(?Z5)^jxNtj4s z^B+~jh?7JHGLlRdvXYJLsNjn1>^(D(pq>xG) zne}IpS;l6rdo9B6eC_QIb-WrVM2%PX#JciON)=8r7*mO=?k_I^1c0 z^<;e-(2zznrU^|wk-OyG+*8E-H&@(C7rI*6%?`TD9@N1d-nAKKlB~C-wDH zKSh5AFp#IM8)V&JIYfJ?9Hu?o_9N6I^^KCx=o_tmwut#3qe$qSASdz)uktqA_3x7J z6BIQS#K}xnvXP@`5XvcYk()f^rI7w&vV<%x%TShbl-F0GXb`UC@83d|6*YC%R@akl z<^B2|mL2KDV|2H!uY8j6>IuB6p3OYA@5FddoiYN!gFV>LCnenD%f+Fp^RF62;8_ zLd9a1@+PZU#|P};L-ujrMz`c`{-s56w`j#Z5xUWx9y~!$deNIc3@ILjhcb-ej9?_A ziYJ2bGm6nXTRe!2A;DP2F`fxbWD=8^!c-6b9MAItFY*%8c$w+EVxKdZ$t+%FHglNE zJm!0_@B$X{8jD!W5?*I1Z?KFvS7IS!wC2V9fTiM2Tc9sZ2AIOh6z(EdkmUCn*>6A{1 zl0NjZc1iQ!TG5R@C4)%cl0j^(x_2pU`ZIt*3}y&Jd6n7BVGsLDx&Cud{V88^o?j?e z#^FZyGCrjAX8`l`Z7AdT-=)~gM|{jlE5DRi?cM2Qh6rECx> zC<|L(L>86BC{77VQi?K^rIIm~6N>tZ1~jCvc7KMH^@)|kASISGh*MioOrv4oX&ZVWYIV3}w~jC{M+T=D)I{3RS5=9qQANMvP|y zlX#97n8wRYX9hEw#cbvn!CBma#lxv63~s#d@l=MxTch)?;9BOK)zpL3j(oZ?GPllaQQ&lCk;^DSrjo*(&%^ZdfE{KoHG00k+;9TcVr zMJYycN>Gwgl%@=2DMxuKR5Je+6_uz=6{=E=>eQenwWv)U>T)Obs80hL(ul@1p(%bs zE4YihxrgT5OAGF!C9M(`t!YDB?x!6O(4Gf*h==JwM>_Ecoq3eUc$_YDr5oMp!4veP z7rp62U!J5NiKi_3Gk}3S%^(IdgrN*$I3pOzD4t<7&oYMe7c!P{jAsH9nZ#tKFqP+c zo)>sAeg0okOygyy^9nPV$t+%FHglNEJm#~2g}lZh7PExcS;`wM<4u;cf|aadHEUR# zKL2ki*0G)qY-AIg*}~hr!&bKOF57vJ9qeQm@3Wf^*u#hHn9OW3FbDR^T|9n=Yau7^KQpM3xB?wicy1Is}NiB74Sx45DcT$h~G@v1kXl#8GnyQ;|R~7Sr zx8fe1&AC_ILf%J9TG5&|w6*Sj*-kzn+sg;#L-JwSL3XSXggePcv^(>t`Z0Zv%P!ho z>89>3d+nF}SaMQudQi(VqbfnegljG$CCNjy2&}5xc!!&g@+hBkoRgg9E579{LDe7}B20uR zG2$eVfs7=Ri4+p47HMQA3t7oVc5;xDT;wJXdC5n93Q(|W5Gh1qbrFhEjN+7_B&8@# z8OlHlAB*GOHm8eV=s#1;W)SxD{s7)Q}awqkuj~`?WH>44bX+l$WS9LSW9zJ9* ze#SMlkB`~UCsp15A29eJ9jf^N(TPXs%%eQU<8+}b-KzNpl^#4nPkPatKJ?{D`tcO~ z8Nfi!+WtGf=LddFSp38}e&#&C7rj0?EFO+ zjF@A#e{_=$7;%msesH-6_Le{hLExy)Z&;cu>Sog3Ut zEOS54ZH<5Be==>kn+NXV9$L|MxwGGLA2Rg-Ml*(2_0MG<^Lbt01~&33hxv+c`ISGo z#GhO-K9OUETP#YiaC=M@byZo7>eQenwWv)U>T)Obs81tWa38H{Lnr&`D*MVO>CbTO z5popI@!|^e|FYs$<}#1@1})$<^&%FthK+1uGh29@ci74`Dz9`0w8|k$H@eeS1y?Bh(}1D4t<7Ggi6TU>~3Hz-kvI zs;+T`q&^MSxC+vEjT032=xa`^ghgxG@c<9fi7s@rL3akKpJot4w1>)3Ji}3}hskOr(%X8kxyLR@^C*w;I9=#UH@eelxi$ta#-G|w`I^l2N*IL0%9iA-WLQ<%zg zJkJZf$V*J)Wv25=`i^EWlUcmVZ00bRdCX@43we!2EM^I>vy?Yj#+xi>1uI#_YSyrp zw^+w|Hjvn8v5C!W;cebwE8BRN?YzehcCw52+06&+;Y0TF5&QU<{d~d!4swW3Im~Aq zNuU3tier4vaZYfOFF3`QoaQUe@HOA?Eob?T@A-ir`H6G<%z1v{0>APbzjKj4xRgHs ze=08X7gzY3t6bwcH@L|^+~PL>@*lxX=K;b*h!P`C5*f%yGMPvrl`LcP^DM|@SQkHU*rveqJL}jW_m1y3w5;JV8%- z(VIT>LcNr(rDP7|#SIGKtAdVJgq@ zJTLGfFENdmna(TBU?#J8mD$W;F7uer0v7UG`U5XwF-v%zrM$s1-efr|Sjj3@vxc?2 z#X8orfsJfpGh29@ci74{-eo)Qv4g};i(S0WZa!cSAF`K^*vH50=MxTakVAaRVLmer zM>xtc?aw*R3GI`7!71%83H}K}A;SOo^WVrnK{!fWM|N_MlU(Gc@IS6}6r~u&DM3j} zQJONwm6H{yL}lu5FKw)AEAOWr573?mdFUV4|A!6eKu0?1e1y(C%40lE7rN4o?)2aZ zdfKKJz3D?=o}?d7(chQ>4CHABF_<9?Wmv*uI3pOzD4t<7&oYMeF=H9WcqTBBNlf-2 zQ<%zgJkJZf$V*H!=4Gby3Nx6=EM6rs+hPuLna6wn zC97D?8rJd_>sZeQHuAY`k8^^P>GS`E;uK$Uny)y+*FF>9$Zt7o!|(WB{R2O$f8rcJ zbDm$gV4Gk0jo-P*A6)t;i2TWA{^AOMbJhHZuNiQi8{Fg{ZrRyw{^dV{TR}KPmMQr5Sf|H}}w-duhRa zw4@cSX+vA?ryUQ_o(FH4|A!P0(}9k3;t@LYD39?tUFb?Ty3@m3_yj%aMQ{4h*SaU^ z$5ZrY00ViNK@4UHLvK0$huLU2BN)jjo?)~B&oYMeX&TEo#xsG5Oky%qn96fJ&kMZB zOHAWsrt=Cjn8_?&P2cfs=6Jxl%ws+aSg8Ls7O|Kmyv|bIU>R?++}IUzC97D?8rJd_ z>sZeQHnNG$Y$5Ts#XD?e8}G86_t?QscJV&D`G7rq$X-5TA0M-yPdLCq4)H06`HUkR z@Y{^Cmd{Qs@E$~CTYgPZ)rEpFS{zx<~TZo6_3CPI`LagxYD#@j(SnM|aRN}B%6 zWFafr$WD%gMNV>&n>^&TK|b=U3s8_k+(BWAP?Ta6r-ZR3DMe|@P*#6A%2PqRB9*92 z6@67@HJPYxQG=S)(pj52)YZO|deo=EZTE+?weEh}@c`|4kcW7f4s@gwkI2#7FHh2sr|56}00#0jgBZ*ZhBA!dj9?_Ac!tqD%NV@d z;jxTkJQHp^|4&p*Vlq>h%5yx=3%tlnOygx{Fq2ul%53H^mwC)*0SkGJMJ#3sud|dl zSjL;_&uBR-Sjj3@d$nuiTHazE>)F6YHnEv4yv;joWgG9Zo%h(mPImD=yZL}Ue8^rB zA6e|V~ z{D*E?5&F-!(EmAw%2e`EgrXFq1f?lMS;|p?id3R9)u>Kw>QI+EsYiVp(1^w~F&-jiXInQPOCKL&|a}5O< z$x1eIlE`h5k2@$#QI1AJq0b3LLqU|RNl^bM_g@RnBN|Gf~LXjCc%oDSj%RCnF8jD!W z5?*I1Z?J-utYS55c#CyxU?cCcBVn|yo3l7AxexmNn{`+$z&phRMN;y7P69!?BpOPxyVf(@{*7I6d+O1q7Zjbm?9LV z7{w_;$>X6=DOs8_l%*WysX#?4QJE@Kr5e?#K}~8=n>y6xPU;;u|Me9OXhXKc1pL0~pBD3}P@t7|Jk)GlG$f;u%KsY{Fs;={c5hjAsH9nZ#tKFqP+c zo)>tLmzd@iy)38m3Nx6=EM8?cbC}CK=Cgo>Bou54QpA)dN#0;O>AZh zZ}SdY*~YuP$4++fKD+sVJ)Y=?@+0=~ar*r4SA4<&4swW3Im~Aq;V8%Woa3C}Bwuif zFFDOuoZ)M};akq~9pCcAPbzjKj4xWu1a<}a@BH&?xyYw|ib zxXC}<;x_;CAHfNSHDMw|iIGHxghfV@NhOWUWFafr$W9J&k()f^B_H`IKq2m+FhwXz zF^W@yl9Zw}Whi?h6e=eZ+j6|7_xt69TZ-eMi=*}z5; zn=Cf7g|~T!t!(36w(}l4*vT&5XEz_PhY#7yN9^;0K9>9WgaaJp5TA0G&p5)-^!Y!g z_?+XM;3QvgiZ40MSDfK%zTsQW@*Usv13&T;=lGfP{K5r(76<6ygpFQ-q=vqc|lfNhwNGhO(5SJQb*TG8C#LE0d^VQI%>`rv^2tMQ!R( zmpiFPeHze^Ml_}gO=-qm+|4~S=U!THA1!G`YueEEr1`&J(T)db&x1U~!*rk{op^-K zJj!D{P8Yh;%`56Id+-E3=|yk)(3dCa$5ZrY00U2&|ECp$7|alcGK}GjU?ih>hS5CB z7}BS1EaMo@1ST?x$xLA?&+$Aj@FFiUjhC68zLQs&!AxfHDzm+!IdU%Zn9l+h@*0a+ z%o1K_DQ~chH(ALlRDuS-#_Y ze&9!b;v7G7o?nc=Ab;gIe&-^8aEU+D=l`*0iDR7wN~p9k$bWfc8AdLp)3eI?{P~ZQfxk+jy7lyvGi9vWxfG%?Iq^L-z8K{q2(< zv!72mz(Ef48AmvpKL5uQpL3iOoa75m@g=AEiZgu8H+;)kzTEMH$Lcj`CEX zB9*92qKZXTs(XMMvZkyh>u@JcxQBbWk5;s%4foTI_O^LgcH|Kr<#D>wjUM!(H+|?! zKc2d1{`)HiFp#Gi#1K0iDu>D8awMa8hS7{+EaMo@1ST?x$xLA?&+!5;@)FZ{nd!X3 zjEm-frec-{dR5MrbL3p+F`osz&QjjsO_sBQm8@b7Yk7-xtY-ro*~DhH*!SD=9l2F* z00k+;9TX-} z#G)w0C{77VQi{@)p)60)--9-~911t42~BCnUEIw*G$)(?kvI=|39flj*BlK*)k$O^ zSv$uy|BG{Jjk+2wY0ranz7`5UCVTUw_5FBSed!;^ESLFu5q0koWCO+1lPm< zJ~!-d9K!zQDjbT+7;%!wK*sBdaFDFXL<*?}rpe4?AuHMRWtTa~NiOZ&G7ovlM}7+E zFDMIf2Zbp@QHoKV5|pGAr74rJC`&oYQ-O+9qB2#eN;Rr`fEuzUwWzIKN7m&|>QSEt zG^7!YX+l$)aTj-U56wy3Yte%HXh|zt(}uR(PdgrB=ZG+x#}UA`h`FjGB?SDDQmeRG+op3edn z@*0a+%o1K_DQ~dM_&4QpRLhplY0!*}I&-eU(l z*`@z|xtkBz!-wqUBlhtziTxIzaKHu!$)ZfgG{ zZ^_&I%YOtnoCgRKAxez+jc_RGhWXE+k&$FFkwU7KX)-fe)LF@<&Q1<;l8fBrAusvJ zZ(IRcP!^JR$ilJ+MJc9TT$Z4ux>UlVv_=`qQjYRepdyv1Ockn9jq22}T}@d_)|Pc- zU3sUhM|~P-H>44bwVTkCX56LkZW8xcH0NGga33vcWl(F`hPK>KJADtx_VPgDN@9LXr2(H_mS>M_zg>__#(p>c9N6PU;(Ca2H;6f39l9MAKD6)*A<(|DQbyuu7- zGK*K4%^cgzmGhX-0v7Tbi&)GOUS}z9u*^1ZvfQ%^tzP0i#xlXQUgL)&I z)SKDD+q}b8w(&08d5<0JWEbzVn-AE-hwSAe`%COoe9V46;Q$A%IK-#w!+gdOj&h98 ztvfDHaFQ=LrSD5ltH0t5Uu%COzvZm@JHA){z>oZ7{zK;!KXaa6xWKRc#_u+`DF5IR ze`;Tre{qGsxvK9P*VQ+;$v@h+-5{)@7C1WOj0pQ#%*A$wOZ9k)Hw-G{#Sl2JxU|rZz!8U!{Vpp?*Q(pmV5t=#y}% zM6e@LK3Es*2o3}zf@r8ws8gszQknQSp##CdP>xW}(5YZ*s86uO@3PqB>wK^)G$@=c zI2rsK91pDt!#^bPe24hLiWVvc^nj?g#3T)&v&YH&E@ zUGf_xQhbNf&Oa7T3$lb0*+N-@lHuB+%He83y`V-g*pB-Jt3yTnc8~o*kI;|7xnO1J zVsI%qVDm_*e5gXGb*N3KU+AgOv0!4TQ0R`(o?vg#B~&alG`X9nR4u6Ot7fQd_@m$_ zuQN}mVNf`9C>Rnt6U+|%>vfk7cMTp2^|z0&6Q*S5it(QB$Y644mRH&**dE#z+7cQb zWXg0XbS88mbTf246b=tHqI|euxM6TP)Fsq2)H{?RoEAi>Ur zjMWPo1dsXpzn;25$5@5nqV4>w$KXVAifZ4DX+ zO@h0EPC;`&Fx)S=FSsk*%D=U36SNIhh1v%lgAPI0;9D9<;d!T@q<)Z2H44k8}uH`+O`7bq#6;^TLCICxXu5w1SD+p_jr7!vlid zk@SbiUvQ)c9^uDBu7)Rt@)n%AVz4)?ua97z=x`si4xyEi{gJR~c_(`UbOsJDl&85tDn zmk9n1SB)GG(*Eb2=+@|OewjueM?|C0%b~A*mEPjDc(5*8 zJX^8sU9&wHJmA%}`QJ3Hp(G1aC z(R<_RV+VMGyW?fz4Wf;rO``Rpck6E%eJ0*hR8m>7S{3%IjX4r{XKg_+pXbkxY@6@m6t1 zRHS|6LCbdkdunxqA<;q6sVO6(qY}~4(P7a<%J`H?Da9gH{&%R2@e2FI8wNQdW240) zHu>L}-WmI3?3=NF#(^1gL}~{M?X_4W(Z;69ktNaTnWp+$m1#qCOLS8- zDY7}!fZ)|k&t;mQX;E}}bXulKnbv1o8C@6sE7P**)=X=oIU)ms2^n`pw@3FzcSUzc z_e2jw_eJ+d4@D10--&MX68>*QEG3?jl#(qaePn@@LMg>kDy39OshV=ch+ZkZ6Db2y z{%_#1=yREtMo&kNM^8pi$?2ZJ8<{GnZprjcrq$8+Grgawacbk#v(cwB&QHmZsZi>G z;G5{IlsPGNg4a^cM3?)KyWJ>#G2S*f++~c8fW=8x;EuzG#a}ejmIXh z80p~kV6m#PiLs@zW3ha(0z?*r?d(*qGSl*v*vNDSnGi zYCQG%*o(30u^F+cv1=*mN9+99!q}qNl2}S={?ui$<*}8q)v>j)b#`$nb~&~wwk5VL zwmr5Zb||(hwmY^bwlB6nb|iK)b}M!~b~1K4_D$?!EG1SvRx(yTRy`Jvr^R!{3&jh^ z%cdsE$Lq%LN^KOsRk>eM>DEC!9M2F>j%SHyi|2^vix-F&k5^8umRdSqHeNYiFST;K zdc0=5cD!L~lho#^4dPAW&Eof`-W_kB+QO&4cT(S^?y3Eg1||(j8j&<6X>?NCc)R#R z@%Hg(JSCndUNl}IHOQJFmMs>}n*LNGS!0Q;O=2x#En}@?55+ooJ@KqbStrLbW?dS~ zlr<%5TGq^2vt-ShHCxuS*!q-w-n`}%~{^h+5dYlm&6)n-4rX7x+|908*7<0 z{f$e1lYWeyi=B^Mh;_`G9Lo~R5z7_JW5ynIB3&M79(y|0CTJG4b2+*0nlQ9{N0*%z zu5}I7Gb0bXBsF)T3A+QhC+O`8*d;nWR5(;U=w_RTounV4rOQ!USH&L2)pQ-4oG`eF z^ZLAS{a{w4bGSpiL%49{v2f>jm+)ipZsBh69^oGGp5fl{t$vqD@kp2W|DDGg1kJ+r zg4W@Zq4wch;ojkX@n+%We(_ZQc$4sqpkH`kyj1XD&^$IU+%jA!)G1sy(my;XKG3f& z85ABJ9}*rL9~K@TFAz%9jtq&9iMREPx&`e+Bf^v8EyBNrMun%w+l9ONEmhCQ+lJeQ z3TK*P1L3prM&b6sq44+d!{Hy}N5bdg$HM31$HN!m zC&RzR%j6j0r$iH%;^)HQr1RknNs9|LcMEzmemQhg)fFLg=dGC#pi~X$LEJv z#utWH$4`Y%htGt+37-vrAO1T}v!uT9i}7euJSjOTC8=yu^+?^MdP$4&Zp?dEzQkSm z?oMi+^i{lcBsG85{H61^&EF!aWm4;;aQam|DCcgbLL|*~J^gByE7ILfQ~Dj#0QYGJO%HJp3Jj7$Q2&z)OjRyE97nl=aoB3IkZZ-LoF5jk=+G%&g&N=xf2zF|NFXC z`BvrCq4eu;dE)>3+Ui1HH7J$5)%O(XU+MQN>FX9{_`hZPjj$gX4_C5}O2O)2W#A7Q zd^a*C7@Ar)c(_873Wa^sFmT20;Ef<_rWE)0)01bc_&&%NS{G^``zk&zd14~DX;3@- zeDX`l(?b=ab;G?J=;>d*-4>>QOVg6p!DR>b{SNi?uMx?eg7l9-C;z_VJ|B;!Zug#W zs<_(>4+xILj=C6p9_tefO5PnA>ArPIWL4<=va8EhCpB{*KJmX>&h&3Wt|l*YqnRik zdM0^e>he&BpkioDa{4)>nH}{Bo)67Sp62GcKsa}%ovG79BZ8vgf|>fck4?W_ob1%_ zS+4WBHWgWybUN3OTm#Ba3(fl9LsbY0crQA8A2zsSKj}rRwwLD4bQ8iEB9$^dmh`Ci zrnG68AM{IJm@ z|2+=RNmUa3jFi~F{J*ake8 z@^q!T#7Qptje9~Fjq?L87T03e8B?6uU@kzt&Dma+!ewwX)DyH`YH!C>lvD`c{}ek{ zwT#=K8qa3pv*2d>P2ycdlgm7pVwV{1XBP_-Fy3F);JVbc(Y48Snd@@bX4e(2Ev_qF z#|gXm-TWT@N&abmKYx%v#P2ZfGA=eQH7+-&() z&CDmd>$>f_y}D<0FX&#?oz=ak>(`ldHUcN;1fyUt+#?FZg!_b%!Wh9>7$^L}=q5ZU zcnA*(UcxlNUzjNb3bTbUAwq~0o?z}B`wVmRfqfj^s_JouaU;h~#^>ACxpmwrb^Ex^ zakkFaTsE->oezyWG_Hj^;M_Z`ci4U=z-5Tbex_5sYg{LH$i2lS3U)u+&9-HN8#4j# zsN)`{#3RgNJQyZ1f5ouRz<;BnYZ%AS@Ydi-6Tl^Kfm|FHJSoB@z-7jiV3+AGtGKNb z)*zRUe7!r>&0L3boAVmyjn12#UpMv-+s#}UHkujE+>cL=GxGpGR~{(o0rh<@Vkg!=85%rbYzAx-p*dm)11d> z>Ap5-M`*i82TwZ0%w*=9QcN$J)G-0xfo7Lf=hW*9>e9^VuchDgjm+}S{V2Dt=*=Pr zi?_w!Vkvvj>TY`48Y51x{i!}yj1kphLqkJtBZ;Xwr=?+~xbjaccdj1Qmbdm0WK{d| zP1T!gHjA4--8`y2rsLy|%Fc70{hb4ympZezShmjCR=@r6uGhL&=0aaod0%T2AOou(H|v&^~XB6Fj;-TajKlKFc)Ot!`R6f-ndyg$}AHYheV zwlKCa_E7A}*ej4RiDMGg$-3mB$*#%e$Y{Xy}+7Q zmnNp&NV}b;ProNUIz2PJJpI*lKFcR-Ue@GXuiO>69l5)6kK|s;y_Wk;9-r@)?^Kjk zR9keg=uFYOMHh?gERGg0(Na`4(duKJZk=VFV@dOTk&zl@X9fjZk1Ci zS66PRtg2dE^-R?(RbN$ItiD`*z1qL_aP86Bin=p(Kh>4j|5)!Mn#EP(%c82m&@iN7 zYJ=FYvf;|o&z4?adTXhnaYW;o#*#*BV@>13%|*?DEs-tRElXCeUU`0{$gG;y8rB-$ zTGP6tHK(nt?O@weYfi7(ws!y8=hv=Z7qR}0^=H?=y}o*L!{(9ghuV*{f7)*DNbAVz zDC{`X@ny&L4pryg&WpGp?OV2N*|%lyR%z?N)^Xb=Z}Z$XV_V9$YuhWk#I8Sdz1a0? z*PC5$cm34$OP6i;J>5>-)4S(($8@)e-P^l&c3(%d1auG|w!=>JVf@(j6IX#9hi~b_ zPC>CDv7xaRR~_5+Yxl1+nHNph7Ho?dnw%QjYMRUxnQNw8iS5BJ-rVi_dTKQP!uMSU z9x&+rp4TY;uSgDXrh`_S4IY#0Fie~8Fj`yaK<>M{w%DP>LGj?N4vG(-;Xo{d9OV}c zVVV!&5@*QHaE(kgmdG??73jrpFWpElMC%LP%`C$tD1LcOp_XcTq|e-hpi-V#m-e--{H92MRaUK1`0 zj|<|5!neXTVXLrOctSWJ3$Nw_gkX7urv{Zc+D??m6Adx;J!x)=9dLbY@+mZjx@Q&QCW>7px1_h3n?& z7U(i`GksdTr%%5%g+A4lQ4d7N#$SoeOd6B;RMH<)#-{mC_sI(KpB0#rcfo&lNJ;)< z({~iMlsMMJFE-@c=iB9bProOBc>b*hm;AP6)0z*3d8}$)o!Tn4U279N*8IBW+`6YX z_Ou7%{`3oc^U-5F=k83}?YS#@=kdp0dFjA21csXIMrEx5IbE_^i91K$Cj%oI47Il-dv6b39YK#~-d;597Kg3~{Rs z+$L_TTD-cu5?>SV?VgKP2ZJ?5K7NY5H6=&z(fDfoH8V89n!~u$|70K{`95f%NfQGa zn5D@B1r&^HK>UUq4{1(o9xzIp%bG7Vw>7iecRcad6Ioi-lP*u@$^o)|m6~oT_#d6@aJ6PsgL=fM3m6Gzlx`F^Dp+0RZiVYh!NXovmb#AFxaWR^ z3>(x{2qAZ?vzg(ee9^=4S|6i_*3amvouQp)G#SISSzHd6Va($SjTWPoD>Yh;Ra_m{ zz?rm(MkkFMf&~W*)UD?YGc*C3Ky*z!YB*td%kYk2z>uc-)DW%7(R^-LsQJoZMcCkm zp%KxAUkns7=+f93hZu(%_sb!IeW24fG_J-UG(T$2X~r7QYkD=8G~ul`@=mWg+4^>g zr|DYkc0Ah5$#1V;5g(rXLuO&sa&cDk3oCwXS-VcXa76KkW&Pzdmn{_6f4im8R2gei zG{QQmd30Cwg3n9eT%NP?RF<@KOv~xCd-E3--?E-t*BsSj?vHsb=beT2)=#Qb4cg|^ z=C9j4J4(0R+|gPi*6v(x=f*!BLZfh3#sjQS-FS=Q4v$6No}E+u89-m~yRv0drp z#nYC|Yd;^ex}n#6);zNA=;Cji#LD)D)ql=XFTB0V&-BUsAsH*m%PQ}U{ki6qs3+2m zrn8osT<68F#-=5ysw0*?x$1?Y{cBq+x3jJ_?##}ue8yC0K4o57@JW4Xesb(x8!-WuZ(#l z|9XB<@%9p-BDu1t#(r^HXR+BMtET%~^WK6L<<0FAV(j8_Q}4+$ zobCZ8t%7%t}Vw0E-JuU|fYbn$a})7M6}Jlwu|uG-`>XYb)&8K3n#+YMw z#A_1@5(1Ou!)@mu0AqX!3*LXzW>vr^pVpHKTRtuX!R zbScA>$)iCTXp&dfYuO`nVsa94EAsBmACaGsUt9D-QBd*4k~3)H=(3VUd#b&yf3!|4 zUsR#3oL)IftQu2wy6R%pjjFlTfz@Ygo<&<9sh_|2#^M09HKyt7rrH%jE1S{QS62PF z>dY#)W&*IL(YSzooGzWqvjLdV69Upw@j(>sT3+uq&FUEsdrc1#~0 z=ojeBd+|B^?Cv$)RjL|Qoob`5UB{?H*Z?+=4P(V%_Aa>c8T189<2)CArBeEFogJD*S@Zufnfh&AL*3YhS`QNNE+lP<4?x#A-_O!3?!MdP(E1< zk%lDMVv^(VNo*=44?JQaTgFyE78=S8wU8>dhOK9pu+3}>y9#vWNs-;h?q?6NFJV6F zHTDhmG<%MHi+!J!*b8hwdx^cue$0N#e#U+Q`HKC9{SI=2{ek_Fy~*BUZ?lXIV`D%V z+TLb}jU!~F&1f4Z8)usbATg|m%@i9S8-K_p)n(OHNVH9iO`J`NO&Vkq+r^4mHm&>` zejUGw@8q}hF@|{bWWL;&I5vSzVpHV4SlANu#47it*ieSvRLT9JW+_b?Y>d*3;b?-V zjTb8Sm8;LPNw7&mwW)H&5g4y1)DvbfiFXDp@^8bH!=muN4U;@bcgE=O-;L5gj}eW} zZzD4}D%!t|$>4an$)oYHJQgAiMz{^NC5eqhAF29$!+v%*yIJ3?r(O*~FWlr@8~Ap9 zD}SCn&-OwtLk>VX^_}_#wt=mKv_g(T&O%n|TlH)8oAn#?&#*6GX768~C;k^s94u%z z+pK?!ZW@e-Vxv3dkm`_%I75Iw1TsfIS0Am9)hFoFAq9|X{UUvZz5%jW-vsG`?9snx ze9!o{vCk;>89#zNsec->A94^=B1iNuL0-`x*S`)q33*%puKsKNcaV=E-$Qos~e!-EEA$T))sWF};~J_zzKBpH&WPX+4qyYwHT)-Oc;&w78u zG((_aq~QbogNCLLLsytbIPs1QKKb0WicHcf;-OV!vUW1&F{WrAf_Orv z-jU~6MZC3&_+XZUWEzPG<$Fi`w2Js++Jq$Njs#;Wgklf2d0=xna&ujns9Z16W0Axu~1))aqCGKdsF429QDQ{5CKR_4=V;pdYVpMuy> zDsHY>w_P!}o{jKfK4M!mYfUq`WpGEUzAQFmb-?$#;2IHx6Q_Zn`-uS6| zFrq3HoV0))B!LO^4vPj?h(oN2X7gu@I2;DEgvid|T>4)22{v$2I5Qdjn1(@1k)xaz zG)`ivS-uD^TCi>9)^O`oQOpm!XM=#0xyOQ%47b_1XREptO|4d^sk^}v;xH56&V{OD z@RJBp2=nwadYgFW<6(Q*X$UKeFJj8B@bOCKBuhP?htWttYl-u@upW$HyPez0m4SG= zu>QiWy+7{_ch<47w#l~Xwz;-Nwu@|yww+*)+l4NnTi68#xm)-HXyqPZukfVs45;Ql zu*v;Gk8l{I^A*s~Gr|YLSxk~Pfr7fQFN?xxbrkan2B8$r3^P{}4X>G2R{vl)d-3Ieu#dz?f>nj2mkFt(6dIq*!Ybid2ZwHul4`* zN)KLVtQT5wG#{aEr`p=byF@3mSEQp~_h40{vDZ6R#qWti5#QfJk8~^7v ztiE;5*7`DP=Om82Xm|L;;qZkg!I1-}mYvox_{Dy3ZN1^rJ|t?NV<%#Dod+CeTKwZL zd#tqBjZQ9QCjHCP2G8+9es>fefr6D{snF+G50phE{Z@)nedwKBO663FNL3G3yh}US z=2GUjR+FRscWwTsCjVnU|NR-an48RYl`j(_;@dA3{!AHsm1Q9FNBEccSNP-n>-Rnp^|Sj-1A6IBz&s}Wfrm@pf@<1S}MeU!5p$9%>(t9Cm7 ztJ}lkI@9E93Nr2IGNPV|dNHapsxEp<^e54Oj_&e0%sp(5Gv}DA%?G$6Uis!W^C7Oo zyxV-%w9ov!xq|=R%*G6OO^*qSvBp%#4D~({L!rEzF`8Iy?7guOvF6y6*z(xLVyq%Z zxw5j;u>-NkxX)w1jn&3=#rejC#}&jq8uvq-eSB$7s(@1TvMdvyp*qe zZu#)@10W~4SLaLfKbbFHpMRSBb-p$=Hr+XO>9h=l7mua3rJmzvFL;Y9Sx~f~e!)iz zE-#qu`^y5iw3%szX<5GKxq9FIX$R8|rJYV|;cutC&%Mv#f5xUer=Re>%K7qH=|$;R ze7B~*nf^igPw7iD{QXAyZOV|i6B*%|k(nl-4$;B`nY%OpnAtBPX!t{>eb$Jq1zFE# zy_*%C-OtU;zLiyw-IRSb`x57z^Lq9Z*&hCdIZJYOuV8Q{Y&*V|sAm zV}+j;?p^pDcX8qH8RFQY8(dtGX@<3^89Xgz z&_y=<_C-AJX7#p4TXlScb+gsL+wyhgtIPKV{Gt4Cz~O*7Vnu4jqKf4erz=LxdZ5yt z52zf%msP&Y*;hI8!}#I+tyz5FjjECS{rqU&i631(1~RwWFK~5rSYUhL1H7rGwdQQi zmB8_QLaiG=kzZT;dF|w&z`D70QFZRT2jr_db?_AajYm#B;u`G9pReB;{AvA_dSew;GhB0<#;&zRA?q7gH|}UW81ro7!N!*xKW>cVuQiTp%5HkP$))M#rcas@ zm&G=nZaUl4+hpQn_-OvY&~KMrUp8!c3ZJt)PUIJaW`-v4NqkXgH0Ct&n+uztXl`k? zU2(a&c7@xDq7}_6J_!A21>T|w&A_a6D*r)Ce~Wk6&6XFv()bMiXxNFcornmpS{>h- z#Xr^hr`8c|)opqF3*4Eum)gE*dx?8@jqln5ej&eU-M8x;)?4@y>wPzTutC~Tzu~8i zLpO;f{OZk{BYHNM@gGOL+J0bxONW)O;!EaM&Arr7!ylMi$0NMeIia(jzuY->-h?fC zwjAGbd&{|bOZXw%0=NIN{aoaik@-6s_#Y!1`DJ`F|6*6d!#{Q1?$UJIbq~WV_bQ&j z3Ja21SOap&iFl(c+!|N7pRP=_2o(nqp)5hLfg)KG5FAjm4>Byk0Xo11m;y99O#`NHN4n=fp>u=&B}2b&*kez5t8F#0na#?HW6y&0%{rYvf9CZh#7KnD~7-#7&z zM}RCn7)%<_S>U%Mp+FQ61H=OfKr;BgD=?m&jbWe7eC`wiFERw*Ga<}kRS4c-3Bl-y zp^PsAFyq-d$UFyzaHPYT34k{{7Y&+=?K~Wt2irWPBVmaIBaXy*v^w!&b&Fz#H%dK6i>o z#&|RYu?_?!g3&h%&;!u^)lrGq(kjO!oR@@jGLV7>q+pwh@77eTD$7LdAQNtWCK{Lt z#+8W(K_=n@nfM<)Y!)h@`TZ;unuVJu3&WnpI0D{SlQv3k4#8 zGQi!wl;K8|q0%xmwhZ|eVQWPhR+MT*xmJ{GMY&cSqu@+AY~|Rmz_?&fgc_x;VB&!$ zz!3-Npf_8IMpU91l}OXrRO7N#V;E_SYLKo)-dY^1gRKsQ*5SN5>@UHwCAgQC;JL6L z_y2xGe)eO3KSq{5+&%K=EC5@&y~RkVXn<}px~=FgpsO6J?g91cK?JM^YZrPLm17TM z184v(pab-PC*TG60Z~9SU=BvuNIV8S4(tK;0#5l zJHC1YjcYATSB=0A>SWz#JePhydmRkw6NN4?H`(hj|V-3>+KotD3<2s=V3d zAfJ7zQlKBpIA~1AvvKNNgk*iO9DO{yMC~QAP3k7~Dz!J;2E%q46WCMgEIj|{WiWcD z;(+Z&)>Xb38}ZT@U64(zC$>(Gr?55JjAw`0OkgMC#UUST{jfhB$gt_bkKwvualNa2 z0dqBtSg)T86amHZ+WjROu?n-Fy_&JwaMlg?GkNlEAQWzh>ANA0?*{JT2LIm;48RS9 z!VT{mxM{JXS$i3?0;SAE#Wg7UZnleD4_4*KAr4-Wd^;4~PgAx(s78Z!DK?T5^M z@-{*yTv!t()2krZgOEQ6Wdy+%gmZ$iAB=P`(zEfXnTyu2KXF ziadU3nlGy^5SXJY*98H=y5+i~!0W&X;G|BY55TKbf5OXC z2Z2Q68vXNljp|RpaeX&t&i3k$>7N1KK(05C=LEKTLx8~m48gSP5KQon#hY8MhL<$% z*osq-@J7NL8N3ZakO0hl2N+rnM-7}Y(P(Q7!@8dUAkpZCm!#Z`4;!ZdVUSOae+J$$ z2HFPNK4rTF*kx<9GuquR%P6~1*pgP;yW6Yn1MFE~DAGf*9c@3uey;so(S9*@n(TEf z<3OSYSU>}CfB@J6cEAw%s2W<0JZj`oBaa$+)OepqjeKgn(W93019#z2sBznLA*aCKdJ>UQY z;t1Y6z&X?vY7W&vx%ILb97H`%(&HpOPSWEfJu1`VL~4csDFcL-irT_%525Bz(Uj9c z-XDS^H0U%o)K8_kWSs#CfKb09aV!!x6C@gv49RpT6roChTT?O}+)UXHg$~&c#Xt$* zX3E8WF4DO;o{Rl_So2}ahbzI1*jaIu+l>D1T;twdILJ7$s-u;I57O5 zy#jC@$H}sH5bYT6sCWo)Y(<=)xCyTR&QPXYUYr-2*52)tAB)-aLYUO7ItcbF$T zFnqM*-uw67|0Iw$B5lOn}xtSw`C1&qyx; z$AHEGM%oB$9Z*R_dR3A?(A=w*y82kDv6q$p(5sOe`!rG$unaf=9PHzyCwoUo%)kuE z*f&G6?F*Eg`+}sAz&KzY;NKrC+4KfW{k>waDz%wN!J@G9qx~m(vjW>YzJNi z-WzyWD(-z)daU0h?ddg1ZP?!IkCI;Mjgm%-eNoce*hw3RlGY4FOWFO=(pSJ!17?Zu zH%s0FF_L3%tmFf10Co+;N{0vHP|o=_hm|7 z0v-cd(#?Tvsk$#m3Izs$`}=b-TDektZ>}gE>dTX+3=~M`dJ7~?Ux8%XUm&dqHVhO> zOuzNESz7^A0o6baPz%%nj{x1|9BH$*!oKu?vmm#n>I?IY`Hvi6m=pR9wRMK@4t zI&HSBLu4H)>o8f*ku@R$$TL^ghy)-VDeH%2ZIX4AtfOUZmUWD*V`U9mkF(-sognK( z(jxJ897vXRimWqboh9pRS?9<)SJrv5&X;w8tP5qmP}W7Vw#XWk7A2R;x=hxKWDQCR zd%3K|3VBm0>nd4S%eqF^wX&|0HRC|1sb$T|S|e+%tT|Z=vewJmAZuG$+sWEq)()~B zBI{e?6e^T8krrgVK-R6Y-X!ZQvQ|Y=9tT+$nJHZ?>*r!9-6!jySW3sq+89UadRbpg zp!AqT(s}bquatFcmPk8^xumbjn$M@SAZs^SyUTiF0olDQq>IW(ACYyfmC_$nk-j49 zfLcl~kae-FKU+$c>$2X}O6e1_etZ1{5j)I2GIj1Jef0q8`X16JkC1kJiS)i>q(fdO zz3v3*TPI1ooh4myj`Xp&NE_ZEo%G%WL`;5*33}qwL2*Ee0@8#Rr77?ydLkr1I~46h zxKK`0WY1HM<$21{J5M=Y=ZVk@g%qRYqZGsPM5u*wQXpljY@aGy=oQ+T2s4l<0X+)k zxv)y`E)2mrS_$QgU$%%75gLn><2g!1$AXyDWUO)_U5;P`2ezAyG_7dDn;!^Gc;lBG z27^l!UZVI>A-0}!got`cX^QSpUoGe*MUJS)q{%{oPSRu%sow|?$tLPOrO84OPSRwd z2&YAkZ;~bp#WxXBlJih}lhS0N*d}SR_{kO;GYVRfB}#4|@GBSM3LE14#O#Ub5uYdaPVA3(y~6IvLWGkv*{N})iT4u~C9QCM z(qt#CFn-dMo{Sgl@gp4ImmG*YMFL1ih}cke;_k%Qsc6!4m>4c`cw+e%W&1_hj1ISNQ&5A(V;Oy#;ZB3OPDZAs1C47gB*r$xe}h3OP1VAs1VL;>kh_FsXRbk3iE! zBu$OED4#_k9Hd40G>Tx*0XpF#LQ51npuj-@!Ud$mg(>)z6;7aN0L221%F%$MhzE?9 z={S*cH<_Ay7`kIiQNIaMRpB#2W3t4JHTGy)x?cOg%m4iEI-m zR>(7@6&6f)=i7OLZrkqrYAR{jF}2}0p5TQFb(hp`~ZJoIxqv62?PMMdZvsC+?o3S E0M=CHcK`qY delta 754 zcmWmBJ!lhg7{~EnFHIWX&$n7_(!}$vFG()u`jXVN&7h!ykm{n~5Jb>LQ(THTC#0{^c zbcTUBm(sP`BIM`}Er%UFq?Hy&^VDm1v_uQTj@GDq-BDA3bK{O;bcp7t?m9YG>tjbn zS~=dSFY({YX`P@`gIQgq zZY8T>T3*U(jGFU(dO!=2=?lgW-*h2h8aIo<(PFR;27IW89|1I=wqJ+!*no}Lgw5Cj z6I-zj+pz;X(TH7W!fxzAGeQWX1rbEC7Z&zmKMvp^4xtroXh#QPh$8_TNgPIMrWka- kH{Q#&IMGpW!9gdw(2X9XkwGtxAd5cqV_?Qx$hqhL1MT0(l>h($