324 lines
13 KiB
C++
324 lines
13 KiB
C++
// © 2016 and later: Unicode, Inc. and others.
|
|
// License & terms of use: http://www.unicode.org/copyright.html
|
|
/*
|
|
**********************************************************************
|
|
* Copyright (C) 1999-2011, International Business Machines
|
|
* Corporation and others. All Rights Reserved.
|
|
**********************************************************************
|
|
*
|
|
* ucnv_cnv.h:
|
|
* Definitions for converter implementations.
|
|
*
|
|
* Modification History:
|
|
*
|
|
* Date Name Description
|
|
* 05/09/00 helena Added implementation to handle fallback mappings.
|
|
* 06/29/2000 helena Major rewrite of the callback APIs.
|
|
*/
|
|
|
|
#ifndef UCNV_CNV_H
|
|
#define UCNV_CNV_H
|
|
|
|
#include "unicode/utypes.h"
|
|
|
|
#if !UCONFIG_NO_CONVERSION
|
|
|
|
#include "unicode/ucnv.h"
|
|
#include "unicode/ucnv_err.h"
|
|
#include "unicode/uset.h"
|
|
#include "uset_imp.h"
|
|
|
|
U_CDECL_BEGIN
|
|
|
|
/* this is used in fromUnicode DBCS tables as an "unassigned" marker */
|
|
#define missingCharMarker 0xFFFF
|
|
|
|
/*
|
|
* #define missingUCharMarker 0xfffe
|
|
*
|
|
* commented out because there are actually two values used in toUnicode tables:
|
|
* U+fffe "unassigned"
|
|
* U+ffff "illegal"
|
|
*/
|
|
|
|
/** Forward declaration, see ucnv_bld.h */
|
|
struct UConverterSharedData;
|
|
typedef struct UConverterSharedData UConverterSharedData;
|
|
|
|
/* function types for UConverterImpl ---------------------------------------- */
|
|
|
|
/* struct with arguments for UConverterLoad and ucnv_load() */
|
|
typedef struct {
|
|
int32_t size; /* sizeof(UConverterLoadArgs) */
|
|
int32_t nestedLoads; /* count nested ucnv_load() calls */
|
|
UBool onlyTestIsLoadable; /* input: don't actually load */
|
|
UBool reserved0; /* reserved - for good alignment of the pointers */
|
|
int16_t reserved; /* reserved - for good alignment of the pointers */
|
|
uint32_t options;
|
|
const char *pkg, *name, *locale;
|
|
} UConverterLoadArgs;
|
|
|
|
#define UCNV_LOAD_ARGS_INITIALIZER \
|
|
{ (int32_t)sizeof(UConverterLoadArgs), 0, false, false, 0, 0, NULL, NULL, NULL }
|
|
|
|
typedef void (*UConverterLoad) (UConverterSharedData *sharedData,
|
|
UConverterLoadArgs *pArgs,
|
|
const uint8_t *raw, UErrorCode *pErrorCode);
|
|
typedef void (*UConverterUnload) (UConverterSharedData *sharedData);
|
|
|
|
typedef void (*UConverterOpen) (UConverter *cnv, UConverterLoadArgs *pArgs, UErrorCode *pErrorCode);
|
|
typedef void (*UConverterClose) (UConverter *cnv);
|
|
|
|
typedef enum UConverterResetChoice {
|
|
UCNV_RESET_BOTH,
|
|
UCNV_RESET_TO_UNICODE,
|
|
UCNV_RESET_FROM_UNICODE
|
|
} UConverterResetChoice;
|
|
|
|
typedef void (*UConverterReset) (UConverter *cnv, UConverterResetChoice choice);
|
|
|
|
/*
|
|
* Converter implementation function(s) for ucnv_toUnicode().
|
|
* If the toUnicodeWithOffsets function pointer is NULL,
|
|
* then the toUnicode function will be used and the offsets will be set to -1.
|
|
*
|
|
* Must maintain state across buffers. Use toUBytes[toULength] for partial input
|
|
* sequences; it will be checked in ucnv.c at the end of the input stream
|
|
* to detect truncated input.
|
|
* Some converters may need additional detection and may then set U_TRUNCATED_CHAR_FOUND.
|
|
*
|
|
* The toUnicodeWithOffsets must write exactly as many offset values as target
|
|
* units. Write offset values of -1 for when the source index corresponding to
|
|
* the output unit is not known (e.g., the character started in an earlier buffer).
|
|
* The pArgs->offsets pointer need not be moved forward.
|
|
*
|
|
* At function return, either one of the following conditions must be true:
|
|
* - U_BUFFER_OVERFLOW_ERROR and the target is full: target==targetLimit
|
|
* - another error code with toUBytes[toULength] set to the offending input
|
|
* - no error, and the source is consumed: source==sourceLimit
|
|
*
|
|
* The ucnv.c code will handle the end of the input (reset)
|
|
* (reset, and truncation detection) and callbacks.
|
|
*/
|
|
typedef void (*UConverterToUnicode) (UConverterToUnicodeArgs *, UErrorCode *);
|
|
|
|
/*
|
|
* Same rules as for UConverterToUnicode.
|
|
* A lead surrogate is kept in fromUChar32 across buffers, and if an error
|
|
* occurs, then the offending input code point must be put into fromUChar32
|
|
* as well.
|
|
*/
|
|
typedef void (*UConverterFromUnicode) (UConverterFromUnicodeArgs *, UErrorCode *);
|
|
|
|
/*
|
|
* Converter implementation function for ucnv_convertEx(), for direct conversion
|
|
* between two charsets without pivoting through UTF-16.
|
|
* The rules are the same as for UConverterToUnicode and UConverterFromUnicode.
|
|
* In addition,
|
|
* - The toUnicode side must behave and keep state exactly like the
|
|
* UConverterToUnicode implementation for the same source charset.
|
|
* - A U_USING_DEFAULT_WARNING can be set to request to temporarily fall back
|
|
* to pivoting. When this function is called, the conversion framework makes
|
|
* sure that this warning is not set on input.
|
|
* - Continuing a partial match and flushing the toUnicode replay buffer
|
|
* are handled by pivoting, using the toUnicode and fromUnicode functions.
|
|
*/
|
|
typedef void (*UConverterConvert) (UConverterFromUnicodeArgs *pFromUArgs,
|
|
UConverterToUnicodeArgs *pToUArgs,
|
|
UErrorCode *pErrorCode);
|
|
|
|
/*
|
|
* Converter implementation function for ucnv_getNextUChar().
|
|
* If the function pointer is NULL, then the toUnicode function will be used.
|
|
*
|
|
* Will be called at a character boundary (toULength==0).
|
|
* May return with
|
|
* - U_INDEX_OUTOFBOUNDS_ERROR if there was no output for the input
|
|
* (the return value will be ignored)
|
|
* - U_TRUNCATED_CHAR_FOUND or another error code (never U_BUFFER_OVERFLOW_ERROR!)
|
|
* with toUBytes[toULength] set to the offending input
|
|
* (the return value will be ignored)
|
|
* - return UCNV_GET_NEXT_UCHAR_USE_TO_U, without moving the source pointer,
|
|
* to indicate that the ucnv.c code shall call the toUnicode function instead
|
|
* - return a real code point result
|
|
*
|
|
* Unless UCNV_GET_NEXT_UCHAR_USE_TO_U is returned, the source bytes must be consumed.
|
|
*
|
|
* The ucnv.c code will handle the end of the input (reset)
|
|
* (except for truncation detection!) and callbacks.
|
|
*/
|
|
typedef UChar32 (*UConverterGetNextUChar) (UConverterToUnicodeArgs *, UErrorCode *);
|
|
|
|
typedef void (*UConverterGetStarters)(const UConverter* converter,
|
|
UBool starters[256],
|
|
UErrorCode *pErrorCode);
|
|
|
|
/* If this function pointer is null or if the function returns null
|
|
* the name field in static data struct should be returned by
|
|
* ucnv_getName() API function
|
|
*/
|
|
typedef const char * (*UConverterGetName) (const UConverter *cnv);
|
|
|
|
/**
|
|
* Write the codepage substitution character.
|
|
* If this function is not set, then ucnv_cbFromUWriteSub() writes
|
|
* the substitution character from UConverter.
|
|
* For stateful converters, it is typically necessary to handle this
|
|
* specifically for the converter in order to properly maintain the state.
|
|
*/
|
|
typedef void (*UConverterWriteSub) (UConverterFromUnicodeArgs *pArgs, int32_t offsetIndex, UErrorCode *pErrorCode);
|
|
|
|
/**
|
|
* For converter-specific safeClone processing
|
|
* If this function is not set, then ucnv_safeClone assumes that the converter has no private data that changes
|
|
* after the converter is done opening.
|
|
* If this function is set, then it is called just after a memcpy() of
|
|
* converter data to the new, empty converter, and is expected to set up
|
|
* the initial state of the converter. It is not expected to increment the
|
|
* reference counts of the standard data types such as the shared data.
|
|
*/
|
|
typedef UConverter * (*UConverterSafeClone) (const UConverter *cnv,
|
|
void *stackBuffer,
|
|
int32_t *pBufferSize,
|
|
UErrorCode *status);
|
|
|
|
/**
|
|
* Filters for some ucnv_getUnicodeSet() implementation code.
|
|
*/
|
|
typedef enum UConverterSetFilter {
|
|
UCNV_SET_FILTER_NONE,
|
|
UCNV_SET_FILTER_DBCS_ONLY,
|
|
UCNV_SET_FILTER_2022_CN,
|
|
UCNV_SET_FILTER_SJIS,
|
|
UCNV_SET_FILTER_GR94DBCS,
|
|
UCNV_SET_FILTER_HZ,
|
|
UCNV_SET_FILTER_COUNT
|
|
} UConverterSetFilter;
|
|
|
|
/**
|
|
* Fills the set of Unicode code points that can be converted by an ICU converter.
|
|
* The API function ucnv_getUnicodeSet() clears the USet before calling
|
|
* the converter's getUnicodeSet() implementation; the converter should only
|
|
* add the appropriate code points to allow recursive use.
|
|
* For example, the ISO-2022-JP converter will call each subconverter's
|
|
* getUnicodeSet() implementation to consecutively add code points to
|
|
* the same USet, which will result in a union of the sets of all subconverters.
|
|
*
|
|
* For more documentation, see ucnv_getUnicodeSet() in ucnv.h.
|
|
*/
|
|
typedef void (*UConverterGetUnicodeSet) (const UConverter *cnv,
|
|
const USetAdder *sa,
|
|
UConverterUnicodeSet which,
|
|
UErrorCode *pErrorCode);
|
|
|
|
UBool CONVERSION_U_SUCCESS (UErrorCode err);
|
|
|
|
/**
|
|
* UConverterImpl contains all the data and functions for a converter type.
|
|
* Its function pointers work much like a C++ vtable.
|
|
* Many converter types need to define only a subset of the functions;
|
|
* when a function pointer is NULL, then a default action will be performed.
|
|
*
|
|
* Every converter type must implement toUnicode, fromUnicode, and getNextUChar,
|
|
* otherwise the converter may crash.
|
|
* Every converter type that has variable-length codepage sequences should
|
|
* also implement toUnicodeWithOffsets and fromUnicodeWithOffsets for
|
|
* correct offset handling.
|
|
* All other functions may or may not be implemented - it depends only on
|
|
* whether the converter type needs them.
|
|
*
|
|
* When open() fails, then close() will be called, if present.
|
|
*/
|
|
struct UConverterImpl {
|
|
UConverterType type;
|
|
|
|
UConverterLoad load;
|
|
UConverterUnload unload;
|
|
|
|
UConverterOpen open;
|
|
UConverterClose close;
|
|
UConverterReset reset;
|
|
|
|
UConverterToUnicode toUnicode;
|
|
UConverterToUnicode toUnicodeWithOffsets;
|
|
UConverterFromUnicode fromUnicode;
|
|
UConverterFromUnicode fromUnicodeWithOffsets;
|
|
UConverterGetNextUChar getNextUChar;
|
|
|
|
UConverterGetStarters getStarters;
|
|
UConverterGetName getName;
|
|
UConverterWriteSub writeSub;
|
|
UConverterSafeClone safeClone;
|
|
UConverterGetUnicodeSet getUnicodeSet;
|
|
|
|
UConverterConvert toUTF8;
|
|
UConverterConvert fromUTF8;
|
|
};
|
|
|
|
extern const UConverterSharedData
|
|
_MBCSData, _Latin1Data,
|
|
_UTF8Data, _UTF16BEData, _UTF16LEData, _UTF32BEData, _UTF32LEData,
|
|
_ISO2022Data,
|
|
_LMBCSData1,_LMBCSData2, _LMBCSData3, _LMBCSData4, _LMBCSData5, _LMBCSData6,
|
|
_LMBCSData8,_LMBCSData11,_LMBCSData16,_LMBCSData17,_LMBCSData18,_LMBCSData19,
|
|
_HZData,_ISCIIData, _SCSUData, _ASCIIData,
|
|
_UTF7Data, _Bocu1Data, _UTF16Data, _UTF32Data, _CESU8Data, _IMAPData, _CompoundTextData;
|
|
|
|
U_CDECL_END
|
|
|
|
/** Always use fallbacks from codepage to Unicode */
|
|
#define TO_U_USE_FALLBACK(useFallback) true
|
|
#define UCNV_TO_U_USE_FALLBACK(cnv) true
|
|
|
|
/** Use fallbacks from Unicode to codepage when cnv->useFallback or for private-use code points */
|
|
#define IS_PRIVATE_USE(c) ((uint32_t)((c)-0xe000)<0x1900 || (uint32_t)((c)-0xf0000)<0x20000)
|
|
#define FROM_U_USE_FALLBACK(useFallback, c) ((useFallback) || IS_PRIVATE_USE(c))
|
|
#define UCNV_FROM_U_USE_FALLBACK(cnv, c) FROM_U_USE_FALLBACK((cnv)->useFallback, c)
|
|
|
|
/**
|
|
* Magic number for ucnv_getNextUChar(), returned by a
|
|
* getNextUChar() implementation to indicate to use the converter's toUnicode()
|
|
* instead of the native function.
|
|
* @internal
|
|
*/
|
|
#define UCNV_GET_NEXT_UCHAR_USE_TO_U -9
|
|
|
|
U_CFUNC void
|
|
ucnv_getCompleteUnicodeSet(const UConverter *cnv,
|
|
const USetAdder *sa,
|
|
UConverterUnicodeSet which,
|
|
UErrorCode *pErrorCode);
|
|
|
|
U_CFUNC void
|
|
ucnv_getNonSurrogateUnicodeSet(const UConverter *cnv,
|
|
const USetAdder *sa,
|
|
UConverterUnicodeSet which,
|
|
UErrorCode *pErrorCode);
|
|
|
|
U_CFUNC void
|
|
ucnv_fromUWriteBytes(UConverter *cnv,
|
|
const char *bytes, int32_t length,
|
|
char **target, const char *targetLimit,
|
|
int32_t **offsets,
|
|
int32_t sourceIndex,
|
|
UErrorCode *pErrorCode);
|
|
U_CFUNC void
|
|
ucnv_toUWriteUChars(UConverter *cnv,
|
|
const UChar *uchars, int32_t length,
|
|
UChar **target, const UChar *targetLimit,
|
|
int32_t **offsets,
|
|
int32_t sourceIndex,
|
|
UErrorCode *pErrorCode);
|
|
|
|
U_CFUNC void
|
|
ucnv_toUWriteCodePoint(UConverter *cnv,
|
|
UChar32 c,
|
|
UChar **target, const UChar *targetLimit,
|
|
int32_t **offsets,
|
|
int32_t sourceIndex,
|
|
UErrorCode *pErrorCode);
|
|
|
|
#endif
|
|
|
|
#endif /* UCNV_CNV */
|