477 lines
17 KiB
C++
477 lines
17 KiB
C++
// © 2016 and later: Unicode, Inc. and others.
|
|
// License & terms of use: http://www.unicode.org/copyright.html
|
|
/*
|
|
***************************************************************************
|
|
* Copyright (C) 1999-2014 International Business Machines Corporation *
|
|
* and others. All rights reserved. *
|
|
***************************************************************************
|
|
*/
|
|
|
|
#include "unicode/utypes.h"
|
|
|
|
#if !UCONFIG_NO_BREAK_ITERATION
|
|
|
|
#include "unicode/ucptrie.h"
|
|
#include "unicode/utypes.h"
|
|
#include "rbbidata.h"
|
|
#include "rbbirb.h"
|
|
#include "udatamem.h"
|
|
#include "cmemory.h"
|
|
#include "cstring.h"
|
|
#include "umutex.h"
|
|
|
|
#include "uassert.h"
|
|
|
|
|
|
U_NAMESPACE_BEGIN
|
|
|
|
//-----------------------------------------------------------------------------
|
|
//
|
|
// Constructors.
|
|
//
|
|
//-----------------------------------------------------------------------------
|
|
RBBIDataWrapper::RBBIDataWrapper(const RBBIDataHeader *data, UErrorCode &status) {
|
|
init0();
|
|
init(data, status);
|
|
}
|
|
|
|
RBBIDataWrapper::RBBIDataWrapper(const RBBIDataHeader *data, enum EDontAdopt, UErrorCode &status) {
|
|
init0();
|
|
init(data, status);
|
|
fDontFreeData = true;
|
|
}
|
|
|
|
RBBIDataWrapper::RBBIDataWrapper(UDataMemory* udm, UErrorCode &status) {
|
|
init0();
|
|
if (U_FAILURE(status)) {
|
|
return;
|
|
}
|
|
const DataHeader *dh = udm->pHeader;
|
|
int32_t headerSize = dh->dataHeader.headerSize;
|
|
if ( !(headerSize >= 20 &&
|
|
dh->info.isBigEndian == U_IS_BIG_ENDIAN &&
|
|
dh->info.charsetFamily == U_CHARSET_FAMILY &&
|
|
dh->info.dataFormat[0] == 0x42 && // dataFormat="Brk "
|
|
dh->info.dataFormat[1] == 0x72 &&
|
|
dh->info.dataFormat[2] == 0x6b &&
|
|
dh->info.dataFormat[3] == 0x20 &&
|
|
isDataVersionAcceptable(dh->info.formatVersion))
|
|
) {
|
|
status = U_INVALID_FORMAT_ERROR;
|
|
return;
|
|
}
|
|
const char *dataAsBytes = reinterpret_cast<const char *>(dh);
|
|
const RBBIDataHeader *rbbidh = reinterpret_cast<const RBBIDataHeader *>(dataAsBytes + headerSize);
|
|
init(rbbidh, status);
|
|
fUDataMem = udm;
|
|
}
|
|
|
|
UBool RBBIDataWrapper::isDataVersionAcceptable(const UVersionInfo version) {
|
|
return RBBI_DATA_FORMAT_VERSION[0] == version[0];
|
|
}
|
|
|
|
|
|
//-----------------------------------------------------------------------------
|
|
//
|
|
// init(). Does most of the work of construction, shared between the
|
|
// constructors.
|
|
//
|
|
//-----------------------------------------------------------------------------
|
|
void RBBIDataWrapper::init0() {
|
|
fHeader = nullptr;
|
|
fForwardTable = nullptr;
|
|
fReverseTable = nullptr;
|
|
fRuleSource = nullptr;
|
|
fRuleStatusTable = nullptr;
|
|
fTrie = nullptr;
|
|
fUDataMem = nullptr;
|
|
fRefCount = 0;
|
|
fDontFreeData = true;
|
|
}
|
|
|
|
void RBBIDataWrapper::init(const RBBIDataHeader *data, UErrorCode &status) {
|
|
if (U_FAILURE(status)) {
|
|
return;
|
|
}
|
|
fHeader = data;
|
|
if (fHeader->fMagic != 0xb1a0 || !isDataVersionAcceptable(fHeader->fFormatVersion)) {
|
|
status = U_INVALID_FORMAT_ERROR;
|
|
return;
|
|
}
|
|
// Note: in ICU version 3.2 and earlier, there was a formatVersion 1
|
|
// that is no longer supported. At that time fFormatVersion was
|
|
// an int32_t field, rather than an array of 4 bytes.
|
|
|
|
fDontFreeData = false;
|
|
if (data->fFTableLen != 0) {
|
|
fForwardTable = (RBBIStateTable *)((char *)data + fHeader->fFTable);
|
|
}
|
|
if (data->fRTableLen != 0) {
|
|
fReverseTable = (RBBIStateTable *)((char *)data + fHeader->fRTable);
|
|
}
|
|
|
|
fTrie = ucptrie_openFromBinary(UCPTRIE_TYPE_FAST,
|
|
UCPTRIE_VALUE_BITS_ANY,
|
|
(uint8_t *)data + fHeader->fTrie,
|
|
fHeader->fTrieLen,
|
|
nullptr, // *actual length
|
|
&status);
|
|
if (U_FAILURE(status)) {
|
|
return;
|
|
}
|
|
|
|
UCPTrieValueWidth width = ucptrie_getValueWidth(fTrie);
|
|
if (!(width == UCPTRIE_VALUE_BITS_8 || width == UCPTRIE_VALUE_BITS_16)) {
|
|
status = U_INVALID_FORMAT_ERROR;
|
|
return;
|
|
}
|
|
|
|
fRuleSource = ((char *)data + fHeader->fRuleSource);
|
|
fRuleString = UnicodeString::fromUTF8(StringPiece(fRuleSource, fHeader->fRuleSourceLen));
|
|
U_ASSERT(data->fRuleSourceLen > 0);
|
|
|
|
fRuleStatusTable = (int32_t *)((char *)data + fHeader->fStatusTable);
|
|
fStatusMaxIdx = data->fStatusTableLen / sizeof(int32_t);
|
|
|
|
fRefCount = 1;
|
|
|
|
#ifdef RBBI_DEBUG
|
|
char *debugEnv = getenv("U_RBBIDEBUG");
|
|
if (debugEnv && uprv_strstr(debugEnv, "data")) {this->printData();}
|
|
#endif
|
|
}
|
|
|
|
|
|
//-----------------------------------------------------------------------------
|
|
//
|
|
// Destructor. Don't call this - use removeReference() instead.
|
|
//
|
|
//-----------------------------------------------------------------------------
|
|
RBBIDataWrapper::~RBBIDataWrapper() {
|
|
U_ASSERT(fRefCount == 0);
|
|
ucptrie_close(fTrie);
|
|
fTrie = nullptr;
|
|
if (fUDataMem) {
|
|
udata_close(fUDataMem);
|
|
} else if (!fDontFreeData) {
|
|
uprv_free((void *)fHeader);
|
|
}
|
|
}
|
|
|
|
|
|
|
|
//-----------------------------------------------------------------------------
|
|
//
|
|
// Operator == Consider two RBBIDataWrappers to be equal if they
|
|
// refer to the same underlying data. Although
|
|
// the data wrappers are normally shared between
|
|
// iterator instances, it's possible to independently
|
|
// open the same data twice, and get two instances, which
|
|
// should still be ==.
|
|
//
|
|
//-----------------------------------------------------------------------------
|
|
bool RBBIDataWrapper::operator ==(const RBBIDataWrapper &other) const {
|
|
if (fHeader == other.fHeader) {
|
|
return true;
|
|
}
|
|
if (fHeader->fLength != other.fHeader->fLength) {
|
|
return false;
|
|
}
|
|
if (uprv_memcmp(fHeader, other.fHeader, fHeader->fLength) == 0) {
|
|
return true;
|
|
}
|
|
return false;
|
|
}
|
|
|
|
int32_t RBBIDataWrapper::hashCode() {
|
|
return fHeader->fFTableLen;
|
|
}
|
|
|
|
|
|
|
|
//-----------------------------------------------------------------------------
|
|
//
|
|
// Reference Counting. A single RBBIDataWrapper object is shared among
|
|
// however many RulesBasedBreakIterator instances are
|
|
// referencing the same data.
|
|
//
|
|
//-----------------------------------------------------------------------------
|
|
void RBBIDataWrapper::removeReference() {
|
|
if (umtx_atomic_dec(&fRefCount) == 0) {
|
|
delete this;
|
|
}
|
|
}
|
|
|
|
|
|
RBBIDataWrapper *RBBIDataWrapper::addReference() {
|
|
umtx_atomic_inc(&fRefCount);
|
|
return this;
|
|
}
|
|
|
|
|
|
|
|
//-----------------------------------------------------------------------------
|
|
//
|
|
// getRuleSourceString
|
|
//
|
|
//-----------------------------------------------------------------------------
|
|
const UnicodeString &RBBIDataWrapper::getRuleSourceString() const {
|
|
return fRuleString;
|
|
}
|
|
|
|
|
|
//-----------------------------------------------------------------------------
|
|
//
|
|
// print - debugging function to dump the runtime data tables.
|
|
//
|
|
//-----------------------------------------------------------------------------
|
|
#ifdef RBBI_DEBUG
|
|
void RBBIDataWrapper::printTable(const char *heading, const RBBIStateTable *table) {
|
|
uint32_t c;
|
|
uint32_t s;
|
|
|
|
RBBIDebugPrintf("%s\n", heading);
|
|
|
|
RBBIDebugPrintf(" fDictCategoriesStart: %d\n", table->fDictCategoriesStart);
|
|
RBBIDebugPrintf(" fLookAheadResultsSize: %d\n", table->fLookAheadResultsSize);
|
|
RBBIDebugPrintf(" Flags: %4x RBBI_LOOKAHEAD_HARD_BREAK=%s RBBI_BOF_REQUIRED=%s RBBI_8BITS_ROWS=%s\n",
|
|
table->fFlags,
|
|
table->fFlags & RBBI_LOOKAHEAD_HARD_BREAK ? "T" : "F",
|
|
table->fFlags & RBBI_BOF_REQUIRED ? "T" : "F",
|
|
table->fFlags & RBBI_8BITS_ROWS ? "T" : "F");
|
|
RBBIDebugPrintf("\nState | Acc LA TagIx");
|
|
for (c=0; c<fHeader->fCatCount; c++) {RBBIDebugPrintf("%3d ", c);}
|
|
RBBIDebugPrintf("\n------|---------------"); for (c=0;c<fHeader->fCatCount; c++) {
|
|
RBBIDebugPrintf("----");
|
|
}
|
|
RBBIDebugPrintf("\n");
|
|
|
|
if (table == nullptr) {
|
|
RBBIDebugPrintf(" N U L L T A B L E\n\n");
|
|
return;
|
|
}
|
|
UBool use8Bits = table->fFlags & RBBI_8BITS_ROWS;
|
|
for (s=0; s<table->fNumStates; s++) {
|
|
RBBIStateTableRow *row = (RBBIStateTableRow *)
|
|
(table->fTableData + (table->fRowLen * s));
|
|
if (use8Bits) {
|
|
RBBIDebugPrintf("%4d | %3d %3d %3d ", s, row->r8.fAccepting, row->r8.fLookAhead, row->r8.fTagsIdx);
|
|
for (c=0; c<fHeader->fCatCount; c++) {
|
|
RBBIDebugPrintf("%3d ", row->r8.fNextState[c]);
|
|
}
|
|
} else {
|
|
RBBIDebugPrintf("%4d | %3d %3d %3d ", s, row->r16.fAccepting, row->r16.fLookAhead, row->r16.fTagsIdx);
|
|
for (c=0; c<fHeader->fCatCount; c++) {
|
|
RBBIDebugPrintf("%3d ", row->r16.fNextState[c]);
|
|
}
|
|
}
|
|
RBBIDebugPrintf("\n");
|
|
}
|
|
RBBIDebugPrintf("\n");
|
|
}
|
|
#endif
|
|
|
|
|
|
void RBBIDataWrapper::printData() {
|
|
#ifdef RBBI_DEBUG
|
|
RBBIDebugPrintf("RBBI Data at %p\n", (void *)fHeader);
|
|
RBBIDebugPrintf(" Version = {%d %d %d %d}\n", fHeader->fFormatVersion[0], fHeader->fFormatVersion[1],
|
|
fHeader->fFormatVersion[2], fHeader->fFormatVersion[3]);
|
|
RBBIDebugPrintf(" total length of data = %d\n", fHeader->fLength);
|
|
RBBIDebugPrintf(" number of character categories = %d\n\n", fHeader->fCatCount);
|
|
|
|
printTable("Forward State Transition Table", fForwardTable);
|
|
printTable("Reverse State Transition Table", fReverseTable);
|
|
|
|
RBBIDebugPrintf("\nOriginal Rules source:\n");
|
|
for (int32_t c=0; fRuleSource[c] != 0; c++) {
|
|
RBBIDebugPrintf("%c", fRuleSource[c]);
|
|
}
|
|
RBBIDebugPrintf("\n\n");
|
|
#endif
|
|
}
|
|
|
|
|
|
U_NAMESPACE_END
|
|
U_NAMESPACE_USE
|
|
|
|
//-----------------------------------------------------------------------------
|
|
//
|
|
// ubrk_swap - byte swap and char encoding swap of RBBI data
|
|
//
|
|
//-----------------------------------------------------------------------------
|
|
|
|
U_CAPI int32_t U_EXPORT2
|
|
ubrk_swap(const UDataSwapper *ds, const void *inData, int32_t length, void *outData,
|
|
UErrorCode *status) {
|
|
|
|
if (status == nullptr || U_FAILURE(*status)) {
|
|
return 0;
|
|
}
|
|
if(ds==nullptr || inData==nullptr || length<-1 || (length>0 && outData==nullptr)) {
|
|
*status=U_ILLEGAL_ARGUMENT_ERROR;
|
|
return 0;
|
|
}
|
|
|
|
//
|
|
// Check that the data header is for for break data.
|
|
// (Header contents are defined in genbrk.cpp)
|
|
//
|
|
const UDataInfo *pInfo = (const UDataInfo *)((const char *)inData+4);
|
|
if(!( pInfo->dataFormat[0]==0x42 && /* dataFormat="Brk " */
|
|
pInfo->dataFormat[1]==0x72 &&
|
|
pInfo->dataFormat[2]==0x6b &&
|
|
pInfo->dataFormat[3]==0x20 &&
|
|
RBBIDataWrapper::isDataVersionAcceptable(pInfo->formatVersion) )) {
|
|
udata_printError(ds, "ubrk_swap(): data format %02x.%02x.%02x.%02x (format version %02x) is not recognized\n",
|
|
pInfo->dataFormat[0], pInfo->dataFormat[1],
|
|
pInfo->dataFormat[2], pInfo->dataFormat[3],
|
|
pInfo->formatVersion[0]);
|
|
*status=U_UNSUPPORTED_ERROR;
|
|
return 0;
|
|
}
|
|
|
|
//
|
|
// Swap the data header. (This is the generic ICU Data Header, not the RBBI Specific
|
|
// RBBIDataHeader). This swap also conveniently gets us
|
|
// the size of the ICU d.h., which lets us locate the start
|
|
// of the RBBI specific data.
|
|
//
|
|
int32_t headerSize=udata_swapDataHeader(ds, inData, length, outData, status);
|
|
|
|
|
|
//
|
|
// Get the RRBI Data Header, and check that it appears to be OK.
|
|
//
|
|
const uint8_t *inBytes =(const uint8_t *)inData+headerSize;
|
|
RBBIDataHeader *rbbiDH = (RBBIDataHeader *)inBytes;
|
|
if (ds->readUInt32(rbbiDH->fMagic) != 0xb1a0 ||
|
|
!RBBIDataWrapper::isDataVersionAcceptable(rbbiDH->fFormatVersion) ||
|
|
ds->readUInt32(rbbiDH->fLength) < sizeof(RBBIDataHeader)) {
|
|
udata_printError(ds, "ubrk_swap(): RBBI Data header is invalid.\n");
|
|
*status=U_UNSUPPORTED_ERROR;
|
|
return 0;
|
|
}
|
|
|
|
//
|
|
// Prefight operation? Just return the size
|
|
//
|
|
int32_t breakDataLength = ds->readUInt32(rbbiDH->fLength);
|
|
int32_t totalSize = headerSize + breakDataLength;
|
|
if (length < 0) {
|
|
return totalSize;
|
|
}
|
|
|
|
//
|
|
// Check that length passed in is consistent with length from RBBI data header.
|
|
//
|
|
if (length < totalSize) {
|
|
udata_printError(ds, "ubrk_swap(): too few bytes (%d after ICU Data header) for break data.\n",
|
|
breakDataLength);
|
|
*status=U_INDEX_OUTOFBOUNDS_ERROR;
|
|
return 0;
|
|
}
|
|
|
|
|
|
//
|
|
// Swap the Data. Do the data itself first, then the RBBI Data Header, because
|
|
// we need to reference the header to locate the data, and an
|
|
// inplace swap of the header leaves it unusable.
|
|
//
|
|
uint8_t *outBytes = (uint8_t *)outData + headerSize;
|
|
RBBIDataHeader *outputDH = (RBBIDataHeader *)outBytes;
|
|
|
|
int32_t tableStartOffset;
|
|
int32_t tableLength;
|
|
|
|
//
|
|
// If not swapping in place, zero out the output buffer before starting.
|
|
// Individual tables and other data items within are aligned to 8 byte boundaries
|
|
// when originally created. Any unused space between items needs to be zero.
|
|
//
|
|
if (inBytes != outBytes) {
|
|
uprv_memset(outBytes, 0, breakDataLength);
|
|
}
|
|
|
|
//
|
|
// Each state table begins with several 32 bit fields. Calculate the size
|
|
// in bytes of these.
|
|
//
|
|
int32_t topSize = offsetof(RBBIStateTable, fTableData);
|
|
|
|
// Forward state table.
|
|
tableStartOffset = ds->readUInt32(rbbiDH->fFTable);
|
|
tableLength = ds->readUInt32(rbbiDH->fFTableLen);
|
|
|
|
if (tableLength > 0) {
|
|
RBBIStateTable *rbbiST = (RBBIStateTable *)(inBytes+tableStartOffset);
|
|
UBool use8Bits = ds->readUInt32(rbbiST->fFlags) & RBBI_8BITS_ROWS;
|
|
|
|
ds->swapArray32(ds, inBytes+tableStartOffset, topSize,
|
|
outBytes+tableStartOffset, status);
|
|
|
|
// Swap the state table if the table is in 16 bits.
|
|
if (use8Bits) {
|
|
if (outBytes != inBytes) {
|
|
uprv_memmove(outBytes+tableStartOffset+topSize,
|
|
inBytes+tableStartOffset+topSize,
|
|
tableLength-topSize);
|
|
}
|
|
} else {
|
|
ds->swapArray16(ds, inBytes+tableStartOffset+topSize, tableLength-topSize,
|
|
outBytes+tableStartOffset+topSize, status);
|
|
}
|
|
}
|
|
|
|
// Reverse state table. Same layout as forward table, above.
|
|
tableStartOffset = ds->readUInt32(rbbiDH->fRTable);
|
|
tableLength = ds->readUInt32(rbbiDH->fRTableLen);
|
|
|
|
if (tableLength > 0) {
|
|
RBBIStateTable *rbbiST = (RBBIStateTable *)(inBytes+tableStartOffset);
|
|
UBool use8Bits = ds->readUInt32(rbbiST->fFlags) & RBBI_8BITS_ROWS;
|
|
|
|
ds->swapArray32(ds, inBytes+tableStartOffset, topSize,
|
|
outBytes+tableStartOffset, status);
|
|
|
|
// Swap the state table if the table is in 16 bits.
|
|
if (use8Bits) {
|
|
if (outBytes != inBytes) {
|
|
uprv_memmove(outBytes+tableStartOffset+topSize,
|
|
inBytes+tableStartOffset+topSize,
|
|
tableLength-topSize);
|
|
}
|
|
} else {
|
|
ds->swapArray16(ds, inBytes+tableStartOffset+topSize, tableLength-topSize,
|
|
outBytes+tableStartOffset+topSize, status);
|
|
}
|
|
}
|
|
|
|
// Trie table for character categories
|
|
ucptrie_swap(ds, inBytes+ds->readUInt32(rbbiDH->fTrie), ds->readUInt32(rbbiDH->fTrieLen),
|
|
outBytes+ds->readUInt32(rbbiDH->fTrie), status);
|
|
|
|
// Source Rules Text. It's UTF8 data
|
|
if (outBytes != inBytes) {
|
|
uprv_memmove(outBytes+ds->readUInt32(rbbiDH->fRuleSource),
|
|
inBytes+ds->readUInt32(rbbiDH->fRuleSource),
|
|
ds->readUInt32(rbbiDH->fRuleSourceLen));
|
|
}
|
|
|
|
// Table of rule status values. It's all int_32 values
|
|
ds->swapArray32(ds, inBytes+ds->readUInt32(rbbiDH->fStatusTable), ds->readUInt32(rbbiDH->fStatusTableLen),
|
|
outBytes+ds->readUInt32(rbbiDH->fStatusTable), status);
|
|
|
|
// And, last, the header.
|
|
// It is all int32_t values except for fFormataVersion, which is an array of four bytes.
|
|
// Swap the whole thing as int32_t, then re-swap the one field.
|
|
//
|
|
ds->swapArray32(ds, inBytes, sizeof(RBBIDataHeader), outBytes, status);
|
|
ds->swapArray32(ds, outputDH->fFormatVersion, 4, outputDH->fFormatVersion, status);
|
|
|
|
return totalSize;
|
|
}
|
|
|
|
|
|
#endif /* #if !UCONFIG_NO_BREAK_ITERATION */
|