// © 2016 and later: Unicode, Inc. and others. // License & terms of use: http://www.unicode.org/copyright.html /* *************************************************************************** * Copyright (C) 1999-2014 International Business Machines Corporation * * and others. All rights reserved. * *************************************************************************** */ #include "unicode/utypes.h" #if !UCONFIG_NO_BREAK_ITERATION #include "unicode/ucptrie.h" #include "unicode/utypes.h" #include "rbbidata.h" #include "rbbirb.h" #include "udatamem.h" #include "cmemory.h" #include "cstring.h" #include "umutex.h" #include "uassert.h" U_NAMESPACE_BEGIN //----------------------------------------------------------------------------- // // Constructors. // //----------------------------------------------------------------------------- RBBIDataWrapper::RBBIDataWrapper(const RBBIDataHeader *data, UErrorCode &status) { init0(); init(data, status); } RBBIDataWrapper::RBBIDataWrapper(const RBBIDataHeader *data, enum EDontAdopt, UErrorCode &status) { init0(); init(data, status); fDontFreeData = TRUE; } RBBIDataWrapper::RBBIDataWrapper(UDataMemory* udm, UErrorCode &status) { init0(); if (U_FAILURE(status)) { return; } const DataHeader *dh = udm->pHeader; int32_t headerSize = dh->dataHeader.headerSize; if ( !(headerSize >= 20 && dh->info.isBigEndian == U_IS_BIG_ENDIAN && dh->info.charsetFamily == U_CHARSET_FAMILY && dh->info.dataFormat[0] == 0x42 && // dataFormat="Brk " dh->info.dataFormat[1] == 0x72 && dh->info.dataFormat[2] == 0x6b && dh->info.dataFormat[3] == 0x20 && isDataVersionAcceptable(dh->info.formatVersion)) ) { status = U_INVALID_FORMAT_ERROR; return; } const char *dataAsBytes = reinterpret_cast<const char *>(dh); const RBBIDataHeader *rbbidh = reinterpret_cast<const RBBIDataHeader *>(dataAsBytes + headerSize); init(rbbidh, status); fUDataMem = udm; } UBool RBBIDataWrapper::isDataVersionAcceptable(const UVersionInfo version) { return RBBI_DATA_FORMAT_VERSION[0] == version[0]; } //----------------------------------------------------------------------------- // // init(). Does most of the work of construction, shared between the // constructors. // //----------------------------------------------------------------------------- void RBBIDataWrapper::init0() { fHeader = NULL; fForwardTable = NULL; fReverseTable = NULL; fRuleSource = NULL; fRuleStatusTable = NULL; fTrie = NULL; fUDataMem = NULL; fRefCount = 0; fDontFreeData = TRUE; } void RBBIDataWrapper::init(const RBBIDataHeader *data, UErrorCode &status) { if (U_FAILURE(status)) { return; } fHeader = data; if (fHeader->fMagic != 0xb1a0 || !isDataVersionAcceptable(fHeader->fFormatVersion)) { status = U_INVALID_FORMAT_ERROR; return; } // Note: in ICU version 3.2 and earlier, there was a formatVersion 1 // that is no longer supported. At that time fFormatVersion was // an int32_t field, rather than an array of 4 bytes. fDontFreeData = FALSE; if (data->fFTableLen != 0) { fForwardTable = (RBBIStateTable *)((char *)data + fHeader->fFTable); } if (data->fRTableLen != 0) { fReverseTable = (RBBIStateTable *)((char *)data + fHeader->fRTable); } fTrie = ucptrie_openFromBinary(UCPTRIE_TYPE_FAST, UCPTRIE_VALUE_BITS_ANY, (uint8_t *)data + fHeader->fTrie, fHeader->fTrieLen, nullptr, // *actual length &status); if (U_FAILURE(status)) { return; } UCPTrieValueWidth width = ucptrie_getValueWidth(fTrie); if (!(width == UCPTRIE_VALUE_BITS_8 || width == UCPTRIE_VALUE_BITS_16)) { status = U_INVALID_FORMAT_ERROR; return; } fRuleSource = ((char *)data + fHeader->fRuleSource); fRuleString = UnicodeString::fromUTF8(StringPiece(fRuleSource, fHeader->fRuleSourceLen)); U_ASSERT(data->fRuleSourceLen > 0); fRuleStatusTable = (int32_t *)((char *)data + fHeader->fStatusTable); fStatusMaxIdx = data->fStatusTableLen / sizeof(int32_t); fRefCount = 1; #ifdef RBBI_DEBUG char *debugEnv = getenv("U_RBBIDEBUG"); if (debugEnv && uprv_strstr(debugEnv, "data")) {this->printData();} #endif } //----------------------------------------------------------------------------- // // Destructor. Don't call this - use removeReference() instead. // //----------------------------------------------------------------------------- RBBIDataWrapper::~RBBIDataWrapper() { U_ASSERT(fRefCount == 0); ucptrie_close(fTrie); fTrie = nullptr; if (fUDataMem) { udata_close(fUDataMem); } else if (!fDontFreeData) { uprv_free((void *)fHeader); } } //----------------------------------------------------------------------------- // // Operator == Consider two RBBIDataWrappers to be equal if they // refer to the same underlying data. Although // the data wrappers are normally shared between // iterator instances, it's possible to independently // open the same data twice, and get two instances, which // should still be ==. // //----------------------------------------------------------------------------- UBool RBBIDataWrapper::operator ==(const RBBIDataWrapper &other) const { if (fHeader == other.fHeader) { return TRUE; } if (fHeader->fLength != other.fHeader->fLength) { return FALSE; } if (uprv_memcmp(fHeader, other.fHeader, fHeader->fLength) == 0) { return TRUE; } return FALSE; } int32_t RBBIDataWrapper::hashCode() { return fHeader->fFTableLen; } //----------------------------------------------------------------------------- // // Reference Counting. A single RBBIDataWrapper object is shared among // however many RulesBasedBreakIterator instances are // referencing the same data. // //----------------------------------------------------------------------------- void RBBIDataWrapper::removeReference() { if (umtx_atomic_dec(&fRefCount) == 0) { delete this; } } RBBIDataWrapper *RBBIDataWrapper::addReference() { umtx_atomic_inc(&fRefCount); return this; } //----------------------------------------------------------------------------- // // getRuleSourceString // //----------------------------------------------------------------------------- const UnicodeString &RBBIDataWrapper::getRuleSourceString() const { return fRuleString; } //----------------------------------------------------------------------------- // // print - debugging function to dump the runtime data tables. // //----------------------------------------------------------------------------- #ifdef RBBI_DEBUG void RBBIDataWrapper::printTable(const char *heading, const RBBIStateTable *table) { uint32_t c; uint32_t s; RBBIDebugPrintf("%s\n", heading); RBBIDebugPrintf(" fDictCategoriesStart: %d\n", table->fDictCategoriesStart); RBBIDebugPrintf(" fLookAheadResultsSize: %d\n", table->fLookAheadResultsSize); RBBIDebugPrintf(" Flags: %4x RBBI_LOOKAHEAD_HARD_BREAK=%s RBBI_BOF_REQUIRED=%s RBBI_8BITS_ROWS=%s\n", table->fFlags, table->fFlags & RBBI_LOOKAHEAD_HARD_BREAK ? "T" : "F", table->fFlags & RBBI_BOF_REQUIRED ? "T" : "F", table->fFlags & RBBI_8BITS_ROWS ? "T" : "F"); RBBIDebugPrintf("\nState | Acc LA TagIx"); for (c=0; c<fHeader->fCatCount; c++) {RBBIDebugPrintf("%3d ", c);} RBBIDebugPrintf("\n------|---------------"); for (c=0;c<fHeader->fCatCount; c++) { RBBIDebugPrintf("----"); } RBBIDebugPrintf("\n"); if (table == NULL) { RBBIDebugPrintf(" N U L L T A B L E\n\n"); return; } UBool use8Bits = table->fFlags & RBBI_8BITS_ROWS; for (s=0; s<table->fNumStates; s++) { RBBIStateTableRow *row = (RBBIStateTableRow *) (table->fTableData + (table->fRowLen * s)); if (use8Bits) { RBBIDebugPrintf("%4d | %3d %3d %3d ", s, row->r8.fAccepting, row->r8.fLookAhead, row->r8.fTagsIdx); for (c=0; c<fHeader->fCatCount; c++) { RBBIDebugPrintf("%3d ", row->r8.fNextState[c]); } } else { RBBIDebugPrintf("%4d | %3d %3d %3d ", s, row->r16.fAccepting, row->r16.fLookAhead, row->r16.fTagsIdx); for (c=0; c<fHeader->fCatCount; c++) { RBBIDebugPrintf("%3d ", row->r16.fNextState[c]); } } RBBIDebugPrintf("\n"); } RBBIDebugPrintf("\n"); } #endif void RBBIDataWrapper::printData() { #ifdef RBBI_DEBUG RBBIDebugPrintf("RBBI Data at %p\n", (void *)fHeader); RBBIDebugPrintf(" Version = {%d %d %d %d}\n", fHeader->fFormatVersion[0], fHeader->fFormatVersion[1], fHeader->fFormatVersion[2], fHeader->fFormatVersion[3]); RBBIDebugPrintf(" total length of data = %d\n", fHeader->fLength); RBBIDebugPrintf(" number of character categories = %d\n\n", fHeader->fCatCount); printTable("Forward State Transition Table", fForwardTable); printTable("Reverse State Transition Table", fReverseTable); RBBIDebugPrintf("\nOrignal Rules source:\n"); for (int32_t c=0; fRuleSource[c] != 0; c++) { RBBIDebugPrintf("%c", fRuleSource[c]); } RBBIDebugPrintf("\n\n"); #endif } U_NAMESPACE_END U_NAMESPACE_USE //----------------------------------------------------------------------------- // // ubrk_swap - byte swap and char encoding swap of RBBI data // //----------------------------------------------------------------------------- U_CAPI int32_t U_EXPORT2 ubrk_swap(const UDataSwapper *ds, const void *inData, int32_t length, void *outData, UErrorCode *status) { if (status == NULL || U_FAILURE(*status)) { return 0; } if(ds==NULL || inData==NULL || length<-1 || (length>0 && outData==NULL)) { *status=U_ILLEGAL_ARGUMENT_ERROR; return 0; } // // Check that the data header is for for break data. // (Header contents are defined in genbrk.cpp) // const UDataInfo *pInfo = (const UDataInfo *)((const char *)inData+4); if(!( pInfo->dataFormat[0]==0x42 && /* dataFormat="Brk " */ pInfo->dataFormat[1]==0x72 && pInfo->dataFormat[2]==0x6b && pInfo->dataFormat[3]==0x20 && RBBIDataWrapper::isDataVersionAcceptable(pInfo->formatVersion) )) { udata_printError(ds, "ubrk_swap(): data format %02x.%02x.%02x.%02x (format version %02x) is not recognized\n", pInfo->dataFormat[0], pInfo->dataFormat[1], pInfo->dataFormat[2], pInfo->dataFormat[3], pInfo->formatVersion[0]); *status=U_UNSUPPORTED_ERROR; return 0; } // // Swap the data header. (This is the generic ICU Data Header, not the RBBI Specific // RBBIDataHeader). This swap also conveniently gets us // the size of the ICU d.h., which lets us locate the start // of the RBBI specific data. // int32_t headerSize=udata_swapDataHeader(ds, inData, length, outData, status); // // Get the RRBI Data Header, and check that it appears to be OK. // const uint8_t *inBytes =(const uint8_t *)inData+headerSize; RBBIDataHeader *rbbiDH = (RBBIDataHeader *)inBytes; if (ds->readUInt32(rbbiDH->fMagic) != 0xb1a0 || !RBBIDataWrapper::isDataVersionAcceptable(rbbiDH->fFormatVersion) || ds->readUInt32(rbbiDH->fLength) < sizeof(RBBIDataHeader)) { udata_printError(ds, "ubrk_swap(): RBBI Data header is invalid.\n"); *status=U_UNSUPPORTED_ERROR; return 0; } // // Prefight operation? Just return the size // int32_t breakDataLength = ds->readUInt32(rbbiDH->fLength); int32_t totalSize = headerSize + breakDataLength; if (length < 0) { return totalSize; } // // Check that length passed in is consistent with length from RBBI data header. // if (length < totalSize) { udata_printError(ds, "ubrk_swap(): too few bytes (%d after ICU Data header) for break data.\n", breakDataLength); *status=U_INDEX_OUTOFBOUNDS_ERROR; return 0; } // // Swap the Data. Do the data itself first, then the RBBI Data Header, because // we need to reference the header to locate the data, and an // inplace swap of the header leaves it unusable. // uint8_t *outBytes = (uint8_t *)outData + headerSize; RBBIDataHeader *outputDH = (RBBIDataHeader *)outBytes; int32_t tableStartOffset; int32_t tableLength; // // If not swapping in place, zero out the output buffer before starting. // Individual tables and other data items within are aligned to 8 byte boundaries // when originally created. Any unused space between items needs to be zero. // if (inBytes != outBytes) { uprv_memset(outBytes, 0, breakDataLength); } // // Each state table begins with several 32 bit fields. Calculate the size // in bytes of these. // int32_t topSize = offsetof(RBBIStateTable, fTableData); // Forward state table. tableStartOffset = ds->readUInt32(rbbiDH->fFTable); tableLength = ds->readUInt32(rbbiDH->fFTableLen); if (tableLength > 0) { RBBIStateTable *rbbiST = (RBBIStateTable *)(inBytes+tableStartOffset); UBool use8Bits = ds->readUInt32(rbbiST->fFlags) & RBBI_8BITS_ROWS; ds->swapArray32(ds, inBytes+tableStartOffset, topSize, outBytes+tableStartOffset, status); // Swap the state table if the table is in 16 bits. if (use8Bits) { if (outBytes != inBytes) { uprv_memmove(outBytes+tableStartOffset+topSize, inBytes+tableStartOffset+topSize, tableLength-topSize); } } else { ds->swapArray16(ds, inBytes+tableStartOffset+topSize, tableLength-topSize, outBytes+tableStartOffset+topSize, status); } } // Reverse state table. Same layout as forward table, above. tableStartOffset = ds->readUInt32(rbbiDH->fRTable); tableLength = ds->readUInt32(rbbiDH->fRTableLen); if (tableLength > 0) { RBBIStateTable *rbbiST = (RBBIStateTable *)(inBytes+tableStartOffset); UBool use8Bits = ds->readUInt32(rbbiST->fFlags) & RBBI_8BITS_ROWS; ds->swapArray32(ds, inBytes+tableStartOffset, topSize, outBytes+tableStartOffset, status); // Swap the state table if the table is in 16 bits. if (use8Bits) { if (outBytes != inBytes) { uprv_memmove(outBytes+tableStartOffset+topSize, inBytes+tableStartOffset+topSize, tableLength-topSize); } } else { ds->swapArray16(ds, inBytes+tableStartOffset+topSize, tableLength-topSize, outBytes+tableStartOffset+topSize, status); } } // Trie table for character categories ucptrie_swap(ds, inBytes+ds->readUInt32(rbbiDH->fTrie), ds->readUInt32(rbbiDH->fTrieLen), outBytes+ds->readUInt32(rbbiDH->fTrie), status); // Source Rules Text. It's UTF8 data if (outBytes != inBytes) { uprv_memmove(outBytes+ds->readUInt32(rbbiDH->fRuleSource), inBytes+ds->readUInt32(rbbiDH->fRuleSource), ds->readUInt32(rbbiDH->fRuleSourceLen)); } // Table of rule status values. It's all int_32 values ds->swapArray32(ds, inBytes+ds->readUInt32(rbbiDH->fStatusTable), ds->readUInt32(rbbiDH->fStatusTableLen), outBytes+ds->readUInt32(rbbiDH->fStatusTable), status); // And, last, the header. // It is all int32_t values except for fFormataVersion, which is an array of four bytes. // Swap the whole thing as int32_t, then re-swap the one field. // ds->swapArray32(ds, inBytes, sizeof(RBBIDataHeader), outBytes, status); ds->swapArray32(ds, outputDH->fFormatVersion, 4, outputDH->fFormatVersion, status); return totalSize; } #endif /* #if !UCONFIG_NO_BREAK_ITERATION */