LeenkxTeam/Kmake
1
0
Fork 1
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Upload Kmake

Browse Source
This commit is contained in:
Gorochu
2026-05-26 23:36:42 -07:00
parent ba051b2f74
commit 555ec72358
41615 changed files with 13344630 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

130
deps/icu-small/source/i18n/BUILD.bazel vendored Normal file
View File

@ -0,0 +1,130 @@
# © 2021 and later: Unicode, Inc. and others.
# License & terms of use: http://www.unicode.org/copyright.html
# This file defines Bazel targets for a subset of the ICU4C "i18n" library header and source files.
# The configuration of dependencies among targets is strongly assisted by the
# file in depstest that maintains such information, at
# icu4c/source/test/depstest/dependencies.txt .
load("@rules_cc//cc:defs.bzl", "cc_binary", "cc_library")
package(
default_visibility = ["//visibility:public"],
)
# When compiling code in the `common` dir, the constant
# `U_I18n_IMPLEMENTATION` needs to be defined. See
# https://unicode-org.github.io/icu/userguide/howtouseicu#c-with-your-own-build-system .
# If linker errors occur, then this may be a sign that the dependencies were
# not specified correctly. Use dependencies.txt in depstest for assistance. See
# https://stackoverflow.com/q/66111709/2077918 .
cc_library(
name = "headers",
hdrs = glob([
"unicode/*.h", # public
"*.h", # internal
]),
# We need to add includes in order to preserve existing source files'
# include directives that use traditional paths, not paths relative to
# Bazel workspace:
# https://stackoverflow.com/a/65635893/2077918
includes = ["."],
local_defines = [
"U_I18N_IMPLEMENTATION",
],
)
cc_library(
name = "collation",
srcs = [
"bocsu.cpp",
"coleitr.cpp",
"coll.cpp",
"collation.cpp",
"collationcompare.cpp",
"collationdata.cpp",
"collationdatareader.cpp",
"collationdatawriter.cpp",
"collationfastlatin.cpp",
# collationfcd.cpp is generated by genuca;
# probably hard to build genuca without depending on the old version.
"collationfcd.cpp",
"collationiterator.cpp",
"collationkeys.cpp",
"collationroot.cpp",
"collationrootelements.cpp",
"collationsets.cpp",
"collationsettings.cpp",
"collationtailoring.cpp",
"rulebasedcollator.cpp",
"sortkey.cpp",
"ucol.cpp",
"ucol_res.cpp",
"ucol_sit.cpp",
"ucoleitr.cpp",
"uitercollationiterator.cpp",
"utf16collationiterator.cpp",
"utf8collationiterator.cpp",
],
includes = ["."],
deps = [
":headers",
":uclean_i18n",
"//icu4c/source/common:bytestream",
"//icu4c/source/common:normalizer2",
"//icu4c/source/common:platform",
"//icu4c/source/common:propname",
"//icu4c/source/common:resourcebundle",
"//icu4c/source/common:service_registration",
"//icu4c/source/common:ucharstrieiterator",
"//icu4c/source/common:uiter",
"//icu4c/source/common:ulist",
"//icu4c/source/common:unifiedcache",
"//icu4c/source/common:uset",
"//icu4c/source/common:usetiter",
"//icu4c/source/common:utrie2",
"//icu4c/source/common:uvector32",
"//icu4c/source/common:uvector64",
],
local_defines = [
"U_I18N_IMPLEMENTATION",
],
)
cc_library(
name = "collation_builder",
srcs = [
"collationbuilder.cpp",
"collationdatabuilder.cpp",
"collationfastlatinbuilder.cpp",
"collationruleparser.cpp",
"collationweights.cpp",
],
includes = ["."],
deps = [
":collation",
"//icu4c/source/common:canonical_iterator",
"//icu4c/source/common:ucharstriebuilder",
"//icu4c/source/common:uset_props"
],
local_defines = [
"U_I18N_IMPLEMENTATION",
],
)
cc_library(
name = "uclean_i18n",
srcs = [
"ucln_in.cpp",
],
hdrs = ["ucln_in.h"],
includes = ["."],
deps = [
"//icu4c/source/common:platform",
],
local_defines = [
"U_I18N_IMPLEMENTATION",
],
)

1235
deps/icu-small/source/i18n/alphaindex.cpp vendored Normal file
View File

File diff suppressed because it is too large Load Diff

411
deps/icu-small/source/i18n/anytrans.cpp vendored Normal file
View File

@ -0,0 +1,411 @@
// © 2016 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html
/*
*****************************************************************
* Copyright (c) 2002-2014, International Business Machines Corporation
* and others. All Rights Reserved.
*****************************************************************
* Date Name Description
* 06/06/2002 aliu Creation.
*****************************************************************
*/
#include "unicode/utypes.h"
#if !UCONFIG_NO_TRANSLITERATION
#include "unicode/uobject.h"
#include "unicode/uscript.h"
#include "anytrans.h"
#include "hash.h"
#include "mutex.h"
#include "nultrans.h"
#include "putilimp.h"
#include "tridpars.h"
#include "uinvchar.h"
#include "uvector.h"
//------------------------------------------------------------
// Constants
static const char16_t TARGET_SEP = 45; // '-'
static const char16_t VARIANT_SEP = 47; // '/'
static const char16_t ANY[] = {0x41,0x6E,0x79,0}; // "Any"
static const char16_t NULL_ID[] = {78,117,108,108,0}; // "Null"
static const char16_t LATIN_PIVOT[] = {0x2D,0x4C,0x61,0x74,0x6E,0x3B,0x4C,0x61,0x74,0x6E,0x2D,0}; // "-Latn;Latn-"
// initial size for an Any-XXXX transform's cache of script-XXXX transforms
// (will grow as necessary, but we don't expect to have source text with more than 7 scripts)
#define ANY_TRANS_CACHE_INIT_SIZE 7
//------------------------------------------------------------
U_CDECL_BEGIN
/**
* Deleter function for Transliterator*.
*/
static void U_CALLCONV
_deleteTransliterator(void *obj) {
delete (icu::Transliterator*) obj;
}
U_CDECL_END
//------------------------------------------------------------
U_NAMESPACE_BEGIN
//------------------------------------------------------------
// ScriptRunIterator
/**
* Returns a series of ranges corresponding to scripts. They will be
* of the form:
*
* ccccSScSSccccTTcTcccc - c = common, S = first script, T = second
* | | - first run (start, limit)
* | | - second run (start, limit)
*
* That is, the runs will overlap. The reason for this is so that a
* transliterator can consider common characters both before and after
* the scripts.
*/
class ScriptRunIterator : public UMemory {
private:
const Replaceable& text;
int32_t textStart;
int32_t textLimit;
public:
/**
* The code of the current run, valid after next() returns. May
* be USCRIPT_INVALID_CODE if and only if the entire text is
* COMMON/INHERITED.
*/
UScriptCode scriptCode;
/**
* The start of the run, inclusive, valid after next() returns.
*/
int32_t start;
/**
* The end of the run, exclusive, valid after next() returns.
*/
int32_t limit;
/**
* Constructs a run iterator over the given text from start
* (inclusive) to limit (exclusive).
*/
ScriptRunIterator(const Replaceable& text, int32_t start, int32_t limit);
/**
* Returns true if there are any more runs. true is always
* returned at least once. Upon return, the caller should
* examine scriptCode, start, and limit.
*/
UBool next();
/**
* Adjusts internal indices for a change in the limit index of the
* given delta. A positive delta means the limit has increased.
*/
void adjustLimit(int32_t delta);
private:
ScriptRunIterator(const ScriptRunIterator &other); // forbid copying of this class
ScriptRunIterator &operator=(const ScriptRunIterator &other); // forbid copying of this class
};
ScriptRunIterator::ScriptRunIterator(const Replaceable& theText,
int32_t myStart, int32_t myLimit) :
text(theText)
{
textStart = myStart;
textLimit = myLimit;
limit = myStart;
}
UBool ScriptRunIterator::next() {
UChar32 ch;
UScriptCode s;
UErrorCode ec = U_ZERO_ERROR;
scriptCode = USCRIPT_INVALID_CODE; // don't know script yet
start = limit;
// Are we done?
if (start == textLimit) {
return false;
}
// Move start back to include adjacent COMMON or INHERITED
// characters
while (start > textStart) {
ch = text.char32At(start - 1); // look back
s = uscript_getScript(ch, &ec);
if (s == USCRIPT_COMMON || s == USCRIPT_INHERITED) {
--start;
} else {
break;
}
}
// Move limit ahead to include COMMON, INHERITED, and characters
// of the current script.
while (limit < textLimit) {
ch = text.char32At(limit); // look ahead
s = uscript_getScript(ch, &ec);
if (s != USCRIPT_COMMON && s != USCRIPT_INHERITED) {
if (scriptCode == USCRIPT_INVALID_CODE) {
scriptCode = s;
} else if (s != scriptCode) {
break;
}
}
++limit;
}
// Return true even if the entire text is COMMON / INHERITED, in
// which case scriptCode will be USCRIPT_INVALID_CODE.
return true;
}
void ScriptRunIterator::adjustLimit(int32_t delta) {
limit += delta;
textLimit += delta;
}
//------------------------------------------------------------
// AnyTransliterator
UOBJECT_DEFINE_RTTI_IMPLEMENTATION(AnyTransliterator)
AnyTransliterator::AnyTransliterator(const UnicodeString& id,
const UnicodeString& theTarget,
const UnicodeString& theVariant,
UScriptCode theTargetScript,
UErrorCode& ec) :
Transliterator(id, nullptr),
targetScript(theTargetScript)
{
cache = uhash_openSize(uhash_hashLong, uhash_compareLong, nullptr, ANY_TRANS_CACHE_INIT_SIZE, &ec);
if (U_FAILURE(ec)) {
return;
}
uhash_setValueDeleter(cache, _deleteTransliterator);
target = theTarget;
if (theVariant.length() > 0) {
target.append(VARIANT_SEP).append(theVariant);
}
}
AnyTransliterator::~AnyTransliterator() {
uhash_close(cache);
}
/**
* Copy constructor.
*/
AnyTransliterator::AnyTransliterator(const AnyTransliterator& o) :
Transliterator(o),
target(o.target),
targetScript(o.targetScript)
{
// Don't copy the cache contents
UErrorCode ec = U_ZERO_ERROR;
cache = uhash_openSize(uhash_hashLong, uhash_compareLong, nullptr, ANY_TRANS_CACHE_INIT_SIZE, &ec);
if (U_FAILURE(ec)) {
return;
}
uhash_setValueDeleter(cache, _deleteTransliterator);
}
/**
* Transliterator API.
*/
AnyTransliterator* AnyTransliterator::clone() const {
return new AnyTransliterator(*this);
}
/**
* Implements {@link Transliterator#handleTransliterate}.
*/
void AnyTransliterator::handleTransliterate(Replaceable& text, UTransPosition& pos,
UBool isIncremental) const {
int32_t allStart = pos.start;
int32_t allLimit = pos.limit;
ScriptRunIterator it(text, pos.contextStart, pos.contextLimit);
while (it.next()) {
// Ignore runs in the ante context
if (it.limit <= allStart) continue;
// Try to instantiate transliterator from it.scriptCode to
// our target or target/variant
Transliterator* t = getTransliterator(it.scriptCode);
if (t == nullptr) {
// We have no transliterator. Do nothing, but keep
// pos.start up to date.
pos.start = it.limit;
continue;
}
// If the run end is before the transliteration limit, do
// a non-incremental transliteration. Otherwise do an
// incremental one.
UBool incremental = isIncremental && (it.limit >= allLimit);
pos.start = uprv_max(allStart, it.start);
pos.limit = uprv_min(allLimit, it.limit);
int32_t limit = pos.limit;
t->filteredTransliterate(text, pos, incremental);
int32_t delta = pos.limit - limit;
allLimit += delta;
it.adjustLimit(delta);
// We're done if we enter the post context
if (it.limit >= allLimit) break;
}
// Restore limit. pos.start is fine where the last transliterator
// left it, or at the end of the last run.
pos.limit = allLimit;
}
Transliterator* AnyTransliterator::getTransliterator(UScriptCode source) const {
if (source == targetScript || source == USCRIPT_INVALID_CODE) {
return nullptr;
}
Transliterator* t = nullptr;
{
Mutex m(nullptr);
t = static_cast<Transliterator*>(uhash_iget(cache, static_cast<int32_t>(source)));
}
if (t == nullptr) {
UErrorCode ec = U_ZERO_ERROR;
UnicodeString sourceName(uscript_getShortName(source), -1, US_INV);
UnicodeString id(sourceName);
id.append(TARGET_SEP).append(target);
t = Transliterator::createInstance(id, UTRANS_FORWARD, ec);
if (U_FAILURE(ec) || t == nullptr) {
delete t;
// Try to pivot around Latin, our most common script
id = sourceName;
id.append(LATIN_PIVOT, -1).append(target);
t = Transliterator::createInstance(id, UTRANS_FORWARD, ec);
if (U_FAILURE(ec) || t == nullptr) {
delete t;
t = nullptr;
}
}
if (t != nullptr) {
Transliterator *rt = nullptr;
{
Mutex m(nullptr);
rt = static_cast<Transliterator*>(uhash_iget(cache, static_cast<int32_t>(source)));
if (rt == nullptr) {
// Common case, no race to cache this new transliterator.
uhash_iput(cache, static_cast<int32_t>(source), t, &ec);
} else {
// Race case, some other thread beat us to caching this transliterator.
Transliterator *temp = rt;
rt = t; // Our newly created transliterator that lost the race & now needs deleting.
t = temp; // The transliterator from the cache that we will return.
}
}
delete rt; // will be non-null only in case of races.
}
}
return t;
}
/**
* Return the script code for a given name, or -1 if not found.
*/
static UScriptCode scriptNameToCode(const UnicodeString& name) {
char buf[128];
UScriptCode code;
UErrorCode ec = U_ZERO_ERROR;
int32_t nameLen = name.length();
UBool isInvariant = uprv_isInvariantUString(name.getBuffer(), nameLen);
if (isInvariant) {
name.extract(0, nameLen, buf, static_cast<int32_t>(sizeof(buf)), US_INV);
buf[127] = 0; // Make sure that we nullptr terminate the string.
}
if (!isInvariant || uscript_getCode(buf, &code, 1, &ec) != 1 || U_FAILURE(ec))
{
code = USCRIPT_INVALID_CODE;
}
return code;
}
/**
* Registers standard transliterators with the system. Called by
* Transliterator during initialization. Scan all current targets and
* register those that are scripts T as Any-T/V.
*/
void AnyTransliterator::registerIDs() {
UErrorCode ec = U_ZERO_ERROR;
Hashtable seen(true, ec);
int32_t sourceCount = Transliterator::_countAvailableSources();
for (int32_t s=0; s<sourceCount; ++s) {
UnicodeString source;
Transliterator::_getAvailableSource(s, source);
// Ignore the "Any" source
if (source.caseCompare(ANY, 3, 0 /*U_FOLD_CASE_DEFAULT*/) == 0) continue;
int32_t targetCount = Transliterator::_countAvailableTargets(source);
for (int32_t t=0; t<targetCount; ++t) {
UnicodeString target;
Transliterator::_getAvailableTarget(t, source, target);
// Only process each target once
if (seen.geti(target) != 0) continue;
ec = U_ZERO_ERROR;
seen.puti(target, 1, ec);
// Get the script code for the target. If not a script, ignore.
UScriptCode targetScript = scriptNameToCode(target);
if (targetScript == USCRIPT_INVALID_CODE) continue;
int32_t variantCount = Transliterator::_countAvailableVariants(source, target);
// assert(variantCount >= 1);
for (int32_t v=0; v<variantCount; ++v) {
UnicodeString variant;
Transliterator::_getAvailableVariant(v, source, target, variant);
UnicodeString id;
TransliteratorIDParser::STVtoID(UnicodeString(true, ANY, 3), target, variant, id);
ec = U_ZERO_ERROR;
AnyTransliterator* tl = new AnyTransliterator(id, target, variant,
targetScript, ec);
if (U_FAILURE(ec)) {
delete tl;
} else {
Transliterator::_registerInstance(tl);
Transliterator::_registerSpecialInverse(target, UnicodeString(true, NULL_ID, 4), false);
}
}
}
}
}
U_NAMESPACE_END
#endif /* #if !UCONFIG_NO_TRANSLITERATION */
//eof

131
deps/icu-small/source/i18n/anytrans.h vendored Normal file
View File

@ -0,0 +1,131 @@
// © 2016 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html
/*
***********************************************************************
* Copyright (c) 2002-2007, International Business Machines Corporation
* and others. All Rights Reserved.
***********************************************************************
* Date Name Description
* 06/06/2002 aliu Creation.
***********************************************************************
*/
#ifndef _ANYTRANS_H_
#define _ANYTRANS_H_
#include "unicode/utypes.h"
#if !UCONFIG_NO_TRANSLITERATION
#include "unicode/translit.h"
#include "unicode/uscript.h"
#include "uhash.h"
U_NAMESPACE_BEGIN
/**
* A transliterator named Any-T or Any-T/V, where T is the target
* script and V is the optional variant, that uses multiple
* transliterators, all going to T or T/V, all with script sources.
* The target must be a script. It partitions text into runs of the
* same script, and then based on the script of each run,
* transliterates from that script to the given target or
* target/variant. Adjacent COMMON or INHERITED script characters are
* included in each run.
*
* @author Alan Liu
*/
class AnyTransliterator : public Transliterator {
/**
* Cache mapping UScriptCode values to Transliterator*.
*/
UHashtable* cache;
/**
* The target or target/variant string.
*/
UnicodeString target;
/**
* The target script code. Never USCRIPT_INVALID_CODE.
*/
UScriptCode targetScript;
public:
/**
* Destructor.
*/
virtual ~AnyTransliterator();
/**
* Copy constructor.
*/
AnyTransliterator(const AnyTransliterator&);
/**
* Transliterator API.
*/
virtual AnyTransliterator* clone() const override;
/**
* Implements {@link Transliterator#handleTransliterate}.
*/
virtual void handleTransliterate(Replaceable& text, UTransPosition& index,
UBool incremental) const override;
/**
* ICU "poor man's RTTI", returns a UClassID for the actual class.
*/
virtual UClassID getDynamicClassID() const override;
/**
* ICU "poor man's RTTI", returns a UClassID for this class.
*/
U_I18N_API static UClassID U_EXPORT2 getStaticClassID();
private:
/**
* Private constructor
* @param id the ID of the form S-T or S-T/V, where T is theTarget
* and V is theVariant. Must not be empty.
* @param theTarget the target name. Must not be empty, and must
* name a script corresponding to theTargetScript.
* @param theVariant the variant name, or the empty string if
* there is no variant
* @param theTargetScript the script code corresponding to
* theTarget.
* @param ec error code, fails if the internal hashtable cannot be
* allocated
*/
AnyTransliterator(const UnicodeString& id,
const UnicodeString& theTarget,
const UnicodeString& theVariant,
UScriptCode theTargetScript,
UErrorCode& ec);
/**
* Returns a transliterator from the given source to our target or
* target/variant. Returns nullptr if the source is the same as our
* target script, or if the source is USCRIPT_INVALID_CODE.
* Caches the result and returns the same transliterator the next
* time. The caller does NOT own the result and must not delete
* it.
*/
Transliterator* getTransliterator(UScriptCode source) const;
/**
* Registers standard transliterators with the system. Called by
* Transliterator during initialization.
*/
static void registerIDs();
friend class Transliterator; // for registerIDs()
};
U_NAMESPACE_END
#endif /* #if !UCONFIG_NO_TRANSLITERATION */
#endif

900
deps/icu-small/source/i18n/astro.cpp vendored Normal file
View File

@ -0,0 +1,900 @@
// © 2016 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html
/************************************************************************
* Copyright (C) 1996-2012, International Business Machines Corporation
* and others. All Rights Reserved.
************************************************************************
* 2003-nov-07 srl Port from Java
*/
#include "astro.h"
#if !UCONFIG_NO_FORMATTING
#include "unicode/calendar.h"
#include <math.h>
#include <float.h>
#include "unicode/putil.h"
#include "uhash.h"
#include "umutex.h"
#include "ucln_in.h"
#include "putilimp.h"
#include <stdio.h> // for toString()
#if defined (PI)
#undef PI
#endif
#ifdef U_DEBUG_ASTRO
# include "uresimp.h" // for debugging
static void debug_astro_loc(const char *f, int32_t l)
{
fprintf(stderr, "%s:%d: ", f, l);
}
static void debug_astro_msg(const char *pat, ...)
{
va_list ap;
va_start(ap, pat);
vfprintf(stderr, pat, ap);
fflush(stderr);
}
#include "unicode/datefmt.h"
#include "unicode/ustring.h"
static const char * debug_astro_date(UDate d) {
static char gStrBuf[1024];
static DateFormat *df = nullptr;
if(df == nullptr) {
df = DateFormat::createDateTimeInstance(DateFormat::MEDIUM, DateFormat::MEDIUM, Locale::getUS());
df->adoptTimeZone(TimeZone::getGMT()->clone());
}
UnicodeString str;
df->format(d,str);
u_austrncpy(gStrBuf,str.getTerminatedBuffer(),sizeof(gStrBuf)-1);
return gStrBuf;
}
// must use double parens, i.e.: U_DEBUG_ASTRO_MSG(("four is: %d",4));
#define U_DEBUG_ASTRO_MSG(x) {debug_astro_loc(__FILE__,__LINE__);debug_astro_msg x;}
#else
#define U_DEBUG_ASTRO_MSG(x)
#endif
static inline UBool isINVALID(double d) {
return(uprv_isNaN(d));
}
static icu::UMutex ccLock;
U_CDECL_BEGIN
static UBool calendar_astro_cleanup() {
return true;
}
U_CDECL_END
U_NAMESPACE_BEGIN
/**
* The number of standard hours in one sidereal day.
* Approximately 24.93.
* @internal
* @deprecated ICU 2.4. This class may be removed or modified.
*/
#define SIDEREAL_DAY (23.93446960027)
/**
* The number of sidereal hours in one mean solar day.
* Approximately 24.07.
* @internal
* @deprecated ICU 2.4. This class may be removed or modified.
*/
#define SOLAR_DAY (24.065709816)
/**
* The average number of solar days from one new moon to the next. This is the time
* it takes for the moon to return the same ecliptic longitude as the sun.
* It is longer than the sidereal month because the sun's longitude increases
* during the year due to the revolution of the earth around the sun.
* Approximately 29.53.
*
* @see #SIDEREAL_MONTH
* @internal
* @deprecated ICU 2.4. This class may be removed or modified.
*/
const double CalendarAstronomer::SYNODIC_MONTH = 29.530588853;
/**
* The average number of days it takes
* for the moon to return to the same ecliptic longitude relative to the
* stellar background. This is referred to as the sidereal month.
* It is shorter than the synodic month due to
* the revolution of the earth around the sun.
* Approximately 27.32.
*
* @see #SYNODIC_MONTH
* @internal
* @deprecated ICU 2.4. This class may be removed or modified.
*/
#define SIDEREAL_MONTH 27.32166
/**
* The average number number of days between successive vernal equinoxes.
* Due to the precession of the earth's
* axis, this is not precisely the same as the sidereal year.
* Approximately 365.24
*
* @see #SIDEREAL_YEAR
* @internal
* @deprecated ICU 2.4. This class may be removed or modified.
*/
#define TROPICAL_YEAR 365.242191
/**
* The average number of days it takes
* for the sun to return to the same position against the fixed stellar
* background. This is the duration of one orbit of the earth about the sun
* as it would appear to an outside observer.
* Due to the precession of the earth's
* axis, this is not precisely the same as the tropical year.
* Approximately 365.25.
*
* @see #TROPICAL_YEAR
* @internal
* @deprecated ICU 2.4. This class may be removed or modified.
*/
#define SIDEREAL_YEAR 365.25636
//-------------------------------------------------------------------------
// Time-related constants
//-------------------------------------------------------------------------
/**
* The number of milliseconds in one second.
* @internal
* @deprecated ICU 2.4. This class may be removed or modified.
*/
#define SECOND_MS U_MILLIS_PER_SECOND
/**
* The number of milliseconds in one minute.
* @internal
* @deprecated ICU 2.4. This class may be removed or modified.
*/
#define MINUTE_MS U_MILLIS_PER_MINUTE
/**
* The number of milliseconds in one hour.
* @internal
* @deprecated ICU 2.4. This class may be removed or modified.
*/
#define HOUR_MS U_MILLIS_PER_HOUR
/**
* The number of milliseconds in one day.
* @internal
* @deprecated ICU 2.4. This class may be removed or modified.
*/
#define DAY_MS U_MILLIS_PER_DAY
/**
* The start of the julian day numbering scheme used by astronomers, which
* is 1/1/4713 BC (Julian), 12:00 GMT. This is given as the number of milliseconds
* since 1/1/1970 AD (Gregorian), a negative number.
* Note that julian day numbers and
* the Julian calendar are <em>not</em> the same thing. Also note that
* julian days start at <em>noon</em>, not midnight.
* @internal
* @deprecated ICU 2.4. This class may be removed or modified.
*/
#define JULIAN_EPOCH_MS -210866760000000.0
/**
* Milliseconds value for 0.0 January 2000 AD.
*/
#define EPOCH_2000_MS 946598400000.0
//-------------------------------------------------------------------------
// Assorted private data used for conversions
//-------------------------------------------------------------------------
// My own copies of these so compilers are more likely to optimize them away
const double CalendarAstronomer::PI = 3.14159265358979323846;
#define CalendarAstronomer_PI2 (CalendarAstronomer::PI*2.0)
#define RAD_HOUR ( 12 / CalendarAstronomer::PI ) // radians -> hours
#define DEG_RAD ( CalendarAstronomer::PI / 180 ) // degrees -> radians
#define RAD_DEG ( 180 / CalendarAstronomer::PI ) // radians -> degrees
/***
* Given 'value', add or subtract 'range' until 0 <= 'value' < range.
* The modulus operator.
*/
inline static double normalize(double value, double range) {
return value - range * ClockMath::floorDivide(value, range);
}
/**
* Normalize an angle so that it's in the range 0 - 2pi.
* For positive angles this is just (angle % 2pi), but the Java
* mod operator doesn't work that way for negative numbers....
*/
inline static double norm2PI(double angle) {
return normalize(angle, CalendarAstronomer::PI * 2.0);
}
/**
* Normalize an angle into the range -PI - PI
*/
inline static double normPI(double angle) {
return normalize(angle + CalendarAstronomer::PI, CalendarAstronomer::PI * 2.0) - CalendarAstronomer::PI;
}
//-------------------------------------------------------------------------
// Constructors
//-------------------------------------------------------------------------
/**
* Construct a new <code>CalendarAstronomer</code> object that is initialized to
* the current date and time.
* @internal
* @deprecated ICU 2.4. This class may be removed or modified.
*/
CalendarAstronomer::CalendarAstronomer():
fTime(Calendar::getNow()), moonPosition(0,0), moonPositionSet(false) {
clearCache();
}
/**
* Construct a new <code>CalendarAstronomer</code> object that is initialized to
* the specified date and time.
* @internal
* @deprecated ICU 2.4. This class may be removed or modified.
*/
CalendarAstronomer::CalendarAstronomer(UDate d): fTime(d), moonPosition(0,0), moonPositionSet(false) {
clearCache();
}
CalendarAstronomer::~CalendarAstronomer()
{
}
//-------------------------------------------------------------------------
// Time and date getters and setters
//-------------------------------------------------------------------------
/**
* Set the current date and time of this <code>CalendarAstronomer</code> object. All
* astronomical calculations are performed based on this time setting.
*
* @param aTime the date and time, expressed as the number of milliseconds since
* 1/1/1970 0:00 GMT (Gregorian).
*
* @see #setDate
* @see #getTime
* @internal
* @deprecated ICU 2.4. This class may be removed or modified.
*/
void CalendarAstronomer::setTime(UDate aTime) {
fTime = aTime;
clearCache();
}
/**
* Get the current time of this <code>CalendarAstronomer</code> object,
* represented as the number of milliseconds since
* 1/1/1970 AD 0:00 GMT (Gregorian).
*
* @see #setTime
* @see #getDate
* @internal
* @deprecated ICU 2.4. This class may be removed or modified.
*/
UDate CalendarAstronomer::getTime() {
return fTime;
}
/**
* Get the current time of this <code>CalendarAstronomer</code> object,
* expressed as a "julian day number", which is the number of elapsed
* days since 1/1/4713 BC (Julian), 12:00 GMT.
*
* @see #setJulianDay
* @see #JULIAN_EPOCH_MS
* @internal
* @deprecated ICU 2.4. This class may be removed or modified.
*/
double CalendarAstronomer::getJulianDay() {
if (isINVALID(julianDay)) {
julianDay = (fTime - JULIAN_EPOCH_MS) / static_cast<double>(DAY_MS);
}
return julianDay;
}
//-------------------------------------------------------------------------
// Coordinate transformations, all based on the current time of this object
//-------------------------------------------------------------------------
/**
* Convert from ecliptic to equatorial coordinates.
*
* @param eclipLong The ecliptic longitude
* @param eclipLat The ecliptic latitude
*
* @return The corresponding point in equatorial coordinates.
* @internal
* @deprecated ICU 2.4. This class may be removed or modified.
*/
CalendarAstronomer::Equatorial& CalendarAstronomer::eclipticToEquatorial(CalendarAstronomer::Equatorial& result, double eclipLong, double eclipLat)
{
// See page 42 of "Practical Astronomy with your Calculator",
// by Peter Duffet-Smith, for details on the algorithm.
double obliq = eclipticObliquity();
double sinE = ::sin(obliq);
double cosE = cos(obliq);
double sinL = ::sin(eclipLong);
double cosL = cos(eclipLong);
double sinB = ::sin(eclipLat);
double cosB = cos(eclipLat);
double tanB = tan(eclipLat);
result.set(atan2(sinL*cosE - tanB*sinE, cosL),
asin(sinB*cosE + cosB*sinE*sinL) );
return result;
}
//-------------------------------------------------------------------------
// The Sun
//-------------------------------------------------------------------------
//
// Parameters of the Sun's orbit as of the epoch Jan 0.0 1990
// Angles are in radians (after multiplying by CalendarAstronomer::PI/180)
//
#define JD_EPOCH 2447891.5 // Julian day of epoch
#define SUN_ETA_G (279.403303 * CalendarAstronomer::PI/180) // Ecliptic longitude at epoch
#define SUN_OMEGA_G (282.768422 * CalendarAstronomer::PI/180) // Ecliptic longitude of perigee
#define SUN_E 0.016713 // Eccentricity of orbit
//double sunR0 1.495585e8 // Semi-major axis in KM
//double sunTheta0 (0.533128 * CalendarAstronomer::PI/180) // Angular diameter at R0
// The following three methods, which compute the sun parameters
// given above for an arbitrary epoch (whatever time the object is
// set to), make only a small difference as compared to using the
// above constants. E.g., Sunset times might differ by ~12
// seconds. Furthermore, the eta-g computation is befuddled by
// Duffet-Smith's incorrect coefficients (p.86). I've corrected
// the first-order coefficient but the others may be off too - no
// way of knowing without consulting another source.
// /**
// * Return the sun's ecliptic longitude at perigee for the current time.
// * See Duffett-Smith, p. 86.
// * @return radians
// */
// private double getSunOmegaG() {
// double T = getJulianCentury();
// return (281.2208444 + (1.719175 + 0.000452778*T)*T) * DEG_RAD;
// }
// /**
// * Return the sun's ecliptic longitude for the current time.
// * See Duffett-Smith, p. 86.
// * @return radians
// */
// private double getSunEtaG() {
// double T = getJulianCentury();
// //return (279.6966778 + (36000.76892 + 0.0003025*T)*T) * DEG_RAD;
// //
// // The above line is from Duffett-Smith, and yields manifestly wrong
// // results. The below constant is derived empirically to match the
// // constant he gives for the 1990 EPOCH.
// //
// return (279.6966778 + (-0.3262541582718024 + 0.0003025*T)*T) * DEG_RAD;
// }
// /**
// * Return the sun's eccentricity of orbit for the current time.
// * See Duffett-Smith, p. 86.
// * @return double
// */
// private double getSunE() {
// double T = getJulianCentury();
// return 0.01675104 - (0.0000418 + 0.000000126*T)*T;
// }
/**
* Find the "true anomaly" (longitude) of an object from
* its mean anomaly and the eccentricity of its orbit. This uses
* an iterative solution to Kepler's equation.
*
* @param meanAnomaly The object's longitude calculated as if it were in
* a regular, circular orbit, measured in radians
* from the point of perigee.
*
* @param eccentricity The eccentricity of the orbit
*
* @return The true anomaly (longitude) measured in radians
*/
static double trueAnomaly(double meanAnomaly, double eccentricity)
{
// First, solve Kepler's equation iteratively
// Duffett-Smith, p.90
double delta;
double E = meanAnomaly;
do {
delta = E - eccentricity * ::sin(E) - meanAnomaly;
E = E - delta / (1 - eccentricity * ::cos(E));
}
while (uprv_fabs(delta) > 1e-5); // epsilon = 1e-5 rad
return 2.0 * ::atan( ::tan(E/2) * ::sqrt( (1+eccentricity)
/(1-eccentricity) ) );
}
/**
* The longitude of the sun at the time specified by this object.
* The longitude is measured in radians along the ecliptic
* from the "first point of Aries," the point at which the ecliptic
* crosses the earth's equatorial plane at the vernal equinox.
* <p>
* Currently, this method uses an approximation of the two-body Kepler's
* equation for the earth and the sun. It does not take into account the
* perturbations caused by the other planets, the moon, etc.
* @internal
* @deprecated ICU 2.4. This class may be removed or modified.
*/
double CalendarAstronomer::getSunLongitude()
{
// See page 86 of "Practical Astronomy with your Calculator",
// by Peter Duffet-Smith, for details on the algorithm.
if (isINVALID(sunLongitude)) {
getSunLongitude(getJulianDay(), sunLongitude, meanAnomalySun);
}
return sunLongitude;
}
/**
* TODO Make this public when the entire class is package-private.
*/
/*public*/ void CalendarAstronomer::getSunLongitude(double jDay, double &longitude, double &meanAnomaly)
{
// See page 86 of "Practical Astronomy with your Calculator",
// by Peter Duffet-Smith, for details on the algorithm.
double day = jDay - JD_EPOCH; // Days since epoch
// Find the angular distance the sun in a fictitious
// circular orbit has travelled since the epoch.
double epochAngle = norm2PI(CalendarAstronomer_PI2/TROPICAL_YEAR*day);
// The epoch wasn't at the sun's perigee; find the angular distance
// since perigee, which is called the "mean anomaly"
meanAnomaly = norm2PI(epochAngle + SUN_ETA_G - SUN_OMEGA_G);
// Now find the "true anomaly", e.g. the real solar longitude
// by solving Kepler's equation for an elliptical orbit
// NOTE: The 3rd ed. of the book lists omega_g and eta_g in different
// equations; omega_g is to be correct.
longitude = norm2PI(trueAnomaly(meanAnomaly, SUN_E) + SUN_OMEGA_G);
}
/**
* Constant representing the winter solstice.
* For use with {@link #getSunTime getSunTime}.
* Note: In this case, "winter" refers to the northern hemisphere's seasons.
* @internal
* @deprecated ICU 2.4. This class may be removed or modified.
*/
double CalendarAstronomer::WINTER_SOLSTICE() {
return ((CalendarAstronomer::PI*3)/2);
}
CalendarAstronomer::AngleFunc::~AngleFunc() {}
/**
* Find the next time at which the sun's ecliptic longitude will have
* the desired value.
* @internal
* @deprecated ICU 2.4. This class may be removed or modified.
*/
class SunTimeAngleFunc : public CalendarAstronomer::AngleFunc {
public:
virtual ~SunTimeAngleFunc();
virtual double eval(CalendarAstronomer& a) override { return a.getSunLongitude(); }
};
SunTimeAngleFunc::~SunTimeAngleFunc() {}
UDate CalendarAstronomer::getSunTime(double desired, UBool next)
{
SunTimeAngleFunc func;
return timeOfAngle( func,
desired,
TROPICAL_YEAR,
MINUTE_MS,
next);
}
//-------------------------------------------------------------------------
// The Moon
//-------------------------------------------------------------------------
#define moonL0 (318.351648 * CalendarAstronomer::PI/180 ) // Mean long. at epoch
#define moonP0 ( 36.340410 * CalendarAstronomer::PI/180 ) // Mean long. of perigee
#define moonN0 ( 318.510107 * CalendarAstronomer::PI/180 ) // Mean long. of node
#define moonI ( 5.145366 * CalendarAstronomer::PI/180 ) // Inclination of orbit
#define moonE ( 0.054900 ) // Eccentricity of orbit
// These aren't used right now
#define moonA ( 3.84401e5 ) // semi-major axis (km)
#define moonT0 ( 0.5181 * CalendarAstronomer::PI/180 ) // Angular size at distance A
#define moonPi ( 0.9507 * CalendarAstronomer::PI/180 ) // Parallax at distance A
/**
* The position of the moon at the time set on this
* object, in equatorial coordinates.
* @internal
* @deprecated ICU 2.4. This class may be removed or modified.
*/
const CalendarAstronomer::Equatorial& CalendarAstronomer::getMoonPosition()
{
//
// See page 142 of "Practical Astronomy with your Calculator",
// by Peter Duffet-Smith, for details on the algorithm.
//
if (moonPositionSet == false) {
// Calculate the solar longitude. Has the side effect of
// filling in "meanAnomalySun" as well.
getSunLongitude();
//
// Find the # of days since the epoch of our orbital parameters.
// TODO: Convert the time of day portion into ephemeris time
//
double day = getJulianDay() - JD_EPOCH; // Days since epoch
// Calculate the mean longitude and anomaly of the moon, based on
// a circular orbit. Similar to the corresponding solar calculation.
double meanLongitude = norm2PI(13.1763966*PI/180*day + moonL0);
double meanAnomalyMoon = norm2PI(meanLongitude - 0.1114041*PI/180 * day - moonP0);
//
// Calculate the following corrections:
// Evection: the sun's gravity affects the moon's eccentricity
// Annual Eqn: variation in the effect due to earth-sun distance
// A3: correction factor (for ???)
//
double evection = 1.2739*PI/180 * ::sin(2 * (meanLongitude - sunLongitude)
- meanAnomalyMoon);
double annual = 0.1858*PI/180 * ::sin(meanAnomalySun);
double a3 = 0.3700*PI/180 * ::sin(meanAnomalySun);
meanAnomalyMoon += evection - annual - a3;
//
// More correction factors:
// center equation of the center correction
// a4 yet another error correction (???)
//
// TODO: Skip the equation of the center correction and solve Kepler's eqn?
//
double center = 6.2886*PI/180 * ::sin(meanAnomalyMoon);
double a4 = 0.2140*PI/180 * ::sin(2 * meanAnomalyMoon);
// Now find the moon's corrected longitude
double moonLongitude = meanLongitude + evection + center - annual + a4;
//
// And finally, find the variation, caused by the fact that the sun's
// gravitational pull on the moon varies depending on which side of
// the earth the moon is on
//
double variation = 0.6583*CalendarAstronomer::PI/180 * ::sin(2*(moonLongitude - sunLongitude));
moonLongitude += variation;
//
// What we've calculated so far is the moon's longitude in the plane
// of its own orbit. Now map to the ecliptic to get the latitude
// and longitude. First we need to find the longitude of the ascending
// node, the position on the ecliptic where it is crossed by the moon's
// orbit as it crosses from the southern to the northern hemisphere.
//
double nodeLongitude = norm2PI(moonN0 - 0.0529539*PI/180 * day);
nodeLongitude -= 0.16*PI/180 * ::sin(meanAnomalySun);
double y = ::sin(moonLongitude - nodeLongitude);
double x = cos(moonLongitude - nodeLongitude);
moonEclipLong = ::atan2(y*cos(moonI), x) + nodeLongitude;
double moonEclipLat = ::asin(y * ::sin(moonI));
eclipticToEquatorial(moonPosition, moonEclipLong, moonEclipLat);
moonPositionSet = true;
}
return moonPosition;
}
/**
* The "age" of the moon at the time specified in this object.
* This is really the angle between the
* current ecliptic longitudes of the sun and the moon,
* measured in radians.
*
* @internal
* @deprecated ICU 2.4. This class may be removed or modified.
*/
double CalendarAstronomer::getMoonAge() {
// See page 147 of "Practical Astronomy with your Calculator",
// by Peter Duffet-Smith, for details on the algorithm.
//
// Force the moon's position to be calculated. We're going to use
// some the intermediate results cached during that calculation.
//
getMoonPosition();
return norm2PI(moonEclipLong - sunLongitude);
}
/**
* Constant representing a new moon.
* For use with {@link #getMoonTime getMoonTime}
* @internal
* @deprecated ICU 2.4. This class may be removed or modified.
*/
CalendarAstronomer::MoonAge CalendarAstronomer::NEW_MOON() {
return CalendarAstronomer::MoonAge(0);
}
/**
* Constant representing the moon's last quarter.
* For use with {@link #getMoonTime getMoonTime}
* @internal
* @deprecated ICU 2.4. This class may be removed or modified.
*/
class MoonTimeAngleFunc : public CalendarAstronomer::AngleFunc {
public:
virtual ~MoonTimeAngleFunc();
virtual double eval(CalendarAstronomer& a) override { return a.getMoonAge(); }
};
MoonTimeAngleFunc::~MoonTimeAngleFunc() {}
/*const CalendarAstronomer::MoonAge CalendarAstronomer::LAST_QUARTER() {
return CalendarAstronomer::MoonAge((CalendarAstronomer::PI*3)/2);
}*/
/**
* Find the next or previous time at which the moon will be in the
* desired phase.
* <p>
* @param desired The desired phase of the moon.
* @param next <tt>true</tt> if the next occurrence of the phase
* is desired, <tt>false</tt> for the previous occurrence.
* @internal
* @deprecated ICU 2.4. This class may be removed or modified.
*/
UDate CalendarAstronomer::getMoonTime(const CalendarAstronomer::MoonAge& desired, UBool next) {
MoonTimeAngleFunc func;
return timeOfAngle( func,
desired.value,
SYNODIC_MONTH,
MINUTE_MS,
next);
}
//-------------------------------------------------------------------------
// Interpolation methods for finding the time at which a given event occurs
//-------------------------------------------------------------------------
UDate CalendarAstronomer::timeOfAngle(AngleFunc& func, double desired,
double periodDays, double epsilon, UBool next)
{
// Find the value of the function at the current time
double lastAngle = func.eval(*this);
// Find out how far we are from the desired angle
double deltaAngle = norm2PI(desired - lastAngle) ;
// Using the average period, estimate the next (or previous) time at
// which the desired angle occurs.
double deltaT = (deltaAngle + (next ? 0.0 : - CalendarAstronomer_PI2 )) * (periodDays*DAY_MS) / CalendarAstronomer_PI2;
double lastDeltaT = deltaT; // Liu
UDate startTime = fTime; // Liu
setTime(fTime + uprv_ceil(deltaT));
// Now iterate until we get the error below epsilon. Throughout
// this loop we use normPI to get values in the range -Pi to Pi,
// since we're using them as correction factors rather than absolute angles.
do {
// Evaluate the function at the time we've estimated
double angle = func.eval(*this);
// Find the # of milliseconds per radian at this point on the curve
double factor = uprv_fabs(deltaT / normPI(angle-lastAngle));
// Correct the time estimate based on how far off the angle is
deltaT = normPI(desired - angle) * factor;
// HACK:
//
// If abs(deltaT) begins to diverge we need to quit this loop.
// This only appears to happen when attempting to locate, for
// example, a new moon on the day of the new moon. E.g.:
//
// This result is correct:
// newMoon(7508(Mon Jul 23 00:00:00 CST 1990,false))=
// Sun Jul 22 10:57:41 CST 1990
//
// But attempting to make the same call a day earlier causes deltaT
// to diverge:
// CalendarAstronomer.timeOfAngle() diverging: 1.348508727575625E9 ->
// 1.3649828540224032E9
// newMoon(7507(Sun Jul 22 00:00:00 CST 1990,false))=
// Sun Jul 08 13:56:15 CST 1990
//
// As a temporary solution, we catch this specific condition and
// adjust our start time by one eighth period days (either forward
// or backward) and try again.
// Liu 11/9/00
if (uprv_fabs(deltaT) > uprv_fabs(lastDeltaT)) {
double delta = uprv_ceil (periodDays * DAY_MS / 8.0);
setTime(startTime + (next ? delta : -delta));
return timeOfAngle(func, desired, periodDays, epsilon, next);
}
lastDeltaT = deltaT;
lastAngle = angle;
setTime(fTime + uprv_ceil(deltaT));
}
while (uprv_fabs(deltaT) > epsilon);
return fTime;
}
/**
* Return the obliquity of the ecliptic (the angle between the ecliptic
* and the earth's equator) at the current time. This varies due to
* the precession of the earth's axis.
*
* @return the obliquity of the ecliptic relative to the equator,
* measured in radians.
*/
double CalendarAstronomer::eclipticObliquity() {
const double epoch = 2451545.0; // 2000 AD, January 1.5
double T = (getJulianDay() - epoch) / 36525;
double eclipObliquity = 23.439292
- 46.815/3600 * T
- 0.0006/3600 * T*T
+ 0.00181/3600 * T*T*T;
return eclipObliquity * DEG_RAD;
}
//-------------------------------------------------------------------------
// Private data
//-------------------------------------------------------------------------
void CalendarAstronomer::clearCache() {
const double INVALID = uprv_getNaN();
julianDay = INVALID;
sunLongitude = INVALID;
meanAnomalySun = INVALID;
moonEclipLong = INVALID;
moonPositionSet = false;
}
// Debugging functions
UnicodeString CalendarAstronomer::Ecliptic::toString() const
{
#ifdef U_DEBUG_ASTRO
char tmp[800];
snprintf(tmp, sizeof(tmp), "[%.5f,%.5f]", longitude*RAD_DEG, latitude*RAD_DEG);
return UnicodeString(tmp, "");
#else
return {};
#endif
}
UnicodeString CalendarAstronomer::Equatorial::toString() const
{
#ifdef U_DEBUG_ASTRO
char tmp[400];
snprintf(tmp, sizeof(tmp), "%f,%f",
(ascension*RAD_DEG), (declination*RAD_DEG));
return UnicodeString(tmp, "");
#else
return {};
#endif
}
// =============== Calendar Cache ================
void CalendarCache::createCache(CalendarCache** cache, UErrorCode& status) {
ucln_i18n_registerCleanup(UCLN_I18N_ASTRO_CALENDAR, calendar_astro_cleanup);
if(cache == nullptr) {
status = U_MEMORY_ALLOCATION_ERROR;
} else {
*cache = new CalendarCache(32, status);
if(U_FAILURE(status)) {
delete *cache;
*cache = nullptr;
}
}
}
int32_t CalendarCache::get(CalendarCache** cache, int32_t key, UErrorCode &status) {
int32_t res;
if(U_FAILURE(status)) {
return 0;
}
umtx_lock(&ccLock);
if(*cache == nullptr) {
createCache(cache, status);
if(U_FAILURE(status)) {
umtx_unlock(&ccLock);
return 0;
}
}
res = uhash_igeti((*cache)->fTable, key);
U_DEBUG_ASTRO_MSG(("%p: GET: [%d] == %d\n", (*cache)->fTable, key, res));
umtx_unlock(&ccLock);
return res;
}
void CalendarCache::put(CalendarCache** cache, int32_t key, int32_t value, UErrorCode &status) {
if(U_FAILURE(status)) {
return;
}
umtx_lock(&ccLock);
if(*cache == nullptr) {
createCache(cache, status);
if(U_FAILURE(status)) {
umtx_unlock(&ccLock);
return;
}
}
uhash_iputi((*cache)->fTable, key, value, &status);
U_DEBUG_ASTRO_MSG(("%p: PUT: [%d] := %d\n", (*cache)->fTable, key, value));
umtx_unlock(&ccLock);
}
CalendarCache::CalendarCache(int32_t size, UErrorCode &status) {
fTable = uhash_openSize(uhash_hashLong, uhash_compareLong, nullptr, size, &status);
U_DEBUG_ASTRO_MSG(("%p: Opening.\n", fTable));
}
CalendarCache::~CalendarCache() {
if(fTable != nullptr) {
U_DEBUG_ASTRO_MSG(("%p: Closing.\n", fTable));
uhash_close(fTable);
}
}
U_NAMESPACE_END
#endif // !UCONFIG_NO_FORMATTING

470
deps/icu-small/source/i18n/astro.h vendored Normal file
View File

@ -0,0 +1,470 @@
// © 2016 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html
/************************************************************************
* Copyright (C) 1996-2008, International Business Machines Corporation *
* and others. All Rights Reserved. *
************************************************************************
* 2003-nov-07 srl Port from Java
*/
#ifndef ASTRO_H
#define ASTRO_H
#include "unicode/utypes.h"
#if !UCONFIG_NO_FORMATTING
#include "gregoimp.h" // for Math
#include "unicode/unistr.h"
U_NAMESPACE_BEGIN
/**
* <code>CalendarAstronomer</code> is a class that can perform the calculations to
* determine the positions of the sun and moon, the time of sunrise and
* sunset, and other astronomy-related data. The calculations it performs
* are in some cases quite complicated, and this utility class saves you
* the trouble of worrying about them.
* <p>
* The measurement of time is a very important part of astronomy. Because
* astronomical bodies are constantly in motion, observations are only valid
* at a given moment in time. Accordingly, each <code>CalendarAstronomer</code>
* object has a <code>time</code> property that determines the date
* and time for which its calculations are performed. You can set and
* retrieve this property with {@link #setTime setTime}, {@link #getTime getTime}
* and related methods.
* <p>
* Almost all of the calculations performed by this class, or by any
* astronomer, are approximations to various degrees of accuracy. The
* calculations in this class are mostly modelled after those described
* in the book
* <a href="http://www.amazon.com/exec/obidos/ISBN=0521356997" target="_top">
* Practical Astronomy With Your Calculator</a>, by Peter J.
* Duffett-Smith, Cambridge University Press, 1990. This is an excellent
* book, and if you want a greater understanding of how these calculations
* are performed it a very good, readable starting point.
* <p>
* <strong>WARNING:</strong> This class is very early in its development, and
* it is highly likely that its API will change to some degree in the future.
* At the moment, it basically does just enough to support {@link IslamicCalendar}
* and {@link ChineseCalendar}.
*
* @author Laura Werner
* @author Alan Liu
* @internal
*/
class U_I18N_API CalendarAstronomer : public UMemory {
public:
// some classes
public:
/**
* Represents the position of an object in the sky relative to the ecliptic,
* the plane of the earth's orbit around the Sun.
* This is a spherical coordinate system in which the latitude
* specifies the position north or south of the plane of the ecliptic.
* The longitude specifies the position along the ecliptic plane
* relative to the "First Point of Aries", which is the Sun's position in the sky
* at the Vernal Equinox.
* <p>
* Note that Ecliptic objects are immutable and cannot be modified
* once they are constructed. This allows them to be passed and returned by
* value without worrying about whether other code will modify them.
*
* @see CalendarAstronomer.Equatorial
* @internal
*/
class U_I18N_API Ecliptic : public UMemory {
public:
/**
* Constructs an Ecliptic coordinate object.
* <p>
* @param lat The ecliptic latitude, measured in radians.
* @param lon The ecliptic longitude, measured in radians.
* @internal
*/
Ecliptic(double lat = 0, double lon = 0) {
latitude = lat;
longitude = lon;
}
/**
* Setter for Ecliptic Coordinate object
* @param lat The ecliptic latitude, measured in radians.
* @param lon The ecliptic longitude, measured in radians.
* @internal
*/
void set(double lat, double lon) {
latitude = lat;
longitude = lon;
}
/**
* Return a string representation of this object
* @internal
*/
UnicodeString toString() const;
/**
* The ecliptic latitude, in radians. This specifies an object's
* position north or south of the plane of the ecliptic,
* with positive angles representing north.
* @internal
*/
double latitude;
/**
* The ecliptic longitude, in radians.
* This specifies an object's position along the ecliptic plane
* relative to the "First Point of Aries", which is the Sun's position
* in the sky at the Vernal Equinox,
* with positive angles representing east.
* <p>
* A bit of trivia: the first point of Aries is currently in the
* constellation Pisces, due to the precession of the earth's axis.
* @internal
*/
double longitude;
};
/**
* Represents the position of an
* object in the sky relative to the plane of the earth's equator.
* The <i>Right Ascension</i> specifies the position east or west
* along the equator, relative to the sun's position at the vernal
* equinox. The <i>Declination</i> is the position north or south
* of the equatorial plane.
* <p>
* Note that Equatorial objects are immutable and cannot be modified
* once they are constructed. This allows them to be passed and returned by
* value without worrying about whether other code will modify them.
*
* @see CalendarAstronomer.Ecliptic
* @internal
*/
class U_I18N_API Equatorial : public UMemory {
public:
/**
* Constructs an Equatorial coordinate object.
* <p>
* @param asc The right ascension, measured in radians.
* @param dec The declination, measured in radians.
* @internal
*/
Equatorial(double asc = 0, double dec = 0)
: ascension(asc), declination(dec) { }
/**
* Setter
* @param asc The right ascension, measured in radians.
* @param dec The declination, measured in radians.
* @internal
*/
void set(double asc, double dec) {
ascension = asc;
declination = dec;
}
/**
* Return a string representation of this object, with the
* angles measured in degrees.
* @internal
*/
UnicodeString toString() const;
/**
* Return a string representation of this object with the right ascension
* measured in hours, minutes, and seconds.
* @internal
*/
//String toHmsString() {
//return radToHms(ascension) + "," + radToDms(declination);
//}
/**
* The right ascension, in radians.
* This is the position east or west along the equator
* relative to the sun's position at the vernal equinox,
* with positive angles representing East.
* @internal
*/
double ascension;
/**
* The declination, in radians.
* This is the position north or south of the equatorial plane,
* with positive angles representing north.
* @internal
*/
double declination;
};
public:
//-------------------------------------------------------------------------
// Assorted private data used for conversions
//-------------------------------------------------------------------------
// My own copies of these so compilers are more likely to optimize them away
static const double PI;
/**
* The average number of solar days from one new moon to the next. This is the time
* it takes for the moon to return the same ecliptic longitude as the sun.
* It is longer than the sidereal month because the sun's longitude increases
* during the year due to the revolution of the earth around the sun.
* Approximately 29.53.
*
* @see #SIDEREAL_MONTH
* @internal
* @deprecated ICU 2.4. This class may be removed or modified.
*/
static const double SYNODIC_MONTH;
//-------------------------------------------------------------------------
// Constructors
//-------------------------------------------------------------------------
/**
* Construct a new <code>CalendarAstronomer</code> object that is initialized to
* the current date and time.
* @internal
*/
CalendarAstronomer();
/**
* Construct a new <code>CalendarAstronomer</code> object that is initialized to
* the specified date and time.
* @internal
*/
CalendarAstronomer(UDate d);
/**
* Destructor
* @internal
*/
~CalendarAstronomer();
//-------------------------------------------------------------------------
// Time and date getters and setters
//-------------------------------------------------------------------------
/**
* Set the current date and time of this <code>CalendarAstronomer</code> object. All
* astronomical calculations are performed based on this time setting.
*
* @param aTime the date and time, expressed as the number of milliseconds since
* 1/1/1970 0:00 GMT (Gregorian).
*
* @see #getTime
* @internal
*/
void setTime(UDate aTime);
/**
* Get the current time of this <code>CalendarAstronomer</code> object,
* represented as the number of milliseconds since
* 1/1/1970 AD 0:00 GMT (Gregorian).
*
* @see #setTime
* @internal
*/
UDate getTime();
/**
* Get the current time of this <code>CalendarAstronomer</code> object,
* expressed as a "julian day number", which is the number of elapsed
* days since 1/1/4713 BC (Julian), 12:00 GMT.
*
* @see #JULIAN_EPOCH_MS
* @internal
*/
double getJulianDay();
public:
/**
* Convert from ecliptic to equatorial coordinates.
*
* @param eclipLong The ecliptic longitude
* @param eclipLat The ecliptic latitude
*
* @return The corresponding point in equatorial coordinates.
* @internal
*/
Equatorial& eclipticToEquatorial(Equatorial& result, double eclipLong, double eclipLat);
//-------------------------------------------------------------------------
// The Sun
//-------------------------------------------------------------------------
/**
* The longitude of the sun at the time specified by this object.
* The longitude is measured in radians along the ecliptic
* from the "first point of Aries," the point at which the ecliptic
* crosses the earth's equatorial plane at the vernal equinox.
* <p>
* Currently, this method uses an approximation of the two-body Kepler's
* equation for the earth and the sun. It does not take into account the
* perturbations caused by the other planets, the moon, etc.
* @internal
*/
double getSunLongitude();
/**
* TODO Make this public when the entire class is package-private.
*/
/*public*/ void getSunLongitude(double julianDay, double &longitude, double &meanAnomaly);
public:
/**
* Constant representing the winter solstice.
* For use with {@link #getSunTime getSunTime}.
* Note: In this case, "winter" refers to the northern hemisphere's seasons.
* @internal
*/
static double WINTER_SOLSTICE();
/**
* Find the next time at which the sun's ecliptic longitude will have
* the desired value.
* @internal
*/
UDate getSunTime(double desired, UBool next);
//-------------------------------------------------------------------------
// The Moon
//-------------------------------------------------------------------------
/**
* The position of the moon at the time set on this
* object, in equatorial coordinates.
* @internal
* @return const reference to internal field of calendar astronomer. Do not use outside of the lifetime of this astronomer.
*/
const Equatorial& getMoonPosition();
/**
* The "age" of the moon at the time specified in this object.
* This is really the angle between the
* current ecliptic longitudes of the sun and the moon,
* measured in radians.
*
* @see #getMoonPhase
* @internal
*/
double getMoonAge();
class U_I18N_API MoonAge : public UMemory {
public:
MoonAge(double l)
: value(l) { }
void set(double l) { value = l; }
double value;
};
/**
* Constant representing a new moon.
* For use with {@link #getMoonTime getMoonTime}
* @internal
*/
static MoonAge NEW_MOON();
/**
* Find the next or previous time at which the Moon's ecliptic
* longitude will have the desired value.
* <p>
* @param desired The desired longitude.
* @param next <tt>true</tt> if the next occurrence of the phase
* is desired, <tt>false</tt> for the previous occurrence.
* @internal
*/
UDate getMoonTime(const MoonAge& desired, UBool next);
//-------------------------------------------------------------------------
// Interpolation methods for finding the time at which a given event occurs
//-------------------------------------------------------------------------
public:
class AngleFunc : public UMemory {
public:
virtual double eval(CalendarAstronomer&) = 0;
virtual ~AngleFunc();
};
friend class AngleFunc;
private:
UDate timeOfAngle(AngleFunc& func, double desired,
double periodDays, double epsilon, UBool next);
//-------------------------------------------------------------------------
// Other utility methods
//-------------------------------------------------------------------------
private:
/**
* Return the obliquity of the ecliptic (the angle between the ecliptic
* and the earth's equator) at the current time. This varies due to
* the precession of the earth's axis.
*
* @return the obliquity of the ecliptic relative to the equator,
* measured in radians.
*/
double eclipticObliquity();
//-------------------------------------------------------------------------
// Private data
//-------------------------------------------------------------------------
private:
/**
* Current time in milliseconds since 1/1/1970 AD
* @see java.util.Date#getTime
*/
UDate fTime;
// The following fields are used to cache calculated results for improved
// performance. These values all depend on the current time setting
// of this object, so the clearCache method is provided.
double julianDay;
double sunLongitude;
double meanAnomalySun;
double moonEclipLong;
void clearCache();
Equatorial moonPosition;
UBool moonPositionSet;
/**
* @internal
*/
// UDate local(UDate localMillis);
};
U_NAMESPACE_END
struct UHashtable;
U_NAMESPACE_BEGIN
/**
* Cache of month -> julian day
* @internal
*/
class CalendarCache : public UMemory {
public:
static int32_t get(CalendarCache** cache, int32_t key, UErrorCode &status);
static void put(CalendarCache** cache, int32_t key, int32_t value, UErrorCode &status);
virtual ~CalendarCache();
private:
CalendarCache(int32_t size, UErrorCode& status);
static void createCache(CalendarCache** cache, UErrorCode& status);
/**
* not implemented
*/
CalendarCache();
UHashtable *fTable;
};
U_NAMESPACE_END
#endif
#endif

544
deps/icu-small/source/i18n/basictz.cpp vendored Normal file
View File

@ -0,0 +1,544 @@
// © 2016 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html
/*
*******************************************************************************
* Copyright (C) 2007-2013, International Business Machines Corporation and
* others. All Rights Reserved.
*******************************************************************************
*/
#include "unicode/utypes.h"
#if !UCONFIG_NO_FORMATTING
#include "unicode/basictz.h"
#include "gregoimp.h"
#include "uvector.h"
#include "cmemory.h"
U_NAMESPACE_BEGIN
#define MILLIS_PER_YEAR (365*24*60*60*1000.0)
BasicTimeZone::BasicTimeZone()
: TimeZone() {
}
BasicTimeZone::BasicTimeZone(const UnicodeString &id)
: TimeZone(id) {
}
BasicTimeZone::BasicTimeZone(const BasicTimeZone& source)
: TimeZone(source) {
}
BasicTimeZone::~BasicTimeZone() {
}
UBool
BasicTimeZone::hasEquivalentTransitions(const BasicTimeZone& tz, UDate start, UDate end,
UBool ignoreDstAmount, UErrorCode& status) const {
if (U_FAILURE(status)) {
return false;
}
if (hasSameRules(tz)) {
return true;
}
// Check the offsets at the start time
int32_t raw1, raw2, dst1, dst2;
getOffset(start, false, raw1, dst1, status);
if (U_FAILURE(status)) {
return false;
}
tz.getOffset(start, false, raw2, dst2, status);
if (U_FAILURE(status)) {
return false;
}
if (ignoreDstAmount) {
if ((raw1 + dst1 != raw2 + dst2)
|| (dst1 != 0 && dst2 == 0)
|| (dst1 == 0 && dst2 != 0)) {
return false;
}
} else {
if (raw1 != raw2 || dst1 != dst2) {
return false;
}
}
// Check transitions in the range
UDate time = start;
TimeZoneTransition tr1, tr2;
while (true) {
UBool avail1 = getNextTransition(time, false, tr1);
UBool avail2 = tz.getNextTransition(time, false, tr2);
if (ignoreDstAmount) {
// Skip a transition which only differ the amount of DST savings
while (true) {
if (avail1
&& tr1.getTime() <= end
&& (tr1.getFrom()->getRawOffset() + tr1.getFrom()->getDSTSavings()
== tr1.getTo()->getRawOffset() + tr1.getTo()->getDSTSavings())
&& (tr1.getFrom()->getDSTSavings() != 0 && tr1.getTo()->getDSTSavings() != 0)) {
getNextTransition(tr1.getTime(), false, tr1);
} else {
break;
}
}
while (true) {
if (avail2
&& tr2.getTime() <= end
&& (tr2.getFrom()->getRawOffset() + tr2.getFrom()->getDSTSavings()
== tr2.getTo()->getRawOffset() + tr2.getTo()->getDSTSavings())
&& (tr2.getFrom()->getDSTSavings() != 0 && tr2.getTo()->getDSTSavings() != 0)) {
tz.getNextTransition(tr2.getTime(), false, tr2);
} else {
break;
}
}
}
UBool inRange1 = (avail1 && tr1.getTime() <= end);
UBool inRange2 = (avail2 && tr2.getTime() <= end);
if (!inRange1 && !inRange2) {
// No more transition in the range
break;
}
if (!inRange1 || !inRange2) {
return false;
}
if (tr1.getTime() != tr2.getTime()) {
return false;
}
if (ignoreDstAmount) {
if (tr1.getTo()->getRawOffset() + tr1.getTo()->getDSTSavings()
!= tr2.getTo()->getRawOffset() + tr2.getTo()->getDSTSavings()
|| (tr1.getTo()->getDSTSavings() != 0 && tr2.getTo()->getDSTSavings() == 0)
|| (tr1.getTo()->getDSTSavings() == 0 && tr2.getTo()->getDSTSavings() != 0)) {
return false;
}
} else {
if (tr1.getTo()->getRawOffset() != tr2.getTo()->getRawOffset() ||
tr1.getTo()->getDSTSavings() != tr2.getTo()->getDSTSavings()) {
return false;
}
}
time = tr1.getTime();
}
return true;
}
void
BasicTimeZone::getSimpleRulesNear(UDate date, InitialTimeZoneRule*& initial,
AnnualTimeZoneRule*& std, AnnualTimeZoneRule*& dst, UErrorCode& status) const {
initial = nullptr;
std = nullptr;
dst = nullptr;
if (U_FAILURE(status)) {
return;
}
int32_t initialRaw, initialDst;
UnicodeString initialName;
AnnualTimeZoneRule *ar1 = nullptr;
AnnualTimeZoneRule *ar2 = nullptr;
UnicodeString name;
UBool avail;
TimeZoneTransition tr;
// Get the next transition
avail = getNextTransition(date, false, tr);
if (avail) {
tr.getFrom()->getName(initialName);
initialRaw = tr.getFrom()->getRawOffset();
initialDst = tr.getFrom()->getDSTSavings();
// Check if the next transition is either DST->STD or STD->DST and
// within roughly 1 year from the specified date
UDate nextTransitionTime = tr.getTime();
if (((tr.getFrom()->getDSTSavings() == 0 && tr.getTo()->getDSTSavings() != 0)
|| (tr.getFrom()->getDSTSavings() != 0 && tr.getTo()->getDSTSavings() == 0))
&& (date + MILLIS_PER_YEAR > nextTransitionTime)) {
int32_t year, mid;
int8_t month, dom, dow;
UDate d;
// Get local wall time for the next transition time
Grego::timeToFields(nextTransitionTime + initialRaw + initialDst,
year, month, dom, dow, mid, status);
if (U_FAILURE(status)) return;
int32_t weekInMonth = Grego::dayOfWeekInMonth(year, month, dom);
// Create DOW rule
DateTimeRule *dtr = new DateTimeRule(month, weekInMonth, dow, mid, DateTimeRule::WALL_TIME);
tr.getTo()->getName(name);
// Note: SimpleTimeZone does not support raw offset change.
// So we always use raw offset of the given time for the rule,
// even raw offset is changed. This will result that the result
// zone to return wrong offset after the transition.
// When we encounter such case, we do not inspect next next
// transition for another rule.
ar1 = new AnnualTimeZoneRule(name, initialRaw, tr.getTo()->getDSTSavings(),
dtr, year, AnnualTimeZoneRule::MAX_YEAR);
if (tr.getTo()->getRawOffset() == initialRaw) {
// Get the next next transition
avail = getNextTransition(nextTransitionTime, false, tr);
if (avail) {
// Check if the next next transition is either DST->STD or STD->DST
// and within roughly 1 year from the next transition
if (((tr.getFrom()->getDSTSavings() == 0 && tr.getTo()->getDSTSavings() != 0)
|| (tr.getFrom()->getDSTSavings() != 0 && tr.getTo()->getDSTSavings() == 0))
&& nextTransitionTime + MILLIS_PER_YEAR > tr.getTime()) {
// Get local wall time for the next transition time
Grego::timeToFields(tr.getTime() + tr.getFrom()->getRawOffset() + tr.getFrom()->getDSTSavings(),
year, month, dom, dow, mid, status);
if (U_FAILURE(status)) return;
weekInMonth = Grego::dayOfWeekInMonth(year, month, dom);
// Generate another DOW rule
dtr = new DateTimeRule(month, weekInMonth, dow, mid, DateTimeRule::WALL_TIME);
tr.getTo()->getName(name);
ar2 = new AnnualTimeZoneRule(name, tr.getTo()->getRawOffset(), tr.getTo()->getDSTSavings(),
dtr, year - 1, AnnualTimeZoneRule::MAX_YEAR);
// Make sure this rule can be applied to the specified date
avail = ar2->getPreviousStart(date, tr.getFrom()->getRawOffset(), tr.getFrom()->getDSTSavings(), true, d);
if (!avail || d > date
|| initialRaw != tr.getTo()->getRawOffset()
|| initialDst != tr.getTo()->getDSTSavings()) {
// We cannot use this rule as the second transition rule
delete ar2;
ar2 = nullptr;
}
}
}
}
if (ar2 == nullptr) {
// Try previous transition
avail = getPreviousTransition(date, true, tr);
if (avail) {
// Check if the previous transition is either DST->STD or STD->DST.
// The actual transition time does not matter here.
if ((tr.getFrom()->getDSTSavings() == 0 && tr.getTo()->getDSTSavings() != 0)
|| (tr.getFrom()->getDSTSavings() != 0 && tr.getTo()->getDSTSavings() == 0)) {
// Generate another DOW rule
Grego::timeToFields(tr.getTime() + tr.getFrom()->getRawOffset() + tr.getFrom()->getDSTSavings(),
year, month, dom, dow, mid, status);
if (U_FAILURE(status)) return;
weekInMonth = Grego::dayOfWeekInMonth(year, month, dom);
dtr = new DateTimeRule(month, weekInMonth, dow, mid, DateTimeRule::WALL_TIME);
tr.getTo()->getName(name);
// second rule raw/dst offsets should match raw/dst offsets
// at the given time
ar2 = new AnnualTimeZoneRule(name, initialRaw, initialDst,
dtr, ar1->getStartYear() - 1, AnnualTimeZoneRule::MAX_YEAR);
// Check if this rule start after the first rule after the specified date
avail = ar2->getNextStart(date, tr.getFrom()->getRawOffset(), tr.getFrom()->getDSTSavings(), false, d);
if (!avail || d <= nextTransitionTime) {
// We cannot use this rule as the second transition rule
delete ar2;
ar2 = nullptr;
}
}
}
}
if (ar2 == nullptr) {
// Cannot find a good pair of AnnualTimeZoneRule
delete ar1;
ar1 = nullptr;
} else {
// The initial rule should represent the rule before the previous transition
ar1->getName(initialName);
initialRaw = ar1->getRawOffset();
initialDst = ar1->getDSTSavings();
}
}
}
else {
// Try the previous one
avail = getPreviousTransition(date, true, tr);
if (avail) {
tr.getTo()->getName(initialName);
initialRaw = tr.getTo()->getRawOffset();
initialDst = tr.getTo()->getDSTSavings();
} else {
// No transitions in the past. Just use the current offsets
getOffset(date, false, initialRaw, initialDst, status);
if (U_FAILURE(status)) {
return;
}
}
}
// Set the initial rule
initial = new InitialTimeZoneRule(initialName, initialRaw, initialDst);
// Set the standard and daylight saving rules
if (ar1 != nullptr && ar2 != nullptr) {
if (ar1->getDSTSavings() != 0) {
dst = ar1;
std = ar2;
} else {
std = ar1;
dst = ar2;
}
}
}
void
BasicTimeZone::getTimeZoneRulesAfter(UDate start, InitialTimeZoneRule*& initial,
UVector*& transitionRules, UErrorCode& status) const {
if (U_FAILURE(status)) {
return;
}
const InitialTimeZoneRule *orgini;
TimeZoneTransition tzt;
bool avail;
int32_t ruleCount;
TimeZoneRule *r = nullptr;
UnicodeString name;
int32_t i;
UDate time, t;
UDate firstStart;
UBool bFinalStd = false, bFinalDst = false;
initial = nullptr;
transitionRules = nullptr;
// Original transition rules
ruleCount = countTransitionRules(status);
if (U_FAILURE(status)) {
return;
}
LocalPointer<UVector> orgRules(
new UVector(uprv_deleteUObject, nullptr, ruleCount, status), status);
if (U_FAILURE(status)) {
return;
}
LocalMemory<const TimeZoneRule *> orgtrs(
static_cast<const TimeZoneRule **>(uprv_malloc(sizeof(TimeZoneRule*)*ruleCount)));
if (orgtrs.isNull()) {
status = U_MEMORY_ALLOCATION_ERROR;
return;
}
getTimeZoneRules(orgini, &orgtrs[0], ruleCount, status);
if (U_FAILURE(status)) {
return;
}
for (i = 0; i < ruleCount; i++) {
LocalPointer<TimeZoneRule> lpRule(orgtrs[i]->clone(), status);
orgRules->adoptElement(lpRule.orphan(), status);
if (U_FAILURE(status)) {
return;
}
}
avail = getPreviousTransition(start, true, tzt);
if (!avail) {
// No need to filter out rules only applicable to time before the start
initial = orgini->clone();
if (initial == nullptr) {
status = U_MEMORY_ALLOCATION_ERROR;
return;
}
transitionRules = orgRules.orphan();
return;
}
LocalMemory<bool> done(static_cast<bool *>(uprv_malloc(sizeof(bool)*ruleCount)));
if (done.isNull()) {
status = U_MEMORY_ALLOCATION_ERROR;
return;
}
LocalPointer<UVector> filteredRules(
new UVector(uprv_deleteUObject, nullptr, status), status);
if (U_FAILURE(status)) {
return;
}
// Create initial rule
tzt.getTo()->getName(name);
LocalPointer<InitialTimeZoneRule> res_initial(
new InitialTimeZoneRule(name, tzt.getTo()->getRawOffset(), tzt.getTo()->getDSTSavings()), status);
if (U_FAILURE(status)) {
return;
}
// Mark rules which does not need to be processed
for (i = 0; i < ruleCount; i++) {
r = static_cast<TimeZoneRule*>(orgRules->elementAt(i));
avail = r->getNextStart(start, res_initial->getRawOffset(), res_initial->getDSTSavings(), false, time);
done[i] = !avail;
}
time = start;
while (!bFinalStd || !bFinalDst) {
avail = getNextTransition(time, false, tzt);
if (!avail) {
break;
}
UDate updatedTime = tzt.getTime();
if (updatedTime == time) {
// Can get here if rules for start & end of daylight time have exactly
// the same time.
// TODO: fix getNextTransition() to prevent it?
status = U_INVALID_STATE_ERROR;
return;
}
time = updatedTime;
const TimeZoneRule *toRule = tzt.getTo();
for (i = 0; i < ruleCount; i++) {
r = static_cast<TimeZoneRule*>(orgRules->elementAt(i));
if (*r == *toRule) {
break;
}
}
if (i >= ruleCount) {
// This case should never happen
status = U_INVALID_STATE_ERROR;
return;
}
if (done[i]) {
continue;
}
const TimeArrayTimeZoneRule *tar = dynamic_cast<const TimeArrayTimeZoneRule *>(toRule);
const AnnualTimeZoneRule *ar;
if (tar != nullptr) {
// Get the previous raw offset and DST savings before the very first start time
TimeZoneTransition tzt0;
t = start;
while (true) {
avail = getNextTransition(t, false, tzt0);
if (!avail) {
break;
}
if (*(tzt0.getTo()) == *tar) {
break;
}
t = tzt0.getTime();
}
if (avail) {
// Check if the entire start times to be added
tar->getFirstStart(tzt.getFrom()->getRawOffset(), tzt.getFrom()->getDSTSavings(), firstStart);
if (firstStart > start) {
// Just add the rule as is
LocalPointer<TimeArrayTimeZoneRule> lpTar(tar->clone(), status);
filteredRules->adoptElement(lpTar.orphan(), status);
if (U_FAILURE(status)) {
return;
}
} else {
// Collect transitions after the start time
int32_t startTimes;
DateTimeRule::TimeRuleType timeType;
int32_t idx;
startTimes = tar->countStartTimes();
timeType = tar->getTimeType();
for (idx = 0; idx < startTimes; idx++) {
tar->getStartTimeAt(idx, t);
if (timeType == DateTimeRule::STANDARD_TIME) {
t -= tzt.getFrom()->getRawOffset();
}
if (timeType == DateTimeRule::WALL_TIME) {
t -= tzt.getFrom()->getDSTSavings();
}
if (t > start) {
break;
}
}
if (U_FAILURE(status)) {
return;
}
int32_t asize = startTimes - idx;
if (asize > 0) {
LocalMemory<UDate> newTimes(static_cast<UDate *>(uprv_malloc(sizeof(UDate) * asize)));
if (newTimes.isNull()) {
status = U_MEMORY_ALLOCATION_ERROR;
return;
}
for (int32_t newidx = 0; newidx < asize; newidx++) {
tar->getStartTimeAt(idx + newidx, newTimes[newidx]);
}
tar->getName(name);
LocalPointer<TimeArrayTimeZoneRule> newTar(new TimeArrayTimeZoneRule(
name, tar->getRawOffset(), tar->getDSTSavings(), &newTimes[0], asize, timeType), status);
filteredRules->adoptElement(newTar.orphan(), status);
if (U_FAILURE(status)) {
return;
}
}
}
}
} else if ((ar = dynamic_cast<const AnnualTimeZoneRule *>(toRule)) != nullptr) {
ar->getFirstStart(tzt.getFrom()->getRawOffset(), tzt.getFrom()->getDSTSavings(), firstStart);
if (firstStart == tzt.getTime()) {
// Just add the rule as is
LocalPointer<AnnualTimeZoneRule> arClone(ar->clone(), status);
filteredRules->adoptElement(arClone.orphan(), status);
if (U_FAILURE(status)) {
return;
}
} else {
// Calculate the transition year
int32_t year = Grego::timeToYear(tzt.getTime(), status);
if (U_FAILURE(status)) {
return;
}
// Re-create the rule
ar->getName(name);
LocalPointer<AnnualTimeZoneRule> newAr(new AnnualTimeZoneRule(name, ar->getRawOffset(), ar->getDSTSavings(),
*(ar->getRule()), year, ar->getEndYear()), status);
filteredRules->adoptElement(newAr.orphan(), status);
if (U_FAILURE(status)) {
return;
}
}
// check if this is a final rule
if (ar->getEndYear() == AnnualTimeZoneRule::MAX_YEAR) {
// After bot final standard and dst rules are processed,
// exit this while loop.
if (ar->getDSTSavings() == 0) {
bFinalStd = true;
} else {
bFinalDst = true;
}
}
}
done[i] = true;
}
// Set the results
initial = res_initial.orphan();
transitionRules = filteredRules.orphan();
}
void
BasicTimeZone::getOffsetFromLocal(UDate /*date*/, UTimeZoneLocalOption /*nonExistingTimeOpt*/,
UTimeZoneLocalOption /*duplicatedTimeOpt*/,
int32_t& /*rawOffset*/, int32_t& /*dstOffset*/,
UErrorCode& status) const {
if (U_FAILURE(status)) {
return;
}
status = U_UNSUPPORTED_ERROR;
}
void BasicTimeZone::getOffsetFromLocal(UDate date, int32_t nonExistingTimeOpt, int32_t duplicatedTimeOpt,
int32_t& rawOffset, int32_t& dstOffset,
UErrorCode& status) const {
getOffsetFromLocal(date, static_cast<UTimeZoneLocalOption>(nonExistingTimeOpt),
static_cast<UTimeZoneLocalOption>(duplicatedTimeOpt), rawOffset, dstOffset, status);
}
U_NAMESPACE_END
#endif /* #if !UCONFIG_NO_FORMATTING */
//eof

144
deps/icu-small/source/i18n/bocsu.cpp vendored Normal file
View File

@ -0,0 +1,144 @@
// © 2016 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html
/*
*******************************************************************************
* Copyright (C) 2001-2014, International Business Machines
* Corporation and others. All Rights Reserved.
*******************************************************************************
* file name: bocsu.cpp
* encoding: UTF-8
* tab size: 8 (not used)
* indentation:4
*
* Author: Markus W. Scherer
*
* Modification history:
* 05/18/2001 weiv Made into separate module
*/
#include "unicode/utypes.h"
#if !UCONFIG_NO_COLLATION
#include "unicode/bytestream.h"
#include "unicode/utf16.h"
#include "bocsu.h"
/*
* encode one difference value -0x10ffff..+0x10ffff in 1..4 bytes,
* preserving lexical order
*/
static uint8_t *
u_writeDiff(int32_t diff, uint8_t *p) {
if(diff>=SLOPE_REACH_NEG_1) {
if(diff<=SLOPE_REACH_POS_1) {
*p++ = static_cast<uint8_t>(SLOPE_MIDDLE + diff);
} else if(diff<=SLOPE_REACH_POS_2) {
*p++ = static_cast<uint8_t>(SLOPE_START_POS_2 + (diff / SLOPE_TAIL_COUNT));
*p++ = static_cast<uint8_t>(SLOPE_MIN + diff % SLOPE_TAIL_COUNT);
} else if(diff<=SLOPE_REACH_POS_3) {
p[2] = static_cast<uint8_t>(SLOPE_MIN + diff % SLOPE_TAIL_COUNT);
diff/=SLOPE_TAIL_COUNT;
p[1] = static_cast<uint8_t>(SLOPE_MIN + diff % SLOPE_TAIL_COUNT);
*p = static_cast<uint8_t>(SLOPE_START_POS_3 + (diff / SLOPE_TAIL_COUNT));
p+=3;
} else {
p[3] = static_cast<uint8_t>(SLOPE_MIN + diff % SLOPE_TAIL_COUNT);
diff/=SLOPE_TAIL_COUNT;
p[2] = static_cast<uint8_t>(SLOPE_MIN + diff % SLOPE_TAIL_COUNT);
diff/=SLOPE_TAIL_COUNT;
p[1] = static_cast<uint8_t>(SLOPE_MIN + diff % SLOPE_TAIL_COUNT);
*p=SLOPE_MAX;
p+=4;
}
} else {
int32_t m;
if(diff>=SLOPE_REACH_NEG_2) {
NEGDIVMOD(diff, SLOPE_TAIL_COUNT, m);
*p++ = static_cast<uint8_t>(SLOPE_START_NEG_2 + diff);
*p++ = static_cast<uint8_t>(SLOPE_MIN + m);
} else if(diff>=SLOPE_REACH_NEG_3) {
NEGDIVMOD(diff, SLOPE_TAIL_COUNT, m);
p[2] = static_cast<uint8_t>(SLOPE_MIN + m);
NEGDIVMOD(diff, SLOPE_TAIL_COUNT, m);
p[1] = static_cast<uint8_t>(SLOPE_MIN + m);
*p = static_cast<uint8_t>(SLOPE_START_NEG_3 + diff);
p+=3;
} else {
NEGDIVMOD(diff, SLOPE_TAIL_COUNT, m);
p[3] = static_cast<uint8_t>(SLOPE_MIN + m);
NEGDIVMOD(diff, SLOPE_TAIL_COUNT, m);
p[2] = static_cast<uint8_t>(SLOPE_MIN + m);
NEGDIVMOD(diff, SLOPE_TAIL_COUNT, m);
p[1] = static_cast<uint8_t>(SLOPE_MIN + m);
*p=SLOPE_MIN;
p+=4;
}
}
return p;
}
/*
* Encode the code points of a string as
* a sequence of byte-encoded differences (slope detection),
* preserving lexical order.
*
* Optimize the difference-taking for runs of Unicode text within
* small scripts:
*
* Most small scripts are allocated within aligned 128-blocks of Unicode
* code points. Lexical order is preserved if "prev" is always moved
* into the middle of such a block.
*
* Additionally, "prev" is moved from anywhere in the Unihan
* area into the middle of that area.
* Note that the identical-level run in a sort key is generated from
* NFD text - there are never Hangul characters included.
*/
U_CFUNC UChar32
u_writeIdenticalLevelRun(UChar32 prev, const char16_t *s, int32_t length, icu::ByteSink &sink) {
char scratch[64];
int32_t capacity;
int32_t i=0;
while(i<length) {
char *buffer=sink.GetAppendBuffer(1, length*2, scratch, (int32_t)sizeof(scratch), &capacity);
uint8_t *p;
// We must have capacity>=SLOPE_MAX_BYTES in case u_writeDiff() writes that much,
// but we do not want to force the sink.GetAppendBuffer() to allocate
// for a large min_capacity because we might actually only write one byte.
if(capacity<16) {
buffer=scratch;
capacity=(int32_t)sizeof(scratch);
}
p=reinterpret_cast<uint8_t *>(buffer);
uint8_t *lastSafe=p+capacity-SLOPE_MAX_BYTES;
while(i<length && p<=lastSafe) {
if(prev<0x4e00 || prev>=0xa000) {
prev=(prev&~0x7f)-SLOPE_REACH_NEG_1;
} else {
/*
* Unihan U+4e00..U+9fa5:
* double-bytes down from the upper end
*/
prev=0x9fff-SLOPE_REACH_POS_2;
}
UChar32 c;
U16_NEXT(s, i, length, c);
if(c==0xfffe) {
*p++=2; // merge separator
prev=0;
} else {
p=u_writeDiff(c-prev, p);
prev=c;
}
}
sink.Append(buffer, (int32_t)(p-reinterpret_cast<uint8_t *>(buffer)));
}
return prev;
}
#endif /* #if !UCONFIG_NO_COLLATION */

161
deps/icu-small/source/i18n/bocsu.h vendored Normal file
View File

@ -0,0 +1,161 @@
// © 2016 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html
/*
*******************************************************************************
* Copyright (C) 2001-2014, International Business Machines
* Corporation and others. All Rights Reserved.
*******************************************************************************
* file name: bocsu.h
* encoding: UTF-8
* tab size: 8 (not used)
* indentation:4
*
* Author: Markus W. Scherer
*
* Modification history:
* 05/18/2001 weiv Made into separate module
*/
#ifndef BOCSU_H
#define BOCSU_H
#include "unicode/utypes.h"
#if !UCONFIG_NO_COLLATION
U_NAMESPACE_BEGIN
class ByteSink;
U_NAMESPACE_END
/*
* "BOCSU"
* Binary Ordered Compression Scheme for Unicode
*
* Specific application:
*
* Encode a Unicode string for the identical level of a sort key.
* Restrictions:
* - byte stream (unsigned 8-bit bytes)
* - lexical order of the identical-level run must be
* the same as code point order for the string
* - avoid byte values 0, 1, 2
*
* Method: Slope Detection
* Remember the previous code point (initial 0).
* For each cp in the string, encode the difference to the previous one.
*
* With a compact encoding of differences, this yields good results for
* small scripts and UTF-like results otherwise.
*
* Encoding of differences:
* - Similar to a UTF, encoding the length of the byte sequence in the lead bytes.
* - Does not need to be friendly for decoding or random access
* (trail byte values may overlap with lead/single byte values).
* - The signedness must be encoded as the most significant part.
*
* We encode differences with few bytes if their absolute values are small.
* For correct ordering, we must treat the entire value range -10ffff..+10ffff
* in ascending order, which forbids encoding the sign and the absolute value separately.
* Instead, we split the lead byte range in the middle and encode non-negative values
* going up and negative values going down.
*
* For very small absolute values, the difference is added to a middle byte value
* for single-byte encoded differences.
* For somewhat larger absolute values, the difference is divided by the number
* of byte values available, the modulo is used for one trail byte, and the remainder
* is added to a lead byte avoiding the single-byte range.
* For large absolute values, the difference is similarly encoded in three bytes.
*
* This encoding does not use byte values 0, 1, 2, but uses all other byte values
* for lead/single bytes so that the middle range of single bytes is as large
* as possible.
* Note that the lead byte ranges overlap some, but that the sequences as a whole
* are well ordered. I.e., even if the lead byte is the same for sequences of different
* lengths, the trail bytes establish correct order.
* It would be possible to encode slightly larger ranges for each length (>1) by
* subtracting the lower bound of the range. However, that would also slow down the
* calculation.
*
* For the actual string encoding, an optimization moves the previous code point value
* to the middle of its Unicode script block to minimize the differences in
* same-script text runs.
*/
/* Do not use byte values 0, 1, 2 because they are separators in sort keys. */
#define SLOPE_MIN 3
#define SLOPE_MAX 0xff
#define SLOPE_MIDDLE 0x81
#define SLOPE_TAIL_COUNT (SLOPE_MAX-SLOPE_MIN+1)
#define SLOPE_MAX_BYTES 4
/*
* Number of lead bytes:
* 1 middle byte for 0
* 2*80=160 single bytes for !=0
* 2*42=84 for double-byte values
* 2*3=6 for 3-byte values
* 2*1=2 for 4-byte values
*
* The sum must be <=SLOPE_TAIL_COUNT.
*
* Why these numbers?
* - There should be >=128 single-byte values to cover 128-blocks
* with small scripts.
* - There should be >=20902 single/double-byte values to cover Unihan.
* - It helps CJK Extension B some if there are 3-byte values that cover
* the distance between them and Unihan.
* This also helps to jump among distant places in the BMP.
* - Four-byte values are necessary to cover the rest of Unicode.
*
* Symmetrical lead byte counts are for convenience.
* With an equal distribution of even and odd differences there is also
* no advantage to asymmetrical lead byte counts.
*/
#define SLOPE_SINGLE 80
#define SLOPE_LEAD_2 42
#define SLOPE_LEAD_3 3
#define SLOPE_LEAD_4 1
/* The difference value range for single-byters. */
#define SLOPE_REACH_POS_1 SLOPE_SINGLE
#define SLOPE_REACH_NEG_1 (-SLOPE_SINGLE)
/* The difference value range for double-byters. */
#define SLOPE_REACH_POS_2 (SLOPE_LEAD_2*SLOPE_TAIL_COUNT+(SLOPE_LEAD_2-1))
#define SLOPE_REACH_NEG_2 (-SLOPE_REACH_POS_2-1)
/* The difference value range for 3-byters. */
#define SLOPE_REACH_POS_3 (SLOPE_LEAD_3*SLOPE_TAIL_COUNT*SLOPE_TAIL_COUNT+(SLOPE_LEAD_3-1)*SLOPE_TAIL_COUNT+(SLOPE_TAIL_COUNT-1))
#define SLOPE_REACH_NEG_3 (-SLOPE_REACH_POS_3-1)
/* The lead byte start values. */
#define SLOPE_START_POS_2 (SLOPE_MIDDLE+SLOPE_SINGLE+1)
#define SLOPE_START_POS_3 (SLOPE_START_POS_2+SLOPE_LEAD_2)
#define SLOPE_START_NEG_2 (SLOPE_MIDDLE+SLOPE_REACH_NEG_1)
#define SLOPE_START_NEG_3 (SLOPE_START_NEG_2-SLOPE_LEAD_2)
/*
* Integer division and modulo with negative numerators
* yields negative modulo results and quotients that are one more than
* what we need here.
*/
#define NEGDIVMOD(n, d, m) UPRV_BLOCK_MACRO_BEGIN { \
(m)=(n)%(d); \
(n)/=(d); \
if((m)<0) { \
--(n); \
(m)+=(d); \
} \
} UPRV_BLOCK_MACRO_END
U_CFUNC UChar32
u_writeIdenticalLevelRun(UChar32 prev, const UChar *s, int32_t length, icu::ByteSink &sink);
#endif /* #if !UCONFIG_NO_COLLATION */
#endif

195
deps/icu-small/source/i18n/brktrans.cpp vendored Normal file
View File

@ -0,0 +1,195 @@
// © 2016 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html
/*
**********************************************************************
* Copyright (C) 2008-2015, International Business Machines
* Corporation and others. All Rights Reserved.
**********************************************************************
* Date Name Description
* 05/11/2008 Andy Heninger Port from Java
**********************************************************************
*/
#include <utility>
#include "unicode/utypes.h"
#if !UCONFIG_NO_TRANSLITERATION && !UCONFIG_NO_BREAK_ITERATION
#include "unicode/brkiter.h"
#include "unicode/localpointer.h"
#include "unicode/uchar.h"
#include "unicode/unifilt.h"
#include "unicode/uniset.h"
#include "brktrans.h"
#include "cmemory.h"
#include "mutex.h"
#include "uprops.h"
#include "uinvchar.h"
#include "util.h"
#include "uvectr32.h"
U_NAMESPACE_BEGIN
UOBJECT_DEFINE_RTTI_IMPLEMENTATION(BreakTransliterator)
static const char16_t SPACE = 32; // ' '
/**
* Constructs a transliterator with the default delimiters '{' and
* '}'.
*/
BreakTransliterator::BreakTransliterator(UnicodeFilter* adoptedFilter) :
Transliterator(UNICODE_STRING("Any-BreakInternal", 17), adoptedFilter),
cachedBI(nullptr), cachedBoundaries(nullptr), fInsertion(SPACE) {
}
/**
* Destructor.
*/
BreakTransliterator::~BreakTransliterator() {
}
/**
* Copy constructor.
*/
BreakTransliterator::BreakTransliterator(const BreakTransliterator& o) :
Transliterator(o), cachedBI(nullptr), cachedBoundaries(nullptr), fInsertion(o.fInsertion) {
}
/**
* Transliterator API.
*/
BreakTransliterator* BreakTransliterator::clone() const {
return new BreakTransliterator(*this);
}
/**
* Implements {@link Transliterator#handleTransliterate}.
*/
void BreakTransliterator::handleTransliterate(Replaceable& text, UTransPosition& offsets,
UBool isIncremental ) const {
UErrorCode status = U_ZERO_ERROR;
LocalPointer<BreakIterator> bi;
LocalPointer<UVector32> boundaries;
{
Mutex m;
BreakTransliterator *nonConstThis = const_cast<BreakTransliterator *>(this);
boundaries = std::move(nonConstThis->cachedBoundaries);
bi = std::move(nonConstThis->cachedBI);
}
if (bi.isNull()) {
bi.adoptInstead(BreakIterator::createWordInstance(Locale::getEnglish(), status));
}
if (boundaries.isNull()) {
boundaries.adoptInstead(new UVector32(status));
}
if (bi.isNull() || boundaries.isNull() || U_FAILURE(status)) {
return;
}
boundaries->removeAllElements();
UnicodeString sText = replaceableAsString(text);
bi->setText(sText);
bi->preceding(offsets.start);
// To make things much easier, we will stack the boundaries, and then insert at the end.
// generally, we won't need too many, since we will be filtered.
int32_t boundary;
for(boundary = bi->next(); boundary != UBRK_DONE && boundary < offsets.limit; boundary = bi->next()) {
if (boundary == 0) continue;
// HACK: Check to see that preceding item was a letter
UChar32 cp = sText.char32At(boundary-1);
int type = u_charType(cp);
//System.out.println(Integer.toString(cp,16) + " (before): " + type);
if ((U_MASK(type) & (U_GC_L_MASK | U_GC_M_MASK)) == 0) continue;
cp = sText.char32At(boundary);
type = u_charType(cp);
//System.out.println(Integer.toString(cp,16) + " (after): " + type);
if ((U_MASK(type) & (U_GC_L_MASK | U_GC_M_MASK)) == 0) continue;
boundaries->addElement(boundary, status);
// printf("Boundary at %d\n", boundary);
}
int delta = 0;
int lastBoundary = 0;
if (boundaries->size() != 0) { // if we found something, adjust
delta = boundaries->size() * fInsertion.length();
lastBoundary = boundaries->lastElementi();
// we do this from the end backwards, so that we don't have to keep updating.
while (boundaries->size() > 0) {
boundary = boundaries->popi();
text.handleReplaceBetween(boundary, boundary, fInsertion);
}
}
// Now fix up the return values
offsets.contextLimit += delta;
offsets.limit += delta;
offsets.start = isIncremental ? lastBoundary + delta : offsets.limit;
// Return break iterator & boundaries vector to the cache.
{
Mutex m;
BreakTransliterator *nonConstThis = const_cast<BreakTransliterator *>(this);
if (nonConstThis->cachedBI.isNull()) {
nonConstThis->cachedBI = std::move(bi);
}
if (nonConstThis->cachedBoundaries.isNull()) {
nonConstThis->cachedBoundaries = std::move(boundaries);
}
}
// TODO: do something with U_FAILURE(status);
// (need to look at transliterators overall, not just here.)
}
//
// getInsertion()
//
const UnicodeString &BreakTransliterator::getInsertion() const {
return fInsertion;
}
//
// setInsertion()
//
void BreakTransliterator::setInsertion(const UnicodeString &insertion) {
this->fInsertion = insertion;
}
//
// replaceableAsString Hack to let break iterators work
// on the replaceable text from transliterators.
// In practice, the only real Replaceable type that we
// will be seeing is UnicodeString, so this function
// will normally be efficient.
//
UnicodeString BreakTransliterator::replaceableAsString(Replaceable &r) {
UnicodeString s;
UnicodeString *rs = dynamic_cast<UnicodeString *>(&r);
if (rs != nullptr) {
s = *rs;
} else {
r.extractBetween(0, r.length(), s);
}
return s;
}
U_NAMESPACE_END
#endif /* #if !UCONFIG_NO_TRANSLITERATION */

104
deps/icu-small/source/i18n/brktrans.h vendored Normal file
View File

@ -0,0 +1,104 @@
// © 2016 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html
/*
**********************************************************************
* Copyright (C) 2008-2015, International Business Machines
* Corporation and others. All Rights Reserved.
**********************************************************************
* Date Name Description
* 05/11/2008 Andy Heninger Ported from Java
**********************************************************************
*/
#ifndef BRKTRANS_H
#define BRKTRANS_H
#include "unicode/utypes.h"
#if !UCONFIG_NO_TRANSLITERATION && !UCONFIG_NO_BREAK_ITERATION
#include "unicode/translit.h"
#include "unicode/localpointer.h"
U_NAMESPACE_BEGIN
class UVector32;
/**
* A transliterator that pInserts the specified characters at word breaks.
* To restrict it to particular characters, use a filter.
* TODO: this is an internal class, and only temporary.
* Remove it once we have \b notation in Transliterator.
*/
class BreakTransliterator : public Transliterator {
public:
/**
* Constructs a transliterator.
* @param adoptedFilter the filter for this transliterator.
*/
BreakTransliterator(UnicodeFilter* adoptedFilter = nullptr);
/**
* Destructor.
*/
virtual ~BreakTransliterator();
/**
* Copy constructor.
*/
BreakTransliterator(const BreakTransliterator&);
/**
* Transliterator API.
* @return A copy of the object.
*/
virtual BreakTransliterator* clone() const override;
virtual const UnicodeString &getInsertion() const;
virtual void setInsertion(const UnicodeString &insertion);
/**
* ICU "poor man's RTTI", returns a UClassID for the actual class.
*/
virtual UClassID getDynamicClassID() const override;
/**
* ICU "poor man's RTTI", returns a UClassID for this class.
*/
U_I18N_API static UClassID U_EXPORT2 getStaticClassID();
protected:
/**
* Implements {@link Transliterator#handleTransliterate}.
* @param text the buffer holding transliterated and
* untransliterated text
* @param offset the start and limit of the text, the position
* of the cursor, and the start and limit of transliteration.
* @param incremental if true, assume more text may be coming after
* pos.contextLimit. Otherwise, assume the text is complete.
*/
virtual void handleTransliterate(Replaceable& text, UTransPosition& offset,
UBool isIncremental) const override;
private:
LocalPointer<BreakIterator> cachedBI;
LocalPointer<UVector32> cachedBoundaries;
UnicodeString fInsertion;
static UnicodeString replaceableAsString(Replaceable &r);
/**
* Assignment operator.
*/
BreakTransliterator& operator=(const BreakTransliterator&);
};
U_NAMESPACE_END
#endif /* #if !UCONFIG_NO_TRANSLITERATION */
#endif

99
deps/icu-small/source/i18n/buddhcal.cpp vendored Normal file
View File

@ -0,0 +1,99 @@
// © 2016 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html
/*
*******************************************************************************
* Copyright (C) 2003-2013, International Business Machines Corporation and *
* others. All Rights Reserved. *
*******************************************************************************
*
* File BUDDHCAL.CPP
*
* Modification History:
* 05/13/2003 srl copied from gregocal.cpp
*
*/
#include "unicode/utypes.h"
#if !UCONFIG_NO_FORMATTING
#include "buddhcal.h"
#include "gregoimp.h"
#include "unicode/gregocal.h"
#include "umutex.h"
#include <float.h>
U_NAMESPACE_BEGIN
UOBJECT_DEFINE_RTTI_IMPLEMENTATION(BuddhistCalendar)
//static const int32_t kMaxEra = 0; // only 1 era
static const int32_t kBuddhistEraStart = -543; // 544 BC (Gregorian)
static const int32_t kGregorianEpoch = 1970; // used as the default value of EXTENDED_YEAR
BuddhistCalendar::BuddhistCalendar(const Locale& aLocale, UErrorCode& success)
: GregorianCalendar(aLocale, success)
{
}
BuddhistCalendar::~BuddhistCalendar()
{
}
BuddhistCalendar::BuddhistCalendar(const BuddhistCalendar& source)
: GregorianCalendar(source)
{
}
BuddhistCalendar* BuddhistCalendar::clone() const
{
return new BuddhistCalendar(*this);
}
const char *BuddhistCalendar::getType() const
{
return "buddhist";
}
int32_t BuddhistCalendar::handleGetExtendedYear(UErrorCode& status)
{
if (U_FAILURE(status)) {
return 0;
}
// EXTENDED_YEAR in BuddhistCalendar is a Gregorian year.
// The default value of EXTENDED_YEAR is 1970 (Buddhist 2513)
if (newerField(UCAL_EXTENDED_YEAR, UCAL_YEAR) == UCAL_EXTENDED_YEAR) {
return internalGet(UCAL_EXTENDED_YEAR, kGregorianEpoch);
}
// extended year is a gregorian year, where 1 = 1AD, 0 = 1BC, -1 = 2BC, etc
int32_t year = internalGet(UCAL_YEAR, kGregorianEpoch - kBuddhistEraStart);
if (uprv_add32_overflow(year, kBuddhistEraStart, &year)) {
status = U_ILLEGAL_ARGUMENT_ERROR;
return 0;
}
return year;
}
void BuddhistCalendar::handleComputeFields(int32_t julianDay, UErrorCode& status)
{
GregorianCalendar::handleComputeFields(julianDay, status);
int32_t y = internalGet(UCAL_EXTENDED_YEAR) - kBuddhistEraStart;
internalSet(UCAL_ERA, 0);
internalSet(UCAL_YEAR, y);
}
int32_t BuddhistCalendar::handleGetLimit(UCalendarDateFields field, ELimitType limitType) const
{
if(field == UCAL_ERA) {
return BE;
}
return GregorianCalendar::handleGetLimit(field,limitType);
}
IMPL_SYSTEM_DEFAULT_CENTURY(BuddhistCalendar, "@calendar=buddhist")
U_NAMESPACE_END
#endif

166
deps/icu-small/source/i18n/buddhcal.h vendored Normal file
View File

@ -0,0 +1,166 @@
// © 2016 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html
/*
********************************************************************************
* Copyright (C) 2003-2013, International Business Machines Corporation
* and others. All Rights Reserved.
********************************************************************************
*
* File BUDDHCAL.H
*
* Modification History:
*
* Date Name Description
* 05/13/2003 srl copied from gregocal.h
********************************************************************************
*/
#ifndef BUDDHCAL_H
#define BUDDHCAL_H
#include "unicode/utypes.h"
#if !UCONFIG_NO_FORMATTING
#include "unicode/calendar.h"
#include "unicode/gregocal.h"
U_NAMESPACE_BEGIN
/**
* Concrete class which provides the Buddhist calendar.
* <P>
* <code>BuddhistCalendar</code> is a subclass of <code>GregorianCalendar</code>
* that numbers years since the birth of the Buddha. This is the civil calendar
* in some predominantly Buddhist countries such as Thailand, and it is used for
* religious purposes elsewhere.
* <p>
* The Buddhist calendar is identical to the Gregorian calendar in all respects
* except for the year and era. Years are numbered since the birth of the
* Buddha in 543 BC (Gregorian), so that 1 AD (Gregorian) is equivalent to 544
* BE (Buddhist Era) and 1998 AD is 2541 BE.
* <p>
* The Buddhist Calendar has only one allowable era: <code>BE</code>. If the
* calendar is not in lenient mode (see <code>setLenient</code>), dates before
* 1/1/1 BE are rejected as an illegal argument.
* <p>
* @internal
*/
class BuddhistCalendar : public GregorianCalendar {
public:
/**
* Useful constants for BuddhistCalendar. Only one Era.
* @internal
*/
enum EEras {
BE
};
/**
* Constructs a BuddhistCalendar based on the current time in the default time zone
* with the given locale.
*
* @param aLocale The given locale.
* @param success Indicates the status of BuddhistCalendar object construction.
* Returns U_ZERO_ERROR if constructed successfully.
* @internal
*/
BuddhistCalendar(const Locale& aLocale, UErrorCode& success);
/**
* Destructor
* @internal
*/
virtual ~BuddhistCalendar();
/**
* Copy constructor
* @param source the object to be copied.
* @internal
*/
BuddhistCalendar(const BuddhistCalendar& source);
/**
* Create and return a polymorphic copy of this calendar.
* @return return a polymorphic copy of this calendar.
* @internal
*/
virtual BuddhistCalendar* clone() const override;
public:
/**
* Override Calendar Returns a unique class ID POLYMORPHICALLY. Pure virtual
* override. This method is to implement a simple version of RTTI, since not all C++
* compilers support genuine RTTI. Polymorphic operator==() and clone() methods call
* this method.
*
* @return The class ID for this object. All objects of a given class have the
* same class ID. Objects of other classes have different class IDs.
* @internal
*/
virtual UClassID getDynamicClassID() const override;
/**
* Return the class ID for this class. This is useful only for comparing to a return
* value from getDynamicClassID(). For example:
*
* Base* polymorphic_pointer = createPolymorphicObject();
* if (polymorphic_pointer->getDynamicClassID() ==
* Derived::getStaticClassID()) ...
*
* @return The class ID for all objects of this class.
* @internal
*/
U_I18N_API static UClassID U_EXPORT2 getStaticClassID();
/**
* return the calendar type, "buddhist".
*
* @return calendar type
* @internal
*/
virtual const char * getType() const override;
private:
BuddhistCalendar(); // default constructor not implemented
protected:
/**
* Return the extended year defined by the current fields. This will
* use the UCAL_EXTENDED_YEAR field or the UCAL_YEAR and supra-year fields (such
* as UCAL_ERA) specific to the calendar system, depending on which set of
* fields is newer.
* @param status
* @return the extended year
* @internal
*/
virtual int32_t handleGetExtendedYear(UErrorCode& status) override;
/**
* Subclasses may override this method to compute several fields
* specific to each calendar system.
* @internal
*/
virtual void handleComputeFields(int32_t julianDay, UErrorCode& status) override;
/**
* Subclass API for defining limits of different types.
* @param field one of the field numbers
* @param limitType one of <code>MINIMUM</code>, <code>GREATEST_MINIMUM</code>,
* <code>LEAST_MAXIMUM</code>, or <code>MAXIMUM</code>
* @internal
*/
virtual int32_t handleGetLimit(UCalendarDateFields field, ELimitType limitType) const override;
virtual bool isEra0CountingBackward() const override { return false; }
DECLARE_OVERRIDE_SYSTEM_DEFAULT_CENTURY
};
U_NAMESPACE_END
#endif /* #if !UCONFIG_NO_FORMATTING */
#endif // _GREGOCAL
//eof

4283
deps/icu-small/source/i18n/calendar.cpp vendored Normal file
View File

File diff suppressed because it is too large Load Diff

193
deps/icu-small/source/i18n/casetrn.cpp vendored Normal file
View File

@ -0,0 +1,193 @@
// © 2016 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html
/*
*******************************************************************************
*
* Copyright (C) 2001-2011, International Business Machines
* Corporation and others. All Rights Reserved.
*
*******************************************************************************
* file name: casetrn.cpp
* encoding: UTF-8
* tab size: 8 (not used)
* indentation:4
*
* created on: 2004sep03
* created by: Markus W. Scherer
*
* Implementation class for lower-/upper-/title-casing transliterators.
*/
#include "unicode/utypes.h"
#if !UCONFIG_NO_TRANSLITERATION
#include "unicode/uchar.h"
#include "unicode/ustring.h"
#include "unicode/utf.h"
#include "unicode/utf16.h"
#include "tolowtrn.h"
#include "ucase.h"
#include "cpputils.h"
/* case context iterator using a Replaceable */
U_CFUNC UChar32 U_CALLCONV
utrans_rep_caseContextIterator(void *context, int8_t dir)
{
U_NAMESPACE_USE
UCaseContext *csc=(UCaseContext *)context;
Replaceable *rep=(Replaceable *)csc->p;
UChar32 c;
if(dir<0) {
/* reset for backward iteration */
csc->index=csc->cpStart;
csc->dir=dir;
} else if(dir>0) {
/* reset for forward iteration */
csc->index=csc->cpLimit;
csc->dir=dir;
} else {
/* continue current iteration direction */
dir=csc->dir;
}
// automatically adjust start and limit if the Replaceable disagrees
// with the original values
if(dir<0) {
if(csc->start<csc->index) {
c=rep->char32At(csc->index-1);
if(c<0) {
csc->start=csc->index;
} else {
csc->index-=U16_LENGTH(c);
return c;
}
}
} else {
// detect, and store in csc->b1, if we hit the limit
if(csc->index<csc->limit) {
c=rep->char32At(csc->index);
if(c<0) {
csc->limit=csc->index;
csc->b1=true;
} else {
csc->index+=U16_LENGTH(c);
return c;
}
} else {
csc->b1=true;
}
}
return U_SENTINEL;
}
U_NAMESPACE_BEGIN
UOBJECT_DEFINE_ABSTRACT_RTTI_IMPLEMENTATION(CaseMapTransliterator)
/**
* Constructs a transliterator.
*/
CaseMapTransliterator::CaseMapTransliterator(const UnicodeString &id, UCaseMapFull *map) :
Transliterator(id, nullptr),
fMap(map)
{
// TODO test incremental mode with context-sensitive text (e.g. greek sigma)
// TODO need to call setMaximumContextLength()?!
}
/**
* Destructor.
*/
CaseMapTransliterator::~CaseMapTransliterator() {
}
/**
* Copy constructor.
*/
CaseMapTransliterator::CaseMapTransliterator(const CaseMapTransliterator& o) :
Transliterator(o),
fMap(o.fMap)
{
}
/**
* Assignment operator.
*/
/*CaseMapTransliterator& CaseMapTransliterator::operator=(const CaseMapTransliterator& o) {
Transliterator::operator=(o);
fMap = o.fMap;
return *this;
}*/
/**
* Transliterator API.
*/
/*CaseMapTransliterator* CaseMapTransliterator::clone() const {
return new CaseMapTransliterator(*this);
}*/
/**
* Implements {@link Transliterator#handleTransliterate}.
*/
void CaseMapTransliterator::handleTransliterate(Replaceable& text,
UTransPosition& offsets,
UBool isIncremental) const
{
if (offsets.start >= offsets.limit) {
return;
}
UCaseContext csc;
uprv_memset(&csc, 0, sizeof(csc));
csc.p = &text;
csc.start = offsets.contextStart;
csc.limit = offsets.contextLimit;
UnicodeString tmp;
const char16_t *s;
UChar32 c;
int32_t textPos, delta, result;
for(textPos=offsets.start; textPos<offsets.limit;) {
csc.cpStart=textPos;
c=text.char32At(textPos);
csc.cpLimit=textPos+=U16_LENGTH(c);
result=fMap(c, utrans_rep_caseContextIterator, &csc, &s, UCASE_LOC_ROOT);
if(csc.b1 && isIncremental) {
// fMap() tried to look beyond the context limit
// wait for more input
offsets.start=csc.cpStart;
return;
}
if(result>=0) {
// replace the current code point with its full case mapping result
// see UCASE_MAX_STRING_LENGTH
if(result<=UCASE_MAX_STRING_LENGTH) {
// string s[result]
tmp.setTo(false, s, result);
delta=result-U16_LENGTH(c);
} else {
// single code point
tmp.setTo(result);
delta=tmp.length()-U16_LENGTH(c);
}
text.handleReplaceBetween(csc.cpStart, textPos, tmp);
if(delta!=0) {
textPos+=delta;
csc.limit=offsets.contextLimit+=delta;
offsets.limit+=delta;
}
}
}
offsets.start=textPos;
}
U_NAMESPACE_END
#endif /* #if !UCONFIG_NO_TRANSLITERATION */

105
deps/icu-small/source/i18n/casetrn.h vendored Normal file
View File

@ -0,0 +1,105 @@
// © 2016 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html
/*
*******************************************************************************
*
* Copyright (C) 2001-2008, International Business Machines
* Corporation and others. All Rights Reserved.
*
*******************************************************************************
* file name: casetrn.h
* encoding: UTF-8
* tab size: 8 (not used)
* indentation:4
*
* created on: 2004sep03
* created by: Markus W. Scherer
*
* Implementation class for lower-/upper-/title-casing transliterators.
*/
#ifndef __CASETRN_H__
#define __CASETRN_H__
#include "unicode/utypes.h"
#if !UCONFIG_NO_TRANSLITERATION
#include "unicode/translit.h"
#include "ucase.h"
U_NAMESPACE_BEGIN
/**
* A transliterator that performs locale-sensitive
* case mapping.
*/
class CaseMapTransliterator : public Transliterator {
public:
/**
* Constructs a transliterator.
* @param loc the given locale.
* @param id the transliterator ID.
* @param map the full case mapping function (see ucase.h)
*/
CaseMapTransliterator(const UnicodeString &id, UCaseMapFull *map);
/**
* Destructor.
*/
virtual ~CaseMapTransliterator();
/**
* Copy constructor.
*/
CaseMapTransliterator(const CaseMapTransliterator&);
/**
* Transliterator API.
* @return a copy of the object.
*/
virtual CaseMapTransliterator* clone() const override = 0;
/**
* ICU "poor man's RTTI", returns a UClassID for the actual class.
*/
//virtual UClassID getDynamicClassID() const;
/**
* ICU "poor man's RTTI", returns a UClassID for this class.
*/
U_I18N_API static UClassID U_EXPORT2 getStaticClassID();
protected:
/**
* Implements {@link Transliterator#handleTransliterate}.
* @param text the buffer holding transliterated and
* untransliterated text
* @param offset the start and limit of the text, the position
* of the cursor, and the start and limit of transliteration.
* @param incremental if true, assume more text may be coming after
* pos.contextLimit. Otherwise, assume the text is complete.
*/
virtual void handleTransliterate(Replaceable& text,
UTransPosition& offsets,
UBool isIncremental) const override;
UCaseMapFull *fMap;
private:
/**
* Assignment operator.
*/
CaseMapTransliterator& operator=(const CaseMapTransliterator&);
};
U_NAMESPACE_END
/** case context iterator using a Replaceable. This must be a C function because it is a callback. */
U_CFUNC UChar32 U_CALLCONV
utrans_rep_caseContextIterator(void *context, int8_t dir);
#endif /* #if !UCONFIG_NO_TRANSLITERATION */
#endif

149
deps/icu-small/source/i18n/cecal.cpp vendored Normal file
View File

@ -0,0 +1,149 @@
// © 2016 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html
/*
*******************************************************************************
* Copyright (C) 2003 - 2009, International Business Machines Corporation and *
* others. All Rights Reserved. *
*******************************************************************************
*/
#include "unicode/utypes.h"
#if !UCONFIG_NO_FORMATTING
#include "cecal.h"
#include "gregoimp.h" //Math
#include "cstring.h"
U_NAMESPACE_BEGIN
static const int32_t LIMITS[UCAL_FIELD_COUNT][4] = {
// Minimum Greatest Least Maximum
// Minimum Maximum
{ 0, 0, 1, 1}, // ERA
{ 1, 1, 5000000, 5000000}, // YEAR
{ 0, 0, 12, 12}, // MONTH
{ 1, 1, 52, 53}, // WEEK_OF_YEAR
{/*N/A*/-1,/*N/A*/-1,/*N/A*/-1,/*N/A*/-1}, // WEEK_OF_MONTH
{ 1, 1, 5, 30}, // DAY_OF_MONTH
{ 1, 1, 365, 366}, // DAY_OF_YEAR
{/*N/A*/-1,/*N/A*/-1,/*N/A*/-1,/*N/A*/-1}, // DAY_OF_WEEK
{ -1, -1, 1, 5}, // DAY_OF_WEEK_IN_MONTH
{/*N/A*/-1,/*N/A*/-1,/*N/A*/-1,/*N/A*/-1}, // AM_PM
{/*N/A*/-1,/*N/A*/-1,/*N/A*/-1,/*N/A*/-1}, // HOUR
{/*N/A*/-1,/*N/A*/-1,/*N/A*/-1,/*N/A*/-1}, // HOUR_OF_DAY
{/*N/A*/-1,/*N/A*/-1,/*N/A*/-1,/*N/A*/-1}, // MINUTE
{/*N/A*/-1,/*N/A*/-1,/*N/A*/-1,/*N/A*/-1}, // SECOND
{/*N/A*/-1,/*N/A*/-1,/*N/A*/-1,/*N/A*/-1}, // MILLISECOND
{/*N/A*/-1,/*N/A*/-1,/*N/A*/-1,/*N/A*/-1}, // ZONE_OFFSET
{/*N/A*/-1,/*N/A*/-1,/*N/A*/-1,/*N/A*/-1}, // DST_OFFSET
{ -5000000, -5000000, 5000000, 5000000}, // YEAR_WOY
{/*N/A*/-1,/*N/A*/-1,/*N/A*/-1,/*N/A*/-1}, // DOW_LOCAL
{ -5000000, -5000000, 5000000, 5000000}, // EXTENDED_YEAR
{/*N/A*/-1,/*N/A*/-1,/*N/A*/-1,/*N/A*/-1}, // JULIAN_DAY
{/*N/A*/-1,/*N/A*/-1,/*N/A*/-1,/*N/A*/-1}, // MILLISECONDS_IN_DAY
{/*N/A*/-1,/*N/A*/-1,/*N/A*/-1,/*N/A*/-1}, // IS_LEAP_MONTH
{ 0, 0, 12, 12}, // ORDINAL_MONTH
};
//-------------------------------------------------------------------------
// Constructors...
//-------------------------------------------------------------------------
CECalendar::CECalendar(const Locale& aLocale, UErrorCode& success)
: Calendar(TimeZone::forLocaleOrDefault(aLocale), aLocale, success)
{
}
CECalendar::CECalendar (const CECalendar& other)
: Calendar(other)
{
}
CECalendar::~CECalendar()
{
}
//-------------------------------------------------------------------------
// Calendar framework
//-------------------------------------------------------------------------
int64_t
CECalendar::handleComputeMonthStart(int32_t eyear,int32_t emonth, UBool /*useMonth*/, UErrorCode& /*status*/) const
{
int64_t year64 = eyear;
// handle month > 12, < 0 (e.g. from add/set)
if ( emonth >= 0 ) {
year64 += emonth/13;
emonth %= 13;
} else {
++emonth;
year64 += emonth/13 - 1;
emonth = emonth%13 + 12;
}
return (
getJDEpochOffset() // difference from Julian epoch to 1,1,1
+ 365LL * year64 // number of days from years
+ ClockMath::floorDivideInt64(year64, 4LL) // extra day of leap year
+ 30 * emonth // number of days from months (months are 0-based)
- 1 // number of days for present month (1 based)
);
}
int32_t
CECalendar::handleGetLimit(UCalendarDateFields field, ELimitType limitType) const
{
return LIMITS[field][limitType];
}
//-------------------------------------------------------------------------
// Calendar system Conversion methods...
//-------------------------------------------------------------------------
void
CECalendar::jdToCE(int32_t julianDay, int32_t jdEpochOffset, int32_t& year, int32_t& month, int32_t& day, UErrorCode& status)
{
int32_t c4; // number of 4 year cycle (1461 days)
int32_t r4; // remainder of 4 year cycle, always positive
if (uprv_add32_overflow(julianDay, -jdEpochOffset, &julianDay)) {
status = U_ILLEGAL_ARGUMENT_ERROR;
return;
}
c4 = ClockMath::floorDivide(julianDay, 1461, &r4);
year = 4 * c4 + (r4/365 - r4/1460); // 4 * <number of 4year cycle> + <years within the last cycle>
int32_t doy = (r4 == 1460) ? 365 : (r4 % 365); // days in present year
month = doy / 30; // 30 -> Coptic/Ethiopic month length up to 12th month
day = (doy % 30) + 1; // 1-based days in a month
}
static const char* kMonthCode13 = "M13";
const char* CECalendar::getTemporalMonthCode(UErrorCode& status) const {
if (get(UCAL_MONTH, status) == 12) {
return kMonthCode13;
}
return Calendar::getTemporalMonthCode(status);
}
void
CECalendar::setTemporalMonthCode(const char* code, UErrorCode& status) {
if (U_FAILURE(status)) {
return;
}
if (uprv_strcmp(code, kMonthCode13) == 0) {
set(UCAL_MONTH, 12);
set(UCAL_IS_LEAP_MONTH, 0);
return;
}
Calendar::setTemporalMonthCode(code, status);
}
U_NAMESPACE_END
#endif /* #if !UCONFIG_NO_FORMATTING */
//eof

129
deps/icu-small/source/i18n/cecal.h vendored Normal file
View File

@ -0,0 +1,129 @@
// © 2016 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html
/*
*******************************************************************************
* Copyright (C) 2003 - 2008, International Business Machines Corporation and *
* others. All Rights Reserved. *
*******************************************************************************
*/
#ifndef CECAL_H
#define CECAL_H
#include "unicode/utypes.h"
#if !UCONFIG_NO_FORMATTING
#include "unicode/calendar.h"
U_NAMESPACE_BEGIN
/**
* Base class for EthiopicCalendar and CopticCalendar.
* @internal
*/
class U_I18N_API CECalendar : public Calendar {
public:
/**
* Gets The Temporal monthCode value corresponding to the month for the date.
* The value is a string identifier that starts with the literal grapheme
* "M" followed by two graphemes representing the zero-padded month number
* of the current month in a normal (non-leap) year. For the short thirteen
* month in each year in the CECalendar, the value is "M13".
*
* @param status ICU Error Code
* @return One of 13 possible strings in {"M01".. "M12", "M13"}.
* @draft ICU 73
*/
virtual const char* getTemporalMonthCode(UErrorCode& status) const override;
/**
* Sets The Temporal monthCode which is a string identifier that starts
* with the literal grapheme "M" followed by two graphemes representing
* the zero-padded month number of the current month in a normal
* (non-leap) year. For CECalendar calendar, the values
* are "M01" .. "M13" while the "M13" is represent the short thirteen month
* in each year.
*
* @param temporalMonth The value to be set for temporal monthCode.
* @param status ICU Error Code
*
* @draft ICU 73
*/
virtual void setTemporalMonthCode(const char* code, UErrorCode& status) override;
protected:
//-------------------------------------------------------------------------
// Constructors...
//-------------------------------------------------------------------------
/**
* Constructs a CECalendar based on the current time in the default time zone
* with the given locale with the Julian epoch offiset
*
* @param aLocale The given locale.
* @param success Indicates the status of CECalendar object construction.
* Returns U_ZERO_ERROR if constructed successfully.
* @internal
*/
CECalendar(const Locale& aLocale, UErrorCode& success);
/**
* Copy Constructor
* @internal
*/
CECalendar (const CECalendar& other);
/**
* Destructor.
* @internal
*/
virtual ~CECalendar();
protected:
//-------------------------------------------------------------------------
// Calendar framework
//-------------------------------------------------------------------------
/**
* Return JD of start of given month/extended year
* @internal
*/
virtual int64_t handleComputeMonthStart(int32_t eyear, int32_t month, UBool useMonth, UErrorCode& status) const override;
/**
* Calculate the limit for a specified type of limit and field
* @internal
*/
virtual int32_t handleGetLimit(UCalendarDateFields field, ELimitType limitType) const override;
protected:
/**
* The Coptic and Ethiopic calendars differ only in their epochs.
* This method must be implemented by CECalendar subclasses to
* return the date offset from Julian
* @internal
*/
virtual int32_t getJDEpochOffset() const = 0;
/**
* Convert a Julian day to an Coptic/Ethiopic year, month and day
*
* @param julianDay the Julian day
* @param jdEpochOffset the epoch offset from Julian epoch
* @param year receives the extended year
* @param month receives the month
* @param date receives the day
* @internal
*/
static void jdToCE(int32_t julianDay, int32_t jdEpochOffset,
int32_t& year, int32_t& month, int32_t& day, UErrorCode& status);
};
U_NAMESPACE_END
#endif /* #if !UCONFIG_NO_FORMATTING */
#endif
//eof

1219
deps/icu-small/source/i18n/chnsecal.cpp vendored Normal file
View File

File diff suppressed because it is too large Load Diff

291
deps/icu-small/source/i18n/chnsecal.h vendored Normal file
View File

@ -0,0 +1,291 @@
// © 2016 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html
/*
*****************************************************************************
* Copyright (C) 2007-2013, International Business Machines Corporation
* and others. All Rights Reserved.
*****************************************************************************
*
* File CHNSECAL.H
*
* Modification History:
*
* Date Name Description
* 9/18/2007 ajmacher ported from java ChineseCalendar
*****************************************************************************
*/
#ifndef CHNSECAL_H
#define CHNSECAL_H
#include "unicode/utypes.h"
#if !UCONFIG_NO_FORMATTING
#include "unicode/calendar.h"
#include "unicode/timezone.h"
U_NAMESPACE_BEGIN
class CalendarCache;
/**
* <code>ChineseCalendar</code> is a concrete subclass of {@link Calendar}
* that implements a traditional Chinese calendar. The traditional Chinese
* calendar is a lunisolar calendar: Each month starts on a new moon, and
* the months are numbered according to solar events, specifically, to
* guarantee that month 11 always contains the winter solstice. In order
* to accomplish this, leap months are inserted in certain years. Leap
* months are numbered the same as the month they follow. The decision of
* which month is a leap month depends on the relative movements of the sun
* and moon.
*
* <p>This class defines one addition field beyond those defined by
* <code>Calendar</code>: The <code>IS_LEAP_MONTH</code> field takes the
* value of 0 for normal months, or 1 for leap months.
*
* <p>All astronomical computations are performed with respect to a time
* zone of GMT+8:00 and a longitude of 120 degrees east. Although some
* calendars implement a historically more accurate convention of using
* Beijing's local longitude (116 degrees 25 minutes east) and time zone
* (GMT+7:45:40) for dates before 1929, we do not implement this here.
*
* <p>Years are counted in two different ways in the Chinese calendar. The
* first method is by sequential numbering from the 61st year of the reign
* of Huang Di, 2637 BCE, which is designated year 1 on the Chinese
* calendar. The second method uses 60-year cycles from the same starting
* point, which is designated year 1 of cycle 1. In this class, the
* <code>EXTENDED_YEAR</code> field contains the sequential year count.
* The <code>ERA</code> field contains the cycle number, and the
* <code>YEAR</code> field contains the year of the cycle, a value between
* 1 and 60.
*
* <p>There is some variation in what is considered the starting point of
* the calendar, with some sources starting in the first year of the reign
* of Huang Di, rather than the 61st. This gives continuous year numbers
* 60 years greater and cycle numbers one greater than what this class
* implements.
*
* <p>Because <code>ChineseCalendar</code> defines an additional field and
* redefines the way the <code>ERA</code> field is used, it requires a new
* format class, <code>ChineseDateFormat</code>. As always, use the
* methods <code>DateFormat.getXxxInstance(Calendar cal,...)</code> to
* obtain a formatter for this calendar.
*
* <p>References:<ul>
*
* <li>Dershowitz and Reingold, <i>Calendrical Calculations</i>,
* Cambridge University Press, 1997</li>
*
* <li>The <a href="http://www.tondering.dk/claus/calendar.html">
* Calendar FAQ</a></li>
*
* </ul>
*
* <p>
* This class should only be subclassed to implement variants of the Chinese lunar calendar.</p>
* <p>
* ChineseCalendar usually should be instantiated using
* {@link com.ibm.icu.util.Calendar#getInstance(ULocale)} passing in a <code>ULocale</code>
* with the tag <code>"@calendar=chinese"</code>.</p>
*
* @see com.ibm.icu.text.ChineseDateFormat
* @see com.ibm.icu.util.Calendar
* @author Alan Liu
* @internal
*/
class U_I18N_API ChineseCalendar : public Calendar {
public:
//-------------------------------------------------------------------------
// Constructors...
//-------------------------------------------------------------------------
/**
* Constructs a ChineseCalendar based on the current time in the default time zone
* with the given locale.
*
* @param aLocale The given locale.
* @param success Indicates the status of ChineseCalendar object construction.
* Returns U_ZERO_ERROR if constructed successfully.
* @internal
*/
ChineseCalendar(const Locale& aLocale, UErrorCode &success);
/**
* Returns true if the date is in a leap year.
*
* @param status ICU Error Code
* @return True if the date in the fields is in a Temporal proposal
* defined leap year. False otherwise.
*/
virtual bool inTemporalLeapYear(UErrorCode &status) const override;
/**
* Gets The Temporal monthCode value corresponding to the month for the date.
* The value is a string identifier that starts with the literal grapheme
* "M" followed by two graphemes representing the zero-padded month number
* of the current month in a normal (non-leap) year and suffixed by an
* optional literal grapheme "L" if this is a leap month in a lunisolar
* calendar. For Chinese calendars (including Dangi), the values are
* "M01" .. "M12" for non-leap year, and "M01" .. "M12" with one of
* "M01L" .. "M12L" for leap year.
*
* @param status ICU Error Code
* @return One of 24 possible strings in
* {"M01" .. "M12", "M01L" .. "M12L"}.
* @draft ICU 73
*/
virtual const char* getTemporalMonthCode(UErrorCode &status) const override;
/**
* Sets The Temporal monthCode which is a string identifier that starts
* with the literal grapheme "M" followed by two graphemes representing
* the zero-padded month number of the current month in a normal
* (non-leap) year and suffixed by an optional literal grapheme "L" if this
* is a leap month in a lunisolar calendar. For Chinese calendars, the values
* are "M01" .. "M12" for non-leap years, and "M01" .. "M12" plus one in
* "M01L" .. "M12L" for leap year.
*
* @param temporalMonth The value to be set for temporal monthCode. One of
* 24 possible strings in {"M01" .. "M12", "M01L" .. "M12L"}.
* @param status ICU Error Code
*
* @draft ICU 73
*/
virtual void setTemporalMonthCode(const char* code, UErrorCode& status) override;
public:
/**
* Copy Constructor
* @internal
*/
ChineseCalendar(const ChineseCalendar& other);
/**
* Destructor.
* @internal
*/
virtual ~ChineseCalendar();
// clone
virtual ChineseCalendar* clone() const override;
private:
//-------------------------------------------------------------------------
// Internal data....
//-------------------------------------------------------------------------
// There is a leap month between the Winter Solstice before and after the
// current date.This is different from leap year because in some year, such as
// 1813 and 2033, the leap month is after the Winter Solstice of that year. So
// this value could be false for a date prior to the Winter Solstice of that
// year but that year still has a leap month and therefor is a leap year.
UBool hasLeapMonthBetweenWinterSolstices;
//----------------------------------------------------------------------
// Calendar framework
//----------------------------------------------------------------------
protected:
virtual int32_t handleGetLimit(UCalendarDateFields field, ELimitType limitType) const override;
virtual int32_t handleGetMonthLength(int32_t extendedYear, int32_t month, UErrorCode& status) const override;
virtual int64_t handleComputeMonthStart(int32_t eyear, int32_t month, UBool useMonth, UErrorCode& status) const override;
virtual int32_t handleGetExtendedYear(UErrorCode& status) override;
virtual void handleComputeFields(int32_t julianDay, UErrorCode &status) override;
virtual const UFieldResolutionTable* getFieldResolutionTable() const override;
private:
int32_t handleGetMonthLengthWithLeap(int32_t extendedYear, int32_t month, bool isLeap, UErrorCode& status) const;
int64_t handleComputeMonthStartWithLeap(int32_t eyear, int32_t month, bool isLeap, UErrorCode& status) const;
public:
virtual void add(UCalendarDateFields field, int32_t amount, UErrorCode &status) override;
virtual void add(EDateFields field, int32_t amount, UErrorCode &status) override;
virtual void roll(UCalendarDateFields field, int32_t amount, UErrorCode &status) override;
virtual void roll(EDateFields field, int32_t amount, UErrorCode &status) override;
/**
* @return The related Gregorian year; will be obtained by modifying the value
* obtained by get from UCAL_EXTENDED_YEAR field
* @internal
*/
virtual int32_t getRelatedYear(UErrorCode &status) const override;
/**
* @param year The related Gregorian year to set; will be modified as necessary then
* set in UCAL_EXTENDED_YEAR field
* @internal
*/
virtual void setRelatedYear(int32_t year) override;
//----------------------------------------------------------------------
// Internal methods & astronomical calculations
//----------------------------------------------------------------------
private:
static const UFieldResolutionTable CHINESE_DATE_PRECEDENCE[];
virtual void offsetMonth(int32_t newMoon, int32_t dom, int32_t delta, UErrorCode& status);
// UObject stuff
public:
/**
* @return The class ID for this object. All objects of a given class have the
* same class ID. Objects of other classes have different class IDs.
* @internal
*/
virtual UClassID getDynamicClassID() const override;
/**
* Return the class ID for this class. This is useful only for comparing to a return
* value from getDynamicClassID(). For example:
*
* Base* polymorphic_pointer = createPolymorphicObject();
* if (polymorphic_pointer->getDynamicClassID() ==
* Derived::getStaticClassID()) ...
*
* @return The class ID for all objects of this class.
* @internal
*/
static UClassID U_EXPORT2 getStaticClassID();
/**
* return the calendar type, "chinese".
*
* @return calendar type
* @internal
*/
virtual const char * getType() const override;
virtual int32_t getActualMaximum(UCalendarDateFields field, UErrorCode& status) const override;
struct Setting {
int32_t epochYear;
const TimeZone* zoneAstroCalc;
CalendarCache** winterSolsticeCache;
CalendarCache** newYearCache;
};
protected:
virtual Setting getSetting(UErrorCode& status) const;
virtual int32_t internalGetMonth(int32_t defaultValue, UErrorCode& status) const override;
virtual int32_t internalGetMonth(UErrorCode& status) const override;
protected:
DECLARE_OVERRIDE_SYSTEM_DEFAULT_CENTURY
private: // default century stuff.
ChineseCalendar() = delete; // default constructor not implemented
#ifdef __CalendarTest__
friend void CalendarTest::TestChineseCalendarComputeMonthStart();
#endif
};
U_NAMESPACE_END
#endif
#endif

577
deps/icu-small/source/i18n/choicfmt.cpp vendored Normal file
View File

@ -0,0 +1,577 @@
// © 2016 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html
/*
*******************************************************************************
* Copyright (C) 1997-2013, International Business Machines Corporation and *
* others. All Rights Reserved. *
*******************************************************************************
*
* File CHOICFMT.CPP
*
* Modification History:
*
* Date Name Description
* 02/19/97 aliu Converted from java.
* 03/20/97 helena Finished first cut of implementation and got rid
* of nextDouble/previousDouble and replaced with
* boolean array.
* 4/10/97 aliu Clean up. Modified to work on AIX.
* 06/04/97 helena Fixed applyPattern(), toPattern() and not to include
* wchar.h.
* 07/09/97 helena Made ParsePosition into a class.
* 08/06/97 nos removed overloaded constructor, fixed 'format(array)'
* 07/22/98 stephen JDK 1.2 Sync - removed UBool array (doubleFlags)
* 02/22/99 stephen Removed character literals for EBCDIC safety
********************************************************************************
*/
#include "unicode/utypes.h"
#if !UCONFIG_NO_FORMATTING
#include "unicode/choicfmt.h"
#include "unicode/numfmt.h"
#include "unicode/locid.h"
#include "cpputils.h"
#include "cstring.h"
#include "messageimpl.h"
#include "putilimp.h"
#include "uassert.h"
#include <stdio.h>
#include <float.h>
// *****************************************************************************
// class ChoiceFormat
// *****************************************************************************
U_NAMESPACE_BEGIN
UOBJECT_DEFINE_RTTI_IMPLEMENTATION(ChoiceFormat)
// Special characters used by ChoiceFormat. There are two characters
// used interchangeably to indicate <=. Either is parsed, but only
// LESS_EQUAL is generated by toPattern().
#define SINGLE_QUOTE ((char16_t)0x0027) /*'*/
#define LESS_THAN ((char16_t)0x003C) /*<*/
#define LESS_EQUAL ((char16_t)0x0023) /*#*/
#define LESS_EQUAL2 ((char16_t)0x2264)
#define VERTICAL_BAR ((char16_t)0x007C) /*|*/
#define MINUS ((char16_t)0x002D) /*-*/
static const char16_t LEFT_CURLY_BRACE = 0x7B; /*{*/
static const char16_t RIGHT_CURLY_BRACE = 0x7D; /*}*/
#ifdef INFINITY
#undef INFINITY
#endif
#define INFINITY ((char16_t)0x221E)
//static const char16_t gPositiveInfinity[] = {INFINITY, 0};
//static const char16_t gNegativeInfinity[] = {MINUS, INFINITY, 0};
#define POSITIVE_INF_STRLEN 1
#define NEGATIVE_INF_STRLEN 2
// -------------------------------------
// Creates a ChoiceFormat instance based on the pattern.
ChoiceFormat::ChoiceFormat(const UnicodeString& newPattern,
UErrorCode& status)
: constructorErrorCode(status),
msgPattern(status)
{
applyPattern(newPattern, status);
}
// -------------------------------------
// Creates a ChoiceFormat instance with the limit array and
// format strings for each limit.
ChoiceFormat::ChoiceFormat(const double* limits,
const UnicodeString* formats,
int32_t cnt )
: constructorErrorCode(U_ZERO_ERROR),
msgPattern(constructorErrorCode)
{
setChoices(limits, nullptr, formats, cnt, constructorErrorCode);
}
// -------------------------------------
ChoiceFormat::ChoiceFormat(const double* limits,
const UBool* closures,
const UnicodeString* formats,
int32_t cnt )
: constructorErrorCode(U_ZERO_ERROR),
msgPattern(constructorErrorCode)
{
setChoices(limits, closures, formats, cnt, constructorErrorCode);
}
// -------------------------------------
// copy constructor
ChoiceFormat::ChoiceFormat(const ChoiceFormat& that)
: NumberFormat(that),
constructorErrorCode(that.constructorErrorCode),
msgPattern(that.msgPattern)
{
}
// -------------------------------------
// Private constructor that creates a
// ChoiceFormat instance based on the
// pattern and populates UParseError
ChoiceFormat::ChoiceFormat(const UnicodeString& newPattern,
UParseError& parseError,
UErrorCode& status)
: constructorErrorCode(status),
msgPattern(status)
{
applyPattern(newPattern,parseError, status);
}
// -------------------------------------
bool
ChoiceFormat::operator==(const Format& that) const
{
if (this == &that) return true;
if (!NumberFormat::operator==(that)) return false;
const ChoiceFormat& thatAlias = static_cast<const ChoiceFormat&>(that);
return msgPattern == thatAlias.msgPattern;
}
// -------------------------------------
// copy constructor
const ChoiceFormat&
ChoiceFormat::operator=(const ChoiceFormat& that)
{
if (this != &that) {
NumberFormat::operator=(that);
constructorErrorCode = that.constructorErrorCode;
msgPattern = that.msgPattern;
}
return *this;
}
// -------------------------------------
ChoiceFormat::~ChoiceFormat()
{
}
// -------------------------------------
/**
* Convert a double value to a string without the overhead of NumberFormat.
*/
UnicodeString&
ChoiceFormat::dtos(double value,
UnicodeString& string)
{
/* Buffer to contain the digits and any extra formatting stuff. */
char temp[DBL_DIG + 16];
char *itrPtr = temp;
char *expPtr;
snprintf(temp, sizeof(temp), "%.*g", DBL_DIG, value);
/* Find and convert the decimal point.
Using setlocale on some machines will cause snprintf to use a comma for certain locales.
*/
while (*itrPtr && (*itrPtr == '-' || isdigit(*itrPtr))) {
itrPtr++;
}
if (*itrPtr != 0 && *itrPtr != 'e') {
/* We reached something that looks like a decimal point.
In case someone used setlocale(), which changes the decimal point. */
*itrPtr = '.';
itrPtr++;
}
/* Search for the exponent */
while (*itrPtr && *itrPtr != 'e') {
itrPtr++;
}
if (*itrPtr == 'e') {
itrPtr++;
/* Verify the exponent sign */
if (*itrPtr == '+' || *itrPtr == '-') {
itrPtr++;
}
/* Remove leading zeros. You will see this on Windows machines. */
expPtr = itrPtr;
while (*itrPtr == '0') {
itrPtr++;
}
if (*itrPtr && expPtr != itrPtr) {
/* Shift the exponent without zeros. */
while (*itrPtr) {
*(expPtr++) = *(itrPtr++);
}
// NUL terminate
*expPtr = 0;
}
}
string = UnicodeString(temp, -1, US_INV); /* invariant codepage */
return string;
}
// -------------------------------------
// calls the overloaded applyPattern method.
void
ChoiceFormat::applyPattern(const UnicodeString& pattern,
UErrorCode& status)
{
msgPattern.parseChoiceStyle(pattern, nullptr, status);
constructorErrorCode = status;
}
// -------------------------------------
// Applies the pattern to this ChoiceFormat instance.
void
ChoiceFormat::applyPattern(const UnicodeString& pattern,
UParseError& parseError,
UErrorCode& status)
{
msgPattern.parseChoiceStyle(pattern, &parseError, status);
constructorErrorCode = status;
}
// -------------------------------------
// Returns the input pattern string.
UnicodeString&
ChoiceFormat::toPattern(UnicodeString& result) const
{
return result = msgPattern.getPatternString();
}
// -------------------------------------
// Sets the limit and format arrays.
void
ChoiceFormat::setChoices( const double* limits,
const UnicodeString* formats,
int32_t cnt )
{
UErrorCode errorCode = U_ZERO_ERROR;
setChoices(limits, nullptr, formats, cnt, errorCode);
}
// -------------------------------------
// Sets the limit and format arrays.
void
ChoiceFormat::setChoices( const double* limits,
const UBool* closures,
const UnicodeString* formats,
int32_t cnt )
{
UErrorCode errorCode = U_ZERO_ERROR;
setChoices(limits, closures, formats, cnt, errorCode);
}
void
ChoiceFormat::setChoices(const double* limits,
const UBool* closures,
const UnicodeString* formats,
int32_t count,
UErrorCode &errorCode) {
if (U_FAILURE(errorCode)) {
return;
}
if (limits == nullptr || formats == nullptr) {
errorCode = U_ILLEGAL_ARGUMENT_ERROR;
return;
}
// Reconstruct the original input pattern.
// Modified version of the pre-ICU 4.8 toPattern() implementation.
UnicodeString result;
for (int32_t i = 0; i < count; ++i) {
if (i != 0) {
result += VERTICAL_BAR;
}
UnicodeString buf;
if (uprv_isPositiveInfinity(limits[i])) {
result += INFINITY;
} else if (uprv_isNegativeInfinity(limits[i])) {
result += MINUS;
result += INFINITY;
} else {
result += dtos(limits[i], buf);
}
if (closures != nullptr && closures[i]) {
result += LESS_THAN;
} else {
result += LESS_EQUAL;
}
// Append formats[i], using quotes if there are special
// characters. Single quotes themselves must be escaped in
// either case.
const UnicodeString& text = formats[i];
int32_t textLength = text.length();
int32_t nestingLevel = 0;
for (int32_t j = 0; j < textLength; ++j) {
char16_t c = text[j];
if (c == SINGLE_QUOTE && nestingLevel == 0) {
// Double each top-level apostrophe.
result.append(c);
} else if (c == VERTICAL_BAR && nestingLevel == 0) {
// Surround each pipe symbol with apostrophes for quoting.
// If the next character is an apostrophe, then that will be doubled,
// and although the parser will see the apostrophe pairs beginning
// and ending one character earlier than our doubling, the result
// is as desired.
// | -> '|'
// |' -> '|'''
// |'' -> '|''''' etc.
result.append(SINGLE_QUOTE).append(c).append(SINGLE_QUOTE);
continue; // Skip the append(c) at the end of the loop body.
} else if (c == LEFT_CURLY_BRACE) {
++nestingLevel;
} else if (c == RIGHT_CURLY_BRACE && nestingLevel > 0) {
--nestingLevel;
}
result.append(c);
}
}
// Apply the reconstructed pattern.
applyPattern(result, errorCode);
}
// -------------------------------------
// Gets the limit array.
const double*
ChoiceFormat::getLimits(int32_t& cnt) const
{
cnt = 0;
return nullptr;
}
// -------------------------------------
// Gets the closures array.
const UBool*
ChoiceFormat::getClosures(int32_t& cnt) const
{
cnt = 0;
return nullptr;
}
// -------------------------------------
// Gets the format array.
const UnicodeString*
ChoiceFormat::getFormats(int32_t& cnt) const
{
cnt = 0;
return nullptr;
}
// -------------------------------------
// Formats an int64 number, it's actually formatted as
// a double. The returned format string may differ
// from the input number because of this.
UnicodeString&
ChoiceFormat::format(int64_t number,
UnicodeString& appendTo,
FieldPosition& status) const
{
return format(static_cast<double>(number), appendTo, status);
}
// -------------------------------------
// Formats an int32_t number, it's actually formatted as
// a double.
UnicodeString&
ChoiceFormat::format(int32_t number,
UnicodeString& appendTo,
FieldPosition& status) const
{
return format(static_cast<double>(number), appendTo, status);
}
// -------------------------------------
// Formats a double number.
UnicodeString&
ChoiceFormat::format(double number,
UnicodeString& appendTo,
FieldPosition& /*pos*/) const
{
if (msgPattern.countParts() == 0) {
// No pattern was applied, or it failed.
return appendTo;
}
// Get the appropriate sub-message.
int32_t msgStart = findSubMessage(msgPattern, 0, number);
if (!MessageImpl::jdkAposMode(msgPattern)) {
int32_t patternStart = msgPattern.getPart(msgStart).getLimit();
int32_t msgLimit = msgPattern.getLimitPartIndex(msgStart);
appendTo.append(msgPattern.getPatternString(),
patternStart,
msgPattern.getPatternIndex(msgLimit) - patternStart);
return appendTo;
}
// JDK compatibility mode: Remove SKIP_SYNTAX.
return MessageImpl::appendSubMessageWithoutSkipSyntax(msgPattern, msgStart, appendTo);
}
int32_t
ChoiceFormat::findSubMessage(const MessagePattern &pattern, int32_t partIndex, double number) {
int32_t count = pattern.countParts();
int32_t msgStart;
// Iterate over (ARG_INT|DOUBLE, ARG_SELECTOR, message) tuples
// until ARG_LIMIT or end of choice-only pattern.
// Ignore the first number and selector and start the loop on the first message.
partIndex += 2;
for (;;) {
// Skip but remember the current sub-message.
msgStart = partIndex;
partIndex = pattern.getLimitPartIndex(partIndex);
if (++partIndex >= count) {
// Reached the end of the choice-only pattern.
// Return with the last sub-message.
break;
}
const MessagePattern::Part &part = pattern.getPart(partIndex++);
UMessagePatternPartType type = part.getType();
if (type == UMSGPAT_PART_TYPE_ARG_LIMIT) {
// Reached the end of the ChoiceFormat style.
// Return with the last sub-message.
break;
}
// part is an ARG_INT or ARG_DOUBLE
U_ASSERT(MessagePattern::Part::hasNumericValue(type));
double boundary = pattern.getNumericValue(part);
// Fetch the ARG_SELECTOR character.
int32_t selectorIndex = pattern.getPatternIndex(partIndex++);
char16_t boundaryChar = pattern.getPatternString().charAt(selectorIndex);
if (boundaryChar == LESS_THAN ? !(number > boundary) : !(number >= boundary)) {
// The number is in the interval between the previous boundary and the current one.
// Return with the sub-message between them.
// The !(a>b) and !(a>=b) comparisons are equivalent to
// (a<=b) and (a<b) except they "catch" NaN.
break;
}
}
return msgStart;
}
// -------------------------------------
// Formats an array of objects. Checks if the data type of the objects
// to get the right value for formatting.
UnicodeString&
ChoiceFormat::format(const Formattable* objs,
int32_t cnt,
UnicodeString& appendTo,
FieldPosition& pos,
UErrorCode& status) const
{
if(cnt < 0) {
status = U_ILLEGAL_ARGUMENT_ERROR;
return appendTo;
}
if (msgPattern.countParts() == 0) {
status = U_INVALID_STATE_ERROR;
return appendTo;
}
for (int32_t i = 0; i < cnt; i++) {
double objDouble = objs[i].getDouble(status);
if (U_SUCCESS(status)) {
format(objDouble, appendTo, pos);
}
}
return appendTo;
}
// -------------------------------------
void
ChoiceFormat::parse(const UnicodeString& text,
Formattable& result,
ParsePosition& pos) const
{
result.setDouble(parseArgument(msgPattern, 0, text, pos));
}
double
ChoiceFormat::parseArgument(
const MessagePattern &pattern, int32_t partIndex,
const UnicodeString &source, ParsePosition &pos) {
// find the best number (defined as the one with the longest parse)
int32_t start = pos.getIndex();
int32_t furthest = start;
double bestNumber = uprv_getNaN();
double tempNumber = 0.0;
int32_t count = pattern.countParts();
while (partIndex < count && pattern.getPartType(partIndex) != UMSGPAT_PART_TYPE_ARG_LIMIT) {
tempNumber = pattern.getNumericValue(pattern.getPart(partIndex));
partIndex += 2; // skip the numeric part and ignore the ARG_SELECTOR
int32_t msgLimit = pattern.getLimitPartIndex(partIndex);
int32_t len = matchStringUntilLimitPart(pattern, partIndex, msgLimit, source, start);
if (len >= 0) {
int32_t newIndex = start + len;
if (newIndex > furthest) {
furthest = newIndex;
bestNumber = tempNumber;
if (furthest == source.length()) {
break;
}
}
}
partIndex = msgLimit + 1;
}
if (furthest == start) {
pos.setErrorIndex(start);
} else {
pos.setIndex(furthest);
}
return bestNumber;
}
int32_t
ChoiceFormat::matchStringUntilLimitPart(
const MessagePattern &pattern, int32_t partIndex, int32_t limitPartIndex,
const UnicodeString &source, int32_t sourceOffset) {
int32_t matchingSourceLength = 0;
const UnicodeString &msgString = pattern.getPatternString();
int32_t prevIndex = pattern.getPart(partIndex).getLimit();
for (;;) {
const MessagePattern::Part &part = pattern.getPart(++partIndex);
if (partIndex == limitPartIndex || part.getType() == UMSGPAT_PART_TYPE_SKIP_SYNTAX) {
int32_t index = part.getIndex();
int32_t length = index - prevIndex;
if (length != 0 && 0 != source.compare(sourceOffset, length, msgString, prevIndex, length)) {
return -1; // mismatch
}
matchingSourceLength += length;
if (partIndex == limitPartIndex) {
return matchingSourceLength;
}
prevIndex = part.getLimit(); // SKIP_SYNTAX
}
}
}
// -------------------------------------
ChoiceFormat*
ChoiceFormat::clone() const
{
ChoiceFormat *aCopy = new ChoiceFormat(*this);
return aCopy;
}
U_NAMESPACE_END
#endif /* #if !UCONFIG_NO_FORMATTING */
//eof

473
deps/icu-small/source/i18n/coleitr.cpp vendored Normal file
View File

@ -0,0 +1,473 @@
// © 2016 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html
/*
*******************************************************************************
* Copyright (C) 1996-2014, International Business Machines Corporation and
* others. All Rights Reserved.
*******************************************************************************
*/
/*
* File coleitr.cpp
*
* Created by: Helena Shih
*
* Modification History:
*
* Date Name Description
*
* 6/23/97 helena Adding comments to make code more readable.
* 08/03/98 erm Synched with 1.2 version of CollationElementIterator.java
* 12/10/99 aliu Ported Thai collation support from Java.
* 01/25/01 swquek Modified to a C++ wrapper calling C APIs (ucoliter.h)
* 02/19/01 swquek Removed CollationElementIterator() since it is
* private constructor and no calls are made to it
* 2012-2014 markus Rewritten in C++ again.
*/
#include "unicode/utypes.h"
#if !UCONFIG_NO_COLLATION
#include "unicode/chariter.h"
#include "unicode/coleitr.h"
#include "unicode/tblcoll.h"
#include "unicode/ustring.h"
#include "cmemory.h"
#include "collation.h"
#include "collationdata.h"
#include "collationiterator.h"
#include "collationsets.h"
#include "collationtailoring.h"
#include "uassert.h"
#include "uhash.h"
#include "utf16collationiterator.h"
#include "uvectr32.h"
/* Constants --------------------------------------------------------------- */
U_NAMESPACE_BEGIN
UOBJECT_DEFINE_RTTI_IMPLEMENTATION(CollationElementIterator)
/* CollationElementIterator public constructor/destructor ------------------ */
CollationElementIterator::CollationElementIterator(
const CollationElementIterator& other)
: UObject(other), iter_(nullptr), rbc_(nullptr), otherHalf_(0), dir_(0), offsets_(nullptr) {
*this = other;
}
CollationElementIterator::~CollationElementIterator()
{
delete iter_;
delete offsets_;
}
/* CollationElementIterator public methods --------------------------------- */
namespace {
uint32_t getFirstHalf(uint32_t p, uint32_t lower32) {
return (p & 0xffff0000) | ((lower32 >> 16) & 0xff00) | ((lower32 >> 8) & 0xff);
}
uint32_t getSecondHalf(uint32_t p, uint32_t lower32) {
return (p << 16) | ((lower32 >> 8) & 0xff00) | (lower32 & 0x3f);
}
UBool ceNeedsTwoParts(int64_t ce) {
return (ce & INT64_C(0xffff00ff003f)) != 0;
}
} // namespace
int32_t CollationElementIterator::getOffset() const
{
if (dir_ < 0 && offsets_ != nullptr && !offsets_->isEmpty()) {
// CollationIterator::previousCE() decrements the CEs length
// while it pops CEs from its internal buffer.
int32_t i = iter_->getCEsLength();
if (otherHalf_ != 0) {
// Return the trailing CE offset while we are in the middle of a 64-bit CE.
++i;
}
U_ASSERT(i < offsets_->size());
return offsets_->elementAti(i);
}
return iter_->getOffset();
}
/**
* Get the ordering priority of the next character in the string.
* @return the next character's ordering. Returns NULLORDER if an error has
* occurred or if the end of string has been reached
*/
int32_t CollationElementIterator::next(UErrorCode& status)
{
if (U_FAILURE(status)) { return NULLORDER; }
if (dir_ > 1) {
// Continue forward iteration. Test this first.
if (otherHalf_ != 0) {
uint32_t oh = otherHalf_;
otherHalf_ = 0;
return oh;
}
} else if (dir_ == 1) {
// next() after setOffset()
dir_ = 2;
} else if (dir_ == 0) {
// The iter_ is already reset to the start of the text.
dir_ = 2;
} else /* dir_ < 0 */ {
// illegal change of direction
status = U_INVALID_STATE_ERROR;
return NULLORDER;
}
// No need to keep all CEs in the buffer when we iterate.
iter_->clearCEsIfNoneRemaining();
int64_t ce = iter_->nextCE(status);
if (ce == Collation::NO_CE) { return NULLORDER; }
// Turn the 64-bit CE into two old-style 32-bit CEs, without quaternary bits.
uint32_t p = static_cast<uint32_t>(ce >> 32);
uint32_t lower32 = static_cast<uint32_t>(ce);
uint32_t firstHalf = getFirstHalf(p, lower32);
uint32_t secondHalf = getSecondHalf(p, lower32);
if (secondHalf != 0) {
otherHalf_ = secondHalf | 0xc0; // continuation CE
}
return firstHalf;
}
bool CollationElementIterator::operator!=(
const CollationElementIterator& other) const
{
return !(*this == other);
}
bool CollationElementIterator::operator==(
const CollationElementIterator& that) const
{
if (this == &that) {
return true;
}
return
(rbc_ == that.rbc_ || *rbc_ == *that.rbc_) &&
otherHalf_ == that.otherHalf_ &&
normalizeDir() == that.normalizeDir() &&
string_ == that.string_ &&
*iter_ == *that.iter_;
}
/**
* Get the ordering priority of the previous collation element in the string.
* @param status the error code status.
* @return the previous element's ordering. Returns NULLORDER if an error has
* occurred or if the start of string has been reached.
*/
int32_t CollationElementIterator::previous(UErrorCode& status)
{
if (U_FAILURE(status)) { return NULLORDER; }
if (dir_ < 0) {
// Continue backwards iteration. Test this first.
if (otherHalf_ != 0) {
uint32_t oh = otherHalf_;
otherHalf_ = 0;
return oh;
}
} else if (dir_ == 0) {
iter_->resetToOffset(string_.length());
dir_ = -1;
} else if (dir_ == 1) {
// previous() after setOffset()
dir_ = -1;
} else /* dir_ > 1 */ {
// illegal change of direction
status = U_INVALID_STATE_ERROR;
return NULLORDER;
}
if (offsets_ == nullptr) {
offsets_ = new UVector32(status);
if (offsets_ == nullptr) {
status = U_MEMORY_ALLOCATION_ERROR;
return NULLORDER;
}
}
// If we already have expansion CEs, then we also have offsets.
// Otherwise remember the trailing offset in case we need to
// write offsets for an artificial expansion.
int32_t limitOffset = iter_->getCEsLength() == 0 ? iter_->getOffset() : 0;
int64_t ce = iter_->previousCE(*offsets_, status);
if (ce == Collation::NO_CE) { return NULLORDER; }
// Turn the 64-bit CE into two old-style 32-bit CEs, without quaternary bits.
uint32_t p = static_cast<uint32_t>(ce >> 32);
uint32_t lower32 = static_cast<uint32_t>(ce);
uint32_t firstHalf = getFirstHalf(p, lower32);
uint32_t secondHalf = getSecondHalf(p, lower32);
if (secondHalf != 0) {
if (offsets_->isEmpty()) {
// When we convert a single 64-bit CE into two 32-bit CEs,
// we need to make this artificial expansion behave like a normal expansion.
// See CollationIterator::previousCE().
offsets_->addElement(iter_->getOffset(), status);
offsets_->addElement(limitOffset, status);
}
otherHalf_ = firstHalf;
return secondHalf | 0xc0; // continuation CE
}
return firstHalf;
}
/**
* Resets the cursor to the beginning of the string.
*/
void CollationElementIterator::reset()
{
iter_ ->resetToOffset(0);
otherHalf_ = 0;
dir_ = 0;
}
void CollationElementIterator::setOffset(int32_t newOffset,
UErrorCode& status)
{
if (U_FAILURE(status)) { return; }
if (0 < newOffset && newOffset < string_.length()) {
int32_t offset = newOffset;
do {
char16_t c = string_.charAt(offset);
if (!rbc_->isUnsafe(c) ||
(U16_IS_LEAD(c) && !rbc_->isUnsafe(string_.char32At(offset)))) {
break;
}
// Back up to before this unsafe character.
--offset;
} while (offset > 0);
if (offset < newOffset) {
// We might have backed up more than necessary.
// For example, contractions "ch" and "cu" make both 'h' and 'u' unsafe,
// but for text "chu" setOffset(2) should remain at 2
// although we initially back up to offset 0.
// Find the last safe offset no greater than newOffset by iterating forward.
int32_t lastSafeOffset = offset;
do {
iter_->resetToOffset(lastSafeOffset);
do {
iter_->nextCE(status);
if (U_FAILURE(status)) { return; }
} while ((offset = iter_->getOffset()) == lastSafeOffset);
if (offset <= newOffset) {
lastSafeOffset = offset;
}
} while (offset < newOffset);
newOffset = lastSafeOffset;
}
}
iter_->resetToOffset(newOffset);
otherHalf_ = 0;
dir_ = 1;
}
/**
* Sets the source to the new source string.
*/
void CollationElementIterator::setText(const UnicodeString& source,
UErrorCode& status)
{
if (U_FAILURE(status)) {
return;
}
string_ = source;
const char16_t *s = string_.getBuffer();
CollationIterator *newIter;
UBool numeric = rbc_->settings->isNumeric();
if (rbc_->settings->dontCheckFCD()) {
newIter = new UTF16CollationIterator(rbc_->data, numeric, s, s, s + string_.length());
} else {
newIter = new FCDUTF16CollationIterator(rbc_->data, numeric, s, s, s + string_.length());
}
if (newIter == nullptr) {
status = U_MEMORY_ALLOCATION_ERROR;
return;
}
delete iter_;
iter_ = newIter;
otherHalf_ = 0;
dir_ = 0;
}
// Sets the source to the new character iterator.
void CollationElementIterator::setText(CharacterIterator& source,
UErrorCode& status)
{
if (U_FAILURE(status))
return;
source.getText(string_);
setText(string_, status);
}
int32_t CollationElementIterator::strengthOrder(int32_t order) const
{
UColAttributeValue s = static_cast<UColAttributeValue>(rbc_->settings->getStrength());
// Mask off the unwanted differences.
if (s == UCOL_PRIMARY) {
order &= 0xffff0000;
}
else if (s == UCOL_SECONDARY) {
order &= 0xffffff00;
}
return order;
}
/* CollationElementIterator private constructors/destructors --------------- */
/**
* This is the "real" constructor for this class; it constructs an iterator
* over the source text using the specified collator
*/
CollationElementIterator::CollationElementIterator(
const UnicodeString &source,
const RuleBasedCollator *coll,
UErrorCode &status)
: iter_(nullptr), rbc_(coll), otherHalf_(0), dir_(0), offsets_(nullptr) {
setText(source, status);
}
/**
* This is the "real" constructor for this class; it constructs an iterator over
* the source text using the specified collator
*/
CollationElementIterator::CollationElementIterator(
const CharacterIterator &source,
const RuleBasedCollator *coll,
UErrorCode &status)
: iter_(nullptr), rbc_(coll), otherHalf_(0), dir_(0), offsets_(nullptr) {
// We only call source.getText() which should be const anyway.
setText(const_cast<CharacterIterator &>(source), status);
}
/* CollationElementIterator private methods -------------------------------- */
const CollationElementIterator& CollationElementIterator::operator=(
const CollationElementIterator& other)
{
if (this == &other) {
return *this;
}
CollationIterator *newIter;
const FCDUTF16CollationIterator *otherFCDIter =
dynamic_cast<const FCDUTF16CollationIterator *>(other.iter_);
if(otherFCDIter != nullptr) {
newIter = new FCDUTF16CollationIterator(*otherFCDIter, string_.getBuffer());
} else {
const UTF16CollationIterator *otherIter =
dynamic_cast<const UTF16CollationIterator *>(other.iter_);
if(otherIter != nullptr) {
newIter = new UTF16CollationIterator(*otherIter, string_.getBuffer());
} else {
newIter = nullptr;
}
}
if(newIter != nullptr) {
delete iter_;
iter_ = newIter;
rbc_ = other.rbc_;
otherHalf_ = other.otherHalf_;
dir_ = other.dir_;
string_ = other.string_;
}
if(other.dir_ < 0 && other.offsets_ != nullptr && !other.offsets_->isEmpty()) {
UErrorCode errorCode = U_ZERO_ERROR;
if(offsets_ == nullptr) {
offsets_ = new UVector32(other.offsets_->size(), errorCode);
}
if(offsets_ != nullptr) {
offsets_->assign(*other.offsets_, errorCode);
}
}
return *this;
}
namespace {
class MaxExpSink : public ContractionsAndExpansions::CESink {
public:
MaxExpSink(UHashtable *h, UErrorCode &ec) : maxExpansions(h), errorCode(ec) {}
virtual ~MaxExpSink();
virtual void handleCE(int64_t /*ce*/) override {}
virtual void handleExpansion(const int64_t ces[], int32_t length) override {
if (length <= 1) {
// We do not need to add single CEs into the map.
return;
}
int32_t count = 0; // number of CE "halves"
for (int32_t i = 0; i < length; ++i) {
count += ceNeedsTwoParts(ces[i]) ? 2 : 1;
}
// last "half" of the last CE
int64_t ce = ces[length - 1];
uint32_t p = static_cast<uint32_t>(ce >> 32);
uint32_t lower32 = static_cast<uint32_t>(ce);
uint32_t lastHalf = getSecondHalf(p, lower32);
if (lastHalf == 0) {
lastHalf = getFirstHalf(p, lower32);
U_ASSERT(lastHalf != 0);
} else {
lastHalf |= 0xc0; // old-style continuation CE
}
if (count > uhash_igeti(maxExpansions, static_cast<int32_t>(lastHalf))) {
uhash_iputi(maxExpansions, static_cast<int32_t>(lastHalf), count, &errorCode);
}
}
private:
UHashtable *maxExpansions;
UErrorCode &errorCode;
};
MaxExpSink::~MaxExpSink() {}
} // namespace
UHashtable *
CollationElementIterator::computeMaxExpansions(const CollationData *data, UErrorCode &errorCode) {
if (U_FAILURE(errorCode)) { return nullptr; }
UHashtable *maxExpansions = uhash_open(uhash_hashLong, uhash_compareLong,
uhash_compareLong, &errorCode);
if (U_FAILURE(errorCode)) { return nullptr; }
MaxExpSink sink(maxExpansions, errorCode);
ContractionsAndExpansions(nullptr, nullptr, &sink, true).forData(data, errorCode);
if (U_FAILURE(errorCode)) {
uhash_close(maxExpansions);
return nullptr;
}
return maxExpansions;
}
int32_t
CollationElementIterator::getMaxExpansion(int32_t order) const {
return getMaxExpansion(rbc_->tailoring->maxExpansions, order);
}
int32_t
CollationElementIterator::getMaxExpansion(const UHashtable *maxExpansions, int32_t order) {
if (order == 0) { return 1; }
int32_t max;
if(maxExpansions != nullptr && (max = uhash_igeti(maxExpansions, order)) != 0) {
return max;
}
if ((order & 0xc0) == 0xc0) {
// old-style continuation CE
return 2;
} else {
return 1;
}
}
U_NAMESPACE_END
#endif /* #if !UCONFIG_NO_COLLATION */

1021
deps/icu-small/source/i18n/coll.cpp vendored Normal file
View File

File diff suppressed because it is too large Load Diff

141
deps/icu-small/source/i18n/collation.cpp vendored Normal file
View File

@ -0,0 +1,141 @@
// © 2016 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html
/*
*******************************************************************************
* Copyright (C) 2010-2014, International Business Machines
* Corporation and others. All Rights Reserved.
*******************************************************************************
* collation.cpp
*
* created on: 2010oct27
* created by: Markus W. Scherer
*/
#include "unicode/utypes.h"
#if !UCONFIG_NO_COLLATION
#include "collation.h"
#include "uassert.h"
U_NAMESPACE_BEGIN
uint32_t
Collation::incTwoBytePrimaryByOffset(uint32_t basePrimary, UBool isCompressible, int32_t offset) {
// Extract the second byte, minus the minimum byte value,
// plus the offset, modulo the number of usable byte values, plus the minimum.
// Reserve the PRIMARY_COMPRESSION_LOW_BYTE and high byte if necessary.
uint32_t primary;
if(isCompressible) {
offset += (static_cast<int32_t>(basePrimary >> 16) & 0xff) - 4;
primary = static_cast<uint32_t>((offset % 251) + 4) << 16;
offset /= 251;
} else {
offset += (static_cast<int32_t>(basePrimary >> 16) & 0xff) - 2;
primary = static_cast<uint32_t>((offset % 254) + 2) << 16;
offset /= 254;
}
// First byte, assume no further overflow.
return primary | ((basePrimary & 0xff000000) + static_cast<uint32_t>(offset << 24));
}
uint32_t
Collation::incThreeBytePrimaryByOffset(uint32_t basePrimary, UBool isCompressible, int32_t offset) {
// Extract the third byte, minus the minimum byte value,
// plus the offset, modulo the number of usable byte values, plus the minimum.
offset += (static_cast<int32_t>(basePrimary >> 8) & 0xff) - 2;
uint32_t primary = static_cast<uint32_t>((offset % 254) + 2) << 8;
offset /= 254;
// Same with the second byte,
// but reserve the PRIMARY_COMPRESSION_LOW_BYTE and high byte if necessary.
if(isCompressible) {
offset += (static_cast<int32_t>(basePrimary >> 16) & 0xff) - 4;
primary |= static_cast<uint32_t>((offset % 251) + 4) << 16;
offset /= 251;
} else {
offset += (static_cast<int32_t>(basePrimary >> 16) & 0xff) - 2;
primary |= static_cast<uint32_t>((offset % 254) + 2) << 16;
offset /= 254;
}
// First byte, assume no further overflow.
return primary | ((basePrimary & 0xff000000) + static_cast<uint32_t>(offset << 24));
}
uint32_t
Collation::decTwoBytePrimaryByOneStep(uint32_t basePrimary, UBool isCompressible, int32_t step) {
// Extract the second byte, minus the minimum byte value,
// minus the step, modulo the number of usable byte values, plus the minimum.
// Reserve the PRIMARY_COMPRESSION_LOW_BYTE and high byte if necessary.
// Assume no further underflow for the first byte.
U_ASSERT(0 < step && step <= 0x7f);
int32_t byte2 = (static_cast<int32_t>(basePrimary >> 16) & 0xff) - step;
if(isCompressible) {
if(byte2 < 4) {
byte2 += 251;
basePrimary -= 0x1000000;
}
} else {
if(byte2 < 2) {
byte2 += 254;
basePrimary -= 0x1000000;
}
}
return (basePrimary & 0xff000000) | (static_cast<uint32_t>(byte2) << 16);
}
uint32_t
Collation::decThreeBytePrimaryByOneStep(uint32_t basePrimary, UBool isCompressible, int32_t step) {
// Extract the third byte, minus the minimum byte value,
// minus the step, modulo the number of usable byte values, plus the minimum.
U_ASSERT(0 < step && step <= 0x7f);
int32_t byte3 = (static_cast<int32_t>(basePrimary >> 8) & 0xff) - step;
if(byte3 >= 2) {
return (basePrimary & 0xffff0000) | (static_cast<uint32_t>(byte3) << 8);
}
byte3 += 254;
// Same with the second byte,
// but reserve the PRIMARY_COMPRESSION_LOW_BYTE and high byte if necessary.
int32_t byte2 = (static_cast<int32_t>(basePrimary >> 16) & 0xff) - 1;
if(isCompressible) {
if(byte2 < 4) {
byte2 = 0xfe;
basePrimary -= 0x1000000;
}
} else {
if(byte2 < 2) {
byte2 = 0xff;
basePrimary -= 0x1000000;
}
}
// First byte, assume no further underflow.
return (basePrimary & 0xff000000) | (static_cast<uint32_t>(byte2) << 16) | (static_cast<uint32_t>(byte3) << 8);
}
uint32_t
Collation::getThreeBytePrimaryForOffsetData(UChar32 c, int64_t dataCE) {
uint32_t p = static_cast<uint32_t>(dataCE >> 32); // three-byte primary pppppp00
int32_t lower32 = static_cast<int32_t>(dataCE); // base code point b & step s: bbbbbbss (bit 7: isCompressible)
int32_t offset = (c - (lower32 >> 8)) * (lower32 & 0x7f); // delta * increment
UBool isCompressible = (lower32 & 0x80) != 0;
return Collation::incThreeBytePrimaryByOffset(p, isCompressible, offset);
}
uint32_t
Collation::unassignedPrimaryFromCodePoint(UChar32 c) {
// Create a gap before U+0000. Use c=-1 for [first unassigned].
++c;
// Fourth byte: 18 values, every 14th byte value (gap of 13).
uint32_t primary = 2 + (c % 18) * 14;
c /= 18;
// Third byte: 254 values.
primary |= (2 + (c % 254)) << 8;
c /= 254;
// Second byte: 251 values 04..FE excluding the primary compression bytes.
primary |= (4 + (c % 251)) << 16;
// One lead byte covers all code points (c < 0x1182B4 = 1*251*254*18).
return primary | (UNASSIGNED_IMPLICIT_BYTE << 24);
}
U_NAMESPACE_END
#endif // !UCONFIG_NO_COLLATION

503
deps/icu-small/source/i18n/collation.h vendored Normal file
View File

@ -0,0 +1,503 @@
// © 2016 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html
/*
*******************************************************************************
* Copyright (C) 2010-2015, International Business Machines
* Corporation and others. All Rights Reserved.
*******************************************************************************
* collation.h
*
* created on: 2010oct27
* created by: Markus W. Scherer
*/
#ifndef __COLLATION_H__
#define __COLLATION_H__
#include "unicode/utypes.h"
#if !UCONFIG_NO_COLLATION
U_NAMESPACE_BEGIN
/**
* Collation v2 basic definitions and static helper functions.
*
* Data structures except for expansion tables store 32-bit CEs which are
* either specials (see tags below) or are compact forms of 64-bit CEs.
*/
class U_I18N_API Collation {
public:
// Special sort key bytes for all levels.
static const uint8_t TERMINATOR_BYTE = 0;
static const uint8_t LEVEL_SEPARATOR_BYTE = 1;
/** The secondary/tertiary lower limit for tailoring before any root elements. */
static const uint32_t BEFORE_WEIGHT16 = 0x0100;
/**
* Merge-sort-key separator.
* Same as the unique primary and identical-level weights of U+FFFE.
* Must not be used as primary compression low terminator.
* Otherwise usable.
*/
static const uint8_t MERGE_SEPARATOR_BYTE = 2;
static const uint32_t MERGE_SEPARATOR_PRIMARY = 0x02000000; // U+FFFE
static const uint32_t MERGE_SEPARATOR_CE32 = 0x02000505; // U+FFFE
/**
* Primary compression low terminator, must be greater than MERGE_SEPARATOR_BYTE.
* Reserved value in primary second byte if the lead byte is compressible.
* Otherwise usable in all CE weight bytes.
*/
static const uint8_t PRIMARY_COMPRESSION_LOW_BYTE = 3;
/**
* Primary compression high terminator.
* Reserved value in primary second byte if the lead byte is compressible.
* Otherwise usable in all CE weight bytes.
*/
static const uint8_t PRIMARY_COMPRESSION_HIGH_BYTE = 0xff;
/** Default secondary/tertiary weight lead byte. */
static const uint8_t COMMON_BYTE = 5;
static const uint32_t COMMON_WEIGHT16 = 0x0500;
/** Middle 16 bits of a CE with a common secondary weight. */
static const uint32_t COMMON_SECONDARY_CE = 0x05000000;
/** Lower 16 bits of a CE with a common tertiary weight. */
static const uint32_t COMMON_TERTIARY_CE = 0x0500;
/** Lower 32 bits of a CE with common secondary and tertiary weights. */
static const uint32_t COMMON_SEC_AND_TER_CE = 0x05000500;
static const uint32_t SECONDARY_MASK = 0xffff0000;
static const uint32_t CASE_MASK = 0xc000;
static const uint32_t SECONDARY_AND_CASE_MASK = SECONDARY_MASK | CASE_MASK;
/** Only the 2*6 bits for the pure tertiary weight. */
static const uint32_t ONLY_TERTIARY_MASK = 0x3f3f;
/** Only the secondary & tertiary bits; no case, no quaternary. */
static const uint32_t ONLY_SEC_TER_MASK = SECONDARY_MASK | ONLY_TERTIARY_MASK;
/** Case bits and tertiary bits. */
static const uint32_t CASE_AND_TERTIARY_MASK = CASE_MASK | ONLY_TERTIARY_MASK;
static const uint32_t QUATERNARY_MASK = 0xc0;
/** Case bits and quaternary bits. */
static const uint32_t CASE_AND_QUATERNARY_MASK = CASE_MASK | QUATERNARY_MASK;
static const uint8_t UNASSIGNED_IMPLICIT_BYTE = 0xfe; // compressible
/**
* First unassigned: AlphabeticIndex overflow boundary.
* We want a 3-byte primary so that it fits into the root elements table.
*
* This 3-byte primary will not collide with
* any unassigned-implicit 4-byte primaries because
* the first few hundred Unicode code points all have real mappings.
*/
static const uint32_t FIRST_UNASSIGNED_PRIMARY = 0xfe040200;
static const uint8_t TRAIL_WEIGHT_BYTE = 0xff; // not compressible
static const uint32_t FIRST_TRAILING_PRIMARY = 0xff020200; // [first trailing]
static const uint32_t MAX_PRIMARY = 0xffff0000; // U+FFFF
static const uint32_t MAX_REGULAR_CE32 = 0xffff0505; // U+FFFF
// CE32 value for U+FFFD as well as illegal UTF-8 byte sequences (which behave like U+FFFD).
// We use the third-highest primary weight for U+FFFD (as in UCA 6.3+).
static const uint32_t FFFD_PRIMARY = MAX_PRIMARY - 0x20000;
static const uint32_t FFFD_CE32 = MAX_REGULAR_CE32 - 0x20000;
/**
* A CE32 is special if its low byte is this or greater.
* Impossible case bits 11 mark special CE32s.
* This value itself is used to indicate a fallback to the base collator.
*/
static const uint8_t SPECIAL_CE32_LOW_BYTE = 0xc0;
static const uint32_t FALLBACK_CE32 = SPECIAL_CE32_LOW_BYTE;
/**
* Low byte of a long-primary special CE32.
*/
static const uint8_t LONG_PRIMARY_CE32_LOW_BYTE = 0xc1; // SPECIAL_CE32_LOW_BYTE | LONG_PRIMARY_TAG
static const uint32_t UNASSIGNED_CE32 = 0xffffffff; // Compute an unassigned-implicit CE.
static const uint32_t NO_CE32 = 1;
/** No CE: End of input. Only used in runtime code, not stored in data. */
static const uint32_t NO_CE_PRIMARY = 1; // not a left-adjusted weight
static const uint32_t NO_CE_WEIGHT16 = 0x0100; // weight of LEVEL_SEPARATOR_BYTE
static const int64_t NO_CE = INT64_C(0x101000100); // NO_CE_PRIMARY, NO_CE_WEIGHT16, NO_CE_WEIGHT16
/** Sort key levels. */
enum Level {
/** Unspecified level. */
NO_LEVEL,
PRIMARY_LEVEL,
SECONDARY_LEVEL,
CASE_LEVEL,
TERTIARY_LEVEL,
QUATERNARY_LEVEL,
IDENTICAL_LEVEL,
/** Beyond sort key bytes. */
ZERO_LEVEL
};
/**
* Sort key level flags: xx_FLAG = 1 << xx_LEVEL.
* In Java, use enum Level with flag() getters, or use EnumSet rather than hand-made bit sets.
*/
static const uint32_t NO_LEVEL_FLAG = 1;
static const uint32_t PRIMARY_LEVEL_FLAG = 2;
static const uint32_t SECONDARY_LEVEL_FLAG = 4;
static const uint32_t CASE_LEVEL_FLAG = 8;
static const uint32_t TERTIARY_LEVEL_FLAG = 0x10;
static const uint32_t QUATERNARY_LEVEL_FLAG = 0x20;
static const uint32_t IDENTICAL_LEVEL_FLAG = 0x40;
static const uint32_t ZERO_LEVEL_FLAG = 0x80;
/**
* Special-CE32 tags, from bits 3..0 of a special 32-bit CE.
* Bits 31..8 are available for tag-specific data.
* Bits 5..4: Reserved. May be used in the future to indicate lccc!=0 and tccc!=0.
*/
enum {
/**
* Fall back to the base collator.
* This is the tag value in SPECIAL_CE32_LOW_BYTE and FALLBACK_CE32.
* Bits 31..8: Unused, 0.
*/
FALLBACK_TAG = 0,
/**
* Long-primary CE with COMMON_SEC_AND_TER_CE.
* Bits 31..8: Three-byte primary.
*/
LONG_PRIMARY_TAG = 1,
/**
* Long-secondary CE with zero primary.
* Bits 31..16: Secondary weight.
* Bits 15.. 8: Tertiary weight.
*/
LONG_SECONDARY_TAG = 2,
/**
* Unused.
* May be used in the future for single-byte secondary CEs (SHORT_SECONDARY_TAG),
* storing the secondary in bits 31..24, the ccc in bits 23..16,
* and the tertiary in bits 15..8.
*/
RESERVED_TAG_3 = 3,
/**
* Latin mini expansions of two simple CEs [pp, 05, tt] [00, ss, 05].
* Bits 31..24: Single-byte primary weight pp of the first CE.
* Bits 23..16: Tertiary weight tt of the first CE.
* Bits 15.. 8: Secondary weight ss of the second CE.
*/
LATIN_EXPANSION_TAG = 4,
/**
* Points to one or more simple/long-primary/long-secondary 32-bit CE32s.
* Bits 31..13: Index into uint32_t table.
* Bits 12.. 8: Length=1..31.
*/
EXPANSION32_TAG = 5,
/**
* Points to one or more 64-bit CEs.
* Bits 31..13: Index into CE table.
* Bits 12.. 8: Length=1..31.
*/
EXPANSION_TAG = 6,
/**
* Builder data, used only in the CollationDataBuilder, not in runtime data.
*
* If bit 8 is 0: Builder context, points to a list of context-sensitive mappings.
* Bits 31..13: Index to the builder's list of ConditionalCE32 for this character.
* Bits 12.. 9: Unused, 0.
*
* If bit 8 is 1 (IS_BUILDER_JAMO_CE32): Builder-only jamoCE32 value.
* The builder fetches the Jamo CE32 from the trie.
* Bits 31..13: Jamo code point.
* Bits 12.. 9: Unused, 0.
*/
BUILDER_DATA_TAG = 7,
/**
* Points to prefix trie.
* Bits 31..13: Index into prefix/contraction data.
* Bits 12.. 8: Unused, 0.
*/
PREFIX_TAG = 8,
/**
* Points to contraction data.
* Bits 31..13: Index into prefix/contraction data.
* Bit 12: Unused, 0.
* Bit 11: CONTRACT_HAS_STARTER flag. (Used by ICU4X only.)
* Bit 10: CONTRACT_TRAILING_CCC flag.
* Bit 9: CONTRACT_NEXT_CCC flag.
* Bit 8: CONTRACT_SINGLE_CP_NO_MATCH flag.
*/
CONTRACTION_TAG = 9,
/**
* Decimal digit.
* Bits 31..13: Index into uint32_t table for non-numeric-collation CE32.
* Bit 12: Unused, 0.
* Bits 11.. 8: Digit value 0..9.
*/
DIGIT_TAG = 10,
/**
* Tag for U+0000, for moving the NUL-termination handling
* from the regular fastpath into specials-handling code.
* Bits 31..8: Unused, 0.
*/
U0000_TAG = 11,
/**
* Tag for a Hangul syllable.
* Bits 31..9: Unused, 0.
* Bit 8: HANGUL_NO_SPECIAL_JAMO flag.
*/
HANGUL_TAG = 12,
/**
* Tag for a lead surrogate code unit.
* Optional optimization for UTF-16 string processing.
* Bits 31..10: Unused, 0.
* 9.. 8: =0: All associated supplementary code points are unassigned-implicit.
* =1: All associated supplementary code points fall back to the base data.
* else: (Normally 2) Look up the data for the supplementary code point.
*/
LEAD_SURROGATE_TAG = 13,
/**
* Tag for CEs with primary weights in code point order.
* Bits 31..13: Index into CE table, for one data "CE".
* Bits 12.. 8: Unused, 0.
*
* This data "CE" has the following bit fields:
* Bits 63..32: Three-byte primary pppppp00.
* 31.. 8: Start/base code point of the in-order range.
* 7: Flag isCompressible primary.
* 6.. 0: Per-code point primary-weight increment.
*/
OFFSET_TAG = 14,
/**
* Implicit CE tag. Compute an unassigned-implicit CE.
* All bits are set (UNASSIGNED_CE32=0xffffffff).
*/
IMPLICIT_TAG = 15
};
static UBool isAssignedCE32(uint32_t ce32) {
return ce32 != FALLBACK_CE32 && ce32 != UNASSIGNED_CE32;
}
/**
* We limit the number of CEs in an expansion
* so that we can use a small number of length bits in the data structure,
* and so that an implementation can copy CEs at runtime without growing a destination buffer.
*/
static const int32_t MAX_EXPANSION_LENGTH = 31;
static const int32_t MAX_INDEX = 0x7ffff;
/**
* Set if there is no match for the single (no-suffix) character itself.
* This is only possible if there is a prefix.
* In this case, discontiguous contraction matching cannot add combining marks
* starting from an empty suffix.
* The default CE32 is used anyway if there is no suffix match.
*/
static const uint32_t CONTRACT_SINGLE_CP_NO_MATCH = 0x100;
/** Set if the first character of every contraction suffix has lccc!=0. */
static const uint32_t CONTRACT_NEXT_CCC = 0x200;
/** Set if any contraction suffix ends with lccc!=0. */
static const uint32_t CONTRACT_TRAILING_CCC = 0x400;
/** Set if any contraction suffix contains a starter. (Used by ICU4X only.) */
static const uint32_t CONTRACT_HAS_STARTER = 0x800;
/** For HANGUL_TAG: None of its Jamo CE32s isSpecialCE32(). */
static const uint32_t HANGUL_NO_SPECIAL_JAMO = 0x100;
static const uint32_t LEAD_ALL_UNASSIGNED = 0;
static const uint32_t LEAD_ALL_FALLBACK = 0x100;
static const uint32_t LEAD_MIXED = 0x200;
static const uint32_t LEAD_TYPE_MASK = 0x300;
static uint32_t makeLongPrimaryCE32(uint32_t p) { return p | LONG_PRIMARY_CE32_LOW_BYTE; }
/** Turns the long-primary CE32 into a primary weight pppppp00. */
static inline uint32_t primaryFromLongPrimaryCE32(uint32_t ce32) {
return ce32 & 0xffffff00;
}
static inline int64_t ceFromLongPrimaryCE32(uint32_t ce32) {
return (static_cast<int64_t>(ce32 & 0xffffff00) << 32) | COMMON_SEC_AND_TER_CE;
}
static uint32_t makeLongSecondaryCE32(uint32_t lower32) {
return lower32 | SPECIAL_CE32_LOW_BYTE | LONG_SECONDARY_TAG;
}
static inline int64_t ceFromLongSecondaryCE32(uint32_t ce32) {
return ce32 & 0xffffff00;
}
/** Makes a special CE32 with tag, index and length. */
static uint32_t makeCE32FromTagIndexAndLength(int32_t tag, int32_t index, int32_t length) {
return (index << 13) | (length << 8) | SPECIAL_CE32_LOW_BYTE | tag;
}
/** Makes a special CE32 with only tag and index. */
static uint32_t makeCE32FromTagAndIndex(int32_t tag, int32_t index) {
return (index << 13) | SPECIAL_CE32_LOW_BYTE | tag;
}
static inline UBool isSpecialCE32(uint32_t ce32) {
return (ce32 & 0xff) >= SPECIAL_CE32_LOW_BYTE;
}
static inline int32_t tagFromCE32(uint32_t ce32) {
return static_cast<int32_t>(ce32 & 0xf);
}
static inline UBool hasCE32Tag(uint32_t ce32, int32_t tag) {
return isSpecialCE32(ce32) && tagFromCE32(ce32) == tag;
}
static inline UBool isLongPrimaryCE32(uint32_t ce32) {
return hasCE32Tag(ce32, LONG_PRIMARY_TAG);
}
static UBool isSimpleOrLongCE32(uint32_t ce32) {
return !isSpecialCE32(ce32) ||
tagFromCE32(ce32) == LONG_PRIMARY_TAG ||
tagFromCE32(ce32) == LONG_SECONDARY_TAG;
}
/**
* @return true if the ce32 yields one or more CEs without further data lookups
*/
static UBool isSelfContainedCE32(uint32_t ce32) {
return !isSpecialCE32(ce32) ||
tagFromCE32(ce32) == LONG_PRIMARY_TAG ||
tagFromCE32(ce32) == LONG_SECONDARY_TAG ||
tagFromCE32(ce32) == LATIN_EXPANSION_TAG;
}
static inline UBool isPrefixCE32(uint32_t ce32) {
return hasCE32Tag(ce32, PREFIX_TAG);
}
static inline UBool isContractionCE32(uint32_t ce32) {
return hasCE32Tag(ce32, CONTRACTION_TAG);
}
static inline UBool ce32HasContext(uint32_t ce32) {
return isSpecialCE32(ce32) &&
(tagFromCE32(ce32) == PREFIX_TAG ||
tagFromCE32(ce32) == CONTRACTION_TAG);
}
/**
* Get the first of the two Latin-expansion CEs encoded in ce32.
* @see LATIN_EXPANSION_TAG
*/
static inline int64_t latinCE0FromCE32(uint32_t ce32) {
return (static_cast<int64_t>(ce32 & 0xff000000) << 32) | COMMON_SECONDARY_CE | ((ce32 & 0xff0000) >> 8);
}
/**
* Get the second of the two Latin-expansion CEs encoded in ce32.
* @see LATIN_EXPANSION_TAG
*/
static inline int64_t latinCE1FromCE32(uint32_t ce32) {
return ((ce32 & 0xff00) << 16) | COMMON_TERTIARY_CE;
}
/**
* Returns the data index from a special CE32.
*/
static inline int32_t indexFromCE32(uint32_t ce32) {
return static_cast<int32_t>(ce32 >> 13);
}
/**
* Returns the data length from a ce32.
*/
static inline int32_t lengthFromCE32(uint32_t ce32) {
return (ce32 >> 8) & 31;
}
/**
* Returns the digit value from a DIGIT_TAG ce32.
*/
static inline char digitFromCE32(uint32_t ce32) {
return static_cast<char>((ce32 >> 8) & 0xf);
}
/** Returns a 64-bit CE from a simple CE32 (not special). */
static inline int64_t ceFromSimpleCE32(uint32_t ce32) {
// normal form ppppsstt -> pppp0000ss00tt00
// assert (ce32 & 0xff) < SPECIAL_CE32_LOW_BYTE
return (static_cast<int64_t>(ce32 & 0xffff0000) << 32) | ((ce32 & 0xff00) << 16) | ((ce32 & 0xff) << 8);
}
/** Returns a 64-bit CE from a simple/long-primary/long-secondary CE32. */
static inline int64_t ceFromCE32(uint32_t ce32) {
uint32_t tertiary = ce32 & 0xff;
if(tertiary < SPECIAL_CE32_LOW_BYTE) {
// normal form ppppsstt -> pppp0000ss00tt00
return (static_cast<int64_t>(ce32 & 0xffff0000) << 32) | ((ce32 & 0xff00) << 16) | (tertiary << 8);
} else {
ce32 -= tertiary;
if((tertiary & 0xf) == LONG_PRIMARY_TAG) {
// long-primary form ppppppC1 -> pppppp00050000500
return (static_cast<int64_t>(ce32) << 32) | COMMON_SEC_AND_TER_CE;
} else {
// long-secondary form ssssttC2 -> 00000000sssstt00
// assert (tertiary & 0xf) == LONG_SECONDARY_TAG
return ce32;
}
}
}
/** Creates a CE from a primary weight. */
static inline int64_t makeCE(uint32_t p) {
return (static_cast<int64_t>(p) << 32) | COMMON_SEC_AND_TER_CE;
}
/**
* Creates a CE from a primary weight,
* 16-bit secondary/tertiary weights, and a 2-bit quaternary.
*/
static inline int64_t makeCE(uint32_t p, uint32_t s, uint32_t t, uint32_t q) {
return (static_cast<int64_t>(p) << 32) | (s << 16) | t | (q << 6);
}
/**
* Increments a 2-byte primary by a code point offset.
*/
static uint32_t incTwoBytePrimaryByOffset(uint32_t basePrimary, UBool isCompressible,
int32_t offset);
/**
* Increments a 3-byte primary by a code point offset.
*/
static uint32_t incThreeBytePrimaryByOffset(uint32_t basePrimary, UBool isCompressible,
int32_t offset);
/**
* Decrements a 2-byte primary by one range step (1..0x7f).
*/
static uint32_t decTwoBytePrimaryByOneStep(uint32_t basePrimary, UBool isCompressible, int32_t step);
/**
* Decrements a 3-byte primary by one range step (1..0x7f).
*/
static uint32_t decThreeBytePrimaryByOneStep(uint32_t basePrimary, UBool isCompressible, int32_t step);
/**
* Computes a 3-byte primary for c's OFFSET_TAG data "CE".
*/
static uint32_t getThreeBytePrimaryForOffsetData(UChar32 c, int64_t dataCE);
/**
* Returns the unassigned-character implicit primary weight for any valid code point c.
*/
static uint32_t unassignedPrimaryFromCodePoint(UChar32 c);
static inline int64_t unassignedCEFromCodePoint(UChar32 c) {
return makeCE(unassignedPrimaryFromCodePoint(c));
}
private:
Collation() = delete; // No instantiation.
};
U_NAMESPACE_END
#endif // !UCONFIG_NO_COLLATION
#endif // __COLLATION_H__

1744
deps/icu-small/source/i18n/collationbuilder.cpp vendored Normal file
View File

File diff suppressed because it is too large Load Diff

411
deps/icu-small/source/i18n/collationbuilder.h vendored Normal file
View File

@ -0,0 +1,411 @@
// © 2016 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html
/*
*******************************************************************************
* Copyright (C) 2013-2014, International Business Machines
* Corporation and others. All Rights Reserved.
*******************************************************************************
* collationbuilder.h
*
* created on: 2013may06
* created by: Markus W. Scherer
*/
#ifndef __COLLATIONBUILDER_H__
#define __COLLATIONBUILDER_H__
#include "unicode/utypes.h"
#if !UCONFIG_NO_COLLATION
#include "unicode/uniset.h"
#include "unicode/unistr.h"
#include "collationrootelements.h"
#include "collationruleparser.h"
#include "uvectr32.h"
#include "uvectr64.h"
struct UParseError;
U_NAMESPACE_BEGIN
struct CollationData;
struct CollationTailoring;
class CEFinalizer;
class CollationDataBuilder;
class Normalizer2;
class Normalizer2Impl;
class U_I18N_API CollationBuilder : public CollationRuleParser::Sink {
public:
CollationBuilder(const CollationTailoring *b, UBool icu4xMode, UErrorCode &errorCode);
CollationBuilder(const CollationTailoring *base, UErrorCode &errorCode);
virtual ~CollationBuilder();
void disableFastLatin() { fastLatinEnabled = false; }
CollationTailoring *parseAndBuild(const UnicodeString &ruleString,
const UVersionInfo rulesVersion,
CollationRuleParser::Importer *importer,
UParseError *outParseError,
UErrorCode &errorCode);
const char *getErrorReason() const { return errorReason; }
private:
friend class CEFinalizer;
/** Implements CollationRuleParser::Sink. */
virtual void addReset(int32_t strength, const UnicodeString &str,
const char *&errorReason, UErrorCode &errorCode) override;
/**
* Returns the secondary or tertiary weight preceding the current node's weight.
* node=nodes[index].
*/
uint32_t getWeight16Before(int32_t index, int64_t node, int32_t level);
int64_t getSpecialResetPosition(const UnicodeString &str,
const char *&parserErrorReason, UErrorCode &errorCode);
/** Implements CollationRuleParser::Sink. */
virtual void addRelation(int32_t strength, const UnicodeString &prefix,
const UnicodeString &str, const UnicodeString &extension,
const char *&errorReason, UErrorCode &errorCode) override;
/**
* Picks one of the current CEs and finds or inserts a node in the graph
* for the CE + strength.
*/
int32_t findOrInsertNodeForCEs(int32_t strength, const char *&parserErrorReason,
UErrorCode &errorCode);
int32_t findOrInsertNodeForRootCE(int64_t ce, int32_t strength, UErrorCode &errorCode);
/** Finds or inserts the node for a root CE's primary weight. */
int32_t findOrInsertNodeForPrimary(uint32_t p, UErrorCode &errorCode);
/** Finds or inserts the node for a secondary or tertiary weight. */
int32_t findOrInsertWeakNode(int32_t index, uint32_t weight16, int32_t level,
UErrorCode &errorCode);
/**
* Makes and inserts a new tailored node into the list, after the one at index.
* Skips over nodes of weaker strength to maintain collation order
* ("postpone insertion").
* @return the new node's index
*/
int32_t insertTailoredNodeAfter(int32_t index, int32_t strength, UErrorCode &errorCode);
/**
* Inserts a new node into the list, between list-adjacent items.
* The node's previous and next indexes must not be set yet.
* @return the new node's index
*/
int32_t insertNodeBetween(int32_t index, int32_t nextIndex, int64_t node,
UErrorCode &errorCode);
/**
* Finds the node which implies or contains a common=05 weight of the given strength
* (secondary or tertiary), if the current node is stronger.
* Skips weaker nodes and tailored nodes if the current node is stronger
* and is followed by an explicit-common-weight node.
* Always returns the input index if that node is no stronger than the given strength.
*/
int32_t findCommonNode(int32_t index, int32_t strength) const;
void setCaseBits(const UnicodeString &nfdString,
const char *&parserErrorReason, UErrorCode &errorCode);
/** Implements CollationRuleParser::Sink. */
virtual void suppressContractions(const UnicodeSet &set, const char *&parserErrorReason,
UErrorCode &errorCode) override;
/** Implements CollationRuleParser::Sink. */
virtual void optimize(const UnicodeSet &set, const char *&parserErrorReason,
UErrorCode &errorCode) override;
/**
* Adds the mapping and its canonical closure.
* Takes ce32=dataBuilder->encodeCEs(...) so that the data builder
* need not re-encode the CEs multiple times.
*/
uint32_t addWithClosure(const UnicodeString &nfdPrefix, const UnicodeString &nfdString,
const int64_t newCEs[], int32_t newCEsLength, uint32_t ce32,
UErrorCode &errorCode);
uint32_t addOnlyClosure(const UnicodeString &nfdPrefix, const UnicodeString &nfdString,
const int64_t newCEs[], int32_t newCEsLength, uint32_t ce32,
UErrorCode &errorCode);
void addTailComposites(const UnicodeString &nfdPrefix, const UnicodeString &nfdString,
UErrorCode &errorCode);
UBool mergeCompositeIntoString(const UnicodeString &nfdString, int32_t indexAfterLastStarter,
UChar32 composite, const UnicodeString &decomp,
UnicodeString &newNFDString, UnicodeString &newString,
UErrorCode &errorCode) const;
UBool ignorePrefix(const UnicodeString &s, UErrorCode &errorCode) const;
UBool ignoreString(const UnicodeString &s, UErrorCode &errorCode) const;
UBool isFCD(const UnicodeString &s, UErrorCode &errorCode) const;
void closeOverComposites(UErrorCode &errorCode);
uint32_t addIfDifferent(const UnicodeString &prefix, const UnicodeString &str,
const int64_t newCEs[], int32_t newCEsLength, uint32_t ce32,
UErrorCode &errorCode);
static UBool sameCEs(const int64_t ces1[], int32_t ces1Length,
const int64_t ces2[], int32_t ces2Length);
/**
* Walks the tailoring graph and overwrites tailored nodes with new CEs.
* After this, the graph is destroyed.
* The nodes array can then be used only as a source of tailored CEs.
*/
void makeTailoredCEs(UErrorCode &errorCode);
/**
* Counts the tailored nodes of the given strength up to the next node
* which is either stronger or has an explicit weight of this strength.
*/
static int32_t countTailoredNodes(const int64_t *nodesArray, int32_t i, int32_t strength);
/** Replaces temporary CEs with the final CEs they point to. */
void finalizeCEs(UErrorCode &errorCode);
/**
* Encodes "temporary CE" data into a CE that fits into the CE32 data structure,
* with 2-byte primary, 1-byte secondary and 6-bit tertiary,
* with valid CE byte values.
*
* The index must not exceed 20 bits (0xfffff).
* The strength must fit into 2 bits (UCOL_PRIMARY..UCOL_QUATERNARY).
*
* Temporary CEs are distinguished from real CEs by their use of
* secondary weights 06..45 which are otherwise reserved for compressed sort keys.
*
* The case bits are unused and available.
*/
static inline int64_t tempCEFromIndexAndStrength(int32_t index, int32_t strength) {
return
// CE byte offsets, to ensure valid CE bytes, and case bits 11
INT64_C(0x4040000006002000) +
// index bits 19..13 -> primary byte 1 = CE bits 63..56 (byte values 40..BF)
(static_cast<int64_t>(index & 0xfe000) << 43) +
// index bits 12..6 -> primary byte 2 = CE bits 55..48 (byte values 40..BF)
(static_cast<int64_t>(index & 0x1fc0) << 42) +
// index bits 5..0 -> secondary byte 1 = CE bits 31..24 (byte values 06..45)
((index & 0x3f) << 24) +
// strength bits 1..0 -> tertiary byte 1 = CE bits 13..8 (byte values 20..23)
(strength << 8);
}
static inline int32_t indexFromTempCE(int64_t tempCE) {
tempCE -= INT64_C(0x4040000006002000);
return
(static_cast<int32_t>(tempCE >> 43) & 0xfe000) |
(static_cast<int32_t>(tempCE >> 42) & 0x1fc0) |
(static_cast<int32_t>(tempCE >> 24) & 0x3f);
}
static inline int32_t strengthFromTempCE(int64_t tempCE) {
return (static_cast<int32_t>(tempCE) >> 8) & 3;
}
static inline UBool isTempCE(int64_t ce) {
uint32_t sec = static_cast<uint32_t>(ce) >> 24;
return 6 <= sec && sec <= 0x45;
}
static inline int32_t indexFromTempCE32(uint32_t tempCE32) {
tempCE32 -= 0x40400620;
return
(static_cast<int32_t>(tempCE32 >> 11) & 0xfe000) |
(static_cast<int32_t>(tempCE32 >> 10) & 0x1fc0) |
(static_cast<int32_t>(tempCE32 >> 8) & 0x3f);
}
static inline UBool isTempCE32(uint32_t ce32) {
return
(ce32 & 0xff) >= 2 && // not a long-primary/long-secondary CE32
6 <= ((ce32 >> 8) & 0xff) && ((ce32 >> 8) & 0xff) <= 0x45;
}
static int32_t ceStrength(int64_t ce);
/** At most 1M nodes, limited by the 20 bits in node bit fields. */
static const int32_t MAX_INDEX = 0xfffff;
/**
* Node bit 6 is set on a primary node if there are nodes
* with secondary values below the common secondary weight (05).
*/
static const int32_t HAS_BEFORE2 = 0x40;
/**
* Node bit 5 is set on a primary or secondary node if there are nodes
* with tertiary values below the common tertiary weight (05).
*/
static const int32_t HAS_BEFORE3 = 0x20;
/**
* Node bit 3 distinguishes a tailored node, which has no weight value,
* from a node with an explicit (root or default) weight.
*/
static const int32_t IS_TAILORED = 8;
static inline int64_t nodeFromWeight32(uint32_t weight32) {
return static_cast<int64_t>(weight32) << 32;
}
static inline int64_t nodeFromWeight16(uint32_t weight16) {
return static_cast<int64_t>(weight16) << 48;
}
static inline int64_t nodeFromPreviousIndex(int32_t previous) {
return static_cast<int64_t>(previous) << 28;
}
static inline int64_t nodeFromNextIndex(int32_t next) {
return next << 8;
}
static inline int64_t nodeFromStrength(int32_t strength) {
return strength;
}
static inline uint32_t weight32FromNode(int64_t node) {
return static_cast<uint32_t>(node >> 32);
}
static inline uint32_t weight16FromNode(int64_t node) {
return static_cast<uint32_t>(node >> 48) & 0xffff;
}
static inline int32_t previousIndexFromNode(int64_t node) {
return static_cast<int32_t>(node >> 28) & MAX_INDEX;
}
static inline int32_t nextIndexFromNode(int64_t node) {
return (static_cast<int32_t>(node) >> 8) & MAX_INDEX;
}
static inline int32_t strengthFromNode(int64_t node) {
return static_cast<int32_t>(node) & 3;
}
static inline UBool nodeHasBefore2(int64_t node) {
return (node & HAS_BEFORE2) != 0;
}
static inline UBool nodeHasBefore3(int64_t node) {
return (node & HAS_BEFORE3) != 0;
}
static inline UBool nodeHasAnyBefore(int64_t node) {
return (node & (HAS_BEFORE2 | HAS_BEFORE3)) != 0;
}
static inline UBool isTailoredNode(int64_t node) {
return (node & IS_TAILORED) != 0;
}
static inline int64_t changeNodePreviousIndex(int64_t node, int32_t previous) {
return (node & INT64_C(0xffff00000fffffff)) | nodeFromPreviousIndex(previous);
}
static inline int64_t changeNodeNextIndex(int64_t node, int32_t next) {
return (node & INT64_C(0xfffffffff00000ff)) | nodeFromNextIndex(next);
}
const Normalizer2 &nfd, &fcd;
const Normalizer2Impl &nfcImpl;
const CollationTailoring *base;
const CollationData *baseData;
const CollationRootElements rootElements;
uint32_t variableTop;
CollationDataBuilder *dataBuilder;
UBool fastLatinEnabled;
UBool icu4xMode;
UnicodeSet optimizeSet;
const char *errorReason;
int64_t ces[Collation::MAX_EXPANSION_LENGTH];
int32_t cesLength;
/**
* Indexes of nodes with root primary weights, sorted by primary.
* Compact form of a TreeMap from root primary to node index.
*
* This is a performance optimization for finding reset positions.
* Without this, we would have to search through the entire nodes list.
* It also allows storing root primary weights in list head nodes,
* without previous index, leaving room in root primary nodes for 32-bit primary weights.
*/
UVector32 rootPrimaryIndexes;
/**
* Data structure for assigning tailored weights and CEs.
* Doubly-linked lists of nodes in mostly collation order.
* Each list starts with a root primary node and ends with a nextIndex of 0.
*
* When there are any nodes in the list, then there is always a root primary node at index 0.
* This allows some code not to have to check explicitly for nextIndex==0.
*
* Root primary nodes have 32-bit weights but do not have previous indexes.
* All other nodes have at most 16-bit weights and do have previous indexes.
*
* Nodes with explicit weights store root collator weights,
* or default weak weights (e.g., secondary 05) for stronger nodes.
* "Tailored" nodes, with the IS_TAILORED bit set,
* do not store explicit weights but rather
* create a difference of a certain strength from the preceding node.
*
* A root node is followed by either
* - a root/default node of the same strength, or
* - a root/default node of the next-weaker strength, or
* - a tailored node of the same strength.
*
* A node of a given strength normally implies "common" weights on weaker levels.
*
* A node with HAS_BEFORE2 must be immediately followed by
* a secondary node with an explicit below-common weight, then a secondary tailored node,
* and later an explicit common-secondary node.
* The below-common weight can be a root weight,
* or it can be BEFORE_WEIGHT16 for tailoring before an implied common weight
* or before the lowest root weight.
* (&[before 2] resets to an explicit secondary node so that
* the following addRelation(secondary) tailors right after that.
* If we did not have this node and instead were to reset on the primary node,
* then addRelation(secondary) would skip forward to the the COMMON_WEIGHT16 node.)
*
* If the flag is not set, then there are no explicit secondary nodes
* with the common or lower weights.
*
* Same for HAS_BEFORE3 for tertiary nodes and weights.
* A node must not have both flags set.
*
* Tailored CEs are initially represented in a CollationDataBuilder as temporary CEs
* which point to stable indexes in this list,
* and temporary CEs stored in a CollationDataBuilder only point to tailored nodes.
*
* A temporary CE in the ces[] array may point to a non-tailored reset-before-position node,
* until the next relation is added.
*
* At the end, the tailored weights are allocated as necessary,
* then the tailored nodes are replaced with final CEs,
* and the CollationData is rewritten by replacing temporary CEs with final ones.
*
* We cannot simply insert new nodes in the middle of the array
* because that would invalidate the indexes stored in existing temporary CEs.
* We need to use a linked graph with stable indexes to existing nodes.
* A doubly-linked list seems easiest to maintain.
*
* Each node is stored as an int64_t, with its fields stored as bit fields.
*
* Root primary node:
* - primary weight: 32 bits 63..32
* - reserved/unused/zero: 4 bits 31..28
*
* Weaker root nodes & tailored nodes:
* - a weight: 16 bits 63..48
* + a root or default weight for a non-tailored node
* + unused/zero for a tailored node
* - index to the previous node: 20 bits 47..28
*
* All types of nodes:
* - index to the next node: 20 bits 27..8
* + nextIndex=0 in last node per root-primary list
* - reserved/unused/zero bits: bits 7, 4, 2
* - HAS_BEFORE2: bit 6
* - HAS_BEFORE3: bit 5
* - IS_TAILORED: bit 3
* - the difference strength (primary/secondary/tertiary/quaternary): 2 bits 1..0
*
* We could allocate structs with pointers, but we would have to store them
* in a pointer list so that they can be indexed from temporary CEs,
* and they would require more memory allocations.
*/
UVector64 nodes;
};
U_NAMESPACE_END
#endif // !UCONFIG_NO_COLLATION
#endif // __COLLATIONBUILDER_H__

356
deps/icu-small/source/i18n/collationcompare.cpp vendored Normal file
View File

@ -0,0 +1,356 @@
// © 2016 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html
/*
*******************************************************************************
* Copyright (C) 1996-2015, International Business Machines
* Corporation and others. All Rights Reserved.
*******************************************************************************
* collationcompare.cpp
*
* created on: 2012feb14 with new and old collation code
* created by: Markus W. Scherer
*/
#include "unicode/utypes.h"
#if !UCONFIG_NO_COLLATION
#include "unicode/ucol.h"
#include "cmemory.h"
#include "collation.h"
#include "collationcompare.h"
#include "collationiterator.h"
#include "collationsettings.h"
#include "uassert.h"
U_NAMESPACE_BEGIN
UCollationResult
CollationCompare::compareUpToQuaternary(CollationIterator &left, CollationIterator &right,
const CollationSettings &settings,
UErrorCode &errorCode) {
if(U_FAILURE(errorCode)) { return UCOL_EQUAL; }
int32_t options = settings.options;
uint32_t variableTop;
if((options & CollationSettings::ALTERNATE_MASK) == 0) {
variableTop = 0;
} else {
// +1 so that we can use "<" and primary ignorables test out early.
variableTop = settings.variableTop + 1;
}
UBool anyVariable = false;
// Fetch CEs, compare primaries, store secondary & tertiary weights.
for(;;) {
// We fetch CEs until we get a non-ignorable primary or reach the end.
uint32_t leftPrimary;
do {
int64_t ce = left.nextCE(errorCode);
leftPrimary = static_cast<uint32_t>(ce >> 32);
if(leftPrimary < variableTop && leftPrimary > Collation::MERGE_SEPARATOR_PRIMARY) {
// Variable CE, shift it to quaternary level.
// Ignore all following primary ignorables, and shift further variable CEs.
anyVariable = true;
do {
// Store only the primary of the variable CE.
left.setCurrentCE(ce & INT64_C(0xffffffff00000000));
for(;;) {
ce = left.nextCE(errorCode);
leftPrimary = static_cast<uint32_t>(ce >> 32);
if(leftPrimary == 0) {
left.setCurrentCE(0);
} else {
break;
}
}
} while(leftPrimary < variableTop &&
leftPrimary > Collation::MERGE_SEPARATOR_PRIMARY);
}
} while(leftPrimary == 0);
uint32_t rightPrimary;
do {
int64_t ce = right.nextCE(errorCode);
rightPrimary = static_cast<uint32_t>(ce >> 32);
if(rightPrimary < variableTop && rightPrimary > Collation::MERGE_SEPARATOR_PRIMARY) {
// Variable CE, shift it to quaternary level.
// Ignore all following primary ignorables, and shift further variable CEs.
anyVariable = true;
do {
// Store only the primary of the variable CE.
right.setCurrentCE(ce & INT64_C(0xffffffff00000000));
for(;;) {
ce = right.nextCE(errorCode);
rightPrimary = static_cast<uint32_t>(ce >> 32);
if(rightPrimary == 0) {
right.setCurrentCE(0);
} else {
break;
}
}
} while(rightPrimary < variableTop &&
rightPrimary > Collation::MERGE_SEPARATOR_PRIMARY);
}
} while(rightPrimary == 0);
if(leftPrimary != rightPrimary) {
// Return the primary difference, with script reordering.
if(settings.hasReordering()) {
leftPrimary = settings.reorder(leftPrimary);
rightPrimary = settings.reorder(rightPrimary);
}
return (leftPrimary < rightPrimary) ? UCOL_LESS : UCOL_GREATER;
}
if(leftPrimary == Collation::NO_CE_PRIMARY) { break; }
}
if(U_FAILURE(errorCode)) { return UCOL_EQUAL; }
// Compare the buffered secondary & tertiary weights.
// We might skip the secondary level but continue with the case level
// which is turned on separately.
if(CollationSettings::getStrength(options) >= UCOL_SECONDARY) {
if((options & CollationSettings::BACKWARD_SECONDARY) == 0) {
int32_t leftIndex = 0;
int32_t rightIndex = 0;
for(;;) {
uint32_t leftSecondary;
do {
leftSecondary = static_cast<uint32_t>(left.getCE(leftIndex++)) >> 16;
} while(leftSecondary == 0);
uint32_t rightSecondary;
do {
rightSecondary = static_cast<uint32_t>(right.getCE(rightIndex++)) >> 16;
} while(rightSecondary == 0);
if(leftSecondary != rightSecondary) {
return (leftSecondary < rightSecondary) ? UCOL_LESS : UCOL_GREATER;
}
if(leftSecondary == Collation::NO_CE_WEIGHT16) { break; }
}
} else {
// The backwards secondary level compares secondary weights backwards
// within segments separated by the merge separator (U+FFFE, weight 02).
int32_t leftStart = 0;
int32_t rightStart = 0;
for(;;) {
// Find the merge separator or the NO_CE terminator.
uint32_t p;
int32_t leftLimit = leftStart;
while ((p = static_cast<uint32_t>(left.getCE(leftLimit) >> 32)) >
Collation::MERGE_SEPARATOR_PRIMARY ||
p == 0) {
++leftLimit;
}
int32_t rightLimit = rightStart;
while ((p = static_cast<uint32_t>(right.getCE(rightLimit) >> 32)) >
Collation::MERGE_SEPARATOR_PRIMARY ||
p == 0) {
++rightLimit;
}
// Compare the segments.
int32_t leftIndex = leftLimit;
int32_t rightIndex = rightLimit;
for(;;) {
int32_t leftSecondary = 0;
while(leftSecondary == 0 && leftIndex > leftStart) {
leftSecondary = static_cast<uint32_t>(left.getCE(--leftIndex)) >> 16;
}
int32_t rightSecondary = 0;
while(rightSecondary == 0 && rightIndex > rightStart) {
rightSecondary = static_cast<uint32_t>(right.getCE(--rightIndex)) >> 16;
}
if(leftSecondary != rightSecondary) {
return (leftSecondary < rightSecondary) ? UCOL_LESS : UCOL_GREATER;
}
if(leftSecondary == 0) { break; }
}
// Did we reach the end of either string?
// Both strings have the same number of merge separators,
// or else there would have been a primary-level difference.
U_ASSERT(left.getCE(leftLimit) == right.getCE(rightLimit));
if(p == Collation::NO_CE_PRIMARY) { break; }
// Skip both merge separators and continue.
leftStart = leftLimit + 1;
rightStart = rightLimit + 1;
}
}
}
if((options & CollationSettings::CASE_LEVEL) != 0) {
int32_t strength = CollationSettings::getStrength(options);
int32_t leftIndex = 0;
int32_t rightIndex = 0;
for(;;) {
uint32_t leftCase, leftLower32, rightCase;
if(strength == UCOL_PRIMARY) {
// Primary+caseLevel: Ignore case level weights of primary ignorables.
// Otherwise we would get a-umlaut > a
// which is not desirable for accent-insensitive sorting.
// Check for (lower 32 bits) == 0 as well because variable CEs are stored
// with only primary weights.
int64_t ce;
do {
ce = left.getCE(leftIndex++);
leftCase = static_cast<uint32_t>(ce);
} while (static_cast<uint32_t>(ce >> 32) == 0 || leftCase == 0);
leftLower32 = leftCase;
leftCase &= 0xc000;
do {
ce = right.getCE(rightIndex++);
rightCase = static_cast<uint32_t>(ce);
} while (static_cast<uint32_t>(ce >> 32) == 0 || rightCase == 0);
rightCase &= 0xc000;
} else {
// Secondary+caseLevel: By analogy with the above,
// ignore case level weights of secondary ignorables.
//
// Note: A tertiary CE has uppercase case bits (0.0.ut)
// to keep tertiary+caseFirst well-formed.
//
// Tertiary+caseLevel: Also ignore case level weights of secondary ignorables.
// Otherwise a tertiary CE's uppercase would be no greater than
// a primary/secondary CE's uppercase.
// (See UCA well-formedness condition 2.)
// We could construct a special case weight higher than uppercase,
// but it's simpler to always ignore case weights of secondary ignorables,
// turning 0.0.ut into 0.0.0.t.
// (See LDML Collation, Case Parameters.)
do {
leftCase = static_cast<uint32_t>(left.getCE(leftIndex++));
} while(leftCase <= 0xffff);
leftLower32 = leftCase;
leftCase &= 0xc000;
do {
rightCase = static_cast<uint32_t>(right.getCE(rightIndex++));
} while(rightCase <= 0xffff);
rightCase &= 0xc000;
}
// No need to handle NO_CE and MERGE_SEPARATOR specially:
// There is one case weight for each previous-level weight,
// so level length differences were handled there.
if(leftCase != rightCase) {
if((options & CollationSettings::UPPER_FIRST) == 0) {
return (leftCase < rightCase) ? UCOL_LESS : UCOL_GREATER;
} else {
return (leftCase < rightCase) ? UCOL_GREATER : UCOL_LESS;
}
}
if((leftLower32 >> 16) == Collation::NO_CE_WEIGHT16) { break; }
}
}
if(CollationSettings::getStrength(options) <= UCOL_SECONDARY) { return UCOL_EQUAL; }
uint32_t tertiaryMask = CollationSettings::getTertiaryMask(options);
int32_t leftIndex = 0;
int32_t rightIndex = 0;
uint32_t anyQuaternaries = 0;
for(;;) {
uint32_t leftLower32, leftTertiary;
do {
leftLower32 = static_cast<uint32_t>(left.getCE(leftIndex++));
anyQuaternaries |= leftLower32;
U_ASSERT((leftLower32 & Collation::ONLY_TERTIARY_MASK) != 0 ||
(leftLower32 & 0xc0c0) == 0);
leftTertiary = leftLower32 & tertiaryMask;
} while(leftTertiary == 0);
uint32_t rightLower32, rightTertiary;
do {
rightLower32 = static_cast<uint32_t>(right.getCE(rightIndex++));
anyQuaternaries |= rightLower32;
U_ASSERT((rightLower32 & Collation::ONLY_TERTIARY_MASK) != 0 ||
(rightLower32 & 0xc0c0) == 0);
rightTertiary = rightLower32 & tertiaryMask;
} while(rightTertiary == 0);
if(leftTertiary != rightTertiary) {
if(CollationSettings::sortsTertiaryUpperCaseFirst(options)) {
// Pass through NO_CE and keep real tertiary weights larger than that.
// Do not change the artificial uppercase weight of a tertiary CE (0.0.ut),
// to keep tertiary CEs well-formed.
// Their case+tertiary weights must be greater than those of
// primary and secondary CEs.
if(leftTertiary > Collation::NO_CE_WEIGHT16) {
if(leftLower32 > 0xffff) {
leftTertiary ^= 0xc000;
} else {
leftTertiary += 0x4000;
}
}
if(rightTertiary > Collation::NO_CE_WEIGHT16) {
if(rightLower32 > 0xffff) {
rightTertiary ^= 0xc000;
} else {
rightTertiary += 0x4000;
}
}
}
return (leftTertiary < rightTertiary) ? UCOL_LESS : UCOL_GREATER;
}
if(leftTertiary == Collation::NO_CE_WEIGHT16) { break; }
}
if(CollationSettings::getStrength(options) <= UCOL_TERTIARY) { return UCOL_EQUAL; }
if(!anyVariable && (anyQuaternaries & 0xc0) == 0) {
// If there are no "variable" CEs and no non-zero quaternary weights,
// then there are no quaternary differences.
return UCOL_EQUAL;
}
leftIndex = 0;
rightIndex = 0;
for(;;) {
uint32_t leftQuaternary;
do {
int64_t ce = left.getCE(leftIndex++);
leftQuaternary = static_cast<uint32_t>(ce) & 0xffff;
if(leftQuaternary <= Collation::NO_CE_WEIGHT16) {
// Variable primary or completely ignorable or NO_CE.
leftQuaternary = static_cast<uint32_t>(ce >> 32);
} else {
// Regular CE, not tertiary ignorable.
// Preserve the quaternary weight in bits 7..6.
leftQuaternary |= 0xffffff3f;
}
} while(leftQuaternary == 0);
uint32_t rightQuaternary;
do {
int64_t ce = right.getCE(rightIndex++);
rightQuaternary = static_cast<uint32_t>(ce) & 0xffff;
if(rightQuaternary <= Collation::NO_CE_WEIGHT16) {
// Variable primary or completely ignorable or NO_CE.
rightQuaternary = static_cast<uint32_t>(ce >> 32);
} else {
// Regular CE, not tertiary ignorable.
// Preserve the quaternary weight in bits 7..6.
rightQuaternary |= 0xffffff3f;
}
} while(rightQuaternary == 0);
if(leftQuaternary != rightQuaternary) {
// Return the difference, with script reordering.
if(settings.hasReordering()) {
leftQuaternary = settings.reorder(leftQuaternary);
rightQuaternary = settings.reorder(rightQuaternary);
}
return (leftQuaternary < rightQuaternary) ? UCOL_LESS : UCOL_GREATER;
}
if(leftQuaternary == Collation::NO_CE_PRIMARY) { break; }
}
return UCOL_EQUAL;
}
U_NAMESPACE_END
#endif // !UCONFIG_NO_COLLATION

38
deps/icu-small/source/i18n/collationcompare.h vendored Normal file
View File

@ -0,0 +1,38 @@
// © 2016 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html
/*
*******************************************************************************
* Copyright (C) 1996-2014, International Business Machines
* Corporation and others. All Rights Reserved.
*******************************************************************************
* collationcompare.h
*
* created on: 2012feb14 with new and old collation code
* created by: Markus W. Scherer
*/
#ifndef __COLLATIONCOMPARE_H__
#define __COLLATIONCOMPARE_H__
#include "unicode/utypes.h"
#if !UCONFIG_NO_COLLATION
#include "unicode/ucol.h"
U_NAMESPACE_BEGIN
class CollationIterator;
struct CollationSettings;
class U_I18N_API CollationCompare /* not : public UObject because all methods are static */ {
public:
static UCollationResult compareUpToQuaternary(CollationIterator &left, CollationIterator &right,
const CollationSettings &settings,
UErrorCode &errorCode);
};
U_NAMESPACE_END
#endif // !UCONFIG_NO_COLLATION
#endif // __COLLATIONCOMPARE_H__

390
deps/icu-small/source/i18n/collationdata.cpp vendored Normal file
View File

@ -0,0 +1,390 @@
// © 2016 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html
/*
*******************************************************************************
* Copyright (C) 2012-2015, International Business Machines
* Corporation and others. All Rights Reserved.
*******************************************************************************
* collationdata.cpp
*
* created on: 2012jul28
* created by: Markus W. Scherer
*/
#include "unicode/utypes.h"
#if !UCONFIG_NO_COLLATION
#include "unicode/ucol.h"
#include "unicode/udata.h"
#include "unicode/uscript.h"
#include "cmemory.h"
#include "collation.h"
#include "collationdata.h"
#include "uassert.h"
#include "utrie2.h"
#include "uvectr32.h"
U_NAMESPACE_BEGIN
uint32_t
CollationData::getIndirectCE32(uint32_t ce32) const {
U_ASSERT(Collation::isSpecialCE32(ce32));
int32_t tag = Collation::tagFromCE32(ce32);
if(tag == Collation::DIGIT_TAG) {
// Fetch the non-numeric-collation CE32.
ce32 = ce32s[Collation::indexFromCE32(ce32)];
} else if(tag == Collation::LEAD_SURROGATE_TAG) {
ce32 = Collation::UNASSIGNED_CE32;
} else if(tag == Collation::U0000_TAG) {
// Fetch the normal ce32 for U+0000.
ce32 = ce32s[0];
}
return ce32;
}
uint32_t
CollationData::getFinalCE32(uint32_t ce32) const {
if(Collation::isSpecialCE32(ce32)) {
ce32 = getIndirectCE32(ce32);
}
return ce32;
}
int64_t
CollationData::getSingleCE(UChar32 c, UErrorCode &errorCode) const {
if(U_FAILURE(errorCode)) { return 0; }
// Keep parallel with CollationDataBuilder::getSingleCE().
const CollationData *d;
uint32_t ce32 = getCE32(c);
if(ce32 == Collation::FALLBACK_CE32) {
d = base;
ce32 = base->getCE32(c);
} else {
d = this;
}
while(Collation::isSpecialCE32(ce32)) {
switch(Collation::tagFromCE32(ce32)) {
case Collation::LATIN_EXPANSION_TAG:
case Collation::BUILDER_DATA_TAG:
case Collation::PREFIX_TAG:
case Collation::CONTRACTION_TAG:
case Collation::HANGUL_TAG:
case Collation::LEAD_SURROGATE_TAG:
errorCode = U_UNSUPPORTED_ERROR;
return 0;
case Collation::FALLBACK_TAG:
case Collation::RESERVED_TAG_3:
errorCode = U_INTERNAL_PROGRAM_ERROR;
return 0;
case Collation::LONG_PRIMARY_TAG:
return Collation::ceFromLongPrimaryCE32(ce32);
case Collation::LONG_SECONDARY_TAG:
return Collation::ceFromLongSecondaryCE32(ce32);
case Collation::EXPANSION32_TAG:
if(Collation::lengthFromCE32(ce32) == 1) {
ce32 = d->ce32s[Collation::indexFromCE32(ce32)];
break;
} else {
errorCode = U_UNSUPPORTED_ERROR;
return 0;
}
case Collation::EXPANSION_TAG: {
if(Collation::lengthFromCE32(ce32) == 1) {
return d->ces[Collation::indexFromCE32(ce32)];
} else {
errorCode = U_UNSUPPORTED_ERROR;
return 0;
}
}
case Collation::DIGIT_TAG:
// Fetch the non-numeric-collation CE32 and continue.
ce32 = d->ce32s[Collation::indexFromCE32(ce32)];
break;
case Collation::U0000_TAG:
U_ASSERT(c == 0);
// Fetch the normal ce32 for U+0000 and continue.
ce32 = d->ce32s[0];
break;
case Collation::OFFSET_TAG:
return d->getCEFromOffsetCE32(c, ce32);
case Collation::IMPLICIT_TAG:
return Collation::unassignedCEFromCodePoint(c);
}
}
return Collation::ceFromSimpleCE32(ce32);
}
uint32_t
CollationData::getFirstPrimaryForGroup(int32_t script) const {
int32_t index = getScriptIndex(script);
return index == 0 ? 0 : static_cast<uint32_t>(scriptStarts[index]) << 16;
}
uint32_t
CollationData::getLastPrimaryForGroup(int32_t script) const {
int32_t index = getScriptIndex(script);
if(index == 0) {
return 0;
}
uint32_t limit = scriptStarts[index + 1];
return (limit << 16) - 1;
}
int32_t
CollationData::getGroupForPrimary(uint32_t p) const {
p >>= 16;
if(p < scriptStarts[1] || scriptStarts[scriptStartsLength - 1] <= p) {
return -1;
}
int32_t index = 1;
while(p >= scriptStarts[index + 1]) { ++index; }
for(int32_t i = 0; i < numScripts; ++i) {
if(scriptsIndex[i] == index) {
return i;
}
}
for(int32_t i = 0; i < MAX_NUM_SPECIAL_REORDER_CODES; ++i) {
if(scriptsIndex[numScripts + i] == index) {
return UCOL_REORDER_CODE_FIRST + i;
}
}
return -1;
}
int32_t
CollationData::getScriptIndex(int32_t script) const {
if(script < 0) {
return 0;
} else if(script < numScripts) {
return scriptsIndex[script];
} else if(script < UCOL_REORDER_CODE_FIRST) {
return 0;
} else {
script -= UCOL_REORDER_CODE_FIRST;
if(script < MAX_NUM_SPECIAL_REORDER_CODES) {
return scriptsIndex[numScripts + script];
} else {
return 0;
}
}
}
int32_t
CollationData::getEquivalentScripts(int32_t script,
int32_t dest[], int32_t capacity,
UErrorCode &errorCode) const {
if(U_FAILURE(errorCode)) { return 0; }
int32_t index = getScriptIndex(script);
if(index == 0) { return 0; }
if(script >= UCOL_REORDER_CODE_FIRST) {
// Special groups have no aliases.
if(capacity > 0) {
dest[0] = script;
} else {
errorCode = U_BUFFER_OVERFLOW_ERROR;
}
return 1;
}
int32_t length = 0;
for(int32_t i = 0; i < numScripts; ++i) {
if(scriptsIndex[i] == index) {
if(length < capacity) {
dest[length] = i;
}
++length;
}
}
if(length > capacity) {
errorCode = U_BUFFER_OVERFLOW_ERROR;
}
return length;
}
void
CollationData::makeReorderRanges(const int32_t *reorder, int32_t length,
UVector32 &ranges, UErrorCode &errorCode) const {
makeReorderRanges(reorder, length, false, ranges, errorCode);
}
void
CollationData::makeReorderRanges(const int32_t *reorder, int32_t length,
UBool latinMustMove,
UVector32 &ranges, UErrorCode &errorCode) const {
if(U_FAILURE(errorCode)) { return; }
ranges.removeAllElements();
if(length == 0 || (length == 1 && reorder[0] == USCRIPT_UNKNOWN)) {
return;
}
// Maps each script-or-group range to a new lead byte.
uint8_t table[MAX_NUM_SCRIPT_RANGES];
uprv_memset(table, 0, sizeof(table));
{
// Set "don't care" values for reserved ranges.
int32_t index = scriptsIndex[
numScripts + REORDER_RESERVED_BEFORE_LATIN - UCOL_REORDER_CODE_FIRST];
if(index != 0) {
table[index] = 0xff;
}
index = scriptsIndex[
numScripts + REORDER_RESERVED_AFTER_LATIN - UCOL_REORDER_CODE_FIRST];
if(index != 0) {
table[index] = 0xff;
}
}
// Never reorder special low and high primary lead bytes.
U_ASSERT(scriptStartsLength >= 2);
U_ASSERT(scriptStarts[0] == 0);
int32_t lowStart = scriptStarts[1];
U_ASSERT(lowStart == ((Collation::MERGE_SEPARATOR_BYTE + 1) << 8));
int32_t highLimit = scriptStarts[scriptStartsLength - 1];
U_ASSERT(highLimit == (Collation::TRAIL_WEIGHT_BYTE << 8));
// Get the set of special reorder codes in the input list.
// This supports a fixed number of special reorder codes;
// it works for data with codes beyond UCOL_REORDER_CODE_LIMIT.
uint32_t specials = 0;
for(int32_t i = 0; i < length; ++i) {
int32_t reorderCode = reorder[i] - UCOL_REORDER_CODE_FIRST;
if(0 <= reorderCode && reorderCode < MAX_NUM_SPECIAL_REORDER_CODES) {
specials |= static_cast<uint32_t>(1) << reorderCode;
}
}
// Start the reordering with the special low reorder codes that do not occur in the input.
for(int32_t i = 0; i < MAX_NUM_SPECIAL_REORDER_CODES; ++i) {
int32_t index = scriptsIndex[numScripts + i];
if (index != 0 && (specials & (static_cast<uint32_t>(1) << i)) == 0) {
lowStart = addLowScriptRange(table, index, lowStart);
}
}
// Skip the reserved range before Latin if Latin is the first script,
// so that we do not move it unnecessarily.
int32_t skippedReserved = 0;
if(specials == 0 && reorder[0] == USCRIPT_LATIN && !latinMustMove) {
int32_t index = scriptsIndex[USCRIPT_LATIN];
U_ASSERT(index != 0);
int32_t start = scriptStarts[index];
U_ASSERT(lowStart <= start);
skippedReserved = start - lowStart;
lowStart = start;
}
// Reorder according to the input scripts, continuing from the bottom of the primary range.
int32_t originalLength = length; // length will be decremented if "others" is in the list.
UBool hasReorderToEnd = false;
for(int32_t i = 0; i < length;) {
int32_t script = reorder[i++];
if(script == USCRIPT_UNKNOWN) {
// Put the remaining scripts at the top.
hasReorderToEnd = true;
while(i < length) {
script = reorder[--length];
if(script == USCRIPT_UNKNOWN || // Must occur at most once.
script == UCOL_REORDER_CODE_DEFAULT) {
errorCode = U_ILLEGAL_ARGUMENT_ERROR;
return;
}
int32_t index = getScriptIndex(script);
if(index == 0) { continue; }
if(table[index] != 0) { // Duplicate or equivalent script.
errorCode = U_ILLEGAL_ARGUMENT_ERROR;
return;
}
highLimit = addHighScriptRange(table, index, highLimit);
}
break;
}
if(script == UCOL_REORDER_CODE_DEFAULT) {
// The default code must be the only one in the list, and that is handled by the caller.
// Otherwise it must not be used.
errorCode = U_ILLEGAL_ARGUMENT_ERROR;
return;
}
int32_t index = getScriptIndex(script);
if(index == 0) { continue; }
if(table[index] != 0) { // Duplicate or equivalent script.
errorCode = U_ILLEGAL_ARGUMENT_ERROR;
return;
}
lowStart = addLowScriptRange(table, index, lowStart);
}
// Put all remaining scripts into the middle.
for(int32_t i = 1; i < scriptStartsLength - 1; ++i) {
int32_t leadByte = table[i];
if(leadByte != 0) { continue; }
int32_t start = scriptStarts[i];
if(!hasReorderToEnd && start > lowStart) {
// No need to move this script.
lowStart = start;
}
lowStart = addLowScriptRange(table, i, lowStart);
}
if(lowStart > highLimit) {
if((lowStart - (skippedReserved & 0xff00)) <= highLimit) {
// Try not skipping the before-Latin reserved range.
makeReorderRanges(reorder, originalLength, true, ranges, errorCode);
return;
}
// We need more primary lead bytes than available, despite the reserved ranges.
errorCode = U_BUFFER_OVERFLOW_ERROR;
return;
}
// Turn lead bytes into a list of (limit, offset) pairs.
// Encode each pair in one list element:
// Upper 16 bits = limit, lower 16 = signed lead byte offset.
int32_t offset = 0;
for(int32_t i = 1;; ++i) {
int32_t nextOffset = offset;
while(i < scriptStartsLength - 1) {
int32_t newLeadByte = table[i];
if(newLeadByte == 0xff) {
// "Don't care" lead byte for reserved range, continue with current offset.
} else {
nextOffset = newLeadByte - (scriptStarts[i] >> 8);
if(nextOffset != offset) { break; }
}
++i;
}
if(offset != 0 || i < scriptStartsLength - 1) {
ranges.addElement((static_cast<int32_t>(scriptStarts[i]) << 16) | (offset & 0xffff), errorCode);
}
if(i == scriptStartsLength - 1) { break; }
offset = nextOffset;
}
}
int32_t
CollationData::addLowScriptRange(uint8_t table[], int32_t index, int32_t lowStart) const {
int32_t start = scriptStarts[index];
if((start & 0xff) < (lowStart & 0xff)) {
lowStart += 0x100;
}
table[index] = static_cast<uint8_t>(lowStart >> 8);
int32_t limit = scriptStarts[index + 1];
lowStart = ((lowStart & 0xff00) + ((limit & 0xff00) - (start & 0xff00))) | (limit & 0xff);
return lowStart;
}
int32_t
CollationData::addHighScriptRange(uint8_t table[], int32_t index, int32_t highLimit) const {
int32_t limit = scriptStarts[index + 1];
if((limit & 0xff) > (highLimit & 0xff)) {
highLimit -= 0x100;
}
int32_t start = scriptStarts[index];
highLimit = ((highLimit & 0xff00) - ((limit & 0xff00) - (start & 0xff00))) | (start & 0xff);
table[index] = static_cast<uint8_t>(highLimit >> 8);
return highLimit;
}
U_NAMESPACE_END
#endif // !UCONFIG_NO_COLLATION

258
deps/icu-small/source/i18n/collationdata.h vendored Normal file
View File

@ -0,0 +1,258 @@
// © 2016 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html
/*
*******************************************************************************
* Copyright (C) 2010-2015, International Business Machines
* Corporation and others. All Rights Reserved.
*******************************************************************************
* collationdata.h
*
* created on: 2010oct27
* created by: Markus W. Scherer
*/
#ifndef __COLLATIONDATA_H__
#define __COLLATIONDATA_H__
#include "unicode/utypes.h"
#if !UCONFIG_NO_COLLATION
#include "unicode/ucol.h"
#include "unicode/uniset.h"
#include "collation.h"
#include "normalizer2impl.h"
#include "utrie2.h"
struct UDataMemory;
U_NAMESPACE_BEGIN
class UVector32;
/**
* Collation data container.
* Immutable data created by a CollationDataBuilder, or loaded from a file,
* or deserialized from API-provided binary data.
*
* Includes data for the collation base (root/default), aliased if this is not the base.
*/
struct U_I18N_API CollationData : public UMemory {
// Note: The ucadata.icu loader could discover the reserved ranges by setting an array
// parallel with the ranges, and resetting ranges that are indexed.
// The reordering builder code could clone the resulting template array.
static constexpr int32_t REORDER_RESERVED_BEFORE_LATIN = UCOL_REORDER_CODE_FIRST + 14;
static constexpr int32_t REORDER_RESERVED_AFTER_LATIN = REORDER_RESERVED_BEFORE_LATIN + 1;
static constexpr int32_t MAX_NUM_SPECIAL_REORDER_CODES = 8;
/** C++ only, data reader check scriptStartsLength. */
static constexpr int32_t MAX_NUM_SCRIPT_RANGES = 256;
CollationData(const Normalizer2Impl &nfc)
: trie(nullptr),
ce32s(nullptr), ces(nullptr), contexts(nullptr), base(nullptr),
jamoCE32s(nullptr),
nfcImpl(nfc),
numericPrimary(0x12000000),
ce32sLength(0), cesLength(0), contextsLength(0),
compressibleBytes(nullptr),
unsafeBackwardSet(nullptr),
fastLatinTable(nullptr), fastLatinTableLength(0),
numScripts(0), scriptsIndex(nullptr), scriptStarts(nullptr), scriptStartsLength(0),
rootElements(nullptr), rootElementsLength(0) {}
uint32_t getCE32(UChar32 c) const {
return UTRIE2_GET32(trie, c);
}
uint32_t getCE32FromSupplementary(UChar32 c) const {
return UTRIE2_GET32_FROM_SUPP(trie, c);
}
UBool isDigit(UChar32 c) const {
return c < 0x660 ? c <= 0x39 && 0x30 <= c :
Collation::hasCE32Tag(getCE32(c), Collation::DIGIT_TAG);
}
UBool isUnsafeBackward(UChar32 c, UBool numeric) const {
return unsafeBackwardSet->contains(c) || (numeric && isDigit(c));
}
UBool isCompressibleLeadByte(uint32_t b) const {
return compressibleBytes[b];
}
inline UBool isCompressiblePrimary(uint32_t p) const {
return isCompressibleLeadByte(p >> 24);
}
/**
* Returns the CE32 from two contexts words.
* Access to the defaultCE32 for contraction and prefix matching.
*/
static uint32_t readCE32(const char16_t *p) {
return (static_cast<uint32_t>(p[0]) << 16) | p[1];
}
/**
* Returns the CE32 for an indirect special CE32 (e.g., with DIGIT_TAG).
* Requires that ce32 is special.
*/
uint32_t getIndirectCE32(uint32_t ce32) const;
/**
* Returns the CE32 for an indirect special CE32 (e.g., with DIGIT_TAG),
* if ce32 is special.
*/
uint32_t getFinalCE32(uint32_t ce32) const;
/**
* Computes a CE from c's ce32 which has the OFFSET_TAG.
*/
int64_t getCEFromOffsetCE32(UChar32 c, uint32_t ce32) const {
int64_t dataCE = ces[Collation::indexFromCE32(ce32)];
return Collation::makeCE(Collation::getThreeBytePrimaryForOffsetData(c, dataCE));
}
/**
* Returns the single CE that c maps to.
* Sets U_UNSUPPORTED_ERROR if c does not map to a single CE.
*/
int64_t getSingleCE(UChar32 c, UErrorCode &errorCode) const;
/**
* Returns the FCD16 value for code point c. c must be >= 0.
*/
uint16_t getFCD16(UChar32 c) const {
return nfcImpl.getFCD16(c);
}
/**
* Returns the first primary for the script's reordering group.
* @return the primary with only the first primary lead byte of the group
* (not necessarily an actual root collator primary weight),
* or 0 if the script is unknown
*/
uint32_t getFirstPrimaryForGroup(int32_t script) const;
/**
* Returns the last primary for the script's reordering group.
* @return the last primary of the group
* (not an actual root collator primary weight),
* or 0 if the script is unknown
*/
uint32_t getLastPrimaryForGroup(int32_t script) const;
/**
* Finds the reordering group which contains the primary weight.
* @return the first script of the group, or -1 if the weight is beyond the last group
*/
int32_t getGroupForPrimary(uint32_t p) const;
int32_t getEquivalentScripts(int32_t script,
int32_t dest[], int32_t capacity, UErrorCode &errorCode) const;
/**
* Writes the permutation of primary-weight ranges
* for the given reordering of scripts and groups.
* The caller checks for illegal arguments and
* takes care of [DEFAULT] and memory allocation.
*
* Each list element will be a (limit, offset) pair as described
* for the CollationSettings::reorderRanges.
* The list will be empty if no ranges are reordered.
*/
void makeReorderRanges(const int32_t *reorder, int32_t length,
UVector32 &ranges, UErrorCode &errorCode) const;
/** @see jamoCE32s */
static const int32_t JAMO_CE32S_LENGTH = 19 + 21 + 27;
/** Main lookup trie. */
const UTrie2 *trie;
/**
* Array of CE32 values.
* At index 0 there must be CE32(U+0000)
* to support U+0000's special-tag for NUL-termination handling.
*/
const uint32_t *ce32s;
/** Array of CE values for expansions and OFFSET_TAG. */
const int64_t *ces;
/** Array of prefix and contraction-suffix matching data. */
const char16_t *contexts;
/** Base collation data, or nullptr if this data itself is a base. */
const CollationData *base;
/**
* Simple array of JAMO_CE32S_LENGTH=19+21+27 CE32s, one per canonical Jamo L/V/T.
* They are normally simple CE32s, rarely expansions.
* For fast handling of HANGUL_TAG.
*/
const uint32_t *jamoCE32s;
const Normalizer2Impl &nfcImpl;
/** The single-byte primary weight (xx000000) for numeric collation. */
uint32_t numericPrimary;
int32_t ce32sLength;
int32_t cesLength;
int32_t contextsLength;
/** 256 flags for which primary-weight lead bytes are compressible. */
const UBool *compressibleBytes;
/**
* Set of code points that are unsafe for starting string comparison after an identical prefix,
* or in backwards CE iteration.
*/
const UnicodeSet *unsafeBackwardSet;
/**
* Fast Latin table for common-Latin-text string comparisons.
* Data structure see class CollationFastLatin.
*/
const uint16_t *fastLatinTable;
int32_t fastLatinTableLength;
/**
* Data for scripts and reordering groups.
* Uses include building a reordering permutation table and
* providing script boundaries to AlphabeticIndex.
*/
int32_t numScripts;
/**
* The length of scriptsIndex is numScripts+16.
* It maps from a UScriptCode or a special reorder code to an entry in scriptStarts.
* 16 special reorder codes (not all used) are mapped starting at numScripts.
* Up to MAX_NUM_SPECIAL_REORDER_CODES are codes for special groups like space/punct/digit.
* There are special codes at the end for reorder-reserved primary ranges.
*
* Multiple scripts may share a range and index, for example Hira & Kana.
*/
const uint16_t *scriptsIndex;
/**
* Start primary weight (top 16 bits only) for a group/script/reserved range
* indexed by scriptsIndex.
* The first range (separators & terminators) and the last range (trailing weights)
* are not reorderable, and no scriptsIndex entry points to them.
*/
const uint16_t *scriptStarts;
int32_t scriptStartsLength;
/**
* Collation elements in the root collator.
* Used by the CollationRootElements class. The data structure is described there.
* nullptr in a tailoring.
*/
const uint32_t *rootElements;
int32_t rootElementsLength;
private:
int32_t getScriptIndex(int32_t script) const;
void makeReorderRanges(const int32_t *reorder, int32_t length,
UBool latinMustMove,
UVector32 &ranges, UErrorCode &errorCode) const;
int32_t addLowScriptRange(uint8_t table[], int32_t index, int32_t lowStart) const;
int32_t addHighScriptRange(uint8_t table[], int32_t index, int32_t highLimit) const;
};
U_NAMESPACE_END
#endif // !UCONFIG_NO_COLLATION
#endif // __COLLATIONDATA_H__

1675
deps/icu-small/source/i18n/collationdatabuilder.cpp vendored Normal file
View File

File diff suppressed because it is too large Load Diff

269
deps/icu-small/source/i18n/collationdatabuilder.h vendored Normal file
View File

@ -0,0 +1,269 @@
// © 2016 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html
/*
*******************************************************************************
* Copyright (C) 2012-2014, International Business Machines
* Corporation and others. All Rights Reserved.
*******************************************************************************
* collationdatabuilder.h
*
* created on: 2012apr01
* created by: Markus W. Scherer
*/
#ifndef __COLLATIONDATABUILDER_H__
#define __COLLATIONDATABUILDER_H__
#include "unicode/utypes.h"
#if !UCONFIG_NO_COLLATION
#include "unicode/uniset.h"
#include "unicode/unistr.h"
#include "unicode/uversion.h"
#include "collation.h"
#include "collationdata.h"
#include "collationsettings.h"
#include "normalizer2impl.h"
#include "utrie2.h"
#include "uvectr32.h"
#include "uvectr64.h"
#include "uvector.h"
U_NAMESPACE_BEGIN
struct ConditionalCE32;
class CollationFastLatinBuilder;
class CopyHelper;
class DataBuilderCollationIterator;
class UCharsTrieBuilder;
/**
* Low-level CollationData builder.
* Takes (character, CE) pairs and builds them into runtime data structures.
* Supports characters with context prefixes and contraction suffixes.
*/
class U_I18N_API CollationDataBuilder : public UObject {
public:
/**
* Collation element modifier. Interface class for a modifier
* that changes a tailoring builder's temporary CEs to final CEs.
* Called for every non-special CE32 and every expansion CE.
*/
class CEModifier : public UObject {
public:
virtual ~CEModifier();
/** Returns a new CE to replace the non-special input CE32, or else Collation::NO_CE. */
virtual int64_t modifyCE32(uint32_t ce32) const = 0;
/** Returns a new CE to replace the input CE, or else Collation::NO_CE. */
virtual int64_t modifyCE(int64_t ce) const = 0;
};
CollationDataBuilder(UBool icu4xMode, UErrorCode &errorCode);
virtual ~CollationDataBuilder();
void initForTailoring(const CollationData *b, UErrorCode &errorCode);
virtual UBool isCompressibleLeadByte(uint32_t b) const;
inline UBool isCompressiblePrimary(uint32_t p) const {
return isCompressibleLeadByte(p >> 24);
}
/**
* @return true if this builder has mappings (e.g., add() has been called)
*/
UBool hasMappings() const { return modified; }
/**
* @return true if c has CEs in this builder
*/
UBool isAssigned(UChar32 c) const;
/**
* @return the three-byte primary if c maps to a single such CE and has no context data,
* otherwise returns 0.
*/
uint32_t getLongPrimaryIfSingleCE(UChar32 c) const;
/**
* @return the single CE for c.
* Sets an error code if c does not have a single CE.
*/
int64_t getSingleCE(UChar32 c, UErrorCode &errorCode) const;
void add(const UnicodeString &prefix, const UnicodeString &s,
const int64_t ces[], int32_t cesLength,
UErrorCode &errorCode);
/**
* Encodes the ces as either the returned ce32 by itself,
* or by storing an expansion, with the returned ce32 referring to that.
*
* add(p, s, ces, cesLength) = addCE32(p, s, encodeCEs(ces, cesLength))
*/
virtual uint32_t encodeCEs(const int64_t ces[], int32_t cesLength, UErrorCode &errorCode);
void addCE32(const UnicodeString &prefix, const UnicodeString &s,
uint32_t ce32, UErrorCode &errorCode);
/**
* Sets three-byte-primary CEs for a range of code points in code point order,
* if it is worth doing; otherwise no change is made.
* None of the code points in the range should have complex mappings so far
* (expansions/contractions/prefixes).
* @param start first code point
* @param end last code point (inclusive)
* @param primary primary weight for 'start'
* @param step per-code point primary-weight increment
* @param errorCode ICU in/out error code
* @return true if an OFFSET_TAG range was used for start..end
*/
UBool maybeSetPrimaryRange(UChar32 start, UChar32 end,
uint32_t primary, int32_t step,
UErrorCode &errorCode);
/**
* Sets three-byte-primary CEs for a range of code points in code point order.
* Sets range values if that is worth doing, or else individual values.
* None of the code points in the range should have complex mappings so far
* (expansions/contractions/prefixes).
* @param start first code point
* @param end last code point (inclusive)
* @param primary primary weight for 'start'
* @param step per-code point primary-weight increment
* @param errorCode ICU in/out error code
* @return the next primary after 'end': start primary incremented by ((end-start)+1)*step
*/
uint32_t setPrimaryRangeAndReturnNext(UChar32 start, UChar32 end,
uint32_t primary, int32_t step,
UErrorCode &errorCode);
/**
* Copies all mappings from the src builder, with modifications.
* This builder here must not be built yet, and should be empty.
*/
void copyFrom(const CollationDataBuilder &src, const CEModifier &modifier,
UErrorCode &errorCode);
void optimize(const UnicodeSet &set, UErrorCode &errorCode);
void suppressContractions(const UnicodeSet &set, UErrorCode &errorCode);
void enableFastLatin() { fastLatinEnabled = true; }
virtual void build(CollationData &data, UErrorCode &errorCode);
/**
* Looks up CEs for s and appends them to the ces array.
* Does not handle normalization: s should be in FCD form.
*
* Does not write completely ignorable CEs.
* Does not write beyond Collation::MAX_EXPANSION_LENGTH.
*
* @return incremented cesLength
*/
int32_t getCEs(const UnicodeString &s, int64_t ces[], int32_t cesLength);
int32_t getCEs(const UnicodeString &prefix, const UnicodeString &s,
int64_t ces[], int32_t cesLength);
protected:
friend class CopyHelper;
friend class DataBuilderCollationIterator;
uint32_t getCE32FromOffsetCE32(UBool fromBase, UChar32 c, uint32_t ce32) const;
int32_t addCE(int64_t ce, UErrorCode &errorCode);
int32_t addCE32(uint32_t ce32, UErrorCode &errorCode);
int32_t addConditionalCE32(const UnicodeString &context, uint32_t ce32, UErrorCode &errorCode);
inline ConditionalCE32 *getConditionalCE32(int32_t index) const {
return static_cast<ConditionalCE32 *>(conditionalCE32s[index]);
}
inline ConditionalCE32 *getConditionalCE32ForCE32(uint32_t ce32) const {
return getConditionalCE32(Collation::indexFromCE32(ce32));
}
static uint32_t makeBuilderContextCE32(int32_t index) {
return Collation::makeCE32FromTagAndIndex(Collation::BUILDER_DATA_TAG, index);
}
static inline UBool isBuilderContextCE32(uint32_t ce32) {
return Collation::hasCE32Tag(ce32, Collation::BUILDER_DATA_TAG);
}
static uint32_t encodeOneCEAsCE32(int64_t ce);
uint32_t encodeOneCE(int64_t ce, UErrorCode &errorCode);
uint32_t encodeExpansion(const int64_t ces[], int32_t length, UErrorCode &errorCode);
uint32_t encodeExpansion32(const int32_t newCE32s[], int32_t length, UErrorCode &errorCode);
uint32_t copyFromBaseCE32(UChar32 c, uint32_t ce32, UBool withContext, UErrorCode &errorCode);
/**
* Copies base contractions to a list of ConditionalCE32.
* Sets cond->next to the index of the first new item
* and returns the index of the last new item.
*/
int32_t copyContractionsFromBaseCE32(UnicodeString &context, UChar32 c, uint32_t ce32,
ConditionalCE32 *cond, UErrorCode &errorCode);
UBool getJamoCE32s(uint32_t jamoCE32s[], UErrorCode &errorCode);
void setDigitTags(UErrorCode &errorCode);
void setLeadSurrogates(UErrorCode &errorCode);
void buildMappings(CollationData &data, UErrorCode &errorCode);
void clearContexts();
void buildContexts(UErrorCode &errorCode);
uint32_t buildContext(ConditionalCE32 *head, UErrorCode &errorCode);
int32_t addContextTrie(uint32_t defaultCE32, UCharsTrieBuilder &trieBuilder,
UErrorCode &errorCode);
void buildFastLatinTable(CollationData &data, UErrorCode &errorCode);
int32_t getCEs(const UnicodeString &s, int32_t start, int64_t ces[], int32_t cesLength);
static UChar32 jamoCpFromIndex(int32_t i) {
// 0 <= i < CollationData::JAMO_CE32S_LENGTH = 19 + 21 + 27
if(i < Hangul::JAMO_L_COUNT) { return Hangul::JAMO_L_BASE + i; }
i -= Hangul::JAMO_L_COUNT;
if(i < Hangul::JAMO_V_COUNT) { return Hangul::JAMO_V_BASE + i; }
i -= Hangul::JAMO_V_COUNT;
// i < 27
return Hangul::JAMO_T_BASE + 1 + i;
}
/** @see Collation::BUILDER_DATA_TAG */
static const uint32_t IS_BUILDER_JAMO_CE32 = 0x100;
const Normalizer2Impl &nfcImpl;
const CollationData *base;
const CollationSettings *baseSettings;
UTrie2 *trie;
UVector32 ce32s;
UVector64 ce64s;
UVector conditionalCE32s; // vector of ConditionalCE32
// Characters that have context (prefixes or contraction suffixes).
UnicodeSet contextChars;
// Serialized UCharsTrie structures for finalized contexts.
UnicodeString contexts;
private:
/**
* The "era" of building intermediate contexts.
* When the array of cached, temporary contexts overflows, then clearContexts()
* removes them all and invalidates the builtCE32 that used to point to built tries.
* See ConditionalCE32::era.
*/
int32_t contextsEra = 0;
protected:
UnicodeSet unsafeBackwardSet;
UBool modified;
UBool icu4xMode;
UBool fastLatinEnabled;
CollationFastLatinBuilder *fastLatinBuilder;
DataBuilderCollationIterator *collIter;
};
U_NAMESPACE_END
#endif // !UCONFIG_NO_COLLATION
#endif // __COLLATIONDATABUILDER_H__

482
deps/icu-small/source/i18n/collationdatareader.cpp vendored Normal file
View File

@ -0,0 +1,482 @@
// © 2016 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html
/*
*******************************************************************************
* Copyright (C) 2013-2015, International Business Machines
* Corporation and others. All Rights Reserved.
*******************************************************************************
* collationdatareader.cpp
*
* created on: 2013feb07
* created by: Markus W. Scherer
*/
#include "unicode/utypes.h"
#if !UCONFIG_NO_COLLATION
#include "unicode/ucol.h"
#include "unicode/udata.h"
#include "unicode/uscript.h"
#include "cmemory.h"
#include "collation.h"
#include "collationdata.h"
#include "collationdatareader.h"
#include "collationfastlatin.h"
#include "collationkeys.h"
#include "collationrootelements.h"
#include "collationsettings.h"
#include "collationtailoring.h"
#include "collunsafe.h"
#include "normalizer2impl.h"
#include "uassert.h"
#include "ucmndata.h"
#include "utrie2.h"
U_NAMESPACE_BEGIN
namespace {
int32_t getIndex(const int32_t *indexes, int32_t length, int32_t i) {
return (i < length) ? indexes[i] : -1;
}
} // namespace
void
CollationDataReader::read(const CollationTailoring *base, const uint8_t *inBytes, int32_t inLength,
CollationTailoring &tailoring, UErrorCode &errorCode) {
if(U_FAILURE(errorCode)) { return; }
if(base != nullptr) {
if(inBytes == nullptr || (0 <= inLength && inLength < 24)) {
errorCode = U_ILLEGAL_ARGUMENT_ERROR;
return;
}
const DataHeader *header = reinterpret_cast<const DataHeader *>(inBytes);
if(!(header->dataHeader.magic1 == 0xda && header->dataHeader.magic2 == 0x27 &&
isAcceptable(tailoring.version, nullptr, nullptr, &header->info))) {
errorCode = U_INVALID_FORMAT_ERROR;
return;
}
if(base->getUCAVersion() != tailoring.getUCAVersion()) {
errorCode = U_COLLATOR_VERSION_MISMATCH;
return;
}
int32_t headerLength = header->dataHeader.headerSize;
inBytes += headerLength;
if(inLength >= 0) {
inLength -= headerLength;
}
}
if(inBytes == nullptr || (0 <= inLength && inLength < 8)) {
errorCode = U_ILLEGAL_ARGUMENT_ERROR;
return;
}
const int32_t *inIndexes = reinterpret_cast<const int32_t *>(inBytes);
int32_t indexesLength = inIndexes[IX_INDEXES_LENGTH];
if(indexesLength < 2 || (0 <= inLength && inLength < indexesLength * 4)) {
errorCode = U_INVALID_FORMAT_ERROR; // Not enough indexes.
return;
}
// Assume that the tailoring data is in initial state,
// with nullptr pointers and 0 lengths.
// Set pointers to non-empty data parts.
// Do this in order of their byte offsets. (Should help porting to Java.)
int32_t index; // one of the indexes[] slots
int32_t offset; // byte offset for the index part
int32_t length; // number of bytes in the index part
if(indexesLength > IX_TOTAL_SIZE) {
length = inIndexes[IX_TOTAL_SIZE];
} else if(indexesLength > IX_REORDER_CODES_OFFSET) {
length = inIndexes[indexesLength - 1];
} else {
length = 0; // only indexes, and inLength was already checked for them
}
if(0 <= inLength && inLength < length) {
errorCode = U_INVALID_FORMAT_ERROR;
return;
}
const CollationData *baseData = base == nullptr ? nullptr : base->data;
const int32_t *reorderCodes = nullptr;
int32_t reorderCodesLength = 0;
const uint32_t *reorderRanges = nullptr;
int32_t reorderRangesLength = 0;
index = IX_REORDER_CODES_OFFSET;
offset = getIndex(inIndexes, indexesLength, index);
length = getIndex(inIndexes, indexesLength, index + 1) - offset;
if(length >= 4) {
if(baseData == nullptr) {
// We assume for collation settings that
// the base data does not have a reordering.
errorCode = U_INVALID_FORMAT_ERROR;
return;
}
reorderCodes = reinterpret_cast<const int32_t *>(inBytes + offset);
reorderCodesLength = length / 4;
// The reorderRanges (if any) are the trailing reorderCodes entries.
// Split the array at the boundary.
// Script or reorder codes do not exceed 16-bit values.
// Range limits are stored in the upper 16 bits, and are never 0.
while(reorderRangesLength < reorderCodesLength &&
(reorderCodes[reorderCodesLength - reorderRangesLength - 1] & 0xffff0000) != 0) {
++reorderRangesLength;
}
U_ASSERT(reorderRangesLength < reorderCodesLength);
if(reorderRangesLength != 0) {
reorderCodesLength -= reorderRangesLength;
reorderRanges = reinterpret_cast<const uint32_t *>(reorderCodes + reorderCodesLength);
}
}
// There should be a reorder table only if there are reorder codes.
// However, when there are reorder codes the reorder table may be omitted to reduce
// the data size.
const uint8_t *reorderTable = nullptr;
index = IX_REORDER_TABLE_OFFSET;
offset = getIndex(inIndexes, indexesLength, index);
length = getIndex(inIndexes, indexesLength, index + 1) - offset;
if(length >= 256) {
if(reorderCodesLength == 0) {
errorCode = U_INVALID_FORMAT_ERROR; // Reordering table without reordering codes.
return;
}
reorderTable = inBytes + offset;
} else {
// If we have reorder codes, then build the reorderTable at the end,
// when the CollationData is otherwise complete.
}
if(baseData != nullptr && baseData->numericPrimary != (inIndexes[IX_OPTIONS] & 0xff000000)) {
errorCode = U_INVALID_FORMAT_ERROR;
return;
}
CollationData *data = nullptr; // Remains nullptr if there are no mappings.
index = IX_TRIE_OFFSET;
offset = getIndex(inIndexes, indexesLength, index);
length = getIndex(inIndexes, indexesLength, index + 1) - offset;
if(length >= 8) {
if(!tailoring.ensureOwnedData(errorCode)) { return; }
data = tailoring.ownedData;
data->base = baseData;
data->numericPrimary = inIndexes[IX_OPTIONS] & 0xff000000;
data->trie = tailoring.trie = utrie2_openFromSerialized(
UTRIE2_32_VALUE_BITS, inBytes + offset, length, nullptr,
&errorCode);
if(U_FAILURE(errorCode)) { return; }
} else if(baseData != nullptr) {
// Use the base data. Only the settings are tailored.
tailoring.data = baseData;
} else {
errorCode = U_INVALID_FORMAT_ERROR; // No mappings.
return;
}
index = IX_CES_OFFSET;
offset = getIndex(inIndexes, indexesLength, index);
length = getIndex(inIndexes, indexesLength, index + 1) - offset;
if(length >= 8) {
if(data == nullptr) {
errorCode = U_INVALID_FORMAT_ERROR; // Tailored ces without tailored trie.
return;
}
data->ces = reinterpret_cast<const int64_t *>(inBytes + offset);
data->cesLength = length / 8;
}
index = IX_CE32S_OFFSET;
offset = getIndex(inIndexes, indexesLength, index);
length = getIndex(inIndexes, indexesLength, index + 1) - offset;
if(length >= 4) {
if(data == nullptr) {
errorCode = U_INVALID_FORMAT_ERROR; // Tailored ce32s without tailored trie.
return;
}
data->ce32s = reinterpret_cast<const uint32_t *>(inBytes + offset);
data->ce32sLength = length / 4;
}
int32_t jamoCE32sStart = getIndex(inIndexes, indexesLength, IX_JAMO_CE32S_START);
if(jamoCE32sStart >= 0) {
if(data == nullptr || data->ce32s == nullptr) {
errorCode = U_INVALID_FORMAT_ERROR; // Index into non-existent ce32s[].
return;
}
data->jamoCE32s = data->ce32s + jamoCE32sStart;
} else if(data == nullptr) {
// Nothing to do.
} else if(baseData != nullptr) {
data->jamoCE32s = baseData->jamoCE32s;
} else {
errorCode = U_INVALID_FORMAT_ERROR; // No Jamo CE32s for Hangul processing.
return;
}
index = IX_ROOT_ELEMENTS_OFFSET;
offset = getIndex(inIndexes, indexesLength, index);
length = getIndex(inIndexes, indexesLength, index + 1) - offset;
if(length >= 4) {
length /= 4;
if(data == nullptr || length <= CollationRootElements::IX_SEC_TER_BOUNDARIES) {
errorCode = U_INVALID_FORMAT_ERROR;
return;
}
data->rootElements = reinterpret_cast<const uint32_t *>(inBytes + offset);
data->rootElementsLength = length;
uint32_t commonSecTer = data->rootElements[CollationRootElements::IX_COMMON_SEC_AND_TER_CE];
if(commonSecTer != Collation::COMMON_SEC_AND_TER_CE) {
errorCode = U_INVALID_FORMAT_ERROR;
return;
}
uint32_t secTerBoundaries = data->rootElements[CollationRootElements::IX_SEC_TER_BOUNDARIES];
if((secTerBoundaries >> 24) < CollationKeys::SEC_COMMON_HIGH) {
// [fixed last secondary common byte] is too low,
// and secondary weights would collide with compressed common secondaries.
errorCode = U_INVALID_FORMAT_ERROR;
return;
}
}
index = IX_CONTEXTS_OFFSET;
offset = getIndex(inIndexes, indexesLength, index);
length = getIndex(inIndexes, indexesLength, index + 1) - offset;
if(length >= 2) {
if(data == nullptr) {
errorCode = U_INVALID_FORMAT_ERROR; // Tailored contexts without tailored trie.
return;
}
data->contexts = reinterpret_cast<const char16_t *>(inBytes + offset);
data->contextsLength = length / 2;
}
index = IX_UNSAFE_BWD_OFFSET;
offset = getIndex(inIndexes, indexesLength, index);
length = getIndex(inIndexes, indexesLength, index + 1) - offset;
if(length >= 2) {
if(data == nullptr) {
errorCode = U_INVALID_FORMAT_ERROR;
return;
}
if(baseData == nullptr) {
#if defined(COLLUNSAFE_COLL_VERSION) && defined (COLLUNSAFE_SERIALIZE)
tailoring.unsafeBackwardSet = new UnicodeSet(unsafe_serializedData, unsafe_serializedCount, UnicodeSet::kSerialized, errorCode);
if(tailoring.unsafeBackwardSet == nullptr) {
errorCode = U_MEMORY_ALLOCATION_ERROR;
return;
} else if (U_FAILURE(errorCode)) {
return;
}
#else
// Create the unsafe-backward set for the root collator.
// Include all non-zero combining marks and trail surrogates.
// We do this at load time, rather than at build time,
// to simplify Unicode version bootstrapping:
// The root data builder only needs the new FractionalUCA.txt data,
// but it need not be built with a version of ICU already updated to
// the corresponding new Unicode Character Database.
//
// The following is an optimized version of
// new UnicodeSet("[[:^lccc=0:][\\udc00-\\udfff]]").
// It is faster and requires fewer code dependencies.
tailoring.unsafeBackwardSet = new UnicodeSet(0xdc00, 0xdfff); // trail surrogates
if(tailoring.unsafeBackwardSet == nullptr) {
errorCode = U_MEMORY_ALLOCATION_ERROR;
return;
}
data->nfcImpl.addLcccChars(*tailoring.unsafeBackwardSet);
#endif // !COLLUNSAFE_SERIALIZE || !COLLUNSAFE_COLL_VERSION
} else {
// Clone the root collator's set contents.
tailoring.unsafeBackwardSet = static_cast<UnicodeSet *>(
baseData->unsafeBackwardSet->cloneAsThawed());
if(tailoring.unsafeBackwardSet == nullptr) {
errorCode = U_MEMORY_ALLOCATION_ERROR;
return;
}
}
// Add the ranges from the data file to the unsafe-backward set.
USerializedSet sset;
const uint16_t *unsafeData = reinterpret_cast<const uint16_t *>(inBytes + offset);
if(!uset_getSerializedSet(&sset, unsafeData, length / 2)) {
errorCode = U_INVALID_FORMAT_ERROR;
return;
}
int32_t count = uset_getSerializedRangeCount(&sset);
for(int32_t i = 0; i < count; ++i) {
UChar32 start, end;
uset_getSerializedRange(&sset, i, &start, &end);
tailoring.unsafeBackwardSet->add(start, end);
}
// Mark each lead surrogate as "unsafe"
// if any of its 1024 associated supplementary code points is "unsafe".
UChar32 c = 0x10000;
for(char16_t lead = 0xd800; lead < 0xdc00; ++lead, c += 0x400) {
if(!tailoring.unsafeBackwardSet->containsNone(c, c + 0x3ff)) {
tailoring.unsafeBackwardSet->add(lead);
}
}
tailoring.unsafeBackwardSet->freeze();
data->unsafeBackwardSet = tailoring.unsafeBackwardSet;
} else if(data == nullptr) {
// Nothing to do.
} else if(baseData != nullptr) {
// No tailoring-specific data: Alias the root collator's set.
data->unsafeBackwardSet = baseData->unsafeBackwardSet;
} else {
errorCode = U_INVALID_FORMAT_ERROR; // No unsafeBackwardSet.
return;
}
// If the fast Latin format version is different,
// or the version is set to 0 for "no fast Latin table",
// then just always use the normal string comparison path.
if(data != nullptr) {
data->fastLatinTable = nullptr;
data->fastLatinTableLength = 0;
if(((inIndexes[IX_OPTIONS] >> 16) & 0xff) == CollationFastLatin::VERSION) {
index = IX_FAST_LATIN_TABLE_OFFSET;
offset = getIndex(inIndexes, indexesLength, index);
length = getIndex(inIndexes, indexesLength, index + 1) - offset;
if(length >= 2) {
data->fastLatinTable = reinterpret_cast<const uint16_t *>(inBytes + offset);
data->fastLatinTableLength = length / 2;
if((*data->fastLatinTable >> 8) != CollationFastLatin::VERSION) {
errorCode = U_INVALID_FORMAT_ERROR; // header vs. table version mismatch
return;
}
} else if(baseData != nullptr) {
data->fastLatinTable = baseData->fastLatinTable;
data->fastLatinTableLength = baseData->fastLatinTableLength;
}
}
}
index = IX_SCRIPTS_OFFSET;
offset = getIndex(inIndexes, indexesLength, index);
length = getIndex(inIndexes, indexesLength, index + 1) - offset;
if(length >= 2) {
if(data == nullptr) {
errorCode = U_INVALID_FORMAT_ERROR;
return;
}
const uint16_t *scripts = reinterpret_cast<const uint16_t *>(inBytes + offset);
int32_t scriptsLength = length / 2;
data->numScripts = scripts[0];
// There must be enough entries for both arrays, including more than two range starts.
data->scriptStartsLength = scriptsLength - (1 + data->numScripts + 16);
if(data->scriptStartsLength <= 2 ||
CollationData::MAX_NUM_SCRIPT_RANGES < data->scriptStartsLength) {
errorCode = U_INVALID_FORMAT_ERROR;
return;
}
data->scriptsIndex = scripts + 1;
data->scriptStarts = scripts + 1 + data->numScripts + 16;
if(!(data->scriptStarts[0] == 0 &&
data->scriptStarts[1] == ((Collation::MERGE_SEPARATOR_BYTE + 1) << 8) &&
data->scriptStarts[data->scriptStartsLength - 1] ==
(Collation::TRAIL_WEIGHT_BYTE << 8))) {
errorCode = U_INVALID_FORMAT_ERROR;
return;
}
} else if(data == nullptr) {
// Nothing to do.
} else if(baseData != nullptr) {
data->numScripts = baseData->numScripts;
data->scriptsIndex = baseData->scriptsIndex;
data->scriptStarts = baseData->scriptStarts;
data->scriptStartsLength = baseData->scriptStartsLength;
}
index = IX_COMPRESSIBLE_BYTES_OFFSET;
offset = getIndex(inIndexes, indexesLength, index);
length = getIndex(inIndexes, indexesLength, index + 1) - offset;
if(length >= 256) {
if(data == nullptr) {
errorCode = U_INVALID_FORMAT_ERROR;
return;
}
data->compressibleBytes = reinterpret_cast<const UBool *>(inBytes + offset);
} else if(data == nullptr) {
// Nothing to do.
} else if(baseData != nullptr) {
data->compressibleBytes = baseData->compressibleBytes;
} else {
errorCode = U_INVALID_FORMAT_ERROR; // No compressibleBytes[].
return;
}
const CollationSettings &ts = *tailoring.settings;
int32_t options = inIndexes[IX_OPTIONS] & 0xffff;
uint16_t fastLatinPrimaries[CollationFastLatin::LATIN_LIMIT];
int32_t fastLatinOptions = CollationFastLatin::getOptions(
tailoring.data, ts, fastLatinPrimaries, UPRV_LENGTHOF(fastLatinPrimaries));
if(options == ts.options && ts.variableTop != 0 &&
reorderCodesLength == ts.reorderCodesLength &&
(reorderCodesLength == 0 ||
uprv_memcmp(reorderCodes, ts.reorderCodes, reorderCodesLength * 4) == 0) &&
fastLatinOptions == ts.fastLatinOptions &&
(fastLatinOptions < 0 ||
uprv_memcmp(fastLatinPrimaries, ts.fastLatinPrimaries,
sizeof(fastLatinPrimaries)) == 0)) {
return;
}
CollationSettings *settings = SharedObject::copyOnWrite(tailoring.settings);
if(settings == nullptr) {
errorCode = U_MEMORY_ALLOCATION_ERROR;
return;
}
settings->options = options;
// Set variableTop from options and scripts data.
settings->variableTop = tailoring.data->getLastPrimaryForGroup(
UCOL_REORDER_CODE_FIRST + int32_t{settings->getMaxVariable()});
if(settings->variableTop == 0) {
errorCode = U_INVALID_FORMAT_ERROR;
return;
}
if(reorderCodesLength != 0) {
settings->aliasReordering(*baseData, reorderCodes, reorderCodesLength,
reorderRanges, reorderRangesLength,
reorderTable, errorCode);
}
settings->fastLatinOptions = CollationFastLatin::getOptions(
tailoring.data, *settings,
settings->fastLatinPrimaries, UPRV_LENGTHOF(settings->fastLatinPrimaries));
}
UBool U_CALLCONV
CollationDataReader::isAcceptable(void *context,
const char * /* type */, const char * /*name*/,
const UDataInfo *pInfo) {
if(
pInfo->size >= 20 &&
pInfo->isBigEndian == U_IS_BIG_ENDIAN &&
pInfo->charsetFamily == U_CHARSET_FAMILY &&
pInfo->dataFormat[0] == 0x55 && // dataFormat="UCol"
pInfo->dataFormat[1] == 0x43 &&
pInfo->dataFormat[2] == 0x6f &&
pInfo->dataFormat[3] == 0x6c &&
pInfo->formatVersion[0] == 5
) {
UVersionInfo *version = static_cast<UVersionInfo *>(context);
if(version != nullptr) {
uprv_memcpy(version, pInfo->dataVersion, 4);
}
return true;
} else {
return false;
}
}
U_NAMESPACE_END
#endif // !UCONFIG_NO_COLLATION

253
deps/icu-small/source/i18n/collationdatareader.h vendored Normal file
View File

@ -0,0 +1,253 @@
// © 2016 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html
/*
*******************************************************************************
* Copyright (C) 2013-2015, International Business Machines
* Corporation and others. All Rights Reserved.
*******************************************************************************
* collationdatareader.h
*
* created on: 2013feb07
* created by: Markus W. Scherer
*/
#ifndef __COLLATIONDATAREADER_H__
#define __COLLATIONDATAREADER_H__
#include "unicode/utypes.h"
#if !UCONFIG_NO_COLLATION
#include "unicode/udata.h"
struct UDataMemory;
U_NAMESPACE_BEGIN
struct CollationTailoring;
/**
* Collation binary data reader.
*/
struct U_I18N_API CollationDataReader /* all static */ {
// The following constants are also copied into source/common/ucol_swp.cpp.
// Keep them in sync!
enum {
/**
* Number of int32_t indexes.
*
* Can be 2 if there are only options.
* Can be 7 or 8 if there are only options and a script reordering.
* The loader treats any index>=indexes[IX_INDEXES_LENGTH] as 0.
*/
IX_INDEXES_LENGTH, // 0
/**
* Bits 31..24: numericPrimary, for numeric collation
* 23..16: fast Latin format version (0 = no fast Latin table)
* 15.. 0: options bit set
*/
IX_OPTIONS,
IX_RESERVED2,
IX_RESERVED3,
/** Array offset to Jamo CE32s in ce32s[], or <0 if none. */
IX_JAMO_CE32S_START, // 4
// Byte offsets from the start of the data, after the generic header.
// The indexes[] are at byte offset 0, other data follows.
// Each data item is aligned properly.
// The data items should be in descending order of unit size,
// to minimize the need for padding.
// Each item's byte length is given by the difference between its offset and
// the next index/offset value.
/** Byte offset to int32_t reorderCodes[]. */
IX_REORDER_CODES_OFFSET,
/**
* Byte offset to uint8_t reorderTable[].
* Empty table if <256 bytes (padding only).
* Otherwise 256 bytes or more (with padding).
*/
IX_REORDER_TABLE_OFFSET,
/** Byte offset to the collation trie. Its length is a multiple of 8 bytes. */
IX_TRIE_OFFSET,
IX_RESERVED8_OFFSET, // 8
/** Byte offset to int64_t ces[]. */
IX_CES_OFFSET,
IX_RESERVED10_OFFSET,
/** Byte offset to uint32_t ce32s[]. */
IX_CE32S_OFFSET,
/** Byte offset to uint32_t rootElements[]. */
IX_ROOT_ELEMENTS_OFFSET, // 12
/** Byte offset to char16_t *contexts[]. */
IX_CONTEXTS_OFFSET,
/** Byte offset to uint16_t [] with serialized unsafeBackwardSet. */
IX_UNSAFE_BWD_OFFSET,
/** Byte offset to uint16_t fastLatinTable[]. */
IX_FAST_LATIN_TABLE_OFFSET,
/** Byte offset to uint16_t scripts[]. */
IX_SCRIPTS_OFFSET, // 16
/**
* Byte offset to UBool compressibleBytes[].
* Empty table if <256 bytes (padding only).
* Otherwise 256 bytes or more (with padding).
*/
IX_COMPRESSIBLE_BYTES_OFFSET,
IX_RESERVED18_OFFSET,
IX_TOTAL_SIZE
};
static void read(const CollationTailoring *base, const uint8_t *inBytes, int32_t inLength,
CollationTailoring &tailoring, UErrorCode &errorCode);
static UBool U_CALLCONV
isAcceptable(void *context, const char *type, const char *name, const UDataInfo *pInfo);
private:
CollationDataReader() = delete; // no constructor
};
/*
* Format of collation data (ucadata.icu, binary data in coll/ *.res files).
* Format version 5.
*
* The root collation data is stored in the ucadata.icu file.
* Tailorings are stored inside .res resource bundle files, with a complete file header.
*
* Collation data begins with a standard ICU data file header
* (DataHeader, see ucmndata.h and unicode/udata.h).
* The UDataInfo.dataVersion field contains the UCA and other version numbers,
* see the comments for CollationTailoring.version.
*
* After the header, the file contains the following parts.
* Constants are defined as enum values of the CollationDataReader class.
* See also the Collation class.
*
* int32_t indexes[indexesLength];
* The indexes array has variable length.
* Some tailorings only need the length and the options,
* others only add reorderCodes and the reorderTable,
* some need to store mappings.
* Only as many indexes are stored as needed to read all of the data.
*
* Index 0: indexesLength
* Index 1: numericPrimary, CollationFastLatin::VERSION, and options: see IX_OPTIONS
* Index 2..3: Unused/reserved/0.
* Index 4: Index into the ce32s array where the CE32s of the conjoining Jamo
* are stored in a short, contiguous part of the ce32s array.
*
* Indexes 5..19 are byte offsets in ascending order.
* Each byte offset marks the start of the next part in the data file,
* and the end of the previous one.
* When two consecutive byte offsets are the same (or too short),
* then the corresponding part is empty.
* Byte offsets are offsets from after the header,
* that is, from the beginning of the indexes[].
* Each part starts at an offset with proper alignment for its data.
* If necessary, the previous part may include padding bytes to achieve this alignment.
* The last byte offset that is stored in the indexes indicates the total size of the data
* (starting with the indexes).
*
* int32_t reorderCodes[]; -- empty in root
* The list of script and reordering codes.
*
* Beginning with format version 5, this array may optionally
* have trailing entries with a full list of reorder ranges
* as described for CollationSettings::reorderRanges.
*
* Script or reorder codes are first and do not exceed 16-bit values.
* Range limits are stored in the upper 16 bits, and are never 0.
* Split this array into reorder codes and ranges at the first entry
* with non-zero upper 16 bits.
*
* If the ranges are missing but needed for split-reordered primary lead bytes,
* then they are regenerated at load time.
*
* uint8_t reorderTable[256]; -- empty in root; can be longer to include padding bytes
* Primary-weight lead byte permutation table.
* Normally present when the reorderCodes are, but can be built at load time.
*
* Beginning with format version 5, a 0 entry at a non-zero index
* (which is otherwise an illegal value)
* means that the primary lead byte is "split"
* (there are different offsets for primaries that share that lead byte)
* and the reordering offset must be determined via the reorder ranges
* that are either stored as part of the reorderCodes array
* or regenerated at load time.
*
* UTrie2 trie; -- see utrie2_impl.h and utrie2.h
* The trie holds the main collation data. Each code point is mapped to a 32-bit value.
* It encodes a simple collation element (CE) in compact form, unless bits 7..6 are both set,
* in which case it is a special CE32 and contains a 4-bit tag and further data.
* See the Collation class for details.
*
* The trie has a value for each lead surrogate code unit with some bits encoding
* collective properties of the 1024 supplementary characters whose UTF-16 form starts with
* the lead surrogate. See Collation::LEAD_SURROGATE_TAG..
*
* int64_t ces[];
* 64-bit CEs and expansions that cannot be stored in a more compact form.
*
* uint32_t ce32s[];
* CE32s for expansions in compact form, and for characters whose trie values
* contain special data.
*
* uint32_t rootElements[]; -- empty in all tailorings
* Compact storage for all of the CEs that occur in the root collation.
* See the CollationRootElements class.
*
* char16_t *contexts[];
* Serialized UCharsTrie structures with prefix (pre-context) and contraction mappings.
*
* uint16_t unsafeBackwardSet[]; -- see UnicodeSet::serialize()
* Serialized form of characters that are unsafe when iterating backwards,
* and at the end of an identical string prefix.
* Back up to a safe character.
* Lead surrogates are "unsafe" when any of their corresponding supplementary
* code points are unsafe.
* Does not include [:^lccc=0:][:^tccc=0:].
* For each tailoring, the root unsafeBackwardSet is subtracted.
* (As a result, in many tailorings no set needs to be stored.)
*
* uint16_t fastLatinTable[];
* Optional optimization for Latin text.
* See the CollationFastLatin class.
*
* uint16_t scripts[]; -- empty in all tailorings
* Format version 5:
* uint16_t numScripts;
* uint16_t scriptsIndex[numScripts+16];
* uint16_t scriptStarts[];
* See CollationData::numScripts etc.
*
* Format version 4:
* Table of the reordering groups with their first and last lead bytes,
* and their script and reordering codes.
* See CollationData::scripts.
*
* UBool compressibleBytes[]; -- empty in all tailorings
* Flag for getSortKey(), indicating primary weight lead bytes that are compressible.
*
* -----------------
* Changes for formatVersion 5 (ICU 55)
*
* Reordering moves single scripts, not groups of scripts.
* Reorder ranges are optionally appended to the reorderCodes,
* and a 0 entry in the reorderTable indicates a split lead byte.
* The scripts data has a new format.
*
* The rootElements may contain secondary and tertiary weights below common=05.
* (Used for small Hiragana letters.)
* Where is occurs, there is also an explicit unit with common secondary & tertiary weights.
* There are no other data structure changes, but builder code needs to be able to handle such data.
*
* The collation element for the merge separator code point U+FFFE
* does not necessarily have special, unique secondary/tertiary weights any more.
*/
U_NAMESPACE_END
#endif // !UCONFIG_NO_COLLATION
#endif // __COLLATIONDATAREADER_H__

352
deps/icu-small/source/i18n/collationdatawriter.cpp vendored Normal file
View File

@ -0,0 +1,352 @@
// © 2016 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html
/*
*******************************************************************************
* Copyright (C) 2013-2015, International Business Machines
* Corporation and others. All Rights Reserved.
*******************************************************************************
* collationdatawriter.cpp
*
* created on: 2013aug06
* created by: Markus W. Scherer
*/
#include "unicode/utypes.h"
#if !UCONFIG_NO_COLLATION
#include "unicode/tblcoll.h"
#include "unicode/udata.h"
#include "unicode/uniset.h"
#include "cmemory.h"
#include "collationdata.h"
#include "collationdatabuilder.h"
#include "collationdatareader.h"
#include "collationdatawriter.h"
#include "collationfastlatin.h"
#include "collationsettings.h"
#include "collationtailoring.h"
#include "uassert.h"
#include "ucmndata.h"
U_NAMESPACE_BEGIN
uint8_t *
RuleBasedCollator::cloneRuleData(int32_t &length, UErrorCode &errorCode) const {
if(U_FAILURE(errorCode)) { return nullptr; }
LocalMemory<uint8_t> buffer(static_cast<uint8_t*>(uprv_malloc(20000)));
if(buffer.isNull()) {
errorCode = U_MEMORY_ALLOCATION_ERROR;
return nullptr;
}
length = cloneBinary(buffer.getAlias(), 20000, errorCode);
if(errorCode == U_BUFFER_OVERFLOW_ERROR) {
if(buffer.allocateInsteadAndCopy(length, 0) == nullptr) {
errorCode = U_MEMORY_ALLOCATION_ERROR;
return nullptr;
}
errorCode = U_ZERO_ERROR;
length = cloneBinary(buffer.getAlias(), length, errorCode);
}
if(U_FAILURE(errorCode)) { return nullptr; }
return buffer.orphan();
}
int32_t
RuleBasedCollator::cloneBinary(uint8_t *dest, int32_t capacity, UErrorCode &errorCode) const {
int32_t indexes[CollationDataReader::IX_TOTAL_SIZE + 1];
return CollationDataWriter::writeTailoring(
*tailoring, *settings, indexes, dest, capacity,
errorCode);
}
static const UDataInfo dataInfo = {
sizeof(UDataInfo),
0,
U_IS_BIG_ENDIAN,
U_CHARSET_FAMILY,
U_SIZEOF_UCHAR,
0,
{ 0x55, 0x43, 0x6f, 0x6c }, // dataFormat="UCol"
{ 5, 0, 0, 0 }, // formatVersion
{ 6, 3, 0, 0 } // dataVersion
};
int32_t
CollationDataWriter::writeBase(const CollationData &data, const CollationSettings &settings,
const void *rootElements, int32_t rootElementsLength,
int32_t indexes[], uint8_t *dest, int32_t capacity,
UErrorCode &errorCode) {
return write(true, nullptr,
data, settings,
rootElements, rootElementsLength,
indexes, dest, capacity, errorCode);
}
int32_t
CollationDataWriter::writeTailoring(const CollationTailoring &t, const CollationSettings &settings,
int32_t indexes[], uint8_t *dest, int32_t capacity,
UErrorCode &errorCode) {
return write(false, t.version,
*t.data, settings,
nullptr, 0,
indexes, dest, capacity, errorCode);
}
int32_t
CollationDataWriter::write(UBool isBase, const UVersionInfo dataVersion,
const CollationData &data, const CollationSettings &settings,
const void *rootElements, int32_t rootElementsLength,
int32_t indexes[], uint8_t *dest, int32_t capacity,
UErrorCode &errorCode) {
if(U_FAILURE(errorCode)) { return 0; }
if(capacity < 0 || (capacity > 0 && dest == nullptr)) {
errorCode = U_ILLEGAL_ARGUMENT_ERROR;
return 0;
}
// Figure out which data items to write before settling on
// the indexes length and writing offsets.
// For any data item, we need to write the start and limit offsets,
// so the indexes length must be at least index-of-start-offset + 2.
int32_t indexesLength;
UBool hasMappings;
UnicodeSet unsafeBackwardSet;
const CollationData *baseData = data.base;
int32_t fastLatinVersion;
if(data.fastLatinTable != nullptr) {
fastLatinVersion = static_cast<int32_t>(CollationFastLatin::VERSION) << 16;
} else {
fastLatinVersion = 0;
}
int32_t fastLatinTableLength = 0;
if(isBase) {
// For the root collator, we write an even number of indexes
// so that we start with an 8-aligned offset.
indexesLength = CollationDataReader::IX_TOTAL_SIZE + 1;
U_ASSERT(settings.reorderCodesLength == 0);
hasMappings = true;
unsafeBackwardSet = *data.unsafeBackwardSet;
fastLatinTableLength = data.fastLatinTableLength;
} else if(baseData == nullptr) {
hasMappings = false;
if(settings.reorderCodesLength == 0) {
// only options
indexesLength = CollationDataReader::IX_OPTIONS + 1; // no limit offset here
} else {
// only options, reorder codes, and the reorder table
indexesLength = CollationDataReader::IX_REORDER_TABLE_OFFSET + 2;
}
} else {
hasMappings = true;
// Tailored mappings, and what else?
// Check in ascending order of optional tailoring data items.
indexesLength = CollationDataReader::IX_CE32S_OFFSET + 2;
if(data.contextsLength != 0) {
indexesLength = CollationDataReader::IX_CONTEXTS_OFFSET + 2;
}
unsafeBackwardSet.addAll(*data.unsafeBackwardSet).removeAll(*baseData->unsafeBackwardSet);
if(!unsafeBackwardSet.isEmpty()) {
indexesLength = CollationDataReader::IX_UNSAFE_BWD_OFFSET + 2;
}
if(data.fastLatinTable != baseData->fastLatinTable) {
fastLatinTableLength = data.fastLatinTableLength;
indexesLength = CollationDataReader::IX_FAST_LATIN_TABLE_OFFSET + 2;
}
}
UVector32 codesAndRanges(errorCode);
const int32_t *reorderCodes = settings.reorderCodes;
int32_t reorderCodesLength = settings.reorderCodesLength;
if(settings.hasReordering() &&
CollationSettings::reorderTableHasSplitBytes(settings.reorderTable)) {
// Rebuild the full list of reorder ranges.
// The list in the settings is truncated for efficiency.
data.makeReorderRanges(reorderCodes, reorderCodesLength, codesAndRanges, errorCode);
// Write the codes, then the ranges.
for(int32_t i = 0; i < reorderCodesLength; ++i) {
codesAndRanges.insertElementAt(reorderCodes[i], i, errorCode);
}
if(U_FAILURE(errorCode)) { return 0; }
reorderCodes = codesAndRanges.getBuffer();
reorderCodesLength = codesAndRanges.size();
}
int32_t headerSize;
if(isBase) {
headerSize = 0; // udata_create() writes the header
} else {
DataHeader header;
header.dataHeader.magic1 = 0xda;
header.dataHeader.magic2 = 0x27;
uprv_memcpy(&header.info, &dataInfo, sizeof(UDataInfo));
uprv_memcpy(header.info.dataVersion, dataVersion, sizeof(UVersionInfo));
headerSize = static_cast<int32_t>(sizeof(header));
U_ASSERT((headerSize & 3) == 0); // multiple of 4 bytes
if(hasMappings && data.cesLength != 0) {
// Sum of the sizes of the data items which are
// not automatically multiples of 8 bytes and which are placed before the CEs.
int32_t sum = headerSize + (indexesLength + reorderCodesLength) * 4;
if((sum & 7) != 0) {
// We need to add padding somewhere so that the 64-bit CEs are 8-aligned.
// We add to the header size here.
// Alternatively, we could increment the indexesLength
// or add a few bytes to the reorderTable.
headerSize += 4;
}
}
header.dataHeader.headerSize = static_cast<uint16_t>(headerSize);
if(headerSize <= capacity) {
uprv_memcpy(dest, &header, sizeof(header));
// Write 00 bytes so that the padding is not mistaken for a copyright string.
uprv_memset(dest + sizeof(header), 0, headerSize - (int32_t)sizeof(header));
dest += headerSize;
capacity -= headerSize;
} else {
dest = nullptr;
capacity = 0;
}
}
indexes[CollationDataReader::IX_INDEXES_LENGTH] = indexesLength;
U_ASSERT((settings.options & ~0xffff) == 0);
indexes[CollationDataReader::IX_OPTIONS] =
data.numericPrimary | fastLatinVersion | settings.options;
indexes[CollationDataReader::IX_RESERVED2] = 0;
indexes[CollationDataReader::IX_RESERVED3] = 0;
// Byte offsets of data items all start from the start of the indexes.
// We add the headerSize at the very end.
int32_t totalSize = indexesLength * 4;
if(hasMappings && (isBase || data.jamoCE32s != baseData->jamoCE32s)) {
indexes[CollationDataReader::IX_JAMO_CE32S_START] = static_cast<int32_t>(data.jamoCE32s - data.ce32s);
} else {
indexes[CollationDataReader::IX_JAMO_CE32S_START] = -1;
}
indexes[CollationDataReader::IX_REORDER_CODES_OFFSET] = totalSize;
totalSize += reorderCodesLength * 4;
indexes[CollationDataReader::IX_REORDER_TABLE_OFFSET] = totalSize;
if(settings.reorderTable != nullptr) {
totalSize += 256;
}
indexes[CollationDataReader::IX_TRIE_OFFSET] = totalSize;
if(hasMappings) {
UErrorCode errorCode2 = U_ZERO_ERROR;
int32_t length;
if(totalSize < capacity) {
length = utrie2_serialize(data.trie, dest + totalSize,
capacity - totalSize, &errorCode2);
} else {
length = utrie2_serialize(data.trie, nullptr, 0, &errorCode2);
}
if(U_FAILURE(errorCode2) && errorCode2 != U_BUFFER_OVERFLOW_ERROR) {
errorCode = errorCode2;
return 0;
}
// The trie size should be a multiple of 8 bytes due to the way
// compactIndex2(UNewTrie2 *trie) currently works.
U_ASSERT((length & 7) == 0);
totalSize += length;
}
indexes[CollationDataReader::IX_RESERVED8_OFFSET] = totalSize;
indexes[CollationDataReader::IX_CES_OFFSET] = totalSize;
if(hasMappings && data.cesLength != 0) {
U_ASSERT(((headerSize + totalSize) & 7) == 0);
totalSize += data.cesLength * 8;
}
indexes[CollationDataReader::IX_RESERVED10_OFFSET] = totalSize;
indexes[CollationDataReader::IX_CE32S_OFFSET] = totalSize;
if(hasMappings) {
totalSize += data.ce32sLength * 4;
}
indexes[CollationDataReader::IX_ROOT_ELEMENTS_OFFSET] = totalSize;
totalSize += rootElementsLength * 4;
indexes[CollationDataReader::IX_CONTEXTS_OFFSET] = totalSize;
if(hasMappings) {
totalSize += data.contextsLength * 2;
}
indexes[CollationDataReader::IX_UNSAFE_BWD_OFFSET] = totalSize;
if(hasMappings && !unsafeBackwardSet.isEmpty()) {
UErrorCode errorCode2 = U_ZERO_ERROR;
int32_t length;
if(totalSize < capacity) {
uint16_t *p = reinterpret_cast<uint16_t *>(dest + totalSize);
length = unsafeBackwardSet.serialize(
p, (capacity - totalSize) / 2, errorCode2);
} else {
length = unsafeBackwardSet.serialize(nullptr, 0, errorCode2);
}
if(U_FAILURE(errorCode2) && errorCode2 != U_BUFFER_OVERFLOW_ERROR) {
errorCode = errorCode2;
return 0;
}
totalSize += length * 2;
}
indexes[CollationDataReader::IX_FAST_LATIN_TABLE_OFFSET] = totalSize;
totalSize += fastLatinTableLength * 2;
UnicodeString scripts;
indexes[CollationDataReader::IX_SCRIPTS_OFFSET] = totalSize;
if(isBase) {
scripts.append(static_cast<char16_t>(data.numScripts));
scripts.append(reinterpret_cast<const char16_t *>(data.scriptsIndex), data.numScripts + 16);
scripts.append(reinterpret_cast<const char16_t *>(data.scriptStarts), data.scriptStartsLength);
totalSize += scripts.length() * 2;
}
indexes[CollationDataReader::IX_COMPRESSIBLE_BYTES_OFFSET] = totalSize;
if(isBase) {
totalSize += 256;
}
indexes[CollationDataReader::IX_RESERVED18_OFFSET] = totalSize;
indexes[CollationDataReader::IX_TOTAL_SIZE] = totalSize;
if(totalSize > capacity) {
errorCode = U_BUFFER_OVERFLOW_ERROR;
return headerSize + totalSize;
}
uprv_memcpy(dest, indexes, indexesLength * 4);
copyData(indexes, CollationDataReader::IX_REORDER_CODES_OFFSET, reorderCodes, dest);
copyData(indexes, CollationDataReader::IX_REORDER_TABLE_OFFSET, settings.reorderTable, dest);
// The trie has already been serialized into the dest buffer.
copyData(indexes, CollationDataReader::IX_CES_OFFSET, data.ces, dest);
copyData(indexes, CollationDataReader::IX_CE32S_OFFSET, data.ce32s, dest);
copyData(indexes, CollationDataReader::IX_ROOT_ELEMENTS_OFFSET, rootElements, dest);
copyData(indexes, CollationDataReader::IX_CONTEXTS_OFFSET, data.contexts, dest);
// The unsafeBackwardSet has already been serialized into the dest buffer.
copyData(indexes, CollationDataReader::IX_FAST_LATIN_TABLE_OFFSET, data.fastLatinTable, dest);
copyData(indexes, CollationDataReader::IX_SCRIPTS_OFFSET, scripts.getBuffer(), dest);
copyData(indexes, CollationDataReader::IX_COMPRESSIBLE_BYTES_OFFSET, data.compressibleBytes, dest);
return headerSize + totalSize;
}
void
CollationDataWriter::copyData(const int32_t indexes[], int32_t startIndex,
const void *src, uint8_t *dest) {
int32_t start = indexes[startIndex];
int32_t limit = indexes[startIndex + 1];
if(start < limit) {
uprv_memcpy(dest + start, src, limit - start);
}
}
U_NAMESPACE_END
#endif // !UCONFIG_NO_COLLATION

57
deps/icu-small/source/i18n/collationdatawriter.h vendored Normal file
View File

@ -0,0 +1,57 @@
// © 2016 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html
/*
*******************************************************************************
* Copyright (C) 2013-2014, International Business Machines
* Corporation and others. All Rights Reserved.
*******************************************************************************
* collationdatawriter.h
*
* created on: 2013aug06
* created by: Markus W. Scherer
*/
#ifndef __COLLATIONDATAWRITER_H__
#define __COLLATIONDATAWRITER_H__
#include "unicode/utypes.h"
#if !UCONFIG_NO_COLLATION
U_NAMESPACE_BEGIN
struct CollationData;
struct CollationSettings;
struct CollationTailoring;
/**
* Collation-related code for tools & demos.
*/
class U_I18N_API CollationDataWriter /* all static */ {
public:
static int32_t writeBase(const CollationData &data, const CollationSettings &settings,
const void *rootElements, int32_t rootElementsLength,
int32_t indexes[], uint8_t *dest, int32_t capacity,
UErrorCode &errorCode);
static int32_t writeTailoring(const CollationTailoring &t, const CollationSettings &settings,
int32_t indexes[], uint8_t *dest, int32_t capacity,
UErrorCode &errorCode);
private:
CollationDataWriter() = delete; // no constructor
static int32_t write(UBool isBase, const UVersionInfo dataVersion,
const CollationData &data, const CollationSettings &settings,
const void *rootElements, int32_t rootElementsLength,
int32_t indexes[], uint8_t *dest, int32_t capacity,
UErrorCode &errorCode);
static void copyData(const int32_t indexes[], int32_t startIndex,
const void *src, uint8_t *dest);
};
U_NAMESPACE_END
#endif // !UCONFIG_NO_COLLATION
#endif // __COLLATIONDATAWRITER_H__

1099
deps/icu-small/source/i18n/collationfastlatin.cpp vendored Normal file
View File

File diff suppressed because it is too large Load Diff

319
deps/icu-small/source/i18n/collationfastlatin.h vendored Normal file
View File

@ -0,0 +1,319 @@
// © 2016 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html
/*
*******************************************************************************
* Copyright (C) 2013-2015, International Business Machines
* Corporation and others. All Rights Reserved.
*******************************************************************************
* collationfastlatin.h
*
* created on: 2013aug09
* created by: Markus W. Scherer
*/
#ifndef __COLLATIONFASTLATIN_H__
#define __COLLATIONFASTLATIN_H__
#include "unicode/utypes.h"
#if !UCONFIG_NO_COLLATION
U_NAMESPACE_BEGIN
struct CollationData;
struct CollationSettings;
class U_I18N_API CollationFastLatin /* all static */ {
public:
/**
* Fast Latin format version (one byte 1..FF).
* Must be incremented for any runtime-incompatible changes,
* in particular, for changes to any of the following constants.
*
* When the major version number of the main data format changes,
* we can reset this fast Latin version to 1.
*/
static const uint16_t VERSION = 2;
static const int32_t LATIN_MAX = 0x17f;
static const int32_t LATIN_LIMIT = LATIN_MAX + 1;
static const int32_t LATIN_MAX_UTF8_LEAD = 0xc5; // UTF-8 lead byte of LATIN_MAX
static const int32_t PUNCT_START = 0x2000;
static const int32_t PUNCT_LIMIT = 0x2040;
// excludes U+FFFE & U+FFFF
static const int32_t NUM_FAST_CHARS = LATIN_LIMIT + (PUNCT_LIMIT - PUNCT_START);
// Note on the supported weight ranges:
// Analysis of UCA 6.3 and CLDR 23 non-search tailorings shows that
// the CEs for characters in the above ranges, excluding expansions with length >2,
// excluding contractions of >2 characters, and other restrictions
// (see the builder's getCEsFromCE32()),
// use at most about 150 primary weights,
// where about 94 primary weights are possibly-variable (space/punct/symbol/currency),
// at most 4 secondary before-common weights,
// at most 4 secondary after-common weights,
// at most 16 secondary high weights (in secondary CEs), and
// at most 4 tertiary after-common weights.
// The following ranges are designed to support slightly more weights than that.
// (en_US_POSIX is unusual: It creates about 64 variable + 116 Latin primaries.)
// Digits may use long primaries (preserving more short ones)
// or short primaries (faster) without changing this data structure.
// (If we supported numeric collation, then digits would have to have long primaries
// so that special handling does not affect the fast path.)
static const uint32_t SHORT_PRIMARY_MASK = 0xfc00; // bits 15..10
static const uint32_t INDEX_MASK = 0x3ff; // bits 9..0 for expansions & contractions
static const uint32_t SECONDARY_MASK = 0x3e0; // bits 9..5
static const uint32_t CASE_MASK = 0x18; // bits 4..3
static const uint32_t LONG_PRIMARY_MASK = 0xfff8; // bits 15..3
static const uint32_t TERTIARY_MASK = 7; // bits 2..0
static const uint32_t CASE_AND_TERTIARY_MASK = CASE_MASK | TERTIARY_MASK;
static const uint32_t TWO_SHORT_PRIMARIES_MASK =
(SHORT_PRIMARY_MASK << 16) | SHORT_PRIMARY_MASK; // 0xfc00fc00
static const uint32_t TWO_LONG_PRIMARIES_MASK =
(LONG_PRIMARY_MASK << 16) | LONG_PRIMARY_MASK; // 0xfff8fff8
static const uint32_t TWO_SECONDARIES_MASK =
(SECONDARY_MASK << 16) | SECONDARY_MASK; // 0x3e003e0
static const uint32_t TWO_CASES_MASK =
(CASE_MASK << 16) | CASE_MASK; // 0x180018
static const uint32_t TWO_TERTIARIES_MASK =
(TERTIARY_MASK << 16) | TERTIARY_MASK; // 0x70007
/**
* Contraction with one fast Latin character.
* Use INDEX_MASK to find the start of the contraction list after the fixed table.
* The first entry contains the default mapping.
* Otherwise use CONTR_CHAR_MASK for the contraction character index
* (in ascending order).
* Use CONTR_LENGTH_SHIFT for the length of the entry
* (1=BAIL_OUT, 2=one CE, 3=two CEs).
*
* Also, U+0000 maps to a contraction entry, so that the fast path need not
* check for NUL termination.
* It usually maps to a contraction list with only the completely ignorable default value.
*/
static const uint32_t CONTRACTION = 0x400;
/**
* An expansion encodes two CEs.
* Use INDEX_MASK to find the pair of CEs after the fixed table.
*
* The higher a mini CE value, the easier it is to process.
* For expansions and higher, no context needs to be considered.
*/
static const uint32_t EXPANSION = 0x800;
/**
* Encodes one CE with a long/low mini primary (there are 128).
* All potentially-variable primaries must be in this range,
* to make the short-primary path as fast as possible.
*/
static const uint32_t MIN_LONG = 0xc00;
static const uint32_t LONG_INC = 8;
static const uint32_t MAX_LONG = 0xff8;
/**
* Encodes one CE with a short/high primary (there are 60),
* plus a secondary CE if the secondary weight is high.
* Fast handling: At least all letter primaries should be in this range.
*/
static const uint32_t MIN_SHORT = 0x1000;
static const uint32_t SHORT_INC = 0x400;
/** The highest primary weight is reserved for U+FFFF. */
static const uint32_t MAX_SHORT = SHORT_PRIMARY_MASK;
static const uint32_t MIN_SEC_BEFORE = 0; // must add SEC_OFFSET
static const uint32_t SEC_INC = 0x20;
static const uint32_t MAX_SEC_BEFORE = MIN_SEC_BEFORE + 4 * SEC_INC; // 5 before common
static const uint32_t COMMON_SEC = MAX_SEC_BEFORE + SEC_INC;
static const uint32_t MIN_SEC_AFTER = COMMON_SEC + SEC_INC;
static const uint32_t MAX_SEC_AFTER = MIN_SEC_AFTER + 5 * SEC_INC; // 6 after common
static const uint32_t MIN_SEC_HIGH = MAX_SEC_AFTER + SEC_INC; // 20 high secondaries
static const uint32_t MAX_SEC_HIGH = SECONDARY_MASK;
/**
* Lookup: Add this offset to secondary weights, except for completely ignorable CEs.
* Must be greater than any special value, e.g., MERGE_WEIGHT.
* The exact value is not relevant for the format version.
*/
static const uint32_t SEC_OFFSET = SEC_INC;
static const uint32_t COMMON_SEC_PLUS_OFFSET = COMMON_SEC + SEC_OFFSET;
static const uint32_t TWO_SEC_OFFSETS =
(SEC_OFFSET << 16) | SEC_OFFSET; // 0x200020
static const uint32_t TWO_COMMON_SEC_PLUS_OFFSET =
(COMMON_SEC_PLUS_OFFSET << 16) | COMMON_SEC_PLUS_OFFSET;
static const uint32_t LOWER_CASE = 8; // case bits include this offset
static const uint32_t TWO_LOWER_CASES = (LOWER_CASE << 16) | LOWER_CASE; // 0x80008
static const uint32_t COMMON_TER = 0; // must add TER_OFFSET
static const uint32_t MAX_TER_AFTER = 7; // 7 after common
/**
* Lookup: Add this offset to tertiary weights, except for completely ignorable CEs.
* Must be greater than any special value, e.g., MERGE_WEIGHT.
* Must be greater than case bits as well, so that with combined case+tertiary weights
* plus the offset the tertiary bits does not spill over into the case bits.
* The exact value is not relevant for the format version.
*/
static const uint32_t TER_OFFSET = SEC_OFFSET;
static const uint32_t COMMON_TER_PLUS_OFFSET = COMMON_TER + TER_OFFSET;
static const uint32_t TWO_TER_OFFSETS = (TER_OFFSET << 16) | TER_OFFSET;
static const uint32_t TWO_COMMON_TER_PLUS_OFFSET =
(COMMON_TER_PLUS_OFFSET << 16) | COMMON_TER_PLUS_OFFSET;
static const uint32_t MERGE_WEIGHT = 3;
static const uint32_t EOS = 2; // end of string
static const uint32_t BAIL_OUT = 1;
/**
* Contraction result first word bits 8..0 contain the
* second contraction character, as a char index 0..NUM_FAST_CHARS-1.
* Each contraction list is terminated with a word containing CONTR_CHAR_MASK.
*/
static const uint32_t CONTR_CHAR_MASK = 0x1ff;
/**
* Contraction result first word bits 10..9 contain the result length:
* 1=bail out, 2=one mini CE, 3=two mini CEs
*/
static const uint32_t CONTR_LENGTH_SHIFT = 9;
/**
* Comparison return value when the regular comparison must be used.
* The exact value is not relevant for the format version.
*/
static const int32_t BAIL_OUT_RESULT = -2;
static inline int32_t getCharIndex(char16_t c) {
if(c <= LATIN_MAX) {
return c;
} else if(PUNCT_START <= c && c < PUNCT_LIMIT) {
return c - (PUNCT_START - LATIN_LIMIT);
} else {
// Not a fast Latin character.
// Note: U+FFFE & U+FFFF are forbidden in tailorings
// and thus do not occur in any contractions.
return -1;
}
}
/**
* Computes the options value for the compare functions
* and writes the precomputed primary weights.
* Returns -1 if the Latin fastpath is not supported for the data and settings.
* The capacity must be LATIN_LIMIT.
*/
static int32_t getOptions(const CollationData *data, const CollationSettings &settings,
uint16_t *primaries, int32_t capacity);
static int32_t compareUTF16(const uint16_t *table, const uint16_t *primaries, int32_t options,
const char16_t *left, int32_t leftLength,
const char16_t *right, int32_t rightLength);
static int32_t compareUTF8(const uint16_t *table, const uint16_t *primaries, int32_t options,
const uint8_t *left, int32_t leftLength,
const uint8_t *right, int32_t rightLength);
private:
static uint32_t lookup(const uint16_t *table, UChar32 c);
static uint32_t lookupUTF8(const uint16_t *table, UChar32 c,
const uint8_t *s8, int32_t &sIndex, int32_t sLength);
static uint32_t lookupUTF8Unsafe(const uint16_t *table, UChar32 c,
const uint8_t *s8, int32_t &sIndex);
static uint32_t nextPair(const uint16_t *table, UChar32 c, uint32_t ce,
const char16_t *s16, const uint8_t *s8, int32_t &sIndex, int32_t &sLength);
static inline uint32_t getPrimaries(uint32_t variableTop, uint32_t pair) {
uint32_t ce = pair & 0xffff;
if(ce >= MIN_SHORT) { return pair & TWO_SHORT_PRIMARIES_MASK; }
if(ce > variableTop) { return pair & TWO_LONG_PRIMARIES_MASK; }
if(ce >= MIN_LONG) { return 0; } // variable
return pair; // special mini CE
}
static inline uint32_t getSecondariesFromOneShortCE(uint32_t ce) {
ce &= SECONDARY_MASK;
if(ce < MIN_SEC_HIGH) {
return ce + SEC_OFFSET;
} else {
return ((ce + SEC_OFFSET) << 16) | COMMON_SEC_PLUS_OFFSET;
}
}
static uint32_t getSecondaries(uint32_t variableTop, uint32_t pair);
static uint32_t getCases(uint32_t variableTop, UBool strengthIsPrimary, uint32_t pair);
static uint32_t getTertiaries(uint32_t variableTop, UBool withCaseBits, uint32_t pair);
static uint32_t getQuaternaries(uint32_t variableTop, uint32_t pair);
private:
CollationFastLatin() = delete; // no constructor
};
/*
* Format of the CollationFastLatin data table.
* CollationFastLatin::VERSION = 2.
*
* This table contains data for a Latin-text collation fastpath.
* The data is stored as an array of uint16_t which contains the following parts.
*
* uint16_t -- version & header length
* Bits 15..8: version, must match the VERSION
* 7..0: length of the header
*
* uint16_t varTops[header length - 1]
* Version 2:
* varTops[m] is the highest CollationFastLatin long-primary weight
* of supported maxVariable group m
* (special reorder group space, punct, symbol, currency).
*
* Version 1:
* Each of these values maps the variable top lead byte of a supported maxVariable group
* to the highest CollationFastLatin long-primary weight.
* The values are stored in ascending order.
* Bits 15..7: max fast-Latin long-primary weight (bits 11..3 shifted left by 4 bits)
* 6..0: regular primary lead byte
*
* uint16_t miniCEs[0x1c0]
* A mini collation element for each character U+0000..U+017F and U+2000..U+203F.
* Each value encodes one or two mini CEs (two are possible if the first one
* has a short mini primary and the second one is a secondary CE, i.e., primary == 0),
* or points to an expansion or to a contraction table.
* U+0000 always has a contraction entry,
* so that NUL-termination need not be tested in the fastpath.
* If the collation elements for a character or contraction cannot be encoded in this format,
* then the BAIL_OUT value is stored.
* For details see the comments for the class constants.
*
* uint16_t expansions[variable length];
* Expansion mini CEs contain an offset relative to just after the miniCEs table.
* An expansions contains exactly 2 mini CEs.
*
* uint16_t contractions[variable length];
* Contraction mini CEs contain an offset relative to just after the miniCEs table.
* It points to a list of tuples which map from a contraction suffix character to a result.
* First uint16_t of each tuple:
* Bits 10..9: Length of the result (1..3), see comments on CONTR_LENGTH_SHIFT.
* Bits 8..0: Contraction character, see comments on CONTR_CHAR_MASK.
* This is followed by 0, 1, or 2 uint16_t according to the length.
* Each list is terminated by an entry with CONTR_CHAR_MASK.
* Each list starts with such an entry which also contains the default result
* for when there is no contraction match.
*
* -----------------
* Changes for version 2 (ICU 55)
*
* Special reorder groups do not necessarily start on whole primary lead bytes any more.
* Therefore, the varTops data has a new format:
* Version 1 stored the lead bytes of the highest root primaries for
* the maxVariable-supported special reorder groups.
* Now the top 16 bits would need to be stored,
* and it is simpler to store only the fast-Latin weights.
*/
U_NAMESPACE_END
#endif // !UCONFIG_NO_COLLATION
#endif // __COLLATIONFASTLATIN_H__

717
deps/icu-small/source/i18n/collationfastlatinbuilder.cpp vendored Normal file
View File

@ -0,0 +1,717 @@
// © 2016 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html
/*
*******************************************************************************
* Copyright (C) 2013-2015, International Business Machines
* Corporation and others. All Rights Reserved.
*******************************************************************************
* collationfastlatinbuilder.cpp
*
* created on: 2013aug09
* created by: Markus W. Scherer
*/
#define DEBUG_COLLATION_FAST_LATIN_BUILDER 0 // 0 or 1 or 2
#if DEBUG_COLLATION_FAST_LATIN_BUILDER
#include <stdio.h>
#include <string>
#endif
#include "unicode/utypes.h"
#if !UCONFIG_NO_COLLATION
#include "unicode/ucol.h"
#include "unicode/ucharstrie.h"
#include "unicode/unistr.h"
#include "unicode/uobject.h"
#include "unicode/uscript.h"
#include "cmemory.h"
#include "collation.h"
#include "collationdata.h"
#include "collationfastlatin.h"
#include "collationfastlatinbuilder.h"
#include "uassert.h"
#include "uvectr64.h"
U_NAMESPACE_BEGIN
struct CollationData;
namespace {
/**
* Compare two signed int64_t values as if they were unsigned.
*/
int32_t
compareInt64AsUnsigned(int64_t a, int64_t b) {
if (static_cast<uint64_t>(a) < static_cast<uint64_t>(b)) {
return -1;
} else if (static_cast<uint64_t>(a) > static_cast<uint64_t>(b)) {
return 1;
} else {
return 0;
}
}
// TODO: Merge this with the near-identical version in collationbasedatabuilder.cpp
/**
* Like Java Collections.binarySearch(List, String, Comparator).
*
* @return the index>=0 where the item was found,
* or the index<0 for inserting the string at ~index in sorted order
*/
int32_t
binarySearch(const int64_t list[], int32_t limit, int64_t ce) {
if (limit == 0) { return ~0; }
int32_t start = 0;
for (;;) {
int32_t i = (start + limit) / 2;
int32_t cmp = compareInt64AsUnsigned(ce, list[i]);
if (cmp == 0) {
return i;
} else if (cmp < 0) {
if (i == start) {
return ~start; // insert ce before i
}
limit = i;
} else {
if (i == start) {
return ~(start + 1); // insert ce after i
}
start = i;
}
}
}
} // namespace
CollationFastLatinBuilder::CollationFastLatinBuilder(UErrorCode &errorCode)
: ce0(0), ce1(0),
contractionCEs(errorCode), uniqueCEs(errorCode),
miniCEs(nullptr),
firstDigitPrimary(0), firstLatinPrimary(0), lastLatinPrimary(0),
firstShortPrimary(0), shortPrimaryOverflow(false),
headerLength(0) {
}
CollationFastLatinBuilder::~CollationFastLatinBuilder() {
uprv_free(miniCEs);
}
UBool
CollationFastLatinBuilder::forData(const CollationData &data, UErrorCode &errorCode) {
if(U_FAILURE(errorCode)) { return false; }
if(!result.isEmpty()) { // This builder is not reusable.
errorCode = U_INVALID_STATE_ERROR;
return false;
}
if(!loadGroups(data, errorCode)) { return false; }
// Fast handling of digits.
firstShortPrimary = firstDigitPrimary;
getCEs(data, errorCode);
if(!encodeUniqueCEs(errorCode)) { return false; }
if(shortPrimaryOverflow) {
// Give digits long mini primaries,
// so that there are more short primaries for letters.
firstShortPrimary = firstLatinPrimary;
resetCEs();
getCEs(data, errorCode);
if(!encodeUniqueCEs(errorCode)) { return false; }
}
// Note: If we still have a short-primary overflow but not a long-primary overflow,
// then we could calculate how many more long primaries would fit,
// and set the firstShortPrimary to that many after the current firstShortPrimary,
// and try again.
// However, this might only benefit the en_US_POSIX tailoring,
// and it is simpler to suppress building fast Latin data for it in genrb,
// or by returning false here if shortPrimaryOverflow.
UBool ok = !shortPrimaryOverflow &&
encodeCharCEs(errorCode) && encodeContractions(errorCode);
contractionCEs.removeAllElements(); // might reduce heap memory usage
uniqueCEs.removeAllElements();
return ok;
}
UBool
CollationFastLatinBuilder::loadGroups(const CollationData &data, UErrorCode &errorCode) {
if(U_FAILURE(errorCode)) { return false; }
headerLength = 1 + NUM_SPECIAL_GROUPS;
uint32_t r0 = (CollationFastLatin::VERSION << 8) | headerLength;
result.append(static_cast<char16_t>(r0));
// The first few reordering groups should be special groups
// (space, punct, ..., digit) followed by Latn, then Grek and other scripts.
for(int32_t i = 0; i < NUM_SPECIAL_GROUPS; ++i) {
lastSpecialPrimaries[i] = data.getLastPrimaryForGroup(UCOL_REORDER_CODE_FIRST + i);
if(lastSpecialPrimaries[i] == 0) {
// missing data
return false;
}
result.append(static_cast<char16_t>(0)); // reserve a slot for this group
}
firstDigitPrimary = data.getFirstPrimaryForGroup(UCOL_REORDER_CODE_DIGIT);
firstLatinPrimary = data.getFirstPrimaryForGroup(USCRIPT_LATIN);
lastLatinPrimary = data.getLastPrimaryForGroup(USCRIPT_LATIN);
if(firstDigitPrimary == 0 || firstLatinPrimary == 0) {
// missing data
return false;
}
return true;
}
UBool
CollationFastLatinBuilder::inSameGroup(uint32_t p, uint32_t q) const {
// Both or neither need to be encoded as short primaries,
// so that we can test only one and use the same bit mask.
if(p >= firstShortPrimary) {
return q >= firstShortPrimary;
} else if(q >= firstShortPrimary) {
return false;
}
// Both or neither must be potentially-variable,
// so that we can test only one and determine if both are variable.
uint32_t lastVariablePrimary = lastSpecialPrimaries[NUM_SPECIAL_GROUPS - 1];
if(p > lastVariablePrimary) {
return q > lastVariablePrimary;
} else if(q > lastVariablePrimary) {
return false;
}
// Both will be encoded with long mini primaries.
// They must be in the same special reordering group,
// so that we can test only one and determine if both are variable.
U_ASSERT(p != 0 && q != 0);
for(int32_t i = 0;; ++i) { // will terminate
uint32_t lastPrimary = lastSpecialPrimaries[i];
if(p <= lastPrimary) {
return q <= lastPrimary;
} else if(q <= lastPrimary) {
return false;
}
}
}
void
CollationFastLatinBuilder::resetCEs() {
contractionCEs.removeAllElements();
uniqueCEs.removeAllElements();
shortPrimaryOverflow = false;
result.truncate(headerLength);
}
void
CollationFastLatinBuilder::getCEs(const CollationData &data, UErrorCode &errorCode) {
if(U_FAILURE(errorCode)) { return; }
int32_t i = 0;
for(char16_t c = 0;; ++i, ++c) {
if(c == CollationFastLatin::LATIN_LIMIT) {
c = CollationFastLatin::PUNCT_START;
} else if(c == CollationFastLatin::PUNCT_LIMIT) {
break;
}
const CollationData *d;
uint32_t ce32 = data.getCE32(c);
if(ce32 == Collation::FALLBACK_CE32) {
d = data.base;
ce32 = d->getCE32(c);
} else {
d = &data;
}
if(getCEsFromCE32(*d, c, ce32, errorCode)) {
charCEs[i][0] = ce0;
charCEs[i][1] = ce1;
addUniqueCE(ce0, errorCode);
addUniqueCE(ce1, errorCode);
} else {
// bail out for c
charCEs[i][0] = ce0 = Collation::NO_CE;
charCEs[i][1] = ce1 = 0;
}
if(c == 0 && !isContractionCharCE(ce0)) {
// Always map U+0000 to a contraction.
// Write a contraction list with only a default value if there is no real contraction.
U_ASSERT(contractionCEs.isEmpty());
addContractionEntry(CollationFastLatin::CONTR_CHAR_MASK, ce0, ce1, errorCode);
charCEs[0][0] = (static_cast<int64_t>(Collation::NO_CE_PRIMARY) << 32) | CONTRACTION_FLAG;
charCEs[0][1] = 0;
}
}
// Terminate the last contraction list.
contractionCEs.addElement(CollationFastLatin::CONTR_CHAR_MASK, errorCode);
}
UBool
CollationFastLatinBuilder::getCEsFromCE32(const CollationData &data, UChar32 c, uint32_t ce32,
UErrorCode &errorCode) {
if(U_FAILURE(errorCode)) { return false; }
ce32 = data.getFinalCE32(ce32);
ce1 = 0;
if(Collation::isSimpleOrLongCE32(ce32)) {
ce0 = Collation::ceFromCE32(ce32);
} else {
switch(Collation::tagFromCE32(ce32)) {
case Collation::LATIN_EXPANSION_TAG:
ce0 = Collation::latinCE0FromCE32(ce32);
ce1 = Collation::latinCE1FromCE32(ce32);
break;
case Collation::EXPANSION32_TAG: {
const uint32_t *ce32s = data.ce32s + Collation::indexFromCE32(ce32);
int32_t length = Collation::lengthFromCE32(ce32);
if(length <= 2) {
ce0 = Collation::ceFromCE32(ce32s[0]);
if(length == 2) {
ce1 = Collation::ceFromCE32(ce32s[1]);
}
break;
} else {
return false;
}
}
case Collation::EXPANSION_TAG: {
const int64_t *ces = data.ces + Collation::indexFromCE32(ce32);
int32_t length = Collation::lengthFromCE32(ce32);
if(length <= 2) {
ce0 = ces[0];
if(length == 2) {
ce1 = ces[1];
}
break;
} else {
return false;
}
}
// Note: We could support PREFIX_TAG (assert c>=0)
// by recursing on its default CE32 and checking that none of the prefixes starts
// with a fast Latin character.
// However, currently (2013) there are only the L-before-middle-dot
// prefix mappings in the Latin range, and those would be rejected anyway.
case Collation::CONTRACTION_TAG:
U_ASSERT(c >= 0);
return getCEsFromContractionCE32(data, ce32, errorCode);
case Collation::OFFSET_TAG:
U_ASSERT(c >= 0);
ce0 = data.getCEFromOffsetCE32(c, ce32);
break;
default:
return false;
}
}
// A mapping can be completely ignorable.
if(ce0 == 0) { return ce1 == 0; }
// We do not support an ignorable ce0 unless it is completely ignorable.
uint32_t p0 = static_cast<uint32_t>(ce0 >> 32);
if(p0 == 0) { return false; }
// We only support primaries up to the Latin script.
if(p0 > lastLatinPrimary) { return false; }
// We support non-common secondary and case weights only together with short primaries.
uint32_t lower32_0 = static_cast<uint32_t>(ce0);
if(p0 < firstShortPrimary) {
uint32_t sc0 = lower32_0 & Collation::SECONDARY_AND_CASE_MASK;
if(sc0 != Collation::COMMON_SECONDARY_CE) { return false; }
}
// No below-common tertiary weights.
if((lower32_0 & Collation::ONLY_TERTIARY_MASK) < Collation::COMMON_WEIGHT16) { return false; }
if(ce1 != 0) {
// Both primaries must be in the same group,
// or both must get short mini primaries,
// or a short-primary CE is followed by a secondary CE.
// This is so that we can test the first primary and use the same mask for both,
// and determine for both whether they are variable.
uint32_t p1 = static_cast<uint32_t>(ce1 >> 32);
if(p1 == 0 ? p0 < firstShortPrimary : !inSameGroup(p0, p1)) { return false; }
uint32_t lower32_1 = static_cast<uint32_t>(ce1);
// No tertiary CEs.
if((lower32_1 >> 16) == 0) { return false; }
// We support non-common secondary and case weights
// only for secondary CEs or together with short primaries.
if(p1 != 0 && p1 < firstShortPrimary) {
uint32_t sc1 = lower32_1 & Collation::SECONDARY_AND_CASE_MASK;
if(sc1 != Collation::COMMON_SECONDARY_CE) { return false; }
}
// No below-common tertiary weights.
if((lower32_1 & Collation::ONLY_TERTIARY_MASK) < Collation::COMMON_WEIGHT16) { return false; }
}
// No quaternary weights.
if(((ce0 | ce1) & Collation::QUATERNARY_MASK) != 0) { return false; }
return true;
}
UBool
CollationFastLatinBuilder::getCEsFromContractionCE32(const CollationData &data, uint32_t ce32,
UErrorCode &errorCode) {
if(U_FAILURE(errorCode)) { return false; }
const char16_t *p = data.contexts + Collation::indexFromCE32(ce32);
ce32 = CollationData::readCE32(p); // Default if no suffix match.
// Since the original ce32 is not a prefix mapping,
// the default ce32 must not be another contraction.
U_ASSERT(!Collation::isContractionCE32(ce32));
int32_t contractionIndex = contractionCEs.size();
if(getCEsFromCE32(data, U_SENTINEL, ce32, errorCode)) {
addContractionEntry(CollationFastLatin::CONTR_CHAR_MASK, ce0, ce1, errorCode);
} else {
// Bail out for c-without-contraction.
addContractionEntry(CollationFastLatin::CONTR_CHAR_MASK, Collation::NO_CE, 0, errorCode);
}
// Handle an encodable contraction unless the next contraction is too long
// and starts with the same character.
int32_t prevX = -1;
UBool addContraction = false;
UCharsTrie::Iterator suffixes(p + 2, 0, errorCode);
while(suffixes.next(errorCode)) {
const UnicodeString &suffix = suffixes.getString();
int32_t x = CollationFastLatin::getCharIndex(suffix.charAt(0));
if(x < 0) { continue; } // ignore anything but fast Latin text
if(x == prevX) {
if(addContraction) {
// Bail out for all contractions starting with this character.
addContractionEntry(x, Collation::NO_CE, 0, errorCode);
addContraction = false;
}
continue;
}
if(addContraction) {
addContractionEntry(prevX, ce0, ce1, errorCode);
}
ce32 = static_cast<uint32_t>(suffixes.getValue());
if(suffix.length() == 1 && getCEsFromCE32(data, U_SENTINEL, ce32, errorCode)) {
addContraction = true;
} else {
addContractionEntry(x, Collation::NO_CE, 0, errorCode);
addContraction = false;
}
prevX = x;
}
if(addContraction) {
addContractionEntry(prevX, ce0, ce1, errorCode);
}
if(U_FAILURE(errorCode)) { return false; }
// Note: There might not be any fast Latin contractions, but
// we need to enter contraction handling anyway so that we can bail out
// when there is a non-fast-Latin character following.
// For example: Danish &Y<<u+umlaut, when we compare Y vs. u\u0308 we need to see the
// following umlaut and bail out, rather than return the difference of Y vs. u.
ce0 = (static_cast<int64_t>(Collation::NO_CE_PRIMARY) << 32) | CONTRACTION_FLAG | contractionIndex;
ce1 = 0;
return true;
}
void
CollationFastLatinBuilder::addContractionEntry(int32_t x, int64_t cce0, int64_t cce1,
UErrorCode &errorCode) {
contractionCEs.addElement(x, errorCode);
contractionCEs.addElement(cce0, errorCode);
contractionCEs.addElement(cce1, errorCode);
addUniqueCE(cce0, errorCode);
addUniqueCE(cce1, errorCode);
}
void
CollationFastLatinBuilder::addUniqueCE(int64_t ce, UErrorCode &errorCode) {
if(U_FAILURE(errorCode)) { return; }
if (ce == 0 || static_cast<uint32_t>(ce >> 32) == Collation::NO_CE_PRIMARY) { return; }
ce &= ~static_cast<int64_t>(Collation::CASE_MASK); // blank out case bits
int32_t i = binarySearch(uniqueCEs.getBuffer(), uniqueCEs.size(), ce);
if(i < 0) {
uniqueCEs.insertElementAt(ce, ~i, errorCode);
}
}
uint32_t
CollationFastLatinBuilder::getMiniCE(int64_t ce) const {
ce &= ~static_cast<int64_t>(Collation::CASE_MASK); // blank out case bits
int32_t index = binarySearch(uniqueCEs.getBuffer(), uniqueCEs.size(), ce);
U_ASSERT(index >= 0);
return miniCEs[index];
}
UBool
CollationFastLatinBuilder::encodeUniqueCEs(UErrorCode &errorCode) {
if(U_FAILURE(errorCode)) { return false; }
uprv_free(miniCEs);
miniCEs = static_cast<uint16_t*>(uprv_malloc(uniqueCEs.size() * 2));
if(miniCEs == nullptr) {
errorCode = U_MEMORY_ALLOCATION_ERROR;
return false;
}
int32_t group = 0;
uint32_t lastGroupPrimary = lastSpecialPrimaries[group];
// The lowest unique CE must be at least a secondary CE.
U_ASSERT(((uint32_t)uniqueCEs.elementAti(0) >> 16) != 0);
uint32_t prevPrimary = 0;
uint32_t prevSecondary = 0;
uint32_t pri = 0;
uint32_t sec = 0;
uint32_t ter = CollationFastLatin::COMMON_TER;
for(int32_t i = 0; i < uniqueCEs.size(); ++i) {
int64_t ce = uniqueCEs.elementAti(i);
// Note: At least one of the p/s/t weights changes from one unique CE to the next.
// (uniqueCEs does not store case bits.)
uint32_t p = static_cast<uint32_t>(ce >> 32);
if(p != prevPrimary) {
while(p > lastGroupPrimary) {
U_ASSERT(pri <= CollationFastLatin::MAX_LONG);
// Set the group's header entry to the
// last "long primary" in or before the group.
result.setCharAt(1 + group, static_cast<char16_t>(pri));
if(++group < NUM_SPECIAL_GROUPS) {
lastGroupPrimary = lastSpecialPrimaries[group];
} else {
lastGroupPrimary = 0xffffffff;
break;
}
}
if(p < firstShortPrimary) {
if(pri == 0) {
pri = CollationFastLatin::MIN_LONG;
} else if(pri < CollationFastLatin::MAX_LONG) {
pri += CollationFastLatin::LONG_INC;
} else {
#if DEBUG_COLLATION_FAST_LATIN_BUILDER
printf("long-primary overflow for %08x\n", p);
#endif
miniCEs[i] = CollationFastLatin::BAIL_OUT;
continue;
}
} else {
if(pri < CollationFastLatin::MIN_SHORT) {
pri = CollationFastLatin::MIN_SHORT;
} else if(pri < (CollationFastLatin::MAX_SHORT - CollationFastLatin::SHORT_INC)) {
// Reserve the highest primary weight for U+FFFF.
pri += CollationFastLatin::SHORT_INC;
} else {
#if DEBUG_COLLATION_FAST_LATIN_BUILDER
printf("short-primary overflow for %08x\n", p);
#endif
shortPrimaryOverflow = true;
miniCEs[i] = CollationFastLatin::BAIL_OUT;
continue;
}
}
prevPrimary = p;
prevSecondary = Collation::COMMON_WEIGHT16;
sec = CollationFastLatin::COMMON_SEC;
ter = CollationFastLatin::COMMON_TER;
}
uint32_t lower32 = static_cast<uint32_t>(ce);
uint32_t s = lower32 >> 16;
if(s != prevSecondary) {
if(pri == 0) {
if(sec == 0) {
sec = CollationFastLatin::MIN_SEC_HIGH;
} else if(sec < CollationFastLatin::MAX_SEC_HIGH) {
sec += CollationFastLatin::SEC_INC;
} else {
miniCEs[i] = CollationFastLatin::BAIL_OUT;
continue;
}
prevSecondary = s;
ter = CollationFastLatin::COMMON_TER;
} else if(s < Collation::COMMON_WEIGHT16) {
if(sec == CollationFastLatin::COMMON_SEC) {
sec = CollationFastLatin::MIN_SEC_BEFORE;
} else if(sec < CollationFastLatin::MAX_SEC_BEFORE) {
sec += CollationFastLatin::SEC_INC;
} else {
miniCEs[i] = CollationFastLatin::BAIL_OUT;
continue;
}
} else if(s == Collation::COMMON_WEIGHT16) {
sec = CollationFastLatin::COMMON_SEC;
} else {
if(sec < CollationFastLatin::MIN_SEC_AFTER) {
sec = CollationFastLatin::MIN_SEC_AFTER;
} else if(sec < CollationFastLatin::MAX_SEC_AFTER) {
sec += CollationFastLatin::SEC_INC;
} else {
miniCEs[i] = CollationFastLatin::BAIL_OUT;
continue;
}
}
prevSecondary = s;
ter = CollationFastLatin::COMMON_TER;
}
U_ASSERT((lower32 & Collation::CASE_MASK) == 0); // blanked out in uniqueCEs
uint32_t t = lower32 & Collation::ONLY_TERTIARY_MASK;
if(t > Collation::COMMON_WEIGHT16) {
if(ter < CollationFastLatin::MAX_TER_AFTER) {
++ter;
} else {
miniCEs[i] = CollationFastLatin::BAIL_OUT;
continue;
}
}
if(CollationFastLatin::MIN_LONG <= pri && pri <= CollationFastLatin::MAX_LONG) {
U_ASSERT(sec == CollationFastLatin::COMMON_SEC);
miniCEs[i] = static_cast<uint16_t>(pri | ter);
} else {
miniCEs[i] = static_cast<uint16_t>(pri | sec | ter);
}
}
#if DEBUG_COLLATION_FAST_LATIN_BUILDER
printf("last mini primary: %04x\n", pri);
#endif
#if DEBUG_COLLATION_FAST_LATIN_BUILDER >= 2
for(int32_t i = 0; i < uniqueCEs.size(); ++i) {
int64_t ce = uniqueCEs.elementAti(i);
printf("unique CE 0x%016lx -> 0x%04x\n", ce, miniCEs[i]);
}
#endif
return U_SUCCESS(errorCode);
}
UBool
CollationFastLatinBuilder::encodeCharCEs(UErrorCode &errorCode) {
if(U_FAILURE(errorCode)) { return false; }
int32_t miniCEsStart = result.length();
for(int32_t i = 0; i < CollationFastLatin::NUM_FAST_CHARS; ++i) {
result.append(static_cast<char16_t>(0)); // initialize to completely ignorable
}
int32_t indexBase = result.length();
for(int32_t i = 0; i < CollationFastLatin::NUM_FAST_CHARS; ++i) {
int64_t ce = charCEs[i][0];
if(isContractionCharCE(ce)) { continue; } // defer contraction
uint32_t miniCE = encodeTwoCEs(ce, charCEs[i][1]);
if(miniCE > 0xffff) {
// Note: There is a chance that this new expansion is the same as a previous one,
// and if so, then we could reuse the other expansion.
// However, that seems unlikely.
int32_t expansionIndex = result.length() - indexBase;
if (expansionIndex > static_cast<int32_t>(CollationFastLatin::INDEX_MASK)) {
miniCE = CollationFastLatin::BAIL_OUT;
} else {
result.append(static_cast<char16_t>(miniCE >> 16)).append(static_cast<char16_t>(miniCE));
miniCE = CollationFastLatin::EXPANSION | expansionIndex;
}
}
result.setCharAt(miniCEsStart + i, static_cast<char16_t>(miniCE));
}
return U_SUCCESS(errorCode);
}
UBool
CollationFastLatinBuilder::encodeContractions(UErrorCode &errorCode) {
// We encode all contraction lists so that the first word of a list
// terminates the previous list, and we only need one additional terminator at the end.
if(U_FAILURE(errorCode)) { return false; }
int32_t indexBase = headerLength + CollationFastLatin::NUM_FAST_CHARS;
int32_t firstContractionIndex = result.length();
for(int32_t i = 0; i < CollationFastLatin::NUM_FAST_CHARS; ++i) {
int64_t ce = charCEs[i][0];
if(!isContractionCharCE(ce)) { continue; }
int32_t contractionIndex = result.length() - indexBase;
if (contractionIndex > static_cast<int32_t>(CollationFastLatin::INDEX_MASK)) {
result.setCharAt(headerLength + i, CollationFastLatin::BAIL_OUT);
continue;
}
UBool firstTriple = true;
for (int32_t index = static_cast<int32_t>(ce) & 0x7fffffff;; index += 3) {
int32_t x = static_cast<int32_t>(contractionCEs.elementAti(index));
if (static_cast<uint32_t>(x) == CollationFastLatin::CONTR_CHAR_MASK && !firstTriple) { break; }
int64_t cce0 = contractionCEs.elementAti(index + 1);
int64_t cce1 = contractionCEs.elementAti(index + 2);
uint32_t miniCE = encodeTwoCEs(cce0, cce1);
if(miniCE == CollationFastLatin::BAIL_OUT) {
result.append(static_cast<char16_t>(x | (1 << CollationFastLatin::CONTR_LENGTH_SHIFT)));
} else if(miniCE <= 0xffff) {
result.append(static_cast<char16_t>(x | (2 << CollationFastLatin::CONTR_LENGTH_SHIFT)));
result.append(static_cast<char16_t>(miniCE));
} else {
result.append(static_cast<char16_t>(x | (3 << CollationFastLatin::CONTR_LENGTH_SHIFT)));
result.append(static_cast<char16_t>(miniCE >> 16)).append(static_cast<char16_t>(miniCE));
}
firstTriple = false;
}
// Note: There is a chance that this new contraction list is the same as a previous one,
// and if so, then we could truncate the result and reuse the other list.
// However, that seems unlikely.
result.setCharAt(headerLength + i,
static_cast<char16_t>(CollationFastLatin::CONTRACTION | contractionIndex));
}
if(result.length() > firstContractionIndex) {
// Terminate the last contraction list.
result.append(static_cast<char16_t>(CollationFastLatin::CONTR_CHAR_MASK));
}
if(result.isBogus()) {
errorCode = U_MEMORY_ALLOCATION_ERROR;
return false;
}
#if DEBUG_COLLATION_FAST_LATIN_BUILDER
printf("** fast Latin %d * 2 = %d bytes\n", result.length(), result.length() * 2);
puts(" header & below-digit groups map");
int32_t i = 0;
for(; i < headerLength; ++i) {
printf(" %04x", result[i]);
}
printf("\n char mini CEs");
U_ASSERT(CollationFastLatin::NUM_FAST_CHARS % 16 == 0);
for(; i < indexBase; i += 16) {
UChar32 c = i - headerLength;
if(c >= CollationFastLatin::LATIN_LIMIT) {
c = CollationFastLatin::PUNCT_START + c - CollationFastLatin::LATIN_LIMIT;
}
printf("\n %04x:", c);
for(int32_t j = 0; j < 16; ++j) {
printf(" %04x", result[i + j]);
}
}
printf("\n expansions & contractions");
for(; i < result.length(); ++i) {
if((i - indexBase) % 16 == 0) { puts(""); }
printf(" %04x", result[i]);
}
puts("");
#endif
return true;
}
uint32_t
CollationFastLatinBuilder::encodeTwoCEs(int64_t first, int64_t second) const {
if(first == 0) {
return 0; // completely ignorable
}
if(first == Collation::NO_CE) {
return CollationFastLatin::BAIL_OUT;
}
U_ASSERT((uint32_t)(first >> 32) != Collation::NO_CE_PRIMARY);
uint32_t miniCE = getMiniCE(first);
if(miniCE == CollationFastLatin::BAIL_OUT) { return miniCE; }
if(miniCE >= CollationFastLatin::MIN_SHORT) {
// Extract & copy the case bits.
// Shift them from normal CE bits 15..14 to mini CE bits 4..3.
uint32_t c = ((static_cast<uint32_t>(first) & Collation::CASE_MASK) >> (14 - 3));
// Only in mini CEs: Ignorable case bits = 0, lowercase = 1.
c += CollationFastLatin::LOWER_CASE;
miniCE |= c;
}
if(second == 0) { return miniCE; }
uint32_t miniCE1 = getMiniCE(second);
if(miniCE1 == CollationFastLatin::BAIL_OUT) { return miniCE1; }
uint32_t case1 = static_cast<uint32_t>(second) & Collation::CASE_MASK;
if(miniCE >= CollationFastLatin::MIN_SHORT &&
(miniCE & CollationFastLatin::SECONDARY_MASK) == CollationFastLatin::COMMON_SEC) {
// Try to combine the two mini CEs into one.
uint32_t sec1 = miniCE1 & CollationFastLatin::SECONDARY_MASK;
uint32_t ter1 = miniCE1 & CollationFastLatin::TERTIARY_MASK;
if(sec1 >= CollationFastLatin::MIN_SEC_HIGH && case1 == 0 &&
ter1 == CollationFastLatin::COMMON_TER) {
// sec1>=sec_high implies pri1==0.
return (miniCE & ~CollationFastLatin::SECONDARY_MASK) | sec1;
}
}
if(miniCE1 <= CollationFastLatin::SECONDARY_MASK || CollationFastLatin::MIN_SHORT <= miniCE1) {
// Secondary CE, or a CE with a short primary, copy the case bits.
case1 = (case1 >> (14 - 3)) + CollationFastLatin::LOWER_CASE;
miniCE1 |= case1;
}
return (miniCE << 16) | miniCE1;
}
U_NAMESPACE_END
#endif // !UCONFIG_NO_COLLATION

100
deps/icu-small/source/i18n/collationfastlatinbuilder.h vendored Normal file
View File

@ -0,0 +1,100 @@
// © 2016 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html
/*
*******************************************************************************
* Copyright (C) 2013-2016, International Business Machines
* Corporation and others. All Rights Reserved.
*******************************************************************************
* collationfastlatinbuilder.h
*
* created on: 2013aug09
* created by: Markus W. Scherer
*/
#ifndef __COLLATIONFASTLATINBUILDER_H__
#define __COLLATIONFASTLATINBUILDER_H__
#include "unicode/utypes.h"
#if !UCONFIG_NO_COLLATION
#include "unicode/ucol.h"
#include "unicode/unistr.h"
#include "unicode/uobject.h"
#include "collation.h"
#include "collationfastlatin.h"
#include "uvectr64.h"
U_NAMESPACE_BEGIN
struct CollationData;
class U_I18N_API CollationFastLatinBuilder : public UObject {
public:
CollationFastLatinBuilder(UErrorCode &errorCode);
~CollationFastLatinBuilder();
UBool forData(const CollationData &data, UErrorCode &errorCode);
const uint16_t *getTable() const {
return reinterpret_cast<const uint16_t *>(result.getBuffer());
}
int32_t lengthOfTable() const { return result.length(); }
private:
// space, punct, symbol, currency (not digit)
enum { NUM_SPECIAL_GROUPS = UCOL_REORDER_CODE_CURRENCY - UCOL_REORDER_CODE_FIRST + 1 };
UBool loadGroups(const CollationData &data, UErrorCode &errorCode);
UBool inSameGroup(uint32_t p, uint32_t q) const;
void resetCEs();
void getCEs(const CollationData &data, UErrorCode &errorCode);
UBool getCEsFromCE32(const CollationData &data, UChar32 c, uint32_t ce32,
UErrorCode &errorCode);
UBool getCEsFromContractionCE32(const CollationData &data, uint32_t ce32,
UErrorCode &errorCode);
void addContractionEntry(int32_t x, int64_t cce0, int64_t cce1, UErrorCode &errorCode);
void addUniqueCE(int64_t ce, UErrorCode &errorCode);
uint32_t getMiniCE(int64_t ce) const;
UBool encodeUniqueCEs(UErrorCode &errorCode);
UBool encodeCharCEs(UErrorCode &errorCode);
UBool encodeContractions(UErrorCode &errorCode);
uint32_t encodeTwoCEs(int64_t first, int64_t second) const;
static UBool isContractionCharCE(int64_t ce) {
return static_cast<uint32_t>(ce >> 32) == Collation::NO_CE_PRIMARY && ce != Collation::NO_CE;
}
static const uint32_t CONTRACTION_FLAG = 0x80000000;
// temporary "buffer"
int64_t ce0, ce1;
int64_t charCEs[CollationFastLatin::NUM_FAST_CHARS][2];
UVector64 contractionCEs;
UVector64 uniqueCEs;
/** One 16-bit mini CE per unique CE. */
uint16_t *miniCEs;
// These are constant for a given root collator.
uint32_t lastSpecialPrimaries[NUM_SPECIAL_GROUPS];
uint32_t firstDigitPrimary;
uint32_t firstLatinPrimary;
uint32_t lastLatinPrimary;
// This determines the first normal primary weight which is mapped to
// a short mini primary. It must be >=firstDigitPrimary.
uint32_t firstShortPrimary;
UBool shortPrimaryOverflow;
UnicodeString result;
int32_t headerLength;
};
U_NAMESPACE_END
#endif // !UCONFIG_NO_COLLATION
#endif // __COLLATIONFASTLATINBUILDER_H__

301
deps/icu-small/source/i18n/collationfcd.cpp vendored Normal file
View File

@ -0,0 +1,301 @@
// Copyright (C) 2016 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html
// Copyright (C) 1999-2016, International Business Machines
// Corporation and others. All Rights Reserved.
//
// file name: collationfcd.cpp
//
// machine-generated by: icu/tools/unicode/c/genuca/genuca.cpp
#include "unicode/utypes.h"
#if !UCONFIG_NO_COLLATION
#include "collationfcd.h"
U_NAMESPACE_BEGIN
const uint8_t CollationFCD::lcccIndex[2048]={
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,1,1,2,3,0,0,0,0,
0,0,0,0,4,0,0,0,0,0,0,0,5,6,7,0,
8,0,9,0xa,0,0,0xb,0xc,0xd,0xe,0xf,0,0,0,0,0x10,
0x11,0x12,0x13,0,0x14,0,0x15,0x16,0,0x17,0x18,0,0,0x17,0x19,0x1a,
0,0x17,0x19,0,0,0x17,0x19,0,0,0x17,0x19,0,0,0,0x19,0,
0,0x17,0x1b,0,0,0x17,0x19,0,0,0x1c,0x19,0,0,0,0x1d,0,
0,0x1e,0x1f,0,0,0x1e,0x1f,0,0x20,0x21,0,0x22,0x23,0,0x24,0,
0,0x25,0,0,0x19,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0x26,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0x27,0x28,0,0,0,0,0x29,0,
0,0,0,0,0,0x2a,0,0,0,0x13,0,0,0,0,0,0,
0x2b,0,0,0x2c,0,0x2d,0x2e,0,0,0x28,0x2f,0x30,0,0x31,0,0x32,
0,0x33,0,0,0,0,0x34,0x35,0,0,0,0,0,0,1,1,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0x36,0x37,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0x38,0,0,0,0x39,0,0,0,1,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0x3a,0,0,0x3b,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0x3c,0x3d,0,0,0x3e,0,0,0,0,0,0,0,0,
0x24,0x3f,0,0,0,0,0x2f,0x40,0,0x41,0x42,0,0,0x42,0x43,0,
0,0,0,0,0,0x44,0x45,0x46,0,0,0,0,0,0,0,0x19,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0x47,0x48,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0x1a,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,3,0,0,0,0,0,0,0,0,0,0,0,0,0,0
};
const uint32_t CollationFCD::lcccBits[73]={
0,0xffffffff,0xffff7fff,0xffff,0xf8,0xfffe0000,0xbfffffff,0xb6,0x7ff0000,0xfffff800,0x10000,0x9fc00000,0x3d9f,0x20000,0xffff0000,0x7ff,
0x200ff800,0xfbc00000,0x3eef,0xe000000,0xff800000,0xfffffc00,0xfffffffb,0x10000000,0x1e2000,0x2000,0x40000000,0x602000,0x18000000,0x400,0x7000000,0xf00,
0x3000000,0x2a00000,0x3c3e0000,0xdf,0x40,0x6800000,0xe0000000,0x300000,0x100000,0x20040000,0x200,0x1800000,0x9fe00001,0xbfff0000,0x7fff,0x10,
0xff800,0xc00,0xc0040,0x800000,0xfff70000,0x31021fd,0x1fff0000,0x1ffe2,0x38000,0x80000000,0xfc00,0x6000000,0x3ff08000,0xc0000000,0x30000,0x1000,
0x3ffff,0x3800,0x80000,1,0xc19d0000,2,0x400000,0xd0000fd,0x7108000
};
const uint8_t CollationFCD::tcccIndex[2048]={
0,0,0,0,0,0,2,3,4,5,6,7,0,8,9,0xa,
0xb,0xc,0,0,0,0,0,0,1,1,0xd,0xe,0xf,0x10,0x11,0,
0x12,0x13,0x14,0x15,0x16,0,0x17,0x18,0,0,0,0,0x19,0x1a,0x1b,0,
0x1c,0x1d,0x1e,0x1f,0,0,0x20,0x21,0x22,0x23,0x24,0,0,0,0,0x25,
0x26,0x27,0x28,0,0x29,0,0x2a,0x2b,0,0x2c,0x2d,0,0,0x2e,0x2f,0x30,
0,0x31,0x32,0,0,0x2e,0x33,0,0,0x2e,0x34,0,0,0,0x33,0,
0,0x2e,0x35,0,0,0x2e,0x33,0,0,0x36,0x33,0,0,0,0x37,0,
0,0x38,0x39,0,0,0x38,0x39,0,0x3a,0x3b,0,0x3c,0x3d,0,0x3e,0,
0,0x3f,0,0,0x33,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0x40,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0x41,0x42,0,0,0,0,0x43,0,
0,0,0,0,0,0x44,0,0,0,0x28,0,0,0,0,0,0,
0x45,0,0,0x46,0,0x47,0x48,0,0,0x42,0x49,0x4a,0,0x4b,0,0x4c,
0,0x4d,0,0,0,0,0x4e,0x4f,0,0,0,0,0,0,1,1,
1,1,1,1,0x50,1,1,0x51,0x52,1,0x53,0x54,1,0x55,0x56,0x57,
0,0,0,0,0,0,0x58,0x59,0,0x5a,0,0,0x5b,0x5c,0x5d,0,
0x5e,0x5f,0x60,0x61,0x62,0x63,0,0x64,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0x2e,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0x65,0,0,0,0x66,0,0,0,1,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0x67,0x68,0x69,0x6a,0x68,0x69,0x6b,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0x6c,0x6d,0,0,0x6e,0,0,0,0,0,0,0,0,
0x3e,0x6f,0,0,0,0,0x49,0x70,0,0x71,0x72,0,0,0x72,0x73,0,
0,0,0,0,0,0x74,0x75,0x76,0,0,0,0,0,0,0,0x33,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0x77,0x78,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0x40,0x79,0x7a,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0xe,0,0,0,0,0,0,0,0,0,0,0,0,0,0
};
const uint32_t CollationFCD::tcccBits[123]={
0,0xffffffff,0x3e7effbf,0xbe7effbf,0xfffcffff,0x7ef1ff3f,0xfff3f1f8,0x7fffff3f,0x18003,0xdfffe000,0xff31ffcf,0xcfffffff,0xfffc0,0xffff7fff,0xffff,0x1d760,
0x1fc00,0x187c00,0x200708b,0x2000000,0x708b0000,0xc00000,0xf8,0xfccf0006,0x33ffcfc,0xfffe0000,0xbfffffff,0xb6,0x7ff0000,0x7c,0xfffff800,0x10000,
0x9fc80005,0x3d9f,0x20000,0xffff0000,0x7ff,0x200ff800,0xfbc00000,0x3eef,0xe000000,0xff800000,0xfffffc00,0xfffffffb,0x10120200,0xff1e2000,0x10000000,0xb0002000,
0x40000000,0x10480000,0x4e002000,0x2000,0x30002000,0x602100,0x18000000,0x24000400,0x7000000,0xf00,0x3000000,0x2a00000,0x3d7e0000,0xdf,0x40,0x6800000,
0xe0000000,0x300000,0x100000,0x20040000,0x200,0x1800000,0x9fe00001,0xbfff0000,0x7fff,0x10,0xff800,0xc00,0xc0040,0x800000,0xfff70000,0x31021fd,
0xbffffff,0x3ffffff,0x3f3fffff,0xaaff3f3f,0x3fffffff,0x1fdfffff,0xefcfffde,0x1fdc7fff,0x1fff0000,0x1ffe2,0x800,0xc000000,0x4000,0xe000,0x1210,0x50,
0x292,0x333e005,0x333,0xf000,0x3c0f,0x38000,0x80000000,0xfc00,0x55555000,0x36db02a5,0x46100000,0x47900000,0x3ff08000,0xc0000000,0x30000,0x1000,
0x3ffff,0x3800,0x80000,1,0xc19d0000,2,0x400000,0xd0000ff,0x7108000,0x5f7ffc00,0x7fdb
};
U_NAMESPACE_END
#endif // !UCONFIG_NO_COLLATION

137
deps/icu-small/source/i18n/collationfcd.h vendored Normal file
View File

@ -0,0 +1,137 @@
// © 2016 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html
/*
*******************************************************************************
* Copyright (C) 2012-2014, International Business Machines
* Corporation and others. All Rights Reserved.
*******************************************************************************
* collationfcd.h
*
* created on: 2012aug18
* created by: Markus W. Scherer
*/
#ifndef __COLLATIONFCD_H__
#define __COLLATIONFCD_H__
#include "unicode/utypes.h"
#if !UCONFIG_NO_COLLATION
#include "unicode/utf16.h"
U_NAMESPACE_BEGIN
/**
* Data and functions for the FCD check fast path.
*
* The fast path looks at a pair of 16-bit code units and checks
* whether there is an FCD boundary between them;
* there is if the first unit has a trailing ccc=0 (!hasTccc(first))
* or the second unit has a leading ccc=0 (!hasLccc(second)),
* or both.
* When the fast path finds a possible non-boundary,
* then the FCD check slow path looks at the actual sequence of FCD values.
*
* This is a pure optimization.
* The fast path must at least find all possible non-boundaries.
* If the fast path is too pessimistic, it costs performance.
*
* For a pair of BMP characters, the fast path tests are precise (1 bit per character).
*
* For a supplementary code point, the two units are its lead and trail surrogates.
* We set hasTccc(lead)=true if any of its 1024 associated supplementary code points
* has lccc!=0 or tccc!=0.
* We set hasLccc(trail)=true for all trail surrogates.
* As a result, we leave the fast path if the lead surrogate might start a
* supplementary code point that is not FCD-inert.
* (So the fast path need not detect that there is a surrogate pair,
* nor look ahead to the next full code point.)
*
* hasLccc(lead)=true if any of its 1024 associated supplementary code points
* has lccc!=0, for fast boundary checking between BMP & supplementary.
*
* hasTccc(trail)=false:
* It should only be tested for unpaired trail surrogates which are FCD-inert.
*/
class U_I18N_API CollationFCD {
public:
static inline UBool hasLccc(UChar32 c) {
// assert c <= 0xffff
// c can be negative, e.g., U_SENTINEL from UCharIterator;
// that is handled in the first test.
int32_t i;
return
// U+0300 is the first character with lccc!=0.
c >= 0x300 &&
(i = lcccIndex[c >> 5]) != 0 &&
(lcccBits[i] & (static_cast<uint32_t>(1) << (c & 0x1f))) != 0;
}
static inline UBool hasTccc(UChar32 c) {
// assert c <= 0xffff
// c can be negative, e.g., U_SENTINEL from UCharIterator;
// that is handled in the first test.
int32_t i;
return
// U+00C0 is the first character with tccc!=0.
c >= 0xc0 &&
(i = tcccIndex[c >> 5]) != 0 &&
(tcccBits[i] & (static_cast<uint32_t>(1) << (c & 0x1f))) != 0;
}
static inline UBool mayHaveLccc(UChar32 c) {
// Handles all of Unicode 0..10FFFF.
// c can be negative, e.g., U_SENTINEL.
// U+0300 is the first character with lccc!=0.
if(c < 0x300) { return false; }
if(c > 0xffff) { c = U16_LEAD(c); }
int32_t i;
return
(i = lcccIndex[c >> 5]) != 0 &&
(lcccBits[i] & (static_cast<uint32_t>(1) << (c & 0x1f))) != 0;
}
/**
* Tibetan composite vowel signs (U+0F73, U+0F75, U+0F81)
* must be decomposed before reaching the core collation code,
* or else some sequences including them, even ones passing the FCD check,
* do not yield canonically equivalent results.
*
* This is a fast and imprecise test.
*
* @param c a code point
* @return true if c is U+0F73, U+0F75 or U+0F81 or one of several other Tibetan characters
*/
static inline UBool maybeTibetanCompositeVowel(UChar32 c) {
return (c & 0x1fff01) == 0xf01;
}
/**
* Tibetan composite vowel signs (U+0F73, U+0F75, U+0F81)
* must be decomposed before reaching the core collation code,
* or else some sequences including them, even ones passing the FCD check,
* do not yield canonically equivalent results.
*
* They have distinct lccc/tccc combinations: 129/130 or 129/132.
*
* @param fcd16 the FCD value (lccc/tccc combination) of a code point
* @return true if fcd16 is from U+0F73, U+0F75 or U+0F81
*/
static inline UBool isFCD16OfTibetanCompositeVowel(uint16_t fcd16) {
return fcd16 == 0x8182 || fcd16 == 0x8184;
}
private:
CollationFCD() = delete; // No instantiation.
static const uint8_t lcccIndex[2048];
static const uint8_t tcccIndex[2048];
static const uint32_t lcccBits[];
static const uint32_t tcccBits[];
};
U_NAMESPACE_END
#endif // !UCONFIG_NO_COLLATION
#endif // __COLLATIONFCD_H__

955
deps/icu-small/source/i18n/collationiterator.cpp vendored Normal file
View File

@ -0,0 +1,955 @@
// © 2016 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html
/*
*******************************************************************************
* Copyright (C) 2010-2014, International Business Machines
* Corporation and others. All Rights Reserved.
*******************************************************************************
* collationiterator.cpp
*
* created on: 2010oct27
* created by: Markus W. Scherer
*/
#include "utypeinfo.h" // for 'typeid' to work
#include "unicode/utypes.h"
#if !UCONFIG_NO_COLLATION
#include "unicode/ucharstrie.h"
#include "unicode/ustringtrie.h"
#include "charstr.h"
#include "cmemory.h"
#include "collation.h"
#include "collationdata.h"
#include "collationfcd.h"
#include "collationiterator.h"
#include "normalizer2impl.h"
#include "uassert.h"
#include "uvectr32.h"
U_NAMESPACE_BEGIN
CollationIterator::CEBuffer::~CEBuffer() {}
UBool
CollationIterator::CEBuffer::ensureAppendCapacity(int32_t appCap, UErrorCode &errorCode) {
int32_t capacity = buffer.getCapacity();
if((length + appCap) <= capacity) { return true; }
if(U_FAILURE(errorCode)) { return false; }
do {
if(capacity < 1000) {
capacity *= 4;
} else {
capacity *= 2;
}
} while(capacity < (length + appCap));
int64_t *p = buffer.resize(capacity, length);
if(p == nullptr) {
errorCode = U_MEMORY_ALLOCATION_ERROR;
return false;
}
return true;
}
// State of combining marks skipped in discontiguous contraction.
// We create a state object on first use and keep it around deactivated between uses.
class SkippedState : public UMemory {
public:
// Born active but empty.
SkippedState() : pos(0), skipLengthAtMatch(0) {}
void clear() {
oldBuffer.remove();
pos = 0;
// The newBuffer is reset by setFirstSkipped().
}
UBool isEmpty() const { return oldBuffer.isEmpty(); }
UBool hasNext() const { return pos < oldBuffer.length(); }
// Requires hasNext().
UChar32 next() {
UChar32 c = oldBuffer.char32At(pos);
pos += U16_LENGTH(c);
return c;
}
// Accounts for one more input code point read beyond the end of the marks buffer.
void incBeyond() {
U_ASSERT(!hasNext());
++pos;
}
// Goes backward through the skipped-marks buffer.
// Returns the number of code points read beyond the skipped marks
// that need to be backtracked through normal input.
int32_t backwardNumCodePoints(int32_t n) {
int32_t length = oldBuffer.length();
int32_t beyond = pos - length;
if(beyond > 0) {
if(beyond >= n) {
// Not back far enough to re-enter the oldBuffer.
pos -= n;
return n;
} else {
// Back out all beyond-oldBuffer code points and re-enter the buffer.
pos = oldBuffer.moveIndex32(length, beyond - n);
return beyond;
}
} else {
// Go backwards from inside the oldBuffer.
pos = oldBuffer.moveIndex32(pos, -n);
return 0;
}
}
void setFirstSkipped(UChar32 c) {
skipLengthAtMatch = 0;
newBuffer.setTo(c);
}
void skip(UChar32 c) {
newBuffer.append(c);
}
void recordMatch() { skipLengthAtMatch = newBuffer.length(); }
// Replaces the characters we consumed with the newly skipped ones.
void replaceMatch() {
// Note: UnicodeString.replace() pins pos to at most length().
oldBuffer.replace(0, pos, newBuffer, 0, skipLengthAtMatch);
pos = 0;
}
void saveTrieState(const UCharsTrie &trie) { trie.saveState(state); }
void resetToTrieState(UCharsTrie &trie) const { trie.resetToState(state); }
private:
// Combining marks skipped in previous discontiguous-contraction matching.
// After that discontiguous contraction was completed, we start reading them from here.
UnicodeString oldBuffer;
// Combining marks newly skipped in current discontiguous-contraction matching.
// These might have been read from the normal text or from the oldBuffer.
UnicodeString newBuffer;
// Reading index in oldBuffer,
// or counter for how many code points have been read beyond oldBuffer (pos-oldBuffer.length()).
int32_t pos;
// newBuffer.length() at the time of the last matching character.
// When a partial match fails, we back out skipped and partial-matching input characters.
int32_t skipLengthAtMatch;
// We save the trie state before we attempt to match a character,
// so that we can skip it and try the next one.
UCharsTrie::State state;
};
CollationIterator::CollationIterator(const CollationIterator &other)
: UObject(other),
trie(other.trie),
data(other.data),
cesIndex(other.cesIndex),
skipped(nullptr),
numCpFwd(other.numCpFwd),
isNumeric(other.isNumeric) {
UErrorCode errorCode = U_ZERO_ERROR;
int32_t length = other.ceBuffer.length;
if(length > 0 && ceBuffer.ensureAppendCapacity(length, errorCode)) {
for(int32_t i = 0; i < length; ++i) {
ceBuffer.set(i, other.ceBuffer.get(i));
}
ceBuffer.length = length;
} else {
cesIndex = 0;
}
}
CollationIterator::~CollationIterator() {
delete skipped;
}
bool
CollationIterator::operator==(const CollationIterator &other) const {
// Subclasses: Call this method and then add more specific checks.
// Compare the iterator state but not the collation data (trie & data fields):
// Assume that the caller compares the data.
// Ignore skipped since that should be unused between calls to nextCE().
// (It only stays around to avoid another memory allocation.)
if(!(typeid(*this) == typeid(other) &&
ceBuffer.length == other.ceBuffer.length &&
cesIndex == other.cesIndex &&
numCpFwd == other.numCpFwd &&
isNumeric == other.isNumeric)) {
return false;
}
for(int32_t i = 0; i < ceBuffer.length; ++i) {
if(ceBuffer.get(i) != other.ceBuffer.get(i)) { return false; }
}
return true;
}
void
CollationIterator::reset() {
cesIndex = ceBuffer.length = 0;
if(skipped != nullptr) { skipped->clear(); }
}
int32_t
CollationIterator::fetchCEs(UErrorCode &errorCode) {
while(U_SUCCESS(errorCode) && nextCE(errorCode) != Collation::NO_CE) {
// No need to loop for each expansion CE.
cesIndex = ceBuffer.length;
}
return ceBuffer.length;
}
uint32_t
CollationIterator::handleNextCE32(UChar32 &c, UErrorCode &errorCode) {
c = nextCodePoint(errorCode);
return (c < 0) ? Collation::FALLBACK_CE32 : data->getCE32(c);
}
char16_t
CollationIterator::handleGetTrailSurrogate() {
return 0;
}
UBool
CollationIterator::foundNULTerminator() {
return false;
}
UBool
CollationIterator::forbidSurrogateCodePoints() const {
return false;
}
uint32_t
CollationIterator::getDataCE32(UChar32 c) const {
return data->getCE32(c);
}
uint32_t
CollationIterator::getCE32FromBuilderData(uint32_t /*ce32*/, UErrorCode &errorCode) {
if(U_SUCCESS(errorCode)) { errorCode = U_INTERNAL_PROGRAM_ERROR; }
return 0;
}
int64_t
CollationIterator::nextCEFromCE32(const CollationData *d, UChar32 c, uint32_t ce32,
UErrorCode &errorCode) {
--ceBuffer.length; // Undo ceBuffer.incLength().
appendCEsFromCE32(d, c, ce32, true, errorCode);
if(U_SUCCESS(errorCode)) {
return ceBuffer.get(cesIndex++);
} else {
return Collation::NO_CE_PRIMARY;
}
}
void
CollationIterator::appendCEsFromCE32(const CollationData *d, UChar32 c, uint32_t ce32,
UBool forward, UErrorCode &errorCode) {
while(Collation::isSpecialCE32(ce32)) {
switch(Collation::tagFromCE32(ce32)) {
case Collation::FALLBACK_TAG:
case Collation::RESERVED_TAG_3:
if(U_SUCCESS(errorCode)) { errorCode = U_INTERNAL_PROGRAM_ERROR; }
return;
case Collation::LONG_PRIMARY_TAG:
ceBuffer.append(Collation::ceFromLongPrimaryCE32(ce32), errorCode);
return;
case Collation::LONG_SECONDARY_TAG:
ceBuffer.append(Collation::ceFromLongSecondaryCE32(ce32), errorCode);
return;
case Collation::LATIN_EXPANSION_TAG:
if(ceBuffer.ensureAppendCapacity(2, errorCode)) {
ceBuffer.set(ceBuffer.length, Collation::latinCE0FromCE32(ce32));
ceBuffer.set(ceBuffer.length + 1, Collation::latinCE1FromCE32(ce32));
ceBuffer.length += 2;
}
return;
case Collation::EXPANSION32_TAG: {
const uint32_t *ce32s = d->ce32s + Collation::indexFromCE32(ce32);
int32_t length = Collation::lengthFromCE32(ce32);
if(ceBuffer.ensureAppendCapacity(length, errorCode)) {
do {
ceBuffer.appendUnsafe(Collation::ceFromCE32(*ce32s++));
} while(--length > 0);
}
return;
}
case Collation::EXPANSION_TAG: {
const int64_t *ces = d->ces + Collation::indexFromCE32(ce32);
int32_t length = Collation::lengthFromCE32(ce32);
if(ceBuffer.ensureAppendCapacity(length, errorCode)) {
do {
ceBuffer.appendUnsafe(*ces++);
} while(--length > 0);
}
return;
}
case Collation::BUILDER_DATA_TAG:
ce32 = getCE32FromBuilderData(ce32, errorCode);
if(U_FAILURE(errorCode)) { return; }
if(ce32 == Collation::FALLBACK_CE32) {
d = data->base;
ce32 = d->getCE32(c);
}
break;
case Collation::PREFIX_TAG:
if(forward) { backwardNumCodePoints(1, errorCode); }
ce32 = getCE32FromPrefix(d, ce32, errorCode);
if(forward) { forwardNumCodePoints(1, errorCode); }
break;
case Collation::CONTRACTION_TAG: {
const char16_t *p = d->contexts + Collation::indexFromCE32(ce32);
uint32_t defaultCE32 = CollationData::readCE32(p); // Default if no suffix match.
if(!forward) {
// Backward contractions are handled by previousCEUnsafe().
// c has contractions but they were not found.
ce32 = defaultCE32;
break;
}
UChar32 nextCp;
if(skipped == nullptr && numCpFwd < 0) {
// Some portion of nextCE32FromContraction() pulled out here as an ASCII fast path,
// avoiding the function call and the nextSkippedCodePoint() overhead.
nextCp = nextCodePoint(errorCode);
if(nextCp < 0) {
// No more text.
ce32 = defaultCE32;
break;
} else if((ce32 & Collation::CONTRACT_NEXT_CCC) != 0 &&
!CollationFCD::mayHaveLccc(nextCp)) {
// All contraction suffixes start with characters with lccc!=0
// but the next code point has lccc==0.
backwardNumCodePoints(1, errorCode);
ce32 = defaultCE32;
break;
}
} else {
nextCp = nextSkippedCodePoint(errorCode);
if(nextCp < 0) {
// No more text.
ce32 = defaultCE32;
break;
} else if((ce32 & Collation::CONTRACT_NEXT_CCC) != 0 &&
!CollationFCD::mayHaveLccc(nextCp)) {
// All contraction suffixes start with characters with lccc!=0
// but the next code point has lccc==0.
backwardNumSkipped(1, errorCode);
ce32 = defaultCE32;
break;
}
}
ce32 = nextCE32FromContraction(d, ce32, p + 2, defaultCE32, nextCp, errorCode);
if(ce32 == Collation::NO_CE32) {
// CEs from a discontiguous contraction plus the skipped combining marks
// have been appended already.
return;
}
break;
}
case Collation::DIGIT_TAG:
if(isNumeric) {
appendNumericCEs(ce32, forward, errorCode);
return;
} else {
// Fetch the non-numeric-collation CE32 and continue.
ce32 = d->ce32s[Collation::indexFromCE32(ce32)];
break;
}
case Collation::U0000_TAG:
U_ASSERT(c == 0);
if(forward && foundNULTerminator()) {
// Handle NUL-termination. (Not needed in Java.)
ceBuffer.append(Collation::NO_CE, errorCode);
return;
} else {
// Fetch the normal ce32 for U+0000 and continue.
ce32 = d->ce32s[0];
break;
}
case Collation::HANGUL_TAG: {
const uint32_t *jamoCE32s = d->jamoCE32s;
c -= Hangul::HANGUL_BASE;
UChar32 t = c % Hangul::JAMO_T_COUNT;
c /= Hangul::JAMO_T_COUNT;
UChar32 v = c % Hangul::JAMO_V_COUNT;
c /= Hangul::JAMO_V_COUNT;
if((ce32 & Collation::HANGUL_NO_SPECIAL_JAMO) != 0) {
// None of the Jamo CE32s are isSpecialCE32().
// Avoid recursive function calls and per-Jamo tests.
if(ceBuffer.ensureAppendCapacity(t == 0 ? 2 : 3, errorCode)) {
ceBuffer.set(ceBuffer.length, Collation::ceFromCE32(jamoCE32s[c]));
ceBuffer.set(ceBuffer.length + 1, Collation::ceFromCE32(jamoCE32s[19 + v]));
ceBuffer.length += 2;
if(t != 0) {
ceBuffer.appendUnsafe(Collation::ceFromCE32(jamoCE32s[39 + t]));
}
}
return;
} else {
// We should not need to compute each Jamo code point.
// In particular, there should be no offset or implicit ce32.
appendCEsFromCE32(d, U_SENTINEL, jamoCE32s[c], forward, errorCode);
appendCEsFromCE32(d, U_SENTINEL, jamoCE32s[19 + v], forward, errorCode);
if(t == 0) { return; }
// offset 39 = 19 + 21 - 1:
// 19 = JAMO_L_COUNT
// 21 = JAMO_T_COUNT
// -1 = omit t==0
ce32 = jamoCE32s[39 + t];
c = U_SENTINEL;
break;
}
}
case Collation::LEAD_SURROGATE_TAG: {
U_ASSERT(forward); // Backward iteration should never see lead surrogate code _unit_ data.
U_ASSERT(U16_IS_LEAD(c));
char16_t trail;
if(U16_IS_TRAIL(trail = handleGetTrailSurrogate())) {
c = U16_GET_SUPPLEMENTARY(c, trail);
ce32 &= Collation::LEAD_TYPE_MASK;
if(ce32 == Collation::LEAD_ALL_UNASSIGNED) {
ce32 = Collation::UNASSIGNED_CE32; // unassigned-implicit
} else if(ce32 == Collation::LEAD_ALL_FALLBACK ||
(ce32 = d->getCE32FromSupplementary(c)) == Collation::FALLBACK_CE32) {
// fall back to the base data
d = d->base;
ce32 = d->getCE32FromSupplementary(c);
}
} else {
// c is an unpaired surrogate.
ce32 = Collation::UNASSIGNED_CE32;
}
break;
}
case Collation::OFFSET_TAG:
U_ASSERT(c >= 0);
ceBuffer.append(d->getCEFromOffsetCE32(c, ce32), errorCode);
return;
case Collation::IMPLICIT_TAG:
U_ASSERT(c >= 0);
if(U_IS_SURROGATE(c) && forbidSurrogateCodePoints()) {
ce32 = Collation::FFFD_CE32;
break;
} else {
ceBuffer.append(Collation::unassignedCEFromCodePoint(c), errorCode);
return;
}
}
}
ceBuffer.append(Collation::ceFromSimpleCE32(ce32), errorCode);
}
uint32_t
CollationIterator::getCE32FromPrefix(const CollationData *d, uint32_t ce32,
UErrorCode &errorCode) {
const char16_t *p = d->contexts + Collation::indexFromCE32(ce32);
ce32 = CollationData::readCE32(p); // Default if no prefix match.
p += 2;
// Number of code points read before the original code point.
int32_t lookBehind = 0;
UCharsTrie prefixes(p);
for(;;) {
UChar32 c = previousCodePoint(errorCode);
if(c < 0) { break; }
++lookBehind;
UStringTrieResult match = prefixes.nextForCodePoint(c);
if(USTRINGTRIE_HAS_VALUE(match)) {
ce32 = static_cast<uint32_t>(prefixes.getValue());
}
if(!USTRINGTRIE_HAS_NEXT(match)) { break; }
}
forwardNumCodePoints(lookBehind, errorCode);
return ce32;
}
UChar32
CollationIterator::nextSkippedCodePoint(UErrorCode &errorCode) {
if(skipped != nullptr && skipped->hasNext()) { return skipped->next(); }
if(numCpFwd == 0) { return U_SENTINEL; }
UChar32 c = nextCodePoint(errorCode);
if(skipped != nullptr && !skipped->isEmpty() && c >= 0) { skipped->incBeyond(); }
if(numCpFwd > 0 && c >= 0) { --numCpFwd; }
return c;
}
void
CollationIterator::backwardNumSkipped(int32_t n, UErrorCode &errorCode) {
if(skipped != nullptr && !skipped->isEmpty()) {
n = skipped->backwardNumCodePoints(n);
}
backwardNumCodePoints(n, errorCode);
if(numCpFwd >= 0) { numCpFwd += n; }
}
uint32_t
CollationIterator::nextCE32FromContraction(const CollationData *d, uint32_t contractionCE32,
const char16_t *p, uint32_t ce32, UChar32 c,
UErrorCode &errorCode) {
// c: next code point after the original one
// Number of code points read beyond the original code point.
// Needed for discontiguous contraction matching.
int32_t lookAhead = 1;
// Number of code points read since the last match (initially only c).
int32_t sinceMatch = 1;
// Normally we only need a contiguous match,
// and therefore need not remember the suffixes state from before a mismatch for retrying.
// If we are already processing skipped combining marks, then we do track the state.
UCharsTrie suffixes(p);
if(skipped != nullptr && !skipped->isEmpty()) { skipped->saveTrieState(suffixes); }
UStringTrieResult match = suffixes.firstForCodePoint(c);
for(;;) {
UChar32 nextCp;
if(USTRINGTRIE_HAS_VALUE(match)) {
ce32 = static_cast<uint32_t>(suffixes.getValue());
if(!USTRINGTRIE_HAS_NEXT(match) || (c = nextSkippedCodePoint(errorCode)) < 0) {
return ce32;
}
if(skipped != nullptr && !skipped->isEmpty()) { skipped->saveTrieState(suffixes); }
sinceMatch = 1;
} else if(match == USTRINGTRIE_NO_MATCH || (nextCp = nextSkippedCodePoint(errorCode)) < 0) {
// No match for c, or partial match (USTRINGTRIE_NO_VALUE) and no further text.
// Back up if necessary, and try a discontiguous contraction.
if((contractionCE32 & Collation::CONTRACT_TRAILING_CCC) != 0 &&
// Discontiguous contraction matching extends an existing match.
// If there is no match yet, then there is nothing to do.
((contractionCE32 & Collation::CONTRACT_SINGLE_CP_NO_MATCH) == 0 ||
sinceMatch < lookAhead)) {
// The last character of at least one suffix has lccc!=0,
// allowing for discontiguous contractions.
// UCA S2.1.1 only processes non-starters immediately following
// "a match in the table" (sinceMatch=1).
if(sinceMatch > 1) {
// Return to the state after the last match.
// (Return to sinceMatch=0 and re-fetch the first partially-matched character.)
backwardNumSkipped(sinceMatch, errorCode);
c = nextSkippedCodePoint(errorCode);
lookAhead -= sinceMatch - 1;
sinceMatch = 1;
}
if(d->getFCD16(c) > 0xff) {
return nextCE32FromDiscontiguousContraction(
d, suffixes, ce32, lookAhead, c, errorCode);
}
}
break;
} else {
// Continue after partial match (USTRINGTRIE_NO_VALUE) for c.
// It does not have a result value, therefore it is not itself "a match in the table".
// If a partially-matched c has ccc!=0 then
// it might be skipped in discontiguous contraction.
c = nextCp;
++sinceMatch;
}
++lookAhead;
match = suffixes.nextForCodePoint(c);
}
backwardNumSkipped(sinceMatch, errorCode);
return ce32;
}
uint32_t
CollationIterator::nextCE32FromDiscontiguousContraction(
const CollationData *d, UCharsTrie &suffixes, uint32_t ce32,
int32_t lookAhead, UChar32 c,
UErrorCode &errorCode) {
if(U_FAILURE(errorCode)) { return 0; }
// UCA section 3.3.2 Contractions:
// Contractions that end with non-starter characters
// are known as discontiguous contractions.
// ... discontiguous contractions must be detected in input text
// whenever the final sequence of non-starter characters could be rearranged
// so as to make a contiguous matching sequence that is canonically equivalent.
// UCA: http://www.unicode.org/reports/tr10/#S2.1
// S2.1 Find the longest initial substring S at each point that has a match in the table.
// S2.1.1 If there are any non-starters following S, process each non-starter C.
// S2.1.2 If C is not blocked from S, find if S + C has a match in the table.
// Note: A non-starter in a string is called blocked
// if there is another non-starter of the same canonical combining class or zero
// between it and the last character of canonical combining class 0.
// S2.1.3 If there is a match, replace S by S + C, and remove C.
// First: Is a discontiguous contraction even possible?
uint16_t fcd16 = d->getFCD16(c);
U_ASSERT(fcd16 > 0xff); // The caller checked this already, as a shortcut.
UChar32 nextCp = nextSkippedCodePoint(errorCode);
if(nextCp < 0) {
// No further text.
backwardNumSkipped(1, errorCode);
return ce32;
}
++lookAhead;
uint8_t prevCC = static_cast<uint8_t>(fcd16);
fcd16 = d->getFCD16(nextCp);
if(fcd16 <= 0xff) {
// The next code point after c is a starter (S2.1.1 "process each non-starter").
backwardNumSkipped(2, errorCode);
return ce32;
}
// We have read and matched (lookAhead-2) code points,
// read non-matching c and peeked ahead at nextCp.
// Return to the state before the mismatch and continue matching with nextCp.
if(skipped == nullptr || skipped->isEmpty()) {
if(skipped == nullptr) {
skipped = new SkippedState();
if(skipped == nullptr) {
errorCode = U_MEMORY_ALLOCATION_ERROR;
return 0;
}
}
suffixes.reset();
if(lookAhead > 2) {
// Replay the partial match so far.
backwardNumCodePoints(lookAhead, errorCode);
suffixes.firstForCodePoint(nextCodePoint(errorCode));
for(int32_t i = 3; i < lookAhead; ++i) {
suffixes.nextForCodePoint(nextCodePoint(errorCode));
}
// Skip c (which did not match) and nextCp (which we will try now).
forwardNumCodePoints(2, errorCode);
}
skipped->saveTrieState(suffixes);
} else {
// Reset to the trie state before the failed match of c.
skipped->resetToTrieState(suffixes);
}
skipped->setFirstSkipped(c);
// Number of code points read since the last match (at this point: c and nextCp).
int32_t sinceMatch = 2;
c = nextCp;
for(;;) {
UStringTrieResult match;
// "If C is not blocked from S, find if S + C has a match in the table." (S2.1.2)
if(prevCC < (fcd16 >> 8) && USTRINGTRIE_HAS_VALUE(match = suffixes.nextForCodePoint(c))) {
// "If there is a match, replace S by S + C, and remove C." (S2.1.3)
// Keep prevCC unchanged.
ce32 = static_cast<uint32_t>(suffixes.getValue());
sinceMatch = 0;
skipped->recordMatch();
if(!USTRINGTRIE_HAS_NEXT(match)) { break; }
skipped->saveTrieState(suffixes);
} else {
// No match for "S + C", skip C.
skipped->skip(c);
skipped->resetToTrieState(suffixes);
prevCC = static_cast<uint8_t>(fcd16);
}
if((c = nextSkippedCodePoint(errorCode)) < 0) { break; }
++sinceMatch;
fcd16 = d->getFCD16(c);
if(fcd16 <= 0xff) {
// The next code point after c is a starter (S2.1.1 "process each non-starter").
break;
}
}
backwardNumSkipped(sinceMatch, errorCode);
UBool isTopDiscontiguous = skipped->isEmpty();
skipped->replaceMatch();
if(isTopDiscontiguous && !skipped->isEmpty()) {
// We did get a match after skipping one or more combining marks,
// and we are not in a recursive discontiguous contraction.
// Append CEs from the contraction ce32
// and then from the combining marks that we skipped before the match.
c = U_SENTINEL;
for(;;) {
appendCEsFromCE32(d, c, ce32, true, errorCode);
// Fetch CE32s for skipped combining marks from the normal data, with fallback,
// rather than from the CollationData where we found the contraction.
if(!skipped->hasNext()) { break; }
c = skipped->next();
ce32 = getDataCE32(c);
if(ce32 == Collation::FALLBACK_CE32) {
d = data->base;
ce32 = d->getCE32(c);
} else {
d = data;
}
// Note: A nested discontiguous-contraction match
// replaces consumed combining marks with newly skipped ones
// and resets the reading position to the beginning.
}
skipped->clear();
ce32 = Collation::NO_CE32; // Signal to the caller that the result is in the ceBuffer.
}
return ce32;
}
void
CollationIterator::appendNumericCEs(uint32_t ce32, UBool forward, UErrorCode &errorCode) {
// Collect digits.
CharString digits;
if(forward) {
for(;;) {
char digit = Collation::digitFromCE32(ce32);
digits.append(digit, errorCode);
if(numCpFwd == 0) { break; }
UChar32 c = nextCodePoint(errorCode);
if(c < 0) { break; }
ce32 = data->getCE32(c);
if(ce32 == Collation::FALLBACK_CE32) {
ce32 = data->base->getCE32(c);
}
if(!Collation::hasCE32Tag(ce32, Collation::DIGIT_TAG)) {
backwardNumCodePoints(1, errorCode);
break;
}
if(numCpFwd > 0) { --numCpFwd; }
}
} else {
for(;;) {
char digit = Collation::digitFromCE32(ce32);
digits.append(digit, errorCode);
UChar32 c = previousCodePoint(errorCode);
if(c < 0) { break; }
ce32 = data->getCE32(c);
if(ce32 == Collation::FALLBACK_CE32) {
ce32 = data->base->getCE32(c);
}
if(!Collation::hasCE32Tag(ce32, Collation::DIGIT_TAG)) {
forwardNumCodePoints(1, errorCode);
break;
}
}
// Reverse the digit string.
char *p = digits.data();
char *q = p + digits.length() - 1;
while(p < q) {
char digit = *p;
*p++ = *q;
*q-- = digit;
}
}
if(U_FAILURE(errorCode)) { return; }
int32_t pos = 0;
do {
// Skip leading zeros.
while(pos < (digits.length() - 1) && digits[pos] == 0) { ++pos; }
// Write a sequence of CEs for at most 254 digits at a time.
int32_t segmentLength = digits.length() - pos;
if(segmentLength > 254) { segmentLength = 254; }
appendNumericSegmentCEs(digits.data() + pos, segmentLength, errorCode);
pos += segmentLength;
} while(U_SUCCESS(errorCode) && pos < digits.length());
}
void
CollationIterator::appendNumericSegmentCEs(const char *digits, int32_t length, UErrorCode &errorCode) {
U_ASSERT(1 <= length && length <= 254);
U_ASSERT(length == 1 || digits[0] != 0);
uint32_t numericPrimary = data->numericPrimary;
// Note: We use primary byte values 2..255: digits are not compressible.
if(length <= 7) {
// Very dense encoding for small numbers.
int32_t value = digits[0];
for(int32_t i = 1; i < length; ++i) {
value = value * 10 + digits[i];
}
// Primary weight second byte values:
// 74 byte values 2.. 75 for small numbers in two-byte primary weights.
// 40 byte values 76..115 for medium numbers in three-byte primary weights.
// 16 byte values 116..131 for large numbers in four-byte primary weights.
// 124 byte values 132..255 for very large numbers with 4..127 digit pairs.
int32_t firstByte = 2;
int32_t numBytes = 74;
if(value < numBytes) {
// Two-byte primary for 0..73, good for day & month numbers etc.
uint32_t primary = numericPrimary | ((firstByte + value) << 16);
ceBuffer.append(Collation::makeCE(primary), errorCode);
return;
}
value -= numBytes;
firstByte += numBytes;
numBytes = 40;
if(value < numBytes * 254) {
// Three-byte primary for 74..10233=74+40*254-1, good for year numbers and more.
uint32_t primary = numericPrimary |
((firstByte + value / 254) << 16) | ((2 + value % 254) << 8);
ceBuffer.append(Collation::makeCE(primary), errorCode);
return;
}
value -= numBytes * 254;
firstByte += numBytes;
numBytes = 16;
if(value < numBytes * 254 * 254) {
// Four-byte primary for 10234..1042489=10234+16*254*254-1.
uint32_t primary = numericPrimary | (2 + value % 254);
value /= 254;
primary |= (2 + value % 254) << 8;
value /= 254;
primary |= (firstByte + value % 254) << 16;
ceBuffer.append(Collation::makeCE(primary), errorCode);
return;
}
// original value > 1042489
}
U_ASSERT(length >= 7);
// The second primary byte value 132..255 indicates the number of digit pairs (4..127),
// then we generate primary bytes with those pairs.
// Omit trailing 00 pairs.
// Decrement the value for the last pair.
// Set the exponent. 4 pairs->132, 5 pairs->133, ..., 127 pairs->255.
int32_t numPairs = (length + 1) / 2;
uint32_t primary = numericPrimary | ((132 - 4 + numPairs) << 16);
// Find the length without trailing 00 pairs.
while(digits[length - 1] == 0 && digits[length - 2] == 0) {
length -= 2;
}
// Read the first pair.
uint32_t pair;
int32_t pos;
if(length & 1) {
// Only "half a pair" if we have an odd number of digits.
pair = digits[0];
pos = 1;
} else {
pair = digits[0] * 10 + digits[1];
pos = 2;
}
pair = 11 + 2 * pair;
// Add the pairs of digits between pos and length.
int32_t shift = 8;
while(pos < length) {
if(shift == 0) {
// Every three pairs/bytes we need to store a 4-byte-primary CE
// and start with a new CE with the '0' primary lead byte.
primary |= pair;
ceBuffer.append(Collation::makeCE(primary), errorCode);
primary = numericPrimary;
shift = 16;
} else {
primary |= pair << shift;
shift -= 8;
}
pair = 11 + 2 * (digits[pos] * 10 + digits[pos + 1]);
pos += 2;
}
primary |= (pair - 1) << shift;
ceBuffer.append(Collation::makeCE(primary), errorCode);
}
int64_t
CollationIterator::previousCE(UVector32 &offsets, UErrorCode &errorCode) {
if(ceBuffer.length > 0) {
// Return the previous buffered CE.
return ceBuffer.get(--ceBuffer.length);
}
offsets.removeAllElements();
int32_t limitOffset = getOffset();
UChar32 c = previousCodePoint(errorCode);
if(c < 0) { return Collation::NO_CE; }
if(data->isUnsafeBackward(c, isNumeric)) {
return previousCEUnsafe(c, offsets, errorCode);
}
// Simple, safe-backwards iteration:
// Get a CE going backwards, handle prefixes but no contractions.
uint32_t ce32 = data->getCE32(c);
const CollationData *d;
if(ce32 == Collation::FALLBACK_CE32) {
d = data->base;
ce32 = d->getCE32(c);
} else {
d = data;
}
if(Collation::isSimpleOrLongCE32(ce32)) {
return Collation::ceFromCE32(ce32);
}
appendCEsFromCE32(d, c, ce32, false, errorCode);
if(U_SUCCESS(errorCode)) {
if(ceBuffer.length > 1) {
offsets.addElement(getOffset(), errorCode);
// For an expansion, the offset of each non-initial CE is the limit offset,
// consistent with forward iteration.
while(offsets.size() <= ceBuffer.length) {
offsets.addElement(limitOffset, errorCode);
}
}
return ceBuffer.get(--ceBuffer.length);
} else {
return Collation::NO_CE_PRIMARY;
}
}
int64_t
CollationIterator::previousCEUnsafe(UChar32 c, UVector32 &offsets, UErrorCode &errorCode) {
// We just move through the input counting safe and unsafe code points
// without collecting the unsafe-backward substring into a buffer and
// switching to it.
// This is to keep the logic simple. Otherwise we would have to handle
// prefix matching going before the backward buffer, switching
// to iteration and back, etc.
// In the most important case of iterating over a normal string,
// reading from the string itself is already maximally fast.
// The only drawback there is that after getting the CEs we always
// skip backward to the safe character rather than switching out
// of a backwardBuffer.
// But this should not be the common case for previousCE(),
// and correctness and maintainability are more important than
// complex optimizations.
// Find the first safe character before c.
int32_t numBackward = 1;
while((c = previousCodePoint(errorCode)) >= 0) {
++numBackward;
if(!data->isUnsafeBackward(c, isNumeric)) {
break;
}
}
// Set the forward iteration limit.
// Note: This counts code points.
// We cannot enforce a limit in the middle of a surrogate pair or similar.
numCpFwd = numBackward;
// Reset the forward iterator.
cesIndex = 0;
U_ASSERT(ceBuffer.length == 0);
// Go forward and collect the CEs.
int32_t offset = getOffset();
while(numCpFwd > 0) {
// nextCE() normally reads one code point.
// Contraction matching and digit specials read more and check numCpFwd.
--numCpFwd;
// Append one or more CEs to the ceBuffer.
(void)nextCE(errorCode);
U_ASSERT(U_FAILURE(errorCode) || ceBuffer.get(ceBuffer.length - 1) != Collation::NO_CE);
// No need to loop for getting each expansion CE from nextCE().
cesIndex = ceBuffer.length;
// However, we need to write an offset for each CE.
// This is for CollationElementIterator::getOffset() to return
// intermediate offsets from the unsafe-backwards segment.
U_ASSERT(offsets.size() < ceBuffer.length);
offsets.addElement(offset, errorCode);
// For an expansion, the offset of each non-initial CE is the limit offset,
// consistent with forward iteration.
offset = getOffset();
while(offsets.size() < ceBuffer.length) {
offsets.addElement(offset, errorCode);
}
}
U_ASSERT(offsets.size() == ceBuffer.length);
// End offset corresponding to just after the unsafe-backwards segment.
offsets.addElement(offset, errorCode);
// Reset the forward iteration limit
// and move backward to before the segment for which we fetched CEs.
numCpFwd = -1;
backwardNumCodePoints(numBackward, errorCode);
// Use the collected CEs and return the last one.
cesIndex = 0; // Avoid cesIndex > ceBuffer.length when that gets decremented.
if(U_SUCCESS(errorCode)) {
return ceBuffer.get(--ceBuffer.length);
} else {
return Collation::NO_CE_PRIMARY;
}
}
U_NAMESPACE_END
#endif // !UCONFIG_NO_COLLATION

336
deps/icu-small/source/i18n/collationiterator.h vendored Normal file
View File

@ -0,0 +1,336 @@
// © 2016 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html
/*
*******************************************************************************
* Copyright (C) 2010-2014, International Business Machines
* Corporation and others. All Rights Reserved.
*******************************************************************************
* collationiterator.h
*
* created on: 2010oct27
* created by: Markus W. Scherer
*/
#ifndef __COLLATIONITERATOR_H__
#define __COLLATIONITERATOR_H__
#include "unicode/utypes.h"
#if !UCONFIG_NO_COLLATION
#include "cmemory.h"
#include "collation.h"
#include "collationdata.h"
U_NAMESPACE_BEGIN
class SkippedState;
class UCharsTrie;
class UVector32;
/* Large enough for CEs of most short strings. */
#define CEBUFFER_INITIAL_CAPACITY 40
// Export an explicit template instantiation of the MaybeStackArray that
// is used as a data member of CEBuffer.
//
// When building DLLs for Windows this is required even though
// no direct access to the MaybeStackArray leaks out of the i18n library.
//
// See digitlst.h, pluralaffix.h, datefmt.h, and others for similar examples.
//
#if U_PF_WINDOWS <= U_PLATFORM && U_PLATFORM <= U_PF_CYGWIN
template class U_I18N_API MaybeStackArray<int64_t, CEBUFFER_INITIAL_CAPACITY>;
#endif
/**
* Collation element iterator and abstract character iterator.
*
* When a method returns a code point value, it must be in 0..10FFFF,
* except it can be negative as a sentinel value.
*/
class U_I18N_API CollationIterator : public UObject {
private:
class U_I18N_API CEBuffer {
private:
/** Large enough for CEs of most short strings. */
static const int32_t INITIAL_CAPACITY = CEBUFFER_INITIAL_CAPACITY;
public:
CEBuffer() : length(0) {}
~CEBuffer();
inline void append(int64_t ce, UErrorCode &errorCode) {
if(length < INITIAL_CAPACITY || ensureAppendCapacity(1, errorCode)) {
buffer[length++] = ce;
}
}
inline void appendUnsafe(int64_t ce) {
buffer[length++] = ce;
}
UBool ensureAppendCapacity(int32_t appCap, UErrorCode &errorCode);
inline UBool incLength(UErrorCode &errorCode) {
// Use INITIAL_CAPACITY for a very simple fastpath.
// (Rather than buffer.getCapacity().)
if(length < INITIAL_CAPACITY || ensureAppendCapacity(1, errorCode)) {
++length;
return true;
} else {
return false;
}
}
inline int64_t set(int32_t i, int64_t ce) {
return buffer[i] = ce;
}
inline int64_t get(int32_t i) const { return buffer[i]; }
const int64_t *getCEs() const { return buffer.getAlias(); }
int32_t length;
private:
CEBuffer(const CEBuffer &) = delete;
void operator=(const CEBuffer &) = delete;
MaybeStackArray<int64_t, INITIAL_CAPACITY> buffer;
};
public:
CollationIterator(const CollationData *d, UBool numeric)
: trie(d->trie),
data(d),
cesIndex(0),
skipped(nullptr),
numCpFwd(-1),
isNumeric(numeric) {}
virtual ~CollationIterator();
virtual bool operator==(const CollationIterator &other) const;
inline bool operator!=(const CollationIterator &other) const {
return !operator==(other);
}
/**
* Resets the iterator state and sets the position to the specified offset.
* Subclasses must implement, and must call the parent class method,
* or CollationIterator::reset().
*/
virtual void resetToOffset(int32_t newOffset) = 0;
virtual int32_t getOffset() const = 0;
/**
* Returns the next collation element.
*/
inline int64_t nextCE(UErrorCode &errorCode) {
if(cesIndex < ceBuffer.length) {
// Return the next buffered CE.
return ceBuffer.get(cesIndex++);
}
// assert cesIndex == ceBuffer.length;
if(!ceBuffer.incLength(errorCode)) {
return Collation::NO_CE;
}
UChar32 c;
uint32_t ce32 = handleNextCE32(c, errorCode);
uint32_t t = ce32 & 0xff;
if(t < Collation::SPECIAL_CE32_LOW_BYTE) { // Forced-inline of isSpecialCE32(ce32).
// Normal CE from the main data.
// Forced-inline of ceFromSimpleCE32(ce32).
return ceBuffer.set(cesIndex++,
(static_cast<int64_t>(ce32 & 0xffff0000) << 32) | ((ce32 & 0xff00) << 16) | (t << 8));
}
const CollationData *d;
// The compiler should be able to optimize the previous and the following
// comparisons of t with the same constant.
if(t == Collation::SPECIAL_CE32_LOW_BYTE) {
if(c < 0) {
return ceBuffer.set(cesIndex++, Collation::NO_CE);
}
d = data->base;
ce32 = d->getCE32(c);
t = ce32 & 0xff;
if(t < Collation::SPECIAL_CE32_LOW_BYTE) {
// Normal CE from the base data.
return ceBuffer.set(cesIndex++,
(static_cast<int64_t>(ce32 & 0xffff0000) << 32) | ((ce32 & 0xff00) << 16) | (t << 8));
}
} else {
d = data;
}
if(t == Collation::LONG_PRIMARY_CE32_LOW_BYTE) {
// Forced-inline of ceFromLongPrimaryCE32(ce32).
return ceBuffer.set(cesIndex++,
(static_cast<int64_t>(ce32 - t) << 32) | Collation::COMMON_SEC_AND_TER_CE);
}
return nextCEFromCE32(d, c, ce32, errorCode);
}
/**
* Fetches all CEs.
* @return getCEsLength()
*/
int32_t fetchCEs(UErrorCode &errorCode);
/**
* Overwrites the current CE (the last one returned by nextCE()).
*/
void setCurrentCE(int64_t ce) {
// assert cesIndex > 0;
ceBuffer.set(cesIndex - 1, ce);
}
/**
* Returns the previous collation element.
*/
int64_t previousCE(UVector32 &offsets, UErrorCode &errorCode);
inline int32_t getCEsLength() const {
return ceBuffer.length;
}
inline int64_t getCE(int32_t i) const {
return ceBuffer.get(i);
}
const int64_t *getCEs() const {
return ceBuffer.getCEs();
}
void clearCEs() {
cesIndex = ceBuffer.length = 0;
}
void clearCEsIfNoneRemaining() {
if(cesIndex == ceBuffer.length) { clearCEs(); }
}
/**
* Returns the next code point (with post-increment).
* Public for identical-level comparison and for testing.
*/
virtual UChar32 nextCodePoint(UErrorCode &errorCode) = 0;
/**
* Returns the previous code point (with pre-decrement).
* Public for identical-level comparison and for testing.
*/
virtual UChar32 previousCodePoint(UErrorCode &errorCode) = 0;
protected:
CollationIterator(const CollationIterator &other);
void reset();
/**
* Returns the next code point and its local CE32 value.
* Returns Collation::FALLBACK_CE32 at the end of the text (c<0)
* or when c's CE32 value is to be looked up in the base data (fallback).
*
* The code point is used for fallbacks, context and implicit weights.
* It is ignored when the returned CE32 is not special (e.g., FFFD_CE32).
*/
virtual uint32_t handleNextCE32(UChar32 &c, UErrorCode &errorCode);
/**
* Called when handleNextCE32() returns a LEAD_SURROGATE_TAG for a lead surrogate code unit.
* Returns the trail surrogate in that case and advances past it,
* if a trail surrogate follows the lead surrogate.
* Otherwise returns any other code unit and does not advance.
*/
virtual char16_t handleGetTrailSurrogate();
/**
* Called when handleNextCE32() returns with c==0, to see whether it is a NUL terminator.
* (Not needed in Java.)
*/
virtual UBool foundNULTerminator();
/**
* @return false if surrogate code points U+D800..U+DFFF
* map to their own implicit primary weights (for UTF-16),
* or true if they map to CE(U+FFFD) (for UTF-8)
*/
virtual UBool forbidSurrogateCodePoints() const;
virtual void forwardNumCodePoints(int32_t num, UErrorCode &errorCode) = 0;
virtual void backwardNumCodePoints(int32_t num, UErrorCode &errorCode) = 0;
/**
* Returns the CE32 from the data trie.
* Normally the same as data->getCE32(), but overridden in the builder.
* Call this only when the faster data->getCE32() cannot be used.
*/
virtual uint32_t getDataCE32(UChar32 c) const;
virtual uint32_t getCE32FromBuilderData(uint32_t ce32, UErrorCode &errorCode);
void appendCEsFromCE32(const CollationData *d, UChar32 c, uint32_t ce32,
UBool forward, UErrorCode &errorCode);
// Main lookup trie of the data object.
const UTrie2 *trie;
const CollationData *data;
private:
int64_t nextCEFromCE32(const CollationData *d, UChar32 c, uint32_t ce32,
UErrorCode &errorCode);
uint32_t getCE32FromPrefix(const CollationData *d, uint32_t ce32,
UErrorCode &errorCode);
UChar32 nextSkippedCodePoint(UErrorCode &errorCode);
void backwardNumSkipped(int32_t n, UErrorCode &errorCode);
uint32_t nextCE32FromContraction(
const CollationData *d, uint32_t contractionCE32,
const char16_t *p, uint32_t ce32, UChar32 c,
UErrorCode &errorCode);
uint32_t nextCE32FromDiscontiguousContraction(
const CollationData *d, UCharsTrie &suffixes, uint32_t ce32,
int32_t lookAhead, UChar32 c,
UErrorCode &errorCode);
/**
* Returns the previous CE when data->isUnsafeBackward(c, isNumeric).
*/
int64_t previousCEUnsafe(UChar32 c, UVector32 &offsets, UErrorCode &errorCode);
/**
* Turns a string of digits (bytes 0..9)
* into a sequence of CEs that will sort in numeric order.
*
* Starts from this ce32's digit value and consumes the following/preceding digits.
* The digits string must not be empty and must not have leading zeros.
*/
void appendNumericCEs(uint32_t ce32, UBool forward, UErrorCode &errorCode);
/**
* Turns 1..254 digits into a sequence of CEs.
* Called by appendNumericCEs() for each segment of at most 254 digits.
*/
void appendNumericSegmentCEs(const char *digits, int32_t length, UErrorCode &errorCode);
CEBuffer ceBuffer;
int32_t cesIndex;
SkippedState *skipped;
// Number of code points to read forward, or -1.
// Used as a forward iteration limit in previousCEUnsafe().
int32_t numCpFwd;
// Numeric collation (CollationSettings::NUMERIC).
UBool isNumeric;
};
U_NAMESPACE_END
#endif // !UCONFIG_NO_COLLATION
#endif // __COLLATIONITERATOR_H__

678
deps/icu-small/source/i18n/collationkeys.cpp vendored Normal file
View File

@ -0,0 +1,678 @@
// © 2016 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html
/*
*******************************************************************************
* Copyright (C) 2012-2015, International Business Machines
* Corporation and others. All Rights Reserved.
*******************************************************************************
* collationkeys.cpp
*
* created on: 2012sep02
* created by: Markus W. Scherer
*/
#include "unicode/utypes.h"
#if !UCONFIG_NO_COLLATION
#include "unicode/bytestream.h"
#include "collation.h"
#include "collationiterator.h"
#include "collationkeys.h"
#include "collationsettings.h"
#include "uassert.h"
U_NAMESPACE_BEGIN
SortKeyByteSink::~SortKeyByteSink() {}
void
SortKeyByteSink::Append(const char *bytes, int32_t n) {
if (n <= 0 || bytes == nullptr) {
return;
}
if (ignore_ > 0) {
int32_t ignoreRest = ignore_ - n;
if (ignoreRest >= 0) {
ignore_ = ignoreRest;
return;
} else {
bytes += ignore_;
n = -ignoreRest;
ignore_ = 0;
}
}
int32_t length = appended_;
appended_ += n;
if ((buffer_ + length) == bytes) {
return; // the caller used GetAppendBuffer() and wrote the bytes already
}
int32_t available = capacity_ - length;
if (n <= available) {
uprv_memcpy(buffer_ + length, bytes, n);
} else {
AppendBeyondCapacity(bytes, n, length);
}
}
char *
SortKeyByteSink::GetAppendBuffer(int32_t min_capacity,
int32_t desired_capacity_hint,
char *scratch,
int32_t scratch_capacity,
int32_t *result_capacity) {
if (min_capacity < 1 || scratch_capacity < min_capacity) {
*result_capacity = 0;
return nullptr;
}
if (ignore_ > 0) {
// Do not write ignored bytes right at the end of the buffer.
*result_capacity = scratch_capacity;
return scratch;
}
int32_t available = capacity_ - appended_;
if (available >= min_capacity) {
*result_capacity = available;
return buffer_ + appended_;
} else if (Resize(desired_capacity_hint, appended_)) {
*result_capacity = capacity_ - appended_;
return buffer_ + appended_;
} else {
*result_capacity = scratch_capacity;
return scratch;
}
}
namespace {
/**
* uint8_t byte buffer, similar to CharString but simpler.
*/
class SortKeyLevel : public UMemory {
public:
SortKeyLevel() : len(0), ok(true) {}
~SortKeyLevel() {}
/** @return false if memory allocation failed */
UBool isOk() const { return ok; }
UBool isEmpty() const { return len == 0; }
int32_t length() const { return len; }
const uint8_t *data() const { return buffer.getAlias(); }
uint8_t operator[](int32_t index) const { return buffer[index]; }
uint8_t *data() { return buffer.getAlias(); }
void appendByte(uint32_t b);
void appendWeight16(uint32_t w);
void appendWeight32(uint32_t w);
void appendReverseWeight16(uint32_t w);
/** Appends all but the last byte to the sink. The last byte should be the 01 terminator. */
void appendTo(ByteSink &sink) const {
U_ASSERT(len > 0 && buffer[len - 1] == 1);
sink.Append(reinterpret_cast<const char *>(buffer.getAlias()), len - 1);
}
private:
MaybeStackArray<uint8_t, 40> buffer;
int32_t len;
UBool ok;
UBool ensureCapacity(int32_t appendCapacity);
SortKeyLevel(const SortKeyLevel &other); // forbid copying of this class
SortKeyLevel &operator=(const SortKeyLevel &other); // forbid copying of this class
};
void SortKeyLevel::appendByte(uint32_t b) {
if(len < buffer.getCapacity() || ensureCapacity(1)) {
buffer[len++] = static_cast<uint8_t>(b);
}
}
void
SortKeyLevel::appendWeight16(uint32_t w) {
U_ASSERT((w & 0xffff) != 0);
uint8_t b0 = static_cast<uint8_t>(w >> 8);
uint8_t b1 = static_cast<uint8_t>(w);
int32_t appendLength = (b1 == 0) ? 1 : 2;
if((len + appendLength) <= buffer.getCapacity() || ensureCapacity(appendLength)) {
buffer[len++] = b0;
if(b1 != 0) {
buffer[len++] = b1;
}
}
}
void
SortKeyLevel::appendWeight32(uint32_t w) {
U_ASSERT(w != 0);
uint8_t bytes[4] = {
static_cast<uint8_t>(w >> 24),
static_cast<uint8_t>(w >> 16),
static_cast<uint8_t>(w >> 8),
static_cast<uint8_t>(w)
};
int32_t appendLength = (bytes[1] == 0) ? 1 : (bytes[2] == 0) ? 2 : (bytes[3] == 0) ? 3 : 4;
if((len + appendLength) <= buffer.getCapacity() || ensureCapacity(appendLength)) {
buffer[len++] = bytes[0];
if(bytes[1] != 0) {
buffer[len++] = bytes[1];
if(bytes[2] != 0) {
buffer[len++] = bytes[2];
if(bytes[3] != 0) {
buffer[len++] = bytes[3];
}
}
}
}
}
void
SortKeyLevel::appendReverseWeight16(uint32_t w) {
U_ASSERT((w & 0xffff) != 0);
uint8_t b0 = static_cast<uint8_t>(w >> 8);
uint8_t b1 = static_cast<uint8_t>(w);
int32_t appendLength = (b1 == 0) ? 1 : 2;
if((len + appendLength) <= buffer.getCapacity() || ensureCapacity(appendLength)) {
if(b1 == 0) {
buffer[len++] = b0;
} else {
buffer[len] = b1;
buffer[len + 1] = b0;
len += 2;
}
}
}
UBool SortKeyLevel::ensureCapacity(int32_t appendCapacity) {
if(!ok) {
return false;
}
int32_t newCapacity = 2 * buffer.getCapacity();
int32_t altCapacity = len + 2 * appendCapacity;
if (newCapacity < altCapacity) {
newCapacity = altCapacity;
}
if (newCapacity < 200) {
newCapacity = 200;
}
if(buffer.resize(newCapacity, len)==nullptr) {
return ok = false;
}
return true;
}
} // namespace
CollationKeys::LevelCallback::~LevelCallback() {}
UBool
CollationKeys::LevelCallback::needToWrite(Collation::Level /*level*/) { return true; }
/**
* Map from collation strength (UColAttributeValue)
* to a mask of Collation::Level bits up to that strength,
* excluding the CASE_LEVEL which is independent of the strength,
* and excluding IDENTICAL_LEVEL which this function does not write.
*/
static const uint32_t levelMasks[UCOL_STRENGTH_LIMIT] = {
2, // UCOL_PRIMARY -> PRIMARY_LEVEL
6, // UCOL_SECONDARY -> up to SECONDARY_LEVEL
0x16, // UCOL_TERTIARY -> up to TERTIARY_LEVEL
0x36, // UCOL_QUATERNARY -> up to QUATERNARY_LEVEL
0, 0, 0, 0,
0, 0, 0, 0,
0, 0, 0,
0x36 // UCOL_IDENTICAL -> up to QUATERNARY_LEVEL
};
void
CollationKeys::writeSortKeyUpToQuaternary(CollationIterator &iter,
const UBool *compressibleBytes,
const CollationSettings &settings,
SortKeyByteSink &sink,
Collation::Level minLevel, LevelCallback &callback,
UBool preflight, UErrorCode &errorCode) {
if(U_FAILURE(errorCode)) { return; }
int32_t options = settings.options;
// Set of levels to process and write.
uint32_t levels = levelMasks[CollationSettings::getStrength(options)];
if((options & CollationSettings::CASE_LEVEL) != 0) {
levels |= Collation::CASE_LEVEL_FLAG;
}
// Minus the levels below minLevel.
levels &= ~((static_cast<uint32_t>(1) << minLevel) - 1);
if(levels == 0) { return; }
uint32_t variableTop;
if((options & CollationSettings::ALTERNATE_MASK) == 0) {
variableTop = 0;
} else {
// +1 so that we can use "<" and primary ignorables test out early.
variableTop = settings.variableTop + 1;
}
uint32_t tertiaryMask = CollationSettings::getTertiaryMask(options);
SortKeyLevel cases;
SortKeyLevel secondaries;
SortKeyLevel tertiaries;
SortKeyLevel quaternaries;
uint32_t prevReorderedPrimary = 0; // 0==no compression
int32_t commonCases = 0;
int32_t commonSecondaries = 0;
int32_t commonTertiaries = 0;
int32_t commonQuaternaries = 0;
uint32_t prevSecondary = 0;
int32_t secSegmentStart = 0;
for(;;) {
// No need to keep all CEs in the buffer when we write a sort key.
iter.clearCEsIfNoneRemaining();
int64_t ce = iter.nextCE(errorCode);
uint32_t p = static_cast<uint32_t>(ce >> 32);
if(p < variableTop && p > Collation::MERGE_SEPARATOR_PRIMARY) {
// Variable CE, shift it to quaternary level.
// Ignore all following primary ignorables, and shift further variable CEs.
if(commonQuaternaries != 0) {
--commonQuaternaries;
while(commonQuaternaries >= QUAT_COMMON_MAX_COUNT) {
quaternaries.appendByte(QUAT_COMMON_MIDDLE);
commonQuaternaries -= QUAT_COMMON_MAX_COUNT;
}
// Shifted primary weights are lower than the common weight.
quaternaries.appendByte(QUAT_COMMON_LOW + commonQuaternaries);
commonQuaternaries = 0;
}
do {
if((levels & Collation::QUATERNARY_LEVEL_FLAG) != 0) {
if(settings.hasReordering()) {
p = settings.reorder(p);
}
if((p >> 24) >= QUAT_SHIFTED_LIMIT_BYTE) {
// Prevent shifted primary lead bytes from
// overlapping with the common compression range.
quaternaries.appendByte(QUAT_SHIFTED_LIMIT_BYTE);
}
quaternaries.appendWeight32(p);
}
do {
ce = iter.nextCE(errorCode);
p = static_cast<uint32_t>(ce >> 32);
} while(p == 0);
} while(p < variableTop && p > Collation::MERGE_SEPARATOR_PRIMARY);
}
// ce could be primary ignorable, or NO_CE, or the merge separator,
// or a regular primary CE, but it is not variable.
// If ce==NO_CE, then write nothing for the primary level but
// terminate compression on all levels and then exit the loop.
if(p > Collation::NO_CE_PRIMARY && (levels & Collation::PRIMARY_LEVEL_FLAG) != 0) {
// Test the un-reordered primary for compressibility.
UBool isCompressible = compressibleBytes[p >> 24];
if(settings.hasReordering()) {
p = settings.reorder(p);
}
uint32_t p1 = p >> 24;
if(!isCompressible || p1 != (prevReorderedPrimary >> 24)) {
if(prevReorderedPrimary != 0) {
if(p < prevReorderedPrimary) {
// No primary compression terminator
// at the end of the level or merged segment.
if(p1 > Collation::MERGE_SEPARATOR_BYTE) {
sink.Append(Collation::PRIMARY_COMPRESSION_LOW_BYTE);
}
} else {
sink.Append(Collation::PRIMARY_COMPRESSION_HIGH_BYTE);
}
}
sink.Append(p1);
if(isCompressible) {
prevReorderedPrimary = p;
} else {
prevReorderedPrimary = 0;
}
}
char p2 = static_cast<char>(p >> 16);
if(p2 != 0) {
char buffer[3] = {p2, static_cast<char>(p >> 8), static_cast<char>(p)};
sink.Append(buffer, (buffer[1] == 0) ? 1 : (buffer[2] == 0) ? 2 : 3);
}
// Optimization for internalNextSortKeyPart():
// When the primary level overflows we can stop because we need not
// calculate (preflight) the whole sort key length.
if(!preflight && sink.Overflowed()) {
if(U_SUCCESS(errorCode) && !sink.IsOk()) {
errorCode = U_MEMORY_ALLOCATION_ERROR;
}
return;
}
}
uint32_t lower32 = static_cast<uint32_t>(ce);
if(lower32 == 0) { continue; } // completely ignorable, no secondary/case/tertiary/quaternary
if((levels & Collation::SECONDARY_LEVEL_FLAG) != 0) {
uint32_t s = lower32 >> 16;
if(s == 0) {
// secondary ignorable
} else if(s == Collation::COMMON_WEIGHT16 &&
((options & CollationSettings::BACKWARD_SECONDARY) == 0 ||
p != Collation::MERGE_SEPARATOR_PRIMARY)) {
// s is a common secondary weight, and
// backwards-secondary is off or the ce is not the merge separator.
++commonSecondaries;
} else if((options & CollationSettings::BACKWARD_SECONDARY) == 0) {
if(commonSecondaries != 0) {
--commonSecondaries;
while(commonSecondaries >= SEC_COMMON_MAX_COUNT) {
secondaries.appendByte(SEC_COMMON_MIDDLE);
commonSecondaries -= SEC_COMMON_MAX_COUNT;
}
uint32_t b;
if(s < Collation::COMMON_WEIGHT16) {
b = SEC_COMMON_LOW + commonSecondaries;
} else {
b = SEC_COMMON_HIGH - commonSecondaries;
}
secondaries.appendByte(b);
commonSecondaries = 0;
}
secondaries.appendWeight16(s);
} else {
if(commonSecondaries != 0) {
--commonSecondaries;
// Append reverse weights. The level will be re-reversed later.
int32_t remainder = commonSecondaries % SEC_COMMON_MAX_COUNT;
uint32_t b;
if(prevSecondary < Collation::COMMON_WEIGHT16) {
b = SEC_COMMON_LOW + remainder;
} else {
b = SEC_COMMON_HIGH - remainder;
}
secondaries.appendByte(b);
commonSecondaries -= remainder;
// commonSecondaries is now a multiple of SEC_COMMON_MAX_COUNT.
while(commonSecondaries > 0) { // same as >= SEC_COMMON_MAX_COUNT
secondaries.appendByte(SEC_COMMON_MIDDLE);
commonSecondaries -= SEC_COMMON_MAX_COUNT;
}
// commonSecondaries == 0
}
if(0 < p && p <= Collation::MERGE_SEPARATOR_PRIMARY) {
// The backwards secondary level compares secondary weights backwards
// within segments separated by the merge separator (U+FFFE).
uint8_t *secs = secondaries.data();
int32_t last = secondaries.length() - 1;
if(secSegmentStart < last) {
uint8_t *q = secs + secSegmentStart;
uint8_t *r = secs + last;
do {
uint8_t b = *q;
*q++ = *r;
*r-- = b;
} while(q < r);
}
secondaries.appendByte(p == Collation::NO_CE_PRIMARY ?
Collation::LEVEL_SEPARATOR_BYTE : Collation::MERGE_SEPARATOR_BYTE);
prevSecondary = 0;
secSegmentStart = secondaries.length();
} else {
secondaries.appendReverseWeight16(s);
prevSecondary = s;
}
}
}
if((levels & Collation::CASE_LEVEL_FLAG) != 0) {
if((CollationSettings::getStrength(options) == UCOL_PRIMARY) ?
p == 0 : lower32 <= 0xffff) {
// Primary+caseLevel: Ignore case level weights of primary ignorables.
// Otherwise: Ignore case level weights of secondary ignorables.
// For details see the comments in the CollationCompare class.
} else {
uint32_t c = (lower32 >> 8) & 0xff; // case bits & tertiary lead byte
U_ASSERT((c & 0xc0) != 0xc0);
if((c & 0xc0) == 0 && c > Collation::LEVEL_SEPARATOR_BYTE) {
++commonCases;
} else {
if((options & CollationSettings::UPPER_FIRST) == 0) {
// lowerFirst: Compress common weights to nibbles 1..7..13, mixed=14, upper=15.
// If there are only common (=lowest) weights in the whole level,
// then we need not write anything.
// Level length differences are handled already on the next-higher level.
if(commonCases != 0 &&
(c > Collation::LEVEL_SEPARATOR_BYTE || !cases.isEmpty())) {
--commonCases;
while(commonCases >= CASE_LOWER_FIRST_COMMON_MAX_COUNT) {
cases.appendByte(CASE_LOWER_FIRST_COMMON_MIDDLE << 4);
commonCases -= CASE_LOWER_FIRST_COMMON_MAX_COUNT;
}
uint32_t b;
if(c <= Collation::LEVEL_SEPARATOR_BYTE) {
b = CASE_LOWER_FIRST_COMMON_LOW + commonCases;
} else {
b = CASE_LOWER_FIRST_COMMON_HIGH - commonCases;
}
cases.appendByte(b << 4);
commonCases = 0;
}
if(c > Collation::LEVEL_SEPARATOR_BYTE) {
c = (CASE_LOWER_FIRST_COMMON_HIGH + (c >> 6)) << 4; // 14 or 15
}
} else {
// upperFirst: Compress common weights to nibbles 3..15, mixed=2, upper=1.
// The compressed common case weights only go up from the "low" value
// because with upperFirst the common weight is the highest one.
if(commonCases != 0) {
--commonCases;
while(commonCases >= CASE_UPPER_FIRST_COMMON_MAX_COUNT) {
cases.appendByte(CASE_UPPER_FIRST_COMMON_LOW << 4);
commonCases -= CASE_UPPER_FIRST_COMMON_MAX_COUNT;
}
cases.appendByte((CASE_UPPER_FIRST_COMMON_LOW + commonCases) << 4);
commonCases = 0;
}
if(c > Collation::LEVEL_SEPARATOR_BYTE) {
c = (CASE_UPPER_FIRST_COMMON_LOW - (c >> 6)) << 4; // 2 or 1
}
}
// c is a separator byte 01,
// or a left-shifted nibble 0x10, 0x20, ... 0xf0.
cases.appendByte(c);
}
}
}
if((levels & Collation::TERTIARY_LEVEL_FLAG) != 0) {
uint32_t t = lower32 & tertiaryMask;
U_ASSERT((lower32 & 0xc000) != 0xc000);
if(t == Collation::COMMON_WEIGHT16) {
++commonTertiaries;
} else if((tertiaryMask & 0x8000) == 0) {
// Tertiary weights without case bits.
// Move lead bytes 06..3F to C6..FF for a large common-weight range.
if(commonTertiaries != 0) {
--commonTertiaries;
while(commonTertiaries >= TER_ONLY_COMMON_MAX_COUNT) {
tertiaries.appendByte(TER_ONLY_COMMON_MIDDLE);
commonTertiaries -= TER_ONLY_COMMON_MAX_COUNT;
}
uint32_t b;
if(t < Collation::COMMON_WEIGHT16) {
b = TER_ONLY_COMMON_LOW + commonTertiaries;
} else {
b = TER_ONLY_COMMON_HIGH - commonTertiaries;
}
tertiaries.appendByte(b);
commonTertiaries = 0;
}
if(t > Collation::COMMON_WEIGHT16) { t += 0xc000; }
tertiaries.appendWeight16(t);
} else if((options & CollationSettings::UPPER_FIRST) == 0) {
// Tertiary weights with caseFirst=lowerFirst.
// Move lead bytes 06..BF to 46..FF for the common-weight range.
if(commonTertiaries != 0) {
--commonTertiaries;
while(commonTertiaries >= TER_LOWER_FIRST_COMMON_MAX_COUNT) {
tertiaries.appendByte(TER_LOWER_FIRST_COMMON_MIDDLE);
commonTertiaries -= TER_LOWER_FIRST_COMMON_MAX_COUNT;
}
uint32_t b;
if(t < Collation::COMMON_WEIGHT16) {
b = TER_LOWER_FIRST_COMMON_LOW + commonTertiaries;
} else {
b = TER_LOWER_FIRST_COMMON_HIGH - commonTertiaries;
}
tertiaries.appendByte(b);
commonTertiaries = 0;
}
if(t > Collation::COMMON_WEIGHT16) { t += 0x4000; }
tertiaries.appendWeight16(t);
} else {
// Tertiary weights with caseFirst=upperFirst.
// Do not change the artificial uppercase weight of a tertiary CE (0.0.ut),
// to keep tertiary CEs well-formed.
// Their case+tertiary weights must be greater than those of
// primary and secondary CEs.
//
// Separator 01 -> 01 (unchanged)
// Lowercase 02..04 -> 82..84 (includes uncased)
// Common weight 05 -> 85..C5 (common-weight compression range)
// Lowercase 06..3F -> C6..FF
// Mixed case 42..7F -> 42..7F
// Uppercase 82..BF -> 02..3F
// Tertiary CE 86..BF -> C6..FF
if(t <= Collation::NO_CE_WEIGHT16) {
// Keep separators unchanged.
} else if(lower32 > 0xffff) {
// Invert case bits of primary & secondary CEs.
t ^= 0xc000;
if(t < (TER_UPPER_FIRST_COMMON_HIGH << 8)) {
t -= 0x4000;
}
} else {
// Keep uppercase bits of tertiary CEs.
U_ASSERT(0x8600 <= t && t <= 0xbfff);
t += 0x4000;
}
if(commonTertiaries != 0) {
--commonTertiaries;
while(commonTertiaries >= TER_UPPER_FIRST_COMMON_MAX_COUNT) {
tertiaries.appendByte(TER_UPPER_FIRST_COMMON_MIDDLE);
commonTertiaries -= TER_UPPER_FIRST_COMMON_MAX_COUNT;
}
uint32_t b;
if(t < (TER_UPPER_FIRST_COMMON_LOW << 8)) {
b = TER_UPPER_FIRST_COMMON_LOW + commonTertiaries;
} else {
b = TER_UPPER_FIRST_COMMON_HIGH - commonTertiaries;
}
tertiaries.appendByte(b);
commonTertiaries = 0;
}
tertiaries.appendWeight16(t);
}
}
if((levels & Collation::QUATERNARY_LEVEL_FLAG) != 0) {
uint32_t q = lower32 & 0xffff;
if((q & 0xc0) == 0 && q > Collation::NO_CE_WEIGHT16) {
++commonQuaternaries;
} else if(q == Collation::NO_CE_WEIGHT16 &&
(options & CollationSettings::ALTERNATE_MASK) == 0 &&
quaternaries.isEmpty()) {
// If alternate=non-ignorable and there are only common quaternary weights,
// then we need not write anything.
// The only weights greater than the merge separator and less than the common weight
// are shifted primary weights, which are not generated for alternate=non-ignorable.
// There are also exactly as many quaternary weights as tertiary weights,
// so level length differences are handled already on tertiary level.
// Any above-common quaternary weight will compare greater regardless.
quaternaries.appendByte(Collation::LEVEL_SEPARATOR_BYTE);
} else {
if(q == Collation::NO_CE_WEIGHT16) {
q = Collation::LEVEL_SEPARATOR_BYTE;
} else {
q = 0xfc + ((q >> 6) & 3);
}
if(commonQuaternaries != 0) {
--commonQuaternaries;
while(commonQuaternaries >= QUAT_COMMON_MAX_COUNT) {
quaternaries.appendByte(QUAT_COMMON_MIDDLE);
commonQuaternaries -= QUAT_COMMON_MAX_COUNT;
}
uint32_t b;
if(q < QUAT_COMMON_LOW) {
b = QUAT_COMMON_LOW + commonQuaternaries;
} else {
b = QUAT_COMMON_HIGH - commonQuaternaries;
}
quaternaries.appendByte(b);
commonQuaternaries = 0;
}
quaternaries.appendByte(q);
}
}
if((lower32 >> 24) == Collation::LEVEL_SEPARATOR_BYTE) { break; } // ce == NO_CE
}
if(U_FAILURE(errorCode)) { return; }
// Append the beyond-primary levels.
UBool ok = true;
if((levels & Collation::SECONDARY_LEVEL_FLAG) != 0) {
if(!callback.needToWrite(Collation::SECONDARY_LEVEL)) { return; }
ok &= secondaries.isOk();
sink.Append(Collation::LEVEL_SEPARATOR_BYTE);
secondaries.appendTo(sink);
}
if((levels & Collation::CASE_LEVEL_FLAG) != 0) {
if(!callback.needToWrite(Collation::CASE_LEVEL)) { return; }
ok &= cases.isOk();
sink.Append(Collation::LEVEL_SEPARATOR_BYTE);
// Write pairs of nibbles as bytes, except separator bytes as themselves.
int32_t length = cases.length() - 1; // Ignore the trailing NO_CE.
uint8_t b = 0;
for(int32_t i = 0; i < length; ++i) {
uint8_t c = cases[i];
U_ASSERT((c & 0xf) == 0 && c != 0);
if(b == 0) {
b = c;
} else {
sink.Append(b | (c >> 4));
b = 0;
}
}
if(b != 0) {
sink.Append(b);
}
}
if((levels & Collation::TERTIARY_LEVEL_FLAG) != 0) {
if(!callback.needToWrite(Collation::TERTIARY_LEVEL)) { return; }
ok &= tertiaries.isOk();
sink.Append(Collation::LEVEL_SEPARATOR_BYTE);
tertiaries.appendTo(sink);
}
if((levels & Collation::QUATERNARY_LEVEL_FLAG) != 0) {
if(!callback.needToWrite(Collation::QUATERNARY_LEVEL)) { return; }
ok &= quaternaries.isOk();
sink.Append(Collation::LEVEL_SEPARATOR_BYTE);
quaternaries.appendTo(sink);
}
if(!ok || !sink.IsOk()) {
errorCode = U_MEMORY_ALLOCATION_ERROR;
}
}
U_NAMESPACE_END
#endif // !UCONFIG_NO_COLLATION

169
deps/icu-small/source/i18n/collationkeys.h vendored Normal file
View File

@ -0,0 +1,169 @@
// © 2016 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html
/*
*******************************************************************************
* Copyright (C) 2012-2014, International Business Machines
* Corporation and others. All Rights Reserved.
*******************************************************************************
* collationkeys.h
*
* created on: 2012sep02
* created by: Markus W. Scherer
*/
#ifndef __COLLATIONKEYS_H__
#define __COLLATIONKEYS_H__
#include "unicode/utypes.h"
#if !UCONFIG_NO_COLLATION
#include "unicode/bytestream.h"
#include "unicode/ucol.h"
#include "charstr.h"
#include "collation.h"
U_NAMESPACE_BEGIN
class CollationIterator;
struct CollationDataReader;
struct CollationSettings;
class SortKeyByteSink : public ByteSink {
public:
SortKeyByteSink(char *dest, int32_t destCapacity)
: buffer_(dest), capacity_(destCapacity),
appended_(0), ignore_(0) {}
virtual ~SortKeyByteSink();
void IgnoreBytes(int32_t numIgnore) { ignore_ = numIgnore; }
virtual void Append(const char *bytes, int32_t n) override;
void Append(uint32_t b) {
if (ignore_ > 0) {
--ignore_;
} else {
if (appended_ < capacity_ || Resize(1, appended_)) {
buffer_[appended_] = static_cast<char>(b);
}
++appended_;
}
}
virtual char *GetAppendBuffer(int32_t min_capacity,
int32_t desired_capacity_hint,
char *scratch, int32_t scratch_capacity,
int32_t *result_capacity) override;
int32_t NumberOfBytesAppended() const { return appended_; }
/**
* @return how many bytes can be appended (including ignored ones)
* without reallocation
*/
int32_t GetRemainingCapacity() const {
// Either ignore_ or appended_ should be 0.
return ignore_ + capacity_ - appended_;
}
UBool Overflowed() const { return appended_ > capacity_; }
/** @return false if memory allocation failed */
UBool IsOk() const { return buffer_ != nullptr; }
protected:
virtual void AppendBeyondCapacity(const char *bytes, int32_t n, int32_t length) = 0;
virtual UBool Resize(int32_t appendCapacity, int32_t length) = 0;
void SetNotOk() {
buffer_ = nullptr;
capacity_ = 0;
}
char *buffer_;
int32_t capacity_;
int32_t appended_;
int32_t ignore_;
private:
SortKeyByteSink(const SortKeyByteSink &); // copy constructor not implemented
SortKeyByteSink &operator=(const SortKeyByteSink &); // assignment operator not implemented
};
class U_I18N_API CollationKeys /* not : public UObject because all methods are static */ {
public:
class LevelCallback : public UMemory {
public:
virtual ~LevelCallback();
/**
* @param level The next level about to be written to the ByteSink.
* @return true if the level is to be written
* (the base class implementation always returns true)
*/
virtual UBool needToWrite(Collation::Level level);
};
/**
* Writes the sort key bytes for minLevel up to the iterator data's strength.
* Optionally writes the case level.
* Stops writing levels when callback.needToWrite(level) returns false.
* Separates levels with the LEVEL_SEPARATOR_BYTE
* but does not write a TERMINATOR_BYTE.
*/
static void writeSortKeyUpToQuaternary(CollationIterator &iter,
const UBool *compressibleBytes,
const CollationSettings &settings,
SortKeyByteSink &sink,
Collation::Level minLevel, LevelCallback &callback,
UBool preflight, UErrorCode &errorCode);
private:
friend struct CollationDataReader;
CollationKeys() = delete; // no instantiation
// Secondary level: Compress up to 33 common weights as 05..25 or 25..45.
static const uint32_t SEC_COMMON_LOW = Collation::COMMON_BYTE;
static const uint32_t SEC_COMMON_MIDDLE = SEC_COMMON_LOW + 0x20;
static const uint32_t SEC_COMMON_HIGH = SEC_COMMON_LOW + 0x40;
static const int32_t SEC_COMMON_MAX_COUNT = 0x21;
// Case level, lowerFirst: Compress up to 7 common weights as 1..7 or 7..13.
static const uint32_t CASE_LOWER_FIRST_COMMON_LOW = 1;
static const uint32_t CASE_LOWER_FIRST_COMMON_MIDDLE = 7;
static const uint32_t CASE_LOWER_FIRST_COMMON_HIGH = 13;
static const int32_t CASE_LOWER_FIRST_COMMON_MAX_COUNT = 7;
// Case level, upperFirst: Compress up to 13 common weights as 3..15.
static const uint32_t CASE_UPPER_FIRST_COMMON_LOW = 3;
static const uint32_t CASE_UPPER_FIRST_COMMON_HIGH = 15;
static const int32_t CASE_UPPER_FIRST_COMMON_MAX_COUNT = 13;
// Tertiary level only (no case): Compress up to 97 common weights as 05..65 or 65..C5.
static const uint32_t TER_ONLY_COMMON_LOW = Collation::COMMON_BYTE;
static const uint32_t TER_ONLY_COMMON_MIDDLE = TER_ONLY_COMMON_LOW + 0x60;
static const uint32_t TER_ONLY_COMMON_HIGH = TER_ONLY_COMMON_LOW + 0xc0;
static const int32_t TER_ONLY_COMMON_MAX_COUNT = 0x61;
// Tertiary with case, lowerFirst: Compress up to 33 common weights as 05..25 or 25..45.
static const uint32_t TER_LOWER_FIRST_COMMON_LOW = Collation::COMMON_BYTE;
static const uint32_t TER_LOWER_FIRST_COMMON_MIDDLE = TER_LOWER_FIRST_COMMON_LOW + 0x20;
static const uint32_t TER_LOWER_FIRST_COMMON_HIGH = TER_LOWER_FIRST_COMMON_LOW + 0x40;
static const int32_t TER_LOWER_FIRST_COMMON_MAX_COUNT = 0x21;
// Tertiary with case, upperFirst: Compress up to 33 common weights as 85..A5 or A5..C5.
static const uint32_t TER_UPPER_FIRST_COMMON_LOW = Collation::COMMON_BYTE + 0x80;
static const uint32_t TER_UPPER_FIRST_COMMON_MIDDLE = TER_UPPER_FIRST_COMMON_LOW + 0x20;
static const uint32_t TER_UPPER_FIRST_COMMON_HIGH = TER_UPPER_FIRST_COMMON_LOW + 0x40;
static const int32_t TER_UPPER_FIRST_COMMON_MAX_COUNT = 0x21;
// Quaternary level: Compress up to 113 common weights as 1C..8C or 8C..FC.
static const uint32_t QUAT_COMMON_LOW = 0x1c;
static const uint32_t QUAT_COMMON_MIDDLE = QUAT_COMMON_LOW + 0x70;
static const uint32_t QUAT_COMMON_HIGH = QUAT_COMMON_LOW + 0xE0;
static const int32_t QUAT_COMMON_MAX_COUNT = 0x71;
// Primary weights shifted to quaternary level must be encoded with
// a lead byte below the common-weight compression range.
static const uint32_t QUAT_SHIFTED_LIMIT_BYTE = QUAT_COMMON_LOW - 1; // 0x1b
};
U_NAMESPACE_END
#endif // !UCONFIG_NO_COLLATION
#endif // __COLLATIONKEYS_H__

140
deps/icu-small/source/i18n/collationroot.cpp vendored Normal file
View File

@ -0,0 +1,140 @@
// © 2016 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html
/*
*******************************************************************************
* Copyright (C) 2012-2014, International Business Machines
* Corporation and others. All Rights Reserved.
*******************************************************************************
* collationroot.cpp
*
* created on: 2012dec17
* created by: Markus W. Scherer
*/
#include "unicode/utypes.h"
#if !UCONFIG_NO_COLLATION
#include "unicode/coll.h"
#include "unicode/udata.h"
#include "collation.h"
#include "collationdata.h"
#include "collationdatareader.h"
#include "collationroot.h"
#include "collationsettings.h"
#include "collationtailoring.h"
#include "normalizer2impl.h"
#include "ucln_in.h"
#include "udatamem.h"
#include "umutex.h"
#include "umapfile.h"
U_NAMESPACE_BEGIN
namespace {
const CollationCacheEntry *rootSingleton = nullptr;
UInitOnce initOnce{};
} // namespace
U_CDECL_BEGIN
static UBool U_CALLCONV uprv_collation_root_cleanup() {
SharedObject::clearPtr(rootSingleton);
initOnce.reset();
return true;
}
U_CDECL_END
UDataMemory*
CollationRoot::loadFromFile(const char* ucadataPath, UErrorCode &errorCode) {
UDataMemory dataMemory;
UDataMemory *rDataMem = nullptr;
if (U_FAILURE(errorCode)) {
return nullptr;
}
if (uprv_mapFile(&dataMemory, ucadataPath, &errorCode)) {
if (dataMemory.pHeader->dataHeader.magic1 == 0xda &&
dataMemory.pHeader->dataHeader.magic2 == 0x27 &&
CollationDataReader::isAcceptable(nullptr, "icu", "ucadata", &dataMemory.pHeader->info)) {
rDataMem = UDataMemory_createNewInstance(&errorCode);
if (U_FAILURE(errorCode)) {
return nullptr;
}
rDataMem->pHeader = dataMemory.pHeader;
rDataMem->mapAddr = dataMemory.mapAddr;
rDataMem->map = dataMemory.map;
return rDataMem;
}
errorCode = U_INVALID_FORMAT_ERROR;
return nullptr;
}
errorCode = U_MISSING_RESOURCE_ERROR;
return nullptr;
}
void U_CALLCONV
CollationRoot::load(const char* ucadataPath, UErrorCode &errorCode) {
if(U_FAILURE(errorCode)) { return; }
LocalPointer<CollationTailoring> t(new CollationTailoring(nullptr));
if(t.isNull() || t->isBogus()) {
errorCode = U_MEMORY_ALLOCATION_ERROR;
return;
}
t->memory = ucadataPath ? CollationRoot::loadFromFile(ucadataPath, errorCode) :
udata_openChoice(U_ICUDATA_NAME U_TREE_SEPARATOR_STRING "coll",
"icu", "ucadata",
CollationDataReader::isAcceptable,
t->version, &errorCode);
if(U_FAILURE(errorCode)) { return; }
const uint8_t *inBytes = static_cast<const uint8_t *>(udata_getMemory(t->memory));
CollationDataReader::read(nullptr, inBytes, udata_getLength(t->memory), *t, errorCode);
if(U_FAILURE(errorCode)) { return; }
ucln_i18n_registerCleanup(UCLN_I18N_COLLATION_ROOT, uprv_collation_root_cleanup);
CollationCacheEntry *entry = new CollationCacheEntry(Locale::getRoot(), t.getAlias());
if(entry != nullptr) {
t.orphan(); // The rootSingleton took ownership of the tailoring.
entry->addRef();
rootSingleton = entry;
}
}
const CollationCacheEntry *
CollationRoot::getRootCacheEntry(UErrorCode &errorCode) {
umtx_initOnce(initOnce, CollationRoot::load, static_cast<const char*>(nullptr), errorCode);
if(U_FAILURE(errorCode)) { return nullptr; }
return rootSingleton;
}
const CollationTailoring *
CollationRoot::getRoot(UErrorCode &errorCode) {
umtx_initOnce(initOnce, CollationRoot::load, static_cast<const char*>(nullptr), errorCode);
if(U_FAILURE(errorCode)) { return nullptr; }
return rootSingleton->tailoring;
}
const CollationData *
CollationRoot::getData(UErrorCode &errorCode) {
const CollationTailoring *root = getRoot(errorCode);
if(U_FAILURE(errorCode)) { return nullptr; }
return root->data;
}
const CollationSettings *
CollationRoot::getSettings(UErrorCode &errorCode) {
const CollationTailoring *root = getRoot(errorCode);
if(U_FAILURE(errorCode)) { return nullptr; }
return root->settings;
}
void
CollationRoot::forceLoadFromFile(const char* ucadataPath, UErrorCode &errorCode) {
umtx_initOnce(initOnce, CollationRoot::load, ucadataPath, errorCode);
}
U_NAMESPACE_END
#endif // !UCONFIG_NO_COLLATION

48
deps/icu-small/source/i18n/collationroot.h vendored Normal file
View File

@ -0,0 +1,48 @@
// © 2016 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html
/*
*******************************************************************************
* Copyright (C) 2012-2014, International Business Machines
* Corporation and others. All Rights Reserved.
*******************************************************************************
* collationroot.h
*
* created on: 2012dec17
* created by: Markus W. Scherer
*/
#ifndef __COLLATIONROOT_H__
#define __COLLATIONROOT_H__
#include "unicode/utypes.h"
#include "unicode/udata.h"
#if !UCONFIG_NO_COLLATION
U_NAMESPACE_BEGIN
struct CollationCacheEntry;
struct CollationData;
struct CollationSettings;
struct CollationTailoring;
/**
* Collation root provider.
*/
class U_I18N_API CollationRoot { // purely static
public:
static const CollationCacheEntry *getRootCacheEntry(UErrorCode &errorCode);
static const CollationTailoring *getRoot(UErrorCode &errorCode);
static const CollationData *getData(UErrorCode &errorCode);
static const CollationSettings *getSettings(UErrorCode &errorCode);
static void U_EXPORT2 forceLoadFromFile(const char* ucadataPath, UErrorCode &errorCode);
private:
static void U_CALLCONV load(const char* ucadataPath, UErrorCode &errorCode);
static UDataMemory* loadFromFile(const char* ucadataPath, UErrorCode &errorCode);
};
U_NAMESPACE_END
#endif // !UCONFIG_NO_COLLATION
#endif // __COLLATIONROOT_H__

341
deps/icu-small/source/i18n/collationrootelements.cpp vendored Normal file
View File

@ -0,0 +1,341 @@
// © 2016 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html
/*
*******************************************************************************
* Copyright (C) 2013-2014, International Business Machines
* Corporation and others. All Rights Reserved.
*******************************************************************************
* collationrootelements.cpp
*
* created on: 2013mar05
* created by: Markus W. Scherer
*/
#include "unicode/utypes.h"
#if !UCONFIG_NO_COLLATION
#include "collation.h"
#include "collationrootelements.h"
#include "uassert.h"
U_NAMESPACE_BEGIN
int64_t
CollationRootElements::lastCEWithPrimaryBefore(uint32_t p) const {
if(p == 0) { return 0; }
U_ASSERT(p > elements[elements[IX_FIRST_PRIMARY_INDEX]]);
int32_t index = findP(p);
uint32_t q = elements[index];
uint32_t secTer;
if(p == (q & 0xffffff00)) {
// p == elements[index] is a root primary. Find the CE before it.
// We must not be in a primary range.
U_ASSERT((q & PRIMARY_STEP_MASK) == 0);
secTer = elements[index - 1];
if((secTer & SEC_TER_DELTA_FLAG) == 0) {
// Primary CE just before p.
p = secTer & 0xffffff00;
secTer = Collation::COMMON_SEC_AND_TER_CE;
} else {
// secTer = last secondary & tertiary for the previous primary
index -= 2;
for(;;) {
p = elements[index];
if((p & SEC_TER_DELTA_FLAG) == 0) {
p &= 0xffffff00;
break;
}
--index;
}
}
} else {
// p > elements[index] which is the previous primary.
// Find the last secondary & tertiary weights for it.
p = q & 0xffffff00;
secTer = Collation::COMMON_SEC_AND_TER_CE;
for(;;) {
q = elements[++index];
if((q & SEC_TER_DELTA_FLAG) == 0) {
// We must not be in a primary range.
U_ASSERT((q & PRIMARY_STEP_MASK) == 0);
break;
}
secTer = q;
}
}
return (static_cast<int64_t>(p) << 32) | (secTer & ~SEC_TER_DELTA_FLAG);
}
int64_t
CollationRootElements::firstCEWithPrimaryAtLeast(uint32_t p) const {
if(p == 0) { return 0; }
int32_t index = findP(p);
if(p != (elements[index] & 0xffffff00)) {
for(;;) {
p = elements[++index];
if((p & SEC_TER_DELTA_FLAG) == 0) {
// First primary after p. We must not be in a primary range.
U_ASSERT((p & PRIMARY_STEP_MASK) == 0);
break;
}
}
}
// The code above guarantees that p has at most 3 bytes: (p & 0xff) == 0.
return (static_cast<int64_t>(p) << 32) | Collation::COMMON_SEC_AND_TER_CE;
}
uint32_t
CollationRootElements::getPrimaryBefore(uint32_t p, UBool isCompressible) const {
int32_t index = findPrimary(p);
int32_t step;
uint32_t q = elements[index];
if(p == (q & 0xffffff00)) {
// Found p itself. Return the previous primary.
// See if p is at the end of a previous range.
step = static_cast<int32_t>(q) & PRIMARY_STEP_MASK;
if(step == 0) {
// p is not at the end of a range. Look for the previous primary.
do {
p = elements[--index];
} while((p & SEC_TER_DELTA_FLAG) != 0);
return p & 0xffffff00;
}
} else {
// p is in a range, and not at the start.
uint32_t nextElement = elements[index + 1];
U_ASSERT(isEndOfPrimaryRange(nextElement));
step = static_cast<int32_t>(nextElement) & PRIMARY_STEP_MASK;
}
// Return the previous range primary.
if((p & 0xffff) == 0) {
return Collation::decTwoBytePrimaryByOneStep(p, isCompressible, step);
} else {
return Collation::decThreeBytePrimaryByOneStep(p, isCompressible, step);
}
}
uint32_t
CollationRootElements::getSecondaryBefore(uint32_t p, uint32_t s) const {
int32_t index;
uint32_t previousSec, sec;
if(p == 0) {
index = static_cast<int32_t>(elements[IX_FIRST_SECONDARY_INDEX]);
// Gap at the beginning of the secondary CE range.
previousSec = 0;
sec = elements[index] >> 16;
} else {
index = findPrimary(p) + 1;
previousSec = Collation::BEFORE_WEIGHT16;
sec = getFirstSecTerForPrimary(index) >> 16;
}
U_ASSERT(s >= sec);
while(s > sec) {
previousSec = sec;
U_ASSERT((elements[index] & SEC_TER_DELTA_FLAG) != 0);
sec = elements[index++] >> 16;
}
U_ASSERT(sec == s);
return previousSec;
}
uint32_t
CollationRootElements::getTertiaryBefore(uint32_t p, uint32_t s, uint32_t t) const {
U_ASSERT((t & ~Collation::ONLY_TERTIARY_MASK) == 0);
int32_t index;
uint32_t previousTer, secTer;
if(p == 0) {
if(s == 0) {
index = static_cast<int32_t>(elements[IX_FIRST_TERTIARY_INDEX]);
// Gap at the beginning of the tertiary CE range.
previousTer = 0;
} else {
index = static_cast<int32_t>(elements[IX_FIRST_SECONDARY_INDEX]);
previousTer = Collation::BEFORE_WEIGHT16;
}
secTer = elements[index] & ~SEC_TER_DELTA_FLAG;
} else {
index = findPrimary(p) + 1;
previousTer = Collation::BEFORE_WEIGHT16;
secTer = getFirstSecTerForPrimary(index);
}
uint32_t st = (s << 16) | t;
while(st > secTer) {
if((secTer >> 16) == s) { previousTer = secTer; }
U_ASSERT((elements[index] & SEC_TER_DELTA_FLAG) != 0);
secTer = elements[index++] & ~SEC_TER_DELTA_FLAG;
}
U_ASSERT(secTer == st);
return previousTer & 0xffff;
}
uint32_t
CollationRootElements::getPrimaryAfter(uint32_t p, int32_t index, UBool isCompressible) const {
U_ASSERT(p == (elements[index] & 0xffffff00) || isEndOfPrimaryRange(elements[index + 1]));
uint32_t q = elements[++index];
int32_t step;
if ((q & SEC_TER_DELTA_FLAG) == 0 && (step = static_cast<int32_t>(q) & PRIMARY_STEP_MASK) != 0) {
// Return the next primary in this range.
if((p & 0xffff) == 0) {
return Collation::incTwoBytePrimaryByOffset(p, isCompressible, step);
} else {
return Collation::incThreeBytePrimaryByOffset(p, isCompressible, step);
}
} else {
// Return the next primary in the list.
while((q & SEC_TER_DELTA_FLAG) != 0) {
q = elements[++index];
}
U_ASSERT((q & PRIMARY_STEP_MASK) == 0);
return q;
}
}
uint32_t
CollationRootElements::getSecondaryAfter(int32_t index, uint32_t s) const {
uint32_t secTer;
uint32_t secLimit;
if(index == 0) {
// primary = 0
U_ASSERT(s != 0);
index = static_cast<int32_t>(elements[IX_FIRST_SECONDARY_INDEX]);
secTer = elements[index];
// Gap at the end of the secondary CE range.
secLimit = 0x10000;
} else {
U_ASSERT(index >= (int32_t)elements[IX_FIRST_PRIMARY_INDEX]);
secTer = getFirstSecTerForPrimary(index + 1);
// If this is an explicit sec/ter unit, then it will be read once more.
// Gap for secondaries of primary CEs.
secLimit = getSecondaryBoundary();
}
for(;;) {
uint32_t sec = secTer >> 16;
if(sec > s) { return sec; }
secTer = elements[++index];
if((secTer & SEC_TER_DELTA_FLAG) == 0) { return secLimit; }
}
}
uint32_t
CollationRootElements::getTertiaryAfter(int32_t index, uint32_t s, uint32_t t) const {
uint32_t secTer;
uint32_t terLimit;
if(index == 0) {
// primary = 0
if(s == 0) {
U_ASSERT(t != 0);
index = static_cast<int32_t>(elements[IX_FIRST_TERTIARY_INDEX]);
// Gap at the end of the tertiary CE range.
terLimit = 0x4000;
} else {
index = static_cast<int32_t>(elements[IX_FIRST_SECONDARY_INDEX]);
// Gap for tertiaries of primary/secondary CEs.
terLimit = getTertiaryBoundary();
}
secTer = elements[index] & ~SEC_TER_DELTA_FLAG;
} else {
U_ASSERT(index >= (int32_t)elements[IX_FIRST_PRIMARY_INDEX]);
secTer = getFirstSecTerForPrimary(index + 1);
// If this is an explicit sec/ter unit, then it will be read once more.
terLimit = getTertiaryBoundary();
}
uint32_t st = (s << 16) | t;
for(;;) {
if(secTer > st) {
U_ASSERT((secTer >> 16) == s);
return secTer & 0xffff;
}
secTer = elements[++index];
// No tertiary greater than t for this primary+secondary.
if((secTer & SEC_TER_DELTA_FLAG) == 0 || (secTer >> 16) > s) { return terLimit; }
secTer &= ~SEC_TER_DELTA_FLAG;
}
}
uint32_t
CollationRootElements::getFirstSecTerForPrimary(int32_t index) const {
uint32_t secTer = elements[index];
if((secTer & SEC_TER_DELTA_FLAG) == 0) {
// No sec/ter delta.
return Collation::COMMON_SEC_AND_TER_CE;
}
secTer &= ~SEC_TER_DELTA_FLAG;
if(secTer > Collation::COMMON_SEC_AND_TER_CE) {
// Implied sec/ter.
return Collation::COMMON_SEC_AND_TER_CE;
}
// Explicit sec/ter below common/common.
return secTer;
}
int32_t
CollationRootElements::findPrimary(uint32_t p) const {
// Requirement: p must occur as a root primary.
U_ASSERT((p & 0xff) == 0); // at most a 3-byte primary
int32_t index = findP(p);
// If p is in a range, then we just assume that p is an actual primary in this range.
// (Too cumbersome/expensive to check.)
// Otherwise, it must be an exact match.
U_ASSERT(isEndOfPrimaryRange(elements[index + 1]) || p == (elements[index] & 0xffffff00));
return index;
}
int32_t
CollationRootElements::findP(uint32_t p) const {
// p need not occur as a root primary.
// For example, it might be a reordering group boundary.
U_ASSERT((p >> 24) != Collation::UNASSIGNED_IMPLICIT_BYTE);
// modified binary search
int32_t start = static_cast<int32_t>(elements[IX_FIRST_PRIMARY_INDEX]);
U_ASSERT(p >= elements[start]);
int32_t limit = length - 1;
U_ASSERT(elements[limit] >= PRIMARY_SENTINEL);
U_ASSERT(p < elements[limit]);
while((start + 1) < limit) {
// Invariant: elements[start] and elements[limit] are primaries,
// and elements[start]<=p<=elements[limit].
int32_t i = (start + limit) / 2;
uint32_t q = elements[i];
if((q & SEC_TER_DELTA_FLAG) != 0) {
// Find the next primary.
int32_t j = i + 1;
for(;;) {
if(j == limit) { break; }
q = elements[j];
if((q & SEC_TER_DELTA_FLAG) == 0) {
i = j;
break;
}
++j;
}
if((q & SEC_TER_DELTA_FLAG) != 0) {
// Find the preceding primary.
j = i - 1;
for(;;) {
if(j == start) { break; }
q = elements[j];
if((q & SEC_TER_DELTA_FLAG) == 0) {
i = j;
break;
}
--j;
}
if((q & SEC_TER_DELTA_FLAG) != 0) {
// No primary between start and limit.
break;
}
}
}
if(p < (q & 0xffffff00)) { // Reset the "step" bits of a range end primary.
limit = i;
} else {
start = i;
}
}
return start;
}
U_NAMESPACE_END
#endif // !UCONFIG_NO_COLLATION

276
deps/icu-small/source/i18n/collationrootelements.h vendored Normal file
View File

@ -0,0 +1,276 @@
// © 2016 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html
/*
*******************************************************************************
* Copyright (C) 2013-2014, International Business Machines
* Corporation and others. All Rights Reserved.
*******************************************************************************
* collationrootelements.h
*
* created on: 2013mar01
* created by: Markus W. Scherer
*/
#ifndef __COLLATIONROOTELEMENTS_H__
#define __COLLATIONROOTELEMENTS_H__
#include "unicode/utypes.h"
#if !UCONFIG_NO_COLLATION
#include "unicode/uobject.h"
#include "collation.h"
U_NAMESPACE_BEGIN
/**
* Container and access methods for collation elements and weights
* that occur in the root collator.
* Needed for finding boundaries for building a tailoring.
*
* This class takes and returns 16-bit secondary and tertiary weights.
*/
class U_I18N_API CollationRootElements : public UMemory {
public:
CollationRootElements(const uint32_t *rootElements, int32_t rootElementsLength)
: elements(rootElements), length(rootElementsLength) {}
/**
* Higher than any root primary.
*/
static const uint32_t PRIMARY_SENTINEL = 0xffffff00;
/**
* Flag in a root element, set if the element contains secondary & tertiary weights,
* rather than a primary.
*/
static const uint32_t SEC_TER_DELTA_FLAG = 0x80;
/**
* Mask for getting the primary range step value from a primary-range-end element.
*/
static const uint8_t PRIMARY_STEP_MASK = 0x7f;
enum {
/**
* Index of the first CE with a non-zero tertiary weight.
* Same as the start of the compact root elements table.
*/
IX_FIRST_TERTIARY_INDEX,
/**
* Index of the first CE with a non-zero secondary weight.
*/
IX_FIRST_SECONDARY_INDEX,
/**
* Index of the first CE with a non-zero primary weight.
*/
IX_FIRST_PRIMARY_INDEX,
/**
* Must match Collation::COMMON_SEC_AND_TER_CE.
*/
IX_COMMON_SEC_AND_TER_CE,
/**
* Secondary & tertiary boundaries.
* Bits 31..24: [fixed last secondary common byte 45]
* Bits 23..16: [fixed first ignorable secondary byte 80]
* Bits 15.. 8: reserved, 0
* Bits 7.. 0: [fixed first ignorable tertiary byte 3C]
*/
IX_SEC_TER_BOUNDARIES,
/**
* The current number of indexes.
* Currently the same as elements[IX_FIRST_TERTIARY_INDEX].
*/
IX_COUNT
};
/**
* Returns the boundary between tertiary weights of primary/secondary CEs
* and those of tertiary CEs.
* This is the upper limit for tertiaries of primary/secondary CEs.
* This minus one is the lower limit for tertiaries of tertiary CEs.
*/
uint32_t getTertiaryBoundary() const {
return (elements[IX_SEC_TER_BOUNDARIES] << 8) & 0xff00;
}
/**
* Returns the first assigned tertiary CE.
*/
uint32_t getFirstTertiaryCE() const {
return elements[elements[IX_FIRST_TERTIARY_INDEX]] & ~SEC_TER_DELTA_FLAG;
}
/**
* Returns the last assigned tertiary CE.
*/
uint32_t getLastTertiaryCE() const {
return elements[elements[IX_FIRST_SECONDARY_INDEX] - 1] & ~SEC_TER_DELTA_FLAG;
}
/**
* Returns the last common secondary weight.
* This is the lower limit for secondaries of primary CEs.
*/
uint32_t getLastCommonSecondary() const {
return (elements[IX_SEC_TER_BOUNDARIES] >> 16) & 0xff00;
}
/**
* Returns the boundary between secondary weights of primary CEs
* and those of secondary CEs.
* This is the upper limit for secondaries of primary CEs.
* This minus one is the lower limit for secondaries of secondary CEs.
*/
uint32_t getSecondaryBoundary() const {
return (elements[IX_SEC_TER_BOUNDARIES] >> 8) & 0xff00;
}
/**
* Returns the first assigned secondary CE.
*/
uint32_t getFirstSecondaryCE() const {
return elements[elements[IX_FIRST_SECONDARY_INDEX]] & ~SEC_TER_DELTA_FLAG;
}
/**
* Returns the last assigned secondary CE.
*/
uint32_t getLastSecondaryCE() const {
return elements[elements[IX_FIRST_PRIMARY_INDEX] - 1] & ~SEC_TER_DELTA_FLAG;
}
/**
* Returns the first assigned primary weight.
*/
uint32_t getFirstPrimary() const {
return elements[elements[IX_FIRST_PRIMARY_INDEX]]; // step=0: cannot be a range end
}
/**
* Returns the first assigned primary CE.
*/
int64_t getFirstPrimaryCE() const {
return Collation::makeCE(getFirstPrimary());
}
/**
* Returns the last root CE with a primary weight before p.
* Intended only for reordering group boundaries.
*/
int64_t lastCEWithPrimaryBefore(uint32_t p) const;
/**
* Returns the first root CE with a primary weight of at least p.
* Intended only for reordering group boundaries.
*/
int64_t firstCEWithPrimaryAtLeast(uint32_t p) const;
/**
* Returns the primary weight before p.
* p must be greater than the first root primary.
*/
uint32_t getPrimaryBefore(uint32_t p, UBool isCompressible) const;
/** Returns the secondary weight before [p, s]. */
uint32_t getSecondaryBefore(uint32_t p, uint32_t s) const;
/** Returns the tertiary weight before [p, s, t]. */
uint32_t getTertiaryBefore(uint32_t p, uint32_t s, uint32_t t) const;
/**
* Finds the index of the input primary.
* p must occur as a root primary, and must not be 0.
*/
int32_t findPrimary(uint32_t p) const;
/**
* Returns the primary weight after p where index=findPrimary(p).
* p must be at least the first root primary.
*/
uint32_t getPrimaryAfter(uint32_t p, int32_t index, UBool isCompressible) const;
/**
* Returns the secondary weight after [p, s] where index=findPrimary(p)
* except use index=0 for p=0.
*
* Must return a weight for every root [p, s] as well as for every weight
* returned by getSecondaryBefore(). If p!=0 then s can be BEFORE_WEIGHT16.
*
* Exception: [0, 0] is handled by the CollationBuilder:
* Both its lower and upper boundaries are special.
*/
uint32_t getSecondaryAfter(int32_t index, uint32_t s) const;
/**
* Returns the tertiary weight after [p, s, t] where index=findPrimary(p)
* except use index=0 for p=0.
*
* Must return a weight for every root [p, s, t] as well as for every weight
* returned by getTertiaryBefore(). If s!=0 then t can be BEFORE_WEIGHT16.
*
* Exception: [0, 0, 0] is handled by the CollationBuilder:
* Both its lower and upper boundaries are special.
*/
uint32_t getTertiaryAfter(int32_t index, uint32_t s, uint32_t t) const;
private:
/**
* Returns the first secondary & tertiary weights for p where index=findPrimary(p)+1.
*/
uint32_t getFirstSecTerForPrimary(int32_t index) const;
/**
* Finds the largest index i where elements[i]<=p.
* Requires first primary<=p<0xffffff00 (PRIMARY_SENTINEL).
* Does not require that p is a root collator primary.
*/
int32_t findP(uint32_t p) const;
static inline UBool isEndOfPrimaryRange(uint32_t q) {
return (q & SEC_TER_DELTA_FLAG) == 0 && (q & PRIMARY_STEP_MASK) != 0;
}
/**
* Data structure:
*
* The first few entries are indexes, up to elements[IX_FIRST_TERTIARY_INDEX].
* See the comments on the IX_ constants.
*
* All other elements are a compact form of the root collator CEs
* in mostly collation order.
*
* A sequence of one or more root CEs with the same primary weight is stored as
* one element with the primary weight, with the SEC_TER_DELTA_FLAG flag not set,
* followed by elements with only the secondary/tertiary weights,
* each with that flag set.
* If the lowest secondary/tertiary combination is Collation::COMMON_SEC_AND_TER_CE,
* then the element for that combination is omitted.
*
* Note: If the first actual secondary/tertiary combination is higher than
* Collation::COMMON_SEC_AND_TER_CE (which is unusual),
* the runtime code will assume anyway that Collation::COMMON_SEC_AND_TER_CE is present.
*
* A range of only-primary CEs with a consistent "step" increment
* from each primary to the next may be stored as a range.
* Only the first and last primary are stored, and the last has the step
* value in the low bits (PRIMARY_STEP_MASK).
*
* An range-end element may also either start a new range or be followed by
* elements with secondary/tertiary deltas.
*
* A primary element that is not a range end has zero step bits.
*
* There is no element for the completely ignorable CE (all weights 0).
*
* Before elements[IX_FIRST_PRIMARY_INDEX], all elements are secondary/tertiary deltas,
* for all of the ignorable root CEs.
*
* There are no elements for unassigned-implicit primary CEs.
* All primaries stored here are at most 3 bytes long.
*/
const uint32_t *elements;
int32_t length;
};
U_NAMESPACE_END
#endif // !UCONFIG_NO_COLLATION
#endif // __COLLATIONROOTELEMENTS_H__

883
deps/icu-small/source/i18n/collationruleparser.cpp vendored Normal file
View File

@ -0,0 +1,883 @@
// © 2016 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html
/*
*******************************************************************************
* Copyright (C) 2013-2015, International Business Machines
* Corporation and others. All Rights Reserved.
*******************************************************************************
* collationruleparser.cpp
*
* (replaced the former ucol_tok.cpp)
*
* created on: 2013apr10
* created by: Markus W. Scherer
*/
#include "unicode/utypes.h"
#if !UCONFIG_NO_COLLATION
#include "unicode/normalizer2.h"
#include "unicode/parseerr.h"
#include "unicode/uchar.h"
#include "unicode/ucol.h"
#include "unicode/uloc.h"
#include "unicode/unistr.h"
#include "unicode/utf16.h"
#include "charstr.h"
#include "cmemory.h"
#include "collation.h"
#include "collationdata.h"
#include "collationruleparser.h"
#include "collationsettings.h"
#include "collationtailoring.h"
#include "cstring.h"
#include "patternprops.h"
#include "uassert.h"
#include "ulocimp.h"
#include "uvectr32.h"
U_NAMESPACE_BEGIN
namespace {
const char16_t BEFORE[] = { 0x5b, 0x62, 0x65, 0x66, 0x6f, 0x72, 0x65, 0 }; // "[before"
const int32_t BEFORE_LENGTH = 7;
} // namespace
CollationRuleParser::Sink::~Sink() {}
void
CollationRuleParser::Sink::suppressContractions(const UnicodeSet &, const char *&, UErrorCode &) {}
void
CollationRuleParser::Sink::optimize(const UnicodeSet &, const char *&, UErrorCode &) {}
CollationRuleParser::Importer::~Importer() {}
CollationRuleParser::CollationRuleParser(const CollationData *base, UErrorCode &errorCode)
: nfd(*Normalizer2::getNFDInstance(errorCode)),
nfc(*Normalizer2::getNFCInstance(errorCode)),
rules(nullptr), baseData(base), settings(nullptr),
parseError(nullptr), errorReason(nullptr),
sink(nullptr), importer(nullptr),
ruleIndex(0) {
}
CollationRuleParser::~CollationRuleParser() {
}
void
CollationRuleParser::parse(const UnicodeString &ruleString,
CollationSettings &outSettings,
UParseError *outParseError,
UErrorCode &errorCode) {
if(U_FAILURE(errorCode)) { return; }
settings = &outSettings;
parseError = outParseError;
if(parseError != nullptr) {
parseError->line = 0;
parseError->offset = -1;
parseError->preContext[0] = 0;
parseError->postContext[0] = 0;
}
errorReason = nullptr;
parse(ruleString, errorCode);
}
void
CollationRuleParser::parse(const UnicodeString &ruleString, UErrorCode &errorCode) {
if(U_FAILURE(errorCode)) { return; }
rules = &ruleString;
ruleIndex = 0;
while(ruleIndex < rules->length()) {
char16_t c = rules->charAt(ruleIndex);
if(PatternProps::isWhiteSpace(c)) {
++ruleIndex;
continue;
}
switch(c) {
case 0x26: // '&'
parseRuleChain(errorCode);
break;
case 0x5b: // '['
parseSetting(errorCode);
break;
case 0x23: // '#' starts a comment, until the end of the line
ruleIndex = skipComment(ruleIndex + 1);
break;
case 0x40: // '@' is equivalent to [backwards 2]
settings->setFlag(CollationSettings::BACKWARD_SECONDARY,
UCOL_ON, 0, errorCode);
++ruleIndex;
break;
case 0x21: // '!' used to turn on Thai/Lao character reversal
// Accept but ignore. The root collator has contractions
// that are equivalent to the character reversal, where appropriate.
++ruleIndex;
break;
default:
setParseError("expected a reset or setting or comment", errorCode);
break;
}
if(U_FAILURE(errorCode)) { return; }
}
}
void
CollationRuleParser::parseRuleChain(UErrorCode &errorCode) {
int32_t resetStrength = parseResetAndPosition(errorCode);
UBool isFirstRelation = true;
for(;;) {
int32_t result = parseRelationOperator(errorCode);
if(U_FAILURE(errorCode)) { return; }
if(result < 0) {
if(ruleIndex < rules->length() && rules->charAt(ruleIndex) == 0x23) {
// '#' starts a comment, until the end of the line
ruleIndex = skipComment(ruleIndex + 1);
continue;
}
if(isFirstRelation) {
setParseError("reset not followed by a relation", errorCode);
}
return;
}
int32_t strength = result & STRENGTH_MASK;
if(resetStrength < UCOL_IDENTICAL) {
// reset-before rule chain
if(isFirstRelation) {
if(strength != resetStrength) {
setParseError("reset-before strength differs from its first relation", errorCode);
return;
}
} else {
if(strength < resetStrength) {
setParseError("reset-before strength followed by a stronger relation", errorCode);
return;
}
}
}
int32_t i = ruleIndex + (result >> OFFSET_SHIFT); // skip over the relation operator
if((result & STARRED_FLAG) == 0) {
parseRelationStrings(strength, i, errorCode);
} else {
parseStarredCharacters(strength, i, errorCode);
}
if(U_FAILURE(errorCode)) { return; }
isFirstRelation = false;
}
}
int32_t
CollationRuleParser::parseResetAndPosition(UErrorCode &errorCode) {
if(U_FAILURE(errorCode)) { return UCOL_DEFAULT; }
int32_t i = skipWhiteSpace(ruleIndex + 1);
int32_t j;
char16_t c;
int32_t resetStrength;
if(rules->compare(i, BEFORE_LENGTH, BEFORE, 0, BEFORE_LENGTH) == 0 &&
(j = i + BEFORE_LENGTH) < rules->length() &&
PatternProps::isWhiteSpace(rules->charAt(j)) &&
((j = skipWhiteSpace(j + 1)) + 1) < rules->length() &&
0x31 <= (c = rules->charAt(j)) && c <= 0x33 &&
rules->charAt(j + 1) == 0x5d) {
// &[before n] with n=1 or 2 or 3
resetStrength = UCOL_PRIMARY + (c - 0x31);
i = skipWhiteSpace(j + 2);
} else {
resetStrength = UCOL_IDENTICAL;
}
if(i >= rules->length()) {
setParseError("reset without position", errorCode);
return UCOL_DEFAULT;
}
UnicodeString str;
if(rules->charAt(i) == 0x5b) { // '['
i = parseSpecialPosition(i, str, errorCode);
} else {
i = parseTailoringString(i, str, errorCode);
}
sink->addReset(resetStrength, str, errorReason, errorCode);
if(U_FAILURE(errorCode)) { setErrorContext(); }
ruleIndex = i;
return resetStrength;
}
int32_t
CollationRuleParser::parseRelationOperator(UErrorCode &errorCode) {
if(U_FAILURE(errorCode)) { return UCOL_DEFAULT; }
ruleIndex = skipWhiteSpace(ruleIndex);
if(ruleIndex >= rules->length()) { return UCOL_DEFAULT; }
int32_t strength;
int32_t i = ruleIndex;
char16_t c = rules->charAt(i++);
switch(c) {
case 0x3c: // '<'
if(i < rules->length() && rules->charAt(i) == 0x3c) { // <<
++i;
if(i < rules->length() && rules->charAt(i) == 0x3c) { // <<<
++i;
if(i < rules->length() && rules->charAt(i) == 0x3c) { // <<<<
++i;
strength = UCOL_QUATERNARY;
} else {
strength = UCOL_TERTIARY;
}
} else {
strength = UCOL_SECONDARY;
}
} else {
strength = UCOL_PRIMARY;
}
if(i < rules->length() && rules->charAt(i) == 0x2a) { // '*'
++i;
strength |= STARRED_FLAG;
}
break;
case 0x3b: // ';' same as <<
strength = UCOL_SECONDARY;
break;
case 0x2c: // ',' same as <<<
strength = UCOL_TERTIARY;
break;
case 0x3d: // '='
strength = UCOL_IDENTICAL;
if(i < rules->length() && rules->charAt(i) == 0x2a) { // '*'
++i;
strength |= STARRED_FLAG;
}
break;
default:
return UCOL_DEFAULT;
}
return ((i - ruleIndex) << OFFSET_SHIFT) | strength;
}
void
CollationRuleParser::parseRelationStrings(int32_t strength, int32_t i, UErrorCode &errorCode) {
// Parse
// prefix | str / extension
// where prefix and extension are optional.
UnicodeString prefix, str, extension;
i = parseTailoringString(i, str, errorCode);
if(U_FAILURE(errorCode)) { return; }
char16_t next = (i < rules->length()) ? rules->charAt(i) : 0;
if(next == 0x7c) { // '|' separates the context prefix from the string.
prefix = str;
i = parseTailoringString(i + 1, str, errorCode);
if(U_FAILURE(errorCode)) { return; }
next = (i < rules->length()) ? rules->charAt(i) : 0;
}
if(next == 0x2f) { // '/' separates the string from the extension.
i = parseTailoringString(i + 1, extension, errorCode);
}
if(!prefix.isEmpty()) {
UChar32 prefix0 = prefix.char32At(0);
UChar32 c = str.char32At(0);
if(!nfc.hasBoundaryBefore(prefix0) || !nfc.hasBoundaryBefore(c)) {
setParseError("in 'prefix|str', prefix and str must each start with an NFC boundary",
errorCode);
return;
}
}
sink->addRelation(strength, prefix, str, extension, errorReason, errorCode);
if(U_FAILURE(errorCode)) { setErrorContext(); }
ruleIndex = i;
}
void
CollationRuleParser::parseStarredCharacters(int32_t strength, int32_t i, UErrorCode &errorCode) {
UnicodeString empty, raw;
i = parseString(skipWhiteSpace(i), raw, errorCode);
if(U_FAILURE(errorCode)) { return; }
if(raw.isEmpty()) {
setParseError("missing starred-relation string", errorCode);
return;
}
UChar32 prev = -1;
int32_t j = 0;
for(;;) {
while(j < raw.length()) {
UChar32 c = raw.char32At(j);
if(!nfd.isInert(c)) {
setParseError("starred-relation string is not all NFD-inert", errorCode);
return;
}
sink->addRelation(strength, empty, UnicodeString(c), empty, errorReason, errorCode);
if(U_FAILURE(errorCode)) {
setErrorContext();
return;
}
j += U16_LENGTH(c);
prev = c;
}
if(i >= rules->length() || rules->charAt(i) != 0x2d) { // '-'
break;
}
if(prev < 0) {
setParseError("range without start in starred-relation string", errorCode);
return;
}
i = parseString(i + 1, raw, errorCode);
if(U_FAILURE(errorCode)) { return; }
if(raw.isEmpty()) {
setParseError("range without end in starred-relation string", errorCode);
return;
}
UChar32 c = raw.char32At(0);
if(c < prev) {
setParseError("range start greater than end in starred-relation string", errorCode);
return;
}
// range prev-c
UnicodeString s;
while(++prev <= c) {
if(!nfd.isInert(prev)) {
setParseError("starred-relation string range is not all NFD-inert", errorCode);
return;
}
if(U_IS_SURROGATE(prev)) {
setParseError("starred-relation string range contains a surrogate", errorCode);
return;
}
if(0xfffd <= prev && prev <= 0xffff) {
setParseError("starred-relation string range contains U+FFFD, U+FFFE or U+FFFF", errorCode);
return;
}
s.setTo(prev);
sink->addRelation(strength, empty, s, empty, errorReason, errorCode);
if(U_FAILURE(errorCode)) {
setErrorContext();
return;
}
}
prev = -1;
j = U16_LENGTH(c);
}
ruleIndex = skipWhiteSpace(i);
}
int32_t
CollationRuleParser::parseTailoringString(int32_t i, UnicodeString &raw, UErrorCode &errorCode) {
i = parseString(skipWhiteSpace(i), raw, errorCode);
if(U_SUCCESS(errorCode) && raw.isEmpty()) {
setParseError("missing relation string", errorCode);
}
return skipWhiteSpace(i);
}
int32_t
CollationRuleParser::parseString(int32_t i, UnicodeString &raw, UErrorCode &errorCode) {
if(U_FAILURE(errorCode)) { return i; }
raw.remove();
while(i < rules->length()) {
UChar32 c = rules->charAt(i++);
if(isSyntaxChar(c)) {
if(c == 0x27) { // apostrophe
if(i < rules->length() && rules->charAt(i) == 0x27) {
// Double apostrophe, encodes a single one.
raw.append(static_cast<char16_t>(0x27));
++i;
continue;
}
// Quote literal text until the next single apostrophe.
for(;;) {
if(i == rules->length()) {
setParseError("quoted literal text missing terminating apostrophe", errorCode);
return i;
}
c = rules->charAt(i++);
if(c == 0x27) {
if(i < rules->length() && rules->charAt(i) == 0x27) {
// Double apostrophe inside quoted literal text,
// still encodes a single apostrophe.
++i;
} else {
break;
}
}
raw.append(static_cast<char16_t>(c));
}
} else if(c == 0x5c) { // backslash
if(i == rules->length()) {
setParseError("backslash escape at the end of the rule string", errorCode);
return i;
}
c = rules->char32At(i);
raw.append(c);
i += U16_LENGTH(c);
} else {
// Any other syntax character terminates a string.
--i;
break;
}
} else if(PatternProps::isWhiteSpace(c)) {
// Unquoted white space terminates a string.
--i;
break;
} else {
raw.append(static_cast<char16_t>(c));
}
}
for(int32_t j = 0; j < raw.length();) {
UChar32 c = raw.char32At(j);
if(U_IS_SURROGATE(c)) {
setParseError("string contains an unpaired surrogate", errorCode);
return i;
}
if(0xfffd <= c && c <= 0xffff) {
setParseError("string contains U+FFFD, U+FFFE or U+FFFF", errorCode);
return i;
}
j += U16_LENGTH(c);
}
return i;
}
namespace {
const char* const positions[] = {
"first tertiary ignorable",
"last tertiary ignorable",
"first secondary ignorable",
"last secondary ignorable",
"first primary ignorable",
"last primary ignorable",
"first variable",
"last variable",
"first regular",
"last regular",
"first implicit",
"last implicit",
"first trailing",
"last trailing"
};
} // namespace
int32_t
CollationRuleParser::parseSpecialPosition(int32_t i, UnicodeString &str, UErrorCode &errorCode) {
if(U_FAILURE(errorCode)) { return 0; }
UnicodeString raw;
int32_t j = readWords(i + 1, raw);
if(j > i && rules->charAt(j) == 0x5d && !raw.isEmpty()) { // words end with ]
++j;
for(int32_t pos = 0; pos < UPRV_LENGTHOF(positions); ++pos) {
if(raw == UnicodeString(positions[pos], -1, US_INV)) {
str.setTo(POS_LEAD).append(static_cast<char16_t>(POS_BASE + pos));
return j;
}
}
if(raw == UNICODE_STRING_SIMPLE("top")) {
str.setTo(POS_LEAD).append(static_cast<char16_t>(POS_BASE + LAST_REGULAR));
return j;
}
if(raw == UNICODE_STRING_SIMPLE("variable top")) {
str.setTo(POS_LEAD).append(static_cast<char16_t>(POS_BASE + LAST_VARIABLE));
return j;
}
}
setParseError("not a valid special reset position", errorCode);
return i;
}
void
CollationRuleParser::parseSetting(UErrorCode &errorCode) {
if(U_FAILURE(errorCode)) { return; }
UnicodeString raw;
int32_t i = ruleIndex + 1;
int32_t j = readWords(i, raw);
if(j <= i || raw.isEmpty()) {
setParseError("expected a setting/option at '['", errorCode);
}
if(rules->charAt(j) == 0x5d) { // words end with ]
++j;
if(raw.startsWith(UNICODE_STRING_SIMPLE("reorder")) &&
(raw.length() == 7 || raw.charAt(7) == 0x20)) {
parseReordering(raw, errorCode);
ruleIndex = j;
return;
}
if(raw == UNICODE_STRING_SIMPLE("backwards 2")) {
settings->setFlag(CollationSettings::BACKWARD_SECONDARY,
UCOL_ON, 0, errorCode);
ruleIndex = j;
return;
}
UnicodeString v;
int32_t valueIndex = raw.lastIndexOf(static_cast<char16_t>(0x20));
if(valueIndex >= 0) {
v.setTo(raw, valueIndex + 1);
raw.truncate(valueIndex);
}
if(raw == UNICODE_STRING_SIMPLE("strength") && v.length() == 1) {
int32_t value = UCOL_DEFAULT;
char16_t c = v.charAt(0);
if(0x31 <= c && c <= 0x34) { // 1..4
value = UCOL_PRIMARY + (c - 0x31);
} else if(c == 0x49) { // 'I'
value = UCOL_IDENTICAL;
}
if(value != UCOL_DEFAULT) {
settings->setStrength(value, 0, errorCode);
ruleIndex = j;
return;
}
} else if(raw == UNICODE_STRING_SIMPLE("alternate")) {
UColAttributeValue value = UCOL_DEFAULT;
if(v == UNICODE_STRING_SIMPLE("non-ignorable")) {
value = UCOL_NON_IGNORABLE;
} else if(v == UNICODE_STRING_SIMPLE("shifted")) {
value = UCOL_SHIFTED;
}
if(value != UCOL_DEFAULT) {
settings->setAlternateHandling(value, 0, errorCode);
ruleIndex = j;
return;
}
} else if(raw == UNICODE_STRING_SIMPLE("maxVariable")) {
int32_t value = UCOL_DEFAULT;
if(v == UNICODE_STRING_SIMPLE("space")) {
value = CollationSettings::MAX_VAR_SPACE;
} else if(v == UNICODE_STRING_SIMPLE("punct")) {
value = CollationSettings::MAX_VAR_PUNCT;
} else if(v == UNICODE_STRING_SIMPLE("symbol")) {
value = CollationSettings::MAX_VAR_SYMBOL;
} else if(v == UNICODE_STRING_SIMPLE("currency")) {
value = CollationSettings::MAX_VAR_CURRENCY;
}
if(value != UCOL_DEFAULT) {
settings->setMaxVariable(value, 0, errorCode);
settings->variableTop = baseData->getLastPrimaryForGroup(
UCOL_REORDER_CODE_FIRST + value);
U_ASSERT(settings->variableTop != 0);
ruleIndex = j;
return;
}
} else if(raw == UNICODE_STRING_SIMPLE("caseFirst")) {
UColAttributeValue value = UCOL_DEFAULT;
if(v == UNICODE_STRING_SIMPLE("off")) {
value = UCOL_OFF;
} else if(v == UNICODE_STRING_SIMPLE("lower")) {
value = UCOL_LOWER_FIRST;
} else if(v == UNICODE_STRING_SIMPLE("upper")) {
value = UCOL_UPPER_FIRST;
}
if(value != UCOL_DEFAULT) {
settings->setCaseFirst(value, 0, errorCode);
ruleIndex = j;
return;
}
} else if(raw == UNICODE_STRING_SIMPLE("caseLevel")) {
UColAttributeValue value = getOnOffValue(v);
if(value != UCOL_DEFAULT) {
settings->setFlag(CollationSettings::CASE_LEVEL, value, 0, errorCode);
ruleIndex = j;
return;
}
} else if(raw == UNICODE_STRING_SIMPLE("normalization")) {
UColAttributeValue value = getOnOffValue(v);
if(value != UCOL_DEFAULT) {
settings->setFlag(CollationSettings::CHECK_FCD, value, 0, errorCode);
ruleIndex = j;
return;
}
} else if(raw == UNICODE_STRING_SIMPLE("numericOrdering")) {
UColAttributeValue value = getOnOffValue(v);
if(value != UCOL_DEFAULT) {
settings->setFlag(CollationSettings::NUMERIC, value, 0, errorCode);
ruleIndex = j;
return;
}
} else if(raw == UNICODE_STRING_SIMPLE("hiraganaQ")) {
UColAttributeValue value = getOnOffValue(v);
if(value != UCOL_DEFAULT) {
if(value == UCOL_ON) {
setParseError("[hiraganaQ on] is not supported", errorCode);
}
ruleIndex = j;
return;
}
} else if(raw == UNICODE_STRING_SIMPLE("import")) {
CharString lang;
lang.appendInvariantChars(v, errorCode);
if(errorCode == U_MEMORY_ALLOCATION_ERROR) { return; }
// BCP 47 language tag -> ICU locale ID
int32_t parsedLength;
CharString localeID = ulocimp_forLanguageTag(lang.data(), -1, &parsedLength, errorCode);
if(U_FAILURE(errorCode) || parsedLength != lang.length()) {
errorCode = U_ZERO_ERROR;
setParseError("expected language tag in [import langTag]", errorCode);
return;
}
// localeID minus all keywords
CharString baseID = ulocimp_getBaseName(localeID.toStringPiece(), errorCode);
if (U_FAILURE(errorCode)) {
errorCode = U_ZERO_ERROR;
setParseError("expected language tag in [import langTag]", errorCode);
return;
}
if (baseID.isEmpty()) {
baseID.copyFrom("root", errorCode);
} else if (baseID[0] == '_') {
// CharString doesn't have any insert() method, only append().
constexpr char und[] = "und";
constexpr int32_t length = sizeof und - 1;
int32_t dummy;
char* tail = baseID.getAppendBuffer(length, length, dummy, errorCode);
char* head = baseID.data();
uprv_memmove(head + length, head, baseID.length());
uprv_memcpy(head, und, length);
baseID.append(tail, length, errorCode);
}
// @collation=type, or length=0 if not specified
CharString collationType = ulocimp_getKeywordValue(localeID.data(), "collation", errorCode);
if(U_FAILURE(errorCode)) {
errorCode = U_ZERO_ERROR;
setParseError("expected language tag in [import langTag]", errorCode);
return;
}
if(importer == nullptr) {
setParseError("[import langTag] is not supported", errorCode);
} else {
UnicodeString importedRules;
importer->getRules(baseID.data(),
!collationType.isEmpty() ? collationType.data() : "standard",
importedRules, errorReason, errorCode);
if(U_FAILURE(errorCode)) {
if(errorReason == nullptr) {
errorReason = "[import langTag] failed";
}
setErrorContext();
return;
}
const UnicodeString *outerRules = rules;
int32_t outerRuleIndex = ruleIndex;
parse(importedRules, errorCode);
if(U_FAILURE(errorCode)) {
if(parseError != nullptr) {
parseError->offset = outerRuleIndex;
}
}
rules = outerRules;
ruleIndex = j;
}
return;
}
} else if(rules->charAt(j) == 0x5b) { // words end with [
UnicodeSet set;
j = parseUnicodeSet(j, set, errorCode);
if(U_FAILURE(errorCode)) { return; }
if(raw == UNICODE_STRING_SIMPLE("optimize")) {
sink->optimize(set, errorReason, errorCode);
if(U_FAILURE(errorCode)) { setErrorContext(); }
ruleIndex = j;
return;
} else if(raw == UNICODE_STRING_SIMPLE("suppressContractions")) {
sink->suppressContractions(set, errorReason, errorCode);
if(U_FAILURE(errorCode)) { setErrorContext(); }
ruleIndex = j;
return;
}
}
setParseError("not a valid setting/option", errorCode);
}
void
CollationRuleParser::parseReordering(const UnicodeString &raw, UErrorCode &errorCode) {
if(U_FAILURE(errorCode)) { return; }
int32_t i = 7; // after "reorder"
if(i == raw.length()) {
// empty [reorder] with no codes
settings->resetReordering();
return;
}
// Parse the codes in [reorder aa bb cc].
UVector32 reorderCodes(errorCode);
if(U_FAILURE(errorCode)) { return; }
CharString word;
while(i < raw.length()) {
++i; // skip the word-separating space
int32_t limit = raw.indexOf(static_cast<char16_t>(0x20), i);
if(limit < 0) { limit = raw.length(); }
word.clear().appendInvariantChars(raw.tempSubStringBetween(i, limit), errorCode);
if(U_FAILURE(errorCode)) { return; }
int32_t code = getReorderCode(word.data());
if(code < 0) {
setParseError("unknown script or reorder code", errorCode);
return;
}
reorderCodes.addElement(code, errorCode);
if(U_FAILURE(errorCode)) { return; }
i = limit;
}
settings->setReordering(*baseData, reorderCodes.getBuffer(), reorderCodes.size(), errorCode);
}
static const char *const gSpecialReorderCodes[] = {
"space", "punct", "symbol", "currency", "digit"
};
int32_t
CollationRuleParser::getReorderCode(const char *word) {
for(int32_t i = 0; i < UPRV_LENGTHOF(gSpecialReorderCodes); ++i) {
if(uprv_stricmp(word, gSpecialReorderCodes[i]) == 0) {
return UCOL_REORDER_CODE_FIRST + i;
}
}
int32_t script = u_getPropertyValueEnum(UCHAR_SCRIPT, word);
if(script >= 0) {
return script;
}
if(uprv_stricmp(word, "others") == 0) {
return UCOL_REORDER_CODE_OTHERS; // same as Zzzz = USCRIPT_UNKNOWN
}
return -1;
}
UColAttributeValue
CollationRuleParser::getOnOffValue(const UnicodeString &s) {
if(s == UNICODE_STRING_SIMPLE("on")) {
return UCOL_ON;
} else if(s == UNICODE_STRING_SIMPLE("off")) {
return UCOL_OFF;
} else {
return UCOL_DEFAULT;
}
}
int32_t
CollationRuleParser::parseUnicodeSet(int32_t i, UnicodeSet &set, UErrorCode &errorCode) {
// Collect a UnicodeSet pattern between a balanced pair of [brackets].
int32_t level = 0;
int32_t j = i;
for(;;) {
if(j == rules->length()) {
setParseError("unbalanced UnicodeSet pattern brackets", errorCode);
return j;
}
char16_t c = rules->charAt(j++);
if(c == 0x5b) { // '['
++level;
} else if(c == 0x5d) { // ']'
if(--level == 0) { break; }
}
}
set.applyPattern(rules->tempSubStringBetween(i, j), errorCode);
if(U_FAILURE(errorCode)) {
errorCode = U_ZERO_ERROR;
setParseError("not a valid UnicodeSet pattern", errorCode);
return j;
}
j = skipWhiteSpace(j);
if(j == rules->length() || rules->charAt(j) != 0x5d) {
setParseError("missing option-terminating ']' after UnicodeSet pattern", errorCode);
return j;
}
return ++j;
}
int32_t
CollationRuleParser::readWords(int32_t i, UnicodeString &raw) const {
static const char16_t sp = 0x20;
raw.remove();
i = skipWhiteSpace(i);
for(;;) {
if(i >= rules->length()) { return 0; }
char16_t c = rules->charAt(i);
if(isSyntaxChar(c) && c != 0x2d && c != 0x5f) { // syntax except -_
if(raw.isEmpty()) { return i; }
if(raw.endsWith(&sp, 1)) { // remove trailing space
raw.truncate(raw.length() - 1);
}
return i;
}
if(PatternProps::isWhiteSpace(c)) {
raw.append(sp);
i = skipWhiteSpace(i + 1);
} else {
raw.append(c);
++i;
}
}
}
int32_t
CollationRuleParser::skipComment(int32_t i) const {
// skip to past the newline
while(i < rules->length()) {
char16_t c = rules->charAt(i++);
// LF or FF or CR or NEL or LS or PS
if(c == 0xa || c == 0xc || c == 0xd || c == 0x85 || c == 0x2028 || c == 0x2029) {
// Unicode Newline Guidelines: "A readline function should stop at NLF, LS, FF, or PS."
// NLF (new line function) = CR or LF or CR+LF or NEL.
// No need to collect all of CR+LF because a following LF will be ignored anyway.
break;
}
}
return i;
}
void
CollationRuleParser::setParseError(const char *reason, UErrorCode &errorCode) {
if(U_FAILURE(errorCode)) { return; }
// Error code consistent with the old parser (from ca. 2001),
// rather than U_PARSE_ERROR;
errorCode = U_INVALID_FORMAT_ERROR;
errorReason = reason;
if(parseError != nullptr) { setErrorContext(); }
}
void
CollationRuleParser::setErrorContext() {
if(parseError == nullptr) { return; }
// Note: This relies on the calling code maintaining the ruleIndex
// at a position that is useful for debugging.
// For example, at the beginning of a reset or relation etc.
parseError->offset = ruleIndex;
parseError->line = 0; // We are not counting line numbers.
// before ruleIndex
int32_t start = ruleIndex - (U_PARSE_CONTEXT_LEN - 1);
if(start < 0) {
start = 0;
} else if(start > 0 && U16_IS_TRAIL(rules->charAt(start))) {
++start;
}
int32_t length = ruleIndex - start;
rules->extract(start, length, parseError->preContext);
parseError->preContext[length] = 0;
// starting from ruleIndex
length = rules->length() - ruleIndex;
if(length >= U_PARSE_CONTEXT_LEN) {
length = U_PARSE_CONTEXT_LEN - 1;
if(U16_IS_LEAD(rules->charAt(ruleIndex + length - 1))) {
--length;
}
}
rules->extract(ruleIndex, length, parseError->postContext);
parseError->postContext[length] = 0;
}
UBool
CollationRuleParser::isSyntaxChar(UChar32 c) {
return 0x21 <= c && c <= 0x7e &&
(c <= 0x2f || (0x3a <= c && c <= 0x40) ||
(0x5b <= c && c <= 0x60) || (0x7b <= c));
}
int32_t
CollationRuleParser::skipWhiteSpace(int32_t i) const {
while(i < rules->length() && PatternProps::isWhiteSpace(rules->charAt(i))) {
++i;
}
return i;
}
U_NAMESPACE_END
#endif // !UCONFIG_NO_COLLATION

197
deps/icu-small/source/i18n/collationruleparser.h vendored Normal file
View File

@ -0,0 +1,197 @@
// © 2016 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html
/*
*******************************************************************************
* Copyright (C) 2013-2014, International Business Machines
* Corporation and others. All Rights Reserved.
*******************************************************************************
* collationruleparser.h
*
* created on: 2013apr10
* created by: Markus W. Scherer
*/
#ifndef __COLLATIONRULEPARSER_H__
#define __COLLATIONRULEPARSER_H__
#include "unicode/utypes.h"
#if !UCONFIG_NO_COLLATION
#include "unicode/ucol.h"
#include "unicode/uniset.h"
#include "unicode/unistr.h"
struct UParseError;
U_NAMESPACE_BEGIN
struct CollationData;
struct CollationTailoring;
class Locale;
class Normalizer2;
struct CollationSettings;
class U_I18N_API CollationRuleParser : public UMemory {
public:
/** Special reset positions. */
enum Position {
FIRST_TERTIARY_IGNORABLE,
LAST_TERTIARY_IGNORABLE,
FIRST_SECONDARY_IGNORABLE,
LAST_SECONDARY_IGNORABLE,
FIRST_PRIMARY_IGNORABLE,
LAST_PRIMARY_IGNORABLE,
FIRST_VARIABLE,
LAST_VARIABLE,
FIRST_REGULAR,
LAST_REGULAR,
FIRST_IMPLICIT,
LAST_IMPLICIT,
FIRST_TRAILING,
LAST_TRAILING
};
/**
* First character of contractions that encode special reset positions.
* U+FFFE cannot be tailored via rule syntax.
*
* The second contraction character is POS_BASE + Position.
*/
static const char16_t POS_LEAD = 0xfffe;
/**
* Base for the second character of contractions that encode special reset positions.
* Braille characters U+28xx are printable and normalization-inert.
* @see POS_LEAD
*/
static const char16_t POS_BASE = 0x2800;
class U_I18N_API Sink : public UObject {
public:
virtual ~Sink();
/**
* Adds a reset.
* strength=UCOL_IDENTICAL for &str.
* strength=UCOL_PRIMARY/UCOL_SECONDARY/UCOL_TERTIARY for &[before n]str where n=1/2/3.
*/
virtual void addReset(int32_t strength, const UnicodeString &str,
const char *&errorReason, UErrorCode &errorCode) = 0;
/**
* Adds a relation with strength and prefix | str / extension.
*/
virtual void addRelation(int32_t strength, const UnicodeString &prefix,
const UnicodeString &str, const UnicodeString &extension,
const char *&errorReason, UErrorCode &errorCode) = 0;
virtual void suppressContractions(const UnicodeSet &set, const char *&errorReason,
UErrorCode &errorCode);
virtual void optimize(const UnicodeSet &set, const char *&errorReason,
UErrorCode &errorCode);
};
class U_I18N_API Importer : public UObject {
public:
virtual ~Importer();
virtual void getRules(
const char *localeID, const char *collationType,
UnicodeString &rules,
const char *&errorReason, UErrorCode &errorCode) = 0;
};
/**
* Constructor.
* The Sink must be set before parsing.
* The Importer can be set, otherwise [import locale] syntax is not supported.
*/
CollationRuleParser(const CollationData *base, UErrorCode &errorCode);
~CollationRuleParser();
/**
* Sets the pointer to a Sink object.
* The pointer is aliased: Pointer copy without cloning or taking ownership.
*/
void setSink(Sink *sinkAlias) {
sink = sinkAlias;
}
/**
* Sets the pointer to an Importer object.
* The pointer is aliased: Pointer copy without cloning or taking ownership.
*/
void setImporter(Importer *importerAlias) {
importer = importerAlias;
}
void parse(const UnicodeString &ruleString,
CollationSettings &outSettings,
UParseError *outParseError,
UErrorCode &errorCode);
const char *getErrorReason() const { return errorReason; }
/**
* Gets a script or reorder code from its string representation.
* @return the script/reorder code, or
* -1 if not recognized
*/
static int32_t getReorderCode(const char *word);
private:
/** UCOL_PRIMARY=0 .. UCOL_IDENTICAL=15 */
static const int32_t STRENGTH_MASK = 0xf;
static const int32_t STARRED_FLAG = 0x10;
static const int32_t OFFSET_SHIFT = 8;
void parse(const UnicodeString &ruleString, UErrorCode &errorCode);
void parseRuleChain(UErrorCode &errorCode);
int32_t parseResetAndPosition(UErrorCode &errorCode);
int32_t parseRelationOperator(UErrorCode &errorCode);
void parseRelationStrings(int32_t strength, int32_t i, UErrorCode &errorCode);
void parseStarredCharacters(int32_t strength, int32_t i, UErrorCode &errorCode);
int32_t parseTailoringString(int32_t i, UnicodeString &raw, UErrorCode &errorCode);
int32_t parseString(int32_t i, UnicodeString &raw, UErrorCode &errorCode);
/**
* Sets str to a contraction of U+FFFE and (U+2800 + Position).
* @return rule index after the special reset position
*/
int32_t parseSpecialPosition(int32_t i, UnicodeString &str, UErrorCode &errorCode);
void parseSetting(UErrorCode &errorCode);
void parseReordering(const UnicodeString &raw, UErrorCode &errorCode);
static UColAttributeValue getOnOffValue(const UnicodeString &s);
int32_t parseUnicodeSet(int32_t i, UnicodeSet &set, UErrorCode &errorCode);
int32_t readWords(int32_t i, UnicodeString &raw) const;
int32_t skipComment(int32_t i) const;
void setParseError(const char *reason, UErrorCode &errorCode);
void setErrorContext();
/**
* ASCII [:P:] and [:S:]:
* [\u0021-\u002F \u003A-\u0040 \u005B-\u0060 \u007B-\u007E]
*/
static UBool isSyntaxChar(UChar32 c);
int32_t skipWhiteSpace(int32_t i) const;
const Normalizer2 &nfd, &nfc;
const UnicodeString *rules;
const CollationData *const baseData;
CollationSettings *settings;
UParseError *parseError;
const char *errorReason;
Sink *sink;
Importer *importer;
int32_t ruleIndex;
};
U_NAMESPACE_END
#endif // !UCONFIG_NO_COLLATION
#endif // __COLLATIONRULEPARSER_H__

612
deps/icu-small/source/i18n/collationsets.cpp vendored Normal file
View File

@ -0,0 +1,612 @@
// © 2016 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html
/*
*******************************************************************************
* Copyright (C) 2013-2014, International Business Machines
* Corporation and others. All Rights Reserved.
*******************************************************************************
* collationsets.cpp
*
* created on: 2013feb09
* created by: Markus W. Scherer
*/
#include "unicode/utypes.h"
#if !UCONFIG_NO_COLLATION
#include "unicode/ucharstrie.h"
#include "unicode/uniset.h"
#include "unicode/unistr.h"
#include "unicode/ustringtrie.h"
#include "collation.h"
#include "collationdata.h"
#include "collationsets.h"
#include "normalizer2impl.h"
#include "uassert.h"
#include "utf16collationiterator.h"
#include "utrie2.h"
U_NAMESPACE_BEGIN
U_CDECL_BEGIN
static UBool U_CALLCONV
enumTailoredRange(const void *context, UChar32 start, UChar32 end, uint32_t ce32) {
if(ce32 == Collation::FALLBACK_CE32) {
return true; // fallback to base, not tailored
}
TailoredSet *ts = (TailoredSet *)context;
return ts->handleCE32(start, end, ce32);
}
U_CDECL_END
void
TailoredSet::forData(const CollationData *d, UErrorCode &ec) {
if(U_FAILURE(ec)) { return; }
errorCode = ec; // Preserve info & warning codes.
data = d;
baseData = d->base;
U_ASSERT(baseData != nullptr);
utrie2_enum(data->trie, nullptr, enumTailoredRange, this);
ec = errorCode;
}
UBool
TailoredSet::handleCE32(UChar32 start, UChar32 end, uint32_t ce32) {
U_ASSERT(ce32 != Collation::FALLBACK_CE32);
if(Collation::isSpecialCE32(ce32)) {
ce32 = data->getIndirectCE32(ce32);
if(ce32 == Collation::FALLBACK_CE32) {
return U_SUCCESS(errorCode);
}
}
do {
uint32_t baseCE32 = baseData->getFinalCE32(baseData->getCE32(start));
// Do not just continue if ce32 == baseCE32 because
// contractions and expansions in different data objects
// normally differ even if they have the same data offsets.
if(Collation::isSelfContainedCE32(ce32) && Collation::isSelfContainedCE32(baseCE32)) {
// fastpath
if(ce32 != baseCE32) {
tailored->add(start);
}
} else {
compare(start, ce32, baseCE32);
}
} while(++start <= end);
return U_SUCCESS(errorCode);
}
void
TailoredSet::compare(UChar32 c, uint32_t ce32, uint32_t baseCE32) {
if(Collation::isPrefixCE32(ce32)) {
const char16_t *p = data->contexts + Collation::indexFromCE32(ce32);
ce32 = data->getFinalCE32(CollationData::readCE32(p));
if(Collation::isPrefixCE32(baseCE32)) {
const char16_t *q = baseData->contexts + Collation::indexFromCE32(baseCE32);
baseCE32 = baseData->getFinalCE32(CollationData::readCE32(q));
comparePrefixes(c, p + 2, q + 2);
} else {
addPrefixes(data, c, p + 2);
}
} else if(Collation::isPrefixCE32(baseCE32)) {
const char16_t *q = baseData->contexts + Collation::indexFromCE32(baseCE32);
baseCE32 = baseData->getFinalCE32(CollationData::readCE32(q));
addPrefixes(baseData, c, q + 2);
}
if(Collation::isContractionCE32(ce32)) {
const char16_t *p = data->contexts + Collation::indexFromCE32(ce32);
if((ce32 & Collation::CONTRACT_SINGLE_CP_NO_MATCH) != 0) {
ce32 = Collation::NO_CE32;
} else {
ce32 = data->getFinalCE32(CollationData::readCE32(p));
}
if(Collation::isContractionCE32(baseCE32)) {
const char16_t *q = baseData->contexts + Collation::indexFromCE32(baseCE32);
if((baseCE32 & Collation::CONTRACT_SINGLE_CP_NO_MATCH) != 0) {
baseCE32 = Collation::NO_CE32;
} else {
baseCE32 = baseData->getFinalCE32(CollationData::readCE32(q));
}
compareContractions(c, p + 2, q + 2);
} else {
addContractions(c, p + 2);
}
} else if(Collation::isContractionCE32(baseCE32)) {
const char16_t *q = baseData->contexts + Collation::indexFromCE32(baseCE32);
baseCE32 = baseData->getFinalCE32(CollationData::readCE32(q));
addContractions(c, q + 2);
}
int32_t tag;
if(Collation::isSpecialCE32(ce32)) {
tag = Collation::tagFromCE32(ce32);
U_ASSERT(tag != Collation::PREFIX_TAG);
U_ASSERT(tag != Collation::CONTRACTION_TAG);
// Currently, the tailoring data builder does not write offset tags.
// They might be useful for saving space,
// but they would complicate the builder,
// and in tailorings we assume that performance of tailored characters is more important.
U_ASSERT(tag != Collation::OFFSET_TAG);
} else {
tag = -1;
}
int32_t baseTag;
if(Collation::isSpecialCE32(baseCE32)) {
baseTag = Collation::tagFromCE32(baseCE32);
U_ASSERT(baseTag != Collation::PREFIX_TAG);
U_ASSERT(baseTag != Collation::CONTRACTION_TAG);
} else {
baseTag = -1;
}
// Non-contextual mappings, expansions, etc.
if(baseTag == Collation::OFFSET_TAG) {
// We might be comparing a tailoring CE which is a copy of
// a base offset-tag CE, via the [optimize [set]] syntax
// or when a single-character mapping was copied for tailored contractions.
// Offset tags always result in long-primary CEs,
// with common secondary/tertiary weights.
if(!Collation::isLongPrimaryCE32(ce32)) {
add(c);
return;
}
int64_t dataCE = baseData->ces[Collation::indexFromCE32(baseCE32)];
uint32_t p = Collation::getThreeBytePrimaryForOffsetData(c, dataCE);
if(Collation::primaryFromLongPrimaryCE32(ce32) != p) {
add(c);
return;
}
}
if(tag != baseTag) {
add(c);
return;
}
if(tag == Collation::EXPANSION32_TAG) {
const uint32_t *ce32s = data->ce32s + Collation::indexFromCE32(ce32);
int32_t length = Collation::lengthFromCE32(ce32);
const uint32_t *baseCE32s = baseData->ce32s + Collation::indexFromCE32(baseCE32);
int32_t baseLength = Collation::lengthFromCE32(baseCE32);
if(length != baseLength) {
add(c);
return;
}
for(int32_t i = 0; i < length; ++i) {
if(ce32s[i] != baseCE32s[i]) {
add(c);
break;
}
}
} else if(tag == Collation::EXPANSION_TAG) {
const int64_t *ces = data->ces + Collation::indexFromCE32(ce32);
int32_t length = Collation::lengthFromCE32(ce32);
const int64_t *baseCEs = baseData->ces + Collation::indexFromCE32(baseCE32);
int32_t baseLength = Collation::lengthFromCE32(baseCE32);
if(length != baseLength) {
add(c);
return;
}
for(int32_t i = 0; i < length; ++i) {
if(ces[i] != baseCEs[i]) {
add(c);
break;
}
}
} else if(tag == Collation::HANGUL_TAG) {
char16_t jamos[3];
int32_t length = Hangul::decompose(c, jamos);
if(tailored->contains(jamos[0]) || tailored->contains(jamos[1]) ||
(length == 3 && tailored->contains(jamos[2]))) {
add(c);
}
} else if(ce32 != baseCE32) {
add(c);
}
}
void
TailoredSet::comparePrefixes(UChar32 c, const char16_t *p, const char16_t *q) {
// Parallel iteration over prefixes of both tables.
UCharsTrie::Iterator prefixes(p, 0, errorCode);
UCharsTrie::Iterator basePrefixes(q, 0, errorCode);
const UnicodeString *tp = nullptr; // Tailoring prefix.
const UnicodeString *bp = nullptr; // Base prefix.
// Use a string with a U+FFFF as the limit sentinel.
// U+FFFF is untailorable and will not occur in prefixes.
UnicodeString none(static_cast<char16_t>(0xffff));
for(;;) {
if(tp == nullptr) {
if(prefixes.next(errorCode)) {
tp = &prefixes.getString();
} else {
tp = &none;
}
}
if(bp == nullptr) {
if(basePrefixes.next(errorCode)) {
bp = &basePrefixes.getString();
} else {
bp = &none;
}
}
if(tp == &none && bp == &none) { break; }
int32_t cmp = tp->compare(*bp);
if(cmp < 0) {
// tp occurs in the tailoring but not in the base.
addPrefix(data, *tp, c, static_cast<uint32_t>(prefixes.getValue()));
tp = nullptr;
} else if(cmp > 0) {
// bp occurs in the base but not in the tailoring.
addPrefix(baseData, *bp, c, static_cast<uint32_t>(basePrefixes.getValue()));
bp = nullptr;
} else {
setPrefix(*tp);
compare(c, static_cast<uint32_t>(prefixes.getValue()), static_cast<uint32_t>(basePrefixes.getValue()));
resetPrefix();
tp = nullptr;
bp = nullptr;
}
}
}
void
TailoredSet::compareContractions(UChar32 c, const char16_t *p, const char16_t *q) {
// Parallel iteration over suffixes of both tables.
UCharsTrie::Iterator suffixes(p, 0, errorCode);
UCharsTrie::Iterator baseSuffixes(q, 0, errorCode);
const UnicodeString *ts = nullptr; // Tailoring suffix.
const UnicodeString *bs = nullptr; // Base suffix.
// Use a string with two U+FFFF as the limit sentinel.
// U+FFFF is untailorable and will not occur in contractions except maybe
// as a single suffix character for a root-collator boundary contraction.
UnicodeString none(static_cast<char16_t>(0xffff));
none.append(static_cast<char16_t>(0xffff));
for(;;) {
if(ts == nullptr) {
if(suffixes.next(errorCode)) {
ts = &suffixes.getString();
} else {
ts = &none;
}
}
if(bs == nullptr) {
if(baseSuffixes.next(errorCode)) {
bs = &baseSuffixes.getString();
} else {
bs = &none;
}
}
if(ts == &none && bs == &none) { break; }
int32_t cmp = ts->compare(*bs);
if(cmp < 0) {
// ts occurs in the tailoring but not in the base.
addSuffix(c, *ts);
ts = nullptr;
} else if(cmp > 0) {
// bs occurs in the base but not in the tailoring.
addSuffix(c, *bs);
bs = nullptr;
} else {
suffix = ts;
compare(c, static_cast<uint32_t>(suffixes.getValue()), static_cast<uint32_t>(baseSuffixes.getValue()));
suffix = nullptr;
ts = nullptr;
bs = nullptr;
}
}
}
void
TailoredSet::addPrefixes(const CollationData *d, UChar32 c, const char16_t *p) {
UCharsTrie::Iterator prefixes(p, 0, errorCode);
while(prefixes.next(errorCode)) {
addPrefix(d, prefixes.getString(), c, static_cast<uint32_t>(prefixes.getValue()));
}
}
void
TailoredSet::addPrefix(const CollationData *d, const UnicodeString &pfx, UChar32 c, uint32_t ce32) {
setPrefix(pfx);
ce32 = d->getFinalCE32(ce32);
if(Collation::isContractionCE32(ce32)) {
const char16_t *p = d->contexts + Collation::indexFromCE32(ce32);
addContractions(c, p + 2);
}
tailored->add(UnicodeString(unreversedPrefix).append(c));
resetPrefix();
}
void
TailoredSet::addContractions(UChar32 c, const char16_t *p) {
UCharsTrie::Iterator suffixes(p, 0, errorCode);
while(suffixes.next(errorCode)) {
addSuffix(c, suffixes.getString());
}
}
void
TailoredSet::addSuffix(UChar32 c, const UnicodeString &sfx) {
tailored->add(UnicodeString(unreversedPrefix).append(c).append(sfx));
}
void
TailoredSet::add(UChar32 c) {
if(unreversedPrefix.isEmpty() && suffix == nullptr) {
tailored->add(c);
} else {
UnicodeString s(unreversedPrefix);
s.append(c);
if(suffix != nullptr) {
s.append(*suffix);
}
tailored->add(s);
}
}
ContractionsAndExpansions::CESink::~CESink() {}
U_CDECL_BEGIN
static UBool U_CALLCONV
enumCnERange(const void *context, UChar32 start, UChar32 end, uint32_t ce32) {
ContractionsAndExpansions *cne = (ContractionsAndExpansions *)context;
if(cne->checkTailored == 0) {
// There is no tailoring.
// No need to collect nor check the tailored set.
} else if(cne->checkTailored < 0) {
// Collect the set of code points with mappings in the tailoring data.
if(ce32 == Collation::FALLBACK_CE32) {
return true; // fallback to base, not tailored
} else {
cne->tailored.add(start, end);
}
// checkTailored > 0: Exclude tailored ranges from the base data enumeration.
} else if(start == end) {
if(cne->tailored.contains(start)) {
return true;
}
} else if(cne->tailored.containsSome(start, end)) {
cne->ranges.set(start, end).removeAll(cne->tailored);
int32_t count = cne->ranges.getRangeCount();
for(int32_t i = 0; i < count; ++i) {
cne->handleCE32(cne->ranges.getRangeStart(i), cne->ranges.getRangeEnd(i), ce32);
}
return U_SUCCESS(cne->errorCode);
}
cne->handleCE32(start, end, ce32);
return U_SUCCESS(cne->errorCode);
}
U_CDECL_END
void
ContractionsAndExpansions::forData(const CollationData *d, UErrorCode &ec) {
if(U_FAILURE(ec)) { return; }
errorCode = ec; // Preserve info & warning codes.
// Add all from the data, can be tailoring or base.
if(d->base != nullptr) {
checkTailored = -1;
}
data = d;
utrie2_enum(data->trie, nullptr, enumCnERange, this);
if(d->base == nullptr || U_FAILURE(errorCode)) {
ec = errorCode;
return;
}
// Add all from the base data but only for un-tailored code points.
tailored.freeze();
checkTailored = 1;
data = d->base;
utrie2_enum(data->trie, nullptr, enumCnERange, this);
ec = errorCode;
}
void
ContractionsAndExpansions::forCodePoint(const CollationData *d, UChar32 c, UErrorCode &ec) {
if(U_FAILURE(ec)) { return; }
errorCode = ec; // Preserve info & warning codes.
uint32_t ce32 = d->getCE32(c);
if(ce32 == Collation::FALLBACK_CE32) {
d = d->base;
ce32 = d->getCE32(c);
}
data = d;
handleCE32(c, c, ce32);
ec = errorCode;
}
void
ContractionsAndExpansions::handleCE32(UChar32 start, UChar32 end, uint32_t ce32) {
for(;;) {
if((ce32 & 0xff) < Collation::SPECIAL_CE32_LOW_BYTE) {
// !isSpecialCE32()
if(sink != nullptr) {
sink->handleCE(Collation::ceFromSimpleCE32(ce32));
}
return;
}
switch(Collation::tagFromCE32(ce32)) {
case Collation::FALLBACK_TAG:
return;
case Collation::RESERVED_TAG_3:
case Collation::BUILDER_DATA_TAG:
case Collation::LEAD_SURROGATE_TAG:
if(U_SUCCESS(errorCode)) { errorCode = U_INTERNAL_PROGRAM_ERROR; }
return;
case Collation::LONG_PRIMARY_TAG:
if(sink != nullptr) {
sink->handleCE(Collation::ceFromLongPrimaryCE32(ce32));
}
return;
case Collation::LONG_SECONDARY_TAG:
if(sink != nullptr) {
sink->handleCE(Collation::ceFromLongSecondaryCE32(ce32));
}
return;
case Collation::LATIN_EXPANSION_TAG:
if(sink != nullptr) {
ces[0] = Collation::latinCE0FromCE32(ce32);
ces[1] = Collation::latinCE1FromCE32(ce32);
sink->handleExpansion(ces, 2);
}
// Optimization: If we have a prefix,
// then the relevant strings have been added already.
if(unreversedPrefix.isEmpty()) {
addExpansions(start, end);
}
return;
case Collation::EXPANSION32_TAG:
if(sink != nullptr) {
const uint32_t *ce32s = data->ce32s + Collation::indexFromCE32(ce32);
int32_t length = Collation::lengthFromCE32(ce32);
for(int32_t i = 0; i < length; ++i) {
ces[i] = Collation::ceFromCE32(*ce32s++);
}
sink->handleExpansion(ces, length);
}
// Optimization: If we have a prefix,
// then the relevant strings have been added already.
if(unreversedPrefix.isEmpty()) {
addExpansions(start, end);
}
return;
case Collation::EXPANSION_TAG:
if(sink != nullptr) {
int32_t length = Collation::lengthFromCE32(ce32);
sink->handleExpansion(data->ces + Collation::indexFromCE32(ce32), length);
}
// Optimization: If we have a prefix,
// then the relevant strings have been added already.
if(unreversedPrefix.isEmpty()) {
addExpansions(start, end);
}
return;
case Collation::PREFIX_TAG:
handlePrefixes(start, end, ce32);
return;
case Collation::CONTRACTION_TAG:
handleContractions(start, end, ce32);
return;
case Collation::DIGIT_TAG:
// Fetch the non-numeric-collation CE32 and continue.
ce32 = data->ce32s[Collation::indexFromCE32(ce32)];
break;
case Collation::U0000_TAG:
U_ASSERT(start == 0 && end == 0);
// Fetch the normal ce32 for U+0000 and continue.
ce32 = data->ce32s[0];
break;
case Collation::HANGUL_TAG:
if(sink != nullptr) {
// TODO: This should be optimized,
// especially if [start..end] is the complete Hangul range. (assert that)
UTF16CollationIterator iter(data, false, nullptr, nullptr, nullptr);
char16_t hangul[1] = { 0 };
for(UChar32 c = start; c <= end; ++c) {
hangul[0] = static_cast<char16_t>(c);
iter.setText(hangul, hangul + 1);
int32_t length = iter.fetchCEs(errorCode);
if(U_FAILURE(errorCode)) { return; }
// Ignore the terminating non-CE.
U_ASSERT(length >= 2 && iter.getCE(length - 1) == Collation::NO_CE);
sink->handleExpansion(iter.getCEs(), length - 1);
}
}
// Optimization: If we have a prefix,
// then the relevant strings have been added already.
if(unreversedPrefix.isEmpty()) {
addExpansions(start, end);
}
return;
case Collation::OFFSET_TAG:
// Currently no need to send offset CEs to the sink.
return;
case Collation::IMPLICIT_TAG:
// Currently no need to send implicit CEs to the sink.
return;
}
}
}
void
ContractionsAndExpansions::handlePrefixes(
UChar32 start, UChar32 end, uint32_t ce32) {
const char16_t *p = data->contexts + Collation::indexFromCE32(ce32);
ce32 = CollationData::readCE32(p); // Default if no prefix match.
handleCE32(start, end, ce32);
if(!addPrefixes) { return; }
UCharsTrie::Iterator prefixes(p + 2, 0, errorCode);
while(prefixes.next(errorCode)) {
setPrefix(prefixes.getString());
// Prefix/pre-context mappings are special kinds of contractions
// that always yield expansions.
addStrings(start, end, contractions);
addStrings(start, end, expansions);
handleCE32(start, end, static_cast<uint32_t>(prefixes.getValue()));
}
resetPrefix();
}
void
ContractionsAndExpansions::handleContractions(
UChar32 start, UChar32 end, uint32_t ce32) {
const char16_t *p = data->contexts + Collation::indexFromCE32(ce32);
if((ce32 & Collation::CONTRACT_SINGLE_CP_NO_MATCH) != 0) {
// No match on the single code point.
// We are underneath a prefix, and the default mapping is just
// a fallback to the mappings for a shorter prefix.
U_ASSERT(!unreversedPrefix.isEmpty());
} else {
ce32 = CollationData::readCE32(p); // Default if no suffix match.
U_ASSERT(!Collation::isContractionCE32(ce32));
handleCE32(start, end, ce32);
}
UCharsTrie::Iterator suffixes(p + 2, 0, errorCode);
while(suffixes.next(errorCode)) {
suffix = &suffixes.getString();
addStrings(start, end, contractions);
if(!unreversedPrefix.isEmpty()) {
addStrings(start, end, expansions);
}
handleCE32(start, end, static_cast<uint32_t>(suffixes.getValue()));
}
suffix = nullptr;
}
void
ContractionsAndExpansions::addExpansions(UChar32 start, UChar32 end) {
if(unreversedPrefix.isEmpty() && suffix == nullptr) {
if(expansions != nullptr) {
expansions->add(start, end);
}
} else {
addStrings(start, end, expansions);
}
}
void
ContractionsAndExpansions::addStrings(UChar32 start, UChar32 end, UnicodeSet *set) {
if(set == nullptr) { return; }
UnicodeString s(unreversedPrefix);
do {
s.append(start);
if(suffix != nullptr) {
s.append(*suffix);
}
set->add(s);
s.truncate(unreversedPrefix.length());
} while(++start <= end);
}
U_NAMESPACE_END
#endif // !UCONFIG_NO_COLLATION

144
deps/icu-small/source/i18n/collationsets.h vendored Normal file
View File

@ -0,0 +1,144 @@
// © 2016 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html
/*
*******************************************************************************
* Copyright (C) 2013-2014, International Business Machines
* Corporation and others. All Rights Reserved.
*******************************************************************************
* collationsets.h
*
* created on: 2013feb09
* created by: Markus W. Scherer
*/
#ifndef __COLLATIONSETS_H__
#define __COLLATIONSETS_H__
#include "unicode/utypes.h"
#if !UCONFIG_NO_COLLATION
#include "unicode/uniset.h"
#include "collation.h"
U_NAMESPACE_BEGIN
struct CollationData;
/**
* Finds the set of characters and strings that sort differently in the tailoring
* from the base data.
*
* Every mapping in the tailoring needs to be compared to the base,
* because some mappings are copied for optimization, and
* all contractions for a character are copied if any contractions for that character
* are added, modified or removed.
*
* It might be simpler to re-parse the rule string, but:
* - That would require duplicating some of the from-rules builder code.
* - That would make the runtime code depend on the builder.
* - That would only work if we have the rule string, and we allow users to
* omit the rule string from data files.
*/
class TailoredSet : public UMemory {
public:
TailoredSet(UnicodeSet *t)
: data(nullptr), baseData(nullptr),
tailored(t),
suffix(nullptr),
errorCode(U_ZERO_ERROR) {}
void forData(const CollationData *d, UErrorCode &errorCode);
/**
* @return U_SUCCESS(errorCode) in C++, void in Java
* @internal only public for access by callback
*/
UBool handleCE32(UChar32 start, UChar32 end, uint32_t ce32);
private:
void compare(UChar32 c, uint32_t ce32, uint32_t baseCE32);
void comparePrefixes(UChar32 c, const char16_t *p, const char16_t *q);
void compareContractions(UChar32 c, const char16_t *p, const char16_t *q);
void addPrefixes(const CollationData *d, UChar32 c, const char16_t *p);
void addPrefix(const CollationData *d, const UnicodeString &pfx, UChar32 c, uint32_t ce32);
void addContractions(UChar32 c, const char16_t *p);
void addSuffix(UChar32 c, const UnicodeString &sfx);
void add(UChar32 c);
/** Prefixes are reversed in the data structure. */
void setPrefix(const UnicodeString &pfx) {
unreversedPrefix = pfx;
unreversedPrefix.reverse();
}
void resetPrefix() {
unreversedPrefix.remove();
}
const CollationData *data;
const CollationData *baseData;
UnicodeSet *tailored;
UnicodeString unreversedPrefix;
const UnicodeString *suffix;
UErrorCode errorCode;
};
class ContractionsAndExpansions : public UMemory {
public:
class CESink : public UMemory {
public:
virtual ~CESink();
virtual void handleCE(int64_t ce) = 0;
virtual void handleExpansion(const int64_t ces[], int32_t length) = 0;
};
ContractionsAndExpansions(UnicodeSet *con, UnicodeSet *exp, CESink *s, UBool prefixes)
: data(nullptr),
contractions(con), expansions(exp),
sink(s),
addPrefixes(prefixes),
checkTailored(0),
suffix(nullptr),
errorCode(U_ZERO_ERROR) {}
void forData(const CollationData *d, UErrorCode &errorCode);
void forCodePoint(const CollationData *d, UChar32 c, UErrorCode &ec);
// all following: @internal, only public for access by callback
void handleCE32(UChar32 start, UChar32 end, uint32_t ce32);
void handlePrefixes(UChar32 start, UChar32 end, uint32_t ce32);
void handleContractions(UChar32 start, UChar32 end, uint32_t ce32);
void addExpansions(UChar32 start, UChar32 end);
void addStrings(UChar32 start, UChar32 end, UnicodeSet *set);
/** Prefixes are reversed in the data structure. */
void setPrefix(const UnicodeString &pfx) {
unreversedPrefix = pfx;
unreversedPrefix.reverse();
}
void resetPrefix() {
unreversedPrefix.remove();
}
const CollationData *data;
UnicodeSet *contractions;
UnicodeSet *expansions;
CESink *sink;
UBool addPrefixes;
int8_t checkTailored; // -1: collected tailored +1: exclude tailored
UnicodeSet tailored;
UnicodeSet ranges;
UnicodeString unreversedPrefix;
const UnicodeString *suffix;
int64_t ces[Collation::MAX_EXPANSION_LENGTH];
UErrorCode errorCode;
};
U_NAMESPACE_END
#endif // !UCONFIG_NO_COLLATION
#endif // __COLLATIONSETS_H__

377
deps/icu-small/source/i18n/collationsettings.cpp vendored Normal file
View File

@ -0,0 +1,377 @@
// © 2016 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html
/*
*******************************************************************************
* Copyright (C) 2013-2015, International Business Machines
* Corporation and others. All Rights Reserved.
*******************************************************************************
* collationsettings.cpp
*
* created on: 2013feb07
* created by: Markus W. Scherer
*/
#include "unicode/utypes.h"
#if !UCONFIG_NO_COLLATION
#include "unicode/ucol.h"
#include "cmemory.h"
#include "collation.h"
#include "collationdata.h"
#include "collationsettings.h"
#include "sharedobject.h"
#include "uassert.h"
#include "umutex.h"
#include "uvectr32.h"
U_NAMESPACE_BEGIN
CollationSettings::CollationSettings(const CollationSettings &other)
: SharedObject(other),
options(other.options), variableTop(other.variableTop),
reorderTable(nullptr),
minHighNoReorder(other.minHighNoReorder),
reorderRanges(nullptr), reorderRangesLength(0),
reorderCodes(nullptr), reorderCodesLength(0), reorderCodesCapacity(0),
fastLatinOptions(other.fastLatinOptions) {
UErrorCode errorCode = U_ZERO_ERROR;
copyReorderingFrom(other, errorCode);
if(fastLatinOptions >= 0) {
uprv_memcpy(fastLatinPrimaries, other.fastLatinPrimaries, sizeof(fastLatinPrimaries));
}
}
CollationSettings::~CollationSettings() {
if(reorderCodesCapacity != 0) {
uprv_free(const_cast<int32_t *>(reorderCodes));
}
}
bool
CollationSettings::operator==(const CollationSettings &other) const {
if(options != other.options) { return false; }
if((options & ALTERNATE_MASK) != 0 && variableTop != other.variableTop) { return false; }
if(reorderCodesLength != other.reorderCodesLength) { return false; }
for(int32_t i = 0; i < reorderCodesLength; ++i) {
if(reorderCodes[i] != other.reorderCodes[i]) { return false; }
}
return true;
}
int32_t
CollationSettings::hashCode() const {
int32_t h = options << 8;
if((options & ALTERNATE_MASK) != 0) { h ^= variableTop; }
h ^= reorderCodesLength;
for(int32_t i = 0; i < reorderCodesLength; ++i) {
h ^= (reorderCodes[i] << i);
}
return h;
}
void
CollationSettings::resetReordering() {
// When we turn off reordering, we want to set a nullptr permutation
// rather than a no-op permutation.
// Keep the memory via reorderCodes and its capacity.
reorderTable = nullptr;
minHighNoReorder = 0;
reorderRangesLength = 0;
reorderCodesLength = 0;
}
void
CollationSettings::aliasReordering(const CollationData &data, const int32_t *codes, int32_t length,
const uint32_t *ranges, int32_t rangesLength,
const uint8_t *table, UErrorCode &errorCode) {
if(U_FAILURE(errorCode)) { return; }
if(table != nullptr &&
(rangesLength == 0 ?
!reorderTableHasSplitBytes(table) :
rangesLength >= 2 &&
// The first offset must be 0. The last offset must not be 0.
(ranges[0] & 0xffff) == 0 && (ranges[rangesLength - 1] & 0xffff) != 0)) {
// We need to release the memory before setting the alias pointer.
if(reorderCodesCapacity != 0) {
uprv_free(const_cast<int32_t *>(reorderCodes));
reorderCodesCapacity = 0;
}
reorderTable = table;
reorderCodes = codes;
reorderCodesLength = length;
// Drop ranges before the first split byte. They are reordered by the table.
// This then speeds up reordering of the remaining ranges.
int32_t firstSplitByteRangeIndex = 0;
while(firstSplitByteRangeIndex < rangesLength &&
(ranges[firstSplitByteRangeIndex] & 0xff0000) == 0) {
// The second byte of the primary limit is 0.
++firstSplitByteRangeIndex;
}
if(firstSplitByteRangeIndex == rangesLength) {
U_ASSERT(!reorderTableHasSplitBytes(table));
minHighNoReorder = 0;
reorderRanges = nullptr;
reorderRangesLength = 0;
} else {
U_ASSERT(table[ranges[firstSplitByteRangeIndex] >> 24] == 0);
minHighNoReorder = ranges[rangesLength - 1] & 0xffff0000;
reorderRanges = ranges + firstSplitByteRangeIndex;
reorderRangesLength = rangesLength - firstSplitByteRangeIndex;
}
return;
}
// Regenerate missing data.
setReordering(data, codes, length, errorCode);
}
void
CollationSettings::setReordering(const CollationData &data,
const int32_t *codes, int32_t codesLength,
UErrorCode &errorCode) {
if(U_FAILURE(errorCode)) { return; }
if(codesLength == 0 || (codesLength == 1 && codes[0] == UCOL_REORDER_CODE_NONE)) {
resetReordering();
return;
}
UVector32 rangesList(errorCode);
data.makeReorderRanges(codes, codesLength, rangesList, errorCode);
if(U_FAILURE(errorCode)) { return; }
int32_t rangesLength = rangesList.size();
if(rangesLength == 0) {
resetReordering();
return;
}
const uint32_t *ranges = reinterpret_cast<uint32_t *>(rangesList.getBuffer());
// ranges[] contains at least two (limit, offset) pairs.
// The first offset must be 0. The last offset must not be 0.
// Separators (at the low end) and trailing weights (at the high end)
// are never reordered.
U_ASSERT(rangesLength >= 2);
U_ASSERT((ranges[0] & 0xffff) == 0 && (ranges[rangesLength - 1] & 0xffff) != 0);
minHighNoReorder = ranges[rangesLength - 1] & 0xffff0000;
// Write the lead byte permutation table.
// Set a 0 for each lead byte that has a range boundary in the middle.
uint8_t table[256];
int32_t b = 0;
int32_t firstSplitByteRangeIndex = -1;
for(int32_t i = 0; i < rangesLength; ++i) {
uint32_t pair = ranges[i];
int32_t limit1 = static_cast<int32_t>(pair >> 24);
while(b < limit1) {
table[b] = static_cast<uint8_t>(b + pair);
++b;
}
// Check the second byte of the limit.
if((pair & 0xff0000) != 0) {
table[limit1] = 0;
b = limit1 + 1;
if(firstSplitByteRangeIndex < 0) {
firstSplitByteRangeIndex = i;
}
}
}
while(b <= 0xff) {
table[b] = static_cast<uint8_t>(b);
++b;
}
if(firstSplitByteRangeIndex < 0) {
// The lead byte permutation table alone suffices for reordering.
rangesLength = 0;
} else {
// Remove the ranges below the first split byte.
ranges += firstSplitByteRangeIndex;
rangesLength -= firstSplitByteRangeIndex;
}
setReorderArrays(codes, codesLength, ranges, rangesLength, table, errorCode);
}
void
CollationSettings::setReorderArrays(const int32_t *codes, int32_t codesLength,
const uint32_t *ranges, int32_t rangesLength,
const uint8_t *table, UErrorCode &errorCode) {
if(U_FAILURE(errorCode)) { return; }
int32_t *ownedCodes;
int32_t totalLength = codesLength + rangesLength;
U_ASSERT(totalLength > 0);
if(totalLength <= reorderCodesCapacity) {
ownedCodes = const_cast<int32_t *>(reorderCodes);
} else {
// Allocate one memory block for the codes, the ranges, and the 16-aligned table.
int32_t capacity = (totalLength + 3) & ~3; // round up to a multiple of 4 ints
ownedCodes = static_cast<int32_t*>(uprv_malloc(capacity * 4 + 256));
if(ownedCodes == nullptr) {
resetReordering();
errorCode = U_MEMORY_ALLOCATION_ERROR;
return;
}
if(reorderCodesCapacity != 0) {
uprv_free(const_cast<int32_t *>(reorderCodes));
}
reorderCodes = ownedCodes;
reorderCodesCapacity = capacity;
}
uprv_memcpy(ownedCodes + reorderCodesCapacity, table, 256);
uprv_memcpy(ownedCodes, codes, codesLength * 4);
uprv_memcpy(ownedCodes + codesLength, ranges, rangesLength * 4);
reorderTable = reinterpret_cast<const uint8_t *>(reorderCodes + reorderCodesCapacity);
reorderCodesLength = codesLength;
reorderRanges = reinterpret_cast<uint32_t *>(ownedCodes) + codesLength;
reorderRangesLength = rangesLength;
}
void
CollationSettings::copyReorderingFrom(const CollationSettings &other, UErrorCode &errorCode) {
if(U_FAILURE(errorCode)) { return; }
if(!other.hasReordering()) {
resetReordering();
return;
}
minHighNoReorder = other.minHighNoReorder;
if(other.reorderCodesCapacity == 0) {
// The reorder arrays are aliased to memory-mapped data.
reorderTable = other.reorderTable;
reorderRanges = other.reorderRanges;
reorderRangesLength = other.reorderRangesLength;
reorderCodes = other.reorderCodes;
reorderCodesLength = other.reorderCodesLength;
} else {
setReorderArrays(other.reorderCodes, other.reorderCodesLength,
other.reorderRanges, other.reorderRangesLength,
other.reorderTable, errorCode);
}
}
UBool
CollationSettings::reorderTableHasSplitBytes(const uint8_t table[256]) {
U_ASSERT(table[0] == 0);
for(int32_t i = 1; i < 256; ++i) {
if(table[i] == 0) {
return true;
}
}
return false;
}
uint32_t
CollationSettings::reorderEx(uint32_t p) const {
if(p >= minHighNoReorder) { return p; }
// Round up p so that its lower 16 bits are >= any offset bits.
// Then compare q directly with (limit, offset) pairs.
uint32_t q = p | 0xffff;
uint32_t r;
const uint32_t *ranges = reorderRanges;
while(q >= (r = *ranges)) { ++ranges; }
return p + (r << 24);
}
void
CollationSettings::setStrength(int32_t value, int32_t defaultOptions, UErrorCode &errorCode) {
if(U_FAILURE(errorCode)) { return; }
int32_t noStrength = options & ~STRENGTH_MASK;
switch(value) {
case UCOL_PRIMARY:
case UCOL_SECONDARY:
case UCOL_TERTIARY:
case UCOL_QUATERNARY:
case UCOL_IDENTICAL:
options = noStrength | (value << STRENGTH_SHIFT);
break;
case UCOL_DEFAULT:
options = noStrength | (defaultOptions & STRENGTH_MASK);
break;
default:
errorCode = U_ILLEGAL_ARGUMENT_ERROR;
break;
}
}
void
CollationSettings::setFlag(int32_t bit, UColAttributeValue value,
int32_t defaultOptions, UErrorCode &errorCode) {
if(U_FAILURE(errorCode)) { return; }
switch(value) {
case UCOL_ON:
options |= bit;
break;
case UCOL_OFF:
options &= ~bit;
break;
case UCOL_DEFAULT:
options = (options & ~bit) | (defaultOptions & bit);
break;
default:
errorCode = U_ILLEGAL_ARGUMENT_ERROR;
break;
}
}
void
CollationSettings::setCaseFirst(UColAttributeValue value,
int32_t defaultOptions, UErrorCode &errorCode) {
if(U_FAILURE(errorCode)) { return; }
int32_t noCaseFirst = options & ~CASE_FIRST_AND_UPPER_MASK;
switch(value) {
case UCOL_OFF:
options = noCaseFirst;
break;
case UCOL_LOWER_FIRST:
options = noCaseFirst | CASE_FIRST;
break;
case UCOL_UPPER_FIRST:
options = noCaseFirst | CASE_FIRST_AND_UPPER_MASK;
break;
case UCOL_DEFAULT:
options = noCaseFirst | (defaultOptions & CASE_FIRST_AND_UPPER_MASK);
break;
default:
errorCode = U_ILLEGAL_ARGUMENT_ERROR;
break;
}
}
void
CollationSettings::setAlternateHandling(UColAttributeValue value,
int32_t defaultOptions, UErrorCode &errorCode) {
if(U_FAILURE(errorCode)) { return; }
int32_t noAlternate = options & ~ALTERNATE_MASK;
switch(value) {
case UCOL_NON_IGNORABLE:
options = noAlternate;
break;
case UCOL_SHIFTED:
options = noAlternate | SHIFTED;
break;
case UCOL_DEFAULT:
options = noAlternate | (defaultOptions & ALTERNATE_MASK);
break;
default:
errorCode = U_ILLEGAL_ARGUMENT_ERROR;
break;
}
}
void
CollationSettings::setMaxVariable(int32_t value, int32_t defaultOptions, UErrorCode &errorCode) {
if(U_FAILURE(errorCode)) { return; }
int32_t noMax = options & ~MAX_VARIABLE_MASK;
switch(value) {
case MAX_VAR_SPACE:
case MAX_VAR_PUNCT:
case MAX_VAR_SYMBOL:
case MAX_VAR_CURRENCY:
options = noMax | (value << MAX_VARIABLE_SHIFT);
break;
case UCOL_DEFAULT:
options = noMax | (defaultOptions & MAX_VARIABLE_MASK);
break;
default:
errorCode = U_ILLEGAL_ARGUMENT_ERROR;
break;
}
}
U_NAMESPACE_END
#endif // !UCONFIG_NO_COLLATION

274
deps/icu-small/source/i18n/collationsettings.h vendored Normal file
View File

@ -0,0 +1,274 @@
// © 2016 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html
/*
*******************************************************************************
* Copyright (C) 2013-2015, International Business Machines
* Corporation and others. All Rights Reserved.
*******************************************************************************
* collationsettings.h
*
* created on: 2013feb07
* created by: Markus W. Scherer
*/
#ifndef __COLLATIONSETTINGS_H__
#define __COLLATIONSETTINGS_H__
#include "unicode/utypes.h"
#if !UCONFIG_NO_COLLATION
#include "unicode/ucol.h"
#include "collation.h"
#include "sharedobject.h"
#include "umutex.h"
U_NAMESPACE_BEGIN
struct CollationData;
/**
* Collation settings/options/attributes.
* These are the values that can be changed via API.
*/
struct U_I18N_API CollationSettings : public SharedObject {
/**
* Options bit 0: Perform the FCD check on the input text and deliver normalized text.
*/
static const int32_t CHECK_FCD = 1;
/**
* Options bit 1: Numeric collation.
* Also known as CODAN = COllate Digits As Numbers.
*
* Treat digit sequences as numbers with CE sequences in numeric order,
* rather than returning a normal CE for each digit.
*/
static const int32_t NUMERIC = 2;
/**
* "Shifted" alternate handling, see ALTERNATE_MASK.
*/
static const int32_t SHIFTED = 4;
/**
* Options bits 3..2: Alternate-handling mask. 0 for non-ignorable.
* Reserve values 8 and 0xc for shift-trimmed and blanked.
*/
static const int32_t ALTERNATE_MASK = 0xc;
/**
* Options bits 6..4: The 3-bit maxVariable value bit field is shifted by this value.
*/
static const int32_t MAX_VARIABLE_SHIFT = 4;
/** maxVariable options bit mask before shifting. */
static const int32_t MAX_VARIABLE_MASK = 0x70;
/** Options bit 7: Reserved/unused/0. */
/**
* Options bit 8: Sort uppercase first if caseLevel or caseFirst is on.
*/
static const int32_t UPPER_FIRST = 0x100;
/**
* Options bit 9: Keep the case bits in the tertiary weight (they trump other tertiary values)
* unless case level is on (when they are *moved* into the separate case level).
* By default, the case bits are removed from the tertiary weight (ignored).
*
* When CASE_FIRST is off, UPPER_FIRST must be off too, corresponding to
* the tri-value UCOL_CASE_FIRST attribute: UCOL_OFF vs. UCOL_LOWER_FIRST vs. UCOL_UPPER_FIRST.
*/
static const int32_t CASE_FIRST = 0x200;
/**
* Options bit mask for caseFirst and upperFirst, before shifting.
* Same value as caseFirst==upperFirst.
*/
static const int32_t CASE_FIRST_AND_UPPER_MASK = CASE_FIRST | UPPER_FIRST;
/**
* Options bit 10: Insert the case level between the secondary and tertiary levels.
*/
static const int32_t CASE_LEVEL = 0x400;
/**
* Options bit 11: Compare secondary weights backwards. ("French secondary")
*/
static const int32_t BACKWARD_SECONDARY = 0x800;
/**
* Options bits 15..12: The 4-bit strength value bit field is shifted by this value.
* It is the top used bit field in the options. (No need to mask after shifting.)
*/
static const int32_t STRENGTH_SHIFT = 12;
/** Strength options bit mask before shifting. */
static const int32_t STRENGTH_MASK = 0xf000;
/** maxVariable values */
enum MaxVariable {
MAX_VAR_SPACE,
MAX_VAR_PUNCT,
MAX_VAR_SYMBOL,
MAX_VAR_CURRENCY
};
CollationSettings()
: options((UCOL_DEFAULT_STRENGTH << STRENGTH_SHIFT) |
(MAX_VAR_PUNCT << MAX_VARIABLE_SHIFT)),
variableTop(0),
reorderTable(nullptr),
minHighNoReorder(0),
reorderRanges(nullptr), reorderRangesLength(0),
reorderCodes(nullptr), reorderCodesLength(0), reorderCodesCapacity(0),
fastLatinOptions(-1) {}
CollationSettings(const CollationSettings &other);
virtual ~CollationSettings();
bool operator==(const CollationSettings &other) const;
inline bool operator!=(const CollationSettings &other) const {
return !operator==(other);
}
int32_t hashCode() const;
void resetReordering();
void aliasReordering(const CollationData &data, const int32_t *codes, int32_t length,
const uint32_t *ranges, int32_t rangesLength,
const uint8_t *table, UErrorCode &errorCode);
void setReordering(const CollationData &data, const int32_t *codes, int32_t codesLength,
UErrorCode &errorCode);
void copyReorderingFrom(const CollationSettings &other, UErrorCode &errorCode);
inline UBool hasReordering() const { return reorderTable != nullptr; }
static UBool reorderTableHasSplitBytes(const uint8_t table[256]);
inline uint32_t reorder(uint32_t p) const {
uint8_t b = reorderTable[p >> 24];
if(b != 0 || p <= Collation::NO_CE_PRIMARY) {
return (static_cast<uint32_t>(b) << 24) | (p & 0xffffff);
} else {
return reorderEx(p);
}
}
void setStrength(int32_t value, int32_t defaultOptions, UErrorCode &errorCode);
static int32_t getStrength(int32_t options) {
return options >> STRENGTH_SHIFT;
}
int32_t getStrength() const {
return getStrength(options);
}
/** Sets the options bit for an on/off attribute. */
void setFlag(int32_t bit, UColAttributeValue value,
int32_t defaultOptions, UErrorCode &errorCode);
UColAttributeValue getFlag(int32_t bit) const {
return ((options & bit) != 0) ? UCOL_ON : UCOL_OFF;
}
void setCaseFirst(UColAttributeValue value, int32_t defaultOptions, UErrorCode &errorCode);
UColAttributeValue getCaseFirst() const {
int32_t option = options & CASE_FIRST_AND_UPPER_MASK;
return (option == 0) ? UCOL_OFF :
(option == CASE_FIRST) ? UCOL_LOWER_FIRST : UCOL_UPPER_FIRST;
}
void setAlternateHandling(UColAttributeValue value,
int32_t defaultOptions, UErrorCode &errorCode);
UColAttributeValue getAlternateHandling() const {
return ((options & ALTERNATE_MASK) == 0) ? UCOL_NON_IGNORABLE : UCOL_SHIFTED;
}
void setMaxVariable(int32_t value, int32_t defaultOptions, UErrorCode &errorCode);
MaxVariable getMaxVariable() const {
return static_cast<MaxVariable>((options & MAX_VARIABLE_MASK) >> MAX_VARIABLE_SHIFT);
}
/**
* Include case bits in the tertiary level if caseLevel=off and caseFirst!=off.
*/
static inline UBool isTertiaryWithCaseBits(int32_t options) {
return (options & (CASE_LEVEL | CASE_FIRST)) == CASE_FIRST;
}
static uint32_t getTertiaryMask(int32_t options) {
// Remove the case bits from the tertiary weight when caseLevel is on or caseFirst is off.
return isTertiaryWithCaseBits(options) ?
Collation::CASE_AND_TERTIARY_MASK : Collation::ONLY_TERTIARY_MASK;
}
static UBool sortsTertiaryUpperCaseFirst(int32_t options) {
// On tertiary level, consider case bits and sort uppercase first
// if caseLevel is off and caseFirst==upperFirst.
return (options & (CASE_LEVEL | CASE_FIRST_AND_UPPER_MASK)) == CASE_FIRST_AND_UPPER_MASK;
}
inline UBool dontCheckFCD() const {
return (options & CHECK_FCD) == 0;
}
inline UBool hasBackwardSecondary() const {
return (options & BACKWARD_SECONDARY) != 0;
}
inline UBool isNumeric() const {
return (options & NUMERIC) != 0;
}
/** CHECK_FCD etc. */
int32_t options;
/** Variable-top primary weight. */
uint32_t variableTop;
/**
* 256-byte table for reordering permutation of primary lead bytes; nullptr if no reordering.
* A 0 entry at a non-zero index means that the primary lead byte is "split"
* (there are different offsets for primaries that share that lead byte)
* and the reordering offset must be determined via the reorderRanges.
*/
const uint8_t *reorderTable;
/** Limit of last reordered range. 0 if no reordering or no split bytes. */
uint32_t minHighNoReorder;
/**
* Primary-weight ranges for script reordering,
* to be used by reorder(p) for split-reordered primary lead bytes.
*
* Each entry is a (limit, offset) pair.
* The upper 16 bits of the entry are the upper 16 bits of the
* exclusive primary limit of a range.
* Primaries between the previous limit and this one have their lead bytes
* modified by the signed offset (-0xff..+0xff) stored in the lower 16 bits.
*
* CollationData::makeReorderRanges() writes a full list where the first range
* (at least for terminators and separators) has a 0 offset.
* The last range has a non-zero offset.
* minHighNoReorder is set to the limit of that last range.
*
* In the settings object, the initial ranges before the first split lead byte
* are omitted for efficiency; they are handled by reorder(p) via the reorderTable.
* If there are no split-reordered lead bytes, then no ranges are needed.
*/
const uint32_t *reorderRanges;
int32_t reorderRangesLength;
/** Array of reorder codes; ignored if reorderCodesLength == 0. */
const int32_t *reorderCodes;
/** Number of reorder codes; 0 if no reordering. */
int32_t reorderCodesLength;
/**
* Capacity of reorderCodes.
* If 0, then the codes, the ranges, and the table are aliases.
* Otherwise, this object owns the memory via the reorderCodes pointer;
* the codes, the ranges, and the table are in the same memory block, in that order.
*/
int32_t reorderCodesCapacity;
/** Options for CollationFastLatin. Negative if disabled. */
int32_t fastLatinOptions;
uint16_t fastLatinPrimaries[0x180];
private:
void setReorderArrays(const int32_t *codes, int32_t codesLength,
const uint32_t *ranges, int32_t rangesLength,
const uint8_t *table, UErrorCode &errorCode);
uint32_t reorderEx(uint32_t p) const;
};
U_NAMESPACE_END
#endif // !UCONFIG_NO_COLLATION
#endif // __COLLATIONSETTINGS_H__

113
deps/icu-small/source/i18n/collationtailoring.cpp vendored Normal file
View File

@ -0,0 +1,113 @@
// © 2016 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html
/*
*******************************************************************************
* Copyright (C) 2013-2015, International Business Machines
* Corporation and others. All Rights Reserved.
*******************************************************************************
* collationtailoring.cpp
*
* created on: 2013mar12
* created by: Markus W. Scherer
*/
#include "unicode/utypes.h"
#if !UCONFIG_NO_COLLATION
#include "unicode/udata.h"
#include "unicode/unistr.h"
#include "unicode/ures.h"
#include "unicode/uversion.h"
#include "unicode/uvernum.h"
#include "cmemory.h"
#include "collationdata.h"
#include "collationsettings.h"
#include "collationtailoring.h"
#include "normalizer2impl.h"
#include "uassert.h"
#include "uhash.h"
#include "umutex.h"
#include "utrie2.h"
U_NAMESPACE_BEGIN
CollationTailoring::CollationTailoring(const CollationSettings *baseSettings)
: data(nullptr), settings(baseSettings),
actualLocale(""),
ownedData(nullptr),
builder(nullptr), memory(nullptr), bundle(nullptr),
trie(nullptr), unsafeBackwardSet(nullptr),
maxExpansions(nullptr) {
if(baseSettings != nullptr) {
U_ASSERT(baseSettings->reorderCodesLength == 0);
U_ASSERT(baseSettings->reorderTable == nullptr);
U_ASSERT(baseSettings->minHighNoReorder == 0);
} else {
settings = new CollationSettings();
}
if(settings != nullptr) {
settings->addRef();
}
rules.getTerminatedBuffer(); // ensure NUL-termination
version[0] = version[1] = version[2] = version[3] = 0;
maxExpansionsInitOnce.reset();
}
CollationTailoring::~CollationTailoring() {
SharedObject::clearPtr(settings);
delete ownedData;
delete builder;
udata_close(memory);
ures_close(bundle);
utrie2_close(trie);
delete unsafeBackwardSet;
uhash_close(maxExpansions);
maxExpansionsInitOnce.reset();
}
UBool
CollationTailoring::ensureOwnedData(UErrorCode &errorCode) {
if(U_FAILURE(errorCode)) { return false; }
if(ownedData == nullptr) {
const Normalizer2Impl *nfcImpl = Normalizer2Factory::getNFCImpl(errorCode);
if(U_FAILURE(errorCode)) { return false; }
ownedData = new CollationData(*nfcImpl);
if(ownedData == nullptr) {
errorCode = U_MEMORY_ALLOCATION_ERROR;
return false;
}
}
data = ownedData;
return true;
}
void
CollationTailoring::makeBaseVersion(const UVersionInfo ucaVersion, UVersionInfo version) {
version[0] = UCOL_BUILDER_VERSION;
version[1] = (ucaVersion[0] << 3) + ucaVersion[1];
version[2] = ucaVersion[2] << 6;
version[3] = 0;
}
void
CollationTailoring::setVersion(const UVersionInfo baseVersion, const UVersionInfo rulesVersion) {
version[0] = UCOL_BUILDER_VERSION;
version[1] = baseVersion[1];
version[2] = (baseVersion[2] & 0xc0) + ((rulesVersion[0] + (rulesVersion[0] >> 6)) & 0x3f);
version[3] = (rulesVersion[1] << 3) + (rulesVersion[1] >> 5) + rulesVersion[2] +
(rulesVersion[3] << 4) + (rulesVersion[3] >> 4);
}
int32_t
CollationTailoring::getUCAVersion() const {
return (static_cast<int32_t>(version[1]) << 4) | (version[2] >> 6);
}
CollationCacheEntry::~CollationCacheEntry() {
SharedObject::clearPtr(tailoring);
}
U_NAMESPACE_END
#endif // !UCONFIG_NO_COLLATION

117
deps/icu-small/source/i18n/collationtailoring.h vendored Normal file
View File

@ -0,0 +1,117 @@
// © 2016 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html
/*
*******************************************************************************
* Copyright (C) 2013-2014, International Business Machines
* Corporation and others. All Rights Reserved.
*******************************************************************************
* collationtailoring.h
*
* created on: 2013mar12
* created by: Markus W. Scherer
*/
#ifndef __COLLATIONTAILORING_H__
#define __COLLATIONTAILORING_H__
#include "unicode/utypes.h"
#if !UCONFIG_NO_COLLATION
#include "unicode/locid.h"
#include "unicode/unistr.h"
#include "unicode/uversion.h"
#include "collationsettings.h"
#include "uhash.h"
#include "umutex.h"
#include "unifiedcache.h"
struct UDataMemory;
struct UResourceBundle;
struct UTrie2;
U_NAMESPACE_BEGIN
struct CollationData;
class UnicodeSet;
/**
* Collation tailoring data & settings.
* This is a container of values for a collation tailoring
* built from rules or deserialized from binary data.
*
* It is logically immutable: Do not modify its values.
* The fields are public for convenience.
*
* It is shared, reference-counted, and auto-deleted; see SharedObject.
*/
struct U_I18N_API CollationTailoring : public SharedObject {
CollationTailoring(const CollationSettings *baseSettings);
virtual ~CollationTailoring();
/**
* Returns true if the constructor could not initialize properly.
*/
UBool isBogus() { return settings == nullptr; }
UBool ensureOwnedData(UErrorCode &errorCode);
static void makeBaseVersion(const UVersionInfo ucaVersion, UVersionInfo version);
void setVersion(const UVersionInfo baseVersion, const UVersionInfo rulesVersion);
int32_t getUCAVersion() const;
// data for sorting etc.
const CollationData *data; // == base data or ownedData
const CollationSettings *settings; // reference-counted
UnicodeString rules;
// The locale is bogus when built from rules or constructed from a binary blob.
// It can then be set by the service registration code which is thread-safe.
mutable Locale actualLocale;
// UCA version u.v.w & rules version r.s.t.q:
// version[0]: builder version (runtime version is mixed in at runtime)
// version[1]: bits 7..3=u, bits 2..0=v
// version[2]: bits 7..6=w, bits 5..0=r
// version[3]= (s<<5)+(s>>3)+t+(q<<4)+(q>>4)
UVersionInfo version;
// owned objects
CollationData *ownedData;
UObject *builder;
UDataMemory *memory;
UResourceBundle *bundle;
UTrie2 *trie;
UnicodeSet *unsafeBackwardSet;
mutable UHashtable *maxExpansions;
mutable UInitOnce maxExpansionsInitOnce;
private:
/**
* No copy constructor: A CollationTailoring cannot be copied.
* It is immutable, and the data trie cannot be copied either.
*/
CollationTailoring(const CollationTailoring &other) = delete;
};
struct U_I18N_API CollationCacheEntry : public SharedObject {
CollationCacheEntry(const Locale &loc, const CollationTailoring *t)
: validLocale(loc), tailoring(t) {
if(t != nullptr) {
t->addRef();
}
}
~CollationCacheEntry();
Locale validLocale;
const CollationTailoring *tailoring;
};
template<> U_I18N_API
const CollationCacheEntry *
LocaleCacheKey<CollationCacheEntry>::createObject(const void *creationContext,
UErrorCode &errorCode) const;
U_NAMESPACE_END
#endif // !UCONFIG_NO_COLLATION
#endif // __COLLATIONTAILORING_H__

570
deps/icu-small/source/i18n/collationweights.cpp vendored Normal file
View File

@ -0,0 +1,570 @@
// © 2016 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html
/*
*******************************************************************************
*
* Copyright (C) 1999-2015, International Business Machines
* Corporation and others. All Rights Reserved.
*
*******************************************************************************
* file name: collationweights.cpp
* encoding: UTF-8
* tab size: 8 (not used)
* indentation:4
*
* created on: 2001mar08 as ucol_wgt.cpp
* created by: Markus W. Scherer
*
* This file contains code for allocating n collation element weights
* between two exclusive limits.
* It is used only internally by the collation tailoring builder.
*/
#include "unicode/utypes.h"
#if !UCONFIG_NO_COLLATION
#include "cmemory.h"
#include "collation.h"
#include "collationweights.h"
#include "uarrsort.h"
#include "uassert.h"
#ifdef UCOL_DEBUG
# include <stdio.h>
#endif
U_NAMESPACE_BEGIN
/* collation element weight allocation -------------------------------------- */
/* helper functions for CE weights */
static inline uint32_t
getWeightTrail(uint32_t weight, int32_t length) {
return (weight >> (8 * (4 - length))) & 0xff;
}
static inline uint32_t
setWeightTrail(uint32_t weight, int32_t length, uint32_t trail) {
length=8*(4-length);
return static_cast<uint32_t>((weight & (0xffffff00 << length)) | (trail << length));
}
static inline uint32_t
getWeightByte(uint32_t weight, int32_t idx) {
return getWeightTrail(weight, idx); /* same calculation */
}
static inline uint32_t
setWeightByte(uint32_t weight, int32_t idx, uint32_t byte) {
uint32_t mask; /* 0xffffffff except a 00 "hole" for the index-th byte */
idx*=8;
if(idx<32) {
mask = (static_cast<uint32_t>(0xffffffff)) >> idx;
} else {
// Do not use uint32_t>>32 because on some platforms that does not shift at all
// while we need it to become 0.
// PowerPC: 0xffffffff>>32 = 0 (wanted)
// x86: 0xffffffff>>32 = 0xffffffff (not wanted)
//
// ANSI C99 6.5.7 Bitwise shift operators:
// "If the value of the right operand is negative
// or is greater than or equal to the width of the promoted left operand,
// the behavior is undefined."
mask=0;
}
idx=32-idx;
mask|=0xffffff00<<idx;
return ((weight & mask) | (byte << idx));
}
static inline uint32_t
truncateWeight(uint32_t weight, int32_t length) {
return static_cast<uint32_t>(weight & (0xffffffff << (8 * (4 - length))));
}
static inline uint32_t
incWeightTrail(uint32_t weight, int32_t length) {
return static_cast<uint32_t>(weight + (1UL << (8 * (4 - length))));
}
static inline uint32_t
decWeightTrail(uint32_t weight, int32_t length) {
return static_cast<uint32_t>(weight - (1UL << (8 * (4 - length))));
}
CollationWeights::CollationWeights()
: middleLength(0), rangeIndex(0), rangeCount(0) {
for(int32_t i = 0; i < 5; ++i) {
minBytes[i] = maxBytes[i] = 0;
}
}
void
CollationWeights::initForPrimary(UBool compressible) {
middleLength=1;
minBytes[1] = Collation::MERGE_SEPARATOR_BYTE + 1;
maxBytes[1] = Collation::TRAIL_WEIGHT_BYTE;
if(compressible) {
minBytes[2] = Collation::PRIMARY_COMPRESSION_LOW_BYTE + 1;
maxBytes[2] = Collation::PRIMARY_COMPRESSION_HIGH_BYTE - 1;
} else {
minBytes[2] = 2;
maxBytes[2] = 0xff;
}
minBytes[3] = 2;
maxBytes[3] = 0xff;
minBytes[4] = 2;
maxBytes[4] = 0xff;
}
void
CollationWeights::initForSecondary() {
// We use only the lower 16 bits for secondary weights.
middleLength=3;
minBytes[1] = 0;
maxBytes[1] = 0;
minBytes[2] = 0;
maxBytes[2] = 0;
minBytes[3] = Collation::LEVEL_SEPARATOR_BYTE + 1;
maxBytes[3] = 0xff;
minBytes[4] = 2;
maxBytes[4] = 0xff;
}
void
CollationWeights::initForTertiary() {
// We use only the lower 16 bits for tertiary weights.
middleLength=3;
minBytes[1] = 0;
maxBytes[1] = 0;
minBytes[2] = 0;
maxBytes[2] = 0;
// We use only 6 bits per byte.
// The other bits are used for case & quaternary weights.
minBytes[3] = Collation::LEVEL_SEPARATOR_BYTE + 1;
maxBytes[3] = 0x3f;
minBytes[4] = 2;
maxBytes[4] = 0x3f;
}
uint32_t
CollationWeights::incWeight(uint32_t weight, int32_t length) const {
for(;;) {
uint32_t byte=getWeightByte(weight, length);
if(byte<maxBytes[length]) {
return setWeightByte(weight, length, byte+1);
} else {
// Roll over, set this byte to the minimum and increment the previous one.
weight=setWeightByte(weight, length, minBytes[length]);
--length;
U_ASSERT(length > 0);
}
}
}
uint32_t
CollationWeights::incWeightByOffset(uint32_t weight, int32_t length, int32_t offset) const {
for(;;) {
offset += getWeightByte(weight, length);
if (static_cast<uint32_t>(offset) <= maxBytes[length]) {
return setWeightByte(weight, length, offset);
} else {
// Split the offset between this byte and the previous one.
offset -= minBytes[length];
weight = setWeightByte(weight, length, minBytes[length] + offset % countBytes(length));
offset /= countBytes(length);
--length;
U_ASSERT(length > 0);
}
}
}
void
CollationWeights::lengthenRange(WeightRange &range) const {
int32_t length=range.length+1;
range.start=setWeightTrail(range.start, length, minBytes[length]);
range.end=setWeightTrail(range.end, length, maxBytes[length]);
range.count*=countBytes(length);
range.length=length;
}
/* for uprv_sortArray: sort ranges in weight order */
static int32_t U_CALLCONV
compareRanges(const void * /*context*/, const void *left, const void *right) {
uint32_t l, r;
l = static_cast<const CollationWeights::WeightRange*>(left)->start;
r = static_cast<const CollationWeights::WeightRange*>(right)->start;
if(l<r) {
return -1;
} else if(l>r) {
return 1;
} else {
return 0;
}
}
UBool
CollationWeights::getWeightRanges(uint32_t lowerLimit, uint32_t upperLimit) {
U_ASSERT(lowerLimit != 0);
U_ASSERT(upperLimit != 0);
/* get the lengths of the limits */
int32_t lowerLength=lengthOfWeight(lowerLimit);
int32_t upperLength=lengthOfWeight(upperLimit);
#ifdef UCOL_DEBUG
printf("length of lower limit 0x%08lx is %ld\n", lowerLimit, lowerLength);
printf("length of upper limit 0x%08lx is %ld\n", upperLimit, upperLength);
#endif
U_ASSERT(lowerLength>=middleLength);
// Permit upperLength<middleLength: The upper limit for secondaries is 0x10000.
if(lowerLimit>=upperLimit) {
#ifdef UCOL_DEBUG
printf("error: no space between lower & upper limits\n");
#endif
return false;
}
/* check that neither is a prefix of the other */
if(lowerLength<upperLength) {
if(lowerLimit==truncateWeight(upperLimit, lowerLength)) {
#ifdef UCOL_DEBUG
printf("error: lower limit 0x%08lx is a prefix of upper limit 0x%08lx\n", lowerLimit, upperLimit);
#endif
return false;
}
}
/* if the upper limit is a prefix of the lower limit then the earlier test lowerLimit>=upperLimit has caught it */
WeightRange lower[5], middle, upper[5]; /* [0] and [1] are not used - this simplifies indexing */
uprv_memset(lower, 0, sizeof(lower));
uprv_memset(&middle, 0, sizeof(middle));
uprv_memset(upper, 0, sizeof(upper));
/*
* With the limit lengths of 1..4, there are up to 7 ranges for allocation:
* range minimum length
* lower[4] 4
* lower[3] 3
* lower[2] 2
* middle 1
* upper[2] 2
* upper[3] 3
* upper[4] 4
*
* We are now going to calculate up to 7 ranges.
* Some of them will typically overlap, so we will then have to merge and eliminate ranges.
*/
uint32_t weight=lowerLimit;
for(int32_t length=lowerLength; length>middleLength; --length) {
uint32_t trail=getWeightTrail(weight, length);
if(trail<maxBytes[length]) {
lower[length].start=incWeightTrail(weight, length);
lower[length].end=setWeightTrail(weight, length, maxBytes[length]);
lower[length].length=length;
lower[length].count=maxBytes[length]-trail;
}
weight=truncateWeight(weight, length-1);
}
if(weight<0xff000000) {
middle.start=incWeightTrail(weight, middleLength);
} else {
// Prevent overflow for primary lead byte FF
// which would yield a middle range starting at 0.
middle.start=0xffffffff; // no middle range
}
weight=upperLimit;
for(int32_t length=upperLength; length>middleLength; --length) {
uint32_t trail=getWeightTrail(weight, length);
if(trail>minBytes[length]) {
upper[length].start=setWeightTrail(weight, length, minBytes[length]);
upper[length].end=decWeightTrail(weight, length);
upper[length].length=length;
upper[length].count=trail-minBytes[length];
}
weight=truncateWeight(weight, length-1);
}
middle.end=decWeightTrail(weight, middleLength);
/* set the middle range */
middle.length=middleLength;
if(middle.end>=middle.start) {
middle.count = static_cast<int32_t>((middle.end - middle.start) >> (8 * (4 - middleLength))) + 1;
} else {
/* no middle range, eliminate overlaps */
for(int32_t length=4; length>middleLength; --length) {
if(lower[length].count>0 && upper[length].count>0) {
// Note: The lowerEnd and upperStart weights are versions of
// lowerLimit and upperLimit (which are lowerLimit<upperLimit),
// truncated (still less-or-equal)
// and then with their last bytes changed to the
// maxByte (for lowerEnd) or minByte (for upperStart).
const uint32_t lowerEnd=lower[length].end;
const uint32_t upperStart=upper[length].start;
UBool merged=false;
if(lowerEnd>upperStart) {
// These two lower and upper ranges collide.
// Since lowerLimit<upperLimit and lowerEnd and upperStart
// are versions with only their last bytes modified
// (and following ones removed/reset to 0),
// lowerEnd>upperStart is only possible
// if the leading bytes are equal
// and lastByte(lowerEnd)>lastByte(upperStart).
U_ASSERT(truncateWeight(lowerEnd, length-1)==
truncateWeight(upperStart, length-1));
// Intersect these two ranges.
lower[length].end=upper[length].end;
lower[length].count=
static_cast<int32_t>(getWeightTrail(lower[length].end, length)) -
static_cast<int32_t>(getWeightTrail(lower[length].start, length)) + 1;
// count might be <=0 in which case there is no room,
// and the range-collecting code below will ignore this range.
merged=true;
} else if(lowerEnd==upperStart) {
// Not possible, unless minByte==maxByte which is not allowed.
U_ASSERT(minBytes[length]<maxBytes[length]);
} else /* lowerEnd<upperStart */ {
if(incWeight(lowerEnd, length)==upperStart) {
// Merge adjacent ranges.
lower[length].end=upper[length].end;
lower[length].count+=upper[length].count; // might be >countBytes
merged=true;
}
}
if(merged) {
// Remove all shorter ranges.
// There was no room available for them between the ranges we just merged.
upper[length].count=0;
while(--length>middleLength) {
lower[length].count=upper[length].count=0;
}
break;
}
}
}
}
#ifdef UCOL_DEBUG
/* print ranges */
for(int32_t length=4; length>=2; --length) {
if(lower[length].count>0) {
printf("lower[%ld] .start=0x%08lx .end=0x%08lx .count=%ld\n", length, lower[length].start, lower[length].end, lower[length].count);
}
}
if(middle.count>0) {
printf("middle .start=0x%08lx .end=0x%08lx .count=%ld\n", middle.start, middle.end, middle.count);
}
for(int32_t length=2; length<=4; ++length) {
if(upper[length].count>0) {
printf("upper[%ld] .start=0x%08lx .end=0x%08lx .count=%ld\n", length, upper[length].start, upper[length].end, upper[length].count);
}
}
#endif
/* copy the ranges, shortest first, into the result array */
rangeCount=0;
if(middle.count>0) {
uprv_memcpy(ranges, &middle, sizeof(WeightRange));
rangeCount=1;
}
for(int32_t length=middleLength+1; length<=4; ++length) {
/* copy upper first so that later the middle range is more likely the first one to use */
if(upper[length].count>0) {
uprv_memcpy(ranges+rangeCount, upper+length, sizeof(WeightRange));
++rangeCount;
}
if(lower[length].count>0) {
uprv_memcpy(ranges+rangeCount, lower+length, sizeof(WeightRange));
++rangeCount;
}
}
return rangeCount>0;
}
UBool
CollationWeights::allocWeightsInShortRanges(int32_t n, int32_t minLength) {
// See if the first few minLength and minLength+1 ranges have enough weights.
for(int32_t i = 0; i < rangeCount && ranges[i].length <= (minLength + 1); ++i) {
if(n <= ranges[i].count) {
// Use the first few minLength and minLength+1 ranges.
if(ranges[i].length > minLength) {
// Reduce the number of weights from the last minLength+1 range
// which might sort before some minLength ranges,
// so that we use all weights in the minLength ranges.
ranges[i].count = n;
}
rangeCount = i + 1;
#ifdef UCOL_DEBUG
printf("take first %ld ranges\n", rangeCount);
#endif
if(rangeCount>1) {
/* sort the ranges by weight values */
UErrorCode errorCode=U_ZERO_ERROR;
uprv_sortArray(ranges, rangeCount, sizeof(WeightRange),
compareRanges, nullptr, false, &errorCode);
/* ignore error code: we know that the internal sort function will not fail here */
}
return true;
}
n -= ranges[i].count; // still >0
}
return false;
}
UBool
CollationWeights::allocWeightsInMinLengthRanges(int32_t n, int32_t minLength) {
// See if the minLength ranges have enough weights
// when we split one and lengthen the following ones.
int32_t count = 0;
int32_t minLengthRangeCount;
for(minLengthRangeCount = 0;
minLengthRangeCount < rangeCount &&
ranges[minLengthRangeCount].length == minLength;
++minLengthRangeCount) {
count += ranges[minLengthRangeCount].count;
}
int32_t nextCountBytes = countBytes(minLength + 1);
if(n > count * nextCountBytes) { return false; }
// Use the minLength ranges. Merge them, and then split again as necessary.
uint32_t start = ranges[0].start;
uint32_t end = ranges[0].end;
for(int32_t i = 1; i < minLengthRangeCount; ++i) {
if(ranges[i].start < start) { start = ranges[i].start; }
if(ranges[i].end > end) { end = ranges[i].end; }
}
// Calculate how to split the range between minLength (count1) and minLength+1 (count2).
// Goal:
// count1 + count2 * nextCountBytes = n
// count1 + count2 = count
// These turn into
// (count - count2) + count2 * nextCountBytes = n
// and then into the following count1 & count2 computations.
int32_t count2 = (n - count) / (nextCountBytes - 1); // number of weights to be lengthened
int32_t count1 = count - count2; // number of minLength weights
if(count2 == 0 || (count1 + count2 * nextCountBytes) < n) {
// round up
++count2;
--count1;
U_ASSERT((count1 + count2 * nextCountBytes) >= n);
}
ranges[0].start = start;
if(count1 == 0) {
// Make one long range.
ranges[0].end = end;
ranges[0].count = count;
lengthenRange(ranges[0]);
rangeCount = 1;
} else {
// Split the range, lengthen the second part.
#ifdef UCOL_DEBUG
printf("split the range number %ld (out of %ld minLength ranges) by %ld:%ld\n",
splitRange, rangeCount, count1, count2);
#endif
// Next start = start + count1. First end = 1 before that.
ranges[0].end = incWeightByOffset(start, minLength, count1 - 1);
ranges[0].count = count1;
ranges[1].start = incWeight(ranges[0].end, minLength);
ranges[1].end = end;
ranges[1].length = minLength; // +1 when lengthened
ranges[1].count = count2; // *countBytes when lengthened
lengthenRange(ranges[1]);
rangeCount = 2;
}
return true;
}
/*
* call getWeightRanges and then determine heuristically
* which ranges to use for a given number of weights between (excluding)
* two limits
*/
UBool
CollationWeights::allocWeights(uint32_t lowerLimit, uint32_t upperLimit, int32_t n) {
#ifdef UCOL_DEBUG
puts("");
#endif
if(!getWeightRanges(lowerLimit, upperLimit)) {
#ifdef UCOL_DEBUG
printf("error: unable to get Weight ranges\n");
#endif
return false;
}
/* try until we find suitably large ranges */
for(;;) {
/* get the smallest number of bytes in a range */
int32_t minLength=ranges[0].length;
if(allocWeightsInShortRanges(n, minLength)) { break; }
if(minLength == 4) {
#ifdef UCOL_DEBUG
printf("error: the maximum number of %ld weights is insufficient for n=%ld\n",
minLengthCount, n);
#endif
return false;
}
if(allocWeightsInMinLengthRanges(n, minLength)) { break; }
/* no good match, lengthen all minLength ranges and iterate */
#ifdef UCOL_DEBUG
printf("lengthen the short ranges from %ld bytes to %ld and iterate\n", minLength, minLength+1);
#endif
for(int32_t i=0; i<rangeCount && ranges[i].length==minLength; ++i) {
lengthenRange(ranges[i]);
}
}
#ifdef UCOL_DEBUG
puts("final ranges:");
for(int32_t i=0; i<rangeCount; ++i) {
printf("ranges[%ld] .start=0x%08lx .end=0x%08lx .length=%ld .count=%ld\n",
i, ranges[i].start, ranges[i].end, ranges[i].length, ranges[i].count);
}
#endif
rangeIndex = 0;
return true;
}
uint32_t
CollationWeights::nextWeight() {
if(rangeIndex >= rangeCount) {
return 0xffffffff;
} else {
/* get the next weight */
WeightRange &range = ranges[rangeIndex];
uint32_t weight = range.start;
if(--range.count == 0) {
/* this range is finished */
++rangeIndex;
} else {
/* increment the weight for the next value */
range.start = incWeight(weight, range.length);
U_ASSERT(range.start <= range.end);
}
return weight;
}
}
U_NAMESPACE_END
#endif /* #if !UCONFIG_NO_COLLATION */

113
deps/icu-small/source/i18n/collationweights.h vendored Normal file
View File

@ -0,0 +1,113 @@
// © 2016 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html
/*
*******************************************************************************
*
* Copyright (C) 1999-2014, International Business Machines
* Corporation and others. All Rights Reserved.
*
*******************************************************************************
* file name: collationweights.h
* encoding: UTF-8
* tab size: 8 (not used)
* indentation:4
*
* created on: 2001mar08 as ucol_wgt.h
* created by: Markus W. Scherer
*/
#ifndef __COLLATIONWEIGHTS_H__
#define __COLLATIONWEIGHTS_H__
#include "unicode/utypes.h"
#if !UCONFIG_NO_COLLATION
#include "unicode/uobject.h"
U_NAMESPACE_BEGIN
/**
* Allocates n collation element weights between two exclusive limits.
* Used only internally by the collation tailoring builder.
*/
class U_I18N_API CollationWeights : public UMemory {
public:
CollationWeights();
static inline int32_t lengthOfWeight(uint32_t weight) {
if((weight&0xffffff)==0) {
return 1;
} else if((weight&0xffff)==0) {
return 2;
} else if((weight&0xff)==0) {
return 3;
} else {
return 4;
}
}
void initForPrimary(UBool compressible);
void initForSecondary();
void initForTertiary();
/**
* Determine heuristically
* what ranges to use for a given number of weights between (excluding)
* two limits.
*
* @param lowerLimit A collation element weight; the ranges will be filled to cover
* weights greater than this one.
* @param upperLimit A collation element weight; the ranges will be filled to cover
* weights less than this one.
* @param n The number of collation element weights w necessary such that
* lowerLimit<w<upperLimit in lexical order.
* @return true if it is possible to fit n elements between the limits
*/
UBool allocWeights(uint32_t lowerLimit, uint32_t upperLimit, int32_t n);
/**
* Given a set of ranges calculated by allocWeights(),
* iterate through the weights.
* The ranges are modified to keep the current iteration state.
*
* @return The next weight in the ranges, or 0xffffffff if there is none left.
*/
uint32_t nextWeight();
/** @internal */
struct WeightRange {
uint32_t start, end;
int32_t length, count;
};
private:
/** @return number of usable byte values for byte idx */
inline int32_t countBytes(int32_t idx) const {
return static_cast<int32_t>(maxBytes[idx] - minBytes[idx] + 1);
}
uint32_t incWeight(uint32_t weight, int32_t length) const;
uint32_t incWeightByOffset(uint32_t weight, int32_t length, int32_t offset) const;
void lengthenRange(WeightRange &range) const;
/**
* Takes two CE weights and calculates the
* possible ranges of weights between the two limits, excluding them.
* For weights with up to 4 bytes there are up to 2*4-1=7 ranges.
*/
UBool getWeightRanges(uint32_t lowerLimit, uint32_t upperLimit);
UBool allocWeightsInShortRanges(int32_t n, int32_t minLength);
UBool allocWeightsInMinLengthRanges(int32_t n, int32_t minLength);
int32_t middleLength;
uint32_t minBytes[5]; // for byte 1, 2, 3, 4
uint32_t maxBytes[5];
WeightRange ranges[7];
int32_t rangeIndex;
int32_t rangeCount;
};
U_NAMESPACE_END
#endif // !UCONFIG_NO_COLLATION
#endif // __COLLATIONWEIGHTS_H__

127
deps/icu-small/source/i18n/collunsafe.h vendored Normal file
View File

@ -0,0 +1,127 @@
// © 2016 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html
// collunsafe.h
// Copyright (C) 2015-2016, International Business Machines Corporation and others.
// All Rights Reserved.
// To be included by collationdatareader.cpp, and generated by gencolusb.
// Machine generated, do not edit.
#ifndef COLLUNSAFE_H
#define COLLUNSAFE_H
#include "unicode/utypes.h"
#define COLLUNSAFE_ICU_VERSION "56.0.1"
#define COLLUNSAFE_COLL_VERSION "9.64"
#define COLLUNSAFE_SERIALIZE 1
static const int32_t unsafe_serializedCount = 850;
static const uint16_t unsafe_serializedData[850] = {
0x8350, 0x01B8, 0x0034, 0x0035, 0x004C, 0x004D, 0x00A0, 0x00A1, // 8
0x0300, 0x034F, 0x0350, 0x0370, 0x03A9, 0x03AA, 0x03E2, 0x03E3, // 16
0x042F, 0x0430, 0x0483, 0x0488, 0x0531, 0x0532, 0x0591, 0x05BE, // 24
0x05BF, 0x05C0, 0x05C1, 0x05C3, 0x05C4, 0x05C6, 0x05C7, 0x05C8, // 32
0x05D0, 0x05D1, 0x0610, 0x061B, 0x0628, 0x0629, 0x064B, 0x0660, // 40
0x0670, 0x0671, 0x06D6, 0x06DD, 0x06DF, 0x06E5, 0x06E7, 0x06E9, // 48
0x06EA, 0x06EE, 0x0710, 0x0712, 0x0730, 0x074B, 0x078C, 0x078D, // 56
0x07D8, 0x07D9, 0x07EB, 0x07F4, 0x0800, 0x0801, 0x0816, 0x081A, // 64
0x081B, 0x0824, 0x0825, 0x0828, 0x0829, 0x082E, 0x0840, 0x0841, // 72
0x0859, 0x085C, 0x08E3, 0x0900, 0x0905, 0x0906, 0x093C, 0x093D, // 80
0x094D, 0x094E, 0x0951, 0x0955, 0x0995, 0x0996, 0x09BC, 0x09BD, // 88
0x09BE, 0x09BF, 0x09CD, 0x09CE, 0x09D7, 0x09D8, 0x0A15, 0x0A16, // 96
0x0A3C, 0x0A3D, 0x0A4D, 0x0A4E, 0x0A95, 0x0A96, 0x0ABC, 0x0ABD, // 104
0x0ACD, 0x0ACE, 0x0B15, 0x0B16, 0x0B3C, 0x0B3D, 0x0B3E, 0x0B3F, // 112
0x0B4D, 0x0B4E, 0x0B56, 0x0B58, 0x0B95, 0x0B96, 0x0BBE, 0x0BBF, // 120
0x0BCD, 0x0BCE, 0x0BD7, 0x0BD8, 0x0C15, 0x0C16, 0x0C4D, 0x0C4E, // 128
0x0C55, 0x0C57, 0x0C95, 0x0C96, 0x0CBC, 0x0CBD, 0x0CC2, 0x0CC3, // 136
0x0CCD, 0x0CCE, 0x0CD5, 0x0CD7, 0x0D15, 0x0D16, 0x0D3E, 0x0D3F, // 144
0x0D4D, 0x0D4E, 0x0D57, 0x0D58, 0x0D85, 0x0D86, 0x0DCA, 0x0DCB, // 152
0x0DCF, 0x0DD0, 0x0DDF, 0x0DE0, 0x0E01, 0x0E2F, 0x0E32, 0x0E33, // 160
0x0E38, 0x0E3B, 0x0E48, 0x0E4C, 0x0E81, 0x0E83, 0x0E84, 0x0E85, // 168
0x0E87, 0x0E89, 0x0E8A, 0x0E8B, 0x0E8D, 0x0E8E, 0x0E94, 0x0E98, // 176
0x0E99, 0x0EA0, 0x0EA1, 0x0EA4, 0x0EA5, 0x0EA6, 0x0EA7, 0x0EA8, // 184
0x0EAA, 0x0EAC, 0x0EAD, 0x0EAF, 0x0EB2, 0x0EB3, 0x0EB8, 0x0EBA, // 192
0x0EC8, 0x0ECC, 0x0EDC, 0x0EE0, 0x0F18, 0x0F1A, 0x0F35, 0x0F36, // 200
0x0F37, 0x0F38, 0x0F39, 0x0F3A, 0x0F40, 0x0F41, 0x0F71, 0x0F76, // 208
0x0F7A, 0x0F7E, 0x0F80, 0x0F85, 0x0F86, 0x0F88, 0x0FC6, 0x0FC7, // 216
0x1000, 0x1001, 0x102E, 0x102F, 0x1037, 0x1038, 0x1039, 0x103B, // 224
0x108D, 0x108E, 0x10D3, 0x10D4, 0x12A0, 0x12A1, 0x135D, 0x1360, // 232
0x13C4, 0x13C5, 0x14C0, 0x14C1, 0x168F, 0x1690, 0x16A0, 0x16A1, // 240
0x1703, 0x1704, 0x1714, 0x1715, 0x1723, 0x1724, 0x1734, 0x1735, // 248
0x1743, 0x1744, 0x1763, 0x1764, 0x1780, 0x1781, 0x17D2, 0x17D3, // 256
0x17DD, 0x17DE, 0x1826, 0x1827, 0x18A9, 0x18AA, 0x1900, 0x1901, // 264
0x1939, 0x193C, 0x1950, 0x1951, 0x1980, 0x19AC, 0x1A00, 0x1A01, // 272
0x1A17, 0x1A19, 0x1A20, 0x1A21, 0x1A60, 0x1A61, 0x1A75, 0x1A7D, // 280
0x1A7F, 0x1A80, 0x1AB0, 0x1ABE, 0x1B05, 0x1B06, 0x1B34, 0x1B36, // 288
0x1B44, 0x1B45, 0x1B6B, 0x1B74, 0x1B83, 0x1B84, 0x1BAA, 0x1BAC, // 296
0x1BC0, 0x1BC1, 0x1BE6, 0x1BE7, 0x1BF2, 0x1BF4, 0x1C00, 0x1C01, // 304
0x1C37, 0x1C38, 0x1C5A, 0x1C5B, 0x1CD0, 0x1CD3, 0x1CD4, 0x1CE1, // 312
0x1CE2, 0x1CE9, 0x1CED, 0x1CEE, 0x1CF4, 0x1CF5, 0x1CF8, 0x1CFA, // 320
0x1DC0, 0x1DF6, 0x1DFC, 0x1E00, 0x201C, 0x201D, 0x20AC, 0x20AD, // 328
0x20D0, 0x20DD, 0x20E1, 0x20E2, 0x20E5, 0x20F1, 0x263A, 0x263B, // 336
0x2C00, 0x2C01, 0x2CEF, 0x2CF2, 0x2D5E, 0x2D5F, 0x2D7F, 0x2D80, // 344
0x2DE0, 0x2E00, 0x302A, 0x3030, 0x304B, 0x304C, 0x3099, 0x309B, // 352
0x30AB, 0x30AC, 0x3105, 0x3106, 0x5B57, 0x5B58, 0xA288, 0xA289, // 360
0xA4E8, 0xA4E9, 0xA549, 0xA54A, 0xA66F, 0xA670, 0xA674, 0xA67E, // 368
0xA69E, 0xA6A1, 0xA6F0, 0xA6F2, 0xA800, 0xA801, 0xA806, 0xA807, // 376
0xA840, 0xA841, 0xA882, 0xA883, 0xA8C4, 0xA8C5, 0xA8E0, 0xA8F2, // 384
0xA90A, 0xA90B, 0xA92B, 0xA92E, 0xA930, 0xA931, 0xA953, 0xA954, // 392
0xA984, 0xA985, 0xA9B3, 0xA9B4, 0xA9C0, 0xA9C1, 0xAA00, 0xAA01, // 400
0xAA80, 0xAAB1, 0xAAB2, 0xAAB5, 0xAAB7, 0xAAB9, 0xAABE, 0xAAC0, // 408
0xAAC1, 0xAAC2, 0xAAF6, 0xAAF7, 0xABC0, 0xABC1, 0xABED, 0xABEE, // 416
0xAC00, 0xAC01, 0xD800, 0xD807, 0xD808, 0xD809, 0xD80C, 0xD80D, // 424
0xD811, 0xD812, 0xD81A, 0xD81C, 0xD82F, 0xD830, 0xD834, 0xD835, // 432
0xD83A, 0xD83B, 0xDC00, 0xE000, 0xFB1E, 0xFB1F, 0xFDD0, 0xFDD1, // 440
0xFE20, 0xFE30, 0x0001, 0x0000, 0x0001, 0x0001, 0x0001, 0x01FD, // 448
0x0001, 0x01FE, 0x0001, 0x0280, 0x0001, 0x0281, 0x0001, 0x02B7, // 456
0x0001, 0x02B8, 0x0001, 0x02E0, 0x0001, 0x02E1, 0x0001, 0x0308, // 464
0x0001, 0x0309, 0x0001, 0x0330, 0x0001, 0x0331, 0x0001, 0x036B, // 472
0x0001, 0x036C, 0x0001, 0x0376, 0x0001, 0x037B, 0x0001, 0x0380, // 480
0x0001, 0x0381, 0x0001, 0x03A0, 0x0001, 0x03A1, 0x0001, 0x0414, // 488
0x0001, 0x0415, 0x0001, 0x0450, 0x0001, 0x0451, 0x0001, 0x0480, // 496
0x0001, 0x0481, 0x0001, 0x0500, 0x0001, 0x0501, 0x0001, 0x0537, // 504
0x0001, 0x0538, 0x0001, 0x0647, 0x0001, 0x0648, 0x0001, 0x0800, // 512
0x0001, 0x0801, 0x0001, 0x0840, 0x0001, 0x0841, 0x0001, 0x0873, // 520
0x0001, 0x0874, 0x0001, 0x0896, 0x0001, 0x0897, 0x0001, 0x08F4, // 528
0x0001, 0x08F5, 0x0001, 0x0900, 0x0001, 0x0901, 0x0001, 0x0920, // 536
0x0001, 0x0921, 0x0001, 0x0980, 0x0001, 0x0981, 0x0001, 0x09A0, // 544
0x0001, 0x09A1, 0x0001, 0x0A00, 0x0001, 0x0A01, 0x0001, 0x0A0D, // 552
0x0001, 0x0A0E, 0x0001, 0x0A0F, 0x0001, 0x0A10, 0x0001, 0x0A38, // 560
0x0001, 0x0A3B, 0x0001, 0x0A3F, 0x0001, 0x0A40, 0x0001, 0x0A60, // 568
0x0001, 0x0A61, 0x0001, 0x0A95, 0x0001, 0x0A96, 0x0001, 0x0AC1, // 576
0x0001, 0x0AC2, 0x0001, 0x0AE5, 0x0001, 0x0AE7, 0x0001, 0x0B00, // 584
0x0001, 0x0B01, 0x0001, 0x0B40, 0x0001, 0x0B41, 0x0001, 0x0B60, // 592
0x0001, 0x0B61, 0x0001, 0x0B8F, 0x0001, 0x0B90, 0x0001, 0x0C00, // 600
0x0001, 0x0C01, 0x0001, 0x0CA1, 0x0001, 0x0CA2, 0x0001, 0x1005, // 608
0x0001, 0x1006, 0x0001, 0x1046, 0x0001, 0x1047, 0x0001, 0x107F, // 616
0x0001, 0x1080, 0x0001, 0x1083, 0x0001, 0x1084, 0x0001, 0x10B9, // 624
0x0001, 0x10BB, 0x0001, 0x10D0, 0x0001, 0x10D1, 0x0001, 0x1100, // 632
0x0001, 0x1104, 0x0001, 0x1127, 0x0001, 0x1128, 0x0001, 0x1133, // 640
0x0001, 0x1135, 0x0001, 0x1152, 0x0001, 0x1153, 0x0001, 0x1173, // 648
0x0001, 0x1174, 0x0001, 0x1183, 0x0001, 0x1184, 0x0001, 0x11C0, // 656
0x0001, 0x11C1, 0x0001, 0x11CA, 0x0001, 0x11CB, 0x0001, 0x1208, // 664
0x0001, 0x1209, 0x0001, 0x1235, 0x0001, 0x1237, 0x0001, 0x128F, // 672
0x0001, 0x1290, 0x0001, 0x12BE, 0x0001, 0x12BF, 0x0001, 0x12E9, // 680
0x0001, 0x12EB, 0x0001, 0x1315, 0x0001, 0x1316, 0x0001, 0x133C, // 688
0x0001, 0x133D, 0x0001, 0x133E, 0x0001, 0x133F, 0x0001, 0x134D, // 696
0x0001, 0x134E, 0x0001, 0x1357, 0x0001, 0x1358, 0x0001, 0x1366, // 704
0x0001, 0x136D, 0x0001, 0x1370, 0x0001, 0x1375, 0x0001, 0x1484, // 712
0x0001, 0x1485, 0x0001, 0x14B0, 0x0001, 0x14B1, 0x0001, 0x14BA, // 720
0x0001, 0x14BB, 0x0001, 0x14BD, 0x0001, 0x14BE, 0x0001, 0x14C2, // 728
0x0001, 0x14C4, 0x0001, 0x158E, 0x0001, 0x158F, 0x0001, 0x15AF, // 736
0x0001, 0x15B0, 0x0001, 0x15BF, 0x0001, 0x15C1, 0x0001, 0x160E, // 744
0x0001, 0x160F, 0x0001, 0x163F, 0x0001, 0x1640, 0x0001, 0x1680, // 752
0x0001, 0x1681, 0x0001, 0x16B6, 0x0001, 0x16B8, 0x0001, 0x1717, // 760
0x0001, 0x1718, 0x0001, 0x172B, 0x0001, 0x172C, 0x0001, 0x18B4, // 768
0x0001, 0x18B5, 0x0001, 0x1AC0, 0x0001, 0x1AC1, 0x0001, 0x2000, // 776
0x0001, 0x2001, 0x0001, 0x3153, 0x0001, 0x3154, 0x0001, 0x4400, // 784
0x0001, 0x4401, 0x0001, 0x6A4F, 0x0001, 0x6A50, 0x0001, 0x6AE6, // 792
0x0001, 0x6AE7, 0x0001, 0x6AF0, 0x0001, 0x6AF5, 0x0001, 0x6B1C, // 800
0x0001, 0x6B1D, 0x0001, 0x6B30, 0x0001, 0x6B37, 0x0001, 0x6F00, // 808
0x0001, 0x6F01, 0x0001, 0xBC20, 0x0001, 0xBC21, 0x0001, 0xBC9E, // 816
0x0001, 0xBC9F, 0x0001, 0xD165, 0x0001, 0xD16A, 0x0001, 0xD16D, // 824
0x0001, 0xD173, 0x0001, 0xD17B, 0x0001, 0xD183, 0x0001, 0xD185, // 832
0x0001, 0xD18C, 0x0001, 0xD1AA, 0x0001, 0xD1AE, 0x0001, 0xD242, // 840
0x0001, 0xD245, 0x0001, 0xE802, 0x0001, 0xE803, 0x0001, 0xE8D0, // 848
0x0001, 0xE8D7};
#endif

75
deps/icu-small/source/i18n/compactdecimalformat.cpp vendored Normal file
View File

@ -0,0 +1,75 @@
// © 2018 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html
#include "unicode/utypes.h"
#if !UCONFIG_NO_FORMATTING
// Allow implicit conversion from char16_t* to UnicodeString for this file:
// Helpful in toString methods and elsewhere.
#define UNISTR_FROM_STRING_EXPLICIT
#include "unicode/compactdecimalformat.h"
#include "number_mapper.h"
#include "number_decimfmtprops.h"
using namespace icu;
UOBJECT_DEFINE_RTTI_IMPLEMENTATION(CompactDecimalFormat)
CompactDecimalFormat*
CompactDecimalFormat::createInstance(const Locale& inLocale, UNumberCompactStyle style,
UErrorCode& status) {
return new CompactDecimalFormat(inLocale, style, status);
}
CompactDecimalFormat::CompactDecimalFormat(const Locale& inLocale, UNumberCompactStyle style,
UErrorCode& status)
: DecimalFormat(new DecimalFormatSymbols(inLocale, status), status) {
if (U_FAILURE(status)) return;
// Minimal properties: let the non-shim code path do most of the logic for us.
fields->properties.compactStyle = style;
fields->properties.groupingSize = -2; // do not forward grouping information
fields->properties.minimumGroupingDigits = 2;
touch(status);
}
CompactDecimalFormat::CompactDecimalFormat(const CompactDecimalFormat& source) = default;
CompactDecimalFormat::~CompactDecimalFormat() = default;
CompactDecimalFormat& CompactDecimalFormat::operator=(const CompactDecimalFormat& rhs) {
DecimalFormat::operator=(rhs);
return *this;
}
CompactDecimalFormat* CompactDecimalFormat::clone() const {
return new CompactDecimalFormat(*this);
}
void
CompactDecimalFormat::parse(
const UnicodeString& /* text */,
Formattable& /* result */,
ParsePosition& /* parsePosition */) const {
}
void
CompactDecimalFormat::parse(
const UnicodeString& /* text */,
Formattable& /* result */,
UErrorCode& status) const {
status = U_UNSUPPORTED_ERROR;
}
CurrencyAmount*
CompactDecimalFormat::parseCurrency(
const UnicodeString& /* text */,
ParsePosition& /* pos */) const {
return nullptr;
}
#endif /* #if !UCONFIG_NO_FORMATTING */

134
deps/icu-small/source/i18n/coptccal.cpp vendored Normal file
View File

@ -0,0 +1,134 @@
// © 2016 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html
/*
*******************************************************************************
* Copyright (C) 2003 - 2013, International Business Machines Corporation and
* others. All Rights Reserved.
*******************************************************************************
*/
#include "unicode/utypes.h"
#if !UCONFIG_NO_FORMATTING
#include "gregoimp.h"
#include "umutex.h"
#include "coptccal.h"
#include "cecal.h"
#include <float.h>
U_NAMESPACE_BEGIN
UOBJECT_DEFINE_RTTI_IMPLEMENTATION(CopticCalendar)
static const int32_t COPTIC_JD_EPOCH_OFFSET = 1824665;
//-------------------------------------------------------------------------
// Constructors...
//-------------------------------------------------------------------------
CopticCalendar::CopticCalendar(const Locale& aLocale, UErrorCode& success)
: CECalendar(aLocale, success)
{
}
CopticCalendar::CopticCalendar (const CopticCalendar& other)
: CECalendar(other)
{
}
CopticCalendar::~CopticCalendar()
{
}
CopticCalendar*
CopticCalendar::clone() const
{
return new CopticCalendar(*this);
}
const char*
CopticCalendar::getType() const
{
return "coptic";
}
//-------------------------------------------------------------------------
// Calendar framework
//-------------------------------------------------------------------------
int32_t
CopticCalendar::handleGetExtendedYear(UErrorCode& status)
{
if (U_FAILURE(status)) {
return 0;
}
if (newerField(UCAL_EXTENDED_YEAR, UCAL_YEAR) == UCAL_EXTENDED_YEAR) {
return internalGet(UCAL_EXTENDED_YEAR, 1); // Default to year 1
}
// The year defaults to the epoch start, the era to CE
int32_t era = internalGet(UCAL_ERA, CE);
if (era == BCE) {
return 1 - internalGet(UCAL_YEAR, 1); // Convert to extended year
}
if (era == CE){
return internalGet(UCAL_YEAR, 1); // Default to year 1
}
status = U_ILLEGAL_ARGUMENT_ERROR;
return 0;
}
void
CopticCalendar::handleComputeFields(int32_t julianDay, UErrorCode& status)
{
int32_t eyear, month, day, era, year;
jdToCE(julianDay, getJDEpochOffset(), eyear, month, day, status);
if (U_FAILURE(status)) return;
if (eyear <= 0) {
era = BCE;
year = 1 - eyear;
} else {
era = CE;
year = eyear;
}
internalSet(UCAL_EXTENDED_YEAR, eyear);
internalSet(UCAL_ERA, era);
internalSet(UCAL_YEAR, year);
internalSet(UCAL_MONTH, month);
internalSet(UCAL_ORDINAL_MONTH, month);
internalSet(UCAL_DATE, day);
internalSet(UCAL_DAY_OF_YEAR, (30 * month) + day);
}
constexpr uint32_t kCopticRelatedYearDiff = 284;
int32_t CopticCalendar::getRelatedYear(UErrorCode &status) const
{
int32_t year = get(UCAL_EXTENDED_YEAR, status);
if (U_FAILURE(status)) {
return 0;
}
return year + kCopticRelatedYearDiff;
}
void CopticCalendar::setRelatedYear(int32_t year)
{
// set extended year
set(UCAL_EXTENDED_YEAR, year - kCopticRelatedYearDiff);
}
IMPL_SYSTEM_DEFAULT_CENTURY(CopticCalendar, "@calendar=coptic")
int32_t
CopticCalendar::getJDEpochOffset() const
{
return COPTIC_JD_EPOCH_OFFSET;
}
U_NAMESPACE_END
#endif /* #if !UCONFIG_NO_FORMATTING */
//eof

247
deps/icu-small/source/i18n/coptccal.h vendored Normal file
View File

@ -0,0 +1,247 @@
// © 2016 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html
/*
*******************************************************************************
* Copyright (C) 2003 - 2013, International Business Machines Corporation and *
* others. All Rights Reserved. *
*******************************************************************************
*/
#ifndef COPTCCAL_H
#define COPTCCAL_H
#include "unicode/utypes.h"
#if !UCONFIG_NO_FORMATTING
#include "unicode/calendar.h"
#include "cecal.h"
U_NAMESPACE_BEGIN
/**
* Implement the Coptic calendar system.
* @internal
*/
class CopticCalendar : public CECalendar {
public:
/**
* Useful constants for CopticCalendar.
* @internal
*/
enum EMonths {
/**
* Constant for &#x03c9;&#x03bf;&#x03b3;&#x03c4;/&#x062a;&#xfeee;&#xfe97;,
* the 1st month of the Coptic year.
*/
TOUT,
/**
* Constant for &#x03a0;&#x03b1;&#x03bf;&#x03c0;&#x03b9;/&#xfeea;&#xfe91;&#xfe8e;&#xfe91;,
* the 2nd month of the Coptic year.
*/
BABA,
/**
* Constant for &#x0391;&#x03b8;&#x03bf;&#x03c1;/&#x0631;&#xfeee;&#xfe97;&#xfe8e;&#xfeeb;,
* the 3rd month of the Coptic year.
*/
HATOR,
/**
* Constant for &#x03a7;&#x03bf;&#x03b9;&#x03b1;&#x03ba;/&#xfeda;&#xfeec;&#xfef4;&#xfedb;,
* the 4th month of the Coptic year.
*/
KIAHK,
/**
* Constant for &#x03a4;&#x03c9;&#x03b2;&#x03b9;/&#x0637;&#xfeee;&#xfe92;&#xfeeb;,
* the 5th month of the Coptic year.
*/
TOBA,
/**
* Constant for &#x039c;&#x03b5;&#x03e3;&#x03b9;&#x03c1;/&#xfeae;&#xfef4;&#xfeb8;&#xfee3;&#x0623;,
* the 6th month of the Coptic year.
*/
AMSHIR,
/**
* Constant for &#x03a0;&#x03b1;&#x03c1;&#x03b5;&#x03bc;&#x03e9;&#x03b1;&#x03c4;/&#x062a;&#xfe8e;&#xfeec;&#xfee3;&#xfeae;&#xfe91;,
* the 7th month of the Coptic year.
*/
BARAMHAT,
/**
* Constant for &#x03a6;&#x03b1;&#x03c1;&#x03bc;&#x03bf;&#x03b8;&#x03b9;/&#x0647;&#x062f;&#xfeee;&#xfee3;&#xfeae;&#xfe91;,
* the 8th month of the Coptic year.
*/
BARAMOUDA,
/**
* Constant for &#x03a0;&#x03b1;&#x03e3;&#x03b1;&#x03bd;/&#xfeb2;&#xfee8;&#xfeb8;&#xfe91;,
* the 9th month of the Coptic year.
*/
BASHANS,
/**
* Constant for &#x03a0;&#x03b1;&#x03c9;&#x03bd;&#x03b9;/&#xfeea;&#xfee7;&#x0624;&#xfeee;&#xfe91;,
* the 10th month of the Coptic year.
*/
PAONA,
/**
* Constant for &#x0395;&#x03c0;&#x03b7;&#x03c0;/&#xfe90;&#xfef4;&#xfe91;&#x0623;,
* the 11th month of the Coptic year.
*/
EPEP,
/**
* Constant for &#x039c;&#x03b5;&#x03f2;&#x03c9;&#x03c1;&#x03b7;/&#x0649;&#xfeae;&#xfeb4;&#xfee3;,
* the 12th month of the Coptic year.
*/
MESRA,
/**
* Constant for &#x03a0;&#x03b9;&#x03ba;&#x03bf;&#x03b3;&#x03eb;&#x03b9;
* &#x03bc;&#x03b1;&#x03b2;&#x03bf;&#x03c4;/&#xfeae;&#xfef4;&#xfed0;&#xfebc;&#xfedf;&#x0627;
* &#xfeae;&#xfeec;&#xfeb8;&#xfedf;&#x0627;,
* the 13th month of the Coptic year.
*/
NASIE
};
enum EEras {
BCE, // Before the epoch
CE // After the epoch
};
/**
* Constructs a CopticCalendar based on the current time in the default time zone
* with the given locale.
*
* @param aLocale The given locale.
* @param success Indicates the status of CopticCalendar object construction.
* Returns U_ZERO_ERROR if constructed successfully.
* @internal
*/
CopticCalendar(const Locale& aLocale, UErrorCode& success);
/**
* Copy Constructor
* @internal
*/
CopticCalendar (const CopticCalendar& other);
/**
* Destructor.
* @internal
*/
virtual ~CopticCalendar();
/**
* Create and return a polymorphic copy of this calendar.
* @return return a polymorphic copy of this calendar.
* @internal
*/
virtual CopticCalendar* clone() const override;
/**
* return the calendar type, "coptic"
* @return calendar type
* @internal
*/
const char * getType() const override;
/**
* @return The related Gregorian year; will be obtained by modifying the value
* obtained by get from UCAL_EXTENDED_YEAR field
* @internal
*/
virtual int32_t getRelatedYear(UErrorCode &status) const override;
/**
* @param year The related Gregorian year to set; will be modified as necessary then
* set in UCAL_EXTENDED_YEAR field
* @internal
*/
virtual void setRelatedYear(int32_t year) override;
protected:
//-------------------------------------------------------------------------
// Calendar framework
//-------------------------------------------------------------------------
/**
* Return the extended year defined by the current fields.
* @internal
*/
virtual int32_t handleGetExtendedYear(UErrorCode& status) override;
/**
* Compute fields from the JD
* @internal
*/
virtual void handleComputeFields(int32_t julianDay, UErrorCode &status) override;
DECLARE_OVERRIDE_SYSTEM_DEFAULT_CENTURY
/**
* Return the date offset from Julian
* @internal
*/
virtual int32_t getJDEpochOffset() const override;
virtual bool isEra0CountingBackward() const override { return true; }
public:
/**
* Override Calendar Returns a unique class ID POLYMORPHICALLY. Pure virtual
* override. This method is to implement a simple version of RTTI, since not all C++
* compilers support genuine RTTI. Polymorphic operator==() and clone() methods call
* this method.
*
* @return The class ID for this object. All objects of a given class have the
* same class ID. Objects of other classes have different class IDs.
* @internal
*/
virtual UClassID getDynamicClassID() const override;
/**
* Return the class ID for this class. This is useful only for comparing to a return
* value from getDynamicClassID(). For example:
*
* Base* polymorphic_pointer = createPolymorphicObject();
* if (polymorphic_pointer->getDynamicClassID() ==
* Derived::getStaticClassID()) ...
*
* @return The class ID for all objects of this class.
* @internal
*/
U_I18N_API static UClassID U_EXPORT2 getStaticClassID();
#if 0
// We do not want to introduce this API in ICU4C.
// It was accidentally introduced in ICU4J as a public API.
public:
//-------------------------------------------------------------------------
// Calendar system Conversion methods...
//-------------------------------------------------------------------------
/**
* Convert an Coptic year, month, and day to a Julian day.
*
* @param year the extended year
* @param month the month
* @param day the day
* @return Julian day
* @internal
*/
static int32_t copticToJD(int32_t year, int32_t month, int32_t day);
#endif
};
U_NAMESPACE_END
#endif /* #if !UCONFIG_NO_FORMATTING */
#endif /* COPTCCAL_H */
//eof

617
deps/icu-small/source/i18n/cpdtrans.cpp vendored Normal file
View File

@ -0,0 +1,617 @@
// © 2016 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html
/*
**********************************************************************
* Copyright (C) 1999-2011, International Business Machines
* Corporation and others. All Rights Reserved.
**********************************************************************
* Date Name Description
* 11/17/99 aliu Creation.
**********************************************************************
*/
#include "unicode/utypes.h"
#if !UCONFIG_NO_TRANSLITERATION
#include "unicode/unifilt.h"
#include "unicode/uniset.h"
#include "cpdtrans.h"
#include "uvector.h"
#include "tridpars.h"
#include "cmemory.h"
// keep in sync with Transliterator
//static const char16_t ID_SEP = 0x002D; /*-*/
static const char16_t ID_DELIM = 0x003B; /*;*/
static const char16_t NEWLINE = 10;
static const char16_t COLON_COLON[] = {0x3A, 0x3A, 0}; //"::"
U_NAMESPACE_BEGIN
const char16_t CompoundTransliterator::PASS_STRING[] = { 0x0025, 0x0050, 0x0061, 0x0073, 0x0073, 0 }; // "%Pass"
UOBJECT_DEFINE_RTTI_IMPLEMENTATION(CompoundTransliterator)
/**
* Constructs a new compound transliterator given an array of
* transliterators. The array of transliterators may be of any
* length, including zero or one, however, useful compound
* transliterators have at least two components.
* @param transliterators array of <code>Transliterator</code>
* objects
* @param transliteratorCount The number of
* <code>Transliterator</code> objects in transliterators.
* @param filter the filter. Any character for which
* <tt>filter.contains()</tt> returns <tt>false</tt> will not be
* altered by this transliterator. If <tt>filter</tt> is
* <tt>null</tt> then no filtering is applied.
*/
CompoundTransliterator::CompoundTransliterator(
Transliterator* const transliterators[],
int32_t transliteratorCount,
UnicodeFilter* adoptedFilter) :
Transliterator(joinIDs(transliterators, transliteratorCount), adoptedFilter),
trans(nullptr), count(0), numAnonymousRBTs(0) {
setTransliterators(transliterators, transliteratorCount);
}
/**
* Splits an ID of the form "ID;ID;..." into a compound using each
* of the IDs.
* @param id of above form
* @param forward if false, does the list in reverse order, and
* takes the inverse of each ID.
*/
CompoundTransliterator::CompoundTransliterator(const UnicodeString& id,
UTransDirection direction,
UnicodeFilter* adoptedFilter,
UParseError& /*parseError*/,
UErrorCode& status) :
Transliterator(id, adoptedFilter),
trans(nullptr), numAnonymousRBTs(0) {
// TODO add code for parseError...currently unused, but
// later may be used by parsing code...
init(id, direction, true, status);
}
CompoundTransliterator::CompoundTransliterator(const UnicodeString& id,
UParseError& /*parseError*/,
UErrorCode& status) :
Transliterator(id, nullptr), // set filter to 0 here!
trans(nullptr), numAnonymousRBTs(0) {
// TODO add code for parseError...currently unused, but
// later may be used by parsing code...
init(id, UTRANS_FORWARD, true, status);
}
/**
* Private constructor for use of TransliteratorAlias
*/
CompoundTransliterator::CompoundTransliterator(const UnicodeString& newID,
UVector& list,
UnicodeFilter* adoptedFilter,
int32_t anonymousRBTs,
UParseError& /*parseError*/,
UErrorCode& status) :
Transliterator(newID, adoptedFilter),
trans(nullptr), numAnonymousRBTs(anonymousRBTs)
{
init(list, UTRANS_FORWARD, false, status);
}
/**
* Private constructor for Transliterator from a vector of
* transliterators. The caller is responsible for fixing up the
* ID.
*/
CompoundTransliterator::CompoundTransliterator(UVector& list,
UParseError& /*parseError*/,
UErrorCode& status) :
Transliterator(UnicodeString(), nullptr),
trans(nullptr), numAnonymousRBTs(0)
{
// TODO add code for parseError...currently unused, but
// later may be used by parsing code...
init(list, UTRANS_FORWARD, false, status);
// assume caller will fixup ID
}
CompoundTransliterator::CompoundTransliterator(UVector& list,
int32_t anonymousRBTs,
UParseError& /*parseError*/,
UErrorCode& status) :
Transliterator(UnicodeString(), nullptr),
trans(nullptr), numAnonymousRBTs(anonymousRBTs)
{
init(list, UTRANS_FORWARD, false, status);
}
/**
* Finish constructing a transliterator: only to be called by
* constructors. Before calling init(), set trans and filter to nullptr.
* @param id the id containing ';'-separated entries
* @param direction either FORWARD or REVERSE
* @param idSplitPoint the index into id at which the
* adoptedSplitTransliterator should be inserted, if there is one, or
* -1 if there is none.
* @param adoptedSplitTransliterator a transliterator to be inserted
* before the entry at offset idSplitPoint in the id string. May be
* nullptr to insert no entry.
* @param fixReverseID if true, then reconstruct the ID of reverse
* entries by calling getID() of component entries. Some constructors
* do not require this because they apply a facade ID anyway.
* @param status the error code indicating success or failure
*/
void CompoundTransliterator::init(const UnicodeString& id,
UTransDirection direction,
UBool fixReverseID,
UErrorCode& status) {
// assert(trans == 0);
if (U_FAILURE(status)) {
return;
}
UVector list(status);
UnicodeSet* compoundFilter = nullptr;
UnicodeString regenID;
if (!TransliteratorIDParser::parseCompoundID(id, direction,
regenID, list, compoundFilter)) {
status = U_INVALID_ID;
delete compoundFilter;
return;
}
TransliteratorIDParser::instantiateList(list, status);
init(list, direction, fixReverseID, status);
if (compoundFilter != nullptr) {
adoptFilter(compoundFilter);
}
}
/**
* Finish constructing a transliterator: only to be called by
* constructors. Before calling init(), set trans and filter to nullptr.
* @param list a vector of transliterator objects to be adopted. It
* should NOT be empty. The list should be in declared order. That
* is, it should be in the FORWARD order; if direction is REVERSE then
* the list order will be reversed.
* @param direction either FORWARD or REVERSE
* @param fixReverseID if true, then reconstruct the ID of reverse
* entries by calling getID() of component entries. Some constructors
* do not require this because they apply a facade ID anyway.
* @param status the error code indicating success or failure
*/
void CompoundTransliterator::init(UVector& list,
UTransDirection direction,
UBool fixReverseID,
UErrorCode& status) {
// assert(trans == 0);
// Allocate array
if (U_SUCCESS(status)) {
count = list.size();
trans = static_cast<Transliterator**>(uprv_malloc(count * sizeof(Transliterator*)));
/* test for nullptr */
if (trans == nullptr) {
status = U_MEMORY_ALLOCATION_ERROR;
return;
}
}
if (U_FAILURE(status) || trans == nullptr) {
// assert(trans == 0);
return;
}
// Move the transliterators from the vector into an array.
// Reverse the order if necessary.
int32_t i;
for (i=0; i<count; ++i) {
int32_t j = (direction == UTRANS_FORWARD) ? i : count - 1 - i;
trans[i] = static_cast<Transliterator*>(list.elementAt(j));
}
// If the direction is UTRANS_REVERSE then we may need to fix the
// ID.
if (direction == UTRANS_REVERSE && fixReverseID) {
UnicodeString newID;
for (i=0; i<count; ++i) {
if (i > 0) {
newID.append(ID_DELIM);
}
newID.append(trans[i]->getID());
}
setID(newID);
}
computeMaximumContextLength();
}
/**
* Return the IDs of the given list of transliterators, concatenated
* with ID_DELIM delimiting them. Equivalent to the perlish expression
* join(ID_DELIM, map($_.getID(), transliterators).
*/
UnicodeString CompoundTransliterator::joinIDs(Transliterator* const transliterators[],
int32_t transCount) {
UnicodeString id;
for (int32_t i=0; i<transCount; ++i) {
if (i > 0) {
id.append(ID_DELIM);
}
id.append(transliterators[i]->getID());
}
return id; // Return temporary
}
/**
* Copy constructor.
*/
CompoundTransliterator::CompoundTransliterator(const CompoundTransliterator& t) :
Transliterator(t), trans(nullptr), count(0), numAnonymousRBTs(-1) {
*this = t;
}
/**
* Destructor
*/
CompoundTransliterator::~CompoundTransliterator() {
freeTransliterators();
}
void CompoundTransliterator::freeTransliterators() {
if (trans != nullptr) {
for (int32_t i=0; i<count; ++i) {
delete trans[i];
}
uprv_free(trans);
}
trans = nullptr;
count = 0;
}
/**
* Assignment operator.
*/
CompoundTransliterator& CompoundTransliterator::operator=(
const CompoundTransliterator& t)
{
if (this == &t) { return *this; } // self-assignment: no-op
Transliterator::operator=(t);
int32_t i = 0;
UBool failed = false;
if (trans != nullptr) {
for (i=0; i<count; ++i) {
delete trans[i];
trans[i] = nullptr;
}
}
if (t.count > count) {
if (trans != nullptr) {
uprv_free(trans);
}
trans = static_cast<Transliterator**>(uprv_malloc(t.count * sizeof(Transliterator*)));
}
count = t.count;
if (trans != nullptr) {
for (i=0; i<count; ++i) {
trans[i] = t.trans[i]->clone();
if (trans[i] == nullptr) {
failed = true;
break;
}
}
}
// if memory allocation failed delete backwards trans array
if (failed && i > 0) {
int32_t n;
for (n = i-1; n >= 0; n--) {
uprv_free(trans[n]);
trans[n] = nullptr;
}
}
numAnonymousRBTs = t.numAnonymousRBTs;
return *this;
}
/**
* Transliterator API.
*/
CompoundTransliterator* CompoundTransliterator::clone() const {
return new CompoundTransliterator(*this);
}
/**
* Returns the number of transliterators in this chain.
* @return number of transliterators in this chain.
*/
int32_t CompoundTransliterator::getCount() const {
return count;
}
/**
* Returns the transliterator at the given index in this chain.
* @param index index into chain, from 0 to <code>getCount() - 1</code>
* @return transliterator at the given index
*/
const Transliterator& CompoundTransliterator::getTransliterator(int32_t index) const {
return *trans[index];
}
void CompoundTransliterator::setTransliterators(Transliterator* const transliterators[],
int32_t transCount) {
Transliterator** a = static_cast<Transliterator**>(uprv_malloc(transCount * sizeof(Transliterator*)));
if (a == nullptr) {
return;
}
int32_t i = 0;
UBool failed = false;
for (i=0; i<transCount; ++i) {
a[i] = transliterators[i]->clone();
if (a[i] == nullptr) {
failed = true;
break;
}
}
if (failed && i > 0) {
int32_t n;
for (n = i-1; n >= 0; n--) {
uprv_free(a[n]);
a[n] = nullptr;
}
return;
}
adoptTransliterators(a, transCount);
}
void CompoundTransliterator::adoptTransliterators(Transliterator* adoptedTransliterators[],
int32_t transCount) {
// First free trans[] and set count to zero. Once this is done,
// orphan the filter. Set up the new trans[].
freeTransliterators();
trans = adoptedTransliterators;
count = transCount;
computeMaximumContextLength();
setID(joinIDs(trans, count));
}
/**
* Append c to buf, unless buf is empty or buf already ends in c.
*/
static void _smartAppend(UnicodeString& buf, char16_t c) {
if (buf.length() != 0 &&
buf.charAt(buf.length() - 1) != c) {
buf.append(c);
}
}
UnicodeString& CompoundTransliterator::toRules(UnicodeString& rulesSource,
UBool escapeUnprintable) const {
// We do NOT call toRules() on our component transliterators, in
// general. If we have several rule-based transliterators, this
// yields a concatenation of the rules -- not what we want. We do
// handle compound RBT transliterators specially -- those for which
// compoundRBTIndex >= 0. For the transliterator at compoundRBTIndex,
// we do call toRules() recursively.
rulesSource.truncate(0);
if (numAnonymousRBTs >= 1 && getFilter() != nullptr) {
// If we are a compound RBT and if we have a global
// filter, then emit it at the top.
UnicodeString pat;
rulesSource.append(COLON_COLON, 2).append(getFilter()->toPattern(pat, escapeUnprintable)).append(ID_DELIM);
}
for (int32_t i=0; i<count; ++i) {
UnicodeString rule;
// Anonymous RuleBasedTransliterators (inline rules and
// ::BEGIN/::END blocks) are given IDs that begin with
// "%Pass": use toRules() to write all the rules to the output
// (and insert "::Null;" if we have two in a row)
if (trans[i]->getID().startsWith(PASS_STRING, 5)) {
trans[i]->toRules(rule, escapeUnprintable);
if (numAnonymousRBTs > 1 && i > 0 && trans[i - 1]->getID().startsWith(PASS_STRING, 5))
rule = UNICODE_STRING_SIMPLE("::Null;") + rule;
// we also use toRules() on CompoundTransliterators (which we
// check for by looking for a semicolon in the ID)-- this gets
// the list of their child transliterators output in the right
// format
} else if (trans[i]->getID().indexOf(ID_DELIM) >= 0) {
trans[i]->toRules(rule, escapeUnprintable);
// for everything else, use Transliterator::toRules()
} else {
trans[i]->Transliterator::toRules(rule, escapeUnprintable);
}
_smartAppend(rulesSource, NEWLINE);
rulesSource.append(rule);
_smartAppend(rulesSource, ID_DELIM);
}
return rulesSource;
}
/**
* Implement Transliterator framework
*/
void CompoundTransliterator::handleGetSourceSet(UnicodeSet& result) const {
UnicodeSet set;
result.clear();
for (int32_t i=0; i<count; ++i) {
result.addAll(trans[i]->getSourceSet(set));
// Take the example of Hiragana-Latin. This is really
// Hiragana-Katakana; Katakana-Latin. The source set of
// these two is roughly [:Hiragana:] and [:Katakana:].
// But the source set for the entire transliterator is
// actually [:Hiragana:] ONLY -- that is, the first
// non-empty source set.
// This is a heuristic, and not 100% reliable.
if (!result.isEmpty()) {
break;
}
}
}
/**
* Override Transliterator framework
*/
UnicodeSet& CompoundTransliterator::getTargetSet(UnicodeSet& result) const {
UnicodeSet set;
result.clear();
for (int32_t i=0; i<count; ++i) {
// This is a heuristic, and not 100% reliable.
result.addAll(trans[i]->getTargetSet(set));
}
return result;
}
/**
* Implements {@link Transliterator#handleTransliterate}.
*/
void CompoundTransliterator::handleTransliterate(Replaceable& text, UTransPosition& index,
UBool incremental) const {
/* Call each transliterator with the same contextStart and
* start, but with the limit as modified
* by preceding transliterators. The start index must be
* reset for each transliterator to give each a chance to
* transliterate the text. The initial contextStart index is known
* to still point to the same place after each transliterator
* is called because each transliterator will not change the
* text between contextStart and the initial start index.
*
* IMPORTANT: After the first transliterator, each subsequent
* transliterator only gets to transliterate text committed by
* preceding transliterators; that is, the start (output
* value) of transliterator i becomes the limit (input value)
* of transliterator i+1. Finally, the overall limit is fixed
* up before we return.
*
* Assumptions we make here:
* (1) contextStart <= start <= limit <= contextLimit <= text.length()
* (2) start <= start' <= limit' ;cursor doesn't move back
* (3) start <= limit' ;text before cursor unchanged
* - start' is the value of start after calling handleKT
* - limit' is the value of limit after calling handleKT
*/
/**
* Example: 3 transliterators. This example illustrates the
* mechanics we need to implement. C, S, and L are the contextStart,
* start, and limit. gl is the globalLimit. contextLimit is
* equal to limit throughout.
*
* 1. h-u, changes hex to Unicode
*
* 4 7 a d 0 4 7 a
* abc/u0061/u => abca/u
* C S L C S L gl=f->a
*
* 2. upup, changes "x" to "XX"
*
* 4 7 a 4 7 a
* abca/u => abcAA/u
* C SL C S
* L gl=a->b
* 3. u-h, changes Unicode to hex
*
* 4 7 a 4 7 a d 0 3
* abcAA/u => abc/u0041/u0041/u
* C S L C S
* L gl=b->15
* 4. return
*
* 4 7 a d 0 3
* abc/u0041/u0041/u
* C S L
*/
if (count < 1) {
index.start = index.limit;
return; // Short circuit for empty compound transliterators
}
// compoundLimit is the limit value for the entire compound
// operation. We overwrite index.limit with the previous
// index.start. After each transliteration, we update
// compoundLimit for insertions or deletions that have happened.
int32_t compoundLimit = index.limit;
// compoundStart is the start for the entire compound
// operation.
int32_t compoundStart = index.start;
int32_t delta = 0; // delta in length
// Give each transliterator a crack at the run of characters.
// See comments at the top of the method for more detail.
for (int32_t i=0; i<count; ++i) {
index.start = compoundStart; // Reset start
int32_t limit = index.limit;
if (index.start == index.limit) {
// Short circuit for empty range
break;
}
trans[i]->filteredTransliterate(text, index, incremental);
// In a properly written transliterator, start == limit after
// handleTransliterate() returns when incremental is false.
// Catch cases where the subclass doesn't do this, and throw
// an exception. (Just pinning start to limit is a bad idea,
// because what's probably happening is that the subclass
// isn't transliterating all the way to the end, and it should
// in non-incremental mode.)
if (!incremental && index.start != index.limit) {
// We can't throw an exception, so just fudge things
index.start = index.limit;
}
// Cumulative delta for insertions/deletions
delta += index.limit - limit;
if (incremental) {
// In the incremental case, only allow subsequent
// transliterators to modify what has already been
// completely processed by prior transliterators. In the
// non-incrmental case, allow each transliterator to
// process the entire text.
index.limit = index.start;
}
}
compoundLimit += delta;
// Start is good where it is -- where the last transliterator left
// it. Limit needs to be put back where it was, modulo
// adjustments for deletions/insertions.
index.limit = compoundLimit;
}
/**
* Sets the length of the longest context required by this transliterator.
* This is <em>preceding</em> context.
*/
void CompoundTransliterator::computeMaximumContextLength() {
int32_t max = 0;
for (int32_t i=0; i<count; ++i) {
int32_t len = trans[i]->getMaximumContextLength();
if (len > max) {
max = len;
}
}
setMaximumContextLength(max);
}
U_NAMESPACE_END
#endif /* #if !UCONFIG_NO_TRANSLITERATION */
/* eof */

232
deps/icu-small/source/i18n/cpdtrans.h vendored Normal file
View File

@ -0,0 +1,232 @@
// © 2016 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html
/*
**********************************************************************
* Copyright (C) 1999-2011, International Business Machines
* Corporation and others. All Rights Reserved.
**********************************************************************
* Date Name Description
* 11/17/99 aliu Creation.
**********************************************************************
*/
#ifndef CPDTRANS_H
#define CPDTRANS_H
#include "unicode/utypes.h"
#if !UCONFIG_NO_TRANSLITERATION
#include "unicode/translit.h"
U_NAMESPACE_BEGIN
class U_COMMON_API UVector;
class TransliteratorRegistry;
/**
* A transliterator that is composed of two or more other
* transliterator objects linked together. For example, if one
* transliterator transliterates from script A to script B, and
* another transliterates from script B to script C, the two may be
* combined to form a new transliterator from A to C.
*
* <p>Composed transliterators may not behave as expected. For
* example, inverses may not combine to form the identity
* transliterator. See the class documentation for {@link
* Transliterator} for details.
*
* @author Alan Liu
*/
class U_I18N_API CompoundTransliterator : public Transliterator {
Transliterator** trans;
int32_t count;
int32_t numAnonymousRBTs;
public:
/**
* Constructs a new compound transliterator given an array of
* transliterators. The array of transliterators may be of any
* length, including zero or one, however, useful compound
* transliterators have at least two components.
* @param transliterators array of <code>Transliterator</code>
* objects
* @param transliteratorCount The number of
* <code>Transliterator</code> objects in transliterators.
* @param adoptedFilter the filter. Any character for which
* <tt>filter.contains()</tt> returns <tt>false</tt> will not be
* altered by this transliterator. If <tt>filter</tt> is
* <tt>null</tt> then no filtering is applied.
*/
CompoundTransliterator(Transliterator* const transliterators[],
int32_t transliteratorCount,
UnicodeFilter* adoptedFilter = nullptr);
/**
* Constructs a new compound transliterator.
* @param id compound ID
* @param dir either UTRANS_FORWARD or UTRANS_REVERSE
* @param adoptedFilter a global filter for this compound transliterator
* or nullptr
*/
CompoundTransliterator(const UnicodeString& id,
UTransDirection dir,
UnicodeFilter* adoptedFilter,
UParseError& parseError,
UErrorCode& status);
/**
* Constructs a new compound transliterator in the FORWARD
* direction with a nullptr filter.
*/
CompoundTransliterator(const UnicodeString& id,
UParseError& parseError,
UErrorCode& status);
/**
* Destructor.
*/
virtual ~CompoundTransliterator();
/**
* Copy constructor.
*/
CompoundTransliterator(const CompoundTransliterator&);
/**
* Transliterator API.
*/
virtual CompoundTransliterator* clone() const override;
/**
* Returns the number of transliterators in this chain.
* @return number of transliterators in this chain.
*/
virtual int32_t getCount() const;
/**
* Returns the transliterator at the given index in this chain.
* @param idx index into chain, from 0 to <code>getCount() - 1</code>
* @return transliterator at the given index
*/
virtual const Transliterator& getTransliterator(int32_t idx) const;
/**
* Sets the transliterators.
*/
void setTransliterators(Transliterator* const transliterators[],
int32_t count);
/**
* Adopts the transliterators.
*/
void adoptTransliterators(Transliterator* adoptedTransliterators[],
int32_t count);
/**
* Override Transliterator:
* Create a rule string that can be passed to createFromRules()
* to recreate this transliterator.
* @param result the string to receive the rules. Previous
* contents will be deleted.
* @param escapeUnprintable if true then convert unprintable
* character to their hex escape representations, \uxxxx or
* \Uxxxxxxxx. Unprintable characters are those other than
* U+000A, U+0020..U+007E.
*/
virtual UnicodeString& toRules(UnicodeString& result,
UBool escapeUnprintable) const override;
protected:
/**
* Implement Transliterator framework
*/
virtual void handleGetSourceSet(UnicodeSet& result) const override;
public:
/**
* Override Transliterator framework
*/
virtual UnicodeSet& getTargetSet(UnicodeSet& result) const override;
protected:
/**
* Implements {@link Transliterator#handleTransliterate}.
*/
virtual void handleTransliterate(Replaceable& text, UTransPosition& idx,
UBool incremental) const override;
public:
/**
* ICU "poor man's RTTI", returns a UClassID for the actual class.
*/
virtual UClassID getDynamicClassID() const override;
/**
* ICU "poor man's RTTI", returns a UClassID for this class.
*/
static UClassID U_EXPORT2 getStaticClassID();
/* @internal */
static const char16_t PASS_STRING[];
private:
friend class Transliterator;
friend class TransliteratorAlias; // to access private ct
/**
* Assignment operator.
*/
CompoundTransliterator& operator=(const CompoundTransliterator&);
/**
* Private constructor for Transliterator.
*/
CompoundTransliterator(const UnicodeString& ID,
UVector& list,
UnicodeFilter* adoptedFilter,
int32_t numAnonymousRBTs,
UParseError& parseError,
UErrorCode& status);
CompoundTransliterator(UVector& list,
UParseError& parseError,
UErrorCode& status);
CompoundTransliterator(UVector& list,
int32_t anonymousRBTs,
UParseError& parseError,
UErrorCode& status);
void init(const UnicodeString& id,
UTransDirection direction,
UBool fixReverseID,
UErrorCode& status);
void init(UVector& list,
UTransDirection direction,
UBool fixReverseID,
UErrorCode& status);
/**
* Return the IDs of the given list of transliterators, concatenated
* with ';' delimiting them. Equivalent to the perlish expression
* join(';', map($_.getID(), transliterators).
*/
UnicodeString joinIDs(Transliterator* const transliterators[],
int32_t transCount);
void freeTransliterators();
void computeMaximumContextLength();
};
U_NAMESPACE_END
#endif /* #if !UCONFIG_NO_TRANSLITERATION */
#endif

492
deps/icu-small/source/i18n/csdetect.cpp vendored Normal file
View File

@ -0,0 +1,492 @@
// © 2016 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html
/*
**********************************************************************
* Copyright (C) 2005-2016, International Business Machines
* Corporation and others. All Rights Reserved.
**********************************************************************
*/
#include "unicode/utypes.h"
#if !UCONFIG_NO_CONVERSION
#include "unicode/ucsdet.h"
#include "csdetect.h"
#include "csmatch.h"
#include "uenumimp.h"
#include "cmemory.h"
#include "cstring.h"
#include "umutex.h"
#include "ucln_in.h"
#include "uarrsort.h"
#include "inputext.h"
#include "csrsbcs.h"
#include "csrmbcs.h"
#include "csrutf8.h"
#include "csrucode.h"
#include "csr2022.h"
#define NEW_ARRAY(type,count) (type *) uprv_malloc((count) * sizeof(type))
#define DELETE_ARRAY(array) uprv_free((void *) (array))
U_NAMESPACE_BEGIN
struct CSRecognizerInfo : public UMemory {
CSRecognizerInfo(CharsetRecognizer *recognizer, UBool isDefaultEnabled)
: recognizer(recognizer), isDefaultEnabled(isDefaultEnabled) {}
~CSRecognizerInfo() {delete recognizer;}
CharsetRecognizer *recognizer;
UBool isDefaultEnabled;
};
U_NAMESPACE_END
static icu::CSRecognizerInfo **fCSRecognizers = nullptr;
static icu::UInitOnce gCSRecognizersInitOnce {};
static int32_t fCSRecognizers_size = 0;
U_CDECL_BEGIN
static UBool U_CALLCONV csdet_cleanup()
{
U_NAMESPACE_USE
if (fCSRecognizers != nullptr) {
for(int32_t r = 0; r < fCSRecognizers_size; r += 1) {
delete fCSRecognizers[r];
fCSRecognizers[r] = nullptr;
}
DELETE_ARRAY(fCSRecognizers);
fCSRecognizers = nullptr;
fCSRecognizers_size = 0;
}
gCSRecognizersInitOnce.reset();
return true;
}
static int32_t U_CALLCONV
charsetMatchComparator(const void * /*context*/, const void *left, const void *right)
{
U_NAMESPACE_USE
const CharsetMatch **csm_l = (const CharsetMatch **) left;
const CharsetMatch **csm_r = (const CharsetMatch **) right;
// NOTE: compare is backwards to sort from highest to lowest.
return (*csm_r)->getConfidence() - (*csm_l)->getConfidence();
}
static void U_CALLCONV initRecognizers(UErrorCode &status) {
U_NAMESPACE_USE
ucln_i18n_registerCleanup(UCLN_I18N_CSDET, csdet_cleanup);
CSRecognizerInfo *tempArray[] = {
new CSRecognizerInfo(new CharsetRecog_UTF8(), true),
new CSRecognizerInfo(new CharsetRecog_UTF_16_BE(), true),
new CSRecognizerInfo(new CharsetRecog_UTF_16_LE(), true),
new CSRecognizerInfo(new CharsetRecog_UTF_32_BE(), true),
new CSRecognizerInfo(new CharsetRecog_UTF_32_LE(), true),
new CSRecognizerInfo(new CharsetRecog_8859_1(), true),
new CSRecognizerInfo(new CharsetRecog_8859_2(), true),
new CSRecognizerInfo(new CharsetRecog_8859_5_ru(), true),
new CSRecognizerInfo(new CharsetRecog_8859_6_ar(), true),
new CSRecognizerInfo(new CharsetRecog_8859_7_el(), true),
new CSRecognizerInfo(new CharsetRecog_8859_8_I_he(), true),
new CSRecognizerInfo(new CharsetRecog_8859_8_he(), true),
new CSRecognizerInfo(new CharsetRecog_windows_1251(), true),
new CSRecognizerInfo(new CharsetRecog_windows_1256(), true),
new CSRecognizerInfo(new CharsetRecog_KOI8_R(), true),
new CSRecognizerInfo(new CharsetRecog_8859_9_tr(), true),
new CSRecognizerInfo(new CharsetRecog_sjis(), true),
new CSRecognizerInfo(new CharsetRecog_gb_18030(), true),
new CSRecognizerInfo(new CharsetRecog_euc_jp(), true),
new CSRecognizerInfo(new CharsetRecog_euc_kr(), true),
new CSRecognizerInfo(new CharsetRecog_big5(), true),
new CSRecognizerInfo(new CharsetRecog_2022JP(), true),
#if !UCONFIG_ONLY_HTML_CONVERSION
new CSRecognizerInfo(new CharsetRecog_2022KR(), true),
new CSRecognizerInfo(new CharsetRecog_2022CN(), true),
new CSRecognizerInfo(new CharsetRecog_IBM424_he_rtl(), false),
new CSRecognizerInfo(new CharsetRecog_IBM424_he_ltr(), false),
new CSRecognizerInfo(new CharsetRecog_IBM420_ar_rtl(), false),
new CSRecognizerInfo(new CharsetRecog_IBM420_ar_ltr(), false)
#endif
};
int32_t rCount = UPRV_LENGTHOF(tempArray);
fCSRecognizers = NEW_ARRAY(CSRecognizerInfo *, rCount);
if (fCSRecognizers == nullptr) {
status = U_MEMORY_ALLOCATION_ERROR;
}
else {
fCSRecognizers_size = rCount;
for (int32_t r = 0; r < rCount; r += 1) {
fCSRecognizers[r] = tempArray[r];
if (fCSRecognizers[r] == nullptr) {
status = U_MEMORY_ALLOCATION_ERROR;
}
}
}
}
U_CDECL_END
U_NAMESPACE_BEGIN
void CharsetDetector::setRecognizers(UErrorCode &status)
{
umtx_initOnce(gCSRecognizersInitOnce, &initRecognizers, status);
}
CharsetDetector::CharsetDetector(UErrorCode &status)
: textIn(new InputText(status)), resultArray(nullptr),
resultCount(0), fStripTags(false), fFreshTextSet(false),
fEnabledRecognizers(nullptr)
{
if (U_FAILURE(status)) {
return;
}
setRecognizers(status);
if (U_FAILURE(status)) {
return;
}
resultArray = static_cast<CharsetMatch**>(uprv_malloc(sizeof(CharsetMatch*) * fCSRecognizers_size));
if (resultArray == nullptr) {
status = U_MEMORY_ALLOCATION_ERROR;
return;
}
for(int32_t i = 0; i < fCSRecognizers_size; i += 1) {
resultArray[i] = new CharsetMatch();
if (resultArray[i] == nullptr) {
status = U_MEMORY_ALLOCATION_ERROR;
break;
}
}
}
CharsetDetector::~CharsetDetector()
{
delete textIn;
for(int32_t i = 0; i < fCSRecognizers_size; i += 1) {
delete resultArray[i];
}
uprv_free(resultArray);
if (fEnabledRecognizers) {
uprv_free(fEnabledRecognizers);
}
}
void CharsetDetector::setText(const char *in, int32_t len)
{
textIn->setText(in, len);
fFreshTextSet = true;
}
UBool CharsetDetector::setStripTagsFlag(UBool flag)
{
UBool temp = fStripTags;
fStripTags = flag;
fFreshTextSet = true;
return temp;
}
UBool CharsetDetector::getStripTagsFlag() const
{
return fStripTags;
}
void CharsetDetector::setDeclaredEncoding(const char *encoding, int32_t len) const
{
textIn->setDeclaredEncoding(encoding,len);
}
int32_t CharsetDetector::getDetectableCount()
{
UErrorCode status = U_ZERO_ERROR;
setRecognizers(status);
return fCSRecognizers_size;
}
const CharsetMatch *CharsetDetector::detect(UErrorCode &status)
{
int32_t maxMatchesFound = 0;
detectAll(maxMatchesFound, status);
if(maxMatchesFound > 0) {
return resultArray[0];
} else {
return nullptr;
}
}
const CharsetMatch * const *CharsetDetector::detectAll(int32_t &maxMatchesFound, UErrorCode &status)
{
if(!textIn->isSet()) {
status = U_MISSING_RESOURCE_ERROR;// TODO: Need to set proper status code for input text not set
return nullptr;
} else if (fFreshTextSet) {
CharsetRecognizer *csr;
int32_t i;
textIn->MungeInput(fStripTags);
// Iterate over all possible charsets, remember all that
// give a match quality > 0.
resultCount = 0;
for (i = 0; i < fCSRecognizers_size; i += 1) {
csr = fCSRecognizers[i]->recognizer;
if (csr->match(textIn, resultArray[resultCount])) {
resultCount++;
}
}
if (resultCount > 1) {
uprv_sortArray(resultArray, resultCount, sizeof resultArray[0], charsetMatchComparator, nullptr, true, &status);
}
fFreshTextSet = false;
}
maxMatchesFound = resultCount;
if (maxMatchesFound == 0) {
status = U_INVALID_CHAR_FOUND;
return nullptr;
}
return resultArray;
}
void CharsetDetector::setDetectableCharset(const char *encoding, UBool enabled, UErrorCode &status)
{
if (U_FAILURE(status)) {
return;
}
int32_t modIdx = -1;
UBool isDefaultVal = false;
for (int32_t i = 0; i < fCSRecognizers_size; i++) {
CSRecognizerInfo *csrinfo = fCSRecognizers[i];
if (uprv_strcmp(csrinfo->recognizer->getName(), encoding) == 0) {
modIdx = i;
isDefaultVal = (csrinfo->isDefaultEnabled == enabled);
break;
}
}
if (modIdx < 0) {
// No matching encoding found
status = U_ILLEGAL_ARGUMENT_ERROR;
return;
}
if (fEnabledRecognizers == nullptr && !isDefaultVal) {
// Create an array storing the non default setting
fEnabledRecognizers = NEW_ARRAY(UBool, fCSRecognizers_size);
if (fEnabledRecognizers == nullptr) {
status = U_MEMORY_ALLOCATION_ERROR;
return;
}
// Initialize the array with default info
for (int32_t i = 0; i < fCSRecognizers_size; i++) {
fEnabledRecognizers[i] = fCSRecognizers[i]->isDefaultEnabled;
}
}
if (fEnabledRecognizers != nullptr) {
fEnabledRecognizers[modIdx] = enabled;
}
}
/*const char *CharsetDetector::getCharsetName(int32_t index, UErrorCode &status) const
{
if( index > fCSRecognizers_size-1 || index < 0) {
status = U_INDEX_OUTOFBOUNDS_ERROR;
return 0;
} else {
return fCSRecognizers[index]->getName();
}
}*/
U_NAMESPACE_END
U_CDECL_BEGIN
typedef struct {
int32_t currIndex;
UBool all;
UBool *enabledRecognizers;
} Context;
static void U_CALLCONV
enumClose(UEnumeration *en) {
if(en->context != nullptr) {
DELETE_ARRAY(en->context);
}
DELETE_ARRAY(en);
}
static int32_t U_CALLCONV
enumCount(UEnumeration *en, UErrorCode *) {
if (((Context *)en->context)->all) {
// ucsdet_getAllDetectableCharsets, all charset detector names
return fCSRecognizers_size;
}
// Otherwise, ucsdet_getDetectableCharsets - only enabled ones
int32_t count = 0;
UBool *enabledArray = ((Context *)en->context)->enabledRecognizers;
if (enabledArray != nullptr) {
// custom set
for (int32_t i = 0; i < fCSRecognizers_size; i++) {
if (enabledArray[i]) {
count++;
}
}
} else {
// default set
for (int32_t i = 0; i < fCSRecognizers_size; i++) {
if (fCSRecognizers[i]->isDefaultEnabled) {
count++;
}
}
}
return count;
}
static const char* U_CALLCONV
enumNext(UEnumeration *en, int32_t *resultLength, UErrorCode * /*status*/) {
const char *currName = nullptr;
if (((Context *)en->context)->currIndex < fCSRecognizers_size) {
if (((Context *)en->context)->all) {
// ucsdet_getAllDetectableCharsets, all charset detector names
currName = fCSRecognizers[((Context *)en->context)->currIndex]->recognizer->getName();
((Context *)en->context)->currIndex++;
} else {
// ucsdet_getDetectableCharsets
UBool *enabledArray = ((Context *)en->context)->enabledRecognizers;
if (enabledArray != nullptr) {
// custom set
while (currName == nullptr && ((Context *)en->context)->currIndex < fCSRecognizers_size) {
if (enabledArray[((Context *)en->context)->currIndex]) {
currName = fCSRecognizers[((Context *)en->context)->currIndex]->recognizer->getName();
}
((Context *)en->context)->currIndex++;
}
} else {
// default set
while (currName == nullptr && ((Context *)en->context)->currIndex < fCSRecognizers_size) {
if (fCSRecognizers[((Context *)en->context)->currIndex]->isDefaultEnabled) {
currName = fCSRecognizers[((Context *)en->context)->currIndex]->recognizer->getName();
}
((Context *)en->context)->currIndex++;
}
}
}
}
if(resultLength != nullptr) {
*resultLength = currName == nullptr ? 0 : (int32_t)uprv_strlen(currName);
}
return currName;
}
static void U_CALLCONV
enumReset(UEnumeration *en, UErrorCode *) {
((Context *)en->context)->currIndex = 0;
}
static const UEnumeration gCSDetEnumeration = {
nullptr,
nullptr,
enumClose,
enumCount,
uenum_unextDefault,
enumNext,
enumReset
};
U_CDECL_END
U_NAMESPACE_BEGIN
UEnumeration * CharsetDetector::getAllDetectableCharsets(UErrorCode &status)
{
/* Initialize recognized charsets. */
setRecognizers(status);
if(U_FAILURE(status)) {
return nullptr;
}
UEnumeration *en = NEW_ARRAY(UEnumeration, 1);
if (en == nullptr) {
status = U_MEMORY_ALLOCATION_ERROR;
return nullptr;
}
memcpy(en, &gCSDetEnumeration, sizeof(UEnumeration));
en->context = (void*)NEW_ARRAY(Context, 1);
if (en->context == nullptr) {
status = U_MEMORY_ALLOCATION_ERROR;
DELETE_ARRAY(en);
return nullptr;
}
uprv_memset(en->context, 0, sizeof(Context));
static_cast<Context*>(en->context)->all = true;
return en;
}
UEnumeration * CharsetDetector::getDetectableCharsets(UErrorCode &status) const
{
if(U_FAILURE(status)) {
return nullptr;
}
UEnumeration *en = NEW_ARRAY(UEnumeration, 1);
if (en == nullptr) {
status = U_MEMORY_ALLOCATION_ERROR;
return nullptr;
}
memcpy(en, &gCSDetEnumeration, sizeof(UEnumeration));
en->context = (void*)NEW_ARRAY(Context, 1);
if (en->context == nullptr) {
status = U_MEMORY_ALLOCATION_ERROR;
DELETE_ARRAY(en);
return nullptr;
}
uprv_memset(en->context, 0, sizeof(Context));
static_cast<Context*>(en->context)->all = false;
static_cast<Context*>(en->context)->enabledRecognizers = fEnabledRecognizers;
return en;
}
U_NAMESPACE_END
#endif

69
deps/icu-small/source/i18n/csdetect.h vendored Normal file
View File

@ -0,0 +1,69 @@
// © 2016 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html
/*
**********************************************************************
* Copyright (C) 2005-2016, International Business Machines
* Corporation and others. All Rights Reserved.
**********************************************************************
*/
#ifndef __CSDETECT_H
#define __CSDETECT_H
#include "unicode/uobject.h"
#if !UCONFIG_NO_CONVERSION
#include "unicode/uenum.h"
U_NAMESPACE_BEGIN
class InputText;
class CharsetRecognizer;
class CharsetMatch;
class CharsetDetector : public UMemory
{
private:
InputText *textIn;
CharsetMatch **resultArray;
int32_t resultCount;
UBool fStripTags; // If true, setText() will strip tags from input text.
UBool fFreshTextSet;
static void setRecognizers(UErrorCode &status);
UBool *fEnabledRecognizers; // If not null, active set of charset recognizers had
// been changed from the default. The array index is
// corresponding to fCSRecognizers. See setDetectableCharset().
public:
CharsetDetector(UErrorCode &status);
~CharsetDetector();
void setText(const char *in, int32_t len);
const CharsetMatch * const *detectAll(int32_t &maxMatchesFound, UErrorCode &status);
const CharsetMatch *detect(UErrorCode& status);
void setDeclaredEncoding(const char *encoding, int32_t len) const;
UBool setStripTagsFlag(UBool flag);
UBool getStripTagsFlag() const;
// const char *getCharsetName(int32_t index, UErrorCode& status) const;
static int32_t getDetectableCount();
static UEnumeration * getAllDetectableCharsets(UErrorCode &status);
UEnumeration * getDetectableCharsets(UErrorCode &status) const;
void setDetectableCharset(const char *encoding, UBool enabled, UErrorCode &status);
};
U_NAMESPACE_END
#endif
#endif /* __CSDETECT_H */

73
deps/icu-small/source/i18n/csmatch.cpp vendored Normal file
View File

@ -0,0 +1,73 @@
// © 2016 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html
/*
**********************************************************************
* Copyright (C) 2005-2012, International Business Machines
* Corporation and others. All Rights Reserved.
**********************************************************************
*/
#include "unicode/utypes.h"
#if !UCONFIG_NO_CONVERSION
#include "unicode/unistr.h"
#include "unicode/ucnv.h"
#include "csmatch.h"
#include "csrecog.h"
#include "inputext.h"
U_NAMESPACE_BEGIN
CharsetMatch::CharsetMatch()
: textIn(nullptr), confidence(0), fCharsetName(nullptr), fLang(nullptr)
{
// nothing else to do.
}
void CharsetMatch::set(InputText *input, const CharsetRecognizer *cr, int32_t conf,
const char *csName, const char *lang)
{
textIn = input;
confidence = conf;
fCharsetName = csName;
fLang = lang;
if (cr != nullptr) {
if (fCharsetName == nullptr) {
fCharsetName = cr->getName();
}
if (fLang == nullptr) {
fLang = cr->getLanguage();
}
}
}
const char* CharsetMatch::getName()const
{
return fCharsetName;
}
const char* CharsetMatch::getLanguage()const
{
return fLang;
}
int32_t CharsetMatch::getConfidence()const
{
return confidence;
}
int32_t CharsetMatch::getUChars(char16_t *buf, int32_t cap, UErrorCode *status) const
{
UConverter *conv = ucnv_open(getName(), status);
int32_t result = ucnv_toUChars(conv, buf, cap, reinterpret_cast<const char*>(textIn->fRawInput), textIn->fRawLength, status);
ucnv_close(conv);
return result;
}
U_NAMESPACE_END
#endif

71
deps/icu-small/source/i18n/csmatch.h vendored Normal file
View File

@ -0,0 +1,71 @@
// © 2016 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html
/*
**********************************************************************
* Copyright (C) 2005-2012, International Business Machines
* Corporation and others. All Rights Reserved.
**********************************************************************
*/
#ifndef __CSMATCH_H
#define __CSMATCH_H
#include "unicode/uobject.h"
#if !UCONFIG_NO_CONVERSION
U_NAMESPACE_BEGIN
class InputText;
class CharsetRecognizer;
/*
* CharsetMatch represents the results produced by one Charset Recognizer for one input text
* Any confidence > 0 indicates a possible match, meaning that the input bytes
* are at least legal.
*
* The full results of a detect are represented by an array of these
* CharsetMatch objects, each representing a possible matching charset.
*
* Note that a single charset recognizer may detect multiple closely related
* charsets, and set different names depending on the exact input bytes seen.
*/
class CharsetMatch : public UMemory
{
private:
InputText *textIn;
int32_t confidence;
const char *fCharsetName;
const char *fLang;
public:
CharsetMatch();
/**
* fully set the state of this CharsetMatch.
* Called by the CharsetRecognizers to record match results.
* Default (nullptr) parameters for names will be filled by calling the
* corresponding getters on the recognizer.
*/
void set(InputText *input,
const CharsetRecognizer *cr,
int32_t conf,
const char *csName=nullptr,
const char *lang=nullptr);
/**
* Return the name of the charset for this Match
*/
const char *getName() const;
const char *getLanguage()const;
int32_t getConfidence()const;
int32_t getUChars(char16_t *buf, int32_t cap, UErrorCode *status) const;
};
U_NAMESPACE_END
#endif
#endif /* __CSMATCH_H */

195
deps/icu-small/source/i18n/csr2022.cpp vendored Normal file
View File

@ -0,0 +1,195 @@
// © 2016 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html
/*
**********************************************************************
* Copyright (C) 2005-2016, International Business Machines
* Corporation and others. All Rights Reserved.
**********************************************************************
*/
#include "unicode/utypes.h"
#if !UCONFIG_NO_CONVERSION
#include "cmemory.h"
#include "cstring.h"
#include "csr2022.h"
#include "csmatch.h"
U_NAMESPACE_BEGIN
/**
* Matching function shared among the 2022 detectors JP, CN and KR
* Counts up the number of legal and unrecognized escape sequences in
* the sample of text, and computes a score based on the total number &
* the proportion that fit the encoding.
*
*
* @param text the byte buffer containing text to analyse
* @param textLen the size of the text in the byte.
* @param escapeSequences the byte escape sequences to test for.
* @return match quality, in the range of 0-100.
*/
int32_t CharsetRecog_2022::match_2022(const uint8_t *text, int32_t textLen, const uint8_t escapeSequences[][5], int32_t escapeSequences_length) const
{
int32_t i, j;
int32_t escN;
int32_t hits = 0;
int32_t misses = 0;
int32_t shifts = 0;
int32_t quality;
i = 0;
while(i < textLen) {
if(text[i] == 0x1B) {
escN = 0;
while(escN < escapeSequences_length) {
const uint8_t *seq = escapeSequences[escN];
int32_t seq_length = static_cast<int32_t>(uprv_strlen((const char*)seq));
if (textLen-i >= seq_length) {
j = 1;
while(j < seq_length) {
if(seq[j] != text[i+j]) {
goto checkEscapes;
}
j += 1;
}
hits += 1;
i += seq_length-1;
goto scanInput;
}
// else we ran out of string to compare this time.
checkEscapes:
escN += 1;
}
misses += 1;
}
if( text[i]== 0x0e || text[i] == 0x0f){
shifts += 1;
}
scanInput:
i += 1;
}
if (hits == 0) {
return 0;
}
//
// Initial quality is based on relative proportion of recognized vs.
// unrecognized escape sequences.
// All good: quality = 100;
// half or less good: quality = 0;
// linear inbetween.
quality = (100*hits - 100*misses) / (hits + misses);
// Back off quality if there were too few escape sequences seen.
// Include shifts in this computation, so that KR does not get penalized
// for having only a single Escape sequence, but many shifts.
if (hits+shifts < 5) {
quality -= (5-(hits+shifts))*10;
}
if (quality < 0) {
quality = 0;
}
return quality;
}
static const uint8_t escapeSequences_2022JP[][5] = {
{0x1b, 0x24, 0x28, 0x43, 0x00}, // KS X 1001:1992
{0x1b, 0x24, 0x28, 0x44, 0x00}, // JIS X 212-1990
{0x1b, 0x24, 0x40, 0x00, 0x00}, // JIS C 6226-1978
{0x1b, 0x24, 0x41, 0x00, 0x00}, // GB 2312-80
{0x1b, 0x24, 0x42, 0x00, 0x00}, // JIS X 208-1983
{0x1b, 0x26, 0x40, 0x00, 0x00}, // JIS X 208 1990, 1997
{0x1b, 0x28, 0x42, 0x00, 0x00}, // ASCII
{0x1b, 0x28, 0x48, 0x00, 0x00}, // JIS-Roman
{0x1b, 0x28, 0x49, 0x00, 0x00}, // Half-width katakana
{0x1b, 0x28, 0x4a, 0x00, 0x00}, // JIS-Roman
{0x1b, 0x2e, 0x41, 0x00, 0x00}, // ISO 8859-1
{0x1b, 0x2e, 0x46, 0x00, 0x00} // ISO 8859-7
};
#if !UCONFIG_ONLY_HTML_CONVERSION
static const uint8_t escapeSequences_2022KR[][5] = {
{0x1b, 0x24, 0x29, 0x43, 0x00}
};
static const uint8_t escapeSequences_2022CN[][5] = {
{0x1b, 0x24, 0x29, 0x41, 0x00}, // GB 2312-80
{0x1b, 0x24, 0x29, 0x47, 0x00}, // CNS 11643-1992 Plane 1
{0x1b, 0x24, 0x2A, 0x48, 0x00}, // CNS 11643-1992 Plane 2
{0x1b, 0x24, 0x29, 0x45, 0x00}, // ISO-IR-165
{0x1b, 0x24, 0x2B, 0x49, 0x00}, // CNS 11643-1992 Plane 3
{0x1b, 0x24, 0x2B, 0x4A, 0x00}, // CNS 11643-1992 Plane 4
{0x1b, 0x24, 0x2B, 0x4B, 0x00}, // CNS 11643-1992 Plane 5
{0x1b, 0x24, 0x2B, 0x4C, 0x00}, // CNS 11643-1992 Plane 6
{0x1b, 0x24, 0x2B, 0x4D, 0x00}, // CNS 11643-1992 Plane 7
{0x1b, 0x4e, 0x00, 0x00, 0x00}, // SS2
{0x1b, 0x4f, 0x00, 0x00, 0x00}, // SS3
};
#endif
CharsetRecog_2022JP::~CharsetRecog_2022JP() {}
const char *CharsetRecog_2022JP::getName() const {
return "ISO-2022-JP";
}
UBool CharsetRecog_2022JP::match(InputText *textIn, CharsetMatch *results) const {
int32_t confidence = match_2022(textIn->fInputBytes,
textIn->fInputLen,
escapeSequences_2022JP,
UPRV_LENGTHOF(escapeSequences_2022JP));
results->set(textIn, this, confidence);
return (confidence > 0);
}
#if !UCONFIG_ONLY_HTML_CONVERSION
CharsetRecog_2022KR::~CharsetRecog_2022KR() {}
const char *CharsetRecog_2022KR::getName() const {
return "ISO-2022-KR";
}
UBool CharsetRecog_2022KR::match(InputText *textIn, CharsetMatch *results) const {
int32_t confidence = match_2022(textIn->fInputBytes,
textIn->fInputLen,
escapeSequences_2022KR,
UPRV_LENGTHOF(escapeSequences_2022KR));
results->set(textIn, this, confidence);
return (confidence > 0);
}
CharsetRecog_2022CN::~CharsetRecog_2022CN() {}
const char *CharsetRecog_2022CN::getName() const {
return "ISO-2022-CN";
}
UBool CharsetRecog_2022CN::match(InputText *textIn, CharsetMatch *results) const {
int32_t confidence = match_2022(textIn->fInputBytes,
textIn->fInputLen,
escapeSequences_2022CN,
UPRV_LENGTHOF(escapeSequences_2022CN));
results->set(textIn, this, confidence);
return (confidence > 0);
}
#endif
CharsetRecog_2022::~CharsetRecog_2022() {
// nothing to do
}
U_NAMESPACE_END
#endif

95
deps/icu-small/source/i18n/csr2022.h vendored Normal file
View File

@ -0,0 +1,95 @@
// © 2016 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html
/*
**********************************************************************
* Copyright (C) 2005-2015, International Business Machines
* Corporation and others. All Rights Reserved.
**********************************************************************
*/
#ifndef __CSR2022_H
#define __CSR2022_H
#include "unicode/utypes.h"
#if !UCONFIG_NO_CONVERSION
#include "csrecog.h"
U_NAMESPACE_BEGIN
class CharsetMatch;
/**
* class CharsetRecog_2022 part of the ICU charset detection implementation.
* This is a superclass for the individual detectors for
* each of the detectable members of the ISO 2022 family
* of encodings.
*
* The separate classes are nested within this class.
*
* @internal
*/
class CharsetRecog_2022 : public CharsetRecognizer
{
public:
virtual ~CharsetRecog_2022() = 0;
protected:
/**
* Matching function shared among the 2022 detectors JP, CN and KR
* Counts up the number of legal an unrecognized escape sequences in
* the sample of text, and computes a score based on the total number &
* the proportion that fit the encoding.
*
*
* @param text the byte buffer containing text to analyse
* @param textLen the size of the text in the byte.
* @param escapeSequences the byte escape sequences to test for.
* @return match quality, in the range of 0-100.
*/
int32_t match_2022(const uint8_t *text,
int32_t textLen,
const uint8_t escapeSequences[][5],
int32_t escapeSequences_length) const;
};
class CharsetRecog_2022JP :public CharsetRecog_2022
{
public:
virtual ~CharsetRecog_2022JP();
const char *getName() const override;
UBool match(InputText *textIn, CharsetMatch *results) const override;
};
#if !UCONFIG_ONLY_HTML_CONVERSION
class CharsetRecog_2022KR :public CharsetRecog_2022 {
public:
virtual ~CharsetRecog_2022KR();
const char *getName() const override;
UBool match(InputText *textIn, CharsetMatch *results) const override;
};
class CharsetRecog_2022CN :public CharsetRecog_2022
{
public:
virtual ~CharsetRecog_2022CN();
const char* getName() const override;
UBool match(InputText *textIn, CharsetMatch *results) const override;
};
#endif
U_NAMESPACE_END
#endif
#endif /* __CSR2022_H */

30
deps/icu-small/source/i18n/csrecog.cpp vendored Normal file
View File

@ -0,0 +1,30 @@
// © 2016 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html
/*
**********************************************************************
* Copyright (C) 2005-2006, International Business Machines
* Corporation and others. All Rights Reserved.
**********************************************************************
*/
#include "unicode/utypes.h"
#if !UCONFIG_NO_CONVERSION
#include "csrecog.h"
U_NAMESPACE_BEGIN
CharsetRecognizer::~CharsetRecognizer()
{
// nothing to do.
}
const char *CharsetRecognizer::getLanguage() const
{
return "";
}
U_NAMESPACE_END
#endif

57
deps/icu-small/source/i18n/csrecog.h vendored Normal file
View File

@ -0,0 +1,57 @@
// © 2016 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html
/*
**********************************************************************
* Copyright (C) 2005-2012, International Business Machines
* Corporation and others. All Rights Reserved.
**********************************************************************
*/
#ifndef __CSRECOG_H
#define __CSRECOG_H
#include "unicode/uobject.h"
#if !UCONFIG_NO_CONVERSION
#include "inputext.h"
U_NAMESPACE_BEGIN
class CharsetMatch;
class CharsetRecognizer : public UMemory
{
public:
/**
* Get the IANA name of this charset.
* Note that some recognizers can recognize more than one charset, but that this API
* assumes just one name per recognizer.
* TODO: need to account for multiple names in public API that enumerates over the
* known detectable charsets.
* @return the charset name.
*/
virtual const char *getName() const = 0;
/**
* Get the ISO language code for this charset.
* @return the language code, or <code>null</code> if the language cannot be determined.
*/
virtual const char *getLanguage() const;
/*
* Try the given input text against this Charset, and fill in the results object
* with the quality of the match plus other information related to the match.
*
* Return true if the the input bytes are a potential match, and
* false if the input data is not compatible with, or illegal in this charset.
*/
virtual UBool match(InputText *textIn, CharsetMatch *results) const = 0;
virtual ~CharsetRecognizer();
};
U_NAMESPACE_END
#endif
#endif /* __CSRECOG_H */

527
deps/icu-small/source/i18n/csrmbcs.cpp vendored Normal file
View File

@ -0,0 +1,527 @@
// © 2016 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html
/*
**********************************************************************
* Copyright (C) 2005-2016, International Business Machines
* Corporation and others. All Rights Reserved.
**********************************************************************
*/
#include "unicode/utypes.h"
#if !UCONFIG_NO_CONVERSION
#include "cmemory.h"
#include "csmatch.h"
#include "csrmbcs.h"
#include <math.h>
U_NAMESPACE_BEGIN
#define min(x,y) (((x)<(y))?(x):(y))
static const uint16_t commonChars_sjis [] = {
// TODO: This set of data comes from the character frequency-
// of-occurrence analysis tool. The data needs to be moved
// into a resource and loaded from there.
0x8140, 0x8141, 0x8142, 0x8145, 0x815b, 0x8169, 0x816a, 0x8175, 0x8176, 0x82a0,
0x82a2, 0x82a4, 0x82a9, 0x82aa, 0x82ab, 0x82ad, 0x82af, 0x82b1, 0x82b3, 0x82b5,
0x82b7, 0x82bd, 0x82be, 0x82c1, 0x82c4, 0x82c5, 0x82c6, 0x82c8, 0x82c9, 0x82cc,
0x82cd, 0x82dc, 0x82e0, 0x82e7, 0x82e8, 0x82e9, 0x82ea, 0x82f0, 0x82f1, 0x8341,
0x8343, 0x834e, 0x834f, 0x8358, 0x835e, 0x8362, 0x8367, 0x8375, 0x8376, 0x8389,
0x838a, 0x838b, 0x838d, 0x8393, 0x8e96, 0x93fa, 0x95aa};
static const uint16_t commonChars_euc_jp[] = {
// TODO: This set of data comes from the character frequency-
// of-occurrence analysis tool. The data needs to be moved
// into a resource and loaded from there.
0xa1a1, 0xa1a2, 0xa1a3, 0xa1a6, 0xa1bc, 0xa1ca, 0xa1cb, 0xa1d6, 0xa1d7, 0xa4a2,
0xa4a4, 0xa4a6, 0xa4a8, 0xa4aa, 0xa4ab, 0xa4ac, 0xa4ad, 0xa4af, 0xa4b1, 0xa4b3,
0xa4b5, 0xa4b7, 0xa4b9, 0xa4bb, 0xa4bd, 0xa4bf, 0xa4c0, 0xa4c1, 0xa4c3, 0xa4c4,
0xa4c6, 0xa4c7, 0xa4c8, 0xa4c9, 0xa4ca, 0xa4cb, 0xa4ce, 0xa4cf, 0xa4d0, 0xa4de,
0xa4df, 0xa4e1, 0xa4e2, 0xa4e4, 0xa4e8, 0xa4e9, 0xa4ea, 0xa4eb, 0xa4ec, 0xa4ef,
0xa4f2, 0xa4f3, 0xa5a2, 0xa5a3, 0xa5a4, 0xa5a6, 0xa5a7, 0xa5aa, 0xa5ad, 0xa5af,
0xa5b0, 0xa5b3, 0xa5b5, 0xa5b7, 0xa5b8, 0xa5b9, 0xa5bf, 0xa5c3, 0xa5c6, 0xa5c7,
0xa5c8, 0xa5c9, 0xa5cb, 0xa5d0, 0xa5d5, 0xa5d6, 0xa5d7, 0xa5de, 0xa5e0, 0xa5e1,
0xa5e5, 0xa5e9, 0xa5ea, 0xa5eb, 0xa5ec, 0xa5ed, 0xa5f3, 0xb8a9, 0xb9d4, 0xbaee,
0xbbc8, 0xbef0, 0xbfb7, 0xc4ea, 0xc6fc, 0xc7bd, 0xcab8, 0xcaf3, 0xcbdc, 0xcdd1};
static const uint16_t commonChars_euc_kr[] = {
// TODO: This set of data comes from the character frequency-
// of-occurrence analysis tool. The data needs to be moved
// into a resource and loaded from there.
0xb0a1, 0xb0b3, 0xb0c5, 0xb0cd, 0xb0d4, 0xb0e6, 0xb0ed, 0xb0f8, 0xb0fa, 0xb0fc,
0xb1b8, 0xb1b9, 0xb1c7, 0xb1d7, 0xb1e2, 0xb3aa, 0xb3bb, 0xb4c2, 0xb4cf, 0xb4d9,
0xb4eb, 0xb5a5, 0xb5b5, 0xb5bf, 0xb5c7, 0xb5e9, 0xb6f3, 0xb7af, 0xb7c2, 0xb7ce,
0xb8a6, 0xb8ae, 0xb8b6, 0xb8b8, 0xb8bb, 0xb8e9, 0xb9ab, 0xb9ae, 0xb9cc, 0xb9ce,
0xb9fd, 0xbab8, 0xbace, 0xbad0, 0xbaf1, 0xbbe7, 0xbbf3, 0xbbfd, 0xbcad, 0xbcba,
0xbcd2, 0xbcf6, 0xbdba, 0xbdc0, 0xbdc3, 0xbdc5, 0xbec6, 0xbec8, 0xbedf, 0xbeee,
0xbef8, 0xbefa, 0xbfa1, 0xbfa9, 0xbfc0, 0xbfe4, 0xbfeb, 0xbfec, 0xbff8, 0xc0a7,
0xc0af, 0xc0b8, 0xc0ba, 0xc0bb, 0xc0bd, 0xc0c7, 0xc0cc, 0xc0ce, 0xc0cf, 0xc0d6,
0xc0da, 0xc0e5, 0xc0fb, 0xc0fc, 0xc1a4, 0xc1a6, 0xc1b6, 0xc1d6, 0xc1df, 0xc1f6,
0xc1f8, 0xc4a1, 0xc5cd, 0xc6ae, 0xc7cf, 0xc7d1, 0xc7d2, 0xc7d8, 0xc7e5, 0xc8ad};
static const uint16_t commonChars_big5[] = {
// TODO: This set of data comes from the character frequency-
// of-occurrence analysis tool. The data needs to be moved
// into a resource and loaded from there.
0xa140, 0xa141, 0xa142, 0xa143, 0xa147, 0xa149, 0xa175, 0xa176, 0xa440, 0xa446,
0xa447, 0xa448, 0xa451, 0xa454, 0xa457, 0xa464, 0xa46a, 0xa46c, 0xa477, 0xa4a3,
0xa4a4, 0xa4a7, 0xa4c1, 0xa4ce, 0xa4d1, 0xa4df, 0xa4e8, 0xa4fd, 0xa540, 0xa548,
0xa558, 0xa569, 0xa5cd, 0xa5e7, 0xa657, 0xa661, 0xa662, 0xa668, 0xa670, 0xa6a8,
0xa6b3, 0xa6b9, 0xa6d3, 0xa6db, 0xa6e6, 0xa6f2, 0xa740, 0xa751, 0xa759, 0xa7da,
0xa8a3, 0xa8a5, 0xa8ad, 0xa8d1, 0xa8d3, 0xa8e4, 0xa8fc, 0xa9c0, 0xa9d2, 0xa9f3,
0xaa6b, 0xaaba, 0xaabe, 0xaacc, 0xaafc, 0xac47, 0xac4f, 0xacb0, 0xacd2, 0xad59,
0xaec9, 0xafe0, 0xb0ea, 0xb16f, 0xb2b3, 0xb2c4, 0xb36f, 0xb44c, 0xb44e, 0xb54c,
0xb5a5, 0xb5bd, 0xb5d0, 0xb5d8, 0xb671, 0xb7ed, 0xb867, 0xb944, 0xbad8, 0xbb44,
0xbba1, 0xbdd1, 0xc2c4, 0xc3b9, 0xc440, 0xc45f};
static const uint16_t commonChars_gb_18030[] = {
// TODO: This set of data comes from the character frequency-
// of-occurrence analysis tool. The data needs to be moved
// into a resource and loaded from there.
0xa1a1, 0xa1a2, 0xa1a3, 0xa1a4, 0xa1b0, 0xa1b1, 0xa1f1, 0xa1f3, 0xa3a1, 0xa3ac,
0xa3ba, 0xb1a8, 0xb1b8, 0xb1be, 0xb2bb, 0xb3c9, 0xb3f6, 0xb4f3, 0xb5bd, 0xb5c4,
0xb5e3, 0xb6af, 0xb6d4, 0xb6e0, 0xb7a2, 0xb7a8, 0xb7bd, 0xb7d6, 0xb7dd, 0xb8b4,
0xb8df, 0xb8f6, 0xb9ab, 0xb9c9, 0xb9d8, 0xb9fa, 0xb9fd, 0xbacd, 0xbba7, 0xbbd6,
0xbbe1, 0xbbfa, 0xbcbc, 0xbcdb, 0xbcfe, 0xbdcc, 0xbecd, 0xbedd, 0xbfb4, 0xbfc6,
0xbfc9, 0xc0b4, 0xc0ed, 0xc1cb, 0xc2db, 0xc3c7, 0xc4dc, 0xc4ea, 0xc5cc, 0xc6f7,
0xc7f8, 0xc8ab, 0xc8cb, 0xc8d5, 0xc8e7, 0xc9cf, 0xc9fa, 0xcab1, 0xcab5, 0xcac7,
0xcad0, 0xcad6, 0xcaf5, 0xcafd, 0xccec, 0xcdf8, 0xceaa, 0xcec4, 0xced2, 0xcee5,
0xcfb5, 0xcfc2, 0xcfd6, 0xd0c2, 0xd0c5, 0xd0d0, 0xd0d4, 0xd1a7, 0xd2aa, 0xd2b2,
0xd2b5, 0xd2bb, 0xd2d4, 0xd3c3, 0xd3d0, 0xd3fd, 0xd4c2, 0xd4da, 0xd5e2, 0xd6d0};
static int32_t binarySearch(const uint16_t *array, int32_t len, uint16_t value)
{
int32_t start = 0, end = len-1;
int32_t mid = (start+end)/2;
while(start <= end) {
if(array[mid] == value) {
return mid;
}
if(array[mid] < value){
start = mid+1;
} else {
end = mid-1;
}
mid = (start+end)/2;
}
return -1;
}
IteratedChar::IteratedChar() :
charValue(0), index(-1), nextIndex(0), error(false), done(false)
{
// nothing else to do.
}
/*void IteratedChar::reset()
{
charValue = 0;
index = -1;
nextIndex = 0;
error = false;
done = false;
}*/
int32_t IteratedChar::nextByte(InputText *det)
{
if (nextIndex >= det->fRawLength) {
done = true;
return -1;
}
return det->fRawInput[nextIndex++];
}
CharsetRecog_mbcs::~CharsetRecog_mbcs()
{
// nothing to do.
}
int32_t CharsetRecog_mbcs::match_mbcs(InputText *det, const uint16_t commonChars[], int32_t commonCharsLen) const {
int32_t doubleByteCharCount = 0;
int32_t commonCharCount = 0;
int32_t badCharCount = 0;
int32_t totalCharCount = 0;
int32_t confidence = 0;
IteratedChar iter;
while (nextChar(&iter, det)) {
totalCharCount++;
if (iter.error) {
badCharCount++;
} else {
if (iter.charValue > 0xFF) {
doubleByteCharCount++;
if (commonChars != nullptr) {
if (binarySearch(commonChars, commonCharsLen, static_cast<uint16_t>(iter.charValue)) >= 0){
commonCharCount += 1;
}
}
}
}
if (badCharCount >= 2 && badCharCount*5 >= doubleByteCharCount) {
// Bail out early if the byte data is not matching the encoding scheme.
// break detectBlock;
return confidence;
}
}
if (doubleByteCharCount <= 10 && badCharCount == 0) {
// Not many multi-byte chars.
if (doubleByteCharCount == 0 && totalCharCount < 10) {
// There weren't any multibyte sequences, and there was a low density of non-ASCII single bytes.
// We don't have enough data to have any confidence.
// Statistical analysis of single byte non-ASCII characters would probably help here.
confidence = 0;
}
else {
// ASCII or ISO file? It's probably not our encoding,
// but is not incompatible with our encoding, so don't give it a zero.
confidence = 10;
}
return confidence;
}
//
// No match if there are too many characters that don't fit the encoding scheme.
// (should we have zero tolerance for these?)
//
if (doubleByteCharCount < 20*badCharCount) {
confidence = 0;
return confidence;
}
if (commonChars == nullptr) {
// We have no statistics on frequently occurring characters.
// Assess confidence purely on having a reasonable number of
// multi-byte characters (the more the better)
confidence = 30 + doubleByteCharCount - 20*badCharCount;
if (confidence > 100) {
confidence = 100;
}
} else {
//
// Frequency of occurrence statistics exist.
//
double maxVal = log(static_cast<double>(doubleByteCharCount) / 4); /*(float)?*/
double scaleFactor = 90.0 / maxVal;
confidence = static_cast<int32_t>(log(static_cast<double>(commonCharCount) + 1) * scaleFactor + 10.0);
confidence = min(confidence, 100);
}
if (confidence < 0) {
confidence = 0;
}
return confidence;
}
CharsetRecog_sjis::~CharsetRecog_sjis()
{
// nothing to do
}
UBool CharsetRecog_sjis::nextChar(IteratedChar* it, InputText* det) const {
it->index = it->nextIndex;
it->error = false;
int32_t firstByte = it->charValue = it->nextByte(det);
if (firstByte < 0) {
return false;
}
if (firstByte <= 0x7F || (firstByte > 0xA0 && firstByte <= 0xDF)) {
return true;
}
int32_t secondByte = it->nextByte(det);
if (secondByte >= 0) {
it->charValue = (firstByte << 8) | secondByte;
}
// else we'll handle the error later.
if (! ((secondByte >= 0x40 && secondByte <= 0x7F) || (secondByte >= 0x80 && secondByte <= 0xFE))) {
// Illegal second byte value.
it->error = true;
}
return true;
}
UBool CharsetRecog_sjis::match(InputText* det, CharsetMatch *results) const {
int32_t confidence = match_mbcs(det, commonChars_sjis, UPRV_LENGTHOF(commonChars_sjis));
results->set(det, this, confidence);
return (confidence > 0);
}
const char *CharsetRecog_sjis::getName() const
{
return "Shift_JIS";
}
const char *CharsetRecog_sjis::getLanguage() const
{
return "ja";
}
CharsetRecog_euc::~CharsetRecog_euc()
{
// nothing to do
}
UBool CharsetRecog_euc::nextChar(IteratedChar* it, InputText* det) const {
int32_t firstByte = 0;
int32_t secondByte = 0;
int32_t thirdByte = 0;
it->index = it->nextIndex;
it->error = false;
firstByte = it->charValue = it->nextByte(det);
if (firstByte < 0) {
// Ran off the end of the input data
return false;
}
if (firstByte <= 0x8D) {
// single byte char
return true;
}
secondByte = it->nextByte(det);
if (secondByte >= 0) {
it->charValue = (it->charValue << 8) | secondByte;
}
// else we'll handle the error later.
if (firstByte >= 0xA1 && firstByte <= 0xFE) {
// Two byte Char
if (secondByte < 0xA1) {
it->error = true;
}
return true;
}
if (firstByte == 0x8E) {
// Code Set 2.
// In EUC-JP, total char size is 2 bytes, only one byte of actual char value.
// In EUC-TW, total char size is 4 bytes, three bytes contribute to char value.
// We don't know which we've got.
// Treat it like EUC-JP. If the data really was EUC-TW, the following two
// bytes will look like a well formed 2 byte char.
if (secondByte < 0xA1) {
it->error = true;
}
return true;
}
if (firstByte == 0x8F) {
// Code set 3.
// Three byte total char size, two bytes of actual char value.
thirdByte = it->nextByte(det);
it->charValue = (it->charValue << 8) | thirdByte;
if (thirdByte < 0xa1) {
// Bad second byte or ran off the end of the input data with a non-ASCII first byte.
it->error = true;
}
}
return true;
}
CharsetRecog_euc_jp::~CharsetRecog_euc_jp()
{
// nothing to do
}
const char *CharsetRecog_euc_jp::getName() const
{
return "EUC-JP";
}
const char *CharsetRecog_euc_jp::getLanguage() const
{
return "ja";
}
UBool CharsetRecog_euc_jp::match(InputText *det, CharsetMatch *results) const
{
int32_t confidence = match_mbcs(det, commonChars_euc_jp, UPRV_LENGTHOF(commonChars_euc_jp));
results->set(det, this, confidence);
return (confidence > 0);
}
CharsetRecog_euc_kr::~CharsetRecog_euc_kr()
{
// nothing to do
}
const char *CharsetRecog_euc_kr::getName() const
{
return "EUC-KR";
}
const char *CharsetRecog_euc_kr::getLanguage() const
{
return "ko";
}
UBool CharsetRecog_euc_kr::match(InputText *det, CharsetMatch *results) const
{
int32_t confidence = match_mbcs(det, commonChars_euc_kr, UPRV_LENGTHOF(commonChars_euc_kr));
results->set(det, this, confidence);
return (confidence > 0);
}
CharsetRecog_big5::~CharsetRecog_big5()
{
// nothing to do
}
UBool CharsetRecog_big5::nextChar(IteratedChar* it, InputText* det) const
{
int32_t firstByte;
it->index = it->nextIndex;
it->error = false;
firstByte = it->charValue = it->nextByte(det);
if (firstByte < 0) {
return false;
}
if (firstByte <= 0x7F || firstByte == 0xFF) {
// single byte character.
return true;
}
int32_t secondByte = it->nextByte(det);
if (secondByte >= 0) {
it->charValue = (it->charValue << 8) | secondByte;
}
// else we'll handle the error later.
if (secondByte < 0x40 || secondByte == 0x7F || secondByte == 0xFF) {
it->error = true;
}
return true;
}
const char *CharsetRecog_big5::getName() const
{
return "Big5";
}
const char *CharsetRecog_big5::getLanguage() const
{
return "zh";
}
UBool CharsetRecog_big5::match(InputText *det, CharsetMatch *results) const
{
int32_t confidence = match_mbcs(det, commonChars_big5, UPRV_LENGTHOF(commonChars_big5));
results->set(det, this, confidence);
return (confidence > 0);
}
CharsetRecog_gb_18030::~CharsetRecog_gb_18030()
{
// nothing to do
}
UBool CharsetRecog_gb_18030::nextChar(IteratedChar* it, InputText* det) const {
int32_t firstByte = 0;
int32_t secondByte = 0;
int32_t thirdByte = 0;
int32_t fourthByte = 0;
it->index = it->nextIndex;
it->error = false;
firstByte = it->charValue = it->nextByte(det);
if (firstByte < 0) {
// Ran off the end of the input data
return false;
}
if (firstByte <= 0x80) {
// single byte char
return true;
}
secondByte = it->nextByte(det);
if (secondByte >= 0) {
it->charValue = (it->charValue << 8) | secondByte;
}
// else we'll handle the error later.
if (firstByte >= 0x81 && firstByte <= 0xFE) {
// Two byte Char
if ((secondByte >= 0x40 && secondByte <= 0x7E) || (secondByte >=80 && secondByte <= 0xFE)) {
return true;
}
// Four byte char
if (secondByte >= 0x30 && secondByte <= 0x39) {
thirdByte = it->nextByte(det);
if (thirdByte >= 0x81 && thirdByte <= 0xFE) {
fourthByte = it->nextByte(det);
if (fourthByte >= 0x30 && fourthByte <= 0x39) {
it->charValue = (it->charValue << 16) | (thirdByte << 8) | fourthByte;
return true;
}
}
}
// Something wasn't valid, or we ran out of data (-1).
it->error = true;
}
return true;
}
const char *CharsetRecog_gb_18030::getName() const
{
return "GB18030";
}
const char *CharsetRecog_gb_18030::getLanguage() const
{
return "zh";
}
UBool CharsetRecog_gb_18030::match(InputText *det, CharsetMatch *results) const
{
int32_t confidence = match_mbcs(det, commonChars_gb_18030, UPRV_LENGTHOF(commonChars_gb_18030));
results->set(det, this, confidence);
return (confidence > 0);
}
U_NAMESPACE_END
#endif

207
deps/icu-small/source/i18n/csrmbcs.h vendored Normal file
View File

@ -0,0 +1,207 @@
// © 2016 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html
/*
**********************************************************************
* Copyright (C) 2005-2012, International Business Machines
* Corporation and others. All Rights Reserved.
**********************************************************************
*/
#ifndef __CSRMBCS_H
#define __CSRMBCS_H
#include "unicode/utypes.h"
#if !UCONFIG_NO_CONVERSION
#include "csrecog.h"
U_NAMESPACE_BEGIN
// "Character" iterated character class.
// Recognizers for specific mbcs encodings make their "characters" available
// by providing a nextChar() function that fills in an instance of IteratedChar
// with the next char from the input.
// The returned characters are not converted to Unicode, but remain as the raw
// bytes (concatenated into an int) from the codepage data.
//
// For Asian charsets, use the raw input rather than the input that has been
// stripped of markup. Detection only considers multi-byte chars, effectively
// stripping markup anyway, and double byte chars do occur in markup too.
//
class IteratedChar : public UMemory
{
public:
uint32_t charValue; // 1-4 bytes from the raw input data
int32_t index;
int32_t nextIndex;
UBool error;
UBool done;
public:
IteratedChar();
//void reset();
int32_t nextByte(InputText* det);
};
class CharsetRecog_mbcs : public CharsetRecognizer {
protected:
/**
* Test the match of this charset with the input text data
* which is obtained via the CharsetDetector object.
*
* @param det The CharsetDetector, which contains the input text
* to be checked for being in this charset.
* @return Two values packed into one int (Damn java, anyhow)
* <br/>
* bits 0-7: the match confidence, ranging from 0-100
* <br/>
* bits 8-15: The match reason, an enum-like value.
*/
int32_t match_mbcs(InputText* det, const uint16_t commonChars[], int32_t commonCharsLen) const;
public:
virtual ~CharsetRecog_mbcs();
/**
* Get the IANA name of this charset.
* @return the charset name.
*/
const char *getName() const override = 0;
const char *getLanguage() const override = 0;
UBool match(InputText* input, CharsetMatch *results) const override = 0;
/**
* Get the next character (however many bytes it is) from the input data
* Subclasses for specific charset encodings must implement this function
* to get characters according to the rules of their encoding scheme.
*
* This function is not a method of class IteratedChar only because
* that would require a lot of extra derived classes, which is awkward.
* @param it The IteratedChar "struct" into which the returned char is placed.
* @param det The charset detector, which is needed to get at the input byte data
* being iterated over.
* @return True if a character was returned, false at end of input.
*/
virtual UBool nextChar(IteratedChar *it, InputText *textIn) const = 0;
};
/**
* Shift-JIS charset recognizer.
*
*/
class CharsetRecog_sjis : public CharsetRecog_mbcs {
public:
virtual ~CharsetRecog_sjis();
UBool nextChar(IteratedChar *it, InputText *det) const override;
UBool match(InputText* input, CharsetMatch *results) const override;
const char *getName() const override;
const char *getLanguage() const override;
};
/**
* EUC charset recognizers. One abstract class that provides the common function
* for getting the next character according to the EUC encoding scheme,
* and nested derived classes for EUC_KR, EUC_JP, EUC_CN.
*
*/
class CharsetRecog_euc : public CharsetRecog_mbcs
{
public:
virtual ~CharsetRecog_euc();
const char *getName() const override = 0;
const char *getLanguage() const override = 0;
UBool match(InputText* input, CharsetMatch *results) const override = 0;
/*
* (non-Javadoc)
* Get the next character value for EUC based encodings.
* Character "value" is simply the raw bytes that make up the character
* packed into an int.
*/
UBool nextChar(IteratedChar *it, InputText *det) const override;
};
/**
* The charset recognize for EUC-JP. A singleton instance of this class
* is created and kept by the public CharsetDetector class
*/
class CharsetRecog_euc_jp : public CharsetRecog_euc
{
public:
virtual ~CharsetRecog_euc_jp();
const char *getName() const override;
const char *getLanguage() const override;
UBool match(InputText* input, CharsetMatch *results) const override;
};
/**
* The charset recognize for EUC-KR. A singleton instance of this class
* is created and kept by the public CharsetDetector class
*/
class CharsetRecog_euc_kr : public CharsetRecog_euc
{
public:
virtual ~CharsetRecog_euc_kr();
const char *getName() const override;
const char *getLanguage() const override;
UBool match(InputText* input, CharsetMatch *results) const override;
};
/**
*
* Big5 charset recognizer.
*
*/
class CharsetRecog_big5 : public CharsetRecog_mbcs
{
public:
virtual ~CharsetRecog_big5();
UBool nextChar(IteratedChar* it, InputText* det) const override;
const char *getName() const override;
const char *getLanguage() const override;
UBool match(InputText* input, CharsetMatch *results) const override;
};
/**
*
* GB-18030 recognizer. Uses simplified Chinese statistics.
*
*/
class CharsetRecog_gb_18030 : public CharsetRecog_mbcs
{
public:
virtual ~CharsetRecog_gb_18030();
UBool nextChar(IteratedChar* it, InputText* det) const override;
const char *getName() const override;
const char *getLanguage() const override;
UBool match(InputText* input, CharsetMatch *results) const override;
};
U_NAMESPACE_END
#endif
#endif /* __CSRMBCS_H */

1271
deps/icu-small/source/i18n/csrsbcs.cpp vendored Normal file
View File

File diff suppressed because it is too large Load Diff

295
deps/icu-small/source/i18n/csrsbcs.h vendored Normal file
View File

@ -0,0 +1,295 @@
// © 2016 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html
/*
**********************************************************************
* Copyright (C) 2005-2015, International Business Machines
* Corporation and others. All Rights Reserved.
**********************************************************************
*/
#ifndef __CSRSBCS_H
#define __CSRSBCS_H
#include "unicode/uobject.h"
#if !UCONFIG_NO_CONVERSION
#include "csrecog.h"
U_NAMESPACE_BEGIN
class NGramParser : public UMemory
{
private:
int32_t ngram;
const int32_t *ngramList;
int32_t ngramCount;
int32_t hitCount;
protected:
int32_t byteIndex;
const uint8_t *charMap;
void addByte(int32_t b);
public:
NGramParser(const int32_t *theNgramList, const uint8_t *theCharMap);
virtual ~NGramParser();
private:
/*
* Binary search for value in table, which must have exactly 64 entries.
*/
int32_t search(const int32_t *table, int32_t value);
void lookup(int32_t thisNgram);
virtual int32_t nextByte(InputText *det);
virtual void parseCharacters(InputText *det);
public:
int32_t parse(InputText *det);
};
#if !UCONFIG_ONLY_HTML_CONVERSION
class NGramParser_IBM420 : public NGramParser
{
public:
NGramParser_IBM420(const int32_t *theNgramList, const uint8_t *theCharMap);
~NGramParser_IBM420();
private:
int32_t alef;
int32_t isLamAlef(int32_t b);
int32_t nextByte(InputText *det) override;
void parseCharacters(InputText *det) override;
};
#endif
class CharsetRecog_sbcs : public CharsetRecognizer
{
public:
CharsetRecog_sbcs();
virtual ~CharsetRecog_sbcs();
virtual const char *getName() const override = 0;
virtual UBool match(InputText *det, CharsetMatch *results) const override = 0;
virtual int32_t match_sbcs(InputText *det, const int32_t ngrams[], const uint8_t charMap[]) const;
};
class CharsetRecog_8859_1 : public CharsetRecog_sbcs
{
public:
virtual ~CharsetRecog_8859_1();
const char *getName() const override;
virtual UBool match(InputText *det, CharsetMatch *results) const override;
};
class CharsetRecog_8859_2 : public CharsetRecog_sbcs
{
public:
virtual ~CharsetRecog_8859_2();
const char *getName() const override;
virtual UBool match(InputText *det, CharsetMatch *results) const override;
};
class CharsetRecog_8859_5 : public CharsetRecog_sbcs
{
public:
virtual ~CharsetRecog_8859_5();
const char *getName() const override;
};
class CharsetRecog_8859_6 : public CharsetRecog_sbcs
{
public:
virtual ~CharsetRecog_8859_6();
const char *getName() const override;
};
class CharsetRecog_8859_7 : public CharsetRecog_sbcs
{
public:
virtual ~CharsetRecog_8859_7();
const char *getName() const override;
};
class CharsetRecog_8859_8 : public CharsetRecog_sbcs
{
public:
virtual ~CharsetRecog_8859_8();
virtual const char *getName() const override;
};
class CharsetRecog_8859_9 : public CharsetRecog_sbcs
{
public:
virtual ~CharsetRecog_8859_9();
const char *getName() const override;
};
class CharsetRecog_8859_5_ru : public CharsetRecog_8859_5
{
public:
virtual ~CharsetRecog_8859_5_ru();
const char *getLanguage() const override;
virtual UBool match(InputText *det, CharsetMatch *results) const override;
};
class CharsetRecog_8859_6_ar : public CharsetRecog_8859_6
{
public:
virtual ~CharsetRecog_8859_6_ar();
const char *getLanguage() const override;
virtual UBool match(InputText *det, CharsetMatch *results) const override;
};
class CharsetRecog_8859_7_el : public CharsetRecog_8859_7
{
public:
virtual ~CharsetRecog_8859_7_el();
const char *getLanguage() const override;
virtual UBool match(InputText *det, CharsetMatch *results) const override;
};
class CharsetRecog_8859_8_I_he : public CharsetRecog_8859_8
{
public:
virtual ~CharsetRecog_8859_8_I_he();
const char *getName() const override;
const char *getLanguage() const override;
virtual UBool match(InputText *det, CharsetMatch *results) const override;
};
class CharsetRecog_8859_8_he : public CharsetRecog_8859_8
{
public:
virtual ~CharsetRecog_8859_8_he ();
const char *getLanguage() const override;
virtual UBool match(InputText *det, CharsetMatch *results) const override;
};
class CharsetRecog_8859_9_tr : public CharsetRecog_8859_9
{
public:
virtual ~CharsetRecog_8859_9_tr ();
const char *getLanguage() const override;
virtual UBool match(InputText *det, CharsetMatch *results) const override;
};
class CharsetRecog_windows_1256 : public CharsetRecog_sbcs
{
public:
virtual ~CharsetRecog_windows_1256();
const char *getName() const override;
const char *getLanguage() const override;
virtual UBool match(InputText *det, CharsetMatch *results) const override;
};
class CharsetRecog_windows_1251 : public CharsetRecog_sbcs
{
public:
virtual ~CharsetRecog_windows_1251();
const char *getName() const override;
const char *getLanguage() const override;
virtual UBool match(InputText *det, CharsetMatch *results) const override;
};
class CharsetRecog_KOI8_R : public CharsetRecog_sbcs
{
public:
virtual ~CharsetRecog_KOI8_R();
const char *getName() const override;
const char *getLanguage() const override;
virtual UBool match(InputText *det, CharsetMatch *results) const override;
};
#if !UCONFIG_ONLY_HTML_CONVERSION
class CharsetRecog_IBM424_he : public CharsetRecog_sbcs
{
public:
virtual ~CharsetRecog_IBM424_he();
const char *getLanguage() const override;
};
class CharsetRecog_IBM424_he_rtl : public CharsetRecog_IBM424_he {
public:
virtual ~CharsetRecog_IBM424_he_rtl();
const char *getName() const override;
virtual UBool match(InputText *det, CharsetMatch *results) const override;
};
class CharsetRecog_IBM424_he_ltr : public CharsetRecog_IBM424_he {
virtual ~CharsetRecog_IBM424_he_ltr();
const char *getName() const override;
virtual UBool match(InputText *det, CharsetMatch *results) const override;
};
class CharsetRecog_IBM420_ar : public CharsetRecog_sbcs
{
public:
virtual ~CharsetRecog_IBM420_ar();
const char *getLanguage() const override;
int32_t match_sbcs(InputText *det, const int32_t ngrams[], const uint8_t charMap[]) const override;
};
class CharsetRecog_IBM420_ar_rtl : public CharsetRecog_IBM420_ar {
public:
virtual ~CharsetRecog_IBM420_ar_rtl();
const char *getName() const override;
virtual UBool match(InputText *det, CharsetMatch *results) const override;
};
class CharsetRecog_IBM420_ar_ltr : public CharsetRecog_IBM420_ar {
virtual ~CharsetRecog_IBM420_ar_ltr();
const char *getName() const override;
virtual UBool match(InputText *det, CharsetMatch *results) const override;
};
#endif
U_NAMESPACE_END
#endif /* !UCONFIG_NO_CONVERSION */
#endif /* __CSRSBCS_H */

200
deps/icu-small/source/i18n/csrucode.cpp vendored Normal file
View File

@ -0,0 +1,200 @@
// © 2016 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html
/*
**********************************************************************
* Copyright (C) 2005-2013, International Business Machines
* Corporation and others. All Rights Reserved.
**********************************************************************
*/
#include "unicode/utypes.h"
#if !UCONFIG_NO_CONVERSION
#include "csrucode.h"
#include "csmatch.h"
U_NAMESPACE_BEGIN
CharsetRecog_Unicode::~CharsetRecog_Unicode()
{
// nothing to do
}
CharsetRecog_UTF_16_BE::~CharsetRecog_UTF_16_BE()
{
// nothing to do
}
const char *CharsetRecog_UTF_16_BE::getName() const
{
return "UTF-16BE";
}
// UTF-16 confidence calculation. Very simple minded, but better than nothing.
// Any 8 bit non-control characters bump the confidence up. These have a zero high byte,
// and are very likely to be UTF-16, although they could also be part of a UTF-32 code.
// NULs are a contra-indication, they will appear commonly if the actual encoding is UTF-32.
// NULs should be rare in actual text.
static int32_t adjustConfidence(char16_t codeUnit, int32_t confidence) {
if (codeUnit == 0) {
confidence -= 10;
} else if ((codeUnit >= 0x20 && codeUnit <= 0xff) || codeUnit == 0x0a) {
confidence += 10;
}
if (confidence < 0) {
confidence = 0;
} else if (confidence > 100) {
confidence = 100;
}
return confidence;
}
UBool CharsetRecog_UTF_16_BE::match(InputText* textIn, CharsetMatch *results) const
{
const uint8_t *input = textIn->fRawInput;
int32_t confidence = 10;
int32_t length = textIn->fRawLength;
int32_t bytesToCheck = (length > 30) ? 30 : length;
for (int32_t charIndex=0; charIndex<bytesToCheck-1; charIndex+=2) {
char16_t codeUnit = (input[charIndex] << 8) | input[charIndex + 1];
if (charIndex == 0 && codeUnit == 0xFEFF) {
confidence = 100;
break;
}
confidence = adjustConfidence(codeUnit, confidence);
if (confidence == 0 || confidence == 100) {
break;
}
}
if (bytesToCheck < 4 && confidence < 100) {
confidence = 0;
}
results->set(textIn, this, confidence);
return (confidence > 0);
}
CharsetRecog_UTF_16_LE::~CharsetRecog_UTF_16_LE()
{
// nothing to do
}
const char *CharsetRecog_UTF_16_LE::getName() const
{
return "UTF-16LE";
}
UBool CharsetRecog_UTF_16_LE::match(InputText* textIn, CharsetMatch *results) const
{
const uint8_t *input = textIn->fRawInput;
int32_t confidence = 10;
int32_t length = textIn->fRawLength;
int32_t bytesToCheck = (length > 30) ? 30 : length;
for (int32_t charIndex=0; charIndex<bytesToCheck-1; charIndex+=2) {
char16_t codeUnit = input[charIndex] | (input[charIndex + 1] << 8);
if (charIndex == 0 && codeUnit == 0xFEFF) {
confidence = 100; // UTF-16 BOM
if (length >= 4 && input[2] == 0 && input[3] == 0) {
confidence = 0; // UTF-32 BOM
}
break;
}
confidence = adjustConfidence(codeUnit, confidence);
if (confidence == 0 || confidence == 100) {
break;
}
}
if (bytesToCheck < 4 && confidence < 100) {
confidence = 0;
}
results->set(textIn, this, confidence);
return (confidence > 0);
}
CharsetRecog_UTF_32::~CharsetRecog_UTF_32()
{
// nothing to do
}
UBool CharsetRecog_UTF_32::match(InputText* textIn, CharsetMatch *results) const
{
const uint8_t *input = textIn->fRawInput;
int32_t limit = (textIn->fRawLength / 4) * 4;
int32_t numValid = 0;
int32_t numInvalid = 0;
bool hasBOM = false;
int32_t confidence = 0;
if (limit > 0 && getChar(input, 0) == 0x0000FEFFUL) {
hasBOM = true;
}
for(int32_t i = 0; i < limit; i += 4) {
int32_t ch = getChar(input, i);
if (ch < 0 || ch >= 0x10FFFF || (ch >= 0xD800 && ch <= 0xDFFF)) {
numInvalid += 1;
} else {
numValid += 1;
}
}
// Cook up some sort of confidence score, based on presence of a BOM
// and the existence of valid and/or invalid multi-byte sequences.
if (hasBOM && numInvalid==0) {
confidence = 100;
} else if (hasBOM && numValid > numInvalid*10) {
confidence = 80;
} else if (numValid > 3 && numInvalid == 0) {
confidence = 100;
} else if (numValid > 0 && numInvalid == 0) {
confidence = 80;
} else if (numValid > numInvalid*10) {
// Probably corrupt UTF-32BE data. Valid sequences aren't likely by chance.
confidence = 25;
}
results->set(textIn, this, confidence);
return (confidence > 0);
}
CharsetRecog_UTF_32_BE::~CharsetRecog_UTF_32_BE()
{
// nothing to do
}
const char *CharsetRecog_UTF_32_BE::getName() const
{
return "UTF-32BE";
}
int32_t CharsetRecog_UTF_32_BE::getChar(const uint8_t *input, int32_t index) const
{
return input[index + 0] << 24 | input[index + 1] << 16 |
input[index + 2] << 8 | input[index + 3];
}
CharsetRecog_UTF_32_LE::~CharsetRecog_UTF_32_LE()
{
// nothing to do
}
const char *CharsetRecog_UTF_32_LE::getName() const
{
return "UTF-32LE";
}
int32_t CharsetRecog_UTF_32_LE::getChar(const uint8_t *input, int32_t index) const
{
return input[index + 3] << 24 | input[index + 2] << 16 |
input[index + 1] << 8 | input[index + 0];
}
U_NAMESPACE_END
#endif

108
deps/icu-small/source/i18n/csrucode.h vendored Normal file
View File

@ -0,0 +1,108 @@
// © 2016 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html
/*
**********************************************************************
* Copyright (C) 2005-2012, International Business Machines
* Corporation and others. All Rights Reserved.
**********************************************************************
*/
#ifndef __CSRUCODE_H
#define __CSRUCODE_H
#include "unicode/utypes.h"
#if !UCONFIG_NO_CONVERSION
#include "csrecog.h"
U_NAMESPACE_BEGIN
/**
* This class matches UTF-16 and UTF-32, both big- and little-endian. The
* BOM will be used if it is present.
*
* @internal
*/
class CharsetRecog_Unicode : public CharsetRecognizer
{
public:
virtual ~CharsetRecog_Unicode();
/* (non-Javadoc)
* @see com.ibm.icu.text.CharsetRecognizer#getName()
*/
const char* getName() const override = 0;
/* (non-Javadoc)
* @see com.ibm.icu.text.CharsetRecognizer#match(com.ibm.icu.text.CharsetDetector)
*/
UBool match(InputText* textIn, CharsetMatch *results) const override = 0;
};
class CharsetRecog_UTF_16_BE : public CharsetRecog_Unicode
{
public:
virtual ~CharsetRecog_UTF_16_BE();
const char *getName() const override;
UBool match(InputText* textIn, CharsetMatch *results) const override;
};
class CharsetRecog_UTF_16_LE : public CharsetRecog_Unicode
{
public:
virtual ~CharsetRecog_UTF_16_LE();
const char *getName() const override;
UBool match(InputText* textIn, CharsetMatch *results) const override;
};
class CharsetRecog_UTF_32 : public CharsetRecog_Unicode
{
protected:
virtual int32_t getChar(const uint8_t *input, int32_t index) const = 0;
public:
virtual ~CharsetRecog_UTF_32();
const char* getName() const override = 0;
UBool match(InputText* textIn, CharsetMatch *results) const override;
};
class CharsetRecog_UTF_32_BE : public CharsetRecog_UTF_32
{
protected:
int32_t getChar(const uint8_t *input, int32_t index) const override;
public:
virtual ~CharsetRecog_UTF_32_BE();
const char *getName() const override;
};
class CharsetRecog_UTF_32_LE : public CharsetRecog_UTF_32
{
protected:
int32_t getChar(const uint8_t *input, int32_t index) const override;
public:
virtual ~CharsetRecog_UTF_32_LE();
const char* getName() const override;
};
U_NAMESPACE_END
#endif
#endif /* __CSRUCODE_H */

111
deps/icu-small/source/i18n/csrutf8.cpp vendored Normal file
View File

@ -0,0 +1,111 @@
// © 2016 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html
/*
**********************************************************************
* Copyright (C) 2005-2014, International Business Machines
* Corporation and others. All Rights Reserved.
**********************************************************************
*/
#include "unicode/utypes.h"
#if !UCONFIG_NO_CONVERSION
#include "csrutf8.h"
#include "csmatch.h"
U_NAMESPACE_BEGIN
CharsetRecog_UTF8::~CharsetRecog_UTF8()
{
// nothing to do
}
const char *CharsetRecog_UTF8::getName() const
{
return "UTF-8";
}
UBool CharsetRecog_UTF8::match(InputText* input, CharsetMatch *results) const {
bool hasBOM = false;
int32_t numValid = 0;
int32_t numInvalid = 0;
const uint8_t *inputBytes = input->fRawInput;
int32_t i;
int32_t trailBytes = 0;
int32_t confidence;
if (input->fRawLength >= 3 &&
inputBytes[0] == 0xEF && inputBytes[1] == 0xBB && inputBytes[2] == 0xBF) {
hasBOM = true;
}
// Scan for multi-byte sequences
for (i=0; i < input->fRawLength; i += 1) {
int32_t b = inputBytes[i];
if ((b & 0x80) == 0) {
continue; // ASCII
}
// Hi bit on char found. Figure out how long the sequence should be
if ((b & 0x0E0) == 0x0C0) {
trailBytes = 1;
} else if ((b & 0x0F0) == 0x0E0) {
trailBytes = 2;
} else if ((b & 0x0F8) == 0xF0) {
trailBytes = 3;
} else {
numInvalid += 1;
continue;
}
// Verify that we've got the right number of trail bytes in the sequence
for (;;) {
i += 1;
if (i >= input->fRawLength) {
break;
}
b = inputBytes[i];
if ((b & 0xC0) != 0x080) {
numInvalid += 1;
break;
}
if (--trailBytes == 0) {
numValid += 1;
break;
}
}
}
// Cook up some sort of confidence score, based on presence of a BOM
// and the existence of valid and/or invalid multi-byte sequences.
confidence = 0;
if (hasBOM && numInvalid == 0) {
confidence = 100;
} else if (hasBOM && numValid > numInvalid*10) {
confidence = 80;
} else if (numValid > 3 && numInvalid == 0) {
confidence = 100;
} else if (numValid > 0 && numInvalid == 0) {
confidence = 80;
} else if (numValid == 0 && numInvalid == 0) {
// Plain ASCII. Confidence must be > 10, it's more likely than UTF-16, which
// accepts ASCII with confidence = 10.
confidence = 15;
} else if (numValid > numInvalid*10) {
// Probably corrupt utf-8 data. Valid sequences aren't likely by chance.
confidence = 25;
}
results->set(input, this, confidence);
return (confidence > 0);
}
U_NAMESPACE_END
#endif

44
deps/icu-small/source/i18n/csrutf8.h vendored Normal file
View File

@ -0,0 +1,44 @@
// © 2016 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html
/*
**********************************************************************
* Copyright (C) 2005-2012, International Business Machines
* Corporation and others. All Rights Reserved.
**********************************************************************
*/
#ifndef __CSRUTF8_H
#define __CSRUTF8_H
#include "unicode/utypes.h"
#if !UCONFIG_NO_CONVERSION
#include "csrecog.h"
U_NAMESPACE_BEGIN
/**
* Charset recognizer for UTF-8
*
* @internal
*/
class CharsetRecog_UTF8: public CharsetRecognizer {
public:
virtual ~CharsetRecog_UTF8();
const char *getName() const override;
/* (non-Javadoc)
* @see com.ibm.icu.text.CharsetRecognizer#match(com.ibm.icu.text.CharsetDetector)
*/
UBool match(InputText *input, CharsetMatch *results) const override;
};
U_NAMESPACE_END
#endif
#endif /* __CSRUTF8_H */

56
deps/icu-small/source/i18n/curramt.cpp vendored Normal file
View File

@ -0,0 +1,56 @@
// © 2016 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html
/*
**********************************************************************
* Copyright (c) 2004, International Business Machines
* Corporation and others. All Rights Reserved.
**********************************************************************
* Author: Alan Liu
* Created: April 26, 2004
* Since: ICU 3.0
**********************************************************************
*/
#include "unicode/utypes.h"
#if !UCONFIG_NO_FORMATTING
#include "unicode/curramt.h"
#include "unicode/currunit.h"
U_NAMESPACE_BEGIN
CurrencyAmount::CurrencyAmount(const Formattable& amount, ConstChar16Ptr isoCode,
UErrorCode& ec) :
Measure(amount, new CurrencyUnit(isoCode, ec), ec) {
}
CurrencyAmount::CurrencyAmount(double amount, ConstChar16Ptr isoCode,
UErrorCode& ec) :
Measure(Formattable(amount), new CurrencyUnit(isoCode, ec), ec) {
}
CurrencyAmount::CurrencyAmount(const CurrencyAmount& other) :
Measure(other) {
}
CurrencyAmount& CurrencyAmount::operator=(const CurrencyAmount& other) {
Measure::operator=(other);
return *this;
}
CurrencyAmount* CurrencyAmount::clone() const {
return new CurrencyAmount(*this);
}
CurrencyAmount::~CurrencyAmount() {
}
const CurrencyUnit& CurrencyAmount::getCurrency() const {
return static_cast<const CurrencyUnit&>(getUnit());
}
UOBJECT_DEFINE_RTTI_IMPLEMENTATION(CurrencyAmount)
U_NAMESPACE_END
#endif // !UCONFIG_NO_FORMATTING

62
deps/icu-small/source/i18n/currfmt.cpp vendored Normal file
View File

@ -0,0 +1,62 @@
// © 2016 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html
/*
**********************************************************************
* Copyright (c) 2004-2014 International Business Machines
* Corporation and others. All Rights Reserved.
**********************************************************************
* Author: Alan Liu
* Created: April 20, 2004
* Since: ICU 3.0
**********************************************************************
*/
#include "unicode/utypes.h"
#if !UCONFIG_NO_FORMATTING
#include "currfmt.h"
#include "unicode/numfmt.h"
#include "unicode/curramt.h"
U_NAMESPACE_BEGIN
CurrencyFormat::CurrencyFormat(const Locale& locale, UErrorCode& ec) :
MeasureFormat(locale, UMEASFMT_WIDTH_WIDE, ec)
{
}
CurrencyFormat::CurrencyFormat(const CurrencyFormat& other) :
MeasureFormat(other)
{
}
CurrencyFormat::~CurrencyFormat() {
}
CurrencyFormat* CurrencyFormat::clone() const {
return new CurrencyFormat(*this);
}
UnicodeString& CurrencyFormat::format(const Formattable& obj,
UnicodeString& appendTo,
FieldPosition& pos,
UErrorCode& ec) const
{
return getCurrencyFormatInternal().format(obj, appendTo, pos, ec);
}
void CurrencyFormat::parseObject(const UnicodeString& source,
Formattable& result,
ParsePosition& pos) const
{
CurrencyAmount* currAmt = getCurrencyFormatInternal().parseCurrency(source, pos);
if (currAmt != nullptr) {
result.adoptObject(currAmt);
}
}
UOBJECT_DEFINE_RTTI_IMPLEMENTATION(CurrencyFormat)
U_NAMESPACE_END
#endif /* #if !UCONFIG_NO_FORMATTING */

94
deps/icu-small/source/i18n/currfmt.h vendored Normal file
View File

@ -0,0 +1,94 @@
// © 2016 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html
/*
**********************************************************************
* Copyright (c) 2004-2014, International Business Machines
* Corporation and others. All Rights Reserved.
**********************************************************************
* Author: Alan Liu
* Created: April 20, 2004
* Since: ICU 3.0
**********************************************************************
*/
#ifndef CURRENCYFORMAT_H
#define CURRENCYFORMAT_H
#include "unicode/utypes.h"
#if !UCONFIG_NO_FORMATTING
#include "unicode/measfmt.h"
U_NAMESPACE_BEGIN
class NumberFormat;
/**
* Temporary internal concrete subclass of MeasureFormat implementing
* parsing and formatting of currency amount objects. This class is
* likely to be redesigned and rewritten in the near future.
*
* <p>This class currently delegates to DecimalFormat for parsing and
* formatting.
*
* @see MeasureFormat
* @author Alan Liu
* @internal
*/
class CurrencyFormat : public MeasureFormat {
public:
/**
* Construct a CurrencyFormat for the given locale.
*/
CurrencyFormat(const Locale& locale, UErrorCode& ec);
/**
* Copy constructor.
*/
CurrencyFormat(const CurrencyFormat& other);
/**
* Destructor.
*/
virtual ~CurrencyFormat();
/**
* Override Format API.
*/
virtual CurrencyFormat* clone() const override;
using MeasureFormat::format;
/**
* Override Format API.
*/
virtual UnicodeString& format(const Formattable& obj,
UnicodeString& appendTo,
FieldPosition& pos,
UErrorCode& ec) const override;
/**
* Override Format API.
*/
virtual void parseObject(const UnicodeString& source,
Formattable& result,
ParsePosition& pos) const override;
/**
* Override Format API.
*/
virtual UClassID getDynamicClassID() const override;
/**
* Returns the class ID for this class.
*/
static UClassID U_EXPORT2 getStaticClassID();
};
U_NAMESPACE_END
#endif // #if !UCONFIG_NO_FORMATTING
#endif // #ifndef CURRENCYFORMAT_H

441
deps/icu-small/source/i18n/currpinf.cpp vendored Normal file
View File

@ -0,0 +1,441 @@
// © 2016 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html
/*
*******************************************************************************
* Copyright (C) 2009-2014, International Business Machines Corporation and
* others. All Rights Reserved.
*******************************************************************************
*/
#include "unicode/currpinf.h"
#if !UCONFIG_NO_FORMATTING
//#define CURRENCY_PLURAL_INFO_DEBUG 1
#ifdef CURRENCY_PLURAL_INFO_DEBUG
#include <iostream>
#endif
#include "unicode/locid.h"
#include "unicode/plurrule.h"
#include "unicode/strenum.h"
#include "unicode/ures.h"
#include "unicode/numsys.h"
#include "cstring.h"
#include "hash.h"
#include "uresimp.h"
#include "ureslocs.h"
U_NAMESPACE_BEGIN
static const char16_t gNumberPatternSeparator = 0x3B; // ;
U_CDECL_BEGIN
/**
* @internal ICU 4.2
*/
static UBool U_CALLCONV ValueComparator(UHashTok val1, UHashTok val2);
UBool
U_CALLCONV ValueComparator(UHashTok val1, UHashTok val2) {
const UnicodeString* affix_1 = (UnicodeString*)val1.pointer;
const UnicodeString* affix_2 = (UnicodeString*)val2.pointer;
return *affix_1 == *affix_2;
}
U_CDECL_END
UOBJECT_DEFINE_RTTI_IMPLEMENTATION(CurrencyPluralInfo)
static const char16_t gDefaultCurrencyPluralPattern[] = {'0', '.', '#', '#', ' ', 0xA4, 0xA4, 0xA4, 0};
static const char16_t gTripleCurrencySign[] = {0xA4, 0xA4, 0xA4, 0};
static const char16_t gPluralCountOther[] = {0x6F, 0x74, 0x68, 0x65, 0x72, 0};
static const char16_t gPart0[] = {0x7B, 0x30, 0x7D, 0};
static const char16_t gPart1[] = {0x7B, 0x31, 0x7D, 0};
static const char gNumberElementsTag[]="NumberElements";
static const char gLatnTag[]="latn";
static const char gPatternsTag[]="patterns";
static const char gDecimalFormatTag[]="decimalFormat";
static const char gCurrUnitPtnTag[]="CurrencyUnitPatterns";
CurrencyPluralInfo::CurrencyPluralInfo(UErrorCode& status)
: fPluralCountToCurrencyUnitPattern(nullptr),
fPluralRules(nullptr),
fLocale(nullptr),
fInternalStatus(U_ZERO_ERROR) {
initialize(Locale::getDefault(), status);
}
CurrencyPluralInfo::CurrencyPluralInfo(const Locale& locale, UErrorCode& status)
: fPluralCountToCurrencyUnitPattern(nullptr),
fPluralRules(nullptr),
fLocale(nullptr),
fInternalStatus(U_ZERO_ERROR) {
initialize(locale, status);
}
CurrencyPluralInfo::CurrencyPluralInfo(const CurrencyPluralInfo& info)
: UObject(info),
fPluralCountToCurrencyUnitPattern(nullptr),
fPluralRules(nullptr),
fLocale(nullptr),
fInternalStatus(U_ZERO_ERROR) {
*this = info;
}
CurrencyPluralInfo&
CurrencyPluralInfo::operator=(const CurrencyPluralInfo& info) {
if (this == &info) {
return *this;
}
fInternalStatus = info.fInternalStatus;
if (U_FAILURE(fInternalStatus)) {
// bail out early if the object we were copying from was already 'invalid'.
return *this;
}
deleteHash(fPluralCountToCurrencyUnitPattern);
fPluralCountToCurrencyUnitPattern = initHash(fInternalStatus);
copyHash(info.fPluralCountToCurrencyUnitPattern,
fPluralCountToCurrencyUnitPattern, fInternalStatus);
if ( U_FAILURE(fInternalStatus) ) {
return *this;
}
delete fPluralRules;
fPluralRules = nullptr;
delete fLocale;
fLocale = nullptr;
if (info.fPluralRules != nullptr) {
fPluralRules = info.fPluralRules->clone();
if (fPluralRules == nullptr) {
fInternalStatus = U_MEMORY_ALLOCATION_ERROR;
return *this;
}
}
if (info.fLocale != nullptr) {
fLocale = info.fLocale->clone();
if (fLocale == nullptr) {
// Note: If clone had an error parameter, then we could check/set that instead.
fInternalStatus = U_MEMORY_ALLOCATION_ERROR;
return *this;
}
// If the other locale wasn't bogus, but our clone'd locale is bogus, then OOM happened
// during the call to clone().
if (!info.fLocale->isBogus() && fLocale->isBogus()) {
fInternalStatus = U_MEMORY_ALLOCATION_ERROR;
return *this;
}
}
return *this;
}
CurrencyPluralInfo::~CurrencyPluralInfo() {
deleteHash(fPluralCountToCurrencyUnitPattern);
fPluralCountToCurrencyUnitPattern = nullptr;
delete fPluralRules;
delete fLocale;
fPluralRules = nullptr;
fLocale = nullptr;
}
bool
CurrencyPluralInfo::operator==(const CurrencyPluralInfo& info) const {
#ifdef CURRENCY_PLURAL_INFO_DEBUG
if (*fPluralRules == *info.fPluralRules) {
std::cout << "same plural rules\n";
}
if (*fLocale == *info.fLocale) {
std::cout << "same locale\n";
}
if (fPluralCountToCurrencyUnitPattern->equals(*info.fPluralCountToCurrencyUnitPattern)) {
std::cout << "same pattern\n";
}
#endif
return *fPluralRules == *info.fPluralRules &&
*fLocale == *info.fLocale &&
fPluralCountToCurrencyUnitPattern->equals(*info.fPluralCountToCurrencyUnitPattern);
}
CurrencyPluralInfo*
CurrencyPluralInfo::clone() const {
CurrencyPluralInfo* newObj = new CurrencyPluralInfo(*this);
// Since clone doesn't have a 'status' parameter, the best we can do is return nullptr
// if the new object was not full constructed properly (an error occurred).
if (newObj != nullptr && U_FAILURE(newObj->fInternalStatus)) {
delete newObj;
newObj = nullptr;
}
return newObj;
}
const PluralRules*
CurrencyPluralInfo::getPluralRules() const {
return fPluralRules;
}
UnicodeString&
CurrencyPluralInfo::getCurrencyPluralPattern(const UnicodeString& pluralCount,
UnicodeString& result) const {
const UnicodeString* currencyPluralPattern =
static_cast<UnicodeString*>(fPluralCountToCurrencyUnitPattern->get(pluralCount));
if (currencyPluralPattern == nullptr) {
// fall back to "other"
if (pluralCount.compare(gPluralCountOther, 5)) {
currencyPluralPattern =
static_cast<UnicodeString*>(fPluralCountToCurrencyUnitPattern->get(UnicodeString(true, gPluralCountOther, 5)));
}
if (currencyPluralPattern == nullptr) {
// no currencyUnitPatterns defined,
// fallback to predefined default.
// This should never happen when ICU resource files are
// available, since currencyUnitPattern of "other" is always
// defined in root.
result = UnicodeString(gDefaultCurrencyPluralPattern);
return result;
}
}
result = *currencyPluralPattern;
return result;
}
const Locale&
CurrencyPluralInfo::getLocale() const {
return *fLocale;
}
void
CurrencyPluralInfo::setPluralRules(const UnicodeString& ruleDescription,
UErrorCode& status) {
if (U_SUCCESS(status)) {
delete fPluralRules;
fPluralRules = PluralRules::createRules(ruleDescription, status);
}
}
void
CurrencyPluralInfo::setCurrencyPluralPattern(const UnicodeString& pluralCount,
const UnicodeString& pattern,
UErrorCode& status) {
if (U_SUCCESS(status)) {
UnicodeString* oldValue = static_cast<UnicodeString*>(
fPluralCountToCurrencyUnitPattern->get(pluralCount));
delete oldValue;
LocalPointer<UnicodeString> p(new UnicodeString(pattern), status);
if (U_SUCCESS(status)) {
// the p object allocated above will be owned by fPluralCountToCurrencyUnitPattern
// after the call to put(), even if the method returns failure.
fPluralCountToCurrencyUnitPattern->put(pluralCount, p.orphan(), status);
}
}
}
void
CurrencyPluralInfo::setLocale(const Locale& loc, UErrorCode& status) {
initialize(loc, status);
}
void
CurrencyPluralInfo::initialize(const Locale& loc, UErrorCode& status) {
if (U_FAILURE(status)) {
return;
}
delete fLocale;
fLocale = nullptr;
delete fPluralRules;
fPluralRules = nullptr;
fLocale = loc.clone();
if (fLocale == nullptr) {
status = U_MEMORY_ALLOCATION_ERROR;
return;
}
// If the locale passed in wasn't bogus, but our clone'd locale is bogus, then OOM happened
// during the call to loc.clone().
if (!loc.isBogus() && fLocale->isBogus()) {
status = U_MEMORY_ALLOCATION_ERROR;
return;
}
fPluralRules = PluralRules::forLocale(loc, status);
setupCurrencyPluralPattern(loc, status);
}
void
CurrencyPluralInfo::setupCurrencyPluralPattern(const Locale& loc, UErrorCode& status) {
if (U_FAILURE(status)) {
return;
}
deleteHash(fPluralCountToCurrencyUnitPattern);
fPluralCountToCurrencyUnitPattern = initHash(status);
if (U_FAILURE(status)) {
return;
}
LocalPointer<NumberingSystem> ns(NumberingSystem::createInstance(loc, status), status);
if (U_FAILURE(status)) {
return;
}
UErrorCode ec = U_ZERO_ERROR;
LocalUResourceBundlePointer rb(ures_open(nullptr, loc.getName(), &ec));
LocalUResourceBundlePointer numElements(ures_getByKeyWithFallback(rb.getAlias(), gNumberElementsTag, nullptr, &ec));
ures_getByKeyWithFallback(numElements.getAlias(), ns->getName(), rb.getAlias(), &ec);
ures_getByKeyWithFallback(rb.getAlias(), gPatternsTag, rb.getAlias(), &ec);
int32_t ptnLen;
const char16_t* numberStylePattern = ures_getStringByKeyWithFallback(rb.getAlias(), gDecimalFormatTag, &ptnLen, &ec);
// Fall back to "latn" if num sys specific pattern isn't there.
if ( ec == U_MISSING_RESOURCE_ERROR && (uprv_strcmp(ns->getName(), gLatnTag) != 0)) {
ec = U_ZERO_ERROR;
ures_getByKeyWithFallback(numElements.getAlias(), gLatnTag, rb.getAlias(), &ec);
ures_getByKeyWithFallback(rb.getAlias(), gPatternsTag, rb.getAlias(), &ec);
numberStylePattern = ures_getStringByKeyWithFallback(rb.getAlias(), gDecimalFormatTag, &ptnLen, &ec);
}
int32_t numberStylePatternLen = ptnLen;
const char16_t* negNumberStylePattern = nullptr;
int32_t negNumberStylePatternLen = 0;
// TODO: Java
// parse to check whether there is ";" separator in the numberStylePattern
UBool hasSeparator = false;
if (U_SUCCESS(ec)) {
for (int32_t styleCharIndex = 0; styleCharIndex < ptnLen; ++styleCharIndex) {
if (numberStylePattern[styleCharIndex] == gNumberPatternSeparator) {
hasSeparator = true;
// split the number style pattern into positive and negative
negNumberStylePattern = numberStylePattern + styleCharIndex + 1;
negNumberStylePatternLen = ptnLen - styleCharIndex - 1;
numberStylePatternLen = styleCharIndex;
}
}
}
if (U_FAILURE(ec)) {
// If OOM occurred during the above code, then we want to report that back to the caller.
if (ec == U_MEMORY_ALLOCATION_ERROR) {
status = ec;
}
return;
}
LocalUResourceBundlePointer currRb(ures_open(U_ICUDATA_CURR, loc.getName(), &ec));
LocalUResourceBundlePointer currencyRes(ures_getByKeyWithFallback(currRb.getAlias(), gCurrUnitPtnTag, nullptr, &ec));
#ifdef CURRENCY_PLURAL_INFO_DEBUG
std::cout << "in set up\n";
#endif
LocalPointer<StringEnumeration> keywords(fPluralRules->getKeywords(ec), ec);
if (U_SUCCESS(ec)) {
const char* pluralCount;
while (((pluralCount = keywords->next(nullptr, ec)) != nullptr) && U_SUCCESS(ec)) {
int32_t ptnLength;
UErrorCode err = U_ZERO_ERROR;
const char16_t* patternChars = ures_getStringByKeyWithFallback(currencyRes.getAlias(), pluralCount, &ptnLength, &err);
if (err == U_MEMORY_ALLOCATION_ERROR || patternChars == nullptr) {
ec = err;
break;
}
if (U_SUCCESS(err) && ptnLength > 0) {
UnicodeString* pattern = new UnicodeString(patternChars, ptnLength);
if (pattern == nullptr) {
ec = U_MEMORY_ALLOCATION_ERROR;
break;
}
#ifdef CURRENCY_PLURAL_INFO_DEBUG
char result_1[1000];
pattern->extract(0, pattern->length(), result_1, "UTF-8");
std::cout << "pluralCount: " << pluralCount << "; pattern: " << result_1 << "\n";
#endif
pattern->findAndReplace(UnicodeString(true, gPart0, 3),
UnicodeString(numberStylePattern, numberStylePatternLen));
pattern->findAndReplace(UnicodeString(true, gPart1, 3), UnicodeString(true, gTripleCurrencySign, 3));
if (hasSeparator) {
UnicodeString negPattern(patternChars, ptnLength);
negPattern.findAndReplace(UnicodeString(true, gPart0, 3),
UnicodeString(negNumberStylePattern, negNumberStylePatternLen));
negPattern.findAndReplace(UnicodeString(true, gPart1, 3), UnicodeString(true, gTripleCurrencySign, 3));
pattern->append(gNumberPatternSeparator);
pattern->append(negPattern);
}
#ifdef CURRENCY_PLURAL_INFO_DEBUG
pattern->extract(0, pattern->length(), result_1, "UTF-8");
std::cout << "pluralCount: " << pluralCount << "; pattern: " << result_1 << "\n";
#endif
// the 'pattern' object allocated above will be owned by the fPluralCountToCurrencyUnitPattern after the call to
// put(), even if the method returns failure.
fPluralCountToCurrencyUnitPattern->put(UnicodeString(pluralCount, -1, US_INV), pattern, status);
}
}
}
// If OOM occurred during the above code, then we want to report that back to the caller.
if (ec == U_MEMORY_ALLOCATION_ERROR) {
status = ec;
}
}
void
CurrencyPluralInfo::deleteHash(Hashtable* hTable) {
if ( hTable == nullptr ) {
return;
}
int32_t pos = UHASH_FIRST;
const UHashElement* element = nullptr;
while ( (element = hTable->nextElement(pos)) != nullptr ) {
const UHashTok valueTok = element->value;
const UnicodeString* value = static_cast<UnicodeString*>(valueTok.pointer);
delete value;
}
delete hTable;
hTable = nullptr;
}
Hashtable*
CurrencyPluralInfo::initHash(UErrorCode& status) {
if (U_FAILURE(status)) {
return nullptr;
}
LocalPointer<Hashtable> hTable(new Hashtable(true, status), status);
if (U_FAILURE(status)) {
return nullptr;
}
hTable->setValueComparator(ValueComparator);
return hTable.orphan();
}
void
CurrencyPluralInfo::copyHash(const Hashtable* source,
Hashtable* target,
UErrorCode& status) {
if (U_FAILURE(status)) {
return;
}
int32_t pos = UHASH_FIRST;
const UHashElement* element = nullptr;
if (source) {
while ( (element = source->nextElement(pos)) != nullptr ) {
const UHashTok keyTok = element->key;
const UnicodeString* key = static_cast<UnicodeString*>(keyTok.pointer);
const UHashTok valueTok = element->value;
const UnicodeString* value = static_cast<UnicodeString*>(valueTok.pointer);
LocalPointer<UnicodeString> copy(new UnicodeString(*value), status);
if (U_FAILURE(status)) {
return;
}
// The HashTable owns the 'copy' object after the call to put().
target->put(UnicodeString(*key), copy.orphan(), status);
if (U_FAILURE(status)) {
return;
}
}
}
}
U_NAMESPACE_END
#endif

126
deps/icu-small/source/i18n/currunit.cpp vendored Normal file
View File

@ -0,0 +1,126 @@
// © 2016 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html
/*
**********************************************************************
* Copyright (c) 2004-2014, International Business Machines
* Corporation and others. All Rights Reserved.
**********************************************************************
* Author: Alan Liu
* Created: April 26, 2004
* Since: ICU 3.0
**********************************************************************
*/
#include "unicode/utypes.h"
#if !UCONFIG_NO_FORMATTING
#include "unicode/currunit.h"
#include "unicode/ustring.h"
#include "unicode/uchar.h"
#include "cstring.h"
#include "uinvchar.h"
#include "charstr.h"
#include "ustr_imp.h"
#include "measunit_impl.h"
U_NAMESPACE_BEGIN
CurrencyUnit::CurrencyUnit(ConstChar16Ptr _isoCode, UErrorCode& ec) {
// The constructor always leaves the CurrencyUnit in a valid state (with a 3-character currency code).
// Note: in ICU4J Currency.getInstance(), we check string length for 3, but in ICU4C we allow a
// non-NUL-terminated string to be passed as an argument, so it is not possible to check length.
// However, we allow a NUL-terminated empty string, which should have the same behavior as nullptr.
// Consider NUL-terminated strings of length 1 or 2 as invalid.
bool useDefault = false;
if (U_FAILURE(ec) || _isoCode == nullptr || _isoCode[0] == 0) {
useDefault = true;
} else if (_isoCode[1] == 0 || _isoCode[2] == 0) {
useDefault = true;
ec = U_ILLEGAL_ARGUMENT_ERROR;
} else if (!uprv_isInvariantUString(_isoCode, 3)) {
// TODO: Perform a more strict ASCII check like in ICU4J isAlpha3Code?
useDefault = true;
ec = U_INVARIANT_CONVERSION_ERROR;
} else {
for (int32_t i=0; i<3; i++) {
isoCode[i] = u_asciiToUpper(_isoCode[i]);
}
isoCode[3] = 0;
}
if (useDefault) {
uprv_memcpy(isoCode, kDefaultCurrency, sizeof(char16_t) * 4);
}
char simpleIsoCode[4];
u_UCharsToChars(isoCode, simpleIsoCode, 4);
initCurrency(simpleIsoCode);
}
CurrencyUnit::CurrencyUnit(StringPiece _isoCode, UErrorCode& ec) {
// Note: unlike the old constructor, reject empty arguments with an error.
char isoCodeBuffer[4];
const char* isoCodeToUse;
// uprv_memchr checks that the string contains no internal NULs
if (_isoCode.length() != 3 || uprv_memchr(_isoCode.data(), 0, 3) != nullptr) {
isoCodeToUse = kDefaultCurrency8;
ec = U_ILLEGAL_ARGUMENT_ERROR;
} else if (!uprv_isInvariantString(_isoCode.data(), 3)) {
// TODO: Perform a more strict ASCII check like in ICU4J isAlpha3Code?
isoCodeToUse = kDefaultCurrency8;
ec = U_INVARIANT_CONVERSION_ERROR;
} else {
// Have to use isoCodeBuffer to ensure the string is NUL-terminated
for (int32_t i=0; i<3; i++) {
isoCodeBuffer[i] = uprv_toupper(_isoCode.data()[i]);
}
isoCodeBuffer[3] = 0;
isoCodeToUse = isoCodeBuffer;
}
u_charsToUChars(isoCodeToUse, isoCode, 4);
initCurrency(isoCodeToUse);
}
CurrencyUnit::CurrencyUnit(const CurrencyUnit& other) : MeasureUnit(other) {
u_strcpy(isoCode, other.isoCode);
}
CurrencyUnit::CurrencyUnit(const MeasureUnit& other, UErrorCode& ec) : MeasureUnit(other) {
// Make sure this is a currency.
// OK to hard-code the string because we are comparing against another hard-coded string.
if (uprv_strcmp("currency", getType()) != 0) {
ec = U_ILLEGAL_ARGUMENT_ERROR;
isoCode[0] = 0;
} else {
// Get the ISO Code from the subtype field.
u_charsToUChars(getSubtype(), isoCode, 4);
isoCode[3] = 0; // make 100% sure it is NUL-terminated
}
}
CurrencyUnit::CurrencyUnit() : MeasureUnit() {
u_strcpy(isoCode, kDefaultCurrency);
char simpleIsoCode[4];
u_UCharsToChars(isoCode, simpleIsoCode, 4);
initCurrency(simpleIsoCode);
}
CurrencyUnit& CurrencyUnit::operator=(const CurrencyUnit& other) {
if (this == &other) {
return *this;
}
MeasureUnit::operator=(other);
u_strcpy(isoCode, other.isoCode);
return *this;
}
CurrencyUnit* CurrencyUnit::clone() const {
return new CurrencyUnit(*this);
}
CurrencyUnit::~CurrencyUnit() {
}
UOBJECT_DEFINE_RTTI_IMPLEMENTATION(CurrencyUnit)
U_NAMESPACE_END
#endif // !UCONFIG_NO_FORMATTING

194
deps/icu-small/source/i18n/dangical.cpp vendored Normal file
View File

@ -0,0 +1,194 @@
// © 2016 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html
/*
******************************************************************************
* Copyright (C) 2013, International Business Machines Corporation
* and others. All Rights Reserved.
******************************************************************************
*
* File DANGICAL.CPP
*****************************************************************************
*/
#include "chnsecal.h"
#include "dangical.h"
#if !UCONFIG_NO_FORMATTING
#include "astro.h" // CalendarCache
#include "gregoimp.h" // Math
#include "uassert.h"
#include "ucln_in.h"
#include "umutex.h"
#include "unicode/rbtz.h"
#include "unicode/tzrule.h"
// --- The cache --
// Lazy Creation & Access synchronized by class CalendarCache with a mutex.
static icu::CalendarCache *gWinterSolsticeCache = nullptr;
static icu::CalendarCache *gNewYearCache = nullptr;
// gAstronomerTimeZone
static icu::TimeZone *gAstronomerTimeZone = nullptr;
static icu::UInitOnce gAstronomerTimeZoneInitOnce {};
/**
* The start year of the Korean traditional calendar (Dan-gi) is the inaugural
* year of Dan-gun (BC 2333).
*/
static const int32_t DANGI_EPOCH_YEAR = -2332; // Gregorian year
U_CDECL_BEGIN
static UBool calendar_dangi_cleanup() {
if (gWinterSolsticeCache) {
delete gWinterSolsticeCache;
gWinterSolsticeCache = nullptr;
}
if (gNewYearCache) {
delete gNewYearCache;
gNewYearCache = nullptr;
}
if (gAstronomerTimeZone) {
delete gAstronomerTimeZone;
gAstronomerTimeZone = nullptr;
}
gAstronomerTimeZoneInitOnce.reset();
return true;
}
U_CDECL_END
U_NAMESPACE_BEGIN
// Implementation of the DangiCalendar class
//-------------------------------------------------------------------------
// Constructors...
//-------------------------------------------------------------------------
const TimeZone* getAstronomerTimeZone(UErrorCode &status);
DangiCalendar::DangiCalendar(const Locale& aLocale, UErrorCode& success)
: ChineseCalendar(aLocale, success)
{
}
DangiCalendar::DangiCalendar (const DangiCalendar& other)
: ChineseCalendar(other)
{
}
DangiCalendar::~DangiCalendar()
{
}
DangiCalendar*
DangiCalendar::clone() const
{
return new DangiCalendar(*this);
}
const char *DangiCalendar::getType() const {
return "dangi";
}
/**
* The time zone used for performing astronomical computations for
* Dangi calendar. In Korea various timezones have been used historically
* (cf. http://www.math.snu.ac.kr/~kye/others/lunar.html):
*
* - 1908/04/01: GMT+8
* 1908/04/01 - 1911/12/31: GMT+8.5
* 1912/01/01 - 1954/03/20: GMT+9
* 1954/03/21 - 1961/08/09: GMT+8.5
* 1961/08/10 - : GMT+9
*
* Note that, in 1908-1911, the government did not apply the timezone change
* but used GMT+8. In addition, 1954-1961's timezone change does not affect
* the lunar date calculation. Therefore, the following simpler rule works:
*
* -1911: GMT+8
* 1912-: GMT+9
*
* Unfortunately, our astronomer's approximation doesn't agree with the
* references (http://www.math.snu.ac.kr/~kye/others/lunar.html and
* http://astro.kasi.re.kr/Life/ConvertSolarLunarForm.aspx?MenuID=115)
* in 1897/7/30. So the following ad hoc fix is used here:
*
* -1896: GMT+8
* 1897: GMT+7
* 1898-1911: GMT+8
* 1912- : GMT+9
*/
static void U_CALLCONV initAstronomerTimeZone(UErrorCode &status) {
U_ASSERT(gAstronomerTimeZone == nullptr);
const UDate millis1897[] = { static_cast<UDate>((1897 - 1970) * 365 * kOneDay) }; // some days of error is not a problem here
const UDate millis1898[] = { static_cast<UDate>((1898 - 1970) * 365 * kOneDay) }; // some days of error is not a problem here
const UDate millis1912[] = { static_cast<UDate>((1912 - 1970) * 365 * kOneDay) }; // this doesn't create an issue for 1911/12/20
LocalPointer<InitialTimeZoneRule> initialTimeZone(new InitialTimeZoneRule(
UnicodeString(u"GMT+8"), 8*kOneHour, 0), status);
LocalPointer<TimeZoneRule> rule1897(new TimeArrayTimeZoneRule(
UnicodeString(u"Korean 1897"), 7*kOneHour, 0, millis1897, 1, DateTimeRule::STANDARD_TIME), status);
LocalPointer<TimeZoneRule> rule1898to1911(new TimeArrayTimeZoneRule(
UnicodeString(u"Korean 1898-1911"), 8*kOneHour, 0, millis1898, 1, DateTimeRule::STANDARD_TIME), status);
LocalPointer<TimeZoneRule> ruleFrom1912(new TimeArrayTimeZoneRule(
UnicodeString(u"Korean 1912-"), 9*kOneHour, 0, millis1912, 1, DateTimeRule::STANDARD_TIME), status);
LocalPointer<RuleBasedTimeZone> zone(new RuleBasedTimeZone(
UnicodeString(u"KOREA_ZONE"), initialTimeZone.orphan()), status); // adopts initialTimeZone
if (U_FAILURE(status)) {
return;
}
zone->addTransitionRule(rule1897.orphan(), status); // adopts rule1897
zone->addTransitionRule(rule1898to1911.orphan(), status);
zone->addTransitionRule(ruleFrom1912.orphan(), status);
zone->complete(status);
if (U_SUCCESS(status)) {
gAstronomerTimeZone = zone.orphan();
}
ucln_i18n_registerCleanup(UCLN_I18N_DANGI_CALENDAR, calendar_dangi_cleanup);
}
const TimeZone* getAstronomerTimeZone(UErrorCode &status) {
umtx_initOnce(gAstronomerTimeZoneInitOnce, &initAstronomerTimeZone, status);
return gAstronomerTimeZone;
}
constexpr uint32_t kDangiRelatedYearDiff = -2333;
int32_t DangiCalendar::getRelatedYear(UErrorCode &status) const
{
int32_t year = get(UCAL_EXTENDED_YEAR, status);
if (U_FAILURE(status)) {
return 0;
}
if (uprv_add32_overflow(year, kDangiRelatedYearDiff, &year)) {
status = U_ILLEGAL_ARGUMENT_ERROR;
return 0;
}
return year;
}
void DangiCalendar::setRelatedYear(int32_t year)
{
// set extended year
set(UCAL_EXTENDED_YEAR, year - kDangiRelatedYearDiff);
}
ChineseCalendar::Setting DangiCalendar::getSetting(UErrorCode& status) const {
return { DANGI_EPOCH_YEAR,
getAstronomerTimeZone(status),
&gWinterSolsticeCache, &gNewYearCache
};
}
UOBJECT_DEFINE_RTTI_IMPLEMENTATION(DangiCalendar)
U_NAMESPACE_END
#endif

135
deps/icu-small/source/i18n/dangical.h vendored Normal file
View File

@ -0,0 +1,135 @@
// © 2016 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html
/*
*****************************************************************************
* Copyright (C) 2013, International Business Machines Corporation
* and others. All Rights Reserved.
*****************************************************************************
*
* File DANGICAL.H
*****************************************************************************
*/
#ifndef DANGICAL_H
#define DANGICAL_H
#include "unicode/utypes.h"
#if !UCONFIG_NO_FORMATTING
#include "unicode/calendar.h"
#include "unicode/timezone.h"
#include "chnsecal.h"
U_NAMESPACE_BEGIN
/**
* <p><code>DangiCalendar</code> is a concrete subclass of {@link Calendar}
* that implements a traditional Korean lunisolar calendar.</p>
*
* <p>DangiCalendar usually should be instantiated using
* {@link com.ibm.icu.util.Calendar#getInstance(ULocale)} passing in a <code>ULocale</code>
* with the tag <code>"@calendar=dangi"</code>.</p>
*
* @internal
*/
class DangiCalendar : public ChineseCalendar {
public:
//-------------------------------------------------------------------------
// Constructors...
//-------------------------------------------------------------------------
/**
* Constructs a DangiCalendar based on the current time in the default time zone
* with the given locale.
*
* @param aLocale The given locale.
* @param success Indicates the status of DangiCalendar object construction.
* Returns U_ZERO_ERROR if constructed successfully.
* @internal
*/
DangiCalendar(const Locale& aLocale, UErrorCode &success);
/**
* Copy Constructor
* @internal
*/
DangiCalendar(const DangiCalendar& other);
/**
* Destructor.
* @internal
*/
virtual ~DangiCalendar();
/**
* Clone.
* @internal
*/
virtual DangiCalendar* clone() const override;
//----------------------------------------------------------------------
// Internal methods & astronomical calculations
//----------------------------------------------------------------------
/**
* @return The related Gregorian year; will be obtained by modifying the value
* obtained by get from UCAL_EXTENDED_YEAR field
* @internal
*/
virtual int32_t getRelatedYear(UErrorCode &status) const override;
/**
* @param year The related Gregorian year to set; will be modified as necessary then
* set in UCAL_EXTENDED_YEAR field
* @internal
*/
virtual void setRelatedYear(int32_t year) override;
private:
// UObject stuff
public:
/**
* @return The class ID for this object. All objects of a given class have the
* same class ID. Objects of other classes have different class IDs.
* @internal
*/
virtual UClassID getDynamicClassID() const override;
/**
* Return the class ID for this class. This is useful only for comparing to a return
* value from getDynamicClassID(). For example:
*
* Base* polymorphic_pointer = createPolymorphicObject();
* if (polymorphic_pointer->getDynamicClassID() ==
* Derived::getStaticClassID()) ...
*
* @return The class ID for all objects of this class.
* @internal
*/
U_I18N_API static UClassID U_EXPORT2 getStaticClassID();
/**
* return the calendar type, "dangi".
*
* @return calendar type
* @internal
*/
const char * getType() const override;
protected:
virtual Setting getSetting(UErrorCode& status) const override;
private:
DangiCalendar(); // default constructor not implemented
};
U_NAMESPACE_END
#endif
#endif

772
deps/icu-small/source/i18n/datefmt.cpp vendored Normal file
View File

@ -0,0 +1,772 @@
// © 2016 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html
/*
*******************************************************************************
* Copyright (C) 1997-2015, International Business Machines Corporation and *
* others. All Rights Reserved. *
*******************************************************************************
*
* File DATEFMT.CPP
*
* Modification History:
*
* Date Name Description
* 02/19/97 aliu Converted from java.
* 03/31/97 aliu Modified extensively to work with 50 locales.
* 04/01/97 aliu Added support for centuries.
* 08/12/97 aliu Fixed operator== to use Calendar::equivalentTo.
* 07/20/98 stephen Changed ParsePosition initialization
********************************************************************************
*/
#include "unicode/utypes.h"
#if !UCONFIG_NO_FORMATTING
#include "unicode/ures.h"
#include "unicode/datefmt.h"
#include "unicode/smpdtfmt.h"
#include "unicode/dtptngen.h"
#include "unicode/udisplaycontext.h"
#include "unicode/gregocal.h"
#include "reldtfmt.h"
#include "sharedobject.h"
#include "unifiedcache.h"
#include "uarrsort.h"
#include "cstring.h"
#include "windtfmt.h"
#if defined( U_DEBUG_CALSVC ) || defined (U_DEBUG_CAL)
#include <stdio.h>
#endif
#include <typeinfo>
// *****************************************************************************
// class DateFormat
// *****************************************************************************
U_NAMESPACE_BEGIN
class DateFmtBestPattern : public SharedObject {
public:
UnicodeString fPattern;
DateFmtBestPattern(const UnicodeString &pattern)
: fPattern(pattern) { }
~DateFmtBestPattern();
};
DateFmtBestPattern::~DateFmtBestPattern() {
}
template<>
const DateFmtBestPattern *LocaleCacheKey<DateFmtBestPattern>::createObject(
const void * /*creationContext*/, UErrorCode &status) const {
status = U_UNSUPPORTED_ERROR;
return nullptr;
}
class DateFmtBestPatternKey : public LocaleCacheKey<DateFmtBestPattern> {
private:
UnicodeString fSkeleton;
protected:
virtual bool equals(const CacheKeyBase &other) const override {
if (!LocaleCacheKey<DateFmtBestPattern>::equals(other)) {
return false;
}
// We know that this and other are of same class if we get this far.
return operator==(static_cast<const DateFmtBestPatternKey &>(other));
}
public:
DateFmtBestPatternKey(
const Locale &loc,
const UnicodeString &skeleton,
UErrorCode &status)
: LocaleCacheKey<DateFmtBestPattern>(loc),
fSkeleton(DateTimePatternGenerator::staticGetSkeleton(skeleton, status)) { }
DateFmtBestPatternKey(const DateFmtBestPatternKey &other) :
LocaleCacheKey<DateFmtBestPattern>(other),
fSkeleton(other.fSkeleton) { }
virtual ~DateFmtBestPatternKey();
virtual int32_t hashCode() const override {
return static_cast<int32_t>(37u * static_cast<uint32_t>(LocaleCacheKey<DateFmtBestPattern>::hashCode()) + static_cast<uint32_t>(fSkeleton.hashCode()));
}
inline bool operator==(const DateFmtBestPatternKey &other) const {
return fSkeleton == other.fSkeleton;
}
virtual CacheKeyBase *clone() const override {
return new DateFmtBestPatternKey(*this);
}
virtual const DateFmtBestPattern *createObject(
const void * /*unused*/, UErrorCode &status) const override {
LocalPointer<DateTimePatternGenerator> dtpg(
DateTimePatternGenerator::createInstance(fLoc, status));
if (U_FAILURE(status)) {
return nullptr;
}
LocalPointer<DateFmtBestPattern> pattern(
new DateFmtBestPattern(
dtpg->getBestPattern(fSkeleton, status)),
status);
if (U_FAILURE(status)) {
return nullptr;
}
DateFmtBestPattern *result = pattern.orphan();
result->addRef();
return result;
}
};
DateFmtBestPatternKey::~DateFmtBestPatternKey() { }
DateFormat::DateFormat()
: fCalendar(nullptr),
fNumberFormat(nullptr),
fCapitalizationContext(UDISPCTX_CAPITALIZATION_NONE)
{
}
//----------------------------------------------------------------------
DateFormat::DateFormat(const DateFormat& other)
: Format(other),
fCalendar(nullptr),
fNumberFormat(nullptr),
fCapitalizationContext(UDISPCTX_CAPITALIZATION_NONE)
{
*this = other;
}
//----------------------------------------------------------------------
DateFormat& DateFormat::operator=(const DateFormat& other)
{
if (this != &other)
{
delete fCalendar;
delete fNumberFormat;
if(other.fCalendar) {
fCalendar = other.fCalendar->clone();
} else {
fCalendar = nullptr;
}
if(other.fNumberFormat) {
fNumberFormat = other.fNumberFormat->clone();
} else {
fNumberFormat = nullptr;
}
fBoolFlags = other.fBoolFlags;
fCapitalizationContext = other.fCapitalizationContext;
}
return *this;
}
//----------------------------------------------------------------------
DateFormat::~DateFormat()
{
delete fCalendar;
delete fNumberFormat;
}
//----------------------------------------------------------------------
bool
DateFormat::operator==(const Format& other) const
{
if (this == &other) {
return true;
}
if (!(Format::operator==(other))) {
return false;
}
// Format::operator== guarantees that this cast is safe
DateFormat* fmt = (DateFormat*)&other;
return fCalendar&&(fCalendar->isEquivalentTo(*fmt->fCalendar)) &&
(fNumberFormat && *fNumberFormat == *fmt->fNumberFormat) &&
(fCapitalizationContext == fmt->fCapitalizationContext);
}
//----------------------------------------------------------------------
UnicodeString&
DateFormat::format(const Formattable& obj,
UnicodeString& appendTo,
FieldPosition& fieldPosition,
UErrorCode& status) const
{
if (U_FAILURE(status)) return appendTo;
// if the type of the Formattable is double or long, treat it as if it were a Date
UDate date = 0;
switch (obj.getType())
{
case Formattable::kDate:
date = obj.getDate();
break;
case Formattable::kDouble:
date = static_cast<UDate>(obj.getDouble());
break;
case Formattable::kLong:
date = static_cast<UDate>(obj.getLong());
break;
default:
status = U_ILLEGAL_ARGUMENT_ERROR;
return appendTo;
}
// Is this right?
//if (fieldPosition.getBeginIndex() == fieldPosition.getEndIndex())
// status = U_ILLEGAL_ARGUMENT_ERROR;
return format(date, appendTo, fieldPosition);
}
//----------------------------------------------------------------------
UnicodeString&
DateFormat::format(const Formattable& obj,
UnicodeString& appendTo,
FieldPositionIterator* posIter,
UErrorCode& status) const
{
if (U_FAILURE(status)) return appendTo;
// if the type of the Formattable is double or long, treat it as if it were a Date
UDate date = 0;
switch (obj.getType())
{
case Formattable::kDate:
date = obj.getDate();
break;
case Formattable::kDouble:
date = static_cast<UDate>(obj.getDouble());
break;
case Formattable::kLong:
date = static_cast<UDate>(obj.getLong());
break;
default:
status = U_ILLEGAL_ARGUMENT_ERROR;
return appendTo;
}
// Is this right?
//if (fieldPosition.getBeginIndex() == fieldPosition.getEndIndex())
// status = U_ILLEGAL_ARGUMENT_ERROR;
return format(date, appendTo, posIter, status);
}
//----------------------------------------------------------------------
// Default implementation for backwards compatibility, subclasses should implement.
UnicodeString&
DateFormat::format(Calendar& /* unused cal */,
UnicodeString& appendTo,
FieldPositionIterator* /* unused posIter */,
UErrorCode& status) const {
if (U_SUCCESS(status)) {
status = U_UNSUPPORTED_ERROR;
}
return appendTo;
}
//----------------------------------------------------------------------
UnicodeString&
DateFormat::format(UDate date, UnicodeString& appendTo, FieldPosition& fieldPosition) const {
if (fCalendar != nullptr) {
UErrorCode ec = U_ZERO_ERROR;
// Avoid a heap allocation and corresponding free for the common case
if (typeid(*fCalendar) == typeid(GregorianCalendar)) {
GregorianCalendar cal(*static_cast<GregorianCalendar*>(fCalendar));
cal.setTime(date, ec);
if (U_SUCCESS(ec)) {
format(cal, appendTo, fieldPosition);
}
} else {
// Use a clone of our calendar instance
Calendar *calClone = fCalendar->clone();
if (calClone != nullptr) {
calClone->setTime(date, ec);
if (U_SUCCESS(ec)) {
format(*calClone, appendTo, fieldPosition);
}
delete calClone;
}
}
}
return appendTo;
}
//----------------------------------------------------------------------
UnicodeString&
DateFormat::format(UDate date, UnicodeString& appendTo, FieldPositionIterator* posIter,
UErrorCode& status) const {
if (fCalendar != nullptr) {
UErrorCode ec = U_ZERO_ERROR;
// Avoid a heap allocation and corresponding free for the common case
if (typeid(*fCalendar) == typeid(GregorianCalendar)) {
GregorianCalendar cal(*static_cast<GregorianCalendar*>(fCalendar));
cal.setTime(date, ec);
if (U_SUCCESS(ec)) {
format(cal, appendTo, posIter, status);
}
} else {
Calendar* calClone = fCalendar->clone();
if (calClone != nullptr) {
calClone->setTime(date, status);
if (U_SUCCESS(status)) {
format(*calClone, appendTo, posIter, status);
}
delete calClone;
}
}
}
return appendTo;
}
//----------------------------------------------------------------------
UnicodeString&
DateFormat::format(UDate date, UnicodeString& appendTo) const
{
// Note that any error information is just lost. That's okay
// for this convenience method.
FieldPosition fpos(FieldPosition::DONT_CARE);
return format(date, appendTo, fpos);
}
//----------------------------------------------------------------------
UDate
DateFormat::parse(const UnicodeString& text,
ParsePosition& pos) const
{
UDate d = 0; // Error return UDate is 0 (the epoch)
if (fCalendar != nullptr) {
Calendar* calClone = fCalendar->clone();
if (calClone != nullptr) {
int32_t start = pos.getIndex();
calClone->clear();
parse(text, *calClone, pos);
if (pos.getIndex() != start) {
UErrorCode ec = U_ZERO_ERROR;
d = calClone->getTime(ec);
if (U_FAILURE(ec)) {
// We arrive here if fCalendar => calClone is non-lenient and
// there is an out-of-range field. We don't know which field
// was illegal so we set the error index to the start.
pos.setIndex(start);
pos.setErrorIndex(start);
d = 0;
}
}
delete calClone;
}
}
return d;
}
//----------------------------------------------------------------------
UDate
DateFormat::parse(const UnicodeString& text,
UErrorCode& status) const
{
if (U_FAILURE(status)) return 0;
ParsePosition pos(0);
UDate result = parse(text, pos);
if (pos.getIndex() == 0) {
#if defined (U_DEBUG_CAL)
fprintf(stderr, "%s:%d - - failed to parse - err index %d\n"
, __FILE__, __LINE__, pos.getErrorIndex() );
#endif
status = U_ILLEGAL_ARGUMENT_ERROR;
}
return result;
}
//----------------------------------------------------------------------
void
DateFormat::parseObject(const UnicodeString& source,
Formattable& result,
ParsePosition& pos) const
{
result.setDate(parse(source, pos));
}
//----------------------------------------------------------------------
DateFormat* U_EXPORT2
DateFormat::createTimeInstance(DateFormat::EStyle style,
const Locale& aLocale)
{
return createDateTimeInstance(kNone, style, aLocale);
}
//----------------------------------------------------------------------
DateFormat* U_EXPORT2
DateFormat::createDateInstance(DateFormat::EStyle style,
const Locale& aLocale)
{
return createDateTimeInstance(style, kNone, aLocale);
}
//----------------------------------------------------------------------
DateFormat* U_EXPORT2
DateFormat::createDateTimeInstance(EStyle dateStyle,
EStyle timeStyle,
const Locale& aLocale)
{
if(dateStyle != kNone)
{
dateStyle = static_cast<EStyle>(dateStyle + kDateOffset);
}
return create(timeStyle, dateStyle, aLocale);
}
//----------------------------------------------------------------------
DateFormat* U_EXPORT2
DateFormat::createInstance()
{
return createDateTimeInstance(kShort, kShort, Locale::getDefault());
}
//----------------------------------------------------------------------
UnicodeString U_EXPORT2
DateFormat::getBestPattern(
const Locale &locale,
const UnicodeString &skeleton,
UErrorCode &status) {
UnifiedCache *cache = UnifiedCache::getInstance(status);
if (U_FAILURE(status)) {
return {};
}
DateFmtBestPatternKey key(locale, skeleton, status);
const DateFmtBestPattern *patternPtr = nullptr;
cache->get(key, patternPtr, status);
if (U_FAILURE(status)) {
return {};
}
UnicodeString result(patternPtr->fPattern);
patternPtr->removeRef();
return result;
}
DateFormat* U_EXPORT2
DateFormat::createInstanceForSkeleton(
Calendar *calendarToAdopt,
const UnicodeString& skeleton,
const Locale &locale,
UErrorCode &status) {
LocalPointer<Calendar> calendar(calendarToAdopt);
if (U_FAILURE(status)) {
return nullptr;
}
if (calendar.isNull()) {
status = U_ILLEGAL_ARGUMENT_ERROR;
return nullptr;
}
Locale localeWithCalendar = locale;
localeWithCalendar.setKeywordValue("calendar", calendar->getType(), status);
if (U_FAILURE(status)) {
return nullptr;
}
DateFormat *result = createInstanceForSkeleton(skeleton, localeWithCalendar, status);
if (U_FAILURE(status)) {
return nullptr;
}
result->adoptCalendar(calendar.orphan());
return result;
}
DateFormat* U_EXPORT2
DateFormat::createInstanceForSkeleton(
const UnicodeString& skeleton,
const Locale &locale,
UErrorCode &status) {
if (U_FAILURE(status)) {
return nullptr;
}
LocalPointer<DateFormat> df(
new SimpleDateFormat(
getBestPattern(locale, skeleton, status),
locale, status),
status);
return U_SUCCESS(status) ? df.orphan() : nullptr;
}
DateFormat* U_EXPORT2
DateFormat::createInstanceForSkeleton(
const UnicodeString& skeleton,
UErrorCode &status) {
return createInstanceForSkeleton(
skeleton, Locale::getDefault(), status);
}
//----------------------------------------------------------------------
DateFormat* U_EXPORT2
DateFormat::create(EStyle timeStyle, EStyle dateStyle, const Locale& locale)
{
UErrorCode status = U_ZERO_ERROR;
#if U_PLATFORM_USES_ONLY_WIN32_API
char buffer[8];
int32_t count = locale.getKeywordValue("compat", buffer, sizeof(buffer), status);
// if the locale has "@compat=host", create a host-specific DateFormat...
if (count > 0 && uprv_strcmp(buffer, "host") == 0) {
Win32DateFormat *f = new Win32DateFormat(timeStyle, dateStyle, locale, status);
if (U_SUCCESS(status)) {
return f;
}
delete f;
}
#endif
// is it relative?
if(/*((timeStyle!=UDAT_NONE)&&(timeStyle & UDAT_RELATIVE)) || */((dateStyle!=kNone)&&((dateStyle-kDateOffset) & UDAT_RELATIVE))) {
RelativeDateFormat* r = new RelativeDateFormat(static_cast<UDateFormatStyle>(timeStyle), static_cast<UDateFormatStyle>(dateStyle - kDateOffset), locale, status);
if(U_SUCCESS(status)) return r;
delete r;
status = U_ZERO_ERROR;
}
// Try to create a SimpleDateFormat of the desired style.
SimpleDateFormat *f = new SimpleDateFormat(timeStyle, dateStyle, locale, status);
if (U_SUCCESS(status)) return f;
delete f;
// If that fails, try to create a format using the default pattern and
// the DateFormatSymbols for this locale.
status = U_ZERO_ERROR;
f = new SimpleDateFormat(locale, status);
if (U_SUCCESS(status)) return f;
delete f;
// This should never really happen, because the preceding constructor
// should always succeed. If the resource data is unavailable, a last
// resort object should be returned.
return nullptr;
}
//----------------------------------------------------------------------
const Locale* U_EXPORT2
DateFormat::getAvailableLocales(int32_t& count)
{
// Get the list of installed locales.
// Even if root has the correct date format for this locale,
// it's still a valid locale (we don't worry about data fallbacks).
return Locale::getAvailableLocales(count);
}
//----------------------------------------------------------------------
void
DateFormat::adoptCalendar(Calendar* newCalendar)
{
delete fCalendar;
fCalendar = newCalendar;
}
//----------------------------------------------------------------------
void
DateFormat::setCalendar(const Calendar& newCalendar)
{
Calendar* newCalClone = newCalendar.clone();
if (newCalClone != nullptr) {
adoptCalendar(newCalClone);
}
}
//----------------------------------------------------------------------
const Calendar*
DateFormat::getCalendar() const
{
return fCalendar;
}
//----------------------------------------------------------------------
void
DateFormat::adoptNumberFormat(NumberFormat* newNumberFormat)
{
delete fNumberFormat;
fNumberFormat = newNumberFormat;
newNumberFormat->setParseIntegerOnly(true);
newNumberFormat->setGroupingUsed(false);
}
//----------------------------------------------------------------------
void
DateFormat::setNumberFormat(const NumberFormat& newNumberFormat)
{
NumberFormat* newNumFmtClone = newNumberFormat.clone();
if (newNumFmtClone != nullptr) {
adoptNumberFormat(newNumFmtClone);
}
}
//----------------------------------------------------------------------
const NumberFormat*
DateFormat::getNumberFormat() const
{
return fNumberFormat;
}
//----------------------------------------------------------------------
void
DateFormat::adoptTimeZone(TimeZone* zone)
{
if (fCalendar != nullptr) {
fCalendar->adoptTimeZone(zone);
}
}
//----------------------------------------------------------------------
void
DateFormat::setTimeZone(const TimeZone& zone)
{
if (fCalendar != nullptr) {
fCalendar->setTimeZone(zone);
}
}
//----------------------------------------------------------------------
const TimeZone&
DateFormat::getTimeZone() const
{
if (fCalendar != nullptr) {
return fCalendar->getTimeZone();
}
// If calendar doesn't exists, create default timezone.
// fCalendar is rarely null
return *(TimeZone::createDefault());
}
//----------------------------------------------------------------------
void
DateFormat::setLenient(UBool lenient)
{
if (fCalendar != nullptr) {
fCalendar->setLenient(lenient);
}
UErrorCode status = U_ZERO_ERROR;
setBooleanAttribute(UDAT_PARSE_ALLOW_WHITESPACE, lenient, status);
setBooleanAttribute(UDAT_PARSE_ALLOW_NUMERIC, lenient, status);
}
//----------------------------------------------------------------------
UBool
DateFormat::isLenient() const
{
UBool lenient = true;
if (fCalendar != nullptr) {
lenient = fCalendar->isLenient();
}
UErrorCode status = U_ZERO_ERROR;
return lenient
&& getBooleanAttribute(UDAT_PARSE_ALLOW_WHITESPACE, status)
&& getBooleanAttribute(UDAT_PARSE_ALLOW_NUMERIC, status);
}
void
DateFormat::setCalendarLenient(UBool lenient)
{
if (fCalendar != nullptr) {
fCalendar->setLenient(lenient);
}
}
//----------------------------------------------------------------------
UBool
DateFormat::isCalendarLenient() const
{
if (fCalendar != nullptr) {
return fCalendar->isLenient();
}
// fCalendar is rarely null
return false;
}
//----------------------------------------------------------------------
void DateFormat::setContext(UDisplayContext value, UErrorCode& status)
{
if (U_FAILURE(status))
return;
if (static_cast<UDisplayContextType>(static_cast<uint32_t>(value) >> 8) == UDISPCTX_TYPE_CAPITALIZATION) {
fCapitalizationContext = value;
} else {
status = U_ILLEGAL_ARGUMENT_ERROR;
}
}
//----------------------------------------------------------------------
UDisplayContext DateFormat::getContext(UDisplayContextType type, UErrorCode& status) const
{
if (U_FAILURE(status))
return static_cast<UDisplayContext>(0);
if (type != UDISPCTX_TYPE_CAPITALIZATION) {
status = U_ILLEGAL_ARGUMENT_ERROR;
return static_cast<UDisplayContext>(0);
}
return fCapitalizationContext;
}
//----------------------------------------------------------------------
DateFormat&
DateFormat::setBooleanAttribute(UDateFormatBooleanAttribute attr,
UBool newValue,
UErrorCode &status) {
if(!fBoolFlags.isValidValue(newValue)) {
status = U_ILLEGAL_ARGUMENT_ERROR;
} else {
fBoolFlags.set(attr, newValue);
}
return *this;
}
//----------------------------------------------------------------------
UBool
DateFormat::getBooleanAttribute(UDateFormatBooleanAttribute attr, UErrorCode &/*status*/) const {
return fBoolFlags.get(attr);
}
U_NAMESPACE_END
#endif /* #if !UCONFIG_NO_FORMATTING */
//eof

514
deps/icu-small/source/i18n/dayperiodrules.cpp vendored Normal file
View File

@ -0,0 +1,514 @@
// © 2016 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html
/*
*******************************************************************************
* Copyright (C) 2016, International Business Machines
* Corporation and others. All Rights Reserved.
*******************************************************************************
* dayperiodrules.cpp
*
* created on: 2016-01-20
* created by: kazede
*/
#include "dayperiodrules.h"
#include "unicode/ures.h"
#include "charstr.h"
#include "cstring.h"
#include "ucln_in.h"
#include "uhash.h"
#include "ulocimp.h"
#include "umutex.h"
#include "uresimp.h"
U_NAMESPACE_BEGIN
namespace {
struct DayPeriodRulesData : public UMemory {
DayPeriodRulesData() : localeToRuleSetNumMap(nullptr), rules(nullptr), maxRuleSetNum(0) {}
UHashtable *localeToRuleSetNumMap;
DayPeriodRules *rules;
int32_t maxRuleSetNum;
} *data = nullptr;
enum CutoffType {
CUTOFF_TYPE_UNKNOWN = -1,
CUTOFF_TYPE_BEFORE,
CUTOFF_TYPE_AFTER, // TODO: AFTER is deprecated in CLDR 29. Remove.
CUTOFF_TYPE_FROM,
CUTOFF_TYPE_AT
};
} // namespace
struct DayPeriodRulesDataSink : public ResourceSink {
DayPeriodRulesDataSink() {
for (int32_t i = 0; i < UPRV_LENGTHOF(cutoffs); ++i) { cutoffs[i] = 0; }
}
virtual ~DayPeriodRulesDataSink();
virtual void put(const char *key, ResourceValue &value, UBool, UErrorCode &errorCode) override {
ResourceTable dayPeriodData = value.getTable(errorCode);
if (U_FAILURE(errorCode)) { return; }
for (int32_t i = 0; dayPeriodData.getKeyAndValue(i, key, value); ++i) {
if (uprv_strcmp(key, "locales") == 0) {
ResourceTable locales = value.getTable(errorCode);
if (U_FAILURE(errorCode)) { return; }
for (int32_t j = 0; locales.getKeyAndValue(j, key, value); ++j) {
UnicodeString setNum_str = value.getUnicodeString(errorCode);
int32_t setNum = parseSetNum(setNum_str, errorCode);
uhash_puti(data->localeToRuleSetNumMap, const_cast<char *>(key), setNum, &errorCode);
}
} else if (uprv_strcmp(key, "rules") == 0) {
// Allocate one more than needed to skip [0]. See comment in parseSetNum().
data->rules = new DayPeriodRules[data->maxRuleSetNum + 1];
if (data->rules == nullptr) {
errorCode = U_MEMORY_ALLOCATION_ERROR;
return;
}
ResourceTable rules = value.getTable(errorCode);
processRules(rules, key, value, errorCode);
if (U_FAILURE(errorCode)) { return; }
}
}
}
void processRules(const ResourceTable &rules, const char *key,
ResourceValue &value, UErrorCode &errorCode) {
if (U_FAILURE(errorCode)) { return; }
for (int32_t i = 0; rules.getKeyAndValue(i, key, value); ++i) {
ruleSetNum = parseSetNum(key, errorCode);
ResourceTable ruleSet = value.getTable(errorCode);
if (U_FAILURE(errorCode)) { return; }
for (int32_t j = 0; ruleSet.getKeyAndValue(j, key, value); ++j) {
period = DayPeriodRules::getDayPeriodFromString(key);
if (period == DayPeriodRules::DAYPERIOD_UNKNOWN) {
errorCode = U_INVALID_FORMAT_ERROR;
return;
}
ResourceTable periodDefinition = value.getTable(errorCode);
if (U_FAILURE(errorCode)) { return; }
for (int32_t k = 0; periodDefinition.getKeyAndValue(k, key, value); ++k) {
if (value.getType() == URES_STRING) {
// Key-value pairs (e.g. before{6:00}).
CutoffType type = getCutoffTypeFromString(key);
addCutoff(type, value.getUnicodeString(errorCode), errorCode);
if (U_FAILURE(errorCode)) { return; }
} else {
// Arrays (e.g. before{6:00, 24:00}).
cutoffType = getCutoffTypeFromString(key);
ResourceArray cutoffArray = value.getArray(errorCode);
if (U_FAILURE(errorCode)) { return; }
int32_t length = cutoffArray.getSize();
for (int32_t l = 0; l < length; ++l) {
cutoffArray.getValue(l, value);
addCutoff(cutoffType, value.getUnicodeString(errorCode), errorCode);
if (U_FAILURE(errorCode)) { return; }
}
}
}
setDayPeriodForHoursFromCutoffs(errorCode);
for (int32_t k = 0; k < UPRV_LENGTHOF(cutoffs); ++k) {
cutoffs[k] = 0;
}
}
if (!data->rules[ruleSetNum].allHoursAreSet()) {
errorCode = U_INVALID_FORMAT_ERROR;
return;
}
}
}
// Members.
int32_t cutoffs[25]; // [0] thru [24]: 24 is allowed in "before 24".
// "Path" to data.
int32_t ruleSetNum;
DayPeriodRules::DayPeriod period;
CutoffType cutoffType;
// Helpers.
static int32_t parseSetNum(const UnicodeString &setNumStr, UErrorCode &errorCode) {
CharString cs;
cs.appendInvariantChars(setNumStr, errorCode);
return parseSetNum(cs.data(), errorCode);
}
static int32_t parseSetNum(const char *setNumStr, UErrorCode &errorCode) {
if (U_FAILURE(errorCode)) { return -1; }
if (uprv_strncmp(setNumStr, "set", 3) != 0) {
errorCode = U_INVALID_FORMAT_ERROR;
return -1;
}
int32_t i = 3;
int32_t setNum = 0;
while (setNumStr[i] != 0) {
int32_t digit = setNumStr[i] - '0';
if (digit < 0 || 9 < digit) {
errorCode = U_INVALID_FORMAT_ERROR;
return -1;
}
setNum = 10 * setNum + digit;
++i;
}
// Rule set number must not be zero. (0 is used to indicate "not found" by hashmap.)
// Currently ICU data conveniently starts numbering rule sets from 1.
if (setNum == 0) {
errorCode = U_INVALID_FORMAT_ERROR;
return -1;
} else {
return setNum;
}
}
void addCutoff(CutoffType type, const UnicodeString &hour_str, UErrorCode &errorCode) {
if (U_FAILURE(errorCode)) { return; }
if (type == CUTOFF_TYPE_UNKNOWN) {
errorCode = U_INVALID_FORMAT_ERROR;
return;
}
int32_t hour = parseHour(hour_str, errorCode);
if (U_FAILURE(errorCode)) { return; }
cutoffs[hour] |= 1 << type;
}
// Translate the cutoffs[] array to day period rules.
void setDayPeriodForHoursFromCutoffs(UErrorCode &errorCode) {
DayPeriodRules &rule = data->rules[ruleSetNum];
for (int32_t startHour = 0; startHour <= 24; ++startHour) {
// AT cutoffs must be either midnight or noon.
if (cutoffs[startHour] & (1 << CUTOFF_TYPE_AT)) {
if (startHour == 0 && period == DayPeriodRules::DAYPERIOD_MIDNIGHT) {
rule.fHasMidnight = true;
} else if (startHour == 12 && period == DayPeriodRules::DAYPERIOD_NOON) {
rule.fHasNoon = true;
} else {
errorCode = U_INVALID_FORMAT_ERROR; // Bad data.
return;
}
}
// FROM/AFTER and BEFORE must come in a pair.
if (cutoffs[startHour] & (1 << CUTOFF_TYPE_FROM) ||
cutoffs[startHour] & (1 << CUTOFF_TYPE_AFTER)) {
for (int32_t hour = startHour + 1;; ++hour) {
if (hour == startHour) {
// We've gone around the array once and can't find a BEFORE.
errorCode = U_INVALID_FORMAT_ERROR;
return;
}
if (hour == 25) { hour = 0; }
if (cutoffs[hour] & (1 << CUTOFF_TYPE_BEFORE)) {
rule.add(startHour, hour, period);
break;
}
}
}
}
}
// Translate "before" to CUTOFF_TYPE_BEFORE, for example.
static CutoffType getCutoffTypeFromString(const char *type_str) {
if (uprv_strcmp(type_str, "from") == 0) {
return CUTOFF_TYPE_FROM;
} else if (uprv_strcmp(type_str, "before") == 0) {
return CUTOFF_TYPE_BEFORE;
} else if (uprv_strcmp(type_str, "after") == 0) {
return CUTOFF_TYPE_AFTER;
} else if (uprv_strcmp(type_str, "at") == 0) {
return CUTOFF_TYPE_AT;
} else {
return CUTOFF_TYPE_UNKNOWN;
}
}
// Gets the numerical value of the hour from the Unicode string.
static int32_t parseHour(const UnicodeString &time, UErrorCode &errorCode) {
if (U_FAILURE(errorCode)) {
return 0;
}
int32_t hourLimit = time.length() - 3;
// `time` must look like "x:00" or "xx:00".
// If length is wrong or `time` doesn't end with ":00", error out.
if ((hourLimit != 1 && hourLimit != 2) ||
time[hourLimit] != 0x3A || time[hourLimit + 1] != 0x30 ||
time[hourLimit + 2] != 0x30) {
errorCode = U_INVALID_FORMAT_ERROR;
return 0;
}
// If `time` doesn't begin with a number in [0, 24], error out.
// Note: "24:00" is possible in "before 24:00".
int32_t hour = time[0] - 0x30;
if (hour < 0 || 9 < hour) {
errorCode = U_INVALID_FORMAT_ERROR;
return 0;
}
if (hourLimit == 2) {
int32_t hourDigit2 = time[1] - 0x30;
if (hourDigit2 < 0 || 9 < hourDigit2) {
errorCode = U_INVALID_FORMAT_ERROR;
return 0;
}
hour = hour * 10 + hourDigit2;
if (hour > 24) {
errorCode = U_INVALID_FORMAT_ERROR;
return 0;
}
}
return hour;
}
}; // struct DayPeriodRulesDataSink
struct DayPeriodRulesCountSink : public ResourceSink {
virtual ~DayPeriodRulesCountSink();
virtual void put(const char *key, ResourceValue &value, UBool, UErrorCode &errorCode) override {
ResourceTable rules = value.getTable(errorCode);
if (U_FAILURE(errorCode)) { return; }
for (int32_t i = 0; rules.getKeyAndValue(i, key, value); ++i) {
int32_t setNum = DayPeriodRulesDataSink::parseSetNum(key, errorCode);
if (setNum > data->maxRuleSetNum) {
data->maxRuleSetNum = setNum;
}
}
}
};
// Out-of-line virtual destructors.
DayPeriodRulesDataSink::~DayPeriodRulesDataSink() {}
DayPeriodRulesCountSink::~DayPeriodRulesCountSink() {}
namespace {
UInitOnce initOnce {};
U_CFUNC UBool U_CALLCONV dayPeriodRulesCleanup() {
delete[] data->rules;
uhash_close(data->localeToRuleSetNumMap);
delete data;
data = nullptr;
return true;
}
} // namespace
void U_CALLCONV DayPeriodRules::load(UErrorCode &errorCode) {
if (U_FAILURE(errorCode)) {
return;
}
data = new DayPeriodRulesData();
data->localeToRuleSetNumMap = uhash_open(uhash_hashChars, uhash_compareChars, nullptr, &errorCode);
LocalUResourceBundlePointer rb_dayPeriods(ures_openDirect(nullptr, "dayPeriods", &errorCode));
// Get the largest rule set number (so we allocate enough objects).
DayPeriodRulesCountSink countSink;
ures_getAllItemsWithFallback(rb_dayPeriods.getAlias(), "rules", countSink, errorCode);
// Populate rules.
DayPeriodRulesDataSink sink;
ures_getAllItemsWithFallback(rb_dayPeriods.getAlias(), "", sink, errorCode);
ucln_i18n_registerCleanup(UCLN_I18N_DAYPERIODRULES, dayPeriodRulesCleanup);
}
const DayPeriodRules *DayPeriodRules::getInstance(const Locale &locale, UErrorCode &errorCode) {
umtx_initOnce(initOnce, DayPeriodRules::load, errorCode);
// If the entire day period rules data doesn't conform to spec (even if the part we want
// does), return nullptr.
if(U_FAILURE(errorCode)) { return nullptr; }
const char *localeCode = locale.getBaseName();
char name[ULOC_FULLNAME_CAPACITY];
if (uprv_strlen(localeCode) < ULOC_FULLNAME_CAPACITY) {
uprv_strcpy(name, localeCode);
// Treat empty string as root.
if (*name == '\0') {
uprv_strcpy(name, "root");
}
} else {
errorCode = U_BUFFER_OVERFLOW_ERROR;
return nullptr;
}
int32_t ruleSetNum = 0; // NB there is no rule set 0 and 0 is returned upon lookup failure.
while (*name != '\0') {
ruleSetNum = uhash_geti(data->localeToRuleSetNumMap, name);
if (ruleSetNum == 0) {
CharString parent = ulocimp_getParent(name, errorCode);
if (parent.isEmpty()) {
// Saves a lookup in the hash table.
break;
}
parent.extract(name, UPRV_LENGTHOF(name), errorCode);
} else {
break;
}
}
if (ruleSetNum <= 0 || data->rules[ruleSetNum].getDayPeriodForHour(0) == DAYPERIOD_UNKNOWN) {
// If day period for hour 0 is UNKNOWN then day period for all hours are UNKNOWN.
// Data doesn't exist even with fallback.
return nullptr;
} else {
return &data->rules[ruleSetNum];
}
}
DayPeriodRules::DayPeriodRules() : fHasMidnight(false), fHasNoon(false) {
for (int32_t i = 0; i < 24; ++i) {
fDayPeriodForHour[i] = DayPeriodRules::DAYPERIOD_UNKNOWN;
}
}
double DayPeriodRules::getMidPointForDayPeriod(
DayPeriodRules::DayPeriod dayPeriod, UErrorCode &errorCode) const {
if (U_FAILURE(errorCode)) { return -1; }
int32_t startHour = getStartHourForDayPeriod(dayPeriod, errorCode);
int32_t endHour = getEndHourForDayPeriod(dayPeriod, errorCode);
// Can't obtain startHour or endHour; bail out.
if (U_FAILURE(errorCode)) { return -1; }
double midPoint = (startHour + endHour) / 2.0;
if (startHour > endHour) {
// dayPeriod wraps around midnight. Shift midPoint by 12 hours, in the direction that
// lands it in [0, 24).
midPoint += 12;
if (midPoint >= 24) {
midPoint -= 24;
}
}
return midPoint;
}
int32_t DayPeriodRules::getStartHourForDayPeriod(
DayPeriodRules::DayPeriod dayPeriod, UErrorCode &errorCode) const {
if (U_FAILURE(errorCode)) { return -1; }
if (dayPeriod == DAYPERIOD_MIDNIGHT) { return 0; }
if (dayPeriod == DAYPERIOD_NOON) { return 12; }
if (fDayPeriodForHour[0] == dayPeriod && fDayPeriodForHour[23] == dayPeriod) {
// dayPeriod wraps around midnight. Start hour is later than end hour.
for (int32_t i = 22; i >= 1; --i) {
if (fDayPeriodForHour[i] != dayPeriod) {
return (i + 1);
}
}
} else {
for (int32_t i = 0; i <= 23; ++i) {
if (fDayPeriodForHour[i] == dayPeriod) {
return i;
}
}
}
// dayPeriod doesn't exist in rule set; set error and exit.
errorCode = U_ILLEGAL_ARGUMENT_ERROR;
return -1;
}
int32_t DayPeriodRules::getEndHourForDayPeriod(
DayPeriodRules::DayPeriod dayPeriod, UErrorCode &errorCode) const {
if (U_FAILURE(errorCode)) { return -1; }
if (dayPeriod == DAYPERIOD_MIDNIGHT) { return 0; }
if (dayPeriod == DAYPERIOD_NOON) { return 12; }
if (fDayPeriodForHour[0] == dayPeriod && fDayPeriodForHour[23] == dayPeriod) {
// dayPeriod wraps around midnight. End hour is before start hour.
for (int32_t i = 1; i <= 22; ++i) {
if (fDayPeriodForHour[i] != dayPeriod) {
// i o'clock is when a new period starts, therefore when the old period ends.
return i;
}
}
} else {
for (int32_t i = 23; i >= 0; --i) {
if (fDayPeriodForHour[i] == dayPeriod) {
return (i + 1);
}
}
}
// dayPeriod doesn't exist in rule set; set error and exit.
errorCode = U_ILLEGAL_ARGUMENT_ERROR;
return -1;
}
DayPeriodRules::DayPeriod DayPeriodRules::getDayPeriodFromString(const char *type_str) {
if (uprv_strcmp(type_str, "midnight") == 0) {
return DAYPERIOD_MIDNIGHT;
} else if (uprv_strcmp(type_str, "noon") == 0) {
return DAYPERIOD_NOON;
} else if (uprv_strcmp(type_str, "morning1") == 0) {
return DAYPERIOD_MORNING1;
} else if (uprv_strcmp(type_str, "afternoon1") == 0) {
return DAYPERIOD_AFTERNOON1;
} else if (uprv_strcmp(type_str, "evening1") == 0) {
return DAYPERIOD_EVENING1;
} else if (uprv_strcmp(type_str, "night1") == 0) {
return DAYPERIOD_NIGHT1;
} else if (uprv_strcmp(type_str, "morning2") == 0) {
return DAYPERIOD_MORNING2;
} else if (uprv_strcmp(type_str, "afternoon2") == 0) {
return DAYPERIOD_AFTERNOON2;
} else if (uprv_strcmp(type_str, "evening2") == 0) {
return DAYPERIOD_EVENING2;
} else if (uprv_strcmp(type_str, "night2") == 0) {
return DAYPERIOD_NIGHT2;
} else if (uprv_strcmp(type_str, "am") == 0) {
return DAYPERIOD_AM;
} else if (uprv_strcmp(type_str, "pm") == 0) {
return DAYPERIOD_PM;
} else {
return DAYPERIOD_UNKNOWN;
}
}
void DayPeriodRules::add(int32_t startHour, int32_t limitHour, DayPeriod period) {
for (int32_t i = startHour; i != limitHour; ++i) {
if (i == 24) { i = 0; }
fDayPeriodForHour[i] = period;
}
}
UBool DayPeriodRules::allHoursAreSet() {
for (int32_t i = 0; i < 24; ++i) {
if (fDayPeriodForHour[i] == DAYPERIOD_UNKNOWN) { return false; }
}
return true;
}
U_NAMESPACE_END

89
deps/icu-small/source/i18n/dayperiodrules.h vendored Normal file
View File

@ -0,0 +1,89 @@
// © 2016 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html
/*
*******************************************************************************
* Copyright (C) 2016, International Business Machines
* Corporation and others. All Rights Reserved.
*******************************************************************************
* dayperiodrules.h
*
* created on: 2016-01-20
* created by: kazede
*/
#ifndef DAYPERIODRULES_H
#define DAYPERIODRULES_H
#include "unicode/locid.h"
#include "unicode/unistr.h"
#include "unicode/uobject.h"
#include "unicode/utypes.h"
#include "resource.h"
#include "uhash.h"
U_NAMESPACE_BEGIN
struct DayPeriodRulesDataSink;
class DayPeriodRules : public UMemory {
friend struct DayPeriodRulesDataSink;
public:
enum DayPeriod {
DAYPERIOD_UNKNOWN = -1,
DAYPERIOD_MIDNIGHT,
DAYPERIOD_NOON,
DAYPERIOD_MORNING1,
DAYPERIOD_AFTERNOON1,
DAYPERIOD_EVENING1,
DAYPERIOD_NIGHT1,
DAYPERIOD_MORNING2,
DAYPERIOD_AFTERNOON2,
DAYPERIOD_EVENING2,
DAYPERIOD_NIGHT2,
DAYPERIOD_AM,
DAYPERIOD_PM
};
static const DayPeriodRules *getInstance(const Locale &locale, UErrorCode &errorCode);
UBool hasMidnight() const { return fHasMidnight; }
UBool hasNoon() const { return fHasNoon; }
DayPeriod getDayPeriodForHour(int32_t hour) const { return fDayPeriodForHour[hour]; }
// Returns the center of dayPeriod. Half hours are indicated with a .5 .
double getMidPointForDayPeriod(DayPeriod dayPeriod, UErrorCode &errorCode) const;
private:
DayPeriodRules();
// Translates "morning1" to DAYPERIOD_MORNING1, for example.
static DayPeriod getDayPeriodFromString(const char *type_str);
static void U_CALLCONV load(UErrorCode &errorCode);
// Sets period type for all hours in [startHour, limitHour).
void add(int32_t startHour, int32_t limitHour, DayPeriod period);
// Returns true if for all i, DayPeriodForHour[i] has a type other than UNKNOWN.
// Values of HasNoon and HasMidnight do not affect the return value.
UBool allHoursAreSet();
// Returns the hour that starts dayPeriod. Returns 0 for MIDNIGHT and 12 for NOON.
int32_t getStartHourForDayPeriod(DayPeriod dayPeriod, UErrorCode &errorCode) const;
// Returns the hour that ends dayPeriod, i.e. that starts the next period.
// E.g. if fDayPeriodForHour[13] thru [16] are AFTERNOON1, then this function returns 17 if
// queried with AFTERNOON1.
// Returns 0 for MIDNIGHT and 12 for NOON.
int32_t getEndHourForDayPeriod(DayPeriod dayPeriod, UErrorCode &errorCode) const;
UBool fHasMidnight;
UBool fHasNoon;
DayPeriod fDayPeriodForHour[24];
};
U_NAMESPACE_END
#endif /* DAYPERIODRULES_H */

607
deps/icu-small/source/i18n/dcfmtsym.cpp vendored Normal file
View File

@ -0,0 +1,607 @@
// © 2016 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html
/*
*******************************************************************************
* Copyright (C) 1997-2016, International Business Machines Corporation and
* others. All Rights Reserved.
*******************************************************************************
*
* File DCFMTSYM.CPP
*
* Modification History:
*
* Date Name Description
* 02/19/97 aliu Converted from java.
* 03/18/97 clhuang Implemented with C++ APIs.
* 03/27/97 helena Updated to pass the simple test after code review.
* 08/26/97 aliu Added currency/intl currency symbol support.
* 07/20/98 stephen Slightly modified initialization of monetarySeparator
********************************************************************************
*/
#include "unicode/utypes.h"
#if !UCONFIG_NO_FORMATTING
#include "unicode/dcfmtsym.h"
#include "unicode/ures.h"
#include "unicode/decimfmt.h"
#include "unicode/ucurr.h"
#include "unicode/choicfmt.h"
#include "unicode/unistr.h"
#include "unicode/numsys.h"
#include "unicode/unum.h"
#include "unicode/utf16.h"
#include "ucurrimp.h"
#include "cstring.h"
#include "locbased.h"
#include "uresimp.h"
#include "ureslocs.h"
#include "charstr.h"
#include "uassert.h"
// *****************************************************************************
// class DecimalFormatSymbols
// *****************************************************************************
U_NAMESPACE_BEGIN
UOBJECT_DEFINE_RTTI_IMPLEMENTATION(DecimalFormatSymbols)
static const char gNumberElements[] = "NumberElements";
static const char gCurrencySpacingTag[] = "currencySpacing";
static const char gBeforeCurrencyTag[] = "beforeCurrency";
static const char gAfterCurrencyTag[] = "afterCurrency";
static const char gCurrencyMatchTag[] = "currencyMatch";
static const char gCurrencySudMatchTag[] = "surroundingMatch";
static const char gCurrencyInsertBtnTag[] = "insertBetween";
static const char gLatn[] = "latn";
static const char gSymbols[] = "symbols";
static const char gNumberElementsLatnSymbols[] = "NumberElements/latn/symbols";
static const char16_t INTL_CURRENCY_SYMBOL_STR[] = {0xa4, 0xa4, 0};
// List of field names to be loaded from the data files.
// These are parallel with the enum ENumberFormatSymbol in unicode/dcfmtsym.h.
static const char *gNumberElementKeys[DecimalFormatSymbols::kFormatSymbolCount] = {
"decimal",
"group",
nullptr, /* #11897: the <list> symbol is NOT the pattern separator symbol */
"percentSign",
nullptr, /* Native zero digit is deprecated from CLDR - get it from the numbering system */
nullptr, /* Pattern digit character is deprecated from CLDR - use # by default always */
"minusSign",
"plusSign",
nullptr, /* currency symbol - Wait until we know the currency before loading from CLDR */
nullptr, /* intl currency symbol - Wait until we know the currency before loading from CLDR */
"currencyDecimal",
"exponential",
"perMille",
nullptr, /* Escape padding character - not in CLDR */
"infinity",
"nan",
nullptr, /* Significant digit symbol - not in CLDR */
"currencyGroup",
nullptr, /* one digit - get it from the numbering system */
nullptr, /* two digit - get it from the numbering system */
nullptr, /* three digit - get it from the numbering system */
nullptr, /* four digit - get it from the numbering system */
nullptr, /* five digit - get it from the numbering system */
nullptr, /* six digit - get it from the numbering system */
nullptr, /* seven digit - get it from the numbering system */
nullptr, /* eight digit - get it from the numbering system */
nullptr, /* nine digit - get it from the numbering system */
"superscriptingExponent", /* Multiplication (x) symbol for exponents */
"approximatelySign" /* Approximately sign symbol */
};
// -------------------------------------
// Initializes this with the decimal format symbols in the default locale.
DecimalFormatSymbols::DecimalFormatSymbols(UErrorCode& status)
: UObject(), locale() {
initialize(locale, status, true);
}
// -------------------------------------
// Initializes this with the decimal format symbols in the desired locale.
DecimalFormatSymbols::DecimalFormatSymbols(const Locale& loc, UErrorCode& status)
: UObject(), locale(loc) {
initialize(locale, status);
}
DecimalFormatSymbols::DecimalFormatSymbols(const Locale& loc, const NumberingSystem& ns, UErrorCode& status)
: UObject(), locale(loc) {
initialize(locale, status, false, &ns);
}
DecimalFormatSymbols::DecimalFormatSymbols()
: UObject(), locale(Locale::getRoot()) {
initialize();
}
DecimalFormatSymbols*
DecimalFormatSymbols::createWithLastResortData(UErrorCode& status) {
if (U_FAILURE(status)) { return nullptr; }
DecimalFormatSymbols* sym = new DecimalFormatSymbols();
if (sym == nullptr) {
status = U_MEMORY_ALLOCATION_ERROR;
}
return sym;
}
// -------------------------------------
DecimalFormatSymbols::~DecimalFormatSymbols()
{
delete actualLocale;
delete validLocale;
}
// -------------------------------------
// copy constructor
DecimalFormatSymbols::DecimalFormatSymbols(const DecimalFormatSymbols &source)
: UObject(source)
{
*this = source;
}
// -------------------------------------
// assignment operator
DecimalFormatSymbols&
DecimalFormatSymbols::operator=(const DecimalFormatSymbols& rhs)
{
if (this != &rhs) {
for (int32_t i = 0; i < static_cast<int32_t>(kFormatSymbolCount); ++i) {
// fastCopyFrom is safe, see docs on fSymbols
fSymbols[static_cast<ENumberFormatSymbol>(i)].fastCopyFrom(rhs.fSymbols[static_cast<ENumberFormatSymbol>(i)]);
}
for (int32_t i = 0; i < static_cast<int32_t>(UNUM_CURRENCY_SPACING_COUNT); ++i) {
currencySpcBeforeSym[i].fastCopyFrom(rhs.currencySpcBeforeSym[i]);
currencySpcAfterSym[i].fastCopyFrom(rhs.currencySpcAfterSym[i]);
}
locale = rhs.locale;
UErrorCode status = U_ZERO_ERROR;
U_LOCALE_BASED(locBased, *this);
locBased.setLocaleIDs(rhs.validLocale, rhs.actualLocale, status);
U_ASSERT(U_SUCCESS(status));
fIsCustomCurrencySymbol = rhs.fIsCustomCurrencySymbol;
fIsCustomIntlCurrencySymbol = rhs.fIsCustomIntlCurrencySymbol;
fCodePointZero = rhs.fCodePointZero;
currPattern = rhs.currPattern;
uprv_strcpy(nsName, rhs.nsName);
}
return *this;
}
// -------------------------------------
bool
DecimalFormatSymbols::operator==(const DecimalFormatSymbols& that) const
{
if (this == &that) {
return true;
}
if (fIsCustomCurrencySymbol != that.fIsCustomCurrencySymbol) {
return false;
}
if (fIsCustomIntlCurrencySymbol != that.fIsCustomIntlCurrencySymbol) {
return false;
}
for (int32_t i = 0; i < static_cast<int32_t>(kFormatSymbolCount); ++i) {
if (fSymbols[static_cast<ENumberFormatSymbol>(i)] != that.fSymbols[static_cast<ENumberFormatSymbol>(i)]) {
return false;
}
}
for (int32_t i = 0; i < static_cast<int32_t>(UNUM_CURRENCY_SPACING_COUNT); ++i) {
if(currencySpcBeforeSym[i] != that.currencySpcBeforeSym[i]) {
return false;
}
if(currencySpcAfterSym[i] != that.currencySpcAfterSym[i]) {
return false;
}
}
// No need to check fCodePointZero since it is based on fSymbols
return locale == that.locale &&
LocaleBased::equalIDs(actualLocale, that.actualLocale) &&
LocaleBased::equalIDs(validLocale, that.validLocale);
}
// -------------------------------------
namespace {
/**
* Sink for enumerating all of the decimal format symbols (more specifically, anything
* under the "NumberElements.symbols" tree).
*
* More specific bundles (en_GB) are enumerated before their parents (en_001, en, root):
* Only store a value if it is still missing, that is, it has not been overridden.
*/
struct DecFmtSymDataSink : public ResourceSink {
// Destination for data, modified via setters.
DecimalFormatSymbols& dfs;
// Boolean array of whether or not we have seen a particular symbol yet.
// Can't simply check fSymbols because it is pre-populated with defaults.
UBool seenSymbol[DecimalFormatSymbols::kFormatSymbolCount];
// Constructor/Destructor
DecFmtSymDataSink(DecimalFormatSymbols& _dfs) : dfs(_dfs) {
uprv_memset(seenSymbol, false, sizeof(seenSymbol));
}
virtual ~DecFmtSymDataSink();
virtual void put(const char *key, ResourceValue &value, UBool /*noFallback*/,
UErrorCode &errorCode) override {
ResourceTable symbolsTable = value.getTable(errorCode);
if (U_FAILURE(errorCode)) { return; }
for (int32_t j = 0; symbolsTable.getKeyAndValue(j, key, value); ++j) {
for (int32_t i=0; i<DecimalFormatSymbols::kFormatSymbolCount; i++) {
if (gNumberElementKeys[i] != nullptr && uprv_strcmp(key, gNumberElementKeys[i]) == 0) {
if (!seenSymbol[i]) {
seenSymbol[i] = true;
dfs.setSymbol(
static_cast<DecimalFormatSymbols::ENumberFormatSymbol>(i),
value.getUnicodeString(errorCode));
if (U_FAILURE(errorCode)) { return; }
}
break;
}
}
}
}
// Returns true if all the symbols have been seen.
UBool seenAll() {
for (int32_t i=0; i<DecimalFormatSymbols::kFormatSymbolCount; i++) {
if (!seenSymbol[i]) {
return false;
}
}
return true;
}
// If monetary decimal or grouping were not explicitly set, then set them to be the
// same as their non-monetary counterparts.
void resolveMissingMonetarySeparators(const UnicodeString* fSymbols) {
if (!seenSymbol[DecimalFormatSymbols::kMonetarySeparatorSymbol]) {
dfs.setSymbol(
DecimalFormatSymbols::kMonetarySeparatorSymbol,
fSymbols[DecimalFormatSymbols::kDecimalSeparatorSymbol]);
}
if (!seenSymbol[DecimalFormatSymbols::kMonetaryGroupingSeparatorSymbol]) {
dfs.setSymbol(
DecimalFormatSymbols::kMonetaryGroupingSeparatorSymbol,
fSymbols[DecimalFormatSymbols::kGroupingSeparatorSymbol]);
}
}
};
struct CurrencySpacingSink : public ResourceSink {
DecimalFormatSymbols& dfs;
UBool hasBeforeCurrency;
UBool hasAfterCurrency;
CurrencySpacingSink(DecimalFormatSymbols& _dfs)
: dfs(_dfs), hasBeforeCurrency(false), hasAfterCurrency(false) {}
virtual ~CurrencySpacingSink();
virtual void put(const char *key, ResourceValue &value, UBool /*noFallback*/,
UErrorCode &errorCode) override {
ResourceTable spacingTypesTable = value.getTable(errorCode);
for (int32_t i = 0; spacingTypesTable.getKeyAndValue(i, key, value); ++i) {
UBool beforeCurrency;
if (uprv_strcmp(key, gBeforeCurrencyTag) == 0) {
beforeCurrency = true;
hasBeforeCurrency = true;
} else if (uprv_strcmp(key, gAfterCurrencyTag) == 0) {
beforeCurrency = false;
hasAfterCurrency = true;
} else {
continue;
}
ResourceTable patternsTable = value.getTable(errorCode);
for (int32_t j = 0; patternsTable.getKeyAndValue(j, key, value); ++j) {
UCurrencySpacing pattern;
if (uprv_strcmp(key, gCurrencyMatchTag) == 0) {
pattern = UNUM_CURRENCY_MATCH;
} else if (uprv_strcmp(key, gCurrencySudMatchTag) == 0) {
pattern = UNUM_CURRENCY_SURROUNDING_MATCH;
} else if (uprv_strcmp(key, gCurrencyInsertBtnTag) == 0) {
pattern = UNUM_CURRENCY_INSERT;
} else {
continue;
}
const UnicodeString& current = dfs.getPatternForCurrencySpacing(
pattern, beforeCurrency, errorCode);
if (current.isEmpty()) {
dfs.setPatternForCurrencySpacing(
pattern, beforeCurrency, value.getUnicodeString(errorCode));
}
}
}
}
void resolveMissing() {
// For consistency with Java, this method overwrites everything with the defaults unless
// both beforeCurrency and afterCurrency were found in CLDR.
static const char* defaults[] = { "[:letter:]", "[:digit:]", " " };
if (!hasBeforeCurrency || !hasAfterCurrency) {
for (int32_t pattern = 0; pattern < UNUM_CURRENCY_SPACING_COUNT; pattern++) {
dfs.setPatternForCurrencySpacing(static_cast<UCurrencySpacing>(pattern),
false, UnicodeString(defaults[pattern], -1, US_INV));
}
for (int32_t pattern = 0; pattern < UNUM_CURRENCY_SPACING_COUNT; pattern++) {
dfs.setPatternForCurrencySpacing(static_cast<UCurrencySpacing>(pattern),
true, UnicodeString(defaults[pattern], -1, US_INV));
}
}
}
};
// Virtual destructors must be defined out of line.
DecFmtSymDataSink::~DecFmtSymDataSink() {}
CurrencySpacingSink::~CurrencySpacingSink() {}
} // namespace
void
DecimalFormatSymbols::initialize(const Locale& loc, UErrorCode& status,
UBool useLastResortData, const NumberingSystem* ns)
{
if (U_FAILURE(status)) { return; }
// First initialize all the symbols to the fallbacks for anything we can't find
initialize();
//
// Next get the numbering system for this locale and set zero digit
// and the digit string based on the numbering system for the locale
//
LocalPointer<NumberingSystem> nsLocal;
if (ns == nullptr) {
// Use the numbering system according to the locale.
// Save it into a LocalPointer so it gets cleaned up.
nsLocal.adoptInstead(NumberingSystem::createInstance(loc, status));
ns = nsLocal.getAlias();
}
const char *nsName;
if (U_SUCCESS(status) && ns->getRadix() == 10 && !ns->isAlgorithmic()) {
nsName = ns->getName();
UnicodeString digitString(ns->getDescription());
int32_t digitIndex = 0;
UChar32 digit = digitString.char32At(0);
fSymbols[kZeroDigitSymbol].setTo(digit);
for (int32_t i = kOneDigitSymbol; i <= kNineDigitSymbol; ++i) {
digitIndex += U16_LENGTH(digit);
digit = digitString.char32At(digitIndex);
fSymbols[i].setTo(digit);
}
} else {
nsName = gLatn;
}
uprv_strcpy(this->nsName, nsName);
// Open resource bundles
const char* locStr = loc.getName();
LocalUResourceBundlePointer resource(ures_open(nullptr, locStr, &status));
LocalUResourceBundlePointer numberElementsRes(
ures_getByKeyWithFallback(resource.getAlias(), gNumberElements, nullptr, &status));
if (U_FAILURE(status)) {
if ( useLastResortData ) {
status = U_USING_DEFAULT_WARNING;
initialize();
}
return;
}
// Set locale IDs
// TODO: Is there a way to do this without depending on the resource bundle instance?
U_LOCALE_BASED(locBased, *this);
locBased.setLocaleIDs(
ures_getLocaleByType(
numberElementsRes.getAlias(),
ULOC_VALID_LOCALE, &status),
ures_getLocaleByType(
numberElementsRes.getAlias(),
ULOC_ACTUAL_LOCALE, &status),
status);
// Now load the rest of the data from the data sink.
// Start with loading this nsName if it is not Latin.
DecFmtSymDataSink sink(*this);
if (uprv_strcmp(nsName, gLatn) != 0) {
CharString path;
path.append(gNumberElements, status)
.append('/', status)
.append(nsName, status)
.append('/', status)
.append(gSymbols, status);
ures_getAllItemsWithFallback(resource.getAlias(), path.data(), sink, status);
// If no symbols exist for the given nsName and resource bundle, silently ignore
// and fall back to Latin.
if (status == U_MISSING_RESOURCE_ERROR) {
status = U_ZERO_ERROR;
} else if (U_FAILURE(status)) {
return;
}
}
// Continue with Latin if necessary.
if (!sink.seenAll()) {
ures_getAllItemsWithFallback(resource.getAlias(), gNumberElementsLatnSymbols, sink, status);
if (U_FAILURE(status)) { return; }
}
// Let the monetary number separators equal the default number separators if necessary.
sink.resolveMissingMonetarySeparators(fSymbols);
// Resolve codePointZero
UChar32 tempCodePointZero = -1;
for (int32_t i=0; i<=9; i++) {
const UnicodeString& stringDigit = getConstDigitSymbol(i);
if (stringDigit.countChar32() != 1) {
tempCodePointZero = -1;
break;
}
UChar32 cp = stringDigit.char32At(0);
if (i == 0) {
tempCodePointZero = cp;
} else if (cp != tempCodePointZero + i) {
tempCodePointZero = -1;
break;
}
}
fCodePointZero = tempCodePointZero;
// Get the default currency from the currency API.
UErrorCode internalStatus = U_ZERO_ERROR; // don't propagate failures out
char16_t curriso[4];
UnicodeString tempStr;
int32_t currisoLength = ucurr_forLocale(locStr, curriso, UPRV_LENGTHOF(curriso), &internalStatus);
if (U_SUCCESS(internalStatus) && currisoLength == 3) {
setCurrency(curriso, status);
} else {
setCurrency(nullptr, status);
}
// Currency Spacing.
LocalUResourceBundlePointer currencyResource(ures_open(U_ICUDATA_CURR, locStr, &status));
CurrencySpacingSink currencySink(*this);
ures_getAllItemsWithFallback(currencyResource.getAlias(), gCurrencySpacingTag, currencySink, status);
currencySink.resolveMissing();
if (U_FAILURE(status)) { return; }
}
void
DecimalFormatSymbols::initialize() {
/*
* These strings used to be in static arrays, but the HP/UX aCC compiler
* cannot initialize a static array with class constructors.
* markus 2000may25
*/
fSymbols[kDecimalSeparatorSymbol] = static_cast<char16_t>(0x2e); // '.' decimal separator
fSymbols[kGroupingSeparatorSymbol].remove(); // group (thousands) separator
fSymbols[kPatternSeparatorSymbol] = static_cast<char16_t>(0x3b); // ';' pattern separator
fSymbols[kPercentSymbol] = static_cast<char16_t>(0x25); // '%' percent sign
fSymbols[kZeroDigitSymbol] = static_cast<char16_t>(0x30); // '0' native 0 digit
fSymbols[kOneDigitSymbol] = static_cast<char16_t>(0x31); // '1' native 1 digit
fSymbols[kTwoDigitSymbol] = static_cast<char16_t>(0x32); // '2' native 2 digit
fSymbols[kThreeDigitSymbol] = static_cast<char16_t>(0x33); // '3' native 3 digit
fSymbols[kFourDigitSymbol] = static_cast<char16_t>(0x34); // '4' native 4 digit
fSymbols[kFiveDigitSymbol] = static_cast<char16_t>(0x35); // '5' native 5 digit
fSymbols[kSixDigitSymbol] = static_cast<char16_t>(0x36); // '6' native 6 digit
fSymbols[kSevenDigitSymbol] = static_cast<char16_t>(0x37); // '7' native 7 digit
fSymbols[kEightDigitSymbol] = static_cast<char16_t>(0x38); // '8' native 8 digit
fSymbols[kNineDigitSymbol] = static_cast<char16_t>(0x39); // '9' native 9 digit
fSymbols[kDigitSymbol] = static_cast<char16_t>(0x23); // '#' pattern digit
fSymbols[kPlusSignSymbol] = static_cast<char16_t>(0x002b); // '+' plus sign
fSymbols[kMinusSignSymbol] = static_cast<char16_t>(0x2d); // '-' minus sign
fSymbols[kCurrencySymbol] = static_cast<char16_t>(0xa4); // 'OX' currency symbol
fSymbols[kIntlCurrencySymbol].setTo(true, INTL_CURRENCY_SYMBOL_STR, 2);
fSymbols[kMonetarySeparatorSymbol] = static_cast<char16_t>(0x2e); // '.' monetary decimal separator
fSymbols[kExponentialSymbol] = static_cast<char16_t>(0x45); // 'E' exponential
fSymbols[kPerMillSymbol] = static_cast<char16_t>(0x2030); // '%o' per mill
fSymbols[kPadEscapeSymbol] = static_cast<char16_t>(0x2a); // '*' pad escape symbol
fSymbols[kInfinitySymbol] = static_cast<char16_t>(0x221e); // 'oo' infinite
fSymbols[kNaNSymbol] = static_cast<char16_t>(0xfffd); // SUB NaN
fSymbols[kSignificantDigitSymbol] = static_cast<char16_t>(0x0040); // '@' significant digit
fSymbols[kMonetaryGroupingSeparatorSymbol].remove(); //
fSymbols[kExponentMultiplicationSymbol] = static_cast<char16_t>(0xd7); // 'x' multiplication symbol for exponents
fSymbols[kApproximatelySignSymbol] = u'~'; // '~' approximately sign
fIsCustomCurrencySymbol = false;
fIsCustomIntlCurrencySymbol = false;
fCodePointZero = 0x30;
U_ASSERT(fCodePointZero == fSymbols[kZeroDigitSymbol].char32At(0));
currPattern = nullptr;
nsName[0] = 0;
}
void DecimalFormatSymbols::setCurrency(const char16_t* currency, UErrorCode& status) {
// TODO: If this method is made public:
// - Adopt ICU4J behavior of not allowing currency to be null.
// - Also verify that the length of currency is 3.
if (!currency) {
return;
}
UnicodeString tempStr;
uprv_getStaticCurrencyName(currency, locale.getName(), tempStr, status);
if (U_SUCCESS(status)) {
fSymbols[kIntlCurrencySymbol].setTo(currency, 3);
fSymbols[kCurrencySymbol] = tempStr;
}
char cc[4]={0};
u_UCharsToChars(currency, cc, 3);
/* An explicit currency was requested */
// TODO(ICU-13297): Move this data loading logic into a centralized place
UErrorCode localStatus = U_ZERO_ERROR;
LocalUResourceBundlePointer rbTop(ures_open(U_ICUDATA_CURR, locale.getName(), &localStatus));
LocalUResourceBundlePointer rb(
ures_getByKeyWithFallback(rbTop.getAlias(), "Currencies", nullptr, &localStatus));
ures_getByKeyWithFallback(rb.getAlias(), cc, rb.getAlias(), &localStatus);
if(U_SUCCESS(localStatus) && ures_getSize(rb.getAlias())>2) { // the length is 3 if more data is present
ures_getByIndex(rb.getAlias(), 2, rb.getAlias(), &localStatus);
int32_t currPatternLen = 0;
currPattern =
ures_getStringByIndex(rb.getAlias(), static_cast<int32_t>(0), &currPatternLen, &localStatus);
UnicodeString decimalSep =
ures_getUnicodeStringByIndex(rb.getAlias(), static_cast<int32_t>(1), &localStatus);
UnicodeString groupingSep =
ures_getUnicodeStringByIndex(rb.getAlias(), static_cast<int32_t>(2), &localStatus);
if(U_SUCCESS(localStatus)){
fSymbols[kMonetaryGroupingSeparatorSymbol] = groupingSep;
fSymbols[kMonetarySeparatorSymbol] = decimalSep;
//pattern.setTo(true, currPattern, currPatternLen);
}
}
/* else An explicit currency was requested and is unknown or locale data is malformed. */
/* ucurr_* API will get the correct value later on. */
}
Locale
DecimalFormatSymbols::getLocale(ULocDataLocaleType type, UErrorCode& status) const {
return LocaleBased::getLocale(validLocale, actualLocale, type, status);
}
const UnicodeString&
DecimalFormatSymbols::getPatternForCurrencySpacing(UCurrencySpacing type,
UBool beforeCurrency,
UErrorCode& status) const {
if (U_FAILURE(status)) {
return fNoSymbol; // always empty.
}
if (beforeCurrency) {
return currencySpcBeforeSym[static_cast<int32_t>(type)];
} else {
return currencySpcAfterSym[static_cast<int32_t>(type)];
}
}
void
DecimalFormatSymbols::setPatternForCurrencySpacing(UCurrencySpacing type,
UBool beforeCurrency,
const UnicodeString& pattern) {
if (beforeCurrency) {
currencySpcBeforeSym[static_cast<int32_t>(type)] = pattern;
} else {
currencySpcAfterSym[static_cast<int32_t>(type)] = pattern;
}
}
U_NAMESPACE_END
#endif /* #if !UCONFIG_NO_FORMATTING */
//eof

432
deps/icu-small/source/i18n/decContext.cpp vendored Normal file
View File

@ -0,0 +1,432 @@
// © 2016 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html
/* ------------------------------------------------------------------ */
/* Decimal Context module */
/* ------------------------------------------------------------------ */
/* Copyright (c) IBM Corporation, 2000-2012. All rights reserved. */
/* */
/* This software is made available under the terms of the */
/* ICU License -- ICU 1.8.1 and later. */
/* */
/* The description and User's Guide ("The decNumber C Library") for */
/* this software is called decNumber.pdf. This document is */
/* available, together with arithmetic and format specifications, */
/* testcases, and Web links, on the General Decimal Arithmetic page. */
/* */
/* Please send comments, suggestions, and corrections to the author: */
/* mfc@uk.ibm.com */
/* Mike Cowlishaw, IBM Fellow */
/* IBM UK, PO Box 31, Birmingham Road, Warwick CV34 5JL, UK */
/* ------------------------------------------------------------------ */
/* This module comprises the routines for handling arithmetic */
/* context structures. */
/* ------------------------------------------------------------------ */
#include <string.h> /* for strcmp */
#include <stdio.h> /* for printf if DECCHECK */
#include "decContext.h" /* context and base types */
#include "decNumberLocal.h" /* decNumber local types, etc. */
#if 0 /* ICU: No need to test endianness at runtime. */
/* compile-time endian tester [assumes sizeof(Int)>1] */
static const Int mfcone=1; /* constant 1 */
static const Flag *mfctop=(Flag *)&mfcone; /* -> top byte */
#define LITEND *mfctop /* named flag; 1=little-endian */
#endif
/* ------------------------------------------------------------------ */
/* decContextClearStatus -- clear bits in current status */
/* */
/* context is the context structure to be queried */
/* mask indicates the bits to be cleared (the status bit that */
/* corresponds to each 1 bit in the mask is cleared) */
/* returns context */
/* */
/* No error is possible. */
/* ------------------------------------------------------------------ */
U_CAPI decContext * U_EXPORT2 uprv_decContextClearStatus(decContext *context, uInt mask) {
context->status&=~mask;
return context;
} /* decContextClearStatus */
/* ------------------------------------------------------------------ */
/* decContextDefault -- initialize a context structure */
/* */
/* context is the structure to be initialized */
/* kind selects the required set of default values, one of: */
/* DEC_INIT_BASE -- select ANSI X3-274 defaults */
/* DEC_INIT_DECIMAL32 -- select IEEE 754 defaults, 32-bit */
/* DEC_INIT_DECIMAL64 -- select IEEE 754 defaults, 64-bit */
/* DEC_INIT_DECIMAL128 -- select IEEE 754 defaults, 128-bit */
/* For any other value a valid context is returned, but with */
/* Invalid_operation set in the status field. */
/* returns a context structure with the appropriate initial values. */
/* ------------------------------------------------------------------ */
U_CAPI decContext * U_EXPORT2 uprv_decContextDefault(decContext *context, Int kind) {
/* set defaults... */
context->digits=9; /* 9 digits */
context->emax=DEC_MAX_EMAX; /* 9-digit exponents */
context->emin=DEC_MIN_EMIN; /* .. balanced */
context->round=DEC_ROUND_HALF_UP; /* 0.5 rises */
context->traps=DEC_Errors; /* all but informational */
context->status=0; /* cleared */
context->clamp=0; /* no clamping */
#if DECSUBSET
context->extended=0; /* cleared */
#endif
switch (kind) {
case DEC_INIT_BASE:
/* [use defaults] */
break;
case DEC_INIT_DECIMAL32:
context->digits=7; /* digits */
context->emax=96; /* Emax */
context->emin=-95; /* Emin */
context->round=DEC_ROUND_HALF_EVEN; /* 0.5 to nearest even */
context->traps=0; /* no traps set */
context->clamp=1; /* clamp exponents */
#if DECSUBSET
context->extended=1; /* set */
#endif
break;
case DEC_INIT_DECIMAL64:
context->digits=16; /* digits */
context->emax=384; /* Emax */
context->emin=-383; /* Emin */
context->round=DEC_ROUND_HALF_EVEN; /* 0.5 to nearest even */
context->traps=0; /* no traps set */
context->clamp=1; /* clamp exponents */
#if DECSUBSET
context->extended=1; /* set */
#endif
break;
case DEC_INIT_DECIMAL128:
context->digits=34; /* digits */
context->emax=6144; /* Emax */
context->emin=-6143; /* Emin */
context->round=DEC_ROUND_HALF_EVEN; /* 0.5 to nearest even */
context->traps=0; /* no traps set */
context->clamp=1; /* clamp exponents */
#if DECSUBSET
context->extended=1; /* set */
#endif
break;
default: /* invalid Kind */
/* use defaults, and .. */
uprv_decContextSetStatus(context, DEC_Invalid_operation); /* trap */
}
return context;} /* decContextDefault */
/* ------------------------------------------------------------------ */
/* decContextGetRounding -- return current rounding mode */
/* */
/* context is the context structure to be queried */
/* returns the rounding mode */
/* */
/* No error is possible. */
/* ------------------------------------------------------------------ */
U_CAPI enum rounding U_EXPORT2 uprv_decContextGetRounding(decContext *context) {
return context->round;
} /* decContextGetRounding */
/* ------------------------------------------------------------------ */
/* decContextGetStatus -- return current status */
/* */
/* context is the context structure to be queried */
/* returns status */
/* */
/* No error is possible. */
/* ------------------------------------------------------------------ */
U_CAPI uInt U_EXPORT2 uprv_decContextGetStatus(decContext *context) {
return context->status;
} /* decContextGetStatus */
/* ------------------------------------------------------------------ */
/* decContextRestoreStatus -- restore bits in current status */
/* */
/* context is the context structure to be updated */
/* newstatus is the source for the bits to be restored */
/* mask indicates the bits to be restored (the status bit that */
/* corresponds to each 1 bit in the mask is set to the value of */
/* the corresponding bit in newstatus) */
/* returns context */
/* */
/* No error is possible. */
/* ------------------------------------------------------------------ */
U_CAPI decContext * U_EXPORT2 uprv_decContextRestoreStatus(decContext *context,
uInt newstatus, uInt mask) {
context->status&=~mask; /* clear the selected bits */
context->status|=(mask&newstatus); /* or in the new bits */
return context;
} /* decContextRestoreStatus */
/* ------------------------------------------------------------------ */
/* decContextSaveStatus -- save bits in current status */
/* */
/* context is the context structure to be queried */
/* mask indicates the bits to be saved (the status bits that */
/* correspond to each 1 bit in the mask are saved) */
/* returns the AND of the mask and the current status */
/* */
/* No error is possible. */
/* ------------------------------------------------------------------ */
U_CAPI uInt U_EXPORT2 uprv_decContextSaveStatus(decContext *context, uInt mask) {
return context->status&mask;
} /* decContextSaveStatus */
/* ------------------------------------------------------------------ */
/* decContextSetRounding -- set current rounding mode */
/* */
/* context is the context structure to be updated */
/* newround is the value which will replace the current mode */
/* returns context */
/* */
/* No error is possible. */
/* ------------------------------------------------------------------ */
U_CAPI decContext * U_EXPORT2 uprv_decContextSetRounding(decContext *context,
enum rounding newround) {
context->round=newround;
return context;
} /* decContextSetRounding */
/* ------------------------------------------------------------------ */
/* decContextSetStatus -- set status and raise trap if appropriate */
/* */
/* context is the context structure to be updated */
/* status is the DEC_ exception code */
/* returns the context structure */
/* */
/* Control may never return from this routine, if there is a signal */
/* handler and it takes a long jump. */
/* ------------------------------------------------------------------ */
U_CAPI decContext * U_EXPORT2 uprv_decContextSetStatus(decContext *context, uInt status) {
context->status|=status;
#if 0 /* ICU: Do not raise signals. */
if (status & context->traps) raise(SIGFPE);
#endif
return context;} /* decContextSetStatus */
/* ------------------------------------------------------------------ */
/* decContextSetStatusFromString -- set status from a string + trap */
/* */
/* context is the context structure to be updated */
/* string is a string exactly equal to one that might be returned */
/* by decContextStatusToString */
/* */
/* The status bit corresponding to the string is set, and a trap */
/* is raised if appropriate. */
/* */
/* returns the context structure, unless the string is equal to */
/* DEC_Condition_MU or is not recognized. In these cases nullptr is */
/* returned. */
/* ------------------------------------------------------------------ */
U_CAPI decContext * U_EXPORT2 uprv_decContextSetStatusFromString(decContext *context,
const char *string) {
if (strcmp(string, DEC_Condition_CS)==0)
return uprv_decContextSetStatus(context, DEC_Conversion_syntax);
if (strcmp(string, DEC_Condition_DZ)==0)
return uprv_decContextSetStatus(context, DEC_Division_by_zero);
if (strcmp(string, DEC_Condition_DI)==0)
return uprv_decContextSetStatus(context, DEC_Division_impossible);
if (strcmp(string, DEC_Condition_DU)==0)
return uprv_decContextSetStatus(context, DEC_Division_undefined);
if (strcmp(string, DEC_Condition_IE)==0)
return uprv_decContextSetStatus(context, DEC_Inexact);
if (strcmp(string, DEC_Condition_IS)==0)
return uprv_decContextSetStatus(context, DEC_Insufficient_storage);
if (strcmp(string, DEC_Condition_IC)==0)
return uprv_decContextSetStatus(context, DEC_Invalid_context);
if (strcmp(string, DEC_Condition_IO)==0)
return uprv_decContextSetStatus(context, DEC_Invalid_operation);
#if DECSUBSET
if (strcmp(string, DEC_Condition_LD)==0)
return uprv_decContextSetStatus(context, DEC_Lost_digits);
#endif
if (strcmp(string, DEC_Condition_OV)==0)
return uprv_decContextSetStatus(context, DEC_Overflow);
if (strcmp(string, DEC_Condition_PA)==0)
return uprv_decContextSetStatus(context, DEC_Clamped);
if (strcmp(string, DEC_Condition_RO)==0)
return uprv_decContextSetStatus(context, DEC_Rounded);
if (strcmp(string, DEC_Condition_SU)==0)
return uprv_decContextSetStatus(context, DEC_Subnormal);
if (strcmp(string, DEC_Condition_UN)==0)
return uprv_decContextSetStatus(context, DEC_Underflow);
if (strcmp(string, DEC_Condition_ZE)==0)
return context;
return nullptr; /* Multiple status, or unknown */
} /* decContextSetStatusFromString */
/* ------------------------------------------------------------------ */
/* decContextSetStatusFromStringQuiet -- set status from a string */
/* */
/* context is the context structure to be updated */
/* string is a string exactly equal to one that might be returned */
/* by decContextStatusToString */
/* */
/* The status bit corresponding to the string is set; no trap is */
/* raised. */
/* */
/* returns the context structure, unless the string is equal to */
/* DEC_Condition_MU or is not recognized. In these cases nullptr is */
/* returned. */
/* ------------------------------------------------------------------ */
U_CAPI decContext * U_EXPORT2 uprv_decContextSetStatusFromStringQuiet(decContext *context,
const char *string) {
if (strcmp(string, DEC_Condition_CS)==0)
return uprv_decContextSetStatusQuiet(context, DEC_Conversion_syntax);
if (strcmp(string, DEC_Condition_DZ)==0)
return uprv_decContextSetStatusQuiet(context, DEC_Division_by_zero);
if (strcmp(string, DEC_Condition_DI)==0)
return uprv_decContextSetStatusQuiet(context, DEC_Division_impossible);
if (strcmp(string, DEC_Condition_DU)==0)
return uprv_decContextSetStatusQuiet(context, DEC_Division_undefined);
if (strcmp(string, DEC_Condition_IE)==0)
return uprv_decContextSetStatusQuiet(context, DEC_Inexact);
if (strcmp(string, DEC_Condition_IS)==0)
return uprv_decContextSetStatusQuiet(context, DEC_Insufficient_storage);
if (strcmp(string, DEC_Condition_IC)==0)
return uprv_decContextSetStatusQuiet(context, DEC_Invalid_context);
if (strcmp(string, DEC_Condition_IO)==0)
return uprv_decContextSetStatusQuiet(context, DEC_Invalid_operation);
#if DECSUBSET
if (strcmp(string, DEC_Condition_LD)==0)
return uprv_decContextSetStatusQuiet(context, DEC_Lost_digits);
#endif
if (strcmp(string, DEC_Condition_OV)==0)
return uprv_decContextSetStatusQuiet(context, DEC_Overflow);
if (strcmp(string, DEC_Condition_PA)==0)
return uprv_decContextSetStatusQuiet(context, DEC_Clamped);
if (strcmp(string, DEC_Condition_RO)==0)
return uprv_decContextSetStatusQuiet(context, DEC_Rounded);
if (strcmp(string, DEC_Condition_SU)==0)
return uprv_decContextSetStatusQuiet(context, DEC_Subnormal);
if (strcmp(string, DEC_Condition_UN)==0)
return uprv_decContextSetStatusQuiet(context, DEC_Underflow);
if (strcmp(string, DEC_Condition_ZE)==0)
return context;
return nullptr; /* Multiple status, or unknown */
} /* decContextSetStatusFromStringQuiet */
/* ------------------------------------------------------------------ */
/* decContextSetStatusQuiet -- set status without trap */
/* */
/* context is the context structure to be updated */
/* status is the DEC_ exception code */
/* returns the context structure */
/* */
/* No error is possible. */
/* ------------------------------------------------------------------ */
U_CAPI decContext * U_EXPORT2 uprv_decContextSetStatusQuiet(decContext *context, uInt status) {
context->status|=status;
return context;} /* decContextSetStatusQuiet */
/* ------------------------------------------------------------------ */
/* decContextStatusToString -- convert status flags to a string */
/* */
/* context is a context with valid status field */
/* */
/* returns a constant string describing the condition. If multiple */
/* (or no) flags are set, a generic constant message is returned. */
/* ------------------------------------------------------------------ */
U_CAPI const char * U_EXPORT2 uprv_decContextStatusToString(const decContext *context) {
Int status=context->status;
/* test the five IEEE first, as some of the others are ambiguous when */
/* DECEXTFLAG=0 */
if (status==DEC_Invalid_operation ) return DEC_Condition_IO;
if (status==DEC_Division_by_zero ) return DEC_Condition_DZ;
if (status==DEC_Overflow ) return DEC_Condition_OV;
if (status==DEC_Underflow ) return DEC_Condition_UN;
if (status==DEC_Inexact ) return DEC_Condition_IE;
if (status==DEC_Division_impossible ) return DEC_Condition_DI;
if (status==DEC_Division_undefined ) return DEC_Condition_DU;
if (status==DEC_Rounded ) return DEC_Condition_RO;
if (status==DEC_Clamped ) return DEC_Condition_PA;
if (status==DEC_Subnormal ) return DEC_Condition_SU;
if (status==DEC_Conversion_syntax ) return DEC_Condition_CS;
if (status==DEC_Insufficient_storage ) return DEC_Condition_IS;
if (status==DEC_Invalid_context ) return DEC_Condition_IC;
#if DECSUBSET
if (status==DEC_Lost_digits ) return DEC_Condition_LD;
#endif
if (status==0 ) return DEC_Condition_ZE;
return DEC_Condition_MU; /* Multiple errors */
} /* decContextStatusToString */
/* ------------------------------------------------------------------ */
/* decContextTestEndian -- test whether DECLITEND is set correctly */
/* */
/* quiet is 1 to suppress message; 0 otherwise */
/* returns 0 if DECLITEND is correct */
/* 1 if DECLITEND is incorrect and should be 1 */
/* -1 if DECLITEND is incorrect and should be 0 */
/* */
/* A message is displayed if the return value is not 0 and quiet==0. */
/* */
/* No error is possible. */
/* ------------------------------------------------------------------ */
#if 0 /* ICU: Unused function. Anyway, do not call printf(). */
U_CAPI Int U_EXPORT2 uprv_decContextTestEndian(Flag quiet) {
Int res=0; /* optimist */
uInt dle=(uInt)DECLITEND; /* unsign */
if (dle>1) dle=1; /* ensure 0 or 1 */
if (LITEND!=DECLITEND) {
const char *adj;
if (!quiet) {
if (LITEND) adj="little";
else adj="big";
printf("Warning: DECLITEND is set to %d, but this computer appears to be %s-endian\n",
DECLITEND, adj);
}
res=(Int)LITEND-dle;
}
return res;
} /* decContextTestEndian */
#endif
/* ------------------------------------------------------------------ */
/* decContextTestSavedStatus -- test bits in saved status */
/* */
/* oldstatus is the status word to be tested */
/* mask indicates the bits to be tested (the oldstatus bits that */
/* correspond to each 1 bit in the mask are tested) */
/* returns 1 if any of the tested bits are 1, or 0 otherwise */
/* */
/* No error is possible. */
/* ------------------------------------------------------------------ */
U_CAPI uInt U_EXPORT2 uprv_decContextTestSavedStatus(uInt oldstatus, uInt mask) {
return (oldstatus&mask)!=0;
} /* decContextTestSavedStatus */
/* ------------------------------------------------------------------ */
/* decContextTestStatus -- test bits in current status */
/* */
/* context is the context structure to be updated */
/* mask indicates the bits to be tested (the status bits that */
/* correspond to each 1 bit in the mask are tested) */
/* returns 1 if any of the tested bits are 1, or 0 otherwise */
/* */
/* No error is possible. */
/* ------------------------------------------------------------------ */
U_CAPI uInt U_EXPORT2 uprv_decContextTestStatus(decContext *context, uInt mask) {
return (context->status&mask)!=0;
} /* decContextTestStatus */
/* ------------------------------------------------------------------ */
/* decContextZeroStatus -- clear all status bits */
/* */
/* context is the context structure to be updated */
/* returns context */
/* */
/* No error is possible. */
/* ------------------------------------------------------------------ */
U_CAPI decContext * U_EXPORT2 uprv_decContextZeroStatus(decContext *context) {
context->status=0;
return context;
} /* decContextZeroStatus */

269
deps/icu-small/source/i18n/decContext.h vendored Normal file
View File

@ -0,0 +1,269 @@
// © 2016 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html
/* ------------------------------------------------------------------ */
/* Decimal Context module header */
/* ------------------------------------------------------------------ */
/* Copyright (c) IBM Corporation, 2000-2011. All rights reserved. */
/* */
/* This software is made available under the terms of the */
/* ICU License -- ICU 1.8.1 and later. */
/* */
/* The description and User's Guide ("The decNumber C Library") for */
/* this software is called decNumber.pdf. This document is */
/* available, together with arithmetic and format specifications, */
/* testcases, and Web links, on the General Decimal Arithmetic page. */
/* */
/* Please send comments, suggestions, and corrections to the author: */
/* mfc@uk.ibm.com */
/* Mike Cowlishaw, IBM Fellow */
/* IBM UK, PO Box 31, Birmingham Road, Warwick CV34 5JL, UK */
/* ------------------------------------------------------------------ */
/* Modified version, for use from within ICU.
* Renamed public functions, to avoid an unwanted export of the
* standard names from the ICU library.
*
* Use ICU's uprv_malloc() and uprv_free()
*
* Revert comment syntax to plain C
*
* Remove a few compiler warnings.
*/
#include "unicode/utypes.h"
#include "putilimp.h"
/* */
/* Context variables must always have valid values: */
/* */
/* status -- [any bits may be cleared, but not set, by user] */
/* round -- must be one of the enumerated rounding modes */
/* */
/* The following variables are implied for fixed size formats (i.e., */
/* they are ignored) but should still be set correctly in case used */
/* with decNumber functions: */
/* */
/* clamp -- must be either 0 or 1 */
/* digits -- must be in the range 1 through 999999999 */
/* emax -- must be in the range 0 through 999999999 */
/* emin -- must be in the range 0 through -999999999 */
/* extended -- must be either 0 or 1 [present only if DECSUBSET] */
/* traps -- only defined bits may be set */
/* */
/* ------------------------------------------------------------------ */
#if !defined(DECCONTEXT)
#define DECCONTEXT
#define DECCNAME "decContext" /* Short name */
#define DECCFULLNAME "Decimal Context Descriptor" /* Verbose name */
#define DECCAUTHOR "Mike Cowlishaw" /* Who to blame */
#if !defined(int32_t)
/* #include <stdint.h> */ /* C99 standard integers */
#endif
#include <stdio.h> /* for printf, etc. */
#include <signal.h> /* for traps */
/* Extended flags setting -- set this to 0 to use only IEEE flags */
#if !defined(DECEXTFLAG)
#define DECEXTFLAG 1 /* 1=enable extended flags */
#endif
/* Conditional code flag -- set this to 0 for best performance */
#if !defined(DECSUBSET)
#define DECSUBSET 0 /* 1=enable subset arithmetic */
#endif
/* Context for operations, with associated constants */
enum rounding {
DEC_ROUND_CEILING, /* round towards +infinity */
DEC_ROUND_UP, /* round away from 0 */
DEC_ROUND_HALF_UP, /* 0.5 rounds up */
DEC_ROUND_HALF_EVEN, /* 0.5 rounds to nearest even */
DEC_ROUND_HALF_DOWN, /* 0.5 rounds down */
DEC_ROUND_DOWN, /* round towards 0 (truncate) */
DEC_ROUND_FLOOR, /* round towards -infinity */
DEC_ROUND_05UP, /* round for reround */
DEC_ROUND_MAX /* enum must be less than this */
};
#define DEC_ROUND_DEFAULT DEC_ROUND_HALF_EVEN;
typedef struct {
int32_t digits; /* working precision */
int32_t emax; /* maximum positive exponent */
int32_t emin; /* minimum negative exponent */
enum rounding round; /* rounding mode */
uint32_t traps; /* trap-enabler flags */
uint32_t status; /* status flags */
uint8_t clamp; /* flag: apply IEEE exponent clamp */
#if DECSUBSET
uint8_t extended; /* flag: special-values allowed */
#endif
} decContext;
/* Maxima and Minima for context settings */
#define DEC_MAX_DIGITS 999999999
#define DEC_MIN_DIGITS 1
#define DEC_MAX_EMAX 999999999
#define DEC_MIN_EMAX 0
#define DEC_MAX_EMIN 0
#define DEC_MIN_EMIN -999999999
#define DEC_MAX_MATH 999999 /* max emax, etc., for math funcs. */
/* Classifications for decimal numbers, aligned with 754 (note that */
/* 'normal' and 'subnormal' are meaningful only with a decContext */
/* or a fixed size format). */
enum decClass {
DEC_CLASS_SNAN,
DEC_CLASS_QNAN,
DEC_CLASS_NEG_INF,
DEC_CLASS_NEG_NORMAL,
DEC_CLASS_NEG_SUBNORMAL,
DEC_CLASS_NEG_ZERO,
DEC_CLASS_POS_ZERO,
DEC_CLASS_POS_SUBNORMAL,
DEC_CLASS_POS_NORMAL,
DEC_CLASS_POS_INF
};
/* Strings for the decClasses */
#define DEC_ClassString_SN "sNaN"
#define DEC_ClassString_QN "NaN"
#define DEC_ClassString_NI "-Infinity"
#define DEC_ClassString_NN "-Normal"
#define DEC_ClassString_NS "-Subnormal"
#define DEC_ClassString_NZ "-Zero"
#define DEC_ClassString_PZ "+Zero"
#define DEC_ClassString_PS "+Subnormal"
#define DEC_ClassString_PN "+Normal"
#define DEC_ClassString_PI "+Infinity"
#define DEC_ClassString_UN "Invalid"
/* Trap-enabler and Status flags (exceptional conditions), and */
/* their names. The top byte is reserved for internal use */
#if DECEXTFLAG
/* Extended flags */
#define DEC_Conversion_syntax 0x00000001
#define DEC_Division_by_zero 0x00000002
#define DEC_Division_impossible 0x00000004
#define DEC_Division_undefined 0x00000008
#define DEC_Insufficient_storage 0x00000010 /* [when malloc fails] */
#define DEC_Inexact 0x00000020
#define DEC_Invalid_context 0x00000040
#define DEC_Invalid_operation 0x00000080
#if DECSUBSET
#define DEC_Lost_digits 0x00000100
#endif
#define DEC_Overflow 0x00000200
#define DEC_Clamped 0x00000400
#define DEC_Rounded 0x00000800
#define DEC_Subnormal 0x00001000
#define DEC_Underflow 0x00002000
#else
/* IEEE flags only */
#define DEC_Conversion_syntax 0x00000010
#define DEC_Division_by_zero 0x00000002
#define DEC_Division_impossible 0x00000010
#define DEC_Division_undefined 0x00000010
#define DEC_Insufficient_storage 0x00000010 /* [when malloc fails] */
#define DEC_Inexact 0x00000001
#define DEC_Invalid_context 0x00000010
#define DEC_Invalid_operation 0x00000010
#if DECSUBSET
#define DEC_Lost_digits 0x00000000
#endif
#define DEC_Overflow 0x00000008
#define DEC_Clamped 0x00000000
#define DEC_Rounded 0x00000000
#define DEC_Subnormal 0x00000000
#define DEC_Underflow 0x00000004
#endif
/* IEEE 754 groupings for the flags */
/* [DEC_Clamped, DEC_Lost_digits, DEC_Rounded, and DEC_Subnormal */
/* are not in IEEE 754] */
#define DEC_IEEE_754_Division_by_zero (DEC_Division_by_zero)
#if DECSUBSET
#define DEC_IEEE_754_Inexact (DEC_Inexact | DEC_Lost_digits)
#else
#define DEC_IEEE_754_Inexact (DEC_Inexact)
#endif
#define DEC_IEEE_754_Invalid_operation (DEC_Conversion_syntax | \
DEC_Division_impossible | \
DEC_Division_undefined | \
DEC_Insufficient_storage | \
DEC_Invalid_context | \
DEC_Invalid_operation)
#define DEC_IEEE_754_Overflow (DEC_Overflow)
#define DEC_IEEE_754_Underflow (DEC_Underflow)
/* flags which are normally errors (result is qNaN, infinite, or 0) */
#define DEC_Errors (DEC_IEEE_754_Division_by_zero | \
DEC_IEEE_754_Invalid_operation | \
DEC_IEEE_754_Overflow | DEC_IEEE_754_Underflow)
/* flags which cause a result to become qNaN */
#define DEC_NaNs DEC_IEEE_754_Invalid_operation
/* flags which are normally for information only (finite results) */
#if DECSUBSET
#define DEC_Information (DEC_Clamped | DEC_Rounded | DEC_Inexact \
| DEC_Lost_digits)
#else
#define DEC_Information (DEC_Clamped | DEC_Rounded | DEC_Inexact)
#endif
/* IEEE 854 names (for compatibility with older decNumber versions) */
#define DEC_IEEE_854_Division_by_zero DEC_IEEE_754_Division_by_zero
#define DEC_IEEE_854_Inexact DEC_IEEE_754_Inexact
#define DEC_IEEE_854_Invalid_operation DEC_IEEE_754_Invalid_operation
#define DEC_IEEE_854_Overflow DEC_IEEE_754_Overflow
#define DEC_IEEE_854_Underflow DEC_IEEE_754_Underflow
/* Name strings for the exceptional conditions */
#define DEC_Condition_CS "Conversion syntax"
#define DEC_Condition_DZ "Division by zero"
#define DEC_Condition_DI "Division impossible"
#define DEC_Condition_DU "Division undefined"
#define DEC_Condition_IE "Inexact"
#define DEC_Condition_IS "Insufficient storage"
#define DEC_Condition_IC "Invalid context"
#define DEC_Condition_IO "Invalid operation"
#if DECSUBSET
#define DEC_Condition_LD "Lost digits"
#endif
#define DEC_Condition_OV "Overflow"
#define DEC_Condition_PA "Clamped"
#define DEC_Condition_RO "Rounded"
#define DEC_Condition_SU "Subnormal"
#define DEC_Condition_UN "Underflow"
#define DEC_Condition_ZE "No status"
#define DEC_Condition_MU "Multiple status"
#define DEC_Condition_Length 21 /* length of the longest string, */
/* including terminator */
/* Initialization descriptors, used by decContextDefault */
#define DEC_INIT_BASE 0
#define DEC_INIT_DECIMAL32 32
#define DEC_INIT_DECIMAL64 64
#define DEC_INIT_DECIMAL128 128
/* Synonyms */
#define DEC_INIT_DECSINGLE DEC_INIT_DECIMAL32
#define DEC_INIT_DECDOUBLE DEC_INIT_DECIMAL64
#define DEC_INIT_DECQUAD DEC_INIT_DECIMAL128
/* decContext routines */
U_CAPI decContext * U_EXPORT2 uprv_decContextClearStatus(decContext *, uint32_t);
U_CAPI decContext * U_EXPORT2 uprv_decContextDefault(decContext *, int32_t);
U_CAPI enum rounding U_EXPORT2 uprv_decContextGetRounding(decContext *);
U_CAPI uint32_t U_EXPORT2 uprv_decContextGetStatus(decContext *);
U_CAPI decContext * U_EXPORT2 uprv_decContextRestoreStatus(decContext *, uint32_t, uint32_t);
U_CAPI uint32_t U_EXPORT2 uprv_decContextSaveStatus(decContext *, uint32_t);
U_CAPI decContext * U_EXPORT2 uprv_decContextSetRounding(decContext *, enum rounding);
U_CAPI decContext * U_EXPORT2 uprv_decContextSetStatus(decContext *, uint32_t);
U_CAPI decContext * U_EXPORT2 uprv_decContextSetStatusFromString(decContext *, const char *);
U_CAPI decContext * U_EXPORT2 uprv_decContextSetStatusFromStringQuiet(decContext *, const char *);
U_CAPI decContext * U_EXPORT2 uprv_decContextSetStatusQuiet(decContext *, uint32_t);
U_CAPI const char * U_EXPORT2 uprv_decContextStatusToString(const decContext *);
U_CAPI uint32_t U_EXPORT2 uprv_decContextTestSavedStatus(uint32_t, uint32_t);
U_CAPI uint32_t U_EXPORT2 uprv_decContextTestStatus(decContext *, uint32_t);
U_CAPI decContext * U_EXPORT2 uprv_decContextZeroStatus(decContext *);
#endif

8185
deps/icu-small/source/i18n/decNumber.cpp vendored Normal file
View File

File diff suppressed because it is too large Load Diff

198
deps/icu-small/source/i18n/decNumber.h vendored Normal file
View File

@ -0,0 +1,198 @@
// © 2016 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html
/* ------------------------------------------------------------------ */
/* Decimal Number arithmetic module header */
/* ------------------------------------------------------------------ */
/* Copyright (c) IBM Corporation, 2000-2010. All rights reserved. */
/* */
/* This software is made available under the terms of the */
/* ICU License -- ICU 1.8.1 and later. */
/* */
/* The description and User's Guide ("The decNumber C Library") for */
/* this software is called decNumber.pdf. This document is */
/* available, together with arithmetic and format specifications, */
/* testcases, and Web links, on the General Decimal Arithmetic page. */
/* */
/* Please send comments, suggestions, and corrections to the author: */
/* mfc@uk.ibm.com */
/* Mike Cowlishaw, IBM Fellow */
/* IBM UK, PO Box 31, Birmingham Road, Warwick CV34 5JL, UK */
/* ------------------------------------------------------------------ */
/* Modified version, for use from within ICU.
* Renamed public functions, to avoid an unwanted export of the
* standard names from the ICU library.
*
* Use ICU's uprv_malloc() and uprv_free()
*
* Revert comment syntax to plain C
*
* Remove a few compiler warnings.
*/
#if !defined(DECNUMBER)
#define DECNUMBER
#define DECNAME "decNumber" /* Short name */
#define DECFULLNAME "Decimal Number Module" /* Verbose name */
#define DECAUTHOR "Mike Cowlishaw" /* Who to blame */
#if !defined(DECCONTEXT)
#include "decContext.h"
#endif
/* Bit settings for decNumber.bits */
#define DECNEG 0x80 /* Sign; 1=negative, 0=positive or zero */
#define DECINF 0x40 /* 1=Infinity */
#define DECNAN 0x20 /* 1=NaN */
#define DECSNAN 0x10 /* 1=sNaN */
/* The remaining bits are reserved; they must be 0 */
#define DECSPECIAL (DECINF|DECNAN|DECSNAN) /* any special value */
/* Define the decNumber data structure. The size and shape of the */
/* units array in the structure is determined by the following */
/* constant. This must not be changed without recompiling the */
/* decNumber library modules. */
/* For ICU, use one digit per byte, to make it easier to emulate the
* old DigitList interface on top of a decNumber
*/
#define DECDPUN 1 /* DECimal Digits Per UNit [must be >0 */
/* and <10; 3 or powers of 2 are best]. */
/* DECNUMDIGITS is the default number of digits that can be held in */
/* the structure. If undefined, 1 is assumed and it is assumed */
/* that the structure will be immediately followed by extra space, */
/* as required. DECNUMDIGITS is always >0. */
#if !defined(DECNUMDIGITS)
#define DECNUMDIGITS 1
#endif
/* The size (integer data type) of each unit is determined by the */
/* number of digits it will hold. */
#if DECDPUN<=2
#define decNumberUnit uint8_t
#elif DECDPUN<=4
#define decNumberUnit uint16_t
#else
#define decNumberUnit uint32_t
#endif
/* The number of units needed is ceil(DECNUMDIGITS/DECDPUN) */
#define DECNUMUNITS ((DECNUMDIGITS+DECDPUN-1)/DECDPUN)
/* The data structure... */
typedef struct {
int32_t digits; /* Count of digits in the coefficient; >0 */
int32_t exponent; /* Unadjusted exponent, unbiased, in */
/* range: -1999999997 through 999999999 */
uint8_t bits; /* Indicator bits (see above) */
/* Coefficient, from least significant unit */
decNumberUnit lsu[DECNUMUNITS];
} decNumber;
/* Notes: */
/* 1. If digits is > DECDPUN then there will one or more */
/* decNumberUnits immediately following the first element of lsu.*/
/* These contain the remaining (more significant) digits of the */
/* number, and may be in the lsu array, or may be guaranteed by */
/* some other mechanism (such as being contained in another */
/* structure, or being overlaid on dynamically allocated */
/* storage). */
/* */
/* Each integer of the coefficient (except potentially the last) */
/* contains DECDPUN digits (e.g., a value in the range 0 through */
/* 99999999 if DECDPUN is 8, or 0 through 999 if DECDPUN is 3). */
/* */
/* 2. A decNumber converted to a string may need up to digits+14 */
/* characters. The worst cases (non-exponential and exponential */
/* formats) are -0.00000{9...}# and -9.{9...}E+999999999# */
/* (where # is '\0') */
/* ---------------------------------------------------------------- */
/* decNumber public functions and macros */
/* ---------------------------------------------------------------- */
/* Conversions */
U_CAPI decNumber * U_EXPORT2 uprv_decNumberFromInt32(decNumber *, int32_t);
U_CAPI decNumber * U_EXPORT2 uprv_decNumberFromUInt32(decNumber *, uint32_t);
U_CAPI decNumber * U_EXPORT2 uprv_decNumberFromString(decNumber *, const char *, decContext *);
U_CAPI char * U_EXPORT2 uprv_decNumberToString(const decNumber *, char *);
U_CAPI char * U_EXPORT2 uprv_decNumberToEngString(const decNumber *, char *);
U_CAPI uint32_t U_EXPORT2 uprv_decNumberToUInt32(const decNumber *, decContext *);
U_CAPI int32_t U_EXPORT2 uprv_decNumberToInt32(const decNumber *, decContext *);
U_CAPI uint8_t * U_EXPORT2 uprv_decNumberGetBCD(const decNumber *, uint8_t *);
U_CAPI decNumber * U_EXPORT2 uprv_decNumberSetBCD(decNumber *, const uint8_t *, uint32_t);
/* Operators and elementary functions */
U_CAPI decNumber * U_EXPORT2 uprv_decNumberAbs(decNumber *, const decNumber *, decContext *);
U_CAPI decNumber * U_EXPORT2 uprv_decNumberAdd(decNumber *, const decNumber *, const decNumber *, decContext *);
U_CAPI decNumber * U_EXPORT2 uprv_decNumberAnd(decNumber *, const decNumber *, const decNumber *, decContext *);
U_CAPI decNumber * U_EXPORT2 uprv_decNumberCompare(decNumber *, const decNumber *, const decNumber *, decContext *);
U_CAPI decNumber * U_EXPORT2 uprv_decNumberCompareSignal(decNumber *, const decNumber *, const decNumber *, decContext *);
U_CAPI decNumber * U_EXPORT2 uprv_decNumberCompareTotal(decNumber *, const decNumber *, const decNumber *, decContext *);
U_CAPI decNumber * U_EXPORT2 uprv_decNumberCompareTotalMag(decNumber *, const decNumber *, const decNumber *, decContext *);
U_CAPI decNumber * U_EXPORT2 uprv_decNumberDivide(decNumber *, const decNumber *, const decNumber *, decContext *);
U_CAPI decNumber * U_EXPORT2 uprv_decNumberDivideInteger(decNumber *, const decNumber *, const decNumber *, decContext *);
U_CAPI decNumber * U_EXPORT2 uprv_decNumberExp(decNumber *, const decNumber *, decContext *);
U_CAPI decNumber * U_EXPORT2 uprv_decNumberFMA(decNumber *, const decNumber *, const decNumber *, const decNumber *, decContext *);
U_CAPI decNumber * U_EXPORT2 uprv_decNumberInvert(decNumber *, const decNumber *, decContext *);
U_CAPI decNumber * U_EXPORT2 uprv_decNumberLn(decNumber *, const decNumber *, decContext *);
U_CAPI decNumber * U_EXPORT2 uprv_decNumberLogB(decNumber *, const decNumber *, decContext *);
U_CAPI decNumber * U_EXPORT2 uprv_decNumberLog10(decNumber *, const decNumber *, decContext *);
U_CAPI decNumber * U_EXPORT2 uprv_decNumberMax(decNumber *, const decNumber *, const decNumber *, decContext *);
U_CAPI decNumber * U_EXPORT2 uprv_decNumberMaxMag(decNumber *, const decNumber *, const decNumber *, decContext *);
U_CAPI decNumber * U_EXPORT2 uprv_decNumberMin(decNumber *, const decNumber *, const decNumber *, decContext *);
U_CAPI decNumber * U_EXPORT2 uprv_decNumberMinMag(decNumber *, const decNumber *, const decNumber *, decContext *);
U_CAPI decNumber * U_EXPORT2 uprv_decNumberMinus(decNumber *, const decNumber *, decContext *);
U_CAPI decNumber * U_EXPORT2 uprv_decNumberMultiply(decNumber *, const decNumber *, const decNumber *, decContext *);
U_CAPI decNumber * U_EXPORT2 uprv_decNumberNormalize(decNumber *, const decNumber *, decContext *);
U_CAPI decNumber * U_EXPORT2 uprv_decNumberOr(decNumber *, const decNumber *, const decNumber *, decContext *);
U_CAPI decNumber * U_EXPORT2 uprv_decNumberPlus(decNumber *, const decNumber *, decContext *);
U_CAPI decNumber * U_EXPORT2 uprv_decNumberPower(decNumber *, const decNumber *, const decNumber *, decContext *);
U_CAPI decNumber * U_EXPORT2 uprv_decNumberQuantize(decNumber *, const decNumber *, const decNumber *, decContext *);
U_CAPI decNumber * U_EXPORT2 uprv_decNumberReduce(decNumber *, const decNumber *, decContext *);
U_CAPI decNumber * U_EXPORT2 uprv_decNumberRemainder(decNumber *, const decNumber *, const decNumber *, decContext *);
U_CAPI decNumber * U_EXPORT2 uprv_decNumberRemainderNear(decNumber *, const decNumber *, const decNumber *, decContext *);
U_CAPI decNumber * U_EXPORT2 uprv_decNumberRescale(decNumber *, const decNumber *, const decNumber *, decContext *);
U_CAPI decNumber * U_EXPORT2 uprv_decNumberRotate(decNumber *, const decNumber *, const decNumber *, decContext *);
U_CAPI decNumber * U_EXPORT2 uprv_decNumberSameQuantum(decNumber *, const decNumber *, const decNumber *);
U_CAPI decNumber * U_EXPORT2 uprv_decNumberScaleB(decNumber *, const decNumber *, const decNumber *, decContext *);
U_CAPI decNumber * U_EXPORT2 uprv_decNumberShift(decNumber *, const decNumber *, const decNumber *, decContext *);
U_CAPI decNumber * U_EXPORT2 uprv_decNumberSquareRoot(decNumber *, const decNumber *, decContext *);
U_CAPI decNumber * U_EXPORT2 uprv_decNumberSubtract(decNumber *, const decNumber *, const decNumber *, decContext *);
U_CAPI decNumber * U_EXPORT2 uprv_decNumberToIntegralExact(decNumber *, const decNumber *, decContext *);
U_CAPI decNumber * U_EXPORT2 uprv_decNumberToIntegralValue(decNumber *, const decNumber *, decContext *);
U_CAPI decNumber * U_EXPORT2 uprv_decNumberXor(decNumber *, const decNumber *, const decNumber *, decContext *);
/* Utilities */
enum decClass uprv_decNumberClass(const decNumber *, decContext *);
U_CAPI const char * U_EXPORT2 uprv_decNumberClassToString(enum decClass);
U_CAPI decNumber * U_EXPORT2 uprv_decNumberCopy(decNumber *, const decNumber *);
U_CAPI decNumber * U_EXPORT2 uprv_decNumberCopyAbs(decNumber *, const decNumber *);
U_CAPI decNumber * U_EXPORT2 uprv_decNumberCopyNegate(decNumber *, const decNumber *);
U_CAPI decNumber * U_EXPORT2 uprv_decNumberCopySign(decNumber *, const decNumber *, const decNumber *);
U_CAPI decNumber * U_EXPORT2 uprv_decNumberNextMinus(decNumber *, const decNumber *, decContext *);
U_CAPI decNumber * U_EXPORT2 uprv_decNumberNextPlus(decNumber *, const decNumber *, decContext *);
U_CAPI decNumber * U_EXPORT2 uprv_decNumberNextToward(decNumber *, const decNumber *, const decNumber *, decContext *);
U_CAPI decNumber * U_EXPORT2 uprv_decNumberTrim(decNumber *);
U_CAPI const char * U_EXPORT2 uprv_decNumberVersion();
U_CAPI decNumber * U_EXPORT2 uprv_decNumberZero(decNumber *);
/* Functions for testing decNumbers (normality depends on context) */
U_CAPI int32_t U_EXPORT2 uprv_decNumberIsNormal(const decNumber *, decContext *);
U_CAPI int32_t U_EXPORT2 uprv_decNumberIsSubnormal(const decNumber *, decContext *);
/* Macros for testing decNumber *dn */
#define decNumberIsCanonical(dn) (1) /* All decNumbers are saintly */
#define decNumberIsFinite(dn) (((dn)->bits&DECSPECIAL)==0)
#define decNumberIsInfinite(dn) (((dn)->bits&DECINF)!=0)
#define decNumberIsNaN(dn) (((dn)->bits&(DECNAN|DECSNAN))!=0)
#define decNumberIsNegative(dn) (((dn)->bits&DECNEG)!=0)
#define decNumberIsQNaN(dn) (((dn)->bits&(DECNAN))!=0)
#define decNumberIsSNaN(dn) (((dn)->bits&(DECSNAN))!=0)
#define decNumberIsSpecial(dn) (((dn)->bits&DECSPECIAL)!=0)
#define decNumberIsZero(dn) (*(dn)->lsu==0 \
&& (dn)->digits==1 \
&& (((dn)->bits&DECSPECIAL)==0))
#define decNumberRadix(dn) (10)
#endif

728
deps/icu-small/source/i18n/decNumberLocal.h vendored Normal file
View File

@ -0,0 +1,728 @@
// © 2016 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html
/* ------------------------------------------------------------------ */
/* decNumber package local type, tuning, and macro definitions */
/* ------------------------------------------------------------------ */
/* Copyright (c) IBM Corporation, 2000-2016. All rights reserved. */
/* */
/* This software is made available under the terms of the */
/* ICU License -- ICU 1.8.1 and later. */
/* */
/* The description and User's Guide ("The decNumber C Library") for */
/* this software is called decNumber.pdf. This document is */
/* available, together with arithmetic and format specifications, */
/* testcases, and Web links, on the General Decimal Arithmetic page. */
/* */
/* Please send comments, suggestions, and corrections to the author: */
/* mfc@uk.ibm.com */
/* Mike Cowlishaw, IBM Fellow */
/* IBM UK, PO Box 31, Birmingham Road, Warwick CV34 5JL, UK */
/* ------------------------------------------------------------------ */
/* This header file is included by all modules in the decNumber */
/* library, and contains local type definitions, tuning parameters, */
/* etc. It should not need to be used by application programs. */
/* decNumber.h or one of decDouble (etc.) must be included first. */
/* ------------------------------------------------------------------ */
#if !defined(DECNUMBERLOC)
#define DECNUMBERLOC
#define DECVERSION "decNumber 3.61" /* Package Version [16 max.] */
#define DECNLAUTHOR "Mike Cowlishaw" /* Who to blame */
#include <stdlib.h> /* for abs */
#include <string.h> /* for memset, strcpy */
#include "decContext.h"
/* Conditional code flag -- set this to match hardware platform */
#if !defined(DECLITEND)
#define DECLITEND 1 /* 1=little-endian, 0=big-endian */
#endif
/* Conditional code flag -- set this to 1 for best performance */
#if !defined(DECUSE64)
#define DECUSE64 1 /* 1=use int64s, 0=int32 & smaller only */
#endif
/* Conditional check flags -- set these to 0 for best performance */
#if !defined(DECCHECK)
#define DECCHECK 0 /* 1 to enable robust checking */
#endif
#if !defined(DECALLOC)
#define DECALLOC 0 /* 1 to enable memory accounting */
#endif
#if !defined(DECTRACE)
#define DECTRACE 0 /* 1 to trace certain internals, etc. */
#endif
/* Tuning parameter for decNumber (arbitrary precision) module */
#if !defined(DECBUFFER)
#define DECBUFFER 36 /* Size basis for local buffers. This */
/* should be a common maximum precision */
/* rounded up to a multiple of 4; must */
/* be zero or positive. */
#endif
/* ---------------------------------------------------------------- */
/* Definitions for all modules (general-purpose) */
/* ---------------------------------------------------------------- */
/* Local names for common types -- for safety, decNumber modules do */
/* not use int or long directly. */
#define Flag uint8_t
#define Byte int8_t
#define uByte uint8_t
#define Short int16_t
#define uShort uint16_t
#define Int int32_t
#define uInt uint32_t
#define Unit decNumberUnit
#if DECUSE64
#define Long int64_t
#define uLong uint64_t
#endif
/* Development-use definitions */
typedef long int LI; /* for printf arguments only */
#define DECNOINT 0 /* 1 to check no internal use of 'int' */
/* or stdint types */
#if DECNOINT
/* if these interfere with your C includes, do not set DECNOINT */
#define int ? /* enable to ensure that plain C 'int' */
#define long ?? /* .. or 'long' types are not used */
#endif
/* LONGMUL32HI -- set w=(u*v)>>32, where w, u, and v are uInts */
/* (that is, sets w to be the high-order word of the 64-bit result; */
/* the low-order word is simply u*v.) */
/* This version is derived from Knuth via Hacker's Delight; */
/* it seems to optimize better than some others tried */
#define LONGMUL32HI(w, u, v) { \
uInt u0, u1, v0, v1, w0, w1, w2, t; \
u0=u & 0xffff; u1=u>>16; \
v0=v & 0xffff; v1=v>>16; \
w0=u0*v0; \
t=u1*v0 + (w0>>16); \
w1=t & 0xffff; w2=t>>16; \
w1=u0*v1 + w1; \
(w)=u1*v1 + w2 + (w1>>16);}
/* ROUNDUP -- round an integer up to a multiple of n */
#define ROUNDUP(i, n) ((((i)+(n)-1)/n)*n)
#define ROUNDUP4(i) (((i)+3)&~3) /* special for n=4 */
/* ROUNDDOWN -- round an integer down to a multiple of n */
#define ROUNDDOWN(i, n) (((i)/n)*n)
#define ROUNDDOWN4(i) ((i)&~3) /* special for n=4 */
/* References to multi-byte sequences under different sizes; these */
/* require locally declared variables, but do not violate strict */
/* aliasing or alignment (as did the UINTAT simple cast to uInt). */
/* Variables needed are uswork, uiwork, etc. [so do not use at same */
/* level in an expression, e.g., UBTOUI(x)==UBTOUI(y) may fail]. */
/* Return a uInt, etc., from bytes starting at a char* or uByte* */
#define UBTOUS(b) (memcpy((void *)&uswork, b, 2), uswork)
#define UBTOUI(b) (memcpy((void *)&uiwork, b, 4), uiwork)
/* Store a uInt, etc., into bytes starting at a char* or uByte*. */
/* Returns i, evaluated, for convenience; has to use uiwork because */
/* i may be an expression. */
#define UBFROMUS(b, i) (uswork=(i), memcpy(b, (void *)&uswork, 2), uswork)
#define UBFROMUI(b, i) (uiwork=(i), memcpy(b, (void *)&uiwork, 4), uiwork)
/* X10 and X100 -- multiply integer i by 10 or 100 */
/* [shifts are usually faster than multiply; could be conditional] */
#define X10(i) (((i)<<1)+((i)<<3))
#define X100(i) (((i)<<2)+((i)<<5)+((i)<<6))
/* MAXI and MINI -- general max & min (not in ANSI) for integers */
#define MAXI(x,y) ((x)<(y)?(y):(x))
#define MINI(x,y) ((x)>(y)?(y):(x))
/* Useful constants */
#define BILLION 1000000000 /* 10**9 */
/* CHARMASK: 0x30303030 for ASCII/UTF8; 0xF0F0F0F0 for EBCDIC */
#define CHARMASK ((((((((uInt)'0')<<8)+'0')<<8)+'0')<<8)+'0')
/* ---------------------------------------------------------------- */
/* Definitions for arbitrary-precision modules (only valid after */
/* decNumber.h has been included) */
/* ---------------------------------------------------------------- */
/* Limits and constants */
#define DECNUMMAXP 999999999 /* maximum precision code can handle */
#define DECNUMMAXE 999999999 /* maximum adjusted exponent ditto */
#define DECNUMMINE -999999999 /* minimum adjusted exponent ditto */
#if (DECNUMMAXP != DEC_MAX_DIGITS)
#error Maximum digits mismatch
#endif
#if (DECNUMMAXE != DEC_MAX_EMAX)
#error Maximum exponent mismatch
#endif
#if (DECNUMMINE != DEC_MIN_EMIN)
#error Minimum exponent mismatch
#endif
/* Set DECDPUNMAX -- the maximum integer that fits in DECDPUN */
/* digits, and D2UTABLE -- the initializer for the D2U table */
#ifndef DECDPUN
// no-op
#elif DECDPUN==1
#define DECDPUNMAX 9
#define D2UTABLE {0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17, \
18,19,20,21,22,23,24,25,26,27,28,29,30,31,32, \
33,34,35,36,37,38,39,40,41,42,43,44,45,46,47, \
48,49}
#elif DECDPUN==2
#define DECDPUNMAX 99
#define D2UTABLE {0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,8,8,9,9,10,10, \
11,11,12,12,13,13,14,14,15,15,16,16,17,17,18, \
18,19,19,20,20,21,21,22,22,23,23,24,24,25}
#elif DECDPUN==3
#define DECDPUNMAX 999
#define D2UTABLE {0,1,1,1,2,2,2,3,3,3,4,4,4,5,5,5,6,6,6,7,7,7, \
8,8,8,9,9,9,10,10,10,11,11,11,12,12,12,13,13, \
13,14,14,14,15,15,15,16,16,16,17}
#elif DECDPUN==4
#define DECDPUNMAX 9999
#define D2UTABLE {0,1,1,1,1,2,2,2,2,3,3,3,3,4,4,4,4,5,5,5,5,6, \
6,6,6,7,7,7,7,8,8,8,8,9,9,9,9,10,10,10,10,11, \
11,11,11,12,12,12,12,13}
#elif DECDPUN==5
#define DECDPUNMAX 99999
#define D2UTABLE {0,1,1,1,1,1,2,2,2,2,2,3,3,3,3,3,4,4,4,4,4,5, \
5,5,5,5,6,6,6,6,6,7,7,7,7,7,8,8,8,8,8,9,9,9, \
9,9,10,10,10,10}
#elif DECDPUN==6
#define DECDPUNMAX 999999
#define D2UTABLE {0,1,1,1,1,1,1,2,2,2,2,2,2,3,3,3,3,3,3,4,4,4, \
4,4,4,5,5,5,5,5,5,6,6,6,6,6,6,7,7,7,7,7,7,8, \
8,8,8,8,8,9}
#elif DECDPUN==7
#define DECDPUNMAX 9999999
#define D2UTABLE {0,1,1,1,1,1,1,1,2,2,2,2,2,2,2,3,3,3,3,3,3,3, \
4,4,4,4,4,4,4,5,5,5,5,5,5,5,6,6,6,6,6,6,6,7, \
7,7,7,7,7,7}
#elif DECDPUN==8
#define DECDPUNMAX 99999999
#define D2UTABLE {0,1,1,1,1,1,1,1,1,2,2,2,2,2,2,2,2,3,3,3,3,3, \
3,3,3,4,4,4,4,4,4,4,4,5,5,5,5,5,5,5,5,6,6,6, \
6,6,6,6,6,7}
#elif DECDPUN==9
#define DECDPUNMAX 999999999
#define D2UTABLE {0,1,1,1,1,1,1,1,1,1,2,2,2,2,2,2,2,2,2,3,3,3, \
3,3,3,3,3,3,4,4,4,4,4,4,4,4,4,5,5,5,5,5,5,5, \
5,5,6,6,6,6}
#else
#error DECDPUN must be in the range 1-9
#endif
/* ----- Shared data (in decNumber.c) ----- */
/* Public lookup table used by the D2U macro (see below) */
#define DECMAXD2U 49
/*extern const uByte d2utable[DECMAXD2U+1];*/
/* ----- Macros ----- */
/* ISZERO -- return true if decNumber dn is a zero */
/* [performance-critical in some situations] */
#define ISZERO(dn) decNumberIsZero(dn) /* now just a local name */
/* D2U -- return the number of Units needed to hold d digits */
/* (runtime version, with table lookaside for small d) */
#if defined(DECDPUN) && DECDPUN==8
#define D2U(d) ((unsigned)((d)<=DECMAXD2U?d2utable[d]:((d)+7)>>3))
#elif defined(DECDPUN) && DECDPUN==4
#define D2U(d) ((unsigned)((d)<=DECMAXD2U?d2utable[d]:((d)+3)>>2))
#else
#define D2U(d) ((d)<=DECMAXD2U?d2utable[d]:((d)+DECDPUN-1)/DECDPUN)
#endif
/* SD2U -- static D2U macro (for compile-time calculation) */
#define SD2U(d) (((d)+DECDPUN-1)/DECDPUN)
/* MSUDIGITS -- returns digits in msu, from digits, calculated */
/* using D2U */
#define MSUDIGITS(d) ((d)-(D2U(d)-1)*DECDPUN)
/* D2N -- return the number of decNumber structs that would be */
/* needed to contain that number of digits (and the initial */
/* decNumber struct) safely. Note that one Unit is included in the */
/* initial structure. Used for allocating space that is aligned on */
/* a decNumber struct boundary. */
#define D2N(d) \
((((SD2U(d)-1)*sizeof(Unit))+sizeof(decNumber)*2-1)/sizeof(decNumber))
/* TODIGIT -- macro to remove the leading digit from the unsigned */
/* integer u at column cut (counting from the right, LSD=0) and */
/* place it as an ASCII character into the character pointed to by */
/* c. Note that cut must be <= 9, and the maximum value for u is */
/* 2,000,000,000 (as is needed for negative exponents of */
/* subnormals). The unsigned integer pow is used as a temporary */
/* variable. */
#define TODIGIT(u, cut, c, pow) UPRV_BLOCK_MACRO_BEGIN { \
*(c)='0'; \
pow=DECPOWERS[cut]*2; \
if ((u)>pow) { \
pow*=4; \
if ((u)>=pow) {(u)-=pow; *(c)+=8;} \
pow/=2; \
if ((u)>=pow) {(u)-=pow; *(c)+=4;} \
pow/=2; \
} \
if ((u)>=pow) {(u)-=pow; *(c)+=2;} \
pow/=2; \
if ((u)>=pow) {(u)-=pow; *(c)+=1;} \
} UPRV_BLOCK_MACRO_END
/* ---------------------------------------------------------------- */
/* Definitions for fixed-precision modules (only valid after */
/* decSingle.h, decDouble.h, or decQuad.h has been included) */
/* ---------------------------------------------------------------- */
/* bcdnum -- a structure describing a format-independent finite */
/* number, whose coefficient is a string of bcd8 uBytes */
typedef struct {
uByte *msd; /* -> most significant digit */
uByte *lsd; /* -> least ditto */
uInt sign; /* 0=positive, DECFLOAT_Sign=negative */
Int exponent; /* Unadjusted signed exponent (q), or */
/* DECFLOAT_NaN etc. for a special */
} bcdnum;
/* Test if exponent or bcdnum exponent must be a special, etc. */
#define EXPISSPECIAL(exp) ((exp)>=DECFLOAT_MinSp)
#define EXPISINF(exp) (exp==DECFLOAT_Inf)
#define EXPISNAN(exp) (exp==DECFLOAT_qNaN || exp==DECFLOAT_sNaN)
#define NUMISSPECIAL(num) (EXPISSPECIAL((num)->exponent))
/* Refer to a 32-bit word or byte in a decFloat (df) by big-endian */
/* (array) notation (the 0 word or byte contains the sign bit), */
/* automatically adjusting for endianness; similarly address a word */
/* in the next-wider format (decFloatWider, or dfw) */
#define DECWORDS (DECBYTES/4)
#define DECWWORDS (DECWBYTES/4)
#if DECLITEND
#define DFBYTE(df, off) ((df)->bytes[DECBYTES-1-(off)])
#define DFWORD(df, off) ((df)->words[DECWORDS-1-(off)])
#define DFWWORD(dfw, off) ((dfw)->words[DECWWORDS-1-(off)])
#else
#define DFBYTE(df, off) ((df)->bytes[off])
#define DFWORD(df, off) ((df)->words[off])
#define DFWWORD(dfw, off) ((dfw)->words[off])
#endif
/* Tests for sign or specials, directly on DECFLOATs */
#define DFISSIGNED(df) (DFWORD(df, 0)&0x80000000)
#define DFISSPECIAL(df) ((DFWORD(df, 0)&0x78000000)==0x78000000)
#define DFISINF(df) ((DFWORD(df, 0)&0x7c000000)==0x78000000)
#define DFISNAN(df) ((DFWORD(df, 0)&0x7c000000)==0x7c000000)
#define DFISQNAN(df) ((DFWORD(df, 0)&0x7e000000)==0x7c000000)
#define DFISSNAN(df) ((DFWORD(df, 0)&0x7e000000)==0x7e000000)
/* Shared lookup tables */
extern const uInt DECCOMBMSD[64]; /* Combination field -> MSD */
extern const uInt DECCOMBFROM[48]; /* exp+msd -> Combination */
/* Private generic (utility) routine */
#if DECCHECK || DECTRACE
extern void decShowNum(const bcdnum *, const char *);
#endif
/* Format-dependent macros and constants */
#if defined(DECPMAX)
/* Useful constants */
#define DECPMAX9 (ROUNDUP(DECPMAX, 9)/9) /* 'Pmax' in 10**9s */
/* Top words for a zero */
#define SINGLEZERO 0x22500000
#define DOUBLEZERO 0x22380000
#define QUADZERO 0x22080000
/* [ZEROWORD is defined to be one of these in the DFISZERO macro] */
/* Format-dependent common tests: */
/* DFISZERO -- test for (any) zero */
/* DFISCCZERO -- test for coefficient continuation being zero */
/* DFISCC01 -- test for coefficient contains only 0s and 1s */
/* DFISINT -- test for finite and exponent q=0 */
/* DFISUINT01 -- test for sign=0, finite, exponent q=0, and */
/* MSD=0 or 1 */
/* ZEROWORD is also defined here. */
/* In DFISZERO the first test checks the least-significant word */
/* (most likely to be non-zero); the penultimate tests MSD and */
/* DPDs in the signword, and the final test excludes specials and */
/* MSD>7. DFISINT similarly has to allow for the two forms of */
/* MSD codes. DFISUINT01 only has to allow for one form of MSD */
/* code. */
#if DECPMAX==7
#define ZEROWORD SINGLEZERO
/* [test macros not needed except for Zero] */
#define DFISZERO(df) ((DFWORD(df, 0)&0x1c0fffff)==0 \
&& (DFWORD(df, 0)&0x60000000)!=0x60000000)
#elif DECPMAX==16
#define ZEROWORD DOUBLEZERO
#define DFISZERO(df) ((DFWORD(df, 1)==0 \
&& (DFWORD(df, 0)&0x1c03ffff)==0 \
&& (DFWORD(df, 0)&0x60000000)!=0x60000000))
#define DFISINT(df) ((DFWORD(df, 0)&0x63fc0000)==0x22380000 \
||(DFWORD(df, 0)&0x7bfc0000)==0x6a380000)
#define DFISUINT01(df) ((DFWORD(df, 0)&0xfbfc0000)==0x22380000)
#define DFISCCZERO(df) (DFWORD(df, 1)==0 \
&& (DFWORD(df, 0)&0x0003ffff)==0)
#define DFISCC01(df) ((DFWORD(df, 0)&~0xfffc9124)==0 \
&& (DFWORD(df, 1)&~0x49124491)==0)
#elif DECPMAX==34
#define ZEROWORD QUADZERO
#define DFISZERO(df) ((DFWORD(df, 3)==0 \
&& DFWORD(df, 2)==0 \
&& DFWORD(df, 1)==0 \
&& (DFWORD(df, 0)&0x1c003fff)==0 \
&& (DFWORD(df, 0)&0x60000000)!=0x60000000))
#define DFISINT(df) ((DFWORD(df, 0)&0x63ffc000)==0x22080000 \
||(DFWORD(df, 0)&0x7bffc000)==0x6a080000)
#define DFISUINT01(df) ((DFWORD(df, 0)&0xfbffc000)==0x22080000)
#define DFISCCZERO(df) (DFWORD(df, 3)==0 \
&& DFWORD(df, 2)==0 \
&& DFWORD(df, 1)==0 \
&& (DFWORD(df, 0)&0x00003fff)==0)
#define DFISCC01(df) ((DFWORD(df, 0)&~0xffffc912)==0 \
&& (DFWORD(df, 1)&~0x44912449)==0 \
&& (DFWORD(df, 2)&~0x12449124)==0 \
&& (DFWORD(df, 3)&~0x49124491)==0)
#endif
/* Macros to test if a certain 10 bits of a uInt or pair of uInts */
/* are a canonical declet [higher or lower bits are ignored]. */
/* declet is at offset 0 (from the right) in a uInt: */
#define CANONDPD(dpd) (((dpd)&0x300)==0 || ((dpd)&0x6e)!=0x6e)
/* declet is at offset k (a multiple of 2) in a uInt: */
#define CANONDPDOFF(dpd, k) (((dpd)&(0x300<<(k)))==0 \
|| ((dpd)&(((uInt)0x6e)<<(k)))!=(((uInt)0x6e)<<(k)))
/* declet is at offset k (a multiple of 2) in a pair of uInts: */
/* [the top 2 bits will always be in the more-significant uInt] */
#define CANONDPDTWO(hi, lo, k) (((hi)&(0x300>>(32-(k))))==0 \
|| ((hi)&(0x6e>>(32-(k))))!=(0x6e>>(32-(k))) \
|| ((lo)&(((uInt)0x6e)<<(k)))!=(((uInt)0x6e)<<(k)))
/* Macro to test whether a full-length (length DECPMAX) BCD8 */
/* coefficient, starting at uByte u, is all zeros */
/* Test just the LSWord first, then the remainder as a sequence */
/* of tests in order to avoid same-level use of UBTOUI */
#if DECPMAX==7
#define ISCOEFFZERO(u) ( \
UBTOUI((u)+DECPMAX-4)==0 \
&& UBTOUS((u)+DECPMAX-6)==0 \
&& *(u)==0)
#elif DECPMAX==16
#define ISCOEFFZERO(u) ( \
UBTOUI((u)+DECPMAX-4)==0 \
&& UBTOUI((u)+DECPMAX-8)==0 \
&& UBTOUI((u)+DECPMAX-12)==0 \
&& UBTOUI(u)==0)
#elif DECPMAX==34
#define ISCOEFFZERO(u) ( \
UBTOUI((u)+DECPMAX-4)==0 \
&& UBTOUI((u)+DECPMAX-8)==0 \
&& UBTOUI((u)+DECPMAX-12)==0 \
&& UBTOUI((u)+DECPMAX-16)==0 \
&& UBTOUI((u)+DECPMAX-20)==0 \
&& UBTOUI((u)+DECPMAX-24)==0 \
&& UBTOUI((u)+DECPMAX-28)==0 \
&& UBTOUI((u)+DECPMAX-32)==0 \
&& UBTOUS(u)==0)
#endif
/* Macros and masks for the exponent continuation field and MSD */
/* Get the exponent continuation from a decFloat *df as an Int */
#define GETECON(df) ((Int)((DFWORD((df), 0)&0x03ffffff)>>(32-6-DECECONL)))
/* Ditto, from the next-wider format */
#define GETWECON(df) ((Int)((DFWWORD((df), 0)&0x03ffffff)>>(32-6-DECWECONL)))
/* Get the biased exponent similarly */
#define GETEXP(df) ((Int)(DECCOMBEXP[DFWORD((df), 0)>>26]+GETECON(df)))
/* Get the unbiased exponent similarly */
#define GETEXPUN(df) ((Int)GETEXP(df)-DECBIAS)
/* Get the MSD similarly (as uInt) */
#define GETMSD(df) (DECCOMBMSD[DFWORD((df), 0)>>26])
/* Compile-time computes of the exponent continuation field masks */
/* full exponent continuation field: */
#define ECONMASK ((0x03ffffff>>(32-6-DECECONL))<<(32-6-DECECONL))
/* same, not including its first digit (the qNaN/sNaN selector): */
#define ECONNANMASK ((0x01ffffff>>(32-6-DECECONL))<<(32-6-DECECONL))
/* Macros to decode the coefficient in a finite decFloat *df into */
/* a BCD string (uByte *bcdin) of length DECPMAX uBytes. */
/* In-line sequence to convert least significant 10 bits of uInt */
/* dpd to three BCD8 digits starting at uByte u. Note that an */
/* extra byte is written to the right of the three digits because */
/* four bytes are moved at a time for speed; the alternative */
/* macro moves exactly three bytes (usually slower). */
#define dpd2bcd8(u, dpd) memcpy(u, &DPD2BCD8[((dpd)&0x3ff)*4], 4)
#define dpd2bcd83(u, dpd) memcpy(u, &DPD2BCD8[((dpd)&0x3ff)*4], 3)
/* Decode the declets. After extracting each one, it is decoded */
/* to BCD8 using a table lookup (also used for variable-length */
/* decode). Each DPD decode is 3 bytes BCD8 plus a one-byte */
/* length which is not used, here). Fixed-length 4-byte moves */
/* are fast, however, almost everywhere, and so are used except */
/* for the final three bytes (to avoid overrun). The code below */
/* is 36 instructions for Doubles and about 70 for Quads, even */
/* on IA32. */
/* Two macros are defined for each format: */
/* GETCOEFF extracts the coefficient of the current format */
/* GETWCOEFF extracts the coefficient of the next-wider format. */
/* The latter is a copy of the next-wider GETCOEFF using DFWWORD. */
#if DECPMAX==7
#define GETCOEFF(df, bcd) { \
uInt sourhi=DFWORD(df, 0); \
*(bcd)=(uByte)DECCOMBMSD[sourhi>>26]; \
dpd2bcd8(bcd+1, sourhi>>10); \
dpd2bcd83(bcd+4, sourhi);}
#define GETWCOEFF(df, bcd) { \
uInt sourhi=DFWWORD(df, 0); \
uInt sourlo=DFWWORD(df, 1); \
*(bcd)=(uByte)DECCOMBMSD[sourhi>>26]; \
dpd2bcd8(bcd+1, sourhi>>8); \
dpd2bcd8(bcd+4, (sourhi<<2) | (sourlo>>30)); \
dpd2bcd8(bcd+7, sourlo>>20); \
dpd2bcd8(bcd+10, sourlo>>10); \
dpd2bcd83(bcd+13, sourlo);}
#elif DECPMAX==16
#define GETCOEFF(df, bcd) { \
uInt sourhi=DFWORD(df, 0); \
uInt sourlo=DFWORD(df, 1); \
*(bcd)=(uByte)DECCOMBMSD[sourhi>>26]; \
dpd2bcd8(bcd+1, sourhi>>8); \
dpd2bcd8(bcd+4, (sourhi<<2) | (sourlo>>30)); \
dpd2bcd8(bcd+7, sourlo>>20); \
dpd2bcd8(bcd+10, sourlo>>10); \
dpd2bcd83(bcd+13, sourlo);}
#define GETWCOEFF(df, bcd) { \
uInt sourhi=DFWWORD(df, 0); \
uInt sourmh=DFWWORD(df, 1); \
uInt sourml=DFWWORD(df, 2); \
uInt sourlo=DFWWORD(df, 3); \
*(bcd)=(uByte)DECCOMBMSD[sourhi>>26]; \
dpd2bcd8(bcd+1, sourhi>>4); \
dpd2bcd8(bcd+4, ((sourhi)<<6) | (sourmh>>26)); \
dpd2bcd8(bcd+7, sourmh>>16); \
dpd2bcd8(bcd+10, sourmh>>6); \
dpd2bcd8(bcd+13, ((sourmh)<<4) | (sourml>>28)); \
dpd2bcd8(bcd+16, sourml>>18); \
dpd2bcd8(bcd+19, sourml>>8); \
dpd2bcd8(bcd+22, ((sourml)<<2) | (sourlo>>30)); \
dpd2bcd8(bcd+25, sourlo>>20); \
dpd2bcd8(bcd+28, sourlo>>10); \
dpd2bcd83(bcd+31, sourlo);}
#elif DECPMAX==34
#define GETCOEFF(df, bcd) { \
uInt sourhi=DFWORD(df, 0); \
uInt sourmh=DFWORD(df, 1); \
uInt sourml=DFWORD(df, 2); \
uInt sourlo=DFWORD(df, 3); \
*(bcd)=(uByte)DECCOMBMSD[sourhi>>26]; \
dpd2bcd8(bcd+1, sourhi>>4); \
dpd2bcd8(bcd+4, ((sourhi)<<6) | (sourmh>>26)); \
dpd2bcd8(bcd+7, sourmh>>16); \
dpd2bcd8(bcd+10, sourmh>>6); \
dpd2bcd8(bcd+13, ((sourmh)<<4) | (sourml>>28)); \
dpd2bcd8(bcd+16, sourml>>18); \
dpd2bcd8(bcd+19, sourml>>8); \
dpd2bcd8(bcd+22, ((sourml)<<2) | (sourlo>>30)); \
dpd2bcd8(bcd+25, sourlo>>20); \
dpd2bcd8(bcd+28, sourlo>>10); \
dpd2bcd83(bcd+31, sourlo);}
#define GETWCOEFF(df, bcd) {??} /* [should never be used] */
#endif
/* Macros to decode the coefficient in a finite decFloat *df into */
/* a base-billion uInt array, with the least-significant */
/* 0-999999999 'digit' at offset 0. */
/* Decode the declets. After extracting each one, it is decoded */
/* to binary using a table lookup. Three tables are used; one */
/* the usual DPD to binary, the other two pre-multiplied by 1000 */
/* and 1000000 to avoid multiplication during decode. These */
/* tables can also be used for multiplying up the MSD as the DPD */
/* code for 0 through 9 is the identity. */
#define DPD2BIN0 DPD2BIN /* for prettier code */
#if DECPMAX==7
#define GETCOEFFBILL(df, buf) { \
uInt sourhi=DFWORD(df, 0); \
(buf)[0]=DPD2BIN0[sourhi&0x3ff] \
+DPD2BINK[(sourhi>>10)&0x3ff] \
+DPD2BINM[DECCOMBMSD[sourhi>>26]];}
#elif DECPMAX==16
#define GETCOEFFBILL(df, buf) { \
uInt sourhi, sourlo; \
sourlo=DFWORD(df, 1); \
(buf)[0]=DPD2BIN0[sourlo&0x3ff] \
+DPD2BINK[(sourlo>>10)&0x3ff] \
+DPD2BINM[(sourlo>>20)&0x3ff]; \
sourhi=DFWORD(df, 0); \
(buf)[1]=DPD2BIN0[((sourhi<<2) | (sourlo>>30))&0x3ff] \
+DPD2BINK[(sourhi>>8)&0x3ff] \
+DPD2BINM[DECCOMBMSD[sourhi>>26]];}
#elif DECPMAX==34
#define GETCOEFFBILL(df, buf) { \
uInt sourhi, sourmh, sourml, sourlo; \
sourlo=DFWORD(df, 3); \
(buf)[0]=DPD2BIN0[sourlo&0x3ff] \
+DPD2BINK[(sourlo>>10)&0x3ff] \
+DPD2BINM[(sourlo>>20)&0x3ff]; \
sourml=DFWORD(df, 2); \
(buf)[1]=DPD2BIN0[((sourml<<2) | (sourlo>>30))&0x3ff] \
+DPD2BINK[(sourml>>8)&0x3ff] \
+DPD2BINM[(sourml>>18)&0x3ff]; \
sourmh=DFWORD(df, 1); \
(buf)[2]=DPD2BIN0[((sourmh<<4) | (sourml>>28))&0x3ff] \
+DPD2BINK[(sourmh>>6)&0x3ff] \
+DPD2BINM[(sourmh>>16)&0x3ff]; \
sourhi=DFWORD(df, 0); \
(buf)[3]=DPD2BIN0[((sourhi<<6) | (sourmh>>26))&0x3ff] \
+DPD2BINK[(sourhi>>4)&0x3ff] \
+DPD2BINM[DECCOMBMSD[sourhi>>26]];}
#endif
/* Macros to decode the coefficient in a finite decFloat *df into */
/* a base-thousand uInt array (of size DECLETS+1, to allow for */
/* the MSD), with the least-significant 0-999 'digit' at offset 0.*/
/* Decode the declets. After extracting each one, it is decoded */
/* to binary using a table lookup. */
#if DECPMAX==7
#define GETCOEFFTHOU(df, buf) { \
uInt sourhi=DFWORD(df, 0); \
(buf)[0]=DPD2BIN[sourhi&0x3ff]; \
(buf)[1]=DPD2BIN[(sourhi>>10)&0x3ff]; \
(buf)[2]=DECCOMBMSD[sourhi>>26];}
#elif DECPMAX==16
#define GETCOEFFTHOU(df, buf) { \
uInt sourhi, sourlo; \
sourlo=DFWORD(df, 1); \
(buf)[0]=DPD2BIN[sourlo&0x3ff]; \
(buf)[1]=DPD2BIN[(sourlo>>10)&0x3ff]; \
(buf)[2]=DPD2BIN[(sourlo>>20)&0x3ff]; \
sourhi=DFWORD(df, 0); \
(buf)[3]=DPD2BIN[((sourhi<<2) | (sourlo>>30))&0x3ff]; \
(buf)[4]=DPD2BIN[(sourhi>>8)&0x3ff]; \
(buf)[5]=DECCOMBMSD[sourhi>>26];}
#elif DECPMAX==34
#define GETCOEFFTHOU(df, buf) { \
uInt sourhi, sourmh, sourml, sourlo; \
sourlo=DFWORD(df, 3); \
(buf)[0]=DPD2BIN[sourlo&0x3ff]; \
(buf)[1]=DPD2BIN[(sourlo>>10)&0x3ff]; \
(buf)[2]=DPD2BIN[(sourlo>>20)&0x3ff]; \
sourml=DFWORD(df, 2); \
(buf)[3]=DPD2BIN[((sourml<<2) | (sourlo>>30))&0x3ff]; \
(buf)[4]=DPD2BIN[(sourml>>8)&0x3ff]; \
(buf)[5]=DPD2BIN[(sourml>>18)&0x3ff]; \
sourmh=DFWORD(df, 1); \
(buf)[6]=DPD2BIN[((sourmh<<4) | (sourml>>28))&0x3ff]; \
(buf)[7]=DPD2BIN[(sourmh>>6)&0x3ff]; \
(buf)[8]=DPD2BIN[(sourmh>>16)&0x3ff]; \
sourhi=DFWORD(df, 0); \
(buf)[9]=DPD2BIN[((sourhi<<6) | (sourmh>>26))&0x3ff]; \
(buf)[10]=DPD2BIN[(sourhi>>4)&0x3ff]; \
(buf)[11]=DECCOMBMSD[sourhi>>26];}
#endif
/* Macros to decode the coefficient in a finite decFloat *df and */
/* add to a base-thousand uInt array (as for GETCOEFFTHOU). */
/* After the addition then most significant 'digit' in the array */
/* might have a value larger then 10 (with a maximum of 19). */
#if DECPMAX==7
#define ADDCOEFFTHOU(df, buf) { \
uInt sourhi=DFWORD(df, 0); \
(buf)[0]+=DPD2BIN[sourhi&0x3ff]; \
if (buf[0]>999) {buf[0]-=1000; buf[1]++;} \
(buf)[1]+=DPD2BIN[(sourhi>>10)&0x3ff]; \
if (buf[1]>999) {buf[1]-=1000; buf[2]++;} \
(buf)[2]+=DECCOMBMSD[sourhi>>26];}
#elif DECPMAX==16
#define ADDCOEFFTHOU(df, buf) { \
uInt sourhi, sourlo; \
sourlo=DFWORD(df, 1); \
(buf)[0]+=DPD2BIN[sourlo&0x3ff]; \
if (buf[0]>999) {buf[0]-=1000; buf[1]++;} \
(buf)[1]+=DPD2BIN[(sourlo>>10)&0x3ff]; \
if (buf[1]>999) {buf[1]-=1000; buf[2]++;} \
(buf)[2]+=DPD2BIN[(sourlo>>20)&0x3ff]; \
if (buf[2]>999) {buf[2]-=1000; buf[3]++;} \
sourhi=DFWORD(df, 0); \
(buf)[3]+=DPD2BIN[((sourhi<<2) | (sourlo>>30))&0x3ff]; \
if (buf[3]>999) {buf[3]-=1000; buf[4]++;} \
(buf)[4]+=DPD2BIN[(sourhi>>8)&0x3ff]; \
if (buf[4]>999) {buf[4]-=1000; buf[5]++;} \
(buf)[5]+=DECCOMBMSD[sourhi>>26];}
#elif DECPMAX==34
#define ADDCOEFFTHOU(df, buf) { \
uInt sourhi, sourmh, sourml, sourlo; \
sourlo=DFWORD(df, 3); \
(buf)[0]+=DPD2BIN[sourlo&0x3ff]; \
if (buf[0]>999) {buf[0]-=1000; buf[1]++;} \
(buf)[1]+=DPD2BIN[(sourlo>>10)&0x3ff]; \
if (buf[1]>999) {buf[1]-=1000; buf[2]++;} \
(buf)[2]+=DPD2BIN[(sourlo>>20)&0x3ff]; \
if (buf[2]>999) {buf[2]-=1000; buf[3]++;} \
sourml=DFWORD(df, 2); \
(buf)[3]+=DPD2BIN[((sourml<<2) | (sourlo>>30))&0x3ff]; \
if (buf[3]>999) {buf[3]-=1000; buf[4]++;} \
(buf)[4]+=DPD2BIN[(sourml>>8)&0x3ff]; \
if (buf[4]>999) {buf[4]-=1000; buf[5]++;} \
(buf)[5]+=DPD2BIN[(sourml>>18)&0x3ff]; \
if (buf[5]>999) {buf[5]-=1000; buf[6]++;} \
sourmh=DFWORD(df, 1); \
(buf)[6]+=DPD2BIN[((sourmh<<4) | (sourml>>28))&0x3ff]; \
if (buf[6]>999) {buf[6]-=1000; buf[7]++;} \
(buf)[7]+=DPD2BIN[(sourmh>>6)&0x3ff]; \
if (buf[7]>999) {buf[7]-=1000; buf[8]++;} \
(buf)[8]+=DPD2BIN[(sourmh>>16)&0x3ff]; \
if (buf[8]>999) {buf[8]-=1000; buf[9]++;} \
sourhi=DFWORD(df, 0); \
(buf)[9]+=DPD2BIN[((sourhi<<6) | (sourmh>>26))&0x3ff]; \
if (buf[9]>999) {buf[9]-=1000; buf[10]++;} \
(buf)[10]+=DPD2BIN[(sourhi>>4)&0x3ff]; \
if (buf[10]>999) {buf[10]-=1000; buf[11]++;} \
(buf)[11]+=DECCOMBMSD[sourhi>>26];}
#endif
/* Set a decFloat to the maximum positive finite number (Nmax) */
#if DECPMAX==7
#define DFSETNMAX(df) \
{DFWORD(df, 0)=0x77f3fcff;}
#elif DECPMAX==16
#define DFSETNMAX(df) \
{DFWORD(df, 0)=0x77fcff3f; \
DFWORD(df, 1)=0xcff3fcff;}
#elif DECPMAX==34
#define DFSETNMAX(df) \
{DFWORD(df, 0)=0x77ffcff3; \
DFWORD(df, 1)=0xfcff3fcf; \
DFWORD(df, 2)=0xf3fcff3f; \
DFWORD(df, 3)=0xcff3fcff;}
#endif
/* [end of format-dependent macros and constants] */
#endif
#else
#error decNumberLocal included more than once
#endif

1892
deps/icu-small/source/i18n/decimfmt.cpp vendored Normal file
View File

File diff suppressed because it is too large Load Diff

Some files were not shown because too many files have changed in this diff Show More