diff -Nru ucto-0.9.6/debian/changelog ucto-0.9.6/debian/changelog --- ucto-0.9.6/debian/changelog 2017-01-23 14:10:00.000000000 +0000 +++ ucto-0.9.6/debian/changelog 2018-11-18 14:40:29.000000000 +0000 @@ -1,3 +1,10 @@ +ucto (0.9.6-1+rpi1) buster-staging; urgency=medium + + * Apply patch from https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=913512 + to fix build with new ICU. + + -- Peter Michael Green Sun, 18 Nov 2018 14:40:29 +0000 + ucto (0.9.6-1) unstable; urgency=high * Updated to new upstream (0.9.6) diff -Nru ucto-0.9.6/debian/patches/series ucto-0.9.6/debian/patches/series --- ucto-0.9.6/debian/patches/series 1970-01-01 00:00:00.000000000 +0000 +++ ucto-0.9.6/debian/patches/series 2018-11-18 14:38:27.000000000 +0000 @@ -0,0 +1 @@ +ucto_ICU_63.1.patch diff -Nru ucto-0.9.6/debian/patches/ucto_ICU_63.1.patch ucto-0.9.6/debian/patches/ucto_ICU_63.1.patch --- ucto-0.9.6/debian/patches/ucto_ICU_63.1.patch 1970-01-01 00:00:00.000000000 +0000 +++ ucto-0.9.6/debian/patches/ucto_ICU_63.1.patch 2018-11-18 14:38:12.000000000 +0000 @@ -0,0 +1,1081 @@ +Description: fix FTBFS with ICU 63.1 + Add icu namespace. +Author: Laszlo Boszormenyi (GCS) +Last-Update: 2018-11-09 + +--- + +--- ucto-0.9.6.orig/include/ucto/setting.h ++++ ucto-0.9.6/include/ucto/setting.h +@@ -38,15 +38,15 @@ namespace Tokenizer { + public: + Rule(): regexp(0){ + }; +- Rule( const UnicodeString& id, const UnicodeString& pattern); ++ Rule( const icu::UnicodeString& id, const icu::UnicodeString& pattern); + ~Rule(); +- UnicodeString id; +- UnicodeString pattern; ++ icu::UnicodeString id; ++ icu::UnicodeString pattern; + UnicodeRegexMatcher *regexp; +- bool matchAll( const UnicodeString&, +- UnicodeString&, +- UnicodeString&, +- std::vector& ); ++ bool matchAll( const icu::UnicodeString&, ++ icu::UnicodeString&, ++ icu::UnicodeString&, ++ std::vector& ); + private: + Rule( const Rule& ); // inhibit copies + Rule& operator=( const Rule& ); // inhibit copies +@@ -56,17 +56,17 @@ namespace Tokenizer { + class Quoting { + friend std::ostream& operator<<( std::ostream&, const Quoting& ); + struct QuotePair { +- UnicodeString openQuote; +- UnicodeString closeQuote; ++ icu::UnicodeString openQuote; ++ icu::UnicodeString closeQuote; + }; + public: +- void add( const UnicodeString&, const UnicodeString& ); +- UnicodeString lookupOpen( const UnicodeString &) const; +- UnicodeString lookupClose( const UnicodeString & ) const; ++ void add( const icu::UnicodeString&, const icu::UnicodeString& ); ++ icu::UnicodeString lookupOpen( const icu::UnicodeString &) const; ++ icu::UnicodeString lookupClose( const icu::UnicodeString & ) const; + bool empty() const { return _quotes.empty(); }; + bool emptyStack() const { return quotestack.empty(); }; + void clearStack() { quoteindexstack.clear(); quotestack.clear(); }; +- int lookup( const UnicodeString&, int& ); ++ int lookup( const icu::UnicodeString&, int& ); + void eraseAtPos( int pos ) { + quotestack.erase( quotestack.begin()+pos ); + quoteindexstack.erase( quoteindexstack.begin()+pos ); +@@ -90,14 +90,14 @@ namespace Tokenizer { + bool readfilters( const std::string& ); + bool readquotes( const std::string& ); + bool readeosmarkers( const std::string& ); +- bool readabbreviations( const std::string&, UnicodeString& ); +- void add_rule( const UnicodeString&, const std::vector& ); +- void sortRules( std::map&, +- const std::vector& ); +- UnicodeString eosmarkers; ++ bool readabbreviations( const std::string&, icu::UnicodeString& ); ++ void add_rule( const icu::UnicodeString&, const std::vector& ); ++ void sortRules( std::map&, ++ const std::vector& ); ++ icu::UnicodeString eosmarkers; + std::vector rules; +- std::map rulesmap; +- std::map rules_index; ++ std::map rulesmap; ++ std::map rules_index; + Quoting quotes; + UnicodeFilter filter; + std::string set_file; // the name of the settingsfile +--- ucto-0.9.6.orig/include/ucto/tokenize.h ++++ ucto-0.9.6/include/ucto/tokenize.h +@@ -78,11 +78,11 @@ namespace Tokenizer { + class Token { + friend std::ostream& operator<< (std::ostream&, const Token& ); + public: +- UnicodeString type; +- UnicodeString us; ++ icu::UnicodeString type; ++ icu::UnicodeString us; + TokenRole role; +- Token( const UnicodeString&, +- const UnicodeString&, ++ Token( const icu::UnicodeString&, ++ const icu::UnicodeString&, + TokenRole role = NOROLE, + const std::string& = "" ); + std::string lc; // ISO 639-3 language code +@@ -130,12 +130,12 @@ namespace Tokenizer { + + // Tokenize a line (a line is NOT a sentence, but an arbitrary string + // of characters, inclusive EOS markers, Newlines etc.) +- int tokenizeLine( const UnicodeString&, ++ int tokenizeLine( const icu::UnicodeString&, + const std::string& = "default" ); // Unicode chars + int tokenizeLine( const std::string&, + const std::string& = "default" ); // UTF8 chars + +- void passthruLine( const UnicodeString&, bool& ); ++ void passthruLine( const icu::UnicodeString&, bool& ); + void passthruLine( const std::string&, bool& ); + + //Processes tokens and initialises the sentence buffer. Returns the amount of sentences found +@@ -209,8 +209,8 @@ namespace Tokenizer { + void setLanguage( const std::string& l ){ default_language = l; }; + + // set eos marker +- UnicodeString setEosMarker( const std::string& s = "") { UnicodeString t = eosmark; eosmark = folia::UTF8ToUnicode(s); return t; }; +- UnicodeString getEosMarker( ) const { return eosmark; } ++ icu::UnicodeString setEosMarker( const std::string& s = "") { icu::UnicodeString t = eosmark; eosmark = folia::UTF8ToUnicode(s); return t; }; ++ icu::UnicodeString getEosMarker( ) const { return eosmark; } + + bool setNormSet( const std::string& ); + +@@ -255,14 +255,14 @@ namespace Tokenizer { + private: + TokenizerClass( const TokenizerClass& ); // inhibit copies + TokenizerClass& operator=( const TokenizerClass& ); // inhibit copies +- void add_rule( const UnicodeString&, +- const std::vector& ); +- void tokenizeWord( const UnicodeString&, ++ void add_rule( const icu::UnicodeString&, ++ const std::vector& ); ++ void tokenizeWord( const icu::UnicodeString&, + bool, + const std::string&, +- const UnicodeString& ="" ); ++ const icu::UnicodeString& ="" ); + +- bool detectEos( size_t, const UnicodeString&, const Quoting& ) const; ++ bool detectEos( size_t, const icu::UnicodeString&, const Quoting& ) const; + void detectSentenceBounds( const int offset, + const std::string& = "default" ); + void detectQuotedSentenceBounds( const int offset, +@@ -272,7 +272,7 @@ namespace Tokenizer { + //Signal the tokeniser that a paragraph is detected + void signalParagraph( bool b=true ) { paragraphsignal = b; }; + +- bool resolveQuote( int, const UnicodeString&, Quoting& ); ++ bool resolveQuote( int, const icu::UnicodeString&, Quoting& ); + bool u_isquote( UChar32, + const Quoting& ) const; + std::string checkBOM( std::istream& ); +@@ -289,9 +289,9 @@ namespace Tokenizer { + UnicodeNormalizer normalizer; + std::string inputEncoding; + +- UnicodeString eosmark; ++ icu::UnicodeString eosmark; + std::vector tokens; +- std::set norm_set; ++ std::set norm_set; + TiCC::LogStream *theErrLog; + + std::string default_language; +--- ucto-0.9.6.orig/include/ucto/unicode.h ++++ ucto-0.9.6/include/ucto/unicode.h +@@ -42,7 +42,7 @@ namespace Tokenizer { + class UnicodeNormalizer { + public: + UnicodeNormalizer(): mode(UNORM_NFC){}; +- UnicodeString normalize( const UnicodeString& ); ++ icu::UnicodeString normalize( const icu::UnicodeString& ); + std::string getMode( ) const; + std::string setMode( const std::string& ); + private: +@@ -52,34 +52,34 @@ namespace Tokenizer { + class UnicodeFilter { + friend std::ostream& operator<<( std::ostream&, const UnicodeFilter& ); + public: +- UnicodeString filter( const UnicodeString& ); ++ icu::UnicodeString filter( const icu::UnicodeString& ); + bool fill( const std::string& ); +- bool add( const UnicodeString& ); ++ bool add( const icu::UnicodeString& ); + bool add( const std::string& ); + bool empty() const { return the_map.empty(); }; + private: +- void add( UChar uc, const UnicodeString& us ) { the_map[uc] = us; }; +- std::map the_map; ++ void add( UChar uc, const icu::UnicodeString& us ) { the_map[uc] = us; }; ++ std::map the_map; + }; + + class UnicodeRegexMatcher { + public: +- UnicodeRegexMatcher( const UnicodeString&, const UnicodeString& name="" ); ++ UnicodeRegexMatcher( const icu::UnicodeString&, const icu::UnicodeString& name="" ); + ~UnicodeRegexMatcher(); +- bool match_all( const UnicodeString&, UnicodeString&, UnicodeString& ); +- const UnicodeString get_match( unsigned int ) const; ++ bool match_all( const icu::UnicodeString&, icu::UnicodeString&, icu::UnicodeString& ); ++ const icu::UnicodeString get_match( unsigned int ) const; + int NumOfMatches() const; +- int split( const UnicodeString&, std::vector& ); +- UnicodeString Pattern() const; ++ int split( const icu::UnicodeString&, std::vector& ); ++ icu::UnicodeString Pattern() const; + private: + UnicodeRegexMatcher( const UnicodeRegexMatcher& ); // inhibit copies + UnicodeRegexMatcher& operator=( const UnicodeRegexMatcher& ); // inhibit copies + std::string failString; +- RegexPattern *pattern; +- RegexMatcher *matcher; ++ icu::RegexPattern *pattern; ++ icu::RegexMatcher *matcher; + UnicodeRegexMatcher(); +- std::vector results; +- const UnicodeString _name; ++ std::vector results; ++ const icu::UnicodeString _name; + }; + + } // namespace +--- ucto-0.9.6.orig/src/setting.cxx ++++ ucto-0.9.6/src/setting.cxx +@@ -54,7 +54,7 @@ namespace Tokenizer { + ORDINALS, EOSMARKERS, QUOTES, CURRENCY, + FILTER, RULEORDER, METARULES }; + +- ConfigMode getMode( const UnicodeString& line ) { ++ ConfigMode getMode( const icu::UnicodeString& line ) { + ConfigMode mode = NONE; + if (line == "[RULES]") { + mode = RULES; +@@ -111,7 +111,7 @@ namespace Tokenizer { + public: + uConfigError( const string& s, const string& f ): + invalid_argument( "ucto: " + s + " (" + f + ")" ){}; +- uConfigError( const UnicodeString& us, const string& f ): ++ uConfigError( const icu::UnicodeString& us, const string& f ): + uConfigError( folia::UnicodeToUTF8(us), f ){}; + }; + +@@ -143,14 +143,14 @@ namespace Tokenizer { + } + } + +- void Quoting::add( const UnicodeString& o, const UnicodeString& c ){ ++ void Quoting::add( const icu::UnicodeString& o, const icu::UnicodeString& c ){ + QuotePair quote; + quote.openQuote = o; + quote.closeQuote = c; + _quotes.push_back( quote ); + } + +- int Quoting::lookup( const UnicodeString& open, int& stackindex ){ ++ int Quoting::lookup( const icu::UnicodeString& open, int& stackindex ){ + if (quotestack.empty() || (quotestack.size() != quoteindexstack.size())) return -1; + auto it = quotestack.crbegin(); + size_t i = quotestack.size(); +@@ -165,7 +165,7 @@ namespace Tokenizer { + return -1; + } + +- UnicodeString Quoting::lookupOpen( const UnicodeString &q ) const { ++ icu::UnicodeString Quoting::lookupOpen( const icu::UnicodeString &q ) const { + for ( const auto& quote : _quotes ){ + if ( quote.openQuote.indexOf(q) >=0 ) + return quote.closeQuote; +@@ -173,8 +173,8 @@ namespace Tokenizer { + return ""; + } + +- UnicodeString Quoting::lookupClose( const UnicodeString &q ) const { +- UnicodeString res; ++ icu::UnicodeString Quoting::lookupClose( const icu::UnicodeString &q ) const { ++ icu::UnicodeString res; + for ( const auto& quote : _quotes ){ + if ( quote.closeQuote.indexOf(q) >= 0 ) + return quote.openQuote; +@@ -186,7 +186,7 @@ namespace Tokenizer { + delete regexp; + } + +- Rule::Rule( const UnicodeString& _id, const UnicodeString& _pattern): ++ Rule::Rule( const icu::UnicodeString& _id, const icu::UnicodeString& _pattern): + id(_id), pattern(_pattern) { + regexp = new UnicodeRegexMatcher( pattern, id ); + } +@@ -200,10 +200,10 @@ namespace Tokenizer { + return os; + } + +- bool Rule::matchAll( const UnicodeString& line, +- UnicodeString& pre, +- UnicodeString& post, +- vector& matches ){ ++ bool Rule::matchAll( const icu::UnicodeString& line, ++ icu::UnicodeString& pre, ++ icu::UnicodeString& post, ++ vector& matches ){ + matches.clear(); + pre = ""; + post = ""; +@@ -244,7 +244,7 @@ namespace Tokenizer { + else { + string rawline; + while ( getline(f,rawline) ){ +- UnicodeString line = folia::UTF8ToUnicode(rawline); ++ icu::UnicodeString line = folia::UTF8ToUnicode(rawline); + line.trim(); + if ((line.length() > 0) && (line[0] != '#')) { + if ( tokDebug >= 5 ){ +@@ -255,8 +255,8 @@ namespace Tokenizer { + throw uConfigError( "invalid RULES entry: " + line, + fname ); + } +- UnicodeString id = UnicodeString( line, 0,splitpoint); +- UnicodeString pattern = UnicodeString( line, splitpoint+1); ++ icu::UnicodeString id = icu::UnicodeString( line, 0,splitpoint); ++ icu::UnicodeString pattern = icu::UnicodeString( line, splitpoint+1); + rulesmap[id] = new Rule( id, pattern); + } + } +@@ -282,7 +282,7 @@ namespace Tokenizer { + else { + string rawline; + while ( getline(f,rawline) ){ +- UnicodeString line = folia::UTF8ToUnicode(rawline); ++ icu::UnicodeString line = folia::UTF8ToUnicode(rawline); + line.trim(); + if ((line.length() > 0) && (line[0] != '#')) { + if ( tokDebug >= 5 ){ +@@ -296,8 +296,8 @@ namespace Tokenizer { + + " (missing whitespace)", + fname ); + } +- UnicodeString open = UnicodeString( line, 0,splitpoint); +- UnicodeString close = UnicodeString( line, splitpoint+1); ++ icu::UnicodeString open = icu::UnicodeString( line, 0,splitpoint); ++ icu::UnicodeString close = icu::UnicodeString( line, splitpoint+1); + open = open.trim().unescape(); + close = close.trim().unescape(); + if ( open.isEmpty() || close.isEmpty() ){ +@@ -323,7 +323,7 @@ namespace Tokenizer { + else { + string rawline; + while ( getline(f,rawline) ){ +- UnicodeString line = folia::UTF8ToUnicode(rawline); ++ icu::UnicodeString line = folia::UTF8ToUnicode(rawline); + line.trim(); + if ((line.length() > 0) && (line[0] != '#')) { + if ( tokDebug >= 5 ){ +@@ -331,7 +331,7 @@ namespace Tokenizer { + } + if ( ( line.startsWith("\\u") && line.length() == 6 ) || + ( line.startsWith("\\U") && line.length() == 10 ) ){ +- UnicodeString uit = line.unescape(); ++ icu::UnicodeString uit = line.unescape(); + if ( uit.isEmpty() ){ + throw uConfigError( "Invalid EOSMARKERS entry: " + line, fname ); + } +@@ -344,7 +344,7 @@ namespace Tokenizer { + } + + bool Setting::readabbreviations( const string& fname, +- UnicodeString& abbreviations ){ ++ icu::UnicodeString& abbreviations ){ + if ( tokDebug > 0 ){ + *theErrLog << "%include " << fname << endl; + } +@@ -355,7 +355,7 @@ namespace Tokenizer { + else { + string rawline; + while ( getline(f,rawline) ){ +- UnicodeString line = folia::UTF8ToUnicode(rawline); ++ icu::UnicodeString line = folia::UTF8ToUnicode(rawline); + line.trim(); + if ((line.length() > 0) && (line[0] != '#')) { + if ( tokDebug >= 5 ){ +@@ -370,17 +370,17 @@ namespace Tokenizer { + return true; + } + +- void Setting::add_rule( const UnicodeString& name, +- const vector& parts ){ +- UnicodeString pat; ++ void Setting::add_rule( const icu::UnicodeString& name, ++ const vector& parts ){ ++ icu::UnicodeString pat; + for ( auto const& part : parts ){ + pat += part; + } + rulesmap[name] = new Rule( name, pat ); + } + +- void Setting::sortRules( map& rulesmap, +- const vector& sort ){ ++ void Setting::sortRules( map& rulesmap, ++ const vector& sort ){ + // LOG << "rules voor sort : " << endl; + // for ( size_t i=0; i < rules.size(); ++i ){ + // LOG << "rule " << i << " " << *rules[i] << endl; +@@ -432,14 +432,14 @@ namespace Tokenizer { + return result; + } + +- void addOrder( vector& order, +- map& reverse_order, ++ void addOrder( vector& order, ++ map& reverse_order, + int& index, +- UnicodeString &line, ++ icu::UnicodeString &line, + const string& fn ){ + try { + UnicodeRegexMatcher m( "\\s+" ); +- vector usv; ++ vector usv; + m.split( line, usv ); + for ( const auto& us : usv ){ + if ( reverse_order.find( us ) != reverse_order.end() ){ +@@ -500,7 +500,7 @@ namespace Tokenizer { + int dbg, LogStream* ls ) { + tokDebug = dbg; + theErrLog = ls; +- map pattern = { { ABBREVIATIONS, "" }, ++ map pattern = { { ABBREVIATIONS, "" }, + { TOKENS, "" }, + { PREFIXES, "" }, + { SUFFIXES, "" }, +@@ -508,7 +508,7 @@ namespace Tokenizer { + { ATTACHEDSUFFIXES, "" }, + { UNITS, "" }, + { ORDINALS, "" } }; +- vector rules_order; ++ vector rules_order; + vector meta_rules; + string conffile = get_filename( settings_name ); + +@@ -572,7 +572,7 @@ namespace Tokenizer { + continue; + } + +- UnicodeString line = folia::UTF8ToUnicode(rawline); ++ icu::UnicodeString line = folia::UTF8ToUnicode(rawline); + line.trim(); + if ((line.length() > 0) && (line[0] != '#')) { + if (line[0] == '[') { +@@ -580,7 +580,7 @@ namespace Tokenizer { + } + else { + if ( line[0] == '\\' && line.length() > 1 && line[1] == '[' ){ +- line = UnicodeString( line, 1 ); ++ line = icu::UnicodeString( line, 1 ); + } + switch( mode ){ + case RULES: { +@@ -589,8 +589,8 @@ namespace Tokenizer { + throw uConfigError( "invalid RULES entry: " + line, + set_file ); + } +- UnicodeString id = UnicodeString( line, 0,splitpoint); +- UnicodeString pattern = UnicodeString( line, splitpoint+1); ++ icu::UnicodeString id = icu::UnicodeString( line, 0,splitpoint); ++ icu::UnicodeString pattern = icu::UnicodeString( line, splitpoint+1); + rulesmap[id] = new Rule( id, pattern); + } + break; +@@ -617,7 +617,7 @@ namespace Tokenizer { + case EOSMARKERS: + if ( ( line.startsWith("\\u") && line.length() == 6 ) || + ( line.startsWith("\\U") && line.length() == 10 ) ){ +- UnicodeString uit = line.unescape(); ++ icu::UnicodeString uit = line.unescape(); + if ( uit.isEmpty() ){ + throw uConfigError( "Invalid EOSMARKERS entry: " + line, + set_file ); +@@ -634,8 +634,8 @@ namespace Tokenizer { + + " (missing whitespace)", + set_file ); + } +- UnicodeString open = UnicodeString( line, 0,splitpoint); +- UnicodeString close = UnicodeString( line, splitpoint+1); ++ icu::UnicodeString open = icu::UnicodeString( line, 0,splitpoint); ++ icu::UnicodeString close = icu::UnicodeString( line, splitpoint+1); + open = open.trim().unescape(); + close = close.trim().unescape(); + if ( open.isEmpty() || close.isEmpty() ){ +@@ -702,7 +702,7 @@ namespace Tokenizer { + } + continue; + } +- UnicodeString name = folia::UTF8ToUnicode( nam ); ++ icu::UnicodeString name = folia::UTF8ToUnicode( nam ); + string rule = mr.substr( pos+1 ); + if ( tokDebug > 5 ){ + LOG << "SPLIT using: '" << split << "'" << endl; +@@ -712,11 +712,11 @@ namespace Tokenizer { + for ( auto& str : parts ){ + str = TiCC::trim( str ); + } +- vector new_parts; +- vector undef_parts; ++ vector new_parts; ++ vector undef_parts; + bool skip_rule = false; + for ( const auto& part : parts ){ +- UnicodeString meta = folia::UTF8ToUnicode( part ); ++ icu::UnicodeString meta = folia::UTF8ToUnicode( part ); + ConfigMode mode = getMode( "[" + meta + "]" ); + switch ( mode ){ + case ORDINALS: +--- ucto-0.9.6.orig/src/tokenize.cxx ++++ ucto-0.9.6/src/tokenize.cxx +@@ -88,11 +88,11 @@ namespace Tokenizer { + }; + + +- UnicodeString convert( const string& line, ++ icu::UnicodeString convert( const string& line, + const string& inputEncoding ){ +- UnicodeString result; ++ icu::UnicodeString result; + try { +- result = UnicodeString( line.c_str(), ++ result = icu::UnicodeString( line.c_str(), + line.length(), + inputEncoding.c_str() ); + } +@@ -108,17 +108,17 @@ namespace Tokenizer { + return result; + } + +- const UnicodeString type_space = "SPACE"; +- const UnicodeString type_currency = "CURRENCY"; +- const UnicodeString type_emoticon = "EMOTICON"; +- const UnicodeString type_word = "WORD"; +- const UnicodeString type_symbol = "SYMBOL"; +- const UnicodeString type_punctuation = "PUNCTUATION"; +- const UnicodeString type_number = "NUMBER"; +- const UnicodeString type_unknown = "UNKNOWN"; ++ const icu::UnicodeString type_space = "SPACE"; ++ const icu::UnicodeString type_currency = "CURRENCY"; ++ const icu::UnicodeString type_emoticon = "EMOTICON"; ++ const icu::UnicodeString type_word = "WORD"; ++ const icu::UnicodeString type_symbol = "SYMBOL"; ++ const icu::UnicodeString type_punctuation = "PUNCTUATION"; ++ const icu::UnicodeString type_number = "NUMBER"; ++ const icu::UnicodeString type_unknown = "UNKNOWN"; + +- Token::Token( const UnicodeString& _type, +- const UnicodeString& _s, ++ Token::Token( const icu::UnicodeString& _type, ++ const icu::UnicodeString& _s, + TokenRole _role, const string& _lc ): + type(_type), us(_s), role(_role), lc(_lc) {} + +@@ -226,7 +226,7 @@ namespace Tokenizer { + << "'" << endl; + } + stripCR( line ); +- UnicodeString input_line; ++ icu::UnicodeString input_line; + if ( line.size() > 0 && line[0] == 0 ){ + // when processing UTF16LE, '0' bytes show up at pos 0 + // we discard them, not for UTF16BE! +@@ -273,7 +273,7 @@ namespace Tokenizer { + LOG << "use textCat to guess language from: " + << input_line << endl; + } +- UnicodeString temp = input_line; ++ icu::UnicodeString temp = input_line; + temp.toLower(); + string lan = tc->get_language( folia::UnicodeToUTF8(temp) ); + if ( settings.find( lan ) != settings.end() ){ +@@ -531,7 +531,7 @@ namespace Tokenizer { + if ( root->hastext( outputclass ) ){ + return; + } +- UnicodeString utxt = root->text( outputclass, false, false ); ++ icu::UnicodeString utxt = root->text( outputclass, false, false ); + // cerr << "untok: '" << utxt << "'" << endl; + // UnicodeString txt = root->text( outputclass, true ); + // cerr << " tok: '" << txt << "'" << endl; +@@ -664,7 +664,7 @@ namespace Tokenizer { + if ( tokDebug > 0 ){ + cerr << "tokenize sentence element: " << element->id() << endl; + } +- UnicodeString line = element->stricttext( inputclass ); ++ icu::UnicodeString line = element->stricttext( inputclass ); + if ( line.isEmpty() ){ + // so no usefull text in this element. skip it + return; +@@ -837,7 +837,7 @@ namespace Tokenizer { + args["space"]= "no"; + } + folia::FoliaElement *w = new folia::Word( args, root->doc() ); +- UnicodeString out = token.us; ++ icu::UnicodeString out = token.us; + if (lowercase) { + out.toLower(); + } +@@ -898,7 +898,7 @@ namespace Tokenizer { + OUT << endl << endl; + } + } +- UnicodeString s = token.us; ++ icu::UnicodeString s = token.us; + if (lowercase) { + s = s.toLower(); + } +@@ -1118,12 +1118,12 @@ namespace Tokenizer { + quote = true; + } + else { +- UnicodeString opening = quotes.lookupOpen( c ); ++ icu::UnicodeString opening = quotes.lookupOpen( c ); + if (!opening.isEmpty()) { + quote = true; + } + else { +- UnicodeString closing = quotes.lookupClose( c ); ++ icu::UnicodeString closing = quotes.lookupClose( c ); + if (!closing.isEmpty()) { + quote = true; + } +@@ -1151,7 +1151,7 @@ namespace Tokenizer { + } + + bool TokenizerClass::resolveQuote( int endindex, +- const UnicodeString& open, ++ const icu::UnicodeString& open, + Quoting& quotes ) { + //resolve a quote + int stackindex = -1; +@@ -1250,7 +1250,7 @@ namespace Tokenizer { + } + + bool TokenizerClass::detectEos( size_t i, +- const UnicodeString& eosmarkers, ++ const icu::UnicodeString& eosmarkers, + const Quoting& quotes ) const { + bool is_eos = false; + UChar32 c = tokens[i].us.char32At(0); +@@ -1288,7 +1288,7 @@ namespace Tokenizer { + Quoting& quotes ) { + UChar32 c = tokens[i].us.char32At(0); + //Detect Quotation marks +- if ((c == '"') || ( UnicodeString(c) == """) ) { ++ if ((c == '"') || ( icu::UnicodeString(c) == """) ) { + if (tokDebug > 1 ){ + LOG << "[detectQuoteBounds] Standard double-quote (ambiguous) found @i="<< i << endl; + } +@@ -1311,7 +1311,7 @@ namespace Tokenizer { + } + } + else { +- UnicodeString close = quotes.lookupOpen( c ); ++ icu::UnicodeString close = quotes.lookupOpen( c ); + if ( !close.isEmpty() ){ // we have a opening quote + if ( tokDebug > 1 ) { + LOG << "[detectQuoteBounds] Opening quote found @i="<< i << ", pushing to stack for resolution later..." << endl; +@@ -1319,7 +1319,7 @@ namespace Tokenizer { + quotes.push( i, c ); // remember it + } + else { +- UnicodeString open = quotes.lookupClose( c ); ++ icu::UnicodeString open = quotes.lookupClose( c ); + if ( !open.isEmpty() ) { // we have a closeing quote + if (tokDebug > 1 ) { + LOG << "[detectQuoteBounds] Closing quote found @i="<< i << ", attempting to resolve..." << endl; +@@ -1484,17 +1484,17 @@ namespace Tokenizer { + + void TokenizerClass::passthruLine( const string& s, bool& bos ) { + // string wrapper +- UnicodeString us = convert( s, inputEncoding );; ++ icu::UnicodeString us = convert( s, inputEncoding );; + passthruLine( us, bos ); + } + +- void TokenizerClass::passthruLine( const UnicodeString& input, bool& bos ) { ++ void TokenizerClass::passthruLine( const icu::UnicodeString& input, bool& bos ) { + if (tokDebug) { + LOG << "[passthruLine] input: line=[" << input << "]" << endl; + } + bool alpha = false, num = false, punct = false; +- UnicodeString word; +- StringCharacterIterator sit(input); ++ icu::UnicodeString word; ++ icu::StringCharacterIterator sit(input); + while ( sit.hasNext() ){ + UChar32 c = sit.current32(); + if ( u_isspace(c)) { +@@ -1514,7 +1514,7 @@ namespace Tokenizer { + bos = true; + } + else { +- UnicodeString type; ++ icu::UnicodeString type; + if (alpha && !num && !punct) { + type = type_word; + } +@@ -1577,7 +1577,7 @@ namespace Tokenizer { + tokens.back().role |= ENDOFSENTENCE; + } + else { +- UnicodeString type; ++ icu::UnicodeString type; + if (alpha && !num && !punct) { + type = type_word; + } +@@ -1653,7 +1653,7 @@ namespace Tokenizer { + // string wrapper + int TokenizerClass::tokenizeLine( const string& s, + const string& lang ){ +- UnicodeString uinputstring = convert( s, inputEncoding ); ++ icu::UnicodeString uinputstring = convert( s, inputEncoding ); + return tokenizeLine( uinputstring, lang ); + } + +@@ -1673,7 +1673,7 @@ namespace Tokenizer { + || u_charType( c ) == U_OTHER_SYMBOL; + } + +- const UnicodeString& detect_type( UChar32 c ){ ++ const icu::UnicodeString& detect_type( UChar32 c ){ + if ( u_isspace(c)) { + return type_space; + } +@@ -1768,7 +1768,7 @@ namespace Tokenizer { + } + } + +- int TokenizerClass::tokenizeLine( const UnicodeString& originput, ++ int TokenizerClass::tokenizeLine( const icu::UnicodeString& originput, + const string& _lang ){ + string lang = _lang; + if ( lang.empty() ){ +@@ -1786,7 +1786,7 @@ namespace Tokenizer { + LOG << "[tokenizeLine] input: line=[" + << originput << "] (" << lang << ")" << endl; + } +- UnicodeString input = normalizer.normalize( originput ); ++ icu::UnicodeString input = normalizer.normalize( originput ); + if ( doFilter ){ + input = settings[lang]->filter.filter( input ); + } +@@ -1808,13 +1808,13 @@ namespace Tokenizer { + bool tokenizeword = false; + bool reset = false; + //iterate over all characters +- UnicodeString word; +- StringCharacterIterator sit(input); ++ icu::UnicodeString word; ++ icu::StringCharacterIterator sit(input); + long int i = 0; + while ( sit.hasNext() ){ + UChar32 c = sit.current32(); + if ( tokDebug > 8 ){ +- UnicodeString s = c; ++ icu::UnicodeString s = c; + int8_t charT = u_charType( c ); + LOG << "examine character: " << s << " type= " + << toString( charT ) << endl; +@@ -1855,7 +1855,7 @@ namespace Tokenizer { + } + int eospos = tokens.size()-1; + if (expliciteosfound > 0) { +- UnicodeString realword; ++ icu::UnicodeString realword; + word.extract(0,expliciteosfound,realword); + if (tokDebug >= 2) { + LOG << "[tokenizeLine] Prefix before EOS: " +@@ -1865,7 +1865,7 @@ namespace Tokenizer { + eospos++; + } + if ( expliciteosfound + eosmark.length() < word.length() ){ +- UnicodeString realword; ++ icu::UnicodeString realword; + word.extract( expliciteosfound+eosmark.length(), + word.length() - expliciteosfound - eosmark.length(), + realword ); +@@ -1941,10 +1941,10 @@ namespace Tokenizer { + return numNewTokens; + } + +- void TokenizerClass::tokenizeWord( const UnicodeString& input, ++ void TokenizerClass::tokenizeWord( const icu::UnicodeString& input, + bool space, + const string& lang, +- const UnicodeString& assigned_type ) { ++ const icu::UnicodeString& assigned_type ) { + bool recurse = !assigned_type.isEmpty(); + + int32_t inpLen = input.countChar32(); +@@ -1977,7 +1977,7 @@ namespace Tokenizer { + if ( inpLen == 1) { + //single character, no need to process all rules, do some simpler (faster) detection + UChar32 c = input.char32At(0); +- UnicodeString type = detect_type( c ); ++ icu::UnicodeString type = detect_type( c ); + if ( type == type_space ){ + return; + } +@@ -1993,7 +1993,7 @@ namespace Tokenizer { + } + } + else { +- UnicodeString word = input; ++ icu::UnicodeString word = input; + if ( norm_set.find( type ) != norm_set.end() ){ + word = "{{" + type + "}}"; + } +@@ -2010,10 +2010,10 @@ namespace Tokenizer { + if ( tokDebug >= 4){ + LOG << "\tTESTING " << rule->id << endl; + } +- UnicodeString type = rule->id; ++ icu::UnicodeString type = rule->id; + //Find first matching rule +- UnicodeString pre, post; +- vector matches; ++ icu::UnicodeString pre, post; ++ vector matches; + if ( rule->matchAll( input, pre, post, matches ) ){ + a_rule_matched = true; + if ( tokDebug >= 4 ){ +@@ -2083,7 +2083,7 @@ namespace Tokenizer { + if ( post.length() > 0 ) { + internal_space = false; + } +- UnicodeString word = matches[m]; ++ icu::UnicodeString word = matches[m]; + if ( norm_set.find( type ) != norm_set.end() ){ + word = "{{" + type + "}}"; + tokens.push_back( Token( type, word, internal_space ? NOROLE : NOSPACE, lang ) ); +--- ucto-0.9.6.orig/src/unicode.cxx ++++ ucto-0.9.6/src/unicode.cxx +@@ -84,10 +84,10 @@ namespace Tokenizer { + return res; + } + +- UnicodeString UnicodeNormalizer::normalize( const UnicodeString& us ){ +- UnicodeString r; ++ icu::UnicodeString UnicodeNormalizer::normalize( const icu::UnicodeString& us ){ ++ icu::UnicodeString r; + UErrorCode status=U_ZERO_ERROR; +- Normalizer::normalize( us, mode, 0, r, status ); ++ icu::Normalizer::normalize( us, mode, 0, r, status ); + if (U_FAILURE(status)){ + throw std::invalid_argument("Normalizer"); + } +@@ -101,18 +101,18 @@ namespace Tokenizer { + else { + auto it=q.the_map.cbegin(); + while ( it != q.the_map.cend() ){ +- os << folia::UnicodeToUTF8(UnicodeString(it->first)) << "\t" << it->second << endl; ++ os << folia::UnicodeToUTF8(icu::UnicodeString(it->first)) << "\t" << it->second << endl; + ++it; + } + } + return os; + } + +- UnicodeString UnicodeFilter::filter( const UnicodeString& s ){ ++ icu::UnicodeString UnicodeFilter::filter( const icu::UnicodeString& s ){ + if ( empty() ) + return s; + else { +- UnicodeString result; ++ icu::UnicodeString result; + for ( int i=0; i < s.length(); ++i ){ + auto it=the_map.find(s[i]); + if ( it != the_map.cend() ) +@@ -125,16 +125,16 @@ namespace Tokenizer { + } + + bool UnicodeFilter::add( const string& s ){ +- UnicodeString line = folia::UTF8ToUnicode(s); ++ icu::UnicodeString line = folia::UTF8ToUnicode(s); + return add( line ); + } + +- bool UnicodeFilter::add( const UnicodeString& s ){ +- UnicodeString line = s; ++ bool UnicodeFilter::add( const icu::UnicodeString& s ){ ++ icu::UnicodeString line = s; + line.trim(); + if ((line.length() > 0) && (line[0] != '#')) { +- UnicodeString open = ""; +- UnicodeString close = ""; ++ icu::UnicodeString open = ""; ++ icu::UnicodeString close = ""; + int splitpoint = line.indexOf(" "); + if ( splitpoint == -1 ) + splitpoint = line.indexOf("\t"); +@@ -142,8 +142,8 @@ namespace Tokenizer { + open = line; + } + else { +- open = UnicodeString( line, 0,splitpoint); +- close = UnicodeString( line, splitpoint+1); ++ open = icu::UnicodeString( line, 0,splitpoint); ++ close = icu::UnicodeString( line, splitpoint+1); + } + open = open.trim().unescape(); + close = close.trim().unescape(); +@@ -175,29 +175,29 @@ namespace Tokenizer { + class uConfigError: public std::invalid_argument { + public: + uConfigError( const string& s ): invalid_argument( "ucto: config file:" + s ){}; +- uConfigError( const UnicodeString& us ): invalid_argument( "ucto: config file:" + folia::UnicodeToUTF8(us) ){}; ++ uConfigError( const icu::UnicodeString& us ): invalid_argument( "ucto: config file:" + folia::UnicodeToUTF8(us) ){}; + }; + + +- UnicodeString UnicodeRegexMatcher::Pattern() const{ ++ icu::UnicodeString UnicodeRegexMatcher::Pattern() const{ + return pattern->pattern(); + } + +- UnicodeRegexMatcher::UnicodeRegexMatcher( const UnicodeString& pat, +- const UnicodeString& name ): ++ UnicodeRegexMatcher::UnicodeRegexMatcher( const icu::UnicodeString& pat, ++ const icu::UnicodeString& name ): + _name(name) + { + failString.clear(); + matcher = NULL; + UErrorCode u_stat = U_ZERO_ERROR; + UParseError errorInfo; +- pattern = RegexPattern::compile( pat, 0, errorInfo, u_stat ); ++ pattern = icu::RegexPattern::compile( pat, 0, errorInfo, u_stat ); + if ( U_FAILURE(u_stat) ){ + string spat = folia::UnicodeToUTF8(pat); + failString = folia::UnicodeToUTF8(_name); + if ( errorInfo.offset >0 ){ + failString += " Invalid regular expression at position " + TiCC::toString( errorInfo.offset ) + "\n"; +- UnicodeString pat1 = UnicodeString( pat, 0, errorInfo.offset -1 ); ++ icu::UnicodeString pat1 = icu::UnicodeString( pat, 0, errorInfo.offset -1 ); + failString += folia::UnicodeToUTF8(pat1) + " <== HERE\n"; + } + else { +@@ -222,9 +222,9 @@ namespace Tokenizer { + + //#define MATCH_DEBUG 1 + +- bool UnicodeRegexMatcher::match_all( const UnicodeString& line, +- UnicodeString& pre, +- UnicodeString& post ){ ++ bool UnicodeRegexMatcher::match_all( const icu::UnicodeString& line, ++ icu::UnicodeString& pre, ++ icu::UnicodeString& post ){ + UErrorCode u_stat = U_ZERO_ERROR; + pre = ""; + post = ""; +@@ -243,21 +243,21 @@ namespace Tokenizer { + #endif + if ( matcher->groupCount() == 0 ){ + // case 1: a rule without capture groups matches +- UnicodeString us = matcher->group(0,u_stat) ; ++ icu::UnicodeString us = matcher->group(0,u_stat) ; + #ifdef MATCH_DEBUG + cerr << "case 1, result = " << us << endl; + #endif + results.push_back( us ); + int start = matcher->start( 0, u_stat ); + if ( start > 0 ){ +- pre = UnicodeString( line, 0, start ); ++ pre = icu::UnicodeString( line, 0, start ); + #ifdef MATCH_DEBUG + cerr << "found pre " << folia::UnicodeToUTF8(pre) << endl; + #endif + } + int end = matcher->end( 0, u_stat ); + if ( end < line.length() ){ +- post = UnicodeString( line, end ); ++ post = icu::UnicodeString( line, end ); + #ifdef MATCH_DEBUG + cerr << "found post " << folia::UnicodeToUTF8(post) << endl; + #endif +@@ -268,20 +268,20 @@ namespace Tokenizer { + // case 2: a rule with one capture group matches + int start = matcher->start( 1, u_stat ); + if ( start >= 0 ){ +- UnicodeString us = matcher->group(1,u_stat) ; ++ icu::UnicodeString us = matcher->group(1,u_stat) ; + #ifdef MATCH_DEBUG + cerr << "case 2a , result = " << us << endl; + #endif + results.push_back( us ); + if ( start > 0 ){ +- pre = UnicodeString( line, 0, start ); ++ pre = icu::UnicodeString( line, 0, start ); + #ifdef MATCH_DEBUG + cerr << "found pre " << pre << endl; + #endif + } + int end = matcher->end( 1, u_stat ); + if ( end < line.length() ){ +- post = UnicodeString( line, end ); ++ post = icu::UnicodeString( line, end ); + #ifdef MATCH_DEBUG + cerr << "found post " << post << endl; + #endif +@@ -289,21 +289,21 @@ namespace Tokenizer { + } + else { + // group 1 is empty, return group 0 +- UnicodeString us = matcher->group(0,u_stat) ; ++ icu::UnicodeString us = matcher->group(0,u_stat) ; + #ifdef MATCH_DEBUG + cerr << "case 2b , result = " << us << endl; + #endif + results.push_back( us ); + start = matcher->start( 0, u_stat ); + if ( start > 0 ){ +- pre = UnicodeString( line, 0, start ); ++ pre = icu::UnicodeString( line, 0, start ); + #ifdef MATCH_DEBUG + cerr << "found pre " << pre << endl; + #endif + } + int end = matcher->end( 0, u_stat ); + if ( end < line.length() ){ +- post = UnicodeString( line, end ); ++ post = icu::UnicodeString( line, end ); + #ifdef MATCH_DEBUG + cerr << "found post " << post << endl; + #endif +@@ -332,7 +332,7 @@ namespace Tokenizer { + else + break; + if ( start > end ){ +- pre = UnicodeString( line, end, start ); ++ pre = icu::UnicodeString( line, end, start ); + #ifdef MATCH_DEBUG + cerr << "found pre " << folia::UnicodeToUTF8(pre) << endl; + #endif +@@ -342,7 +342,7 @@ namespace Tokenizer { + cerr << "end = " << end << endl; + #endif + if (!U_FAILURE(u_stat)){ +- results.push_back( UnicodeString( line, start, end - start ) ); ++ results.push_back( icu::UnicodeString( line, start, end - start ) ); + #ifdef MATCH_DEBUG + cerr << "added result " << folia::UnicodeToUTF8( results.back() ) << endl; + #endif +@@ -351,7 +351,7 @@ namespace Tokenizer { + break; + } + if ( end < line.length() ){ +- post = UnicodeString( line, end ); ++ post = icu::UnicodeString( line, end ); + #ifdef MATCH_DEBUG + cerr << "found post " << folia::UnicodeToUTF8(post) << endl; + #endif +@@ -364,7 +364,7 @@ namespace Tokenizer { + return false; + } + +- const UnicodeString UnicodeRegexMatcher::get_match( unsigned int n ) const{ ++ const icu::UnicodeString UnicodeRegexMatcher::get_match( unsigned int n ) const{ + if ( n < results.size() ) + return results[n]; + else +@@ -378,11 +378,11 @@ namespace Tokenizer { + return 0; + } + +- int UnicodeRegexMatcher::split( const UnicodeString& us, +- vector& result ){ ++ int UnicodeRegexMatcher::split( const icu::UnicodeString& us, ++ vector& result ){ + result.clear(); + const int maxWords = 256; +- UnicodeString words[maxWords]; ++ icu::UnicodeString words[maxWords]; + UErrorCode status = U_ZERO_ERROR; + int numWords = matcher->split( us, words, maxWords, status ); + for ( int i = 0; i < numWords; ++i )