[0169c1]: inc / ucommon / unicode.h  Maximize  Restore  History

Download this file

498 lines (428 with data), 13.4 kB

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
// Copyright (C) 2009-2010 David Sugar, Tycho Softworks.
//
// This file is part of GNU uCommon C++.
//
// GNU uCommon C++ is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published
// by the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// GNU uCommon C++ is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with GNU uCommon C++. If not, see <http://www.gnu.org/licenses/>.
/**
* Basic UCommon Unicode support.
* This includes computing unicode transcoding and supporting a
* UTF8-aware string class (UString). We may add support for a wchar_t
* aware string class as well, as some external api libraries may require
* ucs-2 or 4 encoded strings.
* @file ucommon/unicode.h
*/
/**
* An example of some unicode-utf8 transcoding.
* @example unicode.cpp
*/
#ifndef _UCOMMON_UNICODE_H_
#define _UCOMMON_UNICODE_H_
#ifndef _UCOMMON_STRING_H_
#include <ucommon/string.h>
#endif
NAMESPACE_UCOMMON
/**
* 32 bit unicode character code. We may extract this from a ucs2 or utf8
* string.
*/
typedef int32_t ucs4_t;
/**
* 16 bit unicode character code. Java and some api's like these.
*/
typedef int16_t ucs2_t;
/**
* Resolves issues where wchar_t is not defined.
*/
typedef void *unicode_t;
/**
* A core class of ut8 encoded string functions. This is a foundation for
* all utf8 string processing.
* @author David Sugar
*/
class __EXPORT utf8
{
public:
/**
* Size of "unicode_t" character codes, may not be ucs4_t size.
*/
static const unsigned ucsize;
/**
* A convenient NULL pointer value.
*/
static const char *nil;
/**
* Compute character size of utf8 string codepoint.
* @param codepoint in string.
* @return size of codepoint as utf8 encoded data, 0 if invalid.
*/
static unsigned size(const char *codepoint);
/**
* Count ut8 encoded ucs4 codepoints in string.
* @param string of utf8 data.
* @return codepount count, 0 if empty or invalid.
*/
static size_t count(const char *string);
/**
* Get codepoint offset in a string.
* @param string of utf8 data.
* @param position of codepoint in string, negative offsets are from tail.
* @return offset of codepoint or NULL if invalid.
*/
static char *offset(char *string, ssize_t position);
/**
* Convert a utf8 encoded codepoint to a ucs4 character value.
* @param encoded utf8 codepoint.
* @return ucs4 string or 0 if invalid.
*/
static ucs4_t codepoint(const char *encoded);
/**
* How many chars requires to encode a given wchar string.
* @param string of ucs4 data.
* @return number of chars required to encode given string.
*/
static size_t chars(const unicode_t string);
/**
* How many chars requires to encode a given unicode character.
* @param character tyo encode.
* @return number of chars required to encode given character.
*/
static size_t chars(ucs4_t character);
/**
* Convert a unicode string into utf8.
* @param string of unicode data.
* @param buffer to convert into.
* @param size of conversion buffer.
* @return number of code points converted.
*/
static size_t convert(const unicode_t string, char *buffer, size_t size);
/**
* Convert a utf8 string into a unicode data buffer.
* @param string to copy.
* @param unicode data buffer.
* @param size of unicode data buffer in codepoints.
* @return number of code points converted.
*/
static size_t extract(const char *string, unicode_t unicode, size_t size);
/**
* Find first occurance of character in string.
* @param string to search in.
* @param character code to search for.
* @param start offset in string in codepoints.
* @return pointer to first instance or NULL if not found.
*/
static const char *find(const char *string, ucs4_t character, size_t start = 0);
/**
* Find last occurance of character in string.
* @param string to search in.
* @param character code to search for.
* @param end offset to start from in codepoints.
* @return pointer to last instance or NULL if not found.
*/
static const char *rfind(const char *string, ucs4_t character, size_t end = (size_t)-1l);
/**
* Count occurences of a unicode character in string.
* @param string to search in.
* @param character code to search for.
* @return count of occurences.
*/
static unsigned ccount(const char *string, ucs4_t character);
/**
* Get a unicode character from a file stream.
* @param file to get character from.
* @return unicode character or EOF error.
*/
ucs4_t getch(FILE *file);
/**
* Push a unicode character to a file stream.
* @param character to push to file.
* @param file to push character to.
* @return unicode character or EOF on error.
*/
ucs4_t putch(ucs4_t character, FILE *file);
};
/**
* A copy-on-write utf8 string class that operates by reference count. This
* is derived from the classic uCommon String class by adding operations that
* are utf8 encoding aware.
* @author David Sugar <dyfet@gnutelephony.org>
*/
class __EXPORT UString : public String, public utf8
{
protected:
/**
* Create a new empty utf8 aware string object.
*/
UString();
/**
* Create an empty string with a buffer pre-allocated to a specified size.
* @param size of buffer to allocate.
*/
UString(strsize_t size);
/**
* Create a utf8 aware string for a null terminated unicode string.
* @param text of ucs4 encoded data.
*/
UString(const unicode_t text);
/**
* Create a string from null terminated text up to a maximum specified
* size.
* @param text to use for string.
* @param size limit of new string.
*/
UString(const char *text, strsize_t size);
/**
* Create a string for a substring. The end of the substring is a
* pointer within the substring itself.
* @param text to use for string.
* @param end of text in substring.
*/
UString(const unicode_t *text, const unicode_t *end);
/**
* Construct a copy of a string object. Our copy inherets the same
* reference counted instance of cstring as in the original.
* @param existing string to copy from.
*/
UString(const UString& existing);
/**
* Destroy string. De-reference cstring. If last reference to cstring,
* then also remove cstring from heap.
*/
virtual ~UString();
/**
* Get a new string object as a substring of the current object.
* @param codepoint offset of substring.
* @param size of substring in codepoints or 0 if to end.
* @return string object holding substring.
*/
UString get(strsize_t codepoint, strsize_t size = 0) const;
/**
* Extract a unicode byte sequence from utf8 object.
* @param unicode data buffer.
* @param size of data buffer.
* @return codepoints copied.
*/
inline size_t get(unicode_t unicode, size_t size) const
{return utf8::extract(str->text, unicode, size);};
/**
* Set a utf8 encoded string based on unicode data.
* @param unicode text to set.
*/
void set(const unicode_t unicode);
/**
* Add (append) unicode to a utf8 encoded string.
* @param unicode text to add.
*/
void add(const unicode_t unicode);
/**
* Return unicode character found at a specific codepoint in the string.
* @param position of codepoint in string, negative values computed from end.
* @return character code at specified position in string.
*/
ucs4_t at(int position) const;
/**
* Extract a unicode byte sequence from utf8 object.
* @param unicode data buffer.
* @param size of data buffer.
* @return codepoints copied.
*/
inline size_t operator()(unicode_t unicode, size_t size) const
{return utf8::extract(str->text, unicode, size);};
/**
* Get a new substring through object expression.
* @param codepoint offset of substring.
* @param size of substring or 0 if to end.
* @return string object holding substring.
*/
UString operator()(int codepoint, strsize_t size) const;
/**
* Reference a string in the object by codepoint offset. Positive
* offsets are from the start of the string, negative from the
* end.
* @param offset to string position.
* @return pointer to string data or NULL if invalid offset.
*/
const char *operator()(int offset) const;
/**
* Reference a unicode character in string object by array offset.
* @param position of codepoint offset to character.
* @return character value at offset.
*/
inline ucs4_t operator[](int position) const
{return UString::at(position);};
/**
* Count codepoints in current string.
* @return count of codepoints.
*/
inline strsize_t count(void) const
{return utf8::count(str->text);}
/**
* Count occurences of a unicode character in string.
* @param character code to search for.
* @return count of occurences.
*/
unsigned ccount(ucs4_t character) const;
/**
* Find first occurance of character in string.
* @param character code to search for.
* @param start offset in string in codepoints.
* @return pointer to first instance or NULL if not found.
*/
const char *find(ucs4_t character, strsize_t start = 0) const;
/**
* Find last occurance of character in string.
* @param character code to search for.
* @param end offset to start from in codepoints.
* @return pointer to last instance or NULL if not found.
*/
const char *rfind(ucs4_t character, strsize_t end = npos) const;
};
/**
* Pointer to utf8 encoded character data. This is a kind of "char *" for
* utf8 text.
* @author David Sugar <dyfet@gnutelephony.org>
*/
class __EXPORT utf8_pointer
{
protected:
uint8_t *text;
public:
/**
* Create a utf8 pointer set to NULL.
*/
utf8_pointer();
/**
* Create a utf8 pointer for an existing char pointer.
* @param string pointer to use.
*/
utf8_pointer(const char *string);
/**
* Create a utf8 pointer as a copy of existing utf8 pointer.
* @param copy of object to use.
*/
utf8_pointer(const utf8_pointer& copy);
/**
* Iterative increment of a utf8 pointer to prior codepoint.
* @return object incremented.
*/
utf8_pointer& operator ++();
/**
* Iterative decrement of a utf8 pointer to next codepoint.
* @return object decremented.
*/
utf8_pointer& operator --();
/**
* Adjust utf8 pointer by specified codepoints forward.
* @param offset to increment by.
* @return object incremented.
*/
utf8_pointer& operator +=(long offset);
/**
* Adjust utf8 pointer by specified codepoints backward.
* @param offset to decrement by.
* @return object decremented.
*/
utf8_pointer& operator -=(long offset);
/**
* Get new utf8 string after adding a codepoint offset.
* @param offset to add.
* @return new utf8 pointer pointing to specified offset.
*/
utf8_pointer operator+(long offset) const;
/**
* Get new utf8 string after subtracting a codepoint offset.
* @param offset to subtract.
* @return new utf8 pointer pointing to specified offset.
*/
utf8_pointer operator-(long offset) const;
/**
* Check if text is valid pointer.
* @return true if not NULL.
*/
inline operator bool() const
{return text != NULL;};
/**
* Check if text is an invalid pointer.
* @return false if not NULL.
*/
inline bool operator!() const
{return text == NULL;};
/**
* Extract a unicode character from a specified codepoint.
* @param codepoint offset to extract character from.
* @return unicode character or 0.
*/
ucs4_t operator[](long codepoint) const;
/**
* Assign a utf8 string to point to.
* @param string to point to.
* @return current object after set to string.
*/
utf8_pointer& operator=(const char *string);
/**
* Iterative increment of a utf8 pointer to next codepoint.
*/
void inc(void);
/**
* Iterative decrement of a utf8 pointer to prior codepoint.
*/
void dec(void);
/**
* check if pointer equals another string.
* @param string to check.
* @return true if same memory address.
*/
inline bool operator==(const char *string) const
{return (const char *)text == string;};
/**
* check if pointer does not equal another string.
* @param string to check.
* @return false if same memory address.
*/
inline bool operator!=(const char *string) const
{return (const char *)text != string;};
/**
* Get unicode character pointed to by pointer.
* @return unicode character we are pointing to.
*/
inline ucs4_t operator*() const
{return utf8::codepoint((const char *)text);};
/**
* Get c string we point to.
* @return string we point to.
*/
inline char *c_str(void) const
{return (char *)text;};
/**
* Convert utf8 pointer to a generic string pointer.
* @return generic string pointer.
*/
inline operator char*() const
{return (char *)text;};
/**
* Get length of null terminated utf8 string in codepoints.
* @return codepoint length of string.
*/
inline size_t len(void) const
{return utf8::count((const char *)text);};
};
/**
* Convenience type for utf8 encoded strings.
*/
typedef UString ustring_t;
/**
* Convience type for utf8_pointer strings.
*/
typedef utf8_pointer utf8_t;
END_NAMESPACE
#endif

Get latest updates about Open Source Projects, Conferences and News.

Sign up for the SourceForge newsletter:





No, thanks