Update Files

Kha/Backends/Kinc-hxcpp/khacpp/project/thirdparty/pcre2-10.42-16/maint/utf8.c

@@ -0,0 +1,347 @@
+/****************************************************
+* PCRE maintainers' helper program: UTF-8 converter *
+****************************************************/
+
+/* This is a test program for converting character code points to UTF-8 and
+vice versa. Note that this program conforms to the original definition of
+UTF-8, which allows codepoints up to 7fffffff. The more recent definition
+limits the validity of Unicode UTF-8 codepoints to a maximum of 10ffffff, and
+forbids the "surrogate" code points. This program now gives warnings for these
+invalid code points.
+
+The arguments are either single code point values written as U+hh.. or 0xhh..
+for conversion to UTF-8, or sequences of hex values, written without 0x and
+optionally including spaces (but such arguments must be quoted), for conversion
+from UTF-8 to codepoints. For example:
+
+./utf8 0x1234
+U+00001234 => e1 88 b4
+
+./utf8 "e1 88 b4"
+U+00001234 <= e1 88 b4
+
+In the second case, a number of UTF-8 characters can be present in one
+argument. In other words, each such argument is interpreted (after ignoring
+spaces) as a string of UTF-8 bytes representing a string of characters:
+
+./utf8 "65 e188b4 77"
+0x00000065 <= 65
+0x00001234 <= e1 88 b4
+0x00000077 <= 77
+
+If the option -s is given, the sequence of UTF-bytes is written out between
+angle brackets at the end of the line. On a UTF-8 terminal, this will show the
+appropriate graphic for the code point.
+
+Errors provoke error messages, but the program carries on with the next
+argument. The return code is always zero.
+
+Philip Hazel
+Original creation data: unknown
+Code extended and tidied to avoid compiler warnings: 26 March 2020
+*/
+
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <ctype.h>
+#include <string.h>
+
+/* The valid ranges for UTF-8 characters are:
+
+0000 0000  to  0000 007f   1 byte (ascii)
+0000 0080  to  0000 07ff   2 bytes
+0000 0800  to  0000 ffff   3 bytes
+0001 0000  to  001f ffff   4 bytes
+0020 0000  to  03ff ffff   5 bytes
+0400 0000  to  7fff ffff   6 bytes
+*/
+
+
+static const unsigned int utf8_table1[] = {
+  0x0000007f, 0x000007ff, 0x0000ffff, 0x001fffff, 0x03ffffff, 0x7fffffff};
+
+static const int utf8_table2[] = {
+  0, 0xc0, 0xe0, 0xf0, 0xf8, 0xfc};
+
+static const int utf8_table3[] = {
+  0xff, 0x1f, 0x0f, 0x07, 0x03, 0x01};
+
+
+/*************************************************
+*       Convert character value to UTF-8         *
+*************************************************/
+
+/* This function takes an unsigned long integer value in the range 0 -
+0x7fffffff and encodes it as a UTF-8 character in 1 to 6 bytes.
+
+Arguments:
+  cvalue     the character value
+  buffer     pointer to buffer for result - at least 6 bytes long
+
+Returns:     number of bytes placed in the buffer
+             0 if input code point is too big
+*/
+
+static size_t
+ord2utf8(unsigned long int cvalue, unsigned char *buffer)
+{
+size_t i, j;
+for (i = 0; i < sizeof(utf8_table1)/sizeof(int); i++)
+  if (cvalue <= utf8_table1[i]) break;
+if (i >= sizeof(utf8_table1)/sizeof(int)) return 0;
+buffer += i;
+for (j = i; j > 0; j--)
+ {
+ *buffer-- = 0x80 | (cvalue & 0x3f);
+ cvalue >>= 6;
+ }
+*buffer = utf8_table2[i] | cvalue;
+return i + 1;
+}
+
+
+
+/*************************************************
+*            Convert UTF-8 string to value       *
+*************************************************/
+
+/* This function takes one or more bytes that represent a UTF-8 character from
+the start of a string of bytes. It returns the value of the character, or the
+offset of a malformation. For an overlong encoding that works but is not the
+correct (shortest) one, the error offset is just after the last byte.
+
+Argument:
+  buffer   a pointer to the byte vector
+  buffend  a pointer to the end of the buffer
+  vptr     a pointer to a variable to receive the value
+  lenptr   a pointer to a variable to receive the offset when error detected
+
+Returns:   > 0 => the number of bytes consumed
+             0 => invalid UTF-8: first byte missing 0x40 bit
+            -1 => invalid UTF-8: first byte has too many high-order 1-bits
+            -2 => incomplete sequence at end of string
+            -3 => incomplete sequence within string
+            -4 => overlong code sequence
+*/
+
+static int
+utf82ord(unsigned char *buffer, unsigned char *buffend,
+  long unsigned int *vptr, int *lenptr)
+{
+unsigned int c = *buffer++;
+unsigned int d = c;
+int i, j, s;
+
+/* Check for an ASCII character, or find the number of additional bytes in a
+multibyte character. */
+
+for (i = -1; i < 6; i++)
+  {
+  if ((d & 0x80) == 0) break;
+  d <<= 1;
+  }
+
+switch (i)
+  {
+  case -1:      /* ASCII character; first byte does not have 0x80 bit */
+  *vptr = c;
+  return 1;
+
+  case 0:       /* First byte has 0x80 but is missing 0x40 bit */
+  *lenptr = 0;
+  return 0;
+
+  case 6:
+  *lenptr = 0;  /* Too many high bits */
+  return -1;
+
+  default:
+  break;
+  }
+
+/* i now has a value in the range 1-5 */
+
+s = 6*i;
+d = (c & utf8_table3[i]) << s;
+
+for (j = 0; j < i; j++)
+  {
+  if (buffer >= buffend)
+    {
+    *lenptr = j + 1;
+    return -2;
+    }
+  c = *buffer++;
+  if ((c & 0xc0) != 0x80)
+    {
+    *lenptr = j + 1;
+    return -3;
+    }
+  s -= 6;
+  d |= (c & 0x3f) << s;
+  }
+
+/* Valid UTF-8 syntax */
+
+*vptr = d;
+
+/* Check that encoding was the correct one, not overlong */
+
+for (j = 0; j < (int)(sizeof(utf8_table1)/sizeof(int)); j++)
+  if (d <= utf8_table1[j]) break;
+if (j != i)
+  {
+  *lenptr = i + 1;
+  return -4;
+  }
+
+/* Valid value */
+
+return i + 1;
+}
+
+
+
+/*************************************************
+*                 Main Program                   *
+*************************************************/
+
+int
+main(int argc, char **argv)
+{
+int i = 1;
+int show = 0;
+unsigned char buffer[64];
+
+if (argc > 1 && strcmp(argv[1], "-s") == 0)
+  {
+  show = 1;
+  i = 2;
+  }
+
+for (; i < argc; i++)
+  {
+  char *x = argv[i];
+  char *endptr;
+  if (strncmp(x, "0x", 2) == 0 || strncmp(x, "U+", 2) == 0)
+    {
+    size_t rc, j;
+    unsigned long int d = strtoul(x+2, &endptr, 16);
+    if (*endptr != 0)
+      {
+      printf("** Invalid hex number %s\n", x);
+      continue;   /* With next argument */
+      }
+    rc = ord2utf8(d, buffer);
+    printf("U+%08lx => ", d);
+    if (rc == 0)
+      printf("** Code point greater than 0x7fffffff cannot be encoded");
+    else
+      {
+      for (j = 0; j < rc; j++) printf("%02x ", buffer[j]);
+      if (show)
+        {
+        printf(">");
+        for (j = 0; j < rc; j++) printf("%c", buffer[j]);
+        printf("< ");
+        }
+      if (d >= 0xd800 && d <= 0xdfff)
+        printf("** Invalid Unicode (surrogate)");
+      else if (d > 0x10ffff)
+        printf("** Invalid Unicode (greater than U+10ffff)");
+      }
+    printf("\n");
+    }
+  else
+    {
+    unsigned char *bptr;
+    unsigned char *buffend;
+    int len = 0;
+    int y = 0;
+    int z = 0;
+
+    for (;;)
+      {
+      while (*x == ' ') x++;
+      if (*x == 0 && !z) break;
+      if (!isxdigit(*x))
+        {
+        printf("** Malformed hex string: %s\n", argv[i]);
+        len = -1;
+        break;
+        }
+      y = y * 16 + tolower(*x) - ((isdigit(*x))? '0' : 'W');
+      x++;
+      if (z)
+        {
+        buffer[len++] = y;
+        y = 0;
+        }
+      z ^= 1;
+      }
+
+    if (len < 0) continue;  /* With next argument after malformation */
+
+    bptr = buffer;
+    buffend = buffer + len;
+
+    while (bptr < buffend)
+      {
+      unsigned long int d;
+      int j;
+      int offset;
+      int rc = utf82ord(bptr, buffend, &d, &offset);
+
+      if (rc > 0)
+        {
+        printf("U+%08lx <= ", d);
+        for (j = 0; j < rc; j++) printf("%02x ", bptr[j]);
+        if (show)
+          {
+          printf(">");
+          for (j = 0; j < rc; j++) printf("%c", bptr[j]);
+          printf("<");
+          }
+        printf("\n");
+        bptr += rc;
+        }
+      else if (rc == -4)
+        {
+        printf("U+%08lx <= ", d);
+        for (j = 0; j < offset; j++) printf("%02x ", bptr[j]);
+        printf("** Overlong UTF-8 sequence\n");
+        bptr += offset;
+        }
+      else
+        {
+        switch (rc)
+          {
+          case 0:  printf("** First byte missing 0x40 bit");
+          break;
+
+          case -1: printf("** First byte has too many high-order bits");
+          break;
+
+          case -2: printf("** Incomplete UTF-8 sequence at end of string");
+          break;
+
+          case -3: printf("** Incomplete UTF-8 sequence");
+          break;
+
+          default: printf("** Unexpected return %d from utf82ord()", rc);
+          break;
+          }
+        printf(" at offset %d in string ", offset);
+        while (bptr < buffend) printf("%02x ", *bptr++);
+        printf("\n");
+        break;
+        }
+      }
+    }
+  }
+
+return 0;
+}
+
+/* End */