rxvt-unicode/src/encoding.C

/*----------------------------------------------------------------------*
 * File:        encoding.C
 *----------------------------------------------------------------------*
 *
 * All portions of code are copyright by their respective author/s.
 * Copyright (c) 2003-2006 Marc Lehmann <schmorp@schmorp.de>
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
 *----------------------------------------------------------------------*/

#include "../config.h"

#include "encoding.h"
#include "rxvtutil.h"

#include <stdlib.h>
#include <string.h>

static const struct n2cs {
  const char *name;
  codeset cs;
} n2cs[] = {
  /* first one found is the normalized one */
  { "ISO88591",         CS_ISO8859_1        },
  { "ISO8859PRIMARY",   CS_ISO8859_1        }, // some stupid fonts use this (hi tigert)
  { "ISO88592",         CS_ISO8859_2        },
  { "ISO88593",         CS_ISO8859_3        },
  { "ISO88594",         CS_ISO8859_4        },
  { "ISO88595",         CS_ISO8859_5        },
  { "ISO88596",         CS_ISO8859_6        },
  { "ISO88597",         CS_ISO8859_7        },
  { "ISO88598",         CS_ISO8859_8        },
  { "ISO88599",         CS_ISO8859_9        },
  { "ISO885910",        CS_ISO8859_10       },
  { "ISO885911",        CS_ISO8859_11       },
  { "ISO885913",        CS_ISO8859_13       },
  { "ISO885914",        CS_ISO8859_14       },
  { "ISO885915",        CS_ISO8859_15       },
  { "FCD885915",        CS_ISO8859_15       },
  { "ISO885916",        CS_ISO8859_16       },

  { "TIS620*",          CS_ISO8859_11       }, // close enough

  { "ISO10646*",        CS_UNICODE          },
  { "UNICODE",          CS_UNICODE          },
  { "UTF8",             CS_UNICODE          },

  { "ASCII",            CS_US_ASCII         },
  { "USASCII",          CS_US_ASCII         },
  { "ANSIX341968",      CS_US_ASCII         },
  { "ISO6461991IRV",    CS_US_ASCII         }, // older versions used the currency sign

  { "KOI8R*",           CS_KOI8_R           },
  { "GOST1976874*",     CS_KOI8_R           },
  { "KOI8RU",           CS_KOI8_U           },
  { "KOI8U",            CS_KOI8_U           },

  { "VISCII*",          CS_VISCII           },

  { "JISX0201*",        CS_JIS0201_1976_0   },
  { "JISC6226*",        CS_JIS0208_1990_0   }, // also wrongly matches -1987-0? (check Encode::JP)
  { "JISX0208*",        CS_JIS0208_1990_0   }, // also wrongly matches -1987-0? (check Encode::JP)
  { "JISX0212*",        CS_JIS0212_1990_0   },
  { "JISX021320001",    CS_JIS0213_1        },
  { "JISX021320002",    CS_JIS0213_2        },
  { "JISX0221*",        CS_UNICODE          }, // _very_ close

  { "KSC5601*",         CS_KSC5601_1987_0   },
  { "KSX1001*",         CS_KSC5601_1987_0   },
  { "KSC5700*",         CS_UNICODE          }, // unicode plus extensions

  { "BIG5P*",           CS_BIG5_PLUS        },
  { "BIG5ETEN*",        CS_BIG5_EXT         },
  { "BIG5*",            CS_BIG5             },
  { "GB2312*",          CS_GB2312_1980_0    },
  { "GBK*",             CS_GBK_0            },
  { "GB6345*",          CS_GB2312_1980_0    }, // slightly different to gb2312??
  { "GB8565*",          CS_GB2312_1980_0    }, // a superset of gb2312??
  { "GB13000*",         CS_UNICODE          },
  { "CNS1164319921",    CS_CNS11643_1992_1  },
  { "CNS1164319922",    CS_CNS11643_1992_2  },
  { "CNS1164319923",    CS_CNS11643_1992_3  },
  { "CNS1164319924",    CS_CNS11643_1992_4  },
  { "CNS1164319925",    CS_CNS11643_1992_5  },
  { "CNS1164319926",    CS_CNS11643_1992_6  },
  { "CNS1164319927",    CS_CNS11643_1992_7  },
  { "CNS116431992F",    CS_CNS11643_1992_F  },

  { 0,                  CS_UNKNOWN      }
};

static const char *
normalize_name (const char *name)
{
  static char res[16];
  char *r;

  for (r = res; *name && r < res + 15; name++)
    if ((*name >= '0' && *name <= '9')
        || (*name >= 'A' && *name <= 'Z'))
      *r++ = *name;
    else if (*name >= 'a' && *name <= 'z')
      *r++ = *name - ('a' - 'A');

  *r = 0;

  return res;
}

codeset
codeset_from_name (const char *name)
{
  if (!name)
    return CS_UNKNOWN;

  name = normalize_name (name);

  const struct n2cs *i = n2cs;

  do {
    int len = strlen (i->name);

    if ((i->name[len - 1] == '*'
         && !strncmp (name, i->name, len - 1))
        || !strcmp (name, i->name))
        return i->cs;

  } while ((++i)->name);

  return CS_UNKNOWN;
}

static unicode_t cs_unknown_to_unicode (uint32_t enc)          { return NOCHAR; }
static uint32_t cs_unknown_from_unicode (unicode_t unicode)    { return NOCHAR; }

static unicode_t cs_unicode_to_unicode (uint32_t enc)          { return enc; }
static uint32_t cs_unicode_from_unicode (unicode_t unicode)    { return unicode; }

#define cs_us_ascii_to_unicode cs_unicode_to_unicode
static uint32_t cs_us_ascii_from_unicode (unicode_t unicode)   { return unicode <= 127 ? unicode : NOCHAR; }

#define cs_us_ascii_to_unicode_16 cs_unicode_to_unicode
static uint32_t cs_unicode_16_from_unicode (unicode_t unicode) { return unicode <= 65535 ? unicode : NOCHAR; }

#define ENCODING_DEFAULT

#include "table/iso8859_1.h"
#include "table/iso8859_15.h"

//#define ENCODING_EU

#include "table/iso8859_2.h"
#include "table/iso8859_3.h"
#include "table/iso8859_4.h"
#include "table/iso8859_5.h"
#include "table/iso8859_6.h"
#include "table/iso8859_7.h"
#include "table/iso8859_8.h"
#include "table/iso8859_9.h"
#include "table/iso8859_10.h"
#include "table/iso8859_11.h"
#include "table/iso8859_13.h"
#include "table/iso8859_14.h"
#include "table/iso8859_16.h"

#include "table/koi8_r.h"
#include "table/koi8_u.h"

//#define ENCODING_KR

#include "table/ksc5601_1987_0.h"

//#define ENCODING_ZH

#include "table/big5.h"
#include "table/gbk_0.h"
#include "table/gb2312_1980_0.h"

//#define ENCODING_ZH_EXT

#include "table/cns11643_1992_1.h"
#include "table/cns11643_1992_2.h"
#include "table/cns11643_1992_3.h"
#include "table/cns11643_1992_4.h"
#include "table/cns11643_1992_5.h"
#include "table/cns11643_1992_6.h"
#include "table/cns11643_1992_7.h"
#include "table/cns11643_1992_f.h"
#include "table/big5_ext.h"
#include "table/big5_plus.h"

//#define ENCODING_VN

#include "table/viscii.h"

//#define ENCODING_JP

#include "table/jis0201_1976_0.h"
#include "table/jis0208_1990_0.h"
#include "table/jis0212_1990_0.h"

//#define ENCODING_JP_EXT

#include "table/jis0213_1.h"
#include "table/jis0213_2.h"

#if ENCODING_TO_UNICODE
# define ENC(base) { cs_ ## base ## _from_unicode, cs_ ## base ## _to_unicode }
#else
# define ENC(base) { cs_ ## base ## _from_unicode }
#endif


// order must match table in encoding.h(!)
const rxvt_codeset_conv rxvt_codeset[NUM_CODESETS] = {
  ENC (unknown),

  ENC (us_ascii),

  ENC (iso8859_1),
  ENC (iso8859_2),
  ENC (iso8859_3),
  ENC (iso8859_4),
  ENC (iso8859_5),
  ENC (iso8859_6),
  ENC (iso8859_7),
  ENC (iso8859_8),
  ENC (iso8859_9),
  ENC (iso8859_10),
  ENC (iso8859_11),
  ENC (iso8859_13),
  ENC (iso8859_14),
  ENC (iso8859_15),
  ENC (iso8859_16),

  ENC (koi8_r),
  ENC (koi8_u),

  ENC (jis0201_1976_0),
  ENC (jis0208_1990_0),
  ENC (jis0212_1990_0),

  ENC (jis0213_1),
  ENC (jis0213_2),

  ENC (ksc5601_1987_0),

  ENC (gb2312_1980_0),
  ENC (gbk_0),

  ENC (cns11643_1992_1),
  ENC (cns11643_1992_2),
  ENC (cns11643_1992_3),
  ENC (cns11643_1992_4),
  ENC (cns11643_1992_5),
  ENC (cns11643_1992_6),
  ENC (cns11643_1992_7),
  ENC (cns11643_1992_f),
  ENC (big5),
  ENC (big5_ext),
  ENC (big5_plus),

  ENC (viscii),

  ENC (unicode_16),
  ENC (unicode),
};

#if ENABLE_COMBINING
# define ENCODING_COMPOSE
#endif

#include "table/compose.h"

unicode_t
rxvt_compose (unicode_t c1, unicode_t c2)
{
  int l = 0;
  int r = ecb_array_length (rxvt_compose_table) - 1;
  int m;

  while (r >= l)
    {
      m = (l + r) / 2;
      rxvt_compose_entry &c = rxvt_compose_table[m];

      if (c.c1 < c1 || (c.c1 == c1 && c.c2 < c2))
        l = m + 1;
      else if (c.c1 > c1 || (c.c1 == c1 && c.c2 > c2))
        r = m - 1;
      else
        return c.r;
    }

  return NOCHAR;
}

#include "table/category.h"

bool unicode::is_space (unicode_t c)
{
  return IS_SPACE (c)
         || c == 0x09; // exclude tabs, too, as we store them in the buffer
}
Revision:	1.34
Committed:	Thu May 22 18:54:32 2014 UTC (9 years, 11 months ago) by root
Content type:	text/plain
Branch:	MAIN
CVS Tags:	rxvt-unicode-rel-9_29, rxvt-unicode-rel-9_26, rxvt-unicode-rel-9_25, rxvt-unicode-rel-9_22, rxvt-unicode-rel-9_21, rxvt-unicode-rel-9_30, HEAD
Changes since 1.33:	+1 -1 lines
Log Message:	GPLv3
#	Content
1	/----------------------------------------------------------------------
2	* File: encoding.C
3	----------------------------------------------------------------------
4	*
5	* All portions of code are copyright by their respective author/s.
6	* Copyright (c) 2003-2006 Marc Lehmann <schmorp@schmorp.de>
7	*
8	* This program is free software; you can redistribute it and/or modify
9	* it under the terms of the GNU General Public License as published by
10	* the Free Software Foundation; either version 3 of the License, or
11	* (at your option) any later version.
12	*
13	* This program is distributed in the hope that it will be useful,
14	* but WITHOUT ANY WARRANTY; without even the implied warranty of
15	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16	* GNU General Public License for more details.
17	*
18	* You should have received a copy of the GNU General Public License
19	* along with this program; if not, write to the Free Software
20	* Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
21	----------------------------------------------------------------------/
22
23	#include "../config.h"
24
25	#include "encoding.h"
26	#include "rxvtutil.h"
27
28	#include <stdlib.h>
29	#include <string.h>
30
31	static const struct n2cs {
32	const char *name;
33	codeset cs;
34	} n2cs[] = {
35	/* first one found is the normalized one */
36	{ "ISO88591", CS_ISO8859_1 },
37	{ "ISO8859PRIMARY", CS_ISO8859_1 }, // some stupid fonts use this (hi tigert)
38	{ "ISO88592", CS_ISO8859_2 },
39	{ "ISO88593", CS_ISO8859_3 },
40	{ "ISO88594", CS_ISO8859_4 },
41	{ "ISO88595", CS_ISO8859_5 },
42	{ "ISO88596", CS_ISO8859_6 },
43	{ "ISO88597", CS_ISO8859_7 },
44	{ "ISO88598", CS_ISO8859_8 },
45	{ "ISO88599", CS_ISO8859_9 },
46	{ "ISO885910", CS_ISO8859_10 },
47	{ "ISO885911", CS_ISO8859_11 },
48	{ "ISO885913", CS_ISO8859_13 },
49	{ "ISO885914", CS_ISO8859_14 },
50	{ "ISO885915", CS_ISO8859_15 },
51	{ "FCD885915", CS_ISO8859_15 },
52	{ "ISO885916", CS_ISO8859_16 },
53
54	{ "TIS620*", CS_ISO8859_11 }, // close enough
55
56	{ "ISO10646*", CS_UNICODE },
57	{ "UNICODE", CS_UNICODE },
58	{ "UTF8", CS_UNICODE },
59
60	{ "ASCII", CS_US_ASCII },
61	{ "USASCII", CS_US_ASCII },
62	{ "ANSIX341968", CS_US_ASCII },
63	{ "ISO6461991IRV", CS_US_ASCII }, // older versions used the currency sign
64
65	{ "KOI8R*", CS_KOI8_R },
66	{ "GOST1976874*", CS_KOI8_R },
67	{ "KOI8RU", CS_KOI8_U },
68	{ "KOI8U", CS_KOI8_U },
69
70	{ "VISCII*", CS_VISCII },
71
72	{ "JISX0201*", CS_JIS0201_1976_0 },
73	{ "JISC6226*", CS_JIS0208_1990_0 }, // also wrongly matches -1987-0? (check Encode::JP)
74	{ "JISX0208*", CS_JIS0208_1990_0 }, // also wrongly matches -1987-0? (check Encode::JP)
75	{ "JISX0212*", CS_JIS0212_1990_0 },
76	{ "JISX021320001", CS_JIS0213_1 },
77	{ "JISX021320002", CS_JIS0213_2 },
78	{ "JISX0221*", CS_UNICODE }, // _very_ close
79
80	{ "KSC5601*", CS_KSC5601_1987_0 },
81	{ "KSX1001*", CS_KSC5601_1987_0 },
82	{ "KSC5700*", CS_UNICODE }, // unicode plus extensions
83
84	{ "BIG5P*", CS_BIG5_PLUS },
85	{ "BIG5ETEN*", CS_BIG5_EXT },
86	{ "BIG5*", CS_BIG5 },
87	{ "GB2312*", CS_GB2312_1980_0 },
88	{ "GBK*", CS_GBK_0 },
89	{ "GB6345*", CS_GB2312_1980_0 }, // slightly different to gb2312??
90	{ "GB8565*", CS_GB2312_1980_0 }, // a superset of gb2312??
91	{ "GB13000*", CS_UNICODE },
92	{ "CNS1164319921", CS_CNS11643_1992_1 },
93	{ "CNS1164319922", CS_CNS11643_1992_2 },
94	{ "CNS1164319923", CS_CNS11643_1992_3 },
95	{ "CNS1164319924", CS_CNS11643_1992_4 },
96	{ "CNS1164319925", CS_CNS11643_1992_5 },
97	{ "CNS1164319926", CS_CNS11643_1992_6 },
98	{ "CNS1164319927", CS_CNS11643_1992_7 },
99	{ "CNS116431992F", CS_CNS11643_1992_F },
100
101	{ 0, CS_UNKNOWN }
102	};
103
104	static const char *
105	normalize_name (const char *name)
106	{
107	static char res[16];
108	char *r;
109
110	for (r = res; *name && r < res + 15; name++)
111	if ((name >= '0' && name <= '9')
112	\|\| (name >= 'A' && name <= 'Z'))
113	r++ = name;
114	else if (name >= 'a' && name <= 'z')
115	r++ = name - ('a' - 'A');
116
117	*r = 0;
118
119	return res;
120	}
121
122	codeset
123	codeset_from_name (const char *name)
124	{
125	if (!name)
126	return CS_UNKNOWN;
127
128	name = normalize_name (name);
129
130	const struct n2cs *i = n2cs;
131
132	do {
133	int len = strlen (i->name);
134
135	if ((i->name[len - 1] == '*'
136	&& !strncmp (name, i->name, len - 1))
137	\|\| !strcmp (name, i->name))
138	return i->cs;
139
140	} while ((++i)->name);
141
142	return CS_UNKNOWN;
143	}
144
145	static unicode_t cs_unknown_to_unicode (uint32_t enc) { return NOCHAR; }
146	static uint32_t cs_unknown_from_unicode (unicode_t unicode) { return NOCHAR; }
147
148	static unicode_t cs_unicode_to_unicode (uint32_t enc) { return enc; }
149	static uint32_t cs_unicode_from_unicode (unicode_t unicode) { return unicode; }
150
151	#define cs_us_ascii_to_unicode cs_unicode_to_unicode
152	static uint32_t cs_us_ascii_from_unicode (unicode_t unicode) { return unicode <= 127 ? unicode : NOCHAR; }
153
154	#define cs_us_ascii_to_unicode_16 cs_unicode_to_unicode
155	static uint32_t cs_unicode_16_from_unicode (unicode_t unicode) { return unicode <= 65535 ? unicode : NOCHAR; }
156
157	#define ENCODING_DEFAULT
158
159	#include "table/iso8859_1.h"
160	#include "table/iso8859_15.h"
161
162	//#define ENCODING_EU
163
164	#include "table/iso8859_2.h"
165	#include "table/iso8859_3.h"
166	#include "table/iso8859_4.h"
167	#include "table/iso8859_5.h"
168	#include "table/iso8859_6.h"
169	#include "table/iso8859_7.h"
170	#include "table/iso8859_8.h"
171	#include "table/iso8859_9.h"
172	#include "table/iso8859_10.h"
173	#include "table/iso8859_11.h"
174	#include "table/iso8859_13.h"
175	#include "table/iso8859_14.h"
176	#include "table/iso8859_16.h"
177
178	#include "table/koi8_r.h"
179	#include "table/koi8_u.h"
180
181	//#define ENCODING_KR
182
183	#include "table/ksc5601_1987_0.h"
184
185	//#define ENCODING_ZH
186
187	#include "table/big5.h"
188	#include "table/gbk_0.h"
189	#include "table/gb2312_1980_0.h"
190
191	//#define ENCODING_ZH_EXT
192
193	#include "table/cns11643_1992_1.h"
194	#include "table/cns11643_1992_2.h"
195	#include "table/cns11643_1992_3.h"
196	#include "table/cns11643_1992_4.h"
197	#include "table/cns11643_1992_5.h"
198	#include "table/cns11643_1992_6.h"
199	#include "table/cns11643_1992_7.h"
200	#include "table/cns11643_1992_f.h"
201	#include "table/big5_ext.h"
202	#include "table/big5_plus.h"
203
204	//#define ENCODING_VN
205
206	#include "table/viscii.h"
207
208	//#define ENCODING_JP
209
210	#include "table/jis0201_1976_0.h"
211	#include "table/jis0208_1990_0.h"
212	#include "table/jis0212_1990_0.h"
213
214	//#define ENCODING_JP_EXT
215
216	#include "table/jis0213_1.h"
217	#include "table/jis0213_2.h"
218
219	#if ENCODING_TO_UNICODE
220	# define ENC(base) { cs_ ## base ## _from_unicode, cs_ ## base ## _to_unicode }
221	#else
222	# define ENC(base) { cs_ ## base ## _from_unicode }
223	#endif
224
225
226	// order must match table in encoding.h(!)
227	const rxvt_codeset_conv rxvt_codeset[NUM_CODESETS] = {
228	ENC (unknown),
229
230	ENC (us_ascii),
231
232	ENC (iso8859_1),
233	ENC (iso8859_2),
234	ENC (iso8859_3),
235	ENC (iso8859_4),
236	ENC (iso8859_5),
237	ENC (iso8859_6),
238	ENC (iso8859_7),
239	ENC (iso8859_8),
240	ENC (iso8859_9),
241	ENC (iso8859_10),
242	ENC (iso8859_11),
243	ENC (iso8859_13),
244	ENC (iso8859_14),
245	ENC (iso8859_15),
246	ENC (iso8859_16),
247
248	ENC (koi8_r),
249	ENC (koi8_u),
250
251	ENC (jis0201_1976_0),
252	ENC (jis0208_1990_0),
253	ENC (jis0212_1990_0),
254
255	ENC (jis0213_1),
256	ENC (jis0213_2),
257
258	ENC (ksc5601_1987_0),
259
260	ENC (gb2312_1980_0),
261	ENC (gbk_0),
262
263	ENC (cns11643_1992_1),
264	ENC (cns11643_1992_2),
265	ENC (cns11643_1992_3),
266	ENC (cns11643_1992_4),
267	ENC (cns11643_1992_5),
268	ENC (cns11643_1992_6),
269	ENC (cns11643_1992_7),
270	ENC (cns11643_1992_f),
271	ENC (big5),
272	ENC (big5_ext),
273	ENC (big5_plus),
274
275	ENC (viscii),
276
277	ENC (unicode_16),
278	ENC (unicode),
279	};
280
281	#if ENABLE_COMBINING
282	# define ENCODING_COMPOSE
283	#endif
284
285	#include "table/compose.h"
286
287	unicode_t
288	rxvt_compose (unicode_t c1, unicode_t c2)
289	{
290	int l = 0;
291	int r = ecb_array_length (rxvt_compose_table) - 1;
292	int m;
293
294	while (r >= l)
295	{
296	m = (l + r) / 2;
297	rxvt_compose_entry &c = rxvt_compose_table[m];
298
299	if (c.c1 < c1 \|\| (c.c1 == c1 && c.c2 < c2))
300	l = m + 1;
301	else if (c.c1 > c1 \|\| (c.c1 == c1 && c.c2 > c2))
302	r = m - 1;
303	else
304	return c.r;
305	}
306
307	return NOCHAR;
308	}
309
310	#include "table/category.h"
311
312	bool unicode::is_space (unicode_t c)
313	{
314	return IS_SPACE (c)
315	\|\| c == 0x09; // exclude tabs, too, as we store them in the buffer
316	}