[Freeciv-Dev] (PR#1824) ruleset data is in incompatible charsets

["Jason Short via RT" <rt@xxxxxxxxxxxxxx>]

[rfalke - Wed Jan 29 17:37:32 2003]:

> I'm against adding this patch to CVS HEAD. Stable is ok and useful.

I have updated the patch for S1_14, and made a couple of slight changes
based on feedback from Davide (whose system behaves very differently
from mine).

jason

Index: acconfig.old
===================================================================
RCS file: /home/freeciv/CVS/freeciv/acconfig.old,v
retrieving revision 1.1.2.1
diff -u -r1.1.2.1 acconfig.old
--- acconfig.old        2002/10/19 13:31:30     1.1.2.1
+++ acconfig.old        2003/01/31 00:06:41
@@ -26,6 +26,8 @@
 #undef IS_DEVEL_VERSION
 #undef IS_BETA_VERSION
 #undef VERSION_STRING
+#undef HAVE_ICONV
+#undef ICONV_CONST
 #undef HAVE_LIBICE
 #undef HAVE_LIBSM
 #undef HAVE_LIBX11
Index: configure.ac
===================================================================
RCS file: /home/freeciv/CVS/freeciv/configure.ac,v
retrieving revision 1.22.2.4
diff -u -r1.22.2.4 configure.ac
--- configure.ac        2003/01/17 10:07:48     1.22.2.4
+++ configure.ac        2003/01/31 00:06:42
@@ -296,6 +296,10 @@
   CFLAGS="$CFLAGS -fnative-struct"
 fi
 
+dnl Check for libiconv (which is usually included in glibc, but may be
+dnl distributed separately).
+AM_ICONV
+
 dnl Check and choose clients
 if test x$client != xno; then
 
Index: configure.in
===================================================================
RCS file: /home/freeciv/CVS/freeciv/configure.in,v
retrieving revision 1.201.2.4
diff -u -r1.201.2.4 configure.in
--- configure.in        2003/01/17 10:07:48     1.201.2.4
+++ configure.in        2003/01/31 00:06:42
@@ -290,6 +290,10 @@
   CFLAGS="$CFLAGS -fnative-struct"
 fi
 
+dnl Check for libiconv (which is usually included in glibc, but may be
+dnl distributed separately).
+AM_ICONV
+
 dnl Check and choose clients
 if test x$client != xno; then
 
Index: common/shared.c
===================================================================
RCS file: /home/freeciv/CVS/freeciv/common/shared.c,v
retrieving revision 1.92.2.1
diff -u -r1.92.2.1 shared.c
--- common/shared.c     2002/11/13 18:47:21     1.92.2.1
+++ common/shared.c     2003/01/31 00:06:44
@@ -28,6 +28,9 @@
 #include <sys/types.h>
 #include <sys/stat.h>
 
+#ifdef HAVE_ICONV
+#include <iconv.h>
+#endif
 #ifdef HAVE_UNISTD_H
 #include <unistd.h>
 #endif
@@ -1222,4 +1225,110 @@
     vec2++;
   }
   return FALSE;
+}
+
+#ifdef HAVE_ICONV
+/***************************************************************************
+  Convert the text.  This assumes 'from' is an 8-bit charset.
+***************************************************************************/
+static char *convert_string_malloc(const char *text,
+                                  const char *from, const char *to)
+{
+  iconv_t cd = iconv_open(to, from);
+  size_t from_len = strlen(text) + 1, to_len = from_len;
+  char *result;
+
+  if (cd == (iconv_t) (-1)) {
+    freelog(LOG_ERROR,
+           _("String conversion from %s not possible.  You may\n"
+             "want to set your local encoding manually by setting\n"
+             "the environment variable $FREECIV_LOCAL_ENCODING."
+             "Proceeding anyway..."),
+           from);
+    return mystrdup(text); /* The best we can do? */
+  }
+
+  do {
+    size_t flen = from_len, tlen = to_len, res;
+    const char *mytext = text;
+    char *myresult;
+
+    result = fc_malloc(to_len);
+
+    myresult = result;
+
+    /* Since we may do multiple translations, we may need to reset iconv
+     * in between. */
+    iconv(cd, NULL, NULL, NULL, NULL);
+
+    res = iconv(cd, (ICONV_CONST char **)&mytext, &flen, &myresult, &tlen);
+    if (res == (size_t) (-1)) {
+      if (errno != E2BIG) {
+       /* Invalid input. */
+       freelog(LOG_ERROR,
+               _("The string '%s' is not valid: %s. Ruleset files must\n"
+                 "be encoded as %s; you can change this by setting\n"
+                 "$FREECIV_DATA_ENCODING."),
+               text, strerror(errno), from);
+       free(result);
+       iconv_close(cd);
+       return mystrdup(text); /* The best we can do? */
+      }
+    } else {
+      /* Success. */
+      iconv_close(cd);
+
+      /* There may be wasted space here.  But we don't want to call
+       * mystrdup on result since it might not be in an 8-bit charset. */
+      return result;
+    }
+
+    /* Not enough space; try again. */
+    free(result);
+    to_len *= 2;
+  } while (TRUE);
+}
+#endif
+
+/***************************************************************************
+  We convert from the charset used by the rulesets into the local encoding.
+***************************************************************************/
+char *convert_data_string_malloc(const char *text)
+{
+#ifdef HAVE_ICONV
+  char *local_encoding;
+  char *data_encoding;
+
+  data_encoding = getenv("FREECIV_DATA_ENCODING");
+  if (!data_encoding) {
+    /* Currently the rulesets are in latin1 (ISO-8859-1). */
+    data_encoding = "ISO-8859-1";
+  }
+
+  local_encoding = getenv("FREECIV_LOCAL_ENCODING");
+  if (!local_encoding) {
+    /* This converts into the current 8-bit encoding,
+     * with transliteration. */
+    iconv_t cd;
+
+    cd = iconv_open(data_encoding, "");
+    if (cd == (iconv_t)-1) {
+      /* HACK: on some systems "" doesn't work,
+       *  so we use "char" instead and hope... */
+      local_encoding = "char//TRANSLIT";
+    } else {
+      local_encoding = "//TRANSLIT";
+      iconv_close(cd);
+    }
+  }
+
+  return convert_string_malloc(text, data_encoding, local_encoding);
+#else
+  freelog(LOG_ERROR,
+         _("You are running Freeciv without using iconv.  Unless\n"
+           "you are using the latin1 character set, some characters\n"
+           "may not be displayed properly.  You can download iconv\n"
+           "at http://gnu.org/.";));
+  return mystrdup(text);
+#endif
 }
Index: common/shared.h
===================================================================
RCS file: /home/freeciv/CVS/freeciv/common/shared.h,v
retrieving revision 1.104
diff -u -r1.104 shared.h
--- common/shared.h     2002/10/09 20:54:20     1.104
+++ common/shared.h     2003/01/31 00:06:44
@@ -178,4 +178,6 @@
 
 const char *freeciv_motto(void);
 
+char *convert_data_string_malloc(const char *text);
+
 #endif  /* FC__SHARED_H */
Index: server/ruleset.c
===================================================================
RCS file: /home/freeciv/CVS/freeciv/server/ruleset.c,v
retrieving revision 1.120
diff -u -r1.120 ruleset.c
--- server/ruleset.c    2002/09/27 12:32:46     1.120
+++ server/ruleset.c    2003/01/31 00:06:47
@@ -1945,7 +1945,7 @@
       } /* if (!next) */
     } /* if (name) */
     remove_leading_trailing_spaces(cities[j]);
-    city_names[j].name = mystrdup(cities[j]);
+    city_names[j].name = convert_data_string_malloc(cities[j]);
     if (check_name(city_names[j].name)) {
       /* The ruleset contains a name that is too long.  This shouldn't
         happen - if it does, the author should get immediate feedback */
@@ -1997,7 +1997,7 @@
     }
     pl->leader_count = dim;
     for(j = 0; j < dim; j++) {
-      pl->leader_name[j] = mystrdup(leaders[j]);
+      pl->leader_name[j] = convert_data_string_malloc(leaders[j]);
       if (check_name(leaders[j])) {
        pl->leader_name[j][MAX_LEN_NAME - 1] = 0;
       }