[Freeciv-Dev] Re: (PR#1824) ruleset data is in incompatible charsets

["Davide Pagnin via RT" <rt@xxxxxxxxxxxxxx>]

On Fri, 2003-01-31 at 08:42, Jason Short via RT wrote:
> [jdorje - Fri Jan 31 07:41:30 2003]:
> 
> > Thus the patch becomes what it is here.  Note that transliteration is
> > necessary, since the latin1 characters may very well not be available
> > in
> > the local encoding.
> 
> Sigh.  All these patches are confusing.  _Here_ is the correct one.
> 
> jason
> 
> 
> ______________________________________________________________________
I've updated jason patch to reflex libcharset consideration I've made on
PR#2559 and that are also related to this topic.

As Jason pointed out, this patch is more a band aid than a proper fix,
but we can use a better band aid, no?

        Ciao, Davide

diff -urN -Xfreeciv/diff_ignore freeciv/Makefile.am freeciv-iconv/Makefile.am
--- freeciv/Makefile.am Tue Nov 12 15:09:23 2002
+++ freeciv-iconv/Makefile.am   Sun Feb  2 16:05:10 2003
@@ -65,7 +65,7 @@
                debian/rules                    \
                debian/watch                    \
                m4/ac_path_lib.m4               \
-               m4/codeset.m4                   \
+               m4/locale.m4                    \
                m4/debug.m4                     \
                m4/esd.m4                       \
                m4/gettext.m4                   \
diff -urN -Xfreeciv/diff_ignore freeciv/acconfig.old freeciv-iconv/acconfig.old
--- freeciv/acconfig.old        Tue Nov 12 15:09:23 2002
+++ freeciv-iconv/acconfig.old  Sun Feb  2 16:03:29 2003
@@ -26,6 +26,10 @@
 #undef IS_DEVEL_VERSION
 #undef IS_BETA_VERSION
 #undef VERSION_STRING
+#undef HAVE_ICONV
+#undef ICONV_CONST
+#undef HAVE_LIBCHARSET
+#undef HAVE_LANGINFO_CODESET
 #undef HAVE_LIBICE
 #undef HAVE_LIBSM
 #undef HAVE_LIBX11
diff -urN -Xfreeciv/diff_ignore freeciv/common/shared.c 
freeciv-iconv/common/shared.c
--- freeciv/common/shared.c     Wed Nov 13 19:45:09 2002
+++ freeciv-iconv/common/shared.c       Sun Feb  2 16:14:32 2003
@@ -28,6 +28,16 @@
 #include <sys/types.h>
 #include <sys/stat.h>
 
+#ifdef HAVE_ICONV
+#include <iconv.h>
+#ifdef HAVE_LIBCHARSET
+#include <libcharset.h>
+#else
+#ifdef HAVE_LANGINFO_CODESET
+#include <langinfo.h>
+#endif
+#endif
+#endif
 #ifdef HAVE_UNISTD_H
 #include <unistd.h>
 #endif
@@ -1223,3 +1233,109 @@
   }
   return FALSE;
 }
+
+#ifdef HAVE_ICONV
+/***************************************************************************
+  Convert the text.  This assumes 'from' is an 8-bit charset.
+***************************************************************************/
+static char *convert_string_malloc(const char *text,
+                                  const char *from, const char *to)
+{
+  iconv_t cd;
+  size_t from_len = strlen(text) + 1, to_len = from_len;
+  char *result;
+
+  cd = iconv_open(to, from);
+  if (cd == (iconv_t) (-1)) {
+    freelog(LOG_ERROR,
+           _("String conversion from %s not possible.  You may\n"
+             "want to set your local encoding manually by setting\n"
+             "the environment variable $FREECIV_LOCAL_ENCODING."
+             "Proceeding anyway..."),
+           from);
+    return mystrdup(text); /* The best we can do? */
+  }
+
+  do {
+    size_t flen = from_len, tlen = to_len, res;
+    const char *mytext = text;
+    char *myresult;
+
+    result = fc_malloc(to_len);
+
+    myresult = result;
+
+    /* Since we may do multiple translations, we may need to reset iconv
+     * in between. */
+    iconv(cd, NULL, NULL, NULL, NULL);
+
+    res = iconv(cd, (ICONV_CONST char **)&mytext, &flen, &myresult, &tlen);
+    if (res == (size_t) (-1)) {
+      if (errno != E2BIG) {
+       /* Invalid input. */
+       freelog(LOG_ERROR,
+               _("The string '%s' is not valid: %s. Ruleset files must\n"
+                 "be encoded as %s; you can change this by setting\n"
+                 "$FREECIV_DATA_ENCODING."),
+               text, strerror(errno), from);
+       free(result);
+       iconv_close(cd);
+       return mystrdup(text); /* The best we can do? */
+      }
+    } else {
+      /* Success. */
+      iconv_close(cd);
+
+      /* There may be wasted space here.  But we don't want to call
+       * mystrdup on result since it might not be in an 8-bit charset. */
+      return result;
+    }
+
+    /* Not enough space; try again. */
+    free(result);
+    to_len *= 2;
+  } while (TRUE);
+}
+#endif
+
+/***************************************************************************
+  We convert from the charset used by the rulesets into the local encoding.
+***************************************************************************/
+char *convert_data_string_malloc(const char *text)
+{
+#ifdef HAVE_ICONV
+  char *local_encoding;
+  char *data_encoding;
+  char target[128];
+
+  data_encoding = getenv("FREECIV_DATA_ENCODING");
+  if (!data_encoding) {
+    /* Currently the rulesets are in latin1 (ISO-8859-1). */
+    data_encoding = "ISO-8859-1";
+  }
+
+  local_encoding = getenv("FREECIV_LOCAL_ENCODING");
+  if (!local_encoding) {
+#ifdef HAVE_LIBCHARSET
+    local_encoding = locale_charset();
+#else
+#ifdef HAVE_LANGINFO_CODESET
+    local_encoding = nl_langinfo(CODESET);
+#else
+    local_encoding = "";
+#endif
+#endif
+    my_snprintf(target, sizeof(target), "%s//TRANSLIT", local_encoding);
+    local_encoding = target;
+  }
+
+  return convert_string_malloc(text, data_encoding, local_encoding);
+#else
+  freelog(LOG_ERROR,
+         _("You are running Freeciv without using iconv.  Unless\n"
+           "you are using the latin1 character set, some characters\n"
+           "may not be displayed properly.  You can download iconv\n"
+           "at http://gnu.org/.";));
+  return mystrdup(text);
+#endif
+}
diff -urN -Xfreeciv/diff_ignore freeciv/common/shared.h 
freeciv-iconv/common/shared.h
--- freeciv/common/shared.h     Tue Nov 12 15:09:47 2002
+++ freeciv-iconv/common/shared.h       Sun Feb  2 15:42:34 2003
@@ -178,4 +178,6 @@
 
 const char *freeciv_motto(void);
 
+char *convert_data_string_malloc(const char *text);
+
 #endif  /* FC__SHARED_H */
diff -urN -Xfreeciv/diff_ignore freeciv/configure.ac freeciv-iconv/configure.ac
--- freeciv/configure.ac        Fri Jan 17 10:47:31 2003
+++ freeciv-iconv/configure.ac  Sun Feb  2 16:06:02 2003
@@ -296,6 +296,12 @@
   CFLAGS="$CFLAGS -fnative-struct"
 fi
 
+dnl Check for libiconv (which is usually included in glibc, but may be
+dnl distributed separately) and other locale related functions.
+AM_ICONV
+AM_LIBCHARSET
+AM_LANGINFO_CODESET
+
 dnl Check and choose clients
 if test x$client != xno; then
 
diff -urN -Xfreeciv/diff_ignore freeciv/configure.in freeciv-iconv/configure.in
--- freeciv/configure.in        Fri Jan 17 10:47:31 2003
+++ freeciv-iconv/configure.in  Sun Feb  2 15:55:04 2003
@@ -290,6 +290,12 @@
   CFLAGS="$CFLAGS -fnative-struct"
 fi
 
+dnl Check for libiconv (which is usually included in glibc, but may be
+dnl distributed separately) and other locale related functions.
+AM_ICONV
+AM_LIBCHARSET
+AM_LANGINFO_CODESET
+
 dnl Check and choose clients
 if test x$client != xno; then
 
diff -urN -Xfreeciv/diff_ignore freeciv/m4/codeset.m4 
freeciv-iconv/m4/codeset.m4
--- freeciv/m4/codeset.m4       Tue Nov 12 15:09:50 2002
+++ freeciv-iconv/m4/codeset.m4 Thu Jan  1 01:00:00 1970
@@ -1,16 +0,0 @@
-#serial AM1
-dnl From Bruno Haible.
-
-AC_DEFUN([AM_LANGINFO_CODESET],
-[
-  AC_CACHE_CHECK([for nl_langinfo and CODESET], am_cv_langinfo_codeset,
-    [AC_TRY_LINK([#include <langinfo.h>],
-      [char* cs = nl_langinfo(CODESET);],
-      am_cv_langinfo_codeset=yes,
-      am_cv_langinfo_codeset=no)
-    ])
-  if test $am_cv_langinfo_codeset = yes; then
-    AC_DEFINE(HAVE_LANGINFO_CODESET, 1,
-      [Define if you have <langinfo.h> and nl_langinfo(CODESET).])
-  fi
-])
diff -urN -Xfreeciv/diff_ignore freeciv/m4/locale.m4 freeciv-iconv/m4/locale.m4
--- freeciv/m4/locale.m4        Thu Jan  1 01:00:00 1970
+++ freeciv-iconv/m4/locale.m4  Sun Feb  2 16:04:47 2003
@@ -0,0 +1,32 @@
+#serial AM1
+dnl From Bruno Haible.
+
+AC_DEFUN([AM_LANGINFO_CODESET],
+[
+  AC_CACHE_CHECK([for nl_langinfo and CODESET], am_cv_langinfo_codeset,
+    [AC_TRY_LINK([#include <langinfo.h>],
+      [char* cs = nl_langinfo(CODESET);],
+      am_cv_langinfo_codeset=yes,
+      am_cv_langinfo_codeset=no)
+    ])
+  if test $am_cv_langinfo_codeset = yes; then
+    AC_DEFINE(HAVE_LANGINFO_CODESET, 1,
+      [Define if you have <langinfo.h> and nl_langinfo(CODESET).])
+  fi
+])
+AC_DEFUN([AM_LIBCHARSET],
+[
+  AC_CACHE_CHECK([for libcharset], am_cv_libcharset,
+    [lc_save_LIBS="$LIBS"
+     LIBS="$LIBS $LIBICONV"
+     AC_TRY_LINK([#include <libcharset.h>],
+      [locale_charset()],
+      am_cv_libcharset=yes,
+      am_cv_libcharset=no) 
+      LIBS="$lc_save_LIBS" 
+    ])
+  if test $am_cv_libcharset = yes; then
+    AC_DEFINE(HAVE_LIBCHARSET, 1,
+      [Define if you have <libcharset.h> and locale_charset().])
+  fi
+])
diff -urN -Xfreeciv/diff_ignore freeciv/server/ruleset.c 
freeciv-iconv/server/ruleset.c
--- freeciv/server/ruleset.c    Tue Nov 12 15:09:54 2002
+++ freeciv-iconv/server/ruleset.c      Sun Feb  2 15:42:34 2003
@@ -1945,7 +1945,7 @@
       } /* if (!next) */
     } /* if (name) */
     remove_leading_trailing_spaces(cities[j]);
-    city_names[j].name = mystrdup(cities[j]);
+    city_names[j].name = convert_data_string_malloc(cities[j]);
     if (check_name(city_names[j].name)) {
       /* The ruleset contains a name that is too long.  This shouldn't
         happen - if it does, the author should get immediate feedback */
@@ -1997,7 +1997,7 @@
     }
     pl->leader_count = dim;
     for(j = 0; j < dim; j++) {
-      pl->leader_name[j] = mystrdup(leaders[j]);
+      pl->leader_name[j] = convert_data_string_malloc(leaders[j]);
       if (check_name(leaders[j])) {
        pl->leader_name[j][MAX_LEN_NAME - 1] = 0;
       }