[Freeciv-Dev] (PR#1824) ruleset data is in incompatible charsets

["Jason Short" <jdorje@xxxxxxxxxxxxxxxxxxxxx>]

<URL: http://rt.freeciv.org/Ticket/Display.html?id=1824 >

Here's an update of the patch.  The only change was in Makefile.am.

jason

Index: configure.ac
===================================================================
RCS file: /home/freeciv/CVS/freeciv/configure.ac,v
retrieving revision 1.70
diff -u -r1.70 configure.ac
--- configure.ac        21 Aug 2004 16:20:25 -0000      1.70
+++ configure.ac        3 Sep 2004 04:43:47 -0000
@@ -325,6 +325,11 @@
      FTWL_LIBS=`sdl-config --libs`" -lpng "`freetype-config --libs`
 fi
 
+dnl Check for libiconv (which is usually included in glibc, but may be
+dnl distributed separately).
+AM_ICONV
+LIBS="$LIBS $LIBICONV"
+
 dnl Check and choose clients
 if test x$client != xno; then
 
Index: configure.in
===================================================================
RCS file: /home/freeciv/CVS/freeciv/configure.in,v
retrieving revision 1.246
diff -u -r1.246 configure.in
--- configure.in        21 Aug 2004 16:20:25 -0000      1.246
+++ configure.in        3 Sep 2004 04:43:47 -0000
@@ -318,6 +318,11 @@
      FTWL_LIBS=`sdl-config --libs`" -lpng "`freetype-config --libs`
 fi
 
+dnl Check for libiconv (which is usually included in glibc, but may be
+dnl distributed separately).
+AM_ICONV
+LIBS="$LIBS $LIBICONV"
+
 dnl Check and choose clients
 if test x$client != xno; then
 
Index: client/civclient.c
===================================================================
RCS file: /home/freeciv/CVS/freeciv/client/civclient.c,v
retrieving revision 1.193
diff -u -r1.193 civclient.c
--- client/civclient.c  2 Aug 2004 23:19:35 -0000       1.193
+++ client/civclient.c  3 Sep 2004 04:43:47 -0000
@@ -21,7 +21,9 @@
 #include <time.h>
 
 #include "capstr.h"
+#include "dataio.h"
 #include "diptreaty.h"
+#include "fciconv.h"
 #include "fcintl.h"
 #include "game.h"
 #include "idex.h"
@@ -91,6 +93,61 @@
  */
 bool turn_done_sent = FALSE;
 
+/**************************************************************************
+  Convert a text string from the internal to the data encoding, when it
+  is written to the network.
+**************************************************************************/
+static unsigned char *put_conv(const char *src, size_t *length)
+{
+  char *out = internal_to_data_string_malloc(src);
+
+  if (out) {
+    *length = strlen(out);
+    return out;
+  } else {
+    *length = 0;
+    return NULL;
+  }
+}
+
+/**************************************************************************
+  Convert a text string from the data to the internal encoding when it is
+  first read from the network.  Returns FALSE if the destination isn't
+  large enough or the source was bad.
+**************************************************************************/
+static bool get_conv(char *dst, size_t ndst,
+                    const unsigned char *src, size_t nsrc)
+{
+  char *out = data_to_internal_string_malloc(src);
+  bool ret = TRUE;
+  size_t len;
+
+  if (!out) {
+    dst[0] = '\0';
+    return FALSE;
+  }
+
+  len = strlen(out);
+  if (ndst > 0 && len >= ndst) {
+    ret = FALSE;
+    len = ndst - 1;
+  }
+
+  memcpy(dst, out, len);
+  dst[len] = '\0';
+  free(out);
+
+  return ret;
+}
+
+/**************************************************************************
+  Set up charsets for the client.
+**************************************************************************/
+static void charsets_init(void)
+{
+  dio_set_put_conv_callback(put_conv);
+  dio_set_get_conv_callback(get_conv);
+}
 
 /**************************************************************************
 ...
@@ -203,6 +260,7 @@
   conn_list_init(&game.game_connections);
 
   ui_init();
+  charsets_init();
   my_init_network();
   chatline_common_init();
   init_messages_where();
Index: client/gui-gtk/gui_main.c
===================================================================
RCS file: /home/freeciv/CVS/freeciv/client/gui-gtk/gui_main.c,v
retrieving revision 1.150
diff -u -r1.150 gui_main.c
--- client/gui-gtk/gui_main.c   10 Jul 2004 18:48:18 -0000      1.150
+++ client/gui-gtk/gui_main.c   3 Sep 2004 04:43:48 -0000
@@ -31,6 +31,7 @@
 #include <unistd.h>
 #endif
 
+#include "fciconv.h"
 #include "fcintl.h"
 #include "game.h"
 #include "government.h"
@@ -803,6 +804,7 @@
 **************************************************************************/
 void ui_init(void)
 {
+  init_character_encodings(NULL);
 }
 
 /**************************************************************************
Index: client/gui-gtk-2.0/connectdlg.c
===================================================================
RCS file: /home/freeciv/CVS/freeciv/client/gui-gtk-2.0/connectdlg.c,v
retrieving revision 1.38
diff -u -r1.38 connectdlg.c
--- client/gui-gtk-2.0/connectdlg.c     28 Aug 2004 07:21:01 -0000      1.38
+++ client/gui-gtk-2.0/connectdlg.c     3 Sep 2004 04:43:48 -0000
@@ -260,12 +260,12 @@
       GtkTreeIter it;
       int i;
 
-      row[0] = ntoh_str(pserver->name);
-      row[1] = ntoh_str(pserver->port);
-      row[2] = ntoh_str(pserver->version);
-      row[3] = g_strdup(_(pserver->status));
-      row[4] = ntoh_str(pserver->players);
-      row[5] = ntoh_str(pserver->metastring);
+      row[0] = pserver->name;
+      row[1] = pserver->port;
+      row[2] = pserver->version;
+      row[3] = _(pserver->status);
+      row[4] = pserver->players;
+      row[5] = pserver->metastring;
 
       gtk_list_store_append(storelan, &it);
       gtk_list_store_set(storelan, &it,
@@ -1129,12 +1129,12 @@
     GtkTreeIter it;
     int i;
 
-    row[0] = ntoh_str(pserver->name);
-    row[1] = ntoh_str(pserver->port);
-    row[2] = ntoh_str(pserver->version);
-    row[3] = g_strdup(_(pserver->status));
-    row[4] = ntoh_str(pserver->players);
-    row[5] = ntoh_str(pserver->metastring);
+    row[0] = pserver->name;
+    row[1] = pserver->port;
+    row[2] = pserver->version;
+    row[3] = _(pserver->status);
+    row[4] = pserver->players;
+    row[5] = pserver->metastring;
 
     gtk_list_store_append(storemeta, &it);
     gtk_list_store_set(storemeta, &it,
Index: client/gui-gtk-2.0/gui_main.c
===================================================================
RCS file: /home/freeciv/CVS/freeciv/client/gui-gtk-2.0/gui_main.c,v
retrieving revision 1.74
diff -u -r1.74 gui_main.c
--- client/gui-gtk-2.0/gui_main.c       10 Jul 2004 18:48:18 -0000      1.74
+++ client/gui-gtk-2.0/gui_main.c       3 Sep 2004 04:43:48 -0000
@@ -33,6 +33,7 @@
 #include <gdk/gdkkeysyms.h>
 
 #include "dataio.h"
+#include "fciconv.h"
 #include "fcintl.h"
 #include "game.h"
 #include "government.h"
@@ -173,107 +174,13 @@
 static gint timer_callback(gpointer data);
 static gboolean show_conn_popup(GtkWidget *view, GdkEventButton *ev,
                                gpointer data);
-static char *network_charset = NULL;
-
-
-/**************************************************************************
-Network string charset conversion functions.
-**************************************************************************/
-gchar *ntoh_str(const gchar *netstr)
-{
-  return g_convert(netstr, -1, "UTF-8", network_charset, NULL, NULL, NULL);
-}
-
-/**************************************************************************
-...
-**************************************************************************/
-static unsigned char *put_conv(const char *src, size_t *length)
-{
-  gsize len;
-  gchar *out =
-    g_convert(src, -1, network_charset, "UTF-8", NULL, &len, NULL);
-
-  if (out) {
-    unsigned char *dst = fc_malloc(len + 1);
-
-    memcpy(dst, out, len);
-    g_free(out);
-    dst[len] = '\0';
-
-    *length = len;
-    return dst;
-  } else {
-    *length = 0;
-    return NULL;
-  }
-}
-
-/**************************************************************************
- Returns FALSE if the destination isn't large enough or the source was
- bad.
-**************************************************************************/
-static bool get_conv(char *dst, size_t ndst, const unsigned char *src,
-                    size_t nsrc)
-{
-  gsize len;                   /* length to copy, not including null */
-  gchar *out = g_convert(src, nsrc, "UTF-8", network_charset, NULL, &len, 
NULL);
-  bool ret = TRUE;
-
-  if (!out) {
-    dst[0] = '\0';
-    return FALSE;
-  }
-
-  if (ndst > 0 && len >= ndst) {
-    ret = FALSE;
-    len = ndst - 1;
-  }
-
-  memcpy(dst, out, len);
-  dst[len] = '\0';
-  g_free(out);
-
-  return ret;
-}
-
-/**************************************************************************
-Local log callback functions.
-**************************************************************************/
-static void fprintf_utf8(FILE *stream, const char *format, ...)
-{
-  va_list ap;
-  const gchar *charset;
-  gchar *s;
-
-  va_start(ap, format);
-  s = g_strdup_vprintf(format, ap);
-  va_end(ap);
-
-  if (!g_get_charset(&charset)) {
-    GError *error = NULL;
-    gchar  *s2;
-
-    s2 = g_convert(s, -1, charset, "UTF-8", NULL, NULL, &error);
-
-    if (error) {
-      fprintf(stream, "fprintf_utf8: %s\n", error->message);
-      g_error_free(error);
-    } else {
-      g_free(s);
-      s = s2;
-    }
-  }
-  fputs(s, stream);
-  fflush(stream);
-  g_free(s);
-}
 
 /**************************************************************************
 ...
 **************************************************************************/
 static void log_callback_utf8(int level, const char *message)
 {
-  fprintf_utf8(stderr, "%d: %s\n", level, message);
+  fc_fprintf(stderr, "%d: %s\n", level, message);
 }
 
 /**************************************************************************
@@ -283,7 +190,7 @@
 static void print_usage(const char *argv0)
 {
   /* add client-specific usage information here */
-  fprintf_utf8(stderr, _("Report bugs to <%s>.\n"), BUG_EMAIL_ADDRESS);
+  fc_fprintf(stderr, _("Report bugs to <%s>.\n"), BUG_EMAIL_ADDRESS);
 }
 
 /**************************************************************************
@@ -1020,33 +927,11 @@
 void ui_init(void)
 {
   gchar *s;
-  char *net_charset;
 
-#ifdef ENABLE_NLS
-  bind_textdomain_codeset(PACKAGE, "UTF-8");
-#endif
+  init_character_encodings("UTF-8");
 
   log_set_callback(log_callback_utf8);
 
-  /* set networking string conversion callbacks */
-  if ((net_charset = getenv("FREECIV_NETWORK_ENCODING"))) {
-    network_charset = mystrdup(net_charset);
-  } else {
-    const gchar *charset;
-
-    g_get_charset(&charset);
-
-    if (!strcmp(charset, "ANSI_X3.4-1968")
-       || !strcmp(charset, "ASCII")) {
-      charset = "ISO-8859-1";
-    }
-    
-    network_charset = mystrdup(charset);
-  }
-
-  dio_set_put_conv_callback(put_conv);
-  dio_set_get_conv_callback(get_conv);
-
   /* convert inputs */
   s = g_locale_to_utf8(user_name, -1, NULL, NULL, NULL);
   sz_strlcpy(user_name, s);
Index: client/gui-sdl/gui_iconv.c
===================================================================
RCS file: /home/freeciv/CVS/freeciv/client/gui-sdl/gui_iconv.c,v
retrieving revision 1.8
diff -u -r1.8 gui_iconv.c
--- client/gui-sdl/gui_iconv.c  7 Aug 2003 21:54:29 -0000       1.8
+++ client/gui-sdl/gui_iconv.c  3 Sep 2004 04:43:48 -0000
@@ -66,22 +66,6 @@
 }
 
 /**************************************************************************
-  Return the local charset encoding (which will be passed to iconv).
-**************************************************************************/
-static const char *get_local_encoding(void)
-{
-#ifdef HAVE_LIBCHARSET
-  return locale_charset();
-#else
-#  ifdef HAVE_LANGINFO_CODESET
-  return nl_langinfo(CODESET);
-#  else
-  return "";
-#  endif
-#endif
-}
-
-/**************************************************************************
   Convert string from local encoding (8 bit char) to
   display encoding (16 bit unicode) and resut put in pToUniString.
   if pToUniString == NULL then resulting string will be allocate automaticaly.
@@ -94,7 +78,7 @@
 {
   /* Start Parametrs */
   const char *pTocode = get_display_encoding();
-  const char *pFromcode = get_local_encoding();
+  const char *pFromcode = INTERNAL_ENCODING;
   const char *pStart = pFromString;
   size_t length = strlen(pFromString) + 1;
 
@@ -175,7 +159,7 @@
 {
   /* Start Parametrs */
   const char *pFromcode = get_display_encoding();
-  const char *pTocode = get_local_encoding();
+  const char *pTocode = INTERNAL_ENCODING;
   const char *pStart = (char *) pFromUniString;
   size_t ulength = (unistrlen(pFromUniString) + 1) * 2;
 
Index: client/gui-sdl/gui_iconv.h
===================================================================
RCS file: /home/freeciv/CVS/freeciv/client/gui-sdl/gui_iconv.h,v
retrieving revision 1.5
diff -u -r1.5 gui_iconv.h
--- client/gui-sdl/gui_iconv.h  7 Aug 2003 21:54:29 -0000       1.5
+++ client/gui-sdl/gui_iconv.h  3 Sep 2004 04:43:48 -0000
@@ -26,6 +26,10 @@
 #ifndef FC__GUI_ICONV_H
 #define FC__GUI_ICONV_H
 
+/* The encoding used internally by the client (outside of the GUI code).
+ * This must be a single-width encoding. */
+#define INTERNAL_ENCODING "UTF-8"
+
 Uint16 *convertcopy_to_utf16(Uint16 *pToUniString, size_t ulenght,
                              const char *pFromString);
 char *convertcopy_to_chars(char *pToString, size_t lenght,
Index: client/gui-sdl/gui_main.c
===================================================================
RCS file: /home/freeciv/CVS/freeciv/client/gui-sdl/gui_main.c,v
retrieving revision 1.44
diff -u -r1.44 gui_main.c
--- client/gui-sdl/gui_main.c   24 Jan 2004 02:58:55 -0000      1.44
+++ client/gui-sdl/gui_main.c   3 Sep 2004 04:43:48 -0000
@@ -698,6 +698,8 @@
   SDL_Surface *pBgd, *pTmp;
   Uint32 iSDL_Flags;
 
+  init_character_encodings(INTERNAL_ENCODING);
+
   SDL_Client_Flags = 0;
   iSDL_Flags = SDL_INIT_VIDEO | SDL_INIT_NOPARACHUTE;
   
Index: client/gui-win32/gui_main.c
===================================================================
RCS file: /home/freeciv/CVS/freeciv/client/gui-win32/gui_main.c,v
retrieving revision 1.28
diff -u -r1.28 gui_main.c
--- client/gui-win32/gui_main.c 21 Aug 2004 16:07:49 -0000      1.28
+++ client/gui-win32/gui_main.c 3 Sep 2004 04:43:48 -0000
@@ -21,6 +21,7 @@
 #include <commctrl.h>
 #include <richedit.h>
 
+#include "fciconv.h"
 #include "fcintl.h"
 #include "game.h"
 #include "log.h"
@@ -529,6 +530,7 @@
 **************************************************************************/
 void ui_init(void)
 {
+  init_character_encodings(NULL);
 }
 
 /**************************************************************************
Index: client/gui-xaw/gui_main.c
===================================================================
RCS file: /home/freeciv/CVS/freeciv/client/gui-xaw/gui_main.c,v
retrieving revision 1.92
diff -u -r1.92 gui_main.c
--- client/gui-xaw/gui_main.c   6 Jun 2004 06:00:09 -0000       1.92
+++ client/gui-xaw/gui_main.c   3 Sep 2004 04:43:49 -0000
@@ -37,6 +37,7 @@
 #include "canvas.h"
 #include "pixcomm.h"
 
+#include "fciconv.h"
 #include "fcintl.h"
 #include "game.h"
 #include "government.h"
@@ -261,6 +262,7 @@
 **************************************************************************/
 void ui_init(void)
 {
+  init_character_encodings(NULL);
 }
 
 /**************************************************************************
Index: server/srv_main.c
===================================================================
RCS file: /home/freeciv/CVS/freeciv/server/srv_main.c,v
retrieving revision 1.184
diff -u -r1.184 srv_main.c
--- server/srv_main.c   29 Aug 2004 19:03:32 -0000      1.184
+++ server/srv_main.c   3 Sep 2004 04:43:49 -0000
@@ -49,6 +49,7 @@
 #include "city.h"
 #include "dataio.h"
 #include "events.h"
+#include "fciconv.h"
 #include "fcintl.h"
 #include "game.h"
 #include "log.h"
@@ -184,6 +185,9 @@
   /* mark as initialized */
   has_been_srv_init = TRUE;
 
+  /* init character encodings. */
+  init_character_encodings(DEFAULT_DATA_ENCODING);
+
   /* done */
   return;
 }
Index: utility/Makefile.am
===================================================================
RCS file: /home/freeciv/CVS/freeciv/utility/Makefile.am,v
retrieving revision 1.6
diff -u -r1.6 Makefile.am
--- utility/Makefile.am 27 Aug 2004 08:18:38 -0000      1.6
+++ utility/Makefile.am 3 Sep 2004 04:43:49 -0000
@@ -13,6 +13,8 @@
                astring.h       \
                capability.c    \
                capability.h    \
+               fciconv.c       \
+               fciconv.h       \
                distribute.c    \
                distribute.h    \
                fcintl.c        \
Index: utility/fciconv.c
===================================================================
RCS file: /home/freeciv/CVS/freeciv/utility/fciconv.c,v
retrieving revision 1.1
diff -u -r1.1 fciconv.c
--- utility/fciconv.c   26 Apr 2004 02:13:30 -0000      1.1
+++ utility/fciconv.c   3 Sep 2004 04:43:49 -0000
@@ -0,0 +1,272 @@
+/********************************************************************** 
+ Freeciv - Copyright (C) 2003-2004 - The Freeciv Project
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 2, or (at your option)
+   any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+***********************************************************************/
+
+#ifdef HAVE_CONFIG_H
+#include <config.h>
+#endif
+
+#include <assert.h>
+#include <errno.h>
+#include <stdio.h>
+#include <string.h>
+
+#ifdef HAVE_ICONV
+#include <iconv.h>
+#endif
+
+#ifdef HAVE_LANGINFO_CODESET
+#include <langinfo.h>
+#endif
+
+#include "fciconv.h"
+#include "fcintl.h"
+#include "log.h"
+#include "mem.h"
+#include "support.h"
+
+static bool is_init = FALSE;
+static char convert_buffer[4096];
+
+#ifdef HAVE_ICONV
+static char *local_encoding, *data_encoding, *internal_encoding;
+#endif
+
+/***************************************************************************
+  Must be called during the initialization phase of server and client to
+  initialize the character encodings to be used.
+***************************************************************************/
+void init_character_encodings(char *my_internal_encoding)
+{
+#ifdef HAVE_ICONV
+  /* Set the data encoding - first check $FREECIV_DATA_ENCODING,
+   * then fall back to the default. */
+  data_encoding = getenv("FREECIV_DATA_ENCODING");
+  if (!data_encoding) {
+    /* Currently the rulesets are in latin1 (ISO-8859-1). */
+    data_encoding = DEFAULT_DATA_ENCODING;
+  }
+
+  /* Set the local encoding - first check $FREECIV_LOCAL_ENCODING,
+   * then ask the system. */
+  local_encoding = getenv("FREECIV_LOCAL_ENCODING");
+  if (!local_encoding) {
+#ifdef HAVE_LIBCHARSET
+    local_encoding = locale_charset();
+#else
+#ifdef HAVE_LANGINFO_CODESET
+    local_encoding = nl_langinfo(CODESET);
+#else
+    local_encoding = "";
+#endif
+#endif
+    if (strcasecmp(local_encoding, "ANSI_X3.4-1968") == 0
+       || strcasecmp(local_encoding, "ASCII") == 0) {
+      /* HACK: use latin1 instead of ascii in typical cases when the
+       * encoding is unconfigured. */
+      local_encoding = "ISO-8859-1";
+    }
+  }
+
+  /* Set the internal encoding - first check $FREECIV_INTERNAL_ENCODING,
+   * then check the passed-in default value, then fall back to the local
+   * encoding. */
+  internal_encoding = getenv("FREECIV_INTERNAL_ENCODING");
+  if (!internal_encoding) {
+    internal_encoding = my_internal_encoding;
+
+    if (!internal_encoding) {
+      internal_encoding = local_encoding;
+    }
+  }
+
+#ifdef ENABLE_NLS
+  bind_textdomain_codeset(PACKAGE, internal_encoding);
+#endif
+
+  fprintf(stderr, "Data=%s, Local=%s, Internal=%s\n",
+            data_encoding, local_encoding, internal_encoding);
+#else
+   /* freelog may not work at this point. */
+  fprintf(stderr,
+            _("You are running Freeciv without using iconv.  Unless\n"
+              "you are using the latin1 character set, some characters\n"
+              "may not be displayed properly.  You can download iconv\n"
+              "at http://gnu.org/.\n";));
+#endif
+
+  is_init = TRUE;
+}
+
+const char *get_data_encoding(void)
+{
+  assert(is_init);
+  return data_encoding;
+}
+
+const char *get_local_encoding(void)
+{
+  assert(is_init);
+  return local_encoding;
+}
+
+const char *get_internal_encoding(void)
+{
+  assert(is_init);
+  return internal_encoding;
+}
+
+/***************************************************************************
+  Convert the text.  Both 'from' and 'to' must be 8-bit charsets.  The
+  result will be put into the buf buffer unless it is NULL, in which case it
+  will be allocated on demand.
+***************************************************************************/
+static char *convert_string(const char *text,
+                           const char *from,
+                           const char *to,
+                           char *buf, size_t bufsz)
+{
+#ifdef HAVE_ICONV
+  iconv_t cd = iconv_open(to, from);
+  size_t from_len = strlen(text) + 1, to_len;
+  bool alloc = (buf == NULL);
+
+  assert(is_init && from != NULL && to != NULL);
+  assert(text != NULL);
+
+  if (cd == (iconv_t) (-1)) {
+    freelog(LOG_ERROR,
+           _("Could not convert text from %s to %s: %s"),
+           from, to, strerror(errno));
+    /* The best we can do? */
+    if (alloc) {
+      return mystrdup(text);
+    } else {
+      my_snprintf(buf, bufsz, "%s", text);
+      return buf;
+    }
+  }
+
+  if (alloc) {
+    to_len = from_len;
+  } else {
+    to_len = bufsz;
+  }
+
+  do {
+    size_t flen = from_len, tlen = to_len, res;
+    const char *mytext = text;
+    char *myresult;
+
+    if (alloc) {
+      buf = fc_malloc(to_len);
+    }
+
+    myresult = buf;
+
+    /* Since we may do multiple translations, we may need to reset iconv
+     * in between. */
+    iconv(cd, NULL, NULL, NULL, NULL);
+
+    res = iconv(cd, (char**)&mytext, &flen, &myresult, &tlen);
+    if (res == (size_t) (-1)) {
+      if (errno != E2BIG) {
+       /* Invalid input. */
+       freelog(LOG_ERROR, "Invalid string conversion from %s to %s.",
+               from, to);
+       iconv_close(cd);
+       if (alloc) {
+         free(buf);
+         return mystrdup(text); /* The best we can do? */
+       } else {
+         my_snprintf(buf, bufsz, "%s", text);
+         return buf;
+       }
+      }
+    } else {
+      /* Success. */
+      iconv_close(cd);
+
+      /* There may be wasted space here, but there's nothing we can do
+       * about it. */
+      return buf;
+    }
+
+    if (alloc) {
+      /* Not enough space; try again. */
+      buf[to_len - 1] = 0;
+      freelog(LOG_VERBOSE, "   Result was '%s'.", buf);
+
+      free(buf);
+      to_len *= 2;
+    }
+  } while (alloc);
+
+  return buf;
+#else /* HAVE_ICONV */
+  if (buf) {
+    strncpy(buf, text, bufsz);
+    buf[bufsz - 1] = '\0';
+  } else {
+    return mystrdup(text);
+  }
+#endif /* HAVE_ICONV */
+}
+
+#define CONV_FUNC_MALLOC(src, dst)                                          \
+char *src ## _to_ ## dst ## _string_malloc(const char *text)                \
+{                                                                           \
+  return convert_string(text, (src ## _encoding),                          \
+                       (dst ## _encoding), NULL, 0);                       \
+}
+
+#define CONV_FUNC_BUFFER(src, dst)                                          \
+char *src ## _to_ ## dst ## _string_buffer(const char *text,                \
+                                          char *buf, size_t bufsz)         \
+{                                                                           \
+  return convert_string(text, (src ## _encoding),                          \
+                        (dst ## _encoding), buf, bufsz);                    \
+}
+
+#define CONV_FUNC_STATIC(src, dst)                                          \
+char *src ## _to_ ## dst ## _string_static(const char *text)                \
+{                                                                           \
+  (src ## _to_ ## dst ## _string_buffer)(text,                              \
+                                       convert_buffer,                     \
+                                       sizeof(convert_buffer));            \
+  return convert_buffer;                                                    \
+}
+
+CONV_FUNC_MALLOC(data, internal)
+CONV_FUNC_MALLOC(internal, data)
+
+static CONV_FUNC_BUFFER(internal, local)
+static CONV_FUNC_STATIC(internal, local)
+
+/***************************************************************************
+  Do a printf in the currently bound codeset.
+***************************************************************************/
+void fc_fprintf(FILE *stream, const char *format, ...)
+{
+  va_list ap;
+  char string[4096];
+  const char *output;
+
+  va_start(ap, format);
+  my_vsnprintf(string, sizeof(string), format, ap);
+  va_end(ap);
+
+  output = internal_to_local_string_static(string);
+
+  fputs(output, stream);
+  fflush(stream);
+}
Index: utility/fciconv.h
===================================================================
RCS file: /home/freeciv/CVS/freeciv/utility/fciconv.h,v
retrieving revision 1.1
diff -u -r1.1 fciconv.h
--- utility/fciconv.h   26 Apr 2004 02:13:30 -0000      1.1
+++ utility/fciconv.h   3 Sep 2004 04:43:49 -0000
@@ -0,0 +1,32 @@
+/********************************************************************** 
+ Freeciv - Copyright (C) 2003-2004 - The Freeciv Project
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 2, or (at your option)
+   any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+***********************************************************************/
+#ifndef FC__FCICONV_H
+#define FC__FCICONV_H
+
+#include "shared.h"
+
+#define DEFAULT_DATA_ENCODING "ISO-8859-1"
+
+void init_character_encodings(char *internal_encoding);
+
+const char *get_data_encoding(void);
+const char *get_local_encoding(void);
+const char *get_internal_encoding(void);
+
+char *data_to_internal_string_malloc(const char *text);
+char *internal_to_data_string_malloc(const char *text);
+
+void fc_fprintf(FILE *stream, const char *format, ...)
+      fc__attribute((format (printf, 2, 3)));
+
+#endif /* FC__FCICONV_H */
Index: utility/log.c
===================================================================
RCS file: /home/freeciv/CVS/freeciv/utility/log.c,v
retrieving revision 1.45
diff -u -r1.45 log.c
--- utility/log.c       10 Apr 2004 03:47:49 -0000      1.45
+++ utility/log.c       3 Sep 2004 04:43:49 -0000
@@ -19,6 +19,7 @@
 #include <stdio.h>
 #include <string.h>
 
+#include "fciconv.h"
 #include "fcintl.h"
 #include "mem.h"
 #include "shared.h"
@@ -80,16 +81,16 @@
   }
   if (n == 0) {
     if (sscanf(level_str, "%d", &level) != 1) {
-      fprintf(stderr, _("Bad log level \"%s\".\n"), level_str);
+      fc_fprintf(stderr, _("Bad log level \"%s\".\n"), level_str);
       return -1;
     }
     if (level >= LOG_FATAL && level <= max_level) {
       return level;
     } else {
-      fprintf(stderr, _("Bad log level %d in \"%s\".\n"), level, level_str);
+      fc_fprintf(stderr, _("Bad log level %d in \"%s\".\n"), level, level_str);
       if (level == LOG_DEBUG && max_level < LOG_DEBUG) {
-       fprintf(stderr, _("Freeciv must be compiled with the DEBUG flag"
-                         " to use debug level %d.\n"), LOG_DEBUG);
+       fc_fprintf(stderr, _("Freeciv must be compiled with the DEBUG flag"
+                            " to use debug level %d.\n"), LOG_DEBUG);
       }
       return -1;
     }
@@ -99,12 +100,13 @@
   if (c[0] == ('0' + LOG_DEBUG) && c[1] == ':') {
     level = LOG_DEBUG;
     if (max_level < LOG_DEBUG) {
-      fprintf(stderr, _("Freeciv must be compiled with the DEBUG flag"
-                       " to use debug level %d.\n"), LOG_DEBUG);
+      fc_fprintf(stderr, _("Freeciv must be compiled with the DEBUG flag"
+                          " to use debug level %d.\n"), LOG_DEBUG);
       return -1;
     }
   } else {
-    fprintf(stderr, _("Badly formed log level argument \"%s\".\n"), level_str);
+    fc_fprintf(stderr, _("Badly formed log level argument \"%s\".\n"),
+              level_str);
     return -1;
   }
   logd_num_files = n;
@@ -115,7 +117,8 @@
   tok = strtok(dup, ":");
   
   if (!tok) {
-    fprintf(stderr, _("Badly formed log level argument \"%s\".\n"), level_str);
+    fc_fprintf(stderr, _("Badly formed log level argument \"%s\".\n"),
+              level_str);
     level = -1;
     goto out;
   }
@@ -132,20 +135,20 @@
       if (d && *pc != '\0' && d[1] != '\0') {
        d[0] = '\0';
        if (sscanf(pc, "%d", &logd_files[i].min) != 1) {
-         fprintf(stderr, _("Not an integer: '%s'\n"), pc);
+         fc_fprintf(stderr, _("Not an integer: '%s'\n"), pc);
          level = -1;
          goto out;
        }
        if (sscanf(d + 1, "%d", &logd_files[i].max) != 1) {
-         fprintf(stderr, _("Not an integer: '%s'\n"), d + 1);
+         fc_fprintf(stderr, _("Not an integer: '%s'\n"), d + 1);
          level = -1;
          goto out;
        }
       }
     }
     if(strlen(tok)==0) {
-      fprintf(stderr, _("Empty filename in log level argument \"%s\".\n"),
-             level_str);
+      fc_fprintf(stderr, _("Empty filename in log level argument \"%s\".\n"),
+                level_str);
       level = -1;
       goto out;
     }
@@ -155,7 +158,8 @@
   } while(tok);
 
   if (i!=logd_num_files) {
-    fprintf(stderr, _("Badly formed log level argument \"%s\".\n"), level_str);
+    fc_fprintf(stderr, _("Badly formed log level argument \"%s\".\n"),
+              level_str);
     level = -1;
     goto out;
   }
@@ -236,7 +240,7 @@
     log_callback(level, message);
   }
   if (log_filename || (!log_callback)) {
-    fprintf(fs, "%d: %s\n", level, message);
+    fc_fprintf(fs, "%d: %s\n", level, message);
     fflush(fs);
   }
 }
@@ -264,7 +268,7 @@
 
     if (log_filename) {
       if(!(fs=fopen(log_filename, "a"))) {
-       fprintf(stderr, _("Couldn't open logfile: %s for appending.\n"), 
+       fc_fprintf(stderr, _("Couldn't open logfile: %s for appending.\n"), 
                log_filename);
        exit(EXIT_FAILURE);
       }