[Freeciv-Dev] Re: (PR#1824) ruleset data is in incompatible charsets

["Jason Short" <jdorje@xxxxxxxxxxxxxxxxxxxxx>]

<URL: http://rt.freeciv.org/Ticket/Display.html?id=1824 >

Here's a quick update of the patch to conform to the new design.  It is 
untested and not very closely inspected.

There is some question of whether fciconv should be in utility/ or 
common/.  The code in here is fairly specific to freeciv in that it is 
based on the design outlined in the flowchart.  But none of it is 
directly freeciv-related.  Moreover log.c needs to use functions from 
this file for its output (alternatively we could use a registered 
callback mechanism here).

jason


? diff
Index: configure.ac
===================================================================
RCS file: /home/freeciv/CVS/freeciv/configure.ac,v
retrieving revision 1.64
diff -u -r1.64 configure.ac
--- configure.ac        2 May 2004 12:13:51 -0000       1.64
+++ configure.ac        8 May 2004 07:13:48 -0000
@@ -306,6 +306,11 @@
   AC_MSG_ERROR([zlib found but not zlib.h.  
 You may need to install a zlib \"development\" package.]))
 
+dnl Check for libiconv (which is usually included in glibc, but may be
+dnl distributed separately).
+AM_ICONV
+LIBS="$LIBS $LIBICONV"
+
 dnl Check and choose clients
 if test x$client != xno; then
 
Index: configure.in
===================================================================
RCS file: /home/freeciv/CVS/freeciv/configure.in,v
retrieving revision 1.240
diff -u -r1.240 configure.in
--- configure.in        2 May 2004 12:13:51 -0000       1.240
+++ configure.in        8 May 2004 07:13:48 -0000
@@ -299,6 +299,11 @@
   AC_MSG_ERROR([zlib found but not zlib.h.  
 You may need to install a zlib \"development\" package.]))
 
+dnl Check for libiconv (which is usually included in glibc, but may be
+dnl distributed separately).
+AM_ICONV
+LIBS="$LIBS $LIBICONV"
+
 dnl Check and choose clients
 if test x$client != xno; then
 
Index: client/civclient.c
===================================================================
RCS file: /home/freeciv/CVS/freeciv/client/civclient.c,v
retrieving revision 1.187
diff -u -r1.187 civclient.c
--- client/civclient.c  21 Apr 2004 19:46:37 -0000      1.187
+++ client/civclient.c  8 May 2004 07:13:48 -0000
@@ -32,7 +32,9 @@
 #endif
 
 #include "capstr.h"
+#include "dataio.h"
 #include "diptreaty.h"
+#include "fciconv.h"
 #include "fcintl.h"
 #include "game.h"
 #include "idex.h"
@@ -102,6 +104,61 @@
  */
 bool turn_done_sent = FALSE;
 
+/**************************************************************************
+  Convert a text string from the internal to the data encoding, when it
+  is written to the network.
+**************************************************************************/
+static unsigned char *put_conv(const char *src, size_t *length)
+{
+  char *out = internal_to_data_string_malloc(src);
+
+  if (out) {
+    *length = strlen(out);
+    return out;
+  } else {
+    *length = 0;
+    return NULL;
+  }
+}
+
+/**************************************************************************
+  Convert a text string from the data to the internal encoding when it is
+  first read from the network.  Returns FALSE if the destination isn't
+  large enough or the source was bad.
+**************************************************************************/
+static bool get_conv(char *dst, size_t ndst,
+                    const unsigned char *src, size_t nsrc)
+{
+  char *out = data_to_internal_string_malloc(src);
+  bool ret = TRUE;
+  size_t len;
+
+  if (!out) {
+    dst[0] = '\0';
+    return FALSE;
+  }
+
+  len = strlen(out);
+  if (ndst > 0 && len >= ndst) {
+    ret = FALSE;
+    len = ndst - 1;
+  }
+
+  memcpy(dst, out, len);
+  dst[len] = '\0';
+  free(out);
+
+  return ret;
+}
+
+/**************************************************************************
+  Set up charsets for the client.
+**************************************************************************/
+static void charsets_init(void)
+{
+  dio_set_put_conv_callback(put_conv);
+  dio_set_get_conv_callback(get_conv);
+}
 
 /**************************************************************************
 ...
@@ -214,6 +271,7 @@
   conn_list_init(&game.game_connections);
 
   ui_init();
+  charsets_init();
   my_init_network();
   init_messages_where();
   init_city_report_data();
Index: client/gui-gtk/gui_main.c
===================================================================
RCS file: /home/freeciv/CVS/freeciv/client/gui-gtk/gui_main.c,v
retrieving revision 1.148
diff -u -r1.148 gui_main.c
--- client/gui-gtk/gui_main.c   15 Apr 2004 19:36:01 -0000      1.148
+++ client/gui-gtk/gui_main.c   8 May 2004 07:13:49 -0000
@@ -31,6 +31,7 @@
 #include <unistd.h>
 #endif
 
+#include "fciconv.h"
 #include "fcintl.h"
 #include "game.h"
 #include "government.h"
@@ -798,6 +799,7 @@
 **************************************************************************/
 void ui_init(void)
 {
+  init_character_encodings(NULL);
 }
 
 /**************************************************************************
Index: client/gui-gtk-2.0/connectdlg.c
===================================================================
RCS file: /home/freeciv/CVS/freeciv/client/gui-gtk-2.0/connectdlg.c,v
retrieving revision 1.34
diff -u -r1.34 connectdlg.c
--- client/gui-gtk-2.0/connectdlg.c     1 May 2004 17:33:01 -0000       1.34
+++ client/gui-gtk-2.0/connectdlg.c     8 May 2004 07:13:49 -0000
@@ -260,12 +260,12 @@
       GtkTreeIter it;
       int i;
 
-      row[0] = ntoh_str(pserver->name);
-      row[1] = ntoh_str(pserver->port);
-      row[2] = ntoh_str(pserver->version);
+      row[0] = pserver->name;
+      row[1] = pserver->port;
+      row[2] = pserver->version;
       row[3] = _(pserver->status);
-      row[4] = ntoh_str(pserver->players);
-      row[5] = ntoh_str(pserver->metastring);
+      row[4] = pserver->players;
+      row[5] = pserver->metastring;
 
       gtk_list_store_append(storelan, &it);
       gtk_list_store_set(storelan, &it,
@@ -1137,12 +1137,12 @@
     GtkTreeIter it;
     int i;
 
-    row[0] = ntoh_str(pserver->name);
-    row[1] = ntoh_str(pserver->port);
-    row[2] = ntoh_str(pserver->version);
+    row[0] = pserver->name;
+    row[1] = pserver->port;
+    row[2] = pserver->version;
     row[3] = _(pserver->status);
-    row[4] = ntoh_str(pserver->players);
-    row[5] = ntoh_str(pserver->metastring);
+    row[4] = pserver->players;
+    row[5] = pserver->metastring;
 
     gtk_list_store_append(storemeta, &it);
     gtk_list_store_set(storemeta, &it,
Index: client/gui-gtk-2.0/gui_main.c
===================================================================
RCS file: /home/freeciv/CVS/freeciv/client/gui-gtk-2.0/gui_main.c,v
retrieving revision 1.72
diff -u -r1.72 gui_main.c
--- client/gui-gtk-2.0/gui_main.c       1 May 2004 17:28:47 -0000       1.72
+++ client/gui-gtk-2.0/gui_main.c       8 May 2004 07:13:49 -0000
@@ -33,6 +33,7 @@
 #include <gdk/gdkkeysyms.h>
 
 #include "dataio.h"
+#include "fciconv.h"
 #include "fcintl.h"
 #include "game.h"
 #include "government.h"
@@ -173,107 +174,13 @@
 static gint timer_callback(gpointer data);
 static gboolean show_conn_popup(GtkWidget *view, GdkEventButton *ev,
                                gpointer data);
-static char *network_charset = NULL;
-
-
-/**************************************************************************
-Network string charset conversion functions.
-**************************************************************************/
-gchar *ntoh_str(const gchar *netstr)
-{
-  return g_convert(netstr, -1, "UTF-8", network_charset, NULL, NULL, NULL);
-}
-
-/**************************************************************************
-...
-**************************************************************************/
-static unsigned char *put_conv(const char *src, size_t *length)
-{
-  gsize len;
-  gchar *out =
-    g_convert(src, -1, network_charset, "UTF-8", NULL, &len, NULL);
-
-  if (out) {
-    unsigned char *dst = fc_malloc(len + 1);
-
-    memcpy(dst, out, len);
-    g_free(out);
-    dst[len] = '\0';
-
-    *length = len;
-    return dst;
-  } else {
-    *length = 0;
-    return NULL;
-  }
-}
-
-/**************************************************************************
- Returns FALSE if the destination isn't large enough or the source was
- bad.
-**************************************************************************/
-static bool get_conv(char *dst, size_t ndst, const unsigned char *src,
-                    size_t nsrc)
-{
-  gsize len;                   /* length to copy, not including null */
-  gchar *out = g_convert(src, nsrc, "UTF-8", network_charset, NULL, &len, 
NULL);
-  bool ret = TRUE;
-
-  if (!out) {
-    dst[0] = '\0';
-    return FALSE;
-  }
-
-  if (ndst > 0 && len >= ndst) {
-    ret = FALSE;
-    len = ndst - 1;
-  }
-
-  memcpy(dst, out, len);
-  dst[len] = '\0';
-  g_free(out);
-
-  return ret;
-}
-
-/**************************************************************************
-Local log callback functions.
-**************************************************************************/
-static void fprintf_utf8(FILE *stream, const char *format, ...)
-{
-  va_list ap;
-  const gchar *charset;
-  gchar *s;
-
-  va_start(ap, format);
-  s = g_strdup_vprintf(format, ap);
-  va_end(ap);
-
-  if (!g_get_charset(&charset)) {
-    GError *error = NULL;
-    gchar  *s2;
-
-    s2 = g_convert(s, -1, charset, "UTF-8", NULL, NULL, &error);
-
-    if (error) {
-      fprintf(stream, "fprintf_utf8: %s\n", error->message);
-      g_error_free(error);
-    } else {
-      g_free(s);
-      s = s2;
-    }
-  }
-  fputs(s, stream);
-  fflush(stream);
-  g_free(s);
-}
 
 /**************************************************************************
 ...
 **************************************************************************/
 static void log_callback_utf8(int level, const char *message)
 {
-  fprintf_utf8(stderr, "%d: %s\n", level, message);
+  fc_fprintf(stderr, "%d: %s\n", level, message);
 }
 
 /**************************************************************************
@@ -283,7 +190,7 @@
 static void print_usage(const char *argv0)
 {
   /* add client-specific usage information here */
-  fprintf_utf8(stderr, _("Report bugs to <%s>.\n"), BUG_EMAIL_ADDRESS);
+  fc_fprintf(stderr, _("Report bugs to <%s>.\n"), BUG_EMAIL_ADDRESS);
 }
 
 /**************************************************************************
@@ -1015,33 +922,11 @@
 void ui_init(void)
 {
   gchar *s;
-  char *net_charset;
 
-#ifdef ENABLE_NLS
-  bind_textdomain_codeset(PACKAGE, "UTF-8");
-#endif
+  init_character_encodings("UTF-8");
 
   log_set_callback(log_callback_utf8);
 
-  /* set networking string conversion callbacks */
-  if ((net_charset = getenv("FREECIV_NETWORK_ENCODING"))) {
-    network_charset = mystrdup(net_charset);
-  } else {
-    const gchar *charset;
-
-    g_get_charset(&charset);
-
-    if (!strcmp(charset, "ANSI_X3.4-1968")
-       || !strcmp(charset, "ASCII")) {
-      charset = "ISO-8859-1";
-    }
-    
-    network_charset = mystrdup(charset);
-  }
-
-  dio_set_put_conv_callback(put_conv);
-  dio_set_get_conv_callback(get_conv);
-
   /* convert inputs */
   s = g_locale_to_utf8(user_name, -1, NULL, NULL, NULL);
   sz_strlcpy(user_name, s);
Index: client/gui-sdl/gui_iconv.c
===================================================================
RCS file: /home/freeciv/CVS/freeciv/client/gui-sdl/gui_iconv.c,v
retrieving revision 1.8
diff -u -r1.8 gui_iconv.c
--- client/gui-sdl/gui_iconv.c  7 Aug 2003 21:54:29 -0000       1.8
+++ client/gui-sdl/gui_iconv.c  8 May 2004 07:13:49 -0000
@@ -66,22 +66,6 @@
 }
 
 /**************************************************************************
-  Return the local charset encoding (which will be passed to iconv).
-**************************************************************************/
-static const char *get_local_encoding(void)
-{
-#ifdef HAVE_LIBCHARSET
-  return locale_charset();
-#else
-#  ifdef HAVE_LANGINFO_CODESET
-  return nl_langinfo(CODESET);
-#  else
-  return "";
-#  endif
-#endif
-}
-
-/**************************************************************************
   Convert string from local encoding (8 bit char) to
   display encoding (16 bit unicode) and resut put in pToUniString.
   if pToUniString == NULL then resulting string will be allocate automaticaly.
@@ -94,7 +78,7 @@
 {
   /* Start Parametrs */
   const char *pTocode = get_display_encoding();
-  const char *pFromcode = get_local_encoding();
+  const char *pFromcode = INTERNAL_ENCODING;
   const char *pStart = pFromString;
   size_t length = strlen(pFromString) + 1;
 
@@ -175,7 +159,7 @@
 {
   /* Start Parametrs */
   const char *pFromcode = get_display_encoding();
-  const char *pTocode = get_local_encoding();
+  const char *pTocode = INTERNAL_ENCODING;
   const char *pStart = (char *) pFromUniString;
   size_t ulength = (unistrlen(pFromUniString) + 1) * 2;
 
Index: client/gui-sdl/gui_iconv.h
===================================================================
RCS file: /home/freeciv/CVS/freeciv/client/gui-sdl/gui_iconv.h,v
retrieving revision 1.5
diff -u -r1.5 gui_iconv.h
--- client/gui-sdl/gui_iconv.h  7 Aug 2003 21:54:29 -0000       1.5
+++ client/gui-sdl/gui_iconv.h  8 May 2004 07:13:49 -0000
@@ -26,6 +26,10 @@
 #ifndef FC__GUI_ICONV_H
 #define FC__GUI_ICONV_H
 
+/* The encoding used internally by the client (outside of the GUI code).
+ * This must be a single-width encoding. */
+#define INTERNAL_ENCODING "UTF-8"
+
 Uint16 *convertcopy_to_utf16(Uint16 *pToUniString, size_t ulenght,
                              const char *pFromString);
 char *convertcopy_to_chars(char *pToString, size_t lenght,
Index: client/gui-sdl/gui_main.c
===================================================================
RCS file: /home/freeciv/CVS/freeciv/client/gui-sdl/gui_main.c,v
retrieving revision 1.44
diff -u -r1.44 gui_main.c
--- client/gui-sdl/gui_main.c   24 Jan 2004 02:58:55 -0000      1.44
+++ client/gui-sdl/gui_main.c   8 May 2004 07:13:49 -0000
@@ -698,6 +698,8 @@
   SDL_Surface *pBgd, *pTmp;
   Uint32 iSDL_Flags;
 
+  init_character_encodings(INTERNAL_ENCODING);
+
   SDL_Client_Flags = 0;
   iSDL_Flags = SDL_INIT_VIDEO | SDL_INIT_NOPARACHUTE;
   
Index: client/gui-win32/gui_main.c
===================================================================
RCS file: /home/freeciv/CVS/freeciv/client/gui-win32/gui_main.c,v
retrieving revision 1.25
diff -u -r1.25 gui_main.c
--- client/gui-win32/gui_main.c 17 Mar 2004 09:10:57 -0000      1.25
+++ client/gui-win32/gui_main.c 8 May 2004 07:13:49 -0000
@@ -519,6 +519,7 @@
 **************************************************************************/
 void ui_init(void)
 {
+  init_character_encodings(NULL);
 }
 
 /**************************************************************************
Index: client/gui-xaw/gui_main.c
===================================================================
RCS file: /home/freeciv/CVS/freeciv/client/gui-xaw/gui_main.c,v
retrieving revision 1.91
diff -u -r1.91 gui_main.c
--- client/gui-xaw/gui_main.c   15 Apr 2004 19:36:01 -0000      1.91
+++ client/gui-xaw/gui_main.c   8 May 2004 07:13:50 -0000
@@ -37,6 +37,7 @@
 #include "canvas.h"
 #include "pixcomm.h"
 
+#include "fciconv.h"
 #include "fcintl.h"
 #include "game.h"
 #include "government.h"
@@ -261,6 +262,7 @@
 **************************************************************************/
 void ui_init(void)
 {
+  init_character_encodings(NULL);
 }
 
 /**************************************************************************
Index: common/Makefile.am
===================================================================
RCS file: /home/freeciv/CVS/freeciv/common/Makefile.am,v
retrieving revision 1.50
diff -u -r1.50 Makefile.am
--- common/Makefile.am  2 May 2004 12:13:51 -0000       1.50
+++ common/Makefile.am  8 May 2004 07:13:50 -0000
@@ -24,6 +24,8 @@
                effects.c       \
                effects.h       \
                events.h        \
+               fciconv.c       \
+               fciconv.h       \
                game.c          \
                game.h          \
                generate_packets.py \
Index: common/fciconv.c
===================================================================
RCS file: /home/freeciv/CVS/freeciv/common/fciconv.c,v
retrieving revision 1.1
diff -u -r1.1 fciconv.c
--- common/fciconv.c    26 Apr 2004 02:13:30 -0000      1.1
+++ common/fciconv.c    8 May 2004 07:13:50 -0000
@@ -0,0 +1,272 @@
+/********************************************************************** 
+ Freeciv - Copyright (C) 2003 - The Freeciv Project
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 2, or (at your option)
+   any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+***********************************************************************/
+
+#ifdef HAVE_CONFIG_H
+#include <config.h>
+#endif
+
+#include <assert.h>
+#include <errno.h>
+#include <stdio.h>
+#include <string.h>
+
+#ifdef HAVE_ICONV
+#include <iconv.h>
+#endif
+
+#ifdef HAVE_LANGINFO_CODESET
+#include <langinfo.h>
+#endif
+
+#include "fciconv.h"
+#include "fcintl.h"
+#include "log.h"
+#include "mem.h"
+#include "support.h"
+
+static bool is_init = FALSE;
+static char convert_buffer[4096];
+
+#ifdef HAVE_ICONV
+static char *local_encoding, *data_encoding, *internal_encoding;
+#endif
+
+/***************************************************************************
+  Must be called during the initialization phase of server and client to
+  initialize the character encodings to be used.
+***************************************************************************/
+void init_character_encodings(char *my_internal_encoding)
+{
+#ifdef HAVE_ICONV
+  /* Set the data encoding - first check $FREECIV_DATA_ENCODING,
+   * then fall back to the default. */
+  data_encoding = getenv("FREECIV_DATA_ENCODING");
+  if (!data_encoding) {
+    /* Currently the rulesets are in latin1 (ISO-8859-1). */
+    data_encoding = DEFAULT_DATA_ENCODING;
+  }
+
+  /* Set the local encoding - first check $FREECIV_LOCAL_ENCODING,
+   * then ask the system. */
+  local_encoding = getenv("FREECIV_LOCAL_ENCODING");
+  if (!local_encoding) {
+#ifdef HAVE_LIBCHARSET
+    local_encoding = locale_charset();
+#else
+#ifdef HAVE_LANGINFO_CODESET
+    local_encoding = nl_langinfo(CODESET);
+#else
+    local_encoding = "";
+#endif
+#endif
+    if (strcasecmp(local_encoding, "ANSI_X3.4-1968") == 0
+       || strcasecmp(local_encoding, "ASCII") == 0) {
+      /* HACK: use latin1 instead of ascii in typical cases when the
+       * encoding is unconfigured. */
+      local_encoding = "ISO-8859-1";
+    }
+  }
+
+  /* Set the internal encoding - first check $FREECIV_INTERNAL_ENCODING,
+   * then check the passed-in default value, then fall back to the local
+   * encoding. */
+  internal_encoding = getenv("FREECIV_INTERNAL_ENCODING");
+  if (!internal_encoding) {
+    internal_encoding = my_internal_encoding;
+
+    if (!internal_encoding) {
+      internal_encoding = local_encoding;
+    }
+  }
+
+#ifdef ENABLE_NLS
+  bind_textdomain_codeset(PACKAGE, internal_encoding);
+#endif
+
+  fc_fprintf(stderr, "Data=%s, Local=%s, Internal=%s\n",
+            data_encoding, local_encoding, internal_encoding);
+#else
+   /* freelog may not work at this point. */
+  fc_fprintf(stderr,
+            _("You are running Freeciv without using iconv.  Unless\n"
+              "you are using the latin1 character set, some characters\n"
+              "may not be displayed properly.  You can download iconv\n"
+              "at http://gnu.org/.\n";));
+#endif
+
+  is_init = TRUE;
+}
+
+const char *get_data_encoding(void)
+{
+  assert(is_init);
+  return data_encoding;
+}
+
+const char *get_local_encoding(void)
+{
+  assert(is_init);
+  return local_encoding;
+}
+
+const char *get_internal_encoding(void)
+{
+  assert(is_init);
+  return internal_encoding;
+}
+
+/***************************************************************************
+  Convert the text.  Both 'from' and 'to' must be 8-bit charsets.  The
+  result will be put into the buf buffer unless it is NULL, in which case it
+  will be allocated on demand.
+***************************************************************************/
+static char *convert_string(const char *text,
+                           const char *from,
+                           const char *to,
+                           char *buf, size_t bufsz)
+{
+#ifdef HAVE_ICONV
+  iconv_t cd = iconv_open(to, from);
+  size_t from_len = strlen(text) + 1, to_len;
+  bool alloc = (buf == NULL);
+
+  assert(is_init && from != NULL && to != NULL);
+  assert(text != NULL);
+
+  if (cd == (iconv_t) (-1)) {
+    freelog(LOG_ERROR,
+           _("Could not convert text from %s to %s: %s"),
+           from, to, strerror(errno));
+    /* The best we can do? */
+    if (alloc) {
+      return mystrdup(text);
+    } else {
+      my_snprintf(buf, bufsz, "%s", text);
+      return buf;
+    }
+  }
+
+  if (alloc) {
+    to_len = from_len;
+  } else {
+    to_len = bufsz;
+  }
+
+  do {
+    size_t flen = from_len, tlen = to_len, res;
+    const char *mytext = text;
+    char *myresult;
+
+    if (alloc) {
+      buf = fc_malloc(to_len);
+    }
+
+    myresult = buf;
+
+    /* Since we may do multiple translations, we may need to reset iconv
+     * in between. */
+    iconv(cd, NULL, NULL, NULL, NULL);
+
+    res = iconv(cd, (char**)&mytext, &flen, &myresult, &tlen);
+    if (res == (size_t) (-1)) {
+      if (errno != E2BIG) {
+       /* Invalid input. */
+       freelog(LOG_ERROR, "Invalid string conversion from %s to %s.",
+               from, to);
+       iconv_close(cd);
+       if (alloc) {
+         free(buf);
+         return mystrdup(text); /* The best we can do? */
+       } else {
+         my_snprintf(buf, bufsz, "%s", text);
+         return buf;
+       }
+      }
+    } else {
+      /* Success. */
+      iconv_close(cd);
+
+      /* There may be wasted space here, but there's nothing we can do
+       * about it. */
+      return buf;
+    }
+
+    if (alloc) {
+      /* Not enough space; try again. */
+      buf[to_len - 1] = 0;
+      freelog(LOG_VERBOSE, "   Result was '%s'.", buf);
+
+      free(buf);
+      to_len *= 2;
+    }
+  } while (alloc);
+
+  return buf;
+#else /* HAVE_ICONV */
+  if (buf) {
+    strncpy(buf, text, bufsz);
+    buf[bufsz - 1] = '\0';
+  } else {
+    return mystrdup(text);
+  }
+#endif /* HAVE_ICONV */
+}
+
+#define CONV_FUNC_MALLOC(src, dst)                                          \
+char *src ## _to_ ## dst ## _string_malloc(const char *text)                \
+{                                                                           \
+  return convert_string(text, (src ## _encoding),                          \
+                       (dst ## _encoding), NULL, 0);                       \
+}
+
+#define CONV_FUNC_BUFFER(src, dst)                                          \
+char *src ## _to_ ## dst ## _string_buffer(const char *text,                \
+                                          char *buf, size_t bufsz)         \
+{                                                                           \
+  return convert_string(text, (src ## _encoding),                          \
+                        (dst ## _encoding), buf, bufsz);                    \
+}
+
+#define CONV_FUNC_STATIC(src, dst)                                          \
+char *src ## _to_ ## dst ## _string_static(const char *text)                \
+{                                                                           \
+  (src ## _to_ ## dst ## _string_buffer)(text,                              \
+                                       convert_buffer,                     \
+                                       sizeof(convert_buffer));            \
+  return convert_buffer;                                                    \
+}
+
+CONV_FUNC_MALLOC(data, internal)
+CONV_FUNC_MALLOC(internal, data)
+
+static CONV_FUNC_BUFFER(internal, local)
+static CONV_FUNC_STATIC(internal, local)
+
+/***************************************************************************
+  Do a printf in the currently bound codeset.
+***************************************************************************/
+void fc_fprintf(FILE *stream, const char *format, ...)
+{
+  va_list ap;
+  char string[4096];
+  const char *output;
+
+  va_start(ap, format);
+  my_vsnprintf(string, sizeof(string), format, ap);
+  va_end(ap);
+
+  output = internal_to_local_string_static(string);
+
+  fputs(output, stream);
+  fflush(stream);
+}
Index: common/fciconv.h
===================================================================
RCS file: /home/freeciv/CVS/freeciv/common/fciconv.h,v
retrieving revision 1.1
diff -u -r1.1 fciconv.h
--- common/fciconv.h    26 Apr 2004 02:13:30 -0000      1.1
+++ common/fciconv.h    8 May 2004 07:13:50 -0000
@@ -0,0 +1,32 @@
+/********************************************************************** 
+ Freeciv - Copyright (C) 2003 - The Freeciv Project
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 2, or (at your option)
+   any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+***********************************************************************/
+#ifndef FC__FCICONV_H
+#define FC__FCICONV_H
+
+#include "shared.h"
+
+#define DEFAULT_DATA_ENCODING "ISO-8859-1"
+
+void init_character_encodings(char *internal_encoding);
+
+const char *get_data_encoding(void);
+const char *get_local_encoding(void);
+const char *get_internal_encoding(void);
+
+char *data_to_internal_string_malloc(const char *text);
+char *internal_to_data_string_malloc(const char *text);
+
+void fc_fprintf(FILE *stream, const char *format, ...)
+      fc__attribute((format (printf, 2, 3)));
+
+#endif /* FC__FCICONV_H */
Index: server/srv_main.c
===================================================================
RCS file: /home/freeciv/CVS/freeciv/server/srv_main.c,v
retrieving revision 1.160
diff -u -r1.160 srv_main.c
--- server/srv_main.c   2 May 2004 14:47:12 -0000       1.160
+++ server/srv_main.c   8 May 2004 07:13:50 -0000
@@ -49,6 +49,7 @@
 #include "city.h"
 #include "dataio.h"
 #include "events.h"
+#include "fciconv.h"
 #include "fcintl.h"
 #include "game.h"
 #include "log.h"
@@ -180,6 +181,9 @@
   /* mark as initialized */
   has_been_srv_init = TRUE;
 
+  /* init character encodings. */
+  init_character_encodings(DEFAULT_DATA_ENCODING);
+
   /* done */
   return;
 }
Index: utility/log.c
===================================================================
RCS file: /home/freeciv/CVS/freeciv/utility/log.c,v
retrieving revision 1.45
diff -u -r1.45 log.c
--- utility/log.c       10 Apr 2004 03:47:49 -0000      1.45
+++ utility/log.c       8 May 2004 07:13:50 -0000
@@ -26,6 +26,10 @@
 
 #include "log.h"
 
+/* FIXME: I guess fciconv must be in utility?  But it probably shouldn't
+ * be. */
+#include "../common/fciconv.h"
+
 static const char *log_filename;
 static log_callback_fn log_callback;
 
@@ -80,16 +84,16 @@
   }
   if (n == 0) {
     if (sscanf(level_str, "%d", &level) != 1) {
-      fprintf(stderr, _("Bad log level \"%s\".\n"), level_str);
+      fc_fprintf(stderr, _("Bad log level \"%s\".\n"), level_str);
       return -1;
     }
     if (level >= LOG_FATAL && level <= max_level) {
       return level;
     } else {
-      fprintf(stderr, _("Bad log level %d in \"%s\".\n"), level, level_str);
+      fc_fprintf(stderr, _("Bad log level %d in \"%s\".\n"), level, level_str);
       if (level == LOG_DEBUG && max_level < LOG_DEBUG) {
-       fprintf(stderr, _("Freeciv must be compiled with the DEBUG flag"
-                         " to use debug level %d.\n"), LOG_DEBUG);
+       fc_fprintf(stderr, _("Freeciv must be compiled with the DEBUG flag"
+                            " to use debug level %d.\n"), LOG_DEBUG);
       }
       return -1;
     }
@@ -99,12 +103,13 @@
   if (c[0] == ('0' + LOG_DEBUG) && c[1] == ':') {
     level = LOG_DEBUG;
     if (max_level < LOG_DEBUG) {
-      fprintf(stderr, _("Freeciv must be compiled with the DEBUG flag"
-                       " to use debug level %d.\n"), LOG_DEBUG);
+      fc_fprintf(stderr, _("Freeciv must be compiled with the DEBUG flag"
+                          " to use debug level %d.\n"), LOG_DEBUG);
       return -1;
     }
   } else {
-    fprintf(stderr, _("Badly formed log level argument \"%s\".\n"), level_str);
+    fc_fprintf(stderr, _("Badly formed log level argument \"%s\".\n"),
+              level_str);
     return -1;
   }
   logd_num_files = n;
@@ -115,7 +120,8 @@
   tok = strtok(dup, ":");
   
   if (!tok) {
-    fprintf(stderr, _("Badly formed log level argument \"%s\".\n"), level_str);
+    fc_fprintf(stderr, _("Badly formed log level argument \"%s\".\n"),
+              level_str);
     level = -1;
     goto out;
   }
@@ -132,20 +138,20 @@
       if (d && *pc != '\0' && d[1] != '\0') {
        d[0] = '\0';
        if (sscanf(pc, "%d", &logd_files[i].min) != 1) {
-         fprintf(stderr, _("Not an integer: '%s'\n"), pc);
+         fc_fprintf(stderr, _("Not an integer: '%s'\n"), pc);
          level = -1;
          goto out;
        }
        if (sscanf(d + 1, "%d", &logd_files[i].max) != 1) {
-         fprintf(stderr, _("Not an integer: '%s'\n"), d + 1);
+         fc_fprintf(stderr, _("Not an integer: '%s'\n"), d + 1);
          level = -1;
          goto out;
        }
       }
     }
     if(strlen(tok)==0) {
-      fprintf(stderr, _("Empty filename in log level argument \"%s\".\n"),
-             level_str);
+      fc_fprintf(stderr, _("Empty filename in log level argument \"%s\".\n"),
+                level_str);
       level = -1;
       goto out;
     }
@@ -155,7 +161,8 @@
   } while(tok);
 
   if (i!=logd_num_files) {
-    fprintf(stderr, _("Badly formed log level argument \"%s\".\n"), level_str);
+    fc_fprintf(stderr, _("Badly formed log level argument \"%s\".\n"),
+              level_str);
     level = -1;
     goto out;
   }
@@ -236,7 +243,7 @@
     log_callback(level, message);
   }
   if (log_filename || (!log_callback)) {
-    fprintf(fs, "%d: %s\n", level, message);
+    fc_fprintf(fs, "%d: %s\n", level, message);
     fflush(fs);
   }
 }
@@ -264,7 +271,7 @@
 
     if (log_filename) {
       if(!(fs=fopen(log_filename, "a"))) {
-       fprintf(stderr, _("Couldn't open logfile: %s for appending.\n"), 
+       fc_fprintf(stderr, _("Couldn't open logfile: %s for appending.\n"), 
                log_filename);
        exit(EXIT_FAILURE);
       }