[Freeciv-Dev] (PR#10982) server console character-encoding bugs

["Jason Short" <jdorje@xxxxxxxxxxxxxxxxxxxxx>]

<URL: http://rt.freeciv.org/Ticket/Display.html?id=10982 >

Here is a new patch that should fix all the server console problems.

- Console output is converted to the local encoding before being written.

- Console input is converted to the internal encoding before being 
passed to stdinhand_input.  (There are three different places this must 
be done, all in sernet.c.)

- The readline callback function generic generator must take a string in 
the local encoding and return a string in the local encoding, but must 
do its comparison in the internal encoding.  Thus two conversions are 
necessary.

- For the rest we let readline take care of itself.  History works but 
the persistent history file will be stored in the local encoding, so if 
you change your local encoding it may break.  I won't fix this.  (Did 
anyone know freeciv had persistent history?)

Attached also is a simple game to test it with.  Load this game and try:

- /list.  Check if output is correct.
- /aitoggle k<tab>.  Check if it works.
- /aitoggle <type in full name>.  Check if it works.
- Press <up> for history, and <enter>.  Check if it works.
- Anything else you can think of.

Try it with a non-utf-8 console, with and without the patch.  Without 
the patch it should fail (if it doesn't, you're not in a non-utf-8 
console).  With the patch it works AFAICT.

jason

Index: server/console.c
===================================================================
RCS file: /home/freeciv/CVS/freeciv/server/console.c,v
retrieving revision 1.23
diff -u -r1.23 console.c
--- server/console.c    1 May 2004 03:22:11 -0000       1.23
+++ server/console.c    11 Nov 2004 20:30:26 -0000
@@ -23,6 +23,7 @@
 #include <readline/readline.h>
 #endif
 
+#include "fciconv.h"
 #include "fcintl.h"
 #include "log.h"
 #include "support.h"
@@ -99,12 +100,12 @@
   va_end(args);
 
   if(console_prompt_is_showing) {
-    printf("\n");
+    fc_printf("\n");
   }
   if ((console_rfcstyle) && (rfc_status >= 0)) {
-    printf("%.3d %s", rfc_status, buf);
+    fc_printf("%.3d %s", rfc_status, buf);
   } else {
-    printf("%s", buf);
+    fc_printf("%s", buf);
   }
   console_prompt_is_showing = FALSE;
   return (int) strlen(buf);
@@ -136,12 +137,12 @@
 void con_puts(enum rfc_status rfc_status, const char *str)
 {
   if(console_prompt_is_showing) {
-    printf("\n");
+    fc_printf("\n");
   }
   if ((console_rfcstyle) && (rfc_status >= 0)) {
-    printf("%.3d %s\n", rfc_status, str);
+    fc_printf("%.3d %s\n", rfc_status, str);
   } else {
-    printf("%s\n", str);
+    fc_printf("%s\n", str);
   }
   console_prompt_is_showing = FALSE;
   con_update_prompt();
Index: server/sernet.c
===================================================================
RCS file: /home/freeciv/CVS/freeciv/server/sernet.c,v
retrieving revision 1.124
diff -u -r1.124 sernet.c
--- server/sernet.c     21 Oct 2004 20:27:29 -0000      1.124
+++ server/sernet.c     11 Nov 2004 20:30:26 -0000
@@ -60,6 +60,8 @@
 #include <winsock.h>
 #endif
 
+#include "fciconv.h"
+
 #include "capability.h"
 #include "dataio.h"
 #include "events.h"
@@ -158,6 +160,8 @@
 *****************************************************************************/
 static void handle_readline_input_callback(char *line)
 {
+  char *line_internal;
+
   if (no_input)
     return;
 
@@ -170,7 +174,9 @@
     add_history(line);
 
   con_prompt_enter();          /* just got an 'Enter' hit */
-  (void) handle_stdin_input((struct connection*)NULL, line, FALSE);
+  line_internal = local_to_internal_string_malloc(line);
+  (void) handle_stdin_input(NULL, line_internal, FALSE);
+  free(line_internal);
 
   readline_handled_input = TRUE;
 }
@@ -557,8 +563,11 @@
     
 #ifdef SOCKET_ZERO_ISNT_STDIN
     if (!no_input && (bufptr = my_read_console())) {
+      char bufptr_internal = local_to_internal_string_malloc(bufptr);
+
       con_prompt_enter();      /* will need a new prompt, regardless */
-      handle_stdin_input((struct connection *)NULL, bufptr, FALSE);
+      handle_stdin_input(NULL, bufptr_internal, FALSE);
+      free(bufptr_internal);
     }
 #else  /* !SOCKET_ZERO_ISNT_STDIN */
     if(!no_input && FD_ISSET(0, &readfs)) {    /* input from server operator */
@@ -572,6 +581,7 @@
 #else  /* !HAVE_LIBREADLINE */
       ssize_t didget;
       char buf[BUF_SIZE+1];
+      char *buf_internal;
       
       if((didget=read(0, buf, BUF_SIZE))==-1) {
        die("read from stdin failed");
@@ -583,7 +593,9 @@
 
       *(buf+didget)='\0';
       con_prompt_enter();      /* will need a new prompt, regardless */
-      handle_stdin_input((struct connection *)NULL, buf, FALSE);
+      buf_internal = local_to_internal_string_malloc(buf);
+      handle_stdin_input(NULL, buf_internal, FALSE);
+      free(buf_internal);
 #endif /* !HAVE_LIBREADLINE */
     }
     else
Index: server/stdinhand.c
===================================================================
RCS file: /home/freeciv/CVS/freeciv/server/stdinhand.c,v
retrieving revision 1.369
diff -u -r1.369 stdinhand.c
--- server/stdinhand.c  9 Nov 2004 18:00:31 -0000       1.369
+++ server/stdinhand.c  11 Nov 2004 20:30:27 -0000
@@ -29,6 +29,8 @@
 #endif
 #endif
 
+#include "fciconv.h"
+
 #include "astring.h"
 #include "capability.h"
 #include "events.h"
@@ -3985,13 +3987,21 @@
 {
   static int list_index, len;
   const char *name;
+  char *mytext = local_to_internal_string_malloc(text);
+
+  /* This function takes a string (text) in the local format and must return
+   * a string in the local format.  However comparisons are done against
+   * names that are in the internal format (UTF-8).  Thus we have to convert
+   * the text function from the local to the internal format before doing
+   * the comparison, and convert the string we return *back* to the
+   * local format when returning it. */
 
   /* If this is a new word to complete, initialize now.  This includes
      saving the length of TEXT for efficiency, and initializing the index
      variable to 0. */
   if (state == 0) {
     list_index = 0;
-    len = strlen (text);
+    len = strlen (mytext);
   }
 
   /* Return the next name which partially matches: */
@@ -3999,9 +4009,12 @@
     name = index2str(list_index);
     list_index++;
 
-    if (mystrncasecmp (name, text, len) == 0)
-      return mystrdup(name);
+    if (mystrncasecmp (name, mytext, len) == 0) {
+      free(mytext);
+      return internal_to_local_string_malloc(name);
+    }
   }
+  free(mytext);
 
   /* If no names matched, then return NULL. */
   return ((char *)NULL);
Index: utility/fciconv.c
===================================================================
RCS file: /home/freeciv/CVS/freeciv/utility/fciconv.c,v
retrieving revision 1.9
diff -u -r1.9 fciconv.c
--- utility/fciconv.c   15 Oct 2004 22:46:31 -0000      1.9
+++ utility/fciconv.c   11 Nov 2004 20:30:27 -0000
@@ -297,6 +297,8 @@
 
 CONV_FUNC_MALLOC(data, internal)
 CONV_FUNC_MALLOC(internal, data)
+CONV_FUNC_MALLOC(internal, local)
+CONV_FUNC_MALLOC(local, internal)
 
 static CONV_FUNC_BUFFER(internal, local)
 static CONV_FUNC_STATIC(internal, local)
Index: utility/fciconv.h
===================================================================
RCS file: /home/freeciv/CVS/freeciv/utility/fciconv.h,v
retrieving revision 1.6
diff -u -r1.6 fciconv.h
--- utility/fciconv.h   7 Oct 2004 14:41:05 -0000       1.6
+++ utility/fciconv.h   11 Nov 2004 20:30:27 -0000
@@ -27,7 +27,10 @@
 
 char *data_to_internal_string_malloc(const char *text);
 char *internal_to_data_string_malloc(const char *text);
+char *internal_to_local_string_malloc(const char *text);
+char *local_to_internal_string_malloc(const char *text);
 
+#define fc_printf(...) fc_fprintf(stdout, __VA_ARGS__)
 void fc_fprintf(FILE *stream, const char *format, ...)
       fc__attribute((format (printf, 2, 3)));