[Freeciv-Dev] Re: (PR#11068) savegame file compression

["Marko Lindqvist" <marko.lindqvist@xxxxxxxxxxx>]

<URL: http://bugs.freeciv.org/Ticket/Display.html?id=11068 >

Marko Lindqvist wrote:
> 
>  I consider this ready for commit.

  - ML

diff -Nurd -X.diff_ignore freeciv/client/connectdlg_common.c 
freeciv/client/connectdlg_common.c
--- freeciv/client/connectdlg_common.c  2005-06-30 19:00:40.375000000 +0300
+++ freeciv/client/connectdlg_common.c  2005-06-30 19:43:02.609375000 +0300
@@ -39,6 +39,7 @@
 
 #include "capability.h"
 #include "fcintl.h"
+#include "ioz.h"
 #include "log.h"
 #include "mem.h"
 #include "netintf.h"
@@ -436,7 +437,7 @@
 
     section_file_init(&file);
     secfile_insert_str(&file, req.token, "challenge.token");
-    if (!section_file_save(&file, challenge_fullname, 0)) {
+    if (!section_file_save(&file, challenge_fullname, 0, FZ_PLAIN)) {
       freelog(LOG_ERROR, "Couldn't write token to temporary file: %s",
              challenge_fullname);
     }
diff -Nurd -X.diff_ignore freeciv/client/options.c freeciv/client/options.c
--- freeciv/client/options.c    2005-06-30 19:02:16.781250000 +0300
+++ freeciv/client/options.c    2005-06-30 19:43:02.703125000 +0300
@@ -21,6 +21,7 @@
 #include "events.h"
 #include "fcintl.h"
 #include "game.h"
+#include "ioz.h"
 #include "log.h"
 #include "mem.h"
 #include "registry.h"
@@ -602,7 +603,7 @@
   }
 
   /* save to disk */
-  if (!section_file_save(&sf, name, 0)) {
+  if (!section_file_save(&sf, name, 0, FZ_PLAIN)) {
     my_snprintf(output_buffer, sizeof(output_buffer),
                _("Save failed, cannot write to file %s"), name);
   } else {
diff -Nurd -X.diff_ignore freeciv/common/game.c freeciv/common/game.c
--- freeciv/common/game.c       2005-06-30 19:02:27.859375000 +0300
+++ freeciv/common/game.c       2005-06-30 19:43:02.796875000 +0300
@@ -24,6 +24,7 @@
 #include "fcintl.h"
 #include "government.h"
 #include "idex.h"
+#include "ioz.h"
 #include "log.h"
 #include "map.h"
 #include "mem.h"
@@ -234,10 +235,13 @@
   game.info.watchtower_extra_vision = GAME_DEFAULT_WATCHTOWER_EXTRA_VISION;
   game.info.allowed_city_names = GAME_DEFAULT_ALLOWED_CITY_NAMES;
   game.info.save_nturns   = 10;
-#ifdef HAVE_LIBZ
   game.info.save_compress_level = GAME_DEFAULT_COMPRESS_LEVEL;
+#ifdef HAVE_LIBBZ2
+  game.info.save_compress_type = FZ_BZIP2;
+#elif defined(HAVE_LIBZ)
+  game.info.save_compress_type = FZ_ZLIB;
 #else
-  game.info.save_compress_level = GAME_NO_COMPRESS_LEVEL;
+  game.info.save_compress_type = FZ_PLAIN;
 #endif
   game.info.government_when_anarchy = G_MAGIC;   /* flag */
 
diff -Nurd -X.diff_ignore freeciv/common/game.h freeciv/common/game.h
--- freeciv/common/game.h       2005-06-30 19:02:27.984375000 +0300
+++ freeciv/common/game.h       2005-06-30 19:43:02.890625000 +0300
@@ -341,9 +341,8 @@
 #define GAME_DEFAULT_ALLOW_TAKE      "HAhadOo"
 
 #define GAME_DEFAULT_COMPRESS_LEVEL 6    /* if we have compression */
-#define GAME_MIN_COMPRESS_LEVEL     0
+#define GAME_MIN_COMPRESS_LEVEL     1
 #define GAME_MAX_COMPRESS_LEVEL     9
-#define GAME_NO_COMPRESS_LEVEL      0
 
 #define GAME_DEFAULT_WATCHTOWER_EXTRA_VISION 2
 #define GAME_MIN_WATCHTOWER_EXTRA_VISION 0
diff -Nurd -X.diff_ignore freeciv/common/packets.def freeciv/common/packets.def
--- freeciv/common/packets.def  2005-06-30 19:02:30.687500000 +0300
+++ freeciv/common/packets.def  2005-06-30 19:43:03.078125000 +0300
@@ -437,6 +437,7 @@
 
   UINT8 save_nturns;
   UINT8 save_compress_level;
+  UINT8 save_compress_type;
 
   STRING start_units[MAX_LEN_STARTUNIT];
   
diff -Nurd -X.diff_ignore freeciv/configure.ac freeciv/configure.ac
--- freeciv/configure.ac        2005-06-30 19:02:35.500000000 +0300
+++ freeciv/configure.ac        2005-06-30 19:43:03.171875000 +0300
@@ -344,6 +344,10 @@
   AC_MSG_ERROR([You need the gzip program for compilation.])
 fi
 
+dnl Check for libbzip2
+AC_CHECK_LIB(bz2, BZ2_bzReadOpen)
+AC_CHECK_HEADERS(bzlib.h)
+
 dnl Check and compile ftwl
 if test "$ftwl" = x11 ; then
      FTWL_CFLAGS=`freetype-config --cflags`
diff -Nurd -X.diff_ignore freeciv/server/settings.c freeciv/server/settings.c
--- freeciv/server/settings.c   2005-06-30 19:05:34.437500000 +0300
+++ freeciv/server/settings.c   2005-06-30 19:43:03.390625000 +0300
@@ -17,6 +17,7 @@
 
 #include "fcintl.h"
 #include "game.h"
+#include "ioz.h"
 #include "log.h"
 
 #include "map.h"
@@ -953,31 +954,30 @@
             "turns. Zero means never auto-save."), NULL, 
          0, 200, 10)
 
-  /* Could undef entire option if !HAVE_LIBZ, but this way users get to see
-   * what they're missing out on if they didn't compile with zlib?  --dwp
-   */
-#ifdef HAVE_LIBZ
   GEN_INT("compress", game.info.save_compress_level,
          SSET_META, SSET_INTERNAL, SSET_RARE, SSET_SERVER_ONLY,
          N_("Savegame compression level"),
          N_("If non-zero, saved games will be compressed using zlib "
-            "(gzip format). Larger values will give better "
-            "compression but take longer. If the maximum is zero "
-            "this server was not compiled to use zlib."), NULL,
+            "(gzip format) or bzip2. Larger values will give better "
+            "compression but take longer."), NULL,
 
          GAME_MIN_COMPRESS_LEVEL, GAME_MAX_COMPRESS_LEVEL,
          GAME_DEFAULT_COMPRESS_LEVEL)
-#else
-  GEN_INT("compress", game.info.save_compress_level,
-         SSET_META, SSET_INTERNAL, SSET_RARE, SSET_SERVER_ONLY,
-         N_("Savegame compression level"),
-         N_("If non-zero, saved games will be compressed using zlib "
-            "(gzip format). Larger values will give better "
-            "compression but take longer. If the maximum is zero "
-            "this server was not compiled to use zlib."), NULL, 
 
-         GAME_NO_COMPRESS_LEVEL, GAME_NO_COMPRESS_LEVEL, 
-         GAME_NO_COMPRESS_LEVEL)
+  GEN_INT("compresstype", game.info.save_compress_type,
+          SSET_META, SSET_INTERNAL, SSET_RARE, SSET_SERVER_ONLY,
+          N_("Savegame compression algorithm"),
+          N_("Compression library to use for savegames.\n"
+             " 0 - none\n"
+             " 1 - zlib (gzip format)\n"
+             " 2 - bzip2\n"
+             "Not all servers support all compression methods."), NULL,
+#if !defined(HAVE_LIBBZ2) && !defined(HAVE_LIBZ)
+          FZ_PLAIN, FZ_PLAIN, FZ_PLAIN)
+#elif !defined(HAVE_LIBBZ2) && defined(HAVE_LIBZ)
+          FZ_PLAIN, FZ_ZLIB, FZ_ZLIB)
+#else
+          FZ_PLAIN, FZ_BZIP2, FZ_BZIP2)
 #endif
 
   GEN_STRING("savename", game.save_name,
diff -Nurd -X.diff_ignore freeciv/server/srv_main.c freeciv/server/srv_main.c
--- freeciv/server/srv_main.c   2005-06-30 19:05:34.625000000 +0300
+++ freeciv/server/srv_main.c   2005-06-30 19:47:26.484375000 +0300
@@ -743,8 +743,26 @@
   sz_strlcat(filename, ".sav");
 
   if (game.info.save_compress_level > 0) {
-    /* Append ".gz" to filename. */
-    sz_strlcat(filename, ".gz");
+    switch (game.info.save_compress_type) {
+#ifdef HAVE_LIBZ
+    case FZ_ZLIB:
+      /* Append ".gz" to filename. */
+      sz_strlcat(filename, ".gz");
+      break;
+#endif
+#ifdef HAVE_LIBBZ2
+    case FZ_BZIP2:
+      /* Append ".bz2" to filename. */
+      sz_strlcat(filename, ".bz2");
+      break;
+#endif
+    case FZ_PLAIN:
+      break;
+    default:
+      freelog(LOG_NORMAL, "Unsupported compression type %d",
+              game.info.save_compress_type);
+      break;
+    }
   }
 
   if (!path_is_absolute(filename)) {
@@ -761,7 +779,8 @@
     sz_strlcpy(filename, tmpname);
   }
 
-  if(!section_file_save(&file, filename, game.info.save_compress_level))
+  if (!section_file_save(&file, filename, game.info.save_compress_level,
+                         game.info.save_compress_type))
     con_write(C_FAIL, _("Failed saving game as %s"), filename);
   else
     con_write(C_OK, _("Game saved as %s"), filename);
diff -Nurd -X.diff_ignore freeciv/server/userdb/user_db.c 
freeciv/server/userdb/user_db.c
--- freeciv/server/userdb/user_db.c     2005-06-30 19:05:35.703125000 +0300
+++ freeciv/server/userdb/user_db.c     2005-06-30 19:43:03.578125000 +0300
@@ -159,7 +159,7 @@
   secfile_insert_int(&new, new_num, "db.num_entries");
 
   /* save to file */
-  if (!section_file_save(&new, FC_USER_DATABASE, 0)) {
+  if (!section_file_save(&new, FC_USER_DATABASE, 0, 0)) {
     result = USER_DB_ERROR;
   } else {
     result = USER_DB_SUCCESS;
diff -Nurd -X.diff_ignore freeciv/utility/ioz.c freeciv/utility/ioz.c
--- freeciv/utility/ioz.c       2005-06-30 19:05:40.187500000 +0300
+++ freeciv/utility/ioz.c       2005-06-30 19:43:03.671875000 +0300
@@ -33,6 +33,7 @@
 #include <config.h>
 #endif
 
+#include <assert.h>
 #include <errno.h>
 #include <stdarg.h>
 #include <stdio.h>
@@ -42,6 +43,10 @@
 #include <zlib.h>
 #endif
 
+#ifdef HAVE_BZLIB_H
+#include <bzlib.h>
+#endif
+
 #include "log.h"
 #include "mem.h"
 #include "shared.h"
@@ -49,13 +54,26 @@
 
 #include "ioz.h"
 
+#ifdef HAVE_LIBBZ2
+struct bzip2_struct {
+  BZFILE *file;
+  FILE *plain;
+  int error;
+  int firstbyte;
+};
+#endif
+
 struct fz_FILE_s {
   enum fz_method method;
+  char mode;
   union {
     FILE *plain;               /* FZ_PLAIN */
 #ifdef HAVE_LIBZ
     gzFile zlib;               /* FZ_ZLIB */
 #endif
+#ifdef HAVE_LIBBZ2
+    struct bzip2_struct bz2;
+#endif
   } u;
 };
 
@@ -83,18 +101,58 @@
 
   if (mode[0] == 'w') {
     /* Writing: */
-    if (compress_level == 0) {
-      method = FZ_PLAIN;
-    }
+    fp->mode = 'w';
+
 #ifndef HAVE_LIBZ
-    /* In theory this shouldn't happen, but check anyway. */
     if (method == FZ_ZLIB) {
       freelog(LOG_NORMAL, "Not compiled with zlib support, reverting to 
plain.");
       method = FZ_PLAIN;
     }
 #endif
+#ifndef HAVE_LIBBZ2
+    if (method == FZ_BZIP2) {
+      freelog(LOG_NORMAL, "Not compiled with bzib2 support, reverting to 
plain.");
+      method = FZ_PLAIN;
+    }
+#endif
+
   } else {
     /* Reading: ignore specified method and try best: */
+    fp->mode = 'r';
+#ifdef HAVE_LIBBZ2
+    /* Try to open as bzip2 file */
+    method = FZ_BZIP2;
+    sz_strlcat(mode,"b");
+    fp->u.bz2.plain = fopen(filename, mode);
+    if (fp->u.bz2.plain) {
+      fp->u.bz2.file = BZ2_bzReadOpen(&fp->u.bz2.error, fp->u.bz2.plain, 1, 0,
+                                      NULL, 0);
+    }
+    if (!fp->u.bz2.file) {
+      if (fp->u.bz2.plain) {
+        fclose(fp->u.bz2.plain);
+      }
+      free(fp);
+      return NULL;
+    } else {
+      /* Try to read first byte out of stream so we can figure out if this
+         really is bzip2 file or not. Store byte for later use */
+      char tmp;
+      BZ2_bzRead(&fp->u.bz2.error, fp->u.bz2.file, &tmp, 1);
+      if (fp->u.bz2.error != BZ_DATA_ERROR_MAGIC) { /* bzip2 file */
+        if (fp->u.bz2.error != BZ_OK) {
+          fclose(fp->u.bz2.plain);
+          free(fp);
+          return NULL;
+        }
+        fp->method = FZ_BZIP2;
+        fp->u.bz2.firstbyte = tmp;
+        return fp;
+      }
+      fclose(fp->u.bz2.plain);
+    }
+#endif
+
 #ifdef HAVE_LIBZ
     method = FZ_ZLIB;
 #else
@@ -105,6 +163,26 @@
   fp->method = method;
 
   switch (fp->method) {
+#ifdef HAVE_LIBBZ2
+  case FZ_BZIP2:
+    /*  bz2 files are binary files, so we should add "b" to mode! */
+    sz_strlcat(mode,"b");
+    fp->u.bz2.plain = fopen(filename, mode);
+    if (fp->u.bz2.plain) {
+      /*  Open for read handled earlier */
+      assert(mode[0] == 'w');
+      fp->u.bz2.file = BZ2_bzWriteOpen(&fp->u.bz2.error, fp->u.bz2.plain,
+                                       compress_level, 1, 15);
+    }
+    if (!fp->u.bz2.file) {
+      if (fp->u.bz2.plain) {
+        fclose(fp->u.bz2.plain);
+      }
+      free(fp);
+      fp = NULL;
+    }
+    break;
+#endif
 #ifdef HAVE_LIBZ
   case FZ_ZLIB:
     /*  gz files are binary files, so we should add "b" to mode! */
@@ -163,6 +241,21 @@
   int retval = 0;
   
   switch(fp->method) {
+#ifdef HAVE_LIBBZ2
+  case FZ_BZIP2:
+    if(fp->mode == 'w') {
+      BZ2_bzWriteClose(&fp->u.bz2.error, fp->u.bz2.file, 0, NULL, NULL);
+    } else {
+      BZ2_bzReadClose(&fp->u.bz2.error, fp->u.bz2.file);
+    }
+    if(fp->u.bz2.error == BZ_OK) {
+      retval = 0;
+    } else {
+      retval = 1;
+    }
+    fclose(fp->u.bz2.plain);
+    break;
+#endif
 #ifdef HAVE_LIBZ
   case FZ_ZLIB:
     retval = gzclose(fp->u.zlib);
@@ -189,9 +282,38 @@
 ***************************************************************/
 char *fz_fgets(char *buffer, int size, fz_FILE *fp)
 {
-  char *retval = 0;
+  char *retval = NULL;
   
-  switch(fp->method) {
+  switch (fp->method) {
+#ifdef HAVE_LIBBZ2
+  case FZ_BZIP2:
+    {
+      int i = 0;
+      /* See if first byte is already read and stored */
+      if (fp->u.bz2.firstbyte >= 0) {
+        buffer[0] = fp->u.bz2.firstbyte;
+        fp->u.bz2.firstbyte = -1;
+        i++;
+      } else {
+        BZ2_bzRead(&fp->u.bz2.error, fp->u.bz2.file, buffer + i, 1);
+        i++;
+      }
+      /* Leave space for trailing zero */
+      for (; i < size - 1 && fp->u.bz2.error == BZ_OK && buffer[i - 1] != '\n' 
;
+           i++) {
+        BZ2_bzRead(&fp->u.bz2.error, fp->u.bz2.file, buffer + i, 1);
+      }
+      if (fp->u.bz2.error != BZ_OK &&
+          (fp->u.bz2.error != BZ_STREAM_END ||
+          i == 0)) {
+        retval = NULL;
+      } else {
+        retval = buffer;
+      }
+      buffer[i] = '\0';
+      break;
+    }
+#endif
 #ifdef HAVE_LIBZ
   case FZ_ZLIB:
     retval = gzgets(fp->u.zlib, buffer, size);
@@ -224,8 +346,27 @@
   int retval = 0;
   
   va_start(ap, format);
-  
-  switch(fp->method) {
+
+  switch (fp->method) {
+#ifdef HAVE_LIBBZ2
+  case FZ_BZIP2:
+    {
+      char buffer[65536];
+      int num;
+      num = my_vsnprintf(buffer, sizeof(buffer), format, ap);
+      if (num == -1) {
+         freelog(LOG_ERROR, "Too much data: truncated in fz_fprintf (%lu)",
+                 (unsigned long) sizeof(buffer));
+      }
+      BZ2_bzWrite(&fp->u.bz2.error, fp->u.bz2.file, buffer, strlen(buffer));
+      if (fp->u.bz2.error != BZ_OK) {
+        retval = 0;
+      } else {
+        retval = strlen(buffer);
+      }
+    }
+    break;
+#endif
 #ifdef HAVE_LIBZ
   case FZ_ZLIB:
     {
@@ -258,8 +399,18 @@
 int fz_ferror(fz_FILE *fp)
 {
   int retval = 0;
-  
-  switch(fp->method) {
+
+  switch (fp->method) {
+#ifdef HAVE_LIBBZ2
+  case FZ_BZIP2:
+    if (fp->u.bz2.error != BZ_OK &&
+        fp->u.bz2.error != BZ_STREAM_END) {
+      retval = 1;
+    } else {
+      retval = 0;
+    }
+    break;
+#endif
 #ifdef HAVE_LIBZ
   case FZ_ZLIB:
     (void) gzerror(fp->u.zlib, &retval);       /* ignore string result here */
@@ -291,6 +442,16 @@
   const char *retval = 0;
   
   switch(fp->method) {
+#ifdef HAVE_LIBBZ2
+  case FZ_BZIP2:
+    {
+      static char bzip2error[50];
+      my_snprintf(bzip2error, sizeof(bzip2error), "Bzip2 error %d",
+                  fp->u.bz2.error);
+      retval = bzip2error;
+      break;
+    }
+#endif
 #ifdef HAVE_LIBZ
   case FZ_ZLIB:
     {
diff -Nurd -X.diff_ignore freeciv/utility/ioz.h freeciv/utility/ioz.h
--- freeciv/utility/ioz.h       2005-06-30 19:05:40.187500000 +0300
+++ freeciv/utility/ioz.h       2005-06-30 19:43:03.765625000 +0300
@@ -27,7 +27,7 @@
 typedef struct fz_FILE_s fz_FILE;
 
 /* (possibly) supported methods (depending on config.h) */
-enum fz_method { FZ_PLAIN, FZ_ZLIB, FZ_LAST };
+enum fz_method { FZ_PLAIN = 0, FZ_ZLIB = 1, FZ_BZIP2 = 2, FZ_LAST };
 #define FZ_NOT_USED FZ_LAST
 
 fz_FILE *fz_from_file(const char *filename, const char *in_mode,
diff -Nurd -X.diff_ignore freeciv/utility/registry.c freeciv/utility/registry.c
--- freeciv/utility/registry.c  2005-06-30 19:05:40.515625000 +0300
+++ freeciv/utility/registry.c  2005-06-30 19:43:03.859375000 +0300
@@ -654,7 +654,8 @@
 **************************************************************************/
 bool section_file_save(struct section_file *my_section_file,
                        const char *filename,
-                      int compression_level)
+                       int compression_level,
+                       enum fz_method compression_method)
 {
   char real_filename[1024];
   fz_FILE *fs;
@@ -663,7 +664,7 @@
   int i;
   
   interpret_tilde(real_filename, sizeof(real_filename), filename);
-  fs = fz_from_file(real_filename, "w", FZ_ZLIB, compression_level);
+  fs = fz_from_file(real_filename, "w", compression_method, compression_level);
 
   if (!fs)
     return FALSE;
diff -Nurd -X.diff_ignore freeciv/utility/registry.h freeciv/utility/registry.h
--- freeciv/utility/registry.h  2005-06-30 19:05:40.531250000 +0300
+++ freeciv/utility/registry.h  2005-06-30 19:43:04.046875000 +0300
@@ -37,7 +37,8 @@
 bool section_file_load_from_stream(struct section_file *my_section_file,
                                   fz_FILE * stream);
 bool section_file_save(struct section_file *my_section_file,
-                     const char *filename, int compression_level);
+                       const char *filename, int compression_level,
+                       enum fz_method compression_method);
 void section_file_free(struct section_file *file);
 void section_file_check_unused(struct section_file *file,
                               const char *filename);