[Freeciv-Dev] Re: (PR#11068) savegame file compression

["Marko Lindqvist" <marko.lindqvist@xxxxxxxxxxx>]

<URL: http://bugs.freeciv.org/Ticket/Display.html?id=11068 >


  Pinpointed optimal bzip2 parameters for compressing freeciv saves. 
Resulting files are identical, but compression is much faster than with 
previous version.
  Also style improvements.

  Mainly tweaked workFactor, last parameter to BZ2_bzWriteOpen(). One 
may give it values between 1-250. Default is 30, but I found out that 15 
works much better for freeciv saves. If save format changes drastically, 
one may want to pinpoint new optimal value for that.


  - ML

diff -Nurd -X.diff_ignore freeciv/client/connectdlg_common.c 
freeciv/client/connectdlg_common.c
--- freeciv/client/connectdlg_common.c  2005-06-27 00:51:06.828125000 +0300
+++ freeciv/client/connectdlg_common.c  2005-06-27 14:09:41.468750000 +0300
@@ -39,6 +39,7 @@
 
 #include "capability.h"
 #include "fcintl.h"
+#include "ioz.h"
 #include "log.h"
 #include "mem.h"
 #include "netintf.h"
@@ -436,7 +437,7 @@
 
     section_file_init(&file);
     secfile_insert_str(&file, req.token, "challenge.token");
-    if (!section_file_save(&file, challenge_fullname, 0)) {
+    if (!section_file_save(&file, challenge_fullname, 0, FZ_PLAIN)) {
       freelog(LOG_ERROR, "Couldn't write token to temporary file: %s",
              challenge_fullname);
     }
diff -Nurd -X.diff_ignore freeciv/client/options.c freeciv/client/options.c
--- freeciv/client/options.c    2005-06-27 00:51:06.828125000 +0300
+++ freeciv/client/options.c    2005-06-27 14:09:41.468750000 +0300
@@ -21,6 +21,7 @@
 #include "events.h"
 #include "fcintl.h"
 #include "game.h"
+#include "ioz.h"
 #include "log.h"
 #include "mem.h"
 #include "registry.h"
@@ -602,7 +603,7 @@
   }
 
   /* save to disk */
-  if (!section_file_save(&sf, name, 0)) {
+  if (!section_file_save(&sf, name, 0, FZ_PLAIN)) {
     my_snprintf(output_buffer, sizeof(output_buffer),
                _("Save failed, cannot write to file %s"), name);
   } else {
diff -Nurd -X.diff_ignore freeciv/common/game.c freeciv/common/game.c
--- freeciv/common/game.c       2005-06-27 00:51:06.843750000 +0300
+++ freeciv/common/game.c       2005-06-27 14:09:41.468750000 +0300
@@ -24,6 +24,7 @@
 #include "fcintl.h"
 #include "government.h"
 #include "idex.h"
+#include "ioz.h"
 #include "log.h"
 #include "map.h"
 #include "mem.h"
@@ -234,10 +235,13 @@
   game.info.watchtower_extra_vision = GAME_DEFAULT_WATCHTOWER_EXTRA_VISION;
   game.info.allowed_city_names = GAME_DEFAULT_ALLOWED_CITY_NAMES;
   game.info.save_nturns   = 10;
-#ifdef HAVE_LIBZ
   game.info.save_compress_level = GAME_DEFAULT_COMPRESS_LEVEL;
+#ifdef HAVE_LIBBZ2
+  game.info.save_compress_type = FZ_BZIP2;
+#elif defined(HAVE_LIBZ)
+  game.info.save_compress_type = FZ_ZLIB;
 #else
-  game.info.save_compress_level = GAME_NO_COMPRESS_LEVEL;
+  game.info.save_compress_type = FZ_PLAIN;
 #endif
   game.info.government_when_anarchy = G_MAGIC;   /* flag */
 
diff -Nurd -X.diff_ignore freeciv/common/game.h freeciv/common/game.h
--- freeciv/common/game.h       2005-06-27 00:51:06.843750000 +0300
+++ freeciv/common/game.h       2005-06-27 14:09:41.468750000 +0300
@@ -341,9 +341,8 @@
 #define GAME_DEFAULT_ALLOW_TAKE      "HAhadOo"
 
 #define GAME_DEFAULT_COMPRESS_LEVEL 6    /* if we have compression */
-#define GAME_MIN_COMPRESS_LEVEL     0
+#define GAME_MIN_COMPRESS_LEVEL     1
 #define GAME_MAX_COMPRESS_LEVEL     9
-#define GAME_NO_COMPRESS_LEVEL      0
 
 #define GAME_DEFAULT_WATCHTOWER_EXTRA_VISION 2
 #define GAME_MIN_WATCHTOWER_EXTRA_VISION 0
diff -Nurd -X.diff_ignore freeciv/common/packets.def freeciv/common/packets.def
--- freeciv/common/packets.def  2005-06-27 00:51:06.859375000 +0300
+++ freeciv/common/packets.def  2005-06-27 14:09:41.484375000 +0300
@@ -430,6 +430,7 @@
 
   UINT8 save_nturns;
   UINT8 save_compress_level;
+  UINT8 save_compress_type;
 
   STRING start_units[MAX_LEN_STARTUNIT];
   
diff -Nurd -X.diff_ignore freeciv/configure.ac freeciv/configure.ac
--- freeciv/configure.ac        2005-06-27 00:51:06.875000000 +0300
+++ freeciv/configure.ac        2005-06-27 14:09:41.484375000 +0300
@@ -344,6 +344,10 @@
   AC_MSG_ERROR([You need the gzip program for compilation.])
 fi
 
+dnl Check for libbzip2
+AC_CHECK_LIB(bz2, BZ2_bzReadOpen)
+AC_CHECK_HEADERS(bzlib.h)
+
 dnl Check and compile ftwl
 if test "$ftwl" = x11 ; then
      FTWL_CFLAGS=`freetype-config --cflags`
diff -Nurd -X.diff_ignore freeciv/server/settings.c freeciv/server/settings.c
--- freeciv/server/settings.c   2005-06-27 00:51:07.000000000 +0300
+++ freeciv/server/settings.c   2005-06-27 14:09:41.484375000 +0300
@@ -17,6 +17,7 @@
 
 #include "fcintl.h"
 #include "game.h"
+#include "ioz.h"
 #include "log.h"
 
 #include "map.h"
@@ -953,31 +954,30 @@
             "turns. Zero means never auto-save."), NULL, 
          0, 200, 10)
 
-  /* Could undef entire option if !HAVE_LIBZ, but this way users get to see
-   * what they're missing out on if they didn't compile with zlib?  --dwp
-   */
-#ifdef HAVE_LIBZ
   GEN_INT("compress", game.info.save_compress_level,
          SSET_META, SSET_INTERNAL, SSET_RARE, SSET_SERVER_ONLY,
          N_("Savegame compression level"),
          N_("If non-zero, saved games will be compressed using zlib "
-            "(gzip format). Larger values will give better "
-            "compression but take longer. If the maximum is zero "
-            "this server was not compiled to use zlib."), NULL,
+            "(gzip format) or bzip2. Larger values will give better "
+            "compression but take longer."), NULL,
 
          GAME_MIN_COMPRESS_LEVEL, GAME_MAX_COMPRESS_LEVEL,
          GAME_DEFAULT_COMPRESS_LEVEL)
-#else
-  GEN_INT("compress", game.info.save_compress_level,
-         SSET_META, SSET_INTERNAL, SSET_RARE, SSET_SERVER_ONLY,
-         N_("Savegame compression level"),
-         N_("If non-zero, saved games will be compressed using zlib "
-            "(gzip format). Larger values will give better "
-            "compression but take longer. If the maximum is zero "
-            "this server was not compiled to use zlib."), NULL, 
 
-         GAME_NO_COMPRESS_LEVEL, GAME_NO_COMPRESS_LEVEL, 
-         GAME_NO_COMPRESS_LEVEL)
+  GEN_INT("compresstype", game.info.save_compress_type,
+          SSET_META, SSET_INTERNAL, SSET_RARE, SSET_SERVER_ONLY,
+          N_("Savegame compression algorithm"),
+          N_("Compression library to use for savegames.\n"
+             " 0 - none\n"
+             " 1 - zlib (gzip format)\n"
+             " 2 - bzip2\n"
+             "Not all servers support all compression methods."), NULL,
+#if !defined(HAVE_LIBBZ2) && !defined(HAVE_LIBZ)
+          FZ_PLAIN, FZ_PLAIN, FZ_PLAIN)
+#elif !defined(HAVE_LIBBZ2) && defined(HAVE_LIBZ)
+          FZ_PLAIN, FZ_ZLIB, FZ_ZLIB)
+#else
+          FZ_PLAIN, FZ_BZIP2, FZ_BZIP2)
 #endif
 
   GEN_STRING("savename", game.save_name,
diff -Nurd -X.diff_ignore freeciv/server/srv_main.c freeciv/server/srv_main.c
--- freeciv/server/srv_main.c   2005-06-27 00:51:07.000000000 +0300
+++ freeciv/server/srv_main.c   2005-06-27 14:12:36.156250000 +0300
@@ -743,8 +743,22 @@
   sz_strlcat(filename, ".sav");
 
   if (game.info.save_compress_level > 0) {
-    /* Append ".gz" to filename. */
-    sz_strlcat(filename, ".gz");
+    switch (game.info.save_compress_type) {
+    case FZ_ZLIB:
+      /* Append ".gz" to filename. */
+      sz_strlcat(filename, ".gz");
+      break;
+    case FZ_BZIP2:
+      /* Append ".bz2" to filename. */
+      sz_strlcat(filename, ".bz2");
+      break;
+    case FZ_PLAIN:
+      break;
+    default:
+      freelog(LOG_ERROR, "Unknown compression type %d",
+              game.info.save_compress_type);
+      break;
+    }
   }
 
   if (!path_is_absolute(filename)) {
@@ -761,7 +775,8 @@
     sz_strlcpy(filename, tmpname);
   }
 
-  if(!section_file_save(&file, filename, game.info.save_compress_level))
+  if (!section_file_save(&file, filename, game.info.save_compress_level,
+                         game.info.save_compress_type))
     con_write(C_FAIL, _("Failed saving game as %s"), filename);
   else
     con_write(C_OK, _("Game saved as %s"), filename);
diff -Nurd -X.diff_ignore freeciv/server/userdb/user_db.c 
freeciv/server/userdb/user_db.c
--- freeciv/server/userdb/user_db.c     2005-06-27 00:51:07.031250000 +0300
+++ freeciv/server/userdb/user_db.c     2005-06-27 14:09:41.500000000 +0300
@@ -159,7 +159,7 @@
   secfile_insert_int(&new, new_num, "db.num_entries");
 
   /* save to file */
-  if (!section_file_save(&new, FC_USER_DATABASE, 0)) {
+  if (!section_file_save(&new, FC_USER_DATABASE, 0, 0)) {
     result = USER_DB_ERROR;
   } else {
     result = USER_DB_SUCCESS;
diff -Nurd -X.diff_ignore freeciv/utility/ioz.c freeciv/utility/ioz.c
--- freeciv/utility/ioz.c       2005-06-27 00:51:07.031250000 +0300
+++ freeciv/utility/ioz.c       2005-06-27 14:17:14.015625000 +0300
@@ -33,6 +33,7 @@
 #include <config.h>
 #endif
 
+#include <assert.h>
 #include <errno.h>
 #include <stdarg.h>
 #include <stdio.h>
@@ -42,6 +43,10 @@
 #include <zlib.h>
 #endif
 
+#ifdef HAVE_BZLIB_H
+#include <bzlib.h>
+#endif
+
 #include "log.h"
 #include "mem.h"
 #include "shared.h"
@@ -49,13 +54,26 @@
 
 #include "ioz.h"
 
+#ifdef HAVE_LIBBZ2
+struct bzip2_struct {
+  BZFILE *file;
+  FILE *plain;
+  int error;
+  int firstbyte;
+};
+#endif
+
 struct fz_FILE_s {
   enum fz_method method;
+  char mode;
   union {
     FILE *plain;               /* FZ_PLAIN */
 #ifdef HAVE_LIBZ
     gzFile zlib;               /* FZ_ZLIB */
 #endif
+#ifdef HAVE_LIBBZ2
+    struct bzip2_struct bz2;
+#endif
   } u;
 };
 
@@ -83,18 +101,58 @@
 
   if (mode[0] == 'w') {
     /* Writing: */
-    if (compress_level == 0) {
-      method = FZ_PLAIN;
-    }
+    fp->mode = 'w';
+
 #ifndef HAVE_LIBZ
-    /* In theory this shouldn't happen, but check anyway. */
     if (method == FZ_ZLIB) {
       freelog(LOG_NORMAL, "Not compiled with zlib support, reverting to 
plain.");
       method = FZ_PLAIN;
     }
 #endif
+#ifndef HAVE_LIBBZ2
+    if (method == FZ_BZIP2) {
+      freelog(LOG_NORMAL, "Not compiled with bzib2 support, reverting to 
plain.");
+      method = FZ_PLAIN;
+    }
+#endif
+
   } else {
     /* Reading: ignore specified method and try best: */
+    fp->mode = 'r';
+#ifdef HAVE_LIBBZ2
+    /* Try to open as bzip2 file */
+    method = FZ_BZIP2;
+    sz_strlcat(mode,"b");
+    fp->u.bz2.plain = fopen(filename, mode);
+    if (fp->u.bz2.plain) {
+      fp->u.bz2.file = BZ2_bzReadOpen(&fp->u.bz2.error, fp->u.bz2.plain, 1, 0,
+                                      NULL, 0);
+    }
+    if (!fp->u.bz2.file) {
+      if (fp->u.bz2.plain) {
+        fclose(fp->u.bz2.plain);
+      }
+      free(fp);
+      return NULL;
+    } else {
+      /* Try to read first byte out of stream so we can figure out if this
+         really is bzip2 file or not. Store byte for later use */
+      char tmp;
+      BZ2_bzRead(&fp->u.bz2.error, fp->u.bz2.file, &tmp, 1);
+      if (fp->u.bz2.error != BZ_DATA_ERROR_MAGIC) { /* bzip2 file */
+        if (fp->u.bz2.error != BZ_OK) {
+          fclose(fp->u.bz2.plain);
+          free(fp);
+          return NULL;
+        }
+        fp->method = FZ_BZIP2;
+        fp->u.bz2.firstbyte = tmp;
+        return fp;
+      }
+      fclose(fp->u.bz2.plain);
+    }
+#endif
+
 #ifdef HAVE_LIBZ
     method = FZ_ZLIB;
 #else
@@ -105,6 +163,26 @@
   fp->method = method;
 
   switch (fp->method) {
+#ifdef HAVE_LIBBZ2
+  case FZ_BZIP2:
+    /*  bz2 files are binary files, so we should add "b" to mode! */
+    sz_strlcat(mode,"b");
+    fp->u.bz2.plain = fopen(filename, mode);
+    if (fp->u.bz2.plain) {
+      /*  Open for read handled earlier */
+      assert(mode[0] == 'w');
+      fp->u.bz2.file = BZ2_bzWriteOpen(&fp->u.bz2.error, fp->u.bz2.plain,
+                                       compress_level, 1, 15);
+    }
+    if (!fp->u.bz2.file) {
+      if (fp->u.bz2.plain) {
+        fclose(fp->u.bz2.plain);
+      }
+      free(fp);
+      fp = NULL;
+    }
+    break;
+#endif
 #ifdef HAVE_LIBZ
   case FZ_ZLIB:
     /*  gz files are binary files, so we should add "b" to mode! */
@@ -163,6 +241,21 @@
   int retval = 0;
   
   switch(fp->method) {
+#ifdef HAVE_LIBBZ2
+  case FZ_BZIP2:
+    if(fp->mode == 'w') {
+      BZ2_bzWriteClose(&fp->u.bz2.error, fp->u.bz2.file, 0, NULL, NULL);
+    } else {
+      BZ2_bzReadClose(&fp->u.bz2.error, fp->u.bz2.file);
+    }
+    if(fp->u.bz2.error == BZ_OK) {
+      retval = 0;
+    } else {
+      retval = 1;
+    }
+    fclose(fp->u.bz2.plain);
+    break;
+#endif
 #ifdef HAVE_LIBZ
   case FZ_ZLIB:
     retval = gzclose(fp->u.zlib);
@@ -189,9 +282,38 @@
 ***************************************************************/
 char *fz_fgets(char *buffer, int size, fz_FILE *fp)
 {
-  char *retval = 0;
+  char *retval = NULL;
   
-  switch(fp->method) {
+  switch (fp->method) {
+#ifdef HAVE_LIBBZ2
+  case FZ_BZIP2:
+    {
+      int i = 0;
+      /* See if first byte is already read and stored */
+      if (fp->u.bz2.firstbyte >= 0) {
+        buffer[0] = fp->u.bz2.firstbyte;
+        fp->u.bz2.firstbyte = -1;
+        i++;
+      } else {
+        BZ2_bzRead(&fp->u.bz2.error, fp->u.bz2.file, buffer + i, 1);
+        i++;
+      }
+      /* Leave space for trailing zero */
+      for (; i < size - 1 && fp->u.bz2.error == BZ_OK && buffer[i - 1] != '\n' 
;
+           i++) {
+        BZ2_bzRead(&fp->u.bz2.error, fp->u.bz2.file, buffer + i, 1);
+      }
+      if (fp->u.bz2.error != BZ_OK &&
+          (fp->u.bz2.error != BZ_STREAM_END ||
+          i == 0)) {
+        retval = NULL;
+      } else {
+        retval = buffer;
+      }
+      buffer[i] = '\0';
+      break;
+    }
+#endif
 #ifdef HAVE_LIBZ
   case FZ_ZLIB:
     retval = gzgets(fp->u.zlib, buffer, size);
@@ -224,8 +346,27 @@
   int retval = 0;
   
   va_start(ap, format);
-  
-  switch(fp->method) {
+
+  switch (fp->method) {
+#ifdef HAVE_LIBBZ2
+  case FZ_BZIP2:
+    {
+      char buffer[65536];
+      int num;
+      num = my_vsnprintf(buffer, sizeof(buffer), format, ap);
+      if (num == -1) {
+         freelog(LOG_ERROR, "Too much data: truncated in fz_fprintf (%lu)",
+                 (unsigned long) sizeof(buffer));
+      }
+      BZ2_bzWrite(&fp->u.bz2.error, fp->u.bz2.file, buffer, strlen(buffer));
+      if (fp->u.bz2.error != BZ_OK) {
+        retval = 0;
+      } else {
+        retval = strlen(buffer);
+      }
+    }
+    break;
+#endif
 #ifdef HAVE_LIBZ
   case FZ_ZLIB:
     {
@@ -258,8 +399,18 @@
 int fz_ferror(fz_FILE *fp)
 {
   int retval = 0;
-  
-  switch(fp->method) {
+
+  switch (fp->method) {
+#ifdef HAVE_LIBBZ2
+  case FZ_BZIP2:
+    if (fp->u.bz2.error != BZ_OK &&
+        fp->u.bz2.error != BZ_STREAM_END) {
+      retval = 1;
+    } else {
+      retval = 0;
+    }
+    break;
+#endif
 #ifdef HAVE_LIBZ
   case FZ_ZLIB:
     (void) gzerror(fp->u.zlib, &retval);       /* ignore string result here */
@@ -291,6 +442,16 @@
   const char *retval = 0;
   
   switch(fp->method) {
+#ifdef HAVE_LIBBZ2
+  case FZ_BZIP2:
+    {
+      static char bzip2error[50];
+      my_snprintf(bzip2error, sizeof(bzip2error), "Bzip2 error %d",
+                  fp->u.bz2.error);
+      retval = bzip2error;
+      break;
+    }
+#endif
 #ifdef HAVE_LIBZ
   case FZ_ZLIB:
     {
diff -Nurd -X.diff_ignore freeciv/utility/ioz.h freeciv/utility/ioz.h
--- freeciv/utility/ioz.h       2005-06-27 00:51:07.046875000 +0300
+++ freeciv/utility/ioz.h       2005-06-27 14:09:41.500000000 +0300
@@ -27,7 +27,7 @@
 typedef struct fz_FILE_s fz_FILE;
 
 /* (possibly) supported methods (depending on config.h) */
-enum fz_method { FZ_PLAIN, FZ_ZLIB, FZ_LAST };
+enum fz_method { FZ_PLAIN = 0, FZ_ZLIB = 1, FZ_BZIP2 = 2, FZ_LAST };
 #define FZ_NOT_USED FZ_LAST
 
 fz_FILE *fz_from_file(const char *filename, const char *in_mode,
diff -Nurd -X.diff_ignore freeciv/utility/registry.c freeciv/utility/registry.c
--- freeciv/utility/registry.c  2005-06-27 00:51:07.046875000 +0300
+++ freeciv/utility/registry.c  2005-06-27 14:09:41.515625000 +0300
@@ -654,7 +654,8 @@
 **************************************************************************/
 bool section_file_save(struct section_file *my_section_file,
                        const char *filename,
-                      int compression_level)
+                       int compression_level,
+                       enum fz_method compression_method)
 {
   char real_filename[1024];
   fz_FILE *fs;
@@ -663,7 +664,7 @@
   int i;
   
   interpret_tilde(real_filename, sizeof(real_filename), filename);
-  fs = fz_from_file(real_filename, "w", FZ_ZLIB, compression_level);
+  fs = fz_from_file(real_filename, "w", compression_method, compression_level);
 
   if (!fs)
     return FALSE;
diff -Nurd -X.diff_ignore freeciv/utility/registry.h freeciv/utility/registry.h
--- freeciv/utility/registry.h  2005-06-27 00:51:07.046875000 +0300
+++ freeciv/utility/registry.h  2005-06-27 14:09:41.515625000 +0300
@@ -37,7 +37,8 @@
 bool section_file_load_from_stream(struct section_file *my_section_file,
                                   fz_FILE * stream);
 bool section_file_save(struct section_file *my_section_file,
-                     const char *filename, int compression_level);
+                       const char *filename, int compression_level,
+                       enum fz_method compression_method);
 void section_file_free(struct section_file *file);
 void section_file_check_unused(struct section_file *file,
                               const char *filename);