[Freeciv-Dev] Re: Timing tool

[Paul Zastoupil <paul@xxxxxxxxxxxxx>]

Fwiw, I have a motherboard with cpu (PII 400) and RAM ready for freeciv.  I
will try to put it together this weekend.

On Fri, Apr 05, 2002 at 05:45:43PM +0200, Raimar Falke wrote:
> 
> Attached is a first version of a script which should allow us to
> easily measure performance. I wrote it in python but sh or perl should
> also be possible. The basic program is test_one.py. You can see
> possible uses in test.sh. It does:
>  - copy the base tree
>  - apply any patches
>  - call the configure command
>  - call make
>  - do <runs> autogames
>  - check if the savegames are equal
>  - create some plots
>  - it also handles profiling (just add -p and it will add -pg and run
>  gprof for each run)
> 
> All the results are put into a new directory <name>. There is a
> sub-dir for every run. A sample result dir may look like this:
> 
> .
> |-- configure_command.out
> |-- report
> |-- run_00
> |   |-- civgame+0001.sav.gz
> |   |-- civgame+0020.sav.gz
> |   |-- civgame-0250.sav.gz
> |   |-- civgame-0500.sav.gz
> |   |-- civgame-0750.sav.gz
> |   |-- civgame-1000.sav.gz
> |   |-- civgame-1500.sav.gz
> |   |-- civgame-2000.sav.gz
> |   |-- civgame-2500.sav.gz
> |   |-- civgame-3000.sav.gz
> |   |-- civgame-3500.sav.gz
> |   |-- gamelog
> |   `-- log
> |-- run_01
> |   |-- civgame+0001.sav.gz
> |   |-- civgame+0020.sav.gz
> |   |-- civgame-0250.sav.gz
> |   |-- civgame-0500.sav.gz
> |   |-- civgame-0750.sav.gz
> |   |-- civgame-1000.sav.gz
> |   |-- civgame-1500.sav.gz
> |   |-- civgame-2000.sav.gz
> |   |-- civgame-2500.sav.gz
> |   |-- civgame-3000.sav.gz
> |   |-- civgame-3500.sav.gz
> |   |-- gamelog
> |   `-- log
> |-- run_02
> |   |-- civgame+0001.sav.gz
> |   |-- civgame+0020.sav.gz
> |   |-- civgame-0250.sav.gz
> |   |-- civgame-0500.sav.gz
> |   |-- civgame-0750.sav.gz
> |   |-- civgame-1000.sav.gz
> |   |-- civgame-1500.sav.gz
> |   |-- civgame-2000.sav.gz
> |   |-- civgame-2500.sav.gz
> |   |-- civgame-3000.sav.gz
> |   |-- civgame-3500.sav.gz
> |   |-- gamelog
> |   `-- log
> |-- startup.final
> |-- system.png
> |-- system.times
> |-- user.png
> |-- user.times
> |-- wall.png
> `-- wall.times
> 
> The file "report" contains all relevant data to repeat the experiment
> and also the timing results.
> 
> Sample report:
> 
> <begin quote>
> Name:              O2_no_CHECK_MAP_POS
> Runs:              3
> Base-tree:         /mnt/hdb7/hawk/freeciv/new
> RC-template:       /mnt/hdb7/hawk/freeciv/test/rc.templ
> Configure command: /mnt/hdb7/hawk/freeciv/test/conf
> Profile:           0
> CFLAGS:            -g -O2 -Wall
> CPPFLAGS:          
> 
> Patches:
>   /mnt/hdb7/hawk/freeciv/disable_CHECK_MAP_POS.diff applied with -p0
> 
> Configure command:
> ---START /mnt/hdb7/hawk/freeciv/test/conf---
> #!/bin/sh
> 
> ./configure --enable-client=no
> ---END---
>   w=21.98s, s=3.86s, u=16.15s
> 
> Make:
>   w=152.05s, s=7.1s, u=138.87s
> 
> Startup script:
> ---START /mnt/hdb7/hawk/freeciv/test/rc.templ---
> set randseed 4321
> set seed 1234
> set aifill 7
> hard
> set endyear 1
> #set saveturns 1
> ---END---
> ---START /mnt/hdb7/hawk/freeciv/test/O2_no_CHECK_MAP_POS/startup.final---
> set randseed 4321
> set seed 1234
> set aifill 7
> hard
> set endyear 1
> #set saveturns 1
> 
> set timeout -1
> set savename civgame
> set gamelog 40
> create Caesar
> start
> ---END---
> 
> Runs:
>   0: w=80.27s, s=0.23s, u=76.34s
>   1: w=78.95s, s=0.16s, u=75.99s
>   2: w=79.07s, s=0.13s, u=76.25s
> 
> Average wall time: 79.430000
> Average system time: 0.173333
> Average user time: 76.193333
> Savegames are equal
> 
> <end quote>
> 
> TODO:
>  - add scripts which do the plots and savegame comparison for multiple
>  settings
>  - remove bugs
> 
> I haven't stressed it a lot since the runs are pretty time
> intensive. But it should be usable.
> 
>       Raimar
> 
> -- 
>  email: rf13@xxxxxxxxxxxxxxxxx
>  "> WHY?! Isn't it better to put $(shell cat cscope.files) on the list of
>   I only have a yellow belt in makefile kungfu.  These fancy gnu make things
>   are relatively new to some of us..."
>     -- Mark Frazer to Vassilii Khachaturov in linux-kernel


> #!/bin/sh
> 
> ./configure --enable-client=no

> set randseed 4321
> set seed 1234
> set aifill 7
> hard
> set endyear 1
> #set saveturns 1

> Index: common/map.h
> ===================================================================
> RCS file: /home/freeciv/CVS/freeciv/common/map.h,v
> retrieving revision 1.125
> diff -u -r1.125 map.h
> --- common/map.h      2002/02/27 10:33:08     1.125
> +++ common/map.h      2002/04/05 15:08:39
> @@ -202,7 +202,7 @@
>  void initialize_move_costs(void);
>  void reset_move_costs(int x, int y);
>  
> -#define CHECK_MAP_POS(x,y) assert(is_normal_map_pos((x),(y)))
> +#define CHECK_MAP_POS(x,y) ((void)0)
>  
>  #define map_adjust_x(X)            \
>    ((X) < 0                         \

> #!/usr/bin/env python1.5
> 
> import string, os, re, sys, time
> 
> def safe_strip(str):
>     if str[-1]=="\n":
>         return str[:-1]
>     else:
>         return str
>     
> def shell_quote(str):
>     return "'%s'"%string.replace(str,"'","'\"'\"'")
> 
> def get_tmp_filename():
>     result=os.path.expanduser("%s_%f_%d"%(os.path.basename(sys.argv[0]),
>                                           time.time(),
>                                           os.getpid()))
>     return os.path.abspath(result)
> 
> def system(command,dir=".",pass_errors=0):
>     show=1
>     if show:
>         print command,
>         sys.stdout.flush()
>         
>     cap=get_tmp_filename()
>     time_cap=get_tmp_filename()
>     assert cap!=time_cap
>     
>     whole_command=("cd %s;/usr/bin/time -f '%%e %%S %%U %%x' "+
>                    "-o %s sh -c '%s' >%s 2>&1")%(shell_quote(dir),
>                                                  shell_quote(time_cap),
>                                                  command, shell_quote(cap))
>     #print whole_command
>     os.system(whole_command)
> 
>     result={}
>     
>     file=open(cap,"r")
>     result["output"]=file.read()
>     file.close()
>     os.remove(cap)
> 
>     file=open(time_cap,"r")
>     time_line=map(safe_strip,file.readlines())[-1]
>     file.close()
>     os.remove(time_cap)
> 
>     time_line=string.split(time_line," ")
>     result["wall"]=float(time_line[0])
>     result["system"]=float(time_line[1])
>     result["user"]=float(time_line[2])
>     result["error"]=int(time_line[3])
> 
>     result["times"]="w=%ss, s=%ss, 
> u=%ss"%(result["wall"],result["system"],result["user"])
> 
>     if show: print result["times"], "e=%d"%result["error"]
>     
>     if result["error"]!=0 and not pass_errors:
>         print command
>         print result["output"]
>         assert 0
>     return result
> 
> # Use a class to get some kind of global namespace
> class TestOne:
>     def __init__(self):
>         self.parse_args()
>         self.init()
>         system("rm -rf %s"%shell_quote(self.build))
>         system("cp -ar %s %s"%(shell_quote(self.base_tree),
>                                shell_quote(self.build)))
>         self.patch()
>         self.configure()
>         self.make()
>         self.build_rc()
>         self.report.write("Runs:\n")
>         for i in range(self.runs):
>             self.run(i)
>         self.report.write("\n")
>         self.plot()
>         self.check_savegames()
> 
>     def check_savegames(self):
>         md5sums=[]
>         for run in range(self.runs):
>             md5sums.append(system("md5sum civgame* | md5sum",
>                                   self.run_dir(run))["output"])
>         m={}
>         for i in md5sums:
>             m[i]=1
>         if len(m)>1:
>             self.report.write("Savegames doesn't match: %s\n"%repr(md5sums))
>             assert 0
>         else:
>             self.report.write("Savegames are equal\n")
>         
>     def plot(self):
>         plot=get_tmp_filename()
>         for i in ["wall","system","user"]:
>             sum=0.0
>             j=0
>             for line in open("%s/%s.times"%(self.report_dir,i)).readlines():
>                 sum=sum+float(line)
>                 j=j+1
>             average=sum/j
> 
>             self.report.write("Average %s time: %f\n"%(i,average))
>             
>             f=open(plot,"w")
>             f.write('''
> set terminal png color
> set output "%s"
> plot "%s.times" with lines, %f title "average=%f"
> '''%(self.report_dir+"/"+i+".png",i,average,average))
>             f.close()
>             system("gnuplot %s"%shell_quote(plot), self.report_dir)
>         os.remove(plot)
> 
>     def run_dir(self,run):
>         return "%s/run_%02d"%(self.report_dir,run)
>     
>     def run(self,run):
>         self.report.write("  %d: "%run)
>         dir=self.run_dir(run)
>         os.mkdir(dir)
>         system("rm -f civgame* core gmon.out",self.build)
>         t=system("LANG=C ./server/civserver -r %s -g %s -l %s"%(
>             shell_quote(self.report_dir+"/startup.final"),
>             shell_quote(dir+"/gamelog"),
>             shell_quote(dir+"/log")), self.build)
>         system("mv civgame* %s"%shell_quote(dir),self.build,1)
>         for i in ["wall","system","user"]:
>             f=open("%s/%s.times"%(self.report_dir,i), "a")
>             f.write("%s\n"%t[i])
>             f.close()
>         self.report.write("%s\n"%t["times"])
>         if self.profile:
>             t=system("gprof server/civserver",self.build)
>             open(dir+"/profile","w").write(t["output"])
> 
>     def build_rc(self):
>         self.report.write("Startup script:\n")
>         self.content(self.rc_templ)
>         file=open(self.report_dir+"/startup.final","w")
>         file.write(open(self.rc_templ).read())
>         file.write("""
> set timeout -1
> set savename civgame
> set gamelog 40
> create Caesar
> start
> """)
>         file.close()
>         self.content(self.report_dir+"/startup.final")
>         self.report.write("\n")
> 
>     def patch(self):
>         self.report.write("Patches:\n")
>         for p in self.patches:
>             for i in [0,1]:
>                 t=system("patch --dry-run -p%d --batch 
> <%s"%(i,p),self.build,1)
>                 if not t["error"]:
>                     system("patch -p%d --batch <%s"%(i,shell_quote(p)),
>                            self.build, 1)
>                     self.report.write("  %s applied with -p%d\n"%(p,i))
>                     break
>             else:
>                 self.report.write("  %s can't be applied\n"%p)
>                 assert 0
>         self.report.write("\n")
> 
>     def content(self,file):
>         self.report.write("---START 
> %s---\n%s---END---\n"%(file,open(file).read()))
>         
>     def configure(self):
>         self.report.write("Configure command:\n")
>         if not os.path.isfile(self.config_cmd):
>             self.report.write("  %s not found\n"%self.config_cmd)
>             return
>         self.content(self.config_cmd)
>         t=system(self.config_cmd,self.build)
>         open(self.report_dir+"/configure_command.out","w").write(t["output"])
>         self.report.write("  %s\n\n"%t["times"])
> 
>     def make(self):
>         self.report.write("Make:\n")
>         system("make clean", self.build)
>         t=system('make CFLAGS="%s" CPPFLAGS="%s"'%(self.CFLAGS,self.CPPFLAGS),
>                  self.build)
>         self.report.write("  %s\n\n"%t["times"])
> 
>     def usage(self):
>         print """\
> test_one.py [-p] [CFLAGS=...] [CPPFLAGS=...] <name> <runs> <base-tree> 
> <rc-template> <configure-command> [<patch0>] [<patch1>] ...
> """
>         sys.exit(1)
> 
>     def init(self):
>         self.build=os.path.abspath("BUILD")
>         if self.profile:
>             self.CFLAGS=self.CFLAGS+" -pg"
> 
>         self.report_dir=os.path.abspath(self.name)
>         if os.path.isdir(self.report_dir):
>             print "Warning: Removing old result dir %s"%self.report_dir
>             system("rm -r %s"%shell_quote(self.report_dir))
> 
>         os.mkdir(self.report_dir)
>         self.report=open(self.report_dir+"/report","w")
>         self.report.write("""\
> Name:              %(name)s
> Runs:              %(runs)d
> Base-tree:         %(base_tree)s
> RC-template:       %(rc_templ)s
> Configure command: %(config_cmd)s
> Profile:           %(profile)s
> CFLAGS:            %(CFLAGS)s
> CPPFLAGS:          %(CPPFLAGS)s
> 
> """%self.__dict__)
> 
>     def parse_args(self):
>         self.options={}
>         args=sys.argv[:]
>         del args[0]
>         if len(args)<5:
>             self.usage()
>         if args[0]=="-p":
>             self.profile=1
>             del args[0]
>         else:
>             self.profile=0
>         self.CFLAGS="-g -O2 -Wall"
>         self.CPPFLAGS=""
>         for i in ["CFLAGS", "CPPFLAGS"]:
>             if string.find(args[0],i)==0:
>                 setattr(self, i, args[0])
>                 del args[0]
>         if len(args)<5:
>             self.usage()
>         self.name=args[0]
>         self.runs=int(args[1])
>         self.base_tree=os.path.abspath(args[2])
>         self.rc_templ=os.path.abspath(args[3])
>         self.config_cmd=os.path.abspath(args[4])
>         args=args[5:]
>         self.patches=map(os.path.abspath, args)
> 
> TestOne()


-- 
Paul Zastoupil