microbench/parse-results.py

   1 #!/usr/bin/python2
   2
   3 import os, re, sys
   4
   5 def load_results(prefix):
   6     settings_file = open(prefix + ".settings")
   7     results_file = open(prefix + ".results")
   8
   9     settings = {}
  10     for l in settings_file:
  11         l = l.strip()
  12         try:
  13             (k, v) = l.split('=', 1)
  14             settings[k] = v
  15         except: pass
  16
  17     results = []
  18     result_group = {}
  19     for l in results_file:
  20         l = l.strip()
  21         if len(l) == 0 and len(result_group) > 0:
  22             results.append(result_group)
  23             result_group = {}
  24         m = re.match(r"^(\w+): \[(.*)\]$", l)
  25         if m:
  26             vals = [float(x.strip()) for x in m.group(2).split(',')]
  27             result_group[m.group(1)] = vals
  28     if len(result_group) > 0:
  29         results.append(result_group)
  30
  31     return (settings, results)
  32
  33 def extract(data, params={}, ty='read', index=0):
  34     if ty == 'mixed':
  35         ty = ('read', 'write')
  36     else:
  37         ty = (ty,)
  38
  39     results = {}
  40     for (s, r) in data:
  41         match = True
  42         for (k, v) in params.items():
  43             if s[k] != v: match = False
  44         if not match: continue
  45
  46         ops = int(s['BENCH_OPS'])
  47
  48         vs = 0.0
  49         for t in ty:
  50             vals = [x[t][index] for x in r]
  51             vals = vals[5:]
  52             vs += sum(vals) / (len(vals) or 1)
  53         results[ops] = vs
  54
  55     return results
  56
  57 data = []
  58 if __name__ == '__main__':
  59     dirname = '20110310'
  60     for f in os.listdir(dirname):
  61         if f.endswith('.settings'):
  62             data.append(load_results(dirname + '/' + f[:-len('.settings')]))
  63
  64 ratios = {'read': '0.0', 'write': '1.0', 'mixed': '0.5'}
  65
  66 blocksizes = set(int(s[0]['BENCH_BLOCKSIZE'])
  67                    or int(s[0]['BENCH_FILESIZE'])
  68                  for s in data)
  69 print blocksizes
  70 sizes = set(int(s[0]['BENCH_FILESIZE']) * int(s[0]['BENCH_FILECOUNT']) / 1024**2
  71                 for s in data)
  72 print sizes
  73
  74 for size in sorted(sizes):
  75     for blocksize in sorted(blocksizes):
  76         for ratio in ratios:
  77             params = {'BLUESKY_TARGET': 's3:mvrable-bluesky-west',
  78                       'BENCH_WRITERATIO': ratios[ratio],
  79                       'BENCH_FILECOUNT': str(size),
  80                       'BENCH_BLOCKSIZE': str(blocksize)}
  81             basesize = 1 << 20
  82             if blocksize < basesize:
  83                 params['BENCH_BLOCKSIZE'] = str(blocksize)
  84                 params['BENCH_FILESIZE'] = str(basesize)
  85             elif blocksize >= basesize:
  86                 params['BENCH_BLOCKSIZE'] = str(0)
  87                 params['BENCH_FILESIZE'] = str(blocksize)
  88
  89             d0 = extract(data, params, ty=ratio, index=0)
  90             d1 = extract(data, params, ty=ratio, index=1)
  91
  92             fp = open('%s-%d-%s-%dk.data' % (params['BLUESKY_TARGET'], size, ratio, blocksize / 1024), 'w')
  93             for k in sorted(d0.keys()):
  94                 fp.write("%d\t%f\t%f\n" % (k, d0[k], d1[k]))
  95             fp.close()