[WIP] AmgX adjustments for benchmarking

amgx_precon_postprocess-base.py

@@ -0,0 +1,145 @@
+# Copyright (c) 2017, Lawrence Livermore National Security, LLC. Produced at
+# the Lawrence Livermore National Laboratory. LLNL-CODE-734707. All Rights
+# reserved. See files LICENSE and NOTICE for details.
+#
+# This file is part of CEED, a collection of benchmarks, miniapps, software
+# libraries and APIs for efficient high-order finite element and spectral
+# element discretizations for exascale applications. For more information and
+# source code availability see http://github.com/ceed.
+#
+# The CEED research is supportted by the Exascale Computing Project
+# (17-SC-20-SC), a collaborative effort of two U.S. Department of Energy
+# organizations (Office of Science and the National Nuclear Security
+# Administration) responsible for the planning and preparation of a capable
+# exascale ecosystem, including software, applications, hardware, advanced
+# system engineering and early testbed platforms, in support of the nation's
+# exascale computing imperative.
+
+from sys import stdout as out
+import fileinput
+import pprint
+
+#####   Read all input files specified on the command line, or stdin and parse
+#####   the content, storing it as a list of dictionaries - one dictionary for
+#####   each test run.
+
+it=fileinput.input()
+state=0
+line=''
+i=0
+mesh_p=1
+config='unknown'
+compiler='unknown'
+test='unknown'
+num_procs=0
+num_procs_node=0
+lnfmt='%05i'
+data={}
+runs=[]
+
+while True:
+   ##
+   if state%2==0:
+      ##
+      try:
+         line=it.next()
+         i=i+1
+      except StopIteration:
+         break
+      state=state+1
+      ##
+   elif state==1:
+      ##
+      state=0
+      if 'Reading configuration' in line:
+         ##
+         ## This is the beginning of a new file.
+         ##
+         # out.write('\n'+lnfmt%i+': %s'%line)
+         config=line.split()[2]
+         num_procs=0
+         num_procs_node=0
+      elif 'Setting up compiler' in line:
+         # out.write(lnfmt%i+': %s'%line)
+         compiler=line.split()[3]
+      elif 'Reading test file: ' in line:
+         # out.write(lnfmt%i+': %s'%line)
+         test=line.strip().split('Reading test file: ',1)[-1]
+      elif 'Running the tests using a total of' in line:
+         # out.write(lnfmt%i+': %s'%line)
+         num_procs=int(line.split('a total of ',1)[1].split(None,1)[0])
+      elif 'tasks per node' in line:
+         # out.write(lnfmt%i+': %s'%line)
+         num_procs_node=int(line.split(' tasks per',1)[0].rsplit(None,1)[1])
+      elif line == 'Options used:\n':
+         ##
+         ## This is the beginning of a new run.
+         ##
+         if 'cg-iteration-dps' in data:
+            runs.append(data)
+            # print
+            # pprint.pprint(data)
+         data={}
+         data['file']=fileinput.filename()
+         data['config']=config
+         data['compiler']=compiler
+         data['test']=test
+         data['num-procs']=num_procs
+         data['num-procs-node']=num_procs_node
+         data['mesh-order']=mesh_p
+         data['code']="MFEM"
+         test_=test.rsplit('/',1)[-1]
+         data['case']='scalar' if (('bp1' in test_) or ('bp3' in test_) or
+                                   ('bp5' in test_)) else 'vector'
+         # out.write('\n'+lnfmt%i+': %s'%line)
+         # out.write('*'*len(lnfmt%i)+':    --mesh-p %i\n'%mesh_p)
+         state=2
+      elif 'setup' in line:
+         data['amg-setup'] = float(line.split()[1])
+      #Used when using AmgX as a solver
+      elif 'solve(per iteration)' in line:
+         data['cost-of-precon'] = float(line.split()[2])
+      elif 'Total iterations' in line:
+         data['iterations'] = float(line.split()[2])
+      elif 'Time per CG step' in line:
+         data['time-per-cg-step'] = float(line.split()[4])
+      elif 'Total CG time' in line:
+         data['total-cg-time'] = float(line.split()[3])
+      elif '"DOFs/sec" in CG' in line:
+         # out.write(lnfmt%i+': %s'%line)
+         data['cg-iteration-dps']=1e6*float(line.split(' ')[3])
+      elif 'Number of finite element unknowns' in line:
+         # out.write(lnfmt%i+': %s'%line)
+         data['num-unknowns']=long(line.rsplit(None,1)[1])
+      elif 'Number of qudrature points per element' in line:
+         # out.write(lnfmt%i+': %s'%line)
+         data['quadrature-pts']=int(line.split(' = ',1)[1])
+      elif 'Total number of elements:' in line:
+         # out.write(lnfmt%i+': %s'%line)
+         data['num-elem']=int(line.rsplit(None,1)[1])
+      ##
+   elif state==3:
+      ##
+      if line[:5] == '   --':
+         # out.write(lnfmt%i+': %s'%line)
+         state=2
+         opt=line[5:].strip().split(' ',1)
+         if opt[0] in ('order'):
+            data[opt[0]]=int(opt[1])
+         elif opt[0] in ('device'):
+            data['mfem-device']=opt[1]
+      else:
+         state=1
+
+if 'cg-iteration-dps' in data:
+   runs.append(data)
+   # print
+   # pprint.pprint(data)
+for run in runs:
+   if not 'quadrature-pts' in run:
+      run['quadrature-pts']=0
+# print
+# print
+# pprint.pprint(runs)
+
+print 'Number of test runs read: %i'%len(runs)

amgx_precon_postprocess-plot-2.py

@@ -0,0 +1,220 @@
+# Copyright (c) 2017, Lawrence Livermore National Security, LLC. Produced at
+# the Lawrence Livermore National Laboratory. LLNL-CODE-734707. All Rights
+# reserved. See files LICENSE and NOTICE for details.
+#
+# This file is part of CEED, a collection of benchmarks, miniapps, software
+# libraries and APIs for efficient high-order finite element and spectral
+# element discretizations for exascale applications. For more information and
+# source code availability see http://github.com/ceed.
+#
+# The CEED research is supported by the Exascale Computing Project
+# (17-SC-20-SC), a collaborative effort of two U.S. Department of Energy
+# organizations (Office of Science and the National Nuclear Security
+# Administration) responsible for the planning and preparation of a capable
+# exascale ecosystem, including software, applications, hardware, advanced
+# system engineering and early testbed platforms, in support of the nation's
+# exascale computing imperative.
+
+#my_y_data = 'amg-setup'
+#my_y_data = 'time-per-cg-step'
+my_y_data = 'total-cg-time'
+
+#####   Adjustable plot parameters:
+log_y=0               # use log scale on the y-axis?
+#x_range=(1e3,6e6)    # plot range for the x-axis; comment out for auto
+#y_range=(0,2.2e7)    # plot range for the y-axis; comment out for auto
+#y_range=(0,2.5e6)     # plot range for the y-axis; comment out for auto
+draw_iter_lines=0     # draw the "iter/s" lines?
+ymin_iter_lines=3e5   # minimal y value for the "iter/s" lines
+ymax_iter_lines=8e8   # maximal y value for the "iter/s" lines
+legend_ncol=(2 if log_y else 1)   # number of columns in the legend
+write_figures=1       # save the figures to files?
+show_figures=0        # display the figures on the screen?
+
+
+#####   Load the data
+execfile('amgx_precon_postprocess-base.py')
+
+
+#####   Sample plot output
+
+from matplotlib import use
+if not show_figures:
+   use('pdf')
+from pylab import *
+
+rcParams['font.sans-serif'].insert(0,'Noto Sans')
+rcParams['font.sans-serif'].insert(1,'Open Sans')
+rcParams['figure.figsize']=[10, 8] # default: 8 x 6
+
+cm=get_cmap('Set1') # 'Accent', 'Dark2', 'Set1', 'Set2', 'Set3'
+if '_segmentdata' in cm.__dict__:
+   cm_size=len(cm.__dict__['_segmentdata']['red'])
+elif 'colors' in cm.__dict__:
+   cm_size=len(cm.__dict__['colors'])
+colors=[cm(1.*i/(cm_size-1)) for i in range(cm_size)]
+
+# colors=['blue','green','crimson','turquoise','m','gold','cornflowerblue',
+#         'darkorange']
+
+sel_runs=runs
+configs=list(set([run['config'].rsplit('/',1)[-1].rsplit('.sh',1)[0]
+                  for run in sel_runs]))
+# print 'Present configurations:', configs
+config=configs[0]
+print 'Using configuration:', config
+config_short=config
+sel_runs=[run for run in sel_runs if
+          run['config'].rsplit('/',1)[-1].rsplit('.sh',1)[0]==config]
+
+tests=list(set([run['test'].rsplit('/',1)[-1].rsplit('.sh',1)[0]
+                for run in sel_runs]))
+# print 'Present tests:', tests
+test=tests[0]
+print 'Using test:', test
+test_short=test
+sel_runs=[run for run in sel_runs if
+          run['test'].rsplit('/',1)[-1].rsplit('.sh',1)[0]==test]
+
+vdim=1
+prob=test
+
+codes = list(set([run['code'] for run in sel_runs]))
+code  = codes[0]
+sel_runs=[run for run in sel_runs if run['code']==code]
+
+pl_set=[(run['compiler'],run['num-procs'],run['num-procs-node'],
+         run['mfem-device'])
+        for run in sel_runs]
+pl_set=sorted(set(pl_set))
+print
+pprint.pprint(pl_set)
+
+for plt in pl_set:
+   compiler=plt[0]
+   num_procs=plt[1]
+   num_procs_node=plt[2]
+   mfem_dev=plt[3]
+   num_nodes=num_procs/num_procs_node
+   pl_runs=[run for run in sel_runs
+            if run['compiler']==compiler and
+               run['num-procs']==num_procs and
+               run['num-procs-node']==num_procs_node and
+               run['mfem-device']==mfem_dev]
+   pl_runs=sorted(pl_runs)
+   if len(pl_runs)==0:
+      continue
+
+   print
+   print 'compiler: %s, compute nodes: %i, number of MPI tasks = %i'%(
+      compiler,num_nodes,num_procs)
+   print 'mfem device: %s'%mfem_dev
+
+   figure()
+   i=0
+   sol_p_set=sorted(set([run['order'] for run in pl_runs]))
+   for sol_p in sol_p_set:
+      qpts=sorted(list(set([run['quadrature-pts'] for run in pl_runs
+                            if run['order']==sol_p])))
+      qpts.reverse()
+      print 'Order: %i, quadrature points:'%sol_p, qpts
+      qpts_1d=[int(q**(1./3)+0.5) for q in qpts]
+
+      d=[[run['order'],run['num-elem'],1.*run['num-unknowns']/num_nodes/vdim,
+           (run['num-unknowns']/run[my_y_data])/1e6]
+#          run['cg-iteration-dps']/num_nodes]
+         for run in pl_runs
+         if run['order']==sol_p and
+            run['quadrature-pts']==qpts[0]]
+      # print
+      # print 'order = %i'%sol_p
+      # pprint.pprint(sorted(d))
+      d=[[e[2],e[3]] for e in d if e[0]==sol_p]
+      # (DOFs/[sec/iter]/node)/(DOFs/node) = iter/sec
+      d=[[nun,
+          min([e[1] for e in d if e[0]==nun]),
+          max([e[1] for e in d if e[0]==nun])]
+         for nun in set([e[0] for e in d])]
+      d=asarray(sorted(d))
+      ## Legend with 'q=...'
+      # plot(d[:,0],d[:,2],'o-',color=colors[i%cm_size],
+      #      label='p=%i, q=p+%i'%(sol_p,qpts_1d[0]-sol_p))
+      ## Legend without 'q=...'
+      print(d)
+      plot(d[:,0],d[:,2],'o-',color=colors[i%cm_size],
+           label='p=%i'%sol_p)
+      if list(d[:,1]) != list(d[:,2]):
+         plot(d[:,0],d[:,1],'o-',color=colors[i])
+         fill_between(d[:,0],d[:,1],d[:,2],facecolor=colors[i],alpha=0.2)
+      ##
+      if len(qpts)==1:
+         i=i+1
+         continue
+      d=[[run['order'],run['num-elem'],1.*run['num-unknowns']/num_nodes/vdim,
+          run['cg-iteration-dps']/num_nodes]
+         for run in pl_runs
+         if run['order']==sol_p and (run['quadrature-pts']==qpts[1])]
+      d=[[e[2],e[3]] for e in d if e[0]==sol_p]
+      if len(d)==0:
+         i=i+1
+         continue
+      d=[[nun,
+          min([e[1] for e in d if e[0]==nun]),
+          max([e[1] for e in d if e[0]==nun])]
+         for nun in set([e[0] for e in d])]
+      d=asarray(sorted(d))
+      plot(d[:,0],d[:,2],'s--',color=colors[i],
+           label='p=%i, q=p+%i'%(sol_p,qpts_1d[1]-sol_p))
+      if list(d[:,1]) != list(d[:,2]):
+         plot(d[:,0],d[:,1],'s--',color=colors[i])
+      ##
+      i=i+1
+   ##
+   if draw_iter_lines:
+      y0,y1=ymin_iter_lines,ymax_iter_lines
+      y=asarray([y0,y1]) if log_y else exp(linspace(log(y0), log(y1)))
+      slope1=600.
+      slope2=6000.
+      plot(y/slope1,y,'k--',label='%g iter/s'%(slope1/vdim))
+      plot(y/slope2,y,'k-',label='%g iter/s'%(slope2/vdim))
+
+   title('Config: %s/%s, host: %s (%i node%s, %i task%s/node), %s, %s'%(
+         code,mfem_dev,config_short,num_nodes,'' if num_nodes==1 else 's',
+         num_procs_node,'' if num_procs_node==1 else 's', compiler,
+         test))
+   xscale('log') # subsx=[2,4,6,8]
+   if log_y:
+      yscale('log')
+   if 'x_range' in vars() and len(x_range)==2:
+      xlim(x_range)
+   if 'y_range' in vars() and len(y_range)==2:
+      ylim(y_range)
+   # rng=arange(1e7,1.02e8,1e7)
+   # yticks(rng,['%i'%int(v/1e6) for v in rng])
+   # ylim(min(rng),max(rng))
+   # xlim(0.5,max([run['order'] for run in pl_runs])+0.5)
+   grid('on', color='gray', ls='dotted')
+   grid('on', axis='both', which='minor', color='gray', ls='dotted')
+   gca().set_axisbelow(True)
+   xlabel('Points per compute node')
+   my_y_label = 'DOFS /' + my_y_data
+   ylabel(my_y_label)
+   legend(ncol=legend_ncol, loc='best')
+
+   if write_figures: # write .pdf file?
+      pdf_file='plot2_%s_%s_%s_%s_%s_N%03i_pn%i.pdf'%(
+               code,mfem_dev,test,config_short,compiler,
+               num_nodes,num_procs_node)
+      print 'saving figure --> %s'%pdf_file
+      savefig(pdf_file, format='pdf', bbox_inches='tight')
+
+   if write_figures: # write .png file?
+      png_file='plot2_%s_%s_%s_%s_%s_N%03i_pn%i.png'%(
+               code,mfem_dev,test,config_short,compiler,
+               num_nodes,num_procs_node)
+      print 'saving figure --> %s'%png_file
+      savefig(png_file, format='png', bbox_inches='tight')
+
+if show_figures: # show the figures?
+   print '\nshowing figures ...'
+   show()