solvcon.anchor

1 # -*- coding: UTF-8 -*- 2 # 3 # Copyright (C) 2008-2010 Yung-Yu Chen <yyc@solvcon.net>. 4 # 5 # This program is free software; you can redistribute it and/or modify 6 # it under the terms of the GNU General Public License as published by 7 # the Free Software Foundation; either version 2 of the License, or 8 # (at your option) any later version. 9 # 10 # This program is distributed in the hope that it will be useful, 11 # but WITHOUT ANY WARRANTY; without even the implied warranty of 12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 # GNU General Public License for more details. 14 # 15 # You should have received a copy of the GNU General Public License along 16 # with this program; if not, write to the Free Software Foundation, Inc., 17 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 18 19 """ 20 Anchors attached to the solvers. There's only one base anchor class for 21 subclassing. Any other anchors defined here are for directly installation. 22 """

23 24 -class Anchor(object):

25 """ 26 Anchor that called by solver objects at various stages. 27 28 @ivar svr: the solver object to be attached to. 29 @itype svr: solvcon.solver.Solver 30 @ivar kws: excessive keywords. 31 @itype kws: dict 32 """ 33

34 - def __init__(self, svr, **kw):

35 from .solver import BaseSolver 36 assert isinstance(svr, BaseSolver) 37 self.svr = svr 38 self.kws = dict(kw)

39

40 - def provide(self):

41 pass

42 - def preloop(self):

43 pass

44 - def premarch(self):

45 pass

46 - def prefull(self):

47 pass

48 - def presub(self):

49 pass

50 - def postsub(self):

51 pass

52 - def postfull(self):

53 pass

54 - def postmarch(self):

55 pass

56 - def postloop(self):

57 pass

58 - def exhaust(self):

59 pass

60

61 -class AnchorList(list):

62 """ 63 Anchor container and invoker. 64 65 @ivar svr: solver object. 66 @itype svr: solvcon.solver.BaseSolver 67 """

68 - def __init__(self, svr, *args, **kw):

69 self.svr = svr 70 self.names = dict() 71 super(AnchorList, self).__init__(*args, **kw)

72 - def append(self, obj, **kw):

73 name = kw.pop('name', None) 74 if isinstance(name, int): 75 raise ValueError('name can\'t be integer') 76 if isinstance(obj, type): 77 obj = obj(self.svr, **kw) 78 super(AnchorList, self).append(obj) 79 if name != None: 80 self.names[name] = obj

81 - def __getitem__(self, key):

82 if key in self.names: 83 return self.names[key] 84 else: 85 return super(AnchorList, self).__getitem__(key)

86 - def __call__(self, method):

87 """ 88 Invoke the specified method for each anchor. 89 90 @param method: name of the method to run. 91 @type method: str 92 @return: nothing 93 """ 94 runanchors = self.svr.runanchors 95 if method == 'postloop' or method == 'exhaust': 96 runanchors = reversed(runanchors) 97 for anchor in runanchors: 98 func = getattr(anchor, method, None) 99 if func != None: 100 func()

101

102 ################################################################################ 103 # Solution output. 104 ################################################################################ 105 106 -class MarchSaveAnchor(Anchor):

107 """ 108 Save solution data into VTK XML format for a solver. 109 110 @ivar anames: the arrays in der of solvers to be saved. True means in der. 111 @itype anames: dict 112 @ivar compressor: compressor for binary data. Can only be 'gz' or ''. 113 @itype compressor: str 114 @ivar fpdtype: string for floating point data type (in numpy convention). 115 @itype fpdtype: str 116 @ivar psteps: the interval (in step) to save data. 117 @itype psteps: int 118 @ivar vtkfn_tmpl: the template string for the VTK file. 119 @itype vtkfn_tmpl: str 120 """

121 - def __init__(self, svr, **kw):

122 self.anames = kw.pop('anames', dict()) 123 self.compressor = kw.pop('compressor') 124 self.fpdtype = kw.pop('fpdtype') 125 self.psteps = kw.pop('psteps') 126 self.vtkfn_tmpl = kw.pop('vtkfn_tmpl') 127 super(MarchSaveAnchor, self).__init__(svr, **kw)

128 - def _write(self, istep):

129 from .io.vtkxml import VtkXmlUstGridWriter 130 from .solver import FakeBlockVtk 131 ngstcell = self.svr.ngstcell 132 sarrs = dict() 133 varrs = dict() 134 # collect data. 135 for key in self.anames: 136 # get the array. 137 if self.anames[key]: 138 arr = self.svr.der[key][ngstcell:] 139 else: 140 arr = getattr(self.svr, key)[ngstcell:] 141 # put array in dict. 142 if len(arr.shape) == 1: 143 sarrs[key] = arr 144 elif arr.shape[1] == self.svr.ndim: 145 varrs[key] = arr 146 else: 147 for it in range(arr.shape[1]): 148 sarrs['%s[%d]' % (key, it)] = arr[:,it] 149 # write. 150 wtr = VtkXmlUstGridWriter(FakeBlockVtk(self.svr), fpdtype=self.fpdtype, 151 compressor=self.compressor, scalars=sarrs, vectors=varrs) 152 svrn = self.svr.svrn 153 wtr.write(self.vtkfn_tmpl % (istep if svrn is None else (istep, svrn)))

154 - def preloop(self):

155 self._write(0)

156 - def postmarch(self):

157 psteps = self.psteps 158 istep = self.svr.step_global 159 if istep%psteps == 0: 160 self._write(istep)

161 - def postloop(self):

162 psteps = self.psteps 163 istep = self.svr.step_global 164 if istep%psteps != 0: 165 self._write(istep)

166

167 ################################################################################ 168 # Anchors for in situ visualization. 169 ################################################################################ 170 171 -class VtkAnchor(Anchor):

172 """ 173 Abstract class for VTK filtering anchor. Must override process() method 174 for use. Note: svr.ust is shared by all VtkAnchor instances. 175 176 @ivar anames: the arrays in der of solvers to be saved. True means in der. 177 @itype anames: dict 178 @ivar fpdtype: string for floating point data type (in numpy convention). 179 @itype fpdtype: str 180 @ivar psteps: the interval (in step) to save data. 181 @itype psteps: int 182 @ivar vtkfn_tmpl: the template string for the VTK file. 183 @itype vtkfn_tmpl: str 184 """

185 - def __init__(self, svr, **kw):

186 self.anames = kw.pop('anames', dict()) 187 self.fpdtype = kw.pop('fpdtype') 188 self.psteps = kw.pop('psteps') 189 self.vtkfn_tmpl = kw.pop('vtkfn_tmpl') 190 self.vac = dict() 191 super(VtkAnchor, self).__init__(svr, **kw)

192 @property

193 - def vtkfn(self):

194 """ 195 The correct file name for VTK based on the template. 196 """ 197 istep = self.svr.step_global 198 svrn = self.svr.svrn 199 return self.vtkfn_tmpl % ( 200 istep if svrn is None else (istep, svrn))

201 - def _aggregate(self):

202 """ 203 Aggregate data from solver object to VTK unstructured mesh. 204 205 @return: nothing 206 """ 207 from .visual_vtk import valid_vector, set_array 208 ngstcell = self.svr.ngstcell 209 fpdtype = self.fpdtype 210 ust = self.svr.ust 211 # collect derived. 212 for key, inder, flag in self.anames: 213 # get the array. 214 if inder: 215 arr = self.svr.der[key][ngstcell:] 216 else: 217 arr = getattr(self.svr, key)[ngstcell:] 218 # set array in unstructured mesh. 219 if len(arr.shape) == 1: 220 set_array(arr, key, fpdtype, ust) 221 elif arr.shape[1] == self.svr.ndim: 222 set_array(valid_vector(arr), key, fpdtype, ust) 223 else: 224 for it in range(arr.shape[1]): 225 set_array(arr[:,it], '%s[%d]' % (key, it), fpdtype, ust)

226 - def preloop(self):

227 self.process(0)

228 - def postmarch(self):

229 psteps = self.psteps 230 istep = self.svr.step_global 231 if istep%psteps == 0: 232 self.process(istep)

233 - def postloop(self):

234 psteps = self.psteps 235 istep = self.svr.step_global 236 if istep%psteps != 0: 237 self.process(istep)

238 - def process(self, istep):

239 """ 240 This method implements the VTK filtering operations. Must be 241 overidden. 242 """ 243 raise NotImplementedError

244

245 ################################################################################ 246 # StatAnchor. 247 ################################################################################ 248 249 -class RuntimeStatAnchor(Anchor):

250 """ 251 Report the Linux load average through solver. Reports are made after a 252 full marching interation. 253 254 @ivar reports: list what should be reported. Default is ['loadavg'] only. 255 @itype reports: list 256 @ivar cputotal: flag to use total jiffy for cpu usage percentage. 257 @itype cputotal: bool 258 @ivar cputime: marker for timing cpu usage. 259 @itype cputime: float 260 @ivar jiffytime: the time a jiffy is. Default is 0.01 second. 261 @itype jiffytime: float 262 """ 263 264 PSTAT_KEYS = [ 265 ('pid', int), ('comm', str), ('state', str), 266 ('ppid', int), ('pgrp', int), ('session', int), 267 ('tty_nr', int), ('tpgid', int), ('flags', int), 268 ('minflt', int), ('cminflt', int), ('majflt', int), ('cmajflt', int), 269 ('utime', int), ('stime', int), ('cutime', int), ('cstime', int), 270 ('priority', int), ('nice', int), ('num_threads', int), 271 ('itrealvalue', int), ('starttime', int), 272 ('vsize', int), ('rss', int), ('rsslim', int), 273 ('startcode', int), ('endcode', int), ('startstack', int), 274 ('kstkesp', int), ('kstkeip', int), 275 # obselete, use /proc/[pid]/status. 276 ('signal', int), ('blocked', int), ('sigignore', int), 277 ('sigcatch', int), 278 # process waiting channel. 279 ('wchan', int), 280 # not maintained. 281 ('nswap', int), ('cnswap', int), 282 # signal to parent process when die. 283 ('exit_signal', int), 284 ('processor', int), ('rt_priority', int), 285 ('policy', int), ('delayacct_blkio_ticks', int), 286 ('guest_time', int), ('cguest_time', int), 287 ] 288 289 SETTING_KEYS = ['ibcthread'] 290 ENVAR_KEYS = ['KMP_AFFINITY'] 291 292 CPU_NAMES = ['us', 'sy', 'ni', 'id', 'wa', 'hi', 'si', 'st'] 293 294 @classmethod

295 - def get_pstat(cls):

296 import os 297 pid = os.getpid() 298 f = open('/proc/%d/stat'%pid) 299 sinfo = f.read().split() 300 f.close() 301 pstat = dict() 302 for it in range(len(sinfo)): 303 key, typ = cls.PSTAT_KEYS[it] 304 pstat[key] = typ(sinfo[it]) 305 return pstat

306 307 @staticmethod

308 - def get_cpu_frame():

309 f = open('/proc/stat') 310 totcpu = f.readlines()[0] 311 f.close() 312 return [float(it) for it in totcpu.split()[1:]]

313 314 @staticmethod

315 - def calc_cpu_difference(frame0, frame1):

316 frame = list() 317 for it in range(len(frame0)): 318 frame.append(frame1[it]-frame0[it]) 319 return frame

320 321 @classmethod

322 - def get_envar(cls):

323 import os 324 envar = dict() 325 for key in cls.ENVAR_KEYS: 326 envar[key] = os.environ.get(key, None) 327 return envar

328 329 @staticmethod

330 - def get_loadavg():

331 f = open('/proc/loadavg') 332 loadavg = f.read() 333 f.close() 334 return [float(val) for val in loadavg.split()[:3]]

335

336 - def __init__(self, svr, **kw):

337 self.cputotal = kw.pop('cputotal', True) 338 self.jiffytime = kw.pop('jiffytime', 0.01) 339 self.records = list() 340 super(RuntimeStatAnchor, self).__init__(svr, **kw)

341

342 - def _get_record(self):

343 from time import time 344 record = dict() 345 record['time'] = time() 346 pstat = self.get_pstat() 347 cpu = self.get_cpu_frame() 348 loadavg = self.get_loadavg() 349 # pstat. 350 for key in ['utime', 'stime', 'priority', 'nice', 'num_threads', 351 'vsize', 'rss', 'rt_priority']: 352 record[key] = pstat[key] 353 # cpu usage. 354 record['cpu'] = cpu 355 # loadavg. 356 record['loadavg'] = loadavg 357 # envar. 358 record.update(self.get_envar()) 359 # timer. 360 record.update(self.svr.timer) 361 return record

362

363 - def _msg_setting(self, record):

364 return ' '.join([ 365 '%s=%s' % (key, str(getattr(self.svr, key))) for key in 366 self.SETTING_KEYS 367 ])

368

369 - def _msg_envar(self, record):

370 return ' '.join([ 371 '%s=%s' % (key, str(record[key])) for key in self.ENVAR_KEYS 372 ])

373

374 - def _msg_cpu(self, record):

375 # get the difference to the frame since last run of this method. 376 if len(self.records): 377 oldtime = self.records[-1]['time'] 378 framediff = self.calc_cpu_difference( 379 self.records[-1]['cpu'], record['cpu']) 380 pcpudiff = [ 381 record['utime'] - self.records[-1]['utime'], 382 record['stime'] - self.records[-1]['stime'], 383 ] 384 else: 385 oldtime = record['time'] 386 framediff = record['cpu'] 387 pcpudiff = [record['utime'], record['stime']] 388 # calculate the percentage. 389 if self.cputotal: 390 jiffy = sum(framediff) 391 else: 392 jiffy = (record['time']-oldtime)/self.jiffytime 393 if jiffy == 0.0: jiffy = 1.e100 394 pscale = [it/jiffy*100 for it in pcpudiff] 395 oscale = [it/jiffy*100 for it in framediff] 396 # build message. 397 process = '%.2f%%utime %.2f%%stime' % (pscale[0], pscale[1]) 398 overall = ' '.join(['%s%s' % ('%.2f%%'%oscale[it], self.CPU_NAMES[it]) 399 for it in range(len(self.CPU_NAMES)) 400 ]) 401 return ' '.join(['cputotal=%s'%self.cputotal, process, overall])

402 @staticmethod

403 - def _parse_cpu(line):

404 every = line.split() 405 time = float(every[0]) 406 return [time] + [float(tok.split('%')[0]) for tok in every[2:]]

407 @classmethod

408 - def plot_cpu(cls, lines, ax, xtime=False, showx=True, 409 lloc='right'):

410 arr, xval, xlabel = cls._parse(lines, 'cpu', xtime) 411 arr[0,:] = arr[1,:] 412 ax.plot(xval, arr[:,2:5].sum(axis=1), '-', label='us+st+ni') 413 ax.plot(xval, arr[:,5:7].sum(axis=1), ':', label='id+wa') 414 ax.plot(xval, arr[:,0:2].sum(axis=1), '--', label='utime+stime') 415 if showx: ax.set_xlabel(xlabel) 416 ax.set_ylabel('CPU %') 417 ax.set_ylim([0,100]) 418 ax.legend(loc=lloc)

419

420 - def _msg_mem(self, record):

421 return '%d' % record['vsize']

422 @staticmethod

423 - def _parse_mem(line):

424 time, vsize = line.split() 425 return [float(time), int(vsize)]

426 @classmethod

427 - def plot_mem(cls, lines, ax, xtime=False, showx=True, 428 lloc=None):

429 arr, xval, xlabel = cls._parse(lines, 'mem', xtime) 430 ax.plot(xval, arr[:,0]/1024**2, '-') 431 if showx: ax.set_xlabel(xlabel) 432 ax.set_ylabel('Memory usage (MB)')

433

434 - def _msg_loadavg(self, record):

435 return '%.2f %.2f %.2f' % tuple(record['loadavg'])

436 @staticmethod

437 - def _parse_loadavg(line):

438 return [float(val) for val in line.split()]

439 @classmethod

440 - def plot_loadavg(cls, lines, ax, xtime=False, showx=True, 441 lloc='right'):

442 arr, xval, xlabel = cls._parse(lines, 'loadavg', xtime) 443 ax.plot(xval, arr[:,0], '-', label='1 min') 444 ax.plot(xval, arr[:,1], ':', label='5 min') 445 ax.plot(xval, arr[:,2], '--', label='15 min') 446 if showx: ax.set_xlabel(xlabel) 447 ax.set_ylabel('Load average') 448 ax.legend(loc=lloc)

449 450 @classmethod

451 - def _parse(cls, lines, key, xtime):

452 from numpy import array, arange 453 myhead = 'RT_%s: ' % key 454 nmyhead = len(myhead) 455 mymethod = getattr(cls, '_parse_%s' % key) 456 data = list() 457 for line in lines: 458 loc = line.find(myhead) 459 if loc > -1: 460 data.append(mymethod(line[loc+nmyhead:])) 461 arr = array(data, dtype='float64') 462 xval = arr[:,0]-arr[0,0] if xtime else arange(arr.shape[0])+1 463 xlabel = 'Time (s)' if xtime else 'Steps' 464 return arr[:,1:].copy(), xval.copy(), xlabel

465

466 - def postfull(self):

467 import sys 468 if not sys.platform.startswith('linux'): return 469 rec = self._get_record() 470 # output the messages. 471 time = rec['time'] 472 for mkey in ['cpu', 'loadavg', 'mem', 'setting', 'envar']: 473 method = getattr(self, '_msg_%s'%mkey) 474 self.svr.mesg('RT_%s: %.20e %s\n' % (mkey, time, method(rec))) 475 # save. 476 self.records.append(rec)

477

478 -class MarchStatAnchor(Anchor):

479 """ 480 Report the time used in each methods of marching. 481 """ 482 483 @classmethod

484 - def plot(cls, keys, lines, ax, lloc='best'):

485 maxavg = 0.0 486 for key in keys: 487 arr, xval, xlabel = cls.parse(lines, key) 488 ax.plot(xval, arr[:,0], label='%s'%key) 489 maxavg = max(arr[:,0].mean(), maxavg) 490 ax.set_xlabel(xlabel) 491 ax.set_ylabel('Time (s)') 492 ax.set_ylim([0.0, maxavg*1.5]) 493 ax.legend(loc=lloc)

494 495 @classmethod

496 - def parse(cls, lines, key, diff=True):

497 from numpy import array, arange 498 myhead = 'MA_%s: ' % key 499 nmyhead = len(myhead) 500 data = list() 501 for line in lines: 502 loc = line.find(myhead) 503 if loc > -1: 504 data.append([float(val) for val in line[loc+nmyhead:].split()]) 505 arr = array(data, dtype='float64') 506 if diff: 507 arr[1:,:] = arr[1:,:] - arr[:-1,:] 508 arr[0,:] = arr[1,:] 509 xval = arange(arr.shape[0])+1 510 xlabel = 'Steps' 511 return arr, xval.copy(), xlabel

512

513 - def postfull(self):

514 for key in ['march'] + self.svr.mmnames: 515 val = self.svr.timer.get(key, None) 516 if val == None: 517 continue 518 self.svr.mesg('MA_%s: %g\n' % (key, val))

519

520 -class TpoolStatAnchor(Anchor):

521 """ 522 Report the ticks used in each threads in pool. 523 """ 524 525 @classmethod

526 - def plot(cls, key, lines, ax, showx=True, lloc='best'):

527 arr, xval, xlabel = cls._parse(lines, key) 528 for it in range(arr.shape[1]): 529 ax.plot(xval, arr[:,it], label='%s%d'%(key, it)) 530 if showx: ax.set_xlabel(xlabel) 531 ax.set_ylabel('CPU ticks') 532 ax.legend(loc=lloc)

533 534 @classmethod

535 - def _parse(cls, lines, key):

536 from numpy import array, arange 537 myhead = 'TP_%s: ' % key 538 nmyhead = len(myhead) 539 data = list() 540 for line in lines: 541 loc = line.find(myhead) 542 if loc > -1: 543 data.append([int(val) for val in line[loc+nmyhead:].split()]) 544 arr = array(data, dtype='int32') 545 arr[1:,:] = arr[1:,:] - arr[:-1,:] 546 arr[0,:] = arr[1,:] 547 xval = arange(arr.shape[0])+1 548 xlabel = 'Steps' 549 return arr, xval.copy(), xlabel

550

551 - def postfull(self):

552 for key in self.svr.mmnames: 553 if key not in self.svr.ticker: 554 continue 555 vals = self.svr.ticker[key] 556 nval = len(vals) 557 self.svr.mesg('TP_%s: %s\n' % ( 558 key, ' '.join(['%d'%val for val in vals]) 559 ))

560

561 ################################################################################ 562 # Initialization. 563 ################################################################################ 564 565 -class FillAnchor(Anchor):

566 """ 567 Fill the array with value. 568 """

569 - def __init__(self, svr, **kw):

570 self.keys = kw.pop('keys') 571 self.value = kw.pop('value') 572 super(FillAnchor, self).__init__(svr, **kw)

573 - def provide(self):

574 for key in self.keys: 575 getattr(self.svr, key).fill(self.value)

576

Source Code for Module solvcon.anchor