1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 """
20 Commands for users.
21 """
22
23 from .cmdutil import Command
24
25 -class mpi(Command):
26 """
27 Utilities for MPI.
28 """
29
30 min_args = 0
31
33 from optparse import OptionGroup
34 super(mpi, self).__init__(env)
35 op = self.op
36 opg = OptionGroup(op, 'MPI')
37 opg.add_option('--node-env', action='store',
38 dest='node_env', default='PBS_NODEFILE',
39 help='Environment variable of the host file.',
40 )
41 opg.add_option('--compress-nodelist', action='store_true',
42 dest='compress_nodelist', default=False,
43 help='To compress nodelist on the head node.',
44 )
45 opg.add_option('--head-last', action='store_true',
46 dest='head_last', default=False,
47 help='Make the head node to be last in the host file.',
48 )
49 op.add_option_group(opg)
50 self.opg_arrangement = opg
51
53 import os, sys
54 ops, args = self.opargs
55 f = open(os.environ[ops.node_env])
56 nodes = [node.strip() for node in f.readlines()]
57 f.close()
58 if ops.compress_nodelist:
59 newnodes = []
60 for node in nodes:
61 if not newnodes or newnodes[-1] != node:
62 newnodes.append(node)
63 nodes = newnodes
64 if ops.head_last:
65 first = nodes[0]
66 ind = 1
67 while ind < len(nodes):
68 if nodes[ind] != first:
69 break
70 ind += 1
71 nodes = nodes[ind:] + nodes[:ind]
72 if len(args) > 0:
73 f = open(args[0], 'w')
74 f.write('\n'.join([node.strip() for node in nodes]))
75 f.close()
76 else:
77 sys.stdout.write('\n'.join([node.strip() for node in nodes]))
78
80 """
81 Mesh manipulation.
82 """
83
84 min_args = 1
85
87 from optparse import OptionGroup
88 super(mesh, self).__init__(env)
89 op = self.op
90
91 opg = OptionGroup(op, 'Mesh')
92 opg.add_option('--ascii', action='store_false',
93 dest='binary', default=True,
94 help='Use ASCII in VTK (default is binary).',
95 )
96 opg.add_option('--encoding', action='store', type='string',
97 dest='encoding', default='raw',
98 help='The encoding used in VTK XML binary data (raw/base64). '
99 'Must be base64 for inline binary data.',
100 )
101 opg.add_option('--inline', action='store_false',
102 dest='appended', default=True,
103 help='Inline binary data in VTK XML file.',
104 )
105 opg.add_option('--fpdtype', action='store', type='string',
106 dest='fpdtype', default='float64',
107 help='dtype for floating-point (default is float64).',
108 )
109 opg.add_option('--formats', action='store', type='string',
110 dest='formats', default='',
111 help='Assign the I/O formats as InputIO,OutputIO.',
112 )
113 opg.add_option('--compressor', action='store', type='string',
114 dest='compressor', default='',
115 help='Empty string (no compression), gz or bz2.',
116 )
117 opg.add_option('--split', action='store', type='int',
118 dest='split', default=None,
119 help='Split the loaded block into given number of parts.',
120 )
121 opg.add_option('--bc-reject', action='store', type='string',
122 dest='bc_reject', default='',
123 help='The BC (name) to be rejected in conversion.',
124 )
125 opg.add_option('--no-print-block', action='store_false',
126 dest='print_block', default=True,
127 help='Prevent printing block information.',
128 )
129 opg.add_option('--no-print-bcs', action='store_false',
130 dest='print_bc', default=True,
131 help='Prevent printing BC objects information.',
132 )
133 op.add_option_group(opg)
134 self.opg_arrangement = opg
135
136 @staticmethod
146 @staticmethod
147 - def _save_domain(ops, blk, dirname):
148 import os
149 from time import time
150 from .domain import Collective
151 from .io.domain import DomainIO
152 from .helper import info
153 info('Create domain ... ')
154 timer = time()
155 dom = Collective(blk)
156 info('done. (%gs)\n' % (time()-timer))
157 info('Split domain into %d parts ... ' % ops.split)
158 timer = time()
159 dom.split(ops.split)
160 info('done. (%gs)\n' % (time()-timer))
161 dio = DomainIO(dom=dom, compressor=ops.compressor)
162 if not os.path.exists(dirname):
163 os.makedirs(dirname)
164 info('Save to directory %s/ ... ' % dirname)
165 timer = time()
166 dio.save(dirname=dirname)
167 info('done. (%gs)\n' % (time()-timer))
168 @staticmethod
179 @staticmethod
180 - def _save_vtkxml(ops, blk, vtkfn, appended, binary, encoding, compressor,
181 fpdtype):
196 @staticmethod
236 import os
237 from time import time
238 from .helper import info
239 ops, args = self.opargs
240
241 iio, oio = self._determine_formats(ops, args)
242
243 info('Load %s (%s) ... ' % (args[0], type(iio).__name__))
244 timer = time()
245 blk = iio.load(stream=args[0])
246 info('done. (%gs)\n' % (time()-timer))
247
248 if ops.print_block:
249 info('Block information:')
250 info('\n %s\n' % str(blk))
251 if ops.print_bc:
252 for bc in blk.bclist:
253 info(' %s\n' % str(bc))
254 info(' Cell groups:\n')
255 for igrp in range(len(blk.grpnames)):
256 grpname = blk.grpnames[igrp]
257 info(' grp#%d: %s\n' % (igrp, grpname))
258 info(' Cell volume (min, max, all): %g, %g, %g.\n' % (
259 blk.clvol.min(), blk.clvol.max(), blk.clvol.sum()))
260
261 if oio is not None:
262 path = args[1]
263 if oio == 'BlockIO':
264 self._save_block(ops, blk, path)
265 elif oio == 'DomainIO':
266 if ops.split is None:
267 info('No saving: split must be specified.\n')
268 return
269 self._save_domain(ops, blk, path)
270 elif oio == 'VtkLegacy':
271 self._save_vtklegacy(ops, blk, path, ops.binary, ops.fpdtype)
272 elif oio == 'VtkXml':
273 self._save_vtkxml(ops, blk, path, ops.appended, ops.binary,
274 ops.encoding, ops.compressor, ops.fpdtype)
275
277 """
278 Actions related Solver log.
279 """
280
282 from optparse import OptionGroup
283 super(SolverLog, self).__init__(env)
284 op = self.op
285
286 opg = OptionGroup(op, 'Solver Log')
287 opg.add_option('-f', action='store',
288 dest='filename', default=None,
289 help='Save plot to a file with specified name.',
290 )
291 opg.add_option('--backend', action='store',
292 dest='backend', default='TkAgg',
293 help='The backend for matplotlib. Default is TkAgg.',
294 )
295 op.add_option_group(opg)
296 self.opg_arrangement = opg
297
299 import os, glob
300 ops, args = self.opargs
301 fn = args[0]
302 fns = list()
303 if os.path.isdir(fn):
304 if ops.filename != None:
305 main, ext = os.path.splitext(ops.filename)
306 dsttmpl = main+'%d'+ext
307 else:
308 dsttmpl = None
309 nfn = len(glob.glob(os.path.join(fn, 'solvcon.solver*.log')))
310 for idx in range(nfn):
311 src = os.path.join(fn, 'solvcon.solver%d.log'%idx)
312 if dsttmpl != None:
313 dst = dsttmpl%idx
314 else:
315 dst = None
316 lines = open(src).readlines()
317 fns.append((lines, src, dst))
318 else:
319 lines = open(fn).readlines()
320 fns.append((lines, fn, ops.filename))
321 return fns
322
324 """
325 Show output from RuntimeStatAnchor.
326 """
327
328 min_args = 1
329 PLOTS = ['cpu', 'loadavg', 'mem']
330
332 from optparse import OptionGroup
333 super(log_runtime, self).__init__(env)
334 op = self.op
335
336 opg = OptionGroup(op, 'Show Runtime')
337 opg.add_option('-c', action='store_true',
338 dest='cpu', default=False,
339 help='Plot CPU usage.',
340 )
341 opg.add_option('-l', action='store_true',
342 dest='loadavg', default=False,
343 help='Plot load average.',
344 )
345 opg.add_option('-m', action='store_true',
346 dest='mem', default=False,
347 help='Plot memory usage.',
348 )
349 opg.add_option('-t', action='store_true',
350 dest='xtime', default=False,
351 help='Use time as x-axis.',
352 )
353 opg.add_option('--lloc', action='store',
354 dest='lloc', default=None,
355 help='Legend location. Default is None (by plot).',
356 )
357 opg.add_option('--scale', action='store', type=int,
358 dest='scale', default=0.6,
359 help='The scale when having more than one subplot.'
360 ' Default is 0.6.',
361 )
362 op.add_option_group(opg)
363 self.opg_arrangement = opg
364
366 import matplotlib
367 ops, args = self.opargs
368 figsize = matplotlib.rcParams['figure.figsize']
369 top = matplotlib.rcParams['figure.subplot.top']
370 bottom = matplotlib.rcParams['figure.subplot.bottom']
371 if nplot > 1:
372 upscale = nplot*ops.scale
373 top = 1.0 - (1.0-top)*(1.0-top)/((1.0-top)*upscale)
374 bottom = bottom*bottom/(bottom*upscale)
375 figsize = figsize[0], figsize[1]*upscale
376 matplotlib.rcParams.update({
377 'backend': ops.backend,
378 'figure.figsize': figsize,
379 'figure.subplot.top': top,
380 'figure.subplot.bottom': bottom,
381 })
382 matplotlib.use(ops.backend)
383
385 import os, sys
386 from .anchor import RuntimeStatAnchor
387 ops, args = self.opargs
388
389 nplot = 0
390 for key in self.PLOTS:
391 if getattr(ops, key):
392 nplot += 1
393 self._init_mpl(nplot)
394 from matplotlib import pyplot as plt
395
396 datas = self._get_datas()
397
398 for lines, src, dst in datas:
399 if nplot:
400 fig = plt.figure()
401 iplot = 1
402 for key in self.PLOTS:
403 if getattr(ops, key):
404 ax = fig.add_subplot(nplot, 1, iplot)
405 kws = {
406 'xtime': ops.xtime,
407 'showx': iplot==nplot,
408 }
409 if ops.lloc != None:
410 kws['lloc'] = ops.lloc
411 getattr(RuntimeStatAnchor, 'plot_'+key)(lines, ax, **kws)
412 iplot += 1
413 if nplot:
414 sys.stdout.write('%s processed' % src)
415 if dst != None:
416 plt.savefig(dst)
417 sys.stdout.write(' and written to %s.' % dst)
418 sys.stdout.write('\n')
419
420 if nplot and ops.filename == None:
421 plt.show()
422
424 """
425 Show output from MarchStatAnchor.
426 """
427
428 min_args = 1
429
431 from optparse import OptionGroup
432 super(log_march, self).__init__(env)
433 op = self.op
434
435 opg = OptionGroup(op, 'Show March')
436 opg.add_option('-k', action='store',
437 dest='plotkeys', default='',
438 help='Keys to plot.',
439 )
440 opg.add_option('--lloc', action='store',
441 dest='lloc', default=None,
442 help='Legend location. Default is None (by plot).',
443 )
444 opg.add_option('--scale', action='store', type=int,
445 dest='scale', default=0.6,
446 help='The scale when having more than one subplot.'
447 ' Default is 0.6.',
448 )
449 op.add_option_group(opg)
450 self.opg_arrangement = opg
451
453 import matplotlib
454 ops, args = self.opargs
455 matplotlib.rcParams.update({
456 'backend': ops.backend,
457 })
458 matplotlib.use(ops.backend)
459
461 import os, sys
462 from .anchor import MarchStatAnchor
463 ops, args = self.opargs
464
465 plotkeys = ops.plotkeys.split(',')
466 nplot = len(plotkeys)
467 self._init_mpl(nplot)
468 from matplotlib import pyplot as plt
469
470 datas = self._get_datas()
471
472 if nplot == 0:
473 return
474 for lines, src, dst in datas:
475 fig = plt.figure()
476 ax = fig.add_subplot(1, 1, 1)
477 kws = dict()
478 if ops.lloc != None:
479 kws['lloc'] = ops.lloc
480 MarchStatAnchor.plot(plotkeys, lines, ax, **kws)
481 sys.stdout.write('%s processed' % src)
482 if dst != None:
483 plt.savefig(dst)
484 sys.stdout.write(' and written to %s.' % dst)
485 sys.stdout.write('\n')
486
487 if ops.filename == None:
488 plt.show()
489
491 """
492 Show output from TpoolStatAnchor.
493 """
494
495 min_args = 1
496
498 from optparse import OptionGroup
499 super(log_tpool, self).__init__(env)
500 op = self.op
501
502 opg = OptionGroup(op, 'Show Tpool')
503 opg.add_option('-k', action='store',
504 dest='plotkeys', default='',
505 help='Keys to plot.',
506 )
507 opg.add_option('--lloc', action='store',
508 dest='lloc', default=None,
509 help='Legend location. Default is None (by plot).',
510 )
511 opg.add_option('--scale', action='store', type=int,
512 dest='scale', default=0.6,
513 help='The scale when having more than one subplot.'
514 ' Default is 0.6.',
515 )
516 op.add_option_group(opg)
517 self.opg_arrangement = opg
518
520 import matplotlib
521 ops, args = self.opargs
522 figsize = matplotlib.rcParams['figure.figsize']
523 top = matplotlib.rcParams['figure.subplot.top']
524 bottom = matplotlib.rcParams['figure.subplot.bottom']
525 if nplot > 1:
526 upscale = nplot*ops.scale
527 top = 1.0 - (1.0-top)*(1.0-top)/((1.0-top)*upscale)
528 bottom = bottom*bottom/(bottom*upscale)
529 figsize = figsize[0], figsize[1]*upscale
530 matplotlib.rcParams.update({
531 'backend': ops.backend,
532 'figure.figsize': figsize,
533 'figure.subplot.top': top,
534 'figure.subplot.bottom': bottom,
535 })
536 matplotlib.use(ops.backend)
537
539 import os, sys
540 from .anchor import TpoolStatAnchor
541 ops, args = self.opargs
542
543 plotkeys = ops.plotkeys.split(',')
544 nplot = len(plotkeys)
545 self._init_mpl(nplot)
546 from matplotlib import pyplot as plt
547
548 datas = self._get_datas()
549
550 for lines, src, dst in datas:
551 if nplot:
552 fig = plt.figure()
553 iplot = 1
554 for key in plotkeys:
555 ax = fig.add_subplot(nplot, 1, iplot)
556 kws = {
557 'showx': iplot==nplot,
558 }
559 if ops.lloc != None:
560 kws['lloc'] = ops.lloc
561 TpoolStatAnchor.plot(key, lines, ax, **kws)
562 iplot += 1
563 if nplot:
564 sys.stdout.write('%s processed' % src)
565 if dst != None:
566 plt.savefig(dst)
567 sys.stdout.write(' and written to %s.' % dst)
568 sys.stdout.write('\n')
569
570 if nplot and ops.filename == None:
571 plt.show()
572
574 """
575 @ivar opg_arrangement: group for options for arrangement.
576 @itype opg_arrangement: optparse.OptionGroup
577 """
578
580 from optparse import OptionGroup
581 super(ArrangementCommand, self).__init__(env)
582 op = self.op
583
584 opg = OptionGroup(op, 'Arrangement')
585 opg.add_option('--runlevel', action='store', type=int,
586 dest='runlevel', default=0,
587 help='0: fresh run, 1: restart, 2: init only.',
588 )
589 opg.add_option('--solver-output', action='store_true',
590 dest='solver_output', default=False,
591 help='Turn on the output device in the solver object(s).',
592 )
593 opg.add_option('--npart', action='store', type=int,
594 dest='npart', default=None,
595 help='The number of partitions.',
596 )
597 opg.add_option('--compress-nodelist', action='store_true',
598 dest='compress_nodelist', default=False,
599 help='To compress nodelist on the head node.',
600 )
601 opg.add_option('-e', '--envar', action='store',
602 dest='envar', default=None,
603 help='Additional environmental variable to remote solvers.',
604 )
605 opg.add_option('-b', '--batch', action='store',
606 dest='batch', default='Batch',
607 help='The name of batch system.',
608 )
609 opg.add_option('--use-profiler', action='store_true',
610 dest='use_profiler', default=False,
611 help='Flag to use profiler in running or not.',
612 )
613 opg.add_option('--profiler-sort', action='store', type='string',
614 dest='profiler_sort', default='cum,time',
615 help='Fields for sorting stats in profiler; comma separated.',
616 )
617 opg.add_option('--profiler-dat', action='store', type='string',
618 dest='profiler_dat', default='profiler.dat',
619 help='File name for raw profiler output.',
620 )
621 opg.add_option('--profiler-log', action='store', type='string',
622 dest='profiler_log', default='profiler.log',
623 help='File name for human-readable profiler output.',
624 )
625 opg.add_option('--basedir', action='store',
626 dest='basedir', default='',
627 help='Suggested basedir (may or may not used by arrangement).',
628 )
629 opg.add_option('--test', action='store_true',
630 dest='test', default=False,
631 help='General flags for test run.',
632 )
633 op.add_option_group(opg)
634 self.opg_arrangement = opg
635
636 @property
638 ops, args = self.opargs
639 dct = dict()
640 if ops.envar:
641 for ent in ops.envar.split(','):
642 key, val = [it.strip() for it in ent.split('=')]
643 dct[key] = val
644 return dct
645
646 -class run(ArrangementCommand):
647 """
648 Run arrangement.
649 """
650
651 min_args = 0
652
654 import os
655 import cProfile
656 import pstats
657 from socket import gethostname
658 from .helper import info
659 from .conf import use_application, env
660 from . import domain
661 from .batch import batregy
662 from .case import arrangements
663 from .rpc import Worker, DEFAULT_AUTHKEY
664 ops, args = self.opargs
665 if len(args) > 0:
666 name = args[0]
667 else:
668 name = os.path.basename(os.getcwd())
669
670 batch = batregy[ops.batch]
671
672 npart = ops.npart
673 if npart != None:
674 if batch == batregy.Batch:
675 domaintype = domain.Collective
676 else:
677 domaintype = domain.Distributed
678 else:
679 domaintype = domain.Domain
680
681 funckw = {
682 'envar': self.envar,
683 'runlevel': ops.runlevel,
684 'solver_output': ops.solver_output,
685 'batch': batch,
686 'npart': npart, 'domaintype': domaintype,
687 }
688 func = arrangements[name]
689 if env.mpi and env.mpi.rank != 0:
690 pdata = (
691 ops.profiler_dat,
692 ops.profiler_log,
693 ops.profiler_sort,
694 ) if ops.use_profiler else None
695 wkr = Worker(None, profiler_data=pdata)
696 wkr.run(('0.0.0.0', 0), DEFAULT_AUTHKEY)
697 else:
698 if ops.use_profiler:
699 cProfile.runctx('func(submit=False, **funckw)',
700 globals(), locals(), ops.profiler_dat)
701 plog = open(ops.profiler_log, 'w')
702 p = pstats.Stats(ops.profiler_dat, stream=plog)
703 p.sort_stats(*ops.profiler_sort.split(','))
704 p.dump_stats(ops.profiler_dat)
705 p.print_stats()
706 plog.close()
707 info('*** Profiled information saved in '
708 '%s (raw) and %s (text).\n' % (
709 ops.profiler_dat, ops.profiler_log))
710 else:
711 func(submit=False, **funckw)
712
713 -class submit(ArrangementCommand):
714 """
715 Submit arrangement to batch system.
716 """
717
718 min_args = 1
719
721 from optparse import OptionGroup
722 super(submit, self).__init__(env)
723 op = self.op
724
725 opg = OptionGroup(op, 'Batching')
726 opg.add_option('-l', '--resources', action='store',
727 dest='resources', default='',
728 help='Resource list with "," as delimiter.',
729 )
730 opg.add_option('--use-mpi', action='store_true',
731 dest='use_mpi', default=False,
732 help='Indicate to use MPI as transport layer.',
733 )
734 opg.add_option('--postpone', action='store_true',
735 dest='postpone', default=False,
736 help='Postpone feeding into batch system.',
737 )
738 op.add_option_group(opg)
739 self.opg_batch = opg
740
742 import os
743 from .conf import use_application
744 from .batch import batregy
745 from .case import arrangements
746 ops, args = self.opargs
747 if len(args) > 0:
748 name = args[0]
749 else:
750 name = os.path.basename(os.getcwd())
751
752 for modname in self.env.modnames:
753 use_application(modname)
754
755 resources = dict([(line, None) for line in ops.resources.split(',')])
756
757 batch = batregy[ops.batch]
758
759 arrangements[name](submit=True,
760 use_mpi=ops.use_mpi, postpone=ops.postpone,
761 envar=self.envar,
762 runlevel=ops.runlevel,
763 resources=resources, batch=batch, npart=ops.npart,
764 )
765
766 -class help(Command):
767 """
768 Print general help.
769 """
770
772 ops, args = self.opargs
773 self.op.print_help()
774
775 @property
778