solvcon.io.domain

37 """ 38 @cvar META_GLOBAL: global meta entries. 39 @ctype META_GLOBAL: tuple 40 @cvar META_SWITCH: optional flags. 41 @ctype META_SWITCH: tuple 42 43 @ivar compressor: the compression to use: '', 'gz', or 'bz2' 44 @itype compressor: str 45 @ivar blk_format_rev: the format (revision) of block to be saved. 46 @itype blk_format_rev: str 47 """ 48 49 __metaclass__ = DomainFormatMeta 50 51 FILE_HEADER = '-*- solvcon dom file -*-' 52 DOM_FILENAME = 'domain.dom' 53 WHOLE_FILENAME = 'whole.blk' 54 SPLIT_FILENAME = 'part%d.blk' 55 56 SPEC_OF_META = ( 57 ('GLOBAL', str), 58 ('SWITCH', strbool), 59 ('SHAPE', int), 60 ('IDXINFO', None), 61 ) 62 META_GLOBAL = ('FORMAT_REV', 'compressor', 'blk_format_rev',) 63 META_SWITCH = tuple() 64 META_SHAPE = ('edgecut', 'nnode', 'nface', 'ncell', 65 'npart', 'nifp', 'ndmblk',) 66

67 - def __init__(self, **kw):

68 self.compressor = kw.pop('compressor', '') 69 self.blk_format_rev = kw.pop('blk_format_rev', self.FORMAT_REV) 70 super(DomainFormat, self).__init__()

71 - def read_meta(self, dirname):

72 """ 73 Read meta-data from directory 74 75 @param dirname: directory to be read. 76 @type dirname: str 77 @return: meta-data, raw text lines of meta-data, and the length of 78 meta-data in bytes. 79 @rtype: solvcon.gendata.AttributeDict 80 """ 81 import os 82 stream = open(os.path.join(dirname, self.DOM_FILENAME), 'rb') 83 meta = self._parse_meta(self._get_textpart(stream)[0]) 84 stream.close() 85 return meta

86 - def save(self, dom, dirname):

87 """ 88 Save the dom object into a file. 89 90 @param dom: to-be-written domain object; must be split. 91 @type dom: solvcon.domain.Collective 92 @param dirname: the directory to save data. 93 @type dirname: str 94 """ 95 import os 96 from .block import blfregy 97 stream = open(os.path.join(dirname, self.DOM_FILENAME), 'wb') 98 # text part. 99 stream.write(self.FILE_HEADER + '\n') 100 self._save_meta(dom, stream) 101 self._save_idxinfo_shape(dom, stream) 102 self._save_block_filenames(dirname, dom, stream) 103 stream.write(self.BINARY_MARKER + '\n') 104 # binary part. 105 self._write_array(self.compressor, dom.part, stream) 106 self._write_array(self.compressor, dom.shapes, stream) 107 self._write_array(self.compressor, dom.ifparr, stream) 108 for maparr in dom.mappers: 109 self._write_array(self.compressor, maparr, stream) 110 for mynds, myfcs, mycls in dom.idxinfo: 111 self._write_array(self.compressor, mynds, stream) 112 self._write_array(self.compressor, myfcs, stream) 113 self._write_array(self.compressor, mycls, stream) 114 stream.close() 115 # blocks. 116 blf = blfregy[self.blk_format_rev](compressor=self.compressor) 117 stream = open(os.path.join(dirname, self.WHOLE_FILENAME), 'wb') 118 blf.save(dom.blk, stream) 119 stream.close() 120 for iblk in range(len(dom)): 121 stream = open( 122 os.path.join(dirname, self.SPLIT_FILENAME%iblk), 'wb') 123 blf.save(dom[iblk], stream) 124 stream.close()

125 - def load(self, dirname, bcmapper, with_arrs, with_whole, with_split, 126 return_filenames, domaintype):

127 """ 128 Load domain file in the specified directory with BC mapper applied. 129 130 @param dirname: directory name of domain. 131 @type dirname: str 132 @param bcmapper: BC type mapper. 133 @type bcmapper: dict 134 @param with_arrs: load arrays for domain object. 135 @type with_arrs: bool 136 @param with_whole: load whole block. 137 @type with_whole: bool 138 @param with_split: load split block as well. 139 @type with_split: bool 140 @param return_filenames: also return the relative paths of containing 141 filenames. 142 @type return:filenames: bool 143 @param domaintype: the type used to instantiate domain object. 144 @type domaintype: solvcon.domain.Collective 145 @return: the read domain object. 146 @rtype: solvcon.domain.Collective 147 """ 148 import os 149 from .block import blfregy 150 from ..domain import Collective 151 assert issubclass(domaintype, Collective) 152 dom = domaintype(None) 153 stream = open(os.path.join(dirname, self.DOM_FILENAME), 'rb') 154 # determine the text part and binary part. 155 lines, textlen = self._get_textpart(stream) 156 # create meta-data dict. 157 meta = self._parse_meta(lines) 158 dom.edgecut = meta.edgecut 159 # load idxinfo shape. 160 idxlens = self._load_idxinfo_shape(meta, lines) 161 # load filename. 162 whole, split = self._load_block_filename(meta, lines) 163 # load arrays. 164 stream.seek(textlen) 165 seek_only = not with_arrs # if not reading arrays, seek only. 166 dom.part = self._read_array(meta.compressor, 167 (meta.ncell,), 'int32', stream, seek_only=seek_only) 168 dom.shapes = self._read_array(meta.compressor, 169 (meta.npart, 7), 'int32', stream, seek_only=seek_only) 170 dom.ifparr = self._read_array(meta.compressor, 171 (meta.nifp, 2), 'int32', stream) # this must be read. 172 ndmaps = self._read_array(meta.compressor, 173 (meta.nnode, 1+2*meta.ndmblk), 'int32', stream, seek_only=seek_only) 174 fcmaps = self._read_array(meta.compressor, 175 (meta.nface, 5), 'int32', stream, seek_only=seek_only) 176 clmaps = self._read_array(meta.compressor, 177 (meta.ncell, 2), 'int32', stream, seek_only=seek_only) 178 dom.mappers = (ndmaps, fcmaps, clmaps) 179 idxinfo = list() 180 for nnd, nfc, ncl in idxlens: 181 mynds = self._read_array(meta.compressor, (nnd,), 'int32', 182 stream, seek_only=seek_only) 183 myfcs = self._read_array(meta.compressor, (nfc,), 'int32', 184 stream, seek_only=seek_only) 185 mycls = self._read_array(meta.compressor, (ncl,), 'int32', 186 stream, seek_only=seek_only) 187 idxinfo.append((mynds, myfcs, mycls)) 188 dom.idxinfo = tuple(idxinfo) 189 stream.close() 190 # load blocks. 191 only_meta = not with_whole 192 blf = blfregy[meta.blk_format_rev]() 193 stream = open(os.path.join(dirname, whole), 'rb') 194 dom.blk = blf.load(stream, bcmapper, only_meta=only_meta) 195 stream.close() 196 if with_split: 197 for sfn in split: 198 stream = open(os.path.join(dirname, sfn), 'rb') 199 dom.append(blf.load(stream, bcmapper)) 200 stream.close() 201 if return_filenames: 202 return dom, whole, split 203 else: 204 return dom

205 - def load_block(self, dirname, blkid, bcmapper, blkfn=None):

206 """ 207 Load block file in the specified directory with BC mapper applied. 208 209 @param dirname: directory name of domain. 210 @type dirname: str 211 @param blkid: the id of the block to be loaded. 212 @type blkid: int or None 213 @param bcmapper: BC type mapper. 214 @type bcmapper: dict 215 @keyword blkfn: the file name of the block to be loaded; relative path. 216 @type blkfn: str 217 @return: the read block object. 218 @rtype: solvcon.block.Block 219 """ 220 import os 221 from time import sleep 222 from random import seed, random 223 from .block import BlockIO 224 # load the text part of DOM file for block filename. 225 if blkfn is None: 226 stream = open(os.path.join(dirname, self.DOM_FILENAME), 'rb') 227 lines, textlen = self._get_textpart(stream) 228 meta = self._parse_meta(lines) 229 whole, split = self._load_block_filename(meta, lines) 230 stream.close() 231 blkfn = whole if blkid == None else split[blkid] 232 # load block. 233 blkpath = os.path.join(dirname, blkfn) 234 ## need to capture random parallel input error. 235 seed(blkid) 236 itry = 0 237 while True: 238 try: 239 stream = open(blkpath, 'rb') 240 except IOError, e: 241 if itry <= 100: 242 itry += 1 243 sleep(1.0+random()) 244 else: 245 e.args = list(e.args) + [blkid, itry] 246 raise 247 else: 248 break 249 blk = BlockIO().load(stream=stream, bcmapper=bcmapper) 250 stream.close() 251 return blk

252 253 ############################################################################ 254 # Facilities for writing. 255 ############################################################################

256 - def _save_meta(self, dom, stream):

257 """ 258 @param dom: partitioned domain object to store. 259 @type dom: solvcon.domain.Collective 260 @param stream: output stream. 261 @type stream: file 262 @return: nothing. 263 """ 264 # auto sections. 265 for secname in 'GLOBAL', 'SWITCH': 266 for key in getattr(self, 'META_'+secname): 267 skey = key 268 if hasattr(key, '__iter__'): 269 key, skey = key 270 stream.write('%s = %s\n' % (key, str(getattr(self, key)))) 271 # shape. 272 stream.write('%s = %d\n' % ('edgecut', dom.edgecut)) 273 stream.write('%s = %d\n' % ('nnode', dom.blk.nnode)) 274 stream.write('%s = %d\n' % ('nface', dom.blk.nface)) 275 stream.write('%s = %d\n' % ('ncell', dom.blk.ncell)) 276 stream.write('%s = %d\n' % ('npart', len(dom))) 277 stream.write('%s = %d\n' % ('nifp', dom.ifparr.shape[0])) 278 ndmaps, fcmaps, clmaps = dom.mappers 279 assert ndmaps.shape[1]%2 == 1 280 stream.write('%s = %d\n' % ('ndmblk', (ndmaps.shape[1]-1)/2))

281 @staticmethod

282 - def _save_idxinfo_shape(dom, stream):

283 nblk = len(dom) 284 for iblk in range(nblk): 285 key = 'idxinfo%d' % iblk 286 spe = ' '.join(['%d'%arr.shape[0] for arr in dom.idxinfo[iblk]]) 287 stream.write('%s = %s\n' % (key, spe))

288 @staticmethod

289 - def _save_block_filenames(dirname, dom, stream):

290 import os 291 stream.write('%s = %s\n' % ('whole', 'whole.blk')) 292 for iblk in range(len(dom)): 293 stream.write('%s = %s\n' % ('part%d'%iblk, 'part%d.blk'%iblk))

294 295 ############################################################################ 296 # Facilities for reading. 297 ############################################################################ 298 @classmethod

299 - def _load_idxinfo_shape(cls, meta, lines):

300 """ 301 @param meta: meta information dictionary. 302 @type meta: solvcon.gendata.AttributeDict 303 @param lines: text data 304 @type lines: list 305 @return: length of indices. 306 @rtype: list 307 """ 308 meta_length = cls.meta_length 309 begin = meta_length + 1 310 end = meta_length + 1 + meta.npart 311 idxlens = list() 312 for line in lines[begin:end]: 313 try: 314 toks = line.split('=')[-1].strip().split() 315 if len(toks) != 3: 316 raise IndexError('must have 3 tokens') 317 idxlens.append([int(tok) for tok in toks]) 318 except StandardError as e: 319 e.value += '; wrong format in the %d-th index length' % len( 320 idxlens) 321 raise e 322 return idxlens

323 @classmethod

324 - def _load_block_filename(cls, meta, lines):

325 """ 326 @param meta: meta information dictionary. 327 @type meta: solvcon.gendata.AttributeDict 328 @param lines: text data 329 @type lines: list 330 @return: filenames for whole and split blocks. 331 @rtype: str, list of str 332 """ 333 meta_length = cls.meta_length 334 # filename for whole block. 335 end = meta_length + 1 + meta.npart 336 whole = lines[end].split('=')[-1].strip() 337 # filenames for split blocks. 338 begin = end + 1 339 end = begin + meta.npart 340 split = [line.split('=')[-1].strip() for line in lines[begin:end]] 341 return whole, split

350 """ 351 Proxy to dom directory format. 352 353 @ivar dom: attached block object. 354 @itype dom: solvcon.block.Block 355 @ivar dirname: directory name of domain. 356 @itype dirname: str 357 @ivar fmt: name of the format to use; can be either class name or 358 revision string. 359 @itype fmt: str 360 361 @ivar compressor: the compression to use: '', 'gz', or 'bz2' 362 @itype compressor: str 363 @ivar dmf: the format class for the domain to be read. 364 @itype dmf: DomainFormat 365 """

366 - def __init__(self, **kw):

367 self.dom = kw.pop('dom', None) 368 self.dirname = kw.pop('dirname', None) 369 fmt = kw.pop('fmt', 'TrivialDomainFormat') 370 compressor = kw.pop('compressor', '') 371 super(DomainIO, self).__init__() 372 # create DomainFormat object. 373 self.dmf = dmfregy[fmt](compressor=compressor)

374 - def save(self, dom=None, dirname=None):

375 """ 376 Save the block object into a file. 377 378 @keyword dom: to-be-written domain object. 379 @type dom: solvcon.domain.Domain 380 @keyword dirname: directory name to be read. 381 @type dirname: str 382 """ 383 dom = self.dom if dom == None else dom 384 dirname = self.dirname if dirname == None else dirname 385 self.dmf.save(dom, dirname)

386 - def read_meta(self, dirname=None):

387 """ 388 Read meta-data of dom file from stream. 389 390 @keyword dirname: the directory of domain data. 391 @type dirname: str 392 @return: meta-data, raw text lines of meta-data, and the length of 393 meta-data in bytes. 394 @rtype: solvcon.gendata.AttributeDict, list, int 395 """ 396 dirname = self.dirname if dirname == None else dirname 397 return self.dmf.read_meta(dirname)

398 - def load(self, dirname=None, bcmapper=None, with_arrs=True, 399 with_whole=True, with_split=False, return_filenames=False, 400 domaintype=None):

401 """ 402 Load domain from stream with BC mapper applied. 403 404 @keyword dirname: directory name to be read. 405 @type dirname: str 406 @keyword bcmapper: BC type mapper. 407 @type bcmapper: dict 408 @keyword with_arrs: load arrays for domain object. 409 @type with_arrs: bool 410 @keyword with_whole: load whole block. 411 @type with_whole: bool 412 @keyword with_split: load split block as well. 413 @type with_split: bool 414 @keyword return_filenames: also return the relative paths of containing 415 filenames. 416 @type return:filenames: bool 417 @keyword domaintype: the type used to instantiate domain object. 418 @type domaintype: type 419 @return: the read domain object. 420 @rtype: solvcon.domain.Collective 421 """ 422 from .. import domain 423 dirname = self.dirname if dirname == None else dirname 424 if domaintype is None: 425 domaintype = domain.Collective 426 elif isinstance(domaintype, basestring): 427 domiantype = getattr(domain, domaintype) 428 return self.dmf.load(dirname, bcmapper, with_arrs, with_whole, 429 with_split, return_filenames, domaintype)

430 - def load_block(self, dirname=None, blkid=None, bcmapper=None, blkfn=None):

431 """ 432 Load block from stream with BC mapper applied. 433 434 @keyword dirname: directory name to be read. 435 @type dirname: str 436 @keyword blkid: the id of block to be read. 437 @type blkid: int 438 @keyword bcmapper: BC type mapper. 439 @type bcmapper: dict 440 @keyword blkfn: the file name of the block to be loaded; relative path. 441 @type blkfn: str 442 @return: the read block object. 443 @rtype: solvcon.block.Block 444 """ 445 dirname = self.dirname if dirname == None else dirname 446 return self.dmf.load_block(dirname, blkid, bcmapper, blkfn=blkfn)

Source Code for Module solvcon.io.domain