00001
00002
00003 """:py:class:`PSCalib.CalibFileFinder` is a python version of CalibFileFinder.cpp - finds calibration file
00004
00005 Usage::
00006
00007 from PSCalib.CalibFileFinder import CalibFileFinder
00008
00009 cdir = '/reg/d/psdm/CXI/cxi83714/calib/'
00010 group = 'CsPad::CalibV1' # optional parameter, if not available will be set for src from dict
00011 src = 'CxiDs1.0:Cspad.0'
00012 type = 'pedestals'
00013 rnum = 137
00014
00015 cff = CalibFileFinder(cdir, group, pbits=0377)
00016 #OR
00017 cff = CalibFileFinder(cdir)
00018 fname = cff.findCalibFile(src, type, rnum)
00019
00020 fname_new = cff.makeCalibFileName(src, type, run_start, run_end=None)
00021
00022 #-----------------------------------------------
00023 # ALTERNATIVE usage of direct access methods
00024
00025 from PSCalib.CalibFileFinder import find_calib_file, make_calib_file_name
00026
00027 fname_existing = find_calib_file(cdir, src, type, rnum, pbits=1)
00028 fname_new = make_calib_file_name(cdir, src, type, run_start, run_end=None, pbits=1)
00029
00030 #-----------------------------------------------
00031 # Deploy file or numpy array as a file in the calibration store
00032
00033 # use optional dictionary of comments to save in the HISTORY and file
00034 arr = np.ones((32,185,388))
00035 cmts = {'exp':'cxi12345', 'ifname':'input-file-name', 'app':'my-app-name', 'comment':'my-comment'}
00036 deploy_calib_array(cdir, src, type, run_start, run_end, arr, cmts, fmt='%.1f', pbits=1)
00037
00038 cmts = {'exp':'cxi12345', 'app':'my-app-name', 'comment':'my-comment'}
00039 ifname='path-to-my-own-calibtation-file/file-name.txt'
00040 deploy_calib_file(cdir, src, type, run_start, run_end, ifnameq, cmts, pbits=1)
00041
00042 This software was developed for the SIT project. If you use all or
00043 part of it, please give an appropriate acknowledgment.
00044
00045 Revision: $Revision: 12322 $
00046
00047 @version $Id: CalibFileFinder.py 8469 2014-06-24 22:55:21Z dubrovin@SLAC.STANFORD.EDU $
00048
00049 @author Mikhail S. Dubrovin
00050 """
00051
00052 __version__ = "$Revision: 12322 $"
00053
00054
00055 import os
00056 import sys
00057 import PSCalib.GlobalUtils as gu
00058 from PSCalib.NDArrIO import save_txt
00059 import tempfile
00060
00061
00062
00063 class CalibFile :
00064
00065 rnum_max = 9999
00066
00067 def __init__(self, path='', pbits=1) :
00068 self.path = path
00069 self.pbits = pbits
00070
00071 fname = os.path.basename(path)
00072 basename = os.path.splitext(fname)[0]
00073
00074 if not ('-' in basename) :
00075 if self.pbits : print 'WARNING! FILE NAME "%s" IS NOT VALID - MISSING DASH' % basename
00076 self.valid = False
00077 return
00078
00079 begin, end = basename.split('-')
00080 self.begin = int(begin)
00081 self.end = int(end) if end != 'end' else self.rnum_max
00082 self.valid = True
00083
00084 def get_path(self) :
00085 return self.path
00086
00087 def get_begin(self) :
00088 return self.begin
00089
00090 def get_end(self) :
00091 return self.end
00092
00093 def __cmp__(self, other) :
00094
00095
00096
00097 if self.begin < other.begin : return -1
00098 elif self.begin > other.begin : return 1
00099 else :
00100 if self.end > other.end : return -1
00101 elif self.end < other.end : return 1
00102 else : return 0
00103
00104 def str_attrs(self) :
00105 return 'begin: %4d end: %4d path: %s' % (self.begin, self.end, self.path)
00106
00107
00108
00109 def find_calib_file(cdir, src, type, rnum, pbits=1) :
00110 return CalibFileFinder(cdir, pbits=pbits).findCalibFile(src, type, rnum)
00111
00112
00113
00114 def make_calib_file_name(cdir, src, type, run_start, run_end=None, pbits=1) :
00115 return CalibFileFinder(cdir, pbits=pbits).makeCalibFileName(src, type, run_start, run_end=None)
00116
00117
00118
00119 def deploy_calib_array(cdir, src, type, run_start, run_end=None, arr=None, dcmts={}, fmt='%.1f', pbits=1) :
00120 """Deploys array in calibration file
00121
00122 - makes the new file name using make_calib_file_name(...)
00123 - if file with this name already exists - rename it with current timestamp in the name
00124 - save array in file
00125 - add history record
00126 """
00127
00128 fname = make_calib_file_name(cdir, src, type, run_start, run_end, pbits)
00129 path_history = '%s/HISTORY'%os.path.dirname(fname)
00130
00131 if os.path.exists(fname) :
00132 fname_bkp = '%s-%s'%(fname, gu.str_tstamp(fmt='%Y-%m-%dT%H:%M:%S'))
00133 os.system('cp %s %s'%(fname, fname_bkp))
00134 if pbits & 1 :
00135 print 'Existing file %s\nis backed-up %s' % (fname, fname_bkp)
00136
00137
00138 d = dict(dcmts)
00139 d['run'] = run_start
00140 d['fname'] = os.path.basename(fname)
00141 d['src'] = src
00142 d['ctype'] = type
00143
00144
00145 cmts=['%s %s'%(k.upper().ljust(11),v) for k,v in d.iteritems()]
00146
00147
00148 fntmp = tempfile.NamedTemporaryFile(mode='r+b',suffix='.data')
00149 if pbits & 2 : print 'Save constants in tmp file: %s' % fntmp.name
00150 save_txt(fntmp.name, arr, cmts, fmt='%.1f')
00151
00152 if pbits & 1 : print 'Deploy constants in file: %s' % fname
00153
00154 cmd_cat = 'cat %s > %s' % (fntmp.name, fname)
00155
00156 stream = os.popen(cmd_cat)
00157 resp = stream.read()
00158 msg = 'Command: %s\n - resp: %s' % (cmd_cat, resp)
00159 if pbits & 2 : print msg
00160
00161
00162 hrec = _history_record(d)
00163 if pbits & 1 : print 'Add record: %sto the file: %s' % (hrec, path_history)
00164 gu.save_textfile(hrec, path_history, mode='a')
00165
00166
00167
00168 def deploy_calib_file(cdir, src, type, run_start, run_end=None, ifname='', dcmts={}, pbits=1) :
00169 """Deploys calibration file
00170
00171 - makes the new file name using make_calib_file_name(...)
00172 - if file with this name already exists - rename it with current timestamp in the name
00173 - save array in file
00174 - add history record
00175 """
00176
00177 fname = make_calib_file_name(cdir, src, type, run_start, run_end, pbits)
00178 path_history = '%s/HISTORY'%os.path.dirname(fname)
00179
00180 if os.path.exists(fname) :
00181 fname_bkp = '%s-%s'%(fname, gu.str_tstamp(fmt='%Y-%m-%dT%H:%M:%S'))
00182 os.system('cp %s %s'%(fname, fname_bkp))
00183 if pbits & 1 :
00184 print 'Existing file %s\nis backed-up %s' % (fname, fname_bkp)
00185
00186
00187 d = dict(dcmts)
00188 d['run'] = run_start
00189 d['fname'] = os.path.basename(fname)
00190 d['ifname']= ifname
00191 d['src'] = src
00192 d['ctype'] = type
00193
00194 if pbits & 1 : print 'Deploy constants in file: %s' % fname
00195
00196 cmd_cat = 'cat %s > %s' % (ifname, fname)
00197
00198 stream = os.popen(cmd_cat)
00199 resp = stream.read()
00200 msg = 'Command: %s\n - resp: %s' % (cmd_cat, resp)
00201 if pbits & 2 : print msg
00202
00203
00204 hrec = _history_record(d)
00205 if pbits & 1 : print 'Add record: %sto the file: %s' % (hrec, path_history)
00206 gu.save_textfile(hrec, path_history, mode='a')
00207
00208
00209
00210 def _history_record(dcmts) :
00211 """Returns history record made of dictionary comments and system info
00212 """
00213 user = gu.get_login()
00214 host = gu.get_hostname()
00215 tstamp = gu.str_tstamp(fmt='%Y-%m-%dT%H:%M:%S zone:%Z')
00216 rnum = '%04d' % dcmts.get('run')
00217 exp = '%s' % dcmts.get('exp')
00218 ifname = '%s' % dcmts.get('ifname')
00219 ofname = '%s' % dcmts.get('fname')
00220 app = '%s' % dcmts.get('app')
00221 cmt = '%s' % dcmts.get('comment')
00222
00223 return 'file:%s copy_of:%s exp:%s run:%s app:%s user:%s host:%s cptime:%s comment:%s\n' % \
00224 (ofname.ljust(14),
00225 ifname,
00226 exp.ljust(8),
00227 rnum.ljust(4),
00228 app.ljust(10),
00229 user,
00230 host,
00231 tstamp.ljust(29),
00232 cmt)
00233
00234
00235
00236 class CalibFileFinder :
00237
00238 def __init__(self, cdir='', group='', pbits=1) :
00239 self.cdir = cdir
00240 self.group = group
00241 self.pbits = pbits
00242
00243
00244 def _setGroup(self, src) :
00245 """If not available, sets group from source.
00246 """
00247 if self.group == '' or self.group is None :
00248 dettype = gu.det_type_from_source(src)
00249 self.group = gu.dic_det_type_to_calib_group.get(dettype)
00250 if self.group is None :
00251 if self.pbits & 1 : print 'WARNING! CALIBRATION GROUP IS NOT FOUND FOR SOURCE %s' % src
00252 return False
00253 return True
00254
00255
00256 def makeCalibFileName(self, src, type, run_start, run_end=None) :
00257 """Returns calibration file name.
00258 """
00259 if os.path.basename(self.cdir.rstrip('/')) != 'calib' :
00260 if self.pbits & 1 : print 'WARNING! NOT calib DIRECTORY: %s' % self.cdir
00261 return None
00262
00263
00264
00265 assert os.path.isdir(self.cdir), 'psana calib-dir must exist: '+self.cdir
00266
00267 if not self._setGroup(src) :
00268 return None
00269
00270 if run_start < 0 :
00271 if self.pbits & 1 : print 'WARNING! START RUN NUMBER IS NEGATIVE: %d' % run_start
00272 return None
00273
00274 if run_start > 9999 :
00275 if self.pbits & 1 : print 'WARNING! START RUN NUMBER EXCEEDS 4-DIGITS: %d' % run_start
00276 return None
00277
00278 if run_end is None :
00279 self.cfname = '%d-end.data' % (run_start)
00280
00281 else :
00282
00283 if run_end < 0 :
00284 if self.pbits & 1 : print 'WARNING! END RUN NUMBER IS NEGATIVE: %d' % run_end
00285 return None
00286
00287 if run_end > 9999 :
00288 if self.pbits & 1 : print 'WARNING! END RUN NUMBER IS TOO BIG: %d' % run_end
00289 return None
00290
00291 if run_end < run_start :
00292 if self.pbits & 1 : print 'WARNING! END RUN:%d < START RUN:%d' % (run_end, run_start)
00293 return None
00294
00295 self.cfname = '%d-%d.data' % (run_start, run_end)
00296
00297 dir = self.cdir
00298 for subdir in (self.group, src, type) :
00299 dir = os.path.join(dir, subdir)
00300 gu.create_directory(dir, self.pbits)
00301
00302 return os.path.join(dir, self.cfname)
00303
00304
00305 def findCalibFile(self, src, type, rnum0) :
00306 """Find calibration file.
00307 """
00308 rnum = rnum0 if rnum0 <= CalibFile.rnum_max else CalibFile.rnum_max
00309
00310
00311
00312 assert os.path.isdir(self.cdir), 'psana calib-dir must exist: '+self.cdir
00313
00314 if not self._setGroup(src) : return ''
00315
00316 dir_name = os.path.join(self.cdir, self.group, src, type)
00317 if not os.path.exists(dir_name) :
00318 if self.pbits & 1 : print 'WARNING! NON-EXISTENT DIR: %s' % dir_name
00319 return ''
00320
00321 fnames = os.listdir(dir_name)
00322 files = [os.path.join(dir_name,fname) for fname in fnames]
00323 return self.selectCalibFile(files, rnum)
00324
00325
00326 def selectCalibFile(self, files, rnum) :
00327 """Selects calibration file from a list of file names
00328 """
00329 if self.pbits & 1024 : print '\nUnsorted list of *.data files in the calib directory:'
00330 list_cf = []
00331 for path in files :
00332 fname = os.path.basename(path)
00333
00334 if fname is 'HISTORY' : continue
00335 if os.path.splitext(fname)[1] != '.data' : continue
00336
00337 cf = CalibFile(path)
00338 if cf.valid :
00339 if self.pbits & 1024 : print cf.str_attrs()
00340 list_cf.append(cf)
00341
00342
00343 list_cf_ord = sorted(list_cf)
00344
00345
00346 if self.pbits & 4 :
00347 print '\nSorted list of *.data files in the calib directory:'
00348 for cf in list_cf_ord[::-1] :
00349 if self.pbits & 4 : print cf.str_attrs()
00350
00351
00352 for cf in list_cf_ord[::-1] :
00353 if cf.get_begin() <= rnum and rnum <= cf.get_end() :
00354 if self.pbits & 8 :
00355 print 'Select calib file: %s' % cf.get_path()
00356 return cf.get_path()
00357
00358
00359 return ''
00360
00361
00362
00363 def test01() :
00364
00365
00366
00367
00368
00369 cdir = '/reg/d/psdm/CXI/cxi83714/calib/'
00370
00371 group = 'CsPad::CalibV1'
00372 src = 'CxiDs1.0:Cspad.0'
00373 type = 'pedestals'
00374 rnum = 134
00375
00376
00377
00378
00379 print 80*'_', '\nTest 1'
00380 print 'Finding calib file for\n dir = %s\n grp = %s\n src = %s\n type= %s\n run = %d' % \
00381 (cdir, group, src, type, rnum)
00382
00383 cff = CalibFileFinder(cdir, group, 0377)
00384 fname = cff.findCalibFile(src, type, rnum)
00385
00386
00387
00388 print 80*'_', '\nTest 2'
00389 print 'Test methods find_calib_file and make_calib_file_name'
00390 fname_existing = find_calib_file(cdir, src, type, rnum, pbits=1)
00391 print ' fname_existing : %s' % fname_existing
00392
00393 cdir = './calib'
00394 run_start = 134
00395 gu.create_directory(cdir, True)
00396 fname_new = make_calib_file_name(cdir, src, type, run_start, run_end=None, pbits=0)
00397 print ' fname_new : %s' % fname_new
00398
00399
00400
00401 def test_deploy_calib_array() :
00402 print 80*'_', '\nTest deploy_calib_array'
00403
00404 cdir = './calib'
00405 if not os.path.exists(cdir) : gu.create_directory(cdir, verb=True)
00406
00407
00408 src = 'CxiDs1.0:Cspad.0'
00409 type = 'pedestals'
00410 run_start = 9991
00411 run_end = None
00412 arr= gu.np.ones((32,185,388))
00413 cmts = {'exp':'cxi83714', 'ifname':'input-file-name', 'app':'my-app-name', 'comment':'my-comment'}
00414 deploy_calib_array(cdir, src, type, run_start, run_end, arr, cmts, fmt='%.1f', pbits=3)
00415
00416
00417
00418 def test_deploy_calib_file() :
00419 print 80*'_', '\nTest deploy_calib_file'
00420 cdir = './calib'
00421 if not os.path.exists(cdir) : gu.create_directory(cdir, verb=True)
00422
00423
00424 src = 'CxiDs1.0:Cspad.0'
00425 type = 'geometry'
00426 run_start = 9992
00427 run_end = None
00428 fname = '/reg/g/psdm/detector/alignment/cspad/calib-cxi-camera1-2014-09-24/2016-06-15-geometry-cxil0216-r150-camera1-z95mm.txt'
00429 cmts = {'exp':'cxi83714', 'app':'my-app-name', 'comment':'my-comment'}
00430 deploy_calib_file(cdir, src, type, run_start, run_end, fname, cmts, pbits=3)
00431
00432
00433
00434 if __name__ == "__main__" :
00435
00436 if len(sys.argv)<2 : test01()
00437 elif sys.argv[1]=='1' : test01()
00438 elif sys.argv[1]=='2' : test01()
00439 elif sys.argv[1]=='3' : test_deploy_calib_array()
00440 elif sys.argv[1]=='4' : test_deploy_calib_file()
00441 else : print 'Non-expected arguments: sys.argv=', sys.argv
00442
00443 sys.exit('End of %s' % sys.argv[0])
00444
00445