00001
00002
00003 """:py:class:`PSCalib.NDArrIO` - i/o methods to read/write numpy array in the text file
00004
00005 Usage::
00006
00007 # Import
00008 from PSCalib.NDArrIO import save_txt, load_txt, list_of_comments
00009
00010 # Save n-dimensional numpy array in the text file.
00011 save_txt(fname, arr, cmts=(), fmt='%.1f')
00012
00013 # Load 1-, 2-, n-dimensional array (if metadata available) from file .
00014 arr = load_txt(fname) # this version unpacks data directly in this script
00015 # or
00016 arr = load_txt_v2(fname) # v2 uses numpy.loadtxt(...) to load data (~30% slower then the load_txt)
00017
00018 # Get list of str objects - comment records with '#' in 1st position from file.
00019 cmts = list_of_comments(fname)
00020
00021 #------------------------------
00022 # Example of the file header:
00023 #------------------------------
00024 # TITLE File to load ndarray of calibration parameters
00025 #
00026 # EXPERIMENT amo12345
00027 # DETECTOR Camp.0:pnCCD.1
00028 # CALIB_TYPE pedestals
00029
00030 # DATE_TIME 2014-05-06 15:24:10
00031 # AUTHOR <user-login-name>
00032
00033 # line of comment always begins with #
00034 # Mandatory fields to define the ndarray<TYPE,NDIM> and its shape as unsigned shape[NDIM] = (DIM1,DIM2,DIM3)
00035 # DTYPE float
00036 # NDIM 3
00037 # DIM:1 3
00038 # DIM:2 4
00039 # DIM:3 8
00040 #------------------------------
00041
00042 @see :py:class:`Detector.AreaDetector`
00043
00044 For more detail see `AreaDetector <https://pswww.slac.stanford.edu/swdoc/releases/ana-current/pyana-ref/html/Detector/#module-Detector.AreaDetector>`_.
00045
00046 This software was developed for the SIT project.
00047 If you use all or part of it, please give an appropriate acknowledgment.
00048
00049 $Revision: 12322 $
00050
00051 @version $Id: NDArrIO.py 12322 2016-07-21 23:10:51Z dubrovin@SLAC.STANFORD.EDU $
00052
00053 @author Mikhail S. Dubrovin
00054 """
00055
00056 __version__ = "$Revision: 12322 $"
00057
00058
00059
00060
00061
00062 import numpy as np
00063 import PSCalib.GlobalUtils as gu
00064
00065
00066 def save_txt(fname='nda.txt', arr=None, cmts=(), fmt='%.1f', verbos=False, addmetad=True) :
00067 """Save n-dimensional numpy array to text file with metadata.
00068 - fname - file name for text file,
00069 - arr - numpy array,
00070 - cmts -list of comments which will be saved in the file header.
00071 """
00072
00073 recs = ['# %s' % cmt for cmt in cmts]
00074 recs.append('\n# HOST %s' % gu.get_hostname())
00075 recs.append('# WORK_DIR %s' % gu.get_cwd())
00076 recs.append('# FILE_NAME %s' % fname)
00077 recs.append('# DATE_TIME %s' % gu.str_tstamp(fmt='%Y-%m-%dT%H:%M:%S'))
00078 recs.append('# UID %s' % gu.get_login())
00079 recs.append('# SHAPE %s' % str(arr.shape).replace(' ',''))
00080 recs.append('# DATATYPE %s' % str(arr.dtype))
00081
00082 if addmetad :
00083 recs.append('\n# DTYPE %s' % str(arr.dtype))
00084 recs.append('# NDIM %s' % len(arr.shape))
00085
00086 for i in range(len(arr.shape)) :
00087 recs.append('# DIM:%d %s' % (i, arr.shape[i]))
00088
00089 arr2d = gu.reshape_nda_to_2d(arr)
00090
00091
00092 recs.append('' if len(arr.shape)>1 else '\n')
00093 nline = '\n' if len(arr.shape)>1 else ' '
00094
00095 hdr = '\n'.join(recs)
00096
00097
00098 np.savetxt(fname, arr, fmt, delimiter=' ', newline=nline, header=hdr, comments='')
00099 if verbos : print 'File %s is saved' % fname
00100
00101
00102
00103 def _unpack_data(recs) :
00104 """Reconstruct data records from file to 2-d (or 1-d) list of values.
00105 """
00106 if len(recs) == 0 : return None
00107
00108 if len(recs) == 1 :
00109 for rec in recs :
00110 fields = rec.strip('\n').split()
00111 return [float(v) for v in fields]
00112
00113 arr = []
00114 for rec in recs :
00115 fields = rec.strip('\n').split()
00116 vals = [float(v) for v in fields]
00117 arr.append(vals)
00118
00119 return arr
00120
00121
00122
00123 def _metadata_from_comments(cmts) :
00124 """Returns metadata from the list of comments
00125 """
00126 str_dtype = ''
00127 dtype = None
00128 ndim = None
00129 shape = []
00130
00131 if cmts is not None :
00132 for rec in cmts :
00133 fields = rec.split(' ', 2)
00134 if len(fields)<3 : continue
00135 if fields[1] == 'DTYPE' : str_dtype = fields[2].rstrip('\n').strip(' ')
00136 elif fields[1] == 'NDIM' : ndim = int(fields[2])
00137 elif fields[1][:4] == 'DIM:' : shape.append(int(fields[2]))
00138
00139 dtype = np.dtype(str_dtype) if str_dtype else np.float32
00140
00141 return ndim, shape, dtype
00142
00143
00144
00145 def list_of_comments(fname) :
00146 """Returns list of str objects - comment records from file.
00147 - fname - file name for text file.
00148 """
00149
00150
00151 f=open(fname,'r')
00152
00153 cmts = []
00154 for rec in f :
00155 if rec.isspace() : continue
00156 elif rec[0] == '#' : cmts.append(rec.rstrip('\n'))
00157 else : break
00158
00159 f.close()
00160
00161 if len(cmts)==0 :
00162 return None
00163
00164 return cmts
00165
00166
00167
00168 def load_txt_v2(fname) :
00169 """Reads n-dimensional numpy array from text file with metadata.
00170 - fname - file name for text file.
00171 """
00172 cmts = list_of_comments(fname)
00173 ndim, shape, dtype = _metadata_from_comments(cmts)
00174
00175 nparr = np.loadtxt(fname, dtype=dtype, comments='#')
00176
00177 if dtype is None or ndim is None or shape==[] :
00178
00179 return nparr
00180
00181 if ndim != len(shape) :
00182 str_metad = 'dtype=%s ndim=%d shape=%s' % (str(dtype), ndim, str(shape))
00183 raise IOError('Inconsistent metadata (ndim != len(shape)) in file: %s' % str_metad)
00184
00185 if ndim > 2: nparr.shape = shape
00186
00187 return nparr
00188
00189
00190
00191 def load_txt(fname) :
00192 """Reads n-dimensional numpy array from text file with metadata.
00193 - fname - file name for text file.
00194 """
00195
00196
00197
00198 f=open(fname,'r')
00199 recs = f.readlines()
00200 f.close()
00201
00202
00203 cmts = []
00204 data = []
00205 for rec in recs :
00206 if rec.isspace() : continue
00207 if rec[0] == '#' : cmts.append(rec)
00208 else : data.append(rec)
00209
00210 if data == [] :
00211 raise IOError('Data is missing in the file %s' % fname)
00212
00213
00214 ndim, shape, dtype = _metadata_from_comments(cmts)
00215
00216
00217 nparr = np.array(_unpack_data(data), dtype)
00218
00219 if ndim is None or shape==[] or dtype is None :
00220
00221 return nparr
00222
00223 if ndim != len(shape) :
00224 str_metad = 'dtype=%s ndim=%d shape=%s' % (str(dtype), ndim, str(shape))
00225 raise IOError('Inconsistent metadata (NDIM != len(shape)) in file: %s' % str_metad)
00226
00227 if ndim > 2: nparr.shape = shape
00228
00229 return nparr
00230
00231
00232
00233
00234
00235 def test_save_txt() :
00236
00237 arr3 = (((111,112,113),
00238 (121,122,123)),
00239 ((211,212,213),
00240 (221,222,223)))
00241
00242 arr2 = ((11,12,13),
00243 (21,22,23))
00244
00245 arr1 = (1,2,3,4,5)
00246
00247 npa = np.array(np.array(arr3), dtype=np.int)
00248 save_txt('nda.txt', npa, cmts=('Test of PSCalib.NDArrIO.save_txt(...)', 'save numpy array in the text file'), verbos=True)
00249
00250
00251
00252 def test_load_txt() :
00253
00254 from time import time
00255
00256 fname = 'nda.txt'
00257 fname = '/reg/g/psdm/detector/alignment/andor3d/calib-andor3d-2016-02-09/calib/Andor3d::CalibV1/SxrEndstation.0:DualAndor.0/pedestals/7-end.data'
00258
00259
00260 print 'Test list_of_comments:'
00261 t0_sec = time()
00262 cmts = list_of_comments(fname)
00263 print 'Consumed time for list_of_comments = %10.6f sec' % (time()-t0_sec)
00264 if cmts is not None :
00265 for cmt in cmts: print cmt
00266
00267 t0_sec = time()
00268 nparr = load_txt(fname)
00269
00270
00271 print 'Consumed time for load_txt = %10.6f sec' % (time()-t0_sec)
00272
00273 print 'Test np.array from file:\n', nparr
00274 print 'nparr.shape:', nparr.shape
00275 print 'nparr.dtype:', nparr.dtype
00276
00277
00278
00279 if __name__ == "__main__" :
00280 test_save_txt()
00281 test_load_txt()
00282
00283