1
2
3 """
4
5 License: GPLv2+
6
7 Authors: Henning O. Sorensen & Erik Knudsen
8 Center for Fundamental Research: Metal Structures in Four Dimensions
9 Risoe National Laboratory
10 Frederiksborgvej 399
11 DK-4000 Roskilde
12 email:erik.knudsen@risoe.dk
13
14 + Jon Wright, ESRF
15
16 2011-02-11: Mostly rewritten by Jérôme Kieffer (Jerome.Kieffer@esrf.eu)
17 European Synchrotron Radiation Facility
18 Grenoble (France)
19
20 """
21
22 import numpy as np, logging
23 from fabioimage import fabioimage
24 import gzip, bz2, zlib, os, StringIO
25
26
27 BLOCKSIZE = 512
28 DATA_TYPES = { "SignedByte" : np.int8,
29 "Signed8" : np.int8,
30 "UnsignedByte" : np.uint8,
31 "Unsigned8" : np.uint8,
32 "SignedShort" : np.int16,
33 "Signed16" : np.int16,
34 "UnsignedShort" : np.uint16,
35 "Unsigned16" : np.uint16,
36 "UnsignedShortInteger" : np.uint16,
37 "SignedInteger" : np.int32,
38 "Signed32" : np.int32,
39 "UnsignedInteger": np.uint32,
40 "Unsigned32" : np.uint32,
41 "SignedLong" : np.int32,
42 "UnsignedLong" : np.uint32,
43 "Signed64" : np.int64,
44 "Unsigned64" : np.uint64,
45 "FloatValue" : np.float32,
46 "FLOATVALUE" : np.float32,
47 "FLOAT" : np.float32,
48 "Float" : np.float32,
49 "FloatIEEE32" : np.float32,
50 "Float32" : np.float32,
51 "Double" : np.float64,
52 "DoubleValue" : np.float64,
53 "FloatIEEE64" : np.float64,
54 "DoubleIEEE64" : np.float64
55 }
56
57 NUMPY_EDF_DTYPE = {"int8" :"SignedByte",
58 "int16" :"SignedShort",
59 "int32" :"SignedInteger",
60 "int64" :"Signed64",
61 "uint8" :"UnsignedByte",
62 "uint16" :"UnsignedShort",
63 "uint32" :"UnsignedInteger",
64 "uint64" :"Unsigned64",
65 "float32" :"FloatValue",
66 "float64" :"DoubleValue"
67 }
68
69 MINIMUM_KEYS = ['HEADERID',
70 'IMAGE',
71 'BYTEORDER',
72 'DATATYPE',
73 'DIM_1',
74 'DIM_2',
75 'SIZE']
78 """
79 A class representing a single frame in an EDF file
80 """
81 - def __init__(self, data=None, header=None, header_keys=None, number=None):
115
117 """
118 Parse the header in some EDF format from an already open file
119
120 @param block: string representing the header block
121 @type block: string, should be full ascii
122 @return: size of the binary blob
123 """
124
125 self.header = {}
126 self.capsHeader = {}
127 self.header_keys = []
128 self.size = None
129 calcsize = 1
130 self.dims = []
131
132 for line in block.split(';'):
133 if '=' in line:
134 key, val = line.split('=' , 1)
135 key = key.strip()
136 self.header[key] = val.strip()
137 self.capsHeader[key.upper()] = key
138 self.header_keys.append(key)
139
140
141 if "SIZE" in self.capsHeader:
142 try:
143 self.size = int(self.header[self.capsHeader["SIZE"]])
144 except ValueError:
145 logging.warning("Unable to convert to integer : %s %s " % (self.capsHeader["SIZE"], self.header[self.capsHeader["SIZE"]]))
146 if "DIM_1" in self.capsHeader:
147 try:
148 dim1 = int(self.header[self.capsHeader['DIM_1']])
149 except ValueError:
150 logging.error("Unable to convert to integer Dim_1: %s %s" % (self.capsHeader["DIM_1"], self.header[self.capsHeader["DIM_1"]]))
151 else:
152 calcsize *= dim1
153 self.dims.append(dim1)
154 else:
155 logging.error("No Dim_1 in headers !!!")
156 if "DIM_2" in self.capsHeader:
157 try:
158 dim2 = int(self.header[self.capsHeader['DIM_2']])
159 except ValueError:
160 logging.error("Unable to convert to integer Dim_3: %s %s" % (self.capsHeader["DIM_2"], self.header[self.capsHeader["DIM_2"]]))
161 else:
162 calcsize *= dim2
163 self.dims.append(dim2)
164 else:
165 logging.error("No Dim_2 in headers !!!")
166 iDim = 3
167 while iDim is not None:
168 strDim = "DIM_%i" % iDim
169 if strDim in self.capsHeader:
170 try:
171 dim3 = int(self.header[self.capsHeader[strDim]])
172 except ValueError:
173 logging.error("Unable to convert to integer %s: %s %s"
174 % (strDim, self.capsHeader[strDim], self.header[self.capsHeader[strDim]]))
175 dim3 = None
176 iDim = None
177 else:
178 calcsize *= dim3
179 self.dims.append(dim3)
180 iDim += 1
181 else:
182 logging.debug("No Dim_3 -> it is a 2D image")
183 iDim = None
184 if self.bytecode is None:
185 if "DATATYPE" in self.capsHeader:
186 self.bytecode = DATA_TYPES[self.header[self.capsHeader['DATATYPE']]]
187 else:
188 self.bytecode = np.uint16
189 logging.warning("Defaulting type to uint16")
190 self.bpp = len(np.array(0, self.bytecode).tostring())
191 calcsize *= self.bpp
192 if (self.size is None):
193 self.size = calcsize
194 elif (self.size != calcsize):
195 if ("COMPRESSION" in self.capsHeader) and (self.header[self.capsHeader['COMPRESSION']].upper().startswith("NO")):
196 logging.info("Mismatch between the expected size %s and the calculated one %s" % (self.size, calcsize))
197 self.size = calcsize
198
199 for i, n in enumerate(self.dims):
200 exec "self.dim%i=%i" % (i + 1, n)
201
202 return self.size
203
204
206 """
207 Decide if we need to byteswap
208 """
209 if ('Low' in self.header[self.capsHeader['BYTEORDER']] and np.little_endian) or \
210 ('High' in self.header[self.capsHeader['BYTEORDER']] and not np.little_endian):
211 return False
212 if ('High' in self.header[self.capsHeader['BYTEORDER']] and np.little_endian) or \
213 ('Low' in self.header[self.capsHeader['BYTEORDER']] and not np.little_endian):
214 if self.bpp in [2, 4, 8]:
215 return True
216 else:
217 return False
218
219
221 """
222 Unpack a binary blob according to the specification given in the header
223
224 @return: dataset as numpy.ndarray
225 """
226 if self.data is not None:
227 return self.data
228 if self.rawData is None:
229 return self.data
230
231 if self.bytecode is None:
232 if "DATATYPE" in self.capsHeader:
233 self.bytecode = DATA_TYPES[self.header[self.capsHeader["DATATYPE"]]]
234 else:
235 self.bytecode = np.uint16
236 dims = self.dims[:]
237 dims.reverse()
238
239 if ("COMPRESSION" in self.capsHeader):
240 compression = self.header[self.capsHeader["COMPRESSION"]].upper()
241 uncompressed_size = self.bpp
242 for i in dims:
243 uncompressed_size *= i
244 if "OFFSET" in compression :
245 try:
246 import byte_offset
247 except ImportError:
248 logging.error("Unimplemented compression scheme: %s" % compression)
249 else:
250 myData = byte_offset.analyseCython(self.rawData, size=uncompressed_size)
251 rawData = myData.astype(self.bytecode).tostring()
252 self.size = uncompressed_size
253 elif compression == "NONE":
254 rawData = self.rawData
255 elif "GZIP" in compression:
256 fileobj = StringIO.StringIO(self.rawData)
257 try:
258 rawData = gzip.GzipFile(fileobj=fileobj).read()
259 except IOError:
260 logging.warning("Encounter the python-gzip bug with trailing garbage")
261
262 import subprocess
263 sub = subprocess.Popen(["gzip", "-d", "-f"], stdout=subprocess.PIPE, stdin=subprocess.PIPE, stderr=subprocess.PIPE)
264 rawData, err = sub.communicate(input=self.rawData)
265 logging.debug("Gzip subprocess ended with %s err= %s; I got %s bytes back" % (sub.wait(), err, len(rawData)))
266 self.size = uncompressed_size
267 elif "BZ" in compression :
268 rawData = bz2.decompress(self.rawData)
269 self.size = uncompressed_size
270 elif "Z" in compression :
271 rawData = zlib.decompress(self.rawData)
272 self.size = uncompressed_size
273 else:
274 logging.warning("Unknown compression scheme %s" % compression)
275 rawData = self.rawData
276
277 else:
278 rawData = self.rawData
279
280 expected = self.size
281 obtained = len(rawData)
282 if expected > obtained:
283 logging.error("Data stream is incomplete: %s < expected %s bytes" % (obtained, expected))
284 rawData += "\x00" * (expected - obtained)
285 elif expected < len(rawData):
286 logging.info("Data stream contains trailing junk : %s > expected %s bytes" % (obtained, expected))
287 rawData = rawData[:expected]
288
289 if self.swap_needed():
290 data = np.fromstring(rawData, self.bytecode).byteswap().reshape(tuple(dims))
291 else:
292 data = np.fromstring(rawData, self.bytecode).reshape(tuple(dims))
293 self.data = data
294 self.rawData = None
295 self.bytecode = data.dtype.type
296 return data
297
298
300 """
301 @param force_type: type of the dataset to be enforced like "float64" or "uint16"
302 @type force_type: string or numpy.dtype
303 @return: ascii header block
304 @rtype: python string with the concatenation of the ascii header and the binary data block
305 """
306 if force_type is not None:
307 data = self.getData().astype(force_type)
308 else:
309 data = self.getData()
310
311 for key in self.header:
312 KEY = key.upper()
313 if KEY not in self.capsHeader:
314 self.capsHeader[KEY] = key
315 if key not in self.header_keys:
316 self.header_keys.append(key)
317
318 header = self.header.copy()
319 header_keys = self.header_keys[:]
320 capsHeader = self.capsHeader.copy()
321
322 listHeader = ["{\n"]
323
324 for i in capsHeader:
325 if "DIM_" in i:
326 header.pop(capsHeader[i])
327 header_keys.remove(capsHeader[i])
328 for KEY in ["SIZE", "EDF_BINARYSIZE", "EDF_HEADERSIZE", "BYTEORDER", "DATATYPE", "HEADERID", "IMAGE"]:
329 if KEY in capsHeader:
330 header.pop(capsHeader[KEY])
331 header_keys.remove(capsHeader[KEY])
332 if "EDF_DATABLOCKID" in capsHeader:
333 header_keys.remove(capsHeader["EDF_DATABLOCKID"])
334
335 if capsHeader["EDF_DATABLOCKID"] != "EDF_DataBlockID":
336 header["EDF_DataBlockID"] = header.pop(capsHeader["EDF_DATABLOCKID"])
337 capsHeader["EDF_DATABLOCKID"] = "EDF_DataBlockID"
338
339
340 header_keys.insert(0, "Size")
341 header["Size"] = len(data.tostring())
342 header_keys.insert(0, "HeaderID")
343 header["HeaderID"] = "EH:%06d:000000:000000" % self.iFrame
344 header_keys.insert(0, "Image")
345 header["Image"] = str(self.iFrame)
346
347 dims = list(data.shape)
348 nbdim = len(dims)
349 for i in dims:
350 key = "Dim_%i" % nbdim
351 header[key] = i
352 header_keys.insert(0, key)
353 nbdim -= 1
354 header_keys.insert(0, "DataType")
355 header["DataType"] = NUMPY_EDF_DTYPE[str(np.dtype(data.dtype))]
356 header_keys.insert(0, "ByteOrder")
357 if np.little_endian:
358 header["ByteOrder"] = "LowByteFirst"
359 else:
360 header["ByteOrder"] = "HighByteFirst"
361 approxHeaderSize = 100
362 for key in header:
363 approxHeaderSize += 7 + len(key) + len(str(header[key]))
364 approxHeaderSize = BLOCKSIZE * (approxHeaderSize // BLOCKSIZE + 1)
365 header_keys.insert(0, "EDF_HeaderSize")
366 header["EDF_HeaderSize"] = str(BLOCKSIZE * (approxHeaderSize // BLOCKSIZE + 1))
367 header_keys.insert(0, "EDF_BinarySize")
368 header["EDF_BinarySize"] = len(data.tostring())
369 header_keys.insert(0, "EDF_DataBlockID")
370 if not "EDF_DataBlockID" in header:
371 header["EDF_DataBlockID"] = "%i.Image.Psd" % self.iFrame
372 preciseSize = 4
373 for key in header_keys:
374 line = str("%s = %s ;\n" % (key, header[key]))
375 preciseSize += len(line)
376 listHeader.append(line)
377
378 if preciseSize > approxHeaderSize:
379 logging.error("I expected the header block only at %s in fact it is %s" % (approxHeaderSize, preciseSize))
380 for idx, line in enumerate(listHeader[:]):
381 if line.startswith("EDF_HeaderSize"):
382 headerSize = BLOCKSIZE * (preciseSize // BLOCKSIZE + 1)
383 newline = "EDF_HeaderSize = %s ;\n" % headerSize
384 delta = len(newline) - len(line)
385 if (preciseSize // BLOCKSIZE) != ((preciseSize + delta) // BLOCKSIZE):
386 headerSize = BLOCKSIZE * ((preciseSize + delta) // BLOCKSIZE + 1)
387 newline = "EDF_HeaderSize = %s ;\n" % headerSize
388 preciseSize = preciseSize + delta
389 listHeader[idx] = newline
390 break
391 else:
392 headerSize = approxHeaderSize
393 listHeader.append(" "*(headerSize - preciseSize) + "}\n")
394 return "".join(listHeader) + data.tostring()
395
399 """ Read and try to write the ESRF edf data format """
400
401 - def __init__(self, data=None , header=None, header_keys=None):
408
409
410 @staticmethod
412 """
413 Read in a header in some EDF format from an already open file
414
415 @param infile: file object open in read mode
416 @return: string (or None if no header was found.
417 """
418
419 block = infile.read(BLOCKSIZE)
420 if len(block) < BLOCKSIZE:
421 logging.debug("Under-short header: only %i bytes in %s" % (len(block), infile.name))
422 return
423 if (block.find("{") < 0) :
424
425 logging.warning("no opening {. Corrupt header of EDF file %s" % infile.name)
426 return
427 while '}' not in block:
428 block = block + infile.read(BLOCKSIZE)
429 if len(block) > BLOCKSIZE * 20:
430 logging.warning("Runaway header in EDF file")
431 return
432 start = block.find("{") + 1
433 end = block.find("}")
434
435
436 if block[end: end + 3] == "}\r\n":
437 offset = end + 3 - len(block)
438 elif block[end: end + 2] == "}\n":
439 offset = end + 2 - len(block)
440 else:
441 logging.error("Unable to locate start of the binary section")
442 offset = None
443 if offset is not None:
444 infile.seek(offset, os.SEEK_CUR)
445 return block[start:end]
446
447
449 """
450 Read all headers in a file and populate self.header
451 data is not yet populated
452 @type infile: file object open in read mode
453 """
454 self.nframes = 0
455 self.frames = []
456
457 bContinue = True
458 while bContinue:
459 block = self._readHeaderBlock(infile)
460 if block is None:
461 bContinue = False
462 break
463 frame = Frame(number=self.nframes)
464 self.frames.append(frame)
465 size = frame.parseheader(block)
466 frame.rawData = infile.read(size)
467 if len(frame.rawData) != size:
468 logging.warning("Non complete datablock: got %s, expected %s" % (len(frame.rawData), size))
469 bContinue = False
470 break
471 self.nframes += 1
472
473 for i, frame in enumerate(self.frames):
474 missing = []
475 for item in MINIMUM_KEYS:
476 if item not in frame.capsHeader:
477 missing.append(item)
478 if len(missing) > 0:
479 logging.info("EDF file %s frame %i misses mandatory keys: %s " % (self.filename, i, " ".join(missing)))
480
481 self.currentframe = 0
482
483
484 - def read(self, fname):
485 """
486 Read in header into self.header and
487 the data into self.data
488 """
489 self.resetvals()
490 self.filename = fname
491
492 infile = self._open(fname, "rb")
493 self._readheader(infile)
494 if self.data is None:
495 self.data = self.unpack()
496
497 self.resetvals()
498
499 self.pilimage = None
500 return self
501
503 """
504 Decide if we need to byteswap
505 """
506 if ('Low' in self.header[self.capsHeader['BYTEORDER']] and np.little_endian) or \
507 ('High' in self.header[self.capsHeader['BYTEORDER']] and not np.little_endian):
508 return False
509 if ('High' in self.header[self.capsHeader['BYTEORDER']] and np.little_endian) or \
510 ('Low' in self.header[self.capsHeader['BYTEORDER']] and not np.little_endian):
511 if self.bpp in [2, 4, 8]:
512 return True
513 else:
514 return False
515
516
518 """
519 Unpack a binary blob according to the specification given in the header and return the dataset
520
521 @return: dataset as numpy.ndarray
522 """
523 return self.frames[self.currentframe].getData()
524
525
527 """ returns the file numbered 'num' in the series as a fabioimage """
528 if num in xrange(self.nframes):
529 frame = self.frames[num]
530 newImage = edfimage(data=frame.getData(), header=frame.header, header_keys=frame.header_keys)
531 newImage.frames = self.frames
532 newImage.nframes = self.nframes
533 newImage.currentframe = num
534 newImage.filename = self.filename
535 return newImage
536 else:
537 logging.error("Cannot access frame: %s" % num)
538 raise ValueError("edfimage.getframe: index out of range: %s" % num)
539
540
542 """ returns the previous file in the series as a fabioimage """
543 if self.nframes == 1:
544 return fabioimage.previous(self)
545 else:
546 newFrameId = self.currentframe - 1
547 return self.getframe(newFrameId)
548
549
551 """ returns the next file in the series as a fabioimage """
552 if self.nframes == 1:
553 return fabioimage.previous(self)
554 else:
555 newFrameId = self.currentframe + 1
556 return self.getframe(newFrameId)
557
558
559
560 - def write(self, fname, force_type=None):
561 """
562 Try to write a file
563 check we can write zipped also
564 mimics that fabian was writing uint16 (we sometimes want floats)
565
566 @param force_type: can be numpy.uint16 or simply "float"
567 @return: None
568
569 """
570 outfile = self._open(fname, mode="wb")
571 for i, frame in enumerate(self.frames):
572 frame.iFrame = i
573 outfile.write(frame.getEdfBlock())
574 outfile.close()
575
576
577
578
579
581 """
582 Getter for the headers. used by the property header,
583 """
584 return self.frames[self.currentframe].header
586 """
587 Enforces the propagation of the header to the list of frames
588 """
589 try:
590 self.frames[self.currentframe].header = _dictHeader
591 except AttributeError:
592 self.frames = [Frame(header=_dictHeader)]
593 except IndexError:
594 if self.currentframe < len(self.frames):
595 self.frames.append(Frame(header=_dictHeader))
597 """
598 Deleter for edf header
599 """
600 self.frames[self.currentframe].header = {}
601 header = property(getHeader, setHeader, delHeader, "property: header of EDF file")
602
604 """
605 Getter for edf header_keys
606 """
607 return self.frames[self.currentframe].header_keys
609 """
610 Enforces the propagation of the header_keys to the list of frames
611 @param _listtHeader: list of the (ordered) keys in the header
612 @type _listtHeader: python list
613 """
614 try:
615 self.frames[self.currentframe].header_keys = _listtHeader
616 except AttributeError:
617 self.frames = [Frame(header_keys=_listtHeader)]
618 except IndexError:
619 if self.currentframe < len(self.frames):
620 self.frames.append(Frame(header_keys=_listtHeader))
622 """
623 Deleter for edf header_keys
624 """
625 self.frames[self.currentframe].header_keys = []
626 header_keys = property(getHeaderKeys, setHeaderKeys, delHeaderKeys, "property: header_keys of EDF file")
627
629 """
630 getter for edf Data
631 @return: data for current frame
632 @rtype: numpy.ndarray
633 """
634 data = None
635 try:
636 data = self.frames[self.currentframe].data
637 except AttributeError:
638 self.frames = [Frame()]
639 data = self.frames[self.currentframe].data
640 except IndexError:
641 if self.currentframe < len(self.frames):
642 self.frames.append(Frame())
643 data = self.frames[self.currentframe].data
644 return data
645
647 """
648 Enforces the propagation of the header_keys to the list of frames
649 @param _data: numpy array representing data
650 """
651 try:
652 self.frames[self.currentframe].data = _data
653 except AttributeError:
654 self.frames = [Frame(data=_data)]
655 except IndexError:
656 if self.currentframe < len(self.frames):
657 self.frames.append(Frame(data=_data))
659 """
660 deleter for edf Data
661 """
662 self.frames[self.currentframe].data = None
663 data = property(getData, setData, delData, "property: data of EDF file")
664
666 """
667 getter for edf headers keys in upper case
668 @return: data for current frame
669 @rtype: dict
670 """
671 return self.frames[self.currentframe].capsHeader
673 """
674 Enforces the propagation of the header_keys to the list of frames
675 @param _data: numpy array representing data
676 """
677 self.frames[self.currentframe].capsHeader = _data
679 """
680 deleter for edf capsHeader
681 """
682 self.frames[self.currentframe].capsHeader = {}
683 capsHeader = property(getCapsHeader, setCapsHeader, delCapsHeader, "property: capsHeader of EDF file, i.e. the keys of the header in UPPER case.")
684
686 return self.frames[self.currentframe].dim1
688 try:
689 self.frames[self.currentframe].dim1 = _iVal
690 except AttributeError:
691 self.frames = [Frame()]
692 except IndexError:
693 if self.currentframe < len(self.frames):
694 self.frames.append(Frame())
695 self.frames[self.currentframe].dim1 = _iVal
696 dim1 = property(getDim1, setDim1)
698 return self.frames[self.currentframe].dim2
700 try:
701 self.frames[self.currentframe].dim2 = _iVal
702 except AttributeError:
703 self.frames = [Frame()]
704 except IndexError:
705 if self.currentframe < len(self.frames):
706 self.frames.append(Frame())
707 self.frames[self.currentframe].dim2 = _iVal
708 dim2 = property(getDim2, setDim2)
709
711 return self.frames[self.currentframe].dims
712 dims = property(getDims)
714 return self.frames[self.currentframe].bytecode
716 try:
717 self.frames[self.currentframe].bytecode = _iVal
718 except AttributeError:
719 self.frames = [Frame()]
720 except IndexError:
721 if self.currentframe < len(self.frames):
722 self.frames.append(Frame())
723 self.frames[self.currentframe].bytecode = _iVal
724
725 bytecode = property(getByteCode, setByteCode)
727 return self.frames[self.currentframe].bpp
729 try:
730 self.frames[self.currentframe].bpp = _iVal
731 except AttributeError:
732 self.frames = [Frame()]
733 except IndexError:
734 if self.currentframe < len(self.frames):
735 self.frames.append(Frame())
736 self.frames[self.currentframe].bpp = _iVal
737 bpp = property(getBpp, setBpp)
738