From 848f1649d88ad4287bd659459183817002963bf1 Mon Sep 17 00:00:00 2001 From: "W. Trevor King" Date: Sun, 16 May 2010 12:00:56 -0400 Subject: [PATCH] Add Curve loading to hooke.playlist.Playlist. Highlights: * NotRecognized.__get/setstate__ for Pickling through the Queues. Perhaps this would be better solved in Failure.__get/setstate__? The current fix works for the moment though. * Use Curve.data == None to indicate unloaded curves. * Fix TutorialDriver.is_me(self) -> is_me(self, path) * Added ._load* to Playlist, so it only caches ._max_loaded curves in memory. * Fix broken Playlist.from_string doctest. --- hooke/curve.py | 25 ++++++++++++++++++++----- hooke/driver/__init__.py | 2 +- hooke/driver/tutorial.py | 4 ++-- hooke/playlist.py | 38 +++++++++++++++++++++++++++++--------- 4 files changed, 52 insertions(+), 17 deletions(-) diff --git a/hooke/curve.py b/hooke/curve.py index 816a921..216722d 100644 --- a/hooke/curve.py +++ b/hooke/curve.py @@ -26,9 +26,16 @@ import numpy class NotRecognized (ValueError): def __init__(self, curve): - msg = 'Not a recognizable curve format: %s' % curve.path - ValueError.__init__(self, msg) - self.curve = curve + self.__setstate__(curve) + + def __getstate__(self): + return self.curve + + def __setstate__(self, data): + if isinstance(data, Curve): + msg = 'Not a recognizable curve format: %s' % data.path + super(NotRecognized, self).__init__(msg) + self.curve = data class Data (numpy.ndarray): """Stores a single, continuous data set. @@ -87,7 +94,7 @@ class Curve (object): #the data dictionary contains: {name of data: list of data sets [{[x], [y]}] self.path = path self.driver = None - self.data = [] + self.data = None if info == None: info = {} self.info = info @@ -106,4 +113,12 @@ class Curve (object): def load(self): """Use the driver to read the curve into memory. """ - pass + data,info = self.driver.read(self.path) + self.data = data + for key,value in info.items(): + self.info[key] = value + + def unload(self): + """Release memory intensive :attr:`.data`. + """ + self.data = None diff --git a/hooke/driver/__init__.py b/hooke/driver/__init__.py index 92ae972..b6324cb 100644 --- a/hooke/driver/__init__.py +++ b/hooke/driver/__init__.py @@ -81,7 +81,7 @@ class Driver(object): def read(self, path): """Read data from `path` and return a - (:class:`hooke.curve.Data`, `info`) tuple. + ([:class:`hooke.curve.Data`, ...], `info`) tuple. The `info` :class:`dict` must contain values for the keys: 'filetype' and 'experiment'. See :class:`hooke.curve.Curve` diff --git a/hooke/driver/tutorial.py b/hooke/driver/tutorial.py index d68393e..4553369 100644 --- a/hooke/driver/tutorial.py +++ b/hooke/driver/tutorial.py @@ -93,7 +93,7 @@ class TutorialDriver (Driver): help='Set the units used for the x data.'), ] - def is_me(self): + def is_me(self, path): """YOU MUST OVERRIDE Driver.is_me. RETURNS: Boolean (`True` or `False`) @@ -106,7 +106,7 @@ class TutorialDriver (Driver): automatically. """ - f = open(self.filename, 'r') + f = open(path, 'r') header = f.readline() # we only need the first line f.close() diff --git a/hooke/playlist.py b/hooke/playlist.py index 4cc897a..ca38b4e 100644 --- a/hooke/playlist.py +++ b/hooke/playlist.py @@ -81,10 +81,12 @@ class Playlist (NoteIndexList): def __init__(self, drivers, name=None): super(Playlist, self).__init__(name=name) self.drivers = drivers + self._loaded = [] # List of loaded curves, see :meth:`._load`. + self._max_loaded = 100 # curves to hold in memory simultaneously. def append_curve_by_path(self, path, info=None, identify=True): if self.path != None: - path = os.path.join(self.path, path) + path = os.path.join(os.path.dirname(self.path), path) path = os.path.normpath(path) c = curve.Curve(path, info=info) if identify == True: @@ -92,6 +94,24 @@ class Playlist (NoteIndexList): self.append(c) return c + def current(self): + curve = super(Playlist, self).current() + self._load(curve) + return curve + + def _load(self, curve): + if curve != None and curve not in self._loaded: + if curve not in self: + self.append(curve) + if curve.driver == None: + c.identify(self.drivers) + if curve.data == None: + curve.load() + self._loaded.append(curve) + if len(self._loaded) > self._max_loaded: + oldest = self._loaded.pop(0) + oldest.unload() + class FilePlaylist (Playlist): version = '0.1' @@ -199,7 +219,7 @@ class FilePlaylist (Playlist): root.unlink() # break circular references for garbage collection return string - def _from_xml_doc(self, doc): + def _from_xml_doc(self, doc, identify=True): """Load a playlist from an :class:`xml.dom.minidom.Document` instance. """ @@ -217,10 +237,10 @@ class FilePlaylist (Playlist): path = curve_element.getAttribute('path') info = dict(curve_element.attributes.items()) info.pop('path') - self.append_curve_by_path(path, info, identify=False) + self.append_curve_by_path(path, info, identify=identify) self.jump(self._index) # ensure valid index - def from_string(self, string): + def from_string(self, string, identify=True): """Load a playlist from a string. Examples @@ -233,8 +253,8 @@ class FilePlaylist (Playlist): ... ... ''' >>> p = FilePlaylist(drivers=[], - ... path=os.path.join('path', 'to','playlist')) - >>> p.from_string(string) + ... path=os.path.join('path', 'to', 'my', 'playlist')) + >>> p.from_string(string, identify=False) >>> p._index 1 >>> p.info @@ -245,14 +265,14 @@ class FilePlaylist (Playlist): path/to/curve/two """ doc = xml.dom.minidom.parseString(string) - self._from_xml_doc(doc) + self._from_xml_doc(doc, identify=identify) - def load(self, path=None): + def load(self, path=None, identify=True): """Load a playlist from a file. """ self.set_path(path) doc = xml.dom.minidom.parse(self.path) - self._from_xml_doc(doc) + self._from_xml_doc(doc, identify=identify) self._digest = self.digest() def save(self, path=None): -- 2.26.2