Source code for hyperstream.channel_manager

# The MIT License (MIT) # Copyright (c) 2014-2017 University of Bristol
#
#  Permission is hereby granted, free of charge, to any person obtaining a copy
#  of this software and associated documentation files (the "Software"), to deal
#  in the Software without restriction, including without limitation the rights
#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
#  copies of the Software, and to permit persons to whom the Software is
#  furnished to do so, subject to the following conditions:
#
#  The above copyright notice and this permission notice shall be included in all
#  copies or substantial portions of the Software.
#
#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
#  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
#  MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
#  IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
#  DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
#  OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE
#  OR OTHER DEALINGS IN THE SOFTWARE.
"""
Channel manager module. Defines the ChannelManager - a container for channels, that can load in plugins
"""


import inspect
import logging
from mongoengine import DoesNotExist, MultipleObjectsReturned
from mongoengine.context_managers import switch_db

from models import StreamDefinitionModel, StreamStatusModel
from stream import StreamId, DatabaseStream
from time_interval import TimeIntervals
from utils import Printable, utcnow, MIN_DATE, StreamAlreadyExistsError, ChannelNotFoundError, ToolNotFoundError, \
    ChannelAlreadyExistsError
from channels import ToolChannel, MemoryChannel, DatabaseChannel


[docs]class ChannelManager(dict, Printable): """ Container for channels. """ def __init__(self, plugins, **kwargs): super(ChannelManager, self).__init__(**kwargs) # See this answer http://stackoverflow.com/a/14620633 for why we do the following: self.__dict__ = self self.tools = ToolChannel("tools", "hyperstream/tools", up_to_timestamp=utcnow()) self.memory = MemoryChannel("memory") self.mongo = DatabaseChannel("mongo") for plugin in plugins: for channel in plugin.load(): if channel.channel_id in self: raise ChannelAlreadyExistsError(channel.channel_id) self[channel.channel_id] = channel self.update_channels() @property def tool_channels(self): """ The tool channels as a list """ return [c for c in self.values() if isinstance(c, ToolChannel)] @property def memory_channels(self): """ The memory channels as a list """ return [c for c in self.values() if isinstance(c, MemoryChannel)] @property def database_channels(self): """ The database channels as a list """ return [c for c in self.values() if isinstance(c, DatabaseChannel)]
[docs] def get_channel(self, channel_id): """ Get the channel by id :param channel_id: The channel id :return: The channel object """ try: return self[channel_id] except KeyError: raise ChannelNotFoundError("Channel {} not found".format(channel_id))
[docs] def update_channels(self): """ Pulls out all of the stream definitions from the database, and populates the channels with stream references """ with switch_db(StreamDefinitionModel, 'hyperstream'): for s in StreamDefinitionModel.objects(): stream_id = StreamId(name=s.stream_id.name, meta_data=s.stream_id.meta_data) channel = self.get_channel(s.channel_id) if stream_id in channel.streams: raise StreamAlreadyExistsError(stream_id) from channels import MemoryChannel, DatabaseChannel if isinstance(channel, MemoryChannel): channel.create_stream(stream_id) elif isinstance(channel, DatabaseChannel): calculated_intervals = None with switch_db(StreamStatusModel, db_alias='hyperstream'): try: status = StreamStatusModel.objects.get(__raw__=stream_id.as_raw()) calculated_intervals = TimeIntervals(map(lambda x: (x.start, x.end), status.calculated_intervals)) except DoesNotExist as e: logging.debug(e) status = StreamStatusModel( stream_id=stream_id.as_dict(), calculated_intervals=[], last_accessed=utcnow(), last_updated=utcnow()) status.save() except MultipleObjectsReturned as e: raise e channel.streams[stream_id] = DatabaseStream( channel=channel, stream_id=stream_id, calculated_intervals=calculated_intervals, sandbox=s.sandbox) else: raise NotImplementedError
[docs] def get_tool_class(self, tool): """ Gets the actual class which cna then be instantiated with its parameters :param tool: The tool name or id :return: The tool class :type tool: str | unicode | StreamId :rtype Tool | MultiOutputTool """ if isinstance(tool, (str, unicode)): tool_id = StreamId(tool) elif isinstance(tool, StreamId): tool_id = tool else: raise TypeError(tool) tool_stream_view = None # Look in the main tool channel first if tool_id in self.tools: tool_stream_view = self.tools[tool_id].window((MIN_DATE, self.tools.up_to_timestamp)) else: # Otherwise look through all the channels in the order they were defined for tool_channel in self.tool_channels: if tool_channel == self.tools: continue if tool_id in tool_channel: # noinspection PyTypeChecker tool_stream_view = tool_channel[tool_id].window((MIN_DATE, tool_channel.up_to_timestamp)) if tool_stream_view is None: raise ToolNotFoundError(tool) # TODO: Use tool versions - here we just take the latest one return tool_stream_view.last().value
[docs] def get_tool(self, name, parameters): """ Gets the tool object from the tool channel(s), and instantiates it using the tool parameters :param name: The name or stream id for the tool in the tool channel :param parameters: The parameters for the tool :return: The instantiated tool object """ tool_class = self.get_tool_class(name) # Check that the number of arguments is correct for this tool arg_spec = inspect.getargspec(tool_class.__init__) max_expected = len(arg_spec[0]) if arg_spec.defaults: min_expected = max_expected - len(arg_spec.defaults) else: min_expected = max_expected num_parameters = len(parameters) if parameters is not None else 0 if not (min_expected <= num_parameters + 1 <= max_expected): message = "Tool {} takes a between {} and {} arguments ({} given)".format( tool_class.__name__, min_expected, max_expected, num_parameters + 1) # logging.warn(message) raise ValueError(message) # Instantiate tool name = tool_class(**parameters) if parameters is not None else tool_class() if not name: raise ToolNotFoundError return name