--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/configurationengine/source/cone/storage/webstorage.py Thu Mar 11 17:04:37 2010 +0200
@@ -0,0 +1,621 @@
+#
+# Copyright (c) 2009 Nokia Corporation and/or its subsidiary(-ies).
+# All rights reserved.
+# This component and the accompanying materials are made available
+# under the terms of "Eclipse Public License v1.0"
+# which accompanies this distribution, and is available
+# at the URL "http://www.eclipse.org/legal/epl-v10.html".
+#
+# Initial Contributors:
+# Nokia Corporation - initial contribution.
+#
+# Contributors:
+#
+# Description:
+#
+##
+# @author Teemu Rytkonen
+
+import getpass
+import StringIO
+import os
+import re
+import copy
+import logging
+import httplib
+import urllib, urllib2
+import simplejson
+import posixpath
+
+from cone.public import *
+from cone.carbon import persistentjson, model
+from cone.storage import authenticate
+
+class WebStorage(api.Storage):
+ """
+ A general base class for all storage type classes
+ @param path : the reference to the root of the storage.
+ """
+
+ def __init__(self, path, mode="a", **kwargs):
+ api.Storage.__init__(self,path)
+ self.mode = mode
+ self.extapi = CarbonExtapi(path, **kwargs)
+ self.persistentmodule = persistentjson
+
+ # resource cache is a intermediate solution to create a mapping between carbon objects and Configuration project
+ self._resource_cache = None
+
+
+ def _create_resource(self, path, object):
+ """
+ Create a new resource (configuration|featurelist) to carbon storage.
+ """
+ # Test the path, whether it is a featurelist or configuration
+ try:
+ object_type = object.meta.get('type')
+ except (TypeError,AttributeError):
+ logging.getLogger('cone').error('Cannot dump configuration %s to webstorage without a type.' % path)
+ return False
+ carbonpath = persistentjson.CarbonResourceMapper().map_confml_resource(object_type, path)
+ if object_type == 'featurelist':
+ # Create a featurelist
+ success = self.extapi.create_featurelist(carbonpath, object)
+ if success:
+ self.resource_cache.add_resource_link(path, carbonpath)
+ return success
+ else:
+ # Create a configuration
+ success = self.extapi.create_configuration(carbonpath, object)
+ if success:
+ self.resource_cache.add_resource_link(path, carbonpath)
+ return success
+
+ @classmethod
+ def supported_storage(cls,path):
+ """
+ Class method for determing if the given clas supports a storage by given path.
+ E.g. http://foo.com/
+ @param path:
+ @return: Boolean value. True if the storage of the path is supported. False if not.
+ """
+ if path.startswith('http://'):
+ return True
+ else:
+ return False
+
+ @property
+ def resource_cache(self):
+ """
+ Returns a resource cache dictionary of all the resources inside the Carbon storage. Works as an intermediate
+ solution to link Configuration project concepts to Carbon storage
+
+ """
+ if not self._resource_cache:
+ self._resource_cache = ResourceCache()
+ reslist = self.extapi.list_resources("/", True)
+ # Append all resources to resource cache
+ for res in reslist:
+ self._resource_cache.add_resource(res)
+# if isinstance(res, model.ConfigurationResource):
+# self._resource_cache.add_configuration(res)
+# elif isinstance(res, model.FeatureListResource):
+# self._resource_cache.add_featurelist(res)
+ return self._resource_cache
+
+ def list_resources(self,path, recurse=False, empty_folders=False):
+ """
+ find the resources under certain path/path
+ @param path : reference to path where resources are searched
+ @param recurse : defines whether to return resources directly under the path or does the listing recurse to subfolders.
+ Default value is False. Set to True to enable recursion.
+ @param empty_folders: parameters that defined whether empty folders are included. This parameter is ignored
+ in WebStorage.
+ """
+ return self.resource_cache.list_resources(path,recurse)
+
+
+ def open_resource(self,path,mode="r"):
+ path = utils.resourceref.remove_begin_slash(path)
+ if self.resource_cache.get_resource_object(path):
+ return self.resource_cache.get_resource_object(path)
+ elif self.resource_cache.get_resource_link(path):
+ path = self.resource_cache.get_resource_link(path)
+
+# path = path.replace(".confml", ".configuration")
+# path = utils.resourceref.join_refs([self.get_current_path(), path])
+ try:
+ if self.get_mode(mode) == self.MODE_READ:
+ strio = self.extapi._get_stringio_from_path(path)
+ elif self.get_mode(mode) == self.MODE_APPEND:
+ strio = self.extapi._get_stringio_from_path(path)
+ strio.seek(0, os.SEEK_END)
+ elif self.get_mode(mode) == self.MODE_WRITE:
+ strio = StringIO.StringIO()
+ else:
+ raise StorageException("Unrecognized mode %s" % mode)
+ res = WebResource(self,path,mode,strio)
+ self.__opened__(res)
+ return res
+ except KeyError:
+ raise exceptions.NotResource("The given resource is not found %s" % path)
+
+ def is_resource(self,path):
+ return self.resource_cache.is_resource(path)
+# path = path.replace(".confml", ".configuration")
+# path = utils.resourceref.join_refs([self.get_current_path(), path])
+# try:
+# query = urllib.quote(self._get_action_url('is_resource', path))
+# self.conn.request("GET", query)
+# resp = self.conn.getresponse()
+# reader = persistentjson.HasResourceReader()
+# return reader.loads(resp.read())
+# except exceptions.NotFound:
+# return False
+
+ def save_resource(self, res):
+ """
+ Close resource is no-operation action with webstorage for now
+ """
+ return
+
+ def close_resource(self, path):
+ """
+ Close resource is no-operation action with webstorage
+ """
+ return
+
+ def export_resources(self, paths, storage, empty_folders=False):
+ for path in paths:
+ if not self.is_resource(path):
+ logging.getLogger('cone').warning("The given path is not a Resource in this storage %s! Ignoring from export!" % path)
+ continue
+ wres = storage.open_resource(path,'wb')
+ res = self.open_resource(path,"rb")
+ wres.write(res.read())
+ wres.close()
+ res.close()
+
+ def unload(self, path, object):
+ """
+ Dump a given object to the storage (reference is fetched from the object)
+ @param object: The object to dump to the storage, which is expected to be an instance
+ of Base class.
+ """
+ # Add the current path in front of the given path
+ path = utils.resourceref.join_refs([self.get_current_path(), path])
+ print "unload %s" % path
+ if not isinstance(object, api.Configuration):
+ raise exceptions.StorageException("Cannot dump object type %s" % object.__class__)
+ # Skip the unload storing to storage if the storage is opened in read mode
+ if self.get_mode(self.mode) != api.Storage.MODE_READ:
+ if self.resource_cache.get_resource_link(path):
+ path = self.resource_cache.get_resource_link(path)
+ else:
+ """ otherwise create the new resource first before update"""
+ if self._create_resource(path, object):
+ path = self.resource_cache.get_resource_link(path)
+ else:
+ # Creation failed
+ logging.getLogger('cone').error('Creation of %s resource failed' % path)
+ return
+ data = persistentjson.dumps(object)
+ self.extapi.update_resource(path, data)
+ else:
+ raise exceptions.StorageException("Cannot dump object to readonly storage")
+ return
+
+ def load(self, path):
+ """
+ Load resource from a path.
+ """
+ # Check if the object is already cached or has a cached link to another resource to load
+ path = utils.resourceref.remove_begin_slash(path)
+ print "load %s" % path
+ if self.resource_cache.get_resource_link(path):
+ path = self.resource_cache.get_resource_link(path)
+ elif self.resource_cache.get_mapped_resource(path):
+ path = self.resource_cache.get_mapped_resource(path)
+ elif not utils.resourceref.get_ext(path) == "confml":
+ raise exceptions.StorageException("Cannot load reference type %s" % utils.resourceref.get_ext(path))
+ else:
+ # Add the current path in front of the given path
+ path = utils.resourceref.join_refs([self.get_current_path(), path])
+ if not self.is_resource(path):
+ raise exceptions.NotResource("No such %s resource!" % path)
+
+ res = self.open_resource(path,"r")
+ # read the resource with persistentmodule
+ try:
+ obj = self.persistentmodule.loads(res.read())
+ #obj.set_path(path)
+ res.close()
+ return obj
+ except exceptions.ParseError,e:
+ logging.getLogger('cone').error("Resource %s parsing failed with exception: %s" % (path,e))
+ # returning an empty config in case of xml parsing failure.
+ return api.Configuration(path)
+
+ def close(self):
+ """ No operation in web storage close """
+ pass
+
+
+class WebResource(api.Resource):
+ def __init__(self,storage,path,mode,handle):
+ api.Resource.__init__(self,storage,path,mode)
+ self.handle = handle
+
+ def read(self,bytes=0):
+ if bytes == 0:
+ return self.handle.read()
+ else:
+ return self.handle.read(bytes)
+
+ def write(self,string):
+ if self.get_mode() == api.Storage.MODE_READ:
+ raise exceptions.StorageException("Writing attempted to %s in read-only mode." % self.path)
+ else:
+ self.handle.write(string)
+
+ def truncate(self,size=0):
+ raise exceptions.NotSupportedException()
+
+ def flush(self):
+ self.storage.flush_resource(self)
+
+ def close(self):
+ self.storage.close_resource(self)
+ self.handle.close()
+
+ def get_size(self):
+ if self.get_mode() == api.Storage.MODE_WRITE:
+ raise exceptions.StorageException("Reading resource size attempted to %s in write-only mode." % self.path)
+ return len(self.handle.getvalue())
+
+ def getvalue(self):
+ return self.handle.getvalue()
+
+
+class CarbonExtapi(object):
+ ACTIONS = { 'open_resource' : 'get_resource',
+ 'list_resources' : 'list_resources',
+ 'is_resource' : 'has_resource',
+ 'put_resource' : 'put_resource',
+ 'update_resource' : 'update_resource' }
+
+ """
+ A general container for Carbon extapi action
+ """
+ def __init__(self, path, **kwargs):
+ self.path = path
+ self.server_path = ''
+ self.service_path = ''
+ if path.startswith('http://'):
+ path = path.replace('http://','',1)
+ pathelems = path.split('/',1)
+ self.server_path = pathelems[0]
+ if len(pathelems) > 1:
+ self.service_path = pathelems[1]
+ self._username = kwargs.get('username', None)
+ self._password = kwargs.get('password', None)
+ authhandler = authenticate.CarbonAuthHandler()
+ authhandler.add_password(self.username, self.password)
+ self.conn = urllib2.build_opener(urllib2.HTTPCookieProcessor, authhandler, urllib2.ProxyHandler({}))
+
+ @property
+ def username(self):
+ if self._username == None:
+ self._username = getpass.getuser()
+ return self._username
+
+ @property
+ def password(self):
+ if self._password == None:
+ self._password = getpass.getpass()
+ return self._password
+
+ def checklogin(self):
+ """
+ Checks that we are logged in by loading the main page.
+ If we are not logged in it will redirect us to login page.
+ """
+ loginurl = "http://%(host)s/" % dict(host=self.server_path)
+ loginreq = urllib2.Request(loginurl)
+ print 'Checking login by opening URL %s ...' % loginurl
+ try:
+ resp = self.conn.open(loginreq)
+ except urllib2.URLError, e:
+ print str(e)
+ return False
+ return True
+
+ def _get_stringio_from_path(self,path):
+ """
+ return a StringIO object containing the data under a given path.
+ @return: StringIO buffer
+ @raise exception.NotResource: if the resource under path is not found
+ """
+ path = utils.resourceref.remove_begin_slash(path)
+ action_url = self._get_action_url('open_resource', path)
+ req = urllib2.Request(action_url)
+ try:
+ resp = self.conn.open(req)
+ bytes = resp.read()
+ strio = StringIO.StringIO(bytes)
+ return strio
+ except urllib2.HTTPError,e:
+ raise exceptions.NotResource("The given path %s could not be retrived from this storage. Server returned status %s." % (path, e))
+
+ def _get_action_url(self, action, path, **kwargs):
+ path = utils.resourceref.remove_begin_slash(path)
+ return "%s/%s/%s" % (self.path,self.ACTIONS[action], urllib.quote(path))
+
+ def list_resources_for_type(self,path,type, recurse=False):
+ """
+ find the resources under certain path/path
+ @param path : reference to path where resources are searched
+ @param type : resources for particular carbon specific type.
+ @param recurse : defines whether to return resources directly under the path or does the listing recurse to subfolders.
+ Default value is False. Set to True to enable recursion.
+ """
+ try:
+ path = utils.resourceref.join_refs([path,'.'+type])
+ path = utils.resourceref.remove_begin_slash(path)
+ query = self._get_action_url('list_resources', path)
+ req = urllib2.Request(query)
+ resp = self.conn.open(req)
+ if resp.code == httplib.OK:
+ bytes = resp.read()
+ reader = persistentjson.ResourceListReader()
+ reslist = reader.loads(bytes)
+ return reslist
+ else:
+ return []
+ except exceptions.NotFound:
+ return []
+
+ def list_resources(self,path, recurse=False):
+ """
+ find the resources under certain path/path
+ @param path : reference to path where resources are searched
+ @param recurse : defines whether to return resources directly under the path or does the listing recurse to subfolders.
+ Default value is False. Set to True to enable recursion.
+ """
+ try:
+ path = utils.resourceref.remove_begin_slash(path)
+ query = self._get_action_url('list_resources', path)
+ req = urllib2.Request(query)
+ resp = self.conn.open(req)
+ if resp.code == httplib.OK:
+ bytes = resp.read()
+ reslist = simplejson.loads(bytes)
+ return reslist.get('resources',[])
+ else:
+ return []
+ except exceptions.NotFound:
+ return []
+
+
+ def update_resource(self, path, data):
+ """
+ Update a resource to carbon. The resource can be a CarbonConfiguration or FeatureList object.
+ @param object: The object which is dumped to dict with persistentjson and then updated to server.
+ """
+ try:
+ path = utils.resourceref.remove_begin_slash(path)
+ query = self._get_action_url('update_resource', path)
+ jsondata = simplejson.dumps(data)
+ encdata = urllib.urlencode({'data' : jsondata})
+ req = urllib2.Request(query, encdata)
+
+ resp = self.conn.open(req)
+ if resp.code == httplib.OK:
+ bytes = resp.read()
+ respdata = simplejson.loads(bytes)
+ success = respdata.get('success') == True
+ if success:
+ logging.getLogger('cone').info('Carbon update succeeds to path %s.' % (respdata.get('path')))
+ else:
+ logging.getLogger('cone').error('Carbon update %s failed %s' % (path,respdata.get('errors')))
+ return success
+ else:
+ logging.getLogger('cone').error('Carbon update %s failed %s: %s' % (path,resp.code, resp))
+ return False
+ except urllib2.HTTPError,e:
+ utils.log_exception(logging.getLogger('cone'), "HTTPError in %s, %s" % (query,e))
+ return False
+
+ def create_feature(self,path, feature, parent=None):
+ """
+ Create new Carbon feature based on Feature object.
+ @param path: The path to the featurelist where the feature is created.
+ @param feature: The feature object
+ @param parent: A possible parent feature ref
+ """
+ try:
+ path = utils.resourceref.remove_begin_slash(path)
+ query = self._get_action_url('put_resource', path)
+ data = persistentjson.dumps(feature)
+ if parent:
+ data['parent'] = parent
+ jsondata = simplejson.dumps(data)
+ encdata = urllib.urlencode({'data' : jsondata})
+ req = urllib2.Request(query, encdata)
+
+ resp = self.conn.open(req)
+ if resp.code == httplib.OK:
+ bytes = resp.read()
+ respdata = simplejson.loads(bytes)
+ success = respdata.get('success') == True
+ if success:
+ logging.getLogger('cone').info('New Carbon feature created to path %s.' % (respdata.get('path')))
+ else:
+ logging.getLogger('cone').error('Feature %s creation failed %s' % (feature.fqr,respdata.get('errors')))
+ return success
+ else:
+ logging.getLogger('cone').error('Feature %s creation failed %s: %s' % (feature.fqr,resp.code, resp))
+ return False
+ except urllib2.HTTPError,e:
+ utils.log_exception(logging.getLogger('cone'), "HTTPError in %s, %s" % (query,e))
+ return False
+
+ def create_featurelist(self, path, featurelist):
+ """
+ Create new Carbon featurelist to carbon.
+ @param featurelist: The FeatureList object which is created.
+ @return: tuple (success, created_path) where success indicates the success of the operation
+ and created_path is the newly created featurelist path on success.
+ """
+ try:
+ path = utils.resourceref.remove_begin_slash(path)
+ query = self._get_action_url('put_resource', path)
+ data = persistentjson.FeatureListCreateWriter().dumps(featurelist)
+ jsondata = simplejson.dumps(data)
+ encdata = urllib.urlencode({'data' : jsondata})
+ req = urllib2.Request(query, encdata)
+
+ resp = self.conn.open(req)
+ if resp.code == httplib.OK:
+ bytes = resp.read()
+ respdata = simplejson.loads(bytes)
+ success = respdata.get('success') == True
+ newpath = respdata.get('path')
+ if success:
+ logging.getLogger('cone').info('New Carbon featurelist created to path %s.' % (newpath))
+ else:
+ logging.getLogger('cone').error('FeatureList %s creation failed %s' % (featurelist.path,respdata.get('errors')))
+ return (success, newpath)
+ else:
+ logging.getLogger('cone').error('FeatureList %s creation failed %s: %s' % (featurelist.path,resp.code, resp))
+ return (False,'')
+ except urllib2.HTTPError,e:
+ utils.log_exception(logging.getLogger('cone'), "HTTPError in %s, %s" % (query,e))
+ return (False,'')
+
+ def create_configuration(self, path, configuration):
+ """
+ Create new Carbon configuration to carbon.
+ @param path: The path to the configuration
+ @param configuration: The CarbonConfiguration object
+ @return: tuple (success, created_path) where success indicates the success of the operation
+ and created_path is the newly created configuration path on success.
+ """
+ try:
+ path = utils.resourceref.remove_begin_slash(path)
+ query = self._get_action_url('put_resource', path)
+ data = persistentjson.ConfigurationCreateWriter().dumps(configuration)
+ jsondata = simplejson.dumps(data)
+ encdata = urllib.urlencode({'data' : jsondata})
+ req = urllib2.Request(query, encdata)
+
+ resp = self.conn.open(req)
+ if resp.code == httplib.OK:
+ bytes = resp.read()
+ respdata = simplejson.loads(bytes)
+ success = respdata.get('success') == True
+ newpath = respdata.get('path')
+ if success:
+ logging.getLogger('cone').info('New Carbon configuration created to path %s.' % (newpath))
+ else:
+ logging.getLogger('cone').error('CarbonConfiguration %s creation failed %s' % (configuration.path,respdata.get('errors')))
+ return (success, newpath)
+ else:
+ logging.getLogger('cone').error('CarbonConfiguration %s creation failed %s: %s' % (configuration.path,resp.code, resp))
+ return (False,'')
+ except urllib2.HTTPError,e:
+ utils.log_exception(logging.getLogger('cone'), "HTTPError in %s, %s" % (query,e))
+ return (False,'')
+
+class ResourceCache(object):
+ """
+ Resource cache maintains a list of ConE resource names and their actual links to Carbon resources.
+ """
+ def __init__(self):
+ self._cache = {}
+
+ def add_configuration(self, configuration):
+ """
+ Add a list of Carbon configurations.
+ """
+
+ # Create the configuration as a layer and configuration root
+ self._cache[configuration.get_path()] = configuration.path
+ rootconf_path = configuration.name+'.confml'
+ rootconf = model.CarbonConfiguration(rootconf_path)
+ rootconf.include_configuration(configuration.get_path())
+ self._cache[rootconf_path] = rootconf
+
+ def add_featurelist(self, featurelist):
+ """
+ Add a list of Carbon configurations.
+ """
+ # Add the feature list under feature list folder
+ self._cache["featurelists/"+featurelist.get_path()] = featurelist.get_carbon_path()
+ pass
+
+ def add_resource(self, resourcepath):
+ """
+ Add a resource
+ """
+ confmlpath = persistentjson.CarbonResourceMapper().map_carbon_resource(resourcepath)
+ self._cache[confmlpath] = resourcepath
+
+ def list_resources(self, path, recurse=False):
+ """
+ List ConE resources under certain path
+ """
+ resources = []
+ path = utils.resourceref.insert_begin_slash(path)
+ for res in self._cache.keys():
+ (respath,resname) = posixpath.split(res)
+ respath = utils.resourceref.insert_begin_slash(respath)
+ if recurse:
+ if posixpath.normpath(respath).startswith(posixpath.normpath(path)):
+ resources.append(res)
+ else:
+ if posixpath.normpath(respath) == posixpath.normpath(path):
+ resources.append(res)
+ return resources
+
+ def is_resource(self,path):
+ return self._cache.has_key(path)
+
+ def get_resource_link(self,path):
+ """
+ Get a the actual Carbon resource link if it is found from cached storage.
+ """
+ linkpath = self._cache.get(path, None)
+ if isinstance(linkpath, str):
+ return linkpath
+ else:
+ return None
+
+ def get_mapped_resource(self, path):
+ # Try to make a carbon like resource path for a confml resource
+ if path.startswith('featurelists'):
+ object_type = 'featurelist'
+ elif path.endswith('/root.confml'):
+ object_type = 'configurationlayer'
+ else:
+ object_type = 'configurationroot'
+ carbonpath = persistentjson.CarbonResourceMapper().map_confml_resource(object_type, path)
+ return carbonpath
+
+ def add_resource_link(self,link, path):
+ """
+ Add a actual Carbon resource link. The link is the key which returns path when asked from get_resource_link.
+ @param link: is linking path to the actual carbon resource path
+ @param path: is the actual carbon path
+ """
+ self._cache[link] = path
+
+ def get_resource_object(self,path):
+ """
+ Get a the actual cached Carbon object if it is found.
+ """
+ cachebj = self._cache.get(path, None)
+ if isinstance(cachebj, model.CarbonConfiguration):
+ return cachebj
+ return None
+