diff --git a/Pipfile b/Pipfile new file mode 100644 index 0000000..36df033 --- /dev/null +++ b/Pipfile @@ -0,0 +1,15 @@ +[[source]] +url = "https://pypi.org/simple" +verify_ssl = true +name = "pypi" + +[packages] +redis = "*" +pymemcache = "*" +pytest = "*" +pytest-cov = "*" + +[dev-packages] + +[requires] +python_version = "3.7" diff --git a/Pipfile.lock b/Pipfile.lock new file mode 100644 index 0000000..a65f97f --- /dev/null +++ b/Pipfile.lock @@ -0,0 +1,136 @@ +{ + "_meta": { + "hash": { + "sha256": "955727c5c7783398b2ca8b0d7911ef8280351a3d525e1da4a4159ddb3644f068" + }, + "pipfile-spec": 6, + "requires": { + "python_version": "3.7" + }, + "sources": [ + { + "name": "pypi", + "url": "https://pypi.org/simple", + "verify_ssl": true + } + ] + }, + "default": { + "atomicwrites": { + "hashes": [ + "sha256:0312ad34fcad8fac3704d441f7b317e50af620823353ec657a53e981f92920c0", + "sha256:ec9ae8adaae229e4f8446952d204a3e4b5fdd2d099f9be3aaf556120135fb3ee" + ], + "markers": "python_version != '3.2.*' and python_version != '3.1.*' and python_version >= '2.7' and python_version != '3.3.*' and python_version != '3.0.*'", + "version": "==1.2.1" + }, + "attrs": { + "hashes": [ + "sha256:10cbf6e27dbce8c30807caf056c8eb50917e0eaafe86347671b57254006c3e69", + "sha256:ca4be454458f9dec299268d472aaa5a11f67a4ff70093396e1ceae9c76cf4bbb" + ], + "version": "==18.2.0" + }, + "coverage": { + "hashes": [ + "sha256:03481e81d558d30d230bc12999e3edffe392d244349a90f4ef9b88425fac74ba", + "sha256:0b136648de27201056c1869a6c0d4e23f464750fd9a9ba9750b8336a244429ed", + "sha256:10a46017fef60e16694a30627319f38a2b9b52e90182dddb6e37dcdab0f4bf95", + "sha256:198626739a79b09fa0a2f06e083ffd12eb55449b5f8bfdbeed1df4910b2ca640", + "sha256:23d341cdd4a0371820eb2b0bd6b88f5003a7438bbedb33688cd33b8eae59affd", + "sha256:28b2191e7283f4f3568962e373b47ef7f0392993bb6660d079c62bd50fe9d162", + "sha256:2a5b73210bad5279ddb558d9a2bfedc7f4bf6ad7f3c988641d83c40293deaec1", + "sha256:2eb564bbf7816a9d68dd3369a510be3327f1c618d2357fa6b1216994c2e3d508", + "sha256:337ded681dd2ef9ca04ef5d93cfc87e52e09db2594c296b4a0a3662cb1b41249", + "sha256:3a2184c6d797a125dca8367878d3b9a178b6fdd05fdc2d35d758c3006a1cd694", + "sha256:3c79a6f7b95751cdebcd9037e4d06f8d5a9b60e4ed0cd231342aa8ad7124882a", + "sha256:3d72c20bd105022d29b14a7d628462ebdc61de2f303322c0212a054352f3b287", + "sha256:3eb42bf89a6be7deb64116dd1cc4b08171734d721e7a7e57ad64cc4ef29ed2f1", + "sha256:4635a184d0bbe537aa185a34193898eee409332a8ccb27eea36f262566585000", + "sha256:56e448f051a201c5ebbaa86a5efd0ca90d327204d8b059ab25ad0f35fbfd79f1", + "sha256:5a13ea7911ff5e1796b6d5e4fbbf6952381a611209b736d48e675c2756f3f74e", + "sha256:69bf008a06b76619d3c3f3b1983f5145c75a305a0fea513aca094cae5c40a8f5", + "sha256:6bc583dc18d5979dc0f6cec26a8603129de0304d5ae1f17e57a12834e7235062", + "sha256:701cd6093d63e6b8ad7009d8a92425428bc4d6e7ab8d75efbb665c806c1d79ba", + "sha256:7608a3dd5d73cb06c531b8925e0ef8d3de31fed2544a7de6c63960a1e73ea4bc", + "sha256:76ecd006d1d8f739430ec50cc872889af1f9c1b6b8f48e29941814b09b0fd3cc", + "sha256:7aa36d2b844a3e4a4b356708d79fd2c260281a7390d678a10b91ca595ddc9e99", + "sha256:7d3f553904b0c5c016d1dad058a7554c7ac4c91a789fca496e7d8347ad040653", + "sha256:7e1fe19bd6dce69d9fd159d8e4a80a8f52101380d5d3a4d374b6d3eae0e5de9c", + "sha256:8c3cb8c35ec4d9506979b4cf90ee9918bc2e49f84189d9bf5c36c0c1119c6558", + "sha256:9d6dd10d49e01571bf6e147d3b505141ffc093a06756c60b053a859cb2128b1f", + "sha256:be6cfcd8053d13f5f5eeb284aa8a814220c3da1b0078fa859011c7fffd86dab9", + "sha256:c1bb572fab8208c400adaf06a8133ac0712179a334c09224fb11393e920abcdd", + "sha256:de4418dadaa1c01d497e539210cb6baa015965526ff5afc078c57ca69160108d", + "sha256:e05cb4d9aad6233d67e0541caa7e511fa4047ed7750ec2510d466e806e0255d6", + "sha256:f3f501f345f24383c0000395b26b726e46758b71393267aeae0bd36f8b3ade80" + ], + "markers": "python_version != '3.1.*' and python_version >= '2.6' and python_version != '3.0.*' and python_version != '3.2.*' and python_version < '4'", + "version": "==4.5.1" + }, + "more-itertools": { + "hashes": [ + "sha256:c187a73da93e7a8acc0001572aebc7e3c69daf7bf6881a2cea10650bd4420092", + "sha256:c476b5d3a34e12d40130bc2f935028b5f636df8f372dc2c1c01dc19681b2039e", + "sha256:fcbfeaea0be121980e15bc97b3817b5202ca73d0eae185b4550cbfce2a3ebb3d" + ], + "version": "==4.3.0" + }, + "pluggy": { + "hashes": [ + "sha256:447ba94990e8014ee25ec853339faf7b0fc8050cdc3289d4d71f7f410fb90095", + "sha256:bde19360a8ec4dfd8a20dcb811780a30998101f078fc7ded6162f0076f50508f" + ], + "markers": "python_version != '3.2.*' and python_version != '3.1.*' and python_version >= '2.7' and python_version != '3.3.*' and python_version != '3.0.*'", + "version": "==0.8.0" + }, + "py": { + "hashes": [ + "sha256:bf92637198836372b520efcba9e020c330123be8ce527e535d185ed4b6f45694", + "sha256:e76826342cefe3c3d5f7e8ee4316b80d1dd8a300781612ddbc765c17ba25a6c6" + ], + "markers": "python_version != '3.2.*' and python_version != '3.1.*' and python_version >= '2.7' and python_version != '3.3.*' and python_version != '3.0.*'", + "version": "==1.7.0" + }, + "pymemcache": { + "hashes": [ + "sha256:210ccbe6ec2baf0a7d4009ea6594a746d11dc5e56421f4234cad5084958f44b6", + "sha256:5cfcd9f8393a753897d31fc59dfc506c19d2bfa4a791093b91a72a2677431f79" + ], + "index": "pypi", + "version": "==2.0.0" + }, + "pytest": { + "hashes": [ + "sha256:10e59f84267370ab20cec9305bafe7505ba4d6b93ecbf66a1cce86193ed511d5", + "sha256:8c827e7d4816dfe13e9329c8226aef8e6e75d65b939bc74fda894143b6d1df59" + ], + "index": "pypi", + "version": "==3.9.1" + }, + "pytest-cov": { + "hashes": [ + "sha256:513c425e931a0344944f84ea47f3956be0e416d95acbd897a44970c8d926d5d7", + "sha256:e360f048b7dae3f2f2a9a4d067b2dd6b6a015d384d1577c994a43f3f7cbad762" + ], + "index": "pypi", + "version": "==2.6.0" + }, + "redis": { + "hashes": [ + "sha256:8a1900a9f2a0a44ecf6e8b5eb3e967a9909dfed219ad66df094f27f7d6f330fb", + "sha256:a22ca993cea2962dbb588f9f30d0015ac4afcc45bee27d3978c0dbe9e97c6c0f" + ], + "index": "pypi", + "version": "==2.10.6" + }, + "six": { + "hashes": [ + "sha256:70e8a77beed4562e7f14fe23a786b54f6296e34344c23bc42f07b15018ff98e9", + "sha256:832dc0e10feb1aa2c68dcc57dbb658f1c7e65b9b61af69048abc87a2db00a0eb" + ], + "version": "==1.11.0" + } + }, + "develop": {} +} diff --git a/README.md b/README.md index 3c46fe2..4329929 100644 --- a/README.md +++ b/README.md @@ -8,12 +8,13 @@ Install using pip: `$ pip install cache4py` ### Usage ```python -from cache4py import cache, LRU, Backend +from cache4py.decorators import cache +from cache4py.storage.backends import RedisBackend # You can choose memcached, redis or default (python's dict) as a backend. -redis_backend = Backend(variant=Backend.REDIS, url='', port='') +redis_backend = RedisBackend(url='localhost', port='6379') -@cache(eviction_policy=LRU, backend=redis_backend) +@cache(backend=redis_backend) def my_function_one(*args, **kwargs): # do something awesome return @@ -42,11 +43,15 @@ pip install -r requirements.txt ``` ### Running the tests -Run unit tests using the command: `TODO` +Run unit tests using the command: + +``` +pytest --cov=cache4py --cov-report html tests/ +``` ## Issue tracking -Create issues at [cache4pd/issues](https://github.com/nitinl/cache4py/issues). +Create issues at [cache4py/issues](https://github.com/nitinl/cache4py/issues). ## Authors diff --git a/cache4py/__init__.py b/cache4py/__init__.py new file mode 100644 index 0000000..558e202 --- /dev/null +++ b/cache4py/__init__.py @@ -0,0 +1,5 @@ +""" +Author: nitin +Date: 18/7/17 +Description: +""" \ No newline at end of file diff --git a/cache4py/decorators.py b/cache4py/decorators.py new file mode 100644 index 0000000..f1e196e --- /dev/null +++ b/cache4py/decorators.py @@ -0,0 +1,44 @@ +""" +Author: nitin +Date: 18/7/17 +Description: +""" +from functools import wraps + +from cache4py.exceptions import BackendException +from cache4py.utils import args_to_key, hash_key +from cache4py.storage.redis import RedisBackend + + +def cache(backend=RedisBackend, keys=args_to_key): + """ + Cache decorator. Backend can be a python dict, redis or memcached server. + """ + + def _decorator(func): + @wraps(func) + def wrapper(*args, **kwargs): + if backend is None: + backend_exception_message = 'Invalid input None provided for backend.' + raise BackendException(backend_exception_message) + + key = keys(*args, **kwargs) + hashed_key = hash_key(key) + + result = backend.get(hashed_key) + + if result is not None: + # cache hit + return result + + # cache miss + function_result = func(*args, **kwargs) + + # store result in cache + backend.set(hashed_key, function_result) + return function_result + + return wrapper + + return _decorator + diff --git a/cache4py/exceptions.py b/cache4py/exceptions.py new file mode 100644 index 0000000..bbeb205 --- /dev/null +++ b/cache4py/exceptions.py @@ -0,0 +1,21 @@ +""" +Author: nitin +Date: 18/7/17 +Description: +""" + + +class Cache4PyException(Exception): + pass + + +class BackendException(Cache4PyException): + pass + + +class RedisBackendException(BackendException): + pass + + +class MemcachedBackendException(BackendException): + pass diff --git a/cache4py/storage/__init__.py b/cache4py/storage/__init__.py new file mode 100644 index 0000000..5167051 --- /dev/null +++ b/cache4py/storage/__init__.py @@ -0,0 +1,5 @@ +""" +Author: nitin +Date: 19/10/18 +Description: +""" \ No newline at end of file diff --git a/cache4py/storage/base.py b/cache4py/storage/base.py new file mode 100644 index 0000000..9300702 --- /dev/null +++ b/cache4py/storage/base.py @@ -0,0 +1,41 @@ +""" +Author: nitin +Date: 19/10/18 +Description: Base class for all storage interfaces +""" +import abc + + +class BaseBackend(metaclass=abc.ABCMeta): + """ + Abstract class defining Cache client interface. All implementations of cache clients + must extend this class. + """ + + @abc.abstractmethod + def get(self, key_name): + """ + Return the value at key ``key_name``, or None if the key doesn't exist. + :param key_name: A key. + :return: Value at key ``key_name``, or None if the key doesn't exist. + """ + raise NotImplementedError('Cache client must define a get method.') + + @abc.abstractmethod + def set(self, key_name, value): + """ + Set the value at key ``key_name`` to ``value`` + :param key_name: A key object. + :param value: A value object. + :return: True if set successfully else False. + """ + raise NotImplementedError('Cache client must define a set method.') + + @abc.abstractmethod + def delete(self, key_name): + """ + Deletes key specified by ``key_name``. + :param key_name: A key object. + :return: True if key is successfully deleted else False. Failure can mean connection issue or no value at given key in db. + """ + raise NotImplementedError('Cache client must define a delete method.') \ No newline at end of file diff --git a/cache4py/storage/memcached.py b/cache4py/storage/memcached.py new file mode 100644 index 0000000..78e824c --- /dev/null +++ b/cache4py/storage/memcached.py @@ -0,0 +1,109 @@ +""" +Author: nitin +Date: 19/10/18 +Description: Memcached storage connector +""" +import pickle +import warnings + +from pymemcache.client import Client + +from cache4py.exceptions import MemcachedBackendException +from cache4py.storage.base import BaseBackend + + +class MemcachedBackend(BaseBackend): + """ + Wrapper over memcached client object provided by pymemcached library. Supports storing objects in memcached. + """ + + @staticmethod + def memcached_serializer(key, value): + """ + Pickle objects for storing in memcached. + :param key: A key (length not exceeding 250) + :param value: A value object. + :return: Tuple (Serialized object, type_code) + """ + if type(value) == str: + return value, 1 + return pickle.dumps(value, protocol=pickle.HIGHEST_PROTOCOL), 2 + + @staticmethod + def memcached_deserializer(key, value, flags): + """ + Unpickles objects for recovering them from memcached store. + :param key: A key (length not exceeding 250) + :param value: Pickled object corresponding to above key. + :param flags: Value type code. + :return: Unpickled python object. + """ + if flags == 1: + return value + if flags == 2: + return pickle.loads(value) + raise Exception("Unknown serialization format") + + def __init__(self, server, port=11211): + """ + Initialize memcached client as a cache backend. + :param server: URL of memcached service. + :param port: Port number at which memcached service is exposed. If not specified, uses port 11211 by default. + """ + self.__server = server + self.__port = port + self.__client = Client(server=(server, port), + default_noreply=True, + serializer=MemcachedBackend.memcached_serializer, + deserializer=MemcachedBackend.memcached_deserializer) + + try: + self.__client.stats() + except Exception as exception: + warnings.warn( + 'Error in connecting to memcached server: {0} at port: {1}. More details:\n{2}'.format(server, port, + exception)) + + def get(self, key_name): + """ + Return the value at key ``key_name``, or None if the key doesn't exist. + :param key_name: A key. + :return: Value at key ``key_name``, or None if the key doesn't exist. + """ + try: + retrieved_value = self.__client.get(key_name) + return retrieved_value + except Exception as e: + raise MemcachedBackendException( + 'Error in connecting to memcached server: {0} at port: {1}. More details:\n{2}'.format(self.__server, + self.__port, e)) + + def set(self, key_name, value): + """ + Set the value at key ``key_name`` to ``value`` + :param key_name: A key object. + :param value: A value object. + :return: True if set successfully else False. + """ + try: + set_response = self.__client.set(key_name, value) + return set_response + except Exception as e: + raise MemcachedBackendException( + 'Error in connecting to memcached server: {0} at port: {1}. More details:\n{2}'.format(self.__server, + self.__port, e)) + + def delete(self, key_name): + """ + Deletes key specified by ``key_name``. + :param key_name: A key object. + :return: True if key is successfully deleted else False. Failure can mean connection issue or no value at given key in db. + """ + try: + if self.__client.delete(key_name) >= 0: + return True + return False # given key does not exist in memcached + except Exception as e: + raise MemcachedBackendException( + 'Error in connecting to memcached server: {0} at port: {1}. More details:\n{2}'.format(self.__server, + self.__port, e)) \ No newline at end of file diff --git a/cache4py/storage/redis.py b/cache4py/storage/redis.py new file mode 100644 index 0000000..04a3b01 --- /dev/null +++ b/cache4py/storage/redis.py @@ -0,0 +1,84 @@ +""" +Author: nitin +Date: 19/10/18 +Description: Redis storage connector +""" +import pickle +import warnings + +import redis + +from cache4py.exceptions import RedisBackendException +from cache4py.storage.base import BaseBackend + + +class RedisBackend(BaseBackend): + """ + Wrapper over redis client object provided by python redis library. Supports storing objects in redis. + """ + + def __init__(self, server, port=6379): + """ + Initialize redis client as a cache backend. + :param server: URL of redis service. + :param port: Port number at which redis service is exposed. If not specified, uses port 6379 by default. + """ + self.__server = server + self.__port = port + self.__client = redis.StrictRedis(host=server, port=port) + try: + self.__client.ping() + except redis.ConnectionError as connection_error: + warnings.warn('Failed to connect to redis server: {0} at port: {1}'.format(server, port)) + raise RedisBackendException(connection_error) + + def is_client_valid(self): + """ + Checks if redis client points to an active Redis server. + :return: True if client points to a functional Redis server else False. + """ + if self.__client is None: + return False + try: + self.__client.ping() + except redis.ConnectionError: + return False + return True + + def get(self, key_name): + """ + Return the value at key ``key_name``, or None if the key doesn't exist. + :param key_name: A key. + :return: Value at key ``key_name``, or None if the key doesn't exist. + """ + if not self.is_client_valid(): + raise RedisBackendException('Failed to connect to redis backend {0}:{1}'.format(self.__server, self.__port)) + retrieved_value = self.__client.get(key_name) + if retrieved_value is not None: + retrieved_value = pickle.loads(retrieved_value) + return retrieved_value + + def set(self, key_name, value): + """ + Set the value at key ``key_name`` to ``value`` + :param key_name: A key object. + :param value: A value object. + :return: True if set successfully else False. + """ + if not self.is_client_valid(): + raise RedisBackendException('Failed to connect to redis backend {0}:{1}'.format(self.__server, self.__port)) + set_response = self.__client.set(key_name, pickle.dumps(value)) + return set_response + + def delete(self, key_name): + """ + Deletes key specified by ``key_name``. + :param key_name: A key object. + :return: True if key is successfully deleted else False. Failure can mean connection issue or no value at given key in db. + """ + if not self.is_client_valid(): + raise RedisBackendException('Failed to connect to redis backend {0}:{1}'.format(self.__server, self.__port)) + if self.__client.delete(key_name) >= 0: + return True + + return False \ No newline at end of file diff --git a/cache4py/utils.py b/cache4py/utils.py new file mode 100644 index 0000000..2c7e509 --- /dev/null +++ b/cache4py/utils.py @@ -0,0 +1,34 @@ +""" +Author: nitin +Date: 18/7/17 +Description: +""" + +import pickle +from hashlib import sha224 + + +def args_to_key(*args, **kwargs): + """ + Makes a single object from given args and kwargs. + :param args: Args list. + :param kwargs: Kwargs dictionary. + :return: A combined object of the given args and kwargs. + """ + params = args + if kwargs: + # sort dictionary keys and merge to match inputs where user changes order of kwargs. + _kwmark = (object(),) + params += sum(sorted(kwargs.items()), _kwmark) + return params + + +def hash_key(python_object): + """ + Computes a consistent sha224 hash for given object. + :param python_object: A python object. + :return: Consistent sha224 hash for the key_object. + """ + serialized_key = pickle.dumps(python_object) + hashed_key = sha224(serialized_key).hexdigest() + return hashed_key diff --git a/requirements.in b/requirements.in new file mode 100644 index 0000000..a3b8b31 --- /dev/null +++ b/requirements.in @@ -0,0 +1 @@ +redis==2.10.5 \ No newline at end of file diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..5167051 --- /dev/null +++ b/tests/__init__.py @@ -0,0 +1,5 @@ +""" +Author: nitin +Date: 19/10/18 +Description: +""" \ No newline at end of file diff --git a/tests/test_basic.py b/tests/test_basic.py new file mode 100644 index 0000000..6141a87 --- /dev/null +++ b/tests/test_basic.py @@ -0,0 +1,68 @@ +""" +Author: nitin +Date: 18/7/17 +Description: +""" +import time +from math import factorial + +from cache4py.decorators import cache +from cache4py.storage.memcached import MemcachedBackend +from cache4py.storage.redis import RedisBackend +from cache4py.utils import hash_key + +redis_backend = RedisBackend('localhost', 6379) +memcached_backend = MemcachedBackend('127.0.0.1', 11211) + +@cache(backend=redis_backend) +def redis_target_function(x): + return factorial(x) + +@cache(backend=memcached_backend) +def memcached_target_function(x): + return factorial(x) + + +def uncached_target_function(x): + return factorial(x) + + +def test_redis(): + print("Start redis caching test") + + start_time = time.time() + for i in range(5): + _ = uncached_target_function(75000) + uncached_time = time.time() - start_time + + start_time = time.time() + for i in range(5): + _ = redis_target_function(75000) + cached_time = time.time() - start_time + + print("Time difference: before: {0}, after: {1}".format(uncached_time, cached_time)) + + assert(cached_time < uncached_time) + + hashed_key = hash_key(75000) + redis_backend.delete(key_name=hashed_key) + +def test_memcached(): + print("Start memcached caching test") + + start_time = time.time() + for i in range(5): + _ = uncached_target_function(75000) + uncached_time = time.time() - start_time + + start_time = time.time() + for i in range(5): + _ = memcached_target_function(75000) + cached_time = time.time() - start_time + + print("Time difference: before: {0}, after: {1}".format(uncached_time, cached_time)) + + assert(cached_time < uncached_time) + + hashed_key = hash_key(75000) + memcached_backend.delete(key_name=hashed_key)