diff --git a/README.rst b/README.rst index 414f1cb..87a5e29 100644 --- a/README.rst +++ b/README.rst @@ -121,6 +121,8 @@ django-storage-swift recognises the following options. +----------------------------------------------+----------------+----------------------------------------------------------------------------------------------------------------------------------------------------+ | ``SWIFT_CACHE_HEADERS`` | False | Headers cache on/off switcher | +----------------------------------------------+----------------+----------------------------------------------------------------------------------------------------------------------------------------------------+ +| ``SWIFT_FILE_CACHE`` | False | Enable files metadata cache, before start collectstatic client request all files metadata, in a single request. | ++----------------------------------------------+----------------+----------------------------------------------------------------------------------------------------------------------------------------------------+ SWIFT\_BASE\_URL diff --git a/swift/storage.py b/swift/storage.py index 18dd8eb..875b727 100644 --- a/swift/storage.py +++ b/swift/storage.py @@ -3,6 +3,7 @@ import os import re from datetime import datetime +from email.utils import parsedate_to_datetime from functools import wraps from io import BytesIO, UnsupportedOperation from time import time @@ -12,6 +13,7 @@ from django.core.files import File from django.core.files.storage import Storage from six.moves.urllib import parse as urlparse +import pytz try: from django.utils.deconstruct import deconstructible @@ -29,7 +31,6 @@ def deconstructible(arg): def setting(name, default=None): return getattr(settings, name, default) - def validate_settings(backend): # Check mandatory parameters if not backend.api_auth_url: @@ -155,6 +156,8 @@ class SwiftStorage(Storage): full_listing = setting('SWIFT_FULL_LISTING', True) max_retries = setting('SWIFT_MAX_RETRIES', 5) cache_headers = setting('SWIFT_CACHE_HEADERS', False) + file_cache_enabled = setting('SWIFT_FILE_CACHE', False) + file_cache = None def __init__(self, **settings): # check if some of the settings provided as class attributes @@ -242,6 +245,30 @@ def base_url(self): self._base_url = self.override_base_url return self._base_url + def build_file_cache(self): + if(self.file_cache_enabled and self.file_cache == None): + # Build cache if never built + data = self.swift_conn.get_container( + self.container_name, + prefix=self.name_prefix, + full_listing=self.full_listing + ) + + self.file_cache = {} + for el in data[1]: + ctime_str = el['last_modified'] + name = el['name'] + ctime_dt = datetime.strptime(ctime_str, "%Y-%m-%dT%H:%M:%S.%f").replace(tzinfo=pytz.utc) + hash = el['hash'] + + if settings.USE_TZ: + last_modified = ctime_dt + else: + raise ImproperlyConfigured("settings.USE_TZ cannot be False when using swiftclient") + + self.file_cache[name] = {'last_modified': last_modified, 'hash': hash } + + def _open(self, name, mode='rb'): original_name = name name = self.name_prefix + name @@ -322,6 +349,14 @@ def get_headers(self, name): @prepend_name_prefix def exists(self, name): + if(self.file_cache_enabled): + self.build_file_cache() + try: + self.file_cache[name] + except KeyError: + return False + return True + try: self.get_headers(name) except swiftclient.ClientException: @@ -365,11 +400,6 @@ def get_available_name(self, name, max_length=None): def size(self, name): return int(self.get_headers(name)['content-length']) - @prepend_name_prefix - def modified_time(self, name): - return datetime.fromtimestamp( - float(self.get_headers(name)['x-timestamp'])) - @prepend_name_prefix def url(self, name): return self._path(name) @@ -390,9 +420,6 @@ def _path(self, name): return url - def path(self, name): - raise NotImplementedError - @prepend_name_prefix def isdir(self, name): return '.' not in name @@ -428,7 +455,26 @@ def rmtree(self, abs_path): if obj['name'].startswith(abs_path): self.swift_conn.delete_object(self.container_name, obj['name']) + @prepend_name_prefix + def get_accessed_time(self, name): + # Swift does not get this info, left NotImplemented + raise NotImplementedError('subclasses of Storage must provide a get_accessed_time() method') + @prepend_name_prefix + def get_created_time(self, name): + logger.debug("get_reated_time") + return datetime.fromtimestamp( + float(self.get_headers(name)['x-timestamp'])) + + @prepend_name_prefix + def get_modified_time(self, name): + # Handle case when a cache ready + if(self.file_cache_enabled): + date_timezone = self.file_cache[name]['last_modified'] + else: + data_str = self.get_headers(name)['last-modified'] + date_timezone = parsedate_to_datetime(data_str) + return date_timezone class StaticSwiftStorage(SwiftStorage): container_name = setting('SWIFT_STATIC_CONTAINER_NAME', '')