From 0893f5f7c71b87a9b8803848665176dd931e4aaf Mon Sep 17 00:00:00 2001 From: Eryx-Z Date: Sat, 17 Oct 2026 09:34:06 +0800 Subject: [PATCH] Defer FanoutCache temp directory until first use When directory is omitted, postpone tempfile.mkdtemp and shard construction until the cache is first used (or entered as a context manager). Importing a module that constructs FanoutCache() at module level no longer creates temporary directories as a side effect. Fixes #379 --- diskcache/fanout.py | 92 ++++++++++++++++++++++++++++++++++++-------- tests/test_fanout.py | 42 ++++++++++++++++++++ 2 files changed, 119 insertions(+), 15 deletions(-) diff --git a/diskcache/fanout.py b/diskcache/fanout.py index 9822ee4..740db9e 100644 --- a/diskcache/fanout.py +++ b/diskcache/fanout.py @@ -27,43 +27,72 @@ def __init__( :param disk: `Disk` instance for serialization :param settings: any of `DEFAULT_SETTINGS` - """ - if directory is None: - directory = tempfile.mkdtemp(prefix='diskcache-') - directory = str(directory) - directory = op.expanduser(directory) - directory = op.expandvars(directory) + When `directory` is omitted, the temporary cache directory and shard + databases are created on first use so importing a module that + constructs a module-level `FanoutCache()` has no filesystem side + effect. + """ default_size_limit = DEFAULT_SETTINGS['size_limit'] size_limit = settings.pop('size_limit', default_size_limit) / shards self._count = shards - self._directory = directory self._disk = disk + self._timeout = timeout + self._size_limit = size_limit + self._settings = settings + self._shards = None + self._hash = None + self._caches = {} + self._deques = {} + self._indexes = {} + + if directory is None: + self._directory = None + else: + self._directory = self._normalize_directory(directory) + self._init_shards() + + @staticmethod + def _normalize_directory(directory): + directory = str(directory) + directory = op.expanduser(directory) + directory = op.expandvars(directory) + return directory + + def _ensure_ready(self): + """Create temp directory and shards on first use when deferred.""" + if self._shards is not None: + return + if self._directory is None: + self._directory = tempfile.mkdtemp(prefix='diskcache-') + self._init_shards() + + def _init_shards(self): + directory = self._directory self._shards = tuple( Cache( directory=op.join(directory, '%03d' % num), - timeout=timeout, - disk=disk, - size_limit=size_limit, - **settings, + timeout=self._timeout, + disk=self._disk, + size_limit=self._size_limit, + **self._settings, ) - for num in range(shards) + for num in range(self._count) ) self._hash = self._shards[0].disk.hash - self._caches = {} - self._deques = {} - self._indexes = {} @property def directory(self): """Cache directory.""" + self._ensure_ready() return self._directory def __getattr__(self, name): safe_names = {'timeout', 'disk'} valid_name = name in DEFAULT_SETTINGS or name in safe_names assert valid_name, 'cannot access {} in cache shard'.format(name) + self._ensure_ready() return getattr(self._shards[0], name) @cl.contextmanager @@ -94,6 +123,7 @@ def transact(self, retry=True): """ assert retry, 'retry must be True in FanoutCache' with cl.ExitStack() as stack: + self._ensure_ready() for shard in self._shards: shard_transaction = shard.transact(retry=True) stack.enter_context(shard_transaction) @@ -118,6 +148,7 @@ def set(self, key, value, expire=None, read=False, tag=None, retry=False): :return: True if item was set """ + self._ensure_ready() index = self._hash(key) % self._count shard = self._shards[index] try: @@ -134,6 +165,7 @@ def __setitem__(self, key, value): :param value: value for item """ + self._ensure_ready() index = self._hash(key) % self._count shard = self._shards[index] shard[key] = value @@ -151,6 +183,7 @@ def touch(self, key, expire=None, retry=False): :return: True if key was touched """ + self._ensure_ready() index = self._hash(key) % self._count shard = self._shards[index] try: @@ -182,6 +215,7 @@ def add(self, key, value, expire=None, read=False, tag=None, retry=False): :return: True if item was added """ + self._ensure_ready() index = self._hash(key) % self._count shard = self._shards[index] try: @@ -213,6 +247,7 @@ def incr(self, key, delta=1, default=0, retry=False): :raises KeyError: if key is not found and default is None """ + self._ensure_ready() index = self._hash(key) % self._count shard = self._shards[index] try: @@ -247,6 +282,7 @@ def decr(self, key, delta=1, default=0, retry=False): :raises KeyError: if key is not found and default is None """ + self._ensure_ready() index = self._hash(key) % self._count shard = self._shards[index] try: @@ -279,6 +315,7 @@ def get( :return: value for item if key is found else default """ + self._ensure_ready() index = self._hash(key) % self._count shard = self._shards[index] try: @@ -296,6 +333,7 @@ def __getitem__(self, key): :raises KeyError: if key is not found """ + self._ensure_ready() index = self._hash(key) % self._count shard = self._shards[index] return shard[key] @@ -320,6 +358,7 @@ def __contains__(self, key): :return: True if key is found """ + self._ensure_ready() index = self._hash(key) % self._count shard = self._shards[index] return key in shard @@ -345,6 +384,7 @@ def pop( :return: value for item if key is found else default """ + self._ensure_ready() index = self._hash(key) % self._count shard = self._shards[index] try: @@ -365,6 +405,7 @@ def delete(self, key, retry=False): :return: True if item was deleted """ + self._ensure_ready() index = self._hash(key) % self._count shard = self._shards[index] try: @@ -381,6 +422,7 @@ def __delitem__(self, key): :raises KeyError: if key is not found """ + self._ensure_ready() index = self._hash(key) % self._count shard = self._shards[index] del shard[key] @@ -405,6 +447,7 @@ def check(self, fix=False, retry=False): :raises Timeout: if database timeout occurs """ + self._ensure_ready() warnings = (shard.check(fix, retry) for shard in self._shards) return functools.reduce(operator.iadd, warnings, []) @@ -428,6 +471,7 @@ def create_tag_index(self): :raises Timeout: if database timeout occurs """ + self._ensure_ready() for shard in self._shards: shard.create_tag_index() @@ -437,6 +481,7 @@ def drop_tag_index(self): :raises Timeout: if database timeout occurs """ + self._ensure_ready() for shard in self._shards: shard.drop_tag_index() @@ -479,6 +524,7 @@ def clear(self, retry=False): def _remove(self, name, args=(), retry=False): total = 0 + self._ensure_ready() for shard in self._shards: method = getattr(shard, name) while True: @@ -499,6 +545,7 @@ def stats(self, enable=True, reset=False): :return: (hits, misses) """ + self._ensure_ready() results = [shard.stats(enable, reset) for shard in self._shards] total_hits = sum(hits for hits, _ in results) total_misses = sum(misses for _, misses in results) @@ -510,10 +557,16 @@ def volume(self): :return: size in bytes """ + self._ensure_ready() return sum(shard.volume() for shard in self._shards) def close(self): """Close database connection.""" + if self._shards is None: + self._caches.clear() + self._deques.clear() + self._indexes.clear() + return for shard in self._shards: shard.close() self._caches.clear() @@ -521,12 +574,14 @@ def close(self): self._indexes.clear() def __enter__(self): + self._ensure_ready() return self def __exit__(self, *exception): self.close() def __getstate__(self): + self._ensure_ready() return (self._directory, self._count, self.timeout, type(self.disk)) def __setstate__(self, state): @@ -534,16 +589,19 @@ def __setstate__(self, state): def __iter__(self): """Iterate keys in cache including expired items.""" + self._ensure_ready() iterators = (iter(shard) for shard in self._shards) return it.chain.from_iterable(iterators) def __reversed__(self): """Reverse iterate keys in cache including expired items.""" + self._ensure_ready() iterators = (reversed(shard) for shard in reversed(self._shards)) return it.chain.from_iterable(iterators) def __len__(self): """Count of items in cache including expired items.""" + self._ensure_ready() return sum(len(shard) for shard in self._shards) def reset(self, key, value=ENOVAL): @@ -564,6 +622,7 @@ def reset(self, key, value=ENOVAL): :return: updated value for item """ + self._ensure_ready() for shard in self._shards: while True: try: @@ -602,6 +661,7 @@ def cache(self, name, timeout=60, disk=None, **settings): try: return _caches[name] except KeyError: + self._ensure_ready() parts = name.split('/') directory = op.join(self._directory, 'cache', *parts) temp = Cache( @@ -636,6 +696,7 @@ def deque(self, name, maxlen=None): try: return _deques[name] except KeyError: + self._ensure_ready() parts = name.split('/') directory = op.join(self._directory, 'deque', *parts) cache = Cache( @@ -672,6 +733,7 @@ def index(self, name): try: return _indexes[name] except KeyError: + self._ensure_ready() parts = name.split('/') directory = op.join(self._directory, 'index', *parts) cache = Cache( diff --git a/tests/test_fanout.py b/tests/test_fanout.py index af221b6..688e146 100644 --- a/tests/test_fanout.py +++ b/tests/test_fanout.py @@ -683,3 +683,45 @@ def test_custom_filename_disk(): assert content == str(count) * int(1e5) shutil.rmtree(cache.directory, ignore_errors=True) + + +def test_fanout_defers_tempdir_until_use(tmp_path, monkeypatch): + """FanoutCache() must not create a temp directory until first use (#379).""" + calls = [] + real_mkdtemp = tempfile.mkdtemp + + def tracking_mkdtemp(*args, **kwargs): + path = real_mkdtemp(*args, **kwargs) + calls.append(path) + return path + + monkeypatch.setattr(tempfile, 'mkdtemp', tracking_mkdtemp) + + cache = dc.FanoutCache() + assert calls == [] + assert cache._shards is None + assert cache._directory is None + + cache['key'] = 'value' + assert len(calls) == 1 + assert cache.directory == calls[0] + assert cache['key'] == 'value' + + cache.close() + shutil.rmtree(cache.directory, ignore_errors=True) + + +def test_fanout_close_before_use_is_noop(monkeypatch): + calls = [] + real_mkdtemp = tempfile.mkdtemp + + def tracking_mkdtemp(*args, **kwargs): + path = real_mkdtemp(*args, **kwargs) + calls.append(path) + return path + + monkeypatch.setattr(tempfile, 'mkdtemp', tracking_mkdtemp) + + cache = dc.FanoutCache() + cache.close() + assert calls == []