|
1 | 1 | import functools |
2 | 2 | import pickle |
3 | | -import time |
4 | | -from multiprocessing import Manager, Pool |
| 3 | +from multiprocessing import Manager |
5 | 4 |
|
6 | 5 |
|
7 | 6 | class SharedLRUCache: |
@@ -57,253 +56,3 @@ def wrapper(*args, **kwargs): |
57 | 56 |
|
58 | 57 | def shared_lru_cache(maxsize=128): |
59 | 58 | return SharedLRUCache(maxsize) |
60 | | - |
61 | | - |
62 | | -def test_shared_lru_cache_basic(): |
63 | | - @shared_lru_cache(maxsize=2) |
64 | | - def func(x): |
65 | | - return x * 2 |
66 | | - |
67 | | - assert func(2) == 4 |
68 | | - assert func(3) == 6 |
69 | | - assert func(2) == 4 # Should be cached |
70 | | - |
71 | | - |
72 | | -def test_shared_lru_cache_maxsize(): |
73 | | - @shared_lru_cache(maxsize=2) |
74 | | - def func(x): |
75 | | - return x * 2 |
76 | | - |
77 | | - func(1) |
78 | | - func(2) |
79 | | - func(3) |
80 | | - |
81 | | - # 1 should have been evicted |
82 | | - assert len(func.cache) == 2 |
83 | | - assert "((1,), frozenset())" not in func.cache |
84 | | - |
85 | | - |
86 | | -def worker(x): |
87 | | - @shared_lru_cache(maxsize=10) |
88 | | - def func(x): |
89 | | - return x * 2 |
90 | | - |
91 | | - return func(x) |
92 | | - |
93 | | - |
94 | | -def test_shared_lru_cache_large_data(): |
95 | | - @shared_lru_cache(maxsize=2) |
96 | | - def func(x): |
97 | | - return b"0" * (10**6) # 1MB of data |
98 | | - |
99 | | - data1 = func(1) |
100 | | - data2 = func(2) |
101 | | - |
102 | | - assert len(data1) == 10**6 |
103 | | - assert len(data2) == 10**6 |
104 | | - assert len(func.cache) == 2 |
105 | | - |
106 | | - |
107 | | -shared_cache = shared_lru_cache(maxsize=10) |
108 | | - |
109 | | - |
110 | | -def worker(x): |
111 | | - @shared_cache |
112 | | - def func(x): |
113 | | - return x * 2 |
114 | | - |
115 | | - return func(x) |
116 | | - |
117 | | - |
118 | | -def test_shared_lru_cache_multiprocessing(): |
119 | | - with Pool(processes=4) as pool: |
120 | | - results = pool.map(worker, range(20)) |
121 | | - assert results == [x * 2 for x in range(20)] |
122 | | - |
123 | | - |
124 | | -def test_shared_lru_cache_different_arg_types(): |
125 | | - @shared_lru_cache(maxsize=5) |
126 | | - def func(x): |
127 | | - return str(x) |
128 | | - |
129 | | - assert func(1) == "1" |
130 | | - assert func("a") == "a" |
131 | | - assert func(2.5) == "2.5" |
132 | | - assert func((1, 2)) == "(1, 2)" |
133 | | - assert func({"a": 1}) == "{'a': 1}" |
134 | | - |
135 | | - |
136 | | -def test_shared_lru_cache_with_kwargs(): |
137 | | - @shared_lru_cache(maxsize=3) |
138 | | - def func(x, y=10): |
139 | | - return x + y |
140 | | - |
141 | | - assert func(1) == 11 |
142 | | - assert func(2, y=20) == 22 |
143 | | - assert func(3, y=30) == 33 |
144 | | - assert func(1) == 11 # Should be cached |
145 | | - assert len(func.cache) == 3 |
146 | | - |
147 | | - |
148 | | -def test_shared_lru_cache_eviction_order(): |
149 | | - @shared_lru_cache(maxsize=3) |
150 | | - def func(x): |
151 | | - return x * 2 |
152 | | - |
153 | | - func(1) |
154 | | - func(2) |
155 | | - func(3) |
156 | | - func(4) # This should evict 1 |
157 | | - |
158 | | - assert "((1,), frozenset())" not in func.cache |
159 | | - assert "((2,), frozenset())" in func.cache |
160 | | - assert "((3,), frozenset())" in func.cache |
161 | | - assert "((4,), frozenset())" in func.cache |
162 | | - |
163 | | - |
164 | | -def fibonacci(n): |
165 | | - if n < 2: |
166 | | - return n |
167 | | - return fibonacci(n - 1) + fibonacci(n - 2) |
168 | | - |
169 | | - |
170 | | -cached_fibonacci = shared_lru_cache(maxsize=100)(fibonacci) |
171 | | - |
172 | | - |
173 | | -def test_shared_lru_cache_performance(): |
174 | | - start_time = time.time() |
175 | | - result = cached_fibonacci(30) # Increased from 4 to 30 for a more meaningful test |
176 | | - end_time = time.time() |
177 | | - |
178 | | - assert result == 832040 # The 30th Fibonacci number |
179 | | - assert end_time - start_time < 1 # Should be very fast due to caching |
180 | | - |
181 | | - |
182 | | -def test_shared_lru_cache_vs_functools_lru_cache(): |
183 | | - import functools |
184 | | - import time |
185 | | - |
186 | | - # Define functions with shared_lru_cache and functools.lru_cache |
187 | | - shared_fib = shared_lru_cache(maxsize=100)(fibonacci) |
188 | | - |
189 | | - functools_fib = functools.lru_cache(maxsize=100)(fibonacci) |
190 | | - |
191 | | - # Test performance for a smaller Fibonacci number |
192 | | - n = 32 |
193 | | - |
194 | | - # Measure time for shared_lru_cache |
195 | | - start_time = time.time() |
196 | | - shared_result = shared_fib(n) |
197 | | - shared_time = time.time() - start_time |
198 | | - |
199 | | - # Measure time for functools.lru_cache |
200 | | - start_time = time.time() |
201 | | - functools_result = functools_fib(n) |
202 | | - functools_time = time.time() - start_time |
203 | | - |
204 | | - # Assert that both functions return the same result |
205 | | - assert shared_result == functools_result |
206 | | - |
207 | | - # Print performance comparison |
208 | | - print(f"shared_lru_cache time: {shared_time:.6f} seconds") |
209 | | - print(f"functools.lru_cache time: {functools_time:.6f} seconds") |
210 | | - |
211 | | - # Assert that the performance difference is within an acceptable range |
212 | | - # This is a flexible assertion as performance can vary between runs |
213 | | - assert ( |
214 | | - abs(shared_time - functools_time) < 0.1 |
215 | | - ), "Performance difference is too large" |
216 | | - |
217 | | - |
218 | | -# Simulate loading an image from disk |
219 | | -def load_image(filename): |
220 | | - import numpy as np |
221 | | - |
222 | | - time.sleep(0.2) # Simulate I/O delay |
223 | | - return np.random.rand(2, 2) # Return a random 1000x1000 array |
224 | | - |
225 | | - |
226 | | -shared_load = shared_lru_cache(maxsize=100)(load_image) |
227 | | -functools_load = functools.lru_cache(maxsize=100)(load_image) |
228 | | - |
229 | | - |
230 | | -# Helper function for multiprocessing |
231 | | -def worker_shared(filename): |
232 | | - return shared_load(filename) |
233 | | - |
234 | | - |
235 | | -def worker_functools(filename): |
236 | | - return functools_load(filename) |
237 | | - |
238 | | - |
239 | | -def test_shared_lru_cache_vs_lru_cache_multiprocessing_all_miss(): |
240 | | - import time |
241 | | - from multiprocessing import Pool |
242 | | - |
243 | | - # Define functions with shared_lru_cache and functools.lru_cache |
244 | | - # Test parameters |
245 | | - n_workers = 4 |
246 | | - n_images = 20 |
247 | | - filenames = [f"image_{i}.jpg" for i in range(n_images)] |
248 | | - |
249 | | - # Test shared_lru_cache |
250 | | - start_time = time.time() |
251 | | - with Pool(n_workers) as pool: |
252 | | - shared_results = pool.map(worker_shared, filenames) |
253 | | - shared_time = time.time() - start_time |
254 | | - |
255 | | - # Test functools.lru_cache |
256 | | - start_time = time.time() |
257 | | - with Pool(n_workers) as pool: |
258 | | - functools_results = pool.map(worker_functools, filenames) |
259 | | - functools_time = time.time() - start_time |
260 | | - |
261 | | - # Assert that both methods return the same number of results |
262 | | - assert len(shared_results) == len(functools_results) == n_images |
263 | | - |
264 | | - # Print performance comparison |
265 | | - print(f"shared_lru_cache time: {shared_time:.6f} seconds") |
266 | | - print(f"functools.lru_cache time: {functools_time:.6f} seconds") |
267 | | - |
268 | | - # Assert that shared_lru_cache is faster |
269 | | - assert ( |
270 | | - shared_time < functools_time * 1.5 |
271 | | - ), "shared_lru_cache should be faster in multiprocessing scenario" |
272 | | - |
273 | | - |
274 | | -def test_shared_lru_cache_vs_lru_cache_multiprocessing_with_hits(): |
275 | | - import time |
276 | | - from multiprocessing import Pool |
277 | | - |
278 | | - # Test parameters |
279 | | - n_workers = 4 |
280 | | - n_images = 20 |
281 | | - n_repeats = 3 # Number of times to repeat the process to ensure cache hits |
282 | | - filenames = [f"image_{i}.jpg" for i in range(n_images)] |
283 | | - |
284 | | - def run_test(worker_func): |
285 | | - total_time = 0 |
286 | | - for _ in range(n_repeats): |
287 | | - start_time = time.time() |
288 | | - with Pool(n_workers) as pool: |
289 | | - results = pool.map(worker_func, filenames) |
290 | | - total_time += time.time() - start_time |
291 | | - return total_time / n_repeats, results |
292 | | - |
293 | | - # Test shared_lru_cache |
294 | | - shared_time, shared_results = run_test(worker_shared) |
295 | | - |
296 | | - # Test functools.lru_cache |
297 | | - functools_time, functools_results = run_test(worker_functools) |
298 | | - |
299 | | - # Assert that both methods return the same number of results |
300 | | - assert len(shared_results) == len(functools_results) == n_images |
301 | | - |
302 | | - # Print performance comparison |
303 | | - print(f"shared_lru_cache average time: {shared_time:.6f} seconds") |
304 | | - print(f"functools.lru_cache average time: {functools_time:.6f} seconds") |
305 | | - |
306 | | - # Assert that shared_lru_cache is faster |
307 | | - assert ( |
308 | | - shared_time < functools_time |
309 | | - ), "shared_lru_cache should be faster in multiprocessing scenario with cache hits" |
0 commit comments