Sitelet https://github.com/keranem1/python-diskcache/commit/879a65a1932377d017cc187661fd880346459b0d
Skip to content

Commit 879a65a

Browse files
committed
Refactor file writing logic to retry makedirs
1 parent b86aa9e commit 879a65a

1 file changed

Lines changed: 30 additions & 36 deletions

File tree

‎diskcache/core.py‎

Lines changed: 30 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -187,7 +187,6 @@ def store(self, value, read, key=UNKNOWN):
187187
:return: (size, mode, filename, value) tuple for Cache table
188188
189189
"""
190-
# TODO: Retry mkdirs!!!
191190
# pylint: disable=unidiomatic-typecheck
192191
type_value = type(value)
193192
min_file_size = self.min_file_size
@@ -206,46 +205,18 @@ def store(self, value, read, key=UNKNOWN):
206205
return 0, MODE_RAW, None, sqlite3.Binary(value)
207206
else:
208207
filename, full_path = self.filename(key, value)
209-
full_dir, _ = op.split(full_path)
210-
211-
for count in range(11):
212-
with cl.suppress(OSError):
213-
os.makedirs(full_dir)
214-
215-
try:
216-
# Another cache may have deleted the directory before
217-
# the file could be opened.
218-
writer = open(full_path, 'xb')
219-
except OSError:
220-
if count == 10:
221-
# Give up after 10 tries to open the file.
222-
raise
223-
continue
224-
225-
with writer:
226-
writer.write(value)
227-
228-
break
229-
208+
self._write(full_path, io.BytesIO(value), 'xb')
230209
return len(value), MODE_BINARY, filename, None
231210
elif type_value is str:
232211
filename, full_path = self.filename(key, value)
233-
234-
with open(full_path, 'x', encoding='UTF-8') as writer:
235-
writer.write(value)
236-
212+
self._write(full_path, io.StringIO(value), 'x', 'UTF-8')
237213
size = op.getsize(full_path)
238214
return size, MODE_TEXT, filename, None
239215
elif read:
240-
size = 0
241216
reader = ft.partial(value.read, 2 ** 22)
242217
filename, full_path = self.filename(key, value)
243-
244-
with open(full_path, 'xb') as writer:
245-
for chunk in iter(reader, b''):
246-
size += len(chunk)
247-
writer.write(chunk)
248-
218+
iterator = iter(reader, b'')
219+
size = self._write(full_path, iterator, 'xb')
249220
return size, MODE_BINARY, filename, None
250221
else:
251222
result = pickle.dumps(value, protocol=self.pickle_protocol)
@@ -254,11 +225,34 @@ def store(self, value, read, key=UNKNOWN):
254225
return 0, MODE_PICKLE, None, sqlite3.Binary(result)
255226
else:
256227
filename, full_path = self.filename(key, value)
228+
self._write(full_path, io.BytesIO(result), 'xb')
229+
return len(result), MODE_PICKLE, filename, None
230+
231+
def _write(self, full_path, iterator, mode, encoding=None):
232+
full_dir, _ = op.split(full_path)
257233

258-
with open(full_path, 'xb') as writer:
259-
writer.write(result)
234+
for count in range(1, 11):
235+
with cl.suppress(OSError):
236+
os.makedirs(full_dir)
260237

261-
return len(result), MODE_PICKLE, filename, None
238+
try:
239+
# Another cache may have deleted the directory before
240+
# the file could be opened.
241+
writer = open(full_path, mode, encoding=encoding)
242+
except OSError:
243+
if count == 10:
244+
# Give up after 10 tries to open the file.
245+
raise
246+
continue
247+
248+
with writer:
249+
size = 0
250+
for chunk in iterator:
251+
size += len(chunk)
252+
writer.write(chunk)
253+
return size
254+
255+
break
262256

263257
def fetch(self, mode, filename, value, read):
264258
"""Convert fields `mode`, `filename`, and `value` from Cache table to

0 commit comments

Comments
 (0)