From 0760043d18ac000e290fe82c18db28ebc5c44ebd Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Wed, 25 Oct 2017 16:03:13 -0700 Subject: [PATCH] Add retry when download dataset (#5098) --- python/paddle/v2/dataset/common.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/python/paddle/v2/dataset/common.py b/python/paddle/v2/dataset/common.py index 053ae151c5..e31e501ce9 100644 --- a/python/paddle/v2/dataset/common.py +++ b/python/paddle/v2/dataset/common.py @@ -65,7 +65,14 @@ def download(url, module_name, md5sum): os.makedirs(dirname) filename = os.path.join(dirname, url.split('/')[-1]) - if not (os.path.exists(filename) and md5file(filename) == md5sum): + retry = 0 + retry_limit = 3 + while not (os.path.exists(filename) and md5file(filename) == md5sum): + if retry < retry_limit: + retry += 1 + else: + raise RuntimeError("Cannot download {0} within retry limit {2}". + format(url, retry_limit)) print "Cache file %s not found, downloading %s" % (filename, url) r = requests.get(url, stream=True) total_length = r.headers.get('content-length')