Skip to content

Commit e821096

Browse files
bhass-neonbhass-neon
committed
add handling for windows max path length error, in download_urls and download_file functions
1 parent 32f1304 commit e821096

File tree

1 file changed

+100
-74
lines changed

1 file changed

+100
-74
lines changed

‎src/neonutilities/helper_mods/api_helpers.py

+100-74
Original file line numberDiff line numberDiff line change
@@ -535,42 +535,57 @@ def download_urls(url_set,
535535

536536
for i in tqdm(range(0, len(url_set["z"])), disable=not progress):
537537

538-
try:
539-
if token is None:
540-
j = 0
541-
while j<3:
542-
try:
543-
with open(outpath+url_set["flnm"][i], "wb") as out_file:
544-
content = requests.get(url_set["z"][i], stream=True,
545-
headers={"accept": "application/json",
546-
"User-Agent": usera},
547-
timeout=(10, 120)).content
548-
out_file.write(content)
549-
j = j+5
550-
except Exception:
551-
logging.info(f"File {url_set['flnm'][i]} could not be downloaded. Re-attempting.")
552-
j = j+1
553-
time.sleep(5)
554-
else:
555-
j = 0
556-
while j<3:
557-
try:
558-
with open(outpath+url_set["flnm"][i], "wb") as out_file:
559-
content = requests.get(url_set["z"][i], stream=True,
560-
headers={"X-API-TOKEN": token,
561-
"accept": "application/json",
562-
"User-Agent": usera},
563-
timeout=(10, 120)).content
564-
out_file.write(content)
565-
j = j+5
566-
except Exception:
567-
logging.info(f"File {url_set['flnm'][i]} could not be downloaded. Re-attempting.")
568-
j = j+1
569-
time.sleep(5)
538+
if len(outpath+url_set["flnm"][i]) > 260 and platform.system() == "Windows":
539+
raise OSError(
540+
f'Filepath is {len(outpath+url_set["flnm"][i])} characters long. Filepaths on Windows are limited to 260 characters. Move your working directory closer to the root directory or enable long path support in Windows through the Registry Editor.')
541+
return
570542

571-
except Exception:
572-
logging.info(f"File {url_set['flnm'][i]} could not be downloaded and was skipped. If this issue persists, check your network connection and check the NEON Data Portal for outage alerts.")
573-
pass
543+
else:
544+
try:
545+
if token is None:
546+
j = 0
547+
while j < 3:
548+
try:
549+
with open(outpath+url_set["flnm"][i], "wb") as out_file:
550+
print('length outfile:', len(
551+
outpath+url_set["flnm"][i]))
552+
content = requests.get(url_set["z"][i], stream=True,
553+
headers={"accept": "application/json",
554+
"User-Agent": usera},
555+
timeout=(10, 120)).content
556+
out_file.write(content)
557+
j = j+5
558+
except Exception as e:
559+
logging.info(
560+
f"File {url_set['flnm'][i]} could not be downloaded. Re-attempting.")
561+
print(e)
562+
j = j+1
563+
time.sleep(5)
564+
else:
565+
j = 0
566+
while j < 3:
567+
try:
568+
with open(outpath+url_set["flnm"][i], "wb") as out_file:
569+
print('length outfile:', len(
570+
outpath+url_set["flnm"][i]))
571+
content = requests.get(url_set["z"][i], stream=True,
572+
headers={"X-API-TOKEN": token,
573+
"accept": "application/json",
574+
"User-Agent": usera},
575+
timeout=(10, 120)).content
576+
out_file.write(content)
577+
j = j+5
578+
except Exception as e:
579+
logging.info(
580+
f"File {url_set['flnm'][i]} could not be downloaded. Re-attempting.")
581+
print(e)
582+
j = j+1
583+
time.sleep(5)
584+
585+
except Exception:
586+
logging.info(
587+
f"File {url_set['flnm'][i]} could not be downloaded and was skipped. If this issue persists, check your network connection and check the NEON Data Portal for outage alerts.")
588+
pass
574589

575590
return None
576591

@@ -618,48 +633,59 @@ def download_file(url, savepath, chunk_size=1024, token=None):
618633
file_path = "/".join(pathparts[3:len(pathparts)])
619634

620635
file_fullpath = savepath + "/" + file_path
621-
os.makedirs(os.path.dirname(file_fullpath), exist_ok=True)
636+
file_fullpath_abs = os.path.abspath(file_fullpath) # get the absolute path
622637

623-
try:
624-
if token is None:
625-
j = 0
626-
while j<3:
627-
try:
628-
r = requests.get(url, stream=True,
629-
headers={"accept": "application/json",
630-
"User-Agent": usera},
631-
timeout=(10, 120))
632-
j = j+5
633-
except Exception:
634-
logging.info(f"File {os.path.basename(url)} could not be downloaded. Re-attempting.")
635-
j = j+1
636-
time.sleep(5)
637-
else:
638-
j = 0
639-
while j<3:
640-
try:
641-
r = requests.get(url, stream=True,
642-
headers={"X-API-TOKEN": token,
643-
"accept": "application/json",
644-
"User-Agent": usera},
645-
timeout=(10, 120))
646-
j = j+5
647-
except Exception:
648-
logging.info(f"File {os.path.basename(url)} could not be downloaded. Re-attempting.")
649-
j = j+1
650-
time.sleep(5)
651-
652-
with open(file_fullpath, 'wb') as f:
653-
for chunk in r.iter_content(chunk_size=chunk_size):
654-
if chunk:
655-
f.write(chunk)
656-
r.close()
638+
if len(file_fullpath_abs) > 260 and platform.system() == "Windows":
639+
raise OSError(
640+
f'Filepath is {len(file_fullpath_abs)} characters long. Filepaths on Windows are limited to 260 characters. Set the savepath to be closer to the root directory or enable long path support in Windows through the Registry Editor.')
641+
return
657642

658-
except Exception:
659-
logging.info(f"File {os.path.basename(url)} could not be downloaded and was skipped or partially downloaded. If this issue persists, check your network connection and check the NEON Data Portal for outage alerts.")
660-
pass
643+
else:
644+
os.makedirs(os.path.dirname(file_fullpath), exist_ok=True)
645+
646+
try:
647+
if token is None:
648+
j = 0
649+
while j < 3:
650+
try:
651+
r = requests.get(url, stream=True,
652+
headers={"accept": "application/json",
653+
"User-Agent": usera},
654+
timeout=(10, 120))
655+
j = j+5
656+
except Exception:
657+
logging.info(
658+
f"File {os.path.basename(url)} could not be downloaded. Re-attempting.")
659+
j = j+1
660+
time.sleep(5)
661+
else:
662+
j = 0
663+
while j < 3:
664+
try:
665+
r = requests.get(url, stream=True,
666+
headers={"X-API-TOKEN": token,
667+
"accept": "application/json",
668+
"User-Agent": usera},
669+
timeout=(10, 120))
670+
j = j+5
671+
except Exception:
672+
logging.info(
673+
f"File {os.path.basename(url)} could not be downloaded. Re-attempting.")
674+
j = j+1
675+
time.sleep(5)
676+
677+
with open(file_fullpath, 'wb') as f:
678+
for chunk in r.iter_content(chunk_size=chunk_size):
679+
if chunk:
680+
f.write(chunk)
681+
r.close()
682+
683+
except Exception as e:
684+
logging.info(f"File {os.path.basename(url)} could not be downloaded and was skipped or partially downloaded. If this issue persists, check your network connection and check the NEON Data Portal for outage alerts.")
685+
# print(e)
686+
pass
661687

662-
return
688+
return
663689

664690

665691
def readme_url(readmepath):

0 commit comments

Comments
 (0)