Download a large binary file in chunks with Python ¶

Using requests:

import requests

# https://stackoverflow.com/questions/16694907/download-large-file-in-python-with-requests#16696317
def download_file(url):
    local_filename = url.split('/')[-1]
    # NOTE the stream=True parameter below
    with requests.get(url, stream=True) as r:
        r.raise_for_status()
        with open(local_filename, 'wb') as f:
            for chunk in r.iter_content(chunk_size=8192):
                # If you have chunk encoded response uncomment if
                # and set chunk_size parameter to None.
                #if chunk:
                f.write(chunk)
    return local_filename

Using urlib.request:

import urllib.request

def download_file2(url):
    """
    Use urlib.request instead of requests to avoid "403 Forbidden" errors
    when the remote server tries to prevent scraping (for example when
    is protected behind cloudflare)
    """
    request = urllib.request.Request(url)
    request.add_header('User-Agent', 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:106.0) Gecko/20100101 Firefox/106.0')
    request.add_header('Accept', 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8')
    request.add_header('Accept-Language', 'en-US,en;q=0.5')

    response = urllib.request.urlopen(request)
    local_filename = url.split('/')[-1]
    chunk_size = 8192
    with open(local_filename, 'wb') as f:
        size = 0
        while True:
            info = response.read(chunk_size)
            if len(info) < 1:
                break
            size = size + len(info)
            f.write(info)
    return local_filename