Downloading files with consecutive file name in URL

Posted on

This program is a template for downloading files with consecutive file name in URL.

Usage

Specify the

  1. baseURL and extension
  2. ThreadPool number
  3. Starting and ending index of files

Program

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
from multiprocessing.pool import ThreadPool
from urllib import request

ThreadPool_number = 10
baseURL = "https://www.abc.com/"
extension = "jpg"
starting_index = 1
ending_index = 10

def sendRequest(url, name):
    opener = request.build_opener()
    opener.addheaders = [('User-agent', 'Mozilla/5.0')]
    request.install_opener(opener)
    request.urlretrieve(url, name)

def combineURLAndFileName(i):
    count = str(i).zfill(4)
    url = baseURL + count + "." + extension
    name = count + "." + extension
    return url, name
    
def process(i):
    url, name = combineURLAndFileName(i)
    sendRequest(url, name)

if __name__ == '__main__':
    p = ThreadPool(ThreadPool_number)
    p.map(process, range(starting_index, ending_index))