Home 0P3N Blog Tutorial: Multiple Threads Downloader Source Code
Ready to Start Your Career?
Create Free Account
Bmsr256 s profile image
By: Bmsr256
August 18, 2017

Tutorial: Multiple Threads Downloader Source Code

By: Bmsr256
August 18, 2017
Bmsr256 s profile image
By: Bmsr256
August 18, 2017
This time I will present you here a source code made in Python which you can download a file with multiple threads option. This source code is for educational purposes and I hope is helpful to you. If you find any bugs in this code or have questions, leave a message in the comments section.
`#!/usr/bin/env python#coding=utf-8import osimport sysimport timeimport urllibimport urllib2import threading############################################################################### self-defined exception classes##############################################################################class ConnectionError(Exception): passclass URLUnreachable(Exception):passclass CanotDownload(Exception):pass############################################################################### multiple threads download module starts here##############################################################################class HttpGetThread(threading.Thread):def __init__(self, name, url, filename, range=0):threading.Thread.__init__(self, )self.url = urlself.filename = filenameself.range = rangeself.totalLength = range[1] - range[0] +1try:self.downloaded = os.path.getsize(self.filename)except OSError:self.downloaded = 0self.percent = self.downloaded/float(self.totalLength)*100self.headerrange = (self.range[0]+self.downloaded, self.range[1])self.bufferSize = 8192def run(self):try:self.downloaded = os.path.getsize(self.filename)except OSError:self.downloaded = 0self.percent = self.downloaded/float(self.totalLength)*100#self.headerrange = (self.range[0]+self.downloaded, self.range[1])self.bufferSize = 8192#request = urllib2.Request(self.url)#request.add_header('Range', 'bytes=%d-%d' %self.headerrange)downloadAll = Falseretries = 1while not downloadAll:if retries > 10:breaktry:self.headerrange = (self.range[0]+self.downloaded, self.range[1])request = urllib2.Request(self.url)request.add_header('Range', 'bytes=%d-%d' %self.headerrange)conn = urllib2.urlopen(request)startTime = time.time()data = conn.read(self.bufferSize)while data:f = open(self.filename, 'ab')f.write(data)f.close()self.time = int(time.time() - startTime)self.downloaded += len(data)self.percent = self.downloaded/float(self.totalLength) *100data = conn.read(self.bufferSize)downloadAll = Trueexcept Exception, err:retries += 1time.sleep(1)continuedef Split(size,blocks):ranges = []blocksize = size / blocksfor i in xrange(blocks-1):ranges.append(( i*blocksize, i*blocksize+blocksize-1))ranges.append(( blocksize*(blocks-1), size-1))return rangesdef GetHttpFileSize(url):length = 0try:conn = urllib.urlopen(url)headers = conn.info().headersfor header in headers:if header.find('Length') != -1:length = header.split(':')[-1].strip()length = int(length)except Exception, err:passreturn lengthdef hasLive(ts):for t in ts:if t.isAlive():return Truereturn Falsedef MyHttpGet(url, output=None, connections=4):"""arguments:url, in GBK encodingoutput, default encoding, do no convertionconnections, integer"""length = GetHttpFileSize(url)print lengthmb = length/1024/1024.0if length == 0:raise URLUnreachableblocks = connectionsif output:filename = outputelse:output = url.split('/')[-1]ranges = Split(length, blocks)names = ["%s_%d" %(output,i) for i in xrange(blocks)]ts = []for i in xrange(blocks):t = HttpGetThread(i, url, names[i], ranges[i])t.setDaemon(True)t.start()ts.append(t)live = hasLive(ts)startSize = sum([t.downloaded for t in ts])startTime = time.time()etime = 0while live:try:etime = time.time() - startTimed = sum([t.downloaded for t in ts])/float(length)*100downloadedThistime = sum([t.downloaded for t in ts])-startSizetry:rate = downloadedThistime / float(etime)/1024except:rate = 100.0progressStr = u'rFilesize: %d(%.2fM) Downloaded: %.2f%% Avg rate: %.1fKB/s' %(length, mb, d, rate)sys.stdout.write(progressStr)sys.stdout.flush()#sys.stdout.write('b'*(len(progressStr)+1))live = hasLive(ts)time.sleep(0.2)except KeyboardInterrupt:printprint "Exit..."for n in names:try:os.remove(n)except:passsys.exit(1)print#print u'used time: %d:%d, pingjunsudu:%.2fKB/s' %(int(etime)/60, int(etime)%60,rate)f = open(filename, 'wb')for n in names:f.write(open(n,'rb').read())try:os.remove(n)except:passf.close()if __name__ == '__main__':#This is an example of the get link, you can choose the link you prefer to use and edit the link hereMyHttpGet('https://dldir1.qq.com/qqfile/QQforMac/QQ_V3.1.1.dmg','my_download.file',4)`
Remember, put a message in the comments if you have some questions or opinions. I will be next time with more sample codes.
Schedule Demo

Build your Cybersecurity or IT Career

Accelerate in your role, earn new certifications, and develop cutting-edge skills using the fastest growing catalog in the industry