0

我有一些未重构的代码，如果在单个块中请求下载文件，我使用请求模块下载，但由于我无法弄清楚如果请求拆分为范围（irange in这种情况下）使用多个线程如何使用请求模块来实现相同的。

def _grabAndWriteToDisk(url, saveTo, first=None, queue=None, mode='wb', irange=None):
    """ Function to download file using requests when not multipart range,
        else uses urllib2 when multi threaded multi part file download requests.

        Args:
            url(str): url of file to download
            saveTo(str): path where to save file
            first(int): starting byte of the range
            queue(Queue.Queue): queue object to set status for file download
            mode(str): mode of file to be downloaded
            irange(str): range of byte to download

    """
    fileName = url.split('/')[-1]
    filePath = os.path.join(saveTo, fileName)
    fileSize = int(_fdUtils.getUrlSizeInBytes(url))
    downloadedFileSize = 0 if not first else first
    block_sz = 8192

    def statusSet(que, dlfs, fileSize, fName, savTo):
        STOP_REQUEST.set()
        if que:
            que.task_done()
        _log.info("Download Completed %s%% for file %s, saved to %s",
            dlfs * 100. / fileSize, fName, savTo)

    if not irange:
        resp = requests.get(url, stream=True)
        for fileBuffer in resp.iter_content(block_sz):
            if not fileBuffer:
                break

            with open(filePath, mode) as fd:
                downloadedFileSize += len(fileBuffer)
                fd.write(fileBuffer)
                status = r"%10d  [%3.2f%%]" % (downloadedFileSize, downloadedFileSize * 100. / fileSize)
                status = status + chr(8)*(len(status)+1)
                sys.stdout.write('%s\r' % status)
                time.sleep(.05)
                sys.stdout.flush()
                if downloadedFileSize == fileSize:
                    statusSet(queue, downloadedFileSize, fileSize, fileName, saveTo)

    else:
        req = urllib2.Request(url)
        req.headers['Range'] = 'bytes=%s' % irange

        urlFh = urllib2.urlopen(req)
        with open(filePath, mode) as fh:
            while not STOP_REQUEST.isSet():
                fileBuffer = urlFh.read(block_sz)
                if not fileBuffer:
                    break
                downloadedFileSize += len(fileBuffer)
                fh.write(fileBuffer)

                status = r"%10d  [%3.2f%%]" % (downloadedFileSize, downloadedFileSize * 100. / fileSize)
                status = status + chr(8)*(len(status)+1)
                sys.stdout.write('%s\r' % status)
                time.sleep(.05)
                sys.stdout.flush()
                if downloadedFileSize == fileSize:
                    statusSet(queue, downloadedFileSize, fileSize, fileName, saveTo)

如何像使用 urllib2 一样重用 requests 模块来拆分范围标头？

4

0 回答 0