通过 Python 客户端将大文件流式传输到 CherryPy

Question

我想将一个大文件从 python 客户端发布到cherrypy。我正在使用请求库。

这是我的客户端代码：

def upload(fileName=None):
    url = 'http://localhost:8080/upload'
    files = {'myFile': ( fileName, open(fileName, 'rb') )}
    r = requests.post(url, files=files)

#with open(fileName,'rb') as payload:
    #headers = {'content-type': 'multipart/form-data'}
    #r = requests.post('http://127.0.0.1:8080', data=payload,verify=False,headers=headers)

if __name__ == '__main__':
    upload(sys.argv[1])

问题在于这会将整个文件放入 RAM 内存中。有什么办法可以分片发布文件吗？

class FileDemo(object):


@cherrypy.expose
def upload(self, myFile):

    print myFile.filename
    #size = 0
    #decoder = MultipartDecoder(myFile, 'image/jpeg')
    #for part in decoder.parts:
        #print(part.header['content-type'])

    #while True:

        #advances to the content that hasn't been read
        #myFile.file.seek(size, 0)

        #reads 100mb at a time so it doesn't fill up the RAM
        #data = myFile.file.read(10240000)

        #newFile = open("/home/ivo/Desktop/"+str(myFile.filename), 'a+')
        #newFile.write(data)
        #newFile.close

        #size += len(data)

        #if len(data) < 10240000:
            #break
if __name__ == '__main__':
    cherrypy.quickstart(FileDemo())

这是服务器端的代码。它有很多 cmets，因为我一直在尝试很多东西。现在我只是打印文件名，客户端仍然将整个文件传输到 RAM。

我不知道还能尝试什么。提前感谢您的帮助。

Answer 1

如果是 CherryPy 特定的上传，您可以跳过 multipart/form-data 编码障碍，只需发送文件内容的流式 POST 正文。

客户

#!/usr/bin/env python
# -*- coding: utf-8 -*-


import urllib2
import io
import os


class FileLenIO(io.FileIO):

  def __init__(self, name, mode = 'r', closefd = True):
    io.FileIO.__init__(self, name, mode, closefd)

    self.__size = statinfo = os.stat(name).st_size

  def __len__(self):
    return self.__size


f = FileLenIO('/home/user/Videos/video.mp4', 'rb')
request = urllib2.Request('http://127.0.0.1:8080/upload', f)
request.add_header('Content-Type', 'application/octet-stream')
# you can add custom header with filename if you need it
response = urllib2.urlopen(request)

print response.read()

服务器

#!/usr/bin/env python
# -*- coding: utf-8 -*-


import os
import tempfile
import shutil

import cherrypy


config = {
  'global' : {
    'server.socket_host' : '127.0.0.1',
    'server.socket_port' : 8080,
    'server.thread_pool' : 8,
    # remove any limit on the request body size; cherrypy's default is 100MB
    'server.max_request_body_size' : 0,
    # increase server socket timeout to 60s; cherrypy's defult is 10s
    'server.socket_timeout' : 60
  }
}


class App:

  @cherrypy.config(**{'response.timeout': 3600}) # default is 300s
  @cherrypy.expose()
  def upload(self):
    '''Handle non-multipart upload'''

    destination = os.path.join('/home/user/test-upload')                
    with open(destination, 'wb') as f:
      shutil.copyfileobj(cherrypy.request.body, f)

    return 'Okay'


if __name__ == '__main__':
  cherrypy.quickstart(App(), '/', config)

在 1.3GiB 视频文件上测试。服务器端内存消耗在 10MiB 以下，客户端在 5MiB 以下。

Answer 2

这就是我解决问题的方法：

客户

import poster
def upload(fileName=None):

    register_openers()
    url = 'http://localhost:8080/upload'
    data, headers = multipart_encode({"myFile": open(fileName, "rb")})

    request = urllib2.Request(url, data, headers)
    request.unverifiable = True
    response = urllib2.urlopen(request)
    the_page = response.read()


if __name__ == '__main__':
    upload(sys.argv[1])

服务器

@cherrypy.expose
def upload(self, myFile):

    cherrypy.response.timeout = 3600
    newFile = open("/home/ivo/Desktop/"+str(myFile.filename), 'a+')
    newFile.write(myFile.file.read())
    newFile.close