python从fd直接读入bytearray

我打算使用bytearray和memoryview建议bytearray(mmap),array.fromfile,甚至是自制的os.read(),但FileIO.readinto正在快速尖叫.

import os
import mmap,io,array
import timeit

fn = 'path_to_largish_file'

def fd_readinto_mmap(fd,ba):
    m = mmap.mmap(fd,access=mmap.ACCESS_READ)
    ba.extend(m)
    m.close()

def fd_readinto_fio(fd,ba):
    with io.FileIO(fd,closefd = False) as fio:
    fio.readinto(ba)

def fd_readinto_array(fd,ba):
    ar = array.array('c')
    sz = os.fstat(fd).st_size
    fp = os.fdopen(fd,'rb')
    ar.fromfile(fp,sz)
    ba.extend(ar)

def fd_readinto_mv(fd,ba):
    stat = os.fstat(fd)
    blksize = getattr(stat,'st_blksize',4096)
    bufsize = stat.st_size
    buf = bytearray(bufsize)
    m = memoryview(buf)
    while True:
        b = os.read(fd,blksize)
        s = len(b)
        if not s: break
        m[:s],m = b,m[s:]
    writtenbytes = buffer(buf,bufsize-len(m))
    ba.extend(writtenbytes)

setup = """
from __main__ import fn,fd_readinto_mmap,fd_readinto_fio,fd_readinto_array,fd_readinto_mv,reset_fd
import os
openfd = lambda : os.open(fn,os.O_RDONLY)
closefd = lambda fd: os.close(fd)
"""


reps = 2
tests = {
    'fio' : "fd=openfd(); fd_readinto_fio(fd,bytearray()); closefd(fd)",'mmap': "fd=openfd(); fd_readinto_mmap(fd,'array': "fd=openfd(); fd_readinto_array(fd,bytearray());",'mv' : "fd=openfd(); fd_readinto_mv(fd,}

width = max(map(len,tests))
for n,t in tests.iteritems():
    time = timeit.timeit(t,setup,number=reps)
    print ("{:%s} {}" % width).format(n,time)

在我的系统上,FileIO的速度要快几个数量级. FileIO.readinto在文件大小方面似乎也几乎是恒定的时间. (不确定这是怎么回事.)

mmap  4.86922478676
array 4.19783091545
mv    7.75051403046
fio   9.29832458496e-05