python – 在numpy / scipy中为稀疏的矩阵添加一个非常重复的矩

>>> a = sps.csr_matrix([1,2,3,4])
>>> a.data
array([1,4])
>>> a.indices
array([0,1,3])
>>> a.indptr
array([0,4])

>>> b = sps.csr_matrix((np.array([1,4,5]),...                     np.array([0,0]),5])),shape=(1,4))
>>> b
<1x4 sparse matrix of type '<type 'numpy.int32'>'
    with 5 stored elements in Compressed Sparse Row format>
>>> b.todense()
matrix([[6,4]])

因此,您甚至不必在训练向量和观察矩阵的每一行之间寻找巧合来添加它们：只需用正确的指针填充所有数据,并且需要求和的东西将得到求和何时访问数据.

编辑

鉴于第一个代码的速度很慢,您可以按如下方式将内存换成速度：

def csr_add_sparse_vec(sps_mat,sps_vec) :
    """Adds a sparse vector to every row of a sparse matrix"""
    # No checks done,but both arguments should be sparse matrices in CSR
    # format,both should have the same number of columns,and the vector
    # should be a vector and have only one row.

    rows,cols = sps_mat.shape
    nnz_vec = len(sps_vec.data)
    nnz_per_row = np.diff(sps_mat.indptr)
    longest_row = np.max(nnz_per_row)

    old_data = np.zeros((rows * longest_row,),dtype=sps_mat.data.dtype)
    old_cols = np.zeros((rows * longest_row,dtype=sps_mat.indices.dtype)

    data_idx = np.arange(longest_row) < nnz_per_row[:,None]
    data_idx = data_idx.reshape(-1)
    old_data[data_idx] = sps_mat.data
    old_cols[data_idx] = sps_mat.indices
    old_data = old_data.reshape(rows,-1)
    old_cols = old_cols.reshape(rows,-1)

    new_data = np.zeros((rows,longest_row + nnz_vec,dtype=sps_mat.data.dtype)
    new_data[:,:longest_row] = old_data
    del old_data
    new_cols = np.zeros((rows,dtype=sps_mat.indices.dtype)
    new_cols[:,:longest_row] = old_cols
    del old_cols
    new_data[:,longest_row:] = sps_vec.data
    new_cols[:,longest_row:] = sps_vec.indices
    new_data = new_data.reshape(-1)
    new_cols = new_cols.reshape(-1)
    new_pointer = np.arange(0,(rows + 1) * (longest_row + nnz_vec),longest_row + nnz_vec)

    ret = sps.csr_matrix((new_data,new_cols,new_pointer),shape=sps_mat.shape)
    ret.eliminate_zeros()

    return ret

它没有以前那么快,但它可以在大约1秒内完成10,000行：

In [2]: a
Out[2]: 
<10000x65536 sparse matrix of type '<type 'numpy.float64'>'
    with 15000000 stored elements in Compressed Sparse Row format>

In [3]: b
Out[3]: 
<1x65536 sparse matrix of type '<type 'numpy.float64'>'
    with 1500 stored elements in Compressed Sparse Row format>

In [4]: csr_add_sparse_vec(a,b)
Out[4]: 
<10000x65536 sparse matrix of type '<type 'numpy.float64'>'
    with 30000000 stored elements in Compressed Sparse Row format>

In [5]: %timeit csr_add_sparse_vec(a,b)
1 loops,best of 3: 956 ms per loop

编辑此代码非常非常慢

def csr_add_sparse_vec(sps_mat,cols = sps_mat.shape

    new_data = sps_mat.data
    new_pointer = sps_mat.indptr.copy()
    new_cols = sps_mat.indices

    aux_idx = np.arange(rows + 1)

    for value,col in itertools.izip(sps_vec.data,sps_vec.indices) :
        new_data = np.insert(new_data,new_pointer[1:],[value] * rows)
        new_cols = np.insert(new_cols,[col] * rows)
        new_pointer += aux_idx

    return sps.csr_matrix((new_data,shape=sps_mat.shape)