Sparse data structures in python (scipy.sparse + Pandas)

# -*- coding: utf-8 -*-
"""
Created on Sat Jan 19 12:37:22 2018
@author: Wagner Cipriano


Sparse data structures in python
scipy.sparse + Pandas
"""

#Imports:
from StringIO import StringIO
import pandas as pd
from scipy.sparse import csr_matrix

#flight table data:
TESTDATA = StringIO("""ori des voo
0 3 1
1 2 1
1 4 1
2 3 1
""")

#Split file reading into chunks
chunksize = 2  #1000000
chunks = pd.read_csv( TESTDATA, sep=" ", chunksize=chunksize)
#Concat chunks in dataframe
df = pd.concat( chunk.to_sparse(fill_value=0) for chunk in chunks )
ME = csr_matrix((list(df['voo']), (list(df['ori']), list(df['des']))), shape=[5,5])

#Symmetrization of scipy sparse matrices
rows, cols = ME.nonzero()
ME[cols, rows] = ME[rows, cols]

#Results:
print ME
print ME.todense()