Scanpy: When I ran sce.pp.mnn_correct(), there was an IndexError

[tkisss]

-----------------------------------------------------
IndexError          Traceback (most recent call last)
<ipython-input-22-408c81d5d845> in <module>
      1 t1 = time.time()
----> 2 corrected = sce.pp.mnn_correct(scdata[scdata.obs['batch']==batch_[0]],scdata[scdata.obs['batch']==batch_[1]])
      3 t2 = time.time()
      4 print('Took '+str(timedelta(seconds=t2-t1)))

~/miniconda3/lib/python3.7/site-packages/scanpy/external/pp/_mnn_correct.py in mnn_correct(var_index, var_subset, batch_key, index_unique, batch_categories, k, sigma, cos_norm_in, cos_norm_out, svd_dim, var_adj, compute_angle, mnn_order, svd_mode, do_concatenate, save_raw, n_jobs, *datas, **kwargs)
    152         save_raw=save_raw,
    153         n_jobs=n_jobs,
--> 154         **kwargs,
    155     )
    156     return datas, mnn_list, angle_list

~/miniconda3/lib/python3.7/site-packages/mnnpy/mnn.py in mnn_correct(var_index, var_subset, batch_key, index_unique, batch_categories, k, sigma, cos_norm_in, cos_norm_out, svd_dim, var_adj, compute_angle, mnn_order, svd_mode, do_concatenate, save_raw, n_jobs, *datas, **kwargs)
    124                                 cos_norm_out=cos_norm_out, svd_dim=svd_dim, var_adj=var_adj,
    125                                 compute_angle=compute_angle, mnn_order=mnn_order,
--> 126                                 svd_mode=svd_mode, do_concatenate=do_concatenate, **kwargs)
    127         print('Packing AnnData object...')
    128         if do_concatenate:

~/miniconda3/lib/python3.7/site-packages/mnnpy/mnn.py in mnn_correct(var_index, var_subset, batch_key, index_unique, batch_categories, k, sigma, cos_norm_in, cos_norm_out, svd_dim, var_adj, compute_angle, mnn_order, svd_mode, do_concatenate, save_raw, n_jobs, *datas, **kwargs)
    180         print('  Computing correction vectors...')
    181         correction_in = compute_correction(ref_batch_in, new_batch_in, mnn_ref, mnn_new,
--> 182                                            new_batch_in, sigma)
    183         if not same_set:
    184             correction_out = compute_correction(ref_batch_out, new_batch_out, mnn_ref, mnn_new,

IndexError: arrays used as indices must be of integer (or boolean) type

corrected = sce.pp.mnn_correct(*[scdata[scdata.obs['batch']==batch] for batch in batch_])

Versions:

scanpy==1.4.5 anndata==0.7rc1 umap==0.3.7 numpy==1.18.0 scipy==1.3.1 pandas==0.23.4 scikit-learn==0.19.1 statsmodels==0.10.1 python-igraph==0.7.1 louvain==0.6.1

[adamgayoso]

I believe there are some issues with the newest version of llvmlite.

https://github.com/chriscainx/mnnpy/issues/30

[tkisss]

I believe there are some issues with the newest version of llvmlite.

chriscainx/mnnpy#30

I solved the problem according to your suggestion. Thank you very much!

[flying-sheep]

Hi, please give me a reproducible example that uses only public data or better manually typed data. (np.array([...])).

Also make sure you updated to numpy 0.47 (not 0.46) and llvmlite 0.31 before trying to reproduce the bug.

[adamgayoso]

Here's an example with the newest numba and llvmlite.

I noticed with fewer cells it works.

import scanpy as sc
import anndata
import numpy as np

a = anndata.AnnData(np.random.poisson(size=(4000, 5000)))
b = anndata.AnnData(np.random.poisson(size=(10000, 5000)))

sc.external.pp.mnn_correct(a, b)

Performing cosine normalization...
Starting MNN correct iteration. Reference batch: 0
Step 1 of 1: processing batch 1
  Looking for MNNs...
  Computing correction vectors...
/data/yosef2/users/adamgayoso/.pyenv/versions/3.7.3/envs/anndata_test/lib/python3.7/site-packages/mnnpy/utils.py:102: NumbaWarning: 
Compilation is falling back to object mode WITHOUT looplifting enabled because Function "compute_correction" failed type inference due to: non-precise type pyobject
[1] During: typing of argument at /data/yosef2/users/adamgayoso/.pyenv/versions/3.7.3/envs/anndata_test/lib/python3.7/site-packages/mnnpy/utils.py (107)

File ".pyenv/versions/3.7.3/envs/anndata_test/lib/python3.7/site-packages/mnnpy/utils.py", line 107:
def compute_correction(data1, data2, mnn1, mnn2, data2_or_raw2, sigma):
    <source elided>
    vect_reduced = np.zeros((data2.shape[0], vect.shape[1]), dtype=np.float32)
    for index, ve in zip(mnn2, vect):
    ^

  @jit(float32[:, :](float32[:, :], float32[:, :], int32[:], int32[:], float32[:, :], float32))
/data/yosef2/users/adamgayoso/.pyenv/versions/3.7.3/envs/anndata_test/lib/python3.7/site-packages/numba/object_mode_passes.py:178: NumbaWarning: Function "compute_correction" was compiled in object mode without forceobj=True.

File ".pyenv/versions/3.7.3/envs/anndata_test/lib/python3.7/site-packages/mnnpy/utils.py", line 107:
def compute_correction(data1, data2, mnn1, mnn2, data2_or_raw2, sigma):
    <source elided>
    vect_reduced = np.zeros((data2.shape[0], vect.shape[1]), dtype=np.float32)
    for index, ve in zip(mnn2, vect):
    ^

  state.func_ir.loc))
/data/yosef2/users/adamgayoso/.pyenv/versions/3.7.3/envs/anndata_test/lib/python3.7/site-packages/numba/object_mode_passes.py:188: NumbaDeprecationWarning: 
Fall-back from the nopython compilation path to the object mode compilation path has been detected, this is deprecated behaviour.

For more information visit http://numba.pydata.org/numba-doc/latest/reference/deprecation.html#deprecation-of-object-mode-fall-back-behaviour-when-using-jit

File ".pyenv/versions/3.7.3/envs/anndata_test/lib/python3.7/site-packages/mnnpy/utils.py", line 107:
def compute_correction(data1, data2, mnn1, mnn2, data2_or_raw2, sigma):
    <source elided>
    vect_reduced = np.zeros((data2.shape[0], vect.shape[1]), dtype=np.float32)
    for index, ve in zip(mnn2, vect):
    ^

  state.func_ir.loc))

---------------------------------------------------------------------------
IndexError                                Traceback (most recent call last)
<ipython-input-16-0bc362244a72> in <module>
----> 1 sc.external.pp.mnn_correct(a, b)

/data/yosef2/users/adamgayoso/.pyenv/versions/3.7.3/envs/anndata_test/lib/python3.7/site-packages/scanpy/external/pp/_mnn_correct.py in mnn_correct(var_index, var_subset, batch_key, index_unique, batch_categories, k, sigma, cos_norm_in, cos_norm_out, svd_dim, var_adj, compute_angle, mnn_order, svd_mode, do_concatenate, save_raw, n_jobs, *datas, **kwargs)
    152         save_raw=save_raw,
    153         n_jobs=n_jobs,
--> 154         **kwargs,
    155     )
    156     return datas, mnn_list, angle_list

/data/yosef2/users/adamgayoso/.pyenv/versions/3.7.3/envs/anndata_test/lib/python3.7/site-packages/mnnpy/mnn.py in mnn_correct(var_index, var_subset, batch_key, index_unique, batch_categories, k, sigma, cos_norm_in, cos_norm_out, svd_dim, var_adj, compute_angle, mnn_order, svd_mode, do_concatenate, save_raw, n_jobs, *datas, **kwargs)
    124                                 cos_norm_out=cos_norm_out, svd_dim=svd_dim, var_adj=var_adj,
    125                                 compute_angle=compute_angle, mnn_order=mnn_order,
--> 126                                 svd_mode=svd_mode, do_concatenate=do_concatenate, **kwargs)
    127         print('Packing AnnData object...')
    128         if do_concatenate:

/data/yosef2/users/adamgayoso/.pyenv/versions/3.7.3/envs/anndata_test/lib/python3.7/site-packages/mnnpy/mnn.py in mnn_correct(var_index, var_subset, batch_key, index_unique, batch_categories, k, sigma, cos_norm_in, cos_norm_out, svd_dim, var_adj, compute_angle, mnn_order, svd_mode, do_concatenate, save_raw, n_jobs, *datas, **kwargs)
    180         print('  Computing correction vectors...')
    181         correction_in = compute_correction(ref_batch_in, new_batch_in, mnn_ref, mnn_new,
--> 182                                            new_batch_in, sigma)
    183         if not same_set:
    184             correction_out = compute_correction(ref_batch_out, new_batch_out, mnn_ref, mnn_new,

IndexError: arrays used as indices must be of integer (or boolean) type

[flying-sheep]

Since the bug happens in mnnpy and isn’t caused by the scanpy wrapper, this is not a scanpy bug: chriscainx/mnnpy#30