Spacy: loading existing model throws exception KeyError: 1

Created on 27 Apr 2017 · 7Comments · Source: explosion/spaCy

I've trained a model using BILUO tags after saving when I try to load the mode it throws an error.
Here is the code for saving the model

def save_model(ner, model):
    model_dir = pathlib.Path(model)
    if not model_dir.exists():
        model_dir.mkdir()
    assert model_dir.is_dir()

    with (model_dir / 'config.json').open('w') as file_:
        json.dump(ner.cfg, file_)
    ner.model.dump(str(model_dir / 'model'))
    if not (model_dir / 'vocab').exists():
        (model_dir / 'vocab').mkdir()
    ner.vocab.dump(str(model_dir / 'vocab' / 'lexemes.bin'))
    with (model_dir / 'vocab' / 'strings.json').open('w', encoding='utf8') as file_:
        ner.vocab.strings.dump(file_)

Here is the code for loading the model

def load_model(model):
    model_dir = pathlib.Path(model)
    if not model_dir.exists():
        return False

    nlp = spacy.load('en', parser=False, entity=False, add_vectors=False)
    vocab_dir = pathlib.Path(model_dir / 'vocab')

    with (vocab_dir / 'strings.json').open('r', encoding='utf8') as file_:
        nlp.vocab.strings.load(file_)

    nlp.vocab.load_lexemes(vocab_dir / 'lexemes.bin')

    ner = EntityRecognizer.load(model_dir, nlp.vocab, require=True)
    return (nlp, ner)

This line throws error ner = EntityRecognizer.load(model_dir, nlp.vocab, require=True)
Here is the console output.

line 12, in load_model
    ner = EntityRecognizer.load(model_dir, nlp.vocab, require=True)
  File "spacy/syntax/parser.pyx", line 154, in spacy.syntax.parser.Parser.load (spacy/syntax/parser.cpp:6999)
  File "spacy/syntax/parser.pyx", line 177, in spacy.syntax.parser.Parser.__init__ (spacy/syntax/parser.cpp:7370)
  File "spacy/syntax/ner.pyx", line 71, in spacy.syntax.ner.BiluoPushDown.get_actions (spacy/syntax/ner.cpp:3906)
KeyError: 1

Your Environment

Info about spaCy

spaCy version: 1.8.2
Installed models: en, en_default
Platform: Linux-4.4.0-75-generic-x86_64-with-Ubuntu-16.04-xenial
Python version: 3.5.1+

bug

Source

irfan-zoefit

Most helpful comment

@ines is there a workaround until 2.0 is released?

lukewendling on 6 Jul 2017

👍2

All 7 comments

I am trying to train my own named entity recognizer but I fails exactly in the same way.

thomasgirault on 28 Apr 2017

😕1

Hi Guys, I am trying to use spacy with rasa_nlu and I am getting this error:

python -m rasa_nlu.server -c config.json --server_model_dirs=./model_20170429-200133
INFO:root:Configuration: {
"num_threads": 1,
"log_level": 20,
"language": "en",
"mitie_file": "data/total_word_feature_extractor.dat",
"emulate": null,
"config": "config.json",
"response_log": "/home/ff/logs",
"server_model_dirs": "./model_20170429-200133",
"fine_tune_spacy_ner": false,
"token": null,
"path": "./models",
"log_file": null,
"data": "./data/examples/demo-rasa.json",
"port": 5000,
"backend": "spacy_sklearn"
}
INFO:root:Logging requests to '/home/ff/logs/rasa_nlu_log-20170429-200311-4566.log'.
INFO:root:using spacy + sklearn backend
Traceback (most recent call last):
File "/root/anaconda2/lib/python2.7/runpy.py", line 174, in _run_module_as_main
"__main__", fname, loader, pkg_name)
File "/root/anaconda2/lib/python2.7/runpy.py", line 72, in _run_code
exec code in run_globals
File "/root/anaconda2/lib/python2.7/site-packages/rasa_nlu/server.py", line 99, in
app = WSGIServer(('0.0.0.0', rasa_nlu_config['port']), create_app(rasa_nlu_config))
File "/root/anaconda2/lib/python2.7/site-packages/rasa_nlu/server.py", line 90, in create_app
rasa_nlu_app.data_router = DataRouter(config)
File "/root/anaconda2/lib/python2.7/site-packages/rasa_nlu/data_router.py", line 29, in __init__
self.model_store = self.__create_model_store()
File "/root/anaconda2/lib/python2.7/site-packages/rasa_nlu/data_router.py", line 112, in __create_model_store
interpreter = DataRouter.create_interpreter(metadata, nlp, featurizer)
File "/root/anaconda2/lib/python2.7/site-packages/rasa_nlu/data_router.py", line 170, in create_interpreter
return SpacySklearnInterpreter.load(metadata, nlp, featurizer)
File "/root/anaconda2/lib/python2.7/site-packages/rasa_nlu/interpreters/spacy_sklearn_interpreter.py", line 25, in load
meta.metadata.get("should_fine_tune_spacy_ner"))
File "/root/anaconda2/lib/python2.7/site-packages/rasa_nlu/extractors/spacy_entity_extractor.py", line 15, in __init__
self.ner = EntityRecognizer.load(pathlib.Path(extractor_file), nlp.vocab)
File "spacy/syntax/parser.pyx", line 154, in spacy.syntax.parser.Parser.load (spacy/syntax/parser.cpp:6999)
File "spacy/syntax/parser.pyx", line 177, in spacy.syntax.parser.Parser.__init__ (spacy/syntax/parser.cpp:7370)
File "spacy/syntax/ner.pyx", line 71, in spacy.syntax.ner.BiluoPushDown.get_actions (spacy/syntax/ner.cpp:3906)
KeyError: 1