I have develop a custom NER model with custom labels,but whenever I am giving sentence that is out of training data it is giving either empty output or output with wrong label.Also,the model is not consistent and accurate most of the time.What should I do??
import spacy
from spacy.training import Example
import random
from spacy.util import minibatch
nlp=spacy.load("en_core_web_md")
from spacy.training import offsets_to_biluo_tags
from sklearn.metrics import precision_recall_fscore_support
from sklearn.metrics import accuracy_score
nlp = spacy.load("en_core_web_md")
def validate_alignment(text, entities):
doc = nlp.make_doc(text)
biluo_tags = offsets_to_biluo_tags(doc, entities)
print(f"Text: '{text}'")
print(f"Entities: {entities}")
print(f"BILUO Tags: {biluo_tags}")
TRAINING_DATA = [
("My name is John", {"entities": [(11, 15, "PERSON")]}),
("I am John", {"entities": [(5, 9, "PERSON")]}),
("John this side", {"entities": [(0, 4, "PERSON")]}),
("Hello, my name is Jane Doe.", {"entities": [(17, 26, "PERSON")]}),
("Jane is my friend.", {"entities": [(0, 4, "PERSON")]}),
("I spoke to John yesterday.", {"entities": [(15, 19, "PERSON")]}),
("John and Jane are working together.", {"entities": [(0, 4, "PERSON"), (9, 13, "PERSON")]}),
("Delhi is a great city.", {"entities": [(0, 5, "PLACE")]}),
("Mumbai is a great city.", {"entities": [(0, 6, "PLACE")]}),
("Kolkata is a great city.", {"entities": [(0, 7, "PLACE")]}),
("Cheenai is a great city.", {"entities": [(0, 8, "PLACE")]}),
("Patna is a great city.", {"entities": [(0, 5, "PLACE")]}),
("Lucknow is a great city.", {"entities": [(0, 7, "PLACE")]}),
("The conference will be held in Delhi and Mumbai.", {"entities": [(32, 37, "PLACE"), (42, 48, "PLACE")]}),
("Add the pipeline name Pipeline1.", {"entities": [(22, 31, "PIPELINE")]}),
("Add the pipeline name PipelineXY.", {"entities": [(22, 32, "PIPELINE")]}),
("Add the pipeline name PipelineA.", {"entities": [(22, 31, "PIPELINE")]}),
("Add the pipeline name PipelineTest.", {"entities": [(22, 34, "PIPELINE")]}),
("My pipeline name is PipelineAB.", {"entities": [(22, 31, "PIPELINE")]}),
("The latest version is Pipeline2.", {"entities": [(21, 30, "PIPELINE")]}),
("We are using Pipeline3 for this project.", {"entities": [(19, 28, "PIPELINE")]}),
("The pipeline PipelineX was used for processing data from the SourceName MongoDb.", {"entities": [(4, 14, "PIPELINE"), (37, 44, "SOURCE")]}),
("PipelineY is the new update and it will be integrated with PipelineAB.", {"entities": [(0, 9, "PIPELINE"), (37, 45, "PIPELINE")]}),
("The SourceName will be MongoDb", {"entities": [(23, 30, "SOURCE")]}),
("The SourceName will be Postgres", {"entities": [(23, 31, "SOURCE")]}),
("The SourceName will be SQL", {"entities": [(23, 26, "SOURCE")]}),
("The SourceName will be SNOWFLAKE", {"entities": [(23, 32, "SOURCE")]}),
("Give Source as the Postgres", {"entities": [(19, 27, "SOURCE")]}),
("Give Source as the MongoDb", {"entities": [(19, 26, "SOURCE")]}),
("I want my SourceName to be SQL and DestinationName to be MongoDb.", {"entities": [(27, 30, "SOURCE"), (42, 49, "DESTINATION")]}),
("I can take source as SNOWFLAKE.",{"entities":[(21,30,"SOURCE")]}),
("The DestinationName is MongoDb", {"entities": [(23, 30, "DESTINATION")]}),
("The DestinationName is Postgres", {"entities": [(23, 31, "DESTINATION")]}),
("The DestinationName is SQL", {"entities": [(23, 26, "DESTINATION")]}),
("I want DestinationName as SNOWFLAKE", {"entities": [(26, 35, "DESTINATION")]}),
("I want my DestinationName to be SQL and SourceName to be Postgres.", {"entities": [(32, 35, "DESTINATION"), (50, 57, "SOURCE")]}),
("The Script will be a Spark.", {"entities": [(21, 26, "SCRIPT")]}),
("The Script will be a Python.", {"entities": [(21, 27, "SCRIPT")]}),
("I am using Python for my Script.", {"entities": [(15, 21, "SCRIPT")]}),
("The chosen Script is Spark and it will work with PipelineA.", {"entities": [(18, 23, "SCRIPT"), (42, 51, "PIPELINE")]}),
("I am choosing left-inner join", {"entities": [(14, 24, "JOINS")]}),
("I am choosing left-outer join", {"entities": [(14, 24, "JOINS")]}),
("The process includes a cross join", {"entities": [(18, 23, "JOINS")]}),
("Using a left join is preferred over an inner join", {"entities": [(9, 18, "JOINS"), (32, 41, "JOINS")]}),
("I want a cross join",{"entities":[(9,14,"JOINS")]}),
("I want a inner join",{"entities":[(9,14,"JOINS")]})
]
for text, annotations in TRAINING_DATA:
validate_alignment(text, annotations['entities'])
if "ner" not in nlp.pipe_names:
ner = nlp.create_pipe("ner")
nlp.add_pipe(ner, last=True)
else:
ner = nlp.get_pipe("ner")
custom_labels = ["TECHNOLOGY", "PIPELINE", "PLACE", "SOURCE","DESTINATION","SCRIPT","JOINS"]
for label in custom_labels:
if label not in ner.labels:
ner.add_label(label)
train_data = []
for text, annotations in TRAINING_DATA:
doc = nlp.make_doc(text)
example = Example.from_dict(doc, annotations)
train_data.append(example)
optimizer = nlp.begin_training()
for epoch in range(50):
random.shuffle(train_data)
losses = {}
for batch in minibatch(train_data, size=8):
for example in batch:
nlp.update([example], drop=0.5, losses=losses)
print(f"Epoch {epoch} - Losses: {losses}")
nlp.to_disk("/home/datatroops/Session")
nlp = spacy.load("/home/datatroops/Session")
def extract_entities(text):
doc = nlp(text)
entities = {ent.label_: ent.text for ent in doc.ents}
return entities
texts = [
"PipelineXYZ is the new.",
"The Pipeline name is PipelineA.",
"The SourceName will be SNOWFLAKE",
"The DestinationName is MongoDb",
"The Script will be a Spark.",
"PipelineXY is the new update.",
"I am choosing lefti join",
"I want my SourceName to be SNOWFLAKE",
"SnowFlake",
"I can take source as SNOWFLAKE.",
"Give source as SNOWFLAKE.",
"My script will be Spark.",
# "My name is John.",
# "My name is Shashwat.",
"I want a cross join",
"Using a left join is preferred over an inner join",
"I want a inner join",
"I want an outer join"
]
for text in texts:
print(f"Text: '{text}'")
print("Extracted entities:", extract_entities(text))
