Classify and Extract Compliance Documents with Unstructured and spaCy

                      
                     
from typing import List, Dict, Any
import os
import spacy
from spacy.training import Example
from spacy.pipeline.textcat import Config, ConfigDefaults

# Load spaCy model
nlp = spacy.load('en_core_web_sm')

# Configuration
config = Config().from_str('''[textcat]
model = "simple_cnn"
''' )
text_classifier = nlp.add_pipe('textcat', config=config, last=True)

# Add labels
text_classifier.add_label('compliance')
text_classifier.add_label('non_compliance')

# Training data
train_data: List[Tuple[str, Dict[str, Any]]] = [
    ('This document ensures compliance with regulations.', {'cats': {'compliance': 1, 'non_compliance': 0}}),
    ('This document is not compliant.', {'cats': {'compliance': 0, 'non_compliance': 1}})
]

# Training the model
def train_model(train_data: List[Tuple[str, Dict[str, Any]]]) -> None:
    optimizer = nlp.begin_training()
    for epoch in range(10):  # number of epochs
        for text, annotations in train_data:
            example = Example.from_dict(nlp.make_doc(text), annotations)
            nlp.update([example], drop=0.5, losses={})  # update model

try:
    train_model(train_data)
except Exception as e:
    print(f'Error during training: {e}')

# Prediction function
def predict(text: str) -> str:
    doc = nlp(text)
    return max(doc.cats, key=doc.cats.get)

if __name__ == '__main__':
    test_text = 'This document must meet compliance standards.'
    prediction = predict(test_text)
    print(f'The document is classified as: {prediction}')