Trying to extend the Dependency parser's SBD detection with a custom component.

[MoetasimR]

Hello,

I am using Spacy's "en_core_web_sm" model to split large text strings into sentences and I am using the model's default language processing pipeline. However, I want to implement a custom component that disables the parser's SBD within a certain substring in a large text string.

This is my code for implementing the custom component:

import spacy
nlp = spacy.load("en_core_web_sm")
from spacy.language import Language
from spacy.tokens import Doc

@Language.component("custom_sentence_boundary")
def set_custom_boundaries(doc, start_string="<", end_string=">"): 
    new_sentence_index = 2  # might as well be any number other than -1 
    in_custom_segment = False  # Flag to track whether we're inside a custom segment
    for token in doc:
        if token.text == start_string:
            token.is_sent_start = True
            in_custom_segment = True
        elif token.text == end_string:
            token.is_sent_start = False
            in_custom_segment = False
            new_sentence_index = 0
            # Ensure the next token doesn't start a new sentence
        else:
            # If we're in a custom segment, don't start a new sentence
            if in_custom_segment:
                token.is_sent_start = False
            else:
                # Otherwise, leave sentence segmentation to the parser
                if new_sentence_index == 0:
                    token.is_sent_start = True
                else:
                    token.is_sent_start = None
                new_sentence_index += 1
    return doc

nlp.add_pipe("custom_sentence_boundary", before="parser")

#and this is a sample sentence to test if the parser respects the rules set by my custom component:


text = '<This is first sentence. Next is numbered list. 1. Hello World!.>  < 2. Hello World2!3. Hello World! > *\nThis is the end of the document. This is a new sentence. Hello there. Wise men say, only fools rush in.'
doc = nlp(text)
[sentence for sentence in doc.sents]
[<This is first sentence. Next is numbered list. 1. Hello World!.>,
  ,
 < 2. Hello World2!3. Hello World! >,
 *,
 This is the end of the document.,
 This is a new sentence.,
 Hello there.,
 Wise men say, only fools rush in.]

The problem arises when I try to include more than just a single "<" or ">" in my start_string, end_strings variable.
Exhibit A:

nlp = spacy.load("en_core_web_sm")

@Language.component("custom_sentence_boundary")
def set_custom_boundaries(doc, start_string="<[Start", end_string="End]>"):
    start_pattern = re.compile(re.escape(start_string))
    end_pattern = re.compile(re.escape(end_string))
    in_custom_segment = False  # Flag to track whether we're inside a custom segment

    for token in doc:
        if start_pattern.match(token.text):
            token.is_sent_start = True
            in_custom_segment = True
        elif end_pattern.match(token.text):
            token.is_sent_start = False
            in_custom_segment = False
        else:
            if in_custom_segment:
                token.is_sent_start = False
            else:
                token.is_sent_start = None

    return doc

nlp.add_pipe("custom_sentence_boundary", before="parser")

# Test the custom component
text = '<[Start This is first sentence. Next is numbered list. 1. Hello World!. End]>  <[Start 2. Hello World2!3. Hello World! End]> *\nThis is the end of the document. This is a new sentence. Hello there. Wise men say, only fools rush in.'
doc = nlp(text)
[sentence for sentence in doc.sents]
[<,
 [Start This is first sentence.,
 Next is numbered list.,
 1. Hello World!.,
 End]>  <,
 [Start 2.,
 Hello World2!3.,
 Hello World!,
 End]> *,
 This is the end of the document.,
 This is a new sentence.,
 Hello there.,
 Wise men say, only fools rush in.]

Apparently, the component's rules are now ignored, why is this happening?