Skip to content

⚡️ Speed up function calculate_edit_distance by 150% #269

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
codeflash-ai wants to merge 1 commit into
base: main
Choose a base branch
from
Open

⚡️ Speed up function calculate_edit_distance by 150% #269

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
127 changes: 61 additions & 66 deletions unstructured/metrics/text_extraction.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,63 @@

from unstructured.cleaners.core import clean_bullets, remove_sentence_punctuation

_DOUBLE_QUOTES = {
'"': "U+0022", # noqa 601 # Standard typewriter/programmer's quote
'"': "U+201C", # noqa 601 # Left double quotation mark
'"': "U+201D", # noqa 601 # Right double quotation mark
"„": "U+201E", # Double low-9 quotation mark
"‟": "U+201F", # Double high-reversed-9 quotation mark
"«": "U+00AB", # Left-pointing double angle quotation mark
"»": "U+00BB", # Right-pointing double angle quotation mark
"❝": "U+275D", # Heavy double turned comma quotation mark ornament
"❞": "U+275E", # Heavy double comma quotation mark ornament
"⹂": "U+2E42", # Double low-reversed-9 quotation mark
"🙶": "U+1F676", # SANS-SERIF HEAVY DOUBLE TURNED COMMA QUOTATION MARK ORNAMENT
"🙷": "U+1F677", # SANS-SERIF HEAVY DOUBLE COMMA QUOTATION MARK ORNAMENT
"🙸": "U+1F678", # SANS-SERIF HEAVY LOW DOUBLE COMMA QUOTATION MARK ORNAMENT
"⠦": "U+2826", # Braille double closing quotation mark
"⠴": "U+2834", # Braille double opening quotation mark
"〝": "U+301D", # REVERSED DOUBLE PRIME QUOTATION MARK
"〞": "U+301E", # DOUBLE PRIME QUOTATION MARK
"〟": "U+301F", # LOW DOUBLE PRIME QUOTATION MARK
"＂": "U+FF02", # FULLWIDTH QUOTATION MARK
",,": "U+275E", # LOW HEAVY DOUBLE COMMA ORNAMENT
}

_SINGLE_QUOTES = {
"'": "U+0027", # noqa 601 # Standard typewriter/programmer's quote
"'": "U+2018", # noqa 601 # Left single quotation mark
"'": "U+2019", # noqa 601 # Right single quotation mark # noqa: W605
"‚": "U+201A", # Single low-9 quotation mark
"‛": "U+201B", # Single high-reversed-9 quotation mark
"‹": "U+2039", # Single left-pointing angle quotation mark
"›": "U+203A", # Single right-pointing angle quotation mark
"❛": "U+275B", # Heavy single turned comma quotation mark ornament
"❜": "U+275C", # Heavy single comma quotation mark ornament
"「": "U+300C", # Left corner bracket
"」": "U+300D", # Right corner bracket
"『": "U+300E", # Left white corner bracket
"』": "U+300F", # Right white corner bracket
"﹁": "U+FE41", # PRESENTATION FORM FOR VERTICAL LEFT CORNER BRACKET
"﹂": "U+FE42", # PRESENTATION FORM FOR VERTICAL RIGHT CORNER BRACKET
"﹃": "U+FE43", # PRESENTATION FORM FOR VERTICAL LEFT WHITE CORNER BRACKET
"﹄": "U+FE44", # PRESENTATION FORM FOR VERTICAL RIGHT WHITE CORNER BRACKET
"＇": "U+FF07", # FULLWIDTH APOSTROPHE
"｢": "U+FF62", # HALFWIDTH LEFT CORNER BRACKET
"｣": "U+FF63", # HALFWIDTH RIGHT CORNER BRACKET
}

_DOUBLE_QUOTE_CHARS = {chr(int(val.replace("U+", ""), 16)) for val in _DOUBLE_QUOTES.values()}

_SINGLE_QUOTE_CHARS = {chr(int(val.replace("U+", ""), 16)) for val in _SINGLE_QUOTES.values()}

_QUOTE_TRANSLATION = str.maketrans(
{
**{chr(int(val.replace("U+", ""), 16)): '"' for val in _DOUBLE_QUOTES.values()},
**{chr(int(val.replace("U+", ""), 16)): "'" for val in _SINGLE_QUOTES.values()},
}
)


def calculate_accuracy(
output: Optional[str],
Expand Down Expand Up @@ -54,8 +111,8 @@ def calculate_edit_distance(
- "distance": Returns the raw edit distance value.

"""
return_types = ["score", "distance"]
if return_as not in return_types:
if return_as not in ("score", "distance"):
return_types = ["score", "distance"]
raise ValueError("Invalid return value type. Expected one of: %s" % return_types)
output = standardize_quotes(prepare_str(output, standardize_whitespaces))
source = standardize_quotes(prepare_str(source, standardize_whitespaces))
Expand Down Expand Up @@ -172,70 +229,8 @@ def standardize_quotes(text: str) -> str:
Returns:
str: The text with standardized quotes.
"""
# Double Quotes Dictionary
double_quotes = {
'"': "U+0022", # noqa 601 # Standard typewriter/programmer's quote
'"': "U+201C", # noqa 601 # Left double quotation mark
'"': "U+201D", # noqa 601 # Right double quotation mark
"„": "U+201E", # Double low-9 quotation mark
"‟": "U+201F", # Double high-reversed-9 quotation mark
"«": "U+00AB", # Left-pointing double angle quotation mark
"»": "U+00BB", # Right-pointing double angle quotation mark
"❝": "U+275D", # Heavy double turned comma quotation mark ornament
"❞": "U+275E", # Heavy double comma quotation mark ornament
"⹂": "U+2E42", # Double low-reversed-9 quotation mark
"🙶": "U+1F676", # SANS-SERIF HEAVY DOUBLE TURNED COMMA QUOTATION MARK ORNAMENT
"🙷": "U+1F677", # SANS-SERIF HEAVY DOUBLE COMMA QUOTATION MARK ORNAMENT
"🙸": "U+1F678", # SANS-SERIF HEAVY LOW DOUBLE COMMA QUOTATION MARK ORNAMENT
"⠦": "U+2826", # Braille double closing quotation mark
"⠴": "U+2834", # Braille double opening quotation mark
"〝": "U+301D", # REVERSED DOUBLE PRIME QUOTATION MARK
"〞": "U+301E", # DOUBLE PRIME QUOTATION MARK
"〟": "U+301F", # LOW DOUBLE PRIME QUOTATION MARK
"＂": "U+FF02", # FULLWIDTH QUOTATION MARK
",,": "U+275E", # LOW HEAVY DOUBLE COMMA ORNAMENT
}

# Single Quotes Dictionary
single_quotes = {
"'": "U+0027", # noqa 601 # Standard typewriter/programmer's quote
"'": "U+2018", # noqa 601 # Left single quotation mark
"'": "U+2019", # noqa 601 # Right single quotation mark # noqa: W605
"‚": "U+201A", # Single low-9 quotation mark
"‛": "U+201B", # Single high-reversed-9 quotation mark
"‹": "U+2039", # Single left-pointing angle quotation mark
"›": "U+203A", # Single right-pointing angle quotation mark
"❛": "U+275B", # Heavy single turned comma quotation mark ornament
"❜": "U+275C", # Heavy single comma quotation mark ornament
"「": "U+300C", # Left corner bracket
"」": "U+300D", # Right corner bracket
"『": "U+300E", # Left white corner bracket
"』": "U+300F", # Right white corner bracket
"﹁": "U+FE41", # PRESENTATION FORM FOR VERTICAL LEFT CORNER BRACKET
"﹂": "U+FE42", # PRESENTATION FORM FOR VERTICAL RIGHT CORNER BRACKET
"﹃": "U+FE43", # PRESENTATION FORM FOR VERTICAL LEFT WHITE CORNER BRACKET
"﹄": "U+FE44", # PRESENTATION FORM FOR VERTICAL RIGHT WHITE CORNER BRACKET
"＇": "U+FF07", # FULLWIDTH APOSTROPHE
"｢": "U+FF62", # HALFWIDTH LEFT CORNER BRACKET
"｣": "U+FF63", # HALFWIDTH RIGHT CORNER BRACKET
}

double_quote_standard = '"'
single_quote_standard = "'"

# Apply double quote replacements
for unicode_val in double_quotes.values():
unicode_char = unicode_to_char(unicode_val)
if unicode_char in text:
text = text.replace(unicode_char, double_quote_standard)

# Apply single quote replacements
for unicode_val in single_quotes.values():
unicode_char = unicode_to_char(unicode_val)
if unicode_char in text:
text = text.replace(unicode_char, single_quote_standard)

return text
# Use str.translate() for efficient bulk character replacement
return text.translate(_QUOTE_TRANSLATION)


def unicode_to_char(unicode_val: str) -> str:
Expand Down