Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Catch parsing mistakes #26

Open
wants to merge 3 commits into
base: master
Choose a base branch
from
Open

Catch parsing mistakes #26

Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
9d34e2b
Catch parsing mistakes
manuelrech Mar 20, 2024
5a67ba8
Remove xml - html warning
manuelrech Mar 21, 2024
0d8252d
update checks on wrongly parsed articles
manuelrech Mar 21, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
53 changes: 27 additions & 26 deletions scipdf/pdf/parse_pdf.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -113,7 +113,7 @@ def parse_pdf(
parsed_article = None

if soup and parsed_article is not None:
parsed_article = BeautifulSoup(parsed_article, "lxml")
parsed_article = BeautifulSoup(parsed_article, features='xml')

return parsed_article

Expand All @@ -122,7 +122,7 @@ def parse_authors(article):
"""
Parse authors from a given BeautifulSoup of an article
"""
author_names = article.find("sourcedesc").findAll("persname")
author_names = article.find("fileDesc").find_all("persName")
authors = []
for author in author_names:
firstname = author.find("forename", {"type": "first"})
Expand All @@ -143,7 +143,7 @@ def parse_date(article):
"""
Parse date from a given BeautifulSoup of an article
"""
pub_date = article.find("publicationstmt")
pub_date = article.find("publicationStmt")
year = pub_date.find("date")
year = year.attrs.get("when") if year is not None else ""
return year
Expand Down Expand Up @@ -280,7 +280,7 @@ def parse_figure_caption(article):
figure_id = figure.attrs.get("xml:id") or ""
label = figure.find("label").text
if figure_type == "table":
caption = figure.find("figdesc").text
caption = figure.find("figDesc").text
data = figure.table.text
else:
caption = figure.text
Expand All @@ -289,6 +289,7 @@ def parse_figure_caption(article):
{
"figure_label": label,
"figure_type": figure_type,
"figure_type_label": f"{figure_type.title()}-{label}",
"figure_id": figure_id,
"figure_caption": caption,
"figure_data": data,
Expand Down Expand Up @@ -350,27 +351,28 @@ def convert_article_soup_to_dict(article, as_list: bool = False):
]
}
"""
article_dict = {}
if article is not None:
title = article.find("title", attrs={"type": "main"})
title = title.text.strip() if title is not None else ""

article_dict["title"] = title
article_dict["authors"] = parse_authors(article)
article_dict["pub_date"] = parse_date(article)
article_dict["abstract"] = parse_abstract(article)
article_dict["sections"] = parse_sections(article, as_list=as_list)
article_dict["references"] = parse_references(article)
article_dict["figures"] = parse_figure_caption(article)
article_dict["formulas"] = parse_formulas(article)

doi = article.find("idno", attrs={"type": "DOI"})
doi = doi.text if doi is not None else ""
article_dict["doi"] = doi

return article_dict
else:
if article is None or (article.contents == [] and article.text == ""):
return None

article_dict = {}

title = article.find("title", attrs={"type": "main"})
title = title.text.strip() if title is not None else ""

article_dict["title"] = title
article_dict["authors"] = parse_authors(article)
article_dict["pub_date"] = parse_date(article)
article_dict["abstract"] = parse_abstract(article)
article_dict["sections"] = parse_sections(article, as_list=as_list)
article_dict["references"] = parse_references(article)
article_dict["figures"] = parse_figure_caption(article)
article_dict["formulas"] = parse_formulas(article)

doi = article.find("idno", attrs={"type": "DOI"})
doi = doi.text if doi is not None else ""
article_dict["doi"] = doi

return article_dict


def parse_pdf_to_dict(
Expand All @@ -380,7 +382,6 @@ def parse_pdf_to_dict(
as_list: bool = False,
return_coordinates: bool = True,
grobid_url: str = GROBID_URL,
parse_figures: bool = True,
):
"""
Parse the given PDF and return dictionary of the parsed article
Expand Down Expand Up @@ -455,7 +456,7 @@ def parse_figures(
op.join(op.abspath(figure_path), ""), # end path with "/"
]
_ = subprocess.run(
args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=20
args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=60
)
print("Done parsing figures from PDFs!")
else:
Expand Down