From 92ba3cdbe03979d0ce4bd52a68996cf075133059 Mon Sep 17 00:00:00 2001 From: Christoph Date: Mon, 30 Dec 2024 15:57:23 +0100 Subject: [PATCH] Improve utf8 detection (#12342) * improve utf8 detction with confidence * changelog * refine comment --- CHANGELOG.md | 2 ++ src/main/java/org/jabref/logic/importer/Importer.java | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 29e50ecd0b7..b00ed925c31 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,8 @@ Note that this project **does not** adhere to [Semantic Versioning](https://semv ### Fixed +- We fixed an issue where a bib file with UFF-8 charset was wrongly loaded with a different charset [forum#5369](https://discourse.jabref.org/t/jabref-5-15-opens-bib-files-with-shift-jis-encoding-instead-of-utf-8/5369/) + ### Removed ## [6.0-alpha] – 2024-12-23 diff --git a/src/main/java/org/jabref/logic/importer/Importer.java b/src/main/java/org/jabref/logic/importer/Importer.java index d991b087d3b..7f465176eea 100644 --- a/src/main/java/org/jabref/logic/importer/Importer.java +++ b/src/main/java/org/jabref/logic/importer/Importer.java @@ -121,7 +121,8 @@ protected static Charset getCharset(BufferedInputStream bufferedInputStream) { return defaultCharSet; } - if (Arrays.stream(matches).anyMatch(charset -> "ASCII".equals(charset.getName()))) { + // if we have utf8 with 100 confidence we assume that the file is in utf8, more likely + if (Arrays.stream(matches).anyMatch(charset -> "ASCII".equals(charset.getName()) || ("UTF-8".equals(charset.getName()) && charset.getConfidence() == 100))) { return defaultCharSet; }