diff --git a/Cargo.lock b/Cargo.lock index a1940446..a50acb77 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -173,6 +173,7 @@ dependencies = [ "async-trait", "chrono", "convert_case", + "encoding_rs_io", "float-cmp", "futures", "glob", @@ -321,6 +322,15 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "encoding_rs_io" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1cc3c5651fb62ab8aa3103998dade57efdd028544bd300516baa31840c252a83" +dependencies = [ + "encoding_rs", +] + [[package]] name = "equivalent" version = "1.0.1" diff --git a/Cargo.toml b/Cargo.toml index b47d3eb5..1ffbb1ee 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -28,6 +28,7 @@ async = ["async-trait"] lazy_static = "1.4" serde = "1.0" nom = "7" +encoding_rs_io = "0.1" async-trait = { version = "0.1", optional = true } toml = { version = "0.8", optional = true } diff --git a/src/file/source/file.rs b/src/file/source/file.rs index 8ee5d315..8d7d607d 100644 --- a/src/file/source/file.rs +++ b/src/file/source/file.rs @@ -1,9 +1,11 @@ use std::env; use std::error::Error; -use std::fs; use std::io; +use std::io::Read; use std::path::PathBuf; +use encoding_rs_io::DecodeReaderBytesBuilder; + use crate::file::{ format::ALL_EXTENSIONS, source::FileSourceResult, FileSource, FileStoredFormat, Format, }; @@ -114,8 +116,15 @@ where .and_then(|base| pathdiff::diff_paths(&filename, base)) .unwrap_or_else(|| filename.clone()); - // Read contents from file - let text = fs::read_to_string(filename)?; + let mut text = String::new(); + + DecodeReaderBytesBuilder::new() + // On Windows, we need to check for added Byte Order Mark (BOM). To + // do this, we don't specify an encoding, and enable BOM sniffing. + .encoding(None) + .bom_sniffing(true) + .build(std::fs::File::open(filename)?) + .read_to_string(&mut text)?; Ok(FileSourceResult { uri: Some(uri.to_string_lossy().into_owned()),