Merge pull request #435 from bjorn3/write_dylib_metadata

Read and write dylib metadata

Author: bjorn3

Cargo.lock

@@ -25,6 +25,7 @@ libc = "0.2.53"
 tempfile = "3.0.7"
 gimli = { git = "https://github.com/gimli-rs/gimli.git" }
 indexmap = "1.0.2"
+object = "0.12.0"
 
 # Uncomment to use local checkout of cranelift
 #[patch."https://github.com/CraneStation/cranelift.git"]
@@ -39,5 +40,9 @@ indexmap = "1.0.2"
 [target.'cfg(not(target_arch = "wasm32"))'.dependencies]
 cranelift-simplejit = { git = "https://github.com/CraneStation/cranelift.git" }
 
+[patch.crates-io]
+faerie = { git = "https://github.com/m4b/faerie.git" }
+object = { git = "https://github.com/gimli-rs/object.git" }
+
 [profile.dev.overrides."*"]
 opt-level = 3

Cargo.toml

@@ -20,4 +20,4 @@ fi
 
 export RUSTFLAGS='-Zalways-encode-mir -Cpanic=abort -Cdebuginfo=2 -Zcodegen-backend='$(pwd)'/target/'$channel'/librustc_codegen_cranelift.'$dylib_ext' --sysroot '$(pwd)'/build_sysroot/sysroot'
 RUSTC="rustc $RUSTFLAGS -L crate=target/out --out-dir target/out"
-export RUST_LOG=warn # display metadata load errors
+export RUSTC_LOG=warn # display metadata load errors

config.sh

@@ -191,7 +191,7 @@ impl<'a, 'tcx: 'a> DebugContext<'tcx> {
         let _: Result<()> = sections.for_each_mut(|id, section| {
             if !section.writer.slice().is_empty() {
                 artifact
-                    .declare_with(id.name(), Decl::debug_section(), section.writer.take())
+                    .declare_with(id.name(), Decl::section(SectionKind::Debug), section.writer.take())
                     .unwrap();
             }
             Ok(())

src/debuginfo.rs

@@ -98,7 +98,7 @@ fn run_jit<'a, 'tcx: 'a>(tcx: TyCtxt<'a, 'tcx, 'tcx>, log: &mut Option<File>) ->
 fn run_aot<'a, 'tcx: 'a>(
     tcx: TyCtxt<'a, 'tcx, 'tcx>,
     metadata: EncodedMetadata,
-    _need_metadata_module: bool,
+    need_metadata_module: bool,
     log: &mut Option<File>,
 ) -> Box<CodegenResults> {
     let new_module = |name: String| {
@@ -166,6 +166,37 @@ fn run_aot<'a, 'tcx: 'a>(
     rustc_incremental::save_dep_graph(tcx);
     rustc_incremental::finalize_session_directory(tcx.sess, tcx.crate_hash(LOCAL_CRATE));
 
+    let metadata_module = if need_metadata_module {
+        use rustc::mir::mono::CodegenUnitNameBuilder;
+
+        let cgu_name_builder = &mut CodegenUnitNameBuilder::new(tcx);
+        let metadata_cgu_name = cgu_name_builder
+            .build_cgu_name(LOCAL_CRATE, &["crate"], Some("metadata"))
+            .as_str()
+            .to_string();
+
+        let mut metadata_artifact =
+            faerie::Artifact::new(crate::build_isa(tcx.sess).triple().clone(), metadata_cgu_name.clone());
+        crate::metadata::write_metadata(tcx, &mut metadata_artifact);
+
+        let tmp_file = tcx
+            .output_filenames(LOCAL_CRATE)
+            .temp_path(OutputType::Metadata, Some(&metadata_cgu_name));
+
+        let obj = metadata_artifact.emit().unwrap();
+        std::fs::write(&tmp_file, obj).unwrap();
+
+        Some(CompiledModule {
+            name: metadata_cgu_name,
+            kind: ModuleKind::Metadata,
+            object: Some(tmp_file),
+            bytecode: None,
+            bytecode_compressed: None,
+        })
+    } else {
+        None
+    };
+
     Box::new(CodegenResults {
         crate_name: tcx.crate_name(LOCAL_CRATE),
         modules: vec![emit_module(
@@ -184,13 +215,7 @@ fn run_aot<'a, 'tcx: 'a>(
         } else {
             None
         },
-        metadata_module: Some(CompiledModule {
-            name: "dummy_metadata".to_string(),
-            kind: ModuleKind::Metadata,
-            object: None,
-            bytecode: None,
-            bytecode_compressed: None,
-        }),
+        metadata_module,
         crate_hash: tcx.crate_hash(LOCAL_CRATE),
         metadata,
         windows_subsystem: None, // Windows is not yet supported

src/driver.rs

@@ -1,6 +1,7 @@
 #![feature(rustc_private, never_type, decl_macro)]
 #![allow(intra_doc_link_resolution_failure)]
 
+extern crate flate2;
 extern crate rustc;
 extern crate rustc_allocator;
 extern crate rustc_codegen_ssa;

src/lib.rs

@@ -1,16 +1,20 @@
-use rustc::middle::cstore::MetadataLoader;
+use std::fs::File;
+use std::path::Path;
+
+use rustc::session::config;
+use rustc::ty::TyCtxt;
+use rustc::middle::cstore::{EncodedMetadata, MetadataLoader};
 use rustc_codegen_ssa::METADATA_FILENAME;
 use rustc_data_structures::owning_ref::{self, OwningRef};
 use rustc_data_structures::rustc_erase_owner;
-use std::fs::File;
-use std::path::Path;
+use rustc_target::spec::Target;
 
 pub struct CraneliftMetadataLoader;
 
 impl MetadataLoader for CraneliftMetadataLoader {
     fn get_rlib_metadata(
         &self,
-        _target: &crate::rustc_target::spec::Target,
+        _target: &Target,
         path: &Path,
     ) -> Result<owning_ref::ErasedBoxRef<[u8]>, String> {
         let mut archive = ar::Archive::new(File::open(path).map_err(|e| format!("{:?}", e))?);
@@ -31,9 +35,72 @@ impl MetadataLoader for CraneliftMetadataLoader {
 
     fn get_dylib_metadata(
         &self,
-        _target: &crate::rustc_target::spec::Target,
-        _path: &Path,
+        _target: &Target,
+        path: &Path,
     ) -> Result<owning_ref::ErasedBoxRef<[u8]>, String> {
-        Err("dylib metadata loading is not yet supported".to_string())
+        use object::Object;
+        let file = std::fs::read(path).map_err(|e| format!("read:{:?}", e))?;
+        let file = object::File::parse(&file).map_err(|e| format!("parse: {:?}", e))?;
+        let buf = file.section_data_by_name(".rustc").ok_or("no .rustc section")?.into_owned();
+        let buf: OwningRef<Vec<u8>, [u8]> = OwningRef::new(buf).into();
+        Ok(rustc_erase_owner!(buf.map_owner_box()))
+    }
+}
+
+// Adapted from https://github.com/rust-lang/rust/blob/da573206f87b5510de4b0ee1a9c044127e409bd3/src/librustc_codegen_llvm/base.rs#L47-L112
+pub fn write_metadata<'a, 'gcx>(
+    tcx: TyCtxt<'a, 'gcx, 'gcx>,
+    artifact: &mut faerie::Artifact
+) -> EncodedMetadata {
+    use std::io::Write;
+    use flate2::Compression;
+    use flate2::write::DeflateEncoder;
+
+    #[derive(PartialEq, Eq, PartialOrd, Ord)]
+    enum MetadataKind {
+        None,
+        Uncompressed,
+        Compressed
+    }
+
+    let kind = tcx.sess.crate_types.borrow().iter().map(|ty| {
+        match *ty {
+            config::CrateType::Executable |
+            config::CrateType::Staticlib |
+            config::CrateType::Cdylib => MetadataKind::None,
+
+            config::CrateType::Rlib => MetadataKind::Uncompressed,
+
+            config::CrateType::Dylib |
+            config::CrateType::ProcMacro => MetadataKind::Compressed,
+        }
+    }).max().unwrap_or(MetadataKind::None);
+
+    if kind == MetadataKind::None {
+        return EncodedMetadata::new();
     }
+
+    let metadata = tcx.encode_metadata();
+    if kind == MetadataKind::Uncompressed {
+        return metadata;
+    }
+
+    assert!(kind == MetadataKind::Compressed);
+    let mut compressed = tcx.metadata_encoding_version();
+    DeflateEncoder::new(&mut compressed, Compression::fast())
+        .write_all(&metadata.raw_data).unwrap();
+
+    artifact.declare(".rustc", faerie::Decl::section(faerie::SectionKind::Data)).unwrap();
+    artifact.define_with_symbols(".rustc", compressed, {
+        let mut map = std::collections::BTreeMap::new();
+        // FIXME implement faerie elf backend section custom symbols
+        // For MachO this is necessary to prevent the linker from throwing away the .rustc section,
+        // but for ELF it isn't.
+        if tcx.sess.target.target.options.is_like_osx {
+            map.insert(rustc::middle::exported_symbols::metadata_symbol_name(tcx), 0);
+        }
+        map
+    }).unwrap();
+
+    metadata
 }

src/metadata.rs

@@ -5,7 +5,7 @@ rm -r target/out || true
 mkdir -p target/out/clif
 
 echo "[BUILD] mini_core"
-$RUSTC example/mini_core.rs --crate-name mini_core --crate-type lib
+$RUSTC example/mini_core.rs --crate-name mini_core --crate-type dylib
 
 echo "[BUILD] example"
 $RUSTC example/example.rs --crate-type lib