KFF

Author: marekkokot

kmc_tools/kff_db_reader.h

@@ -13,7 +13,6 @@
 
 //TODO KFF: for dump operation encoding is important: implement it!
 //TODO KFF: considef minimizers sections
-//TODO KFF: only blocks with given k value must be prepared before -> for now maybe just reject KFF with multiple k values
 
 //Forward declaration
 template<unsigned SIZE> class CKFFDbReaderSorted;

kmc_tools/kff_db_writer.h

@@ -55,6 +55,18 @@ template<unsigned SIZE> class CKFFDbWriter : public CDbWriter<SIZE>
 			store_buff();
 		kff_writer.FinishSection();		
 	}
+
+	bool canonical()
+	{
+		auto& headers = CConfig::GetInstance().headers;
+		bool both_stands = true;
+
+		for (auto& input : headers)
+			both_stands = both_stands && input.both_strands; //if any input database is in both strands, output is also in both strands
+
+		return both_stands;
+	}
+
 public:
 
 	CKFFDbWriter(CBundle<SIZE>* bundle, COutputDesc& output_desc) :
@@ -64,9 +76,11 @@ template<unsigned SIZE> class CKFFDbWriter : public CDbWriter<SIZE>
 		bundle(bundle),
 		kff_writer(
 			output_desc.file_src + ".kff",
-			true /*TODO KFF: set this basing on inputs! */,
+			canonical(),
 			CConfig::GetInstance().kmer_len,
-			counter_size
+			counter_size,
+			output_desc.cutoff_min,
+			output_desc.cutoff_max
 			/*TODO KFF: store encoding info!*/),
 		output_desc(output_desc)
 	{

kmc_tools/kff_info_reader.cpp

@@ -29,7 +29,6 @@ CKFFInfoReader::CKFFInfoReader(const std::string& path)
 	if (!file)
 		throw std::runtime_error("Error: cannot open file " + path);
 
-	
 	// Check markers
 	char marker[4];
 	marker[3] = '\0';
@@ -42,8 +41,6 @@ CKFFInfoReader::CKFFInfoReader(const std::string& path)
 	if (strncmp(marker, "KFF", 3) != 0)
 		throw std::runtime_error("Error: missing KFF marker at the end of file " + path);
 
-	//TODO KFF: index may be at the begining, so maybe there is no need to read footer
-
 	my_fseek(file, -23, SEEK_END);
 	char footer_size_str[12];
 	fread(footer_size_str, 1, 12, file);
@@ -73,7 +70,6 @@ CKFFInfoReader::CKFFInfoReader(const std::string& path)
 
 		//std::cerr << "footer nb_vars: " << nb_vars << "\n";
 
-		
 		for (uint64_t i = 0; i < nb_vars; ++i)
 		{
 			auto name = ReadVarName();
@@ -82,6 +78,8 @@ CKFFInfoReader::CKFFInfoReader(const std::string& path)
 			LoadBigEndian(tmp.data(), val);
 			//std::cerr << name << ": " << val << "\n";
 
+			kff_file_struct.footer[name] = val;
+
 			if (name == "first_index")
 				first_index = val;
 		}
@@ -95,14 +93,13 @@ CKFFInfoReader::CKFFInfoReader(const std::string& path)
 	fread(tmp.data(), 1, 1, file);
 	LoadBigEndian(tmp.data(), ver_minor);
 
-	uint8_t encoding;
+
 	fread(tmp.data(), 1, 1, file);
-	LoadBigEndian(tmp.data(), encoding);
+	LoadBigEndian(tmp.data(), kff_file_struct.encoding);
 
-	uint8_t unique;
 	fread(tmp.data(), 1, 1, file);
-	LoadBigEndian(tmp.data(), unique);
-	if (unique == 0)
+	LoadBigEndian(tmp.data(), kff_file_struct.all_unique);
+	if (kff_file_struct.all_unique == 0)
 		throw std::runtime_error("Error: only unique k-mers in KFF file are supported, file " + path);
 
 	uint8_t canonical;
@@ -130,7 +127,6 @@ CKFFInfoReader::CKFFInfoReader(const std::string& path)
 	if (first_index == std::numeric_limits<uint64_t>::max())
 		throw std::runtime_error("Error: no first_index in the footer and first section is not an index, file: " + path);
 
-	//std::vector<std::pair<char, int64_t>> index_pairs;
 
 	while (first_index)
 	{
@@ -181,7 +177,6 @@ CKFFInfoReader::CKFFInfoReader(const std::string& path)
 			throw std::runtime_error("Error: KFF index is inconsistent with file content");
 	}
 
-	//some reading
 	for (auto e : index)
 	{
 		my_fseek(file, e.section_pos, SEEK_SET);
@@ -227,6 +222,8 @@ void CKFFInfoReader::ReadVariableSection()
 			section.data_size = val;
 		else if (var_name == "m")
 			section.minimizer_size = val;
+		else if (var_name == "ordered")
+			section.ordered = val;
 	}
 	if (kff_file_struct.scopes.size())
 	{

kmc_tools/kff_info_reader.h

@@ -2,10 +2,9 @@
 #include <string>
 #include <vector>
 #include <limits>
+#include <string>
+#include <map>
 
-/*
-TODO KFF: add checking if section is ordered!!!!
-*/
 
 template<typename T>
 void LoadBigEndian(const uint8_t* buff, T& data)
@@ -42,12 +41,16 @@ struct CKFFVariables
 	uint64_t data_size = std::numeric_limits<uint64_t>::max(); //counter size
 	uint64_t minimizer_size = std::numeric_limits<uint64_t>::max();
 	uint64_t max_in_block = std::numeric_limits<uint64_t>::max();
+	bool ordered;
 	std::vector<CKFFDataSection> data_sections;
 };
 
 struct CKFFFileStruct
 {	
+	std::map<std::string, uint64_t> footer;
 	bool both_strands;	
+	uint8_t encoding;
+	uint8_t all_unique;
 	std::vector<CKFFVariables> scopes;
 };
 

kmc_tools/kmc1_db_writer.h

@@ -330,9 +330,10 @@ template<unsigned SIZE> void CKMC1DbWriter<SIZE>::finish_writing()
 	write_header_part(output_desc.cutoff_max);
 	write_header_part(added_kmers);
 
-	bool both_stands = false;
+	bool both_stands = true;
+
 	for (auto& input : config.headers)
-		both_stands = both_stands || input.both_strands; //if any input database is in both strands, output is also in both strands
+		both_stands = both_stands && input.both_strands; //if any input database is in not canonical, output is also not canonical
 
 	write_header_part(!both_stands);
 

kmc_tools/kmc_tools.cpp

@@ -158,6 +158,7 @@ template<unsigned SIZE> class CTools
 		{
 			std::cout << "This is KFF file, summary:\n";
 			std::cout << "canonical         :  " << (header.kff_file_struct.both_strands ? "yes" : "no") << "\n";
+			std::cout << "all k-mers unique :  " << (header.kff_file_struct.all_unique ? "yes" : "no") << "\n";
 			//TODO KFF: add encoding printing
 			std::set<uint64_t> k_values;
 			for (auto& e : header.kff_file_struct.scopes)
@@ -167,6 +168,12 @@ template<unsigned SIZE> class CTools
 				std::cout << "max           :  " << e.max_in_block << "\n";
 				if(e.minimizer_size != (std::numeric_limits<uint64_t>::max)())
 					std::cout << "m             :  " << e.minimizer_size << "\n";
+
+				std::cerr << "footer values:\n";
+				for (const auto& e : header.kff_file_struct.footer)
+				{
+					std::cerr << "\t" << e.first << "      :  " << e.second << "\n";
+				}
 				std::cout << "Data sections:\n";
 				uint64_t tot_nb_blocks{};
 				for (auto& s : e.data_sections)
@@ -178,8 +185,8 @@ template<unsigned SIZE> class CTools
 							type = "raw";
 							break;
 						case KFFDataSectionType::MINIMIZER:
-								type = "minimizer";
-								break;
+							type = "minimizer";
+							break;
 						default:
 						{
 							std::cerr << "Error: this should never happen, please contact authors: " << __FILE__ << "\t" << __LINE__ << "\n";

kmc_tools/kmc_tools.vcxproj.user

@@ -1,7 +1,7 @@
 <?xml version="1.0" encoding="utf-8"?>
 <Project ToolsVersion="Current" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
-    <LocalDebuggerCommandArguments>transform o sort ox -okff</LocalDebuggerCommandArguments>
+    <LocalDebuggerCommandArguments>info o</LocalDebuggerCommandArguments>
     <DebuggerFlavor>WindowsLocalDebugger</DebuggerFlavor>
   </PropertyGroup>
 </Project>

kmc_tools/kmer_file_header.cpp

@@ -131,12 +131,26 @@ void CKmerFileHeader::read_from_kff_file(const std::string& fname)
 	for (auto& s : kff_file_struct.scopes)
 	{
 		k_values.insert(s.kmer_size);
+		if (!s.ordered)
+		{
+			std::cerr << "Error: kmc_tools requires all KFF sections to be ordered\n";
+			exit(1);
+		}
+
+		for (auto& section: s.data_sections)
+			if (!(section.type == KFFDataSectionType::RAW))
+			{
+				std::cerr << "Error: currently kmc_tools supports only raw KFF sections\n";
+				exit(1);
+			}
+
 		if (s.data_size > this->counter_size)
 			this->counter_size = s.data_size;
 	}
 
-	this->min_count = 1; //TODO KFF: try to read this values from KFF file, and if not present set defaults
-	this->max_count = std::numeric_limits<uint32>::max();
+	this->min_count = GetFromFooterOrDefault("min_count", 1);
+	this->max_count = GetFromFooterOrDefault("max_count", std::numeric_limits<uint32>::max());
+	this->both_strands = kff_file_struct.both_strands;
 
 	if (k_values.size() == 1)
 		kmer_len = *k_values.begin();

kmc_tools/kmer_file_header.h

@@ -26,6 +26,15 @@ struct CKmerFileHeader
 	bool is_kff_file(std::string& fname);
 	void read_from_kff_file(const std::string& fname);
 	CKFFFileStruct kff_file_struct;
+
+	uint64_t GetFromFooterOrDefault(const std::string& name, uint64_t default)
+	{
+		const auto& m = kff_file_struct.footer;
+		auto r = m.find(name);
+		if (r != m.end())
+			return r->second;
+		return default;
+	}
 public:
 	uint32 kmer_len = 0;
 	uint32 mode = 0;

kmer_counter/kb_completer.cpp

@@ -108,7 +108,7 @@ void CKmerBinCompleter::ProcessBinsFirstStage()
 		}
 		else if (output_type == OutputType::KFF)
 		{			
-			kff_writer = new CKFFWriter(file_name + ".kff", both_strands, kmer_len, counter_size);
+			kff_writer = new CKFFWriter(file_name + ".kff", both_strands, kmer_len, counter_size, cutoff_min, cutoff_max);
 		}
 		else
 		{