From e8e2b6adf21493349cf8a6238960cea57f778b65 Mon Sep 17 00:00:00 2001 From: vrancurel Date: Thu, 6 Dec 2018 18:20:27 -0800 Subject: [PATCH] external C API baseline - compat w/ liberasurecode --- benchmark/benchmark.cpp | 8 +- src/CMakeLists.txt | 1 + src/fec_base.h | 357 +++++++++++++++++++++++++++++++++---- src/misc.cpp | 53 ++++++ src/misc.h | 6 + src/property.h | 38 ++++ src/quadiron_c.cpp | 382 ++++++++++++++++++++++++++++++++++++++++ src/quadiron_c.h | 144 +++++++++++++++ test/ec_driver.cpp | 8 +- 9 files changed, 952 insertions(+), 45 deletions(-) create mode 100644 src/quadiron_c.cpp create mode 100644 src/quadiron_c.h diff --git a/benchmark/benchmark.cpp b/benchmark/benchmark.cpp index de8b2ab9..332afe64 100644 --- a/benchmark/benchmark.cpp +++ b/benchmark/benchmark.cpp @@ -478,9 +478,9 @@ bool Benchmark::encode() reset_c_streams(); if (operation_on_packet) - fec->encode_packet(*d_streams, *c_streams, c_props); + fec->encode_streams_vertical(*d_streams, *c_streams, c_props); else - fec->encode_bufs(*d_streams, *c_streams, c_props); + fec->encode_streams_horizontal(*d_streams, *c_streams, c_props); // update stats enc_stats->add(fec->total_enc_usec); @@ -506,14 +506,14 @@ bool Benchmark::decode() reset_r_streams(); if (operation_on_packet) { - if (!fec->decode_packet( + if (!fec->decode_streams_vertical( d_streams_shuffled, c_streams_shuffled, c_props_shuffled, *r_streams)) return false; } else { - if (!fec->decode_bufs( + if (!fec->decode_streams_horizontal( d_streams_shuffled, c_streams_shuffled, c_props_shuffled, diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 54f48c5e..94421bca 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -35,6 +35,7 @@ set(LIB_SRC ${SOURCE_DIR}/gf_nf4.cpp ${SOURCE_DIR}/gf_ring.cpp ${SOURCE_DIR}/property.cpp + ${SOURCE_DIR}/quadiron_c.cpp CACHE INTERNAL diff --git a/src/fec_base.h b/src/fec_base.h index f28c1c25..87e46696 100644 --- a/src/fec_base.h +++ b/src/fec_base.h @@ -176,17 +176,17 @@ class FecCode { bool read_pkt(char* pkt, std::istream& stream); bool write_pkt(char* pkt, std::ostream& stream); - void encode_bufs( + void encode_streams_horizontal( std::vector input_data_bufs, std::vector output_parities_bufs, std::vector& output_parities_props); - void encode_packet( + void encode_streams_vertical( std::vector input_data_bufs, std::vector output_parities_bufs, std::vector& output_parities_props); - bool decode_bufs( + bool decode_streams_horizontal( std::vector input_data_bufs, std::vector input_parities_bufs, const std::vector& input_parities_props, @@ -197,12 +197,27 @@ class FecCode { size_t size = 0, vec::Buffers* output = nullptr); - bool decode_packet( + bool decode_streams_vertical( std::vector input_data_bufs, std::vector input_parities_bufs, const std::vector& input_parities_props, std::vector output_data_bufs); + void encode_blocks_vertical( + std::vector data_bufs, + std::vector parities_bufs, + std::vector& parities_props, + std::vector wanted_idxs, + size_t block_size_bytes); + + bool decode_blocks_vertical( + std::vector data_bufs, + std::vector parities_bufs, + const std::vector& parities_props, + std::vector missing_idxs, + std::vector wanted_idxs, + size_t block_size_bytes); + const gf::Field& get_gf() { return *gf; @@ -379,19 +394,18 @@ inline bool FecCode::write_pkt(char* pkt, std::ostream& stream) return static_cast(stream.write(pkt, buf_size)); } -/** - * Encode buffers +/** Encode streams * - * @param input_data_bufs must be exactly n_data - * @param output_parities_bufs must be exactly get_n_outputs() (set nullptr when - * not missing/wanted) - * @param output_parities_props must be exactly get_n_outputs() specific - * properties that the called is supposed to store along with parities + * @param input_data_bufs vector size must be exactly n_data + * @param output_parities_bufs vector size must be exactly n_parities + * (set entries no nullptr when not wanted) + * @param output_parities_props vector size must be exactly n_parities + * (set entries no nullptr when not wanted) * * @note all streams must be of equal size */ template -void FecCode::encode_bufs( +void FecCode::encode_streams_horizontal( std::vector input_data_bufs, std::vector output_parities_bufs, std::vector& output_parities_props) @@ -449,7 +463,7 @@ void FecCode::encode_bufs( } template -void FecCode::encode_packet( +void FecCode::encode_streams_vertical( std::vector input_data_bufs, std::vector output_parities_bufs, std::vector& output_parities_props) @@ -522,25 +536,23 @@ void FecCode::encode_packet( } } -/** - * Decode buffers +/** Decode streams * - * @param input_data_bufs if SYSTEMATIC must be exactly n_data otherwise it is - * unused (use nullptr when missing) - * @param input_parities_bufs if SYSTEMATIC must be exactly n_parities otherwise - * get_n_outputs() (use nullptr when missing) - * @param input_parities_props if SYSTEMATIC must be exactly n_parities - * otherwise get_n_outputs() caller is supposed to provide specific information - * bound to parities - * @param output_data_bufs must be exactly n_data (use nullptr when not - * missing/wanted) + * @param input_data_bufs vector size must be exactly n_data + * (set entries to nullptr when missing) + * @param input_parities_bufs vector size must be exactly n_parities + * (set entries to nullptr when missing) + * @param input_parities_props vector size must be exactly n_parities + * (set entries to nullptr when missing) + * @param output_data_bufs vector size must be exactly n_data + * (set entries to nullptr when not wanted) * * @note All streams must be of equal size * * @return true if decode succeded, else false */ template -bool FecCode::decode_bufs( +bool FecCode::decode_streams_horizontal( std::vector input_data_bufs, std::vector input_parities_bufs, const std::vector& input_parities_props, @@ -848,25 +860,23 @@ void FecCode::decode_apply( /********** Decoding over vec::PolyBuf **********/ -/** - * Decode buffers +/** Decode streams * - * @param input_data_bufs if SYSTEMATIC must be exactly n_data otherwise it is - * unused (use nullptr when missing) - * @param input_parities_bufs if SYSTEMATIC must be exactly n_parities otherwise - * get_n_outputs() (use nullptr when missing) - * @param input_parities_props if SYSTEMATIC must be exactly n_parities - * otherwise get_n_outputs() caller is supposed to provide specific information - * bound to parities - * @param output_data_bufs must be exactly n_data (use nullptr when not - * missing/wanted) + * @param input_data_bufs vector size must be exactly n_data + * (set entries to nullptr when missing) + * @param input_parities_bufs vector size must be exactly n_parities + * (set entries to nullptr when missing) + * @param input_parities_props vector size must be exactly n_parities + * (set entries to nullptr when missing) + * @param output_data_bufs vector size must be exactly n_data + * (set entries to nullptr when not wanted) * * @pre All streams must be of equal size * * @return true if decode succeeded, else false */ template -bool FecCode::decode_packet( +bool FecCode::decode_streams_vertical( std::vector input_data_bufs, std::vector input_parities_bufs, const std::vector& input_parities_props, @@ -1005,6 +1015,279 @@ bool FecCode::decode_packet( return true; } +/** Encode blocks + * + * @param data_bufs vector size must be exactly n_data + * (set entries to nullptr when missing) + * @param parities_bufs vector size must be exactly n_parities + * (set entries to nullptr when not wanted) + * @param parities_props vector size must be exactly n_parities + * (set entries to nullptr when not wanted) + * @param wanted_idxs bool array of missing_idxs of len n_parities indicating + * wanted (value 1) or not wanted fragments (value 0) - wanted blocks MUST BE + * allocated by caller + * @param block_size_bytes the block size in bytes + * + * @pre All blocks must be of equal size + */ +template +void FecCode::encode_blocks_vertical( + std::vector data_bufs, + std::vector parities_bufs, + std::vector& parities_props, + std::vector wanted_idxs, + size_t block_size_bytes) +{ + assert(data_bufs.size() == n_data); + assert(parities_bufs.size() == n_outputs); + assert(parities_props.size() == n_outputs); + + // clear property vectors + for (auto& props : parities_props) { + props.clear(); + } + + size_t offset = 0; + size_t block_size = block_size_bytes / word_size; + + // vector of buffers storing data read from chunk + vec::Buffers words_char(n_data, buf_size); + const std::vector words_mem_char = words_char.get_mem(); + // vector of buffers storing data that are performed in encoding, i.e. FFT + vec::Buffers words(n_data, pkt_size); + const std::vector words_mem_T = words.get_mem(); + + int output_len = get_n_outputs(); + + // vector of buffers storing data that are performed in encoding, i.e. FFT + vec::Buffers output(output_len, pkt_size); + const std::vector output_mem_T = output.get_mem(); + // vector of buffers storing data in output chunk + vec::Buffers output_char(output_len, buf_size); + const std::vector output_mem_char = output_char.get_mem(); + + reset_stats_enc(); + + while (offset < block_size) { + size_t remain_size = block_size - offset; + size_t copy_size = std::min(pkt_size, remain_size); + for (unsigned i = 0; i < n_data; i++) { + memcpy( + reinterpret_cast(words_mem_char.at(i)), + data_bufs[i] + offset * word_size, + copy_size * word_size); + } + + // Zero-out trailing part of data + if (copy_size < pkt_size) { + const size_t copy_bytes = copy_size * word_size; + const size_t trailing_bytes = buf_size - copy_bytes; + for (unsigned i = 0; i < n_data; i++) { + memset( + reinterpret_cast(words_mem_char.at(i)) + copy_bytes, + 0, + trailing_bytes); + } + } + + vec::pack( + words_mem_char, words_mem_T, n_data, pkt_size, word_size); + + timeval t1 = tick(); + uint64_t start = hw_timer(); + encode(output, parities_props, offset, words); + uint64_t end = hw_timer(); + uint64_t t2 = hrtime_usec(t1); + + total_enc_usec += t2; + total_encode_cycles += (end - start) / (copy_size * word_size); + n_encode_ops++; + + vec::unpack( + output_mem_T, output_mem_char, output_len, pkt_size, word_size); + + for (unsigned i = 0; i < n_outputs; i++) { + if (wanted_idxs[i]) { + memcpy( + parities_bufs[i] + offset * word_size, + reinterpret_cast(output_mem_char.at(i)), + copy_size * word_size); + } + } + offset += pkt_size; + } +} + +/** Decode blocks + * + * @param data_bufs vector size must be exactly n_data + * (set entries to nullptr when missing) + * @param parities_bufs vector size must be exactly n_parities + * (set entries to nullptr when missing) + * @param parities_props vector size must be exactly n_parities + * (set entries to nullptr when missing) + * @param missing_idxs array of missing indexes of vector size code_len + * indicating presence (value 1) or absence of fragments (value 0). It applies + * for both data and parities + * - non missing blocks MUST BE allocated by caller + * @param wanted_idxs bool array of wanted indexes of vector size n_data + * indicating wanted (value 1) or not wanted fragments (value 0). It applies + * only for data + * - wanted blocks MUST BE allocated + * by caller + * @param block_size_bytes the block size in bytes + * + * @pre All blocks must be of equal size + * + * @return true if decode succeeded, else false + */ +template +bool FecCode::decode_blocks_vertical( + std::vector data_bufs, + std::vector parities_bufs, + const std::vector& parities_props, + std::vector missing_idxs, + std::vector wanted_idxs, + size_t block_size_bytes) +{ + size_t offset = 0; + size_t block_size = block_size_bytes / word_size; + + unsigned fragment_index = 0; + unsigned parity_index = 0; + unsigned avail_data_nb = 0; + + if (type == FecType::SYSTEMATIC) { + assert(data_bufs.size() == n_data); + } + assert(parities_bufs.size() == n_outputs); + assert(parities_props.size() == n_outputs); + + // ids of received fragments, from 0 to codelen-1 + vec::Vector fragments_ids(*(this->gf), n_data); + + if (type == FecType::SYSTEMATIC) { + for (unsigned i = 0; i < n_data; i++) { + if (!missing_idxs[i]) { + decode_add_data(fragment_index, i); + fragments_ids.set(fragment_index, i); + fragment_index++; + } + avail_data_nb = fragment_index; + // data is in clear so nothing to do + if (fragment_index == n_data) + return true; + } + } + + vec::Vector avail_parity_ids(*(this->gf), n_data - avail_data_nb); + + if (fragment_index < n_data) { + // finish with parities available + for (unsigned i = 0; i < n_outputs; i++) { + unsigned j = (type == FecType::SYSTEMATIC) ? n_data + i : i; + if (!missing_idxs[j]) { + decode_add_parities(fragment_index, i); + fragments_ids.set(fragment_index, j); + avail_parity_ids.set(parity_index, i); + fragment_index++; + parity_index++; + // stop when we have enough parities + if (fragment_index == n_data) + break; + } + } + // unable to decode + if (fragment_index < n_data) + return false; + } + fragments_ids.sort(); + + decode_build(); + + // vector of buffers storing data read from chunk + vec::Buffers words_char(n_data, buf_size); + const std::vector words_mem_char = words_char.get_mem(); + // vector of buffers storing data that are performed in encoding, i.e. FFT + vec::Buffers words(n_data, pkt_size); + const std::vector words_mem_T = words.get_mem(); + + int output_len = n_data; + + // vector of buffers storing data that are performed in decoding, i.e. FFT + vec::Buffers output(output_len, pkt_size); + const std::vector output_mem_T = output.get_mem(); + // vector of buffers storing data in output chunk + vec::Buffers output_char(output_len, buf_size); + const std::vector output_mem_char = output_char.get_mem(); + + std::unique_ptr> context = + init_context_dec(fragments_ids, pkt_size, &output); + + reset_stats_dec(); + + while (offset < block_size) { + size_t remain_size = block_size - offset; + size_t copy_size = std::min(pkt_size, remain_size); + if (type == FecType::SYSTEMATIC) { + for (unsigned i = 0; i < avail_data_nb; i++) { + unsigned data_idx = fragments_ids.get(i); + memcpy( + reinterpret_cast(words_mem_char.at(i)), + data_bufs[data_idx] + offset * word_size, + copy_size * word_size); + } + } + for (unsigned i = 0; i < n_data - avail_data_nb; ++i) { + unsigned parity_idx = avail_parity_ids.get(i); + memcpy( + reinterpret_cast(words_mem_char.at(avail_data_nb + i)), + parities_bufs[parity_idx] + offset * word_size, + copy_size * word_size); + } + + // Zero-out trailing part of data + if (copy_size < pkt_size) { + const size_t copy_bytes = copy_size * word_size; + const size_t trailing_bytes = buf_size - copy_bytes; + for (unsigned i = 0; i < n_data; i++) { + memset( + reinterpret_cast(words_mem_char.at(i)) + copy_bytes, + 0, + trailing_bytes); + } + } + + vec::pack( + words_mem_char, words_mem_T, n_data, pkt_size, word_size); + + timeval t1 = tick(); + uint64_t start = hw_timer(); + decode(*context, output, parities_props, offset, words); + uint64_t end = hw_timer(); + uint64_t t2 = hrtime_usec(t1); + + total_dec_usec += t2; + total_decode_cycles += (end - start) / word_size; + n_decode_ops++; + + vec::unpack( + output_mem_T, output_mem_char, output_len, pkt_size, word_size); + + for (unsigned i = 0; i < n_data; i++) { + if (wanted_idxs[i]) { + memcpy( + data_bufs[i] + offset * word_size, + reinterpret_cast(output_mem_char.at(i)), + copy_size * word_size); + } + } + offset += pkt_size; + } + + return true; +} + /** * Perform a Lagrange interpolation to find the coefficients of the * polynomial diff --git a/src/misc.cpp b/src/misc.cpp index de8267b6..8b3f5b80 100644 --- a/src/misc.cpp +++ b/src/misc.cpp @@ -27,6 +27,7 @@ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ +#include #include "misc.h" namespace std { @@ -77,3 +78,55 @@ std::ostream& operator<<(std::ostream& dest, __int128_t value) } } // namespace std + +namespace quadiron { + +std::ostream& hex_dump( + std::ostream& os, + const void* buffer, + std::size_t bufsize, + bool showPrintableChars) +{ + if (buffer == nullptr) { + return os; + } + auto oldFormat = os.flags(); + auto oldFillChar = os.fill(); + constexpr std::size_t maxline{32}; + // create a place to store text version of string + char renderString[maxline + 1]; + char* rsptr{renderString}; + // convenience cast + const unsigned char* buf{reinterpret_cast(buffer)}; + + for (std::size_t linecount = maxline; bufsize; --bufsize, ++buf) { + os << std::setw(2) << std::setfill('0') << std::hex + << static_cast(*buf) << ' '; + *rsptr++ = std::isprint(*buf) ? *buf : '.'; + if (--linecount == 0) { + *rsptr++ = '\0'; // terminate string + if (showPrintableChars) { + os << " | " << renderString; + } + os << '\n'; + rsptr = renderString; + linecount = std::min(maxline, bufsize); + } + } + // emit newline if we haven't already + if (rsptr != renderString) { + if (showPrintableChars) { + for (*rsptr++ = '\0'; rsptr != &renderString[maxline + 1]; + ++rsptr) { + os << " "; + } + os << " | " << renderString; + } + os << '\n'; + } + + os.fill(oldFillChar); + os.flags(oldFormat); + return os; +} +} // namespace quadiron diff --git a/src/misc.h b/src/misc.h index bb3e4044..86e711b1 100644 --- a/src/misc.h +++ b/src/misc.h @@ -80,6 +80,12 @@ static inline uint64_t hw_timer() return x; } +std::ostream& hex_dump( + std::ostream& os, + const void* buffer, + std::size_t bufsize, + bool showPrintableChars); + } // namespace quadiron #endif diff --git a/src/property.h b/src/property.h index 2fcaa61c..87f7861d 100644 --- a/src/property.h +++ b/src/property.h @@ -32,6 +32,7 @@ #define __QUAD_PROPERTY_H__ #include +#include #include #include #include @@ -75,6 +76,39 @@ class Properties { return props; } + /** + * Serialize properties into a buffer (FNT) + * + * @return 0 if OK, else -1 + */ + inline int fnt_serialize(uint32_t* dwords, unsigned n_dwords) + { + unsigned i = 0; + if (props.size() > n_dwords) { + return -1; + } + memset(dwords, 0xff, n_dwords * sizeof(uint32_t)); + for (auto& kv : props) { + dwords[i++] = static_cast(kv.first); + } + return 0; + } + + /** + * Deserialize properties from a buffer (FNT) + * + * @return 0 if OK, else -1 + */ + inline int fnt_deserialize(const uint32_t* dwords, unsigned n_dwords) + { + for (unsigned i = 0; i < n_dwords; i++) { + if (dwords[i] == 0xffffffff) + break; + add(static_cast(dwords[i]), 1); + } + return 0; + } + private: std::unordered_map props; @@ -82,6 +116,10 @@ class Properties { friend std::ostream& operator<<(std::ostream& os, const Properties& props); }; +class FntProperties : public Properties { + public: +}; + } // namespace quadiron #endif diff --git a/src/quadiron_c.cpp b/src/quadiron_c.cpp new file mode 100644 index 00000000..13833b81 --- /dev/null +++ b/src/quadiron_c.cpp @@ -0,0 +1,382 @@ +/* -*- mode: c++ -*- */ +/* + * Copyright 2017-2018 Scality + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ +#include "property.h" +#include "quadiron.h" +#include "quadiron_c.h" + +extern "C" { + +struct QuadironFnt32* +quadiron_fnt32_new(int word_size, int n_data, int n_parities, int systematic) +{ + const size_t pkt_size = 1024; + + if (word_size == 1 || word_size == 2) { + return reinterpret_cast( + new quadiron::fec::RsFnt( + systematic ? quadiron::fec::FecType::SYSTEMATIC + : quadiron::fec::FecType::NON_SYSTEMATIC, + word_size, + n_data, + n_parities, + pkt_size)); + } + + return nullptr; +} + +void quadiron_fnt32_delete(struct QuadironFnt32* fecp) +{ + delete reinterpret_cast*>(fecp); +} + +int quadiron_fnt32_get_metadata_size( + struct QuadironFnt32* /* fecp */, + size_t block_size) +{ + /* + * We assume that a special value of 65536 may occur uniformly. + * We count 4 bytes per special value. + * We see large and roundup by 16 items. + */ + return ((block_size / 65536) + 16) * 4; +} + +int quadiron_fnt32_encode( + struct QuadironFnt32* fecp, + uint8_t** data, + uint8_t** parity, + int* missing_idxs, + size_t block_size) +{ + quadiron::fec::RsFnt* fec = + reinterpret_cast*>(fecp); + std::vector data_vec(fec->n_data, nullptr); + std::vector parities_vec(fec->n_outputs, nullptr); + std::vector parities_props(fec->n_outputs); + std::vector wanted_idxs_vec(fec->code_len); + int metadata_size = quadiron_fnt32_get_metadata_size(fecp, block_size); + + for (unsigned i = 0; i < fec->code_len; i++) { + wanted_idxs_vec[i] = missing_idxs[i] ? true : false; + } + + if (fec->type == quadiron::fec::FecType::SYSTEMATIC) { + for (unsigned i = 0; i < fec->n_data; i++) { + data_vec[i] = data[i] + metadata_size; + } + for (unsigned i = 0; i < fec->n_parities; i++) { + parities_vec[i] = parity[i] + metadata_size; + } + } else { + for (unsigned i = 0; i < fec->n_data; i++) { + data_vec[i] = data[i] + metadata_size; + parities_vec[i] = data[i] + metadata_size; + } + for (unsigned i = 0; i < fec->n_parities; i++) { + parities_vec[fec->n_data + i] = parity[i] + metadata_size; + } + } + + fec->encode_blocks_vertical( + data_vec, parities_vec, parities_props, wanted_idxs_vec, block_size); + + if (fec->type == quadiron::fec::FecType::SYSTEMATIC) { + for (unsigned i = 0; i < fec->n_parities; i++) { + uint32_t* metadata = reinterpret_cast(parity[i]); + int ret = + parities_props[i].fnt_serialize(metadata, metadata_size / 4); + if (ret == -1) { + return -1; + } + } + } else { + for (unsigned i = 0; i < fec->n_data; i++) { + uint32_t* metadata = reinterpret_cast(data[i]); + int ret = + parities_props[i].fnt_serialize(metadata, metadata_size / 4); + if (ret == -1) { + return -1; + } + } + for (unsigned i = 0; i < fec->n_parities; i++) { + uint32_t* metadata = reinterpret_cast(parity[i]); + int ret = parities_props[fec->n_data + i].fnt_serialize( + metadata, metadata_size / 4); + if (ret == -1) { + return -1; + } + } + } + + return 0; +} + +int quadiron_fnt32_decode( + struct QuadironFnt32* fecp, + uint8_t** data, + uint8_t** parity, + int* missing_idxs, + size_t block_size) +{ + quadiron::fec::RsFnt* fec = + reinterpret_cast*>(fecp); + std::vector data_vec(fec->n_data, nullptr); + std::vector parities_vec(fec->n_outputs, nullptr); + std::vector parities_props(fec->n_outputs); + std::vector missing_idxs_vec( + missing_idxs, missing_idxs + fec->code_len); + std::vector wanted_idxs_vec(fec->n_data, true); + int metadata_size = quadiron_fnt32_get_metadata_size(fecp, block_size); + bool res; + + if (fec->type == quadiron::fec::FecType::SYSTEMATIC) { + for (unsigned i = 0; i < fec->n_data; i++) { + data_vec[i] = data[i] + metadata_size; + } + for (unsigned i = 0; i < fec->n_parities; i++) { + if (!missing_idxs[fec->n_data + i]) { + parities_vec[i] = parity[i] + metadata_size; + uint32_t* metadata = reinterpret_cast(parity[i]); + int ret = parities_props[i].fnt_deserialize( + metadata, metadata_size / 4); + if (ret == -1) + return -1; + } + } + } else { + for (unsigned i = 0; i < fec->n_data; i++) { + if (!missing_idxs[i]) { + parities_vec[i] = data[i] + metadata_size; + uint32_t* metadata = reinterpret_cast(data[i]); + int ret = parities_props[i].fnt_deserialize( + metadata, metadata_size / 4); + if (ret == -1) + return -1; + } + data_vec[i] = data[i] + metadata_size; + } + for (unsigned i = 0; i < fec->n_parities; i++) { + if (!missing_idxs[fec->n_data + i]) { + parities_vec[fec->n_data + i] = parity[i] + metadata_size; + uint32_t* metadata = reinterpret_cast(parity[i]); + int ret = parities_props[fec->n_data + i].fnt_deserialize( + metadata, metadata_size / 4); + if (ret == -1) + return -1; + } + } + } + + res = fec->decode_blocks_vertical( + data_vec, + parities_vec, + parities_props, + missing_idxs_vec, + wanted_idxs_vec, + block_size); + if (!res) + return -1; + + return 0; +} + +int quadiron_fnt32_reconstruct( + struct QuadironFnt32* fecp, + uint8_t** data, + uint8_t** parity, + int* missing_idxs, + unsigned int destination_idx, + size_t block_size) +{ + quadiron::fec::RsFnt* fec = + reinterpret_cast*>(fecp); + std::vector data_vec(fec->n_data); + std::vector parities_vec(fec->n_outputs); + std::vector parities_props(fec->n_outputs); + std::vector missing_idxs_vec( + missing_idxs, missing_idxs + fec->code_len); + std::vector wanted_data_vec(fec->n_data, false); + std::vector wanted_idxs_vec(fec->n_outputs, false); + int metadata_size = quadiron_fnt32_get_metadata_size(fecp, block_size); + bool res; + + if (fec->type == quadiron::fec::FecType::SYSTEMATIC) { + for (unsigned i = 0; i < fec->n_data; i++) { + data_vec[i] = data[i] + metadata_size; + } + for (unsigned i = 0; i < fec->n_parities; i++) { + if (!missing_idxs[fec->n_data + i]) { + uint32_t* metadata = reinterpret_cast(parity[i]); + int ret = parities_props[i].fnt_deserialize( + metadata, metadata_size / 4); + if (ret == -1) + return -1; + } + parities_vec[i] = parity[i] + metadata_size; + } + } else { + for (unsigned i = 0; i < fec->n_data; i++) { + if (!missing_idxs[i]) { + uint32_t* metadata = reinterpret_cast(data[i]); + int ret = parities_props[i].fnt_deserialize( + metadata, metadata_size / 4); + if (ret == -1) + return -1; + } + parities_vec[i] = data[i] + metadata_size; + data_vec[i] = data[i] + metadata_size; + } + for (unsigned i = 0; i < fec->n_parities; i++) { + if (!missing_idxs[fec->n_data + i]) { + uint32_t* metadata = reinterpret_cast(parity[i]); + int ret = parities_props[fec->n_data + i].fnt_deserialize( + metadata, metadata_size / 4); + if (ret == -1) + return -1; + } + parities_vec[fec->n_data + i] = parity[i] + metadata_size; + } + } + + int need_decode = 0; + + if (fec->type == quadiron::fec::FecType::SYSTEMATIC) { + /* + * Easy case where the target is a data then simply decode + */ + if (destination_idx < fec->n_data) { + wanted_idxs_vec[destination_idx] = true; + res = fec->decode_blocks_vertical( + data_vec, + parities_vec, + parities_props, + missing_idxs_vec, + wanted_idxs_vec, + block_size); + if (!res) { + return -1; + } + return 0; + } + } + + /* + * At this point we want a parity to be reconstructed + * If systematic we may need to decode if a data is missing. + * If non-systematic we always need to decode + */ + std::vector> blocks(fec->n_data); + if (fec->type == quadiron::fec::FecType::SYSTEMATIC) { + for (unsigned i = 0; i < fec->n_data; i++) { + if (missing_idxs[i]) { + need_decode = 1; + wanted_data_vec[i] = true; + blocks.at(i).resize(block_size); + data_vec[i] = blocks.at(i).data(); + } + } + } else { + need_decode = 1; + for (unsigned i = 0; i < fec->n_data; i++) { + wanted_data_vec[i] = true; + blocks.at(i).resize(block_size); + data_vec[i] = blocks.at(i).data(); + } + } + + if (need_decode) { + res = fec->decode_blocks_vertical( + data_vec, + parities_vec, + parities_props, + missing_idxs_vec, + wanted_data_vec, + block_size); + if (!res) { + return -1; + } + } + + /* + * At this point we have all data blocks. + */ + if (fec->type == quadiron::fec::FecType::SYSTEMATIC) { + wanted_idxs_vec[destination_idx - fec->n_data] = true; + } else { + wanted_idxs_vec[destination_idx] = true; + } + + fec->encode_blocks_vertical( + data_vec, parities_vec, parities_props, wanted_idxs_vec, block_size); + + if (fec->type == quadiron::fec::FecType::SYSTEMATIC) { + for (unsigned i = 0; i < fec->n_parities; i++) { + if (i == destination_idx - fec->n_data) { + uint32_t* metadata = reinterpret_cast(parity[i]); + int ret = parities_props[i].fnt_serialize( + metadata, metadata_size / 4); + if (ret == -1) { + return -1; + } + } + } + } else { + for (unsigned i = 0; i < fec->n_data; i++) { + if (i == destination_idx) { + uint32_t* metadata = reinterpret_cast(data[i]); + int ret = parities_props[i].fnt_serialize( + metadata, metadata_size / 4); + if (ret == -1) { + return -1; + } + } + } + for (unsigned i = 0; i < fec->n_parities; i++) { + if (fec->n_data + i == destination_idx) { + uint32_t* metadata = reinterpret_cast(parity[i]); + int ret = parities_props[fec->n_data + i].fnt_serialize( + metadata, metadata_size / 4); + if (ret == -1) { + return -1; + } + } + } + } + + return 0; +} + +void quadiron_hex_dump(char* buf, int buf_size) +{ + quadiron::hex_dump(std::cerr, buf, buf_size, true); +} +} diff --git a/src/quadiron_c.h b/src/quadiron_c.h new file mode 100644 index 00000000..5ff6311c --- /dev/null +++ b/src/quadiron_c.h @@ -0,0 +1,144 @@ +/* -*- mode: c++ -*- */ +/* + * Copyright 2017-2018 Scality + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ +#ifndef __QUAD_QUADIRON_C_H__ +#define __QUAD_QUADIRON_C_H__ + +#ifdef __cplusplus +extern "C" { +#endif + +/** Create FNT FEC - This FEC is relatively complex because it requires storing + * a metadata header. + * + * @param[in] word_size FNT only supports 1 or 2 + * @param[in] n_data number of data fragments + * @param[in] n_parities number of parity fragments + * @param[in] systematic if 1 then the code is systematic otherwise + * non-systematic + * + * @return the FEC instance pointer + */ +struct QuadironFnt32* +quadiron_fnt32_new(int word_size, int n_data, int n_parities, int systematic); + +/** Delete FEC + * + * @param[in,out] fecp the FEC instance pointer + */ +void quadiron_fnt32_delete(struct QuadironFnt32* fecp); + +/** Return metadata size + * + * FNT requires to store specific information in headers and therefore caller + * needs to preallocate metadata_size in blocks. + * + * @param[in] fecp the FEC instance + * @param[in] block_size the metadata_size is computed acc/to the block_size + * + * @return the metadata size + */ +int quadiron_fnt32_get_metadata_size( + struct QuadironFnt32* fecp, + size_t block_size); + +/** Encode blocks + * + * @param[in] fecp the FEC instance + * @param[in] data must be exactly n_data + * (set entries to NULL when missing) + * buffers must allocate block_size + metadata_size + * @param[out] parity must be exactly n_parities + * (set entries to NULL when missing) + * @param[in] missing_idxs array of missing_idxs of len code_len indicating + * presence (value 1) or absence of fragments (value 0) + * @param[in] block_size the block size in bytes + * + * @return 1 if decode succeeded, else 0 + */ +int quadiron_fnt32_encode( + struct QuadironFnt32* fecp, + uint8_t** data, + uint8_t** parity, + int* missing_idxs, + size_t block_size); + +/** Decode blocks + * + * @param[in] fecp the FEC instance + * @param[in,out] data must be exactly n_data + * (set entries to NULL when missing) + * - callers must allocate block_size + metadata_size + * @param[in] parity must be exactly n_parities + * (set entries to NULL when missing) + * - callers must allocate block_size + metadata_size + * @param[in] missing_idxs array of missing_idxs of len code_len indicating + * presence (value 1) or absence of fragments (value 0) + * @param[in] block_size the block size in bytes + * + * @return 0 if decode succeeded, else -1 + */ +int quadiron_fnt32_decode( + struct QuadironFnt32* fecp, + uint8_t** data, + uint8_t** parity, + int* missing_idxs, + size_t block_size); + +/** Reconstruct block + * + * @param[in] fecp the FEC instance + * @param[in,out] data must be exactly n_data + * (set entries to NULL when missing) + * callers must allocate block_size + metadata_size + * @param[in,out] parity must be exactly n_parities - callers must allocate + * block_size + metadata_size (set entries to NULL when missing) + * @param[in] missing_idxs array of missing_idxs of len code_len indicating + * presence (value 1) or absence of fragments (value 0) + * @param[in] destination_idx index of fragment to reconstruct + * @param[in] block_size the block size in bytes + * + * @return 0 if reconstruct succeeded, else -1 + */ +int quadiron_fnt32_reconstruct( + struct QuadironFnt32* fecp, + uint8_t** data, + uint8_t** parity, + int* missing_idxs, + unsigned int destination_idx, + size_t block_size); + +void quadiron_hex_dump(char* buf, int buf_size); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/test/ec_driver.cpp b/test/ec_driver.cpp index 8729a7f7..0166f848 100644 --- a/test/ec_driver.cpp +++ b/test/ec_driver.cpp @@ -144,9 +144,9 @@ void create_coding_files( } if (operation_on_packet) { - fec->encode_packet(d_files, c_files, c_props); + fec->encode_streams_vertical(d_files, c_files, c_props); } else { - fec->encode_bufs(d_files, c_files, c_props); + fec->encode_streams_horizontal(d_files, c_files, c_props); } for (unsigned i = 0; i < fec->n_data; i++) { @@ -244,9 +244,9 @@ bool repair_data_files( } if (operation_on_packet) { - fec->decode_packet(d_files, c_files, c_props, r_files); + fec->decode_streams_vertical(d_files, c_files, c_props, r_files); } else { - fec->decode_bufs(d_files, c_files, c_props, r_files); + fec->decode_streams_horizontal(d_files, c_files, c_props, r_files); } for (unsigned i = 0; i < fec->n_data; i++) {