diff --git a/benchmark/benchmark.cpp b/benchmark/benchmark.cpp index de8b2ab9..332afe64 100644 --- a/benchmark/benchmark.cpp +++ b/benchmark/benchmark.cpp @@ -478,9 +478,9 @@ bool Benchmark::encode() reset_c_streams(); if (operation_on_packet) - fec->encode_packet(*d_streams, *c_streams, c_props); + fec->encode_streams_vertical(*d_streams, *c_streams, c_props); else - fec->encode_bufs(*d_streams, *c_streams, c_props); + fec->encode_streams_horizontal(*d_streams, *c_streams, c_props); // update stats enc_stats->add(fec->total_enc_usec); @@ -506,14 +506,14 @@ bool Benchmark::decode() reset_r_streams(); if (operation_on_packet) { - if (!fec->decode_packet( + if (!fec->decode_streams_vertical( d_streams_shuffled, c_streams_shuffled, c_props_shuffled, *r_streams)) return false; } else { - if (!fec->decode_bufs( + if (!fec->decode_streams_horizontal( d_streams_shuffled, c_streams_shuffled, c_props_shuffled, diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 54f48c5e..94421bca 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -35,6 +35,7 @@ set(LIB_SRC ${SOURCE_DIR}/gf_nf4.cpp ${SOURCE_DIR}/gf_ring.cpp ${SOURCE_DIR}/property.cpp + ${SOURCE_DIR}/quadiron_c.cpp CACHE INTERNAL diff --git a/src/fec_base.h b/src/fec_base.h index f28c1c25..19dd6f26 100644 --- a/src/fec_base.h +++ b/src/fec_base.h @@ -176,17 +176,17 @@ class FecCode { bool read_pkt(char* pkt, std::istream& stream); bool write_pkt(char* pkt, std::ostream& stream); - void encode_bufs( + void encode_streams_horizontal( std::vector input_data_bufs, std::vector output_parities_bufs, std::vector& output_parities_props); - void encode_packet( + void encode_streams_vertical( std::vector input_data_bufs, std::vector output_parities_bufs, std::vector& output_parities_props); - bool decode_bufs( + bool decode_streams_horizontal( std::vector input_data_bufs, std::vector input_parities_bufs, const std::vector& input_parities_props, @@ -197,12 +197,27 @@ class FecCode { size_t size = 0, vec::Buffers* output = nullptr); - bool decode_packet( + bool decode_streams_vertical( std::vector input_data_bufs, std::vector input_parities_bufs, const std::vector& input_parities_props, std::vector output_data_bufs); + void encode_blocks_vertical( + std::vector data_bufs, + std::vector parities_bufs, + std::vector& parities_props, + std::vector wanted_idxs, + size_t block_size_bytes); + + bool decode_blocks_vertical( + std::vector data_bufs, + std::vector parities_bufs, + const std::vector& parities_props, + std::vector missing_idxs, + std::vector wanted_idxs, + size_t block_size_bytes); + const gf::Field& get_gf() { return *gf; @@ -379,19 +394,18 @@ inline bool FecCode::write_pkt(char* pkt, std::ostream& stream) return static_cast(stream.write(pkt, buf_size)); } -/** - * Encode buffers +/** Encode streams * - * @param input_data_bufs must be exactly n_data - * @param output_parities_bufs must be exactly get_n_outputs() (set nullptr when - * not missing/wanted) - * @param output_parities_props must be exactly get_n_outputs() specific - * properties that the called is supposed to store along with parities + * @param input_data_bufs vector size must be exactly n_data + * @param output_parities_bufs vector size must be exactly n_parities + * (set entries no nullptr when not wanted) + * @param output_parities_props vector size must be exactly n_parities + * (set entries no nullptr when not wanted) * * @note all streams must be of equal size */ template -void FecCode::encode_bufs( +void FecCode::encode_streams_horizontal( std::vector input_data_bufs, std::vector output_parities_bufs, std::vector& output_parities_props) @@ -449,7 +463,7 @@ void FecCode::encode_bufs( } template -void FecCode::encode_packet( +void FecCode::encode_streams_vertical( std::vector input_data_bufs, std::vector output_parities_bufs, std::vector& output_parities_props) @@ -522,25 +536,23 @@ void FecCode::encode_packet( } } -/** - * Decode buffers +/** Decode streams * - * @param input_data_bufs if SYSTEMATIC must be exactly n_data otherwise it is - * unused (use nullptr when missing) - * @param input_parities_bufs if SYSTEMATIC must be exactly n_parities otherwise - * get_n_outputs() (use nullptr when missing) - * @param input_parities_props if SYSTEMATIC must be exactly n_parities - * otherwise get_n_outputs() caller is supposed to provide specific information - * bound to parities - * @param output_data_bufs must be exactly n_data (use nullptr when not - * missing/wanted) + * @param input_data_bufs vector size must be exactly n_data + * (set entries to nullptr when missing) + * @param input_parities_bufs vector size must be exactly n_parities + * (set entries to nullptr when missing) + * @param input_parities_props vector size must be exactly n_parities + * (set entries to nullptr when missing) + * @param output_data_bufs vector size must be exactly n_data + * (set entries to nullptr when not wanted) * * @note All streams must be of equal size * * @return true if decode succeded, else false */ template -bool FecCode::decode_bufs( +bool FecCode::decode_streams_horizontal( std::vector input_data_bufs, std::vector input_parities_bufs, const std::vector& input_parities_props, @@ -848,25 +860,23 @@ void FecCode::decode_apply( /********** Decoding over vec::PolyBuf **********/ -/** - * Decode buffers +/** Decode streams * - * @param input_data_bufs if SYSTEMATIC must be exactly n_data otherwise it is - * unused (use nullptr when missing) - * @param input_parities_bufs if SYSTEMATIC must be exactly n_parities otherwise - * get_n_outputs() (use nullptr when missing) - * @param input_parities_props if SYSTEMATIC must be exactly n_parities - * otherwise get_n_outputs() caller is supposed to provide specific information - * bound to parities - * @param output_data_bufs must be exactly n_data (use nullptr when not - * missing/wanted) + * @param input_data_bufs vector size must be exactly n_data + * (set entries to nullptr when missing) + * @param input_parities_bufs vector size must be exactly n_parities + * (set entries to nullptr when missing) + * @param input_parities_props vector size must be exactly n_parities + * (set entries to nullptr when missing) + * @param output_data_bufs vector size must be exactly n_data + * (set entries to nullptr when not wanted) * * @pre All streams must be of equal size * * @return true if decode succeeded, else false */ template -bool FecCode::decode_packet( +bool FecCode::decode_streams_vertical( std::vector input_data_bufs, std::vector input_parities_bufs, const std::vector& input_parities_props, @@ -1005,6 +1015,279 @@ bool FecCode::decode_packet( return true; } +/** Encode blocks + * + * @param data_bufs vector size must be exactly n_data + * (set entries to nullptr when missing) + * @param parities_bufs vector size must be exactly n_outputs + * (set entries to nullptr when not wanted) + * @param parities_props vector size must be exactly n_outputs + * (set entries to nullptr when not wanted) + * @param wanted_idxs bool array of missing_idxs of len n_outputs indicating + * wanted (value 1) or not wanted fragments (value 0) - wanted blocks MUST BE + * allocated by caller + * @param block_size_bytes the block size in bytes + * + * @pre All blocks must be of equal size + */ +template +void FecCode::encode_blocks_vertical( + std::vector data_bufs, + std::vector parities_bufs, + std::vector& parities_props, + std::vector wanted_idxs, + size_t block_size_bytes) +{ + assert(data_bufs.size() == n_data); + assert(parities_bufs.size() == n_outputs); + assert(parities_props.size() == n_outputs); + + // clear property vectors + for (auto& props : parities_props) { + props.clear(); + } + + size_t offset = 0; + size_t block_size = block_size_bytes / word_size; + + // vector of buffers storing data read from chunk + vec::Buffers words_char(n_data, buf_size); + const std::vector words_mem_char = words_char.get_mem(); + // vector of buffers storing data that are performed in encoding, i.e. FFT + vec::Buffers words(n_data, pkt_size); + const std::vector words_mem_T = words.get_mem(); + + int output_len = get_n_outputs(); + + // vector of buffers storing data that are performed in encoding, i.e. FFT + vec::Buffers output(output_len, pkt_size); + const std::vector output_mem_T = output.get_mem(); + // vector of buffers storing data in output chunk + vec::Buffers output_char(output_len, buf_size); + const std::vector output_mem_char = output_char.get_mem(); + + reset_stats_enc(); + + while (offset < block_size) { + size_t remain_size = block_size - offset; + size_t copy_size = std::min(pkt_size, remain_size); + for (unsigned i = 0; i < n_data; i++) { + memcpy( + reinterpret_cast(words_mem_char.at(i)), + data_bufs[i] + offset * word_size, + copy_size * word_size); + } + + // Zero-out trailing part of data + if (copy_size < pkt_size) { + const size_t copy_bytes = copy_size * word_size; + const size_t trailing_bytes = buf_size - copy_bytes; + for (unsigned i = 0; i < n_data; i++) { + memset( + reinterpret_cast(words_mem_char.at(i)) + copy_bytes, + 0, + trailing_bytes); + } + } + + vec::pack( + words_mem_char, words_mem_T, n_data, pkt_size, word_size); + + timeval t1 = tick(); + uint64_t start = hw_timer(); + encode(output, parities_props, offset, words); + uint64_t end = hw_timer(); + uint64_t t2 = hrtime_usec(t1); + + total_enc_usec += t2; + total_encode_cycles += (end - start) / (copy_size * word_size); + n_encode_ops++; + + vec::unpack( + output_mem_T, output_mem_char, output_len, pkt_size, word_size); + + for (unsigned i = 0; i < n_outputs; i++) { + if (wanted_idxs[i]) { + memcpy( + parities_bufs[i] + offset * word_size, + reinterpret_cast(output_mem_char.at(i)), + copy_size * word_size); + } + } + offset += pkt_size; + } +} + +/** Decode blocks + * + * @param data_bufs vector size must be exactly n_data + * (set entries to nullptr when missing) + * @param parities_bufs vector size must be exactly n_parities + * (set entries to nullptr when missing) + * @param parities_props vector size must be exactly n_parities + * (set entries to nullptr when missing) + * @param missing_idxs array of missing indexes of vector size code_len + * indicating presence (value 1) or absence of fragments (value 0). It applies + * for both data and parities + * - non missing blocks MUST BE allocated by caller + * @param wanted_idxs bool array of wanted indexes of vector size n_data + * indicating wanted (value 1) or not wanted fragments (value 0). It applies + * only for data + * - wanted blocks MUST BE allocated + * by caller + * @param block_size_bytes the block size in bytes + * + * @pre All blocks must be of equal size + * + * @return true if decode succeeded, else false + */ +template +bool FecCode::decode_blocks_vertical( + std::vector data_bufs, + std::vector parities_bufs, + const std::vector& parities_props, + std::vector missing_idxs, + std::vector wanted_idxs, + size_t block_size_bytes) +{ + size_t offset = 0; + size_t block_size = block_size_bytes / word_size; + + unsigned fragment_index = 0; + unsigned parity_index = 0; + unsigned avail_data_nb = 0; + + if (type == FecType::SYSTEMATIC) { + assert(data_bufs.size() == n_data); + } + assert(parities_bufs.size() == n_outputs); + assert(parities_props.size() == n_outputs); + + // ids of received fragments, from 0 to codelen-1 + vec::Vector fragments_ids(*(this->gf), n_data); + + if (type == FecType::SYSTEMATIC) { + for (unsigned i = 0; i < n_data; i++) { + if (!missing_idxs[i]) { + decode_add_data(fragment_index, i); + fragments_ids.set(fragment_index, i); + fragment_index++; + } + avail_data_nb = fragment_index; + // data is in clear so nothing to do + if (fragment_index == n_data) + return true; + } + } + + vec::Vector avail_parity_ids(*(this->gf), n_data - avail_data_nb); + + if (fragment_index < n_data) { + // finish with parities available + for (unsigned i = 0; i < n_outputs; i++) { + unsigned j = (type == FecType::SYSTEMATIC) ? n_data + i : i; + if (!missing_idxs[j]) { + decode_add_parities(fragment_index, i); + fragments_ids.set(fragment_index, j); + avail_parity_ids.set(parity_index, i); + fragment_index++; + parity_index++; + // stop when we have enough parities + if (fragment_index == n_data) + break; + } + } + // unable to decode + if (fragment_index < n_data) + return false; + } + fragments_ids.sort(); + + decode_build(); + + // vector of buffers storing data read from chunk + vec::Buffers words_char(n_data, buf_size); + const std::vector words_mem_char = words_char.get_mem(); + // vector of buffers storing data that are performed in encoding, i.e. FFT + vec::Buffers words(n_data, pkt_size); + const std::vector words_mem_T = words.get_mem(); + + int output_len = n_data; + + // vector of buffers storing data that are performed in decoding, i.e. FFT + vec::Buffers output(output_len, pkt_size); + const std::vector output_mem_T = output.get_mem(); + // vector of buffers storing data in output chunk + vec::Buffers output_char(output_len, buf_size); + const std::vector output_mem_char = output_char.get_mem(); + + std::unique_ptr> context = + init_context_dec(fragments_ids, pkt_size, &output); + + reset_stats_dec(); + + while (offset < block_size) { + size_t remain_size = block_size - offset; + size_t copy_size = std::min(pkt_size, remain_size); + if (type == FecType::SYSTEMATIC) { + for (unsigned i = 0; i < avail_data_nb; i++) { + unsigned data_idx = fragments_ids.get(i); + memcpy( + reinterpret_cast(words_mem_char.at(i)), + data_bufs[data_idx] + offset * word_size, + copy_size * word_size); + } + } + for (unsigned i = 0; i < n_data - avail_data_nb; ++i) { + unsigned parity_idx = avail_parity_ids.get(i); + memcpy( + reinterpret_cast(words_mem_char.at(avail_data_nb + i)), + parities_bufs[parity_idx] + offset * word_size, + copy_size * word_size); + } + + // Zero-out trailing part of data + if (copy_size < pkt_size) { + const size_t copy_bytes = copy_size * word_size; + const size_t trailing_bytes = buf_size - copy_bytes; + for (unsigned i = 0; i < n_data; i++) { + memset( + reinterpret_cast(words_mem_char.at(i)) + copy_bytes, + 0, + trailing_bytes); + } + } + + vec::pack( + words_mem_char, words_mem_T, n_data, pkt_size, word_size); + + timeval t1 = tick(); + uint64_t start = hw_timer(); + decode(*context, output, parities_props, offset, words); + uint64_t end = hw_timer(); + uint64_t t2 = hrtime_usec(t1); + + total_dec_usec += t2; + total_decode_cycles += (end - start) / word_size; + n_decode_ops++; + + vec::unpack( + output_mem_T, output_mem_char, output_len, pkt_size, word_size); + + for (unsigned i = 0; i < n_data; i++) { + if (wanted_idxs[i]) { + memcpy( + data_bufs[i] + offset * word_size, + reinterpret_cast(output_mem_char.at(i)), + copy_size * word_size); + } + } + offset += pkt_size; + } + + return true; +} + /** * Perform a Lagrange interpolation to find the coefficients of the * polynomial diff --git a/src/misc.cpp b/src/misc.cpp index de8267b6..8b3f5b80 100644 --- a/src/misc.cpp +++ b/src/misc.cpp @@ -27,6 +27,7 @@ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ +#include #include "misc.h" namespace std { @@ -77,3 +78,55 @@ std::ostream& operator<<(std::ostream& dest, __int128_t value) } } // namespace std + +namespace quadiron { + +std::ostream& hex_dump( + std::ostream& os, + const void* buffer, + std::size_t bufsize, + bool showPrintableChars) +{ + if (buffer == nullptr) { + return os; + } + auto oldFormat = os.flags(); + auto oldFillChar = os.fill(); + constexpr std::size_t maxline{32}; + // create a place to store text version of string + char renderString[maxline + 1]; + char* rsptr{renderString}; + // convenience cast + const unsigned char* buf{reinterpret_cast(buffer)}; + + for (std::size_t linecount = maxline; bufsize; --bufsize, ++buf) { + os << std::setw(2) << std::setfill('0') << std::hex + << static_cast(*buf) << ' '; + *rsptr++ = std::isprint(*buf) ? *buf : '.'; + if (--linecount == 0) { + *rsptr++ = '\0'; // terminate string + if (showPrintableChars) { + os << " | " << renderString; + } + os << '\n'; + rsptr = renderString; + linecount = std::min(maxline, bufsize); + } + } + // emit newline if we haven't already + if (rsptr != renderString) { + if (showPrintableChars) { + for (*rsptr++ = '\0'; rsptr != &renderString[maxline + 1]; + ++rsptr) { + os << " "; + } + os << " | " << renderString; + } + os << '\n'; + } + + os.fill(oldFillChar); + os.flags(oldFormat); + return os; +} +} // namespace quadiron diff --git a/src/misc.h b/src/misc.h index bb3e4044..86e711b1 100644 --- a/src/misc.h +++ b/src/misc.h @@ -80,6 +80,12 @@ static inline uint64_t hw_timer() return x; } +std::ostream& hex_dump( + std::ostream& os, + const void* buffer, + std::size_t bufsize, + bool showPrintableChars); + } // namespace quadiron #endif diff --git a/src/property.h b/src/property.h index 2fcaa61c..f17d027e 100644 --- a/src/property.h +++ b/src/property.h @@ -32,10 +32,12 @@ #define __QUAD_PROPERTY_H__ #include +#include #include #include #include +#include #include namespace quadiron { @@ -54,6 +56,8 @@ static constexpr unsigned OOR_MARK = 1; */ class Properties { public: + enum { FNT1 = 0x464E5431 }; + inline void add(const off_t loc, const uint32_t data) { props[loc] = data; @@ -75,6 +79,52 @@ class Properties { return props; } + /** + * Serialize properties into a buffer (FNT) + * + * @return 0 if OK, else -1 + */ + inline int fnt_serialize(uint32_t* dwords, unsigned n_dwords) + { + if ((2 + props.size()) > n_dwords) { + return -1; + } + dwords[0] = htonl(FNT1); + unsigned i = 2; + for (auto& kv : props) { + dwords[i++] = htonl(static_cast(kv.first)); + } + dwords[1] = htonl(i - 2); + for (; i < n_dwords; i++) { + dwords[i] = htonl(0); + } + return 0; + } + + /** + * Deserialize properties from a buffer (FNT) + * + * @return 0 if OK, else -1 + */ + inline int fnt_deserialize(const uint32_t* dwords, unsigned n_dwords) + { + if (n_dwords < 2) { + return -1; + } + uint32_t magic = ntohl(dwords[0]); + if (magic != FNT1) { + return -1; + } + uint32_t _n_dwords = ntohl(dwords[1]); + if ((2 + _n_dwords) > n_dwords) { + return -1; + } + for (unsigned i = 0; i < _n_dwords; i++) { + add(static_cast(ntohl(dwords[i + 2])), 1); + } + return 0; + } + private: std::unordered_map props; @@ -82,6 +132,10 @@ class Properties { friend std::ostream& operator<<(std::ostream& os, const Properties& props); }; +class FntProperties : public Properties { + public: +}; + } // namespace quadiron #endif diff --git a/src/quadiron_c.cpp b/src/quadiron_c.cpp new file mode 100644 index 00000000..82542445 --- /dev/null +++ b/src/quadiron_c.cpp @@ -0,0 +1,403 @@ +/* -*- mode: c++ -*- */ +/* + * Copyright 2017-2018 Scality + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ +#include "property.h" +#include "quadiron.h" +#include "quadiron_c.h" + +extern "C" { + +struct QuadironFnt32* +quadiron_fnt32_new(int word_size, int n_data, int n_parities, int systematic) +{ + const size_t pkt_size = 1024; + + if (word_size == 1 || word_size == 2) { + return reinterpret_cast( + new quadiron::fec::RsFnt( + systematic ? quadiron::fec::FecType::SYSTEMATIC + : quadiron::fec::FecType::NON_SYSTEMATIC, + word_size, + n_data, + n_parities, + pkt_size)); + } + + return nullptr; +} + +void quadiron_fnt32_delete(struct QuadironFnt32* fecp) +{ + delete reinterpret_cast*>(fecp); +} + +int quadiron_fnt32_get_metadata_size( + struct QuadironFnt32* /* fecp */, + size_t block_size) +{ + /* + * We assume that a special value of 65536 may occur uniformly. + * We count 4 bytes per special value. + * We see large and roundup by 16 items. + */ + return ((block_size / 65536) + 16) * 4; +} + +int quadiron_fnt32_encode( + struct QuadironFnt32* fecp, + uint8_t** data, + uint8_t** parity, + int* wanted_idxs, + size_t block_size) +{ + quadiron::fec::RsFnt* fec = + reinterpret_cast*>(fecp); + std::vector data_vec(fec->n_data, nullptr); + std::vector parities_vec(fec->n_outputs, nullptr); + std::vector parities_props(fec->n_outputs); + std::vector wanted_idxs_vec(fec->n_outputs); + int metadata_size = quadiron_fnt32_get_metadata_size(fecp, block_size); + + for (unsigned i = 0; i < fec->n_outputs; i++) { + wanted_idxs_vec[i] = wanted_idxs[i] ? true : false; + } + + if (fec->type == quadiron::fec::FecType::SYSTEMATIC) { + for (unsigned i = 0; i < fec->n_data; i++) { + data_vec[i] = data[i] + metadata_size; + } + for (unsigned i = 0; i < fec->n_parities; i++) { + parities_vec[i] = parity[i] + metadata_size; + } + } else { + for (unsigned i = 0; i < fec->n_data; i++) { + data_vec[i] = data[i] + metadata_size; + parities_vec[i] = data[i] + metadata_size; + } + for (unsigned i = 0; i < fec->n_parities; i++) { + parities_vec[fec->n_data + i] = parity[i] + metadata_size; + } + } + + fec->encode_blocks_vertical( + data_vec, parities_vec, parities_props, wanted_idxs_vec, block_size); + + if (fec->type == quadiron::fec::FecType::SYSTEMATIC) { + quadiron::Properties null_prop; + + for (unsigned i = 0; i < fec->n_data; i++) { + uint32_t* metadata = reinterpret_cast(data[i]); + int ret = null_prop.fnt_serialize(metadata, metadata_size / 4); + if (ret == -1) { + return -1; + } + } + for (unsigned i = 0; i < fec->n_parities; i++) { + uint32_t* metadata = reinterpret_cast(parity[i]); + int ret = + parities_props[i].fnt_serialize(metadata, metadata_size / 4); + if (ret == -1) { + return -1; + } + } + } else { + for (unsigned i = 0; i < fec->n_data; i++) { + uint32_t* metadata = reinterpret_cast(data[i]); + int ret = + parities_props[i].fnt_serialize(metadata, metadata_size / 4); + if (ret == -1) { + return -1; + } + } + for (unsigned i = 0; i < fec->n_parities; i++) { + uint32_t* metadata = reinterpret_cast(parity[i]); + int ret = parities_props[fec->n_data + i].fnt_serialize( + metadata, metadata_size / 4); + if (ret == -1) { + return -1; + } + } + } + + return 0; +} + +int quadiron_fnt32_decode( + struct QuadironFnt32* fecp, + uint8_t** data, + uint8_t** parity, + int* missing_idxs, + size_t block_size) +{ + quadiron::fec::RsFnt* fec = + reinterpret_cast*>(fecp); + std::vector data_vec(fec->n_data, nullptr); + std::vector parities_vec(fec->n_outputs, nullptr); + std::vector parities_props(fec->n_outputs); + std::vector missing_idxs_vec( + missing_idxs, missing_idxs + fec->code_len); + std::vector wanted_idxs_vec(fec->n_data, true); + int metadata_size = quadiron_fnt32_get_metadata_size(fecp, block_size); + bool res; + + if (fec->type == quadiron::fec::FecType::SYSTEMATIC) { + for (unsigned i = 0; i < fec->n_data; i++) { + data_vec[i] = data[i] + metadata_size; + } + for (unsigned i = 0; i < fec->n_parities; i++) { + if (!missing_idxs[fec->n_data + i]) { + parities_vec[i] = parity[i] + metadata_size; + uint32_t* metadata = reinterpret_cast(parity[i]); + int ret = parities_props[i].fnt_deserialize( + metadata, metadata_size / 4); + if (ret == -1) + return -1; + } + } + } else { + for (unsigned i = 0; i < fec->n_data; i++) { + if (!missing_idxs[i]) { + parities_vec[i] = data[i] + metadata_size; + uint32_t* metadata = reinterpret_cast(data[i]); + int ret = parities_props[i].fnt_deserialize( + metadata, metadata_size / 4); + if (ret == -1) + return -1; + } + data_vec[i] = data[i] + metadata_size; + } + for (unsigned i = 0; i < fec->n_parities; i++) { + if (!missing_idxs[fec->n_data + i]) { + parities_vec[fec->n_data + i] = parity[i] + metadata_size; + uint32_t* metadata = reinterpret_cast(parity[i]); + int ret = parities_props[fec->n_data + i].fnt_deserialize( + metadata, metadata_size / 4); + if (ret == -1) + return -1; + } + } + } + + res = fec->decode_blocks_vertical( + data_vec, + parities_vec, + parities_props, + missing_idxs_vec, + wanted_idxs_vec, + block_size); + if (!res) + return -1; + + return 0; +} + +int quadiron_fnt32_reconstruct( + struct QuadironFnt32* fecp, + uint8_t** data, + uint8_t** parity, + int* missing_idxs, + unsigned int destination_idx, + size_t block_size) +{ + quadiron::fec::RsFnt* fec = + reinterpret_cast*>(fecp); + std::vector data_vec(fec->n_data); + std::vector parities_vec(fec->n_outputs); + std::vector parities_props(fec->n_outputs); + std::vector missing_idxs_vec( + missing_idxs, missing_idxs + fec->code_len); + std::vector wanted_data_vec(fec->n_data, false); + std::vector wanted_idxs_vec(fec->n_outputs, false); + int metadata_size = quadiron_fnt32_get_metadata_size(fecp, block_size); + bool res; + + if (fec->type == quadiron::fec::FecType::SYSTEMATIC) { + for (unsigned i = 0; i < fec->n_data; i++) { + data_vec[i] = data[i] + metadata_size; + } + for (unsigned i = 0; i < fec->n_parities; i++) { + if (!missing_idxs[fec->n_data + i]) { + uint32_t* metadata = reinterpret_cast(parity[i]); + int ret = parities_props[i].fnt_deserialize( + metadata, metadata_size / 4); + if (ret == -1) + return -1; + } + parities_vec[i] = parity[i] + metadata_size; + } + } else { + for (unsigned i = 0; i < fec->n_data; i++) { + if (!missing_idxs[i]) { + uint32_t* metadata = reinterpret_cast(data[i]); + int ret = parities_props[i].fnt_deserialize( + metadata, metadata_size / 4); + if (ret == -1) + return -1; + } + parities_vec[i] = data[i] + metadata_size; + data_vec[i] = data[i] + metadata_size; + } + for (unsigned i = 0; i < fec->n_parities; i++) { + if (!missing_idxs[fec->n_data + i]) { + uint32_t* metadata = reinterpret_cast(parity[i]); + int ret = parities_props[fec->n_data + i].fnt_deserialize( + metadata, metadata_size / 4); + if (ret == -1) + return -1; + } + parities_vec[fec->n_data + i] = parity[i] + metadata_size; + } + } + + int need_decode = 0; + + if (fec->type == quadiron::fec::FecType::SYSTEMATIC) { + /* + * Easy case where the target is a data then simply decode + */ + if (destination_idx < fec->n_data) { + wanted_idxs_vec[destination_idx] = true; + res = fec->decode_blocks_vertical( + data_vec, + parities_vec, + parities_props, + missing_idxs_vec, + wanted_idxs_vec, + block_size); + if (!res) { + return -1; + } + for (unsigned i = 0; i < fec->n_data; i++) { + quadiron::Properties null_prop; + + if (i == destination_idx) { + uint32_t* metadata = reinterpret_cast(data[i]); + int ret = + null_prop.fnt_serialize(metadata, metadata_size / 4); + if (ret == -1) { + return -1; + } + } + } + return 0; + } + } + + /* + * At this point we want a parity to be reconstructed + * If systematic we may need to decode if a data is missing. + * If non-systematic we always need to decode + */ + std::vector> blocks(fec->n_data); + if (fec->type == quadiron::fec::FecType::SYSTEMATIC) { + for (unsigned i = 0; i < fec->n_data; i++) { + if (missing_idxs[i]) { + need_decode = 1; + wanted_data_vec[i] = true; + blocks.at(i).resize(block_size); + data_vec[i] = blocks.at(i).data(); + } + } + } else { + need_decode = 1; + for (unsigned i = 0; i < fec->n_data; i++) { + wanted_data_vec[i] = true; + blocks.at(i).resize(block_size); + data_vec[i] = blocks.at(i).data(); + } + } + + if (need_decode) { + res = fec->decode_blocks_vertical( + data_vec, + parities_vec, + parities_props, + missing_idxs_vec, + wanted_data_vec, + block_size); + if (!res) { + return -1; + } + } + + /* + * At this point we have all data blocks. + */ + if (fec->type == quadiron::fec::FecType::SYSTEMATIC) { + wanted_idxs_vec[destination_idx - fec->n_data] = true; + } else { + wanted_idxs_vec[destination_idx] = true; + } + + fec->encode_blocks_vertical( + data_vec, parities_vec, parities_props, wanted_idxs_vec, block_size); + + if (fec->type == quadiron::fec::FecType::SYSTEMATIC) { + for (unsigned i = 0; i < fec->n_parities; i++) { + if (i == destination_idx - fec->n_data) { + uint32_t* metadata = reinterpret_cast(parity[i]); + int ret = parities_props[i].fnt_serialize( + metadata, metadata_size / 4); + if (ret == -1) { + return -1; + } + } + } + } else { + for (unsigned i = 0; i < fec->n_data; i++) { + if (i == destination_idx) { + uint32_t* metadata = reinterpret_cast(data[i]); + int ret = parities_props[i].fnt_serialize( + metadata, metadata_size / 4); + if (ret == -1) { + return -1; + } + } + } + for (unsigned i = 0; i < fec->n_parities; i++) { + if (fec->n_data + i == destination_idx) { + uint32_t* metadata = reinterpret_cast(parity[i]); + int ret = parities_props[fec->n_data + i].fnt_serialize( + metadata, metadata_size / 4); + if (ret == -1) { + return -1; + } + } + } + } + + return 0; +} + +void quadiron_hex_dump(uint8_t* buf, size_t size) +{ + quadiron::hex_dump(std::cerr, buf, size, true); +} +} diff --git a/src/quadiron_c.h b/src/quadiron_c.h new file mode 100644 index 00000000..bf6e231e --- /dev/null +++ b/src/quadiron_c.h @@ -0,0 +1,162 @@ +/* -*- mode: c++ -*- */ +/* + * Copyright 2017-2018 Scality + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ +#ifndef __QUAD_QUADIRON_C_H__ +#define __QUAD_QUADIRON_C_H__ + +#ifdef __cplusplus +extern "C" { +#endif + +/** Create FNT FEC - This FEC is relatively complex because it requires storing + * a metadata header. + * + * @param[in] word_size FNT only supports 1 or 2 + * @param[in] n_data number of data fragments + * @param[in] n_parities number of parity fragments + * @param[in] systematic if 1 then the code is systematic otherwise + * non-systematic + * + * @return the FEC instance pointer + */ +struct QuadironFnt32* +quadiron_fnt32_new(int word_size, int n_data, int n_parities, int systematic); + +/** Delete FEC + * + * @param[in,out] fecp the FEC instance pointer + */ +void quadiron_fnt32_delete(struct QuadironFnt32* fecp); + +/** Return metadata size + * + * FNT requires to store specific information in headers and therefore caller + * needs to preallocate metadata_size in blocks. + * + * @param[in] fecp the FEC instance + * @param[in] block_size the metadata_size is computed acc/to the block_size + * + * @return the metadata size + */ +int quadiron_fnt32_get_metadata_size( + struct QuadironFnt32* fecp, + size_t block_size); + +/** Encode blocks + * + * @param[in] fecp the FEC instance + * @param[in] data must be exactly n_data + * buffers must allocate block_size + metadata_size + * @param[out] parity must be exactly n_outputs + * - set entries to NULL when not wanted + * - callers must allocate block_size + metadata_size when wanted + * - n_outputs is n_parities if systematic, and n_data + n_parities if + * non-systematic + * @param[in] wanted_idxs array of length n_outputs indicating + * the wish (value 1) or not (value 0) of parities + * @param[in] block_size the block size in bytes + * + * @return 0 if encode succeeded, else -1 + */ +int quadiron_fnt32_encode( + struct QuadironFnt32* fecp, + uint8_t** data, + uint8_t** parity, + int* wanted_idxs, + size_t block_size); + +/** Decode blocks + * + * @note For non-systematic codes parities must be provided as data and parities + * + * @param[in] fecp the FEC instance + * @param[in,out] data must be exactly n_data + * - callers must allocate block_size + metadata_size for both provided and + * missing data + * - it is not possible to choose not to decode a data (see reconstruct api for + * that) + * @param[in] parity must be exactly n_outputs + * - set entries to NULL when missing + * - n_outputs is n_parities if systematic, and n_data + n_parities if + * non-systematic + * - callers must allocate block_size + metadata_size for provided parities + * @param[in] missing_idxs array of length code_len indicating + * presence (value 1) or absence (value 0) of fragments (data and parities) + * - code_len is n_data + n_parities + * @param[in] block_size the block size in bytes + * + * @return 0 if decode succeeded, else -1 + */ +int quadiron_fnt32_decode( + struct QuadironFnt32* fecp, + uint8_t** data, + uint8_t** parity, + int* missing_idxs, + size_t block_size); + +/** Reconstruct block + * + * @note For non-systematic codes parities must be provided as data and parities + * + * @param[in] fecp the FEC instance + * @param[in,out] data must be exactly n_data + * - set entries to NULL when missing + * - callers must allocate block_size + metadata_size for provided and + * destination_idx data + * @param[in,out] parity must be exactly n_parities + * - set entries to NULL when missing + * - callers must allocate block_size + metadata_size + * @param[in] missing_idxs array of missing_idxs of len code_len indicating + * presence (value 1) or absence (value 0) of fragments (data and parities) + * @param[in] destination_idx index of fragment to reconstruct (data or parity) + * @param[in] block_size the block size in bytes + * + * @return 0 if reconstruct succeeded, else -1 + */ +int quadiron_fnt32_reconstruct( + struct QuadironFnt32* fecp, + uint8_t** data, + uint8_t** parity, + int* missing_idxs, + unsigned int destination_idx, + size_t block_size); + +/** Dump a buffer on stderr (debug function) + * + * @param[in] buf the buffer + * @param[in] size the buffer size + */ +void quadiron_hex_dump(uint8_t* buf, size_t size); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index d8dff84e..e02d1abe 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -123,6 +123,7 @@ set(TEST_SRC ${CMAKE_CURRENT_SOURCE_DIR}/rs_utest.cpp ${CMAKE_CURRENT_SOURCE_DIR}/buffers_utest.cpp ${CMAKE_CURRENT_SOURCE_DIR}/vector_utest.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/quadiron_c_utest.cpp ${CMAKE_CURRENT_SOURCE_DIR}/simd/test_allocator.cpp ${CMAKE_CURRENT_SOURCE_DIR}/simd/test_definitions.cpp ${CMAKE_CURRENT_SOURCE_DIR}/simd/test_simd.cpp diff --git a/test/buffers_utest.cpp b/test/buffers_utest.cpp index a93d57bf..26dcf181 100644 --- a/test/buffers_utest.cpp +++ b/test/buffers_utest.cpp @@ -42,10 +42,15 @@ class BuffersTest : public ::testing::Test { public: quadiron::simd::AlignedAllocator allocator; + BuffersTest() + { + quadiron::prng().seed(time(0)); + } + ~BuffersTest() = default; + std::unique_ptr> gen_buffers_rand_data(int n, int size, int _max = 0) { - std::mt19937 prng; T max_val = 65537; const int max = (_max == 0) ? max_val : _max; std::uniform_int_distribution dis(0, max - 1); @@ -54,7 +59,7 @@ class BuffersTest : public ::testing::Test { for (int i = 0; i < n; i++) { T* buf = this->allocator.allocate(size); for (int j = 0; j < size; j++) { - buf[j] = dis(prng); + buf[j] = dis(quadiron::prng()); } vec->set(i, buf); } diff --git a/test/ec_driver.cpp b/test/ec_driver.cpp index 8729a7f7..0166f848 100644 --- a/test/ec_driver.cpp +++ b/test/ec_driver.cpp @@ -144,9 +144,9 @@ void create_coding_files( } if (operation_on_packet) { - fec->encode_packet(d_files, c_files, c_props); + fec->encode_streams_vertical(d_files, c_files, c_props); } else { - fec->encode_bufs(d_files, c_files, c_props); + fec->encode_streams_horizontal(d_files, c_files, c_props); } for (unsigned i = 0; i < fec->n_data; i++) { @@ -244,9 +244,9 @@ bool repair_data_files( } if (operation_on_packet) { - fec->decode_packet(d_files, c_files, c_props, r_files); + fec->decode_streams_vertical(d_files, c_files, c_props, r_files); } else { - fec->decode_bufs(d_files, c_files, c_props, r_files); + fec->decode_streams_horizontal(d_files, c_files, c_props, r_files); } for (unsigned i = 0; i < fec->n_data; i++) { diff --git a/test/quadiron_c_utest.cpp b/test/quadiron_c_utest.cpp new file mode 100644 index 00000000..f87a0e38 --- /dev/null +++ b/test/quadiron_c_utest.cpp @@ -0,0 +1,245 @@ +/* + * Copyright 2017-2018 Scality + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ +#include +#include +#include "quadiron.h" +#include "quadiron_c.h" + +template +class QuadironCTest : public ::testing::Test { + public: + QuadironCTest() + { + quadiron::prng().seed(time(0)); + } + ~QuadironCTest() = default; + + void randomize_buffer(uint8_t* buf, size_t size) + { + std::uniform_int_distribution<> dis(0, 256); + for (u_int i = 0; i < size; i++) + buf[i] = dis(quadiron::prng()); + } + + /** Generate all combinations of n given k + * + * @param n numbers of items + * @param k number of elements in combinations + * + * @return the vector of combinations + */ + std::vector> generate_combinations(int n, int k) + { + std::vector> combinations; + std::vector selected; + std::vector selector(n); + std::fill(selector.begin(), selector.begin() + k, 1); + do { + for (int i = 0; i < n; i++) { + if (selector[i]) { + selected.push_back(i); + } + } + combinations.push_back(selected); + selected.clear(); + } while (std::prev_permutation(selector.begin(), selector.end())); + return combinations; + } + + /** Convert list of indexes to list of boolean values + * + * @param src input idx_list + * @param n_src number of entries in src + * @param dst output idx list as boolean values (1/0) + * @param n_dst number of entries in dst + */ + void convert_idx_list( + std::vector src, + int n_src, + std::vector& dst, + int n_dst) + { + std::fill_n(dst.begin(), n_dst, 0); + for (int i = 0; i < n_src; i++) { + ASSERT_LT(src[i], n_dst); + dst[src[i]] = 1; + } + } + + /** Test encode/decode/reconstruct + * + * This test will create n_data fragments and generate n_parities parities, + * then decode with a set of missing fragments. Finaly reconstruct them. + * + * @param n_data number of data + * @param n_parities number of parities + * @param block_size size of block in bytes + * @param systematic 1 if systematic else 0 + * @param missing_idxs vector of boolean vales indicating missing fragments + * for decode and reconstruct + * - must be of length n_parities + */ + void test_encode_decode_reconstruct( + int n_data, + int n_parities, + size_t block_size, + int systematic, + std::vector missing_idxs) + { + struct QuadironFnt32* inst = + quadiron_fnt32_new(2, n_data, n_parities, systematic); + size_t metadata_size = + quadiron_fnt32_get_metadata_size(inst, block_size); + std::vector> data(n_data); + std::vector _data(n_data); // for C API + std::vector> ref_data(n_data); + std::vector _ref_data(n_data); // for C API + int n_outputs; + if (systematic) { + n_outputs = n_parities; + } else { + n_outputs = n_data + n_parities; + } + std::vector> parity(n_outputs); + std::vector _parity(n_outputs); // for C API + std::vector wanted_idxs(n_outputs); + + for (int i = 0; i < n_data; i++) { + data.at(i).resize(block_size + metadata_size); + _data[i] = data.at(i).data(); + ref_data.at(i).resize(block_size); + _ref_data[i] = ref_data.at(i).data(); + randomize_buffer(_data[i] + metadata_size, block_size); + std::copy_n(_data[i] + metadata_size, block_size, _ref_data[i]); + } + + for (int i = 0; i < n_outputs; i++) { + parity.at(i).resize(block_size + metadata_size); + _parity[i] = parity.at(i).data(); + } + + // we want all parities + std::fill_n(wanted_idxs.begin(), n_outputs, 1); + + ASSERT_EQ( + quadiron_fnt32_encode( + inst, + _data.data(), + _parity.data(), + wanted_idxs.data(), + block_size), + 0); + + for (int i = 0; i < n_data; i++) { + if (missing_idxs[i]) { + std::fill_n(_data[i], block_size + metadata_size, 0); + } + } + + for (int i = 0; i < n_parities; i++) { + if (missing_idxs[n_data + i]) { + std::fill_n(_parity[i], block_size + metadata_size, 0); + } + } + + ASSERT_EQ( + quadiron_fnt32_decode( + inst, + _data.data(), + _parity.data(), + missing_idxs.data(), + block_size), + 0); + + for (int i = 0; i < n_data; i++) { + ASSERT_TRUE(std::equal( + _ref_data[i], + _ref_data[i] + block_size, + _data[i] + metadata_size)); + } + + for (int i = 0; i < n_data; i++) { + if (missing_idxs[i]) { + ASSERT_EQ( + quadiron_fnt32_reconstruct( + inst, + _data.data(), + _parity.data(), + missing_idxs.data(), + i, + block_size), + 0); + } + } + + for (int i = 0; i < n_parities; i++) { + if (missing_idxs[n_data + i]) { + ASSERT_EQ( + quadiron_fnt32_reconstruct( + inst, + _data.data(), + _parity.data(), + missing_idxs.data(), + n_data + i, + block_size), + 0); + } + } + + quadiron_fnt32_delete(inst); + } + + void test_all_decodable_scenarios(int k, int m, int systematic) + { + for (int i = 0; i <= m; i++) { + const auto combinations = generate_combinations(k + m, i); + for (auto it = combinations.begin(); it != combinations.end(); + it++) { + std::vector missing_idxs(k + m); + convert_idx_list(*it, i, missing_idxs, k + m); + test_encode_decode_reconstruct( + k, m, 10000, systematic, missing_idxs); + } + } + } +}; + +using AllTypes = ::testing::Types; +TYPED_TEST_CASE(QuadironCTest, AllTypes); + +TYPED_TEST(QuadironCTest, TestBasicSys) // NOLINT +{ + this->test_all_decodable_scenarios(3, 3, 1); +} + +TYPED_TEST(QuadironCTest, TestBasicNSys) // NOLINT +{ + this->test_all_decodable_scenarios(3, 3, 0); +}