From 04fa15b9d47535647eac616ba6249e0d934ba9fe Mon Sep 17 00:00:00 2001 From: vrancurel Date: Thu, 6 Dec 2018 18:20:27 -0800 Subject: [PATCH] external C API baseline - compat w/ liberasurecode --- benchmark/benchmark.cpp | 8 +- src/CMakeLists.txt | 1 + src/fec_base.h | 287 ++++++++++++++++++++++++- src/misc.cpp | 53 +++++ src/misc.h | 6 + src/property.h | 33 +++ src/quadiron-c.cpp | 450 ++++++++++++++++++++++++++++++++++++++++ src/quadiron-c.h | 75 +++++++ test/ec_driver.cpp | 8 +- 9 files changed, 905 insertions(+), 16 deletions(-) create mode 100644 src/quadiron-c.cpp create mode 100644 src/quadiron-c.h diff --git a/benchmark/benchmark.cpp b/benchmark/benchmark.cpp index de8b2ab9..332afe64 100644 --- a/benchmark/benchmark.cpp +++ b/benchmark/benchmark.cpp @@ -478,9 +478,9 @@ bool Benchmark::encode() reset_c_streams(); if (operation_on_packet) - fec->encode_packet(*d_streams, *c_streams, c_props); + fec->encode_streams_vertical(*d_streams, *c_streams, c_props); else - fec->encode_bufs(*d_streams, *c_streams, c_props); + fec->encode_streams_horizontal(*d_streams, *c_streams, c_props); // update stats enc_stats->add(fec->total_enc_usec); @@ -506,14 +506,14 @@ bool Benchmark::decode() reset_r_streams(); if (operation_on_packet) { - if (!fec->decode_packet( + if (!fec->decode_streams_vertical( d_streams_shuffled, c_streams_shuffled, c_props_shuffled, *r_streams)) return false; } else { - if (!fec->decode_bufs( + if (!fec->decode_streams_horizontal( d_streams_shuffled, c_streams_shuffled, c_props_shuffled, diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 005bac06..04c70f55 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -34,6 +34,7 @@ set(LIB_SRC ${SOURCE_DIR}/gf_nf4.cpp ${SOURCE_DIR}/gf_ring.cpp ${SOURCE_DIR}/property.cpp + ${SOURCE_DIR}/quadiron-c.cpp CACHE INTERNAL diff --git a/src/fec_base.h b/src/fec_base.h index b4c891fd..2b355e74 100644 --- a/src/fec_base.h +++ b/src/fec_base.h @@ -178,17 +178,17 @@ class FecCode { bool read_pkt(char* pkt, std::istream& stream); bool write_pkt(char* pkt, std::ostream& stream); - void encode_bufs( + void encode_streams_horizontal( std::vector input_data_bufs, std::vector output_parities_bufs, std::vector& output_parities_props); - void encode_packet( + void encode_streams_vertical( std::vector input_data_bufs, std::vector output_parities_bufs, std::vector& output_parities_props); - bool decode_bufs( + bool decode_streams_horizontal( std::vector input_data_bufs, std::vector input_parities_bufs, const std::vector& input_parities_props, @@ -199,12 +199,27 @@ class FecCode { size_t size = 0, vec::Buffers* output = nullptr); - bool decode_packet( + bool decode_streams_vertical( std::vector input_data_bufs, std::vector input_parities_bufs, const std::vector& input_parities_props, std::vector output_data_bufs); + void encode_blocks_vertical( + std::vector data_bufs, + std::vector parities_bufs, + std::vector& parities_props, + std::vector wanted_idxs, + int block_size_bytes); + + bool decode_blocks_vertical( + std::vector data_bufs, + std::vector parities_bufs, + const std::vector& parities_props, + std::vector missing_idxs, + std::vector wanted_idxs, + int block_size_bytes); + const gf::Field& get_gf() { return *gf; @@ -393,7 +408,7 @@ inline bool FecCode::write_pkt(char* pkt, std::ostream& stream) * @note all streams must be of equal size */ template -void FecCode::encode_bufs( +void FecCode::encode_streams_horizontal( std::vector input_data_bufs, std::vector output_parities_bufs, std::vector& output_parities_props) @@ -451,7 +466,7 @@ void FecCode::encode_bufs( } template -void FecCode::encode_packet( +void FecCode::encode_streams_vertical( std::vector input_data_bufs, std::vector output_parities_bufs, std::vector& output_parities_props) @@ -542,7 +557,7 @@ void FecCode::encode_packet( * @return true if decode succeded, else false */ template -bool FecCode::decode_bufs( +bool FecCode::decode_streams_horizontal( std::vector input_data_bufs, std::vector input_parities_bufs, const std::vector& input_parities_props, @@ -868,7 +883,7 @@ void FecCode::decode_apply( * @return true if decode succeeded, else false */ template -bool FecCode::decode_packet( +bool FecCode::decode_streams_vertical( std::vector input_data_bufs, std::vector input_parities_bufs, const std::vector& input_parities_props, @@ -1007,6 +1022,262 @@ bool FecCode::decode_packet( return true; } +/** + * Encode blocks + * + * @param data_bufs if SYSTEMATIC must be exactly n_data otherwise it is + * unused (use nullptr when missing) + * @param parities_bufs if SYSTEMATIC must be exactly n_parities otherwise + * get_n_outputs() (use nullptr when missing) + * @param parities_props if SYSTEMATIC must be exactly n_parities + * otherwise get_n_outputs() caller is supposed to provide specific information + * bound to parities + * @param wanted_idxs array of missing_idxs of len n_parities indicating wanted + * (value 1) or not wanted fragments (value 0) - wanted blocks MUST BE allocated + * by caller + * @param block_size_bytes the block size + * + * @pre All streams must be of equal size + */ +template +void FecCode::encode_blocks_vertical( + std::vector data_bufs, + std::vector parities_bufs, + std::vector& parities_props, + std::vector wanted_idxs, + int block_size_bytes) +{ + assert(data_bufs.size() == n_data); + assert(parities_bufs.size() == n_outputs); + assert(parities_props.size() == n_outputs); + + // clear property vectors + for (auto& props : parities_props) { + props.clear(); + } + + off_t offset = 0; + int block_size = block_size_bytes / word_size; + + // vector of buffers storing data read from chunk + vec::Buffers words_char(n_data, buf_size); + const std::vector words_mem_char = words_char.get_mem(); + // vector of buffers storing data that are performed in encoding, i.e. FFT + vec::Buffers words(n_data, pkt_size); + const std::vector words_mem_T = words.get_mem(); + + int output_len = get_n_outputs(); + + // vector of buffers storing data that are performed in encoding, i.e. FFT + vec::Buffers output(output_len, pkt_size); + const std::vector output_mem_T = output.get_mem(); + // vector of buffers storing data in output chunk + vec::Buffers output_char(output_len, buf_size); + const std::vector output_mem_char = output_char.get_mem(); + + reset_stats_enc(); + + while (offset < block_size) { + size_t remain_size = block_size - offset; + size_t copy_size = std::min(pkt_size, remain_size); + // TODO: get number of read bytes -> true buf size + for (unsigned i = 0; i < n_data; i++) { + memcpy( + reinterpret_cast(words_mem_char.at(i)), + data_bufs[i] + offset * word_size, + copy_size * word_size); + } + + vec::pack( + words_mem_char, words_mem_T, n_data, copy_size, word_size); + + timeval t1 = tick(); + uint64_t start = hw_timer(); + encode(output, parities_props, offset, words); + uint64_t end = hw_timer(); + uint64_t t2 = hrtime_usec(t1); + + total_enc_usec += t2; + total_encode_cycles += (end - start) / (copy_size * word_size); + n_encode_ops++; + + vec::unpack( + output_mem_T, output_mem_char, output_len, pkt_size, word_size); + + for (unsigned i = 0; i < n_outputs; i++) { + if (wanted_idxs[i]) { + memcpy( + parities_bufs[i] + offset * word_size, + reinterpret_cast(output_mem_char.at(i)), + copy_size * word_size); + } + } + offset += pkt_size; + } +} + +/** + * Decode blocks + * + * @param data_bufs if SYSTEMATIC must be exactly n_data otherwise it is + * unused (use nullptr when missing) + * @param parities_bufs if SYSTEMATIC must be exactly n_parities otherwise + * get_n_outputs() (use nullptr when missing) + * @param parities_props if SYSTEMATIC must be exactly n_parities + * otherwise get_n_outputs() caller is supposed to provide specific information + * bound to parities + * @param missing_idxs array of missing_idxs of len code_len indicating presence + * (value 1) or absence of fragments (value 0) - non missing blocks MUST BE + * allocated by caller + * @param wanted_idxs array of missing_idxs of len n_data indicating wanted + * (value 1) or not wanted fragments (value 0) - wanted blocks MUST BE allocated + * by caller + * @param block_size_bytes the block size + * + * @pre All streams must be of equal size + * + * @return true if decode succeeded, else false + */ +template +bool FecCode::decode_blocks_vertical( + std::vector data_bufs, + std::vector parities_bufs, + const std::vector& parities_props, + std::vector missing_idxs, + std::vector wanted_idxs, + int block_size_bytes) +{ + bool cont = true; + off_t offset = 0; + int block_size = block_size_bytes / word_size; + + unsigned fragment_index = 0; + unsigned parity_index = 0; + unsigned avail_data_nb = 0; + + if (type == FecType::SYSTEMATIC) { + assert(data_bufs.size() == n_data); + } + assert(parities_bufs.size() == n_outputs); + assert(parities_props.size() == n_outputs); + + // ids of received fragments, from 0 to codelen-1 + vec::Vector fragments_ids(*(this->gf), n_data); + + if (type == FecType::SYSTEMATIC) { + for (unsigned i = 0; i < n_data; i++) { + if (!missing_idxs[i]) { + decode_add_data(fragment_index, i); + fragments_ids.set(fragment_index, i); + fragment_index++; + } + avail_data_nb = fragment_index; + // data is in clear so nothing to do + if (fragment_index == n_data) + return true; + } + } + + vec::Vector avail_parity_ids(*(this->gf), n_data - avail_data_nb); + + if (fragment_index < n_data) { + // finish with parities available + for (unsigned i = 0; i < n_outputs; i++) { + unsigned j = (type == FecType::SYSTEMATIC) ? n_data + i : i; + if (!missing_idxs[j]) { + decode_add_parities(fragment_index, i); + fragments_ids.set(fragment_index, j); + avail_parity_ids.set(parity_index, i); + fragment_index++; + parity_index++; + // stop when we have enough parities + if (fragment_index == n_data) + break; + } + } + // unable to decode + if (fragment_index < n_data) + return false; + } + fragments_ids.sort(); + + decode_build(); + + // vector of buffers storing data read from chunk + vec::Buffers words_char(n_data, buf_size); + const std::vector words_mem_char = words_char.get_mem(); + // vector of buffers storing data that are performed in encoding, i.e. FFT + vec::Buffers words(n_data, pkt_size); + const std::vector words_mem_T = words.get_mem(); + + int output_len = n_data; + + // vector of buffers storing data that are performed in decoding, i.e. FFT + vec::Buffers output(output_len, pkt_size); + const std::vector output_mem_T = output.get_mem(); + // vector of buffers storing data in output chunk + vec::Buffers output_char(output_len, buf_size); + const std::vector output_mem_char = output_char.get_mem(); + + std::unique_ptr> context = + init_context_dec(fragments_ids, pkt_size, &output); + + reset_stats_dec(); + + while (offset < block_size) { + size_t remain_size = block_size - offset; + size_t copy_size = std::min(pkt_size, remain_size); + // TODO: get number of read bytes -> true buf size + if (type == FecType::SYSTEMATIC) { + for (unsigned i = 0; i < avail_data_nb; i++) { + unsigned data_idx = fragments_ids.get(i); + memcpy( + reinterpret_cast(words_mem_char.at(i)), + data_bufs[data_idx] + offset * word_size, + copy_size * word_size); + } + } + for (unsigned i = 0; i < n_data - avail_data_nb; ++i) { + unsigned parity_idx = avail_parity_ids.get(i); + memcpy( + reinterpret_cast(words_mem_char.at(avail_data_nb + i)), + parities_bufs[parity_idx] + offset * word_size, + copy_size * word_size); + } + + if (!cont) + break; + + vec::pack( + words_mem_char, words_mem_T, n_data, pkt_size, word_size); + + timeval t1 = tick(); + uint64_t start = hw_timer(); + decode(*context, output, parities_props, offset, words); + uint64_t end = hw_timer(); + uint64_t t2 = hrtime_usec(t1); + + total_dec_usec += t2; + total_decode_cycles += (end - start) / word_size; + n_decode_ops++; + + vec::unpack( + output_mem_T, output_mem_char, output_len, pkt_size, word_size); + + for (unsigned i = 0; i < n_data; i++) { + if (wanted_idxs[i]) { + memcpy( + data_bufs[i] + offset * word_size, + reinterpret_cast(output_mem_char.at(i)), + copy_size * word_size); + } + } + offset += pkt_size; + } + + return true; +} + /** * Perform a Lagrange interpolation to find the coefficients of the * polynomial diff --git a/src/misc.cpp b/src/misc.cpp index de8267b6..8b3f5b80 100644 --- a/src/misc.cpp +++ b/src/misc.cpp @@ -27,6 +27,7 @@ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ +#include #include "misc.h" namespace std { @@ -77,3 +78,55 @@ std::ostream& operator<<(std::ostream& dest, __int128_t value) } } // namespace std + +namespace quadiron { + +std::ostream& hex_dump( + std::ostream& os, + const void* buffer, + std::size_t bufsize, + bool showPrintableChars) +{ + if (buffer == nullptr) { + return os; + } + auto oldFormat = os.flags(); + auto oldFillChar = os.fill(); + constexpr std::size_t maxline{32}; + // create a place to store text version of string + char renderString[maxline + 1]; + char* rsptr{renderString}; + // convenience cast + const unsigned char* buf{reinterpret_cast(buffer)}; + + for (std::size_t linecount = maxline; bufsize; --bufsize, ++buf) { + os << std::setw(2) << std::setfill('0') << std::hex + << static_cast(*buf) << ' '; + *rsptr++ = std::isprint(*buf) ? *buf : '.'; + if (--linecount == 0) { + *rsptr++ = '\0'; // terminate string + if (showPrintableChars) { + os << " | " << renderString; + } + os << '\n'; + rsptr = renderString; + linecount = std::min(maxline, bufsize); + } + } + // emit newline if we haven't already + if (rsptr != renderString) { + if (showPrintableChars) { + for (*rsptr++ = '\0'; rsptr != &renderString[maxline + 1]; + ++rsptr) { + os << " "; + } + os << " | " << renderString; + } + os << '\n'; + } + + os.fill(oldFillChar); + os.flags(oldFormat); + return os; +} +} // namespace quadiron diff --git a/src/misc.h b/src/misc.h index bb3e4044..86e711b1 100644 --- a/src/misc.h +++ b/src/misc.h @@ -80,6 +80,12 @@ static inline uint64_t hw_timer() return x; } +std::ostream& hex_dump( + std::ostream& os, + const void* buffer, + std::size_t bufsize, + bool showPrintableChars); + } // namespace quadiron #endif diff --git a/src/property.h b/src/property.h index 766acea7..ce540628 100644 --- a/src/property.h +++ b/src/property.h @@ -32,6 +32,7 @@ #define __QUAD_PROPERTY_H__ #include +#include #include #include #include @@ -73,6 +74,38 @@ class Properties { return props; } + /** + * Serialize F4 properties into a buffer + * + * @return 0 if OK, else -1 + */ + inline int to_buffer(uint32_t* dwords, unsigned n_dwords) + { + unsigned i = 0; + memset(dwords, 0xff, n_dwords * sizeof(uint32_t)); + for (auto& kv : props) { + if (i >= n_dwords) + return -1; + dwords[i++] = static_cast(kv.first); + } + return 0; + } + + /** + * Serialize F4 properties into a buffer + * + * @return 0 if OK, else -1 + */ + inline int from_buffer(uint32_t* dwords, unsigned n_dwords) + { + for (unsigned i = 0; i < n_dwords; i++) { + if (dwords[i] == 0xffffffff) + break; + add(static_cast(dwords[i]), 1); + } + return 0; + } + private: std::unordered_map props; diff --git a/src/quadiron-c.cpp b/src/quadiron-c.cpp new file mode 100644 index 00000000..0406bb05 --- /dev/null +++ b/src/quadiron-c.cpp @@ -0,0 +1,450 @@ +/* -*- mode: c++ -*- */ +/* + * Copyright 2017-2018 Scality + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ +#include "property.h" +#include "quadiron-c.h" +#include "quadiron.h" + +extern "C" { + +/** + * Create FNT FEC - This FEC is relatively complex because it requires storing a + * metadata header. + * + * @param word_size + * @param n_data + * @param n_parities + * + * @return the FEC instance pointer + */ +struct QuadironFnt32* +quadiron_fnt32_new(int word_size, int n_data, int n_parities, int systematic) +{ + size_t pkt_size = 1024; + + if (word_size <= 4) { + return reinterpret_cast( + new quadiron::fec::RsFnt( + systematic ? quadiron::fec::FecType::SYSTEMATIC + : quadiron::fec::FecType::NON_SYSTEMATIC, + word_size, + n_data, + n_parities, + pkt_size)); + } + + return nullptr; +} + +/** + * Delete FEC + * + * @param fec the FEC instance pointer + */ +void quadiron_fnt32_delete(struct QuadironFnt32* fecp) +{ + delete reinterpret_cast*>(fecp); +} + +/** + * Return metadata size + * + * we assume that a special value of 65536 may occur uniformly. + * we count 4 bytes per special value. + * we see large and roundup by 16 items. + */ +int quadiron_fnt32_get_metadata_size( + struct QuadironFnt32* /* fecp */, + int block_size) +{ + return ((block_size / 65536) + 16) * 4; +} + +/** + * Encode blocks + * + * @param data must be exactly n_data (use nullptr when missing) - + * buffers must allocate block_size + metadata_size + * @param parity must be exactly n_parities otherwise + * @param missing_idxs array of missing_idxs of len code_len indicating presence + * (value 1) or absence of fragments (value 0) + * @param block_size the block size in bytes + * + * @return true if decode succeeded, else false + */ +int quadiron_fnt32_encode( + struct QuadironFnt32* fecp, + uint8_t** data, + uint8_t** parity, + int* missing_idxs, + int block_size) +{ + quadiron::fec::RsFnt* fec = + reinterpret_cast*>(fecp); + std::vector data_vec(fec->n_data, nullptr); + std::vector parities_vec(fec->n_outputs, nullptr); + std::vector parities_props(fec->n_outputs); + std::vector missing_idxs_vec( + missing_idxs, missing_idxs + fec->code_len); + int metadata_size = quadiron_fnt32_get_metadata_size(fecp, block_size); + + if (fec->type == quadiron::fec::FecType::SYSTEMATIC) { + for (unsigned i = 0; i < fec->n_data; i++) { + data_vec[i] = data[i] + metadata_size; + } + for (unsigned i = 0; i < fec->n_parities; i++) { + parities_vec[i] = parity[i] + metadata_size; + } + } else { + for (unsigned i = 0; i < fec->n_data; i++) { + data_vec[i] = data[i] + metadata_size; + parities_vec[i] = data[i] + metadata_size; + } + for (unsigned i = 0; i < fec->n_parities; i++) { + parities_vec[fec->n_data + i] = parity[i] + metadata_size; + } + } + + fec->encode_blocks_vertical( + data_vec, parities_vec, parities_props, missing_idxs_vec, block_size); + + if (fec->type == quadiron::fec::FecType::SYSTEMATIC) { + for (unsigned i = 0; i < fec->n_parities; i++) { + uint32_t* metadata = reinterpret_cast(parity[i]); + int ret = parities_props[i].to_buffer(metadata, metadata_size / 4); + if (-1 == ret) { + return -1; + } + } + } else { + for (unsigned i = 0; i < fec->n_data; i++) { + uint32_t* metadata = reinterpret_cast(data[i]); + int ret = parities_props[i].to_buffer(metadata, metadata_size / 4); + if (-1 == ret) { + return -1; + } + } + for (unsigned i = 0; i < fec->n_parities; i++) { + uint32_t* metadata = reinterpret_cast(parity[i]); + int ret = parities_props[fec->n_data + i].to_buffer( + metadata, metadata_size / 4); + if (-1 == ret) { + return -1; + } + } + } + + return 0; +} + +/** + * Decode blocks + * + * @param data must be exactly n_data + * (use nullptr when missing) - buffers must allocate block_size + metadata_size + * @param parity must be exactly n_parities otherwise (use nullptr when missing) + * - buffers must allocate block_size + metadata_size + * @param missing_idxs array of missing_idxs of len code_len indicating presence + * (value 1) or absence of fragments (value 0) + * @param block_size the block size in bytes + * + * @return 0 if decode succeeded, else -1 + */ +int quadiron_fnt32_decode( + struct QuadironFnt32* fecp, + uint8_t** data, + uint8_t** parity, + int* missing_idxs, + int block_size) +{ + quadiron::fec::RsFnt* fec = + reinterpret_cast*>(fecp); + std::vector data_vec(fec->n_data, nullptr); + std::vector parities_vec(fec->n_outputs, nullptr); + std::vector parities_props(fec->n_outputs); + std::vector missing_idxs_vec( + missing_idxs, missing_idxs + fec->code_len); + std::vector wanted_idxs_vec(fec->n_data, 1); + int metadata_size = quadiron_fnt32_get_metadata_size(fecp, block_size); + bool res; + + if (fec->type == quadiron::fec::FecType::SYSTEMATIC) { + for (unsigned i = 0; i < fec->n_data; i++) { + data_vec[i] = data[i] + metadata_size; + } + for (unsigned i = 0; i < fec->n_parities; i++) { + if (!missing_idxs[fec->n_data + i]) { + parities_vec[i] = parity[i] + metadata_size; + uint32_t* metadata = reinterpret_cast(parity[i]); + int ret = + parities_props[i].from_buffer(metadata, metadata_size / 4); + if (-1 == ret) + return -1; + } + } + } else { + for (unsigned i = 0; i < fec->n_data; i++) { + if (!missing_idxs[i]) { + parities_vec[i] = data[i] + metadata_size; + uint32_t* metadata = reinterpret_cast(data[i]); + int ret = + parities_props[i].from_buffer(metadata, metadata_size / 4); + if (-1 == ret) + return -1; + } + data_vec[i] = data[i] + metadata_size; + } + for (unsigned i = 0; i < fec->n_parities; i++) { + if (!missing_idxs[fec->n_data + i]) { + parities_vec[fec->n_data + i] = parity[i] + metadata_size; + uint32_t* metadata = reinterpret_cast(parity[i]); + int ret = parities_props[fec->n_data + i].from_buffer( + metadata, metadata_size / 4); + if (-1 == ret) + return -1; + } + } + } + + res = fec->decode_blocks_vertical( + data_vec, + parities_vec, + parities_props, + missing_idxs_vec, + wanted_idxs_vec, + block_size); + if (!res) + return -1; + + return 0; +} + +/** + * Reconstruct block + * + * @param data must be exactly n_data (use nullptr when missing) - + * buffers must allocate block_size + metadata_size + * @param parity must be exactly n_parities - buffers must allocate block_size + + * metadata_size + * @param missing_idxs array of missing_idxs of len code_len indicating presence + * (value 1) or absence of fragments (value 0) + * @param destination_idx index of fragment to reconstruct + * @param block_size the block size in bytes + * + * @return 0 if reconstruct succeeded, else -1 + */ +int quadiron_fnt32_reconstruct( + struct QuadironFnt32* fecp, + uint8_t** data, + uint8_t** parity, + int* missing_idxs, + unsigned int destination_idx, + int block_size) +{ + quadiron::fec::RsFnt* fec = + reinterpret_cast*>(fecp); + std::vector data_vec(fec->n_data); + std::vector parities_vec(fec->n_outputs); + std::vector parities_props(fec->n_outputs); + std::vector missing_idxs_vec( + missing_idxs, missing_idxs + fec->code_len); + std::vector wanted_data_vec(fec->n_data, 0); + std::vector wanted_idxs_vec(fec->n_outputs, 0); + int metadata_size = quadiron_fnt32_get_metadata_size(fecp, block_size); + bool res; + int ret; + + if (fec->type == quadiron::fec::FecType::SYSTEMATIC) { + for (unsigned i = 0; i < fec->n_data; i++) { + data_vec[i] = data[i] + metadata_size; + } + for (unsigned i = 0; i < fec->n_parities; i++) { + if (!missing_idxs[fec->n_data + i]) { + uint32_t* metadata = reinterpret_cast(parity[i]); + int ret = + parities_props[i].from_buffer(metadata, metadata_size / 4); + if (-1 == ret) + return -1; + } + parities_vec[i] = parity[i] + metadata_size; + } + } else { + for (unsigned i = 0; i < fec->n_data; i++) { + if (!missing_idxs[i]) { + uint32_t* metadata = reinterpret_cast(data[i]); + int ret = + parities_props[i].from_buffer(metadata, metadata_size / 4); + if (-1 == ret) + return -1; + } + parities_vec[i] = data[i] + metadata_size; + data_vec[i] = data[i] + metadata_size; + } + for (unsigned i = 0; i < fec->n_parities; i++) { + if (!missing_idxs[fec->n_data + i]) { + uint32_t* metadata = reinterpret_cast(parity[i]); + int ret = parities_props[fec->n_data + i].from_buffer( + metadata, metadata_size / 4); + if (-1 == ret) + return -1; + } + parities_vec[fec->n_data + i] = parity[i] + metadata_size; + } + } + + int need_decode = 0; + + if (fec->type == quadiron::fec::FecType::SYSTEMATIC) { + /* + * Easy case where the target is a data then simply decode + */ + if (destination_idx < fec->n_data) { + wanted_idxs_vec[destination_idx] = 1; + res = fec->decode_blocks_vertical( + data_vec, + parities_vec, + parities_props, + missing_idxs_vec, + wanted_idxs_vec, + block_size); + if (!res) { + ret = -1; + goto end; + } + ret = 0; + goto end; + } + } + + /* + * At this point we want a parity to be reconstructed + * If systematic we may need to decode if a data is missing. + * If non-systematic we always need to decode + */ + if (fec->type == quadiron::fec::FecType::SYSTEMATIC) { + for (unsigned i = 0; i < fec->n_data; i++) { + if (missing_idxs[i]) { + need_decode = 1; + wanted_data_vec[i] = 1; + data_vec[i] = static_cast(malloc(block_size)); + if (NULL == data_vec[i]) { + ret = -1; + goto end; + } + } + } + } else { + need_decode = 1; + for (unsigned i = 0; i < fec->n_data; i++) { + wanted_data_vec[i] = 1; + data_vec[i] = static_cast(malloc(block_size)); + if (NULL == data_vec[i]) { + ret = -1; + goto end; + } + } + } + + if (need_decode) { + res = fec->decode_blocks_vertical( + data_vec, + parities_vec, + parities_props, + missing_idxs_vec, + wanted_data_vec, + block_size); + if (!res) { + ret = -1; + goto end; + } + } + + /* + * At this point we have all data blocks. + */ + if (fec->type == quadiron::fec::FecType::SYSTEMATIC) { + wanted_idxs_vec[destination_idx - fec->n_data] = 1; + } else { + wanted_idxs_vec[destination_idx] = 1; + } + + fec->encode_blocks_vertical( + data_vec, parities_vec, parities_props, wanted_idxs_vec, block_size); + + if (fec->type == quadiron::fec::FecType::SYSTEMATIC) { + for (unsigned i = 0; i < fec->n_parities; i++) { + if (i == destination_idx - fec->n_data) { + uint32_t* metadata = reinterpret_cast(parity[i]); + int ret = + parities_props[i].to_buffer(metadata, metadata_size / 4); + if (-1 == ret) { + return -1; + } + } + } + } else { + for (unsigned i = 0; i < fec->n_data; i++) { + if (i == destination_idx) { + uint32_t* metadata = reinterpret_cast(data[i]); + int ret = + parities_props[i].to_buffer(metadata, metadata_size / 4); + if (-1 == ret) { + return -1; + } + } + } + for (unsigned i = 0; i < fec->n_parities; i++) { + if (fec->n_data + i == destination_idx) { + uint32_t* metadata = reinterpret_cast(parity[i]); + int ret = parities_props[fec->n_data + i].to_buffer( + metadata, metadata_size / 4); + if (-1 == ret) { + return -1; + } + } + } + } + + ret = 0; +end: + for (unsigned i = 0; i < fec->n_data; i++) { + if (wanted_data_vec[i]) { + free(data_vec[i]); + } + } + + return ret; +} + +void quadiron_hex_dump(char* buf, int buf_size) +{ + quadiron::hex_dump(std::cerr, buf, buf_size, true); +} +} diff --git a/src/quadiron-c.h b/src/quadiron-c.h new file mode 100644 index 00000000..cbf925d7 --- /dev/null +++ b/src/quadiron-c.h @@ -0,0 +1,75 @@ +/* -*- mode: c++ -*- */ +/* + * Copyright 2017-2018 Scality + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ +#ifndef __QUAD_QUADIRON_C_H__ +#define __QUAD_QUADIRON_C_H__ + +#ifdef __cplusplus +extern "C" { +#endif + +struct QuadironFnt32* +quadiron_fnt32_new(int word_size, int n_data, int n_parities, int systematic); + +void quadiron_fnt32_delete(struct QuadironFnt32* fec); + +int quadiron_fnt32_get_metadata_size( + struct QuadironFnt32* fecp, + int block_size); + +int quadiron_fnt32_encode( + struct QuadironFnt32* fecp, + uint8_t** data, + uint8_t** parity, + int* missing_idxs, + int block_size); + +int quadiron_fnt32_decode( + struct QuadironFnt32* fecp, + uint8_t** data, + uint8_t** parity, + int* missing_idxs, + int block_size); + +int quadiron_fnt32_reconstruct( + struct QuadironFnt32* fecp, + uint8_t** data, + uint8_t** parity, + int* missing_idxs, + unsigned int destination_idx, + int block_size); + +void quadiron_hex_dump(char* buf, int buf_size); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/test/ec_driver.cpp b/test/ec_driver.cpp index 8729a7f7..0166f848 100644 --- a/test/ec_driver.cpp +++ b/test/ec_driver.cpp @@ -144,9 +144,9 @@ void create_coding_files( } if (operation_on_packet) { - fec->encode_packet(d_files, c_files, c_props); + fec->encode_streams_vertical(d_files, c_files, c_props); } else { - fec->encode_bufs(d_files, c_files, c_props); + fec->encode_streams_horizontal(d_files, c_files, c_props); } for (unsigned i = 0; i < fec->n_data; i++) { @@ -244,9 +244,9 @@ bool repair_data_files( } if (operation_on_packet) { - fec->decode_packet(d_files, c_files, c_props, r_files); + fec->decode_streams_vertical(d_files, c_files, c_props, r_files); } else { - fec->decode_bufs(d_files, c_files, c_props, r_files); + fec->decode_streams_horizontal(d_files, c_files, c_props, r_files); } for (unsigned i = 0; i < fec->n_data; i++) {