record per block compression and timing

src/lepton/bitops.cc

@@ -70,6 +70,7 @@ abitreader::~abitreader( void )
 
 abitwriter::abitwriter( int size , int max_file_size)
 {
+    bits_written = 0;
     size_bound = max_file_size;
     if (size_bound) {
         size_bound += 8; // 64 bits of padding on the end

src/lepton/bitops.hh

@@ -73,7 +73,7 @@ public:
     int size_bound;
 public:
     void debug() const;
-
+    uint32_t bits_written;
 
     abitwriter( int size, int size_bound);
 	~abitwriter( void );
@@ -117,7 +117,7 @@ public:
 
     void write( unsigned int val, int nbits )
     {
-
+        bits_written += nbits;
         int nbits2 = nbits;
         unsigned int val2 = val;
         dev_assert(nbits <= 64);

src/lepton/lepton_codec.cc

@@ -1,9 +1,83 @@
 #include "lepton_codec.hh"
 #include "uncompressed_components.hh"
 #include "../vp8/decoder/decoder.hh"
+#include <thread>
+#include <mutex>
+#include "jpgcoder.hh"
+std::mutex stats_mutex;
+struct Stats{
+    uint64_t time_us_lep;
+    uint64_t time_us_jpg;
+    uint32_t color;
+    uint16_t x;
+    uint16_t y;
+    uint16_t num_bits_lep;
+    uint16_t num_bits_jpg;
+    bool operator<(const Stats&other) const {
+        if (y != other.y) {
+            return y < other.y;
+        }
+        if (x != other.x) {
+            return x < other.x;
+        }
+        if (color != other.color) {
+            return color < other.color;
+        }
+        return time_us_jpg < other.time_us_jpg;
+    }
+};
 
-
-
+std::vector<Stats> cur_stats;
+void add_lep_bits(uint32_t bits, uint16_t x, uint16_t y, int color) {
+    Stats s;
+    s.time_us_lep = TimingHarness::get_time_us(true);
+    s.time_us_jpg = 0;
+    s.color = (int)color;
+    s.x =x;
+    s.y = y;
+    s.num_bits_lep = bits;
+    s.num_bits_jpg = 0;
+    std::lock_guard<std::mutex> l(stats_mutex);
+    cur_stats.push_back(s);
+
+}
+void add_jpg_bits(uint32_t bits, uint16_t x, uint16_t y, int color) {
+    Stats s;
+    s.time_us_lep = 0;
+    s.time_us_jpg = TimingHarness::get_time_us(true);
+    s.color = (int)color;
+    s.x =x;
+    s.y = y;
+    s.num_bits_jpg = bits;
+    s.num_bits_lep = 0;
+    std::lock_guard<std::mutex> l(stats_mutex);
+    cur_stats.push_back(s);
+}
+FILE * fs = fopen("/tmp/lepton_stats.csv", "w");
+void print_stats() {
+    std::sort(cur_stats.begin(), cur_stats.end());
+    auto s = cur_stats.size();
+    for (size_t i = 0;i < s; ++i) {
+        auto cur = cur_stats[i];
+        if (cur.time_us_lep != 0) {
+            if ( i + 1 < s) {
+                auto next = cur_stats[i+1];
+                if (cur.x == next.x && cur.y == next.y && cur.color == next.color) {
+                    fprintf(fs, "%d,%d,%d, %d,%d, %ld,%ld\n",
+                            cur.x, cur.y, cur.color,
+                            cur.num_bits_lep,
+                            next.num_bits_jpg,
+                            cur.time_us_lep,
+                            next.time_us_jpg);
+                } else {
+                    fprintf(stderr, "MISMATCH %d != %d %d != %d %d != %d\n", cur.x, next.x, cur.y, next.y, cur.color, next.color);
+                }
+            }
+        }
+    }
+    fclose(fs);
+    fs = NULL;
+}
 template<class Left, class Middle, class Right, bool force_memory_optimization>
 void LeptonCodec::ThreadState::decode_row(Left & left_model,
                                           Middle& middle_model,
@@ -14,34 +88,38 @@ void LeptonCodec::ThreadState::decode_row(Left & left_model,
     uint32_t block_width = image_data[(int)middle_model.COLOR]->block_width();
     if (block_width > 0) {
         BlockContext context = context_.at((int)middle_model.COLOR);
-        parse_tokens(context,
+        uint32_t bits = parse_tokens(context,
                      bool_decoder_,
                      left_model,
                      model_); //FIXME
+        add_lep_bits(bits, 0, curr_y, (int)middle_model.COLOR);
         int offset = image_data[middle_model.COLOR]->next(context_.at((int)middle_model.COLOR), true, curr_y);
         if (offset >= component_size_in_block) {
             return;
         }
     }
     for (unsigned int jpeg_x = 1; jpeg_x + 1 < block_width; jpeg_x++) {
         BlockContext context = context_.at((int)middle_model.COLOR);
-        parse_tokens(context,
+        uint32_t bits = parse_tokens(context,
                      bool_decoder_,
                      middle_model,
                      model_); //FIXME
         int offset = image_data[middle_model.COLOR]->next(context_.at((int)middle_model.COLOR),
 							  true,
 							  curr_y);
+        add_lep_bits(bits, jpeg_x, curr_y, (int)middle_model.COLOR);
         if (offset >= component_size_in_block) {
             return;
         }
     }
     if (block_width > 1) {
         BlockContext context = context_.at((int)middle_model.COLOR);
-        parse_tokens(context,
+
+        uint32_t bits = parse_tokens(context,
                      bool_decoder_,
                      right_model,
                      model_);
+        add_lep_bits(bits, block_width - 1, curr_y, (int)middle_model.COLOR);
         image_data[middle_model.COLOR]->next(context_.at((int)middle_model.COLOR), false, curr_y);
     }
 }

src/lepton/lepton_codec.hh

@@ -5,7 +5,8 @@
 #include "bool_decoder.hh"
 #include "base_coders.hh"
 class UncompressedComponents;
-
+void print_stats();
+void add_jpg_bits(uint32_t bits, uint16_t x, uint16_t y, int color);
 class LeptonCodec {
 protected:
     struct ThreadState {

src/lepton/recoder.cc

@@ -343,10 +343,13 @@ bool recode_one_mcu_row(abitwriter *huffw, int mcu,
             lastdc[ cmp ] = dc;
 
             // encode block
+            uint32_t start_bit_pos = huffw->bits_written;
             int eob = encode_block_seq(huffw,
                                        &(hcodes[ 0 ][ cmpnfo[cmp].huffdc ]),
                                        &(hcodes[ 1 ][ cmpnfo[cmp].huffac ]),
                                        block.begin() );
+            uint32_t end_bit_pos = huffw->bits_written;
+            add_jpg_bits(end_bit_pos - start_bit_pos, dpos%cmpnfo[cmp].bch, dpos/cmpnfo[cmp].bch, cmp);
             int old_mcu = mcu;
             // check for errors, proceed if no error encountered
             if ( eob < 0 ) sta = -1;
@@ -842,6 +845,7 @@ bool recode_baseline_jpeg(bounded_iostream*str_out,
         str_out->write( grbgdata, grbs );
     check_decompression_memory_bound_ok();
     str_out->flush();
+    print_stats();
     TimingHarness::timing[0][TimingHarness::TS_JPEG_RECODE_FINISHED] = TimingHarness::get_time_us();
 
     // errormessage if write error

src/vp8/decoder/boolreader.cc

@@ -30,6 +30,7 @@ int vpx_reader_init(vpx_reader *r,
     r->value = 0;
     r->count = -8;
     r->range = 255;
+    r->bits_consumed=0;
     vpx_reader_fill(r);
     return vpx_read(r, 128, Billing::HEADER) != 0;  // marker bit
 }

src/vp8/decoder/boolreader.hh

@@ -172,6 +172,7 @@ typedef struct {
   int count;
   BiRope buffer;
   PacketReader *reader;
+  uint32_t bits_consumed;
 //  vpx_decrypt_cb decrypt_cb;
 //  void *decrypt_state;
 } vpx_reader;
@@ -360,6 +361,7 @@ inline bool vpx_reader_fill_and_read(vpx_reader *r, unsigned int split, Billing
     range <<= shift;
     value <<= shift;
     count -= shift;
+    r->bits_consumed+= shift;
     write_bit_bill(bill, true, shift);
     r->value = value;
     r->count = count;
@@ -392,6 +394,7 @@ inline bool vpx_read(vpx_reader *r, int prob, Billing bill) {
   }
   //unsigned int shift = vpx_norm[range];
   unsigned int shift = count_leading_zeros_uint8(range);
+  r->bits_consumed+= shift;
   range <<= shift;
   value <<= shift;
   count -= shift;

src/vp8/decoder/decoder.cc

@@ -165,10 +165,11 @@ void decode_edge(BlockContext mcontext,
 
 
 template<bool all_neighbors_present, BlockType color>
-void parse_tokens(BlockContext context,
+uint32_t parse_tokens(BlockContext context,
                   BoolDecoder& decoder,
                   ProbabilityTables<all_neighbors_present, color> & probability_tables,
                   ProbabilityTablesBase &pt) {
+    uint32_t start_bits = decoder.bits_consumed();
     context.here().bzero();
     auto num_nonzeros_prob = probability_tables.nonzero_counts_7x7(pt, context.copy());
     uint8_t num_nonzeros_7x7 = 0;
@@ -315,15 +316,17 @@ void parse_tokens(BlockContext context,
     context.num_nonzeros_here->set_vertical(outp_sans_dc.begin(),
                                             ProbabilityTablesBase::quantization_table((int)color),
                                             context.here().dc());
+    uint32_t end_bits = decoder.bits_consumed();
+    return end_bits - start_bits;
 }
 #ifdef ALLOW_FOUR_COLORS
-template void parse_tokens(BlockContext, BoolDecoder&, ProbabilityTables<false, BlockType::Ck>&, ProbabilityTablesBase&);
-template void parse_tokens(BlockContext, BoolDecoder&, ProbabilityTables<true, BlockType::Ck>&, ProbabilityTablesBase&);
+template uint32_t parse_tokens(BlockContext, BoolDecoder&, ProbabilityTables<false, BlockType::Ck>&, ProbabilityTablesBase&);
+template uint32_t parse_tokens(BlockContext, BoolDecoder&, ProbabilityTables<true, BlockType::Ck>&, ProbabilityTablesBase&);
 #endif
 
-template void parse_tokens(BlockContext, BoolDecoder&, ProbabilityTables<false, BlockType::Y>&, ProbabilityTablesBase&);
-template void parse_tokens(BlockContext, BoolDecoder&, ProbabilityTables<false, BlockType::Cb>&, ProbabilityTablesBase&);
-template void parse_tokens(BlockContext, BoolDecoder&, ProbabilityTables<false, BlockType::Cr>&, ProbabilityTablesBase&);
-template void parse_tokens(BlockContext, BoolDecoder&, ProbabilityTables<true, BlockType::Y>&, ProbabilityTablesBase&);
-template void parse_tokens(BlockContext, BoolDecoder&, ProbabilityTables<true, BlockType::Cb>&, ProbabilityTablesBase&);
-template void parse_tokens(BlockContext, BoolDecoder&, ProbabilityTables<true, BlockType::Cr>&, ProbabilityTablesBase&);
+template uint32_t parse_tokens(BlockContext, BoolDecoder&, ProbabilityTables<false, BlockType::Y>&, ProbabilityTablesBase&);
+template uint32_t parse_tokens(BlockContext, BoolDecoder&, ProbabilityTables<false, BlockType::Cb>&, ProbabilityTablesBase&);
+template uint32_t parse_tokens(BlockContext, BoolDecoder&, ProbabilityTables<false, BlockType::Cr>&, ProbabilityTablesBase&);
+template uint32_t parse_tokens(BlockContext, BoolDecoder&, ProbabilityTables<true, BlockType::Y>&, ProbabilityTablesBase&);
+template uint32_t parse_tokens(BlockContext, BoolDecoder&, ProbabilityTables<true, BlockType::Cb>&, ProbabilityTablesBase&);
+template uint32_t parse_tokens(BlockContext, BoolDecoder&, ProbabilityTables<true, BlockType::Cr>&, ProbabilityTablesBase&);

src/vp8/decoder/decoder.hh

@@ -2,7 +2,7 @@
 #ifndef DECODER_HH
 #define DECODER_HH
 template<bool all_neighbors_present, BlockType color>
-void parse_tokens(BlockContext context,
+uint32_t parse_tokens(BlockContext context,
                   BoolDecoder& data,
                   ProbabilityTables<all_neighbors_present, color> & probability_tables,
                   ProbabilityTablesBase&pt);

src/vp8/decoder/vpx_bool_reader.hh

@@ -35,6 +35,9 @@ public:
 #endif
         init(pr);
     }
+    uint32_t bits_consumed() const{
+        return bit_reader.bits_consumed;
+    }
 #ifndef _WIN32
     __attribute__((always_inline))
 #endif