Add changes for b0277c4

Contribution/Development.html

@@ -162,7 +162,7 @@ <h1>MetPX-Sarracenia Developer’s Guide<a class="headerlink" href="#metpx-sarra
 <dd class="field-odd"><p>3.00.54rc1</p>
 </dd>
 <dt class="field-even">date<span class="colon">:</span></dt>
-<dd class="field-even"><p>Jun 13, 2024</p>
+<dd class="field-even"><p>Jun 17, 2024</p>
 </dd>
 </dl>
 <section id="tools-you-need">

Contribution/Release.html

@@ -133,7 +133,7 @@ <h1>Releasing MetPX-Sarracenia<a class="headerlink" href="#releasing-metpx-sarra
 <dd class="field-odd"><p>3.00.54rc1</p>
 </dd>
 <dt class="field-even">date<span class="colon">:</span></dt>
-<dd class="field-even"><p>Jun 13, 2024</p>
+<dd class="field-even"><p>Jun 17, 2024</p>
 </dd>
 </dl>
 <section id="pre-release-overview">

Contribution/man_page_template.html

@@ -122,7 +122,7 @@ <h2>sr_title<a class="headerlink" href="#sr-title" title="Link to this heading">
 <dd class="field-odd"><p>1</p>
 </dd>
 <dt class="field-even">Date<span class="colon">:</span></dt>
-<dd class="field-even"><p>Jun 13, 2024</p>
+<dd class="field-even"><p>Jun 17, 2024</p>
 </dd>
 <dt class="field-odd">Version<span class="colon">:</span></dt>
 <dd class="field-odd"><p>3.00.54rc1</p>

Explanation/SarraPluginDev.html

@@ -141,7 +141,7 @@ <h3>Revision Record<a class="headerlink" href="#revision-record" title="Link to
 <dd class="field-odd"><p>3.00.54rc1</p>
 </dd>
 <dt class="field-even">date<span class="colon">:</span></dt>
-<dd class="field-even"><p>Jun 13, 2024</p>
+<dd class="field-even"><p>Jun 17, 2024</p>
 </dd>
 </dl>
 </section>

How2Guides/source.html

@@ -135,7 +135,7 @@ <h3>Revision Record<a class="headerlink" href="#revision-record" title="Link to
 <dd class="field-odd"><p>3.00.54rc1</p>
 </dd>
 <dt class="field-even">date<span class="colon">:</span></dt>
-<dd class="field-even"><p>Jun 13, 2024</p>
+<dd class="field-even"><p>Jun 17, 2024</p>
 </dd>
 </dl>
 <p>A Sarracenia data pump is a web (or sftp) server with notifications for subscribers

How2Guides/subscriber.html

@@ -133,7 +133,7 @@ <h3>Revision Record<a class="headerlink" href="#revision-record" title="Link to
 <dd class="field-odd"><p>3.00.54rc1</p>
 </dd>
 <dt class="field-even">date<span class="colon">:</span></dt>
-<dd class="field-even"><p>Jun 13, 2024</p>
+<dd class="field-even"><p>Jun 17, 2024</p>
 </dd>
 </dl>
 </section>

Reference/sr3.1.html

@@ -120,7 +120,7 @@ <h2>sr3 Sarracenia CLI<a class="headerlink" href="#sr3-sarracenia-cli" title="Li
 <dd class="field-odd"><p>1</p>
 </dd>
 <dt class="field-even">Date<span class="colon">:</span></dt>
-<dd class="field-even"><p>Jun 13, 2024</p>
+<dd class="field-even"><p>Jun 17, 2024</p>
 </dd>
 <dt class="field-odd">Version<span class="colon">:</span></dt>
 <dd class="field-odd"><p>3.00.54rc1</p>

Reference/sr3_cpump.1.html

@@ -117,7 +117,7 @@ <h2>sr_shovel in C<a class="headerlink" href="#sr-shovel-in-c" title="Link to th
 <dd class="field-odd"><p>1</p>
 </dd>
 <dt class="field-even">Date<span class="colon">:</span></dt>
-<dd class="field-even"><p>Jun 13, 2024</p>
+<dd class="field-even"><p>Jun 17, 2024</p>
 </dd>
 <dt class="field-odd">Version<span class="colon">:</span></dt>
 <dd class="field-odd"><p>3.00.54rc1</p>

Reference/sr3_credentials.7.html

@@ -114,7 +114,7 @@ <h2>SR3 Credential File Format<a class="headerlink" href="#sr3-credential-file-f
 <dd class="field-odd"><p>7</p>
 </dd>
 <dt class="field-even">Date<span class="colon">:</span></dt>
-<dd class="field-even"><p>Jun 13, 2024</p>
+<dd class="field-even"><p>Jun 17, 2024</p>
 </dd>
 <dt class="field-odd">Version<span class="colon">:</span></dt>
 <dd class="field-odd"><p>3.00.54rc1</p>

Reference/sr3_options.7.html

@@ -118,7 +118,7 @@ <h2>SR3 Configuration File Format<a class="headerlink" href="#sr3-configuration-
 <dd class="field-odd"><p>7</p>
 </dd>
 <dt class="field-even">Date<span class="colon">:</span></dt>
-<dd class="field-even"><p>Jun 13, 2024</p>
+<dd class="field-even"><p>Jun 17, 2024</p>
 </dd>
 <dt class="field-odd">Version<span class="colon">:</span></dt>
 <dd class="field-odd"><p>3.00.54rc1</p>

Reference/sr3_post.1.html

@@ -118,7 +118,7 @@ <h2>Publish the Availability of Files<a class="headerlink" href="#publish-the-av
 <dd class="field-odd"><p>1</p>
 </dd>
 <dt class="field-even">Date<span class="colon">:</span></dt>
-<dd class="field-even"><p>Jun 13, 2024</p>
+<dd class="field-even"><p>Jun 17, 2024</p>
 </dd>
 <dt class="field-odd">Version<span class="colon">:</span></dt>
 <dd class="field-odd"><p>3.00.54rc1</p>

Reference/sr_post.7.html

@@ -123,7 +123,7 @@ <h2>Sarracenia v03 Notification Message Format/Protocol<a class="headerlink" hre
 <dd class="field-odd"><p>7</p>
 </dd>
 <dt class="field-even">Date<span class="colon">:</span></dt>
-<dd class="field-even"><p>Jun 13, 2024</p>
+<dd class="field-even"><p>Jun 17, 2024</p>
 </dd>
 <dt class="field-odd">Version<span class="colon">:</span></dt>
 <dd class="field-odd"><p>3.00.54rc1</p>

Tutorials/Install.html

@@ -124,7 +124,7 @@ <h2>Revision Record<a class="headerlink" href="#revision-record" title="Link to
 <dd class="field-odd"><p>3.00.54rc1</p>
 </dd>
 <dt class="field-even">date<span class="colon">:</span></dt>
-<dd class="field-even"><p>Jun 13, 2024</p>
+<dd class="field-even"><p>Jun 17, 2024</p>
 </dd>
 </dl>
 </section>

_modules/sarracenia.html

@@ -461,16 +461,20 @@ <h1>Source code for sarracenia</h1><div class="highlight"><pre>
     <span class="k">return</span> <span class="n">duration</span></div>
 
 
+<span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">  report codes are cribbed from HTTP, when a new situation arises, just peruse a list,</span>
+<span class="sd">  and pick one that fits. Should also be easier for others to use:</span>
+
+<span class="sd">  https://en.wikipedia.org/wiki/List_of_HTTP_status_codes</span>
 
+<span class="sd">&quot;&quot;&quot;</span>
 <span class="n">known_report_codes</span> <span class="o">=</span> <span class="p">{</span>
     <span class="mi">201</span><span class="p">:</span> <span class="s2">&quot;Download successful. (variations: Downloaded, Inserted, Published, Copied, or Linked)&quot;</span><span class="p">,</span>
     <span class="mi">202</span><span class="p">:</span> <span class="s2">&quot;Accepted. mkdir skipped as it already exists&quot;</span><span class="p">,</span> 
     <span class="mi">203</span><span class="p">:</span> <span class="s2">&quot;Non-Authoritative Information: transformed during download.&quot;</span><span class="p">,</span>
-    <span class="mi">205</span><span class="p">:</span>
-    <span class="s2">&quot;Reset Content: truncated. File is shorter than originally expected (changed length during transfer) This only arises during multi-part transfers.&quot;</span><span class="p">,</span>
     <span class="mi">205</span><span class="p">:</span> <span class="s2">&quot;Reset Content: checksum recalculated on receipt.&quot;</span><span class="p">,</span>
-    <span class="mi">304</span><span class="p">:</span>
-    <span class="s2">&quot;Not modified (Checksum validated, unchanged, so no download resulted.)&quot;</span><span class="p">,</span>
+    <span class="mi">206</span><span class="p">:</span> <span class="s2">&quot;Partial Content: received and inserted.&quot;</span><span class="p">,</span>
+    <span class="mi">304</span><span class="p">:</span> <span class="s2">&quot;Not modified (Checksum validated, unchanged, so no download resulted.)&quot;</span><span class="p">,</span>
     <span class="mi">307</span><span class="p">:</span> <span class="s2">&quot;Insertion deferred (writing to temporary part file for the moment.)&quot;</span><span class="p">,</span>
     <span class="mi">417</span><span class="p">:</span> <span class="s2">&quot;Expectation Failed: invalid notification message (corrupt headers)&quot;</span><span class="p">,</span>
     <span class="mi">422</span><span class="p">:</span> <span class="s2">&quot;Unprocessable Content: could not determine path to transfer to&quot;</span><span class="p">,</span>
@@ -874,7 +878,7 @@ <h1>Source code for sarracenia</h1><div class="highlight"><pre>
                 <span class="n">text</span> <span class="o">=</span> <span class="s1">&#39;unknown disposition&#39;</span>
 
         <span class="k">if</span> <span class="s1">&#39;report&#39;</span> <span class="ow">in</span> <span class="n">msg</span><span class="p">:</span>
-            <span class="n">logger</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="s1">&#39;overriding initial report: </span><span class="si">%d</span><span class="s1">: </span><span class="si">%s</span><span class="s1">&#39;</span> <span class="o">%</span>
+            <span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s1">&#39;overriding initial report: </span><span class="si">%d</span><span class="s1">: </span><span class="si">%s</span><span class="s1">&#39;</span> <span class="o">%</span>
                            <span class="p">(</span><span class="n">msg</span><span class="p">[</span><span class="s1">&#39;report&#39;</span><span class="p">][</span><span class="s1">&#39;code&#39;</span><span class="p">],</span> <span class="n">msg</span><span class="p">[</span><span class="s1">&#39;report&#39;</span><span class="p">][</span><span class="s1">&#39;message&#39;</span><span class="p">]))</span>
 
         <span class="n">msg</span><span class="p">[</span><span class="s1">&#39;report&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="p">{</span><span class="s1">&#39;code&#39;</span><span class="p">:</span> <span class="n">code</span><span class="p">,</span> <span class="s1">&#39;timeCompleted&#39;</span><span class="p">:</span> <span class="n">nowstr</span><span class="p">(),</span> <span class="s1">&#39;message&#39;</span><span class="p">:</span> <span class="n">text</span><span class="p">}</span>

_modules/sarracenia/config.html

@@ -185,7 +185,6 @@ <h1>Source code for sarracenia.config</h1><div class="highlight"><pre>
     <span class="s1">&#39;batch&#39;</span> <span class="p">:</span> <span class="mi">100</span><span class="p">,</span>
     <span class="s1">&#39;baseDir&#39;</span><span class="p">:</span> <span class="kc">None</span><span class="p">,</span>
     <span class="s1">&#39;baseUrl_relPath&#39;</span><span class="p">:</span> <span class="kc">False</span><span class="p">,</span>
-    <span class="s1">&#39;block_reassemble&#39;</span><span class="p">:</span> <span class="kc">True</span><span class="p">,</span>
     <span class="s1">&#39;delete&#39;</span><span class="p">:</span> <span class="kc">False</span><span class="p">,</span>
     <span class="s1">&#39;documentRoot&#39;</span><span class="p">:</span> <span class="kc">None</span><span class="p">,</span>
     <span class="s1">&#39;download&#39;</span><span class="p">:</span> <span class="kc">False</span><span class="p">,</span>
@@ -242,7 +241,7 @@ <h1>Source code for sarracenia.config</h1><div class="highlight"><pre>
 
 
 <span class="c1"># all the boolean settings.</span>
-<span class="n">flag_options</span> <span class="o">=</span> <span class="p">[</span> <span class="s1">&#39;acceptSizeWrong&#39;</span><span class="p">,</span> <span class="s1">&#39;acceptUnmatched&#39;</span><span class="p">,</span> <span class="s1">&#39;amqp_consumer&#39;</span><span class="p">,</span> <span class="s1">&#39;baseUrl_relPath&#39;</span><span class="p">,</span> <span class="s1">&#39;block_reassemble&#39;</span><span class="p">,</span> <span class="s1">&#39;debug&#39;</span><span class="p">,</span> \
+<span class="n">flag_options</span> <span class="o">=</span> <span class="p">[</span> <span class="s1">&#39;acceptSizeWrong&#39;</span><span class="p">,</span> <span class="s1">&#39;acceptUnmatched&#39;</span><span class="p">,</span> <span class="s1">&#39;amqp_consumer&#39;</span><span class="p">,</span> <span class="s1">&#39;baseUrl_relPath&#39;</span><span class="p">,</span> <span class="s1">&#39;debug&#39;</span><span class="p">,</span> \
     <span class="s1">&#39;delete&#39;</span><span class="p">,</span> <span class="s1">&#39;discard&#39;</span><span class="p">,</span> <span class="s1">&#39;download&#39;</span><span class="p">,</span> <span class="s1">&#39;dry_run&#39;</span><span class="p">,</span> <span class="s1">&#39;durable&#39;</span><span class="p">,</span> <span class="s1">&#39;exchangeDeclare&#39;</span><span class="p">,</span> <span class="s1">&#39;exchangeSplit&#39;</span><span class="p">,</span> <span class="s1">&#39;logReject&#39;</span><span class="p">,</span> <span class="s1">&#39;realpathFilter&#39;</span><span class="p">,</span> \
     <span class="s1">&#39;follow_symlinks&#39;</span><span class="p">,</span> <span class="s1">&#39;force_polling&#39;</span><span class="p">,</span> <span class="s1">&#39;inline&#39;</span><span class="p">,</span> <span class="s1">&#39;inlineOnly&#39;</span><span class="p">,</span> <span class="s1">&#39;inplace&#39;</span><span class="p">,</span> <span class="s1">&#39;logMetrics&#39;</span><span class="p">,</span> <span class="s1">&#39;logStdout&#39;</span><span class="p">,</span> <span class="s1">&#39;logReject&#39;</span><span class="p">,</span> <span class="s1">&#39;restore&#39;</span><span class="p">,</span> \
     <span class="s1">&#39;messageDebugDump&#39;</span><span class="p">,</span> <span class="s1">&#39;mirror&#39;</span><span class="p">,</span> <span class="s1">&#39;timeCopy&#39;</span><span class="p">,</span> <span class="s1">&#39;notify_only&#39;</span><span class="p">,</span> <span class="s1">&#39;overwrite&#39;</span><span class="p">,</span> <span class="s1">&#39;post_on_start&#39;</span><span class="p">,</span> \
@@ -1673,8 +1672,6 @@ <h1>Source code for sarracenia.config</h1><div class="highlight"><pre>
         <span class="k">if</span> <span class="n">k</span> <span class="o">==</span> <span class="s1">&#39;continue&#39;</span><span class="p">:</span>
             <span class="k">return</span>
 
-        <span class="c1">#FIXME: note for Clea, line conversion to v3 complete here.</span>
-
         <span class="n">line</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="nb">map</span><span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">_varsub</span><span class="p">(</span><span class="n">x</span><span class="p">),</span> <span class="n">line</span><span class="p">))</span>
 
         <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">line</span><span class="p">)</span> <span class="o">==</span> <span class="mi">1</span><span class="p">:</span>

_modules/sarracenia/flow.html

@@ -300,6 +300,9 @@ <h1>Source code for sarracenia.flow</h1><div class="highlight"><pre>
 
         <span class="bp">self</span><span class="o">.</span><span class="n">plugins</span><span class="p">[</span><span class="s1">&#39;load&#39;</span><span class="p">]</span><span class="o">.</span><span class="n">extend</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">o</span><span class="o">.</span><span class="n">destfn_scripts</span><span class="p">)</span>
 
+        <span class="bp">self</span><span class="o">.</span><span class="n">block_reassembly_active</span> <span class="o">=</span> <span class="s1">&#39;block_reassembly&#39;</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">plugins</span><span class="p">[</span><span class="s1">&#39;load&#39;</span><span class="p">]</span> <span class="ow">or</span> \
+                 <span class="s1">&#39;sarracenia.flowcb.block_reassembly&#39;</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">plugins</span><span class="p">[</span><span class="s1">&#39;load&#39;</span><span class="p">]</span>
+
         <span class="c1"># metrics - dictionary with names of plugins as the keys</span>
         <span class="bp">self</span><span class="o">.</span><span class="n">metrics_lastWrite</span><span class="o">=</span><span class="mi">0</span>
         <span class="bp">self</span><span class="o">.</span><span class="n">metricsFlowReset</span><span class="p">()</span>
@@ -2829,11 +2832,11 @@ <h1>Source code for sarracenia.flow</h1><div class="highlight"><pre>
         <span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">&quot;</span><span class="si">%s</span><span class="s2">&quot;</span> <span class="o">%</span> <span class="n">local_file</span><span class="p">)</span>
 
         <span class="c1"># if the file is not partitioned, the the onfly_checksum is for the whole file.</span>
-        <span class="c1"># cache it here, along with the mtime.</span>
-
-        <span class="k">if</span> <span class="p">(</span><span class="s1">&#39;blocks&#39;</span> <span class="ow">in</span> <span class="n">msg</span><span class="p">)</span> <span class="ow">and</span> <span class="n">sarracenia</span><span class="o">.</span><span class="n">features</span><span class="p">[</span><span class="s1">&#39;reassembly&#39;</span><span class="p">][</span><span class="s1">&#39;present&#39;</span><span class="p">]:</span>
-            <span class="k">with</span> <span class="n">sarracenia</span><span class="o">.</span><span class="n">blockmanifest</span><span class="o">.</span><span class="n">BlockManifest</span><span class="p">(</span><span class="n">local_file</span><span class="p">)</span> <span class="k">as</span> <span class="n">y</span><span class="p">:</span>
-                <span class="n">y</span><span class="o">.</span><span class="n">set</span><span class="p">(</span> <span class="n">msg</span><span class="p">[</span><span class="s1">&#39;blocks&#39;</span><span class="p">]</span> <span class="p">)</span>
+        <span class="c1"># cache it here, along with the mtime, unless block_reassembly plugin is active...</span>
+        
+        <span class="k">if</span> <span class="p">(</span><span class="s1">&#39;blocks&#39;</span> <span class="ow">in</span> <span class="n">msg</span><span class="p">)</span> <span class="ow">and</span> <span class="n">sarracenia</span><span class="o">.</span><span class="n">features</span><span class="p">[</span><span class="s1">&#39;reassembly&#39;</span><span class="p">][</span><span class="s1">&#39;present&#39;</span><span class="p">]</span> <span class="ow">and</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">block_reassembly_active</span><span class="p">:</span>
+            <span class="k">with</span> <span class="n">sarracenia</span><span class="o">.</span><span class="n">blockmanifest</span><span class="o">.</span><span class="n">BlockManifest</span><span class="p">(</span><span class="n">local_file</span><span class="p">)</span> <span class="k">as</span> <span class="n">bm</span><span class="p">:</span>
+                <span class="n">bm</span><span class="o">.</span><span class="n">set</span><span class="p">(</span> <span class="n">msg</span><span class="p">[</span><span class="s1">&#39;blocks&#39;</span><span class="p">]</span> <span class="p">)</span>
 
         <span class="n">x</span> <span class="o">=</span> <span class="n">sarracenia</span><span class="o">.</span><span class="n">filemetadata</span><span class="o">.</span><span class="n">FileMetadata</span><span class="p">(</span><span class="n">local_file</span><span class="p">)</span>
         <span class="c1"># FIXME ... what to do when checksums don&#39;t match?</span>