3.5.5

Author: A. Nick Georgieff

bin/fix_task.pl

@@ -0,0 +1,53 @@
+#!/usr/bin/env perl
+
+use MongoDB;
+use Data::Dumper;
+use v5.18;
+
+my $mongo       = MongoDB->connect->db('scot-prod');
+my $collection  = $mongo->get_collection('entry');
+my $cursor      = $collection->find({class => "task"});
+
+print "starting...\n";
+print $cursor->count . " task records\n";
+my %lookup  = ();
+
+while (my $task = $cursor->next) {
+
+    my $id       = $task->{id};
+    print "...task $id\n";
+
+    my $meta = $task->{metadata};
+
+    if ( defined $meta ) {
+
+        if ( defined $meta->{status} 
+             && defined $meta->{when}
+             && defined $meta->{who} ) {
+
+            say "... ... has valid task meta structure";
+
+            my $thref = {
+                status  => $meta->{status},
+                when    => $meta->{when},
+                who     => $meta->{who},
+            };
+
+            delete $meta->{status};
+            delete $meta->{when};
+            delete $meta->{who};
+
+            $meta->{task} = $thref;
+
+            my $set     = {
+                metadata    => $meta
+            };
+
+            $collection->update_one(
+                {id => $id},
+                {'$set' => $set }
+            );
+            say  "Update == ".Dumper($set);
+        }
+    }
+}

install/src/mongodb/reset.js

@@ -22,6 +22,7 @@ db.signature.drop();
 db.sigbody.drop();
 db.apikey.drop();
 db.handler.drop();
+db.stat.drop();
 
 print ("Creating indexes...");
 load("./src/mongodb/indexes.js");

install/src/scot/scot.cfg.pl

@@ -63,6 +63,11 @@
         log_level       => 'DEBUG',
     },
 
+    cgi_ids_config  => {
+        whitelist_file  => '',
+        disable_filters => [],
+    },
+
     # modules to instantiate at Env.pm startup. will be done in 
     # order of the array
     modules => [

lib/Scot/App/Flair.pm

@@ -83,19 +83,36 @@ sub _get_img_munger {
     return $env->img_munger;
 };
 
-#has scot        => (
-#    is          => 'ro',
-#    isa         => 'Scot::Util::Scot2',
-#    required    => 1,
-#    lazy        => 1,
-#    builder     => '_build_scot_scot',
-#);
-
-#sub _build_scot_scot {
-#    my $self    = shift;
-#    my $env     = shift;
-#    return $env->scot;
-#}
+has stomp_host  => (
+    is          => 'ro',
+    isa         => 'Str',
+    required    => 1,
+    lazy        => 1,
+    builder     => '_build_stomp_host',
+);
+
+sub _build_stomp_host {
+    my $self    = shift;
+    my $attr    = "stomp_host";
+    my $default = "localhost";
+    my $envname = "scot_util_stomphost";
+    return $self->get_config_value($attr, $default, $envname);
+}
+has stomp_port  => (
+    is          => 'ro',
+    isa         => 'Int',
+    required    => 1,
+    lazy        => 1,
+    builder     => '_build_stomp_port',
+);
+
+sub _build_stomp_port {
+    my $self    = shift;
+    my $attr    = "stomp_port";
+    my $default = 61613;
+    my $envname = "scot_util_stompport";
+    return $self->get_config_value($attr, $default, $envname);
+}
 
 has interactive => (
     is          => 'ro',
@@ -156,7 +173,14 @@ sub run {
     my $self    = shift;
     my $log     = $self->log;
     my $pm      = AnyEvent::ForkManager->new(max_workers => $self->max_workers);
-    my $stomp   = AnyEvent::STOMP::Client->new();
+    my $stomp;
+
+    if ( $self->stomp_host ne "localhost" ) {
+        $stomp   = AnyEvent::STOMP::Client->new($self->stomp_host, $self->stomp_port);
+    }
+    else {
+        $stomp = AnyEvent::STOMP::Client->new;
+    }
 
     $stomp->connect();
     $stomp->on_connected(sub {
@@ -384,6 +408,7 @@ sub flair_record {
 
             if ( $column =~ /^message[_-]id$/i ) {
                 # the data is telling us that this is a email message_id, so flair
+                $value =~ s/[\<\>]//g;   # cut out the < > brackets if they exist
                 my ($eref, $flair) = $self->process_cell($value, "message_id");
                 if ( ! defined $seen{$eref->{value}} ) {
                     push @entity, $eref;
@@ -409,7 +434,15 @@ sub flair_record {
                 # each link in this field is a <div>filename</div>, 
                 $log->debug("A File attachment Column detected!");
                 $log->debug("value = ",{filter=>\&Dumper, value=>$value});
+                
+                if ( $value eq "" || $value eq " " ) {
+                    next VALUE;
+                }
 
+                if ( $value eq "" || $value eq " " ) {
+                    next VALUE;
+                }
+                
                 my ($eref, $flair) = $self->process_cell($value, "filename");
                 if ( ! defined $seen{$eref->{value}} ) {
                     push @entity, $eref;
@@ -464,6 +497,8 @@ sub process_cell {
     my $log     = $self->env->log;
 
     $log->debug("text   = $text");
+    $text = encode_entities($text);
+    $log->debug("encoded text   = $text");
     $log->debug("header = $header");
     my $flair   = $self->genspan($text,$header);
     $log->debug("text   = $text");

lib/Scot/App/Stretch.pm

@@ -95,6 +95,37 @@ sub _get_max_workers {
     return $self->get_config_value($attr, $default, $envname);
 }
 
+has stomp_host  => (
+    is          => 'ro',
+    isa         => 'Str',
+    required    => 1,
+    lazy        => 1,
+    builder     => '_build_stomp_host',
+);
+
+sub _build_stomp_host {
+    my $self    = shift;
+    my $attr    = "stomp_host";
+    my $default = "localhost";
+    my $envname = "scot_util_stomphost";
+    return $self->get_config_value($attr, $default, $envname);
+}
+has stomp_port  => (
+    is          => 'ro',
+    isa         => 'Int',
+    required    => 1,
+    lazy        => 1,
+    builder     => '_build_stomp_port',
+);
+
+sub _build_stomp_port {
+    my $self    = shift;
+    my $attr    = "stomp_port";
+    my $default = 61613;
+    my $envname = "scot_util_stompport";
+    return $self->get_config_value($attr, $default, $envname);
+}
+
 =head2 Autonomous
 
 $stretch->run();
@@ -112,7 +143,15 @@ sub run {
     $log->debug("Starting STOMP watcher");
     $log->debug("SCOT access mode is ".$self->scot_get_method);
 
-    my $stomp   = AnyEvent::STOMP::Client->new();
+    my $stomp;
+
+    if ( $self->stomp_host eq "localhost" ) {
+        $stomp   = AnyEvent::STOMP::Client->new();
+    }
+    else {
+        $stomp   = AnyEvent::STOMP::Client->new($self->stomp_host, $self->stomp_port);
+    }
+
     my $pm      = AnyEvent::ForkManager->new( 
         max_workers => $self->max_workers 
     );
@@ -182,7 +221,7 @@ sub process_message {
     my $cleanser = Data::Clean::FromJSON->get_cleanser;
     my $record  = $self->get_document($type, $id);
     $cleanser->clean_in_place($record);
-    $es->index($type, $record, 'scot');
+    $es->index('scot', $type, $record);
     $self->put_stat("elastic doc inserted", 1);
 }
 
@@ -291,7 +330,7 @@ sub process_all {
 
             delete $href->{_id};
             try {
-                $es->index($collection, $href, 'scot');
+                $es->index('scot',$collection, $href);
             }
             catch {
                 say "Error: Failed to index $collection : ". $href->{id};

lib/Scot/Collection.pm

@@ -24,6 +24,8 @@ has env => (
     default     => sub { Scot::Env->instance; },
 );
 
+
+
 =item B<create(%args)>
 
 replacing Meerkat's create with one that will generate a integer id.
@@ -518,7 +520,8 @@ sub api_list {
 
     if ( $href->{task_search} ) {
         # $match->{'task.status'}     = {'$exists'    => 1};
-        $match->{'metadata.status'} = {'$exists'    => 1};
+        # $match->{'metadata.status'} = {'$exists'    => 1};
+        $match->{class} = "task";
     }
 
     $self->env->log->debug("match is ",{filter=>\&Dumper, value=>$match});
@@ -624,7 +627,13 @@ sub api_find {
         my $obj = $self->find_iid($id);
         return $obj;
     }
-    my $id  = $href->{id} + 0;
+    my $id  = $href->{id};
+    if ( $id eq "undefined" ) {
+        $id = 0;
+    }
+    else {
+        $id += 0;
+    }
     my $obj = $self->find_iid($id);
     return $obj;
 

lib/Scot/Collection/Apikey.pm

@@ -40,10 +40,11 @@ override api_list => sub {
     my $req     = shift;
     my $user    = shift;
     my $groups  = shift;
+    my $env     = $self->env;
 
     my $match   = $self->build_match_ref($req->{request});
 
-    if (! $self->env->is_admin($user,$groups) ) {
+    if (! $env->is_admin($user,$groups) ) {
         $match->{username}  = $user;
     }
     my $cursor  = $self->find($match);

lib/Scot/Collection/Audit.pm

@@ -31,7 +31,7 @@ sub create_audit_rec {
         groups  => $handler->session('groups'),
         when    => $self->env->now,
         method  => $req->method,
-        url     => $req->url->to_abs,
+        url     => $req->url->to_abs->to_string,
         from    => $handler->tx->remote_address,
         agent   => $req->headers->user_agent,
         params  => $req->params->to_hash,