5.0/create chart images for dashboard emails 2

devel/third-party/ckeditor-4.20.1/README

@@ -46,3 +46,7 @@ RT uses CKEditor as its WYSIWYG text editor. We build a customized download, the
       $ rm -rf skins/bootstrapck/sample
       $ rm -rf plugins/confighelper/docs/
       $ rm -rf plugins/ccmsconfighelper/docs
+
+    * Drop UTF-8 BOM from js files:
+
+      $ find share/static/RichText -name '*.js' | xargs dos2unix

docs/UPGRADING-5.0

@@ -634,4 +634,27 @@ messages, you may need to update your system to match the new format.
 
 =back
 
+=head1 UPGRADING FROM 5.0.5 AND EARLIER
+
+=over 4
+
+=item * Additional options for charts in dashboard emails
+
+While it has been possible to use JSChart to generate chart images in the RT UI,
+because these images are generated client-side it hasn't been possible to include
+them in dashboard emails, so the GD-generated images have been the only option.
+
+It is now possible to use the optional Perl module L<WWW::Mechanize::Chrome> and
+a compatible server-side web brwoser to create images of the JSChart graphs for
+inclusion in emails.
+
+This is accomplished by setting C<$EmailDashboardJSChartImages> to '1' and
+maybe also setting C<$ChromePath> to the path of the executable for your
+chosen Chrome-based browser.
+
+This feature has been tested with Chrome, Chromium, Microsoft Edge, and Opera.
+Other Chrome-based browsers may also work.
+
+=back
+
 =cut

etc/RT_Config.pm.in

@@ -971,6 +971,44 @@ With this enabled, some parts of the email won't look exactly like RT.
 
 Set($EmailDashboardInlineCSS, 0);
 
+=item C<$EmailDashboardJSChartImages>
+
+To use the JSChart-generated images in emailed dashboards, install the
+optional module L<WWW::Mechanize::Chrome> and enable this option.
+
+=cut
+
+Set($EmailDashboardJSChartImages, 0);
+
+=item C<$ChromePath>
+
+This option contains the path for a compatible Chrome-based browser
+executable that will be used to generate static images for JSChart
+graphs for dashboard emails.
+
+See also L<WWW::Mechanize::Chrome/launch_exe>
+
+=cut
+
+Set($ChromePath, 'chromium');
+
+=item C<@ChromeLaunchArguments>
+
+This option contains the launch arguments when initializing
+L<WWW::Mechanize::Chrome>.
+
+If you need to run L<rt-email-dashboards> as root, you probably need to add
+C<--no-sandbox> to get around Chrome's restriction:
+
+    Set(@ChromeLaunchArguments, '--no-sandbox');
+
+See also L<WWW::Mechanize::Chrome/launch_arg>
+
+=cut
+
+Set(@ChromeLaunchArguments, () );
+
+
 =back
 
 

lib/RT/Config.pm

@@ -1994,6 +1994,31 @@ our %META;
     EmailDashboardInlineCSS => {
         Widget => '/Widgets/Form/Boolean',
     },
+    EmailDashboardJSChartImages => {
+        Widget => '/Widgets/Form/Boolean',
+        PostLoadCheck => sub {
+            my $self = shift;
+            return unless $self->Get('EmailDashboardJSChartImages');
+
+            if ( RT::StaticUtil::RequireModule('WWW::Mechanize::Chrome') ) {
+                my $chrome = RT->Config->Get('ChromePath') || 'chromium';
+                if ( !WWW::Mechanize::Chrome->find_executable( $chrome ) ) {
+                    RT->Logger->warning("Can't find chrome executable from \$ChromePath value '$chrome', disabling \$EmailDashboardJSChartImages");
+                    $self->Set( 'EmailDashboardJSChartImages', 0 );
+                }
+            }
+            else {
+                RT->Logger->warning('WWW::Mechanize::Chrome is not installed, disabling $EmailDashboardJSChartImages');
+                $self->Set( 'EmailDashboardJSChartImages', 0 );
+            }
+        },
+    },
+    ChromePath => {
+        Widget => '/Widgets/Form/String',
+    },
+    ChromeLaunchArguments => {
+        Type => 'ARRAY',
+    },
     DefaultErrorMailPrecedence => {
         Widget => '/Widgets/Form/String',
     },

lib/RT/Dashboard/Mailer.pm

@@ -423,8 +423,6 @@ SUMMARY
         }
     }
 
-    $content = ScrubContent($content);
-
     $RT::Logger->debug("Got ".length($content)." characters of output.");
 
     $content = HTML::RewriteAttributes::Links->rewrite(
@@ -536,6 +534,8 @@ sub EmailDashboard {
     $RT::Logger->debug("Done sending dashboard to ".$currentuser->Name." <$email>");
 }
 
+my $chrome;
+
 sub BuildEmail {
     my $self = shift;
     my %args = (
@@ -592,6 +592,133 @@ sub BuildEmail {
         inline_imports => 1,
     );
 
+    # This needs to be done after all of the CSS has been imported (by
+    # inline_css above, which is distinct from the work done by CSS::Inliner
+    # below) and before all of the scripts are scrubbed away.
+    if ( RT->Config->Get('EmailDashboardJSChartImages') ) {
+        # WWW::Mechanize::Chrome uses Log::Log4perl and calls trace sometimes.
+        # Here we merge trace to debug.
+        my $is_debug;
+        for my $type (qw/LogToSyslog LogToSTDERR LogToFile/) {
+            my $log_level = RT->Config->Get($type) or next;
+            if ( $log_level eq 'debug' ) {
+                $is_debug = 1;
+                last;
+            }
+        }
+
+        local *Log::Dispatch::is_trace = sub { $is_debug || 0 };
+        local *Log::Dispatch::trace    = sub {
+            my $self = shift;
+            return $self->debug(@_);
+        };
+
+        my ( $width, $height );
+        my @launch_arguments = RT->Config->Get('ChromeLaunchArguments');
+
+        for my $arg (@launch_arguments) {
+            if ( $arg =~ /^--window-size=(\d+)x(\d+)$/ ) {
+                $width  = $1;
+                $height = $2;
+                last;
+            }
+        }
+
+        $width  ||= 2560;
+        $height ||= 1440;
+
+        $chrome ||= WWW::Mechanize::Chrome->new(
+            autodie          => 0,
+            headless         => 1,
+            autoclose        => 1,
+            separate_session => 1,
+            log              => RT->Logger,
+            launch_arg       => \@launch_arguments,
+            launch_exe       => RT->Config->Get('ChromePath') || 'chromium',
+        );
+
+        # copy the content
+        my $content_with_script = $content;
+
+        # copy in the text of the linked js
+        $content_with_script
+            =~ s{<script type="text/javascript" src="([^"]+)"></script>}{<script type="text/javascript">@{ [(GetResource( $1 ))[0]] }</script>}g;
+
+        # write the complete content to a temp file
+        my $temp_fh = File::Temp->new(
+            UNLINK   => 1,
+            TEMPLATE => 'email-dashboard-XXXXXX',
+            SUFFIX   => '.html',
+            DIR      => $RT::VarPath,               # $chrome can't get the file if saved to /tmp
+        );
+        print $temp_fh Encode::encode( 'UTF-8', $content_with_script );
+        close $temp_fh;
+
+        $chrome->viewport_size( { width => $width, height => $height } );
+        $chrome->get_local( $temp_fh->filename );
+        $chrome->wait_until_visible( selector => 'div.dashboard' );
+
+        # grab the list of canvas elements
+        my @canvases = $chrome->selector('div.chart canvas');
+        if (@canvases) {
+
+            my $max_extent = 0;
+
+            # ... and their coordinates
+            foreach my $canvas_data (@canvases) {
+                my $coords = $canvas_data->{coords} = $chrome->element_coordinates($canvas_data);
+                if ( $max_extent < $coords->{top} + $coords->{height} ) {
+                    $max_extent = int( $coords->{top} + $coords->{height} ) + 1;
+                }
+            }
+
+            # make sure that all of them are "visible" in the headless instance
+            if ( $height < $max_extent ) {
+                $chrome->viewport_size( { width => $width, height => $max_extent } );
+            }
+
+            # capture the entire page as an image
+            my $page_image = $chrome->_content_as_png( undef, { width => $width, height => $height } )->get;
+
+            my $cid = time() . $$;
+            foreach my $canvas_data (@canvases) {
+                $cid++;
+
+                my $coords       = $canvas_data->{coords};
+                my $canvas_image = $page_image->crop(
+                    left   => $coords->{left},
+                    top    => $coords->{top},
+                    width  => $coords->{width},
+                    height => $coords->{height},
+                );
+                my $canvas_data;
+                $canvas_image->write( data => \$canvas_data, type => 'png' );
+
+                # replace each canvas in the original content with an image tag
+                $content =~ s{<canvas [^>]+>}{<img src="cid:$cid"/>};
+
+                push @parts,
+                    MIME::Entity->build(
+                        Top          => 0,
+                        Data         => $canvas_data,
+                        Type         => 'image/png',
+                        Encoding     => 'base64',
+                        Disposition  => 'inline',
+                        'Content-Id' => "<$cid>",
+                    );
+            }
+        }
+
+        # Shut down chrome if it's a test email from web UI, to reduce memory usage.
+        # Unset $chrome so next time it can re-create a new one.
+        if ( $args{Test} ) {
+            $chrome->close;
+            undef $chrome;
+        }
+    }
+
+    $content =~ s{<link rel="shortcut icon"[^>]+/>}{};
+
     # Inline the CSS if CSS::Inliner is installed and can be loaded
     if ( RT->Config->Get('EmailDashboardInlineCSS') ) {
         if ( RT::StaticUtil::RequireModule('CSS::Inliner') ) {
@@ -609,6 +736,8 @@ sub BuildEmail {
         }
     }
 
+    $content = ScrubContent($content);
+
     my $entity = MIME::Entity->build(
         From    => Encode::encode("UTF-8", $args{From}),
         To      => Encode::encode("UTF-8", $args{To}),
@@ -782,6 +911,12 @@ sub GetResource {
     # the rest of this was taken from Email::MIME::CreateHTML::Resolver::LWP
     ($mimetype, $encoding) = MIME::Types::by_suffix($filename);
 
+    if ( $mimetype =~ m{^(?:text/|application/(?:x-)?javascript)}
+        && !utf8::is_utf8($content) )
+    {
+        $content = Encode::decode( 'UTF-8', $content );
+    }
+
     if ($content_type) {
         $mimetype = $content_type;
 

sbin/rt-email-dashboards.in

@@ -89,15 +89,16 @@ RT::LoadConfig();
 # adjust logging to the screen according to options
 RT->Config->Set( LogToSTDERR => $opts{log} ) if $opts{log};
 
-# Disable JS chart as email clients don't support it
-RT->Config->Set( EnableJSChart => 0 );
-
 # Disable inline editing as email clients don't support it
 RT->Config->Set( InlineEdit => 0 );
 
 # Connect to the database and get RT::SystemUser and RT::Nobody loaded
 RT::Init();
 
+# Disable JS chart unless EmailDashboardJSChartImages is true
+# Do this after Init as PostLoadCheck of EmailDashboardJSChartImages might disable the config.
+RT->Config->Set( EnableJSChart => RT->Config->Get( 'EmailDashboardJSChartImages' ) );
+
 require RT::Dashboard::Mailer;
 RT::Dashboard::Mailer->MailDashboards(
     All        => $opts{all},