Merge remote-tracking branch 'upstream/master' into dc-master

app/actions/document_import.rb

@@ -155,7 +155,6 @@ def process_text
     document.id
   end
 
-
   private
 
   def queue_page_text(text, page_number)
@@ -190,6 +189,10 @@ def save_page_text!
   def save_page_aspect_ratios!
     ids = document.pages.order(:page_number).pluck(:id)
 
+    # This is a query which will take a list of pages and aspect ratios
+    # and update all of the pages in a single query.
+    #
+    # Note, the page objects need to all exist for this to work.
     query_template = <<-QUERY
     UPDATE pages 
       SET aspect_ratio = input.aspect_ratio

app/actions/reindex_everything.rb

@@ -16,7 +16,7 @@ def process
         counter += 1
         (sleep(0.25 * counter) and retry) if counter < 5
         LifecycleMailer.exception_notification(e,options).deliver_now
-        outcomes[:failed].push(:id=>doc.id)
+        outcomes[:failed].push(:id=>document.id)
       end
     end
     Sunspot.commit

app/controllers/admin_controller.rb

@@ -264,7 +264,7 @@ def organization
     top_data = @documents.group(:account_id).count.sort_by{|k,v| -v}.first(@top_count).map do |arr|
       m = @memberships.where(account_id: arr.first).first
       fake_account = Struct.new(:full_name, :email, :slug)
-      accountish = m.blank? ? fake_account.new("Deleted User", nil, nil) : m.account
+      accountish = (m.blank? or m.account.nil?) ? fake_account.new("Deleted User", nil, nil) : m.account
       [accountish, arr.last]
     end
     @top_uploaders       = Hash[top_data]
@@ -378,7 +378,7 @@ def login_as
 
     # notify when an Admin logs in as a user.
     hook_url = DC::SECRETS['slack_webhook']
-    text = "#{@current_account.full_name} has logged in as #{acc.full_name} (#{acc.email} on #{Rails.env})."
+    text = "#{@current_account.full_name} (#{@current_account.email}) has logged in as #{acc.full_name} (#{acc.email} on #{Rails.env})."
     data = {:payload => {:text => text, :username => "docbot", :icon_emoji => ":doccloud:"}.to_json}
     RestClient.post(hook_url, data)
 

app/controllers/annotations_controller.rb

@@ -43,6 +43,7 @@ def show
           @exclude_analytics = true
           render template: 'annotations/show_embedded', layout: 'new'
         else
+          @include_analytics = true
           make_oembeddable(current_annotation) unless current_annotation.section_note?
           set_minimal_nav text:    'Read the full document',
                           xs_text: 'Full document',

app/controllers/home_controller.rb

@@ -40,6 +40,7 @@ def terms
     return not_found unless @version_numbers.include? @version
 
     @canonical_url = terms_url if @version == @current_terms['version']
+    @include_analytics = true
     render layout: 'new', template: "home/terms/show"
   end
 
@@ -48,18 +49,22 @@ def api_terms
     return not_found unless @version_numbers.include? @version
 
     @canonical_url = api_terms_url if @version == @current_terms['version']
+    @include_analytics = true
     render layout: 'new', template: "home/api_terms/show"
   end
 
   def status
+    @include_analytics = true
     not_found
   end
 
   def privacy
+    @include_analytics = true
     render layout: 'new'
   end
 
   def faq
+    @include_analytics = true
     render layout: 'new'
   end
 
@@ -98,6 +103,7 @@ def quackbot
 
     end
 
+    @include_analytics = true
     render layout: 'new'
   end
 

app/controllers/pages_controller.rb

@@ -30,6 +30,7 @@ def show
           @exclude_analytics = true
           render template: 'pages/show_embedded'
         else
+          @include_analytics = true
           make_oembeddable(current_page)
           set_minimal_nav text:    'Read the full document',
                           xs_text: 'Full document',

app/controllers/projects_controller.rb

@@ -1,12 +1,37 @@
 class ProjectsController < ApplicationController
 
-  before_action :login_required
+  before_action :login_required, except: :show
 
   READONLY_ACTIONS = [
-    :index, :documents
+    :index, :documents, :show
   ]
   before_action :read_only_error, :except => READONLY_ACTIONS if read_only?
 
+  def show
+    # PLEASE BE CAREFUL WITH THIS ACTION
+    #
+    # It was intentionally authored in spite of ProjectsController#current_project
+    # in order to hack around the notion that projects can be public objects
+    # and not merely objects viewable if you have edit permissions for them.
+    @project = Project.where(hidden:false).find(params[:id])
+    return not_found unless @project
+
+    respond_to do |format|
+      format.html do
+        @organization_id = @project.account.organization_id
+        render(layout: 'new')
+      end
+      format.json do
+        documents = @project.documents.accessible(current_account, current_organization).select(:id,:slug)
+        json({
+          id: @project.slug,
+          title: @project.title,
+          documents: documents.first(10000).map{ |document| document.canonical_url(:html) }
+        }.as_json)
+      end
+    end
+  end
+
   def index
     json Project.visible.accessible(current_account)
   end
@@ -45,11 +70,27 @@ def remove_documents
   end
 
   private
-
+=begin
+  Okay i want to add a notion of publicness.
+  The notions that a project has are:
+    - projects you own
+    - projects shared with you
+  
+  The current project is a project you can edit (which you own or have had shared w/ you)
+  
+  There should be public projects.
+  
+  Having a list of public projects and integer ids means that folks can enumerate projects
+  (they already can via search tho) and find groupings of public documents which may reveal
+  investigations underway.
+  
+  Need a mechanism to indicate the privacy/publicness of a project.
+  Really need to overhaul the search index.
+=end
   def current_project(only_owner=false)
     return @current_project if @current_project
     base = only_owner ? current_account.projects : Project.accessible(current_account)
     @current_project = base.find(params[:id])
   end
-
+  
 end

app/models/document.rb

@@ -293,7 +293,8 @@ def self.upload_statistics(type, id)
 
   # Ensure that titles are stripped of trailing whitespace.
   def title=(title="Untitled Document")
-    self[:title] = self.strip title.strip
+    # Clean the stripped title
+    self[:title] = CGI.unescape_html(self.strip title.strip)
   end
 
   # Save all text assets, including the `combined_page_text`, and the text of
@@ -899,7 +900,7 @@ def self.duplicate(document_ids, account, options={})
   def duplicate!(recipient=nil, options={})
     Document.transaction do
       # Clone the document.
-      newattrs = attributes.merge({
+      newattrs = attributes.dup.merge({
           :access     => PENDING,
           :created_at => Time.now,
           :updated_at => Time.now
@@ -912,7 +913,7 @@ def duplicate!(recipient=nil, options={})
 
       # Clone the docdata.
       if docdata and options['include_docdata']
-        Docdata.create! document_id: copy_id, data: docdata.data
+        Docdata.create! document_id: copy_id, data: docdata.data.dup
       end
 
       # Clone the associations.
@@ -925,7 +926,7 @@ def duplicate!(recipient=nil, options={})
       associations.push project_memberships if options['include_project']
       associations.each do |association|
         association.each do |model|
-          model_attrs = model.attributes.merge newattrs
+          model_attrs = model.attributes.dup.merge newattrs
           model_attrs.delete('id')
           model.class.create! model_attrs
         end

app/views/documents/preview.html.erb

@@ -27,9 +27,5 @@
 
   <%= render(:partial => 'viewer_editing') if @edits_enabled %>
 
-<% if @current_document %>
-  <%= render partial: 'common/google', locals: { pageview_url: '/documents/show' } %>
-<% end %>
-
 </body>
 </html>

app/views/donate/index.html.erb

@@ -18,24 +18,7 @@
     <div class="container-sm">
   <% end %>
 
-      <%= bootstrap_form_tag url: donate_charge_path, html: {id: 'donation-form'} do |f| %>
-        <%= hidden_field_tag(:stripe_token) %>
-        <%= hidden_field_tag(:stripe_email) %>
-        <%= hidden_field_tag(:donation_amount_cents) %>
-        <%= hidden_field_tag(:suggested_donation, @suggested_donation) %>
-        <div id="donation_amounts" class="btn-group" data-toggle="buttons">
-        <% @donation_levels.each do |amount| %>
-          <label class="btn btn-secondary">
-            <input name="donation_amount" id="donation-amount-<%= amount %>" value="<%= amount %>" type="radio" autocomplete="off"> $<%= amount %>
-          </label>
-        <% end %>
-          <label class="sr-only" for="donation_amount">Amount (USD)</label>
-          <input name="donation_amount_other" id="donation-amount-other" type="tel" class="form-control" placeholder="Other">
-        </div>
-        <p><%= f.submit 'Donate!', class: 'btn btn-block btn-success' %></p>
-        <div id="error_explanation"></div>
-        <p class="note">You’re making a tax-deductible donation to DocumentCloud.org</p>
-      <% end %>
+  <iframe src="https://www.muckrock.com/crowdfund/185/embed/" width="100%" height="530px" style="border: none; box-shadow: 2px 2px 2px #00000024; margin: -1em 0 4em 0;"></iframe>
 
   <% if params[:prolix] %>
       </div>

app/views/home/_sidebar.html.erb

@@ -93,12 +93,7 @@
         Show your support for great journalism with a donation to DocumentCloud.<br>
         <%= link_to "Read more about donating", donate_path %>
       </p>
-      <div class="text_input first light">
-        <div class="inner">
-          <input type="tel" placeholder="Amount ($)" name="amount" id="donation-amount-other" />
-        </div>
-      </div>
-      <p><button type="submit" class="minibutton default">Donate</button></p>
+      <a href="/donate"><span class="minibutton default">Donate</span></a>
       <div id="error_explanation" style="color:red"></div>
     <% end %>
   </div>

app/views/home/privacy.html.erb

@@ -3,12 +3,13 @@
 <div class="content">
   <div class="container-md">
     <h1>Privacy Policy</h1>
-    <p>This privacy policy extends DocumentCloud's terms and conditions, which may be found at <a href="https://www.documentcloud.org/terms">https://www.documentcloud.org/terms</a>.</p>
+    <p>This privacy policy extends DocumentCloud's terms and conditions, which may be found at <a href="https://www.documentcloud.org/terms">https://www.documentcloud.org/terms</a>. Your use of DocumentCloud, and any dispute over privacy, is subject to this Policy and our Terms of Service.</p>
     <ol>
     	<li><b>Information DocumentCloud collects about all users, including Document Contributors and non-contributing users:</b></li>
     	<ol>
     		<li>When you read or download documents from DocumentCloud, we collect limited, anonymous data. For instance, our web server records the Internet Protocol address of the computer you are using, information about your browser and operating system, date and time of access and the internet address and page which referred you to our site. The information logged by DocumentCloud is automatically deleted at regular intervals. In addition, we analyze traffic to DocumentCloud using Google Analytics. Google Analytics is subject to the privacy policy of Google, Inc., which you can find at <A HREF="http://www.google.com/privacy">http://www.google.com/privacy</A>.  We use this information to generate statistics and measure site activity to improve the usefulness of the site to our visitors.</li>
     		<li>Documents available through DocumentCloud are stored on s3 servers provided by Amazon Web Services. Amazon Web Services is a subsidiary of Amazon.com (Their privacy policies may be found at <A HREF="http://aws.amazon.com/privacy/">http://aws.amazon.com/privacy/</A> and <A HREF="http://www.amazon.com/privacy">http://www.amazon.com/privacy</A> respectively).  </li>
+				<li>Documents available through DocumentCloud may contain links to, or appear on third-party websites. Any access to and use of such third party websites is not governed by this Policy, but instead is governed by the privacy policies of those third party websites, and DocumentCloud is not responsible for the information practices of such third party websites.</li>
     	</ol>
     	<li><b>Information DocumentCloud collects about Document Contributors:</b></li>
     	<ol>
@@ -32,9 +33,16 @@
     		<li>Communications between you and DocumentCloud may pass through many machines, operating systems, programs, browsers, Web servers, networks, routers, Ethernet switches, Internet service providers, proxy servers, intranets, the public phone system, or other devices on your premises, at DocumentCloud, and in between. Some of these devices create logs of activities that are recorded on computer systems.</li>
     		<li>It is possible that the servers that host DocumentCloud could become compromised by others and that the information on DocumentCloud's computers could be collected and disseminated without the knowledge or consent of DocumentCloud. </li>
     	</ol>
+			<li><b>Additional Terms for EU Users:</b></li>
+			<ol>
+			<li>DocumentCloud processes personal data as both a “processor” and as a “controller” under the European Union’s General Data Protection Regulation. We process personal data as a processor when providing Services to our users, who may then make it available for access and use. We act as a controller when we process the personal data you provide to us through DocumentCloud’s site. This Section applies to our processing of personal data collected via the Site as a controller. To contact DocumentCloud about your personal data, see the section entitled “Contact Us” below.</li>
+			<li>DocumentCloud processes personal data to pursue its legitimate business interests, which are described above.</li>
+			<li>Please be aware that we may transfer your personal data to the United States where our servers are located. We will only transfer your personal information to the United States through the use of appropriate safeguards. DocumentCloud will retain your personal information for as long as necessary to fulfill the purposes for which it was collected or as required by applicable laws or regulations. For users with a registered account on the Site, DocumentCloud will retain the information provided by you for as long as you have an account on the Site.</li>
+		 	<li>You have the right to access your personal information, rectify inaccurate personal information kept about you, request the erasure of personal information that is no longer necessary for the purposes for which we have collected or processed the information, and to request that we restrict the processing of your personal information in certain circumstances, for example if you are contesting the accuracy of the data. If you wish to access your personal information, please contact DocumentCloud through the contact listed below in writing and allow reasonable time for the data to be collected. You also have the right to receive personal information you have provided to us in a commonly used and machine-readable format and to transmit that personal information to any controller without hindrance from us. Finally, you have the right to lodge a complaint with a supervisory authority.</li>
+			</ol>
     	<li><b>Limitation of Liability</b></li>
     	<ol>
-    		<li><b>IMPORTANT:</b> DocumentCloud is not responsible for breaches of privacy that may occur as a result of any user's failure to exercise best practices at public workstations, on public networks, or with respect to secure passwords. DocumentCloud is not responsible for breaches of privacy that may occur as a result of third party access over which we have no control. </li>
+    		<li><b>IMPORTANT:</b> DocumentCloud is not responsible for breaches of privacy that may occur as a result of any user's failure to exercise best practices at public workstations, on public networks, or with respect to secure passwords. DocumentCloud is not responsible for breaches of privacy that may occur as a result of third party access over which we have no control. DocumentCloud is not responsible for breaches of privacy that may occur as a result of your use the Services, including, but not limited to, your disclosure of the private or confidential information of third parties.</li>
     	</ol>
     	<li><b>When DocumentCloud discloses information: </b></li>
     	<ol>
@@ -43,7 +51,7 @@
     	</ol>
     	<li><b>Children's privacy:</b></li>
     	<ol>
-    		<li>DocumentCloud content is intended for adults. We do not knowingly collect personal information from children under 13 years old. If you are a parent or legal guardian of a child under age 13 who you believe has submitted personal information to DocumentCloud, please contact us immediately.</li>
+    		<li>DocumentCloud content is intended for adults. We do not knowingly collect personal information from children under 13 years old. If you are a parent or legal guardian of a child under age 13 who you believe has submitted personal information to DocumentCloud, please contact us immediately. If we discover that we have been provided with the personal information of a child under 13, we will delete such information from our systems.</li>
     	</ol>
     	<li><b>Changes and updates to this notice:</b></li>
     	<ol>
@@ -57,11 +65,11 @@
     	   <ol>
            <li>See <a href="/contact">contact us</a> for the complete array of ways to reach us for technical support or new account questions. Our mailing address is:
              <p style="margin-left:1rem;margin-top:1rem">
-               DocumentCloud<br>
-               Investigative Reporters and Editors<br>
-               141 Neff Annex<br>
-               University of Missouri<br>
-               Columbia, MO 65211
+      				DocumentCloud<br>
+      				PO Box 29401<br>
+      				1602 Frankford Ave<br>
+      				Philadelphia, PA  19125-9998<br>
+      				[email protected]
              </p>
            </li>
         </ol>