Merge pull request #84 from phillmv/fix_static_mode

Fix static mode tests

.github/workflows/continuous_integration.yaml

@@ -26,6 +26,10 @@ jobs:
           tags: arquivo-test:latest
       - name: run the test
         run: docker run arquivo-test:latest
+      - name: run the test
+        run: STATIC_PLS=1 docker run -e STATIC_PLS arquivo-test:latest
+
+
 
 
   # run_rails_test:

.gitignore

@@ -38,6 +38,8 @@ yarn-debug.log*
 /.nix-gems
 /.nix-node
 # /vendor/cache
+# we vendor the gems, but not the installed paths
+/vendor/ruby
 *.DS_Store
 
 # use this to customize your .env entries

Dockerfile

@@ -23,7 +23,7 @@ RUN gem update --system --no-document && \
 RUN --mount=type=cache,id=dev-apt-cache,sharing=locked,target=/var/cache/apt \
     --mount=type=cache,id=dev-apt-lib,sharing=locked,target=/var/lib/apt \
     apt-get update -qq && \
-    apt-get install --no-install-recommends -y curl git openssh-client libsqlite3-0
+    apt-get install --no-install-recommends -y curl git openssh-client libsqlite3-0 wget
 
 
 RUN useradd rails --create-home --shell /bin/bash

app/controllers/application_controller.rb

@@ -20,7 +20,7 @@ def check_imports
   def resync_with_remotes
     last_checked_at = session[:synced_remotes_at]
     if last_checked_at.nil? || last_checked_at < 15.minutes.ago
-      PullAllFromGit.perform_later
+      PullAllFromGit.perform_later unless Arquivo.static?
     end
     session[:synced_remotes_at] = Time.current
   end

app/controllers/entries_controller.rb

@@ -12,6 +12,7 @@ def index
   # GET /entries/1.json
   def show
     if @entry.document?
+      # TODO: why import these at all why not just open the frigging thing?
       blob = @entry.files.blobs.first
       expires_in ActiveStorage.service_urls_expire_in
       redirect_to rails_blob_path(blob, disposition: params[:disposition])

app/helpers/url_helper.rb

@@ -16,7 +16,16 @@ def self.entry_params(entry)
     name_with_path = "#{name}_path".to_sym
     define_method name_with_path do |entry, opts = {}|
       if Arquivo.static?
-        RailsUrlHelpers.send(name_with_path, entry, opts.except(:notebook).merge(format: "html"))
+        # A year and a half later, I don't remember _why_ I started forcing
+        # .html urls in links (isn't this breaking document downloads?)
+        # my impression is I made this distinction in order to force
+        # downloaded urls to be stored with .html extension so they can
+        # be served statically – but i'm not entirely sure this is The Correct Way
+        these_opts = opts.except(:notebook)
+        if entry.append_html_extension?
+          these_opts = these_opts.merge(format: "html")
+        end
+        RailsUrlHelpers.send(name_with_path, entry, these_opts)
       else
         RailsUrlHelpers.send(name_with_path, entry, opts.merge(UrlHelper.entry_params(entry)))
       end

app/models/ad_hoc_markdown_importer.rb

@@ -29,7 +29,8 @@ def find_or_create_notebook
     notebook = Notebook.find_by(name: notebook_name)
     if notebook.nil?
       notebook = Notebook.create(name: notebook_name,
-                                 import_path: notebook_path)
+                                 import_path: notebook_path,
+                                 skip_local_sync: true)
     else
       notebook.update(import_path: notebook_path)
     end
@@ -70,16 +71,10 @@ def process_import_path(notebook)
 
         filename = File.basename(identifier)
         if !entry.files.blobs.find_by(filename: filename)
-          # TODO 2023-11-23: mimic the same blob lifecycle handling from create_blob_and_file to avoid the async stuff noted below.
           blob = ActiveStorage::Blob.create_and_upload!(io: File.open(file_path),
-                                                        filename: filename)
-
-          # run analysis step synchronously, so we skip the async job.
-          # for reasons i don't comprehend, in dev mode at least
-          # ActiveStorage::AnalyzeJob just hangs there indefinitely, doing
-          # naught to improve our lot, and this is very frustrating and further
-          # i have close to zero desire to debug ActiveJob async shenanigans
-          blob.analyze
+                                                        filename: filename,
+                                                        metadata: { "analyzed" => true })
+
           entry.files.create(blob_id: blob.id, created_at: blob.created_at)
         end
       end

app/models/entry.rb

@@ -213,6 +213,10 @@ def clear_cached_blob_filenames
   end
   # --
 
+  def append_html_extension?
+    note? || calendar? || bookmark?
+  end
+
   def note?
     kind.nil?
   end

app/models/entry_linker.rb

@@ -46,16 +46,19 @@ def blacklisted_paths
     end
 
     if Arquivo.static?
-      return []
-    end
+      contact_path = Rails.application.routes.url_helpers.contact_path(query: "")
+      tag_path = Rails.application.routes.url_helpers.tag_path(query: "")
+      @blacklisted_paths = [contact_path, tag_path]
+    else
 
-    # don't want to consider autogen tag and contact urls to be a "link"
-    # since we keep track of those references separately ¯\_(ツ)_/¯
-    # TODO: rewrite this to handle Arquivo.static urls
-    tag_search_path = Rails.application.routes.url_helpers.search_path(owner: notebook.owner, notebook: notebook, query: "#")
-    contact_search_path = Rails.application.routes.url_helpers.search_path(owner: notebook.owner, notebook: notebook, query: "@")
+      # don't want to consider autogen tag and contact urls to be a "link"
+      # since we keep track of those references separately ¯\_(ツ)_/¯
+      # TODO: rewrite this to be handled in their respective pipelines (a data attr? a context variable?) so we're not doing this funny business
+      tag_search_path = Rails.application.routes.url_helpers.search_path(owner: notebook.owner, notebook: notebook, query: "#")
+      contact_search_path = Rails.application.routes.url_helpers.search_path(owner: notebook.owner, notebook: notebook, query: "@")
 
-    @blacklisted_paths = [tag_search_path, contact_search_path]
+      @blacklisted_paths = [tag_search_path, contact_search_path]
+    end
   end
 
   def link!

app/models/sync_from_disk.rb

@@ -164,7 +164,7 @@ def create_blob_and_file(entry, blob_attr, entry_files_path)
       end
       blob_attr["metadata"]["analyzed"] = true
 
-      blob = ActiveStorage::Blob.create(blob_attr)
+      blob = ActiveStorage::Blob.create!(blob_attr)
       blob.upload_without_unfurling(File.open(new_attachment_filepath))
 
       entry.files.create(blob_id: blob.id, created_at: blob.created_at)

bin/mawl-generate

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+PROJECT_FOLDER=`realpath $(dirname "$0")/..` && cd "$PROJECT_FOLDER"
+
+PID=$(cat tmp/pids/server.pid 2>/dev/null) && ps -p $PID > /dev/null && kill $PID
+./bin/rails s -e production -d && ./bin/rails static:generate

bin/mawl-import

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+PROJECT_FOLDER=`realpath $(dirname "$0")/..` && cd "$PROJECT_FOLDER"
+./bin/rails static:import

bin/mawl-import-and-generate

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+PROJECT_FOLDER=`realpath $(dirname "$0")/..` && cd "$PROJECT_FOLDER"
+./bin/mawl-import && ./bin/mawl-generate

bin/start-arquivo

@@ -0,0 +1,75 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+PROJECT_FOLDER=`realpath $(dirname "$0")/..`
+
+if [ -z ${DATA_FOLDER+x} ]; then
+  DATA_FOLDER="$PROJECT_FOLDER"/data
+fi
+
+mkdir -p "$DATA_FOLDER"
+
+if [ -z ${ARQUIVO_PORT+x} ]; then
+  ARQUIVO_PORT=12346
+fi
+
+if [ "$#" -eq 0 ]; then
+  echo "Usage: $0 [console|server]"
+  exit 1
+else
+  if [ "$1" = "console" ]; then
+    shift
+    if [ "$#" -eq 0 ]; then
+      DOCKER_COMMAND="bash"
+    else
+      DOCKER_COMMAND="$@"
+    fi
+  elif [ "$1" = "server" ]; then
+    shift
+    # do the default
+    DOCKER_COMMAND=
+    export RAILS_ENV=development
+  elif [ "$1" = "mawl" ]; then
+    # TODO: rationalize into its own command, plus a server option
+    shift
+
+    if [ -z "${1+x}" ]; then
+      echo "Usage: $0 mawl [input-folder] [output-folder]"
+      exit 1
+    else
+      MAWL_INPUT_PATH=`realpath "$1"`
+    fi
+
+    if [ -z "${2+x}" ]; then
+      MAWL_OUTPUT_PATH="$PROJECT_FOLDER"/tmp/mawl-output
+    else
+      MAWL_OUTPUT_PATH="$2"
+    fi
+
+    export STATIC_PLS=1
+    DOCKER_COMMAND=/arquivo/bin/mawl-import-and-generate
+    # DOCKER_COMMAND="./bin/rails server -d && ./bin/rails static:import static:generate"
+  fi
+fi
+
+export ARQUIVO_USER=phillmv
+export [email protected]
+export ARQUIVO_GIT_NAME="Phill MV"
+export RAILS_MASTER_KEY=$(cat "$PROJECT_FOLDER"/config/master.key)
+export RAILS_BIND=tcp://0.0.0.0:3001
+
+echo "Running arquivo while mounting local filesystem..."
+
+docker run -it -p "$ARQUIVO_PORT":3001 \
+  -e ARQUIVO_USER \
+  -e ARQUIVO_GIT_EMAIL \
+  -e ARQUIVO_GIT_NAME \
+  -e RAILS_ENV \
+  -e RAILS_BIND \
+  -e STATIC_PLS \
+  -v "$MAWL_INPUT_PATH":/mawl-input \
+  -v "$DATA_FOLDER":/data \
+  -v "$MAWL_OUTPUT_PATH":/output \
+  -v "$PROJECT_FOLDER":/arquivo \
+  arquivo-development:latest ${DOCKER_COMMAND}

config/environments/development.rb

@@ -2,6 +2,11 @@
 require 'socket'
 
 Rails.application.configure do
+  if Arquivo.static?
+    # TODO WRITE A TEST TO CHECK IF STUFF IN /ASSETS MAKE SIT THRU
+    # TO STATIC GENERATE
+    config.assets.compile = false
+  end
   config.hosts << "arquivo.io"
 
   # until we figure out how to multitenant this,

config/environments/production.rb

@@ -3,6 +3,9 @@
 Rails.application.configure do
   config.hosts << "arquivo.io"
   config.hosts << "arquivo.localhost"
+  if Arquivo.static?
+    config.hosts << "localhost"
+  end
 
   # until we figure out how to multitenant this,
   config.active_storage.routes_prefix = ENV["ARQUIVO_USER"] || "/phillmv/_"

config/routes.rb

@@ -53,7 +53,7 @@
     # only here to keep the UrlHelper ticking
     get "/settings", to: "settings#index", as: :settings
     get "/*id/edit", to: "entries#edit", as: :edit_entry
-  else
+  else # END STATIC MODE
 
   resources :notebooks
   scope ':owner', defaults: { owner: "owner" } do

lib/tasks/static.rake

@@ -1,18 +1,25 @@
 namespace :static do
   desc 'Generate static site in ./out/ directory'
   task :import => :environment do
-    SyncFromDisk.new(ENV["NOTEBOOK_PATH"]).import!
+    if !Arquivo.static?
+      puts "Buddy, we're not in static mode! Quitting..."
+      exit 1
+    end
+
+    sync_path = ENV["MAWL_INPUT_PATH"] || "/mawl-input"
+    SyncFromDisk.new(sync_path).import!
   end
 
   task :generate do
-    if ENV["MAWB_OUTPUT"]
-      folder = ENV["MAWB_OUTPUT"]
-    else
-      folder = "out"
+    if !Arquivo.static?
+      puts "Buddy, we're not in static mode! Quitting..."
+      exit 1
     end
 
-    Dir.mkdir folder unless File.exist? folder
-    Dir.chdir folder do
+    output_path = ENV["MAWL_OUTPUT_PATH"] || "/output"
+
+    Dir.mkdir output_path unless File.exist? output_path
+    Dir.chdir output_path do
       puts `wget --version`
       puts "######\nwget --domains localhost  --recursive  --page-requisites  --html-extension  --convert-links -nH localhost:3000 localhost:3000/hidden_entries"
       `wget --domains localhost  --recursive  --page-requisites  --html-extension  --convert-links -nH localhost:3000 localhost:3000/hidden_entries`

test/integration/entries_integration_test.rb

@@ -6,6 +6,10 @@ class EntriesIntegrationTest < ActionDispatch::IntegrationTest
   end
 
   test "save a bookmark" do
+    if Arquivo.static?
+      return
+    end
+
     get save_bookmark_path(notebook: @current_notebook), params: { url: "http://example.com", subject: "foo" }
     assert_response :success
 
@@ -52,6 +56,9 @@ class EntriesIntegrationTest < ActionDispatch::IntegrationTest
   end
 
   test "load a gdocs bookmark" do
+    if Arquivo.static?
+      return
+    end
     get save_bookmark_path(notebook: @current_notebook), params: { url: "https://docs.google.com/document/d/1E-V2Qj2OhURTtHVo9ONDjteHS7fRr77lEBbttA6DbIo/edit#heading=h.wwc00h4un5en", subject: "foo" }
     assert_response :success
 
@@ -60,6 +67,10 @@ class EntriesIntegrationTest < ActionDispatch::IntegrationTest
   end
 
   test "create a threaded entry" do
+    if Arquivo.static?
+      return
+    end
+
     entry1 = @current_notebook.entries.create(body: "test 1")
     post create_entry_path(owner: @current_notebook.owner, notebook: @current_notebook), params: { entry: { body: "test mc test", in_reply_to: entry1.identifier } }
 
@@ -76,6 +87,10 @@ class EntriesIntegrationTest < ActionDispatch::IntegrationTest
   end
 
   test "replying to an existing entry will increment the date" do
+    if Arquivo.static?
+      return
+    end
+
     entry = @current_notebook.entries.create(body: "# #daily 2023-03-16", occurred_at: "2023-03-16".to_date)
 
     get new_entry_path(@current_notebook), params: { in_reply_to: entry.identifier }
@@ -87,6 +102,10 @@ class EntriesIntegrationTest < ActionDispatch::IntegrationTest
   end
 
   test "when saved thru a controller, an entry's identifier will be set from its subject" do
+    if Arquivo.static?
+      return
+    end
+
     assert_equal 0, @current_notebook.entries.count
     post create_entry_path(owner: @current_notebook.owner, notebook: @current_notebook), params: { entry: { body: "test mc test"} }