From ca59a6e93468595ff0e6baa3e460e6f5a70804be Mon Sep 17 00:00:00 2001 From: Phill MV Date: Thu, 21 Dec 2023 11:06:32 -0500 Subject: [PATCH 01/18] fixed broken entry linker in static mode. --- app/models/entry_linker.rb | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/app/models/entry_linker.rb b/app/models/entry_linker.rb index 5c4c9b99a..971dd3f1a 100644 --- a/app/models/entry_linker.rb +++ b/app/models/entry_linker.rb @@ -46,16 +46,19 @@ def blacklisted_paths end if Arquivo.static? - return [] - end + contact_path = Rails.application.routes.url_helpers.contact_path(query: "") + tag_path = Rails.application.routes.url_helpers.tag_path(query: "") + @blacklisted_paths = [contact_path, tag_path] + else - # don't want to consider autogen tag and contact urls to be a "link" - # since we keep track of those references separately ¯\_(ツ)_/¯ - # TODO: rewrite this to handle Arquivo.static urls - tag_search_path = Rails.application.routes.url_helpers.search_path(owner: notebook.owner, notebook: notebook, query: "#") - contact_search_path = Rails.application.routes.url_helpers.search_path(owner: notebook.owner, notebook: notebook, query: "@") + # don't want to consider autogen tag and contact urls to be a "link" + # since we keep track of those references separately ¯\_(ツ)_/¯ + # TODO: rewrite this to be handled in their respective pipelines (a data attr? a context variable?) so we're not doing this funny business + tag_search_path = Rails.application.routes.url_helpers.search_path(owner: notebook.owner, notebook: notebook, query: "#") + contact_search_path = Rails.application.routes.url_helpers.search_path(owner: notebook.owner, notebook: notebook, query: "@") - @blacklisted_paths = [tag_search_path, contact_search_path] + @blacklisted_paths = [tag_search_path, contact_search_path] + end end def link! From 8c3d88c003db2e3f4da0f3b44d9b99180ebe6bc1 Mon Sep 17 00:00:00 2001 From: Phill MV Date: Thu, 21 Dec 2023 11:08:26 -0500 Subject: [PATCH 02/18] Skipped tests that are irrelevant in static mode. --- test/integration/entries_with_slashes_test.rb | 4 ++++ test/integration/file_upload_integration_test.rb | 3 +++ test/integration/timeline_test.rb | 5 +++++ 3 files changed, 12 insertions(+) diff --git a/test/integration/entries_with_slashes_test.rb b/test/integration/entries_with_slashes_test.rb index 427fa4ced..28ce02d36 100644 --- a/test/integration/entries_with_slashes_test.rb +++ b/test/integration/entries_with_slashes_test.rb @@ -6,6 +6,10 @@ class EntriesWithSlashesTest < ActionDispatch::IntegrationTest end test "entries with slashes in their identifier can be saved to disk, and read from disk" do + if Arquivo.static? + return + end + enable_local_sync do arquivo_path = Setting.get(:arquivo, :arquivo_path) entry = @current_notebook.entries.create(identifier: "hello/world", body: "this is my test entry") diff --git a/test/integration/file_upload_integration_test.rb b/test/integration/file_upload_integration_test.rb index 6aab0fd9f..dfbdf2ec3 100644 --- a/test/integration/file_upload_integration_test.rb +++ b/test/integration/file_upload_integration_test.rb @@ -7,6 +7,9 @@ class FileUploadIntegrationTest < ActionDispatch::IntegrationTest end test "files direct uploaded to the same entry are renamed if filenames collide" do + if Arquivo.static? + return + end assert_equal 0, @current_notebook.entries.count @entry = @current_notebook.entries.new diff --git a/test/integration/timeline_test.rb b/test/integration/timeline_test.rb index a0b41f8f0..1e3a1d0b4 100644 --- a/test/integration/timeline_test.rb +++ b/test/integration/timeline_test.rb @@ -23,6 +23,11 @@ def User.tz end test "timeline smoke test" do + # no search in static mode + if Arquivo.static? + return + end + get search_path(@current_notebook, query: "is:everything") assert_response :success From 8e87c2bc6767f555a4f0ba05055e8b74bbf50947 Mon Sep 17 00:00:00 2001 From: Phill MV Date: Thu, 21 Dec 2023 11:28:15 -0500 Subject: [PATCH 03/18] Tentative fix for static site import export test. --- app/helpers/url_helper.rb | 11 ++++++++++- app/models/entry.rb | 4 ++++ test/integration/static_site_import_export_test.rb | 2 +- 3 files changed, 15 insertions(+), 2 deletions(-) diff --git a/app/helpers/url_helper.rb b/app/helpers/url_helper.rb index fceeb0f44..4c9531d06 100644 --- a/app/helpers/url_helper.rb +++ b/app/helpers/url_helper.rb @@ -16,7 +16,16 @@ def self.entry_params(entry) name_with_path = "#{name}_path".to_sym define_method name_with_path do |entry, opts = {}| if Arquivo.static? - RailsUrlHelpers.send(name_with_path, entry, opts.except(:notebook).merge(format: "html")) + # A year and a half later, I don't remember _why_ I started forcing + # .html urls in links (isn't this breaking document downloads?) + # my impression is I made this distinction in order to force + # downloaded urls to be stored with .html extension so they can + # be served statically – but i'm not entirely sure this is The Correct Way + these_opts = opts.except(:notebook) + if entry.append_html_extension? + these_opts = these_opts.merge(format: "html") + end + RailsUrlHelpers.send(name_with_path, entry, these_opts) else RailsUrlHelpers.send(name_with_path, entry, opts.merge(UrlHelper.entry_params(entry))) end diff --git a/app/models/entry.rb b/app/models/entry.rb index 5330fc53e..64cf01796 100644 --- a/app/models/entry.rb +++ b/app/models/entry.rb @@ -213,6 +213,10 @@ def clear_cached_blob_filenames end # -- + def append_html_extension? + note? || calendar? || bookmark? + end + def note? kind.nil? end diff --git a/test/integration/static_site_import_export_test.rb b/test/integration/static_site_import_export_test.rb index a169f22dd..577234061 100644 --- a/test/integration/static_site_import_export_test.rb +++ b/test/integration/static_site_import_export_test.rb @@ -96,7 +96,7 @@ class StaticSiteImportExportTest < ActionDispatch::IntegrationTest # i expect exactly two links: assert_equal 3, css_select("a").count assert_select "a[href='/youvechanged.jpg']" - assert_select "a[href='/about']" + assert_select "a[href='/about.html']" assert_select "a[href='/archive']" # try getting individual pages: From 12275aa2d1d954069ebc933000aaebe9579d94a6 Mon Sep 17 00:00:00 2001 From: Phill MV Date: Thu, 21 Dec 2023 13:11:20 -0500 Subject: [PATCH 04/18] skipping tests that test disk syncing behaviour in static mode since we're not meaning to edit posts in static mode. --- test/models/local_syncer_test.rb | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/test/models/local_syncer_test.rb b/test/models/local_syncer_test.rb index 330bc4821..53a21c6c8 100644 --- a/test/models/local_syncer_test.rb +++ b/test/models/local_syncer_test.rb @@ -23,6 +23,10 @@ class LocalSyncerTest < ActiveSupport::TestCase end test "with enable_local_sync we do write to a git repo and get history" do + if Arquivo.static? + return + end + enable_local_sync do arquivo_path = Setting.get(:arquivo, :arquivo_path) refute File.exist?(arquivo_path) @@ -48,6 +52,12 @@ class LocalSyncerTest < ActiveSupport::TestCase end test "when an entry is updated, we write to a local repo" do + # this test _used_ to work in static mode but I believe that I turned off + # syncing in static mode. When did this stop working? + if Arquivo.static? + return + end + notebook = Notebook.create(name: "test-notebook") enable_local_sync do # in the beginning, there is no folder From b96f4109720c3396b8c130762382d064c971c3b7 Mon Sep 17 00:00:00 2001 From: Phill MV Date: Thu, 21 Dec 2023 13:11:58 -0500 Subject: [PATCH 05/18] tweaked entries_with_slashes test to accomodate the difference in the default template in static mode. --- test/integration/entries_with_slashes_test.rb | 37 +++++++++++-------- 1 file changed, 22 insertions(+), 15 deletions(-) diff --git a/test/integration/entries_with_slashes_test.rb b/test/integration/entries_with_slashes_test.rb index 28ce02d36..bc4efe048 100644 --- a/test/integration/entries_with_slashes_test.rb +++ b/test/integration/entries_with_slashes_test.rb @@ -60,7 +60,7 @@ class EntriesWithSlashesTest < ActionDispatch::IntegrationTest get timeline_path(@current_notebook) assert_response :success - expected_hrefs = entries.map do |e| "#{timeline_path(@current_notebook)}/#{e.identifier}" end.to_set + expected_hrefs = entries.map { |e| entry_path(e) }.to_set permalink_hrefs = css_select("a.permalink[href]").map do |a| a["href"] end.to_set assert_equal expected_hrefs, permalink_hrefs @@ -69,22 +69,29 @@ class EntriesWithSlashesTest < ActionDispatch::IntegrationTest get entry_path(entries.last) assert_response :success - # the page displays and lists the identifier name in the timeline top thingy - assert_select("nav li[aria-current=page] h3", text: entries.last.identifier) - # and we're looking at the same right page - assert_select(".entry-body", text: entries.last.body) + if Arquivo.static? + assert_select("entry-subject", html: /#{entries.last.identifier}/) + assert_select("entry-body", text: entries.last.body) + else + # the page displays and lists the identifier name in the timeline top thingy + assert_select("nav li[aria-current=page] h3", text: entries.last.identifier) + # and we're looking at the same right page + assert_select(".entry-body", text: entries.last.body) + end # we can edit it: - get edit_entry_path(entries.last) - assert_response :success - assert_select("textarea#entry_body", text: entries.last.body) - - # we can see its thread - get threaded_entry_path(entries.last) - assert_response :success - - threaded_permalink_hrefs = css_select("a.permalink[href]").map do |a| a["href"] end.to_set - assert_equal expected_hrefs, threaded_permalink_hrefs + if !Arquivo.static? + get edit_entry_path(entries.last) + assert_response :success + assert_select("textarea#entry_body", text: entries.last.body) + + # we can see its thread + get threaded_entry_path(entries.last) + assert_response :success + + threaded_permalink_hrefs = css_select("a.permalink[href]").map do |a| a["href"] end.to_set + assert_equal expected_hrefs, threaded_permalink_hrefs + end # ran out of time, but feel confident about where we're at, so TODO: # - access /files/ path From 63f4eb108d1dcd22e3525631185e6ddfc6975aa9 Mon Sep 17 00:00:00 2001 From: Phill MV Date: Thu, 21 Dec 2023 13:14:25 -0500 Subject: [PATCH 06/18] commented out another local syncer test in static mode. --- test/models/local_syncer_test.rb | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/test/models/local_syncer_test.rb b/test/models/local_syncer_test.rb index 53a21c6c8..0a7e5a586 100644 --- a/test/models/local_syncer_test.rb +++ b/test/models/local_syncer_test.rb @@ -91,6 +91,10 @@ class LocalSyncerTest < ActiveSupport::TestCase end test "when an entry is destroyed, we delete it from the local repo" do + if Arquivo.static? + return + end + notebook = Notebook.create(name: "test-notebook") enable_local_sync do # in the beginning, there is no folder From ae17633891e82a0933a362f517bee42e5cc2e497 Mon Sep 17 00:00:00 2001 From: Phill MV Date: Thu, 21 Dec 2023 13:19:09 -0500 Subject: [PATCH 07/18] Tweaked entry renderer test to account for diff in routing in static mode --- test/models/entry_renderer_test.rb | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/test/models/entry_renderer_test.rb b/test/models/entry_renderer_test.rb index 9cbf9a4de..600ac66af 100644 --- a/test/models/entry_renderer_test.rb +++ b/test/models/entry_renderer_test.rb @@ -69,7 +69,11 @@ def render_html(renderer, str) # of note: bar was not converted, and the generated urls # point to /test/timeline/search - assert_equal "

#foo bar #baz

", hashtag_markdown_html + if Arquivo.static? + assert_equal "

#foo bar #baz

", hashtag_markdown_html + else + assert_equal "

#foo bar #baz

", hashtag_markdown_html + end end test "parses contacts and links them" do @@ -81,7 +85,12 @@ def render_html(renderer, str) # of note: bar is not converted, # it points to /test/timeline/search # .user_mention class - assert_equal "

@foo bar @baz

", contact_markdown_html + + if Arquivo.static? + assert_equal "

@foo bar @baz

", contact_markdown_html + else + assert_equal "

@foo bar @baz

", contact_markdown_html + end end test "adds ids to headers" do From 5687aef88f924e62bc7d6a41c8b17bd267337ba1 Mon Sep 17 00:00:00 2001 From: Phill MV Date: Thu, 21 Dec 2023 13:21:53 -0500 Subject: [PATCH 08/18] commented out more tests that are irrelevant for static mode. --- test/integration/entries_integration_test.rb | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/test/integration/entries_integration_test.rb b/test/integration/entries_integration_test.rb index 12db4dd34..32fa6acc0 100644 --- a/test/integration/entries_integration_test.rb +++ b/test/integration/entries_integration_test.rb @@ -6,6 +6,10 @@ class EntriesIntegrationTest < ActionDispatch::IntegrationTest end test "save a bookmark" do + if Arquivo.static? + return + end + get save_bookmark_path(notebook: @current_notebook), params: { url: "http://example.com", subject: "foo" } assert_response :success @@ -52,6 +56,9 @@ class EntriesIntegrationTest < ActionDispatch::IntegrationTest end test "load a gdocs bookmark" do + if Arquivo.static? + return + end get save_bookmark_path(notebook: @current_notebook), params: { url: "https://docs.google.com/document/d/1E-V2Qj2OhURTtHVo9ONDjteHS7fRr77lEBbttA6DbIo/edit#heading=h.wwc00h4un5en", subject: "foo" } assert_response :success @@ -60,6 +67,10 @@ class EntriesIntegrationTest < ActionDispatch::IntegrationTest end test "create a threaded entry" do + if Arquivo.static? + return + end + entry1 = @current_notebook.entries.create(body: "test 1") post create_entry_path(owner: @current_notebook.owner, notebook: @current_notebook), params: { entry: { body: "test mc test", in_reply_to: entry1.identifier } } @@ -76,6 +87,10 @@ class EntriesIntegrationTest < ActionDispatch::IntegrationTest end test "replying to an existing entry will increment the date" do + if Arquivo.static? + return + end + entry = @current_notebook.entries.create(body: "# #daily 2023-03-16", occurred_at: "2023-03-16".to_date) get new_entry_path(@current_notebook), params: { in_reply_to: entry.identifier } @@ -87,6 +102,10 @@ class EntriesIntegrationTest < ActionDispatch::IntegrationTest end test "when saved thru a controller, an entry's identifier will be set from its subject" do + if Arquivo.static? + return + end + assert_equal 0, @current_notebook.entries.count post create_entry_path(owner: @current_notebook.owner, notebook: @current_notebook), params: { entry: { body: "test mc test"} } From bf4153cb182990d89ca1af03fe1c3853f95a2c5c Mon Sep 17 00:00:00 2001 From: Phill MV Date: Thu, 21 Dec 2023 13:44:57 -0500 Subject: [PATCH 09/18] Added little script for starting arquivo in dev mode. --- bin/start-arquivo | 50 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100755 bin/start-arquivo diff --git a/bin/start-arquivo b/bin/start-arquivo new file mode 100755 index 000000000..5e31b2879 --- /dev/null +++ b/bin/start-arquivo @@ -0,0 +1,50 @@ +#!/usr/bin/env bash + +set -euo pipefail + +PROJECT_FOLDER=`realpath $(dirname "$0")/..` +DEV_DATA_FOLDER="$PROJECT_FOLDER"/tmp/data + +mkdir -p "$DEV_DATA_FOLDER" + +if [ -z ${ARQUIVO_PORT+x} ]; then + ARQUIVO_PORT=12346 +fi + +if [ "$#" -eq 0 ]; then + echo "Usage: $0 [console|server]" + exit 1 +else + if [ "$1" = "console" ]; then + shift + if [ "$#" -eq 0 ]; then + DOCKER_COMMAND="bash" + else + DOCKER_COMMAND="$@" + fi + elif [ "$1" = "server" ]; then + shift + # do the default + DOCKER_COMMAND= + export RAILS_ENV=development + fi +fi + +export ARQUIVO_USER=phillmv +export ARQUIVO_GIT_EMAIL=phillmv@okayfail.com +export ARQUIVO_GIT_NAME="Phill MV" +export RAILS_MASTER_KEY=$(cat "$PROJECT_FOLDER"/config/master.key) +export RAILS_BIND=tcp://0.0.0.0:3001 + +echo "Running arquivo while mounting local filesystem..." + +docker run -it -p "$ARQUIVO_PORT":3001 \ + -e ARQUIVO_USER \ + -e ARQUIVO_GIT_EMAIL \ + -e ARQUIVO_GIT_NAME \ + -e RAILS_ENV \ + -e RAILS_BIND \ + -e STATIC_PLS \ + -v "$DEV_DATA_FOLDER":/data \ + -v "$PROJECT_FOLDER":/arquivo \ + arquivo-development:latest ${DOCKER_COMMAND} From 0289d1f9e6f457233e2c26dcfd54591934bd3d53 Mon Sep 17 00:00:00 2001 From: Phill MV Date: Thu, 21 Dec 2023 13:45:32 -0500 Subject: [PATCH 10/18] want to try to run the tests in static mode going fwd --- .github/workflows/continuous_integration.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/continuous_integration.yaml b/.github/workflows/continuous_integration.yaml index 8bfaa3f58..8471ca6a5 100644 --- a/.github/workflows/continuous_integration.yaml +++ b/.github/workflows/continuous_integration.yaml @@ -26,6 +26,10 @@ jobs: tags: arquivo-test:latest - name: run the test run: docker run arquivo-test:latest + - name: run the test + run: STATIC_PLS=1 docker run -e STATIC_PLS arquivo-test:latest + + # run_rails_test: From de4637ae613777e9e77e98498d0a464f984698f5 Mon Sep 17 00:00:00 2001 From: Phill MV Date: Thu, 21 Dec 2023 13:45:44 -0500 Subject: [PATCH 11/18] ignore vendor/ruby pls --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index 7c8f67ee6..6fa88914d 100644 --- a/.gitignore +++ b/.gitignore @@ -38,6 +38,8 @@ yarn-debug.log* /.nix-gems /.nix-node # /vendor/cache +# we vendor the gems, but not the installed paths +/vendor/ruby *.DS_Store # use this to customize your .env entries From 815d705f463391d32c48f04b7109d5b60905d773 Mon Sep 17 00:00:00 2001 From: Phill MV Date: Thu, 21 Dec 2023 13:46:05 -0500 Subject: [PATCH 12/18] minor comment in routes.rb --- config/routes.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/routes.rb b/config/routes.rb index b31b86bfe..827ab8abc 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -53,7 +53,7 @@ # only here to keep the UrlHelper ticking get "/settings", to: "settings#index", as: :settings get "/*id/edit", to: "entries#edit", as: :edit_entry - else + else # END STATIC MODE resources :notebooks scope ':owner', defaults: { owner: "owner" } do From 223eeba7d4e29114480b0a77941527f5be6d9012 Mon Sep 17 00:00:00 2001 From: Phill MV Date: Fri, 22 Dec 2023 16:34:01 -0500 Subject: [PATCH 13/18] added mawl mode --- bin/start-arquivo | 30 +++++++++++++++++++++++++++--- 1 file changed, 27 insertions(+), 3 deletions(-) diff --git a/bin/start-arquivo b/bin/start-arquivo index 5e31b2879..6aee9fdc6 100755 --- a/bin/start-arquivo +++ b/bin/start-arquivo @@ -3,9 +3,12 @@ set -euo pipefail PROJECT_FOLDER=`realpath $(dirname "$0")/..` -DEV_DATA_FOLDER="$PROJECT_FOLDER"/tmp/data -mkdir -p "$DEV_DATA_FOLDER" +if [ -z ${DATA_FOLDER+x} ]; then + DATA_FOLDER="$PROJECT_FOLDER"/data +fi + +mkdir -p "$DATA_FOLDER" if [ -z ${ARQUIVO_PORT+x} ]; then ARQUIVO_PORT=12346 @@ -27,6 +30,25 @@ else # do the default DOCKER_COMMAND= export RAILS_ENV=development + elif [ "$1" = "mawl" ]; then + shift + + if [ -z "${1+x}" ]; then + echo "Usage: $0 mawl [input-folder] [output-folder]" + exit 1 + else + MAWL_INPUT_PATH=`realpath "$1"` + fi + + if [ -z "${2+x}" ]; then + MAWL_OUTPUT_PATH="$PROJECT_FOLDER"/tmp/mawl-output + else + MAWL_OUTPUT_PATH="$2" + fi + + export STATIC_PLS=1 + DOCKER_COMMAND=bash + # DOCKER_COMMAND="./bin/rails server -d && ./bin/rails static:import static:generate" fi fi @@ -45,6 +67,8 @@ docker run -it -p "$ARQUIVO_PORT":3001 \ -e RAILS_ENV \ -e RAILS_BIND \ -e STATIC_PLS \ - -v "$DEV_DATA_FOLDER":/data \ + -v "$MAWL_INPUT_PATH":/mawl-input \ + -v "$DATA_FOLDER":/data \ + -v "$MAWL_OUTPUT_PATH":/output \ -v "$PROJECT_FOLDER":/arquivo \ arquivo-development:latest ${DOCKER_COMMAND} From d861575d87d84c6a0f49be8ae84cbd4990fbc5b1 Mon Sep 17 00:00:00 2001 From: Phill MV Date: Fri, 22 Dec 2023 16:34:10 -0500 Subject: [PATCH 14/18] need wget for mawl --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 7d1889a15..98b1ce55d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -23,7 +23,7 @@ RUN gem update --system --no-document && \ RUN --mount=type=cache,id=dev-apt-cache,sharing=locked,target=/var/cache/apt \ --mount=type=cache,id=dev-apt-lib,sharing=locked,target=/var/lib/apt \ apt-get update -qq && \ - apt-get install --no-install-recommends -y curl git openssh-client libsqlite3-0 + apt-get install --no-install-recommends -y curl git openssh-client libsqlite3-0 wget RUN useradd rails --create-home --shell /bin/bash From 82695662c75bcf41bde8a9539459490c2c98a2d8 Mon Sep 17 00:00:00 2001 From: Phill MV Date: Sat, 23 Dec 2023 14:08:13 -0500 Subject: [PATCH 15/18] Fixed blob creation in AdHoc importer, added some static? checks where appropriate. --- app/controllers/application_controller.rb | 2 +- app/controllers/entries_controller.rb | 1 + app/models/ad_hoc_markdown_importer.rb | 15 +++++---------- config/environments/development.rb | 5 +++++ config/environments/production.rb | 3 +++ lib/tasks/static.rake | 21 ++++++++++++++------- 6 files changed, 29 insertions(+), 18 deletions(-) diff --git a/app/controllers/application_controller.rb b/app/controllers/application_controller.rb index b41044943..c56c9cd00 100644 --- a/app/controllers/application_controller.rb +++ b/app/controllers/application_controller.rb @@ -20,7 +20,7 @@ def check_imports def resync_with_remotes last_checked_at = session[:synced_remotes_at] if last_checked_at.nil? || last_checked_at < 15.minutes.ago - PullAllFromGit.perform_later + PullAllFromGit.perform_later unless Arquivo.static? end session[:synced_remotes_at] = Time.current end diff --git a/app/controllers/entries_controller.rb b/app/controllers/entries_controller.rb index 58fa774f1..07cc76e07 100644 --- a/app/controllers/entries_controller.rb +++ b/app/controllers/entries_controller.rb @@ -12,6 +12,7 @@ def index # GET /entries/1.json def show if @entry.document? + # TODO: why import these at all why not just open the frigging thing? blob = @entry.files.blobs.first expires_in ActiveStorage.service_urls_expire_in redirect_to rails_blob_path(blob, disposition: params[:disposition]) diff --git a/app/models/ad_hoc_markdown_importer.rb b/app/models/ad_hoc_markdown_importer.rb index 2e28d1767..58cdcc1a1 100644 --- a/app/models/ad_hoc_markdown_importer.rb +++ b/app/models/ad_hoc_markdown_importer.rb @@ -29,7 +29,8 @@ def find_or_create_notebook notebook = Notebook.find_by(name: notebook_name) if notebook.nil? notebook = Notebook.create(name: notebook_name, - import_path: notebook_path) + import_path: notebook_path, + skip_local_sync: true) else notebook.update(import_path: notebook_path) end @@ -70,16 +71,10 @@ def process_import_path(notebook) filename = File.basename(identifier) if !entry.files.blobs.find_by(filename: filename) - # TODO 2023-11-23: mimic the same blob lifecycle handling from create_blob_and_file to avoid the async stuff noted below. blob = ActiveStorage::Blob.create_and_upload!(io: File.open(file_path), - filename: filename) - - # run analysis step synchronously, so we skip the async job. - # for reasons i don't comprehend, in dev mode at least - # ActiveStorage::AnalyzeJob just hangs there indefinitely, doing - # naught to improve our lot, and this is very frustrating and further - # i have close to zero desire to debug ActiveJob async shenanigans - blob.analyze + filename: filename, + metadata: { "analyzed" => true }) + entry.files.create(blob_id: blob.id, created_at: blob.created_at) end end diff --git a/config/environments/development.rb b/config/environments/development.rb index 52e8a158d..9f101c1be 100644 --- a/config/environments/development.rb +++ b/config/environments/development.rb @@ -2,6 +2,11 @@ require 'socket' Rails.application.configure do + if Arquivo.static? + # TODO WRITE A TEST TO CHECK IF STUFF IN /ASSETS MAKE SIT THRU + # TO STATIC GENERATE + config.assets.compile = false + end config.hosts << "arquivo.io" # until we figure out how to multitenant this, diff --git a/config/environments/production.rb b/config/environments/production.rb index c13f2910b..74b0b23b1 100644 --- a/config/environments/production.rb +++ b/config/environments/production.rb @@ -3,6 +3,9 @@ Rails.application.configure do config.hosts << "arquivo.io" config.hosts << "arquivo.localhost" + if Arquivo.static? + config.hosts << "localhost" + end # until we figure out how to multitenant this, config.active_storage.routes_prefix = ENV["ARQUIVO_USER"] || "/phillmv/_" diff --git a/lib/tasks/static.rake b/lib/tasks/static.rake index 44ba463c0..e472e1339 100644 --- a/lib/tasks/static.rake +++ b/lib/tasks/static.rake @@ -1,18 +1,25 @@ namespace :static do desc 'Generate static site in ./out/ directory' task :import => :environment do - SyncFromDisk.new(ENV["NOTEBOOK_PATH"]).import! + if !Arquivo.static? + puts "Buddy, we're not in static mode! Quitting..." + exit 1 + end + + sync_path = ENV["MAWL_INPUT_PATH"] || "/mawl-input" + SyncFromDisk.new(sync_path).import! end task :generate do - if ENV["MAWB_OUTPUT"] - folder = ENV["MAWB_OUTPUT"] - else - folder = "out" + if !Arquivo.static? + puts "Buddy, we're not in static mode! Quitting..." + exit 1 end - Dir.mkdir folder unless File.exist? folder - Dir.chdir folder do + output_path = ENV["MAWL_OUTPUT_PATH"] || "/output" + + Dir.mkdir output_path unless File.exist? output_path + Dir.chdir output_path do puts `wget --version` puts "######\nwget --domains localhost --recursive --page-requisites --html-extension --convert-links -nH localhost:3000 localhost:3000/hidden_entries" `wget --domains localhost --recursive --page-requisites --html-extension --convert-links -nH localhost:3000 localhost:3000/hidden_entries` From 71d34111b866ab6842857acb6e1b4ef02fb2bc3d Mon Sep 17 00:00:00 2001 From: Phill MV Date: Sat, 23 Dec 2023 14:09:09 -0500 Subject: [PATCH 16/18] made SyncFromDisk throw an exception if blob creation fails. --- app/models/sync_from_disk.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/models/sync_from_disk.rb b/app/models/sync_from_disk.rb index 249e12d02..c92bdb689 100644 --- a/app/models/sync_from_disk.rb +++ b/app/models/sync_from_disk.rb @@ -164,7 +164,7 @@ def create_blob_and_file(entry, blob_attr, entry_files_path) end blob_attr["metadata"]["analyzed"] = true - blob = ActiveStorage::Blob.create(blob_attr) + blob = ActiveStorage::Blob.create!(blob_attr) blob.upload_without_unfurling(File.open(new_attachment_filepath)) entry.files.create(blob_id: blob.id, created_at: blob.created_at) From 5c5e262c5cf39c88a18a325eb5b1413b523ca387 Mon Sep 17 00:00:00 2001 From: Phill MV Date: Sat, 23 Dec 2023 14:38:39 -0500 Subject: [PATCH 17/18] added little scripts for kicking off mawl --- bin/mawl-generate | 8 ++++++++ bin/mawl-import | 6 ++++++ bin/mawl-import-and-generate | 6 ++++++ bin/start-arquivo | 2 +- 4 files changed, 21 insertions(+), 1 deletion(-) create mode 100755 bin/mawl-generate create mode 100755 bin/mawl-import create mode 100755 bin/mawl-import-and-generate diff --git a/bin/mawl-generate b/bin/mawl-generate new file mode 100755 index 000000000..377971db8 --- /dev/null +++ b/bin/mawl-generate @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +set -euo pipefail + +PROJECT_FOLDER=`realpath $(dirname "$0")/..` && cd "$PROJECT_FOLDER" + +PID=$(cat tmp/pids/server.pid 2>/dev/null) && ps -p $PID > /dev/null && kill $PID +./bin/rails s -e production -d && ./bin/rails static:generate diff --git a/bin/mawl-import b/bin/mawl-import new file mode 100755 index 000000000..d3445f035 --- /dev/null +++ b/bin/mawl-import @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +set -euo pipefail + +PROJECT_FOLDER=`realpath $(dirname "$0")/..` && cd "$PROJECT_FOLDER" +./bin/rails static:import diff --git a/bin/mawl-import-and-generate b/bin/mawl-import-and-generate new file mode 100755 index 000000000..774549d63 --- /dev/null +++ b/bin/mawl-import-and-generate @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +set -euo pipefail + +PROJECT_FOLDER=`realpath $(dirname "$0")/..` && cd "$PROJECT_FOLDER" +./bin/mawl-import && ./bin/mawl-generate diff --git a/bin/start-arquivo b/bin/start-arquivo index 6aee9fdc6..dccc315a2 100755 --- a/bin/start-arquivo +++ b/bin/start-arquivo @@ -47,7 +47,7 @@ else fi export STATIC_PLS=1 - DOCKER_COMMAND=bash + DOCKER_COMMAND=/arquivo/bin/mawl-import-and-generate # DOCKER_COMMAND="./bin/rails server -d && ./bin/rails static:import static:generate" fi fi From 451d325c9ba0078bb4debc75beb5922e0c3a02b8 Mon Sep 17 00:00:00 2001 From: Phill MV Date: Sun, 31 Dec 2023 10:44:54 -0500 Subject: [PATCH 18/18] minor comment --- bin/start-arquivo | 1 + 1 file changed, 1 insertion(+) diff --git a/bin/start-arquivo b/bin/start-arquivo index dccc315a2..3630104c4 100755 --- a/bin/start-arquivo +++ b/bin/start-arquivo @@ -31,6 +31,7 @@ else DOCKER_COMMAND= export RAILS_ENV=development elif [ "$1" = "mawl" ]; then + # TODO: rationalize into its own command, plus a server option shift if [ -z "${1+x}" ]; then