diff --git a/.github/workflows/ci-debian13.yml b/.github/workflows/ci-debian13.yml index ffc56987e2..64bebc52a0 100644 --- a/.github/workflows/ci-debian13.yml +++ b/.github/workflows/ci-debian13.yml @@ -8,6 +8,7 @@ on: - "!src/Plugins/Windows/**" - "tests/**" - "lolly/**" + - "moebius/**" - "xmake.lua" - "xmake/packages.lua" - "xmake/packages/**" @@ -22,6 +23,7 @@ on: - "!src/Plugins/Windows/**" - "tests/**" - "lolly/**" + - "moebius/**" - "xmake.lua" - "xmake/packages.lua" - "xmake/packages/**" @@ -97,6 +99,19 @@ jobs: - 'lolly/System/**' - 'lolly/lolly/**' - 'lolly/xmake.lua' + moebius: + - 'moebius/Data/**' + - 'moebius/Kernel/**' + - 'moebius/Scheme/**' + - 'moebius/moebius/**' + - 'moebius/tests/**' + - 'moebius/xmake.lua' + moebius_src: + - 'moebius/Data/**' + - 'moebius/Kernel/**' + - 'moebius/Scheme/**' + - 'moebius/moebius/**' + - 'moebius/xmake.lua' mogan: - 'src/**' - 'tests/**' @@ -114,13 +129,19 @@ jobs: - name: test lolly if: steps.filter.outputs.lolly == 'true' run: xmake test "lolly_tests/*" + - name: build moebius + if: steps.filter.outputs.moebius == 'true' + run: xmake build --yes -vD libmoebius + - name: test moebius + if: steps.filter.outputs.moebius == 'true' + run: xmake test "moebius_tests/*" - name: build - if: steps.filter.outputs.lolly_src == 'true' || steps.filter.outputs.mogan == 'true' + if: steps.filter.outputs.lolly_src == 'true' || steps.filter.outputs.moebius_src == 'true' || steps.filter.outputs.mogan == 'true' run: xmake build --yes -vD stem && xmake build --yes -vD --group=tests - name: test - if: steps.filter.outputs.lolly_src == 'true' || steps.filter.outputs.mogan == 'true' + if: steps.filter.outputs.lolly_src == 'true' || steps.filter.outputs.moebius_src == 'true' || steps.filter.outputs.mogan == 'true' run: xmake run --yes -vD --group=tests - name: Scheme tests - if: steps.filter.outputs.lolly_src == 'true' || steps.filter.outputs.mogan == 'true' + if: steps.filter.outputs.lolly_src == 'true' || steps.filter.outputs.moebius_src == 'true' || steps.filter.outputs.mogan == 'true' run: xmake run --yes -vD --group=scheme_tests diff --git a/.github/workflows/ci-fedora.yml b/.github/workflows/ci-fedora.yml index ae7429e3b8..ad443e9ada 100644 --- a/.github/workflows/ci-fedora.yml +++ b/.github/workflows/ci-fedora.yml @@ -7,6 +7,8 @@ on: - "!src/Plugins/Macos/**" - "!src/Plugins/Windows/**" - "tests/**" + - "lolly/**" + - "moebius/**" - "xmake.lua" - "xmake/packages.lua" - "xmake/packages/**" @@ -19,6 +21,8 @@ on: - "!src/Plugins/Macos/**" - "!src/Plugins/Windows/**" - "tests/**" + - "lolly/**" + - "moebius/**" - "xmake.lua" - "xmake/packages.lua" - "xmake/packages/**" @@ -74,11 +78,62 @@ jobs: tmp/build/.build_cache key: ${{ runner.os }}-fedora-build-qt683-${{ hashFiles('**/packages.lua') }} + - name: Detect changes + id: filter + uses: dorny/paths-filter@v3 + with: + filters: | + lolly: + - 'lolly/**' + lolly_src: + - 'lolly/Data/**' + - 'lolly/Kernel/**' + - 'lolly/Plugins/**' + - 'lolly/System/**' + - 'lolly/lolly/**' + - 'lolly/xmake.lua' + moebius: + - 'moebius/Data/**' + - 'moebius/Kernel/**' + - 'moebius/Scheme/**' + - 'moebius/moebius/**' + - 'moebius/tests/**' + - 'moebius/xmake.lua' + moebius_src: + - 'moebius/Data/**' + - 'moebius/Kernel/**' + - 'moebius/Scheme/**' + - 'moebius/moebius/**' + - 'moebius/xmake.lua' + mogan: + - 'src/**' + - 'tests/**' + - 'xmake.lua' + - 'xmake/packages.lua' + - 'xmake/packages/**' + - 'TeXmacs/tests/*.scm' + - name: config run: xmake config -vD --policies=build.ccache -o tmp/build -m releasedbg --yes + - name: build lolly + if: steps.filter.outputs.lolly == 'true' + run: xmake build --yes -vD liblolly + - name: test lolly + if: steps.filter.outputs.lolly == 'true' + run: xmake test "lolly_tests/*" + - name: build moebius + if: steps.filter.outputs.moebius == 'true' + run: xmake build --yes -vD libmoebius + - name: test moebius + if: steps.filter.outputs.moebius == 'true' + run: xmake test "moebius_tests/*" + - name: build + if: steps.filter.outputs.lolly_src == 'true' || steps.filter.outputs.moebius_src == 'true' || steps.filter.outputs.mogan == 'true' run: xmake build --yes -vD stem && xmake build --yes -vD --group=tests - name: test + if: steps.filter.outputs.lolly_src == 'true' || steps.filter.outputs.moebius_src == 'true' || steps.filter.outputs.mogan == 'true' run: xmake run --yes -vD --group=tests - name: Scheme tests + if: steps.filter.outputs.lolly_src == 'true' || steps.filter.outputs.moebius_src == 'true' || steps.filter.outputs.mogan == 'true' run: xmake run --yes -vD --group=scheme_tests diff --git a/.github/workflows/ci-macos-arm64.yml b/.github/workflows/ci-macos-arm64.yml index 6997b20e00..c22486b4c2 100644 --- a/.github/workflows/ci-macos-arm64.yml +++ b/.github/workflows/ci-macos-arm64.yml @@ -19,6 +19,12 @@ on: - 'lolly/lolly/**' - 'lolly/tests/**' - 'lolly/xmake.lua' + - 'moebius/Data/**' + - 'moebius/Kernel/**' + - 'moebius/Scheme/**' + - 'moebius/moebius/**' + - 'moebius/tests/**' + - 'moebius/xmake.lua' - '.github/workflows/ci-macos-arm64.yml' - 'TeXmacs/tests/*.scm' - '3rdparty/**' @@ -40,6 +46,12 @@ on: - 'lolly/lolly/**' - 'lolly/tests/**' - 'lolly/xmake.lua' + - 'moebius/Data/**' + - 'moebius/Kernel/**' + - 'moebius/Scheme/**' + - 'moebius/moebius/**' + - 'moebius/tests/**' + - 'moebius/xmake.lua' - '.github/workflows/ci-macos-arm64.yml' - 'TeXmacs/tests/*.scm' - '3rdparty/**' @@ -78,6 +90,19 @@ jobs: - 'lolly/System/**' - 'lolly/lolly/**' - 'lolly/xmake.lua' + moebius: + - 'moebius/Data/**' + - 'moebius/Kernel/**' + - 'moebius/Scheme/**' + - 'moebius/moebius/**' + - 'moebius/tests/**' + - 'moebius/xmake.lua' + moebius_src: + - 'moebius/Data/**' + - 'moebius/Kernel/**' + - 'moebius/Scheme/**' + - 'moebius/moebius/**' + - 'moebius/xmake.lua' mogan: - 'src/**' - 'tests/**' @@ -95,7 +120,7 @@ jobs: path: | ${{github.workspace}}/build/.build_cache /Users/runner/.xmake - key: xmake-${{ runner.os }}-${{ matrix.arch }}-${{ hashFiles('xmake/vars.lua', 'lolly/xmake.lua') }} + key: xmake-${{ runner.os }}-${{ matrix.arch }}-${{ hashFiles('xmake/vars.lua', 'lolly/xmake.lua', 'moebius/xmake.lua') }} - name: Set Qt Environment Variables run: | @@ -121,7 +146,7 @@ jobs: with: path: | ${{ env.XMAKE_GLOBALDIR }}/.xmake/packages - key: xrepo-${{ runner.os }}-${{ matrix.arch }}-${{ hashFiles('xmake/vars.lua', 'lolly/xmake.lua') }} + key: xrepo-${{ runner.os }}-${{ matrix.arch }}-${{ hashFiles('xmake/vars.lua', 'lolly/xmake.lua', 'moebius/xmake.lua') }} - name: config run: xmake config --policies=build.ccache -o ${{ runner.workspace }}/build -m releasedbg --yes @@ -131,17 +156,23 @@ jobs: - name: test lolly if: steps.filter.outputs.lolly == 'true' run: xmake test "lolly_tests/*" + - name: build moebius + if: steps.filter.outputs.moebius == 'true' + run: xmake build --yes -vD libmoebius + - name: test moebius + if: steps.filter.outputs.moebius == 'true' + run: xmake test "moebius_tests/*" - name: build - if: steps.filter.outputs.lolly_src == 'true' || steps.filter.outputs.mogan == 'true' + if: steps.filter.outputs.lolly_src == 'true' || steps.filter.outputs.moebius_src == 'true' || steps.filter.outputs.mogan == 'true' run: xmake build --yes -vD stem && xmake build --yes -vD --group=tests - name: C++ tests - if: steps.filter.outputs.lolly_src == 'true' || steps.filter.outputs.mogan == 'true' + if: steps.filter.outputs.lolly_src == 'true' || steps.filter.outputs.moebius_src == 'true' || steps.filter.outputs.mogan == 'true' run: xmake run --yes --verbose --diagnosis --group=tests env: QT_QPA_PLATFORM: offscreen - name: Scheme tests - if: steps.filter.outputs.lolly_src == 'true' || steps.filter.outputs.mogan == 'true' + if: steps.filter.outputs.lolly_src == 'true' || steps.filter.outputs.moebius_src == 'true' || steps.filter.outputs.mogan == 'true' run: xmake run --yes -vD --group=scheme_tests # - name: integration test # run: xmake run --yes -vD --group=integration_tests diff --git a/.github/workflows/ci-moebius-debian13.yml b/.github/workflows/ci-moebius-debian13.yml deleted file mode 100644 index dceffedb9f..0000000000 --- a/.github/workflows/ci-moebius-debian13.yml +++ /dev/null @@ -1,76 +0,0 @@ -name: CI for moebius on Debian 13 - -on: - push: - branches: [ main ] - paths: - - 'moebius/Data/**' - - 'moebius/Kernel/**' - - 'moebius/Scheme/**' - - 'moebius/moebius/**' - - 'moebius/tests/**' - - 'moebius/xmake.lua' - - '.github/workflows/ci-moebius-debian13.yml' - pull_request: - branches: [ main ] - paths: - - 'moebius/Data/**' - - 'moebius/Kernel/**' - - 'moebius/Scheme/**' - - 'moebius/moebius/**' - - 'moebius/tests/**' - - 'moebius/xmake.lua' - - '.github/workflows/ci-moebius-debian13.yml' - workflow_dispatch: - -env: - XMAKE_ROOT: y - QT_QPA_PLATFORM: offscreen - -jobs: - build: - container: debian:13 - runs-on: ubuntu-latest - steps: - - name: Install dependencies - run: | - DEBIAN_FRONTEND=noninteractive apt-get update - DEBIAN_FRONTEND=noninteractive apt-get install -y gcc git 7zip unzip curl build-essential \ - fonts-noto-cjk libcurl4-openssl-dev libfreetype-dev libfontconfig-dev \ - libmimalloc-dev libgit2-dev zlib1g-dev libssl-dev libjpeg62-turbo-dev cmake \ - pandoc python3.13 python3.13-venv python3-pip ninja-build \ - libdbus-1-3 libglib2.0-0t64 libegl1 libgl-dev libxkbcommon0 - update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.13 1 - python3 -m pip install --break-system-packages aqtinstall - - uses: actions/checkout@v3 - with: - fetch-depth: 1 - - - uses: xmake-io/github-action-setup-xmake@v1 - with: - xmake-version: v3.0.7 - - - name: git add safe directory - run: git config --global --add safe.directory '*' - - name: set XMAKE_GLOBALDIR - run: echo "XMAKE_GLOBALDIR=${{ runner.workspace }}/xmake-global" >> $GITHUB_ENV - - name: xmake repo update - run: xmake repo --update - - name: cache packages from xrepo - uses: actions/cache@v4 - with: - path: | - ${{ env.XMAKE_GLOBALDIR }}/.xmake/packages - key: ${{ runner.os }}-xrepo-qt683-${{ hashFiles('**/packages.lua') }} - - name: cache xmake - uses: actions/cache@v4 - with: - path: | - build/.build_cache - key: ${{ runner.os }}-build-qt683-${{ hashFiles('**/packages.lua') }} - - name: config - run: xmake config -vD --policies=build.ccache -m releasedbg --yes - - name: build - run: xmake build --yes -vD libmoebius - - name: test - run: xmake test "moebius_tests/*" diff --git a/.github/workflows/ci-moebius-macos.yml b/.github/workflows/ci-moebius-macos.yml deleted file mode 100644 index 80d12cb406..0000000000 --- a/.github/workflows/ci-moebius-macos.yml +++ /dev/null @@ -1,63 +0,0 @@ -name: CI for moebius on macOS - -on: - push: - branches: [ main ] - paths: - - 'moebius/Data/**' - - 'moebius/Kernel/**' - - 'moebius/Scheme/**' - - 'moebius/moebius/**' - - 'moebius/tests/**' - - 'moebius/xmake.lua' - - '.github/workflows/ci-moebius-macos.yml' - pull_request: - branches: [ main ] - paths: - - 'moebius/Data/**' - - 'moebius/Kernel/**' - - 'moebius/Scheme/**' - - 'moebius/moebius/**' - - 'moebius/tests/**' - - 'moebius/xmake.lua' - - '.github/workflows/ci-moebius-macos.yml' - workflow_dispatch: - -jobs: - macosbuild: - strategy: - matrix: - os: [macos-14] - arch: [arm64] - runs-on: ${{ matrix.os }} - timeout-minutes: 45 - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 1 - - name: set XMAKE_GLOBALDIR - run: echo "XMAKE_GLOBALDIR=${{ runner.workspace }}/xmake-global" >> $GITHUB_ENV - - uses: xmake-io/github-action-setup-xmake@v1 - with: - xmake-version: v3.0.4 - actions-cache-folder: '.xmake-cache' - - name: xmake repo update - run: xmake repo --update - - name: cache packages from xrepo - uses: actions/cache@v4 - with: - path: | - ${{ env.XMAKE_GLOBALDIR }}/.xmake/packages - key: xrepo-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('xmake/vars.lua', 'moebius/xmake.lua') }} - - name: cache xmake - uses: actions/cache@v4 - with: - path: | - ${{ github.workspace }}/build/.build_cache - key: xmake-build-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('xmake/vars.lua', 'moebius/xmake.lua') }} - - name: config - run: xmake config --policies=build.ccache -m releasedbg --yes - - name: build - run: xmake build --yes -vD libmoebius - - name: test - run: xmake test "moebius_tests/*" diff --git a/.github/workflows/ci-moebius-windows.yml b/.github/workflows/ci-moebius-windows.yml deleted file mode 100644 index f200bf8398..0000000000 --- a/.github/workflows/ci-moebius-windows.yml +++ /dev/null @@ -1,60 +0,0 @@ -name: CI for moebius on Windows - -on: - push: - branches: [ main ] - paths: - - 'moebius/Data/**' - - 'moebius/Kernel/**' - - 'moebius/Scheme/**' - - 'moebius/moebius/**' - - 'moebius/tests/**' - - 'moebius/xmake.lua' - - '.github/workflows/ci-moebius-windows.yml' - pull_request: - branches: [ main ] - paths: - - 'moebius/Data/**' - - 'moebius/Kernel/**' - - 'moebius/Scheme/**' - - 'moebius/moebius/**' - - 'moebius/tests/**' - - 'moebius/xmake.lua' - - '.github/workflows/ci-moebius-windows.yml' - workflow_dispatch: - -env: - XMAKE_GLOBALDIR: "${{ github.workspace }}\\.xmake-global" - -jobs: - windowsbuild: - runs-on: windows-2025 - steps: - - uses: xmake-io/github-action-setup-xmake@v1 - with: - xmake-version: v3.0.4 - - name: update repo - run: xrepo update-repo - - name: git crlf - run: git config --global core.autocrlf false - - uses: actions/checkout@v3 - with: - fetch-depth: 1 - - name: cache packages from xrepo - uses: actions/cache@v4 - with: - path: | - ${{ env.XMAKE_GLOBALDIR }}/.xmake/packages - key: qt683-${{ runner.os }}-${{ runner.arch }}-xrepo-${{ hashFiles('xmake/vars.lua', 'moebius/xmake.lua') }} - - name: cache xmake - uses: actions/cache@v4 - with: - path: | - ${{ github.workspace }}/build/.build_cache - key: ${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('xmake/vars.lua', 'moebius/xmake.lua') }} - - name: config - run: xmake config --policies=build.ccache --yes -vD -m releasedbg --plat=windows - - name: build - run: xmake build --yes -vD libmoebius - - name: test - run: xmake test "moebius_tests/*" diff --git a/.github/workflows/ci-xmake-windows.yml b/.github/workflows/ci-xmake-windows.yml index 28317c700b..e9152444ad 100644 --- a/.github/workflows/ci-xmake-windows.yml +++ b/.github/workflows/ci-xmake-windows.yml @@ -21,6 +21,12 @@ on: - 'lolly/lolly/**' - 'lolly/tests/**' - 'lolly/xmake.lua' + - 'moebius/Data/**' + - 'moebius/Kernel/**' + - 'moebius/Scheme/**' + - 'moebius/moebius/**' + - 'moebius/tests/**' + - 'moebius/xmake.lua' - '.github/workflows/ci-xmake-windows.yml' - '3rdparty/**' pull_request: @@ -43,6 +49,12 @@ on: - 'lolly/lolly/**' - 'lolly/tests/**' - 'lolly/xmake.lua' + - 'moebius/Data/**' + - 'moebius/Kernel/**' + - 'moebius/Scheme/**' + - 'moebius/moebius/**' + - 'moebius/tests/**' + - 'moebius/xmake.lua' - '.github/workflows/ci-xmake-windows.yml' - '3rdparty/**' workflow_dispatch: @@ -87,6 +99,19 @@ jobs: - 'lolly/System/**' - 'lolly/lolly/**' - 'lolly/xmake.lua' + moebius: + - 'moebius/Data/**' + - 'moebius/Kernel/**' + - 'moebius/Scheme/**' + - 'moebius/moebius/**' + - 'moebius/tests/**' + - 'moebius/xmake.lua' + moebius_src: + - 'moebius/Data/**' + - 'moebius/Kernel/**' + - 'moebius/Scheme/**' + - 'moebius/moebius/**' + - 'moebius/xmake.lua' mogan: - 'src/**' - 'tests/**' @@ -102,13 +127,13 @@ jobs: with: path: | ${{ env.XMAKE_GLOBALDIR }}/.xmake/packages - key: qt683-${{ runner.os }}-${{ runner.arch }}-xrepo-${{ hashFiles('xmake/vars.lua', 'lolly/xmake.lua') }} + key: qt683-${{ runner.os }}-${{ runner.arch }}-xrepo-${{ hashFiles('xmake/vars.lua', 'lolly/xmake.lua', 'moebius/xmake.lua') }} - name: cache xmake uses: actions/cache@v4 with: path: | ${{ github.workspace }}/build/.build_cache - key: ${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('xmake/vars.lua', 'lolly/xmake.lua') }} + key: ${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('xmake/vars.lua', 'lolly/xmake.lua', 'moebius/xmake.lua') }} - name: config run: xmake config --policies=build.ccache --yes -vD -m releasedbg --plat=windows - name: build lolly @@ -117,7 +142,13 @@ jobs: - name: test lolly if: steps.filter.outputs.lolly == 'true' run: xmake test "lolly_tests/*" + - name: build moebius + if: steps.filter.outputs.moebius == 'true' + run: xmake build --yes -vD libmoebius + - name: test moebius + if: steps.filter.outputs.moebius == 'true' + run: xmake test "moebius_tests/*" - name: build stem - if: steps.filter.outputs.lolly_src == 'true' || steps.filter.outputs.mogan == 'true' + if: steps.filter.outputs.lolly_src == 'true' || steps.filter.outputs.moebius_src == 'true' || steps.filter.outputs.mogan == 'true' run: xmake build --yes -vD stem diff --git a/devel/1127.md b/devel/1127.md new file mode 100644 index 0000000000..e8b115d7ab --- /dev/null +++ b/devel/1127.md @@ -0,0 +1,95 @@ +# [1127] 将 TMU 序列化/反序列化迁移到 moebius + +## 1 背景 + +- `src/Data/Convert/Mogan/from_tmu.cpp`(378 行)和 `to_tmu.cpp`(302 行) + 负责在 TeXmacs tree 与新的 TMU 文件格式之间互转。 +- 这两个文件对主工程的依赖,几乎全部已经落在 moebius + lolly 能力范围内: + - `make_tree_label`、`is_func`/`is_compound`/`is_document`/`is_atomic`/ + `is_apply`/`is_expand`、`STD_CODE`、`DOCUMENT`/`CONCAT`/`COLLECTION`/ + `RAW_DATA`/`EXPAND` 等标签 —— moebius `tree_label.hpp` / `tree_helper.hpp` + / `drd/drd_std.hpp` 均已提供。 + - `skip_spaces` / `index_of` / `tokenize` / `starts` —— lolly `analyze.hpp`。 + - `decode_from_utf8` / `from_hex` / `to_Hex` / `binary_to_hexadecimal` —— + lolly `data/numeral` / `data/unicode`。 + - `version_inf` —— moebius `Kernel/Types/path.hpp`。 +- 唯一阻塞点是 `tmu_reader` 构造时调用的 `get_codes(version)`(版本升级映射表), + 其实现位于 `src/Data/Convert/Texmacs/upgradetm.cpp:51`,主工程的 `fromtm.cpp` + 也在共用。该函数只依赖 moebius 已有的 `STD_CODE` + `version_inf`,可以一并下沉。 +- `from_tmu.cpp` 顶部的 `#include "preferences.hpp"` 经确认无任何符号引用, + 是历史残留,顺手删除。 +- 这是继 [1124] glue standalone 迁移、[1126] tbox 包定义迁移之后,「主工程 → + moebius/lolly」剥离工作的延续。 + +## 2 What + +1. 新建 `moebius/Data/Convert/tmu.hpp`,导出三个公共函数: + `tree tmu_to_tree(string)`, `tree tmu_to_tree(string, string)`, + `tree tmu_document_to_tree(string)`, `string tree_to_tmu(tree)`。 +2. 新建 `moebius/Data/Convert/tmu.cpp`,合并 `from_tmu.cpp` + `to_tmu.cpp` + 全部内容(`tmu_reader` / `tmu_writer` / `get_collection` / `flush` 等), + namespace 包进 `moebius`。 +3. `get_codes` 连同其 helper(`rename_feature`/`new_feature`)一并迁入 + `moebius/Data/Convert/tmu.cpp`(以 `static` 形式内联),供 `tmu_reader` + 使用;`tmu.hpp` 不导出 `get_codes`(保留为内部实现)。 +4. 主工程 `convert.hpp` 的 `get_codes` 声明删除,改由 upgradetm.cpp 内部 + 包含 moebius 提供的等价声明 —— 见下方 Why 第 2 条的最终决策。 +5. `is_snippet`:to_tmu 原本依赖主工程的 `is_snippet`。该谓词实现仅 6 行纯 + tree 判断,且主工程 `totm.cpp` / `verbatim.cpp` 仍在用。决策: + - 在 moebius `Data/Tree/tree_helper.{hpp,cpp}` 补一个 `is_snippet`, + 主工程 `convert.hpp` 改为转发声明(moebius 提供),避免重复实现。 +6. 更新 `moebius/xmake.lua`:`Data/Convert/**.cpp` 加入 `moe_files`, + `Data/Convert` 加入 `moe_includedirs` 和 `add_headerfiles`。 +7. 删除 `src/Data/Convert/Mogan/from_tmu.cpp` 与 `to_tmu.cpp`。 +8. 主工程 `convert.hpp` 的 TMU 三函数声明改为 `#include` moebius 头转发, + 保证 `glue_convert.lua` 仍能解析符号。 +9. 清理 `from_tmu.cpp` 顶部无用的 `#include "preferences.hpp"`。 + +## 3 Why + +- **持续剥离主工程**:TMU 是新版序列化格式,无遗留耦合,是低风险迁移对象。 + 下沉到 moebius 后,后续可在脱离 GUI 的场景(如服务端批处理、格式互转工具) + 直接复用,与 [1124]/[1126] 同向。 +- **`get_codes` 必须随迁**:`tmu_reader` 构造函数直接调用,若留在主工程会 + 形成反向依赖。该函数体量有限(~95 行),依赖干净,一并下沉成本最低。 + 下沉后 moebius 同时为后续迁移 `fromtm.cpp` / `upgradetm.cpp` 铺路。 +- **版本号占位锁死到真实值**:`XMACS_VERSION`/`TEXMACS_VERSION` 在 moebius + 编译单元里看不到主工程 `tm_configure.hpp`,用 `#ifndef` 给占位,值锁死到 + 主工程当前版本(`"2.1.4"` / `"2026.3.0"`)。单参 `tmu_to_tree` 直接用 + `STD_CODE`,双参 `tmu_document_to_tree` 的 version 来自文件头解析的值; + `TEXMACS_VERSION` 仅在 doc 缺 TeXmacs 头时补版本号用到。主工程升级版本时 + 需同步 moebius 里的占位。后续可单独 PR 把版本号参数化。 + +## 4 How + +- 合并而非分文件:from/to 同属 TMU 格式,放单文件 `tmu.cpp` 与现有 + `texmacs_to_tree` / `tree_to_texmacs`(仍在主工程)的命名风格一致。 +- namespace:整文件 `namespace moebius {}` 包裹,符合 moebius 现有惯例 + (参考 `moebius/data/scheme_der.cpp`)。 +- `tmu.hpp` 只暴露顶层 4 个函数,内部 struct(`tmu_reader`/`tmu_writer`) + 留在 .cpp 内匿名化或保持外部不可见。 +- `get_codes` 内联为 `static`,不进 `tmu.hpp`,避免污染 moebius 公共 API; + 主工程 `upgradetm.cpp` 暂时保留它自己的 `get_codes` 实现(主工程仍在用, + 且不属于本 PR 范围)—— 见下方「验证」对重复定义的处理。 +- `is_snippet` 走 moebius `tree_helper`,主工程 `generic.cpp` 删除实现, + `convert.hpp` 改为 `#include` moebius 头并去掉本地声明。 + +## 5 涉及文件 + +- `moebius/Data/Convert/tmu.hpp`(新增) +- `moebius/Data/Convert/tmu.cpp`(新增,合并自 from_tmu + to_tmu) +- `moebius/Data/Tree/tree_helper.hpp` / `.cpp`(补 `is_snippet`) +- `moebius/xmake.lua`(`Data/Convert` 加入 moe_files / include / header) +- `src/Data/Convert/convert.hpp`(TMU 声明改为转发 moebius;删 `is_snippet` + 本地声明) +- `src/Data/Convert/Mogan/from_tmu.cpp`(删除) +- `src/Data/Convert/Mogan/to_tmu.cpp`(删除) +- `src/Data/Convert/Generic/generic.cpp`(删 `is_snippet` 实现,转发 moebius) + +## 6 验证 + +- `xmake b libmoebius` 编译通过 +- `xmake b stem` 主工程编译通过(含 `glue_convert` 链接) +- `xmake b` + `xmake r` 跑 TMU 相关单元测试(若存在) +- 手测:`(tree->tmu (string->tree "..."))` / `(tmu->tree ...)` 在 GUI 中 + 往返一致 diff --git a/moebius/Data/Convert/tmu.cpp b/moebius/Data/Convert/tmu.cpp new file mode 100644 index 0000000000..490e78164f --- /dev/null +++ b/moebius/Data/Convert/tmu.cpp @@ -0,0 +1,868 @@ + +/****************************************************************************** + * MODULE : tmu.cpp + * DESCRIPTION: conversion between TeXmacs trees and the TMU file format + * COPYRIGHT : (C) 1999 Joris van der Hoeven + * 2024 Darcy Shen + ******************************************************************************* + * This software falls under the GNU general public license version 3 or later. + * It comes WITHOUT ANY WARRANTY WHATSOEVER. For details, see the file LICENSE + * in the root directory or . + ******************************************************************************/ + +#include "tmu.hpp" + +#include "analyze.hpp" +#include "path.hpp" +#include "tree_helper.hpp" + +#include +#include +#include +#include + +// moebius 编译单元看不到主工程的 tm_configure.hpp,这里给版本宏占位定义。 +// 版本号锁死到主工程当前值(TEXMACS_VERSION "2.1.4" / XMACS_VERSION +// "2026.1.2"),主工程升级时需同步。后续可单独 PR 把版本号参数化。 +#ifndef XMACS_VERSION +#define XMACS_VERSION "2026.3.0" +#endif +#ifndef TEXMACS_VERSION +#define TEXMACS_VERSION "2.1.4" +#endif + +using lolly::data::binary_to_hexadecimal; +using lolly::data::decode_from_utf8; +using lolly::data::from_hex; +using lolly::data::to_Hex; +using moebius::drd::STD_CODE; +using moebius::drd::std_contains; + +using namespace moebius; + +/****************************************************************************** + * Version-dependent feature codes (used by tmu_reader) + ******************************************************************************/ + +static void +rename_feature (hashmap& H, string old_name, string new_name) { + H (old_name)= H[new_name]; + H->reset (new_name); +} + +static void +new_feature (hashmap& H, string new_name) { + H->reset (new_name); +} + +static hashmap +get_codes (string version) { + hashmap H (UNKNOWN); + H->join (STD_CODE); + + if (version_inf ("1.0.7.6", version)) return H; + + rename_feature (H, "group", "rigid"); + rename_feature (H, "postscript", "image"); + + if (version_inf ("1.0.6.9", version)) return H; + + rename_feature (H, "frozen", "freeze"); + + if (version_inf ("1.0.6.2", version)) return H; + + new_feature (H, "expand-as"); + new_feature (H, "locus"); + new_feature (H, "id"); + new_feature (H, "hard-id"); + new_feature (H, "link"); + new_feature (H, "url"); + new_feature (H, "script"); + + if (version_inf ("1.0.4.1", version)) return H; + + new_feature (H, "copy"); + new_feature (H, "cm-length"); + new_feature (H, "mm-length"); + new_feature (H, "in-length"); + new_feature (H, "pt-length"); + new_feature (H, "bp-length"); + new_feature (H, "dd-length"); + new_feature (H, "pc-length"); + new_feature (H, "cc-length"); + new_feature (H, "fs-length"); + new_feature (H, "fbs-length"); + new_feature (H, "em-length"); + new_feature (H, "ln-length"); + new_feature (H, "sep-length"); + new_feature (H, "yfrac-length"); + new_feature (H, "ex-length"); + new_feature (H, "fn-length"); + new_feature (H, "fns-length"); + new_feature (H, "bls-length"); + new_feature (H, "spc-length"); + new_feature (H, "xspc-length"); + new_feature (H, "par-length"); + new_feature (H, "pag-length"); + new_feature (H, "tmpt-length"); + new_feature (H, "px-length"); + new_feature (H, "tmlen"); + + if (version_inf ("1.0.3.12", version)) return H; + + new_feature (H, "unquote*"); + + if (version_inf ("1.0.3.4", version)) return H; + + new_feature (H, "for-each"); + new_feature (H, "quasi"); + rename_feature (H, "hold", "quasiquote"); + rename_feature (H, "release", "unquote"); + + if (version_inf ("1.0.3.3", version)) return H; + + new_feature (H, "quote-value"); + new_feature (H, "quote-arg"); + new_feature (H, "mark"); + new_feature (H, "use-package"); + new_feature (H, "style-only"); + new_feature (H, "style-only*"); + new_feature (H, "rewrite-inactive"); + new_feature (H, "inline-tag"); + new_feature (H, "open-tag"); + new_feature (H, "middle-tag"); + new_feature (H, "close-tag"); + + if (version_inf ("1.0.2.8", version)) return H; + + rename_feature (H, "raw_data", "raw-data"); + rename_feature (H, "sub_table", "subtable"); + rename_feature (H, "drd_props", "drd-props"); + rename_feature (H, "get_label", "get-label"); + rename_feature (H, "get_arity", "get-arity"); + rename_feature (H, "map_args", "map-args"); + rename_feature (H, "eval_args", "eval-args"); + rename_feature (H, "find_file", "find-file"); + rename_feature (H, "is_tuple", "is-tuple"); + rename_feature (H, "look_up", "look-up"); + rename_feature (H, "var_if", "if*"); + rename_feature (H, "var_inactive", "inactive*"); + rename_feature (H, "var_active", "active*"); + rename_feature (H, "text_at", "text-at"); + rename_feature (H, "var_spline", "spline*"); + rename_feature (H, "old_matrix", "old-matrix"); + rename_feature (H, "old_table", "old-table"); + rename_feature (H, "old_mosaic", "old-mosaic"); + rename_feature (H, "old_mosaic_item", "old-mosaic-item"); + rename_feature (H, "var_expand", "expand*"); + rename_feature (H, "hide_expand", "hide-expand"); + + rename_feature (H, "with_limits", "with-limits"); + rename_feature (H, "line_break", "line-break"); + rename_feature (H, "new_line", "new-line"); + rename_feature (H, "line_separator", "line-sep"); + rename_feature (H, "next_line", "next-line"); + rename_feature (H, "no_line_break", "no-break"); + rename_feature (H, "no_first_indentation", "no-indent"); + rename_feature (H, "enable_first_indentation", "yes-indent"); + rename_feature (H, "no_indentation_after", "no-indent*"); + rename_feature (H, "enable_indentation_after", "yes-indent*"); + rename_feature (H, "page_break_before", "page-break*"); + rename_feature (H, "page_break", "page-break"); + rename_feature (H, "no_page_break_before", "no-page-break*"); + rename_feature (H, "no_page_break_after", "no-page-break"); + rename_feature (H, "new_page_before", "new-page*"); + rename_feature (H, "new_page", "new-page"); + rename_feature (H, "new_double_page_before", "new-dpage*"); + rename_feature (H, "new_double_page", "new-dpage"); + + if (version_inf ("1.0.2.5", version)) return H; + + new_feature (H, "compound"); + new_feature (H, "xmacro"); + new_feature (H, "get_label"); + new_feature (H, "get_arity"); + new_feature (H, "map_args"); + new_feature (H, "eval_args"); + new_feature (H, "drd_props"); + + if (version_inf ("1.0.2.0", version)) return H; + + new_feature (H, "with_limits"); + new_feature (H, "line_break"); + new_feature (H, "new_line"); + new_feature (H, "line_separator"); + new_feature (H, "next_line"); + new_feature (H, "no_line_break"); + new_feature (H, "no_first_indentation"); + new_feature (H, "enable_first_indentation"); + new_feature (H, "no_indentation_after"); + new_feature (H, "enable_indentation_after"); + new_feature (H, "page_break_before"); + new_feature (H, "page_break"); + new_feature (H, "no_page_break_before"); + new_feature (H, "no_page_break_after"); + new_feature (H, "new_page_before"); + new_feature (H, "new_page"); + new_feature (H, "new_double_page_before"); + new_feature (H, "new_double_page"); + + if (version_inf ("1.0.1.25", version)) return H; + + new_feature (H, "active"); + new_feature (H, "var_inactive"); + new_feature (H, "var_active"); + new_feature (H, "attr"); + + if (version_inf ("1.0.0.20", version)) return H; + + new_feature (H, "text_at"); + + if (version_inf ("1.0.0.19", version)) return H; + + new_feature (H, "find_file"); + + if (version_inf ("1.0.0.14", version)) return H; + + rename_feature (H, "paragraph", "para"); + + if (version_inf ("1.0.0.5", version)) return H; + + new_feature (H, "var_if"); + new_feature (H, "hide_expand"); + + if (version_inf ("1.0.0.2", version)) return H; + + new_feature (H, "superpose"); + new_feature (H, "spline"); + new_feature (H, "var_spline"); + + return H; +} + +/****************************************************************************** + * Conversion of TMU strings of the present format to TeXmacs trees + ******************************************************************************/ + +struct tmu_reader { + string version; // document was composed using this version + hashmap codes; // codes for to present version + string buf; // the string being read from + int pos; // the current position of the reader + string last; // last read string + + tmu_reader (string buf2) + : version (XMACS_VERSION), codes (STD_CODE), buf (buf2), pos (0), + last ("") {} + tmu_reader (string buf2, string version2) + : version (version2), codes (get_codes (version)), buf (buf2), pos (0), + last ("") {} + + int skip_blank (); + string decode (string s); + string read_char (); + string read_next (); + string read_function_name (); + tree read_apply (string s, bool skip_flag); + tree read (bool skip_flag); +}; + +int +tmu_reader::skip_blank () { + int n= 0, buf_N= N (buf); + for (; pos < buf_N; pos++) { + if (buf[pos] == ' ') continue; + if (buf[pos] == '\t') continue; + if (buf[pos] == '\r') continue; + if (buf[pos] == '\n') { + n++; + continue; + } + break; + } + return n; +} + +string +tmu_reader::decode (string s) { + int i, n= N (s); + string r; + for (i= 0; i < n; i++) + if (((i + 1) < n) && (s[i] == '\\')) { + i++; + if (s[i] == ';') + ; + else if (s[i] == '\\') r << '\\'; + else r << s[i]; + } + else r << s[i]; + return r; +} + +string +tmu_reader::read_char () { + int buf_N= N (buf); + while (((pos + 1) < buf_N) && (buf[pos] == '\\') && (buf[pos + 1] == '\n')) { + pos+= 2; + skip_spaces (buf, pos); + } + if (pos >= buf_N) return ""; + + int start_pos= pos; + decode_from_utf8 (buf, pos); + return buf (start_pos, pos); +} + +string +tmu_reader::read_next () { + int buf_N = N (buf); + int old_pos= pos; + string c = read_char (); + if (c == "") return c; + switch (c[0]) { + case '\t': + case '\n': + case '\r': + case ' ': + pos--; + if (skip_blank () <= 1) return " "; + else return "\n"; + case '<': { + old_pos= pos; + c = read_char (); + if (c == "") return ""; + if (c == "#") return "<#"; + if ((c == "\\") || (c == "|") || (c == "/")) return "<" * c; + if (is_iso_alpha (c[0]) || (c == ">")) { + pos= old_pos; + return "<"; + } + pos= old_pos; + return "<"; + /* + string d= read_char (); + if ((d == "\\") || (d == "|") || (d == "/")) return "<" * c * d; + pos= old_pos; + return "<" * c; + */ + } + case '|': + case '>': + return c; + } + + string r; + pos= old_pos; + while (true) { + old_pos= pos; + c = read_char (); + if (c == "") return r; + else if (c == "\\") { + if ((pos < buf_N) && (buf[pos] == '\\')) { + r << c << "\\"; + pos++; + } + else r << c << read_char (); + } + else if (c == "\t") break; + else if (c == "\r") break; + else if (c == "\n") break; + else if (c == " ") break; + else if (c == "<") break; + else if (c == "|") break; + else if (c == ">") break; + else r << c; + } + pos= old_pos; + return r; +} + +string +tmu_reader::read_function_name () { + string name= decode (read_next ()); + // cout << "==> " << name << "\n"; + while (true) { + last= read_next (); + // cout << "~~> " << last << "\n"; + if ((last == "") || (last == "|") || (last == ">")) break; + } + return name; +} + +static void +get_collection (tree& u, tree t) { + if (is_func (t, COLLECTION) || is_func (t, DOCUMENT) || is_func (t, CONCAT)) { + for (const auto t_i : t) { + get_collection (u, t_i); + } + } + else if (is_compound (t)) u << t; +} + +tree +tmu_reader::read_apply (string name, bool skip_flag) { + // cout << "Read apply " << name << INDENT << LF; + tree t (make_tree_label (name)); + if (codes->contains (name)) { + // cout << " " << name << " -> " << as_string ((tree_label) codes [name]) + // << "\n"; + t= tree ((tree_label) codes[name]); + } + + bool closed= !skip_flag; + int buf_N = N (buf); + while (pos < buf_N) { + // cout << "last= " << last << LF; + bool sub_flag= (skip_flag) && ((last == "") || (last[N (last) - 1] != '|')); + if (sub_flag) (void) skip_blank (); + t << read (sub_flag); + if ((last == "/>") || (last == "/|")) closed= true; + if (closed && ((last == ">") || (last == "/>"))) break; + } + // cout << "last= " << last << UNINDENT << LF; + // cout << "Done" << LF; + + if (is_func (t, COLLECTION)) { + tree u (COLLECTION); + get_collection (u, t); + return u; + } + return t; +} + +static void +flush (tree& D, tree& C, string& S, bool& spc_flag, bool& ret_flag) { + if (spc_flag) S << " "; + if (S != "") { + if ((N (C) == 0) || (!is_atomic (C[N (C) - 1]))) C << S; + else C[N (C) - 1]->label << S; + S = ""; + spc_flag= false; + } + + if (ret_flag) { + if (N (C) == 0) D << ""; + else if (N (C) == 1) D << C[0]; + else D << C; + C = tree (CONCAT); + ret_flag= false; + } +} + +tree +tmu_reader::read (bool skip_flag) { + int buf_N= N (buf); + tree D (DOCUMENT); + tree C (CONCAT); + string S (""); + bool spc_flag= false; + bool ret_flag= false; + + while (true) { + last= read_next (); + // cout << "--> " << last << "\n"; + if (last == "") break; + if (last == "|") break; + if (last == ">") break; + + if (last[0] == '<') { + char tail_char_of_last= last[N (last) - 1]; + if (tail_char_of_last == '\\') { + flush (D, C, S, spc_flag, ret_flag); + string name= read_function_name (); + if (last == ">") last= "\\>"; + else last= "\\|"; + C << read_apply (name, true); + } + else if (tail_char_of_last == '|') { + (void) read_function_name (); + if (last == ">") last= "|>"; + else last= "||"; + break; + } + else if (tail_char_of_last == '/') { + (void) read_function_name (); + if (last == ">") last= "/>"; + else last= "/|"; + break; + } + else if (tail_char_of_last == '#') { + string r; + while ((buf[pos] != '>') && (pos + 2 < buf_N)) { + r << ((char) from_hex (buf (pos, pos + 2))); + pos+= 2; + } + if (buf[pos] == '>') pos++; + flush (D, C, S, spc_flag, ret_flag); + C << tree (RAW_DATA, r); + last= read_next (); + break; + } + else { + flush (D, C, S, spc_flag, ret_flag); + string name= decode (read_next ()); + string sep = ">"; + if (name == ">") name= ""; + else sep= read_next (); + // cout << "==> " << name << "\n"; + // cout << "~~> " << sep << "\n"; + if (sep == "|") { + last= "|"; + C << read_apply (name, false); + } + else { + tree t (make_tree_label (name)); + if (codes->contains (name)) { + // cout << name << " -> " << as_string ((tree_label) codes [name]) + // << "\n"; + t= tree ((tree_label) codes[name]); + } + C << t; + } + } + } + else if (last == " ") spc_flag= true; + else if (last == "\n") ret_flag= true; + else { + flush (D, C, S, spc_flag, ret_flag); + // cout << "<<< " << last << "\n"; + // cout << ">>> " << decode (last) << "\n"; + S << decode (last); + if ((S == "") && (N (C) == 0)) C << ""; + } + } + + if (skip_flag) spc_flag= ret_flag= false; + flush (D, C, S, spc_flag, ret_flag); + if (N (C) == 1) D << C[0]; + else if (N (C) > 1) D << C; + // cout << "*** " << D << "\n"; + if (N (D) == 0) return ""; + if (N (D) == 1) { + if (!skip_flag) return D[0]; + if (is_func (D[0], COLLECTION)) return D[0]; + } + return D; +} + +tree +tmu_to_tree (string s) { + tmu_reader tmr (s); + return tmr.read (true); +} + +tree +tmu_to_tree (string s, string version) { + tmu_reader tmr (s, version); + return tmr.read (true); +} + +/****************************************************************************** + * Conversion of TeXmacs trees to TMU strings + ******************************************************************************/ + +const string TMU_VERSION= "1.1.0"; + +struct tmu_writer { + string buf; // the resulting string + string spc; // "" or " " + string tmp; // not yet flushed characters + + int tab; // number of tabs after CR + bool spc_flag; // true if last printed character was a space or CR + bool ret_flag; // true if last printed character was a CR + + tmu_writer () + : buf (""), spc (""), tmp (""), tab (0), spc_flag (true), + ret_flag (true) {} + + void cr (); + void flush (); + void write_space (); + void write_return (); + void write (string s, bool flag= true, bool encode_space= false); + void br (int indent= 0); + void tag (string before, string s, string after); + void apply (string func, array args); + void write (tree t); +}; + +void +tmu_writer::cr () { + int i, n= N (buf); + for (i= n - 1; i >= 0; i--) + if ((buf[i] != ' ') || ((i > 0) && (buf[i - 1] == '\\'))) break; + if (i < n - 1) { + buf= buf (0, i + 1); + n = n - N (buf); + for (i= 0; i < n; i++) + buf << "\\ "; + } + buf << '\n'; + for (i= 0; i < min (tab, 20); i++) + buf << ' '; +} + +void +tmu_writer::flush () { + int i, m= N (spc), n= N (tmp); + if ((m + n) == 0) return; + buf << spc << tmp; + spc= ""; + tmp= ""; +} + +void +tmu_writer::write_space () { + if (spc_flag) tmp << "\\ "; + else { + flush (); + spc= " "; + } + spc_flag= true; + ret_flag= false; +} + +void +tmu_writer::write_return () { + if (ret_flag) { + buf << "\\;\n"; + cr (); + } + else { + if ((spc == " ") && (tmp == "")) { + spc= ""; + tmp= "\\ "; + } + flush (); + buf << "\n"; + cr (); + } + spc_flag= true; + ret_flag= true; +} + +void +tmu_writer::write (string s, bool flag, bool encode_space) { + if (flag) { + int i, n= N (s); + for (i= 0; i < n; i++) { + char c= s[i]; + if ((c == ' ') && (!encode_space)) write_space (); + else { + if (c == ' ') tmp << "\\ "; + else if (c == '\\') tmp << "\\\\"; + else if (c == '<') tmp << "\\<"; + else if (c == '|') tmp << "\\|"; + else if (c == '>') tmp << "\\>"; + else tmp << c; + spc_flag= false; + ret_flag= false; + } + } + } + else { + tmp << s; + if (N (s) != 0) { + spc_flag= false; + ret_flag= false; + } + } +} + +void +tmu_writer::br (int indent) { + flush (); + tab+= indent; + int i; + int buf_N= N (buf); + for (i= buf_N - 1; i >= 0; i--) { + if (buf[i] == '\n') return; + if (buf[i] != ' ') { + cr (); + spc_flag= true; + ret_flag= false; + return; + } + } +} + +void +tmu_writer::tag (string before, string s, string after) { + write (before, false); + write (s); + write (after, false); +} + +void +tmu_writer::apply (string func, array args) { + int i, last, n= N (args); + for (i= n - 1; i >= 0; i--) + if (is_document (args[i]) || is_func (args[i], COLLECTION)) break; + last= i; + + if (last >= 0) { + for (i= 0; i <= n; i++) { + bool flag= + (i < n) && (is_document (args[i]) || is_func (args[i], COLLECTION)); + if (i == 0) { + write ("<\\", false); + write (func, true, true); + } + else if (i == last + 1) { + write ("", false); + break; + } + + if (flag) { + write (">", false); + br (2); + write (args[i]); + br (-2); + } + else { + write ("|", false); + write (args[i]); + } + } + } + else { + write ("<", false); + write (func, true, true); + for (i= 0; i < n; i++) { + write ("|", false); + write (args[i]); + } + write (">", false); + } +} + +void +tmu_writer::write (tree t) { + if (is_atomic (t)) { + write (t->label); + return; + } + + int i, n= N (t); + switch (L (t)) { + case RAW_DATA: { + write ("<#", false); + string s= as_string (t[0]); + write (binary_to_hexadecimal (s), false); + write (">", false); + break; + } + case DOCUMENT: + spc_flag= true; + ret_flag= true; + for (i= 0; i < n; i++) { + write (t[i]); + if (i < (n - 1)) write_return (); + else if (ret_flag) write ("\\;", false); + } + break; + case CONCAT: + for (i= 0; i < n; i++) + write (t[i]); + break; + case EXPAND: + if ((n >= 1) && is_atomic (t[0])) { + string s= t[0]->label; + if (std_contains (s)) + ; + else if ((N (s) > 0) && (!is_iso_alpha (s))) + ; + else { + apply (s, A (t (1, n))); + break; + } + } + apply (as_string (EXPAND), A (t)); + break; + case COLLECTION: + tag ("<\\", as_string (COLLECTION), ">"); + if (n == 0) br (); + else { + br (2); + for (i= 0; i < n; i++) { + write (t[i]); + if (i < (n - 1)) br (); + } + br (-2); + } + tag (""); + break; + default: + apply (as_string (L (t)), A (t)); + break; + } +} + +string +tree_to_tmu (tree t) { + if (!is_snippet (t)) { + int t_N= N (t); + tree r (t, t_N); + for (int i= 0; i < t_N; i++) { + if (is_compound (t[i], "style", 1)) { + tree style= t[i][0]; + if (is_func (style, TUPLE, 1)) style= style[0]; + r[i] = copy (t[i]); + r[i][0]= style; + } + else if (is_compound (t[i], "TeXmacs")) { + r[i]= compound ("TMU", tuple (TMU_VERSION, string (XMACS_VERSION))); + } + else r[i]= t[i]; + } + t= r; + } + + tmu_writer tmw; + tmw.write (t); + tmw.flush (); + tmw.buf << "\n"; // append an extra newline at the end of TMU file + return tmw.buf; +} + +/****************************************************************************** + * Conversion of TMU strings to TeXmacs trees (document-level) + ******************************************************************************/ +tree +tmu_document_to_tree (string s) { + tree error (ERROR, "bad format or data"); + + if (starts (s, "'); + string version_tuple= s (N (" version_arr = tokenize (version_tuple, "|"); + string tmu_version = version_arr[0]; + string xmacs_version= version_arr[1]; + tree doc = tmu_to_tree (s, xmacs_version); + + if (is_compound (doc, "TeXmacs", 1) || is_expand (doc, "TeXmacs", 1) || + is_apply (doc, "TeXmacs", 1)) + doc= tree (DOCUMENT, doc); + + if (!is_document (doc)) return error; + + if (N (doc) == 0 || !is_compound (doc[0], "TeXmacs", 1)) { + tree d (DOCUMENT); + d << compound ("TeXmacs", string (TEXMACS_VERSION)); + d << A (doc); + doc= d; + } + + return doc; + } + return error; +} diff --git a/moebius/Data/Convert/tmu.hpp b/moebius/Data/Convert/tmu.hpp new file mode 100644 index 0000000000..979c6008b2 --- /dev/null +++ b/moebius/Data/Convert/tmu.hpp @@ -0,0 +1,32 @@ + +/****************************************************************************** + * MODULE : tmu.hpp + * DESCRIPTION: conversion between TeXmacs trees and the TMU file format + * COPYRIGHT : (C) 1999 Joris van der Hoeven + * 2024 Darcy Shen + ******************************************************************************* + * This software falls under the GNU general public license version 3 or later. + * It comes WITHOUT ANY WARRANTY WHATSOEVER. For details, see the file LICENSE + * in the root directory or . + ******************************************************************************/ + +#ifndef TMU_H +#define TMU_H + +#include "tree.hpp" + +/****************************************************************************** + * Conversion of TMU strings to TeXmacs trees + ******************************************************************************/ + +tree tmu_to_tree (string s); +tree tmu_to_tree (string s, string version); +tree tmu_document_to_tree (string s); + +/****************************************************************************** + * Conversion of TeXmacs trees to TMU strings + ******************************************************************************/ + +string tree_to_tmu (tree t); + +#endif // defined TMU_H diff --git a/moebius/Data/Tree/tree_helper.cpp b/moebius/Data/Tree/tree_helper.cpp index 02ac148ef5..6ef3f5bed5 100644 --- a/moebius/Data/Tree/tree_helper.cpp +++ b/moebius/Data/Tree/tree_helper.cpp @@ -84,6 +84,15 @@ is_document (tree t) { return L (t) == DOCUMENT; } +bool +is_snippet (tree doc) { + if (!is_document (doc)) return true; + int i, n= N (doc); + for (i= 0; i < n; i++) + if (is_compound (doc[i], "TeXmacs", 1)) return false; + return true; +} + bool is_concat (tree t) { return L (t) == CONCAT; diff --git a/moebius/Data/Tree/tree_helper.hpp b/moebius/Data/Tree/tree_helper.hpp index 7be72fb2fe..3dbdd099d0 100644 --- a/moebius/Data/Tree/tree_helper.hpp +++ b/moebius/Data/Tree/tree_helper.hpp @@ -301,6 +301,7 @@ document (tree t1, tree t2, tree t3, tree t4, tree t5) { } bool is_document (tree t); +bool is_snippet (tree doc); bool is_concat (tree t); bool is_format (tree t); bool is_formatting (tree t); diff --git a/moebius/xmake.lua b/moebius/xmake.lua index 3c3213185b..fa8af1cdc8 100644 --- a/moebius/xmake.lua +++ b/moebius/xmake.lua @@ -5,6 +5,7 @@ add_rules("mode.debug") local moe_root = os.scriptdir() local moe_files = { + path.join(moe_root, "Data/Convert/**.cpp"), path.join(moe_root, "Data/History/**.cpp"), path.join(moe_root, "Data/Tree/**.cpp"), path.join(moe_root, "Kernel/Types/**.cpp"), @@ -13,6 +14,7 @@ local moe_files = { path.join(moe_root, "moebius/**.cpp"), } local moe_includedirs = { + path.join(moe_root, "Data/Convert"), path.join(moe_root, "Data/History"), path.join(moe_root, "Data/Tree"), path.join(moe_root, "Kernel/Types"), @@ -43,6 +45,7 @@ target("libmoebius") do add_deps("liblolly") add_packages("s7") + add_headerfiles("Data/Convert/(*.hpp)") add_headerfiles("Data/History/(*.hpp)") add_headerfiles("Data/Tree/(*.hpp)") add_headerfiles("Kernel/Types/(*.hpp)") diff --git a/src/Data/Convert/Generic/generic.cpp b/src/Data/Convert/Generic/generic.cpp index 8676e05985..5f8a58beec 100644 --- a/src/Data/Convert/Generic/generic.cpp +++ b/src/Data/Convert/Generic/generic.cpp @@ -16,15 +16,6 @@ static url current_file_focus= url_none (); -bool -is_snippet (tree doc) { - if (!is_document (doc)) return true; - int i, n= N (doc); - for (i= 0; i < n; i++) - if (is_compound (doc[i], "TeXmacs", 1)) return false; - return true; -} - url get_file_focus () { return current_file_focus; diff --git a/src/Data/Convert/Mogan/from_tmu.cpp b/src/Data/Convert/Mogan/from_tmu.cpp deleted file mode 100644 index 530a85bf34..0000000000 --- a/src/Data/Convert/Mogan/from_tmu.cpp +++ /dev/null @@ -1,378 +0,0 @@ - -/****************************************************************************** - * MODULE : from_tmu.cpp - * DESCRIPTION: Convertion from the TMU format - * COPYRIGHT : (C) 1999 Joris van der Hoeven - * 2024 Darcy Shen - ******************************************************************************* - * This software falls under the GNU general public license version 3 or later. - * It comes WITHOUT ANY WARRANTY WHATSOEVER. For details, see the file LICENSE - * in the root directory or . - ******************************************************************************/ - -#include "convert.hpp" -#include "path.hpp" -#include "preferences.hpp" -#include "tree_helper.hpp" - -#include -#include -#include -#include - -using lolly::data::decode_from_utf8; -using lolly::data::from_hex; -using lolly::data::to_Hex; -using moebius::drd::STD_CODE; - -using namespace moebius; - -/****************************************************************************** - * Conversion of TeXmacs strings of the present format to TeXmacs trees - ******************************************************************************/ - -struct tmu_reader { - string version; // document was composed using this version - hashmap codes; // codes for to present version - string buf; // the string being read from - int pos; // the current position of the reader - string last; // last read string - - tmu_reader (string buf2) - : version (XMACS_VERSION), codes (STD_CODE), buf (buf2), pos (0), - last ("") {} - tmu_reader (string buf2, string version2) - : version (version2), codes (get_codes (version)), buf (buf2), pos (0), - last ("") {} - - int skip_blank (); - string decode (string s); - string read_char (); - string read_next (); - string read_function_name (); - tree read_apply (string s, bool skip_flag); - tree read (bool skip_flag); -}; - -int -tmu_reader::skip_blank () { - int n= 0, buf_N= N (buf); - for (; pos < buf_N; pos++) { - if (buf[pos] == ' ') continue; - if (buf[pos] == '\t') continue; - if (buf[pos] == '\r') continue; - if (buf[pos] == '\n') { - n++; - continue; - } - break; - } - return n; -} - -string -tmu_reader::decode (string s) { - int i, n= N (s); - string r; - for (i= 0; i < n; i++) - if (((i + 1) < n) && (s[i] == '\\')) { - i++; - if (s[i] == ';') - ; - else if (s[i] == '\\') r << '\\'; - else r << s[i]; - } - else r << s[i]; - return r; -} - -string -tmu_reader::read_char () { - int buf_N= N (buf); - while (((pos + 1) < buf_N) && (buf[pos] == '\\') && (buf[pos + 1] == '\n')) { - pos+= 2; - skip_spaces (buf, pos); - } - if (pos >= buf_N) return ""; - - int start_pos= pos; - decode_from_utf8 (buf, pos); - return buf (start_pos, pos); -} - -string -tmu_reader::read_next () { - int buf_N = N (buf); - int old_pos= pos; - string c = read_char (); - if (c == "") return c; - switch (c[0]) { - case '\t': - case '\n': - case '\r': - case ' ': - pos--; - if (skip_blank () <= 1) return " "; - else return "\n"; - case '<': { - old_pos= pos; - c = read_char (); - if (c == "") return ""; - if (c == "#") return "<#"; - if ((c == "\\") || (c == "|") || (c == "/")) return "<" * c; - if (is_iso_alpha (c[0]) || (c == ">")) { - pos= old_pos; - return "<"; - } - pos= old_pos; - return "<"; - /* - string d= read_char (); - if ((d == "\\") || (d == "|") || (d == "/")) return "<" * c * d; - pos= old_pos; - return "<" * c; - */ - } - case '|': - case '>': - return c; - } - - string r; - pos= old_pos; - while (true) { - old_pos= pos; - c = read_char (); - if (c == "") return r; - else if (c == "\\") { - if ((pos < buf_N) && (buf[pos] == '\\')) { - r << c << "\\"; - pos++; - } - else r << c << read_char (); - } - else if (c == "\t") break; - else if (c == "\r") break; - else if (c == "\n") break; - else if (c == " ") break; - else if (c == "<") break; - else if (c == "|") break; - else if (c == ">") break; - else r << c; - } - pos= old_pos; - return r; -} - -string -tmu_reader::read_function_name () { - string name= decode (read_next ()); - // cout << "==> " << name << "\n"; - while (true) { - last= read_next (); - // cout << "~~> " << last << "\n"; - if ((last == "") || (last == "|") || (last == ">")) break; - } - return name; -} - -static void -get_collection (tree& u, tree t) { - if (is_func (t, COLLECTION) || is_func (t, DOCUMENT) || is_func (t, CONCAT)) { - for (const auto t_i : t) { - get_collection (u, t_i); - } - } - else if (is_compound (t)) u << t; -} - -tree -tmu_reader::read_apply (string name, bool skip_flag) { - // cout << "Read apply " << name << INDENT << LF; - tree t (make_tree_label (name)); - if (codes->contains (name)) { - // cout << " " << name << " -> " << as_string ((tree_label) codes [name]) - // << "\n"; - t= tree ((tree_label) codes[name]); - } - - bool closed= !skip_flag; - int buf_N = N (buf); - while (pos < buf_N) { - // cout << "last= " << last << LF; - bool sub_flag= (skip_flag) && ((last == "") || (last[N (last) - 1] != '|')); - if (sub_flag) (void) skip_blank (); - t << read (sub_flag); - if ((last == "/>") || (last == "/|")) closed= true; - if (closed && ((last == ">") || (last == "/>"))) break; - } - // cout << "last= " << last << UNINDENT << LF; - // cout << "Done" << LF; - - if (is_func (t, COLLECTION)) { - tree u (COLLECTION); - get_collection (u, t); - return u; - } - return t; -} - -static void -flush (tree& D, tree& C, string& S, bool& spc_flag, bool& ret_flag) { - if (spc_flag) S << " "; - if (S != "") { - if ((N (C) == 0) || (!is_atomic (C[N (C) - 1]))) C << S; - else C[N (C) - 1]->label << S; - S = ""; - spc_flag= false; - } - - if (ret_flag) { - if (N (C) == 0) D << ""; - else if (N (C) == 1) D << C[0]; - else D << C; - C = tree (CONCAT); - ret_flag= false; - } -} - -tree -tmu_reader::read (bool skip_flag) { - int buf_N= N (buf); - tree D (DOCUMENT); - tree C (CONCAT); - string S (""); - bool spc_flag= false; - bool ret_flag= false; - - while (true) { - last= read_next (); - // cout << "--> " << last << "\n"; - if (last == "") break; - if (last == "|") break; - if (last == ">") break; - - if (last[0] == '<') { - char tail_char_of_last= last[N (last) - 1]; - if (tail_char_of_last == '\\') { - flush (D, C, S, spc_flag, ret_flag); - string name= read_function_name (); - if (last == ">") last= "\\>"; - else last= "\\|"; - C << read_apply (name, true); - } - else if (tail_char_of_last == '|') { - (void) read_function_name (); - if (last == ">") last= "|>"; - else last= "||"; - break; - } - else if (tail_char_of_last == '/') { - (void) read_function_name (); - if (last == ">") last= "/>"; - else last= "/|"; - break; - } - else if (tail_char_of_last == '#') { - string r; - while ((buf[pos] != '>') && (pos + 2 < buf_N)) { - r << ((char) from_hex (buf (pos, pos + 2))); - pos+= 2; - } - if (buf[pos] == '>') pos++; - flush (D, C, S, spc_flag, ret_flag); - C << tree (RAW_DATA, r); - last= read_next (); - break; - } - else { - flush (D, C, S, spc_flag, ret_flag); - string name= decode (read_next ()); - string sep = ">"; - if (name == ">") name= ""; - else sep= read_next (); - // cout << "==> " << name << "\n"; - // cout << "~~> " << sep << "\n"; - if (sep == "|") { - last= "|"; - C << read_apply (name, false); - } - else { - tree t (make_tree_label (name)); - if (codes->contains (name)) { - // cout << name << " -> " << as_string ((tree_label) codes [name]) - // << "\n"; - t= tree ((tree_label) codes[name]); - } - C << t; - } - } - } - else if (last == " ") spc_flag= true; - else if (last == "\n") ret_flag= true; - else { - flush (D, C, S, spc_flag, ret_flag); - // cout << "<<< " << last << "\n"; - // cout << ">>> " << decode (last) << "\n"; - S << decode (last); - if ((S == "") && (N (C) == 0)) C << ""; - } - } - - if (skip_flag) spc_flag= ret_flag= false; - flush (D, C, S, spc_flag, ret_flag); - if (N (C) == 1) D << C[0]; - else if (N (C) > 1) D << C; - // cout << "*** " << D << "\n"; - if (N (D) == 0) return ""; - if (N (D) == 1) { - if (!skip_flag) return D[0]; - if (is_func (D[0], COLLECTION)) return D[0]; - } - return D; -} - -tree -tmu_to_tree (string s) { - tmu_reader tmr (s); - return tmr.read (true); -} - -tree -tmu_to_tree (string s, string version) { - tmu_reader tmr (s, version); - return tmr.read (true); -} - -/****************************************************************************** - * Conversion of TeXmacs strings to TeXmacs trees - ******************************************************************************/ -tree -tmu_document_to_tree (string s) { - tree error (ERROR, "bad format or data"); - - if (starts (s, "'); - string version_tuple= s (N (" version_arr = tokenize (version_tuple, "|"); - string tmu_version = version_arr[0]; - string xmacs_version= version_arr[1]; - tree doc = tmu_to_tree (s, xmacs_version); - - if (is_compound (doc, "TeXmacs", 1) || is_expand (doc, "TeXmacs", 1) || - is_apply (doc, "TeXmacs", 1)) - doc= tree (DOCUMENT, doc); - - if (!is_document (doc)) return error; - - if (N (doc) == 0 || !is_compound (doc[0], "TeXmacs", 1)) { - tree d (DOCUMENT); - d << compound ("TeXmacs", string (TEXMACS_VERSION)); - d << A (doc); - doc= d; - } - - return doc; - } - return error; -} diff --git a/src/Data/Convert/Mogan/to_tmu.cpp b/src/Data/Convert/Mogan/to_tmu.cpp deleted file mode 100644 index d0cb1140fd..0000000000 --- a/src/Data/Convert/Mogan/to_tmu.cpp +++ /dev/null @@ -1,302 +0,0 @@ - -/****************************************************************************** - * MODULE : to_tmu.cpp - * DESCRIPTION: conversion of TeXmacs trees to the TMU file format - * COPYRIGHT : (C) 1999 Joris van der Hoeven - * 2024 Darcy Shen - ******************************************************************************* - * This software falls under the GNU general public license version 3 or later. - * It comes WITHOUT ANY WARRANTY WHATSOEVER. For details, see the file LICENSE - * in the root directory or . - ******************************************************************************/ - -#include "convert.hpp" -#include "tree_helper.hpp" - -#include -#include - -using namespace moebius; -using lolly::data::binary_to_hexadecimal; -using moebius::drd::std_contains; - -const string TMU_VERSION= "1.1.0"; - -/****************************************************************************** - * Conversion of TeXmacs trees to the present TeXmacs string format - ******************************************************************************/ - -struct tmu_writer { - string buf; // the resulting string - string spc; // "" or " " - string tmp; // not yet flushed characters - - int tab; // number of tabs after CR - bool spc_flag; // true if last printed character was a space or CR - bool ret_flag; // true if last printed character was a CR - - tmu_writer () - : buf (""), spc (""), tmp (""), tab (0), spc_flag (true), - ret_flag (true) {} - - void cr (); - void flush (); - void write_space (); - void write_return (); - void write (string s, bool flag= true, bool encode_space= false); - void br (int indent= 0); - void tag (string before, string s, string after); - void apply (string func, array args); - void write (tree t); -}; - -void -tmu_writer::cr () { - int i, n= N (buf); - for (i= n - 1; i >= 0; i--) - if ((buf[i] != ' ') || ((i > 0) && (buf[i - 1] == '\\'))) break; - if (i < n - 1) { - buf= buf (0, i + 1); - n = n - N (buf); - for (i= 0; i < n; i++) - buf << "\\ "; - } - buf << '\n'; - for (i= 0; i < min (tab, 20); i++) - buf << ' '; -} - -void -tmu_writer::flush () { - int i, m= N (spc), n= N (tmp); - if ((m + n) == 0) return; - buf << spc << tmp; - spc= ""; - tmp= ""; -} - -void -tmu_writer::write_space () { - if (spc_flag) tmp << "\\ "; - else { - flush (); - spc= " "; - } - spc_flag= true; - ret_flag= false; -} - -void -tmu_writer::write_return () { - if (ret_flag) { - buf << "\\;\n"; - cr (); - } - else { - if ((spc == " ") && (tmp == "")) { - spc= ""; - tmp= "\\ "; - } - flush (); - buf << "\n"; - cr (); - } - spc_flag= true; - ret_flag= true; -} - -void -tmu_writer::write (string s, bool flag, bool encode_space) { - if (flag) { - int i, n= N (s); - for (i= 0; i < n; i++) { - char c= s[i]; - if ((c == ' ') && (!encode_space)) write_space (); - else { - if (c == ' ') tmp << "\\ "; - else if (c == '\\') tmp << "\\\\"; - else if (c == '<') tmp << "\\<"; - else if (c == '|') tmp << "\\|"; - else if (c == '>') tmp << "\\>"; - else tmp << c; - spc_flag= false; - ret_flag= false; - } - } - } - else { - tmp << s; - if (N (s) != 0) { - spc_flag= false; - ret_flag= false; - } - } -} - -void -tmu_writer::br (int indent) { - flush (); - tab+= indent; - int i; - int buf_N= N (buf); - for (i= buf_N - 1; i >= 0; i--) { - if (buf[i] == '\n') return; - if (buf[i] != ' ') { - cr (); - spc_flag= true; - ret_flag= false; - return; - } - } -} - -void -tmu_writer::tag (string before, string s, string after) { - write (before, false); - write (s); - write (after, false); -} - -void -tmu_writer::apply (string func, array args) { - int i, last, n= N (args); - for (i= n - 1; i >= 0; i--) - if (is_document (args[i]) || is_func (args[i], COLLECTION)) break; - last= i; - - if (last >= 0) { - for (i= 0; i <= n; i++) { - bool flag= - (i < n) && (is_document (args[i]) || is_func (args[i], COLLECTION)); - if (i == 0) { - write ("<\\", false); - write (func, true, true); - } - else if (i == last + 1) { - write ("", false); - break; - } - - if (flag) { - write (">", false); - br (2); - write (args[i]); - br (-2); - } - else { - write ("|", false); - write (args[i]); - } - } - } - else { - write ("<", false); - write (func, true, true); - for (i= 0; i < n; i++) { - write ("|", false); - write (args[i]); - } - write (">", false); - } -} - -void -tmu_writer::write (tree t) { - if (is_atomic (t)) { - write (t->label); - return; - } - - int i, n= N (t); - switch (L (t)) { - case RAW_DATA: { - write ("<#", false); - string s= as_string (t[0]); - write (binary_to_hexadecimal (s), false); - write (">", false); - break; - } - case DOCUMENT: - spc_flag= true; - ret_flag= true; - for (i= 0; i < n; i++) { - write (t[i]); - if (i < (n - 1)) write_return (); - else if (ret_flag) write ("\\;", false); - } - break; - case CONCAT: - for (i= 0; i < n; i++) - write (t[i]); - break; - case EXPAND: - if ((n >= 1) && is_atomic (t[0])) { - string s= t[0]->label; - if (std_contains (s)) - ; - else if ((N (s) > 0) && (!is_iso_alpha (s))) - ; - else { - apply (s, A (t (1, n))); - break; - } - } - apply (as_string (EXPAND), A (t)); - break; - case COLLECTION: - tag ("<\\", as_string (COLLECTION), ">"); - if (n == 0) br (); - else { - br (2); - for (i= 0; i < n; i++) { - write (t[i]); - if (i < (n - 1)) br (); - } - br (-2); - } - tag (""); - break; - default: - apply (as_string (L (t)), A (t)); - break; - } -} - -/****************************************************************************** - * Conversion of TeXmacs trees to TeXmacs strings - ******************************************************************************/ - -string -tree_to_tmu (tree t) { - if (!is_snippet (t)) { - int t_N= N (t); - tree r (t, t_N); - for (int i= 0; i < t_N; i++) { - if (is_compound (t[i], "style", 1)) { - tree style= t[i][0]; - if (is_func (style, TUPLE, 1)) style= style[0]; - r[i] = copy (t[i]); - r[i][0]= style; - } - else if (is_compound (t[i], "TeXmacs")) { - r[i]= compound ("TMU", tuple (TMU_VERSION, string (XMACS_VERSION))); - } - else r[i]= t[i]; - } - t= r; - } - - tmu_writer tmw; - tmw.write (t); - tmw.flush (); - tmw.buf << "\n"; // append an extra newline at the end of TMU file - return tmw.buf; -} diff --git a/src/Data/Convert/convert.hpp b/src/Data/Convert/convert.hpp index 5e3db2c3b9..50942a7417 100644 --- a/src/Data/Convert/convert.hpp +++ b/src/Data/Convert/convert.hpp @@ -19,7 +19,6 @@ class object; /*** Miscellaneous ***/ -bool is_snippet (tree doc); void set_file_focus (url u); url get_file_focus (); @@ -54,9 +53,7 @@ tree eqnumber_to_nonumber (tree t); string search_metadata (tree doc, string kind); /*** TMU ***/ -tree tmu_to_tree (string s); -tree tmu_document_to_tree (string s); -string tree_to_tmu (tree t); +#include "tmu.hpp" /*** Verbatim ***/ string tree_to_verbatim (tree t, bool wrap= false, string enc= "default");