Merge branch 'develop' of digital.github.com:digital-go-jp/abr-geocod…

…er into develop
digital-go-jp · Oct 30, 2024 · 0daf23f · 0daf23f
2 parents c164224 + 9d93819
commit 0daf23f
Show file tree

Hide file tree

Showing 13 changed files with 82 additions and 24 deletions.
diff --git a/package.json b/package.json
@@ -56,6 +56,7 @@
     "execa-cjs": "^9.1.1",
     "jest": "^29.7.0",
     "jest-junit": "^16.0.0",
+    "rimraf": "^6.0.1",
     "ts-add-js-extension": "^1.6.4",
     "ts-jest": "^29.2.5",
     "ts-node": "^10.9.2",
@@ -77,7 +78,6 @@
     "node-gyp": "^10.2.0",
     "proj4": "^2.12.1",
     "range-parser": "^1.2.1",
-    "rimraf": "^6.0.1",
     "string-hash": "^1.1.3",
     "unzipper": "^0.11.6",
     "winston": "^3.14.2",

diff --git a/src/domain/services/remove-files.ts b/src/domain/services/remove-files.ts
@@ -0,0 +1,32 @@
+import path from 'node:path';
+import fs from 'node:fs';
+import { RegExpEx } from "@domain/services/reg-exp-ex";
+
+export const removeFiles = async (params: Required<{
+    dir: string;
+    filename: string | RegExp;
+}>) => {
+    const filePattern: RegExp = (() => {
+        if (typeof params.filename === 'string') {
+            return RegExpEx.create(`^${params.filename}$`);
+        } else {
+            return params.filename;
+        }
+    })();
+
+    // 古いキャッシュファイルを削除
+    const dir = await fs.promises.opendir(params.dir);
+    const removedFiles: string[] = [];
+    for await (const dirent  of dir) {
+        if (!filePattern.test(dirent.name)) {
+            continue;
+        }
+        removedFiles.push(dirent.name);
+        try {
+            await fs.promises.unlink(path.join(dirent.parentPath, dirent.name));
+        } catch (e: unknown) {
+            // Do nothing here
+        }
+    }
+    return removedFiles;
+};
diff --git a/src/interface/cli/commands/geocode-command.ts b/src/interface/cli/commands/geocode-command.ts
@@ -239,9 +239,6 @@ const geocodeCommand: CommandModule = {
 
     // 合計のリクエスト数をセット
     if (progressBar) {
-      progressBar.update(0, {
-        message: 'geocoding...',
-      });
       progressBar?.setTotal(total);
     }
 
@@ -284,7 +281,9 @@ const geocodeCommand: CommandModule = {
     const streamCounter = new StreamCounter({
       fps: 10,
       callback(current) {
-        progressBar?.update(current);
+        progressBar?.update(current, {
+          message: 'geocoding...',
+        });
       },
     });
     await streamPromises.pipeline(

diff --git a/src/usecases/download/download-process.ts b/src/usecases/download/download-process.ts
@@ -110,7 +110,7 @@ export class Downloader {
 
     // ダウンロード処理を行う
     // 最大6コネクション
-    const numOfDownloadThreads = Math.max(Math.floor(parallel / 3 * 1), 1);
+    const numOfDownloadThreads = Math.min(Math.max(Math.floor(parallel / 3 * 2), 1), 6);
     const downloadTransform = new DownloadTransform({
       container: this.container,
       maxConcurrency: numOfDownloadThreads,

diff --git a/src/usecases/geocode/models/city-and-ward-trie-finder.ts b/src/usecases/geocode/models/city-and-ward-trie-finder.ts
@@ -7,7 +7,7 @@ import { kan2num } from '../services/kan2num';
 import { toHiragana } from '../services/to-hiragana';
 import { AbrGeocoderDiContainer } from './abr-geocoder-di-container';
 import { TrieAddressFinder } from "./trie/trie-finder";
-import { rimraf } from "rimraf";
+import { removeFiles } from "@domain/services/remove-files";
 
 export class CityAndWardTrieFinder extends TrieAddressFinder<CityMatchingInfo> {
 
@@ -51,7 +51,10 @@ export class CityAndWardTrieFinder extends TrieAddressFinder<CityMatchingInfo> {
     }
 
     // 古いキャッシュファイルを削除
-    await rimraf(`${path.join(cacheDir, 'city-and-ward_*.v8')}`);
+    await removeFiles({
+      dir: cacheDir,
+      filename: 'city-and-ward_.*\.v8'
+    });
 
     // キャッシュがなければ、Databaseからデータをロードして読み込む
     // キャッシュファイルも作成する

diff --git a/src/usecases/geocode/models/county-and-city-trie-finder.ts b/src/usecases/geocode/models/county-and-city-trie-finder.ts
@@ -7,7 +7,7 @@ import { kan2num } from '../services/kan2num';
 import { toHiragana } from '../services/to-hiragana';
 import { AbrGeocoderDiContainer } from './abr-geocoder-di-container';
 import { TrieAddressFinder } from "./trie/trie-finder";
-import { rimraf } from "rimraf";
+import { removeFiles } from "@domain/services/remove-files";
 
 export class CountyAndCityTrieFinder extends TrieAddressFinder<CityMatchingInfo> {
 
@@ -50,7 +50,10 @@ export class CountyAndCityTrieFinder extends TrieAddressFinder<CityMatchingInfo>
     }
 
     // 古いキャッシュファイルを削除
-    await rimraf(`${path.join(diContainer.cacheDir, 'county-and-city_*.v8')}`);
+    await removeFiles({
+      dir: diContainer.cacheDir,
+      filename: 'county-and-city_.*\.v8'
+    });
 
     // キャッシュがなければ、Databaseからデータをロードして読み込む
     // キャッシュファイルも作成する

diff --git a/src/usecases/geocode/models/kyoto-street-trie-finder.ts b/src/usecases/geocode/models/kyoto-street-trie-finder.ts
@@ -11,7 +11,7 @@ import { toHiragana } from '../services/to-hiragana';
 import { AbrGeocoderDiContainer } from './abr-geocoder-di-container';
 import { TrieAddressFinder } from "./trie/trie-finder";
 import { MatchLevel } from "@domain/types/geocode/match-level";
-import { rimraf } from "rimraf";
+import { removeFiles } from "@domain/services/remove-files";
 
 export class KyotoStreetTrieFinder extends TrieAddressFinder<KoazaMachingInfo> {
 
@@ -73,7 +73,10 @@ export class KyotoStreetTrieFinder extends TrieAddressFinder<KoazaMachingInfo> {
     }
 
     // 古いキャッシュファイルを削除
-    await rimraf(`${path.join(diContainer.cacheDir, 'kyoto-street_*.v8')}`);
+    await removeFiles({
+      dir: diContainer.cacheDir,
+      filename: 'kyoto-street_.*\.v8'
+    });
 
     // キャッシュがなければ、Databaseからデータをロードして読み込む
     // キャッシュファイルも作成する

diff --git a/src/usecases/geocode/models/oaza-cho-trie-finder.ts b/src/usecases/geocode/models/oaza-cho-trie-finder.ts
@@ -4,14 +4,14 @@ import { RegExpEx } from "@domain/services/reg-exp-ex";
 import { OazaChoMachingInfo } from "@domain/types/geocode/oaza-cho-info";
 import fs from 'node:fs';
 import path from 'node:path';
-import { rimraf } from "rimraf";
 import { jisKanji } from '../services/jis-kanji';
 import { kan2num } from '../services/kan2num';
 import { toHankakuAlphaNum } from "../services/to-hankaku-alpha-num";
 import { toHiragana } from '../services/to-hiragana';
 import { AbrGeocoderDiContainer } from './abr-geocoder-di-container';
 import { CharNode } from "./trie/char-node";
 import { TrieAddressFinder } from "./trie/trie-finder";
+import { removeFiles } from "@domain/services/remove-files";
 
 export class OazaChoTrieFinder extends TrieAddressFinder<OazaChoMachingInfo> {
 
@@ -107,7 +107,10 @@ export class OazaChoTrieFinder extends TrieAddressFinder<OazaChoMachingInfo> {
       // ここではエラーを殺すだけで良い
     }
     // 古いキャッシュファイルを削除
-    await rimraf(`${path.join(diContainer.cacheDir, 'oaza-cho_*.v8')}`);
+    await removeFiles({
+      dir: diContainer.cacheDir,
+      filename: 'oaza-cho_.*\.v8'
+    });
 
     // キャッシュがなければ、Databaseからデータをロードして読み込む
     // キャッシュファイルも作成する

diff --git a/src/usecases/geocode/models/pref-trie-finder.ts b/src/usecases/geocode/models/pref-trie-finder.ts
@@ -7,7 +7,7 @@ import { kan2num } from '../services/kan2num';
 import { toHiragana } from '../services/to-hiragana';
 import { AbrGeocoderDiContainer } from './abr-geocoder-di-container';
 import { TrieAddressFinder } from "./trie/trie-finder";
-import { rimraf } from "rimraf";
+import { removeFiles } from "@domain/services/remove-files";
 
 export class PrefTrieFinder extends TrieAddressFinder<PrefInfo> {
 
@@ -50,7 +50,10 @@ export class PrefTrieFinder extends TrieAddressFinder<PrefInfo> {
     }
 
     // 古いキャッシュファイルを削除
-    await rimraf(`${path.join(diContainer.cacheDir, 'pref_*.v8')}`);
+    await removeFiles({
+      dir: diContainer.cacheDir,
+      filename: 'pref_.*\.v8'
+    });
 
     // キャッシュがなければ、Databaseからデータをロードして読み込む
     // キャッシュファイルも作成する

diff --git a/src/usecases/geocode/models/tokyo23-town-finder.ts b/src/usecases/geocode/models/tokyo23-town-finder.ts
@@ -9,7 +9,7 @@ import { kan2num } from '../services/kan2num';
 import { toHiragana } from '../services/to-hiragana';
 import { AbrGeocoderDiContainer } from './abr-geocoder-di-container';
 import { TrieAddressFinder } from "./trie/trie-finder";
-import { rimraf } from "rimraf";
+import { removeFiles } from "@domain/services/remove-files";
 
 export class Tokyo23TownTrieFinder extends TrieAddressFinder<TownMatchingInfo> {
 
@@ -55,7 +55,10 @@ export class Tokyo23TownTrieFinder extends TrieAddressFinder<TownMatchingInfo> {
     }
 
     // 古いキャッシュファイルを削除
-    await rimraf(`${path.join(diContainer.cacheDir, 'tokyo23-town_*.v8')}`);
+    await removeFiles({
+      dir: diContainer.cacheDir,
+      filename: 'tokyo23-town_.*\.v8'
+    });
 
     // キャッシュがなければ、Databaseからデータをロードして読み込む
     // キャッシュファイルも作成する

diff --git a/src/usecases/geocode/models/tokyo23-ward-trie-finder.ts b/src/usecases/geocode/models/tokyo23-ward-trie-finder.ts
@@ -9,7 +9,7 @@ import { kan2num } from '../services/kan2num';
 import { toHiragana } from '../services/to-hiragana';
 import { AbrGeocoderDiContainer } from './abr-geocoder-di-container';
 import { TrieAddressFinder } from "./trie/trie-finder";
-import { rimraf } from "rimraf";
+import { removeFiles } from "@domain/services/remove-files";
 
 export class Tokyo23WardTrieFinder extends TrieAddressFinder<CityMatchingInfo> {
 
@@ -55,7 +55,10 @@ export class Tokyo23WardTrieFinder extends TrieAddressFinder<CityMatchingInfo> {
     }
 
     // 古いキャッシュファイルを削除
-    await rimraf(`${path.join(diContainer.cacheDir, 'tokyo23-ward_*.v8')}`);
+    await removeFiles({
+      dir: diContainer.cacheDir,
+      filename: 'tokyo23-ward_.*\.v8'
+    });
 
     // キャッシュがなければ、Databaseからデータをロードして読み込む
     // キャッシュファイルも作成する

diff --git a/src/usecases/geocode/models/ward-and-oaza-trie-finder.ts b/src/usecases/geocode/models/ward-and-oaza-trie-finder.ts
@@ -7,7 +7,7 @@ import { kan2num } from '../services/kan2num';
 import { toHiragana } from '../services/to-hiragana';
 import { AbrGeocoderDiContainer } from './abr-geocoder-di-container';
 import { TrieAddressFinder } from "./trie/trie-finder";
-import { rimraf } from "rimraf";
+import { removeFiles } from "@domain/services/remove-files";
 
 export class WardAndOazaTrieFinder extends TrieAddressFinder<WardAndOazaMatchingInfo> {
 
@@ -49,7 +49,10 @@ export class WardAndOazaTrieFinder extends TrieAddressFinder<WardAndOazaMatching
     }
 
     // 古いキャッシュファイルを削除
-    await rimraf(`${path.join(diContainer.cacheDir, 'ward-and-oaza_*.v8')}`);
+    await removeFiles({
+      dir: diContainer.cacheDir,
+      filename: 'ward-and-oaza_.*\.v8'
+    });
 
     // キャッシュがなければ、Databaseからデータをロードして読み込む
     // キャッシュファイルも作成する

diff --git a/src/usecases/geocode/models/ward-trie-finder.ts b/src/usecases/geocode/models/ward-trie-finder.ts
@@ -7,7 +7,7 @@ import { toHankakuAlphaNum } from "../services/to-hankaku-alpha-num";
 import { toHiragana } from '../services/to-hiragana';
 import { AbrGeocoderDiContainer } from './abr-geocoder-di-container';
 import { TrieAddressFinder } from "./trie/trie-finder";
-import { rimraf } from "rimraf";
+import { removeFiles } from "@domain/services/remove-files";
 
 export class WardTrieFinder extends TrieAddressFinder<WardMatchingInfo> {
 
@@ -49,7 +49,10 @@ export class WardTrieFinder extends TrieAddressFinder<WardMatchingInfo> {
     }
 
     // 古いキャッシュファイルを削除
-    await rimraf(`${path.join(diContainer.cacheDir, 'ward_*.v8')}`);
+    await removeFiles({
+      dir: diContainer.cacheDir,
+      filename: 'ward_.*\.v8'
+    });
 
     // キャッシュがなければ、Databaseからデータをロードして読み込む
     // キャッシュファイルも作成する