Add configuration with include/exclude options (#97)

* Add configuration with include/exclude options

* Create changeset

* Make config file path optional

.changeset/rude-pants-add.md

@@ -0,0 +1,5 @@
+---
+"mddb": minor
+---
+
+- Add configuration with include/exclude options

markdowndb.config.js

@@ -0,0 +1,8 @@
+export default {
+    computedFields: [], // Array of functions to computed fields
+    schemas: {
+        // Add zod schemas
+    },
+    include: [], // Pattern for files to be included
+    exclude: [] // Patten for files to be excluded
+};

package.json

@@ -45,9 +45,11 @@
   "types": "./dist/src/index.d.ts",
   "dependencies": {
     "@portaljs/remark-wiki-link": "^1.0.4",
+    "@types/micromatch": "^4.0.6",
     "chokidar": "^3.5.3",
     "gray-matter": "^4.0.3",
     "knex": "^2.4.2",
+    "micromatch": "^4.0.5",
     "react-markdown": "^9.0.1",
     "remark-gfm": "^3.0.1",
     "remark-parse": "^10.0.1",

src/bin/index.js

@@ -1,17 +1,27 @@
 #!/usr/bin/env node
-
 import { MarkdownDB } from "../lib/markdowndb.js";
 
 // TODO get these from markdowndb.config.js or something
 const dbPath = "markdown.db";
 const ignorePatterns = [/Excalidraw/, /\.obsidian/, /DS_Store/];
-const [contentPath, watchFlag] = process.argv.slice(2);
 
-if (!contentPath) {
-  throw new Error("Invalid/Missing path to markdown content folder");
+let watchFlag;
+const args = process.argv.slice(2);
+
+// Check for the watch flag and its position
+const watchIndex = args.indexOf('--watch');
+if (watchIndex !== -1) {
+  watchFlag = args[watchIndex];
+  args.splice(watchIndex, 1); // Remove the watch flag from the array
 }
 
-const watchEnabled = watchFlag && watchFlag === "--watch";
+// Assign values to contentPath and configFilePath based on their positions
+const [contentPath, configFilePath] = args;
+
+if (!contentPath) {
+  console.error('Invalid/Missing path to markdown content folder');
+  process.exit(1);
+}
 
 const client = new MarkdownDB({
   client: "sqlite3",
@@ -25,9 +35,10 @@ await client.init();
 await client.indexFolder({
   folderPath: contentPath,
   ignorePatterns: ignorePatterns,
-  watch: watchEnabled,
+  watch: watchFlag,
+  configFilePath: configFilePath
 });
 
-if (!watchEnabled) {
+if (!watchFlag) {
   process.exit();
 }

src/lib/CustomConfig.ts

@@ -8,4 +8,6 @@ type Schemas = { [index: string]: ZodObject<any> };
 export interface CustomConfig {
   computedFields?: ComputedFields;
   schemas?: Schemas;
+  include?: string[];
+  exclude?: string[];
 }

src/lib/indexFolder.ts

@@ -2,6 +2,7 @@ import { ZodError } from "zod";
 import { CustomConfig } from "./CustomConfig.js";
 import { FileInfo, processFile } from "./process.js";
 import { recursiveWalkDir } from "./recursiveWalkDir.js";
+import micromatch from "micromatch";
 
 export function indexFolder(
   folderPath: string,
@@ -11,7 +12,12 @@ export function indexFolder(
 ) {
   const filePathsToIndex = recursiveWalkDir(folderPath);
   const filteredFilePathsToIndex = filePathsToIndex.filter((filePath) =>
-    shouldIncludeFile(filePath, ignorePatterns)
+    shouldIncludeFile({
+      filePath,
+      ignorePatterns,
+      includeGlob: config.include,
+      excludeGlob: config.exclude,
+    })
   );
   const files: FileInfo[] = [];
   const computedFields = config.computedFields || [];
@@ -56,11 +62,46 @@ export function indexFolder(
   return files;
 }
 
-export function shouldIncludeFile(
-  filePath: string,
-  ignorePatterns?: RegExp[]
-): boolean {
-  return !(
-    ignorePatterns && ignorePatterns.some((pattern) => pattern.test(filePath))
-  );
+export function shouldIncludeFile({
+  filePath,
+  ignorePatterns,
+  includeGlob,
+  excludeGlob,
+}: {
+  filePath: string;
+  ignorePatterns?: RegExp[];
+  includeGlob?: string[];
+  excludeGlob?: string[];
+}): boolean {
+  const normalizedFilePath = filePath.replace(/\\/g, "/");
+
+  if (
+    ignorePatterns &&
+    ignorePatterns.some((pattern) => pattern.test(normalizedFilePath))
+  ) {
+    return false;
+  }
+
+  // Check if the file should be included based on includeGlob
+  if (
+    includeGlob &&
+    includeGlob.length > 0 &&
+    !includeGlob.some((pattern) =>
+      micromatch.isMatch(normalizedFilePath, pattern)
+    )
+  ) {
+    return false;
+  }
+
+  // Check if the file should be excluded based on excludeGlob
+  if (
+    excludeGlob &&
+    excludeGlob.some((pattern) =>
+      micromatch.isMatch(normalizedFilePath, pattern)
+    )
+  ) {
+    return false;
+  }
+
+  return true;
 }

src/lib/loadConfig.ts

@@ -0,0 +1,18 @@
+import * as path from "path";
+
+export async function loadConfig(configFilePath?: string) {
+  const normalizedPath = path.resolve(configFilePath || "markdowndb.config.js");
+  const fileUrl = new URL(`file://${normalizedPath}`);
+
+  try {
+    // Import the module using the file URL
+    const configModule = await import(fileUrl.href);
+    return configModule.default;
+  } catch (error) {
+    if (configFilePath) {
+      throw new Error(
+        `Error loading configuration file from ${normalizedPath}`
+      );
+    }
+  }
+}

src/lib/markdowndb.ts

@@ -17,6 +17,7 @@ import { CustomConfig } from "./CustomConfig.js";
 import { FileInfo, processFile } from "./process.js";
 import chokidar from "chokidar";
 import { recursiveWalkDir } from "./recursiveWalkDir.js";
+import { loadConfig } from "./loadConfig.js";
 
 const defaultFilePathToUrl = (filePath: string) => {
   let url = filePath
@@ -77,19 +78,22 @@ export class MarkdownDB {
     // TODO support glob patterns
     ignorePatterns = [],
     pathToUrlResolver = defaultFilePathToUrl,
-    customConfig = {},
+    customConfig,
     watch = false,
+    configFilePath,
   }: {
     folderPath: string;
     ignorePatterns?: RegExp[];
     pathToUrlResolver?: (filePath: string) => string;
     customConfig?: CustomConfig;
     watch?: boolean;
+    configFilePath?: string;
   }) {
+    const config = customConfig || (await loadConfig(configFilePath)) || {};
     const fileObjects = indexFolder(
       folderPath,
       pathToUrlResolver,
-      customConfig,
+      config,
       ignorePatterns
     );
     await this.saveDataToDisk(fileObjects);
@@ -100,10 +104,17 @@ export class MarkdownDB {
       });
 
       const filePathsToIndex = recursiveWalkDir(folderPath);
-      const computedFields = customConfig.computedFields || [];
+      const computedFields = config.computedFields || [];
 
       const handleFileEvent = (event: string, filePath: string) => {
-        if (!shouldIncludeFile(filePath, ignorePatterns)) {
+        if (
+          !shouldIncludeFile({
+            filePath,
+            ignorePatterns,
+            includeGlob: config.include,
+            excludeGlob: config.exclude,
+          })
+        ) {
           return;
         }