Merge pull request #15 from HarperFast/tests/more-and-better-unit-tests

More and better unit tests

Author: cap10morgan

.github/workflows/unit-test.yml

@@ -43,14 +43,5 @@ jobs:
       - name: Build
         run: npm run build || true # we currently have type errors so just ignore that
 
-      - name: Setup Harper
-        env:
-          DEFAULTS_MODE: 'dev'
-          HDB_ADMIN_USERNAME: 'admin'
-          HDB_ADMIN_PASSWORD: 'password'
-          ROOTPATH: '/tmp/hdb'
-          NODE_HOSTNAME: 'localhost'
-        run: node --enable-source-maps ./dist/bin/harperdb.js install
-
       - name: Run tests
         run: npm run test:unit:all

.gitignore

@@ -29,4 +29,7 @@ dist/
 !.env.example
 
 # something is generating this file :shrug:
-integration-test.json
+integration-test.json
+
+# test sandboxes
+unitTests/testEnv/

bin/copyDb.ts

@@ -4,7 +4,7 @@ import { join } from 'path';
 import { move, remove } from 'fs-extra';
 import { get } from '../utility/environment/environmentManager.js';
 import OpenEnvironmentObject from '../utility/lmdb/OpenEnvironmentObject.js';
-import { OpenDBIObject } from '../utility/lmdb/OpenDBIObject.js';
+import { OpenDBIObject } from '../utility/lmdb/OpenDBIObject.ts';
 import { INTERNAL_DBIS_NAME, AUDIT_STORE_NAME } from '../utility/lmdb/terms.js';
 import { CONFIG_PARAMS, DATABASES_DIR_NAME } from '../utility/hdbTerms.ts';
 import { AUDIT_STORE_OPTIONS } from '../resources/auditStore.ts';

config/RootConfigWatcher.ts

@@ -1,7 +1,7 @@
 import chokidar, { FSWatcher } from 'chokidar';
 import { readFile } from 'node:fs/promises';
 import { getConfigFilePath } from './configUtils.js';
-import { EventEmitter, once } from 'node:stream';
+import { EventEmitter, once } from 'node:events';
 import { parse } from 'yaml';
 
 export class RootConfigWatcher extends EventEmitter {

dataLayer/harperBridge/ResourceBridge.ts

@@ -12,11 +12,10 @@ import {
 } from '../../utility/hdbTerms.ts';
 import * as signalling from '../../utility/signalling.js';
 import { SchemaEventMsg } from '../../server/threads/itc.js';
-import { asyncSetTimeout } from '../../utility/common_utils.js';
+import { asyncSetTimeout, errorToString } from '../../utility/common_utils.js';
 import { transaction } from '../../resources/transaction.ts';
 import type { Condition, Query, Context, Select, Id, DirectCondition } from '../../resources/ResourceInterface.ts';
 import { collapseData } from '../../resources/tracked.ts';
-import { errorToString } from '../../utility/logging/harper_logger.js';
 
 const { HDB_ERROR_MSGS } = hdbErrors;
 const DEFAULT_DATABASE = 'data';

dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbGetBackup.js

@@ -4,7 +4,7 @@ const { Readable } = require('stream');
 const { getDatabases } = require('../../../../resources/databases.ts');
 const { readSync, openSync, createReadStream } = require('fs');
 const { open } = require('lmdb');
-const { OpenDBIObject } = require('../../../../utility/lmdb/OpenDBIObject.js');
+const { OpenDBIObject } = require('../../../../utility/lmdb/OpenDBIObject.ts');
 const OpenEnvironmentObject = require('../../../../utility/lmdb/OpenEnvironmentObject.js');
 const { AUDIT_STORE_OPTIONS } = require('../../../../resources/auditStore.ts');
 const { INTERNAL_DBIS_NAME, AUDIT_STORE_NAME } = require('../../../../utility/lmdb/terms.js');

resources/RecordEncoder.ts

@@ -17,7 +17,6 @@ import {
 	ACTION_32_BIT,
 } from './auditStore.ts';
 import * as harperLogger from '../utility/logging/harper_logger.js';
-import './blob.ts';
 import { blobsWereEncoded, decodeFromDatabase, deleteBlobsInObject, encodeBlobsWithFilePath } from './blob.ts';
 import { recordAction } from './analytics/write.ts';
 export type Entry = {
@@ -400,7 +399,7 @@ export function recordUpdater(store, tableId, auditStore) {
 			// we use resolveRecord outside of transaction, so must explicitly make it conditional
 			if (resolveRecord) putOptions.ifVersion = ifVersion = existingEntry?.version ?? null;
 			if (existingEntry && existingEntry.value && type !== 'message' && existingEntry.metadataFlags & HAS_BLOBS) {
-				if (!auditStore.getBinaryFast(existingEntry.localTime)) {
+				if (existingEntry.localTime && !auditStore.getBinaryFast(existingEntry.localTime)) {
 					// if it used to have blobs, and it doesn't exist in the audit store, we need to delete the old blobs
 					deleteBlobsInObject(existingEntry.value);
 				}

resources/blob.ts

@@ -34,13 +34,12 @@ import { Readable, pipeline } from 'node:stream';
 import { ensureDirSync } from 'fs-extra';
 import { get as envGet, getHdbBasePath } from '../utility/environment/environmentManager.js';
 import { CONFIG_PARAMS } from '../utility/hdbTerms.ts';
-import { join, dirname } from 'path';
+import { join, dirname } from 'node:path';
 import logger from '../utility/logging/logger.js';
 import type { LMDBStore } from 'lmdb';
 import { asyncSerialization, hasAsyncSerialization } from '../server/serverHelpers/contentTypes.ts';
 import { HAS_BLOBS, readAuditEntry } from './auditStore.ts';
 import { getHeapStatistics } from 'node:v8';
-import * as buffer from 'node:buffer';
 
 type StorageInfo = {
 	storageIndex: number;
@@ -93,7 +92,7 @@ let warnedSaveDeprecation = false;
  * 1. This has the built-in functionality for reading from the file-based storage
  * 2. This support for streams and asynchronous access to data that may not have a known size ahead of time
  * 3. This also avoids the Blob constructor which is expensive due to the transferred setup
- * Harper still supports saving native Blobs, but when they blobs are retrieved from storage, they always use this class.
+ * Harper still supports saving native Blobs, but when the blobs are retrieved from storage, they always use this class.
  */
 class FileBackedBlob extends InstanceOfBlobWithNoConstructor {
 	type = '';
@@ -492,6 +491,7 @@ export type BlobCreationOptions = {
 	size?: number; // the size of the data, if known ahead of time
 	saveBeforeCommit?: boolean; // save the blob before the transaction is committed
 };
+
 /**
  * Create a blob from a readable stream or a buffer by creating a file in the blob storage path with a new unique internal id, that
  * can be saved/stored.

resources/databases.ts

@@ -1,8 +1,8 @@
 import { initSync, getHdbBasePath, get as envGet } from '../utility/environment/environmentManager.js';
 import { INTERNAL_DBIS_NAME } from '../utility/lmdb/terms.js';
 import { open, compareKeys, type Database } from 'lmdb';
-import { join, extname, basename } from 'path';
-import { existsSync, readdirSync } from 'fs';
+import { join, extname, basename } from 'node:path';
+import { existsSync, readdirSync } from 'node:fs';
 import {
 	getBaseSchemaPath,
 	getTransactionAuditStoreBasePath,
@@ -15,20 +15,16 @@ import { _assignPackageExport } from '../globals.js';
 import { getIndexedValues } from '../utility/lmdb/commonUtility.js';
 import * as signalling from '../utility/signalling.js';
 import { SchemaEventMsg } from '../server/threads/itc.js';
-import { workerData } from 'worker_threads';
+import { workerData } from 'node:worker_threads';
 import harperLogger from '../utility/logging/harper_logger.js';
 const { forComponent } = harperLogger;
 import * as manageThreads from '../server/threads/manageThreads.js';
 import { openAuditStore } from './auditStore.ts';
 import { handleLocalTimeForGets } from './RecordEncoder.ts';
 import { deleteRootBlobPathsForDB } from './blob.ts';
 import { CUSTOM_INDEXES } from './indexes/customIndexes.ts';
-import * as OpenDBIObjectModule from '../utility/lmdb/OpenDBIObject.js';
-function OpenDBIObject(dupSort, isPrimary) {
-	// what is going on with esbuild, it suddenly is randomly flip-flopping the module record for OpenDBIObject, sometimes return the correct exports object and sometimes returning the exports as the `default`.
-	let OpenDBIObject = OpenDBIObjectModule.OpenDBIObject ?? OpenDBIObjectModule.default.OpenDBIObject;
-	return new OpenDBIObject(dupSort, isPrimary);
-}
+import { OpenDBIObject } from '../utility/lmdb/OpenDBIObject.ts';
+
 const logger = forComponent('storage');
 
 const DEFAULT_DATABASE_NAME = 'data';

resources/openApi.ts

@@ -1,5 +1,6 @@
 import { packageJson } from '../utility/packageUtils.js';
 import { Resources } from './Resources.ts';
+import { Resource } from './Resource.ts';
 
 const OPENAPI_VERSION = '3.0.3';
 // Maps graphql primitive types to open api types