feat(file-utils): add a utility package for handling file uploads (#2155

)

* feat(file-utils): add a utility package for handling file uploads

gh-0

* feat(file-utils): changes related to publishing

gh-00

---------

Co-authored-by: Yesha Mavani <[email protected]>

.czferc.js

@@ -34,6 +34,7 @@ const scopes = [
   {name: 'core'},
   {name: 'cli'},
   {name: 'cache'},
+  {name: 'file-utils'},
   {name: 'authentication-service'},
   {name: 'in-mail-service'},
   {name: 'notification-service'},

lerna.json

@@ -3,6 +3,7 @@
     "packages/core",
     "packages/cli",
     "packages/cache",
+    "packages/file-utils",
     "packages/feature-toggle",
     "packages/custom-sf-changelog/",
     "services/*",
@@ -83,4 +84,4 @@
   ],
   "version": "independent",
   "$schema": "node_modules/lerna/schemas/lerna-schema.json"
-}
+}

package.json

@@ -52,6 +52,7 @@
     "packages/cli/",
     "packages/cache",
     "packages/feature-toggle",
+    "packages/file-utils",
     "packages/custom-sf-changelog/",
     "services/*",
     "sandbox/*"

packages/file-utils/.eslintignore

@@ -0,0 +1,5 @@
+node_modules/
+dist/
+coverage/
+.eslintrc.js
+mochawesome-report

packages/file-utils/.eslintrc.js

@@ -0,0 +1,12 @@
+module.exports = {
+  extends: '@loopback/eslint-config',
+  rules: {
+    'no-extra-boolean-cast': 'off',
+    '@typescript-eslint/interface-name-prefix': 'off',
+    'no-prototype-builtins': 'off',
+  },
+  parserOptions: {
+    project: './tsconfig.json',
+    tsconfigRootDir: __dirname,
+  },
+};

packages/file-utils/.gitignore

@@ -0,0 +1,66 @@
+# Logs
+logs
+*.log
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+
+# Runtime data
+pids
+*.pid
+*.seed
+*.pid.lock
+
+# Directory for instrumented libs generated by jscoverage/JSCover
+lib-cov
+
+# Coverage directory used by tools like istanbul
+coverage
+
+# nyc test coverage
+.nyc_output
+
+# Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files)
+.grunt
+
+# Bower dependency directory (https://bower.io/)
+bower_components
+
+# node-waf configuration
+.lock-wscript
+
+# Compiled binary addons (http://nodejs.org/api/addons.html)
+build/Release
+
+# Dependency directories
+node_modules/
+jspm_packages/
+
+# Typescript v1 declaration files
+typings/
+
+# Optional npm cache directory
+.npm
+
+# Optional eslint cache
+.eslintcache
+
+# Optional REPL history
+.node_repl_history
+
+# Output of 'npm pack'
+*.tgz
+
+# Yarn Integrity file
+.yarn-integrity
+
+# dotenv environment variables file
+.env
+
+# Transpiled JavaScript files from Typescript
+/dist
+
+# Cache used by TypeScript's incremental build
+*.tsbuildinfo
+
+mochawesome-report

packages/file-utils/.mocharc.json

@@ -0,0 +1,6 @@
+{
+  "exit": true,
+  "recursive": true,
+  "require": "source-map-support/register",
+  "reporter": "mochawesome"
+}

packages/file-utils/.nycrc

@@ -0,0 +1,9 @@
+{
+  "include": ["dist"],
+  "exclude": ["dist/__tests__/"],
+  "extension": [".js", ".ts"],
+  "reporter": ["text", "html", "json"],
+  "exclude-after-remap": false,
+  "check-coverage": true,
+  "lines": 75
+}

packages/file-utils/.prettierignore

@@ -0,0 +1,4 @@
+dist
+*.json
+coverage
+mochawesome-report

packages/file-utils/.prettierrc

@@ -0,0 +1,7 @@
+{
+  "bracketSpacing": false,
+  "singleQuote": true,
+  "printWidth": 80,
+  "trailingComma": "all",
+  "arrowParens": "avoid"
+}

packages/file-utils/.yo-rc.json

@@ -0,0 +1,6 @@
+{
+  "@sourceloop/cli": {
+    "packageManager": "npm",
+    "version": "4.1.0"
+  }
+}

packages/file-utils/DEVELOPING.md

@@ -0,0 +1,36 @@
+# Developer's Guide
+
+We use Visual Studio Code for developing LoopBack and recommend the same to our
+users.
+
+## VSCode setup
+
+Install the following extensions:
+
+- [eslint](https://marketplace.visualstudio.com/items?itemName=dbaeumer.vscode-eslint)
+- [prettier](https://marketplace.visualstudio.com/items?itemName=esbenp.prettier-vscode)
+
+## Development workflow
+
+### Visual Studio Code
+
+1. Start the build task (Cmd+Shift+B) to run TypeScript compiler in the
+   background, watching and recompiling files as you change them. Compilation
+   errors will be shown in the VSCode's "PROBLEMS" window.
+
+2. Execute "Run Rest Task" from the Command Palette (Cmd+Shift+P) to re-run the
+   test suite and lint the code for both programming and style errors. Linting
+   errors will be shown in VSCode's "PROBLEMS" window. Failed tests are printed
+   to terminal output only.
+
+### Other editors/IDEs
+
+1. Open a new terminal window/tab and start the continuous build process via
+   `npm run build:watch`. It will run TypeScript compiler in watch mode,
+   recompiling files as you change them. Any compilation errors will be printed
+   to the terminal.
+
+2. In your main terminal window/tab, run `npm run test:dev` to re-run the test
+   suite and lint the code for both programming and style errors. You should run
+   this command manually whenever you have new changes to test. Test failures
+   and linter errors will be printed to the terminal.

packages/file-utils/README.md

@@ -0,0 +1,117 @@
+# File Utils
+
+This package provides utilities for adding file upload and validations in LoopBack 4 microservices.
+
+## Installation
+
+To install the package, run the following command:
+
+```shell
+npm install @sourceloop/file-utils
+```
+
+## Usage
+
+- bind the component
+
+```ts
+this.component(FileUtilComponent);
+```
+
+- add the `file` decorator to the controller method where you want the handle the file -
+
+```ts
+@file(getJsonSchema(Parent) as SchemaObject, ['file'], ['.csv'])
+data: ParentWithFile,
+```
+
+You can read more about the file decorator and it's parameters [here](#file-decorator)
+
+### File Decorator
+
+This decorator is designed to handle file uploads in a `multipart/form-data` request. It provides a structured way to manage multiple fields, including file fields, in your request body.
+
+#### Parameters
+
+- **schema** (`SchemaObject`):
+
+  - The schema object for the request body. It is useful in cases where you have multiple fields and you want to handle this data in a structured way. For example, if your request has a file field and a name field, you can define a schema object with these fields and pass it to the decorator. The file field will be marked as binary in the schema.
+
+- **fileFields** (`string[]`):
+
+  - The fields in the schema that represent file uploads. So as per the above example, you can pass `['file']` to this parameter. This will mark the file field as binary in the schema.
+
+- **allowedExtensions** (`string[]`):
+
+  - The allowed file extensions for the uploaded files.
+
+- **storageOptions** (`IFileStorageOptions<T>`):
+  - The options for file storage. This supports any multer storage engine class. By default, this uses in-memory storage, and the package also provides an S3 storage engine.
+
+#### S3 Storage Engine
+
+By default, the files are stored in memory, but the package also provides an s3 based storage engine as well, it can be used in two ways -
+
+- if you want to use this strategy for every request, just bind a storage provider on `FileUtilBindings.MulterStorage` key
+
+```ts
+this.bind(FileUtilBindings.MulterStorage).toProvider(MulterStorageProvider);
+```
+
+You can refer the default implementation [here](./src/services/multer-storage.provider.ts).
+
+- if you want to use this strategy for particular requests, you can provide the storage class and it's config with the `@file` decorator -
+
+```ts
+  @post('/parents/s3')
+  @response(200, {
+    description: 'Parent model instance',
+    content: {'application/json': {schema: getModelSchemaRef(Parent)}},
+  })
+  async createS3(
+    @file(getJsonSchema(Parent) as SchemaObject, ['file'], ['.csv'], {
+      storageClass: MulterS3Storage,
+      options: {
+        bucket: process.env.AWS_S3_BUCKET ?? '',
+      },
+    })
+    data: ParentWithFile,
+  ): Promise<void> {
+    this.receiverStub.receive(data);
+  }
+```
+
+### File Validations
+
+The package provides two validators by default -
+
+- `FileTypeValidator` - validates the type of the file as per configurations
+- `FileNameValidator` - validates the name of the file, to avoid certain special characters in the name
+
+You can write your own custom validators and register them using the `@fileValidator` decorator, every validator must implement the `IFileValidator` interface -
+
+```ts
+@fileValidator()
+export class FileSizeValidator implements IFileValidator {
+  ...
+}
+```
+
+### Limits Configurations
+
+You can configure the multer instance by binding the multer limits configuration on the `FileUtilBindings.MulterLimits` key -
+
+```ts
+this.bind(FileUtilBindings.MulterLimits).to({
+  fileSize: FIVE * MB, // 5 MB
+  files: 1,
+});
+```
+
+## Contributing
+
+Contributions are welcome! Please refer to the [contribution guidelines](../../DEVELOPING.md) for more information.
+
+## License
+
+This package is [MIT licensed](../../LICENSE).

packages/file-utils/package.json

@@ -0,0 +1,80 @@
+{
+  "name": "@sourceloop/file-utils",
+  "version": "0.0.1",
+  "description": "file upload and multipart related utilities for loopback4 applications",
+  "keywords": [
+    "loopback-extension",
+    "loopback"
+  ],
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "engines": {
+    "node": ">=18"
+  },
+  "scripts": {
+    "build": "lb-tsc",
+    "build:watch": "lb-tsc --watch",
+    "lint": "npm run eslint && npm run prettier:check",
+    "lint:fix": "npm run eslint:fix && npm run prettier:fix",
+    "prettier:cli": "prettier \"**/*.ts\" \"**/*.js\"",
+    "prettier:check": "npm run prettier:cli -- -l",
+    "prettier:fix": "npm run prettier:cli -- --write",
+    "eslint": "eslint --report-unused-disable-directives .",
+    "eslint:fix": "npm run eslint -- --fix",
+    "pretest": "npm run rebuild",
+    "test": "lb-mocha --timeout=5000 --allow-console-logs \"dist/__tests__\"",
+    "posttest": "npm run lint",
+    "test:dev": "lb-mocha --timeout=5000 --allow-console-logs dist/__tests__/**/*.js && npm run posttest",
+    "clean": "lb-clean dist *.tsbuildinfo .eslintcache",
+    "rebuild": "npm run clean && npm run build",
+    "prune": "npm prune --production",
+    "coverage": "nyc npm run test"
+  },
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/sourcefuse/loopback4-microservice-catalog.git",
+    "directory": "packages/file-utils"
+  },
+  "author": "Sourcefuse",
+  "license": "MIT",
+  "files": [
+    "README.md",
+    "dist",
+    "src",
+    "!*/__tests__"
+  ],
+  "peerDependencies": {
+    "@loopback/core": "^6.0.2"
+  },
+  "dependencies": {
+    "@loopback/core": "^6.0.2",
+    "@loopback/repository": "^7.0.2",
+    "@loopback/rest": "^14.0.2",
+    "@loopback/rest-explorer": "^7.0.2",
+    "@sourceloop/core": "^14.1.1",
+    "@types/multer": "^1.4.11",
+    "file-type": "^16.5.4",
+    "lodash": "^4.17.21",
+    "loopback4-s3": "^9.0.0",
+    "multer": "^1.4.5-lts.1",
+    "multer-s3": "^3.0.1",
+    "tslib": "^2.6.2",
+    "winston": "^3.13.0"
+  },
+  "devDependencies": {
+    "@istanbuljs/nyc-config-typescript": "^1.0.2",
+    "@loopback/build": "^11.0.2",
+    "@loopback/eslint-config": "^15.0.2",
+    "@loopback/testlab": "^7.0.2",
+    "@types/dotenv": "^8.2.0",
+    "@types/lodash": "^4.17.0",
+    "@types/multer-s3": "^3.0.3",
+    "@types/node": "^20.12.7",
+    "dotenv": "^16.4.5",
+    "eslint": "^8.57.0",
+    "form-data": "^4.0.0",
+    "mochawesome": "^7.1.3",
+    "source-map-support": "^0.5.21",
+    "typescript": "^5.4.5"
+  }
+}