feat(KOPPS-1239): Updated/created import script for handling prerequisites (#268)

* Updated/created script for handling prerequisites

* Updated courseSchema and script

* removed csv-parser

* Updated README and config

* Updated README and script

* Updated test

---------

Co-authored-by: Karl Andin <[email protected]>

Author: axelbjo

scripts/importData/README.md

@@ -103,3 +103,33 @@ IMPORTANT: Call it from the project root!
 ```sh
 ./scripts/importSupplementaryInfo.sh ./data/2024-03-07-test.csv
 ```
+
+## Script to import recommendedPrerequisites directly into MongoB
+
+This script can be used to import a CSV containing recommendedPrerequisites. It does clean `NULL` values.
+
+### Data
+
+The script expects a CSV file
+
+- with a comma `,` as separator.
+- without headers
+- with the columns specified in the function [readCSV](./index3.js)
+
+```js
+headers: [
+            'courseCode',
+            'recommendedPrerequisites_sv',
+            'recommendedPrerequisites_en',
+          ],
+```
+
+### Usage
+
+Call `importRecommendedPrerequisites.sh` and pass the path to the file you want to import.
+
+IMPORTANT: Call it from the project root!
+
+```sh
+./scripts/importRecommendedPrerequisites.sh ./data/2024-03-07-test.csv
+```

scripts/importData/filterEmptyCourseInfos.js

@@ -1,3 +1,5 @@
+const log = require('@kth/log')
+
 const filterEmptyCourseInfos = courseInfos => {
   return courseInfos.filter(
     ({ courseCode, supplementaryInfo_sv, supplementaryInfo_en, courseDisposition_sv, courseDisposition_en }) => {
@@ -11,6 +13,19 @@ const filterEmptyCourseInfos = courseInfos => {
     }
   )
 }
+const filterEmptyPrerequisites = courseInfos => {
+  const EMPTY_VALUES = ['', 'NULL']
+
+  return courseInfos.filter(({ courseCode, recommendedPrerequisites_sv, recommendedPrerequisites_en }) => {
+    if (!courseCode) {
+      return false
+    }
+    if (EMPTY_VALUES.includes(recommendedPrerequisites_sv) && EMPTY_VALUES.includes(recommendedPrerequisites_en)) {
+      return false
+    }
+    return true
+  })
+}
 
 const filterEmptySupplementaryInfos = courseInfos => {
   const EMPTY_VALUES = ['', 'NULL']
@@ -29,4 +44,5 @@ const filterEmptySupplementaryInfos = courseInfos => {
 module.exports = {
   filterEmptyCourseInfos,
   filterEmptySupplementaryInfos,
+  filterEmptyPrerequisites,
 }

scripts/importData/index3.js

@@ -0,0 +1,49 @@
+const log = require('@kth/log')
+const { filterEmptyPrerequisites } = require('./filterEmptyCourseInfos')
+const { handleArrayOfCourseInfos } = require('./handleArrayOfCourseInfos')
+const { readCSV, setupLogging, setupDatabase, tryToGetPathToFileFromParams } = require('./utils')
+
+const csvConfig = {
+  separator: ',',
+  headers: ['courseCode', 'recommendedPrerequisites_sv', 'recommendedPrerequisites_en'],
+}
+
+const handleCSV = async file => {
+  const rawCSV = await readCSV(file, csvConfig)
+
+  log.info(`Extracted ${rawCSV.length} courses over all`)
+
+  const filteredList = filterEmptyPrerequisites(rawCSV)
+
+  log.info(`Extracted ${filteredList.length} courses with non-empty texts`)
+
+  const result = await handleArrayOfCourseInfos(filteredList)
+
+  const successful = result.filter(({ success }) => success)
+  const failed = result.filter(({ success }) => !success)
+
+  log.info('-------')
+  log.info(`Extracted ${rawCSV.length} courses over all`)
+  log.info(`Extracted ${filteredList.length} courses with non-empty texts`)
+  log.info(`Handled ${result.length} courseInfos`)
+  log.info(`${successful.length} successful`)
+  log.info(`${failed.length} failed`)
+  log.info('-------')
+
+  if (failed.length) {
+    log.info('Failed attempts:')
+    log.info(failed)
+  }
+}
+
+const run = () => {
+  setupLogging()
+
+  const pathToFile = tryToGetPathToFileFromParams()
+
+  setupDatabase()
+
+  handleCSV(pathToFile)
+}
+
+run()

scripts/importRecommendedPrerequisites.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+node --env-file=./scripts/importData/.env ./scripts/importData/index3.js $1

scripts/test/filterEmptyCourseInfos.test.js

@@ -1,4 +1,9 @@
-const { filterEmptyCourseInfos, filterEmptySupplementaryInfos } = require('../importData/filterEmptyCourseInfos')
+const { recommendedPrerequisites_sv } = require('../../server/models/courseSchema')
+const {
+  filterEmptyCourseInfos,
+  filterEmptySupplementaryInfos,
+  filterEmptyPrerequisites,
+} = require('../importData/filterEmptyCourseInfos')
 
 const filteredCourseInfos = [
   {
@@ -61,6 +66,23 @@ const filteredSupplementaryInfos = [
     courseDisposition_en: '',
   },
 ]
+const filteredPrerequisites = [
+  {
+    courseCode: 'chem6002',
+    recommendedPrerequisites_sv: 'recommendedPrerequisites_sv_2',
+    recommendedPrerequisites_en: 'recommendedPrerequisites_en_2',
+  },
+  {
+    courseCode: 'chem6003',
+    recommendedPrerequisites_sv: 'recommendedPrerequisites_sv_3',
+    recommendedPrerequisites_en: 'recommendedPrerequisites_en_3',
+  },
+  {
+    courseCode: 'chem6004',
+    recommendedPrerequisites_sv: 'recommendedPrerequisites_sv_4',
+    recommendedPrerequisites_en: 'NULL',
+  },
+]
 
 const unfilteredCourseInfos = [
   ...filteredCourseInfos,
@@ -104,6 +126,33 @@ const unfilteredSupplementaryInfos = [
     courseDisposition_en: '',
   },
 ]
+const unfilteredPrerequisites = [
+  {
+    courseCode: 'chem6001',
+    recommendedPrerequisites_sv: 'NULL',
+    recommendedPrerequisites_en: '',
+  },
+  {
+    courseCode: 'chem6002',
+    recommendedPrerequisites_sv: 'recommendedPrerequisites_sv_2',
+    recommendedPrerequisites_en: 'recommendedPrerequisites_en_2',
+  },
+  {
+    courseCode: 'chem6003',
+    recommendedPrerequisites_sv: 'recommendedPrerequisites_sv_3',
+    recommendedPrerequisites_en: 'recommendedPrerequisites_en_3',
+  },
+  {
+    courseCode: 'chem6004',
+    recommendedPrerequisites_sv: 'recommendedPrerequisites_sv_4',
+    recommendedPrerequisites_en: 'NULL',
+  },
+  {
+    courseCode: 'chem6004',
+    recommendedPrerequisites_sv: '',
+    recommendedPrerequisites_en: '',
+  },
+]
 
 describe('filterEmptyCourseInfos', () => {
   test('filters courseInfos', () => {
@@ -120,3 +169,10 @@ describe('filterEmptySupplementaryInfos', () => {
     expect(result).toEqual(filteredSupplementaryInfos)
   })
 })
+describe('filterEmptyPrerequisites', () => {
+  test('filters recommendedPrerequisites', () => {
+    const result = filterEmptyPrerequisites(unfilteredPrerequisites)
+
+    expect(result).toEqual(filteredPrerequisites)
+  })
+})

server/models/courseSchema.js

@@ -121,12 +121,12 @@ module.exports.pre('save', next => {
   }
   if (this.recommendedPrerequisites_sv) {
     if (safeGet(() => this.recommendedPrerequisites_sv)) {
-      this.supplementaryInfo_sv = sanitize(this.recommendedPrerequisites_sv)
+      this.recommendedPrerequisites_sv = sanitize(this.recommendedPrerequisites_sv)
     }
   }
   if (this.recommendedPrerequisites_en) {
     if (safeGet(() => this.recommendedPrerequisites_en)) {
-      this.supplementaryInfo_en = sanitize(this.recommendedPrerequisites_en)
+      this.recommendedPrerequisites_en = sanitize(this.recommendedPrerequisites_en)
     }
   }
   if (this.supplementaryInfo_sv) {