Updated to handle upcoming changes in DefectDojo 2.0 (#10)

* Many minor edits/clean-ups

* Updated to handle upcoming changes in DefectDojo 2.0

The setup directory in DefectDojo's repo has been cleaned out and had a script used by godojo - the script's functionality was added to godojo directly.

database.go

@@ -43,6 +43,32 @@ func installDB(osTar string, dbTar *config.DBTarget, dCmd *osCmds) {
 	return
 }
 
+func installDBClient(osTar string, dbTar *config.DBTarget, dCmd *osCmds) {
+	// Look at the dbTar and call function to install that DB target
+	switch dbTar.Engine {
+	case "SQLite":
+		// Generate commands to install SQLite
+		// A remote SQLite DB makes no sense
+		// TODO: Log this error
+		return
+	case "MariaDB":
+		// Generate commands to install MariaDB
+		// TODO: Write install for MariaDB client
+		//instMariaDBClient(osTar, dCmd)
+		return
+	case "MySQL":
+		// Generate commands to install MySQL
+		// TODO: Write install for MySQL client
+		//instMySQLClient(osTar, dCmd)
+		return
+	case "PostgreSQL":
+		// Generate commands to install PostgreSQL
+		instPostgreSQLClient(osTar, dCmd)
+	}
+	return
+}
+
+
 func startDB(osTar string, dbTar *config.DBTarget, dbCmd *osCmds) {
 	// Look at the dbTar and call function to install that DB target
 	switch dbTar.Engine {

embd/dojoConfig.yml

@@ -14,7 +14,7 @@
 # SecretKey
 
 Install:
-  Version: "1.7.1" # DD_Version - Release version of DefectDojo from Github Releases
+  Version: "1.15.0" # DD_Version - Release version of DefectDojo from Github Releases
   SourceInstall: false # DD_SourceInstall - Boolean if a source install is desired (vs a release)
   # If ^ is true, a souce code install will occur overriding the release version provided
   SourceBranch: "dev" # DD_SourceBranch - The branch's HEAD to be checked out if SourceInstall is true
@@ -24,29 +24,29 @@ Install:
   Redact: true # DD_Redact - Boolean to redact sensitive info from the logs
   DevInstall: false # DD_Dev_Install - Boolean for development installs, uses fixed values
   Prompt: false # DD_Prompt - Boolean to prompt for configuration values - NOT IMPLEMENTED YET
-  Mac: false # DD_Mac - Boolean to set the install target as a Mac - NOT IMPLEMENTED YET
+  Mac: false # DD_Mac - Boolean to set the install target as a Mac - NOT IMPLEMENTED YET - use docker-compose instead
   Root: "/opt/dojo" # DD_Root - Root directory for the DefectDojo app Note: No traiing /
-  Source: "django-DefectDojo" # DD_Source - Directory in DD_Root for DefectDojo source code
+  Source: "django-DefectDojo" # DD_Source - Directory in DD_Root for DefectDojo source code Note: No /'s just the name
   Files: "local" # DD_Files - Directory in DD_Root for local files (static assets, uploads, etc)
   Media: "media" # DD_Media - Directory in DD_Files for uploaded files (screenshots, test artifacts, etc)
   Static: "static" # DD_Static - Directory in DD_Files for static asset files (JS, images, etc)
   App: "dojo" # DD_App - Directory in DD_Source where the DefectDojo Django app is located
-  Sampledata: false # DD_Sampledata - Boolean for installing sample data during the install
+  Sampledata: false # DD_Sampledata - Boolean for installing sample data during the install - NOT IMPLEMENTED YET
   PullSource: true # DD_PullSource - Boolean for installer to download source for DefectDojo Note: Usually for debugging the installer itself
   DB:
-    Engine: "MySQL" # DD_DB_Engine - Database engine to use (SQLite, MySQL, PostgreSQL, MariaDB) Note: CASE sEnSiTiVE!
+    Engine: "PostgreSQL" # DD_DB_Engine - Database engine to use (SQLite, MySQL, PostgreSQL, MariaDB) Note: CASE sEnSiTiVE!
     Local: true # DD_DB_Local - Boolean for when DB is on the same host/server/vm (local)
     Exists: false # DD_DB_Exists - Boolean for when DB for DefectDojo already exists so no install needed
-    Ruser: "root" # DD_DB_Ruser - Superuser for the database Note: this and Rpass below REQUIRED for remote and existing DBs
-    Rpass: "vee0Thoanae1daePooz0ieka" # DD_DB_Rpass - Password for the database superuser Note: set to 24 random characters
+    Ruser: "postgres" # DD_DB_Ruser - Superuser for the database, root for MySQL/MaraiDB & posgres for PostgreSQL. Note: this and Rpass below REQUIRED for remote and existing DBs
+    Rpass: "vee0Thoanae1daePooz0ieka" # DD_DB_Rpass - Password for the database superuser TODO: Note: set to 24 random characters if left blank
     Name: "dojodb" # DD_DB_Name - Name of the database that DefectDojo will use
     User: "dojodbusr" # DD_DB_User - Username of the database user that DefectDojo will use
     Pass: "vee0Thoanae1daePooz0ieka" # DD_DB_Pass - Password for the database user DefectDojo will use Note: set to 24 random characters
     Host: "localhost" # DD_DB_Host - Database hostname
-    Port: 3306 # DD_DB_Port - Port the database is listening on
+    Port: 5432 # DD_DB_Port - Port the database is listening on - 3306 for MySQL/MariaDB and 5432 for PostgreSQL
     Drop: false # DD_DB_Drop - Boolean to tell the installer to drop an existing DB if found
   OS:
-    User: "dojo-srv" # DD_OS_User - OS user to own the DefectDojo instll and files
+    User: "dojosrv" # DD_OS_User - OS user to own the DefectDojo instll and files
     Pass: "wahlieboojoKa8aitheibai3" # DD_OS_Pass - Password for the OS user for DefectDojo Note: set to 24 random characters
     Group: "dojo-srv" # DD_OS_Group - OS Group to own the DefectDojo install and files
     UID: 1337 # DD_OS_UID - User ID for the DefectDojo OS user
@@ -57,7 +57,7 @@ Install:
     Env: "/dojo/settings/.env.prod" # DD_SET_Env - Path to DefectDojo's environmental variables file
   Admin:
     User: "admin" # DD_ADMIN_User - Admin user for the DefectDojo web app
-    Pass: "admin" # DD_ADMIN_Pass - Password for the DefectDojo web app admin user Note: set to 24 random characters
+    Pass: "ddadmin" # DD_ADMIN_Pass - Password for the DefectDojo web app admin user Note: set to 24 random characters
     Email: "admin@localhost" # DD_ADMIN_Email - Email address for the web app admin user
     First: "Default" # DD_ADMIN_First - Web app admin users's first name
     Last: "Admin" # DD_ADMIN_Last - Web app admin users's last name
@@ -154,14 +154,14 @@ Settings:
 # These are optional config values that generally never need to be changed and are used while testing godojo
 # rather then actual installs
 Options:
-  HelpURL: "https://github.com/mtesauro/godojo" # DD_
-  ReleaseURL: "https://github.com/DefectDojo/django-DefectDojo/archive/" # DD_
-  CloneURL: "https://github.com/DefectDojo/django-DefectDojo.git" # DD_
-  YarnGPG: "https://dl.yarnpkg.com/debian/pubkey.gpg" # DD_
-  YarnRepo: "deb https://dl.yarnpkg.com/debian/ stable main" # DD_
-  NodeURL: "https://deb.nodesource.com/setup_12.x" # DD_
-  Embd: false
-  Key: ""
-  Tmpdir: "/opt/.dojo-temp/"
-  UsrInst: true
+  HelpURL: "https://github.com/defectdojo/godojo" #
+  ReleaseURL: "https://github.com/DefectDojo/django-DefectDojo/archive/" #
+  CloneURL: "https://github.com/DefectDojo/django-DefectDojo.git" #
+  YarnGPG: "https://dl.yarnpkg.com/debian/pubkey.gpg" #
+  YarnRepo: "deb https://dl.yarnpkg.com/debian/ stable main" #
+  NodeURL: "https://deb.nodesource.com/setup_12.x" #
+  Embd: false #
+  Key: "" #
+  Tmpdir: "/opt/.dojo-temp/" #
+  UsrInst: false #
 

godojo.go

@@ -24,7 +24,7 @@ import (
 // Global vars
 var (
 	// Installer version
-	ver = "1.1.1"
+	ver = "1.1.2"
 	// Configuration file name
 	cf = "dojoConfig.yml"
 	// Global config struct
@@ -81,7 +81,7 @@ func dojoBanner() {
 	fmt.Println("                                               /___/         ")
 	fmt.Println("    version ", ver)
 	fmt.Println("")
-	fmt.Println("  Welcome to goDojo, the official way to install DefectDojo.")
+	fmt.Println("  Welcome to godojo, the official way to install DefectDojo.")
 	fmt.Println("  For more information on how goDojo does an install, see:")
 	fmt.Printf("  %s", HelpURL)
 	fmt.Println("")
@@ -157,6 +157,19 @@ func getDojoRelease(i *config.InstallConfig) error {
 	traceMsg(fmt.Sprintf("Relese download list is %+v", dwnURL))
 	traceMsg(fmt.Sprintf("File path to write tarball is %+v", tarball))
 
+	// Check for existing tarball before downloading, might be a re-run of godojo
+	_, err = os.Stat(tarball)
+	if err == nil {
+		// File already downloaded so return early
+		err = extractRelease(tarball, i)
+		if err != nil {
+			return err
+		}
+		s.Stop()
+		statusMsg("Tarball already downloaded and extracted the DefectDojo release file")
+		return nil
+	}
+
 	// Setup a custom http client for downloading the Dojo release
 	var ddClient = &http.Client{
 		// Set time to a max of 20 seconds
@@ -201,9 +214,43 @@ func getDojoRelease(i *config.InstallConfig) error {
 		return err
 	}
 
+	// Extract the tarball to create the Dojo source directory
+	err = extractRelease(tarball, i)
+	if err != nil {
+		return err
+	}
+	//traceMsg("Extracting tarball into the Dojo source directory")
+	//tb, err := os.Open(tarball)
+	//if err != nil {
+	//	traceMsg(fmt.Sprintf("Error openging tarball was: %+v", err))
+	//	return err
+	//}
+	//err = Untar(i.Root, tb)
+	//if err != nil {
+	//	traceMsg(fmt.Sprintf("Error extracting tarball was: %+v", err))
+	//	return err
+	//}
+
+	//// Remane source directory to the non-versioned name
+	//traceMsg("Renaming source directory to the non-versioned name")
+	//oldPath := filepath.Join(i.Root, "django-DefectDojo-"+i.Version)
+	//newPath := filepath.Join(i.Root, i.Source)
+	//err = os.Rename(oldPath, newPath)
+	//if err != nil {
+	//	traceMsg(fmt.Sprintf("Error renaming Dojo source directory was: %+v", err))
+	//	return err
+	//}
+
+	// Successfully extracted the file, return nil
+	s.Stop()
+	statusMsg("Successfully downloaded and extracted the DefectDojo release file")
+	return nil
+}
+
+func extractRelease(t string, i *config.InstallConfig) error {
 	// Extract the tarball to create the Dojo source directory
 	traceMsg("Extracting tarball into the Dojo source directory")
-	tb, err := os.Open(tarball)
+	tb, err := os.Open(t)
 	if err != nil {
 		traceMsg(fmt.Sprintf("Error openging tarball was: %+v", err))
 		return err
@@ -223,10 +270,6 @@ func getDojoRelease(i *config.InstallConfig) error {
 		traceMsg(fmt.Sprintf("Error renaming Dojo source directory was: %+v", err))
 		return err
 	}
-
-	// Successfully extracted the file, return nil
-	s.Stop()
-	statusMsg("Successfully downloaded and extracted the DefectDojo release file")
 	return nil
 }
 
@@ -599,6 +642,27 @@ func main() {
 		}
 		Spin.Stop()
 		statusMsg("Installing Database complete")
+	}
+
+	// Install DB clients for remote DBs
+	if !conf.Install.DB.Local {
+		// Setup commands for DB clients
+		dbClient := osCmds{}
+		dbConf := &conf.Install.DB
+		installDBClient(target.id, dbConf, &dbClient)
+
+		// Run the commands to install the chosen DB
+		Spin = spinner.New(spinner.CharSets[34], 100*time.Millisecond)
+		Spin.Prefix = "Installing " + conf.Install.DB.Engine + " database client for DefectDojo..."
+		Spin.Start()
+		for i := range dbClient.cmds {
+			sendCmd(cmdLogger,
+				dbClient.cmds[i],
+				dbClient.errmsg[i],
+				dbClient.hard[i])
+		}
+		Spin.Stop()
+		statusMsg("Installing Database client complete")
 
 	}
 

os.go

@@ -71,6 +71,19 @@ func instPostgreSQL(id string, b *osCmds) {
 	return
 }
 
+func instPostgreSQLClient(id string, b *osCmds) {
+	switch id {
+	case "ubuntu:18.04":
+		fallthrough
+	case "ubuntu:20.04":
+		fallthrough
+	case "ubuntu:20.10":
+		ubuntuInstPostgreSQLClient(id, b)
+	}
+	return
+}
+
+
 func defaultDBCreds(db *config.DBTarget, os string) map[string]string {
 	// Setup a map to return
 	creds := map[string]string{"user": "foo", "pass": "bar"}

prepInstall.go

@@ -83,7 +83,10 @@ func readConfigFile() {
 // overrides any options set in the configuration file. These variables
 // are used to supply either install-time configurations or provide values
 // that are used in DefectDojo's settings.py configuration file
-func readEnvVars() { // Env variables pulled from repo Add newly supported env vars below and to the case statement below after "if match {"
+func readEnvVars() {
+	// Env variables pulled from repo. Add newly supported env vars below and
+	// to the switch statement below after the for that ranges over overrides
+	// TODO: Add non-setting.py ENV vars like DD_SourcCommit
 	dojoEnvs := map[string]bool{
 		"DD_ADMIN_FIRST_NAME":                            true,
 		"DD_ADMIN_LAST_NAME":                             true,

ubuntu.go

@@ -3,6 +3,8 @@ package main
 import (
 	"bufio"
 	"fmt"
+	"io/fs"
+	"io/ioutil"
 	"os"
 	"strings"
 
@@ -136,6 +138,33 @@ func ubuntuInstPostgreSQL(id string, b *osCmds) {
 	return
 }
 
+func ubuntuInstPostgreSQLClient(id string, b *osCmds) {
+	switch id {
+	case "ubuntu:18.04":
+		fallthrough
+	case "ubuntu:20.04":
+		fallthrough
+	case "ubuntu:20.10":
+		b.id = id
+		b.cmds = []string{
+			"DEBIAN_FRONTEND=noninteractive apt-get install -y postgresql-client-12",
+			"groupadd -f postgres",
+			"useradd -s /bin/bash -m -g postgres postgres",
+		}
+		b.errmsg = []string{
+			"Unable to install PostgreSQL client",
+			"Unable to add postgres group",
+			"Unable to add postgres user",
+		}
+		b.hard = []bool{
+			true,
+			true,
+			true,
+		}
+	}
+	return
+}
+
 // Determine the default creds for a database freshly installed in Ubuntu
 func ubuntuDefaultDBCreds(db *config.DBTarget, creds map[string]string) {
 	// Installer currently assumes the default DB passwrod handling won't change by release
@@ -258,6 +287,10 @@ func ubuntuOSPrep(id string, inst *config.InstallConfig, b *osCmds) {
 }
 
 func ubuntuSetupDDjango(id string, inst *config.InstallConfig, b *osCmds) {
+	// Setup expect script needed to set initial admin password
+	traceMsg(fmt.Sprintf("Injecting file %s at %s", "setup-superuser.expect", inst.Root+"/django-DefectDojo"))
+	_ = injectFile("setup-superuser.expect", inst.Root+"/django-DefectDojo", 0755)
+
 	// Django installs - migrations, create Django superuser
 	switch id {
 	case "ubuntu:18.04":
@@ -273,7 +306,7 @@ func ubuntuSetupDDjango(id string, inst *config.InstallConfig, b *osCmds) {
 			"cd " + inst.Root + "/django-DefectDojo && source ../bin/activate && python3 manage.py createsuperuser --noinput --username=\"" +
 				inst.Admin.User + "\" --email=\"" + inst.Admin.Email + "\"",
 			"cd " + inst.Root + "/django-DefectDojo && source ../bin/activate && " +
-				inst.Root + "/django-DefectDojo/setup/scripts/common/setup-superuser.expect " + inst.Admin.User + " " + inst.Admin.Pass,
+				inst.Root + "/django-DefectDojo/setup-superuser.expect " + inst.Admin.User + " " + inst.Admin.Pass,
 			"cd " + inst.Root + "/django-DefectDojo && source ../bin/activate && python3 manage.py loaddata product_type",
 			"cd " + inst.Root + "/django-DefectDojo && source ../bin/activate && python3 manage.py loaddata test_type",
 			"cd " + inst.Root + "/django-DefectDojo && source ../bin/activate && python3 manage.py loaddata development_environment",
@@ -344,3 +377,21 @@ func ubuntuSetupDDjango(id string, inst *config.InstallConfig, b *osCmds) {
 
 	return
 }
+
+func injectFile(n string, p string, mask fs.FileMode) error {
+	loc := emdir + "setup-superuser.expect"
+	d, err := Asset(loc)
+	if err != nil {
+		return err
+	}
+
+	err = ioutil.WriteFile(p+"/"+n, d, mask)
+	if err != nil {
+		// File can't be written
+		return err
+	}
+
+	traceMsg(fmt.Sprintf("Wrote file %s at %s", n, p))
+
+	return nil
+}