Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Concurrency support #9

Open
wants to merge 6 commits into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
74 changes: 70 additions & 4 deletions dll.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@ import (
"go/parser"
"go/token"
"os"
"runtime"
"sync"
)

type report struct {
Expand Down Expand Up @@ -84,16 +86,80 @@ func gather(source string, asFile bool) ([]*report, error) {
func main() {
if len(os.Args) < 2 {
fmt.Fprintln(os.Stderr, "no source files supplied")
os.Exit(1)
}

for _, source := range os.Args[1:] {
files := os.Args[1:]
fileCount := len(files)

reportsChannel := make(chan []*report, fileCount)
cpuCount := runtime.NumCPU()
filesPerCore := splitArrayIntoParts(files, cpuCount)

go func() {
Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think a simpler approach would be to just spawn a new goroutine for each file and add each slice of *reports to another slice guarded by a mutex. Then when the WaitGroup is done(), iterate through the slice of all reports and dump that to stdout.

I don't know how much of an advantage the splitting gives us either as I usually let the Go runtime schedule goroutines for me. :)

Other than that I like the approach. I think if we pare it down to what I tried to describe, it'd be perfect.

var wg sync.WaitGroup

for _, files := range filesPerCore {
wg.Add(1)
go analyseFiles(files, reportsChannel, &wg)
}

wg.Wait()
close(reportsChannel)
}()

for reports := range reportsChannel {
printReports(reports)
}
}

func analyseFiles(files []string, c chan []*report, wg *sync.WaitGroup) {
defer wg.Done()
for _, source := range files {
r, err := gather(source, true)
if err != nil {
fmt.Fprintf(os.Stderr, "error parsing %s: %s\n", source, err)
continue
c <- []*report{}
return
}
for _, rep := range r {
fmt.Println(rep)
c <- r
}
}

func printReports(reports []*report) {
for _, report := range reports {
fmt.Println(report)
}
}

func splitArrayIntoParts(array []string, parts int) [][]string {
arraySize := len(array)

if parts <= 1 {
return [][]string{array}
}

// if there are more parts than strings, it tries to find the next smallest number to destribute them equally.
if arraySize < parts {
for (arraySize % parts) != 0 {
parts--
}
}

stringsPerPart := arraySize / parts
arrayParts := make([][]string, 0, parts)
lastIndex := 0
for i := 0; i < parts; i++ {
arrayParts = append(arrayParts, array[lastIndex:(lastIndex+stringsPerPart)])
lastIndex = lastIndex + stringsPerPart
}

// if not all strings could be splitted equally it will adds the missing ones to the first part.
if stringsPerPart*parts != arraySize {
firstpart := arrayParts[0]
firstpart = append(firstpart, array[stringsPerPart*parts:]...)
arrayParts[0] = firstpart
}

return arrayParts
}
94 changes: 94 additions & 0 deletions dll_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -136,3 +136,97 @@ func TestErrorParsing(t *testing.T) {
t.Error("expected error but got nil")
}
}

func Test_splitArrayIntoParts(t *testing.T) {
getStringArray := func(amount int) []string {
strings := make([]string, 0, amount)
for i := 0; i < amount; i++ {
strings = append(strings, "foo")
}
return strings
}

tests := []struct {
name string
strings []string
parts int
expectedParts int
}{
{
name: "should split the array with one string into one part",
strings: getStringArray(1),
parts: 1,
expectedParts: 1,
},
{
name: "should split the array with one string into zero part",
strings: getStringArray(1),
parts: 0,
expectedParts: 1,
},
{
name: "should split the array with two strings into one part",
strings: getStringArray(2),
parts: 1,
expectedParts: 1,
},
{
name: "should split the array with two strings into four part",
strings: getStringArray(2),
parts: 4,
expectedParts: 2,
},
{
name: "should split the array with one string into two part",
strings: getStringArray(1),
parts: 2,
expectedParts: 1,
},
{
name: "should split the array with four strings into two part",
strings: getStringArray(4),
parts: 2,
expectedParts: 2,
},
{
name: "should split the array with two strings into three part",
strings: getStringArray(2),
parts: 3,
expectedParts: 2,
},
{
name: "should split the array with ten strings into three part",
strings: getStringArray(10),
parts: 3,
expectedParts: 3,
},
}

for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
got := splitArrayIntoParts(test.strings, test.parts)

if len(got) != test.expectedParts {
t.Fatalf("Expect to split the array into '%d' but got '%d'", test.expectedParts, len(got))
}

for _, files := range got {
if len(files) < 1 {
t.Fatalf("Expected to contain at least on string but got none")
}
}
})
}

t.Run("should split the empty array into one part", func(t *testing.T) {
strings := []string{}
parts := 1

got := len(splitArrayIntoParts(strings, parts))
want := 1

if got != want {
t.Fatalf("Expected a length of %d but got %d", want, got)
}
})
}