diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..59bd8d7 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +.idea/ +local/ diff --git a/Makefile b/Makefile index 8f4e185..3c6dbf5 100644 --- a/Makefile +++ b/Makefile @@ -7,3 +7,5 @@ clean: rm -f ./bin/parse rm -f ./bin/count rm -f ./bin/timeseries +local-build: + go build -o ./local/parse ./cmd/parse/parse.go diff --git a/cmd/parse/parse.go b/cmd/parse/parse.go index ea50f37..b18af5e 100644 --- a/cmd/parse/parse.go +++ b/cmd/parse/parse.go @@ -1,226 +1,225 @@ -// -// All software tools within this package are Copyright (c) 2020 Merit Network, Inc., -// and Stanford University. All Rights Reserved. -// - -package main - -import ( - "darknet-events/internal/annotate" - "darknet-events/internal/cache" - "darknet-events/internal/decode" - - "flag" - "io" - "log" - "os" - "runtime" - "runtime/pprof" - "strings" - "time" - - "github.com/google/gopacket/pcapgo" -) - -// Config is a struct containing the program's configuration info. -type Config struct { - SenderExpiry int - MinUniques int - MinScanRate float64 - NumSamples int - CacheInPath string - CacheOutPath string - EventOutPrefix string - OngoingEventOutPrefix string - MMASNInPath string - MMGeoInPath string - Pfx2asInPath string - DNSServers []string - PcapPaths []string - ProfileCPUPath string - ProfileMemPath string - Newdl bool -} - -// config loads configuration information from the given flags. It is expected -// that this slice is os.Args. -func config() *Config { - var cfg Config - - threshold := flag.Int("threshold", 0, "Number of seconds that "+ - "must elapse before an event is considered over.") - minUniques := flag.Int("uniques", 1, "Minimum number of unique "+ - "destinations that must be hit for an event to be considered (must " + - "be a positive number).") - minScanRate := flag.Float64("rate", 0, "Minimum global packet rate for "+ - "an event to be considered.") - numSamples := flag.Int("samples", 0, "Number of packet samples "+ - "to save from each event.") - mmASNInPathString := flag.String("asnin", "", "Path to the MaxMind ASN "+ - "database.") - mmGeoInPathString := flag.String("geoin", "", "Path to the MaxMind "+ - "location database.") - pfx2asInPathString := flag.String("pfx2asin", "", "Path to the pfx2as "+ - "database.") - dnsServersString := flag.String("dns", "", "DNS server IP address(es).") - cacheInPathString := flag.String("cachein", "", "Path to load cache "+ - "entries from before analysis starts.") - cacheOutPathString := flag.String("cacheout", "", "Path to dump "+ - "remaining cache entries to for later use.") - eventOutPrefixString := flag.String("eventout", "", "Prefix of path to "+ - "save captured event data to.") - ongoingEventOutPrefixString := flag.String("ongoingout", "", "Prefix of "+ - "path to save captured event data to.") - pcapPathsString := flag.String("pcap", "", "Path to pcap files. Multiple "+ - "paths must be separated by a space.") - profileCPUPathString := flag.String("cpu", "", "Output path for CPU "+ - "profile. No profiling if not included.") - profileMemPathString := flag.String("mem", "", "Output path for memory "+ - "profile. No profiling if not included.") - newdlBool := flag.Bool("newdl", false, "True if JSON output needs to be "+ - "newline-delimited.") - flag.Parse() - - // Ensure that all non-optional flags are set. - if *threshold == 0 { - log.Fatal("Must specify a threshold timeout.") - } - if *numSamples == 0 { - log.Fatal("Must specify how many samples to take.") - } - if *eventOutPrefixString == "" { - log.Fatal("Must specify a location to write event data to.") - } - if *pcapPathsString == "" { - log.Fatal("Must specify at least one pcap file.") - } - if *minUniques <= 0 { - log.Fatal("Must specify a positive uniques number.") - } - - // Process and save parameters. - cfg.SenderExpiry = *threshold - cfg.MinUniques = *minUniques - cfg.MinScanRate = *minScanRate - cfg.NumSamples = *numSamples - cfg.CacheInPath = *cacheInPathString - cfg.CacheOutPath = *cacheOutPathString - cfg.EventOutPrefix = *eventOutPrefixString - cfg.OngoingEventOutPrefix = *ongoingEventOutPrefixString - cfg.MMASNInPath = *mmASNInPathString - cfg.MMGeoInPath = *mmGeoInPathString - cfg.Pfx2asInPath = *pfx2asInPathString - if *dnsServersString != "" { - cfg.DNSServers = strings.Split(*dnsServersString, " ") - } else { - cfg.DNSServers = make([]string, 0, 0) - } - cfg.PcapPaths = strings.Split(*pcapPathsString, " ") - cfg.ProfileCPUPath = *profileCPUPathString - cfg.ProfileMemPath = *profileMemPathString - cfg.Newdl = *newdlBool - - return &cfg -} - -func main() { - // Capture and log the amount of time the entire program took. - runStart := time.Now() - defer func() { - runEnd := time.Now() - log.Printf("Executed in %s.\n", runEnd.Sub(runStart).String()) - }() - - cfg := config() - - // Start CPU profiling if necessary. - if cfg.ProfileCPUPath != "" { - f, err := os.Create(cfg.ProfileCPUPath) - if err != nil { - log.Fatal("Could not create CPU profile file:", err) - } - defer f.Close() - pprof.StartCPUProfile(f) - defer pprof.StopCPUProfile() - } - // Create memory profile if necessary. - if cfg.ProfileMemPath != "" { - f, err := os.Create(cfg.ProfileMemPath) - if err != nil { - log.Fatal("Could not create memory profile file:", err) - } - defer func() { - defer f.Close() - runtime.GC() - err = pprof.WriteHeapProfile(f) - if err != nil { - log.Fatal("Could not write memory profile: ", err) - } - }() - } - - // Start an annotator (two if ongoing events are desired) and cache. - a := annotate.NewAnnotator(cfg.EventOutPrefix, cfg.MMASNInPath, - cfg.MMGeoInPath, cfg.Pfx2asInPath, cfg.DNSServers, - cfg.MinUniques, cfg.MinScanRate, cfg.Newdl) - var oa *annotate.Annotator - var c *cache.Cache - if cfg.OngoingEventOutPrefix != "" { - log.Println("Creating annotator for ongoing events.") - oa = annotate.NewAnnotator(cfg.OngoingEventOutPrefix, cfg.MMASNInPath, - cfg.MMGeoInPath, cfg.Pfx2asInPath, cfg.DNSServers, - cfg.MinUniques, cfg.MinScanRate, cfg.Newdl) - c = cache.NewCache(cfg.SenderExpiry, cfg.CacheInPath, cfg.CacheOutPath, - a.EventChannel, oa.EventChannel, cfg.NumSamples, cfg.MinUniques) - } else { - c = cache.NewCache(cfg.SenderExpiry, cfg.CacheInPath, cfg.CacheOutPath, - a.EventChannel, nil, cfg.NumSamples, cfg.MinUniques) - } - - // TODO: Remove the temporal dependence on cache and annotator close order. - defer a.Close() - if cfg.OngoingEventOutPrefix != "" { - defer oa.Close() - } - defer c.Close() - - d := decode.NewDecoder() - defer d.Close() - - // Check that all given pcap files exist before running main loop. - for _, path := range cfg.PcapPaths { - pfile, err := os.Open(path) - if err != nil { - log.Fatal("Failed to open pcap file: ", err) - } - pfile.Close() - } - - // Run analysis loop. - for _, path := range cfg.PcapPaths { - pfile, _ := os.Open(path) - defer pfile.Close() - handle, err := pcapgo.NewReader(pfile) - if err != nil { - log.Fatal("Failed to open pcap file: ", err) - } - - for { - read, meta, err := handle.ReadPacketData() - // TODO: Check sampling with: - // read, meta, err := handle.ZeroCopyReadPacketData() - if err != nil { - if err == io.EOF { - break - } - log.Fatal("Could not read packet data: ", err) - } - - // TODO: Is meta.CaptureLength == len(read)? - event, dest, time := d.Decode(read, meta) - c.Add(*event, dest, time, read) - } - - log.Printf("Decoded %s, cache size is %d bytes.\n", path, c.Size()) - } -} +// +// All software tools within this package are Copyright (c) 2020 Merit Network, Inc., +// and Stanford University. All Rights Reserved. +// + +package main + +import ( + "darknet-events/internal/annotate" + "darknet-events/internal/cache" + "darknet-events/internal/decode" + "flag" + "io" + "log" + "os" + "runtime" + "runtime/pprof" + "strings" + "time" + + "github.com/google/gopacket/pcapgo" +) + +// Config is a struct containing the program's configuration info. +type Config struct { + SenderExpiry int + MinUniques int + MinScanRate float64 + NumSamples int + CacheInPath string + CacheOutPath string + EventOutPrefix string + OngoingEventOutPrefix string + MMASNInPath string + MMGeoInPath string + Pfx2asInPath string + DNSServers []string + PcapPaths []string + ProfileCPUPath string + ProfileMemPath string + Newdl bool +} + +// config loads configuration information from the given flags. It is expected +// that this slice is os.Args. +func config() *Config { + var cfg Config + + threshold := flag.Int("threshold", 0, "Number of seconds that "+ + "must elapse before an event is considered over.") + minUniques := flag.Int("uniques", 1, "Minimum number of unique "+ + "destinations that must be hit for an event to be considered (must " + + "be a positive number).") + minScanRate := flag.Float64("rate", 0, "Minimum global packet rate for "+ + "an event to be considered.") + numSamples := flag.Int("samples", 0, "Number of packet samples "+ + "to save from each event.") + mmASNInPathString := flag.String("asnin", "", "Path to the MaxMind ASN "+ + "database.") + mmGeoInPathString := flag.String("geoin", "", "Path to the MaxMind "+ + "location database.") + pfx2asInPathString := flag.String("pfx2asin", "", "Path to the pfx2as "+ + "database.") + dnsServersString := flag.String("dns", "", "DNS server IP address(es).") + cacheInPathString := flag.String("cachein", "", "Path to load cache "+ + "entries from before analysis starts.") + cacheOutPathString := flag.String("cacheout", "", "Path to dump "+ + "remaining cache entries to for later use.") + eventOutPrefixString := flag.String("eventout", "", "Prefix of path to "+ + "save captured event data to.") + ongoingEventOutPrefixString := flag.String("ongoingout", "", "Prefix of "+ + "path to save captured event data to.") + pcapPathsString := flag.String("pcap", "", "Path to pcap files. Multiple "+ + "paths must be separated by a space.") + profileCPUPathString := flag.String("cpu", "", "Output path for CPU "+ + "profile. No profiling if not included.") + profileMemPathString := flag.String("mem", "", "Output path for memory "+ + "profile. No profiling if not included.") + newdlBool := flag.Bool("newdl", false, "True if JSON output needs to be "+ + "newline-delimited.") + flag.Parse() + + // Ensure that all non-optional flags are set. + if *threshold == 0 { + log.Fatal("Must specify a threshold timeout.") + } + if *numSamples == 0 { + log.Fatal("Must specify how many samples to take.") + } + if *eventOutPrefixString == "" { + log.Fatal("Must specify a location to write event data to.") + } + if *pcapPathsString == "" { + log.Fatal("Must specify at least one pcap file.") + } + if *minUniques <= 0 { + log.Fatal("Must specify a positive uniques number.") + } + + // Process and save parameters. + cfg.SenderExpiry = *threshold + cfg.MinUniques = *minUniques + cfg.MinScanRate = *minScanRate + cfg.NumSamples = *numSamples + cfg.CacheInPath = *cacheInPathString + cfg.CacheOutPath = *cacheOutPathString + cfg.EventOutPrefix = *eventOutPrefixString + cfg.OngoingEventOutPrefix = *ongoingEventOutPrefixString + cfg.MMASNInPath = *mmASNInPathString + cfg.MMGeoInPath = *mmGeoInPathString + cfg.Pfx2asInPath = *pfx2asInPathString + if *dnsServersString != "" { + cfg.DNSServers = strings.Split(*dnsServersString, " ") + } else { + cfg.DNSServers = make([]string, 0, 0) + } + cfg.PcapPaths = strings.Split(*pcapPathsString, " ") + cfg.ProfileCPUPath = *profileCPUPathString + cfg.ProfileMemPath = *profileMemPathString + cfg.Newdl = *newdlBool + + return &cfg +} + +func main() { + // Capture and log the amount of time the entire program took. + runStart := time.Now() + defer func() { + runEnd := time.Now() + log.Printf("Executed in %s.\n", runEnd.Sub(runStart).String()) + }() + + cfg := config() + + // Start CPU profiling if necessary. + if cfg.ProfileCPUPath != "" { + f, err := os.Create(cfg.ProfileCPUPath) + if err != nil { + log.Fatal("Could not create CPU profile file:", err) + } + defer f.Close() + pprof.StartCPUProfile(f) + defer pprof.StopCPUProfile() + } + // Create memory profile if necessary. + if cfg.ProfileMemPath != "" { + f, err := os.Create(cfg.ProfileMemPath) + if err != nil { + log.Fatal("Could not create memory profile file:", err) + } + defer func() { + defer f.Close() + runtime.GC() + err = pprof.WriteHeapProfile(f) + if err != nil { + log.Fatal("Could not write memory profile: ", err) + } + }() + } + + // Start an annotator (two if ongoing events are desired) and cache. + a := annotate.NewAnnotator(cfg.EventOutPrefix, cfg.MMASNInPath, + cfg.MMGeoInPath, cfg.Pfx2asInPath, cfg.DNSServers, + cfg.MinUniques, cfg.MinScanRate, cfg.Newdl) + var oa *annotate.Annotator + var c *cache.Cache + if cfg.OngoingEventOutPrefix != "" { + log.Println("Creating annotator for ongoing events.") + oa = annotate.NewAnnotator(cfg.OngoingEventOutPrefix, cfg.MMASNInPath, + cfg.MMGeoInPath, cfg.Pfx2asInPath, cfg.DNSServers, + cfg.MinUniques, cfg.MinScanRate, cfg.Newdl) + c = cache.NewCache(cfg.SenderExpiry, cfg.CacheInPath, cfg.CacheOutPath, + a.EventChannel, oa.EventChannel, cfg.NumSamples, cfg.MinUniques) + } else { + c = cache.NewCache(cfg.SenderExpiry, cfg.CacheInPath, cfg.CacheOutPath, + a.EventChannel, nil, cfg.NumSamples, cfg.MinUniques) + } + + // TODO: Remove the temporal dependence on cache and annotator close order. + defer a.Close() + if cfg.OngoingEventOutPrefix != "" { + defer oa.Close() + } + defer c.Close() + + d := decode.NewDecoder() + defer d.Close() + + // Check that all given pcap files exist before running main loop. + for _, path := range cfg.PcapPaths { + pfile, err := os.Open(path) + if err != nil { + log.Fatal("Failed to open pcap file: ", err) + } + pfile.Close() + } + + // Run analysis loop. + for _, path := range cfg.PcapPaths { + pfile, _ := os.Open(path) + defer pfile.Close() + handle, err := pcapgo.NewReader(pfile) + if err != nil { + log.Fatal("Failed to open pcap file: ", err) + } + + for { + read, meta, err := handle.ReadPacketData() + // TODO: Check sampling with: + // read, meta, err := handle.ZeroCopyReadPacketData() + if err != nil { + if err == io.EOF { + break + } + log.Fatal("Could not read packet data: ", err) + } + + // TODO: Is meta.CaptureLength == len(read)? + event, dest, time := d.Decode(read, meta) + c.Add(event, dest, time, read) + } + + log.Printf("Decoded %s, cache size is %d bytes.\n", path, c.Size()) + } +} diff --git a/go.sum b/go.sum index 1c8a45e..4f59ee7 100644 --- a/go.sum +++ b/go.sum @@ -1,8 +1,7 @@ github.com/asergeyev/nradix v0.0.0-20170505151046-3872ab85bb56 h1:Wi5Tgn8K+jDcBYL+dIMS1+qXYH2r7tpRAyBgqrWfQtw= github.com/asergeyev/nradix v0.0.0-20170505151046-3872ab85bb56/go.mod h1:8BhOLuqtSuT5NZtZMwfvEibi09RO3u79uqfHZzfDTR4= +github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/google/gopacket v1.1.17 h1:rMrlX2ZY2UbvT+sdz3+6J+pp2z+msCq9MxTU6ymxbBY= -github.com/google/gopacket v1.1.17/go.mod h1:UdDNZ1OO62aGYVnPhxT1U6aI7ukYtA/kB8vaU0diBUM= github.com/google/gopacket v1.1.19 h1:ves8RnFZPGiFnTS0uPQStjwru6uO6h+nlr9j6fL7kF8= github.com/google/gopacket v1.1.19/go.mod h1:iJ8V8n6KS+z2U1A8pUwu8bW5SyEMkXJB8Yo/Vo+TKTo= github.com/miekg/dns v1.1.27 h1:aEH/kqUzUxGJ/UHcEKdJY+ugH6WEzsEBBSPa8zuy1aM= @@ -13,8 +12,10 @@ github.com/oschwald/maxminddb-golang v1.6.0 h1:KAJSjdHQ8Kv45nFIbtoLGrGWqHFajOIm7 github.com/oschwald/maxminddb-golang v1.6.0/go.mod h1:DUJFucBg2cvqx42YmDa/+xHvb0elJtOm3o4aFQ/nb/w= github.com/philhofer/fwd v1.0.0 h1:UbZqGr5Y38ApvM/V/jEljVxwocdweyH+vmYvRPBnbqQ= github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/tinylib/msgp v1.1.2 h1:gWmO7n0Ys2RBEb7GPYB9Ujq8Mk5p2U08lRnmMcGy6BQ= github.com/tinylib/msgp v1.1.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= @@ -31,9 +32,9 @@ golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20200226121028-0de0cce0169b h1:0mm1VjtFUOIlE1SbDlwjYaDxZVDP2S5ou6y0gSgXHu8= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e h1:vcxGaoTs7kV8m5Np9uUNQin4BrLOthgV7252N8V+FwY= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190405154228-4b34438f7a67/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191224085550-c709ea063b76/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -43,5 +44,7 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/tools v0.0.0-20191216052735-49a3e744a425/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/internal/analysis/analysis_gen.go b/internal/analysis/analysis_gen.go index a31cf06..8d1acea 100644 --- a/internal/analysis/analysis_gen.go +++ b/internal/analysis/analysis_gen.go @@ -1,6 +1,8 @@ package analysis -// Code generated by github.com/tinylib/msgp DO NOT EDIT. +// NOTE: THIS FILE WAS PRODUCED BY THE +// MSGP CODE GENERATION TOOL (github.com/tinylib/msgp) +// DO NOT EDIT import ( "github.com/tinylib/msgp/msgp" @@ -9,13 +11,12 @@ import ( // DecodeMsg implements msgp.Decodable func (z *TrafficType) DecodeMsg(dc *msgp.Reader) (err error) { { - var zb0001 uint16 - zb0001, err = dc.ReadUint16() - if err != nil { - err = msgp.WrapError(err) - return - } - (*z) = TrafficType(zb0001) + var zxvk uint16 + zxvk, err = dc.ReadUint16() + (*z) = TrafficType(zxvk) + } + if err != nil { + return } return } @@ -24,7 +25,6 @@ func (z *TrafficType) DecodeMsg(dc *msgp.Reader) (err error) { func (z TrafficType) EncodeMsg(en *msgp.Writer) (err error) { err = en.WriteUint16(uint16(z)) if err != nil { - err = msgp.WrapError(err) return } return @@ -40,13 +40,12 @@ func (z TrafficType) MarshalMsg(b []byte) (o []byte, err error) { // UnmarshalMsg implements msgp.Unmarshaler func (z *TrafficType) UnmarshalMsg(bts []byte) (o []byte, err error) { { - var zb0001 uint16 - zb0001, bts, err = msgp.ReadUint16Bytes(bts) - if err != nil { - err = msgp.WrapError(err) - return - } - (*z) = TrafficType(zb0001) + var zbzg uint16 + zbzg, bts, err = msgp.ReadUint16Bytes(bts) + (*z) = TrafficType(zbzg) + } + if err != nil { + return } o = bts return diff --git a/internal/analysis/analysis_gen_test.go b/internal/analysis/analysis_gen_test.go index 27ef632..b840548 100644 --- a/internal/analysis/analysis_gen_test.go +++ b/internal/analysis/analysis_gen_test.go @@ -1,3 +1,5 @@ package analysis -// Code generated by github.com/tinylib/msgp DO NOT EDIT. +// NOTE: THIS FILE WAS PRODUCED BY THE +// MSGP CODE GENERATION TOOL (github.com/tinylib/msgp) +// DO NOT EDIT diff --git a/internal/analysis/event.go b/internal/analysis/event.go index c7da879..67564ca 100644 --- a/internal/analysis/event.go +++ b/internal/analysis/event.go @@ -4,6 +4,7 @@ package analysis import ( "encoding/binary" + "github.com/tinylib/msgp/msgp" "net" "time" "unsafe" @@ -11,29 +12,106 @@ import ( "darknet-events/internal/set" ) +type EventSignature interface { + GetPort() uint16 + GetTraffic() TrafficType + + // Generated by msgp + DecodeMsg(dc *msgp.Reader) error + EncodeMsg(en *msgp.Writer) error + MarshalMsg(o []byte) ([]byte, error) + UnmarshalMsg(bts []byte) ([]byte, error) + Msgsize() int +} + // EventSignature is the data structure used to associate different packets to // a single source and/or being of the same event. -type EventSignature struct { - SourceIP uint32 +type EventSignatureIPv4 struct { + SourceIPv4 uint32 + Port uint16 + Traffic TrafficType +} + +type EventSignatureIPv6 struct { + SourceIPv6 [16]byte Port uint16 Traffic TrafficType } -// NewEventSignature returns a new EventSignature from the given data. -func NewEventSignature(sourceIP net.IP, - p uint16, t TrafficType) *EventSignature { +func NewEventSignatureIPv4(sourceIP net.IP, + p uint16, t TrafficType) EventSignatureIPv4 { + sourceIPv4 := binary.BigEndian.Uint32(sourceIP.To4()) + + es := new(EventSignatureIPv4) + + es.SourceIPv4 = sourceIPv4 + es.Port = p + es.Traffic = t + + return *es +} + +func NewEventSignatureIPv6(sourceIP net.IP, + p uint16, t TrafficType) EventSignatureIPv6 { + var sourceIPv6 [16]byte + copy(sourceIPv6[:], sourceIP) + + es := new(EventSignatureIPv6) + + es.SourceIPv6 = sourceIPv6 + es.Port = p + es.Traffic = t + + return *es +} + +func (es EventSignatureIPv4) GetPort() uint16 { + return es.Port +} + +func (es EventSignatureIPv6) GetPort() uint16 { + return es.Port +} + +func (es EventSignatureIPv4) GetTraffic() TrafficType { + return es.Traffic +} + +func (es EventSignatureIPv6) GetTraffic() TrafficType { + return es.Traffic +} - // The darknet only saves IPv4 packets so calling To4() is safe. - sourceIPInt := binary.BigEndian.Uint32(sourceIP.To4()) +type EventPackets interface { + Add(destIP net.IP, b uint64, t time.Time) int + AddSample(i int, raw []byte) + Size() uintptr + GetFirst() time.Time + GetLatest() time.Time + GetPackets() uint64 + GetBytes() uint64 + GetSamples() [][]byte - es := EventSignature{SourceIP: sourceIPInt, Port: p, Traffic: t} - return &es + // Generated by msgp + DecodeMsg(dc *msgp.Reader) error + EncodeMsg(en *msgp.Writer) error + MarshalMsg(o []byte) ([]byte, error) + UnmarshalMsg(bts []byte) ([]byte, error) + Msgsize() int } // EventPackets collects the data pulled from multiple packets of the same // event. -type EventPackets struct { - Dests *set.Uint32Set +type EventPacketsIPv4 struct { + DestIPv4 *set.Uint32Set // for IPv4 + First time.Time + Latest time.Time + Packets uint64 + Bytes uint64 + Samples [][]byte +} + +type EventPacketsIPv6 struct { + DestIPv6 *set.IPSet // for IPv6 First time.Time Latest time.Time Packets uint64 @@ -42,26 +120,56 @@ type EventPackets struct { } // NewEventPackets returns a new EventPackets object. -func NewEventPackets() *EventPackets { - ep := new(EventPackets) - ep.Dests = set.NewUint32Set() +func NewEventPacketsIPv4() *EventPacketsIPv4 { + ep := new(EventPacketsIPv4) + + ep.DestIPv4 = set.NewUint32Set() ep.Samples = make([][]byte, 0, 1) + + return ep +} + +func NewEventPacketsIPv6() *EventPacketsIPv6 { + ep := new(EventPacketsIPv6) + + ep.DestIPv6 = set.NewIPSet() + ep.Samples = make([][]byte, 0, 1) + return ep } // Add adds the destination IP to the packet collection object and returns the // index it would have been added at (they're actually added to a set). -func (ep *EventPackets) Add(ip uint32, b uint64, t time.Time) int { - ep.Dests.Add(ip) +func (ep *EventPacketsIPv4) Add(destIP net.IP, b uint64, t time.Time) int { + destIPv4 := binary.BigEndian.Uint32(destIP.To4()) + ep.DestIPv4.Add(destIPv4) + if ep.First.IsZero() { ep.First = t } + ep.Latest = t ep.Packets++ ep.Bytes += b return int(ep.Packets) - 1 } +func (ep *EventPacketsIPv6) Add(destIP net.IP, b uint64, t time.Time) int { + var destIPv6 [16]byte + copy(destIPv6[:], destIP) + ep.DestIPv6.Add(destIPv6) + + if ep.First.IsZero() { + ep.First = t + } + + ep.Latest = t + ep.Packets++ + ep.Bytes += b + + return int(ep.Packets) - 1 +} + /* // Append appends the given packet to the EventPackets object and returns the // index that it was appended at. @@ -77,7 +185,15 @@ func (ep *EventPackets) Append(p Packet, t time.Time) int { // AddSample adds a raw packet to an EventPackets object's collections of // samples at the given index -func (ep *EventPackets) AddSample(i int, raw []byte) { +func (ep *EventPacketsIPv4) AddSample(i int, raw []byte) { + if len(ep.Samples) < i+1 { + ep.Samples = append(ep.Samples, raw) + } else { + ep.Samples[i] = raw + } +} + +func (ep *EventPacketsIPv6) AddSample(i int, raw []byte) { if len(ep.Samples) < i+1 { ep.Samples = append(ep.Samples, raw) } else { @@ -87,25 +203,80 @@ func (ep *EventPackets) AddSample(i int, raw []byte) { // Size returns the number of bytes of memory consumed by the EventPackets // object. -func (ep *EventPackets) Size() uintptr { +func (ep *EventPacketsIPv4) Size() uintptr { + var size uintptr + size += unsafe.Sizeof(ep) + size += uintptr(ep.DestIPv4.Size()) + + for _, s := range ep.Samples { + size += uintptr(len(s)) + } + + return size +} + +func (ep *EventPacketsIPv6) Size() uintptr { var size uintptr size += unsafe.Sizeof(ep) - size += uintptr(ep.Dests.ByteSize()) + size += uintptr(ep.DestIPv6.Size()) + for _, s := range ep.Samples { size += uintptr(len(s)) } + return size } + +func (ep *EventPacketsIPv4) GetFirst() time.Time { + return ep.First +} + +func (ep *EventPacketsIPv6) GetFirst() time.Time { + return ep.First +} + +func (ep *EventPacketsIPv4) GetLatest() time.Time { + return ep.Latest +} + +func (ep *EventPacketsIPv6) GetLatest() time.Time { + return ep.Latest +} + +func (ep *EventPacketsIPv4) GetPackets() uint64 { + return ep.Packets +} + +func (ep *EventPacketsIPv6) GetPackets() uint64 { + return ep.Packets +} + +func (ep *EventPacketsIPv4) GetBytes() uint64 { + return ep.Bytes +} + +func (ep *EventPacketsIPv6) GetBytes() uint64 { + return ep.Bytes +} + +func (ep *EventPacketsIPv4) GetSamples() [][]byte { + return ep.Samples +} + +func (ep *EventPacketsIPv6) GetSamples() [][]byte { + return ep.Samples +} + // Event combines an event signature and its collection of packets together. type Event struct { Signature EventSignature - Packets *EventPackets + Packets EventPackets } // NewEvent creates an Event object from the given EventSignature and // EventPackets objects. -func NewEvent(es EventSignature, ep *EventPackets) *Event { +func NewEvent(es EventSignature, ep EventPackets) *Event { e := Event{Signature: es, Packets: ep} return &e } diff --git a/internal/analysis/event_gen.go b/internal/analysis/event_gen.go index 32e9347..7a120dc 100644 --- a/internal/analysis/event_gen.go +++ b/internal/analysis/event_gen.go @@ -1,6 +1,8 @@ package analysis -// Code generated by github.com/tinylib/msgp DO NOT EDIT. +// NOTE: THIS FILE WAS PRODUCED BY THE +// MSGP CODE GENERATION TOOL (github.com/tinylib/msgp) +// DO NOT EDIT import ( "darknet-events/internal/set" @@ -12,83 +14,196 @@ import ( func (z *Event) DecodeMsg(dc *msgp.Reader) (err error) { var field []byte _ = field - var zb0001 uint32 - zb0001, err = dc.ReadMapHeader() + var zxvk uint32 + zxvk, err = dc.ReadMapHeader() if err != nil { - err = msgp.WrapError(err) return } - for zb0001 > 0 { - zb0001-- + for zxvk > 0 { + zxvk-- field, err = dc.ReadMapKeyPtr() if err != nil { - err = msgp.WrapError(err) return } switch msgp.UnsafeString(field) { case "Signature": - var zb0002 uint32 - zb0002, err = dc.ReadMapHeader() + err = z.Signature.DecodeMsg(dc) if err != nil { - err = msgp.WrapError(err, "Signature") return } - for zb0002 > 0 { - zb0002-- - field, err = dc.ReadMapKeyPtr() - if err != nil { - err = msgp.WrapError(err, "Signature") - return - } - switch msgp.UnsafeString(field) { - case "SourceIP": - z.Signature.SourceIP, err = dc.ReadUint32() - if err != nil { - err = msgp.WrapError(err, "Signature", "SourceIP") - return - } - case "Port": - z.Signature.Port, err = dc.ReadUint16() - if err != nil { - err = msgp.WrapError(err, "Signature", "Port") - return - } - case "Traffic": - err = z.Signature.Traffic.DecodeMsg(dc) - if err != nil { - err = msgp.WrapError(err, "Signature", "Traffic") - return - } - default: - err = dc.Skip() - if err != nil { - err = msgp.WrapError(err, "Signature") - return - } - } + case "Packets": + err = z.Packets.DecodeMsg(dc) + if err != nil { + return + } + default: + err = dc.Skip() + if err != nil { + return + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z *Event) EncodeMsg(en *msgp.Writer) (err error) { + // map header, size 2 + // write "Signature" + err = en.Append(0x82, 0xa9, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65) + if err != nil { + return err + } + err = z.Signature.EncodeMsg(en) + if err != nil { + return + } + // write "Packets" + err = en.Append(0xa7, 0x50, 0x61, 0x63, 0x6b, 0x65, 0x74, 0x73) + if err != nil { + return err + } + err = z.Packets.EncodeMsg(en) + if err != nil { + return + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z *Event) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 2 + // string "Signature" + o = append(o, 0x82, 0xa9, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65) + o, err = z.Signature.MarshalMsg(o) + if err != nil { + return + } + // string "Packets" + o = append(o, 0xa7, 0x50, 0x61, 0x63, 0x6b, 0x65, 0x74, 0x73) + o, err = z.Packets.MarshalMsg(o) + if err != nil { + return + } + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *Event) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zbzg uint32 + zbzg, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + return + } + for zbzg > 0 { + zbzg-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + return + } + switch msgp.UnsafeString(field) { + case "Signature": + bts, err = z.Signature.UnmarshalMsg(bts) + if err != nil { + return } case "Packets": + bts, err = z.Packets.UnmarshalMsg(bts) + if err != nil { + return + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *Event) Msgsize() (s int) { + s = 1 + 10 + z.Signature.Msgsize() + 8 + z.Packets.Msgsize() + return +} + +// DecodeMsg implements msgp.Decodable +func (z *EventPacketsIPv4) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zcmr uint32 + zcmr, err = dc.ReadMapHeader() + if err != nil { + return + } + for zcmr > 0 { + zcmr-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + return + } + switch msgp.UnsafeString(field) { + case "DestIPv4": if dc.IsNil() { err = dc.ReadNil() if err != nil { - err = msgp.WrapError(err, "Packets") return } - z.Packets = nil + z.DestIPv4 = nil } else { - if z.Packets == nil { - z.Packets = new(EventPackets) + if z.DestIPv4 == nil { + z.DestIPv4 = new(set.Uint32Set) } - err = z.Packets.DecodeMsg(dc) + err = z.DestIPv4.DecodeMsg(dc) + if err != nil { + return + } + } + case "First": + z.First, err = dc.ReadTime() + if err != nil { + return + } + case "Latest": + z.Latest, err = dc.ReadTime() + if err != nil { + return + } + case "Packets": + z.Packets, err = dc.ReadUint64() + if err != nil { + return + } + case "Bytes": + z.Bytes, err = dc.ReadUint64() + if err != nil { + return + } + case "Samples": + var zajw uint32 + zajw, err = dc.ReadArrayHeader() + if err != nil { + return + } + if cap(z.Samples) >= int(zajw) { + z.Samples = (z.Samples)[:zajw] + } else { + z.Samples = make([][]byte, zajw) + } + for zbai := range z.Samples { + z.Samples[zbai], err = dc.ReadBytes(z.Samples[zbai]) if err != nil { - err = msgp.WrapError(err, "Packets") return } } default: err = dc.Skip() if err != nil { - err = msgp.WrapError(err) return } } @@ -97,178 +212,185 @@ func (z *Event) DecodeMsg(dc *msgp.Reader) (err error) { } // EncodeMsg implements msgp.Encodable -func (z *Event) EncodeMsg(en *msgp.Writer) (err error) { - // map header, size 2 - // write "Signature" - err = en.Append(0x82, 0xa9, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65) +func (z *EventPacketsIPv4) EncodeMsg(en *msgp.Writer) (err error) { + // map header, size 6 + // write "DestIPv4" + err = en.Append(0x86, 0xa8, 0x44, 0x65, 0x73, 0x74, 0x49, 0x50, 0x76, 0x34) if err != nil { - return + return err } - // map header, size 3 - // write "SourceIP" - err = en.Append(0x83, 0xa8, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x49, 0x50) + if z.DestIPv4 == nil { + err = en.WriteNil() + if err != nil { + return + } + } else { + err = z.DestIPv4.EncodeMsg(en) + if err != nil { + return + } + } + // write "First" + err = en.Append(0xa5, 0x46, 0x69, 0x72, 0x73, 0x74) if err != nil { - return + return err } - err = en.WriteUint32(z.Signature.SourceIP) + err = en.WriteTime(z.First) if err != nil { - err = msgp.WrapError(err, "Signature", "SourceIP") return } - // write "Port" - err = en.Append(0xa4, 0x50, 0x6f, 0x72, 0x74) + // write "Latest" + err = en.Append(0xa6, 0x4c, 0x61, 0x74, 0x65, 0x73, 0x74) if err != nil { - return + return err } - err = en.WriteUint16(z.Signature.Port) + err = en.WriteTime(z.Latest) if err != nil { - err = msgp.WrapError(err, "Signature", "Port") return } - // write "Traffic" - err = en.Append(0xa7, 0x54, 0x72, 0x61, 0x66, 0x66, 0x69, 0x63) + // write "Packets" + err = en.Append(0xa7, 0x50, 0x61, 0x63, 0x6b, 0x65, 0x74, 0x73) + if err != nil { + return err + } + err = en.WriteUint64(z.Packets) if err != nil { return } - err = z.Signature.Traffic.EncodeMsg(en) + // write "Bytes" + err = en.Append(0xa5, 0x42, 0x79, 0x74, 0x65, 0x73) + if err != nil { + return err + } + err = en.WriteUint64(z.Bytes) if err != nil { - err = msgp.WrapError(err, "Signature", "Traffic") return } - // write "Packets" - err = en.Append(0xa7, 0x50, 0x61, 0x63, 0x6b, 0x65, 0x74, 0x73) + // write "Samples" + err = en.Append(0xa7, 0x53, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x73) + if err != nil { + return err + } + err = en.WriteArrayHeader(uint32(len(z.Samples))) if err != nil { return } - if z.Packets == nil { - err = en.WriteNil() + for zbai := range z.Samples { + err = en.WriteBytes(z.Samples[zbai]) if err != nil { return } - } else { - err = z.Packets.EncodeMsg(en) - if err != nil { - err = msgp.WrapError(err, "Packets") - return - } } return } // MarshalMsg implements msgp.Marshaler -func (z *Event) MarshalMsg(b []byte) (o []byte, err error) { +func (z *EventPacketsIPv4) MarshalMsg(b []byte) (o []byte, err error) { o = msgp.Require(b, z.Msgsize()) - // map header, size 2 - // string "Signature" - o = append(o, 0x82, 0xa9, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65) - // map header, size 3 - // string "SourceIP" - o = append(o, 0x83, 0xa8, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x49, 0x50) - o = msgp.AppendUint32(o, z.Signature.SourceIP) - // string "Port" - o = append(o, 0xa4, 0x50, 0x6f, 0x72, 0x74) - o = msgp.AppendUint16(o, z.Signature.Port) - // string "Traffic" - o = append(o, 0xa7, 0x54, 0x72, 0x61, 0x66, 0x66, 0x69, 0x63) - o, err = z.Signature.Traffic.MarshalMsg(o) - if err != nil { - err = msgp.WrapError(err, "Signature", "Traffic") - return - } - // string "Packets" - o = append(o, 0xa7, 0x50, 0x61, 0x63, 0x6b, 0x65, 0x74, 0x73) - if z.Packets == nil { + // map header, size 6 + // string "DestIPv4" + o = append(o, 0x86, 0xa8, 0x44, 0x65, 0x73, 0x74, 0x49, 0x50, 0x76, 0x34) + if z.DestIPv4 == nil { o = msgp.AppendNil(o) } else { - o, err = z.Packets.MarshalMsg(o) + o, err = z.DestIPv4.MarshalMsg(o) if err != nil { - err = msgp.WrapError(err, "Packets") return } } + // string "First" + o = append(o, 0xa5, 0x46, 0x69, 0x72, 0x73, 0x74) + o = msgp.AppendTime(o, z.First) + // string "Latest" + o = append(o, 0xa6, 0x4c, 0x61, 0x74, 0x65, 0x73, 0x74) + o = msgp.AppendTime(o, z.Latest) + // string "Packets" + o = append(o, 0xa7, 0x50, 0x61, 0x63, 0x6b, 0x65, 0x74, 0x73) + o = msgp.AppendUint64(o, z.Packets) + // string "Bytes" + o = append(o, 0xa5, 0x42, 0x79, 0x74, 0x65, 0x73) + o = msgp.AppendUint64(o, z.Bytes) + // string "Samples" + o = append(o, 0xa7, 0x53, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x73) + o = msgp.AppendArrayHeader(o, uint32(len(z.Samples))) + for zbai := range z.Samples { + o = msgp.AppendBytes(o, z.Samples[zbai]) + } return } // UnmarshalMsg implements msgp.Unmarshaler -func (z *Event) UnmarshalMsg(bts []byte) (o []byte, err error) { +func (z *EventPacketsIPv4) UnmarshalMsg(bts []byte) (o []byte, err error) { var field []byte _ = field - var zb0001 uint32 - zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + var zwht uint32 + zwht, bts, err = msgp.ReadMapHeaderBytes(bts) if err != nil { - err = msgp.WrapError(err) return } - for zb0001 > 0 { - zb0001-- + for zwht > 0 { + zwht-- field, bts, err = msgp.ReadMapKeyZC(bts) if err != nil { - err = msgp.WrapError(err) return } switch msgp.UnsafeString(field) { - case "Signature": - var zb0002 uint32 - zb0002, bts, err = msgp.ReadMapHeaderBytes(bts) - if err != nil { - err = msgp.WrapError(err, "Signature") - return - } - for zb0002 > 0 { - zb0002-- - field, bts, err = msgp.ReadMapKeyZC(bts) + case "DestIPv4": + if msgp.IsNil(bts) { + bts, err = msgp.ReadNilBytes(bts) if err != nil { - err = msgp.WrapError(err, "Signature") return } - switch msgp.UnsafeString(field) { - case "SourceIP": - z.Signature.SourceIP, bts, err = msgp.ReadUint32Bytes(bts) - if err != nil { - err = msgp.WrapError(err, "Signature", "SourceIP") - return - } - case "Port": - z.Signature.Port, bts, err = msgp.ReadUint16Bytes(bts) - if err != nil { - err = msgp.WrapError(err, "Signature", "Port") - return - } - case "Traffic": - bts, err = z.Signature.Traffic.UnmarshalMsg(bts) - if err != nil { - err = msgp.WrapError(err, "Signature", "Traffic") - return - } - default: - bts, err = msgp.Skip(bts) - if err != nil { - err = msgp.WrapError(err, "Signature") - return - } + z.DestIPv4 = nil + } else { + if z.DestIPv4 == nil { + z.DestIPv4 = new(set.Uint32Set) } - } - case "Packets": - if msgp.IsNil(bts) { - bts, err = msgp.ReadNilBytes(bts) + bts, err = z.DestIPv4.UnmarshalMsg(bts) if err != nil { return } - z.Packets = nil + } + case "First": + z.First, bts, err = msgp.ReadTimeBytes(bts) + if err != nil { + return + } + case "Latest": + z.Latest, bts, err = msgp.ReadTimeBytes(bts) + if err != nil { + return + } + case "Packets": + z.Packets, bts, err = msgp.ReadUint64Bytes(bts) + if err != nil { + return + } + case "Bytes": + z.Bytes, bts, err = msgp.ReadUint64Bytes(bts) + if err != nil { + return + } + case "Samples": + var zhct uint32 + zhct, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + return + } + if cap(z.Samples) >= int(zhct) { + z.Samples = (z.Samples)[:zhct] } else { - if z.Packets == nil { - z.Packets = new(EventPackets) - } - bts, err = z.Packets.UnmarshalMsg(bts) + z.Samples = make([][]byte, zhct) + } + for zbai := range z.Samples { + z.Samples[zbai], bts, err = msgp.ReadBytesBytes(bts, z.Samples[zbai]) if err != nil { - err = msgp.WrapError(err, "Packets") return } } default: bts, err = msgp.Skip(bts) if err != nil { - err = msgp.WrapError(err) return } } @@ -278,99 +400,92 @@ func (z *Event) UnmarshalMsg(bts []byte) (o []byte, err error) { } // Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message -func (z *Event) Msgsize() (s int) { - s = 1 + 10 + 1 + 9 + msgp.Uint32Size + 5 + msgp.Uint16Size + 8 + z.Signature.Traffic.Msgsize() + 8 - if z.Packets == nil { +func (z *EventPacketsIPv4) Msgsize() (s int) { + s = 1 + 9 + if z.DestIPv4 == nil { s += msgp.NilSize } else { - s += z.Packets.Msgsize() + s += z.DestIPv4.Msgsize() + } + s += 6 + msgp.TimeSize + 7 + msgp.TimeSize + 8 + msgp.Uint64Size + 6 + msgp.Uint64Size + 8 + msgp.ArrayHeaderSize + for zbai := range z.Samples { + s += msgp.BytesPrefixSize + len(z.Samples[zbai]) } return } // DecodeMsg implements msgp.Decodable -func (z *EventPackets) DecodeMsg(dc *msgp.Reader) (err error) { +func (z *EventPacketsIPv6) DecodeMsg(dc *msgp.Reader) (err error) { var field []byte _ = field - var zb0001 uint32 - zb0001, err = dc.ReadMapHeader() + var zxhx uint32 + zxhx, err = dc.ReadMapHeader() if err != nil { - err = msgp.WrapError(err) return } - for zb0001 > 0 { - zb0001-- + for zxhx > 0 { + zxhx-- field, err = dc.ReadMapKeyPtr() if err != nil { - err = msgp.WrapError(err) return } switch msgp.UnsafeString(field) { - case "Dests": + case "DestIPv6": if dc.IsNil() { err = dc.ReadNil() if err != nil { - err = msgp.WrapError(err, "Dests") return } - z.Dests = nil + z.DestIPv6 = nil } else { - if z.Dests == nil { - z.Dests = new(set.Uint32Set) + if z.DestIPv6 == nil { + z.DestIPv6 = new(set.IPSet) } - err = z.Dests.DecodeMsg(dc) + err = z.DestIPv6.DecodeMsg(dc) if err != nil { - err = msgp.WrapError(err, "Dests") return } } case "First": z.First, err = dc.ReadTime() if err != nil { - err = msgp.WrapError(err, "First") return } case "Latest": z.Latest, err = dc.ReadTime() if err != nil { - err = msgp.WrapError(err, "Latest") return } case "Packets": z.Packets, err = dc.ReadUint64() if err != nil { - err = msgp.WrapError(err, "Packets") return } case "Bytes": z.Bytes, err = dc.ReadUint64() if err != nil { - err = msgp.WrapError(err, "Bytes") return } case "Samples": - var zb0002 uint32 - zb0002, err = dc.ReadArrayHeader() + var zlqf uint32 + zlqf, err = dc.ReadArrayHeader() if err != nil { - err = msgp.WrapError(err, "Samples") return } - if cap(z.Samples) >= int(zb0002) { - z.Samples = (z.Samples)[:zb0002] + if cap(z.Samples) >= int(zlqf) { + z.Samples = (z.Samples)[:zlqf] } else { - z.Samples = make([][]byte, zb0002) + z.Samples = make([][]byte, zlqf) } - for za0001 := range z.Samples { - z.Samples[za0001], err = dc.ReadBytes(z.Samples[za0001]) + for zcua := range z.Samples { + z.Samples[zcua], err = dc.ReadBytes(z.Samples[zcua]) if err != nil { - err = msgp.WrapError(err, "Samples", za0001) return } } default: err = dc.Skip() if err != nil { - err = msgp.WrapError(err) return } } @@ -379,79 +494,72 @@ func (z *EventPackets) DecodeMsg(dc *msgp.Reader) (err error) { } // EncodeMsg implements msgp.Encodable -func (z *EventPackets) EncodeMsg(en *msgp.Writer) (err error) { +func (z *EventPacketsIPv6) EncodeMsg(en *msgp.Writer) (err error) { // map header, size 6 - // write "Dests" - err = en.Append(0x86, 0xa5, 0x44, 0x65, 0x73, 0x74, 0x73) + // write "DestIPv6" + err = en.Append(0x86, 0xa8, 0x44, 0x65, 0x73, 0x74, 0x49, 0x50, 0x76, 0x36) if err != nil { - return + return err } - if z.Dests == nil { + if z.DestIPv6 == nil { err = en.WriteNil() if err != nil { return } } else { - err = z.Dests.EncodeMsg(en) + err = z.DestIPv6.EncodeMsg(en) if err != nil { - err = msgp.WrapError(err, "Dests") return } } // write "First" err = en.Append(0xa5, 0x46, 0x69, 0x72, 0x73, 0x74) if err != nil { - return + return err } err = en.WriteTime(z.First) if err != nil { - err = msgp.WrapError(err, "First") return } // write "Latest" err = en.Append(0xa6, 0x4c, 0x61, 0x74, 0x65, 0x73, 0x74) if err != nil { - return + return err } err = en.WriteTime(z.Latest) if err != nil { - err = msgp.WrapError(err, "Latest") return } // write "Packets" err = en.Append(0xa7, 0x50, 0x61, 0x63, 0x6b, 0x65, 0x74, 0x73) if err != nil { - return + return err } err = en.WriteUint64(z.Packets) if err != nil { - err = msgp.WrapError(err, "Packets") return } // write "Bytes" err = en.Append(0xa5, 0x42, 0x79, 0x74, 0x65, 0x73) if err != nil { - return + return err } err = en.WriteUint64(z.Bytes) if err != nil { - err = msgp.WrapError(err, "Bytes") return } // write "Samples" err = en.Append(0xa7, 0x53, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x73) if err != nil { - return + return err } err = en.WriteArrayHeader(uint32(len(z.Samples))) if err != nil { - err = msgp.WrapError(err, "Samples") return } - for za0001 := range z.Samples { - err = en.WriteBytes(z.Samples[za0001]) + for zcua := range z.Samples { + err = en.WriteBytes(z.Samples[zcua]) if err != nil { - err = msgp.WrapError(err, "Samples", za0001) return } } @@ -459,17 +567,16 @@ func (z *EventPackets) EncodeMsg(en *msgp.Writer) (err error) { } // MarshalMsg implements msgp.Marshaler -func (z *EventPackets) MarshalMsg(b []byte) (o []byte, err error) { +func (z *EventPacketsIPv6) MarshalMsg(b []byte) (o []byte, err error) { o = msgp.Require(b, z.Msgsize()) // map header, size 6 - // string "Dests" - o = append(o, 0x86, 0xa5, 0x44, 0x65, 0x73, 0x74, 0x73) - if z.Dests == nil { + // string "DestIPv6" + o = append(o, 0x86, 0xa8, 0x44, 0x65, 0x73, 0x74, 0x49, 0x50, 0x76, 0x36) + if z.DestIPv6 == nil { o = msgp.AppendNil(o) } else { - o, err = z.Dests.MarshalMsg(o) + o, err = z.DestIPv6.MarshalMsg(o) if err != nil { - err = msgp.WrapError(err, "Dests") return } } @@ -488,94 +595,84 @@ func (z *EventPackets) MarshalMsg(b []byte) (o []byte, err error) { // string "Samples" o = append(o, 0xa7, 0x53, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x73) o = msgp.AppendArrayHeader(o, uint32(len(z.Samples))) - for za0001 := range z.Samples { - o = msgp.AppendBytes(o, z.Samples[za0001]) + for zcua := range z.Samples { + o = msgp.AppendBytes(o, z.Samples[zcua]) } return } // UnmarshalMsg implements msgp.Unmarshaler -func (z *EventPackets) UnmarshalMsg(bts []byte) (o []byte, err error) { +func (z *EventPacketsIPv6) UnmarshalMsg(bts []byte) (o []byte, err error) { var field []byte _ = field - var zb0001 uint32 - zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + var zdaf uint32 + zdaf, bts, err = msgp.ReadMapHeaderBytes(bts) if err != nil { - err = msgp.WrapError(err) return } - for zb0001 > 0 { - zb0001-- + for zdaf > 0 { + zdaf-- field, bts, err = msgp.ReadMapKeyZC(bts) if err != nil { - err = msgp.WrapError(err) return } switch msgp.UnsafeString(field) { - case "Dests": + case "DestIPv6": if msgp.IsNil(bts) { bts, err = msgp.ReadNilBytes(bts) if err != nil { return } - z.Dests = nil + z.DestIPv6 = nil } else { - if z.Dests == nil { - z.Dests = new(set.Uint32Set) + if z.DestIPv6 == nil { + z.DestIPv6 = new(set.IPSet) } - bts, err = z.Dests.UnmarshalMsg(bts) + bts, err = z.DestIPv6.UnmarshalMsg(bts) if err != nil { - err = msgp.WrapError(err, "Dests") return } } case "First": z.First, bts, err = msgp.ReadTimeBytes(bts) if err != nil { - err = msgp.WrapError(err, "First") return } case "Latest": z.Latest, bts, err = msgp.ReadTimeBytes(bts) if err != nil { - err = msgp.WrapError(err, "Latest") return } case "Packets": z.Packets, bts, err = msgp.ReadUint64Bytes(bts) if err != nil { - err = msgp.WrapError(err, "Packets") return } case "Bytes": z.Bytes, bts, err = msgp.ReadUint64Bytes(bts) if err != nil { - err = msgp.WrapError(err, "Bytes") return } case "Samples": - var zb0002 uint32 - zb0002, bts, err = msgp.ReadArrayHeaderBytes(bts) + var zpks uint32 + zpks, bts, err = msgp.ReadArrayHeaderBytes(bts) if err != nil { - err = msgp.WrapError(err, "Samples") return } - if cap(z.Samples) >= int(zb0002) { - z.Samples = (z.Samples)[:zb0002] + if cap(z.Samples) >= int(zpks) { + z.Samples = (z.Samples)[:zpks] } else { - z.Samples = make([][]byte, zb0002) + z.Samples = make([][]byte, zpks) } - for za0001 := range z.Samples { - z.Samples[za0001], bts, err = msgp.ReadBytesBytes(bts, z.Samples[za0001]) + for zcua := range z.Samples { + z.Samples[zcua], bts, err = msgp.ReadBytesBytes(bts, z.Samples[zcua]) if err != nil { - err = msgp.WrapError(err, "Samples", za0001) return } } default: bts, err = msgp.Skip(bts) if err != nil { - err = msgp.WrapError(err) return } } @@ -585,60 +682,54 @@ func (z *EventPackets) UnmarshalMsg(bts []byte) (o []byte, err error) { } // Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message -func (z *EventPackets) Msgsize() (s int) { - s = 1 + 6 - if z.Dests == nil { +func (z *EventPacketsIPv6) Msgsize() (s int) { + s = 1 + 9 + if z.DestIPv6 == nil { s += msgp.NilSize } else { - s += z.Dests.Msgsize() + s += z.DestIPv6.Msgsize() } - s += 6 + msgp.TimeSize + 7 + msgp.TimeSize + 8 + msgp.Uint32Size + 6 + msgp.Uint32Size + 8 + msgp.ArrayHeaderSize - for za0001 := range z.Samples { - s += msgp.BytesPrefixSize + len(z.Samples[za0001]) + s += 6 + msgp.TimeSize + 7 + msgp.TimeSize + 8 + msgp.Uint64Size + 6 + msgp.Uint64Size + 8 + msgp.ArrayHeaderSize + for zcua := range z.Samples { + s += msgp.BytesPrefixSize + len(z.Samples[zcua]) } return } // DecodeMsg implements msgp.Decodable -func (z *EventSignature) DecodeMsg(dc *msgp.Reader) (err error) { +func (z EventSignatureIPv4) DecodeMsg(dc *msgp.Reader) (err error) { var field []byte _ = field - var zb0001 uint32 - zb0001, err = dc.ReadMapHeader() + var zjfb uint32 + zjfb, err = dc.ReadMapHeader() if err != nil { - err = msgp.WrapError(err) return } - for zb0001 > 0 { - zb0001-- + for zjfb > 0 { + zjfb-- field, err = dc.ReadMapKeyPtr() if err != nil { - err = msgp.WrapError(err) return } switch msgp.UnsafeString(field) { - case "SourceIP": - z.SourceIP, err = dc.ReadUint32() + case "SourceIPv4": + z.SourceIPv4, err = dc.ReadUint32() if err != nil { - err = msgp.WrapError(err, "SourceIP") return } case "Port": z.Port, err = dc.ReadUint16() if err != nil { - err = msgp.WrapError(err, "Port") return } case "Traffic": err = z.Traffic.DecodeMsg(dc) if err != nil { - err = msgp.WrapError(err, "Traffic") return } default: err = dc.Skip() if err != nil { - err = msgp.WrapError(err) return } } @@ -647,48 +738,186 @@ func (z *EventSignature) DecodeMsg(dc *msgp.Reader) (err error) { } // EncodeMsg implements msgp.Encodable -func (z *EventSignature) EncodeMsg(en *msgp.Writer) (err error) { +func (z EventSignatureIPv4) EncodeMsg(en *msgp.Writer) (err error) { // map header, size 3 - // write "SourceIP" - err = en.Append(0x83, 0xa8, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x49, 0x50) + // write "SourceIPv4" + err = en.Append(0x83, 0xaa, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x49, 0x50, 0x76, 0x34) if err != nil { - return + return err } - err = en.WriteUint32(z.SourceIP) + err = en.WriteUint32(z.SourceIPv4) if err != nil { - err = msgp.WrapError(err, "SourceIP") return } // write "Port" err = en.Append(0xa4, 0x50, 0x6f, 0x72, 0x74) if err != nil { - return + return err } err = en.WriteUint16(z.Port) if err != nil { - err = msgp.WrapError(err, "Port") return } // write "Traffic" err = en.Append(0xa7, 0x54, 0x72, 0x61, 0x66, 0x66, 0x69, 0x63) + if err != nil { + return err + } + err = z.Traffic.EncodeMsg(en) + if err != nil { + return + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z EventSignatureIPv4) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 3 + // string "SourceIPv4" + o = append(o, 0x83, 0xaa, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x49, 0x50, 0x76, 0x34) + o = msgp.AppendUint32(o, z.SourceIPv4) + // string "Port" + o = append(o, 0xa4, 0x50, 0x6f, 0x72, 0x74) + o = msgp.AppendUint16(o, z.Port) + // string "Traffic" + o = append(o, 0xa7, 0x54, 0x72, 0x61, 0x66, 0x66, 0x69, 0x63) + o, err = z.Traffic.MarshalMsg(o) + if err != nil { + return + } + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z EventSignatureIPv4) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zcxo uint32 + zcxo, bts, err = msgp.ReadMapHeaderBytes(bts) if err != nil { return } + for zcxo > 0 { + zcxo-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + return + } + switch msgp.UnsafeString(field) { + case "SourceIPv4": + z.SourceIPv4, bts, err = msgp.ReadUint32Bytes(bts) + if err != nil { + return + } + case "Port": + z.Port, bts, err = msgp.ReadUint16Bytes(bts) + if err != nil { + return + } + case "Traffic": + bts, err = z.Traffic.UnmarshalMsg(bts) + if err != nil { + return + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z EventSignatureIPv4) Msgsize() (s int) { + s = 1 + 11 + msgp.Uint32Size + 5 + msgp.Uint16Size + 8 + z.Traffic.Msgsize() + return +} + +// DecodeMsg implements msgp.Decodable +func (z EventSignatureIPv6) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zrsw uint32 + zrsw, err = dc.ReadMapHeader() + if err != nil { + return + } + for zrsw > 0 { + zrsw-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + return + } + switch msgp.UnsafeString(field) { + case "SourceIPv6": + err = dc.ReadExactBytes((z.SourceIPv6)[:]) + if err != nil { + return + } + case "Port": + z.Port, err = dc.ReadUint16() + if err != nil { + return + } + case "Traffic": + err = z.Traffic.DecodeMsg(dc) + if err != nil { + return + } + default: + err = dc.Skip() + if err != nil { + return + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z EventSignatureIPv6) EncodeMsg(en *msgp.Writer) (err error) { + // map header, size 3 + // write "SourceIPv6" + err = en.Append(0x83, 0xaa, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x49, 0x50, 0x76, 0x36) + if err != nil { + return err + } + err = en.WriteBytes((z.SourceIPv6)[:]) + if err != nil { + return + } + // write "Port" + err = en.Append(0xa4, 0x50, 0x6f, 0x72, 0x74) + if err != nil { + return err + } + err = en.WriteUint16(z.Port) + if err != nil { + return + } + // write "Traffic" + err = en.Append(0xa7, 0x54, 0x72, 0x61, 0x66, 0x66, 0x69, 0x63) + if err != nil { + return err + } err = z.Traffic.EncodeMsg(en) if err != nil { - err = msgp.WrapError(err, "Traffic") return } return } // MarshalMsg implements msgp.Marshaler -func (z *EventSignature) MarshalMsg(b []byte) (o []byte, err error) { +func (z EventSignatureIPv6) MarshalMsg(b []byte) (o []byte, err error) { o = msgp.Require(b, z.Msgsize()) // map header, size 3 - // string "SourceIP" - o = append(o, 0x83, 0xa8, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x49, 0x50) - o = msgp.AppendUint32(o, z.SourceIP) + // string "SourceIPv6" + o = append(o, 0x83, 0xaa, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x49, 0x50, 0x76, 0x36) + o = msgp.AppendBytes(o, (z.SourceIPv6)[:]) // string "Port" o = append(o, 0xa4, 0x50, 0x6f, 0x72, 0x74) o = msgp.AppendUint16(o, z.Port) @@ -696,52 +925,45 @@ func (z *EventSignature) MarshalMsg(b []byte) (o []byte, err error) { o = append(o, 0xa7, 0x54, 0x72, 0x61, 0x66, 0x66, 0x69, 0x63) o, err = z.Traffic.MarshalMsg(o) if err != nil { - err = msgp.WrapError(err, "Traffic") return } return } // UnmarshalMsg implements msgp.Unmarshaler -func (z *EventSignature) UnmarshalMsg(bts []byte) (o []byte, err error) { +func (z EventSignatureIPv6) UnmarshalMsg(bts []byte) (o []byte, err error) { var field []byte _ = field - var zb0001 uint32 - zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + var zxpk uint32 + zxpk, bts, err = msgp.ReadMapHeaderBytes(bts) if err != nil { - err = msgp.WrapError(err) return } - for zb0001 > 0 { - zb0001-- + for zxpk > 0 { + zxpk-- field, bts, err = msgp.ReadMapKeyZC(bts) if err != nil { - err = msgp.WrapError(err) return } switch msgp.UnsafeString(field) { - case "SourceIP": - z.SourceIP, bts, err = msgp.ReadUint32Bytes(bts) + case "SourceIPv6": + bts, err = msgp.ReadExactBytes(bts, (z.SourceIPv6)[:]) if err != nil { - err = msgp.WrapError(err, "SourceIP") return } case "Port": z.Port, bts, err = msgp.ReadUint16Bytes(bts) if err != nil { - err = msgp.WrapError(err, "Port") return } case "Traffic": bts, err = z.Traffic.UnmarshalMsg(bts) if err != nil { - err = msgp.WrapError(err, "Traffic") return } default: bts, err = msgp.Skip(bts) if err != nil { - err = msgp.WrapError(err) return } } @@ -751,7 +973,7 @@ func (z *EventSignature) UnmarshalMsg(bts []byte) (o []byte, err error) { } // Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message -func (z *EventSignature) Msgsize() (s int) { - s = 1 + 9 + msgp.Uint32Size + 5 + msgp.Uint16Size + 8 + z.Traffic.Msgsize() +func (z EventSignatureIPv6) Msgsize() (s int) { + s = 1 + 11 + msgp.ArrayHeaderSize + (16 * (msgp.ByteSize)) + 5 + msgp.Uint16Size + 8 + z.Traffic.Msgsize() return } diff --git a/internal/analysis/event_gen_test.go b/internal/analysis/event_gen_test.go index b8f9e84..0f7af63 100644 --- a/internal/analysis/event_gen_test.go +++ b/internal/analysis/event_gen_test.go @@ -1,6 +1,8 @@ package analysis -// Code generated by github.com/tinylib/msgp DO NOT EDIT. +// NOTE: THIS FILE WAS PRODUCED BY THE +// MSGP CODE GENERATION TOOL (github.com/tinylib/msgp) +// DO NOT EDIT import ( "bytes" @@ -74,7 +76,7 @@ func TestEncodeDecodeEvent(t *testing.T) { m := v.Msgsize() if buf.Len() > m { - t.Log("WARNING: TestEncodeDecodeEvent Msgsize() is inaccurate") + t.Logf("WARNING: Msgsize() for %v is inaccurate", v) } vn := Event{} @@ -122,8 +124,8 @@ func BenchmarkDecodeEvent(b *testing.B) { } } -func TestMarshalUnmarshalEventPackets(t *testing.T) { - v := EventPackets{} +func TestMarshalUnmarshalEventPacketsIPv4(t *testing.T) { + v := EventPacketsIPv4{} bts, err := v.MarshalMsg(nil) if err != nil { t.Fatal(err) @@ -145,8 +147,8 @@ func TestMarshalUnmarshalEventPackets(t *testing.T) { } } -func BenchmarkMarshalMsgEventPackets(b *testing.B) { - v := EventPackets{} +func BenchmarkMarshalMsgEventPacketsIPv4(b *testing.B) { + v := EventPacketsIPv4{} b.ReportAllocs() b.ResetTimer() for i := 0; i < b.N; i++ { @@ -154,8 +156,8 @@ func BenchmarkMarshalMsgEventPackets(b *testing.B) { } } -func BenchmarkAppendMsgEventPackets(b *testing.B) { - v := EventPackets{} +func BenchmarkAppendMsgEventPacketsIPv4(b *testing.B) { + v := EventPacketsIPv4{} bts := make([]byte, 0, v.Msgsize()) bts, _ = v.MarshalMsg(bts[0:0]) b.SetBytes(int64(len(bts))) @@ -166,8 +168,8 @@ func BenchmarkAppendMsgEventPackets(b *testing.B) { } } -func BenchmarkUnmarshalEventPackets(b *testing.B) { - v := EventPackets{} +func BenchmarkUnmarshalEventPacketsIPv4(b *testing.B) { + v := EventPacketsIPv4{} bts, _ := v.MarshalMsg(nil) b.ReportAllocs() b.SetBytes(int64(len(bts))) @@ -180,17 +182,17 @@ func BenchmarkUnmarshalEventPackets(b *testing.B) { } } -func TestEncodeDecodeEventPackets(t *testing.T) { - v := EventPackets{} +func TestEncodeDecodeEventPacketsIPv4(t *testing.T) { + v := EventPacketsIPv4{} var buf bytes.Buffer msgp.Encode(&buf, &v) m := v.Msgsize() if buf.Len() > m { - t.Log("WARNING: TestEncodeDecodeEventPackets Msgsize() is inaccurate") + t.Logf("WARNING: Msgsize() for %v is inaccurate", v) } - vn := EventPackets{} + vn := EventPacketsIPv4{} err := msgp.Decode(&buf, &vn) if err != nil { t.Error(err) @@ -204,8 +206,8 @@ func TestEncodeDecodeEventPackets(t *testing.T) { } } -func BenchmarkEncodeEventPackets(b *testing.B) { - v := EventPackets{} +func BenchmarkEncodeEventPacketsIPv4(b *testing.B) { + v := EventPacketsIPv4{} var buf bytes.Buffer msgp.Encode(&buf, &v) b.SetBytes(int64(buf.Len())) @@ -218,8 +220,8 @@ func BenchmarkEncodeEventPackets(b *testing.B) { en.Flush() } -func BenchmarkDecodeEventPackets(b *testing.B) { - v := EventPackets{} +func BenchmarkDecodeEventPacketsIPv4(b *testing.B) { + v := EventPacketsIPv4{} var buf bytes.Buffer msgp.Encode(&buf, &v) b.SetBytes(int64(buf.Len())) @@ -235,8 +237,8 @@ func BenchmarkDecodeEventPackets(b *testing.B) { } } -func TestMarshalUnmarshalEventSignature(t *testing.T) { - v := EventSignature{} +func TestMarshalUnmarshalEventPacketsIPv6(t *testing.T) { + v := EventPacketsIPv6{} bts, err := v.MarshalMsg(nil) if err != nil { t.Fatal(err) @@ -258,8 +260,8 @@ func TestMarshalUnmarshalEventSignature(t *testing.T) { } } -func BenchmarkMarshalMsgEventSignature(b *testing.B) { - v := EventSignature{} +func BenchmarkMarshalMsgEventPacketsIPv6(b *testing.B) { + v := EventPacketsIPv6{} b.ReportAllocs() b.ResetTimer() for i := 0; i < b.N; i++ { @@ -267,8 +269,8 @@ func BenchmarkMarshalMsgEventSignature(b *testing.B) { } } -func BenchmarkAppendMsgEventSignature(b *testing.B) { - v := EventSignature{} +func BenchmarkAppendMsgEventPacketsIPv6(b *testing.B) { + v := EventPacketsIPv6{} bts := make([]byte, 0, v.Msgsize()) bts, _ = v.MarshalMsg(bts[0:0]) b.SetBytes(int64(len(bts))) @@ -279,8 +281,8 @@ func BenchmarkAppendMsgEventSignature(b *testing.B) { } } -func BenchmarkUnmarshalEventSignature(b *testing.B) { - v := EventSignature{} +func BenchmarkUnmarshalEventPacketsIPv6(b *testing.B) { + v := EventPacketsIPv6{} bts, _ := v.MarshalMsg(nil) b.ReportAllocs() b.SetBytes(int64(len(bts))) @@ -293,17 +295,17 @@ func BenchmarkUnmarshalEventSignature(b *testing.B) { } } -func TestEncodeDecodeEventSignature(t *testing.T) { - v := EventSignature{} +func TestEncodeDecodeEventPacketsIPv6(t *testing.T) { + v := EventPacketsIPv6{} var buf bytes.Buffer msgp.Encode(&buf, &v) m := v.Msgsize() if buf.Len() > m { - t.Log("WARNING: TestEncodeDecodeEventSignature Msgsize() is inaccurate") + t.Logf("WARNING: Msgsize() for %v is inaccurate", v) } - vn := EventSignature{} + vn := EventPacketsIPv6{} err := msgp.Decode(&buf, &vn) if err != nil { t.Error(err) @@ -317,8 +319,8 @@ func TestEncodeDecodeEventSignature(t *testing.T) { } } -func BenchmarkEncodeEventSignature(b *testing.B) { - v := EventSignature{} +func BenchmarkEncodeEventPacketsIPv6(b *testing.B) { + v := EventPacketsIPv6{} var buf bytes.Buffer msgp.Encode(&buf, &v) b.SetBytes(int64(buf.Len())) @@ -331,8 +333,234 @@ func BenchmarkEncodeEventSignature(b *testing.B) { en.Flush() } -func BenchmarkDecodeEventSignature(b *testing.B) { - v := EventSignature{} +func BenchmarkDecodeEventPacketsIPv6(b *testing.B) { + v := EventPacketsIPv6{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} + +func TestMarshalUnmarshalEventSignatureIPv4(t *testing.T) { + v := EventSignatureIPv4{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgEventSignatureIPv4(b *testing.B) { + v := EventSignatureIPv4{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgEventSignatureIPv4(b *testing.B) { + v := EventSignatureIPv4{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalEventSignatureIPv4(b *testing.B) { + v := EventSignatureIPv4{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecodeEventSignatureIPv4(t *testing.T) { + v := EventSignatureIPv4{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Logf("WARNING: Msgsize() for %v is inaccurate", v) + } + + vn := EventSignatureIPv4{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncodeEventSignatureIPv4(b *testing.B) { + v := EventSignatureIPv4{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecodeEventSignatureIPv4(b *testing.B) { + v := EventSignatureIPv4{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} + +func TestMarshalUnmarshalEventSignatureIPv6(t *testing.T) { + v := EventSignatureIPv6{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgEventSignatureIPv6(b *testing.B) { + v := EventSignatureIPv6{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgEventSignatureIPv6(b *testing.B) { + v := EventSignatureIPv6{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalEventSignatureIPv6(b *testing.B) { + v := EventSignatureIPv6{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecodeEventSignatureIPv6(t *testing.T) { + v := EventSignatureIPv6{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Logf("WARNING: Msgsize() for %v is inaccurate", v) + } + + vn := EventSignatureIPv6{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncodeEventSignatureIPv6(b *testing.B) { + v := EventSignatureIPv6{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecodeEventSignatureIPv6(b *testing.B) { + v := EventSignatureIPv6{} var buf bytes.Buffer msgp.Encode(&buf, &v) b.SetBytes(int64(buf.Len())) diff --git a/internal/annotate/annotate.go b/internal/annotate/annotate.go index 6d63c71..99e871e 100644 --- a/internal/annotate/annotate.go +++ b/internal/annotate/annotate.go @@ -1,11 +1,10 @@ package annotate import ( + "compress/gzip" "darknet-events/internal/analysis" "darknet-events/internal/seqjson" "darknet-events/internal/set" - - "compress/gzip" "encoding/base64" "encoding/binary" "encoding/csv" @@ -27,8 +26,9 @@ import ( ) // TODO: These globals were copied from count.go - they should have a -// permanent home in an internal package where they can be accessed everywhere. +// permanent home in an internal package where they can be accessed everywhere. +// TODO: Add DarknetSize and DarknetFactor for IPv6 // DarknetSize is the number of IP addresses in the darknet. var DarknetSize uint32 = 475136 @@ -41,6 +41,7 @@ var SmallScanMargin int = int(DarknetSize) / 10 var DarknetFactor float64 = float64(4294967296) / float64(DarknetSize) // Output is the data structure that encapsulates the output for each event. +// Fields receiving pointers do not present when the pointers == nil type Output struct { SourceIP string Port uint16 @@ -51,13 +52,13 @@ type Output struct { Bytes uint64 UniqueDests int UniqueDest24s int - Lat float64 - Long float64 - Country string - City string - ASN int - Org string - Prefix string + Lat *float64 `json:",omitempty"` + Long *float64 `json:",omitempty"` + Country *string `json:",omitempty"` + City *string `json:",omitempty"` + ASN *int `json:",omitempty"` + Org *string `json:",omitempty"` + Prefix *string `json:",omitempty"` RDNS []string Zmap bool Masscan bool @@ -173,6 +174,7 @@ func NewAnnotator(resultsOutPath string, mmASNInPath string, } a.dnsChannel = make(chan ipOutput) for i := 0; i < 1000; i++ { + // TODO: Ask Why 1000? a.dnsWG.Add(1) go a.dnsQuerier() } @@ -183,6 +185,7 @@ func NewAnnotator(resultsOutPath string, mmASNInPath string, a.EventChannel = make(chan *analysis.Event) a.wg.Add(1) + // start a goroutine (running concurrently with other functions) go a.Reader() // Make temporary stats maps. @@ -228,7 +231,6 @@ func (a *Annotator) Close() { log.Printf("At threshold %d, packet coverage was %f%%, "+ "event coverage was %f%%.\n", a.minUniques, currPacketCoverage, currEventCoverage) - // TODO: Perhaps make this a flag for output? /* for i := a.minUniques - 1; i > 0; i-- { @@ -243,6 +245,11 @@ func (a *Annotator) Close() { */ } +type Unique24s struct { + Unique24sIPv4 set.Uint32Set + Unique24sIPv6 set.IPSet +} + // Reader sits in a loop taking in events and annotates them with useful // information such as the event type, AS data, etc. // @@ -253,66 +260,94 @@ func (a *Annotator) Reader() { es := e.Signature ep := e.Packets - // Create sets to count the number of unique dests and /24 dests. - unique24s := set.NewUint32Set() - for k := range *ep.Dests.Map() { - unique24s.Add(k & 0xffffff00) - } + var uniqueDestsSize int + var unique24sSize int + var sourceIP net.IP + + switch ep.(type) { + case *analysis.EventPacketsIPv4: + epIPv4, _ := ep.(*analysis.EventPacketsIPv4) + esIPv4, _ := es.(analysis.EventSignatureIPv4) + // Dealing with IPv4 addresses + // Create sets to count the number of unique dests and /24 dests. + unique24s := set.NewUint32Set() + for k := range *epIPv4.DestIPv4.Map() { + // Collect all IPs under /24 + unique24s.Add(k & 0xffffff00) + } - // Ignore if the number of unique destinations is too low. - if ep.Dests.Size() < a.minUniques { - a.packetsIgnored += ep.Packets - a.eventsIgnored++ + // Ignore if the number of unique destinations is too low. + if epIPv4.DestIPv4.Size() < a.minUniques { + a.packetsIgnored += epIPv4.Packets + a.eventsIgnored++ - // Update temporary stats. - for i := ep.Dests.Size(); i > 0; i-- { - a.packetStats[i] += ep.Packets - a.eventStats[i]++ + // Update temporary stats. + for i := epIPv4.DestIPv4.Size(); i > 0; i-- { + a.packetStats[i] += epIPv4.Packets + a.eventStats[i]++ + } + + continue } - continue - } + // Ignore if the packet rate is too low. + scanDuration := epIPv4.Latest.Sub(epIPv4.First).Seconds() + scanRate := float64(epIPv4.Packets) * DarknetFactor / scanDuration + if scanRate < a.minScanRate { + a.packetsIgnored += epIPv4.Packets + a.eventsIgnored++ + + // Update temporary stats. + for i := epIPv4.DestIPv4.Size(); i > 0; i-- { + a.packetStats[i] += epIPv4.Packets + a.eventStats[i]++ + } - // Ignore if the packet rate is too low. - scanDuration := ep.Latest.Sub(ep.First).Seconds() - scanRate := float64(ep.Packets) * DarknetFactor / scanDuration - if scanRate < a.minScanRate { - a.packetsIgnored += ep.Packets - a.eventsIgnored++ - - // Update temporary stats. - for i := ep.Dests.Size(); i > 0; i-- { - a.packetStats[i] += ep.Packets - a.eventStats[i]++ + continue } - continue - } - - // Convert the source IP to a net.IP object. - sourceIP := make(net.IP, 4) - binary.BigEndian.PutUint32(sourceIP, es.SourceIP) + // Convert the source IP to a net.IP object. + sourceIP = make(net.IP, 4) + binary.BigEndian.PutUint32(sourceIP, esIPv4.SourceIPv4) + uniqueDestsSize = epIPv4.DestIPv4.Size() + unique24sSize = unique24s.Size() + + case *analysis.EventPacketsIPv6: + epIPv6, _ := ep.(*analysis.EventPacketsIPv6) + esIPv6, _ := es.(analysis.EventSignatureIPv6) + // Dealing with IPv6 addresses + // TODO: Do we still consider /24 subnets for IPv6? + + // Convert the source IP to a net.IP object. + sourceIP = make(net.IP, 16) + copy(sourceIP, esIPv6.SourceIPv6[:]) + uniqueDestsSize = epIPv6.DestIPv6.Size() + unique24sSize = -1 + } // switch // If this is a TCP packet, check if its from zmap, masscan, or mirai. var zmap bool var masscan bool var mirai bool - if es.Traffic == analysis.TCPSYN { + rawSamples := ep.GetSamples() + + if es.GetTraffic() == analysis.TCPSYN { zmap = true masscan = true mirai = true - for i := 0; i < len(ep.Samples); i++ { - packet := gopacket.NewPacket(ep.Samples[i], + for i := 0; i < len(rawSamples); i++ { + packet := gopacket.NewPacket(rawSamples[i], layers.LayerTypeEthernet, gopacket.NoCopy) if packet == nil { log.Fatal("Could not parse packet to fingerprint.") } nl := packet.NetworkLayer() - if nl == nil || nl.LayerType() != layers.LayerTypeIPv4 { - log.Fatal("Contradiction in traffic type and IPv4 parse.") + + if nl == nil || (nl.LayerType() != layers.LayerTypeIPv4 && + nl.LayerType() != layers.LayerTypeIPv6) { + log.Fatal("Contradiction in traffic type and IPv4/v6 parse.") } - ip := nl.(*layers.IPv4) tl := packet.TransportLayer() if tl == nil || tl.LayerType() != layers.LayerTypeTCP { @@ -320,85 +355,117 @@ func (a *Annotator) Reader() { } tcp := tl.(*layers.TCP) - dstIP := binary.BigEndian.Uint32(ip.DstIP.To4()) - dstPort := tcp.DstPort - seq := tcp.Seq - id := ip.Id - - if zmap && id != 54321 { - zmap = false - } - if masscan && id != uint16(dstIP^uint32(dstPort)^seq) { - masscan = false - } - if mirai && dstIP != seq { - mirai = false + if nl.LayerType() == layers.LayerTypeIPv4 { + ip := nl.(*layers.IPv4) + dstIP := binary.BigEndian.Uint32(ip.DstIP.To4()) + + dstPort := tcp.DstPort + seq := tcp.Seq + id := ip.Id + + // TODO + if zmap && id != 54321 { + zmap = false + } + if masscan && id != uint16(dstIP^uint32(dstPort)^seq) { + masscan = false + } + if mirai && dstIP != seq { + mirai = false + } } } } // Get geographic data for the source IP. - var latitude float64 - var longitude float64 - var country string - var city string + var latitude *float64 + var longitude *float64 + var country *string + var city *string if a.locations != nil { location, err := a.locations.City(sourceIP) if err != nil { log.Fatalf("Couldn't check location for IP: %s.\n", err) } - latitude = location.Location.Latitude - longitude = location.Location.Longitude - country = location.Country.IsoCode - city = location.City.Names["en"] + latitude = &location.Location.Latitude + longitude = &location.Location.Longitude + country = &location.Country.IsoCode + cityName := location.City.Names["en"] + city = &cityName } - var asnNumber uint - var organisation string + var asnNumber *int + var organisation *string if a.asns != nil { asn, err := a.asns.ASN(sourceIP) if err != nil { log.Fatalf("Couldn't check ASN data for IP: %s\n", err) } - asnNumber = asn.AutonomousSystemNumber - organisation = asn.AutonomousSystemOrganization + temp := int(asn.AutonomousSystemNumber) + asnNumber = &temp + organisation = &asn.AutonomousSystemOrganization } // Get prefix data. - var routedPrefix string + var routedPrefix *string if a.prefixes != nil { val, ok, err := a.prefixes.Get(sourceIP) if err != nil { log.Fatalf("Couldn't look up IP prefix: %s.\n", err) } if err == nil && ok { - routedPrefix = val.(string) + temp := val.(string) + routedPrefix = &temp } } // Convert the saved samples to a string. - samples := make([]string, len(ep.Samples)) - for i := 0; i < len(ep.Samples); i++ { - samples[i] = base64.StdEncoding.EncodeToString(ep.Samples[i]) + samples := make([]string, len(rawSamples)) + for i := 0; i < len(rawSamples); i++ { + samples[i] = base64.StdEncoding.EncodeToString(rawSamples[i]) + } + + traffic := es.GetTraffic() + numPackets := ep.GetPackets() + + // not consume empty string pointers + // notice the difference from invalid pointers + if country != nil && *country == "" { + country = nil + } + if city != nil && *city == "" { + city = nil + } + if organisation != nil && *organisation == "" { + organisation = nil + } + if asnNumber != nil && *asnNumber == 0 { + asnNumber = nil + } + if latitude != nil && longitude != nil { + if *latitude == 0.0 && *longitude == 0.0 { + latitude = nil + longitude = nil + } } output := Output{ SourceIP: sourceIP.String(), - Port: es.Port, - Traffic: uint16(es.Traffic), - First: ep.First, - Last: ep.Latest, - Packets: ep.Packets, - Bytes: ep.Bytes, - UniqueDests: ep.Dests.Size(), - UniqueDest24s: unique24s.Size(), - Lat: latitude, - Long: longitude, - Country: country, - City: city, - ASN: int(asnNumber), - Org: organisation, - Prefix: routedPrefix, + Port: es.GetPort(), + Traffic: uint16(traffic), + First: ep.GetFirst(), + Last: ep.GetLatest(), + Packets: numPackets, + Bytes: ep.GetBytes(), + UniqueDests: uniqueDestsSize, + UniqueDest24s: unique24sSize, + Lat: latitude, // geoip + Long: longitude, // geoip + Country: country, // geoip + City: city, // geoip + ASN: asnNumber, // asn + Org: organisation, // asn + Prefix: routedPrefix, // pfx2as Zmap: zmap, Masscan: masscan, Mirai: mirai, @@ -406,17 +473,17 @@ func (a *Annotator) Reader() { // RDNS is populated asynchronously. } - if es.Traffic.IsTCP() == true { - output.TCP = es.Traffic.ToString() - } else if es.Traffic.IsICMP() == true { - output.ICMP = es.Traffic.ToString() + if traffic.IsTCP() == true { + output.TCP = traffic.ToString() + } else if traffic.IsICMP() == true { + output.ICMP = traffic.ToString() } // Spawn a goroutine to query DNS asynchronously. pair := ipOutput{ip: sourceIP, output: &output} a.dnsChannel <- pair - a.packetsAnnotated += ep.Packets + a.packetsAnnotated += numPackets a.eventsAnnotated++ } diff --git a/internal/cache/cache.go b/internal/cache/cache.go index a5527a7..43d7cd0 100644 --- a/internal/cache/cache.go +++ b/internal/cache/cache.go @@ -5,7 +5,9 @@ package cache import ( "io" "log" + "net" "os" + "runtime" "time" "unsafe" @@ -22,7 +24,7 @@ import ( // of this package. type Cache struct { // The cache itself. - Cache map[analysis.EventSignature]*analysis.EventPackets `msg:"-"` + Cache map[analysis.EventSignature]analysis.EventPackets `msg:"-"` // Last time the cache was cleared. Cleared time.Time // Timestamp of first entry in cache. @@ -71,7 +73,7 @@ func NewCache(timeout int, inPath string, outPath string, c.load(in) } else { - c.Cache = make(map[analysis.EventSignature]*analysis.EventPackets) + c.Cache = make(map[analysis.EventSignature]analysis.EventPackets) } // Check that an output state file can be created if given. This only @@ -95,7 +97,7 @@ func NewCache(timeout int, inPath string, outPath string, // key will be expired beforehand if they have exceeded their timeout. Add may // also add the raw bytes of the packet as a sample. func (c *Cache) Add(es analysis.EventSignature, - ip uint32, t time.Time, raw []byte) { + ip net.IP, t time.Time, raw []byte) { // If we don't have a known first packet time, this is the first packet. if c.First.IsZero() { @@ -103,7 +105,7 @@ func (c *Cache) Add(es analysis.EventSignature, } // If it is time to clear expired entries, stop the world and do so. If - // stw doesn't neet to run but the key exists and is expired, expire it. + // stw doesn't need to run but the key exists and is expired, expire it. if c.Now.Sub(c.Cleared) > c.timeout { c.stw() } else if _, ok := c.Cache[es]; ok && c.check(es, t) { @@ -112,10 +114,15 @@ func (c *Cache) Add(es analysis.EventSignature, // Add the new packet data to the cache. if _, ok := c.Cache[es]; !ok { - c.Cache[es] = analysis.NewEventPackets() + // check whether this is an IPv4/v6 address + if ip.To4() != nil { + c.Cache[es] = analysis.NewEventPacketsIPv4() + } else { + c.Cache[es] = analysis.NewEventPacketsIPv6() + } } - i := c.Cache[es].Add(ip, uint64(len(raw)), t) + i := c.Cache[es].Add(ip, uint64(len(raw)), t) // If this packet should be sampled, do so. // NOTE: We only sample once there have been enough packets to possibly // meet the MinUniques threshold. @@ -138,7 +145,13 @@ func (c *Cache) Size() uintptr { var size uintptr for k, v := range c.Cache { size += unsafe.Sizeof(k) - size += v.Size() + ep, ok := v.(analysis.EventPackets) + if ok { + size += ep.Size() + } else { + assertionError := runtime.TypeAssertionError{} + log.Print(assertionError) + } } return size } @@ -185,7 +198,8 @@ func (c *Cache) Close() { // check checks if the cache entry at the given key k is expired by the time t // and returns true if so. func (c *Cache) check(es analysis.EventSignature, t time.Time) bool { - return t.Sub(c.Cache[es].Latest) > c.timeout + + return t.Sub(c.Cache[es].GetLatest()) > c.timeout } // expire removes the entry at a given key k. The key and entry will be passed @@ -238,7 +252,7 @@ func (c *Cache) load(in io.Reader) { loadEnd.Sub(loadStart).String()) }() - c.Cache = make(map[analysis.EventSignature]*analysis.EventPackets) + c.Cache = make(map[analysis.EventSignature]analysis.EventPackets) r := msgp.NewReader(in) var err error @@ -248,17 +262,18 @@ func (c *Cache) load(in io.Reader) { } i := 0 for err == nil { - k := analysis.EventSignature{} + // TODO: Need to review + k := analysis.EventSignatureIPv6{} err = k.DecodeMsg(r) if err != nil { break } - v := analysis.EventPackets{} + v := analysis.EventPacketsIPv6{} err = v.DecodeMsg(r) if err != nil { break } - c.Cache[k] = &v + c.Cache[&k] = &v i++ } if err != nil && msgp.Cause(err) != io.EOF { diff --git a/internal/cache/cache_gen.go b/internal/cache/cache_gen.go index bbd61be..6f35fb8 100644 --- a/internal/cache/cache_gen.go +++ b/internal/cache/cache_gen.go @@ -1,6 +1,8 @@ package cache -// Code generated by github.com/tinylib/msgp DO NOT EDIT. +// NOTE: THIS FILE WAS PRODUCED BY THE +// MSGP CODE GENERATION TOOL (github.com/tinylib/msgp) +// DO NOT EDIT import ( "github.com/tinylib/msgp/msgp" @@ -10,42 +12,36 @@ import ( func (z *Cache) DecodeMsg(dc *msgp.Reader) (err error) { var field []byte _ = field - var zb0001 uint32 - zb0001, err = dc.ReadMapHeader() + var zxvk uint32 + zxvk, err = dc.ReadMapHeader() if err != nil { - err = msgp.WrapError(err) return } - for zb0001 > 0 { - zb0001-- + for zxvk > 0 { + zxvk-- field, err = dc.ReadMapKeyPtr() if err != nil { - err = msgp.WrapError(err) return } switch msgp.UnsafeString(field) { case "Cleared": z.Cleared, err = dc.ReadTime() if err != nil { - err = msgp.WrapError(err, "Cleared") return } case "First": z.First, err = dc.ReadTime() if err != nil { - err = msgp.WrapError(err, "First") return } case "Now": z.Now, err = dc.ReadTime() if err != nil { - err = msgp.WrapError(err, "Now") return } default: err = dc.Skip() if err != nil { - err = msgp.WrapError(err) return } } @@ -59,31 +55,28 @@ func (z Cache) EncodeMsg(en *msgp.Writer) (err error) { // write "Cleared" err = en.Append(0x83, 0xa7, 0x43, 0x6c, 0x65, 0x61, 0x72, 0x65, 0x64) if err != nil { - return + return err } err = en.WriteTime(z.Cleared) if err != nil { - err = msgp.WrapError(err, "Cleared") return } // write "First" err = en.Append(0xa5, 0x46, 0x69, 0x72, 0x73, 0x74) if err != nil { - return + return err } err = en.WriteTime(z.First) if err != nil { - err = msgp.WrapError(err, "First") return } // write "Now" err = en.Append(0xa3, 0x4e, 0x6f, 0x77) if err != nil { - return + return err } err = en.WriteTime(z.Now) if err != nil { - err = msgp.WrapError(err, "Now") return } return @@ -109,42 +102,36 @@ func (z Cache) MarshalMsg(b []byte) (o []byte, err error) { func (z *Cache) UnmarshalMsg(bts []byte) (o []byte, err error) { var field []byte _ = field - var zb0001 uint32 - zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + var zbzg uint32 + zbzg, bts, err = msgp.ReadMapHeaderBytes(bts) if err != nil { - err = msgp.WrapError(err) return } - for zb0001 > 0 { - zb0001-- + for zbzg > 0 { + zbzg-- field, bts, err = msgp.ReadMapKeyZC(bts) if err != nil { - err = msgp.WrapError(err) return } switch msgp.UnsafeString(field) { case "Cleared": z.Cleared, bts, err = msgp.ReadTimeBytes(bts) if err != nil { - err = msgp.WrapError(err, "Cleared") return } case "First": z.First, bts, err = msgp.ReadTimeBytes(bts) if err != nil { - err = msgp.WrapError(err, "First") return } case "Now": z.Now, bts, err = msgp.ReadTimeBytes(bts) if err != nil { - err = msgp.WrapError(err, "Now") return } default: bts, err = msgp.Skip(bts) if err != nil { - err = msgp.WrapError(err) return } } diff --git a/internal/cache/cache_gen_test.go b/internal/cache/cache_gen_test.go index d990f4b..7617def 100644 --- a/internal/cache/cache_gen_test.go +++ b/internal/cache/cache_gen_test.go @@ -1,6 +1,8 @@ package cache -// Code generated by github.com/tinylib/msgp DO NOT EDIT. +// NOTE: THIS FILE WAS PRODUCED BY THE +// MSGP CODE GENERATION TOOL (github.com/tinylib/msgp) +// DO NOT EDIT import ( "bytes" @@ -74,7 +76,7 @@ func TestEncodeDecodeCache(t *testing.T) { m := v.Msgsize() if buf.Len() > m { - t.Log("WARNING: TestEncodeDecodeCache Msgsize() is inaccurate") + t.Logf("WARNING: Msgsize() for %v is inaccurate", v) } vn := Cache{} diff --git a/internal/decode/decode.go b/internal/decode/decode.go index 56bc465..2bd85cf 100644 --- a/internal/decode/decode.go +++ b/internal/decode/decode.go @@ -2,8 +2,8 @@ package decode import ( "darknet-events/internal/analysis" + "net" - "encoding/binary" "log" "time" @@ -16,7 +16,9 @@ import ( type Decoder struct { eth layers.Ethernet ip4 layers.IPv4 + ip6 layers.IPv6 icmp4 layers.ICMPv4 + icmp6 layers.ICMPv6 tcp layers.TCP udp layers.UDP //sip layers.SIP @@ -32,8 +34,7 @@ type Decoder struct { func NewDecoder() *Decoder { d := new(Decoder) d.parser = gopacket.NewDecodingLayerParser(layers.LayerTypeEthernet, - &d.eth, &d.ip4, &d.icmp4, &d.tcp, &d.udp, - &d.pay) + &d.eth, &d.ip4, &d.ip6, &d.icmp4, &d.icmp6, &d.tcp, &d.udp, &d.pay) //&d.sip, &d.dns, &d.ntp, &d.pay) d.types = make([]gopacket.LayerType, 4, 4) d.parser.IgnoreUnsupported = true @@ -44,29 +45,35 @@ func NewDecoder() *Decoder { // Decode runs gopacket's DecodingLayerParser on a packet and parses the // derived information into event data, packet data, and a timestamp. func (d *Decoder) Decode(read []byte, - meta gopacket.CaptureInfo) (*analysis.EventSignature, uint32, time.Time) { + meta gopacket.CaptureInfo) (analysis.EventSignature, net.IP, time.Time) { err := d.parser.DecodeLayers(read, &d.types) + isErr := false + if err != nil { ult, ok := err.(gopacket.UnsupportedLayerType) if ok { d.unknowns[gopacket.LayerType(ult).String()]++ } - es := analysis.NewEventSignature(d.ip4.SrcIP, - 0, analysis.UnknownTraffic) - // The darknet only saves IPv4 packets so calling To4() is safe. - ip := binary.BigEndian.Uint32(d.ip4.DstIP.To4()) - t := meta.Timestamp - return es, ip, t + isErr = true } // TODO: Hack to get around the ETHERNET, IPV4, [NOTHING] case. - if len(d.types) < 3 { - es := analysis.NewEventSignature(d.ip4.SrcIP, - 0, analysis.UnknownTraffic) - ip := binary.BigEndian.Uint32(d.ip4.DstIP.To4()) + if isErr || len(d.types) < 3 { + var es analysis.EventSignature + var ip net.IP + + if d.ip4.DstIP.To4() != nil { + es = analysis.NewEventSignatureIPv4(d.ip4.SrcIP, + 0, analysis.UnknownTraffic) + ip = d.ip4.DstIP.To4() + } else { + es = analysis.NewEventSignatureIPv6(d.ip6.SrcIP, + 0, analysis.UnknownTraffic) + ip = d.ip4.DstIP.To16() + } t := meta.Timestamp return es, ip, t @@ -75,7 +82,34 @@ func (d *Decoder) Decode(read []byte, var port uint16 var traffic analysis.TrafficType + // Since IP is at the third layer among the four layers (physical, link, + // ip, and transportation) on which we focus, we look at d.types[2] switch d.types[2] { + case layers.LayerTypeICMPv6: + port = 0 + switch d.icmp6.TypeCode.Type() { + /* + https://tools.ietf.org/html/rfc4443 + ICMPv6 is used by IPv6 nodes to report errors encountered in + processing packets, and to perform other internet-layer functions, + such as diagnostics (ICMPv6 "ping"). + */ + // https://github.com/google/gopacket/blob/a9779d139771f6a06fc983b18e0efd23ca30222f/layers/icmp6.go#L19 + case layers.ICMPv6TypeDestinationUnreachable: + traffic = analysis.ICMPDestinationUnreachable + case layers.ICMPv6TypeTimeExceeded: + traffic = analysis.ICMPTimeExceeded + case layers.ICMPv6TypeParameterProblem: + traffic = analysis.ICMPParameterProblem + case layers.ICMPv6TypeEchoRequest: + traffic = analysis.ICMPEchoRequest + case layers.ICMPv6TypeEchoReply: + traffic = analysis.ICMPEchoReply + case layers.ICMPv6TypeRedirect: + traffic = analysis.ICMPRedirect + default: + traffic = analysis.ICMPOther + } case layers.LayerTypeICMPv4: port = 0 switch d.icmp4.TypeCode.Type() { @@ -130,8 +164,16 @@ func (d *Decoder) Decode(read []byte, log.Println("Unknown transport layer:", d.types[2].String()) } - es := analysis.NewEventSignature(d.ip4.SrcIP, port, traffic) - ip := binary.BigEndian.Uint32(d.ip4.DstIP.To4()) + var es analysis.EventSignature + var ip net.IP + + if d.ip4.DstIP.To4() != nil { + es = analysis.NewEventSignatureIPv4(d.ip4.SrcIP, port, traffic) + ip = d.ip4.DstIP.To4() + } else { + es = analysis.NewEventSignatureIPv6(d.ip6.SrcIP, port, traffic) + ip = d.ip6.DstIP.To16() + } t := meta.Timestamp return es, ip, t diff --git a/internal/set/set.go b/internal/set/set.go index 0a3968f..d1c10c3 100644 --- a/internal/set/set.go +++ b/internal/set/set.go @@ -1,5 +1,7 @@ package set +import "github.com/tinylib/msgp/msgp" + // Set is a simple abstraction layer over the Go map type to make it more // useful as a pure set type. Set uses an empty struct as the value type to // minimise the amount of unnecessary memory allocated. @@ -8,4 +10,7 @@ type Set interface { Size() int ByteSize() int Contains(item interface{}) bool + Map() interface{} + DecodeMsg(dc *msgp.Reader) error + EncodeMsg(en *msgp.Writer) error } diff --git a/internal/set/set_netip.go b/internal/set/set_netip.go new file mode 100644 index 0000000..ecbcb51 --- /dev/null +++ b/internal/set/set_netip.go @@ -0,0 +1,149 @@ +package set + +import ( + "github.com/tinylib/msgp/msgp" +) + +type IPSet struct { + M map[[16]byte]struct{} +} + +// NewIPSet initialises a new Set. +func NewIPSet() *IPSet { + s := new(IPSet) + s.M = make(map[[16]byte]struct{}) + return s +} + +// Add adds an item to a Set. If the item is already present, it will not be +// re-added. +func (s *IPSet) Add(k [16]byte) { + if _, found := s.M[k]; found == true { + return + } + var none struct{} + s.M[k] = none +} + +// Size returns the amount of items in the set. +func (s *IPSet) Size() int { + return len(s.M) +} + +// ByteSize approximates the number of bytes taken up by the object. +func (s *IPSet) ByteSize() int { + return len(s.M) * 16 +} + +// Contains returns true if the item is already in the set. +func (s *IPSet) Contains(item [16]byte) bool { + if _, found := s.M[item]; found == true { + return true + } + return false +} + +// Map returns the underlying data structure of the set. +func (s *IPSet) Map() *map[[16]byte]struct{} { + return &s.M +} + +// DecodeMsg implements msgp.Decodable +func (s *IPSet) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "M": + var zb0002 uint32 + zb0002, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err, "M") + return + } + if s.M == nil { + s.M = make(map[[16]byte]struct { + }, zb0002) + } else if len(s.M) > 0 { + for key := range s.M { + delete(s.M, key) + } + } + for zb0002 > 0 { + zb0002-- + var za0001 []byte + err = dc.ReadExactBytes(za0001) + if err != nil { + err = msgp.WrapError(err, "M") + return + } + var za0002 struct{} + var key [16]byte + copy(key[:], za0001) + s.M[key] = za0002 + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (s *IPSet) EncodeMsg(en *msgp.Writer) (err error) { + // map header, size 1 + // write "M" + err = en.Append(0x81, 0xa1, 0x4d) + if err != nil { + return + } + err = en.WriteMapHeader(uint32(len(s.M))) + if err != nil { + err = msgp.WrapError(err, "M") + return + } + for za0001 := range s.M { + var bytes []byte + for _, j := range za0001 { + bytes = append(bytes, j) + } + err = en.WriteBytes(bytes) + if err != nil { + err = msgp.WrapError(err, "M") + return + } + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (s *IPSet) MarshalMsg(b []byte) (o []byte, err error) { + panic("NOT IMPLEMENTED.") +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (s *IPSet) UnmarshalMsg(bts []byte) (o []byte, err error) { + panic("NOT IMPLEMENTED.") +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (s *IPSet) Msgsize() (z int) { + panic("NOT IMPLEMENTED.") +} + diff --git a/run.sh b/run.sh new file mode 100755 index 0000000..fed352d --- /dev/null +++ b/run.sh @@ -0,0 +1,45 @@ +#!/bin/sh + +c=$1 + +case $c in +"asn-only") + ./local/parse --threshold 624 --samples 3 \ + --eventout "../out/results_2021-03-11.01.2.json" \ + --pcap "../data/2021-03-11.01.pcap.gz" \ + --asnin "../data/GeoLite2-ASN.mmdb" + echo "asn only" +;; +"geo-only") + ./local/parse --threshold 624 --samples 3 \ + --eventout "../out/results_2021-03-11.01.2.json" \ + --pcap "../data/2021-03-11.01.pcap.gz" \ + --geoin "../data/GeoLite2-City.mmdb" + echo "geo only" +;; +"pfx2as-only") + ./local/parse --threshold 624 --samples 3 \ + --eventout "../out/results_2021-03-11.01.2.json" \ + --pcap "../data/2021-03-11.01.pcap.gz" \ + --pfx2asin "../data/routeviews-rv2-20201201-1200.pfx2as.gz" + echo "pfx2as only" +;; +"all") + ./local/parse --threshold 624 --samples 3 \ + --eventout "../out/results_2021-03-11.01.2.json" \ + --pcap "../data/2021-03-11.01.pcap.gz" \ + --asnin "../data/GeoLite2-ASN.mmdb" \ + --geoin "../data/GeoLite2-City.mmdb" \ + --pfx2asin "../data/routeviews-rv2-20201201-1200.pfx2as.gz" + echo "all flags present" +;; +"basic") + ./local/parse --threshold 624 --samples 3 \ + --eventout "../out/results_2021-03-11.01.2.json" \ + --pcap "../data/2021-03-11.01.pcap.gz" + echo "basic usage" +;; +esac + +cat "../out/results_2021-03-11.01.2.json" | python3 -m json.tool > "../out/pretty_2021-03-11.01.2.json" +