diff --git a/db/common_test.go b/db/common_test.go index 86ee2e6..c3369a0 100644 --- a/db/common_test.go +++ b/db/common_test.go @@ -5,6 +5,7 @@ import ( "strings" "testing" + "github.com/google/go-cmp/cmp" "github.com/knqyf263/go-cpe/common" "github.com/knqyf263/go-cpe/naming" "github.com/spf13/viper" @@ -26,6 +27,7 @@ func prepareTestData(driver DB) error { {"cpe:2.3:a:vendorName4:productName4:4.0:*:*:*:*:targetSoftware4:targetHardware4:*", false}, {"cpe:2.3:a:vendorName5:productName5:5.0:*:*:*:*:targetSoftware5:targetHardware5:*", false}, {"cpe:2.3:a:vendorName6:productName6:6.0:*:*:*:*:targetSoftware6:targetHardware6:*", true}, + {`cpe:2.3:a:mongodb:c\#_driver:1.10.0:-:*:*:*:mongodb:*:*`, false}, } testCpes := []models.CategorizedCpe{} @@ -66,7 +68,7 @@ func testGetVendorProducts(t *testing.T, driver DB) { } type Expected struct { - VendorProduct []string + VendorProduct []models.VendorProduct ErrString string } @@ -75,22 +77,23 @@ func testGetVendorProducts(t *testing.T, driver DB) { }{ "OK": { Expected: Expected{ - VendorProduct: []string{ - "ntp#ntp", - "responsive_coming_soon_page_project#responsive_coming_soon_page", - "vendorName1#productName1\\-1", // TODO: what's with these slashes? Is it a bug? - "vendorName1#productName1\\-2", // TODO: what's with these slashes? Is it a bug? - "vendorName2#productName2", - "vendorName3#productName3", - "vendorName4#productName4", - "vendorName5#productName5", - "vendorName6#productName6", + VendorProduct: []models.VendorProduct{ + {Vendor: "mongodb", Product: "c\\#_driver"}, + {Vendor: "ntp", Product: "ntp"}, + {Vendor: "responsive_coming_soon_page_project", Product: "responsive_coming_soon_page"}, + {Vendor: "vendorName1", Product: "productName1\\-1"}, // TODO: what's with these slashes? Is it a bug? + {Vendor: "vendorName1", Product: "productName1\\-2"}, // TODO: what's with these slashes? Is it a bug? + {Vendor: "vendorName2", Product: "productName2"}, + {Vendor: "vendorName3", Product: "productName3"}, + {Vendor: "vendorName4", Product: "productName4"}, + {Vendor: "vendorName5", Product: "productName5"}, + {Vendor: "vendorName6", Product: "productName6"}, }, }, }, } for k, tc := range cases { - var vendorProducts []string + var vendorProducts []models.VendorProduct if vendorProducts, err = driver.GetVendorProducts(); err != nil { if !strings.Contains(err.Error(), tc.Expected.ErrString) { t.Errorf("%s : actual %s, expected %s", k, err, tc.Expected.ErrString) @@ -103,8 +106,8 @@ func testGetVendorProducts(t *testing.T, driver DB) { } else if 0 < len(tc.Expected.ErrString) { t.Errorf("%s : actual %s, expected %s", k, err, tc.Expected.ErrString) } - if !reflect.DeepEqual(vendorProducts, tc.Expected.VendorProduct) { - t.Errorf("%s: actual %#v, expected %#v", k, vendorProducts, tc.Expected.VendorProduct) + if diff := cmp.Diff(vendorProducts, tc.Expected.VendorProduct); diff != "" { + t.Errorf("%s: diff %s", k, diff) } } } diff --git a/db/db.go b/db/db.go index ff37e82..3408b04 100644 --- a/db/db.go +++ b/db/db.go @@ -18,7 +18,7 @@ type DB interface { GetFetchMeta() (*models.FetchMeta, error) UpsertFetchMeta(*models.FetchMeta) error - GetVendorProducts() ([]string, error) + GetVendorProducts() ([]models.VendorProduct, error) GetCpesByVendorProduct(string, string) ([]string, []string, error) InsertCpes(models.FetchType, []models.CategorizedCpe) error IsDeprecated(string) (bool, error) diff --git a/db/rdb.go b/db/rdb.go index e70b556..d202ce5 100644 --- a/db/rdb.go +++ b/db/rdb.go @@ -164,22 +164,13 @@ func (r *RDBDriver) UpsertFetchMeta(fetchMeta *models.FetchMeta) error { } // GetVendorProducts : GetVendorProducts -func (r *RDBDriver) GetVendorProducts() (vendorProducts []string, err error) { - var results []struct { - Vendor string - Product string - } - +func (r *RDBDriver) GetVendorProducts() (vendorProducts []models.VendorProduct, err error) { // TODO Is there a better way to use distinct with GORM? Needing // explicit column names seems like an antipattern for an orm. - err = r.conn.Model(&models.CategorizedCpe{}).Distinct("vendor", "product").Find(&results).Error + err = r.conn.Model(&models.CategorizedCpe{}).Distinct("vendor", "product").Find(&vendorProducts).Error if err != nil && errors.Is(err, gorm.ErrRecordNotFound) { return nil, xerrors.Errorf("Failed to select results. err: %w", err) } - - for _, vp := range results { - vendorProducts = append(vendorProducts, fmt.Sprintf("%s#%s", vp.Vendor, vp.Product)) - } return } diff --git a/db/redis.go b/db/redis.go index 2908f2d..df05754 100644 --- a/db/redis.go +++ b/db/redis.go @@ -48,6 +48,7 @@ const ( dialectRedis = "redis" vpKeyFormat = "CPE#VP#%s#%s" vpListKey = "CPE#VendorProducts" + vpSeparator = "##" deprecatedCPEsKey = "CPE#DeprecatedCPEs" depKey = "CPE#DEP" fetchMetaKey = "CPE#FETCHMETA" @@ -148,13 +149,23 @@ func (r *RedisDriver) UpsertFetchMeta(fetchMeta *models.FetchMeta) error { } // GetVendorProducts : GetVendorProducts -func (r *RedisDriver) GetVendorProducts() (vendorProducts []string, err error) { +func (r *RedisDriver) GetVendorProducts() (vendorProducts []models.VendorProduct, err error) { ctx := context.Background() result, err := r.conn.SMembers(ctx, vpListKey).Result() if err != nil { return nil, err } - return result, nil + for _, vp := range result { + vpParts := strings.Split(vp, vpSeparator) + if len(vpParts) != 2 { + continue + } + vendorProducts = append(vendorProducts, models.VendorProduct{ + Vendor: vpParts[0], + Product: vpParts[1], + }) + } + return vendorProducts, nil } // GetCpesByVendorProduct : GetCpesByVendorProduct @@ -217,7 +228,7 @@ func (r *RedisDriver) InsertCpes(fetchType models.FetchType, cpes []models.Categ pipe := r.conn.Pipeline() for _, c := range cpes[idx.From:idx.To] { bar.Increment() - vendorProductStr := fmt.Sprintf("%s#%s", c.Vendor, c.Product) + vendorProductStr := fmt.Sprintf("%s%s%s", c.Vendor, vpSeparator, c.Product) if err := pipe.SAdd(ctx, vpListKey, vendorProductStr).Err(); err != nil { return xerrors.Errorf("Failed to SAdd vendorProduct. err: %w", err) } diff --git a/go.mod b/go.mod index 828e0f2..f8c9177 100644 --- a/go.mod +++ b/go.mod @@ -28,8 +28,6 @@ require ( github.com/yuin/gopher-lua v0.0.0-20200603152657-dc2b0ca8b37e // indirect go.opentelemetry.io/otel/internal/metric v0.21.0 // indirect golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e // indirect - golang.org/x/net v0.0.0-20210614182718-04defd469f4e // indirect - golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c // indirect golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 gorm.io/driver/mysql v1.1.1 gorm.io/driver/postgres v1.1.0 @@ -38,6 +36,12 @@ require ( moul.io/http2curl v1.0.0 // indirect ) +require ( + github.com/google/go-cmp v0.5.6 + golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d // indirect + golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e // indirect +) + require ( github.com/alicebob/gopher-json v0.0.0-20200520072559-a9ecdc9d1d3a // indirect github.com/cespare/xxhash/v2 v2.1.1 // indirect diff --git a/go.sum b/go.sum index 3e33a9c..4fec3a1 100644 --- a/go.sum +++ b/go.sum @@ -684,8 +684,8 @@ golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.0.0-20210614182718-04defd469f4e h1:XpT3nA5TvE525Ne3hInMh6+GETgn27Zfm9dxsThnX2Q= -golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d h1:20cMwl2fHAzkJMEA+8J4JgqBQcQGzbisXo31MIeenXI= +golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -768,8 +768,8 @@ golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c h1:F1jZWGFhYfh0Ci55sIpILtKKK8p3i2/krTr0H1rg74I= -golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e h1:WUoyKPm6nCo1BnNUvPGnFG3T5DUVem42yDJZZ4CNxMA= +golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= diff --git a/models/models.go b/models/models.go index 444a6e3..dfbdcdf 100644 --- a/models/models.go +++ b/models/models.go @@ -47,3 +47,8 @@ type CategorizedCpe struct { Other string `gorm:"type:varchar(255)"` Deprecated bool } + +type VendorProduct struct { + Vendor string + Product string +}