fix(redis) fix separator # to ## (#45)

* fix(redis) fix separator # to ##

* go mod tidy

Co-authored-by: Kota Kanbe <[email protected]>

db/common_test.go

@@ -5,6 +5,7 @@ import (
 	"strings"
 	"testing"
 
+	"github.com/google/go-cmp/cmp"
 	"github.com/knqyf263/go-cpe/common"
 	"github.com/knqyf263/go-cpe/naming"
 	"github.com/spf13/viper"
@@ -26,6 +27,7 @@ func prepareTestData(driver DB) error {
 		{"cpe:2.3:a:vendorName4:productName4:4.0:*:*:*:*:targetSoftware4:targetHardware4:*", false},
 		{"cpe:2.3:a:vendorName5:productName5:5.0:*:*:*:*:targetSoftware5:targetHardware5:*", false},
 		{"cpe:2.3:a:vendorName6:productName6:6.0:*:*:*:*:targetSoftware6:targetHardware6:*", true},
+		{`cpe:2.3:a:mongodb:c\#_driver:1.10.0:-:*:*:*:mongodb:*:*`, false},
 	}
 
 	testCpes := []models.CategorizedCpe{}
@@ -66,7 +68,7 @@ func testGetVendorProducts(t *testing.T, driver DB) {
 	}
 
 	type Expected struct {
-		VendorProduct []string
+		VendorProduct []models.VendorProduct
 		ErrString     string
 	}
 
@@ -75,22 +77,23 @@ func testGetVendorProducts(t *testing.T, driver DB) {
 	}{
 		"OK": {
 			Expected: Expected{
-				VendorProduct: []string{
-					"ntp#ntp",
-					"responsive_coming_soon_page_project#responsive_coming_soon_page",
-					"vendorName1#productName1\\-1", // TODO: what's with these slashes? Is it a bug?
-					"vendorName1#productName1\\-2", // TODO: what's with these slashes? Is it a bug?
-					"vendorName2#productName2",
-					"vendorName3#productName3",
-					"vendorName4#productName4",
-					"vendorName5#productName5",
-					"vendorName6#productName6",
+				VendorProduct: []models.VendorProduct{
+					{Vendor: "mongodb", Product: "c\\#_driver"},
+					{Vendor: "ntp", Product: "ntp"},
+					{Vendor: "responsive_coming_soon_page_project", Product: "responsive_coming_soon_page"},
+					{Vendor: "vendorName1", Product: "productName1\\-1"}, // TODO: what's with these slashes? Is it a bug?
+					{Vendor: "vendorName1", Product: "productName1\\-2"}, // TODO: what's with these slashes? Is it a bug?
+					{Vendor: "vendorName2", Product: "productName2"},
+					{Vendor: "vendorName3", Product: "productName3"},
+					{Vendor: "vendorName4", Product: "productName4"},
+					{Vendor: "vendorName5", Product: "productName5"},
+					{Vendor: "vendorName6", Product: "productName6"},
 				},
 			},
 		},
 	}
 	for k, tc := range cases {
-		var vendorProducts []string
+		var vendorProducts []models.VendorProduct
 		if vendorProducts, err = driver.GetVendorProducts(); err != nil {
 			if !strings.Contains(err.Error(), tc.Expected.ErrString) {
 				t.Errorf("%s : actual %s, expected %s", k, err, tc.Expected.ErrString)
@@ -103,8 +106,8 @@ func testGetVendorProducts(t *testing.T, driver DB) {
 		} else if 0 < len(tc.Expected.ErrString) {
 			t.Errorf("%s : actual %s, expected %s", k, err, tc.Expected.ErrString)
 		}
-		if !reflect.DeepEqual(vendorProducts, tc.Expected.VendorProduct) {
-			t.Errorf("%s: actual %#v, expected %#v", k, vendorProducts, tc.Expected.VendorProduct)
+		if diff := cmp.Diff(vendorProducts, tc.Expected.VendorProduct); diff != "" {
+			t.Errorf("%s: diff %s", k, diff)
 		}
 	}
 }

db/db.go

@@ -18,7 +18,7 @@ type DB interface {
 	GetFetchMeta() (*models.FetchMeta, error)
 	UpsertFetchMeta(*models.FetchMeta) error
 
-	GetVendorProducts() ([]string, error)
+	GetVendorProducts() ([]models.VendorProduct, error)
 	GetCpesByVendorProduct(string, string) ([]string, []string, error)
 	InsertCpes(models.FetchType, []models.CategorizedCpe) error
 	IsDeprecated(string) (bool, error)

db/rdb.go

@@ -164,22 +164,13 @@ func (r *RDBDriver) UpsertFetchMeta(fetchMeta *models.FetchMeta) error {
 }
 
 // GetVendorProducts : GetVendorProducts
-func (r *RDBDriver) GetVendorProducts() (vendorProducts []string, err error) {
-	var results []struct {
-		Vendor  string
-		Product string
-	}
-
+func (r *RDBDriver) GetVendorProducts() (vendorProducts []models.VendorProduct, err error) {
 	// TODO Is there a better way to use distinct with GORM? Needing
 	// explicit column names seems like an antipattern for an orm.
-	err = r.conn.Model(&models.CategorizedCpe{}).Distinct("vendor", "product").Find(&results).Error
+	err = r.conn.Model(&models.CategorizedCpe{}).Distinct("vendor", "product").Find(&vendorProducts).Error
 	if err != nil && errors.Is(err, gorm.ErrRecordNotFound) {
 		return nil, xerrors.Errorf("Failed to select results. err: %w", err)
 	}
-
-	for _, vp := range results {
-		vendorProducts = append(vendorProducts, fmt.Sprintf("%s#%s", vp.Vendor, vp.Product))
-	}
 	return
 }
 

db/redis.go

@@ -48,6 +48,7 @@ const (
 	dialectRedis      = "redis"
 	vpKeyFormat       = "CPE#VP#%s#%s"
 	vpListKey         = "CPE#VendorProducts"
+	vpSeparator       = "##"
 	deprecatedCPEsKey = "CPE#DeprecatedCPEs"
 	depKey            = "CPE#DEP"
 	fetchMetaKey      = "CPE#FETCHMETA"
@@ -148,13 +149,23 @@ func (r *RedisDriver) UpsertFetchMeta(fetchMeta *models.FetchMeta) error {
 }
 
 // GetVendorProducts : GetVendorProducts
-func (r *RedisDriver) GetVendorProducts() (vendorProducts []string, err error) {
+func (r *RedisDriver) GetVendorProducts() (vendorProducts []models.VendorProduct, err error) {
 	ctx := context.Background()
 	result, err := r.conn.SMembers(ctx, vpListKey).Result()
 	if err != nil {
 		return nil, err
 	}
-	return result, nil
+	for _, vp := range result {
+		vpParts := strings.Split(vp, vpSeparator)
+		if len(vpParts) != 2 {
+			continue
+		}
+		vendorProducts = append(vendorProducts, models.VendorProduct{
+			Vendor:  vpParts[0],
+			Product: vpParts[1],
+		})
+	}
+	return vendorProducts, nil
 }
 
 // GetCpesByVendorProduct : GetCpesByVendorProduct
@@ -217,7 +228,7 @@ func (r *RedisDriver) InsertCpes(fetchType models.FetchType, cpes []models.Categ
 		pipe := r.conn.Pipeline()
 		for _, c := range cpes[idx.From:idx.To] {
 			bar.Increment()
-			vendorProductStr := fmt.Sprintf("%s#%s", c.Vendor, c.Product)
+			vendorProductStr := fmt.Sprintf("%s%s%s", c.Vendor, vpSeparator, c.Product)
 			if err := pipe.SAdd(ctx, vpListKey, vendorProductStr).Err(); err != nil {
 				return xerrors.Errorf("Failed to SAdd vendorProduct. err: %w", err)
 			}

go.mod

@@ -28,8 +28,6 @@ require (
 	github.com/yuin/gopher-lua v0.0.0-20200603152657-dc2b0ca8b37e // indirect
 	go.opentelemetry.io/otel/internal/metric v0.21.0 // indirect
 	golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e // indirect
-	golang.org/x/net v0.0.0-20210614182718-04defd469f4e // indirect
-	golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c // indirect
 	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1
 	gorm.io/driver/mysql v1.1.1
 	gorm.io/driver/postgres v1.1.0
@@ -38,6 +36,12 @@ require (
 	moul.io/http2curl v1.0.0 // indirect
 )
 
+require (
+	github.com/google/go-cmp v0.5.6
+	golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d // indirect
+	golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e // indirect
+)
+
 require (
 	github.com/alicebob/gopher-json v0.0.0-20200520072559-a9ecdc9d1d3a // indirect
 	github.com/cespare/xxhash/v2 v2.1.1 // indirect

go.sum

@@ -684,8 +684,8 @@ golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc=
 golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
-golang.org/x/net v0.0.0-20210614182718-04defd469f4e h1:XpT3nA5TvE525Ne3hInMh6+GETgn27Zfm9dxsThnX2Q=
-golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d h1:20cMwl2fHAzkJMEA+8J4JgqBQcQGzbisXo31MIeenXI=
+golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -768,8 +768,8 @@ golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c h1:F1jZWGFhYfh0Ci55sIpILtKKK8p3i2/krTr0H1rg74I=
-golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e h1:WUoyKPm6nCo1BnNUvPGnFG3T5DUVem42yDJZZ4CNxMA=
+golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=

models/models.go

@@ -47,3 +47,8 @@ type CategorizedCpe struct {
 	Other           string    `gorm:"type:varchar(255)"`
 	Deprecated      bool
 }
+
+type VendorProduct struct {
+	Vendor  string
+	Product string
+}