Change GCS Init to use List (kahing#81)

Co-authored-by: Sai Teja Suram <[email protected]>

Author: 2 people

README-gcs.md

@@ -5,11 +5,7 @@
 
 Service Account credentials or user authentication. Ensure that either the service account or user has the proper permissions to the Bucket / Object under GCS.
 
-For example, read-only access should be granted the following permissions:
-```
-storage.objects.get
-storage.objects.list
-````
+To have a successful mount, we require users to have object listing (`storage.objects.list`) permission to the bucket.
 
 ### Service Account credentials
 

internal/backend_gcs.go

@@ -8,6 +8,7 @@ import (
 	"fmt"
 	"io"
 	"net"
+	"path"
 	"strings"
 	"syscall"
 
@@ -78,16 +79,20 @@ func NewGCS(bucket string, config *common.GCSConfig) (*GCSBackend, error) {
 
 // Init checks user's access to bucket.
 func (g *GCSBackend) Init(key string) error {
-	g.logger.Debugf("INIT GCS: %s", key)
-	_, err := g.bucket.Object(key).Attrs(context.Background())
-
-	// Currently we return successful mounts on 404 errors. However, GCS does not differentiate bucket not found and
-	// object not found errors. So, even though we don't know that the bucket exists, we will still mount successfully.
-	if err == storage.ErrObjectNotExist {
-		err = nil
-	}
-
-	return mapGCSError(err)
+	// We will do a successful mount if the user can list on the bucket.
+	// This is different other backends because GCS does not differentiate between object not found and
+	// bucket not found.
+	prefix, _ := path.Split(key)
+	_, err := g.ListBlobs(&ListBlobsInput{
+		MaxKeys: PUInt32(1),
+		Prefix:  PString(prefix),
+	})
+	g.logger.Debugf("INIT GCS: ListStatus = %s", getDebugResponseStatus(err))
+	if err == syscall.ENXIO {
+		return fmt.Errorf("bucket %v does not exist", g.bucketName)
+	}
+	// Errors can be returned directly since ListBlobs converts them to syscall errors.
+	return err
 }
 
 func (g *GCSBackend) Capabilities() *Capabilities {

internal/backend_gcs_test.go

@@ -187,19 +187,18 @@ func (s *GCSBackendTest) TearDownTest(c *C) {
 }
 
 func (s *GCSBackendTest) TestGCSBackend_Init_Authenticated(c *C) {
-	// on existing private bucket
+	// No error when accessing existing private bucket.
 	err := s.gcsBackend.Init(s.bucketName)
 	c.Assert(err, IsNil)
 
-	// on nonexistent bucket
-	gcsBackend, err := s.getGCSTestBackend(RandStringBytesMaskImprSrc(16))
+	// Not Found error when accessing nonexistent bucket.
+	randBktName := RandStringBytesMaskImprSrc(16)
+	gcsBackend, err := s.getGCSTestBackend(randBktName)
 	c.Assert(err, IsNil)
 	err = gcsBackend.Init(RandStringBytesMaskImprSrc(16))
-	// there is no error because we return not found errors as successful
-	// bucket not found has the same 404 error code as object not found
-	c.Assert(err, IsNil)
+	c.Assert(err, ErrorMatches, fmt.Sprintf("bucket %s does not exist", randBktName))
 
-	// on public bucket
+	// No error when accessing public bucket.
 	gcsBackend, err = s.getGCSTestBackend("gcp-public-data-nexrad-l2")
 	c.Assert(err, IsNil)
 	err = gcsBackend.Init(RandStringBytesMaskImprSrc(16))
@@ -218,19 +217,20 @@ func (s *GCSBackendTest) TestGCSBackend_Init_Unauthenticated(c *C) {
 		c.Skip("Skipping this test because credentials still exist in the environment.")
 	}
 
-	// on existing private bucket
+	// Access error on existing private bucket.
 	gcsBackend, err := s.getGCSTestBackend(s.bucketName)
 	c.Assert(err, IsNil)
 	err = gcsBackend.Init(RandStringBytesMaskImprSrc(15))
 	c.Assert(err, Equals, syscall.EACCES)
 
-	// on nonexistent bucket. there is no error because bucket not found is a 404 error
-	gcsBackend, err = s.getGCSTestBackend(RandStringBytesMaskImprSrc(16))
+	// Not Found error when accessing nonexistent bucket.
+	randBktName := RandStringBytesMaskImprSrc(16)
+	gcsBackend, err = s.getGCSTestBackend(randBktName)
 	c.Assert(err, IsNil)
 	err = gcsBackend.Init(RandStringBytesMaskImprSrc(16))
-	c.Assert(err, IsNil)
+	c.Assert(err, ErrorMatches, fmt.Sprintf("bucket %s does not exist", randBktName))
 
-	// on public bucket
+	// No error when accessing public bucket.
 	gcsBackend, err = s.getGCSTestBackend("gcp-public-data-nexrad-l2")
 	c.Assert(err, IsNil)
 	err = gcsBackend.Init(RandStringBytesMaskImprSrc(16))
@@ -249,21 +249,24 @@ func (s *GCSBackendTest) TestGCSBackend_Init_Authenticated_ReadOnlyAccess(c *C)
 	}()
 	os.Setenv("GOOGLE_APPLICATION_CREDENTIALS", os.Getenv("READONLY_GOOGLE_APPLICATION_CREDENTIALS"))
 
-	// on existing private bucket
+	// No error when accessing existing private bucket.
 	gcsBackend, err := s.getGCSTestBackend(s.bucketName)
 	c.Assert(err, IsNil)
+	err = gcsBackend.Init(s.bucketName)
+	c.Assert(err, IsNil)
 
-	// error when trying to modify object
+	// Access error when trying to modify object.
 	_, err = gcsBackend.DeleteBlob(&DeleteBlobInput{Key: s.testSpec.existingObjKey})
 	c.Assert(err, Equals, syscall.EACCES)
 
-	// on nonexistent bucket. there is no error because bucket not found is a 404 error
-	gcsBackend, err = s.getGCSTestBackend(RandStringBytesMaskImprSrc(16))
+	// Not Found error when accessing nonexistent bucket.
+	randBktName := RandStringBytesMaskImprSrc(16)
+	gcsBackend, err = s.getGCSTestBackend(randBktName)
 	c.Assert(err, IsNil)
 	err = gcsBackend.Init(RandStringBytesMaskImprSrc(16))
-	c.Assert(err, IsNil)
+	c.Assert(err, ErrorMatches, fmt.Sprintf("bucket %s does not exist", randBktName))
 
-	// on public bucket
+	// No error when accessing public bucket.
 	gcsBackend, err = s.getGCSTestBackend("gcp-public-data-nexrad-l2")
 	c.Assert(err, IsNil)
 	err = gcsBackend.Init(RandStringBytesMaskImprSrc(16))