Plat 1113 use search for group records (#127)

* [Plat-1113] Update storage.listRecordsSharedWithGroup() to use search service instead of storage service

* [Plat-1113] Fixes listRecordsSharedWithGroup to be backwards compatible

* [Plat-1113] Adding chaching of group AK for files shared with groups found via search

* [Plat-1113] Updates searchResult to cache AK only if the record is a file AND shared via groups

* [PLAT-1113] updates timeout for longrunning secrets test

* [Plat-1113] removes timeout update and delays test start so indexer can finish

* [Plat-1113] updates timeout for all secrets tests

* [PLAT-1113] increases timeout limit for travis tests

Author: lily-tsou

.travis.yml

@@ -13,7 +13,7 @@ env:
   global:
     - TEST_LOCAL_USE_PROD=false
     - TEST_LOCAL_USE_CDN=false
-    - TEST_IDLE_TIMEOUT_MILLISECONDS=100000
+    - TEST_IDLE_TIMEOUT_MILLISECONDS=200000
     - TEST_ENVIRONMENT=remote
     - TEST_BROWSER_VERSION=latest
 jobs:

__tests__/secret.test.js

@@ -5,7 +5,7 @@ const ops = require('./utils/operations')
 const { SECRET_UUID } = require('../lib/utils/constants')
 const { testEmail } = require('./utils')
 
-jest.setTimeout(100000)
+jest.setTimeout(200000)
 
 let realmConfig
 let realm
@@ -234,6 +234,7 @@ describe('Tozny identity client', () => {
     // use local search to wait
     await ops.createSecret(realmConfig, identity, secret)
     // wait for proper indexing of secret
+    await new Promise((r) => setTimeout(r, 5000))
     let result = await ops.getSecrets(realmConfig, identity, 10)
     expect(result.list[result.list.length - 1].data.secretValue).toBe(
       'secret-value'
@@ -247,6 +248,7 @@ describe('Tozny identity client', () => {
       description: 'this is a description',
     }
     const created = await ops.createSecret(realmConfig, identity, secret)
+    await new Promise((r) => setTimeout(r, 5000))
     const returned = await ops.viewSecret(
       realmConfig,
       identity,
@@ -270,6 +272,7 @@ describe('Tozny identity client', () => {
     }
     await ops.createSecret(realmConfig, identity, oldSecret)
     await ops.updateSecret(realmConfig, identity, oldSecret, newSecret)
+    await new Promise((r) => setTimeout(r, 5000))
     const secretsWithUpdatedRecord = await identity.getSecrets(100)
     const newLengthSecrets = secretsWithUpdatedRecord.list.length
     // Tests
@@ -334,6 +337,7 @@ describe('Tozny identity client', () => {
     await ops.updateSecret(realmConfig, identity, oldSecret, newSecret)
     const start = new Date()
     let latestVersionOfSecret
+    await new Promise((r) => setTimeout(r, 5000))
     while (new Date() - start < 30000) {
       latestVersionOfSecret = await ops.getLatestSecret(
         realmConfig,
@@ -479,6 +483,7 @@ describe('Tozny identity client', () => {
       // delay 200 milliseconds between tries
       await new Promise((r) => setTimeout(r, 200))
     }
+    await new Promise((r) => setTimeout(r, 5000))
     const listResponse = await ops.getSecretSharedList(
       realmConfig,
       identity,
@@ -531,6 +536,7 @@ describe('Tozny identity client', () => {
       // delay 200 milliseconds between tries
       await new Promise((r) => setTimeout(r, 200))
     }
+    await new Promise((r) => setTimeout(r, 5000))
     let sharedList = await ops.getSharedSecrets(realmConfig, identity2)
     expect(sharedList.sharedList[0].data.secretValue).toBe('secret-value')
   })
@@ -561,6 +567,7 @@ describe('Tozny identity client', () => {
       // delay 200 milliseconds between tries
       await new Promise((r) => setTimeout(r, 200))
     }
+    await new Promise((r) => setTimeout(r, 5000))
     let sharedList = await ops.getSharedSecrets(realmConfig, identity2)
     expect(sharedList.sharedList[0].data.secretValue).toBe('secret-value')
     let recordView = await ops.viewSecret(
@@ -691,6 +698,7 @@ describe('Tozny identity client', () => {
     )
   })
   it('can delete all secrets created by an identity', async () => {
+    await new Promise((r) => setTimeout(r, 5000))
     let listedSecrets = await ops.getSecrets(realmConfig, identity, 100)
     for (let index = 0; index < listedSecrets.list.length; index++) {
       let deleted = await ops.deleteSecretVersion(

__tests__/storage.test.js

@@ -440,6 +440,7 @@ describe('Tozny', () => {
       created.group.groupID,
       groupMembersToAdd
     )
+    await new Promise((r) => setTimeout(r, 5000))
     let sharedWithGroup = await ops.listRecordsSharedWithGroup(
       readerClient,
       created.group.groupID,
@@ -479,12 +480,13 @@ describe('Tozny', () => {
     const meta = { hola: 'mundo' }
     let recordInfo = await ops.writeRecord(readerClient, type, data, meta)
     await ops.shareRecordWithGroup(readerClient, created.group.groupID, type)
+    await new Promise((r) => setTimeout(r, 5000))
     let sharedWithGroup = await ops.listRecordsSharedWithGroup(
       authorizerClient,
       created.group.groupID,
       [],
       0,
-      10
+      1
     )
     expect(sharedWithGroup[0][0].meta.recordId).toBe(recordInfo.meta.recordId)
   })
@@ -512,7 +514,7 @@ describe('Tozny', () => {
       created.group.groupID,
       groupMembersToAdd
     )
-    const type = 'say-hello'
+    const type = `say-hello-${uuidv4()}`
     const data1 = { hi: 'world' }
     const meta1 = { hola: 'mundo' }
     await ops.writeRecord(readerClient, type, data1, meta1)
@@ -526,6 +528,7 @@ describe('Tozny', () => {
     const meta4 = { hola: 'mundo' }
     await ops.writeRecord(readerClient, type, data4, meta4)
     await ops.shareRecordWithGroup(readerClient, created.group.groupID, type)
+    await new Promise((r) => setTimeout(r, 5000))
     let sharedWithGroup = await ops.listRecordsSharedWithGroup(
       authorizerClient,
       created.group.groupID,
@@ -541,7 +544,7 @@ describe('Tozny', () => {
       0,
       10
     )
-    expect(sharedWithGroup2[0].length).toBe(5)
+    expect(sharedWithGroup2[0].length).toBe(4)
   })
   it('It can share and unshare a record with a group', async () => {
     const groupName = `testGroup-updated-${uuidv4()}`
@@ -572,6 +575,7 @@ describe('Tozny', () => {
     const meta = { hola: 'mundo' }
     let recordInfo = await ops.writeRecord(readerClient, type, data, meta)
     await ops.shareRecordWithGroup(readerClient, created.group.groupID, type)
+    await new Promise((r) => setTimeout(r, 5000))
     let sharedWithGroup = await ops.listRecordsSharedWithGroup(
       authorizerClient,
       created.group.groupID,
@@ -742,7 +746,6 @@ describe('Tozny', () => {
       groupName,
       groupDesciption
     )
-    console.log(created)
     const groupMember = new GroupMember(readerClient.clientId, {
       read: true,
       share: true,

lib/storage/client.js

@@ -624,14 +624,15 @@ class Client extends CryptoConsumer {
   }
 
   async _search(searchRequest) {
+    const searchBody = searchRequest.stringify()
     let response = await this.authenticator.tokenFetch(
       this.config.apiUrl + '/v2/search',
       {
         method: 'POST',
         headers: {
           'Content-Type': 'application/json',
         },
-        body: searchRequest.stringify(),
+        body: searchBody,
       }
     )
     await checkStatus(response)
@@ -1402,67 +1403,23 @@ class Client extends CryptoConsumer {
    *
    * @return {Array}  An array of records returned from TozStore.
    */
-  async listRecordsSharedWithGroup(
+    async listRecordsSharedWithGroup(
     groupId,
     writerIds = [],
     nextToken = null,
-    max = null
+    max = null,
   ) {
-    const listSharedRequest = {
-      group_id: groupId,
-      writer_ids: writerIds,
-      nextToken: nextToken,
-      max: max,
-    }
-    let encodedUrl = urlEncodeData(listSharedRequest)
-    let response = await this.authenticator.tsv1Fetch(
-      `${this.config.apiUrl}/v2/storage/groups/${groupId}/share?${encodedUrl}`,
-      {
-        method: 'GET',
-        headers: {
-          'Content-Type': 'application/json',
-        },
-      }
-    )
-    let returnValues = []
-    let listSharedWithGroups = []
-    let recordsJson = await validateResponseAsJSON(response)
-    if (recordsJson.results == null) {
-      return listSharedWithGroups
-    }
-    for (var i = 0; i < recordsJson.results.length; i++) {
-      let meta = await Meta.decode(recordsJson.results[i].meta)
-      let recordData = await new RecordData(recordsJson.results[i].record_data)
-      let record = await new Record(
-        meta,
-        recordData,
-        recordsJson.results[i].rec_sig
-      )
-      // Note: When fetching group records, particularly for the files use-case
-      // using the cached AK gives us the benefit of placing the AK in the cache
-      // which means later when downloading the file, the AK does not need to
-      // be fetched from the API. The gotcha here for now is that the API does
-      // not currently support AK fetches when access is granted via group
-      // permissions. This hides that by making sure the direct fetch is not
-      // needed, however for it to work the group records must be fetched via
-      // this list method first. This method is currently the only viable way
-      // of reading records that have been shared via the groups API.
-      let ak = await this._getCachedAk(
-        meta.writerId,
-        meta.userId,
-        this.config.clientId,
-        meta.type,
-        recordsJson.results[i].group_access_key
-      )
-      let decryptedRecord = await this.crypto.decryptRecord(record, ak)
-      listSharedWithGroups.push(decryptedRecord)
-    }
-    let NextToken = {
-      nextToken: recordsJson.next_token,
-    }
-    returnValues.push(listSharedWithGroups)
-    returnValues.push(NextToken)
-    return returnValues
+    console.log(writerIds)
+    // Create a search query
+    const groupIds = [groupId]
+    const searchRequest = new Search(true, true, max, nextToken, groupIds, true)
+    // Create search result
+    const resultQuery = await this.search(searchRequest)
+    // Execute search
+    let resultList = await resultQuery.next(true)
+    if (resultList.length == 0)
+      return []
+    return [resultList, {nextToken: resultQuery.request.nextToken}]
   }
 
   /**
@@ -1591,9 +1548,9 @@ class Client extends CryptoConsumer {
       data_start_timestamp: params.DataStartTimestamp,
     }
     if (typeof params.DataEndTimestamp !== 'undefined') {
-      computeEncoded["data_end_timestamp"] = params.DataEndTimestamp 
+      computeEncoded["data_end_timestamp"] = params.DataEndTimestamp
      }
-    if (typeof params.DataRequired !== 'undefined') { 
+    if (typeof params.DataRequired !== 'undefined') {
       let obj =  Object.fromEntries(params.DataRequired);
       computeEncoded["data_required"] = obj
      }

types/search.js

@@ -12,7 +12,9 @@ class Search extends Serializable {
     includeData = false,
     includeAllWriters = false,
     count = DEFAULT_QUERY_COUNT,
-    nextToken
+    nextToken,
+    groupIds = [],
+    includeOnlyGroups = false,
   ) {
     super()
     this.includeData = includeData
@@ -21,6 +23,8 @@ class Search extends Serializable {
     this.nextToken = nextToken
     this.matches = []
     this.excludes = []
+    this.groupIds = groupIds
+    this.includeOnlyGroups = includeOnlyGroups
   }
 
   match(terms, condition, strategy) {
@@ -54,6 +58,8 @@ class Search extends Serializable {
       limit: this.count,
       include_all_writers: this.includeAllWriters,
       include_data: this.includeData,
+      group_ids: this.groupIds,
+      include_only_groups: this.includeOnlyGroups,
     }
     if (this.nextToken) {
       toSerialize.next_token = this.nextToken
@@ -70,6 +76,9 @@ class Search extends Serializable {
     if (this.searchOrder) {
       toSerialize.order = this.searchOrder.serializable()
     }
+    if (this.groupIds){
+      toSerialize.group_ids = this.groupIds
+    }
     return toSerialize
   }
 }

types/searchResult.js

@@ -42,6 +42,15 @@ class SearchResult {
         const meta = await Meta.decode(result.meta)
         const record = new Record(meta, result.record_data)
         if (this.request.includeData) {
+          if (record.isFile && result.sharing_model == "GROUP"){
+            await this.client._getCachedAk(
+              meta.writerId,
+              meta.userId,
+              this.client.config.clientId,
+              meta.type,
+              result.group_access_key
+            )
+          }
           const ak = await this.decodeAccessKey(result.sharing_model, result.access_key, result.group_access_key)
           return this.client.crypto.decryptRecord(record, ak)
         }