diff --git a/changelog.d/1-api-changes/add-columns-to-export b/changelog.d/1-api-changes/add-columns-to-export new file mode 100644 index 00000000000..04633327ba1 --- /dev/null +++ b/changelog.d/1-api-changes/add-columns-to-export @@ -0,0 +1 @@ +The team CSV export endpoint has gained two extra columns: `last_active` and `status`. The streaming behaviour has also been improved. diff --git a/integration/integration.cabal b/integration/integration.cabal index a3989f28e76..edada8586df 100644 --- a/integration/integration.cabal +++ b/integration/integration.cabal @@ -100,6 +100,7 @@ library API.GundeckInternal API.Nginz API.Spar + API.Stern MLS.Util Notifications RunAllTests diff --git a/integration/test/API/Stern.hs b/integration/test/API/Stern.hs new file mode 100644 index 00000000000..b7d93d07178 --- /dev/null +++ b/integration/test/API/Stern.hs @@ -0,0 +1,8 @@ +module API.Stern where + +import Testlib.Prelude + +getTeamActivity :: (HasCallStack, MakesValue domain) => domain -> String -> App Response +getTeamActivity domain tid = + baseRequest domain Stern Unversioned (joinHttpPath ["team-activity-info", tid]) + >>= submit "GET" diff --git a/integration/test/Test/Client.hs b/integration/test/Test/Client.hs index 8c3101737dd..bc39d96d985 100644 --- a/integration/test/Test/Client.hs +++ b/integration/test/Test/Client.hs @@ -6,10 +6,12 @@ import API.Brig import qualified API.Brig as API import API.BrigCommon import API.Gundeck +import API.Stern import Control.Lens hiding ((.=)) import Control.Monad.Codensity import Control.Monad.Reader import Data.Aeson hiding ((.=)) +import qualified Data.ByteString.Char8 as B8 import Data.ProtoLens.Labels () import Data.Time.Clock.POSIX import Data.Time.Clock.System @@ -42,6 +44,30 @@ testClientLastActive = do <$> parseTimeM False defaultTimeLocale "%Y-%m-%dT%H:%M:%SZ" tm1 assertBool "last_active is earlier than expected" $ ts1 >= now +testTeamActivity :: (HasCallStack) => App () +testTeamActivity = do + (alice, tid, [bob, charlie]) <- createTeam OwnDomain 3 + [alice1, _alice2] <- replicateM 2 $ addClient alice def >>= getJSON 201 + [_bob1, _bob2] <- replicateM 2 $ addClient bob def >>= getJSON 201 + charlie1 <- addClient charlie def >>= getJSON 201 + + for_ [(alice, alice1), (charlie, charlie1)] $ \(u, cl) -> do + clientId <- cl %. "id" & asString + void $ getNotifications u def {client = Just clientId} + + let row (u, t) = do + uid <- u %. "id" & asString + pure (uid, t) + + expectedRows <- sort <$> traverse row [(alice, True), (bob, False), (charlie, True)] + + bindResponse (getTeamActivity alice tid) $ \resp -> do + resp.status `shouldMatchInt` 200 + for_ (zip (sort (B8.lines resp.body)) expectedRows) $ \(r, (uid, active)) -> do + let [actualUser, timestamp] = B8.split ',' r + B8.unpack actualUser `shouldMatch` uid + B8.null timestamp `shouldMatch` not active + testListClientsIfBackendIsOffline :: (HasCallStack) => App () testListClientsIfBackendIsOffline = do resourcePool <- asks (.resourcePool) diff --git a/integration/test/Test/Teams.hs b/integration/test/Test/Teams.hs index 623983abcba..5ce10031fba 100644 --- a/integration/test/Test/Teams.hs +++ b/integration/test/Test/Teams.hs @@ -1,3 +1,4 @@ +{-# OPTIONS -Wno-ambiguous-fields #-} -- This file is part of the Wire Server implementation. -- -- Copyright (C) 2024 Wire Swiss GmbH @@ -22,6 +23,7 @@ import qualified API.BrigInternal as I import API.Common import API.Galley (getTeam, getTeamMembers, getTeamMembersCsv, getTeamNotifications) import API.GalleyInternal (setTeamFeatureStatus) +import API.Gundeck import Control.Monad.Codensity (Codensity (runCodensity)) import Control.Monad.Extra (findM) import Control.Monad.Reader (asks) @@ -284,16 +286,28 @@ testUpgradePersonalToTeamAlreadyInATeam = do -- for additional tests of the CSV download particularly with SCIM users, please refer to 'Test.Spar.Scim.UserSpec' testTeamMemberCsvExport :: (HasCallStack) => App () testTeamMemberCsvExport = do - (owner, tid, members) <- createTeam OwnDomain 10 - let numClients = [0, 1, 2] <> repeat 0 - modifiedMembers <- for (zip numClients (owner : members)) $ \(n, m) -> do - handle <- randomHandle - putHandle m handle >>= assertSuccess - replicateM_ n $ addClient m def - void $ I.putSSOId m def {I.scimExternalId = Just "foo"} >>= getBody 200 - setField "handle" handle m - >>= setField "role" (if m == owner then "owner" else "member") - >>= setField "num_clients" (show n) + (owner, tid, members) <- createTeam OwnDomain 5 + + modifiedMembers <- for + ( zip + ([0, 1, 2] <> repeat 0) + (owner : members) + ) + $ \(n, m) -> do + handle <- randomHandle + putHandle m handle >>= assertSuccess + clients <- + replicateM n + $ addClient m def + >>= getJSON 201 + >>= (%. "id") + >>= asString + for_ (listToMaybe clients) $ \c -> + getNotifications m def {client = Just c} + void $ I.putSSOId m def {I.scimExternalId = Just "foo"} >>= getBody 200 + setField "handle" handle m + >>= setField "role" (if m == owner then "owner" else "member") + >>= setField "num_clients" n memberMap :: Map.Map String Value <- fmap Map.fromList $ for (modifiedMembers) $ \m -> do uid <- m %. "id" & asString @@ -302,7 +316,7 @@ testTeamMemberCsvExport = do bindResponse (getTeamMembersCsv owner tid) $ \resp -> do resp.status `shouldMatchInt` 200 let rows = sort $ tail $ B8.lines $ resp.body - length rows `shouldMatchInt` 10 + length rows `shouldMatchInt` 5 for_ rows $ \row -> do let cols = B8.split ',' row let uid = read $ B8.unpack $ cols !! 11 @@ -310,6 +324,8 @@ testTeamMemberCsvExport = do ownerId <- owner %. "id" & asString let ownerMember = memberMap Map.! ownerId + now <- formatTime defaultTimeLocale "%Y-%m-%d" <$> liftIO getCurrentTime + numClients <- mem %. "num_clients" & asInt let parseField = unquote . read . B8.unpack . (cols !!) @@ -319,12 +335,15 @@ testTeamMemberCsvExport = do role <- mem %. "role" & asString parseField 3 `shouldMatch` role when (role /= "owner") $ do - now <- formatTime defaultTimeLocale "%Y-%m-%d" <$> liftIO getCurrentTime take 10 (parseField 4) `shouldMatch` now parseField 5 `shouldMatch` (ownerMember %. "handle") parseField 7 `shouldMatch` "wire" parseField 9 `shouldMatch` "foo" - parseField 12 `shouldMatch` (mem %. "num_clients") + parseField 12 `shouldMatch` show numClients + (if numClients > 0 then shouldNotMatch else shouldMatch) + (parseField 13) + "" + parseField 14 `shouldMatch` "active" where unquote :: String -> String unquote ('\'' : x) = x diff --git a/libs/wire-api/default.nix b/libs/wire-api/default.nix index 2fd02d1acf6..c4f7828d2c3 100644 --- a/libs/wire-api/default.nix +++ b/libs/wire-api/default.nix @@ -57,6 +57,7 @@ , iso3166-country-codes , iso639 , jose +, kan-extensions , lens , lib , memory @@ -165,6 +166,7 @@ mkDerivation { iso3166-country-codes iso639 jose + kan-extensions lens memory metrics-wai diff --git a/libs/wire-api/src/Wire/API/Routes/Internal/Brig.hs b/libs/wire-api/src/Wire/API/Routes/Internal/Brig.hs index 5c6dc34ffd9..31c1b018e33 100644 --- a/libs/wire-api/src/Wire/API/Routes/Internal/Brig.hs +++ b/libs/wire-api/src/Wire/API/Routes/Internal/Brig.hs @@ -76,6 +76,7 @@ import Wire.API.Routes.Internal.LegalHold qualified as LegalHoldInternalAPI import Wire.API.Routes.MultiVerb import Wire.API.Routes.Named import Wire.API.Routes.Public (ZUser) +import Wire.API.Team.Export (TeamExportUser) import Wire.API.Team.Feature import Wire.API.Team.Invitation (Invitation) import Wire.API.Team.LegalHold.Internal @@ -601,6 +602,14 @@ type UserAPI = UpdateUserLocale :<|> DeleteUserLocale :<|> GetDefaultLocale + :<|> Named + "get-user-export-data" + ( Summary "Get user export data" + :> "users" + :> Capture "uid" UserId + :> "export-data" + :> MultiVerb1 'GET '[JSON] (Respond 200 "User export data" (Maybe TeamExportUser)) + ) type UpdateUserLocale = Summary diff --git a/libs/wire-api/src/Wire/API/Routes/LowLevelStream.hs b/libs/wire-api/src/Wire/API/Routes/LowLevelStream.hs index f39080b54f7..0313c04ced8 100644 --- a/libs/wire-api/src/Wire/API/Routes/LowLevelStream.hs +++ b/libs/wire-api/src/Wire/API/Routes/LowLevelStream.hs @@ -18,6 +18,8 @@ module Wire.API.Routes.LowLevelStream where import Control.Lens (at, (.~), (?~), _Just) +import Control.Monad.Codensity +import Control.Monad.Trans.Resource import Data.ByteString.Char8 as B8 import Data.CaseInsensitive qualified as CI import Data.HashMap.Strict.InsOrd qualified as InsOrdHashMap @@ -39,6 +41,10 @@ import Servant.Server hiding (respond) import Servant.Server.Internal import Wire.API.Routes.Version +-- | Used as the return type of a streaming handler. The 'Codensity' wrapper +-- makes it possible to add finalisation logic to the streaming action. +type LowLevelStreamingBody = Codensity IO StreamingBody + -- FUTUREWORK: make it possible to generate headers at runtime data LowLevelStream method status (headers :: [(Symbol, Symbol)]) desc ctype @@ -63,7 +69,9 @@ instance (ReflectMethod method, KnownNat status, RenderHeaders headers, Accept ctype) => HasServer (LowLevelStream method status headers desc ctype) context where - type ServerT (LowLevelStream method status headers desc ctype) m = m StreamingBody + type + ServerT (LowLevelStream method status headers desc ctype) m = + m LowLevelStreamingBody hoistServerWithContext _ _ nt s = nt s route Proxy _ action = leafRouter $ \env request respond -> @@ -71,15 +79,25 @@ instance cmediatype = HTTP.matchAccept [contentType (Proxy @ctype)] accH accCheck = when (isNothing cmediatype) $ delayedFail err406 contentHeader = (hContentType, HTTP.renderHeader . maybeToList $ cmediatype) - in runAction - ( action - `addMethodCheck` methodCheck method request - `addAcceptCheck` accCheck - ) - env - request - respond - $ Route . responseStream status (contentHeader : extraHeaders) + in runResourceT $ do + r <- + runDelayed + ( action + `addMethodCheck` methodCheck method request + `addAcceptCheck` accCheck + ) + env + request + liftIO $ case r of + Route h -> + runHandler h >>= \case + Left e -> respond $ FailFatal e + Right getStreamingBody -> lowerCodensity $ do + body <- getStreamingBody + let resp = responseStream status (contentHeader : extraHeaders) body + lift $ respond $ Route resp + Fail e -> respond $ Fail e + FailFatal e -> respond $ FailFatal e where method = reflectMethod (Proxy :: Proxy method) status = statusFromNat (Proxy :: Proxy status) diff --git a/libs/wire-api/src/Wire/API/Routes/MultiVerb.hs b/libs/wire-api/src/Wire/API/Routes/MultiVerb.hs index 2dfeb16685a..ebda942c2cd 100644 --- a/libs/wire-api/src/Wire/API/Routes/MultiVerb.hs +++ b/libs/wire-api/src/Wire/API/Routes/MultiVerb.hs @@ -110,9 +110,9 @@ type RespondEmpty s desc = RespondAs '() s desc () -- | A type to describe a streaming 'MultiVerb' response. -- --- Includes status code, description, framing strategy and content type. Note --- that the handler return type is hardcoded to be 'SourceIO ByteString'. -data RespondStreaming (s :: Nat) (desc :: Symbol) (framing :: Type) (ct :: Type) +-- Includes status code, description and content type. Note that the handler +-- return type is hardcoded to be 'SourceIO ByteString'. +data RespondStreaming (s :: Nat) (desc :: Symbol) (ct :: Type) -- | The result of parsing a response as a union alternative of type 'a'. -- @@ -268,14 +268,14 @@ instance mempty & S.description .~ Text.pack (symbolVal (Proxy @desc)) -type instance ResponseType (RespondStreaming s desc framing ct) = SourceIO ByteString +type instance ResponseType (RespondStreaming s desc ct) = SourceIO ByteString instance (Accept ct, KnownStatus s) => - IsResponse cs (RespondStreaming s desc framing ct) + IsResponse cs (RespondStreaming s desc ct) where - type ResponseStatus (RespondStreaming s desc framing ct) = s - type ResponseBody (RespondStreaming s desc framing ct) = SourceIO ByteString + type ResponseStatus (RespondStreaming s desc ct) = s + type ResponseBody (RespondStreaming s desc ct) = SourceIO ByteString responseRender _ x = pure . addContentType @ct $ Response @@ -289,7 +289,7 @@ instance guard (responseStatusCode resp == statusVal (Proxy @s)) pure $ responseBody resp -instance (KnownSymbol desc) => IsSwaggerResponse (RespondStreaming s desc framing ct) where +instance (KnownSymbol desc) => IsSwaggerResponse (RespondStreaming s desc ct) where responseSwagger = pure $ mempty diff --git a/libs/wire-api/src/Wire/API/Routes/Public/Cargohold.hs b/libs/wire-api/src/Wire/API/Routes/Public/Cargohold.hs index d9c7ca0ed3e..4b15e9d1df2 100644 --- a/libs/wire-api/src/Wire/API/Routes/Public/Cargohold.hs +++ b/libs/wire-api/src/Wire/API/Routes/Public/Cargohold.hs @@ -80,7 +80,6 @@ type AssetStreaming = RespondStreaming 200 "Asset returned directly with content type `application/octet-stream`" - NoFraming OctetStream type GetAsset = diff --git a/libs/wire-api/src/Wire/API/Team/Export.hs b/libs/wire-api/src/Wire/API/Team/Export.hs index 7a37047c307..c31040c5e42 100644 --- a/libs/wire-api/src/Wire/API/Team/Export.hs +++ b/libs/wire-api/src/Wire/API/Team/Export.hs @@ -17,6 +17,7 @@ module Wire.API.Team.Export (TeamExportUser (..), quoted, unquoted) where +import Data.Aeson qualified as A import Data.Aeson qualified as Aeson import Data.Attoparsec.ByteString.Lazy (parseOnly) import Data.ByteString.Char8 qualified as C @@ -24,18 +25,27 @@ import Data.ByteString.Conversion (FromByteString (..), ToByteString, toByteStri import Data.Csv (DefaultOrdered (..), FromNamedRecord (..), Parser, ToNamedRecord (..), namedRecord, (.:)) import Data.Handle (Handle) import Data.Id (UserId) -import Data.Json.Util (UTCTimeMillis) +import Data.Json.Util (UTCTimeMillis, utcTimeSchema) import Data.Misc (HttpsUrl) +import Data.OpenApi qualified as OpenApi +import Data.Schema +import Data.Text qualified as T +import Data.Text.Encoding qualified as T +import Data.Time.Clock +import Data.Time.Format import Data.Vector (fromList) import Imports import Test.QuickCheck import Wire.API.Team.Role (Role) -import Wire.API.User (Name) +import Wire.API.User (AccountStatus (..), Name) import Wire.API.User.Identity (EmailAddress) import Wire.API.User.Profile (ManagedBy) import Wire.API.User.RichInfo (RichInfo) import Wire.Arbitrary +timestampFormat :: String +timestampFormat = "%Y-%m-%d" + data TeamExportUser = TeamExportUser { tExportDisplayName :: Name, tExportHandle :: Maybe Handle, @@ -49,10 +59,33 @@ data TeamExportUser = TeamExportUser tExportSCIMExternalId :: Text, tExportSCIMRichInfo :: Maybe RichInfo, tExportUserId :: UserId, - tExportNumDevices :: Int + tExportNumDevices :: Int, + tExportLastActive :: Maybe UTCTime, + tExportStatus :: Maybe AccountStatus } deriving (Show, Eq, Generic) deriving (Arbitrary) via (GenericUniform TeamExportUser) + deriving (A.ToJSON, A.FromJSON, OpenApi.ToSchema) via (Schema TeamExportUser) + +instance ToSchema TeamExportUser where + schema = + object "TeamExportUser" $ + TeamExportUser + <$> tExportDisplayName .= field "display_name" schema + <*> tExportHandle .= maybe_ (optField "handle" schema) + <*> tExportEmail .= maybe_ (optField "email" schema) + <*> tExportRole .= maybe_ (optField "role" schema) + <*> tExportCreatedOn .= maybe_ (optField "created_on" schema) + <*> tExportInvitedBy .= maybe_ (optField "invited_by" schema) + <*> tExportIdpIssuer .= maybe_ (optField "idp_issuer" schema) + <*> tExportManagedBy .= field "managed_by" schema + <*> tExportSAMLNamedId .= field "saml_name_id" schema + <*> tExportSCIMExternalId .= field "scim_external_id" schema + <*> tExportSCIMRichInfo .= maybe_ (optField "scim_rich_info" schema) + <*> tExportUserId .= field "user_id" schema + <*> tExportNumDevices .= field "num_devices" schema + <*> tExportLastActive .= maybe_ (optField "last_active" utcTimeSchema) + <*> tExportStatus .= maybe_ (optField "status" schema) instance ToNamedRecord TeamExportUser where toNamedRecord row = @@ -69,7 +102,16 @@ instance ToNamedRecord TeamExportUser where ("scim_external_id", secureCsvFieldToByteString (tExportSCIMExternalId row)), ("scim_rich_info", maybe "" (C.toStrict . Aeson.encode) (tExportSCIMRichInfo row)), ("user_id", secureCsvFieldToByteString (tExportUserId row)), - ("num_devices", secureCsvFieldToByteString (tExportNumDevices row)) + ("num_devices", secureCsvFieldToByteString (tExportNumDevices row)), + ( "last_active", + C.pack + ( maybe + "" + (formatTime defaultTimeLocale timestampFormat) + (tExportLastActive row) + ) + ), + ("status", maybe "" formatAccountStatus (tExportStatus row)) ] secureCsvFieldToByteString :: forall a. (ToByteString a) => a -> ByteString @@ -91,7 +133,9 @@ instance DefaultOrdered TeamExportUser where "scim_external_id", "scim_rich_info", "user_id", - "num_devices" + "num_devices", + "last_active", + "status" ] allowEmpty :: (ByteString -> Parser a) -> ByteString -> Parser (Maybe a) @@ -104,6 +148,26 @@ parseByteString bstr = Left err -> fail err Right thing -> pure thing +parseUTCTime :: ByteString -> Parser UTCTime +parseUTCTime b = do + s <- either (fail . displayException) pure $ T.decodeUtf8' b + parseTimeM False defaultTimeLocale timestampFormat (T.unpack s) + +parseAccountStatus :: ByteString -> Parser AccountStatus +parseAccountStatus "active" = pure Active +parseAccountStatus "suspended" = pure Suspended +parseAccountStatus "deleted" = pure Deleted +parseAccountStatus "ephemeral" = pure Ephemeral +parseAccountStatus "pending-invitation" = pure PendingInvitation +parseAccountStatus _ = fail "invalid account status" + +formatAccountStatus :: AccountStatus -> ByteString +formatAccountStatus Active = "active" +formatAccountStatus Suspended = "suspended" +formatAccountStatus Deleted = "deleted" +formatAccountStatus Ephemeral = "ephemeral" +formatAccountStatus PendingInvitation = "pending-invitation" + instance FromNamedRecord TeamExportUser where parseNamedRecord nrec = TeamExportUser @@ -126,6 +190,8 @@ instance FromNamedRecord TeamExportUser where ) <*> (nrec .: "user_id" >>= parseByteString) <*> (nrec .: "num_devices" >>= parseByteString) + <*> (nrec .: "last_active" >>= allowEmpty parseUTCTime) + <*> (nrec .: "status" >>= allowEmpty parseAccountStatus) quoted :: ByteString -> ByteString quoted bs = case C.uncons bs of diff --git a/libs/wire-api/src/Wire/API/Team/Member.hs b/libs/wire-api/src/Wire/API/Team/Member.hs index 98720fab69b..108e19adb98 100644 --- a/libs/wire-api/src/Wire/API/Team/Member.hs +++ b/libs/wire-api/src/Wire/API/Team/Member.hs @@ -22,6 +22,7 @@ module Wire.API.Team.Member ( -- * TeamMember TeamMember, + newTeamMember, mkTeamMember, userId, permissions, diff --git a/libs/wire-api/wire-api.cabal b/libs/wire-api/wire-api.cabal index 1091c12f7f3..33c53b3ddf3 100644 --- a/libs/wire-api/wire-api.cabal +++ b/libs/wire-api/wire-api.cabal @@ -300,6 +300,7 @@ library , iso3166-country-codes >=0.2 , iso639 >=0.1 , jose + , kan-extensions , lens >=4.12 , memory , metrics-wai diff --git a/libs/wire-subsystems/default.nix b/libs/wire-subsystems/default.nix index ead2f2a9c9d..7582b8b4228 100644 --- a/libs/wire-subsystems/default.nix +++ b/libs/wire-subsystems/default.nix @@ -15,6 +15,7 @@ , bloodhound , bytestring , bytestring-conversion +, case-insensitive , cassandra-util , conduit , containers @@ -104,6 +105,7 @@ mkDerivation { bloodhound bytestring bytestring-conversion + case-insensitive cassandra-util conduit containers diff --git a/libs/wire-subsystems/src/Wire/UserStore.hs b/libs/wire-subsystems/src/Wire/UserStore.hs index 55373c0a37d..6ebb55c71cf 100644 --- a/libs/wire-subsystems/src/Wire/UserStore.hs +++ b/libs/wire-subsystems/src/Wire/UserStore.hs @@ -6,10 +6,12 @@ import Cassandra (PageWithState (..), PagingState) import Data.Default import Data.Handle import Data.Id +import Data.Time.Clock import Imports import Polysemy import Polysemy.Error import Wire.API.User +import Wire.API.User.RichInfo import Wire.Arbitrary import Wire.StoredUser import Wire.UserStore.IndexUser @@ -67,6 +69,8 @@ data UserStore m a where IsActivated :: UserId -> UserStore m Bool LookupLocale :: UserId -> UserStore m (Maybe (Maybe Language, Maybe Country)) UpdateUserTeam :: UserId -> TeamId -> UserStore m () + GetActivityTimestamps :: UserId -> UserStore m [Maybe UTCTime] + GetRichInfo :: UserId -> UserStore m (Maybe RichInfoAssocList) makeSem ''UserStore diff --git a/libs/wire-subsystems/src/Wire/UserStore/Cassandra.hs b/libs/wire-subsystems/src/Wire/UserStore/Cassandra.hs index 66d35568d27..db15b04f4b4 100644 --- a/libs/wire-subsystems/src/Wire/UserStore/Cassandra.hs +++ b/libs/wire-subsystems/src/Wire/UserStore/Cassandra.hs @@ -4,12 +4,14 @@ import Cassandra import Cassandra.Exec (prepared) import Data.Handle import Data.Id +import Data.Time.Clock import Database.CQL.Protocol import Imports import Polysemy import Polysemy.Embed import Polysemy.Error import Wire.API.User hiding (DeleteUser) +import Wire.API.User.RichInfo import Wire.StoredUser import Wire.UserStore import Wire.UserStore.IndexUser hiding (userId) @@ -31,6 +33,8 @@ interpretUserStoreCassandra casClient = IsActivated uid -> isActivatedImpl uid LookupLocale uid -> lookupLocaleImpl uid UpdateUserTeam uid tid -> updateUserTeamImpl uid tid + GetActivityTimestamps uid -> getActivityTimestampsImpl uid + GetRichInfo uid -> getRichInfoImpl uid getUsersImpl :: [UserId] -> Client [StoredUser] getUsersImpl usrs = @@ -169,6 +173,21 @@ updateUserTeamImpl u t = retry x5 $ write userTeamUpdate (params LocalQuorum (t, userTeamUpdate :: PrepQuery W (TeamId, UserId) () userTeamUpdate = "UPDATE user SET team = ? WHERE id = ?" +getActivityTimestampsImpl :: UserId -> Client [Maybe UTCTime] +getActivityTimestampsImpl uid = do + runIdentity <$$> retry x1 (query q (params LocalQuorum (Identity uid))) + where + q :: PrepQuery R (Identity UserId) (Identity (Maybe UTCTime)) + q = "SELECT last_active from clients where user = ?" + +getRichInfoImpl :: UserId -> Client (Maybe RichInfoAssocList) +getRichInfoImpl uid = + fmap runIdentity + <$> retry x1 (query1 q (params LocalQuorum (Identity uid))) + where + q :: PrepQuery R (Identity UserId) (Identity RichInfoAssocList) + q = "SELECT json FROM rich_info WHERE user = ?" + -------------------------------------------------------------------------------- -- Queries diff --git a/libs/wire-subsystems/src/Wire/UserSubsystem.hs b/libs/wire-subsystems/src/Wire/UserSubsystem.hs index 10357641b71..c8237825748 100644 --- a/libs/wire-subsystems/src/Wire/UserSubsystem.hs +++ b/libs/wire-subsystems/src/Wire/UserSubsystem.hs @@ -20,6 +20,7 @@ import Polysemy import Polysemy.Error import Wire.API.Federation.Error import Wire.API.Routes.Internal.Galley.TeamFeatureNoConfigMulti (TeamStatus) +import Wire.API.Team.Export (TeamExportUser) import Wire.API.Team.Feature import Wire.API.Team.Member (IsPerm (..), TeamMember) import Wire.API.User @@ -143,6 +144,7 @@ data UserSubsystem m a where -- migration this would just be an internal detail of the subsystem InternalUpdateSearchIndex :: UserId -> UserSubsystem m () InternalFindTeamInvitation :: Maybe EmailKey -> InvitationCode -> UserSubsystem m StoredInvitation + GetUserExportData :: UserId -> UserSubsystem m (Maybe TeamExportUser) -- | the return type of 'CheckHandle' data CheckHandleResp diff --git a/libs/wire-subsystems/src/Wire/UserSubsystem/Interpreter.hs b/libs/wire-subsystems/src/Wire/UserSubsystem/Interpreter.hs index bbcbe719eb8..8f9ba2566e1 100644 --- a/libs/wire-subsystems/src/Wire/UserSubsystem/Interpreter.hs +++ b/libs/wire-subsystems/src/Wire/UserSubsystem/Interpreter.hs @@ -7,8 +7,10 @@ module Wire.UserSubsystem.Interpreter ) where -import Control.Lens (view) +import Control.Error.Util (hush) +import Control.Lens (view, (^.)) import Control.Monad.Trans.Maybe +import Data.CaseInsensitive qualified as CI import Data.Domain import Data.Handle (Handle) import Data.Handle qualified as Handle @@ -16,7 +18,7 @@ import Data.Id import Data.Json.Util import Data.LegalHold import Data.List.Extra (nubOrd) -import Data.Misc (PlainTextPassword6) +import Data.Misc (HttpsUrl, PlainTextPassword6, mkHttpsUrl) import Data.Qualified import Data.Range import Data.Time.Clock @@ -27,6 +29,7 @@ import Polysemy.Error import Polysemy.Input import Polysemy.TinyLog (TinyLog) import Polysemy.TinyLog qualified as Log +import SAML2.WebSSO qualified as SAML import Servant.Client.Core import System.Logger.Message qualified as Log import Wire.API.Federation.API @@ -34,6 +37,7 @@ import Wire.API.Federation.API.Brig qualified as FedBrig import Wire.API.Federation.Error import Wire.API.Routes.FederationDomainConfig import Wire.API.Routes.Internal.Galley.TeamFeatureNoConfigMulti (TeamStatus (..)) +import Wire.API.Team.Export import Wire.API.Team.Feature import Wire.API.Team.Member import Wire.API.Team.Permission qualified as Permission @@ -41,6 +45,7 @@ import Wire.API.Team.Role (defaultRole) import Wire.API.Team.SearchVisibility import Wire.API.Team.Size (TeamSize (TeamSize)) import Wire.API.User as User +import Wire.API.User.RichInfo import Wire.API.User.Search import Wire.API.UserEvent import Wire.Arbitrary @@ -100,76 +105,78 @@ runUserSubsystem :: Member FederationConfigStore r, Member Metrics r, Member InvitationStore r, - Member TinyLog r + Member TinyLog r, + Member (Input UserSubsystemConfig) r ) => - UserSubsystemConfig -> InterpreterFor AuthenticationSubsystem r -> - InterpreterFor UserSubsystem r -runUserSubsystem cfg authInterpreter = - interpret $ - \case - GetUserProfiles self others -> - runInputConst cfg $ - getUserProfilesImpl self others - GetLocalUserProfiles others -> - runInputConst cfg $ - getLocalUserProfilesImpl others - GetAccountsBy getBy -> - runInputConst cfg $ - getAccountsByImpl getBy - GetAccountsByEmailNoFilter emails -> - runInputConst cfg $ - getAccountsByEmailNoFilterImpl emails - GetAccountNoFilter luid -> - runInputConst cfg $ - getAccountNoFilterImpl luid - GetSelfProfile self -> - runInputConst cfg $ - getSelfProfileImpl self - GetUserProfilesWithErrors self others -> - runInputConst cfg $ - getUserProfilesWithErrorsImpl self others - UpdateUserProfile self mconn mb update -> - runInputConst cfg $ - updateUserProfileImpl self mconn mb update - CheckHandle uhandle -> - runInputConst cfg $ - checkHandleImpl uhandle - CheckHandles hdls cnt -> - runInputConst cfg $ - checkHandlesImpl hdls cnt - UpdateHandle uid mconn mb uhandle -> - runInputConst cfg $ - updateHandleImpl uid mconn mb uhandle - LookupLocaleWithDefault luid -> - runInputConst cfg $ - lookupLocaleOrDefaultImpl luid - IsBlocked email -> - runInputConst cfg $ - isBlockedImpl email - BlockListDelete email -> - runInputConst cfg $ - blockListDeleteImpl email - BlockListInsert email -> - runInputConst cfg $ - blockListInsertImpl email - UpdateTeamSearchVisibilityInbound status -> - runInputConst cfg $ - updateTeamSearchVisibilityInboundImpl status - SearchUsers luid query mDomain mMaxResults -> - runInputConst cfg $ - searchUsersImpl luid query mDomain mMaxResults - BrowseTeam uid browseTeamFilters mMaxResults mPagingState -> - browseTeamImpl uid browseTeamFilters mMaxResults mPagingState - InternalUpdateSearchIndex uid -> - syncUserIndex uid - AcceptTeamInvitation luid pwd code -> - authInterpreter - . runInputConst cfg - $ acceptTeamInvitationImpl luid pwd code - InternalFindTeamInvitation mEmailKey code -> - runInputConst cfg $ - internalFindTeamInvitationImpl mEmailKey code + Sem (UserSubsystem ': r) a -> + Sem r a +runUserSubsystem authInterpreter = interpret $ + \case + GetUserProfiles self others -> + getUserProfilesImpl self others + GetLocalUserProfiles others -> + getLocalUserProfilesImpl others + GetAccountsBy getBy -> + getAccountsByImpl getBy + GetAccountsByEmailNoFilter emails -> + getAccountsByEmailNoFilterImpl emails + GetAccountNoFilter luid -> + getAccountNoFilterImpl luid + GetSelfProfile self -> + getSelfProfileImpl self + GetUserProfilesWithErrors self others -> + getUserProfilesWithErrorsImpl self others + UpdateUserProfile self mconn mb update -> + updateUserProfileImpl self mconn mb update + CheckHandle uhandle -> + checkHandleImpl uhandle + CheckHandles hdls cnt -> + checkHandlesImpl hdls cnt + UpdateHandle uid mconn mb uhandle -> + updateHandleImpl uid mconn mb uhandle + LookupLocaleWithDefault luid -> + lookupLocaleOrDefaultImpl luid + IsBlocked email -> + isBlockedImpl email + BlockListDelete email -> + blockListDeleteImpl email + BlockListInsert email -> + blockListInsertImpl email + UpdateTeamSearchVisibilityInbound status -> + updateTeamSearchVisibilityInboundImpl status + SearchUsers luid query mDomain mMaxResults -> + searchUsersImpl luid query mDomain mMaxResults + BrowseTeam uid browseTeamFilters mMaxResults mPagingState -> + browseTeamImpl uid browseTeamFilters mMaxResults mPagingState + InternalUpdateSearchIndex uid -> + syncUserIndex uid + AcceptTeamInvitation luid pwd code -> + authInterpreter $ + acceptTeamInvitationImpl luid pwd code + InternalFindTeamInvitation mEmailKey code -> + internalFindTeamInvitationImpl mEmailKey code + GetUserExportData uid -> getUserExportDataImpl uid + +scimExtId :: StoredUser -> Maybe Text +scimExtId su = do + m <- su.managedBy + i <- su.identity + sso <- ssoIdentity i + scimExternalId m sso + +userToIdPIssuer :: StoredUser -> Maybe HttpsUrl +userToIdPIssuer su = case su.identity >>= ssoIdentity of + Just (UserSSOId (SAML.UserRef issuer _)) -> + either (const Nothing) Just . mkHttpsUrl $ issuer ^. SAML.fromIssuer + Just _ -> Nothing + Nothing -> Nothing + +samlNamedId :: StoredUser -> Maybe Text +samlNamedId su = + su.identity >>= ssoIdentity >>= \case + (UserSSOId (SAML.UserRef _idp nameId)) -> Just . CI.original . SAML.unsafeShowNameID $ nameId + (UserScimExternalId _) -> Nothing internalFindTeamInvitationImpl :: ( Member InvitationStore r, @@ -939,3 +946,31 @@ acceptTeamInvitationImpl luid pw code = do deleteInvitation inv.teamId inv.invitationId syncUserIndex uid generateUserEvent uid Nothing (teamUpdated uid tid) + +getUserExportDataImpl :: (Member UserStore r) => UserId -> Sem r (Maybe TeamExportUser) +getUserExportDataImpl uid = fmap hush . runError @() $ do + su <- UserStore.getUser uid >>= note () + mRichInfo <- UserStore.getRichInfo uid + timestamps <- UserStore.getActivityTimestamps uid + -- Make sure the list of timestamps is non-empty so that 'maximum' is + -- well-defined and returns 'Nothing' when no valid timestamps are present. + let lastActive = maximum (Nothing : timestamps) + let numClients = length timestamps + pure $ + TeamExportUser + { tExportDisplayName = su.name, + tExportHandle = su.handle, + tExportEmail = su.email, + tExportRole = Nothing, + tExportCreatedOn = Nothing, + tExportInvitedBy = Nothing, + tExportIdpIssuer = userToIdPIssuer su, + tExportManagedBy = fromMaybe ManagedByWire su.managedBy, + tExportSAMLNamedId = fromMaybe "" (samlNamedId su), + tExportSCIMExternalId = fromMaybe "" (scimExtId su), + tExportSCIMRichInfo = fmap RichInfo mRichInfo, + tExportUserId = uid, + tExportNumDevices = numClients, + tExportLastActive = lastActive, + tExportStatus = su.status + } diff --git a/libs/wire-subsystems/test/unit/Wire/MiniBackend.hs b/libs/wire-subsystems/test/unit/Wire/MiniBackend.hs index a13271b863a..ee951963bbc 100644 --- a/libs/wire-subsystems/test/unit/Wire/MiniBackend.hs +++ b/libs/wire-subsystems/test/unit/Wire/MiniBackend.hs @@ -400,7 +400,7 @@ interpretMaybeFederationStackState maybeFederationAPIAccess localBackend teamMem authSubsystemInterpreter = interpretAuthenticationSubsystem userSubsystemInterpreter userSubsystemInterpreter :: InterpreterFor UserSubsystem (MiniBackendLowerEffects `Append` r) - userSubsystemInterpreter = runUserSubsystem cfg authSubsystemInterpreter + userSubsystemInterpreter = runUserSubsystem authSubsystemInterpreter in sequentiallyPerformConcurrency . noOpLogger . maybeFederationAPIAccess diff --git a/libs/wire-subsystems/test/unit/Wire/MockInterpreters/UserStore.hs b/libs/wire-subsystems/test/unit/Wire/MockInterpreters/UserStore.hs index a4c05c44b5c..650aeb60dfa 100644 --- a/libs/wire-subsystems/test/unit/Wire/MockInterpreters/UserStore.hs +++ b/libs/wire-subsystems/test/unit/Wire/MockInterpreters/UserStore.hs @@ -71,6 +71,8 @@ inMemoryUserStoreInterpreter = interpret $ \case modify $ map (\u -> if u.id == uid then u {teamId = Just tid} :: StoredUser else u) + GetActivityTimestamps _ -> pure [] + GetRichInfo _ -> error "rich info not implemented" storedUserToIndexUser :: StoredUser -> IndexUser storedUserToIndexUser storedUser = diff --git a/libs/wire-subsystems/wire-subsystems.cabal b/libs/wire-subsystems/wire-subsystems.cabal index 54ff613f5e4..b19f48031fc 100644 --- a/libs/wire-subsystems/wire-subsystems.cabal +++ b/libs/wire-subsystems/wire-subsystems.cabal @@ -156,6 +156,7 @@ library , bloodhound , bytestring , bytestring-conversion + , case-insensitive , cassandra-util , conduit , containers diff --git a/services/brig/src/Brig/API/Internal.hs b/services/brig/src/Brig/API/Internal.hs index 052c5cdb59f..50b7ad70044 100644 --- a/services/brig/src/Brig/API/Internal.hs +++ b/services/brig/src/Brig/API/Internal.hs @@ -82,6 +82,7 @@ import Wire.API.Routes.FederationDomainConfig import Wire.API.Routes.Internal.Brig qualified as BrigIRoutes import Wire.API.Routes.Internal.Brig.Connection import Wire.API.Routes.Named +import Wire.API.Team.Export import Wire.API.Team.Feature import Wire.API.User import Wire.API.User.Activation @@ -266,6 +267,7 @@ userAPI = updateLocale :<|> deleteLocale :<|> getDefaultUserLocale + :<|> Named @"get-user-export-data" getUserExportDataH clientAPI :: ServerT BrigIRoutes.ClientAPI (Handler r) clientAPI = Named @"update-client-last-active" updateClientLastActive @@ -800,3 +802,9 @@ checkHandleInternalH h = lift $ liftSem do getContactListH :: UserId -> (Handler r) UserIds getContactListH uid = lift . wrapClient $ UserIds <$> API.lookupContactList uid + +getUserExportDataH :: + (Member UserSubsystem r) => + UserId -> + Handler r (Maybe TeamExportUser) +getUserExportDataH = lift . liftSem . getUserExportData diff --git a/services/brig/src/Brig/CanonicalInterpreter.hs b/services/brig/src/Brig/CanonicalInterpreter.hs index b2967854fd6..e0f9672bf57 100644 --- a/services/brig/src/Brig/CanonicalInterpreter.hs +++ b/services/brig/src/Brig/CanonicalInterpreter.hs @@ -134,6 +134,7 @@ type BrigLowerLevelEffects = PropertyStore, SFT, ConnectionStore InternalPaging, + Input UserSubsystemConfig, Input VerificationCodeThrottleTTL, Input UTCTime, Input (Local ()), @@ -213,7 +214,7 @@ runBrigToIO e (AppT ma) = do -- These interpreters depend on each other, we use let recursion to solve that. userSubsystemInterpreter :: (Members BrigLowerLevelEffects r) => InterpreterFor UserSubsystem r - userSubsystemInterpreter = runUserSubsystem userSubsystemConfig authSubsystemInterpreter + userSubsystemInterpreter = runUserSubsystem authSubsystemInterpreter authSubsystemInterpreter :: (Members BrigLowerLevelEffects r) => InterpreterFor AuthenticationSubsystem r authSubsystemInterpreter = interpretAuthenticationSubsystem userSubsystemInterpreter @@ -251,6 +252,7 @@ runBrigToIO e (AppT ma) = do . runInputConst (toLocalUnsafe e.settings.federationDomain ()) . runInputSem (embed getCurrentTime) . runInputConst (fromIntegral $ Opt.twoFACodeGenerationDelaySecs e.settings) + . runInputConst userSubsystemConfig . connectionStoreToCassandra . interpretSFT e.httpManager . interpretPropertyStoreCassandra e.casClient diff --git a/services/brig/src/Brig/Data/User.hs b/services/brig/src/Brig/Data/User.hs index caaa7c160cc..e5772f2912e 100644 --- a/services/brig/src/Brig/Data/User.hs +++ b/services/brig/src/Brig/Data/User.hs @@ -374,6 +374,7 @@ lookupName u = fmap runIdentity <$> retry x1 (query1 nameSelect (params LocalQuorum (Identity u))) +-- TODO: remove this lookupRichInfo :: (MonadClient m) => UserId -> m (Maybe RichInfoAssocList) lookupRichInfo u = fmap runIdentity diff --git a/services/galley/default.nix b/services/galley/default.nix index 446b4c9450e..602549b250b 100644 --- a/services/galley/default.nix +++ b/services/galley/default.nix @@ -19,7 +19,6 @@ , bytestring , bytestring-conversion , call-stack -, case-insensitive , cassandra-util , cassava , cereal @@ -79,7 +78,6 @@ , resourcet , retry , safe-exceptions -, saml2-web-sso , servant , servant-client , servant-client-core @@ -148,7 +146,6 @@ mkDerivation { brig-types bytestring bytestring-conversion - case-insensitive cassandra-util cassava comonad @@ -189,7 +186,6 @@ mkDerivation { resourcet retry safe-exceptions - saml2-web-sso servant servant-client servant-server diff --git a/services/galley/galley.cabal b/services/galley/galley.cabal index ae6bdfc65a4..1db3f95e89c 100644 --- a/services/galley/galley.cabal +++ b/services/galley/galley.cabal @@ -122,6 +122,7 @@ library Galley.API.Push Galley.API.Query Galley.API.Teams + Galley.API.Teams.Export Galley.API.Teams.Features Galley.API.Teams.Features.Get Galley.API.Teams.Notifications @@ -300,7 +301,6 @@ library , brig-types >=0.73.1 , bytestring >=0.9 , bytestring-conversion >=0.2 - , case-insensitive , cassandra-util >=0.16.2 , cassava >=0.5.2 , comonad @@ -341,7 +341,6 @@ library , resourcet >=1.1 , retry >=0.5 , safe-exceptions >=0.1 - , saml2-web-sso >=0.20 , servant , servant-client , servant-server diff --git a/services/galley/src/Galley/API/Public/TeamMember.hs b/services/galley/src/Galley/API/Public/TeamMember.hs index 91956a21712..c6b7d5cd059 100644 --- a/services/galley/src/Galley/API/Public/TeamMember.hs +++ b/services/galley/src/Galley/API/Public/TeamMember.hs @@ -18,6 +18,7 @@ module Galley.API.Public.TeamMember where import Galley.API.Teams +import Galley.API.Teams.Export qualified as Export import Galley.App import Wire.API.Routes.API import Wire.API.Routes.Public.Galley.TeamMember @@ -31,4 +32,4 @@ teamMemberAPI = <@> mkNamedAPI @"delete-team-member" deleteTeamMember <@> mkNamedAPI @"delete-non-binding-team-member" deleteNonBindingTeamMember <@> mkNamedAPI @"update-team-member" updateTeamMember - <@> mkNamedAPI @"get-team-members-csv" getTeamMembersCSV + <@> mkNamedAPI @"get-team-members-csv" Export.getTeamMembersCSV diff --git a/services/galley/src/Galley/API/Teams.hs b/services/galley/src/Galley/API/Teams.hs index b59465923af..e51070d5f5a 100644 --- a/services/galley/src/Galley/API/Teams.hs +++ b/services/galley/src/Galley/API/Teams.hs @@ -31,7 +31,6 @@ module Galley.API.Teams addTeamMember, getTeamConversationRoles, getTeamMembers, - getTeamMembersCSV, bulkGetTeamMembers, getTeamMember, deleteTeamMember, @@ -61,13 +60,9 @@ import Brig.Types.Team (TeamSize (..)) import Cassandra (PageWithState (pwsResults), pwsHasMore) import Cassandra qualified as C import Control.Lens -import Data.ByteString.Builder (lazyByteString) import Data.ByteString.Conversion (List, toByteString) import Data.ByteString.Conversion qualified import Data.ByteString.Lazy qualified as LBS -import Data.CaseInsensitive qualified as CI -import Data.Csv (EncodeOptions (..), Quoting (QuoteAll), encodeDefaultOrderedByNameWith) -import Data.Handle qualified as Handle import Data.Id import Data.Json.Util import Data.LegalHold qualified as LH @@ -75,8 +70,6 @@ import Data.List.Extra qualified as List import Data.List.NonEmpty (NonEmpty (..)) import Data.List1 (list1) import Data.Map qualified as Map -import Data.Map.Strict qualified as M -import Data.Misc (HttpsUrl, mkHttpsUrl) import Data.Proxy import Data.Qualified import Data.Range as Range @@ -112,14 +105,10 @@ import Galley.Types.Conversations.Members qualified as Conv import Galley.Types.Teams import Galley.Types.UserList import Imports hiding (forkIO) -import Network.Wai import Polysemy import Polysemy.Error -import Polysemy.Final import Polysemy.Input -import Polysemy.Output import Polysemy.TinyLog qualified as P -import SAML2.WebSSO qualified as SAML import System.Logger qualified as Log import Wire.API.Conversation (ConversationRemoveMembers (..)) import Wire.API.Conversation.Role (wireConvRoles) @@ -130,7 +119,6 @@ import Wire.API.Event.Conversation qualified as Conv import Wire.API.Event.LeaveReason import Wire.API.Event.Team import Wire.API.Federation.Error -import Wire.API.Message qualified as Conv import Wire.API.Routes.Internal.Galley.TeamsIntra import Wire.API.Routes.MultiTablePaging (MultiTablePage (MultiTablePage), MultiTablePagingState (mtpsState)) import Wire.API.Routes.Public.Galley.TeamMember @@ -138,7 +126,6 @@ import Wire.API.Team import Wire.API.Team qualified as Public import Wire.API.Team.Conversation import Wire.API.Team.Conversation qualified as Public -import Wire.API.Team.Export (TeamExportUser (..)) import Wire.API.Team.Feature import Wire.API.Team.Member import Wire.API.Team.Member qualified as M @@ -147,12 +134,8 @@ import Wire.API.Team.Permission (Perm (..), Permissions (..), SPerm (..), copy, import Wire.API.Team.Role import Wire.API.Team.SearchVisibility import Wire.API.Team.SearchVisibility qualified as Public -import Wire.API.User (ScimUserInfo (..), User, UserIdList, UserSSOId (UserScimExternalId), userSCIMExternalId, userSSOId) import Wire.API.User qualified as U -import Wire.API.User.Identity (UserSSOId (UserSSOId)) -import Wire.API.User.RichInfo (RichInfo) import Wire.NotificationSubsystem -import Wire.Sem.Paging qualified as E import Wire.Sem.Paging.Cassandra getTeamH :: @@ -500,130 +483,6 @@ getTeamMembers lzusr tid mbMaxResults mbPagingState = do (pwsHasMore p) (teamMemberPagingState p) -outputToStreamingBody :: (Member (Final IO) r) => Sem (Output LByteString ': r) () -> Sem r StreamingBody -outputToStreamingBody action = withWeavingToFinal @IO $ \state weave _inspect -> - pure . (<$ state) $ \write flush -> do - let writeChunk c = embedFinal $ do - write (lazyByteString c) - flush - void . weave . (<$ state) $ runOutputSem writeChunk action - -getTeamMembersCSV :: - ( Member BrigAccess r, - Member (ErrorS 'AccessDenied) r, - Member (TeamMemberStore InternalPaging) r, - Member TeamStore r, - Member (Final IO) r, - Member SparAccess r - ) => - Local UserId -> - TeamId -> - Sem r StreamingBody -getTeamMembersCSV lusr tid = do - E.getTeamMember tid (tUnqualified lusr) >>= \case - Nothing -> throwS @'AccessDenied - Just member -> unless (member `hasPermission` DownloadTeamMembersCsv) $ throwS @'AccessDenied - - -- In case an exception is thrown inside the StreamingBody of responseStream - -- the response will not contain a correct error message, but rather be an - -- http error such as 'InvalidChunkHeaders'. The exception however still - -- reaches the middleware and is being tracked in logging and metrics. - outputToStreamingBody $ do - output headerLine - E.withChunks (\mps -> E.listTeamMembers @InternalPaging tid mps maxBound) $ - \members -> do - let uids = fmap (view userId) members - teamExportUser <- - mkTeamExportUser - <$> (lookupUser <$> E.lookupActivatedUsers uids) - <*> lookupInviterHandle members - <*> (lookupRichInfo <$> E.getRichInfoMultiUser uids) - <*> (lookupClients <$> E.lookupClients uids) - <*> (lookupScimUserInfo <$> Spar.lookupScimUserInfos uids) - output @LByteString - ( encodeDefaultOrderedByNameWith - defaultEncodeOptions - (mapMaybe teamExportUser members) - ) - where - headerLine :: LByteString - headerLine = encodeDefaultOrderedByNameWith (defaultEncodeOptions {encIncludeHeader = True}) ([] :: [TeamExportUser]) - - defaultEncodeOptions :: EncodeOptions - defaultEncodeOptions = - EncodeOptions - { encDelimiter = fromIntegral (ord ','), - encUseCrLf = True, -- to be compatible with Mac and Windows - encIncludeHeader = False, -- (so we can flush when the header is on the wire) - encQuoting = QuoteAll - } - - mkTeamExportUser :: - (UserId -> Maybe User) -> - (UserId -> Maybe Handle.Handle) -> - (UserId -> Maybe RichInfo) -> - (UserId -> Int) -> - (UserId -> Maybe ScimUserInfo) -> - TeamMember -> - Maybe TeamExportUser - mkTeamExportUser users inviters richInfos numClients scimUserInfo member = do - let uid = member ^. userId - user <- users uid - pure $ - TeamExportUser - { tExportDisplayName = U.userDisplayName user, - tExportHandle = U.userHandle user, - tExportEmail = U.userIdentity user >>= U.emailIdentity, - tExportRole = permissionsRole . view permissions $ member, - tExportCreatedOn = maybe (scimUserInfo uid >>= suiCreatedOn) (Just . snd) (view invitation member), - tExportInvitedBy = inviters . fst =<< member ^. invitation, - tExportIdpIssuer = userToIdPIssuer user, - tExportManagedBy = U.userManagedBy user, - tExportSAMLNamedId = fromMaybe "" (samlNamedId user), - tExportSCIMExternalId = fromMaybe "" (userSCIMExternalId user), - tExportSCIMRichInfo = richInfos uid, - tExportUserId = U.userId user, - tExportNumDevices = numClients uid - } - - lookupInviterHandle :: (Member BrigAccess r) => [TeamMember] -> Sem r (UserId -> Maybe Handle.Handle) - lookupInviterHandle members = do - let inviterIds :: [UserId] - inviterIds = nub $ mapMaybe (fmap fst . view invitation) members - - userList <- E.getUsers inviterIds - - let userMap :: M.Map UserId Handle.Handle - userMap = M.fromList (mapMaybe extract userList) - where - extract u = (U.userId u,) <$> U.userHandle u - - pure (`M.lookup` userMap) - - userToIdPIssuer :: U.User -> Maybe HttpsUrl - userToIdPIssuer usr = case (U.userIdentity >=> U.ssoIdentity) usr of - Just (U.UserSSOId (SAML.UserRef issuer _)) -> either (const Nothing) Just . mkHttpsUrl $ issuer ^. SAML.fromIssuer - Just _ -> Nothing - Nothing -> Nothing - - lookupScimUserInfo :: [ScimUserInfo] -> (UserId -> Maybe ScimUserInfo) - lookupScimUserInfo infos = (`M.lookup` M.fromList (infos <&> (\sui -> (suiUserId sui, sui)))) - - lookupUser :: [U.User] -> (UserId -> Maybe U.User) - lookupUser users = (`M.lookup` M.fromList (users <&> \user -> (U.userId user, user))) - - lookupRichInfo :: [(UserId, RichInfo)] -> (UserId -> Maybe RichInfo) - lookupRichInfo pairs = (`M.lookup` M.fromList pairs) - - lookupClients :: Conv.UserClients -> UserId -> Int - lookupClients userClients uid = maybe 0 length (M.lookup uid (Conv.userClients userClients)) - - samlNamedId :: User -> Maybe Text - samlNamedId = - userSSOId >=> \case - (UserSSOId (SAML.UserRef _idp nameId)) -> Just . CI.original . SAML.unsafeShowNameID $ nameId - (UserScimExternalId _) -> Nothing - -- | like 'getTeamMembers', but with an explicit list of users we are to return. bulkGetTeamMembers :: ( Member (ErrorS 'BulkGetMemberLimitExceeded) r, @@ -633,7 +492,7 @@ bulkGetTeamMembers :: Local UserId -> TeamId -> Maybe (Range 1 HardTruncationLimit Int32) -> - UserIdList -> + U.UserIdList -> Sem r TeamMemberListOptPerms bulkGetTeamMembers lzusr tid mbMaxResults uids = do unless (length (U.mUsers uids) <= fromIntegral (fromRange (fromMaybe (unsafeRange Public.hardTruncationLimit) mbMaxResults))) $ diff --git a/services/galley/src/Galley/API/Teams/Export.hs b/services/galley/src/Galley/API/Teams/Export.hs new file mode 100644 index 00000000000..273cd18972c --- /dev/null +++ b/services/galley/src/Galley/API/Teams/Export.hs @@ -0,0 +1,180 @@ +module Galley.API.Teams.Export (getTeamMembersCSV) where + +import Control.Concurrent +import Control.Concurrent.Async qualified as Async +import Control.Lens (view, (^.)) +import Control.Monad.Codensity +import Data.ByteString (toStrict) +import Data.ByteString.Builder +import Data.Csv +import Data.Handle +import Data.IORef (atomicModifyIORef, newIORef) +import Data.Id +import Data.Map qualified as Map +import Data.Qualified (Local, tUnqualified) +import Galley.Effects +import Galley.Effects.BrigAccess +import Galley.Effects.SparAccess qualified as Spar +import Galley.Effects.TeamMemberStore (listTeamMembers) +import Galley.Effects.TeamStore +import Imports hiding (atomicModifyIORef, newEmptyMVar, newIORef, putMVar, readMVar, takeMVar, threadDelay, tryPutMVar) +import Polysemy +import Polysemy.Async +import Polysemy.Resource +import Wire.API.Error +import Wire.API.Error.Galley +import Wire.API.Routes.LowLevelStream (LowLevelStreamingBody) +import Wire.API.Team.Export +import Wire.API.Team.Member +import Wire.API.User (ScimUserInfo (suiCreatedOn), User (..)) +import Wire.Sem.Concurrency +import Wire.Sem.Concurrency.IO +import Wire.Sem.Paging qualified as E +import Wire.Sem.Paging.Cassandra (InternalPaging) + +-- | Cache of inviter handles. +-- +-- This is used to make sure that inviters are only looked up once in brig, +-- even if they appear as inviters of several users in the team. +type InviterCache = IORef (Map UserId (MVar (Maybe Handle))) + +lookupInviter :: + (Member Resource r, Member BrigAccess r, Member (Final IO) r) => + InviterCache -> + UserId -> + Sem r (Maybe Handle) +lookupInviter cache uid = flip onException ensureCache $ do + empty <- embedFinal newEmptyMVar + (cached, var) <- + embedFinal $ atomicModifyIORef cache $ \m -> case Map.lookup uid m of + Nothing -> (Map.insert uid empty m, (False, empty)) + Just v -> (m, (True, v)) + -- the cache did not contain this user, so write it in the corresponding MVar + unless cached $ do + u <- listToMaybe <$> getUsers [uid] + embedFinal $ putMVar var (u >>= userHandle) + -- at this point, we know that the MVar contains a value or some other thread + -- is about to write one, so it is safe to just read from the MVar with a + -- blocking call + embedFinal $ readMVar var + where + -- this is run in case of errors to guarantee that other threads will never + -- deadlock while reading the cache + ensureCache = embedFinal $ do + m <- readIORef cache + for_ (Map.lookup uid m) $ \var -> + tryPutMVar var Nothing + +getUserRecord :: + ( Member BrigAccess r, + Member Spar.SparAccess r, + Member (ErrorS TeamMemberNotFound) r, + Member (Final IO) r, + Member Resource r + ) => + InviterCache -> + TeamMember -> + Sem r TeamExportUser +getUserRecord cache member = do + let uid = member ^. userId + export <- getUserExportData uid >>= noteS @TeamMemberNotFound + mCreatedOn <- do + let mFromInvitation = snd <$> member ^. invitation + case mFromInvitation of + Just ts -> pure $ Just ts + Nothing -> do + -- TODO: make this a single user query + suis <- Spar.lookupScimUserInfos [uid] + pure $ listToMaybe suis >>= suiCreatedOn + -- look up inviter handle from the cache + let mInviterId = fst <$> member ^. invitation + invitedBy <- join <$> traverse (lookupInviter cache) mInviterId + pure + export + { tExportInvitedBy = invitedBy, + tExportRole = permissionsRole . view permissions $ member, + tExportCreatedOn = mCreatedOn + } + +-- | Export team info as a CSV, and stream it to the client. +-- +-- We paginate through the team member list, then spawn a thread for each user +-- (out of a thread pool) in order to fetch information for that user from brig +-- and spar. Inviter IDs are resolved to handles via a brig request, then +-- stored in a cache so that they can be reused by subsequent requests. +getTeamMembersCSV :: + forall r. + ( Member BrigAccess r, + Member (ErrorS 'AccessDenied) r, + Member (TeamMemberStore InternalPaging) r, + Member TeamStore r, + Member (Final IO) r, + Member SparAccess r + ) => + Local UserId -> + TeamId -> + Sem r LowLevelStreamingBody +getTeamMembersCSV lusr tid = do + getTeamMember tid (tUnqualified lusr) >>= \case + Nothing -> throwS @'AccessDenied + Just member -> unless (member `hasPermission` DownloadTeamMembersCsv) $ throwS @'AccessDenied + + chan <- embedFinal newChan + cache <- embedFinal $ newIORef mempty + + let encodeRow r = encodeDefaultOrderedByNameWith customEncodeOptions [r] + let produceTeamExportUsers = do + embedFinal $ writeChan chan (Just headerLine) + E.withChunks (\mps -> listTeamMembers @InternalPaging tid mps maxBound) $ + \members -> unsafePooledForConcurrentlyN_ 8 members $ \member -> do + mRecord <- + runErrorS @TeamMemberNotFound $ + getUserRecord cache member + let mRow = encodeRow <$> mRecord + when (isJust mRow) $ + embedFinal $ + writeChan chan mRow + + -- In case an exception is thrown inside the producer thread, the response + -- will not contain a correct error message, but rather be an http error such + -- as 'InvalidChunkHeaders'. The exception however still reaches the + -- middleware and is being tracked in logging and metrics. + let producerThread = + produceTeamExportUsers + `finally` embedFinal (writeChan chan Nothing) + + asyncToIOFinal . resourceToIOFinal . unsafelyPerformConcurrency @_ @Unsafe $ do + -- Here we should really capture the Wai continuation and run the finaliser + -- after that. Unfortunately, this is not really possible with Servant, + -- because the continuation is not exposed by the Handler monad. The best + -- we can do is return a Codensity value with the correct finaliser, but + -- that still leaves a short window between when the resource is acquired + -- and when the finaliser is installed where the resource might be leaked. + -- I don't have a good solution for that. + bracketOnError + (async producerThread) + cancel + $ \producer -> do + pure $ do + void $ Codensity $ \k -> do + r <- k () + Async.cancel producer + pure r + pure $ \write flush -> do + let go = do + readChan chan >>= \case + Nothing -> write "" >> flush + Just line -> write (byteString (toStrict line)) >> flush >> go + go + +headerLine :: LByteString +headerLine = encodeDefaultOrderedByNameWith (customEncodeOptions {encIncludeHeader = True}) ([] :: [TeamExportUser]) + +customEncodeOptions :: EncodeOptions +customEncodeOptions = + EncodeOptions + { encDelimiter = fromIntegral (ord ','), + encUseCrLf = True, -- to be compatible with Mac and Windows + encIncludeHeader = False, -- (so we can flush when the header is on the wire) + encQuoting = QuoteAll + } diff --git a/services/galley/src/Galley/Effects/BrigAccess.hs b/services/galley/src/Galley/Effects/BrigAccess.hs index de7fc43bd5b..0e5725513c4 100644 --- a/services/galley/src/Galley/Effects/BrigAccess.hs +++ b/services/galley/src/Galley/Effects/BrigAccess.hs @@ -35,6 +35,7 @@ module Galley.Effects.BrigAccess deleteUser, getContactList, getRichInfoMultiUser, + getUserExportData, -- * Teams getSize, @@ -71,6 +72,7 @@ import Wire.API.Error.Galley import Wire.API.MLS.CipherSuite import Wire.API.Routes.Internal.Brig.Connection import Wire.API.Routes.Internal.Galley.TeamFeatureNoConfigMulti qualified as Multi +import Wire.API.Team.Export import Wire.API.Team.Feature import Wire.API.Team.Size import Wire.API.User.Auth.ReAuth @@ -126,6 +128,7 @@ data BrigAccess m a where UpdateSearchVisibilityInbound :: Multi.TeamStatus SearchVisibilityInboundConfig -> BrigAccess m () + GetUserExportData :: UserId -> BrigAccess m (Maybe TeamExportUser) makeSem ''BrigAccess diff --git a/services/galley/src/Galley/Intra/Effects.hs b/services/galley/src/Galley/Intra/Effects.hs index ef071400ab0..29e39798ac2 100644 --- a/services/galley/src/Galley/Intra/Effects.hs +++ b/services/galley/src/Galley/Intra/Effects.hs @@ -83,6 +83,9 @@ interpretBrigAccess = interpret $ \case GetRichInfoMultiUser uids -> do logEffect "BrigAccess.GetRichInfoMultiUser" embedApp $ getRichInfoMultiUser uids + GetUserExportData uid -> do + logEffect "BrigAccess.GetUserExportData" + embedApp $ getUserExportData uid GetSize tid -> do logEffect "BrigAccess.GetSize" embedApp $ getSize tid diff --git a/services/galley/src/Galley/Intra/User.hs b/services/galley/src/Galley/Intra/User.hs index 8d6c620fd66..27ced33fdee 100644 --- a/services/galley/src/Galley/Intra/User.hs +++ b/services/galley/src/Galley/Intra/User.hs @@ -30,6 +30,7 @@ module Galley.Intra.User getContactList, chunkify, getRichInfoMultiUser, + getUserExportData, getAccountConferenceCallingConfigClient, updateSearchVisibilityInbound, ) @@ -66,6 +67,7 @@ import Wire.API.Routes.Internal.Brig qualified as IAPI import Wire.API.Routes.Internal.Brig.Connection import Wire.API.Routes.Internal.Galley.TeamFeatureNoConfigMulti qualified as Multi import Wire.API.Routes.Named +import Wire.API.Team.Export import Wire.API.Team.Feature import Wire.API.User import Wire.API.User.Auth.ReAuth @@ -237,6 +239,16 @@ getRichInfoMultiUser = chunkify $ \uids -> do . expect2xx parseResponse (mkError status502 "server-error: could not parse response to `GET brig:/i/users/rich-info`") resp +-- | Calls 'Brig.API.Internal.getUserExportDataH' +getUserExportData :: UserId -> App (Maybe TeamExportUser) +getUserExportData uid = do + resp <- + call Brig $ + method GET + . paths ["i/users", toByteString' uid, "export-data"] + . expect2xx + parseResponse (mkError status502 "server-error: could not parse response to `GET brig:/i/users/:uid/export-data`") resp + getAccountConferenceCallingConfigClient :: (HasCallStack) => UserId -> App (Feature ConferenceCallingConfig) getAccountConferenceCallingConfigClient uid = runHereClientM (namedClient @IAPI.API @"get-account-conference-calling-config" uid) diff --git a/tools/stern/default.nix b/tools/stern/default.nix index 18246b4fc52..81032346144 100644 --- a/tools/stern/default.nix +++ b/tools/stern/default.nix @@ -42,6 +42,7 @@ , tasty-ant-xml , tasty-hunit , text +, time , tinylog , transformers , types-common @@ -83,6 +84,7 @@ mkDerivation { servant-swagger-ui split text + time tinylog transformers types-common diff --git a/tools/stern/src/Stern/API.hs b/tools/stern/src/Stern/API.hs index 3e915b2a69e..53616908c64 100644 --- a/tools/stern/src/Stern/API.hs +++ b/tools/stern/src/Stern/API.hs @@ -62,7 +62,7 @@ import Stern.App import Stern.Intra qualified as Intra import Stern.Options import Stern.Types -import System.Logger.Class hiding (Error, name, trace, (.=)) +import System.Logger.Class hiding (Error, flush, name, trace, (.=)) import Util.Options import Wire.API.Connection import Wire.API.Internal.Notification (QueuedNotification) diff --git a/tools/stern/src/Stern/App.hs b/tools/stern/src/Stern/App.hs index 1056cf37182..612c9c67cd8 100644 --- a/tools/stern/src/Stern/App.hs +++ b/tools/stern/src/Stern/App.hs @@ -124,6 +124,9 @@ runAppT e (AppT ma) = runReaderT ma e type Handler = ExceptT Error App +runHandler :: Env -> Handler a -> IO (Either Error a) +runHandler env = runAppT env . runExceptT + type Continue m = Response -> m ResponseReceived userMsg :: UserId -> Msg -> Msg diff --git a/tools/stern/src/Stern/Intra.hs b/tools/stern/src/Stern/Intra.hs index f72649bba90..d226f49f252 100644 --- a/tools/stern/src/Stern/Intra.hs +++ b/tools/stern/src/Stern/Intra.hs @@ -38,6 +38,7 @@ module Stern.Intra setStatusBindingTeam, deleteBindingTeam, deleteBindingTeamForce, + getTeamMembers, getTeamInfo, getUserBindingTeam, isBlacklisted, @@ -66,6 +67,7 @@ module Stern.Intra getOAuthClient, updateOAuthClient, deleteOAuthClient, + getActivityTimestamp, ) where @@ -92,6 +94,7 @@ import Data.Text.Encoding import Data.Text.Encoding.Error import Data.Text.Lazy as LT (pack) import Data.Text.Lazy.Encoding qualified as TL +import Data.Time.Clock import Imports import Network.HTTP.Types (urlEncode) import Network.HTTP.Types.Method @@ -1037,3 +1040,17 @@ deleteOAuthClient cid = do . expect2xx ) parseResponse (mkError status502 "bad-upstream") r + +getActivityTimestamp :: UserId -> Handler (Maybe UTCTime) +getActivityTimestamp uid = do + b <- asks (.brig) + r <- + catchRpcErrors $ + rpc' + "brig" + b + ( method GET + . Bilge.paths ["i", "users", toByteString' uid, "activity"] + . expect2xx + ) + parseResponse (mkError status502 "bad-upstream") r diff --git a/tools/stern/stern.cabal b/tools/stern/stern.cabal index b7e04c9de2b..36b5a86ca65 100644 --- a/tools/stern/stern.cabal +++ b/tools/stern/stern.cabal @@ -96,6 +96,7 @@ library , servant-swagger-ui , split >=0.2 , text >=1.1 + , time , tinylog >=0.10 , transformers >=0.3 , types-common >=0.4.13