From 0ea0211fca87c453d78e3fe7f509bff88403d592 Mon Sep 17 00:00:00 2001 From: Ankit Tiwari Date: Wed, 20 Mar 2024 12:58:56 +0530 Subject: [PATCH 01/21] feat: Add BulkImport APIs and cron --- src/main/java/io/supertokens/Main.java | 4 + .../io/supertokens/bulkimport/BulkImport.java | 90 +++ .../BulkImportUserPaginationContainer.java | 34 + .../BulkImportUserPaginationToken.java | 53 ++ .../bulkimport/BulkImportUserUtils.java | 551 ++++++++++++++ .../InvalidBulkImportDataException.java | 33 + .../bulkimport/ProcessBulkImportUsers.java | 504 +++++++++++++ .../emailpassword/EmailPassword.java | 68 +- .../java/io/supertokens/inmemorydb/Start.java | 5 + .../passwordless/Passwordless.java | 88 ++- .../storageLayer/StorageLayer.java | 14 +- .../io/supertokens/thirdparty/ThirdParty.java | 39 +- .../supertokens/utils/JsonValidatorUtils.java | 123 ++++ .../io/supertokens/webserver/Webserver.java | 3 + .../api/bulkimport/BulkImportAPI.java | 250 +++++++ .../test/bulkimport/BulkImportTest.java | 265 +++++++ .../test/bulkimport/BulkImportTestUtils.java | 59 ++ .../ProcessBulkImportUsersCronJobTest.java | 297 ++++++++ .../apis/AddBulkImportUsersTest.java | 692 ++++++++++++++++++ .../apis/DeleteBulkImportUsersTest.java | 173 +++++ .../apis/GetBulkImportUsersTest.java | 160 ++++ 21 files changed, 3420 insertions(+), 85 deletions(-) create mode 100644 src/main/java/io/supertokens/bulkimport/BulkImport.java create mode 100644 src/main/java/io/supertokens/bulkimport/BulkImportUserPaginationContainer.java create mode 100644 src/main/java/io/supertokens/bulkimport/BulkImportUserPaginationToken.java create mode 100644 src/main/java/io/supertokens/bulkimport/BulkImportUserUtils.java create mode 100644 src/main/java/io/supertokens/bulkimport/exceptions/InvalidBulkImportDataException.java create mode 100644 src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkImportUsers.java create mode 100644 src/main/java/io/supertokens/utils/JsonValidatorUtils.java create mode 100644 src/main/java/io/supertokens/webserver/api/bulkimport/BulkImportAPI.java create mode 100644 src/test/java/io/supertokens/test/bulkimport/BulkImportTest.java create mode 100644 src/test/java/io/supertokens/test/bulkimport/BulkImportTestUtils.java create mode 100644 src/test/java/io/supertokens/test/bulkimport/ProcessBulkImportUsersCronJobTest.java create mode 100644 src/test/java/io/supertokens/test/bulkimport/apis/AddBulkImportUsersTest.java create mode 100644 src/test/java/io/supertokens/test/bulkimport/apis/DeleteBulkImportUsersTest.java create mode 100644 src/test/java/io/supertokens/test/bulkimport/apis/GetBulkImportUsersTest.java diff --git a/src/main/java/io/supertokens/Main.java b/src/main/java/io/supertokens/Main.java index 2998efb7b..7375e6e08 100644 --- a/src/main/java/io/supertokens/Main.java +++ b/src/main/java/io/supertokens/Main.java @@ -20,6 +20,7 @@ import io.supertokens.config.Config; import io.supertokens.config.CoreConfig; import io.supertokens.cronjobs.Cronjobs; +import io.supertokens.cronjobs.bulkimport.ProcessBulkImportUsers; import io.supertokens.cronjobs.deleteExpiredAccessTokenSigningKeys.DeleteExpiredAccessTokenSigningKeys; import io.supertokens.cronjobs.deleteExpiredDashboardSessions.DeleteExpiredDashboardSessions; import io.supertokens.cronjobs.deleteExpiredEmailVerificationTokens.DeleteExpiredEmailVerificationTokens; @@ -254,6 +255,9 @@ private void init() throws IOException, StorageQueryException { // starts DeleteExpiredAccessTokenSigningKeys cronjob if the access token signing keys can change Cronjobs.addCronjob(this, DeleteExpiredAccessTokenSigningKeys.init(this, uniqueUserPoolIdsTenants)); + // starts ProcessBulkImportUsers cronjob to process bulk import users + Cronjobs.addCronjob(this, ProcessBulkImportUsers.init(this, uniqueUserPoolIdsTenants)); + // this is to ensure tenantInfos are in sync for the new cron job as well MultitenancyHelper.getInstance(this).refreshCronjobs(); diff --git a/src/main/java/io/supertokens/bulkimport/BulkImport.java b/src/main/java/io/supertokens/bulkimport/BulkImport.java new file mode 100644 index 000000000..8bfabc7e1 --- /dev/null +++ b/src/main/java/io/supertokens/bulkimport/BulkImport.java @@ -0,0 +1,90 @@ +/* + * Copyright (c) 2024, VRAI Labs and/or its affiliates. All rights reserved. + * + * This software is licensed under the Apache License, Version 2.0 (the + * "License") as published by the Apache Software Foundation. + * + * You may not use this file except in compliance with the License. You may + * obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package io.supertokens.bulkimport; + +import io.supertokens.pluginInterface.bulkimport.BulkImportStorage.BULK_IMPORT_USER_STATUS; +import io.supertokens.pluginInterface.bulkimport.sqlStorage.BulkImportSQLStorage; +import io.supertokens.pluginInterface.Storage; +import io.supertokens.pluginInterface.StorageUtils; +import io.supertokens.pluginInterface.bulkimport.BulkImportUser; +import io.supertokens.pluginInterface.exceptions.StorageQueryException; +import io.supertokens.pluginInterface.multitenancy.AppIdentifier; +import io.supertokens.pluginInterface.multitenancy.exceptions.TenantOrAppNotFoundException; +import io.supertokens.utils.Utils; + + +import java.util.List; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; + +public class BulkImport { + + public static final int MAX_USERS_TO_ADD = 10000; + public static final int GET_USERS_PAGINATION_LIMIT = 500; + public static final int GET_USERS_DEFAULT_LIMIT = 100; + public static final int DELETE_USERS_LIMIT = 500; + public static final int PROCESS_USERS_BATCH_SIZE = 1000; + public static final int PROCESS_USERS_INTERVAL = 60; + + public static void addUsers(AppIdentifier appIdentifier, Storage storage, List users) + throws StorageQueryException, TenantOrAppNotFoundException { + while (true) { + try { + StorageUtils.getBulkImportStorage(storage).addBulkImportUsers(appIdentifier, users); + break; + } catch (io.supertokens.pluginInterface.bulkimport.exceptions.DuplicateUserIdException ignored) { + // We re-generate the user id for every user and retry + for (BulkImportUser user : users) { + user.id = Utils.getUUID(); + } + } + } + } + + public static BulkImportUserPaginationContainer getUsers(AppIdentifier appIdentifier, Storage storage, + @Nonnull Integer limit, @Nullable BULK_IMPORT_USER_STATUS status, @Nullable String paginationToken) + throws StorageQueryException, BulkImportUserPaginationToken.InvalidTokenException { + List users; + + BulkImportSQLStorage bulkImportStorage = StorageUtils.getBulkImportStorage(storage); + + if (paginationToken == null) { + users = bulkImportStorage + .getBulkImportUsers(appIdentifier, limit + 1, status, null, null); + } else { + BulkImportUserPaginationToken tokenInfo = BulkImportUserPaginationToken.extractTokenInfo(paginationToken); + users = bulkImportStorage + .getBulkImportUsers(appIdentifier, limit + 1, status, tokenInfo.bulkImportUserId, tokenInfo.createdAt); + } + + String nextPaginationToken = null; + int maxLoop = users.size(); + if (users.size() == limit + 1) { + maxLoop = limit; + BulkImportUser user = users.get(limit); + nextPaginationToken = new BulkImportUserPaginationToken(user.id, user.createdAt).generateToken(); + } + + List resultUsers = users.subList(0, maxLoop); + return new BulkImportUserPaginationContainer(resultUsers, nextPaginationToken); + } + + public static List deleteUsers(AppIdentifier appIdentifier, Storage storage, String[] userIds) throws StorageQueryException { + return StorageUtils.getBulkImportStorage(storage).deleteBulkImportUsers(appIdentifier, userIds); + } +} diff --git a/src/main/java/io/supertokens/bulkimport/BulkImportUserPaginationContainer.java b/src/main/java/io/supertokens/bulkimport/BulkImportUserPaginationContainer.java new file mode 100644 index 000000000..d2bd21634 --- /dev/null +++ b/src/main/java/io/supertokens/bulkimport/BulkImportUserPaginationContainer.java @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2024, VRAI Labs and/or its affiliates. All rights reserved. + * + * This software is licensed under the Apache License, Version 2.0 (the + * "License") as published by the Apache Software Foundation. + * + * You may not use this file except in compliance with the License. You may + * obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + + package io.supertokens.bulkimport; + +import java.util.List; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; + +import io.supertokens.pluginInterface.bulkimport.BulkImportUser; + +public class BulkImportUserPaginationContainer { + public final List users; + public final String nextPaginationToken; + + public BulkImportUserPaginationContainer(@Nonnull List users, @Nullable String nextPaginationToken) { + this.users = users; + this.nextPaginationToken = nextPaginationToken; + } +} diff --git a/src/main/java/io/supertokens/bulkimport/BulkImportUserPaginationToken.java b/src/main/java/io/supertokens/bulkimport/BulkImportUserPaginationToken.java new file mode 100644 index 000000000..8a492c2ca --- /dev/null +++ b/src/main/java/io/supertokens/bulkimport/BulkImportUserPaginationToken.java @@ -0,0 +1,53 @@ +/* + * Copyright (c) 2024, VRAI Labs and/or its affiliates. All rights reserved. + * + * This software is licensed under the Apache License, Version 2.0 (the + * "License") as published by the Apache Software Foundation. + * + * You may not use this file except in compliance with the License. You may + * obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package io.supertokens.bulkimport; + +import java.util.Base64; + +public class BulkImportUserPaginationToken { + public final String bulkImportUserId; + public final long createdAt; + + public BulkImportUserPaginationToken(String bulkImportUserId, long createdAt) { + this.bulkImportUserId = bulkImportUserId; + this.createdAt = createdAt; + } + + public static BulkImportUserPaginationToken extractTokenInfo(String token) throws InvalidTokenException { + try { + String decodedPaginationToken = new String(Base64.getDecoder().decode(token)); + String[] splitDecodedToken = decodedPaginationToken.split(";"); + if (splitDecodedToken.length != 2) { + throw new InvalidTokenException(); + } + String bulkImportUserId = splitDecodedToken[0]; + long createdAt = Long.parseLong(splitDecodedToken[1]); + return new BulkImportUserPaginationToken(bulkImportUserId, createdAt); + } catch (Exception e) { + throw new InvalidTokenException(); + } + } + + public String generateToken() { + return new String(Base64.getEncoder().encode((this.bulkImportUserId + ";" + this.createdAt).getBytes())); + } + + public static class InvalidTokenException extends Exception { + + private static final long serialVersionUID = 6289026174830695478L; + } +} diff --git a/src/main/java/io/supertokens/bulkimport/BulkImportUserUtils.java b/src/main/java/io/supertokens/bulkimport/BulkImportUserUtils.java new file mode 100644 index 000000000..834ed0e4a --- /dev/null +++ b/src/main/java/io/supertokens/bulkimport/BulkImportUserUtils.java @@ -0,0 +1,551 @@ +/* + * Copyright (c) 2024, VRAI Labs and/or its affiliates. All rights reserved. + * + * This software is licensed under the Apache License, Version 2.0 (the + * "License") as published by the Apache Software Foundation. + * + * You may not use this file except in compliance with the License. You may + * obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package io.supertokens.bulkimport; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; + +import io.supertokens.Main; +import io.supertokens.bulkimport.exceptions.InvalidBulkImportDataException; +import io.supertokens.config.CoreConfig; +import io.supertokens.emailpassword.PasswordHashingUtils; +import io.supertokens.emailpassword.exceptions.UnsupportedPasswordHashingFormatException; +import io.supertokens.featureflag.EE_FEATURES; +import io.supertokens.featureflag.FeatureFlag; +import io.supertokens.multitenancy.Multitenancy; +import io.supertokens.pluginInterface.Storage; +import io.supertokens.pluginInterface.bulkimport.BulkImportUser; +import io.supertokens.pluginInterface.bulkimport.BulkImportUser.LoginMethod; +import io.supertokens.pluginInterface.bulkimport.BulkImportUser.UserRole; +import io.supertokens.pluginInterface.bulkimport.BulkImportUser.TotpDevice; +import io.supertokens.pluginInterface.exceptions.StorageQueryException; +import io.supertokens.pluginInterface.multitenancy.AppIdentifier; +import io.supertokens.pluginInterface.multitenancy.TenantConfig; +import io.supertokens.pluginInterface.multitenancy.TenantIdentifier; +import io.supertokens.pluginInterface.multitenancy.exceptions.TenantOrAppNotFoundException; +import io.supertokens.storageLayer.StorageLayer; +import io.supertokens.utils.Utils; +import io.supertokens.utils.JsonValidatorUtils.ValueType; + +import static io.supertokens.utils.JsonValidatorUtils.parseAndValidateFieldType; +import static io.supertokens.utils.JsonValidatorUtils.validateJsonFieldType; + +public class BulkImportUserUtils { + public static BulkImportUser createBulkImportUserFromJSON(Main main, AppIdentifier appIdentifier, + JsonObject userData, String id, String[] allUserRoles, Set allExternalUserIds) + throws InvalidBulkImportDataException, StorageQueryException, TenantOrAppNotFoundException { + List errors = new ArrayList<>(); + + String externalUserId = parseAndValidateFieldType(userData, "externalUserId", ValueType.STRING, false, + String.class, + errors, "."); + JsonObject userMetadata = parseAndValidateFieldType(userData, "userMetadata", ValueType.OBJECT, false, + JsonObject.class, errors, "."); + List userRoles = getParsedUserRoles(main, appIdentifier, userData, allUserRoles, errors); + List totpDevices = getParsedTotpDevices(userData, errors); + List loginMethods = getParsedLoginMethods(main, appIdentifier, userData, errors); + + externalUserId = validateAndNormaliseExternalUserId(externalUserId, allExternalUserIds, errors); + + validateTenantIdsForRoleAndLoginMethods(main, appIdentifier, userRoles, loginMethods, errors); + + if (!errors.isEmpty()) { + throw new InvalidBulkImportDataException(errors); + } + return new BulkImportUser(id, externalUserId, userMetadata, userRoles, totpDevices, loginMethods); + } + + private static List getParsedUserRoles(Main main, AppIdentifier appIdentifier, JsonObject userData, + String[] allUserRoles, List errors) throws StorageQueryException, TenantOrAppNotFoundException { + JsonArray jsonUserRoles = parseAndValidateFieldType(userData, "userRoles", ValueType.ARRAY_OF_OBJECT, false, + JsonArray.class, errors, "."); + + if (jsonUserRoles == null) { + return null; + } + + List userRoles = new ArrayList<>(); + + for (JsonElement jsonUserRoleEl : jsonUserRoles) { + JsonObject jsonUserRole = jsonUserRoleEl.getAsJsonObject(); + + String role = parseAndValidateFieldType(jsonUserRole, "role", ValueType.STRING, true, String.class, errors, + " for a user role."); + JsonArray jsonTenantIds = parseAndValidateFieldType(jsonUserRole, "tenantIds", ValueType.ARRAY_OF_STRING, + true, JsonArray.class, errors, " for a user role."); + + role = validateAndNormaliseUserRole(role, allUserRoles, errors); + List normalisedTenantIds = validateAndNormaliseTenantIds(main, appIdentifier, jsonTenantIds, errors, + " for a user role."); + + if (role != null && normalisedTenantIds != null) { + userRoles.add(new UserRole(role, normalisedTenantIds)); + } + } + return userRoles; + } + + private static List getParsedTotpDevices(JsonObject userData, List errors) { + JsonArray jsonTotpDevices = parseAndValidateFieldType(userData, "totpDevices", ValueType.ARRAY_OF_OBJECT, false, + JsonArray.class, errors, "."); + + if (jsonTotpDevices == null) { + return null; + } + + List totpDevices = new ArrayList<>(); + for (JsonElement jsonTotpDeviceEl : jsonTotpDevices) { + JsonObject jsonTotpDevice = jsonTotpDeviceEl.getAsJsonObject(); + + String secretKey = parseAndValidateFieldType(jsonTotpDevice, "secretKey", ValueType.STRING, true, + String.class, errors, " for a totp device."); + Integer period = parseAndValidateFieldType(jsonTotpDevice, "period", ValueType.INTEGER, false, + Integer.class, errors, " for a totp device."); + Integer skew = parseAndValidateFieldType(jsonTotpDevice, "skew", ValueType.INTEGER, false, Integer.class, + errors, " for a totp device."); + String deviceName = parseAndValidateFieldType(jsonTotpDevice, "deviceName", ValueType.STRING, false, + String.class, errors, " for a totp device."); + + secretKey = validateAndNormaliseTotpSecretKey(secretKey, errors); + period = validateAndNormaliseTotpPeriod(period, errors); + skew = validateAndNormaliseTotpSkew(skew, errors); + deviceName = validateAndNormaliseTotpDeviceName(deviceName, errors); + + if (secretKey != null && period != null && skew != null) { + totpDevices.add(new TotpDevice(secretKey, period, skew, deviceName)); + } + } + return totpDevices; + } + + private static List getParsedLoginMethods(Main main, AppIdentifier appIdentifier, JsonObject userData, + List errors) + throws StorageQueryException, TenantOrAppNotFoundException { + JsonArray jsonLoginMethods = parseAndValidateFieldType(userData, "loginMethods", ValueType.ARRAY_OF_OBJECT, + true, JsonArray.class, errors, "."); + + if (jsonLoginMethods == null) { + return new ArrayList<>(); + } + + if (jsonLoginMethods.size() == 0) { + errors.add("At least one loginMethod is required."); + return new ArrayList<>(); + } + + validateAndNormaliseIsPrimaryField(jsonLoginMethods, errors); + + List loginMethods = new ArrayList<>(); + + for (JsonElement jsonLoginMethod : jsonLoginMethods) { + JsonObject jsonLoginMethodObj = jsonLoginMethod.getAsJsonObject(); + + String recipeId = parseAndValidateFieldType(jsonLoginMethodObj, "recipeId", ValueType.STRING, true, + String.class, errors, " for a loginMethod."); + JsonArray tenantIds = parseAndValidateFieldType(jsonLoginMethodObj, "tenantIds", ValueType.ARRAY_OF_STRING, + false, JsonArray.class, errors, " for a loginMethod."); + Boolean isVerified = parseAndValidateFieldType(jsonLoginMethodObj, "isVerified", ValueType.BOOLEAN, false, + Boolean.class, errors, " for a loginMethod."); + Boolean isPrimary = parseAndValidateFieldType(jsonLoginMethodObj, "isPrimary", ValueType.BOOLEAN, false, + Boolean.class, errors, " for a loginMethod."); + Long timeJoined = parseAndValidateFieldType(jsonLoginMethodObj, "timeJoinedInMSSinceEpoch", ValueType.LONG, + false, Long.class, errors, " for a loginMethod"); + + recipeId = validateAndNormaliseRecipeId(recipeId, errors); + List normalisedTenantIds = validateAndNormaliseTenantIds(main, appIdentifier, tenantIds, errors, + " for " + recipeId + " recipe."); + isPrimary = validateAndNormaliseIsPrimary(isPrimary); + isVerified = validateAndNormaliseIsVerified(isVerified); + + long timeJoinedInMSSinceEpoch = validateAndNormaliseTimeJoined(timeJoined, errors); + + if ("emailpassword".equals(recipeId)) { + String email = parseAndValidateFieldType(jsonLoginMethodObj, "email", ValueType.STRING, true, + String.class, errors, " for an emailpassword recipe."); + String passwordHash = parseAndValidateFieldType(jsonLoginMethodObj, "passwordHash", ValueType.STRING, + true, String.class, errors, " for an emailpassword recipe."); + String hashingAlgorithm = parseAndValidateFieldType(jsonLoginMethodObj, "hashingAlgorithm", + ValueType.STRING, true, String.class, errors, " for an emailpassword recipe."); + + email = validateAndNormaliseEmail(email, errors); + CoreConfig.PASSWORD_HASHING_ALG normalisedHashingAlgorithm = validateAndNormaliseHashingAlgorithm( + hashingAlgorithm, errors); + hashingAlgorithm = normalisedHashingAlgorithm != null ? normalisedHashingAlgorithm.toString() + : hashingAlgorithm; + passwordHash = validateAndNormalisePasswordHash(main, appIdentifier, normalisedHashingAlgorithm, + passwordHash, errors); + + loginMethods.add(new LoginMethod(normalisedTenantIds, recipeId, isVerified, isPrimary, + timeJoinedInMSSinceEpoch, email, passwordHash, hashingAlgorithm, null, null, null)); + } else if ("thirdparty".equals(recipeId)) { + String email = parseAndValidateFieldType(jsonLoginMethodObj, "email", ValueType.STRING, true, + String.class, errors, " for a thirdparty recipe."); + String thirdPartyId = parseAndValidateFieldType(jsonLoginMethodObj, "thirdPartyId", ValueType.STRING, + true, String.class, errors, " for a thirdparty recipe."); + String thirdPartyUserId = parseAndValidateFieldType(jsonLoginMethodObj, "thirdPartyUserId", + ValueType.STRING, true, String.class, errors, " for a thirdparty recipe."); + + email = validateAndNormaliseEmail(email, errors); + thirdPartyId = validateAndNormaliseThirdPartyId(thirdPartyId, errors); + thirdPartyUserId = validateAndNormaliseThirdPartyUserId(thirdPartyUserId, errors); + + loginMethods.add(new LoginMethod(normalisedTenantIds, recipeId, isVerified, isPrimary, + timeJoinedInMSSinceEpoch, email, null, null, thirdPartyId, thirdPartyUserId, null)); + } else if ("passwordless".equals(recipeId)) { + String email = parseAndValidateFieldType(jsonLoginMethodObj, "email", ValueType.STRING, false, + String.class, errors, " for a passwordless recipe."); + String phoneNumber = parseAndValidateFieldType(jsonLoginMethodObj, "phoneNumber", ValueType.STRING, + false, String.class, errors, " for a passwordless recipe."); + + email = validateAndNormaliseEmail(email, errors); + phoneNumber = validateAndNormalisePhoneNumber(phoneNumber, errors); + + if (email == null && phoneNumber == null) { + errors.add("Either email or phoneNumber is required for a passwordless recipe."); + } + + loginMethods.add(new LoginMethod(normalisedTenantIds, recipeId, isVerified, isPrimary, + timeJoinedInMSSinceEpoch, email, null, null, null, null, phoneNumber)); + } + } + return loginMethods; + } + + private static String validateAndNormaliseExternalUserId(String externalUserId, Set allExternalUserIds, + List errors) { + if (externalUserId == null) { + return null; + } + + if (externalUserId.length() > 255) { + errors.add("externalUserId " + externalUserId + " is too long. Max length is 128."); + } + + if (!allExternalUserIds.add(externalUserId)) { + errors.add("externalUserId " + externalUserId + " is not unique. It is already used by another user."); + } + + // We just trim the externalUserId as per the UpdateExternalUserIdInfoAPI.java + return externalUserId.trim(); + } + + private static String validateAndNormaliseUserRole(String role, String[] allUserRoles, List errors) { + if (role.length() > 255) { + errors.add("role " + role + " is too long. Max length is 255."); + } + + // We just trim the role as per the CreateRoleAPI.java + String normalisedRole = role.trim(); + + if (!Arrays.asList(allUserRoles).contains(normalisedRole)) { + errors.add("Role " + normalisedRole + " does not exist."); + } + + return normalisedRole; + } + + private static String validateAndNormaliseTotpSecretKey(String secretKey, List errors) { + if (secretKey == null) { + return null; + } + + if (secretKey.length() > 256) { + errors.add("TOTP secretKey " + secretKey + " is too long. Max length is 256."); + } + + // We don't perform any normalisation on the secretKey in ImportTotpDeviceAPI.java + return secretKey; + } + + private static Integer validateAndNormaliseTotpPeriod(Integer period, List errors) { + // We default to 30 if period is null + if (period == null) { + return 30; + } + + if (period.intValue() < 1) { + errors.add("period should be > 0 for a totp device."); + return null; + } + return period; + } + + private static Integer validateAndNormaliseTotpSkew(Integer skew, List errors) { + // We default to 1 if skew is null + if (skew == null) { + return 1; + } + + if (skew.intValue() < 0) { + errors.add("skew should be >= 0 for a totp device."); + return null; + } + return skew; + } + + private static String validateAndNormaliseTotpDeviceName(String deviceName, List errors) { + if (deviceName == null) { + return null; + } + + if (deviceName.length() > 256) { + errors.add("TOTP deviceName " + deviceName + " is too long. Max length is 256."); + } + + // We normalise the deviceName as per the ImportTotpDeviceAPI.java + return deviceName.trim(); + } + + private static void validateAndNormaliseIsPrimaryField(JsonArray jsonLoginMethods, List errors) { + // We are validating that only one loginMethod has isPrimary as true + boolean hasPrimaryLoginMethod = false; + for (JsonElement jsonLoginMethod : jsonLoginMethods) { + JsonObject jsonLoginMethodObj = jsonLoginMethod.getAsJsonObject(); + if (validateJsonFieldType(jsonLoginMethodObj, "isPrimary", ValueType.BOOLEAN)) { + if (jsonLoginMethodObj.get("isPrimary").getAsBoolean()) { + if (hasPrimaryLoginMethod) { + errors.add("No two loginMethods can have isPrimary as true."); + } + hasPrimaryLoginMethod = true; + } + } + } + } + + private static String validateAndNormaliseRecipeId(String recipeId, List errors) { + if (recipeId == null) { + return null; + } + + // We don't perform any normalisation on the recipeId after reading it from request header. + // We will validate it as is. + if (!Arrays.asList("emailpassword", "thirdparty", "passwordless").contains(recipeId)) { + errors.add("Invalid recipeId for loginMethod. Pass one of emailpassword, thirdparty or, passwordless!"); + } + return recipeId; + } + + private static List validateAndNormaliseTenantIds(Main main, AppIdentifier appIdentifier, + JsonArray tenantIds, List errors, String errorSuffix) + throws StorageQueryException, TenantOrAppNotFoundException { + if (tenantIds == null) { + return List.of(TenantIdentifier.DEFAULT_TENANT_ID); // Default to DEFAULT_TENANT_ID ("public") + } + + List normalisedTenantIds = new ArrayList<>(); + + for (JsonElement tenantIdEl : tenantIds) { + String tenantId = tenantIdEl.getAsString(); + tenantId = validateAndNormaliseTenantId(main, appIdentifier, tenantId, errors, errorSuffix); + + if (tenantId != null) { + normalisedTenantIds.add(tenantId); + } + } + return normalisedTenantIds; + } + + private static String validateAndNormaliseTenantId(Main main, AppIdentifier appIdentifier, String tenantId, + List errors, String errorSuffix) + throws StorageQueryException, TenantOrAppNotFoundException { + if (tenantId == null || tenantId.equals(TenantIdentifier.DEFAULT_TENANT_ID)) { + return tenantId; + } + + if (Arrays.stream(FeatureFlag.getInstance(main, appIdentifier).getEnabledFeatures()) + .noneMatch(t -> t == EE_FEATURES.MULTI_TENANCY)) { + errors.add("Multitenancy must be enabled before importing users to a different tenant."); + return null; + } + + // We make the tenantId lowercase while parsing from the request in WebserverAPI.java + String normalisedTenantId = tenantId.trim().toLowerCase(); + TenantConfig[] allTenantConfigs = Multitenancy.getAllTenantsForApp(appIdentifier, main); + Set validTenantIds = new HashSet<>(); + Arrays.stream(allTenantConfigs) + .forEach(tenantConfig -> validTenantIds.add(tenantConfig.tenantIdentifier.getTenantId())); + + if (!validTenantIds.contains(normalisedTenantId)) { + errors.add("Invalid tenantId: " + tenantId + errorSuffix); + return null; + } + return normalisedTenantId; + } + + private static Boolean validateAndNormaliseIsPrimary(Boolean isPrimary) { + // We set the default value as false + return isPrimary == null ? false : isPrimary; + } + + private static Boolean validateAndNormaliseIsVerified(Boolean isVerified) { + // We set the default value as false + return isVerified == null ? false : isVerified; + } + + private static long validateAndNormaliseTimeJoined(Long timeJoined, List errors) { + // We default timeJoined to currentTime if it is null + if (timeJoined == null) { + return System.currentTimeMillis(); + } + + if (timeJoined > System.currentTimeMillis()) { + errors.add("timeJoined cannot be in future for a loginMethod."); + } + + if (timeJoined < 0) { + errors.add("timeJoined cannot be < 0 for a loginMethod."); + } + + return timeJoined.longValue(); + } + + private static String validateAndNormaliseEmail(String email, List errors) { + if (email == null) { + return null; + } + + if (email.length() > 255) { + errors.add("email " + email + " is too long. Max length is 256."); + } + + // We normalise the email as per the SignUpAPI.java + return Utils.normaliseEmail(email); + } + + private static CoreConfig.PASSWORD_HASHING_ALG validateAndNormaliseHashingAlgorithm(String hashingAlgorithm, + List errors) { + if (hashingAlgorithm == null) { + return null; + } + + try { + // We trim the hashingAlgorithm and make it uppercase as per the ImportUserWithPasswordHashAPI.java + return CoreConfig.PASSWORD_HASHING_ALG.valueOf(hashingAlgorithm.trim().toUpperCase()); + } catch (IllegalArgumentException e) { + errors.add( + "Invalid hashingAlgorithm for emailpassword recipe. Pass one of bcrypt, argon2 or, firebase_scrypt!"); + return null; + } + } + + private static String validateAndNormalisePasswordHash(Main main, AppIdentifier appIdentifier, + CoreConfig.PASSWORD_HASHING_ALG hashingAlgorithm, String passwordHash, List errors) + throws TenantOrAppNotFoundException { + if (hashingAlgorithm == null || passwordHash == null) { + return passwordHash; + } + + if (passwordHash.length() > 256) { + errors.add("passwordHash is too long. Max length is 256."); + } + + // We trim the passwordHash and validate it as per ImportUserWithPasswordHashAPI.java + passwordHash = passwordHash.trim(); + + try { + PasswordHashingUtils.assertSuperTokensSupportInputPasswordHashFormat(appIdentifier, main, passwordHash, + hashingAlgorithm); + } catch (UnsupportedPasswordHashingFormatException e) { + errors.add(e.getMessage()); + } + + return passwordHash; + } + + private static String validateAndNormaliseThirdPartyId(String thirdPartyId, List errors) { + if (thirdPartyId == null) { + return null; + } + + if (thirdPartyId.length() > 28) { + errors.add("thirdPartyId " + thirdPartyId + " is too long. Max length is 28."); + } + + // We don't perform any normalisation on the thirdPartyId in SignInUpAPI.java + return thirdPartyId; + } + + private static String validateAndNormaliseThirdPartyUserId(String thirdPartyUserId, List errors) { + if (thirdPartyUserId == null) { + return null; + } + + if (thirdPartyUserId.length() > 256) { + errors.add("thirdPartyUserId " + thirdPartyUserId + " is too long. Max length is 256."); + } + + // We don't perform any normalisation on the thirdPartyUserId in SignInUpAPI.java + return thirdPartyUserId; + } + + private static String validateAndNormalisePhoneNumber(String phoneNumber, List errors) { + if (phoneNumber == null) { + return null; + } + + if (phoneNumber.length() > 256) { + errors.add("phoneNumber " + phoneNumber + " is too long. Max length is 256."); + } + + // We normalise the phoneNumber as per the CreateCodeAPI.java + return Utils.normalizeIfPhoneNumber(phoneNumber); + } + + private static void validateTenantIdsForRoleAndLoginMethods(Main main, AppIdentifier appIdentifier, + List userRoles, List loginMethods, List errors) + throws TenantOrAppNotFoundException { + if (loginMethods == null) { + return; + } + + // First validate that tenantIds provided for userRoles also exist in the loginMethods + if (userRoles != null) { + for (UserRole userRole : userRoles) { + for (String tenantId : userRole.tenantIds) { + if (!tenantId.equals(TenantIdentifier.DEFAULT_TENANT_ID) && loginMethods.stream() + .noneMatch(loginMethod -> loginMethod.tenantIds.contains(tenantId))) { + errors.add("TenantId " + tenantId + " for a user role does not exist in loginMethods."); + } + } + } + } + + // Now validate that all the tenants share the same storage + String commonTenantUserPoolId = null; + for (LoginMethod loginMethod : loginMethods) { + for (String tenantId : loginMethod.tenantIds) { + TenantIdentifier tenantIdentifier = new TenantIdentifier(appIdentifier.getConnectionUriDomain(), + appIdentifier.getAppId(), tenantId); + Storage storage = StorageLayer.getStorage(tenantIdentifier, main); + String tenantUserPoolId = storage.getUserPoolId(); + + if (commonTenantUserPoolId == null) { + commonTenantUserPoolId = tenantUserPoolId; + } else if (!commonTenantUserPoolId.equals(tenantUserPoolId)) { + errors.add("All tenants for a user must share the same storage."); + } + } + } + } +} diff --git a/src/main/java/io/supertokens/bulkimport/exceptions/InvalidBulkImportDataException.java b/src/main/java/io/supertokens/bulkimport/exceptions/InvalidBulkImportDataException.java new file mode 100644 index 000000000..3fbcd8fbd --- /dev/null +++ b/src/main/java/io/supertokens/bulkimport/exceptions/InvalidBulkImportDataException.java @@ -0,0 +1,33 @@ +/* + * Copyright (c) 2024, VRAI Labs and/or its affiliates. All rights reserved. + * + * This software is licensed under the Apache License, Version 2.0 (the + * "License") as published by the Apache Software Foundation. + * + * You may not use this file except in compliance with the License. You may + * obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package io.supertokens.bulkimport.exceptions; + +import java.util.List; + +public class InvalidBulkImportDataException extends Exception { + private static final long serialVersionUID = 1L; + public List errors; + + public InvalidBulkImportDataException(List errors) { + super("Data has missing or invalid fields. Please check the errors field for more details."); + this.errors = errors; + } + + public void addError(String error) { + this.errors.add(error); + } +} diff --git a/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkImportUsers.java b/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkImportUsers.java new file mode 100644 index 000000000..730fd43b4 --- /dev/null +++ b/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkImportUsers.java @@ -0,0 +1,504 @@ +/* + * Copyright (c) 2024. VRAI Labs and/or its affiliates. All rights reserved. + * + * This software is licensed under the Apache License, Version 2.0 (the + * "License") as published by the Apache Software Foundation. + * + * You may not use this file except in compliance with the License. You may + * obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package io.supertokens.cronjobs.bulkimport; + +import java.io.IOException; +import java.sql.Connection; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import io.supertokens.Main; +import io.supertokens.ResourceDistributor; +import io.supertokens.authRecipe.AuthRecipe; +import io.supertokens.authRecipe.exception.AccountInfoAlreadyAssociatedWithAnotherPrimaryUserIdException; +import io.supertokens.authRecipe.exception.InputUserIdIsNotAPrimaryUserException; +import io.supertokens.authRecipe.exception.RecipeUserIdAlreadyLinkedWithAnotherPrimaryUserIdException; +import io.supertokens.authRecipe.exception.RecipeUserIdAlreadyLinkedWithPrimaryUserIdException; +import io.supertokens.bulkimport.BulkImport; +import io.supertokens.config.Config; +import io.supertokens.cronjobs.CronTask; +import io.supertokens.cronjobs.CronTaskTest; +import io.supertokens.emailpassword.EmailPassword; +import io.supertokens.emailpassword.EmailPassword.ImportUserResponse; +import io.supertokens.featureflag.exceptions.FeatureNotEnabledException; +import io.supertokens.multitenancy.Multitenancy; +import io.supertokens.multitenancy.exception.AnotherPrimaryUserWithEmailAlreadyExistsException; +import io.supertokens.multitenancy.exception.AnotherPrimaryUserWithPhoneNumberAlreadyExistsException; +import io.supertokens.multitenancy.exception.AnotherPrimaryUserWithThirdPartyInfoAlreadyExistsException; +import io.supertokens.passwordless.Passwordless; +import io.supertokens.passwordless.exceptions.RestartFlowException; +import io.supertokens.pluginInterface.STORAGE_TYPE; +import io.supertokens.pluginInterface.Storage; +import io.supertokens.pluginInterface.StorageUtils; +import io.supertokens.pluginInterface.authRecipe.AuthRecipeUserInfo; +import io.supertokens.pluginInterface.bulkimport.BulkImportUser; +import io.supertokens.pluginInterface.bulkimport.BulkImportStorage.BULK_IMPORT_USER_STATUS; +import io.supertokens.pluginInterface.bulkimport.BulkImportUser.LoginMethod; +import io.supertokens.pluginInterface.bulkimport.BulkImportUser.TotpDevice; +import io.supertokens.pluginInterface.bulkimport.BulkImportUser.UserRole; +import io.supertokens.pluginInterface.bulkimport.sqlStorage.BulkImportSQLStorage; +import io.supertokens.pluginInterface.emailpassword.exceptions.DuplicateEmailException; +import io.supertokens.pluginInterface.emailpassword.exceptions.UnknownUserIdException; +import io.supertokens.pluginInterface.emailverification.sqlStorage.EmailVerificationSQLStorage; +import io.supertokens.pluginInterface.exceptions.DbInitException; +import io.supertokens.pluginInterface.exceptions.InvalidConfigException; +import io.supertokens.pluginInterface.exceptions.StorageQueryException; +import io.supertokens.pluginInterface.exceptions.StorageTransactionLogicException; +import io.supertokens.pluginInterface.multitenancy.AppIdentifier; +import io.supertokens.pluginInterface.multitenancy.TenantIdentifier; +import io.supertokens.pluginInterface.multitenancy.exceptions.TenantOrAppNotFoundException; +import io.supertokens.pluginInterface.passwordless.exception.DuplicatePhoneNumberException; +import io.supertokens.pluginInterface.sqlStorage.SQLStorage; +import io.supertokens.pluginInterface.sqlStorage.TransactionConnection; +import io.supertokens.pluginInterface.thirdparty.exception.DuplicateThirdPartyUserException; +import io.supertokens.pluginInterface.totp.exception.DeviceAlreadyExistsException; +import io.supertokens.pluginInterface.useridmapping.exception.UnknownSuperTokensUserIdException; +import io.supertokens.pluginInterface.useridmapping.exception.UserIdMappingAlreadyExistsException; +import io.supertokens.pluginInterface.userroles.exception.UnknownRoleException; +import io.supertokens.storageLayer.StorageLayer; +import io.supertokens.thirdparty.ThirdParty; +import io.supertokens.thirdparty.ThirdParty.SignInUpResponse; +import io.supertokens.totp.Totp; +import io.supertokens.useridmapping.UserIdMapping; +import io.supertokens.usermetadata.UserMetadata; +import io.supertokens.userroles.UserRoles; +import jakarta.servlet.ServletException; + +public class ProcessBulkImportUsers extends CronTask { + + public static final String RESOURCE_KEY = "io.supertokens.ee.cronjobs.ProcessBulkImportUsers"; + private Map userPoolToStorageMap = new HashMap<>(); + + private ProcessBulkImportUsers(Main main, List> tenantsInfo) { + super("ProcessBulkImportUsers", main, tenantsInfo, true); + } + + public static ProcessBulkImportUsers init(Main main, List> tenantsInfo) { + return (ProcessBulkImportUsers) main.getResourceDistributor() + .setResource(new TenantIdentifier(null, null, null), RESOURCE_KEY, + new ProcessBulkImportUsers(main, tenantsInfo)); + } + + @Override + protected void doTaskPerApp(AppIdentifier app) + throws TenantOrAppNotFoundException, StorageQueryException, InvalidConfigException, IOException, + DbInitException { + + if (StorageLayer.getBaseStorage(main).getType() != STORAGE_TYPE.SQL) { + return; + } + + BulkImportSQLStorage bulkImportSQLStorage = (BulkImportSQLStorage) StorageLayer + .getStorage(app.getAsPublicTenantIdentifier(), main); + + AppIdentifier appIdentifier = new AppIdentifier(app.getConnectionUriDomain(), app.getAppId()); + + List users = bulkImportSQLStorage.getBulkImportUsersForProcessing(appIdentifier, + BulkImport.PROCESS_USERS_BATCH_SIZE); + + for (BulkImportUser user : users) { + processUser(appIdentifier, user); + } + + closeAllProxyStorages(); + } + + @Override + public int getIntervalTimeSeconds() { + if (Main.isTesting) { + Integer interval = CronTaskTest.getInstance(main).getIntervalInSeconds(RESOURCE_KEY); + if (interval != null) { + return interval; + } + } + return BulkImport.PROCESS_USERS_INTERVAL; + } + + @Override + public int getInitialWaitTimeSeconds() { + // We are setting a non-zero initial wait for tests to avoid race condition with the beforeTest process that deletes data in the storage layer + if (Main.isTesting) { + return 5; + } + return 0; + } + + private Storage getProxyStorage(TenantIdentifier tenantIdentifier) + throws InvalidConfigException, IOException, TenantOrAppNotFoundException, DbInitException { + String userPoolId = StorageLayer.getStorage(tenantIdentifier, main).getUserPoolId(); + if (userPoolToStorageMap.containsKey(userPoolId)) { + return userPoolToStorageMap.get(userPoolId); + } + + SQLStorage bulkImportProxyStorage = (SQLStorage) StorageLayer.getNewBulkImportProxyStorageInstance(main, + Config.getBaseConfigAsJsonObject(main), tenantIdentifier, true); + + userPoolToStorageMap.put(userPoolId, bulkImportProxyStorage); + bulkImportProxyStorage.initStorage(true); + return bulkImportProxyStorage; + } + + public Storage[] getAllProxyStoragesForApp(Main main, AppIdentifier appIdentifier) + throws TenantOrAppNotFoundException, InvalidConfigException, IOException, DbInitException { + List allProxyStorages = new ArrayList<>(); + + Map resources = main + .getResourceDistributor() + .getAllResourcesWithResourceKey(RESOURCE_KEY); + for (ResourceDistributor.KeyClass key : resources.keySet()) { + if (key.getTenantIdentifier().toAppIdentifier().equals(appIdentifier)) { + allProxyStorages.add(getProxyStorage(key.getTenantIdentifier())); + } + } + return allProxyStorages.toArray(new Storage[0]); + } + + private void closeAllProxyStorages() { + for (Storage storage : userPoolToStorageMap.values()) { + storage.close(); + } + } + + private void processUser(AppIdentifier appIdentifier, BulkImportUser user) + throws TenantOrAppNotFoundException, StorageQueryException, InvalidConfigException, IOException, + DbInitException { + // Since all the tenants of a user must share the storage, we will just use the + // storage of the first tenantId of the first loginMethod + + TenantIdentifier firstTenantIdentifier = new TenantIdentifier(appIdentifier.getConnectionUriDomain(), + appIdentifier.getAppId(), user.loginMethods.get(0).tenantIds.get(0)); + + SQLStorage bulkImportProxyStorage = (SQLStorage) getProxyStorage(firstTenantIdentifier); + + LoginMethod primaryLM = getPrimaryLoginMethod(user); + + try { + bulkImportProxyStorage.startTransaction(con -> { + for (LoginMethod lm : user.loginMethods) { + processUserLoginMethod(appIdentifier, bulkImportProxyStorage, lm); + } + + createPrimaryUserAndLinkAccounts(main, appIdentifier, bulkImportProxyStorage, user, primaryLM); + createUserIdMapping(main, appIdentifier, user, primaryLM); + verifyEmailForAllLoginMethods(appIdentifier, con, bulkImportProxyStorage, user.loginMethods); + createTotpDevices(main, appIdentifier, bulkImportProxyStorage, user.totpDevices, primaryLM); + createUserMetadata(appIdentifier, bulkImportProxyStorage, user, primaryLM); + createUserRoles(main, appIdentifier, bulkImportProxyStorage, user); + + ((BulkImportSQLStorage) bulkImportProxyStorage).deleteBulkImportUser_Transaction(appIdentifier, con, + user.id); + + // We need to commit the transaction manually because we have overridden that in the proxy storage + try { + Connection connection = (Connection) con.getConnection(); + connection.commit(); + connection.setAutoCommit(true); + } catch (SQLException e) { + throw new StorageTransactionLogicException(e); + } + + return null; + }); + } catch (StorageTransactionLogicException e) { + handleProcessUserExceptions(appIdentifier, user, (BulkImportSQLStorage) bulkImportProxyStorage, e); + } + } + + private void handleProcessUserExceptions(AppIdentifier appIdentifier, BulkImportUser user, + BulkImportSQLStorage bulkImportSQLStorage, Exception e) + throws StorageQueryException { + + // Java doesn't allow us to reassign local variables inside a lambda expression + // so we have to use an array. + String[] errorMessage = { e.getMessage() }; + + if (e instanceof StorageTransactionLogicException) { + StorageTransactionLogicException exception = (StorageTransactionLogicException) e; + errorMessage[0] = exception.actualException.getMessage(); + } + + String[] userId = { user.id }; + + try { + bulkImportSQLStorage.startTransaction(con -> { + bulkImportSQLStorage.updateBulkImportUserStatus_Transaction(appIdentifier, con, userId, + BULK_IMPORT_USER_STATUS.FAILED, errorMessage[0]); + + // We need to commit the transaction manually because we have overridden that in the proxy storage + try { + Connection connection = (Connection) con.getConnection(); + connection.commit(); + connection.setAutoCommit(true); + } catch (SQLException ex) { + throw new StorageTransactionLogicException(ex); + } + return null; + }); + } catch (StorageTransactionLogicException e1) { + throw new StorageQueryException(e1.actualException); + } + } + + private void processUserLoginMethod(AppIdentifier appIdentifier, Storage storage, + LoginMethod lm) throws StorageTransactionLogicException { + String firstTenant = lm.tenantIds.get(0); + + TenantIdentifier tenantIdentifier = new TenantIdentifier(appIdentifier.getConnectionUriDomain(), + appIdentifier.getAppId(), firstTenant); + + if (lm.recipeId.equals("emailpassword")) { + processEmailPasswordLoginMethod(tenantIdentifier, storage, lm); + } else if (lm.recipeId.equals("thirdparty")) { + processThirdPartyLoginMethod(tenantIdentifier, storage, lm); + } else if (lm.recipeId.equals("passwordless")) { + processPasswordlessLoginMethod(tenantIdentifier, storage, lm); + } else { + throw new StorageTransactionLogicException( + new IllegalArgumentException("Unknown recipeId " + lm.recipeId + " for loginMethod ")); + } + + associateUserToTenants(main, appIdentifier, storage, lm, firstTenant); + } + + private void processEmailPasswordLoginMethod(TenantIdentifier tenantIdentifier, Storage storage, + LoginMethod lm) throws StorageTransactionLogicException { + try { + ImportUserResponse userInfo = EmailPassword.createUserWithPasswordHash(tenantIdentifier, storage, lm.email, + lm.passwordHash, lm.timeJoinedInMSSinceEpoch); + + lm.superTokensOrExternalUserId = userInfo.user.getSupertokensUserId(); + } catch (StorageQueryException | TenantOrAppNotFoundException e) { + throw new StorageTransactionLogicException(e); + } catch (DuplicateEmailException e) { + throw new StorageTransactionLogicException( + new Exception("A user with email " + lm.email + " already exists")); + } + } + + private void processThirdPartyLoginMethod(TenantIdentifier tenantIdentifier, Storage storage, LoginMethod lm) + throws StorageTransactionLogicException { + try { + SignInUpResponse userInfo = ThirdParty.createThirdPartyUser( + tenantIdentifier, storage, lm.thirdPartyId, lm.thirdPartyUserId, lm.email, + lm.timeJoinedInMSSinceEpoch); + + lm.superTokensOrExternalUserId = userInfo.user.getSupertokensUserId(); + } catch (StorageQueryException | TenantOrAppNotFoundException e) { + throw new StorageTransactionLogicException(e); + } catch (DuplicateThirdPartyUserException e) { + throw new StorageTransactionLogicException(new Exception("A user with thirdPartyId " + lm.thirdPartyId + + " and thirdPartyUserId " + lm.thirdPartyUserId + " already exists")); + } + } + + private void processPasswordlessLoginMethod(TenantIdentifier tenantIdentifier, Storage storage, LoginMethod lm) + throws StorageTransactionLogicException { + try { + AuthRecipeUserInfo userInfo = Passwordless.createPasswordlessUser(tenantIdentifier, storage, lm.email, + lm.phoneNumber, lm.timeJoinedInMSSinceEpoch); + + lm.superTokensOrExternalUserId = userInfo.getSupertokensUserId(); + } catch (StorageQueryException | TenantOrAppNotFoundException | RestartFlowException e) { + throw new StorageTransactionLogicException(e); + } + } + + private void associateUserToTenants(Main main, AppIdentifier appIdentifier, Storage storage, LoginMethod lm, + String firstTenant) throws StorageTransactionLogicException { + for (String tenantId : lm.tenantIds) { + try { + if (tenantId.equals(firstTenant)) { + continue; + } + + TenantIdentifier tenantIdentifier = new TenantIdentifier(appIdentifier.getConnectionUriDomain(), + appIdentifier.getAppId(), tenantId); + Multitenancy.addUserIdToTenant(main, tenantIdentifier, storage, lm.superTokensOrExternalUserId); + } catch (TenantOrAppNotFoundException | UnknownUserIdException | StorageQueryException + | FeatureNotEnabledException | DuplicateEmailException | DuplicatePhoneNumberException + | DuplicateThirdPartyUserException | AnotherPrimaryUserWithPhoneNumberAlreadyExistsException + | AnotherPrimaryUserWithEmailAlreadyExistsException + | AnotherPrimaryUserWithThirdPartyInfoAlreadyExistsException e) { + throw new StorageTransactionLogicException(e); + } + } + } + + private void createPrimaryUserAndLinkAccounts(Main main, + AppIdentifier appIdentifier, Storage storage, BulkImportUser user, LoginMethod primaryLM) + throws StorageTransactionLogicException { + if (user.loginMethods.size() == 1) { + return; + } + + try { + AuthRecipe.createPrimaryUser(main, appIdentifier, storage, primaryLM.superTokensOrExternalUserId); + } catch (TenantOrAppNotFoundException | FeatureNotEnabledException | StorageQueryException e) { + throw new StorageTransactionLogicException(e); + } catch (UnknownUserIdException e) { + throw new StorageTransactionLogicException(new Exception( + "We tried to create the primary user for the userId " + primaryLM.superTokensOrExternalUserId + + " but it doesn't exist. This should not happen. Please contact support.")); + } catch (RecipeUserIdAlreadyLinkedWithPrimaryUserIdException + | AccountInfoAlreadyAssociatedWithAnotherPrimaryUserIdException e) { + throw new StorageTransactionLogicException( + new Exception(e.getMessage() + " This should not happen. Please contact support.")); + } + + for (LoginMethod lm : user.loginMethods) { + try { + if (lm.superTokensOrExternalUserId.equals(primaryLM.superTokensOrExternalUserId)) { + continue; + } + + AuthRecipe.linkAccounts(main, appIdentifier, storage, lm.superTokensOrExternalUserId, + primaryLM.superTokensOrExternalUserId); + + } catch (TenantOrAppNotFoundException | FeatureNotEnabledException | StorageQueryException e) { + throw new StorageTransactionLogicException(e); + } catch (UnknownUserIdException e) { + throw new StorageTransactionLogicException( + new Exception("We tried to link the userId " + lm.superTokensOrExternalUserId + + " to the primary userId " + primaryLM.superTokensOrExternalUserId + + " but it doesn't exist. This should not happen. Please contact support.")); + } catch (InputUserIdIsNotAPrimaryUserException e) { + throw new StorageTransactionLogicException( + new Exception("We tried to link the userId " + lm.superTokensOrExternalUserId + + " to the primary userId " + primaryLM.superTokensOrExternalUserId + + " but it is not a primary user. This should not happen. Please contact support.")); + } catch (AccountInfoAlreadyAssociatedWithAnotherPrimaryUserIdException + | RecipeUserIdAlreadyLinkedWithAnotherPrimaryUserIdException e) { + throw new StorageTransactionLogicException( + new Exception(e.getMessage() + " This should not happen. Please contact support.")); + } + } + } + + private void createUserIdMapping(Main main, AppIdentifier appIdentifier, + BulkImportUser user, LoginMethod primaryLM) throws StorageTransactionLogicException { + if (user.externalUserId != null) { + try { + UserIdMapping.createUserIdMapping( + appIdentifier, getAllProxyStoragesForApp(main, appIdentifier), + primaryLM.superTokensOrExternalUserId, user.externalUserId, + null, false, true); + + primaryLM.superTokensOrExternalUserId = user.externalUserId; + } catch (StorageQueryException | ServletException | TenantOrAppNotFoundException | InvalidConfigException + | IOException | DbInitException e) { + throw new StorageTransactionLogicException(e); + } catch (UserIdMappingAlreadyExistsException e) { + throw new StorageTransactionLogicException( + new Exception("A user with externalId " + user.externalUserId + " already exists")); + } catch (UnknownSuperTokensUserIdException e) { + throw new StorageTransactionLogicException( + new Exception("We tried to create the externalUserId mapping for the superTokenUserId " + + primaryLM.superTokensOrExternalUserId + + " but it doesn't exist. This should not happen. Please contact support.")); + } + } + } + + private void createUserMetadata(AppIdentifier appIdentifier, Storage storage, BulkImportUser user, + LoginMethod primaryLM) throws StorageTransactionLogicException { + if (user.userMetadata != null) { + try { + UserMetadata.updateUserMetadata(appIdentifier, storage, primaryLM.superTokensOrExternalUserId, + user.userMetadata); + } catch (StorageQueryException | TenantOrAppNotFoundException e) { + throw new StorageTransactionLogicException(e); + } + } + } + + private void createUserRoles(Main main, AppIdentifier appIdentifier, Storage storage, + BulkImportUser user) throws StorageTransactionLogicException { + if (user.userRoles != null) { + for (UserRole userRole : user.userRoles) { + try { + for (String tenantId : userRole.tenantIds) { + TenantIdentifier tenantIdentifier = new TenantIdentifier( + appIdentifier.getConnectionUriDomain(), appIdentifier.getAppId(), + tenantId); + + UserRoles.addRoleToUser(main, tenantIdentifier, storage, user.externalUserId, userRole.role); + } + } catch (TenantOrAppNotFoundException | StorageQueryException e) { + throw new StorageTransactionLogicException(e); + } catch (UnknownRoleException e) { + throw new StorageTransactionLogicException(new Exception("Role " + userRole.role + + " does not exist! You need pre-create the role before assigning it to the user.")); + } + } + } + } + + private void verifyEmailForAllLoginMethods(AppIdentifier appIdentifier, TransactionConnection con, Storage storage, + List loginMethods) throws StorageTransactionLogicException { + + for (LoginMethod lm : loginMethods) { + try { + + TenantIdentifier tenantIdentifier = new TenantIdentifier(appIdentifier.getConnectionUriDomain(), + appIdentifier.getAppId(), lm.tenantIds.get(0)); + + EmailVerificationSQLStorage emailVerificationSQLStorage = StorageUtils + .getEmailVerificationStorage(storage); + emailVerificationSQLStorage + .updateIsEmailVerified_Transaction(tenantIdentifier.toAppIdentifier(), con, + lm.superTokensOrExternalUserId, lm.email, true); + } catch (TenantOrAppNotFoundException | StorageQueryException e) { + throw new StorageTransactionLogicException(e); + } + } + } + + private void createTotpDevices(Main main, AppIdentifier appIdentifier, Storage storage, + List totpDevices, LoginMethod primaryLM) throws StorageTransactionLogicException { + for (TotpDevice totpDevice : totpDevices) { + try { + Totp.createDevice(main, appIdentifier, storage, primaryLM.superTokensOrExternalUserId, + totpDevice.deviceName, totpDevice.skew, totpDevice.period, totpDevice.secretKey, + true, System.currentTimeMillis()); + } catch (TenantOrAppNotFoundException | StorageQueryException | FeatureNotEnabledException e) { + throw new StorageTransactionLogicException(e); + } catch (DeviceAlreadyExistsException e) { + throw new StorageTransactionLogicException( + new Exception("A totp device with name " + totpDevice.deviceName + " already exists")); + } + } + } + + // Returns the primary loginMethod of the user. If no loginMethod is marked as + // primary, then the oldest loginMethod is returned. + private BulkImportUser.LoginMethod getPrimaryLoginMethod(BulkImportUser user) { + BulkImportUser.LoginMethod oldestLM = user.loginMethods.get(0); + for (BulkImportUser.LoginMethod lm : user.loginMethods) { + if (lm.isPrimary) { + return lm; + } + + if (lm.timeJoinedInMSSinceEpoch < oldestLM.timeJoinedInMSSinceEpoch) { + oldestLM = lm; + } + } + return oldestLM; + } +} diff --git a/src/main/java/io/supertokens/emailpassword/EmailPassword.java b/src/main/java/io/supertokens/emailpassword/EmailPassword.java index 9a92592f3..d35baac2b 100644 --- a/src/main/java/io/supertokens/emailpassword/EmailPassword.java +++ b/src/main/java/io/supertokens/emailpassword/EmailPassword.java @@ -183,41 +183,51 @@ public static ImportUserResponse importUserWithPasswordHash(TenantIdentifier ten tenantIdentifier.toAppIdentifier(), main, passwordHash, hashingAlgorithm); - while (true) { - String userId = Utils.getUUID(); - long timeJoined = System.currentTimeMillis(); + EmailPasswordSQLStorage epStorage = StorageUtils.getEmailPasswordStorage(storage); + ImportUserResponse response = null; - EmailPasswordSQLStorage epStorage = StorageUtils.getEmailPasswordStorage(storage); + try { + long timeJoined = System.currentTimeMillis(); + response = createUserWithPasswordHash(tenantIdentifier, storage, email, passwordHash, timeJoined); + } catch (DuplicateEmailException e) { + AuthRecipeUserInfo[] allUsers = epStorage.listPrimaryUsersByEmail(tenantIdentifier, email); + AuthRecipeUserInfo userInfoToBeUpdated = null; + LoginMethod loginMethod = null; + for (AuthRecipeUserInfo currUser : allUsers) { + for (LoginMethod currLM : currUser.loginMethods) { + if (currLM.email.equals(email) && currLM.recipeId == RECIPE_ID.EMAIL_PASSWORD && currLM.tenantIds.contains(tenantIdentifier.getTenantId())) { + userInfoToBeUpdated = currUser; + loginMethod = currLM; + break; + } + } + } + if (userInfoToBeUpdated != null) { + LoginMethod finalLoginMethod = loginMethod; + epStorage.startTransaction(con -> { + epStorage.updateUsersPassword_Transaction(tenantIdentifier.toAppIdentifier(), con, + finalLoginMethod.getSupertokensUserId(), passwordHash); + return null; + }); + response = new ImportUserResponse(true, userInfoToBeUpdated); + } + } + return response; + } + public static ImportUserResponse createUserWithPasswordHash(TenantIdentifier tenantIdentifier, Storage storage, + @Nonnull String email, + @Nonnull String passwordHash, @Nullable long timeJoined) + throws StorageQueryException, DuplicateEmailException, TenantOrAppNotFoundException { + EmailPasswordSQLStorage epStorage = StorageUtils.getEmailPasswordStorage(storage); + while (true) { + String userId = Utils.getUUID(); try { - AuthRecipeUserInfo userInfo = epStorage.signUp(tenantIdentifier, userId, email, passwordHash, - timeJoined); + AuthRecipeUserInfo userInfo = null; + userInfo = epStorage.signUp(tenantIdentifier, userId, email, passwordHash, timeJoined); return new ImportUserResponse(false, userInfo); } catch (DuplicateUserIdException e) { // we retry with a new userId - } catch (DuplicateEmailException e) { - AuthRecipeUserInfo[] allUsers = epStorage.listPrimaryUsersByEmail(tenantIdentifier, email); - AuthRecipeUserInfo userInfoToBeUpdated = null; - LoginMethod loginMethod = null; - for (AuthRecipeUserInfo currUser : allUsers) { - for (LoginMethod currLM : currUser.loginMethods) { - if (currLM.email.equals(email) && currLM.recipeId == RECIPE_ID.EMAIL_PASSWORD && currLM.tenantIds.contains(tenantIdentifier.getTenantId())) { - userInfoToBeUpdated = currUser; - loginMethod = currLM; - break; - } - } - } - - if (userInfoToBeUpdated != null) { - LoginMethod finalLoginMethod = loginMethod; - epStorage.startTransaction(con -> { - epStorage.updateUsersPassword_Transaction(tenantIdentifier.toAppIdentifier(), con, - finalLoginMethod.getSupertokensUserId(), passwordHash); - return null; - }); - return new ImportUserResponse(true, userInfoToBeUpdated); - } } } } diff --git a/src/main/java/io/supertokens/inmemorydb/Start.java b/src/main/java/io/supertokens/inmemorydb/Start.java index b23434718..d4edd411b 100644 --- a/src/main/java/io/supertokens/inmemorydb/Start.java +++ b/src/main/java/io/supertokens/inmemorydb/Start.java @@ -135,6 +135,11 @@ public void constructor(String processId, boolean silent, boolean isTesting) { Start.isTesting = isTesting; } + @Override + public Storage createBulkImportProxyStorageInstance() { + return this; + } + @Override public STORAGE_TYPE getType() { return STORAGE_TYPE.SQL; diff --git a/src/main/java/io/supertokens/passwordless/Passwordless.java b/src/main/java/io/supertokens/passwordless/Passwordless.java index ba76d7f45..c0ad15f1e 100644 --- a/src/main/java/io/supertokens/passwordless/Passwordless.java +++ b/src/main/java/io/supertokens/passwordless/Passwordless.java @@ -456,52 +456,37 @@ public static ConsumeCodeResponse consumeCode(TenantIdentifier tenantIdentifier, } if (user == null) { - while (true) { - try { - String userId = Utils.getUUID(); - long timeJoined = System.currentTimeMillis(); - user = passwordlessStorage.createUser(tenantIdentifier, userId, consumedDevice.email, - consumedDevice.phoneNumber, timeJoined); + long timeJoined = System.currentTimeMillis(); + user = createPasswordlessUser(tenantIdentifier, storage, consumedDevice.email, + consumedDevice.phoneNumber, timeJoined); - // Set email as verified, if using email - if (setEmailVerified && consumedDevice.email != null) { + // Set email as verified, if using email + if (setEmailVerified && consumedDevice.email != null) { + try { + AuthRecipeUserInfo finalUser = user; + EmailVerificationSQLStorage evStorage = + StorageUtils.getEmailVerificationStorage(storage); + evStorage.startTransaction(con -> { try { - AuthRecipeUserInfo finalUser = user; - EmailVerificationSQLStorage evStorage = - StorageUtils.getEmailVerificationStorage(storage); - evStorage.startTransaction(con -> { - try { - evStorage.updateIsEmailVerified_Transaction(tenantIdentifier.toAppIdentifier(), con, - finalUser.getSupertokensUserId(), consumedDevice.email, true); - evStorage.commitTransaction(con); - - return null; - } catch (TenantOrAppNotFoundException e) { - throw new StorageTransactionLogicException(e); - } - }); - user.loginMethods[0].setVerified(); // newly created user has only one loginMethod - } catch (StorageTransactionLogicException e) { - if (e.actualException instanceof TenantOrAppNotFoundException) { - throw (TenantOrAppNotFoundException) e.actualException; - } - throw new StorageQueryException(e); + evStorage.updateIsEmailVerified_Transaction(tenantIdentifier.toAppIdentifier(), con, + finalUser.getSupertokensUserId(), consumedDevice.email, true); + evStorage.commitTransaction(con); + + return null; + } catch (TenantOrAppNotFoundException e) { + throw new StorageTransactionLogicException(e); } + }); + user.loginMethods[0].setVerified(); // newly created user has only one loginMethod + } catch (StorageTransactionLogicException e) { + if (e.actualException instanceof TenantOrAppNotFoundException) { + throw (TenantOrAppNotFoundException) e.actualException; } - - return new ConsumeCodeResponse(true, user, consumedDevice.email, consumedDevice.phoneNumber, consumedDevice); - } catch (DuplicateEmailException | DuplicatePhoneNumberException e) { - // Getting these would mean that between getting the user and trying creating it: - // 1. the user managed to do a full create+consume flow - // 2. the users email or phoneNumber was updated to the new one (including device cleanup) - // These should be almost impossibly rare, so it's safe to just ask the user to restart. - // Also, both would make the current login fail if done before the transaction - // by cleaning up the device/code this consume would've used. - throw new RestartFlowException(); - } catch (DuplicateUserIdException e) { - // We can retry.. + throw new StorageQueryException(e); } } + + return new ConsumeCodeResponse(true, user, consumedDevice.email, consumedDevice.phoneNumber, consumedDevice); } else { if (setEmailVerified && consumedDevice.email != null) { // Set email verification @@ -541,6 +526,29 @@ public static ConsumeCodeResponse consumeCode(TenantIdentifier tenantIdentifier, return new ConsumeCodeResponse(false, user, consumedDevice.email, consumedDevice.phoneNumber, consumedDevice); } + public static AuthRecipeUserInfo createPasswordlessUser(TenantIdentifier tenantIdentifier, Storage storage, + String email, String phoneNumber, long timeJoined) + throws TenantOrAppNotFoundException, StorageQueryException, RestartFlowException { + PasswordlessSQLStorage passwordlessStorage = StorageUtils.getPasswordlessStorage(storage); + + while (true) { + try { + String userId = Utils.getUUID(); + return passwordlessStorage.createUser(tenantIdentifier, userId, email, phoneNumber, timeJoined); + } catch (DuplicateEmailException | DuplicatePhoneNumberException e) { + // Getting these would mean that between getting the user and trying creating it: + // 1. the user managed to do a full create+consume flow + // 2. the users email or phoneNumber was updated to the new one (including device cleanup) + // These should be almost impossibly rare, so it's safe to just ask the user to restart. + // Also, both would make the current login fail if done before the transaction + // by cleaning up the device/code this consume would've used. + throw new RestartFlowException(); + } catch (DuplicateUserIdException e) { + // We can retry.. + } + } + } + @TestOnly public static void removeCode(Main main, String codeId) throws StorageQueryException, StorageTransactionLogicException { diff --git a/src/main/java/io/supertokens/storageLayer/StorageLayer.java b/src/main/java/io/supertokens/storageLayer/StorageLayer.java index 5145e7c1c..711767703 100644 --- a/src/main/java/io/supertokens/storageLayer/StorageLayer.java +++ b/src/main/java/io/supertokens/storageLayer/StorageLayer.java @@ -56,6 +56,14 @@ public Storage getUnderlyingStorage() { } public static Storage getNewStorageInstance(Main main, JsonObject config, TenantIdentifier tenantIdentifier, boolean doNotLog) throws InvalidConfigException { + return getNewInstance(main, config, tenantIdentifier, doNotLog, false); + } + + public static Storage getNewBulkImportProxyStorageInstance(Main main, JsonObject config, TenantIdentifier tenantIdentifier, boolean doNotLog) throws InvalidConfigException { + return getNewInstance(main, config, tenantIdentifier, doNotLog, true); + } + + private static Storage getNewInstance(Main main, JsonObject config, TenantIdentifier tenantIdentifier, boolean doNotLog, boolean isBulkImportProxy) throws InvalidConfigException { Storage result; if (StorageLayer.ucl == null) { result = new Start(main); @@ -75,7 +83,11 @@ public static Storage getNewStorageInstance(Main main, JsonObject config, Tenant } if (storageLayer != null && !main.isForceInMemoryDB() && (storageLayer.canBeUsed(config) || CLIOptions.get(main).isForceNoInMemoryDB())) { - result = storageLayer; + if (isBulkImportProxy) { + result = storageLayer.createBulkImportProxyStorageInstance(); + } else { + result = storageLayer; + } } else { result = new Start(main); } diff --git a/src/main/java/io/supertokens/thirdparty/ThirdParty.java b/src/main/java/io/supertokens/thirdparty/ThirdParty.java index d49f0a93c..3628aab9d 100644 --- a/src/main/java/io/supertokens/thirdparty/ThirdParty.java +++ b/src/main/java/io/supertokens/thirdparty/ThirdParty.java @@ -206,22 +206,12 @@ private static SignInUpResponse signInUpHelper(TenantIdentifier tenantIdentifier while (true) { // loop for sign in + sign up - while (true) { - // loop for sign up - String userId = Utils.getUUID(); - long timeJoined = System.currentTimeMillis(); + long timeJoined = System.currentTimeMillis(); - try { - AuthRecipeUserInfo createdUser = tpStorage.signUp(tenantIdentifier, userId, email, - new LoginMethod.ThirdParty(thirdPartyId, thirdPartyUserId), timeJoined); - - return new SignInUpResponse(true, createdUser); - } catch (DuplicateUserIdException e) { - // we try again.. - } catch (DuplicateThirdPartyUserException e) { - // we try to sign in - break; - } + try { + return createThirdPartyUser( tenantIdentifier, storage, thirdPartyId, thirdPartyUserId, email, timeJoined); + } catch (DuplicateThirdPartyUserException e) { + // The user already exists, we will try to update the email if needed below } // we try to get user and update their email @@ -341,6 +331,25 @@ private static SignInUpResponse signInUpHelper(TenantIdentifier tenantIdentifier } } + public static SignInUpResponse createThirdPartyUser(TenantIdentifier tenantIdentifier, Storage storage, + String thirdPartyId, String thirdPartyUserId, String email, long timeJoined) + throws StorageQueryException, TenantOrAppNotFoundException, DuplicateThirdPartyUserException { + ThirdPartySQLStorage tpStorage = StorageUtils.getThirdPartyStorage(storage); + + while (true) { + // loop for sign up + String userId = Utils.getUUID(); + + try { + AuthRecipeUserInfo createdUser = tpStorage.signUp(tenantIdentifier, userId, email, + new LoginMethod.ThirdParty(thirdPartyId, thirdPartyUserId), timeJoined); + return new SignInUpResponse(true, createdUser); + } catch (DuplicateUserIdException e) { + // we try again.. + } + } + } + @Deprecated public static AuthRecipeUserInfo getUser(AppIdentifier appIdentifier, Storage storage, String userId) throws StorageQueryException { diff --git a/src/main/java/io/supertokens/utils/JsonValidatorUtils.java b/src/main/java/io/supertokens/utils/JsonValidatorUtils.java new file mode 100644 index 000000000..89a8ea932 --- /dev/null +++ b/src/main/java/io/supertokens/utils/JsonValidatorUtils.java @@ -0,0 +1,123 @@ +/* + * Copyright (c) 2024, VRAI Labs and/or its affiliates. All rights reserved. + * + * This software is licensed under the Apache License, Version 2.0 (the + * "License") as published by the Apache Software Foundation. + * + * You may not use this file except in compliance with the License. You may + * obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package io.supertokens.utils; + +import java.util.ArrayList; +import java.util.List; + +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; + +public class JsonValidatorUtils { + @SuppressWarnings("unchecked") + public static T parseAndValidateFieldType(JsonObject jsonObject, String key, ValueType expectedType, + boolean isRequired, Class targetType, List errors, String errorSuffix) { + if (jsonObject.has(key)) { + if (validateJsonFieldType(jsonObject, key, expectedType)) { + T value; + switch (expectedType) { + case STRING: + value = (T) jsonObject.get(key).getAsString(); + break; + case INTEGER: + Integer intValue = jsonObject.get(key).getAsNumber().intValue(); + value = (T) intValue; + break; + case LONG: + Long longValue = jsonObject.get(key).getAsNumber().longValue(); + value = (T) longValue; + break; + case BOOLEAN: + Boolean boolValue = jsonObject.get(key).getAsBoolean(); + value = (T) boolValue; + break; + case OBJECT: + value = (T) jsonObject.get(key).getAsJsonObject(); + break; + case ARRAY_OF_OBJECT, ARRAY_OF_STRING: + value = (T) jsonObject.get(key).getAsJsonArray(); + break; + default: + value = null; + break; + } + if (value != null) { + return targetType.cast(value); + } else { + errors.add(key + " should be of type " + getTypeForErrorMessage(expectedType) + errorSuffix); + } + } else { + errors.add(key + " should be of type " + getTypeForErrorMessage(expectedType) + errorSuffix); + } + } else if (isRequired) { + errors.add(key + " is required" + errorSuffix); + } + return null; + } + + public enum ValueType { + STRING, + INTEGER, + LONG, + BOOLEAN, + OBJECT, + ARRAY_OF_STRING, + ARRAY_OF_OBJECT + } + + private static String getTypeForErrorMessage(ValueType type) { + return switch (type) { + case STRING -> "string"; + case INTEGER -> "integer"; + case LONG -> "integer"; // choosing integer over long because it is user facing + case BOOLEAN -> "boolean"; + case OBJECT -> "object"; + case ARRAY_OF_STRING -> "array of string"; + case ARRAY_OF_OBJECT -> "array of object"; + }; + } + + public static boolean validateJsonFieldType(JsonObject jsonObject, String key, ValueType expectedType) { + if (jsonObject.has(key)) { + return switch (expectedType) { + case STRING -> jsonObject.get(key).isJsonPrimitive() && jsonObject.getAsJsonPrimitive(key).isString() + && !jsonObject.get(key).getAsString().isBlank(); + case INTEGER, LONG -> jsonObject.get(key).isJsonPrimitive() && jsonObject.getAsJsonPrimitive(key).isNumber(); + case BOOLEAN -> jsonObject.get(key).isJsonPrimitive() && jsonObject.getAsJsonPrimitive(key).isBoolean(); + case OBJECT -> jsonObject.get(key).isJsonObject(); + case ARRAY_OF_OBJECT, ARRAY_OF_STRING -> jsonObject.get(key).isJsonArray() + && validateArrayElements(jsonObject.getAsJsonArray(key), expectedType); + default -> false; + }; + } + return false; + } + + public static boolean validateArrayElements(JsonArray array, ValueType expectedType) { + List elements = new ArrayList<>(); + array.forEach(elements::add); + + return switch (expectedType) { + case ARRAY_OF_OBJECT -> elements.stream().allMatch(JsonElement::isJsonObject); + case ARRAY_OF_STRING -> + elements.stream().allMatch(el -> el.isJsonPrimitive() && el.getAsJsonPrimitive().isString() + && !el.getAsString().isBlank()); + default -> false; + }; + } +} diff --git a/src/main/java/io/supertokens/webserver/Webserver.java b/src/main/java/io/supertokens/webserver/Webserver.java index 700fb4ba1..96716c746 100644 --- a/src/main/java/io/supertokens/webserver/Webserver.java +++ b/src/main/java/io/supertokens/webserver/Webserver.java @@ -26,6 +26,7 @@ import io.supertokens.pluginInterface.multitenancy.TenantIdentifier; import io.supertokens.pluginInterface.multitenancy.exceptions.TenantOrAppNotFoundException; import io.supertokens.webserver.api.accountlinking.*; +import io.supertokens.webserver.api.bulkimport.BulkImportAPI; import io.supertokens.webserver.api.core.*; import io.supertokens.webserver.api.dashboard.*; import io.supertokens.webserver.api.emailpassword.UserAPI; @@ -260,6 +261,8 @@ private void setupRoutes() { addAPI(new RequestStatsAPI(main)); + addAPI(new BulkImportAPI(main)); + StandardContext context = tomcatReference.getContext(); Tomcat tomcat = tomcatReference.getTomcat(); diff --git a/src/main/java/io/supertokens/webserver/api/bulkimport/BulkImportAPI.java b/src/main/java/io/supertokens/webserver/api/bulkimport/BulkImportAPI.java new file mode 100644 index 000000000..0fbf8055f --- /dev/null +++ b/src/main/java/io/supertokens/webserver/api/bulkimport/BulkImportAPI.java @@ -0,0 +1,250 @@ +/* + * Copyright (c) 2024, VRAI Labs and/or its affiliates. All rights reserved. + * + * This software is licensed under the Apache License, Version 2.0 (the + * "License") as published by the Apache Software Foundation. + * + * You may not use this file except in compliance with the License. You may + * obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package io.supertokens.webserver.api.bulkimport; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import com.google.gson.JsonArray; +import com.google.gson.JsonObject; +import com.google.gson.JsonPrimitive; + +import io.supertokens.Main; +import io.supertokens.bulkimport.BulkImport; +import io.supertokens.bulkimport.BulkImportUserPaginationContainer; +import io.supertokens.bulkimport.BulkImportUserPaginationToken; +import io.supertokens.bulkimport.BulkImportUserUtils; +import io.supertokens.multitenancy.exception.BadPermissionException; +import io.supertokens.output.Logging; +import io.supertokens.pluginInterface.bulkimport.BulkImportStorage.BULK_IMPORT_USER_STATUS; +import io.supertokens.pluginInterface.Storage; +import io.supertokens.pluginInterface.StorageUtils; +import io.supertokens.pluginInterface.bulkimport.BulkImportUser; +import io.supertokens.pluginInterface.exceptions.StorageQueryException; +import io.supertokens.pluginInterface.multitenancy.AppIdentifier; +import io.supertokens.pluginInterface.multitenancy.exceptions.TenantOrAppNotFoundException; +import io.supertokens.utils.Utils; +import io.supertokens.webserver.InputParser; +import io.supertokens.webserver.WebserverAPI; +import jakarta.servlet.ServletException; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; + +public class BulkImportAPI extends WebserverAPI { + public BulkImportAPI(Main main) { + super(main, ""); + } + + @Override + public String getPath() { + return "/bulk-import/users"; + } + + @Override + protected void doGet(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException { + String statusString = InputParser.getQueryParamOrThrowError(req, "status", true); + String paginationToken = InputParser.getQueryParamOrThrowError(req, "paginationToken", true); + Integer limit = InputParser.getIntQueryParamOrThrowError(req, "limit", true); + + if (limit != null) { + if (limit > BulkImport.GET_USERS_PAGINATION_LIMIT) { + throw new ServletException( + new BadRequestException("Max limit allowed is " + BulkImport.GET_USERS_PAGINATION_LIMIT)); + } else if (limit < 1) { + throw new ServletException(new BadRequestException("limit must a positive integer with min value 1")); + } + } else { + limit = BulkImport.GET_USERS_DEFAULT_LIMIT; + } + + BULK_IMPORT_USER_STATUS status = null; + if (statusString != null) { + try { + status = BULK_IMPORT_USER_STATUS.valueOf(statusString); + } catch (IllegalArgumentException e) { + throw new ServletException(new BadRequestException("Invalid value for status. Pass one of NEW, PROCESSING, or FAILED!")); + } + } + + AppIdentifier appIdentifier = null; + Storage storage = null; + + try { + appIdentifier = getAppIdentifier(req); + storage = enforcePublicTenantAndGetPublicTenantStorage(req); + } catch (TenantOrAppNotFoundException | BadPermissionException e) { + throw new ServletException(e); + } + + try { + BulkImportUserPaginationContainer users = BulkImport.getUsers(appIdentifier, storage, limit, status, paginationToken); + JsonObject result = new JsonObject(); + result.addProperty("status", "OK"); + + JsonArray usersJson = new JsonArray(); + for (BulkImportUser user : users.users) { + usersJson.add(user.toJsonObject()); + } + result.add("users", usersJson); + + if (users.nextPaginationToken != null) { + result.addProperty("nextPaginationToken", users.nextPaginationToken); + } + super.sendJsonResponse(200, result, resp); + } catch (BulkImportUserPaginationToken.InvalidTokenException e) { + Logging.debug(main, null, Utils.exceptionStacktraceToString(e)); + throw new ServletException(new BadRequestException("invalid pagination token")); + } catch (StorageQueryException e) { + throw new ServletException(e); + } + } + + @Override + protected void doPost(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException { + JsonObject input = InputParser.parseJsonObjectOrThrowError(req); + JsonArray users = InputParser.parseArrayOrThrowError(input, "users", false); + + if (users.size() <= 0 || users.size() > BulkImport.MAX_USERS_TO_ADD) { + JsonObject errorResponseJson = new JsonObject(); + String errorMsg = users.size() <= 0 ? "You need to add at least one user." + : "You can only add " + BulkImport.MAX_USERS_TO_ADD + " users at a time."; + errorResponseJson.addProperty("error", errorMsg); + throw new ServletException(new WebserverAPI.BadRequestException(errorResponseJson.toString())); + } + + AppIdentifier appIdentifier = null; + Storage storage = null; + + try { + appIdentifier = getAppIdentifier(req); + storage = enforcePublicTenantAndGetPublicTenantStorage(req); + } catch (TenantOrAppNotFoundException | BadPermissionException e) { + throw new ServletException(e); + } + + String[] allUserRoles = null; + + try { + allUserRoles = StorageUtils.getUserRolesStorage(storage).getRoles(appIdentifier); + } catch (StorageQueryException e) { + throw new ServletException(e); + } + + JsonArray errorsJson = new JsonArray(); + Set allExternalUserIds = new HashSet<>(); + List usersToAdd = new ArrayList<>(); + + for (int i = 0; i < users.size(); i++) { + try { + BulkImportUser user = BulkImportUserUtils.createBulkImportUserFromJSON(main, appIdentifier, users.get(i).getAsJsonObject(), Utils.getUUID(), allUserRoles, allExternalUserIds); + usersToAdd.add(user); + } catch (io.supertokens.bulkimport.exceptions.InvalidBulkImportDataException e) { + JsonObject errorObj = new JsonObject(); + + JsonArray errors = e.errors.stream() + .map(JsonPrimitive::new) + .collect(JsonArray::new, JsonArray::add, JsonArray::addAll); + + errorObj.addProperty("index", i); + errorObj.add("errors", errors); + errorsJson.add(errorObj); + } catch (StorageQueryException | TenantOrAppNotFoundException e) { + throw new ServletException(e); + } + } + + if (errorsJson.size() > 0) { + JsonObject errorResponseJson = new JsonObject(); + errorResponseJson.addProperty("error", + "Data has missing or invalid fields. Please check the users field for more details."); + errorResponseJson.add("users", errorsJson); + throw new ServletException(new WebserverAPI.BadRequestException(errorResponseJson.toString())); + } + + try { + BulkImport.addUsers(appIdentifier, storage, usersToAdd); + } catch (TenantOrAppNotFoundException | StorageQueryException e) { + throw new ServletException(e); + } + + JsonObject result = new JsonObject(); + result.addProperty("status", "OK"); + super.sendJsonResponse(200, result, resp); + } + + @Override + protected void doDelete(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException { + JsonObject input = InputParser.parseJsonObjectOrThrowError(req); + JsonArray arr = InputParser.parseArrayOrThrowError(input, "ids", false); + + if (arr.size() == 0) { + throw new ServletException(new WebserverAPI.BadRequestException("Field name 'ids' cannot be an empty array")); + } + + if (arr.size() > BulkImport.DELETE_USERS_LIMIT) { + throw new ServletException(new WebserverAPI.BadRequestException("Field name 'ids' cannot contain more than " + + BulkImport.DELETE_USERS_LIMIT + " elements")); + } + + String[] userIds = new String[arr.size()]; + + for (int i = 0; i < userIds.length; i++) { + String userId = InputParser.parseStringFromElementOrThrowError(arr.get(i), "ids", false); + if (userId.isEmpty()) { + throw new ServletException(new WebserverAPI.BadRequestException("Field name 'ids' cannot contain an empty string")); + } + userIds[i] = userId; + } + + AppIdentifier appIdentifier = null; + Storage storage = null; + + try { + appIdentifier = getAppIdentifier(req); + storage = enforcePublicTenantAndGetPublicTenantStorage(req); + } catch (TenantOrAppNotFoundException | BadPermissionException e) { + throw new ServletException(e); + } + + try { + List deletedIds = BulkImport.deleteUsers(appIdentifier, storage, userIds); + + JsonArray deletedIdsJson = new JsonArray(); + JsonArray invalidIds = new JsonArray(); + + for (String userId : userIds) { + if (deletedIds.contains(userId)) { + deletedIdsJson.add(new JsonPrimitive(userId)); + } else { + invalidIds.add(new JsonPrimitive(userId)); + } + } + + JsonObject result = new JsonObject(); + result.add("deletedIds", deletedIdsJson); + result.add("invalidIds", invalidIds); + + super.sendJsonResponse(200, result, resp); + + } catch (StorageQueryException e) { + throw new ServletException(e); + } + } +} diff --git a/src/test/java/io/supertokens/test/bulkimport/BulkImportTest.java b/src/test/java/io/supertokens/test/bulkimport/BulkImportTest.java new file mode 100644 index 000000000..ca0be8534 --- /dev/null +++ b/src/test/java/io/supertokens/test/bulkimport/BulkImportTest.java @@ -0,0 +1,265 @@ +/* + * Copyright (c) 2024, VRAI Labs and/or its affiliates. All rights reserved. + * + * This software is licensed under the Apache License, Version 2.0 (the + * "License") as published by the Apache Software Foundation. + * + * You may not use this file except in compliance with the License. You may + * obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package io.supertokens.test.bulkimport; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; + +import java.util.List; +import java.util.stream.Collectors; + +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TestRule; + +import io.supertokens.ProcessState; +import io.supertokens.bulkimport.BulkImport; +import io.supertokens.bulkimport.BulkImportUserPaginationContainer; +import io.supertokens.pluginInterface.STORAGE_TYPE; +import io.supertokens.pluginInterface.bulkimport.BulkImportStorage; +import io.supertokens.pluginInterface.bulkimport.BulkImportUser; +import io.supertokens.pluginInterface.bulkimport.BulkImportStorage.BULK_IMPORT_USER_STATUS; +import io.supertokens.pluginInterface.bulkimport.sqlStorage.BulkImportSQLStorage; +import io.supertokens.pluginInterface.multitenancy.AppIdentifier; +import io.supertokens.storageLayer.StorageLayer; +import io.supertokens.test.TestingProcessManager; +import io.supertokens.test.Utils; + +import static io.supertokens.test.bulkimport.BulkImportTestUtils.generateBulkImportUser; + +public class BulkImportTest { + @Rule + public TestRule watchman = Utils.getOnFailure(); + + @AfterClass + public static void afterTesting() { + Utils.afterTesting(); + } + + @Before + public void beforeEach() { + Utils.reset(); + } + + @Test + public void shouldAddUsersInBulkImportUsersTable() throws Exception { + String[] args = {"../"}; + + TestingProcessManager.TestingProcess process = TestingProcessManager.start(args); + assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); + + if (StorageLayer.getStorage(process.getProcess()).getType() != STORAGE_TYPE.SQL) { + return; + } + + List users = generateBulkImportUser(10); + + BulkImportStorage storage = (BulkImportStorage) StorageLayer.getStorage(process.main); + BulkImport.addUsers(new AppIdentifier(null, null), storage, users); + + List addedUsers = storage.getBulkImportUsers(new AppIdentifier(null, null), null, BULK_IMPORT_USER_STATUS.NEW, null, null); + + // Verify that all users are present in addedUsers + for (BulkImportUser user : users) { + BulkImportUser matchingUser = addedUsers.stream() + .filter(addedUser -> user.id.equals(addedUser.id)) + .findFirst() + .orElse(null); + + assertNotNull(matchingUser); + assertEquals(BULK_IMPORT_USER_STATUS.NEW, matchingUser.status); + assertEquals(user.toRawDataForDbStorage(), matchingUser.toRawDataForDbStorage()); + } + + process.kill(); + assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STOPPED)); + } + + @Test + public void shouldCreatedNewIdsIfDuplicateIdIsFound() throws Exception { + String[] args = {"../"}; + + TestingProcessManager.TestingProcess process = TestingProcessManager.start(args); + assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); + + if (StorageLayer.getStorage(process.getProcess()).getType() != STORAGE_TYPE.SQL) { + return; + } + + List users = generateBulkImportUser(10); + + // We are setting the id of the second user to be the same as the first user to ensure a duplicate id is present + users.get(1).id = users.get(0).id; + + List initialIds = users.stream().map(user -> user.id).collect(Collectors.toList()); + + BulkImportStorage storage = (BulkImportStorage) StorageLayer.getStorage(process.main); + AppIdentifier appIdentifier = new AppIdentifier(null, null); + BulkImport.addUsers(appIdentifier, storage, users); + + List addedUsers = storage.getBulkImportUsers(appIdentifier, null, BULK_IMPORT_USER_STATUS.NEW, null, null); + + // Verify that the other properties are same but ids changed + for (BulkImportUser user : users) { + BulkImportUser matchingUser = addedUsers.stream() + .filter(addedUser -> user.toRawDataForDbStorage().equals(addedUser.toRawDataForDbStorage())) + .findFirst() + .orElse(null); + + assertNotNull(matchingUser); + assertEquals(BULK_IMPORT_USER_STATUS.NEW, matchingUser.status); + assertFalse(initialIds.contains(matchingUser.id)); + } + + process.kill(); + assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STOPPED)); + } + + @Test + public void testGetUsersStatusFilter() throws Exception { + String[] args = {"../"}; + + TestingProcessManager.TestingProcess process = TestingProcessManager.start(args); + assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); + + if (StorageLayer.getStorage(process.getProcess()).getType() != STORAGE_TYPE.SQL) { + return; + } + + BulkImportSQLStorage storage = (BulkImportSQLStorage) StorageLayer.getStorage(process.main); + AppIdentifier appIdentifier = new AppIdentifier(null, null); + + // Test with status = 'NEW' + { + List users = generateBulkImportUser(10); + BulkImport.addUsers(appIdentifier, storage, users); + + List addedUsers = storage.getBulkImportUsers(appIdentifier, null, BULK_IMPORT_USER_STATUS.NEW, null, null); + assertEquals(10, addedUsers.size()); + } + + // Test with status = 'PROCESSING' + { + List users = generateBulkImportUser(10); + BulkImport.addUsers(appIdentifier, storage, users); + + // Update the users status to PROCESSING + String[] userIds = users.stream().map(user -> user.id).toArray(String[]::new); + + storage.startTransaction(con -> { + storage.updateBulkImportUserStatus_Transaction(appIdentifier, con, userIds, BULK_IMPORT_USER_STATUS.PROCESSING, null); + storage.commitTransaction(con); + return null; + }); + + List addedUsers = storage.getBulkImportUsers(appIdentifier, null, BULK_IMPORT_USER_STATUS.PROCESSING, null, null); + assertEquals(10, addedUsers.size()); + } + + // Test with status = 'FAILED' + { + List users = generateBulkImportUser(10); + BulkImport.addUsers(appIdentifier, storage, users); + + // Update the users status to FAILED + String[] userIds = users.stream().map(user -> user.id).toArray(String[]::new); + + storage.startTransaction(con -> { + storage.updateBulkImportUserStatus_Transaction(appIdentifier, con, userIds, BULK_IMPORT_USER_STATUS.FAILED, null); + storage.commitTransaction(con); + return null; + }); + + List addedUsers = storage.getBulkImportUsers(appIdentifier, null, BULK_IMPORT_USER_STATUS.FAILED, null, null); + assertEquals(10, addedUsers.size()); + } + + process.kill(); + assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STOPPED)); + } + + @Test + public void randomPaginationTest() throws Exception { + String[] args = {"../"}; + + TestingProcessManager.TestingProcess process = TestingProcessManager.start(args); + assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); + + if (StorageLayer.getStorage(process.getProcess()).getType() != STORAGE_TYPE.SQL) { + return; + } + + BulkImportStorage storage = (BulkImportStorage) StorageLayer.getStorage(process.main); + + int numberOfUsers = 500; + // Insert users in batches + { + int batchSize = 100; + for (int i = 0; i < numberOfUsers; i += batchSize) { + List users = generateBulkImportUser(batchSize); + BulkImport.addUsers(new AppIdentifier(null, null), storage, users); + // Adding a delay between each batch to ensure the createdAt different + Thread.sleep(1000); + } + } + + // Get all inserted users + List addedUsers = storage.getBulkImportUsers(new AppIdentifier(null, null), null, null, null, null); + assertEquals(numberOfUsers, addedUsers.size()); + + // We are sorting the users based on createdAt and id like we do in the storage layer + List sortedUsers = addedUsers.stream() + .sorted((user1, user2) -> { + int compareResult = Long.compare(user2.createdAt, user1.createdAt); + if (compareResult == 0) { + return user2.id.compareTo(user1.id); + } + return compareResult; + }) + .collect(Collectors.toList()); + + int[] limits = new int[]{10, 14, 20, 23, 50, 100, 110, 150, 200, 510}; + + for (int limit : limits) { + int indexIntoUsers = 0; + String paginationToken = null; + do { + BulkImportUserPaginationContainer users = BulkImport.getUsers(new AppIdentifier(null, null), storage, limit, null, paginationToken); + + for (BulkImportUser actualUser : users.users) { + BulkImportUser expectedUser = sortedUsers.get(indexIntoUsers); + + assertEquals(expectedUser.id, actualUser.id); + assertEquals(expectedUser.status, actualUser.status); + assertEquals(expectedUser.toRawDataForDbStorage(), actualUser.toRawDataForDbStorage()); + indexIntoUsers++; + } + + paginationToken = users.nextPaginationToken; + } while (paginationToken != null); + + assert (indexIntoUsers == sortedUsers.size()); + } + + process.kill(); + assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STOPPED)); + } + +} diff --git a/src/test/java/io/supertokens/test/bulkimport/BulkImportTestUtils.java b/src/test/java/io/supertokens/test/bulkimport/BulkImportTestUtils.java new file mode 100644 index 000000000..6b822d610 --- /dev/null +++ b/src/test/java/io/supertokens/test/bulkimport/BulkImportTestUtils.java @@ -0,0 +1,59 @@ +/* + * Copyright (c) 2024, VRAI Labs and/or its affiliates. All rights reserved. + * + * This software is licensed under the Apache License, Version 2.0 (the + * "License") as published by the Apache Software Foundation. + * + * You may not use this file except in compliance with the License. You may + * obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + + +package io.supertokens.test.bulkimport; + +import java.util.ArrayList; +import java.util.List; + +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; + +import io.supertokens.pluginInterface.bulkimport.BulkImportUser; +import io.supertokens.pluginInterface.bulkimport.BulkImportUser.LoginMethod; +import io.supertokens.pluginInterface.bulkimport.BulkImportUser.TotpDevice; +import io.supertokens.pluginInterface.bulkimport.BulkImportUser.UserRole; + +public class BulkImportTestUtils { + public static List generateBulkImportUser(int numberOfUsers) { + List users = new ArrayList<>(); + JsonParser parser = new JsonParser(); + + for (int i = 0; i < numberOfUsers; i++) { + String email = "user" + i + "@example.com"; + String id = io.supertokens.utils.Utils.getUUID(); + String externalId = io.supertokens.utils.Utils.getUUID(); + + JsonObject userMetadata = parser.parse("{\"key1\":\"value1\",\"key2\":{\"key3\":\"value3\"}}").getAsJsonObject(); + + List userRoles = new ArrayList<>(); + userRoles.add(new UserRole("role1", List.of("public"))); + userRoles.add(new UserRole("role2", List.of("public"))); + + List totpDevices = new ArrayList<>(); + totpDevices.add(new TotpDevice("secretKey", 30, 1, "deviceName")); + + List loginMethods = new ArrayList<>(); + long currentTimeMillis = System.currentTimeMillis(); + loginMethods.add(new LoginMethod(List.of("public", "t1"), "emailpassword", true, true, currentTimeMillis, email, "$2a", "BCRYPT", null, null, null)); + loginMethods.add(new LoginMethod(List.of("public", "t1"), "thirdparty", true, false, currentTimeMillis, email, null, null, "thirdPartyId" + i, "thirdPartyUserId" + i, null)); + loginMethods.add(new LoginMethod(List.of("public", "t1"), "passwordless", true, false, currentTimeMillis, email, null, null, null, null, null)); + users.add(new BulkImportUser(id, externalId, userMetadata, userRoles, totpDevices, loginMethods)); + } + return users; + } +} diff --git a/src/test/java/io/supertokens/test/bulkimport/ProcessBulkImportUsersCronJobTest.java b/src/test/java/io/supertokens/test/bulkimport/ProcessBulkImportUsersCronJobTest.java new file mode 100644 index 000000000..78bff7e32 --- /dev/null +++ b/src/test/java/io/supertokens/test/bulkimport/ProcessBulkImportUsersCronJobTest.java @@ -0,0 +1,297 @@ +/* + * Copyright (c) 2024, VRAI Labs and/or its affiliates. All rights reserved. + * + * This software is licensed under the Apache License, Version 2.0 (the + * "License") as published by the Apache Software Foundation. + * + * You may not use this file except in compliance with the License. You may + * obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + */ + +package io.supertokens.test.bulkimport; + +import io.supertokens.Main; +import io.supertokens.ProcessState; +import io.supertokens.authRecipe.AuthRecipe; +import io.supertokens.authRecipe.UserPaginationContainer; +import io.supertokens.bulkimport.BulkImport; +import io.supertokens.cronjobs.CronTaskTest; +import io.supertokens.cronjobs.bulkimport.ProcessBulkImportUsers; +import io.supertokens.featureflag.EE_FEATURES; +import io.supertokens.featureflag.FeatureFlagTestContent; +import io.supertokens.featureflag.exceptions.FeatureNotEnabledException; +import io.supertokens.multitenancy.Multitenancy; +import io.supertokens.multitenancy.exception.BadPermissionException; +import io.supertokens.multitenancy.exception.CannotModifyBaseConfigException; +import io.supertokens.pluginInterface.STORAGE_TYPE; +import io.supertokens.pluginInterface.authRecipe.AuthRecipeUserInfo; +import io.supertokens.pluginInterface.authRecipe.LoginMethod; +import io.supertokens.pluginInterface.bulkimport.BulkImportUser; +import io.supertokens.pluginInterface.bulkimport.BulkImportStorage.BULK_IMPORT_USER_STATUS; +import io.supertokens.pluginInterface.bulkimport.BulkImportUser.TotpDevice; +import io.supertokens.pluginInterface.bulkimport.sqlStorage.BulkImportSQLStorage; +import io.supertokens.pluginInterface.exceptions.InvalidConfigException; +import io.supertokens.pluginInterface.exceptions.StorageQueryException; +import io.supertokens.pluginInterface.multitenancy.AppIdentifier; +import io.supertokens.pluginInterface.multitenancy.EmailPasswordConfig; +import io.supertokens.pluginInterface.multitenancy.PasswordlessConfig; +import io.supertokens.pluginInterface.multitenancy.TenantConfig; +import io.supertokens.pluginInterface.multitenancy.TenantIdentifier; +import io.supertokens.pluginInterface.multitenancy.ThirdPartyConfig; +import io.supertokens.pluginInterface.multitenancy.exceptions.TenantOrAppNotFoundException; +import io.supertokens.pluginInterface.totp.TOTPDevice; +import io.supertokens.storageLayer.StorageLayer; +import io.supertokens.test.TestingProcessManager; +import io.supertokens.test.TestingProcessManager.TestingProcess; +import io.supertokens.test.Utils; +import io.supertokens.thirdparty.InvalidProviderConfigException; +import io.supertokens.totp.Totp; +import io.supertokens.useridmapping.UserIdMapping; +import io.supertokens.usermetadata.UserMetadata; +import io.supertokens.userroles.UserRoles; + +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TestRule; + +import com.google.gson.JsonObject; + +import static io.supertokens.test.bulkimport.BulkImportTestUtils.generateBulkImportUser; +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +import java.io.IOException; +import java.util.List; + +public class ProcessBulkImportUsersCronJobTest { + @Rule + public TestRule watchman = Utils.getOnFailure(); + + @AfterClass + public static void afterTesting() { + Utils.afterTesting(); + } + + @Before + public void beforeEach() { + Utils.reset(); + } + + @Test + public void shouldProcessBulkImportUsers() throws Exception { + TestingProcess process = startCronProcess(); + Main main = process.getProcess(); + + // Create user roles before inserting bulk users + { + UserRoles.createNewRoleOrModifyItsPermissions(main, "role1", null); + UserRoles.createNewRoleOrModifyItsPermissions(main, "role2", null); + } + + createTenants(main); + + BulkImportSQLStorage storage = (BulkImportSQLStorage) StorageLayer.getStorage(main); + AppIdentifier appIdentifier = new AppIdentifier(null, null); + + int usersCount = 1; + List users = generateBulkImportUser(usersCount); + BulkImport.addUsers(appIdentifier, storage, users); + + BulkImportUser bulkImportUser = users.get(0); + + // Thread.sleep(600000); + Thread.sleep(6000); + + List usersAfterProcessing = storage.getBulkImportUsers(appIdentifier, null, null, + null, null); + + System.out.println("Users after processing: " + usersAfterProcessing.size()); + assertEquals(0, usersAfterProcessing.size()); + + UserPaginationContainer container = AuthRecipe.getUsers(main, 100, "ASC", null, null, null); + assertEquals(usersCount, container.users.length); + + UserIdMapping.populateExternalUserIdForUsers(appIdentifier, storage, container.users); + + for (AuthRecipeUserInfo user : container.users) { + for (LoginMethod lm1 : user.loginMethods) { + bulkImportUser.loginMethods.forEach(lm2 -> { + if (lm2.recipeId.equals(lm1.recipeId.toString())) { + assertLoginMethodEquals(lm1, lm2); + } + }); + } + + JsonObject createdUserMetadata = UserMetadata.getUserMetadata(main, user.getSupertokensOrExternalUserId()); + assertEquals(bulkImportUser.userMetadata, createdUserMetadata); + + String[] createdUserRoles = UserRoles.getRolesForUser(main, user.getSupertokensOrExternalUserId()); + String[] bulkImportUserRoles = bulkImportUser.userRoles.stream().map(r -> r.role).toArray(String[]::new); + assertArrayEquals(bulkImportUserRoles, createdUserRoles); + + assertEquals(bulkImportUser.externalUserId, user.getSupertokensOrExternalUserId()); + + + TOTPDevice[] createdTotpDevices = Totp.getDevices(main, user.getSupertokensOrExternalUserId()); + assertTotpDevicesEquals(createdTotpDevices, bulkImportUser.totpDevices.toArray(new TotpDevice[0])); + } + + process.kill(); + assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STOPPED)); + } + + @Test + public void shouldDeleteEverythingFromtheDBIfAnythingFails() throws Exception { + // Creating a non-existing user role will result in an error. + // Since, user role creation happens at the last step of the bulk import process, everything should be deleted from the DB. + + TestingProcess process = startCronProcess(); + Main main = process.getProcess(); + + createTenants(main); + + BulkImportSQLStorage storage = (BulkImportSQLStorage) StorageLayer.getStorage(main); + AppIdentifier appIdentifier = new AppIdentifier(null, null); + + List users = generateBulkImportUser(1); + BulkImport.addUsers(appIdentifier, storage, users); + + Thread.sleep(6000); + + List usersAfterProcessing = storage.getBulkImportUsers(appIdentifier, null, null, + null, null); + + assertEquals(1, usersAfterProcessing.size()); + + assertEquals(BULK_IMPORT_USER_STATUS.FAILED, usersAfterProcessing.get(0).status); + assertEquals("Role role1 does not exist! You need pre-create the role before assigning it to the user.", + usersAfterProcessing.get(0).errorMessage); + + UserPaginationContainer container = AuthRecipe.getUsers(main, 100, "ASC", null, null, null); + assertEquals(0, container.users.length); + } + + @Test + public void shouldThrowTenantDoesNotExistError() throws Exception { + TestingProcess process = startCronProcess(); + Main main = process.getProcess(); + + BulkImportSQLStorage storage = (BulkImportSQLStorage) StorageLayer.getStorage(main); + AppIdentifier appIdentifier = new AppIdentifier(null, null); + + List users = generateBulkImportUser(1); + BulkImport.addUsers(appIdentifier, storage, users); + + Thread.sleep(6000); + + List usersAfterProcessing = storage.getBulkImportUsers(appIdentifier, null, null, + null, null); + + assertEquals(1, usersAfterProcessing.size()); + assertEquals(BULK_IMPORT_USER_STATUS.FAILED, usersAfterProcessing.get(0).status); + assertEquals( + "Tenant with the following connectionURIDomain, appId and tenantId combination not found: (, public, t1)", + usersAfterProcessing.get(0).errorMessage); + } + + private TestingProcess startCronProcess() throws InterruptedException { + String[] args = { "../" }; + + TestingProcessManager.TestingProcess process = TestingProcessManager.start(args, false); + + Main main = process.getProcess(); + + FeatureFlagTestContent.getInstance(main) + .setKeyValue(FeatureFlagTestContent.ENABLED_FEATURES, new EE_FEATURES[] { + EE_FEATURES.ACCOUNT_LINKING, EE_FEATURES.MULTI_TENANCY, EE_FEATURES.MFA }); + + CronTaskTest.getInstance(main).setIntervalInSeconds(ProcessBulkImportUsers.RESOURCE_KEY, 100000); + process.startProcess(); + assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); + + if (StorageLayer.getStorage(main).getType() != STORAGE_TYPE.SQL) { + return null; + } + + return process; + } + + private void assertLoginMethodEquals(LoginMethod lm1, + io.supertokens.pluginInterface.bulkimport.BulkImportUser.LoginMethod lm2) { + assertEquals(lm1.email, lm2.email); + assertEquals(lm1.verified, lm2.isVerified); + assertTrue(lm2.tenantIds.containsAll(lm1.tenantIds) && lm1.tenantIds.containsAll(lm2.tenantIds)); + + switch (lm2.recipeId) { + case "emailpassword": + assertEquals(lm1.passwordHash, lm2.passwordHash); + break; + case "thirdparty": + assertEquals(lm1.thirdParty.id, lm2.thirdPartyId); + assertEquals(lm1.thirdParty.userId, lm2.thirdPartyUserId); + break; + case "passwordless": + assertEquals(lm1.phoneNumber, lm2.phoneNumber); + break; + default: + break; + } + } + + private void assertTotpDevicesEquals(TOTPDevice[] createdTotpDevices, TotpDevice[] bulkImportTotpDevices) { + assertEquals(createdTotpDevices.length, bulkImportTotpDevices.length); + for (int i = 0; i < createdTotpDevices.length; i++) { + assertEquals(createdTotpDevices[i].deviceName, bulkImportTotpDevices[i].deviceName); + assertEquals(createdTotpDevices[i].period, bulkImportTotpDevices[i].period); + assertEquals(createdTotpDevices[i].secretKey, bulkImportTotpDevices[i].secretKey); + assertEquals(createdTotpDevices[i].skew, bulkImportTotpDevices[i].skew); + } + } + + private void createTenants(Main main) + throws StorageQueryException, TenantOrAppNotFoundException, InvalidProviderConfigException, + FeatureNotEnabledException, IOException, InvalidConfigException, + CannotModifyBaseConfigException, BadPermissionException { + { // tenant 1 (t1 in the same storage as public tenant) + TenantIdentifier tenantIdentifier = new TenantIdentifier(null, null, "t1"); + + Multitenancy.addNewOrUpdateAppOrTenant( + main, + new TenantIdentifier(null, null, null), + new TenantConfig( + tenantIdentifier, + new EmailPasswordConfig(true), + new ThirdPartyConfig(true, null), + new PasswordlessConfig(true), + null, null, new JsonObject())); + } + { // tenant 2 (t2 in the different storage than public tenant) + TenantIdentifier tenantIdentifier = new TenantIdentifier(null, null, "t2"); + + JsonObject config = new JsonObject(); + + StorageLayer.getStorage(new TenantIdentifier(null, null, null), main) + .modifyConfigToAddANewUserPoolForTesting(config, 1); + Multitenancy.addNewOrUpdateAppOrTenant( + main, + new TenantIdentifier(null, null, null), + new TenantConfig( + tenantIdentifier, + new EmailPasswordConfig(true), + new ThirdPartyConfig(true, null), + new PasswordlessConfig(true), + null, null, config)); + } + } +} diff --git a/src/test/java/io/supertokens/test/bulkimport/apis/AddBulkImportUsersTest.java b/src/test/java/io/supertokens/test/bulkimport/apis/AddBulkImportUsersTest.java new file mode 100644 index 000000000..3303ebca7 --- /dev/null +++ b/src/test/java/io/supertokens/test/bulkimport/apis/AddBulkImportUsersTest.java @@ -0,0 +1,692 @@ +/* + * Copyright (c) 2024, VRAI Labs and/or its affiliates. All rights reserved. + * + * This software is licensed under the Apache License, Version 2.0 (the + * "License") as published by the Apache Software Foundation. + * + * You may not use this file except in compliance with the License. You may + * obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package io.supertokens.test.bulkimport.apis; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.fail; + +import java.io.IOException; +import java.util.HashMap; +import java.util.UUID; + +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TestRule; + +import com.google.gson.JsonArray; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; + +import io.supertokens.Main; +import io.supertokens.ProcessState; +import io.supertokens.featureflag.EE_FEATURES; +import io.supertokens.featureflag.FeatureFlagTestContent; +import io.supertokens.featureflag.exceptions.FeatureNotEnabledException; +import io.supertokens.multitenancy.Multitenancy; +import io.supertokens.multitenancy.exception.BadPermissionException; +import io.supertokens.multitenancy.exception.CannotModifyBaseConfigException; +import io.supertokens.pluginInterface.STORAGE_TYPE; +import io.supertokens.pluginInterface.exceptions.InvalidConfigException; +import io.supertokens.pluginInterface.exceptions.StorageQueryException; +import io.supertokens.pluginInterface.multitenancy.EmailPasswordConfig; +import io.supertokens.pluginInterface.multitenancy.PasswordlessConfig; +import io.supertokens.pluginInterface.multitenancy.TenantConfig; +import io.supertokens.pluginInterface.multitenancy.TenantIdentifier; +import io.supertokens.pluginInterface.multitenancy.ThirdPartyConfig; +import io.supertokens.pluginInterface.multitenancy.exceptions.TenantOrAppNotFoundException; +import io.supertokens.storageLayer.StorageLayer; +import io.supertokens.test.TestingProcessManager; +import io.supertokens.test.Utils; +import io.supertokens.test.httpRequest.HttpRequestForTesting; +import io.supertokens.thirdparty.InvalidProviderConfigException; +import io.supertokens.userroles.UserRoles; + +public class AddBulkImportUsersTest { + @Rule + public TestRule watchman = Utils.getOnFailure(); + + @AfterClass + public static void afterTesting() { + Utils.afterTesting(); + } + + @Before + public void beforeEach() { + Utils.reset(); + } + + public String getResponseMessageFromError(String response) { + return response.substring(response.indexOf("Message: ") + "Message: ".length()); + } + + @Test + public void shouldThrow400Error() throws Exception { + String[] args = { "../" }; + + TestingProcessManager.TestingProcess process = TestingProcessManager.start(args); + assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); + + if (StorageLayer.getStorage(process.getProcess()).getType() != STORAGE_TYPE.SQL) { + return; + } + + // Create user roles + { + UserRoles.createNewRoleOrModifyItsPermissions(process.getProcess(), "role1", null); + } + + String genericErrMsg = "Data has missing or invalid fields. Please check the users field for more details."; + + // users is required in the json body + { + // CASE 1: users field is not present + try { + JsonObject request = new JsonObject(); + HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", + "http://localhost:3567/bulk-import/users", + request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); + fail("The API should have thrown an error"); + } catch (io.supertokens.test.httpRequest.HttpResponseException e) { + String responseString = getResponseMessageFromError(e.getMessage()); + assertEquals(400, e.statusCode); + assertEquals(responseString, "Field name 'users' is invalid in JSON input"); + } + // CASE 2: users field type in incorrect + try { + JsonObject request = new JsonParser().parse("{\"users\": \"string\"}").getAsJsonObject(); + HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", + "http://localhost:3567/bulk-import/users", + request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); + fail("The API should have thrown an error"); + } catch (io.supertokens.test.httpRequest.HttpResponseException e) { + String responseString = getResponseMessageFromError(e.getMessage()); + assertEquals(400, e.statusCode); + assertEquals(responseString, "Field name 'users' is invalid in JSON input"); + } + } + // loginMethod array is required in the user object + { + // CASE 1: loginMethods field is not present + try { + JsonObject request = new JsonParser().parse("{\"users\":[{}]}").getAsJsonObject(); + HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", + "http://localhost:3567/bulk-import/users", + request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); + fail("The API should have thrown an error"); + } catch (io.supertokens.test.httpRequest.HttpResponseException e) { + String responseString = getResponseMessageFromError(e.getMessage()); + assertEquals(400, e.statusCode); + assertEquals(responseString, "{\"error\":\"" + genericErrMsg + "\",\"users\":[{\"index\":0,\"errors\":[\"loginMethods is required.\"]}]}"); + } + // CASE 2: loginMethods field type in incorrect + try { + JsonObject request = new JsonParser().parse("{\"users\":[{\"loginMethods\": \"string\"}]}") + .getAsJsonObject(); + HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", + "http://localhost:3567/bulk-import/users", + request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); + fail("The API should have thrown an error"); + } catch (io.supertokens.test.httpRequest.HttpResponseException e) { + String responseString = getResponseMessageFromError(e.getMessage()); + assertEquals(400, e.statusCode); + assertEquals(responseString, + "{\"error\":\"" + genericErrMsg + "\",\"users\":[{\"index\":0,\"errors\":[\"loginMethods should be of type array of object.\"]}]}"); + } + // CASE 3: loginMethods array is empty + try { + JsonObject request = new JsonParser().parse("{\"users\":[{\"loginMethods\": []}]}").getAsJsonObject(); + HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", + "http://localhost:3567/bulk-import/users", + request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); + fail("The API should have thrown an error"); + } catch (io.supertokens.test.httpRequest.HttpResponseException e) { + String responseString = getResponseMessageFromError(e.getMessage()); + assertEquals(400, e.statusCode); + assertEquals(responseString, "{\"error\":\"" + genericErrMsg + "\",\"users\":[{\"index\":0,\"errors\":[\"At least one loginMethod is required.\"]}]}"); + } + } + // Invalid field type of non required fields outside loginMethod + { + try { + JsonObject request = new JsonParser() + .parse("{\"users\":[{\"externalUserId\":[],\"userMetaData\":[],\"userRoles\":{},\"totpDevices\":{}}]}") + .getAsJsonObject(); + HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", + "http://localhost:3567/bulk-import/users", + request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); + fail("The API should have thrown an error"); + } catch (io.supertokens.test.httpRequest.HttpResponseException e) { + String responseString = getResponseMessageFromError(e.getMessage()); + assertEquals(400, e.statusCode); + assertEquals(responseString, + "{\"error\":\"" + genericErrMsg + "\",\"users\":[{\"index\":0,\"errors\":[\"externalUserId should be of type string.\",\"userRoles should be of type array of object.\",\"totpDevices should be of type array of object.\",\"loginMethods is required.\"]}]}"); + } + // Non-unique externalUserIds + try { + JsonObject request = new JsonParser() + .parse("{\"users\":[{\"externalUserId\":\"id1\"}, {\"externalUserId\":\"id1\"}]}") + .getAsJsonObject(); + HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", + "http://localhost:3567/bulk-import/users", + request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); + fail("The API should have thrown an error"); + } catch (io.supertokens.test.httpRequest.HttpResponseException e) { + String responseString = getResponseMessageFromError(e.getMessage()); + assertEquals(400, e.statusCode); + assertEquals(responseString, + "{\"error\":\"" + genericErrMsg + "\",\"users\":[{\"index\":0,\"errors\":[\"loginMethods is required.\"]},{\"index\":1,\"errors\":[\"loginMethods is required.\",\"externalUserId id1 is not unique. It is already used by another user.\"]}]}"); + } + // secretKey is required in totpDevices + try { + JsonObject request = new JsonParser() + .parse("{\"users\":[{\"totpDevices\":[{\"secret\": \"secret\"}]}]}") + .getAsJsonObject(); + HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", + "http://localhost:3567/bulk-import/users", + request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); + fail("The API should have thrown an error"); + } catch (io.supertokens.test.httpRequest.HttpResponseException e) { + String responseString = getResponseMessageFromError(e.getMessage()); + assertEquals(400, e.statusCode); + assertEquals(responseString, + "{\"error\":\"" + genericErrMsg + "\",\"users\":[{\"index\":0,\"errors\":[\"secretKey is required for a totp device.\",\"loginMethods is required.\"]}]}"); + } + // Invalid role (tenantIds is required) + try { + JsonObject request = new JsonParser() + .parse("{\"users\":[{\"userRoles\":[{\"role\":\"role1\"}]}]}") + .getAsJsonObject(); + HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", + "http://localhost:3567/bulk-import/users", + request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); + fail("The API should have thrown an error"); + } catch (io.supertokens.test.httpRequest.HttpResponseException e) { + String responseString = getResponseMessageFromError(e.getMessage()); + assertEquals(400, e.statusCode); + + assertEquals(responseString, + "{\"error\":\"" + genericErrMsg + "\",\"users\":[{\"index\":0,\"errors\":[\"tenantIds is required for a user role.\",\"loginMethods is required.\"]}]}"); + } + // Invalid role (role doesn't exist) + try { + JsonObject request = new JsonParser() + .parse("{\"users\":[{\"userRoles\":[{\"role\":\"role5\", \"tenantIds\": [\"public\"]}]}]}") + .getAsJsonObject(); + HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", + "http://localhost:3567/bulk-import/users", + request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); + fail("The API should have thrown an error"); + } catch (io.supertokens.test.httpRequest.HttpResponseException e) { + String responseString = getResponseMessageFromError(e.getMessage()); + assertEquals(400, e.statusCode); + + assertEquals(responseString, + "{\"error\":\"" + genericErrMsg + "\",\"users\":[{\"index\":0,\"errors\":[\"Role role5 does not exist.\",\"loginMethods is required.\"]}]}"); + } + } + // Invalid field type of non required fields inside loginMethod + { + try { + JsonObject request = new JsonParser().parse( + "{\"users\":[{\"loginMethods\":[{\"recipeId\":[],\"tenantIds\":{},\"isPrimary\":[],\"isVerified\":[],\"timeJoinedInMSSinceEpoch\":[]}]}]}") + .getAsJsonObject(); + HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", + "http://localhost:3567/bulk-import/users", + request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); + fail("The API should have thrown an error"); + } catch (io.supertokens.test.httpRequest.HttpResponseException e) { + String responseString = getResponseMessageFromError(e.getMessage()); + assertEquals(400, e.statusCode); + assertEquals(responseString, + "{\"error\":\"" + genericErrMsg + "\",\"users\":[{\"index\":0,\"errors\":[\"recipeId should be of type string for a loginMethod.\",\"tenantIds should be of type array of string for a loginMethod.\",\"isVerified should be of type boolean for a loginMethod.\",\"isPrimary should be of type boolean for a loginMethod.\",\"timeJoinedInMSSinceEpoch should be of type integer for a loginMethod\"]}]}"); + } + } + // Invalid recipeId + { + try { + JsonObject request = new JsonParser() + .parse("{\"users\":[{\"loginMethods\":[{\"recipeId\":\"invalid_recipe_id\"}]}]}") + .getAsJsonObject(); + HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", + "http://localhost:3567/bulk-import/users", + request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); + fail("The API should have thrown an error"); + } catch (io.supertokens.test.httpRequest.HttpResponseException e) { + String responseString = getResponseMessageFromError(e.getMessage()); + assertEquals(400, e.statusCode); + assertEquals(responseString, + "{\"error\":\"" + genericErrMsg + "\",\"users\":[{\"index\":0,\"errors\":[\"Invalid recipeId for loginMethod. Pass one of emailpassword, thirdparty or, passwordless!\"]}]}"); + } + } + // Invalid field type in emailpassword recipe + { + // CASE 1: email, passwordHash and hashingAlgorithm are not present + try { + JsonObject request = new JsonParser() + .parse("{\"users\":[{\"loginMethods\":[{\"recipeId\":\"emailpassword\"}]}]}").getAsJsonObject(); + HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", + "http://localhost:3567/bulk-import/users", + request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); + fail("The API should have thrown an error"); + } catch (io.supertokens.test.httpRequest.HttpResponseException e) { + String responseString = getResponseMessageFromError(e.getMessage()); + assertEquals(400, e.statusCode); + assertEquals(responseString, + "{\"error\":\"" + genericErrMsg + "\",\"users\":[{\"index\":0,\"errors\":[\"email is required for an emailpassword recipe.\",\"passwordHash is required for an emailpassword recipe.\",\"hashingAlgorithm is required for an emailpassword recipe.\"]}]}"); + } + // CASE 2: email, passwordHash and hashingAlgorithm field type is incorrect + try { + JsonObject request = new JsonParser().parse( + "{\"users\":[{\"loginMethods\":[{\"recipeId\":\"emailpassword\",\"email\":[],\"passwordHash\":[],\"hashingAlgorithm\":[]}]}]}") + .getAsJsonObject(); + HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", + "http://localhost:3567/bulk-import/users", + request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); + fail("The API should have thrown an error"); + } catch (io.supertokens.test.httpRequest.HttpResponseException e) { + String responseString = getResponseMessageFromError(e.getMessage()); + assertEquals(400, e.statusCode); + assertEquals(responseString, + "{\"error\":\"" + genericErrMsg + "\",\"users\":[{\"index\":0,\"errors\":[\"email should be of type string for an emailpassword recipe.\",\"passwordHash should be of type string for an emailpassword recipe.\",\"hashingAlgorithm should be of type string for an emailpassword recipe.\"]}]}"); + } + // CASE 3: hashingAlgorithm is not one of bcrypt, argon2, firebase_scrypt + try { + JsonObject request = new JsonParser().parse( + "{\"users\":[{\"loginMethods\":[{\"recipeId\":\"emailpassword\",\"email\":\"johndoe@gmail.com\",\"passwordHash\":\"$2a\",\"hashingAlgorithm\":\"invalid_algorithm\"}]}]}") + .getAsJsonObject(); + HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", + "http://localhost:3567/bulk-import/users", + request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); + fail("The API should have thrown an error"); + } catch (io.supertokens.test.httpRequest.HttpResponseException e) { + String responseString = getResponseMessageFromError(e.getMessage()); + assertEquals(400, e.statusCode); + assertEquals(responseString, + "{\"error\":\"" + genericErrMsg + "\",\"users\":[{\"index\":0,\"errors\":[\"Invalid hashingAlgorithm for emailpassword recipe. Pass one of bcrypt, argon2 or, firebase_scrypt!\"]}]}"); + } + } + // Invalid field type in thirdparty recipe + { + // CASE 1: email, thirdPartyId and thirdPartyUserId are not present + try { + JsonObject request = new JsonParser() + .parse("{\"users\":[{\"loginMethods\":[{\"recipeId\":\"thirdparty\"}]}]}").getAsJsonObject(); + HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", + "http://localhost:3567/bulk-import/users", + request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); + fail("The API should have thrown an error"); + } catch (io.supertokens.test.httpRequest.HttpResponseException e) { + String responseString = getResponseMessageFromError(e.getMessage()); + assertEquals(400, e.statusCode); + assertEquals(responseString, + "{\"error\":\"" + genericErrMsg + "\",\"users\":[{\"index\":0,\"errors\":[\"email is required for a thirdparty recipe.\",\"thirdPartyId is required for a thirdparty recipe.\",\"thirdPartyUserId is required for a thirdparty recipe.\"]}]}"); + } + // CASE 2: email, passwordHash and thirdPartyUserId field type is incorrect + try { + JsonObject request = new JsonParser().parse( + "{\"users\":[{\"loginMethods\":[{\"recipeId\":\"thirdparty\",\"email\":[],\"thirdPartyId\":[],\"thirdPartyUserId\":[]}]}]}") + .getAsJsonObject(); + HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", + "http://localhost:3567/bulk-import/users", + request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); + fail("The API should have thrown an error"); + } catch (io.supertokens.test.httpRequest.HttpResponseException e) { + String responseString = getResponseMessageFromError(e.getMessage()); + assertEquals(400, e.statusCode); + assertEquals(responseString, + "{\"error\":\"" + genericErrMsg + "\",\"users\":[{\"index\":0,\"errors\":[\"email should be of type string for a thirdparty recipe.\",\"thirdPartyId should be of type string for a thirdparty recipe.\",\"thirdPartyUserId should be of type string for a thirdparty recipe.\"]}]}"); + } + } + // Invalid field type in passwordless recipe + { + // CASE 1: email and phoneNumber are not present + try { + JsonObject request = new JsonParser() + .parse("{\"users\":[{\"loginMethods\":[{\"recipeId\":\"passwordless\"}]}]}").getAsJsonObject(); + HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", + "http://localhost:3567/bulk-import/users", + request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); + fail("The API should have thrown an error"); + } catch (io.supertokens.test.httpRequest.HttpResponseException e) { + String responseString = getResponseMessageFromError(e.getMessage()); + assertEquals(400, e.statusCode); + assertEquals(responseString, + "{\"error\":\"" + genericErrMsg + "\",\"users\":[{\"index\":0,\"errors\":[\"Either email or phoneNumber is required for a passwordless recipe.\"]}]}"); + } + // CASE 2: email and phoneNumber field type is incorrect + try { + JsonObject request = new JsonParser().parse( + "{\"users\":[{\"loginMethods\":[{\"recipeId\":\"passwordless\",\"email\":[],\"phoneNumber\":[]}]}]}") + .getAsJsonObject(); + HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", + "http://localhost:3567/bulk-import/users", + request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); + fail("The API should have thrown an error"); + } catch (io.supertokens.test.httpRequest.HttpResponseException e) { + String responseString = getResponseMessageFromError(e.getMessage()); + assertEquals(400, e.statusCode); + assertEquals(responseString, + "{\"error\":\"" + genericErrMsg + "\",\"users\":[{\"index\":0,\"errors\":[\"email should be of type string for a passwordless recipe.\",\"phoneNumber should be of type string for a passwordless recipe.\",\"Either email or phoneNumber is required for a passwordless recipe.\"]}]}"); + } + } + // Validate tenantId + { + // CASE 1: Invalid tenantId when multitenancy is not enabled + try { + JsonObject request = new JsonParser().parse( + "{\"users\":[{\"loginMethods\":[{\"tenantIds\":[\"invalid\"],\"recipeId\":\"passwordless\",\"email\":\"johndoe@gmail.com\"}]}]}") + .getAsJsonObject(); + HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", + "http://localhost:3567/bulk-import/users", + request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); + + fail("The API should have thrown an error"); + } catch (io.supertokens.test.httpRequest.HttpResponseException e) { + String responseString = getResponseMessageFromError(e.getMessage()); + assertEquals(400, e.statusCode); + assertEquals(responseString, + "{\"error\":\"" + genericErrMsg + "\",\"users\":[{\"index\":0,\"errors\":[\"Multitenancy must be enabled before importing users to a different tenant.\"]}]}"); + } + // CASE 2: Invalid tenantId when multitenancy is enabled + try { + FeatureFlagTestContent.getInstance(process.getProcess()) + .setKeyValue(FeatureFlagTestContent.ENABLED_FEATURES, new EE_FEATURES[]{EE_FEATURES.MULTI_TENANCY}); + + JsonObject request = new JsonParser().parse( + "{\"users\":[{\"loginMethods\":[{\"tenantIds\":[\"invalid\"],\"recipeId\":\"passwordless\",\"email\":\"johndoe@gmail.com\"}]}]}") + .getAsJsonObject(); + HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", + "http://localhost:3567/bulk-import/users", + request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); + fail("The API should have thrown an error"); + } catch (io.supertokens.test.httpRequest.HttpResponseException e) { + String responseString = getResponseMessageFromError(e.getMessage()); + assertEquals(400, e.statusCode); + assertEquals(responseString, + "{\"error\":\"" + genericErrMsg + "\",\"users\":[{\"index\":0,\"errors\":[\"Invalid tenantId: invalid for passwordless recipe.\"]}]}"); + } + // CASE 3. Two more tenants do not share the same storage + try { + FeatureFlagTestContent.getInstance(process.getProcess()) + .setKeyValue(FeatureFlagTestContent.ENABLED_FEATURES, new EE_FEATURES[]{EE_FEATURES.MULTI_TENANCY}); + + createTenants(process.getProcess()); + + JsonObject request = new JsonParser().parse( + "{\"users\":[{\"loginMethods\":[{\"tenantIds\":[\"public\"],\"recipeId\":\"passwordless\",\"email\":\"johndoe@gmail.com\"}, {\"tenantIds\":[\"t2\"],\"recipeId\":\"thirdparty\", \"email\":\"johndoe@gmail.com\", \"thirdPartyId\":\"id\", \"thirdPartyUserId\":\"id\"}]}]}") + .getAsJsonObject(); + HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", + "http://localhost:3567/bulk-import/users", + request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); + fail("The API should have thrown an error"); + } catch (io.supertokens.test.httpRequest.HttpResponseException e) { + String responseString = getResponseMessageFromError(e.getMessage()); + assertEquals(400, e.statusCode); + assertEquals(responseString, + "{\"error\":\"" + genericErrMsg + "\",\"users\":[{\"index\":0,\"errors\":[\"All tenants for a user must share the same storage.\"]}]}"); + } + } + // No two loginMethods can have isPrimary as true + { + // CASE 1: email, passwordHash and hashingAlgorithm are not present + try { + JsonObject request = new JsonParser() + .parse("{\"users\":[{\"loginMethods\":[{\"recipeId\":\"emailpassword\",\"email\":\"johndoe@gmail.com\",\"passwordHash\":\"$2a\",\"hashingAlgorithm\":\"bcrypt\",\"isPrimary\":true},{\"recipeId\":\"passwordless\",\"email\":\"johndoe@gmail.com\",\"isPrimary\":true}]}]}").getAsJsonObject(); + HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", + "http://localhost:3567/bulk-import/users", + request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); + fail("The API should have thrown an error"); + } catch (io.supertokens.test.httpRequest.HttpResponseException e) { + String responseString = getResponseMessageFromError(e.getMessage()); + assertEquals(400, e.statusCode); + assertEquals(responseString, + "{\"error\":\"" + genericErrMsg + "\",\"users\":[{\"index\":0,\"errors\":[\"No two loginMethods can have isPrimary as true.\"]}]}"); + } + } + // Can't import less than 1 user at a time + { + try { + JsonObject request = generateUsersJson(0); + HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", + "http://localhost:3567/bulk-import/users", + request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); + fail("The API should have thrown an error"); + } catch (io.supertokens.test.httpRequest.HttpResponseException e) { + String responseString = getResponseMessageFromError(e.getMessage()); + assertEquals(400, e.statusCode); + assertEquals(responseString, "{\"error\":\"You need to add at least one user.\"}"); + } + } + // Can't import more than 10000 users at a time + { + try { + JsonObject request = generateUsersJson(10001); + HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", + "http://localhost:3567/bulk-import/users", + request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); + fail("The API should have thrown an error"); + } catch (io.supertokens.test.httpRequest.HttpResponseException e) { + String responseString = getResponseMessageFromError(e.getMessage()); + assertEquals(400, e.statusCode); + assertEquals(responseString, "{\"error\":\"You can only add 10000 users at a time.\"}"); + } + } + + process.kill(); + Assert.assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STOPPED)); + } + + @Test + public void shouldReturn200Response() throws Exception { + String[] args = { "../" }; + + TestingProcessManager.TestingProcess process = TestingProcessManager.start(args); + assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); + + if (StorageLayer.getStorage(process.getProcess()).getType() != STORAGE_TYPE.SQL) { + return; + } + + // Create user roles before inserting bulk users + { + UserRoles.createNewRoleOrModifyItsPermissions(process.getProcess(), "role1", null); + UserRoles.createNewRoleOrModifyItsPermissions(process.getProcess(), "role2", null); + } + + JsonObject request = generateUsersJson(10000); + JsonObject response = HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", + "http://localhost:3567/bulk-import/users", + request, 1000, 10000, null, Utils.getCdiVersionStringLatestForTests(), null); + assertEquals("OK", response.get("status").getAsString()); + + process.kill(); + Assert.assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STOPPED)); + } + + @Test + public void shouldNormaliseFields() throws Exception { + String[] args = { "../" }; + + TestingProcessManager.TestingProcess process = TestingProcessManager.start(args); + assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); + + if (StorageLayer.getStorage(process.getProcess()).getType() != STORAGE_TYPE.SQL) { + return; + } + + // Create user roles before inserting bulk users + { + UserRoles.createNewRoleOrModifyItsPermissions(process.getProcess(), "role1", null); + UserRoles.createNewRoleOrModifyItsPermissions(process.getProcess(), "role2", null); + } + + JsonObject request = generateUsersJson(1); + JsonObject response = HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", + "http://localhost:3567/bulk-import/users", + request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); + assertEquals("OK", response.get("status").getAsString()); + + JsonObject getResponse = HttpRequestForTesting.sendGETRequest(process.getProcess(), "", + "http://localhost:3567/bulk-import/users", + new HashMap<>(), 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); + + assertEquals("OK", getResponse.get("status").getAsString()); + JsonArray bulkImportUsers = getResponse.get("users").getAsJsonArray(); + assertEquals(1, bulkImportUsers.size()); + + JsonObject bulkImportUserJson = bulkImportUsers.get(0).getAsJsonObject(); + + // Test if default values were set in totpDevices + JsonArray totpDevices = bulkImportUserJson.getAsJsonArray("totpDevices"); + for (int i = 0; i < totpDevices.size(); i++) { + JsonObject totpDevice = totpDevices.get(i).getAsJsonObject(); + assertEquals(30, totpDevice.get("period").getAsInt()); + assertEquals(1, totpDevice.get("skew").getAsInt()); + } + + JsonArray loginMethods = bulkImportUserJson.getAsJsonArray("loginMethods"); + for (int i = 0; i < loginMethods.size(); i++) { + JsonObject loginMethod = loginMethods.get(i).getAsJsonObject(); + if (loginMethod.has("email")) { + assertEquals("johndoe+0@gmail.com", loginMethod.get("email").getAsString()); + } + if (loginMethod.has("phoneNumber")) { + assertEquals("+919999999999", loginMethod.get("phoneNumber").getAsString()); + } + if (loginMethod.has("hashingAlgorithm")) { + assertEquals("ARGON2", loginMethod.get("hashingAlgorithm").getAsString()); + } + } + + process.kill(); + Assert.assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STOPPED)); + } + + public static JsonObject generateUsersJson(int numberOfUsers) { + JsonObject userJsonObject = new JsonObject(); + JsonParser parser = new JsonParser(); + + JsonArray usersArray = new JsonArray(); + for (int i = 0; i < numberOfUsers; i++) { + JsonObject user = new JsonObject(); + + user.addProperty("externalUserId", UUID.randomUUID().toString()); + user.add("userMetadata", parser.parse("{\"key1\":\"value1\",\"key2\":{\"key3\":\"value3\"}}")); + user.add("userRoles", parser.parse("[{\"role\":\"role1\", \"tenantIds\": [\"public\"]},{\"role\":\"role2\", \"tenantIds\": [\"public\"]}]")); + user.add("totpDevices", parser.parse("[{\"secretKey\":\"secretKey\",\"deviceName\":\"deviceName\"}]")); + + JsonArray tenanatIds = parser.parse("[\"public\"]").getAsJsonArray(); + String email = " johndoe+" + i + "@gmail.com "; + + JsonArray loginMethodsArray = new JsonArray(); + loginMethodsArray.add(createEmailLoginMethod(email, tenanatIds)); + loginMethodsArray.add(createThirdPartyLoginMethod(email, tenanatIds)); + loginMethodsArray.add(createPasswordlessLoginMethod(email, tenanatIds)); + user.add("loginMethods", loginMethodsArray); + + usersArray.add(user); + } + + userJsonObject.add("users", usersArray); + return userJsonObject; + } + + private static JsonObject createEmailLoginMethod(String email, JsonArray tenantIds) { + JsonObject loginMethod = new JsonObject(); + loginMethod.add("tenantIds", tenantIds); + loginMethod.addProperty("email", email); + loginMethod.addProperty("recipeId", "emailpassword"); + loginMethod.addProperty("passwordHash", "$argon2d$v=19$m=12,t=3,p=1$aGI4enNvMmd0Zm0wMDAwMA$r6p7qbr6HD+8CD7sBi4HVw"); + loginMethod.addProperty("hashingAlgorithm", "argon2"); + loginMethod.addProperty("isVerified", true); + loginMethod.addProperty("isPrimary", true); + loginMethod.addProperty("timeJoinedInMSSinceEpoch", 0); + return loginMethod; + } + + private static JsonObject createThirdPartyLoginMethod(String email, JsonArray tenantIds) { + JsonObject loginMethod = new JsonObject(); + loginMethod.add("tenantIds", tenantIds); + loginMethod.addProperty("recipeId", "thirdparty"); + loginMethod.addProperty("email", email); + loginMethod.addProperty("thirdPartyId", "google"); + loginMethod.addProperty("thirdPartyUserId", "112618388912586834161"); + loginMethod.addProperty("isVerified", true); + loginMethod.addProperty("isPrimary", false); + loginMethod.addProperty("timeJoinedInMSSinceEpoch", 0); + return loginMethod; + } + + private static JsonObject createPasswordlessLoginMethod(String email, JsonArray tenantIds) { + JsonObject loginMethod = new JsonObject(); + loginMethod.add("tenantIds", tenantIds); + loginMethod.addProperty("email", email); + loginMethod.addProperty("recipeId", "passwordless"); + loginMethod.addProperty("phoneNumber", "+91-9999999999"); + loginMethod.addProperty("isVerified", true); + loginMethod.addProperty("isPrimary", false); + loginMethod.addProperty("timeJoinedInMSSinceEpoch", 0); + return loginMethod; + } + + private void createTenants(Main main) + throws StorageQueryException, TenantOrAppNotFoundException, InvalidProviderConfigException, + FeatureNotEnabledException, IOException, InvalidConfigException, + CannotModifyBaseConfigException, BadPermissionException { + // User pool 1 - (null, null, null), (null, null, t1) + // User pool 2 - (null, null, t2) + + { // tenant 1 + TenantIdentifier tenantIdentifier = new TenantIdentifier(null, null, "t1"); + + Multitenancy.addNewOrUpdateAppOrTenant( + main, + new TenantIdentifier(null, null, null), + new TenantConfig( + tenantIdentifier, + new EmailPasswordConfig(true), + new ThirdPartyConfig(true, null), + new PasswordlessConfig(true), + null, null, new JsonObject() + ) + ); + } + { // tenant 2 + JsonObject config = new JsonObject(); + TenantIdentifier tenantIdentifier = new TenantIdentifier(null, null, "t2"); + + StorageLayer.getStorage(new TenantIdentifier(null, null, null), main) + .modifyConfigToAddANewUserPoolForTesting(config, 1); + + Multitenancy.addNewOrUpdateAppOrTenant( + main, + new TenantIdentifier(null, null, null), + new TenantConfig( + tenantIdentifier, + new EmailPasswordConfig(true), + new ThirdPartyConfig(true, null), + new PasswordlessConfig(true), + null, null, config + ) + ); + } + } +} diff --git a/src/test/java/io/supertokens/test/bulkimport/apis/DeleteBulkImportUsersTest.java b/src/test/java/io/supertokens/test/bulkimport/apis/DeleteBulkImportUsersTest.java new file mode 100644 index 000000000..db2fe1707 --- /dev/null +++ b/src/test/java/io/supertokens/test/bulkimport/apis/DeleteBulkImportUsersTest.java @@ -0,0 +1,173 @@ +/* + * Copyright (c) 2024, VRAI Labs and/or its affiliates. All rights reserved. + * + * This software is licensed under the Apache License, Version 2.0 (the + * "License") as published by the Apache Software Foundation. + * + * You may not use this file except in compliance with the License. You may + * obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package io.supertokens.test.bulkimport.apis; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +import java.util.List; + +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TestRule; + +import com.google.gson.JsonArray; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; +import com.google.gson.JsonPrimitive; + +import io.supertokens.ProcessState; +import io.supertokens.bulkimport.BulkImport; +import io.supertokens.pluginInterface.STORAGE_TYPE; +import io.supertokens.pluginInterface.bulkimport.BulkImportStorage; +import io.supertokens.pluginInterface.bulkimport.BulkImportUser; +import io.supertokens.pluginInterface.multitenancy.AppIdentifier; +import io.supertokens.storageLayer.StorageLayer; +import io.supertokens.test.TestingProcessManager; +import io.supertokens.test.Utils; +import io.supertokens.test.httpRequest.HttpRequestForTesting; + +import static io.supertokens.test.bulkimport.BulkImportTestUtils.generateBulkImportUser; + +public class DeleteBulkImportUsersTest { + @Rule + public TestRule watchman = Utils.getOnFailure(); + + @AfterClass + public static void afterTesting() { + Utils.afterTesting(); + } + + @Before + public void beforeEach() { + Utils.reset(); + } + + @Test + public void shouldReturn400Error() throws Exception { + String[] args = { "../" }; + + TestingProcessManager.TestingProcess process = TestingProcessManager.start(args); + assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); + + if (StorageLayer.getStorage(process.getProcess()).getType() != STORAGE_TYPE.SQL) { + return; + } + + { + try { + JsonObject request = new JsonObject(); + HttpRequestForTesting.sendJsonDELETERequest(process.getProcess(), "", + "http://localhost:3567/bulk-import/users", + request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); + } catch (io.supertokens.test.httpRequest.HttpResponseException e) { + assertEquals(400, e.statusCode); + assertEquals("Http error. Status Code: 400. Message: Field name 'ids' is invalid in JSON input", e.getMessage()); + } + } + { + try { + JsonObject request = new JsonParser().parse("{\"ids\":[]}").getAsJsonObject(); + HttpRequestForTesting.sendJsonDELETERequest(process.getProcess(), "", + "http://localhost:3567/bulk-import/users", + request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); + } catch (io.supertokens.test.httpRequest.HttpResponseException e) { + assertEquals(400, e.statusCode); + assertEquals("Http error. Status Code: 400. Message: Field name 'ids' cannot be an empty array", e.getMessage()); + } + } + { + try { + JsonObject request = new JsonParser().parse("{\"ids\":[\"\"]}").getAsJsonObject(); + HttpRequestForTesting.sendJsonDELETERequest(process.getProcess(), "", + "http://localhost:3567/bulk-import/users", + request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); + } catch (io.supertokens.test.httpRequest.HttpResponseException e) { + assertEquals(400, e.statusCode); + assertEquals("Http error. Status Code: 400. Message: Field name 'ids' cannot contain an empty string", e.getMessage()); + } + } + { + try { + // Create a string array of 500 uuids + JsonObject request = new JsonObject(); + JsonArray ids = new JsonArray(); + for (int i = 0; i < 501; i++) { + ids.add(new JsonPrimitive(io.supertokens.utils.Utils.getUUID())); + } + request.add("ids", ids); + + HttpRequestForTesting.sendJsonDELETERequest(process.getProcess(), "", + "http://localhost:3567/bulk-import/users", + request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); + } catch (io.supertokens.test.httpRequest.HttpResponseException e) { + assertEquals(400, e.statusCode); + assertEquals("Http error. Status Code: 400. Message: Field name 'ids' cannot contain more than 500 elements", e.getMessage()); + } + } + + process.kill(); + Assert.assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STOPPED)); + } + + @Test + public void shouldReturn200Response() throws Exception { + String[] args = { "../" }; + + TestingProcessManager.TestingProcess process = TestingProcessManager.start(args); + assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); + + if (StorageLayer.getStorage(process.getProcess()).getType() != STORAGE_TYPE.SQL) { + return; + } + + BulkImportStorage storage = (BulkImportStorage) StorageLayer.getStorage(process.main); + AppIdentifier appIdentifier = new AppIdentifier(null, null); + + // Insert users + List users = generateBulkImportUser(5); + BulkImport.addUsers(appIdentifier, storage, users); + + String invalidId = io.supertokens.utils.Utils.getUUID(); + JsonObject request = new JsonObject(); + JsonArray validIds = new JsonArray(); + for (BulkImportUser user : users) { + validIds.add(new JsonPrimitive(user.id)); + } + validIds.add(new JsonPrimitive(invalidId)); + + request.add("ids", validIds); + + JsonObject response = HttpRequestForTesting.sendJsonDELETERequest(process.getProcess(), "", + "http://localhost:3567/bulk-import/users", + request, 1000000, 1000000, null, Utils.getCdiVersionStringLatestForTests(), null); + + response.get("deletedIds").getAsJsonArray().forEach(id -> { + assertTrue(validIds.contains(id)); + }); + + assertEquals(invalidId, response.get("invalidIds").getAsJsonArray().get(0).getAsString()); + + process.kill(); + Assert.assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STOPPED)); + } + +} diff --git a/src/test/java/io/supertokens/test/bulkimport/apis/GetBulkImportUsersTest.java b/src/test/java/io/supertokens/test/bulkimport/apis/GetBulkImportUsersTest.java new file mode 100644 index 000000000..181bcd336 --- /dev/null +++ b/src/test/java/io/supertokens/test/bulkimport/apis/GetBulkImportUsersTest.java @@ -0,0 +1,160 @@ +/* + * Copyright (c) 2024, VRAI Labs and/or its affiliates. All rights reserved. + * + * This software is licensed under the Apache License, Version 2.0 (the + * "License") as published by the Apache Software Foundation. + * + * You may not use this file except in compliance with the License. You may + * obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package io.supertokens.test.bulkimport.apis; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.fail; + +import java.util.HashMap; +import java.util.Map; + +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TestRule; + +import com.google.gson.JsonArray; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; + +import io.supertokens.ProcessState; +import io.supertokens.pluginInterface.STORAGE_TYPE; +import io.supertokens.pluginInterface.bulkimport.BulkImportUser; +import io.supertokens.storageLayer.StorageLayer; +import io.supertokens.test.TestingProcessManager; +import io.supertokens.test.Utils; +import io.supertokens.test.httpRequest.HttpRequestForTesting; + +public class GetBulkImportUsersTest { + @Rule + public TestRule watchman = Utils.getOnFailure(); + + @AfterClass + public static void afterTesting() { + Utils.afterTesting(); + } + + @Before + public void beforeEach() { + Utils.reset(); + } + + @Test + public void shouldReturn400Error() throws Exception { + String[] args = { "../" }; + + TestingProcessManager.TestingProcess process = TestingProcessManager.start(args); + assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); + + if (StorageLayer.getStorage(process.getProcess()).getType() != STORAGE_TYPE.SQL) { + return; + } + + try { + Map params = new HashMap<>(); + params.put("status", "INVALID_STATUS"); + HttpRequestForTesting.sendGETRequest(process.getProcess(), "", + "http://localhost:3567/bulk-import/users", + params, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); + fail("The API should have thrown an error"); + } catch (io.supertokens.test.httpRequest.HttpResponseException e) { + assertEquals(400, e.statusCode); + assertEquals( + "Http error. Status Code: 400. Message: Invalid value for status. Pass one of NEW, PROCESSING, or FAILED!", + e.getMessage()); + } + + try { + Map params = new HashMap<>(); + params.put("limit", "0"); + HttpRequestForTesting.sendGETRequest(process.getProcess(), "", + "http://localhost:3567/bulk-import/users", + params, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); + fail("The API should have thrown an error"); + } catch (io.supertokens.test.httpRequest.HttpResponseException e) { + assertEquals(400, e.statusCode); + assertEquals("Http error. Status Code: 400. Message: limit must a positive integer with min value 1", + e.getMessage()); + } + + try { + Map params = new HashMap<>(); + params.put("limit", "501"); + HttpRequestForTesting.sendGETRequest(process.getProcess(), "", + "http://localhost:3567/bulk-import/users", + params, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); + fail("The API should have thrown an error"); + } catch (io.supertokens.test.httpRequest.HttpResponseException e) { + assertEquals(400, e.statusCode); + assertEquals("Http error. Status Code: 400. Message: Max limit allowed is 500", e.getMessage()); + } + + try { + Map params = new HashMap<>(); + params.put("paginationToken", "invalid_token"); + HttpRequestForTesting.sendGETRequest(process.getProcess(), "", + "http://localhost:3567/bulk-import/users", + params, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); + fail("The API should have thrown an error"); + } catch (io.supertokens.test.httpRequest.HttpResponseException e) { + assertEquals(400, e.statusCode); + assertEquals("Http error. Status Code: 400. Message: invalid pagination token", e.getMessage()); + } + + process.kill(); + Assert.assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STOPPED)); + } + + @Test + public void shouldReturn200Response() throws Exception { + String[] args = { "../" }; + + TestingProcessManager.TestingProcess process = TestingProcessManager.start(args); + assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); + + if (StorageLayer.getStorage(process.getProcess()).getType() != STORAGE_TYPE.SQL) { + return; + } + + // Create a bulk import user to test the GET API + String rawData = "{\"users\":[{\"loginMethods\":[{\"recipeId\":\"passwordless\",\"email\":\"johndoe@gmail.com\"}]}]}"; + { + JsonObject request = new JsonParser().parse(rawData).getAsJsonObject(); + JsonObject res = HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", + "http://localhost:3567/bulk-import/users", + request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); + assert res.get("status").getAsString().equals("OK"); + } + + Map params = new HashMap<>(); + JsonObject response = HttpRequestForTesting.sendGETRequest(process.getProcess(), "", + "http://localhost:3567/bulk-import/users", + params, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); + assertEquals("OK", response.get("status").getAsString()); + JsonArray bulkImportUsers = response.get("users").getAsJsonArray(); + assertEquals(1, bulkImportUsers.size()); + JsonObject bulkImportUserJson = bulkImportUsers.get(0).getAsJsonObject(); + bulkImportUserJson.get("status").getAsString().equals("NEW"); + BulkImportUser.forTesting_fromJson(bulkImportUserJson).toRawDataForDbStorage().equals(rawData); + + process.kill(); + Assert.assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STOPPED)); + } +} From f86211ab7a9d9d6eaaf0490c583a598f863ab6d2 Mon Sep 17 00:00:00 2001 From: Ankit Tiwari Date: Wed, 20 Mar 2024 13:03:13 +0530 Subject: [PATCH 02/21] chore: update pull request template --- .github/PULL_REQUEST_TEMPLATE.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 94c0f5211..aa8544d05 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -35,6 +35,8 @@ highlighting the necessary changes) latest branch (`git branch --all`) whose `X.Y` is greater than the latest released tag. - If no such branch exists, then create one from the latest released branch. - [ ] If added a foreign key constraint on `app_id_to_user_id` table, make sure to delete from this table when deleting the user as well if `deleteUserIdMappingToo` is false. +- [ ] If added a new recipe, then make sure to update the bulk import API to include the new recipe. + ## Remaining TODOs for this PR - [ ] Item1 From 05dc4a198b36fd0e669f0c0ae362b8b15d51c681 Mon Sep 17 00:00:00 2001 From: Ankit Tiwari Date: Wed, 20 Mar 2024 14:15:38 +0530 Subject: [PATCH 03/21] fix: Use the correct tenant config to create the proxy storage --- .../bulkimport/ProcessBulkImportUsers.java | 48 +++++++++++++------ .../ProcessBulkImportUsersCronJobTest.java | 1 - 2 files changed, 33 insertions(+), 16 deletions(-) diff --git a/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkImportUsers.java b/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkImportUsers.java index 730fd43b4..aa6ff8825 100644 --- a/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkImportUsers.java +++ b/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkImportUsers.java @@ -24,6 +24,8 @@ import java.util.List; import java.util.Map; +import com.google.gson.JsonObject; + import io.supertokens.Main; import io.supertokens.ResourceDistributor; import io.supertokens.authRecipe.AuthRecipe; @@ -62,6 +64,7 @@ import io.supertokens.pluginInterface.exceptions.StorageQueryException; import io.supertokens.pluginInterface.exceptions.StorageTransactionLogicException; import io.supertokens.pluginInterface.multitenancy.AppIdentifier; +import io.supertokens.pluginInterface.multitenancy.TenantConfig; import io.supertokens.pluginInterface.multitenancy.TenantIdentifier; import io.supertokens.pluginInterface.multitenancy.exceptions.TenantOrAppNotFoundException; import io.supertokens.pluginInterface.passwordless.exception.DuplicatePhoneNumberException; @@ -114,7 +117,7 @@ protected void doTaskPerApp(AppIdentifier app) BulkImport.PROCESS_USERS_BATCH_SIZE); for (BulkImportUser user : users) { - processUser(appIdentifier, user); + processUser(appIdentifier, user, bulkImportSQLStorage); } closeAllProxyStorages(); @@ -147,12 +150,23 @@ private Storage getProxyStorage(TenantIdentifier tenantIdentifier) return userPoolToStorageMap.get(userPoolId); } - SQLStorage bulkImportProxyStorage = (SQLStorage) StorageLayer.getNewBulkImportProxyStorageInstance(main, - Config.getBaseConfigAsJsonObject(main), tenantIdentifier, true); + TenantConfig[] allTenants = Multitenancy.getAllTenants(main); + + Map normalisedConfigs = Config.getNormalisedConfigsForAllTenants( + allTenants, + Config.getBaseConfigAsJsonObject(main)); + + for (ResourceDistributor.KeyClass key : normalisedConfigs.keySet()) { + if (key.getTenantIdentifier().equals(tenantIdentifier)) { + SQLStorage bulkImportProxyStorage = (SQLStorage) StorageLayer.getNewBulkImportProxyStorageInstance(main, + normalisedConfigs.get(key), tenantIdentifier, true); - userPoolToStorageMap.put(userPoolId, bulkImportProxyStorage); - bulkImportProxyStorage.initStorage(true); - return bulkImportProxyStorage; + userPoolToStorageMap.put(userPoolId, bulkImportProxyStorage); + bulkImportProxyStorage.initStorage(true); + return bulkImportProxyStorage; + } + } + throw new TenantOrAppNotFoundException(tenantIdentifier); } public Storage[] getAllProxyStoragesForApp(Main main, AppIdentifier appIdentifier) @@ -161,9 +175,10 @@ public Storage[] getAllProxyStoragesForApp(Main main, AppIdentifier appIdentifie Map resources = main .getResourceDistributor() - .getAllResourcesWithResourceKey(RESOURCE_KEY); + .getAllResourcesWithResourceKey(StorageLayer.RESOURCE_KEY); for (ResourceDistributor.KeyClass key : resources.keySet()) { if (key.getTenantIdentifier().toAppIdentifier().equals(appIdentifier)) { + System.out.println("Adding storage for tenant: " + key.getTenantIdentifier().getTenantId()); allProxyStorages.add(getProxyStorage(key.getTenantIdentifier())); } } @@ -176,7 +191,7 @@ private void closeAllProxyStorages() { } } - private void processUser(AppIdentifier appIdentifier, BulkImportUser user) + private void processUser(AppIdentifier appIdentifier, BulkImportUser user, BulkImportSQLStorage baseTenantStorage) throws TenantOrAppNotFoundException, StorageQueryException, InvalidConfigException, IOException, DbInitException { // Since all the tenants of a user must share the storage, we will just use the @@ -202,8 +217,12 @@ private void processUser(AppIdentifier appIdentifier, BulkImportUser user) createUserMetadata(appIdentifier, bulkImportProxyStorage, user, primaryLM); createUserRoles(main, appIdentifier, bulkImportProxyStorage, user); - ((BulkImportSQLStorage) bulkImportProxyStorage).deleteBulkImportUser_Transaction(appIdentifier, con, - user.id); + // NOTE: We need to use the baseTenantStorage as bulkImportProxyStorage could have a different storage than the baseTenantStorage + baseTenantStorage.startTransaction(con2 -> { + baseTenantStorage.deleteBulkImportUser_Transaction(appIdentifier, con2, + user.id); + return null; + }); // We need to commit the transaction manually because we have overridden that in the proxy storage try { @@ -217,12 +236,11 @@ private void processUser(AppIdentifier appIdentifier, BulkImportUser user) return null; }); } catch (StorageTransactionLogicException e) { - handleProcessUserExceptions(appIdentifier, user, (BulkImportSQLStorage) bulkImportProxyStorage, e); + handleProcessUserExceptions(appIdentifier, user, e, baseTenantStorage); } } - private void handleProcessUserExceptions(AppIdentifier appIdentifier, BulkImportUser user, - BulkImportSQLStorage bulkImportSQLStorage, Exception e) + private void handleProcessUserExceptions(AppIdentifier appIdentifier, BulkImportUser user, Exception e, BulkImportSQLStorage baseTenantStorage) throws StorageQueryException { // Java doesn't allow us to reassign local variables inside a lambda expression @@ -237,8 +255,8 @@ private void handleProcessUserExceptions(AppIdentifier appIdentifier, BulkImport String[] userId = { user.id }; try { - bulkImportSQLStorage.startTransaction(con -> { - bulkImportSQLStorage.updateBulkImportUserStatus_Transaction(appIdentifier, con, userId, + baseTenantStorage.startTransaction(con -> { + baseTenantStorage.updateBulkImportUserStatus_Transaction(appIdentifier, con, userId, BULK_IMPORT_USER_STATUS.FAILED, errorMessage[0]); // We need to commit the transaction manually because we have overridden that in the proxy storage diff --git a/src/test/java/io/supertokens/test/bulkimport/ProcessBulkImportUsersCronJobTest.java b/src/test/java/io/supertokens/test/bulkimport/ProcessBulkImportUsersCronJobTest.java index 78bff7e32..00c73bfb6 100644 --- a/src/test/java/io/supertokens/test/bulkimport/ProcessBulkImportUsersCronJobTest.java +++ b/src/test/java/io/supertokens/test/bulkimport/ProcessBulkImportUsersCronJobTest.java @@ -110,7 +110,6 @@ public void shouldProcessBulkImportUsers() throws Exception { BulkImportUser bulkImportUser = users.get(0); - // Thread.sleep(600000); Thread.sleep(6000); List usersAfterProcessing = storage.getBulkImportUsers(appIdentifier, null, null, From 5b4658af1f1479344817ff7ff2687bf8b8b238fb Mon Sep 17 00:00:00 2001 From: Ankit Tiwari Date: Thu, 21 Mar 2024 16:17:37 +0530 Subject: [PATCH 04/21] fix: PR changes --- .../bulkimport/BulkImportUserUtils.java | 83 +++++++++++------- .../io/supertokens/cronjobs/CronTaskTest.java | 10 +++ .../bulkimport/ProcessBulkImportUsers.java | 84 ++++++++----------- .../java/io/supertokens/inmemorydb/Start.java | 22 ++++- .../api/bulkimport/BulkImportAPI.java | 7 +- .../test/bulkimport/BulkImportTest.java | 22 +++-- .../ProcessBulkImportUsersCronJobTest.java | 2 + .../apis/AddBulkImportUsersTest.java | 56 +++++++++++-- 8 files changed, 192 insertions(+), 94 deletions(-) diff --git a/src/main/java/io/supertokens/bulkimport/BulkImportUserUtils.java b/src/main/java/io/supertokens/bulkimport/BulkImportUserUtils.java index 834ed0e4a..f884680d4 100644 --- a/src/main/java/io/supertokens/bulkimport/BulkImportUserUtils.java +++ b/src/main/java/io/supertokens/bulkimport/BulkImportUserUtils.java @@ -52,8 +52,16 @@ import static io.supertokens.utils.JsonValidatorUtils.validateJsonFieldType; public class BulkImportUserUtils { - public static BulkImportUser createBulkImportUserFromJSON(Main main, AppIdentifier appIdentifier, - JsonObject userData, String id, String[] allUserRoles, Set allExternalUserIds) + private String[] allUserRoles; + private Set allExternalUserIds; + + public BulkImportUserUtils(String[] allUserRoles) { + this.allUserRoles = allUserRoles; + this.allExternalUserIds = new HashSet<>(); + } + + public BulkImportUser createBulkImportUserFromJSON(Main main, AppIdentifier appIdentifier, JsonObject userData, + String id) throws InvalidBulkImportDataException, StorageQueryException, TenantOrAppNotFoundException { List errors = new ArrayList<>(); @@ -62,11 +70,11 @@ public static BulkImportUser createBulkImportUserFromJSON(Main main, AppIdentifi errors, "."); JsonObject userMetadata = parseAndValidateFieldType(userData, "userMetadata", ValueType.OBJECT, false, JsonObject.class, errors, "."); - List userRoles = getParsedUserRoles(main, appIdentifier, userData, allUserRoles, errors); - List totpDevices = getParsedTotpDevices(userData, errors); + List userRoles = getParsedUserRoles(main, appIdentifier, userData, errors); + List totpDevices = getParsedTotpDevices(main, appIdentifier, userData, errors); List loginMethods = getParsedLoginMethods(main, appIdentifier, userData, errors); - externalUserId = validateAndNormaliseExternalUserId(externalUserId, allExternalUserIds, errors); + externalUserId = validateAndNormaliseExternalUserId(externalUserId, errors); validateTenantIdsForRoleAndLoginMethods(main, appIdentifier, userRoles, loginMethods, errors); @@ -76,8 +84,8 @@ public static BulkImportUser createBulkImportUserFromJSON(Main main, AppIdentifi return new BulkImportUser(id, externalUserId, userMetadata, userRoles, totpDevices, loginMethods); } - private static List getParsedUserRoles(Main main, AppIdentifier appIdentifier, JsonObject userData, - String[] allUserRoles, List errors) throws StorageQueryException, TenantOrAppNotFoundException { + private List getParsedUserRoles(Main main, AppIdentifier appIdentifier, JsonObject userData, + List errors) throws StorageQueryException, TenantOrAppNotFoundException { JsonArray jsonUserRoles = parseAndValidateFieldType(userData, "userRoles", ValueType.ARRAY_OF_OBJECT, false, JsonArray.class, errors, "."); @@ -95,7 +103,7 @@ private static List getParsedUserRoles(Main main, AppIdentifier appIde JsonArray jsonTenantIds = parseAndValidateFieldType(jsonUserRole, "tenantIds", ValueType.ARRAY_OF_STRING, true, JsonArray.class, errors, " for a user role."); - role = validateAndNormaliseUserRole(role, allUserRoles, errors); + role = validateAndNormaliseUserRole(role, errors); List normalisedTenantIds = validateAndNormaliseTenantIds(main, appIdentifier, jsonTenantIds, errors, " for a user role."); @@ -106,7 +114,8 @@ private static List getParsedUserRoles(Main main, AppIdentifier appIde return userRoles; } - private static List getParsedTotpDevices(JsonObject userData, List errors) { + private List getParsedTotpDevices(Main main, AppIdentifier appIdentifier, JsonObject userData, + List errors) throws StorageQueryException, TenantOrAppNotFoundException { JsonArray jsonTotpDevices = parseAndValidateFieldType(userData, "totpDevices", ValueType.ARRAY_OF_OBJECT, false, JsonArray.class, errors, "."); @@ -114,6 +123,12 @@ private static List getParsedTotpDevices(JsonObject userData, List t == EE_FEATURES.MFA)) { + errors.add("MFA must be enabled to import totp devices."); + return null; + } + List totpDevices = new ArrayList<>(); for (JsonElement jsonTotpDeviceEl : jsonTotpDevices) { JsonObject jsonTotpDevice = jsonTotpDeviceEl.getAsJsonObject(); @@ -139,7 +154,7 @@ private static List getParsedTotpDevices(JsonObject userData, List getParsedLoginMethods(Main main, AppIdentifier appIdentifier, JsonObject userData, + private List getParsedLoginMethods(Main main, AppIdentifier appIdentifier, JsonObject userData, List errors) throws StorageQueryException, TenantOrAppNotFoundException { JsonArray jsonLoginMethods = parseAndValidateFieldType(userData, "loginMethods", ValueType.ARRAY_OF_OBJECT, @@ -154,6 +169,13 @@ private static List getParsedLoginMethods(Main main, AppIdentifier return new ArrayList<>(); } + if (jsonLoginMethods.size() > 1) { + if (Arrays.stream(FeatureFlag.getInstance(main, appIdentifier).getEnabledFeatures()) + .noneMatch(t -> t == EE_FEATURES.ACCOUNT_LINKING || t == EE_FEATURES.MFA)) { + errors.add("Account linking or MFA must be enabled to import multiple loginMethods."); + } + } + validateAndNormaliseIsPrimaryField(jsonLoginMethods, errors); List loginMethods = new ArrayList<>(); @@ -232,8 +254,7 @@ private static List getParsedLoginMethods(Main main, AppIdentifier return loginMethods; } - private static String validateAndNormaliseExternalUserId(String externalUserId, Set allExternalUserIds, - List errors) { + private String validateAndNormaliseExternalUserId(String externalUserId, List errors) { if (externalUserId == null) { return null; } @@ -250,7 +271,7 @@ private static String validateAndNormaliseExternalUserId(String externalUserId, return externalUserId.trim(); } - private static String validateAndNormaliseUserRole(String role, String[] allUserRoles, List errors) { + private String validateAndNormaliseUserRole(String role, List errors) { if (role.length() > 255) { errors.add("role " + role + " is too long. Max length is 255."); } @@ -265,7 +286,7 @@ private static String validateAndNormaliseUserRole(String role, String[] allUser return normalisedRole; } - private static String validateAndNormaliseTotpSecretKey(String secretKey, List errors) { + private String validateAndNormaliseTotpSecretKey(String secretKey, List errors) { if (secretKey == null) { return null; } @@ -278,7 +299,7 @@ private static String validateAndNormaliseTotpSecretKey(String secretKey, List errors) { + private Integer validateAndNormaliseTotpPeriod(Integer period, List errors) { // We default to 30 if period is null if (period == null) { return 30; @@ -291,7 +312,7 @@ private static Integer validateAndNormaliseTotpPeriod(Integer period, List errors) { + private Integer validateAndNormaliseTotpSkew(Integer skew, List errors) { // We default to 1 if skew is null if (skew == null) { return 1; @@ -304,7 +325,7 @@ private static Integer validateAndNormaliseTotpSkew(Integer skew, List e return skew; } - private static String validateAndNormaliseTotpDeviceName(String deviceName, List errors) { + private String validateAndNormaliseTotpDeviceName(String deviceName, List errors) { if (deviceName == null) { return null; } @@ -317,7 +338,7 @@ private static String validateAndNormaliseTotpDeviceName(String deviceName, List return deviceName.trim(); } - private static void validateAndNormaliseIsPrimaryField(JsonArray jsonLoginMethods, List errors) { + private void validateAndNormaliseIsPrimaryField(JsonArray jsonLoginMethods, List errors) { // We are validating that only one loginMethod has isPrimary as true boolean hasPrimaryLoginMethod = false; for (JsonElement jsonLoginMethod : jsonLoginMethods) { @@ -333,7 +354,7 @@ private static void validateAndNormaliseIsPrimaryField(JsonArray jsonLoginMethod } } - private static String validateAndNormaliseRecipeId(String recipeId, List errors) { + private String validateAndNormaliseRecipeId(String recipeId, List errors) { if (recipeId == null) { return null; } @@ -346,7 +367,7 @@ private static String validateAndNormaliseRecipeId(String recipeId, List return recipeId; } - private static List validateAndNormaliseTenantIds(Main main, AppIdentifier appIdentifier, + private List validateAndNormaliseTenantIds(Main main, AppIdentifier appIdentifier, JsonArray tenantIds, List errors, String errorSuffix) throws StorageQueryException, TenantOrAppNotFoundException { if (tenantIds == null) { @@ -366,7 +387,7 @@ private static List validateAndNormaliseTenantIds(Main main, AppIdentifi return normalisedTenantIds; } - private static String validateAndNormaliseTenantId(Main main, AppIdentifier appIdentifier, String tenantId, + private String validateAndNormaliseTenantId(Main main, AppIdentifier appIdentifier, String tenantId, List errors, String errorSuffix) throws StorageQueryException, TenantOrAppNotFoundException { if (tenantId == null || tenantId.equals(TenantIdentifier.DEFAULT_TENANT_ID)) { @@ -393,17 +414,17 @@ private static String validateAndNormaliseTenantId(Main main, AppIdentifier appI return normalisedTenantId; } - private static Boolean validateAndNormaliseIsPrimary(Boolean isPrimary) { + private Boolean validateAndNormaliseIsPrimary(Boolean isPrimary) { // We set the default value as false return isPrimary == null ? false : isPrimary; } - private static Boolean validateAndNormaliseIsVerified(Boolean isVerified) { + private Boolean validateAndNormaliseIsVerified(Boolean isVerified) { // We set the default value as false return isVerified == null ? false : isVerified; } - private static long validateAndNormaliseTimeJoined(Long timeJoined, List errors) { + private long validateAndNormaliseTimeJoined(Long timeJoined, List errors) { // We default timeJoined to currentTime if it is null if (timeJoined == null) { return System.currentTimeMillis(); @@ -420,7 +441,7 @@ private static long validateAndNormaliseTimeJoined(Long timeJoined, List return timeJoined.longValue(); } - private static String validateAndNormaliseEmail(String email, List errors) { + private String validateAndNormaliseEmail(String email, List errors) { if (email == null) { return null; } @@ -433,7 +454,7 @@ private static String validateAndNormaliseEmail(String email, List error return Utils.normaliseEmail(email); } - private static CoreConfig.PASSWORD_HASHING_ALG validateAndNormaliseHashingAlgorithm(String hashingAlgorithm, + private CoreConfig.PASSWORD_HASHING_ALG validateAndNormaliseHashingAlgorithm(String hashingAlgorithm, List errors) { if (hashingAlgorithm == null) { return null; @@ -449,7 +470,7 @@ private static CoreConfig.PASSWORD_HASHING_ALG validateAndNormaliseHashingAlgori } } - private static String validateAndNormalisePasswordHash(Main main, AppIdentifier appIdentifier, + private String validateAndNormalisePasswordHash(Main main, AppIdentifier appIdentifier, CoreConfig.PASSWORD_HASHING_ALG hashingAlgorithm, String passwordHash, List errors) throws TenantOrAppNotFoundException { if (hashingAlgorithm == null || passwordHash == null) { @@ -473,7 +494,7 @@ private static String validateAndNormalisePasswordHash(Main main, AppIdentifier return passwordHash; } - private static String validateAndNormaliseThirdPartyId(String thirdPartyId, List errors) { + private String validateAndNormaliseThirdPartyId(String thirdPartyId, List errors) { if (thirdPartyId == null) { return null; } @@ -486,7 +507,7 @@ private static String validateAndNormaliseThirdPartyId(String thirdPartyId, List return thirdPartyId; } - private static String validateAndNormaliseThirdPartyUserId(String thirdPartyUserId, List errors) { + private String validateAndNormaliseThirdPartyUserId(String thirdPartyUserId, List errors) { if (thirdPartyUserId == null) { return null; } @@ -499,7 +520,7 @@ private static String validateAndNormaliseThirdPartyUserId(String thirdPartyUser return thirdPartyUserId; } - private static String validateAndNormalisePhoneNumber(String phoneNumber, List errors) { + private String validateAndNormalisePhoneNumber(String phoneNumber, List errors) { if (phoneNumber == null) { return null; } @@ -512,7 +533,7 @@ private static String validateAndNormalisePhoneNumber(String phoneNumber, List userRoles, List loginMethods, List errors) throws TenantOrAppNotFoundException { if (loginMethods == null) { diff --git a/src/main/java/io/supertokens/cronjobs/CronTaskTest.java b/src/main/java/io/supertokens/cronjobs/CronTaskTest.java index 477d23cc5..4265c361d 100644 --- a/src/main/java/io/supertokens/cronjobs/CronTaskTest.java +++ b/src/main/java/io/supertokens/cronjobs/CronTaskTest.java @@ -28,6 +28,7 @@ public class CronTaskTest extends SingletonResource { private static final String RESOURCE_ID = "io.supertokens.cronjobs.CronTaskTest"; private Map cronTaskToInterval = new HashMap(); + private Map cronTaskToWaitTime = new HashMap(); private CronTaskTest() { @@ -51,4 +52,13 @@ public void setIntervalInSeconds(String resourceId, int interval) { public Integer getIntervalInSeconds(String resourceId) { return cronTaskToInterval.get(resourceId); } + + @TestOnly + public void setInitialWaitTimeInSeconds(String resourceId, int interval) { + cronTaskToWaitTime.put(resourceId, interval); + } + + public Integer getInitialWaitTimeInSeconds(String resourceId) { + return cronTaskToWaitTime.get(resourceId); + } } diff --git a/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkImportUsers.java b/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkImportUsers.java index aa6ff8825..0be95228a 100644 --- a/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkImportUsers.java +++ b/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkImportUsers.java @@ -17,8 +17,6 @@ package io.supertokens.cronjobs.bulkimport; import java.io.IOException; -import java.sql.Connection; -import java.sql.SQLException; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -87,7 +85,7 @@ public class ProcessBulkImportUsers extends CronTask { public static final String RESOURCE_KEY = "io.supertokens.ee.cronjobs.ProcessBulkImportUsers"; - private Map userPoolToStorageMap = new HashMap<>(); + private Map userPoolToStorageMap = new HashMap<>(); private ProcessBulkImportUsers(Main main, List> tenantsInfo) { super("ProcessBulkImportUsers", main, tenantsInfo, true); @@ -113,7 +111,7 @@ protected void doTaskPerApp(AppIdentifier app) AppIdentifier appIdentifier = new AppIdentifier(app.getConnectionUriDomain(), app.getAppId()); - List users = bulkImportSQLStorage.getBulkImportUsersForProcessing(appIdentifier, + List users = bulkImportSQLStorage.getBulkImportUsersAndChangeStatusToProcessing(appIdentifier, BulkImport.PROCESS_USERS_BATCH_SIZE); for (BulkImportUser user : users) { @@ -136,9 +134,11 @@ public int getIntervalTimeSeconds() { @Override public int getInitialWaitTimeSeconds() { - // We are setting a non-zero initial wait for tests to avoid race condition with the beforeTest process that deletes data in the storage layer if (Main.isTesting) { - return 5; + Integer waitTime = CronTaskTest.getInstance(main).getInitialWaitTimeInSeconds(RESOURCE_KEY); + if (waitTime != null) { + return waitTime; + } } return 0; } @@ -178,17 +178,18 @@ public Storage[] getAllProxyStoragesForApp(Main main, AppIdentifier appIdentifie .getAllResourcesWithResourceKey(StorageLayer.RESOURCE_KEY); for (ResourceDistributor.KeyClass key : resources.keySet()) { if (key.getTenantIdentifier().toAppIdentifier().equals(appIdentifier)) { - System.out.println("Adding storage for tenant: " + key.getTenantIdentifier().getTenantId()); allProxyStorages.add(getProxyStorage(key.getTenantIdentifier())); } } return allProxyStorages.toArray(new Storage[0]); } - private void closeAllProxyStorages() { - for (Storage storage : userPoolToStorageMap.values()) { + private void closeAllProxyStorages() throws StorageQueryException { + for (SQLStorage storage : userPoolToStorageMap.values()) { + storage.closeConnectionForBulkImportProxyStorage(); storage.close(); } + userPoolToStorageMap.clear(); } private void processUser(AppIdentifier appIdentifier, BulkImportUser user, BulkImportSQLStorage baseTenantStorage) @@ -206,41 +207,41 @@ private void processUser(AppIdentifier appIdentifier, BulkImportUser user, BulkI try { bulkImportProxyStorage.startTransaction(con -> { - for (LoginMethod lm : user.loginMethods) { - processUserLoginMethod(appIdentifier, bulkImportProxyStorage, lm); - } + try { + for (LoginMethod lm : user.loginMethods) { + processUserLoginMethod(appIdentifier, bulkImportProxyStorage, lm); + } - createPrimaryUserAndLinkAccounts(main, appIdentifier, bulkImportProxyStorage, user, primaryLM); - createUserIdMapping(main, appIdentifier, user, primaryLM); - verifyEmailForAllLoginMethods(appIdentifier, con, bulkImportProxyStorage, user.loginMethods); - createTotpDevices(main, appIdentifier, bulkImportProxyStorage, user.totpDevices, primaryLM); - createUserMetadata(appIdentifier, bulkImportProxyStorage, user, primaryLM); - createUserRoles(main, appIdentifier, bulkImportProxyStorage, user); - - // NOTE: We need to use the baseTenantStorage as bulkImportProxyStorage could have a different storage than the baseTenantStorage - baseTenantStorage.startTransaction(con2 -> { - baseTenantStorage.deleteBulkImportUser_Transaction(appIdentifier, con2, - user.id); + createPrimaryUserAndLinkAccounts(main, appIdentifier, bulkImportProxyStorage, user, primaryLM); + createUserIdMapping(main, appIdentifier, user, primaryLM); + verifyEmailForAllLoginMethods(appIdentifier, con, bulkImportProxyStorage, user.loginMethods); + createTotpDevices(main, appIdentifier, bulkImportProxyStorage, user.totpDevices, primaryLM); + createUserMetadata(appIdentifier, bulkImportProxyStorage, user, primaryLM); + createUserRoles(main, appIdentifier, bulkImportProxyStorage, user); + + // NOTE: We need to use the baseTenantStorage as bulkImportProxyStorage could have a different storage than the baseTenantStorage + baseTenantStorage.startTransaction(con2 -> { + baseTenantStorage.deleteBulkImportUser_Transaction(appIdentifier, con2, + user.id); + return null; + }); + + // We need to commit the transaction manually because we have overridden that in the proxy storage + bulkImportProxyStorage.commitTransactionForBulkImportProxyStorage(); return null; - }); - - // We need to commit the transaction manually because we have overridden that in the proxy storage - try { - Connection connection = (Connection) con.getConnection(); - connection.commit(); - connection.setAutoCommit(true); - } catch (SQLException e) { - throw new StorageTransactionLogicException(e); + } catch (StorageTransactionLogicException e) { + // We need to rollback the transaction manually because we have overridden that in the proxy storage + bulkImportProxyStorage.rollbackTransactionForBulkImportProxyStorage(); + throw e; } - - return null; }); } catch (StorageTransactionLogicException e) { handleProcessUserExceptions(appIdentifier, user, e, baseTenantStorage); } } - private void handleProcessUserExceptions(AppIdentifier appIdentifier, BulkImportUser user, Exception e, BulkImportSQLStorage baseTenantStorage) + private void handleProcessUserExceptions(AppIdentifier appIdentifier, BulkImportUser user, Exception e, + BulkImportSQLStorage baseTenantStorage) throws StorageQueryException { // Java doesn't allow us to reassign local variables inside a lambda expression @@ -252,21 +253,10 @@ private void handleProcessUserExceptions(AppIdentifier appIdentifier, BulkImport errorMessage[0] = exception.actualException.getMessage(); } - String[] userId = { user.id }; - try { baseTenantStorage.startTransaction(con -> { - baseTenantStorage.updateBulkImportUserStatus_Transaction(appIdentifier, con, userId, + baseTenantStorage.updateBulkImportUserStatus_Transaction(appIdentifier, con, user.id, BULK_IMPORT_USER_STATUS.FAILED, errorMessage[0]); - - // We need to commit the transaction manually because we have overridden that in the proxy storage - try { - Connection connection = (Connection) con.getConnection(); - connection.commit(); - connection.setAutoCommit(true); - } catch (SQLException ex) { - throw new StorageTransactionLogicException(ex); - } return null; }); } catch (StorageTransactionLogicException e1) { diff --git a/src/main/java/io/supertokens/inmemorydb/Start.java b/src/main/java/io/supertokens/inmemorydb/Start.java index d4edd411b..d5cdd89b2 100644 --- a/src/main/java/io/supertokens/inmemorydb/Start.java +++ b/src/main/java/io/supertokens/inmemorydb/Start.java @@ -137,7 +137,27 @@ public void constructor(String processId, boolean silent, boolean isTesting) { @Override public Storage createBulkImportProxyStorageInstance() { - return this; + // throw not implemented error + throw new UnsupportedOperationException("Unimplemented method 'createBulkImportProxyStorageInstance'"); + + } + + @Override + public void closeConnectionForBulkImportProxyStorage() throws StorageQueryException { + throw new UnsupportedOperationException( + "closeConnectionForBulkImportProxyStorage should only be called from BulkImportProxyStorage"); + } + + @Override + public void commitTransactionForBulkImportProxyStorage() throws StorageQueryException { + throw new UnsupportedOperationException( + "commitTransactionForBulkImportProxyStorage should only be called from BulkImportProxyStorage"); + } + + @Override + public void rollbackTransactionForBulkImportProxyStorage() throws StorageQueryException { + throw new UnsupportedOperationException( + "rollbackTransactionForBulkImportProxyStorage should only be called from BulkImportProxyStorage"); } @Override diff --git a/src/main/java/io/supertokens/webserver/api/bulkimport/BulkImportAPI.java b/src/main/java/io/supertokens/webserver/api/bulkimport/BulkImportAPI.java index 0fbf8055f..abb8fe65d 100644 --- a/src/main/java/io/supertokens/webserver/api/bulkimport/BulkImportAPI.java +++ b/src/main/java/io/supertokens/webserver/api/bulkimport/BulkImportAPI.java @@ -59,6 +59,7 @@ public String getPath() { @Override protected void doGet(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException { + // API is app specific String statusString = InputParser.getQueryParamOrThrowError(req, "status", true); String paginationToken = InputParser.getQueryParamOrThrowError(req, "paginationToken", true); Integer limit = InputParser.getIntQueryParamOrThrowError(req, "limit", true); @@ -118,6 +119,7 @@ protected void doGet(HttpServletRequest req, HttpServletResponse resp) throws Se @Override protected void doPost(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException { + // API is app specific JsonObject input = InputParser.parseJsonObjectOrThrowError(req); JsonArray users = InputParser.parseArrayOrThrowError(input, "users", false); @@ -148,12 +150,12 @@ protected void doPost(HttpServletRequest req, HttpServletResponse resp) throws S } JsonArray errorsJson = new JsonArray(); - Set allExternalUserIds = new HashSet<>(); List usersToAdd = new ArrayList<>(); + BulkImportUserUtils bulkImportUserUtils = new BulkImportUserUtils(allUserRoles); for (int i = 0; i < users.size(); i++) { try { - BulkImportUser user = BulkImportUserUtils.createBulkImportUserFromJSON(main, appIdentifier, users.get(i).getAsJsonObject(), Utils.getUUID(), allUserRoles, allExternalUserIds); + BulkImportUser user = bulkImportUserUtils.createBulkImportUserFromJSON(main, appIdentifier, users.get(i).getAsJsonObject(), Utils.getUUID()); usersToAdd.add(user); } catch (io.supertokens.bulkimport.exceptions.InvalidBulkImportDataException e) { JsonObject errorObj = new JsonObject(); @@ -191,6 +193,7 @@ protected void doPost(HttpServletRequest req, HttpServletResponse resp) throws S @Override protected void doDelete(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException { + // API is app specific JsonObject input = InputParser.parseJsonObjectOrThrowError(req); JsonArray arr = InputParser.parseArrayOrThrowError(input, "ids", false); diff --git a/src/test/java/io/supertokens/test/bulkimport/BulkImportTest.java b/src/test/java/io/supertokens/test/bulkimport/BulkImportTest.java index ca0be8534..d7847e36a 100644 --- a/src/test/java/io/supertokens/test/bulkimport/BulkImportTest.java +++ b/src/test/java/io/supertokens/test/bulkimport/BulkImportTest.java @@ -32,6 +32,8 @@ import io.supertokens.ProcessState; import io.supertokens.bulkimport.BulkImport; import io.supertokens.bulkimport.BulkImportUserPaginationContainer; +import io.supertokens.cronjobs.CronTaskTest; +import io.supertokens.cronjobs.bulkimport.ProcessBulkImportUsers; import io.supertokens.pluginInterface.STORAGE_TYPE; import io.supertokens.pluginInterface.bulkimport.BulkImportStorage; import io.supertokens.pluginInterface.bulkimport.BulkImportUser; @@ -161,10 +163,10 @@ public void testGetUsersStatusFilter() throws Exception { BulkImport.addUsers(appIdentifier, storage, users); // Update the users status to PROCESSING - String[] userIds = users.stream().map(user -> user.id).toArray(String[]::new); - storage.startTransaction(con -> { - storage.updateBulkImportUserStatus_Transaction(appIdentifier, con, userIds, BULK_IMPORT_USER_STATUS.PROCESSING, null); + for (BulkImportUser user : users) { + storage.updateBulkImportUserStatus_Transaction(appIdentifier, con, user.id, BULK_IMPORT_USER_STATUS.PROCESSING, null); + } storage.commitTransaction(con); return null; }); @@ -179,10 +181,10 @@ public void testGetUsersStatusFilter() throws Exception { BulkImport.addUsers(appIdentifier, storage, users); // Update the users status to FAILED - String[] userIds = users.stream().map(user -> user.id).toArray(String[]::new); - storage.startTransaction(con -> { - storage.updateBulkImportUserStatus_Transaction(appIdentifier, con, userIds, BULK_IMPORT_USER_STATUS.FAILED, null); + for (BulkImportUser user : users) { + storage.updateBulkImportUserStatus_Transaction(appIdentifier, con, user.id, BULK_IMPORT_USER_STATUS.FAILED, null); + } storage.commitTransaction(con); return null; }); @@ -199,7 +201,13 @@ public void testGetUsersStatusFilter() throws Exception { public void randomPaginationTest() throws Exception { String[] args = {"../"}; - TestingProcessManager.TestingProcess process = TestingProcessManager.start(args); + TestingProcessManager.TestingProcess process = TestingProcessManager.start(args, false); + + // We are setting a high initial wait time to ensure the cron job doesn't run while we are running the tests + CronTaskTest.getInstance(process.getProcess()).setInitialWaitTimeInSeconds(ProcessBulkImportUsers.RESOURCE_KEY, 1000000); + + process.startProcess(); + assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); if (StorageLayer.getStorage(process.getProcess()).getType() != STORAGE_TYPE.SQL) { diff --git a/src/test/java/io/supertokens/test/bulkimport/ProcessBulkImportUsersCronJobTest.java b/src/test/java/io/supertokens/test/bulkimport/ProcessBulkImportUsersCronJobTest.java index 00c73bfb6..b71372d5e 100644 --- a/src/test/java/io/supertokens/test/bulkimport/ProcessBulkImportUsersCronJobTest.java +++ b/src/test/java/io/supertokens/test/bulkimport/ProcessBulkImportUsersCronJobTest.java @@ -215,6 +215,8 @@ private TestingProcess startCronProcess() throws InterruptedException { .setKeyValue(FeatureFlagTestContent.ENABLED_FEATURES, new EE_FEATURES[] { EE_FEATURES.ACCOUNT_LINKING, EE_FEATURES.MULTI_TENANCY, EE_FEATURES.MFA }); + // We are setting a non-zero initial wait for tests to avoid race condition with the beforeTest process that deletes data in the storage layer + CronTaskTest.getInstance(main).setInitialWaitTimeInSeconds(ProcessBulkImportUsers.RESOURCE_KEY, 5); CronTaskTest.getInstance(main).setIntervalInSeconds(ProcessBulkImportUsers.RESOURCE_KEY, 100000); process.startProcess(); assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); diff --git a/src/test/java/io/supertokens/test/bulkimport/apis/AddBulkImportUsersTest.java b/src/test/java/io/supertokens/test/bulkimport/apis/AddBulkImportUsersTest.java index 3303ebca7..d38c18de0 100644 --- a/src/test/java/io/supertokens/test/bulkimport/apis/AddBulkImportUsersTest.java +++ b/src/test/java/io/supertokens/test/bulkimport/apis/AddBulkImportUsersTest.java @@ -194,8 +194,24 @@ public void shouldThrow400Error() throws Exception { assertEquals(responseString, "{\"error\":\"" + genericErrMsg + "\",\"users\":[{\"index\":0,\"errors\":[\"loginMethods is required.\"]},{\"index\":1,\"errors\":[\"loginMethods is required.\",\"externalUserId id1 is not unique. It is already used by another user.\"]}]}"); } + // MFA must be enabled to import totpDevices + try { + JsonObject request = new JsonParser() + .parse("{\"users\":[{\"totpDevices\":[{\"secret\": \"secret\"}]}]}") + .getAsJsonObject(); + HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", + "http://localhost:3567/bulk-import/users", + request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); + fail("The API should have thrown an error"); + } catch (io.supertokens.test.httpRequest.HttpResponseException e) { + String responseString = getResponseMessageFromError(e.getMessage()); + assertEquals(400, e.statusCode); + assertEquals(responseString, + "{\"error\":\"" + genericErrMsg + "\",\"users\":[{\"index\":0,\"errors\":[\"MFA must be enabled to import totp devices.\",\"loginMethods is required.\"]}]}"); + } // secretKey is required in totpDevices try { + setFeatureFlags(process.getProcess(), new EE_FEATURES[]{EE_FEATURES.MFA}); JsonObject request = new JsonParser() .parse("{\"users\":[{\"totpDevices\":[{\"secret\": \"secret\"}]}]}") .getAsJsonObject(); @@ -387,6 +403,28 @@ public void shouldThrow400Error() throws Exception { "{\"error\":\"" + genericErrMsg + "\",\"users\":[{\"index\":0,\"errors\":[\"email should be of type string for a passwordless recipe.\",\"phoneNumber should be of type string for a passwordless recipe.\",\"Either email or phoneNumber is required for a passwordless recipe.\"]}]}"); } } + // Disabling all feature flags to be able to get the desired error messages + { + setFeatureFlags(process.getProcess(), new EE_FEATURES[]{}); + } + // More than two loginMethods when either of account linking or MFA is not enabled + { + // CASE 1: email, passwordHash and hashingAlgorithm are not present + try { + JsonObject request = new JsonParser() + .parse("{\"users\":[{\"loginMethods\":[{\"recipeId\":\"emailpassword\",\"email\":\"johndoe@gmail.com\",\"passwordHash\":\"$2a\",\"hashingAlgorithm\":\"bcrypt\",\"isPrimary\":true},{\"recipeId\":\"passwordless\",\"email\":\"johndoe@gmail.com\"}]}]}").getAsJsonObject(); + HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", + "http://localhost:3567/bulk-import/users", + request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); + fail("The API should have thrown an error"); + } catch (io.supertokens.test.httpRequest.HttpResponseException e) { + String responseString = getResponseMessageFromError(e.getMessage()); + assertEquals(400, e.statusCode); + assertEquals(responseString, + "{\"error\":\"" + genericErrMsg + "\",\"users\":[{\"index\":0,\"errors\":[\"Account linking or MFA must be enabled to import multiple loginMethods.\"]}]}"); + } + } + // Validate tenantId { // CASE 1: Invalid tenantId when multitenancy is not enabled @@ -405,11 +443,12 @@ public void shouldThrow400Error() throws Exception { assertEquals(responseString, "{\"error\":\"" + genericErrMsg + "\",\"users\":[{\"index\":0,\"errors\":[\"Multitenancy must be enabled before importing users to a different tenant.\"]}]}"); } + // Now enabling Account linking and Multitenancy for further tests + { + setFeatureFlags(process.getProcess(), new EE_FEATURES[]{EE_FEATURES.ACCOUNT_LINKING, EE_FEATURES.MULTI_TENANCY}); + } // CASE 2: Invalid tenantId when multitenancy is enabled try { - FeatureFlagTestContent.getInstance(process.getProcess()) - .setKeyValue(FeatureFlagTestContent.ENABLED_FEATURES, new EE_FEATURES[]{EE_FEATURES.MULTI_TENANCY}); - JsonObject request = new JsonParser().parse( "{\"users\":[{\"loginMethods\":[{\"tenantIds\":[\"invalid\"],\"recipeId\":\"passwordless\",\"email\":\"johndoe@gmail.com\"}]}]}") .getAsJsonObject(); @@ -425,9 +464,6 @@ public void shouldThrow400Error() throws Exception { } // CASE 3. Two more tenants do not share the same storage try { - FeatureFlagTestContent.getInstance(process.getProcess()) - .setKeyValue(FeatureFlagTestContent.ENABLED_FEATURES, new EE_FEATURES[]{EE_FEATURES.MULTI_TENANCY}); - createTenants(process.getProcess()); JsonObject request = new JsonParser().parse( @@ -505,6 +541,8 @@ public void shouldReturn200Response() throws Exception { return; } + setFeatureFlags(process.getProcess(), new EE_FEATURES[]{EE_FEATURES.MFA}); + // Create user roles before inserting bulk users { UserRoles.createNewRoleOrModifyItsPermissions(process.getProcess(), "role1", null); @@ -532,6 +570,8 @@ public void shouldNormaliseFields() throws Exception { return; } + setFeatureFlags(process.getProcess(), new EE_FEATURES[]{EE_FEATURES.MFA}); + // Create user roles before inserting bulk users { UserRoles.createNewRoleOrModifyItsPermissions(process.getProcess(), "role1", null); @@ -689,4 +729,8 @@ null, null, new JsonObject() ); } } + + private void setFeatureFlags(Main main, EE_FEATURES[] features) { + FeatureFlagTestContent.getInstance(main).setKeyValue(FeatureFlagTestContent.ENABLED_FEATURES, features); + } } From d721cd5ce63d2426ce619cd7fa75536032c2d3fa Mon Sep 17 00:00:00 2001 From: Ankit Tiwari Date: Fri, 29 Mar 2024 11:49:11 +0530 Subject: [PATCH 05/21] fix: PR changes --- .../bulkimport/ProcessBulkImportUsers.java | 116 ++++++++++++++---- 1 file changed, 92 insertions(+), 24 deletions(-) diff --git a/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkImportUsers.java b/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkImportUsers.java index 0be95228a..918a9cefa 100644 --- a/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkImportUsers.java +++ b/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkImportUsers.java @@ -48,6 +48,7 @@ import io.supertokens.pluginInterface.Storage; import io.supertokens.pluginInterface.StorageUtils; import io.supertokens.pluginInterface.authRecipe.AuthRecipeUserInfo; +import io.supertokens.pluginInterface.authRecipe.sqlStorage.AuthRecipeSQLStorage; import io.supertokens.pluginInterface.bulkimport.BulkImportUser; import io.supertokens.pluginInterface.bulkimport.BulkImportStorage.BULK_IMPORT_USER_STATUS; import io.supertokens.pluginInterface.bulkimport.BulkImportUser.LoginMethod; @@ -205,7 +206,23 @@ private void processUser(AppIdentifier appIdentifier, BulkImportUser user, BulkI LoginMethod primaryLM = getPrimaryLoginMethod(user); + AuthRecipeSQLStorage authRecipeSQLStorage = (AuthRecipeSQLStorage) getProxyStorage(firstTenantIdentifier); try { + // If primaryUserId is not null, it means we may have already processed this user but failed to delete the entry + // If the primaryUserId exists in the database, we'll delete the corresponding entry from the bulkImportUser table and proceed to skip this user. + if (user.primaryUserId != null) { + AuthRecipeUserInfo processedUser = authRecipeSQLStorage.getPrimaryUserById(appIdentifier, + user.primaryUserId); + + if (processedUser != null && isProcessedUserFromSameBulkImportUserEntry(processedUser, user)) { + baseTenantStorage.startTransaction(con2 -> { + baseTenantStorage.deleteBulkImportUser_Transaction(appIdentifier, con2, user.id); + return null; + }); + return; + } + } + bulkImportProxyStorage.startTransaction(con -> { try { for (LoginMethod lm : user.loginMethods) { @@ -219,15 +236,24 @@ private void processUser(AppIdentifier appIdentifier, BulkImportUser user, BulkI createUserMetadata(appIdentifier, bulkImportProxyStorage, user, primaryLM); createUserRoles(main, appIdentifier, bulkImportProxyStorage, user); + // We are updating the primaryUserId in the bulkImportUser entry. This will help us handle the inconsistent transaction commit. + // If this update statement fails then the outer transaction will fail as well and the user will simpl be processed again. No inconsistency will happen in this + // case. + baseTenantStorage.updateBulkImportUserPrimaryUserId(appIdentifier, user.id, + primaryLM.superTokensUserId); + + // We need to commit the transaction manually because we have overridden that in the proxy storage + // If this fails, the primaryUserId will be updated in the bulkImportUser but it wouldn’t actually exist. + // When processing the user again, we'll check if primaryUserId exists with the same email. In this case the user won't exist, and we'll simply re-process it. + bulkImportProxyStorage.commitTransactionForBulkImportProxyStorage(); + // NOTE: We need to use the baseTenantStorage as bulkImportProxyStorage could have a different storage than the baseTenantStorage + // If this fails, the primaryUserId will be updated in the bulkImportUser and it would exist in the database. + // When processing the user again, we'll check if primaryUserId exists with the same email. In this case the user will exist, and we'll simply delete the entry. baseTenantStorage.startTransaction(con2 -> { - baseTenantStorage.deleteBulkImportUser_Transaction(appIdentifier, con2, - user.id); + baseTenantStorage.deleteBulkImportUser_Transaction(appIdentifier, con2, user.id); return null; }); - - // We need to commit the transaction manually because we have overridden that in the proxy storage - bulkImportProxyStorage.commitTransactionForBulkImportProxyStorage(); return null; } catch (StorageTransactionLogicException e) { // We need to rollback the transaction manually because we have overridden that in the proxy storage @@ -291,7 +317,7 @@ private void processEmailPasswordLoginMethod(TenantIdentifier tenantIdentifier, ImportUserResponse userInfo = EmailPassword.createUserWithPasswordHash(tenantIdentifier, storage, lm.email, lm.passwordHash, lm.timeJoinedInMSSinceEpoch); - lm.superTokensOrExternalUserId = userInfo.user.getSupertokensUserId(); + lm.superTokensUserId = userInfo.user.getSupertokensUserId(); } catch (StorageQueryException | TenantOrAppNotFoundException e) { throw new StorageTransactionLogicException(e); } catch (DuplicateEmailException e) { @@ -307,7 +333,7 @@ private void processThirdPartyLoginMethod(TenantIdentifier tenantIdentifier, Sto tenantIdentifier, storage, lm.thirdPartyId, lm.thirdPartyUserId, lm.email, lm.timeJoinedInMSSinceEpoch); - lm.superTokensOrExternalUserId = userInfo.user.getSupertokensUserId(); + lm.superTokensUserId = userInfo.user.getSupertokensUserId(); } catch (StorageQueryException | TenantOrAppNotFoundException e) { throw new StorageTransactionLogicException(e); } catch (DuplicateThirdPartyUserException e) { @@ -322,7 +348,7 @@ private void processPasswordlessLoginMethod(TenantIdentifier tenantIdentifier, S AuthRecipeUserInfo userInfo = Passwordless.createPasswordlessUser(tenantIdentifier, storage, lm.email, lm.phoneNumber, lm.timeJoinedInMSSinceEpoch); - lm.superTokensOrExternalUserId = userInfo.getSupertokensUserId(); + lm.superTokensUserId = userInfo.getSupertokensUserId(); } catch (StorageQueryException | TenantOrAppNotFoundException | RestartFlowException e) { throw new StorageTransactionLogicException(e); } @@ -338,7 +364,7 @@ private void associateUserToTenants(Main main, AppIdentifier appIdentifier, Stor TenantIdentifier tenantIdentifier = new TenantIdentifier(appIdentifier.getConnectionUriDomain(), appIdentifier.getAppId(), tenantId); - Multitenancy.addUserIdToTenant(main, tenantIdentifier, storage, lm.superTokensOrExternalUserId); + Multitenancy.addUserIdToTenant(main, tenantIdentifier, storage, lm.getSuperTokenOrExternalUserId()); } catch (TenantOrAppNotFoundException | UnknownUserIdException | StorageQueryException | FeatureNotEnabledException | DuplicateEmailException | DuplicatePhoneNumberException | DuplicateThirdPartyUserException | AnotherPrimaryUserWithPhoneNumberAlreadyExistsException @@ -357,12 +383,12 @@ private void createPrimaryUserAndLinkAccounts(Main main, } try { - AuthRecipe.createPrimaryUser(main, appIdentifier, storage, primaryLM.superTokensOrExternalUserId); + AuthRecipe.createPrimaryUser(main, appIdentifier, storage, primaryLM.getSuperTokenOrExternalUserId()); } catch (TenantOrAppNotFoundException | FeatureNotEnabledException | StorageQueryException e) { throw new StorageTransactionLogicException(e); } catch (UnknownUserIdException e) { throw new StorageTransactionLogicException(new Exception( - "We tried to create the primary user for the userId " + primaryLM.superTokensOrExternalUserId + "We tried to create the primary user for the userId " + primaryLM.getSuperTokenOrExternalUserId() + " but it doesn't exist. This should not happen. Please contact support.")); } catch (RecipeUserIdAlreadyLinkedWithPrimaryUserIdException | AccountInfoAlreadyAssociatedWithAnotherPrimaryUserIdException e) { @@ -372,24 +398,24 @@ private void createPrimaryUserAndLinkAccounts(Main main, for (LoginMethod lm : user.loginMethods) { try { - if (lm.superTokensOrExternalUserId.equals(primaryLM.superTokensOrExternalUserId)) { + if (lm.getSuperTokenOrExternalUserId().equals(primaryLM.getSuperTokenOrExternalUserId())) { continue; } - AuthRecipe.linkAccounts(main, appIdentifier, storage, lm.superTokensOrExternalUserId, - primaryLM.superTokensOrExternalUserId); + AuthRecipe.linkAccounts(main, appIdentifier, storage, lm.getSuperTokenOrExternalUserId(), + primaryLM.getSuperTokenOrExternalUserId()); } catch (TenantOrAppNotFoundException | FeatureNotEnabledException | StorageQueryException e) { throw new StorageTransactionLogicException(e); } catch (UnknownUserIdException e) { throw new StorageTransactionLogicException( - new Exception("We tried to link the userId " + lm.superTokensOrExternalUserId - + " to the primary userId " + primaryLM.superTokensOrExternalUserId + new Exception("We tried to link the userId " + lm.getSuperTokenOrExternalUserId() + + " to the primary userId " + primaryLM.getSuperTokenOrExternalUserId() + " but it doesn't exist. This should not happen. Please contact support.")); } catch (InputUserIdIsNotAPrimaryUserException e) { throw new StorageTransactionLogicException( - new Exception("We tried to link the userId " + lm.superTokensOrExternalUserId - + " to the primary userId " + primaryLM.superTokensOrExternalUserId + new Exception("We tried to link the userId " + lm.getSuperTokenOrExternalUserId() + + " to the primary userId " + primaryLM.getSuperTokenOrExternalUserId() + " but it is not a primary user. This should not happen. Please contact support.")); } catch (AccountInfoAlreadyAssociatedWithAnotherPrimaryUserIdException | RecipeUserIdAlreadyLinkedWithAnotherPrimaryUserIdException e) { @@ -405,10 +431,10 @@ private void createUserIdMapping(Main main, AppIdentifier appIdentifier, try { UserIdMapping.createUserIdMapping( appIdentifier, getAllProxyStoragesForApp(main, appIdentifier), - primaryLM.superTokensOrExternalUserId, user.externalUserId, + primaryLM.superTokensUserId, user.externalUserId, null, false, true); - primaryLM.superTokensOrExternalUserId = user.externalUserId; + primaryLM.externalUserId = user.externalUserId; } catch (StorageQueryException | ServletException | TenantOrAppNotFoundException | InvalidConfigException | IOException | DbInitException e) { throw new StorageTransactionLogicException(e); @@ -418,7 +444,7 @@ appIdentifier, getAllProxyStoragesForApp(main, appIdentifier), } catch (UnknownSuperTokensUserIdException e) { throw new StorageTransactionLogicException( new Exception("We tried to create the externalUserId mapping for the superTokenUserId " - + primaryLM.superTokensOrExternalUserId + + primaryLM.superTokensUserId + " but it doesn't exist. This should not happen. Please contact support.")); } } @@ -428,7 +454,7 @@ private void createUserMetadata(AppIdentifier appIdentifier, Storage storage, Bu LoginMethod primaryLM) throws StorageTransactionLogicException { if (user.userMetadata != null) { try { - UserMetadata.updateUserMetadata(appIdentifier, storage, primaryLM.superTokensOrExternalUserId, + UserMetadata.updateUserMetadata(appIdentifier, storage, primaryLM.getSuperTokenOrExternalUserId(), user.userMetadata); } catch (StorageQueryException | TenantOrAppNotFoundException e) { throw new StorageTransactionLogicException(e); @@ -471,7 +497,7 @@ private void verifyEmailForAllLoginMethods(AppIdentifier appIdentifier, Transact .getEmailVerificationStorage(storage); emailVerificationSQLStorage .updateIsEmailVerified_Transaction(tenantIdentifier.toAppIdentifier(), con, - lm.superTokensOrExternalUserId, lm.email, true); + lm.getSuperTokenOrExternalUserId(), lm.email, true); } catch (TenantOrAppNotFoundException | StorageQueryException e) { throw new StorageTransactionLogicException(e); } @@ -482,7 +508,7 @@ private void createTotpDevices(Main main, AppIdentifier appIdentifier, Storage s List totpDevices, LoginMethod primaryLM) throws StorageTransactionLogicException { for (TotpDevice totpDevice : totpDevices) { try { - Totp.createDevice(main, appIdentifier, storage, primaryLM.superTokensOrExternalUserId, + Totp.createDevice(main, appIdentifier, storage, primaryLM.getSuperTokenOrExternalUserId(), totpDevice.deviceName, totpDevice.skew, totpDevice.period, totpDevice.secretKey, true, System.currentTimeMillis()); } catch (TenantOrAppNotFoundException | StorageQueryException | FeatureNotEnabledException e) { @@ -509,4 +535,46 @@ private BulkImportUser.LoginMethod getPrimaryLoginMethod(BulkImportUser user) { } return oldestLM; } + + private boolean isProcessedUserFromSameBulkImportUserEntry( + AuthRecipeUserInfo processedUser, BulkImportUser bulkImportUser) { + if (bulkImportUser == null || processedUser == null || bulkImportUser.loginMethods == null || + processedUser.loginMethods == null) { + return false; + } + + for (LoginMethod lm1 : bulkImportUser.loginMethods) { + for (io.supertokens.pluginInterface.authRecipe.LoginMethod lm2 : processedUser.loginMethods) { + if (lm2.recipeId.toString().equals(lm1.recipeId)) { + if (lm1.email != null && !lm1.email.equals(lm2.email)) { + return false; + } + + switch (lm1.recipeId) { + case "emailpassword": + if (lm1.passwordHash != null && !lm1.passwordHash.equals(lm2.passwordHash)) { + return false; + } + break; + case "thirdparty": + if ((lm1.thirdPartyId != null && !lm1.thirdPartyId.equals(lm2.thirdParty.id)) + || (lm1.thirdPartyUserId != null + && !lm1.thirdPartyUserId.equals(lm2.thirdParty.userId))) { + return false; + } + break; + case "passwordless": + if (lm1.phoneNumber != null && !lm1.phoneNumber.equals(lm2.phoneNumber)) { + return false; + } + break; + default: + return false; + } + } + } + } + + return true; + } } From 769cbb648b06c6aa005a2dc99008a9a6250b75b6 Mon Sep 17 00:00:00 2001 From: Ankit Tiwari Date: Tue, 2 Apr 2024 14:39:32 +0530 Subject: [PATCH 06/21] fix: PR changes --- .../io/supertokens/authRecipe/AuthRecipe.java | 8 +- .../bulkimport/BulkImportUserUtils.java | 5 +- .../bulkimport/ProcessBulkImportUsers.java | 6 +- src/main/java/io/supertokens/utils/Utils.java | 8 + .../test/bulkimport/BulkImportTestUtils.java | 17 +- .../ProcessBulkImportUsersCronJobTest.java | 100 ++- .../apis/AddBulkImportUsersTest.java | 809 ++++++++---------- 7 files changed, 479 insertions(+), 474 deletions(-) diff --git a/src/main/java/io/supertokens/authRecipe/AuthRecipe.java b/src/main/java/io/supertokens/authRecipe/AuthRecipe.java index 271b93d52..8b4ffa91c 100644 --- a/src/main/java/io/supertokens/authRecipe/AuthRecipe.java +++ b/src/main/java/io/supertokens/authRecipe/AuthRecipe.java @@ -42,6 +42,8 @@ import io.supertokens.session.Session; import io.supertokens.storageLayer.StorageLayer; import io.supertokens.useridmapping.UserIdType; +import io.supertokens.utils.Utils; + import org.jetbrains.annotations.TestOnly; import javax.annotation.Nullable; @@ -327,8 +329,7 @@ public static LinkAccountsResult linkAccounts(Main main, AppIdentifier appIdenti RecipeUserIdAlreadyLinkedWithAnotherPrimaryUserIdException, InputUserIdIsNotAPrimaryUserException, UnknownUserIdException, TenantOrAppNotFoundException, FeatureNotEnabledException { - if (Arrays.stream(FeatureFlag.getInstance(main, appIdentifier).getEnabledFeatures()) - .noneMatch(t -> t == EE_FEATURES.ACCOUNT_LINKING || t == EE_FEATURES.MFA)) { + if (!Utils.isAccountLinkingEnabled(main, appIdentifier)) { throw new FeatureNotEnabledException( "Account linking feature is not enabled for this app. Please contact support to enable it."); } @@ -532,8 +533,7 @@ public static CreatePrimaryUserResult createPrimaryUser(Main main, RecipeUserIdAlreadyLinkedWithPrimaryUserIdException, UnknownUserIdException, TenantOrAppNotFoundException, FeatureNotEnabledException { - if (Arrays.stream(FeatureFlag.getInstance(main, appIdentifier).getEnabledFeatures()) - .noneMatch(t -> t == EE_FEATURES.ACCOUNT_LINKING || t == EE_FEATURES.MFA)) { + if (!Utils.isAccountLinkingEnabled(main, appIdentifier)) { throw new FeatureNotEnabledException( "Account linking feature is not enabled for this app. Please contact support to enable it."); } diff --git a/src/main/java/io/supertokens/bulkimport/BulkImportUserUtils.java b/src/main/java/io/supertokens/bulkimport/BulkImportUserUtils.java index f884680d4..7e81c0125 100644 --- a/src/main/java/io/supertokens/bulkimport/BulkImportUserUtils.java +++ b/src/main/java/io/supertokens/bulkimport/BulkImportUserUtils.java @@ -170,9 +170,8 @@ private List getParsedLoginMethods(Main main, AppIdentifier appIden } if (jsonLoginMethods.size() > 1) { - if (Arrays.stream(FeatureFlag.getInstance(main, appIdentifier).getEnabledFeatures()) - .noneMatch(t -> t == EE_FEATURES.ACCOUNT_LINKING || t == EE_FEATURES.MFA)) { - errors.add("Account linking or MFA must be enabled to import multiple loginMethods."); + if (!Utils.isAccountLinkingEnabled(main, appIdentifier)) { + errors.add("Account linking must be enabled to import multiple loginMethods."); } } diff --git a/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkImportUsers.java b/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkImportUsers.java index 918a9cefa..1d80ab6fa 100644 --- a/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkImportUsers.java +++ b/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkImportUsers.java @@ -110,13 +110,11 @@ protected void doTaskPerApp(AppIdentifier app) BulkImportSQLStorage bulkImportSQLStorage = (BulkImportSQLStorage) StorageLayer .getStorage(app.getAsPublicTenantIdentifier(), main); - AppIdentifier appIdentifier = new AppIdentifier(app.getConnectionUriDomain(), app.getAppId()); - - List users = bulkImportSQLStorage.getBulkImportUsersAndChangeStatusToProcessing(appIdentifier, + List users = bulkImportSQLStorage.getBulkImportUsersAndChangeStatusToProcessing(app, BulkImport.PROCESS_USERS_BATCH_SIZE); for (BulkImportUser user : users) { - processUser(appIdentifier, user, bulkImportSQLStorage); + processUser(app, user, bulkImportSQLStorage); } closeAllProxyStorages(); diff --git a/src/main/java/io/supertokens/utils/Utils.java b/src/main/java/io/supertokens/utils/Utils.java index ecd3a0479..eb84cff5c 100644 --- a/src/main/java/io/supertokens/utils/Utils.java +++ b/src/main/java/io/supertokens/utils/Utils.java @@ -25,6 +25,8 @@ import com.google.i18n.phonenumbers.Phonenumber; import io.supertokens.Main; import io.supertokens.config.Config; +import io.supertokens.featureflag.EE_FEATURES; +import io.supertokens.featureflag.FeatureFlag; import io.supertokens.jwt.exceptions.UnsupportedJWTSigningAlgorithmException; import io.supertokens.pluginInterface.exceptions.StorageQueryException; import io.supertokens.pluginInterface.exceptions.StorageTransactionLogicException; @@ -50,6 +52,7 @@ import java.security.spec.KeySpec; import java.security.spec.PKCS8EncodedKeySpec; import java.security.spec.X509EncodedKeySpec; +import java.util.Arrays; import java.util.Base64; import java.util.Base64.Decoder; import java.util.Base64.Encoder; @@ -427,4 +430,9 @@ public static JsonObject addLegacySigningKeyInfos(AppIdentifier appIdentifier, M public static JsonElement toJsonTreeWithNulls(Object src) { return new GsonBuilder().serializeNulls().create().toJsonTree(src); } + + public static boolean isAccountLinkingEnabled(Main main, AppIdentifier appIdentifier) throws StorageQueryException, TenantOrAppNotFoundException { + return Arrays.stream(FeatureFlag.getInstance(main, appIdentifier).getEnabledFeatures()) + .anyMatch(t -> t == EE_FEATURES.ACCOUNT_LINKING || t == EE_FEATURES.MFA); + } } diff --git a/src/test/java/io/supertokens/test/bulkimport/BulkImportTestUtils.java b/src/test/java/io/supertokens/test/bulkimport/BulkImportTestUtils.java index 6b822d610..cfee7fa64 100644 --- a/src/test/java/io/supertokens/test/bulkimport/BulkImportTestUtils.java +++ b/src/test/java/io/supertokens/test/bulkimport/BulkImportTestUtils.java @@ -29,11 +29,16 @@ import io.supertokens.pluginInterface.bulkimport.BulkImportUser.UserRole; public class BulkImportTestUtils { + public static List generateBulkImportUser(int numberOfUsers) { + return generateBulkImportUser(numberOfUsers, List.of("public", "t1"), 0); + } + + public static List generateBulkImportUser(int numberOfUsers, List tenants, int startIndex) { List users = new ArrayList<>(); JsonParser parser = new JsonParser(); - for (int i = 0; i < numberOfUsers; i++) { + for (int i = startIndex; i < numberOfUsers + startIndex; i++) { String email = "user" + i + "@example.com"; String id = io.supertokens.utils.Utils.getUUID(); String externalId = io.supertokens.utils.Utils.getUUID(); @@ -41,17 +46,17 @@ public static List generateBulkImportUser(int numberOfUsers) { JsonObject userMetadata = parser.parse("{\"key1\":\"value1\",\"key2\":{\"key3\":\"value3\"}}").getAsJsonObject(); List userRoles = new ArrayList<>(); - userRoles.add(new UserRole("role1", List.of("public"))); - userRoles.add(new UserRole("role2", List.of("public"))); + userRoles.add(new UserRole("role1", tenants)); + userRoles.add(new UserRole("role2", tenants)); List totpDevices = new ArrayList<>(); totpDevices.add(new TotpDevice("secretKey", 30, 1, "deviceName")); List loginMethods = new ArrayList<>(); long currentTimeMillis = System.currentTimeMillis(); - loginMethods.add(new LoginMethod(List.of("public", "t1"), "emailpassword", true, true, currentTimeMillis, email, "$2a", "BCRYPT", null, null, null)); - loginMethods.add(new LoginMethod(List.of("public", "t1"), "thirdparty", true, false, currentTimeMillis, email, null, null, "thirdPartyId" + i, "thirdPartyUserId" + i, null)); - loginMethods.add(new LoginMethod(List.of("public", "t1"), "passwordless", true, false, currentTimeMillis, email, null, null, null, null, null)); + loginMethods.add(new LoginMethod(tenants, "emailpassword", true, true, currentTimeMillis, email, "$2a", "BCRYPT", null, null, null)); + loginMethods.add(new LoginMethod(tenants, "thirdparty", true, false, currentTimeMillis, email, null, null, "thirdPartyId" + i, "thirdPartyUserId" + i, null)); + loginMethods.add(new LoginMethod(tenants, "passwordless", true, false, currentTimeMillis, email, null, null, null, null, null)); users.add(new BulkImportUser(id, externalId, userMetadata, userRoles, totpDevices, loginMethods)); } return users; diff --git a/src/test/java/io/supertokens/test/bulkimport/ProcessBulkImportUsersCronJobTest.java b/src/test/java/io/supertokens/test/bulkimport/ProcessBulkImportUsersCronJobTest.java index b71372d5e..8e29c106a 100644 --- a/src/test/java/io/supertokens/test/bulkimport/ProcessBulkImportUsersCronJobTest.java +++ b/src/test/java/io/supertokens/test/bulkimport/ProcessBulkImportUsersCronJobTest.java @@ -29,8 +29,9 @@ import io.supertokens.featureflag.exceptions.FeatureNotEnabledException; import io.supertokens.multitenancy.Multitenancy; import io.supertokens.multitenancy.exception.BadPermissionException; -import io.supertokens.multitenancy.exception.CannotModifyBaseConfigException; +import io.supertokens.multitenancy.exception.CannotModifyBaseConfigException; import io.supertokens.pluginInterface.STORAGE_TYPE; +import io.supertokens.pluginInterface.Storage; import io.supertokens.pluginInterface.authRecipe.AuthRecipeUserInfo; import io.supertokens.pluginInterface.authRecipe.LoginMethod; import io.supertokens.pluginInterface.bulkimport.BulkImportUser; @@ -89,7 +90,7 @@ public void beforeEach() { } @Test - public void shouldProcessBulkImportUsers() throws Exception { + public void shouldProcessBulkImportUsersInTheSameTenant() throws Exception { TestingProcess process = startCronProcess(); Main main = process.getProcess(); @@ -115,7 +116,6 @@ public void shouldProcessBulkImportUsers() throws Exception { List usersAfterProcessing = storage.getBulkImportUsers(appIdentifier, null, null, null, null); - System.out.println("Users after processing: " + usersAfterProcessing.size()); assertEquals(0, usersAfterProcessing.size()); UserPaginationContainer container = AuthRecipe.getUsers(main, 100, "ASC", null, null, null); @@ -123,29 +123,65 @@ public void shouldProcessBulkImportUsers() throws Exception { UserIdMapping.populateExternalUserIdForUsers(appIdentifier, storage, container.users); - for (AuthRecipeUserInfo user : container.users) { - for (LoginMethod lm1 : user.loginMethods) { - bulkImportUser.loginMethods.forEach(lm2 -> { - if (lm2.recipeId.equals(lm1.recipeId.toString())) { - assertLoginMethodEquals(lm1, lm2); - } - }); - } + TenantIdentifier publicTenant = new TenantIdentifier(null, null, "public"); - JsonObject createdUserMetadata = UserMetadata.getUserMetadata(main, user.getSupertokensOrExternalUserId()); - assertEquals(bulkImportUser.userMetadata, createdUserMetadata); + assertBulkImportUserAndAuthRecipeUserAreEqual(appIdentifier, publicTenant, storage, bulkImportUser, + container.users[0]); - String[] createdUserRoles = UserRoles.getRolesForUser(main, user.getSupertokensOrExternalUserId()); - String[] bulkImportUserRoles = bulkImportUser.userRoles.stream().map(r -> r.role).toArray(String[]::new); - assertArrayEquals(bulkImportUserRoles, createdUserRoles); - - assertEquals(bulkImportUser.externalUserId, user.getSupertokensOrExternalUserId()); + process.kill(); + assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STOPPED)); + } + @Test + public void shouldProcessBulkImportUsersInMultipleTenantsWithDifferentStorages() throws Exception { + TestingProcess process = startCronProcess(); + Main main = process.getProcess(); - TOTPDevice[] createdTotpDevices = Totp.getDevices(main, user.getSupertokensOrExternalUserId()); - assertTotpDevicesEquals(createdTotpDevices, bulkImportUser.totpDevices.toArray(new TotpDevice[0])); + // Create user roles before inserting bulk users + { + UserRoles.createNewRoleOrModifyItsPermissions(main, "role1", null); + UserRoles.createNewRoleOrModifyItsPermissions(main, "role2", null); } + createTenants(main); + + TenantIdentifier t1 = new TenantIdentifier(null, null, "t1"); + TenantIdentifier t2 = new TenantIdentifier(null, null, "t2"); + + BulkImportSQLStorage storage = (BulkImportSQLStorage) StorageLayer.getStorage(main); + AppIdentifier appIdentifier = new AppIdentifier(null, null); + + List usersT1 = generateBulkImportUser(1, List.of(t1.getTenantId()), 0); + List usersT2 = generateBulkImportUser(1, List.of(t2.getTenantId()), 1); + + BulkImportUser bulkImportUserT1 = usersT1.get(0); + BulkImportUser bulkImportUserT2 = usersT2.get(0); + + BulkImport.addUsers(appIdentifier, storage, List.of(bulkImportUserT1, bulkImportUserT2)); + + Thread.sleep(6000); + + List usersAfterProcessing = storage.getBulkImportUsers(appIdentifier, null, null, + null, null); + + assertEquals(0, usersAfterProcessing.size()); + + Storage storageT1 = StorageLayer.getStorage(t1, main); + Storage storageT2 = StorageLayer.getStorage(t2, main); + + UserPaginationContainer containerT1 = AuthRecipe.getUsers(t1, storageT1, 100, "ASC", null, null, null); + UserPaginationContainer containerT2 = AuthRecipe.getUsers(t2, storageT2, 100, "ASC", null, null, null); + + assertEquals(usersT1.size() + usersT2.size(), containerT1.users.length + containerT2.users.length); + + UserIdMapping.populateExternalUserIdForUsers(appIdentifier, storageT1, containerT1.users); + UserIdMapping.populateExternalUserIdForUsers(appIdentifier, storageT2, containerT2.users); + + assertBulkImportUserAndAuthRecipeUserAreEqual(appIdentifier, t1, storageT1, bulkImportUserT1, + containerT1.users[0]); + assertBulkImportUserAndAuthRecipeUserAreEqual(appIdentifier, t2, storageT2, bulkImportUserT2, + containerT2.users[0]); + process.kill(); assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STOPPED)); } @@ -228,6 +264,30 @@ private TestingProcess startCronProcess() throws InterruptedException { return process; } + private void assertBulkImportUserAndAuthRecipeUserAreEqual(AppIdentifier appIdentifier, + TenantIdentifier tenantIdentifier, Storage storage, BulkImportUser bulkImportUser, + AuthRecipeUserInfo authRecipeUser) throws StorageQueryException, TenantOrAppNotFoundException { + for (LoginMethod lm1 : authRecipeUser.loginMethods) { + bulkImportUser.loginMethods.forEach(lm2 -> { + if (lm2.recipeId.equals(lm1.recipeId.toString())) { + assertLoginMethodEquals(lm1, lm2); + } + }); + } + assertEquals(bulkImportUser.externalUserId, authRecipeUser.getSupertokensOrExternalUserId()); + assertEquals(bulkImportUser.userMetadata, + UserMetadata.getUserMetadata(appIdentifier, storage, authRecipeUser.getSupertokensOrExternalUserId())); + + String[] createdUserRoles = UserRoles.getRolesForUser(tenantIdentifier, storage, + authRecipeUser.getSupertokensOrExternalUserId()); + String[] bulkImportUserRoles = bulkImportUser.userRoles.stream().map(r -> r.role).toArray(String[]::new); + assertArrayEquals(bulkImportUserRoles, createdUserRoles); + + TOTPDevice[] createdTotpDevices = Totp.getDevices(appIdentifier, storage, + authRecipeUser.getSupertokensOrExternalUserId()); + assertTotpDevicesEquals(createdTotpDevices, bulkImportUser.totpDevices.toArray(new TotpDevice[0])); + } + private void assertLoginMethodEquals(LoginMethod lm1, io.supertokens.pluginInterface.bulkimport.BulkImportUser.LoginMethod lm2) { assertEquals(lm1.email, lm2.email); diff --git a/src/test/java/io/supertokens/test/bulkimport/apis/AddBulkImportUsersTest.java b/src/test/java/io/supertokens/test/bulkimport/apis/AddBulkImportUsersTest.java index d38c18de0..774d841f3 100644 --- a/src/test/java/io/supertokens/test/bulkimport/apis/AddBulkImportUsersTest.java +++ b/src/test/java/io/supertokens/test/bulkimport/apis/AddBulkImportUsersTest.java @@ -60,6 +60,8 @@ import io.supertokens.userroles.UserRoles; public class AddBulkImportUsersTest { + private String genericErrMsg = "Data has missing or invalid fields. Please check the users field for more details."; + @Rule public TestRule watchman = Utils.getOnFailure(); @@ -73,459 +75,376 @@ public void beforeEach() { Utils.reset(); } - public String getResponseMessageFromError(String response) { - return response.substring(response.indexOf("Message: ") + "Message: ".length()); + @Test + public void shouldThrow400IfUsersAreMissingInRequestBody() throws Exception { + TestingProcessManager.TestingProcess process = TestingProcessManager.start(new String[] { "../" }); + assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); + + if (StorageLayer.getStorage(process.getProcess()).getType() != STORAGE_TYPE.SQL) { + return; + } + + // CASE 1: users field is not present + testBadRequest(process.getProcess(), new JsonObject(), "Field name 'users' is invalid in JSON input"); + + // CASE 2: users field type in incorrect + testBadRequest(process.getProcess(), new JsonParser().parse("{\"users\": \"string\"}").getAsJsonObject(), + "Field name 'users' is invalid in JSON input"); + + // CASE 3: users array is empty + testBadRequest(process.getProcess(), generateUsersJson(0).getAsJsonObject(), + "{\"error\":\"You need to add at least one user.\"}"); + + // CASE 4: users array length is greater than 10000 + testBadRequest(process.getProcess(), generateUsersJson(10001).getAsJsonObject(), + "{\"error\":\"You can only add 10000 users at a time.\"}"); + + process.kill(); + Assert.assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STOPPED)); } @Test - public void shouldThrow400Error() throws Exception { - String[] args = { "../" }; + public void shouldThrow400IfLoginMethodsAreMissingInUserObject() throws Exception { + TestingProcessManager.TestingProcess process = TestingProcessManager.start(new String[] { "../" }); + assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); - TestingProcessManager.TestingProcess process = TestingProcessManager.start(args); + if (StorageLayer.getStorage(process.getProcess()).getType() != STORAGE_TYPE.SQL) { + return; + } + + // CASE 1: loginMethods field is not present + testBadRequest(process.getProcess(), new JsonParser().parse("{\"users\":[{}]}").getAsJsonObject(), + "{\"error\":\"" + genericErrMsg + + "\",\"users\":[{\"index\":0,\"errors\":[\"loginMethods is required.\"]}]}"); + + // CASE 2: loginMethods field type in incorrect + testBadRequest(process.getProcess(), + new JsonParser().parse("{\"users\":[{\"loginMethods\": \"string\"}]}").getAsJsonObject(), + "{\"error\":\"" + genericErrMsg + + "\",\"users\":[{\"index\":0,\"errors\":[\"loginMethods should be of type array of object.\"]}]}"); + + // CASE 3: loginMethods array is empty + testBadRequest(process.getProcess(), + new JsonParser().parse("{\"users\":[{\"loginMethods\": []}]}").getAsJsonObject(), + "{\"error\":\"" + genericErrMsg + + "\",\"users\":[{\"index\":0,\"errors\":[\"At least one loginMethod is required.\"]}]}"); + + process.kill(); + Assert.assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STOPPED)); + } + + @Test + public void shouldThrow400IfNonRequiredFieldsHaveInvalidType() throws Exception { + TestingProcessManager.TestingProcess process = TestingProcessManager.start(new String[] { "../" }); assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); - + if (StorageLayer.getStorage(process.getProcess()).getType() != STORAGE_TYPE.SQL) { return; } - // Create user roles - { - UserRoles.createNewRoleOrModifyItsPermissions(process.getProcess(), "role1", null); + JsonObject requestBody = new JsonParser() + .parse("{\"users\":[{\"externalUserId\":[],\"userMetaData\":[],\"userRoles\":{},\"totpDevices\":{}}]}") + .getAsJsonObject(); + + testBadRequest(process.getProcess(), requestBody, + "{\"error\":\"" + genericErrMsg + + "\",\"users\":[{\"index\":0,\"errors\":[\"externalUserId should be of type string.\",\"userRoles should be of type array of object.\",\"totpDevices should be of type array of object.\",\"loginMethods is required.\"]}]}"); + + process.kill(); + Assert.assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STOPPED)); + } + + @Test + public void shouldThrow400IfNonUniqueExternalIdsArePassed() throws Exception { + TestingProcessManager.TestingProcess process = TestingProcessManager.start(new String[] { "../" }); + assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); + + if (StorageLayer.getStorage(process.getProcess()).getType() != STORAGE_TYPE.SQL) { + return; } + JsonObject requestBody = new JsonParser() + .parse("{\"users\":[{\"externalUserId\":\"id1\"}, {\"externalUserId\":\"id1\"}]}") + .getAsJsonObject(); - String genericErrMsg = "Data has missing or invalid fields. Please check the users field for more details."; + testBadRequest(process.getProcess(), requestBody, "{\"error\":\"" + genericErrMsg + + "\",\"users\":[{\"index\":0,\"errors\":[\"loginMethods is required.\"]},{\"index\":1,\"errors\":[\"loginMethods is required.\",\"externalUserId id1 is not unique. It is already used by another user.\"]}]}"); - // users is required in the json body - { - // CASE 1: users field is not present - try { - JsonObject request = new JsonObject(); - HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", - "http://localhost:3567/bulk-import/users", - request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); - fail("The API should have thrown an error"); - } catch (io.supertokens.test.httpRequest.HttpResponseException e) { - String responseString = getResponseMessageFromError(e.getMessage()); - assertEquals(400, e.statusCode); - assertEquals(responseString, "Field name 'users' is invalid in JSON input"); - } - // CASE 2: users field type in incorrect - try { - JsonObject request = new JsonParser().parse("{\"users\": \"string\"}").getAsJsonObject(); - HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", - "http://localhost:3567/bulk-import/users", - request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); - fail("The API should have thrown an error"); - } catch (io.supertokens.test.httpRequest.HttpResponseException e) { - String responseString = getResponseMessageFromError(e.getMessage()); - assertEquals(400, e.statusCode); - assertEquals(responseString, "Field name 'users' is invalid in JSON input"); - } + process.kill(); + Assert.assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STOPPED)); + } + + @Test + public void shouldThrow400IfTotpDevicesAreNotPassedCorrectly() throws Exception { + TestingProcessManager.TestingProcess process = TestingProcessManager.start(new String[] { "../" }); + assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); + + if (StorageLayer.getStorage(process.getProcess()).getType() != STORAGE_TYPE.SQL) { + return; } - // loginMethod array is required in the user object - { - // CASE 1: loginMethods field is not present - try { - JsonObject request = new JsonParser().parse("{\"users\":[{}]}").getAsJsonObject(); - HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", - "http://localhost:3567/bulk-import/users", - request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); - fail("The API should have thrown an error"); - } catch (io.supertokens.test.httpRequest.HttpResponseException e) { - String responseString = getResponseMessageFromError(e.getMessage()); - assertEquals(400, e.statusCode); - assertEquals(responseString, "{\"error\":\"" + genericErrMsg + "\",\"users\":[{\"index\":0,\"errors\":[\"loginMethods is required.\"]}]}"); - } - // CASE 2: loginMethods field type in incorrect - try { - JsonObject request = new JsonParser().parse("{\"users\":[{\"loginMethods\": \"string\"}]}") - .getAsJsonObject(); - HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", - "http://localhost:3567/bulk-import/users", - request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); - fail("The API should have thrown an error"); - } catch (io.supertokens.test.httpRequest.HttpResponseException e) { - String responseString = getResponseMessageFromError(e.getMessage()); - assertEquals(400, e.statusCode); - assertEquals(responseString, - "{\"error\":\"" + genericErrMsg + "\",\"users\":[{\"index\":0,\"errors\":[\"loginMethods should be of type array of object.\"]}]}"); - } - // CASE 3: loginMethods array is empty - try { - JsonObject request = new JsonParser().parse("{\"users\":[{\"loginMethods\": []}]}").getAsJsonObject(); - HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", - "http://localhost:3567/bulk-import/users", - request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); - fail("The API should have thrown an error"); - } catch (io.supertokens.test.httpRequest.HttpResponseException e) { - String responseString = getResponseMessageFromError(e.getMessage()); - assertEquals(400, e.statusCode); - assertEquals(responseString, "{\"error\":\"" + genericErrMsg + "\",\"users\":[{\"index\":0,\"errors\":[\"At least one loginMethod is required.\"]}]}"); - } + + // CASE 1: MFA must be enabled to import totp devices + JsonObject requestBody = new JsonParser() + .parse("{\"users\":[{\"totpDevices\":[{\"secret\": \"secret\"}]}]}") + .getAsJsonObject(); + + testBadRequest(process.getProcess(), requestBody, "{\"error\":\"" + genericErrMsg + + "\",\"users\":[{\"index\":0,\"errors\":[\"MFA must be enabled to import totp devices.\",\"loginMethods is required.\"]}]}"); + + // CASE 2: secretKey is required in totpDevices + setFeatureFlags(process.getProcess(), new EE_FEATURES[] { EE_FEATURES.MFA }); + testBadRequest(process.getProcess(), requestBody, "{\"error\":\"" + genericErrMsg + + "\",\"users\":[{\"index\":0,\"errors\":[\"secretKey is required for a totp device.\",\"loginMethods is required.\"]}]}"); + + process.kill(); + Assert.assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STOPPED)); + } + + @Test + public void shouldThrow400IfUserRolesAreNotPassedCorrectly() throws Exception { + TestingProcessManager.TestingProcess process = TestingProcessManager.start(new String[] { "../" }); + assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); + + if (StorageLayer.getStorage(process.getProcess()).getType() != STORAGE_TYPE.SQL) { + return; } - // Invalid field type of non required fields outside loginMethod + + // Create user roles { - try { - JsonObject request = new JsonParser() - .parse("{\"users\":[{\"externalUserId\":[],\"userMetaData\":[],\"userRoles\":{},\"totpDevices\":{}}]}") - .getAsJsonObject(); - HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", - "http://localhost:3567/bulk-import/users", - request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); - fail("The API should have thrown an error"); - } catch (io.supertokens.test.httpRequest.HttpResponseException e) { - String responseString = getResponseMessageFromError(e.getMessage()); - assertEquals(400, e.statusCode); - assertEquals(responseString, - "{\"error\":\"" + genericErrMsg + "\",\"users\":[{\"index\":0,\"errors\":[\"externalUserId should be of type string.\",\"userRoles should be of type array of object.\",\"totpDevices should be of type array of object.\",\"loginMethods is required.\"]}]}"); - } - // Non-unique externalUserIds - try { - JsonObject request = new JsonParser() - .parse("{\"users\":[{\"externalUserId\":\"id1\"}, {\"externalUserId\":\"id1\"}]}") - .getAsJsonObject(); - HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", - "http://localhost:3567/bulk-import/users", - request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); - fail("The API should have thrown an error"); - } catch (io.supertokens.test.httpRequest.HttpResponseException e) { - String responseString = getResponseMessageFromError(e.getMessage()); - assertEquals(400, e.statusCode); - assertEquals(responseString, - "{\"error\":\"" + genericErrMsg + "\",\"users\":[{\"index\":0,\"errors\":[\"loginMethods is required.\"]},{\"index\":1,\"errors\":[\"loginMethods is required.\",\"externalUserId id1 is not unique. It is already used by another user.\"]}]}"); - } - // MFA must be enabled to import totpDevices - try { - JsonObject request = new JsonParser() - .parse("{\"users\":[{\"totpDevices\":[{\"secret\": \"secret\"}]}]}") - .getAsJsonObject(); - HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", - "http://localhost:3567/bulk-import/users", - request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); - fail("The API should have thrown an error"); - } catch (io.supertokens.test.httpRequest.HttpResponseException e) { - String responseString = getResponseMessageFromError(e.getMessage()); - assertEquals(400, e.statusCode); - assertEquals(responseString, - "{\"error\":\"" + genericErrMsg + "\",\"users\":[{\"index\":0,\"errors\":[\"MFA must be enabled to import totp devices.\",\"loginMethods is required.\"]}]}"); - } - // secretKey is required in totpDevices - try { - setFeatureFlags(process.getProcess(), new EE_FEATURES[]{EE_FEATURES.MFA}); - JsonObject request = new JsonParser() - .parse("{\"users\":[{\"totpDevices\":[{\"secret\": \"secret\"}]}]}") - .getAsJsonObject(); - HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", - "http://localhost:3567/bulk-import/users", - request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); - fail("The API should have thrown an error"); - } catch (io.supertokens.test.httpRequest.HttpResponseException e) { - String responseString = getResponseMessageFromError(e.getMessage()); - assertEquals(400, e.statusCode); - assertEquals(responseString, - "{\"error\":\"" + genericErrMsg + "\",\"users\":[{\"index\":0,\"errors\":[\"secretKey is required for a totp device.\",\"loginMethods is required.\"]}]}"); - } - // Invalid role (tenantIds is required) - try { - JsonObject request = new JsonParser() - .parse("{\"users\":[{\"userRoles\":[{\"role\":\"role1\"}]}]}") - .getAsJsonObject(); - HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", - "http://localhost:3567/bulk-import/users", - request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); - fail("The API should have thrown an error"); - } catch (io.supertokens.test.httpRequest.HttpResponseException e) { - String responseString = getResponseMessageFromError(e.getMessage()); - assertEquals(400, e.statusCode); - - assertEquals(responseString, - "{\"error\":\"" + genericErrMsg + "\",\"users\":[{\"index\":0,\"errors\":[\"tenantIds is required for a user role.\",\"loginMethods is required.\"]}]}"); - } - // Invalid role (role doesn't exist) - try { - JsonObject request = new JsonParser() - .parse("{\"users\":[{\"userRoles\":[{\"role\":\"role5\", \"tenantIds\": [\"public\"]}]}]}") - .getAsJsonObject(); - HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", - "http://localhost:3567/bulk-import/users", - request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); - fail("The API should have thrown an error"); - } catch (io.supertokens.test.httpRequest.HttpResponseException e) { - String responseString = getResponseMessageFromError(e.getMessage()); - assertEquals(400, e.statusCode); - - assertEquals(responseString, - "{\"error\":\"" + genericErrMsg + "\",\"users\":[{\"index\":0,\"errors\":[\"Role role5 does not exist.\",\"loginMethods is required.\"]}]}"); - } + UserRoles.createNewRoleOrModifyItsPermissions(process.getProcess(), "role1", null); } - // Invalid field type of non required fields inside loginMethod - { - try { - JsonObject request = new JsonParser().parse( - "{\"users\":[{\"loginMethods\":[{\"recipeId\":[],\"tenantIds\":{},\"isPrimary\":[],\"isVerified\":[],\"timeJoinedInMSSinceEpoch\":[]}]}]}") - .getAsJsonObject(); - HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", - "http://localhost:3567/bulk-import/users", - request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); - fail("The API should have thrown an error"); - } catch (io.supertokens.test.httpRequest.HttpResponseException e) { - String responseString = getResponseMessageFromError(e.getMessage()); - assertEquals(400, e.statusCode); - assertEquals(responseString, - "{\"error\":\"" + genericErrMsg + "\",\"users\":[{\"index\":0,\"errors\":[\"recipeId should be of type string for a loginMethod.\",\"tenantIds should be of type array of string for a loginMethod.\",\"isVerified should be of type boolean for a loginMethod.\",\"isPrimary should be of type boolean for a loginMethod.\",\"timeJoinedInMSSinceEpoch should be of type integer for a loginMethod\"]}]}"); - } + + // CASE 1: tenantIds is required for a user role + JsonObject requestBody = new JsonParser() + .parse("{\"users\":[{\"userRoles\":[{\"role\":\"role1\"}]}]}") + .getAsJsonObject(); + + testBadRequest(process.getProcess(), requestBody, "{\"error\":\"" + genericErrMsg + + "\",\"users\":[{\"index\":0,\"errors\":[\"tenantIds is required for a user role.\",\"loginMethods is required.\"]}]}"); + + // CASE 2: Role doesn't exist + JsonObject requestBody2 = new JsonParser() + .parse("{\"users\":[{\"userRoles\":[{\"role\":\"role5\", \"tenantIds\": [\"public\"]}]}]}") + .getAsJsonObject(); + + testBadRequest(process.getProcess(), requestBody2, "{\"error\":\"" + genericErrMsg + + "\",\"users\":[{\"index\":0,\"errors\":[\"Role role5 does not exist.\",\"loginMethods is required.\"]}]}"); + + process.kill(); + Assert.assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STOPPED)); + } + + @Test + public void shouldThrow400IfLoginMethodsHaveInvalidFieldType() throws Exception { + TestingProcessManager.TestingProcess process = TestingProcessManager.start(new String[] { "../" }); + assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); + + if (StorageLayer.getStorage(process.getProcess()).getType() != STORAGE_TYPE.SQL) { + return; } - // Invalid recipeId - { - try { - JsonObject request = new JsonParser() - .parse("{\"users\":[{\"loginMethods\":[{\"recipeId\":\"invalid_recipe_id\"}]}]}") - .getAsJsonObject(); - HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", - "http://localhost:3567/bulk-import/users", - request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); - fail("The API should have thrown an error"); - } catch (io.supertokens.test.httpRequest.HttpResponseException e) { - String responseString = getResponseMessageFromError(e.getMessage()); - assertEquals(400, e.statusCode); - assertEquals(responseString, - "{\"error\":\"" + genericErrMsg + "\",\"users\":[{\"index\":0,\"errors\":[\"Invalid recipeId for loginMethod. Pass one of emailpassword, thirdparty or, passwordless!\"]}]}"); - } + + // CASE 1: Field type is invalid + JsonObject requestBody = new JsonParser() + .parse( + "{\"users\":[{\"loginMethods\":[{\"recipeId\":[],\"tenantIds\":{},\"isPrimary\":[],\"isVerified\":[],\"timeJoinedInMSSinceEpoch\":[]}]}]}") + .getAsJsonObject(); + + testBadRequest(process.getProcess(), requestBody, "{\"error\":\"" + genericErrMsg + + "\",\"users\":[{\"index\":0,\"errors\":[\"recipeId should be of type string for a loginMethod.\",\"tenantIds should be of type array of string for a loginMethod.\",\"isVerified should be of type boolean for a loginMethod.\",\"isPrimary should be of type boolean for a loginMethod.\",\"timeJoinedInMSSinceEpoch should be of type integer for a loginMethod\"]}]}"); + + // CASE 2: recipeId is invalid + JsonObject requestBody2 = new JsonParser() + .parse("{\"users\":[{\"loginMethods\":[{\"recipeId\":\"invalid_recipe_id\"}]}]}") + .getAsJsonObject(); + + testBadRequest(process.getProcess(), requestBody2, "{\"error\":\"" + genericErrMsg + + "\",\"users\":[{\"index\":0,\"errors\":[\"Invalid recipeId for loginMethod. Pass one of emailpassword, thirdparty or, passwordless!\"]}]}"); + + process.kill(); + Assert.assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STOPPED)); + } + + @Test + public void shouldThrow400IfEmailPasswordRecipeHasInvalidFieldTypes() throws Exception { + TestingProcessManager.TestingProcess process = TestingProcessManager.start(new String[] { "../" }); + assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); + + if (StorageLayer.getStorage(process.getProcess()).getType() != STORAGE_TYPE.SQL) { + return; } - // Invalid field type in emailpassword recipe - { - // CASE 1: email, passwordHash and hashingAlgorithm are not present - try { - JsonObject request = new JsonParser() - .parse("{\"users\":[{\"loginMethods\":[{\"recipeId\":\"emailpassword\"}]}]}").getAsJsonObject(); - HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", - "http://localhost:3567/bulk-import/users", - request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); - fail("The API should have thrown an error"); - } catch (io.supertokens.test.httpRequest.HttpResponseException e) { - String responseString = getResponseMessageFromError(e.getMessage()); - assertEquals(400, e.statusCode); - assertEquals(responseString, - "{\"error\":\"" + genericErrMsg + "\",\"users\":[{\"index\":0,\"errors\":[\"email is required for an emailpassword recipe.\",\"passwordHash is required for an emailpassword recipe.\",\"hashingAlgorithm is required for an emailpassword recipe.\"]}]}"); - } - // CASE 2: email, passwordHash and hashingAlgorithm field type is incorrect - try { - JsonObject request = new JsonParser().parse( + + // CASE 1: email, passwordHash and hashingAlgorithm are not present + JsonObject requestBody = new JsonParser() + .parse("{\"users\":[{\"loginMethods\":[{\"recipeId\":\"emailpassword\"}]}]}") + .getAsJsonObject(); + + testBadRequest(process.getProcess(), requestBody, "{\"error\":\"" + genericErrMsg + + "\",\"users\":[{\"index\":0,\"errors\":[\"email is required for an emailpassword recipe.\",\"passwordHash is required for an emailpassword recipe.\",\"hashingAlgorithm is required for an emailpassword recipe.\"]}]}"); + + // CASE 2: email, passwordHash and hashingAlgorithm field type is incorrect + JsonObject requestBody2 = new JsonParser() + .parse( "{\"users\":[{\"loginMethods\":[{\"recipeId\":\"emailpassword\",\"email\":[],\"passwordHash\":[],\"hashingAlgorithm\":[]}]}]}") - .getAsJsonObject(); - HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", - "http://localhost:3567/bulk-import/users", - request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); - fail("The API should have thrown an error"); - } catch (io.supertokens.test.httpRequest.HttpResponseException e) { - String responseString = getResponseMessageFromError(e.getMessage()); - assertEquals(400, e.statusCode); - assertEquals(responseString, - "{\"error\":\"" + genericErrMsg + "\",\"users\":[{\"index\":0,\"errors\":[\"email should be of type string for an emailpassword recipe.\",\"passwordHash should be of type string for an emailpassword recipe.\",\"hashingAlgorithm should be of type string for an emailpassword recipe.\"]}]}"); - } - // CASE 3: hashingAlgorithm is not one of bcrypt, argon2, firebase_scrypt - try { - JsonObject request = new JsonParser().parse( + .getAsJsonObject(); + + testBadRequest(process.getProcess(), requestBody2, "{\"error\":\"" + genericErrMsg + + "\",\"users\":[{\"index\":0,\"errors\":[\"email should be of type string for an emailpassword recipe.\",\"passwordHash should be of type string for an emailpassword recipe.\",\"hashingAlgorithm should be of type string for an emailpassword recipe.\"]}]}"); + + // CASE 3: hashingAlgorithm is not one of bcrypt, argon2, firebase_scrypt + JsonObject requestBody3 = new JsonParser() + .parse( "{\"users\":[{\"loginMethods\":[{\"recipeId\":\"emailpassword\",\"email\":\"johndoe@gmail.com\",\"passwordHash\":\"$2a\",\"hashingAlgorithm\":\"invalid_algorithm\"}]}]}") - .getAsJsonObject(); - HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", - "http://localhost:3567/bulk-import/users", - request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); - fail("The API should have thrown an error"); - } catch (io.supertokens.test.httpRequest.HttpResponseException e) { - String responseString = getResponseMessageFromError(e.getMessage()); - assertEquals(400, e.statusCode); - assertEquals(responseString, - "{\"error\":\"" + genericErrMsg + "\",\"users\":[{\"index\":0,\"errors\":[\"Invalid hashingAlgorithm for emailpassword recipe. Pass one of bcrypt, argon2 or, firebase_scrypt!\"]}]}"); - } + .getAsJsonObject(); + + testBadRequest(process.getProcess(), requestBody3, "{\"error\":\"" + genericErrMsg + + "\",\"users\":[{\"index\":0,\"errors\":[\"Invalid hashingAlgorithm for emailpassword recipe. Pass one of bcrypt, argon2 or, firebase_scrypt!\"]}]}"); + + process.kill(); + Assert.assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STOPPED)); + } + + @Test + public void shouldThrow400IfThirdPartyRecipeHasInvalidFieldTypes() throws Exception { + TestingProcessManager.TestingProcess process = TestingProcessManager.start(new String[] { "../" }); + assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); + + if (StorageLayer.getStorage(process.getProcess()).getType() != STORAGE_TYPE.SQL) { + return; } - // Invalid field type in thirdparty recipe - { - // CASE 1: email, thirdPartyId and thirdPartyUserId are not present - try { - JsonObject request = new JsonParser() - .parse("{\"users\":[{\"loginMethods\":[{\"recipeId\":\"thirdparty\"}]}]}").getAsJsonObject(); - HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", - "http://localhost:3567/bulk-import/users", - request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); - fail("The API should have thrown an error"); - } catch (io.supertokens.test.httpRequest.HttpResponseException e) { - String responseString = getResponseMessageFromError(e.getMessage()); - assertEquals(400, e.statusCode); - assertEquals(responseString, - "{\"error\":\"" + genericErrMsg + "\",\"users\":[{\"index\":0,\"errors\":[\"email is required for a thirdparty recipe.\",\"thirdPartyId is required for a thirdparty recipe.\",\"thirdPartyUserId is required for a thirdparty recipe.\"]}]}"); - } - // CASE 2: email, passwordHash and thirdPartyUserId field type is incorrect - try { - JsonObject request = new JsonParser().parse( + + // CASE 1: email, thirdPartyId and thirdPartyUserId are not present + JsonObject requestBody = new JsonParser() + .parse("{\"users\":[{\"loginMethods\":[{\"recipeId\":\"thirdparty\"}]}]}") + .getAsJsonObject(); + + testBadRequest(process.getProcess(), requestBody, "{\"error\":\"" + genericErrMsg + + "\",\"users\":[{\"index\":0,\"errors\":[\"email is required for a thirdparty recipe.\",\"thirdPartyId is required for a thirdparty recipe.\",\"thirdPartyUserId is required for a thirdparty recipe.\"]}]}"); + + // CASE 2: email, passwordHash and thirdPartyUserId field type is incorrect + JsonObject requestBody2 = new JsonParser() + .parse( "{\"users\":[{\"loginMethods\":[{\"recipeId\":\"thirdparty\",\"email\":[],\"thirdPartyId\":[],\"thirdPartyUserId\":[]}]}]}") - .getAsJsonObject(); - HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", - "http://localhost:3567/bulk-import/users", - request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); - fail("The API should have thrown an error"); - } catch (io.supertokens.test.httpRequest.HttpResponseException e) { - String responseString = getResponseMessageFromError(e.getMessage()); - assertEquals(400, e.statusCode); - assertEquals(responseString, - "{\"error\":\"" + genericErrMsg + "\",\"users\":[{\"index\":0,\"errors\":[\"email should be of type string for a thirdparty recipe.\",\"thirdPartyId should be of type string for a thirdparty recipe.\",\"thirdPartyUserId should be of type string for a thirdparty recipe.\"]}]}"); - } + .getAsJsonObject(); + + testBadRequest(process.getProcess(), requestBody2, "{\"error\":\"" + genericErrMsg + + "\",\"users\":[{\"index\":0,\"errors\":[\"email should be of type string for a thirdparty recipe.\",\"thirdPartyId should be of type string for a thirdparty recipe.\",\"thirdPartyUserId should be of type string for a thirdparty recipe.\"]}]}"); + + process.kill(); + Assert.assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STOPPED)); + } + + @Test + public void shouldThrow400IfPasswordlessRecipeHasInvalidFieldTypes() throws Exception { + TestingProcessManager.TestingProcess process = TestingProcessManager.start(new String[] { "../" }); + assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); + + if (StorageLayer.getStorage(process.getProcess()).getType() != STORAGE_TYPE.SQL) { + return; } - // Invalid field type in passwordless recipe - { - // CASE 1: email and phoneNumber are not present - try { - JsonObject request = new JsonParser() - .parse("{\"users\":[{\"loginMethods\":[{\"recipeId\":\"passwordless\"}]}]}").getAsJsonObject(); - HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", - "http://localhost:3567/bulk-import/users", - request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); - fail("The API should have thrown an error"); - } catch (io.supertokens.test.httpRequest.HttpResponseException e) { - String responseString = getResponseMessageFromError(e.getMessage()); - assertEquals(400, e.statusCode); - assertEquals(responseString, - "{\"error\":\"" + genericErrMsg + "\",\"users\":[{\"index\":0,\"errors\":[\"Either email or phoneNumber is required for a passwordless recipe.\"]}]}"); - } - // CASE 2: email and phoneNumber field type is incorrect - try { - JsonObject request = new JsonParser().parse( + + // CASE 1: email and phoneNumber are not present + JsonObject requestBody = new JsonParser() + .parse("{\"users\":[{\"loginMethods\":[{\"recipeId\":\"passwordless\"}]}]}") + .getAsJsonObject(); + + testBadRequest(process.getProcess(), requestBody, "{\"error\":\"" + genericErrMsg + + "\",\"users\":[{\"index\":0,\"errors\":[\"Either email or phoneNumber is required for a passwordless recipe.\"]}]}"); + + // CASE 2: email and phoneNumber field type is incorrect + JsonObject requestBody2 = new JsonParser() + .parse( "{\"users\":[{\"loginMethods\":[{\"recipeId\":\"passwordless\",\"email\":[],\"phoneNumber\":[]}]}]}") - .getAsJsonObject(); - HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", - "http://localhost:3567/bulk-import/users", - request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); - fail("The API should have thrown an error"); - } catch (io.supertokens.test.httpRequest.HttpResponseException e) { - String responseString = getResponseMessageFromError(e.getMessage()); - assertEquals(400, e.statusCode); - assertEquals(responseString, - "{\"error\":\"" + genericErrMsg + "\",\"users\":[{\"index\":0,\"errors\":[\"email should be of type string for a passwordless recipe.\",\"phoneNumber should be of type string for a passwordless recipe.\",\"Either email or phoneNumber is required for a passwordless recipe.\"]}]}"); - } - } - // Disabling all feature flags to be able to get the desired error messages - { - setFeatureFlags(process.getProcess(), new EE_FEATURES[]{}); + .getAsJsonObject(); + + testBadRequest(process.getProcess(), requestBody2, "{\"error\":\"" + genericErrMsg + + "\",\"users\":[{\"index\":0,\"errors\":[\"email should be of type string for a passwordless recipe.\",\"phoneNumber should be of type string for a passwordless recipe.\",\"Either email or phoneNumber is required for a passwordless recipe.\"]}]}"); + + process.kill(); + Assert.assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STOPPED)); + } + + @Test + public void shouldThrow400IfAUserHasMultipleLoginMethodsAndAccountLinkingIsDisabled() throws Exception { + TestingProcessManager.TestingProcess process = TestingProcessManager.start(new String[] { "../" }); + assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); + + if (StorageLayer.getStorage(process.getProcess()).getType() != STORAGE_TYPE.SQL) { + return; } - // More than two loginMethods when either of account linking or MFA is not enabled - { - // CASE 1: email, passwordHash and hashingAlgorithm are not present - try { - JsonObject request = new JsonParser() - .parse("{\"users\":[{\"loginMethods\":[{\"recipeId\":\"emailpassword\",\"email\":\"johndoe@gmail.com\",\"passwordHash\":\"$2a\",\"hashingAlgorithm\":\"bcrypt\",\"isPrimary\":true},{\"recipeId\":\"passwordless\",\"email\":\"johndoe@gmail.com\"}]}]}").getAsJsonObject(); - HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", - "http://localhost:3567/bulk-import/users", - request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); - fail("The API should have thrown an error"); - } catch (io.supertokens.test.httpRequest.HttpResponseException e) { - String responseString = getResponseMessageFromError(e.getMessage()); - assertEquals(400, e.statusCode); - assertEquals(responseString, - "{\"error\":\"" + genericErrMsg + "\",\"users\":[{\"index\":0,\"errors\":[\"Account linking or MFA must be enabled to import multiple loginMethods.\"]}]}"); - } + JsonObject requestBody = new JsonParser() + .parse("{\"users\":[{\"loginMethods\":[{\"recipeId\":\"emailpassword\",\"email\":\"johndoe@gmail.com\",\"passwordHash\":\"$2a\",\"hashingAlgorithm\":\"bcrypt\",\"isPrimary\":true},{\"recipeId\":\"passwordless\",\"email\":\"johndoe@gmail.com\"}]}]}") + .getAsJsonObject(); + + testBadRequest(process.getProcess(), requestBody, "{\"error\":\"" + genericErrMsg + + "\",\"users\":[{\"index\":0,\"errors\":[\"Account linking must be enabled to import multiple loginMethods.\"]}]}"); + + process.kill(); + Assert.assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STOPPED)); + } + + @Test + public void shouldThrow400IfInvalidTenantIdIsPassed() throws Exception { + TestingProcessManager.TestingProcess process = TestingProcessManager.start(new String[] { "../" }); + assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); + + if (StorageLayer.getStorage(process.getProcess()).getType() != STORAGE_TYPE.SQL) { + return; } - // Validate tenantId - { - // CASE 1: Invalid tenantId when multitenancy is not enabled - try { - JsonObject request = new JsonParser().parse( + // CASE 1: Multitenancy is not enabled + JsonObject requestBody = new JsonParser() + .parse( "{\"users\":[{\"loginMethods\":[{\"tenantIds\":[\"invalid\"],\"recipeId\":\"passwordless\",\"email\":\"johndoe@gmail.com\"}]}]}") - .getAsJsonObject(); - HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", - "http://localhost:3567/bulk-import/users", - request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); - - fail("The API should have thrown an error"); - } catch (io.supertokens.test.httpRequest.HttpResponseException e) { - String responseString = getResponseMessageFromError(e.getMessage()); - assertEquals(400, e.statusCode); - assertEquals(responseString, - "{\"error\":\"" + genericErrMsg + "\",\"users\":[{\"index\":0,\"errors\":[\"Multitenancy must be enabled before importing users to a different tenant.\"]}]}"); - } - // Now enabling Account linking and Multitenancy for further tests - { - setFeatureFlags(process.getProcess(), new EE_FEATURES[]{EE_FEATURES.ACCOUNT_LINKING, EE_FEATURES.MULTI_TENANCY}); - } - // CASE 2: Invalid tenantId when multitenancy is enabled - try { - JsonObject request = new JsonParser().parse( + .getAsJsonObject(); + + testBadRequest(process.getProcess(), requestBody, "{\"error\":\"" + genericErrMsg + + "\",\"users\":[{\"index\":0,\"errors\":[\"Multitenancy must be enabled before importing users to a different tenant.\"]}]}"); + + // CASE 2: Invalid tenantId + setFeatureFlags(process.getProcess(), + new EE_FEATURES[] { EE_FEATURES.ACCOUNT_LINKING, EE_FEATURES.MULTI_TENANCY }); + + JsonObject requestBody2 = new JsonParser() + .parse( "{\"users\":[{\"loginMethods\":[{\"tenantIds\":[\"invalid\"],\"recipeId\":\"passwordless\",\"email\":\"johndoe@gmail.com\"}]}]}") - .getAsJsonObject(); - HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", - "http://localhost:3567/bulk-import/users", - request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); - fail("The API should have thrown an error"); - } catch (io.supertokens.test.httpRequest.HttpResponseException e) { - String responseString = getResponseMessageFromError(e.getMessage()); - assertEquals(400, e.statusCode); - assertEquals(responseString, - "{\"error\":\"" + genericErrMsg + "\",\"users\":[{\"index\":0,\"errors\":[\"Invalid tenantId: invalid for passwordless recipe.\"]}]}"); - } - // CASE 3. Two more tenants do not share the same storage - try { - createTenants(process.getProcess()); - - JsonObject request = new JsonParser().parse( - "{\"users\":[{\"loginMethods\":[{\"tenantIds\":[\"public\"],\"recipeId\":\"passwordless\",\"email\":\"johndoe@gmail.com\"}, {\"tenantIds\":[\"t2\"],\"recipeId\":\"thirdparty\", \"email\":\"johndoe@gmail.com\", \"thirdPartyId\":\"id\", \"thirdPartyUserId\":\"id\"}]}]}") - .getAsJsonObject(); - HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", - "http://localhost:3567/bulk-import/users", - request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); - fail("The API should have thrown an error"); - } catch (io.supertokens.test.httpRequest.HttpResponseException e) { - String responseString = getResponseMessageFromError(e.getMessage()); - assertEquals(400, e.statusCode); - assertEquals(responseString, - "{\"error\":\"" + genericErrMsg + "\",\"users\":[{\"index\":0,\"errors\":[\"All tenants for a user must share the same storage.\"]}]}"); - } - } - // No two loginMethods can have isPrimary as true - { - // CASE 1: email, passwordHash and hashingAlgorithm are not present - try { - JsonObject request = new JsonParser() - .parse("{\"users\":[{\"loginMethods\":[{\"recipeId\":\"emailpassword\",\"email\":\"johndoe@gmail.com\",\"passwordHash\":\"$2a\",\"hashingAlgorithm\":\"bcrypt\",\"isPrimary\":true},{\"recipeId\":\"passwordless\",\"email\":\"johndoe@gmail.com\",\"isPrimary\":true}]}]}").getAsJsonObject(); - HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", - "http://localhost:3567/bulk-import/users", - request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); - fail("The API should have thrown an error"); - } catch (io.supertokens.test.httpRequest.HttpResponseException e) { - String responseString = getResponseMessageFromError(e.getMessage()); - assertEquals(400, e.statusCode); - assertEquals(responseString, - "{\"error\":\"" + genericErrMsg + "\",\"users\":[{\"index\":0,\"errors\":[\"No two loginMethods can have isPrimary as true.\"]}]}"); - } - } - // Can't import less than 1 user at a time - { - try { - JsonObject request = generateUsersJson(0); - HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", - "http://localhost:3567/bulk-import/users", - request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); - fail("The API should have thrown an error"); - } catch (io.supertokens.test.httpRequest.HttpResponseException e) { - String responseString = getResponseMessageFromError(e.getMessage()); - assertEquals(400, e.statusCode); - assertEquals(responseString, "{\"error\":\"You need to add at least one user.\"}"); - } - } - // Can't import more than 10000 users at a time - { - try { - JsonObject request = generateUsersJson(10001); - HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", - "http://localhost:3567/bulk-import/users", - request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); - fail("The API should have thrown an error"); - } catch (io.supertokens.test.httpRequest.HttpResponseException e) { - String responseString = getResponseMessageFromError(e.getMessage()); - assertEquals(400, e.statusCode); - assertEquals(responseString, "{\"error\":\"You can only add 10000 users at a time.\"}"); - } + .getAsJsonObject(); + + testBadRequest(process.getProcess(), requestBody2, "{\"error\":\"" + genericErrMsg + + "\",\"users\":[{\"index\":0,\"errors\":[\"Invalid tenantId: invalid for passwordless recipe.\"]}]}"); + + // CASE 3: Two or more tenants do not share the same storage + + createTenants(process.getProcess()); + + JsonObject requestBody3 = new JsonParser().parse( + "{\"users\":[{\"loginMethods\":[{\"tenantIds\":[\"public\"],\"recipeId\":\"passwordless\",\"email\":\"johndoe@gmail.com\"}, {\"tenantIds\":[\"t2\"],\"recipeId\":\"thirdparty\", \"email\":\"johndoe@gmail.com\", \"thirdPartyId\":\"id\", \"thirdPartyUserId\":\"id\"}]}]}") + .getAsJsonObject(); + + testBadRequest(process.getProcess(), requestBody3, "{\"error\":\"" + genericErrMsg + + "\",\"users\":[{\"index\":0,\"errors\":[\"All tenants for a user must share the same storage.\"]}]}"); + + process.kill(); + Assert.assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STOPPED)); + } + + @Test + public void shouldThrow400IfTwoLoginMethodsHaveIsPrimaryTrue() throws Exception { + TestingProcessManager.TestingProcess process = TestingProcessManager.start(new String[] { "../" }); + assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); + + if (StorageLayer.getStorage(process.getProcess()).getType() != STORAGE_TYPE.SQL) { + return; } + setFeatureFlags(process.getProcess(), + new EE_FEATURES[] { EE_FEATURES.ACCOUNT_LINKING, EE_FEATURES.MULTI_TENANCY }); + + JsonObject requestBody = new JsonParser() + .parse("{\"users\":[{\"loginMethods\":[{\"recipeId\":\"emailpassword\",\"email\":\"johndoe@gmail.com\",\"passwordHash\":\"$2a\",\"hashingAlgorithm\":\"bcrypt\",\"isPrimary\":true},{\"recipeId\":\"passwordless\",\"email\":\"johndoe@gmail.com\",\"isPrimary\":true}]}]}") + .getAsJsonObject(); + + testBadRequest(process.getProcess(), requestBody, "{\"error\":\"" + genericErrMsg + + "\",\"users\":[{\"index\":0,\"errors\":[\"No two loginMethods can have isPrimary as true.\"]}]}"); + process.kill(); Assert.assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STOPPED)); } @@ -541,7 +460,7 @@ public void shouldReturn200Response() throws Exception { return; } - setFeatureFlags(process.getProcess(), new EE_FEATURES[]{EE_FEATURES.MFA}); + setFeatureFlags(process.getProcess(), new EE_FEATURES[] { EE_FEATURES.MFA }); // Create user roles before inserting bulk users { @@ -551,8 +470,8 @@ public void shouldReturn200Response() throws Exception { JsonObject request = generateUsersJson(10000); JsonObject response = HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", - "http://localhost:3567/bulk-import/users", - request, 1000, 10000, null, Utils.getCdiVersionStringLatestForTests(), null); + "http://localhost:3567/bulk-import/users", + request, 1000, 10000, null, Utils.getCdiVersionStringLatestForTests(), null); assertEquals("OK", response.get("status").getAsString()); process.kill(); @@ -570,7 +489,7 @@ public void shouldNormaliseFields() throws Exception { return; } - setFeatureFlags(process.getProcess(), new EE_FEATURES[]{EE_FEATURES.MFA}); + setFeatureFlags(process.getProcess(), new EE_FEATURES[] { EE_FEATURES.MFA }); // Create user roles before inserting bulk users { @@ -580,8 +499,8 @@ public void shouldNormaliseFields() throws Exception { JsonObject request = generateUsersJson(1); JsonObject response = HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", - "http://localhost:3567/bulk-import/users", - request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); + "http://localhost:3567/bulk-import/users", + request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); assertEquals("OK", response.get("status").getAsString()); JsonObject getResponse = HttpRequestForTesting.sendGETRequest(process.getProcess(), "", @@ -620,6 +539,24 @@ public void shouldNormaliseFields() throws Exception { Assert.assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STOPPED)); } + private String getResponseMessageFromError(String response) { + return response.substring(response.indexOf("Message: ") + "Message: ".length()); + } + + private void testBadRequest(Main main, JsonObject requestBody, String expectedErrorMessage) throws Exception { + try { + HttpRequestForTesting.sendJsonPOSTRequest(main, "", + "http://localhost:3567/bulk-import/users", + requestBody, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); + + fail("The API should have thrown an error"); + } catch (io.supertokens.test.httpRequest.HttpResponseException e) { + String responseString = getResponseMessageFromError(e.getMessage()); + assertEquals(400, e.statusCode); + assertEquals(responseString, expectedErrorMessage); + } + } + public static JsonObject generateUsersJson(int numberOfUsers) { JsonObject userJsonObject = new JsonObject(); JsonParser parser = new JsonParser(); @@ -630,7 +567,8 @@ public static JsonObject generateUsersJson(int numberOfUsers) { user.addProperty("externalUserId", UUID.randomUUID().toString()); user.add("userMetadata", parser.parse("{\"key1\":\"value1\",\"key2\":{\"key3\":\"value3\"}}")); - user.add("userRoles", parser.parse("[{\"role\":\"role1\", \"tenantIds\": [\"public\"]},{\"role\":\"role2\", \"tenantIds\": [\"public\"]}]")); + user.add("userRoles", parser.parse( + "[{\"role\":\"role1\", \"tenantIds\": [\"public\"]},{\"role\":\"role2\", \"tenantIds\": [\"public\"]}]")); user.add("totpDevices", parser.parse("[{\"secretKey\":\"secretKey\",\"deviceName\":\"deviceName\"}]")); JsonArray tenanatIds = parser.parse("[\"public\"]").getAsJsonArray(); @@ -654,11 +592,12 @@ private static JsonObject createEmailLoginMethod(String email, JsonArray tenantI loginMethod.add("tenantIds", tenantIds); loginMethod.addProperty("email", email); loginMethod.addProperty("recipeId", "emailpassword"); - loginMethod.addProperty("passwordHash", "$argon2d$v=19$m=12,t=3,p=1$aGI4enNvMmd0Zm0wMDAwMA$r6p7qbr6HD+8CD7sBi4HVw"); + loginMethod.addProperty("passwordHash", + "$argon2d$v=19$m=12,t=3,p=1$aGI4enNvMmd0Zm0wMDAwMA$r6p7qbr6HD+8CD7sBi4HVw"); loginMethod.addProperty("hashingAlgorithm", "argon2"); loginMethod.addProperty("isVerified", true); loginMethod.addProperty("isPrimary", true); - loginMethod.addProperty("timeJoinedInMSSinceEpoch", 0); + loginMethod.addProperty("timeJoinedInMSSinceEpoch", 0); return loginMethod; } @@ -705,9 +644,7 @@ private void createTenants(Main main) new EmailPasswordConfig(true), new ThirdPartyConfig(true, null), new PasswordlessConfig(true), - null, null, new JsonObject() - ) - ); + null, null, new JsonObject())); } { // tenant 2 JsonObject config = new JsonObject(); @@ -724,9 +661,7 @@ null, null, new JsonObject() new EmailPasswordConfig(true), new ThirdPartyConfig(true, null), new PasswordlessConfig(true), - null, null, config - ) - ); + null, null, config)); } } From f443a8ca52824ca8a9b8d5852fd91021e7e7f701 Mon Sep 17 00:00:00 2001 From: Ankit Tiwari Date: Tue, 2 Apr 2024 18:58:41 +0530 Subject: [PATCH 07/21] fix: PR changes --- .../cronjobs/bulkimport/ProcessBulkImportUsers.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkImportUsers.java b/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkImportUsers.java index 1d80ab6fa..4f6104ef4 100644 --- a/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkImportUsers.java +++ b/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkImportUsers.java @@ -116,8 +116,6 @@ protected void doTaskPerApp(AppIdentifier app) for (BulkImportUser user : users) { processUser(app, user, bulkImportSQLStorage); } - - closeAllProxyStorages(); } @Override @@ -257,6 +255,8 @@ private void processUser(AppIdentifier appIdentifier, BulkImportUser user, BulkI // We need to rollback the transaction manually because we have overridden that in the proxy storage bulkImportProxyStorage.rollbackTransactionForBulkImportProxyStorage(); throw e; + } finally { + closeAllProxyStorages(); } }); } catch (StorageTransactionLogicException e) { From 73b75a8bbe9b31d633b46526e436b7da85c05185 Mon Sep 17 00:00:00 2001 From: Ankit Tiwari Date: Thu, 4 Apr 2024 19:24:20 +0530 Subject: [PATCH 08/21] fix: PR changes --- .../io/supertokens/bulkimport/BulkImport.java | 2 +- .../bulkimport/ProcessBulkImportUsers.java | 2 +- .../apis/AddBulkImportUsersTest.java | 57 +++++++++++++++++++ 3 files changed, 59 insertions(+), 2 deletions(-) diff --git a/src/main/java/io/supertokens/bulkimport/BulkImport.java b/src/main/java/io/supertokens/bulkimport/BulkImport.java index 8bfabc7e1..e552834b7 100644 --- a/src/main/java/io/supertokens/bulkimport/BulkImport.java +++ b/src/main/java/io/supertokens/bulkimport/BulkImport.java @@ -39,7 +39,7 @@ public class BulkImport { public static final int GET_USERS_DEFAULT_LIMIT = 100; public static final int DELETE_USERS_LIMIT = 500; public static final int PROCESS_USERS_BATCH_SIZE = 1000; - public static final int PROCESS_USERS_INTERVAL = 60; + public static final int PROCESS_USERS_INTERVAL_SECONDS = 60; public static void addUsers(AppIdentifier appIdentifier, Storage storage, List users) throws StorageQueryException, TenantOrAppNotFoundException { diff --git a/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkImportUsers.java b/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkImportUsers.java index 4f6104ef4..4ce47d81c 100644 --- a/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkImportUsers.java +++ b/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkImportUsers.java @@ -126,7 +126,7 @@ public int getIntervalTimeSeconds() { return interval; } } - return BulkImport.PROCESS_USERS_INTERVAL; + return BulkImport.PROCESS_USERS_INTERVAL_SECONDS; } @Override diff --git a/src/test/java/io/supertokens/test/bulkimport/apis/AddBulkImportUsersTest.java b/src/test/java/io/supertokens/test/bulkimport/apis/AddBulkImportUsersTest.java index 774d841f3..e199928c2 100644 --- a/src/test/java/io/supertokens/test/bulkimport/apis/AddBulkImportUsersTest.java +++ b/src/test/java/io/supertokens/test/bulkimport/apis/AddBulkImportUsersTest.java @@ -16,13 +16,18 @@ package io.supertokens.test.bulkimport.apis; +import static io.supertokens.test.bulkimport.BulkImportTestUtils.generateBulkImportUser; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.fail; import java.io.IOException; +import java.lang.reflect.Field; +import java.util.Arrays; import java.util.HashMap; +import java.util.List; import java.util.UUID; +import java.util.stream.Collectors; import org.junit.AfterClass; import org.junit.Assert; @@ -44,6 +49,7 @@ import io.supertokens.multitenancy.exception.BadPermissionException; import io.supertokens.multitenancy.exception.CannotModifyBaseConfigException; import io.supertokens.pluginInterface.STORAGE_TYPE; +import io.supertokens.pluginInterface.bulkimport.BulkImportUser; import io.supertokens.pluginInterface.exceptions.InvalidConfigException; import io.supertokens.pluginInterface.exceptions.StorageQueryException; import io.supertokens.pluginInterface.multitenancy.EmailPasswordConfig; @@ -539,6 +545,57 @@ public void shouldNormaliseFields() throws Exception { Assert.assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STOPPED)); } + @Test + public void shouldFailIfANewFieldWasAddedToBulkImportUser() throws Exception { + List bulkImportUsers = generateBulkImportUser(1); + BulkImportUser user = bulkImportUsers.get(0); + + checkFields(user, "BulkImportUser", + Arrays.asList("id", "externalUserId", "userMetadata", "userRoles", "totpDevices", + "loginMethods", "status", "primaryUserId", "errorMessage", "createdAt", + "updatedAt")); + + checkLoginMethodFields(user.loginMethods.get(0), "LoginMethod", + Arrays.asList("tenantIds", "isVerified", "isPrimary", "timeJoinedInMSSinceEpoch", + "recipeId", "email", "passwordHash", "hashingAlgorithm", + "phoneNumber", "thirdPartyId", "thirdPartyUserId", "externalUserId", "superTokensUserId")); + + checkTotpDeviceFields(user.totpDevices.get(0), "TotpDevice", + Arrays.asList("secretKey", "period", "skew", "deviceName")); + + checkUserRoleFields(user.userRoles.get(0), "UserRole", + Arrays.asList("role", "tenantIds")); + } + + private void checkFields(Object object, String objectType, List expectedFields) { + Field[] actualFields = object.getClass().getDeclaredFields(); + List actualFieldNames = Arrays.stream(actualFields) + .map(Field::getName) + .collect(Collectors.toList()); + + List extraFields = actualFieldNames.stream() + .filter(fieldName -> !expectedFields.contains(fieldName)) + .collect(Collectors.toList()); + + if (!extraFields.isEmpty()) { + fail("The following extra field(s) are present in " + objectType + ": " + String.join(", ", extraFields)); + } + } + + private void checkLoginMethodFields(BulkImportUser.LoginMethod loginMethod, String objectType, + List expectedFields) { + checkFields(loginMethod, objectType, expectedFields); + } + + private void checkTotpDeviceFields(BulkImportUser.TotpDevice totpDevice, String objectType, + List expectedFields) { + checkFields(totpDevice, objectType, expectedFields); + } + + private void checkUserRoleFields(BulkImportUser.UserRole userRole, String objectType, List expectedFields) { + checkFields(userRole, objectType, expectedFields); + } + private String getResponseMessageFromError(String response) { return response.substring(response.indexOf("Message: ") + "Message: ".length()); } From 6d59616db147bb5f155abf80d1268b3e6fd67c5c Mon Sep 17 00:00:00 2001 From: Ankit Tiwari Date: Mon, 8 Apr 2024 12:29:43 +0530 Subject: [PATCH 09/21] fix: PR changes --- src/main/java/io/supertokens/Main.java | 2 + .../bulkimport/BulkImportUserUtils.java | 3 +- .../bulkimport/ProcessBulkImportUsers.java | 38 ++++++++++++----- .../api/bulkimport/BulkImportAPI.java | 2 - src/test/java/io/supertokens/test/Utils.java | 1 + .../ProcessBulkImportUsersCronJobTest.java | 42 ++++++++++++++++++- .../apis/AddBulkImportUsersTest.java | 2 +- 7 files changed, 74 insertions(+), 16 deletions(-) diff --git a/src/main/java/io/supertokens/Main.java b/src/main/java/io/supertokens/Main.java index 7375e6e08..efbc3712d 100644 --- a/src/main/java/io/supertokens/Main.java +++ b/src/main/java/io/supertokens/Main.java @@ -65,6 +65,8 @@ public class Main { // this is a special variable that will be set to true by TestingProcessManager public static boolean isTesting = false; + // this flag is used in ProcessBulkImportUsersCronJobTest to skip the user validation + public static boolean isTesting_skipBulkImportUserValidationInCronJob = false; // this is a special variable that will be set to true by TestingProcessManager public static boolean makeConsolePrintSilent = false; diff --git a/src/main/java/io/supertokens/bulkimport/BulkImportUserUtils.java b/src/main/java/io/supertokens/bulkimport/BulkImportUserUtils.java index 7e81c0125..127d30ded 100644 --- a/src/main/java/io/supertokens/bulkimport/BulkImportUserUtils.java +++ b/src/main/java/io/supertokens/bulkimport/BulkImportUserUtils.java @@ -563,7 +563,8 @@ private void validateTenantIdsForRoleAndLoginMethods(Main main, AppIdentifier ap if (commonTenantUserPoolId == null) { commonTenantUserPoolId = tenantUserPoolId; } else if (!commonTenantUserPoolId.equals(tenantUserPoolId)) { - errors.add("All tenants for a user must share the same storage."); + errors.add("All tenants for a user must share the same storage for " + loginMethod.recipeId + + " recipe."); } } } diff --git a/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkImportUsers.java b/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkImportUsers.java index 4ce47d81c..a28b55c4c 100644 --- a/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkImportUsers.java +++ b/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkImportUsers.java @@ -32,6 +32,8 @@ import io.supertokens.authRecipe.exception.RecipeUserIdAlreadyLinkedWithAnotherPrimaryUserIdException; import io.supertokens.authRecipe.exception.RecipeUserIdAlreadyLinkedWithPrimaryUserIdException; import io.supertokens.bulkimport.BulkImport; +import io.supertokens.bulkimport.BulkImportUserUtils; +import io.supertokens.bulkimport.exceptions.InvalidBulkImportDataException; import io.supertokens.config.Config; import io.supertokens.cronjobs.CronTask; import io.supertokens.cronjobs.CronTaskTest; @@ -113,8 +115,11 @@ protected void doTaskPerApp(AppIdentifier app) List users = bulkImportSQLStorage.getBulkImportUsersAndChangeStatusToProcessing(app, BulkImport.PROCESS_USERS_BATCH_SIZE); + String[] allUserRoles = StorageUtils.getUserRolesStorage(bulkImportSQLStorage).getRoles(app); + BulkImportUserUtils bulkImportUserUtils = new BulkImportUserUtils(allUserRoles); + for (BulkImportUser user : users) { - processUser(app, user, bulkImportSQLStorage); + processUser(app, user, bulkImportUserUtils, bulkImportSQLStorage); } } @@ -189,21 +194,30 @@ private void closeAllProxyStorages() throws StorageQueryException { userPoolToStorageMap.clear(); } - private void processUser(AppIdentifier appIdentifier, BulkImportUser user, BulkImportSQLStorage baseTenantStorage) + private void processUser(AppIdentifier appIdentifier, BulkImportUser user, BulkImportUserUtils bulkImportUserUtils, + BulkImportSQLStorage baseTenantStorage) throws TenantOrAppNotFoundException, StorageQueryException, InvalidConfigException, IOException, DbInitException { - // Since all the tenants of a user must share the storage, we will just use the - // storage of the first tenantId of the first loginMethod - TenantIdentifier firstTenantIdentifier = new TenantIdentifier(appIdentifier.getConnectionUriDomain(), - appIdentifier.getAppId(), user.loginMethods.get(0).tenantIds.get(0)); + try { + if (Main.isTesting && Main.isTesting_skipBulkImportUserValidationInCronJob) { + // Skip validation when the flag is enabled during testing + } else { + // Validate the user + bulkImportUserUtils.createBulkImportUserFromJSON(main, appIdentifier, user.toJsonObject(), user.id); + } - SQLStorage bulkImportProxyStorage = (SQLStorage) getProxyStorage(firstTenantIdentifier); + // Since all the tenants of a user must share the storage, we will just use the + // storage of the first tenantId of the first loginMethod - LoginMethod primaryLM = getPrimaryLoginMethod(user); + TenantIdentifier firstTenantIdentifier = new TenantIdentifier(appIdentifier.getConnectionUriDomain(), + appIdentifier.getAppId(), user.loginMethods.get(0).tenantIds.get(0)); - AuthRecipeSQLStorage authRecipeSQLStorage = (AuthRecipeSQLStorage) getProxyStorage(firstTenantIdentifier); - try { + SQLStorage bulkImportProxyStorage = (SQLStorage) getProxyStorage(firstTenantIdentifier); + + LoginMethod primaryLM = getPrimaryLoginMethod(user); + + AuthRecipeSQLStorage authRecipeSQLStorage = (AuthRecipeSQLStorage) getProxyStorage(firstTenantIdentifier); // If primaryUserId is not null, it means we may have already processed this user but failed to delete the entry // If the primaryUserId exists in the database, we'll delete the corresponding entry from the bulkImportUser table and proceed to skip this user. if (user.primaryUserId != null) { @@ -259,7 +273,7 @@ private void processUser(AppIdentifier appIdentifier, BulkImportUser user, BulkI closeAllProxyStorages(); } }); - } catch (StorageTransactionLogicException e) { + } catch (StorageTransactionLogicException | InvalidBulkImportDataException e) { handleProcessUserExceptions(appIdentifier, user, e, baseTenantStorage); } } @@ -275,6 +289,8 @@ private void handleProcessUserExceptions(AppIdentifier appIdentifier, BulkImport if (e instanceof StorageTransactionLogicException) { StorageTransactionLogicException exception = (StorageTransactionLogicException) e; errorMessage[0] = exception.actualException.getMessage(); + } else if (e instanceof InvalidBulkImportDataException) { + errorMessage[0] = ((InvalidBulkImportDataException) e).errors.toString(); } try { diff --git a/src/main/java/io/supertokens/webserver/api/bulkimport/BulkImportAPI.java b/src/main/java/io/supertokens/webserver/api/bulkimport/BulkImportAPI.java index abb8fe65d..c3c196beb 100644 --- a/src/main/java/io/supertokens/webserver/api/bulkimport/BulkImportAPI.java +++ b/src/main/java/io/supertokens/webserver/api/bulkimport/BulkImportAPI.java @@ -18,9 +18,7 @@ import java.io.IOException; import java.util.ArrayList; -import java.util.HashSet; import java.util.List; -import java.util.Set; import com.google.gson.JsonArray; import com.google.gson.JsonObject; diff --git a/src/test/java/io/supertokens/test/Utils.java b/src/test/java/io/supertokens/test/Utils.java index 7a133782c..2c86519bb 100644 --- a/src/test/java/io/supertokens/test/Utils.java +++ b/src/test/java/io/supertokens/test/Utils.java @@ -80,6 +80,7 @@ public static String getCdiVersionStringLatestForTests() { public static void reset() { Main.isTesting = true; + Main.isTesting_skipBulkImportUserValidationInCronJob = false; PluginInterfaceTesting.isTesting = true; Main.makeConsolePrintSilent = true; String installDir = "../"; diff --git a/src/test/java/io/supertokens/test/bulkimport/ProcessBulkImportUsersCronJobTest.java b/src/test/java/io/supertokens/test/bulkimport/ProcessBulkImportUsersCronJobTest.java index 8e29c106a..26b3814cc 100644 --- a/src/test/java/io/supertokens/test/bulkimport/ProcessBulkImportUsersCronJobTest.java +++ b/src/test/java/io/supertokens/test/bulkimport/ProcessBulkImportUsersCronJobTest.java @@ -191,6 +191,9 @@ public void shouldDeleteEverythingFromtheDBIfAnythingFails() throws Exception { // Creating a non-existing user role will result in an error. // Since, user role creation happens at the last step of the bulk import process, everything should be deleted from the DB. + // NOTE: We will also need to disable the bulk import user validation in the cron job for this test to work. + Main.isTesting_skipBulkImportUserValidationInCronJob = true; + TestingProcess process = startCronProcess(); Main main = process.getProcess(); @@ -225,6 +228,12 @@ public void shouldThrowTenantDoesNotExistError() throws Exception { BulkImportSQLStorage storage = (BulkImportSQLStorage) StorageLayer.getStorage(main); AppIdentifier appIdentifier = new AppIdentifier(null, null); + // Create user roles before inserting bulk users + { + UserRoles.createNewRoleOrModifyItsPermissions(main, "role1", null); + UserRoles.createNewRoleOrModifyItsPermissions(main, "role2", null); + } + List users = generateBulkImportUser(1); BulkImport.addUsers(appIdentifier, storage, users); @@ -236,7 +245,38 @@ public void shouldThrowTenantDoesNotExistError() throws Exception { assertEquals(1, usersAfterProcessing.size()); assertEquals(BULK_IMPORT_USER_STATUS.FAILED, usersAfterProcessing.get(0).status); assertEquals( - "Tenant with the following connectionURIDomain, appId and tenantId combination not found: (, public, t1)", + "[Invalid tenantId: t1 for a user role., Invalid tenantId: t1 for a user role., Invalid tenantId: t1 for emailpassword recipe., Invalid tenantId: t1 for thirdparty recipe., Invalid tenantId: t1 for passwordless recipe.]", + usersAfterProcessing.get(0).errorMessage); + } + + @Test + public void shouldThrowTenantHaveDifferentStoragesError() throws Exception { + TestingProcess process = startCronProcess(); + Main main = process.getProcess(); + + BulkImportSQLStorage storage = (BulkImportSQLStorage) StorageLayer.getStorage(main); + AppIdentifier appIdentifier = new AppIdentifier(null, null); + + // Create user roles before inserting bulk users + { + UserRoles.createNewRoleOrModifyItsPermissions(main, "role1", null); + UserRoles.createNewRoleOrModifyItsPermissions(main, "role2", null); + } + createTenants(main); + + + List users = generateBulkImportUser(1, List.of("t1", "t2"), 0); + BulkImport.addUsers(appIdentifier, storage, users); + + Thread.sleep(6000); + + List usersAfterProcessing = storage.getBulkImportUsers(appIdentifier, null, null, + null, null); + + assertEquals(1, usersAfterProcessing.size()); + assertEquals(BULK_IMPORT_USER_STATUS.FAILED, usersAfterProcessing.get(0).status); + assertEquals( + "[All tenants for a user must share the same storage for emailpassword recipe., All tenants for a user must share the same storage for thirdparty recipe., All tenants for a user must share the same storage for passwordless recipe.]", usersAfterProcessing.get(0).errorMessage); } diff --git a/src/test/java/io/supertokens/test/bulkimport/apis/AddBulkImportUsersTest.java b/src/test/java/io/supertokens/test/bulkimport/apis/AddBulkImportUsersTest.java index e199928c2..baf205057 100644 --- a/src/test/java/io/supertokens/test/bulkimport/apis/AddBulkImportUsersTest.java +++ b/src/test/java/io/supertokens/test/bulkimport/apis/AddBulkImportUsersTest.java @@ -426,7 +426,7 @@ public void shouldThrow400IfInvalidTenantIdIsPassed() throws Exception { .getAsJsonObject(); testBadRequest(process.getProcess(), requestBody3, "{\"error\":\"" + genericErrMsg - + "\",\"users\":[{\"index\":0,\"errors\":[\"All tenants for a user must share the same storage.\"]}]}"); + + "\",\"users\":[{\"index\":0,\"errors\":[\"All tenants for a user must share the same storage for thirdparty recipe.\"]}]}"); process.kill(); Assert.assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STOPPED)); From b6c63f4ad4f68f4e33ec1b788c108c1b4bf507ea Mon Sep 17 00:00:00 2001 From: Ankit Tiwari Date: Tue, 9 Apr 2024 12:49:16 +0530 Subject: [PATCH 10/21] fix: Update version --- CHANGELOG.md | 8 ++++++++ build.gradle | 2 +- coreDriverInterfaceSupported.json | 3 ++- src/main/java/io/supertokens/utils/SemVer.java | 1 + 4 files changed, 12 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index aaf903f10..699f96a71 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,14 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [9.1.0] - 2024-04-10 + +### Added + +- Adds APIs to bulk import users +- Adds `ProcessBulkImportUsers` cron job to process bulk import users + + ## [9.0.0] - 2024-03-13 ### Added diff --git a/build.gradle b/build.gradle index 6080bc20d..08687c8db 100644 --- a/build.gradle +++ b/build.gradle @@ -19,7 +19,7 @@ compileTestJava { options.encoding = "UTF-8" } // } //} -version = "9.0.0" +version = "9.1.0" repositories { diff --git a/coreDriverInterfaceSupported.json b/coreDriverInterfaceSupported.json index 00fa393ac..c27931c8e 100644 --- a/coreDriverInterfaceSupported.json +++ b/coreDriverInterfaceSupported.json @@ -18,6 +18,7 @@ "2.21", "3.0", "4.0", - "5.0" + "5.0", + "5.1" ] } diff --git a/src/main/java/io/supertokens/utils/SemVer.java b/src/main/java/io/supertokens/utils/SemVer.java index e02de95fb..73939d94b 100644 --- a/src/main/java/io/supertokens/utils/SemVer.java +++ b/src/main/java/io/supertokens/utils/SemVer.java @@ -35,6 +35,7 @@ public class SemVer implements Comparable { public static final SemVer v3_0 = new SemVer("3.0"); public static final SemVer v4_0 = new SemVer("4.0"); public static final SemVer v5_0 = new SemVer("5.0"); + public static final SemVer v5_1 = new SemVer("5.1"); final private String version; From 173e7fc07a9c934af7eaaac932724e14f9bd798c Mon Sep 17 00:00:00 2001 From: Ankit Tiwari Date: Wed, 10 Apr 2024 19:48:37 +0530 Subject: [PATCH 11/21] fix: PR changes --- .../cronjobs/bulkimport/ProcessBulkImportUsers.java | 5 +++-- .../supertokens/test/bulkimport/BulkImportTest.java | 12 ++++++------ .../ProcessBulkImportUsersCronJobTest.java | 10 +++++----- .../bulkimport/apis/DeleteBulkImportUsersTest.java | 2 +- 4 files changed, 15 insertions(+), 14 deletions(-) diff --git a/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkImportUsers.java b/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkImportUsers.java index a28b55c4c..671e38884 100644 --- a/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkImportUsers.java +++ b/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkImportUsers.java @@ -146,7 +146,7 @@ public int getInitialWaitTimeSeconds() { } private Storage getProxyStorage(TenantIdentifier tenantIdentifier) - throws InvalidConfigException, IOException, TenantOrAppNotFoundException, DbInitException { + throws InvalidConfigException, IOException, TenantOrAppNotFoundException, DbInitException, StorageQueryException { String userPoolId = StorageLayer.getStorage(tenantIdentifier, main).getUserPoolId(); if (userPoolToStorageMap.containsKey(userPoolId)) { return userPoolToStorageMap.get(userPoolId); @@ -165,6 +165,7 @@ private Storage getProxyStorage(TenantIdentifier tenantIdentifier) userPoolToStorageMap.put(userPoolId, bulkImportProxyStorage); bulkImportProxyStorage.initStorage(true); + bulkImportProxyStorage.commitTransactionForBulkImportProxyStorage(); return bulkImportProxyStorage; } } @@ -172,7 +173,7 @@ private Storage getProxyStorage(TenantIdentifier tenantIdentifier) } public Storage[] getAllProxyStoragesForApp(Main main, AppIdentifier appIdentifier) - throws TenantOrAppNotFoundException, InvalidConfigException, IOException, DbInitException { + throws TenantOrAppNotFoundException, InvalidConfigException, IOException, DbInitException, StorageQueryException { List allProxyStorages = new ArrayList<>(); Map resources = main diff --git a/src/test/java/io/supertokens/test/bulkimport/BulkImportTest.java b/src/test/java/io/supertokens/test/bulkimport/BulkImportTest.java index d7847e36a..f430f443f 100644 --- a/src/test/java/io/supertokens/test/bulkimport/BulkImportTest.java +++ b/src/test/java/io/supertokens/test/bulkimport/BulkImportTest.java @@ -76,7 +76,7 @@ public void shouldAddUsersInBulkImportUsersTable() throws Exception { BulkImportStorage storage = (BulkImportStorage) StorageLayer.getStorage(process.main); BulkImport.addUsers(new AppIdentifier(null, null), storage, users); - List addedUsers = storage.getBulkImportUsers(new AppIdentifier(null, null), null, BULK_IMPORT_USER_STATUS.NEW, null, null); + List addedUsers = storage.getBulkImportUsers(new AppIdentifier(null, null), 100, BULK_IMPORT_USER_STATUS.NEW, null, null); // Verify that all users are present in addedUsers for (BulkImportUser user : users) { @@ -116,7 +116,7 @@ public void shouldCreatedNewIdsIfDuplicateIdIsFound() throws Exception { AppIdentifier appIdentifier = new AppIdentifier(null, null); BulkImport.addUsers(appIdentifier, storage, users); - List addedUsers = storage.getBulkImportUsers(appIdentifier, null, BULK_IMPORT_USER_STATUS.NEW, null, null); + List addedUsers = storage.getBulkImportUsers(appIdentifier, 1000, BULK_IMPORT_USER_STATUS.NEW, null, null); // Verify that the other properties are same but ids changed for (BulkImportUser user : users) { @@ -153,7 +153,7 @@ public void testGetUsersStatusFilter() throws Exception { List users = generateBulkImportUser(10); BulkImport.addUsers(appIdentifier, storage, users); - List addedUsers = storage.getBulkImportUsers(appIdentifier, null, BULK_IMPORT_USER_STATUS.NEW, null, null); + List addedUsers = storage.getBulkImportUsers(appIdentifier, 100, BULK_IMPORT_USER_STATUS.NEW, null, null); assertEquals(10, addedUsers.size()); } @@ -171,7 +171,7 @@ public void testGetUsersStatusFilter() throws Exception { return null; }); - List addedUsers = storage.getBulkImportUsers(appIdentifier, null, BULK_IMPORT_USER_STATUS.PROCESSING, null, null); + List addedUsers = storage.getBulkImportUsers(appIdentifier, 100, BULK_IMPORT_USER_STATUS.PROCESSING, null, null); assertEquals(10, addedUsers.size()); } @@ -189,7 +189,7 @@ public void testGetUsersStatusFilter() throws Exception { return null; }); - List addedUsers = storage.getBulkImportUsers(appIdentifier, null, BULK_IMPORT_USER_STATUS.FAILED, null, null); + List addedUsers = storage.getBulkImportUsers(appIdentifier, 100, BULK_IMPORT_USER_STATUS.FAILED, null, null); assertEquals(10, addedUsers.size()); } @@ -229,7 +229,7 @@ public void randomPaginationTest() throws Exception { } // Get all inserted users - List addedUsers = storage.getBulkImportUsers(new AppIdentifier(null, null), null, null, null, null); + List addedUsers = storage.getBulkImportUsers(new AppIdentifier(null, null), 1000, null, null, null); assertEquals(numberOfUsers, addedUsers.size()); // We are sorting the users based on createdAt and id like we do in the storage layer diff --git a/src/test/java/io/supertokens/test/bulkimport/ProcessBulkImportUsersCronJobTest.java b/src/test/java/io/supertokens/test/bulkimport/ProcessBulkImportUsersCronJobTest.java index 26b3814cc..6e763009c 100644 --- a/src/test/java/io/supertokens/test/bulkimport/ProcessBulkImportUsersCronJobTest.java +++ b/src/test/java/io/supertokens/test/bulkimport/ProcessBulkImportUsersCronJobTest.java @@ -113,7 +113,7 @@ public void shouldProcessBulkImportUsersInTheSameTenant() throws Exception { Thread.sleep(6000); - List usersAfterProcessing = storage.getBulkImportUsers(appIdentifier, null, null, + List usersAfterProcessing = storage.getBulkImportUsers(appIdentifier, 100, null, null, null); assertEquals(0, usersAfterProcessing.size()); @@ -161,7 +161,7 @@ public void shouldProcessBulkImportUsersInMultipleTenantsWithDifferentStorages() Thread.sleep(6000); - List usersAfterProcessing = storage.getBulkImportUsers(appIdentifier, null, null, + List usersAfterProcessing = storage.getBulkImportUsers(appIdentifier, 100, null, null, null); assertEquals(0, usersAfterProcessing.size()); @@ -207,7 +207,7 @@ public void shouldDeleteEverythingFromtheDBIfAnythingFails() throws Exception { Thread.sleep(6000); - List usersAfterProcessing = storage.getBulkImportUsers(appIdentifier, null, null, + List usersAfterProcessing = storage.getBulkImportUsers(appIdentifier, 100, null, null, null); assertEquals(1, usersAfterProcessing.size()); @@ -239,7 +239,7 @@ public void shouldThrowTenantDoesNotExistError() throws Exception { Thread.sleep(6000); - List usersAfterProcessing = storage.getBulkImportUsers(appIdentifier, null, null, + List usersAfterProcessing = storage.getBulkImportUsers(appIdentifier, 100, null, null, null); assertEquals(1, usersAfterProcessing.size()); @@ -270,7 +270,7 @@ public void shouldThrowTenantHaveDifferentStoragesError() throws Exception { Thread.sleep(6000); - List usersAfterProcessing = storage.getBulkImportUsers(appIdentifier, null, null, + List usersAfterProcessing = storage.getBulkImportUsers(appIdentifier, 100, null, null, null); assertEquals(1, usersAfterProcessing.size()); diff --git a/src/test/java/io/supertokens/test/bulkimport/apis/DeleteBulkImportUsersTest.java b/src/test/java/io/supertokens/test/bulkimport/apis/DeleteBulkImportUsersTest.java index db2fe1707..7dad2ac1b 100644 --- a/src/test/java/io/supertokens/test/bulkimport/apis/DeleteBulkImportUsersTest.java +++ b/src/test/java/io/supertokens/test/bulkimport/apis/DeleteBulkImportUsersTest.java @@ -158,7 +158,7 @@ public void shouldReturn200Response() throws Exception { JsonObject response = HttpRequestForTesting.sendJsonDELETERequest(process.getProcess(), "", "http://localhost:3567/bulk-import/users", - request, 1000000, 1000000, null, Utils.getCdiVersionStringLatestForTests(), null); + request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); response.get("deletedIds").getAsJsonArray().forEach(id -> { assertTrue(validIds.contains(id)); From 4f6ab132a9d22592f523daf516ed55a95a8877e4 Mon Sep 17 00:00:00 2001 From: Ankit Tiwari Date: Thu, 18 Apr 2024 16:48:30 +0530 Subject: [PATCH 12/21] fix: PR changes --- .../java/io/supertokens/bulkimport/BulkImportUserUtils.java | 2 +- .../test/bulkimport/ProcessBulkImportUsersCronJobTest.java | 2 +- .../test/bulkimport/apis/AddBulkImportUsersTest.java | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/java/io/supertokens/bulkimport/BulkImportUserUtils.java b/src/main/java/io/supertokens/bulkimport/BulkImportUserUtils.java index 127d30ded..1d090b7fe 100644 --- a/src/main/java/io/supertokens/bulkimport/BulkImportUserUtils.java +++ b/src/main/java/io/supertokens/bulkimport/BulkImportUserUtils.java @@ -563,7 +563,7 @@ private void validateTenantIdsForRoleAndLoginMethods(Main main, AppIdentifier ap if (commonTenantUserPoolId == null) { commonTenantUserPoolId = tenantUserPoolId; } else if (!commonTenantUserPoolId.equals(tenantUserPoolId)) { - errors.add("All tenants for a user must share the same storage for " + loginMethod.recipeId + errors.add("All tenants for a user must share the same database for " + loginMethod.recipeId + " recipe."); } } diff --git a/src/test/java/io/supertokens/test/bulkimport/ProcessBulkImportUsersCronJobTest.java b/src/test/java/io/supertokens/test/bulkimport/ProcessBulkImportUsersCronJobTest.java index 6e763009c..da6b9050f 100644 --- a/src/test/java/io/supertokens/test/bulkimport/ProcessBulkImportUsersCronJobTest.java +++ b/src/test/java/io/supertokens/test/bulkimport/ProcessBulkImportUsersCronJobTest.java @@ -276,7 +276,7 @@ public void shouldThrowTenantHaveDifferentStoragesError() throws Exception { assertEquals(1, usersAfterProcessing.size()); assertEquals(BULK_IMPORT_USER_STATUS.FAILED, usersAfterProcessing.get(0).status); assertEquals( - "[All tenants for a user must share the same storage for emailpassword recipe., All tenants for a user must share the same storage for thirdparty recipe., All tenants for a user must share the same storage for passwordless recipe.]", + "[All tenants for a user must share the same database for emailpassword recipe., All tenants for a user must share the same database for thirdparty recipe., All tenants for a user must share the same database for passwordless recipe.]", usersAfterProcessing.get(0).errorMessage); } diff --git a/src/test/java/io/supertokens/test/bulkimport/apis/AddBulkImportUsersTest.java b/src/test/java/io/supertokens/test/bulkimport/apis/AddBulkImportUsersTest.java index baf205057..90fad26cb 100644 --- a/src/test/java/io/supertokens/test/bulkimport/apis/AddBulkImportUsersTest.java +++ b/src/test/java/io/supertokens/test/bulkimport/apis/AddBulkImportUsersTest.java @@ -426,7 +426,7 @@ public void shouldThrow400IfInvalidTenantIdIsPassed() throws Exception { .getAsJsonObject(); testBadRequest(process.getProcess(), requestBody3, "{\"error\":\"" + genericErrMsg - + "\",\"users\":[{\"index\":0,\"errors\":[\"All tenants for a user must share the same storage for thirdparty recipe.\"]}]}"); + + "\",\"users\":[{\"index\":0,\"errors\":[\"All tenants for a user must share the same database for thirdparty recipe.\"]}]}"); process.kill(); Assert.assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STOPPED)); From fcdfb54dea5a5d13fdb75c22945f8acb0d8d1175 Mon Sep 17 00:00:00 2001 From: Ankit Tiwari Date: Mon, 29 Apr 2024 13:08:04 +0530 Subject: [PATCH 13/21] fix: Rename DeleteBulkImportUser API path --- .../bulkimport/ProcessBulkImportUsers.java | 4 +- .../io/supertokens/webserver/Webserver.java | 2 + .../api/bulkimport/BulkImportAPI.java | 60 ---------- .../bulkimport/DeleteBulkImportUserAPI.java | 108 ++++++++++++++++++ .../apis/DeleteBulkImportUsersTest.java | 20 ++-- 5 files changed, 123 insertions(+), 71 deletions(-) create mode 100644 src/main/java/io/supertokens/webserver/api/bulkimport/DeleteBulkImportUserAPI.java diff --git a/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkImportUsers.java b/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkImportUsers.java index 843813241..4584b9360 100644 --- a/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkImportUsers.java +++ b/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkImportUsers.java @@ -19,8 +19,10 @@ import java.io.IOException; import java.util.ArrayList; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; import com.google.gson.JsonObject; @@ -164,7 +166,7 @@ private Storage getProxyStorage(TenantIdentifier tenantIdentifier) normalisedConfigs.get(key), tenantIdentifier, true); userPoolToStorageMap.put(userPoolId, bulkImportProxyStorage); - bulkImportProxyStorage.initStorage(true, new ArrayList<>(List.of(tenantIdentifier))); + bulkImportProxyStorage.initStorage(true, new ArrayList<>()); bulkImportProxyStorage.commitTransactionForBulkImportProxyStorage(); return bulkImportProxyStorage; } diff --git a/src/main/java/io/supertokens/webserver/Webserver.java b/src/main/java/io/supertokens/webserver/Webserver.java index 96716c746..c36f9aec2 100644 --- a/src/main/java/io/supertokens/webserver/Webserver.java +++ b/src/main/java/io/supertokens/webserver/Webserver.java @@ -27,6 +27,7 @@ import io.supertokens.pluginInterface.multitenancy.exceptions.TenantOrAppNotFoundException; import io.supertokens.webserver.api.accountlinking.*; import io.supertokens.webserver.api.bulkimport.BulkImportAPI; +import io.supertokens.webserver.api.bulkimport.DeleteBulkImportUserAPI; import io.supertokens.webserver.api.core.*; import io.supertokens.webserver.api.dashboard.*; import io.supertokens.webserver.api.emailpassword.UserAPI; @@ -262,6 +263,7 @@ private void setupRoutes() { addAPI(new RequestStatsAPI(main)); addAPI(new BulkImportAPI(main)); + addAPI(new DeleteBulkImportUserAPI(main)); StandardContext context = tomcatReference.getContext(); Tomcat tomcat = tomcatReference.getTomcat(); diff --git a/src/main/java/io/supertokens/webserver/api/bulkimport/BulkImportAPI.java b/src/main/java/io/supertokens/webserver/api/bulkimport/BulkImportAPI.java index c3c196beb..3fc53e263 100644 --- a/src/main/java/io/supertokens/webserver/api/bulkimport/BulkImportAPI.java +++ b/src/main/java/io/supertokens/webserver/api/bulkimport/BulkImportAPI.java @@ -188,64 +188,4 @@ protected void doPost(HttpServletRequest req, HttpServletResponse resp) throws S result.addProperty("status", "OK"); super.sendJsonResponse(200, result, resp); } - - @Override - protected void doDelete(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException { - // API is app specific - JsonObject input = InputParser.parseJsonObjectOrThrowError(req); - JsonArray arr = InputParser.parseArrayOrThrowError(input, "ids", false); - - if (arr.size() == 0) { - throw new ServletException(new WebserverAPI.BadRequestException("Field name 'ids' cannot be an empty array")); - } - - if (arr.size() > BulkImport.DELETE_USERS_LIMIT) { - throw new ServletException(new WebserverAPI.BadRequestException("Field name 'ids' cannot contain more than " - + BulkImport.DELETE_USERS_LIMIT + " elements")); - } - - String[] userIds = new String[arr.size()]; - - for (int i = 0; i < userIds.length; i++) { - String userId = InputParser.parseStringFromElementOrThrowError(arr.get(i), "ids", false); - if (userId.isEmpty()) { - throw new ServletException(new WebserverAPI.BadRequestException("Field name 'ids' cannot contain an empty string")); - } - userIds[i] = userId; - } - - AppIdentifier appIdentifier = null; - Storage storage = null; - - try { - appIdentifier = getAppIdentifier(req); - storage = enforcePublicTenantAndGetPublicTenantStorage(req); - } catch (TenantOrAppNotFoundException | BadPermissionException e) { - throw new ServletException(e); - } - - try { - List deletedIds = BulkImport.deleteUsers(appIdentifier, storage, userIds); - - JsonArray deletedIdsJson = new JsonArray(); - JsonArray invalidIds = new JsonArray(); - - for (String userId : userIds) { - if (deletedIds.contains(userId)) { - deletedIdsJson.add(new JsonPrimitive(userId)); - } else { - invalidIds.add(new JsonPrimitive(userId)); - } - } - - JsonObject result = new JsonObject(); - result.add("deletedIds", deletedIdsJson); - result.add("invalidIds", invalidIds); - - super.sendJsonResponse(200, result, resp); - - } catch (StorageQueryException e) { - throw new ServletException(e); - } - } } diff --git a/src/main/java/io/supertokens/webserver/api/bulkimport/DeleteBulkImportUserAPI.java b/src/main/java/io/supertokens/webserver/api/bulkimport/DeleteBulkImportUserAPI.java new file mode 100644 index 000000000..b78f12c11 --- /dev/null +++ b/src/main/java/io/supertokens/webserver/api/bulkimport/DeleteBulkImportUserAPI.java @@ -0,0 +1,108 @@ +/* + * Copyright (c) 2024, VRAI Labs and/or its affiliates. All rights reserved. + * + * This software is licensed under the Apache License, Version 2.0 (the + * "License") as published by the Apache Software Foundation. + * + * You may not use this file except in compliance with the License. You may + * obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package io.supertokens.webserver.api.bulkimport; + +import java.io.IOException; +import java.util.List; + +import com.google.gson.JsonArray; +import com.google.gson.JsonObject; +import com.google.gson.JsonPrimitive; + +import io.supertokens.Main; +import io.supertokens.bulkimport.BulkImport; +import io.supertokens.multitenancy.exception.BadPermissionException; +import io.supertokens.pluginInterface.Storage; +import io.supertokens.pluginInterface.exceptions.StorageQueryException; +import io.supertokens.pluginInterface.multitenancy.AppIdentifier; +import io.supertokens.pluginInterface.multitenancy.exceptions.TenantOrAppNotFoundException; +import io.supertokens.webserver.InputParser; +import io.supertokens.webserver.WebserverAPI; +import jakarta.servlet.ServletException; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; + +public class DeleteBulkImportUserAPI extends WebserverAPI { + public DeleteBulkImportUserAPI(Main main) { + super(main, ""); + } + + @Override + public String getPath() { + return "/bulk-import/users/remove"; + } + + @Override + protected void doPost(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException { + // API is app specific + JsonObject input = InputParser.parseJsonObjectOrThrowError(req); + JsonArray arr = InputParser.parseArrayOrThrowError(input, "ids", false); + + if (arr.size() == 0) { + throw new ServletException(new WebserverAPI.BadRequestException("Field name 'ids' cannot be an empty array")); + } + + if (arr.size() > BulkImport.DELETE_USERS_LIMIT) { + throw new ServletException(new WebserverAPI.BadRequestException("Field name 'ids' cannot contain more than " + + BulkImport.DELETE_USERS_LIMIT + " elements")); + } + + String[] userIds = new String[arr.size()]; + + for (int i = 0; i < userIds.length; i++) { + String userId = InputParser.parseStringFromElementOrThrowError(arr.get(i), "ids", false); + if (userId.isEmpty()) { + throw new ServletException(new WebserverAPI.BadRequestException("Field name 'ids' cannot contain an empty string")); + } + userIds[i] = userId; + } + + AppIdentifier appIdentifier = null; + Storage storage = null; + + try { + appIdentifier = getAppIdentifier(req); + storage = enforcePublicTenantAndGetPublicTenantStorage(req); + } catch (TenantOrAppNotFoundException | BadPermissionException e) { + throw new ServletException(e); + } + + try { + List deletedIds = BulkImport.deleteUsers(appIdentifier, storage, userIds); + + JsonArray deletedIdsJson = new JsonArray(); + JsonArray invalidIds = new JsonArray(); + + for (String userId : userIds) { + if (deletedIds.contains(userId)) { + deletedIdsJson.add(new JsonPrimitive(userId)); + } else { + invalidIds.add(new JsonPrimitive(userId)); + } + } + + JsonObject result = new JsonObject(); + result.add("deletedIds", deletedIdsJson); + result.add("invalidIds", invalidIds); + + super.sendJsonResponse(200, result, resp); + + } catch (StorageQueryException e) { + throw new ServletException(e); + } + } +} diff --git a/src/test/java/io/supertokens/test/bulkimport/apis/DeleteBulkImportUsersTest.java b/src/test/java/io/supertokens/test/bulkimport/apis/DeleteBulkImportUsersTest.java index 7dad2ac1b..50042e7d6 100644 --- a/src/test/java/io/supertokens/test/bulkimport/apis/DeleteBulkImportUsersTest.java +++ b/src/test/java/io/supertokens/test/bulkimport/apis/DeleteBulkImportUsersTest.java @@ -75,8 +75,8 @@ public void shouldReturn400Error() throws Exception { { try { JsonObject request = new JsonObject(); - HttpRequestForTesting.sendJsonDELETERequest(process.getProcess(), "", - "http://localhost:3567/bulk-import/users", + HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", + "http://localhost:3567/bulk-import/users/remove", request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); } catch (io.supertokens.test.httpRequest.HttpResponseException e) { assertEquals(400, e.statusCode); @@ -86,8 +86,8 @@ public void shouldReturn400Error() throws Exception { { try { JsonObject request = new JsonParser().parse("{\"ids\":[]}").getAsJsonObject(); - HttpRequestForTesting.sendJsonDELETERequest(process.getProcess(), "", - "http://localhost:3567/bulk-import/users", + HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", + "http://localhost:3567/bulk-import/users/remove", request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); } catch (io.supertokens.test.httpRequest.HttpResponseException e) { assertEquals(400, e.statusCode); @@ -97,8 +97,8 @@ public void shouldReturn400Error() throws Exception { { try { JsonObject request = new JsonParser().parse("{\"ids\":[\"\"]}").getAsJsonObject(); - HttpRequestForTesting.sendJsonDELETERequest(process.getProcess(), "", - "http://localhost:3567/bulk-import/users", + HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", + "http://localhost:3567/bulk-import/users/remove", request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); } catch (io.supertokens.test.httpRequest.HttpResponseException e) { assertEquals(400, e.statusCode); @@ -115,8 +115,8 @@ public void shouldReturn400Error() throws Exception { } request.add("ids", ids); - HttpRequestForTesting.sendJsonDELETERequest(process.getProcess(), "", - "http://localhost:3567/bulk-import/users", + HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", + "http://localhost:3567/bulk-import/users/remove", request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); } catch (io.supertokens.test.httpRequest.HttpResponseException e) { assertEquals(400, e.statusCode); @@ -156,8 +156,8 @@ public void shouldReturn200Response() throws Exception { request.add("ids", validIds); - JsonObject response = HttpRequestForTesting.sendJsonDELETERequest(process.getProcess(), "", - "http://localhost:3567/bulk-import/users", + JsonObject response = HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", + "http://localhost:3567/bulk-import/users/remove", request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); response.get("deletedIds").getAsJsonArray().forEach(id -> { From ddfaa1c39775e63393f36b86f7d187ec7c1f3811 Mon Sep 17 00:00:00 2001 From: Ankit Tiwari Date: Mon, 29 Apr 2024 13:38:56 +0530 Subject: [PATCH 14/21] fix: disable bulk import for in-memory db --- .../bulkimport/ProcessBulkImportUsers.java | 2 +- .../java/io/supertokens/inmemorydb/Start.java | 3 +- .../api/bulkimport/BulkImportAPI.java | 11 ++ .../bulkimport/DeleteBulkImportUserAPI.java | 6 + .../test/bulkimport/BulkImportTest.java | 13 +- .../ProcessBulkImportUsersCronJobTest.java | 20 +++ .../apis/AddBulkImportUsersTest.java | 127 ++++++++++-------- .../apis/DeleteBulkImportUsersTest.java | 17 ++- .../apis/GetBulkImportUsersTest.java | 19 +-- 9 files changed, 140 insertions(+), 78 deletions(-) diff --git a/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkImportUsers.java b/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkImportUsers.java index 4584b9360..daf1160e8 100644 --- a/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkImportUsers.java +++ b/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkImportUsers.java @@ -107,7 +107,7 @@ protected void doTaskPerApp(AppIdentifier app) throws TenantOrAppNotFoundException, StorageQueryException, InvalidConfigException, IOException, DbInitException { - if (StorageLayer.getBaseStorage(main).getType() != STORAGE_TYPE.SQL) { + if (StorageLayer.getBaseStorage(main).getType() != STORAGE_TYPE.SQL || StorageLayer.isInMemDb(main)) { return; } diff --git a/src/main/java/io/supertokens/inmemorydb/Start.java b/src/main/java/io/supertokens/inmemorydb/Start.java index 77b611a10..ae9b57ba9 100644 --- a/src/main/java/io/supertokens/inmemorydb/Start.java +++ b/src/main/java/io/supertokens/inmemorydb/Start.java @@ -137,8 +137,7 @@ public void constructor(String processId, boolean silent, boolean isTesting) { @Override public Storage createBulkImportProxyStorageInstance() { - // throw not implemented error - throw new UnsupportedOperationException("Unimplemented method 'createBulkImportProxyStorageInstance'"); + throw new UnsupportedOperationException("'createBulkImportProxyStorageInstance' is not supported for in-memory db"); } diff --git a/src/main/java/io/supertokens/webserver/api/bulkimport/BulkImportAPI.java b/src/main/java/io/supertokens/webserver/api/bulkimport/BulkImportAPI.java index 3fc53e263..496fe3ba2 100644 --- a/src/main/java/io/supertokens/webserver/api/bulkimport/BulkImportAPI.java +++ b/src/main/java/io/supertokens/webserver/api/bulkimport/BulkImportAPI.java @@ -38,6 +38,7 @@ import io.supertokens.pluginInterface.exceptions.StorageQueryException; import io.supertokens.pluginInterface.multitenancy.AppIdentifier; import io.supertokens.pluginInterface.multitenancy.exceptions.TenantOrAppNotFoundException; +import io.supertokens.storageLayer.StorageLayer; import io.supertokens.utils.Utils; import io.supertokens.webserver.InputParser; import io.supertokens.webserver.WebserverAPI; @@ -58,6 +59,11 @@ public String getPath() { @Override protected void doGet(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException { // API is app specific + + if (StorageLayer.isInMemDb(main)) { + throw new ServletException(new BadRequestException("This API is not supported in the in-memory database.")); + } + String statusString = InputParser.getQueryParamOrThrowError(req, "status", true); String paginationToken = InputParser.getQueryParamOrThrowError(req, "paginationToken", true); Integer limit = InputParser.getIntQueryParamOrThrowError(req, "limit", true); @@ -118,6 +124,11 @@ protected void doGet(HttpServletRequest req, HttpServletResponse resp) throws Se @Override protected void doPost(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException { // API is app specific + + if (StorageLayer.isInMemDb(main)) { + throw new ServletException(new BadRequestException("This API is not supported in the in-memory database.")); + } + JsonObject input = InputParser.parseJsonObjectOrThrowError(req); JsonArray users = InputParser.parseArrayOrThrowError(input, "users", false); diff --git a/src/main/java/io/supertokens/webserver/api/bulkimport/DeleteBulkImportUserAPI.java b/src/main/java/io/supertokens/webserver/api/bulkimport/DeleteBulkImportUserAPI.java index b78f12c11..a9032cc18 100644 --- a/src/main/java/io/supertokens/webserver/api/bulkimport/DeleteBulkImportUserAPI.java +++ b/src/main/java/io/supertokens/webserver/api/bulkimport/DeleteBulkImportUserAPI.java @@ -30,6 +30,7 @@ import io.supertokens.pluginInterface.exceptions.StorageQueryException; import io.supertokens.pluginInterface.multitenancy.AppIdentifier; import io.supertokens.pluginInterface.multitenancy.exceptions.TenantOrAppNotFoundException; +import io.supertokens.storageLayer.StorageLayer; import io.supertokens.webserver.InputParser; import io.supertokens.webserver.WebserverAPI; import jakarta.servlet.ServletException; @@ -49,6 +50,11 @@ public String getPath() { @Override protected void doPost(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException { // API is app specific + + if (StorageLayer.isInMemDb(main)) { + throw new ServletException(new BadRequestException("This API is not supported in the in-memory database.")); + } + JsonObject input = InputParser.parseJsonObjectOrThrowError(req); JsonArray arr = InputParser.parseArrayOrThrowError(input, "ids", false); diff --git a/src/test/java/io/supertokens/test/bulkimport/BulkImportTest.java b/src/test/java/io/supertokens/test/bulkimport/BulkImportTest.java index f430f443f..5eaf38842 100644 --- a/src/test/java/io/supertokens/test/bulkimport/BulkImportTest.java +++ b/src/test/java/io/supertokens/test/bulkimport/BulkImportTest.java @@ -29,6 +29,7 @@ import org.junit.Test; import org.junit.rules.TestRule; +import io.supertokens.Main; import io.supertokens.ProcessState; import io.supertokens.bulkimport.BulkImport; import io.supertokens.bulkimport.BulkImportUserPaginationContainer; @@ -66,8 +67,9 @@ public void shouldAddUsersInBulkImportUsersTable() throws Exception { TestingProcessManager.TestingProcess process = TestingProcessManager.start(args); assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); + Main main = process.getProcess(); - if (StorageLayer.getStorage(process.getProcess()).getType() != STORAGE_TYPE.SQL) { + if (StorageLayer.getStorage(main).getType() != STORAGE_TYPE.SQL || StorageLayer.isInMemDb(main)) { return; } @@ -100,8 +102,9 @@ public void shouldCreatedNewIdsIfDuplicateIdIsFound() throws Exception { TestingProcessManager.TestingProcess process = TestingProcessManager.start(args); assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); + Main main = process.getProcess(); - if (StorageLayer.getStorage(process.getProcess()).getType() != STORAGE_TYPE.SQL) { + if (StorageLayer.getStorage(main).getType() != STORAGE_TYPE.SQL || StorageLayer.isInMemDb(main)) { return; } @@ -140,8 +143,9 @@ public void testGetUsersStatusFilter() throws Exception { TestingProcessManager.TestingProcess process = TestingProcessManager.start(args); assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); + Main main = process.getProcess(); - if (StorageLayer.getStorage(process.getProcess()).getType() != STORAGE_TYPE.SQL) { + if (StorageLayer.getStorage(main).getType() != STORAGE_TYPE.SQL || StorageLayer.isInMemDb(main)) { return; } @@ -209,8 +213,9 @@ public void randomPaginationTest() throws Exception { process.startProcess(); assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); + Main main = process.getProcess(); - if (StorageLayer.getStorage(process.getProcess()).getType() != STORAGE_TYPE.SQL) { + if (StorageLayer.getStorage(main).getType() != STORAGE_TYPE.SQL || StorageLayer.isInMemDb(main)) { return; } diff --git a/src/test/java/io/supertokens/test/bulkimport/ProcessBulkImportUsersCronJobTest.java b/src/test/java/io/supertokens/test/bulkimport/ProcessBulkImportUsersCronJobTest.java index da6b9050f..a8e4cbc52 100644 --- a/src/test/java/io/supertokens/test/bulkimport/ProcessBulkImportUsersCronJobTest.java +++ b/src/test/java/io/supertokens/test/bulkimport/ProcessBulkImportUsersCronJobTest.java @@ -94,6 +94,10 @@ public void shouldProcessBulkImportUsersInTheSameTenant() throws Exception { TestingProcess process = startCronProcess(); Main main = process.getProcess(); + if (StorageLayer.getBaseStorage(main).getType() != STORAGE_TYPE.SQL || StorageLayer.isInMemDb(main)) { + return; + } + // Create user roles before inserting bulk users { UserRoles.createNewRoleOrModifyItsPermissions(main, "role1", null); @@ -137,6 +141,10 @@ public void shouldProcessBulkImportUsersInMultipleTenantsWithDifferentStorages() TestingProcess process = startCronProcess(); Main main = process.getProcess(); + if (StorageLayer.getBaseStorage(main).getType() != STORAGE_TYPE.SQL || StorageLayer.isInMemDb(main)) { + return; + } + // Create user roles before inserting bulk users { UserRoles.createNewRoleOrModifyItsPermissions(main, "role1", null); @@ -197,6 +205,10 @@ public void shouldDeleteEverythingFromtheDBIfAnythingFails() throws Exception { TestingProcess process = startCronProcess(); Main main = process.getProcess(); + if (StorageLayer.getBaseStorage(main).getType() != STORAGE_TYPE.SQL || StorageLayer.isInMemDb(main)) { + return; + } + createTenants(main); BulkImportSQLStorage storage = (BulkImportSQLStorage) StorageLayer.getStorage(main); @@ -225,6 +237,10 @@ public void shouldThrowTenantDoesNotExistError() throws Exception { TestingProcess process = startCronProcess(); Main main = process.getProcess(); + if (StorageLayer.getBaseStorage(main).getType() != STORAGE_TYPE.SQL || StorageLayer.isInMemDb(main)) { + return; + } + BulkImportSQLStorage storage = (BulkImportSQLStorage) StorageLayer.getStorage(main); AppIdentifier appIdentifier = new AppIdentifier(null, null); @@ -254,6 +270,10 @@ public void shouldThrowTenantHaveDifferentStoragesError() throws Exception { TestingProcess process = startCronProcess(); Main main = process.getProcess(); + if (StorageLayer.getBaseStorage(main).getType() != STORAGE_TYPE.SQL || StorageLayer.isInMemDb(main)) { + return; + } + BulkImportSQLStorage storage = (BulkImportSQLStorage) StorageLayer.getStorage(main); AppIdentifier appIdentifier = new AppIdentifier(null, null); diff --git a/src/test/java/io/supertokens/test/bulkimport/apis/AddBulkImportUsersTest.java b/src/test/java/io/supertokens/test/bulkimport/apis/AddBulkImportUsersTest.java index 90fad26cb..2ff85b4a4 100644 --- a/src/test/java/io/supertokens/test/bulkimport/apis/AddBulkImportUsersTest.java +++ b/src/test/java/io/supertokens/test/bulkimport/apis/AddBulkImportUsersTest.java @@ -85,24 +85,25 @@ public void beforeEach() { public void shouldThrow400IfUsersAreMissingInRequestBody() throws Exception { TestingProcessManager.TestingProcess process = TestingProcessManager.start(new String[] { "../" }); assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); + Main main = process.getProcess(); - if (StorageLayer.getStorage(process.getProcess()).getType() != STORAGE_TYPE.SQL) { + if (StorageLayer.getBaseStorage(main).getType() != STORAGE_TYPE.SQL || StorageLayer.isInMemDb(main)) { return; } // CASE 1: users field is not present - testBadRequest(process.getProcess(), new JsonObject(), "Field name 'users' is invalid in JSON input"); + testBadRequest(main, new JsonObject(), "Field name 'users' is invalid in JSON input"); // CASE 2: users field type in incorrect - testBadRequest(process.getProcess(), new JsonParser().parse("{\"users\": \"string\"}").getAsJsonObject(), + testBadRequest(main, new JsonParser().parse("{\"users\": \"string\"}").getAsJsonObject(), "Field name 'users' is invalid in JSON input"); // CASE 3: users array is empty - testBadRequest(process.getProcess(), generateUsersJson(0).getAsJsonObject(), + testBadRequest(main, generateUsersJson(0).getAsJsonObject(), "{\"error\":\"You need to add at least one user.\"}"); // CASE 4: users array length is greater than 10000 - testBadRequest(process.getProcess(), generateUsersJson(10001).getAsJsonObject(), + testBadRequest(main, generateUsersJson(10001).getAsJsonObject(), "{\"error\":\"You can only add 10000 users at a time.\"}"); process.kill(); @@ -113,24 +114,25 @@ public void shouldThrow400IfUsersAreMissingInRequestBody() throws Exception { public void shouldThrow400IfLoginMethodsAreMissingInUserObject() throws Exception { TestingProcessManager.TestingProcess process = TestingProcessManager.start(new String[] { "../" }); assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); + Main main = process.getProcess(); - if (StorageLayer.getStorage(process.getProcess()).getType() != STORAGE_TYPE.SQL) { + if (StorageLayer.getBaseStorage(main).getType() != STORAGE_TYPE.SQL || StorageLayer.isInMemDb(main)) { return; } // CASE 1: loginMethods field is not present - testBadRequest(process.getProcess(), new JsonParser().parse("{\"users\":[{}]}").getAsJsonObject(), + testBadRequest(main, new JsonParser().parse("{\"users\":[{}]}").getAsJsonObject(), "{\"error\":\"" + genericErrMsg + "\",\"users\":[{\"index\":0,\"errors\":[\"loginMethods is required.\"]}]}"); // CASE 2: loginMethods field type in incorrect - testBadRequest(process.getProcess(), + testBadRequest(main, new JsonParser().parse("{\"users\":[{\"loginMethods\": \"string\"}]}").getAsJsonObject(), "{\"error\":\"" + genericErrMsg + "\",\"users\":[{\"index\":0,\"errors\":[\"loginMethods should be of type array of object.\"]}]}"); // CASE 3: loginMethods array is empty - testBadRequest(process.getProcess(), + testBadRequest(main, new JsonParser().parse("{\"users\":[{\"loginMethods\": []}]}").getAsJsonObject(), "{\"error\":\"" + genericErrMsg + "\",\"users\":[{\"index\":0,\"errors\":[\"At least one loginMethod is required.\"]}]}"); @@ -143,8 +145,9 @@ public void shouldThrow400IfLoginMethodsAreMissingInUserObject() throws Exceptio public void shouldThrow400IfNonRequiredFieldsHaveInvalidType() throws Exception { TestingProcessManager.TestingProcess process = TestingProcessManager.start(new String[] { "../" }); assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); + Main main = process.getProcess(); - if (StorageLayer.getStorage(process.getProcess()).getType() != STORAGE_TYPE.SQL) { + if (StorageLayer.getBaseStorage(main).getType() != STORAGE_TYPE.SQL || StorageLayer.isInMemDb(main)) { return; } @@ -152,7 +155,7 @@ public void shouldThrow400IfNonRequiredFieldsHaveInvalidType() throws Exception .parse("{\"users\":[{\"externalUserId\":[],\"userMetaData\":[],\"userRoles\":{},\"totpDevices\":{}}]}") .getAsJsonObject(); - testBadRequest(process.getProcess(), requestBody, + testBadRequest(main, requestBody, "{\"error\":\"" + genericErrMsg + "\",\"users\":[{\"index\":0,\"errors\":[\"externalUserId should be of type string.\",\"userRoles should be of type array of object.\",\"totpDevices should be of type array of object.\",\"loginMethods is required.\"]}]}"); @@ -164,15 +167,16 @@ public void shouldThrow400IfNonRequiredFieldsHaveInvalidType() throws Exception public void shouldThrow400IfNonUniqueExternalIdsArePassed() throws Exception { TestingProcessManager.TestingProcess process = TestingProcessManager.start(new String[] { "../" }); assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); + Main main = process.getProcess(); - if (StorageLayer.getStorage(process.getProcess()).getType() != STORAGE_TYPE.SQL) { + if (StorageLayer.getBaseStorage(main).getType() != STORAGE_TYPE.SQL || StorageLayer.isInMemDb(main)) { return; } JsonObject requestBody = new JsonParser() .parse("{\"users\":[{\"externalUserId\":\"id1\"}, {\"externalUserId\":\"id1\"}]}") .getAsJsonObject(); - testBadRequest(process.getProcess(), requestBody, "{\"error\":\"" + genericErrMsg + testBadRequest(main, requestBody, "{\"error\":\"" + genericErrMsg + "\",\"users\":[{\"index\":0,\"errors\":[\"loginMethods is required.\"]},{\"index\":1,\"errors\":[\"loginMethods is required.\",\"externalUserId id1 is not unique. It is already used by another user.\"]}]}"); process.kill(); @@ -183,8 +187,9 @@ public void shouldThrow400IfNonUniqueExternalIdsArePassed() throws Exception { public void shouldThrow400IfTotpDevicesAreNotPassedCorrectly() throws Exception { TestingProcessManager.TestingProcess process = TestingProcessManager.start(new String[] { "../" }); assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); + Main main = process.getProcess(); - if (StorageLayer.getStorage(process.getProcess()).getType() != STORAGE_TYPE.SQL) { + if (StorageLayer.getBaseStorage(main).getType() != STORAGE_TYPE.SQL || StorageLayer.isInMemDb(main)) { return; } @@ -193,12 +198,12 @@ public void shouldThrow400IfTotpDevicesAreNotPassedCorrectly() throws Exception .parse("{\"users\":[{\"totpDevices\":[{\"secret\": \"secret\"}]}]}") .getAsJsonObject(); - testBadRequest(process.getProcess(), requestBody, "{\"error\":\"" + genericErrMsg + testBadRequest(main, requestBody, "{\"error\":\"" + genericErrMsg + "\",\"users\":[{\"index\":0,\"errors\":[\"MFA must be enabled to import totp devices.\",\"loginMethods is required.\"]}]}"); // CASE 2: secretKey is required in totpDevices - setFeatureFlags(process.getProcess(), new EE_FEATURES[] { EE_FEATURES.MFA }); - testBadRequest(process.getProcess(), requestBody, "{\"error\":\"" + genericErrMsg + setFeatureFlags(main, new EE_FEATURES[] { EE_FEATURES.MFA }); + testBadRequest(main, requestBody, "{\"error\":\"" + genericErrMsg + "\",\"users\":[{\"index\":0,\"errors\":[\"secretKey is required for a totp device.\",\"loginMethods is required.\"]}]}"); process.kill(); @@ -209,14 +214,15 @@ public void shouldThrow400IfTotpDevicesAreNotPassedCorrectly() throws Exception public void shouldThrow400IfUserRolesAreNotPassedCorrectly() throws Exception { TestingProcessManager.TestingProcess process = TestingProcessManager.start(new String[] { "../" }); assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); + Main main = process.getProcess(); - if (StorageLayer.getStorage(process.getProcess()).getType() != STORAGE_TYPE.SQL) { + if (StorageLayer.getBaseStorage(main).getType() != STORAGE_TYPE.SQL || StorageLayer.isInMemDb(main)) { return; } // Create user roles { - UserRoles.createNewRoleOrModifyItsPermissions(process.getProcess(), "role1", null); + UserRoles.createNewRoleOrModifyItsPermissions(main, "role1", null); } // CASE 1: tenantIds is required for a user role @@ -224,7 +230,7 @@ public void shouldThrow400IfUserRolesAreNotPassedCorrectly() throws Exception { .parse("{\"users\":[{\"userRoles\":[{\"role\":\"role1\"}]}]}") .getAsJsonObject(); - testBadRequest(process.getProcess(), requestBody, "{\"error\":\"" + genericErrMsg + testBadRequest(main, requestBody, "{\"error\":\"" + genericErrMsg + "\",\"users\":[{\"index\":0,\"errors\":[\"tenantIds is required for a user role.\",\"loginMethods is required.\"]}]}"); // CASE 2: Role doesn't exist @@ -232,7 +238,7 @@ public void shouldThrow400IfUserRolesAreNotPassedCorrectly() throws Exception { .parse("{\"users\":[{\"userRoles\":[{\"role\":\"role5\", \"tenantIds\": [\"public\"]}]}]}") .getAsJsonObject(); - testBadRequest(process.getProcess(), requestBody2, "{\"error\":\"" + genericErrMsg + testBadRequest(main, requestBody2, "{\"error\":\"" + genericErrMsg + "\",\"users\":[{\"index\":0,\"errors\":[\"Role role5 does not exist.\",\"loginMethods is required.\"]}]}"); process.kill(); @@ -243,8 +249,9 @@ public void shouldThrow400IfUserRolesAreNotPassedCorrectly() throws Exception { public void shouldThrow400IfLoginMethodsHaveInvalidFieldType() throws Exception { TestingProcessManager.TestingProcess process = TestingProcessManager.start(new String[] { "../" }); assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); + Main main = process.getProcess(); - if (StorageLayer.getStorage(process.getProcess()).getType() != STORAGE_TYPE.SQL) { + if (StorageLayer.getBaseStorage(main).getType() != STORAGE_TYPE.SQL || StorageLayer.isInMemDb(main)) { return; } @@ -254,7 +261,7 @@ public void shouldThrow400IfLoginMethodsHaveInvalidFieldType() throws Exception "{\"users\":[{\"loginMethods\":[{\"recipeId\":[],\"tenantIds\":{},\"isPrimary\":[],\"isVerified\":[],\"timeJoinedInMSSinceEpoch\":[]}]}]}") .getAsJsonObject(); - testBadRequest(process.getProcess(), requestBody, "{\"error\":\"" + genericErrMsg + testBadRequest(main, requestBody, "{\"error\":\"" + genericErrMsg + "\",\"users\":[{\"index\":0,\"errors\":[\"recipeId should be of type string for a loginMethod.\",\"tenantIds should be of type array of string for a loginMethod.\",\"isVerified should be of type boolean for a loginMethod.\",\"isPrimary should be of type boolean for a loginMethod.\",\"timeJoinedInMSSinceEpoch should be of type integer for a loginMethod\"]}]}"); // CASE 2: recipeId is invalid @@ -262,7 +269,7 @@ public void shouldThrow400IfLoginMethodsHaveInvalidFieldType() throws Exception .parse("{\"users\":[{\"loginMethods\":[{\"recipeId\":\"invalid_recipe_id\"}]}]}") .getAsJsonObject(); - testBadRequest(process.getProcess(), requestBody2, "{\"error\":\"" + genericErrMsg + testBadRequest(main, requestBody2, "{\"error\":\"" + genericErrMsg + "\",\"users\":[{\"index\":0,\"errors\":[\"Invalid recipeId for loginMethod. Pass one of emailpassword, thirdparty or, passwordless!\"]}]}"); process.kill(); @@ -273,8 +280,9 @@ public void shouldThrow400IfLoginMethodsHaveInvalidFieldType() throws Exception public void shouldThrow400IfEmailPasswordRecipeHasInvalidFieldTypes() throws Exception { TestingProcessManager.TestingProcess process = TestingProcessManager.start(new String[] { "../" }); assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); + Main main = process.getProcess(); - if (StorageLayer.getStorage(process.getProcess()).getType() != STORAGE_TYPE.SQL) { + if (StorageLayer.getBaseStorage(main).getType() != STORAGE_TYPE.SQL || StorageLayer.isInMemDb(main)) { return; } @@ -283,7 +291,7 @@ public void shouldThrow400IfEmailPasswordRecipeHasInvalidFieldTypes() throws Exc .parse("{\"users\":[{\"loginMethods\":[{\"recipeId\":\"emailpassword\"}]}]}") .getAsJsonObject(); - testBadRequest(process.getProcess(), requestBody, "{\"error\":\"" + genericErrMsg + testBadRequest(main, requestBody, "{\"error\":\"" + genericErrMsg + "\",\"users\":[{\"index\":0,\"errors\":[\"email is required for an emailpassword recipe.\",\"passwordHash is required for an emailpassword recipe.\",\"hashingAlgorithm is required for an emailpassword recipe.\"]}]}"); // CASE 2: email, passwordHash and hashingAlgorithm field type is incorrect @@ -292,7 +300,7 @@ public void shouldThrow400IfEmailPasswordRecipeHasInvalidFieldTypes() throws Exc "{\"users\":[{\"loginMethods\":[{\"recipeId\":\"emailpassword\",\"email\":[],\"passwordHash\":[],\"hashingAlgorithm\":[]}]}]}") .getAsJsonObject(); - testBadRequest(process.getProcess(), requestBody2, "{\"error\":\"" + genericErrMsg + testBadRequest(main, requestBody2, "{\"error\":\"" + genericErrMsg + "\",\"users\":[{\"index\":0,\"errors\":[\"email should be of type string for an emailpassword recipe.\",\"passwordHash should be of type string for an emailpassword recipe.\",\"hashingAlgorithm should be of type string for an emailpassword recipe.\"]}]}"); // CASE 3: hashingAlgorithm is not one of bcrypt, argon2, firebase_scrypt @@ -301,7 +309,7 @@ public void shouldThrow400IfEmailPasswordRecipeHasInvalidFieldTypes() throws Exc "{\"users\":[{\"loginMethods\":[{\"recipeId\":\"emailpassword\",\"email\":\"johndoe@gmail.com\",\"passwordHash\":\"$2a\",\"hashingAlgorithm\":\"invalid_algorithm\"}]}]}") .getAsJsonObject(); - testBadRequest(process.getProcess(), requestBody3, "{\"error\":\"" + genericErrMsg + testBadRequest(main, requestBody3, "{\"error\":\"" + genericErrMsg + "\",\"users\":[{\"index\":0,\"errors\":[\"Invalid hashingAlgorithm for emailpassword recipe. Pass one of bcrypt, argon2 or, firebase_scrypt!\"]}]}"); process.kill(); @@ -312,8 +320,9 @@ public void shouldThrow400IfEmailPasswordRecipeHasInvalidFieldTypes() throws Exc public void shouldThrow400IfThirdPartyRecipeHasInvalidFieldTypes() throws Exception { TestingProcessManager.TestingProcess process = TestingProcessManager.start(new String[] { "../" }); assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); + Main main = process.getProcess(); - if (StorageLayer.getStorage(process.getProcess()).getType() != STORAGE_TYPE.SQL) { + if (StorageLayer.getBaseStorage(main).getType() != STORAGE_TYPE.SQL || StorageLayer.isInMemDb(main)) { return; } @@ -322,7 +331,7 @@ public void shouldThrow400IfThirdPartyRecipeHasInvalidFieldTypes() throws Except .parse("{\"users\":[{\"loginMethods\":[{\"recipeId\":\"thirdparty\"}]}]}") .getAsJsonObject(); - testBadRequest(process.getProcess(), requestBody, "{\"error\":\"" + genericErrMsg + testBadRequest(main, requestBody, "{\"error\":\"" + genericErrMsg + "\",\"users\":[{\"index\":0,\"errors\":[\"email is required for a thirdparty recipe.\",\"thirdPartyId is required for a thirdparty recipe.\",\"thirdPartyUserId is required for a thirdparty recipe.\"]}]}"); // CASE 2: email, passwordHash and thirdPartyUserId field type is incorrect @@ -331,7 +340,7 @@ public void shouldThrow400IfThirdPartyRecipeHasInvalidFieldTypes() throws Except "{\"users\":[{\"loginMethods\":[{\"recipeId\":\"thirdparty\",\"email\":[],\"thirdPartyId\":[],\"thirdPartyUserId\":[]}]}]}") .getAsJsonObject(); - testBadRequest(process.getProcess(), requestBody2, "{\"error\":\"" + genericErrMsg + testBadRequest(main, requestBody2, "{\"error\":\"" + genericErrMsg + "\",\"users\":[{\"index\":0,\"errors\":[\"email should be of type string for a thirdparty recipe.\",\"thirdPartyId should be of type string for a thirdparty recipe.\",\"thirdPartyUserId should be of type string for a thirdparty recipe.\"]}]}"); process.kill(); @@ -342,8 +351,9 @@ public void shouldThrow400IfThirdPartyRecipeHasInvalidFieldTypes() throws Except public void shouldThrow400IfPasswordlessRecipeHasInvalidFieldTypes() throws Exception { TestingProcessManager.TestingProcess process = TestingProcessManager.start(new String[] { "../" }); assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); + Main main = process.getProcess(); - if (StorageLayer.getStorage(process.getProcess()).getType() != STORAGE_TYPE.SQL) { + if (StorageLayer.getBaseStorage(main).getType() != STORAGE_TYPE.SQL || StorageLayer.isInMemDb(main)) { return; } @@ -352,7 +362,7 @@ public void shouldThrow400IfPasswordlessRecipeHasInvalidFieldTypes() throws Exce .parse("{\"users\":[{\"loginMethods\":[{\"recipeId\":\"passwordless\"}]}]}") .getAsJsonObject(); - testBadRequest(process.getProcess(), requestBody, "{\"error\":\"" + genericErrMsg + testBadRequest(main, requestBody, "{\"error\":\"" + genericErrMsg + "\",\"users\":[{\"index\":0,\"errors\":[\"Either email or phoneNumber is required for a passwordless recipe.\"]}]}"); // CASE 2: email and phoneNumber field type is incorrect @@ -361,7 +371,7 @@ public void shouldThrow400IfPasswordlessRecipeHasInvalidFieldTypes() throws Exce "{\"users\":[{\"loginMethods\":[{\"recipeId\":\"passwordless\",\"email\":[],\"phoneNumber\":[]}]}]}") .getAsJsonObject(); - testBadRequest(process.getProcess(), requestBody2, "{\"error\":\"" + genericErrMsg + testBadRequest(main, requestBody2, "{\"error\":\"" + genericErrMsg + "\",\"users\":[{\"index\":0,\"errors\":[\"email should be of type string for a passwordless recipe.\",\"phoneNumber should be of type string for a passwordless recipe.\",\"Either email or phoneNumber is required for a passwordless recipe.\"]}]}"); process.kill(); @@ -372,15 +382,16 @@ public void shouldThrow400IfPasswordlessRecipeHasInvalidFieldTypes() throws Exce public void shouldThrow400IfAUserHasMultipleLoginMethodsAndAccountLinkingIsDisabled() throws Exception { TestingProcessManager.TestingProcess process = TestingProcessManager.start(new String[] { "../" }); assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); + Main main = process.getProcess(); - if (StorageLayer.getStorage(process.getProcess()).getType() != STORAGE_TYPE.SQL) { + if (StorageLayer.getBaseStorage(main).getType() != STORAGE_TYPE.SQL || StorageLayer.isInMemDb(main)) { return; } JsonObject requestBody = new JsonParser() .parse("{\"users\":[{\"loginMethods\":[{\"recipeId\":\"emailpassword\",\"email\":\"johndoe@gmail.com\",\"passwordHash\":\"$2a\",\"hashingAlgorithm\":\"bcrypt\",\"isPrimary\":true},{\"recipeId\":\"passwordless\",\"email\":\"johndoe@gmail.com\"}]}]}") .getAsJsonObject(); - testBadRequest(process.getProcess(), requestBody, "{\"error\":\"" + genericErrMsg + testBadRequest(main, requestBody, "{\"error\":\"" + genericErrMsg + "\",\"users\":[{\"index\":0,\"errors\":[\"Account linking must be enabled to import multiple loginMethods.\"]}]}"); process.kill(); @@ -391,8 +402,9 @@ public void shouldThrow400IfAUserHasMultipleLoginMethodsAndAccountLinkingIsDisab public void shouldThrow400IfInvalidTenantIdIsPassed() throws Exception { TestingProcessManager.TestingProcess process = TestingProcessManager.start(new String[] { "../" }); assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); + Main main = process.getProcess(); - if (StorageLayer.getStorage(process.getProcess()).getType() != STORAGE_TYPE.SQL) { + if (StorageLayer.getBaseStorage(main).getType() != STORAGE_TYPE.SQL || StorageLayer.isInMemDb(main)) { return; } @@ -402,11 +414,11 @@ public void shouldThrow400IfInvalidTenantIdIsPassed() throws Exception { "{\"users\":[{\"loginMethods\":[{\"tenantIds\":[\"invalid\"],\"recipeId\":\"passwordless\",\"email\":\"johndoe@gmail.com\"}]}]}") .getAsJsonObject(); - testBadRequest(process.getProcess(), requestBody, "{\"error\":\"" + genericErrMsg + testBadRequest(main, requestBody, "{\"error\":\"" + genericErrMsg + "\",\"users\":[{\"index\":0,\"errors\":[\"Multitenancy must be enabled before importing users to a different tenant.\"]}]}"); // CASE 2: Invalid tenantId - setFeatureFlags(process.getProcess(), + setFeatureFlags(main, new EE_FEATURES[] { EE_FEATURES.ACCOUNT_LINKING, EE_FEATURES.MULTI_TENANCY }); JsonObject requestBody2 = new JsonParser() @@ -414,18 +426,18 @@ public void shouldThrow400IfInvalidTenantIdIsPassed() throws Exception { "{\"users\":[{\"loginMethods\":[{\"tenantIds\":[\"invalid\"],\"recipeId\":\"passwordless\",\"email\":\"johndoe@gmail.com\"}]}]}") .getAsJsonObject(); - testBadRequest(process.getProcess(), requestBody2, "{\"error\":\"" + genericErrMsg + testBadRequest(main, requestBody2, "{\"error\":\"" + genericErrMsg + "\",\"users\":[{\"index\":0,\"errors\":[\"Invalid tenantId: invalid for passwordless recipe.\"]}]}"); // CASE 3: Two or more tenants do not share the same storage - createTenants(process.getProcess()); + createTenants(main); JsonObject requestBody3 = new JsonParser().parse( "{\"users\":[{\"loginMethods\":[{\"tenantIds\":[\"public\"],\"recipeId\":\"passwordless\",\"email\":\"johndoe@gmail.com\"}, {\"tenantIds\":[\"t2\"],\"recipeId\":\"thirdparty\", \"email\":\"johndoe@gmail.com\", \"thirdPartyId\":\"id\", \"thirdPartyUserId\":\"id\"}]}]}") .getAsJsonObject(); - testBadRequest(process.getProcess(), requestBody3, "{\"error\":\"" + genericErrMsg + testBadRequest(main, requestBody3, "{\"error\":\"" + genericErrMsg + "\",\"users\":[{\"index\":0,\"errors\":[\"All tenants for a user must share the same database for thirdparty recipe.\"]}]}"); process.kill(); @@ -436,19 +448,20 @@ public void shouldThrow400IfInvalidTenantIdIsPassed() throws Exception { public void shouldThrow400IfTwoLoginMethodsHaveIsPrimaryTrue() throws Exception { TestingProcessManager.TestingProcess process = TestingProcessManager.start(new String[] { "../" }); assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); + Main main = process.getProcess(); - if (StorageLayer.getStorage(process.getProcess()).getType() != STORAGE_TYPE.SQL) { + if (StorageLayer.getBaseStorage(main).getType() != STORAGE_TYPE.SQL || StorageLayer.isInMemDb(main)) { return; } - setFeatureFlags(process.getProcess(), + setFeatureFlags(main, new EE_FEATURES[] { EE_FEATURES.ACCOUNT_LINKING, EE_FEATURES.MULTI_TENANCY }); JsonObject requestBody = new JsonParser() .parse("{\"users\":[{\"loginMethods\":[{\"recipeId\":\"emailpassword\",\"email\":\"johndoe@gmail.com\",\"passwordHash\":\"$2a\",\"hashingAlgorithm\":\"bcrypt\",\"isPrimary\":true},{\"recipeId\":\"passwordless\",\"email\":\"johndoe@gmail.com\",\"isPrimary\":true}]}]}") .getAsJsonObject(); - testBadRequest(process.getProcess(), requestBody, "{\"error\":\"" + genericErrMsg + testBadRequest(main, requestBody, "{\"error\":\"" + genericErrMsg + "\",\"users\":[{\"index\":0,\"errors\":[\"No two loginMethods can have isPrimary as true.\"]}]}"); process.kill(); @@ -461,21 +474,22 @@ public void shouldReturn200Response() throws Exception { TestingProcessManager.TestingProcess process = TestingProcessManager.start(args); assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); + Main main = process.getProcess(); - if (StorageLayer.getStorage(process.getProcess()).getType() != STORAGE_TYPE.SQL) { + if (StorageLayer.getBaseStorage(main).getType() != STORAGE_TYPE.SQL || StorageLayer.isInMemDb(main)) { return; } - setFeatureFlags(process.getProcess(), new EE_FEATURES[] { EE_FEATURES.MFA }); + setFeatureFlags(main, new EE_FEATURES[] { EE_FEATURES.MFA }); // Create user roles before inserting bulk users { - UserRoles.createNewRoleOrModifyItsPermissions(process.getProcess(), "role1", null); - UserRoles.createNewRoleOrModifyItsPermissions(process.getProcess(), "role2", null); + UserRoles.createNewRoleOrModifyItsPermissions(main, "role1", null); + UserRoles.createNewRoleOrModifyItsPermissions(main, "role2", null); } JsonObject request = generateUsersJson(10000); - JsonObject response = HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", + JsonObject response = HttpRequestForTesting.sendJsonPOSTRequest(main, "", "http://localhost:3567/bulk-import/users", request, 1000, 10000, null, Utils.getCdiVersionStringLatestForTests(), null); assertEquals("OK", response.get("status").getAsString()); @@ -490,26 +504,27 @@ public void shouldNormaliseFields() throws Exception { TestingProcessManager.TestingProcess process = TestingProcessManager.start(args); assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); + Main main = process.getProcess(); - if (StorageLayer.getStorage(process.getProcess()).getType() != STORAGE_TYPE.SQL) { + if (StorageLayer.getBaseStorage(main).getType() != STORAGE_TYPE.SQL || StorageLayer.isInMemDb(main)) { return; } - setFeatureFlags(process.getProcess(), new EE_FEATURES[] { EE_FEATURES.MFA }); + setFeatureFlags(main, new EE_FEATURES[] { EE_FEATURES.MFA }); // Create user roles before inserting bulk users { - UserRoles.createNewRoleOrModifyItsPermissions(process.getProcess(), "role1", null); - UserRoles.createNewRoleOrModifyItsPermissions(process.getProcess(), "role2", null); + UserRoles.createNewRoleOrModifyItsPermissions(main, "role1", null); + UserRoles.createNewRoleOrModifyItsPermissions(main, "role2", null); } JsonObject request = generateUsersJson(1); - JsonObject response = HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", + JsonObject response = HttpRequestForTesting.sendJsonPOSTRequest(main, "", "http://localhost:3567/bulk-import/users", request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); assertEquals("OK", response.get("status").getAsString()); - JsonObject getResponse = HttpRequestForTesting.sendGETRequest(process.getProcess(), "", + JsonObject getResponse = HttpRequestForTesting.sendGETRequest(main, "", "http://localhost:3567/bulk-import/users", new HashMap<>(), 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); diff --git a/src/test/java/io/supertokens/test/bulkimport/apis/DeleteBulkImportUsersTest.java b/src/test/java/io/supertokens/test/bulkimport/apis/DeleteBulkImportUsersTest.java index 50042e7d6..ebd9b3624 100644 --- a/src/test/java/io/supertokens/test/bulkimport/apis/DeleteBulkImportUsersTest.java +++ b/src/test/java/io/supertokens/test/bulkimport/apis/DeleteBulkImportUsersTest.java @@ -34,6 +34,7 @@ import com.google.gson.JsonParser; import com.google.gson.JsonPrimitive; +import io.supertokens.Main; import io.supertokens.ProcessState; import io.supertokens.bulkimport.BulkImport; import io.supertokens.pluginInterface.STORAGE_TYPE; @@ -67,15 +68,16 @@ public void shouldReturn400Error() throws Exception { TestingProcessManager.TestingProcess process = TestingProcessManager.start(args); assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); + Main main = process.getProcess(); - if (StorageLayer.getStorage(process.getProcess()).getType() != STORAGE_TYPE.SQL) { + if (StorageLayer.getStorage(main).getType() != STORAGE_TYPE.SQL || StorageLayer.isInMemDb(main)) { return; } { try { JsonObject request = new JsonObject(); - HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", + HttpRequestForTesting.sendJsonPOSTRequest(main, "", "http://localhost:3567/bulk-import/users/remove", request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); } catch (io.supertokens.test.httpRequest.HttpResponseException e) { @@ -86,7 +88,7 @@ public void shouldReturn400Error() throws Exception { { try { JsonObject request = new JsonParser().parse("{\"ids\":[]}").getAsJsonObject(); - HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", + HttpRequestForTesting.sendJsonPOSTRequest(main, "", "http://localhost:3567/bulk-import/users/remove", request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); } catch (io.supertokens.test.httpRequest.HttpResponseException e) { @@ -97,7 +99,7 @@ public void shouldReturn400Error() throws Exception { { try { JsonObject request = new JsonParser().parse("{\"ids\":[\"\"]}").getAsJsonObject(); - HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", + HttpRequestForTesting.sendJsonPOSTRequest(main, "", "http://localhost:3567/bulk-import/users/remove", request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); } catch (io.supertokens.test.httpRequest.HttpResponseException e) { @@ -115,7 +117,7 @@ public void shouldReturn400Error() throws Exception { } request.add("ids", ids); - HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", + HttpRequestForTesting.sendJsonPOSTRequest(main, "", "http://localhost:3567/bulk-import/users/remove", request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); } catch (io.supertokens.test.httpRequest.HttpResponseException e) { @@ -134,8 +136,9 @@ public void shouldReturn200Response() throws Exception { TestingProcessManager.TestingProcess process = TestingProcessManager.start(args); assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); + Main main = process.getProcess(); - if (StorageLayer.getStorage(process.getProcess()).getType() != STORAGE_TYPE.SQL) { + if (StorageLayer.getStorage(main).getType() != STORAGE_TYPE.SQL || StorageLayer.isInMemDb(main)) { return; } @@ -156,7 +159,7 @@ public void shouldReturn200Response() throws Exception { request.add("ids", validIds); - JsonObject response = HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", + JsonObject response = HttpRequestForTesting.sendJsonPOSTRequest(main, "", "http://localhost:3567/bulk-import/users/remove", request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); diff --git a/src/test/java/io/supertokens/test/bulkimport/apis/GetBulkImportUsersTest.java b/src/test/java/io/supertokens/test/bulkimport/apis/GetBulkImportUsersTest.java index 181bcd336..8db075610 100644 --- a/src/test/java/io/supertokens/test/bulkimport/apis/GetBulkImportUsersTest.java +++ b/src/test/java/io/supertokens/test/bulkimport/apis/GetBulkImportUsersTest.java @@ -34,6 +34,7 @@ import com.google.gson.JsonObject; import com.google.gson.JsonParser; +import io.supertokens.Main; import io.supertokens.ProcessState; import io.supertokens.pluginInterface.STORAGE_TYPE; import io.supertokens.pluginInterface.bulkimport.BulkImportUser; @@ -62,15 +63,16 @@ public void shouldReturn400Error() throws Exception { TestingProcessManager.TestingProcess process = TestingProcessManager.start(args); assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); + Main main = process.getProcess(); - if (StorageLayer.getStorage(process.getProcess()).getType() != STORAGE_TYPE.SQL) { + if (StorageLayer.getStorage(main).getType() != STORAGE_TYPE.SQL || StorageLayer.isInMemDb(main)) { return; } try { Map params = new HashMap<>(); params.put("status", "INVALID_STATUS"); - HttpRequestForTesting.sendGETRequest(process.getProcess(), "", + HttpRequestForTesting.sendGETRequest(main, "", "http://localhost:3567/bulk-import/users", params, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); fail("The API should have thrown an error"); @@ -84,7 +86,7 @@ public void shouldReturn400Error() throws Exception { try { Map params = new HashMap<>(); params.put("limit", "0"); - HttpRequestForTesting.sendGETRequest(process.getProcess(), "", + HttpRequestForTesting.sendGETRequest(main, "", "http://localhost:3567/bulk-import/users", params, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); fail("The API should have thrown an error"); @@ -97,7 +99,7 @@ public void shouldReturn400Error() throws Exception { try { Map params = new HashMap<>(); params.put("limit", "501"); - HttpRequestForTesting.sendGETRequest(process.getProcess(), "", + HttpRequestForTesting.sendGETRequest(main, "", "http://localhost:3567/bulk-import/users", params, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); fail("The API should have thrown an error"); @@ -109,7 +111,7 @@ public void shouldReturn400Error() throws Exception { try { Map params = new HashMap<>(); params.put("paginationToken", "invalid_token"); - HttpRequestForTesting.sendGETRequest(process.getProcess(), "", + HttpRequestForTesting.sendGETRequest(main, "", "http://localhost:3567/bulk-import/users", params, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); fail("The API should have thrown an error"); @@ -128,8 +130,9 @@ public void shouldReturn200Response() throws Exception { TestingProcessManager.TestingProcess process = TestingProcessManager.start(args); assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); + Main main = process.getProcess(); - if (StorageLayer.getStorage(process.getProcess()).getType() != STORAGE_TYPE.SQL) { + if (StorageLayer.getStorage(main).getType() != STORAGE_TYPE.SQL || StorageLayer.isInMemDb(main)) { return; } @@ -137,14 +140,14 @@ public void shouldReturn200Response() throws Exception { String rawData = "{\"users\":[{\"loginMethods\":[{\"recipeId\":\"passwordless\",\"email\":\"johndoe@gmail.com\"}]}]}"; { JsonObject request = new JsonParser().parse(rawData).getAsJsonObject(); - JsonObject res = HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", + JsonObject res = HttpRequestForTesting.sendJsonPOSTRequest(main, "", "http://localhost:3567/bulk-import/users", request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); assert res.get("status").getAsString().equals("OK"); } Map params = new HashMap<>(); - JsonObject response = HttpRequestForTesting.sendGETRequest(process.getProcess(), "", + JsonObject response = HttpRequestForTesting.sendGETRequest(main, "", "http://localhost:3567/bulk-import/users", params, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); assertEquals("OK", response.get("status").getAsString()); From c7ae1e486eddf358ac7580161ddbab5721d38c87 Mon Sep 17 00:00:00 2001 From: Ankit Tiwari Date: Mon, 29 Apr 2024 17:57:16 +0530 Subject: [PATCH 15/21] fix: a bug with createTotpDevices --- .../bulkimport/ProcessBulkImportUsers.java | 26 ++++++++++--------- 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkImportUsers.java b/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkImportUsers.java index daf1160e8..44066ce04 100644 --- a/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkImportUsers.java +++ b/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkImportUsers.java @@ -245,7 +245,7 @@ private void processUser(AppIdentifier appIdentifier, BulkImportUser user, BulkI createPrimaryUserAndLinkAccounts(main, appIdentifier, bulkImportProxyStorage, user, primaryLM); createUserIdMapping(main, appIdentifier, user, primaryLM); verifyEmailForAllLoginMethods(appIdentifier, con, bulkImportProxyStorage, user.loginMethods); - createTotpDevices(main, appIdentifier, bulkImportProxyStorage, user.totpDevices, primaryLM); + createTotpDevices(main, appIdentifier, bulkImportProxyStorage, user, primaryLM); createUserMetadata(appIdentifier, bulkImportProxyStorage, user, primaryLM); createUserRoles(main, appIdentifier, bulkImportProxyStorage, user); @@ -522,17 +522,19 @@ private void verifyEmailForAllLoginMethods(AppIdentifier appIdentifier, Transact } private void createTotpDevices(Main main, AppIdentifier appIdentifier, Storage storage, - List totpDevices, LoginMethod primaryLM) throws StorageTransactionLogicException { - for (TotpDevice totpDevice : totpDevices) { - try { - Totp.createDevice(main, appIdentifier, storage, primaryLM.getSuperTokenOrExternalUserId(), - totpDevice.deviceName, totpDevice.skew, totpDevice.period, totpDevice.secretKey, - true, System.currentTimeMillis()); - } catch (TenantOrAppNotFoundException | StorageQueryException | FeatureNotEnabledException e) { - throw new StorageTransactionLogicException(e); - } catch (DeviceAlreadyExistsException e) { - throw new StorageTransactionLogicException( - new Exception("A totp device with name " + totpDevice.deviceName + " already exists")); + BulkImportUser user, LoginMethod primaryLM) throws StorageTransactionLogicException { + if (user.totpDevices != null) { + for (TotpDevice totpDevice : user.totpDevices) { + try { + Totp.createDevice(main, appIdentifier, storage, primaryLM.getSuperTokenOrExternalUserId(), + totpDevice.deviceName, totpDevice.skew, totpDevice.period, totpDevice.secretKey, + true, System.currentTimeMillis()); + } catch (TenantOrAppNotFoundException | StorageQueryException | FeatureNotEnabledException e) { + throw new StorageTransactionLogicException(e); + } catch (DeviceAlreadyExistsException e) { + throw new StorageTransactionLogicException( + new Exception("A totp device with name " + totpDevice.deviceName + " already exists")); + } } } } From d24bebfb0c7482139446d934f25d06d01c5c89b0 Mon Sep 17 00:00:00 2001 From: Ankit Tiwari Date: Wed, 8 May 2024 10:57:14 +0530 Subject: [PATCH 16/21] fix: PR changes --- .../io/supertokens/bulkimport/BulkImport.java | 13 +++- .../bulkimport/BulkImportUserUtils.java | 1 + .../bulkimport/ProcessBulkImportUsers.java | 74 ++++++++++++------- .../storageLayer/StorageLayer.java | 3 + .../api/bulkimport/BulkImportAPI.java | 4 +- .../bulkimport/DeleteBulkImportUserAPI.java | 4 +- 6 files changed, 65 insertions(+), 34 deletions(-) diff --git a/src/main/java/io/supertokens/bulkimport/BulkImport.java b/src/main/java/io/supertokens/bulkimport/BulkImport.java index e552834b7..918b1e903 100644 --- a/src/main/java/io/supertokens/bulkimport/BulkImport.java +++ b/src/main/java/io/supertokens/bulkimport/BulkImport.java @@ -29,16 +29,21 @@ import java.util.List; -import javax.annotation.Nonnull; import javax.annotation.Nullable; public class BulkImport { + // Maximum number of users that can be added in a single /bulk-import/users POST request public static final int MAX_USERS_TO_ADD = 10000; - public static final int GET_USERS_PAGINATION_LIMIT = 500; + // Maximum number of users to return in a single page when calling /bulk-import/users GET + public static final int GET_USERS_PAGINATION_MAX_LIMIT = 500; + // Default number of users to return when no specific limit is given in /bulk-import/users GET public static final int GET_USERS_DEFAULT_LIMIT = 100; - public static final int DELETE_USERS_LIMIT = 500; + // Maximum number of users that can be deleted in a single operation + public static final int DELETE_USERS_MAX_LIMIT = 500; + // Number of users to process in a single batch of ProcessBulkImportUsers Cron Job public static final int PROCESS_USERS_BATCH_SIZE = 1000; + // Time interval in seconds between two consecutive runs of ProcessBulkImportUsers Cron Job public static final int PROCESS_USERS_INTERVAL_SECONDS = 60; public static void addUsers(AppIdentifier appIdentifier, Storage storage, List users) @@ -57,7 +62,7 @@ public static void addUsers(AppIdentifier appIdentifier, Storage storage, List users; diff --git a/src/main/java/io/supertokens/bulkimport/BulkImportUserUtils.java b/src/main/java/io/supertokens/bulkimport/BulkImportUserUtils.java index 1d090b7fe..4c0bc73ba 100644 --- a/src/main/java/io/supertokens/bulkimport/BulkImportUserUtils.java +++ b/src/main/java/io/supertokens/bulkimport/BulkImportUserUtils.java @@ -565,6 +565,7 @@ private void validateTenantIdsForRoleAndLoginMethods(Main main, AppIdentifier ap } else if (!commonTenantUserPoolId.equals(tenantUserPoolId)) { errors.add("All tenants for a user must share the same database for " + loginMethod.recipeId + " recipe."); + break; // Break to avoid adding the same error multiple times for the same loginMethod } } } diff --git a/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkImportUsers.java b/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkImportUsers.java index 44066ce04..847bbaf01 100644 --- a/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkImportUsers.java +++ b/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkImportUsers.java @@ -147,7 +147,7 @@ public int getInitialWaitTimeSeconds() { return 0; } - private Storage getProxyStorage(TenantIdentifier tenantIdentifier) + private synchronized Storage getProxyStorage(TenantIdentifier tenantIdentifier) throws InvalidConfigException, IOException, TenantOrAppNotFoundException, DbInitException, StorageQueryException { String userPoolId = StorageLayer.getStorage(tenantIdentifier, main).getUserPoolId(); if (userPoolToStorageMap.containsKey(userPoolId)) { @@ -166,7 +166,13 @@ private Storage getProxyStorage(TenantIdentifier tenantIdentifier) normalisedConfigs.get(key), tenantIdentifier, true); userPoolToStorageMap.put(userPoolId, bulkImportProxyStorage); - bulkImportProxyStorage.initStorage(true, new ArrayList<>()); + bulkImportProxyStorage.initStorage(false, new ArrayList<>()); + // `BulkImportProxyStorage` uses `BulkImportProxyConnection`, which overrides the `.commit()` method on the Connection object. + // The `initStorage()` method runs `select * from table_name limit 1` queries to check if the tables exist but these queries + // don't get committed due to the overridden `.commit()`, so we need to manually commit the transaction to remove any locks on the tables. + + // Without this commit, a call to `select * from bulk_import_users limit 1` in `doesTableExist()` locks the `bulk_import_users` table, + // causing other queries to stall indefinitely. bulkImportProxyStorage.commitTransactionForBulkImportProxyStorage(); return bulkImportProxyStorage; } @@ -178,13 +184,9 @@ public Storage[] getAllProxyStoragesForApp(Main main, AppIdentifier appIdentifie throws TenantOrAppNotFoundException, InvalidConfigException, IOException, DbInitException, StorageQueryException { List allProxyStorages = new ArrayList<>(); - Map resources = main - .getResourceDistributor() - .getAllResourcesWithResourceKey(StorageLayer.RESOURCE_KEY); - for (ResourceDistributor.KeyClass key : resources.keySet()) { - if (key.getTenantIdentifier().toAppIdentifier().equals(appIdentifier)) { - allProxyStorages.add(getProxyStorage(key.getTenantIdentifier())); - } + TenantConfig[] tenantConfigs = Multitenancy.getAllTenantsForApp(appIdentifier, main); + for (TenantConfig tenantConfig : tenantConfigs) { + allProxyStorages.add(getProxyStorage(tenantConfig.tenantIdentifier)); } return allProxyStorages.toArray(new Storage[0]); } @@ -221,17 +223,39 @@ private void processUser(AppIdentifier appIdentifier, BulkImportUser user, BulkI LoginMethod primaryLM = getPrimaryLoginMethod(user); AuthRecipeSQLStorage authRecipeSQLStorage = (AuthRecipeSQLStorage) getProxyStorage(firstTenantIdentifier); - // If primaryUserId is not null, it means we may have already processed this user but failed to delete the entry - // If the primaryUserId exists in the database, we'll delete the corresponding entry from the bulkImportUser table and proceed to skip this user. + + /* + * We use two separate storage instances: one for importing the user and another for managing bulk_import_users entries. + * This is necessary because the bulk_import_users entries are always in the public tenant storage, + * but the actual user data could be in a different storage. + * + * If transactions are committed individually, in this order: + * 1. Commit the transaction that imports the user. + * 2. Commit the transaction that deletes the corresponding bulk import entry. + * + * There's a risk where the first commit succeeds, but the second fails. This creates a situation where + * the bulk import entry is re-processed, even though the user has already been imported into the database. + * + * To resolve this, we added a `primaryUserId` field to the `bulk_import_users` table. + * The processing logic now follows these steps: + * + * 1. Import the user and get the `primaryUserId` (transaction uncommitted). + * 2. Update the `primaryUserId` in the corresponding bulk import entry. + * 3. Commit the import transaction from step 1. + * 4. Delete the bulk import entry. + * + * If step 2 or any earlier step fails, nothing is committed, preventing partial state. + * If step 3 fails, the `primaryUserId` in the bulk import entry is updated, but the user doesn't exist in the database—this results in re-processing on the next run. + * If step 4 fails, the user exists but the bulk import entry remains; this will be handled by deleting it in the next run. + * + * The following code implements this logic. + */ if (user.primaryUserId != null) { - AuthRecipeUserInfo processedUser = authRecipeSQLStorage.getPrimaryUserById(appIdentifier, + AuthRecipeUserInfo importedUser = authRecipeSQLStorage.getPrimaryUserById(appIdentifier, user.primaryUserId); - if (processedUser != null && isProcessedUserFromSameBulkImportUserEntry(processedUser, user)) { - baseTenantStorage.startTransaction(con2 -> { - baseTenantStorage.deleteBulkImportUser_Transaction(appIdentifier, con2, user.id); - return null; - }); + if (importedUser != null && isProcessedUserFromSameBulkImportUserEntry(importedUser, user)) { + baseTenantStorage.deleteBulkImportUsers(appIdentifier, new String[] { user.id }); return; } } @@ -263,10 +287,7 @@ private void processUser(AppIdentifier appIdentifier, BulkImportUser user, BulkI // NOTE: We need to use the baseTenantStorage as bulkImportProxyStorage could have a different storage than the baseTenantStorage // If this fails, the primaryUserId will be updated in the bulkImportUser and it would exist in the database. // When processing the user again, we'll check if primaryUserId exists with the same email. In this case the user will exist, and we'll simply delete the entry. - baseTenantStorage.startTransaction(con2 -> { - baseTenantStorage.deleteBulkImportUser_Transaction(appIdentifier, con2, user.id); - return null; - }); + baseTenantStorage.deleteBulkImportUsers(appIdentifier, new String[] { user.id }); return null; } catch (StorageTransactionLogicException e) { // We need to rollback the transaction manually because we have overridden that in the proxy storage @@ -555,15 +576,16 @@ private BulkImportUser.LoginMethod getPrimaryLoginMethod(BulkImportUser user) { return oldestLM; } + // Checks if the importedUser was processed from the same bulkImportUser entry. private boolean isProcessedUserFromSameBulkImportUserEntry( - AuthRecipeUserInfo processedUser, BulkImportUser bulkImportUser) { - if (bulkImportUser == null || processedUser == null || bulkImportUser.loginMethods == null || - processedUser.loginMethods == null) { + AuthRecipeUserInfo importedUser, BulkImportUser bulkImportEntry) { + if (bulkImportEntry == null || importedUser == null || bulkImportEntry.loginMethods == null || + importedUser.loginMethods == null) { return false; } - for (LoginMethod lm1 : bulkImportUser.loginMethods) { - for (io.supertokens.pluginInterface.authRecipe.LoginMethod lm2 : processedUser.loginMethods) { + for (LoginMethod lm1 : bulkImportEntry.loginMethods) { + for (io.supertokens.pluginInterface.authRecipe.LoginMethod lm2 : importedUser.loginMethods) { if (lm2.recipeId.toString().equals(lm1.recipeId)) { if (lm1.email != null && !lm1.email.equals(lm2.email)) { return false; diff --git a/src/main/java/io/supertokens/storageLayer/StorageLayer.java b/src/main/java/io/supertokens/storageLayer/StorageLayer.java index 2d09299d0..928bc9c08 100644 --- a/src/main/java/io/supertokens/storageLayer/StorageLayer.java +++ b/src/main/java/io/supertokens/storageLayer/StorageLayer.java @@ -89,6 +89,9 @@ private static Storage getNewInstance(Main main, JsonObject config, TenantIdenti result = storageLayer; } } else { + if (isBulkImportProxy) { + throw new QuitProgramException("Creating a bulk import proxy storage instance with in-memory DB is not supported."); + } result = new Start(main); } } diff --git a/src/main/java/io/supertokens/webserver/api/bulkimport/BulkImportAPI.java b/src/main/java/io/supertokens/webserver/api/bulkimport/BulkImportAPI.java index 496fe3ba2..c30f63f82 100644 --- a/src/main/java/io/supertokens/webserver/api/bulkimport/BulkImportAPI.java +++ b/src/main/java/io/supertokens/webserver/api/bulkimport/BulkImportAPI.java @@ -69,9 +69,9 @@ protected void doGet(HttpServletRequest req, HttpServletResponse resp) throws Se Integer limit = InputParser.getIntQueryParamOrThrowError(req, "limit", true); if (limit != null) { - if (limit > BulkImport.GET_USERS_PAGINATION_LIMIT) { + if (limit > BulkImport.GET_USERS_PAGINATION_MAX_LIMIT) { throw new ServletException( - new BadRequestException("Max limit allowed is " + BulkImport.GET_USERS_PAGINATION_LIMIT)); + new BadRequestException("Max limit allowed is " + BulkImport.GET_USERS_PAGINATION_MAX_LIMIT)); } else if (limit < 1) { throw new ServletException(new BadRequestException("limit must a positive integer with min value 1")); } diff --git a/src/main/java/io/supertokens/webserver/api/bulkimport/DeleteBulkImportUserAPI.java b/src/main/java/io/supertokens/webserver/api/bulkimport/DeleteBulkImportUserAPI.java index a9032cc18..5562fafca 100644 --- a/src/main/java/io/supertokens/webserver/api/bulkimport/DeleteBulkImportUserAPI.java +++ b/src/main/java/io/supertokens/webserver/api/bulkimport/DeleteBulkImportUserAPI.java @@ -62,9 +62,9 @@ protected void doPost(HttpServletRequest req, HttpServletResponse resp) throws S throw new ServletException(new WebserverAPI.BadRequestException("Field name 'ids' cannot be an empty array")); } - if (arr.size() > BulkImport.DELETE_USERS_LIMIT) { + if (arr.size() > BulkImport.DELETE_USERS_MAX_LIMIT) { throw new ServletException(new WebserverAPI.BadRequestException("Field name 'ids' cannot contain more than " - + BulkImport.DELETE_USERS_LIMIT + " elements")); + + BulkImport.DELETE_USERS_MAX_LIMIT + " elements")); } String[] userIds = new String[arr.size()]; From 09c1b85d338f679382bb1afeda84d8df02be6adf Mon Sep 17 00:00:00 2001 From: Ankit Tiwari Date: Thu, 23 May 2024 12:38:14 +0530 Subject: [PATCH 17/21] feat: Add an api to import user in sync --- .../io/supertokens/bulkimport/BulkImport.java | 417 +++++++++++++++++- .../bulkimport/ProcessBulkImportUsers.java | 326 ++------------ .../io/supertokens/webserver/Webserver.java | 2 + .../api/bulkimport/ImportUserAPI.java | 103 +++++ .../test/bulkimport/BulkImportTest.java | 193 +++++++- .../test/bulkimport/BulkImportTestUtils.java | 137 +++++- .../ProcessBulkImportUsersCronJobTest.java | 133 +----- .../apis/AddBulkImportUsersTest.java | 56 +-- .../test/bulkimport/apis/ImportUserTest.java | 151 +++++++ 9 files changed, 1016 insertions(+), 502 deletions(-) create mode 100644 src/main/java/io/supertokens/webserver/api/bulkimport/ImportUserAPI.java create mode 100644 src/test/java/io/supertokens/test/bulkimport/apis/ImportUserTest.java diff --git a/src/main/java/io/supertokens/bulkimport/BulkImport.java b/src/main/java/io/supertokens/bulkimport/BulkImport.java index 918b1e903..ef6d5d18d 100644 --- a/src/main/java/io/supertokens/bulkimport/BulkImport.java +++ b/src/main/java/io/supertokens/bulkimport/BulkImport.java @@ -18,19 +18,69 @@ import io.supertokens.pluginInterface.bulkimport.BulkImportStorage.BULK_IMPORT_USER_STATUS; import io.supertokens.pluginInterface.bulkimport.sqlStorage.BulkImportSQLStorage; +import io.supertokens.pluginInterface.emailpassword.exceptions.DuplicateEmailException; +import io.supertokens.pluginInterface.emailpassword.exceptions.UnknownUserIdException; +import io.supertokens.pluginInterface.emailverification.sqlStorage.EmailVerificationSQLStorage; +import io.supertokens.Main; +import io.supertokens.ResourceDistributor; +import io.supertokens.authRecipe.AuthRecipe; +import io.supertokens.authRecipe.exception.AccountInfoAlreadyAssociatedWithAnotherPrimaryUserIdException; +import io.supertokens.authRecipe.exception.InputUserIdIsNotAPrimaryUserException; +import io.supertokens.authRecipe.exception.RecipeUserIdAlreadyLinkedWithAnotherPrimaryUserIdException; +import io.supertokens.authRecipe.exception.RecipeUserIdAlreadyLinkedWithPrimaryUserIdException; +import io.supertokens.config.Config; +import io.supertokens.emailpassword.EmailPassword; +import io.supertokens.emailpassword.EmailPassword.ImportUserResponse; +import io.supertokens.featureflag.exceptions.FeatureNotEnabledException; +import io.supertokens.multitenancy.Multitenancy; +import io.supertokens.multitenancy.exception.AnotherPrimaryUserWithEmailAlreadyExistsException; +import io.supertokens.multitenancy.exception.AnotherPrimaryUserWithPhoneNumberAlreadyExistsException; +import io.supertokens.multitenancy.exception.AnotherPrimaryUserWithThirdPartyInfoAlreadyExistsException; +import io.supertokens.passwordless.Passwordless; +import io.supertokens.passwordless.exceptions.RestartFlowException; import io.supertokens.pluginInterface.Storage; import io.supertokens.pluginInterface.StorageUtils; +import io.supertokens.pluginInterface.authRecipe.AuthRecipeUserInfo; import io.supertokens.pluginInterface.bulkimport.BulkImportUser; +import io.supertokens.pluginInterface.bulkimport.BulkImportUser.LoginMethod; +import io.supertokens.pluginInterface.bulkimport.BulkImportUser.TotpDevice; +import io.supertokens.pluginInterface.bulkimport.BulkImportUser.UserRole; +import io.supertokens.pluginInterface.exceptions.DbInitException; +import io.supertokens.pluginInterface.exceptions.InvalidConfigException; import io.supertokens.pluginInterface.exceptions.StorageQueryException; +import io.supertokens.pluginInterface.exceptions.StorageTransactionLogicException; import io.supertokens.pluginInterface.multitenancy.AppIdentifier; +import io.supertokens.pluginInterface.multitenancy.TenantConfig; +import io.supertokens.pluginInterface.multitenancy.TenantIdentifier; import io.supertokens.pluginInterface.multitenancy.exceptions.TenantOrAppNotFoundException; +import io.supertokens.pluginInterface.passwordless.exception.DuplicatePhoneNumberException; +import io.supertokens.pluginInterface.sqlStorage.SQLStorage; +import io.supertokens.pluginInterface.sqlStorage.TransactionConnection; +import io.supertokens.pluginInterface.thirdparty.exception.DuplicateThirdPartyUserException; +import io.supertokens.pluginInterface.totp.exception.DeviceAlreadyExistsException; +import io.supertokens.pluginInterface.useridmapping.exception.UnknownSuperTokensUserIdException; +import io.supertokens.pluginInterface.useridmapping.exception.UserIdMappingAlreadyExistsException; +import io.supertokens.pluginInterface.userroles.exception.UnknownRoleException; +import io.supertokens.storageLayer.StorageLayer; +import io.supertokens.thirdparty.ThirdParty; +import io.supertokens.thirdparty.ThirdParty.SignInUpResponse; +import io.supertokens.totp.Totp; +import io.supertokens.useridmapping.UserIdMapping; +import io.supertokens.usermetadata.UserMetadata; +import io.supertokens.userroles.UserRoles; import io.supertokens.utils.Utils; +import jakarta.servlet.ServletException; - +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; import javax.annotation.Nullable; +import com.google.gson.JsonObject; + public class BulkImport { // Maximum number of users that can be added in a single /bulk-import/users POST request @@ -46,6 +96,9 @@ public class BulkImport { // Time interval in seconds between two consecutive runs of ProcessBulkImportUsers Cron Job public static final int PROCESS_USERS_INTERVAL_SECONDS = 60; + // This map allows reusing proxy storage for all tenants in the app and closing connections after import. + private static Map userPoolToStorageMap = new HashMap<>(); + public static void addUsers(AppIdentifier appIdentifier, Storage storage, List users) throws StorageQueryException, TenantOrAppNotFoundException { while (true) { @@ -74,7 +127,8 @@ public static BulkImportUserPaginationContainer getUsers(AppIdentifier appIdenti } else { BulkImportUserPaginationToken tokenInfo = BulkImportUserPaginationToken.extractTokenInfo(paginationToken); users = bulkImportStorage - .getBulkImportUsers(appIdentifier, limit + 1, status, tokenInfo.bulkImportUserId, tokenInfo.createdAt); + .getBulkImportUsers(appIdentifier, limit + 1, status, tokenInfo.bulkImportUserId, + tokenInfo.createdAt); } String nextPaginationToken = null; @@ -89,7 +143,364 @@ public static BulkImportUserPaginationContainer getUsers(AppIdentifier appIdenti return new BulkImportUserPaginationContainer(resultUsers, nextPaginationToken); } - public static List deleteUsers(AppIdentifier appIdentifier, Storage storage, String[] userIds) throws StorageQueryException { + public static List deleteUsers(AppIdentifier appIdentifier, Storage storage, String[] userIds) + throws StorageQueryException { return StorageUtils.getBulkImportStorage(storage).deleteBulkImportUsers(appIdentifier, userIds); } + + public static synchronized AuthRecipeUserInfo importUser(Main main, AppIdentifier appIdentifier, + BulkImportUser user) + throws StorageQueryException, InvalidConfigException, IOException, TenantOrAppNotFoundException, + DbInitException { + // Since all the tenants of a user must share the storage, we will just use the + // storage of the first tenantId of the first loginMethod + TenantIdentifier firstTenantIdentifier = new TenantIdentifier(appIdentifier.getConnectionUriDomain(), + appIdentifier.getAppId(), user.loginMethods.get(0).tenantIds.get(0)); + + SQLStorage bulkImportProxyStorage = (SQLStorage) getBulkImportProxyStorage(main, firstTenantIdentifier); + + LoginMethod primaryLM = getPrimaryLoginMethod(user); + + try { + return bulkImportProxyStorage.startTransaction(con -> { + try { + for (LoginMethod lm : user.loginMethods) { + processUserLoginMethod(main, appIdentifier, bulkImportProxyStorage, lm); + } + + createPrimaryUserAndLinkAccounts(main, appIdentifier, bulkImportProxyStorage, user, primaryLM); + + Storage[] allStoragesForApp = getAllProxyStoragesForApp(main, appIdentifier); + createUserIdMapping(appIdentifier, user, primaryLM, allStoragesForApp); + + verifyEmailForAllLoginMethods(appIdentifier, con, bulkImportProxyStorage, user.loginMethods); + createTotpDevices(main, appIdentifier, bulkImportProxyStorage, user, primaryLM); + createUserMetadata(appIdentifier, bulkImportProxyStorage, user, primaryLM); + createUserRoles(main, appIdentifier, bulkImportProxyStorage, user); + + bulkImportProxyStorage.commitTransactionForBulkImportProxyStorage(); + + AuthRecipeUserInfo importedUser = AuthRecipe.getUserById(appIdentifier, bulkImportProxyStorage, primaryLM.superTokensUserId); + io.supertokens.useridmapping.UserIdMapping.populateExternalUserIdForUsers(appIdentifier, bulkImportProxyStorage, new AuthRecipeUserInfo[]{importedUser}); + + return importedUser; + } catch (StorageTransactionLogicException e) { + // We need to rollback the transaction manually because we have overridden that in the proxy storage + bulkImportProxyStorage.rollbackTransactionForBulkImportProxyStorage(); + throw e; + } finally { + closeAllProxyStorages(); + } + }); + } catch (StorageTransactionLogicException e) { + throw new StorageQueryException(e.actualException); + } + } + + public static void processUserLoginMethod(Main main, AppIdentifier appIdentifier, Storage storage, + LoginMethod lm) throws StorageTransactionLogicException { + String firstTenant = lm.tenantIds.get(0); + + TenantIdentifier tenantIdentifier = new TenantIdentifier(appIdentifier.getConnectionUriDomain(), + appIdentifier.getAppId(), firstTenant); + + if (lm.recipeId.equals("emailpassword")) { + processEmailPasswordLoginMethod(tenantIdentifier, storage, lm); + } else if (lm.recipeId.equals("thirdparty")) { + processThirdPartyLoginMethod(tenantIdentifier, storage, lm); + } else if (lm.recipeId.equals("passwordless")) { + processPasswordlessLoginMethod(tenantIdentifier, storage, lm); + } else { + throw new StorageTransactionLogicException( + new IllegalArgumentException("Unknown recipeId " + lm.recipeId + " for loginMethod ")); + } + + associateUserToTenants(main, appIdentifier, storage, lm, firstTenant); + } + + private static void processEmailPasswordLoginMethod(TenantIdentifier tenantIdentifier, Storage storage, + LoginMethod lm) throws StorageTransactionLogicException { + try { + ImportUserResponse userInfo = EmailPassword.createUserWithPasswordHash(tenantIdentifier, storage, lm.email, + lm.passwordHash, lm.timeJoinedInMSSinceEpoch); + + lm.superTokensUserId = userInfo.user.getSupertokensUserId(); + } catch (StorageQueryException | TenantOrAppNotFoundException e) { + throw new StorageTransactionLogicException(e); + } catch (DuplicateEmailException e) { + throw new StorageTransactionLogicException( + new Exception("A user with email " + lm.email + " already exists")); + } + } + + private static void processThirdPartyLoginMethod(TenantIdentifier tenantIdentifier, Storage storage, LoginMethod lm) + throws StorageTransactionLogicException { + try { + SignInUpResponse userInfo = ThirdParty.createThirdPartyUser( + tenantIdentifier, storage, lm.thirdPartyId, lm.thirdPartyUserId, lm.email, + lm.timeJoinedInMSSinceEpoch); + + lm.superTokensUserId = userInfo.user.getSupertokensUserId(); + } catch (StorageQueryException | TenantOrAppNotFoundException e) { + throw new StorageTransactionLogicException(e); + } catch (DuplicateThirdPartyUserException e) { + throw new StorageTransactionLogicException(new Exception("A user with thirdPartyId " + lm.thirdPartyId + + " and thirdPartyUserId " + lm.thirdPartyUserId + " already exists")); + } + } + + private static void processPasswordlessLoginMethod(TenantIdentifier tenantIdentifier, Storage storage, + LoginMethod lm) + throws StorageTransactionLogicException { + try { + AuthRecipeUserInfo userInfo = Passwordless.createPasswordlessUser(tenantIdentifier, storage, lm.email, + lm.phoneNumber, lm.timeJoinedInMSSinceEpoch); + + lm.superTokensUserId = userInfo.getSupertokensUserId(); + } catch (StorageQueryException | TenantOrAppNotFoundException | RestartFlowException e) { + throw new StorageTransactionLogicException(e); + } + } + + private static void associateUserToTenants(Main main, AppIdentifier appIdentifier, Storage storage, LoginMethod lm, + String firstTenant) throws StorageTransactionLogicException { + for (String tenantId : lm.tenantIds) { + try { + if (tenantId.equals(firstTenant)) { + continue; + } + + TenantIdentifier tenantIdentifier = new TenantIdentifier(appIdentifier.getConnectionUriDomain(), + appIdentifier.getAppId(), tenantId); + Multitenancy.addUserIdToTenant(main, tenantIdentifier, storage, lm.getSuperTokenOrExternalUserId()); + } catch (TenantOrAppNotFoundException | UnknownUserIdException | StorageQueryException + | FeatureNotEnabledException | DuplicateEmailException | DuplicatePhoneNumberException + | DuplicateThirdPartyUserException | AnotherPrimaryUserWithPhoneNumberAlreadyExistsException + | AnotherPrimaryUserWithEmailAlreadyExistsException + | AnotherPrimaryUserWithThirdPartyInfoAlreadyExistsException e) { + throw new StorageTransactionLogicException(e); + } + } + } + + public static void createPrimaryUserAndLinkAccounts(Main main, + AppIdentifier appIdentifier, Storage storage, BulkImportUser user, LoginMethod primaryLM) + throws StorageTransactionLogicException { + if (user.loginMethods.size() == 1) { + return; + } + + try { + AuthRecipe.createPrimaryUser(main, appIdentifier, storage, primaryLM.getSuperTokenOrExternalUserId()); + } catch (TenantOrAppNotFoundException | FeatureNotEnabledException | StorageQueryException e) { + throw new StorageTransactionLogicException(e); + } catch (UnknownUserIdException e) { + throw new StorageTransactionLogicException(new Exception( + "We tried to create the primary user for the userId " + primaryLM.getSuperTokenOrExternalUserId() + + " but it doesn't exist. This should not happen. Please contact support.")); + } catch (RecipeUserIdAlreadyLinkedWithPrimaryUserIdException + | AccountInfoAlreadyAssociatedWithAnotherPrimaryUserIdException e) { + throw new StorageTransactionLogicException( + new Exception(e.getMessage() + " This should not happen. Please contact support.")); + } + + for (LoginMethod lm : user.loginMethods) { + try { + if (lm.getSuperTokenOrExternalUserId().equals(primaryLM.getSuperTokenOrExternalUserId())) { + continue; + } + + AuthRecipe.linkAccounts(main, appIdentifier, storage, lm.getSuperTokenOrExternalUserId(), + primaryLM.getSuperTokenOrExternalUserId()); + + } catch (TenantOrAppNotFoundException | FeatureNotEnabledException | StorageQueryException e) { + throw new StorageTransactionLogicException(e); + } catch (UnknownUserIdException e) { + throw new StorageTransactionLogicException( + new Exception("We tried to link the userId " + lm.getSuperTokenOrExternalUserId() + + " to the primary userId " + primaryLM.getSuperTokenOrExternalUserId() + + " but it doesn't exist. This should not happen. Please contact support.")); + } catch (InputUserIdIsNotAPrimaryUserException e) { + throw new StorageTransactionLogicException( + new Exception("We tried to link the userId " + lm.getSuperTokenOrExternalUserId() + + " to the primary userId " + primaryLM.getSuperTokenOrExternalUserId() + + " but it is not a primary user. This should not happen. Please contact support.")); + } catch (AccountInfoAlreadyAssociatedWithAnotherPrimaryUserIdException + | RecipeUserIdAlreadyLinkedWithAnotherPrimaryUserIdException e) { + throw new StorageTransactionLogicException( + new Exception(e.getMessage() + " This should not happen. Please contact support.")); + } + } + } + + public static void createUserIdMapping(AppIdentifier appIdentifier, + BulkImportUser user, LoginMethod primaryLM, Storage[] storages) throws StorageTransactionLogicException { + if (user.externalUserId != null) { + try { + UserIdMapping.createUserIdMapping( + appIdentifier, storages, + primaryLM.superTokensUserId, user.externalUserId, + null, false, true); + + primaryLM.externalUserId = user.externalUserId; + } catch (StorageQueryException | ServletException | TenantOrAppNotFoundException e) { + throw new StorageTransactionLogicException(e); + } catch (UserIdMappingAlreadyExistsException e) { + throw new StorageTransactionLogicException( + new Exception("A user with externalId " + user.externalUserId + " already exists")); + } catch (UnknownSuperTokensUserIdException e) { + throw new StorageTransactionLogicException( + new Exception("We tried to create the externalUserId mapping for the superTokenUserId " + + primaryLM.superTokensUserId + + " but it doesn't exist. This should not happen. Please contact support.")); + } + } + } + + public static void createUserMetadata(AppIdentifier appIdentifier, Storage storage, BulkImportUser user, + LoginMethod primaryLM) throws StorageTransactionLogicException { + if (user.userMetadata != null) { + try { + UserMetadata.updateUserMetadata(appIdentifier, storage, primaryLM.getSuperTokenOrExternalUserId(), + user.userMetadata); + } catch (StorageQueryException | TenantOrAppNotFoundException e) { + throw new StorageTransactionLogicException(e); + } + } + } + + public static void createUserRoles(Main main, AppIdentifier appIdentifier, Storage storage, + BulkImportUser user) throws StorageTransactionLogicException { + if (user.userRoles != null) { + for (UserRole userRole : user.userRoles) { + try { + for (String tenantId : userRole.tenantIds) { + TenantIdentifier tenantIdentifier = new TenantIdentifier( + appIdentifier.getConnectionUriDomain(), appIdentifier.getAppId(), + tenantId); + + UserRoles.addRoleToUser(main, tenantIdentifier, storage, user.externalUserId, userRole.role); + } + } catch (TenantOrAppNotFoundException | StorageQueryException e) { + throw new StorageTransactionLogicException(e); + } catch (UnknownRoleException e) { + throw new StorageTransactionLogicException(new Exception("Role " + userRole.role + + " does not exist! You need pre-create the role before assigning it to the user.")); + } + } + } + } + + public static void verifyEmailForAllLoginMethods(AppIdentifier appIdentifier, TransactionConnection con, + Storage storage, + List loginMethods) throws StorageTransactionLogicException { + + for (LoginMethod lm : loginMethods) { + try { + + TenantIdentifier tenantIdentifier = new TenantIdentifier(appIdentifier.getConnectionUriDomain(), + appIdentifier.getAppId(), lm.tenantIds.get(0)); + + EmailVerificationSQLStorage emailVerificationSQLStorage = StorageUtils + .getEmailVerificationStorage(storage); + emailVerificationSQLStorage + .updateIsEmailVerified_Transaction(tenantIdentifier.toAppIdentifier(), con, + lm.getSuperTokenOrExternalUserId(), lm.email, true); + } catch (TenantOrAppNotFoundException | StorageQueryException e) { + throw new StorageTransactionLogicException(e); + } + } + } + + public static void createTotpDevices(Main main, AppIdentifier appIdentifier, Storage storage, + BulkImportUser user, LoginMethod primaryLM) throws StorageTransactionLogicException { + if (user.totpDevices != null) { + for (TotpDevice totpDevice : user.totpDevices) { + try { + Totp.createDevice(main, appIdentifier, storage, primaryLM.getSuperTokenOrExternalUserId(), + totpDevice.deviceName, totpDevice.skew, totpDevice.period, totpDevice.secretKey, + true, System.currentTimeMillis()); + } catch (TenantOrAppNotFoundException | StorageQueryException | FeatureNotEnabledException e) { + throw new StorageTransactionLogicException(e); + } catch (DeviceAlreadyExistsException e) { + throw new StorageTransactionLogicException( + new Exception("A totp device with name " + totpDevice.deviceName + " already exists")); + } + } + } + } + + // Returns the primary loginMethod of the user. If no loginMethod is marked as + // primary, then the oldest loginMethod is returned. + public static BulkImportUser.LoginMethod getPrimaryLoginMethod(BulkImportUser user) { + BulkImportUser.LoginMethod oldestLM = user.loginMethods.get(0); + for (BulkImportUser.LoginMethod lm : user.loginMethods) { + if (lm.isPrimary) { + return lm; + } + + if (lm.timeJoinedInMSSinceEpoch < oldestLM.timeJoinedInMSSinceEpoch) { + oldestLM = lm; + } + } + return oldestLM; + } + + private static synchronized Storage getBulkImportProxyStorage(Main main, TenantIdentifier tenantIdentifier) + throws InvalidConfigException, IOException, TenantOrAppNotFoundException, DbInitException, + StorageQueryException { + String userPoolId = StorageLayer.getStorage(tenantIdentifier, main).getUserPoolId(); + if (userPoolToStorageMap.containsKey(userPoolId)) { + return userPoolToStorageMap.get(userPoolId); + } + + TenantConfig[] allTenants = Multitenancy.getAllTenants(main); + + Map normalisedConfigs = Config.getNormalisedConfigsForAllTenants( + allTenants, + Config.getBaseConfigAsJsonObject(main)); + + for (ResourceDistributor.KeyClass key : normalisedConfigs.keySet()) { + if (key.getTenantIdentifier().equals(tenantIdentifier)) { + SQLStorage bulkImportProxyStorage = (SQLStorage) StorageLayer.getNewBulkImportProxyStorageInstance(main, + normalisedConfigs.get(key), tenantIdentifier, true); + + userPoolToStorageMap.put(userPoolId, bulkImportProxyStorage); + bulkImportProxyStorage.initStorage(false, new ArrayList<>()); + // `BulkImportProxyStorage` uses `BulkImportProxyConnection`, which overrides the `.commit()` method on the Connection object. + // The `initStorage()` method runs `select * from table_name limit 1` queries to check if the tables exist but these queries + // don't get committed due to the overridden `.commit()`, so we need to manually commit the transaction to remove any locks on the tables. + + // Without this commit, a call to `select * from bulk_import_users limit 1` in `doesTableExist()` locks the `bulk_import_users` table, + // causing other queries to stall indefinitely. + bulkImportProxyStorage.commitTransactionForBulkImportProxyStorage(); + return bulkImportProxyStorage; + } + } + throw new TenantOrAppNotFoundException(tenantIdentifier); + } + + private static Storage[] getAllProxyStoragesForApp(Main main, AppIdentifier appIdentifier) + throws StorageTransactionLogicException { + + try { + List allProxyStorages = new ArrayList<>(); + + TenantConfig[] tenantConfigs = Multitenancy.getAllTenantsForApp(appIdentifier, main); + for (TenantConfig tenantConfig : tenantConfigs) { + allProxyStorages.add(getBulkImportProxyStorage(main, tenantConfig.tenantIdentifier)); + } + return allProxyStorages.toArray(new Storage[0]); + } catch (TenantOrAppNotFoundException | InvalidConfigException | IOException | DbInitException | StorageQueryException e) { + throw new StorageTransactionLogicException(e); + } + } + + private static void closeAllProxyStorages() throws StorageQueryException { + for (SQLStorage storage : userPoolToStorageMap.values()) { + storage.closeConnectionForBulkImportProxyStorage(); + storage.close(); + } + userPoolToStorageMap.clear(); + } } diff --git a/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkImportUsers.java b/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkImportUsers.java index 847bbaf01..7e1ef2fdf 100644 --- a/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkImportUsers.java +++ b/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkImportUsers.java @@ -19,35 +19,20 @@ import java.io.IOException; import java.util.ArrayList; import java.util.HashMap; -import java.util.HashSet; import java.util.List; import java.util.Map; -import java.util.Set; import com.google.gson.JsonObject; import io.supertokens.Main; import io.supertokens.ResourceDistributor; -import io.supertokens.authRecipe.AuthRecipe; -import io.supertokens.authRecipe.exception.AccountInfoAlreadyAssociatedWithAnotherPrimaryUserIdException; -import io.supertokens.authRecipe.exception.InputUserIdIsNotAPrimaryUserException; -import io.supertokens.authRecipe.exception.RecipeUserIdAlreadyLinkedWithAnotherPrimaryUserIdException; -import io.supertokens.authRecipe.exception.RecipeUserIdAlreadyLinkedWithPrimaryUserIdException; import io.supertokens.bulkimport.BulkImport; import io.supertokens.bulkimport.BulkImportUserUtils; import io.supertokens.bulkimport.exceptions.InvalidBulkImportDataException; import io.supertokens.config.Config; import io.supertokens.cronjobs.CronTask; import io.supertokens.cronjobs.CronTaskTest; -import io.supertokens.emailpassword.EmailPassword; -import io.supertokens.emailpassword.EmailPassword.ImportUserResponse; -import io.supertokens.featureflag.exceptions.FeatureNotEnabledException; import io.supertokens.multitenancy.Multitenancy; -import io.supertokens.multitenancy.exception.AnotherPrimaryUserWithEmailAlreadyExistsException; -import io.supertokens.multitenancy.exception.AnotherPrimaryUserWithPhoneNumberAlreadyExistsException; -import io.supertokens.multitenancy.exception.AnotherPrimaryUserWithThirdPartyInfoAlreadyExistsException; -import io.supertokens.passwordless.Passwordless; -import io.supertokens.passwordless.exceptions.RestartFlowException; import io.supertokens.pluginInterface.STORAGE_TYPE; import io.supertokens.pluginInterface.Storage; import io.supertokens.pluginInterface.StorageUtils; @@ -56,12 +41,7 @@ import io.supertokens.pluginInterface.bulkimport.BulkImportUser; import io.supertokens.pluginInterface.bulkimport.BulkImportStorage.BULK_IMPORT_USER_STATUS; import io.supertokens.pluginInterface.bulkimport.BulkImportUser.LoginMethod; -import io.supertokens.pluginInterface.bulkimport.BulkImportUser.TotpDevice; -import io.supertokens.pluginInterface.bulkimport.BulkImportUser.UserRole; import io.supertokens.pluginInterface.bulkimport.sqlStorage.BulkImportSQLStorage; -import io.supertokens.pluginInterface.emailpassword.exceptions.DuplicateEmailException; -import io.supertokens.pluginInterface.emailpassword.exceptions.UnknownUserIdException; -import io.supertokens.pluginInterface.emailverification.sqlStorage.EmailVerificationSQLStorage; import io.supertokens.pluginInterface.exceptions.DbInitException; import io.supertokens.pluginInterface.exceptions.InvalidConfigException; import io.supertokens.pluginInterface.exceptions.StorageQueryException; @@ -70,22 +50,8 @@ import io.supertokens.pluginInterface.multitenancy.TenantConfig; import io.supertokens.pluginInterface.multitenancy.TenantIdentifier; import io.supertokens.pluginInterface.multitenancy.exceptions.TenantOrAppNotFoundException; -import io.supertokens.pluginInterface.passwordless.exception.DuplicatePhoneNumberException; import io.supertokens.pluginInterface.sqlStorage.SQLStorage; -import io.supertokens.pluginInterface.sqlStorage.TransactionConnection; -import io.supertokens.pluginInterface.thirdparty.exception.DuplicateThirdPartyUserException; -import io.supertokens.pluginInterface.totp.exception.DeviceAlreadyExistsException; -import io.supertokens.pluginInterface.useridmapping.exception.UnknownSuperTokensUserIdException; -import io.supertokens.pluginInterface.useridmapping.exception.UserIdMappingAlreadyExistsException; -import io.supertokens.pluginInterface.userroles.exception.UnknownRoleException; import io.supertokens.storageLayer.StorageLayer; -import io.supertokens.thirdparty.ThirdParty; -import io.supertokens.thirdparty.ThirdParty.SignInUpResponse; -import io.supertokens.totp.Totp; -import io.supertokens.useridmapping.UserIdMapping; -import io.supertokens.usermetadata.UserMetadata; -import io.supertokens.userroles.UserRoles; -import jakarta.servlet.ServletException; public class ProcessBulkImportUsers extends CronTask { @@ -147,7 +113,7 @@ public int getInitialWaitTimeSeconds() { return 0; } - private synchronized Storage getProxyStorage(TenantIdentifier tenantIdentifier) + private synchronized Storage getBulkImportProxyStorage(TenantIdentifier tenantIdentifier) throws InvalidConfigException, IOException, TenantOrAppNotFoundException, DbInitException, StorageQueryException { String userPoolId = StorageLayer.getStorage(tenantIdentifier, main).getUserPoolId(); if (userPoolToStorageMap.containsKey(userPoolId)) { @@ -180,15 +146,20 @@ private synchronized Storage getProxyStorage(TenantIdentifier tenantIdentifier) throw new TenantOrAppNotFoundException(tenantIdentifier); } - public Storage[] getAllProxyStoragesForApp(Main main, AppIdentifier appIdentifier) - throws TenantOrAppNotFoundException, InvalidConfigException, IOException, DbInitException, StorageQueryException { - List allProxyStorages = new ArrayList<>(); + private Storage[] getAllProxyStoragesForApp(Main main, AppIdentifier appIdentifier) + throws StorageTransactionLogicException { - TenantConfig[] tenantConfigs = Multitenancy.getAllTenantsForApp(appIdentifier, main); - for (TenantConfig tenantConfig : tenantConfigs) { - allProxyStorages.add(getProxyStorage(tenantConfig.tenantIdentifier)); + try { + List allProxyStorages = new ArrayList<>(); + + TenantConfig[] tenantConfigs = Multitenancy.getAllTenantsForApp(appIdentifier, main); + for (TenantConfig tenantConfig : tenantConfigs) { + allProxyStorages.add(getBulkImportProxyStorage(tenantConfig.tenantIdentifier)); + } + return allProxyStorages.toArray(new Storage[0]); + } catch (TenantOrAppNotFoundException | InvalidConfigException | IOException | DbInitException | StorageQueryException e) { + throw new StorageTransactionLogicException(e); } - return allProxyStorages.toArray(new Storage[0]); } private void closeAllProxyStorages() throws StorageQueryException { @@ -218,11 +189,11 @@ private void processUser(AppIdentifier appIdentifier, BulkImportUser user, BulkI TenantIdentifier firstTenantIdentifier = new TenantIdentifier(appIdentifier.getConnectionUriDomain(), appIdentifier.getAppId(), user.loginMethods.get(0).tenantIds.get(0)); - SQLStorage bulkImportProxyStorage = (SQLStorage) getProxyStorage(firstTenantIdentifier); + SQLStorage bulkImportProxyStorage = (SQLStorage) getBulkImportProxyStorage(firstTenantIdentifier); - LoginMethod primaryLM = getPrimaryLoginMethod(user); + LoginMethod primaryLM = BulkImport.getPrimaryLoginMethod(user); - AuthRecipeSQLStorage authRecipeSQLStorage = (AuthRecipeSQLStorage) getProxyStorage(firstTenantIdentifier); + AuthRecipeSQLStorage authRecipeSQLStorage = (AuthRecipeSQLStorage) getBulkImportProxyStorage(firstTenantIdentifier); /* * We use two separate storage instances: one for importing the user and another for managing bulk_import_users entries. @@ -263,15 +234,18 @@ private void processUser(AppIdentifier appIdentifier, BulkImportUser user, BulkI bulkImportProxyStorage.startTransaction(con -> { try { for (LoginMethod lm : user.loginMethods) { - processUserLoginMethod(appIdentifier, bulkImportProxyStorage, lm); + BulkImport.processUserLoginMethod(main, appIdentifier, bulkImportProxyStorage, lm); } - createPrimaryUserAndLinkAccounts(main, appIdentifier, bulkImportProxyStorage, user, primaryLM); - createUserIdMapping(main, appIdentifier, user, primaryLM); - verifyEmailForAllLoginMethods(appIdentifier, con, bulkImportProxyStorage, user.loginMethods); - createTotpDevices(main, appIdentifier, bulkImportProxyStorage, user, primaryLM); - createUserMetadata(appIdentifier, bulkImportProxyStorage, user, primaryLM); - createUserRoles(main, appIdentifier, bulkImportProxyStorage, user); + BulkImport.createPrimaryUserAndLinkAccounts(main, appIdentifier, bulkImportProxyStorage, user, primaryLM); + + Storage[] allStoragesForApp = getAllProxyStoragesForApp(main, appIdentifier); + BulkImport.createUserIdMapping(appIdentifier, user, primaryLM, allStoragesForApp); + + BulkImport.verifyEmailForAllLoginMethods(appIdentifier, con, bulkImportProxyStorage, user.loginMethods); + BulkImport.createTotpDevices(main, appIdentifier, bulkImportProxyStorage, user, primaryLM); + BulkImport.createUserMetadata(appIdentifier, bulkImportProxyStorage, user, primaryLM); + BulkImport.createUserRoles(main, appIdentifier, bulkImportProxyStorage, user); // We are updating the primaryUserId in the bulkImportUser entry. This will help us handle the inconsistent transaction commit. // If this update statement fails then the outer transaction will fail as well and the user will simpl be processed again. No inconsistency will happen in this @@ -328,254 +302,6 @@ private void handleProcessUserExceptions(AppIdentifier appIdentifier, BulkImport } } - private void processUserLoginMethod(AppIdentifier appIdentifier, Storage storage, - LoginMethod lm) throws StorageTransactionLogicException { - String firstTenant = lm.tenantIds.get(0); - - TenantIdentifier tenantIdentifier = new TenantIdentifier(appIdentifier.getConnectionUriDomain(), - appIdentifier.getAppId(), firstTenant); - - if (lm.recipeId.equals("emailpassword")) { - processEmailPasswordLoginMethod(tenantIdentifier, storage, lm); - } else if (lm.recipeId.equals("thirdparty")) { - processThirdPartyLoginMethod(tenantIdentifier, storage, lm); - } else if (lm.recipeId.equals("passwordless")) { - processPasswordlessLoginMethod(tenantIdentifier, storage, lm); - } else { - throw new StorageTransactionLogicException( - new IllegalArgumentException("Unknown recipeId " + lm.recipeId + " for loginMethod ")); - } - - associateUserToTenants(main, appIdentifier, storage, lm, firstTenant); - } - - private void processEmailPasswordLoginMethod(TenantIdentifier tenantIdentifier, Storage storage, - LoginMethod lm) throws StorageTransactionLogicException { - try { - ImportUserResponse userInfo = EmailPassword.createUserWithPasswordHash(tenantIdentifier, storage, lm.email, - lm.passwordHash, lm.timeJoinedInMSSinceEpoch); - - lm.superTokensUserId = userInfo.user.getSupertokensUserId(); - } catch (StorageQueryException | TenantOrAppNotFoundException e) { - throw new StorageTransactionLogicException(e); - } catch (DuplicateEmailException e) { - throw new StorageTransactionLogicException( - new Exception("A user with email " + lm.email + " already exists")); - } - } - - private void processThirdPartyLoginMethod(TenantIdentifier tenantIdentifier, Storage storage, LoginMethod lm) - throws StorageTransactionLogicException { - try { - SignInUpResponse userInfo = ThirdParty.createThirdPartyUser( - tenantIdentifier, storage, lm.thirdPartyId, lm.thirdPartyUserId, lm.email, - lm.timeJoinedInMSSinceEpoch); - - lm.superTokensUserId = userInfo.user.getSupertokensUserId(); - } catch (StorageQueryException | TenantOrAppNotFoundException e) { - throw new StorageTransactionLogicException(e); - } catch (DuplicateThirdPartyUserException e) { - throw new StorageTransactionLogicException(new Exception("A user with thirdPartyId " + lm.thirdPartyId - + " and thirdPartyUserId " + lm.thirdPartyUserId + " already exists")); - } - } - - private void processPasswordlessLoginMethod(TenantIdentifier tenantIdentifier, Storage storage, LoginMethod lm) - throws StorageTransactionLogicException { - try { - AuthRecipeUserInfo userInfo = Passwordless.createPasswordlessUser(tenantIdentifier, storage, lm.email, - lm.phoneNumber, lm.timeJoinedInMSSinceEpoch); - - lm.superTokensUserId = userInfo.getSupertokensUserId(); - } catch (StorageQueryException | TenantOrAppNotFoundException | RestartFlowException e) { - throw new StorageTransactionLogicException(e); - } - } - - private void associateUserToTenants(Main main, AppIdentifier appIdentifier, Storage storage, LoginMethod lm, - String firstTenant) throws StorageTransactionLogicException { - for (String tenantId : lm.tenantIds) { - try { - if (tenantId.equals(firstTenant)) { - continue; - } - - TenantIdentifier tenantIdentifier = new TenantIdentifier(appIdentifier.getConnectionUriDomain(), - appIdentifier.getAppId(), tenantId); - Multitenancy.addUserIdToTenant(main, tenantIdentifier, storage, lm.getSuperTokenOrExternalUserId()); - } catch (TenantOrAppNotFoundException | UnknownUserIdException | StorageQueryException - | FeatureNotEnabledException | DuplicateEmailException | DuplicatePhoneNumberException - | DuplicateThirdPartyUserException | AnotherPrimaryUserWithPhoneNumberAlreadyExistsException - | AnotherPrimaryUserWithEmailAlreadyExistsException - | AnotherPrimaryUserWithThirdPartyInfoAlreadyExistsException e) { - throw new StorageTransactionLogicException(e); - } - } - } - - private void createPrimaryUserAndLinkAccounts(Main main, - AppIdentifier appIdentifier, Storage storage, BulkImportUser user, LoginMethod primaryLM) - throws StorageTransactionLogicException { - if (user.loginMethods.size() == 1) { - return; - } - - try { - AuthRecipe.createPrimaryUser(main, appIdentifier, storage, primaryLM.getSuperTokenOrExternalUserId()); - } catch (TenantOrAppNotFoundException | FeatureNotEnabledException | StorageQueryException e) { - throw new StorageTransactionLogicException(e); - } catch (UnknownUserIdException e) { - throw new StorageTransactionLogicException(new Exception( - "We tried to create the primary user for the userId " + primaryLM.getSuperTokenOrExternalUserId() - + " but it doesn't exist. This should not happen. Please contact support.")); - } catch (RecipeUserIdAlreadyLinkedWithPrimaryUserIdException - | AccountInfoAlreadyAssociatedWithAnotherPrimaryUserIdException e) { - throw new StorageTransactionLogicException( - new Exception(e.getMessage() + " This should not happen. Please contact support.")); - } - - for (LoginMethod lm : user.loginMethods) { - try { - if (lm.getSuperTokenOrExternalUserId().equals(primaryLM.getSuperTokenOrExternalUserId())) { - continue; - } - - AuthRecipe.linkAccounts(main, appIdentifier, storage, lm.getSuperTokenOrExternalUserId(), - primaryLM.getSuperTokenOrExternalUserId()); - - } catch (TenantOrAppNotFoundException | FeatureNotEnabledException | StorageQueryException e) { - throw new StorageTransactionLogicException(e); - } catch (UnknownUserIdException e) { - throw new StorageTransactionLogicException( - new Exception("We tried to link the userId " + lm.getSuperTokenOrExternalUserId() - + " to the primary userId " + primaryLM.getSuperTokenOrExternalUserId() - + " but it doesn't exist. This should not happen. Please contact support.")); - } catch (InputUserIdIsNotAPrimaryUserException e) { - throw new StorageTransactionLogicException( - new Exception("We tried to link the userId " + lm.getSuperTokenOrExternalUserId() - + " to the primary userId " + primaryLM.getSuperTokenOrExternalUserId() - + " but it is not a primary user. This should not happen. Please contact support.")); - } catch (AccountInfoAlreadyAssociatedWithAnotherPrimaryUserIdException - | RecipeUserIdAlreadyLinkedWithAnotherPrimaryUserIdException e) { - throw new StorageTransactionLogicException( - new Exception(e.getMessage() + " This should not happen. Please contact support.")); - } - } - } - - private void createUserIdMapping(Main main, AppIdentifier appIdentifier, - BulkImportUser user, LoginMethod primaryLM) throws StorageTransactionLogicException { - if (user.externalUserId != null) { - try { - UserIdMapping.createUserIdMapping( - appIdentifier, getAllProxyStoragesForApp(main, appIdentifier), - primaryLM.superTokensUserId, user.externalUserId, - null, false, true); - - primaryLM.externalUserId = user.externalUserId; - } catch (StorageQueryException | ServletException | TenantOrAppNotFoundException | InvalidConfigException - | IOException | DbInitException e) { - throw new StorageTransactionLogicException(e); - } catch (UserIdMappingAlreadyExistsException e) { - throw new StorageTransactionLogicException( - new Exception("A user with externalId " + user.externalUserId + " already exists")); - } catch (UnknownSuperTokensUserIdException e) { - throw new StorageTransactionLogicException( - new Exception("We tried to create the externalUserId mapping for the superTokenUserId " - + primaryLM.superTokensUserId - + " but it doesn't exist. This should not happen. Please contact support.")); - } - } - } - - private void createUserMetadata(AppIdentifier appIdentifier, Storage storage, BulkImportUser user, - LoginMethod primaryLM) throws StorageTransactionLogicException { - if (user.userMetadata != null) { - try { - UserMetadata.updateUserMetadata(appIdentifier, storage, primaryLM.getSuperTokenOrExternalUserId(), - user.userMetadata); - } catch (StorageQueryException | TenantOrAppNotFoundException e) { - throw new StorageTransactionLogicException(e); - } - } - } - - private void createUserRoles(Main main, AppIdentifier appIdentifier, Storage storage, - BulkImportUser user) throws StorageTransactionLogicException { - if (user.userRoles != null) { - for (UserRole userRole : user.userRoles) { - try { - for (String tenantId : userRole.tenantIds) { - TenantIdentifier tenantIdentifier = new TenantIdentifier( - appIdentifier.getConnectionUriDomain(), appIdentifier.getAppId(), - tenantId); - - UserRoles.addRoleToUser(main, tenantIdentifier, storage, user.externalUserId, userRole.role); - } - } catch (TenantOrAppNotFoundException | StorageQueryException e) { - throw new StorageTransactionLogicException(e); - } catch (UnknownRoleException e) { - throw new StorageTransactionLogicException(new Exception("Role " + userRole.role - + " does not exist! You need pre-create the role before assigning it to the user.")); - } - } - } - } - - private void verifyEmailForAllLoginMethods(AppIdentifier appIdentifier, TransactionConnection con, Storage storage, - List loginMethods) throws StorageTransactionLogicException { - - for (LoginMethod lm : loginMethods) { - try { - - TenantIdentifier tenantIdentifier = new TenantIdentifier(appIdentifier.getConnectionUriDomain(), - appIdentifier.getAppId(), lm.tenantIds.get(0)); - - EmailVerificationSQLStorage emailVerificationSQLStorage = StorageUtils - .getEmailVerificationStorage(storage); - emailVerificationSQLStorage - .updateIsEmailVerified_Transaction(tenantIdentifier.toAppIdentifier(), con, - lm.getSuperTokenOrExternalUserId(), lm.email, true); - } catch (TenantOrAppNotFoundException | StorageQueryException e) { - throw new StorageTransactionLogicException(e); - } - } - } - - private void createTotpDevices(Main main, AppIdentifier appIdentifier, Storage storage, - BulkImportUser user, LoginMethod primaryLM) throws StorageTransactionLogicException { - if (user.totpDevices != null) { - for (TotpDevice totpDevice : user.totpDevices) { - try { - Totp.createDevice(main, appIdentifier, storage, primaryLM.getSuperTokenOrExternalUserId(), - totpDevice.deviceName, totpDevice.skew, totpDevice.period, totpDevice.secretKey, - true, System.currentTimeMillis()); - } catch (TenantOrAppNotFoundException | StorageQueryException | FeatureNotEnabledException e) { - throw new StorageTransactionLogicException(e); - } catch (DeviceAlreadyExistsException e) { - throw new StorageTransactionLogicException( - new Exception("A totp device with name " + totpDevice.deviceName + " already exists")); - } - } - } - } - - // Returns the primary loginMethod of the user. If no loginMethod is marked as - // primary, then the oldest loginMethod is returned. - private BulkImportUser.LoginMethod getPrimaryLoginMethod(BulkImportUser user) { - BulkImportUser.LoginMethod oldestLM = user.loginMethods.get(0); - for (BulkImportUser.LoginMethod lm : user.loginMethods) { - if (lm.isPrimary) { - return lm; - } - - if (lm.timeJoinedInMSSinceEpoch < oldestLM.timeJoinedInMSSinceEpoch) { - oldestLM = lm; - } - } - return oldestLM; - } - // Checks if the importedUser was processed from the same bulkImportUser entry. private boolean isProcessedUserFromSameBulkImportUserEntry( AuthRecipeUserInfo importedUser, BulkImportUser bulkImportEntry) { diff --git a/src/main/java/io/supertokens/webserver/Webserver.java b/src/main/java/io/supertokens/webserver/Webserver.java index c36f9aec2..23e953ef9 100644 --- a/src/main/java/io/supertokens/webserver/Webserver.java +++ b/src/main/java/io/supertokens/webserver/Webserver.java @@ -28,6 +28,7 @@ import io.supertokens.webserver.api.accountlinking.*; import io.supertokens.webserver.api.bulkimport.BulkImportAPI; import io.supertokens.webserver.api.bulkimport.DeleteBulkImportUserAPI; +import io.supertokens.webserver.api.bulkimport.ImportUserAPI; import io.supertokens.webserver.api.core.*; import io.supertokens.webserver.api.dashboard.*; import io.supertokens.webserver.api.emailpassword.UserAPI; @@ -264,6 +265,7 @@ private void setupRoutes() { addAPI(new BulkImportAPI(main)); addAPI(new DeleteBulkImportUserAPI(main)); + addAPI(new ImportUserAPI(main)); StandardContext context = tomcatReference.getContext(); Tomcat tomcat = tomcatReference.getTomcat(); diff --git a/src/main/java/io/supertokens/webserver/api/bulkimport/ImportUserAPI.java b/src/main/java/io/supertokens/webserver/api/bulkimport/ImportUserAPI.java new file mode 100644 index 000000000..ae29fdd63 --- /dev/null +++ b/src/main/java/io/supertokens/webserver/api/bulkimport/ImportUserAPI.java @@ -0,0 +1,103 @@ +/* + * Copyright (c) 2024, VRAI Labs and/or its affiliates. All rights reserved. + * + * This software is licensed under the Apache License, Version 2.0 (the + * "License") as published by the Apache Software Foundation. + * + * You may not use this file except in compliance with the License. You may + * obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package io.supertokens.webserver.api.bulkimport; + +import java.io.IOException; + +import com.google.gson.JsonArray; +import com.google.gson.JsonObject; +import com.google.gson.JsonPrimitive; + +import io.supertokens.Main; +import io.supertokens.bulkimport.BulkImport; +import io.supertokens.bulkimport.BulkImportUserUtils; +import io.supertokens.multitenancy.exception.BadPermissionException; +import io.supertokens.pluginInterface.Storage; +import io.supertokens.pluginInterface.StorageUtils; +import io.supertokens.pluginInterface.authRecipe.AuthRecipeUserInfo; +import io.supertokens.pluginInterface.bulkimport.BulkImportUser; +import io.supertokens.pluginInterface.exceptions.DbInitException; +import io.supertokens.pluginInterface.exceptions.InvalidConfigException; +import io.supertokens.pluginInterface.exceptions.StorageQueryException; +import io.supertokens.pluginInterface.multitenancy.AppIdentifier; +import io.supertokens.pluginInterface.multitenancy.exceptions.TenantOrAppNotFoundException; +import io.supertokens.storageLayer.StorageLayer; +import io.supertokens.utils.Utils; +import io.supertokens.webserver.InputParser; +import io.supertokens.webserver.WebserverAPI; +import jakarta.servlet.ServletException; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; + +public class ImportUserAPI extends WebserverAPI { + public ImportUserAPI(Main main) { + super(main, ""); + } + + @Override + public String getPath() { + return "/bulk-import/import"; + } + + @Override + protected void doPost(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException { + // API is app specific + + if (StorageLayer.isInMemDb(main)) { + throw new ServletException(new BadRequestException("This API is not supported in the in-memory database.")); + } + + JsonObject input = InputParser.parseJsonObjectOrThrowError(req); + JsonObject jsonUser = InputParser.parseJsonObjectOrThrowError(input, "user", false); + + AppIdentifier appIdentifier = null; + Storage storage = null; + String[] allUserRoles = null; + + try { + appIdentifier = getAppIdentifier(req); + storage = enforcePublicTenantAndGetPublicTenantStorage(req); + allUserRoles = StorageUtils.getUserRolesStorage(storage).getRoles(appIdentifier); + } catch (TenantOrAppNotFoundException | BadPermissionException | StorageQueryException e) { + throw new ServletException(e); + } + + BulkImportUserUtils bulkImportUserUtils = new BulkImportUserUtils(allUserRoles); + + try { + BulkImportUser user = bulkImportUserUtils.createBulkImportUserFromJSON(main, appIdentifier, jsonUser, + Utils.getUUID()); + + AuthRecipeUserInfo importedUser = BulkImport.importUser(main, appIdentifier, user); + + JsonObject result = new JsonObject(); + result.addProperty("status", "OK"); + result.add("user", importedUser.toJson()); + super.sendJsonResponse(200, result, resp); + } catch (io.supertokens.bulkimport.exceptions.InvalidBulkImportDataException e) { + JsonArray errors = e.errors.stream() + .map(JsonPrimitive::new) + .collect(JsonArray::new, JsonArray::add, JsonArray::addAll); + + JsonObject errorResponseJson = new JsonObject(); + errorResponseJson.add("errors", errors); + throw new ServletException(new WebserverAPI.BadRequestException(errorResponseJson.toString())); + } catch (StorageQueryException | TenantOrAppNotFoundException | InvalidConfigException | DbInitException e) { + throw new ServletException(e); + } + } +} diff --git a/src/test/java/io/supertokens/test/bulkimport/BulkImportTest.java b/src/test/java/io/supertokens/test/bulkimport/BulkImportTest.java index 5eaf38842..a673b7682 100644 --- a/src/test/java/io/supertokens/test/bulkimport/BulkImportTest.java +++ b/src/test/java/io/supertokens/test/bulkimport/BulkImportTest.java @@ -20,7 +20,12 @@ import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; +import java.util.ArrayList; import java.util.List; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import org.junit.AfterClass; @@ -35,15 +40,21 @@ import io.supertokens.bulkimport.BulkImportUserPaginationContainer; import io.supertokens.cronjobs.CronTaskTest; import io.supertokens.cronjobs.bulkimport.ProcessBulkImportUsers; +import io.supertokens.featureflag.EE_FEATURES; +import io.supertokens.featureflag.FeatureFlagTestContent; import io.supertokens.pluginInterface.STORAGE_TYPE; +import io.supertokens.pluginInterface.Storage; +import io.supertokens.pluginInterface.authRecipe.AuthRecipeUserInfo; import io.supertokens.pluginInterface.bulkimport.BulkImportStorage; import io.supertokens.pluginInterface.bulkimport.BulkImportUser; import io.supertokens.pluginInterface.bulkimport.BulkImportStorage.BULK_IMPORT_USER_STATUS; import io.supertokens.pluginInterface.bulkimport.sqlStorage.BulkImportSQLStorage; import io.supertokens.pluginInterface.multitenancy.AppIdentifier; +import io.supertokens.pluginInterface.multitenancy.TenantIdentifier; import io.supertokens.storageLayer.StorageLayer; import io.supertokens.test.TestingProcessManager; import io.supertokens.test.Utils; +import io.supertokens.userroles.UserRoles; import static io.supertokens.test.bulkimport.BulkImportTestUtils.generateBulkImportUser; @@ -63,7 +74,7 @@ public void beforeEach() { @Test public void shouldAddUsersInBulkImportUsersTable() throws Exception { - String[] args = {"../"}; + String[] args = { "../" }; TestingProcessManager.TestingProcess process = TestingProcessManager.start(args); assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); @@ -78,7 +89,8 @@ public void shouldAddUsersInBulkImportUsersTable() throws Exception { BulkImportStorage storage = (BulkImportStorage) StorageLayer.getStorage(process.main); BulkImport.addUsers(new AppIdentifier(null, null), storage, users); - List addedUsers = storage.getBulkImportUsers(new AppIdentifier(null, null), 100, BULK_IMPORT_USER_STATUS.NEW, null, null); + List addedUsers = storage.getBulkImportUsers(new AppIdentifier(null, null), 100, + BULK_IMPORT_USER_STATUS.NEW, null, null); // Verify that all users are present in addedUsers for (BulkImportUser user : users) { @@ -98,7 +110,7 @@ public void shouldAddUsersInBulkImportUsersTable() throws Exception { @Test public void shouldCreatedNewIdsIfDuplicateIdIsFound() throws Exception { - String[] args = {"../"}; + String[] args = { "../" }; TestingProcessManager.TestingProcess process = TestingProcessManager.start(args); assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); @@ -119,7 +131,8 @@ public void shouldCreatedNewIdsIfDuplicateIdIsFound() throws Exception { AppIdentifier appIdentifier = new AppIdentifier(null, null); BulkImport.addUsers(appIdentifier, storage, users); - List addedUsers = storage.getBulkImportUsers(appIdentifier, 1000, BULK_IMPORT_USER_STATUS.NEW, null, null); + List addedUsers = storage.getBulkImportUsers(appIdentifier, 1000, BULK_IMPORT_USER_STATUS.NEW, + null, null); // Verify that the other properties are same but ids changed for (BulkImportUser user : users) { @@ -139,7 +152,7 @@ public void shouldCreatedNewIdsIfDuplicateIdIsFound() throws Exception { @Test public void testGetUsersStatusFilter() throws Exception { - String[] args = {"../"}; + String[] args = { "../" }; TestingProcessManager.TestingProcess process = TestingProcessManager.start(args); assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); @@ -148,7 +161,7 @@ public void testGetUsersStatusFilter() throws Exception { if (StorageLayer.getStorage(main).getType() != STORAGE_TYPE.SQL || StorageLayer.isInMemDb(main)) { return; } - + BulkImportSQLStorage storage = (BulkImportSQLStorage) StorageLayer.getStorage(process.main); AppIdentifier appIdentifier = new AppIdentifier(null, null); @@ -157,7 +170,8 @@ public void testGetUsersStatusFilter() throws Exception { List users = generateBulkImportUser(10); BulkImport.addUsers(appIdentifier, storage, users); - List addedUsers = storage.getBulkImportUsers(appIdentifier, 100, BULK_IMPORT_USER_STATUS.NEW, null, null); + List addedUsers = storage.getBulkImportUsers(appIdentifier, 100, + BULK_IMPORT_USER_STATUS.NEW, null, null); assertEquals(10, addedUsers.size()); } @@ -169,13 +183,15 @@ public void testGetUsersStatusFilter() throws Exception { // Update the users status to PROCESSING storage.startTransaction(con -> { for (BulkImportUser user : users) { - storage.updateBulkImportUserStatus_Transaction(appIdentifier, con, user.id, BULK_IMPORT_USER_STATUS.PROCESSING, null); + storage.updateBulkImportUserStatus_Transaction(appIdentifier, con, user.id, + BULK_IMPORT_USER_STATUS.PROCESSING, null); } storage.commitTransaction(con); return null; }); - List addedUsers = storage.getBulkImportUsers(appIdentifier, 100, BULK_IMPORT_USER_STATUS.PROCESSING, null, null); + List addedUsers = storage.getBulkImportUsers(appIdentifier, 100, + BULK_IMPORT_USER_STATUS.PROCESSING, null, null); assertEquals(10, addedUsers.size()); } @@ -187,13 +203,15 @@ public void testGetUsersStatusFilter() throws Exception { // Update the users status to FAILED storage.startTransaction(con -> { for (BulkImportUser user : users) { - storage.updateBulkImportUserStatus_Transaction(appIdentifier, con, user.id, BULK_IMPORT_USER_STATUS.FAILED, null); + storage.updateBulkImportUserStatus_Transaction(appIdentifier, con, user.id, + BULK_IMPORT_USER_STATUS.FAILED, null); } storage.commitTransaction(con); return null; }); - List addedUsers = storage.getBulkImportUsers(appIdentifier, 100, BULK_IMPORT_USER_STATUS.FAILED, null, null); + List addedUsers = storage.getBulkImportUsers(appIdentifier, 100, + BULK_IMPORT_USER_STATUS.FAILED, null, null); assertEquals(10, addedUsers.size()); } @@ -203,12 +221,13 @@ public void testGetUsersStatusFilter() throws Exception { @Test public void randomPaginationTest() throws Exception { - String[] args = {"../"}; + String[] args = { "../" }; TestingProcessManager.TestingProcess process = TestingProcessManager.start(args, false); // We are setting a high initial wait time to ensure the cron job doesn't run while we are running the tests - CronTaskTest.getInstance(process.getProcess()).setInitialWaitTimeInSeconds(ProcessBulkImportUsers.RESOURCE_KEY, 1000000); + CronTaskTest.getInstance(process.getProcess()).setInitialWaitTimeInSeconds(ProcessBulkImportUsers.RESOURCE_KEY, + 1000000); process.startProcess(); @@ -234,7 +253,8 @@ public void randomPaginationTest() throws Exception { } // Get all inserted users - List addedUsers = storage.getBulkImportUsers(new AppIdentifier(null, null), 1000, null, null, null); + List addedUsers = storage.getBulkImportUsers(new AppIdentifier(null, null), 1000, null, null, + null); assertEquals(numberOfUsers, addedUsers.size()); // We are sorting the users based on createdAt and id like we do in the storage layer @@ -248,13 +268,14 @@ public void randomPaginationTest() throws Exception { }) .collect(Collectors.toList()); - int[] limits = new int[]{10, 14, 20, 23, 50, 100, 110, 150, 200, 510}; + int[] limits = new int[] { 10, 14, 20, 23, 50, 100, 110, 150, 200, 510 }; for (int limit : limits) { int indexIntoUsers = 0; String paginationToken = null; do { - BulkImportUserPaginationContainer users = BulkImport.getUsers(new AppIdentifier(null, null), storage, limit, null, paginationToken); + BulkImportUserPaginationContainer users = BulkImport.getUsers(new AppIdentifier(null, null), storage, + limit, null, paginationToken); for (BulkImportUser actualUser : users.users) { BulkImportUser expectedUser = sortedUsers.get(indexIntoUsers); @@ -275,4 +296,144 @@ public void randomPaginationTest() throws Exception { assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STOPPED)); } + @Test + public void shouldImportTheUserInTheSameTenant() throws Exception { + String[] args = { "../" }; + + TestingProcessManager.TestingProcess process = TestingProcessManager.start(args); + assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); + Main main = process.getProcess(); + + if (StorageLayer.getStorage(main).getType() != STORAGE_TYPE.SQL || StorageLayer.isInMemDb(main)) { + return; + } + + FeatureFlagTestContent.getInstance(main).setKeyValue(FeatureFlagTestContent.ENABLED_FEATURES, + new EE_FEATURES[] { EE_FEATURES.MULTI_TENANCY, EE_FEATURES.MFA, EE_FEATURES.ACCOUNT_LINKING }); + + // Create tenants + BulkImportTestUtils.createTenants(main); + + // Create user roles + { + UserRoles.createNewRoleOrModifyItsPermissions(main, "role1", null); + UserRoles.createNewRoleOrModifyItsPermissions(main, "role2", null); + } + + AppIdentifier appIdentifier = new AppIdentifier(null, null); + List users = generateBulkImportUser(1); + + AuthRecipeUserInfo importedUser = BulkImport.importUser(main, appIdentifier, users.get(0)); + + BulkImportTestUtils.assertBulkImportUserAndAuthRecipeUserAreEqual(appIdentifier, + appIdentifier.getAsPublicTenantIdentifier(), StorageLayer.getStorage(main), users.get(0), importedUser); + + process.kill(); + assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STOPPED)); + } + + @Test + public void shouldImportTheUserInMultipleTenantsWithDifferentStorages() throws Exception { + String[] args = { "../" }; + + TestingProcessManager.TestingProcess process = TestingProcessManager.start(args); + assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); + Main main = process.getProcess(); + + if (StorageLayer.getStorage(main).getType() != STORAGE_TYPE.SQL || StorageLayer.isInMemDb(main)) { + return; + } + + FeatureFlagTestContent.getInstance(main).setKeyValue(FeatureFlagTestContent.ENABLED_FEATURES, + new EE_FEATURES[] { EE_FEATURES.MULTI_TENANCY, EE_FEATURES.MFA, EE_FEATURES.ACCOUNT_LINKING }); + + // Create tenants + BulkImportTestUtils.createTenants(main); + + // Create user roles + { + UserRoles.createNewRoleOrModifyItsPermissions(main, "role1", null); + UserRoles.createNewRoleOrModifyItsPermissions(main, "role2", null); + } + + TenantIdentifier t1 = new TenantIdentifier(null, null, "t1"); + TenantIdentifier t2 = new TenantIdentifier(null, null, "t2"); + + Storage storageT1 = StorageLayer.getStorage(t1, main); + Storage storageT2 = StorageLayer.getStorage(t2, main); + + AppIdentifier appIdentifier = new AppIdentifier(null, null); + + List usersT1 = generateBulkImportUser(1, List.of(t1.getTenantId()), 0); + List usersT2 = generateBulkImportUser(1, List.of(t2.getTenantId()), 1); + + BulkImportUser bulkImportUserT1 = usersT1.get(0); + BulkImportUser bulkImportUserT2 = usersT2.get(0); + + AuthRecipeUserInfo importedUser1 = BulkImport.importUser(main, appIdentifier, bulkImportUserT1); + AuthRecipeUserInfo importedUser2 = BulkImport.importUser(main, appIdentifier, bulkImportUserT2); + + BulkImportTestUtils.assertBulkImportUserAndAuthRecipeUserAreEqual(appIdentifier, t1, storageT1, + bulkImportUserT1, + importedUser1); + BulkImportTestUtils.assertBulkImportUserAndAuthRecipeUserAreEqual(appIdentifier, t2, storageT2, + bulkImportUserT2, + importedUser2); + + process.kill(); + assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STOPPED)); + } + + @Test + public void shouldImportUsersConcurrently() throws Exception { + String[] args = { "../" }; + + TestingProcessManager.TestingProcess process = TestingProcessManager.start(args); + assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); + Main main = process.getProcess(); + + if (StorageLayer.getStorage(main).getType() != STORAGE_TYPE.SQL || StorageLayer.isInMemDb(main)) { + return; + } + + FeatureFlagTestContent.getInstance(main).setKeyValue(FeatureFlagTestContent.ENABLED_FEATURES, + new EE_FEATURES[] { EE_FEATURES.MULTI_TENANCY, EE_FEATURES.MFA, EE_FEATURES.ACCOUNT_LINKING }); + + // Create tenants + BulkImportTestUtils.createTenants(main); + + // Create user roles + { + UserRoles.createNewRoleOrModifyItsPermissions(main, "role1", null); + UserRoles.createNewRoleOrModifyItsPermissions(main, "role2", null); + } + + AppIdentifier appIdentifier = new AppIdentifier(null, null); + List users = generateBulkImportUser(10); + + // Concurrently import users + ExecutorService executor = Executors.newFixedThreadPool(10); + List> futures = new ArrayList<>(); + + for (BulkImportUser user : users) { + Future future = executor.submit(() -> { + return BulkImport.importUser(main, appIdentifier, user); + }); + futures.add(future); + } + + executor.shutdown(); + executor.awaitTermination(1, TimeUnit.MINUTES); + + for (int i = 0; i < users.size(); i++) { + AuthRecipeUserInfo importedUser = futures.get(i).get(); + BulkImportTestUtils.assertBulkImportUserAndAuthRecipeUserAreEqual(appIdentifier, + appIdentifier.getAsPublicTenantIdentifier(), StorageLayer.getStorage(main), users.get(i), + importedUser); + } + + process.kill(); + assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STOPPED)); + } + } diff --git a/src/test/java/io/supertokens/test/bulkimport/BulkImportTestUtils.java b/src/test/java/io/supertokens/test/bulkimport/BulkImportTestUtils.java index cfee7fa64..4faea47ab 100644 --- a/src/test/java/io/supertokens/test/bulkimport/BulkImportTestUtils.java +++ b/src/test/java/io/supertokens/test/bulkimport/BulkImportTestUtils.java @@ -14,19 +14,45 @@ * under the License. */ - package io.supertokens.test.bulkimport; +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.io.IOException; import java.util.ArrayList; import java.util.List; import com.google.gson.JsonObject; import com.google.gson.JsonParser; +import io.supertokens.Main; +import io.supertokens.featureflag.exceptions.FeatureNotEnabledException; +import io.supertokens.multitenancy.Multitenancy; +import io.supertokens.multitenancy.exception.BadPermissionException; +import io.supertokens.multitenancy.exception.CannotModifyBaseConfigException; +import io.supertokens.pluginInterface.Storage; +import io.supertokens.pluginInterface.authRecipe.AuthRecipeUserInfo; import io.supertokens.pluginInterface.bulkimport.BulkImportUser; import io.supertokens.pluginInterface.bulkimport.BulkImportUser.LoginMethod; import io.supertokens.pluginInterface.bulkimport.BulkImportUser.TotpDevice; import io.supertokens.pluginInterface.bulkimport.BulkImportUser.UserRole; +import io.supertokens.pluginInterface.exceptions.InvalidConfigException; +import io.supertokens.pluginInterface.exceptions.StorageQueryException; +import io.supertokens.pluginInterface.multitenancy.AppIdentifier; +import io.supertokens.pluginInterface.multitenancy.EmailPasswordConfig; +import io.supertokens.pluginInterface.multitenancy.PasswordlessConfig; +import io.supertokens.pluginInterface.multitenancy.TenantConfig; +import io.supertokens.pluginInterface.multitenancy.TenantIdentifier; +import io.supertokens.pluginInterface.multitenancy.ThirdPartyConfig; +import io.supertokens.pluginInterface.multitenancy.exceptions.TenantOrAppNotFoundException; +import io.supertokens.pluginInterface.totp.TOTPDevice; +import io.supertokens.storageLayer.StorageLayer; +import io.supertokens.thirdparty.InvalidProviderConfigException; +import io.supertokens.totp.Totp; +import io.supertokens.usermetadata.UserMetadata; +import io.supertokens.userroles.UserRoles; public class BulkImportTestUtils { @@ -43,7 +69,8 @@ public static List generateBulkImportUser(int numberOfUsers, Lis String id = io.supertokens.utils.Utils.getUUID(); String externalId = io.supertokens.utils.Utils.getUUID(); - JsonObject userMetadata = parser.parse("{\"key1\":\"value1\",\"key2\":{\"key3\":\"value3\"}}").getAsJsonObject(); + JsonObject userMetadata = parser.parse("{\"key1\":\"value1\",\"key2\":{\"key3\":\"value3\"}}") + .getAsJsonObject(); List userRoles = new ArrayList<>(); userRoles.add(new UserRole("role1", tenants)); @@ -53,12 +80,110 @@ public static List generateBulkImportUser(int numberOfUsers, Lis totpDevices.add(new TotpDevice("secretKey", 30, 1, "deviceName")); List loginMethods = new ArrayList<>(); - long currentTimeMillis = System.currentTimeMillis(); - loginMethods.add(new LoginMethod(tenants, "emailpassword", true, true, currentTimeMillis, email, "$2a", "BCRYPT", null, null, null)); - loginMethods.add(new LoginMethod(tenants, "thirdparty", true, false, currentTimeMillis, email, null, null, "thirdPartyId" + i, "thirdPartyUserId" + i, null)); - loginMethods.add(new LoginMethod(tenants, "passwordless", true, false, currentTimeMillis, email, null, null, null, null, null)); + long currentTimeMillis = System.currentTimeMillis(); + loginMethods.add(new LoginMethod(tenants, "emailpassword", true, true, currentTimeMillis, email, "$2a", + "BCRYPT", null, null, null)); + loginMethods.add(new LoginMethod(tenants, "thirdparty", true, false, currentTimeMillis, email, null, null, + "thirdPartyId" + i, "thirdPartyUserId" + i, null)); + loginMethods.add(new LoginMethod(tenants, "passwordless", true, false, currentTimeMillis, email, null, null, + null, null, null)); users.add(new BulkImportUser(id, externalId, userMetadata, userRoles, totpDevices, loginMethods)); } return users; } + + public static void createTenants(Main main) + throws StorageQueryException, TenantOrAppNotFoundException, InvalidProviderConfigException, + FeatureNotEnabledException, IOException, InvalidConfigException, + CannotModifyBaseConfigException, BadPermissionException { + // User pool 1 - (null, null, null), (null, null, t1) + // User pool 2 - (null, null, t2) + + { // tenant 1 + TenantIdentifier tenantIdentifier = new TenantIdentifier(null, null, "t1"); + + Multitenancy.addNewOrUpdateAppOrTenant( + main, + new TenantIdentifier(null, null, null), + new TenantConfig( + tenantIdentifier, + new EmailPasswordConfig(true), + new ThirdPartyConfig(true, null), + new PasswordlessConfig(true), + null, null, new JsonObject())); + } + { // tenant 2 + JsonObject config = new JsonObject(); + TenantIdentifier tenantIdentifier = new TenantIdentifier(null, null, "t2"); + + StorageLayer.getStorage(new TenantIdentifier(null, null, null), main) + .modifyConfigToAddANewUserPoolForTesting(config, 1); + + Multitenancy.addNewOrUpdateAppOrTenant( + main, + new TenantIdentifier(null, null, null), + new TenantConfig( + tenantIdentifier, + new EmailPasswordConfig(true), + new ThirdPartyConfig(true, null), + new PasswordlessConfig(true), + null, null, config)); + } + } + + public static void assertBulkImportUserAndAuthRecipeUserAreEqual(AppIdentifier appIdentifier, + TenantIdentifier tenantIdentifier, Storage storage, BulkImportUser bulkImportUser, + AuthRecipeUserInfo authRecipeUser) throws StorageQueryException, TenantOrAppNotFoundException { + for (io.supertokens.pluginInterface.authRecipe.LoginMethod lm1 : authRecipeUser.loginMethods) { + bulkImportUser.loginMethods.forEach(lm2 -> { + if (lm2.recipeId.equals(lm1.recipeId.toString())) { + assertLoginMethodEquals(lm1, lm2); + } + }); + } + assertEquals(bulkImportUser.externalUserId, authRecipeUser.getSupertokensOrExternalUserId()); + assertEquals(bulkImportUser.userMetadata, + UserMetadata.getUserMetadata(appIdentifier, storage, authRecipeUser.getSupertokensOrExternalUserId())); + + String[] createdUserRoles = UserRoles.getRolesForUser(tenantIdentifier, storage, + authRecipeUser.getSupertokensOrExternalUserId()); + String[] bulkImportUserRoles = bulkImportUser.userRoles.stream().map(r -> r.role).toArray(String[]::new); + assertArrayEquals(bulkImportUserRoles, createdUserRoles); + + TOTPDevice[] createdTotpDevices = Totp.getDevices(appIdentifier, storage, + authRecipeUser.getSupertokensOrExternalUserId()); + assertTotpDevicesEquals(createdTotpDevices, bulkImportUser.totpDevices.toArray(new TotpDevice[0])); + } + + private static void assertLoginMethodEquals(io.supertokens.pluginInterface.authRecipe.LoginMethod lm1, + io.supertokens.pluginInterface.bulkimport.BulkImportUser.LoginMethod lm2) { + assertEquals(lm1.email, lm2.email); + assertEquals(lm1.verified, lm2.isVerified); + assertTrue(lm2.tenantIds.containsAll(lm1.tenantIds) && lm1.tenantIds.containsAll(lm2.tenantIds)); + + switch (lm2.recipeId) { + case "emailpassword": + assertEquals(lm1.passwordHash, lm2.passwordHash); + break; + case "thirdparty": + assertEquals(lm1.thirdParty.id, lm2.thirdPartyId); + assertEquals(lm1.thirdParty.userId, lm2.thirdPartyUserId); + break; + case "passwordless": + assertEquals(lm1.phoneNumber, lm2.phoneNumber); + break; + default: + break; + } + } + + private static void assertTotpDevicesEquals(TOTPDevice[] createdTotpDevices, TotpDevice[] bulkImportTotpDevices) { + assertEquals(createdTotpDevices.length, bulkImportTotpDevices.length); + for (int i = 0; i < createdTotpDevices.length; i++) { + assertEquals(createdTotpDevices[i].deviceName, bulkImportTotpDevices[i].deviceName); + assertEquals(createdTotpDevices[i].period, bulkImportTotpDevices[i].period); + assertEquals(createdTotpDevices[i].secretKey, bulkImportTotpDevices[i].secretKey); + assertEquals(createdTotpDevices[i].skew, bulkImportTotpDevices[i].skew); + } + } } diff --git a/src/test/java/io/supertokens/test/bulkimport/ProcessBulkImportUsersCronJobTest.java b/src/test/java/io/supertokens/test/bulkimport/ProcessBulkImportUsersCronJobTest.java index a8e4cbc52..a17f842e7 100644 --- a/src/test/java/io/supertokens/test/bulkimport/ProcessBulkImportUsersCronJobTest.java +++ b/src/test/java/io/supertokens/test/bulkimport/ProcessBulkImportUsersCronJobTest.java @@ -26,36 +26,18 @@ import io.supertokens.cronjobs.bulkimport.ProcessBulkImportUsers; import io.supertokens.featureflag.EE_FEATURES; import io.supertokens.featureflag.FeatureFlagTestContent; -import io.supertokens.featureflag.exceptions.FeatureNotEnabledException; -import io.supertokens.multitenancy.Multitenancy; -import io.supertokens.multitenancy.exception.BadPermissionException; -import io.supertokens.multitenancy.exception.CannotModifyBaseConfigException; import io.supertokens.pluginInterface.STORAGE_TYPE; import io.supertokens.pluginInterface.Storage; -import io.supertokens.pluginInterface.authRecipe.AuthRecipeUserInfo; -import io.supertokens.pluginInterface.authRecipe.LoginMethod; import io.supertokens.pluginInterface.bulkimport.BulkImportUser; import io.supertokens.pluginInterface.bulkimport.BulkImportStorage.BULK_IMPORT_USER_STATUS; -import io.supertokens.pluginInterface.bulkimport.BulkImportUser.TotpDevice; import io.supertokens.pluginInterface.bulkimport.sqlStorage.BulkImportSQLStorage; -import io.supertokens.pluginInterface.exceptions.InvalidConfigException; -import io.supertokens.pluginInterface.exceptions.StorageQueryException; import io.supertokens.pluginInterface.multitenancy.AppIdentifier; -import io.supertokens.pluginInterface.multitenancy.EmailPasswordConfig; -import io.supertokens.pluginInterface.multitenancy.PasswordlessConfig; -import io.supertokens.pluginInterface.multitenancy.TenantConfig; import io.supertokens.pluginInterface.multitenancy.TenantIdentifier; -import io.supertokens.pluginInterface.multitenancy.ThirdPartyConfig; -import io.supertokens.pluginInterface.multitenancy.exceptions.TenantOrAppNotFoundException; -import io.supertokens.pluginInterface.totp.TOTPDevice; import io.supertokens.storageLayer.StorageLayer; import io.supertokens.test.TestingProcessManager; import io.supertokens.test.TestingProcessManager.TestingProcess; import io.supertokens.test.Utils; -import io.supertokens.thirdparty.InvalidProviderConfigException; -import io.supertokens.totp.Totp; import io.supertokens.useridmapping.UserIdMapping; -import io.supertokens.usermetadata.UserMetadata; import io.supertokens.userroles.UserRoles; import org.junit.AfterClass; @@ -64,15 +46,10 @@ import org.junit.Test; import org.junit.rules.TestRule; -import com.google.gson.JsonObject; - import static io.supertokens.test.bulkimport.BulkImportTestUtils.generateBulkImportUser; -import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; -import java.io.IOException; import java.util.List; public class ProcessBulkImportUsersCronJobTest { @@ -104,7 +81,7 @@ public void shouldProcessBulkImportUsersInTheSameTenant() throws Exception { UserRoles.createNewRoleOrModifyItsPermissions(main, "role2", null); } - createTenants(main); + BulkImportTestUtils.createTenants(main); BulkImportSQLStorage storage = (BulkImportSQLStorage) StorageLayer.getStorage(main); AppIdentifier appIdentifier = new AppIdentifier(null, null); @@ -129,7 +106,8 @@ public void shouldProcessBulkImportUsersInTheSameTenant() throws Exception { TenantIdentifier publicTenant = new TenantIdentifier(null, null, "public"); - assertBulkImportUserAndAuthRecipeUserAreEqual(appIdentifier, publicTenant, storage, bulkImportUser, + BulkImportTestUtils.assertBulkImportUserAndAuthRecipeUserAreEqual(appIdentifier, publicTenant, storage, + bulkImportUser, container.users[0]); process.kill(); @@ -151,7 +129,7 @@ public void shouldProcessBulkImportUsersInMultipleTenantsWithDifferentStorages() UserRoles.createNewRoleOrModifyItsPermissions(main, "role2", null); } - createTenants(main); + BulkImportTestUtils.createTenants(main); TenantIdentifier t1 = new TenantIdentifier(null, null, "t1"); TenantIdentifier t2 = new TenantIdentifier(null, null, "t2"); @@ -185,9 +163,11 @@ public void shouldProcessBulkImportUsersInMultipleTenantsWithDifferentStorages() UserIdMapping.populateExternalUserIdForUsers(appIdentifier, storageT1, containerT1.users); UserIdMapping.populateExternalUserIdForUsers(appIdentifier, storageT2, containerT2.users); - assertBulkImportUserAndAuthRecipeUserAreEqual(appIdentifier, t1, storageT1, bulkImportUserT1, + BulkImportTestUtils.assertBulkImportUserAndAuthRecipeUserAreEqual(appIdentifier, t1, storageT1, + bulkImportUserT1, containerT1.users[0]); - assertBulkImportUserAndAuthRecipeUserAreEqual(appIdentifier, t2, storageT2, bulkImportUserT2, + BulkImportTestUtils.assertBulkImportUserAndAuthRecipeUserAreEqual(appIdentifier, t2, storageT2, + bulkImportUserT2, containerT2.users[0]); process.kill(); @@ -209,7 +189,7 @@ public void shouldDeleteEverythingFromtheDBIfAnythingFails() throws Exception { return; } - createTenants(main); + BulkImportTestUtils.createTenants(main); BulkImportSQLStorage storage = (BulkImportSQLStorage) StorageLayer.getStorage(main); AppIdentifier appIdentifier = new AppIdentifier(null, null); @@ -282,8 +262,7 @@ public void shouldThrowTenantHaveDifferentStoragesError() throws Exception { UserRoles.createNewRoleOrModifyItsPermissions(main, "role1", null); UserRoles.createNewRoleOrModifyItsPermissions(main, "role2", null); } - createTenants(main); - + BulkImportTestUtils.createTenants(main); List users = generateBulkImportUser(1, List.of("t1", "t2"), 0); BulkImport.addUsers(appIdentifier, storage, users); @@ -323,96 +302,4 @@ private TestingProcess startCronProcess() throws InterruptedException { return process; } - - private void assertBulkImportUserAndAuthRecipeUserAreEqual(AppIdentifier appIdentifier, - TenantIdentifier tenantIdentifier, Storage storage, BulkImportUser bulkImportUser, - AuthRecipeUserInfo authRecipeUser) throws StorageQueryException, TenantOrAppNotFoundException { - for (LoginMethod lm1 : authRecipeUser.loginMethods) { - bulkImportUser.loginMethods.forEach(lm2 -> { - if (lm2.recipeId.equals(lm1.recipeId.toString())) { - assertLoginMethodEquals(lm1, lm2); - } - }); - } - assertEquals(bulkImportUser.externalUserId, authRecipeUser.getSupertokensOrExternalUserId()); - assertEquals(bulkImportUser.userMetadata, - UserMetadata.getUserMetadata(appIdentifier, storage, authRecipeUser.getSupertokensOrExternalUserId())); - - String[] createdUserRoles = UserRoles.getRolesForUser(tenantIdentifier, storage, - authRecipeUser.getSupertokensOrExternalUserId()); - String[] bulkImportUserRoles = bulkImportUser.userRoles.stream().map(r -> r.role).toArray(String[]::new); - assertArrayEquals(bulkImportUserRoles, createdUserRoles); - - TOTPDevice[] createdTotpDevices = Totp.getDevices(appIdentifier, storage, - authRecipeUser.getSupertokensOrExternalUserId()); - assertTotpDevicesEquals(createdTotpDevices, bulkImportUser.totpDevices.toArray(new TotpDevice[0])); - } - - private void assertLoginMethodEquals(LoginMethod lm1, - io.supertokens.pluginInterface.bulkimport.BulkImportUser.LoginMethod lm2) { - assertEquals(lm1.email, lm2.email); - assertEquals(lm1.verified, lm2.isVerified); - assertTrue(lm2.tenantIds.containsAll(lm1.tenantIds) && lm1.tenantIds.containsAll(lm2.tenantIds)); - - switch (lm2.recipeId) { - case "emailpassword": - assertEquals(lm1.passwordHash, lm2.passwordHash); - break; - case "thirdparty": - assertEquals(lm1.thirdParty.id, lm2.thirdPartyId); - assertEquals(lm1.thirdParty.userId, lm2.thirdPartyUserId); - break; - case "passwordless": - assertEquals(lm1.phoneNumber, lm2.phoneNumber); - break; - default: - break; - } - } - - private void assertTotpDevicesEquals(TOTPDevice[] createdTotpDevices, TotpDevice[] bulkImportTotpDevices) { - assertEquals(createdTotpDevices.length, bulkImportTotpDevices.length); - for (int i = 0; i < createdTotpDevices.length; i++) { - assertEquals(createdTotpDevices[i].deviceName, bulkImportTotpDevices[i].deviceName); - assertEquals(createdTotpDevices[i].period, bulkImportTotpDevices[i].period); - assertEquals(createdTotpDevices[i].secretKey, bulkImportTotpDevices[i].secretKey); - assertEquals(createdTotpDevices[i].skew, bulkImportTotpDevices[i].skew); - } - } - - private void createTenants(Main main) - throws StorageQueryException, TenantOrAppNotFoundException, InvalidProviderConfigException, - FeatureNotEnabledException, IOException, InvalidConfigException, - CannotModifyBaseConfigException, BadPermissionException { - { // tenant 1 (t1 in the same storage as public tenant) - TenantIdentifier tenantIdentifier = new TenantIdentifier(null, null, "t1"); - - Multitenancy.addNewOrUpdateAppOrTenant( - main, - new TenantIdentifier(null, null, null), - new TenantConfig( - tenantIdentifier, - new EmailPasswordConfig(true), - new ThirdPartyConfig(true, null), - new PasswordlessConfig(true), - null, null, new JsonObject())); - } - { // tenant 2 (t2 in the different storage than public tenant) - TenantIdentifier tenantIdentifier = new TenantIdentifier(null, null, "t2"); - - JsonObject config = new JsonObject(); - - StorageLayer.getStorage(new TenantIdentifier(null, null, null), main) - .modifyConfigToAddANewUserPoolForTesting(config, 1); - Multitenancy.addNewOrUpdateAppOrTenant( - main, - new TenantIdentifier(null, null, null), - new TenantConfig( - tenantIdentifier, - new EmailPasswordConfig(true), - new ThirdPartyConfig(true, null), - new PasswordlessConfig(true), - null, null, config)); - } - } } diff --git a/src/test/java/io/supertokens/test/bulkimport/apis/AddBulkImportUsersTest.java b/src/test/java/io/supertokens/test/bulkimport/apis/AddBulkImportUsersTest.java index 2ff85b4a4..c4ac061d8 100644 --- a/src/test/java/io/supertokens/test/bulkimport/apis/AddBulkImportUsersTest.java +++ b/src/test/java/io/supertokens/test/bulkimport/apis/AddBulkImportUsersTest.java @@ -21,7 +21,6 @@ import static org.junit.Assert.assertNotNull; import static org.junit.Assert.fail; -import java.io.IOException; import java.lang.reflect.Field; import java.util.Arrays; import java.util.HashMap; @@ -44,25 +43,13 @@ import io.supertokens.ProcessState; import io.supertokens.featureflag.EE_FEATURES; import io.supertokens.featureflag.FeatureFlagTestContent; -import io.supertokens.featureflag.exceptions.FeatureNotEnabledException; -import io.supertokens.multitenancy.Multitenancy; -import io.supertokens.multitenancy.exception.BadPermissionException; -import io.supertokens.multitenancy.exception.CannotModifyBaseConfigException; import io.supertokens.pluginInterface.STORAGE_TYPE; import io.supertokens.pluginInterface.bulkimport.BulkImportUser; -import io.supertokens.pluginInterface.exceptions.InvalidConfigException; -import io.supertokens.pluginInterface.exceptions.StorageQueryException; -import io.supertokens.pluginInterface.multitenancy.EmailPasswordConfig; -import io.supertokens.pluginInterface.multitenancy.PasswordlessConfig; -import io.supertokens.pluginInterface.multitenancy.TenantConfig; -import io.supertokens.pluginInterface.multitenancy.TenantIdentifier; -import io.supertokens.pluginInterface.multitenancy.ThirdPartyConfig; -import io.supertokens.pluginInterface.multitenancy.exceptions.TenantOrAppNotFoundException; import io.supertokens.storageLayer.StorageLayer; import io.supertokens.test.TestingProcessManager; import io.supertokens.test.Utils; +import io.supertokens.test.bulkimport.BulkImportTestUtils; import io.supertokens.test.httpRequest.HttpRequestForTesting; -import io.supertokens.thirdparty.InvalidProviderConfigException; import io.supertokens.userroles.UserRoles; public class AddBulkImportUsersTest { @@ -431,7 +418,7 @@ public void shouldThrow400IfInvalidTenantIdIsPassed() throws Exception { // CASE 3: Two or more tenants do not share the same storage - createTenants(main); + BulkImportTestUtils.createTenants(main); JsonObject requestBody3 = new JsonParser().parse( "{\"users\":[{\"loginMethods\":[{\"tenantIds\":[\"public\"],\"recipeId\":\"passwordless\",\"email\":\"johndoe@gmail.com\"}, {\"tenantIds\":[\"t2\"],\"recipeId\":\"thirdparty\", \"email\":\"johndoe@gmail.com\", \"thirdPartyId\":\"id\", \"thirdPartyUserId\":\"id\"}]}]}") @@ -698,45 +685,6 @@ private static JsonObject createPasswordlessLoginMethod(String email, JsonArray return loginMethod; } - private void createTenants(Main main) - throws StorageQueryException, TenantOrAppNotFoundException, InvalidProviderConfigException, - FeatureNotEnabledException, IOException, InvalidConfigException, - CannotModifyBaseConfigException, BadPermissionException { - // User pool 1 - (null, null, null), (null, null, t1) - // User pool 2 - (null, null, t2) - - { // tenant 1 - TenantIdentifier tenantIdentifier = new TenantIdentifier(null, null, "t1"); - - Multitenancy.addNewOrUpdateAppOrTenant( - main, - new TenantIdentifier(null, null, null), - new TenantConfig( - tenantIdentifier, - new EmailPasswordConfig(true), - new ThirdPartyConfig(true, null), - new PasswordlessConfig(true), - null, null, new JsonObject())); - } - { // tenant 2 - JsonObject config = new JsonObject(); - TenantIdentifier tenantIdentifier = new TenantIdentifier(null, null, "t2"); - - StorageLayer.getStorage(new TenantIdentifier(null, null, null), main) - .modifyConfigToAddANewUserPoolForTesting(config, 1); - - Multitenancy.addNewOrUpdateAppOrTenant( - main, - new TenantIdentifier(null, null, null), - new TenantConfig( - tenantIdentifier, - new EmailPasswordConfig(true), - new ThirdPartyConfig(true, null), - new PasswordlessConfig(true), - null, null, config)); - } - } - private void setFeatureFlags(Main main, EE_FEATURES[] features) { FeatureFlagTestContent.getInstance(main).setKeyValue(FeatureFlagTestContent.ENABLED_FEATURES, features); } diff --git a/src/test/java/io/supertokens/test/bulkimport/apis/ImportUserTest.java b/src/test/java/io/supertokens/test/bulkimport/apis/ImportUserTest.java new file mode 100644 index 000000000..e1d5206a3 --- /dev/null +++ b/src/test/java/io/supertokens/test/bulkimport/apis/ImportUserTest.java @@ -0,0 +1,151 @@ +/* + * Copyright (c) 2024, VRAI Labs and/or its affiliates. All rights reserved. + * + * This software is licensed under the Apache License, Version 2.0 (the + * "License") as published by the Apache Software Foundation. + * + * You may not use this file except in compliance with the License. You may + * obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package io.supertokens.test.bulkimport.apis; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.fail; + +import java.util.List; + +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TestRule; + +import com.google.gson.JsonObject; + +import io.supertokens.Main; +import io.supertokens.ProcessState; +import io.supertokens.featureflag.EE_FEATURES; +import io.supertokens.featureflag.FeatureFlagTestContent; +import io.supertokens.pluginInterface.STORAGE_TYPE; +import io.supertokens.pluginInterface.bulkimport.BulkImportUser; +import io.supertokens.storageLayer.StorageLayer; +import io.supertokens.test.TestingProcessManager; +import io.supertokens.test.Utils; +import io.supertokens.test.bulkimport.BulkImportTestUtils; +import io.supertokens.test.httpRequest.HttpRequestForTesting; +import io.supertokens.userroles.UserRoles; + +public class ImportUserTest { + @Rule + public TestRule watchman = Utils.getOnFailure(); + + @AfterClass + public static void afterTesting() { + Utils.afterTesting(); + } + + @Before + public void beforeEach() { + Utils.reset(); + } + + @Test + public void shouldReturn400Error() throws Exception { + String[] args = { "../" }; + + TestingProcessManager.TestingProcess process = TestingProcessManager.start(args); + assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); + Main main = process.getProcess(); + + if (StorageLayer.getStorage(main).getType() != STORAGE_TYPE.SQL || StorageLayer.isInMemDb(main)) { + return; + } + + { + try { + JsonObject request = new JsonObject(); + HttpRequestForTesting.sendJsonPOSTRequest(main, "", + "http://localhost:3567/bulk-import/import", + request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); + fail("The API should have thrown an error"); + } catch (io.supertokens.test.httpRequest.HttpResponseException e) { + assertEquals(400, e.statusCode); + assertEquals("Http error. Status Code: 400. Message: Field name 'user' is invalid in JSON input", + e.getMessage()); + } + } + + { + FeatureFlagTestContent.getInstance(main).setKeyValue(FeatureFlagTestContent.ENABLED_FEATURES, + new EE_FEATURES[] { EE_FEATURES.MULTI_TENANCY, EE_FEATURES.MFA, EE_FEATURES.ACCOUNT_LINKING }); + + try { + JsonObject request = new JsonObject(); + List users = BulkImportTestUtils.generateBulkImportUser(1); + request.add("user", users.get(0).toJsonObject()); + + HttpRequestForTesting.sendJsonPOSTRequest(main, "", + "http://localhost:3567/bulk-import/import", + request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); + fail("The API should have thrown an error"); + } catch (io.supertokens.test.httpRequest.HttpResponseException e) { + assertEquals(400, e.statusCode); + assertEquals( + "Http error. Status Code: 400. Message: {\"errors\":[\"Role role1 does not exist.\",\"Invalid tenantId: t1 for a user role.\",\"Role role2 does not exist.\",\"Invalid tenantId: t1 for a user role.\",\"Invalid tenantId: t1 for emailpassword recipe.\",\"Invalid tenantId: t1 for thirdparty recipe.\",\"Invalid tenantId: t1 for passwordless recipe.\"]}", + e.getMessage()); + } + } + + process.kill(); + Assert.assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STOPPED)); + } + + @Test + public void shouldReturn200Response() throws Exception { + String[] args = { "../" }; + + TestingProcessManager.TestingProcess process = TestingProcessManager.start(args); + assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); + Main main = process.getProcess(); + + if (StorageLayer.getStorage(main).getType() != STORAGE_TYPE.SQL || StorageLayer.isInMemDb(main)) { + return; + } + + FeatureFlagTestContent.getInstance(main).setKeyValue(FeatureFlagTestContent.ENABLED_FEATURES, + new EE_FEATURES[] { EE_FEATURES.MULTI_TENANCY, EE_FEATURES.MFA, EE_FEATURES.ACCOUNT_LINKING }); + + // Create tenants + BulkImportTestUtils.createTenants(main); + + // Create user roles + { + UserRoles.createNewRoleOrModifyItsPermissions(main, "role1", null); + UserRoles.createNewRoleOrModifyItsPermissions(main, "role2", null); + } + + JsonObject request = new JsonObject(); + List users = BulkImportTestUtils.generateBulkImportUser(1); + request.add("user", users.get(0).toJsonObject()); + + JsonObject response = HttpRequestForTesting.sendJsonPOSTRequest(main, "", + "http://localhost:3567/bulk-import/import", + request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); + + assertEquals("OK", response.get("status").getAsString()); + assertNotNull(response.get("user")); + + process.kill(); + Assert.assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STOPPED)); + } + +} From 119ca1dd582e968d86605c412ce957b757c51c2a Mon Sep 17 00:00:00 2001 From: Ankit Tiwari Date: Thu, 23 May 2024 17:35:24 +0530 Subject: [PATCH 18/21] feat: Add an api to get count of bulk import users --- .../io/supertokens/bulkimport/BulkImport.java | 23 ++- .../bulkimport/ProcessBulkImportUsers.java | 76 ++++----- .../io/supertokens/webserver/Webserver.java | 2 + .../bulkimport/CountBulkImportUsersAPI.java | 86 ++++++++++ .../test/bulkimport/BulkImportTest.java | 66 ++++++++ .../apis/CountBulkImportUsersTest.java | 148 ++++++++++++++++++ 6 files changed, 361 insertions(+), 40 deletions(-) create mode 100644 src/main/java/io/supertokens/webserver/api/bulkimport/CountBulkImportUsersAPI.java create mode 100644 src/test/java/io/supertokens/test/bulkimport/apis/CountBulkImportUsersTest.java diff --git a/src/main/java/io/supertokens/bulkimport/BulkImport.java b/src/main/java/io/supertokens/bulkimport/BulkImport.java index ef6d5d18d..8fa6c9e38 100644 --- a/src/main/java/io/supertokens/bulkimport/BulkImport.java +++ b/src/main/java/io/supertokens/bulkimport/BulkImport.java @@ -148,6 +148,12 @@ public static List deleteUsers(AppIdentifier appIdentifier, Storage stor return StorageUtils.getBulkImportStorage(storage).deleteBulkImportUsers(appIdentifier, userIds); } + public static long getBulkImportUsersCount(AppIdentifier appIdentifier, Storage storage, + @Nullable BULK_IMPORT_USER_STATUS status) + throws StorageQueryException { + return StorageUtils.getBulkImportStorage(storage).getBulkImportUsersCount(appIdentifier, status); + } + public static synchronized AuthRecipeUserInfo importUser(Main main, AppIdentifier appIdentifier, BulkImportUser user) throws StorageQueryException, InvalidConfigException, IOException, TenantOrAppNotFoundException, @@ -180,8 +186,10 @@ public static synchronized AuthRecipeUserInfo importUser(Main main, AppIdentifie bulkImportProxyStorage.commitTransactionForBulkImportProxyStorage(); - AuthRecipeUserInfo importedUser = AuthRecipe.getUserById(appIdentifier, bulkImportProxyStorage, primaryLM.superTokensUserId); - io.supertokens.useridmapping.UserIdMapping.populateExternalUserIdForUsers(appIdentifier, bulkImportProxyStorage, new AuthRecipeUserInfo[]{importedUser}); + AuthRecipeUserInfo importedUser = AuthRecipe.getUserById(appIdentifier, bulkImportProxyStorage, + primaryLM.superTokensUserId); + io.supertokens.useridmapping.UserIdMapping.populateExternalUserIdForUsers(appIdentifier, + bulkImportProxyStorage, new AuthRecipeUserInfo[] { importedUser }); return importedUser; } catch (StorageTransactionLogicException e) { @@ -257,7 +265,11 @@ private static void processPasswordlessLoginMethod(TenantIdentifier tenantIdenti lm.phoneNumber, lm.timeJoinedInMSSinceEpoch); lm.superTokensUserId = userInfo.getSupertokensUserId(); - } catch (StorageQueryException | TenantOrAppNotFoundException | RestartFlowException e) { + } catch (RestartFlowException e) { + String errorMessage = lm.email != null ? "A user with email " + lm.email + " already exists." + : "A user with phoneNumber " + lm.phoneNumber + " already exists."; + throw new StorageTransactionLogicException(new Exception(errorMessage)); + } catch (StorageQueryException | TenantOrAppNotFoundException e) { throw new StorageTransactionLogicException(e); } } @@ -485,13 +497,14 @@ private static Storage[] getAllProxyStoragesForApp(Main main, AppIdentifier appI try { List allProxyStorages = new ArrayList<>(); - + TenantConfig[] tenantConfigs = Multitenancy.getAllTenantsForApp(appIdentifier, main); for (TenantConfig tenantConfig : tenantConfigs) { allProxyStorages.add(getBulkImportProxyStorage(main, tenantConfig.tenantIdentifier)); } return allProxyStorages.toArray(new Storage[0]); - } catch (TenantOrAppNotFoundException | InvalidConfigException | IOException | DbInitException | StorageQueryException e) { + } catch (TenantOrAppNotFoundException | InvalidConfigException | IOException | DbInitException + | StorageQueryException e) { throw new StorageTransactionLogicException(e); } } diff --git a/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkImportUsers.java b/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkImportUsers.java index 7e1ef2fdf..1bdf4ebb0 100644 --- a/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkImportUsers.java +++ b/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkImportUsers.java @@ -70,8 +70,7 @@ public static ProcessBulkImportUsers init(Main main, List @Override protected void doTaskPerApp(AppIdentifier app) - throws TenantOrAppNotFoundException, StorageQueryException, InvalidConfigException, IOException, - DbInitException { + throws TenantOrAppNotFoundException, StorageQueryException, IOException, DbInitException { if (StorageLayer.getBaseStorage(main).getType() != STORAGE_TYPE.SQL || StorageLayer.isInMemDb(main)) { return; @@ -114,7 +113,8 @@ public int getInitialWaitTimeSeconds() { } private synchronized Storage getBulkImportProxyStorage(TenantIdentifier tenantIdentifier) - throws InvalidConfigException, IOException, TenantOrAppNotFoundException, DbInitException, StorageQueryException { + throws InvalidConfigException, IOException, TenantOrAppNotFoundException, DbInitException, + StorageQueryException { String userPoolId = StorageLayer.getStorage(tenantIdentifier, main).getUserPoolId(); if (userPoolToStorageMap.containsKey(userPoolId)) { return userPoolToStorageMap.get(userPoolId); @@ -151,13 +151,13 @@ private Storage[] getAllProxyStoragesForApp(Main main, AppIdentifier appIdentifi try { List allProxyStorages = new ArrayList<>(); - TenantConfig[] tenantConfigs = Multitenancy.getAllTenantsForApp(appIdentifier, main); for (TenantConfig tenantConfig : tenantConfigs) { allProxyStorages.add(getBulkImportProxyStorage(tenantConfig.tenantIdentifier)); } return allProxyStorages.toArray(new Storage[0]); - } catch (TenantOrAppNotFoundException | InvalidConfigException | IOException | DbInitException | StorageQueryException e) { + } catch (TenantOrAppNotFoundException | InvalidConfigException | IOException | DbInitException + | StorageQueryException e) { throw new StorageTransactionLogicException(e); } } @@ -172,7 +172,7 @@ private void closeAllProxyStorages() throws StorageQueryException { private void processUser(AppIdentifier appIdentifier, BulkImportUser user, BulkImportUserUtils bulkImportUserUtils, BulkImportSQLStorage baseTenantStorage) - throws TenantOrAppNotFoundException, StorageQueryException, InvalidConfigException, IOException, + throws TenantOrAppNotFoundException, StorageQueryException, IOException, DbInitException { try { @@ -193,34 +193,36 @@ private void processUser(AppIdentifier appIdentifier, BulkImportUser user, BulkI LoginMethod primaryLM = BulkImport.getPrimaryLoginMethod(user); - AuthRecipeSQLStorage authRecipeSQLStorage = (AuthRecipeSQLStorage) getBulkImportProxyStorage(firstTenantIdentifier); + AuthRecipeSQLStorage authRecipeSQLStorage = (AuthRecipeSQLStorage) getBulkImportProxyStorage( + firstTenantIdentifier); /* - * We use two separate storage instances: one for importing the user and another for managing bulk_import_users entries. - * This is necessary because the bulk_import_users entries are always in the public tenant storage, - * but the actual user data could be in a different storage. - * - * If transactions are committed individually, in this order: - * 1. Commit the transaction that imports the user. - * 2. Commit the transaction that deletes the corresponding bulk import entry. - * - * There's a risk where the first commit succeeds, but the second fails. This creates a situation where - * the bulk import entry is re-processed, even though the user has already been imported into the database. - * - * To resolve this, we added a `primaryUserId` field to the `bulk_import_users` table. - * The processing logic now follows these steps: - * - * 1. Import the user and get the `primaryUserId` (transaction uncommitted). - * 2. Update the `primaryUserId` in the corresponding bulk import entry. - * 3. Commit the import transaction from step 1. - * 4. Delete the bulk import entry. - * - * If step 2 or any earlier step fails, nothing is committed, preventing partial state. - * If step 3 fails, the `primaryUserId` in the bulk import entry is updated, but the user doesn't exist in the database—this results in re-processing on the next run. - * If step 4 fails, the user exists but the bulk import entry remains; this will be handled by deleting it in the next run. - * - * The following code implements this logic. - */ + * We use two separate storage instances: one for importing the user and another for managing bulk_import_users entries. + * This is necessary because the bulk_import_users entries are always in the public tenant storage, + * but the actual user data could be in a different storage. + * + * If transactions are committed individually, in this order: + * 1. Commit the transaction that imports the user. + * 2. Commit the transaction that deletes the corresponding bulk import entry. + * + * There's a risk where the first commit succeeds, but the second fails. This creates a situation where + * the bulk import entry is re-processed, even though the user has already been imported into the database. + * + * To resolve this, we added a `primaryUserId` field to the `bulk_import_users` table. + * The processing logic now follows these steps: + * + * 1. Import the user and get the `primaryUserId` (transaction uncommitted). + * 2. Update the `primaryUserId` in the corresponding bulk import entry. + * 3. Commit the import transaction from step 1. + * 4. Delete the bulk import entry. + * + * If step 2 or any earlier step fails, nothing is committed, preventing partial state. + * If step 3 fails, the `primaryUserId` in the bulk import entry is updated, but the user doesn't exist in the database—this results in re-processing on the + * next run. + * If step 4 fails, the user exists but the bulk import entry remains; this will be handled by deleting it in the next run. + * + * The following code implements this logic. + */ if (user.primaryUserId != null) { AuthRecipeUserInfo importedUser = authRecipeSQLStorage.getPrimaryUserById(appIdentifier, user.primaryUserId); @@ -237,12 +239,14 @@ private void processUser(AppIdentifier appIdentifier, BulkImportUser user, BulkI BulkImport.processUserLoginMethod(main, appIdentifier, bulkImportProxyStorage, lm); } - BulkImport.createPrimaryUserAndLinkAccounts(main, appIdentifier, bulkImportProxyStorage, user, primaryLM); + BulkImport.createPrimaryUserAndLinkAccounts(main, appIdentifier, bulkImportProxyStorage, user, + primaryLM); Storage[] allStoragesForApp = getAllProxyStoragesForApp(main, appIdentifier); BulkImport.createUserIdMapping(appIdentifier, user, primaryLM, allStoragesForApp); - BulkImport.verifyEmailForAllLoginMethods(appIdentifier, con, bulkImportProxyStorage, user.loginMethods); + BulkImport.verifyEmailForAllLoginMethods(appIdentifier, con, bulkImportProxyStorage, + user.loginMethods); BulkImport.createTotpDevices(main, appIdentifier, bulkImportProxyStorage, user, primaryLM); BulkImport.createUserMetadata(appIdentifier, bulkImportProxyStorage, user, primaryLM); BulkImport.createUserRoles(main, appIdentifier, bulkImportProxyStorage, user); @@ -271,7 +275,7 @@ private void processUser(AppIdentifier appIdentifier, BulkImportUser user, BulkI closeAllProxyStorages(); } }); - } catch (StorageTransactionLogicException | InvalidBulkImportDataException e) { + } catch (StorageTransactionLogicException | InvalidBulkImportDataException | InvalidConfigException e) { handleProcessUserExceptions(appIdentifier, user, e, baseTenantStorage); } } @@ -289,6 +293,8 @@ private void handleProcessUserExceptions(AppIdentifier appIdentifier, BulkImport errorMessage[0] = exception.actualException.getMessage(); } else if (e instanceof InvalidBulkImportDataException) { errorMessage[0] = ((InvalidBulkImportDataException) e).errors.toString(); + } else if (e instanceof InvalidConfigException) { + errorMessage[0] = e.getMessage(); } try { diff --git a/src/main/java/io/supertokens/webserver/Webserver.java b/src/main/java/io/supertokens/webserver/Webserver.java index 23e953ef9..47dfeb682 100644 --- a/src/main/java/io/supertokens/webserver/Webserver.java +++ b/src/main/java/io/supertokens/webserver/Webserver.java @@ -27,6 +27,7 @@ import io.supertokens.pluginInterface.multitenancy.exceptions.TenantOrAppNotFoundException; import io.supertokens.webserver.api.accountlinking.*; import io.supertokens.webserver.api.bulkimport.BulkImportAPI; +import io.supertokens.webserver.api.bulkimport.CountBulkImportUsersAPI; import io.supertokens.webserver.api.bulkimport.DeleteBulkImportUserAPI; import io.supertokens.webserver.api.bulkimport.ImportUserAPI; import io.supertokens.webserver.api.core.*; @@ -266,6 +267,7 @@ private void setupRoutes() { addAPI(new BulkImportAPI(main)); addAPI(new DeleteBulkImportUserAPI(main)); addAPI(new ImportUserAPI(main)); + addAPI(new CountBulkImportUsersAPI(main)); StandardContext context = tomcatReference.getContext(); Tomcat tomcat = tomcatReference.getTomcat(); diff --git a/src/main/java/io/supertokens/webserver/api/bulkimport/CountBulkImportUsersAPI.java b/src/main/java/io/supertokens/webserver/api/bulkimport/CountBulkImportUsersAPI.java new file mode 100644 index 000000000..9edb8be57 --- /dev/null +++ b/src/main/java/io/supertokens/webserver/api/bulkimport/CountBulkImportUsersAPI.java @@ -0,0 +1,86 @@ +/* + * Copyright (c) 2024, VRAI Labs and/or its affiliates. All rights reserved. + * + * This software is licensed under the Apache License, Version 2.0 (the + * "License") as published by the Apache Software Foundation. + * + * You may not use this file except in compliance with the License. You may + * obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package io.supertokens.webserver.api.bulkimport; + +import java.io.IOException; + +import com.google.gson.JsonObject; + +import io.supertokens.Main; +import io.supertokens.bulkimport.BulkImport; +import io.supertokens.multitenancy.exception.BadPermissionException; +import io.supertokens.pluginInterface.Storage; +import io.supertokens.pluginInterface.bulkimport.BulkImportStorage.BULK_IMPORT_USER_STATUS; +import io.supertokens.pluginInterface.exceptions.StorageQueryException; +import io.supertokens.pluginInterface.multitenancy.AppIdentifier; +import io.supertokens.pluginInterface.multitenancy.exceptions.TenantOrAppNotFoundException; +import io.supertokens.storageLayer.StorageLayer; +import io.supertokens.webserver.InputParser; +import io.supertokens.webserver.WebserverAPI; +import jakarta.servlet.ServletException; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; + +public class CountBulkImportUsersAPI extends WebserverAPI { + public CountBulkImportUsersAPI(Main main) { + super(main, ""); + } + + @Override + public String getPath() { + return "/bulk-import/users/count"; + } + + @Override + protected void doGet(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException { + // API is app specific + + if (StorageLayer.isInMemDb(main)) { + throw new ServletException(new BadRequestException("This API is not supported in the in-memory database.")); + } + + String statusString = InputParser.getQueryParamOrThrowError(req, "status", true); + + BULK_IMPORT_USER_STATUS status = null; + if (statusString != null) { + try { + status = BULK_IMPORT_USER_STATUS.valueOf(statusString); + } catch (IllegalArgumentException e) { + throw new ServletException( + new BadRequestException("Invalid value for status. Pass one of NEW, PROCESSING, or FAILED!")); + } + } + + AppIdentifier appIdentifier = null; + Storage storage = null; + + try { + appIdentifier = getAppIdentifier(req); + storage = enforcePublicTenantAndGetPublicTenantStorage(req); + + long count = BulkImport.getBulkImportUsersCount(appIdentifier, storage, status); + + JsonObject result = new JsonObject(); + result.addProperty("status", "OK"); + result.addProperty("count", count); + super.sendJsonResponse(200, result, resp); + + } catch (TenantOrAppNotFoundException | BadPermissionException | StorageQueryException e) { + throw new ServletException(e); + } + } +} diff --git a/src/test/java/io/supertokens/test/bulkimport/BulkImportTest.java b/src/test/java/io/supertokens/test/bulkimport/BulkImportTest.java index a673b7682..ac298d4ce 100644 --- a/src/test/java/io/supertokens/test/bulkimport/BulkImportTest.java +++ b/src/test/java/io/supertokens/test/bulkimport/BulkImportTest.java @@ -296,6 +296,72 @@ public void randomPaginationTest() throws Exception { assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STOPPED)); } + @Test + public void testGetBulkImportUsersCount() throws Exception { + String[] args = { "../" }; + + TestingProcessManager.TestingProcess process = TestingProcessManager.start(args); + assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); + Main main = process.getProcess(); + + if (StorageLayer.getStorage(main).getType() != STORAGE_TYPE.SQL || StorageLayer.isInMemDb(main)) { + return; + } + + BulkImportSQLStorage storage = (BulkImportSQLStorage) StorageLayer.getStorage(process.main); + AppIdentifier appIdentifier = new AppIdentifier(null, null); + + // Test with status = 'NEW' + { + List users = generateBulkImportUser(10); + BulkImport.addUsers(appIdentifier, storage, users); + + long count = BulkImport.getBulkImportUsersCount(appIdentifier, storage, BULK_IMPORT_USER_STATUS.NEW); + assertEquals(10, count); + } + + // Test with status = 'PROCESSING' + { + List users = generateBulkImportUser(10); + BulkImport.addUsers(appIdentifier, storage, users); + + // Update the users status to PROCESSING + storage.startTransaction(con -> { + for (BulkImportUser user : users) { + storage.updateBulkImportUserStatus_Transaction(appIdentifier, con, user.id, + BULK_IMPORT_USER_STATUS.PROCESSING, null); + } + storage.commitTransaction(con); + return null; + }); + + long count = BulkImport.getBulkImportUsersCount(appIdentifier, storage, BULK_IMPORT_USER_STATUS.PROCESSING); + assertEquals(10, count); + } + + // Test with status = 'FAILED' + { + List users = generateBulkImportUser(10); + BulkImport.addUsers(appIdentifier, storage, users); + + // Update the users status to FAILED + storage.startTransaction(con -> { + for (BulkImportUser user : users) { + storage.updateBulkImportUserStatus_Transaction(appIdentifier, con, user.id, + BULK_IMPORT_USER_STATUS.FAILED, null); + } + storage.commitTransaction(con); + return null; + }); + + long count = BulkImport.getBulkImportUsersCount(appIdentifier, storage, BULK_IMPORT_USER_STATUS.FAILED); + assertEquals(10, count); + } + + process.kill(); + assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STOPPED)); + } + @Test public void shouldImportTheUserInTheSameTenant() throws Exception { String[] args = { "../" }; diff --git a/src/test/java/io/supertokens/test/bulkimport/apis/CountBulkImportUsersTest.java b/src/test/java/io/supertokens/test/bulkimport/apis/CountBulkImportUsersTest.java new file mode 100644 index 000000000..6d1e14a61 --- /dev/null +++ b/src/test/java/io/supertokens/test/bulkimport/apis/CountBulkImportUsersTest.java @@ -0,0 +1,148 @@ +/* + * Copyright (c) 2024, VRAI Labs and/or its affiliates. All rights reserved. + * + * This software is licensed under the Apache License, Version 2.0 (the + * "License") as published by the Apache Software Foundation. + * + * You may not use this file except in compliance with the License. You may + * obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package io.supertokens.test.bulkimport.apis; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import java.util.HashMap; +import java.util.Map; + +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TestRule; + +import com.google.gson.JsonObject; + +import io.supertokens.Main; +import io.supertokens.ProcessState; +import io.supertokens.pluginInterface.STORAGE_TYPE; +import io.supertokens.storageLayer.StorageLayer; +import io.supertokens.test.TestingProcessManager; +import io.supertokens.test.Utils; +import io.supertokens.test.httpRequest.HttpRequestForTesting; + + +public class CountBulkImportUsersTest { + @Rule + public TestRule watchman = Utils.getOnFailure(); + + @AfterClass + public static void afterTesting() { + Utils.afterTesting(); + } + + @Before + public void beforeEach() { + Utils.reset(); + } + + @Test + public void shouldReturn400Error() throws Exception { + String[] args = { "../" }; + + TestingProcessManager.TestingProcess process = TestingProcessManager.start(args); + assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); + Main main = process.getProcess(); + + if (StorageLayer.getStorage(main).getType() != STORAGE_TYPE.SQL || StorageLayer.isInMemDb(main)) { + return; + } + + try { + Map params = new HashMap<>(); + params.put("status", "INVALID_STATUS"); + HttpRequestForTesting.sendGETRequest(main, "", + "http://localhost:3567/bulk-import/users/count", + params, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); + fail("The API should have thrown an error"); + } catch (io.supertokens.test.httpRequest.HttpResponseException e) { + assertEquals(400, e.statusCode); + assertEquals( + "Http error. Status Code: 400. Message: Invalid value for status. Pass one of NEW, PROCESSING, or FAILED!", + e.getMessage()); + } + + process.kill(); + Assert.assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STOPPED)); + } + + @Test + public void shouldReturn200Response() throws Exception { + String[] args = { "../" }; + + TestingProcessManager.TestingProcess process = TestingProcessManager.start(args); + assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); + Main main = process.getProcess(); + + if (StorageLayer.getStorage(main).getType() != STORAGE_TYPE.SQL || StorageLayer.isInMemDb(main)) { + return; + } + + { + Map params = new HashMap<>(); + JsonObject response = HttpRequestForTesting.sendGETRequest(main, "", + "http://localhost:3567/bulk-import/users/count", + params, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); + + assertEquals("OK", response.get("status").getAsString()); + assertEquals(0, response.get("count").getAsLong()); + } + + { + Map params = new HashMap<>(); + params.put("status", "NEW"); + JsonObject response = HttpRequestForTesting.sendGETRequest(main, "", + "http://localhost:3567/bulk-import/users/count", + params, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); + + assertEquals("OK", response.get("status").getAsString()); + assertEquals(0, response.get("count").getAsLong()); + } + + { + Map params = new HashMap<>(); + params.put("status", "PROCESSING"); + JsonObject response = HttpRequestForTesting.sendGETRequest(main, "", + "http://localhost:3567/bulk-import/users/count", + params, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); + + assertEquals("OK", response.get("status").getAsString()); + assertEquals(0, response.get("count").getAsLong()); + } + + { + Map params = new HashMap<>(); + params.put("status", "FAILED"); + JsonObject response = HttpRequestForTesting.sendGETRequest(main, "", + "http://localhost:3567/bulk-import/users/count", + params, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); + + assertEquals("OK", response.get("status").getAsString()); + assertEquals(0, response.get("count").getAsLong()); + } + + process.kill(); + Assert.assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STOPPED)); + } + +} From 79a1e67ce9a152462500b08ade467aae709605c4 Mon Sep 17 00:00:00 2001 From: Ankit Tiwari Date: Mon, 27 May 2024 17:53:59 +0530 Subject: [PATCH 19/21] fix: PR changes --- .../io/supertokens/bulkimport/BulkImport.java | 8 +- .../bulkimport/ProcessBulkImportUsers.java | 14 +--- .../api/bulkimport/BulkImportAPI.java | 9 ++- .../bulkimport/DeleteBulkImportUserAPI.java | 5 +- .../api/bulkimport/ImportUserAPI.java | 3 +- .../apis/AddBulkImportUsersTest.java | 6 +- .../apis/DeleteBulkImportUsersTest.java | 79 +++++++++++-------- .../test/bulkimport/apis/ImportUserTest.java | 21 +---- 8 files changed, 66 insertions(+), 79 deletions(-) diff --git a/src/main/java/io/supertokens/bulkimport/BulkImport.java b/src/main/java/io/supertokens/bulkimport/BulkImport.java index 8fa6c9e38..7173d93c0 100644 --- a/src/main/java/io/supertokens/bulkimport/BulkImport.java +++ b/src/main/java/io/supertokens/bulkimport/BulkImport.java @@ -237,7 +237,7 @@ private static void processEmailPasswordLoginMethod(TenantIdentifier tenantIdent throw new StorageTransactionLogicException(e); } catch (DuplicateEmailException e) { throw new StorageTransactionLogicException( - new Exception("A user with email " + lm.email + " already exists")); + new Exception("A user with email " + lm.email + " already exists in emailpassword loginMethod.")); } } @@ -253,7 +253,7 @@ private static void processThirdPartyLoginMethod(TenantIdentifier tenantIdentifi throw new StorageTransactionLogicException(e); } catch (DuplicateThirdPartyUserException e) { throw new StorageTransactionLogicException(new Exception("A user with thirdPartyId " + lm.thirdPartyId - + " and thirdPartyUserId " + lm.thirdPartyUserId + " already exists")); + + " and thirdPartyUserId " + lm.thirdPartyUserId + " already exists in thirdparty loginMethod.")); } } @@ -266,8 +266,8 @@ private static void processPasswordlessLoginMethod(TenantIdentifier tenantIdenti lm.superTokensUserId = userInfo.getSupertokensUserId(); } catch (RestartFlowException e) { - String errorMessage = lm.email != null ? "A user with email " + lm.email + " already exists." - : "A user with phoneNumber " + lm.phoneNumber + " already exists."; + String errorMessage = lm.email != null ? "A user with email " + lm.email + " already exists in passwordless loginMethod." + : "A user with phoneNumber " + lm.phoneNumber + " already exists in passwordless loginMethod."; throw new StorageTransactionLogicException(new Exception(errorMessage)); } catch (StorageQueryException | TenantOrAppNotFoundException e) { throw new StorageTransactionLogicException(e); diff --git a/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkImportUsers.java b/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkImportUsers.java index 1bdf4ebb0..be3dfb744 100644 --- a/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkImportUsers.java +++ b/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkImportUsers.java @@ -113,8 +113,7 @@ public int getInitialWaitTimeSeconds() { } private synchronized Storage getBulkImportProxyStorage(TenantIdentifier tenantIdentifier) - throws InvalidConfigException, IOException, TenantOrAppNotFoundException, DbInitException, - StorageQueryException { + throws InvalidConfigException, IOException, TenantOrAppNotFoundException, DbInitException { String userPoolId = StorageLayer.getStorage(tenantIdentifier, main).getUserPoolId(); if (userPoolToStorageMap.containsKey(userPoolId)) { return userPoolToStorageMap.get(userPoolId); @@ -133,13 +132,6 @@ private synchronized Storage getBulkImportProxyStorage(TenantIdentifier tenantId userPoolToStorageMap.put(userPoolId, bulkImportProxyStorage); bulkImportProxyStorage.initStorage(false, new ArrayList<>()); - // `BulkImportProxyStorage` uses `BulkImportProxyConnection`, which overrides the `.commit()` method on the Connection object. - // The `initStorage()` method runs `select * from table_name limit 1` queries to check if the tables exist but these queries - // don't get committed due to the overridden `.commit()`, so we need to manually commit the transaction to remove any locks on the tables. - - // Without this commit, a call to `select * from bulk_import_users limit 1` in `doesTableExist()` locks the `bulk_import_users` table, - // causing other queries to stall indefinitely. - bulkImportProxyStorage.commitTransactionForBulkImportProxyStorage(); return bulkImportProxyStorage; } } @@ -156,8 +148,7 @@ private Storage[] getAllProxyStoragesForApp(Main main, AppIdentifier appIdentifi allProxyStorages.add(getBulkImportProxyStorage(tenantConfig.tenantIdentifier)); } return allProxyStorages.toArray(new Storage[0]); - } catch (TenantOrAppNotFoundException | InvalidConfigException | IOException | DbInitException - | StorageQueryException e) { + } catch (TenantOrAppNotFoundException | InvalidConfigException | IOException | DbInitException e) { throw new StorageTransactionLogicException(e); } } @@ -165,7 +156,6 @@ private Storage[] getAllProxyStoragesForApp(Main main, AppIdentifier appIdentifi private void closeAllProxyStorages() throws StorageQueryException { for (SQLStorage storage : userPoolToStorageMap.values()) { storage.closeConnectionForBulkImportProxyStorage(); - storage.close(); } userPoolToStorageMap.clear(); } diff --git a/src/main/java/io/supertokens/webserver/api/bulkimport/BulkImportAPI.java b/src/main/java/io/supertokens/webserver/api/bulkimport/BulkImportAPI.java index c30f63f82..6e8d0157f 100644 --- a/src/main/java/io/supertokens/webserver/api/bulkimport/BulkImportAPI.java +++ b/src/main/java/io/supertokens/webserver/api/bulkimport/BulkImportAPI.java @@ -132,7 +132,14 @@ protected void doPost(HttpServletRequest req, HttpServletResponse resp) throws S JsonObject input = InputParser.parseJsonObjectOrThrowError(req); JsonArray users = InputParser.parseArrayOrThrowError(input, "users", false); - if (users.size() <= 0 || users.size() > BulkImport.MAX_USERS_TO_ADD) { + if (users.size() == 0) { + JsonObject result = new JsonObject(); + result.addProperty("status", "OK"); + super.sendJsonResponse(200, result, resp); + return; + } + + if (users.size() > BulkImport.MAX_USERS_TO_ADD) { JsonObject errorResponseJson = new JsonObject(); String errorMsg = users.size() <= 0 ? "You need to add at least one user." : "You can only add " + BulkImport.MAX_USERS_TO_ADD + " users at a time."; diff --git a/src/main/java/io/supertokens/webserver/api/bulkimport/DeleteBulkImportUserAPI.java b/src/main/java/io/supertokens/webserver/api/bulkimport/DeleteBulkImportUserAPI.java index 5562fafca..0dd604fcd 100644 --- a/src/main/java/io/supertokens/webserver/api/bulkimport/DeleteBulkImportUserAPI.java +++ b/src/main/java/io/supertokens/webserver/api/bulkimport/DeleteBulkImportUserAPI.java @@ -59,7 +59,10 @@ protected void doPost(HttpServletRequest req, HttpServletResponse resp) throws S JsonArray arr = InputParser.parseArrayOrThrowError(input, "ids", false); if (arr.size() == 0) { - throw new ServletException(new WebserverAPI.BadRequestException("Field name 'ids' cannot be an empty array")); + JsonObject result = new JsonObject(); + result.add("deletedIds", new JsonArray()); + result.add("invalidIds", new JsonArray()); + super.sendJsonResponse(200, result, resp); } if (arr.size() > BulkImport.DELETE_USERS_MAX_LIMIT) { diff --git a/src/main/java/io/supertokens/webserver/api/bulkimport/ImportUserAPI.java b/src/main/java/io/supertokens/webserver/api/bulkimport/ImportUserAPI.java index ae29fdd63..599801680 100644 --- a/src/main/java/io/supertokens/webserver/api/bulkimport/ImportUserAPI.java +++ b/src/main/java/io/supertokens/webserver/api/bulkimport/ImportUserAPI.java @@ -61,8 +61,7 @@ protected void doPost(HttpServletRequest req, HttpServletResponse resp) throws S throw new ServletException(new BadRequestException("This API is not supported in the in-memory database.")); } - JsonObject input = InputParser.parseJsonObjectOrThrowError(req); - JsonObject jsonUser = InputParser.parseJsonObjectOrThrowError(input, "user", false); + JsonObject jsonUser = InputParser.parseJsonObjectOrThrowError(req); AppIdentifier appIdentifier = null; Storage storage = null; diff --git a/src/test/java/io/supertokens/test/bulkimport/apis/AddBulkImportUsersTest.java b/src/test/java/io/supertokens/test/bulkimport/apis/AddBulkImportUsersTest.java index c4ac061d8..641ba3822 100644 --- a/src/test/java/io/supertokens/test/bulkimport/apis/AddBulkImportUsersTest.java +++ b/src/test/java/io/supertokens/test/bulkimport/apis/AddBulkImportUsersTest.java @@ -85,11 +85,7 @@ public void shouldThrow400IfUsersAreMissingInRequestBody() throws Exception { testBadRequest(main, new JsonParser().parse("{\"users\": \"string\"}").getAsJsonObject(), "Field name 'users' is invalid in JSON input"); - // CASE 3: users array is empty - testBadRequest(main, generateUsersJson(0).getAsJsonObject(), - "{\"error\":\"You need to add at least one user.\"}"); - - // CASE 4: users array length is greater than 10000 + // CASE 3: users array length is greater than 10000 testBadRequest(main, generateUsersJson(10001).getAsJsonObject(), "{\"error\":\"You can only add 10000 users at a time.\"}"); diff --git a/src/test/java/io/supertokens/test/bulkimport/apis/DeleteBulkImportUsersTest.java b/src/test/java/io/supertokens/test/bulkimport/apis/DeleteBulkImportUsersTest.java index ebd9b3624..30c07597c 100644 --- a/src/test/java/io/supertokens/test/bulkimport/apis/DeleteBulkImportUsersTest.java +++ b/src/test/java/io/supertokens/test/bulkimport/apis/DeleteBulkImportUsersTest.java @@ -19,6 +19,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; import java.util.List; @@ -80,9 +81,11 @@ public void shouldReturn400Error() throws Exception { HttpRequestForTesting.sendJsonPOSTRequest(main, "", "http://localhost:3567/bulk-import/users/remove", request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); + fail("The API should have thrown an error"); } catch (io.supertokens.test.httpRequest.HttpResponseException e) { assertEquals(400, e.statusCode); - assertEquals("Http error. Status Code: 400. Message: Field name 'ids' is invalid in JSON input", e.getMessage()); + assertEquals("Http error. Status Code: 400. Message: Field name 'ids' is invalid in JSON input", + e.getMessage()); } } { @@ -91,20 +94,11 @@ public void shouldReturn400Error() throws Exception { HttpRequestForTesting.sendJsonPOSTRequest(main, "", "http://localhost:3567/bulk-import/users/remove", request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); + fail("The API should have thrown an error"); } catch (io.supertokens.test.httpRequest.HttpResponseException e) { assertEquals(400, e.statusCode); - assertEquals("Http error. Status Code: 400. Message: Field name 'ids' cannot be an empty array", e.getMessage()); - } - } - { - try { - JsonObject request = new JsonParser().parse("{\"ids\":[\"\"]}").getAsJsonObject(); - HttpRequestForTesting.sendJsonPOSTRequest(main, "", - "http://localhost:3567/bulk-import/users/remove", - request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); - } catch (io.supertokens.test.httpRequest.HttpResponseException e) { - assertEquals(400, e.statusCode); - assertEquals("Http error. Status Code: 400. Message: Field name 'ids' cannot contain an empty string", e.getMessage()); + assertEquals("Http error. Status Code: 400. Message: Field name 'ids' cannot be an empty array", + e.getMessage()); } } { @@ -122,7 +116,9 @@ public void shouldReturn400Error() throws Exception { request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); } catch (io.supertokens.test.httpRequest.HttpResponseException e) { assertEquals(400, e.statusCode); - assertEquals("Http error. Status Code: 400. Message: Field name 'ids' cannot contain more than 500 elements", e.getMessage()); + assertEquals( + "Http error. Status Code: 400. Message: Field name 'ids' cannot contain more than 500 elements", + e.getMessage()); } } @@ -142,33 +138,46 @@ public void shouldReturn200Response() throws Exception { return; } - BulkImportStorage storage = (BulkImportStorage) StorageLayer.getStorage(process.main); - AppIdentifier appIdentifier = new AppIdentifier(null, null); - - // Insert users - List users = generateBulkImportUser(5); - BulkImport.addUsers(appIdentifier, storage, users); + // Call the API with empty array + { + JsonObject request = new JsonParser().parse("{\"ids\":[\"\"]}").getAsJsonObject(); + JsonObject resonse = HttpRequestForTesting.sendJsonPOSTRequest(main, "", + "http://localhost:3567/bulk-import/users/remove", + request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); - String invalidId = io.supertokens.utils.Utils.getUUID(); - JsonObject request = new JsonObject(); - JsonArray validIds = new JsonArray(); - for (BulkImportUser user : users) { - validIds.add(new JsonPrimitive(user.id)); + assertEquals(0, resonse.get("deletedIds").getAsJsonArray().size()); + assertEquals(0, resonse.get("invalidIds").getAsJsonArray().size()); } - validIds.add(new JsonPrimitive(invalidId)); - - request.add("ids", validIds); - JsonObject response = HttpRequestForTesting.sendJsonPOSTRequest(main, "", - "http://localhost:3567/bulk-import/users/remove", - request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); + { - response.get("deletedIds").getAsJsonArray().forEach(id -> { - assertTrue(validIds.contains(id)); - }); + BulkImportStorage storage = (BulkImportStorage) StorageLayer.getStorage(process.main); + AppIdentifier appIdentifier = new AppIdentifier(null, null); - assertEquals(invalidId, response.get("invalidIds").getAsJsonArray().get(0).getAsString()); + // Insert users + List users = generateBulkImportUser(5); + BulkImport.addUsers(appIdentifier, storage, users); + + String invalidId = io.supertokens.utils.Utils.getUUID(); + JsonObject request = new JsonObject(); + JsonArray validIds = new JsonArray(); + for (BulkImportUser user : users) { + validIds.add(new JsonPrimitive(user.id)); + } + validIds.add(new JsonPrimitive(invalidId)); + request.add("ids", validIds); + + JsonObject response = HttpRequestForTesting.sendJsonPOSTRequest(main, "", + "http://localhost:3567/bulk-import/users/remove", + request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); + + response.get("deletedIds").getAsJsonArray().forEach(id -> { + assertTrue(validIds.contains(id)); + }); + + assertEquals(invalidId, response.get("invalidIds").getAsJsonArray().get(0).getAsString()); + } process.kill(); Assert.assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STOPPED)); } diff --git a/src/test/java/io/supertokens/test/bulkimport/apis/ImportUserTest.java b/src/test/java/io/supertokens/test/bulkimport/apis/ImportUserTest.java index e1d5206a3..5a1b1db85 100644 --- a/src/test/java/io/supertokens/test/bulkimport/apis/ImportUserTest.java +++ b/src/test/java/io/supertokens/test/bulkimport/apis/ImportUserTest.java @@ -70,28 +70,13 @@ public void shouldReturn400Error() throws Exception { return; } - { - try { - JsonObject request = new JsonObject(); - HttpRequestForTesting.sendJsonPOSTRequest(main, "", - "http://localhost:3567/bulk-import/import", - request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); - fail("The API should have thrown an error"); - } catch (io.supertokens.test.httpRequest.HttpResponseException e) { - assertEquals(400, e.statusCode); - assertEquals("Http error. Status Code: 400. Message: Field name 'user' is invalid in JSON input", - e.getMessage()); - } - } - { FeatureFlagTestContent.getInstance(main).setKeyValue(FeatureFlagTestContent.ENABLED_FEATURES, new EE_FEATURES[] { EE_FEATURES.MULTI_TENANCY, EE_FEATURES.MFA, EE_FEATURES.ACCOUNT_LINKING }); try { - JsonObject request = new JsonObject(); List users = BulkImportTestUtils.generateBulkImportUser(1); - request.add("user", users.get(0).toJsonObject()); + JsonObject request = users.get(0).toJsonObject(); HttpRequestForTesting.sendJsonPOSTRequest(main, "", "http://localhost:3567/bulk-import/import", @@ -132,10 +117,8 @@ public void shouldReturn200Response() throws Exception { UserRoles.createNewRoleOrModifyItsPermissions(main, "role1", null); UserRoles.createNewRoleOrModifyItsPermissions(main, "role2", null); } - - JsonObject request = new JsonObject(); List users = BulkImportTestUtils.generateBulkImportUser(1); - request.add("user", users.get(0).toJsonObject()); + JsonObject request = users.get(0).toJsonObject(); JsonObject response = HttpRequestForTesting.sendJsonPOSTRequest(main, "", "http://localhost:3567/bulk-import/import", From 77cb57c08f2016a36a9e24d568e1d0bff44a0cc9 Mon Sep 17 00:00:00 2001 From: Ankit Tiwari Date: Wed, 29 May 2024 10:38:44 +0530 Subject: [PATCH 20/21] fix: Add error codes and plainTextPassword import --- .../io/supertokens/bulkimport/BulkImport.java | 216 ++++++++++++------ .../bulkimport/BulkImportUserUtils.java | 16 +- .../bulkimport/ProcessBulkImportUsers.java | 33 ++- .../supertokens/webserver/WebserverAPI.java | 3 +- .../test/bulkimport/BulkImportTest.java | 55 ++++- .../test/bulkimport/BulkImportTestUtils.java | 35 ++- .../ProcessBulkImportUsersCronJobTest.java | 8 +- .../apis/AddBulkImportUsersTest.java | 6 +- 8 files changed, 262 insertions(+), 110 deletions(-) diff --git a/src/main/java/io/supertokens/bulkimport/BulkImport.java b/src/main/java/io/supertokens/bulkimport/BulkImport.java index 7173d93c0..cb8111a78 100644 --- a/src/main/java/io/supertokens/bulkimport/BulkImport.java +++ b/src/main/java/io/supertokens/bulkimport/BulkImport.java @@ -31,6 +31,7 @@ import io.supertokens.config.Config; import io.supertokens.emailpassword.EmailPassword; import io.supertokens.emailpassword.EmailPassword.ImportUserResponse; +import io.supertokens.emailpassword.PasswordHashing; import io.supertokens.featureflag.exceptions.FeatureNotEnabledException; import io.supertokens.multitenancy.Multitenancy; import io.supertokens.multitenancy.exception.AnotherPrimaryUserWithEmailAlreadyExistsException; @@ -81,6 +82,9 @@ import com.google.gson.JsonObject; +// Error codes ensure globally unique and identifiable errors in Bulk Import. +// Current range: E001 to E046. + public class BulkImport { // Maximum number of users that can be added in a single /bulk-import/users POST request @@ -170,19 +174,9 @@ public static synchronized AuthRecipeUserInfo importUser(Main main, AppIdentifie try { return bulkImportProxyStorage.startTransaction(con -> { try { - for (LoginMethod lm : user.loginMethods) { - processUserLoginMethod(main, appIdentifier, bulkImportProxyStorage, lm); - } - - createPrimaryUserAndLinkAccounts(main, appIdentifier, bulkImportProxyStorage, user, primaryLM); - Storage[] allStoragesForApp = getAllProxyStoragesForApp(main, appIdentifier); - createUserIdMapping(appIdentifier, user, primaryLM, allStoragesForApp); - - verifyEmailForAllLoginMethods(appIdentifier, con, bulkImportProxyStorage, user.loginMethods); - createTotpDevices(main, appIdentifier, bulkImportProxyStorage, user, primaryLM); - createUserMetadata(appIdentifier, bulkImportProxyStorage, user, primaryLM); - createUserRoles(main, appIdentifier, bulkImportProxyStorage, user); + processUserImportSteps(main, con, appIdentifier, bulkImportProxyStorage, user, primaryLM, + allStoragesForApp); bulkImportProxyStorage.commitTransactionForBulkImportProxyStorage(); @@ -205,6 +199,21 @@ public static synchronized AuthRecipeUserInfo importUser(Main main, AppIdentifie } } + public static void processUserImportSteps(Main main, TransactionConnection con, AppIdentifier appIdentifier, + Storage bulkImportProxyStorage, BulkImportUser user, LoginMethod primaryLM, Storage[] allStoragesForApp) + throws StorageTransactionLogicException { + for (LoginMethod lm : user.loginMethods) { + processUserLoginMethod(main, appIdentifier, bulkImportProxyStorage, lm); + } + + createPrimaryUserAndLinkAccounts(main, appIdentifier, bulkImportProxyStorage, user, primaryLM); + createUserIdMapping(appIdentifier, user, primaryLM, allStoragesForApp); + verifyEmailForAllLoginMethods(appIdentifier, con, bulkImportProxyStorage, user.loginMethods); + createTotpDevices(main, appIdentifier, bulkImportProxyStorage, user, primaryLM); + createUserMetadata(appIdentifier, bulkImportProxyStorage, user, primaryLM); + createUserRoles(main, appIdentifier, bulkImportProxyStorage, user); + } + public static void processUserLoginMethod(Main main, AppIdentifier appIdentifier, Storage storage, LoginMethod lm) throws StorageTransactionLogicException { String firstTenant = lm.tenantIds.get(0); @@ -213,31 +222,41 @@ public static void processUserLoginMethod(Main main, AppIdentifier appIdentifier appIdentifier.getAppId(), firstTenant); if (lm.recipeId.equals("emailpassword")) { - processEmailPasswordLoginMethod(tenantIdentifier, storage, lm); + processEmailPasswordLoginMethod(main, tenantIdentifier, storage, lm); } else if (lm.recipeId.equals("thirdparty")) { processThirdPartyLoginMethod(tenantIdentifier, storage, lm); } else if (lm.recipeId.equals("passwordless")) { processPasswordlessLoginMethod(tenantIdentifier, storage, lm); } else { throw new StorageTransactionLogicException( - new IllegalArgumentException("Unknown recipeId " + lm.recipeId + " for loginMethod ")); + new IllegalArgumentException("E001: Unknown recipeId " + lm.recipeId + " for loginMethod.")); } associateUserToTenants(main, appIdentifier, storage, lm, firstTenant); } - private static void processEmailPasswordLoginMethod(TenantIdentifier tenantIdentifier, Storage storage, + private static void processEmailPasswordLoginMethod(Main main, TenantIdentifier tenantIdentifier, Storage storage, LoginMethod lm) throws StorageTransactionLogicException { try { + + String passwordHash = lm.passwordHash; + if (passwordHash == null && lm.plainTextPassword != null) { + passwordHash = PasswordHashing.getInstance(main) + .createHashWithSalt(tenantIdentifier.toAppIdentifier(), lm.plainTextPassword); + } + ImportUserResponse userInfo = EmailPassword.createUserWithPasswordHash(tenantIdentifier, storage, lm.email, - lm.passwordHash, lm.timeJoinedInMSSinceEpoch); + passwordHash, lm.timeJoinedInMSSinceEpoch); lm.superTokensUserId = userInfo.user.getSupertokensUserId(); - } catch (StorageQueryException | TenantOrAppNotFoundException e) { + } catch (StorageQueryException e) { throw new StorageTransactionLogicException(e); + } catch (TenantOrAppNotFoundException e) { + throw new StorageTransactionLogicException(new Exception("E002: " + e.getMessage())); } catch (DuplicateEmailException e) { throw new StorageTransactionLogicException( - new Exception("A user with email " + lm.email + " already exists in emailpassword loginMethod.")); + new Exception( + "E003: A user with email " + lm.email + " already exists in emailpassword loginMethod.")); } } @@ -249,10 +268,12 @@ private static void processThirdPartyLoginMethod(TenantIdentifier tenantIdentifi lm.timeJoinedInMSSinceEpoch); lm.superTokensUserId = userInfo.user.getSupertokensUserId(); - } catch (StorageQueryException | TenantOrAppNotFoundException e) { + } catch (StorageQueryException e) { throw new StorageTransactionLogicException(e); + } catch (TenantOrAppNotFoundException e) { + throw new StorageTransactionLogicException(new Exception("E004: " + e.getMessage())); } catch (DuplicateThirdPartyUserException e) { - throw new StorageTransactionLogicException(new Exception("A user with thirdPartyId " + lm.thirdPartyId + throw new StorageTransactionLogicException(new Exception("E005: A user with thirdPartyId " + lm.thirdPartyId + " and thirdPartyUserId " + lm.thirdPartyUserId + " already exists in thirdparty loginMethod.")); } } @@ -266,11 +287,15 @@ private static void processPasswordlessLoginMethod(TenantIdentifier tenantIdenti lm.superTokensUserId = userInfo.getSupertokensUserId(); } catch (RestartFlowException e) { - String errorMessage = lm.email != null ? "A user with email " + lm.email + " already exists in passwordless loginMethod." - : "A user with phoneNumber " + lm.phoneNumber + " already exists in passwordless loginMethod."; + String errorMessage = lm.email != null + ? "E006: A user with email " + lm.email + " already exists in passwordless loginMethod." + : "E007: A user with phoneNumber " + lm.phoneNumber + + " already exists in passwordless loginMethod."; throw new StorageTransactionLogicException(new Exception(errorMessage)); - } catch (StorageQueryException | TenantOrAppNotFoundException e) { + } catch (StorageQueryException e) { throw new StorageTransactionLogicException(e); + } catch (TenantOrAppNotFoundException e) { + throw new StorageTransactionLogicException(new Exception("E008: " + e.getMessage())); } } @@ -285,12 +310,42 @@ private static void associateUserToTenants(Main main, AppIdentifier appIdentifie TenantIdentifier tenantIdentifier = new TenantIdentifier(appIdentifier.getConnectionUriDomain(), appIdentifier.getAppId(), tenantId); Multitenancy.addUserIdToTenant(main, tenantIdentifier, storage, lm.getSuperTokenOrExternalUserId()); - } catch (TenantOrAppNotFoundException | UnknownUserIdException | StorageQueryException - | FeatureNotEnabledException | DuplicateEmailException | DuplicatePhoneNumberException - | DuplicateThirdPartyUserException | AnotherPrimaryUserWithPhoneNumberAlreadyExistsException - | AnotherPrimaryUserWithEmailAlreadyExistsException - | AnotherPrimaryUserWithThirdPartyInfoAlreadyExistsException e) { + } catch (TenantOrAppNotFoundException e) { + throw new StorageTransactionLogicException(new Exception("E009: " + e.getMessage())); + } catch (StorageQueryException e) { throw new StorageTransactionLogicException(e); + } catch (UnknownUserIdException e) { + throw new StorageTransactionLogicException(new Exception("E010: " + "We tried to add the userId " + + lm.getSuperTokenOrExternalUserId() + " to the tenantId " + tenantId + + " but it doesn't exist. This should not happen. Please contact support.")); + } catch (AnotherPrimaryUserWithEmailAlreadyExistsException e) { + throw new StorageTransactionLogicException(new Exception("E011: " + "We tried to add the userId " + + lm.getSuperTokenOrExternalUserId() + " to the tenantId " + tenantId + + " but another primary user with email " + lm.email + " already exists.")); + } catch (AnotherPrimaryUserWithPhoneNumberAlreadyExistsException e) { + throw new StorageTransactionLogicException(new Exception("E012: " + "We tried to add the userId " + + lm.getSuperTokenOrExternalUserId() + " to the tenantId " + tenantId + + " but another primary user with phoneNumber " + lm.phoneNumber + " already exists.")); + } catch (AnotherPrimaryUserWithThirdPartyInfoAlreadyExistsException e) { + throw new StorageTransactionLogicException(new Exception("E013: " + "We tried to add the userId " + + lm.getSuperTokenOrExternalUserId() + " to the tenantId " + tenantId + + " but another primary user with thirdPartyId " + lm.thirdPartyId + " and thirdPartyUserId " + + lm.thirdPartyUserId + " already exists.")); + } catch (DuplicateEmailException e) { + throw new StorageTransactionLogicException(new Exception("E014: " + "We tried to add the userId " + + lm.getSuperTokenOrExternalUserId() + " to the tenantId " + tenantId + + " but another user with email " + lm.email + " already exists.")); + } catch (DuplicatePhoneNumberException e) { + throw new StorageTransactionLogicException(new Exception("E015: " + "We tried to add the userId " + + lm.getSuperTokenOrExternalUserId() + " to the tenantId " + tenantId + + " but another user with phoneNumber " + lm.phoneNumber + " already exists.")); + } catch (DuplicateThirdPartyUserException e) { + throw new StorageTransactionLogicException(new Exception("E016: " + "We tried to add the userId " + + lm.getSuperTokenOrExternalUserId() + " to the tenantId " + tenantId + + " but another user with thirdPartyId " + lm.thirdPartyId + " and thirdPartyUserId " + + lm.thirdPartyUserId + " already exists.")); + } catch (FeatureNotEnabledException e) { + throw new StorageTransactionLogicException(new Exception("E017: " + e.getMessage())); } } } @@ -304,16 +359,27 @@ public static void createPrimaryUserAndLinkAccounts(Main main, try { AuthRecipe.createPrimaryUser(main, appIdentifier, storage, primaryLM.getSuperTokenOrExternalUserId()); - } catch (TenantOrAppNotFoundException | FeatureNotEnabledException | StorageQueryException e) { + } catch (TenantOrAppNotFoundException e) { + throw new StorageTransactionLogicException(new Exception("E018: " + e.getMessage())); + } catch (StorageQueryException e) { throw new StorageTransactionLogicException(e); + } catch (FeatureNotEnabledException e) { + throw new StorageTransactionLogicException(new Exception("E019: " + e.getMessage())); } catch (UnknownUserIdException e) { throw new StorageTransactionLogicException(new Exception( - "We tried to create the primary user for the userId " + primaryLM.getSuperTokenOrExternalUserId() + "E020: We tried to create the primary user for the userId " + + primaryLM.getSuperTokenOrExternalUserId() + " but it doesn't exist. This should not happen. Please contact support.")); - } catch (RecipeUserIdAlreadyLinkedWithPrimaryUserIdException - | AccountInfoAlreadyAssociatedWithAnotherPrimaryUserIdException e) { - throw new StorageTransactionLogicException( - new Exception(e.getMessage() + " This should not happen. Please contact support.")); + } catch (RecipeUserIdAlreadyLinkedWithPrimaryUserIdException e) { + throw new StorageTransactionLogicException(new Exception( + "E021: We tried to create the primary user for the userId " + + primaryLM.getSuperTokenOrExternalUserId() + + " but it is already linked with another primary user.")); + } catch (AccountInfoAlreadyAssociatedWithAnotherPrimaryUserIdException e) { + throw new StorageTransactionLogicException(new Exception( + "E022: We tried to create the primary user for the userId " + + primaryLM.getSuperTokenOrExternalUserId() + + " but the account info is already associated with another primary user.")); } for (LoginMethod lm : user.loginMethods) { @@ -325,22 +391,32 @@ public static void createPrimaryUserAndLinkAccounts(Main main, AuthRecipe.linkAccounts(main, appIdentifier, storage, lm.getSuperTokenOrExternalUserId(), primaryLM.getSuperTokenOrExternalUserId()); - } catch (TenantOrAppNotFoundException | FeatureNotEnabledException | StorageQueryException e) { + } catch (TenantOrAppNotFoundException e) { + throw new StorageTransactionLogicException(new Exception("E023: " + e.getMessage())); + } catch (StorageQueryException e) { throw new StorageTransactionLogicException(e); + } catch (FeatureNotEnabledException e) { + throw new StorageTransactionLogicException(new Exception("E024: " + e.getMessage())); } catch (UnknownUserIdException e) { throw new StorageTransactionLogicException( - new Exception("We tried to link the userId " + lm.getSuperTokenOrExternalUserId() + new Exception("E025: We tried to link the userId " + lm.getSuperTokenOrExternalUserId() + " to the primary userId " + primaryLM.getSuperTokenOrExternalUserId() - + " but it doesn't exist. This should not happen. Please contact support.")); + + " but it doesn't exist.")); } catch (InputUserIdIsNotAPrimaryUserException e) { throw new StorageTransactionLogicException( - new Exception("We tried to link the userId " + lm.getSuperTokenOrExternalUserId() + new Exception("E026: We tried to link the userId " + lm.getSuperTokenOrExternalUserId() + " to the primary userId " + primaryLM.getSuperTokenOrExternalUserId() - + " but it is not a primary user. This should not happen. Please contact support.")); - } catch (AccountInfoAlreadyAssociatedWithAnotherPrimaryUserIdException - | RecipeUserIdAlreadyLinkedWithAnotherPrimaryUserIdException e) { - throw new StorageTransactionLogicException( - new Exception(e.getMessage() + " This should not happen. Please contact support.")); + + " but it is not a primary user.")); + } catch (AccountInfoAlreadyAssociatedWithAnotherPrimaryUserIdException e) { + throw new StorageTransactionLogicException(new Exception( + "E027: We tried to link the userId " + lm.getSuperTokenOrExternalUserId() + + " to the primary userId " + primaryLM.getSuperTokenOrExternalUserId() + + " but the account info is already associated with another primary user.")); + } catch (RecipeUserIdAlreadyLinkedWithAnotherPrimaryUserIdException e) { + throw new StorageTransactionLogicException(new Exception( + "E028: We tried to link the userId " + lm.getSuperTokenOrExternalUserId() + + " to the primary userId " + primaryLM.getSuperTokenOrExternalUserId() + + " but it is already linked with another primary user.")); } } } @@ -355,14 +431,18 @@ public static void createUserIdMapping(AppIdentifier appIdentifier, null, false, true); primaryLM.externalUserId = user.externalUserId; - } catch (StorageQueryException | ServletException | TenantOrAppNotFoundException e) { + } catch (TenantOrAppNotFoundException e) { + throw new StorageTransactionLogicException(new Exception("E029: " + e.getMessage())); + } catch (StorageQueryException e) { throw new StorageTransactionLogicException(e); + } catch (ServletException e) { + throw new StorageTransactionLogicException(new Exception("E030: " + e.getMessage())); } catch (UserIdMappingAlreadyExistsException e) { throw new StorageTransactionLogicException( - new Exception("A user with externalId " + user.externalUserId + " already exists")); + new Exception("E031: A user with externalId " + user.externalUserId + " already exists")); } catch (UnknownSuperTokensUserIdException e) { throw new StorageTransactionLogicException( - new Exception("We tried to create the externalUserId mapping for the superTokenUserId " + new Exception("E032: We tried to create the externalUserId mapping for the superTokenUserId " + primaryLM.superTokensUserId + " but it doesn't exist. This should not happen. Please contact support.")); } @@ -375,7 +455,9 @@ public static void createUserMetadata(AppIdentifier appIdentifier, Storage stora try { UserMetadata.updateUserMetadata(appIdentifier, storage, primaryLM.getSuperTokenOrExternalUserId(), user.userMetadata); - } catch (StorageQueryException | TenantOrAppNotFoundException e) { + } catch (TenantOrAppNotFoundException e) { + throw new StorageTransactionLogicException(new Exception("E040: " + e.getMessage())); + } catch (StorageQueryException e) { throw new StorageTransactionLogicException(e); } } @@ -393,10 +475,12 @@ public static void createUserRoles(Main main, AppIdentifier appIdentifier, Stora UserRoles.addRoleToUser(main, tenantIdentifier, storage, user.externalUserId, userRole.role); } - } catch (TenantOrAppNotFoundException | StorageQueryException e) { + } catch (TenantOrAppNotFoundException e) { + throw new StorageTransactionLogicException(new Exception("E033: " + e.getMessage())); + } catch (StorageQueryException e) { throw new StorageTransactionLogicException(e); } catch (UnknownRoleException e) { - throw new StorageTransactionLogicException(new Exception("Role " + userRole.role + throw new StorageTransactionLogicException(new Exception("E034: Role " + userRole.role + " does not exist! You need pre-create the role before assigning it to the user.")); } } @@ -418,7 +502,9 @@ public static void verifyEmailForAllLoginMethods(AppIdentifier appIdentifier, Tr emailVerificationSQLStorage .updateIsEmailVerified_Transaction(tenantIdentifier.toAppIdentifier(), con, lm.getSuperTokenOrExternalUserId(), lm.email, true); - } catch (TenantOrAppNotFoundException | StorageQueryException e) { + } catch (TenantOrAppNotFoundException e) { + throw new StorageTransactionLogicException(new Exception("E035: " + e.getMessage())); + } catch (StorageQueryException e) { throw new StorageTransactionLogicException(e); } } @@ -432,11 +518,16 @@ public static void createTotpDevices(Main main, AppIdentifier appIdentifier, Sto Totp.createDevice(main, appIdentifier, storage, primaryLM.getSuperTokenOrExternalUserId(), totpDevice.deviceName, totpDevice.skew, totpDevice.period, totpDevice.secretKey, true, System.currentTimeMillis()); - } catch (TenantOrAppNotFoundException | StorageQueryException | FeatureNotEnabledException e) { + } catch (TenantOrAppNotFoundException e) { + throw new StorageTransactionLogicException(new Exception("E036: " + e.getMessage())); + } catch (StorageQueryException e) { throw new StorageTransactionLogicException(e); + } catch (FeatureNotEnabledException e) { + throw new StorageTransactionLogicException(new Exception("E037: " + e.getMessage())); } catch (DeviceAlreadyExistsException e) { throw new StorageTransactionLogicException( - new Exception("A totp device with name " + totpDevice.deviceName + " already exists")); + new Exception( + "E038: A totp device with name " + totpDevice.deviceName + " already exists")); } } } @@ -459,8 +550,7 @@ public static BulkImportUser.LoginMethod getPrimaryLoginMethod(BulkImportUser us } private static synchronized Storage getBulkImportProxyStorage(Main main, TenantIdentifier tenantIdentifier) - throws InvalidConfigException, IOException, TenantOrAppNotFoundException, DbInitException, - StorageQueryException { + throws InvalidConfigException, IOException, TenantOrAppNotFoundException, DbInitException { String userPoolId = StorageLayer.getStorage(tenantIdentifier, main).getUserPoolId(); if (userPoolToStorageMap.containsKey(userPoolId)) { return userPoolToStorageMap.get(userPoolId); @@ -479,13 +569,6 @@ private static synchronized Storage getBulkImportProxyStorage(Main main, TenantI userPoolToStorageMap.put(userPoolId, bulkImportProxyStorage); bulkImportProxyStorage.initStorage(false, new ArrayList<>()); - // `BulkImportProxyStorage` uses `BulkImportProxyConnection`, which overrides the `.commit()` method on the Connection object. - // The `initStorage()` method runs `select * from table_name limit 1` queries to check if the tables exist but these queries - // don't get committed due to the overridden `.commit()`, so we need to manually commit the transaction to remove any locks on the tables. - - // Without this commit, a call to `select * from bulk_import_users limit 1` in `doesTableExist()` locks the `bulk_import_users` table, - // causing other queries to stall indefinitely. - bulkImportProxyStorage.commitTransactionForBulkImportProxyStorage(); return bulkImportProxyStorage; } } @@ -503,9 +586,14 @@ private static Storage[] getAllProxyStoragesForApp(Main main, AppIdentifier appI allProxyStorages.add(getBulkImportProxyStorage(main, tenantConfig.tenantIdentifier)); } return allProxyStorages.toArray(new Storage[0]); - } catch (TenantOrAppNotFoundException | InvalidConfigException | IOException | DbInitException - | StorageQueryException e) { - throw new StorageTransactionLogicException(e); + } catch (TenantOrAppNotFoundException e) { + throw new StorageTransactionLogicException(new Exception("E039: " + e.getMessage())); + } catch (InvalidConfigException e) { + throw new StorageTransactionLogicException(new InvalidConfigException("E040: " + e.getMessage())); + } catch (DbInitException e) { + throw new StorageTransactionLogicException(new DbInitException("E041: " + e.getMessage())); + } catch (IOException e) { + throw new StorageTransactionLogicException(new IOException("E042: " + e.getMessage())); } } diff --git a/src/main/java/io/supertokens/bulkimport/BulkImportUserUtils.java b/src/main/java/io/supertokens/bulkimport/BulkImportUserUtils.java index 4c0bc73ba..7c6aac760 100644 --- a/src/main/java/io/supertokens/bulkimport/BulkImportUserUtils.java +++ b/src/main/java/io/supertokens/bulkimport/BulkImportUserUtils.java @@ -205,9 +205,15 @@ private List getParsedLoginMethods(Main main, AppIdentifier appIden String email = parseAndValidateFieldType(jsonLoginMethodObj, "email", ValueType.STRING, true, String.class, errors, " for an emailpassword recipe."); String passwordHash = parseAndValidateFieldType(jsonLoginMethodObj, "passwordHash", ValueType.STRING, - true, String.class, errors, " for an emailpassword recipe."); + false, String.class, errors, " for an emailpassword recipe."); String hashingAlgorithm = parseAndValidateFieldType(jsonLoginMethodObj, "hashingAlgorithm", - ValueType.STRING, true, String.class, errors, " for an emailpassword recipe."); + ValueType.STRING, false, String.class, errors, " for an emailpassword recipe."); + String plainTextPassword = parseAndValidateFieldType(jsonLoginMethodObj, "plainTextPassword", + ValueType.STRING, false, String.class, errors, " for an emailpassword recipe."); + + if ((passwordHash == null || hashingAlgorithm == null) && plainTextPassword == null) { + errors.add("Either (passwordHash, hashingAlgorithm) or plainTextPassword is required for an emailpassword recipe."); + } email = validateAndNormaliseEmail(email, errors); CoreConfig.PASSWORD_HASHING_ALG normalisedHashingAlgorithm = validateAndNormaliseHashingAlgorithm( @@ -218,7 +224,7 @@ private List getParsedLoginMethods(Main main, AppIdentifier appIden passwordHash, errors); loginMethods.add(new LoginMethod(normalisedTenantIds, recipeId, isVerified, isPrimary, - timeJoinedInMSSinceEpoch, email, passwordHash, hashingAlgorithm, null, null, null)); + timeJoinedInMSSinceEpoch, email, passwordHash, hashingAlgorithm, null, null, null, null)); } else if ("thirdparty".equals(recipeId)) { String email = parseAndValidateFieldType(jsonLoginMethodObj, "email", ValueType.STRING, true, String.class, errors, " for a thirdparty recipe."); @@ -232,7 +238,7 @@ private List getParsedLoginMethods(Main main, AppIdentifier appIden thirdPartyUserId = validateAndNormaliseThirdPartyUserId(thirdPartyUserId, errors); loginMethods.add(new LoginMethod(normalisedTenantIds, recipeId, isVerified, isPrimary, - timeJoinedInMSSinceEpoch, email, null, null, thirdPartyId, thirdPartyUserId, null)); + timeJoinedInMSSinceEpoch, email, null, null, null, thirdPartyId, thirdPartyUserId, null)); } else if ("passwordless".equals(recipeId)) { String email = parseAndValidateFieldType(jsonLoginMethodObj, "email", ValueType.STRING, false, String.class, errors, " for a passwordless recipe."); @@ -247,7 +253,7 @@ private List getParsedLoginMethods(Main main, AppIdentifier appIden } loginMethods.add(new LoginMethod(normalisedTenantIds, recipeId, isVerified, isPrimary, - timeJoinedInMSSinceEpoch, email, null, null, null, null, phoneNumber)); + timeJoinedInMSSinceEpoch, email, null, null, null, null, null, phoneNumber)); } } return loginMethods; diff --git a/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkImportUsers.java b/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkImportUsers.java index be3dfb744..a4b235da0 100644 --- a/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkImportUsers.java +++ b/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkImportUsers.java @@ -33,6 +33,7 @@ import io.supertokens.cronjobs.CronTask; import io.supertokens.cronjobs.CronTaskTest; import io.supertokens.multitenancy.Multitenancy; +import io.supertokens.output.Logging; import io.supertokens.pluginInterface.STORAGE_TYPE; import io.supertokens.pluginInterface.Storage; import io.supertokens.pluginInterface.StorageUtils; @@ -148,8 +149,14 @@ private Storage[] getAllProxyStoragesForApp(Main main, AppIdentifier appIdentifi allProxyStorages.add(getBulkImportProxyStorage(tenantConfig.tenantIdentifier)); } return allProxyStorages.toArray(new Storage[0]); - } catch (TenantOrAppNotFoundException | InvalidConfigException | IOException | DbInitException e) { - throw new StorageTransactionLogicException(e); + } catch (TenantOrAppNotFoundException e) { + throw new StorageTransactionLogicException(new Exception("E043: " + e.getMessage())); + } catch (InvalidConfigException e) { + throw new StorageTransactionLogicException(new InvalidConfigException("E044: " + e.getMessage())); + } catch (DbInitException e) { + throw new StorageTransactionLogicException(new DbInitException("E045: " + e.getMessage())); + } catch (IOException e) { + throw new StorageTransactionLogicException(new IOException("E046: " + e.getMessage())); } } @@ -225,21 +232,8 @@ private void processUser(AppIdentifier appIdentifier, BulkImportUser user, BulkI bulkImportProxyStorage.startTransaction(con -> { try { - for (LoginMethod lm : user.loginMethods) { - BulkImport.processUserLoginMethod(main, appIdentifier, bulkImportProxyStorage, lm); - } - - BulkImport.createPrimaryUserAndLinkAccounts(main, appIdentifier, bulkImportProxyStorage, user, - primaryLM); - Storage[] allStoragesForApp = getAllProxyStoragesForApp(main, appIdentifier); - BulkImport.createUserIdMapping(appIdentifier, user, primaryLM, allStoragesForApp); - - BulkImport.verifyEmailForAllLoginMethods(appIdentifier, con, bulkImportProxyStorage, - user.loginMethods); - BulkImport.createTotpDevices(main, appIdentifier, bulkImportProxyStorage, user, primaryLM); - BulkImport.createUserMetadata(appIdentifier, bulkImportProxyStorage, user, primaryLM); - BulkImport.createUserRoles(main, appIdentifier, bulkImportProxyStorage, user); + BulkImport.processUserImportSteps(main, con, appIdentifier, bulkImportProxyStorage, user, primaryLM, allStoragesForApp); // We are updating the primaryUserId in the bulkImportUser entry. This will help us handle the inconsistent transaction commit. // If this update statement fails then the outer transaction will fail as well and the user will simpl be processed again. No inconsistency will happen in this @@ -273,13 +267,18 @@ private void processUser(AppIdentifier appIdentifier, BulkImportUser user, BulkI private void handleProcessUserExceptions(AppIdentifier appIdentifier, BulkImportUser user, Exception e, BulkImportSQLStorage baseTenantStorage) throws StorageQueryException { - // Java doesn't allow us to reassign local variables inside a lambda expression // so we have to use an array. String[] errorMessage = { e.getMessage() }; if (e instanceof StorageTransactionLogicException) { StorageTransactionLogicException exception = (StorageTransactionLogicException) e; + // If the exception is due to a StorageQueryException, we want to retry the entry after sometime instead + // of marking it as FAILED. We will return early in that case. + if (exception.actualException instanceof StorageQueryException) { + Logging.error(main, null, "We got an StorageQueryException while processing a bulk import user entry. It will be retried again. Error Message: " + e.getMessage(), true); + return; + } errorMessage[0] = exception.actualException.getMessage(); } else if (e instanceof InvalidBulkImportDataException) { errorMessage[0] = ((InvalidBulkImportDataException) e).errors.toString(); diff --git a/src/main/java/io/supertokens/webserver/WebserverAPI.java b/src/main/java/io/supertokens/webserver/WebserverAPI.java index 616413635..4ca52c83c 100644 --- a/src/main/java/io/supertokens/webserver/WebserverAPI.java +++ b/src/main/java/io/supertokens/webserver/WebserverAPI.java @@ -74,10 +74,11 @@ public abstract class WebserverAPI extends HttpServlet { supportedVersions.add(SemVer.v3_0); supportedVersions.add(SemVer.v4_0); supportedVersions.add(SemVer.v5_0); + supportedVersions.add(SemVer.v5_1); } public static SemVer getLatestCDIVersion() { - return SemVer.v5_0; + return SemVer.v5_1; } public SemVer getLatestCDIVersionForRequest(HttpServletRequest req) diff --git a/src/test/java/io/supertokens/test/bulkimport/BulkImportTest.java b/src/test/java/io/supertokens/test/bulkimport/BulkImportTest.java index ac298d4ce..cd399e61c 100644 --- a/src/test/java/io/supertokens/test/bulkimport/BulkImportTest.java +++ b/src/test/java/io/supertokens/test/bulkimport/BulkImportTest.java @@ -47,6 +47,7 @@ import io.supertokens.pluginInterface.authRecipe.AuthRecipeUserInfo; import io.supertokens.pluginInterface.bulkimport.BulkImportStorage; import io.supertokens.pluginInterface.bulkimport.BulkImportUser; +import io.supertokens.pluginInterface.bulkimport.BulkImportUser.LoginMethod; import io.supertokens.pluginInterface.bulkimport.BulkImportStorage.BULK_IMPORT_USER_STATUS; import io.supertokens.pluginInterface.bulkimport.sqlStorage.BulkImportSQLStorage; import io.supertokens.pluginInterface.multitenancy.AppIdentifier; @@ -391,7 +392,7 @@ public void shouldImportTheUserInTheSameTenant() throws Exception { AuthRecipeUserInfo importedUser = BulkImport.importUser(main, appIdentifier, users.get(0)); - BulkImportTestUtils.assertBulkImportUserAndAuthRecipeUserAreEqual(appIdentifier, + BulkImportTestUtils.assertBulkImportUserAndAuthRecipeUserAreEqual(main, appIdentifier, appIdentifier.getAsPublicTenantIdentifier(), StorageLayer.getStorage(main), users.get(0), importedUser); process.kill(); @@ -439,10 +440,10 @@ public void shouldImportTheUserInMultipleTenantsWithDifferentStorages() throws E AuthRecipeUserInfo importedUser1 = BulkImport.importUser(main, appIdentifier, bulkImportUserT1); AuthRecipeUserInfo importedUser2 = BulkImport.importUser(main, appIdentifier, bulkImportUserT2); - BulkImportTestUtils.assertBulkImportUserAndAuthRecipeUserAreEqual(appIdentifier, t1, storageT1, + BulkImportTestUtils.assertBulkImportUserAndAuthRecipeUserAreEqual(main, appIdentifier, t1, storageT1, bulkImportUserT1, importedUser1); - BulkImportTestUtils.assertBulkImportUserAndAuthRecipeUserAreEqual(appIdentifier, t2, storageT2, + BulkImportTestUtils.assertBulkImportUserAndAuthRecipeUserAreEqual(main, appIdentifier, t2, storageT2, bulkImportUserT2, importedUser2); @@ -493,7 +494,7 @@ public void shouldImportUsersConcurrently() throws Exception { for (int i = 0; i < users.size(); i++) { AuthRecipeUserInfo importedUser = futures.get(i).get(); - BulkImportTestUtils.assertBulkImportUserAndAuthRecipeUserAreEqual(appIdentifier, + BulkImportTestUtils.assertBulkImportUserAndAuthRecipeUserAreEqual(main, appIdentifier, appIdentifier.getAsPublicTenantIdentifier(), StorageLayer.getStorage(main), users.get(i), importedUser); } @@ -502,4 +503,50 @@ public void shouldImportUsersConcurrently() throws Exception { assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STOPPED)); } + @Test + public void shouldImportWithPlainTextPassword() throws Exception { + String[] args = { "../" }; + + TestingProcessManager.TestingProcess process = TestingProcessManager.start(args); + assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); + Main main = process.getProcess(); + + if (StorageLayer.getStorage(main).getType() != STORAGE_TYPE.SQL || StorageLayer.isInMemDb(main)) { + return; + } + + FeatureFlagTestContent.getInstance(main).setKeyValue(FeatureFlagTestContent.ENABLED_FEATURES, + new EE_FEATURES[] { EE_FEATURES.MULTI_TENANCY, EE_FEATURES.MFA, EE_FEATURES.ACCOUNT_LINKING }); + + // Create tenants + BulkImportTestUtils.createTenants(main); + + // Create user roles + { + UserRoles.createNewRoleOrModifyItsPermissions(main, "role1", null); + UserRoles.createNewRoleOrModifyItsPermissions(main, "role2", null); + } + + AppIdentifier appIdentifier = new AppIdentifier(null, null); + List users = generateBulkImportUser(1); + BulkImportUser bulkImportUser = users.get(0); + + // Set passwordHash to null and plainTextPassword to a value to ensure we do a plainTextPassword import + for (LoginMethod lm : bulkImportUser.loginMethods) { + if (lm.recipeId == "emailpassword") { + lm.passwordHash = null; + lm.hashingAlgorithm = null; + lm.plainTextPassword = "testPass@123"; + } + } + + AuthRecipeUserInfo importedUser = BulkImport.importUser(main, appIdentifier, bulkImportUser); + + BulkImportTestUtils.assertBulkImportUserAndAuthRecipeUserAreEqual(main, appIdentifier, + appIdentifier.getAsPublicTenantIdentifier(), StorageLayer.getStorage(main), bulkImportUser, importedUser); + + process.kill(); + assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STOPPED)); + } + } diff --git a/src/test/java/io/supertokens/test/bulkimport/BulkImportTestUtils.java b/src/test/java/io/supertokens/test/bulkimport/BulkImportTestUtils.java index 4faea47ab..61740021b 100644 --- a/src/test/java/io/supertokens/test/bulkimport/BulkImportTestUtils.java +++ b/src/test/java/io/supertokens/test/bulkimport/BulkImportTestUtils.java @@ -28,6 +28,7 @@ import com.google.gson.JsonParser; import io.supertokens.Main; +import io.supertokens.emailpassword.PasswordHashing; import io.supertokens.featureflag.exceptions.FeatureNotEnabledException; import io.supertokens.multitenancy.Multitenancy; import io.supertokens.multitenancy.exception.BadPermissionException; @@ -82,11 +83,13 @@ public static List generateBulkImportUser(int numberOfUsers, Lis List loginMethods = new ArrayList<>(); long currentTimeMillis = System.currentTimeMillis(); loginMethods.add(new LoginMethod(tenants, "emailpassword", true, true, currentTimeMillis, email, "$2a", - "BCRYPT", null, null, null)); - loginMethods.add(new LoginMethod(tenants, "thirdparty", true, false, currentTimeMillis, email, null, null, - "thirdPartyId" + i, "thirdPartyUserId" + i, null)); - loginMethods.add(new LoginMethod(tenants, "passwordless", true, false, currentTimeMillis, email, null, null, - null, null, null)); + "BCRYPT", null, null, null, null)); + loginMethods + .add(new LoginMethod(tenants, "thirdparty", true, false, currentTimeMillis, email, null, null, null, + "thirdPartyId" + i, "thirdPartyUserId" + i, null)); + loginMethods.add( + new LoginMethod(tenants, "passwordless", true, false, currentTimeMillis, email, null, null, null, + null, null, null)); users.add(new BulkImportUser(id, externalId, userMetadata, userRoles, totpDevices, loginMethods)); } return users; @@ -131,15 +134,15 @@ public static void createTenants(Main main) } } - public static void assertBulkImportUserAndAuthRecipeUserAreEqual(AppIdentifier appIdentifier, + public static void assertBulkImportUserAndAuthRecipeUserAreEqual(Main main, AppIdentifier appIdentifier, TenantIdentifier tenantIdentifier, Storage storage, BulkImportUser bulkImportUser, AuthRecipeUserInfo authRecipeUser) throws StorageQueryException, TenantOrAppNotFoundException { for (io.supertokens.pluginInterface.authRecipe.LoginMethod lm1 : authRecipeUser.loginMethods) { - bulkImportUser.loginMethods.forEach(lm2 -> { + for (LoginMethod lm2 : bulkImportUser.loginMethods) { if (lm2.recipeId.equals(lm1.recipeId.toString())) { - assertLoginMethodEquals(lm1, lm2); + assertLoginMethodEquals(main, lm1, lm2); } - }); + } } assertEquals(bulkImportUser.externalUserId, authRecipeUser.getSupertokensOrExternalUserId()); assertEquals(bulkImportUser.userMetadata, @@ -155,15 +158,23 @@ public static void assertBulkImportUserAndAuthRecipeUserAreEqual(AppIdentifier a assertTotpDevicesEquals(createdTotpDevices, bulkImportUser.totpDevices.toArray(new TotpDevice[0])); } - private static void assertLoginMethodEquals(io.supertokens.pluginInterface.authRecipe.LoginMethod lm1, - io.supertokens.pluginInterface.bulkimport.BulkImportUser.LoginMethod lm2) { + private static void assertLoginMethodEquals(Main main, io.supertokens.pluginInterface.authRecipe.LoginMethod lm1, + io.supertokens.pluginInterface.bulkimport.BulkImportUser.LoginMethod lm2) + throws TenantOrAppNotFoundException { assertEquals(lm1.email, lm2.email); assertEquals(lm1.verified, lm2.isVerified); assertTrue(lm2.tenantIds.containsAll(lm1.tenantIds) && lm1.tenantIds.containsAll(lm2.tenantIds)); switch (lm2.recipeId) { case "emailpassword": - assertEquals(lm1.passwordHash, lm2.passwordHash); + // If lm2.passwordHash is null then the user was imported using plainTextPassword + // We check if the plainTextPassword matches the stored passwordHash + if (lm2.passwordHash == null) { + assertTrue(PasswordHashing.getInstance(main).verifyPasswordWithHash(lm2.plainTextPassword, + lm1.passwordHash)); + } else { + assertEquals(lm1.passwordHash, lm2.passwordHash); + } break; case "thirdparty": assertEquals(lm1.thirdParty.id, lm2.thirdPartyId); diff --git a/src/test/java/io/supertokens/test/bulkimport/ProcessBulkImportUsersCronJobTest.java b/src/test/java/io/supertokens/test/bulkimport/ProcessBulkImportUsersCronJobTest.java index a17f842e7..667287f5a 100644 --- a/src/test/java/io/supertokens/test/bulkimport/ProcessBulkImportUsersCronJobTest.java +++ b/src/test/java/io/supertokens/test/bulkimport/ProcessBulkImportUsersCronJobTest.java @@ -106,7 +106,7 @@ public void shouldProcessBulkImportUsersInTheSameTenant() throws Exception { TenantIdentifier publicTenant = new TenantIdentifier(null, null, "public"); - BulkImportTestUtils.assertBulkImportUserAndAuthRecipeUserAreEqual(appIdentifier, publicTenant, storage, + BulkImportTestUtils.assertBulkImportUserAndAuthRecipeUserAreEqual(main, appIdentifier, publicTenant, storage, bulkImportUser, container.users[0]); @@ -163,10 +163,10 @@ public void shouldProcessBulkImportUsersInMultipleTenantsWithDifferentStorages() UserIdMapping.populateExternalUserIdForUsers(appIdentifier, storageT1, containerT1.users); UserIdMapping.populateExternalUserIdForUsers(appIdentifier, storageT2, containerT2.users); - BulkImportTestUtils.assertBulkImportUserAndAuthRecipeUserAreEqual(appIdentifier, t1, storageT1, + BulkImportTestUtils.assertBulkImportUserAndAuthRecipeUserAreEqual(main, appIdentifier, t1, storageT1, bulkImportUserT1, containerT1.users[0]); - BulkImportTestUtils.assertBulkImportUserAndAuthRecipeUserAreEqual(appIdentifier, t2, storageT2, + BulkImportTestUtils.assertBulkImportUserAndAuthRecipeUserAreEqual(main, appIdentifier, t2, storageT2, bulkImportUserT2, containerT2.users[0]); @@ -205,7 +205,7 @@ public void shouldDeleteEverythingFromtheDBIfAnythingFails() throws Exception { assertEquals(1, usersAfterProcessing.size()); assertEquals(BULK_IMPORT_USER_STATUS.FAILED, usersAfterProcessing.get(0).status); - assertEquals("Role role1 does not exist! You need pre-create the role before assigning it to the user.", + assertEquals("E034: Role role1 does not exist! You need pre-create the role before assigning it to the user.", usersAfterProcessing.get(0).errorMessage); UserPaginationContainer container = AuthRecipe.getUsers(main, 100, "ASC", null, null, null); diff --git a/src/test/java/io/supertokens/test/bulkimport/apis/AddBulkImportUsersTest.java b/src/test/java/io/supertokens/test/bulkimport/apis/AddBulkImportUsersTest.java index 641ba3822..ff7d12f02 100644 --- a/src/test/java/io/supertokens/test/bulkimport/apis/AddBulkImportUsersTest.java +++ b/src/test/java/io/supertokens/test/bulkimport/apis/AddBulkImportUsersTest.java @@ -275,7 +275,7 @@ public void shouldThrow400IfEmailPasswordRecipeHasInvalidFieldTypes() throws Exc .getAsJsonObject(); testBadRequest(main, requestBody, "{\"error\":\"" + genericErrMsg - + "\",\"users\":[{\"index\":0,\"errors\":[\"email is required for an emailpassword recipe.\",\"passwordHash is required for an emailpassword recipe.\",\"hashingAlgorithm is required for an emailpassword recipe.\"]}]}"); + + "\",\"users\":[{\"index\":0,\"errors\":[\"email is required for an emailpassword recipe.\",\"Either (passwordHash, hashingAlgorithm) or plainTextPassword is required for an emailpassword recipe.\"]}]}"); // CASE 2: email, passwordHash and hashingAlgorithm field type is incorrect JsonObject requestBody2 = new JsonParser() @@ -284,7 +284,7 @@ public void shouldThrow400IfEmailPasswordRecipeHasInvalidFieldTypes() throws Exc .getAsJsonObject(); testBadRequest(main, requestBody2, "{\"error\":\"" + genericErrMsg - + "\",\"users\":[{\"index\":0,\"errors\":[\"email should be of type string for an emailpassword recipe.\",\"passwordHash should be of type string for an emailpassword recipe.\",\"hashingAlgorithm should be of type string for an emailpassword recipe.\"]}]}"); + + "\",\"users\":[{\"index\":0,\"errors\":[\"email should be of type string for an emailpassword recipe.\",\"passwordHash should be of type string for an emailpassword recipe.\",\"hashingAlgorithm should be of type string for an emailpassword recipe.\",\"Either (passwordHash, hashingAlgorithm) or plainTextPassword is required for an emailpassword recipe.\"]}]}"); // CASE 3: hashingAlgorithm is not one of bcrypt, argon2, firebase_scrypt JsonObject requestBody3 = new JsonParser() @@ -555,7 +555,7 @@ public void shouldFailIfANewFieldWasAddedToBulkImportUser() throws Exception { checkLoginMethodFields(user.loginMethods.get(0), "LoginMethod", Arrays.asList("tenantIds", "isVerified", "isPrimary", "timeJoinedInMSSinceEpoch", - "recipeId", "email", "passwordHash", "hashingAlgorithm", + "recipeId", "email", "passwordHash", "plainTextPassword", "hashingAlgorithm", "phoneNumber", "thirdPartyId", "thirdPartyUserId", "externalUserId", "superTokensUserId")); checkTotpDeviceFields(user.totpDevices.get(0), "TotpDevice", From 78979f4a455883ed5f1af3e0651ad71f31c14d62 Mon Sep 17 00:00:00 2001 From: Ankit Tiwari Date: Tue, 18 Jun 2024 12:05:07 +0530 Subject: [PATCH 21/21] fix: PR changes --- .../bulkimport/DeleteBulkImportUserAPI.java | 1 + .../apis/DeleteBulkImportUsersTest.java | 21 ++++--------------- 2 files changed, 5 insertions(+), 17 deletions(-) diff --git a/src/main/java/io/supertokens/webserver/api/bulkimport/DeleteBulkImportUserAPI.java b/src/main/java/io/supertokens/webserver/api/bulkimport/DeleteBulkImportUserAPI.java index 0dd604fcd..d95e0c268 100644 --- a/src/main/java/io/supertokens/webserver/api/bulkimport/DeleteBulkImportUserAPI.java +++ b/src/main/java/io/supertokens/webserver/api/bulkimport/DeleteBulkImportUserAPI.java @@ -63,6 +63,7 @@ protected void doPost(HttpServletRequest req, HttpServletResponse resp) throws S result.add("deletedIds", new JsonArray()); result.add("invalidIds", new JsonArray()); super.sendJsonResponse(200, result, resp); + return; } if (arr.size() > BulkImport.DELETE_USERS_MAX_LIMIT) { diff --git a/src/test/java/io/supertokens/test/bulkimport/apis/DeleteBulkImportUsersTest.java b/src/test/java/io/supertokens/test/bulkimport/apis/DeleteBulkImportUsersTest.java index 30c07597c..28906ba95 100644 --- a/src/test/java/io/supertokens/test/bulkimport/apis/DeleteBulkImportUsersTest.java +++ b/src/test/java/io/supertokens/test/bulkimport/apis/DeleteBulkImportUsersTest.java @@ -88,19 +88,6 @@ public void shouldReturn400Error() throws Exception { e.getMessage()); } } - { - try { - JsonObject request = new JsonParser().parse("{\"ids\":[]}").getAsJsonObject(); - HttpRequestForTesting.sendJsonPOSTRequest(main, "", - "http://localhost:3567/bulk-import/users/remove", - request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); - fail("The API should have thrown an error"); - } catch (io.supertokens.test.httpRequest.HttpResponseException e) { - assertEquals(400, e.statusCode); - assertEquals("Http error. Status Code: 400. Message: Field name 'ids' cannot be an empty array", - e.getMessage()); - } - } { try { // Create a string array of 500 uuids @@ -140,13 +127,13 @@ public void shouldReturn200Response() throws Exception { // Call the API with empty array { - JsonObject request = new JsonParser().parse("{\"ids\":[\"\"]}").getAsJsonObject(); - JsonObject resonse = HttpRequestForTesting.sendJsonPOSTRequest(main, "", + JsonObject request = new JsonParser().parse("{\"ids\":[]}").getAsJsonObject(); + JsonObject response = HttpRequestForTesting.sendJsonPOSTRequest(main, "", "http://localhost:3567/bulk-import/users/remove", request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); - assertEquals(0, resonse.get("deletedIds").getAsJsonArray().size()); - assertEquals(0, resonse.get("invalidIds").getAsJsonArray().size()); + assertEquals(0, response.get("deletedIds").getAsJsonArray().size()); + assertEquals(0, response.get("invalidIds").getAsJsonArray().size()); } {