Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

UUID bulk fetching #451

Open
wants to merge 4 commits into
base: master
Choose a base branch
from
Open

UUID bulk fetching #451

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
a14b3b8
Revert "fix: update the Mojang API endpoint (#413)"
Tomut0 Oct 18, 2024
314ed76
fix: uuid bulk fetching over Mojang API
Tomut0 Oct 18, 2024
067fde2
feat: add Mojang API as UUID fetcher fallback
Tomut0 Nov 9, 2024
3fac455
suppress warning log on tryFetchUUIDWithFallback
Tomut0 Jan 31, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
91 changes: 62 additions & 29 deletions src/main/java/net/sacredlabyrinth/phaed/simpleclans/managers/StorageManager.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,9 @@
import java.text.MessageFormat;
import java.time.LocalDateTime;
import java.util.*;
import java.util.concurrent.ExecutionException;
import java.util.logging.Level;
import java.util.stream.Collectors;

import static net.sacredlabyrinth.phaed.simpleclans.SimpleClans.lang;
import static net.sacredlabyrinth.phaed.simpleclans.managers.SettingsManager.ConfigField.*;
Expand Down Expand Up @@ -537,7 +539,7 @@ public List<ClanPlayer> retrieveClanPlayers() {

if (last_seen == 0) {
last_seen = (new Date()).getTime();
}
}

ClanPlayer cp = new ClanPlayer();
if (uuid != null) {
Expand Down Expand Up @@ -1182,47 +1184,78 @@ private void updateDatabase() {
* Updates the database to the latest version
*
*/

private void updatePlayersToUUID() {
plugin.getLogger().log(Level.WARNING, "Starting Migration to UUID Players !");
plugin.getLogger().log(Level.WARNING, "==================== ATTENTION DONT STOP BUKKIT ! ==================== ");
plugin.getLogger().log(Level.WARNING, "==================== ATTENTION DONT STOP BUKKIT ! ==================== ");
plugin.getLogger().log(Level.WARNING, "==================== ATTENTION DONT STOP BUKKIT ! ==================== ");
SimpleClans.getInstance().setUUID(false);
logMigrationStart();

List<ClanPlayer> cps = retrieveClanPlayers();
Map<String, UUID> uuidMap = fetchUUIDs(cps);

int i = 1;
for (ClanPlayer cp : cps) {
int totalPlayers = cps.size();
for (int i = 0; i < totalPlayers; i++) {
ClanPlayer cp = cps.get(i);
try {
UUID uuidPlayer;
if (SimpleClans.getInstance().getServer().getOnlineMode()) {
uuidPlayer = UUIDFetcher.getUUIDOfThrottled(cp.getName());
} else {
uuidPlayer = UUID.nameUUIDFromBytes(("OfflinePlayer:" + cp.getName()).getBytes(Charsets.UTF_8));
}
String query = "UPDATE `" + getPrefixedTable("players") + "` SET uuid = '" + uuidPlayer.toString() + "' WHERE name = '" + cp.getName() + "';";
core.executeUpdate(query);
UUID uuid = uuidMap.getOrDefault(cp.getName(), cp.getUniqueId());
updatePlayerInDatabase(cp.getName(), uuid);
logSuccess(i + 1, totalPlayers, cp.getName(), uuid);
} catch (Exception ex) {
logFailure(i + 1, totalPlayers, cp.getName(), ex);
}
}

logMigrationEnd(totalPlayers);
}

String query2 = "UPDATE `" + getPrefixedTable("kills") + "` SET attacker_uuid = '" + uuidPlayer + "' WHERE attacker = '" + cp.getName() + "';";
core.executeUpdate(query2);
private void updatePlayerInDatabase(String playerName, UUID uuid) {
String[] tables = {"players", "kills", "kills"};
String[] columns = {"uuid", "attacker_uuid", "victim_uuid"};
String[] conditions = {"name", "attacker", "victim"};

String query3 = "UPDATE `" + getPrefixedTable("kills") + "` SET victim_uuid = '" + uuidPlayer + "' WHERE victim = '" + cp.getName() + "';";
core.executeUpdate(query3);
plugin.getLogger().info("[" + i + " / " + cps.size() + "] Success: " + cp.getName() + "; UUID: " + uuidPlayer);
} catch (Exception ex) {
plugin.getLogger().log(Level.WARNING, "[" + i + " / " + cps.size() + "] Failed [ERRO]: " + cp.getName() + "; UUID: ???");
for (int i = 0; i < tables.length; i++) {
String query = String.format("UPDATE `%s` SET %s = '%s' WHERE %s = '%s';",
getPrefixedTable(tables[i]), columns[i], uuid.toString(), conditions[i], playerName);
core.executeUpdate(query);
}
}

private Map<String, UUID> fetchUUIDs(List<ClanPlayer> clanPlayers) {
Map<String, UUID> uuidMap = new HashMap<>();

try {
if (SimpleClans.getInstance().getServer().getOnlineMode()) {
uuidMap = UUIDFetcher.fetchUUIDsForClanPlayers(clanPlayers);
} else {
uuidMap = clanPlayers.stream().collect(Collectors.toMap(ClanPlayer::getName, ClanPlayer::getUniqueId));
}
i++;
} catch (InterruptedException | ExecutionException ex) {
plugin.getLogger().log(Level.SEVERE, "Error fetching UUIDs in bulk: " + ex.getMessage(), ex);
}

return uuidMap;
}

private void logSuccess(int current, int total, String playerName, UUID uuid) {
plugin.getLogger().info(String.format("[%d / %d] Success: %s; UUID: %s", current, total, playerName, uuid));
}

private void logFailure(int current, int total, String playerName, Exception ex) {
plugin.getLogger().log(Level.WARNING, String.format("[%d / %d] Failed [ERROR]: %s; UUID: ???", current, total, playerName), ex);
}

private void logMigrationStart() {
plugin.getLogger().log(Level.WARNING, "Starting Migration to UUID Players!");
plugin.getLogger().log(Level.WARNING, "==================== ATTENTION DON'T STOP BUKKIT! ====================");
plugin.getLogger().log(Level.WARNING, "==================== ATTENTION DON'T STOP BUKKIT! ====================");
plugin.getLogger().log(Level.WARNING, "==================== ATTENTION DON'T STOP BUKKIT! ====================");
}

private void logMigrationEnd(int totalPlayers) {
plugin.getLogger().log(Level.WARNING, "==================== END OF MIGRATION ====================");
plugin.getLogger().log(Level.WARNING, "==================== END OF MIGRATION ====================");
plugin.getLogger().log(Level.WARNING, "==================== END OF MIGRATION ====================");


if (!cps.isEmpty()) {
plugin.getLogger().info(MessageFormat.format(lang("clan.players"), cps.size()));
if (totalPlayers > 0) {
plugin.getLogger().info(MessageFormat.format(lang("clan.players"), totalPlayers));
}
SimpleClans.getInstance().setUUID(true);
}

private String getPrefixedTable(String name) {
Expand Down
197 changes: 125 additions & 72 deletions src/main/java/net/sacredlabyrinth/phaed/simpleclans/uuid/UUIDFetcher.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,114 +1,167 @@
package net.sacredlabyrinth.phaed.simpleclans.uuid;

import com.google.common.collect.ImmutableList;
import com.google.gson.Gson;
import com.google.gson.JsonArray;
import com.google.gson.JsonElement;
import com.google.gson.JsonObject;
import net.sacredlabyrinth.phaed.simpleclans.ClanPlayer;
import net.sacredlabyrinth.phaed.simpleclans.SimpleClans;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;

import java.io.IOException;
import java.io.InputStreamReader;
import java.io.OutputStream;
import java.net.HttpURLConnection;
import java.net.URI;
import java.net.URL;
import java.nio.ByteBuffer;
import java.util.*;
import java.util.concurrent.Callable;
import java.util.concurrent.*;
import java.util.stream.Collectors;

import static java.nio.charset.StandardCharsets.UTF_8;
import static java.util.Collections.singletonList;

/**
* @author evilmidget38
*
* <a href="http://forums.bukkit.org/threads/250926/">...</a>
*
* <a href="https://gist.github.com/evilmidget38/26d70114b834f71fb3b4">...</a>
* @author evilmidget38 (previous author)
* @see <a href="http://forums.bukkit.org/threads/250926/">Bukkit Thread</a>
* @see <a href="http://web.archive.org/web/20140909143249/https://gist.github.com/evilmidget38/26d70114b834f71fb3b4">Github Gist</a>
*/
public class UUIDFetcher implements Callable<Map<String, UUID>> {
public final class UUIDFetcher {
private static final String PROFILE_URL = "https://api.minetools.eu/uuid/";
private static final String FALLBACK_PROFILE_URL = "https://api.mojang.com/users/profiles/minecraft/";
private static final int BATCH_SIZE = 100;
private static final Gson gson = new Gson();

private static final double PROFILES_PER_REQUEST = 100;
private static final String PROFILE_URL = "https://api.minecraftservices.com/minecraft/profile/lookup/bulk/byname";
private final Gson gson = new Gson();
private final List<String> names;
private final boolean rateLimiting;
private UUIDFetcher() {
// Private constructor to prevent instantiation
}

public UUIDFetcher(List<String> names, boolean rateLimiting) {
this.names = ImmutableList.copyOf(names);
this.rateLimiting = rateLimiting;
/**
* Fetches UUIDs for a list of ClanPlayer objects.
* This method extracts the names from the ClanPlayer objects and fetches their corresponding UUIDs.
*
* @param clanPlayers A list of ClanPlayer objects for which to fetch UUIDs
* @return A Map where the keys are player names and the values are their corresponding UUIDs
* @throws InterruptedException If the operation is interrupted while waiting
* @throws ExecutionException If the computation threw an exception
*/
public static Map<String, UUID> fetchUUIDsForClanPlayers(List<ClanPlayer> clanPlayers) throws InterruptedException, ExecutionException {
List<String> names = clanPlayers.stream().map(ClanPlayer::getName).collect(Collectors.toList());
return fetchUUIDsConcurrently(names);
}

public UUIDFetcher(List<String> names) {
this(names, true);
/**
* Fetches the UUID for a single player name.
* This method is a convenience wrapper around the batch UUID fetching process.
*
* @param name The name of the player whose UUID is to be fetched
* @return The UUID of the specified player, or null if not found
* @throws InterruptedException If the operation is interrupted while waiting
* @throws ExecutionException If the computation threw an exception
*/
public static @Nullable UUID getUUIDOf(@NotNull String name) throws InterruptedException, ExecutionException {
return fetchUUIDsConcurrently(singletonList(name)).get(name);
}

private static void writeBody(HttpURLConnection connection, String body) throws IOException {
OutputStream stream = connection.getOutputStream();
stream.write(body.getBytes(UTF_8));
stream.flush();
stream.close();
// Fetch UUIDs in batches with concurrency
private static Map<String, UUID> fetchUUIDsConcurrently(List<String> names) throws InterruptedException, ExecutionException {
Map<String, UUID> resultMap = new ConcurrentHashMap<>();
ExecutorService executorService = Executors.newFixedThreadPool(10); // Thread pool for parallel execution
List<Callable<Map<String, UUID>>> tasks = new ArrayList<>();

// Split the list of names into batches and create tasks
for (int i = 0; i < names.size(); i += BATCH_SIZE) {
List<String> batch = names.subList(i, Math.min(i + BATCH_SIZE, names.size()));
tasks.add(createTask(batch));
}

// Execute all tasks in parallel
List<Future<Map<String, UUID>>> futures = executorService.invokeAll(tasks);

// Collect results from all batches
for (Future<Map<String, UUID>> future : futures) {
resultMap.putAll(future.get()); // Merge each batch result into the final result map
}

// Shutdown the executor service
executorService.shutdownNow();
if (!executorService.awaitTermination(5, TimeUnit.SECONDS)) {
SimpleClans.getInstance().getLogger().warning("Executor did not terminate in time.");
}

return resultMap;
}

private static HttpURLConnection createConnection() throws IOException {
URL url = new URL(PROFILE_URL);
HttpURLConnection connection = (HttpURLConnection) url.openConnection();
connection.setRequestMethod("POST");
// Create connection for each name
private static HttpURLConnection createConnection(@NotNull URL url) throws IOException {
var connection = (HttpURLConnection) url.openConnection();

connection.setRequestMethod("GET");
connection.setRequestProperty("Content-Type", "application/json");
connection.setUseCaches(false);
connection.setDoInput(true);
connection.setDoOutput(true);

return connection;
}

private static UUID getUUID(String id) {
// Get UUID by name
private static @NotNull UUID getUUID(@NotNull String id) {
return UUID.fromString(id.substring(0, 8) + "-" + id.substring(8, 12) + "-" + id.substring(12, 16) + "-" + id.substring(16, 20) + "-" + id.substring(20, 32));
}

public static byte[] toBytes(UUID uuid) {
ByteBuffer byteBuffer = ByteBuffer.wrap(new byte[16]);
byteBuffer.putLong(uuid.getMostSignificantBits());
byteBuffer.putLong(uuid.getLeastSignificantBits());
return byteBuffer.array();
}
// Handle single batch of names
private static Map<String, UUID> fetchUUIDsForBatch(List<String> batch) {
Map<String, UUID> uuidMap = new HashMap<>();

public static UUID fromBytes(byte[] array) {
if (array.length != 16) {
throw new IllegalArgumentException("Illegal byte array length: " + array.length);
for (String name : batch) {
uuidMap.computeIfAbsent(name, k -> {
try {
return tryFetchUUIDWithFallback(k);
} catch (IOException e) {
throw new RuntimeException(e);
}
});
}
ByteBuffer byteBuffer = ByteBuffer.wrap(array);
long mostSignificant = byteBuffer.getLong();
long leastSignificant = byteBuffer.getLong();
return new UUID(mostSignificant, leastSignificant);
}

public static UUID getUUIDOf(String name) throws IOException, InterruptedException {
return new UUIDFetcher(Collections.singletonList(name)).call().get(name);
return uuidMap;
}

public static UUID getUUIDOfThrottled(String name) throws IOException, InterruptedException {
return new UUIDFetcher(Collections.singletonList(name), true).call().get(name);
private static @Nullable UUID tryFetchUUIDWithFallback(@NotNull String name) throws IOException {
try {
// Try the primary URL
return fetchUUID(URI.create(PROFILE_URL + name).toURL());
} catch (IOException e) {
// If the primary URL fails, attempt the fallback URL
SimpleClans.debug(String.format("Failed to fetch %s UUID by MineTools API. Trying to use Mojang API instead...", name));
return fetchUUID(URI.create(FALLBACK_PROFILE_URL + name).toURL());
}
}

@Override
public Map<String, UUID> call() throws IOException, InterruptedException {
Map<String, UUID> uuidMap = new HashMap<>();
int requests = (int) Math.ceil(names.size() / PROFILES_PER_REQUEST);
for (int i = 0; i < requests; i++) {
HttpURLConnection connection = createConnection();
List<String> namesToFetch = names.subList(i * 100, Math.min((i + 1) * 100, names.size()));
String body = gson.toJson(namesToFetch);
writeBody(connection, body);
JsonArray array = gson.fromJson(new InputStreamReader(connection.getInputStream(), UTF_8), JsonArray.class);
for (JsonElement profile : array) {
JsonObject jsonProfile = profile.getAsJsonObject();
String id = jsonProfile.get("id").getAsString();
String name = jsonProfile.get("name").getAsString();
UUID uuid = UUIDFetcher.getUUID(id);
uuidMap.put(name, uuid);
}
if (rateLimiting && i != requests - 1) {
Thread.sleep(10L);
}
private static @Nullable UUID fetchUUID(@NotNull URL url) throws IOException {
var connection = createConnection(url);
if (connection.getResponseCode() != HttpURLConnection.HTTP_OK) {
throw new IOException(String.format("Unexpected response code: %d. Response: %s",
connection.getResponseCode(), connection.getResponseMessage()));
}
return uuidMap;

JsonObject response = gson.fromJson(new InputStreamReader(connection.getInputStream(), UTF_8), JsonObject.class);

if (!response.has("id")) {
return null;
}

JsonElement id = response.get("id");
JsonElement status = response.get("status");

if (id.isJsonNull() ||
(response.has("status") && status.getAsString().equals("ERR"))) {
throw new IOException(String.format("Invalid UUID: %s", id));
}

return getUUID(id.getAsString());
}

// Callable task for batch processing
private static Callable<Map<String, UUID>> createTask(List<String> batch) {
return () -> fetchUUIDsForBatch(batch);
}
}
}
10 changes: 3 additions & 7 deletions src/test/java/net/sacredlabyrinth/phaed/simpleclans/uuid/UUIDFetcherTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,10 +3,9 @@
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;

import java.io.IOException;
import java.util.UUID;

import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.*;

public class UUIDFetcherTest {

Expand All @@ -19,11 +18,8 @@ public void setup() {

@Test
public void getUUIDOf() throws Exception {
assertEquals(ghostUUID, UUIDFetcher.getUUIDOf("GhostTheWolf"));
}
assertEquals(ghostUUID, UUIDFetcher.getUUIDOf("GhostTheWolf"), "Assert fetching UUID from right name");

@Test
public void getUUIDOfThrottled() throws IOException, InterruptedException {
assertEquals(ghostUUID, UUIDFetcher.getUUIDOfThrottled("GhostTheWolf"));
assertNull(UUIDFetcher.getUUIDOf("AnyNameLongerThan16Chars"), "Assert fetching UUID from wrong name");
}
}