diff --git a/avni-server-api/src/main/java/org/avni/server/service/MetadataBundleAndFileHandler.java b/avni-server-api/src/main/java/org/avni/server/service/MetadataBundleAndFileHandler.java new file mode 100644 index 000000000..aa8fd44a8 --- /dev/null +++ b/avni-server-api/src/main/java/org/avni/server/service/MetadataBundleAndFileHandler.java @@ -0,0 +1,106 @@ +package org.avni.server.service; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Service; +import org.springframework.web.multipart.MultipartFile; + +import java.io.*; +import java.nio.file.Files; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.zip.ZipEntry; +import java.util.zip.ZipInputStream; + +@Service +public class MetadataBundleAndFileHandler { + + private static final ObjectMapper objectMapper = new ObjectMapper(); + + protected File extractZip(MultipartFile zipFile) throws IOException { + File tempDir = Files.createTempDirectory("metadata-zip").toFile(); + + try (ZipInputStream zipInputStream = new ZipInputStream(zipFile.getInputStream())) { + ZipEntry entry; + while ((entry = zipInputStream.getNextEntry()) != null) { + if (!entry.isDirectory()) { + File file = new File(tempDir, entry.getName()); + File parentDir = file.getParentFile(); + if (!parentDir.exists()) { + parentDir.mkdirs(); + } + + try (OutputStream outputStream = new FileOutputStream(file)) { + byte[] buffer = new byte[1024]; + int length; + while ((length = zipInputStream.read(buffer)) > 0) { + outputStream.write(buffer, 0, length); + } + } + } + zipInputStream.closeEntry(); + } + } + return tempDir; + } + + protected List listJsonFiles(File directory) { + List jsonFiles = new ArrayList<>(); + File[] files = directory.listFiles(); + if (files != null) { + for (File file : files) { + if (file.isDirectory()) { + jsonFiles.addAll(listJsonFiles(file)); + } else if (file.isFile() && file.getName().toLowerCase().endsWith(".json")) { + jsonFiles.add(file); + } + } + } + return jsonFiles; + } + + protected Map> parseJsonFiles(List files, File rootDir) throws IOException { + Map> jsonMap = new HashMap<>(); + + for (File file : files) { + String relativePath = getRelativePath(file, rootDir); + try (BufferedReader reader = new BufferedReader(new FileReader(file))) { + StringBuilder jsonContent = new StringBuilder(); + String line; + while ((line = reader.readLine()) != null) { + jsonContent.append(line); + } + + Map jsonMapFile = new HashMap<>(); + if (jsonContent.toString().trim().startsWith("[")) { + List> jsonArray = objectMapper.readValue(jsonContent.toString(), new TypeReference>>() {}); + for (Map jsonObject : jsonArray) { + String uuid = (String) jsonObject.get("uuid"); + if (uuid != null) { + jsonObject.remove("filename"); + jsonMapFile.put(uuid, jsonObject); + } + } + } else { + Map jsonObject = objectMapper.readValue(jsonContent.toString(), new TypeReference>() {}); + String uuid = (String) jsonObject.get("uuid"); + if (uuid != null) { + jsonObject.remove("filename"); + jsonMapFile.put(uuid, jsonObject); + } + } + jsonMap.put(relativePath, jsonMapFile); + } + } + return jsonMap; + } + private String getRelativePath(File file, File rootDir) { + String filePath = file.getPath(); + String rootPath = rootDir.getPath(); + return filePath.substring(rootPath.length() + 1); + } +} diff --git a/avni-server-api/src/main/java/org/avni/server/service/MetadataDiffChecker.java b/avni-server-api/src/main/java/org/avni/server/service/MetadataDiffChecker.java new file mode 100644 index 000000000..224c15eab --- /dev/null +++ b/avni-server-api/src/main/java/org/avni/server/service/MetadataDiffChecker.java @@ -0,0 +1,152 @@ +package org.avni.server.service; + +import org.springframework.stereotype.Service; + +import java.util.*; +import java.util.function.Function; +import java.util.stream.Collectors; + +@Service +public class MetadataDiffChecker { + + public static final String MODIFIED = "modified"; + public static final String ADDED = "added"; + public static final String REMOVED = "removed"; + public static final String NO_MODIFICATION = "noModification"; + + MetadataDiffOutputGenerator metadataDiffOutputGenerator; + + public MetadataDiffChecker(MetadataDiffOutputGenerator metadataDiffOutputGenerator) { + this.metadataDiffOutputGenerator = metadataDiffOutputGenerator; + } + + protected Map findDifferences(Map jsonMap1, Map jsonMap2) { + Map differences = new HashMap<>(); + boolean hasDifferences = false; + String uuid = "null"; + for (Map.Entry entry : jsonMap1.entrySet()) { + uuid = entry.getKey(); + Object json1 = entry.getValue(); + Object json2 = jsonMap2.get(uuid); + if (json2 != null) { + Map diff = findJsonDifferences(castToStringObjectMap(json1), castToStringObjectMap(json2)); + if (!diff.isEmpty()) { + differences.put(uuid, diff); + hasDifferences = true; + } + } else { + differences.put(uuid, metadataDiffOutputGenerator.createFieldDiff( json1, null,REMOVED)); + hasDifferences = true; + } + } + + for (Map.Entry entry : jsonMap2.entrySet()) { + String uuid2 = entry.getKey(); + if (!jsonMap1.containsKey(uuid2)) { + differences.put(uuid2, metadataDiffOutputGenerator.createFieldDiff(null, entry.getValue(), ADDED)); + hasDifferences = true; + } + } + + if (!hasDifferences) { + differences.put(uuid, metadataDiffOutputGenerator.createFieldDiff(null, null, NO_MODIFICATION)); + } + return differences; + } + + protected Map findJsonDifferences(Map json1, Map json2) { + Map differences = new LinkedHashMap<>(); + if (json1 == null && json2 == null) { + return differences; + } + + if (json1 == null) { + json2.forEach((key, value) -> differences.put(key, metadataDiffOutputGenerator.createFieldDiff(null, value, ADDED))); + return differences; + } + + if (json2 == null) { + json1.forEach((key, value) -> differences.put(key, metadataDiffOutputGenerator.createFieldDiff(value, null, REMOVED))); + return differences; + } + + for (Map.Entry entry : json1.entrySet()) { + String key = entry.getKey(); + Object value1 = entry.getValue(); + Object value2 = json2.get(key); + + if (key.equals("id")) { + continue; + } + if (value2 == null) { + differences.put(key, metadataDiffOutputGenerator.createFieldDiff(value1, null, REMOVED)); + } else { + if (value1 instanceof Map && value2 instanceof Map) { + Map subDiff = findJsonDifferences((Map) value1, (Map) value2); + if (!subDiff.isEmpty()) { + differences.put(key, metadataDiffOutputGenerator.createObjectDiff(subDiff, MODIFIED)); + } + } else if (value1 instanceof List && value2 instanceof List) { + List> listDiff = findArrayDifferences((List) value1, (List) value2); + if (!listDiff.isEmpty()) { + differences.put(key, metadataDiffOutputGenerator.createArrayDiff(listDiff, MODIFIED)); + } + } else if (!value1.equals(value2)) { + differences.put(key, metadataDiffOutputGenerator.createFieldDiff(value1, value2, MODIFIED)); + } + } + } + + for (Map.Entry entry : json2.entrySet()) { + String key = entry.getKey(); + if (!json1.containsKey(key)) { + differences.put(key, metadataDiffOutputGenerator.createFieldDiff(null, entry.getValue(), ADDED)); + } + } + + return differences; + } + + protected List> findArrayDifferences(List array1, List array2) { + List> differences = new ArrayList<>(); + + Function, String> getUuid = obj -> (String) obj.get("uuid"); + + Map> map1 = array1.stream() + .filter(obj -> obj instanceof Map) + .map(obj -> (Map) obj) + .collect(Collectors.toMap(getUuid, Function.identity(), (e1, e2) -> e1)); + + Map> map2 = array2.stream() + .filter(obj -> obj instanceof Map) + .map(obj -> (Map) obj) + .collect(Collectors.toMap(getUuid, Function.identity(), (e1, e2) -> e1)); + + for (String uuid : map2.keySet()) { + if (!map1.containsKey(uuid)) { + differences.add(metadataDiffOutputGenerator.createFieldDiff(null, map2.get(uuid), ADDED)); + } else { + Map obj1 = map1.get(uuid); + Map obj2 = map2.get(uuid); + + Map subDiff = findJsonDifferences(obj1, obj2); + if (!subDiff.isEmpty()) { + differences.add(metadataDiffOutputGenerator.createObjectDiff(subDiff, MODIFIED)); + } + } + } + + for (String uuid : map1.keySet()) { + if (!map2.containsKey(uuid)) { + differences.add(metadataDiffOutputGenerator.createFieldDiff(map1.get(uuid), null, REMOVED)); + } + } + return differences; + } + private Map castToStringObjectMap(Object obj) { + if (obj instanceof Map) { + return (Map) obj; + } + return new HashMap<>(); + } +} diff --git a/avni-server-api/src/main/java/org/avni/server/service/MetadataDiffOutputGenerator.java b/avni-server-api/src/main/java/org/avni/server/service/MetadataDiffOutputGenerator.java new file mode 100644 index 000000000..07c8b4932 --- /dev/null +++ b/avni-server-api/src/main/java/org/avni/server/service/MetadataDiffOutputGenerator.java @@ -0,0 +1,77 @@ +package org.avni.server.service; + +import org.springframework.stereotype.Service; + +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +@Service +public class MetadataDiffOutputGenerator { + + public static final String NO_MODIFICATION = "noModification"; + public static final String OLD_VALUE = "oldValue"; + public static final String NEW_VALUE = "newValue"; + public static final String CHANGE_TYPE = "changeType"; + public static final String DATA_TYPE = "dataType"; + public static final String OBJECT = "object"; + public static final String FIELD = "field"; + public static final String ARRAY = "array"; + public static final String PRIMITIVE = "primitive"; + public static final String ITEMS = "items"; + + protected Map createFieldDiff(Object oldValue, Object newValue, String changeType) { + Map fieldDiff = new LinkedHashMap<>(); + + if(!NO_MODIFICATION.equals(changeType)) { + if (oldValue == null && newValue != null) { + fieldDiff.put(DATA_TYPE, getDataType(newValue)); + } else if (oldValue != null && newValue == null) { + fieldDiff.put(DATA_TYPE, getDataType(oldValue)); + } else if (oldValue != null && newValue != null) { + fieldDiff.put(DATA_TYPE, getDataType(newValue)); + } else { + fieldDiff.put(DATA_TYPE, OBJECT); + } + } + fieldDiff.put(CHANGE_TYPE, changeType); + if (oldValue != null) { + fieldDiff.put(OLD_VALUE, oldValue); + } + if (newValue != null) { + fieldDiff.put(NEW_VALUE, newValue); + } + return fieldDiff; + } + + protected Map createObjectDiff(Map fieldsDiff, String changeType) { + Map objectDiff = new LinkedHashMap<>(); + + if (!fieldsDiff.isEmpty() && !NO_MODIFICATION.equals(changeType)) { + objectDiff.put(DATA_TYPE, OBJECT); + objectDiff.put(CHANGE_TYPE, changeType); + objectDiff.put(FIELD, fieldsDiff); + } + return objectDiff; + } + + protected Map createArrayDiff(List> itemsDiff, String changeType) { + Map arrayDiff = new LinkedHashMap<>(); + + if (!itemsDiff.isEmpty() && !NO_MODIFICATION.equals(changeType)) { + arrayDiff.put(DATA_TYPE, ARRAY); + arrayDiff.put(CHANGE_TYPE, changeType); + arrayDiff.put(ITEMS, itemsDiff); + } + return arrayDiff; + } + private String getDataType(Object value) { + if (value instanceof Map) { + return OBJECT; + } else if (value instanceof List) { + return ARRAY; + } else { + return PRIMITIVE; + } + } +} diff --git a/avni-server-api/src/main/java/org/avni/server/service/MetadataDiffService.java b/avni-server-api/src/main/java/org/avni/server/service/MetadataDiffService.java new file mode 100644 index 000000000..10348a2ab --- /dev/null +++ b/avni-server-api/src/main/java/org/avni/server/service/MetadataDiffService.java @@ -0,0 +1,111 @@ +package org.avni.server.service; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Service; +import org.springframework.web.multipart.MultipartFile; +import com.fasterxml.jackson.databind.ObjectMapper; + +import java.io.*; +import java.util.*; + +@Service +public class MetadataDiffService { + + private static final Logger logger = LoggerFactory.getLogger(MetadataDiffService.class); + private static final ObjectMapper objectMapper = new ObjectMapper(); + + private final MetadataBundleAndFileHandler bundleAndFileHandler; + private final MetadataDiffChecker diffChecker; + private final MetadataDiffOutputGenerator outputGenerator; + + public MetadataDiffService(MetadataBundleAndFileHandler bundleAndFileHandler, MetadataDiffChecker diffChecker, MetadataDiffOutputGenerator outputGenerator) { + this.bundleAndFileHandler = bundleAndFileHandler; + this.diffChecker = diffChecker; + this.outputGenerator = outputGenerator; + } + + public Map compareMetadataZips(MultipartFile zipFile1, MultipartFile zipFile2) throws IOException { + Map result = new HashMap<>(); + File tempDir1 = null, tempDir2 = null; + + try { + tempDir1 = bundleAndFileHandler.extractZip(zipFile1); + tempDir2 = bundleAndFileHandler.extractZip(zipFile2); + + List files1 = bundleAndFileHandler.listJsonFiles(tempDir1); + List files2 = bundleAndFileHandler.listJsonFiles(tempDir2); + + Map> jsonMap1 = bundleAndFileHandler.parseJsonFiles(files1, tempDir1); + Map> jsonMap2 = bundleAndFileHandler.parseJsonFiles(files2, tempDir2); + + Set fileNames1 = jsonMap1.keySet(); + Set fileNames2 = jsonMap2.keySet(); + + Set commonFileNames = new HashSet<>(fileNames1); + commonFileNames.retainAll(fileNames2); + + for (String fileName : commonFileNames) { + Map jsonMapFile1 = jsonMap1.get(fileName); + Map jsonMapFile2 = jsonMap2.get(fileName); + + if (jsonMapFile1 != null && jsonMapFile2 != null) { + Map fileDifferences = diffChecker.findDifferences(jsonMapFile1, jsonMapFile2); + if (!fileDifferences.isEmpty()) { + result.put(fileName, fileDifferences); + } + } + } + + Set missingInZip1 = findMissingFiles(fileNames1, fileNames2); + if (!missingInZip1.isEmpty()) { + result.putAll(missingFilesMap(missingInZip1, "Missing Files in UAT ZIP")); + } + + Set missingInZip2 = findMissingFiles(fileNames2, fileNames1); + if (!missingInZip2.isEmpty()) { + result.putAll(missingFilesMap(missingInZip2, "Missing Files in PROD ZIP")); + } + + } catch (IOException e) { + logger.error("Error comparing metadata ZIPs: " + e.getMessage(), e); + Map errorResult = new HashMap<>(); + errorResult.put("error", "Error comparing metadata ZIPs: " + e.getMessage()); + result.put("error", errorResult); + } finally { + if (tempDir1 != null) { + deleteDirectory(tempDir1); + } + if (tempDir2 != null) { + deleteDirectory(tempDir2); + } + } + return result; + } + + protected Set findMissingFiles(Set fileNames1, Set fileNames2) { + Set missingFiles = new HashSet<>(fileNames1); + missingFiles.removeAll(fileNames2); + return missingFiles; + } + + protected Map missingFilesMap(Set missingFiles, String message) { + Map missingFilesMap = new LinkedHashMap<>(); + missingFilesMap.put(message, missingFiles); + return missingFilesMap; + } + + protected void deleteDirectory(File directory) { + File[] files = directory.listFiles(); + if (files != null) { + for (File file : files) { + if (file.isDirectory()) { + deleteDirectory(file); + } else { + file.delete(); + } + } + } + directory.delete(); + } +} diff --git a/avni-server-api/src/main/java/org/avni/server/web/MetadataDiffController.java b/avni-server-api/src/main/java/org/avni/server/web/MetadataDiffController.java new file mode 100644 index 000000000..ce4199669 --- /dev/null +++ b/avni-server-api/src/main/java/org/avni/server/web/MetadataDiffController.java @@ -0,0 +1,32 @@ +package org.avni.server.web; + +import org.avni.server.service.MetadataDiffService; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.http.*; +import org.springframework.security.access.prepost.PreAuthorize; +import org.springframework.web.bind.annotation.*; +import org.springframework.web.multipart.MultipartFile; + +import java.io.IOException; +import java.util.Map; + +@RestController +@RequestMapping("/api") +public class MetadataDiffController { + private final MetadataDiffService metadatadiffService; + + @Autowired + public MetadataDiffController(MetadataDiffService metadatadiffService) { + this.metadatadiffService = metadatadiffService; + } + + @PostMapping("/compare-metadata") + @PreAuthorize("hasAnyAuthority('user')") + public ResponseEntity compareMetadataZips(@RequestParam("file1") MultipartFile file1, + @RequestParam("file2") MultipartFile file2) throws IOException { + + Map result = metadatadiffService.compareMetadataZips(file1, file2); + + return ResponseEntity.ok(result); + } +} diff --git a/avni-server-api/src/test/java/org/avni/server/service/MetadataDiffServiceTest.java b/avni-server-api/src/test/java/org/avni/server/service/MetadataDiffServiceTest.java new file mode 100644 index 000000000..53b5c1ebc --- /dev/null +++ b/avni-server-api/src/test/java/org/avni/server/service/MetadataDiffServiceTest.java @@ -0,0 +1,98 @@ +package org.avni.server.service; + +import static org.junit.Assert.*; +import static org.mockito.Mockito.*; + +import org.junit.Before; +import org.junit.Test; +import org.springframework.mock.web.MockMultipartFile; +import org.springframework.web.multipart.MultipartFile; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.util.*; +import java.util.HashMap; +import java.util.Map; +import java.util.zip.ZipEntry; +import java.util.zip.ZipOutputStream; + +public class MetadataDiffServiceTest { + + private MetadataBundleAndFileHandler bundleAndFileHandler; + private MetadataDiffChecker diffChecker; + private MetadataDiffOutputGenerator outputGenerator; + private MetadataDiffService metadataDiffService; + + @Before + public void setUp() { + bundleAndFileHandler = mock(MetadataBundleAndFileHandler.class); + diffChecker = mock(MetadataDiffChecker.class); + outputGenerator = mock(MetadataDiffOutputGenerator.class); + metadataDiffService = new MetadataDiffService(bundleAndFileHandler, diffChecker, outputGenerator); + } + + @Test + public void testCompareMetadataZips() throws IOException { + MultipartFile zipFile1 = createMultipartFile("file1.json", "{\"key\":\"value1\"}"); + MultipartFile zipFile2 = createMultipartFile("file1.json", "{\"key\":\"value2\"}"); + + assertNotNull(zipFile1); + assertNotNull(zipFile2); + + Map differences = metadataDiffService.compareMetadataZips(zipFile1, zipFile2); + + assertNotNull(differences); + assertEquals(1, differences.size()); + } + + private MultipartFile createMultipartFile(String fileName, String jsonContent) throws IOException { + ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); + try (ZipOutputStream zipOutputStream = new ZipOutputStream(byteArrayOutputStream)) { + ZipEntry zipEntry = new ZipEntry(fileName); + zipOutputStream.putNextEntry(zipEntry); + zipOutputStream.write(jsonContent.getBytes()); + zipOutputStream.closeEntry(); + } + return new MockMultipartFile("file", "test.zip", "application/zip", byteArrayOutputStream.toByteArray()); + } + + @Test + public void testFindDifferences() { + Map jsonMap1 = new HashMap<>(); + Map jsonMap2 = new HashMap<>(); + + jsonMap1.put("uuid1", createJsonObject("value1")); + jsonMap2.put("uuid1", createJsonObject("value2")); + jsonMap2.put("uuid2", createJsonObject("value3")); + + Map differences = diffChecker.findDifferences(jsonMap1, jsonMap2); + + assertNotNull(differences); + + assertTrue(differences.containsKey("uuid1")); + assertTrue(differences.containsKey("uuid2")); + } + + @Test + public void testFindMissingFiles() { + Set fileNames1 = new HashSet<>(); + Set fileNames2 = new HashSet<>(); + + fileNames1.add("file1.json"); + fileNames1.add("file2.json"); + fileNames2.add("file1.json"); + + Set missingFiles = metadataDiffService.findMissingFiles(fileNames1, fileNames2); + + assertNotNull(missingFiles); + + assertTrue(missingFiles.contains("file2.json")); + assertFalse(missingFiles.contains("file1.json")); + } + + private Map createJsonObject(String value) { + Map jsonObject = new HashMap<>(); + jsonObject.put("key", value); + return jsonObject; + } +} diff --git a/avni-server-api/src/test/java/org/avni/server/web/MetadataDiffControllerIntegrationTest.java b/avni-server-api/src/test/java/org/avni/server/web/MetadataDiffControllerIntegrationTest.java new file mode 100644 index 000000000..71eb2caa8 --- /dev/null +++ b/avni-server-api/src/test/java/org/avni/server/web/MetadataDiffControllerIntegrationTest.java @@ -0,0 +1,25 @@ +package org.avni.server.web; + +import org.avni.server.common.AbstractControllerIntegrationTest; +import org.junit.Test; +import org.springframework.http.MediaType; +import org.springframework.mock.web.MockMultipartFile; +import org.springframework.test.web.servlet.request.MockMvcRequestBuilders; + +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; + + +public class MetadataDiffControllerIntegrationTest extends AbstractControllerIntegrationTest { + + @Test + public void testCompareMetadataZips() throws Exception { + setUser("demo-user"); + MockMultipartFile file1 = new MockMultipartFile("file1", "file1.zip", MediaType.MULTIPART_FORM_DATA_VALUE, "zip file content".getBytes()); + MockMultipartFile file2 = new MockMultipartFile("file2", "file2.zip", MediaType.MULTIPART_FORM_DATA_VALUE, "zip file content".getBytes()); + + mockMvc.perform(MockMvcRequestBuilders.multipart("/api/compare-metadata") + .file(file1) + .file(file2)) + .andExpect(status().isOk()); + } +} \ No newline at end of file