Skip to content

Commit

Permalink
Merge pull request #20 from Best-offer-finder/BOF-42-Run-timer-on-eve…
Browse files Browse the repository at this point in the history 
…ry-scheme-edition

BOF-42 Run timer on every scheme edition
  • Loading branch information
@Vertonowsky
Vertonowsky authored Jun 10, 2024
2 parents 658a5c2 + 8358bb1 commit 31d52a9
Show file tree
Hide file tree
Showing 3 changed files with 201 additions and 196 deletions.
2 changes: 1 addition & 1 deletion src/main/java/com/example/backend/common/ErrorCode.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ public enum ErrorCode {

APPLICATION_SETTING_NOT_FOUND("APPLICATION_SETTING_NOT_FOUND"),
INPUT_DATA_INVALID("INPUT_DATA_INVALID"),
JSON_FORMAT_INVALID("JSON_FORMAT_INVALID"),
SCHEME_DATA_INVALID("SCHEME_DATA_INVALID"),
SCHEME_DATA_REQUIRED("SCHEME_DATA_REQUIRED"),
SCHEME_NOT_FOUND("SCHEME_NOT_FOUND"),
SCHEME_NAME_REQUIRED("SCHEME_NAME_REQUIRED"),
Expand Down
207 changes: 195 additions & 12 deletions src/main/java/com/example/backend/scheme/service/SchemeService.java
View file
Original file line number Diff line number Diff line change
@@ -1,42 +1,66 @@
package com.example.backend.scheme.service;

import com.example.backend.common.Collections;
import com.example.backend.common.ErrorCode;
import com.example.backend.exception.exceptions.EntityNotFoundException;
import com.example.backend.exception.exceptions.IllegalInputException;
import com.example.backend.scheme.CarStatus;
import com.example.backend.scheme.dto.SchemeDto;
import com.example.backend.scheme.model.Scheme;
import com.example.backend.scheme.model.SchemeToCar;
import com.example.backend.scheme.repository.SchemeRepository;
import com.example.backend.scheme.repository.SchemeToCarRepository;
import com.example.backend.setting.Key;
import com.example.backend.setting.service.ApplicationSettingService;
import com.example.backend.timer.otomoto.request.OtomotoDto;
import com.example.backend.timer.otomoto.response.*;
import com.example.backend.user.model.User;
import com.example.backend.user.service.UserService;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import jakarta.persistence.EntityManager;
import jakarta.transaction.Transactional;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.Pageable;
import org.springframework.http.HttpEntity;
import org.springframework.http.HttpHeaders;
import org.springframework.http.MediaType;
import org.springframework.http.ResponseEntity;
import org.springframework.stereotype.Service;
import org.springframework.web.client.RestTemplate;

import java.util.List;
import java.util.Set;
import java.util.*;
import java.util.stream.Collectors;

import static com.example.backend.common.Strings.isNullOrEmpty;

@Service
public class SchemeService {

public static final String TIMER_NAME = "OtomotoTimer";
private static final String EMPTY_JSON_OBJECT = "{}";
private static final Logger LOGGER = LoggerFactory.getLogger(SchemeService.class);
private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
private static final int PAGE_SIZE = 32;

@Value("${otomoto.search.api.url}")
private String destinationPath;

private final EntityManager em;
private final SchemeRepository schemeRepository;
private final SchemeToCarRepository schemeToCarRepository;
private final ApplicationSettingService applicationSettingService;
private final UserService userService;
private static final String EMPTY_JSON_OBJECT = "{}";

public SchemeService(EntityManager em, SchemeRepository schemeRepository, SchemeToCarRepository schemeToCarRepository, UserService userService) {
public SchemeService(EntityManager em, SchemeRepository schemeRepository, SchemeToCarRepository schemeToCarRepository,
ApplicationSettingService applicationSettingService, UserService userService) {
this.em = em;
this.schemeRepository = schemeRepository;
this.schemeToCarRepository = schemeToCarRepository;
this.applicationSettingService = applicationSettingService;
this.userService = userService;
}

Expand All @@ -54,7 +78,7 @@ public Scheme create(String email, SchemeDto schemeDto) throws EntityNotFoundExc
if (isNullOrEmpty(schemeDto.getName()))
throw new IllegalInputException(ErrorCode.SCHEME_NAME_REQUIRED);

validateJson(schemeDto.getData());
validateData(schemeDto.getData());

Scheme scheme = new Scheme();
scheme.setUser(user);
Expand All @@ -70,11 +94,15 @@ public Scheme edit(Long id, String email, SchemeDto schemeDto) throws IllegalInp
if (!isNullOrEmpty(schemeDto.getName()))
scheme.setName(schemeDto.getName());

boolean sameData = scheme.getData().equals(schemeDto.getData());
if (!isNullOrEmpty(schemeDto.getData())) {
validateJson(schemeDto.getData());
validateData(schemeDto.getData());
scheme.setData(schemeDto.getData());
}

if (!scheme.getData().equals(EMPTY_JSON_OBJECT) && !sameData)
loadDataForSingleScheme(scheme, getTimerQuery(), true);

return schemeRepository.save(scheme);
}

Expand Down Expand Up @@ -105,15 +133,170 @@ public void addCarsToScheme(Set<SchemeToCar> schemeToCarSet) {
schemeToCarRepository.saveAll(schemeToCarSet);
}

private void validateJson(String string) throws IllegalInputException {
if (string.equals(EMPTY_JSON_OBJECT))
throw new IllegalInputException(ErrorCode.JSON_FORMAT_INVALID);
public String getTimerQuery() throws EntityNotFoundException {
String query = applicationSettingService.getString(Key.OTOMOTO_QUERY_STRING);

try {
return OBJECT_MAPPER.readValue(query, OtomotoDto.class).getQuery();
} catch (JsonProcessingException e) {
LOGGER.error("[{}] Invalid query: {}", TIMER_NAME, query, e);
return "";
}
}

/**
* Prepare dto which will be sent to otomoto REST API
* and run loop searching for new cars.
* <p>
* Loop is finished when there are no more new cars
* (objects from API are repeating with objects inside database)
*
* @param scheme currently processed scheme
* @param query final GraphQL indicating String query
*/
public void loadDataForSingleScheme(Scheme scheme, String query, boolean oneTime) {
OtomotoDto otomotoDto;
try {
otomotoDto = OBJECT_MAPPER.readValue(scheme.getData(), OtomotoDto.class);
otomotoDto.setQuery(query);
} catch (JsonProcessingException e) {
LOGGER.error("[{}][SchemeId: {}] There was an error while parsing data.", TIMER_NAME, scheme.getId(), e);
return;
}

if (otomotoDto.getVariables() == null) {
LOGGER.error("[{}][SchemeId: {}] Scheme has incorrect json data.", TIMER_NAME, scheme.getId());
return;
}

int page = 1;
boolean shouldFinish = false;
while (!shouldFinish) {
shouldFinish = processSinglePage(otomotoDto, scheme, page);
page++;
if (oneTime)
shouldFinish = true;
}
}

/**
* Create dto with next pages and send POST request
*
* @param otomotoDto - dto sent to REST API
* @param scheme currently processed scheme
* @param page page number. There are 32 elements per page
* @return true if there are no more new elements and timer for certain scheme should be finished
*/
private boolean processSinglePage(OtomotoDto otomotoDto, Scheme scheme, int page) {
try {
otomotoDto.getVariables().setPage(page);

OtomotoResponseDto otomotoResponseDto = getPostResponse(OBJECT_MAPPER.writeValueAsString(otomotoDto));
return calculatePageWithOtomotoData(scheme, otomotoResponseDto);

} catch (JsonProcessingException e) {
LOGGER.error("[{}] Error generating POST request.", TIMER_NAME, e);
return true;
}
}

/**
* Send POST request to the destination path
*
* @param jsonData json combining scheme data specified by user
* @return OtomotoResponseDto containing car data
*/
private OtomotoResponseDto getPostResponse(String jsonData) {
HttpHeaders headers = new HttpHeaders();
headers.setContentType(MediaType.APPLICATION_JSON);

HttpEntity<String> requestEntity = new HttpEntity<>(jsonData, headers);
ResponseEntity<String> responseEntity = new RestTemplate().postForEntity(destinationPath, requestEntity, String.class);

LOGGER.debug("Response code: {}", responseEntity.getStatusCode());

try {
return OBJECT_MAPPER.readValue(responseEntity.getBody(), OtomotoResponseDto.class);
} catch (JsonProcessingException e) {
LOGGER.error("[{}] There was an error while parsing data.", TIMER_NAME, e);
return new OtomotoResponseDto();
}
}

/**
* Calculate current scheme.
* If there are new cars, then they should be saved inside the database.
* Otherwise, scheme is being ignored.
*
* @param scheme currently processed scheme
* @param otomotoResponseDto response from REST API
* @return true if the process should stop and there won't be any more cars
*/
private boolean calculatePageWithOtomotoData(Scheme scheme, OtomotoResponseDto otomotoResponseDto) {
Set<NodeParentDto> newCars = Optional.ofNullable(otomotoResponseDto)
.map(OtomotoResponseDto::getData)
.map(DataDto::getAdvertSearch)
.map(AdvertSearchDto::getEdges)
.orElse(new HashSet<>());

if (Collections.isNullOrEmpty(newCars))
return true;

fetchCars(scheme);
Set<Long> newCarsIds = newCars.stream().map(NodeParentDto::getNode).map(NodeDto::getId).collect(Collectors.toSet());
Set<Long> oldCarsIds = Collections.isNullOrEmpty(scheme.getSchemeToCars()) ? new HashSet<>() : scheme.getSchemeToCars().stream().map(SchemeToCar::getCarId).collect(Collectors.toSet());

Set<Long> diff = newCarsIds.stream().filter(newCarElement -> !oldCarsIds.contains(newCarElement)).collect(Collectors.toSet());
Map<Long, NodeDto> carIdToNodeDtoMap = newCars.stream()
.map(NodeParentDto::getNode).filter((node) -> diff.contains(node.getId()))
.collect(Collectors.toMap(NodeDto::getId, node -> {
node.setId(null);
return node;
}));

LOGGER.info("[{}][SchemeId: {}] Found {} new cars: {}", TIMER_NAME, scheme.getId(), diff.size(), Collections.toString(diff));

// Save all new cars into the database
boolean firstInitialization = scheme.getSchemeToCars().isEmpty();
saveNewCarsToDatabase(carIdToNodeDtoMap, scheme, firstInitialization);

if (firstInitialization)
return true;

if (diff.size() == PAGE_SIZE)
return false;

return diff.isEmpty() || diff.size() < PAGE_SIZE;
}

/**
* Insert car's ids into the database.
*
* @param carIdToNodeDtoMap Set of car's ids and their response data (NodeDto)
* @param scheme currently processed scheme
* @param firstInitialization true if this is the first ever loop for the scheme
*/
private void saveNewCarsToDatabase(Map<Long, NodeDto> carIdToNodeDtoMap, Scheme scheme, boolean firstInitialization) {
Set<SchemeToCar> schemeToCarSet = carIdToNodeDtoMap.entrySet().stream()
.map(entry -> {
try {
String carNode = OBJECT_MAPPER.writeValueAsString(entry.getValue());
return SchemeToCar.of(scheme, entry.getKey(), carNode, firstInitialization ? CarStatus.FIRST_INITIALIZATION : CarStatus.NEW);
} catch (JsonProcessingException ex) {
LOGGER.error("[{}][SchemeId: {}] There was an error while parsing car data for carId: {}.", TIMER_NAME, scheme.getId(), entry.getKey(), ex);
return null;
}
})
.collect(Collectors.toSet());

addCarsToScheme(schemeToCarSet);
}

private void validateData(String data) throws IllegalInputException {
try {
ObjectMapper mapper = new ObjectMapper();
mapper.readTree(string);
OBJECT_MAPPER.readValue(data, OtomotoDto.class);
} catch (JsonProcessingException e) {
throw new IllegalInputException(ErrorCode.JSON_FORMAT_INVALID);
throw new IllegalInputException(ErrorCode.SCHEME_DATA_INVALID);
}
}

Expand Down
Loading

0 comments on commit 31d52a9

Please sign in to comment.