diff --git a/build.gradle b/build.gradle index 6171aaf..5484cad 100644 --- a/build.gradle +++ b/build.gradle @@ -30,6 +30,8 @@ dependencies { annotationProcessor 'org.projectlombok:lombok' testImplementation 'org.springframework.boot:spring-boot-starter-test' + implementation 'org.jsoup:jsoup:1.13.1' + // swagger // implementation 'org.springdoc:springdoc-openapi-ui:1.6.15' // implementation 'io.springfox:springfox-swagger2:2.9.2' @@ -41,6 +43,13 @@ dependencies { // S3 설정 implementation "org.springframework.cloud:spring-cloud-starter-aws:2.2.6.RELEASE" + + // 캐싱 + implementation "org.springframework.boot:spring-boot-starter-cache" + + // Redis + implementation 'it.ozimov:embedded-redis:0.7.2' + implementation 'org.springframework.boot:spring-boot-starter-data-redis' } tasks.named('bootBuildImage') { diff --git a/src/main/java/dev/neordinary/zero/WebScraperTwo.java b/src/main/java/dev/neordinary/zero/WebScraperTwo.java new file mode 100644 index 0000000..87eaa84 --- /dev/null +++ b/src/main/java/dev/neordinary/zero/WebScraperTwo.java @@ -0,0 +1,87 @@ +package dev.neordinary.zero; + +import org.jsoup.Jsoup; +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; +import org.jsoup.select.Elements; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public class WebScraperTwo { + + public static void main(String[] args) { + String url = "https://www.fatsecret.kr/칼로리-영양소"+"/search?q=2%ED%94%84%EB%A1%9C"; + + try { + Document doc = Jsoup.connect(url).get(); + List productList = new ArrayList<>(); + + Elements productLinks = doc.select("a.prominent"); + + for (Element link : productLinks) { + String productName = link.text().trim(); + String productUrl = "https://www.fatsecret.kr" + link.attr("href"); + String productSugar = ""; + String productKcal = ""; + String productSize = ""; + try { + Document infoDoc = Jsoup.connect(productUrl).get(); + Elements body = infoDoc.getAllElements(); + System.out.println(body.text()); + Elements sugarElements = infoDoc.getElementsMatchingOwnText("설탕당"); + if (!sugarElements.isEmpty()) { + Element sugarElement = sugarElements.first(); + Element nextElement = sugarElement.nextElementSibling(); + if (nextElement != null) { + productSugar = nextElement.text(); + } else { + System.out.println("설탕당 값이 없습니다."); + } + } else { + System.out.println("설탕당을 찾을 수 없습니다."); + } + + Elements kcalElements = infoDoc.getElementsMatchingOwnText("열량"); + if (!kcalElements.isEmpty()) { + Element kcalElement = kcalElements.first(); + Element nextElement = kcalElement.nextElementSibling().nextElementSibling().nextElementSibling().nextElementSibling(); + if (nextElement != null) { + productKcal = nextElement.text(); + } else { + System.out.println("칼로리 값이 없습니다."); + } + } else { + System.out.println("칼로리를 찾을 수 없습니다."); + } + + String sizeElement = infoDoc.getElementsContainingOwnText("ml").toString(); + // 정규 표현식을 사용하여 괄호 안의 숫자 추출 + Pattern pattern = Pattern.compile("(\\d+) ml"); + Matcher matcher = pattern.matcher(sizeElement); + + // 정규 표현식과 일치하는 부분이 있는지 확인하고 숫자 추출 + if (matcher.find()) { + productSize = matcher.group(1); + } else { + System.out.println("용량이 없습니다."); + } + + } catch (IOException e) { + e.printStackTrace(); + } + productList.add(new String[]{productName, productSugar, productKcal, productSize}); + } + + for (String[] product : productList) { + System.out.println("제품 이름: " + product[0] + ", 당: " + product[1] + ", 칼로리: " + product[2] + ", 용량: " + product[3]); + } + + } catch (IOException e) { + e.printStackTrace(); + } + } +} diff --git a/src/main/java/dev/neordinary/zero/base/BaseException.java b/src/main/java/dev/neordinary/zero/base/BaseException.java new file mode 100644 index 0000000..e6b2515 --- /dev/null +++ b/src/main/java/dev/neordinary/zero/base/BaseException.java @@ -0,0 +1,12 @@ +package dev.neordinary.zero.base; + +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.Setter; + +@Getter +@Setter +@AllArgsConstructor +public class BaseException extends RuntimeException { + BaseResponseStatus errorStatus; +} diff --git a/src/main/java/dev/neordinary/zero/base/BaseExceptionHandler.java b/src/main/java/dev/neordinary/zero/base/BaseExceptionHandler.java new file mode 100644 index 0000000..3a1bc80 --- /dev/null +++ b/src/main/java/dev/neordinary/zero/base/BaseExceptionHandler.java @@ -0,0 +1,13 @@ +package dev.neordinary.zero.base; + +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.ControllerAdvice; +import org.springframework.web.bind.annotation.ExceptionHandler; + +@ControllerAdvice +public class BaseExceptionHandler { + @ExceptionHandler(BaseException.class) + protected ResponseEntity handleBaseException(BaseException e) { + return BaseResponse.toResponseEntityContainsStatus(e.getErrorStatus()); + } +} diff --git a/src/main/java/dev/neordinary/zero/base/BaseResponse.java b/src/main/java/dev/neordinary/zero/base/BaseResponse.java new file mode 100644 index 0000000..a1fa62c --- /dev/null +++ b/src/main/java/dev/neordinary/zero/base/BaseResponse.java @@ -0,0 +1,56 @@ +package dev.neordinary.zero.base; + + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonPropertyOrder; +import lombok.Builder; +import lombok.Data; +import org.springframework.http.ResponseEntity; + +import static dev.neordinary.zero.base.BaseResponseStatus.SUCCESS; + +@Data +@Builder +@JsonPropertyOrder({"status", "code", "message", "result"}) +public class BaseResponse { // BaseResponse 객체를 사용할때 성공, 실패 경우 + private final int status; + private final String code; + private final String message; + @JsonInclude(JsonInclude.Include.NON_NULL) + private Object result; + + // Custom Status를 포함한 Response + public static ResponseEntity toResponseEntityContainsStatus(BaseResponseStatus baseResponseStatus) { + return ResponseEntity + .status(baseResponseStatus.getStatus()) + .body(BaseResponse.builder() + .status(baseResponseStatus.getStatus().value()) + .code(baseResponseStatus.getCode()) + .message(baseResponseStatus.getMessage()) + .build()); + } + + // Http 200, Result를 포함한 Response + public static ResponseEntity toResponseEntityContainsResult(Object result) { + return ResponseEntity + .status(SUCCESS.getStatus()) + .body(BaseResponse.builder() + .status(SUCCESS.getStatus().value()) + .code(SUCCESS.getCode()) + .message(SUCCESS.getMessage()) + .result(result) + .build()); + } + + // Custom Status, Result를 포함한 Response + public static ResponseEntity toResponseEntityContainsStatusAndResult(BaseResponseStatus baseResponseStatus, Object result) { + return ResponseEntity + .status(baseResponseStatus.getStatus()) + .body(BaseResponse.builder() + .status(baseResponseStatus.getStatus().value()) + .code(baseResponseStatus.getCode()) + .message(baseResponseStatus.getMessage()) + .result(result) + .build()); + } +} diff --git a/src/main/java/dev/neordinary/zero/base/BaseResponseStatus.java b/src/main/java/dev/neordinary/zero/base/BaseResponseStatus.java new file mode 100644 index 0000000..260ac11 --- /dev/null +++ b/src/main/java/dev/neordinary/zero/base/BaseResponseStatus.java @@ -0,0 +1,48 @@ +package dev.neordinary.zero.base; + +import lombok.AllArgsConstructor; +import lombok.Getter; +import org.springframework.http.HttpStatus; + +@Getter +@AllArgsConstructor +public enum BaseResponseStatus implements BaseResponseStatusImpl { + /** + * 100 : 진행 정보 + */ + + /** + * 200 : 요청 성공 + */ + SUCCESS(HttpStatus.OK, "SUCCESS", "요청에 성공했습니다."), + CREATED(HttpStatus.CREATED, "CREATED", "요청에 성공했으며 리소스가 정상적으로 생성되었습니다."), + ACCEPTED(HttpStatus.ACCEPTED, "ACCEPTED", "요청에 성공했으나 처리가 완료되지 않았습니다."), + DELETED(HttpStatus.NO_CONTENT, "DELETED", "요청에 성공했으며 더 이상 응답할 내용이 존재하지 않습니다."), + + /** + * 300 : 리다이렉션 + */ + SEE_OTHER(HttpStatus.SEE_OTHER, "REDIRECT", "다른 주소로 요청해주세요."), + + /** + * 400 : 요청 실패 + */ + INPUT_INVALID_VALUE(HttpStatus.BAD_REQUEST, "REQUEST_ERROR_001", "잘못된 요청입니다."), + INVALID_ENUM(HttpStatus.BAD_REQUEST, "REQUEST_ERROR_002", "Enum 타입으로 변경할 수 없습니다."), + + // Product + CRAWLING_SERVER_ERROR(HttpStatus.INTERNAL_SERVER_ERROR, "PRODUCT_ERROR_001", "크롤링 서버와의 연결에 실패했습니다."), + PRODUCT_INDEX_OUT_OF_RANGE(HttpStatus.BAD_REQUEST, "PRODUCT_ERROR_002", "제품 리스트의 끝입니다."), + + /** + * 500 : 응답 실패 + */ + INTERNAL_SERVER_ERROR(HttpStatus.INTERNAL_SERVER_ERROR, "RESPONSE_ERROR_001", "서버와의 연결에 실패했습니다."), + BAD_GATEWAY(HttpStatus.BAD_GATEWAY, "RESPONSE_ERROR_002", "다른 서버로부터 잘못된 응답이 수신되었습니다."), + INSUFFICIENT_STORAGE(HttpStatus.INSUFFICIENT_STORAGE, "RESPONSE_ERROR_003", "서버의 용량이 부족해 요청에 실패했습니다."), + ; + + private final HttpStatus status; + private final String code; + private final String message; +} diff --git a/src/main/java/dev/neordinary/zero/base/BaseResponseStatusImpl.java b/src/main/java/dev/neordinary/zero/base/BaseResponseStatusImpl.java new file mode 100644 index 0000000..7676e16 --- /dev/null +++ b/src/main/java/dev/neordinary/zero/base/BaseResponseStatusImpl.java @@ -0,0 +1,9 @@ +package dev.neordinary.zero.base; + +import org.springframework.http.HttpStatus; + +public interface BaseResponseStatusImpl { + HttpStatus getStatus(); + String getCode(); + String getMessage(); +} diff --git a/src/main/java/dev/neordinary/zero/config/RedisConfig.java b/src/main/java/dev/neordinary/zero/config/RedisConfig.java new file mode 100644 index 0000000..df8a82d --- /dev/null +++ b/src/main/java/dev/neordinary/zero/config/RedisConfig.java @@ -0,0 +1,52 @@ +package dev.neordinary.zero.config; + +import org.springframework.cache.CacheManager; +import org.springframework.cache.annotation.EnableCaching; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.data.redis.cache.CacheKeyPrefix; +import org.springframework.data.redis.cache.RedisCacheConfiguration; +import org.springframework.data.redis.cache.RedisCacheManager; +import org.springframework.data.redis.connection.RedisConnectionFactory; +import org.springframework.data.redis.core.RedisKeyValueAdapter; +import org.springframework.data.redis.repository.configuration.EnableRedisRepositories; +import org.springframework.data.redis.serializer.GenericJackson2JsonRedisSerializer; +import org.springframework.data.redis.serializer.RedisSerializationContext; +import org.springframework.data.redis.serializer.StringRedisSerializer; + +import java.util.HashMap; +import java.util.Map; + +@EnableRedisRepositories(enableKeyspaceEvents = RedisKeyValueAdapter.EnableKeyspaceEvents.ON_STARTUP) +@Configuration +@EnableCaching +public class RedisConfig { + @Bean + public CacheManager redisCacheManager(RedisConnectionFactory redisConnectionFactory){ + RedisCacheConfiguration configuration = RedisCacheConfiguration.defaultCacheConfig() + .disableCachingNullValues() + .computePrefixWith(CacheKeyPrefix.simple()) + .serializeKeysWith( + RedisSerializationContext.SerializationPair + .fromSerializer(new StringRedisSerializer())) + + .serializeValuesWith( + RedisSerializationContext.SerializationPair + .fromSerializer(new GenericJackson2JsonRedisSerializer())); + + + // 캐시키 별 default 유효시간 설정 + Map cacheConfiguration = new HashMap<>(); + /* + cacheConfiguration.put(CacheKey.ZONE, RedisCacheConfiguration.defaultCacheConfig() + .entryTtl(Duration.ofSeconds(CacheKey.ZONE_EXPIRE_SEC))); + */ + + return RedisCacheManager.RedisCacheManagerBuilder + .fromConnectionFactory(redisConnectionFactory) + .cacheDefaults(configuration) + .withInitialCacheConfigurations(cacheConfiguration) + .build(); + } +} + diff --git a/src/main/java/dev/neordinary/zero/controller/ProductController.java b/src/main/java/dev/neordinary/zero/controller/ProductController.java new file mode 100644 index 0000000..b022281 --- /dev/null +++ b/src/main/java/dev/neordinary/zero/controller/ProductController.java @@ -0,0 +1,23 @@ +package dev.neordinary.zero.controller; + +import dev.neordinary.zero.base.BaseResponse; +import dev.neordinary.zero.service.ProductService; +import jakarta.validation.constraints.NotBlank; +import lombok.RequiredArgsConstructor; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; + +@RequiredArgsConstructor +@RestController +public class ProductController { + + private final ProductService productService; + + @GetMapping("/api/v1/product") + public ResponseEntity getProductResponse(@RequestParam @NotBlank String keyword, + @RequestParam(defaultValue = "0", required = false) Integer lastProductId) { + return BaseResponse.toResponseEntityContainsResult(productService.getProductResponse(keyword, lastProductId)); + } +} diff --git a/src/main/java/dev/neordinary/zero/domain/Product.java b/src/main/java/dev/neordinary/zero/domain/Product.java new file mode 100644 index 0000000..e4c6f9d --- /dev/null +++ b/src/main/java/dev/neordinary/zero/domain/Product.java @@ -0,0 +1,30 @@ +package dev.neordinary.zero.domain; + +import dev.neordinary.zero.dto.ProductInfo; +import jakarta.persistence.Id; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Getter; +import org.springframework.data.redis.core.RedisHash; +import org.springframework.data.redis.core.index.Indexed; + +import java.util.List; + +@Getter +@RedisHash(value = "product", timeToLive = 3600) +@AllArgsConstructor +@Builder +public class Product { + @Id + private String id; + + @Indexed + private List productInfoList; + + public static Product createProduct(String id, List productInfoList) { + return Product.builder() + .id(id) + .productInfoList(productInfoList) + .build(); + } +} diff --git a/src/main/java/dev/neordinary/zero/domain/ProductRedisRepository.java b/src/main/java/dev/neordinary/zero/domain/ProductRedisRepository.java new file mode 100644 index 0000000..6b7aa6a --- /dev/null +++ b/src/main/java/dev/neordinary/zero/domain/ProductRedisRepository.java @@ -0,0 +1,6 @@ +package dev.neordinary.zero.domain; + +import org.springframework.data.repository.CrudRepository; + +public interface ProductRedisRepository extends CrudRepository { +} diff --git a/src/main/java/dev/neordinary/zero/dto/ProductInfo.java b/src/main/java/dev/neordinary/zero/dto/ProductInfo.java new file mode 100644 index 0000000..c8e5a12 --- /dev/null +++ b/src/main/java/dev/neordinary/zero/dto/ProductInfo.java @@ -0,0 +1,15 @@ +package dev.neordinary.zero.dto; + +import lombok.Builder; + +@Builder +public record ProductInfo( + String productName, + + Double productSugar, + + Integer productKcal, + + Integer productSize +) { +} diff --git a/src/main/java/dev/neordinary/zero/dto/ProductResponse.java b/src/main/java/dev/neordinary/zero/dto/ProductResponse.java new file mode 100644 index 0000000..07dbb24 --- /dev/null +++ b/src/main/java/dev/neordinary/zero/dto/ProductResponse.java @@ -0,0 +1,13 @@ +package dev.neordinary.zero.dto; + +import lombok.Builder; + +import java.util.List; + +@Builder +public record ProductResponse( + List productInfoList, + + Integer lastProductId +) { +} diff --git a/src/main/java/dev/neordinary/zero/service/ProductService.java b/src/main/java/dev/neordinary/zero/service/ProductService.java new file mode 100644 index 0000000..cabfa74 --- /dev/null +++ b/src/main/java/dev/neordinary/zero/service/ProductService.java @@ -0,0 +1,165 @@ +package dev.neordinary.zero.service; + +import dev.neordinary.zero.base.BaseException; +import dev.neordinary.zero.base.BaseResponseStatus; +import dev.neordinary.zero.domain.Product; +import dev.neordinary.zero.domain.ProductRedisRepository; +import dev.neordinary.zero.dto.ProductInfo; +import dev.neordinary.zero.dto.ProductResponse; +import lombok.RequiredArgsConstructor; +import org.jsoup.Jsoup; +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; +import org.jsoup.select.Elements; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +@RequiredArgsConstructor +@Service +public class ProductService { + + private final ProductRedisRepository productRedisRepository; + + private static final String MAIN_URL = "https://www.fatsecret.kr"; + private static final String PRODUCT_LIST_URL = MAIN_URL + "/칼로리-영양소/search?q="; + private static final String PAGE = "&pg="; + private static final int MAX_PRODUCT_PER_PAGE = 10; + private static final int MAX_PRODUCT_PER_VIEW = 5; + + public ProductResponse getProductResponse(String keyword, Integer lastProductId) { + Optional product = findProductList(keyword); + if (product.isPresent()) { + List productInfoList = product.get().getProductInfoList(); + if (lastProductId + 1 >= productInfoList.size()) { + throw new BaseException(BaseResponseStatus.PRODUCT_INDEX_OUT_OF_RANGE); + } + if (lastProductId + MAX_PRODUCT_PER_VIEW >= productInfoList.size()) + return ProductResponse.builder() + .productInfoList(productInfoList.subList(lastProductId + 1, productInfoList.size())) + .lastProductId(productInfoList.size() - 1) + .build(); + return ProductResponse.builder() + .productInfoList(productInfoList.subList(lastProductId + 1, lastProductId + MAX_PRODUCT_PER_VIEW + 1)) + .lastProductId(lastProductId + MAX_PRODUCT_PER_VIEW) + .build(); + } + + int pageId = 0; + String url = PRODUCT_LIST_URL + keyword + PAGE + pageId; + int productTotal = getProductTotal(url); + int pageTotal = productTotal % MAX_PRODUCT_PER_PAGE == 0 ? productTotal / MAX_PRODUCT_PER_PAGE : productTotal / MAX_PRODUCT_PER_PAGE + 1; + List productList = new ArrayList<>(); + + while (pageId < pageTotal) { + url = PRODUCT_LIST_URL + keyword + PAGE + pageId; + + try { + Document doc = Jsoup.connect(url).get(); + Elements productLinks = doc.select("a.prominent"); + + for (Element link : productLinks) { + String productName = link.text().trim().replaceAll("\\(\\w+\\)", "").strip(); + String productUrl = MAIN_URL + link.attr("href"); + String productSugar = ""; + String productKcal = ""; + String productSize = ""; + + try { + Document infoDoc = Jsoup.connect(productUrl).get(); + Elements body = infoDoc.getAllElements(); + System.out.println(body.text()); + productName = body.text().split(" ")[0] + " " + productName; + productSugar = getProductSugar(infoDoc); + productKcal = getProductKcal(infoDoc); + productSize = getProductSize(infoDoc); + } catch (IOException e) { + throw new BaseException(BaseResponseStatus.CRAWLING_SERVER_ERROR); + } + + if (productSugar != null && productKcal != null && productSize != null) + productList.add(ProductInfo.builder() + .productName(productName) + .productSugar(Double.valueOf(productSugar)) + .productKcal(Integer.valueOf(productKcal)) + .productSize(Integer.valueOf(productSize)) + .build()); + } + } catch (IOException e) { + throw new BaseException(BaseResponseStatus.CRAWLING_SERVER_ERROR); + } + + pageId++; + } + addProductList(keyword, productList); + return ProductResponse.builder() + .productInfoList(productList.subList(0, MAX_PRODUCT_PER_VIEW)) + .lastProductId(MAX_PRODUCT_PER_VIEW - 1) + .build(); + } + + @Transactional + public void addProductList(String keyword, List productInfoList) { + productRedisRepository.save(Product.createProduct(keyword, productInfoList)); + } + + @Transactional + public Optional findProductList(String keyword) { + return productRedisRepository.findById(keyword); + } + + private int getProductTotal(String url) { + try { + Document doc = Jsoup.connect(url).get(); + return Integer.parseInt(doc.select("div.searchResultSummary").text().split("중")[0]); + } catch (IOException e) { + throw new BaseException(BaseResponseStatus.CRAWLING_SERVER_ERROR); + } + } + + private String getProductSugar(Document infoDoc) { + Elements sugarElements = infoDoc.getElementsMatchingOwnText("설탕당"); + + if (!sugarElements.isEmpty()) { + Element sugarElement = sugarElements.first(); + Element nextElement = sugarElement.nextElementSibling(); + if (nextElement != null) { + return nextElement.text().replaceAll("g", ""); + } + return null; + } + return null; + } + + private String getProductKcal(Document infoDoc) { + Elements kcalElements = infoDoc.getElementsMatchingOwnText("열량"); + + if (!kcalElements.isEmpty()) { + Element kcalElement = kcalElements.first(); + Element nextElement = kcalElement.nextElementSibling().nextElementSibling().nextElementSibling().nextElementSibling(); + if (nextElement != null) { + return nextElement.text().replaceAll(" kcal", ""); + } + return null; + } + return null; + } + + private String getProductSize(Document infoDoc) { + String sizeElement = infoDoc.getElementsContainingOwnText("ml").toString(); + + Pattern pattern = Pattern.compile("(\\d+) ml"); + Matcher matcher = pattern.matcher(sizeElement); + + if (matcher.find()) { + return matcher.group(1); + } + return null; + } +}