Merge pull request #7 from NEORDINARY-TEAM-N/wiz (#14)

* FEAT: 웹 크롤링 테스트용 코드 작성

* BUILD: Redis 사용을 위한 의존성 추가

* FEAT: Response 템플릿 추가

* FEAT: Redis 설정

* FEAT: Product 캐싱, 크롤링 및 무한스크롤 로직 작성

---------

Co-authored-by: 길지운 <[email protected]>

build.gradle

@@ -30,6 +30,8 @@ dependencies {
 	annotationProcessor 'org.projectlombok:lombok'
 	testImplementation 'org.springframework.boot:spring-boot-starter-test'
 
+	implementation 'org.jsoup:jsoup:1.13.1'
+
 	// swagger
 //	implementation 'org.springdoc:springdoc-openapi-ui:1.6.15'
 //	implementation 'io.springfox:springfox-swagger2:2.9.2'
@@ -41,6 +43,13 @@ dependencies {
 
 	// S3 설정
 	implementation "org.springframework.cloud:spring-cloud-starter-aws:2.2.6.RELEASE"
+
+	// 캐싱
+	implementation "org.springframework.boot:spring-boot-starter-cache"
+
+	// Redis
+	implementation 'it.ozimov:embedded-redis:0.7.2'
+	implementation 'org.springframework.boot:spring-boot-starter-data-redis'
 }
 
 tasks.named('bootBuildImage') {

src/main/java/dev/neordinary/zero/WebScraperTwo.java

@@ -0,0 +1,87 @@
+package dev.neordinary.zero;
+
+import org.jsoup.Jsoup;
+import org.jsoup.nodes.Document;
+import org.jsoup.nodes.Element;
+import org.jsoup.select.Elements;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+public class WebScraperTwo {
+
+    public static void main(String[] args) {
+        String url = "https://www.fatsecret.kr/칼로리-영양소"+"/search?q=2%ED%94%84%EB%A1%9C";
+
+        try {
+            Document doc = Jsoup.connect(url).get();
+            List<String[]> productList = new ArrayList<>();
+
+            Elements productLinks = doc.select("a.prominent");
+
+            for (Element link : productLinks) {
+                String productName = link.text().trim();
+                String productUrl = "https://www.fatsecret.kr" + link.attr("href");
+                String productSugar = "";
+                String productKcal = "";
+                String productSize = "";
+                try {
+                    Document infoDoc = Jsoup.connect(productUrl).get();
+                    Elements body = infoDoc.getAllElements();
+                    System.out.println(body.text());
+                    Elements sugarElements = infoDoc.getElementsMatchingOwnText("설탕당");
+                    if (!sugarElements.isEmpty()) {
+                        Element sugarElement = sugarElements.first();
+                        Element nextElement = sugarElement.nextElementSibling();
+                        if (nextElement != null) {
+                            productSugar = nextElement.text();
+                        } else {
+                            System.out.println("설탕당 값이 없습니다.");
+                        }
+                    } else {
+                        System.out.println("설탕당을 찾을 수 없습니다.");
+                    }
+
+                    Elements kcalElements = infoDoc.getElementsMatchingOwnText("열량");
+                    if (!kcalElements.isEmpty()) {
+                        Element kcalElement = kcalElements.first();
+                        Element nextElement = kcalElement.nextElementSibling().nextElementSibling().nextElementSibling().nextElementSibling();
+                        if (nextElement != null) {
+                            productKcal = nextElement.text();
+                        } else {
+                            System.out.println("칼로리 값이 없습니다.");
+                        }
+                    } else {
+                        System.out.println("칼로리를 찾을 수 없습니다.");
+                    }
+
+                    String sizeElement = infoDoc.getElementsContainingOwnText("ml").toString();
+                    // 정규 표현식을 사용하여 괄호 안의 숫자 추출
+                    Pattern pattern = Pattern.compile("(\\d+) ml");
+                    Matcher matcher = pattern.matcher(sizeElement);
+
+                    // 정규 표현식과 일치하는 부분이 있는지 확인하고 숫자 추출
+                    if (matcher.find()) {
+                        productSize = matcher.group(1);
+                    } else {
+                        System.out.println("용량이 없습니다.");
+                    }
+
+                } catch (IOException e) {
+                    e.printStackTrace();
+                }
+                productList.add(new String[]{productName, productSugar, productKcal, productSize});
+            }
+
+            for (String[] product : productList) {
+                System.out.println("제품 이름: " + product[0] + ", 당: " + product[1] + ", 칼로리: " + product[2] + ", 용량: " + product[3]);
+            }
+
+        } catch (IOException e) {
+            e.printStackTrace();
+        }
+    }
+}

src/main/java/dev/neordinary/zero/base/BaseException.java

@@ -0,0 +1,12 @@
+package dev.neordinary.zero.base;
+
+import lombok.AllArgsConstructor;
+import lombok.Getter;
+import lombok.Setter;
+
+@Getter
+@Setter
+@AllArgsConstructor
+public class BaseException extends RuntimeException {
+    BaseResponseStatus errorStatus;
+}

src/main/java/dev/neordinary/zero/base/BaseExceptionHandler.java

@@ -0,0 +1,13 @@
+package dev.neordinary.zero.base;
+
+import org.springframework.http.ResponseEntity;
+import org.springframework.web.bind.annotation.ControllerAdvice;
+import org.springframework.web.bind.annotation.ExceptionHandler;
+
+@ControllerAdvice
+public class BaseExceptionHandler {
+    @ExceptionHandler(BaseException.class)
+    protected ResponseEntity<BaseResponse> handleBaseException(BaseException e) {
+        return BaseResponse.toResponseEntityContainsStatus(e.getErrorStatus());
+    }
+}

src/main/java/dev/neordinary/zero/base/BaseResponse.java

@@ -0,0 +1,56 @@
+package dev.neordinary.zero.base;
+
+
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonPropertyOrder;
+import lombok.Builder;
+import lombok.Data;
+import org.springframework.http.ResponseEntity;
+
+import static dev.neordinary.zero.base.BaseResponseStatus.SUCCESS;
+
+@Data
+@Builder
+@JsonPropertyOrder({"status", "code", "message", "result"})
+public class BaseResponse { // BaseResponse 객체를 사용할때 성공, 실패 경우
+    private final int status;
+    private final String code;
+    private final String message;
+    @JsonInclude(JsonInclude.Include.NON_NULL)
+    private Object result;
+
+    // Custom Status를 포함한 Response
+    public static ResponseEntity<BaseResponse> toResponseEntityContainsStatus(BaseResponseStatus baseResponseStatus) {
+        return ResponseEntity
+                .status(baseResponseStatus.getStatus())
+                .body(BaseResponse.builder()
+                        .status(baseResponseStatus.getStatus().value())
+                        .code(baseResponseStatus.getCode())
+                        .message(baseResponseStatus.getMessage())
+                        .build());
+    }
+
+    // Http 200, Result를 포함한 Response
+    public static ResponseEntity<BaseResponse> toResponseEntityContainsResult(Object result) {
+        return ResponseEntity
+                .status(SUCCESS.getStatus())
+                .body(BaseResponse.builder()
+                        .status(SUCCESS.getStatus().value())
+                        .code(SUCCESS.getCode())
+                        .message(SUCCESS.getMessage())
+                        .result(result)
+                        .build());
+    }
+
+    // Custom Status, Result를 포함한 Response
+    public static ResponseEntity<BaseResponse> toResponseEntityContainsStatusAndResult(BaseResponseStatus baseResponseStatus, Object result) {
+        return ResponseEntity
+                .status(baseResponseStatus.getStatus())
+                .body(BaseResponse.builder()
+                        .status(baseResponseStatus.getStatus().value())
+                        .code(baseResponseStatus.getCode())
+                        .message(baseResponseStatus.getMessage())
+                        .result(result)
+                        .build());
+    }
+}

src/main/java/dev/neordinary/zero/base/BaseResponseStatus.java

@@ -0,0 +1,48 @@
+package dev.neordinary.zero.base;
+
+import lombok.AllArgsConstructor;
+import lombok.Getter;
+import org.springframework.http.HttpStatus;
+
+@Getter
+@AllArgsConstructor
+public enum BaseResponseStatus implements BaseResponseStatusImpl {
+    /**
+     * 100 : 진행 정보
+     */
+
+    /**
+     * 200 : 요청 성공
+     */
+    SUCCESS(HttpStatus.OK, "SUCCESS", "요청에 성공했습니다."),
+    CREATED(HttpStatus.CREATED, "CREATED", "요청에 성공했으며 리소스가 정상적으로 생성되었습니다."),
+    ACCEPTED(HttpStatus.ACCEPTED, "ACCEPTED", "요청에 성공했으나 처리가 완료되지 않았습니다."),
+    DELETED(HttpStatus.NO_CONTENT, "DELETED", "요청에 성공했으며 더 이상 응답할 내용이 존재하지 않습니다."),
+
+    /**
+     * 300 : 리다이렉션
+     */
+    SEE_OTHER(HttpStatus.SEE_OTHER, "REDIRECT", "다른 주소로 요청해주세요."),
+
+    /**
+     * 400 : 요청 실패
+     */
+    INPUT_INVALID_VALUE(HttpStatus.BAD_REQUEST, "REQUEST_ERROR_001", "잘못된 요청입니다."),
+    INVALID_ENUM(HttpStatus.BAD_REQUEST, "REQUEST_ERROR_002", "Enum 타입으로 변경할 수 없습니다."),
+
+    // Product
+    CRAWLING_SERVER_ERROR(HttpStatus.INTERNAL_SERVER_ERROR, "PRODUCT_ERROR_001", "크롤링 서버와의 연결에 실패했습니다."),
+    PRODUCT_INDEX_OUT_OF_RANGE(HttpStatus.BAD_REQUEST, "PRODUCT_ERROR_002", "제품 리스트의 끝입니다."),
+
+    /**
+     * 500 : 응답 실패
+     */
+    INTERNAL_SERVER_ERROR(HttpStatus.INTERNAL_SERVER_ERROR, "RESPONSE_ERROR_001", "서버와의 연결에 실패했습니다."),
+    BAD_GATEWAY(HttpStatus.BAD_GATEWAY, "RESPONSE_ERROR_002", "다른 서버로부터 잘못된 응답이 수신되었습니다."),
+    INSUFFICIENT_STORAGE(HttpStatus.INSUFFICIENT_STORAGE, "RESPONSE_ERROR_003", "서버의 용량이 부족해 요청에 실패했습니다."),
+    ;
+
+    private final HttpStatus status;
+    private final String code;
+    private final String message;
+}

src/main/java/dev/neordinary/zero/base/BaseResponseStatusImpl.java

@@ -0,0 +1,9 @@
+package dev.neordinary.zero.base;
+
+import org.springframework.http.HttpStatus;
+
+public interface BaseResponseStatusImpl {
+    HttpStatus getStatus();
+    String getCode();
+    String getMessage();
+}

src/main/java/dev/neordinary/zero/config/RedisConfig.java

@@ -0,0 +1,52 @@
+package dev.neordinary.zero.config;
+
+import org.springframework.cache.CacheManager;
+import org.springframework.cache.annotation.EnableCaching;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.data.redis.cache.CacheKeyPrefix;
+import org.springframework.data.redis.cache.RedisCacheConfiguration;
+import org.springframework.data.redis.cache.RedisCacheManager;
+import org.springframework.data.redis.connection.RedisConnectionFactory;
+import org.springframework.data.redis.core.RedisKeyValueAdapter;
+import org.springframework.data.redis.repository.configuration.EnableRedisRepositories;
+import org.springframework.data.redis.serializer.GenericJackson2JsonRedisSerializer;
+import org.springframework.data.redis.serializer.RedisSerializationContext;
+import org.springframework.data.redis.serializer.StringRedisSerializer;
+
+import java.util.HashMap;
+import java.util.Map;
+
+@EnableRedisRepositories(enableKeyspaceEvents = RedisKeyValueAdapter.EnableKeyspaceEvents.ON_STARTUP)
+@Configuration
+@EnableCaching
+public class RedisConfig {
+    @Bean
+    public CacheManager redisCacheManager(RedisConnectionFactory redisConnectionFactory){
+        RedisCacheConfiguration configuration = RedisCacheConfiguration.defaultCacheConfig()
+                .disableCachingNullValues()
+                .computePrefixWith(CacheKeyPrefix.simple())
+                .serializeKeysWith(
+                        RedisSerializationContext.SerializationPair
+                                .fromSerializer(new StringRedisSerializer()))
+
+                .serializeValuesWith(
+                        RedisSerializationContext.SerializationPair
+                                .fromSerializer(new GenericJackson2JsonRedisSerializer()));
+
+
+        // 캐시키 별 default 유효시간 설정
+        Map<String, RedisCacheConfiguration> cacheConfiguration = new HashMap<>();
+        /*
+        cacheConfiguration.put(CacheKey.ZONE, RedisCacheConfiguration.defaultCacheConfig()
+                .entryTtl(Duration.ofSeconds(CacheKey.ZONE_EXPIRE_SEC)));
+         */
+
+        return RedisCacheManager.RedisCacheManagerBuilder
+                .fromConnectionFactory(redisConnectionFactory)
+                .cacheDefaults(configuration)
+                .withInitialCacheConfigurations(cacheConfiguration)
+                .build();
+    }
+}
+

src/main/java/dev/neordinary/zero/controller/ProductController.java

@@ -0,0 +1,23 @@
+package dev.neordinary.zero.controller;
+
+import dev.neordinary.zero.base.BaseResponse;
+import dev.neordinary.zero.service.ProductService;
+import jakarta.validation.constraints.NotBlank;
+import lombok.RequiredArgsConstructor;
+import org.springframework.http.ResponseEntity;
+import org.springframework.web.bind.annotation.GetMapping;
+import org.springframework.web.bind.annotation.RequestParam;
+import org.springframework.web.bind.annotation.RestController;
+
+@RequiredArgsConstructor
+@RestController
+public class ProductController {
+
+    private final ProductService productService;
+
+    @GetMapping("/api/v1/product")
+    public ResponseEntity<BaseResponse> getProductResponse(@RequestParam @NotBlank String keyword,
+                                                           @RequestParam(defaultValue = "0", required = false) Integer lastProductId) {
+        return BaseResponse.toResponseEntityContainsResult(productService.getProductResponse(keyword, lastProductId));
+    }
+}

src/main/java/dev/neordinary/zero/domain/Product.java

@@ -0,0 +1,30 @@
+package dev.neordinary.zero.domain;
+
+import dev.neordinary.zero.dto.ProductInfo;
+import jakarta.persistence.Id;
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Getter;
+import org.springframework.data.redis.core.RedisHash;
+import org.springframework.data.redis.core.index.Indexed;
+
+import java.util.List;
+
+@Getter
+@RedisHash(value = "product", timeToLive = 3600)
+@AllArgsConstructor
+@Builder
+public class Product {
+    @Id
+    private String id;
+
+    @Indexed
+    private List<ProductInfo> productInfoList;
+
+    public static Product createProduct(String id, List<ProductInfo> productInfoList) {
+        return Product.builder()
+                .id(id)
+                .productInfoList(productInfoList)
+                .build();
+    }
+}

src/main/java/dev/neordinary/zero/domain/ProductRedisRepository.java

@@ -0,0 +1,6 @@
+package dev.neordinary.zero.domain;
+
+import org.springframework.data.repository.CrudRepository;
+
+public interface ProductRedisRepository extends CrudRepository<Product, String> {
+}

src/main/java/dev/neordinary/zero/dto/ProductInfo.java

@@ -0,0 +1,15 @@
+package dev.neordinary.zero.dto;
+
+import lombok.Builder;
+
+@Builder
+public record ProductInfo(
+        String productName,
+
+        Double productSugar,
+
+        Integer productKcal,
+
+        Integer productSize
+) {
+}

src/main/java/dev/neordinary/zero/dto/ProductResponse.java

@@ -0,0 +1,13 @@
+package dev.neordinary.zero.dto;
+
+import lombok.Builder;
+
+import java.util.List;
+
+@Builder
+public record ProductResponse(
+        List<ProductInfo> productInfoList,
+
+        Integer lastProductId
+) {
+}