From 7512e124601f97a01adb8417299da9562323458f Mon Sep 17 00:00:00 2001 From: ddnjs Date: Tue, 31 Oct 2023 22:31:07 +0900 Subject: [PATCH 01/37] feat: add exhibition crawler --- build.gradle | 3 + .../common/service/CatcherCrawlerService.java | 83 +++++++++++++++++++ .../batch/resource/CrawlerController.java | 26 ++++++ 3 files changed, 112 insertions(+) create mode 100644 src/main/java/com/catcher/batch/common/service/CatcherCrawlerService.java create mode 100644 src/main/java/com/catcher/batch/resource/CrawlerController.java diff --git a/build.gradle b/build.gradle index d220628..309d2db 100644 --- a/build.gradle +++ b/build.gradle @@ -54,6 +54,9 @@ dependencies { //swagger implementation 'org.springdoc:springdoc-openapi-starter-webmvc-ui:2.2.0' + //crawling + implementation 'org.seleniumhq.selenium:selenium-java:4.14.1' + implementation 'com.opencsv:opencsv:5.7.1' } dependencyManagement { diff --git a/src/main/java/com/catcher/batch/common/service/CatcherCrawlerService.java b/src/main/java/com/catcher/batch/common/service/CatcherCrawlerService.java new file mode 100644 index 0000000..f39ff3e --- /dev/null +++ b/src/main/java/com/catcher/batch/common/service/CatcherCrawlerService.java @@ -0,0 +1,83 @@ +package com.catcher.batch.common.service; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.openqa.selenium.By; +import org.openqa.selenium.WebDriver; +import org.openqa.selenium.WebElement; +import org.openqa.selenium.chrome.ChromeDriver; +import org.springframework.stereotype.Service; + +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; + +@Service +public class CatcherCrawlerService { + public List exhibitionCrawling() { + + //TODO 드라이버 경로 상황에 맞게 설정, 그에 따른 크롬 및 gradle 의존성 버전도 알맞게 수정 + System.setProperty("webdriver.chrome.driver", "C:/Users/dong/Downloads/chromedriver-win64/chromedriver-win64/chromedriver.exe"); + + WebDriver driver = new ChromeDriver(); + + //TODO 파라미터 받아서 월 조절하는 것으로 변경 + driver.get("https://www.akei.or.kr/bbs/board.php?bo_table=schedule"); + driver.manage().timeouts().implicitlyWait(Duration.ofSeconds(20)); + + int page = 1; + + ObjectMapper mapper = new ObjectMapper(); + List exhibition_list = new ArrayList<>(); + + String[] columns = {"eng_name", "abbreviation_name", "host", "period", "location", "field", "homepage"}; + + while (true) { + WebElement exhibitList = driver.findElement(By.cssSelector(".exhibit_list")); + List contentElements = exhibitList.findElements(By.cssSelector(".content_sc_li")); + + for (WebElement contentElement : contentElements) { + WebElement tbody = contentElement.findElement(By.tagName("tbody")); + List tr_elements = tbody.findElements(By.tagName("tr")); + + List data = new ArrayList<>(); + ObjectNode json = mapper.createObjectNode(); + + for (WebElement td_element : tr_elements) { + WebElement element = td_element.findElement(By.tagName("td")); + data.add(element.getAttribute("innerText")); + } + for (int i = 0; i < columns.length; i++) { + json.put(columns[i], data.get(i)); + } + exhibition_list.add(json); + } + + page++; + + boolean nextPageExists = isNextPageExists(driver, page); + if (!nextPageExists) { + break; + } + + // 다음 페이지 2,3,4... + WebElement nextPage = driver.findElement(By.xpath("//a[@class='pg_page' and text()='" + page + "']")); + + nextPage.click(); + driver.manage().timeouts().implicitlyWait(Duration.ofSeconds(20)); + } + //TODO driver.quit 에러 안나는 방법 서치 +// driver.quit(); + return exhibition_list; + } + + // 전시회 사이트 다음페이지 여부 검사 근데 어차피 다른 페이지에는 못써서 안에 넣어야할듯? + private static boolean isNextPageExists(WebDriver driver, int nextPage) { + try { + driver.findElement(By.xpath("//a[@class='pg_page' and text()='" + nextPage + "']")); + return true; + } catch (Exception e) { + return false; + } + } +} diff --git a/src/main/java/com/catcher/batch/resource/CrawlerController.java b/src/main/java/com/catcher/batch/resource/CrawlerController.java new file mode 100644 index 0000000..fba5547 --- /dev/null +++ b/src/main/java/com/catcher/batch/resource/CrawlerController.java @@ -0,0 +1,26 @@ +package com.catcher.batch.resource; + +import com.catcher.batch.common.service.CatcherCrawlerService; +import com.catcher.batch.core.dto.FestivalApiResponse; +import com.fasterxml.jackson.databind.node.ObjectNode; +import lombok.RequiredArgsConstructor; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; + +import java.util.List; + +@RequiredArgsConstructor +@RestController +@RequestMapping("/crawler") +public class CrawlerController { + + private final CatcherCrawlerService catcherCrawlerService; + + @GetMapping("/exhibition") + public ResponseEntity> getExhibitionData() { + List exhibitionCrawlingResponse = catcherCrawlerService.exhibitionCrawling(); + return ResponseEntity.ok(exhibitionCrawlingResponse); + } +} From aacb01ffc3535724a9bad884086c412dbe3e4986 Mon Sep 17 00:00:00 2001 From: cheolwon1994 Date: Fri, 10 Nov 2023 14:28:38 +0900 Subject: [PATCH 02/37] reset db, kms, ssh values --- src/main/resources/application-dev.properties | 14 ++++++------- .../resources/application-local.properties | 14 ++++++------- src/main/resources/application.properties | 20 ++++++++++++------- 3 files changed, 27 insertions(+), 21 deletions(-) diff --git a/src/main/resources/application-dev.properties b/src/main/resources/application-dev.properties index eee38e3..531f0bc 100644 --- a/src/main/resources/application-dev.properties +++ b/src/main/resources/application-dev.properties @@ -2,19 +2,19 @@ spring.datasource.driver-class-name=org.mariadb.jdbc.Driver ## spring datasource spring.datasource.url=AQICAHg2j0pl3GguoQNkjcreuBPYJqI3OPeLEOelKIkMSzn01AEZqo9kRIKYisZIm0Z0xpuOAAAAiDCBhQYJKoZIhvcNAQcGoHgwdgIBADBxBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDMoZI5smgDwlG+0kBgIBEIBE1VE7k3neYIx1Pkx6YqUrTf7LEBAxddoUbZVTZxC7HQJpNJKNlwMoaXXyHdPTkBW15dJiCw5jUzuY9CRV7o/vpogIg9Q= -spring.datasource.username=AQICAHg2j0pl3GguoQNkjcreuBPYJqI3OPeLEOelKIkMSzn01AGLdMIjOGdWmBOvj9YZ5685AAAAajBoBgkqhkiG9w0BBwagWzBZAgEAMFQGCSqGSIb3DQEHATAeBglghkgBZQMEAS4wEQQMDgyL4QtIvG2hM9/5AgEQgCdVSGC6ySfTUn8ZyXKzj2x9eFu+05ACK2sP55fJtoGP0hncckeuVmQ= -spring.datasource.password=AQICAHg2j0pl3GguoQNkjcreuBPYJqI3OPeLEOelKIkMSzn01AGVzdO6dwAvzNw42TIj6JZ1AAAAaTBnBgkqhkiG9w0BBwagWjBYAgEAMFMGCSqGSIb3DQEHATAeBglghkgBZQMEAS4wEQQMjonbIr69Og3D+ujOAgEQgCatIXU02fuulW5mm6Bn+X9e/FPMq3Fk/GhOLyYpudTHfSkRtMORQQ== +spring.datasource.username=AQICAHg2j0pl3GguoQNkjcreuBPYJqI3OPeLEOelKIkMSzn01AFDjW/RUZSn5wxJhd1XeZOgAAAAYzBhBgkqhkiG9w0BBwagVDBSAgEAME0GCSqGSIb3DQEHATAeBglghkgBZQMEAS4wEQQM0eyz4OI2Vkkc/uTUAgEQgCAXcQUOKDsI26jXFzQfba7Cv2EHO51213zwOT0hzc0SnQ== +spring.datasource.password=AQICAHg2j0pl3GguoQNkjcreuBPYJqI3OPeLEOelKIkMSzn01AGiM8q8PoYW89muC9gXzZ5GAAAAaTBnBgkqhkiG9w0BBwagWjBYAgEAMFMGCSqGSIb3DQEHATAeBglghkgBZQMEAS4wEQQMxGIq7N/hB2XW5NODAgEQgCZeBBgSs9LrXXuCsMAHE53Rsg1WGKfDzjMaAoFRDjEKoGTT9uFThQ== ## ssh -ssh.host=AQICAHg2j0pl3GguoQNkjcreuBPYJqI3OPeLEOelKIkMSzn01AGU6PSF+1N5z7We+p8jZ0o0AAAAaDBmBgkqhkiG9w0BBwagWTBXAgEAMFIGCSqGSIb3DQEHATAeBglghkgBZQMEAS4wEQQM9b3AkdokJsb66zbHAgEQgCXIPbqF5l817FjPbh2LXBzXndYJiSUqyDGZXzyfppZi3qqlSYZV -ssh.username=AQICAHg2j0pl3GguoQNkjcreuBPYJqI3OPeLEOelKIkMSzn01AE68oCmRn5p/q+G/NzwrKpHAAAAYjBgBgkqhkiG9w0BBwagUzBRAgEAMEwGCSqGSIb3DQEHATAeBglghkgBZQMEAS4wEQQMCABMThPwxXQHcl5bAgEQgB8+W4uWWbUm8bpo7QTCtjYyJ/ZB5hv4SKstc3Xa4Ltt -ssh.password=AQICAHg2j0pl3GguoQNkjcreuBPYJqI3OPeLEOelKIkMSzn01AGlm0DJ1SuCTcwtPCyXZLYjAAAAaTBnBgkqhkiG9w0BBwagWjBYAgEAMFMGCSqGSIb3DQEHATAeBglghkgBZQMEAS4wEQQM1awvQRELJm1vQL/oAgEQgCasN+qZQ0q7w4FOhwRTCzPYzoMnraA1yx4T9hz2U3j4SGn3+3yEaQ== -ssh.datasource.origin=AQICAHg2j0pl3GguoQNkjcreuBPYJqI3OPeLEOelKIkMSzn01AGuDWPowi/9gzZpLApcSpl9AAAAmzCBmAYJKoZIhvcNAQcGoIGKMIGHAgEAMIGBBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDNqU18Wps+61WNJv6QIBEIBUziHPbeZlunGk/0EvzFYVr2TG4HYuwKd9bQcUdRL7PPkZqO+zdFXeYRtgfhyq/isQqKav0E7LAsnrruFaM0BfGmoqHChECFdSABUJ7+uSdWF2F/vV +ssh.host=AQICAHg2j0pl3GguoQNkjcreuBPYJqI3OPeLEOelKIkMSzn01AGc1mlusKIbT/FmW65+2rl4AAAAbDBqBgkqhkiG9w0BBwagXTBbAgEAMFYGCSqGSIb3DQEHATAeBglghkgBZQMEAS4wEQQMvijHTTZUUuGjjF4jAgEQgCmKq00a3J3yAUkdljTk342nBjYaeLr5BQeJ2kIISM3P/WPMXKQB/0WzlQ== +ssh.username=AQICAHg2j0pl3GguoQNkjcreuBPYJqI3OPeLEOelKIkMSzn01AE26DRJsDEHLdAnA5QIkHfYAAAAZjBkBgkqhkiG9w0BBwagVzBVAgEAMFAGCSqGSIb3DQEHATAeBglghkgBZQMEAS4wEQQMd4p4zWU1dg+tbLAKAgEQgCNWOoyj2Ual/lQaCe7yz2JkY6TXbUR25FO6SkPkYaAncbgkcA== +ssh.password=AQICAHg2j0pl3GguoQNkjcreuBPYJqI3OPeLEOelKIkMSzn01AFNH5Ud+NVY/FVqOa99UOeMAAAAaTBnBgkqhkiG9w0BBwagWjBYAgEAMFMGCSqGSIb3DQEHATAeBglghkgBZQMEAS4wEQQMsRopSwW2oDrqOTNrAgEQgCacSRENpF2QbZBfiyN/mlQlY21Qd6PSHef54Rg29FfM5rhWlEuFfg== +ssh.datasource.origin=AQICAHg2j0pl3GguoQNkjcreuBPYJqI3OPeLEOelKIkMSzn01AFY15QxnieQA6WGHi+ZBdpTAAAAtDCBsQYJKoZIhvcNAQcGoIGjMIGgAgEAMIGaBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDDHB9KlaPh/a8J528wIBEIBtds8VUJXlJZSbxWDeV9mnr+16QcsLiVr4C9GlnR44Wb9opywAAUFBCBYqN1SX4czjhkdEgryY9/QYyzSx+VHLT5ORfVTFZ58IbMRGBpZdoXVgXfCH9WSXz3xDDBZQQ7MDlMAj2dECNsgLAA6i2A== ssh.port=22 ssh.local-port=3306 ## kms -aws.kms.keyId=03703854-534e-40ec-838c-cf65541f88f8 +aws.kms.keyId=5d1c783d-6c8e-4661-a8ed-a12654aac8c3 aws.kms.encryptionAlgorithm=SYMMETRIC_DEFAULT aws.kms.profile=default diff --git a/src/main/resources/application-local.properties b/src/main/resources/application-local.properties index eee38e3..531f0bc 100644 --- a/src/main/resources/application-local.properties +++ b/src/main/resources/application-local.properties @@ -2,19 +2,19 @@ spring.datasource.driver-class-name=org.mariadb.jdbc.Driver ## spring datasource spring.datasource.url=AQICAHg2j0pl3GguoQNkjcreuBPYJqI3OPeLEOelKIkMSzn01AEZqo9kRIKYisZIm0Z0xpuOAAAAiDCBhQYJKoZIhvcNAQcGoHgwdgIBADBxBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDMoZI5smgDwlG+0kBgIBEIBE1VE7k3neYIx1Pkx6YqUrTf7LEBAxddoUbZVTZxC7HQJpNJKNlwMoaXXyHdPTkBW15dJiCw5jUzuY9CRV7o/vpogIg9Q= -spring.datasource.username=AQICAHg2j0pl3GguoQNkjcreuBPYJqI3OPeLEOelKIkMSzn01AGLdMIjOGdWmBOvj9YZ5685AAAAajBoBgkqhkiG9w0BBwagWzBZAgEAMFQGCSqGSIb3DQEHATAeBglghkgBZQMEAS4wEQQMDgyL4QtIvG2hM9/5AgEQgCdVSGC6ySfTUn8ZyXKzj2x9eFu+05ACK2sP55fJtoGP0hncckeuVmQ= -spring.datasource.password=AQICAHg2j0pl3GguoQNkjcreuBPYJqI3OPeLEOelKIkMSzn01AGVzdO6dwAvzNw42TIj6JZ1AAAAaTBnBgkqhkiG9w0BBwagWjBYAgEAMFMGCSqGSIb3DQEHATAeBglghkgBZQMEAS4wEQQMjonbIr69Og3D+ujOAgEQgCatIXU02fuulW5mm6Bn+X9e/FPMq3Fk/GhOLyYpudTHfSkRtMORQQ== +spring.datasource.username=AQICAHg2j0pl3GguoQNkjcreuBPYJqI3OPeLEOelKIkMSzn01AFDjW/RUZSn5wxJhd1XeZOgAAAAYzBhBgkqhkiG9w0BBwagVDBSAgEAME0GCSqGSIb3DQEHATAeBglghkgBZQMEAS4wEQQM0eyz4OI2Vkkc/uTUAgEQgCAXcQUOKDsI26jXFzQfba7Cv2EHO51213zwOT0hzc0SnQ== +spring.datasource.password=AQICAHg2j0pl3GguoQNkjcreuBPYJqI3OPeLEOelKIkMSzn01AGiM8q8PoYW89muC9gXzZ5GAAAAaTBnBgkqhkiG9w0BBwagWjBYAgEAMFMGCSqGSIb3DQEHATAeBglghkgBZQMEAS4wEQQMxGIq7N/hB2XW5NODAgEQgCZeBBgSs9LrXXuCsMAHE53Rsg1WGKfDzjMaAoFRDjEKoGTT9uFThQ== ## ssh -ssh.host=AQICAHg2j0pl3GguoQNkjcreuBPYJqI3OPeLEOelKIkMSzn01AGU6PSF+1N5z7We+p8jZ0o0AAAAaDBmBgkqhkiG9w0BBwagWTBXAgEAMFIGCSqGSIb3DQEHATAeBglghkgBZQMEAS4wEQQM9b3AkdokJsb66zbHAgEQgCXIPbqF5l817FjPbh2LXBzXndYJiSUqyDGZXzyfppZi3qqlSYZV -ssh.username=AQICAHg2j0pl3GguoQNkjcreuBPYJqI3OPeLEOelKIkMSzn01AE68oCmRn5p/q+G/NzwrKpHAAAAYjBgBgkqhkiG9w0BBwagUzBRAgEAMEwGCSqGSIb3DQEHATAeBglghkgBZQMEAS4wEQQMCABMThPwxXQHcl5bAgEQgB8+W4uWWbUm8bpo7QTCtjYyJ/ZB5hv4SKstc3Xa4Ltt -ssh.password=AQICAHg2j0pl3GguoQNkjcreuBPYJqI3OPeLEOelKIkMSzn01AGlm0DJ1SuCTcwtPCyXZLYjAAAAaTBnBgkqhkiG9w0BBwagWjBYAgEAMFMGCSqGSIb3DQEHATAeBglghkgBZQMEAS4wEQQM1awvQRELJm1vQL/oAgEQgCasN+qZQ0q7w4FOhwRTCzPYzoMnraA1yx4T9hz2U3j4SGn3+3yEaQ== -ssh.datasource.origin=AQICAHg2j0pl3GguoQNkjcreuBPYJqI3OPeLEOelKIkMSzn01AGuDWPowi/9gzZpLApcSpl9AAAAmzCBmAYJKoZIhvcNAQcGoIGKMIGHAgEAMIGBBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDNqU18Wps+61WNJv6QIBEIBUziHPbeZlunGk/0EvzFYVr2TG4HYuwKd9bQcUdRL7PPkZqO+zdFXeYRtgfhyq/isQqKav0E7LAsnrruFaM0BfGmoqHChECFdSABUJ7+uSdWF2F/vV +ssh.host=AQICAHg2j0pl3GguoQNkjcreuBPYJqI3OPeLEOelKIkMSzn01AGc1mlusKIbT/FmW65+2rl4AAAAbDBqBgkqhkiG9w0BBwagXTBbAgEAMFYGCSqGSIb3DQEHATAeBglghkgBZQMEAS4wEQQMvijHTTZUUuGjjF4jAgEQgCmKq00a3J3yAUkdljTk342nBjYaeLr5BQeJ2kIISM3P/WPMXKQB/0WzlQ== +ssh.username=AQICAHg2j0pl3GguoQNkjcreuBPYJqI3OPeLEOelKIkMSzn01AE26DRJsDEHLdAnA5QIkHfYAAAAZjBkBgkqhkiG9w0BBwagVzBVAgEAMFAGCSqGSIb3DQEHATAeBglghkgBZQMEAS4wEQQMd4p4zWU1dg+tbLAKAgEQgCNWOoyj2Ual/lQaCe7yz2JkY6TXbUR25FO6SkPkYaAncbgkcA== +ssh.password=AQICAHg2j0pl3GguoQNkjcreuBPYJqI3OPeLEOelKIkMSzn01AFNH5Ud+NVY/FVqOa99UOeMAAAAaTBnBgkqhkiG9w0BBwagWjBYAgEAMFMGCSqGSIb3DQEHATAeBglghkgBZQMEAS4wEQQMsRopSwW2oDrqOTNrAgEQgCacSRENpF2QbZBfiyN/mlQlY21Qd6PSHef54Rg29FfM5rhWlEuFfg== +ssh.datasource.origin=AQICAHg2j0pl3GguoQNkjcreuBPYJqI3OPeLEOelKIkMSzn01AFY15QxnieQA6WGHi+ZBdpTAAAAtDCBsQYJKoZIhvcNAQcGoIGjMIGgAgEAMIGaBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDDHB9KlaPh/a8J528wIBEIBtds8VUJXlJZSbxWDeV9mnr+16QcsLiVr4C9GlnR44Wb9opywAAUFBCBYqN1SX4czjhkdEgryY9/QYyzSx+VHLT5ORfVTFZ58IbMRGBpZdoXVgXfCH9WSXz3xDDBZQQ7MDlMAj2dECNsgLAA6i2A== ssh.port=22 ssh.local-port=3306 ## kms -aws.kms.keyId=03703854-534e-40ec-838c-cf65541f88f8 +aws.kms.keyId=5d1c783d-6c8e-4661-a8ed-a12654aac8c3 aws.kms.encryptionAlgorithm=SYMMETRIC_DEFAULT aws.kms.profile=default diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index 4b8f1b6..2dc9ecf 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -4,28 +4,34 @@ spring.datasource.driver-class-name=org.mariadb.jdbc.Driver ## spring datasource spring.datasource.url=AQICAHg2j0pl3GguoQNkjcreuBPYJqI3OPeLEOelKIkMSzn01AEZqo9kRIKYisZIm0Z0xpuOAAAAiDCBhQYJKoZIhvcNAQcGoHgwdgIBADBxBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDMoZI5smgDwlG+0kBgIBEIBE1VE7k3neYIx1Pkx6YqUrTf7LEBAxddoUbZVTZxC7HQJpNJKNlwMoaXXyHdPTkBW15dJiCw5jUzuY9CRV7o/vpogIg9Q= -spring.datasource.username=AQICAHg2j0pl3GguoQNkjcreuBPYJqI3OPeLEOelKIkMSzn01AGLdMIjOGdWmBOvj9YZ5685AAAAajBoBgkqhkiG9w0BBwagWzBZAgEAMFQGCSqGSIb3DQEHATAeBglghkgBZQMEAS4wEQQMDgyL4QtIvG2hM9/5AgEQgCdVSGC6ySfTUn8ZyXKzj2x9eFu+05ACK2sP55fJtoGP0hncckeuVmQ= -spring.datasource.password=AQICAHg2j0pl3GguoQNkjcreuBPYJqI3OPeLEOelKIkMSzn01AGVzdO6dwAvzNw42TIj6JZ1AAAAaTBnBgkqhkiG9w0BBwagWjBYAgEAMFMGCSqGSIb3DQEHATAeBglghkgBZQMEAS4wEQQMjonbIr69Og3D+ujOAgEQgCatIXU02fuulW5mm6Bn+X9e/FPMq3Fk/GhOLyYpudTHfSkRtMORQQ== +spring.datasource.username=AQICAHg2j0pl3GguoQNkjcreuBPYJqI3OPeLEOelKIkMSzn01AFDjW/RUZSn5wxJhd1XeZOgAAAAYzBhBgkqhkiG9w0BBwagVDBSAgEAME0GCSqGSIb3DQEHATAeBglghkgBZQMEAS4wEQQM0eyz4OI2Vkkc/uTUAgEQgCAXcQUOKDsI26jXFzQfba7Cv2EHO51213zwOT0hzc0SnQ== +spring.datasource.password=AQICAHg2j0pl3GguoQNkjcreuBPYJqI3OPeLEOelKIkMSzn01AGiM8q8PoYW89muC9gXzZ5GAAAAaTBnBgkqhkiG9w0BBwagWjBYAgEAMFMGCSqGSIb3DQEHATAeBglghkgBZQMEAS4wEQQMxGIq7N/hB2XW5NODAgEQgCZeBBgSs9LrXXuCsMAHE53Rsg1WGKfDzjMaAoFRDjEKoGTT9uFThQ== ## ssh -ssh.host=AQICAHg2j0pl3GguoQNkjcreuBPYJqI3OPeLEOelKIkMSzn01AGU6PSF+1N5z7We+p8jZ0o0AAAAaDBmBgkqhkiG9w0BBwagWTBXAgEAMFIGCSqGSIb3DQEHATAeBglghkgBZQMEAS4wEQQM9b3AkdokJsb66zbHAgEQgCXIPbqF5l817FjPbh2LXBzXndYJiSUqyDGZXzyfppZi3qqlSYZV -ssh.username=AQICAHg2j0pl3GguoQNkjcreuBPYJqI3OPeLEOelKIkMSzn01AE68oCmRn5p/q+G/NzwrKpHAAAAYjBgBgkqhkiG9w0BBwagUzBRAgEAMEwGCSqGSIb3DQEHATAeBglghkgBZQMEAS4wEQQMCABMThPwxXQHcl5bAgEQgB8+W4uWWbUm8bpo7QTCtjYyJ/ZB5hv4SKstc3Xa4Ltt -ssh.password=AQICAHg2j0pl3GguoQNkjcreuBPYJqI3OPeLEOelKIkMSzn01AGlm0DJ1SuCTcwtPCyXZLYjAAAAaTBnBgkqhkiG9w0BBwagWjBYAgEAMFMGCSqGSIb3DQEHATAeBglghkgBZQMEAS4wEQQM1awvQRELJm1vQL/oAgEQgCasN+qZQ0q7w4FOhwRTCzPYzoMnraA1yx4T9hz2U3j4SGn3+3yEaQ== -ssh.datasource.origin=AQICAHg2j0pl3GguoQNkjcreuBPYJqI3OPeLEOelKIkMSzn01AGuDWPowi/9gzZpLApcSpl9AAAAmzCBmAYJKoZIhvcNAQcGoIGKMIGHAgEAMIGBBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDNqU18Wps+61WNJv6QIBEIBUziHPbeZlunGk/0EvzFYVr2TG4HYuwKd9bQcUdRL7PPkZqO+zdFXeYRtgfhyq/isQqKav0E7LAsnrruFaM0BfGmoqHChECFdSABUJ7+uSdWF2F/vV +ssh.host=AQICAHg2j0pl3GguoQNkjcreuBPYJqI3OPeLEOelKIkMSzn01AGc1mlusKIbT/FmW65+2rl4AAAAbDBqBgkqhkiG9w0BBwagXTBbAgEAMFYGCSqGSIb3DQEHATAeBglghkgBZQMEAS4wEQQMvijHTTZUUuGjjF4jAgEQgCmKq00a3J3yAUkdljTk342nBjYaeLr5BQeJ2kIISM3P/WPMXKQB/0WzlQ== +ssh.username=AQICAHg2j0pl3GguoQNkjcreuBPYJqI3OPeLEOelKIkMSzn01AE26DRJsDEHLdAnA5QIkHfYAAAAZjBkBgkqhkiG9w0BBwagVzBVAgEAMFAGCSqGSIb3DQEHATAeBglghkgBZQMEAS4wEQQMd4p4zWU1dg+tbLAKAgEQgCNWOoyj2Ual/lQaCe7yz2JkY6TXbUR25FO6SkPkYaAncbgkcA== +ssh.password=AQICAHg2j0pl3GguoQNkjcreuBPYJqI3OPeLEOelKIkMSzn01AFNH5Ud+NVY/FVqOa99UOeMAAAAaTBnBgkqhkiG9w0BBwagWjBYAgEAMFMGCSqGSIb3DQEHATAeBglghkgBZQMEAS4wEQQMsRopSwW2oDrqOTNrAgEQgCacSRENpF2QbZBfiyN/mlQlY21Qd6PSHef54Rg29FfM5rhWlEuFfg== +ssh.datasource.origin=AQICAHg2j0pl3GguoQNkjcreuBPYJqI3OPeLEOelKIkMSzn01AFY15QxnieQA6WGHi+ZBdpTAAAAtDCBsQYJKoZIhvcNAQcGoIGjMIGgAgEAMIGaBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDDHB9KlaPh/a8J528wIBEIBtds8VUJXlJZSbxWDeV9mnr+16QcsLiVr4C9GlnR44Wb9opywAAUFBCBYqN1SX4czjhkdEgryY9/QYyzSx+VHLT5ORfVTFZ58IbMRGBpZdoXVgXfCH9WSXz3xDDBZQQ7MDlMAj2dECNsgLAA6i2A== ssh.port=22 ssh.local-port=3306 ## kms -aws.kms.keyId=03703854-534e-40ec-838c-cf65541f88f8 +aws.kms.keyId=5d1c783d-6c8e-4661-a8ed-a12654aac8c3 aws.kms.encryptionAlgorithm=SYMMETRIC_DEFAULT aws.kms.profile=default ## movie api movie.baseUrl=http://www.kobis.or.kr/kobisopenapi/webservice/rest/boxoffice/searchDailyBoxOfficeList.json movie.key=AQICAHg2j0pl3GguoQNkjcreuBPYJqI3OPeLEOelKIkMSzn01AEdAQDVPqdOhpfNbxFhaX/rAAAAfjB8BgkqhkiG9w0BBwagbzBtAgEAMGgGCSqGSIb3DQEHATAeBglghkgBZQMEAS4wEQQMnYW/M8ND+IPKIhQ8AgEQgDv37SLjLvGtPBohoi8i1Z+SFBaopteVK3e+36yH3RB44h/Ik0ZyxE69hlJ0T8SHnTLWzLKsk5TKP0uwFA== + +## festival api festival.baseUrl=http://api.data.go.kr/openapi/tn_pubr_public_cltur_fstvl_api festival.key=AQICAHg2j0pl3GguoQNkjcreuBPYJqI3OPeLEOelKIkMSzn01AGEXII6Ufa8dIEGD6MQNyM9AAAAujCBtwYJKoZIhvcNAQcGoIGpMIGmAgEAMIGgBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDEphbmyNQTiCTHtF/gIBEIBz8Fj7Ij/xk+O5hXWQfMOsIfc6uAYrvF2Xew+p0qLkX3DO0Plzjt4EMzaUP504RIMr7s+Yhx2y2Lq5SZ9I/cZ4swMYIOwj8FXDLeFy/k3dfwFdnBRh1kmvIDiMSg5kj4kgk05nzHyJ5KloYXXngp/ZQzsdpA== +## movie api tmdb +movie.tmdb.baseUrl=https://api.themoviedb.org/3/discover/movie +movie.tmdb.key=AQICAHg2j0pl3GguoQNkjcreuBPYJqI3OPeLEOelKIkMSzn01AELxjw41yAPSTnmp5zrZNWQAAAAfjB8BgkqhkiG9w0BBwagbzBtAgEAMGgGCSqGSIb3DQEHATAeBglghkgBZQMEAS4wEQQMHCIOEwhqioIojbKiAgEQgDsmsdMzlgQamoyDnGglRz5A2BNtMvn/4BIVT75wmxUz82NvOoh+Tno236+8lcoAoYdWzfgFVk7cAMNZdw== + #update the schema with the given values. spring.jpa.hibernate.ddl-auto=update #To beautify or pretty print the SQL From 8f919314b851ded220f2c27658a9fe00ae644107 Mon Sep 17 00:00:00 2001 From: cheolwon1994 Date: Fri, 10 Nov 2023 14:50:05 +0900 Subject: [PATCH 03/37] remove unneeded code --- src/main/resources/application-dev.properties | 1 - src/main/resources/application-local.properties | 1 - src/main/resources/application.properties | 1 - 3 files changed, 3 deletions(-) diff --git a/src/main/resources/application-dev.properties b/src/main/resources/application-dev.properties index 531f0bc..bdf9a5a 100644 --- a/src/main/resources/application-dev.properties +++ b/src/main/resources/application-dev.properties @@ -16,7 +16,6 @@ ssh.local-port=3306 ## kms aws.kms.keyId=5d1c783d-6c8e-4661-a8ed-a12654aac8c3 aws.kms.encryptionAlgorithm=SYMMETRIC_DEFAULT -aws.kms.profile=default ## movie api movie.baseUrl=http://www.kobis.or.kr/kobisopenapi/webservice/rest/boxoffice/searchDailyBoxOfficeList.json diff --git a/src/main/resources/application-local.properties b/src/main/resources/application-local.properties index 531f0bc..bdf9a5a 100644 --- a/src/main/resources/application-local.properties +++ b/src/main/resources/application-local.properties @@ -16,7 +16,6 @@ ssh.local-port=3306 ## kms aws.kms.keyId=5d1c783d-6c8e-4661-a8ed-a12654aac8c3 aws.kms.encryptionAlgorithm=SYMMETRIC_DEFAULT -aws.kms.profile=default ## movie api movie.baseUrl=http://www.kobis.or.kr/kobisopenapi/webservice/rest/boxoffice/searchDailyBoxOfficeList.json diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index 2dc9ecf..2f7bd75 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -18,7 +18,6 @@ ssh.local-port=3306 ## kms aws.kms.keyId=5d1c783d-6c8e-4661-a8ed-a12654aac8c3 aws.kms.encryptionAlgorithm=SYMMETRIC_DEFAULT -aws.kms.profile=default ## movie api movie.baseUrl=http://www.kobis.or.kr/kobisopenapi/webservice/rest/boxoffice/searchDailyBoxOfficeList.json From f7d11edc2e2431da0052e1c8f634548c69f8a400 Mon Sep 17 00:00:00 2001 From: HongGeun Date: Sat, 11 Nov 2023 14:13:52 +0900 Subject: [PATCH 04/37] add regions and apply ssh tunneling --- .../config/DatabaseConfiguration.java | 43 ++++++++----------- .../batch/infrastructure/utils/KmsUtils.java | 2 + 2 files changed, 21 insertions(+), 24 deletions(-) diff --git a/src/main/java/com/catcher/batch/datasource/config/DatabaseConfiguration.java b/src/main/java/com/catcher/batch/datasource/config/DatabaseConfiguration.java index f04cd47..6c8541b 100644 --- a/src/main/java/com/catcher/batch/datasource/config/DatabaseConfiguration.java +++ b/src/main/java/com/catcher/batch/datasource/config/DatabaseConfiguration.java @@ -53,32 +53,27 @@ public class DatabaseConfiguration { @Bean public DataSource dataSource() throws Exception { // KMS 활용한 연결 -// JSch jsch = new JSch(); -// Session session = jsch.getSession( -// KmsUtils.decrypt(sshUsername), -// KmsUtils.decrypt(sshHost), -// sshPort -// ); -// session.setPassword(KmsUtils.decrypt(sshPassword)); -// session.setConfig("StrictHostKeyChecking", "no"); -// session.connect(); -// -// int assignedPort = session.setPortForwardingL(0, -// KmsUtils.decrypt(originUrl), -// localPort -// ); // TODO: lport 값(현재 0)은 추후 서버 올릴때는 지정해줘야함 -// -// return DataSourceBuilder.create() -// .url(KmsUtils.decrypt(databaseUrl).replace(Integer.toString(localPort), Integer.toString(assignedPort))) -// .username(KmsUtils.decrypt(databaseUsername)) -// .password(KmsUtils.decrypt(databasePassword)) -// .build(); - // EKS ConfigMap & Secret + JSch jsch = new JSch(); + Session session = jsch.getSession( + KmsUtils.decrypt(sshUsername), + KmsUtils.decrypt(sshHost), + sshPort + ); + session.setPassword(KmsUtils.decrypt(sshPassword)); + session.setConfig("StrictHostKeyChecking", "no"); + session.connect(); + + int assignedPort = session.setPortForwardingL(0, + KmsUtils.decrypt(originUrl), + localPort + ); // TODO: lport 값(현재 0)은 추후 서버 올릴때는 지정해줘야함 + DriverManagerDataSource dataSource = new DriverManagerDataSource(); dataSource.setDriverClassName(driverClassName); - dataSource.setUrl(databaseUrl); - dataSource.setUsername(databaseUsername); - dataSource.setPassword(databasePassword); + dataSource.setUrl(KmsUtils.decrypt(databaseUrl).replace(Integer.toString(localPort), Integer.toString(assignedPort))); + dataSource.setUsername(KmsUtils.decrypt(databaseUsername)); + dataSource.setPassword(KmsUtils.decrypt(databasePassword)); + return dataSource; } } diff --git a/src/main/java/com/catcher/batch/infrastructure/utils/KmsUtils.java b/src/main/java/com/catcher/batch/infrastructure/utils/KmsUtils.java index 26f4136..cc00443 100644 --- a/src/main/java/com/catcher/batch/infrastructure/utils/KmsUtils.java +++ b/src/main/java/com/catcher/batch/infrastructure/utils/KmsUtils.java @@ -32,6 +32,7 @@ public class KmsUtils { public static String encrypt(String text) { try { AWSKMS kmsClient = AWSKMSClientBuilder.standard() + .withRegion(Regions.AP_NORTHEAST_2) .withCredentials(DefaultAWSCredentialsProviderChain.getInstance()) .build(); @@ -50,6 +51,7 @@ public static String encrypt(String text) { public static String decrypt(String cipherBase64) { try { AWSKMS kmsClient = AWSKMSClientBuilder.standard() + .withRegion(Regions.AP_NORTHEAST_2) .withCredentials(DefaultAWSCredentialsProviderChain.getInstance()) .build(); From 9c3313c0f73cbef63353e09b5f3048e75961ceca Mon Sep 17 00:00:00 2001 From: inyoung Date: Wed, 8 Nov 2023 00:32:10 +0900 Subject: [PATCH 05/37] =?UTF-8?q?feat:=20=EB=B0=B0=EC=B9=98=20=EA=B4=80?= =?UTF-8?q?=EB=A0=A8=20entity=20=EA=B5=AC=ED=98=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../core/domain/entity/BaseTimeEntity.java | 32 +++++++++ .../batch/core/domain/entity/CatcherItem.java | 69 +++++++++++++++++++ .../batch/core/domain/entity/Category.java | 26 +++++++ .../batch/core/domain/entity/Location.java | 24 +++++++ 4 files changed, 151 insertions(+) create mode 100644 src/main/java/com/catcher/batch/core/domain/entity/BaseTimeEntity.java create mode 100644 src/main/java/com/catcher/batch/core/domain/entity/CatcherItem.java create mode 100644 src/main/java/com/catcher/batch/core/domain/entity/Category.java create mode 100644 src/main/java/com/catcher/batch/core/domain/entity/Location.java diff --git a/src/main/java/com/catcher/batch/core/domain/entity/BaseTimeEntity.java b/src/main/java/com/catcher/batch/core/domain/entity/BaseTimeEntity.java new file mode 100644 index 0000000..fa0f69f --- /dev/null +++ b/src/main/java/com/catcher/batch/core/domain/entity/BaseTimeEntity.java @@ -0,0 +1,32 @@ +package com.catcher.batch.core.domain.entity; + +import jakarta.persistence.*; +import lombok.Getter; +import org.springframework.data.annotation.CreatedDate; +import org.springframework.data.annotation.LastModifiedDate; +import org.springframework.data.jpa.domain.support.AuditingEntityListener; + +import java.time.LocalDateTime; +import java.time.ZonedDateTime; + +@EntityListeners(AuditingEntityListener.class) +@MappedSuperclass +@Getter +public class BaseTimeEntity { + @Column(name = "created_at") + private ZonedDateTime createdAt; + + @Column(name = "updated_at") + private ZonedDateTime updatedAt; + + @PrePersist + private void prePersist() { + this.createdAt = ZonedDateTime.now(); + this.updatedAt = ZonedDateTime.now(); + } + + @PreUpdate + private void preUpdate() { + this.updatedAt = ZonedDateTime.now(); + } +} diff --git a/src/main/java/com/catcher/batch/core/domain/entity/CatcherItem.java b/src/main/java/com/catcher/batch/core/domain/entity/CatcherItem.java new file mode 100644 index 0000000..f0efc75 --- /dev/null +++ b/src/main/java/com/catcher/batch/core/domain/entity/CatcherItem.java @@ -0,0 +1,69 @@ +package com.catcher.batch.core.domain.entity; + +import jakarta.persistence.*; +import lombok.*; +import org.hibernate.annotations.Where; + +import java.time.LocalDateTime; +import java.time.ZonedDateTime; +import java.util.HashMap; +import java.util.Map; + +@Entity +@Builder +@Getter +@AllArgsConstructor(access = AccessLevel.PROTECTED) +@NoArgsConstructor(access = AccessLevel.PROTECTED) +@Where(clause = "delete_at = 'N'") +@Table(name = "catcher_item") +public class CatcherItem extends BaseTimeEntity { + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + private Long id; + + @ManyToOne(fetch = FetchType.LAZY) + @JoinColumn(name = "category_id", nullable = false) + private Category category; + + @ManyToOne(fetch = FetchType.LAZY) + @JoinColumn(name = "location_id") + private Location location; + +// @ElementCollection +// @CollectionTable( +// name = "item_hash_value_map", +// joinColumns = {@JoinColumn(name = "catcher_item_id", referencedColumnName = "id")} +// ) +// @MapKeyColumn(name="item_key") +// @Column(name = "item", unique = true, nullable = false) +// private Map itemHashValue = new HashMap<>(); + + @Column(unique = true, nullable = false) + private String itemHashValue; + + @Column(nullable = false) + private String title; + + private String description; + + private String thumbnailUrl; + + private String resourceUrl; + + @Column(name = "start_at") + private LocalDateTime startAt; + + @Column(name = "end_at") + private LocalDateTime endAt; + + @Column(name = "delete_at") + private ZonedDateTime deletedAt; + +// public void addHashValue(String key,Long value){ +// if (this.itemHashValue == null) { +// this.itemHashValue = new HashMap<>(); +// } +// +// this.itemHashValue.put(key,value); +// } +} diff --git a/src/main/java/com/catcher/batch/core/domain/entity/Category.java b/src/main/java/com/catcher/batch/core/domain/entity/Category.java new file mode 100644 index 0000000..2f96783 --- /dev/null +++ b/src/main/java/com/catcher/batch/core/domain/entity/Category.java @@ -0,0 +1,26 @@ +package com.catcher.batch.core.domain.entity; + +import jakarta.persistence.*; +import lombok.AccessLevel; +import lombok.Builder; +import lombok.Getter; +import lombok.NoArgsConstructor; + +@Entity +@Getter +@NoArgsConstructor(access = AccessLevel.PROTECTED) +@Table(name = "category") +public class Category { + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + private Long id; + + @Column(nullable = false) + private String name; + + public static Category create(String name) { + Category category = new Category(); + category.name = name; + return category; + } +} diff --git a/src/main/java/com/catcher/batch/core/domain/entity/Location.java b/src/main/java/com/catcher/batch/core/domain/entity/Location.java new file mode 100644 index 0000000..99995eb --- /dev/null +++ b/src/main/java/com/catcher/batch/core/domain/entity/Location.java @@ -0,0 +1,24 @@ +package com.catcher.batch.core.domain.entity; + +import jakarta.persistence.*; +import lombok.AccessLevel; +import lombok.Getter; +import lombok.NoArgsConstructor; + +@Entity +@Getter +@NoArgsConstructor(access = AccessLevel.PROTECTED) +@Table(name = "location") +public class Location { + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + private Long id; + + private String areaCode; + + private String latitude; + + private String longitude; + + private String description; +} From 93bfef4512401dbfbad38e8fb24f79b1cd9c5a8b Mon Sep 17 00:00:00 2001 From: inyoung Date: Wed, 8 Nov 2023 00:32:57 +0900 Subject: [PATCH 06/37] =?UTF-8?q?feat:=20camping=20property=20=EA=B5=AC?= =?UTF-8?q?=ED=98=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../properties/CampingProperties.java | 57 +++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100644 src/main/java/com/catcher/batch/infrastructure/properties/CampingProperties.java diff --git a/src/main/java/com/catcher/batch/infrastructure/properties/CampingProperties.java b/src/main/java/com/catcher/batch/infrastructure/properties/CampingProperties.java new file mode 100644 index 0000000..c226dcc --- /dev/null +++ b/src/main/java/com/catcher/batch/infrastructure/properties/CampingProperties.java @@ -0,0 +1,57 @@ +package com.catcher.batch.infrastructure.properties; + +import com.catcher.batch.core.dto.CampingApiResponse; +import com.catcher.batch.core.properties.PropertyBase; +import com.catcher.batch.infrastructure.utils.KmsUtils; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; +import org.springframework.web.util.UriComponentsBuilder; + +import java.io.UnsupportedEncodingException; +import java.net.URI; +import java.net.URLEncoder; +import java.util.Map; + +@Component +public class CampingProperties extends PropertyBase { + + @Value("${camping.key}") + private String serviceKey; + + public CampingProperties(@Value("${camping.baseUrl}") String endPoint) { + super(endPoint); + } + + @Override + public boolean support(Class clazz) { + return clazz.isAssignableFrom(CampingApiResponse.class); + } + + @Override + public URI getURI() { + try { + String key = URLEncoder.encode(KmsUtils.decrypt(serviceKey), "UTF-8"); + + UriComponentsBuilder uriBuilder = UriComponentsBuilder + .fromUriString(this.getEndPoint()) + .queryParam("MobileOS", "ETC") + .queryParam("MobileApp", "AppTest") + .queryParam("serviceKey", key) + .queryParam("_type", "json"); + + return this.addParams(uriBuilder) + .build(true).toUri(); + } catch (UnsupportedEncodingException e) { + throw new RuntimeException(e); + } + } + + private UriComponentsBuilder addParams(UriComponentsBuilder uriComponentsBuilder ) { + Map params = getParams(); + for (String key : params.keySet()) { + uriComponentsBuilder + .queryParam(key, params.get(key)); + } + return uriComponentsBuilder; + } +} From e0fe9097b3aa4e963281b80eefda0efd5462adde Mon Sep 17 00:00:00 2001 From: inyoung Date: Wed, 8 Nov 2023 00:33:59 +0900 Subject: [PATCH 07/37] =?UTF-8?q?feat:=20camping=20feign=20=ED=98=B8?= =?UTF-8?q?=EC=B6=9C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/catcher/batch/BatchApplication.java | 2 + .../batch/core/dto/CampingApiResponse.java | 74 +++++++++++++++++++ .../batch/resource/CampingController.java | 48 ++++++++++++ src/main/resources/application.properties | 2 + 4 files changed, 126 insertions(+) create mode 100644 src/main/java/com/catcher/batch/core/dto/CampingApiResponse.java create mode 100644 src/main/java/com/catcher/batch/resource/CampingController.java diff --git a/src/main/java/com/catcher/batch/BatchApplication.java b/src/main/java/com/catcher/batch/BatchApplication.java index 452d22e..374b10f 100644 --- a/src/main/java/com/catcher/batch/BatchApplication.java +++ b/src/main/java/com/catcher/batch/BatchApplication.java @@ -3,9 +3,11 @@ import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.cloud.openfeign.EnableFeignClients; +import org.springframework.data.jpa.repository.config.EnableJpaAuditing; @SpringBootApplication @EnableFeignClients(basePackages = "com.catcher.batch.resource.external") +@EnableJpaAuditing public class BatchApplication { public static void main(String[] args) { diff --git a/src/main/java/com/catcher/batch/core/dto/CampingApiResponse.java b/src/main/java/com/catcher/batch/core/dto/CampingApiResponse.java new file mode 100644 index 0000000..6a06307 --- /dev/null +++ b/src/main/java/com/catcher/batch/core/dto/CampingApiResponse.java @@ -0,0 +1,74 @@ +package com.catcher.batch.core.dto; + +import com.catcher.batch.annotation.CatcherJson; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.Getter; + +import java.util.List; + +@Getter +@JsonIgnoreProperties(ignoreUnknown = true) +@CatcherJson(path = "response.body") +public class CampingApiResponse { + + @JsonProperty("items") + private CampingItems items; + + @JsonProperty("totalCount") + private Integer totalCount; + + @JsonProperty("numOfRows") + private Integer numOfRows; + + @JsonProperty("pageNo") + private Integer pageNo; + + @Getter + @JsonIgnoreProperties(ignoreUnknown = true) + public static class CampingItems { + @JsonProperty("item") + private List item; + } + + @Getter + @JsonIgnoreProperties(ignoreUnknown = true) + public static class CampingItem { + + @JsonProperty("facltNm") + private String name; + + @JsonProperty("contentId") + private String key; + + @JsonProperty("lineIntro") + private String description; + + @JsonProperty("doNm") + private String province; + + @JsonProperty("sigunguNm") + private String city; + + @JsonProperty("addr1") + private String address; + + @JsonProperty("induty") + private String category; + + @JsonProperty("zipcode") + private String zipcode; + + @JsonProperty("homepage") + private String homepage; + + @JsonProperty("mapX") + private String mapX; + + @JsonProperty("mapY") + private String mapY; + + @JsonProperty("firstImageUrl") + private String thumbnailUrl; + } +} diff --git a/src/main/java/com/catcher/batch/resource/CampingController.java b/src/main/java/com/catcher/batch/resource/CampingController.java new file mode 100644 index 0000000..cf4d16d --- /dev/null +++ b/src/main/java/com/catcher/batch/resource/CampingController.java @@ -0,0 +1,48 @@ +package com.catcher.batch.resource; + +import com.catcher.batch.common.response.CommonResponse; +import com.catcher.batch.common.service.CatcherFeignService; +import com.catcher.batch.core.domain.CommandExecutor; +import com.catcher.batch.core.domain.command.RegisterCampingDataCommand; +import com.catcher.batch.core.dto.CampingApiResponse; +import com.catcher.batch.core.service.CampingService; +import lombok.RequiredArgsConstructor; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.*; +import java.util.HashMap; + +@RequiredArgsConstructor +@RestController +@RequestMapping("/camping") +public class CampingController { + private final CatcherFeignService catcherFeignService; + private final CampingService campingService; + private final CommandExecutor commandExecutor; + + @GetMapping("/feign-batch") + public CommonResponse getCampingDataByFeign( + @RequestParam(defaultValue = "1") Integer page, + @RequestParam(defaultValue = "5") Integer count + ) { + HashMap params = new HashMap<>(); + params.put("pageNo", page); + params.put("numOfRows", count); + CampingApiResponse campingApiResponse = catcherFeignService.parseService(params, CampingApiResponse.class); + + return CommonResponse.success(200, campingApiResponse); + } + + @PostMapping("/batch") + public CommonResponse batchCampingData( + @RequestParam(defaultValue = "1") Integer page, + @RequestParam(defaultValue = "1") Integer count + ) { + HashMap params = new HashMap<>(); + params.put("pageNo", page); + params.put("numOfRows", count); + CampingApiResponse campingApiResponse = catcherFeignService.parseService(params, CampingApiResponse.class); + + commandExecutor.run(new RegisterCampingDataCommand(campingService, campingApiResponse)); + return CommonResponse.success(201, null); + } +} diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index 2f7bd75..77ec0fb 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -26,6 +26,8 @@ movie.key=AQICAHg2j0pl3GguoQNkjcreuBPYJqI3OPeLEOelKIkMSzn01AEdAQDVPqdOhpfNbxFhaX ## festival api festival.baseUrl=http://api.data.go.kr/openapi/tn_pubr_public_cltur_fstvl_api festival.key=AQICAHg2j0pl3GguoQNkjcreuBPYJqI3OPeLEOelKIkMSzn01AGEXII6Ufa8dIEGD6MQNyM9AAAAujCBtwYJKoZIhvcNAQcGoIGpMIGmAgEAMIGgBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDEphbmyNQTiCTHtF/gIBEIBz8Fj7Ij/xk+O5hXWQfMOsIfc6uAYrvF2Xew+p0qLkX3DO0Plzjt4EMzaUP504RIMr7s+Yhx2y2Lq5SZ9I/cZ4swMYIOwj8FXDLeFy/k3dfwFdnBRh1kmvIDiMSg5kj4kgk05nzHyJ5KloYXXngp/ZQzsdpA== +camping.key=AQICAHg2j0pl3GguoQNkjcreuBPYJqI3OPeLEOelKIkMSzn01AF5N6HoIJmz3ef5mw7h81sOAAAAujCBtwYJKoZIhvcNAQcGoIGpMIGmAgEAMIGgBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDDvX80CMep9JqAHwVgIBEIBzD41zp/0NHhhvJ4nFvikmteZp501dy09Q/RjJ631ggAcTPaskdGKn6+qwvQVLw3vWesWOSycm3NiMeatVHUwSBZ4AKFGxfhHIsinf1Hyl2RdKjbbeYLYTtTiJY8jh4kZrXUPn6zQ4/vVr75WcIi9eny6lhw== +camping.baseUrl=http://apis.data.go.kr/B551011/GoCamping/basedList ## movie api tmdb movie.tmdb.baseUrl=https://api.themoviedb.org/3/discover/movie From 7eb6f2c9ebfdb9d78b825dff5f4c47523f20857a Mon Sep 17 00:00:00 2001 From: inyoung Date: Wed, 8 Nov 2023 00:34:25 +0900 Subject: [PATCH 08/37] =?UTF-8?q?test:=20camping=20feign=20=ED=98=B8?= =?UTF-8?q?=EC=B6=9C=20test=20=EC=BD=94=EB=93=9C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../batch/resource/CampingControllerTest.java | 52 +++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 src/test/java/com/catcher/batch/resource/CampingControllerTest.java diff --git a/src/test/java/com/catcher/batch/resource/CampingControllerTest.java b/src/test/java/com/catcher/batch/resource/CampingControllerTest.java new file mode 100644 index 0000000..1a814c8 --- /dev/null +++ b/src/test/java/com/catcher/batch/resource/CampingControllerTest.java @@ -0,0 +1,52 @@ +package com.catcher.batch.resource; + +import com.catcher.batch.common.service.CatcherFeignService; +import com.catcher.batch.core.domain.CommandExecutor; +import com.catcher.batch.core.domain.command.RegisterCampingDataCommand; +import com.catcher.batch.core.dto.CampingApiResponse; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.http.MediaType; +import org.springframework.test.web.servlet.MockMvc; +import org.springframework.test.web.servlet.request.MockMvcRequestBuilders; +import org.springframework.test.web.servlet.result.MockMvcResultMatchers; + +import java.util.HashMap; + +import static org.mockito.Mockito.when; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; + +@SpringBootTest(classes = CampingController.class) +@AutoConfigureMockMvc +class CampingControllerTest { + + @MockBean + private CatcherFeignService catcherFeignService; + + @MockBean + private CommandExecutor commandExecutor; + + @Autowired + private MockMvc mockMvc; + + @DisplayName("SUCCESS : 캠핑 api 응답 200 테스트") + @Test + void getCampingDataByFeign() throws Exception { + CampingApiResponse campingApiResponse = new CampingApiResponse(); + + when(catcherFeignService.parseService(new HashMap<>(), CampingApiResponse.class)) + .thenReturn(campingApiResponse); + + mockMvc.perform(MockMvcRequestBuilders + .get("/camping/feign-batch") + .param("page", "1") + .param("count", "5") + .accept(MediaType.APPLICATION_JSON)) + .andExpect(status().isOk()); + } +} From 5f5c79579b7bb20d38d906bee90703a5f9e1d134 Mon Sep 17 00:00:00 2001 From: inyoung Date: Wed, 8 Nov 2023 00:35:42 +0900 Subject: [PATCH 09/37] =?UTF-8?q?feat:=20=EC=BA=A0=ED=95=91=20=EC=95=84?= =?UTF-8?q?=EC=9D=B4=ED=85=9C=20db=20=EC=A0=80=EC=9E=A5=20=EA=B5=AC?= =?UTF-8?q?=ED=98=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../core/database/CatcherItemRepository.java | 12 ++++ .../core/database/LocationRepository.java | 9 +++ .../core/domain/CatcherItemExecutor.java | 12 ++++ .../batch/core/domain/CommandExecutor.java | 7 ++ .../batch/core/domain/command/Command.java | 5 ++ .../command/RegisterCampingDataCommand.java | 18 +++++ .../batch/core/service/CampingService.java | 70 +++++++++++++++++++ .../datasource/CatcherItemJpaRepository.java | 10 +++ .../datasource/CatcherItemRepositoryImpl.java | 25 +++++++ .../batch/datasource/CategoryRepository.java | 10 +++ .../datasource/LocationJpaRepository.java | 10 +++ .../datasource/LocationRepositoryImpl.java | 19 +++++ 12 files changed, 207 insertions(+) create mode 100644 src/main/java/com/catcher/batch/core/database/CatcherItemRepository.java create mode 100644 src/main/java/com/catcher/batch/core/database/LocationRepository.java create mode 100644 src/main/java/com/catcher/batch/core/domain/CatcherItemExecutor.java create mode 100644 src/main/java/com/catcher/batch/core/domain/CommandExecutor.java create mode 100644 src/main/java/com/catcher/batch/core/domain/command/Command.java create mode 100644 src/main/java/com/catcher/batch/core/domain/command/RegisterCampingDataCommand.java create mode 100644 src/main/java/com/catcher/batch/core/service/CampingService.java create mode 100644 src/main/java/com/catcher/batch/datasource/CatcherItemJpaRepository.java create mode 100644 src/main/java/com/catcher/batch/datasource/CatcherItemRepositoryImpl.java create mode 100644 src/main/java/com/catcher/batch/datasource/CategoryRepository.java create mode 100644 src/main/java/com/catcher/batch/datasource/LocationJpaRepository.java create mode 100644 src/main/java/com/catcher/batch/datasource/LocationRepositoryImpl.java diff --git a/src/main/java/com/catcher/batch/core/database/CatcherItemRepository.java b/src/main/java/com/catcher/batch/core/database/CatcherItemRepository.java new file mode 100644 index 0000000..21b69a8 --- /dev/null +++ b/src/main/java/com/catcher/batch/core/database/CatcherItemRepository.java @@ -0,0 +1,12 @@ +package com.catcher.batch.core.database; + +import com.catcher.batch.core.domain.entity.CatcherItem; + +import java.util.List; +import java.util.Optional; + +public interface CatcherItemRepository { + void saveAll(List catcherItems); + + Optional findByItemHashValue(String hashKey); +} diff --git a/src/main/java/com/catcher/batch/core/database/LocationRepository.java b/src/main/java/com/catcher/batch/core/database/LocationRepository.java new file mode 100644 index 0000000..872d45c --- /dev/null +++ b/src/main/java/com/catcher/batch/core/database/LocationRepository.java @@ -0,0 +1,9 @@ +package com.catcher.batch.core.database; + +import com.catcher.batch.core.domain.entity.Location; + +import java.util.Optional; + +public interface LocationRepository { + Optional findByDescription(String province, String city); +} diff --git a/src/main/java/com/catcher/batch/core/domain/CatcherItemExecutor.java b/src/main/java/com/catcher/batch/core/domain/CatcherItemExecutor.java new file mode 100644 index 0000000..a3941a5 --- /dev/null +++ b/src/main/java/com/catcher/batch/core/domain/CatcherItemExecutor.java @@ -0,0 +1,12 @@ +package com.catcher.batch.core.domain; + +import com.catcher.batch.core.domain.command.Command; +import org.springframework.stereotype.Component; + +@Component +public class CatcherItemExecutor implements CommandExecutor { + @Override + public T run(Command command) { + return command.execute(); + } +} diff --git a/src/main/java/com/catcher/batch/core/domain/CommandExecutor.java b/src/main/java/com/catcher/batch/core/domain/CommandExecutor.java new file mode 100644 index 0000000..939a173 --- /dev/null +++ b/src/main/java/com/catcher/batch/core/domain/CommandExecutor.java @@ -0,0 +1,7 @@ +package com.catcher.batch.core.domain; + +import com.catcher.batch.core.domain.command.Command; + +public interface CommandExecutor { + T run(Command command); +} diff --git a/src/main/java/com/catcher/batch/core/domain/command/Command.java b/src/main/java/com/catcher/batch/core/domain/command/Command.java new file mode 100644 index 0000000..89b044e --- /dev/null +++ b/src/main/java/com/catcher/batch/core/domain/command/Command.java @@ -0,0 +1,5 @@ +package com.catcher.batch.core.domain.command; + +public interface Command { + T execute(); +} diff --git a/src/main/java/com/catcher/batch/core/domain/command/RegisterCampingDataCommand.java b/src/main/java/com/catcher/batch/core/domain/command/RegisterCampingDataCommand.java new file mode 100644 index 0000000..d9099ca --- /dev/null +++ b/src/main/java/com/catcher/batch/core/domain/command/RegisterCampingDataCommand.java @@ -0,0 +1,18 @@ +package com.catcher.batch.core.domain.command; + +import com.catcher.batch.core.dto.CampingApiResponse; +import com.catcher.batch.core.service.CampingService; +import lombok.RequiredArgsConstructor; + + +@RequiredArgsConstructor +public class RegisterCampingDataCommand implements Command { + private final CampingService campingService; + private final CampingApiResponse campingApiResponse; + + @Override + public Void execute() { + campingService.batch(campingApiResponse); + return null; + } +} diff --git a/src/main/java/com/catcher/batch/core/service/CampingService.java b/src/main/java/com/catcher/batch/core/service/CampingService.java new file mode 100644 index 0000000..f2d092a --- /dev/null +++ b/src/main/java/com/catcher/batch/core/service/CampingService.java @@ -0,0 +1,70 @@ +package com.catcher.batch.core.service; + +import com.catcher.batch.core.database.CatcherItemRepository; +import com.catcher.batch.core.database.LocationRepository; +import com.catcher.batch.core.domain.entity.CatcherItem; +import com.catcher.batch.core.domain.entity.Category; +import com.catcher.batch.core.domain.entity.Location; +import com.catcher.batch.core.dto.CampingApiResponse; +import com.catcher.batch.datasource.CategoryRepository; +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + +import java.util.ArrayList; +import java.util.List; + +@Service +@RequiredArgsConstructor +public class CampingService { + private final CatcherItemRepository catcherItemRepository; + private final CategoryRepository categoryRepository; + private final LocationRepository locationRepository; + public static final String CATEGORY_NAME = "camping"; + + @Transactional + public void batch(CampingApiResponse campingApiResponse) { + Category category = categoryRepository.findByName(CATEGORY_NAME) + .orElseGet(() -> categoryRepository.save(Category.create(CATEGORY_NAME))); + + List campingItems = campingApiResponse.getItems().getItem(); + List catcherItems = new ArrayList<>(); + + for (CampingApiResponse.CampingItem campingItem : campingItems) { + Location location = getLocationByDescription(campingItem.getProvince(), campingItem.getCity()); + + String hashKey = campingItem.getKey(); + + // 중복 해시값 체크 + if (isDuplicateHashValue(hashKey)) { + System.out.println("중복됨"); + continue; + } + + CatcherItem catcherItem = CatcherItem.builder() + .category(category) + .location(location) + .title(campingItem.getName()) + .description(campingItem.getDescription()) + .thumbnailUrl(campingItem.getThumbnailUrl()) + .itemHashValue(hashKey) + .build(); + + catcherItems.add(catcherItem); + } + + catcherItemRepository.saveAll(catcherItems); + } + + private Location getLocationByDescription(String province, String city) { + String withoutDo = province.replace("도", ""); + + return locationRepository.findByDescription(withoutDo, city) + .orElseThrow(); + } + + // TODO : 중복체크가 안됨 ㅜ.ㅜ + private boolean isDuplicateHashValue(String hashKey) { + return catcherItemRepository.findByItemHashValue(hashKey).isPresent(); + } +} diff --git a/src/main/java/com/catcher/batch/datasource/CatcherItemJpaRepository.java b/src/main/java/com/catcher/batch/datasource/CatcherItemJpaRepository.java new file mode 100644 index 0000000..283f810 --- /dev/null +++ b/src/main/java/com/catcher/batch/datasource/CatcherItemJpaRepository.java @@ -0,0 +1,10 @@ +package com.catcher.batch.datasource; + +import com.catcher.batch.core.domain.entity.CatcherItem; +import org.springframework.data.jpa.repository.JpaRepository; + +import java.util.Optional; + +public interface CatcherItemJpaRepository extends JpaRepository { + Optional findByItemHashValue(String itemHashValue); +} diff --git a/src/main/java/com/catcher/batch/datasource/CatcherItemRepositoryImpl.java b/src/main/java/com/catcher/batch/datasource/CatcherItemRepositoryImpl.java new file mode 100644 index 0000000..13e7f1c --- /dev/null +++ b/src/main/java/com/catcher/batch/datasource/CatcherItemRepositoryImpl.java @@ -0,0 +1,25 @@ +package com.catcher.batch.datasource; + +import com.catcher.batch.core.database.CatcherItemRepository; +import com.catcher.batch.core.domain.entity.CatcherItem; +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Repository; + +import java.util.List; +import java.util.Optional; + +@Repository +@RequiredArgsConstructor +public class CatcherItemRepositoryImpl implements CatcherItemRepository { + private final CatcherItemJpaRepository catcherItemJpaRepository; + + @Override + public void saveAll(List catcherItems) { + catcherItemJpaRepository.saveAll(catcherItems); + } + + @Override + public Optional findByItemHashValue(String hashKey) { + return catcherItemJpaRepository.findByItemHashValue(hashKey); + } +} diff --git a/src/main/java/com/catcher/batch/datasource/CategoryRepository.java b/src/main/java/com/catcher/batch/datasource/CategoryRepository.java new file mode 100644 index 0000000..9a1b7fa --- /dev/null +++ b/src/main/java/com/catcher/batch/datasource/CategoryRepository.java @@ -0,0 +1,10 @@ +package com.catcher.batch.datasource; + +import com.catcher.batch.core.domain.entity.Category; +import org.springframework.data.jpa.repository.JpaRepository; + +import java.util.Optional; + +public interface CategoryRepository extends JpaRepository { + Optional findByName(String name); +} diff --git a/src/main/java/com/catcher/batch/datasource/LocationJpaRepository.java b/src/main/java/com/catcher/batch/datasource/LocationJpaRepository.java new file mode 100644 index 0000000..3d9ba71 --- /dev/null +++ b/src/main/java/com/catcher/batch/datasource/LocationJpaRepository.java @@ -0,0 +1,10 @@ +package com.catcher.batch.datasource; + +import com.catcher.batch.core.domain.entity.Location; +import org.springframework.data.jpa.repository.JpaRepository; + +import java.util.Optional; + +public interface LocationJpaRepository extends JpaRepository { + Optional findByDescriptionStartingWithAndDescriptionEndingWith(String start, String end); +} diff --git a/src/main/java/com/catcher/batch/datasource/LocationRepositoryImpl.java b/src/main/java/com/catcher/batch/datasource/LocationRepositoryImpl.java new file mode 100644 index 0000000..28c55b3 --- /dev/null +++ b/src/main/java/com/catcher/batch/datasource/LocationRepositoryImpl.java @@ -0,0 +1,19 @@ +package com.catcher.batch.datasource; + +import com.catcher.batch.core.database.LocationRepository; +import com.catcher.batch.core.domain.entity.Location; +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Repository; + +import java.util.Optional; + +@Repository +@RequiredArgsConstructor +public class LocationRepositoryImpl implements LocationRepository { + private final LocationJpaRepository locationJpaRepository; + + @Override + public Optional findByDescription(String start, String end) { + return locationJpaRepository.findByDescriptionStartingWithAndDescriptionEndingWith(start, end); + } +} From 56efafe8aedd38c7097c5493735c92db445bb6e5 Mon Sep 17 00:00:00 2001 From: inyoung Date: Wed, 8 Nov 2023 01:03:18 +0900 Subject: [PATCH 10/37] =?UTF-8?q?fix:=20test=20=EC=BD=94=EB=93=9C=20?= =?UTF-8?q?=EC=98=A4=EB=A5=98=20=EC=88=98=EC=A0=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/catcher/batch/resource/CampingController.java | 4 +++- .../com/catcher/batch/resource/CampingControllerTest.java | 8 +++++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/src/main/java/com/catcher/batch/resource/CampingController.java b/src/main/java/com/catcher/batch/resource/CampingController.java index cf4d16d..1edc514 100644 --- a/src/main/java/com/catcher/batch/resource/CampingController.java +++ b/src/main/java/com/catcher/batch/resource/CampingController.java @@ -7,13 +7,15 @@ import com.catcher.batch.core.dto.CampingApiResponse; import com.catcher.batch.core.service.CampingService; import lombok.RequiredArgsConstructor; -import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.*; +import org.springframework.web.servlet.config.annotation.EnableWebMvc; + import java.util.HashMap; @RequiredArgsConstructor @RestController @RequestMapping("/camping") +@EnableWebMvc public class CampingController { private final CatcherFeignService catcherFeignService; private final CampingService campingService; diff --git a/src/test/java/com/catcher/batch/resource/CampingControllerTest.java b/src/test/java/com/catcher/batch/resource/CampingControllerTest.java index 1a814c8..c6562ce 100644 --- a/src/test/java/com/catcher/batch/resource/CampingControllerTest.java +++ b/src/test/java/com/catcher/batch/resource/CampingControllerTest.java @@ -2,11 +2,10 @@ import com.catcher.batch.common.service.CatcherFeignService; import com.catcher.batch.core.domain.CommandExecutor; -import com.catcher.batch.core.domain.command.RegisterCampingDataCommand; import com.catcher.batch.core.dto.CampingApiResponse; +import com.catcher.batch.core.service.CampingService; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; -import org.mockito.Mockito; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc; import org.springframework.boot.test.context.SpringBootTest; @@ -14,7 +13,6 @@ import org.springframework.http.MediaType; import org.springframework.test.web.servlet.MockMvc; import org.springframework.test.web.servlet.request.MockMvcRequestBuilders; -import org.springframework.test.web.servlet.result.MockMvcResultMatchers; import java.util.HashMap; @@ -31,6 +29,10 @@ class CampingControllerTest { @MockBean private CommandExecutor commandExecutor; + @MockBean + private CampingService campingService; + + @Autowired private MockMvc mockMvc; From f5b3817fa946a4868f2a98ec33fce057f8cf1019 Mon Sep 17 00:00:00 2001 From: inyoung Date: Wed, 8 Nov 2023 01:53:46 +0900 Subject: [PATCH 11/37] =?UTF-8?q?fix:=20test=20=EC=BD=94=EB=93=9C=EC=97=90?= =?UTF-8?q?=20MockBean=20=EC=B6=94=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../java/com/catcher/batch/resource/CampingControllerTest.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/test/java/com/catcher/batch/resource/CampingControllerTest.java b/src/test/java/com/catcher/batch/resource/CampingControllerTest.java index c6562ce..bc3bd85 100644 --- a/src/test/java/com/catcher/batch/resource/CampingControllerTest.java +++ b/src/test/java/com/catcher/batch/resource/CampingControllerTest.java @@ -10,6 +10,7 @@ import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.data.jpa.mapping.JpaMetamodelMappingContext; import org.springframework.http.MediaType; import org.springframework.test.web.servlet.MockMvc; import org.springframework.test.web.servlet.request.MockMvcRequestBuilders; @@ -20,6 +21,7 @@ import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; @SpringBootTest(classes = CampingController.class) +@MockBean(JpaMetamodelMappingContext.class) @AutoConfigureMockMvc class CampingControllerTest { From e861f472cba09a2cc87bd8cab45705f8624ce6e1 Mon Sep 17 00:00:00 2001 From: inyoung Date: Wed, 8 Nov 2023 02:03:36 +0900 Subject: [PATCH 12/37] =?UTF-8?q?fix:=20test=20=EC=BD=94=EB=93=9C=EC=97=90?= =?UTF-8?q?=20MockBean=20=EC=B6=94=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/test/java/com/catcher/batch/BatchApplicationTests.java | 3 +++ .../com/catcher/batch/resource/HealthCheckControllerTest.java | 3 +++ 2 files changed, 6 insertions(+) diff --git a/src/test/java/com/catcher/batch/BatchApplicationTests.java b/src/test/java/com/catcher/batch/BatchApplicationTests.java index 5360bca..95d3a8b 100644 --- a/src/test/java/com/catcher/batch/BatchApplicationTests.java +++ b/src/test/java/com/catcher/batch/BatchApplicationTests.java @@ -2,8 +2,11 @@ import org.junit.jupiter.api.Test; import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.data.jpa.mapping.JpaMetamodelMappingContext; @SpringBootTest +@MockBean(JpaMetamodelMappingContext.class) class BatchApplicationTests { @Test diff --git a/src/test/java/com/catcher/batch/resource/HealthCheckControllerTest.java b/src/test/java/com/catcher/batch/resource/HealthCheckControllerTest.java index 597e375..207aef3 100644 --- a/src/test/java/com/catcher/batch/resource/HealthCheckControllerTest.java +++ b/src/test/java/com/catcher/batch/resource/HealthCheckControllerTest.java @@ -5,6 +5,8 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.autoconfigure.security.servlet.SecurityAutoConfiguration; import org.springframework.boot.test.autoconfigure.web.servlet.WebMvcTest; +import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.data.jpa.mapping.JpaMetamodelMappingContext; import org.springframework.http.MediaType; import org.springframework.test.web.servlet.MockMvc; import org.springframework.test.web.servlet.ResultActions; @@ -16,6 +18,7 @@ value = HealthCheckController.class, excludeAutoConfiguration = {SecurityAutoConfiguration.class} ) +@MockBean(JpaMetamodelMappingContext.class) class HealthCheckControllerTest { @Autowired From bd27fd4e294650dfd9a591f7869fd2f01fa6f1b6 Mon Sep 17 00:00:00 2001 From: inyoung Date: Wed, 8 Nov 2023 02:10:10 +0900 Subject: [PATCH 13/37] =?UTF-8?q?fix:=20@SpringBootTest=20=EC=A3=BC?= =?UTF-8?q?=EC=84=9D=EC=B2=98=EB=A6=AC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/test/java/com/catcher/batch/BatchApplicationTests.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/com/catcher/batch/BatchApplicationTests.java b/src/test/java/com/catcher/batch/BatchApplicationTests.java index 95d3a8b..048cdcf 100644 --- a/src/test/java/com/catcher/batch/BatchApplicationTests.java +++ b/src/test/java/com/catcher/batch/BatchApplicationTests.java @@ -5,7 +5,7 @@ import org.springframework.boot.test.mock.mockito.MockBean; import org.springframework.data.jpa.mapping.JpaMetamodelMappingContext; -@SpringBootTest +//@SpringBootTest @MockBean(JpaMetamodelMappingContext.class) class BatchApplicationTests { From 39b9957715c6a63fc5193a679705cbc9fbd2e147 Mon Sep 17 00:00:00 2001 From: inyoung Date: Wed, 8 Nov 2023 02:18:59 +0900 Subject: [PATCH 14/37] =?UTF-8?q?fix:=20@SpringBootTest=20=EC=A0=9C?= =?UTF-8?q?=EA=B1=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/catcher/batch/resource/CampingControllerTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/java/com/catcher/batch/resource/CampingControllerTest.java b/src/test/java/com/catcher/batch/resource/CampingControllerTest.java index bc3bd85..bb9f8d4 100644 --- a/src/test/java/com/catcher/batch/resource/CampingControllerTest.java +++ b/src/test/java/com/catcher/batch/resource/CampingControllerTest.java @@ -8,7 +8,7 @@ import org.junit.jupiter.api.Test; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc; -import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.autoconfigure.web.servlet.WebMvcTest; import org.springframework.boot.test.mock.mockito.MockBean; import org.springframework.data.jpa.mapping.JpaMetamodelMappingContext; import org.springframework.http.MediaType; @@ -20,7 +20,7 @@ import static org.mockito.Mockito.when; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; -@SpringBootTest(classes = CampingController.class) +@WebMvcTest(value = CampingController.class) @MockBean(JpaMetamodelMappingContext.class) @AutoConfigureMockMvc class CampingControllerTest { From 386ce4cc9af7c10d35f56003f065727c45c50c8d Mon Sep 17 00:00:00 2001 From: inyoung Date: Wed, 8 Nov 2023 02:20:25 +0900 Subject: [PATCH 15/37] =?UTF-8?q?fix:=20security=20=EB=B9=84=ED=99=9C?= =?UTF-8?q?=EC=84=B1=ED=99=94?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/catcher/batch/resource/CampingControllerTest.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/test/java/com/catcher/batch/resource/CampingControllerTest.java b/src/test/java/com/catcher/batch/resource/CampingControllerTest.java index bb9f8d4..7117165 100644 --- a/src/test/java/com/catcher/batch/resource/CampingControllerTest.java +++ b/src/test/java/com/catcher/batch/resource/CampingControllerTest.java @@ -7,6 +7,7 @@ import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.security.servlet.SecurityAutoConfiguration; import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc; import org.springframework.boot.test.autoconfigure.web.servlet.WebMvcTest; import org.springframework.boot.test.mock.mockito.MockBean; @@ -20,7 +21,9 @@ import static org.mockito.Mockito.when; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; -@WebMvcTest(value = CampingController.class) +@WebMvcTest( + value = CampingController.class, + excludeAutoConfiguration = {SecurityAutoConfiguration.class}) @MockBean(JpaMetamodelMappingContext.class) @AutoConfigureMockMvc class CampingControllerTest { From 9b5f24613a1e2dd93853e796e6ab64b28fb65391 Mon Sep 17 00:00:00 2001 From: inyoung Date: Thu, 9 Nov 2023 00:27:30 +0900 Subject: [PATCH 16/37] =?UTF-8?q?refactor:=20hashValue=20=EC=95=94?= =?UTF-8?q?=ED=98=B8=ED=99=94?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../batch/core/domain/entity/CatcherItem.java | 18 +----------------- .../batch/core/service/CampingService.java | 7 ++++++- .../batch/resource/CampingController.java | 2 +- 3 files changed, 8 insertions(+), 19 deletions(-) diff --git a/src/main/java/com/catcher/batch/core/domain/entity/CatcherItem.java b/src/main/java/com/catcher/batch/core/domain/entity/CatcherItem.java index f0efc75..565d2b0 100644 --- a/src/main/java/com/catcher/batch/core/domain/entity/CatcherItem.java +++ b/src/main/java/com/catcher/batch/core/domain/entity/CatcherItem.java @@ -2,6 +2,7 @@ import jakarta.persistence.*; import lombok.*; +import org.apache.commons.codec.digest.DigestUtils; import org.hibernate.annotations.Where; import java.time.LocalDateTime; @@ -29,15 +30,6 @@ public class CatcherItem extends BaseTimeEntity { @JoinColumn(name = "location_id") private Location location; -// @ElementCollection -// @CollectionTable( -// name = "item_hash_value_map", -// joinColumns = {@JoinColumn(name = "catcher_item_id", referencedColumnName = "id")} -// ) -// @MapKeyColumn(name="item_key") -// @Column(name = "item", unique = true, nullable = false) -// private Map itemHashValue = new HashMap<>(); - @Column(unique = true, nullable = false) private String itemHashValue; @@ -58,12 +50,4 @@ public class CatcherItem extends BaseTimeEntity { @Column(name = "delete_at") private ZonedDateTime deletedAt; - -// public void addHashValue(String key,Long value){ -// if (this.itemHashValue == null) { -// this.itemHashValue = new HashMap<>(); -// } -// -// this.itemHashValue.put(key,value); -// } } diff --git a/src/main/java/com/catcher/batch/core/service/CampingService.java b/src/main/java/com/catcher/batch/core/service/CampingService.java index f2d092a..0c31507 100644 --- a/src/main/java/com/catcher/batch/core/service/CampingService.java +++ b/src/main/java/com/catcher/batch/core/service/CampingService.java @@ -8,6 +8,7 @@ import com.catcher.batch.core.dto.CampingApiResponse; import com.catcher.batch.datasource.CategoryRepository; import lombok.RequiredArgsConstructor; +import org.apache.commons.codec.digest.DigestUtils; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; @@ -33,7 +34,7 @@ public void batch(CampingApiResponse campingApiResponse) { for (CampingApiResponse.CampingItem campingItem : campingItems) { Location location = getLocationByDescription(campingItem.getProvince(), campingItem.getCity()); - String hashKey = campingItem.getKey(); + String hashKey = hashString(campingItem.getKey()); // 중복 해시값 체크 if (isDuplicateHashValue(hashKey)) { @@ -67,4 +68,8 @@ private Location getLocationByDescription(String province, String city) { private boolean isDuplicateHashValue(String hashKey) { return catcherItemRepository.findByItemHashValue(hashKey).isPresent(); } + + private String hashString(String input) { + return DigestUtils.sha256Hex(input); + } } diff --git a/src/main/java/com/catcher/batch/resource/CampingController.java b/src/main/java/com/catcher/batch/resource/CampingController.java index 1edc514..64ef054 100644 --- a/src/main/java/com/catcher/batch/resource/CampingController.java +++ b/src/main/java/com/catcher/batch/resource/CampingController.java @@ -37,7 +37,7 @@ public CommonResponse getCampingDataByFeign( @PostMapping("/batch") public CommonResponse batchCampingData( @RequestParam(defaultValue = "1") Integer page, - @RequestParam(defaultValue = "1") Integer count + @RequestParam(defaultValue = "5") Integer count ) { HashMap params = new HashMap<>(); params.put("pageNo", page); From bdd09025c5612fa2c34f7425f6fb487712b6880f Mon Sep 17 00:00:00 2001 From: inyoung Date: Thu, 9 Nov 2023 00:28:01 +0900 Subject: [PATCH 17/37] =?UTF-8?q?chore:=20=EC=82=AC=EC=9A=A9=ED=95=98?= =?UTF-8?q?=EC=A7=80=20=EC=95=8A=EB=8A=94=20import=EB=AC=B8=20=EC=A0=9C?= =?UTF-8?q?=EA=B1=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../java/com/catcher/batch/core/domain/entity/CatcherItem.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/main/java/com/catcher/batch/core/domain/entity/CatcherItem.java b/src/main/java/com/catcher/batch/core/domain/entity/CatcherItem.java index 565d2b0..61604e8 100644 --- a/src/main/java/com/catcher/batch/core/domain/entity/CatcherItem.java +++ b/src/main/java/com/catcher/batch/core/domain/entity/CatcherItem.java @@ -2,13 +2,10 @@ import jakarta.persistence.*; import lombok.*; -import org.apache.commons.codec.digest.DigestUtils; import org.hibernate.annotations.Where; import java.time.LocalDateTime; import java.time.ZonedDateTime; -import java.util.HashMap; -import java.util.Map; @Entity @Builder From 99768e5cfc868dfe7f39e4da8577db5a769e7320 Mon Sep 17 00:00:00 2001 From: inyoung Date: Thu, 9 Nov 2023 00:28:33 +0900 Subject: [PATCH 18/37] =?UTF-8?q?fix:=20=EB=B3=91=ED=95=A9=EC=B6=A9?= =?UTF-8?q?=EB=8F=8C=ED=95=B4=EA=B2=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/test/java/com/catcher/batch/BatchApplicationTests.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/com/catcher/batch/BatchApplicationTests.java b/src/test/java/com/catcher/batch/BatchApplicationTests.java index 048cdcf..95d3a8b 100644 --- a/src/test/java/com/catcher/batch/BatchApplicationTests.java +++ b/src/test/java/com/catcher/batch/BatchApplicationTests.java @@ -5,7 +5,7 @@ import org.springframework.boot.test.mock.mockito.MockBean; import org.springframework.data.jpa.mapping.JpaMetamodelMappingContext; -//@SpringBootTest +@SpringBootTest @MockBean(JpaMetamodelMappingContext.class) class BatchApplicationTests { From 813d2355b7db7622d0b61a93a50594be44eb94f1 Mon Sep 17 00:00:00 2001 From: inyoung Date: Sun, 12 Nov 2023 17:56:33 +0900 Subject: [PATCH 19/37] =?UTF-8?q?refactor:=20localdatetime=EC=9D=84=20zone?= =?UTF-8?q?=EC=9C=BC=EB=A1=9C=20=EB=B3=80=EA=B2=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/catcher/batch/core/domain/entity/CatcherItem.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/main/java/com/catcher/batch/core/domain/entity/CatcherItem.java b/src/main/java/com/catcher/batch/core/domain/entity/CatcherItem.java index 61604e8..0172fb4 100644 --- a/src/main/java/com/catcher/batch/core/domain/entity/CatcherItem.java +++ b/src/main/java/com/catcher/batch/core/domain/entity/CatcherItem.java @@ -4,7 +4,6 @@ import lombok.*; import org.hibernate.annotations.Where; -import java.time.LocalDateTime; import java.time.ZonedDateTime; @Entity @@ -40,10 +39,10 @@ public class CatcherItem extends BaseTimeEntity { private String resourceUrl; @Column(name = "start_at") - private LocalDateTime startAt; + private ZonedDateTime startAt; @Column(name = "end_at") - private LocalDateTime endAt; + private ZonedDateTime endAt; @Column(name = "delete_at") private ZonedDateTime deletedAt; From 036fa0e095d8078e3dc7e7ab9ccf815c591b457b Mon Sep 17 00:00:00 2001 From: inyoung Date: Sun, 12 Nov 2023 17:58:41 +0900 Subject: [PATCH 20/37] =?UTF-8?q?refactor:=20=EC=9C=84=EB=8F=84,=20?= =?UTF-8?q?=EA=B2=BD=EB=8F=84=20=EB=B3=80=EC=88=98=EB=AA=85=20=EB=B3=80?= =?UTF-8?q?=EA=B2=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../java/com/catcher/batch/core/dto/CampingApiResponse.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/catcher/batch/core/dto/CampingApiResponse.java b/src/main/java/com/catcher/batch/core/dto/CampingApiResponse.java index 6a06307..042fecc 100644 --- a/src/main/java/com/catcher/batch/core/dto/CampingApiResponse.java +++ b/src/main/java/com/catcher/batch/core/dto/CampingApiResponse.java @@ -63,10 +63,10 @@ public static class CampingItem { private String homepage; @JsonProperty("mapX") - private String mapX; + private String latitude; @JsonProperty("mapY") - private String mapY; + private String longitude; @JsonProperty("firstImageUrl") private String thumbnailUrl; From c5143458ed504cbc52e78c90f30fe65f05ea48ee Mon Sep 17 00:00:00 2001 From: ddnjs Date: Sun, 12 Nov 2023 20:48:56 +0900 Subject: [PATCH 21/37] =?UTF-8?q?=EC=A0=84=EC=8B=9C=ED=9A=8C=20db=20?= =?UTF-8?q?=EC=A0=80=EC=9E=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../service/ExhibitionService.java} | 33 +++++++++++++++++-- .../batch/resource/CrawlerController.java | 7 ++-- 2 files changed, 33 insertions(+), 7 deletions(-) rename src/main/java/com/catcher/batch/{common/service/CatcherCrawlerService.java => core/service/ExhibitionService.java} (71%) diff --git a/src/main/java/com/catcher/batch/common/service/CatcherCrawlerService.java b/src/main/java/com/catcher/batch/core/service/ExhibitionService.java similarity index 71% rename from src/main/java/com/catcher/batch/common/service/CatcherCrawlerService.java rename to src/main/java/com/catcher/batch/core/service/ExhibitionService.java index f39ff3e..e8a7739 100644 --- a/src/main/java/com/catcher/batch/common/service/CatcherCrawlerService.java +++ b/src/main/java/com/catcher/batch/core/service/ExhibitionService.java @@ -1,7 +1,12 @@ -package com.catcher.batch.common.service; +package com.catcher.batch.core.service; +import com.catcher.batch.core.database.CatcherItemRepository; +import com.catcher.batch.core.domain.entity.CatcherItem; +import com.catcher.batch.core.domain.entity.Category; +import com.catcher.batch.datasource.CategoryRepository; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; +import lombok.RequiredArgsConstructor; import org.openqa.selenium.By; import org.openqa.selenium.WebDriver; import org.openqa.selenium.WebElement; @@ -11,10 +16,18 @@ import java.time.Duration; import java.util.ArrayList; import java.util.List; +import java.util.UUID; @Service -public class CatcherCrawlerService { +@RequiredArgsConstructor +public class ExhibitionService { + private final CatcherItemRepository catcherItemRepository; + private final CategoryRepository categoryRepository; + public List exhibitionCrawling() { + List catcherItems = new ArrayList<>(); + Category category = categoryRepository.findByName("exhibition") + .orElseGet(() -> categoryRepository.save(Category.create("exhibition"))); //TODO 드라이버 경로 상황에 맞게 설정, 그에 따른 크롬 및 gradle 의존성 버전도 알맞게 수정 System.setProperty("webdriver.chrome.driver", "C:/Users/dong/Downloads/chromedriver-win64/chromedriver-win64/chromedriver.exe"); @@ -68,6 +81,20 @@ public List exhibitionCrawling() { } //TODO driver.quit 에러 안나는 방법 서치 // driver.quit(); + + for (ObjectNode exhibitionInfo : exhibition_list) { + CatcherItem catcherItem = CatcherItem.builder() + .category(category) + .title(exhibitionInfo.get("eng_name").asText()) + .itemHashValue(UUID.randomUUID().toString()) + .description(exhibitionInfo.get("period").asText()) + .resourceUrl(exhibitionInfo.get("homepage").asText()) + .build(); + + catcherItems.add(catcherItem); + } + catcherItemRepository.saveAll(catcherItems); + return exhibition_list; } @@ -80,4 +107,4 @@ private static boolean isNextPageExists(WebDriver driver, int nextPage) { return false; } } -} +} \ No newline at end of file diff --git a/src/main/java/com/catcher/batch/resource/CrawlerController.java b/src/main/java/com/catcher/batch/resource/CrawlerController.java index fba5547..c9a8a3a 100644 --- a/src/main/java/com/catcher/batch/resource/CrawlerController.java +++ b/src/main/java/com/catcher/batch/resource/CrawlerController.java @@ -1,7 +1,6 @@ package com.catcher.batch.resource; -import com.catcher.batch.common.service.CatcherCrawlerService; -import com.catcher.batch.core.dto.FestivalApiResponse; +import com.catcher.batch.core.service.ExhibitionService; import com.fasterxml.jackson.databind.node.ObjectNode; import lombok.RequiredArgsConstructor; import org.springframework.http.ResponseEntity; @@ -16,11 +15,11 @@ @RequestMapping("/crawler") public class CrawlerController { - private final CatcherCrawlerService catcherCrawlerService; + private final ExhibitionService exhibitionService; @GetMapping("/exhibition") public ResponseEntity> getExhibitionData() { - List exhibitionCrawlingResponse = catcherCrawlerService.exhibitionCrawling(); + List exhibitionCrawlingResponse = exhibitionService.exhibitionCrawling(); return ResponseEntity.ok(exhibitionCrawlingResponse); } } From 1bbecf7f0de75af0a10350759c06e6deca02074d Mon Sep 17 00:00:00 2001 From: inyoung Date: Mon, 13 Nov 2023 02:32:01 +0900 Subject: [PATCH 22/37] =?UTF-8?q?refactor:=20stream=EC=9C=BC=EB=A1=9C=20?= =?UTF-8?q?=EB=A6=AC=ED=8C=A9=ED=86=A0=EB=A7=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../batch/core/service/CampingService.java | 40 ++++++++----------- 1 file changed, 16 insertions(+), 24 deletions(-) diff --git a/src/main/java/com/catcher/batch/core/service/CampingService.java b/src/main/java/com/catcher/batch/core/service/CampingService.java index 0c31507..d0e5b5a 100644 --- a/src/main/java/com/catcher/batch/core/service/CampingService.java +++ b/src/main/java/com/catcher/batch/core/service/CampingService.java @@ -12,8 +12,8 @@ import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; -import java.util.ArrayList; import java.util.List; +import java.util.stream.Collectors; @Service @RequiredArgsConstructor @@ -29,30 +29,23 @@ public void batch(CampingApiResponse campingApiResponse) { .orElseGet(() -> categoryRepository.save(Category.create(CATEGORY_NAME))); List campingItems = campingApiResponse.getItems().getItem(); - List catcherItems = new ArrayList<>(); - for (CampingApiResponse.CampingItem campingItem : campingItems) { - Location location = getLocationByDescription(campingItem.getProvince(), campingItem.getCity()); + List catcherItems = campingItems.stream() + .filter(campingItem -> !isDuplicateHashValue(hashString(campingItem.getKey()))) + .map(campingItem -> { + Location location = getLocationByDescription(campingItem.getProvince(), campingItem.getCity()); + String hashKey = hashString(campingItem.getKey()); - String hashKey = hashString(campingItem.getKey()); - - // 중복 해시값 체크 - if (isDuplicateHashValue(hashKey)) { - System.out.println("중복됨"); - continue; - } - - CatcherItem catcherItem = CatcherItem.builder() - .category(category) - .location(location) - .title(campingItem.getName()) - .description(campingItem.getDescription()) - .thumbnailUrl(campingItem.getThumbnailUrl()) - .itemHashValue(hashKey) - .build(); - - catcherItems.add(catcherItem); - } + return CatcherItem.builder() + .category(category) + .location(location) + .title(campingItem.getName()) + .description(campingItem.getDescription()) + .thumbnailUrl(campingItem.getThumbnailUrl()) + .itemHashValue(hashKey) + .build(); + }) + .collect(Collectors.toList()); catcherItemRepository.saveAll(catcherItems); } @@ -64,7 +57,6 @@ private Location getLocationByDescription(String province, String city) { .orElseThrow(); } - // TODO : 중복체크가 안됨 ㅜ.ㅜ private boolean isDuplicateHashValue(String hashKey) { return catcherItemRepository.findByItemHashValue(hashKey).isPresent(); } From 66463928f07584aee2d7e45ae889d12ed913f1c5 Mon Sep 17 00:00:00 2001 From: inyoung Date: Mon, 13 Nov 2023 02:36:21 +0900 Subject: [PATCH 23/37] =?UTF-8?q?feat:=20header=20=EC=84=A4=EC=A0=95=20?= =?UTF-8?q?=EA=B5=AC=ED=98=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../common/service/CatcherFeignService.java | 9 +++++ .../core/properties/HeaderPropertyProxy.java | 37 +++++++++++++++++++ .../batch/core/properties/HeaderSupport.java | 7 ++++ .../resource/external/ExternalFeign.java | 3 ++ 4 files changed, 56 insertions(+) create mode 100644 src/main/java/com/catcher/batch/core/properties/HeaderPropertyProxy.java create mode 100644 src/main/java/com/catcher/batch/core/properties/HeaderSupport.java diff --git a/src/main/java/com/catcher/batch/common/service/CatcherFeignService.java b/src/main/java/com/catcher/batch/common/service/CatcherFeignService.java index ab6b25f..d812a30 100644 --- a/src/main/java/com/catcher/batch/common/service/CatcherFeignService.java +++ b/src/main/java/com/catcher/batch/common/service/CatcherFeignService.java @@ -2,11 +2,13 @@ import com.catcher.batch.core.converter.CatcherConverter; import com.catcher.batch.core.properties.PropertyBase; +import com.catcher.batch.core.properties.HeaderSupport; import com.catcher.batch.resource.external.ExternalFeign; import jakarta.annotation.PostConstruct; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.context.ApplicationContext; +import org.springframework.http.HttpHeaders; import org.springframework.scheduling.annotation.Async; import org.springframework.stereotype.Service; @@ -40,6 +42,13 @@ public T parseService(Map params, Class requestType) { property.setParams(params); URI uri = property.getURI(); + // 헤더가 있는 경우 + HttpHeaders headers; + if (property instanceof HeaderSupport headerSupport) { + headers = headerSupport.addHeaders(); + return catcherConverter.parse(externalFeign.getInfoWithHeader(headers.getFirst("Authorization"), uri), requestType); + } + return catcherConverter.parse(externalFeign.getInfo(uri), requestType); } diff --git a/src/main/java/com/catcher/batch/core/properties/HeaderPropertyProxy.java b/src/main/java/com/catcher/batch/core/properties/HeaderPropertyProxy.java new file mode 100644 index 0000000..fc200b5 --- /dev/null +++ b/src/main/java/com/catcher/batch/core/properties/HeaderPropertyProxy.java @@ -0,0 +1,37 @@ +package com.catcher.batch.core.properties; + +import org.springframework.http.HttpHeaders; + +import java.net.URI; +import java.util.Map; + +public class HeaderPropertyProxy extends PropertyBase { + private final PropertyBase property; + + public HeaderPropertyProxy(PropertyBase property) { + super(property.getEndPoint()); + this.property = property; + } + + @Override + public boolean support(Class clazz) { + return property.support(clazz); + } + + @Override + public URI getURI() { + return property.getURI(); + } + + @Override + public void setParams(Map params) { + property.setParams(params); + } + + public HttpHeaders addHeaders() { + if (property instanceof HeaderSupport) { + return ((HeaderSupport) property).addHeaders(); + } + return new HttpHeaders(); + } +} diff --git a/src/main/java/com/catcher/batch/core/properties/HeaderSupport.java b/src/main/java/com/catcher/batch/core/properties/HeaderSupport.java new file mode 100644 index 0000000..9b6d3b6 --- /dev/null +++ b/src/main/java/com/catcher/batch/core/properties/HeaderSupport.java @@ -0,0 +1,7 @@ +package com.catcher.batch.core.properties; + +import org.springframework.http.HttpHeaders; + +public interface HeaderSupport { + HttpHeaders addHeaders(); +} diff --git a/src/main/java/com/catcher/batch/resource/external/ExternalFeign.java b/src/main/java/com/catcher/batch/resource/external/ExternalFeign.java index b79ac64..f48c163 100644 --- a/src/main/java/com/catcher/batch/resource/external/ExternalFeign.java +++ b/src/main/java/com/catcher/batch/resource/external/ExternalFeign.java @@ -2,6 +2,7 @@ import org.springframework.cloud.openfeign.FeignClient; import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.RequestHeader; import java.net.URI; @@ -10,4 +11,6 @@ public interface ExternalFeign { @GetMapping String getInfo(URI url); + @GetMapping + String getInfoWithHeader(@RequestHeader("Authorization") String authorizationHeader, URI url); } From aebfe891d2aeac1505331fafdd5000c0c1103a97 Mon Sep 17 00:00:00 2001 From: inyoung Date: Mon, 13 Nov 2023 02:37:52 +0900 Subject: [PATCH 24/37] =?UTF-8?q?feat:=20=EC=9D=8C=EC=8B=9D=EC=A0=90=20db?= =?UTF-8?q?=20=EC=A0=80=EC=9E=A5=20=EA=B5=AC=ED=98=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../core/converter/CatcherConverter.java | 3 + .../RegisterRestaurantDataCommand.java | 17 +++++ .../batch/core/dto/RestaurantApiResponse.java | 57 ++++++++++++++++ .../batch/core/service/RestaurantService.java | 67 +++++++++++++++++++ .../properties/RestaurantProperties.java | 57 ++++++++++++++++ .../batch/resource/RestaurantController.java | 38 +++++++++++ src/main/resources/application.properties | 6 ++ 7 files changed, 245 insertions(+) create mode 100644 src/main/java/com/catcher/batch/core/domain/command/RegisterRestaurantDataCommand.java create mode 100644 src/main/java/com/catcher/batch/core/dto/RestaurantApiResponse.java create mode 100644 src/main/java/com/catcher/batch/core/service/RestaurantService.java create mode 100644 src/main/java/com/catcher/batch/infrastructure/properties/RestaurantProperties.java create mode 100644 src/main/java/com/catcher/batch/resource/RestaurantController.java diff --git a/src/main/java/com/catcher/batch/core/converter/CatcherConverter.java b/src/main/java/com/catcher/batch/core/converter/CatcherConverter.java index 4b666c9..6129453 100644 --- a/src/main/java/com/catcher/batch/core/converter/CatcherConverter.java +++ b/src/main/java/com/catcher/batch/core/converter/CatcherConverter.java @@ -3,6 +3,7 @@ import com.catcher.batch.annotation.CatcherJson; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.commons.lang3.StringUtils; import org.json.JSONObject; import org.springframework.stereotype.Component; @@ -33,6 +34,8 @@ private String getPath(Class responseType) { private JSONObject getJsonObject(String json, String jsonPath) { if(jsonPath == null) { throw new IllegalStateException(); + } else if (StringUtils.isBlank(jsonPath)) { + return new JSONObject(json); } JSONObject jsonObject = new JSONObject(json); diff --git a/src/main/java/com/catcher/batch/core/domain/command/RegisterRestaurantDataCommand.java b/src/main/java/com/catcher/batch/core/domain/command/RegisterRestaurantDataCommand.java new file mode 100644 index 0000000..738cb2e --- /dev/null +++ b/src/main/java/com/catcher/batch/core/domain/command/RegisterRestaurantDataCommand.java @@ -0,0 +1,17 @@ +package com.catcher.batch.core.domain.command; + +import com.catcher.batch.core.dto.RestaurantApiResponse; +import com.catcher.batch.core.service.RestaurantService; +import lombok.RequiredArgsConstructor; + +@RequiredArgsConstructor +public class RegisterRestaurantDataCommand implements Command { + private final RestaurantService restaurantService; + private final RestaurantApiResponse restaurantApiResponse; + + @Override + public Void execute() { + restaurantService.batch(restaurantApiResponse); + return null; + } +} diff --git a/src/main/java/com/catcher/batch/core/dto/RestaurantApiResponse.java b/src/main/java/com/catcher/batch/core/dto/RestaurantApiResponse.java new file mode 100644 index 0000000..045d7b4 --- /dev/null +++ b/src/main/java/com/catcher/batch/core/dto/RestaurantApiResponse.java @@ -0,0 +1,57 @@ +package com.catcher.batch.core.dto; + +import com.catcher.batch.annotation.CatcherJson; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.Getter; + +import java.util.List; + +@Getter +@JsonIgnoreProperties(ignoreUnknown = true) +@CatcherJson(path = "") +public class RestaurantApiResponse { + + @JsonProperty("meta") + private Meta meta; + + @JsonProperty("documents") + private List items; + + @Getter + @JsonIgnoreProperties(ignoreUnknown = true) + public static class Meta { + + @JsonProperty("pageable_count") + private int pageableCount; + + @JsonProperty("total_count") + private int totalCount; + + @JsonProperty("is_end") + private boolean isEnd; + } + + @Getter + @JsonIgnoreProperties(ignoreUnknown = true) + public static class RestaurantItem { + + @JsonProperty("id") + private String key; + + @JsonProperty("place_name") + private String name; + + @JsonProperty("place_url") + private String resourceUrl; + + @JsonProperty("address_name") + private String address; + + @JsonProperty("x") + private String latitude; + + @JsonProperty("y") + private String longitude; + } +} diff --git a/src/main/java/com/catcher/batch/core/service/RestaurantService.java b/src/main/java/com/catcher/batch/core/service/RestaurantService.java new file mode 100644 index 0000000..0af0656 --- /dev/null +++ b/src/main/java/com/catcher/batch/core/service/RestaurantService.java @@ -0,0 +1,67 @@ +package com.catcher.batch.core.service; + +import com.catcher.batch.core.database.CatcherItemRepository; +import com.catcher.batch.core.database.LocationRepository; +import com.catcher.batch.core.domain.entity.CatcherItem; +import com.catcher.batch.core.domain.entity.Category; +import com.catcher.batch.core.domain.entity.Location; +import com.catcher.batch.core.dto.RestaurantApiResponse; +import com.catcher.batch.datasource.CategoryRepository; +import lombok.RequiredArgsConstructor; +import org.apache.commons.codec.digest.DigestUtils; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + +import java.util.List; +import java.util.stream.Collectors; + +@Service +@RequiredArgsConstructor +public class RestaurantService { + private final CatcherItemRepository catcherItemRepository; + private final CategoryRepository categoryRepository; + private final LocationRepository locationRepository; + public static final String CATEGORY_NAME = "restaurant"; + + @Transactional + public void batch(RestaurantApiResponse restaurantApiResponse) { + Category category = categoryRepository.findByName(CATEGORY_NAME) + .orElseGet(() -> categoryRepository.save(Category.create(CATEGORY_NAME))); + + List catcherItems = restaurantApiResponse.getItems().stream() + .filter(item -> !isDuplicateHashValue(hashString(item.getKey()))) + .map(item -> { + Location location = getLocation(item.getAddress()); + String hashKey = hashString(item.getKey()); + + return CatcherItem.builder() + .category(category) + .location(location) + .title(item.getName()) + .resourceUrl(item.getResourceUrl()) + .itemHashValue(hashKey) + .build(); + }) + .collect(Collectors.toList()); + + catcherItemRepository.saveAll(catcherItems); + } + + private Location getLocation(String address) { + String[] parts = address.split("\\s+"); + + String province = parts[0]; + String city = parts[1]; + + return locationRepository.findByDescription(province, city) + .orElseThrow(); + } + + private boolean isDuplicateHashValue(String hashKey) { + return catcherItemRepository.findByItemHashValue(hashKey).isPresent(); + } + + private String hashString(String input) { + return DigestUtils.sha256Hex(input); + } +} diff --git a/src/main/java/com/catcher/batch/infrastructure/properties/RestaurantProperties.java b/src/main/java/com/catcher/batch/infrastructure/properties/RestaurantProperties.java new file mode 100644 index 0000000..36ef559 --- /dev/null +++ b/src/main/java/com/catcher/batch/infrastructure/properties/RestaurantProperties.java @@ -0,0 +1,57 @@ +package com.catcher.batch.infrastructure.properties; + +import com.catcher.batch.core.dto.RestaurantApiResponse; +import com.catcher.batch.core.properties.HeaderSupport; +import com.catcher.batch.core.properties.PropertyBase; +import com.catcher.batch.infrastructure.utils.KmsUtils; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.http.HttpHeaders; +import org.springframework.http.MediaType; +import org.springframework.stereotype.Component; +import org.springframework.web.util.UriComponentsBuilder; + +import java.net.URI; +import java.util.Map; + +@Component +public class RestaurantProperties extends PropertyBase implements HeaderSupport { + + @Value("${restaurant.key}") + private String serviceKey; + + public RestaurantProperties(@Value("${restaurant.baseUrl}") String endPoint) { + super(endPoint); + } + + @Override + public boolean support(Class clazz) { + return clazz.isAssignableFrom(RestaurantApiResponse.class); + } + + @Override + public URI getURI() { + UriComponentsBuilder uriBuilder = UriComponentsBuilder + .fromUriString(this.getEndPoint()) + .queryParam("category_group_code", "FD6"); + + return this.addParams(uriBuilder) + .build().toUri(); + } + + private UriComponentsBuilder addParams(UriComponentsBuilder uriComponentsBuilder ) { + Map params = getParams(); + for (String key : params.keySet()) { + uriComponentsBuilder + .queryParam(key, params.get(key)); + } + return uriComponentsBuilder; + } + + @Override + public HttpHeaders addHeaders() { + HttpHeaders headers = new HttpHeaders(); + headers.set("Authorization", "KakaoAK " + KmsUtils.decrypt(serviceKey)); + headers.setContentType(MediaType.APPLICATION_JSON); + return headers; + } +} diff --git a/src/main/java/com/catcher/batch/resource/RestaurantController.java b/src/main/java/com/catcher/batch/resource/RestaurantController.java new file mode 100644 index 0000000..2bf5f36 --- /dev/null +++ b/src/main/java/com/catcher/batch/resource/RestaurantController.java @@ -0,0 +1,38 @@ +package com.catcher.batch.resource; + +import com.catcher.batch.common.response.CommonResponse; +import com.catcher.batch.common.service.CatcherFeignService; +import com.catcher.batch.core.domain.CommandExecutor; +import com.catcher.batch.core.domain.command.RegisterRestaurantDataCommand; +import com.catcher.batch.core.dto.RestaurantApiResponse; +import com.catcher.batch.core.service.RestaurantService; +import lombok.RequiredArgsConstructor; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; + +import java.util.HashMap; + +@RequiredArgsConstructor +@RestController +@RequestMapping("/restaurant") +public class RestaurantController { + private final CatcherFeignService catcherFeignService; + private final RestaurantService restaurantService; + private final CommandExecutor commandExecutor; + + @PostMapping("/batch") + public CommonResponse batchRestaurantData( + @RequestParam(defaultValue = "1") Integer page, + @RequestParam(defaultValue = "5") Integer count + ) { + HashMap params = new HashMap<>(); + params.put("page", page); + params.put("size", count); + RestaurantApiResponse restaurantApiResponse = catcherFeignService.parseService(params, RestaurantApiResponse.class); + + commandExecutor.run(new RegisterRestaurantDataCommand(restaurantService, restaurantApiResponse)); + return CommonResponse.success(201, null); + } +} diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index 77ec0fb..2b3d843 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -26,6 +26,8 @@ movie.key=AQICAHg2j0pl3GguoQNkjcreuBPYJqI3OPeLEOelKIkMSzn01AEdAQDVPqdOhpfNbxFhaX ## festival api festival.baseUrl=http://api.data.go.kr/openapi/tn_pubr_public_cltur_fstvl_api festival.key=AQICAHg2j0pl3GguoQNkjcreuBPYJqI3OPeLEOelKIkMSzn01AGEXII6Ufa8dIEGD6MQNyM9AAAAujCBtwYJKoZIhvcNAQcGoIGpMIGmAgEAMIGgBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDEphbmyNQTiCTHtF/gIBEIBz8Fj7Ij/xk+O5hXWQfMOsIfc6uAYrvF2Xew+p0qLkX3DO0Plzjt4EMzaUP504RIMr7s+Yhx2y2Lq5SZ9I/cZ4swMYIOwj8FXDLeFy/k3dfwFdnBRh1kmvIDiMSg5kj4kgk05nzHyJ5KloYXXngp/ZQzsdpA== + +## camping api camping.key=AQICAHg2j0pl3GguoQNkjcreuBPYJqI3OPeLEOelKIkMSzn01AF5N6HoIJmz3ef5mw7h81sOAAAAujCBtwYJKoZIhvcNAQcGoIGpMIGmAgEAMIGgBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDDvX80CMep9JqAHwVgIBEIBzD41zp/0NHhhvJ4nFvikmteZp501dy09Q/RjJ631ggAcTPaskdGKn6+qwvQVLw3vWesWOSycm3NiMeatVHUwSBZ4AKFGxfhHIsinf1Hyl2RdKjbbeYLYTtTiJY8jh4kZrXUPn6zQ4/vVr75WcIi9eny6lhw== camping.baseUrl=http://apis.data.go.kr/B551011/GoCamping/basedList @@ -33,6 +35,10 @@ camping.baseUrl=http://apis.data.go.kr/B551011/GoCamping/basedList movie.tmdb.baseUrl=https://api.themoviedb.org/3/discover/movie movie.tmdb.key=AQICAHg2j0pl3GguoQNkjcreuBPYJqI3OPeLEOelKIkMSzn01AELxjw41yAPSTnmp5zrZNWQAAAAfjB8BgkqhkiG9w0BBwagbzBtAgEAMGgGCSqGSIb3DQEHATAeBglghkgBZQMEAS4wEQQMHCIOEwhqioIojbKiAgEQgDsmsdMzlgQamoyDnGglRz5A2BNtMvn/4BIVT75wmxUz82NvOoh+Tno236+8lcoAoYdWzfgFVk7cAMNZdw== +## restaurant api +restaurant.key=AQICAHgxTvGwZVD/2MLMvR9/01Wy8IeL4nGHGGgc5XMYIhkQ2gFwIra357LjWgWTfXdGHOTMAAAAfjB8BgkqhkiG9w0BBwagbzBtAgEAMGgGCSqGSIb3DQEHATAeBglghkgBZQMEAS4wEQQM0t0Y5XX4gbI1BsAnAgEQgDtbOGiG1iS5qgxCIfnvMf/pEad1kQAOJrPoc39lLxiTk/Vw+bw/DqE/Sdmj/ibi3fakUgdXhdK111insQ== +restaurant.baseUrl=https://dapi.kakao.com/v2/local/search/category + #update the schema with the given values. spring.jpa.hibernate.ddl-auto=update #To beautify or pretty print the SQL From 16f6768fad961512117295fcbaf59af21b8705c3 Mon Sep 17 00:00:00 2001 From: inyoung Date: Mon, 13 Nov 2023 02:38:37 +0900 Subject: [PATCH 25/37] =?UTF-8?q?chore:=20@Where=20=EC=A3=BC=EC=84=9D=20?= =?UTF-8?q?=EC=B2=98=EB=A6=AC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../java/com/catcher/batch/core/domain/entity/CatcherItem.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/catcher/batch/core/domain/entity/CatcherItem.java b/src/main/java/com/catcher/batch/core/domain/entity/CatcherItem.java index 0172fb4..d768209 100644 --- a/src/main/java/com/catcher/batch/core/domain/entity/CatcherItem.java +++ b/src/main/java/com/catcher/batch/core/domain/entity/CatcherItem.java @@ -11,7 +11,7 @@ @Getter @AllArgsConstructor(access = AccessLevel.PROTECTED) @NoArgsConstructor(access = AccessLevel.PROTECTED) -@Where(clause = "delete_at = 'N'") +//@Where(clause = "delete_at = 'N'") @Table(name = "catcher_item") public class CatcherItem extends BaseTimeEntity { @Id From 70b7e9530fd72cfddcd0c833501aff1dee0c0400 Mon Sep 17 00:00:00 2001 From: inyoung Date: Mon, 13 Nov 2023 03:06:26 +0900 Subject: [PATCH 26/37] =?UTF-8?q?test:=20=EC=9D=8C=EC=8B=9D=EC=A0=90=20?= =?UTF-8?q?=EC=A0=80=EC=9E=A5=20=EC=84=B1=EA=B3=B5=20=ED=85=8C=EC=8A=A4?= =?UTF-8?q?=ED=8A=B8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../core/service/RestaurantServiceTest.java | 88 +++++++++++++++++++ 1 file changed, 88 insertions(+) create mode 100644 src/test/java/com/catcher/batch/core/service/RestaurantServiceTest.java diff --git a/src/test/java/com/catcher/batch/core/service/RestaurantServiceTest.java b/src/test/java/com/catcher/batch/core/service/RestaurantServiceTest.java new file mode 100644 index 0000000..2ffe5ea --- /dev/null +++ b/src/test/java/com/catcher/batch/core/service/RestaurantServiceTest.java @@ -0,0 +1,88 @@ +package com.catcher.batch.core.service; + +import com.catcher.batch.core.database.CatcherItemRepository; +import com.catcher.batch.core.database.LocationRepository; +import com.catcher.batch.core.domain.entity.Category; +import com.catcher.batch.core.domain.entity.Location; +import com.catcher.batch.core.dto.RestaurantApiResponse; +import com.catcher.batch.datasource.CategoryRepository; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.jupiter.MockitoExtension; + +import java.util.Arrays; +import java.util.Collections; +import java.util.Optional; + +@ExtendWith(MockitoExtension.class) +class RestaurantServiceTest { + + @Mock + private CatcherItemRepository catcherItemRepository; + + @Mock + private LocationRepository locationRepository; + + @Mock + private CategoryRepository categoryRepository; + + @InjectMocks + private RestaurantService restaurantService; + + @DisplayName("SUCCESS : 음식점 db 저장 성공 테스트") + @Test + void batchTest_SuccessfulSave() { + // Given + RestaurantApiResponse.RestaurantItem restaurantItem = createItem(); + + RestaurantApiResponse restaurantApiResponse = RestaurantApiResponse.builder() + .items(Collections.singletonList(restaurantItem)) + .build(); + + Category category = Category.create("restaurant"); + Mockito.when(categoryRepository.findByName("restaurant")).thenReturn(Optional.of(category)); + Mockito.when(locationRepository.findByDescription("서울", "관악구")).thenReturn(Optional.of(Location.create("1162000000", "37.4783683761333", "126.951561853868", "서울 관악구"))); + Mockito.when(catcherItemRepository.findByItemHashValue(Mockito.anyString())).thenReturn(Optional.empty()); + + // When + restaurantService.batch(restaurantApiResponse); + + // Then + Mockito.verify(catcherItemRepository, Mockito.times(1)).saveAll(Mockito.anyList()); + } + + @DisplayName("SUCCESS: 음식점 db 저장 성공 테스트 - 중복 객체 확인") + @Test + void batchTest_SuccessfulSaveWithDuplicate() { + // Given + RestaurantApiResponse.RestaurantItem restaurantItem = createItem(); + + RestaurantApiResponse restaurantApiResponse = RestaurantApiResponse.builder() + .items(Arrays.asList(restaurantItem, restaurantItem)) + .build(); + + Category category = Category.create("restaurant"); + Mockito.when(categoryRepository.findByName("restaurant")).thenReturn(Optional.of(category)); + Mockito.when(locationRepository.findByDescription("서울", "관악구")).thenReturn(Optional.of(Location.create("1162000000", "37.4783683761333", "126.951561853868", "서울 관악구"))); + Mockito.when(catcherItemRepository.findByItemHashValue(Mockito.anyString())).thenReturn(Optional.empty()); + + // When + restaurantService.batch(restaurantApiResponse); + + // Then + Mockito.verify(catcherItemRepository, Mockito.times(1)).saveAll(Mockito.anyList()); + } + + private RestaurantApiResponse.RestaurantItem createItem() { + return RestaurantApiResponse.RestaurantItem.builder() + .key("key") + .address("서울 관악구") + .name("맛집") + .resourceUrl("url") + .build(); + } +} \ No newline at end of file From cb43ed16bdd9dbee50b727399f4209255e2cf9fb Mon Sep 17 00:00:00 2001 From: inyoung Date: Mon, 13 Nov 2023 18:36:31 +0900 Subject: [PATCH 27/37] =?UTF-8?q?refactor:=20hashKey=20=EC=A4=91=EB=B3=B5?= =?UTF-8?q?=20=EA=B2=80=EC=82=AC=20=EB=A1=9C=EC=A7=81=20=EB=A6=AC=ED=8C=A9?= =?UTF-8?q?=ED=86=A0=EB=A7=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../batch/common/utils/HashCodeGenerator.java | 9 +++++++++ .../core/database/CatcherItemRepository.java | 5 +++++ .../batch/core/service/CampingService.java | 19 ++++++++----------- .../batch/core/service/RestaurantService.java | 19 ++++++++----------- .../datasource/CatcherItemJpaRepository.java | 4 ++++ .../datasource/CatcherItemRepositoryImpl.java | 11 +++++++++++ 6 files changed, 45 insertions(+), 22 deletions(-) create mode 100644 src/main/java/com/catcher/batch/common/utils/HashCodeGenerator.java diff --git a/src/main/java/com/catcher/batch/common/utils/HashCodeGenerator.java b/src/main/java/com/catcher/batch/common/utils/HashCodeGenerator.java new file mode 100644 index 0000000..491ac40 --- /dev/null +++ b/src/main/java/com/catcher/batch/common/utils/HashCodeGenerator.java @@ -0,0 +1,9 @@ +package com.catcher.batch.common.utils; + +import org.apache.commons.codec.digest.DigestUtils; + +public class HashCodeGenerator { + public static String hashString(String category, String input) { + return DigestUtils.sha256Hex(category + "-" + input); + } +} diff --git a/src/main/java/com/catcher/batch/core/database/CatcherItemRepository.java b/src/main/java/com/catcher/batch/core/database/CatcherItemRepository.java index 21b69a8..772858a 100644 --- a/src/main/java/com/catcher/batch/core/database/CatcherItemRepository.java +++ b/src/main/java/com/catcher/batch/core/database/CatcherItemRepository.java @@ -1,6 +1,7 @@ package com.catcher.batch.core.database; import com.catcher.batch.core.domain.entity.CatcherItem; +import com.catcher.batch.core.domain.entity.Category; import java.util.List; import java.util.Optional; @@ -8,5 +9,9 @@ public interface CatcherItemRepository { void saveAll(List catcherItems); + void save(CatcherItem catcherItem); + Optional findByItemHashValue(String hashKey); + + List findByCategory(Category category); } diff --git a/src/main/java/com/catcher/batch/core/service/CampingService.java b/src/main/java/com/catcher/batch/core/service/CampingService.java index d0e5b5a..c041342 100644 --- a/src/main/java/com/catcher/batch/core/service/CampingService.java +++ b/src/main/java/com/catcher/batch/core/service/CampingService.java @@ -8,13 +8,15 @@ import com.catcher.batch.core.dto.CampingApiResponse; import com.catcher.batch.datasource.CategoryRepository; import lombok.RequiredArgsConstructor; -import org.apache.commons.codec.digest.DigestUtils; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; import java.util.List; +import java.util.Map; import java.util.stream.Collectors; +import static com.catcher.batch.common.utils.HashCodeGenerator.hashString; + @Service @RequiredArgsConstructor public class CampingService { @@ -28,13 +30,16 @@ public void batch(CampingApiResponse campingApiResponse) { Category category = categoryRepository.findByName(CATEGORY_NAME) .orElseGet(() -> categoryRepository.save(Category.create(CATEGORY_NAME))); + Map itemMap = catcherItemRepository.findByCategory(category).stream() + .collect(Collectors.toMap(CatcherItem::getItemHashValue, CatcherItem::getTitle)); + List campingItems = campingApiResponse.getItems().getItem(); List catcherItems = campingItems.stream() - .filter(campingItem -> !isDuplicateHashValue(hashString(campingItem.getKey()))) + .filter(campingItem -> !itemMap.containsKey(hashString(CATEGORY_NAME, campingItem.getKey()))) .map(campingItem -> { Location location = getLocationByDescription(campingItem.getProvince(), campingItem.getCity()); - String hashKey = hashString(campingItem.getKey()); + String hashKey = hashString(CATEGORY_NAME, campingItem.getKey()); return CatcherItem.builder() .category(category) @@ -56,12 +61,4 @@ private Location getLocationByDescription(String province, String city) { return locationRepository.findByDescription(withoutDo, city) .orElseThrow(); } - - private boolean isDuplicateHashValue(String hashKey) { - return catcherItemRepository.findByItemHashValue(hashKey).isPresent(); - } - - private String hashString(String input) { - return DigestUtils.sha256Hex(input); - } } diff --git a/src/main/java/com/catcher/batch/core/service/RestaurantService.java b/src/main/java/com/catcher/batch/core/service/RestaurantService.java index 0af0656..1f2d463 100644 --- a/src/main/java/com/catcher/batch/core/service/RestaurantService.java +++ b/src/main/java/com/catcher/batch/core/service/RestaurantService.java @@ -8,13 +8,15 @@ import com.catcher.batch.core.dto.RestaurantApiResponse; import com.catcher.batch.datasource.CategoryRepository; import lombok.RequiredArgsConstructor; -import org.apache.commons.codec.digest.DigestUtils; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; import java.util.List; +import java.util.Map; import java.util.stream.Collectors; +import static com.catcher.batch.common.utils.HashCodeGenerator.hashString; + @Service @RequiredArgsConstructor public class RestaurantService { @@ -28,11 +30,14 @@ public void batch(RestaurantApiResponse restaurantApiResponse) { Category category = categoryRepository.findByName(CATEGORY_NAME) .orElseGet(() -> categoryRepository.save(Category.create(CATEGORY_NAME))); + Map itemMap = catcherItemRepository.findByCategory(category).stream() + .collect(Collectors.toMap(CatcherItem::getItemHashValue, CatcherItem::getTitle)); + List catcherItems = restaurantApiResponse.getItems().stream() - .filter(item -> !isDuplicateHashValue(hashString(item.getKey()))) + .filter(item -> !itemMap.containsKey(hashString(CATEGORY_NAME, item.getKey()))) .map(item -> { Location location = getLocation(item.getAddress()); - String hashKey = hashString(item.getKey()); + String hashKey = hashString(CATEGORY_NAME, item.getKey()); return CatcherItem.builder() .category(category) @@ -56,12 +61,4 @@ private Location getLocation(String address) { return locationRepository.findByDescription(province, city) .orElseThrow(); } - - private boolean isDuplicateHashValue(String hashKey) { - return catcherItemRepository.findByItemHashValue(hashKey).isPresent(); - } - - private String hashString(String input) { - return DigestUtils.sha256Hex(input); - } } diff --git a/src/main/java/com/catcher/batch/datasource/CatcherItemJpaRepository.java b/src/main/java/com/catcher/batch/datasource/CatcherItemJpaRepository.java index 283f810..51d3bfe 100644 --- a/src/main/java/com/catcher/batch/datasource/CatcherItemJpaRepository.java +++ b/src/main/java/com/catcher/batch/datasource/CatcherItemJpaRepository.java @@ -1,10 +1,14 @@ package com.catcher.batch.datasource; import com.catcher.batch.core.domain.entity.CatcherItem; +import com.catcher.batch.core.domain.entity.Category; import org.springframework.data.jpa.repository.JpaRepository; +import java.util.List; import java.util.Optional; public interface CatcherItemJpaRepository extends JpaRepository { Optional findByItemHashValue(String itemHashValue); + + List findByCategory(Category category); } diff --git a/src/main/java/com/catcher/batch/datasource/CatcherItemRepositoryImpl.java b/src/main/java/com/catcher/batch/datasource/CatcherItemRepositoryImpl.java index 13e7f1c..ff48699 100644 --- a/src/main/java/com/catcher/batch/datasource/CatcherItemRepositoryImpl.java +++ b/src/main/java/com/catcher/batch/datasource/CatcherItemRepositoryImpl.java @@ -2,6 +2,7 @@ import com.catcher.batch.core.database.CatcherItemRepository; import com.catcher.batch.core.domain.entity.CatcherItem; +import com.catcher.batch.core.domain.entity.Category; import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Repository; @@ -18,8 +19,18 @@ public void saveAll(List catcherItems) { catcherItemJpaRepository.saveAll(catcherItems); } + @Override + public void save(CatcherItem catcherItem) { + catcherItemJpaRepository.save(catcherItem); + } + @Override public Optional findByItemHashValue(String hashKey) { return catcherItemJpaRepository.findByItemHashValue(hashKey); } + + @Override + public List findByCategory(Category category) { + return catcherItemJpaRepository.findByCategory(category); + } } From b5900d4f906bbf73efea84672d9e2dc773186474 Mon Sep 17 00:00:00 2001 From: inyoung Date: Mon, 13 Nov 2023 18:50:34 +0900 Subject: [PATCH 28/37] =?UTF-8?q?chore:=20delete=20=EC=BB=AC=EB=9F=BC?= =?UTF-8?q?=EB=AA=85=20=EC=88=98=EC=A0=95,=20@Where=20=EC=88=98=EC=A0=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/catcher/batch/core/domain/entity/CatcherItem.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/catcher/batch/core/domain/entity/CatcherItem.java b/src/main/java/com/catcher/batch/core/domain/entity/CatcherItem.java index d768209..a10abcf 100644 --- a/src/main/java/com/catcher/batch/core/domain/entity/CatcherItem.java +++ b/src/main/java/com/catcher/batch/core/domain/entity/CatcherItem.java @@ -11,7 +11,7 @@ @Getter @AllArgsConstructor(access = AccessLevel.PROTECTED) @NoArgsConstructor(access = AccessLevel.PROTECTED) -//@Where(clause = "delete_at = 'N'") +@Where(clause = "deleted_at IS NULL") @Table(name = "catcher_item") public class CatcherItem extends BaseTimeEntity { @Id @@ -44,6 +44,6 @@ public class CatcherItem extends BaseTimeEntity { @Column(name = "end_at") private ZonedDateTime endAt; - @Column(name = "delete_at") + @Column(name = "deleted_at") private ZonedDateTime deletedAt; } From 325eddbb17ec9100cfe36376b6a05f3e038d7f67 Mon Sep 17 00:00:00 2001 From: inyoung Date: Mon, 13 Nov 2023 19:53:54 +0900 Subject: [PATCH 29/37] =?UTF-8?q?test:=20=EC=9D=8C=EC=8B=9D=EC=A0=90=20?= =?UTF-8?q?=EB=8B=A8=EC=9C=84=20=ED=85=8C=EC=8A=A4=ED=8A=B8=20=EC=BD=94?= =?UTF-8?q?=EB=93=9C=20=EB=A6=AC=ED=8C=A9=ED=86=A0=EB=A7=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../core/service/RestaurantServiceTest.java | 60 ++++++++++--------- 1 file changed, 32 insertions(+), 28 deletions(-) diff --git a/src/test/java/com/catcher/batch/core/service/RestaurantServiceTest.java b/src/test/java/com/catcher/batch/core/service/RestaurantServiceTest.java index 2ffe5ea..cda6360 100644 --- a/src/test/java/com/catcher/batch/core/service/RestaurantServiceTest.java +++ b/src/test/java/com/catcher/batch/core/service/RestaurantServiceTest.java @@ -6,6 +6,7 @@ import com.catcher.batch.core.domain.entity.Location; import com.catcher.batch.core.dto.RestaurantApiResponse; import com.catcher.batch.datasource.CategoryRepository; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -13,11 +14,17 @@ import org.mockito.Mock; import org.mockito.Mockito; import org.mockito.junit.jupiter.MockitoExtension; +import org.mockito.junit.jupiter.MockitoSettings; +import org.mockito.quality.Strictness; import java.util.Arrays; -import java.util.Collections; +import java.util.List; import java.util.Optional; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +@MockitoSettings(strictness = Strictness.LENIENT) @ExtendWith(MockitoExtension.class) class RestaurantServiceTest { @@ -33,20 +40,21 @@ class RestaurantServiceTest { @InjectMocks private RestaurantService restaurantService; + private RestaurantApiResponse restaurantApiResponse; + private RestaurantApiResponse.RestaurantItem restaurantItem; + + @BeforeEach + void beforeEach() { + restaurantApiResponse = mock(RestaurantApiResponse.class); + restaurantItem = Mockito.mock(RestaurantApiResponse.RestaurantItem.class); + } + @DisplayName("SUCCESS : 음식점 db 저장 성공 테스트") @Test void batchTest_SuccessfulSave() { // Given - RestaurantApiResponse.RestaurantItem restaurantItem = createItem(); - - RestaurantApiResponse restaurantApiResponse = RestaurantApiResponse.builder() - .items(Collections.singletonList(restaurantItem)) - .build(); - - Category category = Category.create("restaurant"); - Mockito.when(categoryRepository.findByName("restaurant")).thenReturn(Optional.of(category)); - Mockito.when(locationRepository.findByDescription("서울", "관악구")).thenReturn(Optional.of(Location.create("1162000000", "37.4783683761333", "126.951561853868", "서울 관악구"))); - Mockito.when(catcherItemRepository.findByItemHashValue(Mockito.anyString())).thenReturn(Optional.empty()); + setUpRestaurantItem(); + when(restaurantApiResponse.getItems()).thenReturn(List.of(restaurantItem)); // When restaurantService.batch(restaurantApiResponse); @@ -59,16 +67,8 @@ void batchTest_SuccessfulSave() { @Test void batchTest_SuccessfulSaveWithDuplicate() { // Given - RestaurantApiResponse.RestaurantItem restaurantItem = createItem(); - - RestaurantApiResponse restaurantApiResponse = RestaurantApiResponse.builder() - .items(Arrays.asList(restaurantItem, restaurantItem)) - .build(); - - Category category = Category.create("restaurant"); - Mockito.when(categoryRepository.findByName("restaurant")).thenReturn(Optional.of(category)); - Mockito.when(locationRepository.findByDescription("서울", "관악구")).thenReturn(Optional.of(Location.create("1162000000", "37.4783683761333", "126.951561853868", "서울 관악구"))); - Mockito.when(catcherItemRepository.findByItemHashValue(Mockito.anyString())).thenReturn(Optional.empty()); + setUpRestaurantItem(); + when(restaurantApiResponse.getItems()).thenReturn(Arrays.asList(restaurantItem, restaurantItem)); // When restaurantService.batch(restaurantApiResponse); @@ -77,12 +77,16 @@ void batchTest_SuccessfulSaveWithDuplicate() { Mockito.verify(catcherItemRepository, Mockito.times(1)).saveAll(Mockito.anyList()); } - private RestaurantApiResponse.RestaurantItem createItem() { - return RestaurantApiResponse.RestaurantItem.builder() - .key("key") - .address("서울 관악구") - .name("맛집") - .resourceUrl("url") - .build(); + private void setUpRestaurantItem() { + when(restaurantItem.getKey()).thenReturn("key"); + when(restaurantItem.getName()).thenReturn("맛집"); + when(restaurantItem.getResourceUrl()).thenReturn("url"); + when(restaurantItem.getAddress()).thenReturn("서울 관악구"); + when(restaurantItem.getLatitude()).thenReturn("37.4783683761333"); + when(restaurantItem.getLongitude()).thenReturn("126.951561853868"); + + when(restaurantApiResponse.getItems()).thenReturn(List.of(restaurantItem)); + Mockito.when(categoryRepository.findByName("restaurant")).thenReturn(Optional.of(Category.create("restaurant"))); + Mockito.when(locationRepository.findByDescription("서울", "관악구")).thenReturn(Optional.of(Location.create("1162000000", "37.4783683761333", "126.951561853868", "서울 관악구"))); } } \ No newline at end of file From 0ebd09e5ac0f50f56ddc955b01ec0874f150056d Mon Sep 17 00:00:00 2001 From: inyoung Date: Tue, 14 Nov 2023 19:37:35 +0900 Subject: [PATCH 30/37] =?UTF-8?q?refactor:=20category=20repository=20?= =?UTF-8?q?=EA=B5=AC=EC=A1=B0=20=EB=B3=80=EA=B2=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../core/database/CategoryRepository.java | 11 +++++++++ .../batch/core/service/CampingService.java | 2 +- .../batch/core/service/RestaurantService.java | 2 +- ...sitory.java => CategoryJpaRepository.java} | 2 +- .../datasource/CategoryRepositoryImpl.java | 24 +++++++++++++++++++ 5 files changed, 38 insertions(+), 3 deletions(-) create mode 100644 src/main/java/com/catcher/batch/core/database/CategoryRepository.java rename src/main/java/com/catcher/batch/datasource/{CategoryRepository.java => CategoryJpaRepository.java} (74%) create mode 100644 src/main/java/com/catcher/batch/datasource/CategoryRepositoryImpl.java diff --git a/src/main/java/com/catcher/batch/core/database/CategoryRepository.java b/src/main/java/com/catcher/batch/core/database/CategoryRepository.java new file mode 100644 index 0000000..015bddd --- /dev/null +++ b/src/main/java/com/catcher/batch/core/database/CategoryRepository.java @@ -0,0 +1,11 @@ +package com.catcher.batch.core.database; + +import com.catcher.batch.core.domain.entity.Category; + +import java.util.Optional; + +public interface CategoryRepository { + Optional findByName(String name); + + Category save(Category category); +} diff --git a/src/main/java/com/catcher/batch/core/service/CampingService.java b/src/main/java/com/catcher/batch/core/service/CampingService.java index c041342..aade016 100644 --- a/src/main/java/com/catcher/batch/core/service/CampingService.java +++ b/src/main/java/com/catcher/batch/core/service/CampingService.java @@ -1,12 +1,12 @@ package com.catcher.batch.core.service; import com.catcher.batch.core.database.CatcherItemRepository; +import com.catcher.batch.core.database.CategoryRepository; import com.catcher.batch.core.database.LocationRepository; import com.catcher.batch.core.domain.entity.CatcherItem; import com.catcher.batch.core.domain.entity.Category; import com.catcher.batch.core.domain.entity.Location; import com.catcher.batch.core.dto.CampingApiResponse; -import com.catcher.batch.datasource.CategoryRepository; import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; diff --git a/src/main/java/com/catcher/batch/core/service/RestaurantService.java b/src/main/java/com/catcher/batch/core/service/RestaurantService.java index 1f2d463..cd530ef 100644 --- a/src/main/java/com/catcher/batch/core/service/RestaurantService.java +++ b/src/main/java/com/catcher/batch/core/service/RestaurantService.java @@ -1,12 +1,12 @@ package com.catcher.batch.core.service; import com.catcher.batch.core.database.CatcherItemRepository; +import com.catcher.batch.core.database.CategoryRepository; import com.catcher.batch.core.database.LocationRepository; import com.catcher.batch.core.domain.entity.CatcherItem; import com.catcher.batch.core.domain.entity.Category; import com.catcher.batch.core.domain.entity.Location; import com.catcher.batch.core.dto.RestaurantApiResponse; -import com.catcher.batch.datasource.CategoryRepository; import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; diff --git a/src/main/java/com/catcher/batch/datasource/CategoryRepository.java b/src/main/java/com/catcher/batch/datasource/CategoryJpaRepository.java similarity index 74% rename from src/main/java/com/catcher/batch/datasource/CategoryRepository.java rename to src/main/java/com/catcher/batch/datasource/CategoryJpaRepository.java index 9a1b7fa..c37e7e8 100644 --- a/src/main/java/com/catcher/batch/datasource/CategoryRepository.java +++ b/src/main/java/com/catcher/batch/datasource/CategoryJpaRepository.java @@ -5,6 +5,6 @@ import java.util.Optional; -public interface CategoryRepository extends JpaRepository { +public interface CategoryJpaRepository extends JpaRepository { Optional findByName(String name); } diff --git a/src/main/java/com/catcher/batch/datasource/CategoryRepositoryImpl.java b/src/main/java/com/catcher/batch/datasource/CategoryRepositoryImpl.java new file mode 100644 index 0000000..32791ab --- /dev/null +++ b/src/main/java/com/catcher/batch/datasource/CategoryRepositoryImpl.java @@ -0,0 +1,24 @@ +package com.catcher.batch.datasource; + +import com.catcher.batch.core.database.CategoryRepository; +import com.catcher.batch.core.domain.entity.Category; +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Repository; + +import java.util.Optional; + +@Repository +@RequiredArgsConstructor +public class CategoryRepositoryImpl implements CategoryRepository { + private final CategoryJpaRepository categoryJpaRepository; + + @Override + public Optional findByName(String name) { + return categoryJpaRepository.findByName(name); + } + + @Override + public Category save(Category category) { + return categoryJpaRepository.save(category); + } +} From 9ae0da7f7ed168a918b817e617157c63cc4e3ff5 Mon Sep 17 00:00:00 2001 From: inyoung Date: Tue, 14 Nov 2023 21:14:20 +0900 Subject: [PATCH 31/37] =?UTF-8?q?refactor:=20saveAll()=20=EB=8C=80?= =?UTF-8?q?=EC=8B=A0=20save()=20=EC=82=AC=EC=9A=A9=ED=95=98=EB=8F=84?= =?UTF-8?q?=EB=A1=9D=20=EB=B3=80=EA=B2=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../batch/core/service/RestaurantService.java | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/src/main/java/com/catcher/batch/core/service/RestaurantService.java b/src/main/java/com/catcher/batch/core/service/RestaurantService.java index cd530ef..a550c5c 100644 --- a/src/main/java/com/catcher/batch/core/service/RestaurantService.java +++ b/src/main/java/com/catcher/batch/core/service/RestaurantService.java @@ -11,7 +11,6 @@ import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; -import java.util.List; import java.util.Map; import java.util.stream.Collectors; @@ -33,23 +32,22 @@ public void batch(RestaurantApiResponse restaurantApiResponse) { Map itemMap = catcherItemRepository.findByCategory(category).stream() .collect(Collectors.toMap(CatcherItem::getItemHashValue, CatcherItem::getTitle)); - List catcherItems = restaurantApiResponse.getItems().stream() + restaurantApiResponse.getItems().stream() .filter(item -> !itemMap.containsKey(hashString(CATEGORY_NAME, item.getKey()))) - .map(item -> { + .forEach(item -> { Location location = getLocation(item.getAddress()); String hashKey = hashString(CATEGORY_NAME, item.getKey()); - return CatcherItem.builder() + CatcherItem catcherItem = CatcherItem.builder() .category(category) .location(location) .title(item.getName()) .resourceUrl(item.getResourceUrl()) .itemHashValue(hashKey) .build(); - }) - .collect(Collectors.toList()); - catcherItemRepository.saveAll(catcherItems); + catcherItemRepository.save(catcherItem); + }); } private Location getLocation(String address) { From d49d1b4a62e73b11d2ea1d78f54f68551b5921f2 Mon Sep 17 00:00:00 2001 From: inyoung Date: Tue, 14 Nov 2023 21:15:12 +0900 Subject: [PATCH 32/37] =?UTF-8?q?test:=20=EC=9D=8C=EC=8B=9D=EC=A0=90=20?= =?UTF-8?q?=EB=8B=A8=EC=9C=84=20=ED=85=8C=EC=8A=A4=ED=8A=B8=20-=20?= =?UTF-8?q?=EC=A4=91=EB=B3=B5=20=EA=B0=9D=EC=B2=B4=20=EA=B2=80=EC=82=AC=20?= =?UTF-8?q?=EC=BD=94=EB=93=9C=20=EB=A6=AC=ED=8C=A9=ED=86=A0=EB=A7=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../core/service/RestaurantServiceTest.java | 37 +++++++++++++------ 1 file changed, 26 insertions(+), 11 deletions(-) diff --git a/src/test/java/com/catcher/batch/core/service/RestaurantServiceTest.java b/src/test/java/com/catcher/batch/core/service/RestaurantServiceTest.java index cda6360..ae5b28e 100644 --- a/src/test/java/com/catcher/batch/core/service/RestaurantServiceTest.java +++ b/src/test/java/com/catcher/batch/core/service/RestaurantServiceTest.java @@ -1,11 +1,13 @@ package com.catcher.batch.core.service; import com.catcher.batch.core.database.CatcherItemRepository; +import com.catcher.batch.core.database.CategoryRepository; import com.catcher.batch.core.database.LocationRepository; +import com.catcher.batch.core.domain.entity.CatcherItem; import com.catcher.batch.core.domain.entity.Category; import com.catcher.batch.core.domain.entity.Location; import com.catcher.batch.core.dto.RestaurantApiResponse; -import com.catcher.batch.datasource.CategoryRepository; +import org.apache.commons.codec.digest.DigestUtils; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; @@ -17,11 +19,11 @@ import org.mockito.junit.jupiter.MockitoSettings; import org.mockito.quality.Strictness; -import java.util.Arrays; +import java.util.Collections; import java.util.List; import java.util.Optional; -import static org.mockito.Mockito.mock; +import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.when; @MockitoSettings(strictness = Strictness.LENIENT) @@ -42,11 +44,17 @@ class RestaurantServiceTest { private RestaurantApiResponse restaurantApiResponse; private RestaurantApiResponse.RestaurantItem restaurantItem; + private CatcherItem catcherItem; + private Location location; + private Category category; @BeforeEach void beforeEach() { - restaurantApiResponse = mock(RestaurantApiResponse.class); + restaurantApiResponse = Mockito.mock(RestaurantApiResponse.class); restaurantItem = Mockito.mock(RestaurantApiResponse.RestaurantItem.class); + catcherItem = Mockito.mock(CatcherItem.class); + location = Mockito.mock(Location.class); + category = Mockito.mock((Category.class)); } @DisplayName("SUCCESS : 음식점 db 저장 성공 테스트") @@ -54,27 +62,34 @@ void beforeEach() { void batchTest_SuccessfulSave() { // Given setUpRestaurantItem(); - when(restaurantApiResponse.getItems()).thenReturn(List.of(restaurantItem)); // When restaurantService.batch(restaurantApiResponse); // Then - Mockito.verify(catcherItemRepository, Mockito.times(1)).saveAll(Mockito.anyList()); + Mockito.verify(catcherItemRepository, Mockito.times(1)).save(Mockito.any()); } @DisplayName("SUCCESS: 음식점 db 저장 성공 테스트 - 중복 객체 확인") @Test void batchTest_SuccessfulSaveWithDuplicate() { // Given + String hashKey = DigestUtils.sha256Hex("restaurant" + "-" + "key"); + + when(catcherItem.getItemHashValue()).thenReturn(hashKey); + when(catcherItem.getLocation()).thenReturn(location); + when(catcherItem.getTitle()).thenReturn("저장된 맛집"); + when(catcherItem.getCategory()).thenReturn(category); + + when(catcherItemRepository.findByCategory(any(Category.class))).thenAnswer(invocation -> Collections.singletonList(catcherItem)); + setUpRestaurantItem(); - when(restaurantApiResponse.getItems()).thenReturn(Arrays.asList(restaurantItem, restaurantItem)); // When restaurantService.batch(restaurantApiResponse); // Then - Mockito.verify(catcherItemRepository, Mockito.times(1)).saveAll(Mockito.anyList()); + Mockito.verify(catcherItemRepository, Mockito.never()).save(Mockito.any()); } private void setUpRestaurantItem() { @@ -85,8 +100,8 @@ private void setUpRestaurantItem() { when(restaurantItem.getLatitude()).thenReturn("37.4783683761333"); when(restaurantItem.getLongitude()).thenReturn("126.951561853868"); - when(restaurantApiResponse.getItems()).thenReturn(List.of(restaurantItem)); - Mockito.when(categoryRepository.findByName("restaurant")).thenReturn(Optional.of(Category.create("restaurant"))); - Mockito.when(locationRepository.findByDescription("서울", "관악구")).thenReturn(Optional.of(Location.create("1162000000", "37.4783683761333", "126.951561853868", "서울 관악구"))); + Mockito.when(restaurantApiResponse.getItems()).thenReturn(List.of(restaurantItem)); + Mockito.when(categoryRepository.findByName("restaurant")).thenReturn(Optional.of(category)); + Mockito.when(locationRepository.findByDescription("서울", "관악구")).thenReturn(Optional.of(location)); } } \ No newline at end of file From 76d885cdc9a54523ada3425b31aa24dfcb449c2a Mon Sep 17 00:00:00 2001 From: inyoung Date: Tue, 14 Nov 2023 21:33:27 +0900 Subject: [PATCH 33/37] =?UTF-8?q?feat:=20=ED=82=A4=EA=B0=92=20=EC=A4=91?= =?UTF-8?q?=EB=B3=B5=20=EA=B2=80=EC=82=AC=20=EB=A1=9C=EC=A7=81=20=EC=B6=94?= =?UTF-8?q?=EA=B0=80=20=EA=B5=AC=ED=98=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../batch/core/service/CampingService.java | 6 +++++- .../batch/core/service/RestaurantService.java | 16 +++++++++++----- 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/src/main/java/com/catcher/batch/core/service/CampingService.java b/src/main/java/com/catcher/batch/core/service/CampingService.java index aade016..4edc71e 100644 --- a/src/main/java/com/catcher/batch/core/service/CampingService.java +++ b/src/main/java/com/catcher/batch/core/service/CampingService.java @@ -41,6 +41,8 @@ public void batch(CampingApiResponse campingApiResponse) { Location location = getLocationByDescription(campingItem.getProvince(), campingItem.getCity()); String hashKey = hashString(CATEGORY_NAME, campingItem.getKey()); + itemMap.put(hashKey, campingItem.getName()); + return CatcherItem.builder() .category(category) .location(location) @@ -52,7 +54,9 @@ public void batch(CampingApiResponse campingApiResponse) { }) .collect(Collectors.toList()); - catcherItemRepository.saveAll(catcherItems); + if (!catcherItems.isEmpty()) { + catcherItemRepository.saveAll(catcherItems); + } } private Location getLocationByDescription(String province, String city) { diff --git a/src/main/java/com/catcher/batch/core/service/RestaurantService.java b/src/main/java/com/catcher/batch/core/service/RestaurantService.java index a550c5c..dc8e868 100644 --- a/src/main/java/com/catcher/batch/core/service/RestaurantService.java +++ b/src/main/java/com/catcher/batch/core/service/RestaurantService.java @@ -11,6 +11,7 @@ import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; +import java.util.List; import java.util.Map; import java.util.stream.Collectors; @@ -32,22 +33,27 @@ public void batch(RestaurantApiResponse restaurantApiResponse) { Map itemMap = catcherItemRepository.findByCategory(category).stream() .collect(Collectors.toMap(CatcherItem::getItemHashValue, CatcherItem::getTitle)); - restaurantApiResponse.getItems().stream() + List catcherItems = restaurantApiResponse.getItems().stream() .filter(item -> !itemMap.containsKey(hashString(CATEGORY_NAME, item.getKey()))) - .forEach(item -> { + .map(item -> { Location location = getLocation(item.getAddress()); String hashKey = hashString(CATEGORY_NAME, item.getKey()); - CatcherItem catcherItem = CatcherItem.builder() + itemMap.put(hashKey, item.getName()); + + return CatcherItem.builder() .category(category) .location(location) .title(item.getName()) .resourceUrl(item.getResourceUrl()) .itemHashValue(hashKey) .build(); + }) + .collect(Collectors.toList()); - catcherItemRepository.save(catcherItem); - }); + if (!catcherItems.isEmpty()) { + catcherItemRepository.saveAll(catcherItems); + } } private Location getLocation(String address) { From 2ed9747dba96b769ab24ced46f6830cdbe9d3fa9 Mon Sep 17 00:00:00 2001 From: inyoung Date: Tue, 14 Nov 2023 21:35:01 +0900 Subject: [PATCH 34/37] =?UTF-8?q?test:=20=EC=BD=94=EB=93=9C=20=EB=B3=80?= =?UTF-8?q?=EA=B2=BD=EC=9C=BC=EB=A1=9C=20=EC=9D=B8=ED=95=9C=20=EC=88=98?= =?UTF-8?q?=EC=A0=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/catcher/batch/core/service/RestaurantServiceTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/java/com/catcher/batch/core/service/RestaurantServiceTest.java b/src/test/java/com/catcher/batch/core/service/RestaurantServiceTest.java index ae5b28e..5def08a 100644 --- a/src/test/java/com/catcher/batch/core/service/RestaurantServiceTest.java +++ b/src/test/java/com/catcher/batch/core/service/RestaurantServiceTest.java @@ -67,7 +67,7 @@ void batchTest_SuccessfulSave() { restaurantService.batch(restaurantApiResponse); // Then - Mockito.verify(catcherItemRepository, Mockito.times(1)).save(Mockito.any()); + Mockito.verify(catcherItemRepository, Mockito.times(1)).saveAll(Mockito.anyList()); } @DisplayName("SUCCESS: 음식점 db 저장 성공 테스트 - 중복 객체 확인") @@ -89,7 +89,7 @@ void batchTest_SuccessfulSaveWithDuplicate() { restaurantService.batch(restaurantApiResponse); // Then - Mockito.verify(catcherItemRepository, Mockito.never()).save(Mockito.any()); + Mockito.verify(catcherItemRepository, Mockito.never()).saveAll(Mockito.anyList()); } private void setUpRestaurantItem() { From ca49bdb25c730fce7c3ed348a0dbac7ed7a3e344 Mon Sep 17 00:00:00 2001 From: HongGeun Date: Wed, 15 Nov 2023 00:03:23 +0900 Subject: [PATCH 35/37] change datasource --- src/main/resources/application-dev.properties | 2 +- src/main/resources/application-local.properties | 2 +- src/main/resources/application.properties | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/resources/application-dev.properties b/src/main/resources/application-dev.properties index bdf9a5a..568f206 100644 --- a/src/main/resources/application-dev.properties +++ b/src/main/resources/application-dev.properties @@ -9,7 +9,7 @@ spring.datasource.password=AQICAHg2j0pl3GguoQNkjcreuBPYJqI3OPeLEOelKIkMSzn01AGiM ssh.host=AQICAHg2j0pl3GguoQNkjcreuBPYJqI3OPeLEOelKIkMSzn01AGc1mlusKIbT/FmW65+2rl4AAAAbDBqBgkqhkiG9w0BBwagXTBbAgEAMFYGCSqGSIb3DQEHATAeBglghkgBZQMEAS4wEQQMvijHTTZUUuGjjF4jAgEQgCmKq00a3J3yAUkdljTk342nBjYaeLr5BQeJ2kIISM3P/WPMXKQB/0WzlQ== ssh.username=AQICAHg2j0pl3GguoQNkjcreuBPYJqI3OPeLEOelKIkMSzn01AE26DRJsDEHLdAnA5QIkHfYAAAAZjBkBgkqhkiG9w0BBwagVzBVAgEAMFAGCSqGSIb3DQEHATAeBglghkgBZQMEAS4wEQQMd4p4zWU1dg+tbLAKAgEQgCNWOoyj2Ual/lQaCe7yz2JkY6TXbUR25FO6SkPkYaAncbgkcA== ssh.password=AQICAHg2j0pl3GguoQNkjcreuBPYJqI3OPeLEOelKIkMSzn01AFNH5Ud+NVY/FVqOa99UOeMAAAAaTBnBgkqhkiG9w0BBwagWjBYAgEAMFMGCSqGSIb3DQEHATAeBglghkgBZQMEAS4wEQQMsRopSwW2oDrqOTNrAgEQgCacSRENpF2QbZBfiyN/mlQlY21Qd6PSHef54Rg29FfM5rhWlEuFfg== -ssh.datasource.origin=AQICAHg2j0pl3GguoQNkjcreuBPYJqI3OPeLEOelKIkMSzn01AFY15QxnieQA6WGHi+ZBdpTAAAAtDCBsQYJKoZIhvcNAQcGoIGjMIGgAgEAMIGaBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDDHB9KlaPh/a8J528wIBEIBtds8VUJXlJZSbxWDeV9mnr+16QcsLiVr4C9GlnR44Wb9opywAAUFBCBYqN1SX4czjhkdEgryY9/QYyzSx+VHLT5ORfVTFZ58IbMRGBpZdoXVgXfCH9WSXz3xDDBZQQ7MDlMAj2dECNsgLAA6i2A== +ssh.datasource.origin=AQICAHgxTvGwZVD/2MLMvR9/01Wy8IeL4nGHGGgc5XMYIhkQ2gFoX7QJe7YHVfT6QKf90FzWAAAAojCBnwYJKoZIhvcNAQcGoIGRMIGOAgEAMIGIBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDHCkPDcljb9kXkBq9AIBEIBbyA8xiSyWUYw6KV4WGF4BrlT/LcAt4JrHB/C038I5EveqJ82NhmKU4HnNSDWMTWZP/g8Es1wd2jcGFihOYWbJ664V78jCoJ7X1sVh6F+FrBo7xx6jByc6sVHFlg== ssh.port=22 ssh.local-port=3306 diff --git a/src/main/resources/application-local.properties b/src/main/resources/application-local.properties index bdf9a5a..568f206 100644 --- a/src/main/resources/application-local.properties +++ b/src/main/resources/application-local.properties @@ -9,7 +9,7 @@ spring.datasource.password=AQICAHg2j0pl3GguoQNkjcreuBPYJqI3OPeLEOelKIkMSzn01AGiM ssh.host=AQICAHg2j0pl3GguoQNkjcreuBPYJqI3OPeLEOelKIkMSzn01AGc1mlusKIbT/FmW65+2rl4AAAAbDBqBgkqhkiG9w0BBwagXTBbAgEAMFYGCSqGSIb3DQEHATAeBglghkgBZQMEAS4wEQQMvijHTTZUUuGjjF4jAgEQgCmKq00a3J3yAUkdljTk342nBjYaeLr5BQeJ2kIISM3P/WPMXKQB/0WzlQ== ssh.username=AQICAHg2j0pl3GguoQNkjcreuBPYJqI3OPeLEOelKIkMSzn01AE26DRJsDEHLdAnA5QIkHfYAAAAZjBkBgkqhkiG9w0BBwagVzBVAgEAMFAGCSqGSIb3DQEHATAeBglghkgBZQMEAS4wEQQMd4p4zWU1dg+tbLAKAgEQgCNWOoyj2Ual/lQaCe7yz2JkY6TXbUR25FO6SkPkYaAncbgkcA== ssh.password=AQICAHg2j0pl3GguoQNkjcreuBPYJqI3OPeLEOelKIkMSzn01AFNH5Ud+NVY/FVqOa99UOeMAAAAaTBnBgkqhkiG9w0BBwagWjBYAgEAMFMGCSqGSIb3DQEHATAeBglghkgBZQMEAS4wEQQMsRopSwW2oDrqOTNrAgEQgCacSRENpF2QbZBfiyN/mlQlY21Qd6PSHef54Rg29FfM5rhWlEuFfg== -ssh.datasource.origin=AQICAHg2j0pl3GguoQNkjcreuBPYJqI3OPeLEOelKIkMSzn01AFY15QxnieQA6WGHi+ZBdpTAAAAtDCBsQYJKoZIhvcNAQcGoIGjMIGgAgEAMIGaBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDDHB9KlaPh/a8J528wIBEIBtds8VUJXlJZSbxWDeV9mnr+16QcsLiVr4C9GlnR44Wb9opywAAUFBCBYqN1SX4czjhkdEgryY9/QYyzSx+VHLT5ORfVTFZ58IbMRGBpZdoXVgXfCH9WSXz3xDDBZQQ7MDlMAj2dECNsgLAA6i2A== +ssh.datasource.origin=AQICAHgxTvGwZVD/2MLMvR9/01Wy8IeL4nGHGGgc5XMYIhkQ2gFoX7QJe7YHVfT6QKf90FzWAAAAojCBnwYJKoZIhvcNAQcGoIGRMIGOAgEAMIGIBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDHCkPDcljb9kXkBq9AIBEIBbyA8xiSyWUYw6KV4WGF4BrlT/LcAt4JrHB/C038I5EveqJ82NhmKU4HnNSDWMTWZP/g8Es1wd2jcGFihOYWbJ664V78jCoJ7X1sVh6F+FrBo7xx6jByc6sVHFlg== ssh.port=22 ssh.local-port=3306 diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index 77ec0fb..b7c279f 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -11,7 +11,7 @@ spring.datasource.password=AQICAHg2j0pl3GguoQNkjcreuBPYJqI3OPeLEOelKIkMSzn01AGiM ssh.host=AQICAHg2j0pl3GguoQNkjcreuBPYJqI3OPeLEOelKIkMSzn01AGc1mlusKIbT/FmW65+2rl4AAAAbDBqBgkqhkiG9w0BBwagXTBbAgEAMFYGCSqGSIb3DQEHATAeBglghkgBZQMEAS4wEQQMvijHTTZUUuGjjF4jAgEQgCmKq00a3J3yAUkdljTk342nBjYaeLr5BQeJ2kIISM3P/WPMXKQB/0WzlQ== ssh.username=AQICAHg2j0pl3GguoQNkjcreuBPYJqI3OPeLEOelKIkMSzn01AE26DRJsDEHLdAnA5QIkHfYAAAAZjBkBgkqhkiG9w0BBwagVzBVAgEAMFAGCSqGSIb3DQEHATAeBglghkgBZQMEAS4wEQQMd4p4zWU1dg+tbLAKAgEQgCNWOoyj2Ual/lQaCe7yz2JkY6TXbUR25FO6SkPkYaAncbgkcA== ssh.password=AQICAHg2j0pl3GguoQNkjcreuBPYJqI3OPeLEOelKIkMSzn01AFNH5Ud+NVY/FVqOa99UOeMAAAAaTBnBgkqhkiG9w0BBwagWjBYAgEAMFMGCSqGSIb3DQEHATAeBglghkgBZQMEAS4wEQQMsRopSwW2oDrqOTNrAgEQgCacSRENpF2QbZBfiyN/mlQlY21Qd6PSHef54Rg29FfM5rhWlEuFfg== -ssh.datasource.origin=AQICAHg2j0pl3GguoQNkjcreuBPYJqI3OPeLEOelKIkMSzn01AFY15QxnieQA6WGHi+ZBdpTAAAAtDCBsQYJKoZIhvcNAQcGoIGjMIGgAgEAMIGaBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDDHB9KlaPh/a8J528wIBEIBtds8VUJXlJZSbxWDeV9mnr+16QcsLiVr4C9GlnR44Wb9opywAAUFBCBYqN1SX4czjhkdEgryY9/QYyzSx+VHLT5ORfVTFZ58IbMRGBpZdoXVgXfCH9WSXz3xDDBZQQ7MDlMAj2dECNsgLAA6i2A== +ssh.datasource.origin=AQICAHgxTvGwZVD/2MLMvR9/01Wy8IeL4nGHGGgc5XMYIhkQ2gFoX7QJe7YHVfT6QKf90FzWAAAAojCBnwYJKoZIhvcNAQcGoIGRMIGOAgEAMIGIBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDHCkPDcljb9kXkBq9AIBEIBbyA8xiSyWUYw6KV4WGF4BrlT/LcAt4JrHB/C038I5EveqJ82NhmKU4HnNSDWMTWZP/g8Es1wd2jcGFihOYWbJ664V78jCoJ7X1sVh6F+FrBo7xx6jByc6sVHFlg== ssh.port=22 ssh.local-port=3306 From 290459a9210c0658466b59a6acf5f74548aa17f1 Mon Sep 17 00:00:00 2001 From: nyoung <86757234+inyoung0215@users.noreply.github.com> Date: Wed, 15 Nov 2023 04:55:35 +0000 Subject: [PATCH 36/37] =?UTF-8?q?style:=20=ED=95=84=EC=9A=94=EC=97=86?= =?UTF-8?q?=EB=8A=94=20=EA=B4=84=ED=98=B8=20=EC=A0=9C=EA=B1=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: cheolwon1994 <32122076+cheolwon1994@users.noreply.github.com> --- .../com/catcher/batch/core/service/RestaurantServiceTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/com/catcher/batch/core/service/RestaurantServiceTest.java b/src/test/java/com/catcher/batch/core/service/RestaurantServiceTest.java index 5def08a..81fa8a9 100644 --- a/src/test/java/com/catcher/batch/core/service/RestaurantServiceTest.java +++ b/src/test/java/com/catcher/batch/core/service/RestaurantServiceTest.java @@ -54,7 +54,7 @@ void beforeEach() { restaurantItem = Mockito.mock(RestaurantApiResponse.RestaurantItem.class); catcherItem = Mockito.mock(CatcherItem.class); location = Mockito.mock(Location.class); - category = Mockito.mock((Category.class)); + category = Mockito.mock(Category.class); } @DisplayName("SUCCESS : 음식점 db 저장 성공 테스트") From c5ccd3cca46677bb5109a88bb697eb07e95c6db7 Mon Sep 17 00:00:00 2001 From: ddnjs Date: Fri, 17 Nov 2023 22:15:15 +0900 Subject: [PATCH 37/37] =?UTF-8?q?=EC=A0=84=EC=8B=9C=ED=9A=8C=20DB=EC=A0=80?= =?UTF-8?q?=EC=9E=A5=20=EB=8D=B0=EC=9D=B4=ED=84=B0=20=EC=88=98=EC=A0=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../batch/core/service/ExhibitionService.java | 39 ++++++++++++++----- .../batch/resource/CrawlerController.java | 5 ++- 2 files changed, 33 insertions(+), 11 deletions(-) diff --git a/src/main/java/com/catcher/batch/core/service/ExhibitionService.java b/src/main/java/com/catcher/batch/core/service/ExhibitionService.java index e8a7739..c3ad2a4 100644 --- a/src/main/java/com/catcher/batch/core/service/ExhibitionService.java +++ b/src/main/java/com/catcher/batch/core/service/ExhibitionService.java @@ -1,9 +1,9 @@ package com.catcher.batch.core.service; import com.catcher.batch.core.database.CatcherItemRepository; +import com.catcher.batch.core.database.CategoryRepository; import com.catcher.batch.core.domain.entity.CatcherItem; import com.catcher.batch.core.domain.entity.Category; -import com.catcher.batch.datasource.CategoryRepository; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; import lombok.RequiredArgsConstructor; @@ -16,15 +16,19 @@ import java.time.Duration; import java.util.ArrayList; import java.util.List; -import java.util.UUID; +import java.util.function.Function; +import java.util.stream.Collectors; + +import static com.catcher.batch.common.utils.HashCodeGenerator.hashString; @Service @RequiredArgsConstructor public class ExhibitionService { private final CatcherItemRepository catcherItemRepository; private final CategoryRepository categoryRepository; + public static final String CATEGORY_NAME = "exhibition"; - public List exhibitionCrawling() { + public List exhibitionCrawling() { List catcherItems = new ArrayList<>(); Category category = categoryRepository.findByName("exhibition") .orElseGet(() -> categoryRepository.save(Category.create("exhibition"))); @@ -43,13 +47,21 @@ public List exhibitionCrawling() { ObjectMapper mapper = new ObjectMapper(); List exhibition_list = new ArrayList<>(); - String[] columns = {"eng_name", "abbreviation_name", "host", "period", "location", "field", "homepage"}; + String[] columns = {"engName", "abbreviationName", "host", "period", "location", "field", "homepage", "thumbnailUrl"}; while (true) { WebElement exhibitList = driver.findElement(By.cssSelector(".exhibit_list")); List contentElements = exhibitList.findElements(By.cssSelector(".content_sc_li")); for (WebElement contentElement : contentElements) { + String thumbnailUrl = contentElement.findElement(By.cssSelector("div.schedule_view div.img img")).getAttribute("src"); + String title = contentElement.findElement(By.cssSelector("div.txt strong p")).getText(); + + // 제목에서 인증전시회\n 2023 서울국제소싱페어 이런 케이스 제거 + if (title.contains("\n")) { + title = title.substring(title.indexOf("\n") + 1); + } + WebElement tbody = contentElement.findElement(By.tagName("tbody")); List tr_elements = tbody.findElements(By.tagName("tr")); @@ -60,9 +72,11 @@ public List exhibitionCrawling() { WebElement element = td_element.findElement(By.tagName("td")); data.add(element.getAttribute("innerText")); } - for (int i = 0; i < columns.length; i++) { + for (int i = 0; i < columns.length - 1; i++) { json.put(columns[i], data.get(i)); } + json.put(columns[7], thumbnailUrl); + json.put(columns[0], title); exhibition_list.add(json); } @@ -85,17 +99,24 @@ public List exhibitionCrawling() { for (ObjectNode exhibitionInfo : exhibition_list) { CatcherItem catcherItem = CatcherItem.builder() .category(category) - .title(exhibitionInfo.get("eng_name").asText()) - .itemHashValue(UUID.randomUUID().toString()) + .title(exhibitionInfo.get("engName").asText()) + .itemHashValue(hashString(CATEGORY_NAME, exhibitionInfo.get("abbreviationName").asText())) .description(exhibitionInfo.get("period").asText()) .resourceUrl(exhibitionInfo.get("homepage").asText()) + .thumbnailUrl(exhibitionInfo.get("thumbnailUrl").asText()) .build(); catcherItems.add(catcherItem); } - catcherItemRepository.saveAll(catcherItems); + // 사이트 자체에 중복 데이터가 있기 때문에 hashValue 기준으로 제거 + List uniqueCatcherItems = catcherItems.stream() + .collect(Collectors.toMap(CatcherItem::getItemHashValue, Function.identity(), (existing, replacement) -> existing)) + .values().stream() + .collect(Collectors.toList()); + + catcherItemRepository.saveAll(uniqueCatcherItems); - return exhibition_list; + return uniqueCatcherItems; } // 전시회 사이트 다음페이지 여부 검사 근데 어차피 다른 페이지에는 못써서 안에 넣어야할듯? diff --git a/src/main/java/com/catcher/batch/resource/CrawlerController.java b/src/main/java/com/catcher/batch/resource/CrawlerController.java index c9a8a3a..477e3c7 100644 --- a/src/main/java/com/catcher/batch/resource/CrawlerController.java +++ b/src/main/java/com/catcher/batch/resource/CrawlerController.java @@ -1,5 +1,6 @@ package com.catcher.batch.resource; +import com.catcher.batch.core.domain.entity.CatcherItem; import com.catcher.batch.core.service.ExhibitionService; import com.fasterxml.jackson.databind.node.ObjectNode; import lombok.RequiredArgsConstructor; @@ -18,8 +19,8 @@ public class CrawlerController { private final ExhibitionService exhibitionService; @GetMapping("/exhibition") - public ResponseEntity> getExhibitionData() { - List exhibitionCrawlingResponse = exhibitionService.exhibitionCrawling(); + public ResponseEntity> getExhibitionData() { + List exhibitionCrawlingResponse = exhibitionService.exhibitionCrawling(); return ResponseEntity.ok(exhibitionCrawlingResponse); } }