전시회 크롤링입니다

build.gradle

@@ -54,6 +54,9 @@ dependencies {
     //swagger
     implementation 'org.springdoc:springdoc-openapi-starter-webmvc-ui:2.2.0'
 
+    //crawling
+    implementation 'org.seleniumhq.selenium:selenium-java:4.14.1'
+    implementation 'com.opencsv:opencsv:5.7.1'
 }
 
 dependencyManagement {

src/main/java/com/catcher/batch/BatchApplication.java

@@ -3,9 +3,11 @@
 import org.springframework.boot.SpringApplication;
 import org.springframework.boot.autoconfigure.SpringBootApplication;
 import org.springframework.cloud.openfeign.EnableFeignClients;
+import org.springframework.data.jpa.repository.config.EnableJpaAuditing;
 
 @SpringBootApplication
 @EnableFeignClients(basePackages = "com.catcher.batch.resource.external")
+@EnableJpaAuditing
 public class BatchApplication {
 
     public static void main(String[] args) {

src/main/java/com/catcher/batch/common/service/CatcherFeignService.java

@@ -2,11 +2,13 @@
 
 import com.catcher.batch.core.converter.CatcherConverter;
 import com.catcher.batch.core.properties.PropertyBase;
+import com.catcher.batch.core.properties.HeaderSupport;
 import com.catcher.batch.resource.external.ExternalFeign;
 import jakarta.annotation.PostConstruct;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.context.ApplicationContext;
+import org.springframework.http.HttpHeaders;
 import org.springframework.scheduling.annotation.Async;
 import org.springframework.stereotype.Service;
 
@@ -40,6 +42,13 @@ public <T> T parseService(Map<String, Object> params, Class<T> requestType) {
         property.setParams(params);
         URI uri = property.getURI();
 
+        // 헤더가 있는 경우
+        HttpHeaders headers;
+        if (property instanceof HeaderSupport headerSupport) {
+            headers = headerSupport.addHeaders();
+            return catcherConverter.parse(externalFeign.getInfoWithHeader(headers.getFirst("Authorization"), uri), requestType);
+        }
+
         return catcherConverter.parse(externalFeign.getInfo(uri), requestType);
     }
 

src/main/java/com/catcher/batch/common/utils/HashCodeGenerator.java

@@ -0,0 +1,9 @@
+package com.catcher.batch.common.utils;
+
+import org.apache.commons.codec.digest.DigestUtils;
+
+public class HashCodeGenerator {
+    public static String hashString(String category, String input) {
+        return DigestUtils.sha256Hex(category + "-" + input);
+    }
+}

src/main/java/com/catcher/batch/core/converter/CatcherConverter.java

@@ -3,6 +3,7 @@
 import com.catcher.batch.annotation.CatcherJson;
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.ObjectMapper;
+import org.apache.commons.lang3.StringUtils;
 import org.json.JSONObject;
 import org.springframework.stereotype.Component;
 
@@ -33,6 +34,8 @@ private <T> String getPath(Class<T> responseType) {
     private JSONObject getJsonObject(String json, String jsonPath) {
         if(jsonPath == null) {
             throw new IllegalStateException();
+        } else if (StringUtils.isBlank(jsonPath)) {
+            return new JSONObject(json);
         }
 
         JSONObject jsonObject = new JSONObject(json);

src/main/java/com/catcher/batch/core/database/CatcherItemRepository.java

@@ -0,0 +1,17 @@
+package com.catcher.batch.core.database;
+
+import com.catcher.batch.core.domain.entity.CatcherItem;
+import com.catcher.batch.core.domain.entity.Category;
+
+import java.util.List;
+import java.util.Optional;
+
+public interface CatcherItemRepository {
+    void saveAll(List<CatcherItem> catcherItems);
+
+    void save(CatcherItem catcherItem);
+
+    Optional<CatcherItem> findByItemHashValue(String hashKey);
+
+    List<CatcherItem> findByCategory(Category category);
+}

src/main/java/com/catcher/batch/core/database/CategoryRepository.java

@@ -0,0 +1,11 @@
+package com.catcher.batch.core.database;
+
+import com.catcher.batch.core.domain.entity.Category;
+
+import java.util.Optional;
+
+public interface CategoryRepository {
+    Optional<Category> findByName(String name);
+
+    Category save(Category category);
+}

src/main/java/com/catcher/batch/core/database/LocationRepository.java

@@ -0,0 +1,9 @@
+package com.catcher.batch.core.database;
+
+import com.catcher.batch.core.domain.entity.Location;
+
+import java.util.Optional;
+
+public interface LocationRepository {
+    Optional<Location> findByDescription(String province, String city);
+}

src/main/java/com/catcher/batch/core/domain/CatcherItemExecutor.java

@@ -0,0 +1,12 @@
+package com.catcher.batch.core.domain;
+
+import com.catcher.batch.core.domain.command.Command;
+import org.springframework.stereotype.Component;
+
+@Component
+public class CatcherItemExecutor implements CommandExecutor {
+    @Override
+    public <T> T run(Command<T> command) {
+        return command.execute();
+    }
+}

src/main/java/com/catcher/batch/core/domain/CommandExecutor.java

@@ -0,0 +1,7 @@
+package com.catcher.batch.core.domain;
+
+import com.catcher.batch.core.domain.command.Command;
+
+public interface CommandExecutor {
+    <T> T run(Command<T> command);
+}

src/main/java/com/catcher/batch/core/domain/command/Command.java

@@ -0,0 +1,5 @@
+package com.catcher.batch.core.domain.command;
+
+public interface Command<T> {
+    T execute();
+}

src/main/java/com/catcher/batch/core/domain/command/RegisterCampingDataCommand.java

@@ -0,0 +1,18 @@
+package com.catcher.batch.core.domain.command;
+
+import com.catcher.batch.core.dto.CampingApiResponse;
+import com.catcher.batch.core.service.CampingService;
+import lombok.RequiredArgsConstructor;
+
+
+@RequiredArgsConstructor
+public class RegisterCampingDataCommand implements Command<Void> {
+    private final CampingService campingService;
+    private final CampingApiResponse campingApiResponse;
+
+    @Override
+    public Void execute() {
+        campingService.batch(campingApiResponse);
+        return null;
+    }
+}

src/main/java/com/catcher/batch/core/domain/command/RegisterRestaurantDataCommand.java

@@ -0,0 +1,17 @@
+package com.catcher.batch.core.domain.command;
+
+import com.catcher.batch.core.dto.RestaurantApiResponse;
+import com.catcher.batch.core.service.RestaurantService;
+import lombok.RequiredArgsConstructor;
+
+@RequiredArgsConstructor
+public class RegisterRestaurantDataCommand implements Command<Void> {
+    private final RestaurantService restaurantService;
+    private final RestaurantApiResponse restaurantApiResponse;
+
+    @Override
+    public Void execute() {
+        restaurantService.batch(restaurantApiResponse);
+        return null;
+    }
+}

src/main/java/com/catcher/batch/core/domain/entity/BaseTimeEntity.java

@@ -0,0 +1,32 @@
+package com.catcher.batch.core.domain.entity;
+
+import jakarta.persistence.*;
+import lombok.Getter;
+import org.springframework.data.annotation.CreatedDate;
+import org.springframework.data.annotation.LastModifiedDate;
+import org.springframework.data.jpa.domain.support.AuditingEntityListener;
+
+import java.time.LocalDateTime;
+import java.time.ZonedDateTime;
+
+@EntityListeners(AuditingEntityListener.class)
+@MappedSuperclass
+@Getter
+public class BaseTimeEntity {
+    @Column(name = "created_at")
+    private ZonedDateTime createdAt;
+
+    @Column(name = "updated_at")
+    private ZonedDateTime updatedAt;
+
+    @PrePersist
+    private void prePersist() {
+        this.createdAt = ZonedDateTime.now();
+        this.updatedAt = ZonedDateTime.now();
+    }
+
+    @PreUpdate
+    private void preUpdate() {
+        this.updatedAt = ZonedDateTime.now();
+    }
+}

src/main/java/com/catcher/batch/core/domain/entity/CatcherItem.java

@@ -0,0 +1,49 @@
+package com.catcher.batch.core.domain.entity;
+
+import jakarta.persistence.*;
+import lombok.*;
+import org.hibernate.annotations.Where;
+
+import java.time.ZonedDateTime;
+
+@Entity
+@Builder
+@Getter
+@AllArgsConstructor(access = AccessLevel.PROTECTED)
+@NoArgsConstructor(access = AccessLevel.PROTECTED)
+@Where(clause = "deleted_at IS NULL")
+@Table(name = "catcher_item")
+public class CatcherItem extends BaseTimeEntity {
+    @Id
+    @GeneratedValue(strategy = GenerationType.IDENTITY)
+    private Long id;
+
+    @ManyToOne(fetch = FetchType.LAZY)
+    @JoinColumn(name = "category_id", nullable = false)
+    private Category category;
+
+    @ManyToOne(fetch = FetchType.LAZY)
+    @JoinColumn(name = "location_id")
+    private Location location;
+
+    @Column(unique = true, nullable = false)
+    private String itemHashValue;
+
+    @Column(nullable = false)
+    private String title;
+
+    private String description;
+
+    private String thumbnailUrl;
+
+    private String resourceUrl;
+
+    @Column(name = "start_at")
+    private ZonedDateTime startAt;
+
+    @Column(name = "end_at")
+    private ZonedDateTime endAt;
+
+    @Column(name = "deleted_at")
+    private ZonedDateTime deletedAt;
+}

src/main/java/com/catcher/batch/core/domain/entity/Category.java

@@ -0,0 +1,26 @@
+package com.catcher.batch.core.domain.entity;
+
+import jakarta.persistence.*;
+import lombok.AccessLevel;
+import lombok.Builder;
+import lombok.Getter;
+import lombok.NoArgsConstructor;
+
+@Entity
+@Getter
+@NoArgsConstructor(access = AccessLevel.PROTECTED)
+@Table(name = "category")
+public class Category {
+    @Id
+    @GeneratedValue(strategy = GenerationType.IDENTITY)
+    private Long id;
+
+    @Column(nullable = false)
+    private String name;
+
+    public static Category create(String name) {
+        Category category = new Category();
+        category.name = name;
+        return category;
+    }
+}

src/main/java/com/catcher/batch/core/domain/entity/Location.java

@@ -0,0 +1,24 @@
+package com.catcher.batch.core.domain.entity;
+
+import jakarta.persistence.*;
+import lombok.AccessLevel;
+import lombok.Getter;
+import lombok.NoArgsConstructor;
+
+@Entity
+@Getter
+@NoArgsConstructor(access = AccessLevel.PROTECTED)
+@Table(name = "location")
+public class Location {
+    @Id
+    @GeneratedValue(strategy = GenerationType.IDENTITY)
+    private Long id;
+
+    private String areaCode;
+
+    private String latitude;
+
+    private String longitude;
+
+    private String description;
+}

src/main/java/com/catcher/batch/core/dto/CampingApiResponse.java

@@ -0,0 +1,74 @@
+package com.catcher.batch.core.dto;
+
+import com.catcher.batch.annotation.CatcherJson;
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import lombok.Getter;
+
+import java.util.List;
+
+@Getter
+@JsonIgnoreProperties(ignoreUnknown = true)
+@CatcherJson(path = "response.body")
+public class CampingApiResponse {
+
+    @JsonProperty("items")
+    private CampingItems items;
+
+    @JsonProperty("totalCount")
+    private Integer totalCount;
+
+    @JsonProperty("numOfRows")
+    private Integer numOfRows;
+
+    @JsonProperty("pageNo")
+    private Integer pageNo;
+
+    @Getter
+    @JsonIgnoreProperties(ignoreUnknown = true)
+    public static class CampingItems {
+        @JsonProperty("item")
+        private List<CampingItem> item;
+    }
+
+    @Getter
+    @JsonIgnoreProperties(ignoreUnknown = true)
+    public static class CampingItem {
+
+        @JsonProperty("facltNm")
+        private String name;
+
+        @JsonProperty("contentId")
+        private String key;
+
+        @JsonProperty("lineIntro")
+        private String description;
+
+        @JsonProperty("doNm")
+        private String province;
+
+        @JsonProperty("sigunguNm")
+        private String city;
+
+        @JsonProperty("addr1")
+        private String address;
+
+        @JsonProperty("induty")
+        private String category;
+
+        @JsonProperty("zipcode")
+        private String zipcode;
+
+        @JsonProperty("homepage")
+        private String homepage;
+
+        @JsonProperty("mapX")
+        private String latitude;
+
+        @JsonProperty("mapY")
+        private String longitude;
+
+        @JsonProperty("firstImageUrl")
+        private String thumbnailUrl;
+    }
+}