nature文章信息爬取，存储db

qdlgxiemaosheng
1 parent ea294a6a
Showing 11 changed files with 142 additions and 50 deletions
sql/webmagic.sql
src/main/java/com/canrd/webmagic/controller/NatureArticleController.java
src/main/java/com/canrd/webmagic/domain/dto/NatureArticleDO.java
src/main/java/com/canrd/webmagic/domain/vo/NatureArticleQueryVO.java
src/main/java/com/canrd/webmagic/domain/vo/NatureArticleStartVO.java
src/main/java/com/canrd/webmagic/domain/vo/NatureArticleVO.java
src/main/java/com/canrd/webmagic/processor/NatureArticlePipeline.java
src/main/java/com/canrd/webmagic/processor/NatureSearchPageProcessor.java
src/main/java/com/canrd/webmagic/service/NatureArticleService.java
src/main/java/com/canrd/webmagic/service/impl/NatureArticleServiceImpl.java
src/main/resources/application-local.yml
 DROP TABLE IF EXISTS `nature_article`;
 CREATE TABLE `nature_article` (
           `id` bigint NOT NULL AUTO_INCREMENT,
+          `article_code` varchar(64) DEFAULT NULL COMMENT '文章标识',
           `author_name` varchar(256) DEFAULT NULL COMMENT '作者名称',
           `title` varchar(256) DEFAULT NULL COMMENT '文章标题',
           `publish_time` varchar(64) DEFAULT NULL COMMENT '发布时间',
           `email_info` varchar(512) DEFAULT NULL COMMENT '邮箱信息',
           PRIMARY KEY (`id`)
-) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COMMENT='nature-文章信息';
 \ No newline at end of file
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COMMENT='nature-文章信息';
+
+
+alter table `nature_article` add column `article_code` varchar(256) DEFAULT NULL COMMENT '文章标识';
 \ No newline at end of file
@@ -4,12 +4,12 @@ import com.canrd.webmagic.common.constant.ServerResult;
 import com.canrd.webmagic.common.jsr303.OperateGroup;
 import com.canrd.webmagic.domain.vo.NatureArticleQueryVO;
 import com.canrd.webmagic.domain.vo.NatureArticleVO;
+import com.canrd.webmagic.processor.NatureArticlePipeline;
+import com.canrd.webmagic.processor.NatureSearchPageProcessor;
 import com.canrd.webmagic.service.NatureArticleService;
 import org.springframework.validation.annotation.Validated;
-import org.springframework.web.bind.annotation.PostMapping;
-import org.springframework.web.bind.annotation.RequestBody;
-import org.springframework.web.bind.annotation.RequestMapping;
-import org.springframework.web.bind.annotation.RestController;
+import org.springframework.web.bind.annotation.*;
+import us.codecraft.webmagic.Spider;
 import javax.annotation.Resource;
@@ -20,7 +20,7 @@ import javax.annotation.Resource;
  * @since 2024-04-07 18:39:41
  */
 @RestController
-@RequestMapping("/gwms/xxx")
+@RequestMapping("/nature/article")
 public class NatureArticleController {
     /**
      * 服务对象
@@ -28,6 +28,29 @@ public class NatureArticleController {
     @Resource
     private NatureArticleService natureArticleService;
+    @Resource
+    private NatureSearchPageProcessor natureSearchPageProcessor;
+
+    @Resource
+    private NatureArticlePipeline articlePipeline;
+
+    /**
+     * @return
+     */
+    @GetMapping("/start")
+    public ServerResult start(@RequestParam(value = "keyword") String keyword, @RequestParam(value = "indexSize") Integer indexSize) {
+        for (int i = 1; i <= indexSize; i++) {
+            Spider.create(natureSearchPageProcessor)
+                    // 添加这个Spider要爬取的网页地址
+                    .addUrl("https://www.nature.com/search?q=" + keyword + "&page=" + i)
+                    .addPipeline(articlePipeline)
+                    // 开启5个线程执行，并开始爬取
+                    .thread(5).run();
+        }
+
+        return ServerResult.success();
+    }
+
     /**
      * 分页查询
      *
 package com.canrd.webmagic.domain.dto;
-.dto;
-
-
-import java.io.Serializable;
-
 import com.baomidou.mybatisplus.annotation.TableName;
-import com.gree.gaolan.common.dto.BaseDO;
 import lombok.*;
 import lombok.experimental.SuperBuilder;
+import java.io.Serializable;
+
 /**
  * nature-文章信息(NatureArticle)实体类
  *
@@ -23,7 +19,7 @@ import lombok.experimental.SuperBuilder;
 @NoArgsConstructor
 @EqualsAndHashCode(callSuper = false)
 @SuperBuilder
-public class NatureArticleDO extends BaseDO implements Serializable {
+public class NatureArticleDO implements Serializable {
     private static final long serialVersionUID = 890672868109538541L;
     private Long id;
@@ -31,14 +27,22 @@ public class NatureArticleDO extends BaseDO implements Serializable {
      * 作者名称
      */
     private String authorName;
+
+    /**
+     * 文章标识
+     */
+    private String articleCode;
+
     /**
      * 文章标题
      */
     private String title;
+
     /**
      * 发布时间
      */
     private String publishTime;
+
     /**
      * 邮箱信息
      */
 package com.canrd.webmagic.domain.vo;
-.vo;
-
-import java.io.Serializable;
-
 import lombok.*;
 import lombok.experimental.SuperBuilder;
-import com.gree.gaolan.common.vo.common.BasePageVO;
-import java.lang.Long;
+import java.io.Serializable;
 import java.util.List;
 /**
+package com.canrd.webmagic.domain.vo;
+
+import lombok.*;
+import lombok.experimental.SuperBuilder;
+
+import java.io.Serializable;
+import java.util.List;
+
+/**
+ * nature-文章信息(NatureArticle)实体类
+ *
+ * @author makejava
+ * @since 2024-04-07 18:39:41
+ */
+@Data
+@AllArgsConstructor
+@ToString
+@NoArgsConstructor
+@EqualsAndHashCode(callSuper = false)
+@SuperBuilder
+public class NatureArticleStartVO implements Serializable {
+
+    /**
+     * 爬取的总页数
+     */
+    private Integer indexSize;
+
+    /**
+     * 爬取的关键字
+     */
+    private String keyword;
+}
+
 package com.canrd.webmagic.domain.vo;
-.vo;
-
-
-import java.io.Serializable;
-
 import lombok.*;
 import lombok.experimental.SuperBuilder;
+import java.io.Serializable;
+
 /**
  * nature-文章信息(NatureArticle)实体类
  *
+package com.canrd.webmagic.processor;
+
+import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
+import com.canrd.webmagic.domain.dto.NatureArticleDO;
+import com.canrd.webmagic.service.NatureArticleService;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+import us.codecraft.webmagic.ResultItems;
+import us.codecraft.webmagic.Task;
+import us.codecraft.webmagic.pipeline.Pipeline;
+
+import java.util.Objects;
+
+@Component
+public class NatureArticlePipeline implements Pipeline {
+
+    private NatureArticleService natureArticleService;
+
+    @Autowired
+    public void setNatureArticleService(NatureArticleService natureArticleService) {
+        this.natureArticleService = natureArticleService;
+    }
+
+    @Override
+    public void process(ResultItems resultItems, Task task) {
+        NatureArticleDO articleDO = resultItems.get("article");
+        if (Objects.nonNull(articleDO)) {
+            NatureArticleDO natureArticleDO = natureArticleService.getOne(new LambdaQueryWrapper<NatureArticleDO>().eq(NatureArticleDO::getArticleCode, articleDO.getArticleCode()));
+            if (Objects.nonNull(natureArticleDO)) {
+                return;
+            }
+            natureArticleService.save(articleDO);
+        }
+    }
+}
@@ -3,6 +3,8 @@ package com.canrd.webmagic.processor;
 import com.alibaba.fastjson.JSONArray;
 import com.alibaba.fastjson.JSONObject;
 import com.canrd.webmagic.common.utils.StringUtils;
+import com.canrd.webmagic.domain.dto.NatureArticleDO;
+import org.springframework.stereotype.Component;
 import us.codecraft.webmagic.Page;
 import us.codecraft.webmagic.Site;
 import us.codecraft.webmagic.Spider;
@@ -20,6 +22,7 @@ import java.util.Objects;
  * @date: 2024/4/1 14:19
  * @version: 1.0
  */
+@Component
 public class NatureSearchPageProcessor implements PageProcessor {
     // 抓取网站的相关配置，包括编码、抓取间隔、重试次数等
@@ -32,7 +35,7 @@ public class NatureSearchPageProcessor implements PageProcessor {
      */
     @Override
     public void process(Page page) {
-        if ("https://www.nature.com/search?q=battery&page=1".equals(page.getUrl().get())) {
+        if (page.getUrl().get().contains("search")) {
             doArticleList(page);
         } else {
             doArticleContent(page);
@@ -44,7 +47,7 @@ public class NatureSearchPageProcessor implements PageProcessor {
         //解析页面
         Html html = page.getHtml();
         String[] urlArr = page.getUrl().get().split("/");
-        String articleId = urlArr[urlArr.length - 1];
+        String articleCode = urlArr[urlArr.length - 1];
         Selectable headSelectable = html.xpath("//div[@class='c-article-header']/header");
         List<Selectable> authorEmailSelectables = html.xpath("//p[@id='corresponding-author-list']/a").nodes();
@@ -69,7 +72,14 @@ public class NatureSearchPageProcessor implements PageProcessor {
             jsonObject.put("email", email);
             array.add(jsonObject);
         }
-        System.out.println("id:" + articleId + ",发布时间：" + publishTime + ",标题：" + title + ",作者：" + authorName + "，邮箱信息：" + array.toJSONString());
+        System.out.println("code:" + articleCode + ",发布时间：" + publishTime + ",标题：" + title + ",作者：" + authorName + "，邮箱信息：" + array.toJSONString());
+
+        page.putField("article", NatureArticleDO.builder()
+                .articleCode(articleCode)
+                .authorName(authorName.toString())
+                .title(title)
+                .publishTime(publishTime)
+                .emailInfo(array.toJSONString()).build());
     }
     private void doArticleList(Page page) {
@@ -96,7 +106,6 @@ public class NatureSearchPageProcessor implements PageProcessor {
             String title = node.$("a", "text").get();
             System.out.printf("%d、%s，访问地址：%s%n", i, title, link1);
         }
-//        page.addTargetRequest("https://www.nature.com/search?q=battery&page=" + pageIndex);
     }
     @Override
@@ -109,6 +118,7 @@ public class NatureSearchPageProcessor implements PageProcessor {
         Spider.create(new NatureSearchPageProcessor())
                 // 添加这个Spider要爬取的网页地址
                 .addUrl("https://www.nature.com/search?q=battery&page=1")
+                .addPipeline(new NatureArticlePipeline())
                 // 开启5个线程执行，并开始爬取
                 .thread(5).run();
     }
 package com.canrd.webmagic.service;
 import com.baomidou.mybatisplus.extension.service.IService;
+import com.canrd.webmagic.common.constant.ServerResult;
 import com.canrd.webmagic.domain.dto.NatureArticleDO;
-import com.gree.gaolan.common.constant.ServerResult;
+import com.canrd.webmagic.domain.vo.NatureArticleQueryVO;
+import com.canrd.webmagic.domain.vo.NatureArticleVO;
 /**
  * nature-文章信息(NatureArticle)表服务接口
 package com.canrd.webmagic.service.impl;
-import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
+import cn.hutool.core.bean.BeanUtil;
+import cn.hutool.core.collection.CollUtil;
 import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
-import com.baomidou.mybatisplus.core.metadata.IPage;
-import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
 import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
+import com.canrd.webmagic.common.constant.ServerResult;
 import com.canrd.webmagic.domain.dto.NatureArticleDO;
-import com.central.common.utils.CopyBeanUtils;
-import com.gree.doraemon.base.core.bean.BeanUtil;
-import com.gree.doraemon.base.core.collection.CollUtil;
-import com.gree.gaolan.common.constant.Constant;
-import com.gree.gaolan.common.constant.ServerResult;
-import com.gree.gaolan.common.util.PageUtils;
+import com.canrd.webmagic.domain.vo.NatureArticleQueryVO;
+import com.canrd.webmagic.domain.vo.NatureArticleVO;
+import com.canrd.webmagic.mapper.NatureArticleMapper;
+import com.canrd.webmagic.service.NatureArticleService;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.stereotype.Service;
@@ -56,15 +54,7 @@ public class NatureArticleServiceImpl extends ServiceImpl&lt;NatureArticleMapper, N
      */
     @Override
     public ServerResult list(NatureArticleQueryVO natureArticleQueryVO) {
-
-        LambdaQueryWrapper<NatureArticleDO> queryWapper = new LambdaQueryWrapper<NatureArticleDO>()
-                .eq(NatureArticleDO::getEnableFlag, Constant.ENABLE_TEN)
-                .orderByDesc(NatureArticleDO::getId);
-        Page page = new Page<>(natureArticleQueryVO.getCurrent(), natureArticleQueryVO.getSize());
-        IPage<NatureArticleDO> iPage = page(page, queryWapper);
-        natureArticleQueryVO.setTotal(Long.valueOf(iPage.getTotal()).intValue());
-        List<NatureArticleVO> result = CopyBeanUtils.trans(iPage.getRecords(), NatureArticleVO.class);
-        return ServerResult.success(PageUtils.getPageReturn(result, natureArticleQueryVO));
+        return ServerResult.success(ServerResult.success());
     }
     /**
@@ -123,8 +113,7 @@ public class NatureArticleServiceImpl extends ServiceImpl&lt;NatureArticleMapper, N
         }
         //todo 校验是否可以逻辑删除
         LambdaUpdateWrapper<NatureArticleDO> updateWrapper = new LambdaUpdateWrapper<NatureArticleDO>()
-                .in(NatureArticleDO::getId, ids)
-                .set(NatureArticleDO::getEnableFlag, Constant.UNABLE_TWENTY);
+                .in(NatureArticleDO::getId, ids);
         update(updateWrapper);
         return ServerResult.success();
     }
@@ -59,7 +59,7 @@ spring:
     testOnReturn: true
     password: canrd@2024
     time-between-eviction-runs-millis: 1000
-    url: jdbc:mysql://39.108.227.113:3307/order-erp1?useUnicode=true&characterEncoding=UTF-8&useJDBCCompliantTimezoneShift=true&useLegacyDatetimeCode=false&serverTimezone=Asia/Shanghai&useSSL=false&autoReconnect=true&failOverReadOnly=false&maxReconnects=10&allowMultiQueries=true&useAffectedRows=true&autoReconnectForPools=true
+    url: jdbc:mysql://39.108.227.113:3307/webmagic?useUnicode=true&characterEncoding=UTF-8&useJDBCCompliantTimezoneShift=true&useLegacyDatetimeCode=false&serverTimezone=Asia/Shanghai&useSSL=false&autoReconnect=true&failOverReadOnly=false&maxReconnects=10&allowMultiQueries=true&useAffectedRows=true&autoReconnectForPools=true
     username: root
   redis:
     database: 0