feat: nature article爬取

谢茂盛
1 parent 47bdaf78
Showing 11 changed files with 157 additions and 61 deletions
sql/webmagic.sql
src/main/java/com/canrd/webmagic/controller/NatureArticleController.java → src/main/java/com/canrd/webmagic/controller/ArticleController.java
src/main/java/com/canrd/webmagic/domain/ArticleTypeEnum.java
src/main/java/com/canrd/webmagic/domain/dto/NatureArticleDO.java → src/main/java/com/canrd/webmagic/domain/dto/ArticleDO.java
src/main/java/com/canrd/webmagic/mapper/NatureArticleMapper.java → src/main/java/com/canrd/webmagic/mapper/ArticleMapper.java
src/main/java/com/canrd/webmagic/processor/NatureSearchPageProcessor.java
src/main/java/com/canrd/webmagic/processor/config/UpdateIp.java
src/main/java/com/canrd/webmagic/processor/pipeline/NatureArticlePipeline.java
src/main/java/com/canrd/webmagic/service/NatureArticleService.java → src/main/java/com/canrd/webmagic/service/ArticleService.java
src/main/java/com/canrd/webmagic/service/impl/NatureArticleServiceImpl.java → src/main/java/com/canrd/webmagic/service/impl/ArticleServiceImpl.java
src/test/java/com/canrd/webmagic/utils/DateTimeUtilTest.java
-DROP TABLE IF EXISTS `nature_article`;
-CREATE TABLE `nature_article` (
+DROP TABLE IF EXISTS `article`;
+CREATE TABLE `article` (
           `id` bigint NOT NULL AUTO_INCREMENT,
+          `article_type` varchar(32) DEFAULT NULL COMMENT '文章类型',
           `article_code` varchar(64) DEFAULT NULL COMMENT '文章标识',
           `author_name` varchar(256) DEFAULT NULL COMMENT '作者名称',
           `title` varchar(256) DEFAULT NULL COMMENT '文章标题',
           `publish_time` varchar(64) DEFAULT NULL COMMENT '发布时间',
           `email_info` varchar(512) DEFAULT NULL COMMENT '邮箱信息',
+          `article_desc` text DEFAULT NULL COMMENT '文章摘要',
+          `author_address` text DEFAULT NULL COMMENT '作者地址信息',
+          `reference_info` text DEFAULT NULL COMMENT '相关文章引用信息',
           PRIMARY KEY (`id`)
-) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COMMENT='nature-文章信息';
-
-
-alter table `nature_article` add column `article_code` varchar(256) DEFAULT NULL COMMENT '文章标识';
 \ No newline at end of file
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COMMENT='文章信息';
 \ No newline at end of file
@@ -6,7 +6,7 @@ import com.canrd.webmagic.domain.vo.NatureArticleQueryVO;
 import com.canrd.webmagic.domain.vo.NatureArticleVO;
 import com.canrd.webmagic.processor.NatureSearchPageProcessor;
 import com.canrd.webmagic.processor.pipeline.NatureArticlePipeline;
-import com.canrd.webmagic.service.NatureArticleService;
+import com.canrd.webmagic.service.ArticleService;
 import org.springframework.validation.annotation.Validated;
 import org.springframework.web.bind.annotation.*;
 import us.codecraft.webmagic.Spider;
@@ -21,12 +21,12 @@ import javax.annotation.Resource;
  */
 @RestController
 @RequestMapping("/nature/article")
-public class NatureArticleController {
+public class ArticleController {
     /**
      * 服务对象
      */
     @Resource
-    private NatureArticleService natureArticleService;
+    private ArticleService articleService;
     @Resource
     private NatureSearchPageProcessor natureSearchPageProcessor;
@@ -60,7 +60,7 @@ public class NatureArticleController {
      */
     @PostMapping("/list")
     public ServerResult list(@RequestBody @Validated({OperateGroup.List.class}) NatureArticleQueryVO natureArticleQueryVO) {
-        return natureArticleService.list(natureArticleQueryVO);
+        return articleService.list(natureArticleQueryVO);
     }
     /**
@@ -71,7 +71,7 @@ public class NatureArticleController {
      */
     @PostMapping("/query_by_id")
     public ServerResult queryById(@RequestBody NatureArticleQueryVO natureArticleQueryVO) {
-        return natureArticleService.queryById(natureArticleQueryVO);
+        return articleService.queryById(natureArticleQueryVO);
     }
     /**
@@ -82,7 +82,7 @@ public class NatureArticleController {
      */
     @PostMapping("/add")
     public ServerResult add(@RequestBody NatureArticleVO natureArticleVO) {
-        return natureArticleService.add(natureArticleVO);
+        return articleService.add(natureArticleVO);
     }
     /**
@@ -93,7 +93,7 @@ public class NatureArticleController {
      */
     @PostMapping("/edit")
     public ServerResult edit(@RequestBody NatureArticleVO natureArticleVO) {
-        return natureArticleService.edit(natureArticleVO);
+        return articleService.edit(natureArticleVO);
     }
     /**
@@ -104,7 +104,7 @@ public class NatureArticleController {
      */
     @PostMapping("/delete_by_id")
     public ServerResult deleteById(@RequestBody NatureArticleQueryVO natureArticleQueryVO) {
-        return natureArticleService.deleteById(natureArticleQueryVO);
+        return articleService.deleteById(natureArticleQueryVO);
     }
 }
+package com.canrd.webmagic.domain;
+
+import lombok.AllArgsConstructor;
+import lombok.Getter;
+import lombok.NoArgsConstructor;
+
+/**
+ * @author: xms
+ * @description: TODO
+ * @date: 2024/4/11 16:52
+ * @version: 1.0
+ */
+@Getter
+@AllArgsConstructor
+@NoArgsConstructor
+public enum ArticleTypeEnum {
+    NATURE("nature", "nature网址"),
+    ;
+    private String type;
+    private String desc;
+
+}
@@ -12,17 +12,23 @@ import java.io.Serializable;
  * @author makejava
  * @since 2024-04-07 18:39:38
  */
-@TableName("nature_article")
+@TableName("article")
 @Data
 @AllArgsConstructor
 @ToString
 @NoArgsConstructor
 @EqualsAndHashCode(callSuper = false)
 @SuperBuilder
-public class NatureArticleDO implements Serializable {
+public class ArticleDO implements Serializable {
     private static final long serialVersionUID = 890672868109538541L;
     private Long id;
+
+    /**
+     * 文章类型：ArticleTypeEnum
+     */
+    private String articleType;
+
     /**
      * 作者名称
      */
@@ -48,4 +54,19 @@ public class NatureArticleDO implements Serializable {
      */
     private String emailInfo;
+    /**
+     * 文章摘要
+     */
+    private String articleDesc;
+
+    /**
+     * 作者地址
+     */
+    private String authorAddress;
+
+    /**
+     * 相关文章引用信息
+     */
+    private String referenceInfo;
+
 }
 package com.canrd.webmagic.mapper;
 import com.baomidou.mybatisplus.core.mapper.BaseMapper;
-import com.canrd.webmagic.domain.dto.NatureArticleDO;
+import com.canrd.webmagic.domain.dto.ArticleDO;
 /**
  * nature-文章信息(NatureArticle)表数据库访问层
@@ -9,7 +9,7 @@ import com.canrd.webmagic.domain.dto.NatureArticleDO;
  * @author makejava
  * @since 2024-04-07 18:39:47
  */
-public interface NatureArticleMapper extends BaseMapper<NatureArticleDO> {
+public interface ArticleMapper extends BaseMapper<ArticleDO> {
 }
@@ -2,10 +2,13 @@ package com.canrd.webmagic.processor;
 import com.alibaba.fastjson.JSONArray;
 import com.alibaba.fastjson.JSONObject;
+import com.baomidou.mybatisplus.core.toolkit.CollectionUtils;
 import com.canrd.webmagic.common.utils.StringUtils;
-import com.canrd.webmagic.domain.dto.NatureArticleDO;
+import com.canrd.webmagic.domain.ArticleTypeEnum;
+import com.canrd.webmagic.domain.dto.ArticleDO;
 import com.canrd.webmagic.processor.config.Agent;
 import com.canrd.webmagic.processor.pipeline.NatureArticlePipeline;
+import lombok.extern.slf4j.Slf4j;
 import org.springframework.stereotype.Component;
 import us.codecraft.webmagic.Page;
 import us.codecraft.webmagic.Site;
@@ -15,8 +18,10 @@ import us.codecraft.webmagic.selector.Html;
 import us.codecraft.webmagic.selector.Selectable;
 import us.codecraft.webmagic.selector.XpathSelector;
+import java.util.ArrayList;
 import java.util.List;
 import java.util.Objects;
+import java.util.stream.Collectors;
 /**
  * @author: xms
@@ -24,12 +29,13 @@ import java.util.Objects;
  * @date: 2024/4/1 14:19
  * @version: 1.0
  */
+@Slf4j
 @Component
 public class NatureSearchPageProcessor implements PageProcessor {
     private String agent = Agent.getRandom();
     // 抓取网站的相关配置，包括编码、抓取间隔、重试次数等
-    private Site site = Site.me().setRetryTimes(3).setSleepTime(100).setUserAgent(agent);
+    private Site site = Site.me().setRetryTimes(3).setSleepTime(100);
     /**
      * 定制爬虫逻辑的核心接口，在这里编写抽取逻辑
@@ -42,17 +48,16 @@ public class NatureSearchPageProcessor implements PageProcessor {
             doArticleList(page);
         } else if (page.getUrl().get().contains("research-articles")) {
             doArticleList4ReSearch(page);
-        }else {
+        } else {
             doArticleContent(page);
         }
     }
     /**
-     *
      * @param page
      */
-    private void doArticleList4ReSearch(Page page){
+    private void doArticleList4ReSearch(Page page) {
         String url = page.getUrl().get();
         String[] split = url.split("=");
         Integer pageIndex = Integer.parseInt(split[split.length - 1]);
@@ -79,17 +84,23 @@ public class NatureSearchPageProcessor implements PageProcessor {
     }
     private void doArticleContent(Page page) {
+        if (page.getUrl().get().contains("redirect") || !page.getUrl().get().contains("nature")) {
+            return;
+        }
         //解析页面
         Html html = page.getHtml();
         String[] urlArr = page.getUrl().get().split("/");
         String articleCode = urlArr[urlArr.length - 1];
         Selectable headSelectable = html.xpath("//div[@class='c-article-header']/header");
         List<Selectable> authorEmailSelectables = html.xpath("//p[@id='corresponding-author-list']/a").nodes();
+        Selectable referencesSelectable = html.xpath("//ol[@class='c-article-references']").select(new XpathSelector("li[@class='c-article-references__item js-c-reading-companion-references-item']"));
+        Selectable authorAddressSelectable = html.xpath("//ol[@class='c-article-author-affiliation__list']").select(new XpathSelector("li"));
         String title = headSelectable.xpath("//div/h1/text()").get();
         if (StringUtils.isBlank(title)) {
             title = headSelectable.xpath("//h1/text()").get();
         }
+        String articleDesc = html.xpath("//div[@class='c-article-section__content']/p/text()").get();
         String publishTime = headSelectable.xpath("//ul").nodes().get(0).xpath("//li").nodes().get(2).xpath("//li/time/text()").get();
         Selectable authorSelectable = headSelectable.xpath("//ul").nodes().get(1).select(new XpathSelector("li[@class='c-article-author-list__item']"));
         List<Selectable> authorNodes = authorSelectable.nodes();
@@ -97,7 +108,41 @@ public class NatureSearchPageProcessor implements PageProcessor {
         for (Selectable node : authorNodes) {
             authorName.append(node.xpath("//a/text()"));
         }
-        JSONArray array = new JSONArray();
+
+        JSONArray authorAddress = new JSONArray();
+        List<Selectable> authorAddressList = authorAddressSelectable.nodes();
+        if (CollectionUtils.isNotEmpty(authorAddressList)) {
+            for (Selectable selectable : authorAddressList) {
+                String address = selectable.xpath("//p").xpath("//p[@class='c-article-author-affiliation__address']/text()").get();
+                String authorNames = selectable.xpath("//p").xpath("//p[@class='c-article-author-affiliation__authors-list']/text()").get();
+                JSONObject object = new JSONObject();
+                object.put("address", address);
+                object.put("authorNames", authorNames);
+                authorAddress.add(object);
+            }
+        }
+
+        JSONArray references = new JSONArray();
+        List<Selectable> referenceList = referencesSelectable.nodes();
+        if (CollectionUtils.isNotEmpty(referenceList)) {
+            for (Selectable reference : referenceList) {
+                String referenceTitle = reference.xpath("//p").xpath("//p[@class='c-article-references__text']/text()").get();
+                List<Selectable> referenceLinks = reference.xpath("//p").xpath("//p[@class='c-article-references__links u-hide-print']").links().nodes();
+                List<String> links = new ArrayList<>();
+                if (CollectionUtils.isNotEmpty(referenceLinks)) {
+                    links = referenceLinks.stream().map(x -> x.get()).collect(Collectors.toList());
+                }
+                JSONObject object = new JSONObject();
+                object.put("referenceTitle", referenceTitle);
+                object.put("links", links);
+                if (CollectionUtils.isNotEmpty(links)) {
+                    page.addTargetRequests(links.stream().filter(x -> x.contains("nature")).collect(Collectors.toList()));
+                }
+                references.add(object);
+            }
+        }
+
+        JSONArray authorEmail = new JSONArray();
         for (Selectable authorEmailSelectable : authorEmailSelectables) {
             String[] split = authorEmailSelectable.xpath("//a").links().get().split(":");
             String email = Objects.isNull(split) ? "" : split[split.length - 1];
@@ -105,16 +150,21 @@ public class NatureSearchPageProcessor implements PageProcessor {
             JSONObject jsonObject = new JSONObject();
             jsonObject.put("authorEmailName", authorEmailName);
             jsonObject.put("email", email);
-            array.add(jsonObject);
+            authorEmail.add(jsonObject);
         }
-        System.out.println("code:" + articleCode + ",发布时间：" + publishTime + ",标题：" + title + ",作者：" + authorName + "，邮箱信息：" + array.toJSONString());
+        System.out.println("code:" + articleCode + ",发布时间：" + publishTime + ",标题：" + title + ",作者：" + authorName + "，邮箱信息：" + authorEmail.toJSONString());
+
-        page.putField("article", NatureArticleDO.builder()
+        page.putField("article", ArticleDO.builder()
+                .articleType(ArticleTypeEnum.NATURE.getType())
                 .articleCode(articleCode)
                 .authorName(authorName.toString())
                 .title(title)
                 .publishTime(publishTime)
-                .emailInfo(array.toJSONString()).build());
+                .emailInfo(authorEmail.toJSONString())
+                .articleDesc(articleDesc)
+                .authorAddress(authorAddress.toJSONString())
+                .referenceInfo(references.toJSONString()).build());
     }
     private void doArticleList(Page page) {
@@ -152,7 +202,7 @@ public class NatureSearchPageProcessor implements PageProcessor {
         // 创建一个Spider，并把我们的处理器放进去
         Spider.create(new NatureSearchPageProcessor())
                 // 添加这个Spider要爬取的网页地址
-                .addUrl("https://www.nature.com/search?q=battery&page=1")
+                .addUrl("https://www.nature.com/nature/research-articles?sort=PubDate&page=1")
                 .addPipeline(new NatureArticlePipeline())
                 // 开启5个线程执行，并开始爬取
                 .thread(5).run();
@@ -31,7 +31,7 @@ public class UpdateIp {
     @Autowired
     private RedisTemplate redisTemplate;
-    @Scheduled(cron = "*/20 * * * * ?")
+//    @Scheduled(cron = "*/20 * * * * ?")
     void update() {
         List<String> range = redisTemplate.opsForList().range("ip", 0, -1);
         for (String ip : range) {
@@ -42,7 +42,7 @@ public class UpdateIp {
         }
     }
-    @Scheduled(cron = "*/15 * * * * ?")
+//    @Scheduled(cron = "*/15 * * * * ?")
     void ips() {
         String string = null;
         try {
 package com.canrd.webmagic.processor.pipeline;
 import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
-import com.canrd.webmagic.domain.dto.NatureArticleDO;
-import com.canrd.webmagic.service.NatureArticleService;
+import com.baomidou.mybatisplus.core.toolkit.CollectionUtils;
+import com.canrd.webmagic.domain.dto.ArticleDO;
+import com.canrd.webmagic.service.ArticleService;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Component;
 import us.codecraft.webmagic.ResultItems;
 import us.codecraft.webmagic.Task;
 import us.codecraft.webmagic.pipeline.Pipeline;
+import java.util.List;
 import java.util.Objects;
 @Component
 public class NatureArticlePipeline implements Pipeline {
-    private NatureArticleService natureArticleService;
+    private ArticleService articleService;
     @Autowired
-    public void setNatureArticleService(NatureArticleService natureArticleService) {
-        this.natureArticleService = natureArticleService;
+    public void setNatureArticleService(ArticleService articleService) {
+        this.articleService = articleService;
     }
     @Override
     public void process(ResultItems resultItems, Task task) {
-        NatureArticleDO articleDO = resultItems.get("article");
+        ArticleDO articleDO = resultItems.get("article");
         if (Objects.nonNull(articleDO)) {
-            NatureArticleDO natureArticleDO = natureArticleService.getOne(new LambdaQueryWrapper<NatureArticleDO>().eq(NatureArticleDO::getArticleCode, articleDO.getArticleCode()));
-            if (Objects.nonNull(natureArticleDO)) {
+            List<ArticleDO> natureArticleDO = articleService.list(new LambdaQueryWrapper<ArticleDO>().eq(ArticleDO::getArticleCode, articleDO.getArticleCode()));
+            if (CollectionUtils.isNotEmpty(natureArticleDO)) {
                 return;
             }
-            natureArticleService.save(articleDO);
+            articleService.save(articleDO);
         }
     }
 }
@@ -2,7 +2,7 @@ package com.canrd.webmagic.service;
 import com.baomidou.mybatisplus.extension.service.IService;
 import com.canrd.webmagic.common.constant.ServerResult;
-import com.canrd.webmagic.domain.dto.NatureArticleDO;
+import com.canrd.webmagic.domain.dto.ArticleDO;
 import com.canrd.webmagic.domain.vo.NatureArticleQueryVO;
 import com.canrd.webmagic.domain.vo.NatureArticleVO;
@@ -12,7 +12,7 @@ import com.canrd.webmagic.domain.vo.NatureArticleVO;
  * @author makejava
  * @since 2024-04-07 18:39:48
  */
-public interface NatureArticleService extends IService<NatureArticleDO> {
+public interface ArticleService extends IService<ArticleDO> {
     /**
      * 通过ID查询单条数据
@@ -5,11 +5,11 @@ import cn.hutool.core.collection.CollUtil;
 import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
 import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
 import com.canrd.webmagic.common.constant.ServerResult;
-import com.canrd.webmagic.domain.dto.NatureArticleDO;
+import com.canrd.webmagic.domain.dto.ArticleDO;
 import com.canrd.webmagic.domain.vo.NatureArticleQueryVO;
 import com.canrd.webmagic.domain.vo.NatureArticleVO;
-import com.canrd.webmagic.mapper.NatureArticleMapper;
-import com.canrd.webmagic.service.NatureArticleService;
+import com.canrd.webmagic.mapper.ArticleMapper;
+import com.canrd.webmagic.service.ArticleService;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.stereotype.Service;
@@ -24,7 +24,7 @@ import java.util.Objects;
  */
 @Slf4j
 @Service
-public class NatureArticleServiceImpl extends ServiceImpl<NatureArticleMapper, NatureArticleDO> implements NatureArticleService {
+public class ArticleServiceImpl extends ServiceImpl<ArticleMapper, ArticleDO> implements ArticleService {
     /**
@@ -39,11 +39,11 @@ public class NatureArticleServiceImpl extends ServiceImpl&lt;NatureArticleMapper, N
         if (Objects.isNull(natureArticleQueryVO.getId())) {
             return ServerResult.fail("id 不能为空");
         }
-        NatureArticleDO NatureArticleDo = getById(natureArticleQueryVO.getId());
-        if (Objects.isNull(NatureArticleDo)) {
+        ArticleDO articleDo = getById(natureArticleQueryVO.getId());
+        if (Objects.isNull(articleDo)) {
             return ServerResult.success(null);
         }
-        return ServerResult.success(BeanUtil.copyProperties(NatureArticleDo, NatureArticleVO.class));
+        return ServerResult.success(BeanUtil.copyProperties(articleDo, NatureArticleVO.class));
     }
     /**
@@ -69,9 +69,9 @@ public class NatureArticleServiceImpl extends ServiceImpl&lt;NatureArticleMapper, N
         if (Objects.nonNull(natureArticleVO.getId())) {
             natureArticleVO.setId(null);
         }
-        NatureArticleDO natureArticleDo = BeanUtil.copyProperties(natureArticleVO, NatureArticleDO.class);
+        ArticleDO articleDo = BeanUtil.copyProperties(natureArticleVO, ArticleDO.class);
-        save(natureArticleDo);
+        save(articleDo);
         return ServerResult.success();
     }
@@ -88,9 +88,9 @@ public class NatureArticleServiceImpl extends ServiceImpl&lt;NatureArticleMapper, N
         if (Objects.isNull(natureArticleVO.getId())) {
             return ServerResult.fail("id 不能为空");
         }
-        NatureArticleDO natureArticleDo = BeanUtil.copyProperties(natureArticleVO, NatureArticleDO.class);
+        ArticleDO articleDo = BeanUtil.copyProperties(natureArticleVO, ArticleDO.class);
-        updateById(natureArticleDo);
+        updateById(articleDo);
         return ServerResult.success();
     }
@@ -107,13 +107,13 @@ public class NatureArticleServiceImpl extends ServiceImpl&lt;NatureArticleMapper, N
         if (CollUtil.isEmpty(ids)) {
             return ServerResult.fail("ids 参数不能为空");
         }
-        List<NatureArticleDO> natureArticleList = listByIds(ids);
+        List<ArticleDO> natureArticleList = listByIds(ids);
         if (CollUtil.isEmpty(natureArticleList)) {
             return ServerResult.success();
         }
         //todo 校验是否可以逻辑删除
-        LambdaUpdateWrapper<NatureArticleDO> updateWrapper = new LambdaUpdateWrapper<NatureArticleDO>()
-                .in(NatureArticleDO::getId, ids);
+        LambdaUpdateWrapper<ArticleDO> updateWrapper = new LambdaUpdateWrapper<ArticleDO>()
+                .in(ArticleDO::getId, ids);
         update(updateWrapper);
         return ServerResult.success();
     }
@@ -4,8 +4,8 @@ import com.alibaba.fastjson.JSON;
 import com.alibaba.fastjson.JSONArray;
 import com.alibaba.fastjson.JSONObject;
 import com.canrd.webmagic.BaseTest;
-import com.canrd.webmagic.domain.dto.NatureArticleDO;
-import com.canrd.webmagic.service.NatureArticleService;
+import com.canrd.webmagic.domain.dto.ArticleDO;
+import com.canrd.webmagic.service.ArticleService;
 import org.junit.Test;
 import javax.annotation.Resource;
@@ -20,13 +20,13 @@ import java.util.List;
 public class DateTimeUtilTest extends BaseTest {
     @Resource
-    private NatureArticleService natureArticleService;
+    private ArticleService articleService;
     @Test
     public void export() {
-        List<NatureArticleDO> articleDOList = natureArticleService.list();
+        List<ArticleDO> articleDOList = articleService.list();
         JSONArray array = new JSONArray();
-        for (NatureArticleDO articleDO : articleDOList) {
+        for (ArticleDO articleDO : articleDOList) {
             JSONArray jsonArray = JSONArray.parseArray(articleDO.getEmailInfo());
             array.addAll(jsonArray);
         }