feat: nature article爬取

谢茂盛
1 parent 0c4251f2
Showing 2 changed files with 12 additions and 15 deletions
src/main/java/com/canrd/webmagic/processor/NatureSearchPageProcessor.java
src/test/java/com/canrd/webmagic/utils/DateTimeUtilTest.java
@@ -59,8 +59,6 @@ public class NatureSearchPageProcessor implements PageProcessor {
      */
     private void doArticleList4ReSearch(Page page) {
         String url = page.getUrl().get();
-        String[] split = url.split("=");
-        Integer pageIndex = Integer.parseInt(split[split.length - 1]);
         /**
          * 通过page.getHtml()可以获取到main函数中Spider.create(new BaiduHotSearchPageProcessor()).addUrl中的地址的网页内容
          * 1、通过$或css()方法获取到该page html下某元素dom
@@ -77,9 +75,8 @@ public class NatureSearchPageProcessor implements PageProcessor {
             Selectable node = nodes.get(i).$(".u-full-height").nodes().get(2).nodes().get(0).$(".u-full-height").select(new XpathSelector("a[@class='c-card__link u-link-inherit']")).nodes().get(0);
             String link = node.$("a", "href").get();
             page.addTargetRequest(link);
-            String link1 = node.links().get();
             String title = node.$("a", "text").get();
-            System.out.printf("%d、%s，访问地址：%s%n", i, title, link1);
+            log.info("research文章列表链接：{},标题:{},文章链接：{}", url, title, link);
         }
     }
  
@@ -89,8 +86,7 @@ public class NatureSearchPageProcessor implements PageProcessor {
         }
         //解析页面
         Html html = page.getHtml();
-        String[] urlArr = page.getUrl().get().split("/");
-        String articleCode = urlArr[urlArr.length - 1];
+        String articleCode = page.getUrl().get();
         Selectable headSelectable = html.xpath("//div[@class='c-article-header']/header");
         List<Selectable> authorEmailSelectables = html.xpath("//p[@id='corresponding-author-list']/a").nodes();
         Selectable referencesSelectable = html.xpath("//ol[@class='c-article-references']").select(new XpathSelector("li[@class='c-article-references__item js-c-reading-companion-references-item']"));
@@ -104,10 +100,10 @@ public class NatureSearchPageProcessor implements PageProcessor {
         String publishTime;
         try {
             publishTime = headSelectable.xpath("//ul").nodes().get(0).xpath("//li").nodes().get(2).xpath("//li/time/text()").get();
-        }catch (Exception e) {
+        } catch (Exception e) {
             try {
                 publishTime = headSelectable.xpath("//ul").nodes().get(0).xpath("//li").nodes().get(1).xpath("//li/time/text()").get();
-            }catch (Exception e1) {
+            } catch (Exception e1) {
                 publishTime = headSelectable.xpath("//ul").nodes().get(0).xpath("//li").nodes().get(0).xpath("//li/time/text()").get();
             }
         }
@@ -161,8 +157,7 @@ public class NatureSearchPageProcessor implements PageProcessor {
             jsonObject.put("email", email);
             authorEmail.add(jsonObject);
         }
-        System.out.println("code:" + articleCode + ",发布时间：" + publishTime + ",标题：" + title + ",作者：" + authorName + "，邮箱信息：" + authorEmail.toJSONString());
-
+        log.info("文章链接：{},发布时间:{},标题:{},作者:{},邮箱信息:{}", articleCode, publishTime, title, authorEmail.toJSONString());
  
         page.putField("article", ArticleDO.builder()
                 .articleType(ArticleTypeEnum.NATURE.getType())
@@ -178,8 +173,6 @@ public class NatureSearchPageProcessor implements PageProcessor {
  
     private void doArticleList(Page page) {
         String url = page.getUrl().get();
-        String[] split = url.split("=");
-        Integer pageIndex = Integer.parseInt(split[split.length - 1]);
         /**
          * 通过page.getHtml()可以获取到main函数中Spider.create(new BaiduHotSearchPageProcessor()).addUrl中的地址的网页内容
          * 1、通过$或css()方法获取到该page html下某元素dom
@@ -196,9 +189,8 @@ public class NatureSearchPageProcessor implements PageProcessor {
             Selectable node = nodes.get(i).$(".u-full-height").nodes().get(2).nodes().get(0).$(".u-full-height").select(new XpathSelector("a[@class='c-card__link u-link-inherit']")).nodes().get(0);
             String link = node.$("a", "href").get();
             page.addTargetRequest(link);
-            String link1 = node.links().get();
             String title = node.$("a", "text").get();
-            System.out.printf("%d、%s，访问地址：%s%n", i, title, link1);
+            log.info("关键字文章列表链接：{},标题:{},文章链接：{}", url, title, link);
         }
     }
  
@@ -3,7 +3,9 @@ package com.canrd.webmagic.utils;
 import com.alibaba.fastjson.JSON;
 import com.alibaba.fastjson.JSONArray;
 import com.alibaba.fastjson.JSONObject;
+import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
 import com.canrd.webmagic.BaseTest;
+import com.canrd.webmagic.common.utils.StringUtils;
 import com.canrd.webmagic.domain.dto.ArticleDO;
 import com.canrd.webmagic.service.ArticleService;
 import org.junit.Test;
@@ -24,9 +26,12 @@ public class DateTimeUtilTest extends BaseTest {
  
     @Test
     public void export() {
-        List<ArticleDO> articleDOList = articleService.list();
+        List<ArticleDO> articleDOList = articleService.list(new LambdaQueryWrapper<ArticleDO>().select(ArticleDO::getEmailInfo));
         JSONArray array = new JSONArray();
         for (ArticleDO articleDO : articleDOList) {
+            if (StringUtils.isBlank(articleDO.getEmailInfo())) {
+                continue;
+            }
             JSONArray jsonArray = JSONArray.parseArray(articleDO.getEmailInfo());
             array.addAll(jsonArray);
         }