feat:

1、science 爬取

feat:
1、science 爬取
谢茂盛
1 parent fde047e7
Showing 3 changed files with 78 additions and 3 deletions
src/main/java/com/canrd/webmagic/controller/Science4JournalController.java
src/main/java/com/canrd/webmagic/processor/Science4JournalArticlePageProcessor.java
src/main/java/com/canrd/webmagic/processor/Science4JournalSearchPageProcessor.java
 package com.canrd.webmagic.controller;
 import com.canrd.webmagic.common.constant.ServerResult;
+import com.canrd.webmagic.common.utils.KeywordUtil;
 import com.canrd.webmagic.processor.Science4JournalSearchPageProcessor;
 import com.canrd.webmagic.processor.download.SeleniumDownloader;
 import org.apache.logging.log4j.core.util.UuidUtil;
@@ -32,7 +33,7 @@ public class Science4JournalController {
      * @return
      */
     @GetMapping("/start")
-    public ServerResult start(@RequestParam(value = "keyword") String keyword, @RequestParam(value = "indexSize") Integer indexSize) {
+    public ServerResult start(@RequestParam(value = "indexSize") Integer indexSize) {
         for (int i = 0; i <= indexSize; i++) {
             Spider.create(science4JournalSearchPageProcessor)
                     .addUrl("https://www.science.org/journal/science/insights?startPage=" + i)
@@ -46,5 +47,24 @@ public class Science4JournalController {
     }
+    /**
+     * @return
+     */
+    @GetMapping("/search")
+    public ServerResult search(@RequestParam(value = "indexSize") Integer indexSize) {
+        for (int i = 0; i <= indexSize; i++) {
+            for (String keyword : KeywordUtil.getKeyWordList()) {
+                Spider.create(science4JournalSearchPageProcessor)
+                        .addUrl("https://www.science.org/action/doSearch?AllField=" + keyword + "&pageSize=20&startPage=" + i)
+                        // 开启5个线程执行，并开始爬取
+                        .setUUID(UuidUtil.getTimeBasedUuid().toString())
+                        .setDownloader(seleniumDownloader)
+                        .thread(5).run();
+            }
+        }
+
+        return ServerResult.success();
+    }
+
 }
@@ -3,6 +3,7 @@ package com.canrd.webmagic.processor;
 import com.alibaba.fastjson.JSONArray;
 import com.alibaba.fastjson.JSONObject;
 import com.canrd.webmagic.common.utils.DateUtil;
+import com.canrd.webmagic.common.utils.StringUtils;
 import com.canrd.webmagic.domain.ArticleTypeEnum;
 import com.canrd.webmagic.domain.dto.ArticleDO;
 import com.canrd.webmagic.processor.config.Agent;
@@ -57,8 +58,14 @@ public class Science4JournalArticlePageProcessor implements PageProcessor {
         Selectable headSelectable = html.xpath("//div[@class='article-container']/article/header");
         String title = html.xpath("//div[@class='article-container']/article/header/div").xpath("//div[@class='core-lede']/div/text()").get();
+        if (StringUtils.isBlank(title)) {
+            title = html.xpath("//div[@class='article-container']/article/header").xpath("//h1[@property='name']/text()").get();
+        }
         String articleDesc = html.xpath("//div[@class='article-container']/article").xpath("//section[@id='bodymatter']/div/div/text()").get();
+        if (StringUtils.isBlank(articleDesc)) {
+            articleDesc = html.xpath("//div[@class='article-container']/article").xpath("//div[@role='paragraph']/text()").get();
+        }
         String publishTime = html.xpath("//div[@class='article-container']/article/header/div").xpath("//div[@class='core-self-citation']").xpath("//div[@class='core-date-published']/span/text()").get();
         Date publishTimeDateTime = null;
@@ -72,7 +79,11 @@ public class Science4JournalArticlePageProcessor implements PageProcessor {
         List<Selectable> authorNodes = html.xpath("//div[@class='article-container']/article/header/div").xpath("//div[@class='contributors']/span/span/span").nodes();
         StringBuffer authorName = new StringBuffer();
         for (Selectable node : authorNodes) {
-            authorName.append(node.xpath("//a/span/text()").get()).append(" ");
+            String name = node.xpath("//a/span/text()").get();
+            if (StringUtils.isBlank(name)) {
+                continue;
+            }
+            authorName.append(name).append(" ");
         }
@@ -53,7 +53,51 @@ public class Science4JournalSearchPageProcessor implements PageProcessor {
      */
     @Override
     public void process(Page page) {
-        doArticleList(page);
+        if (page.getUrl().get().contains("doSearch")) {
+            doSearch(page);
+        } else {
+            doArticleList(page);
+        }
+    }
+
+    /**
+     * @param page
+     */
+    private void doSearch(Page page) {
+        String url = page.getUrl().get();
+        /**
+         * 通过page.getHtml()可以获取到main函数中Spider.create(new BaiduHotSearchPageProcessor()).addUrl中的地址的网页内容
+         * 1、通过$或css()方法获取到该page html下某元素dom
+         */
+        Selectable selectable = page.getHtml().xpath("//div[@class=' search-result__body titles-results ']").select(new XpathSelector("div[@class='card pb-3 mb-4 border-bottom']"));
+        List<Selectable> nodes = selectable.nodes();
+
+        /**
+         * 获取到指定的dom后，从这些dom中提取元素内容。
+         */
+        for (int i = 0; i <= nodes.size() - 1; i++) {
+            String title = nodes.get(i).xpath("//div[@class='card pb-3 mb-4 border-bottom']/div").xpath("//div[@class='d-flex justify-content-between align-items-end']/div/span/h2/a/text()").get();
+            String time = nodes.get(i).xpath("//div[@class='card-meta align-middle mb-2 text-uppercase text-darker-gray']/span").nodes().get(2).xpath("//time/text()").get();
+            String link = nodes.get(i).links().get();
+            SimpleDateFormat formatter = new SimpleDateFormat("dd MMMM yyyy", Locale.ENGLISH);
+            try {
+                Date publishTimeDateTime = formatter.parse(time);
+                if (!publishTimeDateTime.before(DateUtil.localDate2Date(DateUtil.parseDate("2000-01-01", DateUtil.DATE)))) {
+//                        page.addTargetRequest(link);
+                    Spider.create(science4JournalArticlePageProcessor)
+                            .addUrl(link)
+                            .addPipeline(articlePipeline)
+                            .setDownloader(seleniumDownloader)
+                            .setUUID(UuidUtil.getTimeBasedUuid().toString())
+                            // 开启5个线程执行，并开始爬取
+                            .thread(1).run();
+                    log.info("关键字文章列表链接：{},标题:{},文章链接：{}", url, title, link);
+                }
+            } catch (ParseException e) {
+                e.printStackTrace();
+            }
+        }
+
     }
     /**