feat: nature article爬取

谢茂盛
1 parent b1b31dc6
Showing 2 changed files with 35 additions and 2 deletions
src/main/java/com/canrd/webmagic/controller/NatureArticleController.java
src/main/java/com/canrd/webmagic/processor/NatureSearchPageProcessor.java
@@ -4,8 +4,8 @@ import com.canrd.webmagic.common.constant.ServerResult;
 import com.canrd.webmagic.common.jsr303.OperateGroup;
 import com.canrd.webmagic.domain.vo.NatureArticleQueryVO;
 import com.canrd.webmagic.domain.vo.NatureArticleVO;
-import com.canrd.webmagic.processor.pipeline.NatureArticlePipeline;
 import com.canrd.webmagic.processor.NatureSearchPageProcessor;
+import com.canrd.webmagic.processor.pipeline.NatureArticlePipeline;
 import com.canrd.webmagic.service.NatureArticleService;
 import org.springframework.validation.annotation.Validated;
 import org.springframework.web.bind.annotation.*;
@@ -43,6 +43,7 @@ public class NatureArticleController {
             Spider.create(natureSearchPageProcessor)
                     // 添加这个Spider要爬取的网页地址
                     .addUrl("https://www.nature.com/search?q=" + keyword + "&page=" + i)
+                    .addUrl("https://www.nature.com/nature/research-articles?sort=PubDate&page=" + i)
                     .addPipeline(articlePipeline)
                     // 开启5个线程执行，并开始爬取
                     .thread(5).run();
@@ -40,12 +40,44 @@ public class NatureSearchPageProcessor implements PageProcessor {
     public void process(Page page) {
         if (page.getUrl().get().contains("search")) {
             doArticleList(page);
-        } else {
+        } else if (page.getUrl().get().contains("research-articles")) {
+            doArticleList4ReSearch(page);
+        }else {
             doArticleContent(page);
         }
     }
+    /**
+     *
+     * @param page
+     */
+    private void doArticleList4ReSearch(Page page){
+        String url = page.getUrl().get();
+        String[] split = url.split("=");
+        Integer pageIndex = Integer.parseInt(split[split.length - 1]);
+        /**
+         * 通过page.getHtml()可以获取到main函数中Spider.create(new BaiduHotSearchPageProcessor()).addUrl中的地址的网页内容
+         * 1、通过$或css()方法获取到该page html下某元素dom
+         */
+        Selectable selectable = page.getHtml().$(".app-article-list-row").select(
+                new XpathSelector("li[@class='app-article-list-row__item']")
+        );
+        List<Selectable> nodes = selectable.nodes();
+
+        /**
+         * 获取到指定的dom后，从这些dom中提取元素内容。
+         */
+        for (int i = 1; i <= nodes.size() - 1; i++) {
+            Selectable node = nodes.get(i).$(".u-full-height").nodes().get(2).nodes().get(0).$(".u-full-height").select(new XpathSelector("a[@class='c-card__link u-link-inherit']")).nodes().get(0);
+            String link = node.$("a", "href").get();
+            page.addTargetRequest(link);
+            String link1 = node.links().get();
+            String title = node.$("a", "text").get();
+            System.out.printf("%d、%s，访问地址：%s%n", i, title, link1);
+        }
+    }
+
     private void doArticleContent(Page page) {
         //解析页面
         Html html = page.getHtml();