feat:

1、nature 爬取调整

feat:
1、nature 爬取调整
谢茂盛
1 parent 72c685be
Showing 4 changed files with 48 additions and 12 deletions
src/main/java/com/canrd/webmagic/controller/ArticleController.java → src/main/java/com/canrd/webmagic/controller/NatureController.java
src/main/java/com/canrd/webmagic/job/NatureJob.java
src/main/java/com/canrd/webmagic/processor/NatureSearchPageProcessor.java
src/main/resources/application-test.yml
 package com.canrd.webmagic.controller;
 import com.canrd.webmagic.common.constant.ServerResult;
+import com.canrd.webmagic.common.utils.KeywordUtil;
+import com.canrd.webmagic.common.utils.StringUtils;
 import com.canrd.webmagic.processor.NatureSearchPageProcessor;
 import com.canrd.webmagic.processor.download.Downloader;
 import com.canrd.webmagic.processor.pipeline.ArticlePipeline;
+import org.apache.logging.log4j.core.util.UuidUtil;
 import org.springframework.web.bind.annotation.GetMapping;
 import org.springframework.web.bind.annotation.RequestMapping;
 import org.springframework.web.bind.annotation.RequestParam;
@@ -20,7 +23,7 @@ import javax.annotation.Resource;
  */
 @RestController
 @RequestMapping("/nature/article")
-public class ArticleController {
+public class NatureController {
     @Resource
     private NatureSearchPageProcessor natureSearchPageProcessor;
@@ -35,7 +38,7 @@ public class ArticleController {
      * @return
      */
     @GetMapping("/start")
-    public ServerResult start(@RequestParam(value = "keyword") String keyword, @RequestParam(value = "indexSize") Integer indexSize) {
+    public ServerResult start(@RequestParam(value = "indexSize") Integer indexSize) {
         for (int i = 1; i <= indexSize; i++) {
             Spider.create(natureSearchPageProcessor)
                     .addUrl("https://www.nature.com/nature/research-articles?sort=PubDate&page=" + i)
@@ -47,5 +50,33 @@ public class ArticleController {
         return ServerResult.success();
     }
+
+    /**
+     * @return
+     */
+    @GetMapping("/search")
+    public ServerResult search(@RequestParam(value = "indexSize") Integer indexSize,
+                               @RequestParam(value = "orderBy") String orderBy,
+                               @RequestParam(value = "journal") String journal) {
+        StringBuffer url = new StringBuffer("https://www.nature.com/search?q=");
+        for (int i = 1; i <= indexSize; i++) {
+            for (String keyword : KeywordUtil.getKeyWordList()) {
+                if (StringUtils.isBlank(orderBy)) {
+                    url.append(keyword).append("&page=" + i);
+                } else {
+                    url.append(keyword).append("&order=" + orderBy).append("&page=" + i);
+                }
+                if (StringUtils.isNotBlank(journal)) {
+                    url.append("&journal=" + journal);
+                }
+                Spider.create(natureSearchPageProcessor)
+                        .addUrl(url.toString())
+                        // 开启5个线程执行，并开始爬取
+                        .setUUID(UuidUtil.getTimeBasedUuid().toString())
+                        .thread(5).run();
+            }
+        }
+        return ServerResult.success();
+    }
 }
@@ -33,13 +33,18 @@ public class NatureJob {
         for (String keyword : KeywordUtil.getKeyWordList()) {
             Spider.create(natureSearchPageProcessor)
                     // 添加这个Spider要爬取的网页地址
-                    .addUrl("https://www.nature.com/search?q=" + keyword + "&page=" + 1)
-                    .addUrl("https://www.nature.com/search?q=" + keyword + "&page=" + 2)
-                    .addUrl("https://www.nature.com/search?q=" + keyword + "&page=" + 3)
+                    .addUrl("https://www.nature.com/search?q=" + keyword + "&order=date_desc&page=" + 1)
+                    .addUrl("https://www.nature.com/search?q=" + keyword + "&order=date_desc&page=" + 2)
+                    .addUrl("https://www.nature.com/search?q=" + keyword + "&order=date_desc&page=" + 3)
+                    .addUrl("https://www.nature.com/search?q=" + keyword + "&journal=nnano&order=date_desc&page=" + 1)
+                    .addUrl("https://www.nature.com/search?q=" + keyword + "&journal=nnano&order=date_desc&page=" + 2)
+                    .addUrl("https://www.nature.com/search?q=" + keyword + "&journal=nnano&order=date_desc&page=" + 3)
+                    .addUrl("https://www.nature.com/search?q=" + keyword + "&journal=nphys&order=date_desc&page=" + 1)
+                    .addUrl("https://www.nature.com/search?q=" + keyword + "&journal=nphys&order=date_desc&page=" + 2)
+                    .addUrl("https://www.nature.com/search?q=" + keyword + "&journal=nphys&order=date_desc&page=" + 3)
                     .addPipeline(articlePipeline)
-//                    .setDownloader(downloader.newIpDownloader())
-                    // 开启5个线程执行，并开始爬取
-                    .thread(5).run();
+                    // 开启20个线程执行，并开始爬取
+                    .thread(20).run();
         }
     }
 }
@@ -155,9 +155,9 @@ public class NatureSearchPageProcessor implements PageProcessor {
                 JSONObject object = new JSONObject();
                 object.put("referenceTitle", referenceTitle);
                 object.put("links", links);
-                if (CollectionUtils.isNotEmpty(links)) {
-                    page.addTargetRequests(links.stream().filter(x -> x.contains("nature")).collect(Collectors.toList()));
-                }
+//                if (CollectionUtils.isNotEmpty(links)) {
+//                    page.addTargetRequests(links.stream().filter(x -> x.contains("nature")).collect(Collectors.toList()));
+//                }
                 references.add(object);
             }
         }
@@ -57,7 +57,7 @@ spring:
     testWhileIdle: true
     testOnBorrow: true
     testOnReturn: true
-    password: 123456
+    password: Canrd@2023
     time-between-eviction-runs-millis: 1000
     url: jdbc:mysql://localhost:3306/webmagic?useUnicode=true&characterEncoding=UTF-8&useJDBCCompliantTimezoneShift=true&useLegacyDatetimeCode=false&serverTimezone=Asia/Shanghai&useSSL=false&autoReconnect=true&failOverReadOnly=false&maxReconnects=10&allowMultiQueries=true&useAffectedRows=true&autoReconnectForPools=true
     username: root