feat: nature article爬取

谢茂盛
1 parent 89ec05fb
Showing 3 changed files with 22 additions and 20 deletions
src/main/java/com/canrd/webmagic/controller/ArticleController.java
src/main/java/com/canrd/webmagic/processor/config/Downloader.java
src/main/java/com/canrd/webmagic/processor/config/UpdateIp.java
@@ -5,6 +5,7 @@ import com.canrd.webmagic.common.jsr303.OperateGroup;
 import com.canrd.webmagic.domain.vo.NatureArticleQueryVO;
 import com.canrd.webmagic.domain.vo.NatureArticleVO;
 import com.canrd.webmagic.processor.NatureSearchPageProcessor;
+import com.canrd.webmagic.processor.config.Downloader;
 import com.canrd.webmagic.processor.pipeline.NatureArticlePipeline;
 import com.canrd.webmagic.service.ArticleService;
 import org.springframework.validation.annotation.Validated;
@@ -34,6 +35,9 @@ public class ArticleController {
     @Resource
     private NatureArticlePipeline articlePipeline;
  
+    @Resource
+    private Downloader downloader;
+
     /**
      * @return
      */
@@ -45,6 +49,7 @@ public class ArticleController {
                     .addUrl("https://www.nature.com/search?q=" + keyword + "&page=" + i)
                     .addUrl("https://www.nature.com/nature/research-articles?sort=PubDate&page=" + i)
                     .addPipeline(articlePipeline)
+                    .setDownloader(downloader.newIpDownloader())
                     // 开启5个线程执行，并开始爬取
                     .thread(5).run();
         }
@@ -28,10 +28,9 @@ public class Downloader {
     }
  
     /**
-     *
      * @return
      */
-    public static HttpClientDownloader newIpDownloader() {
+    public HttpClientDownloader newIpDownloader() {
         HttpClientDownloader downloader = new HttpClientDownloader() {
             @Override
             protected void onError(Request request) {
@@ -39,10 +38,12 @@ public class Downloader {
                 setProxyProvider(SimpleProxyProvider.from(new Proxy(ips[0], Integer.parseInt(ips[1]))));
             }
         };
+        String[] ips = newIp();
+        downloader.setProxyProvider(SimpleProxyProvider.from(new Proxy(ips[0], Integer.parseInt(ips[1]))));
         return downloader;
     }
  
-    static String[] newIp() {
+    private String[] newIp() {
         Long size = redisTemplate.opsForList().size("ip");
         String ip = redisTemplate.opsForList().index("ip", new Random().nextInt(size.intValue())).toString();
         log.info("获取ip===========>" + ip);
 package com.canrd.webmagic.processor.config;
  
-import com.baomidou.mybatisplus.core.toolkit.StringUtils;
 import org.apache.commons.io.IOUtils;
 import org.jsoup.Jsoup;
 import org.jsoup.nodes.Document;
@@ -31,7 +30,7 @@ public class UpdateIp {
     @Autowired
     private RedisTemplate redisTemplate;
  
-//    @Scheduled(cron = "*/20 * * * * ?")
+    @Scheduled(cron = "*/20 * * * * ?")
     void update() {
         List<String> range = redisTemplate.opsForList().range("ip", 0, -1);
         for (String ip : range) {
@@ -44,25 +43,22 @@ public class UpdateIp {
  
 //    @Scheduled(cron = "*/15 * * * * ?")
     void ips() {
-        String string = null;
         try {
-            Document document = Jsoup.connect("https://www.xicidaili.com/nn").timeout(3000).get();
-            Elements tags = document.select("#ip_list > tbody > tr");
-            for (Element element : tags) {
-                //取得ip地址节点
-                Elements tdChilds = element.select("tr > td:nth-child(2)");
-                //取得端口号节点
-                Elements tcpd = element.select("tr > td:nth-child(3)");
-                if (StringUtils.isNotBlank(tdChilds.text()) && StringUtils.isNotBlank(tcpd.text())) {
-                    string = tdChilds.text() + ":" + tcpd.text();
-                    if (!ifUseless(string)) {
+            for (int i = 1; i < 10; i++) {
+                Document document = Jsoup.connect("https://www.zdaye.com/free/" + i + "/?sAdr=taiwan").timeout(3000).get();
+                Elements tags = document.selectXpath("//table[@id='ipc']/tbody/tr");
+                for (Element element : tags) {
+                    String ip = element.getElementsByTag("td").get(0).text();
+                    String port = document.selectXpath("//table[@id='ipc']/tbody/tr").get(0).getElementsByTag("td").get(1).text();
+                    String uri = ip + ":" + port;
+                    if (!ifUseless(uri)) {
                         List<String> range = redisTemplate.opsForList().range("ip", 0, -1);
-                        if (!range.contains(string)) {
-                            System.err.println(string + "  存进redis");
+                        if (!range.contains(uri)) {
+                            System.err.println(uri + "  存进redis");
                             if (redisTemplate.opsForList().size("ip") > 100) {
-                                redisTemplate.opsForList().rightPopAndLeftPush("ip", string);
+                                redisTemplate.opsForList().rightPopAndLeftPush("ip", uri);
                             } else {
-                                redisTemplate.opsForList().leftPush("ip", string);
+                                redisTemplate.opsForList().leftPush("ip", uri);
                             }
                         }
                     }